Merge tag 'v3.7.1' into HEAD
libaom 3.7.1
2023-11-17 v3.7.1
This release includes several bug fixes. This release is ABI
compatible with the last release. See
https://aomedia.googlesource.com/aom/+log/v3.7.0..v3.7.1 for all the
commits in this release.
- Bug Fixes
* aomedia:3349: heap overflow when increasing resolution
* aomedia:3478: GCC 12.2.0 emits a -Wstringop-overflow warning on
aom/av1/encoder/motion_search_facade.c
* aomedia:3489: Detect encoder and image high bit depth mismatch
* aomedia:3491: heap-buffer-overflow on frame size change
* b/303023614: Segfault at encoding time for high bit depth images
Bug: aomedia:3513
Change-Id: Iecf1f155b4f0ea2604ef27fef0d6111499ea9bad
diff --git a/.mailmap b/.mailmap
index 7d31a70..6d6e6302 100644
--- a/.mailmap
+++ b/.mailmap
@@ -1,12 +1,16 @@
+Aasaipriya Chandran <aasaipriya.c@ittiam.com>
+Aasaipriya Chandran <aasaipriya.c@ittiam.com> Aasaipriya C <100778@ittiam.com>
Adrian Grange <agrange@google.com>
-Aℓex Converse <aconverse@google.com>
-Aℓex Converse <aconverse@google.com> <alex.converse@gmail.com>
+Adrian Grange <agrange@google.com> <agrange@agrange-macbookpro.roam.corp.google.com>
+Alexander Bokov <alexanderbokov@google.com>
Alexis Ballier <aballier@gentoo.org> <alexis.ballier@gmail.com>
Alpha Lam <hclam@google.com> <hclam@chromium.org>
Andrey Norkin <anorkin@netflix.com>
Angie Chiang <angiebird@google.com>
Arild Fuldseth <arilfuld@cisco.com> <arild.fuldseth@gmail.com>
Arild Fuldseth <arilfuld@cisco.com> <arilfuld@cisco.com>
+Aℓex Converse <aconverse@google.com>
+Aℓex Converse <aconverse@google.com> <alex.converse@gmail.com>
Aasaipriya Chandran <aasaipriya.c@ittiam.com>
Aasaipriya Chandran <aasaipriya.c@ittiam.com> Aasaipriya C <100778@ittiam.com>
Apurve Pandey <apurve.pandey@ittiam.com>
@@ -27,9 +31,10 @@
Grant Hsu <grant.hsu@cidana.com> <grant.hsu@gmail.com>
Guillaume Martres <smarter@ubuntu.com>
Guillaume Martres <smarter@ubuntu.com> <gmartres@google.com>
-Guillaume Martres <smarter@ubuntu.com> <smarter3@gmail.com>
Guillaume Martres <smarter@ubuntu.com> <gmartres@mozilla.com>
+Guillaume Martres <smarter@ubuntu.com> <smarter3@gmail.com>
Hangyu Kuang <hkuang@google.com>
+Hangyu Kuang <hkuang@google.com> <hkuang@hkuang-macbookpro.roam.corp.google.com>
Hui Su <huisu@google.com>
Iole Moccagatta <iole.moccagatta@gmail.com>
Jacky Chen <jackychen@google.com>
@@ -40,13 +45,14 @@
Johann Koenig <johannkoenig@google.com> <johann.koenig@duck.com>
Johann Koenig <johannkoenig@google.com> <johann.koenig@gmail.com>
Johann Koenig <johannkoenig@google.com> <johannkoenig@chromium.org>
+Johann Koenig <johannkoenig@google.com> <johannkoenig@dhcp-172-19-7-52.mtv.corp.google.com>
John Koleszar <jkoleszar@google.com>
Joshua Litt <joshualitt@google.com> <joshualitt@chromium.org>
Kyle Siefring <siekyleb@amazon.com>
Kyle Siefring <siekyleb@amazon.com> <kylesiefring@gmail.com>
Lin Zheng <linzhen@google.com>
-Lokeshwar Reddy B <lokeshwar.reddy@ittiam.com>
Logan Goldberg <logangw@google.com>
+Lokeshwar Reddy B <lokeshwar.reddy@ittiam.com>
Luc Trudeau <luc@trud.ca>
Luc Trudeau <luc@trud.ca> <ltrudeau@mozilla.com>
Marco Paniconi <marpan@google.com>
@@ -56,6 +62,7 @@
Mingliang Chen <mlchen@google.com>
Monty Montgomery <cmontgomery@mozilla.com>
Mudassir Galaganath <mudassir.galaganath@ittiam.com>
+Narayan Kalaburgi <narayan.kalaburgi@ittiam.com>
Mudassir Galaganath <mudassir.galaganath@ittiam.com> Mudassir Galagnath
Nathan E. Egge <negge@mozilla.com>
Nathan E. Egge <negge@mozilla.com> <negge@dgql.org>
@@ -72,13 +79,14 @@
Remya Prakasan <remya.prakasan@ittiam.com>
Roger Zhou <youzhou@microsoft.com>
Ronald S. Bultje <rsbultje@gmail.com> <rbultje@google.com>
-Ryan Lei <ryanlei@fb.com> <ryan.z.lei@intel.com>
Ryan Lei <ryanlei@fb.com> <ryan.lei@intel.com>
+Ryan Lei <ryanlei@fb.com> <ryan.z.lei@intel.com>
Ryan Lei <ryanlei@fb.com> <zlei3@ZLEI3-DESK.amr.corp.intel.com>
Sachin Kumar Garg <sachin.kumargarg@ittiam.com>
Sai Deng <sdeng@google.com>
Sami Pietilä <samipietila@google.com>
Sarah Parker <sarahparker@google.com>
+Susanna D'Souza <susannad@google.com>
Tamar Levy <tamar.levy@intel.com>
Tamar Levy <tamar.levy@intel.com> <levytamar82@gmail.com>
Tero Rintaluoma <teror@google.com> <tero.rintaluoma@on2.com>
@@ -90,14 +98,16 @@
Tom Finegan <tomfinegan@google.com> <tomfinegan@chromium.org>
Tristan Matthews <tmatth@videolan.org> <le.businessman@gmail.com>
Venkat Sanampudi <sanampudi.venkatarao@ittiam.com>
+Vitalii Dziumenko <vdziumenko@luxoft.com> <vdziumenko@luxoft.corp-partner.google.com>
Wei-Ting Lin <weitinglin@google.com>
Wei-Ting Lin <weitinglin@google.com> <weitingco@gmail.com>
Wenyao Liu <wenyao.liu@cidana.com>
Will Bresnahan <bill.wresnahan@gmail.com>
+Yaowu Xu <yaowu@google.com> <Yaowu Xu>
Yaowu Xu <yaowu@google.com> <adam@xuyaowu.com>
+Yaowu Xu <yaowu@google.com> <yaowu.google.com>
+Yaowu Xu <yaowu@google.com> <yaowu@YAOWU2-W.ad.corp.google.com>
Yaowu Xu <yaowu@google.com> <yaowu@xuyaowu.com>
Yaowu Xu <yaowu@google.com> <yaowu@yaowu-macbookpro.roam.corp.google.com>
-Yaowu Xu <yaowu@google.com> <Yaowu Xu>
-Yaowu Xu <yaowu@google.com> <yaowu.google.com>
Zhipin Deng <zhipin.deng@intel.com>
Zoe Liu <zoeliu@gmail.com> <zoeliu@google.com>
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8e0b65f..8e6ca6b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -288,9 +288,9 @@
add_library(aom_static STATIC ${target_objs_aom} $<TARGET_OBJECTS:aom_rtcd>)
set_target_properties(aom_static PROPERTIES OUTPUT_NAME aom)
if(MSVC OR (WIN32 AND NOT MINGW))
- # Fix race condition on the export library file between the two versions.
- # Affects MSVC in all three flavors (stock, Clang/CL, LLVM-- the latter sets
- # MSVC and MINGW both to FALSE).
+ # Fix race condition between the import library and the static library.
+ # Affects MSVC in all three flavors (stock, clang-cl, LLVM -- the latter
+ # sets MSVC and MINGW both to FALSE).
set_target_properties(aom PROPERTIES ARCHIVE_OUTPUT_NAME "aom_dll")
endif()
@@ -323,7 +323,7 @@
endif()
endif()
-if(CONFIG_AV1_ENCODER AND NOT CONFIG_REALTIME_ONLY AND NOT BUILD_SHARED_LIBS)
+if(CONFIG_AV1_ENCODER AND NOT BUILD_SHARED_LIBS)
list(APPEND AOM_AV1_RC_SOURCES "${AOM_ROOT}/av1/ratectrl_rtc.h"
"${AOM_ROOT}/av1/ratectrl_rtc.cc")
add_library(aom_av1_rc ${AOM_AV1_RC_SOURCES})
@@ -336,7 +336,7 @@
# List of object and static library targets.
set(AOM_LIB_TARGETS ${AOM_LIB_TARGETS} aom_rtcd aom_mem aom_scale aom)
-if(CONFIG_AV1_ENCODER AND NOT CONFIG_REALTIME_ONLY AND NOT BUILD_SHARED_LIBS)
+if(CONFIG_AV1_ENCODER AND NOT BUILD_SHARED_LIBS)
set(AOM_LIB_TARGETS ${AOM_LIB_TARGETS} aom_av1_rc)
endif()
if(BUILD_SHARED_LIBS)
@@ -387,13 +387,6 @@
endif()
endif()
-if((CONFIG_AV1_DECODER OR CONFIG_AV1_ENCODER) AND ENABLE_EXAMPLES)
- add_executable(resize_util "${AOM_ROOT}/examples/resize_util.c"
- $<TARGET_OBJECTS:aom_common_app_util>)
- set_property(TARGET ${example} PROPERTY FOLDER examples)
- list(APPEND AOM_APP_TARGETS resize_util)
-endif()
-
if(CONFIG_AV1_DECODER AND ENABLE_EXAMPLES)
add_executable(aomdec "${AOM_ROOT}/apps/aomdec.c"
$<TARGET_OBJECTS:aom_common_app_util>
@@ -494,14 +487,18 @@
$<TARGET_OBJECTS:aom_common_app_util>
$<TARGET_OBJECTS:aom_encoder_app_util>)
- add_executable(svc_encoder_rtc "${AOM_ROOT}/examples/svc_encoder_rtc.cc"
- $<TARGET_OBJECTS:aom_common_app_util>
- $<TARGET_OBJECTS:aom_encoder_app_util>)
-
# Maintain a list of encoder example targets.
list(APPEND AOM_ENCODER_EXAMPLE_TARGETS aomenc lossless_encoder noise_model
photon_noise_table set_maps simple_encoder scalable_encoder
- twopass_encoder svc_encoder_rtc)
+ twopass_encoder)
+
+ if(NOT BUILD_SHARED_LIBS)
+ add_executable(svc_encoder_rtc "${AOM_ROOT}/examples/svc_encoder_rtc.cc"
+ $<TARGET_OBJECTS:aom_common_app_util>
+ $<TARGET_OBJECTS:aom_encoder_app_util>)
+ target_link_libraries(svc_encoder_rtc ${AOM_LIB_LINK_TYPE} aom_av1_rc)
+ list(APPEND AOM_ENCODER_EXAMPLE_TARGETS svc_encoder_rtc)
+ endif()
endif()
if(ENABLE_TOOLS)
@@ -852,7 +849,7 @@
# Aomedia documentation rule.
set(DOXYGEN_VERSION_VALUE 0)
if(ENABLE_DOCS)
- include(FindDoxygen)
+ find_package(Doxygen)
if(DOXYGEN_FOUND)
# Check if Doxygen version is >= minimum required version(i.e. 1.8.10).
set(MINIMUM_DOXYGEN_VERSION 1008010)
@@ -942,7 +939,8 @@
get_cmake_property(all_cmake_vars VARIABLES)
foreach(var ${all_cmake_vars})
if("${var}" MATCHES "SOURCES$\|_INTRIN_\|_ASM_"
- AND NOT "${var}" MATCHES "DOXYGEN\|LIBYUV\|_PKG_\|TEST")
+ AND NOT "${var}" MATCHES "DOXYGEN\|LIBYUV\|_PKG_\|TEST"
+ AND NOT "${var}" MATCHES "_ASM_NASM\|_ASM_COMPILER_")
list(APPEND aom_source_vars ${var})
endif()
endforeach()
diff --git a/README.md b/README.md
index d7b66e0..4e2eb27 100644
--- a/README.md
+++ b/README.md
@@ -159,6 +159,7 @@
The toolchain files available at the time of this writing are:
- arm64-ios.cmake
+ - arm64-linux-clang.cmake
- arm64-linux-gcc.cmake
- arm64-mingw-gcc.cmake
- armv7-ios.cmake
diff --git a/aom/aom_encoder.h b/aom/aom_encoder.h
index e3d8d29..5d0bbe1 100644
--- a/aom/aom_encoder.h
+++ b/aom/aom_encoder.h
@@ -1006,11 +1006,11 @@
aom_fixed_buf_t *aom_codec_get_global_headers(aom_codec_ctx_t *ctx);
/*!\brief usage parameter analogous to AV1 GOOD QUALITY mode. */
-#define AOM_USAGE_GOOD_QUALITY (0)
+#define AOM_USAGE_GOOD_QUALITY 0u
/*!\brief usage parameter analogous to AV1 REALTIME mode. */
-#define AOM_USAGE_REALTIME (1)
+#define AOM_USAGE_REALTIME 1u
/*!\brief usage parameter analogous to AV1 all intra mode. */
-#define AOM_USAGE_ALL_INTRA (2)
+#define AOM_USAGE_ALL_INTRA 2u
/*!\brief Encode a frame
*
diff --git a/aom/aomcx.h b/aom/aomcx.h
index a5db0a5..f061be3 100644
--- a/aom/aomcx.h
+++ b/aom/aomcx.h
@@ -208,14 +208,14 @@
* encoding process, values greater than 0 will increase encoder speed at
* the expense of quality.
*
- * Valid range: 0..10. 0 runs the slowest, and 10 runs the fastest;
+ * Valid range: 0..11. 0 runs the slowest, and 11 runs the fastest;
* quality improves as speed decreases (since more compression
* possibilities are explored).
*
- * NOTE: 10 is only allowed in AOM_USAGE_REALTIME. In AOM_USAGE_GOOD_QUALITY
- * and AOM_USAGE_ALL_INTRA, 9 is the highest allowed value. However,
- * AOM_USAGE_GOOD_QUALITY treats 7..9 the same as 6. Also, AOM_USAGE_REALTIME
- * treats 0..4 the same as 5.
+ * NOTE: 10 and 11 are only allowed in AOM_USAGE_REALTIME. In
+ * AOM_USAGE_GOOD_QUALITY and AOM_USAGE_ALL_INTRA, 9 is the highest allowed
+ * value. However, AOM_USAGE_GOOD_QUALITY treats 7..9 the same as 6. Also,
+ * AOM_USAGE_REALTIME treats 0..4 the same as 5.
*/
AOME_SET_CPUUSED = 13,
@@ -1527,6 +1527,12 @@
*/
AV1E_SET_BITRATE_ONE_PASS_CBR = 163,
+ /*!\brief Codec control to set the maximum number of consecutive frame drops
+ * allowed for the frame dropper in 1 pass CBR mode, int parameter. Value of
+ * zero has no effect.
+ */
+ AV1E_SET_MAX_CONSEC_FRAME_DROP_CBR = 164,
+
// Any new encoder control IDs should be added above.
// Maximum allowed encoder control ID is 229.
// No encoder control ID should be added below.
@@ -1678,10 +1684,10 @@
/*!brief Parameters for setting ref frame config */
typedef struct aom_svc_ref_frame_config {
- // 7 references: LAST_FRAME (0), LAST2_FRAME(1), LAST3_FRAME(2),
- // GOLDEN_FRAME(3), BWDREF_FRAME(4), ALTREF2_FRAME(5), ALTREF_FRAME(6).
+ // 7 references: The index 0 - 6 refers to the references:
+ // last(0), last2(1), last3(2), golden(3), bwdref(4), altref2(5), altref(6).
int reference[7]; /**< Reference flag for each of the 7 references. */
- /*! Buffer slot index for each of 7 references. */
+ /*! Buffer slot index for each of 7 references indexed above. */
int ref_idx[7];
int refresh[8]; /**< Refresh flag for each of the 8 slots. */
} aom_svc_ref_frame_config_t;
@@ -2172,6 +2178,9 @@
AOM_CTRL_USE_TYPE(AV1E_SET_BITRATE_ONE_PASS_CBR, unsigned int)
#define AOM_CTRL_AV1E_SET_BITRATE_ONE_PASS_CBR
+AOM_CTRL_USE_TYPE(AV1E_SET_MAX_CONSEC_FRAME_DROP_CBR, int)
+#define AOM_CTRL_AV1E_SET_MAX_CONSEC_FRAME_DROP_CBR
+
/*!\endcond */
/*! @} - end defgroup aom_encoder */
#ifdef __cplusplus
diff --git a/aom_dsp/aom_dsp.cmake b/aom_dsp/aom_dsp.cmake
index 4c60e5c..f8f2cbb 100644
--- a/aom_dsp/aom_dsp.cmake
+++ b/aom_dsp/aom_dsp.cmake
@@ -115,12 +115,17 @@
"${AOM_ROOT}/aom_dsp/arm/aom_convolve8_neon.c"
"${AOM_ROOT}/aom_dsp/arm/fwd_txfm_neon.c"
"${AOM_ROOT}/aom_dsp/arm/loopfilter_neon.c"
- "${AOM_ROOT}/aom_dsp/arm/highbd_intrapred_neon.c"
"${AOM_ROOT}/aom_dsp/arm/intrapred_neon.c"
"${AOM_ROOT}/aom_dsp/arm/subtract_neon.c"
"${AOM_ROOT}/aom_dsp/arm/blend_a64_mask_neon.c"
"${AOM_ROOT}/aom_dsp/arm/avg_pred_neon.c")
+list(APPEND AOM_DSP_COMMON_INTRIN_NEON_DOTPROD
+ "${AOM_ROOT}/aom_dsp/arm/aom_convolve8_neon_dotprod.c")
+
+list(APPEND AOM_DSP_COMMON_INTRIN_NEON_I8MM
+ "${AOM_ROOT}/aom_dsp/arm/aom_convolve8_neon_i8mm.c")
+
if(CONFIG_AV1_HIGHBITDEPTH)
list(APPEND AOM_DSP_COMMON_INTRIN_SSE2
"${AOM_ROOT}/aom_dsp/x86/highbd_convolve_sse2.c"
@@ -134,6 +139,11 @@
"${AOM_ROOT}/aom_dsp/x86/highbd_loopfilter_avx2.c")
list(APPEND AOM_DSP_COMMON_INTRIN_NEON
+ "${AOM_ROOT}/aom_dsp/arm/highbd_blend_a64_hmask_neon.c"
+ "${AOM_ROOT}/aom_dsp/arm/highbd_blend_a64_mask_neon.c"
+ "${AOM_ROOT}/aom_dsp/arm/highbd_blend_a64_vmask_neon.c"
+ "${AOM_ROOT}/aom_dsp/arm/highbd_convolve8_neon.c"
+ "${AOM_ROOT}/aom_dsp/arm/highbd_intrapred_neon.c"
"${AOM_ROOT}/aom_dsp/arm/highbd_loopfilter_neon.c")
endif()
@@ -191,6 +201,9 @@
list(APPEND AOM_DSP_ENCODER_INTRIN_AVX2
"${AOM_ROOT}/aom_dsp/flow_estimation/x86/corner_match_avx2.c")
+
+ list(APPEND AOM_DSP_ENCODER_INTRIN_NEON
+ "${AOM_ROOT}/aom_dsp/flow_estimation/arm/disflow_neon.c")
endif()
list(APPEND AOM_DSP_ENCODER_ASM_SSE2 "${AOM_ROOT}/aom_dsp/x86/sad4d_sse2.asm"
@@ -269,7 +282,15 @@
"${AOM_ROOT}/aom_dsp/arm/obmc_variance_neon.c"
"${AOM_ROOT}/aom_dsp/arm/obmc_sad_neon.c"
"${AOM_ROOT}/aom_dsp/arm/sse_neon.c"
- "${AOM_ROOT}/aom_dsp/arm/sum_squares_neon.c")
+ "${AOM_ROOT}/aom_dsp/arm/sum_squares_neon.c"
+ "${AOM_ROOT}/aom_dsp/arm/blk_sse_sum_neon.c")
+
+ list(APPEND AOM_DSP_ENCODER_INTRIN_NEON_DOTPROD
+ "${AOM_ROOT}/aom_dsp/arm/sad_neon_dotprod.c"
+ "${AOM_ROOT}/aom_dsp/arm/sadxd_neon_dotprod.c"
+ "${AOM_ROOT}/aom_dsp/arm/sse_neon_dotprod.c"
+ "${AOM_ROOT}/aom_dsp/arm/sum_squares_neon_dotprod.c"
+ "${AOM_ROOT}/aom_dsp/arm/variance_neon_dotprod.c")
if(CONFIG_AV1_HIGHBITDEPTH)
list(APPEND AOM_DSP_ENCODER_ASM_SSE2
@@ -292,11 +313,20 @@
list(APPEND AOM_DSP_ENCODER_INTRIN_NEON
"${AOM_ROOT}/aom_dsp/arm/highbd_avg_neon.c"
+ "${AOM_ROOT}/aom_dsp/arm/highbd_avg_pred_neon.c"
"${AOM_ROOT}/aom_dsp/arm/highbd_hadamard_neon.c"
+ "${AOM_ROOT}/aom_dsp/arm/highbd_masked_sad_neon.c"
+ "${AOM_ROOT}/aom_dsp/arm/highbd_obmc_sad_neon.c"
+ "${AOM_ROOT}/aom_dsp/arm/highbd_obmc_variance_neon.c"
"${AOM_ROOT}/aom_dsp/arm/highbd_quantize_neon.c"
"${AOM_ROOT}/aom_dsp/arm/highbd_sad_neon.c"
- "${AOM_ROOT}/aom_dsp/arm/highbd_sad4d_neon.c"
+ "${AOM_ROOT}/aom_dsp/arm/highbd_sadxd_neon.c"
+ "${AOM_ROOT}/aom_dsp/arm/highbd_sse_neon.c"
+ "${AOM_ROOT}/aom_dsp/arm/highbd_subpel_variance_neon.c"
"${AOM_ROOT}/aom_dsp/arm/highbd_variance_neon.c")
+
+ list(APPEND AOM_DSP_ENCODER_INTRIN_NEON_DOTPROD
+ "${AOM_ROOT}/aom_dsp/arm/highbd_variance_neon_dotprod.c")
endif()
if(CONFIG_INTERNAL_STATS)
@@ -326,6 +356,10 @@
list(REMOVE_ITEM AOM_DSP_ENCODER_INTRIN_SSE2
"${AOM_ROOT}/aom_dsp/x86/adaptive_quantize_sse2.c")
+
+ list(REMOVE_ITEM AOM_DSP_ENCODER_INTRIN_NEON
+ "${AOM_ROOT}/aom_dsp/arm/highbd_obmc_variance_neon.c"
+ "${AOM_ROOT}/aom_dsp/arm/obmc_variance_neon.c")
endif()
endif()
@@ -433,6 +467,23 @@
endif()
endif()
+ if(HAVE_NEON_DOTPROD)
+ add_intrinsics_object_library("${AOM_NEON_DOTPROD_FLAG}" "neon_dotprod"
+ "aom_dsp_common"
+ "AOM_DSP_COMMON_INTRIN_NEON_DOTPROD")
+ if(CONFIG_AV1_ENCODER)
+ add_intrinsics_object_library("${AOM_NEON_DOTPROD_FLAG}" "neon_dotprod"
+ "aom_dsp_encoder"
+ "AOM_DSP_ENCODER_INTRIN_NEON_DOTPROD")
+ endif()
+ endif()
+
+ if(HAVE_NEON_I8MM)
+ add_intrinsics_object_library("${AOM_NEON_I8MM_FLAG}" "neon_i8mm"
+ "aom_dsp_common"
+ "AOM_DSP_COMMON_INTRIN_NEON_I8MM")
+ endif()
+
target_sources(aom PRIVATE $<TARGET_OBJECTS:aom_dsp>)
if(BUILD_SHARED_LIBS)
target_sources(aom_static PRIVATE $<TARGET_OBJECTS:aom_dsp>)
diff --git a/aom_dsp/aom_dsp_common.h b/aom_dsp/aom_dsp_common.h
index efb634a..85dc005 100644
--- a/aom_dsp/aom_dsp_common.h
+++ b/aom_dsp/aom_dsp_common.h
@@ -23,10 +23,6 @@
#define PI 3.141592653589793238462643383279502884
-#ifndef MAX_SB_SIZE
-#define MAX_SB_SIZE 128
-#endif // ndef MAX_SB_SIZE
-
#define AOMMIN(x, y) (((x) < (y)) ? (x) : (y))
#define AOMMAX(x, y) (((x) > (y)) ? (x) : (y))
#define AOMSIGN(x) ((x) < 0 ? -1 : 0)
diff --git a/aom_dsp/aom_dsp_rtcd_defs.pl b/aom_dsp/aom_dsp_rtcd_defs.pl
index e738971..c9b2682 100755
--- a/aom_dsp/aom_dsp_rtcd_defs.pl
+++ b/aom_dsp/aom_dsp_rtcd_defs.pl
@@ -497,22 +497,22 @@
add_proto qw/void aom_convolve8_horiz/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h";
add_proto qw/void aom_convolve8_vert/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h";
-specialize qw/aom_convolve_copy neon sse2 avx2/;
-specialize qw/aom_convolve8_horiz neon sse2 ssse3/, "$avx2_ssse3";
-specialize qw/aom_convolve8_vert neon sse2 ssse3/, "$avx2_ssse3";
+specialize qw/aom_convolve_copy neon sse2 avx2/;
+specialize qw/aom_convolve8_horiz neon neon_dotprod neon_i8mm sse2 ssse3/, "$avx2_ssse3";
+specialize qw/aom_convolve8_vert neon neon_dotprod neon_i8mm sse2 ssse3/, "$avx2_ssse3";
add_proto qw/void aom_scaled_2d/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h";
specialize qw/aom_scaled_2d ssse3 neon/;
if (aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") {
add_proto qw/void aom_highbd_convolve_copy/, "const uint16_t *src, ptrdiff_t src_stride, uint16_t *dst, ptrdiff_t dst_stride, int w, int h";
- specialize qw/aom_highbd_convolve_copy sse2 avx2/;
+ specialize qw/aom_highbd_convolve_copy sse2 avx2 neon/;
add_proto qw/void aom_highbd_convolve8_horiz/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, int bd";
- specialize qw/aom_highbd_convolve8_horiz sse2 avx2/;
+ specialize qw/aom_highbd_convolve8_horiz sse2 avx2 neon/;
add_proto qw/void aom_highbd_convolve8_vert/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, int bd";
- specialize qw/aom_highbd_convolve8_vert sse2 avx2/;
+ specialize qw/aom_highbd_convolve8_vert sse2 avx2 neon/;
}
#
@@ -750,7 +750,7 @@
add_proto qw/void aom_blend_a64_mask/, "uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subw, int subh";
add_proto qw/void aom_blend_a64_hmask/, "uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, int w, int h";
add_proto qw/void aom_blend_a64_vmask/, "uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, int w, int h";
-specialize "aom_blend_a64_mask", qw/sse4_1 avx2/;
+specialize "aom_blend_a64_mask", qw/sse4_1 neon avx2/;
specialize "aom_blend_a64_hmask", qw/sse4_1 neon/;
specialize "aom_blend_a64_vmask", qw/sse4_1 neon/;
@@ -759,10 +759,10 @@
add_proto qw/void aom_highbd_blend_a64_hmask/, "uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, int w, int h, int bd";
add_proto qw/void aom_highbd_blend_a64_vmask/, "uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, int w, int h, int bd";
add_proto qw/void aom_highbd_blend_a64_d16_mask/, "uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subw, int subh, ConvolveParams *conv_params, const int bd";
- specialize "aom_highbd_blend_a64_mask", qw/sse4_1/;
- specialize "aom_highbd_blend_a64_hmask", qw/sse4_1/;
- specialize "aom_highbd_blend_a64_vmask", qw/sse4_1/;
- specialize "aom_highbd_blend_a64_d16_mask", qw/sse4_1 avx2/;
+ specialize "aom_highbd_blend_a64_mask", qw/sse4_1 neon/;
+ specialize "aom_highbd_blend_a64_hmask", qw/sse4_1 neon/;
+ specialize "aom_highbd_blend_a64_vmask", qw/sse4_1 neon/;
+ specialize "aom_highbd_blend_a64_d16_mask", qw/sse4_1 neon avx2/;
}
if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
@@ -773,35 +773,33 @@
specialize qw/aom_subtract_block neon sse2 avx2/;
add_proto qw/int64_t/, "aom_sse", "const uint8_t *a, int a_stride, const uint8_t *b,int b_stride, int width, int height";
- specialize qw/aom_sse sse4_1 avx2 neon/;
+ specialize qw/aom_sse sse4_1 avx2 neon neon_dotprod/;
add_proto qw/void/, "aom_get_blk_sse_sum", "const int16_t *data, int stride, int bw, int bh, int *x_sum, int64_t *x2_sum";
- specialize qw/aom_get_blk_sse_sum sse2 avx2/;
+ specialize qw/aom_get_blk_sse_sum sse2 avx2 neon/;
if (aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") {
add_proto qw/void aom_highbd_subtract_block/, "int rows, int cols, int16_t *diff_ptr, ptrdiff_t diff_stride, const uint8_t *src_ptr, ptrdiff_t src_stride, const uint8_t *pred_ptr, ptrdiff_t pred_stride";
specialize qw/aom_highbd_subtract_block sse2 neon/;
add_proto qw/int64_t/, "aom_highbd_sse", "const uint8_t *a8, int a_stride, const uint8_t *b8,int b_stride, int width, int height";
- specialize qw/aom_highbd_sse sse4_1 avx2 neon/;
+ specialize qw/aom_highbd_sse sse4_1 avx2 neon/;
}
- if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
- #
- # Sum of Squares
- #
- add_proto qw/uint64_t aom_sum_squares_2d_i16/, "const int16_t *src, int stride, int width, int height";
- specialize qw/aom_sum_squares_2d_i16 sse2 avx2 neon/;
+ #
+ # Sum of Squares
+ #
+ add_proto qw/uint64_t aom_sum_squares_2d_i16/, "const int16_t *src, int stride, int width, int height";
+ specialize qw/aom_sum_squares_2d_i16 sse2 avx2 neon/;
- add_proto qw/uint64_t aom_sum_squares_i16/, "const int16_t *src, uint32_t N";
- specialize qw/aom_sum_squares_i16 sse2 neon/;
+ add_proto qw/uint64_t aom_sum_squares_i16/, "const int16_t *src, uint32_t N";
+ specialize qw/aom_sum_squares_i16 sse2 neon/;
- add_proto qw/uint64_t aom_var_2d_u8/, "uint8_t *src, int src_stride, int width, int height";
- specialize qw/aom_var_2d_u8 sse2 avx2 neon/;
+ add_proto qw/uint64_t aom_var_2d_u8/, "uint8_t *src, int src_stride, int width, int height";
+ specialize qw/aom_var_2d_u8 sse2 avx2 neon neon_dotprod/;
- add_proto qw/uint64_t aom_var_2d_u16/, "uint8_t *src, int src_stride, int width, int height";
- specialize qw/aom_var_2d_u16 sse2 avx2 neon/;
- }
+ add_proto qw/uint64_t aom_var_2d_u16/, "uint8_t *src, int src_stride, int width, int height";
+ specialize qw/aom_var_2d_u16 sse2 avx2 neon/;
#
# Single block SAD / Single block Avg SAD
@@ -816,65 +814,65 @@
add_proto qw/uint64_t aom_sum_sse_2d_i16/, "const int16_t *src, int src_stride, int width, int height, int *sum";
specialize qw/aom_sum_sse_2d_i16 avx2 neon sse2/;
- specialize qw/aom_sad128x128 avx2 neon sse2/;
- specialize qw/aom_sad128x64 avx2 neon sse2/;
- specialize qw/aom_sad64x128 avx2 neon sse2/;
- specialize qw/aom_sad64x64 avx2 neon sse2/;
- specialize qw/aom_sad64x32 avx2 neon sse2/;
- specialize qw/aom_sad32x64 avx2 neon sse2/;
- specialize qw/aom_sad32x32 avx2 neon sse2/;
- specialize qw/aom_sad32x16 avx2 neon sse2/;
- specialize qw/aom_sad16x32 neon sse2/;
- specialize qw/aom_sad16x16 neon sse2/;
- specialize qw/aom_sad16x8 neon sse2/;
- specialize qw/aom_sad8x16 neon sse2/;
- specialize qw/aom_sad8x8 neon sse2/;
- specialize qw/aom_sad8x4 neon sse2/;
- specialize qw/aom_sad4x8 neon sse2/;
- specialize qw/aom_sad4x4 neon sse2/;
+ specialize qw/aom_sad128x128 avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad128x64 avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad64x128 avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad64x64 avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad64x32 avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad32x64 avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad32x32 avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad32x16 avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad16x32 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad16x16 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad16x8 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad8x16 sse2 neon/;
+ specialize qw/aom_sad8x8 sse2 neon/;
+ specialize qw/aom_sad8x4 sse2 neon/;
+ specialize qw/aom_sad4x8 sse2 neon/;
+ specialize qw/aom_sad4x4 sse2 neon/;
- specialize qw/aom_sad4x16 neon sse2/;
- specialize qw/aom_sad16x4 neon sse2/;
- specialize qw/aom_sad8x32 neon sse2/;
- specialize qw/aom_sad32x8 neon sse2/;
- specialize qw/aom_sad16x64 neon sse2/;
- specialize qw/aom_sad64x16 neon sse2/;
+ specialize qw/aom_sad4x16 sse2 neon/;
+ specialize qw/aom_sad16x4 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad8x32 sse2 neon/;
+ specialize qw/aom_sad32x8 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad16x64 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad64x16 sse2 neon neon_dotprod/;
- specialize qw/aom_sad_skip_128x128 avx2 sse2 neon/;
- specialize qw/aom_sad_skip_128x64 avx2 sse2 neon/;
- specialize qw/aom_sad_skip_64x128 avx2 sse2 neon/;
- specialize qw/aom_sad_skip_64x64 avx2 sse2 neon/;
- specialize qw/aom_sad_skip_64x32 avx2 sse2 neon/;
- specialize qw/aom_sad_skip_32x64 avx2 sse2 neon/;
- specialize qw/aom_sad_skip_32x32 avx2 sse2 neon/;
- specialize qw/aom_sad_skip_32x16 avx2 sse2 neon/;
- specialize qw/aom_sad_skip_16x32 sse2 neon/;
- specialize qw/aom_sad_skip_16x16 sse2 neon/;
- specialize qw/aom_sad_skip_16x8 sse2 neon/;
- specialize qw/aom_sad_skip_8x16 sse2 neon/;
- specialize qw/aom_sad_skip_8x8 sse2 neon/;
- specialize qw/aom_sad_skip_8x4 neon/;
- specialize qw/aom_sad_skip_4x8 sse2 neon/;
- specialize qw/aom_sad_skip_4x4 neon/;
+ specialize qw/aom_sad_skip_128x128 avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad_skip_128x64 avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad_skip_64x128 avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad_skip_64x64 avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad_skip_64x32 avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad_skip_32x64 avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad_skip_32x32 avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad_skip_32x16 avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad_skip_16x32 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad_skip_16x16 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad_skip_16x8 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad_skip_8x16 sse2 neon/;
+ specialize qw/aom_sad_skip_8x8 sse2 neon/;
+ specialize qw/aom_sad_skip_8x4 neon/;
+ specialize qw/aom_sad_skip_4x8 sse2 neon/;
+ specialize qw/aom_sad_skip_4x4 neon/;
- specialize qw/aom_sad_skip_4x16 sse2 neon/;
- specialize qw/aom_sad_skip_16x4 neon/;
- specialize qw/aom_sad_skip_8x32 sse2 neon/;
- specialize qw/aom_sad_skip_32x8 sse2 neon/;
- specialize qw/aom_sad_skip_16x64 sse2 neon/;
- specialize qw/aom_sad_skip_64x16 sse2 neon/;
+ specialize qw/aom_sad_skip_4x16 sse2 neon/;
+ specialize qw/aom_sad_skip_16x4 neon neon_dotprod/;
+ specialize qw/aom_sad_skip_8x32 sse2 neon/;
+ specialize qw/aom_sad_skip_32x8 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad_skip_16x64 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad_skip_64x16 sse2 neon neon_dotprod/;
- specialize qw/aom_sad128x128_avg avx2 sse2 neon/;
- specialize qw/aom_sad128x64_avg avx2 sse2 neon/;
- specialize qw/aom_sad64x128_avg avx2 sse2 neon/;
- specialize qw/aom_sad64x64_avg avx2 sse2 neon/;
- specialize qw/aom_sad64x32_avg avx2 sse2 neon/;
- specialize qw/aom_sad32x64_avg avx2 sse2 neon/;
- specialize qw/aom_sad32x32_avg avx2 sse2 neon/;
- specialize qw/aom_sad32x16_avg avx2 sse2 neon/;
- specialize qw/aom_sad16x32_avg sse2 neon/;
- specialize qw/aom_sad16x16_avg sse2 neon/;
- specialize qw/aom_sad16x8_avg sse2 neon/;
+ specialize qw/aom_sad128x128_avg avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad128x64_avg avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad64x128_avg avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad64x64_avg avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad64x32_avg avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad32x64_avg avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad32x32_avg avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad32x16_avg avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad16x32_avg sse2 neon neon_dotprod/;
+ specialize qw/aom_sad16x16_avg sse2 neon neon_dotprod/;
+ specialize qw/aom_sad16x8_avg sse2 neon neon_dotprod/;
specialize qw/aom_sad8x16_avg sse2 neon/;
specialize qw/aom_sad8x8_avg sse2 neon/;
specialize qw/aom_sad8x4_avg sse2 neon/;
@@ -882,36 +880,36 @@
specialize qw/aom_sad4x4_avg sse2 neon/;
specialize qw/aom_sad4x16_avg sse2 neon/;
- specialize qw/aom_sad16x4_avg sse2 neon/;
+ specialize qw/aom_sad16x4_avg sse2 neon neon_dotprod/;
specialize qw/aom_sad8x32_avg sse2 neon/;
- specialize qw/aom_sad32x8_avg sse2 neon/;
- specialize qw/aom_sad16x64_avg sse2 neon/;
- specialize qw/aom_sad64x16_avg sse2 neon/;
+ specialize qw/aom_sad32x8_avg sse2 neon neon_dotprod/;
+ specialize qw/aom_sad16x64_avg sse2 neon neon_dotprod/;
+ specialize qw/aom_sad64x16_avg sse2 neon neon_dotprod/;
- specialize qw/aom_dist_wtd_sad128x128_avg sse2/;
- specialize qw/aom_dist_wtd_sad128x64_avg sse2/;
- specialize qw/aom_dist_wtd_sad64x128_avg sse2/;
- specialize qw/aom_dist_wtd_sad64x64_avg sse2/;
- specialize qw/aom_dist_wtd_sad64x32_avg sse2/;
- specialize qw/aom_dist_wtd_sad32x64_avg sse2/;
- specialize qw/aom_dist_wtd_sad32x32_avg sse2/;
- specialize qw/aom_dist_wtd_sad32x16_avg sse2/;
- specialize qw/aom_dist_wtd_sad16x32_avg sse2/;
- specialize qw/aom_dist_wtd_sad16x16_avg sse2/;
- specialize qw/aom_dist_wtd_sad16x8_avg sse2/;
- specialize qw/aom_dist_wtd_sad8x16_avg sse2/;
- specialize qw/aom_dist_wtd_sad8x8_avg sse2/;
- specialize qw/aom_dist_wtd_sad8x4_avg sse2/;
- specialize qw/aom_dist_wtd_sad4x8_avg sse2/;
- specialize qw/aom_dist_wtd_sad4x4_avg sse2/;
+ specialize qw/aom_dist_wtd_sad128x128_avg sse2 neon neon_dotprod/;
+ specialize qw/aom_dist_wtd_sad128x64_avg sse2 neon neon_dotprod/;
+ specialize qw/aom_dist_wtd_sad64x128_avg sse2 neon neon_dotprod/;
+ specialize qw/aom_dist_wtd_sad64x64_avg sse2 neon neon_dotprod/;
+ specialize qw/aom_dist_wtd_sad64x32_avg sse2 neon neon_dotprod/;
+ specialize qw/aom_dist_wtd_sad32x64_avg sse2 neon neon_dotprod/;
+ specialize qw/aom_dist_wtd_sad32x32_avg sse2 neon neon_dotprod/;
+ specialize qw/aom_dist_wtd_sad32x16_avg sse2 neon neon_dotprod/;
+ specialize qw/aom_dist_wtd_sad16x32_avg sse2 neon neon_dotprod/;
+ specialize qw/aom_dist_wtd_sad16x16_avg sse2 neon neon_dotprod/;
+ specialize qw/aom_dist_wtd_sad16x8_avg sse2 neon neon_dotprod/;
+ specialize qw/aom_dist_wtd_sad8x16_avg sse2 neon/;
+ specialize qw/aom_dist_wtd_sad8x8_avg sse2 neon/;
+ specialize qw/aom_dist_wtd_sad8x4_avg sse2 neon/;
+ specialize qw/aom_dist_wtd_sad4x8_avg sse2 neon/;
+ specialize qw/aom_dist_wtd_sad4x4_avg sse2 neon/;
if (aom_config("CONFIG_REALTIME_ONLY") ne "yes") {
- specialize qw/aom_dist_wtd_sad4x16_avg sse2/;
- specialize qw/aom_dist_wtd_sad16x4_avg sse2/;
- specialize qw/aom_dist_wtd_sad8x32_avg sse2/;
- specialize qw/aom_dist_wtd_sad32x8_avg sse2/;
- specialize qw/aom_dist_wtd_sad16x64_avg sse2/;
- specialize qw/aom_dist_wtd_sad64x16_avg sse2/;
+ specialize qw/aom_dist_wtd_sad4x16_avg sse2 neon/;
+ specialize qw/aom_dist_wtd_sad16x4_avg sse2 neon neon_dotprod/;
+ specialize qw/aom_dist_wtd_sad8x32_avg sse2 neon/;
+ specialize qw/aom_dist_wtd_sad32x8_avg sse2 neon neon_dotprod/;
+ specialize qw/aom_dist_wtd_sad16x64_avg sse2 neon neon_dotprod/;
+ specialize qw/aom_dist_wtd_sad64x16_avg sse2 neon neon_dotprod/;
}
if (aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") {
@@ -974,27 +972,29 @@
specialize qw/aom_highbd_sad_skip_16x64 avx2 sse2 neon/;
specialize qw/aom_highbd_sad_skip_64x16 avx2 sse2 neon/;
- specialize qw/aom_highbd_sad128x128_avg avx2/;
- specialize qw/aom_highbd_sad128x64_avg avx2/;
- specialize qw/aom_highbd_sad64x128_avg avx2/;
- specialize qw/aom_highbd_sad64x64_avg avx2 sse2/;
- specialize qw/aom_highbd_sad64x32_avg avx2 sse2/;
- specialize qw/aom_highbd_sad32x64_avg avx2 sse2/;
- specialize qw/aom_highbd_sad32x32_avg avx2 sse2/;
- specialize qw/aom_highbd_sad32x16_avg avx2 sse2/;
- specialize qw/aom_highbd_sad16x32_avg avx2 sse2/;
- specialize qw/aom_highbd_sad16x16_avg avx2 sse2/;
- specialize qw/aom_highbd_sad16x8_avg avx2 sse2/;
- specialize qw/aom_highbd_sad8x4_avg sse2/;
- specialize qw/aom_highbd_sad4x8_avg sse2/;
- specialize qw/aom_highbd_sad4x4_avg sse2/;
+ specialize qw/aom_highbd_sad128x128_avg avx2 neon/;
+ specialize qw/aom_highbd_sad128x64_avg avx2 neon/;
+ specialize qw/aom_highbd_sad64x128_avg avx2 neon/;
+ specialize qw/aom_highbd_sad64x64_avg avx2 sse2 neon/;
+ specialize qw/aom_highbd_sad64x32_avg avx2 sse2 neon/;
+ specialize qw/aom_highbd_sad32x64_avg avx2 sse2 neon/;
+ specialize qw/aom_highbd_sad32x32_avg avx2 sse2 neon/;
+ specialize qw/aom_highbd_sad32x16_avg avx2 sse2 neon/;
+ specialize qw/aom_highbd_sad16x32_avg avx2 sse2 neon/;
+ specialize qw/aom_highbd_sad16x16_avg avx2 sse2 neon/;
+ specialize qw/aom_highbd_sad16x8_avg avx2 sse2 neon/;
+ specialize qw/aom_highbd_sad8x16_avg neon/;
+ specialize qw/aom_highbd_sad8x8_avg neon/;
+ specialize qw/aom_highbd_sad8x4_avg sse2 neon/;
+ specialize qw/aom_highbd_sad4x8_avg sse2 neon/;
+ specialize qw/aom_highbd_sad4x4_avg sse2 neon/;
- specialize qw/aom_highbd_sad4x16_avg sse2/;
- specialize qw/aom_highbd_sad16x4_avg avx2 sse2/;
- specialize qw/aom_highbd_sad8x32_avg sse2/;
- specialize qw/aom_highbd_sad32x8_avg avx2 sse2/;
- specialize qw/aom_highbd_sad16x64_avg avx2 sse2/;
- specialize qw/aom_highbd_sad64x16_avg avx2 sse2/;
+ specialize qw/aom_highbd_sad4x16_avg sse2 neon/;
+ specialize qw/aom_highbd_sad8x32_avg sse2 neon/;
+ specialize qw/aom_highbd_sad16x4_avg avx2 sse2 neon/;
+ specialize qw/aom_highbd_sad16x64_avg avx2 sse2 neon/;
+ specialize qw/aom_highbd_sad32x8_avg avx2 sse2 neon/;
+ specialize qw/aom_highbd_sad64x16_avg avx2 sse2 neon/;
}
#
# Masked SAD
@@ -1009,7 +1009,7 @@
foreach (@encoder_block_sizes) {
($w, $h) = @$_;
add_proto qw/unsigned int/, "aom_highbd_masked_sad${w}x${h}", "const uint8_t *src8, int src_stride, const uint8_t *ref8, int ref_stride, const uint8_t *second_pred8, const uint8_t *msk, int msk_stride, int invert_mask";
- specialize "aom_highbd_masked_sad${w}x${h}", qw/ssse3 avx2/;
+ specialize "aom_highbd_masked_sad${w}x${h}", qw/ssse3 avx2 neon/;
}
}
@@ -1030,7 +1030,7 @@
($w, $h) = @$_;
add_proto qw/unsigned int/, "aom_highbd_obmc_sad${w}x${h}", "const uint8_t *pre, int pre_stride, const int32_t *wsrc, const int32_t *mask";
if (! (($w == 128 && $h == 32) || ($w == 32 && $h == 128))) {
- specialize "aom_highbd_obmc_sad${w}x${h}", qw/sse4_1 avx2/;
+ specialize "aom_highbd_obmc_sad${w}x${h}", qw/sse4_1 avx2 neon/;
}
}
}
@@ -1047,47 +1047,47 @@
add_proto qw/void/, "aom_masked_sad${w}x${h}x4d", "const uint8_t *src, int src_stride, const uint8_t *ref[4], int ref_stride, const uint8_t *second_pred, const uint8_t *msk, int msk_stride, int invert_mask, unsigned sads[4]";
}
- specialize qw/aom_sad128x128x4d avx2 neon sse2/;
- specialize qw/aom_sad128x64x4d avx2 neon sse2/;
- specialize qw/aom_sad64x128x4d avx2 neon sse2/;
- specialize qw/aom_sad64x64x4d avx2 neon sse2/;
- specialize qw/aom_sad64x32x4d avx2 neon sse2/;
- specialize qw/aom_sad32x64x4d avx2 neon sse2/;
- specialize qw/aom_sad32x32x4d avx2 neon sse2/;
- specialize qw/aom_sad32x16x4d avx2 neon sse2/;
- specialize qw/aom_sad16x32x4d avx2 neon sse2/;
- specialize qw/aom_sad16x16x4d avx2 neon sse2/;
- specialize qw/aom_sad16x8x4d avx2 neon sse2/;
+ specialize qw/aom_sad128x128x4d avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad128x64x4d avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad64x128x4d avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad64x64x4d avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad64x32x4d avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad32x64x4d avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad32x32x4d avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad32x16x4d avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad16x32x4d avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad16x16x4d avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad16x8x4d avx2 sse2 neon neon_dotprod/;
- specialize qw/aom_sad8x16x4d neon sse2/;
- specialize qw/aom_sad8x8x4d neon sse2/;
- specialize qw/aom_sad8x4x4d neon sse2/;
- specialize qw/aom_sad4x8x4d neon sse2/;
- specialize qw/aom_sad4x4x4d neon sse2/;
+ specialize qw/aom_sad8x16x4d sse2 neon/;
+ specialize qw/aom_sad8x8x4d sse2 neon/;
+ specialize qw/aom_sad8x4x4d sse2 neon/;
+ specialize qw/aom_sad4x8x4d sse2 neon/;
+ specialize qw/aom_sad4x4x4d sse2 neon/;
- specialize qw/aom_sad64x16x4d avx2 neon sse2/;
- specialize qw/aom_sad32x8x4d avx2 neon sse2/;
- specialize qw/aom_sad16x64x4d avx2 neon sse2/;
- specialize qw/aom_sad16x4x4d avx2 neon sse2/;
- specialize qw/aom_sad8x32x4d neon sse2/;
- specialize qw/aom_sad4x16x4d neon sse2/;
+ specialize qw/aom_sad64x16x4d avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad32x8x4d avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad16x64x4d avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad16x4x4d avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad8x32x4d sse2 neon/;
+ specialize qw/aom_sad4x16x4d sse2 neon/;
- specialize qw/aom_sad_skip_128x128x4d avx2 sse2 neon/;
- specialize qw/aom_sad_skip_128x64x4d avx2 sse2 neon/;
- specialize qw/aom_sad_skip_64x128x4d avx2 sse2 neon/;
- specialize qw/aom_sad_skip_64x64x4d avx2 sse2 neon/;
- specialize qw/aom_sad_skip_64x32x4d avx2 sse2 neon/;
- specialize qw/aom_sad_skip_64x16x4d avx2 sse2 neon/;
- specialize qw/aom_sad_skip_32x64x4d avx2 sse2 neon/;
- specialize qw/aom_sad_skip_32x32x4d avx2 sse2 neon/;
- specialize qw/aom_sad_skip_32x16x4d avx2 sse2 neon/;
- specialize qw/aom_sad_skip_32x8x4d avx2 sse2 neon/;
+ specialize qw/aom_sad_skip_128x128x4d avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad_skip_128x64x4d avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad_skip_64x128x4d avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad_skip_64x64x4d avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad_skip_64x32x4d avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad_skip_64x16x4d avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad_skip_32x64x4d avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad_skip_32x32x4d avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad_skip_32x16x4d avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad_skip_32x8x4d avx2 sse2 neon neon_dotprod/;
- specialize qw/aom_sad_skip_16x64x4d avx2 sse2 neon/;
- specialize qw/aom_sad_skip_16x32x4d avx2 sse2 neon/;
- specialize qw/aom_sad_skip_16x16x4d avx2 sse2 neon/;
- specialize qw/aom_sad_skip_16x8x4d avx2 sse2 neon/;
- specialize qw/aom_sad_skip_16x4x4d neon/;
+ specialize qw/aom_sad_skip_16x64x4d avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad_skip_16x32x4d avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad_skip_16x16x4d avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad_skip_16x8x4d avx2 sse2 neon neon_dotprod/;
+ specialize qw/aom_sad_skip_16x4x4d neon neon_dotprod/;
specialize qw/aom_sad_skip_8x32x4d sse2 neon/;
specialize qw/aom_sad_skip_8x16x4d sse2 neon/;
specialize qw/aom_sad_skip_8x8x4d sse2 neon/;
@@ -1096,29 +1096,29 @@
specialize qw/aom_sad_skip_4x8x4d sse2 neon/;
specialize qw/aom_sad_skip_4x4x4d neon/;
- specialize qw/aom_sad128x128x3d neon avx2/;
- specialize qw/aom_sad128x64x3d neon avx2/;
- specialize qw/aom_sad64x128x3d neon avx2/;
- specialize qw/aom_sad64x64x3d neon avx2/;
- specialize qw/aom_sad64x32x3d neon avx2/;
- specialize qw/aom_sad32x64x3d neon avx2/;
- specialize qw/aom_sad32x32x3d neon avx2/;
- specialize qw/aom_sad32x16x3d neon avx2/;
- specialize qw/aom_sad16x32x3d neon avx2/;
- specialize qw/aom_sad16x16x3d neon avx2/;
- specialize qw/aom_sad16x8x3d neon avx2/;
- specialize qw/aom_sad8x16x3d neon/;
- specialize qw/aom_sad8x8x3d neon/;
- specialize qw/aom_sad8x4x3d neon/;
- specialize qw/aom_sad4x8x3d neon/;
- specialize qw/aom_sad4x4x3d neon/;
+ specialize qw/aom_sad128x128x3d avx2 neon neon_dotprod/;
+ specialize qw/aom_sad128x64x3d avx2 neon neon_dotprod/;
+ specialize qw/aom_sad64x128x3d avx2 neon neon_dotprod/;
+ specialize qw/aom_sad64x64x3d avx2 neon neon_dotprod/;
+ specialize qw/aom_sad64x32x3d avx2 neon neon_dotprod/;
+ specialize qw/aom_sad32x64x3d avx2 neon neon_dotprod/;
+ specialize qw/aom_sad32x32x3d avx2 neon neon_dotprod/;
+ specialize qw/aom_sad32x16x3d avx2 neon neon_dotprod/;
+ specialize qw/aom_sad16x32x3d avx2 neon neon_dotprod/;
+ specialize qw/aom_sad16x16x3d avx2 neon neon_dotprod/;
+ specialize qw/aom_sad16x8x3d avx2 neon neon_dotprod/;
+ specialize qw/aom_sad8x16x3d neon/;
+ specialize qw/aom_sad8x8x3d neon/;
+ specialize qw/aom_sad8x4x3d neon/;
+ specialize qw/aom_sad4x8x3d neon/;
+ specialize qw/aom_sad4x4x3d neon/;
- specialize qw/aom_sad64x16x3d neon avx2/;
- specialize qw/aom_sad32x8x3d neon avx2/;
- specialize qw/aom_sad16x64x3d neon avx2/;
- specialize qw/aom_sad16x4x3d neon/;
- specialize qw/aom_sad8x32x3d neon/;
- specialize qw/aom_sad4x16x3d neon/;
+ specialize qw/aom_sad64x16x3d avx2 neon neon_dotprod/;
+ specialize qw/aom_sad32x8x3d avx2 neon neon_dotprod/;
+ specialize qw/aom_sad16x64x3d avx2 neon neon_dotprod/;
+ specialize qw/aom_sad16x4x3d neon neon_dotprod/;
+ specialize qw/aom_sad8x32x3d neon/;
+ specialize qw/aom_sad4x16x3d neon/;
specialize qw/aom_masked_sad128x128x4d ssse3 neon/;
specialize qw/aom_masked_sad128x64x4d ssse3 neon/;
@@ -1153,9 +1153,9 @@
if (aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") {
foreach (@encoder_block_sizes) {
($w, $h) = @$_;
- add_proto qw/void/, "aom_highbd_sad${w}x${h}x4d", "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_ptr[], int ref_stride, uint32_t *sad_array";
- add_proto qw/void/, "aom_highbd_sad${w}x${h}x3d", "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_ptr[], int ref_stride, uint32_t *sad_array";
- add_proto qw/void/, "aom_highbd_sad_skip_${w}x${h}x4d", "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_ptr[], int ref_stride, uint32_t *sad_array";
+ add_proto qw/void/, "aom_highbd_sad${w}x${h}x4d", "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_ptr[4], int ref_stride, uint32_t sad_array[4]";
+ add_proto qw/void/, "aom_highbd_sad${w}x${h}x3d", "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_ptr[4], int ref_stride, uint32_t sad_array[4]";
+ add_proto qw/void/, "aom_highbd_sad_skip_${w}x${h}x4d", "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_ptr[4], int ref_stride, uint32_t sad_array[4]";
if ($w != 128 && $h != 128) {
specialize "aom_highbd_sad${w}x${h}x4d", qw/sse2/;
}
@@ -1208,22 +1208,29 @@
specialize qw/aom_highbd_sad_skip_16x64x4d avx2 sse2 neon/;
specialize qw/aom_highbd_sad_skip_64x16x4d avx2 sse2 neon/;
- specialize qw/aom_highbd_sad128x128x3d avx2/;
- specialize qw/aom_highbd_sad128x64x3d avx2/;
- specialize qw/aom_highbd_sad64x128x3d avx2/;
- specialize qw/aom_highbd_sad64x64x3d avx2/;
- specialize qw/aom_highbd_sad64x32x3d avx2/;
- specialize qw/aom_highbd_sad32x64x3d avx2/;
- specialize qw/aom_highbd_sad32x32x3d avx2/;
- specialize qw/aom_highbd_sad32x16x3d avx2/;
- specialize qw/aom_highbd_sad16x32x3d avx2/;
- specialize qw/aom_highbd_sad16x16x3d avx2/;
- specialize qw/aom_highbd_sad16x8x3d avx2/;
+ specialize qw/aom_highbd_sad128x128x3d avx2 neon/;
+ specialize qw/aom_highbd_sad128x64x3d avx2 neon/;
+ specialize qw/aom_highbd_sad64x128x3d avx2 neon/;
+ specialize qw/aom_highbd_sad64x64x3d avx2 neon/;
+ specialize qw/aom_highbd_sad64x32x3d avx2 neon/;
+ specialize qw/aom_highbd_sad32x64x3d avx2 neon/;
+ specialize qw/aom_highbd_sad32x32x3d avx2 neon/;
+ specialize qw/aom_highbd_sad32x16x3d avx2 neon/;
+ specialize qw/aom_highbd_sad16x32x3d avx2 neon/;
+ specialize qw/aom_highbd_sad16x16x3d avx2 neon/;
+ specialize qw/aom_highbd_sad16x8x3d avx2 neon/;
+ specialize qw/aom_highbd_sad8x16x3d neon/;
+ specialize qw/aom_highbd_sad8x8x3d neon/;
+ specialize qw/aom_highbd_sad8x4x3d neon/;
+ specialize qw/aom_highbd_sad4x8x3d neon/;
+ specialize qw/aom_highbd_sad4x4x3d neon/;
- specialize qw/aom_highbd_sad16x4x3d avx2/;
- specialize qw/aom_highbd_sad32x8x3d avx2/;
- specialize qw/aom_highbd_sad16x64x3d avx2/;
- specialize qw/aom_highbd_sad64x16x3d avx2/;
+ specialize qw/aom_highbd_sad64x16x3d avx2 neon/;
+ specialize qw/aom_highbd_sad32x8x3d avx2 neon/;
+ specialize qw/aom_highbd_sad16x64x3d avx2 neon/;
+ specialize qw/aom_highbd_sad16x4x3d avx2 neon/;
+ specialize qw/aom_highbd_sad8x32x3d neon/;
+ specialize qw/aom_highbd_sad4x16x3d neon/;
}
#
# Avg
@@ -1323,20 +1330,20 @@
# Specialty Variance
#
add_proto qw/void aom_get_var_sse_sum_8x8_quad/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse8x8, int *sum8x8, unsigned int *tot_sse, int *tot_sum, uint32_t *var8x8";
- specialize qw/aom_get_var_sse_sum_8x8_quad avx2 sse2 neon/;
+ specialize qw/aom_get_var_sse_sum_8x8_quad avx2 sse2 neon neon_dotprod/;
add_proto qw/void aom_get_var_sse_sum_16x16_dual/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse16x16, unsigned int *tot_sse, int *tot_sum, uint32_t *var16x16";
- specialize qw/aom_get_var_sse_sum_16x16_dual avx2 sse2 neon/;
+ specialize qw/aom_get_var_sse_sum_16x16_dual avx2 sse2 neon neon_dotprod/;
add_proto qw/unsigned int aom_mse16x16/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int recon_stride, unsigned int *sse";
add_proto qw/unsigned int aom_mse16x8/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int recon_stride, unsigned int *sse";
add_proto qw/unsigned int aom_mse8x16/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int recon_stride, unsigned int *sse";
add_proto qw/unsigned int aom_mse8x8/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int recon_stride, unsigned int *sse";
- specialize qw/aom_mse16x16 sse2 avx2 neon/;
- specialize qw/aom_mse16x8 sse2 neon/;
- specialize qw/aom_mse8x16 sse2 neon/;
- specialize qw/aom_mse8x8 sse2 neon/;
+ specialize qw/aom_mse16x16 sse2 avx2 neon neon_dotprod/;
+ specialize qw/aom_mse16x8 sse2 neon neon_dotprod/;
+ specialize qw/aom_mse8x16 sse2 neon neon_dotprod/;
+ specialize qw/aom_mse8x8 sse2 neon neon_dotprod/;
if (aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") {
foreach $bd (8, 10, 12) {
@@ -1345,31 +1352,32 @@
add_proto qw/unsigned int/, "aom_highbd_${bd}_mse8x16", "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int recon_stride, unsigned int *sse";
add_proto qw/unsigned int/, "aom_highbd_${bd}_mse8x8", "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int recon_stride, unsigned int *sse";
- specialize "aom_highbd_${bd}_mse16x16", qw/sse2/;
- specialize "aom_highbd_${bd}_mse8x8", qw/sse2/;
+ specialize "aom_highbd_${bd}_mse16x16", qw/sse2 neon/;
+ specialize "aom_highbd_${bd}_mse16x8", qw/neon/;
+ specialize "aom_highbd_${bd}_mse8x16", qw/neon/;
+ specialize "aom_highbd_${bd}_mse8x8", qw/sse2 neon/;
}
+
+ specialize "aom_highbd_8_mse16x16", qw/neon_dotprod/;
+ specialize "aom_highbd_8_mse16x8", qw/neon_dotprod/;
+ specialize "aom_highbd_8_mse8x16", qw/neon_dotprod/;
+ specialize "aom_highbd_8_mse8x8", qw/neon_dotprod/;
}
#
#
#
add_proto qw/unsigned int aom_get_mb_ss/, "const int16_t *";
- specialize qw/aom_get_mb_ss sse2/;
+ specialize qw/aom_get_mb_ss sse2 neon/;
#
# Variance / Subpixel Variance / Subpixel Avg Variance
#
- add_proto qw/unsigned int/, "aom_variance2x2", "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
-
- add_proto qw/unsigned int/, "aom_variance2x4", "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
-
- add_proto qw/unsigned int/, "aom_variance4x2", "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
-
add_proto qw/uint64_t/, "aom_mse_wxh_16bit", "uint8_t *dst, int dstride,uint16_t *src, int sstride, int w, int h";
specialize qw/aom_mse_wxh_16bit sse2 avx2 neon/;
add_proto qw/uint64_t/, "aom_mse_16xh_16bit", "uint8_t *dst, int dstride,uint16_t *src, int w, int h";
- specialize qw/aom_mse_16xh_16bit sse2 avx2/;
+ specialize qw/aom_mse_16xh_16bit sse2 avx2 neon/;
foreach (@encoder_block_sizes) {
($w, $h) = @$_;
@@ -1378,22 +1386,22 @@
add_proto qw/uint32_t/, "aom_sub_pixel_avg_variance${w}x${h}", "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
add_proto qw/uint32_t/, "aom_dist_wtd_sub_pixel_avg_variance${w}x${h}", "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param";
}
- specialize qw/aom_variance128x128 sse2 avx2 neon/;
- specialize qw/aom_variance128x64 sse2 avx2 neon/;
- specialize qw/aom_variance64x128 sse2 avx2 neon/;
- specialize qw/aom_variance64x64 sse2 avx2 neon/;
- specialize qw/aom_variance64x32 sse2 avx2 neon/;
- specialize qw/aom_variance32x64 sse2 avx2 neon/;
- specialize qw/aom_variance32x32 sse2 avx2 neon/;
- specialize qw/aom_variance32x16 sse2 avx2 neon/;
- specialize qw/aom_variance16x32 sse2 avx2 neon/;
- specialize qw/aom_variance16x16 sse2 avx2 neon/;
- specialize qw/aom_variance16x8 sse2 avx2 neon/;
- specialize qw/aom_variance8x16 sse2 neon/;
- specialize qw/aom_variance8x8 sse2 neon/;
- specialize qw/aom_variance8x4 sse2 neon/;
- specialize qw/aom_variance4x8 sse2 neon/;
- specialize qw/aom_variance4x4 sse2 neon/;
+ specialize qw/aom_variance128x128 sse2 avx2 neon neon_dotprod/;
+ specialize qw/aom_variance128x64 sse2 avx2 neon neon_dotprod/;
+ specialize qw/aom_variance64x128 sse2 avx2 neon neon_dotprod/;
+ specialize qw/aom_variance64x64 sse2 avx2 neon neon_dotprod/;
+ specialize qw/aom_variance64x32 sse2 avx2 neon neon_dotprod/;
+ specialize qw/aom_variance32x64 sse2 avx2 neon neon_dotprod/;
+ specialize qw/aom_variance32x32 sse2 avx2 neon neon_dotprod/;
+ specialize qw/aom_variance32x16 sse2 avx2 neon neon_dotprod/;
+ specialize qw/aom_variance16x32 sse2 avx2 neon neon_dotprod/;
+ specialize qw/aom_variance16x16 sse2 avx2 neon neon_dotprod/;
+ specialize qw/aom_variance16x8 sse2 avx2 neon neon_dotprod/;
+ specialize qw/aom_variance8x16 sse2 neon neon_dotprod/;
+ specialize qw/aom_variance8x8 sse2 neon neon_dotprod/;
+ specialize qw/aom_variance8x4 sse2 neon neon_dotprod/;
+ specialize qw/aom_variance4x8 sse2 neon neon_dotprod/;
+ specialize qw/aom_variance4x4 sse2 neon neon_dotprod/;
specialize qw/aom_sub_pixel_variance128x128 avx2 neon sse2 ssse3/;
specialize qw/aom_sub_pixel_variance128x64 avx2 neon sse2 ssse3/;
@@ -1430,12 +1438,12 @@
specialize qw/aom_sub_pixel_avg_variance4x4 neon sse2 ssse3/;
if (aom_config("CONFIG_REALTIME_ONLY") ne "yes") {
- specialize qw/aom_variance4x16 neon sse2/;
- specialize qw/aom_variance16x4 neon sse2 avx2/;
- specialize qw/aom_variance8x32 neon sse2/;
- specialize qw/aom_variance32x8 neon sse2 avx2/;
- specialize qw/aom_variance16x64 neon sse2 avx2/;
- specialize qw/aom_variance64x16 neon sse2 avx2/;
+ specialize qw/aom_variance4x16 neon neon_dotprod sse2/;
+ specialize qw/aom_variance16x4 neon neon_dotprod sse2 avx2/;
+ specialize qw/aom_variance8x32 neon neon_dotprod sse2/;
+ specialize qw/aom_variance32x8 neon neon_dotprod sse2 avx2/;
+ specialize qw/aom_variance16x64 neon neon_dotprod sse2 avx2/;
+ specialize qw/aom_variance64x16 neon neon_dotprod sse2 avx2/;
specialize qw/aom_sub_pixel_variance4x16 neon sse2 ssse3/;
specialize qw/aom_sub_pixel_variance16x4 neon avx2 sse2 ssse3/;
@@ -1450,82 +1458,259 @@
specialize qw/aom_sub_pixel_avg_variance16x64 neon sse2 ssse3/;
specialize qw/aom_sub_pixel_avg_variance64x16 neon sse2 ssse3/;
- specialize qw/aom_dist_wtd_sub_pixel_avg_variance4x16 ssse3/;
- specialize qw/aom_dist_wtd_sub_pixel_avg_variance16x4 ssse3/;
- specialize qw/aom_dist_wtd_sub_pixel_avg_variance8x32 ssse3/;
- specialize qw/aom_dist_wtd_sub_pixel_avg_variance32x8 ssse3/;
- specialize qw/aom_dist_wtd_sub_pixel_avg_variance16x64 ssse3/;
- specialize qw/aom_dist_wtd_sub_pixel_avg_variance64x16 ssse3/;
+ specialize qw/aom_dist_wtd_sub_pixel_avg_variance4x16 neon ssse3/;
+ specialize qw/aom_dist_wtd_sub_pixel_avg_variance16x4 neon ssse3/;
+ specialize qw/aom_dist_wtd_sub_pixel_avg_variance8x32 neon ssse3/;
+ specialize qw/aom_dist_wtd_sub_pixel_avg_variance32x8 neon ssse3/;
+ specialize qw/aom_dist_wtd_sub_pixel_avg_variance16x64 neon ssse3/;
+ specialize qw/aom_dist_wtd_sub_pixel_avg_variance64x16 neon ssse3/;
}
- specialize qw/aom_dist_wtd_sub_pixel_avg_variance64x64 ssse3/;
- specialize qw/aom_dist_wtd_sub_pixel_avg_variance64x32 ssse3/;
- specialize qw/aom_dist_wtd_sub_pixel_avg_variance32x64 ssse3/;
- specialize qw/aom_dist_wtd_sub_pixel_avg_variance32x32 ssse3/;
- specialize qw/aom_dist_wtd_sub_pixel_avg_variance32x16 ssse3/;
- specialize qw/aom_dist_wtd_sub_pixel_avg_variance16x32 ssse3/;
- specialize qw/aom_dist_wtd_sub_pixel_avg_variance16x16 ssse3/;
- specialize qw/aom_dist_wtd_sub_pixel_avg_variance16x8 ssse3/;
- specialize qw/aom_dist_wtd_sub_pixel_avg_variance8x16 ssse3/;
- specialize qw/aom_dist_wtd_sub_pixel_avg_variance8x8 ssse3/;
- specialize qw/aom_dist_wtd_sub_pixel_avg_variance8x4 ssse3/;
- specialize qw/aom_dist_wtd_sub_pixel_avg_variance4x8 ssse3/;
- specialize qw/aom_dist_wtd_sub_pixel_avg_variance4x4 ssse3/;
+ specialize qw/aom_dist_wtd_sub_pixel_avg_variance64x64 neon ssse3/;
+ specialize qw/aom_dist_wtd_sub_pixel_avg_variance64x32 neon ssse3/;
+ specialize qw/aom_dist_wtd_sub_pixel_avg_variance32x64 neon ssse3/;
+ specialize qw/aom_dist_wtd_sub_pixel_avg_variance32x32 neon ssse3/;
+ specialize qw/aom_dist_wtd_sub_pixel_avg_variance32x16 neon ssse3/;
+ specialize qw/aom_dist_wtd_sub_pixel_avg_variance16x32 neon ssse3/;
+ specialize qw/aom_dist_wtd_sub_pixel_avg_variance16x16 neon ssse3/;
+ specialize qw/aom_dist_wtd_sub_pixel_avg_variance16x8 neon ssse3/;
+ specialize qw/aom_dist_wtd_sub_pixel_avg_variance8x16 neon ssse3/;
+ specialize qw/aom_dist_wtd_sub_pixel_avg_variance8x8 neon ssse3/;
+ specialize qw/aom_dist_wtd_sub_pixel_avg_variance8x4 neon ssse3/;
+ specialize qw/aom_dist_wtd_sub_pixel_avg_variance4x8 neon ssse3/;
+ specialize qw/aom_dist_wtd_sub_pixel_avg_variance4x4 neon ssse3/;
- specialize qw/aom_dist_wtd_sub_pixel_avg_variance128x128 ssse3/;
- specialize qw/aom_dist_wtd_sub_pixel_avg_variance128x64 ssse3/;
- specialize qw/aom_dist_wtd_sub_pixel_avg_variance64x128 ssse3/;
+ specialize qw/aom_dist_wtd_sub_pixel_avg_variance128x128 neon ssse3/;
+ specialize qw/aom_dist_wtd_sub_pixel_avg_variance128x64 neon ssse3/;
+ specialize qw/aom_dist_wtd_sub_pixel_avg_variance64x128 neon ssse3/;
if (aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") {
foreach $bd (8, 10, 12) {
- add_proto qw/unsigned int/, "aom_highbd_${bd}_variance2x2", "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
-
- add_proto qw/unsigned int/, "aom_highbd_${bd}_variance2x4", "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
-
- add_proto qw/unsigned int/, "aom_highbd_${bd}_variance4x2", "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
-
foreach (@encoder_block_sizes) {
($w, $h) = @$_;
add_proto qw/unsigned int/, "aom_highbd_${bd}_variance${w}x${h}", "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
add_proto qw/uint32_t/, "aom_highbd_${bd}_sub_pixel_variance${w}x${h}", "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
add_proto qw/uint32_t/, "aom_highbd_${bd}_sub_pixel_avg_variance${w}x${h}", "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- if ($w != 128 && $h != 128 && $w != 4 && $h != 4) {
- if ($bd == 10) {
- specialize "aom_highbd_${bd}_variance${w}x${h}", qw/sse2 neon/;
- } else {
- specialize "aom_highbd_${bd}_variance${w}x${h}", "sse2";
- }
- }
-
- if ($w == 4 || $h == 4) {
- # TODO(rachelbarker): When ext-partition-types is enabled, we currently
- # don't have vectorized 4x16 highbd variance functions
- if ($w == 4 && $h == 4) {
- if ($bd == 10) {
- specialize "aom_highbd_${bd}_variance${w}x${h}", qw/sse4_1 neon/;
- } else {
- specialize "aom_highbd_${bd}_variance${w}x${h}", "sse4_1";
- }
- } else {
- if ($bd == 10) {
- specialize "aom_highbd_${bd}_variance${w}x${h}", qw/neon/;
- }
- }
- }
-
-
- if ($w != 128 && $h != 128 && $w != 4) {
- specialize "aom_highbd_${bd}_sub_pixel_variance${w}x${h}", qw/sse2/;
- specialize "aom_highbd_${bd}_sub_pixel_avg_variance${w}x${h}", qw/sse2/;
- }
- if ($w == 4 && $h == 4) {
- specialize "aom_highbd_${bd}_sub_pixel_variance${w}x${h}", "sse4_1";
- specialize "aom_highbd_${bd}_sub_pixel_avg_variance${w}x${h}", "sse4_1";
- }
-
add_proto qw/uint32_t/, "aom_highbd_${bd}_dist_wtd_sub_pixel_avg_variance${w}x${h}", "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS* jcp_param";
}
}
+
+ specialize qw/aom_highbd_12_variance128x128 sse2 neon/;
+ specialize qw/aom_highbd_12_variance128x64 sse2 neon/;
+ specialize qw/aom_highbd_12_variance64x128 sse2 neon/;
+ specialize qw/aom_highbd_12_variance64x64 sse2 neon/;
+ specialize qw/aom_highbd_12_variance64x32 sse2 neon/;
+ specialize qw/aom_highbd_12_variance32x64 sse2 neon/;
+ specialize qw/aom_highbd_12_variance32x32 sse2 neon/;
+ specialize qw/aom_highbd_12_variance32x16 sse2 neon/;
+ specialize qw/aom_highbd_12_variance16x32 sse2 neon/;
+ specialize qw/aom_highbd_12_variance16x16 sse2 neon/;
+ specialize qw/aom_highbd_12_variance16x8 sse2 neon/;
+ specialize qw/aom_highbd_12_variance8x16 sse2 neon/;
+ specialize qw/aom_highbd_12_variance8x8 sse2 neon/;
+ specialize qw/aom_highbd_12_variance8x4 neon/;
+ specialize qw/aom_highbd_12_variance4x8 neon/;
+ specialize qw/aom_highbd_12_variance4x4 sse4_1 neon/;
+
+ specialize qw/aom_highbd_10_variance128x128 sse2 avx2 neon/;
+ specialize qw/aom_highbd_10_variance128x64 sse2 avx2 neon/;
+ specialize qw/aom_highbd_10_variance64x128 sse2 avx2 neon/;
+ specialize qw/aom_highbd_10_variance64x64 sse2 avx2 neon/;
+ specialize qw/aom_highbd_10_variance64x32 sse2 avx2 neon/;
+ specialize qw/aom_highbd_10_variance32x64 sse2 avx2 neon/;
+ specialize qw/aom_highbd_10_variance32x32 sse2 avx2 neon/;
+ specialize qw/aom_highbd_10_variance32x16 sse2 avx2 neon/;
+ specialize qw/aom_highbd_10_variance16x32 sse2 avx2 neon/;
+ specialize qw/aom_highbd_10_variance16x16 sse2 avx2 neon/;
+ specialize qw/aom_highbd_10_variance16x8 sse2 avx2 neon/;
+ specialize qw/aom_highbd_10_variance8x16 sse2 avx2 neon/;
+ specialize qw/aom_highbd_10_variance8x8 sse2 avx2 neon/;
+ specialize qw/aom_highbd_10_variance8x4 neon/;
+ specialize qw/aom_highbd_10_variance4x8 neon/;
+ specialize qw/aom_highbd_10_variance4x4 sse4_1 neon/;
+
+ specialize qw/aom_highbd_8_variance128x128 sse2 neon/;
+ specialize qw/aom_highbd_8_variance128x64 sse2 neon/;
+ specialize qw/aom_highbd_8_variance64x128 sse2 neon/;
+ specialize qw/aom_highbd_8_variance64x64 sse2 neon/;
+ specialize qw/aom_highbd_8_variance64x32 sse2 neon/;
+ specialize qw/aom_highbd_8_variance32x64 sse2 neon/;
+ specialize qw/aom_highbd_8_variance32x32 sse2 neon/;
+ specialize qw/aom_highbd_8_variance32x16 sse2 neon/;
+ specialize qw/aom_highbd_8_variance16x32 sse2 neon/;
+ specialize qw/aom_highbd_8_variance16x16 sse2 neon/;
+ specialize qw/aom_highbd_8_variance16x8 sse2 neon/;
+ specialize qw/aom_highbd_8_variance8x16 sse2 neon/;
+ specialize qw/aom_highbd_8_variance8x8 sse2 neon/;
+ specialize qw/aom_highbd_8_variance8x4 neon/;
+ specialize qw/aom_highbd_8_variance4x8 neon/;
+ specialize qw/aom_highbd_8_variance4x4 sse4_1 neon/;
+
+ if (aom_config("CONFIG_REALTIME_ONLY") ne "yes") {
+ foreach $bd (8, 10, 12) {
+ my $avx2 = ($bd == 10) ? "avx2" : "";
+ specialize "aom_highbd_${bd}_variance64x16" , $avx2, qw/sse2 neon/;
+ specialize "aom_highbd_${bd}_variance32x8" , $avx2, qw/sse2 neon/;
+ specialize "aom_highbd_${bd}_variance16x64" , $avx2, qw/sse2 neon/;
+ specialize "aom_highbd_${bd}_variance16x4" , qw/neon/;
+ specialize "aom_highbd_${bd}_variance8x32" , $avx2, qw/sse2 neon/;
+ specialize "aom_highbd_${bd}_variance4x16" , qw/neon/;
+ }
+ }
+
+ specialize qw/aom_highbd_12_sub_pixel_variance128x128 sse2 neon/;
+ specialize qw/aom_highbd_12_sub_pixel_variance128x64 sse2 neon/;
+ specialize qw/aom_highbd_12_sub_pixel_variance64x128 sse2 neon/;
+ specialize qw/aom_highbd_12_sub_pixel_variance64x64 sse2 neon/;
+ specialize qw/aom_highbd_12_sub_pixel_variance64x32 sse2 neon/;
+ specialize qw/aom_highbd_12_sub_pixel_variance32x64 sse2 neon/;
+ specialize qw/aom_highbd_12_sub_pixel_variance32x32 sse2 neon/;
+ specialize qw/aom_highbd_12_sub_pixel_variance32x16 sse2 neon/;
+ specialize qw/aom_highbd_12_sub_pixel_variance16x32 sse2 neon/;
+ specialize qw/aom_highbd_12_sub_pixel_variance16x16 sse2 neon/;
+ specialize qw/aom_highbd_12_sub_pixel_variance16x8 sse2 neon/;
+ specialize qw/aom_highbd_12_sub_pixel_variance8x16 sse2 neon/;
+ specialize qw/aom_highbd_12_sub_pixel_variance8x8 sse2 neon/;
+ specialize qw/aom_highbd_12_sub_pixel_variance8x4 sse2 neon/;
+ specialize qw/aom_highbd_12_sub_pixel_variance4x8 neon/;
+ specialize qw/aom_highbd_12_sub_pixel_variance4x4 sse4_1 neon/;
+
+ specialize qw/aom_highbd_10_sub_pixel_variance128x128 sse2 avx2 neon/;
+ specialize qw/aom_highbd_10_sub_pixel_variance128x64 sse2 avx2 neon/;
+ specialize qw/aom_highbd_10_sub_pixel_variance64x128 sse2 avx2 neon/;
+ specialize qw/aom_highbd_10_sub_pixel_variance64x64 sse2 avx2 neon/;
+ specialize qw/aom_highbd_10_sub_pixel_variance64x32 sse2 avx2 neon/;
+ specialize qw/aom_highbd_10_sub_pixel_variance32x64 sse2 avx2 neon/;
+ specialize qw/aom_highbd_10_sub_pixel_variance32x32 sse2 avx2 neon/;
+ specialize qw/aom_highbd_10_sub_pixel_variance32x16 sse2 avx2 neon/;
+ specialize qw/aom_highbd_10_sub_pixel_variance16x32 sse2 avx2 neon/;
+ specialize qw/aom_highbd_10_sub_pixel_variance16x16 sse2 avx2 neon/;
+ specialize qw/aom_highbd_10_sub_pixel_variance16x8 sse2 avx2 neon/;
+ specialize qw/aom_highbd_10_sub_pixel_variance8x16 sse2 avx2 neon/;
+ specialize qw/aom_highbd_10_sub_pixel_variance8x8 sse2 avx2 neon/;
+ specialize qw/aom_highbd_10_sub_pixel_variance8x4 sse2 neon/;
+ specialize qw/aom_highbd_10_sub_pixel_variance4x8 neon/;
+ specialize qw/aom_highbd_10_sub_pixel_variance4x4 sse4_1 neon/;
+
+ specialize qw/aom_highbd_8_sub_pixel_variance128x128 sse2 neon/;
+ specialize qw/aom_highbd_8_sub_pixel_variance128x64 sse2 neon/;
+ specialize qw/aom_highbd_8_sub_pixel_variance64x128 sse2 neon/;
+ specialize qw/aom_highbd_8_sub_pixel_variance64x64 sse2 neon/;
+ specialize qw/aom_highbd_8_sub_pixel_variance64x32 sse2 neon/;
+ specialize qw/aom_highbd_8_sub_pixel_variance32x64 sse2 neon/;
+ specialize qw/aom_highbd_8_sub_pixel_variance32x32 sse2 neon/;
+ specialize qw/aom_highbd_8_sub_pixel_variance32x16 sse2 neon/;
+ specialize qw/aom_highbd_8_sub_pixel_variance16x32 sse2 neon/;
+ specialize qw/aom_highbd_8_sub_pixel_variance16x16 sse2 neon/;
+ specialize qw/aom_highbd_8_sub_pixel_variance16x8 sse2 neon/;
+ specialize qw/aom_highbd_8_sub_pixel_variance8x16 sse2 neon/;
+ specialize qw/aom_highbd_8_sub_pixel_variance8x8 sse2 neon/;
+ specialize qw/aom_highbd_8_sub_pixel_variance8x4 sse2 neon/;
+ specialize qw/aom_highbd_8_sub_pixel_variance4x8 neon/;
+ specialize qw/aom_highbd_8_sub_pixel_variance4x4 sse4_1 neon/;
+
+ if (aom_config("CONFIG_REALTIME_ONLY") ne "yes") {
+ foreach $bd (8, 10, 12) {
+ specialize "aom_highbd_${bd}_sub_pixel_variance64x16" , qw/sse2 neon/;
+ specialize "aom_highbd_${bd}_sub_pixel_variance32x8" , qw/sse2 neon/;
+ specialize "aom_highbd_${bd}_sub_pixel_variance16x64" , qw/sse2 neon/;
+ specialize "aom_highbd_${bd}_sub_pixel_variance16x4" , qw/sse2 neon/;
+ specialize "aom_highbd_${bd}_sub_pixel_variance8x32" , qw/sse2 neon/;
+ specialize "aom_highbd_${bd}_sub_pixel_variance4x16" , qw/neon/;
+ }
+ }
+
+ specialize qw/aom_highbd_12_sub_pixel_avg_variance128x128 neon/;
+ specialize qw/aom_highbd_12_sub_pixel_avg_variance128x64 neon/;
+ specialize qw/aom_highbd_12_sub_pixel_avg_variance64x128 neon/;
+ specialize qw/aom_highbd_12_sub_pixel_avg_variance64x64 sse2 neon/;
+ specialize qw/aom_highbd_12_sub_pixel_avg_variance64x32 sse2 neon/;
+ specialize qw/aom_highbd_12_sub_pixel_avg_variance32x64 sse2 neon/;
+ specialize qw/aom_highbd_12_sub_pixel_avg_variance32x32 sse2 neon/;
+ specialize qw/aom_highbd_12_sub_pixel_avg_variance32x16 sse2 neon/;
+ specialize qw/aom_highbd_12_sub_pixel_avg_variance16x32 sse2 neon/;
+ specialize qw/aom_highbd_12_sub_pixel_avg_variance16x16 sse2 neon/;
+ specialize qw/aom_highbd_12_sub_pixel_avg_variance16x8 sse2 neon/;
+ specialize qw/aom_highbd_12_sub_pixel_avg_variance8x16 sse2 neon/;
+ specialize qw/aom_highbd_12_sub_pixel_avg_variance8x8 sse2 neon/;
+ specialize qw/aom_highbd_12_sub_pixel_avg_variance8x4 sse2 neon/;
+ specialize qw/aom_highbd_12_sub_pixel_avg_variance4x8 neon/;
+ specialize qw/aom_highbd_12_sub_pixel_avg_variance4x4 sse4_1 neon/;
+
+ specialize qw/aom_highbd_10_sub_pixel_avg_variance128x128 neon/;
+ specialize qw/aom_highbd_10_sub_pixel_avg_variance128x64 neon/;
+ specialize qw/aom_highbd_10_sub_pixel_avg_variance64x128 neon/;
+ specialize qw/aom_highbd_10_sub_pixel_avg_variance64x64 sse2 neon/;
+ specialize qw/aom_highbd_10_sub_pixel_avg_variance64x32 sse2 neon/;
+ specialize qw/aom_highbd_10_sub_pixel_avg_variance32x64 sse2 neon/;
+ specialize qw/aom_highbd_10_sub_pixel_avg_variance32x32 sse2 neon/;
+ specialize qw/aom_highbd_10_sub_pixel_avg_variance32x16 sse2 neon/;
+ specialize qw/aom_highbd_10_sub_pixel_avg_variance16x32 sse2 neon/;
+ specialize qw/aom_highbd_10_sub_pixel_avg_variance16x16 sse2 neon/;
+ specialize qw/aom_highbd_10_sub_pixel_avg_variance16x8 sse2 neon/;
+ specialize qw/aom_highbd_10_sub_pixel_avg_variance8x16 sse2 neon/;
+ specialize qw/aom_highbd_10_sub_pixel_avg_variance8x8 sse2 neon/;
+ specialize qw/aom_highbd_10_sub_pixel_avg_variance8x4 sse2 neon/;
+ specialize qw/aom_highbd_10_sub_pixel_avg_variance4x8 neon/;
+ specialize qw/aom_highbd_10_sub_pixel_avg_variance4x4 sse4_1 neon/;
+
+ specialize qw/aom_highbd_8_sub_pixel_avg_variance128x128 neon/;
+ specialize qw/aom_highbd_8_sub_pixel_avg_variance128x64 neon/;
+ specialize qw/aom_highbd_8_sub_pixel_avg_variance64x128 neon/;
+ specialize qw/aom_highbd_8_sub_pixel_avg_variance64x64 sse2 neon/;
+ specialize qw/aom_highbd_8_sub_pixel_avg_variance64x32 sse2 neon/;
+ specialize qw/aom_highbd_8_sub_pixel_avg_variance32x64 sse2 neon/;
+ specialize qw/aom_highbd_8_sub_pixel_avg_variance32x32 sse2 neon/;
+ specialize qw/aom_highbd_8_sub_pixel_avg_variance32x16 sse2 neon/;
+ specialize qw/aom_highbd_8_sub_pixel_avg_variance16x32 sse2 neon/;
+ specialize qw/aom_highbd_8_sub_pixel_avg_variance16x16 sse2 neon/;
+ specialize qw/aom_highbd_8_sub_pixel_avg_variance16x8 sse2 neon/;
+ specialize qw/aom_highbd_8_sub_pixel_avg_variance8x16 sse2 neon/;
+ specialize qw/aom_highbd_8_sub_pixel_avg_variance8x8 sse2 neon/;
+ specialize qw/aom_highbd_8_sub_pixel_avg_variance8x4 sse2 neon/;
+ specialize qw/aom_highbd_8_sub_pixel_avg_variance4x8 neon/;
+ specialize qw/aom_highbd_8_sub_pixel_avg_variance4x4 sse4_1 neon/;
+
+ if (aom_config("CONFIG_REALTIME_ONLY") ne "yes") {
+ foreach $bd (8, 10, 12) {
+ specialize "aom_highbd_${bd}_sub_pixel_avg_variance64x16" , qw/sse2 neon/;
+ specialize "aom_highbd_${bd}_sub_pixel_avg_variance32x8" , qw/sse2 neon/;
+ specialize "aom_highbd_${bd}_sub_pixel_avg_variance16x64" , qw/sse2 neon/;
+ specialize "aom_highbd_${bd}_sub_pixel_avg_variance16x4" , qw/sse2 neon/;
+ specialize "aom_highbd_${bd}_sub_pixel_avg_variance8x32" , qw/sse2 neon/;
+ specialize "aom_highbd_${bd}_sub_pixel_avg_variance4x16" , qw/neon/;
+ }
+ }
+
+ foreach $bd (8, 10, 12) {
+ specialize "aom_highbd_${bd}_dist_wtd_sub_pixel_avg_variance128x128", qw/neon/;
+ specialize "aom_highbd_${bd}_dist_wtd_sub_pixel_avg_variance128x64" , qw/neon/;
+ specialize "aom_highbd_${bd}_dist_wtd_sub_pixel_avg_variance64x128" , qw/neon/;
+ specialize "aom_highbd_${bd}_dist_wtd_sub_pixel_avg_variance64x64" , qw/neon/;
+ specialize "aom_highbd_${bd}_dist_wtd_sub_pixel_avg_variance64x32" , qw/neon/;
+ specialize "aom_highbd_${bd}_dist_wtd_sub_pixel_avg_variance32x64" , qw/neon/;
+ specialize "aom_highbd_${bd}_dist_wtd_sub_pixel_avg_variance32x32" , qw/neon/;
+ specialize "aom_highbd_${bd}_dist_wtd_sub_pixel_avg_variance32x16" , qw/neon/;
+ specialize "aom_highbd_${bd}_dist_wtd_sub_pixel_avg_variance16x32" , qw/neon/;
+ specialize "aom_highbd_${bd}_dist_wtd_sub_pixel_avg_variance16x16" , qw/neon/;
+ specialize "aom_highbd_${bd}_dist_wtd_sub_pixel_avg_variance16x8" , qw/neon/;
+ specialize "aom_highbd_${bd}_dist_wtd_sub_pixel_avg_variance8x16" , qw/neon/;
+ specialize "aom_highbd_${bd}_dist_wtd_sub_pixel_avg_variance8x8" , qw/neon/;
+ specialize "aom_highbd_${bd}_dist_wtd_sub_pixel_avg_variance8x4" , qw/neon/;
+ specialize "aom_highbd_${bd}_dist_wtd_sub_pixel_avg_variance4x8" , qw/neon/;
+ specialize "aom_highbd_${bd}_dist_wtd_sub_pixel_avg_variance4x4" , qw/neon/;
+ }
+
+ if (aom_config("CONFIG_REALTIME_ONLY") ne "yes") {
+ foreach $bd (8, 10, 12) {
+ specialize "aom_highbd_${bd}_dist_wtd_sub_pixel_avg_variance64x16", qw/neon/;
+ specialize "aom_highbd_${bd}_dist_wtd_sub_pixel_avg_variance32x8" , qw/neon/;
+ specialize "aom_highbd_${bd}_dist_wtd_sub_pixel_avg_variance16x64", qw/neon/;
+ specialize "aom_highbd_${bd}_dist_wtd_sub_pixel_avg_variance16x4" , qw/neon/;
+ specialize "aom_highbd_${bd}_dist_wtd_sub_pixel_avg_variance8x32" , qw/neon/;
+ specialize "aom_highbd_${bd}_dist_wtd_sub_pixel_avg_variance4x16" , qw/neon/;
+ }
+ }
}
#
# Masked Variance / Masked Subpixel Variance
@@ -1541,7 +1726,7 @@
foreach (@encoder_block_sizes) {
($w, $h) = @$_;
add_proto qw/unsigned int/, "aom_highbd${bd}masked_sub_pixel_variance${w}x${h}", "const uint8_t *src, int src_stride, int xoffset, int yoffset, const uint8_t *ref, int ref_stride, const uint8_t *second_pred, const uint8_t *msk, int msk_stride, int invert_mask, unsigned int *sse";
- specialize "aom_highbd${bd}masked_sub_pixel_variance${w}x${h}", qw/ssse3/;
+ specialize "aom_highbd${bd}masked_sub_pixel_variance${w}x${h}", qw/ssse3 neon/;
}
}
}
@@ -1559,56 +1744,18 @@
}
if (aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") {
- foreach $bd ("_", "_10_", "_12_") {
+ foreach $bd ("_8_", "_10_", "_12_") {
foreach (@encoder_block_sizes) {
($w, $h) = @$_;
add_proto qw/unsigned int/, "aom_highbd${bd}obmc_variance${w}x${h}", "const uint8_t *pre, int pre_stride, const int32_t *wsrc, const int32_t *mask, unsigned int *sse";
add_proto qw/unsigned int/, "aom_highbd${bd}obmc_sub_pixel_variance${w}x${h}", "const uint8_t *pre, int pre_stride, int xoffset, int yoffset, const int32_t *wsrc, const int32_t *mask, unsigned int *sse";
- specialize "aom_highbd${bd}obmc_variance${w}x${h}", qw/sse4_1/;
+ specialize "aom_highbd${bd}obmc_variance${w}x${h}", qw/sse4_1 neon/;
+ specialize "aom_highbd${bd}obmc_sub_pixel_variance${w}x${h}", qw/neon/;
}
}
}
}
- add_proto qw/uint32_t aom_sub_pixel_avg_variance64x64/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_sub_pixel_avg_variance64x64 avx2 sse2 ssse3/;
-
- add_proto qw/uint32_t aom_sub_pixel_avg_variance64x32/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_sub_pixel_avg_variance64x32 sse2 ssse3/;
-
- add_proto qw/uint32_t aom_sub_pixel_avg_variance32x64/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_sub_pixel_avg_variance32x64 sse2 ssse3/;
-
- add_proto qw/uint32_t aom_sub_pixel_avg_variance32x32/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_sub_pixel_avg_variance32x32 avx2 sse2 ssse3/;
-
- add_proto qw/uint32_t aom_sub_pixel_avg_variance32x16/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_sub_pixel_avg_variance32x16 sse2 ssse3/;
-
- add_proto qw/uint32_t aom_sub_pixel_avg_variance16x32/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_sub_pixel_avg_variance16x32 sse2 ssse3/;
-
- add_proto qw/uint32_t aom_sub_pixel_avg_variance16x16/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_sub_pixel_avg_variance16x16 sse2 ssse3/;
-
- add_proto qw/uint32_t aom_sub_pixel_avg_variance16x8/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_sub_pixel_avg_variance16x8 sse2 ssse3/;
-
- add_proto qw/uint32_t aom_sub_pixel_avg_variance8x16/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_sub_pixel_avg_variance8x16 sse2 ssse3/;
-
- add_proto qw/uint32_t aom_sub_pixel_avg_variance8x8/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_sub_pixel_avg_variance8x8 sse2 ssse3/;
-
- add_proto qw/uint32_t aom_sub_pixel_avg_variance8x4/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_sub_pixel_avg_variance8x4 sse2 ssse3/;
-
- add_proto qw/uint32_t aom_sub_pixel_avg_variance4x8/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_sub_pixel_avg_variance4x8 sse2 ssse3/;
-
- add_proto qw/uint32_t aom_sub_pixel_avg_variance4x4/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_sub_pixel_avg_variance4x4 sse2 ssse3/;
-
#
# Comp Avg
#
@@ -1616,469 +1763,25 @@
specialize qw/aom_comp_avg_pred avx2 neon/;
add_proto qw/void aom_dist_wtd_comp_avg_pred/, "uint8_t *comp_pred, const uint8_t *pred, int width, int height, const uint8_t *ref, int ref_stride, const DIST_WTD_COMP_PARAMS *jcp_param";
- specialize qw/aom_dist_wtd_comp_avg_pred ssse3/;
+ specialize qw/aom_dist_wtd_comp_avg_pred ssse3 neon/;
if (aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") {
-
- add_proto qw/unsigned int aom_highbd_12_variance128x128/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_12_variance128x128 sse2 neon/;
-
- add_proto qw/unsigned int aom_highbd_12_variance128x64/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_12_variance128x64 sse2 neon/;
-
- add_proto qw/unsigned int aom_highbd_12_variance64x128/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_12_variance64x128 sse2 neon/;
-
- add_proto qw/unsigned int aom_highbd_12_variance64x64/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_12_variance64x64 sse2 neon/;
-
- add_proto qw/unsigned int aom_highbd_12_variance64x32/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_12_variance64x32 sse2 neon/;
-
- add_proto qw/unsigned int aom_highbd_12_variance32x64/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_12_variance32x64 sse2 neon/;
-
- add_proto qw/unsigned int aom_highbd_12_variance32x32/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_12_variance32x32 sse2 neon/;
-
- add_proto qw/unsigned int aom_highbd_12_variance32x16/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_12_variance32x16 sse2 neon/;
-
- add_proto qw/unsigned int aom_highbd_12_variance16x32/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_12_variance16x32 sse2 neon/;
-
- add_proto qw/unsigned int aom_highbd_12_variance16x16/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_12_variance16x16 sse2 neon/;
-
- add_proto qw/unsigned int aom_highbd_12_variance16x8/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_12_variance16x8 sse2 neon/;
-
- add_proto qw/unsigned int aom_highbd_12_variance8x16/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_12_variance8x16 sse2 neon/;
-
- add_proto qw/unsigned int aom_highbd_12_variance8x8/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_12_variance8x8 sse2 neon/;
-
- add_proto qw/unsigned int aom_highbd_12_variance8x4/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_12_variance8x4 neon/;
-
- add_proto qw/unsigned int aom_highbd_12_variance4x8/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_12_variance4x8 neon/;
-
- add_proto qw/unsigned int aom_highbd_12_variance4x4/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_12_variance4x4 neon/;
-
- add_proto qw/unsigned int aom_highbd_10_variance128x128/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_10_variance128x128 sse2 avx2 neon/;
-
- add_proto qw/unsigned int aom_highbd_10_variance128x64/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_10_variance128x64 sse2 avx2 neon/;
-
- add_proto qw/unsigned int aom_highbd_10_variance64x128/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_10_variance64x128 sse2 avx2 neon/;
-
- add_proto qw/unsigned int aom_highbd_10_variance64x64/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_10_variance64x64 sse2 avx2 neon/;
-
- add_proto qw/unsigned int aom_highbd_10_variance64x32/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_10_variance64x32 sse2 avx2 neon/;
-
- add_proto qw/unsigned int aom_highbd_10_variance32x64/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_10_variance32x64 sse2 avx2 neon/;
-
- add_proto qw/unsigned int aom_highbd_10_variance32x32/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_10_variance32x32 sse2 avx2 neon/;
-
- add_proto qw/unsigned int aom_highbd_10_variance32x16/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_10_variance32x16 sse2 avx2 neon/;
-
- add_proto qw/unsigned int aom_highbd_10_variance16x32/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_10_variance16x32 sse2 avx2 neon/;
-
- add_proto qw/unsigned int aom_highbd_10_variance16x16/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_10_variance16x16 sse2 avx2 neon/;
-
- add_proto qw/unsigned int aom_highbd_10_variance16x8/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_10_variance16x8 sse2 avx2 neon/;
-
- add_proto qw/unsigned int aom_highbd_10_variance8x16/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_10_variance8x16 sse2 avx2 neon/;
-
- add_proto qw/unsigned int aom_highbd_10_variance8x8/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_10_variance8x8 sse2 avx2 neon/;
-
- add_proto qw/unsigned int aom_highbd_10_variance8x4/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_10_variance8x4 neon/;
-
- add_proto qw/unsigned int aom_highbd_10_variance4x8/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_10_variance4x8 neon/;
-
- add_proto qw/unsigned int aom_highbd_10_variance4x4/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_10_variance4x4 neon/;
-
- add_proto qw/unsigned int aom_highbd_8_variance128x128/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_8_variance128x128 sse2 neon/;
-
- add_proto qw/unsigned int aom_highbd_8_variance128x64/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_8_variance128x64 sse2 neon/;
-
- add_proto qw/unsigned int aom_highbd_8_variance64x128/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_8_variance64x128 sse2 neon/;
-
- add_proto qw/unsigned int aom_highbd_8_variance64x64/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_8_variance64x64 sse2 neon/;
-
- add_proto qw/unsigned int aom_highbd_8_variance64x32/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_8_variance64x32 sse2 neon/;
-
- add_proto qw/unsigned int aom_highbd_8_variance32x64/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_8_variance32x64 sse2 neon/;
-
- add_proto qw/unsigned int aom_highbd_8_variance32x32/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_8_variance32x32 sse2 neon/;
-
- add_proto qw/unsigned int aom_highbd_8_variance32x16/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_8_variance32x16 sse2 neon/;
-
- add_proto qw/unsigned int aom_highbd_8_variance16x32/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_8_variance16x32 sse2 neon/;
-
- add_proto qw/unsigned int aom_highbd_8_variance16x16/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_8_variance16x16 sse2 neon/;
-
- add_proto qw/unsigned int aom_highbd_8_variance16x8/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_8_variance16x8 sse2 neon/;
-
- add_proto qw/unsigned int aom_highbd_8_variance8x16/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_8_variance8x16 sse2 neon/;
-
- add_proto qw/unsigned int aom_highbd_8_variance8x8/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_8_variance8x8 sse2 neon/;
-
- add_proto qw/unsigned int aom_highbd_8_variance8x4/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_8_variance8x4 neon/;
-
- add_proto qw/unsigned int aom_highbd_8_variance4x8/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_8_variance4x8 neon/;
-
- add_proto qw/unsigned int aom_highbd_8_variance4x4/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize qw/aom_highbd_8_variance4x4 neon/;
-
- if (aom_config("CONFIG_REALTIME_ONLY") ne "yes") {
- foreach $bd (8, 10, 12) {
- add_proto qw/unsigned int/, "aom_highbd_${bd}_variance64x16", "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize "aom_highbd_${bd}_variance64x16" , qw/neon/;
-
- add_proto qw/unsigned int/, "aom_highbd_${bd}_variance32x8", "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize "aom_highbd_${bd}_variance32x8" , qw/neon/;
-
- add_proto qw/unsigned int/, "aom_highbd_${bd}_variance16x64", "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize "aom_highbd_${bd}_variance16x64" , qw/neon/;
-
- add_proto qw/unsigned int/, "aom_highbd_${bd}_variance16x4", "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize "aom_highbd_${bd}_variance16x4" , qw/neon/;
-
- add_proto qw/unsigned int/, "aom_highbd_${bd}_variance8x32", "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize "aom_highbd_${bd}_variance8x32" , qw/neon/;
-
- add_proto qw/unsigned int/, "aom_highbd_${bd}_variance4x16", "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
- specialize "aom_highbd_${bd}_variance4x16" , qw/neon/;
- }
- }
-
- add_proto qw/unsigned int aom_highbd_8_mse16x16/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int recon_stride, unsigned int *sse";
- specialize qw/aom_highbd_8_mse16x16 sse2 neon/;
-
- add_proto qw/unsigned int aom_highbd_8_mse16x8/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int recon_stride, unsigned int *sse";
- specialize qw/aom_highbd_8_mse16x8 neon/;
- add_proto qw/unsigned int aom_highbd_8_mse8x16/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int recon_stride, unsigned int *sse";
- specialize qw/aom_highbd_8_mse8x16 neon/;
- add_proto qw/unsigned int aom_highbd_8_mse8x8/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int recon_stride, unsigned int *sse";
- specialize qw/aom_highbd_8_mse8x8 sse2 neon/;
-
- add_proto qw/unsigned int aom_highbd_10_mse16x16/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int recon_stride, unsigned int *sse";
- specialize qw/aom_highbd_10_mse16x16 sse2 neon/;
-
- add_proto qw/unsigned int aom_highbd_10_mse16x8/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int recon_stride, unsigned int *sse";
- specialize qw/aom_highbd_10_mse16x8 neon/;
- add_proto qw/unsigned int aom_highbd_10_mse8x16/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int recon_stride, unsigned int *sse";
- specialize qw/aom_highbd_10_mse8x16 neon/;
- add_proto qw/unsigned int aom_highbd_10_mse8x8/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int recon_stride, unsigned int *sse";
- specialize qw/aom_highbd_10_mse8x8 sse2 neon/;
-
- add_proto qw/unsigned int aom_highbd_12_mse16x16/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int recon_stride, unsigned int *sse";
- specialize qw/aom_highbd_12_mse16x16 sse2 neon/;
-
- add_proto qw/unsigned int aom_highbd_12_mse16x8/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int recon_stride, unsigned int *sse";
- specialize qw/aom_highbd_12_mse16x8 neon/;
- add_proto qw/unsigned int aom_highbd_12_mse8x16/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int recon_stride, unsigned int *sse";
- specialize qw/aom_highbd_12_mse8x16 neon/;
- add_proto qw/unsigned int aom_highbd_12_mse8x8/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int recon_stride, unsigned int *sse";
- specialize qw/aom_highbd_12_mse8x8 sse2 neon/;
-
add_proto qw/void aom_highbd_comp_avg_pred/, "uint8_t *comp_pred8, const uint8_t *pred8, int width, int height, const uint8_t *ref8, int ref_stride";
+ specialize qw/aom_highbd_comp_avg_pred neon/;
add_proto qw/void aom_highbd_dist_wtd_comp_avg_pred/, "uint8_t *comp_pred8, const uint8_t *pred8, int width, int height, const uint8_t *ref8, int ref_stride, const DIST_WTD_COMP_PARAMS *jcp_param";
- specialize qw/aom_highbd_dist_wtd_comp_avg_pred sse2/;
+ specialize qw/aom_highbd_dist_wtd_comp_avg_pred sse2 neon/;
add_proto qw/uint64_t/, "aom_mse_wxh_16bit_highbd", "uint16_t *dst, int dstride,uint16_t *src, int sstride, int w, int h";
- specialize qw/aom_mse_wxh_16bit_highbd sse2 avx2/;
+ specialize qw/aom_mse_wxh_16bit_highbd sse2 avx2 neon/;
}
- #
- # Subpixel Variance
- #
- if (aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") {
- add_proto qw/uint32_t aom_highbd_12_sub_pixel_variance128x128/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_12_sub_pixel_variance128x128 sse2/;
-
- add_proto qw/uint32_t aom_highbd_12_sub_pixel_variance128x64/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_12_sub_pixel_variance128x64 sse2/;
-
- add_proto qw/uint32_t aom_highbd_12_sub_pixel_variance64x128/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_12_sub_pixel_variance64x128 sse2/;
-
- add_proto qw/uint32_t aom_highbd_12_sub_pixel_variance64x64/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_12_sub_pixel_variance64x64 sse2/;
-
- add_proto qw/uint32_t aom_highbd_12_sub_pixel_variance64x32/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_12_sub_pixel_variance64x32 sse2/;
-
- add_proto qw/uint32_t aom_highbd_12_sub_pixel_variance32x64/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_12_sub_pixel_variance32x64 sse2/;
-
- add_proto qw/uint32_t aom_highbd_12_sub_pixel_variance32x32/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_12_sub_pixel_variance32x32 sse2/;
-
- add_proto qw/uint32_t aom_highbd_12_sub_pixel_variance32x16/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_12_sub_pixel_variance32x16 sse2/;
-
- add_proto qw/uint32_t aom_highbd_12_sub_pixel_variance16x32/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_12_sub_pixel_variance16x32 sse2/;
-
- add_proto qw/uint32_t aom_highbd_12_sub_pixel_variance16x16/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_12_sub_pixel_variance16x16 sse2/;
-
- add_proto qw/uint32_t aom_highbd_12_sub_pixel_variance16x8/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_12_sub_pixel_variance16x8 sse2/;
-
- add_proto qw/uint32_t aom_highbd_12_sub_pixel_variance8x16/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_12_sub_pixel_variance8x16 sse2/;
-
- add_proto qw/uint32_t aom_highbd_12_sub_pixel_variance8x8/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_12_sub_pixel_variance8x8 sse2/;
-
- add_proto qw/uint32_t aom_highbd_12_sub_pixel_variance8x4/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_12_sub_pixel_variance8x4 sse2/;
-
- add_proto qw/uint32_t aom_highbd_12_sub_pixel_variance4x8/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- add_proto qw/uint32_t aom_highbd_12_sub_pixel_variance4x4/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
-
- add_proto qw/uint32_t aom_highbd_10_sub_pixel_variance128x128/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_10_sub_pixel_variance128x128 sse2 avx2/;
-
- add_proto qw/uint32_t aom_highbd_10_sub_pixel_variance128x64/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_10_sub_pixel_variance128x64 sse2 avx2/;
-
- add_proto qw/uint32_t aom_highbd_10_sub_pixel_variance64x128/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_10_sub_pixel_variance64x128 sse2 avx2/;
-
- add_proto qw/uint32_t aom_highbd_10_sub_pixel_variance64x64/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_10_sub_pixel_variance64x64 sse2 avx2/;
-
- add_proto qw/uint32_t aom_highbd_10_sub_pixel_variance64x32/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_10_sub_pixel_variance64x32 sse2 avx2/;
-
- add_proto qw/uint32_t aom_highbd_10_sub_pixel_variance32x64/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_10_sub_pixel_variance32x64 sse2 avx2/;
-
- add_proto qw/uint32_t aom_highbd_10_sub_pixel_variance32x32/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_10_sub_pixel_variance32x32 sse2 avx2/;
-
- add_proto qw/uint32_t aom_highbd_10_sub_pixel_variance32x16/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_10_sub_pixel_variance32x16 sse2 avx2/;
-
- add_proto qw/uint32_t aom_highbd_10_sub_pixel_variance16x32/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_10_sub_pixel_variance16x32 sse2 avx2/;
-
- add_proto qw/uint32_t aom_highbd_10_sub_pixel_variance16x16/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_10_sub_pixel_variance16x16 sse2 avx2/;
-
- add_proto qw/uint32_t aom_highbd_10_sub_pixel_variance16x8/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_10_sub_pixel_variance16x8 sse2 avx2/;
-
- add_proto qw/uint32_t aom_highbd_10_sub_pixel_variance8x16/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_10_sub_pixel_variance8x16 sse2 avx2/;
-
- add_proto qw/uint32_t aom_highbd_10_sub_pixel_variance8x8/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_10_sub_pixel_variance8x8 sse2 avx2/;
-
- add_proto qw/uint32_t aom_highbd_10_sub_pixel_variance8x4/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_10_sub_pixel_variance8x4 sse2/;
-
- add_proto qw/uint32_t aom_highbd_10_sub_pixel_variance4x8/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- add_proto qw/uint32_t aom_highbd_10_sub_pixel_variance4x4/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
-
- add_proto qw/uint32_t aom_highbd_8_sub_pixel_variance128x128/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_8_sub_pixel_variance128x128 sse2/;
-
- add_proto qw/uint32_t aom_highbd_8_sub_pixel_variance128x64/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_8_sub_pixel_variance128x64 sse2/;
-
- add_proto qw/uint32_t aom_highbd_8_sub_pixel_variance64x128/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_8_sub_pixel_variance64x128 sse2/;
-
- add_proto qw/uint32_t aom_highbd_8_sub_pixel_variance64x64/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_8_sub_pixel_variance64x64 sse2/;
-
- add_proto qw/uint32_t aom_highbd_8_sub_pixel_variance64x32/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_8_sub_pixel_variance64x32 sse2/;
-
- add_proto qw/uint32_t aom_highbd_8_sub_pixel_variance32x64/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_8_sub_pixel_variance32x64 sse2/;
-
- add_proto qw/uint32_t aom_highbd_8_sub_pixel_variance32x32/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_8_sub_pixel_variance32x32 sse2/;
-
- add_proto qw/uint32_t aom_highbd_8_sub_pixel_variance32x16/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_8_sub_pixel_variance32x16 sse2/;
-
- add_proto qw/uint32_t aom_highbd_8_sub_pixel_variance16x32/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_8_sub_pixel_variance16x32 sse2/;
-
- add_proto qw/uint32_t aom_highbd_8_sub_pixel_variance16x16/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_8_sub_pixel_variance16x16 sse2/;
-
- add_proto qw/uint32_t aom_highbd_8_sub_pixel_variance16x8/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_8_sub_pixel_variance16x8 sse2/;
-
- add_proto qw/uint32_t aom_highbd_8_sub_pixel_variance8x16/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_8_sub_pixel_variance8x16 sse2/;
-
- add_proto qw/uint32_t aom_highbd_8_sub_pixel_variance8x8/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_8_sub_pixel_variance8x8 sse2/;
-
- add_proto qw/uint32_t aom_highbd_8_sub_pixel_variance8x4/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- specialize qw/aom_highbd_8_sub_pixel_variance8x4 sse2/;
-
- add_proto qw/uint32_t aom_highbd_8_sub_pixel_variance4x8/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
- add_proto qw/uint32_t aom_highbd_8_sub_pixel_variance4x4/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
-
- add_proto qw/uint32_t aom_highbd_12_sub_pixel_avg_variance64x64/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_highbd_12_sub_pixel_avg_variance64x64 sse2/;
-
- add_proto qw/uint32_t aom_highbd_12_sub_pixel_avg_variance64x32/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_highbd_12_sub_pixel_avg_variance64x32 sse2/;
-
- add_proto qw/uint32_t aom_highbd_12_sub_pixel_avg_variance32x64/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_highbd_12_sub_pixel_avg_variance32x64 sse2/;
-
- add_proto qw/uint32_t aom_highbd_12_sub_pixel_avg_variance32x32/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_highbd_12_sub_pixel_avg_variance32x32 sse2/;
-
- add_proto qw/uint32_t aom_highbd_12_sub_pixel_avg_variance32x16/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_highbd_12_sub_pixel_avg_variance32x16 sse2/;
-
- add_proto qw/uint32_t aom_highbd_12_sub_pixel_avg_variance16x32/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_highbd_12_sub_pixel_avg_variance16x32 sse2/;
-
- add_proto qw/uint32_t aom_highbd_12_sub_pixel_avg_variance16x16/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_highbd_12_sub_pixel_avg_variance16x16 sse2/;
-
- add_proto qw/uint32_t aom_highbd_12_sub_pixel_avg_variance16x8/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_highbd_12_sub_pixel_avg_variance16x8 sse2/;
-
- add_proto qw/uint32_t aom_highbd_12_sub_pixel_avg_variance8x16/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_highbd_12_sub_pixel_avg_variance8x16 sse2/;
-
- add_proto qw/uint32_t aom_highbd_12_sub_pixel_avg_variance8x8/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_highbd_12_sub_pixel_avg_variance8x8 sse2/;
-
- add_proto qw/uint32_t aom_highbd_12_sub_pixel_avg_variance8x4/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_highbd_12_sub_pixel_avg_variance8x4 sse2/;
-
- add_proto qw/uint32_t aom_highbd_12_sub_pixel_avg_variance4x8/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- add_proto qw/uint32_t aom_highbd_12_sub_pixel_avg_variance4x4/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
-
- add_proto qw/uint32_t aom_highbd_10_sub_pixel_avg_variance64x64/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_highbd_10_sub_pixel_avg_variance64x64 sse2/;
-
- add_proto qw/uint32_t aom_highbd_10_sub_pixel_avg_variance64x32/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_highbd_10_sub_pixel_avg_variance64x32 sse2/;
-
- add_proto qw/uint32_t aom_highbd_10_sub_pixel_avg_variance32x64/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_highbd_10_sub_pixel_avg_variance32x64 sse2/;
-
- add_proto qw/uint32_t aom_highbd_10_sub_pixel_avg_variance32x32/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_highbd_10_sub_pixel_avg_variance32x32 sse2/;
-
- add_proto qw/uint32_t aom_highbd_10_sub_pixel_avg_variance32x16/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_highbd_10_sub_pixel_avg_variance32x16 sse2/;
-
- add_proto qw/uint32_t aom_highbd_10_sub_pixel_avg_variance16x32/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_highbd_10_sub_pixel_avg_variance16x32 sse2/;
-
- add_proto qw/uint32_t aom_highbd_10_sub_pixel_avg_variance16x16/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_highbd_10_sub_pixel_avg_variance16x16 sse2/;
-
- add_proto qw/uint32_t aom_highbd_10_sub_pixel_avg_variance16x8/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_highbd_10_sub_pixel_avg_variance16x8 sse2/;
-
- add_proto qw/uint32_t aom_highbd_10_sub_pixel_avg_variance8x16/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_highbd_10_sub_pixel_avg_variance8x16 sse2/;
-
- add_proto qw/uint32_t aom_highbd_10_sub_pixel_avg_variance8x8/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_highbd_10_sub_pixel_avg_variance8x8 sse2/;
-
- add_proto qw/uint32_t aom_highbd_10_sub_pixel_avg_variance8x4/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_highbd_10_sub_pixel_avg_variance8x4 sse2/;
-
- add_proto qw/uint32_t aom_highbd_10_sub_pixel_avg_variance4x8/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- add_proto qw/uint32_t aom_highbd_10_sub_pixel_avg_variance4x4/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
-
- add_proto qw/uint32_t aom_highbd_8_sub_pixel_avg_variance64x64/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_highbd_8_sub_pixel_avg_variance64x64 sse2/;
-
- add_proto qw/uint32_t aom_highbd_8_sub_pixel_avg_variance64x32/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_highbd_8_sub_pixel_avg_variance64x32 sse2/;
-
- add_proto qw/uint32_t aom_highbd_8_sub_pixel_avg_variance32x64/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_highbd_8_sub_pixel_avg_variance32x64 sse2/;
-
- add_proto qw/uint32_t aom_highbd_8_sub_pixel_avg_variance32x32/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_highbd_8_sub_pixel_avg_variance32x32 sse2/;
-
- add_proto qw/uint32_t aom_highbd_8_sub_pixel_avg_variance32x16/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_highbd_8_sub_pixel_avg_variance32x16 sse2/;
-
- add_proto qw/uint32_t aom_highbd_8_sub_pixel_avg_variance16x32/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_highbd_8_sub_pixel_avg_variance16x32 sse2/;
-
- add_proto qw/uint32_t aom_highbd_8_sub_pixel_avg_variance16x16/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_highbd_8_sub_pixel_avg_variance16x16 sse2/;
-
- add_proto qw/uint32_t aom_highbd_8_sub_pixel_avg_variance16x8/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_highbd_8_sub_pixel_avg_variance16x8 sse2/;
-
- add_proto qw/uint32_t aom_highbd_8_sub_pixel_avg_variance8x16/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_highbd_8_sub_pixel_avg_variance8x16 sse2/;
-
- add_proto qw/uint32_t aom_highbd_8_sub_pixel_avg_variance8x8/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_highbd_8_sub_pixel_avg_variance8x8 sse2/;
-
- add_proto qw/uint32_t aom_highbd_8_sub_pixel_avg_variance8x4/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- specialize qw/aom_highbd_8_sub_pixel_avg_variance8x4 sse2/;
-
- add_proto qw/uint32_t aom_highbd_8_sub_pixel_avg_variance4x8/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- add_proto qw/uint32_t aom_highbd_8_sub_pixel_avg_variance4x4/, "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
- }
-
add_proto qw/void aom_comp_mask_pred/, "uint8_t *comp_pred, const uint8_t *pred, int width, int height, const uint8_t *ref, int ref_stride, const uint8_t *mask, int mask_stride, int invert_mask";
specialize qw/aom_comp_mask_pred ssse3 avx2 neon/;
if (aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") {
add_proto qw/void aom_highbd_comp_mask_pred/, "uint8_t *comp_pred, const uint8_t *pred8, int width, int height, const uint8_t *ref8, int ref_stride, const uint8_t *mask, int mask_stride, int invert_mask";
- specialize qw/aom_highbd_comp_mask_pred sse2 avx2/;
+ specialize qw/aom_highbd_comp_mask_pred sse2 avx2 neon/;
}
# Flow estimation library
@@ -2087,7 +1790,7 @@
specialize qw/av1_compute_cross_correlation sse4_1 avx2/;
add_proto qw/void aom_compute_flow_at_point/, "const uint8_t *src, const uint8_t *ref, int x, int y, int width, int height, int stride, double *u, double *v";
- specialize qw/aom_compute_flow_at_point sse4_1/;
+ specialize qw/aom_compute_flow_at_point sse4_1 neon/;
}
} # CONFIG_AV1_ENCODER
diff --git a/aom_dsp/aom_simd.h b/aom_dsp/aom_simd.h
index ab950ca..69da8f2 100644
--- a/aom_dsp/aom_simd.h
+++ b/aom_dsp/aom_simd.h
@@ -24,12 +24,10 @@
#define SIMD_CHECK 1 // Sanity checks in C equivalents
-#if HAVE_NEON
-#include "simd/v256_intrinsics_arm.h"
// VS compiling for 32 bit targets does not support vector types in
// structs as arguments, which makes the v256 type of the intrinsics
// hard to support, so optimizations for this target are disabled.
-#elif HAVE_SSE2 && (defined(_WIN64) || !defined(_MSC_VER) || defined(__clang__))
+#if HAVE_SSE2 && (defined(_WIN64) || !defined(_MSC_VER) || defined(__clang__))
#include "simd/v256_intrinsics_x86.h"
#else
#include "simd/v256_intrinsics.h"
diff --git a/aom_dsp/arm/aom_convolve8_neon.c b/aom_dsp/arm/aom_convolve8_neon.c
index 3d07a0f..c8ee780 100644
--- a/aom_dsp/arm/aom_convolve8_neon.c
+++ b/aom_dsp/arm/aom_convolve8_neon.c
@@ -24,826 +24,6 @@
#include "aom_dsp/arm/transpose_neon.h"
#include "aom_ports/mem.h"
-#if AOM_ARCH_AARCH64 && \
- (defined(__ARM_FEATURE_DOTPROD) || defined(__ARM_FEATURE_MATMUL_INT8))
-
-DECLARE_ALIGNED(16, static const uint8_t, dot_prod_permute_tbl[48]) = {
- 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6,
- 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10,
- 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
-};
-
-DECLARE_ALIGNED(16, static const uint8_t, dot_prod_tran_concat_tbl[32]) = {
- 0, 8, 16, 24, 1, 9, 17, 25, 2, 10, 18, 26, 3, 11, 19, 27,
- 4, 12, 20, 28, 5, 13, 21, 29, 6, 14, 22, 30, 7, 15, 23, 31
-};
-
-DECLARE_ALIGNED(16, static const uint8_t, dot_prod_merge_block_tbl[48]) = {
- /* Shift left and insert new last column in transposed 4x4 block. */
- 1, 2, 3, 16, 5, 6, 7, 20, 9, 10, 11, 24, 13, 14, 15, 28,
- /* Shift left and insert two new columns in transposed 4x4 block. */
- 2, 3, 16, 17, 6, 7, 20, 21, 10, 11, 24, 25, 14, 15, 28, 29,
- /* Shift left and insert three new columns in transposed 4x4 block. */
- 3, 16, 17, 18, 7, 20, 21, 22, 11, 24, 25, 26, 15, 28, 29, 30
-};
-
-#if defined(__ARM_FEATURE_MATMUL_INT8)
-
-static INLINE int16x4_t convolve8_4_usdot(uint8x16_t samples,
- const int8x8_t filter,
- const uint8x16x2_t permute_tbl) {
- uint8x16_t permuted_samples[2];
- int32x4_t sum;
-
- /* Permute samples ready for dot product. */
- /* { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } */
- permuted_samples[0] = vqtbl1q_u8(samples, permute_tbl.val[0]);
- /* { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } */
- permuted_samples[1] = vqtbl1q_u8(samples, permute_tbl.val[1]);
-
- /* Accumulate dot product into 'correction' to account for range clamp. */
- sum = vusdotq_lane_s32(vdupq_n_s32(0), permuted_samples[0], filter, 0);
- sum = vusdotq_lane_s32(sum, permuted_samples[1], filter, 1);
-
- /* Further narrowing and packing is performed by the caller. */
- return vqmovn_s32(sum);
-}
-
-static INLINE uint8x8_t convolve8_8_usdot(uint8x16_t samples,
- const int8x8_t filter,
- const uint8x16x3_t permute_tbl) {
- uint8x16_t permuted_samples[3];
- int32x4_t sum0, sum1;
- int16x8_t sum;
-
- /* Permute samples ready for dot product. */
- /* { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } */
- permuted_samples[0] = vqtbl1q_u8(samples, permute_tbl.val[0]);
- /* { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } */
- permuted_samples[1] = vqtbl1q_u8(samples, permute_tbl.val[1]);
- /* { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 } */
- permuted_samples[2] = vqtbl1q_u8(samples, permute_tbl.val[2]);
-
- /* First 4 output values. */
- sum0 = vusdotq_lane_s32(vdupq_n_s32(0), permuted_samples[0], filter, 0);
- sum0 = vusdotq_lane_s32(sum0, permuted_samples[1], filter, 1);
- /* Second 4 output values. */
- sum1 = vusdotq_lane_s32(vdupq_n_s32(0), permuted_samples[1], filter, 0);
- sum1 = vusdotq_lane_s32(sum1, permuted_samples[2], filter, 1);
-
- /* Narrow and re-pack. */
- sum = vcombine_s16(vqmovn_s32(sum0), vqmovn_s32(sum1));
- return vqrshrun_n_s16(sum, FILTER_BITS);
-}
-
-void aom_convolve8_horiz_neon(const uint8_t *src, ptrdiff_t src_stride,
- uint8_t *dst, ptrdiff_t dst_stride,
- const int16_t *filter_x, int x_step_q4,
- const int16_t *filter_y, int y_step_q4, int w,
- int h) {
- const int8x8_t filter = vmovn_s16(vld1q_s16(filter_x));
- uint8x16_t s0, s1, s2, s3;
-
- assert((intptr_t)dst % 4 == 0);
- assert(dst_stride % 4 == 0);
-
- (void)x_step_q4;
- (void)filter_y;
- (void)y_step_q4;
-
- src -= ((SUBPEL_TAPS / 2) - 1);
-
- if (w == 4) {
- const uint8x16x2_t perm_tbl = vld1q_u8_x2(dot_prod_permute_tbl);
- do {
- int16x4_t t0, t1, t2, t3;
- uint8x8_t d01, d23;
-
- load_u8_16x4(src, src_stride, &s0, &s1, &s2, &s3);
-
- t0 = convolve8_4_usdot(s0, filter, perm_tbl);
- t1 = convolve8_4_usdot(s1, filter, perm_tbl);
- t2 = convolve8_4_usdot(s2, filter, perm_tbl);
- t3 = convolve8_4_usdot(s3, filter, perm_tbl);
- d01 = vqrshrun_n_s16(vcombine_s16(t0, t1), FILTER_BITS);
- d23 = vqrshrun_n_s16(vcombine_s16(t2, t3), FILTER_BITS);
-
- store_u8_4x1(dst + 0 * dst_stride, d01, 0);
- store_u8_4x1(dst + 1 * dst_stride, d01, 1);
- store_u8_4x1(dst + 2 * dst_stride, d23, 0);
- store_u8_4x1(dst + 3 * dst_stride, d23, 1);
-
- src += 4 * src_stride;
- dst += 4 * dst_stride;
- h -= 4;
- } while (h > 0);
- } else {
- const uint8x16x3_t perm_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
- const uint8_t *s;
- uint8_t *d;
- int width;
- uint8x8_t d0, d1, d2, d3;
-
- do {
- width = w;
- s = src;
- d = dst;
- do {
- load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3);
-
- d0 = convolve8_8_usdot(s0, filter, perm_tbl);
- d1 = convolve8_8_usdot(s1, filter, perm_tbl);
- d2 = convolve8_8_usdot(s2, filter, perm_tbl);
- d3 = convolve8_8_usdot(s3, filter, perm_tbl);
-
- store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
-
- s += 8;
- d += 8;
- width -= 8;
- } while (width != 0);
- src += 4 * src_stride;
- dst += 4 * dst_stride;
- h -= 4;
- } while (h > 0);
- }
-}
-
-static INLINE void transpose_concat_4x4(uint8x8_t a0, uint8x8_t a1,
- uint8x8_t a2, uint8x8_t a3,
- uint8x16_t *b,
- const uint8x16_t permute_tbl) {
- /* Transpose 8-bit elements and concatenate result rows as follows:
- * a0: 00, 01, 02, 03, XX, XX, XX, XX
- * a1: 10, 11, 12, 13, XX, XX, XX, XX
- * a2: 20, 21, 22, 23, XX, XX, XX, XX
- * a3: 30, 31, 32, 33, XX, XX, XX, XX
- *
- * b: 00, 10, 20, 30, 01, 11, 21, 31, 02, 12, 22, 32, 03, 13, 23, 33
- *
- * The 'permute_tbl' is always 'dot_prod_tran_concat_tbl' above. Passing it
- * as an argument is preferable to loading it directly from memory as this
- * inline helper is called many times from the same parent function.
- */
-
- uint8x16x2_t samples = { { vcombine_u8(a0, a1), vcombine_u8(a2, a3) } };
- *b = vqtbl2q_u8(samples, permute_tbl);
-}
-
-static INLINE void transpose_concat_8x4(uint8x8_t a0, uint8x8_t a1,
- uint8x8_t a2, uint8x8_t a3,
- uint8x16_t *b0, uint8x16_t *b1,
- const uint8x16x2_t permute_tbl) {
- /* Transpose 8-bit elements and concatenate result rows as follows:
- * a0: 00, 01, 02, 03, 04, 05, 06, 07
- * a1: 10, 11, 12, 13, 14, 15, 16, 17
- * a2: 20, 21, 22, 23, 24, 25, 26, 27
- * a3: 30, 31, 32, 33, 34, 35, 36, 37
- *
- * b0: 00, 10, 20, 30, 01, 11, 21, 31, 02, 12, 22, 32, 03, 13, 23, 33
- * b1: 04, 14, 24, 34, 05, 15, 25, 35, 06, 16, 26, 36, 07, 17, 27, 37
- *
- * The 'permute_tbl' is always 'dot_prod_tran_concat_tbl' above. Passing it
- * as an argument is preferable to loading it directly from memory as this
- * inline helper is called many times from the same parent function.
- */
-
- uint8x16x2_t samples = { { vcombine_u8(a0, a1), vcombine_u8(a2, a3) } };
- *b0 = vqtbl2q_u8(samples, permute_tbl.val[0]);
- *b1 = vqtbl2q_u8(samples, permute_tbl.val[1]);
-}
-
-static INLINE int16x4_t convolve8_4_usdot_partial(const uint8x16_t samples_lo,
- const uint8x16_t samples_hi,
- const int8x8_t filter) {
- /* Sample permutation is performed by the caller. */
- int32x4_t sum;
-
- sum = vusdotq_lane_s32(vdupq_n_s32(0), samples_lo, filter, 0);
- sum = vusdotq_lane_s32(sum, samples_hi, filter, 1);
-
- /* Further narrowing and packing is performed by the caller. */
- return vqmovn_s32(sum);
-}
-
-static INLINE uint8x8_t convolve8_8_usdot_partial(const uint8x16_t samples0_lo,
- const uint8x16_t samples0_hi,
- const uint8x16_t samples1_lo,
- const uint8x16_t samples1_hi,
- const int8x8_t filter) {
- /* Sample permutation is performed by the caller. */
- int32x4_t sum0, sum1;
- int16x8_t sum;
-
- /* First 4 output values. */
- sum0 = vusdotq_lane_s32(vdupq_n_s32(0), samples0_lo, filter, 0);
- sum0 = vusdotq_lane_s32(sum0, samples0_hi, filter, 1);
- /* Second 4 output values. */
- sum1 = vusdotq_lane_s32(vdupq_n_s32(0), samples1_lo, filter, 0);
- sum1 = vusdotq_lane_s32(sum1, samples1_hi, filter, 1);
-
- /* Narrow and re-pack. */
- sum = vcombine_s16(vqmovn_s32(sum0), vqmovn_s32(sum1));
- return vqrshrun_n_s16(sum, FILTER_BITS);
-}
-
-void aom_convolve8_vert_neon(const uint8_t *src, ptrdiff_t src_stride,
- uint8_t *dst, ptrdiff_t dst_stride,
- const int16_t *filter_x, int x_step_q4,
- const int16_t *filter_y, int y_step_q4, int w,
- int h) {
- const int8x8_t filter = vmovn_s16(vld1q_s16(filter_y));
- const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(dot_prod_merge_block_tbl);
- uint8x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
- uint8x16x2_t samples_LUT;
-
- assert((intptr_t)dst % 4 == 0);
- assert(dst_stride % 4 == 0);
-
- (void)filter_x;
- (void)x_step_q4;
- (void)y_step_q4;
-
- src -= ((SUBPEL_TAPS / 2) - 1) * src_stride;
-
- if (w == 4) {
- const uint8x16_t tran_concat_tbl = vld1q_u8(dot_prod_tran_concat_tbl);
- uint8x16_t s0123, s1234, s2345, s3456, s4567, s5678, s6789, s78910;
- int16x4_t d0, d1, d2, d3;
- uint8x8_t d01, d23;
-
- load_u8_8x7(src, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
- src += 7 * src_stride;
-
- s7 = vdup_n_u8(0);
- s8 = vdup_n_u8(0);
- s9 = vdup_n_u8(0);
-
- /* This operation combines a conventional transpose and the sample permute
- * (see horizontal case) required before computing the dot product.
- */
- transpose_concat_4x4(s0, s1, s2, s3, &s0123, tran_concat_tbl);
- transpose_concat_4x4(s1, s2, s3, s4, &s1234, tran_concat_tbl);
- transpose_concat_4x4(s2, s3, s4, s5, &s2345, tran_concat_tbl);
- transpose_concat_4x4(s3, s4, s5, s6, &s3456, tran_concat_tbl);
- transpose_concat_4x4(s4, s5, s6, s7, &s4567, tran_concat_tbl);
- transpose_concat_4x4(s5, s6, s7, s8, &s5678, tran_concat_tbl);
- transpose_concat_4x4(s6, s7, s8, s9, &s6789, tran_concat_tbl);
-
- do {
- load_u8_8x4(src, src_stride, &s7, &s8, &s9, &s10);
-
- transpose_concat_4x4(s7, s8, s9, s10, &s78910, tran_concat_tbl);
-
- /* Merge new data into block from previous iteration. */
- samples_LUT.val[0] = s3456;
- samples_LUT.val[1] = s78910;
- s4567 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[0]);
- s5678 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[1]);
- s6789 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[2]);
-
- d0 = convolve8_4_usdot_partial(s0123, s4567, filter);
- d1 = convolve8_4_usdot_partial(s1234, s5678, filter);
- d2 = convolve8_4_usdot_partial(s2345, s6789, filter);
- d3 = convolve8_4_usdot_partial(s3456, s78910, filter);
- d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS);
- d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS);
-
- store_u8_4x1(dst + 0 * dst_stride, d01, 0);
- store_u8_4x1(dst + 1 * dst_stride, d01, 1);
- store_u8_4x1(dst + 2 * dst_stride, d23, 0);
- store_u8_4x1(dst + 3 * dst_stride, d23, 1);
-
- /* Prepare block for next iteration - re-using as much as possible. */
- /* Shuffle everything up four rows. */
- s0123 = s4567;
- s1234 = s5678;
- s2345 = s6789;
- s3456 = s78910;
-
- src += 4 * src_stride;
- dst += 4 * dst_stride;
- h -= 4;
- } while (h != 0);
- } else {
- const uint8x16x2_t tran_concat_tbl = vld1q_u8_x2(dot_prod_tran_concat_tbl);
- uint8x16_t s0123_lo, s0123_hi, s1234_lo, s1234_hi, s2345_lo, s2345_hi,
- s3456_lo, s3456_hi, s4567_lo, s4567_hi, s5678_lo, s5678_hi, s6789_lo,
- s6789_hi, s78910_lo, s78910_hi;
- uint8x8_t d0, d1, d2, d3;
- const uint8_t *s;
- uint8_t *d;
- int height;
-
- do {
- height = h;
- s = src;
- d = dst;
-
- load_u8_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
- s += 7 * src_stride;
-
- s7 = vdup_n_u8(0);
- s8 = vdup_n_u8(0);
- s9 = vdup_n_u8(0);
-
- /* This operation combines a conventional transpose and the sample permute
- * (see horizontal case) required before computing the dot product.
- */
- transpose_concat_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi,
- tran_concat_tbl);
- transpose_concat_8x4(s1, s2, s3, s4, &s1234_lo, &s1234_hi,
- tran_concat_tbl);
- transpose_concat_8x4(s2, s3, s4, s5, &s2345_lo, &s2345_hi,
- tran_concat_tbl);
- transpose_concat_8x4(s3, s4, s5, s6, &s3456_lo, &s3456_hi,
- tran_concat_tbl);
- transpose_concat_8x4(s4, s5, s6, s7, &s4567_lo, &s4567_hi,
- tran_concat_tbl);
- transpose_concat_8x4(s5, s6, s7, s8, &s5678_lo, &s5678_hi,
- tran_concat_tbl);
- transpose_concat_8x4(s6, s7, s8, s9, &s6789_lo, &s6789_hi,
- tran_concat_tbl);
-
- do {
- load_u8_8x4(s, src_stride, &s7, &s8, &s9, &s10);
-
- transpose_concat_8x4(s7, s8, s9, s10, &s78910_lo, &s78910_hi,
- tran_concat_tbl);
-
- /* Merge new data into block from previous iteration. */
- samples_LUT.val[0] = s3456_lo;
- samples_LUT.val[1] = s78910_lo;
- s4567_lo = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[0]);
- s5678_lo = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[1]);
- s6789_lo = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[2]);
-
- samples_LUT.val[0] = s3456_hi;
- samples_LUT.val[1] = s78910_hi;
- s4567_hi = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[0]);
- s5678_hi = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[1]);
- s6789_hi = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[2]);
-
- d0 = convolve8_8_usdot_partial(s0123_lo, s4567_lo, s0123_hi, s4567_hi,
- filter);
- d1 = convolve8_8_usdot_partial(s1234_lo, s5678_lo, s1234_hi, s5678_hi,
- filter);
- d2 = convolve8_8_usdot_partial(s2345_lo, s6789_lo, s2345_hi, s6789_hi,
- filter);
- d3 = convolve8_8_usdot_partial(s3456_lo, s78910_lo, s3456_hi, s78910_hi,
- filter);
-
- store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
-
- /* Prepare block for next iteration - re-using as much as possible. */
- /* Shuffle everything up four rows. */
- s0123_lo = s4567_lo;
- s0123_hi = s4567_hi;
- s1234_lo = s5678_lo;
- s1234_hi = s5678_hi;
- s2345_lo = s6789_lo;
- s2345_hi = s6789_hi;
- s3456_lo = s78910_lo;
- s3456_hi = s78910_hi;
-
- s += 4 * src_stride;
- d += 4 * dst_stride;
- height -= 4;
- } while (height != 0);
- src += 8;
- dst += 8;
- w -= 8;
- } while (w != 0);
- }
-}
-
-#else // !defined(__ARM_FEATURE_MATMUL_INT8)
-
-static INLINE int16x4_t convolve8_4_sdot(uint8x16_t samples,
- const int8x8_t filter,
- const int32x4_t correction,
- const uint8x16_t range_limit,
- const uint8x16x2_t permute_tbl) {
- int8x16_t clamped_samples, permuted_samples[2];
- int32x4_t sum;
-
- /* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
- clamped_samples = vreinterpretq_s8_u8(vsubq_u8(samples, range_limit));
-
- /* Permute samples ready for dot product. */
- /* { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } */
- permuted_samples[0] = vqtbl1q_s8(clamped_samples, permute_tbl.val[0]);
- /* { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } */
- permuted_samples[1] = vqtbl1q_s8(clamped_samples, permute_tbl.val[1]);
-
- /* Accumulate dot product into 'correction' to account for range clamp. */
- sum = vdotq_lane_s32(correction, permuted_samples[0], filter, 0);
- sum = vdotq_lane_s32(sum, permuted_samples[1], filter, 1);
-
- /* Further narrowing and packing is performed by the caller. */
- return vqmovn_s32(sum);
-}
-
-static INLINE uint8x8_t convolve8_8_sdot(uint8x16_t samples,
- const int8x8_t filter,
- const int32x4_t correction,
- const uint8x16_t range_limit,
- const uint8x16x3_t permute_tbl) {
- int8x16_t clamped_samples, permuted_samples[3];
- int32x4_t sum0, sum1;
- int16x8_t sum;
-
- /* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
- clamped_samples = vreinterpretq_s8_u8(vsubq_u8(samples, range_limit));
-
- /* Permute samples ready for dot product. */
- /* { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } */
- permuted_samples[0] = vqtbl1q_s8(clamped_samples, permute_tbl.val[0]);
- /* { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } */
- permuted_samples[1] = vqtbl1q_s8(clamped_samples, permute_tbl.val[1]);
- /* { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 } */
- permuted_samples[2] = vqtbl1q_s8(clamped_samples, permute_tbl.val[2]);
-
- /* Accumulate dot product into 'correction' to account for range clamp. */
- /* First 4 output values. */
- sum0 = vdotq_lane_s32(correction, permuted_samples[0], filter, 0);
- sum0 = vdotq_lane_s32(sum0, permuted_samples[1], filter, 1);
- /* Second 4 output values. */
- sum1 = vdotq_lane_s32(correction, permuted_samples[1], filter, 0);
- sum1 = vdotq_lane_s32(sum1, permuted_samples[2], filter, 1);
-
- /* Narrow and re-pack. */
- sum = vcombine_s16(vqmovn_s32(sum0), vqmovn_s32(sum1));
- return vqrshrun_n_s16(sum, FILTER_BITS);
-}
-
-void aom_convolve8_horiz_neon(const uint8_t *src, ptrdiff_t src_stride,
- uint8_t *dst, ptrdiff_t dst_stride,
- const int16_t *filter_x, int x_step_q4,
- const int16_t *filter_y, int y_step_q4, int w,
- int h) {
- const int8x8_t filter = vmovn_s16(vld1q_s16(filter_x));
- const int16x8_t correct_tmp = vmulq_n_s16(vld1q_s16(filter_x), 128);
- const int32x4_t correction = vdupq_n_s32((int32_t)vaddvq_s16(correct_tmp));
- const uint8x16_t range_limit = vdupq_n_u8(128);
- uint8x16_t s0, s1, s2, s3;
-
- assert((intptr_t)dst % 4 == 0);
- assert(dst_stride % 4 == 0);
-
- (void)x_step_q4;
- (void)filter_y;
- (void)y_step_q4;
-
- src -= ((SUBPEL_TAPS / 2) - 1);
-
- if (w == 4) {
- const uint8x16x2_t perm_tbl = vld1q_u8_x2(dot_prod_permute_tbl);
- do {
- int16x4_t t0, t1, t2, t3;
- uint8x8_t d01, d23;
-
- load_u8_16x4(src, src_stride, &s0, &s1, &s2, &s3);
-
- t0 = convolve8_4_sdot(s0, filter, correction, range_limit, perm_tbl);
- t1 = convolve8_4_sdot(s1, filter, correction, range_limit, perm_tbl);
- t2 = convolve8_4_sdot(s2, filter, correction, range_limit, perm_tbl);
- t3 = convolve8_4_sdot(s3, filter, correction, range_limit, perm_tbl);
- d01 = vqrshrun_n_s16(vcombine_s16(t0, t1), FILTER_BITS);
- d23 = vqrshrun_n_s16(vcombine_s16(t2, t3), FILTER_BITS);
-
- store_u8_4x1(dst + 0 * dst_stride, d01, 0);
- store_u8_4x1(dst + 1 * dst_stride, d01, 1);
- store_u8_4x1(dst + 2 * dst_stride, d23, 0);
- store_u8_4x1(dst + 3 * dst_stride, d23, 1);
-
- src += 4 * src_stride;
- dst += 4 * dst_stride;
- h -= 4;
- } while (h > 0);
- } else {
- const uint8x16x3_t perm_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
- const uint8_t *s;
- uint8_t *d;
- int width;
- uint8x8_t d0, d1, d2, d3;
-
- do {
- width = w;
- s = src;
- d = dst;
- do {
- load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3);
-
- d0 = convolve8_8_sdot(s0, filter, correction, range_limit, perm_tbl);
- d1 = convolve8_8_sdot(s1, filter, correction, range_limit, perm_tbl);
- d2 = convolve8_8_sdot(s2, filter, correction, range_limit, perm_tbl);
- d3 = convolve8_8_sdot(s3, filter, correction, range_limit, perm_tbl);
-
- store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
-
- s += 8;
- d += 8;
- width -= 8;
- } while (width != 0);
- src += 4 * src_stride;
- dst += 4 * dst_stride;
- h -= 4;
- } while (h > 0);
- }
-}
-
-static INLINE void transpose_concat_4x4(int8x8_t a0, int8x8_t a1, int8x8_t a2,
- int8x8_t a3, int8x16_t *b,
- const uint8x16_t permute_tbl) {
- /* Transpose 8-bit elements and concatenate result rows as follows:
- * a0: 00, 01, 02, 03, XX, XX, XX, XX
- * a1: 10, 11, 12, 13, XX, XX, XX, XX
- * a2: 20, 21, 22, 23, XX, XX, XX, XX
- * a3: 30, 31, 32, 33, XX, XX, XX, XX
- *
- * b: 00, 10, 20, 30, 01, 11, 21, 31, 02, 12, 22, 32, 03, 13, 23, 33
- *
- * The 'permute_tbl' is always 'dot_prod_tran_concat_tbl' above. Passing it
- * as an argument is preferable to loading it directly from memory as this
- * inline helper is called many times from the same parent function.
- */
-
- int8x16x2_t samples = { { vcombine_s8(a0, a1), vcombine_s8(a2, a3) } };
- *b = vqtbl2q_s8(samples, permute_tbl);
-}
-
-static INLINE void transpose_concat_8x4(int8x8_t a0, int8x8_t a1, int8x8_t a2,
- int8x8_t a3, int8x16_t *b0,
- int8x16_t *b1,
- const uint8x16x2_t permute_tbl) {
- /* Transpose 8-bit elements and concatenate result rows as follows:
- * a0: 00, 01, 02, 03, 04, 05, 06, 07
- * a1: 10, 11, 12, 13, 14, 15, 16, 17
- * a2: 20, 21, 22, 23, 24, 25, 26, 27
- * a3: 30, 31, 32, 33, 34, 35, 36, 37
- *
- * b0: 00, 10, 20, 30, 01, 11, 21, 31, 02, 12, 22, 32, 03, 13, 23, 33
- * b1: 04, 14, 24, 34, 05, 15, 25, 35, 06, 16, 26, 36, 07, 17, 27, 37
- *
- * The 'permute_tbl' is always 'dot_prod_tran_concat_tbl' above. Passing it
- * as an argument is preferable to loading it directly from memory as this
- * inline helper is called many times from the same parent function.
- */
-
- int8x16x2_t samples = { { vcombine_s8(a0, a1), vcombine_s8(a2, a3) } };
- *b0 = vqtbl2q_s8(samples, permute_tbl.val[0]);
- *b1 = vqtbl2q_s8(samples, permute_tbl.val[1]);
-}
-
-static INLINE int16x4_t convolve8_4_sdot_partial(const int8x16_t samples_lo,
- const int8x16_t samples_hi,
- const int32x4_t correction,
- const int8x8_t filter) {
- /* Sample range-clamping and permutation are performed by the caller. */
- int32x4_t sum;
-
- /* Accumulate dot product into 'correction' to account for range clamp. */
- sum = vdotq_lane_s32(correction, samples_lo, filter, 0);
- sum = vdotq_lane_s32(sum, samples_hi, filter, 1);
-
- /* Further narrowing and packing is performed by the caller. */
- return vqmovn_s32(sum);
-}
-
-static INLINE uint8x8_t convolve8_8_sdot_partial(const int8x16_t samples0_lo,
- const int8x16_t samples0_hi,
- const int8x16_t samples1_lo,
- const int8x16_t samples1_hi,
- const int32x4_t correction,
- const int8x8_t filter) {
- /* Sample range-clamping and permutation are performed by the caller. */
- int32x4_t sum0, sum1;
- int16x8_t sum;
-
- /* Accumulate dot product into 'correction' to account for range clamp. */
- /* First 4 output values. */
- sum0 = vdotq_lane_s32(correction, samples0_lo, filter, 0);
- sum0 = vdotq_lane_s32(sum0, samples0_hi, filter, 1);
- /* Second 4 output values. */
- sum1 = vdotq_lane_s32(correction, samples1_lo, filter, 0);
- sum1 = vdotq_lane_s32(sum1, samples1_hi, filter, 1);
-
- /* Narrow and re-pack. */
- sum = vcombine_s16(vqmovn_s32(sum0), vqmovn_s32(sum1));
- return vqrshrun_n_s16(sum, FILTER_BITS);
-}
-
-void aom_convolve8_vert_neon(const uint8_t *src, ptrdiff_t src_stride,
- uint8_t *dst, ptrdiff_t dst_stride,
- const int16_t *filter_x, int x_step_q4,
- const int16_t *filter_y, int y_step_q4, int w,
- int h) {
- const int8x8_t filter = vmovn_s16(vld1q_s16(filter_y));
- const int16x8_t correct_tmp = vmulq_n_s16(vld1q_s16(filter_y), 128);
- const int32x4_t correction = vdupq_n_s32((int32_t)vaddvq_s16(correct_tmp));
- const uint8x8_t range_limit = vdup_n_u8(128);
- const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(dot_prod_merge_block_tbl);
- uint8x8_t t0, t1, t2, t3, t4, t5, t6;
- int8x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
- int8x16x2_t samples_LUT;
-
- assert((intptr_t)dst % 4 == 0);
- assert(dst_stride % 4 == 0);
-
- (void)filter_x;
- (void)x_step_q4;
- (void)y_step_q4;
-
- src -= ((SUBPEL_TAPS / 2) - 1) * src_stride;
-
- if (w == 4) {
- const uint8x16_t tran_concat_tbl = vld1q_u8(dot_prod_tran_concat_tbl);
- int8x16_t s0123, s1234, s2345, s3456, s4567, s5678, s6789, s78910;
- int16x4_t d0, d1, d2, d3;
- uint8x8_t d01, d23;
-
- load_u8_8x7(src, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6);
- src += 7 * src_stride;
-
- /* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
- s0 = vreinterpret_s8_u8(vsub_u8(t0, range_limit));
- s1 = vreinterpret_s8_u8(vsub_u8(t1, range_limit));
- s2 = vreinterpret_s8_u8(vsub_u8(t2, range_limit));
- s3 = vreinterpret_s8_u8(vsub_u8(t3, range_limit));
- s4 = vreinterpret_s8_u8(vsub_u8(t4, range_limit));
- s5 = vreinterpret_s8_u8(vsub_u8(t5, range_limit));
- s6 = vreinterpret_s8_u8(vsub_u8(t6, range_limit));
- s7 = vdup_n_s8(0);
- s8 = vdup_n_s8(0);
- s9 = vdup_n_s8(0);
-
- /* This operation combines a conventional transpose and the sample permute
- * (see horizontal case) required before computing the dot product.
- */
- transpose_concat_4x4(s0, s1, s2, s3, &s0123, tran_concat_tbl);
- transpose_concat_4x4(s1, s2, s3, s4, &s1234, tran_concat_tbl);
- transpose_concat_4x4(s2, s3, s4, s5, &s2345, tran_concat_tbl);
- transpose_concat_4x4(s3, s4, s5, s6, &s3456, tran_concat_tbl);
- transpose_concat_4x4(s4, s5, s6, s7, &s4567, tran_concat_tbl);
- transpose_concat_4x4(s5, s6, s7, s8, &s5678, tran_concat_tbl);
- transpose_concat_4x4(s6, s7, s8, s9, &s6789, tran_concat_tbl);
-
- do {
- uint8x8_t t7, t8, t9, t10;
-
- load_u8_8x4(src, src_stride, &t7, &t8, &t9, &t10);
-
- s7 = vreinterpret_s8_u8(vsub_u8(t7, range_limit));
- s8 = vreinterpret_s8_u8(vsub_u8(t8, range_limit));
- s9 = vreinterpret_s8_u8(vsub_u8(t9, range_limit));
- s10 = vreinterpret_s8_u8(vsub_u8(t10, range_limit));
-
- transpose_concat_4x4(s7, s8, s9, s10, &s78910, tran_concat_tbl);
-
- /* Merge new data into block from previous iteration. */
- samples_LUT.val[0] = s3456;
- samples_LUT.val[1] = s78910;
- s4567 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]);
- s5678 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]);
- s6789 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]);
-
- d0 = convolve8_4_sdot_partial(s0123, s4567, correction, filter);
- d1 = convolve8_4_sdot_partial(s1234, s5678, correction, filter);
- d2 = convolve8_4_sdot_partial(s2345, s6789, correction, filter);
- d3 = convolve8_4_sdot_partial(s3456, s78910, correction, filter);
- d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS);
- d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS);
-
- store_u8_4x1(dst + 0 * dst_stride, d01, 0);
- store_u8_4x1(dst + 1 * dst_stride, d01, 1);
- store_u8_4x1(dst + 2 * dst_stride, d23, 0);
- store_u8_4x1(dst + 3 * dst_stride, d23, 1);
-
- /* Prepare block for next iteration - re-using as much as possible. */
- /* Shuffle everything up four rows. */
- s0123 = s4567;
- s1234 = s5678;
- s2345 = s6789;
- s3456 = s78910;
-
- src += 4 * src_stride;
- dst += 4 * dst_stride;
- h -= 4;
- } while (h != 0);
- } else {
- const uint8x16x2_t tran_concat_tbl = vld1q_u8_x2(dot_prod_tran_concat_tbl);
- int8x16_t s0123_lo, s0123_hi, s1234_lo, s1234_hi, s2345_lo, s2345_hi,
- s3456_lo, s3456_hi, s4567_lo, s4567_hi, s5678_lo, s5678_hi, s6789_lo,
- s6789_hi, s78910_lo, s78910_hi;
- uint8x8_t d0, d1, d2, d3;
- const uint8_t *s;
- uint8_t *d;
- int height;
-
- do {
- height = h;
- s = src;
- d = dst;
-
- load_u8_8x7(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6);
- s += 7 * src_stride;
-
- /* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
- s0 = vreinterpret_s8_u8(vsub_u8(t0, range_limit));
- s1 = vreinterpret_s8_u8(vsub_u8(t1, range_limit));
- s2 = vreinterpret_s8_u8(vsub_u8(t2, range_limit));
- s3 = vreinterpret_s8_u8(vsub_u8(t3, range_limit));
- s4 = vreinterpret_s8_u8(vsub_u8(t4, range_limit));
- s5 = vreinterpret_s8_u8(vsub_u8(t5, range_limit));
- s6 = vreinterpret_s8_u8(vsub_u8(t6, range_limit));
- s7 = vdup_n_s8(0);
- s8 = vdup_n_s8(0);
- s9 = vdup_n_s8(0);
-
- /* This operation combines a conventional transpose and the sample permute
- * (see horizontal case) required before computing the dot product.
- */
- transpose_concat_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi,
- tran_concat_tbl);
- transpose_concat_8x4(s1, s2, s3, s4, &s1234_lo, &s1234_hi,
- tran_concat_tbl);
- transpose_concat_8x4(s2, s3, s4, s5, &s2345_lo, &s2345_hi,
- tran_concat_tbl);
- transpose_concat_8x4(s3, s4, s5, s6, &s3456_lo, &s3456_hi,
- tran_concat_tbl);
- transpose_concat_8x4(s4, s5, s6, s7, &s4567_lo, &s4567_hi,
- tran_concat_tbl);
- transpose_concat_8x4(s5, s6, s7, s8, &s5678_lo, &s5678_hi,
- tran_concat_tbl);
- transpose_concat_8x4(s6, s7, s8, s9, &s6789_lo, &s6789_hi,
- tran_concat_tbl);
-
- do {
- uint8x8_t t7, t8, t9, t10;
-
- load_u8_8x4(s, src_stride, &t7, &t8, &t9, &t10);
-
- s7 = vreinterpret_s8_u8(vsub_u8(t7, range_limit));
- s8 = vreinterpret_s8_u8(vsub_u8(t8, range_limit));
- s9 = vreinterpret_s8_u8(vsub_u8(t9, range_limit));
- s10 = vreinterpret_s8_u8(vsub_u8(t10, range_limit));
-
- transpose_concat_8x4(s7, s8, s9, s10, &s78910_lo, &s78910_hi,
- tran_concat_tbl);
-
- /* Merge new data into block from previous iteration. */
- samples_LUT.val[0] = s3456_lo;
- samples_LUT.val[1] = s78910_lo;
- s4567_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]);
- s5678_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]);
- s6789_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]);
-
- samples_LUT.val[0] = s3456_hi;
- samples_LUT.val[1] = s78910_hi;
- s4567_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]);
- s5678_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]);
- s6789_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]);
-
- d0 = convolve8_8_sdot_partial(s0123_lo, s4567_lo, s0123_hi, s4567_hi,
- correction, filter);
- d1 = convolve8_8_sdot_partial(s1234_lo, s5678_lo, s1234_hi, s5678_hi,
- correction, filter);
- d2 = convolve8_8_sdot_partial(s2345_lo, s6789_lo, s2345_hi, s6789_hi,
- correction, filter);
- d3 = convolve8_8_sdot_partial(s3456_lo, s78910_lo, s3456_hi, s78910_hi,
- correction, filter);
-
- store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
-
- /* Prepare block for next iteration - re-using as much as possible. */
- /* Shuffle everything up four rows. */
- s0123_lo = s4567_lo;
- s0123_hi = s4567_hi;
- s1234_lo = s5678_lo;
- s1234_hi = s5678_hi;
- s2345_lo = s6789_lo;
- s2345_hi = s6789_hi;
- s3456_lo = s78910_lo;
- s3456_hi = s78910_hi;
-
- s += 4 * src_stride;
- d += 4 * dst_stride;
- height -= 4;
- } while (height != 0);
- src += 8;
- dst += 8;
- w -= 8;
- } while (w != 0);
- }
-}
-
-#endif // defined(__ARM_FEATURE_MATMUL_INT8)
-
-#else // !(AOM_ARCH_AARCH64 &&
- // (defined(__ARM_FEATURE_DOTPROD) ||
- // defined(__ARM_FEATURE_MATMUL_INT8)))
-
static INLINE int16x4_t convolve8_4(const int16x4_t s0, const int16x4_t s1,
const int16x4_t s2, const int16x4_t s3,
const int16x4_t s4, const int16x4_t s5,
@@ -905,7 +85,7 @@
int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, d0, d1, d2, d3;
load_u8_8x4(src, src_stride, &t0, &t1, &t2, &t3);
- transpose_u8_8x4(&t0, &t1, &t2, &t3);
+ transpose_elems_inplace_u8_8x4(&t0, &t1, &t2, &t3);
s0 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
s1 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
s2 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
@@ -918,7 +98,7 @@
do {
load_u8_8x4(src, src_stride, &t0, &t1, &t2, &t3);
- transpose_u8_8x4(&t0, &t1, &t2, &t3);
+ transpose_elems_inplace_u8_8x4(&t0, &t1, &t2, &t3);
s7 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
s8 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
s9 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
@@ -931,7 +111,7 @@
d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS);
d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS);
- transpose_u8_4x4(&d01, &d23);
+ transpose_elems_inplace_u8_4x4(&d01, &d23);
store_u8_4x1(dst + 0 * dst_stride, d01, 0);
store_u8_4x1(dst + 1 * dst_stride, d23, 0);
@@ -956,7 +136,7 @@
if (w == 4) {
do {
load_u8_8x8(src, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
- transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
+ transpose_elems_inplace_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
@@ -967,7 +147,8 @@
load_u8_8x8(src + 7, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6,
&t7);
- transpose_u8_4x8(&t0, &t1, &t2, &t3, t4, t5, t6, t7);
+ transpose_elems_u8_4x8(t0, t1, t2, t3, t4, t5, t6, t7, &t0, &t1, &t2,
+ &t3);
s7 = vreinterpretq_s16_u16(vmovl_u8(t0));
s8 = vreinterpretq_s16_u16(vmovl_u8(t1));
s9 = vreinterpretq_s16_u16(vmovl_u8(t2));
@@ -978,7 +159,7 @@
d2 = convolve8_8(s2, s3, s4, s5, s6, s7, s8, s9, filter);
d3 = convolve8_8(s3, s4, s5, s6, s7, s8, s9, s10, filter);
- transpose_u8_8x4(&d0, &d1, &d2, &d3);
+ transpose_elems_inplace_u8_8x4(&d0, &d1, &d2, &d3);
store_u8_4x1(dst + 0 * dst_stride, d0, 0);
store_u8_4x1(dst + 1 * dst_stride, d1, 0);
@@ -1002,7 +183,7 @@
do {
load_u8_8x8(src, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
- transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
+ transpose_elems_inplace_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
@@ -1017,7 +198,8 @@
do {
load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
- transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
+ transpose_elems_inplace_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6,
+ &t7);
s7 = vreinterpretq_s16_u16(vmovl_u8(t0));
s8 = vreinterpretq_s16_u16(vmovl_u8(t1));
s9 = vreinterpretq_s16_u16(vmovl_u8(t2));
@@ -1036,7 +218,8 @@
d6 = convolve8_8(s6, s7, s8, s9, s10, s11, s12, s13, filter);
d7 = convolve8_8(s7, s8, s9, s10, s11, s12, s13, s14, filter);
- transpose_u8_8x8(&d0, &d1, &d2, &d3, &d4, &d5, &d6, &d7);
+ transpose_elems_inplace_u8_8x8(&d0, &d1, &d2, &d3, &d4, &d5, &d6,
+ &d7);
store_u8_8x8(d, dst_stride, d0, d1, d2, d3, d4, d5, d6, d7);
@@ -1172,5 +355,3 @@
} while (w != 0);
}
}
-
-#endif // AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_DOTPROD)
diff --git a/aom_dsp/arm/aom_convolve8_neon_dotprod.c b/aom_dsp/arm/aom_convolve8_neon_dotprod.c
new file mode 100644
index 0000000..e565414
--- /dev/null
+++ b/aom_dsp/arm/aom_convolve8_neon_dotprod.c
@@ -0,0 +1,464 @@
+/*
+ * Copyright (c) 2014 The WebM project authors. All Rights Reserved.
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+#include <assert.h>
+#include <string.h>
+
+#include "config/aom_config.h"
+#include "config/aom_dsp_rtcd.h"
+
+#include "aom/aom_integer.h"
+#include "aom_dsp/aom_dsp_common.h"
+#include "aom_dsp/aom_filter.h"
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_dsp/arm/transpose_neon.h"
+#include "aom_ports/mem.h"
+
+DECLARE_ALIGNED(16, static const uint8_t, dot_prod_permute_tbl[48]) = {
+ 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6,
+ 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10,
+ 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
+};
+
+DECLARE_ALIGNED(16, static const uint8_t, dot_prod_tran_concat_tbl[32]) = {
+ 0, 8, 16, 24, 1, 9, 17, 25, 2, 10, 18, 26, 3, 11, 19, 27,
+ 4, 12, 20, 28, 5, 13, 21, 29, 6, 14, 22, 30, 7, 15, 23, 31
+};
+
+DECLARE_ALIGNED(16, static const uint8_t, dot_prod_merge_block_tbl[48]) = {
+ /* Shift left and insert new last column in transposed 4x4 block. */
+ 1, 2, 3, 16, 5, 6, 7, 20, 9, 10, 11, 24, 13, 14, 15, 28,
+ /* Shift left and insert two new columns in transposed 4x4 block. */
+ 2, 3, 16, 17, 6, 7, 20, 21, 10, 11, 24, 25, 14, 15, 28, 29,
+ /* Shift left and insert three new columns in transposed 4x4 block. */
+ 3, 16, 17, 18, 7, 20, 21, 22, 11, 24, 25, 26, 15, 28, 29, 30
+};
+
+static INLINE int16x4_t convolve8_4_sdot(uint8x16_t samples,
+ const int8x8_t filter,
+ const int32x4_t correction,
+ const uint8x16_t range_limit,
+ const uint8x16x2_t permute_tbl) {
+ int8x16_t clamped_samples, permuted_samples[2];
+ int32x4_t sum;
+
+ /* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
+ clamped_samples = vreinterpretq_s8_u8(vsubq_u8(samples, range_limit));
+
+ /* Permute samples ready for dot product. */
+ /* { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } */
+ permuted_samples[0] = vqtbl1q_s8(clamped_samples, permute_tbl.val[0]);
+ /* { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } */
+ permuted_samples[1] = vqtbl1q_s8(clamped_samples, permute_tbl.val[1]);
+
+ /* Accumulate dot product into 'correction' to account for range clamp. */
+ sum = vdotq_lane_s32(correction, permuted_samples[0], filter, 0);
+ sum = vdotq_lane_s32(sum, permuted_samples[1], filter, 1);
+
+ /* Further narrowing and packing is performed by the caller. */
+ return vqmovn_s32(sum);
+}
+
+static INLINE uint8x8_t convolve8_8_sdot(uint8x16_t samples,
+ const int8x8_t filter,
+ const int32x4_t correction,
+ const uint8x16_t range_limit,
+ const uint8x16x3_t permute_tbl) {
+ int8x16_t clamped_samples, permuted_samples[3];
+ int32x4_t sum0, sum1;
+ int16x8_t sum;
+
+ /* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
+ clamped_samples = vreinterpretq_s8_u8(vsubq_u8(samples, range_limit));
+
+ /* Permute samples ready for dot product. */
+ /* { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } */
+ permuted_samples[0] = vqtbl1q_s8(clamped_samples, permute_tbl.val[0]);
+ /* { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } */
+ permuted_samples[1] = vqtbl1q_s8(clamped_samples, permute_tbl.val[1]);
+ /* { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 } */
+ permuted_samples[2] = vqtbl1q_s8(clamped_samples, permute_tbl.val[2]);
+
+ /* Accumulate dot product into 'correction' to account for range clamp. */
+ /* First 4 output values. */
+ sum0 = vdotq_lane_s32(correction, permuted_samples[0], filter, 0);
+ sum0 = vdotq_lane_s32(sum0, permuted_samples[1], filter, 1);
+ /* Second 4 output values. */
+ sum1 = vdotq_lane_s32(correction, permuted_samples[1], filter, 0);
+ sum1 = vdotq_lane_s32(sum1, permuted_samples[2], filter, 1);
+
+ /* Narrow and re-pack. */
+ sum = vcombine_s16(vqmovn_s32(sum0), vqmovn_s32(sum1));
+ return vqrshrun_n_s16(sum, FILTER_BITS);
+}
+
+void aom_convolve8_horiz_neon_dotprod(const uint8_t *src, ptrdiff_t src_stride,
+ uint8_t *dst, ptrdiff_t dst_stride,
+ const int16_t *filter_x, int x_step_q4,
+ const int16_t *filter_y, int y_step_q4,
+ int w, int h) {
+ const int8x8_t filter = vmovn_s16(vld1q_s16(filter_x));
+ const int16x8_t correct_tmp = vmulq_n_s16(vld1q_s16(filter_x), 128);
+ const int32x4_t correction = vdupq_n_s32((int32_t)vaddvq_s16(correct_tmp));
+ const uint8x16_t range_limit = vdupq_n_u8(128);
+ uint8x16_t s0, s1, s2, s3;
+
+ assert((intptr_t)dst % 4 == 0);
+ assert(dst_stride % 4 == 0);
+
+ (void)x_step_q4;
+ (void)filter_y;
+ (void)y_step_q4;
+
+ src -= ((SUBPEL_TAPS / 2) - 1);
+
+ if (w == 4) {
+ const uint8x16x2_t perm_tbl = vld1q_u8_x2(dot_prod_permute_tbl);
+ do {
+ int16x4_t t0, t1, t2, t3;
+ uint8x8_t d01, d23;
+
+ load_u8_16x4(src, src_stride, &s0, &s1, &s2, &s3);
+
+ t0 = convolve8_4_sdot(s0, filter, correction, range_limit, perm_tbl);
+ t1 = convolve8_4_sdot(s1, filter, correction, range_limit, perm_tbl);
+ t2 = convolve8_4_sdot(s2, filter, correction, range_limit, perm_tbl);
+ t3 = convolve8_4_sdot(s3, filter, correction, range_limit, perm_tbl);
+ d01 = vqrshrun_n_s16(vcombine_s16(t0, t1), FILTER_BITS);
+ d23 = vqrshrun_n_s16(vcombine_s16(t2, t3), FILTER_BITS);
+
+ store_u8_4x1(dst + 0 * dst_stride, d01, 0);
+ store_u8_4x1(dst + 1 * dst_stride, d01, 1);
+ store_u8_4x1(dst + 2 * dst_stride, d23, 0);
+ store_u8_4x1(dst + 3 * dst_stride, d23, 1);
+
+ src += 4 * src_stride;
+ dst += 4 * dst_stride;
+ h -= 4;
+ } while (h > 0);
+ } else {
+ const uint8x16x3_t perm_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
+ const uint8_t *s;
+ uint8_t *d;
+ int width;
+ uint8x8_t d0, d1, d2, d3;
+
+ do {
+ width = w;
+ s = src;
+ d = dst;
+ do {
+ load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3);
+
+ d0 = convolve8_8_sdot(s0, filter, correction, range_limit, perm_tbl);
+ d1 = convolve8_8_sdot(s1, filter, correction, range_limit, perm_tbl);
+ d2 = convolve8_8_sdot(s2, filter, correction, range_limit, perm_tbl);
+ d3 = convolve8_8_sdot(s3, filter, correction, range_limit, perm_tbl);
+
+ store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src += 4 * src_stride;
+ dst += 4 * dst_stride;
+ h -= 4;
+ } while (h > 0);
+ }
+}
+
+static INLINE void transpose_concat_4x4(int8x8_t a0, int8x8_t a1, int8x8_t a2,
+ int8x8_t a3, int8x16_t *b,
+ const uint8x16_t permute_tbl) {
+ /* Transpose 8-bit elements and concatenate result rows as follows:
+ * a0: 00, 01, 02, 03, XX, XX, XX, XX
+ * a1: 10, 11, 12, 13, XX, XX, XX, XX
+ * a2: 20, 21, 22, 23, XX, XX, XX, XX
+ * a3: 30, 31, 32, 33, XX, XX, XX, XX
+ *
+ * b: 00, 10, 20, 30, 01, 11, 21, 31, 02, 12, 22, 32, 03, 13, 23, 33
+ *
+ * The 'permute_tbl' is always 'dot_prod_tran_concat_tbl' above. Passing it
+ * as an argument is preferable to loading it directly from memory as this
+ * inline helper is called many times from the same parent function.
+ */
+
+ int8x16x2_t samples = { { vcombine_s8(a0, a1), vcombine_s8(a2, a3) } };
+ *b = vqtbl2q_s8(samples, permute_tbl);
+}
+
+static INLINE void transpose_concat_8x4(int8x8_t a0, int8x8_t a1, int8x8_t a2,
+ int8x8_t a3, int8x16_t *b0,
+ int8x16_t *b1,
+ const uint8x16x2_t permute_tbl) {
+ /* Transpose 8-bit elements and concatenate result rows as follows:
+ * a0: 00, 01, 02, 03, 04, 05, 06, 07
+ * a1: 10, 11, 12, 13, 14, 15, 16, 17
+ * a2: 20, 21, 22, 23, 24, 25, 26, 27
+ * a3: 30, 31, 32, 33, 34, 35, 36, 37
+ *
+ * b0: 00, 10, 20, 30, 01, 11, 21, 31, 02, 12, 22, 32, 03, 13, 23, 33
+ * b1: 04, 14, 24, 34, 05, 15, 25, 35, 06, 16, 26, 36, 07, 17, 27, 37
+ *
+ * The 'permute_tbl' is always 'dot_prod_tran_concat_tbl' above. Passing it
+ * as an argument is preferable to loading it directly from memory as this
+ * inline helper is called many times from the same parent function.
+ */
+
+ int8x16x2_t samples = { { vcombine_s8(a0, a1), vcombine_s8(a2, a3) } };
+ *b0 = vqtbl2q_s8(samples, permute_tbl.val[0]);
+ *b1 = vqtbl2q_s8(samples, permute_tbl.val[1]);
+}
+
+static INLINE int16x4_t convolve8_4_sdot_partial(const int8x16_t samples_lo,
+ const int8x16_t samples_hi,
+ const int32x4_t correction,
+ const int8x8_t filter) {
+ /* Sample range-clamping and permutation are performed by the caller. */
+ int32x4_t sum;
+
+ /* Accumulate dot product into 'correction' to account for range clamp. */
+ sum = vdotq_lane_s32(correction, samples_lo, filter, 0);
+ sum = vdotq_lane_s32(sum, samples_hi, filter, 1);
+
+ /* Further narrowing and packing is performed by the caller. */
+ return vqmovn_s32(sum);
+}
+
+static INLINE uint8x8_t convolve8_8_sdot_partial(const int8x16_t samples0_lo,
+ const int8x16_t samples0_hi,
+ const int8x16_t samples1_lo,
+ const int8x16_t samples1_hi,
+ const int32x4_t correction,
+ const int8x8_t filter) {
+ /* Sample range-clamping and permutation are performed by the caller. */
+ int32x4_t sum0, sum1;
+ int16x8_t sum;
+
+ /* Accumulate dot product into 'correction' to account for range clamp. */
+ /* First 4 output values. */
+ sum0 = vdotq_lane_s32(correction, samples0_lo, filter, 0);
+ sum0 = vdotq_lane_s32(sum0, samples0_hi, filter, 1);
+ /* Second 4 output values. */
+ sum1 = vdotq_lane_s32(correction, samples1_lo, filter, 0);
+ sum1 = vdotq_lane_s32(sum1, samples1_hi, filter, 1);
+
+ /* Narrow and re-pack. */
+ sum = vcombine_s16(vqmovn_s32(sum0), vqmovn_s32(sum1));
+ return vqrshrun_n_s16(sum, FILTER_BITS);
+}
+
+void aom_convolve8_vert_neon_dotprod(const uint8_t *src, ptrdiff_t src_stride,
+ uint8_t *dst, ptrdiff_t dst_stride,
+ const int16_t *filter_x, int x_step_q4,
+ const int16_t *filter_y, int y_step_q4,
+ int w, int h) {
+ const int8x8_t filter = vmovn_s16(vld1q_s16(filter_y));
+ const int16x8_t correct_tmp = vmulq_n_s16(vld1q_s16(filter_y), 128);
+ const int32x4_t correction = vdupq_n_s32((int32_t)vaddvq_s16(correct_tmp));
+ const uint8x8_t range_limit = vdup_n_u8(128);
+ const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(dot_prod_merge_block_tbl);
+ uint8x8_t t0, t1, t2, t3, t4, t5, t6;
+ int8x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
+ int8x16x2_t samples_LUT;
+
+ assert((intptr_t)dst % 4 == 0);
+ assert(dst_stride % 4 == 0);
+
+ (void)filter_x;
+ (void)x_step_q4;
+ (void)y_step_q4;
+
+ src -= ((SUBPEL_TAPS / 2) - 1) * src_stride;
+
+ if (w == 4) {
+ const uint8x16_t tran_concat_tbl = vld1q_u8(dot_prod_tran_concat_tbl);
+ int8x16_t s0123, s1234, s2345, s3456, s4567, s5678, s6789, s78910;
+ int16x4_t d0, d1, d2, d3;
+ uint8x8_t d01, d23;
+
+ load_u8_8x7(src, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6);
+ src += 7 * src_stride;
+
+ /* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
+ s0 = vreinterpret_s8_u8(vsub_u8(t0, range_limit));
+ s1 = vreinterpret_s8_u8(vsub_u8(t1, range_limit));
+ s2 = vreinterpret_s8_u8(vsub_u8(t2, range_limit));
+ s3 = vreinterpret_s8_u8(vsub_u8(t3, range_limit));
+ s4 = vreinterpret_s8_u8(vsub_u8(t4, range_limit));
+ s5 = vreinterpret_s8_u8(vsub_u8(t5, range_limit));
+ s6 = vreinterpret_s8_u8(vsub_u8(t6, range_limit));
+ s7 = vdup_n_s8(0);
+ s8 = vdup_n_s8(0);
+ s9 = vdup_n_s8(0);
+
+ /* This operation combines a conventional transpose and the sample permute
+ * (see horizontal case) required before computing the dot product.
+ */
+ transpose_concat_4x4(s0, s1, s2, s3, &s0123, tran_concat_tbl);
+ transpose_concat_4x4(s1, s2, s3, s4, &s1234, tran_concat_tbl);
+ transpose_concat_4x4(s2, s3, s4, s5, &s2345, tran_concat_tbl);
+ transpose_concat_4x4(s3, s4, s5, s6, &s3456, tran_concat_tbl);
+ transpose_concat_4x4(s4, s5, s6, s7, &s4567, tran_concat_tbl);
+ transpose_concat_4x4(s5, s6, s7, s8, &s5678, tran_concat_tbl);
+ transpose_concat_4x4(s6, s7, s8, s9, &s6789, tran_concat_tbl);
+
+ do {
+ uint8x8_t t7, t8, t9, t10;
+
+ load_u8_8x4(src, src_stride, &t7, &t8, &t9, &t10);
+
+ s7 = vreinterpret_s8_u8(vsub_u8(t7, range_limit));
+ s8 = vreinterpret_s8_u8(vsub_u8(t8, range_limit));
+ s9 = vreinterpret_s8_u8(vsub_u8(t9, range_limit));
+ s10 = vreinterpret_s8_u8(vsub_u8(t10, range_limit));
+
+ transpose_concat_4x4(s7, s8, s9, s10, &s78910, tran_concat_tbl);
+
+ /* Merge new data into block from previous iteration. */
+ samples_LUT.val[0] = s3456;
+ samples_LUT.val[1] = s78910;
+ s4567 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]);
+ s5678 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]);
+ s6789 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]);
+
+ d0 = convolve8_4_sdot_partial(s0123, s4567, correction, filter);
+ d1 = convolve8_4_sdot_partial(s1234, s5678, correction, filter);
+ d2 = convolve8_4_sdot_partial(s2345, s6789, correction, filter);
+ d3 = convolve8_4_sdot_partial(s3456, s78910, correction, filter);
+ d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS);
+ d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS);
+
+ store_u8_4x1(dst + 0 * dst_stride, d01, 0);
+ store_u8_4x1(dst + 1 * dst_stride, d01, 1);
+ store_u8_4x1(dst + 2 * dst_stride, d23, 0);
+ store_u8_4x1(dst + 3 * dst_stride, d23, 1);
+
+ /* Prepare block for next iteration - re-using as much as possible. */
+ /* Shuffle everything up four rows. */
+ s0123 = s4567;
+ s1234 = s5678;
+ s2345 = s6789;
+ s3456 = s78910;
+
+ src += 4 * src_stride;
+ dst += 4 * dst_stride;
+ h -= 4;
+ } while (h != 0);
+ } else {
+ const uint8x16x2_t tran_concat_tbl = vld1q_u8_x2(dot_prod_tran_concat_tbl);
+ int8x16_t s0123_lo, s0123_hi, s1234_lo, s1234_hi, s2345_lo, s2345_hi,
+ s3456_lo, s3456_hi, s4567_lo, s4567_hi, s5678_lo, s5678_hi, s6789_lo,
+ s6789_hi, s78910_lo, s78910_hi;
+ uint8x8_t d0, d1, d2, d3;
+ const uint8_t *s;
+ uint8_t *d;
+ int height;
+
+ do {
+ height = h;
+ s = src;
+ d = dst;
+
+ load_u8_8x7(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6);
+ s += 7 * src_stride;
+
+ /* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
+ s0 = vreinterpret_s8_u8(vsub_u8(t0, range_limit));
+ s1 = vreinterpret_s8_u8(vsub_u8(t1, range_limit));
+ s2 = vreinterpret_s8_u8(vsub_u8(t2, range_limit));
+ s3 = vreinterpret_s8_u8(vsub_u8(t3, range_limit));
+ s4 = vreinterpret_s8_u8(vsub_u8(t4, range_limit));
+ s5 = vreinterpret_s8_u8(vsub_u8(t5, range_limit));
+ s6 = vreinterpret_s8_u8(vsub_u8(t6, range_limit));
+ s7 = vdup_n_s8(0);
+ s8 = vdup_n_s8(0);
+ s9 = vdup_n_s8(0);
+
+ /* This operation combines a conventional transpose and the sample permute
+ * (see horizontal case) required before computing the dot product.
+ */
+ transpose_concat_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi,
+ tran_concat_tbl);
+ transpose_concat_8x4(s1, s2, s3, s4, &s1234_lo, &s1234_hi,
+ tran_concat_tbl);
+ transpose_concat_8x4(s2, s3, s4, s5, &s2345_lo, &s2345_hi,
+ tran_concat_tbl);
+ transpose_concat_8x4(s3, s4, s5, s6, &s3456_lo, &s3456_hi,
+ tran_concat_tbl);
+ transpose_concat_8x4(s4, s5, s6, s7, &s4567_lo, &s4567_hi,
+ tran_concat_tbl);
+ transpose_concat_8x4(s5, s6, s7, s8, &s5678_lo, &s5678_hi,
+ tran_concat_tbl);
+ transpose_concat_8x4(s6, s7, s8, s9, &s6789_lo, &s6789_hi,
+ tran_concat_tbl);
+
+ do {
+ uint8x8_t t7, t8, t9, t10;
+
+ load_u8_8x4(s, src_stride, &t7, &t8, &t9, &t10);
+
+ s7 = vreinterpret_s8_u8(vsub_u8(t7, range_limit));
+ s8 = vreinterpret_s8_u8(vsub_u8(t8, range_limit));
+ s9 = vreinterpret_s8_u8(vsub_u8(t9, range_limit));
+ s10 = vreinterpret_s8_u8(vsub_u8(t10, range_limit));
+
+ transpose_concat_8x4(s7, s8, s9, s10, &s78910_lo, &s78910_hi,
+ tran_concat_tbl);
+
+ /* Merge new data into block from previous iteration. */
+ samples_LUT.val[0] = s3456_lo;
+ samples_LUT.val[1] = s78910_lo;
+ s4567_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]);
+ s5678_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]);
+ s6789_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]);
+
+ samples_LUT.val[0] = s3456_hi;
+ samples_LUT.val[1] = s78910_hi;
+ s4567_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]);
+ s5678_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]);
+ s6789_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]);
+
+ d0 = convolve8_8_sdot_partial(s0123_lo, s4567_lo, s0123_hi, s4567_hi,
+ correction, filter);
+ d1 = convolve8_8_sdot_partial(s1234_lo, s5678_lo, s1234_hi, s5678_hi,
+ correction, filter);
+ d2 = convolve8_8_sdot_partial(s2345_lo, s6789_lo, s2345_hi, s6789_hi,
+ correction, filter);
+ d3 = convolve8_8_sdot_partial(s3456_lo, s78910_lo, s3456_hi, s78910_hi,
+ correction, filter);
+
+ store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ /* Prepare block for next iteration - re-using as much as possible. */
+ /* Shuffle everything up four rows. */
+ s0123_lo = s4567_lo;
+ s0123_hi = s4567_hi;
+ s1234_lo = s5678_lo;
+ s1234_hi = s5678_hi;
+ s2345_lo = s6789_lo;
+ s2345_hi = s6789_hi;
+ s3456_lo = s78910_lo;
+ s3456_hi = s78910_hi;
+
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ height -= 4;
+ } while (height != 0);
+ src += 8;
+ dst += 8;
+ w -= 8;
+ } while (w != 0);
+ }
+}
diff --git a/aom_dsp/arm/aom_convolve8_neon_i8mm.c b/aom_dsp/arm/aom_convolve8_neon_i8mm.c
new file mode 100644
index 0000000..d778e8a
--- /dev/null
+++ b/aom_dsp/arm/aom_convolve8_neon_i8mm.c
@@ -0,0 +1,413 @@
+/*
+ * Copyright (c) 2014 The WebM project authors. All Rights Reserved.
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+#include <assert.h>
+#include <string.h>
+
+#include "config/aom_config.h"
+#include "config/aom_dsp_rtcd.h"
+
+#include "aom/aom_integer.h"
+#include "aom_dsp/aom_dsp_common.h"
+#include "aom_dsp/aom_filter.h"
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_dsp/arm/transpose_neon.h"
+#include "aom_ports/mem.h"
+
+DECLARE_ALIGNED(16, static const uint8_t, dot_prod_permute_tbl[48]) = {
+ 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6,
+ 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10,
+ 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
+};
+
+DECLARE_ALIGNED(16, static const uint8_t, dot_prod_tran_concat_tbl[32]) = {
+ 0, 8, 16, 24, 1, 9, 17, 25, 2, 10, 18, 26, 3, 11, 19, 27,
+ 4, 12, 20, 28, 5, 13, 21, 29, 6, 14, 22, 30, 7, 15, 23, 31
+};
+
+DECLARE_ALIGNED(16, static const uint8_t, dot_prod_merge_block_tbl[48]) = {
+ /* Shift left and insert new last column in transposed 4x4 block. */
+ 1, 2, 3, 16, 5, 6, 7, 20, 9, 10, 11, 24, 13, 14, 15, 28,
+ /* Shift left and insert two new columns in transposed 4x4 block. */
+ 2, 3, 16, 17, 6, 7, 20, 21, 10, 11, 24, 25, 14, 15, 28, 29,
+ /* Shift left and insert three new columns in transposed 4x4 block. */
+ 3, 16, 17, 18, 7, 20, 21, 22, 11, 24, 25, 26, 15, 28, 29, 30
+};
+
+static INLINE int16x4_t convolve8_4_usdot(uint8x16_t samples,
+ const int8x8_t filter,
+ const uint8x16x2_t permute_tbl) {
+ uint8x16_t permuted_samples[2];
+ int32x4_t sum;
+
+ /* Permute samples ready for dot product. */
+ /* { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } */
+ permuted_samples[0] = vqtbl1q_u8(samples, permute_tbl.val[0]);
+ /* { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } */
+ permuted_samples[1] = vqtbl1q_u8(samples, permute_tbl.val[1]);
+
+ /* Accumulate dot product into 'correction' to account for range clamp. */
+ sum = vusdotq_lane_s32(vdupq_n_s32(0), permuted_samples[0], filter, 0);
+ sum = vusdotq_lane_s32(sum, permuted_samples[1], filter, 1);
+
+ /* Further narrowing and packing is performed by the caller. */
+ return vqmovn_s32(sum);
+}
+
+static INLINE uint8x8_t convolve8_8_usdot(uint8x16_t samples,
+ const int8x8_t filter,
+ const uint8x16x3_t permute_tbl) {
+ uint8x16_t permuted_samples[3];
+ int32x4_t sum0, sum1;
+ int16x8_t sum;
+
+ /* Permute samples ready for dot product. */
+ /* { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } */
+ permuted_samples[0] = vqtbl1q_u8(samples, permute_tbl.val[0]);
+ /* { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } */
+ permuted_samples[1] = vqtbl1q_u8(samples, permute_tbl.val[1]);
+ /* { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 } */
+ permuted_samples[2] = vqtbl1q_u8(samples, permute_tbl.val[2]);
+
+ /* First 4 output values. */
+ sum0 = vusdotq_lane_s32(vdupq_n_s32(0), permuted_samples[0], filter, 0);
+ sum0 = vusdotq_lane_s32(sum0, permuted_samples[1], filter, 1);
+ /* Second 4 output values. */
+ sum1 = vusdotq_lane_s32(vdupq_n_s32(0), permuted_samples[1], filter, 0);
+ sum1 = vusdotq_lane_s32(sum1, permuted_samples[2], filter, 1);
+
+ /* Narrow and re-pack. */
+ sum = vcombine_s16(vqmovn_s32(sum0), vqmovn_s32(sum1));
+ return vqrshrun_n_s16(sum, FILTER_BITS);
+}
+
+void aom_convolve8_horiz_neon_i8mm(const uint8_t *src, ptrdiff_t src_stride,
+ uint8_t *dst, ptrdiff_t dst_stride,
+ const int16_t *filter_x, int x_step_q4,
+ const int16_t *filter_y, int y_step_q4,
+ int w, int h) {
+ const int8x8_t filter = vmovn_s16(vld1q_s16(filter_x));
+ uint8x16_t s0, s1, s2, s3;
+
+ assert((intptr_t)dst % 4 == 0);
+ assert(dst_stride % 4 == 0);
+
+ (void)x_step_q4;
+ (void)filter_y;
+ (void)y_step_q4;
+
+ src -= ((SUBPEL_TAPS / 2) - 1);
+
+ if (w == 4) {
+ const uint8x16x2_t perm_tbl = vld1q_u8_x2(dot_prod_permute_tbl);
+ do {
+ int16x4_t t0, t1, t2, t3;
+ uint8x8_t d01, d23;
+
+ load_u8_16x4(src, src_stride, &s0, &s1, &s2, &s3);
+
+ t0 = convolve8_4_usdot(s0, filter, perm_tbl);
+ t1 = convolve8_4_usdot(s1, filter, perm_tbl);
+ t2 = convolve8_4_usdot(s2, filter, perm_tbl);
+ t3 = convolve8_4_usdot(s3, filter, perm_tbl);
+ d01 = vqrshrun_n_s16(vcombine_s16(t0, t1), FILTER_BITS);
+ d23 = vqrshrun_n_s16(vcombine_s16(t2, t3), FILTER_BITS);
+
+ store_u8_4x1(dst + 0 * dst_stride, d01, 0);
+ store_u8_4x1(dst + 1 * dst_stride, d01, 1);
+ store_u8_4x1(dst + 2 * dst_stride, d23, 0);
+ store_u8_4x1(dst + 3 * dst_stride, d23, 1);
+
+ src += 4 * src_stride;
+ dst += 4 * dst_stride;
+ h -= 4;
+ } while (h > 0);
+ } else {
+ const uint8x16x3_t perm_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
+ const uint8_t *s;
+ uint8_t *d;
+ int width;
+ uint8x8_t d0, d1, d2, d3;
+
+ do {
+ width = w;
+ s = src;
+ d = dst;
+ do {
+ load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3);
+
+ d0 = convolve8_8_usdot(s0, filter, perm_tbl);
+ d1 = convolve8_8_usdot(s1, filter, perm_tbl);
+ d2 = convolve8_8_usdot(s2, filter, perm_tbl);
+ d3 = convolve8_8_usdot(s3, filter, perm_tbl);
+
+ store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src += 4 * src_stride;
+ dst += 4 * dst_stride;
+ h -= 4;
+ } while (h > 0);
+ }
+}
+
+static INLINE void transpose_concat_4x4(uint8x8_t a0, uint8x8_t a1,
+ uint8x8_t a2, uint8x8_t a3,
+ uint8x16_t *b,
+ const uint8x16_t permute_tbl) {
+ /* Transpose 8-bit elements and concatenate result rows as follows:
+ * a0: 00, 01, 02, 03, XX, XX, XX, XX
+ * a1: 10, 11, 12, 13, XX, XX, XX, XX
+ * a2: 20, 21, 22, 23, XX, XX, XX, XX
+ * a3: 30, 31, 32, 33, XX, XX, XX, XX
+ *
+ * b: 00, 10, 20, 30, 01, 11, 21, 31, 02, 12, 22, 32, 03, 13, 23, 33
+ *
+ * The 'permute_tbl' is always 'dot_prod_tran_concat_tbl' above. Passing it
+ * as an argument is preferable to loading it directly from memory as this
+ * inline helper is called many times from the same parent function.
+ */
+
+ uint8x16x2_t samples = { { vcombine_u8(a0, a1), vcombine_u8(a2, a3) } };
+ *b = vqtbl2q_u8(samples, permute_tbl);
+}
+
+static INLINE void transpose_concat_8x4(uint8x8_t a0, uint8x8_t a1,
+ uint8x8_t a2, uint8x8_t a3,
+ uint8x16_t *b0, uint8x16_t *b1,
+ const uint8x16x2_t permute_tbl) {
+ /* Transpose 8-bit elements and concatenate result rows as follows:
+ * a0: 00, 01, 02, 03, 04, 05, 06, 07
+ * a1: 10, 11, 12, 13, 14, 15, 16, 17
+ * a2: 20, 21, 22, 23, 24, 25, 26, 27
+ * a3: 30, 31, 32, 33, 34, 35, 36, 37
+ *
+ * b0: 00, 10, 20, 30, 01, 11, 21, 31, 02, 12, 22, 32, 03, 13, 23, 33
+ * b1: 04, 14, 24, 34, 05, 15, 25, 35, 06, 16, 26, 36, 07, 17, 27, 37
+ *
+ * The 'permute_tbl' is always 'dot_prod_tran_concat_tbl' above. Passing it
+ * as an argument is preferable to loading it directly from memory as this
+ * inline helper is called many times from the same parent function.
+ */
+
+ uint8x16x2_t samples = { { vcombine_u8(a0, a1), vcombine_u8(a2, a3) } };
+ *b0 = vqtbl2q_u8(samples, permute_tbl.val[0]);
+ *b1 = vqtbl2q_u8(samples, permute_tbl.val[1]);
+}
+
+static INLINE int16x4_t convolve8_4_usdot_partial(const uint8x16_t samples_lo,
+ const uint8x16_t samples_hi,
+ const int8x8_t filter) {
+ /* Sample permutation is performed by the caller. */
+ int32x4_t sum;
+
+ sum = vusdotq_lane_s32(vdupq_n_s32(0), samples_lo, filter, 0);
+ sum = vusdotq_lane_s32(sum, samples_hi, filter, 1);
+
+ /* Further narrowing and packing is performed by the caller. */
+ return vqmovn_s32(sum);
+}
+
+static INLINE uint8x8_t convolve8_8_usdot_partial(const uint8x16_t samples0_lo,
+ const uint8x16_t samples0_hi,
+ const uint8x16_t samples1_lo,
+ const uint8x16_t samples1_hi,
+ const int8x8_t filter) {
+ /* Sample permutation is performed by the caller. */
+ int32x4_t sum0, sum1;
+ int16x8_t sum;
+
+ /* First 4 output values. */
+ sum0 = vusdotq_lane_s32(vdupq_n_s32(0), samples0_lo, filter, 0);
+ sum0 = vusdotq_lane_s32(sum0, samples0_hi, filter, 1);
+ /* Second 4 output values. */
+ sum1 = vusdotq_lane_s32(vdupq_n_s32(0), samples1_lo, filter, 0);
+ sum1 = vusdotq_lane_s32(sum1, samples1_hi, filter, 1);
+
+ /* Narrow and re-pack. */
+ sum = vcombine_s16(vqmovn_s32(sum0), vqmovn_s32(sum1));
+ return vqrshrun_n_s16(sum, FILTER_BITS);
+}
+
+void aom_convolve8_vert_neon_i8mm(const uint8_t *src, ptrdiff_t src_stride,
+ uint8_t *dst, ptrdiff_t dst_stride,
+ const int16_t *filter_x, int x_step_q4,
+ const int16_t *filter_y, int y_step_q4, int w,
+ int h) {
+ const int8x8_t filter = vmovn_s16(vld1q_s16(filter_y));
+ const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(dot_prod_merge_block_tbl);
+ uint8x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
+ uint8x16x2_t samples_LUT;
+
+ assert((intptr_t)dst % 4 == 0);
+ assert(dst_stride % 4 == 0);
+
+ (void)filter_x;
+ (void)x_step_q4;
+ (void)y_step_q4;
+
+ src -= ((SUBPEL_TAPS / 2) - 1) * src_stride;
+
+ if (w == 4) {
+ const uint8x16_t tran_concat_tbl = vld1q_u8(dot_prod_tran_concat_tbl);
+ uint8x16_t s0123, s1234, s2345, s3456, s4567, s5678, s6789, s78910;
+ int16x4_t d0, d1, d2, d3;
+ uint8x8_t d01, d23;
+
+ load_u8_8x7(src, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
+ src += 7 * src_stride;
+
+ s7 = vdup_n_u8(0);
+ s8 = vdup_n_u8(0);
+ s9 = vdup_n_u8(0);
+
+ /* This operation combines a conventional transpose and the sample permute
+ * (see horizontal case) required before computing the dot product.
+ */
+ transpose_concat_4x4(s0, s1, s2, s3, &s0123, tran_concat_tbl);
+ transpose_concat_4x4(s1, s2, s3, s4, &s1234, tran_concat_tbl);
+ transpose_concat_4x4(s2, s3, s4, s5, &s2345, tran_concat_tbl);
+ transpose_concat_4x4(s3, s4, s5, s6, &s3456, tran_concat_tbl);
+ transpose_concat_4x4(s4, s5, s6, s7, &s4567, tran_concat_tbl);
+ transpose_concat_4x4(s5, s6, s7, s8, &s5678, tran_concat_tbl);
+ transpose_concat_4x4(s6, s7, s8, s9, &s6789, tran_concat_tbl);
+
+ do {
+ load_u8_8x4(src, src_stride, &s7, &s8, &s9, &s10);
+
+ transpose_concat_4x4(s7, s8, s9, s10, &s78910, tran_concat_tbl);
+
+ /* Merge new data into block from previous iteration. */
+ samples_LUT.val[0] = s3456;
+ samples_LUT.val[1] = s78910;
+ s4567 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[0]);
+ s5678 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[1]);
+ s6789 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[2]);
+
+ d0 = convolve8_4_usdot_partial(s0123, s4567, filter);
+ d1 = convolve8_4_usdot_partial(s1234, s5678, filter);
+ d2 = convolve8_4_usdot_partial(s2345, s6789, filter);
+ d3 = convolve8_4_usdot_partial(s3456, s78910, filter);
+ d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS);
+ d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS);
+
+ store_u8_4x1(dst + 0 * dst_stride, d01, 0);
+ store_u8_4x1(dst + 1 * dst_stride, d01, 1);
+ store_u8_4x1(dst + 2 * dst_stride, d23, 0);
+ store_u8_4x1(dst + 3 * dst_stride, d23, 1);
+
+ /* Prepare block for next iteration - re-using as much as possible. */
+ /* Shuffle everything up four rows. */
+ s0123 = s4567;
+ s1234 = s5678;
+ s2345 = s6789;
+ s3456 = s78910;
+
+ src += 4 * src_stride;
+ dst += 4 * dst_stride;
+ h -= 4;
+ } while (h != 0);
+ } else {
+ const uint8x16x2_t tran_concat_tbl = vld1q_u8_x2(dot_prod_tran_concat_tbl);
+ uint8x16_t s0123_lo, s0123_hi, s1234_lo, s1234_hi, s2345_lo, s2345_hi,
+ s3456_lo, s3456_hi, s4567_lo, s4567_hi, s5678_lo, s5678_hi, s6789_lo,
+ s6789_hi, s78910_lo, s78910_hi;
+ uint8x8_t d0, d1, d2, d3;
+ const uint8_t *s;
+ uint8_t *d;
+ int height;
+
+ do {
+ height = h;
+ s = src;
+ d = dst;
+
+ load_u8_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
+ s += 7 * src_stride;
+
+ s7 = vdup_n_u8(0);
+ s8 = vdup_n_u8(0);
+ s9 = vdup_n_u8(0);
+
+ /* This operation combines a conventional transpose and the sample permute
+ * (see horizontal case) required before computing the dot product.
+ */
+ transpose_concat_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi,
+ tran_concat_tbl);
+ transpose_concat_8x4(s1, s2, s3, s4, &s1234_lo, &s1234_hi,
+ tran_concat_tbl);
+ transpose_concat_8x4(s2, s3, s4, s5, &s2345_lo, &s2345_hi,
+ tran_concat_tbl);
+ transpose_concat_8x4(s3, s4, s5, s6, &s3456_lo, &s3456_hi,
+ tran_concat_tbl);
+ transpose_concat_8x4(s4, s5, s6, s7, &s4567_lo, &s4567_hi,
+ tran_concat_tbl);
+ transpose_concat_8x4(s5, s6, s7, s8, &s5678_lo, &s5678_hi,
+ tran_concat_tbl);
+ transpose_concat_8x4(s6, s7, s8, s9, &s6789_lo, &s6789_hi,
+ tran_concat_tbl);
+
+ do {
+ load_u8_8x4(s, src_stride, &s7, &s8, &s9, &s10);
+
+ transpose_concat_8x4(s7, s8, s9, s10, &s78910_lo, &s78910_hi,
+ tran_concat_tbl);
+
+ /* Merge new data into block from previous iteration. */
+ samples_LUT.val[0] = s3456_lo;
+ samples_LUT.val[1] = s78910_lo;
+ s4567_lo = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[0]);
+ s5678_lo = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[1]);
+ s6789_lo = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[2]);
+
+ samples_LUT.val[0] = s3456_hi;
+ samples_LUT.val[1] = s78910_hi;
+ s4567_hi = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[0]);
+ s5678_hi = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[1]);
+ s6789_hi = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[2]);
+
+ d0 = convolve8_8_usdot_partial(s0123_lo, s4567_lo, s0123_hi, s4567_hi,
+ filter);
+ d1 = convolve8_8_usdot_partial(s1234_lo, s5678_lo, s1234_hi, s5678_hi,
+ filter);
+ d2 = convolve8_8_usdot_partial(s2345_lo, s6789_lo, s2345_hi, s6789_hi,
+ filter);
+ d3 = convolve8_8_usdot_partial(s3456_lo, s78910_lo, s3456_hi, s78910_hi,
+ filter);
+
+ store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ /* Prepare block for next iteration - re-using as much as possible. */
+ /* Shuffle everything up four rows. */
+ s0123_lo = s4567_lo;
+ s0123_hi = s4567_hi;
+ s1234_lo = s5678_lo;
+ s1234_hi = s5678_hi;
+ s2345_lo = s6789_lo;
+ s2345_hi = s6789_hi;
+ s3456_lo = s78910_lo;
+ s3456_hi = s78910_hi;
+
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ height -= 4;
+ } while (height != 0);
+ src += 8;
+ dst += 8;
+ w -= 8;
+ } while (w != 0);
+ }
+}
diff --git a/aom_dsp/arm/aom_convolve_copy_neon.c b/aom_dsp/arm/aom_convolve_copy_neon.c
index 583d832..d746f9e 100644
--- a/aom_dsp/arm/aom_convolve_copy_neon.c
+++ b/aom_dsp/arm/aom_convolve_copy_neon.c
@@ -50,3 +50,104 @@
}
}
}
+
+#if CONFIG_AV1_HIGHBITDEPTH
+void aom_highbd_convolve_copy_neon(const uint16_t *src, ptrdiff_t src_stride,
+ uint16_t *dst, ptrdiff_t dst_stride, int w,
+ int h) {
+ if (w < 8) { // copy4
+ uint16x4_t s0, s1;
+ do {
+ s0 = vld1_u16(src);
+ src += src_stride;
+ s1 = vld1_u16(src);
+ src += src_stride;
+
+ vst1_u16(dst, s0);
+ dst += dst_stride;
+ vst1_u16(dst, s1);
+ dst += dst_stride;
+ h -= 2;
+ } while (h != 0);
+ } else if (w == 8) { // copy8
+ uint16x8_t s0, s1;
+ do {
+ s0 = vld1q_u16(src);
+ src += src_stride;
+ s1 = vld1q_u16(src);
+ src += src_stride;
+
+ vst1q_u16(dst, s0);
+ dst += dst_stride;
+ vst1q_u16(dst, s1);
+ dst += dst_stride;
+ h -= 2;
+ } while (h != 0);
+ } else if (w < 32) { // copy16
+ uint16x8_t s0, s1, s2, s3;
+ do {
+ s0 = vld1q_u16(src);
+ s1 = vld1q_u16(src + 8);
+ src += src_stride;
+ s2 = vld1q_u16(src);
+ s3 = vld1q_u16(src + 8);
+ src += src_stride;
+
+ vst1q_u16(dst, s0);
+ vst1q_u16(dst + 8, s1);
+ dst += dst_stride;
+ vst1q_u16(dst, s2);
+ vst1q_u16(dst + 8, s3);
+ dst += dst_stride;
+ h -= 2;
+ } while (h != 0);
+ } else if (w == 32) { // copy32
+ uint16x8_t s0, s1, s2, s3;
+ do {
+ s0 = vld1q_u16(src);
+ s1 = vld1q_u16(src + 8);
+ s2 = vld1q_u16(src + 16);
+ s3 = vld1q_u16(src + 24);
+ src += src_stride;
+
+ vst1q_u16(dst, s0);
+ vst1q_u16(dst + 8, s1);
+ vst1q_u16(dst + 16, s2);
+ vst1q_u16(dst + 24, s3);
+ dst += dst_stride;
+ } while (--h != 0);
+ } else { // copy64
+ uint16x8_t s0, s1, s2, s3, s4, s5, s6, s7;
+ do {
+ const uint16_t *s = src;
+ uint16_t *d = dst;
+ int width = w;
+ do {
+ s0 = vld1q_u16(s);
+ s1 = vld1q_u16(s + 8);
+ s2 = vld1q_u16(s + 16);
+ s3 = vld1q_u16(s + 24);
+ s4 = vld1q_u16(s + 32);
+ s5 = vld1q_u16(s + 40);
+ s6 = vld1q_u16(s + 48);
+ s7 = vld1q_u16(s + 56);
+
+ vst1q_u16(d, s0);
+ vst1q_u16(d + 8, s1);
+ vst1q_u16(d + 16, s2);
+ vst1q_u16(d + 24, s3);
+ vst1q_u16(d + 32, s4);
+ vst1q_u16(d + 40, s5);
+ vst1q_u16(d + 48, s6);
+ vst1q_u16(d + 56, s7);
+ s += 64;
+ d += 64;
+ width -= 64;
+ } while (width > 0);
+ src += src_stride;
+ dst += dst_stride;
+ } while (--h != 0);
+ }
+}
+
+#endif // CONFIG_AV1_HIGHBITDEPTH
diff --git a/aom_dsp/arm/avg_neon.c b/aom_dsp/arm/avg_neon.c
index ef2f3af..2e79b2e 100644
--- a/aom_dsp/arm/avg_neon.c
+++ b/aom_dsp/arm/avg_neon.c
@@ -10,6 +10,7 @@
#include <arm_neon.h>
#include <assert.h>
+#include <stdlib.h>
#include "config/aom_config.h"
#include "config/aom_dsp_rtcd.h"
@@ -19,75 +20,68 @@
#include "aom_dsp/arm/transpose_neon.h"
#include "aom_ports/mem.h"
-#if !AOM_ARCH_AARCH64
-static INLINE uint32x2_t horizontal_add_u16x8_v(const uint16x8_t a) {
- const uint32x4_t b = vpaddlq_u16(a);
- const uint64x2_t c = vpaddlq_u32(b);
- return vadd_u32(vreinterpret_u32_u64(vget_low_u64(c)),
- vreinterpret_u32_u64(vget_high_u64(c)));
-}
-#endif
+unsigned int aom_avg_4x4_neon(const uint8_t *p, int stride) {
+ const uint8x8_t s0 = load_unaligned_u8(p, stride);
+ const uint8x8_t s1 = load_unaligned_u8(p + 2 * stride, stride);
-unsigned int aom_avg_4x4_neon(const uint8_t *a, int a_stride) {
- const uint8x16_t b = load_unaligned_u8q(a, a_stride);
- const uint16x8_t c = vaddl_u8(vget_low_u8(b), vget_high_u8(b));
-#if AOM_ARCH_AARCH64
- const uint32_t d = vaddlvq_u16(c);
- return (d + 8) >> 4;
-#else
- const uint32x2_t d = horizontal_add_u16x8_v(c);
- return vget_lane_u32(vrshr_n_u32(d, 4), 0);
-#endif
+ const uint32_t sum = horizontal_add_u16x8(vaddl_u8(s0, s1));
+ return (sum + (1 << 3)) >> 4;
}
-unsigned int aom_avg_8x8_neon(const uint8_t *a, int a_stride) {
- uint16x8_t sum;
- uint8x8_t b = vld1_u8(a);
- a += a_stride;
- uint8x8_t c = vld1_u8(a);
- a += a_stride;
- sum = vaddl_u8(b, c);
+unsigned int aom_avg_8x8_neon(const uint8_t *p, int stride) {
+ uint8x8_t s0 = vld1_u8(p);
+ p += stride;
+ uint8x8_t s1 = vld1_u8(p);
+ p += stride;
+ uint16x8_t acc = vaddl_u8(s0, s1);
- for (int i = 0; i < 6; ++i) {
- const uint8x8_t e = vld1_u8(a);
- a += a_stride;
- sum = vaddw_u8(sum, e);
- }
+ int i = 0;
+ do {
+ const uint8x8_t si = vld1_u8(p);
+ p += stride;
+ acc = vaddw_u8(acc, si);
+ } while (++i < 6);
-#if AOM_ARCH_AARCH64
- const uint32_t d = vaddlvq_u16(sum);
- return (d + 32) >> 6;
-#else
- const uint32x2_t d = horizontal_add_u16x8_v(sum);
- return vget_lane_u32(vrshr_n_u32(d, 6), 0);
-#endif
+ const uint32_t sum = horizontal_add_u16x8(acc);
+ return (sum + (1 << 5)) >> 6;
}
void aom_avg_8x8_quad_neon(const uint8_t *s, int p, int x16_idx, int y16_idx,
int *avg) {
- for (int k = 0; k < 4; k++) {
- const int x8_idx = x16_idx + ((k & 1) << 3);
- const int y8_idx = y16_idx + ((k >> 1) << 3);
- const uint8_t *s_tmp = s + y8_idx * p + x8_idx;
- avg[k] = aom_avg_8x8_neon(s_tmp, p);
- }
+ avg[0] = aom_avg_8x8_neon(s + y16_idx * p + x16_idx, p);
+ avg[1] = aom_avg_8x8_neon(s + y16_idx * p + (x16_idx + 8), p);
+ avg[2] = aom_avg_8x8_neon(s + (y16_idx + 8) * p + x16_idx, p);
+ avg[3] = aom_avg_8x8_neon(s + (y16_idx + 8) * p + (x16_idx + 8), p);
}
int aom_satd_lp_neon(const int16_t *coeff, int length) {
- const int16x4_t zero = vdup_n_s16(0);
- int32x4_t accum = vdupq_n_s32(0);
+ int16x8_t s0 = vld1q_s16(coeff);
+ int16x8_t s1 = vld1q_s16(coeff + 8);
- do {
- const int16x8_t src0 = vld1q_s16(coeff);
- const int16x8_t src8 = vld1q_s16(coeff + 8);
- accum = vabal_s16(accum, vget_low_s16(src0), zero);
- accum = vabal_s16(accum, vget_high_s16(src0), zero);
- accum = vabal_s16(accum, vget_low_s16(src8), zero);
- accum = vabal_s16(accum, vget_high_s16(src8), zero);
+ int16x8_t abs0 = vabsq_s16(s0);
+ int16x8_t abs1 = vabsq_s16(s1);
+
+ int32x4_t acc0 = vpaddlq_s16(abs0);
+ int32x4_t acc1 = vpaddlq_s16(abs1);
+
+ length -= 16;
+ coeff += 16;
+
+ while (length != 0) {
+ s0 = vld1q_s16(coeff);
+ s1 = vld1q_s16(coeff + 8);
+
+ abs0 = vabsq_s16(s0);
+ abs1 = vabsq_s16(s1);
+
+ acc0 = vpadalq_s16(acc0, abs0);
+ acc1 = vpadalq_s16(acc1, abs1);
+
length -= 16;
coeff += 16;
- } while (length != 0);
+ }
+ int32x4_t accum = vaddq_s32(acc0, acc1);
return horizontal_add_s32x4(accum);
}
@@ -180,56 +174,84 @@
} while (h < height);
}
-// coeff: 16 bits, dynamic range [-32640, 32640].
-// length: value range {16, 64, 256, 1024}.
+// coeff: 20 bits, dynamic range [-524287, 524287].
+// length: value range {16, 32, 64, 128, 256, 512, 1024}.
int aom_satd_neon(const tran_low_t *coeff, int length) {
const int32x4_t zero = vdupq_n_s32(0);
- int32x4_t accum = zero;
- do {
- const int32x4_t src0 = vld1q_s32(&coeff[0]);
- const int32x4_t src8 = vld1q_s32(&coeff[4]);
- const int32x4_t src16 = vld1q_s32(&coeff[8]);
- const int32x4_t src24 = vld1q_s32(&coeff[12]);
- accum = vabaq_s32(accum, src0, zero);
- accum = vabaq_s32(accum, src8, zero);
- accum = vabaq_s32(accum, src16, zero);
- accum = vabaq_s32(accum, src24, zero);
+
+ int32x4_t s0 = vld1q_s32(&coeff[0]);
+ int32x4_t s1 = vld1q_s32(&coeff[4]);
+ int32x4_t s2 = vld1q_s32(&coeff[8]);
+ int32x4_t s3 = vld1q_s32(&coeff[12]);
+
+ int32x4_t accum0 = vabsq_s32(s0);
+ int32x4_t accum1 = vabsq_s32(s2);
+ accum0 = vabaq_s32(accum0, s1, zero);
+ accum1 = vabaq_s32(accum1, s3, zero);
+
+ length -= 16;
+ coeff += 16;
+
+ while (length != 0) {
+ s0 = vld1q_s32(&coeff[0]);
+ s1 = vld1q_s32(&coeff[4]);
+ s2 = vld1q_s32(&coeff[8]);
+ s3 = vld1q_s32(&coeff[12]);
+
+ accum0 = vabaq_s32(accum0, s0, zero);
+ accum1 = vabaq_s32(accum1, s1, zero);
+ accum0 = vabaq_s32(accum0, s2, zero);
+ accum1 = vabaq_s32(accum1, s3, zero);
+
length -= 16;
coeff += 16;
- } while (length != 0);
+ }
- // satd: 26 bits, dynamic range [-32640 * 1024, 32640 * 1024]
- return horizontal_add_s32x4(accum);
+ // satd: 30 bits, dynamic range [-524287 * 1024, 524287 * 1024]
+ return horizontal_add_s32x4(vaddq_s32(accum0, accum1));
}
int aom_vector_var_neon(const int16_t *ref, const int16_t *src, int bwl) {
- int32x4_t v_mean = vdupq_n_s32(0);
- int32x4_t v_sse = v_mean;
- int16x8_t v_ref, v_src;
- int16x4_t v_low;
+ assert(bwl >= 2 && bwl <= 5);
+ int width = 4 << bwl;
- int i, width = 4 << bwl;
- for (i = 0; i < width; i += 8) {
- v_ref = vld1q_s16(&ref[i]);
- v_src = vld1q_s16(&src[i]);
- const int16x8_t diff = vsubq_s16(v_ref, v_src);
- // diff: dynamic range [-510, 510], 10 bits.
- v_mean = vpadalq_s16(v_mean, diff);
- v_low = vget_low_s16(diff);
- v_sse = vmlal_s16(v_sse, v_low, v_low);
-#if AOM_ARCH_AARCH64
- v_sse = vmlal_high_s16(v_sse, diff, diff);
-#else
- const int16x4_t v_high = vget_high_s16(diff);
- v_sse = vmlal_s16(v_sse, v_high, v_high);
-#endif
- }
- const int mean = horizontal_add_s32x4(v_mean);
- const int sse = horizontal_add_s32x4(v_sse);
- const unsigned int mean_abs = mean >= 0 ? mean : -mean;
- // (mean * mean): dynamic range 31 bits.
- const int var = sse - ((mean_abs * mean_abs) >> (bwl + 2));
- return var;
+ int16x8_t r = vld1q_s16(ref);
+ int16x8_t s = vld1q_s16(src);
+
+ // diff: dynamic range [-510, 510] 10 (signed) bits.
+ int16x8_t diff = vsubq_s16(r, s);
+ // v_mean: dynamic range 16 * diff -> [-8160, 8160], 14 (signed) bits.
+ int16x8_t v_mean = diff;
+ // v_sse: dynamic range 2 * 16 * diff^2 -> [0, 8,323,200], 24 (signed) bits.
+ int32x4_t v_sse[2];
+ v_sse[0] = vmull_s16(vget_low_s16(diff), vget_low_s16(diff));
+ v_sse[1] = vmull_s16(vget_high_s16(diff), vget_high_s16(diff));
+
+ ref += 8;
+ src += 8;
+ width -= 8;
+
+ do {
+ r = vld1q_s16(ref);
+ s = vld1q_s16(src);
+
+ diff = vsubq_s16(r, s);
+ v_mean = vaddq_s16(v_mean, diff);
+
+ v_sse[0] = vmlal_s16(v_sse[0], vget_low_s16(diff), vget_low_s16(diff));
+ v_sse[1] = vmlal_s16(v_sse[1], vget_high_s16(diff), vget_high_s16(diff));
+
+ ref += 8;
+ src += 8;
+ width -= 8;
+ } while (width != 0);
+
+ // Dynamic range [0, 65280], 16 (unsigned) bits.
+ const uint32_t mean_abs = abs(horizontal_add_s16x8(v_mean));
+ const int32_t sse = horizontal_add_s32x4(vaddq_s32(v_sse[0], v_sse[1]));
+
+ // (mean_abs * mean_abs): dynamic range 32 (unsigned) bits.
+ return sse - ((mean_abs * mean_abs) >> (bwl + 2));
}
void aom_minmax_8x8_neon(const uint8_t *a, int a_stride, const uint8_t *b,
diff --git a/aom_dsp/arm/avg_pred_neon.c b/aom_dsp/arm/avg_pred_neon.c
index 04e0904..b17f7fc 100644
--- a/aom_dsp/arm/avg_pred_neon.c
+++ b/aom_dsp/arm/avg_pred_neon.c
@@ -13,6 +13,9 @@
#include <assert.h>
#include "config/aom_dsp_rtcd.h"
+
+#include "aom_dsp/arm/blend_neon.h"
+#include "aom_dsp/arm/dist_wtd_avg_neon.h"
#include "aom_dsp/arm/mem_neon.h"
#include "aom_dsp/blend.h"
@@ -74,6 +77,75 @@
}
}
+void aom_dist_wtd_comp_avg_pred_neon(uint8_t *comp_pred, const uint8_t *pred,
+ int width, int height, const uint8_t *ref,
+ int ref_stride,
+ const DIST_WTD_COMP_PARAMS *jcp_param) {
+ const uint8x16_t fwd_offset = vdupq_n_u8(jcp_param->fwd_offset);
+ const uint8x16_t bck_offset = vdupq_n_u8(jcp_param->bck_offset);
+
+ if (width > 8) {
+ do {
+ const uint8_t *pred_ptr = pred;
+ const uint8_t *ref_ptr = ref;
+ uint8_t *comp_pred_ptr = comp_pred;
+ int w = width;
+
+ do {
+ const uint8x16_t p = vld1q_u8(pred_ptr);
+ const uint8x16_t r = vld1q_u8(ref_ptr);
+
+ const uint8x16_t wtd_avg =
+ dist_wtd_avg_u8x16(r, p, fwd_offset, bck_offset);
+
+ vst1q_u8(comp_pred_ptr, wtd_avg);
+
+ ref_ptr += 16;
+ pred_ptr += 16;
+ comp_pred_ptr += 16;
+ w -= 16;
+ } while (w != 0);
+
+ ref += ref_stride;
+ pred += width;
+ comp_pred += width;
+ } while (--height != 0);
+ } else if (width == 8) {
+ int h = height / 2;
+
+ do {
+ const uint8x16_t p = vld1q_u8(pred);
+ const uint8x16_t r = load_u8_8x2(ref, ref_stride);
+
+ const uint8x16_t wtd_avg =
+ dist_wtd_avg_u8x16(r, p, fwd_offset, bck_offset);
+
+ vst1q_u8(comp_pred, wtd_avg);
+
+ ref += 2 * ref_stride;
+ pred += 16;
+ comp_pred += 16;
+ } while (--h != 0);
+ } else {
+ int h = height / 2;
+ assert(width == 4);
+
+ do {
+ const uint8x8_t p = vld1_u8(pred);
+ const uint8x8_t r = load_unaligned_u8_4x2(ref, ref_stride);
+
+ const uint8x8_t wtd_avg = dist_wtd_avg_u8x8(r, p, vget_low_u8(fwd_offset),
+ vget_low_u8(bck_offset));
+
+ vst1_u8(comp_pred, wtd_avg);
+
+ ref += 2 * ref_stride;
+ pred += 8;
+ comp_pred += 8;
+ } while (--h != 0);
+ }
+}
+
void aom_comp_mask_pred_neon(uint8_t *comp_pred, const uint8_t *pred, int width,
int height, const uint8_t *ref, int ref_stride,
const uint8_t *mask, int mask_stride,
@@ -84,7 +156,6 @@
const int src_stride1 = invert_mask ? ref_stride : width;
if (width > 8) {
- const uint8x16_t max_alpha = vdupq_n_u8(AOM_BLEND_A64_MAX_ALPHA);
do {
const uint8_t *src0_ptr = src0;
const uint8_t *src1_ptr = src1;
@@ -97,19 +168,7 @@
const uint8x16_t s1 = vld1q_u8(src1_ptr);
const uint8x16_t m0 = vld1q_u8(mask_ptr);
- uint8x16_t m0_inv = vsubq_u8(max_alpha, m0);
- uint16x8_t blend_u16_lo = vmull_u8(vget_low_u8(s0), vget_low_u8(m0));
- uint16x8_t blend_u16_hi = vmull_u8(vget_high_u8(s0), vget_high_u8(m0));
- blend_u16_lo =
- vmlal_u8(blend_u16_lo, vget_low_u8(s1), vget_low_u8(m0_inv));
- blend_u16_hi =
- vmlal_u8(blend_u16_hi, vget_high_u8(s1), vget_high_u8(m0_inv));
-
- uint8x8_t blend_u8_lo =
- vrshrn_n_u16(blend_u16_lo, AOM_BLEND_A64_ROUND_BITS);
- uint8x8_t blend_u8_hi =
- vrshrn_n_u16(blend_u16_hi, AOM_BLEND_A64_ROUND_BITS);
- uint8x16_t blend_u8 = vcombine_u8(blend_u8_lo, blend_u8_hi);
+ uint8x16_t blend_u8 = alpha_blend_a64_u8x16(m0, s0, s1);
vst1q_u8(comp_pred_ptr, blend_u8);
@@ -126,17 +185,12 @@
comp_pred += width;
} while (--height != 0);
} else if (width == 8) {
- const uint8x8_t max_alpha = vdup_n_u8(AOM_BLEND_A64_MAX_ALPHA);
-
do {
const uint8x8_t s0 = vld1_u8(src0);
const uint8x8_t s1 = vld1_u8(src1);
const uint8x8_t m0 = vld1_u8(mask);
- uint8x8_t m0_inv = vsub_u8(max_alpha, m0);
- uint16x8_t blend_u16 = vmull_u8(s0, m0);
- blend_u16 = vmlal_u8(blend_u16, s1, m0_inv);
- uint8x8_t blend_u8 = vrshrn_n_u16(blend_u16, AOM_BLEND_A64_ROUND_BITS);
+ uint8x8_t blend_u8 = alpha_blend_a64_u8x8(m0, s0, s1);
vst1_u8(comp_pred, blend_u8);
@@ -146,7 +200,6 @@
comp_pred += 8;
} while (--height != 0);
} else {
- const uint8x8_t max_alpha = vdup_n_u8(AOM_BLEND_A64_MAX_ALPHA);
int h = height / 2;
assert(width == 4);
@@ -155,10 +208,7 @@
const uint8x8_t s1 = load_unaligned_u8(src1, src_stride1);
const uint8x8_t m0 = load_unaligned_u8(mask, mask_stride);
- uint8x8_t m0_inv = vsub_u8(max_alpha, m0);
- uint16x8_t blend_u16 = vmull_u8(s0, m0);
- blend_u16 = vmlal_u8(blend_u16, s1, m0_inv);
- uint8x8_t blend_u8 = vrshrn_n_u16(blend_u16, AOM_BLEND_A64_ROUND_BITS);
+ uint8x8_t blend_u8 = alpha_blend_a64_u8x8(m0, s0, s1);
vst1_u8(comp_pred, blend_u8);
diff --git a/aom_dsp/arm/blend_a64_mask_neon.c b/aom_dsp/arm/blend_a64_mask_neon.c
index c3ee0b7..7b1b66a 100644
--- a/aom_dsp/arm/blend_a64_mask_neon.c
+++ b/aom_dsp/arm/blend_a64_mask_neon.c
@@ -12,117 +12,34 @@
#include <arm_neon.h>
#include <assert.h>
-#include "aom/aom_integer.h"
-#include "aom_dsp/aom_dsp_common.h"
-#include "aom_dsp/blend.h"
-#include "aom_dsp/arm/mem_neon.h"
-#include "aom_ports/mem.h"
#include "config/aom_dsp_rtcd.h"
-static INLINE void blend8x1(int16x8_t mask, int16x8_t src_0, int16x8_t src_1,
- const int16x8_t v_maxval, int16x8_t *res) {
- int32x4_t im_res_low, im_res_high;
- const int16x8_t max_minus_mask = vsubq_s16(v_maxval, mask);
+#include "aom/aom_integer.h"
+#include "aom_dsp/aom_dsp_common.h"
+#include "aom_dsp/arm/blend_neon.h"
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_dsp/blend.h"
- im_res_low = vmull_s16(vget_low_s16(mask), vget_low_s16(src_0));
- im_res_low =
- vmlal_s16(im_res_low, vget_low_s16(max_minus_mask), vget_low_s16(src_1));
+uint8x8_t alpha_blend_a64_d16_u16x8(uint16x8_t m, uint16x8_t a, uint16x8_t b,
+ uint16x8_t round_offset) {
+ const uint16x8_t m_inv = vsubq_u16(vdupq_n_u16(AOM_BLEND_A64_MAX_ALPHA), m);
- im_res_high = vmull_s16(vget_high_s16(mask), vget_high_s16(src_0));
- im_res_high = vmlal_s16(im_res_high, vget_high_s16(max_minus_mask),
- vget_high_s16(src_1));
+ uint32x4_t blend_u32_lo = vmull_u16(vget_low_u16(m), vget_low_u16(a));
+ uint32x4_t blend_u32_hi = vmull_u16(vget_high_u16(m), vget_high_u16(a));
- *res = vcombine_s16(vshrn_n_s32(im_res_low, AOM_BLEND_A64_ROUND_BITS),
- vshrn_n_s32(im_res_high, AOM_BLEND_A64_ROUND_BITS));
-}
+ blend_u32_lo = vmlal_u16(blend_u32_lo, vget_low_u16(m_inv), vget_low_u16(b));
+ blend_u32_hi =
+ vmlal_u16(blend_u32_hi, vget_high_u16(m_inv), vget_high_u16(b));
-static INLINE void blend_8x4(uint8_t *dst, uint32_t dst_stride,
- const CONV_BUF_TYPE *src0, uint32_t src0_stride,
- const CONV_BUF_TYPE *src1, uint32_t src1_stride,
- int16x8_t mask0, int16x8_t mask1, int16x8_t mask2,
- int16x8_t mask3, const int16x8_t v_maxval,
- const uint16x8_t vec_round_offset,
- const int16x8_t vec_round_bits) {
- int16x8_t src0_0, src0_1, src0_2, src0_3;
- int16x8_t src1_0, src1_1, src1_2, src1_3;
- int16x8_t im_res_0, im_res_1, im_res_2, im_res_3;
+ uint16x4_t blend_u16_lo = vshrn_n_u32(blend_u32_lo, AOM_BLEND_A64_ROUND_BITS);
+ uint16x4_t blend_u16_hi = vshrn_n_u32(blend_u32_hi, AOM_BLEND_A64_ROUND_BITS);
- load_s16_8x4((int16_t *)src0, (int32_t)src0_stride, &src0_0, &src0_1, &src0_2,
- &src0_3);
- load_s16_8x4((int16_t *)src1, (int32_t)src1_stride, &src1_0, &src1_1, &src1_2,
- &src1_3);
+ uint16x8_t res = vcombine_u16(blend_u16_lo, blend_u16_hi);
- blend8x1(mask0, src0_0, src1_0, v_maxval, &im_res_0);
- blend8x1(mask1, src0_1, src1_1, v_maxval, &im_res_1);
- blend8x1(mask2, src0_2, src1_2, v_maxval, &im_res_2);
- blend8x1(mask3, src0_3, src1_3, v_maxval, &im_res_3);
+ res = vqsubq_u16(res, round_offset);
- uint16x8_t im_res1_0 =
- vqsubq_u16(vreinterpretq_u16_s16(im_res_0), vec_round_offset);
- uint16x8_t im_res1_1 =
- vqsubq_u16(vreinterpretq_u16_s16(im_res_1), vec_round_offset);
- uint16x8_t im_res1_2 =
- vqsubq_u16(vreinterpretq_u16_s16(im_res_2), vec_round_offset);
- uint16x8_t im_res1_3 =
- vqsubq_u16(vreinterpretq_u16_s16(im_res_3), vec_round_offset);
-
- im_res_0 = vshlq_s16(vreinterpretq_s16_u16(im_res1_0), vec_round_bits);
- im_res_1 = vshlq_s16(vreinterpretq_s16_u16(im_res1_1), vec_round_bits);
- im_res_2 = vshlq_s16(vreinterpretq_s16_u16(im_res1_2), vec_round_bits);
- im_res_3 = vshlq_s16(vreinterpretq_s16_u16(im_res1_3), vec_round_bits);
-
- vst1_u8((dst + 0 * dst_stride), vqmovun_s16(im_res_0));
- vst1_u8((dst + 1 * dst_stride), vqmovun_s16(im_res_1));
- vst1_u8((dst + 2 * dst_stride), vqmovun_s16(im_res_2));
- vst1_u8((dst + 3 * dst_stride), vqmovun_s16(im_res_3));
-}
-
-static INLINE void blend_4x4(uint8_t *dst, uint32_t dst_stride,
- const CONV_BUF_TYPE *src0, uint32_t src0_stride,
- const CONV_BUF_TYPE *src1, uint32_t src1_stride,
- int16x4_t mask0, int16x4_t mask1, int16x4_t mask2,
- int16x4_t mask3, const int16x8_t v_maxval,
- const uint16x8_t vec_round_offset,
- const int16x8_t vec_round_bits) {
- int16x8_t src0_0, src0_1;
- int16x8_t src1_0, src1_1;
- uint16x8_t tu0 = vdupq_n_u16(0);
- uint16x8_t tu1 = vdupq_n_u16(0);
- uint16x8_t tu2 = vdupq_n_u16(0);
- uint16x8_t tu3 = vdupq_n_u16(0);
- int16x8_t mask0_1, mask2_3;
- int16x8_t res0, res1;
-
- load_unaligned_u16_4x4(src0, src0_stride, &tu0, &tu1);
- load_unaligned_u16_4x4(src1, src1_stride, &tu2, &tu3);
-
- src0_0 = vreinterpretq_s16_u16(tu0);
- src0_1 = vreinterpretq_s16_u16(tu1);
-
- src1_0 = vreinterpretq_s16_u16(tu2);
- src1_1 = vreinterpretq_s16_u16(tu3);
-
- mask0_1 = vcombine_s16(mask0, mask1);
- mask2_3 = vcombine_s16(mask2, mask3);
-
- blend8x1(mask0_1, src0_0, src1_0, v_maxval, &res0);
- blend8x1(mask2_3, src0_1, src1_1, v_maxval, &res1);
-
- uint16x8_t im_res_0 =
- vqsubq_u16(vreinterpretq_u16_s16(res0), vec_round_offset);
- uint16x8_t im_res_1 =
- vqsubq_u16(vreinterpretq_u16_s16(res1), vec_round_offset);
-
- src0_0 = vshlq_s16(vreinterpretq_s16_u16(im_res_0), vec_round_bits);
- src0_1 = vshlq_s16(vreinterpretq_s16_u16(im_res_1), vec_round_bits);
-
- uint8x8_t res_0 = vqmovun_s16(src0_0);
- uint8x8_t res_1 = vqmovun_s16(src0_1);
-
- store_unaligned_u8_4x1(dst + 0 * dst_stride, res_0, 0);
- store_unaligned_u8_4x1(dst + 1 * dst_stride, res_0, 1);
- store_unaligned_u8_4x1(dst + 2 * dst_stride, res_1, 0);
- store_unaligned_u8_4x1(dst + 3 * dst_stride, res_1, 1);
+ return vqrshrn_n_u16(res,
+ 2 * FILTER_BITS - ROUND0_BITS - COMPOUND_ROUND1_BITS);
}
void aom_lowbd_blend_a64_d16_mask_neon(
@@ -130,19 +47,13 @@
uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride,
const uint8_t *mask, uint32_t mask_stride, int w, int h, int subw, int subh,
ConvolveParams *conv_params) {
- int i = 0;
- const int bd = 8;
- int w_tmp = w;
- const uint8_t *mask_tmp = mask;
- const CONV_BUF_TYPE *src0_tmp = src0;
- const CONV_BUF_TYPE *src1_tmp = src1;
- uint8_t *dst_tmp = dst;
+ (void)conv_params;
- const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0;
- const int round_offset = (1 << (offset_bits - conv_params->round_1)) +
- (1 << (offset_bits - conv_params->round_1 - 1));
- const int round_bits =
- 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1;
+ const int bd = 8;
+ const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
+ const int round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
+ (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
+ const uint16x8_t offset_vec = vdupq_n_u16(round_offset);
assert(IMPLIES((void *)src0 == dst, src0_stride == dst_stride));
assert(IMPLIES((void *)src1 == dst, src1_stride == dst_stride));
@@ -152,294 +63,430 @@
assert(IS_POWER_OF_TWO(h));
assert(IS_POWER_OF_TWO(w));
- uint8x8_t s0 = vdup_n_u8(0);
- uint8x8_t s1 = vdup_n_u8(0);
- uint8x8_t s2 = vdup_n_u8(0);
- uint8x8_t s3 = vdup_n_u8(0);
- uint8x16_t t0, t1, t2, t3, t4, t5, t6, t7;
- int16x8_t mask0, mask1, mask2, mask3;
- int16x8_t mask4, mask5, mask6, mask7;
- int32x4_t m0_32, m1_32, m2_32, m3_32;
- int32x4_t m4_32, m5_32, m6_32, m7_32;
- uint8x8_t mask0_l, mask1_l, mask2_l, mask3_l;
- uint8x8_t mask4_l, mask5_l, mask6_l, mask7_l;
- int16x4_t mask0_low, mask1_low, mask2_low, mask3_low;
- const uint16x4_t vec_zero = vdup_n_u16(0);
- const uint16_t offset = round_offset - (1 << (round_bits - 1));
- const int16x8_t v_maxval = vdupq_n_s16(AOM_BLEND_A64_MAX_ALPHA);
- const int16x8_t vec_round_bits = vdupq_n_s16(-round_bits);
- const uint16x8_t vec_offset = vdupq_n_u16(offset);
-
if (subw == 0 && subh == 0) {
- if (w_tmp > 7) {
+ if (w >= 8) {
do {
- w_tmp = w;
+ int i = 0;
do {
- load_u8_8x4(mask_tmp, mask_stride, &s0, &s1, &s2, &s3);
+ uint16x8_t m0 = vmovl_u8(vld1_u8(mask + i));
+ uint16x8_t s0 = vld1q_u16(src0 + i);
+ uint16x8_t s1 = vld1q_u16(src1 + i);
- mask0 = vmovl_s8(vreinterpret_s8_u8(s0));
- mask1 = vmovl_s8(vreinterpret_s8_u8(s1));
- mask2 = vmovl_s8(vreinterpret_s8_u8(s2));
- mask3 = vmovl_s8(vreinterpret_s8_u8(s3));
+ uint8x8_t blend = alpha_blend_a64_d16_u16x8(m0, s0, s1, offset_vec);
- blend_8x4(dst_tmp, dst_stride, src0_tmp, src0_stride, src1_tmp,
- src1_stride, mask0, mask1, mask2, mask3, v_maxval,
- vec_offset, vec_round_bits);
+ vst1_u8(dst + i, blend);
+ i += 8;
+ } while (i < w);
- w_tmp -= 8;
- mask_tmp += 8;
- dst_tmp += 8;
- src0_tmp += 8;
- src1_tmp += 8;
- } while (w_tmp > 7);
- i += 4;
- mask_tmp += (4 * mask_stride) - w;
- dst_tmp += (4 * dst_stride) - w;
- src0_tmp += (4 * src0_stride) - w;
- src1_tmp += (4 * src1_stride) - w;
- } while (i < h);
+ mask += mask_stride;
+ src0 += src0_stride;
+ src1 += src1_stride;
+ dst += dst_stride;
+ } while (--h != 0);
} else {
do {
- load_unaligned_u8_4x4(mask_tmp, mask_stride, &s0, &s1);
+ uint16x8_t m0 = vmovl_u8(load_unaligned_u8_4x2(mask, mask_stride));
+ uint16x8_t s0 = load_unaligned_u16_4x2(src0, src0_stride);
+ uint16x8_t s1 = load_unaligned_u16_4x2(src1, src1_stride);
- mask0 = vreinterpretq_s16_u16(vmovl_u8(s0));
- mask1 = vreinterpretq_s16_u16(vmovl_u8(s1));
+ uint8x8_t blend = alpha_blend_a64_d16_u16x8(m0, s0, s1, offset_vec);
- mask0_low = vget_low_s16(mask0);
- mask1_low = vget_high_s16(mask0);
- mask2_low = vget_low_s16(mask1);
- mask3_low = vget_high_s16(mask1);
+ store_unaligned_u8_4x2(dst, dst_stride, blend);
- blend_4x4(dst_tmp, dst_stride, src0_tmp, src0_stride, src1_tmp,
- src1_stride, mask0_low, mask1_low, mask2_low, mask3_low,
- v_maxval, vec_offset, vec_round_bits);
-
- i += 4;
- mask_tmp += (4 * mask_stride);
- dst_tmp += (4 * dst_stride);
- src0_tmp += (4 * src0_stride);
- src1_tmp += (4 * src1_stride);
- } while (i < h);
+ mask += 2 * mask_stride;
+ src0 += 2 * src0_stride;
+ src1 += 2 * src1_stride;
+ dst += 2 * dst_stride;
+ h -= 2;
+ } while (h != 0);
}
} else if (subw == 1 && subh == 1) {
- if (w_tmp > 7) {
+ if (w >= 8) {
do {
- w_tmp = w;
+ int i = 0;
do {
- load_u8_16x8(mask_tmp, mask_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6,
- &t7);
+ uint8x8_t m0 = vld1_u8(mask + 0 * mask_stride + 2 * i);
+ uint8x8_t m1 = vld1_u8(mask + 1 * mask_stride + 2 * i);
+ uint8x8_t m2 = vld1_u8(mask + 0 * mask_stride + 2 * i + 8);
+ uint8x8_t m3 = vld1_u8(mask + 1 * mask_stride + 2 * i + 8);
+ uint16x8_t s0 = vld1q_u16(src0 + i);
+ uint16x8_t s1 = vld1q_u16(src1 + i);
- mask0 =
- vreinterpretq_s16_u16(vaddl_u8(vget_low_u8(t0), vget_low_u8(t1)));
- mask1 =
- vreinterpretq_s16_u16(vaddl_u8(vget_low_u8(t2), vget_low_u8(t3)));
- mask2 =
- vreinterpretq_s16_u16(vaddl_u8(vget_low_u8(t4), vget_low_u8(t5)));
- mask3 =
- vreinterpretq_s16_u16(vaddl_u8(vget_low_u8(t6), vget_low_u8(t7)));
+ uint16x8_t m_avg =
+ vmovl_u8(avg_blend_pairwise_u8x8_4(m0, m1, m2, m3));
- mask4 = vreinterpretq_s16_u16(
- vaddl_u8(vget_high_u8(t0), vget_high_u8(t1)));
- mask5 = vreinterpretq_s16_u16(
- vaddl_u8(vget_high_u8(t2), vget_high_u8(t3)));
- mask6 = vreinterpretq_s16_u16(
- vaddl_u8(vget_high_u8(t4), vget_high_u8(t5)));
- mask7 = vreinterpretq_s16_u16(
- vaddl_u8(vget_high_u8(t6), vget_high_u8(t7)));
+ uint8x8_t blend =
+ alpha_blend_a64_d16_u16x8(m_avg, s0, s1, offset_vec);
- m0_32 = vpaddlq_s16(mask0);
- m1_32 = vpaddlq_s16(mask1);
- m2_32 = vpaddlq_s16(mask2);
- m3_32 = vpaddlq_s16(mask3);
+ vst1_u8(dst + i, blend);
+ i += 8;
+ } while (i < w);
- m4_32 = vpaddlq_s16(mask4);
- m5_32 = vpaddlq_s16(mask5);
- m6_32 = vpaddlq_s16(mask6);
- m7_32 = vpaddlq_s16(mask7);
-
- mask0 =
- vcombine_s16(vqrshrn_n_s32(m0_32, 2), vqrshrn_n_s32(m4_32, 2));
- mask1 =
- vcombine_s16(vqrshrn_n_s32(m1_32, 2), vqrshrn_n_s32(m5_32, 2));
- mask2 =
- vcombine_s16(vqrshrn_n_s32(m2_32, 2), vqrshrn_n_s32(m6_32, 2));
- mask3 =
- vcombine_s16(vqrshrn_n_s32(m3_32, 2), vqrshrn_n_s32(m7_32, 2));
-
- blend_8x4(dst_tmp, dst_stride, src0_tmp, src0_stride, src1_tmp,
- src1_stride, mask0, mask1, mask2, mask3, v_maxval,
- vec_offset, vec_round_bits);
-
- w_tmp -= 8;
- mask_tmp += 16;
- dst_tmp += 8;
- src0_tmp += 8;
- src1_tmp += 8;
- } while (w_tmp > 7);
- i += 4;
- mask_tmp += (8 * mask_stride) - (2 * w);
- dst_tmp += (4 * dst_stride) - w;
- src0_tmp += (4 * src0_stride) - w;
- src1_tmp += (4 * src1_stride) - w;
- } while (i < h);
+ mask += 2 * mask_stride;
+ src0 += src0_stride;
+ src1 += src1_stride;
+ dst += dst_stride;
+ } while (--h != 0);
} else {
do {
- load_u8_8x8(mask_tmp, mask_stride, &mask0_l, &mask1_l, &mask2_l,
- &mask3_l, &mask4_l, &mask5_l, &mask6_l, &mask7_l);
+ uint8x8_t m0 = vld1_u8(mask + 0 * mask_stride);
+ uint8x8_t m1 = vld1_u8(mask + 1 * mask_stride);
+ uint8x8_t m2 = vld1_u8(mask + 2 * mask_stride);
+ uint8x8_t m3 = vld1_u8(mask + 3 * mask_stride);
+ uint16x8_t s0 = load_unaligned_u16_4x2(src0, src0_stride);
+ uint16x8_t s1 = load_unaligned_u16_4x2(src1, src1_stride);
- mask0 = vreinterpretq_s16_u16(vaddl_u8(mask0_l, mask1_l));
- mask1 = vreinterpretq_s16_u16(vaddl_u8(mask2_l, mask3_l));
- mask2 = vreinterpretq_s16_u16(vaddl_u8(mask4_l, mask5_l));
- mask3 = vreinterpretq_s16_u16(vaddl_u8(mask6_l, mask7_l));
+ uint16x8_t m_avg = vmovl_u8(avg_blend_pairwise_u8x8_4(m0, m1, m2, m3));
+ uint8x8_t blend = alpha_blend_a64_d16_u16x8(m_avg, s0, s1, offset_vec);
- m0_32 = vpaddlq_s16(mask0);
- m1_32 = vpaddlq_s16(mask1);
- m2_32 = vpaddlq_s16(mask2);
- m3_32 = vpaddlq_s16(mask3);
+ store_unaligned_u8_4x2(dst, dst_stride, blend);
- mask0_low = vqrshrn_n_s32(m0_32, 2);
- mask1_low = vqrshrn_n_s32(m1_32, 2);
- mask2_low = vqrshrn_n_s32(m2_32, 2);
- mask3_low = vqrshrn_n_s32(m3_32, 2);
-
- blend_4x4(dst_tmp, dst_stride, src0_tmp, src0_stride, src1_tmp,
- src1_stride, mask0_low, mask1_low, mask2_low, mask3_low,
- v_maxval, vec_offset, vec_round_bits);
-
- i += 4;
- mask_tmp += (8 * mask_stride);
- dst_tmp += (4 * dst_stride);
- src0_tmp += (4 * src0_stride);
- src1_tmp += (4 * src1_stride);
- } while (i < h);
+ mask += 4 * mask_stride;
+ src0 += 2 * src0_stride;
+ src1 += 2 * src1_stride;
+ dst += 2 * dst_stride;
+ h -= 2;
+ } while (h != 0);
}
} else if (subw == 1 && subh == 0) {
- if (w_tmp > 7) {
+ if (w >= 8) {
do {
- w_tmp = w;
+ int i = 0;
do {
- load_u8_16x4(mask_tmp, mask_stride, &t0, &t1, &t2, &t3);
+ uint8x8_t m0 = vld1_u8(mask + 2 * i);
+ uint8x8_t m1 = vld1_u8(mask + 2 * i + 8);
+ uint16x8_t s0 = vld1q_u16(src0 + i);
+ uint16x8_t s1 = vld1q_u16(src1 + i);
- mask0 = vreinterpretq_s16_u16(vcombine_u16(
- vpaddl_u8(vget_low_u8(t0)), vpaddl_u8(vget_high_u8(t0))));
- mask1 = vreinterpretq_s16_u16(vcombine_u16(
- vpaddl_u8(vget_low_u8(t1)), vpaddl_u8(vget_high_u8(t1))));
- mask2 = vreinterpretq_s16_u16(vcombine_u16(
- vpaddl_u8(vget_low_u8(t2)), vpaddl_u8(vget_high_u8(t2))));
- mask3 = vreinterpretq_s16_u16(vcombine_u16(
- vpaddl_u8(vget_low_u8(t3)), vpaddl_u8(vget_high_u8(t3))));
+ uint16x8_t m_avg = vmovl_u8(avg_blend_pairwise_u8x8(m0, m1));
+ uint8x8_t blend =
+ alpha_blend_a64_d16_u16x8(m_avg, s0, s1, offset_vec);
- mask0 = vmovl_s8(vqrshrn_n_s16(mask0, 1));
- mask1 = vmovl_s8(vqrshrn_n_s16(mask1, 1));
- mask2 = vmovl_s8(vqrshrn_n_s16(mask2, 1));
- mask3 = vmovl_s8(vqrshrn_n_s16(mask3, 1));
+ vst1_u8(dst + i, blend);
+ i += 8;
+ } while (i < w);
- blend_8x4(dst_tmp, dst_stride, src0_tmp, src0_stride, src1_tmp,
- src1_stride, mask0, mask1, mask2, mask3, v_maxval,
- vec_offset, vec_round_bits);
- w_tmp -= 8;
- mask_tmp += 16;
- dst_tmp += 8;
- src0_tmp += 8;
- src1_tmp += 8;
- } while (w_tmp > 7);
- i += 4;
- mask_tmp += (4 * mask_stride) - (2 * w);
- dst_tmp += (4 * dst_stride) - w;
- src0_tmp += (4 * src0_stride) - w;
- src1_tmp += (4 * src1_stride) - w;
- } while (i < h);
+ mask += mask_stride;
+ src0 += src0_stride;
+ src1 += src1_stride;
+ dst += dst_stride;
+ } while (--h != 0);
} else {
do {
- load_u8_8x4(mask_tmp, mask_stride, &mask0_l, &mask1_l, &mask2_l,
- &mask3_l);
+ uint8x8_t m0 = vld1_u8(mask + 0 * mask_stride);
+ uint8x8_t m1 = vld1_u8(mask + 1 * mask_stride);
+ uint16x8_t s0 = load_unaligned_u16_4x2(src0, src0_stride);
+ uint16x8_t s1 = load_unaligned_u16_4x2(src1, src1_stride);
- mask0 =
- vreinterpretq_s16_u16(vcombine_u16(vpaddl_u8(mask0_l), vec_zero));
- mask1 =
- vreinterpretq_s16_u16(vcombine_u16(vpaddl_u8(mask1_l), vec_zero));
- mask2 =
- vreinterpretq_s16_u16(vcombine_u16(vpaddl_u8(mask2_l), vec_zero));
- mask3 =
- vreinterpretq_s16_u16(vcombine_u16(vpaddl_u8(mask3_l), vec_zero));
+ uint16x8_t m_avg = vmovl_u8(avg_blend_pairwise_u8x8(m0, m1));
+ uint8x8_t blend = alpha_blend_a64_d16_u16x8(m_avg, s0, s1, offset_vec);
- mask0_low = vget_low_s16(vmovl_s8(vqrshrn_n_s16(mask0, 1)));
- mask1_low = vget_low_s16(vmovl_s8(vqrshrn_n_s16(mask1, 1)));
- mask2_low = vget_low_s16(vmovl_s8(vqrshrn_n_s16(mask2, 1)));
- mask3_low = vget_low_s16(vmovl_s8(vqrshrn_n_s16(mask3, 1)));
+ store_unaligned_u8_4x2(dst, dst_stride, blend);
- blend_4x4(dst_tmp, dst_stride, src0_tmp, src0_stride, src1_tmp,
- src1_stride, mask0_low, mask1_low, mask2_low, mask3_low,
- v_maxval, vec_offset, vec_round_bits);
-
- i += 4;
- mask_tmp += (4 * mask_stride);
- dst_tmp += (4 * dst_stride);
- src0_tmp += (4 * src0_stride);
- src1_tmp += (4 * src1_stride);
- } while (i < h);
+ mask += 2 * mask_stride;
+ src0 += 2 * src0_stride;
+ src1 += 2 * src1_stride;
+ dst += 2 * dst_stride;
+ h -= 2;
+ } while (h != 0);
}
} else {
- if (w_tmp > 7) {
+ if (w >= 8) {
do {
- w_tmp = w;
+ int i = 0;
do {
- load_u8_8x8(mask_tmp, mask_stride, &mask0_l, &mask1_l, &mask2_l,
- &mask3_l, &mask4_l, &mask5_l, &mask6_l, &mask7_l);
+ uint8x8_t m0 = vld1_u8(mask + 0 * mask_stride + i);
+ uint8x8_t m1 = vld1_u8(mask + 1 * mask_stride + i);
+ uint16x8_t s0 = vld1q_u16(src0 + i);
+ uint16x8_t s1 = vld1q_u16(src1 + i);
- mask0 = vreinterpretq_s16_u16(vaddl_u8(mask0_l, mask1_l));
- mask1 = vreinterpretq_s16_u16(vaddl_u8(mask2_l, mask3_l));
- mask2 = vreinterpretq_s16_u16(vaddl_u8(mask4_l, mask5_l));
- mask3 = vreinterpretq_s16_u16(vaddl_u8(mask6_l, mask7_l));
+ uint16x8_t m_avg = vmovl_u8(avg_blend_u8x8(m0, m1));
+ uint8x8_t blend =
+ alpha_blend_a64_d16_u16x8(m_avg, s0, s1, offset_vec);
- mask0 = vmovl_s8(vqrshrn_n_s16(mask0, 1));
- mask1 = vmovl_s8(vqrshrn_n_s16(mask1, 1));
- mask2 = vmovl_s8(vqrshrn_n_s16(mask2, 1));
- mask3 = vmovl_s8(vqrshrn_n_s16(mask3, 1));
+ vst1_u8(dst + i, blend);
+ i += 8;
+ } while (i < w);
- blend_8x4(dst_tmp, dst_stride, src0_tmp, src0_stride, src1_tmp,
- src1_stride, mask0, mask1, mask2, mask3, v_maxval,
- vec_offset, vec_round_bits);
-
- w_tmp -= 8;
- mask_tmp += 8;
- dst_tmp += 8;
- src0_tmp += 8;
- src1_tmp += 8;
- } while (w_tmp > 7);
- i += 4;
- mask_tmp += (8 * mask_stride) - w;
- dst_tmp += (4 * dst_stride) - w;
- src0_tmp += (4 * src0_stride) - w;
- src1_tmp += (4 * src1_stride) - w;
- } while (i < h);
+ mask += 2 * mask_stride;
+ src0 += src0_stride;
+ src1 += src1_stride;
+ dst += dst_stride;
+ } while (--h != 0);
} else {
do {
- load_unaligned_u8_4x4(mask_tmp, 2 * mask_stride, &s0, &s1);
- load_unaligned_u8_4x4(mask_tmp + mask_stride, 2 * mask_stride, &s2,
- &s3);
+ uint8x8_t m0_2 =
+ load_unaligned_u8_4x2(mask + 0 * mask_stride, 2 * mask_stride);
+ uint8x8_t m1_3 =
+ load_unaligned_u8_4x2(mask + 1 * mask_stride, 2 * mask_stride);
+ uint16x8_t s0 = load_unaligned_u16_4x2(src0, src0_stride);
+ uint16x8_t s1 = load_unaligned_u16_4x2(src1, src1_stride);
- mask0 = vreinterpretq_s16_u16(vaddl_u8(s0, s2));
- mask1 = vreinterpretq_s16_u16(vaddl_u8(s1, s3));
+ uint16x8_t m_avg = vmovl_u8(avg_blend_u8x8(m0_2, m1_3));
+ uint8x8_t blend = alpha_blend_a64_d16_u16x8(m_avg, s0, s1, offset_vec);
- mask0 = vmovl_s8(vqrshrn_n_s16(mask0, 1));
- mask1 = vmovl_s8(vqrshrn_n_s16(mask1, 1));
+ store_unaligned_u8_4x2(dst, dst_stride, blend);
- mask0_low = vget_low_s16(mask0);
- mask1_low = vget_high_s16(mask0);
- mask2_low = vget_low_s16(mask1);
- mask3_low = vget_high_s16(mask1);
+ mask += 4 * mask_stride;
+ src0 += 2 * src0_stride;
+ src1 += 2 * src1_stride;
+ dst += 2 * dst_stride;
+ h -= 2;
+ } while (h != 0);
+ }
+ }
+}
- blend_4x4(dst_tmp, dst_stride, src0_tmp, src0_stride, src1_tmp,
- src1_stride, mask0_low, mask1_low, mask2_low, mask3_low,
- v_maxval, vec_offset, vec_round_bits);
+void aom_blend_a64_mask_neon(uint8_t *dst, uint32_t dst_stride,
+ const uint8_t *src0, uint32_t src0_stride,
+ const uint8_t *src1, uint32_t src1_stride,
+ const uint8_t *mask, uint32_t mask_stride, int w,
+ int h, int subw, int subh) {
+ assert(IMPLIES(src0 == dst, src0_stride == dst_stride));
+ assert(IMPLIES(src1 == dst, src1_stride == dst_stride));
- i += 4;
- mask_tmp += (8 * mask_stride);
- dst_tmp += (4 * dst_stride);
- src0_tmp += (4 * src0_stride);
- src1_tmp += (4 * src1_stride);
- } while (i < h);
+ assert(h >= 1);
+ assert(w >= 1);
+ assert(IS_POWER_OF_TWO(h));
+ assert(IS_POWER_OF_TWO(w));
+
+ if ((subw | subh) == 0) {
+ if (w > 8) {
+ do {
+ int i = 0;
+ do {
+ uint8x16_t m0 = vld1q_u8(mask + i);
+ uint8x16_t s0 = vld1q_u8(src0 + i);
+ uint8x16_t s1 = vld1q_u8(src1 + i);
+
+ uint8x16_t blend = alpha_blend_a64_u8x16(m0, s0, s1);
+
+ vst1q_u8(dst + i, blend);
+ i += 16;
+ } while (i < w);
+
+ mask += mask_stride;
+ src0 += src0_stride;
+ src1 += src1_stride;
+ dst += dst_stride;
+ } while (--h != 0);
+ } else if (w == 8) {
+ do {
+ uint8x8_t m0 = vld1_u8(mask);
+ uint8x8_t s0 = vld1_u8(src0);
+ uint8x8_t s1 = vld1_u8(src1);
+
+ uint8x8_t blend = alpha_blend_a64_u8x8(m0, s0, s1);
+
+ vst1_u8(dst, blend);
+
+ mask += mask_stride;
+ src0 += src0_stride;
+ src1 += src1_stride;
+ dst += dst_stride;
+ } while (--h != 0);
+ } else {
+ do {
+ uint8x8_t m0 = load_unaligned_u8_4x2(mask, mask_stride);
+ uint8x8_t s0 = load_unaligned_u8_4x2(src0, src0_stride);
+ uint8x8_t s1 = load_unaligned_u8_4x2(src1, src1_stride);
+
+ uint8x8_t blend = alpha_blend_a64_u8x8(m0, s0, s1);
+
+ store_unaligned_u8_4x2(dst, dst_stride, blend);
+
+ mask += 2 * mask_stride;
+ src0 += 2 * src0_stride;
+ src1 += 2 * src1_stride;
+ dst += 2 * dst_stride;
+ h -= 2;
+ } while (h != 0);
+ }
+ } else if ((subw & subh) == 1) {
+ if (w > 8) {
+ do {
+ int i = 0;
+ do {
+ uint8x16_t m0 = vld1q_u8(mask + 0 * mask_stride + 2 * i);
+ uint8x16_t m1 = vld1q_u8(mask + 1 * mask_stride + 2 * i);
+ uint8x16_t m2 = vld1q_u8(mask + 0 * mask_stride + 2 * i + 16);
+ uint8x16_t m3 = vld1q_u8(mask + 1 * mask_stride + 2 * i + 16);
+ uint8x16_t s0 = vld1q_u8(src0 + i);
+ uint8x16_t s1 = vld1q_u8(src1 + i);
+
+ uint8x16_t m_avg = avg_blend_pairwise_u8x16_4(m0, m1, m2, m3);
+ uint8x16_t blend = alpha_blend_a64_u8x16(m_avg, s0, s1);
+
+ vst1q_u8(dst + i, blend);
+
+ i += 16;
+ } while (i < w);
+
+ mask += 2 * mask_stride;
+ src0 += src0_stride;
+ src1 += src1_stride;
+ dst += dst_stride;
+ } while (--h != 0);
+ } else if (w == 8) {
+ do {
+ uint8x8_t m0 = vld1_u8(mask + 0 * mask_stride);
+ uint8x8_t m1 = vld1_u8(mask + 1 * mask_stride);
+ uint8x8_t m2 = vld1_u8(mask + 0 * mask_stride + 8);
+ uint8x8_t m3 = vld1_u8(mask + 1 * mask_stride + 8);
+ uint8x8_t s0 = vld1_u8(src0);
+ uint8x8_t s1 = vld1_u8(src1);
+
+ uint8x8_t m_avg = avg_blend_pairwise_u8x8_4(m0, m1, m2, m3);
+ uint8x8_t blend = alpha_blend_a64_u8x8(m_avg, s0, s1);
+
+ vst1_u8(dst, blend);
+
+ mask += 2 * mask_stride;
+ src0 += src0_stride;
+ src1 += src1_stride;
+ dst += dst_stride;
+ } while (--h != 0);
+ } else {
+ do {
+ uint8x8_t m0 = vld1_u8(mask + 0 * mask_stride);
+ uint8x8_t m1 = vld1_u8(mask + 1 * mask_stride);
+ uint8x8_t m2 = vld1_u8(mask + 2 * mask_stride);
+ uint8x8_t m3 = vld1_u8(mask + 3 * mask_stride);
+ uint8x8_t s0 = load_unaligned_u8_4x2(src0, src0_stride);
+ uint8x8_t s1 = load_unaligned_u8_4x2(src1, src1_stride);
+
+ uint8x8_t m_avg = avg_blend_pairwise_u8x8_4(m0, m1, m2, m3);
+ uint8x8_t blend = alpha_blend_a64_u8x8(m_avg, s0, s1);
+
+ store_unaligned_u8_4x2(dst, dst_stride, blend);
+
+ mask += 4 * mask_stride;
+ src0 += 2 * src0_stride;
+ src1 += 2 * src1_stride;
+ dst += 2 * dst_stride;
+ h -= 2;
+ } while (h != 0);
+ }
+ } else if (subw == 1 && subh == 0) {
+ if (w > 8) {
+ do {
+ int i = 0;
+
+ do {
+ uint8x16_t m0 = vld1q_u8(mask + 2 * i);
+ uint8x16_t m1 = vld1q_u8(mask + 2 * i + 16);
+ uint8x16_t s0 = vld1q_u8(src0 + i);
+ uint8x16_t s1 = vld1q_u8(src1 + i);
+
+ uint8x16_t m_avg = avg_blend_pairwise_u8x16(m0, m1);
+ uint8x16_t blend = alpha_blend_a64_u8x16(m_avg, s0, s1);
+
+ vst1q_u8(dst + i, blend);
+
+ i += 16;
+ } while (i < w);
+
+ mask += mask_stride;
+ src0 += src0_stride;
+ src1 += src1_stride;
+ dst += dst_stride;
+ } while (--h != 0);
+ } else if (w == 8) {
+ do {
+ uint8x8_t m0 = vld1_u8(mask);
+ uint8x8_t m1 = vld1_u8(mask + 8);
+ uint8x8_t s0 = vld1_u8(src0);
+ uint8x8_t s1 = vld1_u8(src1);
+
+ uint8x8_t m_avg = avg_blend_pairwise_u8x8(m0, m1);
+ uint8x8_t blend = alpha_blend_a64_u8x8(m_avg, s0, s1);
+
+ vst1_u8(dst, blend);
+
+ mask += mask_stride;
+ src0 += src0_stride;
+ src1 += src1_stride;
+ dst += dst_stride;
+ } while (--h != 0);
+ } else {
+ do {
+ uint8x8_t m0 = vld1_u8(mask + 0 * mask_stride);
+ uint8x8_t m1 = vld1_u8(mask + 1 * mask_stride);
+ uint8x8_t s0 = load_unaligned_u8_4x2(src0, src0_stride);
+ uint8x8_t s1 = load_unaligned_u8_4x2(src1, src1_stride);
+
+ uint8x8_t m_avg = avg_blend_pairwise_u8x8(m0, m1);
+ uint8x8_t blend = alpha_blend_a64_u8x8(m_avg, s0, s1);
+
+ store_unaligned_u8_4x2(dst, dst_stride, blend);
+
+ mask += 2 * mask_stride;
+ src0 += 2 * src0_stride;
+ src1 += 2 * src1_stride;
+ dst += 2 * dst_stride;
+ h -= 2;
+ } while (h != 0);
+ }
+ } else {
+ if (w > 8) {
+ do {
+ int i = 0;
+ do {
+ uint8x16_t m0 = vld1q_u8(mask + 0 * mask_stride + i);
+ uint8x16_t m1 = vld1q_u8(mask + 1 * mask_stride + i);
+ uint8x16_t s0 = vld1q_u8(src0 + i);
+ uint8x16_t s1 = vld1q_u8(src1 + i);
+
+ uint8x16_t m_avg = avg_blend_u8x16(m0, m1);
+ uint8x16_t blend = alpha_blend_a64_u8x16(m_avg, s0, s1);
+
+ vst1q_u8(dst + i, blend);
+
+ i += 16;
+ } while (i < w);
+
+ mask += 2 * mask_stride;
+ src0 += src0_stride;
+ src1 += src1_stride;
+ dst += dst_stride;
+ } while (--h != 0);
+ } else if (w == 8) {
+ do {
+ uint8x8_t m0 = vld1_u8(mask + 0 * mask_stride);
+ uint8x8_t m1 = vld1_u8(mask + 1 * mask_stride);
+ uint8x8_t s0 = vld1_u8(src0);
+ uint8x8_t s1 = vld1_u8(src1);
+
+ uint8x8_t m_avg = avg_blend_u8x8(m0, m1);
+ uint8x8_t blend = alpha_blend_a64_u8x8(m_avg, s0, s1);
+
+ vst1_u8(dst, blend);
+
+ mask += 2 * mask_stride;
+ src0 += src0_stride;
+ src1 += src1_stride;
+ dst += dst_stride;
+ } while (--h != 0);
+ } else {
+ do {
+ uint8x8_t m0_2 =
+ load_unaligned_u8_4x2(mask + 0 * mask_stride, 2 * mask_stride);
+ uint8x8_t m1_3 =
+ load_unaligned_u8_4x2(mask + 1 * mask_stride, 2 * mask_stride);
+ uint8x8_t s0 = load_unaligned_u8_4x2(src0, src0_stride);
+ uint8x8_t s1 = load_unaligned_u8_4x2(src1, src1_stride);
+
+ uint8x8_t m_avg = avg_blend_u8x8(m0_2, m1_3);
+ uint8x8_t blend = alpha_blend_a64_u8x8(m_avg, s0, s1);
+
+ store_unaligned_u8_4x2(dst, dst_stride, blend);
+
+ mask += 4 * mask_stride;
+ src0 += 2 * src0_stride;
+ src1 += 2 * src1_stride;
+ dst += 2 * dst_stride;
+ h -= 2;
+ } while (h != 0);
}
}
}
diff --git a/aom_dsp/arm/blend_neon.h b/aom_dsp/arm/blend_neon.h
new file mode 100644
index 0000000..c8a03224
--- /dev/null
+++ b/aom_dsp/arm/blend_neon.h
@@ -0,0 +1,125 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#ifndef AOM_AOM_DSP_ARM_BLEND_NEON_H_
+#define AOM_AOM_DSP_ARM_BLEND_NEON_H_
+
+#include <arm_neon.h>
+
+#include "aom_dsp/blend.h"
+
+static INLINE uint8x16_t alpha_blend_a64_u8x16(uint8x16_t m, uint8x16_t a,
+ uint8x16_t b) {
+ const uint8x16_t m_inv = vsubq_u8(vdupq_n_u8(AOM_BLEND_A64_MAX_ALPHA), m);
+
+ uint16x8_t blend_u16_lo = vmull_u8(vget_low_u8(m), vget_low_u8(a));
+ uint16x8_t blend_u16_hi = vmull_u8(vget_high_u8(m), vget_high_u8(a));
+
+ blend_u16_lo = vmlal_u8(blend_u16_lo, vget_low_u8(m_inv), vget_low_u8(b));
+ blend_u16_hi = vmlal_u8(blend_u16_hi, vget_high_u8(m_inv), vget_high_u8(b));
+
+ uint8x8_t blend_u8_lo = vrshrn_n_u16(blend_u16_lo, AOM_BLEND_A64_ROUND_BITS);
+ uint8x8_t blend_u8_hi = vrshrn_n_u16(blend_u16_hi, AOM_BLEND_A64_ROUND_BITS);
+
+ return vcombine_u8(blend_u8_lo, blend_u8_hi);
+}
+
+static INLINE uint8x8_t alpha_blend_a64_u8x8(uint8x8_t m, uint8x8_t a,
+ uint8x8_t b) {
+ const uint8x8_t m_inv = vsub_u8(vdup_n_u8(AOM_BLEND_A64_MAX_ALPHA), m);
+
+ uint16x8_t blend_u16 = vmull_u8(m, a);
+
+ blend_u16 = vmlal_u8(blend_u16, m_inv, b);
+
+ return vrshrn_n_u16(blend_u16, AOM_BLEND_A64_ROUND_BITS);
+}
+
+#if CONFIG_AV1_HIGHBITDEPTH
+static INLINE uint16x8_t alpha_blend_a64_u16x8(uint16x8_t m, uint16x8_t a,
+ uint16x8_t b) {
+ uint16x8_t m_inv = vsubq_u16(vdupq_n_u16(AOM_BLEND_A64_MAX_ALPHA), m);
+
+ uint32x4_t blend_u32_lo = vmull_u16(vget_low_u16(a), vget_low_u16(m));
+ uint32x4_t blend_u32_hi = vmull_u16(vget_high_u16(a), vget_high_u16(m));
+
+ blend_u32_lo = vmlal_u16(blend_u32_lo, vget_low_u16(b), vget_low_u16(m_inv));
+ blend_u32_hi =
+ vmlal_u16(blend_u32_hi, vget_high_u16(b), vget_high_u16(m_inv));
+
+ uint16x4_t blend_u16_lo =
+ vrshrn_n_u32(blend_u32_lo, AOM_BLEND_A64_ROUND_BITS);
+ uint16x4_t blend_u16_hi =
+ vrshrn_n_u32(blend_u32_hi, AOM_BLEND_A64_ROUND_BITS);
+
+ return vcombine_u16(blend_u16_lo, blend_u16_hi);
+}
+
+static INLINE uint16x4_t alpha_blend_a64_u16x4(uint16x4_t m, uint16x4_t a,
+ uint16x4_t b) {
+ const uint16x4_t m_inv = vsub_u16(vdup_n_u16(AOM_BLEND_A64_MAX_ALPHA), m);
+
+ uint32x4_t blend_u16 = vmull_u16(m, a);
+
+ blend_u16 = vmlal_u16(blend_u16, m_inv, b);
+
+ return vrshrn_n_u32(blend_u16, AOM_BLEND_A64_ROUND_BITS);
+}
+#endif // CONFIG_AV1_HIGHBITDEPTH
+
+static INLINE uint8x8_t avg_blend_u8x8(uint8x8_t a, uint8x8_t b) {
+ return vrhadd_u8(a, b);
+}
+
+static INLINE uint8x16_t avg_blend_u8x16(uint8x16_t a, uint8x16_t b) {
+ return vrhaddq_u8(a, b);
+}
+
+static INLINE uint8x8_t avg_blend_pairwise_u8x8(uint8x8_t a, uint8x8_t b) {
+ return vrshr_n_u8(vpadd_u8(a, b), 1);
+}
+
+static INLINE uint8x16_t avg_blend_pairwise_u8x16(uint8x16_t a, uint8x16_t b) {
+#if AOM_ARCH_AARCH64
+ return vrshrq_n_u8(vpaddq_u8(a, b), 1);
+#else
+ uint8x8_t sum_pairwise_a = vpadd_u8(vget_low_u8(a), vget_high_u8(a));
+ uint8x8_t sum_pairwise_b = vpadd_u8(vget_low_u8(b), vget_high_u8(b));
+ return vrshrq_n_u8(vcombine_u8(sum_pairwise_a, sum_pairwise_b), 1);
+#endif // AOM_ARCH_AARCH64
+}
+
+static INLINE uint8x8_t avg_blend_pairwise_u8x8_4(uint8x8_t a, uint8x8_t b,
+ uint8x8_t c, uint8x8_t d) {
+ uint8x8_t a_c = vpadd_u8(a, c);
+ uint8x8_t b_d = vpadd_u8(b, d);
+ return vrshr_n_u8(vqadd_u8(a_c, b_d), 2);
+}
+
+static INLINE uint8x16_t avg_blend_pairwise_u8x16_4(uint8x16_t a, uint8x16_t b,
+ uint8x16_t c,
+ uint8x16_t d) {
+#if AOM_ARCH_AARCH64
+ uint8x16_t a_c = vpaddq_u8(a, c);
+ uint8x16_t b_d = vpaddq_u8(b, d);
+ return vrshrq_n_u8(vqaddq_u8(a_c, b_d), 2);
+#else
+ uint8x8_t sum_pairwise_a = vpadd_u8(vget_low_u8(a), vget_high_u8(a));
+ uint8x8_t sum_pairwise_b = vpadd_u8(vget_low_u8(b), vget_high_u8(b));
+ uint8x8_t sum_pairwise_c = vpadd_u8(vget_low_u8(c), vget_high_u8(c));
+ uint8x8_t sum_pairwise_d = vpadd_u8(vget_low_u8(d), vget_high_u8(d));
+ uint8x16_t a_c = vcombine_u8(sum_pairwise_a, sum_pairwise_c);
+ uint8x16_t b_d = vcombine_u8(sum_pairwise_b, sum_pairwise_d);
+ return vrshrq_n_u8(vqaddq_u8(a_c, b_d), 2);
+#endif // AOM_ARCH_AARCH64
+}
+
+#endif // AOM_AOM_DSP_ARM_BLEND_NEON_H_
diff --git a/aom_dsp/arm/blk_sse_sum_neon.c b/aom_dsp/arm/blk_sse_sum_neon.c
new file mode 100644
index 0000000..f2ada93
--- /dev/null
+++ b/aom_dsp/arm/blk_sse_sum_neon.c
@@ -0,0 +1,124 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+#include <assert.h>
+
+#include "config/aom_dsp_rtcd.h"
+#include "config/aom_config.h"
+
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_dsp/arm/sum_neon.h"
+
+static INLINE void get_blk_sse_sum_4xh_neon(const int16_t *data, int stride,
+ int bh, int *x_sum,
+ int64_t *x2_sum) {
+ int i = bh;
+ int32x4_t sum = vdupq_n_s32(0);
+ int32x4_t sse = vdupq_n_s32(0);
+
+ do {
+ int16x8_t d = vcombine_s16(vld1_s16(data), vld1_s16(data + stride));
+
+ sum = vpadalq_s16(sum, d);
+
+ sse = vmlal_s16(sse, vget_low_s16(d), vget_low_s16(d));
+ sse = vmlal_s16(sse, vget_high_s16(d), vget_high_s16(d));
+
+ data += 2 * stride;
+ i -= 2;
+ } while (i != 0);
+
+ *x_sum = horizontal_add_s32x4(sum);
+ *x2_sum = horizontal_long_add_s32x4(sse);
+}
+
+static INLINE void get_blk_sse_sum_8xh_neon(const int16_t *data, int stride,
+ int bh, int *x_sum,
+ int64_t *x2_sum) {
+ int i = bh;
+ int32x4_t sum = vdupq_n_s32(0);
+ int32x4_t sse = vdupq_n_s32(0);
+
+ // Input is 12-bit wide, so we can add up to 127 squared elements in a signed
+ // 32-bits element. Since we're accumulating into an int32x4_t and the maximum
+ // value for bh is 32, we don't have to worry about sse overflowing.
+
+ do {
+ int16x8_t d = vld1q_s16(data);
+
+ sum = vpadalq_s16(sum, d);
+
+ sse = vmlal_s16(sse, vget_low_s16(d), vget_low_s16(d));
+ sse = vmlal_s16(sse, vget_high_s16(d), vget_high_s16(d));
+
+ data += stride;
+ } while (--i != 0);
+
+ *x_sum = horizontal_add_s32x4(sum);
+ *x2_sum = horizontal_long_add_s32x4(sse);
+}
+
+static INLINE void get_blk_sse_sum_large_neon(const int16_t *data, int stride,
+ int bw, int bh, int *x_sum,
+ int64_t *x2_sum) {
+ int32x4_t sum = vdupq_n_s32(0);
+ int64x2_t sse = vdupq_n_s64(0);
+
+ // Input is 12-bit wide, so we can add up to 127 squared elements in a signed
+ // 32-bits element. Since we're accumulating into an int32x4_t vector that
+ // means we can process up to (127*4)/bw rows before we need to widen to
+ // 64 bits.
+
+ int i_limit = (127 * 4) / bw;
+ int i_tmp = bh > i_limit ? i_limit : bh;
+
+ int i = 0;
+ do {
+ int32x4_t sse_s32 = vdupq_n_s32(0);
+ do {
+ int j = bw;
+ const int16_t *data_ptr = data;
+ do {
+ int16x8_t d = vld1q_s16(data_ptr);
+
+ sum = vpadalq_s16(sum, d);
+
+ sse_s32 = vmlal_s16(sse_s32, vget_low_s16(d), vget_low_s16(d));
+ sse_s32 = vmlal_s16(sse_s32, vget_high_s16(d), vget_high_s16(d));
+
+ data_ptr += 8;
+ j -= 8;
+ } while (j != 0);
+
+ data += stride;
+ i++;
+ } while (i < i_tmp && i < bh);
+
+ sse = vpadalq_s32(sse, sse_s32);
+ i_tmp += i_limit;
+ } while (i < bh);
+
+ *x_sum = horizontal_add_s32x4(sum);
+ *x2_sum = horizontal_add_s64x2(sse);
+}
+
+void aom_get_blk_sse_sum_neon(const int16_t *data, int stride, int bw, int bh,
+ int *x_sum, int64_t *x2_sum) {
+ if (bw == 4) {
+ get_blk_sse_sum_4xh_neon(data, stride, bh, x_sum, x2_sum);
+ } else if (bw == 8) {
+ get_blk_sse_sum_8xh_neon(data, stride, bh, x_sum, x2_sum);
+ } else {
+ assert(bw % 8 == 0);
+ get_blk_sse_sum_large_neon(data, stride, bw, bh, x_sum, x2_sum);
+ }
+}
diff --git a/aom_dsp/arm/dist_wtd_avg_neon.h b/aom_dsp/arm/dist_wtd_avg_neon.h
new file mode 100644
index 0000000..19c9b04
--- /dev/null
+++ b/aom_dsp/arm/dist_wtd_avg_neon.h
@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef AOM_AOM_DSP_ARM_DIST_WTD_AVG_NEON_H_
+#define AOM_AOM_DSP_ARM_DIST_WTD_AVG_NEON_H_
+
+#include <arm_neon.h>
+
+#include "aom_dsp/aom_dsp_common.h"
+#include "av1/common/enums.h"
+
+static INLINE uint8x8_t dist_wtd_avg_u8x8(uint8x8_t a, uint8x8_t b,
+ uint8x8_t wta, uint8x8_t wtb) {
+ uint16x8_t wtd_sum = vmull_u8(a, wta);
+
+ wtd_sum = vmlal_u8(wtd_sum, b, wtb);
+
+ return vrshrn_n_u16(wtd_sum, DIST_PRECISION_BITS);
+}
+
+static INLINE uint16x4_t dist_wtd_avg_u16x4(uint16x4_t a, uint16x4_t b,
+ uint16x4_t wta, uint16x4_t wtb) {
+ uint32x4_t wtd_sum = vmull_u16(a, wta);
+
+ wtd_sum = vmlal_u16(wtd_sum, b, wtb);
+
+ return vrshrn_n_u32(wtd_sum, DIST_PRECISION_BITS);
+}
+
+static INLINE uint8x16_t dist_wtd_avg_u8x16(uint8x16_t a, uint8x16_t b,
+ uint8x16_t wta, uint8x16_t wtb) {
+ uint16x8_t wtd_sum_lo = vmull_u8(vget_low_u8(a), vget_low_u8(wta));
+ uint16x8_t wtd_sum_hi = vmull_u8(vget_high_u8(a), vget_high_u8(wta));
+
+ wtd_sum_lo = vmlal_u8(wtd_sum_lo, vget_low_u8(b), vget_low_u8(wtb));
+ wtd_sum_hi = vmlal_u8(wtd_sum_hi, vget_high_u8(b), vget_high_u8(wtb));
+
+ uint8x8_t wtd_avg_lo = vrshrn_n_u16(wtd_sum_lo, DIST_PRECISION_BITS);
+ uint8x8_t wtd_avg_hi = vrshrn_n_u16(wtd_sum_hi, DIST_PRECISION_BITS);
+
+ return vcombine_u8(wtd_avg_lo, wtd_avg_hi);
+}
+
+static INLINE uint16x8_t dist_wtd_avg_u16x8(uint16x8_t a, uint16x8_t b,
+ uint16x8_t wta, uint16x8_t wtb) {
+ uint32x4_t wtd_sum_lo = vmull_u16(vget_low_u16(a), vget_low_u16(wta));
+ uint32x4_t wtd_sum_hi = vmull_u16(vget_high_u16(a), vget_high_u16(wta));
+
+ wtd_sum_lo = vmlal_u16(wtd_sum_lo, vget_low_u16(b), vget_low_u16(wtb));
+ wtd_sum_hi = vmlal_u16(wtd_sum_hi, vget_high_u16(b), vget_high_u16(wtb));
+
+ uint16x4_t wtd_avg_lo = vrshrn_n_u32(wtd_sum_lo, DIST_PRECISION_BITS);
+ uint16x4_t wtd_avg_hi = vrshrn_n_u32(wtd_sum_hi, DIST_PRECISION_BITS);
+
+ return vcombine_u16(wtd_avg_lo, wtd_avg_hi);
+}
+
+#endif // AOM_AOM_DSP_ARM_DIST_WTD_AVG_NEON_H_
diff --git a/aom_dsp/arm/fwd_txfm_neon.c b/aom_dsp/arm/fwd_txfm_neon.c
index a7d66b3..fb4cda7 100644
--- a/aom_dsp/arm/fwd_txfm_neon.c
+++ b/aom_dsp/arm/fwd_txfm_neon.c
@@ -48,8 +48,8 @@
// Must expand all elements to s32. See 'needs32' comment in fwd_txfm.c.
const int32x4_t s_0_p_s_1 = vaddl_s16(s_0, s_1);
const int32x4_t s_0_m_s_1 = vsubl_s16(s_0, s_1);
- const int32x4_t temp1 = vmulq_n_s32(s_0_p_s_1, cospi_16_64);
- const int32x4_t temp2 = vmulq_n_s32(s_0_m_s_1, cospi_16_64);
+ const int32x4_t temp1 = vmulq_n_s32(s_0_p_s_1, (int32_t)cospi_16_64);
+ const int32x4_t temp2 = vmulq_n_s32(s_0_m_s_1, (int32_t)cospi_16_64);
// fdct_round_shift
int16x4_t out_0 = vrshrn_n_s32(temp1, DCT_CONST_BITS);
@@ -57,11 +57,13 @@
// s_3 * cospi_8_64 + s_2 * cospi_24_64
// s_3 * cospi_24_64 - s_2 * cospi_8_64
- const int32x4_t s_3_cospi_8_64 = vmull_n_s16(s_3, cospi_8_64);
- const int32x4_t s_3_cospi_24_64 = vmull_n_s16(s_3, cospi_24_64);
+ const int32x4_t s_3_cospi_8_64 = vmull_n_s16(s_3, (int32_t)cospi_8_64);
+ const int32x4_t s_3_cospi_24_64 = vmull_n_s16(s_3, (int32_t)cospi_24_64);
- const int32x4_t temp3 = vmlal_n_s16(s_3_cospi_8_64, s_2, cospi_24_64);
- const int32x4_t temp4 = vmlsl_n_s16(s_3_cospi_24_64, s_2, cospi_8_64);
+ const int32x4_t temp3 =
+ vmlal_n_s16(s_3_cospi_8_64, s_2, (int32_t)cospi_24_64);
+ const int32x4_t temp4 =
+ vmlsl_n_s16(s_3_cospi_24_64, s_2, (int32_t)cospi_8_64);
// fdct_round_shift
int16x4_t out_1 = vrshrn_n_s32(temp3, DCT_CONST_BITS);
@@ -69,7 +71,7 @@
// Only transpose the first pass
if (i == 0) {
- transpose_s16_4x4d(&out_0, &out_1, &out_2, &out_3);
+ transpose_elems_inplace_s16_4x4(&out_0, &out_1, &out_2, &out_3);
}
*input_0 = out_0;
diff --git a/aom_dsp/arm/hadamard_neon.c b/aom_dsp/arm/hadamard_neon.c
index 82ce0cd..d0f5922 100644
--- a/aom_dsp/arm/hadamard_neon.c
+++ b/aom_dsp/arm/hadamard_neon.c
@@ -37,7 +37,7 @@
hadamard_4x4_one_pass(&a0, &a1, &a2, &a3);
- transpose_s16_4x4d(&a0, &a1, &a2, &a3);
+ transpose_elems_inplace_s16_4x4(&a0, &a1, &a2, &a3);
hadamard_4x4_one_pass(&a0, &a1, &a2, &a3);
@@ -91,7 +91,7 @@
hadamard8x8_one_pass(&a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7);
- transpose_s16_8x8(&a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7);
+ transpose_elems_inplace_s16_8x8(&a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7);
hadamard8x8_one_pass(&a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7);
@@ -120,7 +120,7 @@
hadamard8x8_one_pass(&a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7);
- transpose_s16_8x8(&a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7);
+ transpose_elems_inplace_s16_8x8(&a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7);
hadamard8x8_one_pass(&a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7);
@@ -196,56 +196,90 @@
/* Bottom right. */
aom_hadamard_8x8_neon(src_diff + 8 + 8 * src_stride, src_stride, coeff + 192);
+ // Each iteration of the loop operates on entire rows (16 samples each)
+ // because we need to swap the second and third quarters of every row in the
+ // output to match AVX2 output (i.e., aom_hadamard_16x16_avx2). See the for
+ // loop at the end of aom_hadamard_16x16_c.
for (int i = 0; i < 64; i += 16) {
- const int16x8_t a00 = load_tran_low_to_s16q(coeff + 0);
- const int16x8_t a01 = load_tran_low_to_s16q(coeff + 64);
- const int16x8_t a02 = load_tran_low_to_s16q(coeff + 128);
- const int16x8_t a03 = load_tran_low_to_s16q(coeff + 192);
+ const int32x4_t a00 = vld1q_s32(coeff + 0);
+ const int32x4_t a01 = vld1q_s32(coeff + 64);
+ const int32x4_t a02 = vld1q_s32(coeff + 128);
+ const int32x4_t a03 = vld1q_s32(coeff + 192);
- const int16x8_t b00 = vhaddq_s16(a00, a01);
- const int16x8_t b01 = vhsubq_s16(a00, a01);
- const int16x8_t b02 = vhaddq_s16(a02, a03);
- const int16x8_t b03 = vhsubq_s16(a02, a03);
+ const int32x4_t b00 = vhaddq_s32(a00, a01);
+ const int32x4_t b01 = vhsubq_s32(a00, a01);
+ const int32x4_t b02 = vhaddq_s32(a02, a03);
+ const int32x4_t b03 = vhsubq_s32(a02, a03);
- const int16x8_t c00 = vaddq_s16(b00, b02);
- const int16x8_t c01 = vaddq_s16(b01, b03);
- const int16x8_t c02 = vsubq_s16(b00, b02);
- const int16x8_t c03 = vsubq_s16(b01, b03);
+ const int32x4_t c00 = vaddq_s32(b00, b02);
+ const int32x4_t c01 = vaddq_s32(b01, b03);
+ const int32x4_t c02 = vsubq_s32(b00, b02);
+ const int32x4_t c03 = vsubq_s32(b01, b03);
- const int16x8_t a10 = load_tran_low_to_s16q(coeff + 8 + 0);
- const int16x8_t a11 = load_tran_low_to_s16q(coeff + 8 + 64);
- const int16x8_t a12 = load_tran_low_to_s16q(coeff + 8 + 128);
- const int16x8_t a13 = load_tran_low_to_s16q(coeff + 8 + 192);
+ const int32x4_t a10 = vld1q_s32(coeff + 4 + 0);
+ const int32x4_t a11 = vld1q_s32(coeff + 4 + 64);
+ const int32x4_t a12 = vld1q_s32(coeff + 4 + 128);
+ const int32x4_t a13 = vld1q_s32(coeff + 4 + 192);
- const int16x8_t b10 = vhaddq_s16(a10, a11);
- const int16x8_t b11 = vhsubq_s16(a10, a11);
- const int16x8_t b12 = vhaddq_s16(a12, a13);
- const int16x8_t b13 = vhsubq_s16(a12, a13);
+ const int32x4_t b10 = vhaddq_s32(a10, a11);
+ const int32x4_t b11 = vhsubq_s32(a10, a11);
+ const int32x4_t b12 = vhaddq_s32(a12, a13);
+ const int32x4_t b13 = vhsubq_s32(a12, a13);
- const int16x8_t c10 = vaddq_s16(b10, b12);
- const int16x8_t c11 = vaddq_s16(b11, b13);
- const int16x8_t c12 = vsubq_s16(b10, b12);
- const int16x8_t c13 = vsubq_s16(b11, b13);
+ const int32x4_t c10 = vaddq_s32(b10, b12);
+ const int32x4_t c11 = vaddq_s32(b11, b13);
+ const int32x4_t c12 = vsubq_s32(b10, b12);
+ const int32x4_t c13 = vsubq_s32(b11, b13);
- store_s16_to_tran_low(coeff + 0 + 0, vget_low_s16(c00));
- store_s16_to_tran_low(coeff + 0 + 4, vget_low_s16(c10));
- store_s16_to_tran_low(coeff + 0 + 8, vget_high_s16(c00));
- store_s16_to_tran_low(coeff + 0 + 12, vget_high_s16(c10));
+ const int32x4_t a20 = vld1q_s32(coeff + 8 + 0);
+ const int32x4_t a21 = vld1q_s32(coeff + 8 + 64);
+ const int32x4_t a22 = vld1q_s32(coeff + 8 + 128);
+ const int32x4_t a23 = vld1q_s32(coeff + 8 + 192);
- store_s16_to_tran_low(coeff + 64 + 0, vget_low_s16(c01));
- store_s16_to_tran_low(coeff + 64 + 4, vget_low_s16(c11));
- store_s16_to_tran_low(coeff + 64 + 8, vget_high_s16(c01));
- store_s16_to_tran_low(coeff + 64 + 12, vget_high_s16(c11));
+ const int32x4_t b20 = vhaddq_s32(a20, a21);
+ const int32x4_t b21 = vhsubq_s32(a20, a21);
+ const int32x4_t b22 = vhaddq_s32(a22, a23);
+ const int32x4_t b23 = vhsubq_s32(a22, a23);
- store_s16_to_tran_low(coeff + 128 + 0, vget_low_s16(c02));
- store_s16_to_tran_low(coeff + 128 + 4, vget_low_s16(c12));
- store_s16_to_tran_low(coeff + 128 + 8, vget_high_s16(c02));
- store_s16_to_tran_low(coeff + 128 + 12, vget_high_s16(c12));
+ const int32x4_t c20 = vaddq_s32(b20, b22);
+ const int32x4_t c21 = vaddq_s32(b21, b23);
+ const int32x4_t c22 = vsubq_s32(b20, b22);
+ const int32x4_t c23 = vsubq_s32(b21, b23);
- store_s16_to_tran_low(coeff + 192 + 0, vget_low_s16(c03));
- store_s16_to_tran_low(coeff + 192 + 4, vget_low_s16(c13));
- store_s16_to_tran_low(coeff + 192 + 8, vget_high_s16(c03));
- store_s16_to_tran_low(coeff + 192 + 12, vget_high_s16(c13));
+ const int32x4_t a30 = vld1q_s32(coeff + 12 + 0);
+ const int32x4_t a31 = vld1q_s32(coeff + 12 + 64);
+ const int32x4_t a32 = vld1q_s32(coeff + 12 + 128);
+ const int32x4_t a33 = vld1q_s32(coeff + 12 + 192);
+
+ const int32x4_t b30 = vhaddq_s32(a30, a31);
+ const int32x4_t b31 = vhsubq_s32(a30, a31);
+ const int32x4_t b32 = vhaddq_s32(a32, a33);
+ const int32x4_t b33 = vhsubq_s32(a32, a33);
+
+ const int32x4_t c30 = vaddq_s32(b30, b32);
+ const int32x4_t c31 = vaddq_s32(b31, b33);
+ const int32x4_t c32 = vsubq_s32(b30, b32);
+ const int32x4_t c33 = vsubq_s32(b31, b33);
+
+ vst1q_s32(coeff + 0 + 0, c00);
+ vst1q_s32(coeff + 0 + 4, c20);
+ vst1q_s32(coeff + 0 + 8, c10);
+ vst1q_s32(coeff + 0 + 12, c30);
+
+ vst1q_s32(coeff + 64 + 0, c01);
+ vst1q_s32(coeff + 64 + 4, c21);
+ vst1q_s32(coeff + 64 + 8, c11);
+ vst1q_s32(coeff + 64 + 12, c31);
+
+ vst1q_s32(coeff + 128 + 0, c02);
+ vst1q_s32(coeff + 128 + 4, c22);
+ vst1q_s32(coeff + 128 + 8, c12);
+ vst1q_s32(coeff + 128 + 12, c32);
+
+ vst1q_s32(coeff + 192 + 0, c03);
+ vst1q_s32(coeff + 192 + 4, c23);
+ vst1q_s32(coeff + 192 + 8, c13);
+ vst1q_s32(coeff + 192 + 12, c33);
coeff += 16;
}
diff --git a/aom_dsp/arm/highbd_avg_pred_neon.c b/aom_dsp/arm/highbd_avg_pred_neon.c
new file mode 100644
index 0000000..531309b
--- /dev/null
+++ b/aom_dsp/arm/highbd_avg_pred_neon.c
@@ -0,0 +1,190 @@
+/*
+ * Copyright (c) 2023 The WebM project authors. All Rights Reserved.
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+#include <assert.h>
+
+#include "config/aom_config.h"
+#include "config/aom_dsp_rtcd.h"
+
+#include "aom_dsp/arm/blend_neon.h"
+#include "aom_dsp/arm/dist_wtd_avg_neon.h"
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_dsp/blend.h"
+
+void aom_highbd_comp_avg_pred_neon(uint8_t *comp_pred8, const uint8_t *pred8,
+ int width, int height, const uint8_t *ref8,
+ int ref_stride) {
+ const uint16_t *pred = CONVERT_TO_SHORTPTR(pred8);
+ const uint16_t *ref = CONVERT_TO_SHORTPTR(ref8);
+ uint16_t *comp_pred = CONVERT_TO_SHORTPTR(comp_pred8);
+
+ int i = height;
+ if (width > 8) {
+ do {
+ int j = 0;
+ do {
+ const uint16x8_t p = vld1q_u16(pred + j);
+ const uint16x8_t r = vld1q_u16(ref + j);
+
+ uint16x8_t avg = vrhaddq_u16(p, r);
+ vst1q_u16(comp_pred + j, avg);
+
+ j += 8;
+ } while (j < width);
+
+ comp_pred += width;
+ pred += width;
+ ref += ref_stride;
+ } while (--i != 0);
+ } else if (width == 8) {
+ do {
+ const uint16x8_t p = vld1q_u16(pred);
+ const uint16x8_t r = vld1q_u16(ref);
+
+ uint16x8_t avg = vrhaddq_u16(p, r);
+ vst1q_u16(comp_pred, avg);
+
+ comp_pred += width;
+ pred += width;
+ ref += ref_stride;
+ } while (--i != 0);
+ } else {
+ assert(width == 4);
+ do {
+ const uint16x4_t p = vld1_u16(pred);
+ const uint16x4_t r = vld1_u16(ref);
+
+ uint16x4_t avg = vrhadd_u16(p, r);
+ vst1_u16(comp_pred, avg);
+
+ comp_pred += width;
+ pred += width;
+ ref += ref_stride;
+ } while (--i != 0);
+ }
+}
+
+void aom_highbd_comp_mask_pred_neon(uint8_t *comp_pred8, const uint8_t *pred8,
+ int width, int height, const uint8_t *ref8,
+ int ref_stride, const uint8_t *mask,
+ int mask_stride, int invert_mask) {
+ uint16_t *pred = CONVERT_TO_SHORTPTR(pred8);
+ uint16_t *ref = CONVERT_TO_SHORTPTR(ref8);
+ uint16_t *comp_pred = CONVERT_TO_SHORTPTR(comp_pred8);
+
+ const uint16_t *src0 = invert_mask ? pred : ref;
+ const uint16_t *src1 = invert_mask ? ref : pred;
+ const int src_stride0 = invert_mask ? width : ref_stride;
+ const int src_stride1 = invert_mask ? ref_stride : width;
+
+ if (width >= 8) {
+ do {
+ int j = 0;
+
+ do {
+ const uint16x8_t s0 = vld1q_u16(src0 + j);
+ const uint16x8_t s1 = vld1q_u16(src1 + j);
+ const uint16x8_t m0 = vmovl_u8(vld1_u8(mask + j));
+
+ uint16x8_t blend_u16 = alpha_blend_a64_u16x8(m0, s0, s1);
+
+ vst1q_u16(comp_pred + j, blend_u16);
+
+ j += 8;
+ } while (j < width);
+
+ src0 += src_stride0;
+ src1 += src_stride1;
+ mask += mask_stride;
+ comp_pred += width;
+ } while (--height != 0);
+ } else {
+ assert(width == 4);
+
+ do {
+ const uint16x4_t s0 = vld1_u16(src0);
+ const uint16x4_t s1 = vld1_u16(src1);
+ const uint16x4_t m0 = vget_low_u16(vmovl_u8(load_unaligned_u8_4x1(mask)));
+
+ uint16x4_t blend_u16 = alpha_blend_a64_u16x4(m0, s0, s1);
+
+ vst1_u16(comp_pred, blend_u16);
+
+ src0 += src_stride0;
+ src1 += src_stride1;
+ mask += mask_stride;
+ comp_pred += 4;
+ } while (--height != 0);
+ }
+}
+
+void aom_highbd_dist_wtd_comp_avg_pred_neon(
+ uint8_t *comp_pred8, const uint8_t *pred8, int width, int height,
+ const uint8_t *ref8, int ref_stride,
+ const DIST_WTD_COMP_PARAMS *jcp_param) {
+ const uint16x8_t fwd_offset_u16 = vdupq_n_u16(jcp_param->fwd_offset);
+ const uint16x8_t bck_offset_u16 = vdupq_n_u16(jcp_param->bck_offset);
+ const uint16_t *pred = CONVERT_TO_SHORTPTR(pred8);
+ const uint16_t *ref = CONVERT_TO_SHORTPTR(ref8);
+ uint16_t *comp_pred = CONVERT_TO_SHORTPTR(comp_pred8);
+
+ if (width > 8) {
+ do {
+ int j = 0;
+ do {
+ const uint16x8_t p = vld1q_u16(pred + j);
+ const uint16x8_t r = vld1q_u16(ref + j);
+
+ const uint16x8_t avg =
+ dist_wtd_avg_u16x8(r, p, fwd_offset_u16, bck_offset_u16);
+
+ vst1q_u16(comp_pred + j, avg);
+
+ j += 8;
+ } while (j < width);
+
+ comp_pred += width;
+ pred += width;
+ ref += ref_stride;
+ } while (--height != 0);
+ } else if (width == 8) {
+ do {
+ const uint16x8_t p = vld1q_u16(pred);
+ const uint16x8_t r = vld1q_u16(ref);
+
+ const uint16x8_t avg =
+ dist_wtd_avg_u16x8(r, p, fwd_offset_u16, bck_offset_u16);
+
+ vst1q_u16(comp_pred, avg);
+
+ comp_pred += width;
+ pred += width;
+ ref += ref_stride;
+ } while (--height != 0);
+ } else {
+ assert(width == 4);
+ do {
+ const uint16x4_t p = vld1_u16(pred);
+ const uint16x4_t r = vld1_u16(ref);
+
+ const uint16x4_t avg = dist_wtd_avg_u16x4(
+ r, p, vget_low_u16(fwd_offset_u16), vget_low_u16(bck_offset_u16));
+
+ vst1_u16(comp_pred, avg);
+
+ comp_pred += width;
+ pred += width;
+ ref += ref_stride;
+ } while (--height != 0);
+ }
+}
diff --git a/aom_dsp/arm/highbd_blend_a64_hmask_neon.c b/aom_dsp/arm/highbd_blend_a64_hmask_neon.c
new file mode 100644
index 0000000..bdd2177
--- /dev/null
+++ b/aom_dsp/arm/highbd_blend_a64_hmask_neon.c
@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+#include <assert.h>
+
+#include "config/aom_config.h"
+#include "config/aom_dsp_rtcd.h"
+
+#include "aom_dsp/arm/blend_neon.h"
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_dsp/blend.h"
+
+void aom_highbd_blend_a64_hmask_neon(uint8_t *dst_8, uint32_t dst_stride,
+ const uint8_t *src0_8,
+ uint32_t src0_stride,
+ const uint8_t *src1_8,
+ uint32_t src1_stride, const uint8_t *mask,
+ int w, int h, int bd) {
+ (void)bd;
+
+ const uint16_t *src0 = CONVERT_TO_SHORTPTR(src0_8);
+ const uint16_t *src1 = CONVERT_TO_SHORTPTR(src1_8);
+ uint16_t *dst = CONVERT_TO_SHORTPTR(dst_8);
+
+ assert(IMPLIES(src0 == dst, src0_stride == dst_stride));
+ assert(IMPLIES(src1 == dst, src1_stride == dst_stride));
+
+ assert(h >= 1);
+ assert(w >= 1);
+ assert(IS_POWER_OF_TWO(h));
+ assert(IS_POWER_OF_TWO(w));
+
+ assert(bd == 8 || bd == 10 || bd == 12);
+
+ if (w >= 8) {
+ do {
+ int i = 0;
+ do {
+ uint16x8_t m0 = vmovl_u8(vld1_u8(mask + i));
+ uint16x8_t s0 = vld1q_u16(src0 + i);
+ uint16x8_t s1 = vld1q_u16(src1 + i);
+
+ uint16x8_t blend = alpha_blend_a64_u16x8(m0, s0, s1);
+
+ vst1q_u16(dst + i, blend);
+ i += 8;
+ } while (i < w);
+
+ src0 += src0_stride;
+ src1 += src1_stride;
+ dst += dst_stride;
+ } while (--h != 0);
+ } else if (w == 4) {
+ const uint16x8_t m0 = vmovl_u8(load_unaligned_dup_u8_4x2(mask));
+ do {
+ uint16x8_t s0 = load_unaligned_u16_4x2(src0, src0_stride);
+ uint16x8_t s1 = load_unaligned_u16_4x2(src1, src1_stride);
+
+ uint16x8_t blend = alpha_blend_a64_u16x8(m0, s0, s1);
+
+ store_unaligned_u16_4x2(dst, dst_stride, blend);
+
+ src0 += 2 * src0_stride;
+ src1 += 2 * src1_stride;
+ dst += 2 * dst_stride;
+ h -= 2;
+ } while (h != 0);
+ } else if (w == 2 && h >= 8) {
+ const uint16x4_t m0 =
+ vget_low_u16(vmovl_u8(load_unaligned_dup_u8_2x4(mask)));
+ do {
+ uint16x4_t s0 = load_unaligned_u16_2x2(src0, src0_stride);
+ uint16x4_t s1 = load_unaligned_u16_2x2(src1, src1_stride);
+
+ uint16x4_t blend = alpha_blend_a64_u16x4(m0, s0, s1);
+
+ store_unaligned_u16_2x2(dst, dst_stride, blend);
+
+ src0 += 2 * src0_stride;
+ src1 += 2 * src1_stride;
+ dst += 2 * dst_stride;
+ h -= 2;
+ } while (h != 0);
+ } else {
+ aom_highbd_blend_a64_hmask_c(dst_8, dst_stride, src0_8, src0_stride, src1_8,
+ src1_stride, mask, w, h, bd);
+ }
+}
diff --git a/aom_dsp/arm/highbd_blend_a64_mask_neon.c b/aom_dsp/arm/highbd_blend_a64_mask_neon.c
new file mode 100644
index 0000000..36d763a
--- /dev/null
+++ b/aom_dsp/arm/highbd_blend_a64_mask_neon.c
@@ -0,0 +1,473 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+#include <assert.h>
+
+#include "config/aom_config.h"
+#include "config/aom_dsp_rtcd.h"
+
+#include "aom_dsp/arm/blend_neon.h"
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_dsp/blend.h"
+
+#define HBD_BLEND_A64_D16_MASK(bd, round0_bits) \
+ static INLINE uint16x8_t alpha_##bd##_blend_a64_d16_u16x8( \
+ uint16x8_t m, uint16x8_t a, uint16x8_t b, int32x4_t round_offset) { \
+ const uint16x8_t m_inv = \
+ vsubq_u16(vdupq_n_u16(AOM_BLEND_A64_MAX_ALPHA), m); \
+ \
+ uint32x4_t blend_u32_lo = vmlal_u16(vreinterpretq_u32_s32(round_offset), \
+ vget_low_u16(m), vget_low_u16(a)); \
+ uint32x4_t blend_u32_hi = vmlal_u16(vreinterpretq_u32_s32(round_offset), \
+ vget_high_u16(m), vget_high_u16(a)); \
+ \
+ blend_u32_lo = \
+ vmlal_u16(blend_u32_lo, vget_low_u16(m_inv), vget_low_u16(b)); \
+ blend_u32_hi = \
+ vmlal_u16(blend_u32_hi, vget_high_u16(m_inv), vget_high_u16(b)); \
+ \
+ uint16x4_t blend_u16_lo = \
+ vqrshrun_n_s32(vreinterpretq_s32_u32(blend_u32_lo), \
+ AOM_BLEND_A64_ROUND_BITS + 2 * FILTER_BITS - \
+ round0_bits - COMPOUND_ROUND1_BITS); \
+ uint16x4_t blend_u16_hi = \
+ vqrshrun_n_s32(vreinterpretq_s32_u32(blend_u32_hi), \
+ AOM_BLEND_A64_ROUND_BITS + 2 * FILTER_BITS - \
+ round0_bits - COMPOUND_ROUND1_BITS); \
+ \
+ uint16x8_t blend_u16 = vcombine_u16(blend_u16_lo, blend_u16_hi); \
+ blend_u16 = vminq_u16(blend_u16, vdupq_n_u16((1 << bd) - 1)); \
+ \
+ return blend_u16; \
+ } \
+ \
+ static INLINE void highbd_##bd##_blend_a64_d16_mask_neon( \
+ uint16_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, \
+ uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, \
+ const uint8_t *mask, uint32_t mask_stride, int w, int h, int subw, \
+ int subh) { \
+ const int offset_bits = bd + 2 * FILTER_BITS - round0_bits; \
+ int32_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + \
+ (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); \
+ int32x4_t offset = \
+ vdupq_n_s32(-(round_offset << AOM_BLEND_A64_ROUND_BITS)); \
+ \
+ if ((subw | subh) == 0) { \
+ if (w >= 8) { \
+ do { \
+ int i = 0; \
+ do { \
+ uint16x8_t m0 = vmovl_u8(vld1_u8(mask + i)); \
+ uint16x8_t s0 = vld1q_u16(src0 + i); \
+ uint16x8_t s1 = vld1q_u16(src1 + i); \
+ \
+ uint16x8_t blend = \
+ alpha_##bd##_blend_a64_d16_u16x8(m0, s0, s1, offset); \
+ \
+ vst1q_u16(dst + i, blend); \
+ i += 8; \
+ } while (i < w); \
+ \
+ mask += mask_stride; \
+ src0 += src0_stride; \
+ src1 += src1_stride; \
+ dst += dst_stride; \
+ } while (--h != 0); \
+ } else { \
+ do { \
+ uint16x8_t m0 = vmovl_u8(load_unaligned_u8_4x2(mask, mask_stride)); \
+ uint16x8_t s0 = load_unaligned_u16_4x2(src0, src0_stride); \
+ uint16x8_t s1 = load_unaligned_u16_4x2(src1, src1_stride); \
+ \
+ uint16x8_t blend = \
+ alpha_##bd##_blend_a64_d16_u16x8(m0, s0, s1, offset); \
+ \
+ store_unaligned_u16_4x2(dst, dst_stride, blend); \
+ \
+ mask += 2 * mask_stride; \
+ src0 += 2 * src0_stride; \
+ src1 += 2 * src1_stride; \
+ dst += 2 * dst_stride; \
+ h -= 2; \
+ } while (h != 0); \
+ } \
+ } else if ((subw & subh) == 1) { \
+ if (w >= 8) { \
+ do { \
+ int i = 0; \
+ do { \
+ uint8x16_t m0 = vld1q_u8(mask + 0 * mask_stride + 2 * i); \
+ uint8x16_t m1 = vld1q_u8(mask + 1 * mask_stride + 2 * i); \
+ uint16x8_t s0 = vld1q_u16(src0 + i); \
+ uint16x8_t s1 = vld1q_u16(src1 + i); \
+ \
+ uint16x8_t m_avg = vmovl_u8(avg_blend_pairwise_u8x8_4( \
+ vget_low_u8(m0), vget_low_u8(m1), vget_high_u8(m0), \
+ vget_high_u8(m1))); \
+ uint16x8_t blend = \
+ alpha_##bd##_blend_a64_d16_u16x8(m_avg, s0, s1, offset); \
+ \
+ vst1q_u16(dst + i, blend); \
+ i += 8; \
+ } while (i < w); \
+ \
+ mask += 2 * mask_stride; \
+ src0 += src0_stride; \
+ src1 += src1_stride; \
+ dst += dst_stride; \
+ } while (--h != 0); \
+ } else { \
+ do { \
+ uint8x8_t m0 = vld1_u8(mask + 0 * mask_stride); \
+ uint8x8_t m1 = vld1_u8(mask + 1 * mask_stride); \
+ uint8x8_t m2 = vld1_u8(mask + 2 * mask_stride); \
+ uint8x8_t m3 = vld1_u8(mask + 3 * mask_stride); \
+ uint16x8_t s0 = load_unaligned_u16_4x2(src0, src0_stride); \
+ uint16x8_t s1 = load_unaligned_u16_4x2(src1, src1_stride); \
+ \
+ uint16x8_t m_avg = \
+ vmovl_u8(avg_blend_pairwise_u8x8_4(m0, m1, m2, m3)); \
+ uint16x8_t blend = \
+ alpha_##bd##_blend_a64_d16_u16x8(m_avg, s0, s1, offset); \
+ \
+ store_unaligned_u16_4x2(dst, dst_stride, blend); \
+ \
+ mask += 4 * mask_stride; \
+ src0 += 2 * src0_stride; \
+ src1 += 2 * src1_stride; \
+ dst += 2 * dst_stride; \
+ h -= 2; \
+ } while (h != 0); \
+ } \
+ } else if (subw == 1 && subh == 0) { \
+ if (w >= 8) { \
+ do { \
+ int i = 0; \
+ do { \
+ uint8x8_t m0 = vld1_u8(mask + 2 * i); \
+ uint8x8_t m1 = vld1_u8(mask + 2 * i + 8); \
+ uint16x8_t s0 = vld1q_u16(src0 + i); \
+ uint16x8_t s1 = vld1q_u16(src1 + i); \
+ \
+ uint16x8_t m_avg = vmovl_u8(avg_blend_pairwise_u8x8(m0, m1)); \
+ uint16x8_t blend = \
+ alpha_##bd##_blend_a64_d16_u16x8(m_avg, s0, s1, offset); \
+ \
+ vst1q_u16(dst + i, blend); \
+ i += 8; \
+ } while (i < w); \
+ \
+ mask += mask_stride; \
+ src0 += src0_stride; \
+ src1 += src1_stride; \
+ dst += dst_stride; \
+ } while (--h != 0); \
+ } else { \
+ do { \
+ uint8x8_t m0 = vld1_u8(mask + 0 * mask_stride); \
+ uint8x8_t m1 = vld1_u8(mask + 1 * mask_stride); \
+ uint16x8_t s0 = load_unaligned_u16_4x2(src0, src0_stride); \
+ uint16x8_t s1 = load_unaligned_u16_4x2(src1, src1_stride); \
+ \
+ uint16x8_t m_avg = vmovl_u8(avg_blend_pairwise_u8x8(m0, m1)); \
+ uint16x8_t blend = \
+ alpha_##bd##_blend_a64_d16_u16x8(m_avg, s0, s1, offset); \
+ \
+ store_unaligned_u16_4x2(dst, dst_stride, blend); \
+ \
+ mask += 2 * mask_stride; \
+ src0 += 2 * src0_stride; \
+ src1 += 2 * src1_stride; \
+ dst += 2 * dst_stride; \
+ h -= 2; \
+ } while (h != 0); \
+ } \
+ } else { \
+ if (w >= 8) { \
+ do { \
+ int i = 0; \
+ do { \
+ uint8x8_t m0 = vld1_u8(mask + 0 * mask_stride + i); \
+ uint8x8_t m1 = vld1_u8(mask + 1 * mask_stride + i); \
+ uint16x8_t s0 = vld1q_u16(src0 + i); \
+ uint16x8_t s1 = vld1q_u16(src1 + i); \
+ \
+ uint16x8_t m_avg = vmovl_u8(avg_blend_u8x8(m0, m1)); \
+ uint16x8_t blend = \
+ alpha_##bd##_blend_a64_d16_u16x8(m_avg, s0, s1, offset); \
+ \
+ vst1q_u16(dst + i, blend); \
+ i += 8; \
+ } while (i < w); \
+ \
+ mask += 2 * mask_stride; \
+ src0 += src0_stride; \
+ src1 += src1_stride; \
+ dst += dst_stride; \
+ } while (--h != 0); \
+ } else { \
+ do { \
+ uint8x8_t m0_2 = \
+ load_unaligned_u8_4x2(mask + 0 * mask_stride, 2 * mask_stride); \
+ uint8x8_t m1_3 = \
+ load_unaligned_u8_4x2(mask + 1 * mask_stride, 2 * mask_stride); \
+ uint16x8_t s0 = load_unaligned_u16_4x2(src0, src0_stride); \
+ uint16x8_t s1 = load_unaligned_u16_4x2(src1, src1_stride); \
+ \
+ uint16x8_t m_avg = vmovl_u8(avg_blend_u8x8(m0_2, m1_3)); \
+ uint16x8_t blend = \
+ alpha_##bd##_blend_a64_d16_u16x8(m_avg, s0, s1, offset); \
+ \
+ store_unaligned_u16_4x2(dst, dst_stride, blend); \
+ \
+ mask += 4 * mask_stride; \
+ src0 += 2 * src0_stride; \
+ src1 += 2 * src1_stride; \
+ dst += 2 * dst_stride; \
+ h -= 2; \
+ } while (h != 0); \
+ } \
+ } \
+ }
+
+// 12 bitdepth
+HBD_BLEND_A64_D16_MASK(12, (ROUND0_BITS + 2))
+// 10 bitdepth
+HBD_BLEND_A64_D16_MASK(10, ROUND0_BITS)
+// 8 bitdepth
+HBD_BLEND_A64_D16_MASK(8, ROUND0_BITS)
+
+void aom_highbd_blend_a64_d16_mask_neon(
+ uint8_t *dst_8, uint32_t dst_stride, const CONV_BUF_TYPE *src0,
+ uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride,
+ const uint8_t *mask, uint32_t mask_stride, int w, int h, int subw, int subh,
+ ConvolveParams *conv_params, const int bd) {
+ (void)conv_params;
+ assert(h >= 1);
+ assert(w >= 1);
+ assert(IS_POWER_OF_TWO(h));
+ assert(IS_POWER_OF_TWO(w));
+
+ uint16_t *dst = CONVERT_TO_SHORTPTR(dst_8);
+ assert(IMPLIES(src0 == dst, src0_stride == dst_stride));
+ assert(IMPLIES(src1 == dst, src1_stride == dst_stride));
+
+ if (bd == 12) {
+ highbd_12_blend_a64_d16_mask_neon(dst, dst_stride, src0, src0_stride, src1,
+ src1_stride, mask, mask_stride, w, h,
+ subw, subh);
+ } else if (bd == 10) {
+ highbd_10_blend_a64_d16_mask_neon(dst, dst_stride, src0, src0_stride, src1,
+ src1_stride, mask, mask_stride, w, h,
+ subw, subh);
+ } else {
+ highbd_8_blend_a64_d16_mask_neon(dst, dst_stride, src0, src0_stride, src1,
+ src1_stride, mask, mask_stride, w, h, subw,
+ subh);
+ }
+}
+
+void aom_highbd_blend_a64_mask_neon(uint8_t *dst_8, uint32_t dst_stride,
+ const uint8_t *src0_8, uint32_t src0_stride,
+ const uint8_t *src1_8, uint32_t src1_stride,
+ const uint8_t *mask, uint32_t mask_stride,
+ int w, int h, int subw, int subh, int bd) {
+ (void)bd;
+
+ const uint16_t *src0 = CONVERT_TO_SHORTPTR(src0_8);
+ const uint16_t *src1 = CONVERT_TO_SHORTPTR(src1_8);
+ uint16_t *dst = CONVERT_TO_SHORTPTR(dst_8);
+
+ assert(IMPLIES(src0 == dst, src0_stride == dst_stride));
+ assert(IMPLIES(src1 == dst, src1_stride == dst_stride));
+
+ assert(h >= 1);
+ assert(w >= 1);
+ assert(IS_POWER_OF_TWO(h));
+ assert(IS_POWER_OF_TWO(w));
+
+ assert(bd == 8 || bd == 10 || bd == 12);
+
+ if ((subw | subh) == 0) {
+ if (w >= 8) {
+ do {
+ int i = 0;
+ do {
+ uint16x8_t m0 = vmovl_u8(vld1_u8(mask + i));
+ uint16x8_t s0 = vld1q_u16(src0 + i);
+ uint16x8_t s1 = vld1q_u16(src1 + i);
+
+ uint16x8_t blend = alpha_blend_a64_u16x8(m0, s0, s1);
+
+ vst1q_u16(dst + i, blend);
+ i += 8;
+ } while (i < w);
+
+ mask += mask_stride;
+ src0 += src0_stride;
+ src1 += src1_stride;
+ dst += dst_stride;
+ } while (--h != 0);
+ } else {
+ do {
+ uint16x8_t m0 = vmovl_u8(load_unaligned_u8_4x2(mask, mask_stride));
+ uint16x8_t s0 = load_unaligned_u16_4x2(src0, src0_stride);
+ uint16x8_t s1 = load_unaligned_u16_4x2(src1, src1_stride);
+
+ uint16x8_t blend = alpha_blend_a64_u16x8(m0, s0, s1);
+
+ store_unaligned_u16_4x2(dst, dst_stride, blend);
+
+ mask += 2 * mask_stride;
+ src0 += 2 * src0_stride;
+ src1 += 2 * src1_stride;
+ dst += 2 * dst_stride;
+ h -= 2;
+ } while (h != 0);
+ }
+ } else if ((subw & subh) == 1) {
+ if (w >= 8) {
+ do {
+ int i = 0;
+ do {
+ uint8x8_t m0 = vld1_u8(mask + 0 * mask_stride + 2 * i);
+ uint8x8_t m1 = vld1_u8(mask + 1 * mask_stride + 2 * i);
+ uint8x8_t m2 = vld1_u8(mask + 0 * mask_stride + 2 * i + 8);
+ uint8x8_t m3 = vld1_u8(mask + 1 * mask_stride + 2 * i + 8);
+ uint16x8_t s0 = vld1q_u16(src0 + i);
+ uint16x8_t s1 = vld1q_u16(src1 + i);
+
+ uint16x8_t m_avg =
+ vmovl_u8(avg_blend_pairwise_u8x8_4(m0, m1, m2, m3));
+
+ uint16x8_t blend = alpha_blend_a64_u16x8(m_avg, s0, s1);
+
+ vst1q_u16(dst + i, blend);
+
+ i += 8;
+ } while (i < w);
+
+ mask += 2 * mask_stride;
+ src0 += src0_stride;
+ src1 += src1_stride;
+ dst += dst_stride;
+ } while (--h != 0);
+ } else {
+ do {
+ uint8x8_t m0 = vld1_u8(mask + 0 * mask_stride);
+ uint8x8_t m1 = vld1_u8(mask + 1 * mask_stride);
+ uint8x8_t m2 = vld1_u8(mask + 2 * mask_stride);
+ uint8x8_t m3 = vld1_u8(mask + 3 * mask_stride);
+ uint16x8_t s0 = load_unaligned_u16_4x2(src0, src0_stride);
+ uint16x8_t s1 = load_unaligned_u16_4x2(src1, src1_stride);
+
+ uint16x8_t m_avg = vmovl_u8(avg_blend_pairwise_u8x8_4(m0, m1, m2, m3));
+ uint16x8_t blend = alpha_blend_a64_u16x8(m_avg, s0, s1);
+
+ store_unaligned_u16_4x2(dst, dst_stride, blend);
+
+ mask += 4 * mask_stride;
+ src0 += 2 * src0_stride;
+ src1 += 2 * src1_stride;
+ dst += 2 * dst_stride;
+ h -= 2;
+ } while (h != 0);
+ }
+ } else if (subw == 1 && subh == 0) {
+ if (w >= 8) {
+ do {
+ int i = 0;
+
+ do {
+ uint8x8_t m0 = vld1_u8(mask + 2 * i);
+ uint8x8_t m1 = vld1_u8(mask + 2 * i + 8);
+ uint16x8_t s0 = vld1q_u16(src0 + i);
+ uint16x8_t s1 = vld1q_u16(src1 + i);
+
+ uint16x8_t m_avg = vmovl_u8(avg_blend_pairwise_u8x8(m0, m1));
+ uint16x8_t blend = alpha_blend_a64_u16x8(m_avg, s0, s1);
+
+ vst1q_u16(dst + i, blend);
+
+ i += 8;
+ } while (i < w);
+
+ mask += mask_stride;
+ src0 += src0_stride;
+ src1 += src1_stride;
+ dst += dst_stride;
+ } while (--h != 0);
+ } else {
+ do {
+ uint8x8_t m0 = vld1_u8(mask + 0 * mask_stride);
+ uint8x8_t m1 = vld1_u8(mask + 1 * mask_stride);
+ uint16x8_t s0 = load_unaligned_u16_4x2(src0, src0_stride);
+ uint16x8_t s1 = load_unaligned_u16_4x2(src1, src1_stride);
+
+ uint16x8_t m_avg = vmovl_u8(avg_blend_pairwise_u8x8(m0, m1));
+ uint16x8_t blend = alpha_blend_a64_u16x8(m_avg, s0, s1);
+
+ store_unaligned_u16_4x2(dst, dst_stride, blend);
+
+ mask += 2 * mask_stride;
+ src0 += 2 * src0_stride;
+ src1 += 2 * src1_stride;
+ dst += 2 * dst_stride;
+ h -= 2;
+ } while (h != 0);
+ }
+ } else {
+ if (w >= 8) {
+ do {
+ int i = 0;
+ do {
+ uint8x8_t m0 = vld1_u8(mask + 0 * mask_stride + i);
+ uint8x8_t m1 = vld1_u8(mask + 1 * mask_stride + i);
+ uint16x8_t s0 = vld1q_u16(src0 + i);
+ uint16x8_t s1 = vld1q_u16(src1 + i);
+
+ uint16x8_t m_avg = vmovl_u8(avg_blend_u8x8(m0, m1));
+ uint16x8_t blend = alpha_blend_a64_u16x8(m_avg, s0, s1);
+
+ vst1q_u16(dst + i, blend);
+
+ i += 8;
+ } while (i < w);
+
+ mask += 2 * mask_stride;
+ src0 += src0_stride;
+ src1 += src1_stride;
+ dst += dst_stride;
+ } while (--h != 0);
+ } else {
+ do {
+ uint8x8_t m0_2 =
+ load_unaligned_u8_4x2(mask + 0 * mask_stride, 2 * mask_stride);
+ uint8x8_t m1_3 =
+ load_unaligned_u8_4x2(mask + 1 * mask_stride, 2 * mask_stride);
+ uint16x8_t s0 = load_unaligned_u16_4x2(src0, src0_stride);
+ uint16x8_t s1 = load_unaligned_u16_4x2(src1, src1_stride);
+
+ uint16x8_t m_avg = vmovl_u8(avg_blend_u8x8(m0_2, m1_3));
+ uint16x8_t blend = alpha_blend_a64_u16x8(m_avg, s0, s1);
+
+ store_unaligned_u16_4x2(dst, dst_stride, blend);
+
+ mask += 4 * mask_stride;
+ src0 += 2 * src0_stride;
+ src1 += 2 * src1_stride;
+ dst += 2 * dst_stride;
+ h -= 2;
+ } while (h != 0);
+ }
+ }
+}
diff --git a/aom_dsp/arm/highbd_blend_a64_vmask_neon.c b/aom_dsp/arm/highbd_blend_a64_vmask_neon.c
new file mode 100644
index 0000000..ea3d655
--- /dev/null
+++ b/aom_dsp/arm/highbd_blend_a64_vmask_neon.c
@@ -0,0 +1,105 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+#include <assert.h>
+
+#include "config/aom_config.h"
+#include "config/aom_dsp_rtcd.h"
+
+#include "aom_dsp/arm/blend_neon.h"
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_dsp/blend.h"
+
+void aom_highbd_blend_a64_vmask_neon(uint8_t *dst_8, uint32_t dst_stride,
+ const uint8_t *src0_8,
+ uint32_t src0_stride,
+ const uint8_t *src1_8,
+ uint32_t src1_stride, const uint8_t *mask,
+ int w, int h, int bd) {
+ (void)bd;
+
+ const uint16_t *src0 = CONVERT_TO_SHORTPTR(src0_8);
+ const uint16_t *src1 = CONVERT_TO_SHORTPTR(src1_8);
+ uint16_t *dst = CONVERT_TO_SHORTPTR(dst_8);
+
+ assert(IMPLIES(src0 == dst, src0_stride == dst_stride));
+ assert(IMPLIES(src1 == dst, src1_stride == dst_stride));
+
+ assert(h >= 1);
+ assert(w >= 1);
+ assert(IS_POWER_OF_TWO(h));
+ assert(IS_POWER_OF_TWO(w));
+
+ assert(bd == 8 || bd == 10 || bd == 12);
+
+ if (w >= 8) {
+ do {
+ uint16x8_t m = vmovl_u8(vdup_n_u8(mask[0]));
+ int i = 0;
+ do {
+ uint16x8_t s0 = vld1q_u16(src0 + i);
+ uint16x8_t s1 = vld1q_u16(src1 + i);
+
+ uint16x8_t blend = alpha_blend_a64_u16x8(m, s0, s1);
+
+ vst1q_u16(dst + i, blend);
+ i += 8;
+ } while (i < w);
+
+ mask += 1;
+ src0 += src0_stride;
+ src1 += src1_stride;
+ dst += dst_stride;
+ } while (--h != 0);
+ } else if (w == 4) {
+ do {
+ uint16x4_t m1 = vdup_n_u16((uint16_t)mask[0]);
+ uint16x4_t m2 = vdup_n_u16((uint16_t)mask[1]);
+ uint16x8_t m = vcombine_u16(m1, m2);
+ uint16x8_t s0 = load_unaligned_u16_4x2(src0, src0_stride);
+ uint16x8_t s1 = load_unaligned_u16_4x2(src1, src1_stride);
+
+ uint16x8_t blend = alpha_blend_a64_u16x8(m, s0, s1);
+
+ store_unaligned_u16_4x2(dst, dst_stride, blend);
+
+ mask += 2;
+ src0 += 2 * src0_stride;
+ src1 += 2 * src1_stride;
+ dst += 2 * dst_stride;
+ h -= 2;
+ } while (h != 0);
+ } else if (w == 2 && h >= 8) {
+ do {
+ uint16x4_t m0 = vdup_n_u16(0);
+ m0 = vld1_lane_u16((uint16_t *)mask, m0, 0);
+ uint8x8_t m0_zip =
+ vzip_u8(vreinterpret_u8_u16(m0), vreinterpret_u8_u16(m0)).val[0];
+ m0 = vget_low_u16(vmovl_u8(m0_zip));
+ uint16x4_t s0 = load_unaligned_u16_2x2(src0, src0_stride);
+ uint16x4_t s1 = load_unaligned_u16_2x2(src1, src1_stride);
+
+ uint16x4_t blend = alpha_blend_a64_u16x4(m0, s0, s1);
+
+ store_unaligned_u16_2x2(dst, dst_stride, blend);
+
+ mask += 2;
+ src0 += 2 * src0_stride;
+ src1 += 2 * src1_stride;
+ dst += 2 * dst_stride;
+ h -= 2;
+ } while (h != 0);
+ } else {
+ aom_highbd_blend_a64_vmask_c(dst_8, dst_stride, src0_8, src0_stride, src1_8,
+ src1_stride, mask, w, h, bd);
+ }
+}
diff --git a/aom_dsp/arm/highbd_convolve8_neon.c b/aom_dsp/arm/highbd_convolve8_neon.c
new file mode 100644
index 0000000..e25438c
--- /dev/null
+++ b/aom_dsp/arm/highbd_convolve8_neon.c
@@ -0,0 +1,363 @@
+/*
+ * Copyright (c) 2014 The WebM project authors. All Rights Reserved.
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+#include <assert.h>
+
+#include "config/aom_config.h"
+#include "config/aom_dsp_rtcd.h"
+
+#include "aom/aom_integer.h"
+#include "aom_dsp/aom_dsp_common.h"
+#include "aom_dsp/aom_filter.h"
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_dsp/arm/transpose_neon.h"
+#include "aom_ports/mem.h"
+
+static INLINE int32x4_t highbd_convolve8_4_s32(
+ const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
+ const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
+ const int16x4_t s6, const int16x4_t s7, const int16x8_t y_filter) {
+ const int16x4_t y_filter_lo = vget_low_s16(y_filter);
+ const int16x4_t y_filter_hi = vget_high_s16(y_filter);
+
+ int32x4_t sum = vmull_lane_s16(s0, y_filter_lo, 0);
+ sum = vmlal_lane_s16(sum, s1, y_filter_lo, 1);
+ sum = vmlal_lane_s16(sum, s2, y_filter_lo, 2);
+ sum = vmlal_lane_s16(sum, s3, y_filter_lo, 3);
+ sum = vmlal_lane_s16(sum, s4, y_filter_hi, 0);
+ sum = vmlal_lane_s16(sum, s5, y_filter_hi, 1);
+ sum = vmlal_lane_s16(sum, s6, y_filter_hi, 2);
+ sum = vmlal_lane_s16(sum, s7, y_filter_hi, 3);
+
+ return sum;
+}
+
+static INLINE uint16x4_t highbd_convolve8_4_s32_s16(
+ const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
+ const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
+ const int16x4_t s6, const int16x4_t s7, const int16x8_t y_filter) {
+ int32x4_t sum =
+ highbd_convolve8_4_s32(s0, s1, s2, s3, s4, s5, s6, s7, y_filter);
+
+ return vqrshrun_n_s32(sum, FILTER_BITS);
+}
+
+static INLINE int32x4_t highbd_convolve8_horiz4_s32(
+ const int16x8_t s0, const int16x8_t s1, const int16x8_t x_filter_0_7) {
+ const int16x8_t s2 = vextq_s16(s0, s1, 1);
+ const int16x8_t s3 = vextq_s16(s0, s1, 2);
+ const int16x8_t s4 = vextq_s16(s0, s1, 3);
+ const int16x4_t s0_lo = vget_low_s16(s0);
+ const int16x4_t s1_lo = vget_low_s16(s2);
+ const int16x4_t s2_lo = vget_low_s16(s3);
+ const int16x4_t s3_lo = vget_low_s16(s4);
+ const int16x4_t s4_lo = vget_high_s16(s0);
+ const int16x4_t s5_lo = vget_high_s16(s2);
+ const int16x4_t s6_lo = vget_high_s16(s3);
+ const int16x4_t s7_lo = vget_high_s16(s4);
+
+ return highbd_convolve8_4_s32(s0_lo, s1_lo, s2_lo, s3_lo, s4_lo, s5_lo, s6_lo,
+ s7_lo, x_filter_0_7);
+}
+
+static INLINE uint16x4_t highbd_convolve8_horiz4_s32_s16(
+ const int16x8_t s0, const int16x8_t s1, const int16x8_t x_filter_0_7) {
+ int32x4_t sum = highbd_convolve8_horiz4_s32(s0, s1, x_filter_0_7);
+
+ return vqrshrun_n_s32(sum, FILTER_BITS);
+}
+
+static INLINE void highbd_convolve8_8_s32(
+ const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
+ const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t s6, const int16x8_t s7, const int16x8_t y_filter,
+ int32x4_t *sum0, int32x4_t *sum1) {
+ const int16x4_t y_filter_lo = vget_low_s16(y_filter);
+ const int16x4_t y_filter_hi = vget_high_s16(y_filter);
+
+ *sum0 = vmull_lane_s16(vget_low_s16(s0), y_filter_lo, 0);
+ *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s1), y_filter_lo, 1);
+ *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s2), y_filter_lo, 2);
+ *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s3), y_filter_lo, 3);
+ *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s4), y_filter_hi, 0);
+ *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s5), y_filter_hi, 1);
+ *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s6), y_filter_hi, 2);
+ *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s7), y_filter_hi, 3);
+
+ *sum1 = vmull_lane_s16(vget_high_s16(s0), y_filter_lo, 0);
+ *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s1), y_filter_lo, 1);
+ *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s2), y_filter_lo, 2);
+ *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s3), y_filter_lo, 3);
+ *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s4), y_filter_hi, 0);
+ *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s5), y_filter_hi, 1);
+ *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s6), y_filter_hi, 2);
+ *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s7), y_filter_hi, 3);
+}
+
+static INLINE void highbd_convolve8_horiz8_s32(const int16x8_t s0,
+ const int16x8_t s0_hi,
+ const int16x8_t x_filter_0_7,
+ int32x4_t *sum0,
+ int32x4_t *sum1) {
+ const int16x8_t s1 = vextq_s16(s0, s0_hi, 1);
+ const int16x8_t s2 = vextq_s16(s0, s0_hi, 2);
+ const int16x8_t s3 = vextq_s16(s0, s0_hi, 3);
+ const int16x8_t s4 = vextq_s16(s0, s0_hi, 4);
+ const int16x8_t s5 = vextq_s16(s0, s0_hi, 5);
+ const int16x8_t s6 = vextq_s16(s0, s0_hi, 6);
+ const int16x8_t s7 = vextq_s16(s0, s0_hi, 7);
+
+ highbd_convolve8_8_s32(s0, s1, s2, s3, s4, s5, s6, s7, x_filter_0_7, sum0,
+ sum1);
+}
+
+static INLINE uint16x8_t highbd_convolve8_horiz8_s32_s16(
+ const int16x8_t s0, const int16x8_t s1, const int16x8_t x_filter_0_7) {
+ int32x4_t sum0, sum1;
+ highbd_convolve8_horiz8_s32(s0, s1, x_filter_0_7, &sum0, &sum1);
+
+ return vcombine_u16(vqrshrun_n_s32(sum0, FILTER_BITS),
+ vqrshrun_n_s32(sum1, FILTER_BITS));
+}
+
+static INLINE uint16x8_t highbd_convolve8_8_s32_s16(
+ const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
+ const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t s6, const int16x8_t s7, const int16x8_t y_filter) {
+ int32x4_t sum0;
+ int32x4_t sum1;
+ highbd_convolve8_8_s32(s0, s1, s2, s3, s4, s5, s6, s7, y_filter, &sum0,
+ &sum1);
+
+ return vcombine_u16(vqrshrun_n_s32(sum0, FILTER_BITS),
+ vqrshrun_n_s32(sum1, FILTER_BITS));
+}
+
+static void highbd_convolve_horiz_neon(const uint16_t *src_ptr,
+ ptrdiff_t src_stride, uint16_t *dst_ptr,
+ ptrdiff_t dst_stride,
+ const int16_t *x_filter_ptr,
+ int x_step_q4, int w, int h, int bd) {
+ assert(w >= 4 && h >= 4);
+ const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
+ const int16x8_t x_filter = vld1q_s16(x_filter_ptr);
+
+ if (w == 4) {
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ do {
+ int16x8_t s0, s1, s2, s3;
+ load_s16_8x2(s, src_stride, &s0, &s2);
+ load_s16_8x2(s + 8, src_stride, &s1, &s3);
+
+ uint16x4_t d0 = highbd_convolve8_horiz4_s32_s16(s0, s1, x_filter);
+ uint16x4_t d1 = highbd_convolve8_horiz4_s32_s16(s2, s3, x_filter);
+
+ uint16x8_t d01 = vcombine_u16(d0, d1);
+ d01 = vminq_u16(d01, max);
+
+ vst1_u16(d + 0 * dst_stride, vget_low_u16(d01));
+ vst1_u16(d + 1 * dst_stride, vget_high_u16(d01));
+
+ s += 2 * src_stride;
+ d += 2 * dst_stride;
+ h -= 2;
+ } while (h > 0);
+ } else {
+ int height = h;
+
+ do {
+ int width = w;
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+ int x_q4 = 0;
+
+ const int16_t *src_x = &s[x_q4 >> SUBPEL_BITS];
+ int16x8_t s0, s2, s4, s6;
+ load_s16_8x4(src_x, src_stride, &s0, &s2, &s4, &s6);
+ src_x += 8;
+
+ do {
+ int16x8_t s1, s3, s5, s7;
+ load_s16_8x4(src_x, src_stride, &s1, &s3, &s5, &s7);
+
+ uint16x8_t d0 = highbd_convolve8_horiz8_s32_s16(s0, s1, x_filter);
+ uint16x8_t d1 = highbd_convolve8_horiz8_s32_s16(s2, s3, x_filter);
+ uint16x8_t d2 = highbd_convolve8_horiz8_s32_s16(s4, s5, x_filter);
+ uint16x8_t d3 = highbd_convolve8_horiz8_s32_s16(s6, s7, x_filter);
+
+ d0 = vminq_u16(d0, max);
+ d1 = vminq_u16(d1, max);
+ d2 = vminq_u16(d2, max);
+ d3 = vminq_u16(d3, max);
+
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s0 = s1;
+ s2 = s3;
+ s4 = s5;
+ s6 = s7;
+ src_x += 8;
+ d += 8;
+ width -= 8;
+ x_q4 += 8 * x_step_q4;
+ } while (width > 0);
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ height -= 4;
+ } while (height > 0);
+ }
+}
+
+void aom_highbd_convolve8_horiz_neon(const uint8_t *src8, ptrdiff_t src_stride,
+ uint8_t *dst8, ptrdiff_t dst_stride,
+ const int16_t *filter_x, int x_step_q4,
+ const int16_t *filter_y, int y_step_q4,
+ int w, int h, int bd) {
+ if (x_step_q4 != 16) {
+ aom_highbd_convolve8_horiz_c(src8, src_stride, dst8, dst_stride, filter_x,
+ x_step_q4, filter_y, y_step_q4, w, h, bd);
+ } else {
+ (void)filter_y;
+ (void)y_step_q4;
+
+ uint16_t *src = CONVERT_TO_SHORTPTR(src8);
+ uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
+
+ src -= SUBPEL_TAPS / 2 - 1;
+ highbd_convolve_horiz_neon(src, src_stride, dst, dst_stride, filter_x,
+ x_step_q4, w, h, bd);
+ }
+}
+
+static void highbd_convolve_vert_neon(const uint16_t *src_ptr,
+ ptrdiff_t src_stride, uint16_t *dst_ptr,
+ ptrdiff_t dst_stride,
+ const int16_t *y_filter_ptr, int w, int h,
+ int bd) {
+ assert(w >= 4 && h >= 4);
+ const int16x8_t y_filter = vld1q_s16(y_filter_ptr);
+ const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
+
+ if (w == 4) {
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ int16x4_t s0, s1, s2, s3, s4, s5, s6;
+ load_s16_4x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
+ s += 7 * src_stride;
+
+ do {
+ int16x4_t s7, s8, s9, s10;
+ load_s16_4x4(s, src_stride, &s7, &s8, &s9, &s10);
+
+ uint16x4_t d0 =
+ highbd_convolve8_4_s32_s16(s0, s1, s2, s3, s4, s5, s6, s7, y_filter);
+ uint16x4_t d1 =
+ highbd_convolve8_4_s32_s16(s1, s2, s3, s4, s5, s6, s7, s8, y_filter);
+ uint16x4_t d2 =
+ highbd_convolve8_4_s32_s16(s2, s3, s4, s5, s6, s7, s8, s9, y_filter);
+ uint16x4_t d3 =
+ highbd_convolve8_4_s32_s16(s3, s4, s5, s6, s7, s8, s9, s10, y_filter);
+
+ uint16x8_t d01 = vcombine_u16(d0, d1);
+ uint16x8_t d23 = vcombine_u16(d2, d3);
+
+ d01 = vminq_u16(d01, max);
+ d23 = vminq_u16(d23, max);
+
+ vst1_u16(d + 0 * dst_stride, vget_low_u16(d01));
+ vst1_u16(d + 1 * dst_stride, vget_high_u16(d01));
+ vst1_u16(d + 2 * dst_stride, vget_low_u16(d23));
+ vst1_u16(d + 3 * dst_stride, vget_high_u16(d23));
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s5 = s9;
+ s6 = s10;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ h -= 4;
+ } while (h > 0);
+ } else {
+ do {
+ int height = h;
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ int16x8_t s0, s1, s2, s3, s4, s5, s6;
+ load_s16_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
+ s += 7 * src_stride;
+
+ do {
+ int16x8_t s7, s8, s9, s10;
+ load_s16_8x4(s, src_stride, &s7, &s8, &s9, &s10);
+
+ uint16x8_t d0 = highbd_convolve8_8_s32_s16(s0, s1, s2, s3, s4, s5, s6,
+ s7, y_filter);
+ uint16x8_t d1 = highbd_convolve8_8_s32_s16(s1, s2, s3, s4, s5, s6, s7,
+ s8, y_filter);
+ uint16x8_t d2 = highbd_convolve8_8_s32_s16(s2, s3, s4, s5, s6, s7, s8,
+ s9, y_filter);
+ uint16x8_t d3 = highbd_convolve8_8_s32_s16(s3, s4, s5, s6, s7, s8, s9,
+ s10, y_filter);
+
+ d0 = vminq_u16(d0, max);
+ d1 = vminq_u16(d1, max);
+ d2 = vminq_u16(d2, max);
+ d3 = vminq_u16(d3, max);
+
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s5 = s9;
+ s6 = s10;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ height -= 4;
+ } while (height > 0);
+ src_ptr += 8;
+ dst_ptr += 8;
+ w -= 8;
+ } while (w > 0);
+ }
+}
+
+void aom_highbd_convolve8_vert_neon(const uint8_t *src8, ptrdiff_t src_stride,
+ uint8_t *dst8, ptrdiff_t dst_stride,
+ const int16_t *filter_x, int x_step_q4,
+ const int16_t *filter_y, int y_step_q4,
+ int w, int h, int bd) {
+ if (y_step_q4 != 16) {
+ aom_highbd_convolve8_vert_c(src8, src_stride, dst8, dst_stride, filter_x,
+ x_step_q4, filter_y, y_step_q4, w, h, bd);
+ } else {
+ (void)filter_x;
+ (void)x_step_q4;
+
+ uint16_t *src = CONVERT_TO_SHORTPTR(src8);
+ uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
+
+ src -= (SUBPEL_TAPS / 2 - 1) * src_stride;
+ highbd_convolve_vert_neon(src, src_stride, dst, dst_stride, filter_y, w, h,
+ bd);
+ }
+}
diff --git a/aom_dsp/arm/highbd_hadamard_neon.c b/aom_dsp/arm/highbd_hadamard_neon.c
index aad2046..d28617c 100644
--- a/aom_dsp/arm/highbd_hadamard_neon.c
+++ b/aom_dsp/arm/highbd_hadamard_neon.c
@@ -109,7 +109,7 @@
// For the first pass we can stay in 16-bit elements (4095*8 = 32760).
hadamard_highbd_col8_first_pass(&s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7);
- transpose_s16_8x8(&s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7);
+ transpose_elems_inplace_s16_8x8(&s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7);
// For the second pass we need to widen to 32-bit elements, so we're
// processing 4 columns at a time.
diff --git a/aom_dsp/arm/highbd_intrapred_neon.c b/aom_dsp/arm/highbd_intrapred_neon.c
index 63f53c3..366ca3f 100644
--- a/aom_dsp/arm/highbd_intrapred_neon.c
+++ b/aom_dsp/arm/highbd_intrapred_neon.c
@@ -15,6 +15,7 @@
#include "config/aom_dsp_rtcd.h"
#include "aom/aom_integer.h"
+#include "aom_dsp/arm/sum_neon.h"
#include "aom_dsp/intrapred_common.h"
// -----------------------------------------------------------------------------
@@ -191,7 +192,7 @@
uint16x8_t sum_above = highbd_dc_load_partial_sum_##w(above); \
uint16x8_t sum_left = highbd_dc_load_partial_sum_##h(left); \
uint16x8_t sum_vec = vaddq_u16(sum_left, sum_above); \
- int sum = horizontal_add_and_broadcast_long_u16x8(sum_vec)[0]; \
+ int sum = horizontal_add_u16x8(sum_vec); \
int dc0 = highbd_dc_predictor_rect((w), (h), sum, (shift), (mult)); \
highbd_dc_store_##w##xh(dst, stride, (h), vdup##q##_n_u16(dc0)); \
}
diff --git a/aom_dsp/arm/highbd_loopfilter_neon.c b/aom_dsp/arm/highbd_loopfilter_neon.c
index 2b5128e..77727b7 100644
--- a/aom_dsp/arm/highbd_loopfilter_neon.c
+++ b/aom_dsp/arm/highbd_loopfilter_neon.c
@@ -298,7 +298,7 @@
uint16x4_t src[4] = { vld1_u16(dst_p1), vld1_u16(dst_p0), vld1_u16(dst_q0),
vld1_u16(dst_q1) };
- transpose_u16_4x4(src);
+ transpose_array_inplace_u16_4x4(src);
// Adjust thresholds to bitdepth.
const int outer_thresh = *blimit << (bd - 8);
@@ -344,7 +344,7 @@
vget_high_u16(p0q0_output),
vget_high_u16(p1q1_output),
};
- transpose_u16_4x4(output);
+ transpose_array_inplace_u16_4x4(output);
vst1_u16(dst_p1, output[0]);
vst1_u16(dst_p0, output[1]);
@@ -386,7 +386,7 @@
// p1q1 = p2q2 + 2 * (p2q2 + p1q1 + p0q0) + q0p0
// ^^^^^^ ^^^^^^
// Should dual issue with the left shift.
- const uint16x8_t q0p0 = transpose64_u16q(p0q0);
+ const uint16x8_t q0p0 = vextq_u16(p0q0, p0q0, 4);
const uint16x8_t outer_sum = vaddq_u16(p2q2, q0p0);
sum = vaddq_u16(sum, outer_sum);
@@ -401,7 +401,7 @@
// p0q0 = p1q1 - (2 * p2q2) + q0p0 + q1p1
// ^^^^^^^^
sum = vsubq_u16(sum, p2q2_double);
- const uint16x8_t q1p1 = transpose64_u16q(p1q1);
+ const uint16x8_t q1p1 = vextq_u16(p1q1, p1q1, 4);
sum = vaddq_u16(sum, vaddq_u16(q0p0, q1p1));
*p0q0_output = vrshrq_n_u16(sum, 3);
@@ -505,7 +505,7 @@
// and src_raw[3] after transpose.
uint16x8_t src_raw[4] = { vld1q_u16(dst_0), vld1q_u16(dst_1),
vld1q_u16(dst_2), vld1q_u16(dst_3) };
- transpose_u16_4x8q(src_raw);
+ transpose_array_inplace_u16_4x8(src_raw);
// p2, p1, p0, q0, q1, q2
const uint16x4_t src[6] = {
vget_low_u16(src_raw[0]), vget_low_u16(src_raw[1]),
@@ -574,7 +574,7 @@
vget_high_u16(p0q0_output),
vget_high_u16(p1q1_output),
};
- transpose_u16_4x4(output);
+ transpose_array_inplace_u16_4x4(output);
// dst_n starts at p2, so adjust to p1.
vst1_u16(dst_0 + 1, output[0]);
@@ -626,7 +626,7 @@
// p2q2 = p3q3 + 2 * (p3q3 + p2q2) + p1q1 + p0q0 + q0p0
// ^^^^^^
- const uint16x8_t q0p0 = transpose64_u16q(p0q0);
+ const uint16x8_t q0p0 = vextq_u16(p0q0, p0q0, 4);
sum = vaddq_u16(sum, q0p0);
*p2q2_output = vrshrq_n_u16(sum, 3);
@@ -635,7 +635,7 @@
// p1 = p2 - p3 - p2 + p1 + q1
// q1 = q2 - q3 - q2 + q0 + p1
sum = vsubq_u16(sum, p23q23);
- const uint16x8_t q1p1 = transpose64_u16q(p1q1);
+ const uint16x8_t q1p1 = vextq_u16(p1q1, p1q1, 4);
sum = vaddq_u16(sum, vaddq_u16(p1q1, q1p1));
*p1q1_output = vrshrq_n_u16(sum, 3);
@@ -644,7 +644,7 @@
// p0 = p1 - p3 - p1 + p0 + q2
// q0 = q1 - q3 - q1 + q0 + p2
sum = vsubq_u16(sum, vaddq_u16(p3q3, p1q1));
- const uint16x8_t q2p2 = transpose64_u16q(p2q2);
+ const uint16x8_t q2p2 = vextq_u16(p2q2, p2q2, 4);
sum = vaddq_u16(sum, vaddq_u16(p0q0, q2p2));
*p0q0_output = vrshrq_n_u16(sum, 3);
@@ -827,7 +827,7 @@
uint16x8_t output[4] = { p0q0_output, p1q1_output, p2q2_output, p3q3 };
// After transpose, |output| will contain rows of the form:
// p0 p1 p2 p3 q0 q1 q2 q3
- transpose_u16_4x8q(output);
+ transpose_array_inplace_u16_4x8(output);
// Reverse p values to produce original order:
// p3 p2 p1 p0 q0 q1 q2 q3
@@ -883,7 +883,7 @@
// ^^
// q5 = p0 + q0 + q1 + q2 + q3 + (2 * q4) + (2 * q5) + (7 * q6)
// ^^
- const uint16x8_t q0p0 = transpose64_u16q(p0q0);
+ const uint16x8_t q0p0 = vextq_u16(p0q0, p0q0, 4);
sum = vaddq_u16(sum, q0p0);
*p5q5_output = vrshrq_n_u16(sum, 4);
@@ -892,7 +892,7 @@
// p4 = p5 - (2 * p6) + p3 + q1
// q4 = q5 - (2 * q6) + q3 + p1
sum = vsubq_u16(sum, vshlq_n_u16(p6q6, 1));
- const uint16x8_t q1p1 = transpose64_u16q(p1q1);
+ const uint16x8_t q1p1 = vextq_u16(p1q1, p1q1, 4);
sum = vaddq_u16(vaddq_u16(p3q3, q1p1), sum);
*p4q4_output = vrshrq_n_u16(sum, 4);
@@ -901,7 +901,7 @@
// p3 = p4 - p6 - p5 + p2 + q2
// q3 = q4 - q6 - q5 + q2 + p2
sum = vsubq_u16(sum, vaddq_u16(p6q6, p5q5));
- const uint16x8_t q2p2 = transpose64_u16q(p2q2);
+ const uint16x8_t q2p2 = vextq_u16(p2q2, p2q2, 4);
sum = vaddq_u16(vaddq_u16(p2q2, q2p2), sum);
*p3q3_output = vrshrq_n_u16(sum, 4);
@@ -910,7 +910,7 @@
// p2 = p3 - p6 - p4 + p1 + q3
// q2 = q3 - q6 - q4 + q1 + p3
sum = vsubq_u16(sum, vaddq_u16(p6q6, p4q4));
- const uint16x8_t q3p3 = transpose64_u16q(p3q3);
+ const uint16x8_t q3p3 = vextq_u16(p3q3, p3q3, 4);
sum = vaddq_u16(vaddq_u16(p1q1, q3p3), sum);
*p2q2_output = vrshrq_n_u16(sum, 4);
@@ -919,7 +919,7 @@
// p1 = p2 - p6 - p3 + p0 + q4
// q1 = q2 - q6 - q3 + q0 + p4
sum = vsubq_u16(sum, vaddq_u16(p6q6, p3q3));
- const uint16x8_t q4p4 = transpose64_u16q(p4q4);
+ const uint16x8_t q4p4 = vextq_u16(p4q4, p4q4, 4);
sum = vaddq_u16(vaddq_u16(p0q0, q4p4), sum);
*p1q1_output = vrshrq_n_u16(sum, 4);
@@ -928,7 +928,7 @@
// p0 = p1 - p6 - p2 + q0 + q5
// q0 = q1 - q6 - q2 + p0 + p5
sum = vsubq_u16(sum, vaddq_u16(p6q6, p2q2));
- const uint16x8_t q5p5 = transpose64_u16q(p5q5);
+ const uint16x8_t q5p5 = vextq_u16(p5q5, p5q5, 4);
sum = vaddq_u16(vaddq_u16(q0p0, q5p5), sum);
*p0q0_output = vrshrq_n_u16(sum, 4);
@@ -1118,14 +1118,14 @@
uint16x8_t src_p[4] = { vld1q_u16(dst_0), vld1q_u16(dst_1), vld1q_u16(dst_2),
vld1q_u16(dst_3) };
// p7 will be the low half of src_p[0]. Not used until the end.
- transpose_u16_4x8q(src_p);
+ transpose_array_inplace_u16_4x8(src_p);
// Low halves: q0 q1 q2 q3
// High halves: q4 q5 q6 q7
uint16x8_t src_q[4] = { vld1q_u16(dst_0 + 8), vld1q_u16(dst_1 + 8),
vld1q_u16(dst_2 + 8), vld1q_u16(dst_3 + 8) };
// q7 will be the high half of src_q[3]. Not used until the end.
- transpose_u16_4x8q(src_q);
+ transpose_array_inplace_u16_4x8(src_q);
// Adjust thresholds to bitdepth.
const int outer_thresh = *blimit << (bd - 8);
@@ -1238,10 +1238,10 @@
const uint16x8x2_t p4p0_q0q4 = permute_acdb64(p4q4_output, p0q0_output);
uint16x8_t output_p[4] = { p7p3_q3q7.val[0], p6p2_q2q6.val[0],
p5p1_q1q5.val[0], p4p0_q0q4.val[0] };
- transpose_u16_4x8q(output_p);
+ transpose_array_inplace_u16_4x8(output_p);
uint16x8_t output_q[4] = { p4p0_q0q4.val[1], p5p1_q1q5.val[1],
p6p2_q2q6.val[1], p7p3_q3q7.val[1] };
- transpose_u16_4x8q(output_q);
+ transpose_array_inplace_u16_4x8(output_q);
// Reverse p values to produce original order:
// p3 p2 p1 p0 q0 q1 q2 q3
diff --git a/aom_dsp/arm/highbd_masked_sad_neon.c b/aom_dsp/arm/highbd_masked_sad_neon.c
new file mode 100644
index 0000000..9262d81
--- /dev/null
+++ b/aom_dsp/arm/highbd_masked_sad_neon.c
@@ -0,0 +1,354 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+
+#include "config/aom_config.h"
+#include "config/aom_dsp_rtcd.h"
+
+#include "aom/aom_integer.h"
+#include "aom_dsp/arm/blend_neon.h"
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_dsp/arm/sum_neon.h"
+#include "aom_dsp/blend.h"
+
+static INLINE uint16x8_t masked_sad_8x1_neon(uint16x8_t sad,
+ const uint16_t *src,
+ const uint16_t *a,
+ const uint16_t *b,
+ const uint8_t *m) {
+ const uint16x8_t s0 = vld1q_u16(src);
+ const uint16x8_t a0 = vld1q_u16(a);
+ const uint16x8_t b0 = vld1q_u16(b);
+ const uint16x8_t m0 = vmovl_u8(vld1_u8(m));
+
+ uint16x8_t blend_u16 = alpha_blend_a64_u16x8(m0, a0, b0);
+
+ return vaddq_u16(sad, vabdq_u16(blend_u16, s0));
+}
+
+static INLINE uint16x8_t masked_sad_16x1_neon(uint16x8_t sad,
+ const uint16_t *src,
+ const uint16_t *a,
+ const uint16_t *b,
+ const uint8_t *m) {
+ sad = masked_sad_8x1_neon(sad, src, a, b, m);
+ return masked_sad_8x1_neon(sad, &src[8], &a[8], &b[8], &m[8]);
+}
+
+static INLINE uint16x8_t masked_sad_32x1_neon(uint16x8_t sad,
+ const uint16_t *src,
+ const uint16_t *a,
+ const uint16_t *b,
+ const uint8_t *m) {
+ sad = masked_sad_16x1_neon(sad, src, a, b, m);
+ return masked_sad_16x1_neon(sad, &src[16], &a[16], &b[16], &m[16]);
+}
+
+static INLINE unsigned int masked_sad_128xh_large_neon(
+ const uint8_t *src8, int src_stride, const uint8_t *a8, int a_stride,
+ const uint8_t *b8, int b_stride, const uint8_t *m, int m_stride,
+ int height) {
+ const uint16_t *src = CONVERT_TO_SHORTPTR(src8);
+ const uint16_t *a = CONVERT_TO_SHORTPTR(a8);
+ const uint16_t *b = CONVERT_TO_SHORTPTR(b8);
+ uint32x4_t sad_u32[] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
+ vdupq_n_u32(0) };
+
+ do {
+ uint16x8_t sad[] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0),
+ vdupq_n_u16(0) };
+ for (int h = 0; h < 4; ++h) {
+ sad[0] = masked_sad_32x1_neon(sad[0], src, a, b, m);
+ sad[1] = masked_sad_32x1_neon(sad[1], &src[32], &a[32], &b[32], &m[32]);
+ sad[2] = masked_sad_32x1_neon(sad[2], &src[64], &a[64], &b[64], &m[64]);
+ sad[3] = masked_sad_32x1_neon(sad[3], &src[96], &a[96], &b[96], &m[96]);
+
+ src += src_stride;
+ a += a_stride;
+ b += b_stride;
+ m += m_stride;
+ }
+
+ sad_u32[0] = vpadalq_u16(sad_u32[0], sad[0]);
+ sad_u32[1] = vpadalq_u16(sad_u32[1], sad[1]);
+ sad_u32[2] = vpadalq_u16(sad_u32[2], sad[2]);
+ sad_u32[3] = vpadalq_u16(sad_u32[3], sad[3]);
+ height -= 4;
+ } while (height != 0);
+
+ sad_u32[0] = vaddq_u32(sad_u32[0], sad_u32[1]);
+ sad_u32[2] = vaddq_u32(sad_u32[2], sad_u32[3]);
+ sad_u32[0] = vaddq_u32(sad_u32[0], sad_u32[2]);
+
+ return horizontal_add_u32x4(sad_u32[0]);
+}
+
+static INLINE unsigned int masked_sad_64xh_large_neon(
+ const uint8_t *src8, int src_stride, const uint8_t *a8, int a_stride,
+ const uint8_t *b8, int b_stride, const uint8_t *m, int m_stride,
+ int height) {
+ const uint16_t *src = CONVERT_TO_SHORTPTR(src8);
+ const uint16_t *a = CONVERT_TO_SHORTPTR(a8);
+ const uint16_t *b = CONVERT_TO_SHORTPTR(b8);
+ uint32x4_t sad_u32[] = { vdupq_n_u32(0), vdupq_n_u32(0) };
+
+ do {
+ uint16x8_t sad[] = { vdupq_n_u16(0), vdupq_n_u16(0) };
+ for (int h = 0; h < 4; ++h) {
+ sad[0] = masked_sad_32x1_neon(sad[0], src, a, b, m);
+ sad[1] = masked_sad_32x1_neon(sad[1], &src[32], &a[32], &b[32], &m[32]);
+
+ src += src_stride;
+ a += a_stride;
+ b += b_stride;
+ m += m_stride;
+ }
+
+ sad_u32[0] = vpadalq_u16(sad_u32[0], sad[0]);
+ sad_u32[1] = vpadalq_u16(sad_u32[1], sad[1]);
+ height -= 4;
+ } while (height != 0);
+
+ return horizontal_add_u32x4(vaddq_u32(sad_u32[0], sad_u32[1]));
+}
+
+static INLINE unsigned int masked_sad_32xh_large_neon(
+ const uint8_t *src8, int src_stride, const uint8_t *a8, int a_stride,
+ const uint8_t *b8, int b_stride, const uint8_t *m, int m_stride,
+ int height) {
+ const uint16_t *src = CONVERT_TO_SHORTPTR(src8);
+ const uint16_t *a = CONVERT_TO_SHORTPTR(a8);
+ const uint16_t *b = CONVERT_TO_SHORTPTR(b8);
+ uint32x4_t sad_u32 = vdupq_n_u32(0);
+
+ do {
+ uint16x8_t sad = vdupq_n_u16(0);
+ for (int h = 0; h < 4; ++h) {
+ sad = masked_sad_32x1_neon(sad, src, a, b, m);
+
+ src += src_stride;
+ a += a_stride;
+ b += b_stride;
+ m += m_stride;
+ }
+
+ sad_u32 = vpadalq_u16(sad_u32, sad);
+ height -= 4;
+ } while (height != 0);
+
+ return horizontal_add_u32x4(sad_u32);
+}
+
+static INLINE unsigned int masked_sad_16xh_large_neon(
+ const uint8_t *src8, int src_stride, const uint8_t *a8, int a_stride,
+ const uint8_t *b8, int b_stride, const uint8_t *m, int m_stride,
+ int height) {
+ const uint16_t *src = CONVERT_TO_SHORTPTR(src8);
+ const uint16_t *a = CONVERT_TO_SHORTPTR(a8);
+ const uint16_t *b = CONVERT_TO_SHORTPTR(b8);
+ uint32x4_t sad_u32 = vdupq_n_u32(0);
+
+ do {
+ uint16x8_t sad_u16 = vdupq_n_u16(0);
+
+ for (int h = 0; h < 8; ++h) {
+ sad_u16 = masked_sad_16x1_neon(sad_u16, src, a, b, m);
+
+ src += src_stride;
+ a += a_stride;
+ b += b_stride;
+ m += m_stride;
+ }
+
+ sad_u32 = vpadalq_u16(sad_u32, sad_u16);
+ height -= 8;
+ } while (height != 0);
+
+ return horizontal_add_u32x4(sad_u32);
+}
+
+#if !CONFIG_REALTIME_ONLY
+static INLINE unsigned int masked_sad_8xh_large_neon(
+ const uint8_t *src8, int src_stride, const uint8_t *a8, int a_stride,
+ const uint8_t *b8, int b_stride, const uint8_t *m, int m_stride,
+ int height) {
+ const uint16_t *src = CONVERT_TO_SHORTPTR(src8);
+ const uint16_t *a = CONVERT_TO_SHORTPTR(a8);
+ const uint16_t *b = CONVERT_TO_SHORTPTR(b8);
+ uint32x4_t sad_u32 = vdupq_n_u32(0);
+
+ do {
+ uint16x8_t sad_u16 = vdupq_n_u16(0);
+
+ for (int h = 0; h < 16; ++h) {
+ sad_u16 = masked_sad_8x1_neon(sad_u16, src, a, b, m);
+
+ src += src_stride;
+ a += a_stride;
+ b += b_stride;
+ m += m_stride;
+ }
+
+ sad_u32 = vpadalq_u16(sad_u32, sad_u16);
+ height -= 16;
+ } while (height != 0);
+
+ return horizontal_add_u32x4(sad_u32);
+}
+#endif // !CONFIG_REALTIME_ONLY
+
+static INLINE unsigned int masked_sad_16xh_small_neon(
+ const uint8_t *src8, int src_stride, const uint8_t *a8, int a_stride,
+ const uint8_t *b8, int b_stride, const uint8_t *m, int m_stride,
+ int height) {
+ // For 12-bit data, we can only accumulate up to 128 elements in the
+ // uint16x8_t type sad accumulator, so we can only process up to 8 rows
+ // before we have to accumulate into 32-bit elements.
+ assert(height <= 8);
+ const uint16_t *src = CONVERT_TO_SHORTPTR(src8);
+ const uint16_t *a = CONVERT_TO_SHORTPTR(a8);
+ const uint16_t *b = CONVERT_TO_SHORTPTR(b8);
+ uint16x8_t sad = vdupq_n_u16(0);
+
+ do {
+ sad = masked_sad_16x1_neon(sad, src, a, b, m);
+
+ src += src_stride;
+ a += a_stride;
+ b += b_stride;
+ m += m_stride;
+ } while (--height != 0);
+
+ return horizontal_add_u16x8(sad);
+}
+
+static INLINE unsigned int masked_sad_8xh_small_neon(
+ const uint8_t *src8, int src_stride, const uint8_t *a8, int a_stride,
+ const uint8_t *b8, int b_stride, const uint8_t *m, int m_stride,
+ int height) {
+ // For 12-bit data, we can only accumulate up to 128 elements in the
+ // uint16x8_t type sad accumulator, so we can only process up to 16 rows
+ // before we have to accumulate into 32-bit elements.
+ assert(height <= 16);
+ const uint16_t *src = CONVERT_TO_SHORTPTR(src8);
+ const uint16_t *a = CONVERT_TO_SHORTPTR(a8);
+ const uint16_t *b = CONVERT_TO_SHORTPTR(b8);
+ uint16x8_t sad = vdupq_n_u16(0);
+
+ do {
+ sad = masked_sad_8x1_neon(sad, src, a, b, m);
+
+ src += src_stride;
+ a += a_stride;
+ b += b_stride;
+ m += m_stride;
+ } while (--height != 0);
+
+ return horizontal_add_u16x8(sad);
+}
+
+static INLINE unsigned int masked_sad_4xh_small_neon(
+ const uint8_t *src8, int src_stride, const uint8_t *a8, int a_stride,
+ const uint8_t *b8, int b_stride, const uint8_t *m, int m_stride,
+ int height) {
+ // For 12-bit data, we can only accumulate up to 64 elements in the
+ // uint16x4_t type sad accumulator, so we can only process up to 16 rows
+ // before we have to accumulate into 32-bit elements.
+ assert(height <= 16);
+ const uint16_t *src = CONVERT_TO_SHORTPTR(src8);
+ const uint16_t *a = CONVERT_TO_SHORTPTR(a8);
+ const uint16_t *b = CONVERT_TO_SHORTPTR(b8);
+
+ uint16x4_t sad = vdup_n_u16(0);
+ do {
+ uint16x4_t m0 = vget_low_u16(vmovl_u8(load_unaligned_u8_4x1(m)));
+ uint16x4_t a0 = load_unaligned_u16_4x1(a);
+ uint16x4_t b0 = load_unaligned_u16_4x1(b);
+ uint16x4_t s0 = load_unaligned_u16_4x1(src);
+
+ uint16x4_t blend_u16 = alpha_blend_a64_u16x4(m0, a0, b0);
+
+ sad = vadd_u16(sad, vabd_u16(blend_u16, s0));
+
+ src += src_stride;
+ a += a_stride;
+ b += b_stride;
+ m += m_stride;
+ } while (--height != 0);
+
+ return horizontal_add_u16x4(sad);
+}
+
+#define HIGHBD_MASKED_SAD_WXH_SMALL_NEON(w, h) \
+ unsigned int aom_highbd_masked_sad##w##x##h##_neon( \
+ const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
+ const uint8_t *second_pred, const uint8_t *msk, int msk_stride, \
+ int invert_mask) { \
+ if (!invert_mask) \
+ return masked_sad_##w##xh_small_neon(src, src_stride, ref, ref_stride, \
+ second_pred, w, msk, msk_stride, \
+ h); \
+ else \
+ return masked_sad_##w##xh_small_neon(src, src_stride, second_pred, w, \
+ ref, ref_stride, msk, msk_stride, \
+ h); \
+ }
+
+#define HIGHBD_MASKED_SAD_WXH_LARGE_NEON(w, h) \
+ unsigned int aom_highbd_masked_sad##w##x##h##_neon( \
+ const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
+ const uint8_t *second_pred, const uint8_t *msk, int msk_stride, \
+ int invert_mask) { \
+ if (!invert_mask) \
+ return masked_sad_##w##xh_large_neon(src, src_stride, ref, ref_stride, \
+ second_pred, w, msk, msk_stride, \
+ h); \
+ else \
+ return masked_sad_##w##xh_large_neon(src, src_stride, second_pred, w, \
+ ref, ref_stride, msk, msk_stride, \
+ h); \
+ }
+
+HIGHBD_MASKED_SAD_WXH_SMALL_NEON(4, 4)
+HIGHBD_MASKED_SAD_WXH_SMALL_NEON(4, 8)
+
+HIGHBD_MASKED_SAD_WXH_SMALL_NEON(8, 4)
+HIGHBD_MASKED_SAD_WXH_SMALL_NEON(8, 8)
+HIGHBD_MASKED_SAD_WXH_SMALL_NEON(8, 16)
+
+HIGHBD_MASKED_SAD_WXH_SMALL_NEON(16, 8)
+HIGHBD_MASKED_SAD_WXH_LARGE_NEON(16, 16)
+HIGHBD_MASKED_SAD_WXH_LARGE_NEON(16, 32)
+
+HIGHBD_MASKED_SAD_WXH_LARGE_NEON(32, 16)
+HIGHBD_MASKED_SAD_WXH_LARGE_NEON(32, 32)
+HIGHBD_MASKED_SAD_WXH_LARGE_NEON(32, 64)
+
+HIGHBD_MASKED_SAD_WXH_LARGE_NEON(64, 32)
+HIGHBD_MASKED_SAD_WXH_LARGE_NEON(64, 64)
+HIGHBD_MASKED_SAD_WXH_LARGE_NEON(64, 128)
+
+HIGHBD_MASKED_SAD_WXH_LARGE_NEON(128, 64)
+HIGHBD_MASKED_SAD_WXH_LARGE_NEON(128, 128)
+
+#if !CONFIG_REALTIME_ONLY
+HIGHBD_MASKED_SAD_WXH_SMALL_NEON(4, 16)
+
+HIGHBD_MASKED_SAD_WXH_LARGE_NEON(8, 32)
+
+HIGHBD_MASKED_SAD_WXH_SMALL_NEON(16, 4)
+HIGHBD_MASKED_SAD_WXH_LARGE_NEON(16, 64)
+
+HIGHBD_MASKED_SAD_WXH_LARGE_NEON(32, 8)
+
+HIGHBD_MASKED_SAD_WXH_LARGE_NEON(64, 16)
+#endif // !CONFIG_REALTIME_ONLY
diff --git a/aom_dsp/arm/highbd_obmc_sad_neon.c b/aom_dsp/arm/highbd_obmc_sad_neon.c
new file mode 100644
index 0000000..28699e6
--- /dev/null
+++ b/aom_dsp/arm/highbd_obmc_sad_neon.c
@@ -0,0 +1,211 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+
+#include "config/aom_config.h"
+#include "config/aom_dsp_rtcd.h"
+
+#include "aom/aom_integer.h"
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_dsp/arm/sum_neon.h"
+
+static INLINE void highbd_obmc_sad_8x1_s16_neon(uint16x8_t ref,
+ const int32_t *mask,
+ const int32_t *wsrc,
+ uint32x4_t *sum) {
+ int16x8_t ref_s16 = vreinterpretq_s16_u16(ref);
+
+ int32x4_t wsrc_lo = vld1q_s32(wsrc);
+ int32x4_t wsrc_hi = vld1q_s32(wsrc + 4);
+
+ int32x4_t mask_lo = vld1q_s32(mask);
+ int32x4_t mask_hi = vld1q_s32(mask + 4);
+
+ int16x8_t mask_s16 = vcombine_s16(vmovn_s32(mask_lo), vmovn_s32(mask_hi));
+
+ int32x4_t pre_lo = vmull_s16(vget_low_s16(ref_s16), vget_low_s16(mask_s16));
+ int32x4_t pre_hi = vmull_s16(vget_high_s16(ref_s16), vget_high_s16(mask_s16));
+
+ uint32x4_t abs_lo = vreinterpretq_u32_s32(vabdq_s32(wsrc_lo, pre_lo));
+ uint32x4_t abs_hi = vreinterpretq_u32_s32(vabdq_s32(wsrc_hi, pre_hi));
+
+ *sum = vrsraq_n_u32(*sum, abs_lo, 12);
+ *sum = vrsraq_n_u32(*sum, abs_hi, 12);
+}
+
+static INLINE unsigned int highbd_obmc_sad_4xh_neon(const uint8_t *ref,
+ int ref_stride,
+ const int32_t *wsrc,
+ const int32_t *mask,
+ int height) {
+ const uint16_t *ref_ptr = CONVERT_TO_SHORTPTR(ref);
+ uint32x4_t sum = vdupq_n_u32(0);
+
+ int h = height / 2;
+ do {
+ uint16x8_t r = load_unaligned_u16_4x2(ref_ptr, ref_stride);
+
+ highbd_obmc_sad_8x1_s16_neon(r, mask, wsrc, &sum);
+
+ ref_ptr += 2 * ref_stride;
+ wsrc += 8;
+ mask += 8;
+ } while (--h != 0);
+
+ return horizontal_add_u32x4(sum);
+}
+
+static INLINE unsigned int highbd_obmc_sad_8xh_neon(const uint8_t *ref,
+ int ref_stride,
+ const int32_t *wsrc,
+ const int32_t *mask,
+ int height) {
+ const uint16_t *ref_ptr = CONVERT_TO_SHORTPTR(ref);
+ uint32x4_t sum = vdupq_n_u32(0);
+
+ do {
+ uint16x8_t r = vld1q_u16(ref_ptr);
+
+ highbd_obmc_sad_8x1_s16_neon(r, mask, wsrc, &sum);
+
+ ref_ptr += ref_stride;
+ wsrc += 8;
+ mask += 8;
+ } while (--height != 0);
+
+ return horizontal_add_u32x4(sum);
+}
+
+static INLINE unsigned int highbd_obmc_sad_large_neon(const uint8_t *ref,
+ int ref_stride,
+ const int32_t *wsrc,
+ const int32_t *mask,
+ int width, int height) {
+ const uint16_t *ref_ptr = CONVERT_TO_SHORTPTR(ref);
+ uint32x4_t sum[2] = { vdupq_n_u32(0), vdupq_n_u32(0) };
+
+ do {
+ int i = 0;
+ do {
+ uint16x8_t r0 = vld1q_u16(ref_ptr + i);
+ highbd_obmc_sad_8x1_s16_neon(r0, mask, wsrc, &sum[0]);
+
+ uint16x8_t r1 = vld1q_u16(ref_ptr + i + 8);
+ highbd_obmc_sad_8x1_s16_neon(r1, mask + 8, wsrc + 8, &sum[1]);
+
+ wsrc += 16;
+ mask += 16;
+ i += 16;
+ } while (i < width);
+
+ ref_ptr += ref_stride;
+ } while (--height != 0);
+
+ return horizontal_add_u32x4(vaddq_u32(sum[0], sum[1]));
+}
+
+static INLINE unsigned int highbd_obmc_sad_16xh_neon(const uint8_t *ref,
+ int ref_stride,
+ const int32_t *wsrc,
+ const int32_t *mask,
+ int h) {
+ return highbd_obmc_sad_large_neon(ref, ref_stride, wsrc, mask, 16, h);
+}
+
+static INLINE unsigned int highbd_obmc_sad_32xh_neon(const uint8_t *ref,
+ int ref_stride,
+ const int32_t *wsrc,
+ const int32_t *mask,
+ int height) {
+ uint32x4_t sum[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
+ vdupq_n_u32(0) };
+ const uint16_t *ref_ptr = CONVERT_TO_SHORTPTR(ref);
+
+ do {
+ uint16x8_t r0 = vld1q_u16(ref_ptr);
+ uint16x8_t r1 = vld1q_u16(ref_ptr + 8);
+ uint16x8_t r2 = vld1q_u16(ref_ptr + 16);
+ uint16x8_t r3 = vld1q_u16(ref_ptr + 24);
+
+ highbd_obmc_sad_8x1_s16_neon(r0, mask, wsrc, &sum[0]);
+ highbd_obmc_sad_8x1_s16_neon(r1, mask + 8, wsrc + 8, &sum[1]);
+ highbd_obmc_sad_8x1_s16_neon(r2, mask + 16, wsrc + 16, &sum[2]);
+ highbd_obmc_sad_8x1_s16_neon(r3, mask + 24, wsrc + 24, &sum[3]);
+
+ wsrc += 32;
+ mask += 32;
+ ref_ptr += ref_stride;
+ } while (--height != 0);
+
+ sum[0] = vaddq_u32(sum[0], sum[1]);
+ sum[2] = vaddq_u32(sum[2], sum[3]);
+
+ return horizontal_add_u32x4(vaddq_u32(sum[0], sum[2]));
+}
+
+static INLINE unsigned int highbd_obmc_sad_64xh_neon(const uint8_t *ref,
+ int ref_stride,
+ const int32_t *wsrc,
+ const int32_t *mask,
+ int h) {
+ return highbd_obmc_sad_large_neon(ref, ref_stride, wsrc, mask, 64, h);
+}
+
+static INLINE unsigned int highbd_obmc_sad_128xh_neon(const uint8_t *ref,
+ int ref_stride,
+ const int32_t *wsrc,
+ const int32_t *mask,
+ int h) {
+ return highbd_obmc_sad_large_neon(ref, ref_stride, wsrc, mask, 128, h);
+}
+
+#define HIGHBD_OBMC_SAD_WXH_NEON(w, h) \
+ unsigned int aom_highbd_obmc_sad##w##x##h##_neon( \
+ const uint8_t *ref, int ref_stride, const int32_t *wsrc, \
+ const int32_t *mask) { \
+ return highbd_obmc_sad_##w##xh_neon(ref, ref_stride, wsrc, mask, h); \
+ }
+
+HIGHBD_OBMC_SAD_WXH_NEON(4, 4)
+HIGHBD_OBMC_SAD_WXH_NEON(4, 8)
+
+HIGHBD_OBMC_SAD_WXH_NEON(8, 4)
+HIGHBD_OBMC_SAD_WXH_NEON(8, 8)
+HIGHBD_OBMC_SAD_WXH_NEON(8, 16)
+
+HIGHBD_OBMC_SAD_WXH_NEON(16, 8)
+HIGHBD_OBMC_SAD_WXH_NEON(16, 16)
+HIGHBD_OBMC_SAD_WXH_NEON(16, 32)
+
+HIGHBD_OBMC_SAD_WXH_NEON(32, 16)
+HIGHBD_OBMC_SAD_WXH_NEON(32, 32)
+HIGHBD_OBMC_SAD_WXH_NEON(32, 64)
+
+HIGHBD_OBMC_SAD_WXH_NEON(64, 32)
+HIGHBD_OBMC_SAD_WXH_NEON(64, 64)
+HIGHBD_OBMC_SAD_WXH_NEON(64, 128)
+
+HIGHBD_OBMC_SAD_WXH_NEON(128, 64)
+HIGHBD_OBMC_SAD_WXH_NEON(128, 128)
+
+#if !CONFIG_REALTIME_ONLY
+HIGHBD_OBMC_SAD_WXH_NEON(4, 16)
+
+HIGHBD_OBMC_SAD_WXH_NEON(8, 32)
+
+HIGHBD_OBMC_SAD_WXH_NEON(16, 4)
+HIGHBD_OBMC_SAD_WXH_NEON(16, 64)
+
+HIGHBD_OBMC_SAD_WXH_NEON(32, 8)
+
+HIGHBD_OBMC_SAD_WXH_NEON(64, 16)
+#endif // !CONFIG_REALTIME_ONLY
diff --git a/aom_dsp/arm/highbd_obmc_variance_neon.c b/aom_dsp/arm/highbd_obmc_variance_neon.c
new file mode 100644
index 0000000..d592246
--- /dev/null
+++ b/aom_dsp/arm/highbd_obmc_variance_neon.c
@@ -0,0 +1,369 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+
+#include "config/aom_config.h"
+#include "config/aom_dsp_rtcd.h"
+
+#include "aom/aom_integer.h"
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_dsp/arm/sum_neon.h"
+
+static INLINE void highbd_obmc_variance_8x1_s16_neon(uint16x8_t pre,
+ const int32_t *wsrc,
+ const int32_t *mask,
+ uint32x4_t *sse,
+ int32x4_t *sum) {
+ int16x8_t pre_s16 = vreinterpretq_s16_u16(pre);
+ int32x4_t wsrc_lo = vld1q_s32(&wsrc[0]);
+ int32x4_t wsrc_hi = vld1q_s32(&wsrc[4]);
+
+ int32x4_t mask_lo = vld1q_s32(&mask[0]);
+ int32x4_t mask_hi = vld1q_s32(&mask[4]);
+
+ int16x8_t mask_s16 = vcombine_s16(vmovn_s32(mask_lo), vmovn_s32(mask_hi));
+
+ int32x4_t diff_lo = vmull_s16(vget_low_s16(pre_s16), vget_low_s16(mask_s16));
+ int32x4_t diff_hi =
+ vmull_s16(vget_high_s16(pre_s16), vget_high_s16(mask_s16));
+
+ diff_lo = vsubq_s32(wsrc_lo, diff_lo);
+ diff_hi = vsubq_s32(wsrc_hi, diff_hi);
+
+ // ROUND_POWER_OF_TWO_SIGNED(value, 12) rounds to nearest with ties away
+ // from zero, however vrshrq_n_s32 rounds to nearest with ties rounded up.
+ // This difference only affects the bit patterns at the rounding breakpoints
+ // exactly, so we can add -1 to all negative numbers to move the breakpoint
+ // one value across and into the correct rounding region.
+ diff_lo = vsraq_n_s32(diff_lo, diff_lo, 31);
+ diff_hi = vsraq_n_s32(diff_hi, diff_hi, 31);
+ int32x4_t round_lo = vrshrq_n_s32(diff_lo, 12);
+ int32x4_t round_hi = vrshrq_n_s32(diff_hi, 12);
+
+ *sum = vaddq_s32(*sum, round_lo);
+ *sum = vaddq_s32(*sum, round_hi);
+ *sse = vmlaq_u32(*sse, vreinterpretq_u32_s32(round_lo),
+ vreinterpretq_u32_s32(round_lo));
+ *sse = vmlaq_u32(*sse, vreinterpretq_u32_s32(round_hi),
+ vreinterpretq_u32_s32(round_hi));
+}
+
+// For 12-bit data, we can only accumulate up to 256 elements in the unsigned
+// 32-bit elements (4095*4095*256 = 4292870400) before we have to accumulate
+// into 64-bit elements. Therefore blocks of size 32x64, 64x32, 64x64, 64x128,
+// 128x64, 128x128 are processed in a different helper function.
+static INLINE void highbd_obmc_variance_xlarge_neon(
+ const uint8_t *pre, int pre_stride, const int32_t *wsrc,
+ const int32_t *mask, int width, int h, int h_limit, uint64_t *sse,
+ int64_t *sum) {
+ uint16_t *pre_ptr = CONVERT_TO_SHORTPTR(pre);
+ int32x4_t sum_s32 = vdupq_n_s32(0);
+ uint64x2_t sse_u64 = vdupq_n_u64(0);
+
+ // 'h_limit' is the number of 'w'-width rows we can process before our 32-bit
+ // accumulator overflows. After hitting this limit we accumulate into 64-bit
+ // elements.
+ int h_tmp = h > h_limit ? h_limit : h;
+
+ do {
+ uint32x4_t sse_u32[2] = { vdupq_n_u32(0), vdupq_n_u32(0) };
+ int j = 0;
+
+ do {
+ int i = 0;
+
+ do {
+ uint16x8_t pre0 = vld1q_u16(pre_ptr + i);
+ highbd_obmc_variance_8x1_s16_neon(pre0, wsrc, mask, &sse_u32[0],
+ &sum_s32);
+
+ uint16x8_t pre1 = vld1q_u16(pre_ptr + i + 8);
+ highbd_obmc_variance_8x1_s16_neon(pre1, wsrc + 8, mask + 8, &sse_u32[1],
+ &sum_s32);
+
+ i += 16;
+ wsrc += 16;
+ mask += 16;
+ } while (i < width);
+
+ pre_ptr += pre_stride;
+ j++;
+ } while (j < h_tmp);
+
+ sse_u64 = vpadalq_u32(sse_u64, sse_u32[0]);
+ sse_u64 = vpadalq_u32(sse_u64, sse_u32[1]);
+ h -= h_tmp;
+ } while (h != 0);
+
+ *sse = horizontal_add_u64x2(sse_u64);
+ *sum = horizontal_long_add_s32x4(sum_s32);
+}
+
+static INLINE void highbd_obmc_variance_xlarge_neon_128xh(
+ const uint8_t *pre, int pre_stride, const int32_t *wsrc,
+ const int32_t *mask, int h, uint64_t *sse, int64_t *sum) {
+ highbd_obmc_variance_xlarge_neon(pre, pre_stride, wsrc, mask, 128, h, 16, sse,
+ sum);
+}
+
+static INLINE void highbd_obmc_variance_xlarge_neon_64xh(
+ const uint8_t *pre, int pre_stride, const int32_t *wsrc,
+ const int32_t *mask, int h, uint64_t *sse, int64_t *sum) {
+ highbd_obmc_variance_xlarge_neon(pre, pre_stride, wsrc, mask, 64, h, 32, sse,
+ sum);
+}
+
+static INLINE void highbd_obmc_variance_xlarge_neon_32xh(
+ const uint8_t *pre, int pre_stride, const int32_t *wsrc,
+ const int32_t *mask, int h, uint64_t *sse, int64_t *sum) {
+ highbd_obmc_variance_xlarge_neon(pre, pre_stride, wsrc, mask, 32, h, 64, sse,
+ sum);
+}
+
+static INLINE void highbd_obmc_variance_large_neon(
+ const uint8_t *pre, int pre_stride, const int32_t *wsrc,
+ const int32_t *mask, int width, int h, uint64_t *sse, int64_t *sum) {
+ uint16_t *pre_ptr = CONVERT_TO_SHORTPTR(pre);
+ uint32x4_t sse_u32 = vdupq_n_u32(0);
+ int32x4_t sum_s32 = vdupq_n_s32(0);
+
+ do {
+ int i = 0;
+ do {
+ uint16x8_t pre0 = vld1q_u16(pre_ptr + i);
+ highbd_obmc_variance_8x1_s16_neon(pre0, wsrc, mask, &sse_u32, &sum_s32);
+
+ uint16x8_t pre1 = vld1q_u16(pre_ptr + i + 8);
+ highbd_obmc_variance_8x1_s16_neon(pre1, wsrc + 8, mask + 8, &sse_u32,
+ &sum_s32);
+
+ i += 16;
+ wsrc += 16;
+ mask += 16;
+ } while (i < width);
+
+ pre_ptr += pre_stride;
+ } while (--h != 0);
+
+ *sse = horizontal_long_add_u32x4(sse_u32);
+ *sum = horizontal_long_add_s32x4(sum_s32);
+}
+
+static INLINE void highbd_obmc_variance_neon_128xh(
+ const uint8_t *pre, int pre_stride, const int32_t *wsrc,
+ const int32_t *mask, int h, uint64_t *sse, int64_t *sum) {
+ highbd_obmc_variance_large_neon(pre, pre_stride, wsrc, mask, 128, h, sse,
+ sum);
+}
+
+static INLINE void highbd_obmc_variance_neon_64xh(const uint8_t *pre,
+ int pre_stride,
+ const int32_t *wsrc,
+ const int32_t *mask, int h,
+ uint64_t *sse, int64_t *sum) {
+ highbd_obmc_variance_large_neon(pre, pre_stride, wsrc, mask, 64, h, sse, sum);
+}
+
+static INLINE void highbd_obmc_variance_neon_32xh(const uint8_t *pre,
+ int pre_stride,
+ const int32_t *wsrc,
+ const int32_t *mask, int h,
+ uint64_t *sse, int64_t *sum) {
+ highbd_obmc_variance_large_neon(pre, pre_stride, wsrc, mask, 32, h, sse, sum);
+}
+
+static INLINE void highbd_obmc_variance_neon_16xh(const uint8_t *pre,
+ int pre_stride,
+ const int32_t *wsrc,
+ const int32_t *mask, int h,
+ uint64_t *sse, int64_t *sum) {
+ highbd_obmc_variance_large_neon(pre, pre_stride, wsrc, mask, 16, h, sse, sum);
+}
+
+static INLINE void highbd_obmc_variance_neon_8xh(const uint8_t *pre8,
+ int pre_stride,
+ const int32_t *wsrc,
+ const int32_t *mask, int h,
+ uint64_t *sse, int64_t *sum) {
+ uint16_t *pre = CONVERT_TO_SHORTPTR(pre8);
+ uint32x4_t sse_u32 = vdupq_n_u32(0);
+ int32x4_t sum_s32 = vdupq_n_s32(0);
+
+ do {
+ uint16x8_t pre_u16 = vld1q_u16(pre);
+
+ highbd_obmc_variance_8x1_s16_neon(pre_u16, wsrc, mask, &sse_u32, &sum_s32);
+
+ pre += pre_stride;
+ wsrc += 8;
+ mask += 8;
+ } while (--h != 0);
+
+ *sse = horizontal_long_add_u32x4(sse_u32);
+ *sum = horizontal_long_add_s32x4(sum_s32);
+}
+
+static INLINE void highbd_obmc_variance_neon_4xh(const uint8_t *pre8,
+ int pre_stride,
+ const int32_t *wsrc,
+ const int32_t *mask, int h,
+ uint64_t *sse, int64_t *sum) {
+ assert(h % 2 == 0);
+ uint16_t *pre = CONVERT_TO_SHORTPTR(pre8);
+ uint32x4_t sse_u32 = vdupq_n_u32(0);
+ int32x4_t sum_s32 = vdupq_n_s32(0);
+
+ do {
+ uint16x8_t pre_u16 = load_unaligned_u16_4x2(pre, pre_stride);
+
+ highbd_obmc_variance_8x1_s16_neon(pre_u16, wsrc, mask, &sse_u32, &sum_s32);
+
+ pre += 2 * pre_stride;
+ wsrc += 8;
+ mask += 8;
+ h -= 2;
+ } while (h != 0);
+
+ *sse = horizontal_long_add_u32x4(sse_u32);
+ *sum = horizontal_long_add_s32x4(sum_s32);
+}
+
+static INLINE void highbd_8_obmc_variance_cast(int64_t sum64, uint64_t sse64,
+ int *sum, unsigned int *sse) {
+ *sum = (int)sum64;
+ *sse = (unsigned int)sse64;
+}
+
+static INLINE void highbd_10_obmc_variance_cast(int64_t sum64, uint64_t sse64,
+ int *sum, unsigned int *sse) {
+ *sum = (int)ROUND_POWER_OF_TWO(sum64, 2);
+ *sse = (unsigned int)ROUND_POWER_OF_TWO(sse64, 4);
+}
+
+static INLINE void highbd_12_obmc_variance_cast(int64_t sum64, uint64_t sse64,
+ int *sum, unsigned int *sse) {
+ *sum = (int)ROUND_POWER_OF_TWO(sum64, 4);
+ *sse = (unsigned int)ROUND_POWER_OF_TWO(sse64, 8);
+}
+
+#define HIGHBD_OBMC_VARIANCE_WXH_NEON(w, h, bitdepth) \
+ unsigned int aom_highbd_##bitdepth##_obmc_variance##w##x##h##_neon( \
+ const uint8_t *pre, int pre_stride, const int32_t *wsrc, \
+ const int32_t *mask, unsigned int *sse) { \
+ int sum; \
+ int64_t sum64; \
+ uint64_t sse64; \
+ highbd_obmc_variance_neon_##w##xh(pre, pre_stride, wsrc, mask, h, &sse64, \
+ &sum64); \
+ highbd_##bitdepth##_obmc_variance_cast(sum64, sse64, &sum, sse); \
+ return *sse - (unsigned int)(((int64_t)sum * sum) / (w * h)); \
+ }
+
+#define HIGHBD_OBMC_VARIANCE_WXH_XLARGE_NEON(w, h, bitdepth) \
+ unsigned int aom_highbd_##bitdepth##_obmc_variance##w##x##h##_neon( \
+ const uint8_t *pre, int pre_stride, const int32_t *wsrc, \
+ const int32_t *mask, unsigned int *sse) { \
+ int sum; \
+ int64_t sum64; \
+ uint64_t sse64; \
+ highbd_obmc_variance_xlarge_neon_##w##xh(pre, pre_stride, wsrc, mask, h, \
+ &sse64, &sum64); \
+ highbd_##bitdepth##_obmc_variance_cast(sum64, sse64, &sum, sse); \
+ return *sse - (unsigned int)(((int64_t)sum * sum) / (w * h)); \
+ }
+
+// 8-bit
+HIGHBD_OBMC_VARIANCE_WXH_NEON(4, 4, 8)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(4, 8, 8)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(4, 16, 8)
+
+HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 4, 8)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 8, 8)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 16, 8)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 32, 8)
+
+HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 4, 8)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 8, 8)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 16, 8)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 32, 8)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 64, 8)
+
+HIGHBD_OBMC_VARIANCE_WXH_NEON(32, 8, 8)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(32, 16, 8)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(32, 32, 8)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(32, 64, 8)
+
+HIGHBD_OBMC_VARIANCE_WXH_NEON(64, 16, 8)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(64, 32, 8)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(64, 64, 8)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(64, 128, 8)
+
+HIGHBD_OBMC_VARIANCE_WXH_NEON(128, 64, 8)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(128, 128, 8)
+
+// 10-bit
+HIGHBD_OBMC_VARIANCE_WXH_NEON(4, 4, 10)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(4, 8, 10)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(4, 16, 10)
+
+HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 4, 10)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 8, 10)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 16, 10)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 32, 10)
+
+HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 4, 10)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 8, 10)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 16, 10)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 32, 10)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 64, 10)
+
+HIGHBD_OBMC_VARIANCE_WXH_NEON(32, 8, 10)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(32, 16, 10)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(32, 32, 10)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(32, 64, 10)
+
+HIGHBD_OBMC_VARIANCE_WXH_NEON(64, 16, 10)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(64, 32, 10)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(64, 64, 10)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(64, 128, 10)
+
+HIGHBD_OBMC_VARIANCE_WXH_NEON(128, 64, 10)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(128, 128, 10)
+
+// 12-bit
+HIGHBD_OBMC_VARIANCE_WXH_NEON(4, 4, 12)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(4, 8, 12)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(4, 16, 12)
+
+HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 4, 12)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 8, 12)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 16, 12)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 32, 12)
+
+HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 4, 12)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 8, 12)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 16, 12)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 32, 12)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 64, 12)
+
+HIGHBD_OBMC_VARIANCE_WXH_NEON(32, 8, 12)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(32, 16, 12)
+HIGHBD_OBMC_VARIANCE_WXH_NEON(32, 32, 12)
+HIGHBD_OBMC_VARIANCE_WXH_XLARGE_NEON(32, 64, 12)
+
+HIGHBD_OBMC_VARIANCE_WXH_NEON(64, 16, 12)
+HIGHBD_OBMC_VARIANCE_WXH_XLARGE_NEON(64, 32, 12)
+HIGHBD_OBMC_VARIANCE_WXH_XLARGE_NEON(64, 64, 12)
+HIGHBD_OBMC_VARIANCE_WXH_XLARGE_NEON(64, 128, 12)
+
+HIGHBD_OBMC_VARIANCE_WXH_XLARGE_NEON(128, 64, 12)
+HIGHBD_OBMC_VARIANCE_WXH_XLARGE_NEON(128, 128, 12)
diff --git a/aom_dsp/arm/highbd_quantize_neon.c b/aom_dsp/arm/highbd_quantize_neon.c
index 77a7aac..6149c9f 100644
--- a/aom_dsp/arm/highbd_quantize_neon.c
+++ b/aom_dsp/arm/highbd_quantize_neon.c
@@ -11,14 +11,11 @@
#include <arm_neon.h>
#include <assert.h>
+#include <string.h>
#include "config/aom_config.h"
#include "aom_dsp/quantize.h"
-#include "aom_dsp/arm/mem_neon.h"
-
-#include "av1/common/quant_common.h"
-#include "av1/encoder/av1_quantize.h"
static INLINE uint32_t sum_abs_coeff(const uint32x4_t a) {
#if AOM_ARCH_AARCH64
@@ -83,6 +80,7 @@
return vmaxq_s16(v_eobmax, v_nz_iscan);
}
+#if !CONFIG_REALTIME_ONLY
static INLINE void get_min_max_lane_eob(const int16_t *iscan,
int16x8_t *v_eobmin,
int16x8_t *v_eobmax, uint16x8_t v_mask,
@@ -91,13 +89,14 @@
const int16x8_t v_nz_iscan_max = vbslq_s16(v_mask, v_iscan, vdupq_n_s16(-1));
#if SKIP_EOB_FACTOR_ADJUST
const int16x8_t v_nz_iscan_min =
- vbslq_s16(v_mask, v_iscan, vdupq_n_s16(n_coeffs));
+ vbslq_s16(v_mask, v_iscan, vdupq_n_s16((int16_t)n_coeffs));
*v_eobmin = vminq_s16(*v_eobmin, v_nz_iscan_min);
#else
(void)v_eobmin;
#endif
*v_eobmax = vmaxq_s16(*v_eobmax, v_nz_iscan_max);
}
+#endif // !CONFIG_REALTIME_ONLY
static INLINE uint16_t get_max_eob(int16x8_t v_eobmax) {
#if AOM_ARCH_AARCH64
@@ -117,6 +116,7 @@
#endif
}
+#if SKIP_EOB_FACTOR_ADJUST && !CONFIG_REALTIME_ONLY
static INLINE uint16_t get_min_eob(int16x8_t v_eobmin) {
#if AOM_ARCH_AARCH64
return (uint16_t)vminvq_s16(v_eobmin);
@@ -134,6 +134,7 @@
return (uint16_t)vget_lane_s16(v_eobmin_final, 0);
#endif
}
+#endif // SKIP_EOB_FACTOR_ADJUST && !CONFIG_REALTIME_ONLY
static void highbd_quantize_b_neon(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
@@ -298,7 +299,7 @@
int32x4_t v_zbin_s32 = vmovl_s16(v_zbin);
uint16x4_t v_mask_lo, v_mask_hi;
int16x8_t v_eobmax = vdupq_n_s16(-1);
- int16x8_t v_eobmin = vdupq_n_s16(n_coeffs);
+ int16x8_t v_eobmin = vdupq_n_s16((int16_t)n_coeffs);
assert(n_coeffs > 8);
// Pre-scan pass
diff --git a/aom_dsp/arm/highbd_sad4d_neon.c b/aom_dsp/arm/highbd_sad4d_neon.c
deleted file mode 100644
index f2fda36..0000000
--- a/aom_dsp/arm/highbd_sad4d_neon.c
+++ /dev/null
@@ -1,360 +0,0 @@
-/*
- * Copyright (c) 2023 The WebM project authors. All Rights Reserved.
- * Copyright (c) 2023, Alliance for Open Media. All rights reserved
- *
- * This source code is subject to the terms of the BSD 2 Clause License and
- * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
- * was not distributed with this source code in the LICENSE file, you can
- * obtain it at www.aomedia.org/license/software. If the Alliance for Open
- * Media Patent License 1.0 was not distributed with this source code in the
- * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
- */
-
-#include <arm_neon.h>
-
-#include "config/aom_config.h"
-#include "config/aom_dsp_rtcd.h"
-
-#include "aom/aom_integer.h"
-#include "aom_dsp/arm/mem_neon.h"
-#include "aom_dsp/arm/sum_neon.h"
-
-static INLINE void highbd_sad4xhx4d_small_neon(const uint8_t *src_ptr,
- int src_stride,
- const uint8_t *const ref_ptr[4],
- int ref_stride, uint32_t res[4],
- int h) {
- const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr);
- const uint16_t *ref16_ptr0 = CONVERT_TO_SHORTPTR(ref_ptr[0]);
- const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]);
- const uint16_t *ref16_ptr2 = CONVERT_TO_SHORTPTR(ref_ptr[2]);
- const uint16_t *ref16_ptr3 = CONVERT_TO_SHORTPTR(ref_ptr[3]);
-
- uint32x4_t sum[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
- vdupq_n_u32(0) };
-
- int i = 0;
- do {
- uint16x4_t s = vld1_u16(src16_ptr + i * src_stride);
- uint16x4_t r0 = vld1_u16(ref16_ptr0 + i * ref_stride);
- uint16x4_t r1 = vld1_u16(ref16_ptr1 + i * ref_stride);
- uint16x4_t r2 = vld1_u16(ref16_ptr2 + i * ref_stride);
- uint16x4_t r3 = vld1_u16(ref16_ptr3 + i * ref_stride);
-
- sum[0] = vabal_u16(sum[0], s, r0);
- sum[1] = vabal_u16(sum[1], s, r1);
- sum[2] = vabal_u16(sum[2], s, r2);
- sum[3] = vabal_u16(sum[3], s, r3);
-
- } while (++i < h);
-
- vst1q_u32(res, horizontal_add_4d_u32x4(sum));
-}
-
-static INLINE void highbd_sad8xhx4d_small_neon(const uint8_t *src_ptr,
- int src_stride,
- const uint8_t *const ref_ptr[4],
- int ref_stride, uint32_t res[4],
- int h) {
- const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr);
- const uint16_t *ref16_ptr0 = CONVERT_TO_SHORTPTR(ref_ptr[0]);
- const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]);
- const uint16_t *ref16_ptr2 = CONVERT_TO_SHORTPTR(ref_ptr[2]);
- const uint16_t *ref16_ptr3 = CONVERT_TO_SHORTPTR(ref_ptr[3]);
-
- uint16x8_t sum[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0),
- vdupq_n_u16(0) };
- uint32x4_t sum_u32[4];
-
- int i = 0;
- do {
- uint16x8_t s = vld1q_u16(src16_ptr + i * src_stride);
-
- sum[0] = vabaq_u16(sum[0], s, vld1q_u16(ref16_ptr0 + i * ref_stride));
- sum[1] = vabaq_u16(sum[1], s, vld1q_u16(ref16_ptr1 + i * ref_stride));
- sum[2] = vabaq_u16(sum[2], s, vld1q_u16(ref16_ptr2 + i * ref_stride));
- sum[3] = vabaq_u16(sum[3], s, vld1q_u16(ref16_ptr3 + i * ref_stride));
-
- } while (++i < h);
-
- sum_u32[0] = vpaddlq_u16(sum[0]);
- sum_u32[1] = vpaddlq_u16(sum[1]);
- sum_u32[2] = vpaddlq_u16(sum[2]);
- sum_u32[3] = vpaddlq_u16(sum[3]);
- vst1q_u32(res, horizontal_add_4d_u32x4(sum_u32));
-}
-
-static INLINE void sad8_neon(uint16x8_t src, uint16x8_t ref,
- uint32x4_t *const sad_sum) {
- uint16x8_t abs_diff = vabdq_u16(src, ref);
- *sad_sum = vpadalq_u16(*sad_sum, abs_diff);
-}
-
-static INLINE void highbd_sad8xhx4d_large_neon(const uint8_t *src_ptr,
- int src_stride,
- const uint8_t *const ref_ptr[4],
- int ref_stride, uint32_t res[4],
- int h) {
- const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr);
- const uint16_t *ref16_ptr0 = CONVERT_TO_SHORTPTR(ref_ptr[0]);
- const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]);
- const uint16_t *ref16_ptr2 = CONVERT_TO_SHORTPTR(ref_ptr[2]);
- const uint16_t *ref16_ptr3 = CONVERT_TO_SHORTPTR(ref_ptr[3]);
-
- uint32x4_t sum[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
- vdupq_n_u32(0) };
-
- int i = 0;
- do {
- uint16x8_t s = vld1q_u16(src16_ptr + i * src_stride);
- sad8_neon(s, vld1q_u16(ref16_ptr0 + i * ref_stride), &sum[0]);
- sad8_neon(s, vld1q_u16(ref16_ptr1 + i * ref_stride), &sum[1]);
- sad8_neon(s, vld1q_u16(ref16_ptr2 + i * ref_stride), &sum[2]);
- sad8_neon(s, vld1q_u16(ref16_ptr3 + i * ref_stride), &sum[3]);
-
- } while (++i < h);
-
- vst1q_u32(res, horizontal_add_4d_u32x4(sum));
-}
-
-static INLINE void highbd_sad16xhx4d_large_neon(const uint8_t *src_ptr,
- int src_stride,
- const uint8_t *const ref_ptr[4],
- int ref_stride, uint32_t res[4],
- int h) {
- const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr);
- const uint16_t *ref16_ptr0 = CONVERT_TO_SHORTPTR(ref_ptr[0]);
- const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]);
- const uint16_t *ref16_ptr2 = CONVERT_TO_SHORTPTR(ref_ptr[2]);
- const uint16_t *ref16_ptr3 = CONVERT_TO_SHORTPTR(ref_ptr[3]);
-
- uint32x4_t sum_lo[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
- vdupq_n_u32(0) };
- uint32x4_t sum_hi[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
- vdupq_n_u32(0) };
- uint32x4_t sum[4];
-
- int i = 0;
- do {
- uint16x8_t s0 = vld1q_u16(src16_ptr + i * src_stride);
- sad8_neon(s0, vld1q_u16(ref16_ptr0 + i * ref_stride), &sum_lo[0]);
- sad8_neon(s0, vld1q_u16(ref16_ptr1 + i * ref_stride), &sum_lo[1]);
- sad8_neon(s0, vld1q_u16(ref16_ptr2 + i * ref_stride), &sum_lo[2]);
- sad8_neon(s0, vld1q_u16(ref16_ptr3 + i * ref_stride), &sum_lo[3]);
-
- uint16x8_t s1 = vld1q_u16(src16_ptr + i * src_stride + 8);
- sad8_neon(s1, vld1q_u16(ref16_ptr0 + i * ref_stride + 8), &sum_hi[0]);
- sad8_neon(s1, vld1q_u16(ref16_ptr1 + i * ref_stride + 8), &sum_hi[1]);
- sad8_neon(s1, vld1q_u16(ref16_ptr2 + i * ref_stride + 8), &sum_hi[2]);
- sad8_neon(s1, vld1q_u16(ref16_ptr3 + i * ref_stride + 8), &sum_hi[3]);
-
- } while (++i < h);
-
- sum[0] = vaddq_u32(sum_lo[0], sum_hi[0]);
- sum[1] = vaddq_u32(sum_lo[1], sum_hi[1]);
- sum[2] = vaddq_u32(sum_lo[2], sum_hi[2]);
- sum[3] = vaddq_u32(sum_lo[3], sum_hi[3]);
-
- vst1q_u32(res, horizontal_add_4d_u32x4(sum));
-}
-
-static INLINE void highbd_sadwxhx4d_large_neon(const uint8_t *src_ptr,
- int src_stride,
- const uint8_t *const ref_ptr[4],
- int ref_stride, uint32_t res[4],
- int w, int h) {
- const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr);
- const uint16_t *ref16_ptr0 = CONVERT_TO_SHORTPTR(ref_ptr[0]);
- const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]);
- const uint16_t *ref16_ptr2 = CONVERT_TO_SHORTPTR(ref_ptr[2]);
- const uint16_t *ref16_ptr3 = CONVERT_TO_SHORTPTR(ref_ptr[3]);
-
- uint32x4_t sum_lo[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
- vdupq_n_u32(0) };
- uint32x4_t sum_hi[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
- vdupq_n_u32(0) };
- uint32x4_t sum[4];
-
- int i = 0;
- do {
- int j = 0;
- do {
- uint16x8_t s0 = vld1q_u16(src16_ptr + i * src_stride + j);
- sad8_neon(s0, vld1q_u16(ref16_ptr0 + i * ref_stride + j), &sum_lo[0]);
- sad8_neon(s0, vld1q_u16(ref16_ptr1 + i * ref_stride + j), &sum_lo[1]);
- sad8_neon(s0, vld1q_u16(ref16_ptr2 + i * ref_stride + j), &sum_lo[2]);
- sad8_neon(s0, vld1q_u16(ref16_ptr3 + i * ref_stride + j), &sum_lo[3]);
-
- uint16x8_t s1 = vld1q_u16(src16_ptr + i * src_stride + j + 8);
- sad8_neon(s1, vld1q_u16(ref16_ptr0 + i * ref_stride + j + 8), &sum_hi[0]);
- sad8_neon(s1, vld1q_u16(ref16_ptr1 + i * ref_stride + j + 8), &sum_hi[1]);
- sad8_neon(s1, vld1q_u16(ref16_ptr2 + i * ref_stride + j + 8), &sum_hi[2]);
- sad8_neon(s1, vld1q_u16(ref16_ptr3 + i * ref_stride + j + 8), &sum_hi[3]);
-
- uint16x8_t s2 = vld1q_u16(src16_ptr + i * src_stride + j + 16);
- sad8_neon(s2, vld1q_u16(ref16_ptr0 + i * ref_stride + j + 16),
- &sum_lo[0]);
- sad8_neon(s2, vld1q_u16(ref16_ptr1 + i * ref_stride + j + 16),
- &sum_lo[1]);
- sad8_neon(s2, vld1q_u16(ref16_ptr2 + i * ref_stride + j + 16),
- &sum_lo[2]);
- sad8_neon(s2, vld1q_u16(ref16_ptr3 + i * ref_stride + j + 16),
- &sum_lo[3]);
-
- uint16x8_t s3 = vld1q_u16(src16_ptr + i * src_stride + j + 24);
- sad8_neon(s3, vld1q_u16(ref16_ptr0 + i * ref_stride + j + 24),
- &sum_hi[0]);
- sad8_neon(s3, vld1q_u16(ref16_ptr1 + i * ref_stride + j + 24),
- &sum_hi[1]);
- sad8_neon(s3, vld1q_u16(ref16_ptr2 + i * ref_stride + j + 24),
- &sum_hi[2]);
- sad8_neon(s3, vld1q_u16(ref16_ptr3 + i * ref_stride + j + 24),
- &sum_hi[3]);
-
- j += 32;
- } while (j < w);
-
- } while (++i < h);
-
- sum[0] = vaddq_u32(sum_lo[0], sum_hi[0]);
- sum[1] = vaddq_u32(sum_lo[1], sum_hi[1]);
- sum[2] = vaddq_u32(sum_lo[2], sum_hi[2]);
- sum[3] = vaddq_u32(sum_lo[3], sum_hi[3]);
-
- vst1q_u32(res, horizontal_add_4d_u32x4(sum));
-}
-
-static INLINE void highbd_sad128xhx4d_large_neon(
- const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_ptr[4],
- int ref_stride, uint32_t res[4], int h) {
- highbd_sadwxhx4d_large_neon(src_ptr, src_stride, ref_ptr, ref_stride, res,
- 128, h);
-}
-
-static INLINE void highbd_sad64xhx4d_large_neon(const uint8_t *src_ptr,
- int src_stride,
- const uint8_t *const ref_ptr[4],
- int ref_stride, uint32_t res[4],
- int h) {
- highbd_sadwxhx4d_large_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 64,
- h);
-}
-
-static INLINE void highbd_sad32xhx4d_large_neon(const uint8_t *src_ptr,
- int src_stride,
- const uint8_t *const ref_ptr[4],
- int ref_stride, uint32_t res[4],
- int h) {
- highbd_sadwxhx4d_large_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 32,
- h);
-}
-
-#define HBD_SAD_WXH_4D_SMALL_NEON(w, h) \
- void aom_highbd_sad##w##x##h##x4d_neon( \
- const uint8_t *src, int src_stride, const uint8_t *const ref_array[4], \
- int ref_stride, uint32_t sad_array[4]) { \
- highbd_sad##w##xhx4d_small_neon(src, src_stride, ref_array, ref_stride, \
- sad_array, (h)); \
- }
-
-#define HBD_SAD_WXH_4D_LARGE_NEON(w, h) \
- void aom_highbd_sad##w##x##h##x4d_neon( \
- const uint8_t *src, int src_stride, const uint8_t *const ref_array[4], \
- int ref_stride, uint32_t sad_array[4]) { \
- highbd_sad##w##xhx4d_large_neon(src, src_stride, ref_array, ref_stride, \
- sad_array, (h)); \
- }
-
-HBD_SAD_WXH_4D_SMALL_NEON(4, 4)
-HBD_SAD_WXH_4D_SMALL_NEON(4, 8)
-
-HBD_SAD_WXH_4D_SMALL_NEON(8, 4)
-HBD_SAD_WXH_4D_SMALL_NEON(8, 8)
-HBD_SAD_WXH_4D_SMALL_NEON(8, 16)
-
-HBD_SAD_WXH_4D_LARGE_NEON(16, 8)
-HBD_SAD_WXH_4D_LARGE_NEON(16, 16)
-HBD_SAD_WXH_4D_LARGE_NEON(16, 32)
-
-HBD_SAD_WXH_4D_LARGE_NEON(32, 16)
-HBD_SAD_WXH_4D_LARGE_NEON(32, 32)
-HBD_SAD_WXH_4D_LARGE_NEON(32, 64)
-
-HBD_SAD_WXH_4D_LARGE_NEON(64, 32)
-HBD_SAD_WXH_4D_LARGE_NEON(64, 64)
-HBD_SAD_WXH_4D_LARGE_NEON(64, 128)
-
-HBD_SAD_WXH_4D_LARGE_NEON(128, 64)
-HBD_SAD_WXH_4D_LARGE_NEON(128, 128)
-
-#if !CONFIG_REALTIME_ONLY
-HBD_SAD_WXH_4D_SMALL_NEON(4, 16)
-
-HBD_SAD_WXH_4D_LARGE_NEON(8, 32)
-
-HBD_SAD_WXH_4D_LARGE_NEON(16, 4)
-HBD_SAD_WXH_4D_LARGE_NEON(16, 64)
-
-HBD_SAD_WXH_4D_LARGE_NEON(32, 8)
-
-HBD_SAD_WXH_4D_LARGE_NEON(64, 16)
-#endif // !CONFIG_REALTIME_ONLY
-
-#define HBD_SAD_SKIP_WXH_4D_SMALL_NEON(w, h) \
- void aom_highbd_sad_skip_##w##x##h##x4d_neon( \
- const uint8_t *src, int src_stride, const uint8_t *const ref_array[4], \
- int ref_stride, uint32_t sad_array[4]) { \
- highbd_sad##w##xhx4d_small_neon(src, 2 * src_stride, ref_array, \
- 2 * ref_stride, sad_array, ((h) >> 1)); \
- sad_array[0] <<= 1; \
- sad_array[1] <<= 1; \
- sad_array[2] <<= 1; \
- sad_array[3] <<= 1; \
- }
-
-#define HBD_SAD_SKIP_WXH_4D_LARGE_NEON(w, h) \
- void aom_highbd_sad_skip_##w##x##h##x4d_neon( \
- const uint8_t *src, int src_stride, const uint8_t *const ref_array[4], \
- int ref_stride, uint32_t sad_array[4]) { \
- highbd_sad##w##xhx4d_large_neon(src, 2 * src_stride, ref_array, \
- 2 * ref_stride, sad_array, ((h) >> 1)); \
- sad_array[0] <<= 1; \
- sad_array[1] <<= 1; \
- sad_array[2] <<= 1; \
- sad_array[3] <<= 1; \
- }
-
-HBD_SAD_SKIP_WXH_4D_SMALL_NEON(4, 4)
-HBD_SAD_SKIP_WXH_4D_SMALL_NEON(4, 8)
-
-HBD_SAD_SKIP_WXH_4D_SMALL_NEON(8, 4)
-HBD_SAD_SKIP_WXH_4D_SMALL_NEON(8, 8)
-HBD_SAD_SKIP_WXH_4D_SMALL_NEON(8, 16)
-
-HBD_SAD_SKIP_WXH_4D_LARGE_NEON(16, 8)
-HBD_SAD_SKIP_WXH_4D_LARGE_NEON(16, 16)
-HBD_SAD_SKIP_WXH_4D_LARGE_NEON(16, 32)
-
-HBD_SAD_SKIP_WXH_4D_LARGE_NEON(32, 16)
-HBD_SAD_SKIP_WXH_4D_LARGE_NEON(32, 32)
-HBD_SAD_SKIP_WXH_4D_LARGE_NEON(32, 64)
-
-HBD_SAD_SKIP_WXH_4D_LARGE_NEON(64, 32)
-HBD_SAD_SKIP_WXH_4D_LARGE_NEON(64, 64)
-HBD_SAD_SKIP_WXH_4D_LARGE_NEON(64, 128)
-
-HBD_SAD_SKIP_WXH_4D_LARGE_NEON(128, 64)
-HBD_SAD_SKIP_WXH_4D_LARGE_NEON(128, 128)
-
-#if !CONFIG_REALTIME_ONLY
-HBD_SAD_SKIP_WXH_4D_SMALL_NEON(4, 16)
-
-HBD_SAD_SKIP_WXH_4D_SMALL_NEON(8, 32)
-
-HBD_SAD_SKIP_WXH_4D_LARGE_NEON(16, 4)
-HBD_SAD_SKIP_WXH_4D_LARGE_NEON(16, 64)
-
-HBD_SAD_SKIP_WXH_4D_LARGE_NEON(32, 8)
-
-HBD_SAD_SKIP_WXH_4D_LARGE_NEON(64, 16)
-#endif // !CONFIG_REALTIME_ONLY
diff --git a/aom_dsp/arm/highbd_sad_neon.c b/aom_dsp/arm/highbd_sad_neon.c
index 919eb55..d51f639 100644
--- a/aom_dsp/arm/highbd_sad_neon.c
+++ b/aom_dsp/arm/highbd_sad_neon.c
@@ -61,6 +61,7 @@
return horizontal_add_u16x8(sum);
}
+#if !CONFIG_REALTIME_ONLY
static INLINE uint32_t highbd_sad8xh_large_neon(const uint8_t *src_ptr,
int src_stride,
const uint8_t *ref_ptr,
@@ -82,6 +83,7 @@
return horizontal_add_u32x4(sum_u32);
}
+#endif // !CONFIG_REALTIME_ONLY
static INLINE uint32_t highbd_sad16xh_large_neon(const uint8_t *src_ptr,
int src_stride,
@@ -283,3 +285,225 @@
HBD_SAD_SKIP_WXH_LARGE_NEON(64, 16)
#endif // !CONFIG_REALTIME_ONLY
+
+static INLINE uint32_t highbd_sad4xh_avg_neon(const uint8_t *src_ptr,
+ int src_stride,
+ const uint8_t *ref_ptr,
+ int ref_stride, int h,
+ const uint8_t *second_pred) {
+ const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr);
+ const uint16_t *ref16_ptr = CONVERT_TO_SHORTPTR(ref_ptr);
+ const uint16_t *pred16_ptr = CONVERT_TO_SHORTPTR(second_pred);
+ uint32x4_t sum = vdupq_n_u32(0);
+
+ int i = h;
+ do {
+ uint16x4_t s = vld1_u16(src16_ptr);
+ uint16x4_t r = vld1_u16(ref16_ptr);
+ uint16x4_t p = vld1_u16(pred16_ptr);
+
+ uint16x4_t avg = vrhadd_u16(r, p);
+ sum = vabal_u16(sum, s, avg);
+
+ src16_ptr += src_stride;
+ ref16_ptr += ref_stride;
+ pred16_ptr += 4;
+ } while (--i != 0);
+
+ return horizontal_add_u32x4(sum);
+}
+
+static INLINE uint32_t highbd_sad8xh_avg_neon(const uint8_t *src_ptr,
+ int src_stride,
+ const uint8_t *ref_ptr,
+ int ref_stride, int h,
+ const uint8_t *second_pred) {
+ const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr);
+ const uint16_t *ref16_ptr = CONVERT_TO_SHORTPTR(ref_ptr);
+ const uint16_t *pred16_ptr = CONVERT_TO_SHORTPTR(second_pred);
+ uint32x4_t sum = vdupq_n_u32(0);
+
+ int i = h;
+ do {
+ uint16x8_t s = vld1q_u16(src16_ptr);
+ uint16x8_t r = vld1q_u16(ref16_ptr);
+ uint16x8_t p = vld1q_u16(pred16_ptr);
+
+ uint16x8_t avg = vrhaddq_u16(r, p);
+ uint16x8_t diff = vabdq_u16(s, avg);
+ sum = vpadalq_u16(sum, diff);
+
+ src16_ptr += src_stride;
+ ref16_ptr += ref_stride;
+ pred16_ptr += 8;
+ } while (--i != 0);
+
+ return horizontal_add_u32x4(sum);
+}
+
+static INLINE uint32_t highbd_sad16xh_avg_neon(const uint8_t *src_ptr,
+ int src_stride,
+ const uint8_t *ref_ptr,
+ int ref_stride, int h,
+ const uint8_t *second_pred) {
+ const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr);
+ const uint16_t *ref16_ptr = CONVERT_TO_SHORTPTR(ref_ptr);
+ const uint16_t *pred16_ptr = CONVERT_TO_SHORTPTR(second_pred);
+ uint32x4_t sum[2] = { vdupq_n_u32(0), vdupq_n_u32(0) };
+
+ int i = h;
+ do {
+ uint16x8_t s0, s1, r0, r1, p0, p1;
+ uint16x8_t avg0, avg1, diff0, diff1;
+
+ s0 = vld1q_u16(src16_ptr);
+ r0 = vld1q_u16(ref16_ptr);
+ p0 = vld1q_u16(pred16_ptr);
+ avg0 = vrhaddq_u16(r0, p0);
+ diff0 = vabdq_u16(s0, avg0);
+ sum[0] = vpadalq_u16(sum[0], diff0);
+
+ s1 = vld1q_u16(src16_ptr + 8);
+ r1 = vld1q_u16(ref16_ptr + 8);
+ p1 = vld1q_u16(pred16_ptr + 8);
+ avg1 = vrhaddq_u16(r1, p1);
+ diff1 = vabdq_u16(s1, avg1);
+ sum[1] = vpadalq_u16(sum[1], diff1);
+
+ src16_ptr += src_stride;
+ ref16_ptr += ref_stride;
+ pred16_ptr += 16;
+ } while (--i != 0);
+
+ sum[0] = vaddq_u32(sum[0], sum[1]);
+ return horizontal_add_u32x4(sum[0]);
+}
+
+static INLINE uint32_t highbd_sadwxh_avg_neon(const uint8_t *src_ptr,
+ int src_stride,
+ const uint8_t *ref_ptr,
+ int ref_stride, int w, int h,
+ const uint8_t *second_pred) {
+ const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr);
+ const uint16_t *ref16_ptr = CONVERT_TO_SHORTPTR(ref_ptr);
+ const uint16_t *pred16_ptr = CONVERT_TO_SHORTPTR(second_pred);
+ uint32x4_t sum[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
+ vdupq_n_u32(0) };
+
+ int i = h;
+ do {
+ int j = 0;
+ do {
+ uint16x8_t s0, s1, s2, s3, r0, r1, r2, r3, p0, p1, p2, p3;
+ uint16x8_t avg0, avg1, avg2, avg3, diff0, diff1, diff2, diff3;
+
+ s0 = vld1q_u16(src16_ptr + j);
+ r0 = vld1q_u16(ref16_ptr + j);
+ p0 = vld1q_u16(pred16_ptr + j);
+ avg0 = vrhaddq_u16(r0, p0);
+ diff0 = vabdq_u16(s0, avg0);
+ sum[0] = vpadalq_u16(sum[0], diff0);
+
+ s1 = vld1q_u16(src16_ptr + j + 8);
+ r1 = vld1q_u16(ref16_ptr + j + 8);
+ p1 = vld1q_u16(pred16_ptr + j + 8);
+ avg1 = vrhaddq_u16(r1, p1);
+ diff1 = vabdq_u16(s1, avg1);
+ sum[1] = vpadalq_u16(sum[1], diff1);
+
+ s2 = vld1q_u16(src16_ptr + j + 16);
+ r2 = vld1q_u16(ref16_ptr + j + 16);
+ p2 = vld1q_u16(pred16_ptr + j + 16);
+ avg2 = vrhaddq_u16(r2, p2);
+ diff2 = vabdq_u16(s2, avg2);
+ sum[2] = vpadalq_u16(sum[2], diff2);
+
+ s3 = vld1q_u16(src16_ptr + j + 24);
+ r3 = vld1q_u16(ref16_ptr + j + 24);
+ p3 = vld1q_u16(pred16_ptr + j + 24);
+ avg3 = vrhaddq_u16(r3, p3);
+ diff3 = vabdq_u16(s3, avg3);
+ sum[3] = vpadalq_u16(sum[3], diff3);
+
+ j += 32;
+ } while (j < w);
+
+ src16_ptr += src_stride;
+ ref16_ptr += ref_stride;
+ pred16_ptr += w;
+ } while (--i != 0);
+
+ sum[0] = vaddq_u32(sum[0], sum[1]);
+ sum[2] = vaddq_u32(sum[2], sum[3]);
+ sum[0] = vaddq_u32(sum[0], sum[2]);
+
+ return horizontal_add_u32x4(sum[0]);
+}
+
+static INLINE unsigned int highbd_sad128xh_avg_neon(
+ const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr,
+ int ref_stride, int h, const uint8_t *second_pred) {
+ return highbd_sadwxh_avg_neon(src_ptr, src_stride, ref_ptr, ref_stride, 128,
+ h, second_pred);
+}
+
+static INLINE unsigned int highbd_sad64xh_avg_neon(const uint8_t *src_ptr,
+ int src_stride,
+ const uint8_t *ref_ptr,
+ int ref_stride, int h,
+ const uint8_t *second_pred) {
+ return highbd_sadwxh_avg_neon(src_ptr, src_stride, ref_ptr, ref_stride, 64, h,
+ second_pred);
+}
+
+static INLINE unsigned int highbd_sad32xh_avg_neon(const uint8_t *src_ptr,
+ int src_stride,
+ const uint8_t *ref_ptr,
+ int ref_stride, int h,
+ const uint8_t *second_pred) {
+ return highbd_sadwxh_avg_neon(src_ptr, src_stride, ref_ptr, ref_stride, 32, h,
+ second_pred);
+}
+
+#define HBD_SAD_WXH_AVG_NEON(w, h) \
+ uint32_t aom_highbd_sad##w##x##h##_avg_neon( \
+ const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
+ const uint8_t *second_pred) { \
+ return highbd_sad##w##xh_avg_neon(src, src_stride, ref, ref_stride, (h), \
+ second_pred); \
+ }
+
+HBD_SAD_WXH_AVG_NEON(4, 4)
+HBD_SAD_WXH_AVG_NEON(4, 8)
+
+HBD_SAD_WXH_AVG_NEON(8, 4)
+HBD_SAD_WXH_AVG_NEON(8, 8)
+HBD_SAD_WXH_AVG_NEON(8, 16)
+
+HBD_SAD_WXH_AVG_NEON(16, 8)
+HBD_SAD_WXH_AVG_NEON(16, 16)
+HBD_SAD_WXH_AVG_NEON(16, 32)
+
+HBD_SAD_WXH_AVG_NEON(32, 16)
+HBD_SAD_WXH_AVG_NEON(32, 32)
+HBD_SAD_WXH_AVG_NEON(32, 64)
+
+HBD_SAD_WXH_AVG_NEON(64, 32)
+HBD_SAD_WXH_AVG_NEON(64, 64)
+HBD_SAD_WXH_AVG_NEON(64, 128)
+
+HBD_SAD_WXH_AVG_NEON(128, 64)
+HBD_SAD_WXH_AVG_NEON(128, 128)
+
+#if !CONFIG_REALTIME_ONLY
+HBD_SAD_WXH_AVG_NEON(4, 16)
+
+HBD_SAD_WXH_AVG_NEON(8, 32)
+
+HBD_SAD_WXH_AVG_NEON(16, 4)
+HBD_SAD_WXH_AVG_NEON(16, 64)
+
+HBD_SAD_WXH_AVG_NEON(32, 8)
+
+HBD_SAD_WXH_AVG_NEON(64, 16)
+#endif // !CONFIG_REALTIME_ONLY
diff --git a/aom_dsp/arm/highbd_sadxd_neon.c b/aom_dsp/arm/highbd_sadxd_neon.c
new file mode 100644
index 0000000..85ca673
--- /dev/null
+++ b/aom_dsp/arm/highbd_sadxd_neon.c
@@ -0,0 +1,617 @@
+/*
+ * Copyright (c) 2023 The WebM project authors. All Rights Reserved.
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+
+#include "config/aom_config.h"
+#include "config/aom_dsp_rtcd.h"
+
+#include "aom/aom_integer.h"
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_dsp/arm/sum_neon.h"
+
+static INLINE void highbd_sad4xhx4d_small_neon(const uint8_t *src_ptr,
+ int src_stride,
+ const uint8_t *const ref_ptr[4],
+ int ref_stride, uint32_t res[4],
+ int h) {
+ const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr);
+ const uint16_t *ref16_ptr0 = CONVERT_TO_SHORTPTR(ref_ptr[0]);
+ const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]);
+ const uint16_t *ref16_ptr2 = CONVERT_TO_SHORTPTR(ref_ptr[2]);
+ const uint16_t *ref16_ptr3 = CONVERT_TO_SHORTPTR(ref_ptr[3]);
+
+ uint32x4_t sum[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
+ vdupq_n_u32(0) };
+
+ int i = 0;
+ do {
+ uint16x4_t s = vld1_u16(src16_ptr + i * src_stride);
+ uint16x4_t r0 = vld1_u16(ref16_ptr0 + i * ref_stride);
+ uint16x4_t r1 = vld1_u16(ref16_ptr1 + i * ref_stride);
+ uint16x4_t r2 = vld1_u16(ref16_ptr2 + i * ref_stride);
+ uint16x4_t r3 = vld1_u16(ref16_ptr3 + i * ref_stride);
+
+ sum[0] = vabal_u16(sum[0], s, r0);
+ sum[1] = vabal_u16(sum[1], s, r1);
+ sum[2] = vabal_u16(sum[2], s, r2);
+ sum[3] = vabal_u16(sum[3], s, r3);
+
+ } while (++i < h);
+
+ vst1q_u32(res, horizontal_add_4d_u32x4(sum));
+}
+
+static INLINE void highbd_sad8xhx4d_small_neon(const uint8_t *src_ptr,
+ int src_stride,
+ const uint8_t *const ref_ptr[4],
+ int ref_stride, uint32_t res[4],
+ int h) {
+ const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr);
+ const uint16_t *ref16_ptr0 = CONVERT_TO_SHORTPTR(ref_ptr[0]);
+ const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]);
+ const uint16_t *ref16_ptr2 = CONVERT_TO_SHORTPTR(ref_ptr[2]);
+ const uint16_t *ref16_ptr3 = CONVERT_TO_SHORTPTR(ref_ptr[3]);
+
+ uint16x8_t sum[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0),
+ vdupq_n_u16(0) };
+ uint32x4_t sum_u32[4];
+
+ int i = 0;
+ do {
+ uint16x8_t s = vld1q_u16(src16_ptr + i * src_stride);
+
+ sum[0] = vabaq_u16(sum[0], s, vld1q_u16(ref16_ptr0 + i * ref_stride));
+ sum[1] = vabaq_u16(sum[1], s, vld1q_u16(ref16_ptr1 + i * ref_stride));
+ sum[2] = vabaq_u16(sum[2], s, vld1q_u16(ref16_ptr2 + i * ref_stride));
+ sum[3] = vabaq_u16(sum[3], s, vld1q_u16(ref16_ptr3 + i * ref_stride));
+
+ } while (++i < h);
+
+ sum_u32[0] = vpaddlq_u16(sum[0]);
+ sum_u32[1] = vpaddlq_u16(sum[1]);
+ sum_u32[2] = vpaddlq_u16(sum[2]);
+ sum_u32[3] = vpaddlq_u16(sum[3]);
+ vst1q_u32(res, horizontal_add_4d_u32x4(sum_u32));
+}
+
+static INLINE void sad8_neon(uint16x8_t src, uint16x8_t ref,
+ uint32x4_t *const sad_sum) {
+ uint16x8_t abs_diff = vabdq_u16(src, ref);
+ *sad_sum = vpadalq_u16(*sad_sum, abs_diff);
+}
+
+#if !CONFIG_REALTIME_ONLY
+static INLINE void highbd_sad8xhx4d_large_neon(const uint8_t *src_ptr,
+ int src_stride,
+ const uint8_t *const ref_ptr[4],
+ int ref_stride, uint32_t res[4],
+ int h) {
+ const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr);
+ const uint16_t *ref16_ptr0 = CONVERT_TO_SHORTPTR(ref_ptr[0]);
+ const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]);
+ const uint16_t *ref16_ptr2 = CONVERT_TO_SHORTPTR(ref_ptr[2]);
+ const uint16_t *ref16_ptr3 = CONVERT_TO_SHORTPTR(ref_ptr[3]);
+
+ uint32x4_t sum[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
+ vdupq_n_u32(0) };
+
+ int i = 0;
+ do {
+ uint16x8_t s = vld1q_u16(src16_ptr + i * src_stride);
+ sad8_neon(s, vld1q_u16(ref16_ptr0 + i * ref_stride), &sum[0]);
+ sad8_neon(s, vld1q_u16(ref16_ptr1 + i * ref_stride), &sum[1]);
+ sad8_neon(s, vld1q_u16(ref16_ptr2 + i * ref_stride), &sum[2]);
+ sad8_neon(s, vld1q_u16(ref16_ptr3 + i * ref_stride), &sum[3]);
+
+ } while (++i < h);
+
+ vst1q_u32(res, horizontal_add_4d_u32x4(sum));
+}
+#endif // !CONFIG_REALTIME_ONLY
+
+static INLINE void highbd_sad16xhx4d_large_neon(const uint8_t *src_ptr,
+ int src_stride,
+ const uint8_t *const ref_ptr[4],
+ int ref_stride, uint32_t res[4],
+ int h) {
+ const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr);
+ const uint16_t *ref16_ptr0 = CONVERT_TO_SHORTPTR(ref_ptr[0]);
+ const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]);
+ const uint16_t *ref16_ptr2 = CONVERT_TO_SHORTPTR(ref_ptr[2]);
+ const uint16_t *ref16_ptr3 = CONVERT_TO_SHORTPTR(ref_ptr[3]);
+
+ uint32x4_t sum_lo[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
+ vdupq_n_u32(0) };
+ uint32x4_t sum_hi[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
+ vdupq_n_u32(0) };
+ uint32x4_t sum[4];
+
+ int i = 0;
+ do {
+ uint16x8_t s0 = vld1q_u16(src16_ptr + i * src_stride);
+ sad8_neon(s0, vld1q_u16(ref16_ptr0 + i * ref_stride), &sum_lo[0]);
+ sad8_neon(s0, vld1q_u16(ref16_ptr1 + i * ref_stride), &sum_lo[1]);
+ sad8_neon(s0, vld1q_u16(ref16_ptr2 + i * ref_stride), &sum_lo[2]);
+ sad8_neon(s0, vld1q_u16(ref16_ptr3 + i * ref_stride), &sum_lo[3]);
+
+ uint16x8_t s1 = vld1q_u16(src16_ptr + i * src_stride + 8);
+ sad8_neon(s1, vld1q_u16(ref16_ptr0 + i * ref_stride + 8), &sum_hi[0]);
+ sad8_neon(s1, vld1q_u16(ref16_ptr1 + i * ref_stride + 8), &sum_hi[1]);
+ sad8_neon(s1, vld1q_u16(ref16_ptr2 + i * ref_stride + 8), &sum_hi[2]);
+ sad8_neon(s1, vld1q_u16(ref16_ptr3 + i * ref_stride + 8), &sum_hi[3]);
+
+ } while (++i < h);
+
+ sum[0] = vaddq_u32(sum_lo[0], sum_hi[0]);
+ sum[1] = vaddq_u32(sum_lo[1], sum_hi[1]);
+ sum[2] = vaddq_u32(sum_lo[2], sum_hi[2]);
+ sum[3] = vaddq_u32(sum_lo[3], sum_hi[3]);
+
+ vst1q_u32(res, horizontal_add_4d_u32x4(sum));
+}
+
+static INLINE void highbd_sadwxhx4d_large_neon(const uint8_t *src_ptr,
+ int src_stride,
+ const uint8_t *const ref_ptr[4],
+ int ref_stride, uint32_t res[4],
+ int w, int h) {
+ const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr);
+ const uint16_t *ref16_ptr0 = CONVERT_TO_SHORTPTR(ref_ptr[0]);
+ const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]);
+ const uint16_t *ref16_ptr2 = CONVERT_TO_SHORTPTR(ref_ptr[2]);
+ const uint16_t *ref16_ptr3 = CONVERT_TO_SHORTPTR(ref_ptr[3]);
+
+ uint32x4_t sum_lo[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
+ vdupq_n_u32(0) };
+ uint32x4_t sum_hi[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
+ vdupq_n_u32(0) };
+ uint32x4_t sum[4];
+
+ int i = 0;
+ do {
+ int j = 0;
+ do {
+ uint16x8_t s0 = vld1q_u16(src16_ptr + i * src_stride + j);
+ sad8_neon(s0, vld1q_u16(ref16_ptr0 + i * ref_stride + j), &sum_lo[0]);
+ sad8_neon(s0, vld1q_u16(ref16_ptr1 + i * ref_stride + j), &sum_lo[1]);
+ sad8_neon(s0, vld1q_u16(ref16_ptr2 + i * ref_stride + j), &sum_lo[2]);
+ sad8_neon(s0, vld1q_u16(ref16_ptr3 + i * ref_stride + j), &sum_lo[3]);
+
+ uint16x8_t s1 = vld1q_u16(src16_ptr + i * src_stride + j + 8);
+ sad8_neon(s1, vld1q_u16(ref16_ptr0 + i * ref_stride + j + 8), &sum_hi[0]);
+ sad8_neon(s1, vld1q_u16(ref16_ptr1 + i * ref_stride + j + 8), &sum_hi[1]);
+ sad8_neon(s1, vld1q_u16(ref16_ptr2 + i * ref_stride + j + 8), &sum_hi[2]);
+ sad8_neon(s1, vld1q_u16(ref16_ptr3 + i * ref_stride + j + 8), &sum_hi[3]);
+
+ uint16x8_t s2 = vld1q_u16(src16_ptr + i * src_stride + j + 16);
+ sad8_neon(s2, vld1q_u16(ref16_ptr0 + i * ref_stride + j + 16),
+ &sum_lo[0]);
+ sad8_neon(s2, vld1q_u16(ref16_ptr1 + i * ref_stride + j + 16),
+ &sum_lo[1]);
+ sad8_neon(s2, vld1q_u16(ref16_ptr2 + i * ref_stride + j + 16),
+ &sum_lo[2]);
+ sad8_neon(s2, vld1q_u16(ref16_ptr3 + i * ref_stride + j + 16),
+ &sum_lo[3]);
+
+ uint16x8_t s3 = vld1q_u16(src16_ptr + i * src_stride + j + 24);
+ sad8_neon(s3, vld1q_u16(ref16_ptr0 + i * ref_stride + j + 24),
+ &sum_hi[0]);
+ sad8_neon(s3, vld1q_u16(ref16_ptr1 + i * ref_stride + j + 24),
+ &sum_hi[1]);
+ sad8_neon(s3, vld1q_u16(ref16_ptr2 + i * ref_stride + j + 24),
+ &sum_hi[2]);
+ sad8_neon(s3, vld1q_u16(ref16_ptr3 + i * ref_stride + j + 24),
+ &sum_hi[3]);
+
+ j += 32;
+ } while (j < w);
+
+ } while (++i < h);
+
+ sum[0] = vaddq_u32(sum_lo[0], sum_hi[0]);
+ sum[1] = vaddq_u32(sum_lo[1], sum_hi[1]);
+ sum[2] = vaddq_u32(sum_lo[2], sum_hi[2]);
+ sum[3] = vaddq_u32(sum_lo[3], sum_hi[3]);
+
+ vst1q_u32(res, horizontal_add_4d_u32x4(sum));
+}
+
+static INLINE void highbd_sad128xhx4d_large_neon(
+ const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_ptr[4],
+ int ref_stride, uint32_t res[4], int h) {
+ highbd_sadwxhx4d_large_neon(src_ptr, src_stride, ref_ptr, ref_stride, res,
+ 128, h);
+}
+
+static INLINE void highbd_sad64xhx4d_large_neon(const uint8_t *src_ptr,
+ int src_stride,
+ const uint8_t *const ref_ptr[4],
+ int ref_stride, uint32_t res[4],
+ int h) {
+ highbd_sadwxhx4d_large_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 64,
+ h);
+}
+
+static INLINE void highbd_sad32xhx4d_large_neon(const uint8_t *src_ptr,
+ int src_stride,
+ const uint8_t *const ref_ptr[4],
+ int ref_stride, uint32_t res[4],
+ int h) {
+ highbd_sadwxhx4d_large_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 32,
+ h);
+}
+
+#define HBD_SAD_WXH_4D_SMALL_NEON(w, h) \
+ void aom_highbd_sad##w##x##h##x4d_neon( \
+ const uint8_t *src, int src_stride, const uint8_t *const ref_array[4], \
+ int ref_stride, uint32_t sad_array[4]) { \
+ highbd_sad##w##xhx4d_small_neon(src, src_stride, ref_array, ref_stride, \
+ sad_array, (h)); \
+ }
+
+#define HBD_SAD_WXH_4D_LARGE_NEON(w, h) \
+ void aom_highbd_sad##w##x##h##x4d_neon( \
+ const uint8_t *src, int src_stride, const uint8_t *const ref_array[4], \
+ int ref_stride, uint32_t sad_array[4]) { \
+ highbd_sad##w##xhx4d_large_neon(src, src_stride, ref_array, ref_stride, \
+ sad_array, (h)); \
+ }
+
+HBD_SAD_WXH_4D_SMALL_NEON(4, 4)
+HBD_SAD_WXH_4D_SMALL_NEON(4, 8)
+
+HBD_SAD_WXH_4D_SMALL_NEON(8, 4)
+HBD_SAD_WXH_4D_SMALL_NEON(8, 8)
+HBD_SAD_WXH_4D_SMALL_NEON(8, 16)
+
+HBD_SAD_WXH_4D_LARGE_NEON(16, 8)
+HBD_SAD_WXH_4D_LARGE_NEON(16, 16)
+HBD_SAD_WXH_4D_LARGE_NEON(16, 32)
+
+HBD_SAD_WXH_4D_LARGE_NEON(32, 16)
+HBD_SAD_WXH_4D_LARGE_NEON(32, 32)
+HBD_SAD_WXH_4D_LARGE_NEON(32, 64)
+
+HBD_SAD_WXH_4D_LARGE_NEON(64, 32)
+HBD_SAD_WXH_4D_LARGE_NEON(64, 64)
+HBD_SAD_WXH_4D_LARGE_NEON(64, 128)
+
+HBD_SAD_WXH_4D_LARGE_NEON(128, 64)
+HBD_SAD_WXH_4D_LARGE_NEON(128, 128)
+
+#if !CONFIG_REALTIME_ONLY
+HBD_SAD_WXH_4D_SMALL_NEON(4, 16)
+
+HBD_SAD_WXH_4D_LARGE_NEON(8, 32)
+
+HBD_SAD_WXH_4D_LARGE_NEON(16, 4)
+HBD_SAD_WXH_4D_LARGE_NEON(16, 64)
+
+HBD_SAD_WXH_4D_LARGE_NEON(32, 8)
+
+HBD_SAD_WXH_4D_LARGE_NEON(64, 16)
+#endif // !CONFIG_REALTIME_ONLY
+
+#define HBD_SAD_SKIP_WXH_4D_SMALL_NEON(w, h) \
+ void aom_highbd_sad_skip_##w##x##h##x4d_neon( \
+ const uint8_t *src, int src_stride, const uint8_t *const ref_array[4], \
+ int ref_stride, uint32_t sad_array[4]) { \
+ highbd_sad##w##xhx4d_small_neon(src, 2 * src_stride, ref_array, \
+ 2 * ref_stride, sad_array, ((h) >> 1)); \
+ sad_array[0] <<= 1; \
+ sad_array[1] <<= 1; \
+ sad_array[2] <<= 1; \
+ sad_array[3] <<= 1; \
+ }
+
+#define HBD_SAD_SKIP_WXH_4D_LARGE_NEON(w, h) \
+ void aom_highbd_sad_skip_##w##x##h##x4d_neon( \
+ const uint8_t *src, int src_stride, const uint8_t *const ref_array[4], \
+ int ref_stride, uint32_t sad_array[4]) { \
+ highbd_sad##w##xhx4d_large_neon(src, 2 * src_stride, ref_array, \
+ 2 * ref_stride, sad_array, ((h) >> 1)); \
+ sad_array[0] <<= 1; \
+ sad_array[1] <<= 1; \
+ sad_array[2] <<= 1; \
+ sad_array[3] <<= 1; \
+ }
+
+HBD_SAD_SKIP_WXH_4D_SMALL_NEON(4, 4)
+HBD_SAD_SKIP_WXH_4D_SMALL_NEON(4, 8)
+
+HBD_SAD_SKIP_WXH_4D_SMALL_NEON(8, 4)
+HBD_SAD_SKIP_WXH_4D_SMALL_NEON(8, 8)
+HBD_SAD_SKIP_WXH_4D_SMALL_NEON(8, 16)
+
+HBD_SAD_SKIP_WXH_4D_LARGE_NEON(16, 8)
+HBD_SAD_SKIP_WXH_4D_LARGE_NEON(16, 16)
+HBD_SAD_SKIP_WXH_4D_LARGE_NEON(16, 32)
+
+HBD_SAD_SKIP_WXH_4D_LARGE_NEON(32, 16)
+HBD_SAD_SKIP_WXH_4D_LARGE_NEON(32, 32)
+HBD_SAD_SKIP_WXH_4D_LARGE_NEON(32, 64)
+
+HBD_SAD_SKIP_WXH_4D_LARGE_NEON(64, 32)
+HBD_SAD_SKIP_WXH_4D_LARGE_NEON(64, 64)
+HBD_SAD_SKIP_WXH_4D_LARGE_NEON(64, 128)
+
+HBD_SAD_SKIP_WXH_4D_LARGE_NEON(128, 64)
+HBD_SAD_SKIP_WXH_4D_LARGE_NEON(128, 128)
+
+#if !CONFIG_REALTIME_ONLY
+HBD_SAD_SKIP_WXH_4D_SMALL_NEON(4, 16)
+
+HBD_SAD_SKIP_WXH_4D_SMALL_NEON(8, 32)
+
+HBD_SAD_SKIP_WXH_4D_LARGE_NEON(16, 4)
+HBD_SAD_SKIP_WXH_4D_LARGE_NEON(16, 64)
+
+HBD_SAD_SKIP_WXH_4D_LARGE_NEON(32, 8)
+
+HBD_SAD_SKIP_WXH_4D_LARGE_NEON(64, 16)
+#endif // !CONFIG_REALTIME_ONLY
+
+static INLINE void highbd_sad4xhx3d_small_neon(const uint8_t *src_ptr,
+ int src_stride,
+ const uint8_t *const ref_ptr[4],
+ int ref_stride, uint32_t res[4],
+ int h) {
+ const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr);
+ const uint16_t *ref16_ptr0 = CONVERT_TO_SHORTPTR(ref_ptr[0]);
+ const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]);
+ const uint16_t *ref16_ptr2 = CONVERT_TO_SHORTPTR(ref_ptr[2]);
+
+ uint32x4_t sum[3] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0) };
+
+ int i = 0;
+ do {
+ uint16x4_t s = vld1_u16(src16_ptr + i * src_stride);
+ uint16x4_t r0 = vld1_u16(ref16_ptr0 + i * ref_stride);
+ uint16x4_t r1 = vld1_u16(ref16_ptr1 + i * ref_stride);
+ uint16x4_t r2 = vld1_u16(ref16_ptr2 + i * ref_stride);
+
+ sum[0] = vabal_u16(sum[0], s, r0);
+ sum[1] = vabal_u16(sum[1], s, r1);
+ sum[2] = vabal_u16(sum[2], s, r2);
+
+ } while (++i < h);
+
+ res[0] = horizontal_add_u32x4(sum[0]);
+ res[1] = horizontal_add_u32x4(sum[1]);
+ res[2] = horizontal_add_u32x4(sum[2]);
+}
+
+static INLINE void highbd_sad8xhx3d_small_neon(const uint8_t *src_ptr,
+ int src_stride,
+ const uint8_t *const ref_ptr[4],
+ int ref_stride, uint32_t res[4],
+ int h) {
+ const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr);
+ const uint16_t *ref16_ptr0 = CONVERT_TO_SHORTPTR(ref_ptr[0]);
+ const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]);
+ const uint16_t *ref16_ptr2 = CONVERT_TO_SHORTPTR(ref_ptr[2]);
+
+ uint16x8_t sum[3] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0) };
+
+ int i = 0;
+ do {
+ uint16x8_t s = vld1q_u16(src16_ptr + i * src_stride);
+
+ sum[0] = vabaq_u16(sum[0], s, vld1q_u16(ref16_ptr0 + i * ref_stride));
+ sum[1] = vabaq_u16(sum[1], s, vld1q_u16(ref16_ptr1 + i * ref_stride));
+ sum[2] = vabaq_u16(sum[2], s, vld1q_u16(ref16_ptr2 + i * ref_stride));
+
+ } while (++i < h);
+
+ res[0] = horizontal_add_u32x4(vpaddlq_u16(sum[0]));
+ res[1] = horizontal_add_u32x4(vpaddlq_u16(sum[1]));
+ res[2] = horizontal_add_u32x4(vpaddlq_u16(sum[2]));
+}
+
+#if !CONFIG_REALTIME_ONLY
+static INLINE void highbd_sad8xhx3d_large_neon(const uint8_t *src_ptr,
+ int src_stride,
+ const uint8_t *const ref_ptr[4],
+ int ref_stride, uint32_t res[4],
+ int h) {
+ const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr);
+ const uint16_t *ref16_ptr0 = CONVERT_TO_SHORTPTR(ref_ptr[0]);
+ const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]);
+ const uint16_t *ref16_ptr2 = CONVERT_TO_SHORTPTR(ref_ptr[2]);
+
+ uint32x4_t sum[3] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0) };
+
+ int i = 0;
+ do {
+ uint16x8_t s = vld1q_u16(src16_ptr + i * src_stride);
+ uint16x8_t r0 = vld1q_u16(ref16_ptr0 + i * ref_stride);
+ uint16x8_t r1 = vld1q_u16(ref16_ptr1 + i * ref_stride);
+ uint16x8_t r2 = vld1q_u16(ref16_ptr2 + i * ref_stride);
+
+ sad8_neon(s, r0, &sum[0]);
+ sad8_neon(s, r1, &sum[1]);
+ sad8_neon(s, r2, &sum[2]);
+
+ } while (++i < h);
+
+ res[0] = horizontal_add_u32x4(sum[0]);
+ res[1] = horizontal_add_u32x4(sum[1]);
+ res[2] = horizontal_add_u32x4(sum[2]);
+}
+#endif // !CONFIG_REALTIME_ONLY
+
+static INLINE void highbd_sad16xhx3d_large_neon(const uint8_t *src_ptr,
+ int src_stride,
+ const uint8_t *const ref_ptr[4],
+ int ref_stride, uint32_t res[4],
+ int h) {
+ const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr);
+ const uint16_t *ref16_ptr0 = CONVERT_TO_SHORTPTR(ref_ptr[0]);
+ const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]);
+ const uint16_t *ref16_ptr2 = CONVERT_TO_SHORTPTR(ref_ptr[2]);
+
+ uint32x4_t sum_lo[3] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0) };
+ uint32x4_t sum_hi[3] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0) };
+
+ int i = 0;
+ do {
+ uint16x8_t s0 = vld1q_u16(src16_ptr + i * src_stride);
+ sad8_neon(s0, vld1q_u16(ref16_ptr0 + i * ref_stride), &sum_lo[0]);
+ sad8_neon(s0, vld1q_u16(ref16_ptr1 + i * ref_stride), &sum_lo[1]);
+ sad8_neon(s0, vld1q_u16(ref16_ptr2 + i * ref_stride), &sum_lo[2]);
+
+ uint16x8_t s1 = vld1q_u16(src16_ptr + i * src_stride + 8);
+ sad8_neon(s1, vld1q_u16(ref16_ptr0 + i * ref_stride + 8), &sum_hi[0]);
+ sad8_neon(s1, vld1q_u16(ref16_ptr1 + i * ref_stride + 8), &sum_hi[1]);
+ sad8_neon(s1, vld1q_u16(ref16_ptr2 + i * ref_stride + 8), &sum_hi[2]);
+
+ } while (++i < h);
+
+ res[0] = horizontal_add_u32x4(vaddq_u32(sum_lo[0], sum_hi[0]));
+ res[1] = horizontal_add_u32x4(vaddq_u32(sum_lo[1], sum_hi[1]));
+ res[2] = horizontal_add_u32x4(vaddq_u32(sum_lo[2], sum_hi[2]));
+}
+
+static INLINE void highbd_sadwxhx3d_large_neon(const uint8_t *src_ptr,
+ int src_stride,
+ const uint8_t *const ref_ptr[4],
+ int ref_stride, uint32_t res[4],
+ int w, int h) {
+ const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr);
+ const uint16_t *ref16_ptr0 = CONVERT_TO_SHORTPTR(ref_ptr[0]);
+ const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]);
+ const uint16_t *ref16_ptr2 = CONVERT_TO_SHORTPTR(ref_ptr[2]);
+
+ uint32x4_t sum_lo[3] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0) };
+ uint32x4_t sum_hi[3] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0) };
+ uint32x4_t sum[3];
+
+ int i = 0;
+ do {
+ int j = 0;
+ do {
+ uint16x8_t s0 = vld1q_u16(src16_ptr + i * src_stride + j);
+ sad8_neon(s0, vld1q_u16(ref16_ptr0 + i * ref_stride + j), &sum_lo[0]);
+ sad8_neon(s0, vld1q_u16(ref16_ptr1 + i * ref_stride + j), &sum_lo[1]);
+ sad8_neon(s0, vld1q_u16(ref16_ptr2 + i * ref_stride + j), &sum_lo[2]);
+
+ uint16x8_t s1 = vld1q_u16(src16_ptr + i * src_stride + j + 8);
+ sad8_neon(s1, vld1q_u16(ref16_ptr0 + i * ref_stride + j + 8), &sum_hi[0]);
+ sad8_neon(s1, vld1q_u16(ref16_ptr1 + i * ref_stride + j + 8), &sum_hi[1]);
+ sad8_neon(s1, vld1q_u16(ref16_ptr2 + i * ref_stride + j + 8), &sum_hi[2]);
+
+ uint16x8_t s2 = vld1q_u16(src16_ptr + i * src_stride + j + 16);
+ sad8_neon(s2, vld1q_u16(ref16_ptr0 + i * ref_stride + j + 16),
+ &sum_lo[0]);
+ sad8_neon(s2, vld1q_u16(ref16_ptr1 + i * ref_stride + j + 16),
+ &sum_lo[1]);
+ sad8_neon(s2, vld1q_u16(ref16_ptr2 + i * ref_stride + j + 16),
+ &sum_lo[2]);
+
+ uint16x8_t s3 = vld1q_u16(src16_ptr + i * src_stride + j + 24);
+ sad8_neon(s3, vld1q_u16(ref16_ptr0 + i * ref_stride + j + 24),
+ &sum_hi[0]);
+ sad8_neon(s3, vld1q_u16(ref16_ptr1 + i * ref_stride + j + 24),
+ &sum_hi[1]);
+ sad8_neon(s3, vld1q_u16(ref16_ptr2 + i * ref_stride + j + 24),
+ &sum_hi[2]);
+
+ j += 32;
+ } while (j < w);
+
+ } while (++i < h);
+
+ sum[0] = vaddq_u32(sum_lo[0], sum_hi[0]);
+ sum[1] = vaddq_u32(sum_lo[1], sum_hi[1]);
+ sum[2] = vaddq_u32(sum_lo[2], sum_hi[2]);
+
+ res[0] = horizontal_add_u32x4(sum[0]);
+ res[1] = horizontal_add_u32x4(sum[1]);
+ res[2] = horizontal_add_u32x4(sum[2]);
+}
+
+static INLINE void highbd_sad128xhx3d_large_neon(
+ const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_ptr[4],
+ int ref_stride, uint32_t res[4], int h) {
+ highbd_sadwxhx3d_large_neon(src_ptr, src_stride, ref_ptr, ref_stride, res,
+ 128, h);
+}
+
+static INLINE void highbd_sad64xhx3d_large_neon(const uint8_t *src_ptr,
+ int src_stride,
+ const uint8_t *const ref_ptr[4],
+ int ref_stride, uint32_t res[4],
+ int h) {
+ highbd_sadwxhx3d_large_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 64,
+ h);
+}
+
+static INLINE void highbd_sad32xhx3d_large_neon(const uint8_t *src_ptr,
+ int src_stride,
+ const uint8_t *const ref_ptr[4],
+ int ref_stride, uint32_t res[4],
+ int h) {
+ highbd_sadwxhx3d_large_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 32,
+ h);
+}
+
+#define HBD_SAD_WXH_3D_SMALL_NEON(w, h) \
+ void aom_highbd_sad##w##x##h##x3d_neon( \
+ const uint8_t *src, int src_stride, const uint8_t *const ref_array[4], \
+ int ref_stride, uint32_t sad_array[4]) { \
+ highbd_sad##w##xhx3d_small_neon(src, src_stride, ref_array, ref_stride, \
+ sad_array, (h)); \
+ }
+
+#define HBD_SAD_WXH_3D_LARGE_NEON(w, h) \
+ void aom_highbd_sad##w##x##h##x3d_neon( \
+ const uint8_t *src, int src_stride, const uint8_t *const ref_array[4], \
+ int ref_stride, uint32_t sad_array[4]) { \
+ highbd_sad##w##xhx3d_large_neon(src, src_stride, ref_array, ref_stride, \
+ sad_array, (h)); \
+ }
+
+HBD_SAD_WXH_3D_SMALL_NEON(4, 4)
+HBD_SAD_WXH_3D_SMALL_NEON(4, 8)
+
+HBD_SAD_WXH_3D_SMALL_NEON(8, 4)
+HBD_SAD_WXH_3D_SMALL_NEON(8, 8)
+HBD_SAD_WXH_3D_SMALL_NEON(8, 16)
+
+HBD_SAD_WXH_3D_LARGE_NEON(16, 8)
+HBD_SAD_WXH_3D_LARGE_NEON(16, 16)
+HBD_SAD_WXH_3D_LARGE_NEON(16, 32)
+
+HBD_SAD_WXH_3D_LARGE_NEON(32, 16)
+HBD_SAD_WXH_3D_LARGE_NEON(32, 32)
+HBD_SAD_WXH_3D_LARGE_NEON(32, 64)
+
+HBD_SAD_WXH_3D_LARGE_NEON(64, 32)
+HBD_SAD_WXH_3D_LARGE_NEON(64, 64)
+HBD_SAD_WXH_3D_LARGE_NEON(64, 128)
+
+HBD_SAD_WXH_3D_LARGE_NEON(128, 64)
+HBD_SAD_WXH_3D_LARGE_NEON(128, 128)
+
+#if !CONFIG_REALTIME_ONLY
+HBD_SAD_WXH_3D_SMALL_NEON(4, 16)
+
+HBD_SAD_WXH_3D_LARGE_NEON(8, 32)
+
+HBD_SAD_WXH_3D_LARGE_NEON(16, 4)
+HBD_SAD_WXH_3D_LARGE_NEON(16, 64)
+
+HBD_SAD_WXH_3D_LARGE_NEON(32, 8)
+
+HBD_SAD_WXH_3D_LARGE_NEON(64, 16)
+#endif // !CONFIG_REALTIME_ONLY
diff --git a/aom_dsp/arm/highbd_sse_neon.c b/aom_dsp/arm/highbd_sse_neon.c
new file mode 100644
index 0000000..184e9f9
--- /dev/null
+++ b/aom_dsp/arm/highbd_sse_neon.c
@@ -0,0 +1,284 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <arm_neon.h>
+
+#include "config/aom_dsp_rtcd.h"
+#include "aom_dsp/arm/sum_neon.h"
+
+static INLINE void highbd_sse_8x1_init_neon(const uint16_t *src,
+ const uint16_t *ref,
+ uint32x4_t *sse_acc0,
+ uint32x4_t *sse_acc1) {
+ uint16x8_t s = vld1q_u16(src);
+ uint16x8_t r = vld1q_u16(ref);
+
+ uint16x8_t abs_diff = vabdq_u16(s, r);
+ uint16x4_t abs_diff_lo = vget_low_u16(abs_diff);
+ uint16x4_t abs_diff_hi = vget_high_u16(abs_diff);
+
+ *sse_acc0 = vmull_u16(abs_diff_lo, abs_diff_lo);
+ *sse_acc1 = vmull_u16(abs_diff_hi, abs_diff_hi);
+}
+
+static INLINE void highbd_sse_8x1_neon(const uint16_t *src, const uint16_t *ref,
+ uint32x4_t *sse_acc0,
+ uint32x4_t *sse_acc1) {
+ uint16x8_t s = vld1q_u16(src);
+ uint16x8_t r = vld1q_u16(ref);
+
+ uint16x8_t abs_diff = vabdq_u16(s, r);
+ uint16x4_t abs_diff_lo = vget_low_u16(abs_diff);
+ uint16x4_t abs_diff_hi = vget_high_u16(abs_diff);
+
+ *sse_acc0 = vmlal_u16(*sse_acc0, abs_diff_lo, abs_diff_lo);
+ *sse_acc1 = vmlal_u16(*sse_acc1, abs_diff_hi, abs_diff_hi);
+}
+
+static INLINE int64_t highbd_sse_128xh_neon(const uint16_t *src, int src_stride,
+ const uint16_t *ref, int ref_stride,
+ int height) {
+ uint32x4_t sse[16];
+ highbd_sse_8x1_init_neon(src + 0 * 8, ref + 0 * 8, &sse[0], &sse[1]);
+ highbd_sse_8x1_init_neon(src + 1 * 8, ref + 1 * 8, &sse[2], &sse[3]);
+ highbd_sse_8x1_init_neon(src + 2 * 8, ref + 2 * 8, &sse[4], &sse[5]);
+ highbd_sse_8x1_init_neon(src + 3 * 8, ref + 3 * 8, &sse[6], &sse[7]);
+ highbd_sse_8x1_init_neon(src + 4 * 8, ref + 4 * 8, &sse[8], &sse[9]);
+ highbd_sse_8x1_init_neon(src + 5 * 8, ref + 5 * 8, &sse[10], &sse[11]);
+ highbd_sse_8x1_init_neon(src + 6 * 8, ref + 6 * 8, &sse[12], &sse[13]);
+ highbd_sse_8x1_init_neon(src + 7 * 8, ref + 7 * 8, &sse[14], &sse[15]);
+ highbd_sse_8x1_neon(src + 8 * 8, ref + 8 * 8, &sse[0], &sse[1]);
+ highbd_sse_8x1_neon(src + 9 * 8, ref + 9 * 8, &sse[2], &sse[3]);
+ highbd_sse_8x1_neon(src + 10 * 8, ref + 10 * 8, &sse[4], &sse[5]);
+ highbd_sse_8x1_neon(src + 11 * 8, ref + 11 * 8, &sse[6], &sse[7]);
+ highbd_sse_8x1_neon(src + 12 * 8, ref + 12 * 8, &sse[8], &sse[9]);
+ highbd_sse_8x1_neon(src + 13 * 8, ref + 13 * 8, &sse[10], &sse[11]);
+ highbd_sse_8x1_neon(src + 14 * 8, ref + 14 * 8, &sse[12], &sse[13]);
+ highbd_sse_8x1_neon(src + 15 * 8, ref + 15 * 8, &sse[14], &sse[15]);
+
+ src += src_stride;
+ ref += ref_stride;
+
+ while (--height != 0) {
+ highbd_sse_8x1_neon(src + 0 * 8, ref + 0 * 8, &sse[0], &sse[1]);
+ highbd_sse_8x1_neon(src + 1 * 8, ref + 1 * 8, &sse[2], &sse[3]);
+ highbd_sse_8x1_neon(src + 2 * 8, ref + 2 * 8, &sse[4], &sse[5]);
+ highbd_sse_8x1_neon(src + 3 * 8, ref + 3 * 8, &sse[6], &sse[7]);
+ highbd_sse_8x1_neon(src + 4 * 8, ref + 4 * 8, &sse[8], &sse[9]);
+ highbd_sse_8x1_neon(src + 5 * 8, ref + 5 * 8, &sse[10], &sse[11]);
+ highbd_sse_8x1_neon(src + 6 * 8, ref + 6 * 8, &sse[12], &sse[13]);
+ highbd_sse_8x1_neon(src + 7 * 8, ref + 7 * 8, &sse[14], &sse[15]);
+ highbd_sse_8x1_neon(src + 8 * 8, ref + 8 * 8, &sse[0], &sse[1]);
+ highbd_sse_8x1_neon(src + 9 * 8, ref + 9 * 8, &sse[2], &sse[3]);
+ highbd_sse_8x1_neon(src + 10 * 8, ref + 10 * 8, &sse[4], &sse[5]);
+ highbd_sse_8x1_neon(src + 11 * 8, ref + 11 * 8, &sse[6], &sse[7]);
+ highbd_sse_8x1_neon(src + 12 * 8, ref + 12 * 8, &sse[8], &sse[9]);
+ highbd_sse_8x1_neon(src + 13 * 8, ref + 13 * 8, &sse[10], &sse[11]);
+ highbd_sse_8x1_neon(src + 14 * 8, ref + 14 * 8, &sse[12], &sse[13]);
+ highbd_sse_8x1_neon(src + 15 * 8, ref + 15 * 8, &sse[14], &sse[15]);
+
+ src += src_stride;
+ ref += ref_stride;
+ }
+
+ return horizontal_long_add_u32x4_x16(sse);
+}
+
+static INLINE int64_t highbd_sse_64xh_neon(const uint16_t *src, int src_stride,
+ const uint16_t *ref, int ref_stride,
+ int height) {
+ uint32x4_t sse[8];
+ highbd_sse_8x1_init_neon(src + 0 * 8, ref + 0 * 8, &sse[0], &sse[1]);
+ highbd_sse_8x1_init_neon(src + 1 * 8, ref + 1 * 8, &sse[2], &sse[3]);
+ highbd_sse_8x1_init_neon(src + 2 * 8, ref + 2 * 8, &sse[4], &sse[5]);
+ highbd_sse_8x1_init_neon(src + 3 * 8, ref + 3 * 8, &sse[6], &sse[7]);
+ highbd_sse_8x1_neon(src + 4 * 8, ref + 4 * 8, &sse[0], &sse[1]);
+ highbd_sse_8x1_neon(src + 5 * 8, ref + 5 * 8, &sse[2], &sse[3]);
+ highbd_sse_8x1_neon(src + 6 * 8, ref + 6 * 8, &sse[4], &sse[5]);
+ highbd_sse_8x1_neon(src + 7 * 8, ref + 7 * 8, &sse[6], &sse[7]);
+
+ src += src_stride;
+ ref += ref_stride;
+
+ while (--height != 0) {
+ highbd_sse_8x1_neon(src + 0 * 8, ref + 0 * 8, &sse[0], &sse[1]);
+ highbd_sse_8x1_neon(src + 1 * 8, ref + 1 * 8, &sse[2], &sse[3]);
+ highbd_sse_8x1_neon(src + 2 * 8, ref + 2 * 8, &sse[4], &sse[5]);
+ highbd_sse_8x1_neon(src + 3 * 8, ref + 3 * 8, &sse[6], &sse[7]);
+ highbd_sse_8x1_neon(src + 4 * 8, ref + 4 * 8, &sse[0], &sse[1]);
+ highbd_sse_8x1_neon(src + 5 * 8, ref + 5 * 8, &sse[2], &sse[3]);
+ highbd_sse_8x1_neon(src + 6 * 8, ref + 6 * 8, &sse[4], &sse[5]);
+ highbd_sse_8x1_neon(src + 7 * 8, ref + 7 * 8, &sse[6], &sse[7]);
+
+ src += src_stride;
+ ref += ref_stride;
+ }
+
+ return horizontal_long_add_u32x4_x8(sse);
+}
+
+static INLINE int64_t highbd_sse_32xh_neon(const uint16_t *src, int src_stride,
+ const uint16_t *ref, int ref_stride,
+ int height) {
+ uint32x4_t sse[8];
+ highbd_sse_8x1_init_neon(src + 0 * 8, ref + 0 * 8, &sse[0], &sse[1]);
+ highbd_sse_8x1_init_neon(src + 1 * 8, ref + 1 * 8, &sse[2], &sse[3]);
+ highbd_sse_8x1_init_neon(src + 2 * 8, ref + 2 * 8, &sse[4], &sse[5]);
+ highbd_sse_8x1_init_neon(src + 3 * 8, ref + 3 * 8, &sse[6], &sse[7]);
+
+ src += src_stride;
+ ref += ref_stride;
+
+ while (--height != 0) {
+ highbd_sse_8x1_neon(src + 0 * 8, ref + 0 * 8, &sse[0], &sse[1]);
+ highbd_sse_8x1_neon(src + 1 * 8, ref + 1 * 8, &sse[2], &sse[3]);
+ highbd_sse_8x1_neon(src + 2 * 8, ref + 2 * 8, &sse[4], &sse[5]);
+ highbd_sse_8x1_neon(src + 3 * 8, ref + 3 * 8, &sse[6], &sse[7]);
+
+ src += src_stride;
+ ref += ref_stride;
+ }
+
+ return horizontal_long_add_u32x4_x8(sse);
+}
+
+static INLINE int64_t highbd_sse_16xh_neon(const uint16_t *src, int src_stride,
+ const uint16_t *ref, int ref_stride,
+ int height) {
+ uint32x4_t sse[4];
+ highbd_sse_8x1_init_neon(src + 0 * 8, ref + 0 * 8, &sse[0], &sse[1]);
+ highbd_sse_8x1_init_neon(src + 1 * 8, ref + 1 * 8, &sse[2], &sse[3]);
+
+ src += src_stride;
+ ref += ref_stride;
+
+ while (--height != 0) {
+ highbd_sse_8x1_neon(src + 0 * 8, ref + 0 * 8, &sse[0], &sse[1]);
+ highbd_sse_8x1_neon(src + 1 * 8, ref + 1 * 8, &sse[2], &sse[3]);
+
+ src += src_stride;
+ ref += ref_stride;
+ }
+
+ return horizontal_long_add_u32x4_x4(sse);
+}
+
+static INLINE int64_t highbd_sse_8xh_neon(const uint16_t *src, int src_stride,
+ const uint16_t *ref, int ref_stride,
+ int height) {
+ uint32x4_t sse[2];
+ highbd_sse_8x1_init_neon(src, ref, &sse[0], &sse[1]);
+
+ src += src_stride;
+ ref += ref_stride;
+
+ while (--height != 0) {
+ highbd_sse_8x1_neon(src, ref, &sse[0], &sse[1]);
+
+ src += src_stride;
+ ref += ref_stride;
+ }
+
+ return horizontal_long_add_u32x4_x2(sse);
+}
+
+static INLINE int64_t highbd_sse_4xh_neon(const uint16_t *src, int src_stride,
+ const uint16_t *ref, int ref_stride,
+ int height) {
+ // Peel the first loop iteration.
+ uint16x4_t s = vld1_u16(src);
+ uint16x4_t r = vld1_u16(ref);
+
+ uint16x4_t abs_diff = vabd_u16(s, r);
+ uint32x4_t sse = vmull_u16(abs_diff, abs_diff);
+
+ src += src_stride;
+ ref += ref_stride;
+
+ while (--height != 0) {
+ s = vld1_u16(src);
+ r = vld1_u16(ref);
+
+ abs_diff = vabd_u16(s, r);
+ sse = vmlal_u16(sse, abs_diff, abs_diff);
+
+ src += src_stride;
+ ref += ref_stride;
+ }
+
+ return horizontal_long_add_u32x4(sse);
+}
+
+static INLINE int64_t highbd_sse_wxh_neon(const uint16_t *src, int src_stride,
+ const uint16_t *ref, int ref_stride,
+ int width, int height) {
+ // { 0, 1, 2, 3, 4, 5, 6, 7 }
+ uint16x8_t k01234567 = vmovl_u8(vcreate_u8(0x0706050403020100));
+ uint16x8_t remainder_mask = vcltq_u16(k01234567, vdupq_n_u16(width & 7));
+ uint64_t sse = 0;
+
+ do {
+ int w = width;
+ int offset = 0;
+
+ do {
+ uint16x8_t s = vld1q_u16(src + offset);
+ uint16x8_t r = vld1q_u16(ref + offset);
+
+ if (w < 8) {
+ // Mask out-of-range elements.
+ s = vandq_u16(s, remainder_mask);
+ r = vandq_u16(r, remainder_mask);
+ }
+
+ uint16x8_t abs_diff = vabdq_u16(s, r);
+ uint16x4_t abs_diff_lo = vget_low_u16(abs_diff);
+ uint16x4_t abs_diff_hi = vget_high_u16(abs_diff);
+
+ uint32x4_t sse_u32 = vmull_u16(abs_diff_lo, abs_diff_lo);
+ sse_u32 = vmlal_u16(sse_u32, abs_diff_hi, abs_diff_hi);
+
+ sse += horizontal_long_add_u32x4(sse_u32);
+
+ offset += 8;
+ w -= 8;
+ } while (w > 0);
+
+ src += src_stride;
+ ref += ref_stride;
+ } while (--height != 0);
+
+ return sse;
+}
+
+int64_t aom_highbd_sse_neon(const uint8_t *src8, int src_stride,
+ const uint8_t *ref8, int ref_stride, int width,
+ int height) {
+ uint16_t *src = CONVERT_TO_SHORTPTR(src8);
+ uint16_t *ref = CONVERT_TO_SHORTPTR(ref8);
+
+ switch (width) {
+ case 4:
+ return highbd_sse_4xh_neon(src, src_stride, ref, ref_stride, height);
+ case 8:
+ return highbd_sse_8xh_neon(src, src_stride, ref, ref_stride, height);
+ case 16:
+ return highbd_sse_16xh_neon(src, src_stride, ref, ref_stride, height);
+ case 32:
+ return highbd_sse_32xh_neon(src, src_stride, ref, ref_stride, height);
+ case 64:
+ return highbd_sse_64xh_neon(src, src_stride, ref, ref_stride, height);
+ case 128:
+ return highbd_sse_128xh_neon(src, src_stride, ref, ref_stride, height);
+ default:
+ return highbd_sse_wxh_neon(src, src_stride, ref, ref_stride, width,
+ height);
+ }
+}
diff --git a/aom_dsp/arm/highbd_subpel_variance_neon.c b/aom_dsp/arm/highbd_subpel_variance_neon.c
new file mode 100644
index 0000000..bdbbf70
--- /dev/null
+++ b/aom_dsp/arm/highbd_subpel_variance_neon.c
@@ -0,0 +1,1497 @@
+/*
+ * Copyright (c) 2023 The WebM project authors. All Rights Reserved.
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+
+#include "config/aom_config.h"
+#include "config/aom_dsp_rtcd.h"
+
+#include "aom_dsp/aom_filter.h"
+#include "aom_dsp/arm/dist_wtd_avg_neon.h"
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_dsp/arm/sum_neon.h"
+#include "aom_dsp/variance.h"
+
+// The bilinear filters look like this:
+//
+// {{ 128, 0 }, { 112, 16 }, { 96, 32 }, { 80, 48 },
+// { 64, 64 }, { 48, 80 }, { 32, 96 }, { 16, 112 }}
+//
+// We can factor out the highest common multiple, such that the sum of both
+// weights will be 8 instead of 128. The benefits of this are two-fold:
+//
+// 1) We can infer the filter values from the filter_offset parameter in the
+// bilinear filter functions below - we don't have to actually load the values
+// from memory:
+// f0 = 8 - filter_offset
+// f1 = filter_offset
+//
+// 2) Scaling the pixel values by 8, instead of 128 enables us to operate on
+// 16-bit data types at all times, rather than widening out to 32-bit and
+// requiring double the number of data processing instructions. (12-bit * 8 =
+// 15-bit.)
+
+// Process a block exactly 4 wide and any height.
+static void highbd_var_filter_block2d_bil_w4(const uint16_t *src_ptr,
+ uint16_t *dst_ptr, int src_stride,
+ int pixel_step, int dst_height,
+ int filter_offset) {
+ const uint16x4_t f0 = vdup_n_u16(8 - filter_offset);
+ const uint16x4_t f1 = vdup_n_u16(filter_offset);
+
+ int i = dst_height;
+ do {
+ uint16x4_t s0 = load_unaligned_u16_4x1(src_ptr);
+ uint16x4_t s1 = load_unaligned_u16_4x1(src_ptr + pixel_step);
+
+ uint16x4_t blend = vmul_u16(s0, f0);
+ blend = vmla_u16(blend, s1, f1);
+ blend = vrshr_n_u16(blend, 3);
+
+ vst1_u16(dst_ptr, blend);
+
+ src_ptr += src_stride;
+ dst_ptr += 4;
+ } while (--i != 0);
+}
+
+// Process a block which is a multiple of 8 and any height.
+static void highbd_var_filter_block2d_bil_large(const uint16_t *src_ptr,
+ uint16_t *dst_ptr,
+ int src_stride, int pixel_step,
+ int dst_width, int dst_height,
+ int filter_offset) {
+ const uint16x8_t f0 = vdupq_n_u16(8 - filter_offset);
+ const uint16x8_t f1 = vdupq_n_u16(filter_offset);
+
+ int i = dst_height;
+ do {
+ int j = 0;
+ do {
+ uint16x8_t s0 = vld1q_u16(src_ptr + j);
+ uint16x8_t s1 = vld1q_u16(src_ptr + j + pixel_step);
+
+ uint16x8_t blend = vmulq_u16(s0, f0);
+ blend = vmlaq_u16(blend, s1, f1);
+ blend = vrshrq_n_u16(blend, 3);
+
+ vst1q_u16(dst_ptr + j, blend);
+
+ j += 8;
+ } while (j < dst_width);
+
+ src_ptr += src_stride;
+ dst_ptr += dst_width;
+ } while (--i != 0);
+}
+
+static void highbd_var_filter_block2d_bil_w8(const uint16_t *src_ptr,
+ uint16_t *dst_ptr, int src_stride,
+ int pixel_step, int dst_height,
+ int filter_offset) {
+ highbd_var_filter_block2d_bil_large(src_ptr, dst_ptr, src_stride, pixel_step,
+ 8, dst_height, filter_offset);
+}
+
+static void highbd_var_filter_block2d_bil_w16(const uint16_t *src_ptr,
+ uint16_t *dst_ptr, int src_stride,
+ int pixel_step, int dst_height,
+ int filter_offset) {
+ highbd_var_filter_block2d_bil_large(src_ptr, dst_ptr, src_stride, pixel_step,
+ 16, dst_height, filter_offset);
+}
+
+static void highbd_var_filter_block2d_bil_w32(const uint16_t *src_ptr,
+ uint16_t *dst_ptr, int src_stride,
+ int pixel_step, int dst_height,
+ int filter_offset) {
+ highbd_var_filter_block2d_bil_large(src_ptr, dst_ptr, src_stride, pixel_step,
+ 32, dst_height, filter_offset);
+}
+
+static void highbd_var_filter_block2d_bil_w64(const uint16_t *src_ptr,
+ uint16_t *dst_ptr, int src_stride,
+ int pixel_step, int dst_height,
+ int filter_offset) {
+ highbd_var_filter_block2d_bil_large(src_ptr, dst_ptr, src_stride, pixel_step,
+ 64, dst_height, filter_offset);
+}
+
+static void highbd_var_filter_block2d_bil_w128(const uint16_t *src_ptr,
+ uint16_t *dst_ptr,
+ int src_stride, int pixel_step,
+ int dst_height,
+ int filter_offset) {
+ highbd_var_filter_block2d_bil_large(src_ptr, dst_ptr, src_stride, pixel_step,
+ 128, dst_height, filter_offset);
+}
+
+static void highbd_var_filter_block2d_avg(const uint16_t *src_ptr,
+ uint16_t *dst_ptr, int src_stride,
+ int pixel_step, int dst_width,
+ int dst_height) {
+ int i = dst_height;
+
+ // We only specialize on the filter values for large block sizes (>= 16x16.)
+ assert(dst_width >= 16 && dst_width % 16 == 0);
+
+ do {
+ int j = 0;
+ do {
+ uint16x8_t s0 = vld1q_u16(src_ptr + j);
+ uint16x8_t s1 = vld1q_u16(src_ptr + j + pixel_step);
+ uint16x8_t avg = vrhaddq_u16(s0, s1);
+ vst1q_u16(dst_ptr + j, avg);
+
+ j += 8;
+ } while (j < dst_width);
+
+ src_ptr += src_stride;
+ dst_ptr += dst_width;
+ } while (--i != 0);
+}
+
+#define HBD_SUBPEL_VARIANCE_WXH_NEON(bitdepth, w, h) \
+ unsigned int aom_highbd_##bitdepth##_sub_pixel_variance##w##x##h##_neon( \
+ const uint8_t *src, int src_stride, int xoffset, int yoffset, \
+ const uint8_t *ref, int ref_stride, uint32_t *sse) { \
+ uint16_t tmp0[w * (h + 1)]; \
+ uint16_t tmp1[w * h]; \
+ uint16_t *src_ptr = CONVERT_TO_SHORTPTR(src); \
+ \
+ highbd_var_filter_block2d_bil_w##w(src_ptr, tmp0, src_stride, 1, (h + 1), \
+ xoffset); \
+ highbd_var_filter_block2d_bil_w##w(tmp0, tmp1, w, w, h, yoffset); \
+ \
+ return aom_highbd_##bitdepth##_variance##w##x##h(CONVERT_TO_BYTEPTR(tmp1), \
+ w, ref, ref_stride, sse); \
+ }
+
+#define HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(bitdepth, w, h) \
+ unsigned int aom_highbd_##bitdepth##_sub_pixel_variance##w##x##h##_neon( \
+ const uint8_t *src, int src_stride, int xoffset, int yoffset, \
+ const uint8_t *ref, int ref_stride, unsigned int *sse) { \
+ uint16_t *src_ptr = CONVERT_TO_SHORTPTR(src); \
+ \
+ if (xoffset == 0) { \
+ if (yoffset == 0) { \
+ return aom_highbd_##bitdepth##_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(src_ptr), src_stride, ref, ref_stride, sse); \
+ } else if (yoffset == 4) { \
+ uint16_t tmp[w * h]; \
+ highbd_var_filter_block2d_avg(src_ptr, tmp, src_stride, src_stride, w, \
+ h); \
+ return aom_highbd_##bitdepth##_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp), w, ref, ref_stride, sse); \
+ } else { \
+ uint16_t tmp[w * h]; \
+ highbd_var_filter_block2d_bil_w##w(src_ptr, tmp, src_stride, \
+ src_stride, h, yoffset); \
+ return aom_highbd_##bitdepth##_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp), w, ref, ref_stride, sse); \
+ } \
+ } else if (xoffset == 4) { \
+ uint16_t tmp0[w * (h + 1)]; \
+ if (yoffset == 0) { \
+ highbd_var_filter_block2d_avg(src_ptr, tmp0, src_stride, 1, w, h); \
+ return aom_highbd_##bitdepth##_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp0), w, ref, ref_stride, sse); \
+ } else if (yoffset == 4) { \
+ uint16_t tmp1[w * (h + 1)]; \
+ highbd_var_filter_block2d_avg(src_ptr, tmp0, src_stride, 1, w, \
+ (h + 1)); \
+ highbd_var_filter_block2d_avg(tmp0, tmp1, w, w, w, h); \
+ return aom_highbd_##bitdepth##_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp1), w, ref, ref_stride, sse); \
+ } else { \
+ uint16_t tmp1[w * (h + 1)]; \
+ highbd_var_filter_block2d_avg(src_ptr, tmp0, src_stride, 1, w, \
+ (h + 1)); \
+ highbd_var_filter_block2d_bil_w##w(tmp0, tmp1, w, w, h, yoffset); \
+ return aom_highbd_##bitdepth##_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp1), w, ref, ref_stride, sse); \
+ } \
+ } else { \
+ uint16_t tmp0[w * (h + 1)]; \
+ if (yoffset == 0) { \
+ highbd_var_filter_block2d_bil_w##w(src_ptr, tmp0, src_stride, 1, h, \
+ xoffset); \
+ return aom_highbd_##bitdepth##_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp0), w, ref, ref_stride, sse); \
+ } else if (yoffset == 4) { \
+ uint16_t tmp1[w * h]; \
+ highbd_var_filter_block2d_bil_w##w(src_ptr, tmp0, src_stride, 1, \
+ (h + 1), xoffset); \
+ highbd_var_filter_block2d_avg(tmp0, tmp1, w, w, w, h); \
+ return aom_highbd_##bitdepth##_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp1), w, ref, ref_stride, sse); \
+ } else { \
+ uint16_t tmp1[w * h]; \
+ highbd_var_filter_block2d_bil_w##w(src_ptr, tmp0, src_stride, 1, \
+ (h + 1), xoffset); \
+ highbd_var_filter_block2d_bil_w##w(tmp0, tmp1, w, w, h, yoffset); \
+ return aom_highbd_##bitdepth##_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp1), w, ref, ref_stride, sse); \
+ } \
+ } \
+ }
+
+// 8-bit
+HBD_SUBPEL_VARIANCE_WXH_NEON(8, 4, 4)
+HBD_SUBPEL_VARIANCE_WXH_NEON(8, 4, 8)
+
+HBD_SUBPEL_VARIANCE_WXH_NEON(8, 8, 4)
+HBD_SUBPEL_VARIANCE_WXH_NEON(8, 8, 8)
+HBD_SUBPEL_VARIANCE_WXH_NEON(8, 8, 16)
+
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(8, 16, 8)
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(8, 16, 16)
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(8, 16, 32)
+
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(8, 32, 16)
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(8, 32, 32)
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(8, 32, 64)
+
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(8, 64, 32)
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(8, 64, 64)
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(8, 64, 128)
+
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(8, 128, 64)
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(8, 128, 128)
+
+#if !CONFIG_REALTIME_ONLY
+HBD_SUBPEL_VARIANCE_WXH_NEON(8, 4, 16)
+
+HBD_SUBPEL_VARIANCE_WXH_NEON(8, 8, 32)
+
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(8, 16, 4)
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(8, 16, 64)
+
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(8, 32, 8)
+
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(8, 64, 16)
+#endif // !CONFIG_REALTIME_ONLY
+
+// 10-bit
+HBD_SUBPEL_VARIANCE_WXH_NEON(10, 4, 4)
+HBD_SUBPEL_VARIANCE_WXH_NEON(10, 4, 8)
+
+HBD_SUBPEL_VARIANCE_WXH_NEON(10, 8, 4)
+HBD_SUBPEL_VARIANCE_WXH_NEON(10, 8, 8)
+HBD_SUBPEL_VARIANCE_WXH_NEON(10, 8, 16)
+
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(10, 16, 8)
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(10, 16, 16)
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(10, 16, 32)
+
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(10, 32, 16)
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(10, 32, 32)
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(10, 32, 64)
+
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(10, 64, 32)
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(10, 64, 64)
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(10, 64, 128)
+
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(10, 128, 64)
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(10, 128, 128)
+
+#if !CONFIG_REALTIME_ONLY
+HBD_SUBPEL_VARIANCE_WXH_NEON(10, 4, 16)
+
+HBD_SUBPEL_VARIANCE_WXH_NEON(10, 8, 32)
+
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(10, 16, 4)
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(10, 16, 64)
+
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(10, 32, 8)
+
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(10, 64, 16)
+#endif // !CONFIG_REALTIME_ONLY
+
+// 12-bit
+HBD_SUBPEL_VARIANCE_WXH_NEON(12, 4, 4)
+HBD_SUBPEL_VARIANCE_WXH_NEON(12, 4, 8)
+
+HBD_SUBPEL_VARIANCE_WXH_NEON(12, 8, 4)
+HBD_SUBPEL_VARIANCE_WXH_NEON(12, 8, 8)
+HBD_SUBPEL_VARIANCE_WXH_NEON(12, 8, 16)
+
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(12, 16, 8)
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(12, 16, 16)
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(12, 16, 32)
+
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(12, 32, 16)
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(12, 32, 32)
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(12, 32, 64)
+
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(12, 64, 32)
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(12, 64, 64)
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(12, 64, 128)
+
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(12, 128, 64)
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(12, 128, 128)
+
+#if !CONFIG_REALTIME_ONLY
+HBD_SUBPEL_VARIANCE_WXH_NEON(12, 4, 16)
+
+HBD_SUBPEL_VARIANCE_WXH_NEON(12, 8, 32)
+
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(12, 16, 4)
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(12, 16, 64)
+
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(12, 32, 8)
+
+HBD_SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(12, 64, 16)
+#endif // !CONFIG_REALTIME_ONLY
+
+// Combine bilinear filter with aom_highbd_comp_avg_pred for blocks having
+// width 4.
+static void highbd_avg_pred_var_filter_block2d_bil_w4(
+ const uint16_t *src_ptr, uint16_t *dst_ptr, int src_stride, int pixel_step,
+ int dst_height, int filter_offset, const uint16_t *second_pred) {
+ const uint16x4_t f0 = vdup_n_u16(8 - filter_offset);
+ const uint16x4_t f1 = vdup_n_u16(filter_offset);
+
+ int i = dst_height;
+ do {
+ uint16x4_t s0 = load_unaligned_u16_4x1(src_ptr);
+ uint16x4_t s1 = load_unaligned_u16_4x1(src_ptr + pixel_step);
+ uint16x4_t p = vld1_u16(second_pred);
+
+ uint16x4_t blend = vmul_u16(s0, f0);
+ blend = vmla_u16(blend, s1, f1);
+ blend = vrshr_n_u16(blend, 3);
+
+ vst1_u16(dst_ptr, vrhadd_u16(blend, p));
+
+ src_ptr += src_stride;
+ dst_ptr += 4;
+ second_pred += 4;
+ } while (--i != 0);
+}
+
+// Combine bilinear filter with aom_highbd_comp_avg_pred for large blocks.
+static void highbd_avg_pred_var_filter_block2d_bil_large(
+ const uint16_t *src_ptr, uint16_t *dst_ptr, int src_stride, int pixel_step,
+ int dst_width, int dst_height, int filter_offset,
+ const uint16_t *second_pred) {
+ const uint16x8_t f0 = vdupq_n_u16(8 - filter_offset);
+ const uint16x8_t f1 = vdupq_n_u16(filter_offset);
+
+ int i = dst_height;
+ do {
+ int j = 0;
+ do {
+ uint16x8_t s0 = vld1q_u16(src_ptr + j);
+ uint16x8_t s1 = vld1q_u16(src_ptr + j + pixel_step);
+ uint16x8_t p = vld1q_u16(second_pred);
+
+ uint16x8_t blend = vmulq_u16(s0, f0);
+ blend = vmlaq_u16(blend, s1, f1);
+ blend = vrshrq_n_u16(blend, 3);
+
+ vst1q_u16(dst_ptr + j, vrhaddq_u16(blend, p));
+
+ j += 8;
+ second_pred += 8;
+ } while (j < dst_width);
+
+ src_ptr += src_stride;
+ dst_ptr += dst_width;
+ } while (--i != 0);
+}
+
+static void highbd_avg_pred_var_filter_block2d_bil_w8(
+ const uint16_t *src_ptr, uint16_t *dst_ptr, int src_stride, int pixel_step,
+ int dst_height, int filter_offset, const uint16_t *second_pred) {
+ highbd_avg_pred_var_filter_block2d_bil_large(src_ptr, dst_ptr, src_stride,
+ pixel_step, 8, dst_height,
+ filter_offset, second_pred);
+}
+
+static void highbd_avg_pred_var_filter_block2d_bil_w16(
+ const uint16_t *src_ptr, uint16_t *dst_ptr, int src_stride, int pixel_step,
+ int dst_height, int filter_offset, const uint16_t *second_pred) {
+ highbd_avg_pred_var_filter_block2d_bil_large(src_ptr, dst_ptr, src_stride,
+ pixel_step, 16, dst_height,
+ filter_offset, second_pred);
+}
+
+static void highbd_avg_pred_var_filter_block2d_bil_w32(
+ const uint16_t *src_ptr, uint16_t *dst_ptr, int src_stride, int pixel_step,
+ int dst_height, int filter_offset, const uint16_t *second_pred) {
+ highbd_avg_pred_var_filter_block2d_bil_large(src_ptr, dst_ptr, src_stride,
+ pixel_step, 32, dst_height,
+ filter_offset, second_pred);
+}
+
+static void highbd_avg_pred_var_filter_block2d_bil_w64(
+ const uint16_t *src_ptr, uint16_t *dst_ptr, int src_stride, int pixel_step,
+ int dst_height, int filter_offset, const uint16_t *second_pred) {
+ highbd_avg_pred_var_filter_block2d_bil_large(src_ptr, dst_ptr, src_stride,
+ pixel_step, 64, dst_height,
+ filter_offset, second_pred);
+}
+
+static void highbd_avg_pred_var_filter_block2d_bil_w128(
+ const uint16_t *src_ptr, uint16_t *dst_ptr, int src_stride, int pixel_step,
+ int dst_height, int filter_offset, const uint16_t *second_pred) {
+ highbd_avg_pred_var_filter_block2d_bil_large(src_ptr, dst_ptr, src_stride,
+ pixel_step, 128, dst_height,
+ filter_offset, second_pred);
+}
+
+// Combine averaging subpel filter with aom_highbd_comp_avg_pred.
+static void highbd_avg_pred_var_filter_block2d_avg(
+ const uint16_t *src_ptr, uint16_t *dst_ptr, int src_stride, int pixel_step,
+ int dst_width, int dst_height, const uint16_t *second_pred) {
+ int i = dst_height;
+
+ // We only specialize on the filter values for large block sizes (>= 16x16.)
+ assert(dst_width >= 16 && dst_width % 16 == 0);
+
+ do {
+ int j = 0;
+ do {
+ uint16x8_t s0 = vld1q_u16(src_ptr + j);
+ uint16x8_t s1 = vld1q_u16(src_ptr + j + pixel_step);
+ uint16x8_t avg = vrhaddq_u16(s0, s1);
+
+ uint16x8_t p = vld1q_u16(second_pred);
+ avg = vrhaddq_u16(avg, p);
+
+ vst1q_u16(dst_ptr + j, avg);
+
+ j += 8;
+ second_pred += 8;
+ } while (j < dst_width);
+
+ src_ptr += src_stride;
+ dst_ptr += dst_width;
+ } while (--i != 0);
+}
+
+// Implementation of aom_highbd_comp_avg_pred for blocks having width >= 16.
+static void highbd_avg_pred(const uint16_t *src_ptr, uint16_t *dst_ptr,
+ int src_stride, int dst_width, int dst_height,
+ const uint16_t *second_pred) {
+ int i = dst_height;
+
+ // We only specialize on the filter values for large block sizes (>= 16x16.)
+ assert(dst_width >= 16 && dst_width % 16 == 0);
+
+ do {
+ int j = 0;
+ do {
+ uint16x8_t s = vld1q_u16(src_ptr + j);
+ uint16x8_t p = vld1q_u16(second_pred);
+
+ uint16x8_t avg = vrhaddq_u16(s, p);
+
+ vst1q_u16(dst_ptr + j, avg);
+
+ j += 8;
+ second_pred += 8;
+ } while (j < dst_width);
+
+ src_ptr += src_stride;
+ dst_ptr += dst_width;
+ } while (--i != 0);
+}
+
+#define HBD_SUBPEL_AVG_VARIANCE_WXH_NEON(bitdepth, w, h) \
+ uint32_t aom_highbd_##bitdepth##_sub_pixel_avg_variance##w##x##h##_neon( \
+ const uint8_t *src, int src_stride, int xoffset, int yoffset, \
+ const uint8_t *ref, int ref_stride, uint32_t *sse, \
+ const uint8_t *second_pred) { \
+ uint16_t tmp0[w * (h + 1)]; \
+ uint16_t tmp1[w * h]; \
+ uint16_t *src_ptr = CONVERT_TO_SHORTPTR(src); \
+ \
+ highbd_var_filter_block2d_bil_w##w(src_ptr, tmp0, src_stride, 1, (h + 1), \
+ xoffset); \
+ highbd_avg_pred_var_filter_block2d_bil_w##w( \
+ tmp0, tmp1, w, w, h, yoffset, CONVERT_TO_SHORTPTR(second_pred)); \
+ \
+ return aom_highbd_##bitdepth##_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp1), w, ref, ref_stride, sse); \
+ }
+
+#define HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(bitdepth, w, h) \
+ unsigned int aom_highbd_##bitdepth##_sub_pixel_avg_variance##w##x##h##_neon( \
+ const uint8_t *src, int source_stride, int xoffset, int yoffset, \
+ const uint8_t *ref, int ref_stride, uint32_t *sse, \
+ const uint8_t *second_pred) { \
+ uint16_t *src_ptr = CONVERT_TO_SHORTPTR(src); \
+ \
+ if (xoffset == 0) { \
+ uint16_t tmp[w * h]; \
+ if (yoffset == 0) { \
+ highbd_avg_pred(src_ptr, tmp, source_stride, w, h, \
+ CONVERT_TO_SHORTPTR(second_pred)); \
+ return aom_highbd_##bitdepth##_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp), w, ref, ref_stride, sse); \
+ } else if (yoffset == 4) { \
+ highbd_avg_pred_var_filter_block2d_avg( \
+ src_ptr, tmp, source_stride, source_stride, w, h, \
+ CONVERT_TO_SHORTPTR(second_pred)); \
+ return aom_highbd_##bitdepth##_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp), w, ref, ref_stride, sse); \
+ } else { \
+ highbd_avg_pred_var_filter_block2d_bil_w##w( \
+ src_ptr, tmp, source_stride, source_stride, h, yoffset, \
+ CONVERT_TO_SHORTPTR(second_pred)); \
+ return aom_highbd_##bitdepth##_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp), w, ref, ref_stride, sse); \
+ } \
+ } else if (xoffset == 4) { \
+ uint16_t tmp0[w * (h + 1)]; \
+ if (yoffset == 0) { \
+ highbd_avg_pred_var_filter_block2d_avg( \
+ src_ptr, tmp0, source_stride, 1, w, h, \
+ CONVERT_TO_SHORTPTR(second_pred)); \
+ return aom_highbd_##bitdepth##_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp0), w, ref, ref_stride, sse); \
+ } else if (yoffset == 4) { \
+ uint16_t tmp1[w * (h + 1)]; \
+ highbd_var_filter_block2d_avg(src_ptr, tmp0, source_stride, 1, w, \
+ (h + 1)); \
+ highbd_avg_pred_var_filter_block2d_avg( \
+ tmp0, tmp1, w, w, w, h, CONVERT_TO_SHORTPTR(second_pred)); \
+ return aom_highbd_##bitdepth##_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp1), w, ref, ref_stride, sse); \
+ } else { \
+ uint16_t tmp1[w * (h + 1)]; \
+ highbd_var_filter_block2d_avg(src_ptr, tmp0, source_stride, 1, w, \
+ (h + 1)); \
+ highbd_avg_pred_var_filter_block2d_bil_w##w( \
+ tmp0, tmp1, w, w, h, yoffset, CONVERT_TO_SHORTPTR(second_pred)); \
+ return aom_highbd_##bitdepth##_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp1), w, ref, ref_stride, sse); \
+ } \
+ } else { \
+ uint16_t tmp0[w * (h + 1)]; \
+ if (yoffset == 0) { \
+ highbd_avg_pred_var_filter_block2d_bil_w##w( \
+ src_ptr, tmp0, source_stride, 1, h, xoffset, \
+ CONVERT_TO_SHORTPTR(second_pred)); \
+ return aom_highbd_##bitdepth##_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp0), w, ref, ref_stride, sse); \
+ } else if (yoffset == 4) { \
+ uint16_t tmp1[w * h]; \
+ highbd_var_filter_block2d_bil_w##w(src_ptr, tmp0, source_stride, 1, \
+ (h + 1), xoffset); \
+ highbd_avg_pred_var_filter_block2d_avg( \
+ tmp0, tmp1, w, w, w, h, CONVERT_TO_SHORTPTR(second_pred)); \
+ return aom_highbd_##bitdepth##_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp1), w, ref, ref_stride, sse); \
+ } else { \
+ uint16_t tmp1[w * h]; \
+ highbd_var_filter_block2d_bil_w##w(src_ptr, tmp0, source_stride, 1, \
+ (h + 1), xoffset); \
+ highbd_avg_pred_var_filter_block2d_bil_w##w( \
+ tmp0, tmp1, w, w, h, yoffset, CONVERT_TO_SHORTPTR(second_pred)); \
+ return aom_highbd_##bitdepth##_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp1), w, ref, ref_stride, sse); \
+ } \
+ } \
+ }
+
+// 8-bit
+HBD_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 4, 4)
+HBD_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 4, 8)
+
+HBD_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 8, 4)
+HBD_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 8, 8)
+HBD_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 8, 16)
+
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 16, 8)
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 16, 16)
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 16, 32)
+
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 32, 16)
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 32, 32)
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 32, 64)
+
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 64, 32)
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 64, 64)
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 64, 128)
+
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 128, 64)
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 128, 128)
+
+#if !CONFIG_REALTIME_ONLY
+HBD_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 4, 16)
+
+HBD_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 8, 32)
+
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 16, 4)
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 16, 64)
+
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 32, 8)
+
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 64, 16)
+#endif // !CONFIG_REALTIME_ONLY
+
+// 10-bit
+HBD_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 4, 4)
+HBD_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 4, 8)
+
+HBD_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 8, 4)
+HBD_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 8, 8)
+HBD_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 8, 16)
+
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 16, 8)
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 16, 16)
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 16, 32)
+
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 32, 16)
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 32, 32)
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 32, 64)
+
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 64, 32)
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 64, 64)
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 64, 128)
+
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 128, 64)
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 128, 128)
+
+#if !CONFIG_REALTIME_ONLY
+HBD_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 4, 16)
+
+HBD_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 8, 32)
+
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 16, 4)
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 16, 64)
+
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 32, 8)
+
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 64, 16)
+#endif // !CONFIG_REALTIME_ONLY
+
+// 12-bit
+HBD_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 4, 4)
+HBD_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 4, 8)
+
+HBD_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 8, 4)
+HBD_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 8, 8)
+HBD_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 8, 16)
+
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 16, 8)
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 16, 16)
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 16, 32)
+
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 32, 16)
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 32, 32)
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 32, 64)
+
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 64, 32)
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 64, 64)
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 64, 128)
+
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 128, 64)
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 128, 128)
+
+#if !CONFIG_REALTIME_ONLY
+HBD_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 4, 16)
+
+HBD_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 8, 32)
+
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 16, 4)
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 16, 64)
+
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 32, 8)
+
+HBD_SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 64, 16)
+#endif // !CONFIG_REALTIME_ONLY
+
+#define HBD_MASKED_SUBPEL_VARIANCE_WXH_NEON(bitdepth, w, h) \
+ unsigned int \
+ aom_highbd_##bitdepth##_masked_sub_pixel_variance##w##x##h##_neon( \
+ const uint8_t *src, int src_stride, int xoffset, int yoffset, \
+ const uint8_t *ref, int ref_stride, const uint8_t *second_pred, \
+ const uint8_t *msk, int msk_stride, int invert_mask, \
+ unsigned int *sse) { \
+ uint16_t tmp0[w * (h + 1)]; \
+ uint16_t tmp1[w * (h + 1)]; \
+ uint16_t tmp2[w * h]; \
+ uint16_t *src_ptr = CONVERT_TO_SHORTPTR(src); \
+ highbd_var_filter_block2d_bil_w##w(src_ptr, tmp0, src_stride, 1, (h + 1), \
+ xoffset); \
+ highbd_var_filter_block2d_bil_w##w(tmp0, tmp1, w, w, h, yoffset); \
+ aom_highbd_comp_mask_pred_neon(CONVERT_TO_BYTEPTR(tmp2), second_pred, w, \
+ h, CONVERT_TO_BYTEPTR(tmp1), w, msk, \
+ msk_stride, invert_mask); \
+ return aom_highbd_##bitdepth##_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp2), w, ref, ref_stride, sse); \
+ }
+
+#define HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(bitdepth, w, h) \
+ unsigned int \
+ aom_highbd_##bitdepth##_masked_sub_pixel_variance##w##x##h##_neon( \
+ const uint8_t *src, int src_stride, int xoffset, int yoffset, \
+ const uint8_t *ref, int ref_stride, const uint8_t *second_pred, \
+ const uint8_t *msk, int msk_stride, int invert_mask, \
+ unsigned int *sse) { \
+ uint16_t *src_ptr = CONVERT_TO_SHORTPTR(src); \
+ if (xoffset == 0) { \
+ uint16_t tmp0[w * h]; \
+ if (yoffset == 0) { \
+ aom_highbd_comp_mask_pred_neon(CONVERT_TO_BYTEPTR(tmp0), second_pred, \
+ w, h, src, src_stride, msk, msk_stride, \
+ invert_mask); \
+ return aom_highbd_##bitdepth##_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp0), w, ref, ref_stride, sse); \
+ } else if (yoffset == 4) { \
+ uint16_t tmp1[w * h]; \
+ highbd_var_filter_block2d_avg(src_ptr, tmp0, src_stride, src_stride, \
+ w, h); \
+ aom_highbd_comp_mask_pred_neon(CONVERT_TO_BYTEPTR(tmp1), second_pred, \
+ w, h, CONVERT_TO_BYTEPTR(tmp0), w, msk, \
+ msk_stride, invert_mask); \
+ return aom_highbd_##bitdepth##_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp1), w, ref, ref_stride, sse); \
+ } else { \
+ uint16_t tmp1[w * h]; \
+ highbd_var_filter_block2d_bil_w##w(src_ptr, tmp0, src_stride, \
+ src_stride, h, yoffset); \
+ aom_highbd_comp_mask_pred_neon(CONVERT_TO_BYTEPTR(tmp1), second_pred, \
+ w, h, CONVERT_TO_BYTEPTR(tmp0), w, msk, \
+ msk_stride, invert_mask); \
+ return aom_highbd_##bitdepth##_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp1), w, ref, ref_stride, sse); \
+ } \
+ } else if (xoffset == 4) { \
+ uint16_t tmp0[w * (h + 1)]; \
+ if (yoffset == 0) { \
+ uint16_t tmp1[w * h]; \
+ highbd_var_filter_block2d_avg(src_ptr, tmp0, src_stride, 1, w, h); \
+ aom_highbd_comp_mask_pred_neon(CONVERT_TO_BYTEPTR(tmp1), second_pred, \
+ w, h, CONVERT_TO_BYTEPTR(tmp0), w, msk, \
+ msk_stride, invert_mask); \
+ return aom_highbd_##bitdepth##_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp1), w, ref, ref_stride, sse); \
+ } else if (yoffset == 4) { \
+ uint16_t tmp1[w * h]; \
+ uint16_t tmp2[w * h]; \
+ highbd_var_filter_block2d_avg(src_ptr, tmp0, src_stride, 1, w, \
+ (h + 1)); \
+ highbd_var_filter_block2d_avg(tmp0, tmp1, w, w, w, h); \
+ aom_highbd_comp_mask_pred_neon(CONVERT_TO_BYTEPTR(tmp2), second_pred, \
+ w, h, CONVERT_TO_BYTEPTR(tmp1), w, msk, \
+ msk_stride, invert_mask); \
+ return aom_highbd_##bitdepth##_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp2), w, ref, ref_stride, sse); \
+ } else { \
+ uint16_t tmp1[w * h]; \
+ uint16_t tmp2[w * h]; \
+ highbd_var_filter_block2d_avg(src_ptr, tmp0, src_stride, 1, w, \
+ (h + 1)); \
+ highbd_var_filter_block2d_bil_w##w(tmp0, tmp1, w, w, h, yoffset); \
+ aom_highbd_comp_mask_pred_neon(CONVERT_TO_BYTEPTR(tmp2), second_pred, \
+ w, h, CONVERT_TO_BYTEPTR(tmp1), w, msk, \
+ msk_stride, invert_mask); \
+ return aom_highbd_##bitdepth##_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp2), w, ref, ref_stride, sse); \
+ } \
+ } else { \
+ if (yoffset == 0) { \
+ uint16_t tmp0[w * h]; \
+ uint16_t tmp1[w * h]; \
+ highbd_var_filter_block2d_bil_w##w(src_ptr, tmp0, src_stride, 1, h, \
+ xoffset); \
+ aom_highbd_comp_mask_pred_neon(CONVERT_TO_BYTEPTR(tmp1), second_pred, \
+ w, h, CONVERT_TO_BYTEPTR(tmp0), w, msk, \
+ msk_stride, invert_mask); \
+ return aom_highbd_##bitdepth##_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp1), w, ref, ref_stride, sse); \
+ } else if (yoffset == 4) { \
+ uint16_t tmp0[w * (h + 1)]; \
+ uint16_t tmp1[w * h]; \
+ uint16_t tmp2[w * h]; \
+ highbd_var_filter_block2d_bil_w##w(src_ptr, tmp0, src_stride, 1, \
+ (h + 1), xoffset); \
+ highbd_var_filter_block2d_avg(tmp0, tmp1, w, w, w, h); \
+ aom_highbd_comp_mask_pred_neon(CONVERT_TO_BYTEPTR(tmp2), second_pred, \
+ w, h, CONVERT_TO_BYTEPTR(tmp1), w, msk, \
+ msk_stride, invert_mask); \
+ return aom_highbd_##bitdepth##_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp2), w, ref, ref_stride, sse); \
+ } else { \
+ uint16_t tmp0[w * (h + 1)]; \
+ uint16_t tmp1[w * (h + 1)]; \
+ uint16_t tmp2[w * h]; \
+ highbd_var_filter_block2d_bil_w##w(src_ptr, tmp0, src_stride, 1, \
+ (h + 1), xoffset); \
+ highbd_var_filter_block2d_bil_w##w(tmp0, tmp1, w, w, h, yoffset); \
+ aom_highbd_comp_mask_pred_neon(CONVERT_TO_BYTEPTR(tmp2), second_pred, \
+ w, h, CONVERT_TO_BYTEPTR(tmp1), w, msk, \
+ msk_stride, invert_mask); \
+ return aom_highbd_##bitdepth##_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp2), w, ref, ref_stride, sse); \
+ } \
+ } \
+ }
+
+// 8-bit
+HBD_MASKED_SUBPEL_VARIANCE_WXH_NEON(8, 4, 4)
+HBD_MASKED_SUBPEL_VARIANCE_WXH_NEON(8, 4, 8)
+
+HBD_MASKED_SUBPEL_VARIANCE_WXH_NEON(8, 8, 4)
+HBD_MASKED_SUBPEL_VARIANCE_WXH_NEON(8, 8, 8)
+HBD_MASKED_SUBPEL_VARIANCE_WXH_NEON(8, 8, 16)
+
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(8, 16, 8)
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(8, 16, 16)
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(8, 16, 32)
+
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(8, 32, 16)
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(8, 32, 32)
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(8, 32, 64)
+
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(8, 64, 32)
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(8, 64, 64)
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(8, 64, 128)
+
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(8, 128, 64)
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(8, 128, 128)
+
+#if !CONFIG_REALTIME_ONLY
+HBD_MASKED_SUBPEL_VARIANCE_WXH_NEON(8, 4, 16)
+
+HBD_MASKED_SUBPEL_VARIANCE_WXH_NEON(8, 8, 32)
+
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(8, 16, 4)
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(8, 16, 64)
+
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(8, 32, 8)
+
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(8, 64, 16)
+#endif // !CONFIG_REALTIME_ONLY
+
+// 10-bit
+HBD_MASKED_SUBPEL_VARIANCE_WXH_NEON(10, 4, 4)
+HBD_MASKED_SUBPEL_VARIANCE_WXH_NEON(10, 4, 8)
+
+HBD_MASKED_SUBPEL_VARIANCE_WXH_NEON(10, 8, 4)
+HBD_MASKED_SUBPEL_VARIANCE_WXH_NEON(10, 8, 8)
+HBD_MASKED_SUBPEL_VARIANCE_WXH_NEON(10, 8, 16)
+
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(10, 16, 8)
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(10, 16, 16)
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(10, 16, 32)
+
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(10, 32, 16)
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(10, 32, 32)
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(10, 32, 64)
+
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(10, 64, 32)
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(10, 64, 64)
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(10, 64, 128)
+
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(10, 128, 64)
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(10, 128, 128)
+
+#if !CONFIG_REALTIME_ONLY
+HBD_MASKED_SUBPEL_VARIANCE_WXH_NEON(10, 4, 16)
+
+HBD_MASKED_SUBPEL_VARIANCE_WXH_NEON(10, 8, 32)
+
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(10, 16, 4)
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(10, 16, 64)
+
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(10, 32, 8)
+
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(10, 64, 16)
+#endif // !CONFIG_REALTIME_ONLY
+
+// 12-bit
+HBD_MASKED_SUBPEL_VARIANCE_WXH_NEON(12, 4, 4)
+HBD_MASKED_SUBPEL_VARIANCE_WXH_NEON(12, 4, 8)
+
+HBD_MASKED_SUBPEL_VARIANCE_WXH_NEON(12, 8, 4)
+HBD_MASKED_SUBPEL_VARIANCE_WXH_NEON(12, 8, 8)
+HBD_MASKED_SUBPEL_VARIANCE_WXH_NEON(12, 8, 16)
+
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(12, 16, 8)
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(12, 16, 16)
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(12, 16, 32)
+
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(12, 32, 16)
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(12, 32, 32)
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(12, 32, 64)
+
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(12, 64, 32)
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(12, 64, 64)
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(12, 64, 128)
+
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(12, 128, 64)
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(12, 128, 128)
+
+#if !CONFIG_REALTIME_ONLY
+HBD_MASKED_SUBPEL_VARIANCE_WXH_NEON(12, 4, 16)
+
+HBD_MASKED_SUBPEL_VARIANCE_WXH_NEON(12, 8, 32)
+
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(12, 16, 4)
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(12, 16, 64)
+
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(12, 32, 8)
+
+HBD_SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(12, 64, 16)
+#endif // !CONFIG_REALTIME_ONLY
+
+#if !CONFIG_REALTIME_ONLY
+#define HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(bitdepth, w, h) \
+ unsigned int \
+ aom_highbd_##bitdepth##_obmc_sub_pixel_variance##w##x##h##_neon( \
+ const uint8_t *pre, int pre_stride, int xoffset, int yoffset, \
+ const int32_t *wsrc, const int32_t *mask, unsigned int *sse) { \
+ uint16_t *pre_ptr = CONVERT_TO_SHORTPTR(pre); \
+ uint16_t tmp0[w * (h + 1)]; \
+ uint16_t tmp1[w * h]; \
+ highbd_var_filter_block2d_bil_w##w(pre_ptr, tmp0, pre_stride, 1, h + 1, \
+ xoffset); \
+ highbd_var_filter_block2d_bil_w##w(tmp0, tmp1, w, w, h, yoffset); \
+ return aom_highbd_##bitdepth##_obmc_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp1), w, wsrc, mask, sse); \
+ }
+
+#define SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(bitdepth, w, h) \
+ unsigned int \
+ aom_highbd_##bitdepth##_obmc_sub_pixel_variance##w##x##h##_neon( \
+ const uint8_t *pre, int pre_stride, int xoffset, int yoffset, \
+ const int32_t *wsrc, const int32_t *mask, unsigned int *sse) { \
+ uint16_t *pre_ptr = CONVERT_TO_SHORTPTR(pre); \
+ if (xoffset == 0) { \
+ if (yoffset == 0) { \
+ return aom_highbd_##bitdepth##_obmc_variance##w##x##h##_neon( \
+ pre, pre_stride, wsrc, mask, sse); \
+ } else if (yoffset == 4) { \
+ uint16_t tmp[w * h]; \
+ highbd_var_filter_block2d_avg(pre_ptr, tmp, pre_stride, pre_stride, w, \
+ h); \
+ return aom_highbd_##bitdepth##_obmc_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp), w, wsrc, mask, sse); \
+ } else { \
+ uint16_t tmp[w * h]; \
+ highbd_var_filter_block2d_bil_w##w(pre_ptr, tmp, pre_stride, \
+ pre_stride, h, yoffset); \
+ return aom_highbd_##bitdepth##_obmc_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp), w, wsrc, mask, sse); \
+ } \
+ } else if (xoffset == 4) { \
+ uint16_t tmp0[w * (h + 1)]; \
+ if (yoffset == 0) { \
+ highbd_var_filter_block2d_avg(pre_ptr, tmp0, pre_stride, 1, w, h); \
+ return aom_highbd_##bitdepth##_obmc_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp0), w, wsrc, mask, sse); \
+ } else if (yoffset == 4) { \
+ uint16_t tmp1[w * (h + 1)]; \
+ highbd_var_filter_block2d_avg(pre_ptr, tmp0, pre_stride, 1, w, h + 1); \
+ highbd_var_filter_block2d_avg(tmp0, tmp1, w, w, w, h); \
+ return aom_highbd_##bitdepth##_obmc_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp1), w, wsrc, mask, sse); \
+ } else { \
+ uint16_t tmp1[w * (h + 1)]; \
+ highbd_var_filter_block2d_avg(pre_ptr, tmp0, pre_stride, 1, w, h + 1); \
+ highbd_var_filter_block2d_bil_w##w(tmp0, tmp1, w, w, h, yoffset); \
+ return aom_highbd_##bitdepth##_obmc_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp1), w, wsrc, mask, sse); \
+ } \
+ } else { \
+ uint16_t tmp0[w * (h + 1)]; \
+ if (yoffset == 0) { \
+ highbd_var_filter_block2d_bil_w##w(pre_ptr, tmp0, pre_stride, 1, h, \
+ xoffset); \
+ return aom_highbd_##bitdepth##_obmc_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp0), w, wsrc, mask, sse); \
+ } else if (yoffset == 4) { \
+ uint16_t tmp1[w * h]; \
+ highbd_var_filter_block2d_bil_w##w(pre_ptr, tmp0, pre_stride, 1, \
+ h + 1, xoffset); \
+ highbd_var_filter_block2d_avg(tmp0, tmp1, w, w, w, h); \
+ return aom_highbd_##bitdepth##_obmc_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp1), w, wsrc, mask, sse); \
+ } else { \
+ uint16_t tmp1[w * h]; \
+ highbd_var_filter_block2d_bil_w##w(pre_ptr, tmp0, pre_stride, 1, \
+ h + 1, xoffset); \
+ highbd_var_filter_block2d_bil_w##w(tmp0, tmp1, w, w, h, yoffset); \
+ return aom_highbd_##bitdepth##_obmc_variance##w##x##h##_neon( \
+ CONVERT_TO_BYTEPTR(tmp1), w, wsrc, mask, sse); \
+ } \
+ } \
+ }
+
+// 8-bit
+HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(8, 4, 4)
+HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(8, 4, 8)
+HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(8, 4, 16)
+
+HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(8, 8, 4)
+HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(8, 8, 8)
+HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(8, 8, 16)
+HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(8, 8, 32)
+
+HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(8, 16, 4)
+HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(8, 16, 8)
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(8, 16, 16)
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(8, 16, 32)
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(8, 16, 64)
+
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(8, 32, 8)
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(8, 32, 16)
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(8, 32, 32)
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(8, 32, 64)
+
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(8, 64, 16)
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(8, 64, 32)
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(8, 64, 64)
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(8, 64, 128)
+
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(8, 128, 64)
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(8, 128, 128)
+
+// 10-bit
+HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(10, 4, 4)
+HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(10, 4, 8)
+HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(10, 4, 16)
+
+HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(10, 8, 4)
+HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(10, 8, 8)
+HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(10, 8, 16)
+HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(10, 8, 32)
+
+HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(10, 16, 4)
+HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(10, 16, 8)
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(10, 16, 16)
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(10, 16, 32)
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(10, 16, 64)
+
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(10, 32, 8)
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(10, 32, 16)
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(10, 32, 32)
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(10, 32, 64)
+
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(10, 64, 16)
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(10, 64, 32)
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(10, 64, 64)
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(10, 64, 128)
+
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(10, 128, 64)
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(10, 128, 128)
+
+// 12-bit
+HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(12, 4, 4)
+HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(12, 4, 8)
+HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(12, 4, 16)
+
+HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(12, 8, 4)
+HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(12, 8, 8)
+HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(12, 8, 16)
+HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(12, 8, 32)
+
+HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(12, 16, 4)
+HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(12, 16, 8)
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(12, 16, 16)
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(12, 16, 32)
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(12, 16, 64)
+
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(12, 32, 8)
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(12, 32, 16)
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(12, 32, 32)
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(12, 32, 64)
+
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(12, 64, 16)
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(12, 64, 32)
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(12, 64, 64)
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(12, 64, 128)
+
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(12, 128, 64)
+SPECIALIZED_HIGHBD_OBMC_SUBPEL_VARIANCE_WXH_NEON(12, 128, 128)
+#endif // !CONFIG_REALTIME_ONLY
+
+static void highbd_dist_wtd_avg_pred(const uint16_t *src_ptr, uint16_t *dst_ptr,
+ int src_stride, int dst_width,
+ int dst_height,
+ const uint16_t *second_pred,
+ const DIST_WTD_COMP_PARAMS *jcp_param) {
+ // We only specialise on the filter values for large block sizes (>= 16x16.)
+ assert(dst_width >= 16 && dst_width % 16 == 0);
+ const uint16x8_t fwd_offset = vdupq_n_u16(jcp_param->fwd_offset);
+ const uint16x8_t bck_offset = vdupq_n_u16(jcp_param->bck_offset);
+
+ int i = dst_height;
+ do {
+ int j = 0;
+ do {
+ uint16x8_t s = vld1q_u16(src_ptr + j);
+ uint16x8_t p = vld1q_u16(second_pred);
+
+ uint16x8_t avg = dist_wtd_avg_u16x8(s, p, fwd_offset, bck_offset);
+
+ vst1q_u16(dst_ptr + j, avg);
+
+ second_pred += 8;
+ j += 8;
+ } while (j < dst_width);
+
+ src_ptr += src_stride;
+ dst_ptr += dst_width;
+ } while (--i != 0);
+}
+
+static void highbd_dist_wtd_avg_pred_var_filter_block2d_avg(
+ const uint16_t *src_ptr, uint16_t *dst_ptr, int src_stride, int pixel_step,
+ int dst_width, int dst_height, const uint16_t *second_pred,
+ const DIST_WTD_COMP_PARAMS *jcp_param) {
+ // We only specialise on the filter values for large block sizes (>= 16x16.)
+ assert(dst_width >= 16 && dst_width % 16 == 0);
+ const uint16x8_t fwd_offset = vdupq_n_u16(jcp_param->fwd_offset);
+ const uint16x8_t bck_offset = vdupq_n_u16(jcp_param->bck_offset);
+
+ int i = dst_height;
+ do {
+ int j = 0;
+ do {
+ uint16x8_t s0 = vld1q_u16(src_ptr + j);
+ uint16x8_t s1 = vld1q_u16(src_ptr + j + pixel_step);
+ uint16x8_t p = vld1q_u16(second_pred);
+ uint16x8_t avg = vrhaddq_u16(s0, s1);
+ avg = dist_wtd_avg_u16x8(avg, p, fwd_offset, bck_offset);
+
+ vst1q_u16(dst_ptr + j, avg);
+
+ second_pred += 8;
+ j += 8;
+ } while (j < dst_width);
+
+ src_ptr += src_stride;
+ dst_ptr += dst_width;
+ } while (--i != 0);
+}
+
+static void highbd_dist_wtd_avg_pred_var_filter_block2d_bil_w4(
+ const uint16_t *src_ptr, uint16_t *dst_ptr, int src_stride, int pixel_step,
+ int dst_height, int filter_offset, const uint16_t *second_pred,
+ const DIST_WTD_COMP_PARAMS *jcp_param) {
+ const uint16x4_t fwd_offset = vdup_n_u16(jcp_param->fwd_offset);
+ const uint16x4_t bck_offset = vdup_n_u16(jcp_param->bck_offset);
+ const uint16x4_t f0 = vdup_n_u16(8 - filter_offset);
+ const uint16x4_t f1 = vdup_n_u16(filter_offset);
+
+ int i = dst_height;
+ do {
+ uint16x4_t s0 = load_unaligned_u16_4x1(src_ptr);
+ uint16x4_t s1 = load_unaligned_u16_4x1(src_ptr + pixel_step);
+ uint16x4_t p = vld1_u16(second_pred);
+
+ uint16x4_t blend = vmul_u16(s0, f0);
+ blend = vmla_u16(blend, s1, f1);
+ blend = vrshr_n_u16(blend, 3);
+
+ uint16x4_t avg = dist_wtd_avg_u16x4(blend, p, fwd_offset, bck_offset);
+
+ vst1_u16(dst_ptr, avg);
+
+ src_ptr += src_stride;
+ dst_ptr += 4;
+ second_pred += 4;
+ } while (--i != 0);
+}
+
+// Combine bilinear filter with aom_dist_wtd_comp_avg_pred for large blocks.
+static void highbd_dist_wtd_avg_pred_var_filter_block2d_bil_large(
+ const uint16_t *src_ptr, uint16_t *dst_ptr, int src_stride, int pixel_step,
+ int dst_width, int dst_height, int filter_offset,
+ const uint16_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) {
+ const uint16x8_t fwd_offset = vdupq_n_u16(jcp_param->fwd_offset);
+ const uint16x8_t bck_offset = vdupq_n_u16(jcp_param->bck_offset);
+ const uint16x8_t f0 = vdupq_n_u16(8 - filter_offset);
+ const uint16x8_t f1 = vdupq_n_u16(filter_offset);
+
+ int i = dst_height;
+ do {
+ int j = 0;
+ do {
+ uint16x8_t s0 = vld1q_u16(src_ptr + j);
+ uint16x8_t s1 = vld1q_u16(src_ptr + j + pixel_step);
+ uint16x8_t p = vld1q_u16(second_pred);
+
+ uint16x8_t blend = vmulq_u16(s0, f0);
+ blend = vmlaq_u16(blend, s1, f1);
+ blend = vrshrq_n_u16(blend, 3);
+
+ uint16x8_t avg = dist_wtd_avg_u16x8(blend, p, fwd_offset, bck_offset);
+
+ vst1q_u16(dst_ptr + j, avg);
+
+ second_pred += 8;
+ j += 8;
+ } while (j < dst_width);
+
+ src_ptr += src_stride;
+ dst_ptr += dst_width;
+ } while (--i != 0);
+}
+
+static void highbd_dist_wtd_avg_pred_var_filter_block2d_bil_w8(
+ const uint16_t *src_ptr, uint16_t *dst_ptr, int src_stride, int pixel_step,
+ int dst_height, int filter_offset, const uint16_t *second_pred,
+ const DIST_WTD_COMP_PARAMS *jcp_param) {
+ highbd_dist_wtd_avg_pred_var_filter_block2d_bil_large(
+ src_ptr, dst_ptr, src_stride, pixel_step, 8, dst_height, filter_offset,
+ second_pred, jcp_param);
+}
+
+// Combine bilinear filter with aom_comp_avg_pred for blocks having width 16.
+static void highbd_dist_wtd_avg_pred_var_filter_block2d_bil_w16(
+ const uint16_t *src_ptr, uint16_t *dst_ptr, int src_stride, int pixel_step,
+ int dst_height, int filter_offset, const uint16_t *second_pred,
+ const DIST_WTD_COMP_PARAMS *jcp_param) {
+ highbd_dist_wtd_avg_pred_var_filter_block2d_bil_large(
+ src_ptr, dst_ptr, src_stride, pixel_step, 16, dst_height, filter_offset,
+ second_pred, jcp_param);
+}
+
+// Combine bilinear filter with aom_comp_avg_pred for blocks having width 32.
+static void highbd_dist_wtd_avg_pred_var_filter_block2d_bil_w32(
+ const uint16_t *src_ptr, uint16_t *dst_ptr, int src_stride, int pixel_step,
+ int dst_height, int filter_offset, const uint16_t *second_pred,
+ const DIST_WTD_COMP_PARAMS *jcp_param) {
+ highbd_dist_wtd_avg_pred_var_filter_block2d_bil_large(
+ src_ptr, dst_ptr, src_stride, pixel_step, 32, dst_height, filter_offset,
+ second_pred, jcp_param);
+}
+
+// Combine bilinear filter with aom_comp_avg_pred for blocks having width 64.
+static void highbd_dist_wtd_avg_pred_var_filter_block2d_bil_w64(
+ const uint16_t *src_ptr, uint16_t *dst_ptr, int src_stride, int pixel_step,
+ int dst_height, int filter_offset, const uint16_t *second_pred,
+ const DIST_WTD_COMP_PARAMS *jcp_param) {
+ highbd_dist_wtd_avg_pred_var_filter_block2d_bil_large(
+ src_ptr, dst_ptr, src_stride, pixel_step, 64, dst_height, filter_offset,
+ second_pred, jcp_param);
+}
+
+// Combine bilinear filter with aom_comp_avg_pred for blocks having width 128.
+static void highbd_dist_wtd_avg_pred_var_filter_block2d_bil_w128(
+ const uint16_t *src_ptr, uint16_t *dst_ptr, int src_stride, int pixel_step,
+ int dst_height, int filter_offset, const uint16_t *second_pred,
+ const DIST_WTD_COMP_PARAMS *jcp_param) {
+ highbd_dist_wtd_avg_pred_var_filter_block2d_bil_large(
+ src_ptr, dst_ptr, src_stride, pixel_step, 128, dst_height, filter_offset,
+ second_pred, jcp_param);
+}
+
+#define HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(bitdepth, w, h) \
+ unsigned int \
+ aom_highbd_##bitdepth##_dist_wtd_sub_pixel_avg_variance##w##x##h##_neon( \
+ const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, \
+ const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, \
+ const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \
+ uint16_t *src = CONVERT_TO_SHORTPTR(src_ptr); \
+ uint16_t *second = CONVERT_TO_SHORTPTR(second_pred); \
+ uint16_t tmp0[w * (h + 1)]; \
+ uint16_t tmp1[w * h]; \
+ highbd_var_filter_block2d_bil_w##w(src, tmp0, source_stride, 1, h + 1, \
+ xoffset); \
+ highbd_dist_wtd_avg_pred_var_filter_block2d_bil_w##w( \
+ tmp0, tmp1, w, w, h, yoffset, second, jcp_param); \
+ return aom_highbd_##bitdepth##_variance##w##x##h( \
+ CONVERT_TO_BYTEPTR(tmp1), w, ref_ptr, ref_stride, sse); \
+ }
+
+#define SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(bitdepth, w, h) \
+ unsigned int \
+ aom_highbd_##bitdepth##_dist_wtd_sub_pixel_avg_variance##w##x##h##_neon( \
+ const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, \
+ const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, \
+ const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \
+ uint16_t *src = CONVERT_TO_SHORTPTR(src_ptr); \
+ uint16_t *second = CONVERT_TO_SHORTPTR(second_pred); \
+ if (xoffset == 0) { \
+ uint16_t tmp[w * h]; \
+ if (yoffset == 0) { \
+ highbd_dist_wtd_avg_pred(src, tmp, source_stride, w, h, second, \
+ jcp_param); \
+ return aom_highbd_##bitdepth##_variance##w##x##h( \
+ CONVERT_TO_BYTEPTR(tmp), w, ref_ptr, ref_stride, sse); \
+ } else if (yoffset == 4) { \
+ highbd_dist_wtd_avg_pred_var_filter_block2d_avg( \
+ src, tmp, source_stride, source_stride, w, h, second, jcp_param); \
+ return aom_highbd_##bitdepth##_variance##w##x##h( \
+ CONVERT_TO_BYTEPTR(tmp), w, ref_ptr, ref_stride, sse); \
+ } else { \
+ highbd_dist_wtd_avg_pred_var_filter_block2d_bil_w##w( \
+ src, tmp, source_stride, source_stride, h, yoffset, second, \
+ jcp_param); \
+ return aom_highbd_##bitdepth##_variance##w##x##h( \
+ CONVERT_TO_BYTEPTR(tmp), w, ref_ptr, ref_stride, sse); \
+ } \
+ } else if (xoffset == 4) { \
+ uint16_t tmp0[w * (h + 1)]; \
+ if (yoffset == 0) { \
+ highbd_dist_wtd_avg_pred_var_filter_block2d_avg( \
+ src, tmp0, source_stride, 1, w, h, second, jcp_param); \
+ return aom_highbd_##bitdepth##_variance##w##x##h( \
+ CONVERT_TO_BYTEPTR(tmp0), w, ref_ptr, ref_stride, sse); \
+ } else if (yoffset == 4) { \
+ uint16_t tmp1[w * (h + 1)]; \
+ highbd_var_filter_block2d_avg(src, tmp0, source_stride, 1, w, h + 1); \
+ highbd_dist_wtd_avg_pred_var_filter_block2d_avg(tmp0, tmp1, w, w, w, \
+ h, second, jcp_param); \
+ return aom_highbd_##bitdepth##_variance##w##x##h( \
+ CONVERT_TO_BYTEPTR(tmp1), w, ref_ptr, ref_stride, sse); \
+ } else { \
+ uint16_t tmp1[w * (h + 1)]; \
+ highbd_var_filter_block2d_avg(src, tmp0, source_stride, 1, w, h + 1); \
+ highbd_dist_wtd_avg_pred_var_filter_block2d_bil_w##w( \
+ tmp0, tmp1, w, w, h, yoffset, second, jcp_param); \
+ return aom_highbd_##bitdepth##_variance##w##x##h( \
+ CONVERT_TO_BYTEPTR(tmp1), w, ref_ptr, ref_stride, sse); \
+ } \
+ } else { \
+ uint16_t tmp0[w * (h + 1)]; \
+ if (yoffset == 0) { \
+ highbd_dist_wtd_avg_pred_var_filter_block2d_bil_w##w( \
+ src, tmp0, source_stride, 1, h, xoffset, second, jcp_param); \
+ return aom_highbd_##bitdepth##_variance##w##x##h( \
+ CONVERT_TO_BYTEPTR(tmp0), w, ref_ptr, ref_stride, sse); \
+ } else if (yoffset == 4) { \
+ uint16_t tmp1[w * h]; \
+ highbd_var_filter_block2d_bil_w##w(src, tmp0, source_stride, 1, h + 1, \
+ xoffset); \
+ highbd_dist_wtd_avg_pred_var_filter_block2d_avg(tmp0, tmp1, w, w, w, \
+ h, second, jcp_param); \
+ return aom_highbd_##bitdepth##_variance##w##x##h( \
+ CONVERT_TO_BYTEPTR(tmp1), w, ref_ptr, ref_stride, sse); \
+ } else { \
+ uint16_t tmp1[w * h]; \
+ highbd_var_filter_block2d_bil_w##w(src, tmp0, source_stride, 1, h + 1, \
+ xoffset); \
+ highbd_dist_wtd_avg_pred_var_filter_block2d_bil_w##w( \
+ tmp0, tmp1, w, w, h, yoffset, second, jcp_param); \
+ return aom_highbd_##bitdepth##_variance##w##x##h( \
+ CONVERT_TO_BYTEPTR(tmp1), w, ref_ptr, ref_stride, sse); \
+ } \
+ } \
+ }
+
+// 8-bit
+HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 4, 4)
+HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 4, 8)
+
+HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 8, 4)
+HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 8, 8)
+HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 8, 16)
+
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 16, 8)
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 16, 16)
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 16, 32)
+
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 32, 16)
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 32, 32)
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 32, 64)
+
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 64, 32)
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 64, 64)
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 64, 128)
+
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 128, 64)
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 128, 128)
+
+#if !CONFIG_REALTIME_ONLY
+HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 4, 16)
+
+HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 8, 32)
+
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 16, 4)
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 16, 64)
+
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 32, 8)
+
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 64, 16)
+#endif // !CONFIG_REALTIME_ONLY
+
+// 10-bit
+HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 4, 4)
+HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 4, 8)
+
+HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 8, 4)
+HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 8, 8)
+HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 8, 16)
+
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 16, 8)
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 16, 16)
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 16, 32)
+
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 32, 16)
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 32, 32)
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 32, 64)
+
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 64, 32)
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 64, 64)
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 64, 128)
+
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 128, 64)
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 128, 128)
+
+#if !CONFIG_REALTIME_ONLY
+HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 4, 16)
+
+HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 8, 32)
+
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 16, 4)
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 16, 64)
+
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 32, 8)
+
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(10, 64, 16)
+#endif // !CONFIG_REALTIME_ONLY
+
+// 12-bit
+HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 4, 4)
+HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 4, 8)
+
+HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 8, 4)
+HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 8, 8)
+HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 8, 16)
+
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 16, 8)
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 16, 16)
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 16, 32)
+
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 32, 16)
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 32, 32)
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 32, 64)
+
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 64, 32)
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 64, 64)
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 64, 128)
+
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 128, 64)
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 128, 128)
+
+#if !CONFIG_REALTIME_ONLY
+HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 4, 16)
+
+HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 8, 32)
+
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 16, 4)
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 16, 64)
+
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 32, 8)
+
+SPECIALIZED_HBD_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(12, 64, 16)
+#endif // !CONFIG_REALTIME_ONLY
diff --git a/aom_dsp/arm/highbd_variance_neon.c b/aom_dsp/arm/highbd_variance_neon.c
index 948f2f7..e54fc18 100644
--- a/aom_dsp/arm/highbd_variance_neon.c
+++ b/aom_dsp/arm/highbd_variance_neon.c
@@ -15,10 +15,10 @@
#include "config/aom_config.h"
#include "config/aom_dsp_rtcd.h"
-#include "aom_dsp/variance.h"
#include "aom_dsp/aom_filter.h"
#include "aom_dsp/arm/mem_neon.h"
#include "aom_dsp/arm/sum_neon.h"
+#include "aom_dsp/variance.h"
// Process a block of width 4 two rows at a time.
static INLINE void highbd_variance_4xh_neon(const uint16_t *src_ptr,
@@ -412,67 +412,6 @@
return *sse;
}
-#if defined(__ARM_FEATURE_DOTPROD)
-
-static INLINE uint32_t highbd_mse8_8xh_neon(const uint16_t *src_ptr,
- int src_stride,
- const uint16_t *ref_ptr,
- int ref_stride, int h,
- unsigned int *sse) {
- uint32x4_t sse_u32 = vdupq_n_u32(0);
-
- int i = h / 2;
- do {
- uint16x8_t s0 = vld1q_u16(src_ptr);
- src_ptr += src_stride;
- uint16x8_t s1 = vld1q_u16(src_ptr);
- src_ptr += src_stride;
- uint16x8_t r0 = vld1q_u16(ref_ptr);
- ref_ptr += ref_stride;
- uint16x8_t r1 = vld1q_u16(ref_ptr);
- ref_ptr += ref_stride;
-
- uint8x16_t s = vcombine_u8(vmovn_u16(s0), vmovn_u16(s1));
- uint8x16_t r = vcombine_u8(vmovn_u16(r0), vmovn_u16(r1));
-
- uint8x16_t diff = vabdq_u8(s, r);
- sse_u32 = vdotq_u32(sse_u32, diff, diff);
- } while (--i != 0);
-
- *sse = horizontal_add_u32x4(sse_u32);
- return *sse;
-}
-
-static INLINE uint32_t highbd_mse8_16xh_neon(const uint16_t *src_ptr,
- int src_stride,
- const uint16_t *ref_ptr,
- int ref_stride, int h,
- unsigned int *sse) {
- uint32x4_t sse_u32 = vdupq_n_u32(0);
-
- int i = h;
- do {
- uint16x8_t s0 = vld1q_u16(src_ptr);
- uint16x8_t s1 = vld1q_u16(src_ptr + 8);
- uint16x8_t r0 = vld1q_u16(ref_ptr);
- uint16x8_t r1 = vld1q_u16(ref_ptr + 8);
-
- uint8x16_t s = vcombine_u8(vmovn_u16(s0), vmovn_u16(s1));
- uint8x16_t r = vcombine_u8(vmovn_u16(r0), vmovn_u16(r1));
-
- uint8x16_t diff = vabdq_u8(s, r);
- sse_u32 = vdotq_u32(sse_u32, diff, diff);
-
- src_ptr += src_stride;
- ref_ptr += ref_stride;
- } while (--i != 0);
-
- *sse = horizontal_add_u32x4(sse_u32);
- return *sse;
-}
-
-#else // !defined(__ARM_FEATURE_DOTPROD)
-
static INLINE uint32_t highbd_mse8_8xh_neon(const uint16_t *src_ptr,
int src_stride,
const uint16_t *ref_ptr,
@@ -491,8 +430,6 @@
sse);
}
-#endif // defined(__ARM_FEATURE_DOTPROD)
-
#define HIGHBD_MSE_WXH_NEON(w, h) \
uint32_t aom_highbd_8_mse##w##x##h##_neon( \
const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \
@@ -529,3 +466,55 @@
HIGHBD_MSE_WXH_NEON(8, 8)
#undef HIGHBD_MSE_WXH_NEON
+
+static INLINE uint64x2_t mse_accumulate_u16_8x2(uint64x2_t sum, uint16x8_t s0,
+ uint16x8_t s1, uint16x8_t d0,
+ uint16x8_t d1) {
+ uint16x8_t e0 = vabdq_u16(s0, d0);
+ uint16x8_t e1 = vabdq_u16(s1, d1);
+
+ uint32x4_t mse = vmull_u16(vget_low_u16(e0), vget_low_u16(e0));
+ mse = vmlal_u16(mse, vget_high_u16(e0), vget_high_u16(e0));
+ mse = vmlal_u16(mse, vget_low_u16(e1), vget_low_u16(e1));
+ mse = vmlal_u16(mse, vget_high_u16(e1), vget_high_u16(e1));
+
+ return vpadalq_u32(sum, mse);
+}
+
+uint64_t aom_mse_wxh_16bit_highbd_neon(uint16_t *dst, int dstride,
+ uint16_t *src, int sstride, int w,
+ int h) {
+ assert((w == 8 || w == 4) && (h == 8 || h == 4));
+
+ uint64x2_t sum = vdupq_n_u64(0);
+
+ if (w == 8) {
+ do {
+ uint16x8_t d0 = vld1q_u16(dst + 0 * dstride);
+ uint16x8_t d1 = vld1q_u16(dst + 1 * dstride);
+ uint16x8_t s0 = vld1q_u16(src + 0 * sstride);
+ uint16x8_t s1 = vld1q_u16(src + 1 * sstride);
+
+ sum = mse_accumulate_u16_8x2(sum, s0, s1, d0, d1);
+
+ dst += 2 * dstride;
+ src += 2 * sstride;
+ h -= 2;
+ } while (h != 0);
+ } else { // w == 4
+ do {
+ uint16x8_t d0 = load_unaligned_u16_4x2(dst + 0 * dstride, dstride);
+ uint16x8_t d1 = load_unaligned_u16_4x2(dst + 2 * dstride, dstride);
+ uint16x8_t s0 = load_unaligned_u16_4x2(src + 0 * sstride, sstride);
+ uint16x8_t s1 = load_unaligned_u16_4x2(src + 2 * sstride, sstride);
+
+ sum = mse_accumulate_u16_8x2(sum, s0, s1, d0, d1);
+
+ dst += 4 * dstride;
+ src += 4 * sstride;
+ h -= 4;
+ } while (h != 0);
+ }
+
+ return horizontal_add_u64x2(sum);
+}
diff --git a/aom_dsp/arm/highbd_variance_neon_dotprod.c b/aom_dsp/arm/highbd_variance_neon_dotprod.c
new file mode 100644
index 0000000..d56ae97
--- /dev/null
+++ b/aom_dsp/arm/highbd_variance_neon_dotprod.c
@@ -0,0 +1,92 @@
+/*
+ * Copyright (c) 2023 The WebM project authors. All Rights Reserved.
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+
+#include "aom_dsp/arm/sum_neon.h"
+#include "config/aom_config.h"
+#include "config/aom_dsp_rtcd.h"
+
+static INLINE uint32_t highbd_mse8_8xh_neon_dotprod(const uint16_t *src_ptr,
+ int src_stride,
+ const uint16_t *ref_ptr,
+ int ref_stride, int h,
+ unsigned int *sse) {
+ uint32x4_t sse_u32 = vdupq_n_u32(0);
+
+ int i = h / 2;
+ do {
+ uint16x8_t s0 = vld1q_u16(src_ptr);
+ src_ptr += src_stride;
+ uint16x8_t s1 = vld1q_u16(src_ptr);
+ src_ptr += src_stride;
+ uint16x8_t r0 = vld1q_u16(ref_ptr);
+ ref_ptr += ref_stride;
+ uint16x8_t r1 = vld1q_u16(ref_ptr);
+ ref_ptr += ref_stride;
+
+ uint8x16_t s = vcombine_u8(vmovn_u16(s0), vmovn_u16(s1));
+ uint8x16_t r = vcombine_u8(vmovn_u16(r0), vmovn_u16(r1));
+
+ uint8x16_t diff = vabdq_u8(s, r);
+ sse_u32 = vdotq_u32(sse_u32, diff, diff);
+ } while (--i != 0);
+
+ *sse = horizontal_add_u32x4(sse_u32);
+ return *sse;
+}
+
+static INLINE uint32_t highbd_mse8_16xh_neon_dotprod(const uint16_t *src_ptr,
+ int src_stride,
+ const uint16_t *ref_ptr,
+ int ref_stride, int h,
+ unsigned int *sse) {
+ uint32x4_t sse_u32 = vdupq_n_u32(0);
+
+ int i = h;
+ do {
+ uint16x8_t s0 = vld1q_u16(src_ptr);
+ uint16x8_t s1 = vld1q_u16(src_ptr + 8);
+ uint16x8_t r0 = vld1q_u16(ref_ptr);
+ uint16x8_t r1 = vld1q_u16(ref_ptr + 8);
+
+ uint8x16_t s = vcombine_u8(vmovn_u16(s0), vmovn_u16(s1));
+ uint8x16_t r = vcombine_u8(vmovn_u16(r0), vmovn_u16(r1));
+
+ uint8x16_t diff = vabdq_u8(s, r);
+ sse_u32 = vdotq_u32(sse_u32, diff, diff);
+
+ src_ptr += src_stride;
+ ref_ptr += ref_stride;
+ } while (--i != 0);
+
+ *sse = horizontal_add_u32x4(sse_u32);
+ return *sse;
+}
+
+#define HIGHBD_MSE_WXH_NEON_DOTPROD(w, h) \
+ uint32_t aom_highbd_8_mse##w##x##h##_neon_dotprod( \
+ const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \
+ int ref_stride, uint32_t *sse) { \
+ uint16_t *src = CONVERT_TO_SHORTPTR(src_ptr); \
+ uint16_t *ref = CONVERT_TO_SHORTPTR(ref_ptr); \
+ highbd_mse8_##w##xh_neon_dotprod(src, src_stride, ref, ref_stride, h, \
+ sse); \
+ return *sse; \
+ }
+
+HIGHBD_MSE_WXH_NEON_DOTPROD(16, 16)
+HIGHBD_MSE_WXH_NEON_DOTPROD(16, 8)
+HIGHBD_MSE_WXH_NEON_DOTPROD(8, 16)
+HIGHBD_MSE_WXH_NEON_DOTPROD(8, 8)
+
+#undef HIGHBD_MSE_WXH_NEON_DOTPROD
diff --git a/aom_dsp/arm/intrapred_neon.c b/aom_dsp/arm/intrapred_neon.c
index 2161378..41f070e 100644
--- a/aom_dsp/arm/intrapred_neon.c
+++ b/aom_dsp/arm/intrapred_neon.c
@@ -24,7 +24,7 @@
// DC 4x4
static INLINE uint16x8_t dc_load_sum_4(const uint8_t *in) {
- const uint8x8_t a = load_u8_4x1_lane0(in);
+ const uint8x8_t a = load_u8_4x1(in);
const uint16x4_t p0 = vpaddl_u8(a);
const uint16x4_t p1 = vpadd_u16(p0, p0);
return vcombine_u16(p1, vdup_n_u16(0));
@@ -354,7 +354,7 @@
void aom_dc_predictor_4x8_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
- uint8x8_t a = load_u8_4x1_lane0(above);
+ uint8x8_t a = load_u8_4x1(above);
uint8x8_t l = vld1_u8(left);
uint32_t sum = horizontal_add_u16x8(vaddl_u8(a, l));
uint32_t dc = calculate_dc_from_sum(4, 8, sum, 2, DC_MULTIPLIER_1X2);
@@ -364,7 +364,7 @@
void aom_dc_predictor_8x4_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
uint8x8_t a = vld1_u8(above);
- uint8x8_t l = load_u8_4x1_lane0(left);
+ uint8x8_t l = load_u8_4x1(left);
uint32_t sum = horizontal_add_u16x8(vaddl_u8(a, l));
uint32_t dc = calculate_dc_from_sum(8, 4, sum, 2, DC_MULTIPLIER_1X2);
dc_store_8xh(dst, stride, 4, vdup_n_u8(dc));
@@ -372,7 +372,7 @@
void aom_dc_predictor_4x16_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
- uint8x8_t a = load_u8_4x1_lane0(above);
+ uint8x8_t a = load_u8_4x1(above);
uint8x16_t l = vld1q_u8(left);
uint16x8_t sum_al = vaddw_u8(vpaddlq_u8(l), a);
uint32_t sum = horizontal_add_u16x8(sum_al);
@@ -383,7 +383,7 @@
void aom_dc_predictor_16x4_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
uint8x16_t a = vld1q_u8(above);
- uint8x8_t l = load_u8_4x1_lane0(left);
+ uint8x8_t l = load_u8_4x1(left);
uint16x8_t sum_al = vaddw_u8(vpaddlq_u8(a), l);
uint32_t sum = horizontal_add_u16x8(sum_al);
uint32_t dc = calculate_dc_from_sum(16, 4, sum, 2, DC_MULTIPLIER_1X4);
@@ -620,7 +620,7 @@
void aom_v_predictor_4x4_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
(void)left;
- v_store_4xh(dst, stride, 4, load_u8_4x1_lane0(above));
+ v_store_4xh(dst, stride, 4, load_u8_4x1(above));
}
void aom_v_predictor_8x8_neon(uint8_t *dst, ptrdiff_t stride,
@@ -646,13 +646,13 @@
void aom_v_predictor_4x8_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
(void)left;
- v_store_4xh(dst, stride, 8, load_u8_4x1_lane0(above));
+ v_store_4xh(dst, stride, 8, load_u8_4x1(above));
}
void aom_v_predictor_4x16_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
(void)left;
- v_store_4xh(dst, stride, 16, load_u8_4x1_lane0(above));
+ v_store_4xh(dst, stride, 16, load_u8_4x1(above));
}
void aom_v_predictor_8x4_neon(uint8_t *dst, ptrdiff_t stride,
@@ -856,7 +856,7 @@
void aom_h_predictor_4x4_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
- const uint8x8_t d0 = load_u8_4x1_lane0(left);
+ const uint8x8_t d0 = load_u8_4x1(left);
(void)above;
store_u8_4x1(dst + 0 * stride, vdup_lane_u8(d0, 0), 0);
store_u8_4x1(dst + 1 * stride, vdup_lane_u8(d0, 1), 0);
@@ -907,7 +907,7 @@
void aom_h_predictor_8x4_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
- const uint8x8_t d0 = load_u8_4x1_lane0(left);
+ const uint8x8_t d0 = load_u8_4x1(left);
(void)above;
vst1_u8(dst + 0 * stride, vdup_lane_u8(d0, 0));
vst1_u8(dst + 1 * stride, vdup_lane_u8(d0, 1));
@@ -936,7 +936,7 @@
void aom_h_predictor_16x4_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
- const uint8x8_t d0 = load_u8_4x1_lane0(left);
+ const uint8x8_t d0 = load_u8_4x1(left);
(void)above;
vst1q_u8(dst + 0 * stride, vdupq_lane_u8(d0, 0));
vst1q_u8(dst + 1 * stride, vdupq_lane_u8(d0, 1));
@@ -1594,8 +1594,10 @@
base_y_c64 = vbic_s16(base_y_c64, vreinterpret_s16_u16(mask64));
#if AOM_ARCH_AARCH64
- uint8x8_t left_idx0 = vreinterpret_u8_s16(base_y_c64 + 2); // [0, 16]
- uint8x8_t left_idx1 = vreinterpret_u8_s16(base_y_c64 + 3); // [1, 17]
+ uint8x8_t left_idx0 =
+ vreinterpret_u8_s16(vadd_s16(base_y_c64, vdup_n_s16(2))); // [0, 16]
+ uint8x8_t left_idx1 =
+ vreinterpret_u8_s16(vadd_s16(base_y_c64, vdup_n_s16(3))); // [1, 17]
uint8x8_t a0_y = vtrn1_u8(vqtbl2_u8(left_vals, left_idx0), v_zero_u8);
uint8x8_t a1_y = vtrn1_u8(vqtbl2_u8(left_vals, left_idx1), v_zero_u8);
@@ -1777,8 +1779,10 @@
base_y_c128 = vbicq_s16(base_y_c128, vreinterpretq_s16_u16(mask128));
#if AOM_ARCH_AARCH64
- uint8x16_t left_idx0 = vreinterpretq_u8_s16(base_y_c128 + 2); // [0, 33]
- uint8x16_t left_idx1 = vreinterpretq_u8_s16(base_y_c128 + 3); // [1, 34]
+ uint8x16_t left_idx0 = vreinterpretq_u8_s16(
+ vaddq_s16(base_y_c128, vdupq_n_s16(2))); // [0, 33]
+ uint8x16_t left_idx1 = vreinterpretq_u8_s16(
+ vaddq_s16(base_y_c128, vdupq_n_s16(3))); // [1, 34]
uint8x16_t left_idx01 = vuzp1q_u8(left_idx0, left_idx1);
uint8x16_t a01_x = vqtbl3q_u8(left_vals, left_idx01);
@@ -2025,8 +2029,10 @@
#if AOM_ARCH_AARCH64
// Values in left_idx{0,1} range from 0 through 63 inclusive.
- uint8x16_t left_idx0 = vreinterpretq_u8_s16(base_y_c256.val[0] + 1);
- uint8x16_t left_idx1 = vreinterpretq_u8_s16(base_y_c256.val[1] + 1);
+ uint8x16_t left_idx0 = vreinterpretq_u8_s16(
+ vaddq_s16(base_y_c256.val[0], vdupq_n_s16(1)));
+ uint8x16_t left_idx1 = vreinterpretq_u8_s16(
+ vaddq_s16(base_y_c256.val[1], vdupq_n_s16(1)));
uint8x16_t left_idx01 = vuzp1q_u8(left_idx0, left_idx1);
@@ -3168,12 +3174,10 @@
const uint8_t bottom_left = left_column[height - 1];
const uint8_t *const weights_y = smooth_weights + height - 4;
- uint8x8_t UNINITIALIZED_IS_SAFE(top_v);
- load_u8_4x1(top_row, &top_v, 0);
+ uint8x8_t top_v = load_u8_4x1(top_row);
const uint8x8_t top_right_v = vdup_n_u8(top_right);
const uint8x8_t bottom_left_v = vdup_n_u8(bottom_left);
- uint8x8_t UNINITIALIZED_IS_SAFE(weights_x_v);
- load_u8_4x1(smooth_weights, &weights_x_v, 0);
+ uint8x8_t weights_x_v = load_u8_4x1(smooth_weights);
const uint8x8_t scaled_weights_x = negate_s8(weights_x_v);
const uint16x8_t weighted_tr = vmull_u8(scaled_weights_x, top_right_v);
@@ -3403,9 +3407,9 @@
const uint8_t bottom_left = left_column[height - 1]; \
const uint8_t *const weights_y = smooth_weights + height - 4; \
\
- uint8x8_t UNINITIALIZED_IS_SAFE(top_v); \
+ uint8x8_t top_v; \
if ((W) == 4) { \
- load_u8_4x1(top_row, &top_v, 0); \
+ top_v = load_u8_4x1(top_row); \
} else { /* width == 8 */ \
top_v = vld1_u8(top_row); \
} \
@@ -3717,9 +3721,9 @@
int width, int height) {
const uint8x8_t top_left = vdup_n_u8(top_row[-1]);
const uint16x8_t top_left_x2 = vdupq_n_u16(top_row[-1] + top_row[-1]);
- uint8x8_t UNINITIALIZED_IS_SAFE(top);
+ uint8x8_t top;
if (width == 4) {
- load_u8_4x1(top_row, &top, 0);
+ top = load_u8_4x1(top_row);
} else { // width == 8
top = vld1_u8(top_row);
}
diff --git a/aom_dsp/arm/loopfilter_neon.c b/aom_dsp/arm/loopfilter_neon.c
index 8fc7ccb..0e683a7 100644
--- a/aom_dsp/arm/loopfilter_neon.c
+++ b/aom_dsp/arm/loopfilter_neon.c
@@ -634,13 +634,13 @@
p6p2 = vget_low_u8(row1);
p5p1 = vget_low_u8(row2);
p4p0 = vget_low_u8(row3);
- transpose_u8_8x4(&pxp3, &p6p2, &p5p1, &p4p0);
+ transpose_elems_inplace_u8_8x4(&pxp3, &p6p2, &p5p1, &p4p0);
q0q4 = vget_high_u8(row0);
q1q5 = vget_high_u8(row1);
q2q6 = vget_high_u8(row2);
q3qy = vget_high_u8(row3);
- transpose_u8_8x4(&q0q4, &q1q5, &q2q6, &q3qy);
+ transpose_elems_inplace_u8_8x4(&q0q4, &q1q5, &q2q6, &q3qy);
pq_rev = vrev64_u32(vreinterpret_u32_u8(q3qy));
pxqx_p3q3 = vtrn_u32(vreinterpret_u32_u8(pxp3), pq_rev);
@@ -679,13 +679,13 @@
q1q5 = vreinterpret_u8_u32(p5q5_p1q1.val[1]);
q2q6 = vreinterpret_u8_u32(p6q6_p2q2.val[1]);
q3qy = vreinterpret_u8_u32(pxqx_p3q3.val[1]);
- transpose_u8_8x4(&q0q4, &q1q5, &q2q6, &q3qy);
+ transpose_elems_inplace_u8_8x4(&q0q4, &q1q5, &q2q6, &q3qy);
pxp3 = vreinterpret_u8_u32(pxqx_p3q3.val[0]);
p6p2 = vreinterpret_u8_u32(p6q6_p2q2.val[0]);
p5p1 = vreinterpret_u8_u32(p5q5_p1q1.val[0]);
p4p0 = vreinterpret_u8_u32(p4q4_p0q0.val[0]);
- transpose_u8_8x4(&pxp3, &p6p2, &p5p1, &p4p0);
+ transpose_elems_inplace_u8_8x4(&pxp3, &p6p2, &p5p1, &p4p0);
row0 = vcombine_u8(pxp3, q0q4);
row1 = vcombine_u8(p6p2, q1q5);
@@ -725,7 +725,7 @@
// row3: p3 p2 p1 p0 | q0 q1 q2 q3
load_u8_8x4(src - 4, stride, &p3q0, &p2q1, &p1q2, &p0q3);
- transpose_u8_8x4(&p3q0, &p2q1, &p1q2, &p0q3);
+ transpose_elems_inplace_u8_8x4(&p3q0, &p2q1, &p1q2, &p0q3);
pq_rev = vrev64_u32(vreinterpret_u32_u8(p0q3));
p3q3_p0q0 = vtrn_u32(vreinterpret_u32_u8(p3q0), pq_rev);
@@ -750,7 +750,7 @@
p1q2 = vreinterpret_u8_u32(vrev64_u32(p2q2_p1q1.val[1]));
p2q1 = vreinterpret_u8_u32(p2q2_p1q1.val[0]);
p3q0 = vreinterpret_u8_u32(p3q3_p0q0.val[0]);
- transpose_u8_8x4(&p3q0, &p2q1, &p1q2, &p0q3);
+ transpose_elems_inplace_u8_8x4(&p3q0, &p2q1, &p1q2, &p0q3);
store_u8_8x4(src - 4, stride, p3q0, p2q1, p1q2, p0q3);
}
@@ -784,7 +784,7 @@
// row3: px p2 p1 p0 | q0 q1 q2 qy
load_u8_8x4(src - 4, stride, &pxq0, &p2q1, &p1q2, &p0qy);
- transpose_u8_8x4(&pxq0, &p2q1, &p1q2, &p0qy);
+ transpose_elems_inplace_u8_8x4(&pxq0, &p2q1, &p1q2, &p0qy);
pq_rev = vrev64_u32(vreinterpret_u32_u8(p0qy));
pxqy_p0q0 = vtrn_u32(vreinterpret_u32_u8(pxq0), pq_rev);
@@ -809,7 +809,7 @@
p1q2 = vreinterpret_u8_u32(vrev64_u32(p2q2_p1q1.val[1]));
p2q1 = vreinterpret_u8_u32(p2q2_p1q1.val[0]);
pxq0 = vreinterpret_u8_u32(pxqy_p0q0.val[0]);
- transpose_u8_8x4(&pxq0, &p2q1, &p1q2, &p0qy);
+ transpose_elems_inplace_u8_8x4(&pxq0, &p2q1, &p1q2, &p0qy);
store_u8_8x4(src - 4, stride, pxq0, p2q1, p1q2, p0qy);
}
@@ -834,7 +834,7 @@
const uint8_t *limit, const uint8_t *thresh) {
uint32x2x2_t p1q0_p0q1, p1q1_p0q0, p1p0_q1q0;
uint32x2_t pq_rev;
- uint8x8_t UNINITIALIZED_IS_SAFE(p1p0), UNINITIALIZED_IS_SAFE(q0q1);
+ uint8x8_t p1p0, q0q1;
uint8x8_t p0q0, p1q1;
// row0: p1 p0 | q0 q1
@@ -843,7 +843,7 @@
// row3: p1 p0 | q0 q1
load_unaligned_u8_4x4(src - 2, stride, &p1p0, &q0q1);
- transpose_u8_4x4(&p1p0, &q0q1);
+ transpose_elems_inplace_u8_4x4(&p1p0, &q0q1);
p1q0_p0q1 = vtrn_u32(vreinterpret_u32_u8(p1p0), vreinterpret_u32_u8(q0q1));
@@ -860,7 +860,7 @@
p1p0 = vreinterpret_u8_u32(p1p0_q1q0.val[0]);
q0q1 = vreinterpret_u8_u32(vrev64_u32(p1p0_q1q0.val[1]));
- transpose_u8_4x4(&p1p0, &q0q1);
+ transpose_elems_inplace_u8_4x4(&p1p0, &q0q1);
store_unaligned_u8_4x1(src - 2, p1p0, 0);
store_unaligned_u8_4x1((src - 2) + 1 * stride, q0q1, 0);
@@ -886,25 +886,13 @@
void aom_lpf_horizontal_14_neon(uint8_t *src, int stride, const uint8_t *blimit,
const uint8_t *limit, const uint8_t *thresh) {
- uint8x8_t UNINITIALIZED_IS_SAFE(p0q0), UNINITIALIZED_IS_SAFE(p1q1),
- UNINITIALIZED_IS_SAFE(p2q2), UNINITIALIZED_IS_SAFE(p3q3),
- UNINITIALIZED_IS_SAFE(p4q4), UNINITIALIZED_IS_SAFE(p5q5),
- UNINITIALIZED_IS_SAFE(p6q6);
-
- load_u8_4x1(src - 7 * stride, &p6q6, 0);
- load_u8_4x1(src - 6 * stride, &p5q5, 0);
- load_u8_4x1(src - 5 * stride, &p4q4, 0);
- load_u8_4x1(src - 4 * stride, &p3q3, 0);
- load_u8_4x1(src - 3 * stride, &p2q2, 0);
- load_u8_4x1(src - 2 * stride, &p1q1, 0);
- load_u8_4x1(src - 1 * stride, &p0q0, 0);
- load_u8_4x1(src + 0 * stride, &p0q0, 1);
- load_u8_4x1(src + 1 * stride, &p1q1, 1);
- load_u8_4x1(src + 2 * stride, &p2q2, 1);
- load_u8_4x1(src + 3 * stride, &p3q3, 1);
- load_u8_4x1(src + 4 * stride, &p4q4, 1);
- load_u8_4x1(src + 5 * stride, &p5q5, 1);
- load_u8_4x1(src + 6 * stride, &p6q6, 1);
+ uint8x8_t p6q6 = load_u8_4x2(src - 7 * stride, 13 * stride);
+ uint8x8_t p5q5 = load_u8_4x2(src - 6 * stride, 11 * stride);
+ uint8x8_t p4q4 = load_u8_4x2(src - 5 * stride, 9 * stride);
+ uint8x8_t p3q3 = load_u8_4x2(src - 4 * stride, 7 * stride);
+ uint8x8_t p2q2 = load_u8_4x2(src - 3 * stride, 5 * stride);
+ uint8x8_t p1q1 = load_u8_4x2(src - 2 * stride, 3 * stride);
+ uint8x8_t p0q0 = load_u8_4x2(src - 1 * stride, 1 * stride);
lpf_14_neon(&p6q6, &p5q5, &p4q4, &p3q3, &p2q2, &p1q1, &p0q0, *blimit, *limit,
*thresh);
@@ -1036,12 +1024,8 @@
void aom_lpf_horizontal_4_neon(uint8_t *src, int stride, const uint8_t *blimit,
const uint8_t *limit, const uint8_t *thresh) {
- uint8x8_t UNINITIALIZED_IS_SAFE(p0q0), UNINITIALIZED_IS_SAFE(p1q1);
-
- load_u8_4x1(src - 2 * stride, &p1q1, 0);
- load_u8_4x1(src - 1 * stride, &p0q0, 0);
- load_u8_4x1(src + 0 * stride, &p0q0, 1);
- load_u8_4x1(src + 1 * stride, &p1q1, 1);
+ uint8x8_t p1q1 = load_u8_4x2(src - 2 * stride, 3 * stride);
+ uint8x8_t p0q0 = load_u8_4x2(src - 1 * stride, 1 * stride);
lpf_4_neon(&p1q1, &p0q0, *blimit, *limit, *thresh);
diff --git a/aom_dsp/arm/masked_sad4d_neon.c b/aom_dsp/arm/masked_sad4d_neon.c
index 98daeda..8f65b80 100644
--- a/aom_dsp/arm/masked_sad4d_neon.c
+++ b/aom_dsp/arm/masked_sad4d_neon.c
@@ -516,19 +516,18 @@
vst1q_u32(res, horizontal_add_4d_u16x8(sum));
}
-#define MASKED_SAD4D_WXH_NEON(w, h) \
- void aom_masked_sad##w##x##h##x4d_neon( \
- const uint8_t *src, int src_stride, const uint8_t *ref[4], \
- int ref_stride, const uint8_t *second_pred, const uint8_t *msk, \
- int msk_stride, int invert_mask, uint32_t res[4]) { \
- if (invert_mask) { \
- return masked_inv_sad##w##xhx4d_neon(src, src_stride, ref, ref_stride, \
- second_pred, msk, msk_stride, res, \
- h); \
- } else { \
- return masked_sad##w##xhx4d_neon(src, src_stride, ref, ref_stride, \
- second_pred, msk, msk_stride, res, h); \
- } \
+#define MASKED_SAD4D_WXH_NEON(w, h) \
+ void aom_masked_sad##w##x##h##x4d_neon( \
+ const uint8_t *src, int src_stride, const uint8_t *ref[4], \
+ int ref_stride, const uint8_t *second_pred, const uint8_t *msk, \
+ int msk_stride, int invert_mask, uint32_t res[4]) { \
+ if (invert_mask) { \
+ masked_inv_sad##w##xhx4d_neon(src, src_stride, ref, ref_stride, \
+ second_pred, msk, msk_stride, res, h); \
+ } else { \
+ masked_sad##w##xhx4d_neon(src, src_stride, ref, ref_stride, second_pred, \
+ msk, msk_stride, res, h); \
+ } \
}
MASKED_SAD4D_WXH_NEON(4, 8)
diff --git a/aom_dsp/arm/masked_sad_neon.c b/aom_dsp/arm/masked_sad_neon.c
index 340df05..9d26310 100644
--- a/aom_dsp/arm/masked_sad_neon.c
+++ b/aom_dsp/arm/masked_sad_neon.c
@@ -15,9 +15,10 @@
#include "config/aom_dsp_rtcd.h"
#include "aom/aom_integer.h"
+#include "aom_dsp/arm/blend_neon.h"
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_dsp/arm/sum_neon.h"
#include "aom_dsp/blend.h"
-#include "mem_neon.h"
-#include "sum_neon.h"
static INLINE uint16x8_t masked_sad_16x1_neon(uint16x8_t sad,
const uint8_t *src,
@@ -29,15 +30,7 @@
uint8x16_t b0 = vld1q_u8(b);
uint8x16_t s0 = vld1q_u8(src);
- uint8x16_t m0_inv = vsubq_u8(vdupq_n_u8(AOM_BLEND_A64_MAX_ALPHA), m0);
- uint16x8_t blend_u16_lo = vmull_u8(vget_low_u8(m0), vget_low_u8(a0));
- uint16x8_t blend_u16_hi = vmull_u8(vget_high_u8(m0), vget_high_u8(a0));
- blend_u16_lo = vmlal_u8(blend_u16_lo, vget_low_u8(m0_inv), vget_low_u8(b0));
- blend_u16_hi = vmlal_u8(blend_u16_hi, vget_high_u8(m0_inv), vget_high_u8(b0));
-
- uint8x8_t blend_u8_lo = vrshrn_n_u16(blend_u16_lo, AOM_BLEND_A64_ROUND_BITS);
- uint8x8_t blend_u8_hi = vrshrn_n_u16(blend_u16_hi, AOM_BLEND_A64_ROUND_BITS);
- uint8x16_t blend_u8 = vcombine_u8(blend_u8_lo, blend_u8_hi);
+ uint8x16_t blend_u8 = alpha_blend_a64_u8x16(m0, a0, b0);
return vpadalq_u8(sad, vabdq_u8(blend_u8, s0));
}
@@ -164,10 +157,7 @@
uint8x8_t b0 = vld1_u8(b);
uint8x8_t s0 = vld1_u8(src);
- uint8x8_t m0_inv = vsub_u8(vdup_n_u8(AOM_BLEND_A64_MAX_ALPHA), m0);
- uint16x8_t blend_u16 = vmull_u8(m0, a0);
- blend_u16 = vmlal_u8(blend_u16, m0_inv, b0);
- uint8x8_t blend_u8 = vrshrn_n_u16(blend_u16, AOM_BLEND_A64_ROUND_BITS);
+ uint8x8_t blend_u8 = alpha_blend_a64_u8x8(m0, a0, b0);
sad = vpadal_u8(sad, vabd_u8(blend_u8, s0));
@@ -199,10 +189,7 @@
uint8x8_t b0 = load_unaligned_u8(b, b_stride);
uint8x8_t s0 = load_unaligned_u8(src, src_stride);
- uint8x8_t m0_inv = vsub_u8(vdup_n_u8(AOM_BLEND_A64_MAX_ALPHA), m0);
- uint16x8_t blend_u16 = vmull_u8(m0, a0);
- blend_u16 = vmlal_u8(blend_u16, m0_inv, b0);
- uint8x8_t blend_u8 = vrshrn_n_u16(blend_u16, AOM_BLEND_A64_ROUND_BITS);
+ uint8x8_t blend_u8 = alpha_blend_a64_u8x8(m0, a0, b0);
sad = vpadal_u8(sad, vabd_u8(blend_u8, s0));
diff --git a/aom_dsp/arm/mem_neon.h b/aom_dsp/arm/mem_neon.h
index 16d44c5..d1ac648 100644
--- a/aom_dsp/arm/mem_neon.h
+++ b/aom_dsp/arm/mem_neon.h
@@ -43,6 +43,11 @@
return res;
}
+static INLINE uint16x8x2_t vld1q_u16_x2(const uint16_t *ptr) {
+ uint16x8x2_t res = { { vld1q_u16(ptr + 0), vld1q_u16(ptr + 8) } };
+ return res;
+}
+
static INLINE uint16x8x4_t vld1q_u16_x4(const uint16_t *ptr) {
uint16x8x4_t res = { { vld1q_u16(ptr + 0 * 8), vld1q_u16(ptr + 1 * 8),
vld1q_u16(ptr + 2 * 8), vld1q_u16(ptr + 3 * 8) } };
@@ -85,18 +90,31 @@
return vcombine_u8(vld1_u8(s), vld1_u8(s + p));
}
-/* These intrinsics require immediate values, so we must use #defines
- to enforce that. */
-#define load_u8_4x1(s, s0, lane) \
- do { \
- *(s0) = vreinterpret_u8_u32( \
- vld1_lane_u32((uint32_t *)(s), vreinterpret_u32_u8(*(s0)), lane)); \
- } while (0)
-
// Load four bytes into the low half of a uint8x8_t, zero the upper half.
-static INLINE uint8x8_t load_u8_4x1_lane0(const uint8_t *p) {
+static INLINE uint8x8_t load_u8_4x1(const uint8_t *p) {
uint8x8_t ret = vdup_n_u8(0);
- load_u8_4x1(p, &ret, 0);
+ ret = vreinterpret_u8_u32(
+ vld1_lane_u32((const uint32_t *)p, vreinterpret_u32_u8(ret), 0));
+ return ret;
+}
+
+static INLINE uint8x8_t load_u8_4x2(const uint8_t *p, int stride) {
+ uint8x8_t ret = vdup_n_u8(0);
+ ret = vreinterpret_u8_u32(
+ vld1_lane_u32((const uint32_t *)p, vreinterpret_u32_u8(ret), 0));
+ p += stride;
+ ret = vreinterpret_u8_u32(
+ vld1_lane_u32((const uint32_t *)p, vreinterpret_u32_u8(ret), 1));
+ return ret;
+}
+
+static INLINE uint16x4_t load_u16_2x2(const uint16_t *p, int stride) {
+ uint16x4_t ret = vdup_n_u16(0);
+ ret = vreinterpret_u16_u32(
+ vld1_lane_u32((const uint32_t *)p, vreinterpret_u32_u16(ret), 0));
+ p += stride;
+ ret = vreinterpret_u16_u32(
+ vld1_lane_u32((const uint32_t *)p, vreinterpret_u32_u16(ret), 1));
return ret;
}
@@ -214,6 +232,38 @@
s += p;
}
+static INLINE void load_s16_4x12(const int16_t *s, ptrdiff_t p,
+ int16x4_t *const s0, int16x4_t *const s1,
+ int16x4_t *const s2, int16x4_t *const s3,
+ int16x4_t *const s4, int16x4_t *const s5,
+ int16x4_t *const s6, int16x4_t *const s7,
+ int16x4_t *const s8, int16x4_t *const s9,
+ int16x4_t *const s10, int16x4_t *const s11) {
+ *s0 = vld1_s16(s);
+ s += p;
+ *s1 = vld1_s16(s);
+ s += p;
+ *s2 = vld1_s16(s);
+ s += p;
+ *s3 = vld1_s16(s);
+ s += p;
+ *s4 = vld1_s16(s);
+ s += p;
+ *s5 = vld1_s16(s);
+ s += p;
+ *s6 = vld1_s16(s);
+ s += p;
+ *s7 = vld1_s16(s);
+ s += p;
+ *s8 = vld1_s16(s);
+ s += p;
+ *s9 = vld1_s16(s);
+ s += p;
+ *s10 = vld1_s16(s);
+ s += p;
+ *s11 = vld1_s16(s);
+}
+
static INLINE void load_s16_4x11(const int16_t *s, ptrdiff_t p,
int16x4_t *const s0, int16x4_t *const s1,
int16x4_t *const s2, int16x4_t *const s3,
@@ -316,6 +366,23 @@
*s6 = vld1_s16(s);
}
+static INLINE void load_s16_4x6(const int16_t *s, ptrdiff_t p,
+ int16x4_t *const s0, int16x4_t *const s1,
+ int16x4_t *const s2, int16x4_t *const s3,
+ int16x4_t *const s4, int16x4_t *const s5) {
+ *s0 = vld1_s16(s);
+ s += p;
+ *s1 = vld1_s16(s);
+ s += p;
+ *s2 = vld1_s16(s);
+ s += p;
+ *s3 = vld1_s16(s);
+ s += p;
+ *s4 = vld1_s16(s);
+ s += p;
+ *s5 = vld1_s16(s);
+}
+
static INLINE void load_s16_4x5(const int16_t *s, ptrdiff_t p,
int16x4_t *const s0, int16x4_t *const s1,
int16x4_t *const s2, int16x4_t *const s3,
@@ -592,6 +659,33 @@
*s10 = vld1_u8(s);
}
+static INLINE void load_s16_8x10(const int16_t *s, ptrdiff_t p,
+ int16x8_t *const s0, int16x8_t *const s1,
+ int16x8_t *const s2, int16x8_t *const s3,
+ int16x8_t *const s4, int16x8_t *const s5,
+ int16x8_t *const s6, int16x8_t *const s7,
+ int16x8_t *const s8, int16x8_t *const s9) {
+ *s0 = vld1q_s16(s);
+ s += p;
+ *s1 = vld1q_s16(s);
+ s += p;
+ *s2 = vld1q_s16(s);
+ s += p;
+ *s3 = vld1q_s16(s);
+ s += p;
+ *s4 = vld1q_s16(s);
+ s += p;
+ *s5 = vld1q_s16(s);
+ s += p;
+ *s6 = vld1q_s16(s);
+ s += p;
+ *s7 = vld1q_s16(s);
+ s += p;
+ *s8 = vld1q_s16(s);
+ s += p;
+ *s9 = vld1q_s16(s);
+}
+
static INLINE void load_s16_8x11(const int16_t *s, ptrdiff_t p,
int16x8_t *const s0, int16x8_t *const s1,
int16x8_t *const s2, int16x8_t *const s3,
@@ -622,6 +716,38 @@
*s10 = vld1q_s16(s);
}
+static INLINE void load_s16_8x12(const int16_t *s, ptrdiff_t p,
+ int16x8_t *const s0, int16x8_t *const s1,
+ int16x8_t *const s2, int16x8_t *const s3,
+ int16x8_t *const s4, int16x8_t *const s5,
+ int16x8_t *const s6, int16x8_t *const s7,
+ int16x8_t *const s8, int16x8_t *const s9,
+ int16x8_t *const s10, int16x8_t *const s11) {
+ *s0 = vld1q_s16(s);
+ s += p;
+ *s1 = vld1q_s16(s);
+ s += p;
+ *s2 = vld1q_s16(s);
+ s += p;
+ *s3 = vld1q_s16(s);
+ s += p;
+ *s4 = vld1q_s16(s);
+ s += p;
+ *s5 = vld1q_s16(s);
+ s += p;
+ *s6 = vld1q_s16(s);
+ s += p;
+ *s7 = vld1q_s16(s);
+ s += p;
+ *s8 = vld1q_s16(s);
+ s += p;
+ *s9 = vld1q_s16(s);
+ s += p;
+ *s10 = vld1q_s16(s);
+ s += p;
+ *s11 = vld1q_s16(s);
+}
+
static INLINE void load_u16_8x11(const uint16_t *s, ptrdiff_t p,
uint16x8_t *const s0, uint16x8_t *const s1,
uint16x8_t *const s2, uint16x8_t *const s3,
@@ -714,6 +840,23 @@
*s6 = vld1q_s16(s);
}
+static INLINE void load_s16_8x6(const int16_t *s, ptrdiff_t p,
+ int16x8_t *const s0, int16x8_t *const s1,
+ int16x8_t *const s2, int16x8_t *const s3,
+ int16x8_t *const s4, int16x8_t *const s5) {
+ *s0 = vld1q_s16(s);
+ s += p;
+ *s1 = vld1q_s16(s);
+ s += p;
+ *s2 = vld1q_s16(s);
+ s += p;
+ *s3 = vld1q_s16(s);
+ s += p;
+ *s4 = vld1q_s16(s);
+ s += p;
+ *s5 = vld1q_s16(s);
+}
+
static INLINE void load_s16_8x5(const int16_t *s, ptrdiff_t p,
int16x8_t *const s0, int16x8_t *const s1,
int16x8_t *const s2, int16x8_t *const s3,
@@ -793,6 +936,24 @@
return vreinterpret_u8_u32(a_u32);
}
+static INLINE uint8x8_t load_unaligned_dup_u8_4x2(const uint8_t *buf) {
+ uint32_t a;
+ uint32x2_t a_u32;
+
+ memcpy(&a, buf, 4);
+ a_u32 = vdup_n_u32(a);
+ return vreinterpret_u8_u32(a_u32);
+}
+
+static INLINE uint8x8_t load_unaligned_dup_u8_2x4(const uint8_t *buf) {
+ uint16_t a;
+ uint16x4_t a_u32;
+
+ memcpy(&a, buf, 2);
+ a_u32 = vdup_n_u16(a);
+ return vreinterpret_u8_u16(a_u32);
+}
+
static INLINE uint8x8_t load_unaligned_u8_4x2(const uint8_t *buf, int stride) {
uint32_t a;
uint32x2_t a_u32;
@@ -844,6 +1005,20 @@
memcpy(dst, &a, 2); \
} while (0)
+#define store_unaligned_u16_2x1(dst, src, lane) \
+ do { \
+ uint32_t a; \
+ a = vget_lane_u32(vreinterpret_u32_u16(src), lane); \
+ memcpy(dst, &a, 4); \
+ } while (0)
+
+#define store_unaligned_u16_4x1(dst, src, lane) \
+ do { \
+ uint64_t a; \
+ a = vgetq_lane_u64(vreinterpretq_u64_u16(src), lane); \
+ memcpy(dst, &a, 8); \
+ } while (0)
+
static INLINE void load_u8_16x8(const uint8_t *s, ptrdiff_t p,
uint8x16_t *const s0, uint8x16_t *const s1,
uint8x16_t *const s2, uint8x16_t *const s3,
@@ -917,6 +1092,27 @@
*s7 = vld1q_u16(s + 8);
}
+static INLINE uint16x4_t load_unaligned_u16_2x2(const uint16_t *buf,
+ int stride) {
+ uint32_t a;
+ uint32x2_t a_u32;
+
+ memcpy(&a, buf, 4);
+ buf += stride;
+ a_u32 = vdup_n_u32(a);
+ memcpy(&a, buf, 4);
+ a_u32 = vset_lane_u32(a, a_u32, 1);
+ return vreinterpret_u16_u32(a_u32);
+}
+
+static INLINE uint16x4_t load_unaligned_u16_4x1(const uint16_t *buf) {
+ uint64_t a;
+ uint64x1_t a_u64 = vdup_n_u64(0);
+ memcpy(&a, buf, 8);
+ a_u64 = vset_lane_u64(a, a_u64, 0);
+ return vreinterpret_u16_u64(a_u64);
+}
+
static INLINE uint16x8_t load_unaligned_u16_4x2(const uint16_t *buf,
uint32_t stride) {
uint64_t a;
@@ -1004,4 +1200,32 @@
vst1q_s32(buf, v0);
}
+static INLINE void store_unaligned_u8_2x2(uint8_t *dst, uint32_t dst_stride,
+ uint8x8_t src) {
+ store_unaligned_u8_2x1(dst, src, 0);
+ dst += dst_stride;
+ store_unaligned_u8_2x1(dst, src, 1);
+}
+
+static INLINE void store_unaligned_u8_4x2(uint8_t *dst, uint32_t dst_stride,
+ uint8x8_t src) {
+ store_unaligned_u8_4x1(dst, src, 0);
+ dst += dst_stride;
+ store_unaligned_u8_4x1(dst, src, 1);
+}
+
+static INLINE void store_unaligned_u16_2x2(uint16_t *dst, uint32_t dst_stride,
+ uint16x4_t src) {
+ store_unaligned_u16_2x1(dst, src, 0);
+ dst += dst_stride;
+ store_unaligned_u16_2x1(dst, src, 1);
+}
+
+static INLINE void store_unaligned_u16_4x2(uint16_t *dst, uint32_t dst_stride,
+ uint16x8_t src) {
+ store_unaligned_u16_4x1(dst, src, 0);
+ dst += dst_stride;
+ store_unaligned_u16_4x1(dst, src, 1);
+}
+
#endif // AOM_AOM_DSP_ARM_MEM_NEON_H_
diff --git a/aom_dsp/arm/sad_neon.c b/aom_dsp/arm/sad_neon.c
index 60efef8..46a1666 100644
--- a/aom_dsp/arm/sad_neon.c
+++ b/aom_dsp/arm/sad_neon.c
@@ -15,93 +15,10 @@
#include "config/aom_dsp_rtcd.h"
#include "aom/aom_integer.h"
+#include "aom_dsp/arm/dist_wtd_avg_neon.h"
#include "aom_dsp/arm/mem_neon.h"
#include "aom_dsp/arm/sum_neon.h"
-#if defined(__ARM_FEATURE_DOTPROD)
-
-static INLINE unsigned int sadwxh_neon(const uint8_t *src_ptr, int src_stride,
- const uint8_t *ref_ptr, int ref_stride,
- int w, int h) {
- // Only two accumulators are required for optimal instruction throughput of
- // the ABD, UDOT sequence on CPUs with either 2 or 4 Neon pipes.
- uint32x4_t sum[2] = { vdupq_n_u32(0), vdupq_n_u32(0) };
-
- int i = h;
- do {
- int j = 0;
- do {
- uint8x16_t s0, s1, r0, r1, diff0, diff1;
-
- s0 = vld1q_u8(src_ptr + j);
- r0 = vld1q_u8(ref_ptr + j);
- diff0 = vabdq_u8(s0, r0);
- sum[0] = vdotq_u32(sum[0], diff0, vdupq_n_u8(1));
-
- s1 = vld1q_u8(src_ptr + j + 16);
- r1 = vld1q_u8(ref_ptr + j + 16);
- diff1 = vabdq_u8(s1, r1);
- sum[1] = vdotq_u32(sum[1], diff1, vdupq_n_u8(1));
-
- j += 32;
- } while (j < w);
-
- src_ptr += src_stride;
- ref_ptr += ref_stride;
- } while (--i != 0);
-
- return horizontal_add_u32x4(vaddq_u32(sum[0], sum[1]));
-}
-
-static INLINE unsigned int sad128xh_neon(const uint8_t *src_ptr, int src_stride,
- const uint8_t *ref_ptr, int ref_stride,
- int h) {
- return sadwxh_neon(src_ptr, src_stride, ref_ptr, ref_stride, 128, h);
-}
-
-static INLINE unsigned int sad64xh_neon(const uint8_t *src_ptr, int src_stride,
- const uint8_t *ref_ptr, int ref_stride,
- int h) {
- return sadwxh_neon(src_ptr, src_stride, ref_ptr, ref_stride, 64, h);
-}
-
-static INLINE unsigned int sad32xh_neon(const uint8_t *src_ptr, int src_stride,
- const uint8_t *ref_ptr, int ref_stride,
- int h) {
- return sadwxh_neon(src_ptr, src_stride, ref_ptr, ref_stride, 32, h);
-}
-
-static INLINE unsigned int sad16xh_neon(const uint8_t *src_ptr, int src_stride,
- const uint8_t *ref_ptr, int ref_stride,
- int h) {
- uint32x4_t sum[2] = { vdupq_n_u32(0), vdupq_n_u32(0) };
-
- int i = h / 2;
- do {
- uint8x16_t s0, s1, r0, r1, diff0, diff1;
-
- s0 = vld1q_u8(src_ptr);
- r0 = vld1q_u8(ref_ptr);
- diff0 = vabdq_u8(s0, r0);
- sum[0] = vdotq_u32(sum[0], diff0, vdupq_n_u8(1));
-
- src_ptr += src_stride;
- ref_ptr += ref_stride;
-
- s1 = vld1q_u8(src_ptr);
- r1 = vld1q_u8(ref_ptr);
- diff1 = vabdq_u8(s1, r1);
- sum[1] = vdotq_u32(sum[1], diff1, vdupq_n_u8(1));
-
- src_ptr += src_stride;
- ref_ptr += ref_stride;
- } while (--i != 0);
-
- return horizontal_add_u32x4(vaddq_u32(sum[0], sum[1]));
-}
-
-#else // !defined(__ARM_FEATURE_DOTPROD)
-
static INLINE unsigned int sad128xh_neon(const uint8_t *src_ptr, int src_stride,
const uint8_t *ref_ptr, int ref_stride,
int h) {
@@ -220,28 +137,25 @@
static INLINE unsigned int sad32xh_neon(const uint8_t *src_ptr, int src_stride,
const uint8_t *ref_ptr, int ref_stride,
int h) {
- uint32x4_t sum = vdupq_n_u32(0);
+ uint16x8_t sum[2] = { vdupq_n_u16(0), vdupq_n_u16(0) };
int i = h;
do {
uint8x16_t s0 = vld1q_u8(src_ptr);
uint8x16_t r0 = vld1q_u8(ref_ptr);
uint8x16_t diff0 = vabdq_u8(s0, r0);
- uint16x8_t sum0 = vpaddlq_u8(diff0);
+ sum[0] = vpadalq_u8(sum[0], diff0);
uint8x16_t s1 = vld1q_u8(src_ptr + 16);
uint8x16_t r1 = vld1q_u8(ref_ptr + 16);
uint8x16_t diff1 = vabdq_u8(s1, r1);
- uint16x8_t sum1 = vpaddlq_u8(diff1);
-
- sum = vpadalq_u16(sum, sum0);
- sum = vpadalq_u16(sum, sum1);
+ sum[1] = vpadalq_u8(sum[1], diff1);
src_ptr += src_stride;
ref_ptr += ref_stride;
} while (--i != 0);
- return horizontal_add_u32x4(sum);
+ return horizontal_add_u16x8(vaddq_u16(sum[0], sum[1]));
}
static INLINE unsigned int sad16xh_neon(const uint8_t *src_ptr, int src_stride,
@@ -264,8 +178,6 @@
return horizontal_add_u16x8(sum);
}
-#endif // defined(__ARM_FEATURE_DOTPROD)
-
static INLINE unsigned int sad8xh_neon(const uint8_t *src_ptr, int src_stride,
const uint8_t *ref_ptr, int ref_stride,
int h) {
@@ -384,114 +296,6 @@
#undef SAD_SKIP_WXH_NEON
-#if defined(__ARM_FEATURE_DOTPROD)
-
-static INLINE unsigned int sadwxh_avg_neon(const uint8_t *src_ptr,
- int src_stride,
- const uint8_t *ref_ptr,
- int ref_stride, int w, int h,
- const uint8_t *second_pred) {
- // Only two accumulators are required for optimal instruction throughput of
- // the ABD, UDOT sequence on CPUs with either 2 or 4 Neon pipes.
- uint32x4_t sum[2] = { vdupq_n_u32(0), vdupq_n_u32(0) };
-
- int i = h;
- do {
- int j = 0;
- do {
- uint8x16_t s0, s1, r0, r1, p0, p1, avg0, avg1, diff0, diff1;
-
- s0 = vld1q_u8(src_ptr + j);
- r0 = vld1q_u8(ref_ptr + j);
- p0 = vld1q_u8(second_pred);
- avg0 = vrhaddq_u8(r0, p0);
- diff0 = vabdq_u8(s0, avg0);
- sum[0] = vdotq_u32(sum[0], diff0, vdupq_n_u8(1));
-
- s1 = vld1q_u8(src_ptr + j + 16);
- r1 = vld1q_u8(ref_ptr + j + 16);
- p1 = vld1q_u8(second_pred + 16);
- avg1 = vrhaddq_u8(r1, p1);
- diff1 = vabdq_u8(s1, avg1);
- sum[1] = vdotq_u32(sum[1], diff1, vdupq_n_u8(1));
-
- j += 32;
- second_pred += 32;
- } while (j < w);
-
- src_ptr += src_stride;
- ref_ptr += ref_stride;
- } while (--i != 0);
-
- return horizontal_add_u32x4(vaddq_u32(sum[0], sum[1]));
-}
-
-static INLINE unsigned int sad128xh_avg_neon(const uint8_t *src_ptr,
- int src_stride,
- const uint8_t *ref_ptr,
- int ref_stride, int h,
- const uint8_t *second_pred) {
- return sadwxh_avg_neon(src_ptr, src_stride, ref_ptr, ref_stride, 128, h,
- second_pred);
-}
-
-static INLINE unsigned int sad64xh_avg_neon(const uint8_t *src_ptr,
- int src_stride,
- const uint8_t *ref_ptr,
- int ref_stride, int h,
- const uint8_t *second_pred) {
- return sadwxh_avg_neon(src_ptr, src_stride, ref_ptr, ref_stride, 64, h,
- second_pred);
-}
-
-static INLINE unsigned int sad32xh_avg_neon(const uint8_t *src_ptr,
- int src_stride,
- const uint8_t *ref_ptr,
- int ref_stride, int h,
- const uint8_t *second_pred) {
- return sadwxh_avg_neon(src_ptr, src_stride, ref_ptr, ref_stride, 32, h,
- second_pred);
-}
-
-static INLINE unsigned int sad16xh_avg_neon(const uint8_t *src_ptr,
- int src_stride,
- const uint8_t *ref_ptr,
- int ref_stride, int h,
- const uint8_t *second_pred) {
- uint32x4_t sum[2] = { vdupq_n_u32(0), vdupq_n_u32(0) };
-
- int i = h / 2;
- do {
- uint8x16_t s0, s1, r0, r1, p0, p1, avg0, avg1, diff0, diff1;
-
- s0 = vld1q_u8(src_ptr);
- r0 = vld1q_u8(ref_ptr);
- p0 = vld1q_u8(second_pred);
- avg0 = vrhaddq_u8(r0, p0);
- diff0 = vabdq_u8(s0, avg0);
- sum[0] = vdotq_u32(sum[0], diff0, vdupq_n_u8(1));
-
- src_ptr += src_stride;
- ref_ptr += ref_stride;
- second_pred += 16;
-
- s1 = vld1q_u8(src_ptr);
- r1 = vld1q_u8(ref_ptr);
- p1 = vld1q_u8(second_pred);
- avg1 = vrhaddq_u8(r1, p1);
- diff1 = vabdq_u8(s1, avg1);
- sum[1] = vdotq_u32(sum[1], diff1, vdupq_n_u8(1));
-
- src_ptr += src_stride;
- ref_ptr += ref_stride;
- second_pred += 16;
- } while (--i != 0);
-
- return horizontal_add_u32x4(vaddq_u32(sum[0], sum[1]));
-}
-
-#else // !defined(__ARM_FEATURE_DOTPROD)
-
static INLINE unsigned int sad128xh_avg_neon(const uint8_t *src_ptr,
int src_stride,
const uint8_t *ref_ptr,
@@ -644,7 +448,7 @@
const uint8_t *ref_ptr,
int ref_stride, int h,
const uint8_t *second_pred) {
- uint32x4_t sum = vdupq_n_u32(0);
+ uint16x8_t sum[2] = { vdupq_n_u16(0), vdupq_n_u16(0) };
int i = h;
do {
@@ -653,24 +457,21 @@
uint8x16_t p0 = vld1q_u8(second_pred);
uint8x16_t avg0 = vrhaddq_u8(r0, p0);
uint8x16_t diff0 = vabdq_u8(s0, avg0);
- uint16x8_t sum0 = vpaddlq_u8(diff0);
+ sum[0] = vpadalq_u8(sum[0], diff0);
uint8x16_t s1 = vld1q_u8(src_ptr + 16);
uint8x16_t r1 = vld1q_u8(ref_ptr + 16);
uint8x16_t p1 = vld1q_u8(second_pred + 16);
uint8x16_t avg1 = vrhaddq_u8(r1, p1);
uint8x16_t diff1 = vabdq_u8(s1, avg1);
- uint16x8_t sum1 = vpaddlq_u8(diff1);
-
- sum = vpadalq_u16(sum, sum0);
- sum = vpadalq_u16(sum, sum1);
+ sum[1] = vpadalq_u8(sum[1], diff1);
src_ptr += src_stride;
ref_ptr += ref_stride;
second_pred += 32;
} while (--i != 0);
- return horizontal_add_u32x4(sum);
+ return horizontal_add_u16x8(vaddq_u16(sum[0], sum[1]));
}
static INLINE unsigned int sad16xh_avg_neon(const uint8_t *src_ptr,
@@ -698,8 +499,6 @@
return horizontal_add_u16x8(sum);
}
-#endif // defined(__ARM_FEATURE_DOTPROD)
-
static INLINE unsigned int sad8xh_avg_neon(const uint8_t *src_ptr,
int src_stride,
const uint8_t *ref_ptr,
@@ -788,3 +587,287 @@
#endif // !CONFIG_REALTIME_ONLY
#undef SAD_WXH_AVG_NEON
+
+static INLINE unsigned int dist_wtd_sad128xh_avg_neon(
+ const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr,
+ int ref_stride, int h, const uint8_t *second_pred,
+ const DIST_WTD_COMP_PARAMS *jcp_param) {
+ const uint8x16_t fwd_offset = vdupq_n_u8(jcp_param->fwd_offset);
+ const uint8x16_t bck_offset = vdupq_n_u8(jcp_param->bck_offset);
+ // We use 8 accumulators to prevent overflow for large values of 'h', as well
+ // as enabling optimal UADALP instruction throughput on CPUs that have either
+ // 2 or 4 Neon pipes.
+ uint16x8_t sum[8] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0),
+ vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0),
+ vdupq_n_u16(0), vdupq_n_u16(0) };
+
+ do {
+ uint8x16_t s0 = vld1q_u8(src_ptr);
+ uint8x16_t r0 = vld1q_u8(ref_ptr);
+ uint8x16_t p0 = vld1q_u8(second_pred);
+ uint8x16_t wtd_avg0 = dist_wtd_avg_u8x16(p0, r0, bck_offset, fwd_offset);
+ uint8x16_t diff0 = vabdq_u8(s0, wtd_avg0);
+ sum[0] = vpadalq_u8(sum[0], diff0);
+
+ uint8x16_t s1 = vld1q_u8(src_ptr + 16);
+ uint8x16_t r1 = vld1q_u8(ref_ptr + 16);
+ uint8x16_t p1 = vld1q_u8(second_pred + 16);
+ uint8x16_t wtd_avg1 = dist_wtd_avg_u8x16(p1, r1, bck_offset, fwd_offset);
+ uint8x16_t diff1 = vabdq_u8(s1, wtd_avg1);
+ sum[1] = vpadalq_u8(sum[1], diff1);
+
+ uint8x16_t s2 = vld1q_u8(src_ptr + 32);
+ uint8x16_t r2 = vld1q_u8(ref_ptr + 32);
+ uint8x16_t p2 = vld1q_u8(second_pred + 32);
+ uint8x16_t wtd_avg2 = dist_wtd_avg_u8x16(p2, r2, bck_offset, fwd_offset);
+ uint8x16_t diff2 = vabdq_u8(s2, wtd_avg2);
+ sum[2] = vpadalq_u8(sum[2], diff2);
+
+ uint8x16_t s3 = vld1q_u8(src_ptr + 48);
+ uint8x16_t r3 = vld1q_u8(ref_ptr + 48);
+ uint8x16_t p3 = vld1q_u8(second_pred + 48);
+ uint8x16_t wtd_avg3 = dist_wtd_avg_u8x16(p3, r3, bck_offset, fwd_offset);
+ uint8x16_t diff3 = vabdq_u8(s3, wtd_avg3);
+ sum[3] = vpadalq_u8(sum[3], diff3);
+
+ uint8x16_t s4 = vld1q_u8(src_ptr + 64);
+ uint8x16_t r4 = vld1q_u8(ref_ptr + 64);
+ uint8x16_t p4 = vld1q_u8(second_pred + 64);
+ uint8x16_t wtd_avg4 = dist_wtd_avg_u8x16(p4, r4, bck_offset, fwd_offset);
+ uint8x16_t diff4 = vabdq_u8(s4, wtd_avg4);
+ sum[4] = vpadalq_u8(sum[4], diff4);
+
+ uint8x16_t s5 = vld1q_u8(src_ptr + 80);
+ uint8x16_t r5 = vld1q_u8(ref_ptr + 80);
+ uint8x16_t p5 = vld1q_u8(second_pred + 80);
+ uint8x16_t wtd_avg5 = dist_wtd_avg_u8x16(p5, r5, bck_offset, fwd_offset);
+ uint8x16_t diff5 = vabdq_u8(s5, wtd_avg5);
+ sum[5] = vpadalq_u8(sum[5], diff5);
+
+ uint8x16_t s6 = vld1q_u8(src_ptr + 96);
+ uint8x16_t r6 = vld1q_u8(ref_ptr + 96);
+ uint8x16_t p6 = vld1q_u8(second_pred + 96);
+ uint8x16_t wtd_avg6 = dist_wtd_avg_u8x16(p6, r6, bck_offset, fwd_offset);
+ uint8x16_t diff6 = vabdq_u8(s6, wtd_avg6);
+ sum[6] = vpadalq_u8(sum[6], diff6);
+
+ uint8x16_t s7 = vld1q_u8(src_ptr + 112);
+ uint8x16_t r7 = vld1q_u8(ref_ptr + 112);
+ uint8x16_t p7 = vld1q_u8(second_pred + 112);
+ uint8x16_t wtd_avg7 = dist_wtd_avg_u8x16(p7, r7, bck_offset, fwd_offset);
+ uint8x16_t diff7 = vabdq_u8(s7, wtd_avg7);
+ sum[7] = vpadalq_u8(sum[7], diff7);
+
+ src_ptr += src_stride;
+ ref_ptr += ref_stride;
+ second_pred += 128;
+ } while (--h != 0);
+
+ uint32x4_t sum_u32 = vpaddlq_u16(sum[0]);
+ sum_u32 = vpadalq_u16(sum_u32, sum[1]);
+ sum_u32 = vpadalq_u16(sum_u32, sum[2]);
+ sum_u32 = vpadalq_u16(sum_u32, sum[3]);
+ sum_u32 = vpadalq_u16(sum_u32, sum[4]);
+ sum_u32 = vpadalq_u16(sum_u32, sum[5]);
+ sum_u32 = vpadalq_u16(sum_u32, sum[6]);
+ sum_u32 = vpadalq_u16(sum_u32, sum[7]);
+
+ return horizontal_add_u32x4(sum_u32);
+}
+
+static INLINE unsigned int dist_wtd_sad64xh_avg_neon(
+ const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr,
+ int ref_stride, int h, const uint8_t *second_pred,
+ const DIST_WTD_COMP_PARAMS *jcp_param) {
+ const uint8x16_t fwd_offset = vdupq_n_u8(jcp_param->fwd_offset);
+ const uint8x16_t bck_offset = vdupq_n_u8(jcp_param->bck_offset);
+ uint16x8_t sum[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0),
+ vdupq_n_u16(0) };
+
+ do {
+ uint8x16_t s0 = vld1q_u8(src_ptr);
+ uint8x16_t r0 = vld1q_u8(ref_ptr);
+ uint8x16_t p0 = vld1q_u8(second_pred);
+ uint8x16_t wtd_avg0 = dist_wtd_avg_u8x16(p0, r0, bck_offset, fwd_offset);
+ uint8x16_t diff0 = vabdq_u8(s0, wtd_avg0);
+ sum[0] = vpadalq_u8(sum[0], diff0);
+
+ uint8x16_t s1 = vld1q_u8(src_ptr + 16);
+ uint8x16_t r1 = vld1q_u8(ref_ptr + 16);
+ uint8x16_t p1 = vld1q_u8(second_pred + 16);
+ uint8x16_t wtd_avg1 = dist_wtd_avg_u8x16(p1, r1, bck_offset, fwd_offset);
+ uint8x16_t diff1 = vabdq_u8(s1, wtd_avg1);
+ sum[1] = vpadalq_u8(sum[1], diff1);
+
+ uint8x16_t s2 = vld1q_u8(src_ptr + 32);
+ uint8x16_t r2 = vld1q_u8(ref_ptr + 32);
+ uint8x16_t p2 = vld1q_u8(second_pred + 32);
+ uint8x16_t wtd_avg2 = dist_wtd_avg_u8x16(p2, r2, bck_offset, fwd_offset);
+ uint8x16_t diff2 = vabdq_u8(s2, wtd_avg2);
+ sum[2] = vpadalq_u8(sum[2], diff2);
+
+ uint8x16_t s3 = vld1q_u8(src_ptr + 48);
+ uint8x16_t r3 = vld1q_u8(ref_ptr + 48);
+ uint8x16_t p3 = vld1q_u8(second_pred + 48);
+ uint8x16_t wtd_avg3 = dist_wtd_avg_u8x16(p3, r3, bck_offset, fwd_offset);
+ uint8x16_t diff3 = vabdq_u8(s3, wtd_avg3);
+ sum[3] = vpadalq_u8(sum[3], diff3);
+
+ src_ptr += src_stride;
+ ref_ptr += ref_stride;
+ second_pred += 64;
+ } while (--h != 0);
+
+ uint32x4_t sum_u32 = vpaddlq_u16(sum[0]);
+ sum_u32 = vpadalq_u16(sum_u32, sum[1]);
+ sum_u32 = vpadalq_u16(sum_u32, sum[2]);
+ sum_u32 = vpadalq_u16(sum_u32, sum[3]);
+
+ return horizontal_add_u32x4(sum_u32);
+}
+
+static INLINE unsigned int dist_wtd_sad32xh_avg_neon(
+ const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr,
+ int ref_stride, int h, const uint8_t *second_pred,
+ const DIST_WTD_COMP_PARAMS *jcp_param) {
+ const uint8x16_t fwd_offset = vdupq_n_u8(jcp_param->fwd_offset);
+ const uint8x16_t bck_offset = vdupq_n_u8(jcp_param->bck_offset);
+ uint16x8_t sum[2] = { vdupq_n_u16(0), vdupq_n_u16(0) };
+
+ do {
+ uint8x16_t s0 = vld1q_u8(src_ptr);
+ uint8x16_t r0 = vld1q_u8(ref_ptr);
+ uint8x16_t p0 = vld1q_u8(second_pred);
+ uint8x16_t wtd_avg0 = dist_wtd_avg_u8x16(p0, r0, bck_offset, fwd_offset);
+ uint8x16_t diff0 = vabdq_u8(s0, wtd_avg0);
+ sum[0] = vpadalq_u8(sum[0], diff0);
+
+ uint8x16_t s1 = vld1q_u8(src_ptr + 16);
+ uint8x16_t r1 = vld1q_u8(ref_ptr + 16);
+ uint8x16_t p1 = vld1q_u8(second_pred + 16);
+ uint8x16_t wtd_avg1 = dist_wtd_avg_u8x16(p1, r1, bck_offset, fwd_offset);
+ uint8x16_t diff1 = vabdq_u8(s1, wtd_avg1);
+ sum[1] = vpadalq_u8(sum[1], diff1);
+
+ src_ptr += src_stride;
+ ref_ptr += ref_stride;
+ second_pred += 32;
+ } while (--h != 0);
+
+ return horizontal_add_u16x8(vaddq_u16(sum[0], sum[1]));
+}
+
+static INLINE unsigned int dist_wtd_sad16xh_avg_neon(
+ const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr,
+ int ref_stride, int h, const uint8_t *second_pred,
+ const DIST_WTD_COMP_PARAMS *jcp_param) {
+ const uint8x16_t fwd_offset = vdupq_n_u8(jcp_param->fwd_offset);
+ const uint8x16_t bck_offset = vdupq_n_u8(jcp_param->bck_offset);
+ uint16x8_t sum = vdupq_n_u16(0);
+
+ do {
+ uint8x16_t s = vld1q_u8(src_ptr);
+ uint8x16_t r = vld1q_u8(ref_ptr);
+ uint8x16_t p = vld1q_u8(second_pred);
+
+ uint8x16_t wtd_avg = dist_wtd_avg_u8x16(p, r, bck_offset, fwd_offset);
+ uint8x16_t diff = vabdq_u8(s, wtd_avg);
+ sum = vpadalq_u8(sum, diff);
+
+ src_ptr += src_stride;
+ ref_ptr += ref_stride;
+ second_pred += 16;
+ } while (--h != 0);
+
+ return horizontal_add_u16x8(sum);
+}
+
+static INLINE unsigned int dist_wtd_sad8xh_avg_neon(
+ const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr,
+ int ref_stride, int h, const uint8_t *second_pred,
+ const DIST_WTD_COMP_PARAMS *jcp_param) {
+ const uint8x8_t fwd_offset = vdup_n_u8(jcp_param->fwd_offset);
+ const uint8x8_t bck_offset = vdup_n_u8(jcp_param->bck_offset);
+ uint16x8_t sum = vdupq_n_u16(0);
+
+ do {
+ uint8x8_t s = vld1_u8(src_ptr);
+ uint8x8_t r = vld1_u8(ref_ptr);
+ uint8x8_t p = vld1_u8(second_pred);
+
+ uint8x8_t wtd_avg = dist_wtd_avg_u8x8(p, r, bck_offset, fwd_offset);
+ sum = vabal_u8(sum, s, wtd_avg);
+
+ src_ptr += src_stride;
+ ref_ptr += ref_stride;
+ second_pred += 8;
+ } while (--h != 0);
+
+ return horizontal_add_u16x8(sum);
+}
+
+static INLINE unsigned int dist_wtd_sad4xh_avg_neon(
+ const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr,
+ int ref_stride, int h, const uint8_t *second_pred,
+ const DIST_WTD_COMP_PARAMS *jcp_param) {
+ const uint8x8_t fwd_offset = vdup_n_u8(jcp_param->fwd_offset);
+ const uint8x8_t bck_offset = vdup_n_u8(jcp_param->bck_offset);
+ uint16x8_t sum = vdupq_n_u16(0);
+
+ int i = h / 2;
+ do {
+ uint8x8_t s = load_unaligned_u8(src_ptr, src_stride);
+ uint8x8_t r = load_unaligned_u8(ref_ptr, ref_stride);
+ uint8x8_t p = vld1_u8(second_pred);
+
+ uint8x8_t wtd_avg = dist_wtd_avg_u8x8(p, r, bck_offset, fwd_offset);
+ sum = vabal_u8(sum, s, wtd_avg);
+
+ src_ptr += 2 * src_stride;
+ ref_ptr += 2 * ref_stride;
+ second_pred += 8;
+ } while (--i != 0);
+
+ return horizontal_add_u16x8(sum);
+}
+
+#define DIST_WTD_SAD_WXH_AVG_NEON(w, h) \
+ unsigned int aom_dist_wtd_sad##w##x##h##_avg_neon( \
+ const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
+ const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \
+ return dist_wtd_sad##w##xh_avg_neon(src, src_stride, ref, ref_stride, (h), \
+ second_pred, jcp_param); \
+ }
+
+DIST_WTD_SAD_WXH_AVG_NEON(4, 4)
+DIST_WTD_SAD_WXH_AVG_NEON(4, 8)
+
+DIST_WTD_SAD_WXH_AVG_NEON(8, 4)
+DIST_WTD_SAD_WXH_AVG_NEON(8, 8)
+DIST_WTD_SAD_WXH_AVG_NEON(8, 16)
+
+DIST_WTD_SAD_WXH_AVG_NEON(16, 8)
+DIST_WTD_SAD_WXH_AVG_NEON(16, 16)
+DIST_WTD_SAD_WXH_AVG_NEON(16, 32)
+
+DIST_WTD_SAD_WXH_AVG_NEON(32, 16)
+DIST_WTD_SAD_WXH_AVG_NEON(32, 32)
+DIST_WTD_SAD_WXH_AVG_NEON(32, 64)
+
+DIST_WTD_SAD_WXH_AVG_NEON(64, 32)
+DIST_WTD_SAD_WXH_AVG_NEON(64, 64)
+DIST_WTD_SAD_WXH_AVG_NEON(64, 128)
+
+DIST_WTD_SAD_WXH_AVG_NEON(128, 64)
+DIST_WTD_SAD_WXH_AVG_NEON(128, 128)
+
+#if !CONFIG_REALTIME_ONLY
+DIST_WTD_SAD_WXH_AVG_NEON(4, 16)
+DIST_WTD_SAD_WXH_AVG_NEON(8, 32)
+DIST_WTD_SAD_WXH_AVG_NEON(16, 4)
+DIST_WTD_SAD_WXH_AVG_NEON(16, 64)
+DIST_WTD_SAD_WXH_AVG_NEON(32, 8)
+DIST_WTD_SAD_WXH_AVG_NEON(64, 16)
+#endif // !CONFIG_REALTIME_ONLY
+
+#undef DIST_WTD_SAD_WXH_AVG_NEON
diff --git a/aom_dsp/arm/sad_neon_dotprod.c b/aom_dsp/arm/sad_neon_dotprod.c
new file mode 100644
index 0000000..5504c68
--- /dev/null
+++ b/aom_dsp/arm/sad_neon_dotprod.c
@@ -0,0 +1,530 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+
+#include "config/aom_config.h"
+#include "config/aom_dsp_rtcd.h"
+
+#include "aom/aom_integer.h"
+#include "aom_dsp/arm/dist_wtd_avg_neon.h"
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_dsp/arm/sum_neon.h"
+
+static INLINE unsigned int sadwxh_neon_dotprod(const uint8_t *src_ptr,
+ int src_stride,
+ const uint8_t *ref_ptr,
+ int ref_stride, int w, int h) {
+ // Only two accumulators are required for optimal instruction throughput of
+ // the ABD, UDOT sequence on CPUs with either 2 or 4 Neon pipes.
+ uint32x4_t sum[2] = { vdupq_n_u32(0), vdupq_n_u32(0) };
+
+ int i = h;
+ do {
+ int j = 0;
+ do {
+ uint8x16_t s0, s1, r0, r1, diff0, diff1;
+
+ s0 = vld1q_u8(src_ptr + j);
+ r0 = vld1q_u8(ref_ptr + j);
+ diff0 = vabdq_u8(s0, r0);
+ sum[0] = vdotq_u32(sum[0], diff0, vdupq_n_u8(1));
+
+ s1 = vld1q_u8(src_ptr + j + 16);
+ r1 = vld1q_u8(ref_ptr + j + 16);
+ diff1 = vabdq_u8(s1, r1);
+ sum[1] = vdotq_u32(sum[1], diff1, vdupq_n_u8(1));
+
+ j += 32;
+ } while (j < w);
+
+ src_ptr += src_stride;
+ ref_ptr += ref_stride;
+ } while (--i != 0);
+
+ return horizontal_add_u32x4(vaddq_u32(sum[0], sum[1]));
+}
+
+static INLINE unsigned int sad128xh_neon_dotprod(const uint8_t *src_ptr,
+ int src_stride,
+ const uint8_t *ref_ptr,
+ int ref_stride, int h) {
+ return sadwxh_neon_dotprod(src_ptr, src_stride, ref_ptr, ref_stride, 128, h);
+}
+
+static INLINE unsigned int sad64xh_neon_dotprod(const uint8_t *src_ptr,
+ int src_stride,
+ const uint8_t *ref_ptr,
+ int ref_stride, int h) {
+ return sadwxh_neon_dotprod(src_ptr, src_stride, ref_ptr, ref_stride, 64, h);
+}
+
+static INLINE unsigned int sad32xh_neon_dotprod(const uint8_t *src_ptr,
+ int src_stride,
+ const uint8_t *ref_ptr,
+ int ref_stride, int h) {
+ return sadwxh_neon_dotprod(src_ptr, src_stride, ref_ptr, ref_stride, 32, h);
+}
+
+static INLINE unsigned int sad16xh_neon_dotprod(const uint8_t *src_ptr,
+ int src_stride,
+ const uint8_t *ref_ptr,
+ int ref_stride, int h) {
+ uint32x4_t sum[2] = { vdupq_n_u32(0), vdupq_n_u32(0) };
+
+ int i = h / 2;
+ do {
+ uint8x16_t s0, s1, r0, r1, diff0, diff1;
+
+ s0 = vld1q_u8(src_ptr);
+ r0 = vld1q_u8(ref_ptr);
+ diff0 = vabdq_u8(s0, r0);
+ sum[0] = vdotq_u32(sum[0], diff0, vdupq_n_u8(1));
+
+ src_ptr += src_stride;
+ ref_ptr += ref_stride;
+
+ s1 = vld1q_u8(src_ptr);
+ r1 = vld1q_u8(ref_ptr);
+ diff1 = vabdq_u8(s1, r1);
+ sum[1] = vdotq_u32(sum[1], diff1, vdupq_n_u8(1));
+
+ src_ptr += src_stride;
+ ref_ptr += ref_stride;
+ } while (--i != 0);
+
+ return horizontal_add_u32x4(vaddq_u32(sum[0], sum[1]));
+}
+
+#define SAD_WXH_NEON_DOTPROD(w, h) \
+ unsigned int aom_sad##w##x##h##_neon_dotprod( \
+ const uint8_t *src, int src_stride, const uint8_t *ref, \
+ int ref_stride) { \
+ return sad##w##xh_neon_dotprod(src, src_stride, ref, ref_stride, (h)); \
+ }
+
+SAD_WXH_NEON_DOTPROD(16, 8)
+SAD_WXH_NEON_DOTPROD(16, 16)
+SAD_WXH_NEON_DOTPROD(16, 32)
+
+SAD_WXH_NEON_DOTPROD(32, 16)
+SAD_WXH_NEON_DOTPROD(32, 32)
+SAD_WXH_NEON_DOTPROD(32, 64)
+
+SAD_WXH_NEON_DOTPROD(64, 32)
+SAD_WXH_NEON_DOTPROD(64, 64)
+SAD_WXH_NEON_DOTPROD(64, 128)
+
+SAD_WXH_NEON_DOTPROD(128, 64)
+SAD_WXH_NEON_DOTPROD(128, 128)
+
+#if !CONFIG_REALTIME_ONLY
+SAD_WXH_NEON_DOTPROD(16, 4)
+SAD_WXH_NEON_DOTPROD(16, 64)
+SAD_WXH_NEON_DOTPROD(32, 8)
+SAD_WXH_NEON_DOTPROD(64, 16)
+#endif // !CONFIG_REALTIME_ONLY
+
+#undef SAD_WXH_NEON_DOTPROD
+
+#define SAD_SKIP_WXH_NEON_DOTPROD(w, h) \
+ unsigned int aom_sad_skip_##w##x##h##_neon_dotprod( \
+ const uint8_t *src, int src_stride, const uint8_t *ref, \
+ int ref_stride) { \
+ return 2 * sad##w##xh_neon_dotprod(src, 2 * src_stride, ref, \
+ 2 * ref_stride, (h) / 2); \
+ }
+
+SAD_SKIP_WXH_NEON_DOTPROD(16, 8)
+SAD_SKIP_WXH_NEON_DOTPROD(16, 16)
+SAD_SKIP_WXH_NEON_DOTPROD(16, 32)
+
+SAD_SKIP_WXH_NEON_DOTPROD(32, 16)
+SAD_SKIP_WXH_NEON_DOTPROD(32, 32)
+SAD_SKIP_WXH_NEON_DOTPROD(32, 64)
+
+SAD_SKIP_WXH_NEON_DOTPROD(64, 32)
+SAD_SKIP_WXH_NEON_DOTPROD(64, 64)
+SAD_SKIP_WXH_NEON_DOTPROD(64, 128)
+
+SAD_SKIP_WXH_NEON_DOTPROD(128, 64)
+SAD_SKIP_WXH_NEON_DOTPROD(128, 128)
+
+#if !CONFIG_REALTIME_ONLY
+SAD_SKIP_WXH_NEON_DOTPROD(16, 4)
+SAD_SKIP_WXH_NEON_DOTPROD(16, 64)
+SAD_SKIP_WXH_NEON_DOTPROD(32, 8)
+SAD_SKIP_WXH_NEON_DOTPROD(64, 16)
+#endif // !CONFIG_REALTIME_ONLY
+
+#undef SAD_SKIP_WXH_NEON_DOTPROD
+
+static INLINE unsigned int sadwxh_avg_neon_dotprod(const uint8_t *src_ptr,
+ int src_stride,
+ const uint8_t *ref_ptr,
+ int ref_stride, int w, int h,
+ const uint8_t *second_pred) {
+ // Only two accumulators are required for optimal instruction throughput of
+ // the ABD, UDOT sequence on CPUs with either 2 or 4 Neon pipes.
+ uint32x4_t sum[2] = { vdupq_n_u32(0), vdupq_n_u32(0) };
+
+ int i = h;
+ do {
+ int j = 0;
+ do {
+ uint8x16_t s0, s1, r0, r1, p0, p1, avg0, avg1, diff0, diff1;
+
+ s0 = vld1q_u8(src_ptr + j);
+ r0 = vld1q_u8(ref_ptr + j);
+ p0 = vld1q_u8(second_pred);
+ avg0 = vrhaddq_u8(r0, p0);
+ diff0 = vabdq_u8(s0, avg0);
+ sum[0] = vdotq_u32(sum[0], diff0, vdupq_n_u8(1));
+
+ s1 = vld1q_u8(src_ptr + j + 16);
+ r1 = vld1q_u8(ref_ptr + j + 16);
+ p1 = vld1q_u8(second_pred + 16);
+ avg1 = vrhaddq_u8(r1, p1);
+ diff1 = vabdq_u8(s1, avg1);
+ sum[1] = vdotq_u32(sum[1], diff1, vdupq_n_u8(1));
+
+ j += 32;
+ second_pred += 32;
+ } while (j < w);
+
+ src_ptr += src_stride;
+ ref_ptr += ref_stride;
+ } while (--i != 0);
+
+ return horizontal_add_u32x4(vaddq_u32(sum[0], sum[1]));
+}
+
+static INLINE unsigned int sad128xh_avg_neon_dotprod(
+ const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr,
+ int ref_stride, int h, const uint8_t *second_pred) {
+ return sadwxh_avg_neon_dotprod(src_ptr, src_stride, ref_ptr, ref_stride, 128,
+ h, second_pred);
+}
+
+static INLINE unsigned int sad64xh_avg_neon_dotprod(
+ const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr,
+ int ref_stride, int h, const uint8_t *second_pred) {
+ return sadwxh_avg_neon_dotprod(src_ptr, src_stride, ref_ptr, ref_stride, 64,
+ h, second_pred);
+}
+
+static INLINE unsigned int sad32xh_avg_neon_dotprod(
+ const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr,
+ int ref_stride, int h, const uint8_t *second_pred) {
+ return sadwxh_avg_neon_dotprod(src_ptr, src_stride, ref_ptr, ref_stride, 32,
+ h, second_pred);
+}
+
+static INLINE unsigned int sad16xh_avg_neon_dotprod(
+ const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr,
+ int ref_stride, int h, const uint8_t *second_pred) {
+ uint32x4_t sum[2] = { vdupq_n_u32(0), vdupq_n_u32(0) };
+
+ int i = h / 2;
+ do {
+ uint8x16_t s0, s1, r0, r1, p0, p1, avg0, avg1, diff0, diff1;
+
+ s0 = vld1q_u8(src_ptr);
+ r0 = vld1q_u8(ref_ptr);
+ p0 = vld1q_u8(second_pred);
+ avg0 = vrhaddq_u8(r0, p0);
+ diff0 = vabdq_u8(s0, avg0);
+ sum[0] = vdotq_u32(sum[0], diff0, vdupq_n_u8(1));
+
+ src_ptr += src_stride;
+ ref_ptr += ref_stride;
+ second_pred += 16;
+
+ s1 = vld1q_u8(src_ptr);
+ r1 = vld1q_u8(ref_ptr);
+ p1 = vld1q_u8(second_pred);
+ avg1 = vrhaddq_u8(r1, p1);
+ diff1 = vabdq_u8(s1, avg1);
+ sum[1] = vdotq_u32(sum[1], diff1, vdupq_n_u8(1));
+
+ src_ptr += src_stride;
+ ref_ptr += ref_stride;
+ second_pred += 16;
+ } while (--i != 0);
+
+ return horizontal_add_u32x4(vaddq_u32(sum[0], sum[1]));
+}
+
+#define SAD_WXH_AVG_NEON_DOTPROD(w, h) \
+ unsigned int aom_sad##w##x##h##_avg_neon_dotprod( \
+ const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
+ const uint8_t *second_pred) { \
+ return sad##w##xh_avg_neon_dotprod(src, src_stride, ref, ref_stride, (h), \
+ second_pred); \
+ }
+
+SAD_WXH_AVG_NEON_DOTPROD(16, 8)
+SAD_WXH_AVG_NEON_DOTPROD(16, 16)
+SAD_WXH_AVG_NEON_DOTPROD(16, 32)
+
+SAD_WXH_AVG_NEON_DOTPROD(32, 16)
+SAD_WXH_AVG_NEON_DOTPROD(32, 32)
+SAD_WXH_AVG_NEON_DOTPROD(32, 64)
+
+SAD_WXH_AVG_NEON_DOTPROD(64, 32)
+SAD_WXH_AVG_NEON_DOTPROD(64, 64)
+SAD_WXH_AVG_NEON_DOTPROD(64, 128)
+
+SAD_WXH_AVG_NEON_DOTPROD(128, 64)
+SAD_WXH_AVG_NEON_DOTPROD(128, 128)
+
+#if !CONFIG_REALTIME_ONLY
+SAD_WXH_AVG_NEON_DOTPROD(16, 4)
+SAD_WXH_AVG_NEON_DOTPROD(16, 64)
+SAD_WXH_AVG_NEON_DOTPROD(32, 8)
+SAD_WXH_AVG_NEON_DOTPROD(64, 16)
+#endif // !CONFIG_REALTIME_ONLY
+
+#undef SAD_WXH_AVG_NEON_DOTPROD
+
+static INLINE unsigned int dist_wtd_sad128xh_avg_neon_dotprod(
+ const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr,
+ int ref_stride, int h, const uint8_t *second_pred,
+ const DIST_WTD_COMP_PARAMS *jcp_param) {
+ const uint8x16_t fwd_offset = vdupq_n_u8(jcp_param->fwd_offset);
+ const uint8x16_t bck_offset = vdupq_n_u8(jcp_param->bck_offset);
+ // We use 8 accumulators to minimize the accumulation and loop carried
+ // dependencies for better instruction throughput.
+ uint32x4_t sum[8] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
+ vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
+ vdupq_n_u32(0), vdupq_n_u32(0) };
+
+ do {
+ uint8x16_t s0 = vld1q_u8(src_ptr);
+ uint8x16_t r0 = vld1q_u8(ref_ptr);
+ uint8x16_t p0 = vld1q_u8(second_pred);
+ uint8x16_t wtd_avg0 = dist_wtd_avg_u8x16(p0, r0, bck_offset, fwd_offset);
+ uint8x16_t diff0 = vabdq_u8(s0, wtd_avg0);
+ sum[0] = vdotq_u32(sum[0], diff0, vdupq_n_u8(1));
+
+ uint8x16_t s1 = vld1q_u8(src_ptr + 16);
+ uint8x16_t r1 = vld1q_u8(ref_ptr + 16);
+ uint8x16_t p1 = vld1q_u8(second_pred + 16);
+ uint8x16_t wtd_avg1 = dist_wtd_avg_u8x16(p1, r1, bck_offset, fwd_offset);
+ uint8x16_t diff1 = vabdq_u8(s1, wtd_avg1);
+ sum[1] = vdotq_u32(sum[1], diff1, vdupq_n_u8(1));
+
+ uint8x16_t s2 = vld1q_u8(src_ptr + 32);
+ uint8x16_t r2 = vld1q_u8(ref_ptr + 32);
+ uint8x16_t p2 = vld1q_u8(second_pred + 32);
+ uint8x16_t wtd_avg2 = dist_wtd_avg_u8x16(p2, r2, bck_offset, fwd_offset);
+ uint8x16_t diff2 = vabdq_u8(s2, wtd_avg2);
+ sum[2] = vdotq_u32(sum[2], diff2, vdupq_n_u8(1));
+
+ uint8x16_t s3 = vld1q_u8(src_ptr + 48);
+ uint8x16_t r3 = vld1q_u8(ref_ptr + 48);
+ uint8x16_t p3 = vld1q_u8(second_pred + 48);
+ uint8x16_t wtd_avg3 = dist_wtd_avg_u8x16(p3, r3, bck_offset, fwd_offset);
+ uint8x16_t diff3 = vabdq_u8(s3, wtd_avg3);
+ sum[3] = vdotq_u32(sum[3], diff3, vdupq_n_u8(1));
+
+ uint8x16_t s4 = vld1q_u8(src_ptr + 64);
+ uint8x16_t r4 = vld1q_u8(ref_ptr + 64);
+ uint8x16_t p4 = vld1q_u8(second_pred + 64);
+ uint8x16_t wtd_avg4 = dist_wtd_avg_u8x16(p4, r4, bck_offset, fwd_offset);
+ uint8x16_t diff4 = vabdq_u8(s4, wtd_avg4);
+ sum[4] = vdotq_u32(sum[4], diff4, vdupq_n_u8(1));
+
+ uint8x16_t s5 = vld1q_u8(src_ptr + 80);
+ uint8x16_t r5 = vld1q_u8(ref_ptr + 80);
+ uint8x16_t p5 = vld1q_u8(second_pred + 80);
+ uint8x16_t wtd_avg5 = dist_wtd_avg_u8x16(p5, r5, bck_offset, fwd_offset);
+ uint8x16_t diff5 = vabdq_u8(s5, wtd_avg5);
+ sum[5] = vdotq_u32(sum[5], diff5, vdupq_n_u8(1));
+
+ uint8x16_t s6 = vld1q_u8(src_ptr + 96);
+ uint8x16_t r6 = vld1q_u8(ref_ptr + 96);
+ uint8x16_t p6 = vld1q_u8(second_pred + 96);
+ uint8x16_t wtd_avg6 = dist_wtd_avg_u8x16(p6, r6, bck_offset, fwd_offset);
+ uint8x16_t diff6 = vabdq_u8(s6, wtd_avg6);
+ sum[6] = vdotq_u32(sum[6], diff6, vdupq_n_u8(1));
+
+ uint8x16_t s7 = vld1q_u8(src_ptr + 112);
+ uint8x16_t r7 = vld1q_u8(ref_ptr + 112);
+ uint8x16_t p7 = vld1q_u8(second_pred + 112);
+ uint8x16_t wtd_avg7 = dist_wtd_avg_u8x16(p7, r7, bck_offset, fwd_offset);
+ uint8x16_t diff7 = vabdq_u8(s7, wtd_avg7);
+ sum[7] = vdotq_u32(sum[7], diff7, vdupq_n_u8(1));
+
+ src_ptr += src_stride;
+ ref_ptr += ref_stride;
+ second_pred += 128;
+ } while (--h != 0);
+
+ sum[0] = vaddq_u32(sum[0], sum[1]);
+ sum[2] = vaddq_u32(sum[2], sum[3]);
+ sum[4] = vaddq_u32(sum[4], sum[5]);
+ sum[6] = vaddq_u32(sum[6], sum[7]);
+ sum[0] = vaddq_u32(sum[0], sum[2]);
+ sum[4] = vaddq_u32(sum[4], sum[6]);
+ sum[0] = vaddq_u32(sum[0], sum[4]);
+ return horizontal_add_u32x4(sum[0]);
+}
+
+static INLINE unsigned int dist_wtd_sad64xh_avg_neon_dotprod(
+ const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr,
+ int ref_stride, int h, const uint8_t *second_pred,
+ const DIST_WTD_COMP_PARAMS *jcp_param) {
+ const uint8x16_t fwd_offset = vdupq_n_u8(jcp_param->fwd_offset);
+ const uint8x16_t bck_offset = vdupq_n_u8(jcp_param->bck_offset);
+ uint32x4_t sum[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
+ vdupq_n_u32(0) };
+
+ do {
+ uint8x16_t s0 = vld1q_u8(src_ptr);
+ uint8x16_t r0 = vld1q_u8(ref_ptr);
+ uint8x16_t p0 = vld1q_u8(second_pred);
+ uint8x16_t wtd_avg0 = dist_wtd_avg_u8x16(p0, r0, bck_offset, fwd_offset);
+ uint8x16_t diff0 = vabdq_u8(s0, wtd_avg0);
+ sum[0] = vdotq_u32(sum[0], diff0, vdupq_n_u8(1));
+
+ uint8x16_t s1 = vld1q_u8(src_ptr + 16);
+ uint8x16_t r1 = vld1q_u8(ref_ptr + 16);
+ uint8x16_t p1 = vld1q_u8(second_pred + 16);
+ uint8x16_t wtd_avg1 = dist_wtd_avg_u8x16(p1, r1, bck_offset, fwd_offset);
+ uint8x16_t diff1 = vabdq_u8(s1, wtd_avg1);
+ sum[1] = vdotq_u32(sum[1], diff1, vdupq_n_u8(1));
+
+ uint8x16_t s2 = vld1q_u8(src_ptr + 32);
+ uint8x16_t r2 = vld1q_u8(ref_ptr + 32);
+ uint8x16_t p2 = vld1q_u8(second_pred + 32);
+ uint8x16_t wtd_avg2 = dist_wtd_avg_u8x16(p2, r2, bck_offset, fwd_offset);
+ uint8x16_t diff2 = vabdq_u8(s2, wtd_avg2);
+ sum[2] = vdotq_u32(sum[2], diff2, vdupq_n_u8(1));
+
+ uint8x16_t s3 = vld1q_u8(src_ptr + 48);
+ uint8x16_t r3 = vld1q_u8(ref_ptr + 48);
+ uint8x16_t p3 = vld1q_u8(second_pred + 48);
+ uint8x16_t wtd_avg3 = dist_wtd_avg_u8x16(p3, r3, bck_offset, fwd_offset);
+ uint8x16_t diff3 = vabdq_u8(s3, wtd_avg3);
+ sum[3] = vdotq_u32(sum[3], diff3, vdupq_n_u8(1));
+
+ src_ptr += src_stride;
+ ref_ptr += ref_stride;
+ second_pred += 64;
+ } while (--h != 0);
+
+ sum[0] = vaddq_u32(sum[0], sum[1]);
+ sum[2] = vaddq_u32(sum[2], sum[3]);
+ sum[0] = vaddq_u32(sum[0], sum[2]);
+ return horizontal_add_u32x4(sum[0]);
+}
+
+static INLINE unsigned int dist_wtd_sad32xh_avg_neon_dotprod(
+ const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr,
+ int ref_stride, int h, const uint8_t *second_pred,
+ const DIST_WTD_COMP_PARAMS *jcp_param) {
+ const uint8x16_t fwd_offset = vdupq_n_u8(jcp_param->fwd_offset);
+ const uint8x16_t bck_offset = vdupq_n_u8(jcp_param->bck_offset);
+ uint32x4_t sum[2] = { vdupq_n_u32(0), vdupq_n_u32(0) };
+
+ do {
+ uint8x16_t s0 = vld1q_u8(src_ptr);
+ uint8x16_t r0 = vld1q_u8(ref_ptr);
+ uint8x16_t p0 = vld1q_u8(second_pred);
+ uint8x16_t wtd_avg0 = dist_wtd_avg_u8x16(p0, r0, bck_offset, fwd_offset);
+ uint8x16_t diff0 = vabdq_u8(s0, wtd_avg0);
+ sum[0] = vdotq_u32(sum[0], diff0, vdupq_n_u8(1));
+
+ uint8x16_t s1 = vld1q_u8(src_ptr + 16);
+ uint8x16_t r1 = vld1q_u8(ref_ptr + 16);
+ uint8x16_t p1 = vld1q_u8(second_pred + 16);
+ uint8x16_t wtd_avg1 = dist_wtd_avg_u8x16(p1, r1, bck_offset, fwd_offset);
+ uint8x16_t diff1 = vabdq_u8(s1, wtd_avg1);
+ sum[1] = vdotq_u32(sum[1], diff1, vdupq_n_u8(1));
+
+ src_ptr += src_stride;
+ ref_ptr += ref_stride;
+ second_pred += 32;
+ } while (--h != 0);
+
+ sum[0] = vaddq_u32(sum[0], sum[1]);
+ return horizontal_add_u32x4(sum[0]);
+}
+
+static INLINE unsigned int dist_wtd_sad16xh_avg_neon_dotprod(
+ const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr,
+ int ref_stride, int h, const uint8_t *second_pred,
+ const DIST_WTD_COMP_PARAMS *jcp_param) {
+ const uint8x16_t fwd_offset = vdupq_n_u8(jcp_param->fwd_offset);
+ const uint8x16_t bck_offset = vdupq_n_u8(jcp_param->bck_offset);
+ uint32x4_t sum[2] = { vdupq_n_u32(0), vdupq_n_u32(0) };
+
+ int i = h / 2;
+ do {
+ uint8x16_t s0 = vld1q_u8(src_ptr);
+ uint8x16_t r0 = vld1q_u8(ref_ptr);
+ uint8x16_t p0 = vld1q_u8(second_pred);
+ uint8x16_t wtd_avg0 = dist_wtd_avg_u8x16(p0, r0, bck_offset, fwd_offset);
+ uint8x16_t diff0 = vabdq_u8(s0, wtd_avg0);
+ sum[0] = vdotq_u32(sum[0], diff0, vdupq_n_u8(1));
+
+ src_ptr += src_stride;
+ ref_ptr += ref_stride;
+ second_pred += 16;
+
+ uint8x16_t s1 = vld1q_u8(src_ptr);
+ uint8x16_t r1 = vld1q_u8(ref_ptr);
+ uint8x16_t p1 = vld1q_u8(second_pred);
+ uint8x16_t wtd_avg1 = dist_wtd_avg_u8x16(p1, r1, bck_offset, fwd_offset);
+ uint8x16_t diff1 = vabdq_u8(s1, wtd_avg1);
+ sum[1] = vdotq_u32(sum[1], diff1, vdupq_n_u8(1));
+
+ src_ptr += src_stride;
+ ref_ptr += ref_stride;
+ second_pred += 16;
+ } while (--i != 0);
+
+ sum[0] = vaddq_u32(sum[0], sum[1]);
+ return horizontal_add_u32x4(sum[0]);
+}
+
+#define DIST_WTD_SAD_WXH_AVG_NEON_DOTPROD(w, h) \
+ unsigned int aom_dist_wtd_sad##w##x##h##_avg_neon_dotprod( \
+ const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
+ const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \
+ return dist_wtd_sad##w##xh_avg_neon_dotprod( \
+ src, src_stride, ref, ref_stride, (h), second_pred, jcp_param); \
+ }
+
+DIST_WTD_SAD_WXH_AVG_NEON_DOTPROD(16, 8)
+DIST_WTD_SAD_WXH_AVG_NEON_DOTPROD(16, 16)
+DIST_WTD_SAD_WXH_AVG_NEON_DOTPROD(16, 32)
+
+DIST_WTD_SAD_WXH_AVG_NEON_DOTPROD(32, 16)
+DIST_WTD_SAD_WXH_AVG_NEON_DOTPROD(32, 32)
+DIST_WTD_SAD_WXH_AVG_NEON_DOTPROD(32, 64)
+
+DIST_WTD_SAD_WXH_AVG_NEON_DOTPROD(64, 32)
+DIST_WTD_SAD_WXH_AVG_NEON_DOTPROD(64, 64)
+DIST_WTD_SAD_WXH_AVG_NEON_DOTPROD(64, 128)
+
+DIST_WTD_SAD_WXH_AVG_NEON_DOTPROD(128, 64)
+DIST_WTD_SAD_WXH_AVG_NEON_DOTPROD(128, 128)
+
+#if !CONFIG_REALTIME_ONLY
+DIST_WTD_SAD_WXH_AVG_NEON_DOTPROD(16, 4)
+DIST_WTD_SAD_WXH_AVG_NEON_DOTPROD(16, 64)
+DIST_WTD_SAD_WXH_AVG_NEON_DOTPROD(32, 8)
+DIST_WTD_SAD_WXH_AVG_NEON_DOTPROD(64, 16)
+#endif // !CONFIG_REALTIME_ONLY
+
+#undef DIST_WTD_SAD_WXH_AVG_NEON_DOTPROD
diff --git a/aom_dsp/arm/sadxd_neon.c b/aom_dsp/arm/sadxd_neon.c
index 81803b1..e89e1c5 100644
--- a/aom_dsp/arm/sadxd_neon.c
+++ b/aom_dsp/arm/sadxd_neon.c
@@ -18,90 +18,6 @@
#include "aom_dsp/arm/mem_neon.h"
#include "aom_dsp/arm/sum_neon.h"
-#if defined(__ARM_FEATURE_DOTPROD)
-
-static INLINE void sad16_neon(uint8x16_t src, uint8x16_t ref,
- uint32x4_t *const sad_sum) {
- uint8x16_t abs_diff = vabdq_u8(src, ref);
- *sad_sum = vdotq_u32(*sad_sum, abs_diff, vdupq_n_u8(1));
-}
-
-static INLINE void sadwxhx3d_large_neon(const uint8_t *src, int src_stride,
- const uint8_t *const ref[4],
- int ref_stride, uint32_t res[4], int w,
- int h) {
- uint32x4_t sum_lo[3] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0) };
- uint32x4_t sum_hi[3] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0) };
-
- int ref_offset = 0;
- int i = h;
- do {
- int j = 0;
- do {
- const uint8x16_t s0 = vld1q_u8(src + j);
- sad16_neon(s0, vld1q_u8(ref[0] + ref_offset + j), &sum_lo[0]);
- sad16_neon(s0, vld1q_u8(ref[1] + ref_offset + j), &sum_lo[1]);
- sad16_neon(s0, vld1q_u8(ref[2] + ref_offset + j), &sum_lo[2]);
-
- const uint8x16_t s1 = vld1q_u8(src + j + 16);
- sad16_neon(s1, vld1q_u8(ref[0] + ref_offset + j + 16), &sum_hi[0]);
- sad16_neon(s1, vld1q_u8(ref[1] + ref_offset + j + 16), &sum_hi[1]);
- sad16_neon(s1, vld1q_u8(ref[2] + ref_offset + j + 16), &sum_hi[2]);
-
- j += 32;
- } while (j < w);
-
- src += src_stride;
- ref_offset += ref_stride;
- } while (--i != 0);
-
- res[0] = horizontal_add_u32x4(vaddq_u32(sum_lo[0], sum_hi[0]));
- res[1] = horizontal_add_u32x4(vaddq_u32(sum_lo[1], sum_hi[1]));
- res[2] = horizontal_add_u32x4(vaddq_u32(sum_lo[2], sum_hi[2]));
-}
-
-static INLINE void sad128xhx3d_neon(const uint8_t *src, int src_stride,
- const uint8_t *const ref[4], int ref_stride,
- uint32_t res[4], int h) {
- sadwxhx3d_large_neon(src, src_stride, ref, ref_stride, res, 128, h);
-}
-
-static INLINE void sad64xhx3d_neon(const uint8_t *src, int src_stride,
- const uint8_t *const ref[4], int ref_stride,
- uint32_t res[4], int h) {
- sadwxhx3d_large_neon(src, src_stride, ref, ref_stride, res, 64, h);
-}
-
-static INLINE void sad32xhx3d_neon(const uint8_t *src, int src_stride,
- const uint8_t *const ref[4], int ref_stride,
- uint32_t res[4], int h) {
- sadwxhx3d_large_neon(src, src_stride, ref, ref_stride, res, 32, h);
-}
-
-static INLINE void sad16xhx3d_neon(const uint8_t *src, int src_stride,
- const uint8_t *const ref[4], int ref_stride,
- uint32_t res[4], int h) {
- uint32x4_t sum[3] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0) };
-
- int ref_offset = 0;
- int i = h;
- do {
- const uint8x16_t s = vld1q_u8(src);
- sad16_neon(s, vld1q_u8(ref[0] + ref_offset), &sum[0]);
- sad16_neon(s, vld1q_u8(ref[1] + ref_offset), &sum[1]);
- sad16_neon(s, vld1q_u8(ref[2] + ref_offset), &sum[2]);
-
- src += src_stride;
- ref_offset += ref_stride;
- } while (--i != 0);
-
- res[0] = horizontal_add_u32x4(sum[0]);
- res[1] = horizontal_add_u32x4(sum[1]);
- res[2] = horizontal_add_u32x4(sum[2]);
-}
-
-#else // !(defined(__ARM_FEATURE_DOTPROD))
-
static INLINE void sad16_neon(uint8x16_t src, uint8x16_t ref,
uint16x8_t *const sad_sum) {
uint8x16_t abs_diff = vabdq_u8(src, ref);
@@ -218,8 +134,6 @@
res[2] = horizontal_add_u16x8(sum[2]);
}
-#endif // defined(__ARM_FEATURE_DOTPROD)
-
static INLINE void sad8xhx3d_neon(const uint8_t *src, int src_stride,
const uint8_t *const ref[3], int ref_stride,
uint32_t res[3], int h) {
@@ -325,92 +239,6 @@
#undef SAD_WXH_3D_NEON
-#if defined(__ARM_FEATURE_DOTPROD)
-
-static INLINE void sadwxhx4d_large_neon(const uint8_t *src, int src_stride,
- const uint8_t *const ref[4],
- int ref_stride, uint32_t res[4], int w,
- int h) {
- uint32x4_t sum_lo[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
- vdupq_n_u32(0) };
- uint32x4_t sum_hi[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
- vdupq_n_u32(0) };
- uint32x4_t sum[4];
-
- int ref_offset = 0;
- int i = h;
- do {
- int j = 0;
- do {
- const uint8x16_t s0 = vld1q_u8(src + j);
- sad16_neon(s0, vld1q_u8(ref[0] + ref_offset + j), &sum_lo[0]);
- sad16_neon(s0, vld1q_u8(ref[1] + ref_offset + j), &sum_lo[1]);
- sad16_neon(s0, vld1q_u8(ref[2] + ref_offset + j), &sum_lo[2]);
- sad16_neon(s0, vld1q_u8(ref[3] + ref_offset + j), &sum_lo[3]);
-
- const uint8x16_t s1 = vld1q_u8(src + j + 16);
- sad16_neon(s1, vld1q_u8(ref[0] + ref_offset + j + 16), &sum_hi[0]);
- sad16_neon(s1, vld1q_u8(ref[1] + ref_offset + j + 16), &sum_hi[1]);
- sad16_neon(s1, vld1q_u8(ref[2] + ref_offset + j + 16), &sum_hi[2]);
- sad16_neon(s1, vld1q_u8(ref[3] + ref_offset + j + 16), &sum_hi[3]);
-
- j += 32;
- } while (j < w);
-
- src += src_stride;
- ref_offset += ref_stride;
- } while (--i != 0);
-
- sum[0] = vaddq_u32(sum_lo[0], sum_hi[0]);
- sum[1] = vaddq_u32(sum_lo[1], sum_hi[1]);
- sum[2] = vaddq_u32(sum_lo[2], sum_hi[2]);
- sum[3] = vaddq_u32(sum_lo[3], sum_hi[3]);
-
- vst1q_u32(res, horizontal_add_4d_u32x4(sum));
-}
-
-static INLINE void sad128xhx4d_neon(const uint8_t *src, int src_stride,
- const uint8_t *const ref[4], int ref_stride,
- uint32_t res[4], int h) {
- sadwxhx4d_large_neon(src, src_stride, ref, ref_stride, res, 128, h);
-}
-
-static INLINE void sad64xhx4d_neon(const uint8_t *src, int src_stride,
- const uint8_t *const ref[4], int ref_stride,
- uint32_t res[4], int h) {
- sadwxhx4d_large_neon(src, src_stride, ref, ref_stride, res, 64, h);
-}
-
-static INLINE void sad32xhx4d_neon(const uint8_t *src, int src_stride,
- const uint8_t *const ref[4], int ref_stride,
- uint32_t res[4], int h) {
- sadwxhx4d_large_neon(src, src_stride, ref, ref_stride, res, 32, h);
-}
-
-static INLINE void sad16xhx4d_neon(const uint8_t *src, int src_stride,
- const uint8_t *const ref[4], int ref_stride,
- uint32_t res[4], int h) {
- uint32x4_t sum[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
- vdupq_n_u32(0) };
-
- int ref_offset = 0;
- int i = h;
- do {
- const uint8x16_t s = vld1q_u8(src);
- sad16_neon(s, vld1q_u8(ref[0] + ref_offset), &sum[0]);
- sad16_neon(s, vld1q_u8(ref[1] + ref_offset), &sum[1]);
- sad16_neon(s, vld1q_u8(ref[2] + ref_offset), &sum[2]);
- sad16_neon(s, vld1q_u8(ref[3] + ref_offset), &sum[3]);
-
- src += src_stride;
- ref_offset += ref_stride;
- } while (--i != 0);
-
- vst1q_u32(res, horizontal_add_4d_u32x4(sum));
-}
-
-#else // !(defined(__ARM_FEATURE_DOTPROD))
-
static INLINE void sadwxhx4d_large_neon(const uint8_t *src, int src_stride,
const uint8_t *const ref[4],
int ref_stride, uint32_t res[4], int w,
@@ -534,8 +362,6 @@
vst1q_u32(res, horizontal_add_4d_u32x4(sum_u32));
}
-#endif // defined(__ARM_FEATURE_DOTPROD)
-
static INLINE void sad8xhx4d_neon(const uint8_t *src, int src_stride,
const uint8_t *const ref[4], int ref_stride,
uint32_t res[4], int h) {
diff --git a/aom_dsp/arm/sadxd_neon_dotprod.c b/aom_dsp/arm/sadxd_neon_dotprod.c
new file mode 100644
index 0000000..3d11d1c
--- /dev/null
+++ b/aom_dsp/arm/sadxd_neon_dotprod.c
@@ -0,0 +1,289 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+
+#include "config/aom_config.h"
+#include "config/aom_dsp_rtcd.h"
+
+#include "aom/aom_integer.h"
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_dsp/arm/sum_neon.h"
+
+static INLINE void sad16_neon(uint8x16_t src, uint8x16_t ref,
+ uint32x4_t *const sad_sum) {
+ uint8x16_t abs_diff = vabdq_u8(src, ref);
+ *sad_sum = vdotq_u32(*sad_sum, abs_diff, vdupq_n_u8(1));
+}
+
+static INLINE void sadwxhx3d_large_neon_dotprod(const uint8_t *src,
+ int src_stride,
+ const uint8_t *const ref[4],
+ int ref_stride, uint32_t res[4],
+ int w, int h) {
+ uint32x4_t sum_lo[3] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0) };
+ uint32x4_t sum_hi[3] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0) };
+
+ int ref_offset = 0;
+ int i = h;
+ do {
+ int j = 0;
+ do {
+ const uint8x16_t s0 = vld1q_u8(src + j);
+ sad16_neon(s0, vld1q_u8(ref[0] + ref_offset + j), &sum_lo[0]);
+ sad16_neon(s0, vld1q_u8(ref[1] + ref_offset + j), &sum_lo[1]);
+ sad16_neon(s0, vld1q_u8(ref[2] + ref_offset + j), &sum_lo[2]);
+
+ const uint8x16_t s1 = vld1q_u8(src + j + 16);
+ sad16_neon(s1, vld1q_u8(ref[0] + ref_offset + j + 16), &sum_hi[0]);
+ sad16_neon(s1, vld1q_u8(ref[1] + ref_offset + j + 16), &sum_hi[1]);
+ sad16_neon(s1, vld1q_u8(ref[2] + ref_offset + j + 16), &sum_hi[2]);
+
+ j += 32;
+ } while (j < w);
+
+ src += src_stride;
+ ref_offset += ref_stride;
+ } while (--i != 0);
+
+ res[0] = horizontal_add_u32x4(vaddq_u32(sum_lo[0], sum_hi[0]));
+ res[1] = horizontal_add_u32x4(vaddq_u32(sum_lo[1], sum_hi[1]));
+ res[2] = horizontal_add_u32x4(vaddq_u32(sum_lo[2], sum_hi[2]));
+}
+
+static INLINE void sad128xhx3d_neon_dotprod(const uint8_t *src, int src_stride,
+ const uint8_t *const ref[4],
+ int ref_stride, uint32_t res[4],
+ int h) {
+ sadwxhx3d_large_neon_dotprod(src, src_stride, ref, ref_stride, res, 128, h);
+}
+
+static INLINE void sad64xhx3d_neon_dotprod(const uint8_t *src, int src_stride,
+ const uint8_t *const ref[4],
+ int ref_stride, uint32_t res[4],
+ int h) {
+ sadwxhx3d_large_neon_dotprod(src, src_stride, ref, ref_stride, res, 64, h);
+}
+
+static INLINE void sad32xhx3d_neon_dotprod(const uint8_t *src, int src_stride,
+ const uint8_t *const ref[4],
+ int ref_stride, uint32_t res[4],
+ int h) {
+ sadwxhx3d_large_neon_dotprod(src, src_stride, ref, ref_stride, res, 32, h);
+}
+
+static INLINE void sad16xhx3d_neon_dotprod(const uint8_t *src, int src_stride,
+ const uint8_t *const ref[4],
+ int ref_stride, uint32_t res[4],
+ int h) {
+ uint32x4_t sum[3] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0) };
+
+ int ref_offset = 0;
+ int i = h;
+ do {
+ const uint8x16_t s = vld1q_u8(src);
+ sad16_neon(s, vld1q_u8(ref[0] + ref_offset), &sum[0]);
+ sad16_neon(s, vld1q_u8(ref[1] + ref_offset), &sum[1]);
+ sad16_neon(s, vld1q_u8(ref[2] + ref_offset), &sum[2]);
+
+ src += src_stride;
+ ref_offset += ref_stride;
+ } while (--i != 0);
+
+ res[0] = horizontal_add_u32x4(sum[0]);
+ res[1] = horizontal_add_u32x4(sum[1]);
+ res[2] = horizontal_add_u32x4(sum[2]);
+}
+
+#define SAD_WXH_3D_NEON_DOTPROD(w, h) \
+ void aom_sad##w##x##h##x3d_neon_dotprod(const uint8_t *src, int src_stride, \
+ const uint8_t *const ref[4], \
+ int ref_stride, uint32_t res[4]) { \
+ sad##w##xhx3d_neon_dotprod(src, src_stride, ref, ref_stride, res, (h)); \
+ }
+
+SAD_WXH_3D_NEON_DOTPROD(16, 8)
+SAD_WXH_3D_NEON_DOTPROD(16, 16)
+SAD_WXH_3D_NEON_DOTPROD(16, 32)
+
+SAD_WXH_3D_NEON_DOTPROD(32, 16)
+SAD_WXH_3D_NEON_DOTPROD(32, 32)
+SAD_WXH_3D_NEON_DOTPROD(32, 64)
+
+SAD_WXH_3D_NEON_DOTPROD(64, 32)
+SAD_WXH_3D_NEON_DOTPROD(64, 64)
+SAD_WXH_3D_NEON_DOTPROD(64, 128)
+
+SAD_WXH_3D_NEON_DOTPROD(128, 64)
+SAD_WXH_3D_NEON_DOTPROD(128, 128)
+
+#if !CONFIG_REALTIME_ONLY
+SAD_WXH_3D_NEON_DOTPROD(16, 4)
+SAD_WXH_3D_NEON_DOTPROD(16, 64)
+SAD_WXH_3D_NEON_DOTPROD(32, 8)
+SAD_WXH_3D_NEON_DOTPROD(64, 16)
+#endif // !CONFIG_REALTIME_ONLY
+
+#undef SAD_WXH_3D_NEON_DOTPROD
+
+static INLINE void sadwxhx4d_large_neon_dotprod(const uint8_t *src,
+ int src_stride,
+ const uint8_t *const ref[4],
+ int ref_stride, uint32_t res[4],
+ int w, int h) {
+ uint32x4_t sum_lo[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
+ vdupq_n_u32(0) };
+ uint32x4_t sum_hi[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
+ vdupq_n_u32(0) };
+ uint32x4_t sum[4];
+
+ int ref_offset = 0;
+ int i = h;
+ do {
+ int j = 0;
+ do {
+ const uint8x16_t s0 = vld1q_u8(src + j);
+ sad16_neon(s0, vld1q_u8(ref[0] + ref_offset + j), &sum_lo[0]);
+ sad16_neon(s0, vld1q_u8(ref[1] + ref_offset + j), &sum_lo[1]);
+ sad16_neon(s0, vld1q_u8(ref[2] + ref_offset + j), &sum_lo[2]);
+ sad16_neon(s0, vld1q_u8(ref[3] + ref_offset + j), &sum_lo[3]);
+
+ const uint8x16_t s1 = vld1q_u8(src + j + 16);
+ sad16_neon(s1, vld1q_u8(ref[0] + ref_offset + j + 16), &sum_hi[0]);
+ sad16_neon(s1, vld1q_u8(ref[1] + ref_offset + j + 16), &sum_hi[1]);
+ sad16_neon(s1, vld1q_u8(ref[2] + ref_offset + j + 16), &sum_hi[2]);
+ sad16_neon(s1, vld1q_u8(ref[3] + ref_offset + j + 16), &sum_hi[3]);
+
+ j += 32;
+ } while (j < w);
+
+ src += src_stride;
+ ref_offset += ref_stride;
+ } while (--i != 0);
+
+ sum[0] = vaddq_u32(sum_lo[0], sum_hi[0]);
+ sum[1] = vaddq_u32(sum_lo[1], sum_hi[1]);
+ sum[2] = vaddq_u32(sum_lo[2], sum_hi[2]);
+ sum[3] = vaddq_u32(sum_lo[3], sum_hi[3]);
+
+ vst1q_u32(res, horizontal_add_4d_u32x4(sum));
+}
+
+static INLINE void sad128xhx4d_neon_dotprod(const uint8_t *src, int src_stride,
+ const uint8_t *const ref[4],
+ int ref_stride, uint32_t res[4],
+ int h) {
+ sadwxhx4d_large_neon_dotprod(src, src_stride, ref, ref_stride, res, 128, h);
+}
+
+static INLINE void sad64xhx4d_neon_dotprod(const uint8_t *src, int src_stride,
+ const uint8_t *const ref[4],
+ int ref_stride, uint32_t res[4],
+ int h) {
+ sadwxhx4d_large_neon_dotprod(src, src_stride, ref, ref_stride, res, 64, h);
+}
+
+static INLINE void sad32xhx4d_neon_dotprod(const uint8_t *src, int src_stride,
+ const uint8_t *const ref[4],
+ int ref_stride, uint32_t res[4],
+ int h) {
+ sadwxhx4d_large_neon_dotprod(src, src_stride, ref, ref_stride, res, 32, h);
+}
+
+static INLINE void sad16xhx4d_neon_dotprod(const uint8_t *src, int src_stride,
+ const uint8_t *const ref[4],
+ int ref_stride, uint32_t res[4],
+ int h) {
+ uint32x4_t sum[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0),
+ vdupq_n_u32(0) };
+
+ int ref_offset = 0;
+ int i = h;
+ do {
+ const uint8x16_t s = vld1q_u8(src);
+ sad16_neon(s, vld1q_u8(ref[0] + ref_offset), &sum[0]);
+ sad16_neon(s, vld1q_u8(ref[1] + ref_offset), &sum[1]);
+ sad16_neon(s, vld1q_u8(ref[2] + ref_offset), &sum[2]);
+ sad16_neon(s, vld1q_u8(ref[3] + ref_offset), &sum[3]);
+
+ src += src_stride;
+ ref_offset += ref_stride;
+ } while (--i != 0);
+
+ vst1q_u32(res, horizontal_add_4d_u32x4(sum));
+}
+
+#define SAD_WXH_4D_NEON_DOTPROD(w, h) \
+ void aom_sad##w##x##h##x4d_neon_dotprod(const uint8_t *src, int src_stride, \
+ const uint8_t *const ref[4], \
+ int ref_stride, uint32_t res[4]) { \
+ sad##w##xhx4d_neon_dotprod(src, src_stride, ref, ref_stride, res, (h)); \
+ }
+
+SAD_WXH_4D_NEON_DOTPROD(16, 8)
+SAD_WXH_4D_NEON_DOTPROD(16, 16)
+SAD_WXH_4D_NEON_DOTPROD(16, 32)
+
+SAD_WXH_4D_NEON_DOTPROD(32, 16)
+SAD_WXH_4D_NEON_DOTPROD(32, 32)
+SAD_WXH_4D_NEON_DOTPROD(32, 64)
+
+SAD_WXH_4D_NEON_DOTPROD(64, 32)
+SAD_WXH_4D_NEON_DOTPROD(64, 64)
+SAD_WXH_4D_NEON_DOTPROD(64, 128)
+
+SAD_WXH_4D_NEON_DOTPROD(128, 64)
+SAD_WXH_4D_NEON_DOTPROD(128, 128)
+
+#if !CONFIG_REALTIME_ONLY
+SAD_WXH_4D_NEON_DOTPROD(16, 4)
+SAD_WXH_4D_NEON_DOTPROD(16, 64)
+SAD_WXH_4D_NEON_DOTPROD(32, 8)
+SAD_WXH_4D_NEON_DOTPROD(64, 16)
+#endif // !CONFIG_REALTIME_ONLY
+
+#undef SAD_WXH_4D_NEON_DOTPROD
+
+#define SAD_SKIP_WXH_4D_NEON_DOTPROD(w, h) \
+ void aom_sad_skip_##w##x##h##x4d_neon_dotprod( \
+ const uint8_t *src, int src_stride, const uint8_t *const ref[4], \
+ int ref_stride, uint32_t res[4]) { \
+ sad##w##xhx4d_neon_dotprod(src, 2 * src_stride, ref, 2 * ref_stride, res, \
+ ((h) >> 1)); \
+ res[0] <<= 1; \
+ res[1] <<= 1; \
+ res[2] <<= 1; \
+ res[3] <<= 1; \
+ }
+
+SAD_SKIP_WXH_4D_NEON_DOTPROD(16, 8)
+SAD_SKIP_WXH_4D_NEON_DOTPROD(16, 16)
+SAD_SKIP_WXH_4D_NEON_DOTPROD(16, 32)
+
+SAD_SKIP_WXH_4D_NEON_DOTPROD(32, 16)
+SAD_SKIP_WXH_4D_NEON_DOTPROD(32, 32)
+SAD_SKIP_WXH_4D_NEON_DOTPROD(32, 64)
+
+SAD_SKIP_WXH_4D_NEON_DOTPROD(64, 32)
+SAD_SKIP_WXH_4D_NEON_DOTPROD(64, 64)
+SAD_SKIP_WXH_4D_NEON_DOTPROD(64, 128)
+
+SAD_SKIP_WXH_4D_NEON_DOTPROD(128, 64)
+SAD_SKIP_WXH_4D_NEON_DOTPROD(128, 128)
+
+#if !CONFIG_REALTIME_ONLY
+SAD_SKIP_WXH_4D_NEON_DOTPROD(16, 4)
+SAD_SKIP_WXH_4D_NEON_DOTPROD(16, 64)
+SAD_SKIP_WXH_4D_NEON_DOTPROD(32, 8)
+SAD_SKIP_WXH_4D_NEON_DOTPROD(64, 16)
+#endif // !CONFIG_REALTIME_ONLY
+
+#undef SAD_SKIP_WXH_4D_NEON_DOTPROD
diff --git a/aom_dsp/arm/sse_neon.c b/aom_dsp/arm/sse_neon.c
index d1d3d93..ec8f0ee 100644
--- a/aom_dsp/arm/sse_neon.c
+++ b/aom_dsp/arm/sse_neon.c
@@ -11,119 +11,8 @@
#include <arm_neon.h>
#include "config/aom_dsp_rtcd.h"
-#include "aom/aom_integer.h"
#include "aom_dsp/arm/mem_neon.h"
#include "aom_dsp/arm/sum_neon.h"
-#include "aom_dsp/arm/transpose_neon.h"
-
-#if defined(__ARM_FEATURE_DOTPROD)
-
-static INLINE void sse_16x1_neon(const uint8_t *src, const uint8_t *ref,
- uint32x4_t *sse) {
- uint8x16_t s = vld1q_u8(src);
- uint8x16_t r = vld1q_u8(ref);
-
- uint8x16_t abs_diff = vabdq_u8(s, r);
-
- *sse = vdotq_u32(*sse, abs_diff, abs_diff);
-}
-
-static INLINE void sse_8x1_neon(const uint8_t *src, const uint8_t *ref,
- uint32x2_t *sse) {
- uint8x8_t s = vld1_u8(src);
- uint8x8_t r = vld1_u8(ref);
-
- uint8x8_t abs_diff = vabd_u8(s, r);
-
- *sse = vdot_u32(*sse, abs_diff, abs_diff);
-}
-
-static INLINE void sse_4x2_neon(const uint8_t *src, int src_stride,
- const uint8_t *ref, int ref_stride,
- uint32x2_t *sse) {
- uint8x8_t s = load_unaligned_u8(src, src_stride);
- uint8x8_t r = load_unaligned_u8(ref, ref_stride);
-
- uint8x8_t abs_diff = vabd_u8(s, r);
-
- *sse = vdot_u32(*sse, abs_diff, abs_diff);
-}
-
-static INLINE uint32_t sse_8xh_neon(const uint8_t *src, int src_stride,
- const uint8_t *ref, int ref_stride,
- int height) {
- uint32x2_t sse[2] = { vdup_n_u32(0), vdup_n_u32(0) };
-
- int i = height;
- do {
- sse_8x1_neon(src, ref, &sse[0]);
- src += src_stride;
- ref += ref_stride;
- sse_8x1_neon(src, ref, &sse[1]);
- src += src_stride;
- ref += ref_stride;
- i -= 2;
- } while (i != 0);
-
- return horizontal_add_u32x4(vcombine_u32(sse[0], sse[1]));
-}
-
-static INLINE uint32_t sse_4xh_neon(const uint8_t *src, int src_stride,
- const uint8_t *ref, int ref_stride,
- int height) {
- uint32x2_t sse = vdup_n_u32(0);
-
- int i = height;
- do {
- sse_4x2_neon(src, src_stride, ref, ref_stride, &sse);
-
- src += 2 * src_stride;
- ref += 2 * ref_stride;
- i -= 2;
- } while (i != 0);
-
- return horizontal_add_u32x2(sse);
-}
-
-static INLINE uint32_t sse_wxh_neon(const uint8_t *src, int src_stride,
- const uint8_t *ref, int ref_stride,
- int width, int height) {
- uint32x2_t sse[2] = { vdup_n_u32(0), vdup_n_u32(0) };
-
- if ((width & 0x07) && ((width & 0x07) < 5)) {
- int i = height;
- do {
- int j = 0;
- do {
- sse_8x1_neon(src + j, ref + j, &sse[0]);
- sse_8x1_neon(src + j + src_stride, ref + j + ref_stride, &sse[1]);
- j += 8;
- } while (j + 4 < width);
-
- sse_4x2_neon(src + j, src_stride, ref + j, ref_stride, &sse[0]);
- src += 2 * src_stride;
- ref += 2 * ref_stride;
- i -= 2;
- } while (i != 0);
- } else {
- int i = height;
- do {
- int j = 0;
- do {
- sse_8x1_neon(src + j, ref + j, &sse[0]);
- sse_8x1_neon(src + j + src_stride, ref + j + ref_stride, &sse[1]);
- j += 8;
- } while (j < width);
-
- src += 2 * src_stride;
- ref += 2 * ref_stride;
- i -= 2;
- } while (i != 0);
- }
- return horizontal_add_u32x4(vcombine_u32(sse[0], sse[1]));
-}
-
-#else // !defined(__ARM_FEATURE_DOTPROD)
static INLINE void sse_16x1_neon(const uint8_t *src, const uint8_t *ref,
uint32x4_t *sse) {
@@ -159,39 +48,6 @@
*sse = vpadalq_u16(*sse, vmull_u8(abs_diff, abs_diff));
}
-static INLINE uint32_t sse_8xh_neon(const uint8_t *src, int src_stride,
- const uint8_t *ref, int ref_stride,
- int height) {
- uint32x4_t sse = vdupq_n_u32(0);
-
- int i = height;
- do {
- sse_8x1_neon(src, ref, &sse);
-
- src += src_stride;
- ref += ref_stride;
- } while (--i != 0);
-
- return horizontal_add_u32x4(sse);
-}
-
-static INLINE uint32_t sse_4xh_neon(const uint8_t *src, int src_stride,
- const uint8_t *ref, int ref_stride,
- int height) {
- uint32x4_t sse = vdupq_n_u32(0);
-
- int i = height;
- do {
- sse_4x2_neon(src, src_stride, ref, ref_stride, &sse);
-
- src += 2 * src_stride;
- ref += 2 * ref_stride;
- i -= 2;
- } while (i != 0);
-
- return horizontal_add_u32x4(sse);
-}
-
static INLINE uint32_t sse_wxh_neon(const uint8_t *src, int src_stride,
const uint8_t *ref, int ref_stride,
int width, int height) {
@@ -228,8 +84,6 @@
return horizontal_add_u32x4(sse);
}
-#endif // defined(__ARM_FEATURE_DOTPROD)
-
static INLINE uint32_t sse_128xh_neon(const uint8_t *src, int src_stride,
const uint8_t *ref, int ref_stride,
int height) {
@@ -308,6 +162,39 @@
return horizontal_add_u32x4(vaddq_u32(sse[0], sse[1]));
}
+static INLINE uint32_t sse_8xh_neon(const uint8_t *src, int src_stride,
+ const uint8_t *ref, int ref_stride,
+ int height) {
+ uint32x4_t sse = vdupq_n_u32(0);
+
+ int i = height;
+ do {
+ sse_8x1_neon(src, ref, &sse);
+
+ src += src_stride;
+ ref += ref_stride;
+ } while (--i != 0);
+
+ return horizontal_add_u32x4(sse);
+}
+
+static INLINE uint32_t sse_4xh_neon(const uint8_t *src, int src_stride,
+ const uint8_t *ref, int ref_stride,
+ int height) {
+ uint32x4_t sse = vdupq_n_u32(0);
+
+ int i = height;
+ do {
+ sse_4x2_neon(src, src_stride, ref, ref_stride, &sse);
+
+ src += 2 * src_stride;
+ ref += 2 * ref_stride;
+ i -= 2;
+ } while (i != 0);
+
+ return horizontal_add_u32x4(sse);
+}
+
int64_t aom_sse_neon(const uint8_t *src, int src_stride, const uint8_t *ref,
int ref_stride, int width, int height) {
switch (width) {
@@ -321,268 +208,3 @@
return sse_wxh_neon(src, src_stride, ref, ref_stride, width, height);
}
}
-
-#if CONFIG_AV1_HIGHBITDEPTH
-static INLINE uint32_t highbd_sse_W8x1_neon(uint16x8_t q2, uint16x8_t q3) {
- uint32_t sse;
- const uint32_t sse1 = 0;
- const uint32x4_t q1 = vld1q_dup_u32(&sse1);
-
- uint16x8_t q4 = vabdq_u16(q2, q3); // diff = abs(a[x] - b[x])
- uint16x4_t d0 = vget_low_u16(q4);
- uint16x4_t d1 = vget_high_u16(q4);
-
- uint32x4_t q6 = vmlal_u16(q1, d0, d0);
- uint32x4_t q7 = vmlal_u16(q1, d1, d1);
-
- uint32x2_t d4 = vadd_u32(vget_low_u32(q6), vget_high_u32(q6));
- uint32x2_t d5 = vadd_u32(vget_low_u32(q7), vget_high_u32(q7));
-
- uint32x2_t d6 = vadd_u32(d4, d5);
-
- sse = vget_lane_u32(d6, 0);
- sse += vget_lane_u32(d6, 1);
-
- return sse;
-}
-
-int64_t aom_highbd_sse_neon(const uint8_t *a8, int a_stride, const uint8_t *b8,
- int b_stride, int width, int height) {
- static const uint16_t k01234567[8] = { 0, 1, 2, 3, 4, 5, 6, 7 };
- const uint16x8_t q0 = vld1q_u16(k01234567);
- int64_t sse = 0;
- uint16_t *a = CONVERT_TO_SHORTPTR(a8);
- uint16_t *b = CONVERT_TO_SHORTPTR(b8);
- int x, y;
- int addinc;
- uint16x4_t d0, d1, d2, d3;
- uint16_t dx;
- uint16x8_t q2, q3, q4, q5;
-
- switch (width) {
- case 4:
- for (y = 0; y < height; y += 2) {
- d0 = vld1_u16(a); // load 4 data
- a += a_stride;
- d1 = vld1_u16(a);
- a += a_stride;
-
- d2 = vld1_u16(b);
- b += b_stride;
- d3 = vld1_u16(b);
- b += b_stride;
- q2 = vcombine_u16(d0, d1); // make a 8 data vector
- q3 = vcombine_u16(d2, d3);
-
- sse += highbd_sse_W8x1_neon(q2, q3);
- }
- break;
- case 8:
- for (y = 0; y < height; y++) {
- q2 = vld1q_u16(a);
- q3 = vld1q_u16(b);
-
- sse += highbd_sse_W8x1_neon(q2, q3);
-
- a += a_stride;
- b += b_stride;
- }
- break;
- case 16:
- for (y = 0; y < height; y++) {
- q2 = vld1q_u16(a);
- q3 = vld1q_u16(b);
-
- sse += highbd_sse_W8x1_neon(q2, q3);
-
- q2 = vld1q_u16(a + 8);
- q3 = vld1q_u16(b + 8);
-
- sse += highbd_sse_W8x1_neon(q2, q3);
-
- a += a_stride;
- b += b_stride;
- }
- break;
- case 32:
- for (y = 0; y < height; y++) {
- q2 = vld1q_u16(a);
- q3 = vld1q_u16(b);
-
- sse += highbd_sse_W8x1_neon(q2, q3);
-
- q2 = vld1q_u16(a + 8);
- q3 = vld1q_u16(b + 8);
-
- sse += highbd_sse_W8x1_neon(q2, q3);
-
- q2 = vld1q_u16(a + 16);
- q3 = vld1q_u16(b + 16);
-
- sse += highbd_sse_W8x1_neon(q2, q3);
-
- q2 = vld1q_u16(a + 24);
- q3 = vld1q_u16(b + 24);
-
- sse += highbd_sse_W8x1_neon(q2, q3);
-
- a += a_stride;
- b += b_stride;
- }
- break;
- case 64:
- for (y = 0; y < height; y++) {
- q2 = vld1q_u16(a);
- q3 = vld1q_u16(b);
-
- sse += highbd_sse_W8x1_neon(q2, q3);
-
- q2 = vld1q_u16(a + 8);
- q3 = vld1q_u16(b + 8);
-
- sse += highbd_sse_W8x1_neon(q2, q3);
-
- q2 = vld1q_u16(a + 16);
- q3 = vld1q_u16(b + 16);
-
- sse += highbd_sse_W8x1_neon(q2, q3);
-
- q2 = vld1q_u16(a + 24);
- q3 = vld1q_u16(b + 24);
-
- sse += highbd_sse_W8x1_neon(q2, q3);
-
- q2 = vld1q_u16(a + 32);
- q3 = vld1q_u16(b + 32);
-
- sse += highbd_sse_W8x1_neon(q2, q3);
-
- q2 = vld1q_u16(a + 40);
- q3 = vld1q_u16(b + 40);
-
- sse += highbd_sse_W8x1_neon(q2, q3);
-
- q2 = vld1q_u16(a + 48);
- q3 = vld1q_u16(b + 48);
-
- sse += highbd_sse_W8x1_neon(q2, q3);
-
- q2 = vld1q_u16(a + 56);
- q3 = vld1q_u16(b + 56);
-
- sse += highbd_sse_W8x1_neon(q2, q3);
-
- a += a_stride;
- b += b_stride;
- }
- break;
- case 128:
- for (y = 0; y < height; y++) {
- q2 = vld1q_u16(a);
- q3 = vld1q_u16(b);
-
- sse += highbd_sse_W8x1_neon(q2, q3);
-
- q2 = vld1q_u16(a + 8);
- q3 = vld1q_u16(b + 8);
-
- sse += highbd_sse_W8x1_neon(q2, q3);
-
- q2 = vld1q_u16(a + 16);
- q3 = vld1q_u16(b + 16);
-
- sse += highbd_sse_W8x1_neon(q2, q3);
-
- q2 = vld1q_u16(a + 24);
- q3 = vld1q_u16(b + 24);
-
- sse += highbd_sse_W8x1_neon(q2, q3);
-
- q2 = vld1q_u16(a + 32);
- q3 = vld1q_u16(b + 32);
-
- sse += highbd_sse_W8x1_neon(q2, q3);
-
- q2 = vld1q_u16(a + 40);
- q3 = vld1q_u16(b + 40);
-
- sse += highbd_sse_W8x1_neon(q2, q3);
-
- q2 = vld1q_u16(a + 48);
- q3 = vld1q_u16(b + 48);
-
- sse += highbd_sse_W8x1_neon(q2, q3);
-
- q2 = vld1q_u16(a + 56);
- q3 = vld1q_u16(b + 56);
-
- sse += highbd_sse_W8x1_neon(q2, q3);
-
- q2 = vld1q_u16(a + 64);
- q3 = vld1q_u16(b + 64);
-
- sse += highbd_sse_W8x1_neon(q2, q3);
-
- q2 = vld1q_u16(a + 72);
- q3 = vld1q_u16(b + 72);
-
- sse += highbd_sse_W8x1_neon(q2, q3);
-
- q2 = vld1q_u16(a + 80);
- q3 = vld1q_u16(b + 80);
-
- sse += highbd_sse_W8x1_neon(q2, q3);
-
- q2 = vld1q_u16(a + 88);
- q3 = vld1q_u16(b + 88);
-
- sse += highbd_sse_W8x1_neon(q2, q3);
-
- q2 = vld1q_u16(a + 96);
- q3 = vld1q_u16(b + 96);
-
- sse += highbd_sse_W8x1_neon(q2, q3);
-
- q2 = vld1q_u16(a + 104);
- q3 = vld1q_u16(b + 104);
-
- sse += highbd_sse_W8x1_neon(q2, q3);
-
- q2 = vld1q_u16(a + 112);
- q3 = vld1q_u16(b + 112);
-
- sse += highbd_sse_W8x1_neon(q2, q3);
-
- q2 = vld1q_u16(a + 120);
- q3 = vld1q_u16(b + 120);
-
- sse += highbd_sse_W8x1_neon(q2, q3);
- a += a_stride;
- b += b_stride;
- }
- break;
- default:
-
- for (y = 0; y < height; y++) {
- x = width;
- while (x > 0) {
- addinc = width - x;
- q2 = vld1q_u16(a + addinc);
- q3 = vld1q_u16(b + addinc);
- if (x < 8) {
- dx = x;
- q4 = vld1q_dup_u16(&dx);
- q5 = vcltq_u16(q0, q4);
- q2 = vandq_u16(q2, q5);
- q3 = vandq_u16(q3, q5);
- }
- sse += highbd_sse_W8x1_neon(q2, q3);
- x -= 8;
- }
- a += a_stride;
- b += b_stride;
- }
- }
- return (int64_t)sse;
-}
-#endif
diff --git a/aom_dsp/arm/sse_neon_dotprod.c b/aom_dsp/arm/sse_neon_dotprod.c
new file mode 100644
index 0000000..9790497
--- /dev/null
+++ b/aom_dsp/arm/sse_neon_dotprod.c
@@ -0,0 +1,223 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <arm_neon.h>
+
+#include "config/aom_dsp_rtcd.h"
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_dsp/arm/sum_neon.h"
+
+static INLINE void sse_16x1_neon_dotprod(const uint8_t *src, const uint8_t *ref,
+ uint32x4_t *sse) {
+ uint8x16_t s = vld1q_u8(src);
+ uint8x16_t r = vld1q_u8(ref);
+
+ uint8x16_t abs_diff = vabdq_u8(s, r);
+
+ *sse = vdotq_u32(*sse, abs_diff, abs_diff);
+}
+
+static INLINE void sse_8x1_neon_dotprod(const uint8_t *src, const uint8_t *ref,
+ uint32x2_t *sse) {
+ uint8x8_t s = vld1_u8(src);
+ uint8x8_t r = vld1_u8(ref);
+
+ uint8x8_t abs_diff = vabd_u8(s, r);
+
+ *sse = vdot_u32(*sse, abs_diff, abs_diff);
+}
+
+static INLINE void sse_4x2_neon_dotprod(const uint8_t *src, int src_stride,
+ const uint8_t *ref, int ref_stride,
+ uint32x2_t *sse) {
+ uint8x8_t s = load_unaligned_u8(src, src_stride);
+ uint8x8_t r = load_unaligned_u8(ref, ref_stride);
+
+ uint8x8_t abs_diff = vabd_u8(s, r);
+
+ *sse = vdot_u32(*sse, abs_diff, abs_diff);
+}
+
+static INLINE uint32_t sse_wxh_neon_dotprod(const uint8_t *src, int src_stride,
+ const uint8_t *ref, int ref_stride,
+ int width, int height) {
+ uint32x2_t sse[2] = { vdup_n_u32(0), vdup_n_u32(0) };
+
+ if ((width & 0x07) && ((width & 0x07) < 5)) {
+ int i = height;
+ do {
+ int j = 0;
+ do {
+ sse_8x1_neon_dotprod(src + j, ref + j, &sse[0]);
+ sse_8x1_neon_dotprod(src + j + src_stride, ref + j + ref_stride,
+ &sse[1]);
+ j += 8;
+ } while (j + 4 < width);
+
+ sse_4x2_neon_dotprod(src + j, src_stride, ref + j, ref_stride, &sse[0]);
+ src += 2 * src_stride;
+ ref += 2 * ref_stride;
+ i -= 2;
+ } while (i != 0);
+ } else {
+ int i = height;
+ do {
+ int j = 0;
+ do {
+ sse_8x1_neon_dotprod(src + j, ref + j, &sse[0]);
+ sse_8x1_neon_dotprod(src + j + src_stride, ref + j + ref_stride,
+ &sse[1]);
+ j += 8;
+ } while (j < width);
+
+ src += 2 * src_stride;
+ ref += 2 * ref_stride;
+ i -= 2;
+ } while (i != 0);
+ }
+ return horizontal_add_u32x4(vcombine_u32(sse[0], sse[1]));
+}
+
+static INLINE uint32_t sse_128xh_neon_dotprod(const uint8_t *src,
+ int src_stride,
+ const uint8_t *ref,
+ int ref_stride, int height) {
+ uint32x4_t sse[2] = { vdupq_n_u32(0), vdupq_n_u32(0) };
+
+ int i = height;
+ do {
+ sse_16x1_neon_dotprod(src, ref, &sse[0]);
+ sse_16x1_neon_dotprod(src + 16, ref + 16, &sse[1]);
+ sse_16x1_neon_dotprod(src + 32, ref + 32, &sse[0]);
+ sse_16x1_neon_dotprod(src + 48, ref + 48, &sse[1]);
+ sse_16x1_neon_dotprod(src + 64, ref + 64, &sse[0]);
+ sse_16x1_neon_dotprod(src + 80, ref + 80, &sse[1]);
+ sse_16x1_neon_dotprod(src + 96, ref + 96, &sse[0]);
+ sse_16x1_neon_dotprod(src + 112, ref + 112, &sse[1]);
+
+ src += src_stride;
+ ref += ref_stride;
+ } while (--i != 0);
+
+ return horizontal_add_u32x4(vaddq_u32(sse[0], sse[1]));
+}
+
+static INLINE uint32_t sse_64xh_neon_dotprod(const uint8_t *src, int src_stride,
+ const uint8_t *ref, int ref_stride,
+ int height) {
+ uint32x4_t sse[2] = { vdupq_n_u32(0), vdupq_n_u32(0) };
+
+ int i = height;
+ do {
+ sse_16x1_neon_dotprod(src, ref, &sse[0]);
+ sse_16x1_neon_dotprod(src + 16, ref + 16, &sse[1]);
+ sse_16x1_neon_dotprod(src + 32, ref + 32, &sse[0]);
+ sse_16x1_neon_dotprod(src + 48, ref + 48, &sse[1]);
+
+ src += src_stride;
+ ref += ref_stride;
+ } while (--i != 0);
+
+ return horizontal_add_u32x4(vaddq_u32(sse[0], sse[1]));
+}
+
+static INLINE uint32_t sse_32xh_neon_dotprod(const uint8_t *src, int src_stride,
+ const uint8_t *ref, int ref_stride,
+ int height) {
+ uint32x4_t sse[2] = { vdupq_n_u32(0), vdupq_n_u32(0) };
+
+ int i = height;
+ do {
+ sse_16x1_neon_dotprod(src, ref, &sse[0]);
+ sse_16x1_neon_dotprod(src + 16, ref + 16, &sse[1]);
+
+ src += src_stride;
+ ref += ref_stride;
+ } while (--i != 0);
+
+ return horizontal_add_u32x4(vaddq_u32(sse[0], sse[1]));
+}
+
+static INLINE uint32_t sse_16xh_neon_dotprod(const uint8_t *src, int src_stride,
+ const uint8_t *ref, int ref_stride,
+ int height) {
+ uint32x4_t sse[2] = { vdupq_n_u32(0), vdupq_n_u32(0) };
+
+ int i = height;
+ do {
+ sse_16x1_neon_dotprod(src, ref, &sse[0]);
+ src += src_stride;
+ ref += ref_stride;
+ sse_16x1_neon_dotprod(src, ref, &sse[1]);
+ src += src_stride;
+ ref += ref_stride;
+ i -= 2;
+ } while (i != 0);
+
+ return horizontal_add_u32x4(vaddq_u32(sse[0], sse[1]));
+}
+
+static INLINE uint32_t sse_8xh_neon_dotprod(const uint8_t *src, int src_stride,
+ const uint8_t *ref, int ref_stride,
+ int height) {
+ uint32x2_t sse[2] = { vdup_n_u32(0), vdup_n_u32(0) };
+
+ int i = height;
+ do {
+ sse_8x1_neon_dotprod(src, ref, &sse[0]);
+ src += src_stride;
+ ref += ref_stride;
+ sse_8x1_neon_dotprod(src, ref, &sse[1]);
+ src += src_stride;
+ ref += ref_stride;
+ i -= 2;
+ } while (i != 0);
+
+ return horizontal_add_u32x4(vcombine_u32(sse[0], sse[1]));
+}
+
+static INLINE uint32_t sse_4xh_neon_dotprod(const uint8_t *src, int src_stride,
+ const uint8_t *ref, int ref_stride,
+ int height) {
+ uint32x2_t sse = vdup_n_u32(0);
+
+ int i = height;
+ do {
+ sse_4x2_neon_dotprod(src, src_stride, ref, ref_stride, &sse);
+
+ src += 2 * src_stride;
+ ref += 2 * ref_stride;
+ i -= 2;
+ } while (i != 0);
+
+ return horizontal_add_u32x2(sse);
+}
+
+int64_t aom_sse_neon_dotprod(const uint8_t *src, int src_stride,
+ const uint8_t *ref, int ref_stride, int width,
+ int height) {
+ switch (width) {
+ case 4:
+ return sse_4xh_neon_dotprod(src, src_stride, ref, ref_stride, height);
+ case 8:
+ return sse_8xh_neon_dotprod(src, src_stride, ref, ref_stride, height);
+ case 16:
+ return sse_16xh_neon_dotprod(src, src_stride, ref, ref_stride, height);
+ case 32:
+ return sse_32xh_neon_dotprod(src, src_stride, ref, ref_stride, height);
+ case 64:
+ return sse_64xh_neon_dotprod(src, src_stride, ref, ref_stride, height);
+ case 128:
+ return sse_128xh_neon_dotprod(src, src_stride, ref, ref_stride, height);
+ default:
+ return sse_wxh_neon_dotprod(src, src_stride, ref, ref_stride, width,
+ height);
+ }
+}
diff --git a/aom_dsp/arm/subpel_variance_neon.c b/aom_dsp/arm/subpel_variance_neon.c
index 9599ae0..2e6e738 100644
--- a/aom_dsp/arm/subpel_variance_neon.c
+++ b/aom_dsp/arm/subpel_variance_neon.c
@@ -18,6 +18,7 @@
#include "aom/aom_integer.h"
#include "aom_dsp/variance.h"
+#include "aom_dsp/arm/dist_wtd_avg_neon.h"
#include "aom_dsp/arm/mem_neon.h"
static void var_filter_block2d_bil_w4(const uint8_t *src_ptr, uint8_t *dst_ptr,
@@ -154,59 +155,58 @@
return aom_variance##w##x##h(tmp1, w, ref, ref_stride, sse); \
}
-#define SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(w, h, padding) \
- unsigned int aom_sub_pixel_variance##w##x##h##_neon( \
- const uint8_t *src, int src_stride, int xoffset, int yoffset, \
- const uint8_t *ref, int ref_stride, unsigned int *sse) { \
- if (xoffset == 0) { \
- if (yoffset == 0) { \
- return aom_variance##w##x##h##_neon(src, src_stride, ref, ref_stride, \
- sse); \
- } else if (yoffset == 4) { \
- uint8_t tmp[w * h]; \
- var_filter_block2d_avg(src, tmp, src_stride, src_stride, w, h); \
- return aom_variance##w##x##h##_neon(tmp, w, ref, ref_stride, sse); \
- } else { \
- uint8_t tmp[w * h]; \
- var_filter_block2d_bil_w##w(src, tmp, src_stride, src_stride, h, \
- yoffset); \
- return aom_variance##w##x##h##_neon(tmp, w, ref, ref_stride, sse); \
- } \
- } else if (xoffset == 4) { \
- uint8_t tmp0[w * (h + padding)]; \
- if (yoffset == 0) { \
- var_filter_block2d_avg(src, tmp0, src_stride, 1, w, h); \
- return aom_variance##w##x##h##_neon(tmp0, w, ref, ref_stride, sse); \
- } else if (yoffset == 4) { \
- uint8_t tmp1[w * (h + padding)]; \
- var_filter_block2d_avg(src, tmp0, src_stride, 1, w, (h + padding)); \
- var_filter_block2d_avg(tmp0, tmp1, w, w, w, h); \
- return aom_variance##w##x##h##_neon(tmp1, w, ref, ref_stride, sse); \
- } else { \
- uint8_t tmp1[w * (h + padding)]; \
- var_filter_block2d_avg(src, tmp0, src_stride, 1, w, (h + padding)); \
- var_filter_block2d_bil_w##w(tmp0, tmp1, w, w, h, yoffset); \
- return aom_variance##w##x##h##_neon(tmp1, w, ref, ref_stride, sse); \
- } \
- } else { \
- uint8_t tmp0[w * (h + padding)]; \
- if (yoffset == 0) { \
- var_filter_block2d_bil_w##w(src, tmp0, src_stride, 1, h, xoffset); \
- return aom_variance##w##x##h##_neon(tmp0, w, ref, ref_stride, sse); \
- } else if (yoffset == 4) { \
- uint8_t tmp1[w * h]; \
- var_filter_block2d_bil_w##w(src, tmp0, src_stride, 1, (h + padding), \
- xoffset); \
- var_filter_block2d_avg(tmp0, tmp1, w, w, w, h); \
- return aom_variance##w##x##h##_neon(tmp1, w, ref, ref_stride, sse); \
- } else { \
- uint8_t tmp1[w * h]; \
- var_filter_block2d_bil_w##w(src, tmp0, src_stride, 1, (h + padding), \
- xoffset); \
- var_filter_block2d_bil_w##w(tmp0, tmp1, w, w, h, yoffset); \
- return aom_variance##w##x##h##_neon(tmp1, w, ref, ref_stride, sse); \
- } \
- } \
+#define SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(w, h, padding) \
+ unsigned int aom_sub_pixel_variance##w##x##h##_neon( \
+ const uint8_t *src, int src_stride, int xoffset, int yoffset, \
+ const uint8_t *ref, int ref_stride, unsigned int *sse) { \
+ if (xoffset == 0) { \
+ if (yoffset == 0) { \
+ return aom_variance##w##x##h(src, src_stride, ref, ref_stride, sse); \
+ } else if (yoffset == 4) { \
+ uint8_t tmp[w * h]; \
+ var_filter_block2d_avg(src, tmp, src_stride, src_stride, w, h); \
+ return aom_variance##w##x##h(tmp, w, ref, ref_stride, sse); \
+ } else { \
+ uint8_t tmp[w * h]; \
+ var_filter_block2d_bil_w##w(src, tmp, src_stride, src_stride, h, \
+ yoffset); \
+ return aom_variance##w##x##h(tmp, w, ref, ref_stride, sse); \
+ } \
+ } else if (xoffset == 4) { \
+ uint8_t tmp0[w * (h + padding)]; \
+ if (yoffset == 0) { \
+ var_filter_block2d_avg(src, tmp0, src_stride, 1, w, h); \
+ return aom_variance##w##x##h(tmp0, w, ref, ref_stride, sse); \
+ } else if (yoffset == 4) { \
+ uint8_t tmp1[w * (h + padding)]; \
+ var_filter_block2d_avg(src, tmp0, src_stride, 1, w, (h + padding)); \
+ var_filter_block2d_avg(tmp0, tmp1, w, w, w, h); \
+ return aom_variance##w##x##h(tmp1, w, ref, ref_stride, sse); \
+ } else { \
+ uint8_t tmp1[w * (h + padding)]; \
+ var_filter_block2d_avg(src, tmp0, src_stride, 1, w, (h + padding)); \
+ var_filter_block2d_bil_w##w(tmp0, tmp1, w, w, h, yoffset); \
+ return aom_variance##w##x##h(tmp1, w, ref, ref_stride, sse); \
+ } \
+ } else { \
+ uint8_t tmp0[w * (h + padding)]; \
+ if (yoffset == 0) { \
+ var_filter_block2d_bil_w##w(src, tmp0, src_stride, 1, h, xoffset); \
+ return aom_variance##w##x##h(tmp0, w, ref, ref_stride, sse); \
+ } else if (yoffset == 4) { \
+ uint8_t tmp1[w * h]; \
+ var_filter_block2d_bil_w##w(src, tmp0, src_stride, 1, (h + padding), \
+ xoffset); \
+ var_filter_block2d_avg(tmp0, tmp1, w, w, w, h); \
+ return aom_variance##w##x##h(tmp1, w, ref, ref_stride, sse); \
+ } else { \
+ uint8_t tmp1[w * h]; \
+ var_filter_block2d_bil_w##w(src, tmp0, src_stride, 1, (h + padding), \
+ xoffset); \
+ var_filter_block2d_bil_w##w(tmp0, tmp1, w, w, h, yoffset); \
+ return aom_variance##w##x##h(tmp1, w, ref, ref_stride, sse); \
+ } \
+ } \
}
SUBPEL_VARIANCE_WXH_NEON(4, 4, 2)
@@ -279,6 +279,36 @@
} while (i != 0);
}
+// Combine bilinear filter with aom_dist_wtd_comp_avg_pred for blocks having
+// width 4.
+static void dist_wtd_avg_pred_var_filter_block2d_bil_w4(
+ const uint8_t *src_ptr, uint8_t *dst_ptr, int src_stride, int pixel_step,
+ int dst_height, int filter_offset, const uint8_t *second_pred,
+ const DIST_WTD_COMP_PARAMS *jcp_param) {
+ const uint8x8_t fwd_offset = vdup_n_u8(jcp_param->fwd_offset);
+ const uint8x8_t bck_offset = vdup_n_u8(jcp_param->bck_offset);
+ const uint8x8_t f0 = vdup_n_u8(8 - filter_offset);
+ const uint8x8_t f1 = vdup_n_u8(filter_offset);
+
+ int i = dst_height;
+ do {
+ uint8x8_t s0 = load_unaligned_u8(src_ptr, src_stride);
+ uint8x8_t s1 = load_unaligned_u8(src_ptr + pixel_step, src_stride);
+ uint8x8_t p = vld1_u8(second_pred);
+ uint16x8_t blend = vmull_u8(s0, f0);
+ blend = vmlal_u8(blend, s1, f1);
+ uint8x8_t blend_u8 = vrshrn_n_u16(blend, 3);
+ uint8x8_t avg = dist_wtd_avg_u8x8(blend_u8, p, fwd_offset, bck_offset);
+
+ vst1_u8(dst_ptr, avg);
+
+ src_ptr += 2 * src_stride;
+ dst_ptr += 2 * 4;
+ second_pred += 2 * 4;
+ i -= 2;
+ } while (i != 0);
+}
+
// Combine bilinear filter with aom_comp_avg_pred for blocks having width 8.
static void avg_pred_var_filter_block2d_bil_w8(const uint8_t *src_ptr,
uint8_t *dst_ptr, int src_stride,
@@ -307,6 +337,35 @@
} while (--i > 0);
}
+// Combine bilinear filter with aom_dist_wtd_comp_avg_pred for blocks having
+// width 8.
+static void dist_wtd_avg_pred_var_filter_block2d_bil_w8(
+ const uint8_t *src_ptr, uint8_t *dst_ptr, int src_stride, int pixel_step,
+ int dst_height, int filter_offset, const uint8_t *second_pred,
+ const DIST_WTD_COMP_PARAMS *jcp_param) {
+ const uint8x8_t fwd_offset = vdup_n_u8(jcp_param->fwd_offset);
+ const uint8x8_t bck_offset = vdup_n_u8(jcp_param->bck_offset);
+ const uint8x8_t f0 = vdup_n_u8(8 - filter_offset);
+ const uint8x8_t f1 = vdup_n_u8(filter_offset);
+
+ int i = dst_height;
+ do {
+ uint8x8_t s0 = vld1_u8(src_ptr);
+ uint8x8_t s1 = vld1_u8(src_ptr + pixel_step);
+ uint8x8_t p = vld1_u8(second_pred);
+ uint16x8_t blend = vmull_u8(s0, f0);
+ blend = vmlal_u8(blend, s1, f1);
+ uint8x8_t blend_u8 = vrshrn_n_u16(blend, 3);
+ uint8x8_t avg = dist_wtd_avg_u8x8(blend_u8, p, fwd_offset, bck_offset);
+
+ vst1_u8(dst_ptr, avg);
+
+ src_ptr += src_stride;
+ dst_ptr += 8;
+ second_pred += 8;
+ } while (--i > 0);
+}
+
// Combine bilinear filter with aom_comp_avg_pred for large blocks.
static void avg_pred_var_filter_block2d_bil_large(
const uint8_t *src_ptr, uint8_t *dst_ptr, int src_stride, int pixel_step,
@@ -342,6 +401,43 @@
} while (--i != 0);
}
+// Combine bilinear filter with aom_dist_wtd_comp_avg_pred for large blocks.
+static void dist_wtd_avg_pred_var_filter_block2d_bil_large(
+ const uint8_t *src_ptr, uint8_t *dst_ptr, int src_stride, int pixel_step,
+ int dst_width, int dst_height, int filter_offset,
+ const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) {
+ const uint8x16_t fwd_offset = vdupq_n_u8(jcp_param->fwd_offset);
+ const uint8x16_t bck_offset = vdupq_n_u8(jcp_param->bck_offset);
+ const uint8x8_t f0 = vdup_n_u8(8 - filter_offset);
+ const uint8x8_t f1 = vdup_n_u8(filter_offset);
+
+ int i = dst_height;
+ do {
+ int j = 0;
+ do {
+ uint8x16_t s0 = vld1q_u8(src_ptr + j);
+ uint8x16_t s1 = vld1q_u8(src_ptr + j + pixel_step);
+ uint16x8_t blend_l = vmull_u8(vget_low_u8(s0), f0);
+ blend_l = vmlal_u8(blend_l, vget_low_u8(s1), f1);
+ uint16x8_t blend_h = vmull_u8(vget_high_u8(s0), f0);
+ blend_h = vmlal_u8(blend_h, vget_high_u8(s1), f1);
+ uint8x16_t blend_u8 =
+ vcombine_u8(vrshrn_n_u16(blend_l, 3), vrshrn_n_u16(blend_h, 3));
+
+ uint8x16_t p = vld1q_u8(second_pred);
+ uint8x16_t avg = dist_wtd_avg_u8x16(blend_u8, p, fwd_offset, bck_offset);
+
+ vst1q_u8(dst_ptr + j, avg);
+
+ j += 16;
+ second_pred += 16;
+ } while (j < dst_width);
+
+ src_ptr += src_stride;
+ dst_ptr += dst_width;
+ } while (--i != 0);
+}
+
// Combine bilinear filter with aom_comp_avg_pred for blocks having width 16.
static void avg_pred_var_filter_block2d_bil_w16(
const uint8_t *src_ptr, uint8_t *dst_ptr, int src_stride, int pixel_step,
@@ -378,6 +474,46 @@
filter_offset, second_pred);
}
+// Combine bilinear filter with aom_comp_avg_pred for blocks having width 16.
+static void dist_wtd_avg_pred_var_filter_block2d_bil_w16(
+ const uint8_t *src_ptr, uint8_t *dst_ptr, int src_stride, int pixel_step,
+ int dst_height, int filter_offset, const uint8_t *second_pred,
+ const DIST_WTD_COMP_PARAMS *jcp_param) {
+ dist_wtd_avg_pred_var_filter_block2d_bil_large(
+ src_ptr, dst_ptr, src_stride, pixel_step, 16, dst_height, filter_offset,
+ second_pred, jcp_param);
+}
+
+// Combine bilinear filter with aom_comp_avg_pred for blocks having width 32.
+static void dist_wtd_avg_pred_var_filter_block2d_bil_w32(
+ const uint8_t *src_ptr, uint8_t *dst_ptr, int src_stride, int pixel_step,
+ int dst_height, int filter_offset, const uint8_t *second_pred,
+ const DIST_WTD_COMP_PARAMS *jcp_param) {
+ dist_wtd_avg_pred_var_filter_block2d_bil_large(
+ src_ptr, dst_ptr, src_stride, pixel_step, 32, dst_height, filter_offset,
+ second_pred, jcp_param);
+}
+
+// Combine bilinear filter with aom_comp_avg_pred for blocks having width 64.
+static void dist_wtd_avg_pred_var_filter_block2d_bil_w64(
+ const uint8_t *src_ptr, uint8_t *dst_ptr, int src_stride, int pixel_step,
+ int dst_height, int filter_offset, const uint8_t *second_pred,
+ const DIST_WTD_COMP_PARAMS *jcp_param) {
+ dist_wtd_avg_pred_var_filter_block2d_bil_large(
+ src_ptr, dst_ptr, src_stride, pixel_step, 64, dst_height, filter_offset,
+ second_pred, jcp_param);
+}
+
+// Combine bilinear filter with aom_comp_avg_pred for blocks having width 128.
+static void dist_wtd_avg_pred_var_filter_block2d_bil_w128(
+ const uint8_t *src_ptr, uint8_t *dst_ptr, int src_stride, int pixel_step,
+ int dst_height, int filter_offset, const uint8_t *second_pred,
+ const DIST_WTD_COMP_PARAMS *jcp_param) {
+ dist_wtd_avg_pred_var_filter_block2d_bil_large(
+ src_ptr, dst_ptr, src_stride, pixel_step, 128, dst_height, filter_offset,
+ second_pred, jcp_param);
+}
+
// Combine averaging subpel filter with aom_comp_avg_pred.
static void avg_pred_var_filter_block2d_avg(const uint8_t *src_ptr,
uint8_t *dst_ptr, int src_stride,
@@ -409,6 +545,37 @@
} while (--i != 0);
}
+// Combine averaging subpel filter with aom_dist_wtd_comp_avg_pred.
+static void dist_wtd_avg_pred_var_filter_block2d_avg(
+ const uint8_t *src_ptr, uint8_t *dst_ptr, int src_stride, int pixel_step,
+ int dst_width, int dst_height, const uint8_t *second_pred,
+ const DIST_WTD_COMP_PARAMS *jcp_param) {
+ // We only specialise on the filter values for large block sizes (>= 16x16.)
+ assert(dst_width >= 16 && dst_width % 16 == 0);
+ const uint8x16_t fwd_offset = vdupq_n_u8(jcp_param->fwd_offset);
+ const uint8x16_t bck_offset = vdupq_n_u8(jcp_param->bck_offset);
+
+ int i = dst_height;
+ do {
+ int j = 0;
+ do {
+ uint8x16_t s0 = vld1q_u8(src_ptr + j);
+ uint8x16_t s1 = vld1q_u8(src_ptr + j + pixel_step);
+ uint8x16_t p = vld1q_u8(second_pred);
+ uint8x16_t avg = vrhaddq_u8(s0, s1);
+ avg = dist_wtd_avg_u8x16(avg, p, fwd_offset, bck_offset);
+
+ vst1q_u8(dst_ptr + j, avg);
+
+ j += 16;
+ second_pred += 16;
+ } while (j < dst_width);
+
+ src_ptr += src_stride;
+ dst_ptr += dst_width;
+ } while (--i != 0);
+}
+
// Implementation of aom_comp_avg_pred for blocks having width >= 16.
static void avg_pred(const uint8_t *src_ptr, uint8_t *dst_ptr, int src_stride,
int dst_width, int dst_height,
@@ -436,6 +603,36 @@
} while (--i != 0);
}
+// Implementation of aom_dist_wtd_comp_avg_pred for blocks having width >= 16.
+static void dist_wtd_avg_pred(const uint8_t *src_ptr, uint8_t *dst_ptr,
+ int src_stride, int dst_width, int dst_height,
+ const uint8_t *second_pred,
+ const DIST_WTD_COMP_PARAMS *jcp_param) {
+ // We only specialise on the filter values for large block sizes (>= 16x16.)
+ assert(dst_width >= 16 && dst_width % 16 == 0);
+ const uint8x16_t fwd_offset = vdupq_n_u8(jcp_param->fwd_offset);
+ const uint8x16_t bck_offset = vdupq_n_u8(jcp_param->bck_offset);
+
+ int i = dst_height;
+ do {
+ int j = 0;
+ do {
+ uint8x16_t s = vld1q_u8(src_ptr + j);
+ uint8x16_t p = vld1q_u8(second_pred);
+
+ uint8x16_t avg = dist_wtd_avg_u8x16(s, p, fwd_offset, bck_offset);
+
+ vst1q_u8(dst_ptr + j, avg);
+
+ j += 16;
+ second_pred += 16;
+ } while (j < dst_width);
+
+ src_ptr += src_stride;
+ dst_ptr += dst_width;
+ } while (--i != 0);
+}
+
#define SUBPEL_AVG_VARIANCE_WXH_NEON(w, h, padding) \
unsigned int aom_sub_pixel_avg_variance##w##x##h##_neon( \
const uint8_t *src, int source_stride, int xoffset, int yoffset, \
@@ -459,53 +656,53 @@
uint8_t tmp[w * h]; \
if (yoffset == 0) { \
avg_pred(src, tmp, source_stride, w, h, second_pred); \
- return aom_variance##w##x##h##_neon(tmp, w, ref, ref_stride, sse); \
+ return aom_variance##w##x##h(tmp, w, ref, ref_stride, sse); \
} else if (yoffset == 4) { \
avg_pred_var_filter_block2d_avg(src, tmp, source_stride, \
source_stride, w, h, second_pred); \
- return aom_variance##w##x##h##_neon(tmp, w, ref, ref_stride, sse); \
+ return aom_variance##w##x##h(tmp, w, ref, ref_stride, sse); \
} else { \
avg_pred_var_filter_block2d_bil_w##w( \
src, tmp, source_stride, source_stride, h, yoffset, second_pred); \
- return aom_variance##w##x##h##_neon(tmp, w, ref, ref_stride, sse); \
+ return aom_variance##w##x##h(tmp, w, ref, ref_stride, sse); \
} \
} else if (xoffset == 4) { \
uint8_t tmp0[w * (h + padding)]; \
if (yoffset == 0) { \
avg_pred_var_filter_block2d_avg(src, tmp0, source_stride, 1, w, h, \
second_pred); \
- return aom_variance##w##x##h##_neon(tmp0, w, ref, ref_stride, sse); \
+ return aom_variance##w##x##h(tmp0, w, ref, ref_stride, sse); \
} else if (yoffset == 4) { \
uint8_t tmp1[w * (h + padding)]; \
var_filter_block2d_avg(src, tmp0, source_stride, 1, w, (h + padding)); \
avg_pred_var_filter_block2d_avg(tmp0, tmp1, w, w, w, h, second_pred); \
- return aom_variance##w##x##h##_neon(tmp1, w, ref, ref_stride, sse); \
+ return aom_variance##w##x##h(tmp1, w, ref, ref_stride, sse); \
} else { \
uint8_t tmp1[w * (h + padding)]; \
var_filter_block2d_avg(src, tmp0, source_stride, 1, w, (h + padding)); \
avg_pred_var_filter_block2d_bil_w##w(tmp0, tmp1, w, w, h, yoffset, \
second_pred); \
- return aom_variance##w##x##h##_neon(tmp1, w, ref, ref_stride, sse); \
+ return aom_variance##w##x##h(tmp1, w, ref, ref_stride, sse); \
} \
} else { \
uint8_t tmp0[w * (h + padding)]; \
if (yoffset == 0) { \
avg_pred_var_filter_block2d_bil_w##w(src, tmp0, source_stride, 1, h, \
xoffset, second_pred); \
- return aom_variance##w##x##h##_neon(tmp0, w, ref, ref_stride, sse); \
+ return aom_variance##w##x##h(tmp0, w, ref, ref_stride, sse); \
} else if (yoffset == 4) { \
uint8_t tmp1[w * h]; \
var_filter_block2d_bil_w##w(src, tmp0, source_stride, 1, \
(h + padding), xoffset); \
avg_pred_var_filter_block2d_avg(tmp0, tmp1, w, w, w, h, second_pred); \
- return aom_variance##w##x##h##_neon(tmp1, w, ref, ref_stride, sse); \
+ return aom_variance##w##x##h(tmp1, w, ref, ref_stride, sse); \
} else { \
uint8_t tmp1[w * h]; \
var_filter_block2d_bil_w##w(src, tmp0, source_stride, 1, \
(h + padding), xoffset); \
avg_pred_var_filter_block2d_bil_w##w(tmp0, tmp1, w, w, h, yoffset, \
second_pred); \
- return aom_variance##w##x##h##_neon(tmp1, w, ref, ref_stride, sse); \
+ return aom_variance##w##x##h(tmp1, w, ref, ref_stride, sse); \
} \
} \
}
@@ -550,6 +747,125 @@
#undef SUBPEL_AVG_VARIANCE_WXH_NEON
#undef SPECIALIZED_SUBPEL_AVG_VARIANCE_WXH_NEON
+#define DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(w, h, padding) \
+ unsigned int aom_dist_wtd_sub_pixel_avg_variance##w##x##h##_neon( \
+ const uint8_t *src, int source_stride, int xoffset, int yoffset, \
+ const uint8_t *ref, int ref_stride, uint32_t *sse, \
+ const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \
+ uint8_t tmp0[w * (h + padding)]; \
+ uint8_t tmp1[w * h]; \
+ var_filter_block2d_bil_w##w(src, tmp0, source_stride, 1, (h + padding), \
+ xoffset); \
+ dist_wtd_avg_pred_var_filter_block2d_bil_w##w( \
+ tmp0, tmp1, w, w, h, yoffset, second_pred, jcp_param); \
+ return aom_variance##w##x##h(tmp1, w, ref, ref_stride, sse); \
+ }
+
+#define SPECIALIZED_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(w, h, padding) \
+ unsigned int aom_dist_wtd_sub_pixel_avg_variance##w##x##h##_neon( \
+ const uint8_t *src, int source_stride, int xoffset, int yoffset, \
+ const uint8_t *ref, int ref_stride, unsigned int *sse, \
+ const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \
+ if (xoffset == 0) { \
+ uint8_t tmp[w * h]; \
+ if (yoffset == 0) { \
+ dist_wtd_avg_pred(src, tmp, source_stride, w, h, second_pred, \
+ jcp_param); \
+ return aom_variance##w##x##h(tmp, w, ref, ref_stride, sse); \
+ } else if (yoffset == 4) { \
+ dist_wtd_avg_pred_var_filter_block2d_avg(src, tmp, source_stride, \
+ source_stride, w, h, \
+ second_pred, jcp_param); \
+ return aom_variance##w##x##h(tmp, w, ref, ref_stride, sse); \
+ } else { \
+ dist_wtd_avg_pred_var_filter_block2d_bil_w##w( \
+ src, tmp, source_stride, source_stride, h, yoffset, second_pred, \
+ jcp_param); \
+ return aom_variance##w##x##h(tmp, w, ref, ref_stride, sse); \
+ } \
+ } else if (xoffset == 4) { \
+ uint8_t tmp0[w * (h + padding)]; \
+ if (yoffset == 0) { \
+ dist_wtd_avg_pred_var_filter_block2d_avg( \
+ src, tmp0, source_stride, 1, w, h, second_pred, jcp_param); \
+ return aom_variance##w##x##h(tmp0, w, ref, ref_stride, sse); \
+ } else if (yoffset == 4) { \
+ uint8_t tmp1[w * (h + padding)]; \
+ var_filter_block2d_avg(src, tmp0, source_stride, 1, w, (h + padding)); \
+ dist_wtd_avg_pred_var_filter_block2d_avg(tmp0, tmp1, w, w, w, h, \
+ second_pred, jcp_param); \
+ return aom_variance##w##x##h(tmp1, w, ref, ref_stride, sse); \
+ } else { \
+ uint8_t tmp1[w * (h + padding)]; \
+ var_filter_block2d_avg(src, tmp0, source_stride, 1, w, (h + padding)); \
+ dist_wtd_avg_pred_var_filter_block2d_bil_w##w( \
+ tmp0, tmp1, w, w, h, yoffset, second_pred, jcp_param); \
+ return aom_variance##w##x##h(tmp1, w, ref, ref_stride, sse); \
+ } \
+ } else { \
+ uint8_t tmp0[w * (h + padding)]; \
+ if (yoffset == 0) { \
+ dist_wtd_avg_pred_var_filter_block2d_bil_w##w( \
+ src, tmp0, source_stride, 1, h, xoffset, second_pred, jcp_param); \
+ return aom_variance##w##x##h(tmp0, w, ref, ref_stride, sse); \
+ } else if (yoffset == 4) { \
+ uint8_t tmp1[w * h]; \
+ var_filter_block2d_bil_w##w(src, tmp0, source_stride, 1, \
+ (h + padding), xoffset); \
+ dist_wtd_avg_pred_var_filter_block2d_avg(tmp0, tmp1, w, w, w, h, \
+ second_pred, jcp_param); \
+ return aom_variance##w##x##h(tmp1, w, ref, ref_stride, sse); \
+ } else { \
+ uint8_t tmp1[w * h]; \
+ var_filter_block2d_bil_w##w(src, tmp0, source_stride, 1, \
+ (h + padding), xoffset); \
+ dist_wtd_avg_pred_var_filter_block2d_bil_w##w( \
+ tmp0, tmp1, w, w, h, yoffset, second_pred, jcp_param); \
+ return aom_variance##w##x##h(tmp1, w, ref, ref_stride, sse); \
+ } \
+ } \
+ }
+
+DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(4, 4, 2)
+DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(4, 8, 2)
+
+DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 4, 1)
+DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 8, 1)
+DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 16, 1)
+
+DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(16, 8, 1)
+SPECIALIZED_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(16, 16, 1)
+SPECIALIZED_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(16, 32, 1)
+
+SPECIALIZED_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(32, 16, 1)
+SPECIALIZED_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(32, 32, 1)
+SPECIALIZED_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(32, 64, 1)
+
+SPECIALIZED_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(64, 32, 1)
+SPECIALIZED_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(64, 64, 1)
+SPECIALIZED_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(64, 128, 1)
+
+SPECIALIZED_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(128, 64, 1)
+SPECIALIZED_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(128, 128, 1)
+
+#if !CONFIG_REALTIME_ONLY
+
+DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(4, 16, 2)
+
+DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(8, 32, 1)
+
+DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(16, 4, 1)
+SPECIALIZED_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(16, 64, 1)
+
+SPECIALIZED_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(32, 8, 1)
+
+SPECIALIZED_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON(64, 16, 1)
+
+#endif // !CONFIG_REALTIME_ONLY
+
+#undef DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON
+#undef SPECIALIZED_DIST_WTD_SUBPEL_AVG_VARIANCE_WXH_NEON
+
#if !CONFIG_REALTIME_ONLY
#define OBMC_SUBPEL_VARIANCE_WXH_NEON(w, h, padding) \
@@ -665,7 +981,7 @@
var_filter_block2d_bil_w##w(tmp0, tmp1, w, w, h, yoffset); \
aom_comp_mask_pred_neon(tmp2, second_pred, w, h, tmp1, w, msk, msk_stride, \
invert_mask); \
- return aom_variance##w##x##h##_neon(tmp2, w, ref, ref_stride, sse); \
+ return aom_variance##w##x##h(tmp2, w, ref, ref_stride, sse); \
}
#define SPECIALIZED_MASKED_SUBPEL_VARIANCE_WXH_NEON(w, h, padding) \
@@ -679,20 +995,20 @@
if (yoffset == 0) { \
aom_comp_mask_pred_neon(tmp0, second_pred, w, h, src, src_stride, msk, \
msk_stride, invert_mask); \
- return aom_variance##w##x##h##_neon(tmp0, w, ref, ref_stride, sse); \
+ return aom_variance##w##x##h(tmp0, w, ref, ref_stride, sse); \
} else if (yoffset == 4) { \
uint8_t tmp1[w * h]; \
var_filter_block2d_avg(src, tmp0, src_stride, src_stride, w, h); \
aom_comp_mask_pred_neon(tmp1, second_pred, w, h, tmp0, w, msk, \
msk_stride, invert_mask); \
- return aom_variance##w##x##h##_neon(tmp1, w, ref, ref_stride, sse); \
+ return aom_variance##w##x##h(tmp1, w, ref, ref_stride, sse); \
} else { \
uint8_t tmp1[w * h]; \
var_filter_block2d_bil_w##w(src, tmp0, src_stride, src_stride, h, \
yoffset); \
aom_comp_mask_pred_neon(tmp1, second_pred, w, h, tmp0, w, msk, \
msk_stride, invert_mask); \
- return aom_variance##w##x##h##_neon(tmp1, w, ref, ref_stride, sse); \
+ return aom_variance##w##x##h(tmp1, w, ref, ref_stride, sse); \
} \
} else if (xoffset == 4) { \
uint8_t tmp0[w * (h + padding)]; \
@@ -701,7 +1017,7 @@
var_filter_block2d_avg(src, tmp0, src_stride, 1, w, h); \
aom_comp_mask_pred_neon(tmp1, second_pred, w, h, tmp0, w, msk, \
msk_stride, invert_mask); \
- return aom_variance##w##x##h##_neon(tmp1, w, ref, ref_stride, sse); \
+ return aom_variance##w##x##h(tmp1, w, ref, ref_stride, sse); \
} else if (yoffset == 4) { \
uint8_t tmp1[w * h]; \
uint8_t tmp2[w * h]; \
@@ -709,7 +1025,7 @@
var_filter_block2d_avg(tmp0, tmp1, w, w, w, h); \
aom_comp_mask_pred_neon(tmp2, second_pred, w, h, tmp1, w, msk, \
msk_stride, invert_mask); \
- return aom_variance##w##x##h##_neon(tmp2, w, ref, ref_stride, sse); \
+ return aom_variance##w##x##h(tmp2, w, ref, ref_stride, sse); \
} else { \
uint8_t tmp1[w * h]; \
uint8_t tmp2[w * h]; \
@@ -717,7 +1033,7 @@
var_filter_block2d_bil_w##w(tmp0, tmp1, w, w, h, yoffset); \
aom_comp_mask_pred_neon(tmp2, second_pred, w, h, tmp1, w, msk, \
msk_stride, invert_mask); \
- return aom_variance##w##x##h##_neon(tmp2, w, ref, ref_stride, sse); \
+ return aom_variance##w##x##h(tmp2, w, ref, ref_stride, sse); \
} \
} else { \
if (yoffset == 0) { \
@@ -726,7 +1042,7 @@
var_filter_block2d_bil_w##w(src, tmp0, src_stride, 1, h, xoffset); \
aom_comp_mask_pred_neon(tmp1, second_pred, w, h, tmp0, w, msk, \
msk_stride, invert_mask); \
- return aom_variance##w##x##h##_neon(tmp1, w, ref, ref_stride, sse); \
+ return aom_variance##w##x##h(tmp1, w, ref, ref_stride, sse); \
} else if (yoffset == 4) { \
uint8_t tmp0[w * (h + padding)]; \
uint8_t tmp1[w * h]; \
@@ -736,7 +1052,7 @@
var_filter_block2d_avg(tmp0, tmp1, w, w, w, h); \
aom_comp_mask_pred_neon(tmp2, second_pred, w, h, tmp1, w, msk, \
msk_stride, invert_mask); \
- return aom_variance##w##x##h##_neon(tmp2, w, ref, ref_stride, sse); \
+ return aom_variance##w##x##h(tmp2, w, ref, ref_stride, sse); \
} else { \
uint8_t tmp0[w * (h + padding)]; \
uint8_t tmp1[w * (h + padding)]; \
@@ -746,7 +1062,7 @@
var_filter_block2d_bil_w##w(tmp0, tmp1, w, w, h, yoffset); \
aom_comp_mask_pred_neon(tmp2, second_pred, w, h, tmp1, w, msk, \
msk_stride, invert_mask); \
- return aom_variance##w##x##h##_neon(tmp2, w, ref, ref_stride, sse); \
+ return aom_variance##w##x##h(tmp2, w, ref, ref_stride, sse); \
} \
} \
}
diff --git a/aom_dsp/arm/sum_neon.h b/aom_dsp/arm/sum_neon.h
index ff68c12..b5a8b97 100644
--- a/aom_dsp/arm/sum_neon.h
+++ b/aom_dsp/arm/sum_neon.h
@@ -8,6 +8,9 @@
* be found in the AUTHORS file in the root of the source tree.
*/
+#ifndef AOM_AOM_DSP_ARM_SUM_NEON_H_
+#define AOM_AOM_DSP_ARM_SUM_NEON_H_
+
#include "config/aom_dsp_rtcd.h"
#include "config/aom_config.h"
@@ -62,7 +65,16 @@
#endif
}
-static INLINE unsigned int horizontal_add_u32x4(const uint32x4_t a) {
+static INLINE int64_t horizontal_long_add_s32x4(const int32x4_t a) {
+#if AOM_ARCH_AARCH64
+ return vaddlvq_s32(a);
+#else
+ const int64x2_t b = vpaddlq_s32(a);
+ return vgetq_lane_s64(b, 0) + vgetq_lane_s64(b, 1);
+#endif
+}
+
+static INLINE uint32_t horizontal_add_u32x4(const uint32x4_t a) {
#if AOM_ARCH_AARCH64
return vaddvq_u32(a);
#else
@@ -88,6 +100,21 @@
#endif
}
+static INLINE int32x4_t horizontal_add_4d_s32x4(const int32x4_t sum[4]) {
+#if AOM_ARCH_AARCH64
+ int32x4_t res01 = vpaddq_s32(sum[0], sum[1]);
+ int32x4_t res23 = vpaddq_s32(sum[2], sum[3]);
+ return vpaddq_s32(res01, res23);
+#else
+ int32x4_t res = vdupq_n_s32(0);
+ res = vsetq_lane_s32(horizontal_add_s32x4(sum[0]), res, 0);
+ res = vsetq_lane_s32(horizontal_add_s32x4(sum[1]), res, 1);
+ res = vsetq_lane_s32(horizontal_add_s32x4(sum[2]), res, 2);
+ res = vsetq_lane_s32(horizontal_add_s32x4(sum[3]), res, 3);
+ return res;
+#endif
+}
+
static INLINE uint32_t horizontal_long_add_u16x8(const uint16x8_t vec_lo,
const uint16x8_t vec_hi) {
#if AOM_ARCH_AARCH64
@@ -186,3 +213,72 @@
return vget_lane_u32(vreinterpret_u32_u64(c), 0);
#endif
}
+
+static INLINE int32x4_t horizontal_add_2d_s32(int32x4_t a, int32x4_t b) {
+#if AOM_ARCH_AARCH64
+ return vpaddq_s32(a, b);
+#else
+ const int32x2_t a0 = vpadd_s32(vget_low_s32(a), vget_high_s32(a));
+ const int32x2_t b0 = vpadd_s32(vget_low_s32(b), vget_high_s32(b));
+ return vcombine_s32(a0, b0);
+#endif
+}
+
+static INLINE int32x2_t add_pairwise_s32x4(int32x4_t a) {
+#if AOM_ARCH_AARCH64
+ return vget_low_s32(vpaddq_s32(a, a));
+#else
+ return vpadd_s32(vget_low_s32(a), vget_high_s32(a));
+#endif
+}
+
+static INLINE uint64_t horizontal_long_add_u32x4_x2(const uint32x4_t a[2]) {
+ return horizontal_long_add_u32x4(a[0]) + horizontal_long_add_u32x4(a[1]);
+}
+
+static INLINE uint64_t horizontal_long_add_u32x4_x4(const uint32x4_t a[4]) {
+ uint64x2_t sum = vpaddlq_u32(a[0]);
+ sum = vpadalq_u32(sum, a[1]);
+ sum = vpadalq_u32(sum, a[2]);
+ sum = vpadalq_u32(sum, a[3]);
+
+ return horizontal_add_u64x2(sum);
+}
+
+static INLINE uint64_t horizontal_long_add_u32x4_x8(const uint32x4_t a[8]) {
+ uint64x2_t sum[2];
+ sum[0] = vpaddlq_u32(a[0]);
+ sum[1] = vpaddlq_u32(a[1]);
+ sum[0] = vpadalq_u32(sum[0], a[2]);
+ sum[1] = vpadalq_u32(sum[1], a[3]);
+ sum[0] = vpadalq_u32(sum[0], a[4]);
+ sum[1] = vpadalq_u32(sum[1], a[5]);
+ sum[0] = vpadalq_u32(sum[0], a[6]);
+ sum[1] = vpadalq_u32(sum[1], a[7]);
+
+ return horizontal_add_u64x2(vaddq_u64(sum[0], sum[1]));
+}
+
+static INLINE uint64_t horizontal_long_add_u32x4_x16(const uint32x4_t a[16]) {
+ uint64x2_t sum[2];
+ sum[0] = vpaddlq_u32(a[0]);
+ sum[1] = vpaddlq_u32(a[1]);
+ sum[0] = vpadalq_u32(sum[0], a[2]);
+ sum[1] = vpadalq_u32(sum[1], a[3]);
+ sum[0] = vpadalq_u32(sum[0], a[4]);
+ sum[1] = vpadalq_u32(sum[1], a[5]);
+ sum[0] = vpadalq_u32(sum[0], a[6]);
+ sum[1] = vpadalq_u32(sum[1], a[7]);
+ sum[0] = vpadalq_u32(sum[0], a[8]);
+ sum[1] = vpadalq_u32(sum[1], a[9]);
+ sum[0] = vpadalq_u32(sum[0], a[10]);
+ sum[1] = vpadalq_u32(sum[1], a[11]);
+ sum[0] = vpadalq_u32(sum[0], a[12]);
+ sum[1] = vpadalq_u32(sum[1], a[13]);
+ sum[0] = vpadalq_u32(sum[0], a[14]);
+ sum[1] = vpadalq_u32(sum[1], a[15]);
+
+ return horizontal_add_u64x2(vaddq_u64(sum[0], sum[1]));
+}
+
+#endif // AOM_AOM_DSP_ARM_SUM_NEON_H_
diff --git a/aom_dsp/arm/sum_squares_neon.c b/aom_dsp/arm/sum_squares_neon.c
index 626cf21..424b2b4 100644
--- a/aom_dsp/arm/sum_squares_neon.c
+++ b/aom_dsp/arm/sum_squares_neon.c
@@ -287,130 +287,6 @@
return aom_sum_squares_i16_c(src, n);
}
-#if defined(__ARM_FEATURE_DOTPROD)
-
-static INLINE uint64_t aom_var_2d_u8_4xh_neon(uint8_t *src, int src_stride,
- int width, int height) {
- uint64_t sum = 0;
- uint64_t sse = 0;
- uint32x2_t sum_u32 = vdup_n_u32(0);
- uint32x2_t sse_u32 = vdup_n_u32(0);
-
- int h = height / 2;
- do {
- int w = width;
- uint8_t *src_ptr = src;
- do {
- uint8x8_t s0 = load_unaligned_u8(src_ptr, src_stride);
-
- sum_u32 = vdot_u32(sum_u32, s0, vdup_n_u8(1));
-
- sse_u32 = vdot_u32(sse_u32, s0, s0);
-
- src_ptr += 8;
- w -= 8;
- } while (w >= 8);
-
- // Process remaining columns in the row using C.
- while (w > 0) {
- int idx = width - w;
- const uint8_t v = src[idx];
- sum += v;
- sse += v * v;
- w--;
- }
-
- src += 2 * src_stride;
- } while (--h != 0);
-
- sum += horizontal_long_add_u32x2(sum_u32);
- sse += horizontal_long_add_u32x2(sse_u32);
-
- return sse - sum * sum / (width * height);
-}
-
-static INLINE uint64_t aom_var_2d_u8_8xh_neon(uint8_t *src, int src_stride,
- int width, int height) {
- uint64_t sum = 0;
- uint64_t sse = 0;
- uint32x2_t sum_u32 = vdup_n_u32(0);
- uint32x2_t sse_u32 = vdup_n_u32(0);
-
- int h = height;
- do {
- int w = width;
- uint8_t *src_ptr = src;
- do {
- uint8x8_t s0 = vld1_u8(src_ptr);
-
- sum_u32 = vdot_u32(sum_u32, s0, vdup_n_u8(1));
-
- sse_u32 = vdot_u32(sse_u32, s0, s0);
-
- src_ptr += 8;
- w -= 8;
- } while (w >= 8);
-
- // Process remaining columns in the row using C.
- while (w > 0) {
- int idx = width - w;
- const uint8_t v = src[idx];
- sum += v;
- sse += v * v;
- w--;
- }
-
- src += src_stride;
- } while (--h != 0);
-
- sum += horizontal_long_add_u32x2(sum_u32);
- sse += horizontal_long_add_u32x2(sse_u32);
-
- return sse - sum * sum / (width * height);
-}
-
-static INLINE uint64_t aom_var_2d_u8_16xh_neon(uint8_t *src, int src_stride,
- int width, int height) {
- uint64_t sum = 0;
- uint64_t sse = 0;
- uint32x4_t sum_u32 = vdupq_n_u32(0);
- uint32x4_t sse_u32 = vdupq_n_u32(0);
-
- int h = height;
- do {
- int w = width;
- uint8_t *src_ptr = src;
- do {
- uint8x16_t s0 = vld1q_u8(src_ptr);
-
- sum_u32 = vdotq_u32(sum_u32, s0, vdupq_n_u8(1));
-
- sse_u32 = vdotq_u32(sse_u32, s0, s0);
-
- src_ptr += 16;
- w -= 16;
- } while (w >= 16);
-
- // Process remaining columns in the row using C.
- while (w > 0) {
- int idx = width - w;
- const uint8_t v = src[idx];
- sum += v;
- sse += v * v;
- w--;
- }
-
- src += src_stride;
- } while (--h != 0);
-
- sum += horizontal_long_add_u32x4(sum_u32);
- sse += horizontal_long_add_u32x4(sse_u32);
-
- return sse - sum * sum / (width * height);
-}
-
-#else // !defined(__ARM_FEATURE_DOTPROD)
-
static INLINE uint64_t aom_var_2d_u8_4xh_neon(uint8_t *src, int src_stride,
int width, int height) {
uint64_t sum = 0;
@@ -584,8 +460,6 @@
return sse - sum * sum / (width * height);
}
-#endif // defined(__ARM_FEATURE_DOTPROD)
-
uint64_t aom_var_2d_u8_neon(uint8_t *src, int src_stride, int width,
int height) {
if (width >= 16) {
diff --git a/aom_dsp/arm/sum_squares_neon_dotprod.c b/aom_dsp/arm/sum_squares_neon_dotprod.c
new file mode 100644
index 0000000..44462a6
--- /dev/null
+++ b/aom_dsp/arm/sum_squares_neon_dotprod.c
@@ -0,0 +1,154 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+#include <assert.h>
+
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_dsp/arm/sum_neon.h"
+#include "config/aom_dsp_rtcd.h"
+
+static INLINE uint64_t aom_var_2d_u8_4xh_neon_dotprod(uint8_t *src,
+ int src_stride, int width,
+ int height) {
+ uint64_t sum = 0;
+ uint64_t sse = 0;
+ uint32x2_t sum_u32 = vdup_n_u32(0);
+ uint32x2_t sse_u32 = vdup_n_u32(0);
+
+ int h = height / 2;
+ do {
+ int w = width;
+ uint8_t *src_ptr = src;
+ do {
+ uint8x8_t s0 = load_unaligned_u8(src_ptr, src_stride);
+
+ sum_u32 = vdot_u32(sum_u32, s0, vdup_n_u8(1));
+
+ sse_u32 = vdot_u32(sse_u32, s0, s0);
+
+ src_ptr += 8;
+ w -= 8;
+ } while (w >= 8);
+
+ // Process remaining columns in the row using C.
+ while (w > 0) {
+ int idx = width - w;
+ const uint8_t v = src[idx];
+ sum += v;
+ sse += v * v;
+ w--;
+ }
+
+ src += 2 * src_stride;
+ } while (--h != 0);
+
+ sum += horizontal_long_add_u32x2(sum_u32);
+ sse += horizontal_long_add_u32x2(sse_u32);
+
+ return sse - sum * sum / (width * height);
+}
+
+static INLINE uint64_t aom_var_2d_u8_8xh_neon_dotprod(uint8_t *src,
+ int src_stride, int width,
+ int height) {
+ uint64_t sum = 0;
+ uint64_t sse = 0;
+ uint32x2_t sum_u32 = vdup_n_u32(0);
+ uint32x2_t sse_u32 = vdup_n_u32(0);
+
+ int h = height;
+ do {
+ int w = width;
+ uint8_t *src_ptr = src;
+ do {
+ uint8x8_t s0 = vld1_u8(src_ptr);
+
+ sum_u32 = vdot_u32(sum_u32, s0, vdup_n_u8(1));
+
+ sse_u32 = vdot_u32(sse_u32, s0, s0);
+
+ src_ptr += 8;
+ w -= 8;
+ } while (w >= 8);
+
+ // Process remaining columns in the row using C.
+ while (w > 0) {
+ int idx = width - w;
+ const uint8_t v = src[idx];
+ sum += v;
+ sse += v * v;
+ w--;
+ }
+
+ src += src_stride;
+ } while (--h != 0);
+
+ sum += horizontal_long_add_u32x2(sum_u32);
+ sse += horizontal_long_add_u32x2(sse_u32);
+
+ return sse - sum * sum / (width * height);
+}
+
+static INLINE uint64_t aom_var_2d_u8_16xh_neon_dotprod(uint8_t *src,
+ int src_stride,
+ int width, int height) {
+ uint64_t sum = 0;
+ uint64_t sse = 0;
+ uint32x4_t sum_u32 = vdupq_n_u32(0);
+ uint32x4_t sse_u32 = vdupq_n_u32(0);
+
+ int h = height;
+ do {
+ int w = width;
+ uint8_t *src_ptr = src;
+ do {
+ uint8x16_t s0 = vld1q_u8(src_ptr);
+
+ sum_u32 = vdotq_u32(sum_u32, s0, vdupq_n_u8(1));
+
+ sse_u32 = vdotq_u32(sse_u32, s0, s0);
+
+ src_ptr += 16;
+ w -= 16;
+ } while (w >= 16);
+
+ // Process remaining columns in the row using C.
+ while (w > 0) {
+ int idx = width - w;
+ const uint8_t v = src[idx];
+ sum += v;
+ sse += v * v;
+ w--;
+ }
+
+ src += src_stride;
+ } while (--h != 0);
+
+ sum += horizontal_long_add_u32x4(sum_u32);
+ sse += horizontal_long_add_u32x4(sse_u32);
+
+ return sse - sum * sum / (width * height);
+}
+
+uint64_t aom_var_2d_u8_neon_dotprod(uint8_t *src, int src_stride, int width,
+ int height) {
+ if (width >= 16) {
+ return aom_var_2d_u8_16xh_neon_dotprod(src, src_stride, width, height);
+ }
+ if (width >= 8) {
+ return aom_var_2d_u8_8xh_neon_dotprod(src, src_stride, width, height);
+ }
+ if (width >= 4 && height % 2 == 0) {
+ return aom_var_2d_u8_4xh_neon_dotprod(src, src_stride, width, height);
+ }
+ return aom_var_2d_u8_c(src, src_stride, width, height);
+}
diff --git a/aom_dsp/arm/transpose_neon.h b/aom_dsp/arm/transpose_neon.h
index 8218140..b215f6a 100644
--- a/aom_dsp/arm/transpose_neon.h
+++ b/aom_dsp/arm/transpose_neon.h
@@ -13,16 +13,14 @@
#include <arm_neon.h>
+#include "aom/aom_integer.h" // For AOM_FORCE_INLINE.
#include "config/aom_config.h"
-// Swap high and low halves.
-static INLINE uint16x8_t transpose64_u16q(const uint16x8_t a) {
- return vextq_u16(a, a, 4);
-}
-
-static INLINE void transpose_u8_8x8(uint8x8_t *a0, uint8x8_t *a1, uint8x8_t *a2,
- uint8x8_t *a3, uint8x8_t *a4, uint8x8_t *a5,
- uint8x8_t *a6, uint8x8_t *a7) {
+static INLINE void transpose_elems_inplace_u8_8x8(uint8x8_t *a0, uint8x8_t *a1,
+ uint8x8_t *a2, uint8x8_t *a3,
+ uint8x8_t *a4, uint8x8_t *a5,
+ uint8x8_t *a6,
+ uint8x8_t *a7) {
// Swap 8 bit elements. Goes from:
// a0: 00 01 02 03 04 05 06 07
// a1: 10 11 12 13 14 15 16 17
@@ -74,8 +72,9 @@
*a7 = vreinterpret_u8_u32(vget_high_u32(d1.val[1]));
}
-static INLINE void transpose_u8_8x4(uint8x8_t *a0, uint8x8_t *a1, uint8x8_t *a2,
- uint8x8_t *a3) {
+static INLINE void transpose_elems_inplace_u8_8x4(uint8x8_t *a0, uint8x8_t *a1,
+ uint8x8_t *a2,
+ uint8x8_t *a3) {
// Swap 8 bit elements. Goes from:
// a0: 00 01 02 03 04 05 06 07
// a1: 10 11 12 13 14 15 16 17
@@ -107,7 +106,8 @@
*a3 = vreinterpret_u8_u16(c1.val[1]);
}
-static INLINE void transpose_u8_4x4(uint8x8_t *a0, uint8x8_t *a1) {
+static INLINE void transpose_elems_inplace_u8_4x4(uint8x8_t *a0,
+ uint8x8_t *a1) {
// Swap 16 bit elements. Goes from:
// a0: 00 01 02 03 10 11 12 13
// a1: 20 21 22 23 30 31 32 33
@@ -136,10 +136,12 @@
*a1 = d0.val[1];
}
-static INLINE void transpose_u8_4x8(uint8x8_t *a0, uint8x8_t *a1, uint8x8_t *a2,
- uint8x8_t *a3, const uint8x8_t a4,
- const uint8x8_t a5, const uint8x8_t a6,
- const uint8x8_t a7) {
+static INLINE void transpose_elems_u8_4x8(uint8x8_t a0, uint8x8_t a1,
+ uint8x8_t a2, uint8x8_t a3,
+ uint8x8_t a4, uint8x8_t a5,
+ uint8x8_t a6, uint8x8_t a7,
+ uint8x8_t *o0, uint8x8_t *o1,
+ uint8x8_t *o2, uint8x8_t *o3) {
// Swap 32 bit elements. Goes from:
// a0: 00 01 02 03 XX XX XX XX
// a1: 10 11 12 13 XX XX XX XX
@@ -156,13 +158,13 @@
// b3.val[0]: 30 31 32 33 70 71 72 73
const uint32x2x2_t b0 =
- vtrn_u32(vreinterpret_u32_u8(*a0), vreinterpret_u32_u8(a4));
+ vtrn_u32(vreinterpret_u32_u8(a0), vreinterpret_u32_u8(a4));
const uint32x2x2_t b1 =
- vtrn_u32(vreinterpret_u32_u8(*a1), vreinterpret_u32_u8(a5));
+ vtrn_u32(vreinterpret_u32_u8(a1), vreinterpret_u32_u8(a5));
const uint32x2x2_t b2 =
- vtrn_u32(vreinterpret_u32_u8(*a2), vreinterpret_u32_u8(a6));
+ vtrn_u32(vreinterpret_u32_u8(a2), vreinterpret_u32_u8(a6));
const uint32x2x2_t b3 =
- vtrn_u32(vreinterpret_u32_u8(*a3), vreinterpret_u32_u8(a7));
+ vtrn_u32(vreinterpret_u32_u8(a3), vreinterpret_u32_u8(a7));
// Swap 16 bit elements resulting in:
// c0.val[0]: 00 01 20 21 40 41 60 61
@@ -186,23 +188,19 @@
const uint8x8x2_t d1 =
vtrn_u8(vreinterpret_u8_u16(c0.val[1]), vreinterpret_u8_u16(c1.val[1]));
- *a0 = d0.val[0];
- *a1 = d0.val[1];
- *a2 = d1.val[0];
- *a3 = d1.val[1];
+ *o0 = d0.val[0];
+ *o1 = d0.val[1];
+ *o2 = d1.val[0];
+ *o3 = d1.val[1];
}
-// Input:
-// 00 01 02 03
-// 10 11 12 13
-// 20 21 22 23
-// 30 31 32 33
-// Output:
-// 00 10 20 30
-// 01 11 21 31
-// 02 12 22 32
-// 03 13 23 33
-static INLINE void transpose_u16_4x4(uint16x4_t a[4]) {
+static INLINE void transpose_array_inplace_u16_4x4(uint16x4_t a[4]) {
+ // Input:
+ // 00 01 02 03
+ // 10 11 12 13
+ // 20 21 22 23
+ // 30 31 32 33
+
// b:
// 00 10 02 12
// 01 11 03 13
@@ -221,23 +219,25 @@
// 03 13 23 33
const uint32x2x2_t e =
vtrn_u32(vreinterpret_u32_u16(b.val[1]), vreinterpret_u32_u16(c.val[1]));
+
+ // Output:
+ // 00 10 20 30
+ // 01 11 21 31
+ // 02 12 22 32
+ // 03 13 23 33
a[0] = vreinterpret_u16_u32(d.val[0]);
a[1] = vreinterpret_u16_u32(e.val[0]);
a[2] = vreinterpret_u16_u32(d.val[1]);
a[3] = vreinterpret_u16_u32(e.val[1]);
}
-// 4x8 Input:
-// a[0]: 00 01 02 03 04 05 06 07
-// a[1]: 10 11 12 13 14 15 16 17
-// a[2]: 20 21 22 23 24 25 26 27
-// a[3]: 30 31 32 33 34 35 36 37
-// 8x4 Output:
-// a[0]: 00 10 20 30 04 14 24 34
-// a[1]: 01 11 21 31 05 15 25 35
-// a[2]: 02 12 22 32 06 16 26 36
-// a[3]: 03 13 23 33 07 17 27 37
-static INLINE void transpose_u16_4x8q(uint16x8_t a[4]) {
+static INLINE void transpose_array_inplace_u16_4x8(uint16x8_t a[4]) {
+ // 4x8 Input:
+ // a[0]: 00 01 02 03 04 05 06 07
+ // a[1]: 10 11 12 13 14 15 16 17
+ // a[2]: 20 21 22 23 24 25 26 27
+ // a[3]: 30 31 32 33 34 35 36 37
+
// b0.val[0]: 00 10 02 12 04 14 06 16
// b0.val[1]: 01 11 03 13 05 15 07 17
// b1.val[0]: 20 30 22 32 24 34 26 36
@@ -254,6 +254,11 @@
const uint32x4x2_t c1 = vtrnq_u32(vreinterpretq_u32_u16(b0.val[1]),
vreinterpretq_u32_u16(b1.val[1]));
+ // 8x4 Output:
+ // a[0]: 00 10 20 30 04 14 24 34
+ // a[1]: 01 11 21 31 05 15 25 35
+ // a[2]: 02 12 22 32 06 16 26 36
+ // a[3]: 03 13 23 33 07 17 27 37
a[0] = vreinterpretq_u16_u32(c0.val[0]);
a[1] = vreinterpretq_u16_u32(c1.val[0]);
a[2] = vreinterpretq_u16_u32(c0.val[1]);
@@ -345,12 +350,11 @@
a[3] = d0.val[0]; // p3q3
}
-static INLINE void transpose_u16_4x8(uint16x4_t *a0, uint16x4_t *a1,
- uint16x4_t *a2, uint16x4_t *a3,
- uint16x4_t *a4, uint16x4_t *a5,
- uint16x4_t *a6, uint16x4_t *a7,
- uint16x8_t *o0, uint16x8_t *o1,
- uint16x8_t *o2, uint16x8_t *o3) {
+static INLINE void transpose_elems_u16_4x8(
+ const uint16x4_t a0, const uint16x4_t a1, const uint16x4_t a2,
+ const uint16x4_t a3, const uint16x4_t a4, const uint16x4_t a5,
+ const uint16x4_t a6, const uint16x4_t a7, uint16x8_t *o0, uint16x8_t *o1,
+ uint16x8_t *o2, uint16x8_t *o3) {
// Combine rows. Goes from:
// a0: 00 01 02 03
// a1: 10 11 12 13
@@ -366,10 +370,10 @@
// b2: 20 21 22 23 60 61 62 63
// b3: 30 31 32 33 70 71 72 73
- const uint16x8_t b0 = vcombine_u16(*a0, *a4);
- const uint16x8_t b1 = vcombine_u16(*a1, *a5);
- const uint16x8_t b2 = vcombine_u16(*a2, *a6);
- const uint16x8_t b3 = vcombine_u16(*a3, *a7);
+ const uint16x8_t b0 = vcombine_u16(a0, a4);
+ const uint16x8_t b1 = vcombine_u16(a1, a5);
+ const uint16x8_t b2 = vcombine_u16(a2, a6);
+ const uint16x8_t b3 = vcombine_u16(a3, a7);
// Swap 16 bit elements resulting in:
// c0.val[0]: 00 10 02 12 40 50 42 52
@@ -397,12 +401,11 @@
*o3 = vreinterpretq_u16_u32(d1.val[1]);
}
-static INLINE void transpose_s16_4x8(int16x4_t *a0, int16x4_t *a1,
- int16x4_t *a2, int16x4_t *a3,
- int16x4_t *a4, int16x4_t *a5,
- int16x4_t *a6, int16x4_t *a7,
- int16x8_t *o0, int16x8_t *o1,
- int16x8_t *o2, int16x8_t *o3) {
+static INLINE void transpose_elems_s16_4x8(
+ const int16x4_t a0, const int16x4_t a1, const int16x4_t a2,
+ const int16x4_t a3, const int16x4_t a4, const int16x4_t a5,
+ const int16x4_t a6, const int16x4_t a7, int16x8_t *o0, int16x8_t *o1,
+ int16x8_t *o2, int16x8_t *o3) {
// Combine rows. Goes from:
// a0: 00 01 02 03
// a1: 10 11 12 13
@@ -418,10 +421,10 @@
// b2: 20 21 22 23 60 61 62 63
// b3: 30 31 32 33 70 71 72 73
- const int16x8_t b0 = vcombine_s16(*a0, *a4);
- const int16x8_t b1 = vcombine_s16(*a1, *a5);
- const int16x8_t b2 = vcombine_s16(*a2, *a6);
- const int16x8_t b3 = vcombine_s16(*a3, *a7);
+ const int16x8_t b0 = vcombine_s16(a0, a4);
+ const int16x8_t b1 = vcombine_s16(a1, a5);
+ const int16x8_t b2 = vcombine_s16(a2, a6);
+ const int16x8_t b3 = vcombine_s16(a3, a7);
// Swap 16 bit elements resulting in:
// c0.val[0]: 00 10 02 12 40 50 42 52
@@ -449,10 +452,9 @@
*o3 = vreinterpretq_s16_s32(d1.val[1]);
}
-static INLINE void transpose_u16_8x8(uint16x8_t *a0, uint16x8_t *a1,
- uint16x8_t *a2, uint16x8_t *a3,
- uint16x8_t *a4, uint16x8_t *a5,
- uint16x8_t *a6, uint16x8_t *a7) {
+static INLINE void transpose_elems_inplace_u16_8x8(
+ uint16x8_t *a0, uint16x8_t *a1, uint16x8_t *a2, uint16x8_t *a3,
+ uint16x8_t *a4, uint16x8_t *a5, uint16x8_t *a6, uint16x8_t *a7) {
// Swap 16 bit elements. Goes from:
// a0: 00 01 02 03 04 05 06 07
// a1: 10 11 12 13 14 15 16 17
@@ -537,10 +539,11 @@
return b0;
}
-static INLINE void transpose_s16_8x8(int16x8_t *a0, int16x8_t *a1,
- int16x8_t *a2, int16x8_t *a3,
- int16x8_t *a4, int16x8_t *a5,
- int16x8_t *a6, int16x8_t *a7) {
+static INLINE void transpose_elems_inplace_s16_8x8(int16x8_t *a0, int16x8_t *a1,
+ int16x8_t *a2, int16x8_t *a3,
+ int16x8_t *a4, int16x8_t *a5,
+ int16x8_t *a6,
+ int16x8_t *a7) {
// Swap 16 bit elements. Goes from:
// a0: 00 01 02 03 04 05 06 07
// a1: 10 11 12 13 14 15 16 17
@@ -609,7 +612,8 @@
*a7 = d3.val[1];
}
-static INLINE void transpose_s16_8x8q(int16x8_t *a, int16x8_t *out) {
+static INLINE void transpose_arrays_s16_8x8(const int16x8_t *a,
+ int16x8_t *out) {
// Swap 16 bit elements. Goes from:
// a0: 00 01 02 03 04 05 06 07
// a1: 10 11 12 13 14 15 16 17
@@ -678,8 +682,10 @@
out[7] = d3.val[1];
}
-static INLINE void transpose_u16_4x4d(uint16x4_t *a0, uint16x4_t *a1,
- uint16x4_t *a2, uint16x4_t *a3) {
+static INLINE void transpose_elems_inplace_u16_4x4(uint16x4_t *a0,
+ uint16x4_t *a1,
+ uint16x4_t *a2,
+ uint16x4_t *a3) {
// Swap 16 bit elements. Goes from:
// a0: 00 01 02 03
// a1: 10 11 12 13
@@ -711,8 +717,9 @@
*a3 = vreinterpret_u16_u32(c1.val[1]);
}
-static INLINE void transpose_s16_4x4d(int16x4_t *a0, int16x4_t *a1,
- int16x4_t *a2, int16x4_t *a3) {
+static INLINE void transpose_elems_inplace_s16_4x4(int16x4_t *a0, int16x4_t *a1,
+ int16x4_t *a2,
+ int16x4_t *a3) {
// Swap 16 bit elements. Goes from:
// a0: 00 01 02 03
// a1: 10 11 12 13
@@ -758,8 +765,12 @@
return b0;
}
-static INLINE void transpose_s32_4x4(int32x4_t *a0, int32x4_t *a1,
- int32x4_t *a2, int32x4_t *a3) {
+static INLINE void transpose_elems_s32_4x4(const int32x4_t a0,
+ const int32x4_t a1,
+ const int32x4_t a2,
+ const int32x4_t a3, int32x4_t *o0,
+ int32x4_t *o1, int32x4_t *o2,
+ int32x4_t *o3) {
// Swap 32 bit elements. Goes from:
// a0: 00 01 02 03
// a1: 10 11 12 13
@@ -771,8 +782,8 @@
// b1.val[0]: 20 30 22 32
// b1.val[1]: 21 31 23 33
- const int32x4x2_t b0 = vtrnq_s32(*a0, *a1);
- const int32x4x2_t b1 = vtrnq_s32(*a2, *a3);
+ const int32x4x2_t b0 = vtrnq_s32(a0, a1);
+ const int32x4x2_t b1 = vtrnq_s32(a2, a3);
// Swap 64 bit elements resulting in:
// c0.val[0]: 00 10 20 30
@@ -783,10 +794,267 @@
const int32x4x2_t c0 = aom_vtrnq_s64_to_s32(b0.val[0], b1.val[0]);
const int32x4x2_t c1 = aom_vtrnq_s64_to_s32(b0.val[1], b1.val[1]);
- *a0 = c0.val[0];
- *a1 = c1.val[0];
- *a2 = c0.val[1];
- *a3 = c1.val[1];
+ *o0 = c0.val[0];
+ *o1 = c1.val[0];
+ *o2 = c0.val[1];
+ *o3 = c1.val[1];
+}
+
+static INLINE void transpose_elems_inplace_s32_4x4(int32x4_t *a0, int32x4_t *a1,
+ int32x4_t *a2,
+ int32x4_t *a3) {
+ transpose_elems_s32_4x4(*a0, *a1, *a2, *a3, a0, a1, a2, a3);
+}
+
+static INLINE void transpose_arrays_s32_4x4(const int32x4_t *in,
+ int32x4_t *out) {
+ transpose_elems_s32_4x4(in[0], in[1], in[2], in[3], &out[0], &out[1], &out[2],
+ &out[3]);
+}
+
+static AOM_FORCE_INLINE void transpose_arrays_s32_4nx4n(const int32x4_t *in,
+ int32x4_t *out,
+ const int width,
+ const int height) {
+ const int h = height >> 2;
+ const int w = width >> 2;
+ for (int j = 0; j < w; j++) {
+ for (int i = 0; i < h; i++) {
+ transpose_arrays_s32_4x4(in + j * height + i * 4,
+ out + i * width + j * 4);
+ }
+ }
+}
+
+#define TRANSPOSE_ARRAYS_S32_WXH_NEON(w, h) \
+ static AOM_FORCE_INLINE void transpose_arrays_s32_##w##x##h( \
+ const int32x4_t *in, int32x4_t *out) { \
+ transpose_arrays_s32_4nx4n(in, out, w, h); \
+ }
+
+TRANSPOSE_ARRAYS_S32_WXH_NEON(4, 8)
+TRANSPOSE_ARRAYS_S32_WXH_NEON(4, 16)
+TRANSPOSE_ARRAYS_S32_WXH_NEON(8, 4)
+TRANSPOSE_ARRAYS_S32_WXH_NEON(8, 8)
+TRANSPOSE_ARRAYS_S32_WXH_NEON(8, 16)
+TRANSPOSE_ARRAYS_S32_WXH_NEON(8, 32)
+TRANSPOSE_ARRAYS_S32_WXH_NEON(16, 8)
+TRANSPOSE_ARRAYS_S32_WXH_NEON(16, 16)
+TRANSPOSE_ARRAYS_S32_WXH_NEON(16, 32)
+TRANSPOSE_ARRAYS_S32_WXH_NEON(16, 64)
+TRANSPOSE_ARRAYS_S32_WXH_NEON(32, 8)
+TRANSPOSE_ARRAYS_S32_WXH_NEON(32, 16)
+TRANSPOSE_ARRAYS_S32_WXH_NEON(32, 32)
+TRANSPOSE_ARRAYS_S32_WXH_NEON(32, 64)
+TRANSPOSE_ARRAYS_S32_WXH_NEON(64, 16)
+TRANSPOSE_ARRAYS_S32_WXH_NEON(64, 32)
+
+#undef TRANSPOSE_ARRAYS_S32_WXH_NEON
+
+static INLINE int64x2_t aom_vtrn1q_s64(int64x2_t a, int64x2_t b) {
+#if AOM_ARCH_AARCH64
+ return vtrn1q_s64(a, b);
+#else
+ return vcombine_s64(vget_low_s64(a), vget_low_s64(b));
+#endif
+}
+
+static INLINE int64x2_t aom_vtrn2q_s64(int64x2_t a, int64x2_t b) {
+#if AOM_ARCH_AARCH64
+ return vtrn2q_s64(a, b);
+#else
+ return vcombine_s64(vget_high_s64(a), vget_high_s64(b));
+#endif
+}
+
+static INLINE void transpose_elems_s32_4x8(int32x4_t a0, int32x4_t a1,
+ int32x4_t a2, int32x4_t a3,
+ int32x4_t a4, int32x4_t a5,
+ int32x4_t a6, int32x4_t a7,
+ int32x4x2_t *o0, int32x4x2_t *o1,
+ int32x4x2_t *o2, int32x4x2_t *o3) {
+ // Perform a 4 x 8 matrix transpose by building on top of the existing 4 x 4
+ // matrix transpose implementation:
+ // [ A ]^T => [ A^T B^T ]
+ // [ B ]
+
+ transpose_elems_inplace_s32_4x4(&a0, &a1, &a2, &a3); // A^T
+ transpose_elems_inplace_s32_4x4(&a4, &a5, &a6, &a7); // B^T
+
+ o0->val[0] = a0;
+ o1->val[0] = a1;
+ o2->val[0] = a2;
+ o3->val[0] = a3;
+
+ o0->val[1] = a4;
+ o1->val[1] = a5;
+ o2->val[1] = a6;
+ o3->val[1] = a7;
+}
+
+static INLINE void transpose_elems_inplace_s32_8x8(
+ int32x4x2_t *a0, int32x4x2_t *a1, int32x4x2_t *a2, int32x4x2_t *a3,
+ int32x4x2_t *a4, int32x4x2_t *a5, int32x4x2_t *a6, int32x4x2_t *a7) {
+ // Perform an 8 x 8 matrix transpose by building on top of the existing 4 x 4
+ // matrix transpose implementation:
+ // [ A B ]^T => [ A^T C^T ]
+ // [ C D ] [ B^T D^T ]
+
+ int32x4_t q0_v1 = a0->val[0];
+ int32x4_t q0_v2 = a1->val[0];
+ int32x4_t q0_v3 = a2->val[0];
+ int32x4_t q0_v4 = a3->val[0];
+
+ int32x4_t q1_v1 = a0->val[1];
+ int32x4_t q1_v2 = a1->val[1];
+ int32x4_t q1_v3 = a2->val[1];
+ int32x4_t q1_v4 = a3->val[1];
+
+ int32x4_t q2_v1 = a4->val[0];
+ int32x4_t q2_v2 = a5->val[0];
+ int32x4_t q2_v3 = a6->val[0];
+ int32x4_t q2_v4 = a7->val[0];
+
+ int32x4_t q3_v1 = a4->val[1];
+ int32x4_t q3_v2 = a5->val[1];
+ int32x4_t q3_v3 = a6->val[1];
+ int32x4_t q3_v4 = a7->val[1];
+
+ transpose_elems_inplace_s32_4x4(&q0_v1, &q0_v2, &q0_v3, &q0_v4); // A^T
+ transpose_elems_inplace_s32_4x4(&q1_v1, &q1_v2, &q1_v3, &q1_v4); // B^T
+ transpose_elems_inplace_s32_4x4(&q2_v1, &q2_v2, &q2_v3, &q2_v4); // C^T
+ transpose_elems_inplace_s32_4x4(&q3_v1, &q3_v2, &q3_v3, &q3_v4); // D^T
+
+ a0->val[0] = q0_v1;
+ a1->val[0] = q0_v2;
+ a2->val[0] = q0_v3;
+ a3->val[0] = q0_v4;
+
+ a0->val[1] = q2_v1;
+ a1->val[1] = q2_v2;
+ a2->val[1] = q2_v3;
+ a3->val[1] = q2_v4;
+
+ a4->val[0] = q1_v1;
+ a5->val[0] = q1_v2;
+ a6->val[0] = q1_v3;
+ a7->val[0] = q1_v4;
+
+ a4->val[1] = q3_v1;
+ a5->val[1] = q3_v2;
+ a6->val[1] = q3_v3;
+ a7->val[1] = q3_v4;
+}
+
+static INLINE void transpose_arrays_s16_4x4(const int16x4_t *const in,
+ int16x4_t *const out) {
+ int16x4_t a0 = in[0];
+ int16x4_t a1 = in[1];
+ int16x4_t a2 = in[2];
+ int16x4_t a3 = in[3];
+
+ transpose_elems_inplace_s16_4x4(&a0, &a1, &a2, &a3);
+
+ out[0] = a0;
+ out[1] = a1;
+ out[2] = a2;
+ out[3] = a3;
+}
+
+static INLINE void transpose_arrays_s16_4x8(const int16x4_t *const in,
+ int16x8_t *const out) {
+#if AOM_ARCH_AARCH64
+ const int16x8_t a0 = vzip1q_s16(vcombine_s16(in[0], vdup_n_s16(0)),
+ vcombine_s16(in[1], vdup_n_s16(0)));
+ const int16x8_t a1 = vzip1q_s16(vcombine_s16(in[2], vdup_n_s16(0)),
+ vcombine_s16(in[3], vdup_n_s16(0)));
+ const int16x8_t a2 = vzip1q_s16(vcombine_s16(in[4], vdup_n_s16(0)),
+ vcombine_s16(in[5], vdup_n_s16(0)));
+ const int16x8_t a3 = vzip1q_s16(vcombine_s16(in[6], vdup_n_s16(0)),
+ vcombine_s16(in[7], vdup_n_s16(0)));
+#else
+ int16x4x2_t temp;
+ temp = vzip_s16(in[0], in[1]);
+ const int16x8_t a0 = vcombine_s16(temp.val[0], temp.val[1]);
+ temp = vzip_s16(in[2], in[3]);
+ const int16x8_t a1 = vcombine_s16(temp.val[0], temp.val[1]);
+ temp = vzip_s16(in[4], in[5]);
+ const int16x8_t a2 = vcombine_s16(temp.val[0], temp.val[1]);
+ temp = vzip_s16(in[6], in[7]);
+ const int16x8_t a3 = vcombine_s16(temp.val[0], temp.val[1]);
+#endif
+
+ const int32x4x2_t b02 =
+ vzipq_s32(vreinterpretq_s32_s16(a0), vreinterpretq_s32_s16(a1));
+ const int32x4x2_t b13 =
+ vzipq_s32(vreinterpretq_s32_s16(a2), vreinterpretq_s32_s16(a3));
+
+#if AOM_ARCH_AARCH64
+ out[0] = vreinterpretq_s16_s64(vzip1q_s64(vreinterpretq_s64_s32(b02.val[0]),
+ vreinterpretq_s64_s32(b13.val[0])));
+ out[1] = vreinterpretq_s16_s64(vzip2q_s64(vreinterpretq_s64_s32(b02.val[0]),
+ vreinterpretq_s64_s32(b13.val[0])));
+ out[2] = vreinterpretq_s16_s64(vzip1q_s64(vreinterpretq_s64_s32(b02.val[1]),
+ vreinterpretq_s64_s32(b13.val[1])));
+ out[3] = vreinterpretq_s16_s64(vzip2q_s64(vreinterpretq_s64_s32(b02.val[1]),
+ vreinterpretq_s64_s32(b13.val[1])));
+#else
+ out[0] = vreinterpretq_s16_s32(
+ vextq_s32(vextq_s32(b02.val[0], b02.val[0], 2), b13.val[0], 2));
+ out[2] = vreinterpretq_s16_s32(
+ vextq_s32(vextq_s32(b02.val[1], b02.val[1], 2), b13.val[1], 2));
+ out[1] = vreinterpretq_s16_s32(
+ vextq_s32(b02.val[0], vextq_s32(b13.val[0], b13.val[0], 2), 2));
+ out[3] = vreinterpretq_s16_s32(
+ vextq_s32(b02.val[1], vextq_s32(b13.val[1], b13.val[1], 2), 2));
+#endif
+}
+
+static INLINE void transpose_arrays_s16_8x4(const int16x8_t *const in,
+ int16x4_t *const out) {
+ // Swap 16 bit elements. Goes from:
+ // in[0]: 00 01 02 03 04 05 06 07
+ // in[1]: 10 11 12 13 14 15 16 17
+ // in[2]: 20 21 22 23 24 25 26 27
+ // in[3]: 30 31 32 33 34 35 36 37
+ // to:
+ // b0.val[0]: 00 10 02 12 04 14 06 16
+ // b0.val[1]: 01 11 03 13 05 15 07 17
+ // b1.val[0]: 20 30 22 32 24 34 26 36
+ // b1.val[1]: 21 31 23 33 25 35 27 37
+
+ const int16x8x2_t b0 = vtrnq_s16(in[0], in[1]);
+ const int16x8x2_t b1 = vtrnq_s16(in[2], in[3]);
+
+ // Swap 32 bit elements resulting in:
+ // c0.val[0]: 00 10 20 30 04 14 24 34
+ // c0.val[1]: 02 12 22 32 06 16 26 36
+ // c1.val[0]: 01 11 21 31 05 15 25 35
+ // c1.val[1]: 03 13 23 33 07 17 27 37
+
+ const uint32x4x2_t c0 = vtrnq_u32(vreinterpretq_u32_s16(b0.val[0]),
+ vreinterpretq_u32_s16(b1.val[0]));
+ const uint32x4x2_t c1 = vtrnq_u32(vreinterpretq_u32_s16(b0.val[1]),
+ vreinterpretq_u32_s16(b1.val[1]));
+
+ // Unpack 64 bit elements resulting in:
+ // out[0]: 00 10 20 30
+ // out[1]: 01 11 21 31
+ // out[2]: 02 12 22 32
+ // out[3]: 03 13 23 33
+ // out[4]: 04 14 24 34
+ // out[5]: 05 15 25 35
+ // out[6]: 06 16 26 36
+ // out[7]: 07 17 27 37
+
+ out[0] = vget_low_s16(vreinterpretq_s16_u32(c0.val[0]));
+ out[1] = vget_low_s16(vreinterpretq_s16_u32(c1.val[0]));
+ out[2] = vget_low_s16(vreinterpretq_s16_u32(c0.val[1]));
+ out[3] = vget_low_s16(vreinterpretq_s16_u32(c1.val[1]));
+ out[4] = vget_high_s16(vreinterpretq_s16_u32(c0.val[0]));
+ out[5] = vget_high_s16(vreinterpretq_s16_u32(c1.val[0]));
+ out[6] = vget_high_s16(vreinterpretq_s16_u32(c0.val[1]));
+ out[7] = vget_high_s16(vreinterpretq_s16_u32(c1.val[1]));
}
#endif // AOM_AOM_DSP_ARM_TRANSPOSE_NEON_H_
diff --git a/aom_dsp/arm/variance_neon.c b/aom_dsp/arm/variance_neon.c
index 5e33996..9e4e8c0 100644
--- a/aom_dsp/arm/variance_neon.c
+++ b/aom_dsp/arm/variance_neon.c
@@ -11,153 +11,12 @@
#include <arm_neon.h>
-#include "config/aom_dsp_rtcd.h"
-#include "config/aom_config.h"
+#include "aom/aom_integer.h"
#include "aom_dsp/arm/mem_neon.h"
#include "aom_dsp/arm/sum_neon.h"
-#include "aom/aom_integer.h"
#include "aom_ports/mem.h"
-
-#if defined(__ARM_FEATURE_DOTPROD)
-
-static INLINE void variance_4xh_neon(const uint8_t *src, int src_stride,
- const uint8_t *ref, int ref_stride, int h,
- uint32_t *sse, int *sum) {
- uint32x4_t src_sum = vdupq_n_u32(0);
- uint32x4_t ref_sum = vdupq_n_u32(0);
- uint32x4_t sse_u32 = vdupq_n_u32(0);
-
- int i = h;
- do {
- uint8x16_t s = load_unaligned_u8q(src, src_stride);
- uint8x16_t r = load_unaligned_u8q(ref, ref_stride);
-
- src_sum = vdotq_u32(src_sum, s, vdupq_n_u8(1));
- ref_sum = vdotq_u32(ref_sum, r, vdupq_n_u8(1));
-
- uint8x16_t abs_diff = vabdq_u8(s, r);
- sse_u32 = vdotq_u32(sse_u32, abs_diff, abs_diff);
-
- src += 4 * src_stride;
- ref += 4 * ref_stride;
- i -= 4;
- } while (i != 0);
-
- int32x4_t sum_diff =
- vsubq_s32(vreinterpretq_s32_u32(src_sum), vreinterpretq_s32_u32(ref_sum));
- *sum = horizontal_add_s32x4(sum_diff);
- *sse = horizontal_add_u32x4(sse_u32);
-}
-
-static INLINE void variance_8xh_neon(const uint8_t *src, int src_stride,
- const uint8_t *ref, int ref_stride, int h,
- uint32_t *sse, int *sum) {
- uint32x4_t src_sum = vdupq_n_u32(0);
- uint32x4_t ref_sum = vdupq_n_u32(0);
- uint32x4_t sse_u32 = vdupq_n_u32(0);
-
- int i = h;
- do {
- uint8x16_t s = vcombine_u8(vld1_u8(src), vld1_u8(src + src_stride));
- uint8x16_t r = vcombine_u8(vld1_u8(ref), vld1_u8(ref + ref_stride));
-
- src_sum = vdotq_u32(src_sum, s, vdupq_n_u8(1));
- ref_sum = vdotq_u32(ref_sum, r, vdupq_n_u8(1));
-
- uint8x16_t abs_diff = vabdq_u8(s, r);
- sse_u32 = vdotq_u32(sse_u32, abs_diff, abs_diff);
-
- src += 2 * src_stride;
- ref += 2 * ref_stride;
- i -= 2;
- } while (i != 0);
-
- int32x4_t sum_diff =
- vsubq_s32(vreinterpretq_s32_u32(src_sum), vreinterpretq_s32_u32(ref_sum));
- *sum = horizontal_add_s32x4(sum_diff);
- *sse = horizontal_add_u32x4(sse_u32);
-}
-
-static INLINE void variance_16xh_neon(const uint8_t *src, int src_stride,
- const uint8_t *ref, int ref_stride, int h,
- uint32_t *sse, int *sum) {
- uint32x4_t src_sum = vdupq_n_u32(0);
- uint32x4_t ref_sum = vdupq_n_u32(0);
- uint32x4_t sse_u32 = vdupq_n_u32(0);
-
- int i = h;
- do {
- uint8x16_t s = vld1q_u8(src);
- uint8x16_t r = vld1q_u8(ref);
-
- src_sum = vdotq_u32(src_sum, s, vdupq_n_u8(1));
- ref_sum = vdotq_u32(ref_sum, r, vdupq_n_u8(1));
-
- uint8x16_t abs_diff = vabdq_u8(s, r);
- sse_u32 = vdotq_u32(sse_u32, abs_diff, abs_diff);
-
- src += src_stride;
- ref += ref_stride;
- } while (--i != 0);
-
- int32x4_t sum_diff =
- vsubq_s32(vreinterpretq_s32_u32(src_sum), vreinterpretq_s32_u32(ref_sum));
- *sum = horizontal_add_s32x4(sum_diff);
- *sse = horizontal_add_u32x4(sse_u32);
-}
-
-static INLINE void variance_large_neon(const uint8_t *src, int src_stride,
- const uint8_t *ref, int ref_stride,
- int w, int h, uint32_t *sse, int *sum) {
- uint32x4_t src_sum = vdupq_n_u32(0);
- uint32x4_t ref_sum = vdupq_n_u32(0);
- uint32x4_t sse_u32 = vdupq_n_u32(0);
-
- int i = h;
- do {
- int j = 0;
- do {
- uint8x16_t s = vld1q_u8(src + j);
- uint8x16_t r = vld1q_u8(ref + j);
-
- src_sum = vdotq_u32(src_sum, s, vdupq_n_u8(1));
- ref_sum = vdotq_u32(ref_sum, r, vdupq_n_u8(1));
-
- uint8x16_t abs_diff = vabdq_u8(s, r);
- sse_u32 = vdotq_u32(sse_u32, abs_diff, abs_diff);
-
- j += 16;
- } while (j < w);
-
- src += src_stride;
- ref += ref_stride;
- } while (--i != 0);
-
- int32x4_t sum_diff =
- vsubq_s32(vreinterpretq_s32_u32(src_sum), vreinterpretq_s32_u32(ref_sum));
- *sum = horizontal_add_s32x4(sum_diff);
- *sse = horizontal_add_u32x4(sse_u32);
-}
-
-static INLINE void variance_32xh_neon(const uint8_t *src, int src_stride,
- const uint8_t *ref, int ref_stride, int h,
- uint32_t *sse, int *sum) {
- variance_large_neon(src, src_stride, ref, ref_stride, 32, h, sse, sum);
-}
-
-static INLINE void variance_64xh_neon(const uint8_t *src, int src_stride,
- const uint8_t *ref, int ref_stride, int h,
- uint32_t *sse, int *sum) {
- variance_large_neon(src, src_stride, ref, ref_stride, 64, h, sse, sum);
-}
-
-static INLINE void variance_128xh_neon(const uint8_t *src, int src_stride,
- const uint8_t *ref, int ref_stride,
- int h, uint32_t *sse, int *sum) {
- variance_large_neon(src, src_stride, ref, ref_stride, 128, h, sse, sum);
-}
-
-#else // !defined(__ARM_FEATURE_DOTPROD)
+#include "config/aom_config.h"
+#include "config/aom_dsp_rtcd.h"
static INLINE void variance_4xh_neon(const uint8_t *src, int src_stride,
const uint8_t *ref, int ref_stride, int h,
@@ -333,8 +192,6 @@
variance_large_neon(src, src_stride, ref, ref_stride, 128, h, 16, sse, sum);
}
-#endif // defined(__ARM_FEATURE_DOTPROD)
-
#define VARIANCE_WXH_NEON(w, h, shift) \
unsigned int aom_variance##w##x##h##_neon( \
const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
@@ -382,7 +239,7 @@
uint32_t *sse8x8, int *sum8x8,
unsigned int *tot_sse, int *tot_sum,
uint32_t *var8x8) {
- // Loop over 4 8x8 blocks. Process one 8x32 block.
+ // Loop over four 8x8 blocks. Process one 8x32 block.
for (int k = 0; k < 4; k++) {
variance_8xh_neon(src + (k * 8), src_stride, ref + (k * 8), ref_stride, 8,
&sse8x8[k], &sum8x8[k]);
@@ -390,8 +247,9 @@
*tot_sse += sse8x8[0] + sse8x8[1] + sse8x8[2] + sse8x8[3];
*tot_sum += sum8x8[0] + sum8x8[1] + sum8x8[2] + sum8x8[3];
- for (int i = 0; i < 4; i++)
+ for (int i = 0; i < 4; i++) {
var8x8[i] = sse8x8[i] - (uint32_t)(((int64_t)sum8x8[i] * sum8x8[i]) >> 6);
+ }
}
void aom_get_var_sse_sum_16x16_dual_neon(const uint8_t *src, int src_stride,
@@ -400,7 +258,7 @@
unsigned int *tot_sse, int *tot_sum,
uint32_t *var16x16) {
int sum16x16[2] = { 0 };
- // Loop over 2 16x16 blocks. Process one 16x32 block.
+ // Loop over two 16x16 blocks. Process one 16x32 block.
for (int k = 0; k < 2; k++) {
variance_16xh_neon(src + (k * 16), src_stride, ref + (k * 16), ref_stride,
16, &sse16x16[k], &sum16x16[k]);
@@ -408,65 +266,12 @@
*tot_sse += sse16x16[0] + sse16x16[1];
*tot_sum += sum16x16[0] + sum16x16[1];
- for (int i = 0; i < 2; i++)
+ for (int i = 0; i < 2; i++) {
var16x16[i] =
sse16x16[i] - (uint32_t)(((int64_t)sum16x16[i] * sum16x16[i]) >> 8);
+ }
}
-#if defined(__ARM_FEATURE_DOTPROD)
-
-static INLINE unsigned int mse8xh_neon(const uint8_t *src, int src_stride,
- const uint8_t *ref, int ref_stride,
- unsigned int *sse, int h) {
- uint32x4_t sse_u32 = vdupq_n_u32(0);
-
- int i = h;
- do {
- uint8x16_t s = vcombine_u8(vld1_u8(src), vld1_u8(src + src_stride));
- uint8x16_t r = vcombine_u8(vld1_u8(ref), vld1_u8(ref + ref_stride));
-
- uint8x16_t abs_diff = vabdq_u8(s, r);
-
- sse_u32 = vdotq_u32(sse_u32, abs_diff, abs_diff);
-
- src += 2 * src_stride;
- ref += 2 * ref_stride;
- i -= 2;
- } while (i != 0);
-
- *sse = horizontal_add_u32x4(sse_u32);
- return horizontal_add_u32x4(sse_u32);
-}
-
-static INLINE unsigned int mse16xh_neon(const uint8_t *src, int src_stride,
- const uint8_t *ref, int ref_stride,
- unsigned int *sse, int h) {
- uint32x4_t sse_u32[2] = { vdupq_n_u32(0), vdupq_n_u32(0) };
-
- int i = h;
- do {
- uint8x16_t s0 = vld1q_u8(src);
- uint8x16_t s1 = vld1q_u8(src + src_stride);
- uint8x16_t r0 = vld1q_u8(ref);
- uint8x16_t r1 = vld1q_u8(ref + ref_stride);
-
- uint8x16_t abs_diff0 = vabdq_u8(s0, r0);
- uint8x16_t abs_diff1 = vabdq_u8(s1, r1);
-
- sse_u32[0] = vdotq_u32(sse_u32[0], abs_diff0, abs_diff0);
- sse_u32[1] = vdotq_u32(sse_u32[1], abs_diff1, abs_diff1);
-
- src += 2 * src_stride;
- ref += 2 * ref_stride;
- i -= 2;
- } while (i != 0);
-
- *sse = horizontal_add_u32x4(vaddq_u32(sse_u32[0], sse_u32[1]));
- return horizontal_add_u32x4(vaddq_u32(sse_u32[0], sse_u32[1]));
-}
-
-#else // !defined(__ARM_FEATURE_DOTPROD)
-
static INLINE unsigned int mse8xh_neon(const uint8_t *src, int src_stride,
const uint8_t *ref, int ref_stride,
unsigned int *sse, int h) {
@@ -564,8 +369,6 @@
return horizontal_add_u32x4(vreinterpretq_u32_s32(sse_s32[0]));
}
-#endif // defined(__ARM_FEATURE_DOTPROD)
-
#define MSE_WXH_NEON(w, h) \
unsigned int aom_mse##w##x##h##_neon(const uint8_t *src, int src_stride, \
const uint8_t *ref, int ref_stride, \
@@ -581,98 +384,87 @@
#undef MSE_WXH_NEON
-#define COMPUTE_MSE_16BIT(src_16x8, dst_16x8) \
- /* r7 r6 r5 r4 r3 r2 r1 r0 - 16 bit */ \
- const uint16x8_t diff = vabdq_u16(src_16x8, dst_16x8); \
- /*r3 r2 r1 r0 - 16 bit */ \
- const uint16x4_t res0_low_16x4 = vget_low_u16(diff); \
- /*r7 r6 r5 r4 - 16 bit */ \
- const uint16x4_t res0_high_16x4 = vget_high_u16(diff); \
- /* (r3*r3)= b3 (r2*r2)= b2 (r1*r1)= b1 (r0*r0)= b0 - 32 bit */ \
- const uint32x4_t res0_32x4 = vmull_u16(res0_low_16x4, res0_low_16x4); \
- /* (r7*r7)= b7 (r6*r6)= b6 (r5*r5)= b5 (r4*r4)= b4 - 32 bit*/ \
- /* b3+b7 b2+b6 b1+b5 b0+b4 - 32 bit*/ \
- const uint32x4_t res_32x4 = \
- vmlal_u16(res0_32x4, res0_high_16x4, res0_high_16x4); \
- \
- /*a1 a0 - 64 bit*/ \
- const uint64x2_t vl = vpaddlq_u32(res_32x4); \
- /*a1+a2= f1 a3+a0= f0*/ \
- square_result = vaddq_u64(square_result, vl);
+static INLINE uint64x2_t mse_accumulate_u16_u8_8x2(uint64x2_t sum,
+ uint16x8_t s0, uint16x8_t s1,
+ uint8x8_t d0, uint8x8_t d1) {
+ int16x8_t e0 = vreinterpretq_s16_u16(vsubw_u8(s0, d0));
+ int16x8_t e1 = vreinterpretq_s16_u16(vsubw_u8(s1, d1));
-static AOM_INLINE uint64_t mse_4xh_16bit_neon(uint8_t *dst, int dstride,
- uint16_t *src, int sstride,
- int h) {
- uint64x2_t square_result = vdupq_n_u64(0);
- uint32_t d0, d1;
- int i = h;
- uint8_t *dst_ptr = dst;
- uint16_t *src_ptr = src;
- do {
- // d03 d02 d01 d00 - 8 bit
- memcpy(&d0, dst_ptr, 4);
- dst_ptr += dstride;
- // d13 d12 d11 d10 - 8 bit
- memcpy(&d1, dst_ptr, 4);
- dst_ptr += dstride;
- // duplication
- uint8x8_t tmp0_8x8 = vreinterpret_u8_u32(vdup_n_u32(d0));
- // d03 d02 d01 d00 - 16 bit
- const uint16x4_t dst0_16x4 = vget_low_u16(vmovl_u8(tmp0_8x8));
- // duplication
- tmp0_8x8 = vreinterpret_u8_u32(vdup_n_u32(d1));
- // d13 d12 d11 d10 - 16 bit
- const uint16x4_t dst1_16x4 = vget_low_u16(vmovl_u8(tmp0_8x8));
- // d13 d12 d11 d10 d03 d02 d01 d00 - 16 bit
- const uint16x8_t dst_16x8 = vcombine_u16(dst0_16x4, dst1_16x4);
+ int32x4_t mse = vmull_s16(vget_low_s16(e0), vget_low_s16(e0));
+ mse = vmlal_s16(mse, vget_high_s16(e0), vget_high_s16(e0));
+ mse = vmlal_s16(mse, vget_low_s16(e1), vget_low_s16(e1));
+ mse = vmlal_s16(mse, vget_high_s16(e1), vget_high_s16(e1));
- // b1r0 - s03 s02 s01 s00 - 16 bit
- const uint16x4_t src0_16x4 = vld1_u16(src_ptr);
- src_ptr += sstride;
- // b1r1 - s13 s12 s11 s10 - 16 bit
- const uint16x4_t src1_16x4 = vld1_u16(src_ptr);
- src_ptr += sstride;
- // s13 s12 s11 s10 s03 s02 s01 s00 - 16 bit
- const uint16x8_t src_16x8 = vcombine_u16(src0_16x4, src1_16x4);
-
- COMPUTE_MSE_16BIT(src_16x8, dst_16x8)
- i -= 2;
- } while (i != 0);
- uint64x1_t sum =
- vadd_u64(vget_high_u64(square_result), vget_low_u64(square_result));
- return vget_lane_u64(sum, 0);
+ return vpadalq_u32(sum, vreinterpretq_u32_s32(mse));
}
-static AOM_INLINE uint64_t mse_8xh_16bit_neon(uint8_t *dst, int dstride,
- uint16_t *src, int sstride,
- int h) {
- uint64x2_t square_result = vdupq_n_u64(0);
- int i = h;
- do {
- // d7 d6 d5 d4 d3 d2 d1 d0 - 8 bit
- const uint16x8_t dst_16x8 = vmovl_u8(vld1_u8(dst));
- // s7 s6 s5 s4 s3 s2 s1 s0 - 16 bit
- const uint16x8_t src_16x8 = vld1q_u16(src);
+static uint64x2_t mse_wxh_16bit(uint8_t *dst, int dstride, const uint16_t *src,
+ int sstride, int w, int h) {
+ assert((w == 8 || w == 4) && (h == 8 || h == 4));
- COMPUTE_MSE_16BIT(src_16x8, dst_16x8)
+ uint64x2_t sum = vdupq_n_u64(0);
- dst += dstride;
- src += sstride;
- } while (--i != 0);
- uint64x1_t sum =
- vadd_u64(vget_high_u64(square_result), vget_low_u64(square_result));
- return vget_lane_u64(sum, 0);
+ if (w == 8) {
+ do {
+ uint8x8_t d0 = vld1_u8(dst + 0 * dstride);
+ uint8x8_t d1 = vld1_u8(dst + 1 * dstride);
+ uint16x8_t s0 = vld1q_u16(src + 0 * sstride);
+ uint16x8_t s1 = vld1q_u16(src + 1 * sstride);
+
+ sum = mse_accumulate_u16_u8_8x2(sum, s0, s1, d0, d1);
+
+ dst += 2 * dstride;
+ src += 2 * sstride;
+ h -= 2;
+ } while (h != 0);
+ } else {
+ do {
+ uint8x8_t d0 = load_unaligned_u8_4x2(dst + 0 * dstride, dstride);
+ uint8x8_t d1 = load_unaligned_u8_4x2(dst + 2 * dstride, dstride);
+ uint16x8_t s0 = load_unaligned_u16_4x2(src + 0 * sstride, sstride);
+ uint16x8_t s1 = load_unaligned_u16_4x2(src + 2 * sstride, sstride);
+
+ sum = mse_accumulate_u16_u8_8x2(sum, s0, s1, d0, d1);
+
+ dst += 4 * dstride;
+ src += 4 * sstride;
+ h -= 4;
+ } while (h != 0);
+ }
+
+ return sum;
}
// Computes mse for a given block size. This function gets called for specific
// block sizes, which are 8x8, 8x4, 4x8 and 4x4.
uint64_t aom_mse_wxh_16bit_neon(uint8_t *dst, int dstride, uint16_t *src,
int sstride, int w, int h) {
- assert((w == 8 || w == 4) && (h == 8 || h == 4) &&
- "w=8/4 and h=8/4 must satisfy");
- switch (w) {
- case 4: return mse_4xh_16bit_neon(dst, dstride, src, sstride, h);
- case 8: return mse_8xh_16bit_neon(dst, dstride, src, sstride, h);
- default: assert(0 && "unsupported width"); return -1;
+ return horizontal_add_u64x2(mse_wxh_16bit(dst, dstride, src, sstride, w, h));
+}
+
+uint32_t aom_get_mb_ss_neon(const int16_t *a) {
+ int32x4_t sse[2] = { vdupq_n_s32(0), vdupq_n_s32(0) };
+
+ for (int i = 0; i < 256; i = i + 8) {
+ int16x8_t a_s16 = vld1q_s16(a + i);
+
+ sse[0] = vmlal_s16(sse[0], vget_low_s16(a_s16), vget_low_s16(a_s16));
+ sse[1] = vmlal_s16(sse[1], vget_high_s16(a_s16), vget_high_s16(a_s16));
}
+
+ return horizontal_add_s32x4(vaddq_s32(sse[0], sse[1]));
+}
+
+uint64_t aom_mse_16xh_16bit_neon(uint8_t *dst, int dstride, uint16_t *src,
+ int w, int h) {
+ uint64x2_t sum = vdupq_n_u64(0);
+
+ int num_blks = 16 / w;
+ do {
+ sum = vaddq_u64(sum, mse_wxh_16bit(dst, dstride, src, w, w, h));
+ dst += w;
+ src += w * h;
+ } while (--num_blks != 0);
+
+ return horizontal_add_u64x2(sum);
}
diff --git a/aom_dsp/arm/variance_neon_dotprod.c b/aom_dsp/arm/variance_neon_dotprod.c
new file mode 100644
index 0000000..9fb52e1
--- /dev/null
+++ b/aom_dsp/arm/variance_neon_dotprod.c
@@ -0,0 +1,314 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+
+#include "aom/aom_integer.h"
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_dsp/arm/sum_neon.h"
+#include "aom_ports/mem.h"
+#include "config/aom_config.h"
+#include "config/aom_dsp_rtcd.h"
+
+static INLINE void variance_4xh_neon_dotprod(const uint8_t *src, int src_stride,
+ const uint8_t *ref, int ref_stride,
+ int h, uint32_t *sse, int *sum) {
+ uint32x4_t src_sum = vdupq_n_u32(0);
+ uint32x4_t ref_sum = vdupq_n_u32(0);
+ uint32x4_t sse_u32 = vdupq_n_u32(0);
+
+ int i = h;
+ do {
+ uint8x16_t s = load_unaligned_u8q(src, src_stride);
+ uint8x16_t r = load_unaligned_u8q(ref, ref_stride);
+
+ src_sum = vdotq_u32(src_sum, s, vdupq_n_u8(1));
+ ref_sum = vdotq_u32(ref_sum, r, vdupq_n_u8(1));
+
+ uint8x16_t abs_diff = vabdq_u8(s, r);
+ sse_u32 = vdotq_u32(sse_u32, abs_diff, abs_diff);
+
+ src += 4 * src_stride;
+ ref += 4 * ref_stride;
+ i -= 4;
+ } while (i != 0);
+
+ int32x4_t sum_diff =
+ vsubq_s32(vreinterpretq_s32_u32(src_sum), vreinterpretq_s32_u32(ref_sum));
+ *sum = horizontal_add_s32x4(sum_diff);
+ *sse = horizontal_add_u32x4(sse_u32);
+}
+
+static INLINE void variance_8xh_neon_dotprod(const uint8_t *src, int src_stride,
+ const uint8_t *ref, int ref_stride,
+ int h, uint32_t *sse, int *sum) {
+ uint32x4_t src_sum = vdupq_n_u32(0);
+ uint32x4_t ref_sum = vdupq_n_u32(0);
+ uint32x4_t sse_u32 = vdupq_n_u32(0);
+
+ int i = h;
+ do {
+ uint8x16_t s = vcombine_u8(vld1_u8(src), vld1_u8(src + src_stride));
+ uint8x16_t r = vcombine_u8(vld1_u8(ref), vld1_u8(ref + ref_stride));
+
+ src_sum = vdotq_u32(src_sum, s, vdupq_n_u8(1));
+ ref_sum = vdotq_u32(ref_sum, r, vdupq_n_u8(1));
+
+ uint8x16_t abs_diff = vabdq_u8(s, r);
+ sse_u32 = vdotq_u32(sse_u32, abs_diff, abs_diff);
+
+ src += 2 * src_stride;
+ ref += 2 * ref_stride;
+ i -= 2;
+ } while (i != 0);
+
+ int32x4_t sum_diff =
+ vsubq_s32(vreinterpretq_s32_u32(src_sum), vreinterpretq_s32_u32(ref_sum));
+ *sum = horizontal_add_s32x4(sum_diff);
+ *sse = horizontal_add_u32x4(sse_u32);
+}
+
+static INLINE void variance_16xh_neon_dotprod(const uint8_t *src,
+ int src_stride,
+ const uint8_t *ref,
+ int ref_stride, int h,
+ uint32_t *sse, int *sum) {
+ uint32x4_t src_sum = vdupq_n_u32(0);
+ uint32x4_t ref_sum = vdupq_n_u32(0);
+ uint32x4_t sse_u32 = vdupq_n_u32(0);
+
+ int i = h;
+ do {
+ uint8x16_t s = vld1q_u8(src);
+ uint8x16_t r = vld1q_u8(ref);
+
+ src_sum = vdotq_u32(src_sum, s, vdupq_n_u8(1));
+ ref_sum = vdotq_u32(ref_sum, r, vdupq_n_u8(1));
+
+ uint8x16_t abs_diff = vabdq_u8(s, r);
+ sse_u32 = vdotq_u32(sse_u32, abs_diff, abs_diff);
+
+ src += src_stride;
+ ref += ref_stride;
+ } while (--i != 0);
+
+ int32x4_t sum_diff =
+ vsubq_s32(vreinterpretq_s32_u32(src_sum), vreinterpretq_s32_u32(ref_sum));
+ *sum = horizontal_add_s32x4(sum_diff);
+ *sse = horizontal_add_u32x4(sse_u32);
+}
+
+static INLINE void variance_large_neon_dotprod(const uint8_t *src,
+ int src_stride,
+ const uint8_t *ref,
+ int ref_stride, int w, int h,
+ uint32_t *sse, int *sum) {
+ uint32x4_t src_sum = vdupq_n_u32(0);
+ uint32x4_t ref_sum = vdupq_n_u32(0);
+ uint32x4_t sse_u32 = vdupq_n_u32(0);
+
+ int i = h;
+ do {
+ int j = 0;
+ do {
+ uint8x16_t s = vld1q_u8(src + j);
+ uint8x16_t r = vld1q_u8(ref + j);
+
+ src_sum = vdotq_u32(src_sum, s, vdupq_n_u8(1));
+ ref_sum = vdotq_u32(ref_sum, r, vdupq_n_u8(1));
+
+ uint8x16_t abs_diff = vabdq_u8(s, r);
+ sse_u32 = vdotq_u32(sse_u32, abs_diff, abs_diff);
+
+ j += 16;
+ } while (j < w);
+
+ src += src_stride;
+ ref += ref_stride;
+ } while (--i != 0);
+
+ int32x4_t sum_diff =
+ vsubq_s32(vreinterpretq_s32_u32(src_sum), vreinterpretq_s32_u32(ref_sum));
+ *sum = horizontal_add_s32x4(sum_diff);
+ *sse = horizontal_add_u32x4(sse_u32);
+}
+
+static INLINE void variance_32xh_neon_dotprod(const uint8_t *src,
+ int src_stride,
+ const uint8_t *ref,
+ int ref_stride, int h,
+ uint32_t *sse, int *sum) {
+ variance_large_neon_dotprod(src, src_stride, ref, ref_stride, 32, h, sse,
+ sum);
+}
+
+static INLINE void variance_64xh_neon_dotprod(const uint8_t *src,
+ int src_stride,
+ const uint8_t *ref,
+ int ref_stride, int h,
+ uint32_t *sse, int *sum) {
+ variance_large_neon_dotprod(src, src_stride, ref, ref_stride, 64, h, sse,
+ sum);
+}
+
+static INLINE void variance_128xh_neon_dotprod(const uint8_t *src,
+ int src_stride,
+ const uint8_t *ref,
+ int ref_stride, int h,
+ uint32_t *sse, int *sum) {
+ variance_large_neon_dotprod(src, src_stride, ref, ref_stride, 128, h, sse,
+ sum);
+}
+
+#define VARIANCE_WXH_NEON_DOTPROD(w, h, shift) \
+ unsigned int aom_variance##w##x##h##_neon_dotprod( \
+ const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
+ unsigned int *sse) { \
+ int sum; \
+ variance_##w##xh_neon_dotprod(src, src_stride, ref, ref_stride, h, sse, \
+ &sum); \
+ return *sse - (uint32_t)(((int64_t)sum * sum) >> shift); \
+ }
+
+VARIANCE_WXH_NEON_DOTPROD(4, 4, 4)
+VARIANCE_WXH_NEON_DOTPROD(4, 8, 5)
+VARIANCE_WXH_NEON_DOTPROD(4, 16, 6)
+
+VARIANCE_WXH_NEON_DOTPROD(8, 4, 5)
+VARIANCE_WXH_NEON_DOTPROD(8, 8, 6)
+VARIANCE_WXH_NEON_DOTPROD(8, 16, 7)
+VARIANCE_WXH_NEON_DOTPROD(8, 32, 8)
+
+VARIANCE_WXH_NEON_DOTPROD(16, 4, 6)
+VARIANCE_WXH_NEON_DOTPROD(16, 8, 7)
+VARIANCE_WXH_NEON_DOTPROD(16, 16, 8)
+VARIANCE_WXH_NEON_DOTPROD(16, 32, 9)
+VARIANCE_WXH_NEON_DOTPROD(16, 64, 10)
+
+VARIANCE_WXH_NEON_DOTPROD(32, 8, 8)
+VARIANCE_WXH_NEON_DOTPROD(32, 16, 9)
+VARIANCE_WXH_NEON_DOTPROD(32, 32, 10)
+VARIANCE_WXH_NEON_DOTPROD(32, 64, 11)
+
+VARIANCE_WXH_NEON_DOTPROD(64, 16, 10)
+VARIANCE_WXH_NEON_DOTPROD(64, 32, 11)
+VARIANCE_WXH_NEON_DOTPROD(64, 64, 12)
+VARIANCE_WXH_NEON_DOTPROD(64, 128, 13)
+
+VARIANCE_WXH_NEON_DOTPROD(128, 64, 13)
+VARIANCE_WXH_NEON_DOTPROD(128, 128, 14)
+
+#undef VARIANCE_WXH_NEON_DOTPROD
+
+void aom_get_var_sse_sum_8x8_quad_neon_dotprod(
+ const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride,
+ uint32_t *sse8x8, int *sum8x8, unsigned int *tot_sse, int *tot_sum,
+ uint32_t *var8x8) {
+ // Loop over four 8x8 blocks. Process one 8x32 block.
+ for (int k = 0; k < 4; k++) {
+ variance_8xh_neon_dotprod(src + (k * 8), src_stride, ref + (k * 8),
+ ref_stride, 8, &sse8x8[k], &sum8x8[k]);
+ }
+
+ *tot_sse += sse8x8[0] + sse8x8[1] + sse8x8[2] + sse8x8[3];
+ *tot_sum += sum8x8[0] + sum8x8[1] + sum8x8[2] + sum8x8[3];
+ for (int i = 0; i < 4; i++) {
+ var8x8[i] = sse8x8[i] - (uint32_t)(((int64_t)sum8x8[i] * sum8x8[i]) >> 6);
+ }
+}
+
+void aom_get_var_sse_sum_16x16_dual_neon_dotprod(
+ const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride,
+ uint32_t *sse16x16, unsigned int *tot_sse, int *tot_sum,
+ uint32_t *var16x16) {
+ int sum16x16[2] = { 0 };
+ // Loop over two 16x16 blocks. Process one 16x32 block.
+ for (int k = 0; k < 2; k++) {
+ variance_16xh_neon_dotprod(src + (k * 16), src_stride, ref + (k * 16),
+ ref_stride, 16, &sse16x16[k], &sum16x16[k]);
+ }
+
+ *tot_sse += sse16x16[0] + sse16x16[1];
+ *tot_sum += sum16x16[0] + sum16x16[1];
+ for (int i = 0; i < 2; i++) {
+ var16x16[i] =
+ sse16x16[i] - (uint32_t)(((int64_t)sum16x16[i] * sum16x16[i]) >> 8);
+ }
+}
+
+static INLINE unsigned int mse8xh_neon_dotprod(const uint8_t *src,
+ int src_stride,
+ const uint8_t *ref,
+ int ref_stride,
+ unsigned int *sse, int h) {
+ uint32x4_t sse_u32 = vdupq_n_u32(0);
+
+ int i = h;
+ do {
+ uint8x16_t s = vcombine_u8(vld1_u8(src), vld1_u8(src + src_stride));
+ uint8x16_t r = vcombine_u8(vld1_u8(ref), vld1_u8(ref + ref_stride));
+
+ uint8x16_t abs_diff = vabdq_u8(s, r);
+
+ sse_u32 = vdotq_u32(sse_u32, abs_diff, abs_diff);
+
+ src += 2 * src_stride;
+ ref += 2 * ref_stride;
+ i -= 2;
+ } while (i != 0);
+
+ *sse = horizontal_add_u32x4(sse_u32);
+ return horizontal_add_u32x4(sse_u32);
+}
+
+static INLINE unsigned int mse16xh_neon_dotprod(const uint8_t *src,
+ int src_stride,
+ const uint8_t *ref,
+ int ref_stride,
+ unsigned int *sse, int h) {
+ uint32x4_t sse_u32[2] = { vdupq_n_u32(0), vdupq_n_u32(0) };
+
+ int i = h;
+ do {
+ uint8x16_t s0 = vld1q_u8(src);
+ uint8x16_t s1 = vld1q_u8(src + src_stride);
+ uint8x16_t r0 = vld1q_u8(ref);
+ uint8x16_t r1 = vld1q_u8(ref + ref_stride);
+
+ uint8x16_t abs_diff0 = vabdq_u8(s0, r0);
+ uint8x16_t abs_diff1 = vabdq_u8(s1, r1);
+
+ sse_u32[0] = vdotq_u32(sse_u32[0], abs_diff0, abs_diff0);
+ sse_u32[1] = vdotq_u32(sse_u32[1], abs_diff1, abs_diff1);
+
+ src += 2 * src_stride;
+ ref += 2 * ref_stride;
+ i -= 2;
+ } while (i != 0);
+
+ *sse = horizontal_add_u32x4(vaddq_u32(sse_u32[0], sse_u32[1]));
+ return horizontal_add_u32x4(vaddq_u32(sse_u32[0], sse_u32[1]));
+}
+
+#define MSE_WXH_NEON_DOTPROD(w, h) \
+ unsigned int aom_mse##w##x##h##_neon_dotprod( \
+ const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
+ unsigned int *sse) { \
+ return mse##w##xh_neon_dotprod(src, src_stride, ref, ref_stride, sse, h); \
+ }
+
+MSE_WXH_NEON_DOTPROD(8, 8)
+MSE_WXH_NEON_DOTPROD(8, 16)
+
+MSE_WXH_NEON_DOTPROD(16, 8)
+MSE_WXH_NEON_DOTPROD(16, 16)
+
+#undef MSE_WXH_NEON_DOTPROD
diff --git a/aom_dsp/avg.c b/aom_dsp/avg.c
index 7b36bf3..893f9c2 100644
--- a/aom_dsp/avg.c
+++ b/aom_dsp/avg.c
@@ -504,14 +504,14 @@
}
#endif // CONFIG_AV1_HIGHBITDEPTH
-// coeff: 16 bits, dynamic range [-32640, 32640].
-// length: value range {16, 64, 256, 1024}.
+// coeff: 20 bits, dynamic range [-524287, 524287].
+// length: value range {16, 32, 64, 128, 256, 512, 1024}.
int aom_satd_c(const tran_low_t *coeff, int length) {
int i;
int satd = 0;
for (i = 0; i < length; ++i) satd += abs(coeff[i]);
- // satd: 26 bits, dynamic range [-32640 * 1024, 32640 * 1024]
+ // satd: 30 bits, dynamic range [-524287 * 1024, 524287 * 1024]
return satd;
}
diff --git a/aom_dsp/entenc.h b/aom_dsp/entenc.h
index 467e47b..d26f027 100644
--- a/aom_dsp/entenc.h
+++ b/aom_dsp/entenc.h
@@ -13,7 +13,7 @@
#define AOM_AOM_DSP_ENTENC_H_
#include <stddef.h>
#include "aom_dsp/entcode.h"
-#include "aom_ports/bitops.h"
+#include "aom_util/endian_inl.h"
#ifdef __cplusplus
extern "C" {
@@ -87,13 +87,14 @@
} while (carry);
}
-// Reverse byte order and write data to buffer adding the carry-bit
+// Convert to big-endian byte order and write data to buffer adding the
+// carry-bit
static AOM_INLINE void write_enc_data_to_out_buf(unsigned char *out,
uint32_t offs, uint64_t output,
uint64_t carry,
uint32_t *enc_offs,
uint8_t num_bytes_ready) {
- const uint64_t reg = get_byteswap64(output) >> ((8 - num_bytes_ready) << 3);
+ const uint64_t reg = HToBE64(output << ((8 - num_bytes_ready) << 3));
memcpy(&out[offs], ®, 8);
// Propagate carry backwards if exists
if (carry) {
diff --git a/aom_dsp/fft.c b/aom_dsp/fft.c
index cad4a65..a44dbf7 100644
--- a/aom_dsp/fft.c
+++ b/aom_dsp/fft.c
@@ -11,6 +11,7 @@
#include "aom_dsp/aom_dsp_common.h"
#include "aom_dsp/fft_common.h"
+#include "config/aom_dsp_rtcd.h"
static INLINE void simple_transpose(const float *A, float *B, int n) {
for (int y = 0; y < n; y++) {
diff --git a/aom_dsp/fft_common.h b/aom_dsp/fft_common.h
index 5137331..3de1a04 100644
--- a/aom_dsp/fft_common.h
+++ b/aom_dsp/fft_common.h
@@ -47,10 +47,16 @@
// Declare some of the forward non-vectorized transforms which are used in some
// of the vectorized implementations
+void aom_fft1d_2_float(const float *input, float *output, int stride);
void aom_fft1d_4_float(const float *input, float *output, int stride);
void aom_fft1d_8_float(const float *input, float *output, int stride);
void aom_fft1d_16_float(const float *input, float *output, int stride);
void aom_fft1d_32_float(const float *input, float *output, int stride);
+void aom_ifft1d_2_float(const float *input, float *output, int stride);
+void aom_ifft1d_4_float(const float *input, float *output, int stride);
+void aom_ifft1d_8_float(const float *input, float *output, int stride);
+void aom_ifft1d_16_float(const float *input, float *output, int stride);
+void aom_ifft1d_32_float(const float *input, float *output, int stride);
/**\!brief Function pointer for transposing a matrix of floats.
*
diff --git a/aom_dsp/flow_estimation/arm/disflow_neon.c b/aom_dsp/flow_estimation/arm/disflow_neon.c
new file mode 100644
index 0000000..e2ba0e0
--- /dev/null
+++ b/aom_dsp/flow_estimation/arm/disflow_neon.c
@@ -0,0 +1,361 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include "aom_dsp/flow_estimation/disflow.h"
+
+#include <arm_neon.h>
+#include <math.h>
+
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_dsp/arm/sum_neon.h"
+#include "config/aom_config.h"
+#include "config/aom_dsp_rtcd.h"
+
+static INLINE void get_cubic_kernel_dbl(double x, double *kernel) {
+ assert(0 <= x && x < 1);
+ double x2 = x * x;
+ double x3 = x2 * x;
+ kernel[0] = -0.5 * x + x2 - 0.5 * x3;
+ kernel[1] = 1.0 - 2.5 * x2 + 1.5 * x3;
+ kernel[2] = 0.5 * x + 2.0 * x2 - 1.5 * x3;
+ kernel[3] = -0.5 * x2 + 0.5 * x3;
+}
+
+static INLINE void get_cubic_kernel_int(double x, int *kernel) {
+ double kernel_dbl[4];
+ get_cubic_kernel_dbl(x, kernel_dbl);
+
+ kernel[0] = (int)rint(kernel_dbl[0] * (1 << DISFLOW_INTERP_BITS));
+ kernel[1] = (int)rint(kernel_dbl[1] * (1 << DISFLOW_INTERP_BITS));
+ kernel[2] = (int)rint(kernel_dbl[2] * (1 << DISFLOW_INTERP_BITS));
+ kernel[3] = (int)rint(kernel_dbl[3] * (1 << DISFLOW_INTERP_BITS));
+}
+
+// Compare two regions of width x height pixels, one rooted at position
+// (x, y) in src and the other at (x + u, y + v) in ref.
+// This function returns the sum of squared pixel differences between
+// the two regions.
+static INLINE void compute_flow_error(const uint8_t *src, const uint8_t *ref,
+ int width, int height, int stride, int x,
+ int y, double u, double v, int16_t *dt) {
+ // Split offset into integer and fractional parts, and compute cubic
+ // interpolation kernels
+ const int u_int = (int)floor(u);
+ const int v_int = (int)floor(v);
+ const double u_frac = u - floor(u);
+ const double v_frac = v - floor(v);
+
+ int h_kernel[4];
+ int v_kernel[4];
+ get_cubic_kernel_int(u_frac, h_kernel);
+ get_cubic_kernel_int(v_frac, v_kernel);
+
+ int16_t tmp_[DISFLOW_PATCH_SIZE * (DISFLOW_PATCH_SIZE + 3)];
+
+ // Clamp coordinates so that all pixels we fetch will remain within the
+ // allocated border region, but allow them to go far enough out that
+ // the border pixels' values do not change.
+ // Since we are calculating an 8x8 block, the bottom-right pixel
+ // in the block has coordinates (x0 + 7, y0 + 7). Then, the cubic
+ // interpolation has 4 taps, meaning that the output of pixel
+ // (x_w, y_w) depends on the pixels in the range
+ // ([x_w - 1, x_w + 2], [y_w - 1, y_w + 2]).
+ //
+ // Thus the most extreme coordinates which will be fetched are
+ // (x0 - 1, y0 - 1) and (x0 + 9, y0 + 9).
+ const int x0 = clamp(x + u_int, -9, width);
+ const int y0 = clamp(y + v_int, -9, height);
+
+ // Horizontal convolution.
+ const uint8_t *ref_start = ref + (y0 - 1) * stride + (x0 - 1);
+ int16x4_t h_filter = vmovn_s32(vld1q_s32(h_kernel));
+
+ for (int i = 0; i < DISFLOW_PATCH_SIZE + 3; ++i) {
+ uint8x16_t r = vld1q_u8(ref_start + i * stride);
+ uint16x8_t r0 = vmovl_u8(vget_low_u8(r));
+ uint16x8_t r1 = vmovl_u8(vget_high_u8(r));
+
+ int16x8_t s0 = vreinterpretq_s16_u16(r0);
+ int16x8_t s1 = vreinterpretq_s16_u16(vextq_u16(r0, r1, 1));
+ int16x8_t s2 = vreinterpretq_s16_u16(vextq_u16(r0, r1, 2));
+ int16x8_t s3 = vreinterpretq_s16_u16(vextq_u16(r0, r1, 3));
+
+ int32x4_t sum_lo = vmull_lane_s16(vget_low_s16(s0), h_filter, 0);
+ sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(s1), h_filter, 1);
+ sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(s2), h_filter, 2);
+ sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(s3), h_filter, 3);
+
+ int32x4_t sum_hi = vmull_lane_s16(vget_high_s16(s0), h_filter, 0);
+ sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(s1), h_filter, 1);
+ sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(s2), h_filter, 2);
+ sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(s3), h_filter, 3);
+
+ // 6 is the maximum allowable number of extra bits which will avoid
+ // the intermediate values overflowing an int16_t. The most extreme
+ // intermediate value occurs when:
+ // * The input pixels are [0, 255, 255, 0]
+ // * u_frac = 0.5
+ // In this case, the un-scaled output is 255 * 1.125 = 286.875.
+ // As an integer with 6 fractional bits, that is 18360, which fits
+ // in an int16_t. But with 7 fractional bits it would be 36720,
+ // which is too large.
+
+ int16x8_t sum = vcombine_s16(vrshrn_n_s32(sum_lo, DISFLOW_INTERP_BITS - 6),
+ vrshrn_n_s32(sum_hi, DISFLOW_INTERP_BITS - 6));
+ vst1q_s16(tmp_ + i * DISFLOW_PATCH_SIZE, sum);
+ }
+
+ // Vertical convolution.
+ int16x4_t v_filter = vmovn_s32(vld1q_s32(v_kernel));
+ int16_t *tmp_start = tmp_ + DISFLOW_PATCH_SIZE;
+
+ for (int i = 0; i < DISFLOW_PATCH_SIZE; ++i) {
+ int16x8_t t0 = vld1q_s16(tmp_start + (i - 1) * DISFLOW_PATCH_SIZE);
+ int16x8_t t1 = vld1q_s16(tmp_start + i * DISFLOW_PATCH_SIZE);
+ int16x8_t t2 = vld1q_s16(tmp_start + (i + 1) * DISFLOW_PATCH_SIZE);
+ int16x8_t t3 = vld1q_s16(tmp_start + (i + 2) * DISFLOW_PATCH_SIZE);
+
+ int32x4_t sum_lo = vmull_lane_s16(vget_low_s16(t0), v_filter, 0);
+ sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(t1), v_filter, 1);
+ sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(t2), v_filter, 2);
+ sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(t3), v_filter, 3);
+
+ int32x4_t sum_hi = vmull_lane_s16(vget_high_s16(t0), v_filter, 0);
+ sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(t1), v_filter, 1);
+ sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(t2), v_filter, 2);
+ sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(t3), v_filter, 3);
+
+ uint8x8_t s = vld1_u8(src + (i + y) * stride + x);
+ int16x8_t s_s16 = vreinterpretq_s16_u16(vshll_n_u8(s, 3));
+
+ // This time, we have to round off the 6 extra bits which were kept
+ // earlier, but we also want to keep DISFLOW_DERIV_SCALE_LOG2 extra bits
+ // of precision to match the scale of the dx and dy arrays.
+ sum_lo = vrshrq_n_s32(sum_lo,
+ DISFLOW_INTERP_BITS + 6 - DISFLOW_DERIV_SCALE_LOG2);
+ sum_hi = vrshrq_n_s32(sum_hi,
+ DISFLOW_INTERP_BITS + 6 - DISFLOW_DERIV_SCALE_LOG2);
+ int32x4_t err_lo = vsubw_s16(sum_lo, vget_low_s16(s_s16));
+ int32x4_t err_hi = vsubw_s16(sum_hi, vget_high_s16(s_s16));
+ vst1q_s16(dt + i * DISFLOW_PATCH_SIZE,
+ vcombine_s16(vmovn_s32(err_lo), vmovn_s32(err_hi)));
+ }
+}
+
+static INLINE void sobel_filter_x(const uint8_t *src, int src_stride,
+ int16_t *dst, int dst_stride) {
+ int16_t tmp[DISFLOW_PATCH_SIZE * (DISFLOW_PATCH_SIZE + 2)];
+
+ // Horizontal filter, using kernel {1, 0, -1}.
+ const uint8_t *src_start = src - 1 * src_stride - 1;
+
+ for (int i = 0; i < DISFLOW_PATCH_SIZE + 2; i++) {
+ uint8x16_t s = vld1q_u8(src_start + i * src_stride);
+ uint8x8_t s0 = vget_low_u8(s);
+ uint8x8_t s2 = vget_low_u8(vextq_u8(s, s, 2));
+
+ // Given that the kernel is {1, 0, -1} the convolution is a simple
+ // subtraction.
+ int16x8_t diff = vreinterpretq_s16_u16(vsubl_u8(s0, s2));
+
+ vst1q_s16(tmp + i * DISFLOW_PATCH_SIZE, diff);
+ }
+
+ // Vertical filter, using kernel {1, 2, 1}.
+ // This kernel can be split into two 2-taps kernels of value {1, 1}.
+ // That way we need only 3 add operations to perform the convolution, one of
+ // which can be reused for the next line.
+ int16x8_t s0 = vld1q_s16(tmp);
+ int16x8_t s1 = vld1q_s16(tmp + DISFLOW_PATCH_SIZE);
+ int16x8_t sum01 = vaddq_s16(s0, s1);
+ for (int i = 0; i < DISFLOW_PATCH_SIZE; i++) {
+ int16x8_t s2 = vld1q_s16(tmp + (i + 2) * DISFLOW_PATCH_SIZE);
+
+ int16x8_t sum12 = vaddq_s16(s1, s2);
+ int16x8_t sum = vaddq_s16(sum01, sum12);
+
+ vst1q_s16(dst + i * dst_stride, sum);
+
+ sum01 = sum12;
+ s1 = s2;
+ }
+}
+
+static INLINE void sobel_filter_y(const uint8_t *src, int src_stride,
+ int16_t *dst, int dst_stride) {
+ int16_t tmp[DISFLOW_PATCH_SIZE * (DISFLOW_PATCH_SIZE + 2)];
+
+ // Horizontal filter, using kernel {1, 2, 1}.
+ // This kernel can be split into two 2-taps kernels of value {1, 1}.
+ // That way we need only 3 add operations to perform the convolution.
+ const uint8_t *src_start = src - 1 * src_stride - 1;
+
+ for (int i = 0; i < DISFLOW_PATCH_SIZE + 2; i++) {
+ uint8x16_t s = vld1q_u8(src_start + i * src_stride);
+ uint8x8_t s0 = vget_low_u8(s);
+ uint8x8_t s1 = vget_low_u8(vextq_u8(s, s, 1));
+ uint8x8_t s2 = vget_low_u8(vextq_u8(s, s, 2));
+
+ uint16x8_t sum01 = vaddl_u8(s0, s1);
+ uint16x8_t sum12 = vaddl_u8(s1, s2);
+ uint16x8_t sum = vaddq_u16(sum01, sum12);
+
+ vst1q_s16(tmp + i * DISFLOW_PATCH_SIZE, vreinterpretq_s16_u16(sum));
+ }
+
+ // Vertical filter, using kernel {1, 0, -1}.
+ // Load the whole block at once to avoid redundant loads during convolution.
+ int16x8_t t[10];
+ load_s16_8x10(tmp, DISFLOW_PATCH_SIZE, &t[0], &t[1], &t[2], &t[3], &t[4],
+ &t[5], &t[6], &t[7], &t[8], &t[9]);
+
+ for (int i = 0; i < DISFLOW_PATCH_SIZE; i++) {
+ // Given that the kernel is {1, 0, -1} the convolution is a simple
+ // subtraction.
+ int16x8_t diff = vsubq_s16(t[i], t[i + 2]);
+
+ vst1q_s16(dst + i * dst_stride, diff);
+ }
+}
+
+// Computes the components of the system of equations used to solve for
+// a flow vector.
+//
+// The flow equations are a least-squares system, derived as follows:
+//
+// For each pixel in the patch, we calculate the current error `dt`,
+// and the x and y gradients `dx` and `dy` of the source patch.
+// This means that, to first order, the squared error for this pixel is
+//
+// (dt + u * dx + v * dy)^2
+//
+// where (u, v) are the incremental changes to the flow vector.
+//
+// We then want to find the values of u and v which minimize the sum
+// of the squared error across all pixels. Conveniently, this fits exactly
+// into the form of a least squares problem, with one equation
+//
+// u * dx + v * dy = -dt
+//
+// for each pixel.
+//
+// Summing across all pixels in a square window of size DISFLOW_PATCH_SIZE,
+// and absorbing the - sign elsewhere, this results in the least squares system
+//
+// M = |sum(dx * dx) sum(dx * dy)|
+// |sum(dx * dy) sum(dy * dy)|
+//
+// b = |sum(dx * dt)|
+// |sum(dy * dt)|
+static INLINE void compute_flow_matrix(const int16_t *dx, int dx_stride,
+ const int16_t *dy, int dy_stride,
+ double *M_inv) {
+ int32x4_t sum[4] = { vdupq_n_s32(0), vdupq_n_s32(0), vdupq_n_s32(0),
+ vdupq_n_s32(0) };
+
+ for (int i = 0; i < DISFLOW_PATCH_SIZE; i++) {
+ int16x8_t x = vld1q_s16(dx + i * dx_stride);
+ int16x8_t y = vld1q_s16(dy + i * dy_stride);
+ sum[0] = vmlal_s16(sum[0], vget_low_s16(x), vget_low_s16(x));
+ sum[0] = vmlal_s16(sum[0], vget_high_s16(x), vget_high_s16(x));
+
+ sum[1] = vmlal_s16(sum[1], vget_low_s16(x), vget_low_s16(y));
+ sum[1] = vmlal_s16(sum[1], vget_high_s16(x), vget_high_s16(y));
+
+ sum[3] = vmlal_s16(sum[3], vget_low_s16(y), vget_low_s16(y));
+ sum[3] = vmlal_s16(sum[3], vget_high_s16(y), vget_high_s16(y));
+ }
+ sum[2] = sum[1];
+
+ int32x4_t res = horizontal_add_4d_s32x4(sum);
+
+ // Apply regularization
+ // We follow the standard regularization method of adding `k * I` before
+ // inverting. This ensures that the matrix will be invertible.
+ //
+ // Setting the regularization strength k to 1 seems to work well here, as
+ // typical values coming from the other equations are very large (1e5 to
+ // 1e6, with an upper limit of around 6e7, at the time of writing).
+ // It also preserves the property that all matrix values are whole numbers,
+ // which is convenient for integerized SIMD implementation.
+
+ double M0 = (double)vgetq_lane_s32(res, 0) + 1;
+ double M1 = (double)vgetq_lane_s32(res, 1);
+ double M2 = (double)vgetq_lane_s32(res, 2);
+ double M3 = (double)vgetq_lane_s32(res, 3) + 1;
+
+ // Invert matrix M.
+ double det = (M0 * M3) - (M1 * M2);
+ assert(det >= 1);
+ const double det_inv = 1 / det;
+
+ M_inv[0] = M3 * det_inv;
+ M_inv[1] = -M1 * det_inv;
+ M_inv[2] = -M2 * det_inv;
+ M_inv[3] = M0 * det_inv;
+}
+
+static INLINE void compute_flow_vector(const int16_t *dx, int dx_stride,
+ const int16_t *dy, int dy_stride,
+ const int16_t *dt, int dt_stride,
+ int *b) {
+ int32x4_t b_s32[2] = { vdupq_n_s32(0), vdupq_n_s32(0) };
+
+ for (int i = 0; i < DISFLOW_PATCH_SIZE; i++) {
+ int16x8_t dx16 = vld1q_s16(dx + i * dx_stride);
+ int16x8_t dy16 = vld1q_s16(dy + i * dy_stride);
+ int16x8_t dt16 = vld1q_s16(dt + i * dt_stride);
+
+ b_s32[0] = vmlal_s16(b_s32[0], vget_low_s16(dx16), vget_low_s16(dt16));
+ b_s32[0] = vmlal_s16(b_s32[0], vget_high_s16(dx16), vget_high_s16(dt16));
+
+ b_s32[1] = vmlal_s16(b_s32[1], vget_low_s16(dy16), vget_low_s16(dt16));
+ b_s32[1] = vmlal_s16(b_s32[1], vget_high_s16(dy16), vget_high_s16(dt16));
+ }
+
+ int32x4_t b_red = horizontal_add_2d_s32(b_s32[0], b_s32[1]);
+ vst1_s32(b, add_pairwise_s32x4(b_red));
+}
+
+void aom_compute_flow_at_point_neon(const uint8_t *src, const uint8_t *ref,
+ int x, int y, int width, int height,
+ int stride, double *u, double *v) {
+ double M_inv[4];
+ int b[2];
+ int16_t dt[DISFLOW_PATCH_SIZE * DISFLOW_PATCH_SIZE];
+ int16_t dx[DISFLOW_PATCH_SIZE * DISFLOW_PATCH_SIZE];
+ int16_t dy[DISFLOW_PATCH_SIZE * DISFLOW_PATCH_SIZE];
+
+ // Compute gradients within this patch
+ const uint8_t *src_patch = &src[y * stride + x];
+ sobel_filter_x(src_patch, stride, dx, DISFLOW_PATCH_SIZE);
+ sobel_filter_y(src_patch, stride, dy, DISFLOW_PATCH_SIZE);
+
+ compute_flow_matrix(dx, DISFLOW_PATCH_SIZE, dy, DISFLOW_PATCH_SIZE, M_inv);
+
+ for (int itr = 0; itr < DISFLOW_MAX_ITR; itr++) {
+ compute_flow_error(src, ref, width, height, stride, x, y, *u, *v, dt);
+ compute_flow_vector(dx, DISFLOW_PATCH_SIZE, dy, DISFLOW_PATCH_SIZE, dt,
+ DISFLOW_PATCH_SIZE, b);
+
+ // Solve flow equations to find a better estimate for the flow vector
+ // at this point
+ const double step_u = M_inv[0] * b[0] + M_inv[1] * b[1];
+ const double step_v = M_inv[2] * b[0] + M_inv[3] * b[1];
+ *u += fclamp(step_u * DISFLOW_STEP_SIZE, -2, 2);
+ *v += fclamp(step_v * DISFLOW_STEP_SIZE, -2, 2);
+
+ if (fabs(step_u) + fabs(step_v) < DISFLOW_STEP_SIZE_THRESOLD) {
+ // Stop iteration when we're close to convergence
+ break;
+ }
+ }
+}
diff --git a/aom_dsp/flow_estimation/corner_detect.c b/aom_dsp/flow_estimation/corner_detect.c
index 7848295..284d1bd 100644
--- a/aom_dsp/flow_estimation/corner_detect.c
+++ b/aom_dsp/flow_estimation/corner_detect.c
@@ -24,10 +24,10 @@
#define FAST_BARRIER 18
-size_t av1_get_corner_list_size() { return sizeof(CornerList); }
+size_t av1_get_corner_list_size(void) { return sizeof(CornerList); }
-CornerList *av1_alloc_corner_list() {
- CornerList *corners = (CornerList *)aom_calloc(1, sizeof(CornerList));
+CornerList *av1_alloc_corner_list(void) {
+ CornerList *corners = (CornerList *)aom_calloc(1, sizeof(*corners));
if (!corners) {
return NULL;
}
@@ -39,7 +39,7 @@
return corners;
}
-void compute_corner_list(const ImagePyramid *pyr, CornerList *corners) {
+static bool compute_corner_list(const ImagePyramid *pyr, CornerList *corners) {
const uint8_t *buf = pyr->layers[0].buffer;
int width = pyr->layers[0].width;
int height = pyr->layers[0].height;
@@ -49,14 +49,14 @@
int num_corners;
xy *const frame_corners_xy = aom_fast9_detect_nonmax(
buf, width, height, stride, FAST_BARRIER, &scores, &num_corners);
+ if (num_corners < 0) return false;
- if (num_corners <= 0) {
- // Some error occured, so no corners are available
- corners->num_corners = 0;
- } else if (num_corners <= MAX_CORNERS) {
+ if (num_corners <= MAX_CORNERS) {
// Use all detected corners
- memcpy(corners->corners, frame_corners_xy,
- sizeof(*frame_corners_xy) * num_corners);
+ if (num_corners != 0) {
+ memcpy(corners->corners, frame_corners_xy,
+ sizeof(*frame_corners_xy) * num_corners);
+ }
corners->num_corners = num_corners;
} else {
// There are more than MAX_CORNERS corners avilable, so pick out a subset
@@ -96,9 +96,10 @@
free(scores);
free(frame_corners_xy);
+ return true;
}
-void av1_compute_corner_list(const ImagePyramid *pyr, CornerList *corners) {
+bool av1_compute_corner_list(const ImagePyramid *pyr, CornerList *corners) {
assert(corners);
#if CONFIG_MULTITHREAD
@@ -106,13 +107,14 @@
#endif // CONFIG_MULTITHREAD
if (!corners->valid) {
- compute_corner_list(pyr, corners);
- corners->valid = true;
+ corners->valid = compute_corner_list(pyr, corners);
}
+ bool valid = corners->valid;
#if CONFIG_MULTITHREAD
pthread_mutex_unlock(&corners->mutex);
#endif // CONFIG_MULTITHREAD
+ return valid;
}
#ifndef NDEBUG
diff --git a/aom_dsp/flow_estimation/corner_detect.h b/aom_dsp/flow_estimation/corner_detect.h
index c77813e..d05846c 100644
--- a/aom_dsp/flow_estimation/corner_detect.h
+++ b/aom_dsp/flow_estimation/corner_detect.h
@@ -53,11 +53,11 @@
int corners[2 * MAX_CORNERS];
} CornerList;
-size_t av1_get_corner_list_size();
+size_t av1_get_corner_list_size(void);
-CornerList *av1_alloc_corner_list();
+CornerList *av1_alloc_corner_list(void);
-void av1_compute_corner_list(const ImagePyramid *pyr, CornerList *corners);
+bool av1_compute_corner_list(const ImagePyramid *pyr, CornerList *corners);
#ifndef NDEBUG
// Check if a corner list has already been computed.
diff --git a/aom_dsp/flow_estimation/corner_match.c b/aom_dsp/flow_estimation/corner_match.c
index f34178e..cef719b 100644
--- a/aom_dsp/flow_estimation/corner_match.c
+++ b/aom_dsp/flow_estimation/corner_match.c
@@ -147,13 +147,13 @@
}
}
-int aom_determine_correspondence(const unsigned char *src,
- const int *src_corners, int num_src_corners,
- const unsigned char *ref,
- const int *ref_corners, int num_ref_corners,
- int width, int height, int src_stride,
- int ref_stride,
- Correspondence *correspondences) {
+static int determine_correspondence(const unsigned char *src,
+ const int *src_corners, int num_src_corners,
+ const unsigned char *ref,
+ const int *ref_corners, int num_ref_corners,
+ int width, int height, int src_stride,
+ int ref_stride,
+ Correspondence *correspondences) {
// TODO(sarahparker) Improve this to include 2-way match
int i, j;
int num_correspondences = 0;
@@ -202,7 +202,8 @@
bool av1_compute_global_motion_feature_match(
TransformationType type, YV12_BUFFER_CONFIG *src, YV12_BUFFER_CONFIG *ref,
- int bit_depth, MotionModel *motion_models, int num_motion_models) {
+ int bit_depth, MotionModel *motion_models, int num_motion_models,
+ bool *mem_alloc_failed) {
int num_correspondences;
Correspondence *correspondences;
ImagePyramid *src_pyramid = src->y_pyramid;
@@ -211,10 +212,22 @@
CornerList *ref_corners = ref->corners;
// Precompute information we will need about each frame
- aom_compute_pyramid(src, bit_depth, src_pyramid);
- av1_compute_corner_list(src_pyramid, src_corners);
- aom_compute_pyramid(ref, bit_depth, ref_pyramid);
- av1_compute_corner_list(ref_pyramid, ref_corners);
+ if (!aom_compute_pyramid(src, bit_depth, src_pyramid)) {
+ *mem_alloc_failed = true;
+ return false;
+ }
+ if (!av1_compute_corner_list(src_pyramid, src_corners)) {
+ *mem_alloc_failed = true;
+ return false;
+ }
+ if (!aom_compute_pyramid(ref, bit_depth, ref_pyramid)) {
+ *mem_alloc_failed = true;
+ return false;
+ }
+ if (!av1_compute_corner_list(src_pyramid, src_corners)) {
+ *mem_alloc_failed = true;
+ return false;
+ }
const uint8_t *src_buffer = src_pyramid->layers[0].buffer;
const int src_width = src_pyramid->layers[0].width;
@@ -229,14 +242,17 @@
// find correspondences between the two images
correspondences = (Correspondence *)aom_malloc(src_corners->num_corners *
sizeof(*correspondences));
- if (!correspondences) return false;
- num_correspondences = aom_determine_correspondence(
+ if (!correspondences) {
+ *mem_alloc_failed = true;
+ return false;
+ }
+ num_correspondences = determine_correspondence(
src_buffer, src_corners->corners, src_corners->num_corners, ref_buffer,
ref_corners->corners, ref_corners->num_corners, src_width, src_height,
src_stride, ref_stride, correspondences);
bool result = ransac(correspondences, num_correspondences, type,
- motion_models, num_motion_models);
+ motion_models, num_motion_models, mem_alloc_failed);
aom_free(correspondences);
return result;
diff --git a/aom_dsp/flow_estimation/corner_match.h b/aom_dsp/flow_estimation/corner_match.h
index bb69944..4435d2c 100644
--- a/aom_dsp/flow_estimation/corner_match.h
+++ b/aom_dsp/flow_estimation/corner_match.h
@@ -29,17 +29,10 @@
#define MATCH_SZ_BY2 ((MATCH_SZ - 1) / 2)
#define MATCH_SZ_SQ (MATCH_SZ * MATCH_SZ)
-int aom_determine_correspondence(const unsigned char *src,
- const int *src_corners, int num_src_corners,
- const unsigned char *ref,
- const int *ref_corners, int num_ref_corners,
- int width, int height, int src_stride,
- int ref_stride,
- Correspondence *correspondences);
-
bool av1_compute_global_motion_feature_match(
TransformationType type, YV12_BUFFER_CONFIG *src, YV12_BUFFER_CONFIG *ref,
- int bit_depth, MotionModel *motion_models, int num_motion_models);
+ int bit_depth, MotionModel *motion_models, int num_motion_models,
+ bool *mem_alloc_failed);
#ifdef __cplusplus
}
diff --git a/aom_dsp/flow_estimation/disflow.c b/aom_dsp/flow_estimation/disflow.c
index a8e7b06..a010c81 100644
--- a/aom_dsp/flow_estimation/disflow.c
+++ b/aom_dsp/flow_estimation/disflow.c
@@ -154,9 +154,13 @@
// (x, y) in src and the other at (x + u, y + v) in ref.
// This function returns the sum of squared pixel differences between
// the two regions.
-static INLINE void compute_flow_error(const uint8_t *src, const uint8_t *ref,
- int width, int height, int stride, int x,
- int y, double u, double v, int16_t *dt) {
+static INLINE void compute_flow_vector(const uint8_t *src, const uint8_t *ref,
+ int width, int height, int stride, int x,
+ int y, double u, double v,
+ const int16_t *dx, const int16_t *dy,
+ int *b) {
+ memset(b, 0, 2 * sizeof(*b));
+
// Split offset into integer and fractional parts, and compute cubic
// interpolation kernels
const int u_int = (int)floor(u);
@@ -230,8 +234,9 @@
const int round_bits = DISFLOW_INTERP_BITS + 6 - DISFLOW_DERIV_SCALE_LOG2;
const int warped = ROUND_POWER_OF_TWO(result, round_bits);
const int src_px = src[(x + j) + (y + i) * stride] << 3;
- const int err = warped - src_px;
- dt[i * DISFLOW_PATCH_SIZE + j] = err;
+ const int dt = warped - src_px;
+ b[0] += dx[i * DISFLOW_PATCH_SIZE + j] * dt;
+ b[1] += dy[i * DISFLOW_PATCH_SIZE + j] * dt;
}
}
}
@@ -351,20 +356,6 @@
M[3] = (double)tmp[3];
}
-static INLINE void compute_flow_vector(const int16_t *dx, int dx_stride,
- const int16_t *dy, int dy_stride,
- const int16_t *dt, int dt_stride,
- int *b) {
- memset(b, 0, 2 * sizeof(*b));
-
- for (int i = 0; i < DISFLOW_PATCH_SIZE; i++) {
- for (int j = 0; j < DISFLOW_PATCH_SIZE; j++) {
- b[0] += dx[i * dx_stride + j] * dt[i * dt_stride + j];
- b[1] += dy[i * dy_stride + j] * dt[i * dt_stride + j];
- }
- }
-}
-
// Try to invert the matrix M
// Note: Due to the nature of how a least-squares matrix is constructed, all of
// the eigenvalues will be >= 0, and therefore det M >= 0 as well.
@@ -388,7 +379,6 @@
double M[4];
double M_inv[4];
int b[2];
- int16_t dt[DISFLOW_PATCH_SIZE * DISFLOW_PATCH_SIZE];
int16_t dx[DISFLOW_PATCH_SIZE * DISFLOW_PATCH_SIZE];
int16_t dy[DISFLOW_PATCH_SIZE * DISFLOW_PATCH_SIZE];
@@ -401,9 +391,8 @@
invert_2x2(M, M_inv);
for (int itr = 0; itr < DISFLOW_MAX_ITR; itr++) {
- compute_flow_error(src, ref, width, height, stride, x, y, *u, *v, dt);
- compute_flow_vector(dx, DISFLOW_PATCH_SIZE, dy, DISFLOW_PATCH_SIZE, dt,
- DISFLOW_PATCH_SIZE, b);
+ compute_flow_vector(src, ref, width, height, stride, x, y, *u, *v, dx, dy,
+ b);
// Solve flow equations to find a better estimate for the flow vector
// at this point
@@ -463,8 +452,9 @@
}
// make sure flow_u and flow_v start at 0
-static void compute_flow_field(const ImagePyramid *src_pyr,
+static bool compute_flow_field(const ImagePyramid *src_pyr,
const ImagePyramid *ref_pyr, FlowField *flow) {
+ bool mem_status = true;
assert(src_pyr->n_levels == ref_pyr->n_levels);
double *flow_u = flow->u;
@@ -473,6 +463,10 @@
const size_t flow_size = flow->stride * (size_t)flow->height;
double *u_upscale = aom_malloc(flow_size * sizeof(*u_upscale));
double *v_upscale = aom_malloc(flow_size * sizeof(*v_upscale));
+ if (!u_upscale || !v_upscale) {
+ mem_status = false;
+ goto free_uvscale;
+ }
// Compute flow field from coarsest to finest level of the pyramid
for (int level = src_pyr->n_levels - 1; level >= 0; --level) {
@@ -522,12 +516,16 @@
const int upscale_flow_height = cur_flow_height << 1;
const int upscale_stride = flow->stride;
- av1_upscale_plane_double_prec(
+ bool upscale_u_plane = av1_upscale_plane_double_prec(
flow_u, cur_flow_height, cur_flow_width, cur_flow_stride, u_upscale,
upscale_flow_height, upscale_flow_width, upscale_stride);
- av1_upscale_plane_double_prec(
+ bool upscale_v_plane = av1_upscale_plane_double_prec(
flow_v, cur_flow_height, cur_flow_width, cur_flow_stride, v_upscale,
upscale_flow_height, upscale_flow_width, upscale_stride);
+ if (!upscale_u_plane || !upscale_v_plane) {
+ mem_status = false;
+ goto free_uvscale;
+ }
// Multiply all flow vectors by 2.
// When we move down a pyramid level, the image resolution doubles.
@@ -569,8 +567,10 @@
}
}
}
+free_uvscale:
aom_free(u_upscale);
aom_free(v_upscale);
+ return mem_status;
}
static FlowField *alloc_flow_field(int frame_width, int frame_height) {
@@ -612,14 +612,24 @@
YV12_BUFFER_CONFIG *src,
YV12_BUFFER_CONFIG *ref, int bit_depth,
MotionModel *motion_models,
- int num_motion_models) {
+ int num_motion_models,
+ bool *mem_alloc_failed) {
// Precompute information we will need about each frame
ImagePyramid *src_pyramid = src->y_pyramid;
CornerList *src_corners = src->corners;
ImagePyramid *ref_pyramid = ref->y_pyramid;
- aom_compute_pyramid(src, bit_depth, src_pyramid);
- av1_compute_corner_list(src_pyramid, src_corners);
- aom_compute_pyramid(ref, bit_depth, ref_pyramid);
+ if (!aom_compute_pyramid(src, bit_depth, src_pyramid)) {
+ *mem_alloc_failed = true;
+ return false;
+ }
+ if (!av1_compute_corner_list(src_pyramid, src_corners)) {
+ *mem_alloc_failed = true;
+ return false;
+ }
+ if (!aom_compute_pyramid(ref, bit_depth, ref_pyramid)) {
+ *mem_alloc_failed = true;
+ return false;
+ }
const int src_width = src_pyramid->layers[0].width;
const int src_height = src_pyramid->layers[0].height;
@@ -627,14 +637,22 @@
assert(ref_pyramid->layers[0].height == src_height);
FlowField *flow = alloc_flow_field(src_width, src_height);
- if (!flow) return false;
+ if (!flow) {
+ *mem_alloc_failed = true;
+ return false;
+ }
- compute_flow_field(src_pyramid, ref_pyramid, flow);
+ if (!compute_flow_field(src_pyramid, ref_pyramid, flow)) {
+ *mem_alloc_failed = true;
+ free_flow_field(flow);
+ return false;
+ }
// find correspondences between the two images using the flow field
Correspondence *correspondences =
aom_malloc(src_corners->num_corners * sizeof(*correspondences));
if (!correspondences) {
+ *mem_alloc_failed = true;
free_flow_field(flow);
return false;
}
@@ -643,7 +661,7 @@
determine_disflow_correspondence(src_corners, flow, correspondences);
bool result = ransac(correspondences, num_correspondences, type,
- motion_models, num_motion_models);
+ motion_models, num_motion_models, mem_alloc_failed);
aom_free(correspondences);
free_flow_field(flow);
diff --git a/aom_dsp/flow_estimation/disflow.h b/aom_dsp/flow_estimation/disflow.h
index 2e97ba2..d772c8a 100644
--- a/aom_dsp/flow_estimation/disflow.h
+++ b/aom_dsp/flow_estimation/disflow.h
@@ -93,7 +93,8 @@
YV12_BUFFER_CONFIG *src,
YV12_BUFFER_CONFIG *ref, int bit_depth,
MotionModel *motion_models,
- int num_motion_models);
+ int num_motion_models,
+ bool *mem_alloc_failed);
#ifdef __cplusplus
}
diff --git a/aom_dsp/flow_estimation/flow_estimation.c b/aom_dsp/flow_estimation/flow_estimation.c
index a6bf942..0f47f86 100644
--- a/aom_dsp/flow_estimation/flow_estimation.c
+++ b/aom_dsp/flow_estimation/flow_estimation.c
@@ -44,15 +44,17 @@
YV12_BUFFER_CONFIG *ref, int bit_depth,
GlobalMotionMethod gm_method,
MotionModel *motion_models,
- int num_motion_models) {
+ int num_motion_models, bool *mem_alloc_failed) {
switch (gm_method) {
case GLOBAL_MOTION_METHOD_FEATURE_MATCH:
return av1_compute_global_motion_feature_match(
- type, src, ref, bit_depth, motion_models, num_motion_models);
+ type, src, ref, bit_depth, motion_models, num_motion_models,
+ mem_alloc_failed);
case GLOBAL_MOTION_METHOD_DISFLOW:
- return av1_compute_global_motion_disflow(
- type, src, ref, bit_depth, motion_models, num_motion_models);
+ return av1_compute_global_motion_disflow(type, src, ref, bit_depth,
+ motion_models, num_motion_models,
+ mem_alloc_failed);
default: assert(0 && "Unknown global motion estimation type");
}
- return 0;
+ return false;
}
diff --git a/aom_dsp/flow_estimation/flow_estimation.h b/aom_dsp/flow_estimation/flow_estimation.h
index 4f2192c..2dfae24 100644
--- a/aom_dsp/flow_estimation/flow_estimation.h
+++ b/aom_dsp/flow_estimation/flow_estimation.h
@@ -86,7 +86,7 @@
YV12_BUFFER_CONFIG *ref, int bit_depth,
GlobalMotionMethod gm_method,
MotionModel *motion_models,
- int num_motion_models);
+ int num_motion_models, bool *mem_alloc_failed);
#ifdef __cplusplus
}
diff --git a/aom_dsp/flow_estimation/ransac.c b/aom_dsp/flow_estimation/ransac.c
index 81c5f2c..b88a07b 100644
--- a/aom_dsp/flow_estimation/ransac.c
+++ b/aom_dsp/flow_estimation/ransac.c
@@ -246,7 +246,8 @@
// Returns true on success, false on error
static bool ransac_internal(const Correspondence *matched_points, int npoints,
MotionModel *motion_models, int num_desired_motions,
- const RansacModelInfo *model_info) {
+ const RansacModelInfo *model_info,
+ bool *mem_alloc_failed) {
assert(npoints >= 0);
int i = 0;
int minpts = model_info->minpts;
@@ -297,6 +298,7 @@
if (!(points1 && points2 && corners1 && corners2 && projected_corners &&
motions && inlier_buffer)) {
ret_val = false;
+ *mem_alloc_failed = true;
goto finish_ransac;
}
@@ -469,7 +471,7 @@
// Returns true on success, false on error
bool ransac(const Correspondence *matched_points, int npoints,
TransformationType type, MotionModel *motion_models,
- int num_desired_motions) {
+ int num_desired_motions, bool *mem_alloc_failed) {
#if ALLOW_TRANSLATION_MODELS
assert(type > IDENTITY && type < TRANS_TYPES);
#else
@@ -477,5 +479,6 @@
#endif // ALLOW_TRANSLATION_MODELS
return ransac_internal(matched_points, npoints, motion_models,
- num_desired_motions, &ransac_model_info[type]);
+ num_desired_motions, &ransac_model_info[type],
+ mem_alloc_failed);
}
diff --git a/aom_dsp/flow_estimation/ransac.h b/aom_dsp/flow_estimation/ransac.h
index 6047580..0529b6e 100644
--- a/aom_dsp/flow_estimation/ransac.h
+++ b/aom_dsp/flow_estimation/ransac.h
@@ -26,7 +26,7 @@
bool ransac(const Correspondence *matched_points, int npoints,
TransformationType type, MotionModel *motion_models,
- int num_desired_motions);
+ int num_desired_motions, bool *mem_alloc_failed);
#ifdef __cplusplus
}
diff --git a/aom_dsp/flow_estimation/x86/disflow_sse4.c b/aom_dsp/flow_estimation/x86/disflow_sse4.c
index a62e9a4..77784ee 100644
--- a/aom_dsp/flow_estimation/x86/disflow_sse4.c
+++ b/aom_dsp/flow_estimation/x86/disflow_sse4.c
@@ -61,12 +61,23 @@
//
// TODO(rachelbarker): Test speed/quality impact of using bilinear interpolation
// instad of bicubic interpolation
-static INLINE void compute_flow_error(const uint8_t *src, const uint8_t *ref,
- int width, int height, int stride, int x,
- int y, double u, double v, int16_t *dt) {
+static INLINE void compute_flow_vector(const uint8_t *src, const uint8_t *ref,
+ int width, int height, int stride, int x,
+ int y, double u, double v,
+ const int16_t *dx, const int16_t *dy,
+ int *b) {
// This function is written to do 8x8 convolutions only
assert(DISFLOW_PATCH_SIZE == 8);
+ // Accumulate 4 32-bit partial sums for each element of b
+ // These will be flattened at the end.
+ __m128i b0_acc = _mm_setzero_si128();
+ __m128i b1_acc = _mm_setzero_si128();
+#if CHECK_RESULTS
+ // Also keep a running sum using the C algorithm, for cross-checking
+ int c_result[2] = { 0 };
+#endif // CHECK_RESULTS
+
// Split offset into integer and fractional parts, and compute cubic
// interpolation kernels
const int u_int = (int)floor(u);
@@ -231,10 +242,20 @@
__m128i src_pixels = _mm_slli_epi16(_mm_cvtepu8_epi16(src_pixels_u8), 3);
// Calculate delta from the target patch
- __m128i err = _mm_sub_epi16(warped, src_pixels);
- _mm_storeu_si128((__m128i *)&dt[i * DISFLOW_PATCH_SIZE], err);
+ __m128i dt = _mm_sub_epi16(warped, src_pixels);
+
+ // Load 8 elements each of dx and dt, to pair with the 8 elements of dt
+ // that we have just computed. Then compute 8 partial sums of dx * dt
+ // and dy * dt, implicitly sum to give 4 partial sums of each, and
+ // accumulate.
+ __m128i dx_row = _mm_loadu_si128((__m128i *)&dx[i * DISFLOW_PATCH_SIZE]);
+ __m128i dy_row = _mm_loadu_si128((__m128i *)&dy[i * DISFLOW_PATCH_SIZE]);
+ b0_acc = _mm_add_epi32(b0_acc, _mm_madd_epi16(dx_row, dt));
+ b1_acc = _mm_add_epi32(b1_acc, _mm_madd_epi16(dy_row, dt));
#if CHECK_RESULTS
+ int16_t dt_arr[8];
+ memcpy(dt_arr, &dt, 8 * sizeof(*dt_arr));
for (int j = 0; j < DISFLOW_PATCH_SIZE; ++j) {
int16_t *p = &tmp[i * DISFLOW_PATCH_SIZE + j];
int arr[4] = { p[-DISFLOW_PATCH_SIZE], p[0], p[DISFLOW_PATCH_SIZE],
@@ -247,12 +268,28 @@
// of precision to match the scale of the dx and dy arrays.
const int c_warped = ROUND_POWER_OF_TWO(result, round_bits);
const int c_src_px = src[(x + j) + (y + i) * stride] << 3;
- const int c_err = c_warped - c_src_px;
- (void)c_err;
- assert(dt[i * DISFLOW_PATCH_SIZE + j] == c_err);
+ const int c_dt = c_warped - c_src_px;
+
+ assert(dt_arr[j] == c_dt);
+
+ c_result[0] += dx[i * DISFLOW_PATCH_SIZE + j] * c_dt;
+ c_result[1] += dy[i * DISFLOW_PATCH_SIZE + j] * c_dt;
}
#endif // CHECK_RESULTS
}
+
+ // Flatten the two sets of partial sums to find the final value of b
+ // We need to set b[0] = sum(b0_acc), b[1] = sum(b1_acc).
+ // We need to do 6 additions in total; a `hadd` instruction can take care
+ // of four of them, leaving two scalar additions.
+ __m128i partial_sum = _mm_hadd_epi32(b0_acc, b1_acc);
+ b[0] = _mm_extract_epi32(partial_sum, 0) + _mm_extract_epi32(partial_sum, 1);
+ b[1] = _mm_extract_epi32(partial_sum, 2) + _mm_extract_epi32(partial_sum, 3);
+
+#if CHECK_RESULTS
+ assert(b[0] == c_result[0]);
+ assert(b[1] == c_result[1]);
+#endif // CHECK_RESULTS
}
static INLINE void sobel_filter_x(const uint8_t *src, int src_stride,
@@ -401,50 +438,6 @@
}
}
-static INLINE void compute_flow_vector(const int16_t *dx, int dx_stride,
- const int16_t *dy, int dy_stride,
- const int16_t *dt, int dt_stride,
- int *b) {
- __m128i b0_acc = _mm_setzero_si128();
- __m128i b1_acc = _mm_setzero_si128();
-
- for (int i = 0; i < DISFLOW_PATCH_SIZE; i++) {
- // Need to load 8 values of dx, 8 of dy, 8 of dt, which conveniently
- // works out to one register each. Then just calculate dx * dt, dy * dt,
- // and (implicitly) sum horizontally in pairs.
- // This gives four 32-bit partial sums for each of b[0] and b[1],
- // which can be accumulated and summed at the end.
- __m128i dx_row = _mm_loadu_si128((__m128i *)&dx[i * dx_stride]);
- __m128i dy_row = _mm_loadu_si128((__m128i *)&dy[i * dy_stride]);
- __m128i dt_row = _mm_loadu_si128((__m128i *)&dt[i * dt_stride]);
-
- b0_acc = _mm_add_epi32(b0_acc, _mm_madd_epi16(dx_row, dt_row));
- b1_acc = _mm_add_epi32(b1_acc, _mm_madd_epi16(dy_row, dt_row));
- }
-
- // We need to set b[0] = sum(b0_acc), b[1] = sum(b1_acc).
- // We might as well use a `hadd` instruction to do 4 of the additions
- // needed here. Then that just leaves two more additions, which can be
- // done in scalar code
- __m128i partial_sum = _mm_hadd_epi32(b0_acc, b1_acc);
- b[0] = _mm_extract_epi32(partial_sum, 0) + _mm_extract_epi32(partial_sum, 1);
- b[1] = _mm_extract_epi32(partial_sum, 2) + _mm_extract_epi32(partial_sum, 3);
-
-#if CHECK_RESULTS
- int c_result[2] = { 0 };
-
- for (int i = 0; i < DISFLOW_PATCH_SIZE; i++) {
- for (int j = 0; j < DISFLOW_PATCH_SIZE; j++) {
- c_result[0] += dx[i * dx_stride + j] * dt[i * dt_stride + j];
- c_result[1] += dy[i * dy_stride + j] * dt[i * dt_stride + j];
- }
- }
-
- assert(b[0] == c_result[0]);
- assert(b[1] == c_result[1]);
-#endif // CHECK_RESULTS
-}
-
static INLINE void compute_flow_matrix(const int16_t *dx, int dx_stride,
const int16_t *dy, int dy_stride,
double *M) {
@@ -528,7 +521,6 @@
double M[4];
double M_inv[4];
int b[2];
- int16_t dt[DISFLOW_PATCH_SIZE * DISFLOW_PATCH_SIZE];
int16_t dx[DISFLOW_PATCH_SIZE * DISFLOW_PATCH_SIZE];
int16_t dy[DISFLOW_PATCH_SIZE * DISFLOW_PATCH_SIZE];
@@ -541,9 +533,8 @@
invert_2x2(M, M_inv);
for (int itr = 0; itr < DISFLOW_MAX_ITR; itr++) {
- compute_flow_error(src, ref, width, height, stride, x, y, *u, *v, dt);
- compute_flow_vector(dx, DISFLOW_PATCH_SIZE, dy, DISFLOW_PATCH_SIZE, dt,
- DISFLOW_PATCH_SIZE, b);
+ compute_flow_vector(src, ref, width, height, stride, x, y, *u, *v, dx, dy,
+ b);
// Solve flow equations to find a better estimate for the flow vector
// at this point
diff --git a/aom_dsp/pyramid.c b/aom_dsp/pyramid.c
index a26d302..324a18b 100644
--- a/aom_dsp/pyramid.c
+++ b/aom_dsp/pyramid.c
@@ -112,7 +112,7 @@
return NULL;
}
- pyr->layers = aom_calloc(n_levels, sizeof(PyramidLayer));
+ pyr->layers = aom_calloc(n_levels, sizeof(*pyr->layers));
if (!pyr->layers) {
aom_free(pyr);
return NULL;
@@ -125,10 +125,10 @@
// These are gathered up first, so that we can allocate all pyramid levels
// in a single buffer
size_t buffer_size = 0;
- size_t *layer_offsets = aom_calloc(n_levels, sizeof(size_t));
+ size_t *layer_offsets = aom_calloc(n_levels, sizeof(*layer_offsets));
if (!layer_offsets) {
- aom_free(pyr);
aom_free(pyr->layers);
+ aom_free(pyr);
return NULL;
}
@@ -195,8 +195,8 @@
pyr->buffer_alloc =
aom_memalign(PYRAMID_ALIGNMENT, buffer_size * sizeof(*pyr->buffer_alloc));
if (!pyr->buffer_alloc) {
- aom_free(pyr);
aom_free(pyr->layers);
+ aom_free(pyr);
aom_free(layer_offsets);
return NULL;
}
@@ -250,7 +250,7 @@
// Compute coarse to fine pyramids for a frame
// This must only be called while holding frame_pyr->mutex
-static INLINE void fill_pyramid(const YV12_BUFFER_CONFIG *frame, int bit_depth,
+static INLINE bool fill_pyramid(const YV12_BUFFER_CONFIG *frame, int bit_depth,
ImagePyramid *frame_pyr) {
int n_levels = frame_pyr->n_levels;
const int frame_width = frame->y_crop_width;
@@ -312,11 +312,13 @@
// 2) Up/downsampling by a factor of 2 can be implemented much more
// efficiently than up/downsampling by a generic ratio.
// TODO(rachelbarker): Use optimized downsample-by-2 function
- av1_resize_plane(prev_buffer, this_height << 1, this_width << 1,
- prev_stride, this_buffer, this_height, this_width,
- this_stride);
+ if (!av1_resize_plane(prev_buffer, this_height << 1, this_width << 1,
+ prev_stride, this_buffer, this_height, this_width,
+ this_stride))
+ return false;
fill_border(this_buffer, this_width, this_height, this_stride);
}
+ return true;
}
// Fill out a downsampling pyramid for a given frame.
@@ -331,7 +333,7 @@
//
// However, if the input frame has a side of length < MIN_PYRAMID_SIZE,
// we will still construct the top level.
-void aom_compute_pyramid(const YV12_BUFFER_CONFIG *frame, int bit_depth,
+bool aom_compute_pyramid(const YV12_BUFFER_CONFIG *frame, int bit_depth,
ImagePyramid *pyr) {
assert(pyr);
@@ -344,9 +346,9 @@
#endif // CONFIG_MULTITHREAD
if (!pyr->valid) {
- fill_pyramid(frame, bit_depth, pyr);
- pyr->valid = true;
+ pyr->valid = fill_pyramid(frame, bit_depth, pyr);
}
+ bool valid = pyr->valid;
// At this point, the pyramid is guaranteed to be valid, and can be safely
// read from without holding the mutex any more
@@ -354,6 +356,7 @@
#if CONFIG_MULTITHREAD
pthread_mutex_unlock(&pyr->mutex);
#endif // CONFIG_MULTITHREAD
+ return valid;
}
#ifndef NDEBUG
diff --git a/aom_dsp/pyramid.h b/aom_dsp/pyramid.h
index 812aae1..9442a1f 100644
--- a/aom_dsp/pyramid.h
+++ b/aom_dsp/pyramid.h
@@ -100,7 +100,7 @@
//
// However, if the input frame has a side of length < MIN_PYRAMID_SIZE,
// we will still construct the top level.
-void aom_compute_pyramid(const YV12_BUFFER_CONFIG *frame, int bit_depth,
+bool aom_compute_pyramid(const YV12_BUFFER_CONFIG *frame, int bit_depth,
ImagePyramid *pyr);
#ifndef NDEBUG
diff --git a/aom_dsp/quantize.c b/aom_dsp/quantize.c
index 8dd5b0b..e5c960b 100644
--- a/aom_dsp/quantize.c
+++ b/aom_dsp/quantize.c
@@ -11,6 +11,7 @@
#include "aom_dsp/quantize.h"
#include "aom_mem/aom_mem.h"
+#include "config/aom_dsp_rtcd.h"
#if !CONFIG_REALTIME_ONLY
void aom_quantize_b_adaptive_helper_c(
diff --git a/aom_dsp/sad.c b/aom_dsp/sad.c
index 341a5ff..8d69e3b 100644
--- a/aom_dsp/sad.c
+++ b/aom_dsp/sad.c
@@ -257,32 +257,32 @@
highbd_sad(src, 2 * src_stride, ref, 2 * ref_stride, (m), (n / 2)); \
}
-#define HIGHBD_SAD_MXNX4D(m, n) \
- void aom_highbd_sad##m##x##n##x4d_c(const uint8_t *src, int src_stride, \
- const uint8_t *const ref_array[], \
- int ref_stride, uint32_t *sad_array) { \
- int i; \
- for (i = 0; i < 4; ++i) { \
- sad_array[i] = aom_highbd_sad##m##x##n##_c(src, src_stride, \
- ref_array[i], ref_stride); \
- } \
- } \
- void aom_highbd_sad_skip_##m##x##n##x4d_c( \
- const uint8_t *src, int src_stride, const uint8_t *const ref_array[], \
- int ref_stride, uint32_t *sad_array) { \
- int i; \
- for (i = 0; i < 4; ++i) { \
- sad_array[i] = 2 * highbd_sad(src, 2 * src_stride, ref_array[i], \
- 2 * ref_stride, (m), (n / 2)); \
- } \
+#define HIGHBD_SAD_MXNX4D(m, n) \
+ void aom_highbd_sad##m##x##n##x4d_c(const uint8_t *src, int src_stride, \
+ const uint8_t *const ref_array[4], \
+ int ref_stride, uint32_t sad_array[4]) { \
+ int i; \
+ for (i = 0; i < 4; ++i) { \
+ sad_array[i] = aom_highbd_sad##m##x##n##_c(src, src_stride, \
+ ref_array[i], ref_stride); \
+ } \
+ } \
+ void aom_highbd_sad_skip_##m##x##n##x4d_c( \
+ const uint8_t *src, int src_stride, const uint8_t *const ref_array[4], \
+ int ref_stride, uint32_t sad_array[4]) { \
+ int i; \
+ for (i = 0; i < 4; ++i) { \
+ sad_array[i] = 2 * highbd_sad(src, 2 * src_stride, ref_array[i], \
+ 2 * ref_stride, (m), (n / 2)); \
+ } \
}
// Call SIMD version of aom_highbd_sad_mxnx4d if the 3d version is unavailable.
-#define HIGHBD_SAD_MXNX3D(m, n) \
- void aom_highbd_sad##m##x##n##x3d_c(const uint8_t *src, int src_stride, \
- const uint8_t *const ref_array[], \
- int ref_stride, uint32_t *sad_array) { \
- aom_highbd_sad##m##x##n##x4d(src, src_stride, ref_array, ref_stride, \
- sad_array); \
+#define HIGHBD_SAD_MXNX3D(m, n) \
+ void aom_highbd_sad##m##x##n##x3d_c(const uint8_t *src, int src_stride, \
+ const uint8_t *const ref_array[4], \
+ int ref_stride, uint32_t sad_array[4]) { \
+ aom_highbd_sad##m##x##n##x4d(src, src_stride, ref_array, ref_stride, \
+ sad_array); \
}
// 128x128
diff --git a/aom_dsp/simd/v128_intrinsics_arm.h b/aom_dsp/simd/v128_intrinsics_arm.h
deleted file mode 100644
index 6488de7..0000000
--- a/aom_dsp/simd/v128_intrinsics_arm.h
+++ /dev/null
@@ -1,977 +0,0 @@
-/*
- * Copyright (c) 2016, Alliance for Open Media. All rights reserved
- *
- * This source code is subject to the terms of the BSD 2 Clause License and
- * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
- * was not distributed with this source code in the LICENSE file, you can
- * obtain it at www.aomedia.org/license/software. If the Alliance for Open
- * Media Patent License 1.0 was not distributed with this source code in the
- * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
- */
-
-#ifndef AOM_AOM_DSP_SIMD_V128_INTRINSICS_ARM_H_
-#define AOM_AOM_DSP_SIMD_V128_INTRINSICS_ARM_H_
-
-#include <arm_neon.h>
-
-#include "config/aom_config.h"
-
-#include "aom_dsp/simd/v64_intrinsics_arm.h"
-
-typedef int64x2_t v128;
-
-SIMD_INLINE uint32_t v128_low_u32(v128 a) {
- return v64_low_u32(vget_low_s64(a));
-}
-
-SIMD_INLINE v64 v128_low_v64(v128 a) { return vget_low_s64(a); }
-
-SIMD_INLINE v64 v128_high_v64(v128 a) { return vget_high_s64(a); }
-
-SIMD_INLINE v128 v128_from_v64(v64 a, v64 b) { return vcombine_s64(b, a); }
-
-SIMD_INLINE v128 v128_from_64(uint64_t a, uint64_t b) {
- return vcombine_s64(vcreate_s64(b), vcreate_s64(a));
-}
-
-SIMD_INLINE v128 v128_from_32(uint32_t a, uint32_t b, uint32_t c, uint32_t d) {
- return vcombine_s64(v64_from_32(c, d), v64_from_32(a, b));
-}
-
-SIMD_INLINE v128 v128_load_aligned(const void *p) {
- return vreinterpretq_s64_u8(vld1q_u8((const uint8_t *)p));
-}
-
-SIMD_INLINE v128 v128_load_unaligned(const void *p) {
- return v128_load_aligned(p);
-}
-
-SIMD_INLINE void v128_store_aligned(void *p, v128 r) {
- vst1q_u8((uint8_t *)p, vreinterpretq_u8_s64(r));
-}
-
-SIMD_INLINE void v128_store_unaligned(void *p, v128 r) {
- vst1q_u8((uint8_t *)p, vreinterpretq_u8_s64(r));
-}
-
-SIMD_INLINE v128 v128_align(v128 a, v128 b, unsigned int c) {
-// The following functions require an immediate.
-// Some compilers will check this during optimisation, others wont.
-#if defined(__OPTIMIZE__) && __OPTIMIZE__ && !defined(__clang__)
- return c ? vreinterpretq_s64_s8(
- vextq_s8(vreinterpretq_s8_s64(b), vreinterpretq_s8_s64(a), c))
- : b;
-#else
- return c < 8 ? v128_from_v64(v64_align(v128_low_v64(a), v128_high_v64(b), c),
- v64_align(v128_high_v64(b), v128_low_v64(b), c))
- : v128_from_v64(
- v64_align(v128_high_v64(a), v128_low_v64(a), c - 8),
- v64_align(v128_low_v64(a), v128_high_v64(b), c - 8));
-#endif
-}
-
-SIMD_INLINE v128 v128_zero(void) { return vreinterpretq_s64_u8(vdupq_n_u8(0)); }
-
-SIMD_INLINE v128 v128_ones(void) {
- return vreinterpretq_s64_u8(vdupq_n_u8(-1));
-}
-
-SIMD_INLINE v128 v128_dup_8(uint8_t x) {
- return vreinterpretq_s64_u8(vdupq_n_u8(x));
-}
-
-SIMD_INLINE v128 v128_dup_16(uint16_t x) {
- return vreinterpretq_s64_u16(vdupq_n_u16(x));
-}
-
-SIMD_INLINE v128 v128_dup_32(uint32_t x) {
- return vreinterpretq_s64_u32(vdupq_n_u32(x));
-}
-
-SIMD_INLINE v128 v128_dup_64(uint64_t x) {
- return vreinterpretq_s64_u64(vdupq_n_u64(x));
-}
-
-SIMD_INLINE int64_t v128_dotp_su8(v128 a, v128 b) {
- int16x8_t t1 = vmulq_s16(
- vmovl_s8(vreinterpret_s8_s64(vget_low_s64(a))),
- vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_s64(vget_low_s64(b)))));
- int16x8_t t2 = vmulq_s16(
- vmovl_s8(vreinterpret_s8_s64(vget_high_s64(a))),
- vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_s64(vget_high_s64(b)))));
-#if AOM_ARCH_AARCH64
- return vaddlvq_s16(t1) + vaddlvq_s16(t2);
-#else
- int64x2_t t = vpaddlq_s32(vaddq_s32(vpaddlq_s16(t1), vpaddlq_s16(t2)));
- return vget_lane_s64(vadd_s64(vget_high_s64(t), vget_low_s64(t)), 0);
-#endif
-}
-
-SIMD_INLINE int64_t v128_dotp_s16(v128 a, v128 b) {
- return v64_dotp_s16(vget_high_s64(a), vget_high_s64(b)) +
- v64_dotp_s16(vget_low_s64(a), vget_low_s64(b));
-}
-
-SIMD_INLINE int64_t v128_dotp_s32(v128 a, v128 b) {
- int64x2_t t = vpaddlq_s32(
- vmulq_s32(vreinterpretq_s32_s64(a), vreinterpretq_s32_s64(b)));
- return vget_lane_s64(vadd_s64(vget_high_s64(t), vget_low_s64(t)), 0);
-}
-
-SIMD_INLINE uint64_t v128_hadd_u8(v128 x) {
-#if AOM_ARCH_AARCH64
- return vaddlvq_u8(vreinterpretq_u8_s64(x));
-#else
- uint64x2_t t = vpaddlq_u32(vpaddlq_u16(vpaddlq_u8(vreinterpretq_u8_s64(x))));
- return vget_lane_s32(
- vreinterpret_s32_u64(vadd_u64(vget_high_u64(t), vget_low_u64(t))), 0);
-#endif
-}
-
-SIMD_INLINE v128 v128_padd_s16(v128 a) {
- return vreinterpretq_s64_s32(vpaddlq_s16(vreinterpretq_s16_s64(a)));
-}
-
-SIMD_INLINE v128 v128_padd_u8(v128 a) {
- return vreinterpretq_s64_u16(vpaddlq_u8(vreinterpretq_u8_s64(a)));
-}
-
-typedef struct {
- sad64_internal hi, lo;
-} sad128_internal;
-
-SIMD_INLINE sad128_internal v128_sad_u8_init(void) {
- sad128_internal s;
- s.hi = s.lo = vdupq_n_u16(0);
- return s;
-}
-
-/* Implementation dependent return value. Result must be finalised with
- v128_sad_u8_sum().
- The result for more than 32 v128_sad_u8() calls is undefined. */
-SIMD_INLINE sad128_internal v128_sad_u8(sad128_internal s, v128 a, v128 b) {
- sad128_internal r;
- r.hi = v64_sad_u8(s.hi, vget_high_s64(a), vget_high_s64(b));
- r.lo = v64_sad_u8(s.lo, vget_low_s64(a), vget_low_s64(b));
- return r;
-}
-
-SIMD_INLINE uint32_t v128_sad_u8_sum(sad128_internal s) {
-#if AOM_ARCH_AARCH64
- return vaddlvq_u16(s.hi) + vaddlvq_u16(s.lo);
-#else
- uint64x2_t t = vpaddlq_u32(vpaddlq_u16(vaddq_u16(s.hi, s.lo)));
- return (uint32_t)vget_lane_u64(vadd_u64(vget_high_u64(t), vget_low_u64(t)),
- 0);
-#endif
-}
-
-typedef struct {
- ssd64_internal hi, lo;
-} ssd128_internal;
-
-SIMD_INLINE ssd128_internal v128_ssd_u8_init(void) {
- ssd128_internal s;
- s.hi = s.lo = v64_ssd_u8_init();
- return s;
-}
-
-/* Implementation dependent return value. Result must be finalised with
- * v128_ssd_u8_sum(). */
-SIMD_INLINE ssd128_internal v128_ssd_u8(ssd128_internal s, v128 a, v128 b) {
- ssd128_internal r;
- r.hi = v64_ssd_u8(s.hi, vget_high_s64(a), vget_high_s64(b));
- r.lo = v64_ssd_u8(s.lo, vget_low_s64(a), vget_low_s64(b));
- return r;
-}
-
-SIMD_INLINE uint32_t v128_ssd_u8_sum(ssd128_internal s) {
- return (uint32_t)(v64_ssd_u8_sum(s.hi) + v64_ssd_u8_sum(s.lo));
-}
-
-SIMD_INLINE v128 v128_or(v128 x, v128 y) { return vorrq_s64(x, y); }
-
-SIMD_INLINE v128 v128_xor(v128 x, v128 y) { return veorq_s64(x, y); }
-
-SIMD_INLINE v128 v128_and(v128 x, v128 y) { return vandq_s64(x, y); }
-
-SIMD_INLINE v128 v128_andn(v128 x, v128 y) { return vbicq_s64(x, y); }
-
-SIMD_INLINE v128 v128_add_8(v128 x, v128 y) {
- return vreinterpretq_s64_u8(
- vaddq_u8(vreinterpretq_u8_s64(x), vreinterpretq_u8_s64(y)));
-}
-
-SIMD_INLINE v128 v128_sadd_u8(v128 x, v128 y) {
- return vreinterpretq_s64_u8(
- vqaddq_u8(vreinterpretq_u8_s64(x), vreinterpretq_u8_s64(y)));
-}
-
-SIMD_INLINE v128 v128_sadd_s8(v128 x, v128 y) {
- return vreinterpretq_s64_s8(
- vqaddq_s8(vreinterpretq_s8_s64(x), vreinterpretq_s8_s64(y)));
-}
-
-SIMD_INLINE v128 v128_add_16(v128 x, v128 y) {
- return vreinterpretq_s64_s16(
- vaddq_s16(vreinterpretq_s16_s64(x), vreinterpretq_s16_s64(y)));
-}
-
-SIMD_INLINE v128 v128_sadd_s16(v128 x, v128 y) {
- return vreinterpretq_s64_s16(
- vqaddq_s16(vreinterpretq_s16_s64(x), vreinterpretq_s16_s64(y)));
-}
-
-SIMD_INLINE v128 v128_add_32(v128 x, v128 y) {
- return vreinterpretq_s64_u32(
- vaddq_u32(vreinterpretq_u32_s64(x), vreinterpretq_u32_s64(y)));
-}
-
-SIMD_INLINE v128 v128_add_64(v128 x, v128 y) {
- return vreinterpretq_s64_u64(
- vaddq_u64(vreinterpretq_u64_s64(x), vreinterpretq_u64_s64(y)));
-}
-
-SIMD_INLINE v128 v128_sub_8(v128 x, v128 y) {
- return vreinterpretq_s64_u8(
- vsubq_u8(vreinterpretq_u8_s64(x), vreinterpretq_u8_s64(y)));
-}
-
-SIMD_INLINE v128 v128_sub_16(v128 x, v128 y) {
- return vreinterpretq_s64_s16(
- vsubq_s16(vreinterpretq_s16_s64(x), vreinterpretq_s16_s64(y)));
-}
-
-SIMD_INLINE v128 v128_ssub_s16(v128 x, v128 y) {
- return vreinterpretq_s64_s16(
- vqsubq_s16(vreinterpretq_s16_s64(x), vreinterpretq_s16_s64(y)));
-}
-
-SIMD_INLINE v128 v128_ssub_u16(v128 x, v128 y) {
- return vreinterpretq_s64_u16(
- vqsubq_u16(vreinterpretq_u16_s64(x), vreinterpretq_u16_s64(y)));
-}
-
-SIMD_INLINE v128 v128_ssub_u8(v128 x, v128 y) {
- return vreinterpretq_s64_u8(
- vqsubq_u8(vreinterpretq_u8_s64(x), vreinterpretq_u8_s64(y)));
-}
-
-SIMD_INLINE v128 v128_ssub_s8(v128 x, v128 y) {
- return vreinterpretq_s64_s8(
- vqsubq_s8(vreinterpretq_s8_s64(x), vreinterpretq_s8_s64(y)));
-}
-
-SIMD_INLINE v128 v128_sub_32(v128 x, v128 y) {
- return vreinterpretq_s64_s32(
- vsubq_s32(vreinterpretq_s32_s64(x), vreinterpretq_s32_s64(y)));
-}
-
-SIMD_INLINE v128 v128_sub_64(v128 x, v128 y) { return vsubq_s64(x, y); }
-
-SIMD_INLINE v128 v128_abs_s16(v128 x) {
- return vreinterpretq_s64_s16(vabsq_s16(vreinterpretq_s16_s64(x)));
-}
-
-SIMD_INLINE v128 v128_abs_s8(v128 x) {
- return vreinterpretq_s64_s8(vabsq_s8(vreinterpretq_s8_s64(x)));
-}
-
-SIMD_INLINE v128 v128_mul_s16(v64 a, v64 b) {
- return vreinterpretq_s64_s32(
- vmull_s16(vreinterpret_s16_s64(a), vreinterpret_s16_s64(b)));
-}
-
-SIMD_INLINE v128 v128_mullo_s16(v128 a, v128 b) {
- return vreinterpretq_s64_s16(
- vmulq_s16(vreinterpretq_s16_s64(a), vreinterpretq_s16_s64(b)));
-}
-
-SIMD_INLINE v128 v128_mulhi_s16(v128 a, v128 b) {
-#if AOM_ARCH_AARCH64
- return vreinterpretq_s64_s16(vuzp2q_s16(
- vreinterpretq_s16_s32(vmull_s16(vreinterpret_s16_s64(vget_low_s64(a)),
- vreinterpret_s16_s64(vget_low_s64(b)))),
- vreinterpretq_s16_s32(
- vmull_high_s16(vreinterpretq_s16_s64(a), vreinterpretq_s16_s64(b)))));
-#else
- return v128_from_v64(v64_mulhi_s16(vget_high_s64(a), vget_high_s64(b)),
- v64_mulhi_s16(vget_low_s64(a), vget_low_s64(b)));
-#endif
-}
-
-SIMD_INLINE v128 v128_mullo_s32(v128 a, v128 b) {
- return vreinterpretq_s64_s32(
- vmulq_s32(vreinterpretq_s32_s64(a), vreinterpretq_s32_s64(b)));
-}
-
-SIMD_INLINE v128 v128_madd_s16(v128 a, v128 b) {
-#if AOM_ARCH_AARCH64
- int32x4_t t1 = vmull_s16(vreinterpret_s16_s64(vget_low_s64(a)),
- vreinterpret_s16_s64(vget_low_s64(b)));
- int32x4_t t2 =
- vmull_high_s16(vreinterpretq_s16_s64(a), vreinterpretq_s16_s64(b));
- return vreinterpretq_s64_s32(vpaddq_s32(t1, t2));
-#else
- return v128_from_v64(v64_madd_s16(vget_high_s64(a), vget_high_s64(b)),
- v64_madd_s16(vget_low_s64(a), vget_low_s64(b)));
-#endif
-}
-
-SIMD_INLINE v128 v128_madd_us8(v128 a, v128 b) {
-#if AOM_ARCH_AARCH64
- int16x8_t t1 = vmulq_s16(
- vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_s64(vget_low_s64(a)))),
- vmovl_s8(vreinterpret_s8_s64(vget_low_s64(b))));
- int16x8_t t2 = vmulq_s16(
- vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_s64(vget_high_s64(a)))),
- vmovl_s8(vreinterpret_s8_s64(vget_high_s64(b))));
- return vreinterpretq_s64_s16(
- vqaddq_s16(vuzp1q_s16(t1, t2), vuzp2q_s16(t1, t2)));
-#else
- return v128_from_v64(v64_madd_us8(vget_high_s64(a), vget_high_s64(b)),
- v64_madd_us8(vget_low_s64(a), vget_low_s64(b)));
-#endif
-}
-
-SIMD_INLINE v128 v128_avg_u8(v128 x, v128 y) {
- return vreinterpretq_s64_u8(
- vrhaddq_u8(vreinterpretq_u8_s64(x), vreinterpretq_u8_s64(y)));
-}
-
-SIMD_INLINE v128 v128_rdavg_u8(v128 x, v128 y) {
- return vreinterpretq_s64_u8(
- vhaddq_u8(vreinterpretq_u8_s64(x), vreinterpretq_u8_s64(y)));
-}
-
-SIMD_INLINE v128 v128_rdavg_u16(v128 x, v128 y) {
- return vreinterpretq_s64_u16(
- vhaddq_u16(vreinterpretq_u16_s64(x), vreinterpretq_u16_s64(y)));
-}
-
-SIMD_INLINE v128 v128_avg_u16(v128 x, v128 y) {
- return vreinterpretq_s64_u16(
- vrhaddq_u16(vreinterpretq_u16_s64(x), vreinterpretq_u16_s64(y)));
-}
-
-SIMD_INLINE v128 v128_min_u8(v128 x, v128 y) {
- return vreinterpretq_s64_u8(
- vminq_u8(vreinterpretq_u8_s64(x), vreinterpretq_u8_s64(y)));
-}
-
-SIMD_INLINE v128 v128_max_u8(v128 x, v128 y) {
- return vreinterpretq_s64_u8(
- vmaxq_u8(vreinterpretq_u8_s64(x), vreinterpretq_u8_s64(y)));
-}
-
-SIMD_INLINE v128 v128_min_s8(v128 x, v128 y) {
- return vreinterpretq_s64_s8(
- vminq_s8(vreinterpretq_s8_s64(x), vreinterpretq_s8_s64(y)));
-}
-
-SIMD_INLINE uint32_t v128_movemask_8(v128 a) {
- a = vreinterpretq_s64_u8(vcltq_s8(vreinterpretq_s8_s64(a), vdupq_n_s8(0)));
-#if AOM_ARCH_AARCH64
- uint8x16_t m =
- vandq_u8(vreinterpretq_u8_s64(a),
- vreinterpretq_u8_u64(vdupq_n_u64(0x8040201008040201ULL)));
- return vaddv_u8(vget_low_u8(m)) + (vaddv_u8(vget_high_u8(m)) << 8);
-#else
- uint64x2_t m = vpaddlq_u32(vpaddlq_u16(vpaddlq_u8(
- vandq_u8(vreinterpretq_u8_s64(a),
- vreinterpretq_u8_u64(vdupq_n_u64(0x8040201008040201ULL))))));
- int64x2_t s = vreinterpretq_s64_u64(m);
- return v64_low_u32(v64_ziplo_8(vget_high_s64(s), vget_low_s64(s)));
-#endif
-}
-
-SIMD_INLINE v128 v128_blend_8(v128 a, v128 b, v128 c) {
- c = vreinterpretq_s64_u8(vcltq_s8(vreinterpretq_s8_s64(c), vdupq_n_s8(0)));
- return v128_or(v128_and(b, c), v128_andn(a, c));
-}
-
-SIMD_INLINE v128 v128_max_s8(v128 x, v128 y) {
- return vreinterpretq_s64_s8(
- vmaxq_s8(vreinterpretq_s8_s64(x), vreinterpretq_s8_s64(y)));
-}
-
-SIMD_INLINE v128 v128_min_s16(v128 x, v128 y) {
- return vreinterpretq_s64_s16(
- vminq_s16(vreinterpretq_s16_s64(x), vreinterpretq_s16_s64(y)));
-}
-
-SIMD_INLINE v128 v128_max_s16(v128 x, v128 y) {
- return vreinterpretq_s64_s16(
- vmaxq_s16(vreinterpretq_s16_s64(x), vreinterpretq_s16_s64(y)));
-}
-
-SIMD_INLINE v128 v128_min_s32(v128 x, v128 y) {
- return vreinterpretq_s64_s32(
- vminq_s32(vreinterpretq_s32_s64(x), vreinterpretq_s32_s64(y)));
-}
-
-SIMD_INLINE v128 v128_max_s32(v128 x, v128 y) {
- return vreinterpretq_s64_s32(
- vmaxq_s32(vreinterpretq_s32_s64(x), vreinterpretq_s32_s64(y)));
-}
-
-SIMD_INLINE v128 v128_ziplo_8(v128 x, v128 y) {
-#if AOM_ARCH_AARCH64
- return vreinterpretq_s64_u8(
- vzip1q_u8(vreinterpretq_u8_s64(y), vreinterpretq_u8_s64(x)));
-#else
- uint8x16x2_t r = vzipq_u8(vreinterpretq_u8_s64(y), vreinterpretq_u8_s64(x));
- return vreinterpretq_s64_u8(r.val[0]);
-#endif
-}
-
-SIMD_INLINE v128 v128_ziphi_8(v128 x, v128 y) {
-#if AOM_ARCH_AARCH64
- return vreinterpretq_s64_u8(
- vzip2q_u8(vreinterpretq_u8_s64(y), vreinterpretq_u8_s64(x)));
-#else
- uint8x16x2_t r = vzipq_u8(vreinterpretq_u8_s64(y), vreinterpretq_u8_s64(x));
- return vreinterpretq_s64_u8(r.val[1]);
-#endif
-}
-
-SIMD_INLINE v128 v128_zip_8(v64 x, v64 y) {
- uint8x8x2_t r = vzip_u8(vreinterpret_u8_s64(y), vreinterpret_u8_s64(x));
- return vreinterpretq_s64_u8(vcombine_u8(r.val[0], r.val[1]));
-}
-
-SIMD_INLINE v128 v128_ziplo_16(v128 x, v128 y) {
-#if AOM_ARCH_AARCH64
- return vreinterpretq_s64_u16(
- vzip1q_u16(vreinterpretq_u16_s64(y), vreinterpretq_u16_s64(x)));
-#else
- int16x8x2_t r = vzipq_s16(vreinterpretq_s16_s64(y), vreinterpretq_s16_s64(x));
- return vreinterpretq_s64_s16(r.val[0]);
-#endif
-}
-
-SIMD_INLINE v128 v128_ziphi_16(v128 x, v128 y) {
-#if AOM_ARCH_AARCH64
- return vreinterpretq_s64_u16(
- vzip2q_u16(vreinterpretq_u16_s64(y), vreinterpretq_u16_s64(x)));
-#else
- int16x8x2_t r = vzipq_s16(vreinterpretq_s16_s64(y), vreinterpretq_s16_s64(x));
- return vreinterpretq_s64_s16(r.val[1]);
-#endif
-}
-
-SIMD_INLINE v128 v128_zip_16(v64 x, v64 y) {
- uint16x4x2_t r = vzip_u16(vreinterpret_u16_s64(y), vreinterpret_u16_s64(x));
- return vreinterpretq_s64_u16(vcombine_u16(r.val[0], r.val[1]));
-}
-
-SIMD_INLINE v128 v128_ziplo_32(v128 x, v128 y) {
-#if AOM_ARCH_AARCH64
- return vreinterpretq_s64_u32(
- vzip1q_u32(vreinterpretq_u32_s64(y), vreinterpretq_u32_s64(x)));
-#else
- int32x4x2_t r = vzipq_s32(vreinterpretq_s32_s64(y), vreinterpretq_s32_s64(x));
- return vreinterpretq_s64_s32(r.val[0]);
-#endif
-}
-
-SIMD_INLINE v128 v128_ziphi_32(v128 x, v128 y) {
-#if AOM_ARCH_AARCH64
- return vreinterpretq_s64_u32(
- vzip2q_u32(vreinterpretq_u32_s64(y), vreinterpretq_u32_s64(x)));
-#else
- int32x4x2_t r = vzipq_s32(vreinterpretq_s32_s64(y), vreinterpretq_s32_s64(x));
- return vreinterpretq_s64_s32(r.val[1]);
-#endif
-}
-
-SIMD_INLINE v128 v128_zip_32(v64 x, v64 y) {
- uint32x2x2_t r = vzip_u32(vreinterpret_u32_s64(y), vreinterpret_u32_s64(x));
- return vreinterpretq_s64_u32(vcombine_u32(r.val[0], r.val[1]));
-}
-
-SIMD_INLINE v128 v128_ziplo_64(v128 a, v128 b) {
- return v128_from_v64(vget_low_s64(a), vget_low_s64(b));
-}
-
-SIMD_INLINE v128 v128_ziphi_64(v128 a, v128 b) {
- return v128_from_v64(vget_high_s64(a), vget_high_s64(b));
-}
-
-SIMD_INLINE v128 v128_unziplo_8(v128 x, v128 y) {
-#if AOM_ARCH_AARCH64
- return vreinterpretq_s64_u8(
- vuzp1q_u8(vreinterpretq_u8_s64(y), vreinterpretq_u8_s64(x)));
-#else
- uint8x16x2_t r = vuzpq_u8(vreinterpretq_u8_s64(y), vreinterpretq_u8_s64(x));
- return vreinterpretq_s64_u8(r.val[0]);
-#endif
-}
-
-SIMD_INLINE v128 v128_unziphi_8(v128 x, v128 y) {
-#if AOM_ARCH_AARCH64
- return vreinterpretq_s64_u8(
- vuzp2q_u8(vreinterpretq_u8_s64(y), vreinterpretq_u8_s64(x)));
-#else
- uint8x16x2_t r = vuzpq_u8(vreinterpretq_u8_s64(y), vreinterpretq_u8_s64(x));
- return vreinterpretq_s64_u8(r.val[1]);
-#endif
-}
-
-SIMD_INLINE v128 v128_unziplo_16(v128 x, v128 y) {
-#if AOM_ARCH_AARCH64
- return vreinterpretq_s64_u16(
- vuzp1q_u16(vreinterpretq_u16_s64(y), vreinterpretq_u16_s64(x)));
-#else
- uint16x8x2_t r =
- vuzpq_u16(vreinterpretq_u16_s64(y), vreinterpretq_u16_s64(x));
- return vreinterpretq_s64_u16(r.val[0]);
-#endif
-}
-
-SIMD_INLINE v128 v128_unziphi_16(v128 x, v128 y) {
-#if AOM_ARCH_AARCH64
- return vreinterpretq_s64_u16(
- vuzp2q_u16(vreinterpretq_u16_s64(y), vreinterpretq_u16_s64(x)));
-#else
- uint16x8x2_t r =
- vuzpq_u16(vreinterpretq_u16_s64(y), vreinterpretq_u16_s64(x));
- return vreinterpretq_s64_u16(r.val[1]);
-#endif
-}
-
-SIMD_INLINE v128 v128_unziplo_32(v128 x, v128 y) {
-#if AOM_ARCH_AARCH64
- return vreinterpretq_s64_u32(
- vuzp1q_u32(vreinterpretq_u32_s64(y), vreinterpretq_u32_s64(x)));
-#else
- uint32x4x2_t r =
- vuzpq_u32(vreinterpretq_u32_s64(y), vreinterpretq_u32_s64(x));
- return vreinterpretq_s64_u32(r.val[0]);
-#endif
-}
-
-SIMD_INLINE v128 v128_unziphi_32(v128 x, v128 y) {
-#if AOM_ARCH_AARCH64
- return vreinterpretq_s64_u32(
- vuzp2q_u32(vreinterpretq_u32_s64(y), vreinterpretq_u32_s64(x)));
-#else
- uint32x4x2_t r =
- vuzpq_u32(vreinterpretq_u32_s64(y), vreinterpretq_u32_s64(x));
- return vreinterpretq_s64_u32(r.val[1]);
-#endif
-}
-
-SIMD_INLINE v128 v128_unpack_u8_s16(v64 a) {
- return vreinterpretq_s64_u16(vmovl_u8(vreinterpret_u8_s64(a)));
-}
-
-SIMD_INLINE v128 v128_unpacklo_u8_s16(v128 a) {
- return vreinterpretq_s64_u16(vmovl_u8(vreinterpret_u8_s64(vget_low_s64(a))));
-}
-
-SIMD_INLINE v128 v128_unpackhi_u8_s16(v128 a) {
- return vreinterpretq_s64_u16(vmovl_u8(vreinterpret_u8_s64(vget_high_s64(a))));
-}
-
-SIMD_INLINE v128 v128_unpack_s8_s16(v64 a) {
- return vreinterpretq_s64_s16(vmovl_s8(vreinterpret_s8_s64(a)));
-}
-
-SIMD_INLINE v128 v128_unpacklo_s8_s16(v128 a) {
- return vreinterpretq_s64_s16(vmovl_s8(vreinterpret_s8_s64(vget_low_s64(a))));
-}
-
-SIMD_INLINE v128 v128_unpackhi_s8_s16(v128 a) {
- return vreinterpretq_s64_s16(vmovl_s8(vreinterpret_s8_s64(vget_high_s64(a))));
-}
-
-SIMD_INLINE v128 v128_pack_s32_s16(v128 a, v128 b) {
- return v128_from_v64(
- vreinterpret_s64_s16(vqmovn_s32(vreinterpretq_s32_s64(a))),
- vreinterpret_s64_s16(vqmovn_s32(vreinterpretq_s32_s64(b))));
-}
-
-SIMD_INLINE v128 v128_pack_s32_u16(v128 a, v128 b) {
- return v128_from_v64(
- vreinterpret_s64_u16(vqmovun_s32(vreinterpretq_s32_s64(a))),
- vreinterpret_s64_u16(vqmovun_s32(vreinterpretq_s32_s64(b))));
-}
-
-SIMD_INLINE v128 v128_pack_s16_u8(v128 a, v128 b) {
- return v128_from_v64(
- vreinterpret_s64_u8(vqmovun_s16(vreinterpretq_s16_s64(a))),
- vreinterpret_s64_u8(vqmovun_s16(vreinterpretq_s16_s64(b))));
-}
-
-SIMD_INLINE v128 v128_pack_s16_s8(v128 a, v128 b) {
- return v128_from_v64(
- vreinterpret_s64_s8(vqmovn_s16(vreinterpretq_s16_s64(a))),
- vreinterpret_s64_s8(vqmovn_s16(vreinterpretq_s16_s64(b))));
-}
-
-SIMD_INLINE v128 v128_unpack_u16_s32(v64 a) {
- return vreinterpretq_s64_u32(vmovl_u16(vreinterpret_u16_s64(a)));
-}
-
-SIMD_INLINE v128 v128_unpack_s16_s32(v64 a) {
- return vreinterpretq_s64_s32(vmovl_s16(vreinterpret_s16_s64(a)));
-}
-
-SIMD_INLINE v128 v128_unpacklo_u16_s32(v128 a) {
- return vreinterpretq_s64_u32(
- vmovl_u16(vreinterpret_u16_s64(vget_low_s64(a))));
-}
-
-SIMD_INLINE v128 v128_unpacklo_s16_s32(v128 a) {
- return vreinterpretq_s64_s32(
- vmovl_s16(vreinterpret_s16_s64(vget_low_s64(a))));
-}
-
-SIMD_INLINE v128 v128_unpackhi_u16_s32(v128 a) {
- return vreinterpretq_s64_u32(
- vmovl_u16(vreinterpret_u16_s64(vget_high_s64(a))));
-}
-
-SIMD_INLINE v128 v128_unpackhi_s16_s32(v128 a) {
- return vreinterpretq_s64_s32(
- vmovl_s16(vreinterpret_s16_s64(vget_high_s64(a))));
-}
-
-SIMD_INLINE v128 v128_shuffle_8(v128 x, v128 pattern) {
-#if AOM_ARCH_AARCH64
- return vreinterpretq_s64_u8(
- vqtbl1q_u8(vreinterpretq_u8_s64(x), vreinterpretq_u8_s64(pattern)));
-#else
- uint8x8x2_t p = { { vget_low_u8(vreinterpretq_u8_s64(x)),
- vget_high_u8(vreinterpretq_u8_s64(x)) } };
- uint8x8_t shuffle_hi =
- vtbl2_u8(p, vreinterpret_u8_s64(vget_high_s64(pattern)));
- uint8x8_t shuffle_lo =
- vtbl2_u8(p, vreinterpret_u8_s64(vget_low_s64(pattern)));
- return v128_from_64(vget_lane_u64(vreinterpret_u64_u8(shuffle_hi), 0),
- vget_lane_u64(vreinterpret_u64_u8(shuffle_lo), 0));
-#endif
-}
-
-SIMD_INLINE v128 v128_cmpgt_s8(v128 x, v128 y) {
- return vreinterpretq_s64_u8(
- vcgtq_s8(vreinterpretq_s8_s64(x), vreinterpretq_s8_s64(y)));
-}
-
-SIMD_INLINE v128 v128_cmplt_s8(v128 x, v128 y) {
- return vreinterpretq_s64_u8(
- vcltq_s8(vreinterpretq_s8_s64(x), vreinterpretq_s8_s64(y)));
-}
-
-SIMD_INLINE v128 v128_cmpeq_8(v128 x, v128 y) {
- return vreinterpretq_s64_u8(
- vceqq_u8(vreinterpretq_u8_s64(x), vreinterpretq_u8_s64(y)));
-}
-
-SIMD_INLINE v128 v128_cmpgt_s16(v128 x, v128 y) {
- return vreinterpretq_s64_u16(
- vcgtq_s16(vreinterpretq_s16_s64(x), vreinterpretq_s16_s64(y)));
-}
-
-SIMD_INLINE v128 v128_cmplt_s16(v128 x, v128 y) {
- return vreinterpretq_s64_u16(
- vcltq_s16(vreinterpretq_s16_s64(x), vreinterpretq_s16_s64(y)));
-}
-
-SIMD_INLINE v128 v128_cmpeq_16(v128 x, v128 y) {
- return vreinterpretq_s64_u16(
- vceqq_s16(vreinterpretq_s16_s64(x), vreinterpretq_s16_s64(y)));
-}
-
-SIMD_INLINE v128 v128_cmpgt_s32(v128 x, v128 y) {
- return vreinterpretq_s64_u32(
- vcgtq_s32(vreinterpretq_s32_s64(x), vreinterpretq_s32_s64(y)));
-}
-
-SIMD_INLINE v128 v128_cmplt_s32(v128 x, v128 y) {
- return vreinterpretq_s64_u32(
- vcltq_s32(vreinterpretq_s32_s64(x), vreinterpretq_s32_s64(y)));
-}
-
-SIMD_INLINE v128 v128_cmpeq_32(v128 x, v128 y) {
- return vreinterpretq_s64_u32(
- vceqq_s32(vreinterpretq_s32_s64(x), vreinterpretq_s32_s64(y)));
-}
-
-SIMD_INLINE v128 v128_shl_8(v128 a, unsigned int c) {
- return (c > 7) ? v128_zero()
- : vreinterpretq_s64_u8(vshlq_u8(vreinterpretq_u8_s64(a),
- vdupq_n_s8((int8_t)c)));
-}
-
-SIMD_INLINE v128 v128_shr_u8(v128 a, unsigned int c) {
- return (c > 7) ? v128_zero()
- : vreinterpretq_s64_u8(vshlq_u8(vreinterpretq_u8_s64(a),
- vdupq_n_s8(-(int8_t)c)));
-}
-
-SIMD_INLINE v128 v128_shr_s8(v128 a, unsigned int c) {
- return (c > 7) ? v128_ones()
- : vreinterpretq_s64_s8(vshlq_s8(vreinterpretq_s8_s64(a),
- vdupq_n_s8(-(int8_t)c)));
-}
-
-SIMD_INLINE v128 v128_shl_16(v128 a, unsigned int c) {
- return (c > 15) ? v128_zero()
- : vreinterpretq_s64_u16(vshlq_u16(vreinterpretq_u16_s64(a),
- vdupq_n_s16((int16_t)c)));
-}
-
-SIMD_INLINE v128 v128_shr_u16(v128 a, unsigned int c) {
- return (c > 15) ? v128_zero()
- : vreinterpretq_s64_u16(vshlq_u16(vreinterpretq_u16_s64(a),
- vdupq_n_s16(-(int16_t)c)));
-}
-
-SIMD_INLINE v128 v128_shr_s16(v128 a, unsigned int c) {
- return (c > 15) ? v128_ones()
- : vreinterpretq_s64_s16(vshlq_s16(vreinterpretq_s16_s64(a),
- vdupq_n_s16(-(int16_t)c)));
-}
-
-SIMD_INLINE v128 v128_shl_32(v128 a, unsigned int c) {
- return (c > 31) ? v128_zero()
- : vreinterpretq_s64_u32(vshlq_u32(vreinterpretq_u32_s64(a),
- vdupq_n_s32((int32_t)c)));
-}
-
-SIMD_INLINE v128 v128_shr_u32(v128 a, unsigned int c) {
- return (c > 31) ? v128_zero()
- : vreinterpretq_s64_u32(vshlq_u32(vreinterpretq_u32_s64(a),
- vdupq_n_s32(-(int32_t)c)));
-}
-
-SIMD_INLINE v128 v128_shr_s32(v128 a, unsigned int c) {
- return (c > 31) ? v128_ones()
- : vreinterpretq_s64_s32(vshlq_s32(vreinterpretq_s32_s64(a),
- vdupq_n_s32(-(int32_t)c)));
-}
-
-SIMD_INLINE v128 v128_shl_64(v128 a, unsigned int c) {
- return (c > 63) ? v128_zero()
- : vreinterpretq_s64_u64(vshlq_u64(vreinterpretq_u64_s64(a),
- vdupq_n_s64((int64_t)c)));
-}
-
-SIMD_INLINE v128 v128_shr_u64(v128 a, unsigned int c) {
- return (c > 63) ? v128_zero()
- : vreinterpretq_s64_u64(vshlq_u64(vreinterpretq_u64_s64(a),
- vdupq_n_s64(-(int64_t)c)));
-}
-
-SIMD_INLINE v128 v128_shr_s64(v128 a, unsigned int c) {
- return (c > 63) ? v128_ones() : vshlq_s64(a, vdupq_n_s64(-(int64_t)c));
-}
-
-#if defined(__OPTIMIZE__) && __OPTIMIZE__ && !defined(__clang__)
-
-SIMD_INLINE v128 v128_shl_n_byte(v128 a, unsigned int n) {
- return n < 8
- ? v128_from_64(
- (uint64_t)vorr_u64(
- vshl_n_u64(vreinterpret_u64_s64(vget_high_s64(a)),
- n * 8),
- vshr_n_u64(vreinterpret_u64_s64(vget_low_s64(a)),
- (8 - n) * 8)),
- (uint64_t)vshl_n_u64(vreinterpret_u64_s64(vget_low_s64(a)),
- n * 8))
- : (n == 8 ? v128_from_64(
- (uint64_t)vreinterpret_u64_s64(vget_low_s64(a)), 0)
- : v128_from_64((uint64_t)vshl_n_u64(
- vreinterpret_u64_s64(vget_low_s64(a)),
- (n - 8) * 8),
- 0));
-}
-
-SIMD_INLINE v128 v128_shr_n_byte(v128 a, unsigned int n) {
- return n == 0
- ? a
- : (n < 8
- ? v128_from_64(
- (uint64_t)vshr_n_u64(
- vreinterpret_u64_s64(vget_high_s64(a)), n * 8),
- (uint64_t)vorr_u64(
- vshr_n_u64(vreinterpret_u64_s64(vget_low_s64(a)),
- n * 8),
- vshl_n_u64(vreinterpret_u64_s64(vget_high_s64(a)),
- (8 - n) * 8)))
- : (n == 8 ? v128_from_64(0, (uint64_t)vreinterpret_u64_s64(
- vget_high_s64(a)))
- : v128_from_64(0, (uint64_t)vshr_n_u64(
- vreinterpret_u64_s64(
- vget_high_s64(a)),
- (n - 8) * 8))));
-}
-
-SIMD_INLINE v128 v128_shl_n_8(v128 a, unsigned int c) {
- return c ? vreinterpretq_s64_u8(vshlq_n_u8(vreinterpretq_u8_s64(a), c)) : a;
-}
-
-SIMD_INLINE v128 v128_shr_n_u8(v128 a, unsigned int c) {
- return c ? vreinterpretq_s64_u8(vshrq_n_u8(vreinterpretq_u8_s64(a), c)) : a;
-}
-
-SIMD_INLINE v128 v128_shr_n_s8(v128 a, unsigned int c) {
- return c ? vreinterpretq_s64_s8(vshrq_n_s8(vreinterpretq_s8_s64(a), c)) : a;
-}
-
-SIMD_INLINE v128 v128_shl_n_16(v128 a, unsigned int c) {
- return c ? vreinterpretq_s64_u16(vshlq_n_u16(vreinterpretq_u16_s64(a), c))
- : a;
-}
-
-SIMD_INLINE v128 v128_shr_n_u16(v128 a, unsigned int c) {
- return c ? vreinterpretq_s64_u16(vshrq_n_u16(vreinterpretq_u16_s64(a), c))
- : a;
-}
-
-SIMD_INLINE v128 v128_shr_n_s16(v128 a, unsigned int c) {
- return c ? vreinterpretq_s64_s16(vshrq_n_s16(vreinterpretq_s16_s64(a), c))
- : a;
-}
-
-SIMD_INLINE v128 v128_shl_n_32(v128 a, unsigned int c) {
- return c ? vreinterpretq_s64_u32(vshlq_n_u32(vreinterpretq_u32_s64(a), c))
- : a;
-}
-
-SIMD_INLINE v128 v128_shr_n_u32(v128 a, unsigned int c) {
- return c ? vreinterpretq_s64_u32(vshrq_n_u32(vreinterpretq_u32_s64(a), c))
- : a;
-}
-
-SIMD_INLINE v128 v128_shr_n_s32(v128 a, unsigned int c) {
- return c ? vreinterpretq_s64_s32(vshrq_n_s32(vreinterpretq_s32_s64(a), c))
- : a;
-}
-
-SIMD_INLINE v128 v128_shl_n_64(v128 a, unsigned int c) {
- return c ? vreinterpretq_s64_u64(vshlq_n_u64(vreinterpretq_u64_s64(a), c))
- : a;
-}
-
-SIMD_INLINE v128 v128_shr_n_u64(v128 a, unsigned int c) {
- return c ? vreinterpretq_s64_u64(vshrq_n_u64(vreinterpretq_u64_s64(a), c))
- : a;
-}
-
-SIMD_INLINE v128 v128_shr_n_s64(v128 a, unsigned int c) {
- return c ? vshrq_n_s64(a, c) : a;
-}
-
-#else
-
-SIMD_INLINE v128 v128_shl_n_byte(v128 a, unsigned int n) {
- if (n < 8)
- return v128_from_v64(v64_or(v64_shl_n_byte(v128_high_v64(a), n),
- v64_shr_n_byte(v128_low_v64(a), 8 - n)),
- v64_shl_n_byte(v128_low_v64(a), n));
- else
- return v128_from_v64(v64_shl_n_byte(v128_low_v64(a), n - 8), v64_zero());
-}
-
-SIMD_INLINE v128 v128_shr_n_byte(v128 a, unsigned int n) {
- if (n < 8)
- return v128_from_v64(v64_shr_n_byte(v128_high_v64(a), n),
- v64_or(v64_shr_n_byte(v128_low_v64(a), n),
- v64_shl_n_byte(v128_high_v64(a), 8 - n)));
- else
- return v128_from_v64(v64_zero(), v64_shr_n_byte(v128_high_v64(a), n - 8));
-}
-
-SIMD_INLINE v128 v128_shl_n_8(v128 a, unsigned int c) {
- return v128_shl_8(a, c);
-}
-
-SIMD_INLINE v128 v128_shr_n_u8(v128 a, unsigned int c) {
- return v128_shr_u8(a, c);
-}
-
-SIMD_INLINE v128 v128_shr_n_s8(v128 a, unsigned int c) {
- return v128_shr_s8(a, c);
-}
-
-SIMD_INLINE v128 v128_shl_n_16(v128 a, unsigned int c) {
- return v128_shl_16(a, c);
-}
-
-SIMD_INLINE v128 v128_shr_n_u16(v128 a, unsigned int c) {
- return v128_shr_u16(a, c);
-}
-
-SIMD_INLINE v128 v128_shr_n_s16(v128 a, unsigned int c) {
- return v128_shr_s16(a, c);
-}
-
-SIMD_INLINE v128 v128_shl_n_32(v128 a, unsigned int c) {
- return v128_shl_32(a, c);
-}
-
-SIMD_INLINE v128 v128_shr_n_u32(v128 a, unsigned int c) {
- return v128_shr_u32(a, c);
-}
-
-SIMD_INLINE v128 v128_shr_n_s32(v128 a, unsigned int c) {
- return v128_shr_s32(a, c);
-}
-
-SIMD_INLINE v128 v128_shl_n_64(v128 a, unsigned int c) {
- return v128_shl_64(a, c);
-}
-
-SIMD_INLINE v128 v128_shr_n_u64(v128 a, unsigned int c) {
- return v128_shr_u64(a, c);
-}
-
-SIMD_INLINE v128 v128_shr_n_s64(v128 a, unsigned int c) {
- return v128_shr_s64(a, c);
-}
-
-#endif
-
-typedef uint32x4_t sad128_internal_u16;
-
-SIMD_INLINE sad128_internal_u16 v128_sad_u16_init(void) {
- return vdupq_n_u32(0);
-}
-
-/* Implementation dependent return value. Result must be finalised with
- * v128_sad_u16_sum(). */
-SIMD_INLINE sad128_internal_u16 v128_sad_u16(sad128_internal_u16 s, v128 a,
- v128 b) {
- return vaddq_u32(
- s, vpaddlq_u16(vsubq_u16(
- vmaxq_u16(vreinterpretq_u16_s64(a), vreinterpretq_u16_s64(b)),
- vminq_u16(vreinterpretq_u16_s64(a), vreinterpretq_u16_s64(b)))));
-}
-
-SIMD_INLINE uint32_t v128_sad_u16_sum(sad128_internal_u16 s) {
- uint64x2_t t = vpaddlq_u32(s);
- return (uint32_t)vget_lane_u64(vadd_u64(vget_high_u64(t), vget_low_u64(t)),
- 0);
-}
-
-typedef v128 ssd128_internal_s16;
-SIMD_INLINE ssd128_internal_s16 v128_ssd_s16_init(void) { return v128_zero(); }
-
-/* Implementation dependent return value. Result must be finalised with
- * v128_ssd_s16_sum(). */
-SIMD_INLINE ssd128_internal_s16 v128_ssd_s16(ssd128_internal_s16 s, v128 a,
- v128 b) {
- v128 d = v128_sub_16(a, b);
- d = v128_madd_s16(d, d);
- return v128_add_64(
- s, vreinterpretq_s64_u64(vpaddlq_u32(vreinterpretq_u32_s64(d))));
-}
-
-SIMD_INLINE uint64_t v128_ssd_s16_sum(ssd128_internal_s16 s) {
- return v64_u64(v128_low_v64(s)) + v64_u64(v128_high_v64(s));
-}
-
-#endif // AOM_AOM_DSP_SIMD_V128_INTRINSICS_ARM_H_
diff --git a/aom_dsp/simd/v256_intrinsics_arm.h b/aom_dsp/simd/v256_intrinsics_arm.h
deleted file mode 100644
index bd86ea1..0000000
--- a/aom_dsp/simd/v256_intrinsics_arm.h
+++ /dev/null
@@ -1,17 +0,0 @@
-/*
- * Copyright (c) 2016, Alliance for Open Media. All rights reserved
- *
- * This source code is subject to the terms of the BSD 2 Clause License and
- * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
- * was not distributed with this source code in the LICENSE file, you can
- * obtain it at www.aomedia.org/license/software. If the Alliance for Open
- * Media Patent License 1.0 was not distributed with this source code in the
- * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
- */
-
-#ifndef AOM_AOM_DSP_SIMD_V256_INTRINSICS_ARM_H_
-#define AOM_AOM_DSP_SIMD_V256_INTRINSICS_ARM_H_
-
-#include "aom_dsp/simd/v256_intrinsics_v128.h"
-
-#endif // AOM_AOM_DSP_SIMD_V256_INTRINSICS_ARM_H_
diff --git a/aom_dsp/simd/v64_intrinsics_arm.h b/aom_dsp/simd/v64_intrinsics_arm.h
deleted file mode 100644
index 8d07c34..0000000
--- a/aom_dsp/simd/v64_intrinsics_arm.h
+++ /dev/null
@@ -1,694 +0,0 @@
-/*
- * Copyright (c) 2016, Alliance for Open Media. All rights reserved
- *
- * This source code is subject to the terms of the BSD 2 Clause License and
- * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
- * was not distributed with this source code in the LICENSE file, you can
- * obtain it at www.aomedia.org/license/software. If the Alliance for Open
- * Media Patent License 1.0 was not distributed with this source code in the
- * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
- */
-
-#ifndef AOM_AOM_DSP_SIMD_V64_INTRINSICS_ARM_H_
-#define AOM_AOM_DSP_SIMD_V64_INTRINSICS_ARM_H_
-
-#include <arm_neon.h>
-#include <string.h>
-
-#include "config/aom_config.h"
-
-#include "aom_dsp/simd/v64_intrinsics_arm.h"
-#include "aom_ports/arm.h"
-
-#ifdef AOM_INCOMPATIBLE_GCC
-#error Incompatible gcc
-#endif
-
-typedef int64x1_t v64;
-
-SIMD_INLINE uint32_t v64_low_u32(v64 a) {
- return vget_lane_u32(vreinterpret_u32_s64(a), 0);
-}
-
-SIMD_INLINE uint32_t v64_high_u32(v64 a) {
- return vget_lane_u32(vreinterpret_u32_s64(a), 1);
-}
-
-SIMD_INLINE int32_t v64_low_s32(v64 a) {
- return vget_lane_s32(vreinterpret_s32_s64(a), 0);
-}
-
-SIMD_INLINE int32_t v64_high_s32(v64 a) {
- return vget_lane_s32(vreinterpret_s32_s64(a), 1);
-}
-
-SIMD_INLINE v64 v64_from_16(uint16_t a, uint16_t b, uint16_t c, uint16_t d) {
- return vcreate_s64((uint64_t)a << 48 | (uint64_t)b << 32 | (uint64_t)c << 16 |
- d);
-}
-
-SIMD_INLINE v64 v64_from_32(uint32_t x, uint32_t y) {
- return vcreate_s64((uint64_t)x << 32 | y);
-}
-
-SIMD_INLINE v64 v64_from_64(uint64_t x) { return vcreate_s64(x); }
-
-SIMD_INLINE uint64_t v64_u64(v64 x) { return (uint64_t)vget_lane_s64(x, 0); }
-
-SIMD_INLINE uint32_t u32_load_aligned(const void *p) {
- return *((uint32_t *)p);
-}
-
-SIMD_INLINE uint32_t u32_load_unaligned(const void *p) {
- return vget_lane_u32(vreinterpret_u32_u8(vld1_u8((const uint8_t *)p)), 0);
-}
-
-SIMD_INLINE void u32_store_aligned(void *p, uint32_t a) {
- *((uint32_t *)p) = a;
-}
-
-SIMD_INLINE void u32_store_unaligned(void *p, uint32_t a) {
-#if defined(__clang__)
- vst1_lane_u32((uint32_t *)p, vreinterpret_u32_s64((uint64x1_t)(uint64_t)a),
- 0);
-#elif defined(__CC_ARM)
- *(__packed uint32_t *)p) = a;
-#elif defined(__GNUC__)
- struct Unaligned32Struct {
- uint32_t value;
- uint8_t dummy; // To make the size non-power-of-two.
- } __attribute__((__packed__));
- ((struct Unaligned32Struct *)p)->value = a;
-#else
- memcpy(p, &a, 4);
-#endif
-}
-
-SIMD_INLINE v64 v64_load_aligned(const void *p) {
- return vreinterpret_s64_u8(vld1_u8((const uint8_t *)p));
-}
-
-SIMD_INLINE v64 v64_load_unaligned(const void *p) {
- return v64_load_aligned(p);
-}
-
-SIMD_INLINE void v64_store_aligned(void *p, v64 r) {
- vst1_u8((uint8_t *)p, vreinterpret_u8_s64(r));
-}
-
-SIMD_INLINE void v64_store_unaligned(void *p, v64 r) {
- vst1_u8((uint8_t *)p, vreinterpret_u8_s64(r));
-}
-
-// The following function requires an immediate.
-// Some compilers will check this if it's optimising, others wont.
-SIMD_INLINE v64 v64_align(v64 a, v64 b, unsigned int c) {
-#if defined(__OPTIMIZE__) && __OPTIMIZE__ && !defined(__clang__)
- return c ? vreinterpret_s64_s8(
- vext_s8(vreinterpret_s8_s64(b), vreinterpret_s8_s64(a), c))
- : b;
-#else
- return c ? v64_from_64(((uint64_t)vget_lane_s64(b, 0) >> c * 8) |
- ((uint64_t)vget_lane_s64(a, 0) << (8 - c) * 8))
- : b;
-#endif
-}
-
-SIMD_INLINE v64 v64_zero(void) { return vreinterpret_s64_u8(vdup_n_u8(0)); }
-
-SIMD_INLINE v64 v64_dup_8(uint8_t x) {
- return vreinterpret_s64_u8(vdup_n_u8(x));
-}
-
-SIMD_INLINE v64 v64_dup_16(uint16_t x) {
- return vreinterpret_s64_u16(vdup_n_u16(x));
-}
-
-SIMD_INLINE v64 v64_dup_32(uint32_t x) {
- return vreinterpret_s64_u32(vdup_n_u32(x));
-}
-
-SIMD_INLINE int64_t v64_dotp_su8(v64 x, v64 y) {
- int16x8_t t =
- vmulq_s16(vmovl_s8(vreinterpret_s8_s64(x)),
- vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_s64(y))));
-#if AOM_ARCH_AARCH64
- return vaddlvq_s16(t);
-#else
- int64x2_t r = vpaddlq_s32(vpaddlq_s16(t));
- return vget_lane_s64(vadd_s64(vget_high_s64(r), vget_low_s64(r)), 0);
-#endif
-}
-
-SIMD_INLINE int64_t v64_dotp_s16(v64 x, v64 y) {
-#if AOM_ARCH_AARCH64
- return vaddlvq_s32(
- vmull_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y)));
-#else
- int64x2_t r =
- vpaddlq_s32(vmull_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y)));
- return vget_lane_s64(vadd_s64(vget_high_s64(r), vget_low_s64(r)), 0);
-#endif
-}
-
-SIMD_INLINE uint64_t v64_hadd_u8(v64 x) {
-#if AOM_ARCH_AARCH64
- return vaddlv_u8(vreinterpret_u8_s64(x));
-#else
- return vget_lane_u64(
- vpaddl_u32(vpaddl_u16(vpaddl_u8(vreinterpret_u8_s64(x)))), 0);
-#endif
-}
-
-SIMD_INLINE int64_t v64_hadd_s16(v64 a) {
- return vget_lane_s64(vpaddl_s32(vpaddl_s16(vreinterpret_s16_s64(a))), 0);
-}
-
-typedef uint16x8_t sad64_internal;
-
-SIMD_INLINE sad64_internal v64_sad_u8_init(void) { return vdupq_n_u16(0); }
-
-// Implementation dependent return value. Result must be finalised with
-// v64_sad_u8_sum().
-SIMD_INLINE sad64_internal v64_sad_u8(sad64_internal s, v64 a, v64 b) {
- return vabal_u8(s, vreinterpret_u8_s64(a), vreinterpret_u8_s64(b));
-}
-
-SIMD_INLINE uint32_t v64_sad_u8_sum(sad64_internal s) {
-#if AOM_ARCH_AARCH64
- return vaddlvq_u16(s);
-#else
- uint64x2_t r = vpaddlq_u32(vpaddlq_u16(s));
- return (uint32_t)vget_lane_u64(vadd_u64(vget_high_u64(r), vget_low_u64(r)),
- 0);
-#endif
-}
-
-typedef uint32x4_t ssd64_internal;
-
-SIMD_INLINE ssd64_internal v64_ssd_u8_init(void) { return vdupq_n_u32(0); }
-
-// Implementation dependent return value. Result must be finalised with
-// v64_ssd_u8_sum().
-SIMD_INLINE ssd64_internal v64_ssd_u8(ssd64_internal s, v64 a, v64 b) {
- uint8x8_t t = vabd_u8(vreinterpret_u8_s64(a), vreinterpret_u8_s64(b));
- return vaddq_u32(s, vpaddlq_u16(vmull_u8(t, t)));
-}
-
-SIMD_INLINE uint32_t v64_ssd_u8_sum(ssd64_internal s) {
-#if AOM_ARCH_AARCH64
- return vaddvq_u32(s);
-#else
- uint64x2_t t = vpaddlq_u32(s);
- return vget_lane_u32(
- vreinterpret_u32_u64(vadd_u64(vget_high_u64(t), vget_low_u64(t))), 0);
-#endif
-}
-
-SIMD_INLINE v64 v64_or(v64 x, v64 y) { return vorr_s64(x, y); }
-
-SIMD_INLINE v64 v64_xor(v64 x, v64 y) { return veor_s64(x, y); }
-
-SIMD_INLINE v64 v64_and(v64 x, v64 y) { return vand_s64(x, y); }
-
-SIMD_INLINE v64 v64_andn(v64 x, v64 y) { return vbic_s64(x, y); }
-
-SIMD_INLINE v64 v64_add_8(v64 x, v64 y) {
- return vreinterpret_s64_u8(
- vadd_u8(vreinterpret_u8_s64(x), vreinterpret_u8_s64(y)));
-}
-
-SIMD_INLINE v64 v64_sadd_u8(v64 x, v64 y) {
- return vreinterpret_s64_u8(
- vqadd_u8(vreinterpret_u8_s64(x), vreinterpret_u8_s64(y)));
-}
-
-SIMD_INLINE v64 v64_sadd_s8(v64 x, v64 y) {
- return vreinterpret_s64_s8(
- vqadd_s8(vreinterpret_s8_s64(x), vreinterpret_s8_s64(y)));
-}
-
-SIMD_INLINE v64 v64_add_16(v64 x, v64 y) {
- return vreinterpret_s64_s16(
- vadd_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y)));
-}
-
-SIMD_INLINE v64 v64_sadd_s16(v64 x, v64 y) {
- return vreinterpret_s64_s16(
- vqadd_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y)));
-}
-
-SIMD_INLINE v64 v64_add_32(v64 x, v64 y) {
- return vreinterpret_s64_u32(
- vadd_u32(vreinterpret_u32_s64(x), vreinterpret_u32_s64(y)));
-}
-
-SIMD_INLINE v64 v64_sub_8(v64 x, v64 y) {
- return vreinterpret_s64_u8(
- vsub_u8(vreinterpret_u8_s64(x), vreinterpret_u8_s64(y)));
-}
-
-SIMD_INLINE v64 v64_sub_16(v64 x, v64 y) {
- return vreinterpret_s64_s16(
- vsub_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y)));
-}
-
-SIMD_INLINE v64 v64_ssub_s16(v64 x, v64 y) {
- return vreinterpret_s64_s16(
- vqsub_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y)));
-}
-
-SIMD_INLINE v64 v64_ssub_u16(v64 x, v64 y) {
- return vreinterpret_s64_u16(
- vqsub_u16(vreinterpret_u16_s64(x), vreinterpret_u16_s64(y)));
-}
-
-SIMD_INLINE v64 v64_ssub_u8(v64 x, v64 y) {
- return vreinterpret_s64_u8(
- vqsub_u8(vreinterpret_u8_s64(x), vreinterpret_u8_s64(y)));
-}
-
-SIMD_INLINE v64 v64_ssub_s8(v64 x, v64 y) {
- return vreinterpret_s64_s8(
- vqsub_s8(vreinterpret_s8_s64(x), vreinterpret_s8_s64(y)));
-}
-
-SIMD_INLINE v64 v64_sub_32(v64 x, v64 y) {
- return vreinterpret_s64_s32(
- vsub_s32(vreinterpret_s32_s64(x), vreinterpret_s32_s64(y)));
-}
-
-SIMD_INLINE v64 v64_abs_s16(v64 x) {
- return vreinterpret_s64_s16(vabs_s16(vreinterpret_s16_s64(x)));
-}
-
-SIMD_INLINE v64 v64_abs_s8(v64 x) {
- return vreinterpret_s64_s8(vabs_s8(vreinterpret_s8_s64(x)));
-}
-
-SIMD_INLINE v64 v64_mullo_s16(v64 x, v64 y) {
- return vreinterpret_s64_s16(
- vmul_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y)));
-}
-
-SIMD_INLINE v64 v64_mulhi_s16(v64 x, v64 y) {
-#if AOM_ARCH_AARCH64
- int16x8_t t = vreinterpretq_s16_s32(
- vmull_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y)));
- return vget_low_s64(vreinterpretq_s64_s16(vuzp2q_s16(t, t)));
-#else
- return vreinterpret_s64_s16(vmovn_s32(vshrq_n_s32(
- vmull_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y)), 16)));
-#endif
-}
-
-SIMD_INLINE v64 v64_mullo_s32(v64 x, v64 y) {
- return vreinterpret_s64_s32(
- vmul_s32(vreinterpret_s32_s64(x), vreinterpret_s32_s64(y)));
-}
-
-SIMD_INLINE v64 v64_madd_s16(v64 x, v64 y) {
- int32x4_t t = vmull_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y));
- return vreinterpret_s64_s32(
- vpadd_s32(vreinterpret_s32_s64(vget_low_s64(vreinterpretq_s64_s32(t))),
- vreinterpret_s32_s64(vget_high_s64(vreinterpretq_s64_s32(t)))));
-}
-
-SIMD_INLINE v64 v64_madd_us8(v64 x, v64 y) {
- int16x8_t t =
- vmulq_s16(vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_s64(x))),
- vmovl_s8(vreinterpret_s8_s64(y)));
- return vreinterpret_s64_s16(vqmovn_s32(vpaddlq_s16(t)));
-}
-
-SIMD_INLINE v64 v64_avg_u8(v64 x, v64 y) {
- return vreinterpret_s64_u8(
- vrhadd_u8(vreinterpret_u8_s64(x), vreinterpret_u8_s64(y)));
-}
-
-SIMD_INLINE v64 v64_rdavg_u8(v64 x, v64 y) {
- return vreinterpret_s64_u8(
- vhadd_u8(vreinterpret_u8_s64(x), vreinterpret_u8_s64(y)));
-}
-
-SIMD_INLINE v64 v64_rdavg_u16(v64 x, v64 y) {
- return vreinterpret_s64_u16(
- vhadd_u16(vreinterpret_u16_s64(x), vreinterpret_u16_s64(y)));
-}
-
-SIMD_INLINE v64 v64_avg_u16(v64 x, v64 y) {
- return vreinterpret_s64_u16(
- vrhadd_u16(vreinterpret_u16_s64(x), vreinterpret_u16_s64(y)));
-}
-
-SIMD_INLINE v64 v64_max_u8(v64 x, v64 y) {
- return vreinterpret_s64_u8(
- vmax_u8(vreinterpret_u8_s64(x), vreinterpret_u8_s64(y)));
-}
-
-SIMD_INLINE v64 v64_min_u8(v64 x, v64 y) {
- return vreinterpret_s64_u8(
- vmin_u8(vreinterpret_u8_s64(x), vreinterpret_u8_s64(y)));
-}
-
-SIMD_INLINE v64 v64_max_s8(v64 x, v64 y) {
- return vreinterpret_s64_s8(
- vmax_s8(vreinterpret_s8_s64(x), vreinterpret_s8_s64(y)));
-}
-
-SIMD_INLINE v64 v64_min_s8(v64 x, v64 y) {
- return vreinterpret_s64_s8(
- vmin_s8(vreinterpret_s8_s64(x), vreinterpret_s8_s64(y)));
-}
-
-SIMD_INLINE v64 v64_max_s16(v64 x, v64 y) {
- return vreinterpret_s64_s16(
- vmax_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y)));
-}
-
-SIMD_INLINE v64 v64_min_s16(v64 x, v64 y) {
- return vreinterpret_s64_s16(
- vmin_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y)));
-}
-
-SIMD_INLINE v64 v64_ziplo_8(v64 x, v64 y) {
-#if AOM_ARCH_AARCH64
- return vreinterpret_s64_u8(
- vzip1_u8(vreinterpret_u8_s64(y), vreinterpret_u8_s64(x)));
-#else
- uint8x8x2_t r = vzip_u8(vreinterpret_u8_s64(y), vreinterpret_u8_s64(x));
- return vreinterpret_s64_u8(r.val[0]);
-#endif
-}
-
-SIMD_INLINE v64 v64_ziphi_8(v64 x, v64 y) {
-#if AOM_ARCH_AARCH64
- return vreinterpret_s64_u8(
- vzip2_u8(vreinterpret_u8_s64(y), vreinterpret_u8_s64(x)));
-#else
- uint8x8x2_t r = vzip_u8(vreinterpret_u8_s64(y), vreinterpret_u8_s64(x));
- return vreinterpret_s64_u8(r.val[1]);
-#endif
-}
-
-SIMD_INLINE v64 v64_ziplo_16(v64 x, v64 y) {
-#if AOM_ARCH_AARCH64
- return vreinterpret_s64_u16(
- vzip1_u16(vreinterpret_u16_s64(y), vreinterpret_u16_s64(x)));
-#else
- int16x4x2_t r = vzip_s16(vreinterpret_s16_s64(y), vreinterpret_s16_s64(x));
- return vreinterpret_s64_s16(r.val[0]);
-#endif
-}
-
-SIMD_INLINE v64 v64_ziphi_16(v64 x, v64 y) {
-#if AOM_ARCH_AARCH64
- return vreinterpret_s64_u16(
- vzip2_u16(vreinterpret_u16_s64(y), vreinterpret_u16_s64(x)));
-#else
- int16x4x2_t r = vzip_s16(vreinterpret_s16_s64(y), vreinterpret_s16_s64(x));
- return vreinterpret_s64_s16(r.val[1]);
-#endif
-}
-
-SIMD_INLINE v64 v64_ziplo_32(v64 x, v64 y) {
-#if AOM_ARCH_AARCH64
- return vreinterpret_s64_u32(
- vzip1_u32(vreinterpret_u32_s64(y), vreinterpret_u32_s64(x)));
-#else
- int32x2x2_t r = vzip_s32(vreinterpret_s32_s64(y), vreinterpret_s32_s64(x));
- return vreinterpret_s64_s32(r.val[0]);
-#endif
-}
-
-SIMD_INLINE v64 v64_ziphi_32(v64 x, v64 y) {
-#if AOM_ARCH_AARCH64
- return vreinterpret_s64_u32(
- vzip2_u32(vreinterpret_u32_s64(y), vreinterpret_u32_s64(x)));
-#else
- int32x2x2_t r = vzip_s32(vreinterpret_s32_s64(y), vreinterpret_s32_s64(x));
- return vreinterpret_s64_s32(r.val[1]);
-#endif
-}
-
-SIMD_INLINE v64 v64_unpacklo_u8_s16(v64 a) {
- return vreinterpret_s64_u16(vget_low_u16(vmovl_u8(vreinterpret_u8_s64(a))));
-}
-
-SIMD_INLINE v64 v64_unpackhi_u8_s16(v64 a) {
- return vreinterpret_s64_u16(vget_high_u16(vmovl_u8(vreinterpret_u8_s64(a))));
-}
-
-SIMD_INLINE v64 v64_unpacklo_s8_s16(v64 a) {
- return vreinterpret_s64_s16(vget_low_s16(vmovl_s8(vreinterpret_s8_s64(a))));
-}
-
-SIMD_INLINE v64 v64_unpackhi_s8_s16(v64 a) {
- return vreinterpret_s64_s16(vget_high_s16(vmovl_s8(vreinterpret_s8_s64(a))));
-}
-
-SIMD_INLINE v64 v64_pack_s32_s16(v64 x, v64 y) {
- return vreinterpret_s64_s16(vqmovn_s32(
- vcombine_s32(vreinterpret_s32_s64(y), vreinterpret_s32_s64(x))));
-}
-
-SIMD_INLINE v64 v64_pack_s32_u16(v64 x, v64 y) {
- return vreinterpret_s64_u16(vqmovun_s32(
- vcombine_s32(vreinterpret_s32_s64(y), vreinterpret_s32_s64(x))));
-}
-
-SIMD_INLINE v64 v64_pack_s16_u8(v64 x, v64 y) {
- return vreinterpret_s64_u8(vqmovun_s16(vreinterpretq_s16_s32(
- vcombine_s32(vreinterpret_s32_s64(y), vreinterpret_s32_s64(x)))));
-}
-
-SIMD_INLINE v64 v64_pack_s16_s8(v64 x, v64 y) {
- return vreinterpret_s64_s8(vqmovn_s16(vreinterpretq_s16_s32(
- vcombine_s32(vreinterpret_s32_s64(y), vreinterpret_s32_s64(x)))));
-}
-
-SIMD_INLINE v64 v64_unziplo_8(v64 x, v64 y) {
-#if AOM_ARCH_AARCH64
- return vreinterpret_s64_u8(
- vuzp1_u8(vreinterpret_u8_s64(y), vreinterpret_u8_s64(x)));
-#else
- uint8x8x2_t r = vuzp_u8(vreinterpret_u8_s64(y), vreinterpret_u8_s64(x));
- return vreinterpret_s64_u8(r.val[0]);
-#endif
-}
-
-SIMD_INLINE v64 v64_unziphi_8(v64 x, v64 y) {
-#if AOM_ARCH_AARCH64
- return vreinterpret_s64_u8(
- vuzp2_u8(vreinterpret_u8_s64(y), vreinterpret_u8_s64(x)));
-#else
- uint8x8x2_t r = vuzp_u8(vreinterpret_u8_s64(y), vreinterpret_u8_s64(x));
- return vreinterpret_s64_u8(r.val[1]);
-#endif
-}
-
-SIMD_INLINE v64 v64_unziplo_16(v64 x, v64 y) {
-#if AOM_ARCH_AARCH64
- return vreinterpret_s64_u16(
- vuzp1_u16(vreinterpret_u16_s64(y), vreinterpret_u16_s64(x)));
-#else
- uint16x4x2_t r = vuzp_u16(vreinterpret_u16_s64(y), vreinterpret_u16_s64(x));
- return vreinterpret_s64_u16(r.val[0]);
-#endif
-}
-
-SIMD_INLINE v64 v64_unziphi_16(v64 x, v64 y) {
-#if AOM_ARCH_AARCH64
- return vreinterpret_s64_u16(
- vuzp2_u16(vreinterpret_u16_s64(y), vreinterpret_u16_s64(x)));
-#else
- uint16x4x2_t r = vuzp_u16(vreinterpret_u16_s64(y), vreinterpret_u16_s64(x));
- return vreinterpret_s64_u16(r.val[1]);
-#endif
-}
-
-SIMD_INLINE v64 v64_unpacklo_s16_s32(v64 x) {
- return vreinterpret_s64_s32(vget_low_s32(vmovl_s16(vreinterpret_s16_s64(x))));
-}
-
-SIMD_INLINE v64 v64_unpacklo_u16_s32(v64 x) {
- return vreinterpret_s64_u32(vget_low_u32(vmovl_u16(vreinterpret_u16_s64(x))));
-}
-
-SIMD_INLINE v64 v64_unpackhi_s16_s32(v64 x) {
- return vreinterpret_s64_s32(
- vget_high_s32(vmovl_s16(vreinterpret_s16_s64(x))));
-}
-
-SIMD_INLINE v64 v64_unpackhi_u16_s32(v64 x) {
- return vreinterpret_s64_u32(
- vget_high_u32(vmovl_u16(vreinterpret_u16_s64(x))));
-}
-
-SIMD_INLINE v64 v64_shuffle_8(v64 x, v64 pattern) {
- return vreinterpret_s64_u8(
- vtbl1_u8(vreinterpret_u8_s64(x), vreinterpret_u8_s64(pattern)));
-}
-
-SIMD_INLINE v64 v64_cmpgt_s8(v64 x, v64 y) {
- return vreinterpret_s64_u8(
- vcgt_s8(vreinterpret_s8_s64(x), vreinterpret_s8_s64(y)));
-}
-
-SIMD_INLINE v64 v64_cmplt_s8(v64 x, v64 y) {
- return vreinterpret_s64_u8(
- vclt_s8(vreinterpret_s8_s64(x), vreinterpret_s8_s64(y)));
-}
-
-SIMD_INLINE v64 v64_cmpeq_8(v64 x, v64 y) {
- return vreinterpret_s64_u8(
- vceq_u8(vreinterpret_u8_s64(x), vreinterpret_u8_s64(y)));
-}
-
-SIMD_INLINE v64 v64_cmpgt_s16(v64 x, v64 y) {
- return vreinterpret_s64_u16(
- vcgt_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y)));
-}
-
-SIMD_INLINE v64 v64_cmplt_s16(v64 x, v64 y) {
- return vreinterpret_s64_u16(
- vclt_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y)));
-}
-
-SIMD_INLINE v64 v64_cmpeq_16(v64 x, v64 y) {
- return vreinterpret_s64_u16(
- vceq_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y)));
-}
-
-SIMD_INLINE v64 v64_shl_8(v64 a, unsigned int c) {
- return vreinterpret_s64_u8(
- vshl_u8(vreinterpret_u8_s64(a), vdup_n_s8((int8_t)c)));
-}
-
-SIMD_INLINE v64 v64_shr_u8(v64 a, unsigned int c) {
- return vreinterpret_s64_u8(
- vshl_u8(vreinterpret_u8_s64(a), vdup_n_s8(-(int8_t)c)));
-}
-
-SIMD_INLINE v64 v64_shr_s8(v64 a, unsigned int c) {
- return vreinterpret_s64_s8(
- vshl_s8(vreinterpret_s8_s64(a), vdup_n_s8(-(int8_t)c)));
-}
-
-SIMD_INLINE v64 v64_shl_16(v64 a, unsigned int c) {
- return vreinterpret_s64_u16(
- vshl_u16(vreinterpret_u16_s64(a), vdup_n_s16((int16_t)c)));
-}
-
-SIMD_INLINE v64 v64_shr_u16(v64 a, unsigned int c) {
- return vreinterpret_s64_u16(
- vshl_u16(vreinterpret_u16_s64(a), vdup_n_s16(-(int16_t)c)));
-}
-
-SIMD_INLINE v64 v64_shr_s16(v64 a, unsigned int c) {
- return vreinterpret_s64_s16(
- vshl_s16(vreinterpret_s16_s64(a), vdup_n_s16(-(int16_t)c)));
-}
-
-SIMD_INLINE v64 v64_shl_32(v64 a, unsigned int c) {
- return vreinterpret_s64_u32(
- vshl_u32(vreinterpret_u32_s64(a), vdup_n_s32((int32_t)c)));
-}
-
-SIMD_INLINE v64 v64_shr_u32(v64 a, unsigned int c) {
- return vreinterpret_s64_u32(
- vshl_u32(vreinterpret_u32_s64(a), vdup_n_s32(-(int32_t)c)));
-}
-
-SIMD_INLINE v64 v64_shr_s32(v64 a, unsigned int c) {
- return vreinterpret_s64_s32(
- vshl_s32(vreinterpret_s32_s64(a), vdup_n_s32(-(int32_t)c)));
-}
-
-// The following functions require an immediate.
-// Some compilers will check this during optimisation, others wont.
-#if defined(__OPTIMIZE__) && __OPTIMIZE__ && !defined(__clang__)
-
-SIMD_INLINE v64 v64_shl_n_byte(v64 a, unsigned int c) {
- return vshl_n_s64(a, c * 8);
-}
-
-SIMD_INLINE v64 v64_shr_n_byte(v64 a, unsigned int c) {
- return c ? (v64)vshr_n_u64(vreinterpret_u64_s64(a), c * 8) : a;
-}
-
-SIMD_INLINE v64 v64_shl_n_8(v64 a, unsigned int c) {
- return c ? vreinterpret_s64_u8(vshl_n_u8(vreinterpret_u8_s64(a), c)) : a;
-}
-
-SIMD_INLINE v64 v64_shr_n_u8(v64 a, unsigned int c) {
- return c ? vreinterpret_s64_u8(vshr_n_u8(vreinterpret_u8_s64(a), c)) : a;
-}
-
-SIMD_INLINE v64 v64_shr_n_s8(v64 a, unsigned int c) {
- return c ? vreinterpret_s64_s8(vshr_n_s8(vreinterpret_s8_s64(a), c)) : a;
-}
-
-SIMD_INLINE v64 v64_shl_n_16(v64 a, unsigned int c) {
- return c ? vreinterpret_s64_u16(vshl_n_u16(vreinterpret_u16_s64(a), c)) : a;
-}
-
-SIMD_INLINE v64 v64_shr_n_u16(v64 a, unsigned int c) {
- return c ? vreinterpret_s64_u16(vshr_n_u16(vreinterpret_u16_s64(a), c)) : a;
-}
-
-SIMD_INLINE v64 v64_shr_n_s16(v64 a, unsigned int c) {
- return c ? vreinterpret_s64_s16(vshr_n_s16(vreinterpret_s16_s64(a), c)) : a;
-}
-
-SIMD_INLINE v64 v64_shl_n_32(v64 a, unsigned int c) {
- return c ? vreinterpret_s64_u32(vshl_n_u32(vreinterpret_u32_s64(a), c)) : a;
-}
-
-SIMD_INLINE v64 v64_shr_n_u32(v64 a, unsigned int c) {
- return c ? vreinterpret_s64_u32(vshr_n_u32(vreinterpret_u32_s64(a), c)) : a;
-}
-
-SIMD_INLINE v64 v64_shr_n_s32(v64 a, unsigned int c) {
- return c ? vreinterpret_s64_s32(vshr_n_s32(vreinterpret_s32_s64(a), c)) : a;
-}
-
-#else
-
-SIMD_INLINE v64 v64_shl_n_byte(v64 a, unsigned int c) {
- return v64_from_64(v64_u64(a) << c * 8);
-}
-
-SIMD_INLINE v64 v64_shr_n_byte(v64 a, unsigned int c) {
- return v64_from_64(v64_u64(a) >> c * 8);
-}
-
-SIMD_INLINE v64 v64_shl_n_8(v64 a, unsigned int c) { return v64_shl_8(a, c); }
-
-SIMD_INLINE v64 v64_shr_n_u8(v64 a, unsigned int c) { return v64_shr_u8(a, c); }
-
-SIMD_INLINE v64 v64_shr_n_s8(v64 a, unsigned int c) { return v64_shr_s8(a, c); }
-
-SIMD_INLINE v64 v64_shl_n_16(v64 a, unsigned int c) { return v64_shl_16(a, c); }
-
-SIMD_INLINE v64 v64_shr_n_u16(v64 a, unsigned int c) {
- return v64_shr_u16(a, c);
-}
-
-SIMD_INLINE v64 v64_shr_n_s16(v64 a, unsigned int c) {
- return v64_shr_s16(a, c);
-}
-
-SIMD_INLINE v64 v64_shl_n_32(v64 a, unsigned int c) { return v64_shl_32(a, c); }
-
-SIMD_INLINE v64 v64_shr_n_u32(v64 a, unsigned int c) {
- return v64_shr_u32(a, c);
-}
-
-SIMD_INLINE v64 v64_shr_n_s32(v64 a, unsigned int c) {
- return v64_shr_s32(a, c);
-}
-
-#endif
-
-#endif // AOM_AOM_DSP_SIMD_V64_INTRINSICS_ARM_H_
diff --git a/aom_dsp/sse.c b/aom_dsp/sse.c
index 16f6b58..bfe76ed 100644
--- a/aom_dsp/sse.c
+++ b/aom_dsp/sse.c
@@ -9,7 +9,12 @@
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
-/* Sum the difference between every corresponding element of the buffers. */
+/*
+ * Sum the square of the difference between every corresponding element of the
+ * buffers.
+ */
+
+#include <stdlib.h>
#include "config/aom_config.h"
#include "config/aom_dsp_rtcd.h"
diff --git a/aom_dsp/variance.c b/aom_dsp/variance.c
index 63c1e5f..f02c307 100644
--- a/aom_dsp/variance.c
+++ b/aom_dsp/variance.c
@@ -1058,7 +1058,7 @@
}
#define HIGHBD_OBMC_VAR(W, H) \
- unsigned int aom_highbd_obmc_variance##W##x##H##_c( \
+ unsigned int aom_highbd_8_obmc_variance##W##x##H##_c( \
const uint8_t *pre, int pre_stride, const int32_t *wsrc, \
const int32_t *mask, unsigned int *sse) { \
int sum; \
@@ -1087,7 +1087,7 @@
}
#define HIGHBD_OBMC_SUBPIX_VAR(W, H) \
- unsigned int aom_highbd_obmc_sub_pixel_variance##W##x##H##_c( \
+ unsigned int aom_highbd_8_obmc_sub_pixel_variance##W##x##H##_c( \
const uint8_t *pre, int pre_stride, int xoffset, int yoffset, \
const int32_t *wsrc, const int32_t *mask, unsigned int *sse) { \
uint16_t fdata3[(H + 1) * W]; \
@@ -1098,8 +1098,8 @@
aom_highbd_var_filter_block2d_bil_second_pass( \
fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \
\
- return aom_highbd_obmc_variance##W##x##H##_c(CONVERT_TO_BYTEPTR(temp2), W, \
- wsrc, mask, sse); \
+ return aom_highbd_8_obmc_variance##W##x##H##_c(CONVERT_TO_BYTEPTR(temp2), \
+ W, wsrc, mask, sse); \
} \
\
unsigned int aom_highbd_10_obmc_sub_pixel_variance##W##x##H##_c( \
diff --git a/aom_dsp/x86/aom_subpixel_8t_intrin_ssse3.c b/aom_dsp/x86/aom_subpixel_8t_intrin_ssse3.c
index 5823059..245fda1 100644
--- a/aom_dsp/x86/aom_subpixel_8t_intrin_ssse3.c
+++ b/aom_dsp/x86/aom_subpixel_8t_intrin_ssse3.c
@@ -23,32 +23,6 @@
#include "aom_ports/mem.h"
#include "aom_ports/emmintrin_compat.h"
-// filters only for the 4_h8 convolution
-DECLARE_ALIGNED(16, static const uint8_t, filt1_4_h8[16]) = { 0, 1, 1, 2, 2, 3,
- 3, 4, 2, 3, 3, 4,
- 4, 5, 5, 6 };
-
-DECLARE_ALIGNED(16, static const uint8_t, filt2_4_h8[16]) = { 4, 5, 5, 6, 6, 7,
- 7, 8, 6, 7, 7, 8,
- 8, 9, 9, 10 };
-
-// filters for 8_h8 and 16_h8
-DECLARE_ALIGNED(16, static const uint8_t,
- filt1_global[16]) = { 0, 1, 1, 2, 2, 3, 3, 4,
- 4, 5, 5, 6, 6, 7, 7, 8 };
-
-DECLARE_ALIGNED(16, static const uint8_t,
- filt2_global[16]) = { 2, 3, 3, 4, 4, 5, 5, 6,
- 6, 7, 7, 8, 8, 9, 9, 10 };
-
-DECLARE_ALIGNED(16, static const uint8_t,
- filt3_global[16]) = { 4, 5, 5, 6, 6, 7, 7, 8,
- 8, 9, 9, 10, 10, 11, 11, 12 };
-
-DECLARE_ALIGNED(16, static const uint8_t,
- filt4_global[16]) = { 6, 7, 7, 8, 8, 9, 9, 10,
- 10, 11, 11, 12, 12, 13, 13, 14 };
-
DECLARE_ALIGNED(32, static const uint8_t, filt_h4[]) = {
0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 0, 1, 1,
2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 2, 3, 3, 4, 4, 5,
@@ -64,11 +38,6 @@
2, 3, 4, 5, 3, 4, 5, 6, 4, 5, 6, 7, 5, 6, 7, 8,
};
-// These are reused by the avx2 intrinsics.
-filter8_1dfunction aom_filter_block1d8_v8_intrin_ssse3;
-filter8_1dfunction aom_filter_block1d8_h8_intrin_ssse3;
-filter8_1dfunction aom_filter_block1d4_h8_intrin_ssse3;
-
static void aom_filter_block1d4_h4_ssse3(
const uint8_t *src_ptr, ptrdiff_t src_pixels_per_line, uint8_t *output_ptr,
ptrdiff_t output_pitch, uint32_t output_height, const int16_t *filter) {
@@ -196,74 +165,6 @@
}
}
-void aom_filter_block1d4_h8_intrin_ssse3(
- const uint8_t *src_ptr, ptrdiff_t src_pixels_per_line, uint8_t *output_ptr,
- ptrdiff_t output_pitch, uint32_t output_height, const int16_t *filter) {
- __m128i firstFilters, secondFilters, shuffle1, shuffle2;
- __m128i srcRegFilt1, srcRegFilt2, srcRegFilt3, srcRegFilt4;
- __m128i addFilterReg64, filtersReg, srcReg, minReg;
- unsigned int i;
-
- // create a register with 0,64,0,64,0,64,0,64,0,64,0,64,0,64,0,64
- addFilterReg64 = _mm_set1_epi32((int)0x0400040u);
- filtersReg = _mm_loadu_si128((const __m128i *)filter);
- // converting the 16 bit (short) to 8 bit (byte) and have the same data
- // in both lanes of 128 bit register.
- filtersReg = _mm_packs_epi16(filtersReg, filtersReg);
-
- // duplicate only the first 16 bits in the filter into the first lane
- firstFilters = _mm_shufflelo_epi16(filtersReg, 0);
- // duplicate only the third 16 bit in the filter into the first lane
- secondFilters = _mm_shufflelo_epi16(filtersReg, 0xAAu);
- // duplicate only the seconds 16 bits in the filter into the second lane
- // firstFilters: k0 k1 k0 k1 k0 k1 k0 k1 k2 k3 k2 k3 k2 k3 k2 k3
- firstFilters = _mm_shufflehi_epi16(firstFilters, 0x55u);
- // duplicate only the forth 16 bits in the filter into the second lane
- // secondFilters: k4 k5 k4 k5 k4 k5 k4 k5 k6 k7 k6 k7 k6 k7 k6 k7
- secondFilters = _mm_shufflehi_epi16(secondFilters, 0xFFu);
-
- // loading the local filters
- shuffle1 = _mm_load_si128((__m128i const *)filt1_4_h8);
- shuffle2 = _mm_load_si128((__m128i const *)filt2_4_h8);
-
- for (i = 0; i < output_height; i++) {
- srcReg = _mm_loadu_si128((const __m128i *)(src_ptr - 3));
-
- // filter the source buffer
- srcRegFilt1 = _mm_shuffle_epi8(srcReg, shuffle1);
- srcRegFilt2 = _mm_shuffle_epi8(srcReg, shuffle2);
-
- // multiply 2 adjacent elements with the filter and add the result
- srcRegFilt1 = _mm_maddubs_epi16(srcRegFilt1, firstFilters);
- srcRegFilt2 = _mm_maddubs_epi16(srcRegFilt2, secondFilters);
-
- // extract the higher half of the lane
- srcRegFilt3 = _mm_srli_si128(srcRegFilt1, 8);
- srcRegFilt4 = _mm_srli_si128(srcRegFilt2, 8);
-
- minReg = _mm_min_epi16(srcRegFilt3, srcRegFilt2);
-
- // add and saturate all the results together
- srcRegFilt1 = _mm_adds_epi16(srcRegFilt1, srcRegFilt4);
- srcRegFilt3 = _mm_max_epi16(srcRegFilt3, srcRegFilt2);
- srcRegFilt1 = _mm_adds_epi16(srcRegFilt1, minReg);
- srcRegFilt1 = _mm_adds_epi16(srcRegFilt1, srcRegFilt3);
- srcRegFilt1 = _mm_adds_epi16(srcRegFilt1, addFilterReg64);
-
- // shift by 7 bit each 16 bits
- srcRegFilt1 = _mm_srai_epi16(srcRegFilt1, 7);
-
- // shrink to 8 bit each 16 bits
- srcRegFilt1 = _mm_packus_epi16(srcRegFilt1, srcRegFilt1);
- src_ptr += src_pixels_per_line;
-
- // save only 4 bytes
- *((int *)&output_ptr[0]) = _mm_cvtsi128_si32(srcRegFilt1);
-
- output_ptr += output_pitch;
- }
-}
-
static void aom_filter_block1d8_h4_ssse3(
const uint8_t *src_ptr, ptrdiff_t src_pixels_per_line, uint8_t *output_ptr,
ptrdiff_t output_pitch, uint32_t output_height, const int16_t *filter) {
@@ -403,168 +304,6 @@
}
}
-void aom_filter_block1d8_h8_intrin_ssse3(
- const uint8_t *src_ptr, ptrdiff_t src_pixels_per_line, uint8_t *output_ptr,
- ptrdiff_t output_pitch, uint32_t output_height, const int16_t *filter) {
- __m128i firstFilters, secondFilters, thirdFilters, forthFilters, srcReg;
- __m128i filt1Reg, filt2Reg, filt3Reg, filt4Reg;
- __m128i srcRegFilt1, srcRegFilt2, srcRegFilt3, srcRegFilt4;
- __m128i addFilterReg64, filtersReg, minReg;
- unsigned int i;
-
- // create a register with 0,64,0,64,0,64,0,64,0,64,0,64,0,64,0,64
- addFilterReg64 = _mm_set1_epi32((int)0x0400040u);
- filtersReg = _mm_loadu_si128((const __m128i *)filter);
- // converting the 16 bit (short) to 8 bit (byte) and have the same data
- // in both lanes of 128 bit register.
- filtersReg = _mm_packs_epi16(filtersReg, filtersReg);
-
- // duplicate only the first 16 bits (first and second byte)
- // across 128 bit register
- firstFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x100u));
- // duplicate only the second 16 bits (third and forth byte)
- // across 128 bit register
- secondFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x302u));
- // duplicate only the third 16 bits (fifth and sixth byte)
- // across 128 bit register
- thirdFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x504u));
- // duplicate only the forth 16 bits (seventh and eighth byte)
- // across 128 bit register
- forthFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x706u));
-
- filt1Reg = _mm_load_si128((__m128i const *)filt1_global);
- filt2Reg = _mm_load_si128((__m128i const *)filt2_global);
- filt3Reg = _mm_load_si128((__m128i const *)filt3_global);
- filt4Reg = _mm_load_si128((__m128i const *)filt4_global);
-
- for (i = 0; i < output_height; i++) {
- srcReg = _mm_loadu_si128((const __m128i *)(src_ptr - 3));
-
- // filter the source buffer
- srcRegFilt1 = _mm_shuffle_epi8(srcReg, filt1Reg);
- srcRegFilt2 = _mm_shuffle_epi8(srcReg, filt2Reg);
-
- // multiply 2 adjacent elements with the filter and add the result
- srcRegFilt1 = _mm_maddubs_epi16(srcRegFilt1, firstFilters);
- srcRegFilt2 = _mm_maddubs_epi16(srcRegFilt2, secondFilters);
-
- // filter the source buffer
- srcRegFilt3 = _mm_shuffle_epi8(srcReg, filt3Reg);
- srcRegFilt4 = _mm_shuffle_epi8(srcReg, filt4Reg);
-
- // multiply 2 adjacent elements with the filter and add the result
- srcRegFilt3 = _mm_maddubs_epi16(srcRegFilt3, thirdFilters);
- srcRegFilt4 = _mm_maddubs_epi16(srcRegFilt4, forthFilters);
-
- // add and saturate all the results together
- minReg = _mm_min_epi16(srcRegFilt2, srcRegFilt3);
- srcRegFilt1 = _mm_adds_epi16(srcRegFilt1, srcRegFilt4);
-
- srcRegFilt2 = _mm_max_epi16(srcRegFilt2, srcRegFilt3);
- srcRegFilt1 = _mm_adds_epi16(srcRegFilt1, minReg);
- srcRegFilt1 = _mm_adds_epi16(srcRegFilt1, srcRegFilt2);
- srcRegFilt1 = _mm_adds_epi16(srcRegFilt1, addFilterReg64);
-
- // shift by 7 bit each 16 bits
- srcRegFilt1 = _mm_srai_epi16(srcRegFilt1, 7);
-
- // shrink to 8 bit each 16 bits
- srcRegFilt1 = _mm_packus_epi16(srcRegFilt1, srcRegFilt1);
-
- src_ptr += src_pixels_per_line;
-
- // save only 8 bytes
- _mm_storel_epi64((__m128i *)&output_ptr[0], srcRegFilt1);
-
- output_ptr += output_pitch;
- }
-}
-
-void aom_filter_block1d8_v8_intrin_ssse3(
- const uint8_t *src_ptr, ptrdiff_t src_pitch, uint8_t *output_ptr,
- ptrdiff_t out_pitch, uint32_t output_height, const int16_t *filter) {
- __m128i addFilterReg64, filtersReg, minReg;
- __m128i firstFilters, secondFilters, thirdFilters, forthFilters;
- __m128i srcRegFilt1, srcRegFilt2, srcRegFilt3, srcRegFilt5;
- __m128i srcReg1, srcReg2, srcReg3, srcReg4, srcReg5, srcReg6, srcReg7;
- __m128i srcReg8;
- unsigned int i;
-
- // create a register with 0,64,0,64,0,64,0,64,0,64,0,64,0,64,0,64
- addFilterReg64 = _mm_set1_epi32((int)0x0400040u);
- filtersReg = _mm_loadu_si128((const __m128i *)filter);
- // converting the 16 bit (short) to 8 bit (byte) and have the same data
- // in both lanes of 128 bit register.
- filtersReg = _mm_packs_epi16(filtersReg, filtersReg);
-
- // duplicate only the first 16 bits in the filter
- firstFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x100u));
- // duplicate only the second 16 bits in the filter
- secondFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x302u));
- // duplicate only the third 16 bits in the filter
- thirdFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x504u));
- // duplicate only the forth 16 bits in the filter
- forthFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x706u));
-
- // load the first 7 rows of 8 bytes
- srcReg1 = _mm_loadl_epi64((const __m128i *)src_ptr);
- srcReg2 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch));
- srcReg3 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 2));
- srcReg4 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 3));
- srcReg5 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 4));
- srcReg6 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 5));
- srcReg7 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 6));
-
- for (i = 0; i < output_height; i++) {
- // load the last 8 bytes
- srcReg8 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 7));
-
- // merge the result together
- srcRegFilt1 = _mm_unpacklo_epi8(srcReg1, srcReg2);
- srcRegFilt3 = _mm_unpacklo_epi8(srcReg3, srcReg4);
-
- // merge the result together
- srcRegFilt2 = _mm_unpacklo_epi8(srcReg5, srcReg6);
- srcRegFilt5 = _mm_unpacklo_epi8(srcReg7, srcReg8);
-
- // multiply 2 adjacent elements with the filter and add the result
- srcRegFilt1 = _mm_maddubs_epi16(srcRegFilt1, firstFilters);
- srcRegFilt3 = _mm_maddubs_epi16(srcRegFilt3, secondFilters);
- srcRegFilt2 = _mm_maddubs_epi16(srcRegFilt2, thirdFilters);
- srcRegFilt5 = _mm_maddubs_epi16(srcRegFilt5, forthFilters);
-
- // add and saturate the results together
- minReg = _mm_min_epi16(srcRegFilt2, srcRegFilt3);
- srcRegFilt1 = _mm_adds_epi16(srcRegFilt1, srcRegFilt5);
- srcRegFilt2 = _mm_max_epi16(srcRegFilt2, srcRegFilt3);
- srcRegFilt1 = _mm_adds_epi16(srcRegFilt1, minReg);
- srcRegFilt1 = _mm_adds_epi16(srcRegFilt1, srcRegFilt2);
- srcRegFilt1 = _mm_adds_epi16(srcRegFilt1, addFilterReg64);
-
- // shift by 7 bit each 16 bit
- srcRegFilt1 = _mm_srai_epi16(srcRegFilt1, 7);
-
- // shrink to 8 bit each 16 bits
- srcRegFilt1 = _mm_packus_epi16(srcRegFilt1, srcRegFilt1);
-
- src_ptr += src_pitch;
-
- // shift down a row
- srcReg1 = srcReg2;
- srcReg2 = srcReg3;
- srcReg3 = srcReg4;
- srcReg4 = srcReg5;
- srcReg5 = srcReg6;
- srcReg6 = srcReg7;
- srcReg7 = srcReg8;
-
- // save only 8 bytes convolve result
- _mm_storel_epi64((__m128i *)&output_ptr[0], srcRegFilt1);
-
- output_ptr += out_pitch;
- }
-}
-
static void aom_filter_block1d16_h4_ssse3(
const uint8_t *src_ptr, ptrdiff_t src_pixels_per_line, uint8_t *output_ptr,
ptrdiff_t output_pitch, uint32_t output_height, const int16_t *filter) {
diff --git a/aom_dsp/x86/avg_intrin_sse2.c b/aom_dsp/x86/avg_intrin_sse2.c
index ca2752e..9ab9143 100644
--- a/aom_dsp/x86/avg_intrin_sse2.c
+++ b/aom_dsp/x86/avg_intrin_sse2.c
@@ -25,6 +25,11 @@
*out_hi = _mm_unpackhi_epi16(in, sign_bits);
}
+static INLINE __m128i invert_sign_32_sse2(__m128i a, __m128i sign) {
+ a = _mm_xor_si128(a, sign);
+ return _mm_sub_epi32(a, sign);
+}
+
void aom_minmax_8x8_sse2(const uint8_t *s, int p, const uint8_t *d, int dp,
int *min, int *max) {
__m128i u0, s0, d0, diff, maxabsdiff, minabsdiff, negdiff, absdiff0, absdiff;
@@ -583,21 +588,14 @@
int aom_satd_sse2(const tran_low_t *coeff, int length) {
int i;
const __m128i zero = _mm_setzero_si128();
- const __m128i one = _mm_set1_epi16(1);
__m128i accum = zero;
- for (i = 0; i < length; i += 16) {
- const __m128i src_line0 = load_tran_low(coeff);
- const __m128i src_line1 = load_tran_low(coeff + 8);
- const __m128i inv0 = _mm_sub_epi16(zero, src_line0);
- const __m128i inv1 = _mm_sub_epi16(zero, src_line1);
- const __m128i abs0 = _mm_max_epi16(src_line0, inv0); // abs(src_line)
- const __m128i abs1 = _mm_max_epi16(src_line1, inv1); // abs(src_line)
- const __m128i sum0 = _mm_madd_epi16(abs0, one);
- const __m128i sum1 = _mm_madd_epi16(abs1, one);
- accum = _mm_add_epi32(accum, sum0);
- accum = _mm_add_epi32(accum, sum1);
- coeff += 16;
+ for (i = 0; i < length; i += 4) {
+ const __m128i src_line = _mm_load_si128((const __m128i *)coeff);
+ const __m128i coeff_sign = _mm_srai_epi32(src_line, 31);
+ const __m128i abs_coeff = invert_sign_32_sse2(src_line, coeff_sign);
+ accum = _mm_add_epi32(accum, abs_coeff);
+ coeff += 4;
}
{ // cascading summation of accum
diff --git a/aom_dsp/x86/convolve.h b/aom_dsp/x86/convolve.h
index b4ff697..4ca214f 100644
--- a/aom_dsp/x86/convolve.h
+++ b/aom_dsp/x86/convolve.h
@@ -14,6 +14,7 @@
#include <assert.h>
#include "config/aom_config.h"
+#include "config/aom_dsp_rtcd.h"
#include "aom/aom_integer.h"
#include "aom_ports/mem.h"
diff --git a/aom_dsp/x86/fft_sse2.c b/aom_dsp/x86/fft_sse2.c
index c6023af..bdd235b 100644
--- a/aom_dsp/x86/fft_sse2.c
+++ b/aom_dsp/x86/fft_sse2.c
@@ -28,6 +28,9 @@
_mm_store_ps(&B[3 * ldb], row4);
}
+// Referenced by fft_avx2.c.
+void aom_transpose_float_sse2(const float *A, float *B, int n);
+
void aom_transpose_float_sse2(const float *A, float *B, int n) {
for (int y = 0; y < n; y += 4) {
for (int x = 0; x < n; x += 4) {
@@ -36,6 +39,9 @@
}
}
+// Referenced by fft_avx2.c.
+void aom_fft_unpack_2d_output_sse2(const float *packed, float *output, int n);
+
void aom_fft_unpack_2d_output_sse2(const float *packed, float *output, int n) {
const int n2 = n / 2;
output[0] = packed[0];
diff --git a/aom_dsp/x86/highbd_convolve_avx2.c b/aom_dsp/x86/highbd_convolve_avx2.c
index 8361e2f..11e4577 100644
--- a/aom_dsp/x86/highbd_convolve_avx2.c
+++ b/aom_dsp/x86/highbd_convolve_avx2.c
@@ -11,7 +11,7 @@
#include <immintrin.h>
#include <string.h>
-#include "config/aom_dsp_rtcd.h"
+#include "config/av1_rtcd.h"
#include "aom_dsp/x86/convolve.h"
#include "aom_dsp/x86/convolve_avx2.h"
diff --git a/aom_dsp/x86/highbd_convolve_ssse3.c b/aom_dsp/x86/highbd_convolve_ssse3.c
index 21389db..31c3c31 100644
--- a/aom_dsp/x86/highbd_convolve_ssse3.c
+++ b/aom_dsp/x86/highbd_convolve_ssse3.c
@@ -12,7 +12,7 @@
#include <tmmintrin.h>
#include <assert.h>
-#include "config/aom_dsp_rtcd.h"
+#include "config/av1_rtcd.h"
#include "aom_dsp/x86/convolve_sse2.h"
#include "aom_dsp/x86/convolve_common_intrin.h"
diff --git a/aom_dsp/x86/highbd_quantize_intrin_sse2.c b/aom_dsp/x86/highbd_quantize_intrin_sse2.c
index a5c450a..3b0c42c 100644
--- a/aom_dsp/x86/highbd_quantize_intrin_sse2.c
+++ b/aom_dsp/x86/highbd_quantize_intrin_sse2.c
@@ -14,6 +14,7 @@
#include "aom_dsp/aom_dsp_common.h"
#include "aom_mem/aom_mem.h"
#include "aom_ports/mem.h"
+#include "config/aom_dsp_rtcd.h"
void aom_highbd_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t count,
const int16_t *zbin_ptr,
diff --git a/aom_dsp/x86/highbd_sad_avx2.c b/aom_dsp/x86/highbd_sad_avx2.c
index e11754e..6c78eee 100644
--- a/aom_dsp/x86/highbd_sad_avx2.c
+++ b/aom_dsp/x86/highbd_sad_avx2.c
@@ -604,7 +604,7 @@
static AOM_FORCE_INLINE void aom_highbd_sadMxNxD_avx2(
int M, int N, int D, const uint8_t *src, int src_stride,
- const uint8_t *const ref_array[], int ref_stride, uint32_t *sad_array) {
+ const uint8_t *const ref_array[4], int ref_stride, uint32_t sad_array[4]) {
__m256i sad_vec[4];
const uint16_t *refp[4];
const uint16_t *keep = CONVERT_TO_SHORTPTR(src);
@@ -639,26 +639,26 @@
#define HIGHBD_SAD_MXNX4D_AVX2(m, n) \
void aom_highbd_sad##m##x##n##x4d_avx2( \
- const uint8_t *src, int src_stride, const uint8_t *const ref_array[], \
- int ref_stride, uint32_t *sad_array) { \
+ const uint8_t *src, int src_stride, const uint8_t *const ref_array[4], \
+ int ref_stride, uint32_t sad_array[4]) { \
aom_highbd_sadMxNxD_avx2(m, n, 4, src, src_stride, ref_array, ref_stride, \
sad_array); \
}
-#define HIGHBD_SAD_SKIP_MXNX4D_AVX2(m, n) \
- void aom_highbd_sad_skip_##m##x##n##x4d_avx2( \
- const uint8_t *src, int src_stride, const uint8_t *const ref_array[], \
- int ref_stride, uint32_t *sad_array) { \
- aom_highbd_sadMxNxD_avx2(m, (n / 2), 4, src, 2 * src_stride, ref_array, \
- 2 * ref_stride, sad_array); \
- sad_array[0] <<= 1; \
- sad_array[1] <<= 1; \
- sad_array[2] <<= 1; \
- sad_array[3] <<= 1; \
+#define HIGHBD_SAD_SKIP_MXNX4D_AVX2(m, n) \
+ void aom_highbd_sad_skip_##m##x##n##x4d_avx2( \
+ const uint8_t *src, int src_stride, const uint8_t *const ref_array[4], \
+ int ref_stride, uint32_t sad_array[4]) { \
+ aom_highbd_sadMxNxD_avx2(m, (n / 2), 4, src, 2 * src_stride, ref_array, \
+ 2 * ref_stride, sad_array); \
+ sad_array[0] <<= 1; \
+ sad_array[1] <<= 1; \
+ sad_array[2] <<= 1; \
+ sad_array[3] <<= 1; \
}
#define HIGHBD_SAD_MXNX3D_AVX2(m, n) \
void aom_highbd_sad##m##x##n##x3d_avx2( \
- const uint8_t *src, int src_stride, const uint8_t *const ref_array[], \
- int ref_stride, uint32_t *sad_array) { \
+ const uint8_t *src, int src_stride, const uint8_t *const ref_array[4], \
+ int ref_stride, uint32_t sad_array[4]) { \
aom_highbd_sadMxNxD_avx2(m, n, 3, src, src_stride, ref_array, ref_stride, \
sad_array); \
}
diff --git a/aom_dsp/x86/highbd_variance_avx2.c b/aom_dsp/x86/highbd_variance_avx2.c
index 36e6473..b4ff91d 100644
--- a/aom_dsp/x86/highbd_variance_avx2.c
+++ b/aom_dsp/x86/highbd_variance_avx2.c
@@ -729,14 +729,16 @@
VAR_FN(16, 32, 16, 9)
VAR_FN(16, 16, 16, 8)
VAR_FN(16, 8, 8, 7)
-VAR_FN(16, 4, 16, 6)
-VAR_FN(8, 32, 8, 8)
-VAR_FN(32, 8, 8, 8)
-VAR_FN(16, 64, 16, 10)
-VAR_FN(64, 16, 16, 10)
VAR_FN(8, 16, 8, 7)
VAR_FN(8, 8, 8, 6)
+#if !CONFIG_REALTIME_ONLY
+VAR_FN(16, 64, 16, 10)
+VAR_FN(32, 8, 8, 8)
+VAR_FN(64, 16, 16, 10)
+VAR_FN(8, 32, 8, 8)
+#endif // !CONFIG_REALTIME_ONLY
+
#undef VAR_FN
#define SSE2_HEIGHT(H) \
diff --git a/aom_dsp/x86/intrapred_avx2.c b/aom_dsp/x86/intrapred_avx2.c
index 621ef7a..242a548 100644
--- a/aom_dsp/x86/intrapred_avx2.c
+++ b/aom_dsp/x86/intrapred_avx2.c
@@ -11,7 +11,7 @@
#include <immintrin.h>
-#include "config/aom_dsp_rtcd.h"
+#include "config/av1_rtcd.h"
#include "aom_dsp/x86/intrapred_x86.h"
#include "aom_dsp/x86/intrapred_utils.h"
#include "aom_dsp/x86/lpf_common_sse2.h"
diff --git a/aom_dsp/x86/intrapred_sse4.c b/aom_dsp/x86/intrapred_sse4.c
index fb30420..9de8bf3 100644
--- a/aom_dsp/x86/intrapred_sse4.c
+++ b/aom_dsp/x86/intrapred_sse4.c
@@ -12,7 +12,7 @@
#include <emmintrin.h> // SSE2
#include <smmintrin.h> /* SSE4.1 */
-#include "config/aom_dsp_rtcd.h"
+#include "config/av1_rtcd.h"
#include "aom_dsp/x86/intrapred_x86.h"
#include "aom_dsp/x86/intrapred_utils.h"
#include "aom_dsp/x86/lpf_common_sse2.h"
diff --git a/aom_dsp/x86/obmc_sad_avx2.c b/aom_dsp/x86/obmc_sad_avx2.c
index 2aa2a05..9d1b7d4 100644
--- a/aom_dsp/x86/obmc_sad_avx2.c
+++ b/aom_dsp/x86/obmc_sad_avx2.c
@@ -13,6 +13,7 @@
#include <immintrin.h>
#include "config/aom_config.h"
+#include "config/aom_dsp_rtcd.h"
#include "aom_ports/mem.h"
#include "aom/aom_integer.h"
diff --git a/aom_dsp/x86/obmc_sad_sse4.c b/aom_dsp/x86/obmc_sad_sse4.c
index 0338a8c..542572c 100644
--- a/aom_dsp/x86/obmc_sad_sse4.c
+++ b/aom_dsp/x86/obmc_sad_sse4.c
@@ -13,6 +13,7 @@
#include <immintrin.h>
#include "config/aom_config.h"
+#include "config/aom_dsp_rtcd.h"
#include "aom_ports/mem.h"
#include "aom/aom_integer.h"
diff --git a/aom_dsp/x86/obmc_variance_avx2.c b/aom_dsp/x86/obmc_variance_avx2.c
index b2df8a9..c23d8c4 100644
--- a/aom_dsp/x86/obmc_variance_avx2.c
+++ b/aom_dsp/x86/obmc_variance_avx2.c
@@ -13,6 +13,7 @@
#include <immintrin.h>
#include "config/aom_config.h"
+#include "config/aom_dsp_rtcd.h"
#include "aom_ports/mem.h"
#include "aom/aom_integer.h"
diff --git a/aom_dsp/x86/obmc_variance_sse4.c b/aom_dsp/x86/obmc_variance_sse4.c
index aa73c39..89b050e 100644
--- a/aom_dsp/x86/obmc_variance_sse4.c
+++ b/aom_dsp/x86/obmc_variance_sse4.c
@@ -13,6 +13,7 @@
#include <immintrin.h>
#include "config/aom_config.h"
+#include "config/aom_dsp_rtcd.h"
#include "aom_ports/mem.h"
#include "aom/aom_integer.h"
@@ -258,10 +259,10 @@
*sse += xx_hsum_epi32_si64(v_sse_d);
}
-static INLINE void highbd_obmc_variance(const uint8_t *pre8, int pre_stride,
- const int32_t *wsrc,
- const int32_t *mask, int w, int h,
- unsigned int *sse, int *sum) {
+static INLINE void highbd_8_obmc_variance(const uint8_t *pre8, int pre_stride,
+ const int32_t *wsrc,
+ const int32_t *mask, int w, int h,
+ unsigned int *sse, int *sum) {
int64_t sum64 = 0;
uint64_t sse64 = 0;
if (w == 4) {
@@ -328,11 +329,11 @@
}
#define HBD_OBMCVARWXH(W, H) \
- unsigned int aom_highbd_obmc_variance##W##x##H##_sse4_1( \
+ unsigned int aom_highbd_8_obmc_variance##W##x##H##_sse4_1( \
const uint8_t *pre, int pre_stride, const int32_t *wsrc, \
const int32_t *mask, unsigned int *sse) { \
int sum; \
- highbd_obmc_variance(pre, pre_stride, wsrc, mask, W, H, sse, &sum); \
+ highbd_8_obmc_variance(pre, pre_stride, wsrc, mask, W, H, sse, &sum); \
return *sse - (unsigned int)(((int64_t)sum * sum) / (W * H)); \
} \
\
diff --git a/aom_dsp/x86/sse_avx2.c b/aom_dsp/x86/sse_avx2.c
index e6ee2fc..c5a5f5c 100644
--- a/aom_dsp/x86/sse_avx2.c
+++ b/aom_dsp/x86/sse_avx2.c
@@ -8,9 +8,11 @@
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
+
#include <smmintrin.h>
#include <immintrin.h>
+#include "config/aom_config.h"
#include "config/aom_dsp_rtcd.h"
#include "aom_ports/mem.h"
@@ -85,6 +87,7 @@
const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w);
*sum = _mm256_add_epi32(*sum, _mm256_madd_epi16(v_d_w, v_d_w));
}
+
static INLINE void sse_w8x2_avx2(const uint8_t *a, int a_stride,
const uint8_t *b, int b_stride, __m256i *sum) {
const __m128i v_a0 = xx_loadl_64(a);
@@ -96,6 +99,7 @@
const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w);
*sum = _mm256_add_epi32(*sum, _mm256_madd_epi16(v_d_w, v_d_w));
}
+
int64_t aom_sse_avx2(const uint8_t *a, int a_stride, const uint8_t *b,
int b_stride, int width, int height) {
int32_t y = 0;
@@ -249,6 +253,7 @@
const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w);
*sum = _mm256_add_epi32(*sum, _mm256_madd_epi16(v_d_w, v_d_w));
}
+
int64_t aom_highbd_sse_avx2(const uint8_t *a8, int a_stride, const uint8_t *b8,
int b_stride, int width, int height) {
int32_t y = 0;
diff --git a/aom_dsp/x86/sse_sse4.c b/aom_dsp/x86/sse_sse4.c
index 5f95eb9..7e74554 100644
--- a/aom_dsp/x86/sse_sse4.c
+++ b/aom_dsp/x86/sse_sse4.c
@@ -13,6 +13,7 @@
#include <smmintrin.h>
#include "config/aom_config.h"
+#include "config/aom_dsp_rtcd.h"
#include "aom_ports/mem.h"
#include "aom/aom_integer.h"
@@ -62,6 +63,7 @@
const __m128i v_d_w = _mm_sub_epi16(v_a_w, v_b_w);
*sum = _mm_add_epi32(*sum, _mm_madd_epi16(v_d_w, v_d_w));
}
+
static INLINE void sse8_sse4_1(const uint8_t *a, const uint8_t *b,
__m128i *sum) {
const __m128i v_a0 = xx_loadl_64(a);
diff --git a/aom_dsp/x86/synonyms_avx2.h b/aom_dsp/x86/synonyms_avx2.h
index 4d6ee6a..b729e5f 100644
--- a/aom_dsp/x86/synonyms_avx2.h
+++ b/aom_dsp/x86/synonyms_avx2.h
@@ -62,8 +62,8 @@
}
static INLINE __m256i yy_loadu2_128(const void *hi, const void *lo) {
- __m128i mhi = _mm_loadu_si128((__m128i *)(hi));
- __m128i mlo = _mm_loadu_si128((__m128i *)(lo));
+ __m128i mhi = _mm_loadu_si128((const __m128i *)(hi));
+ __m128i mlo = _mm_loadu_si128((const __m128i *)(lo));
return yy_set_m128i(mhi, mlo);
}
diff --git a/aom_ports/aarch32_cpudetect.c b/aom_ports/aarch32_cpudetect.c
new file mode 100644
index 0000000..753f957
--- /dev/null
+++ b/aom_ports/aarch32_cpudetect.c
@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+// Feature detection code for Armv7-A / AArch32.
+
+#include "arm_cpudetect.h"
+
+#if !CONFIG_RUNTIME_CPU_DETECT
+
+static int arm_get_cpu_caps(void) {
+ // This function should actually be a no-op. There is no way to adjust any of
+ // these because the RTCD tables do not exist: the functions are called
+ // statically.
+ int flags = 0;
+#if HAVE_NEON
+ flags |= HAS_NEON;
+#endif // HAVE_NEON
+ return flags;
+}
+
+#elif defined(_MSC_VER) // end !CONFIG_RUNTIME_CPU_DETECT
+
+static int arm_get_cpu_caps(void) {
+ int flags = 0;
+#if HAVE_NEON
+ // MSVC has no inline __asm support for Arm, but it does let you __emit
+ // instructions via their assembled hex code.
+ // All of these instructions should be essentially nops.
+ __try {
+ // VORR q0,q0,q0
+ __emit(0xF2200150);
+ flags |= HAS_NEON;
+ } __except (GetExceptionCode() == EXCEPTION_ILLEGAL_INSTRUCTION) {
+ // Ignore exception.
+ }
+#endif // HAVE_NEON
+ return flags;
+}
+
+#elif defined(ANDROID_USE_CPU_FEATURES_LIB)
+
+static int arm_get_cpu_caps(void) {
+ int flags = 0;
+#if HAVE_NEON
+ uint64_t features = android_getCpuFeatures();
+ if (features & ANDROID_CPU_ARM_FEATURE_NEON) flags |= HAS_NEON;
+#endif // HAVE_NEON
+ return flags;
+}
+
+#elif defined(__linux__) // end defined(AOM_USE_ANDROID_CPU_FEATURES)
+
+#include <sys/auxv.h>
+
+// Define hwcap values ourselves: building with an old auxv header where these
+// hwcap values are not defined should not prevent features from being enabled.
+#define AOM_AARCH32_HWCAP_NEON (1 << 12)
+
+static int arm_get_cpu_caps(void) {
+ int flags = 0;
+ unsigned long hwcap = getauxval(AT_HWCAP);
+#if HAVE_NEON
+ if (hwcap & AOM_AARCH32_HWCAP_NEON) flags |= HAS_NEON;
+#endif // HAVE_NEON
+ return flags;
+}
+#else // end __linux__
+#error \
+ "Runtime CPU detection selected, but no CPU detection method " \
+"available for your platform. Rerun cmake with -DCONFIG_RUNTIME_CPU_DETECT=0."
+#endif
+
+int aom_arm_cpu_caps(void) {
+ int flags = 0;
+ if (arm_cpu_env_flags(&flags)) {
+ return flags;
+ }
+ return arm_get_cpu_caps() & arm_cpu_env_mask();
+}
diff --git a/aom_ports/aarch64_cpudetect.c b/aom_ports/aarch64_cpudetect.c
new file mode 100644
index 0000000..43d5a14
--- /dev/null
+++ b/aom_ports/aarch64_cpudetect.c
@@ -0,0 +1,188 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include "arm_cpudetect.h"
+
+#if defined(__APPLE__)
+#include <sys/sysctl.h>
+#endif
+
+#if !CONFIG_RUNTIME_CPU_DETECT
+
+static int arm_get_cpu_caps(void) {
+ // This function should actually be a no-op. There is no way to adjust any of
+ // these because the RTCD tables do not exist: the functions are called
+ // statically.
+ int flags = 0;
+#if HAVE_NEON
+ flags |= HAS_NEON;
+#endif // HAVE_NEON
+ return flags;
+}
+
+#elif defined(__APPLE__) // end !CONFIG_RUNTIME_CPU_DETECT
+
+// sysctlbyname() parameter documentation for instruction set characteristics:
+// https://developer.apple.com/documentation/kernel/1387446-sysctlbyname/determining_instruction_set_characteristics
+static INLINE bool have_feature(const char *feature) {
+ int64_t feature_present = 0;
+ size_t size = sizeof(feature_present);
+ if (sysctlbyname(feature, &feature_present, &size, NULL, 0) != 0) {
+ return false;
+ }
+ return feature_present;
+}
+
+static int arm_get_cpu_caps(void) {
+ int flags = 0;
+#if HAVE_NEON
+ flags |= HAS_NEON;
+#endif // HAVE_NEON
+#if HAVE_ARM_CRC32
+ if (have_feature("hw.optional.armv8_crc32")) flags |= HAS_ARM_CRC32;
+#endif // HAVE_ARM_CRC32
+#if HAVE_NEON_DOTPROD
+ if (have_feature("hw.optional.arm.FEAT_DotProd")) flags |= HAS_NEON_DOTPROD;
+#endif // HAVE_NEON_DOTPROD
+#if HAVE_NEON_I8MM
+ if (have_feature("hw.optional.arm.FEAT_I8MM")) flags |= HAS_NEON_I8MM;
+#endif // HAVE_NEON_I8MM
+ return flags;
+}
+
+#elif defined(_WIN32) // end __APPLE__
+
+static int arm_get_cpu_caps(void) {
+ int flags = 0;
+// IsProcessorFeaturePresent() parameter documentation:
+// https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-isprocessorfeaturepresent#parameters
+#if HAVE_NEON
+ flags |= HAS_NEON; // Neon is mandatory in Armv8.0-A.
+#endif // HAVE_NEON
+#if HAVE_ARM_CRC32
+ if (IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE)) {
+ flags |= HAS_ARM_CRC32;
+ }
+#endif // HAVE_ARM_CRC32
+#if HAVE_NEON_DOTPROD
+// Support for PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE was added in Windows SDK
+// 20348, supported by Windows 11 and Windows Server 2022.
+#if defined(PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE)
+ if (IsProcessorFeaturePresent(PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE)) {
+ flags |= HAS_NEON_DOTPROD;
+ }
+#endif // defined(PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE)
+#endif // HAVE_NEON_DOTPROD
+ // No I8MM or SVE feature detection available on Windows at time of writing.
+ return flags;
+}
+
+#elif defined(ANDROID_USE_CPU_FEATURES_LIB)
+
+static int arm_get_cpu_caps(void) {
+ int flags = 0;
+#if HAVE_NEON
+ flags |= HAS_NEON; // Neon is mandatory in Armv8.0-A.
+#endif // HAVE_NEON
+ return flags;
+}
+
+#elif defined(__linux__) // end defined(AOM_USE_ANDROID_CPU_FEATURES)
+
+#include <sys/auxv.h>
+
+// Define hwcap values ourselves: building with an old auxv header where these
+// hwcap values are not defined should not prevent features from being enabled.
+#define AOM_AARCH64_HWCAP_CRC32 (1 << 7)
+#define AOM_AARCH64_HWCAP_ASIMDDP (1 << 20)
+#define AOM_AARCH64_HWCAP_SVE (1 << 22)
+#define AOM_AARCH64_HWCAP2_I8MM (1 << 13)
+
+static int arm_get_cpu_caps(void) {
+ int flags = 0;
+ unsigned long hwcap = getauxval(AT_HWCAP);
+ unsigned long hwcap2 = getauxval(AT_HWCAP2);
+#if HAVE_NEON
+ flags |= HAS_NEON; // Neon is mandatory in Armv8.0-A.
+#endif // HAVE_NEON
+#if HAVE_ARM_CRC32
+ if (hwcap & AOM_AARCH64_HWCAP_CRC32) flags |= HAS_ARM_CRC32;
+#endif // HAVE_ARM_CRC32
+#if HAVE_NEON_DOTPROD
+ if (hwcap & AOM_AARCH64_HWCAP_ASIMDDP) flags |= HAS_NEON_DOTPROD;
+#endif // HAVE_NEON_DOTPROD
+#if HAVE_NEON_I8MM
+ if (hwcap2 & AOM_AARCH64_HWCAP2_I8MM) flags |= HAS_NEON_I8MM;
+#endif // HAVE_NEON_I8MM
+#if HAVE_SVE
+ if (hwcap & AOM_AARCH64_HWCAP_SVE) flags |= HAS_SVE;
+#endif // HAVE_SVE
+ return flags;
+}
+
+#elif defined(__Fuchsia__) // end __linux__
+
+#include <zircon/features.h>
+#include <zircon/syscalls.h>
+
+// Added in https://fuchsia-review.googlesource.com/c/fuchsia/+/894282.
+#ifndef ZX_ARM64_FEATURE_ISA_I8MM
+#define ZX_ARM64_FEATURE_ISA_I8MM ((uint32_t)(1u << 19))
+#endif
+// Added in https://fuchsia-review.googlesource.com/c/fuchsia/+/895083.
+#ifndef ZX_ARM64_FEATURE_ISA_SVE
+#define ZX_ARM64_FEATURE_ISA_SVE ((uint32_t)(1u << 20))
+#endif
+
+static int arm_get_cpu_caps(void) {
+ int flags = 0;
+#if HAVE_NEON
+ flags |= HAS_NEON; // Neon is mandatory in Armv8.0-A.
+#endif // HAVE_NEON
+ uint32_t features;
+ zx_status_t status = zx_system_get_features(ZX_FEATURE_KIND_CPU, &features);
+ if (status != ZX_OK) return flags;
+#if HAVE_ARM_CRC32
+ if (features & ZX_ARM64_FEATURE_ISA_CRC32) flags |= HAS_ARM_CRC32;
+#endif // HAVE_ARM_CRC32
+#if HAVE_NEON_DOTPROD
+ if (features & ZX_ARM64_FEATURE_ISA_DP) flags |= HAS_NEON_DOTPROD;
+#endif // HAVE_NEON_DOTPROD
+#if HAVE_NEON_I8MM
+ if (features & ZX_ARM64_FEATURE_ISA_I8MM) flags |= HAS_NEON_I8MM;
+#endif // HAVE_NEON_I8MM
+#if HAVE_SVE
+ if (features & ZX_ARM64_FEATURE_ISA_SVE) flags |= HAS_SVE;
+#endif // HAVE_SVE
+ return flags;
+}
+
+#else // end __Fuchsia__
+#error \
+ "Runtime CPU detection selected, but no CPU detection method " \
+"available for your platform. Rerun cmake with -DCONFIG_RUNTIME_CPU_DETECT=0."
+#endif
+
+int aom_arm_cpu_caps(void) {
+ int flags = 0;
+ if (!arm_cpu_env_flags(&flags)) {
+ flags = arm_get_cpu_caps() & arm_cpu_env_mask();
+ }
+
+ // Restrict flags: FEAT_I8MM assumes that FEAT_DotProd is available.
+ if (!(flags & HAS_NEON_DOTPROD)) flags &= ~HAS_NEON_I8MM;
+
+ // Restrict flags: SVE assumes that FEAT_{DotProd,I8MM} are available.
+ if (!(flags & HAS_NEON_DOTPROD)) flags &= ~HAS_SVE;
+ if (!(flags & HAS_NEON_I8MM)) flags &= ~HAS_SVE;
+
+ return flags;
+}
diff --git a/aom_ports/aom_ports.cmake b/aom_ports/aom_ports.cmake
index e3b67e4..8fd2ffd 100644
--- a/aom_ports/aom_ports.cmake
+++ b/aom_ports/aom_ports.cmake
@@ -24,8 +24,10 @@
list(APPEND AOM_PORTS_INCLUDES_X86 "${AOM_ROOT}/aom_ports/x86_abi_support.asm")
-list(APPEND AOM_PORTS_SOURCES_ARM "${AOM_ROOT}/aom_ports/arm.h"
- "${AOM_ROOT}/aom_ports/arm_cpudetect.c")
+list(APPEND AOM_PORTS_SOURCES_AARCH32
+ "${AOM_ROOT}/aom_ports/aarch32_cpudetect.c")
+list(APPEND AOM_PORTS_SOURCES_AARCH64
+ "${AOM_ROOT}/aom_ports/aarch64_cpudetect.c")
if(CONFIG_RUNTIME_CPU_DETECT AND ANDROID_NDK)
include_directories(${ANDROID_NDK}/sources/android/cpufeatures)
@@ -57,8 +59,11 @@
elseif(WIN32 AND "${AOM_TARGET_CPU}" STREQUAL "x86_64")
add_asm_library("aom_ports" "AOM_PORTS_ASM_X86")
set(aom_ports_has_symbols 1)
+ elseif("${AOM_TARGET_CPU}" STREQUAL "arm64")
+ add_library(aom_ports OBJECT ${AOM_PORTS_SOURCES_AARCH64})
+ set(aom_ports_has_symbols 1)
elseif("${AOM_TARGET_CPU}" MATCHES "arm")
- add_library(aom_ports OBJECT ${AOM_PORTS_SOURCES_ARM})
+ add_library(aom_ports OBJECT ${AOM_PORTS_SOURCES_AARCH32})
set(aom_ports_has_symbols 1)
elseif("${AOM_TARGET_CPU}" MATCHES "ppc")
add_library(aom_ports OBJECT ${AOM_PORTS_SOURCES_PPC})
diff --git a/aom_ports/arm.h b/aom_ports/arm.h
index cb1fb9b..853741d 100644
--- a/aom_ports/arm.h
+++ b/aom_ports/arm.h
@@ -19,12 +19,16 @@
extern "C" {
#endif
-/*ARMv5TE "Enhanced DSP" instructions.*/
-#define HAS_EDSP 0x01
-/*ARMv6 "Parallel" or "Media" instructions.*/
-#define HAS_MEDIA 0x02
-/*ARMv7 optional NEON instructions.*/
-#define HAS_NEON 0x04
+// Armv7-A optional Neon instructions, mandatory from Armv8.0-A.
+#define HAS_NEON (1 << 0)
+// Armv8.0-A optional CRC32 instructions, mandatory from Armv8.1-A.
+#define HAS_ARM_CRC32 (1 << 1)
+// Armv8.2-A optional Neon dot-product instructions, mandatory from Armv8.4-A.
+#define HAS_NEON_DOTPROD (1 << 2)
+// Armv8.2-A optional Neon i8mm instructions, mandatory from Armv8.6-A.
+#define HAS_NEON_I8MM (1 << 3)
+// Armv8.2-A optional SVE instructions, mandatory from Armv9.0-A.
+#define HAS_SVE (1 << 4)
int aom_arm_cpu_caps(void);
diff --git a/aom_ports/arm_cpudetect.c b/aom_ports/arm_cpudetect.c
deleted file mode 100644
index 276ef61..0000000
--- a/aom_ports/arm_cpudetect.c
+++ /dev/null
@@ -1,158 +0,0 @@
-/*
- * Copyright (c) 2016, Alliance for Open Media. All rights reserved
- *
- * This source code is subject to the terms of the BSD 2 Clause License and
- * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
- * was not distributed with this source code in the LICENSE file, you can
- * obtain it at www.aomedia.org/license/software. If the Alliance for Open
- * Media Patent License 1.0 was not distributed with this source code in the
- * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
- */
-
-#include <stdlib.h>
-#include <string.h>
-#include "aom_ports/arm.h"
-#include "config/aom_config.h"
-
-#ifdef WINAPI_FAMILY
-#include <winapifamily.h>
-#if !WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP)
-#define getenv(x) NULL
-#endif
-#endif
-
-static int arm_cpu_env_flags(int *flags) {
- char *env;
- env = getenv("AOM_SIMD_CAPS");
- if (env && *env) {
- *flags = (int)strtol(env, NULL, 0);
- return 0;
- }
- *flags = 0;
- return -1;
-}
-
-static int arm_cpu_env_mask(void) {
- char *env;
- env = getenv("AOM_SIMD_CAPS_MASK");
- return env && *env ? (int)strtol(env, NULL, 0) : ~0;
-}
-
-#if !CONFIG_RUNTIME_CPU_DETECT || defined(__APPLE__)
-
-int aom_arm_cpu_caps(void) {
- /* This function should actually be a no-op. There is no way to adjust any of
- * these because the RTCD tables do not exist: the functions are called
- * statically */
- int flags;
- int mask;
- if (!arm_cpu_env_flags(&flags)) {
- return flags;
- }
- mask = arm_cpu_env_mask();
-#if HAVE_NEON
- flags |= HAS_NEON;
-#endif /* HAVE_NEON */
- return flags & mask;
-}
-
-#elif defined(_MSC_VER) /* end !CONFIG_RUNTIME_CPU_DETECT || __APPLE__ */
-#if HAVE_NEON && !AOM_ARCH_AARCH64
-/*For GetExceptionCode() and EXCEPTION_ILLEGAL_INSTRUCTION.*/
-#undef WIN32_LEAN_AND_MEAN
-#define WIN32_LEAN_AND_MEAN
-#undef WIN32_EXTRA_LEAN
-#define WIN32_EXTRA_LEAN
-#include <windows.h>
-#endif // HAVE_NEON && !AOM_ARCH_AARCH64
-
-int aom_arm_cpu_caps(void) {
- int flags;
- int mask;
- if (!arm_cpu_env_flags(&flags)) {
- return flags;
- }
- mask = arm_cpu_env_mask();
-#if AOM_ARCH_AARCH64
- return HAS_NEON & mask;
-#else
-/* MSVC has no inline __asm support for ARM, but it does let you __emit
- * instructions via their assembled hex code.
- * All of these instructions should be essentially nops.
- */
-#if HAVE_NEON
- if (mask & HAS_NEON) {
- __try {
- /*VORR q0,q0,q0*/
- __emit(0xF2200150);
- flags |= HAS_NEON;
- } __except (GetExceptionCode() == EXCEPTION_ILLEGAL_INSTRUCTION) {
- /*Ignore exception.*/
- }
- }
-#endif /* HAVE_NEON */
- return flags & mask;
-#endif // AOM_ARCH_AARCH64
-}
-
-#elif defined(__ANDROID__) /* end _MSC_VER */
-#include <cpu-features.h>
-
-int aom_arm_cpu_caps(void) {
- int flags;
- int mask;
- uint64_t features;
- if (!arm_cpu_env_flags(&flags)) {
- return flags;
- }
- mask = arm_cpu_env_mask();
- features = android_getCpuFeatures();
-
-#if HAVE_NEON
- if (features & ANDROID_CPU_ARM_FEATURE_NEON) flags |= HAS_NEON;
-#endif /* HAVE_NEON */
- return flags & mask;
-}
-
-#elif defined(__linux__) /* end __ANDROID__ */
-
-#include <stdio.h>
-
-int aom_arm_cpu_caps(void) {
- FILE *fin;
- int flags;
- int mask;
- if (!arm_cpu_env_flags(&flags)) {
- return flags;
- }
- mask = arm_cpu_env_mask();
- /* Reading /proc/self/auxv would be easier, but that doesn't work reliably
- * on Android.
- * This also means that detection will fail in Scratchbox.
- */
- fin = fopen("/proc/cpuinfo", "r");
- if (fin != NULL) {
- /* 512 should be enough for anybody (it's even enough for all the flags
- * that x86 has accumulated... so far).
- */
- char buf[512];
- while (fgets(buf, 511, fin) != NULL) {
-#if HAVE_NEON
- if (memcmp(buf, "Features", 8) == 0) {
- char *p;
- p = strstr(buf, " neon");
- if (p != NULL && (p[5] == ' ' || p[5] == '\n')) {
- flags |= HAS_NEON;
- }
- }
-#endif /* HAVE_NEON */
- }
- fclose(fin);
- }
- return flags & mask;
-}
-#else /* end __linux__ */
-#error \
- "Runtime CPU detection selected, but no CPU detection method " \
-"available for your platform. Rerun cmake with -DCONFIG_RUNTIME_CPU_DETECT=0."
-#endif
diff --git a/aom_ports/arm_cpudetect.h b/aom_ports/arm_cpudetect.h
new file mode 100644
index 0000000..33c2d1b
--- /dev/null
+++ b/aom_ports/arm_cpudetect.h
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include "aom_ports/arm.h"
+#include "config/aom_config.h"
+
+#include <stdbool.h>
+#include <stdlib.h>
+#include <string.h>
+
+#if defined(_WIN32)
+#undef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#undef WIN32_EXTRA_LEAN
+#define WIN32_EXTRA_LEAN
+#include <windows.h>
+#endif
+
+#ifdef WINAPI_FAMILY
+#include <winapifamily.h>
+#if !WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP)
+#define getenv(x) NULL
+#endif
+#endif
+
+#if defined(__ANDROID__) && (__ANDROID_API__ < 18)
+#define ANDROID_USE_CPU_FEATURES_LIB 1
+// Use getauxval() when targeting (64-bit) Android with API level >= 18.
+// getauxval() is supported since Android API level 18 (Android 4.3.)
+// First Android version with 64-bit support was Android 5.x (API level 21).
+#include <cpu-features.h>
+#endif
+
+static bool arm_cpu_env_flags(int *flags) {
+ const char *env = getenv("AOM_SIMD_CAPS");
+ if (env && *env) {
+ *flags = (int)strtol(env, NULL, 0);
+ return true;
+ }
+ return false;
+}
+
+static int arm_cpu_env_mask(void) {
+ const char *env = getenv("AOM_SIMD_CAPS_MASK");
+ return env && *env ? (int)strtol(env, NULL, 0) : ~0;
+}
diff --git a/aom_ports/bitops.h b/aom_ports/bitops.h
index 3c5b992..7f4c165 100644
--- a/aom_ports/bitops.h
+++ b/aom_ports/bitops.h
@@ -13,7 +13,6 @@
#define AOM_AOM_PORTS_BITOPS_H_
#include <assert.h>
-#include <stdint.h>
#include "aom_ports/msvc.h"
#include "config/aom_config.h"
@@ -34,12 +33,8 @@
// These versions of get_msb() are only valid when n != 0 because all
// of the optimized versions are undefined when n == 0:
-// get_byteswap64:
-// Returns the number (uint64_t) with byte-positions reversed
-// e.g. input 0x123456789ABCDEF0 returns 0xF0DEBC9A78563412
-
// GCC compiler: https://gcc.gnu.org/onlinedocs/gcc/Other-Builtins.html
-// MSVC: https://learn.microsoft.com/en-us/cpp/c-runtime-library/
+// MSVC: https://learn.microsoft.com/en-us/cpp/intrinsics/compiler-intrinsics
// use GNU builtins where available.
#if defined(__GNUC__) && \
@@ -48,10 +43,6 @@
assert(n != 0);
return 31 ^ __builtin_clz(n);
}
-
-static INLINE uint64_t get_byteswap64(uint64_t num) {
- return __builtin_bswap64(num);
-}
#elif defined(USE_MSC_INTRINSICS)
#pragma intrinsic(_BitScanReverse)
@@ -61,10 +52,6 @@
_BitScanReverse(&first_set_bit, n);
return first_set_bit;
}
-
-static INLINE uint64_t get_byteswap64(uint64_t num) {
- return _byteswap_uint64(num);
-}
#undef USE_MSC_INTRINSICS
#else
static INLINE int get_msb(unsigned int n) {
@@ -82,26 +69,6 @@
}
return log;
}
-
-static INLINE uint64_t get_byteswap64(uint64_t num) {
- uint64_t out = 0x00;
- uint64_t mask = 0xFF00000000000000;
- int bit_shift = 56; // 7 bytes
- // 4 ms bytes
- do {
- out |= (num & mask) >> bit_shift;
- mask >>= 8;
- bit_shift -= 16;
- } while (bit_shift >= 0);
- // 4 ls bytes
- bit_shift = 8; // 1 byte
- do {
- out |= (num & mask) << bit_shift;
- mask >>= 8;
- bit_shift += 16;
- } while (bit_shift <= 56);
- return out;
-}
#endif
#ifdef __cplusplus
diff --git a/aom_ports/mem.h b/aom_ports/mem.h
index e396842..a70ce82 100644
--- a/aom_ports/mem.h
+++ b/aom_ports/mem.h
@@ -24,16 +24,6 @@
#define DECLARE_ALIGNED(n, typ, val) typ val
#endif
-/* Indicates that the usage of the specified variable has been audited to assure
- * that it's safe to use uninitialized. Silences 'may be used uninitialized'
- * warnings on gcc.
- */
-#if defined(__GNUC__) && __GNUC__
-#define UNINITIALIZED_IS_SAFE(x) x = x
-#else
-#define UNINITIALIZED_IS_SAFE(x) x
-#endif
-
#if HAVE_NEON && defined(_MSC_VER)
#define __builtin_prefetch(x)
#endif
diff --git a/aom_scale/generic/yv12config.c b/aom_scale/generic/yv12config.c
index 82376f4..94b400b 100644
--- a/aom_scale/generic/yv12config.c
+++ b/aom_scale/generic/yv12config.c
@@ -193,7 +193,9 @@
if (num_pyramid_levels > 0) {
ybf->y_pyramid = aom_alloc_pyramid(width, height, num_pyramid_levels,
use_highbitdepth);
+ if (!ybf->y_pyramid) return AOM_CODEC_MEM_ERROR;
ybf->corners = av1_alloc_corner_list();
+ if (!ybf->corners) return AOM_CODEC_MEM_ERROR;
}
#endif // CONFIG_AV1_ENCODER && !CONFIG_REALTIME_ONLY
diff --git a/aom_util/aom_thread.c b/aom_util/aom_thread.c
index 2c62b24..fa3b0a2 100644
--- a/aom_util/aom_thread.c
+++ b/aom_util/aom_thread.c
@@ -24,6 +24,7 @@
#include <string.h> // for memset()
#include "aom_mem/aom_mem.h"
+#include "aom_ports/sanitizer.h"
#include "aom_util/aom_thread.h"
#if CONFIG_MULTITHREAD
@@ -144,11 +145,30 @@
pthread_mutex_destroy(&worker->impl_->mutex_);
goto Error;
}
+ pthread_attr_t attr;
+ if (pthread_attr_init(&attr)) goto Error2;
+ // Debug ASan builds require at least ~1MiB of stack; prevents
+ // failures on macOS arm64 where the default is 512KiB.
+ // See: https://crbug.com/aomedia/3379
+#if defined(AOM_ADDRESS_SANITIZER) && defined(__APPLE__) && AOM_ARCH_ARM && \
+ !defined(NDEBUG)
+ size_t stacksize;
+ if (!pthread_attr_getstacksize(&attr, &stacksize)) {
+ const size_t kMinStackSize = 1 << 20; // 1 MiB
+ if (stacksize < kMinStackSize &&
+ pthread_attr_setstacksize(&attr, kMinStackSize)) {
+ pthread_attr_destroy(&attr);
+ goto Error2;
+ }
+ }
+#endif
pthread_mutex_lock(&worker->impl_->mutex_);
- ok = !pthread_create(&worker->impl_->thread_, NULL, thread_loop, worker);
+ ok = !pthread_create(&worker->impl_->thread_, &attr, thread_loop, worker);
if (ok) worker->status_ = OK;
pthread_mutex_unlock(&worker->impl_->mutex_);
+ pthread_attr_destroy(&attr);
if (!ok) {
+ Error2:
pthread_mutex_destroy(&worker->impl_->mutex_);
pthread_cond_destroy(&worker->impl_->condition_);
Error:
diff --git a/aom_util/aom_thread.h b/aom_util/aom_thread.h
index 2df190f..ec2ea43 100644
--- a/aom_util/aom_thread.h
+++ b/aom_util/aom_thread.h
@@ -37,6 +37,7 @@
#include <process.h> // NOLINT
#include <windows.h> // NOLINT
typedef HANDLE pthread_t;
+typedef int pthread_attr_t;
typedef CRITICAL_SECTION pthread_mutex_t;
#if _WIN32_WINNT < 0x0600
@@ -60,7 +61,18 @@
#define THREADFN unsigned int __stdcall
#define THREAD_RETURN(val) (unsigned int)((DWORD_PTR)val)
-static INLINE int pthread_create(pthread_t *const thread, const void *attr,
+static INLINE int pthread_attr_init(pthread_attr_t *attr) {
+ (void)attr;
+ return 0;
+}
+
+static INLINE int pthread_attr_destroy(pthread_attr_t *attr) {
+ (void)attr;
+ return 0;
+}
+
+static INLINE int pthread_create(pthread_t *const thread,
+ const pthread_attr_t *attr,
unsigned int(__stdcall *start)(void *),
void *arg) {
(void)attr;
diff --git a/apps/aomenc.c b/apps/aomenc.c
index 09306f2..c3f5c33 100644
--- a/apps/aomenc.c
+++ b/apps/aomenc.c
@@ -74,7 +74,10 @@
if (detail) fprintf(stderr, " %s\n", detail);
- if (fatal) exit(EXIT_FAILURE);
+ if (fatal) {
+ aom_codec_destroy(ctx);
+ exit(EXIT_FAILURE);
+ }
}
}
diff --git a/av1/arg_defs.c b/av1/arg_defs.c
index 35a2ab4..0575654 100644
--- a/av1/arg_defs.c
+++ b/av1/arg_defs.c
@@ -303,7 +303,7 @@
ARG_DEF(NULL, "max-intra-rate", 1, "Max I-frame bitrate (pct)"),
#if CONFIG_AV1_ENCODER
.cpu_used_av1 = ARG_DEF(NULL, "cpu-used", 1,
- "Speed setting (0..6 in good mode, 5..10 in realtime "
+ "Speed setting (0..6 in good mode, 5..11 in realtime "
"mode, 0..9 in all intra mode)"),
.rowmtarg =
ARG_DEF(NULL, "row-mt", 1,
diff --git a/av1/arg_defs.h b/av1/arg_defs.h
index b9d0cfe..73c78ca 100644
--- a/av1/arg_defs.h
+++ b/av1/arg_defs.h
@@ -21,7 +21,6 @@
#include "common/webmenc.h"
#endif
#include "aom/aomcx.h"
-#include "aom_dsp/flow_estimation/flow_estimation.h"
enum TestDecodeFatality {
TEST_DECODE_OFF,
diff --git a/av1/av1.cmake b/av1/av1.cmake
index 43b7665..1bb0539 100644
--- a/av1/av1.cmake
+++ b/av1/av1.cmake
@@ -267,7 +267,6 @@
"${AOM_ROOT}/av1/common/x86/convolve_2d_sse2.c"
"${AOM_ROOT}/av1/common/x86/convolve_sse2.c"
"${AOM_ROOT}/av1/common/x86/jnt_convolve_sse2.c"
- "${AOM_ROOT}/av1/common/x86/warp_plane_sse2.c"
"${AOM_ROOT}/av1/common/x86/wiener_convolve_sse2.c")
list(APPEND AOM_AV1_COMMON_INTRIN_SSSE3
@@ -319,7 +318,8 @@
"${AOM_ROOT}/av1/encoder/x86/temporal_filter_sse2.c"
"${AOM_ROOT}/av1/encoder/x86/wedge_utils_sse2.c")
-list(APPEND AOM_AV1_ENCODER_INTRIN_SSE3 "${AOM_ROOT}/av1/encoder/x86/ml_sse3.c")
+list(APPEND AOM_AV1_ENCODER_INTRIN_SSE3 "${AOM_ROOT}/av1/encoder/x86/ml_sse3.c"
+ "${AOM_ROOT}/av1/encoder/x86/ml_sse3.h")
list(APPEND AOM_AV1_ENCODER_INTRIN_SSSE3
"${AOM_ROOT}/av1/encoder/x86/reconinter_enc_ssse3.c")
@@ -347,26 +347,30 @@
"${AOM_ROOT}/av1/encoder/x86/av1_k_means_avx2.c"
"${AOM_ROOT}/av1/encoder/x86/temporal_filter_avx2.c"
"${AOM_ROOT}/av1/encoder/x86/pickrst_avx2.c"
- "${AOM_ROOT}/av1/encoder/x86/cnn_avx2.c")
+ "${AOM_ROOT}/av1/encoder/x86/cnn_avx2.c"
+ "${AOM_ROOT}/av1/encoder/x86/ml_avx2.c")
list(APPEND AOM_AV1_ENCODER_INTRIN_NEON
- "${AOM_ROOT}/av1/encoder/arm/neon/quantize_neon.c"
- "${AOM_ROOT}/av1/encoder/arm/neon/av1_highbd_quantize_neon.c"
- "${AOM_ROOT}/av1/encoder/arm/neon/ml_neon.c"
- "${AOM_ROOT}/av1/encoder/arm/neon/picksrt_neon.c"
- "${AOM_ROOT}/av1/encoder/arm/neon/rdopt_neon.c"
"${AOM_ROOT}/av1/encoder/arm/neon/av1_error_neon.c"
- "${AOM_ROOT}/av1/encoder/arm/neon/encodetxb_neon.c"
- "${AOM_ROOT}/av1/encoder/arm/neon/hybrid_fwd_txfm_neon.c"
- "${AOM_ROOT}/av1/encoder/arm/neon/av1_k_means_neon.c"
"${AOM_ROOT}/av1/encoder/arm/neon/av1_fwd_txfm2d_neon.c"
+ "${AOM_ROOT}/av1/encoder/arm/neon/av1_highbd_quantize_neon.c"
+ "${AOM_ROOT}/av1/encoder/arm/neon/av1_k_means_neon.c"
+ "${AOM_ROOT}/av1/encoder/arm/neon/encodetxb_neon.c"
"${AOM_ROOT}/av1/encoder/arm/neon/highbd_fwd_txfm_neon.c"
- "${AOM_ROOT}/av1/encoder/arm/neon/wedge_utils_neon.c"
+ "${AOM_ROOT}/av1/encoder/arm/neon/hybrid_fwd_txfm_neon.c"
+ "${AOM_ROOT}/av1/encoder/arm/neon/ml_neon.c"
+ "${AOM_ROOT}/av1/encoder/arm/neon/pickrst_neon.c"
+ "${AOM_ROOT}/av1/encoder/arm/neon/quantize_neon.c"
+ "${AOM_ROOT}/av1/encoder/arm/neon/rdopt_neon.c"
"${AOM_ROOT}/av1/encoder/arm/neon/reconinter_enc_neon.c"
- "${AOM_ROOT}/av1/encoder/arm/neon/temporal_filter_neon.c")
+ "${AOM_ROOT}/av1/encoder/arm/neon/temporal_filter_neon.c"
+ "${AOM_ROOT}/av1/encoder/arm/neon/wedge_utils_neon.c")
+
+list(APPEND AOM_AV1_ENCODER_INTRIN_NEON_DOTPROD
+ "${AOM_ROOT}/av1/encoder/arm/neon/temporal_filter_neon_dotprod.c")
list(APPEND AOM_AV1_ENCODER_INTRIN_ARM_CRC32
- "${AOM_ROOT}/av1/encoder/arm/crc32/hash_crc32.c")
+ "${AOM_ROOT}/av1/encoder/arm/crc32/hash_arm_crc32.c")
list(APPEND AOM_AV1_COMMON_INTRIN_NEON
"${AOM_ROOT}/av1/common/arm/av1_inv_txfm_neon.c"
@@ -376,10 +380,10 @@
"${AOM_ROOT}/av1/common/arm/blend_a64_vmask_neon.c"
"${AOM_ROOT}/av1/common/arm/cdef_block_neon.c"
"${AOM_ROOT}/av1/common/arm/cfl_neon.c"
+ "${AOM_ROOT}/av1/common/arm/compound_convolve_neon.c"
"${AOM_ROOT}/av1/common/arm/convolve_neon.c"
"${AOM_ROOT}/av1/common/arm/convolve_neon.h"
"${AOM_ROOT}/av1/common/arm/highbd_inv_txfm_neon.c"
- "${AOM_ROOT}/av1/common/arm/jnt_convolve_neon.c"
"${AOM_ROOT}/av1/common/arm/reconinter_neon.c"
"${AOM_ROOT}/av1/common/arm/reconintra_neon.c"
"${AOM_ROOT}/av1/common/arm/resize_neon.c"
@@ -387,6 +391,18 @@
"${AOM_ROOT}/av1/common/arm/warp_plane_neon.c"
"${AOM_ROOT}/av1/common/arm/wiener_convolve_neon.c")
+list(APPEND AOM_AV1_COMMON_INTRIN_NEON_DOTPROD
+ "${AOM_ROOT}/av1/common/arm/compound_convolve_neon_dotprod.c"
+ "${AOM_ROOT}/av1/common/arm/convolve_neon_dotprod.c")
+
+list(APPEND AOM_AV1_COMMON_INTRIN_NEON_I8MM
+ "${AOM_ROOT}/av1/common/arm/compound_convolve_neon_i8mm.c"
+ "${AOM_ROOT}/av1/common/arm/convolve_neon_i8mm.c"
+ "${AOM_ROOT}/av1/common/arm/warp_plane_neon_i8mm.c")
+
+list(APPEND AOM_AV1_COMMON_INTRIN_SVE
+ "${AOM_ROOT}/av1/common/arm/warp_plane_sve.c")
+
list(APPEND AOM_AV1_ENCODER_INTRIN_SSE4_2
"${AOM_ROOT}/av1/encoder/x86/hash_sse42.c")
@@ -446,7 +462,14 @@
"${AOM_ROOT}/av1/common/x86/highbd_warp_affine_avx2.c")
list(APPEND AOM_AV1_COMMON_INTRIN_NEON
- "${AOM_ROOT}/av1/common/arm/highbd_convolve_neon.c")
+ "${AOM_ROOT}/av1/common/arm/highbd_compound_convolve_neon.c"
+ "${AOM_ROOT}/av1/common/arm/highbd_convolve_horiz_rs_neon.c"
+ "${AOM_ROOT}/av1/common/arm/highbd_convolve_neon.c"
+ "${AOM_ROOT}/av1/common/arm/highbd_convolve_scale_neon.c"
+ "${AOM_ROOT}/av1/common/arm/highbd_reconinter_neon.c"
+ "${AOM_ROOT}/av1/common/arm/highbd_reconintra_neon.c"
+ "${AOM_ROOT}/av1/common/arm/highbd_warp_plane_neon.c"
+ "${AOM_ROOT}/av1/common/arm/highbd_wiener_convolve_neon.c")
list(APPEND AOM_AV1_ENCODER_INTRIN_SSE2
"${AOM_ROOT}/av1/encoder/x86/highbd_block_error_intrin_sse2.c"
@@ -459,6 +482,11 @@
"${AOM_ROOT}/av1/encoder/x86/av1_highbd_quantize_avx2.c"
"${AOM_ROOT}/av1/encoder/x86/highbd_block_error_intrin_avx2.c"
"${AOM_ROOT}/av1/encoder/x86/highbd_temporal_filter_avx2.c")
+
+ list(APPEND AOM_AV1_ENCODER_INTRIN_NEON
+ "${AOM_ROOT}/av1/encoder/arm/neon/highbd_pickrst_neon.c"
+ "${AOM_ROOT}/av1/encoder/arm/neon/highbd_rdopt_neon.c"
+ "${AOM_ROOT}/av1/encoder/arm/neon/highbd_temporal_filter_neon.c")
endif()
if(CONFIG_ACCOUNTING)
@@ -623,31 +651,45 @@
endif()
if(HAVE_NEON)
- if(AOM_AV1_COMMON_INTRIN_NEON)
- add_intrinsics_object_library("${AOM_NEON_INTRIN_FLAG}" "neon"
- "aom_av1_common"
- "AOM_AV1_COMMON_INTRIN_NEON")
- endif()
-
+ add_intrinsics_object_library("${AOM_NEON_INTRIN_FLAG}" "neon"
+ "aom_av1_common" "AOM_AV1_COMMON_INTRIN_NEON")
if(CONFIG_AV1_ENCODER)
- if(AOM_AV1_ENCODER_INTRIN_NEON)
- add_intrinsics_object_library("${AOM_NEON_INTRIN_FLAG}" "neon"
- "aom_av1_encoder"
- "AOM_AV1_ENCODER_INTRIN_NEON")
- endif()
+ add_intrinsics_object_library("${AOM_NEON_INTRIN_FLAG}" "neon"
+ "aom_av1_encoder"
+ "AOM_AV1_ENCODER_INTRIN_NEON")
endif()
+ endif()
- if(HAVE_ARM_CRC32)
- if(CONFIG_AV1_ENCODER)
- if(AOM_AV1_ENCODER_INTRIN_ARM_CRC32)
- add_intrinsics_object_library("${AOM_ARM_CRC32_FLAG}" "crc32"
- "aom_av1_encoder"
- "AOM_AV1_ENCODER_INTRIN_ARM_CRC32")
- endif()
- endif()
+ if(HAVE_ARM_CRC32)
+ if(CONFIG_AV1_ENCODER)
+ add_intrinsics_object_library("${AOM_ARM_CRC32_FLAG}" "arm_crc32"
+ "aom_av1_encoder"
+ "AOM_AV1_ENCODER_INTRIN_ARM_CRC32")
endif()
endif()
+ if(HAVE_NEON_DOTPROD)
+ add_intrinsics_object_library("${AOM_NEON_DOTPROD_FLAG}" "neon_dotprod"
+ "aom_av1_common"
+ "AOM_AV1_COMMON_INTRIN_NEON_DOTPROD")
+ if(CONFIG_AV1_ENCODER)
+ add_intrinsics_object_library("${AOM_NEON_DOTPROD_FLAG}" "neon_dotprod"
+ "aom_av1_encoder"
+ "AOM_AV1_ENCODER_INTRIN_NEON_DOTPROD")
+ endif()
+ endif()
+
+ if(HAVE_NEON_I8MM)
+ add_intrinsics_object_library("${AOM_NEON_I8MM_FLAG}" "neon_i8mm"
+ "aom_av1_common"
+ "AOM_AV1_COMMON_INTRIN_NEON_I8MM")
+ endif()
+
+ if(HAVE_SVE)
+ add_intrinsics_object_library("${AOM_SVE_FLAG}" "sve" "aom_av1_common"
+ "AOM_AV1_COMMON_INTRIN_SVE")
+ endif()
+
if(HAVE_VSX)
if(AOM_AV1_COMMON_INTRIN_VSX)
add_intrinsics_object_library("-mvsx -maltivec" "vsx" "aom_av1_common"
diff --git a/av1/av1_cx_iface.c b/av1/av1_cx_iface.c
index 182a098..3c8a39b 100644
--- a/av1/av1_cx_iface.c
+++ b/av1/av1_cx_iface.c
@@ -8,6 +8,7 @@
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
+#include <limits.h>
#include <stdlib.h>
#include <string.h>
@@ -543,6 +544,7 @@
// Number of stats buffers required for look ahead
int num_lap_buffers;
STATS_BUFFER_CTX stats_buf_context;
+ bool monochrome_on_init;
};
static INLINE int gcd(int64_t a, int b) {
@@ -644,6 +646,18 @@
if (cfg->g_forced_max_frame_height) {
RANGE_CHECK_HI(cfg, g_h, cfg->g_forced_max_frame_height);
}
+ // To avoid integer overflows when multiplying width by height (or values
+ // derived from width and height) using the int type, impose a maximum frame
+ // area (width * height) of 2^30.
+ const unsigned int max_frame_width =
+ cfg->g_forced_max_frame_width ? cfg->g_forced_max_frame_width : cfg->g_w;
+ const unsigned int max_frame_height = cfg->g_forced_max_frame_height
+ ? cfg->g_forced_max_frame_height
+ : cfg->g_h;
+ const int64_t max_frame_area = (int64_t)max_frame_width * max_frame_height;
+ if (max_frame_area > (1 << 30)) {
+ ERROR("max_frame_area out of range [..2^30]");
+ }
RANGE_CHECK(cfg, g_timebase.den, 1, 1000000000);
RANGE_CHECK(cfg, g_timebase.num, 1, cfg->g_timebase.den);
RANGE_CHECK_HI(cfg, g_profile, MAX_PROFILES - 1);
@@ -668,11 +682,7 @@
RANGE_CHECK(cfg, kf_mode, AOM_KF_DISABLED, AOM_KF_AUTO);
RANGE_CHECK_HI(cfg, rc_dropframe_thresh, 100);
RANGE_CHECK(cfg, g_pass, AOM_RC_ONE_PASS, AOM_RC_THIRD_PASS);
- if (cfg->g_pass == AOM_RC_ONE_PASS) {
- RANGE_CHECK_HI(cfg, g_lag_in_frames, MAX_TOTAL_BUFFERS);
- } else {
- RANGE_CHECK_HI(cfg, g_lag_in_frames, MAX_LAG_BUFFERS);
- }
+ RANGE_CHECK_HI(cfg, g_lag_in_frames, MAX_LAG_BUFFERS);
if (cfg->g_usage == AOM_USAGE_ALL_INTRA) {
RANGE_CHECK_HI(cfg, g_lag_in_frames, 0);
RANGE_CHECK_HI(cfg, kf_max_dist, 0);
@@ -1505,6 +1515,12 @@
force_key = 1;
}
+ if (ctx->monochrome_on_init && cfg->monochrome == 0) {
+ // TODO(aomedia:3465): Allow this case to work without requiring re-init
+ // of encoder.
+ ERROR("Cannot change to monochrome = 0 after init with monochrome");
+ }
+
// Prevent increasing lag_in_frames. This check is stricter than it needs
// to be -- the limit is not increasing past the first lag_in_frames
// value, but we don't track the initial config, only the last successful
@@ -1585,11 +1601,26 @@
bool is_sb_size_changed = false;
av1_change_config_seq(ctx->ppi, &ctx->oxcf, &is_sb_size_changed);
for (int i = 0; i < ctx->ppi->num_fp_contexts; i++) {
- av1_change_config(ctx->ppi->parallel_cpi[i], &ctx->oxcf,
- is_sb_size_changed);
+ AV1_COMP *const cpi = ctx->ppi->parallel_cpi[i];
+ struct aom_internal_error_info *const error = cpi->common.error;
+ if (setjmp(error->jmp)) {
+ error->setjmp = 0;
+ return error->error_code;
+ }
+ error->setjmp = 1;
+ av1_change_config(cpi, &ctx->oxcf, is_sb_size_changed);
+ error->setjmp = 0;
}
if (ctx->ppi->cpi_lap != NULL) {
- av1_change_config(ctx->ppi->cpi_lap, &ctx->oxcf, is_sb_size_changed);
+ AV1_COMP *const cpi_lap = ctx->ppi->cpi_lap;
+ struct aom_internal_error_info *const error = cpi_lap->common.error;
+ if (setjmp(error->jmp)) {
+ error->setjmp = 0;
+ return error->error_code;
+ }
+ error->setjmp = 1;
+ av1_change_config(cpi_lap, &ctx->oxcf, is_sb_size_changed);
+ error->setjmp = 0;
}
}
return res;
@@ -2569,6 +2600,17 @@
return AOM_CODEC_OK;
}
+static aom_codec_err_t ctrl_set_max_consec_frame_drop_cbr(
+ aom_codec_alg_priv_t *ctx, va_list args) {
+ AV1_PRIMARY *const ppi = ctx->ppi;
+ AV1_COMP *const cpi = ppi->cpi;
+ const int max_consec_drop = CAST(AV1E_SET_MAX_CONSEC_FRAME_DROP_CBR, args);
+ if (max_consec_drop < 0) return AOM_CODEC_INVALID_PARAM;
+ cpi->rc.max_consec_drop = max_consec_drop;
+ cpi->rc.drop_count_consec = 0;
+ return AOM_CODEC_OK;
+}
+
#if !CONFIG_REALTIME_ONLY
aom_codec_err_t av1_create_stats_buffer(FIRSTPASS_STATS **frame_stats_buffer,
STATS_BUFFER_CTX *stats_buf_context,
@@ -2722,6 +2764,8 @@
priv->oxcf.use_highbitdepth =
(ctx->init_flags & AOM_CODEC_USE_HIGHBITDEPTH) ? 1 : 0;
+ priv->monochrome_on_init = priv->cfg.monochrome;
+
priv->ppi = av1_create_primary_compressor(&priv->pkt_list.head,
*num_lap_buffers, &priv->oxcf);
if (!priv->ppi) return AOM_CODEC_MEM_ERROR;
@@ -3144,7 +3188,8 @@
const int status = av1_get_compressed_data(cpi_lap, &cpi_lap_data);
if (status != -1) {
if (status != AOM_CODEC_OK) {
- aom_internal_error(&ppi->error, AOM_CODEC_ERROR, NULL);
+ aom_internal_error(&ppi->error, cpi->common.error->error_code, "%s",
+ cpi->common.error->detail);
}
}
av1_post_encode_updates(cpi_lap, &cpi_lap_data);
@@ -3157,12 +3202,6 @@
ppi->num_fp_contexts = av1_compute_num_fp_contexts(ppi, &cpi->oxcf);
}
- // Reset gf_frame_index in case it reaches MAX_STATIC_GF_GROUP_LENGTH for
- // real time encoding.
- if (is_one_pass_rt_params(cpi) &&
- cpi->gf_frame_index == MAX_STATIC_GF_GROUP_LENGTH)
- cpi->gf_frame_index = 0;
-
// Get the next visible frame. Invisible frames get packed with the next
// visible frame.
while (cpi_data.cx_data_sz >= ctx->cx_data_sz / 2 && !is_frame_visible) {
@@ -3200,7 +3239,8 @@
}
if (status == -1) break;
if (status != AOM_CODEC_OK) {
- aom_internal_error(&ppi->error, AOM_CODEC_ERROR, NULL);
+ aom_internal_error(&ppi->error, cpi->common.error->error_code, "%s",
+ cpi->common.error->detail);
}
if (ppi->num_fp_contexts > 0 && frame_is_intra_only(&cpi->common)) {
av1_init_sc_decisions(ppi);
@@ -3557,7 +3597,12 @@
lc->min_q = params->min_quantizers[layer];
lc->scaling_factor_num = params->scaling_factor_num[sl];
lc->scaling_factor_den = params->scaling_factor_den[sl];
- lc->layer_target_bitrate = 1000 * params->layer_target_bitrate[layer];
+ const int layer_target_bitrate = params->layer_target_bitrate[layer];
+ if (layer_target_bitrate > INT_MAX / 1000) {
+ lc->layer_target_bitrate = INT_MAX;
+ } else {
+ lc->layer_target_bitrate = 1000 * layer_target_bitrate;
+ }
lc->framerate_factor = params->framerate_factor[tl];
if (tl == ppi->number_temporal_layers - 1)
target_bandwidth += lc->layer_target_bitrate;
@@ -4344,6 +4389,7 @@
{ AV1E_SET_RTC_EXTERNAL_RC, ctrl_set_rtc_external_rc },
{ AV1E_SET_QUANTIZER_ONE_PASS, ctrl_set_quantizer_one_pass },
{ AV1E_SET_BITRATE_ONE_PASS_CBR, ctrl_set_bitrate_one_pass_cbr },
+ { AV1E_SET_MAX_CONSEC_FRAME_DROP_CBR, ctrl_set_max_consec_frame_drop_cbr },
// Getters
{ AOME_GET_LAST_QUANTIZER, ctrl_get_quantizer },
diff --git a/av1/av1_dx_iface.c b/av1/av1_dx_iface.c
index a1e7558..29c63e2 100644
--- a/av1/av1_dx_iface.c
+++ b/av1/av1_dx_iface.c
@@ -144,6 +144,7 @@
aom_free(ctx->frame_worker);
aom_free(ctx->buffer_pool);
+ assert(!ctx->img.self_allocd);
aom_img_free(&ctx->img);
aom_free(ctx);
return AOM_CODEC_OK;
diff --git a/av1/common/alloccommon.c b/av1/common/alloccommon.c
index 6e95f70..5e6ffc9 100644
--- a/av1/common/alloccommon.c
+++ b/av1/common/alloccommon.c
@@ -288,11 +288,9 @@
cdef_info->allocated_mi_rows);
}
-// Assumes cm->rst_info[p].restoration_unit_size is already initialized
+// Allocate buffers which are independent of restoration_unit_size
void av1_alloc_restoration_buffers(AV1_COMMON *cm, bool is_sgr_enabled) {
const int num_planes = av1_num_planes(cm);
- for (int p = 0; p < num_planes; ++p)
- av1_alloc_restoration_struct(cm, &cm->rst_info[p], p > 0);
if (cm->rst_tmpbuf == NULL && is_sgr_enabled) {
CHECK_MEM_ERROR(cm, cm->rst_tmpbuf,
@@ -303,21 +301,13 @@
CHECK_MEM_ERROR(cm, cm->rlbs, aom_malloc(sizeof(RestorationLineBuffers)));
}
- // For striped loop restoration, we divide each row of tiles into "stripes",
+ // For striped loop restoration, we divide each plane into "stripes",
// of height 64 luma pixels but with an offset by RESTORATION_UNIT_OFFSET
// luma pixels to match the output from CDEF. We will need to store 2 *
- // RESTORATION_CTX_VERT lines of data for each stripe, and also need to be
- // able to quickly answer the question "Where is the <n>'th stripe for tile
- // row <m>?" To make that efficient, we generate the rst_last_stripe array.
- int num_stripes = 0;
- for (int i = 0; i < cm->tiles.rows; ++i) {
- TileInfo tile_info;
- av1_tile_set_row(&tile_info, cm, i);
- const int mi_h = tile_info.mi_row_end - tile_info.mi_row_start;
- const int ext_h = RESTORATION_UNIT_OFFSET + (mi_h << MI_SIZE_LOG2);
- const int tile_stripes = (ext_h + 63) / 64;
- num_stripes += tile_stripes;
- }
+ // RESTORATION_CTX_VERT lines of data for each stripe.
+ int mi_h = cm->mi_params.mi_rows;
+ const int ext_h = RESTORATION_UNIT_OFFSET + (mi_h << MI_SIZE_LOG2);
+ const int num_stripes = (ext_h + 63) / 64;
// Now we need to allocate enough space to store the line buffers for the
// stripes
diff --git a/av1/common/arm/av1_inv_txfm_neon.c b/av1/common/arm/av1_inv_txfm_neon.c
index 8afcd1f..09e5166 100644
--- a/av1/common/arm/av1_inv_txfm_neon.c
+++ b/av1/common/arm/av1_inv_txfm_neon.c
@@ -3606,7 +3606,7 @@
identity_txfm_round_neon(cur_a, cur_a, txw_idx, buf_size_nonzero_w,
-shift[0]);
for (int j = 0; j < buf_size_w_div8; ++j) {
- transpose_s16_8x8q(&cur_a[j * 8], &b[temp_b + txfm_size_row * j]);
+ transpose_arrays_s16_8x8(&cur_a[j * 8], &b[temp_b + txfm_size_row * j]);
}
temp_b += 8;
}
@@ -3665,14 +3665,14 @@
if (lr_flip == 1) {
for (int j = 0; j < buf_size_w_div8; ++j) {
flip_buf_ud_neon(&cur_a[j * 8], 8);
- transpose_s16_8x8q(
+ transpose_arrays_s16_8x8(
&cur_a[j * 8],
&b[temp_b + txfm_size_row * (buf_size_w_div8 - 1 - j)]);
}
temp_b += 8;
} else {
for (int j = 0; j < buf_size_w_div8; ++j) {
- transpose_s16_8x8q(&cur_a[j * 8], &b[temp_b + txfm_size_row * j]);
+ transpose_arrays_s16_8x8(&cur_a[j * 8], &b[temp_b + txfm_size_row * j]);
}
temp_b += 8;
}
@@ -3730,7 +3730,7 @@
identity_txfm_round_neon(cur_a, cur_a, txw_idx, buf_size_nonzero_w,
-shift[0]);
for (int j = 0; j < buf_size_w_div8; ++j) {
- transpose_s16_8x8q(&cur_a[j * 8], &b[temp_b + txfm_size_row * j]);
+ transpose_arrays_s16_8x8(&cur_a[j * 8], &b[temp_b + txfm_size_row * j]);
}
temp_b += 8;
}
@@ -3768,7 +3768,7 @@
int32_t *buf = temp_out + buf_offset;
int32_t *buf_ptr = buf;
const int8_t stage_range[MAX_TXFM_STAGE_NUM] = { 16, 16, 16, 16, 16, 16, 16 };
- int r, bd = 8;
+ int r;
const transform_1d_neon row_txfm =
lowbd_txfm_all_1d_arr[txw_idx][hitx_1d_tab[tx_type]];
const transform_1d_neon col_txfm =
@@ -3795,20 +3795,20 @@
for (r = 0; r < txfm_size_row; ++r)
temp_in[r] = buf[r * txfm_size_col + (txfm_size_col - c - 1)];
}
- clamp_buf(temp_in, txfm_size_row, bd + 8);
+ clamp_buf(temp_in, txfm_size_row, 16);
col_txfm(temp_in, temp_out, INV_COS_BIT, stage_range);
av1_round_shift_array(temp_out, txfm_size_row, -shift[1]);
if (ud_flip == 0) {
for (r = 0; r < txfm_size_row; ++r) {
output[r * stride + c] =
- highbd_clip_pixel_add(output[r * stride + c], temp_out[r], bd);
+ clip_pixel(output[r * stride + c] + temp_out[r]);
}
} else {
// flip upside down
for (r = 0; r < txfm_size_row; ++r) {
- output[r * stride + c] = highbd_clip_pixel_add(
- output[r * stride + c], temp_out[txfm_size_row - r - 1], bd);
+ output[r * stride + c] = clip_pixel(output[r * stride + c] +
+ temp_out[txfm_size_row - r - 1]);
}
}
}
@@ -3832,7 +3832,7 @@
int32_t *buf_ptr = buf;
const int8_t stage_range[MAX_TXFM_STAGE_NUM] = { 16, 16, 16, 16,
16, 16, 16, 16 };
- int r, bd = 8;
+ int r;
const transform_1d_neon row_txfm =
lowbd_txfm_all_1d_arr[txw_idx][hitx_1d_tab[tx_type]];
const transform_1d_neon col_txfm =
@@ -3860,20 +3860,20 @@
for (r = 0; r < txfm_size_row; ++r)
temp_in[r] = buf[r * txfm_size_col + (txfm_size_col - c - 1)];
}
- clamp_buf(temp_in, txfm_size_row, bd + 8);
+ clamp_buf(temp_in, txfm_size_row, 16);
col_txfm(temp_in, temp_out, INV_COS_BIT, stage_range);
av1_round_shift_array(temp_out, txfm_size_row, -shift[1]);
if (ud_flip == 0) {
for (r = 0; r < txfm_size_row; ++r) {
output[r * stride + c] =
- highbd_clip_pixel_add(output[r * stride + c], temp_out[r], bd);
+ clip_pixel(output[r * stride + c] + temp_out[r]);
}
} else {
// flip upside down
for (r = 0; r < txfm_size_row; ++r) {
- output[r * stride + c] = highbd_clip_pixel_add(
- output[r * stride + c], temp_out[txfm_size_row - r - 1], bd);
+ output[r * stride + c] = clip_pixel(output[r * stride + c] +
+ temp_out[txfm_size_row - r - 1]);
}
}
}
@@ -3897,7 +3897,7 @@
int32_t *buf_ptr = buf;
const int8_t stage_range[MAX_TXFM_STAGE_NUM] = { 16, 16, 16, 16,
16, 16, 16, 16 };
- int r, bd = 8;
+ int r;
const transform_1d_neon row_txfm =
lowbd_txfm_all_1d_arr[txw_idx][hitx_1d_tab[tx_type]];
const transform_1d_neon col_txfm =
@@ -3925,20 +3925,20 @@
for (r = 0; r < txfm_size_row; ++r)
temp_in[r] = buf[r * txfm_size_col + (txfm_size_col - c - 1)];
}
- clamp_buf(temp_in, txfm_size_row, bd + 8);
+ clamp_buf(temp_in, txfm_size_row, 16);
col_txfm(temp_in, temp_out, INV_COS_BIT, stage_range);
av1_round_shift_array(temp_out, txfm_size_row, -shift[1]);
if (ud_flip == 0) {
for (r = 0; r < txfm_size_row; ++r) {
output[r * stride + c] =
- highbd_clip_pixel_add(output[r * stride + c], temp_out[r], bd);
+ clip_pixel(output[r * stride + c] + temp_out[r]);
}
} else {
// flip upside down
for (r = 0; r < txfm_size_row; ++r) {
- output[r * stride + c] = highbd_clip_pixel_add(
- output[r * stride + c], temp_out[txfm_size_row - r - 1], bd);
+ output[r * stride + c] = clip_pixel(output[r * stride + c] +
+ temp_out[txfm_size_row - r - 1]);
}
}
}
@@ -3962,7 +3962,7 @@
int32_t *buf_ptr = buf;
const int8_t stage_range[MAX_TXFM_STAGE_NUM] = { 16, 16, 16, 16, 16,
16, 16, 16, 16, 16 };
- int r, bd = 8;
+ int r;
const transform_1d_neon row_txfm =
lowbd_txfm_all_1d_arr[txw_idx][hitx_1d_tab[tx_type]];
const transform_1d_neon col_txfm =
@@ -3989,20 +3989,20 @@
for (r = 0; r < txfm_size_row; ++r)
temp_in[r] = buf[r * txfm_size_col + (txfm_size_col - c - 1)];
}
- clamp_buf(temp_in, txfm_size_row, bd + 8);
+ clamp_buf(temp_in, txfm_size_row, 16);
col_txfm(temp_in, temp_out, INV_COS_BIT, stage_range);
av1_round_shift_array(temp_out, txfm_size_row, -shift[1]);
if (ud_flip == 0) {
for (r = 0; r < txfm_size_row; ++r) {
output[r * stride + c] =
- highbd_clip_pixel_add(output[r * stride + c], temp_out[r], bd);
+ clip_pixel(output[r * stride + c] + temp_out[r]);
}
} else {
// flip upside down
for (r = 0; r < txfm_size_row; ++r) {
- output[r * stride + c] = highbd_clip_pixel_add(
- output[r * stride + c], temp_out[txfm_size_row - r - 1], bd);
+ output[r * stride + c] = clip_pixel(output[r * stride + c] +
+ temp_out[txfm_size_row - r - 1]);
}
}
}
@@ -4026,7 +4026,7 @@
int32_t *buf_ptr = buf;
const int8_t stage_range[MAX_TXFM_STAGE_NUM] = { 16, 16, 16, 16, 16,
16, 16, 16, 16, 16 };
- int r, bd = 8;
+ int r;
const transform_1d_neon row_txfm =
lowbd_txfm_all_1d_arr[txw_idx][hitx_1d_tab[tx_type]];
const transform_1d_neon col_txfm =
@@ -4053,20 +4053,20 @@
for (r = 0; r < txfm_size_row; ++r)
temp_in[r] = buf[r * txfm_size_col + (txfm_size_col - c - 1)];
}
- clamp_buf(temp_in, txfm_size_row, bd + 8);
+ clamp_buf(temp_in, txfm_size_row, 16);
col_txfm(temp_in, temp_out, INV_COS_BIT, stage_range);
av1_round_shift_array(temp_out, txfm_size_row, -shift[1]);
if (ud_flip == 0) {
for (r = 0; r < txfm_size_row; ++r) {
output[r * stride + c] =
- highbd_clip_pixel_add(output[r * stride + c], temp_out[r], bd);
+ clip_pixel(output[r * stride + c] + temp_out[r]);
}
} else {
// flip upside down
for (r = 0; r < txfm_size_row; ++r) {
- output[r * stride + c] = highbd_clip_pixel_add(
- output[r * stride + c], temp_out[txfm_size_row - r - 1], bd);
+ output[r * stride + c] = clip_pixel(output[r * stride + c] +
+ temp_out[txfm_size_row - r - 1]);
}
}
}
@@ -4116,14 +4116,14 @@
if (lr_flip == 1) {
for (int j = 0; j < buf_size_w_div8; ++j) {
flip_buf_ud_neon(&cur_a[j * 8], 8);
- transpose_s16_8x8q(
+ transpose_arrays_s16_8x8(
&cur_a[j * 8],
&b[temp_b + txfm_size_row * (buf_size_w_div8 - 1 - j)]);
}
temp_b += 8;
} else {
for (int j = 0; j < buf_size_w_div8; ++j) {
- transpose_s16_8x8q(&cur_a[j * 8], &b[temp_b + txfm_size_row * j]);
+ transpose_arrays_s16_8x8(&cur_a[j * 8], &b[temp_b + txfm_size_row * j]);
}
temp_b += 8;
}
diff --git a/av1/common/arm/blend_a64_hmask_neon.c b/av1/common/arm/blend_a64_hmask_neon.c
index baad328..22d2977 100644
--- a/av1/common/arm/blend_a64_hmask_neon.c
+++ b/av1/common/arm/blend_a64_hmask_neon.c
@@ -13,12 +13,12 @@
#include <arm_neon.h>
#include <assert.h>
+#include "config/aom_dsp_rtcd.h"
+
#include "aom/aom_integer.h"
#include "aom_dsp/aom_dsp_common.h"
-#include "aom_dsp/blend.h"
+#include "aom_dsp/arm/blend_neon.h"
#include "aom_dsp/arm/mem_neon.h"
-#include "aom_ports/mem.h"
-#include "config/aom_dsp_rtcd.h"
void aom_blend_a64_hmask_neon(uint8_t *dst, uint32_t dst_stride,
const uint8_t *src0, uint32_t src0_stride,
@@ -31,94 +31,72 @@
assert(w >= 2);
assert(IS_POWER_OF_TWO(h));
assert(IS_POWER_OF_TWO(w));
- uint8x8_t tmp0, tmp1;
- uint8x16_t res_q;
- uint16x8_t res, res_low, res_high;
- const uint8x8_t vdup_64 = vdup_n_u8((uint8_t)64);
- if (w >= 16) {
- const uint8x16_t vdup_64_q = vdupq_n_u8((uint8_t)64);
- for (int i = 0; i < h; ++i) {
- for (int j = 0; j < w; j += 16) {
- __builtin_prefetch(src0);
- __builtin_prefetch(src1);
- const uint8x16_t tmp0_q = vld1q_u8(src0);
- const uint8x16_t tmp1_q = vld1q_u8(src1);
- const uint8x16_t m_q = vld1q_u8(mask);
- const uint8x16_t max_minus_m_q = vsubq_u8(vdup_64_q, m_q);
- res_low = vmull_u8(vget_low_u8(m_q), vget_low_u8(tmp0_q));
- res_low =
- vmlal_u8(res_low, vget_low_u8(max_minus_m_q), vget_low_u8(tmp1_q));
- res_high = vmull_u8(vget_high_u8(m_q), vget_high_u8(tmp0_q));
- res_high = vmlal_u8(res_high, vget_high_u8(max_minus_m_q),
- vget_high_u8(tmp1_q));
- res_q = vcombine_u8(vrshrn_n_u16(res_low, AOM_BLEND_A64_ROUND_BITS),
- vrshrn_n_u16(res_high, AOM_BLEND_A64_ROUND_BITS));
- vst1q_u8(dst, res_q);
- src0 += 16;
- src1 += 16;
- dst += 16;
- mask += 16;
- }
- src0 += src0_stride - w;
- src1 += src1_stride - w;
- dst += dst_stride - w;
- mask -= w;
- }
- } else if (w == 8) {
- const uint8x8_t m = vld1_u8(mask);
- const uint8x8_t max_minus_m = vsub_u8(vdup_64, m);
- for (int i = 0; i < h; ++i) {
- __builtin_prefetch(src0);
- __builtin_prefetch(src1);
- tmp0 = vld1_u8(src0);
- tmp1 = vld1_u8(src1);
- res = vmull_u8(m, tmp0);
- res = vmlal_u8(res, max_minus_m, tmp1);
- vst1_u8(dst, vrshrn_n_u16(res, AOM_BLEND_A64_ROUND_BITS));
+ if (w > 8) {
+ do {
+ int i = 0;
+ do {
+ uint8x16_t m0 = vld1q_u8(mask + i);
+ uint8x16_t s0 = vld1q_u8(src0 + i);
+ uint8x16_t s1 = vld1q_u8(src1 + i);
+
+ uint8x16_t blend = alpha_blend_a64_u8x16(m0, s0, s1);
+
+ vst1q_u8(dst + i, blend);
+
+ i += 16;
+ } while (i < w);
+
src0 += src0_stride;
src1 += src1_stride;
dst += dst_stride;
- }
+ } while (--h != 0);
+ } else if (w == 8) {
+ const uint8x8_t m0 = vld1_u8(mask);
+ do {
+ uint8x8_t s0 = vld1_u8(src0);
+ uint8x8_t s1 = vld1_u8(src1);
+
+ uint8x8_t blend = alpha_blend_a64_u8x8(m0, s0, s1);
+
+ vst1_u8(dst, blend);
+
+ src0 += src0_stride;
+ src1 += src1_stride;
+ dst += dst_stride;
+ } while (--h != 0);
} else if (w == 4) {
- assert(((uintptr_t)mask & 3) == 0);
- const uint8x8_t m = vreinterpret_u8_u32(vld1_dup_u32((uint32_t *)mask));
- const uint8x8_t max_minus_m = vsub_u8(vdup_64, m);
- for (int i = 0; i < h; i += 2) {
- __builtin_prefetch(src0 + 0 * src0_stride);
- __builtin_prefetch(src0 + 1 * src0_stride);
- __builtin_prefetch(src1 + 0 * src1_stride);
- __builtin_prefetch(src1 + 1 * src1_stride);
- tmp0 = load_unaligned_u8_4x2(src0, src0_stride);
- tmp1 = load_unaligned_u8_4x2(src1, src1_stride);
- res = vmull_u8(m, tmp0);
- res = vmlal_u8(res, max_minus_m, tmp1);
- const uint8x8_t result = vrshrn_n_u16(res, AOM_BLEND_A64_ROUND_BITS);
- store_unaligned_u8_4x1(dst + 0 * dst_stride, result, 0);
- store_unaligned_u8_4x1(dst + 1 * dst_stride, result, 1);
- src0 += (2 * src0_stride);
- src1 += (2 * src1_stride);
- dst += (2 * dst_stride);
- }
- } else if (w == 2) {
- assert(((uintptr_t)mask & 1) == 0);
- const uint8x8_t m = vreinterpret_u8_u16(vld1_dup_u16((uint16_t *)mask));
- const uint8x8_t max_minus_m = vsub_u8(vdup_64, m);
- for (int i = 0; i < h; i += 2) {
- __builtin_prefetch(src0 + 0 * src0_stride);
- __builtin_prefetch(src0 + 1 * src0_stride);
- __builtin_prefetch(src1 + 0 * src1_stride);
- __builtin_prefetch(src1 + 1 * src1_stride);
- tmp0 = load_unaligned_u8_2x2(src0, src0_stride);
- tmp1 = load_unaligned_u8_2x2(src1, src1_stride);
- res = vmull_u8(m, tmp0);
- res = vmlal_u8(res, max_minus_m, tmp1);
- const uint8x8_t result = vrshrn_n_u16(res, AOM_BLEND_A64_ROUND_BITS);
- store_unaligned_u8_2x1(dst + 0 * dst_stride, result, 0);
- store_unaligned_u8_2x1(dst + 1 * dst_stride, result, 1);
- src0 += (2 * src0_stride);
- src1 += (2 * src1_stride);
- dst += (2 * dst_stride);
- }
+ const uint8x8_t m0 = load_unaligned_dup_u8_4x2(mask);
+ do {
+ uint8x8_t s0 = load_unaligned_u8_4x2(src0, src0_stride);
+ uint8x8_t s1 = load_unaligned_u8_4x2(src1, src1_stride);
+
+ uint8x8_t blend = alpha_blend_a64_u8x8(m0, s0, s1);
+
+ store_unaligned_u8_4x2(dst, dst_stride, blend);
+
+ src0 += 2 * src0_stride;
+ src1 += 2 * src1_stride;
+ dst += 2 * dst_stride;
+ h -= 2;
+ } while (h != 0);
+ } else if (w == 2 && h >= 16) {
+ const uint8x8_t m0 = vreinterpret_u8_u16(vld1_dup_u16((uint16_t *)mask));
+ do {
+ uint8x8_t s0 = load_unaligned_u8_2x2(src0, src0_stride);
+ uint8x8_t s1 = load_unaligned_u8_2x2(src1, src1_stride);
+
+ uint8x8_t blend = alpha_blend_a64_u8x8(m0, s0, s1);
+
+ store_unaligned_u8_2x2(dst, dst_stride, blend);
+
+ src0 += 2 * src0_stride;
+ src1 += 2 * src1_stride;
+ dst += 2 * dst_stride;
+ h -= 2;
+ } while (h != 0);
+ } else {
+ aom_blend_a64_hmask_c(dst, dst_stride, src0, src0_stride, src1, src1_stride,
+ mask, w, h);
}
}
diff --git a/av1/common/arm/blend_a64_vmask_neon.c b/av1/common/arm/blend_a64_vmask_neon.c
index c316977..d53d363 100644
--- a/av1/common/arm/blend_a64_vmask_neon.c
+++ b/av1/common/arm/blend_a64_vmask_neon.c
@@ -16,6 +16,7 @@
#include "aom/aom_integer.h"
#include "aom_dsp/aom_dsp_common.h"
#include "aom_dsp/blend.h"
+#include "aom_dsp/arm/blend_neon.h"
#include "aom_dsp/arm/mem_neon.h"
#include "aom_ports/mem.h"
#include "config/aom_dsp_rtcd.h"
@@ -24,9 +25,6 @@
const uint8_t *src0, uint32_t src0_stride,
const uint8_t *src1, uint32_t src1_stride,
const uint8_t *mask, int w, int h) {
- uint8x8_t tmp0, tmp1;
- uint8x16_t tmp0_q, tmp1_q, res_q;
- uint16x8_t res, res_low, res_high;
assert(IMPLIES(src0 == dst, src0_stride == dst_stride));
assert(IMPLIES(src1 == dst, src1_stride == dst_stride));
@@ -35,95 +33,80 @@
assert(IS_POWER_OF_TWO(h));
assert(IS_POWER_OF_TWO(w));
- if (w >= 16) {
- for (int i = 0; i < h; ++i) {
- const uint8x8_t m = vdup_n_u8((uint8_t)mask[i]);
- const uint8x8_t max_minus_m = vdup_n_u8(64 - (uint8_t)mask[i]);
- for (int j = 0; j < w; j += 16) {
- __builtin_prefetch(src0);
- __builtin_prefetch(src1);
- tmp0_q = vld1q_u8(src0);
- tmp1_q = vld1q_u8(src1);
- res_low = vmull_u8(m, vget_low_u8(tmp0_q));
- res_low = vmlal_u8(res_low, max_minus_m, vget_low_u8(tmp1_q));
- res_high = vmull_u8(m, vget_high_u8(tmp0_q));
- res_high = vmlal_u8(res_high, max_minus_m, vget_high_u8(tmp1_q));
- res_q = vcombine_u8(vrshrn_n_u16(res_low, AOM_BLEND_A64_ROUND_BITS),
- vrshrn_n_u16(res_high, AOM_BLEND_A64_ROUND_BITS));
- vst1q_u8(dst, res_q);
- src0 += 16;
- src1 += 16;
- dst += 16;
- }
- src0 += src0_stride - w;
- src1 += src1_stride - w;
- dst += dst_stride - w;
- }
- } else if (w == 8) {
- for (int i = 0; i < h; ++i) {
- __builtin_prefetch(src0);
- __builtin_prefetch(src1);
- const uint8x8_t m = vdup_n_u8((uint8_t)mask[i]);
- const uint8x8_t max_minus_m = vdup_n_u8(64 - (uint8_t)mask[i]);
- tmp0 = vld1_u8(src0);
- tmp1 = vld1_u8(src1);
- res = vmull_u8(m, tmp0);
- res = vmlal_u8(res, max_minus_m, tmp1);
- vst1_u8(dst, vrshrn_n_u16(res, AOM_BLEND_A64_ROUND_BITS));
+ if (w > 8) {
+ do {
+ uint8x16_t m0 = vdupq_n_u8(mask[0]);
+ int i = 0;
+ do {
+ uint8x16_t s0 = vld1q_u8(src0 + i);
+ uint8x16_t s1 = vld1q_u8(src1 + i);
+
+ uint8x16_t blend = alpha_blend_a64_u8x16(m0, s0, s1);
+
+ vst1q_u8(dst + i, blend);
+
+ i += 16;
+ } while (i < w);
+
+ mask += 1;
src0 += src0_stride;
src1 += src1_stride;
dst += dst_stride;
- }
+ } while (--h != 0);
+ } else if (w == 8) {
+ do {
+ uint8x8_t m0 = vdup_n_u8(mask[0]);
+ uint8x8_t s0 = vld1_u8(src0);
+ uint8x8_t s1 = vld1_u8(src1);
+
+ uint8x8_t blend = alpha_blend_a64_u8x8(m0, s0, s1);
+
+ vst1_u8(dst, blend);
+
+ mask += 1;
+ src0 += src0_stride;
+ src1 += src1_stride;
+ dst += dst_stride;
+ } while (--h != 0);
} else if (w == 4) {
- for (int i = 0; i < h; i += 2) {
- __builtin_prefetch(src0 + 0 * src0_stride);
- __builtin_prefetch(src0 + 1 * src0_stride);
- __builtin_prefetch(src1 + 0 * src1_stride);
- __builtin_prefetch(src1 + 1 * src1_stride);
- const uint16x4_t m1 = vdup_n_u16((uint16_t)mask[i]);
- const uint16x4_t m2 = vdup_n_u16((uint16_t)mask[i + 1]);
- const uint8x8_t m = vmovn_u16(vcombine_u16(m1, m2));
- const uint16x4_t max_minus_m1 = vdup_n_u16(64 - (uint16_t)mask[i]);
- const uint16x4_t max_minus_m2 = vdup_n_u16(64 - (uint16_t)mask[i + 1]);
- const uint8x8_t max_minus_m =
- vmovn_u16(vcombine_u16(max_minus_m1, max_minus_m2));
- tmp0 = load_unaligned_u8_4x2(src0, src0_stride);
- tmp1 = load_unaligned_u8_4x2(src1, src1_stride);
- res = vmull_u8(m, tmp0);
- res = vmlal_u8(res, max_minus_m, tmp1);
- const uint8x8_t result = vrshrn_n_u16(res, AOM_BLEND_A64_ROUND_BITS);
- store_unaligned_u8_4x1(dst + 0 * dst_stride, result, 0);
- store_unaligned_u8_4x1(dst + 1 * dst_stride, result, 1);
- src0 += (2 * src0_stride);
- src1 += (2 * src1_stride);
- dst += (2 * dst_stride);
- }
- } else if (w == 2) {
- for (int i = 0; i < h; i += 2) {
- __builtin_prefetch(src0 + 0 * src0_stride);
- __builtin_prefetch(src0 + 1 * src0_stride);
- __builtin_prefetch(src1 + 0 * src1_stride);
- __builtin_prefetch(src1 + 1 * src1_stride);
- const uint8x8_t m1 = vdup_n_u8(mask[i]);
- const uint8x8_t m2 = vdup_n_u8(mask[i + 1]);
- const uint16x4x2_t m_trn =
- vtrn_u16(vreinterpret_u16_u8(m1), vreinterpret_u16_u8(m2));
- const uint8x8_t m = vreinterpret_u8_u16(m_trn.val[0]);
- const uint8x8_t max_minus_m1 = vdup_n_u8(64 - mask[i]);
- const uint8x8_t max_minus_m2 = vdup_n_u8(64 - mask[i + 1]);
- const uint16x4x2_t max_minus_m_trn = vtrn_u16(
- vreinterpret_u16_u8(max_minus_m1), vreinterpret_u16_u8(max_minus_m2));
- const uint8x8_t max_minus_m = vreinterpret_u8_u16(max_minus_m_trn.val[0]);
- tmp0 = load_unaligned_u8_2x2(src0, src0_stride);
- tmp1 = load_unaligned_u8_2x2(src1, src1_stride);
- res = vmull_u8(m, tmp0);
- res = vmlal_u8(res, max_minus_m, tmp1);
- const uint8x8_t result = vrshrn_n_u16(res, AOM_BLEND_A64_ROUND_BITS);
- store_unaligned_u8_2x1(dst + 0 * dst_stride, result, 0);
- store_unaligned_u8_2x1(dst + 1 * dst_stride, result, 1);
- src0 += (2 * src0_stride);
- src1 += (2 * src1_stride);
- dst += (2 * dst_stride);
- }
+ do {
+ const uint16x4_t m0 = vdup_n_u16((uint16_t)mask[0]);
+ const uint16x4_t m1 = vdup_n_u16((uint16_t)mask[1]);
+ const uint8x8_t m = vmovn_u16(vcombine_u16(m0, m1));
+ uint8x8_t s0 = load_unaligned_u8_4x2(src0, src0_stride);
+ uint8x8_t s1 = load_unaligned_u8_4x2(src1, src1_stride);
+
+ uint8x8_t blend = alpha_blend_a64_u8x8(m, s0, s1);
+
+ store_unaligned_u8_4x2(dst, dst_stride, blend);
+
+ mask += 2;
+ src0 += 2 * src0_stride;
+ src1 += 2 * src1_stride;
+ dst += 2 * dst_stride;
+ h -= 2;
+ } while (h != 0);
+ } else if (w == 2 && h >= 16) {
+ do {
+ uint16x4_t m0 = vdup_n_u16(0);
+ m0 = vld1_lane_u16((uint16_t *)mask, m0, 0);
+ uint8x8_t m =
+ vzip_u8(vreinterpret_u8_u16(m0), vreinterpret_u8_u16(m0)).val[0];
+ uint8x8_t s0 = load_unaligned_u8_2x2(src0, src0_stride);
+ uint8x8_t s1 = load_unaligned_u8_2x2(src1, src1_stride);
+
+ uint8x8_t blend = alpha_blend_a64_u8x8(m, s0, s1);
+
+ store_unaligned_u8_2x2(dst, dst_stride, blend);
+
+ mask += 2;
+ src0 += 2 * src0_stride;
+ src1 += 2 * src1_stride;
+ dst += 2 * dst_stride;
+ h -= 2;
+ } while (h != 0);
+ } else {
+ aom_blend_a64_vmask_c(dst, dst_stride, src0, src0_stride, src1, src1_stride,
+ mask, w, h);
}
}
diff --git a/av1/common/arm/cdef_block_neon.c b/av1/common/arm/cdef_block_neon.c
index 4397a47..1bcf6a1 100644
--- a/av1/common/arm/cdef_block_neon.c
+++ b/av1/common/arm/cdef_block_neon.c
@@ -9,23 +9,408 @@
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
-#include "aom_dsp/aom_simd.h"
-#define SIMD_FUNC(name) name##_neon
-#include "av1/common/cdef_block_simd.h"
+#include <arm_neon.h>
+#include <assert.h>
+
+#include "config/aom_config.h"
+#include "config/av1_rtcd.h"
+
+#include "aom_dsp/arm/mem_neon.h"
+#include "av1/common/cdef_block.h"
void cdef_copy_rect8_8bit_to_16bit_neon(uint16_t *dst, int dstride,
const uint8_t *src, int sstride,
int width, int height) {
- int j;
- for (int i = 0; i < height; i++) {
- for (j = 0; j < (width & ~0x7); j += 8) {
- v64 row = v64_load_unaligned(&src[i * sstride + j]);
- v128_store_unaligned(&dst[i * dstride + j], v128_unpack_u8_s16(row));
+ do {
+ const uint8_t *src_ptr = src;
+ uint16_t *dst_ptr = dst;
+
+ int w = 0;
+ while (w <= width - 16) {
+ uint8x16_t row = vld1q_u8(src_ptr + w);
+ uint8x16x2_t row_u16 = { { row, vdupq_n_u8(0) } };
+ vst2q_u8((uint8_t *)(dst_ptr + w), row_u16);
+
+ w += 16;
}
- for (; j < width; j++) {
- dst[i * dstride + j] = src[i * sstride + j];
+ if (width - w == 8) {
+ uint8x8_t row = vld1_u8(src_ptr + w);
+ vst1q_u16(dst_ptr + w, vmovl_u8(row));
+ } else if (width - w == 4) {
+ for (int i = 0; i < 4; i++) {
+ dst_ptr[i] = src_ptr[i];
+ }
}
+
+ src += sstride;
+ dst += dstride;
+ } while (--height != 0);
+}
+
+void cdef_copy_rect8_16bit_to_16bit_neon(uint16_t *dst, int dstride,
+ const uint16_t *src, int sstride,
+ int width, int height) {
+ do {
+ const uint16_t *src_ptr = src;
+ uint16_t *dst_ptr = dst;
+
+ int w = 0;
+ while (width - w >= 8) {
+ uint16x8_t row = vld1q_u16(src_ptr + w);
+ vst1q_u16(dst_ptr + w, row);
+
+ w += 8;
+ }
+ if (width == 4) {
+ uint16x4_t row = vld1_u16(src_ptr);
+ vst1_u16(dst_ptr, row);
+ }
+
+ src += sstride;
+ dst += dstride;
+ } while (--height != 0);
+}
+
+static INLINE int16x8_t v128_from_64_neon(int64_t a, int64_t b) {
+ return vreinterpretq_s16_s64(vcombine_s64(vcreate_s64(a), vcreate_s64(b)));
+}
+
+#define SHL_HIGH_NEON(n) \
+ static INLINE int16x8_t v128_shl_##n##_byte_neon(int16x8_t a) { \
+ int64x2_t a_s64 = vreinterpretq_s64_s16(a); \
+ return v128_from_64_neon( \
+ 0, vget_lane_u64(vshl_n_u64(vreinterpret_u64_s64(vget_low_s64(a_s64)), \
+ (n - 8) * 8), \
+ 0)); \
}
+
+#define SHL_NEON(n) \
+ static INLINE int16x8_t v128_shl_##n##_byte_neon(int16x8_t a) { \
+ int64x2_t a_s64 = vreinterpretq_s64_s16(a); \
+ return v128_from_64_neon( \
+ 0, vget_lane_u64(vreinterpret_u64_s64(vget_low_s64(a_s64)), 0)); \
+ }
+
+#define SHL_LOW_NEON(n) \
+ static INLINE int16x8_t v128_shl_##n##_byte_neon(int16x8_t a) { \
+ int64x2_t a_s64 = vreinterpretq_s64_s16(a); \
+ return v128_from_64_neon( \
+ vget_lane_u64( \
+ vshl_n_u64(vreinterpret_u64_s64(vget_low_s64(a_s64)), n * 8), 0), \
+ vget_lane_u64( \
+ vorr_u64( \
+ vshl_n_u64(vreinterpret_u64_s64(vget_high_s64(a_s64)), n * 8), \
+ vshr_n_u64(vreinterpret_u64_s64(vget_low_s64(a_s64)), \
+ (8 - n) * 8)), \
+ 0)); \
+ }
+
+SHL_HIGH_NEON(14)
+SHL_HIGH_NEON(12)
+SHL_HIGH_NEON(10)
+SHL_NEON(8)
+SHL_LOW_NEON(6)
+SHL_LOW_NEON(4)
+SHL_LOW_NEON(2)
+
+#define v128_shl_n_byte_neon(a, n) v128_shl_##n##_byte_neon(a)
+
+#define SHR_HIGH_NEON(n) \
+ static INLINE int16x8_t v128_shr_##n##_byte_neon(int16x8_t a) { \
+ int64x2_t a_s64 = vreinterpretq_s64_s16(a); \
+ return v128_from_64_neon( \
+ vget_lane_u64(vshr_n_u64(vreinterpret_u64_s64(vget_high_s64(a_s64)), \
+ (n - 8) * 8), \
+ 0), \
+ 0); \
+ }
+
+#define SHR_NEON(n) \
+ static INLINE int16x8_t v128_shr_##n##_byte_neon(int16x8_t a) { \
+ int64x2_t a_s64 = vreinterpretq_s64_s16(a); \
+ return v128_from_64_neon( \
+ vget_lane_u64(vreinterpret_u64_s64(vget_high_s64(a_s64)), 0), 0); \
+ }
+
+#define SHR_LOW_NEON(n) \
+ static INLINE int16x8_t v128_shr_##n##_byte_neon(int16x8_t a) { \
+ int64x2_t a_s64 = vreinterpretq_s64_s16(a); \
+ return v128_from_64_neon( \
+ vget_lane_u64( \
+ vorr_u64( \
+ vshr_n_u64(vreinterpret_u64_s64(vget_low_s64(a_s64)), n * 8), \
+ vshl_n_u64(vreinterpret_u64_s64(vget_high_s64(a_s64)), \
+ (8 - n) * 8)), \
+ 0), \
+ vget_lane_u64( \
+ vshr_n_u64(vreinterpret_u64_s64(vget_high_s64(a_s64)), n * 8), \
+ 0)); \
+ }
+
+SHR_HIGH_NEON(14)
+SHR_HIGH_NEON(12)
+SHR_HIGH_NEON(10)
+SHR_NEON(8)
+SHR_LOW_NEON(6)
+SHR_LOW_NEON(4)
+SHR_LOW_NEON(2)
+
+#define v128_shr_n_byte_neon(a, n) v128_shr_##n##_byte_neon(a)
+
+static INLINE uint32x4_t v128_madd_s16_neon(int16x8_t a, int16x8_t b) {
+ uint32x4_t t1 =
+ vreinterpretq_u32_s32(vmull_s16(vget_low_s16(a), vget_low_s16(b)));
+ uint32x4_t t2 =
+ vreinterpretq_u32_s32(vmull_s16(vget_high_s16(a), vget_high_s16(b)));
+#if AOM_ARCH_AARCH64
+ return vpaddq_u32(t1, t2);
+#else
+ return vcombine_u32(vpadd_u32(vget_low_u32(t1), vget_high_u32(t1)),
+ vpadd_u32(vget_low_u32(t2), vget_high_u32(t2)));
+#endif
+}
+
+// partial A is a 16-bit vector of the form:
+// [x8 x7 x6 x5 x4 x3 x2 x1] and partial B has the form:
+// [0 y1 y2 y3 y4 y5 y6 y7].
+// This function computes (x1^2+y1^2)*C1 + (x2^2+y2^2)*C2 + ...
+// (x7^2+y2^7)*C7 + (x8^2+0^2)*C8 where the C1..C8 constants are in const1
+// and const2.
+static INLINE uint32x4_t fold_mul_and_sum_neon(int16x8_t partiala,
+ int16x8_t partialb,
+ uint32x4_t const1,
+ uint32x4_t const2) {
+ int16x8_t tmp;
+ // Reverse partial B.
+ uint8x16_t pattern = vreinterpretq_u8_u64(
+ vcombine_u64(vcreate_u64((uint64_t)0x07060908 << 32 | 0x0b0a0d0c),
+ vcreate_u64((uint64_t)0x0f0e0100 << 32 | 0x03020504)));
+
+#if AOM_ARCH_AARCH64
+ partialb =
+ vreinterpretq_s16_s8(vqtbl1q_s8(vreinterpretq_s8_s16(partialb), pattern));
+#else
+ int8x8x2_t p = { { vget_low_s8(vreinterpretq_s8_s16(partialb)),
+ vget_high_s8(vreinterpretq_s8_s16(partialb)) } };
+ int8x8_t shuffle_hi = vtbl2_s8(p, vget_high_s8(vreinterpretq_s8_u8(pattern)));
+ int8x8_t shuffle_lo = vtbl2_s8(p, vget_low_s8(vreinterpretq_s8_u8(pattern)));
+ partialb = vreinterpretq_s16_s8(vcombine_s8(shuffle_lo, shuffle_hi));
+#endif
+
+ // Interleave the x and y values of identical indices and pair x8 with 0.
+ tmp = partiala;
+ partiala = vzipq_s16(partiala, partialb).val[0];
+ partialb = vzipq_s16(tmp, partialb).val[1];
+ // Square and add the corresponding x and y values.
+ uint32x4_t partiala_u32 = v128_madd_s16_neon(partiala, partiala);
+ uint32x4_t partialb_u32 = v128_madd_s16_neon(partialb, partialb);
+
+ // Multiply by constant.
+ partiala_u32 = vmulq_u32(partiala_u32, const1);
+ partialb_u32 = vmulq_u32(partialb_u32, const2);
+
+ // Sum all results.
+ partiala_u32 = vaddq_u32(partiala_u32, partialb_u32);
+ return partiala_u32;
+}
+
+static INLINE uint64x2_t ziplo_u64(uint32x4_t a, uint32x4_t b) {
+ return vcombine_u64(vget_low_u64(vreinterpretq_u64_u32(a)),
+ vget_low_u64(vreinterpretq_u64_u32(b)));
+}
+
+static INLINE uint64x2_t ziphi_u64(uint32x4_t a, uint32x4_t b) {
+ return vcombine_u64(vget_high_u64(vreinterpretq_u64_u32(a)),
+ vget_high_u64(vreinterpretq_u64_u32(b)));
+}
+
+static INLINE uint32x4_t hsum4_neon(uint32x4_t x0, uint32x4_t x1, uint32x4_t x2,
+ uint32x4_t x3) {
+ uint32x4_t t0, t1, t2, t3;
+ t0 = vzipq_u32(x0, x1).val[0];
+ t1 = vzipq_u32(x2, x3).val[0];
+ t2 = vzipq_u32(x0, x1).val[1];
+ t3 = vzipq_u32(x2, x3).val[1];
+ x0 = vreinterpretq_u32_u64(ziplo_u64(t0, t1));
+ x1 = vreinterpretq_u32_u64(ziphi_u64(t0, t1));
+ x2 = vreinterpretq_u32_u64(ziplo_u64(t2, t3));
+ x3 = vreinterpretq_u32_u64(ziphi_u64(t2, t3));
+ return vaddq_u32(vaddq_u32(x0, x1), vaddq_u32(x2, x3));
+}
+
+static INLINE uint32x4_t compute_directions_neon(int16x8_t lines[8],
+ uint32_t cost[4]) {
+ int16x8_t partial4a, partial4b, partial5a, partial5b, partial6, partial7a,
+ partial7b;
+ int16x8_t tmp;
+
+ // Partial sums for lines 0 and 1.
+ partial4a = v128_shl_n_byte_neon(lines[0], 14);
+ partial4b = v128_shr_n_byte_neon(lines[0], 2);
+ partial4a = vaddq_s16(partial4a, v128_shl_n_byte_neon(lines[1], 12));
+ partial4b = vaddq_s16(partial4b, v128_shr_n_byte_neon(lines[1], 4));
+ tmp = vaddq_s16(lines[0], lines[1]);
+ partial5a = v128_shl_n_byte_neon(tmp, 10);
+ partial5b = v128_shr_n_byte_neon(tmp, 6);
+ partial7a = v128_shl_n_byte_neon(tmp, 4);
+ partial7b = v128_shr_n_byte_neon(tmp, 12);
+ partial6 = tmp;
+
+ // Partial sums for lines 2 and 3.
+ partial4a = vaddq_s16(partial4a, v128_shl_n_byte_neon(lines[2], 10));
+ partial4b = vaddq_s16(partial4b, v128_shr_n_byte_neon(lines[2], 6));
+ partial4a = vaddq_s16(partial4a, v128_shl_n_byte_neon(lines[3], 8));
+ partial4b = vaddq_s16(partial4b, v128_shr_n_byte_neon(lines[3], 8));
+ tmp = vaddq_s16(lines[2], lines[3]);
+ partial5a = vaddq_s16(partial5a, v128_shl_n_byte_neon(tmp, 8));
+ partial5b = vaddq_s16(partial5b, v128_shr_n_byte_neon(tmp, 8));
+ partial7a = vaddq_s16(partial7a, v128_shl_n_byte_neon(tmp, 6));
+ partial7b = vaddq_s16(partial7b, v128_shr_n_byte_neon(tmp, 10));
+ partial6 = vaddq_s16(partial6, tmp);
+
+ // Partial sums for lines 4 and 5.
+ partial4a = vaddq_s16(partial4a, v128_shl_n_byte_neon(lines[4], 6));
+ partial4b = vaddq_s16(partial4b, v128_shr_n_byte_neon(lines[4], 10));
+ partial4a = vaddq_s16(partial4a, v128_shl_n_byte_neon(lines[5], 4));
+ partial4b = vaddq_s16(partial4b, v128_shr_n_byte_neon(lines[5], 12));
+ tmp = vaddq_s16(lines[4], lines[5]);
+ partial5a = vaddq_s16(partial5a, v128_shl_n_byte_neon(tmp, 6));
+ partial5b = vaddq_s16(partial5b, v128_shr_n_byte_neon(tmp, 10));
+ partial7a = vaddq_s16(partial7a, v128_shl_n_byte_neon(tmp, 8));
+ partial7b = vaddq_s16(partial7b, v128_shr_n_byte_neon(tmp, 8));
+ partial6 = vaddq_s16(partial6, tmp);
+
+ // Partial sums for lines 6 and 7.
+ partial4a = vaddq_s16(partial4a, v128_shl_n_byte_neon(lines[6], 2));
+ partial4b = vaddq_s16(partial4b, v128_shr_n_byte_neon(lines[6], 14));
+ partial4a = vaddq_s16(partial4a, lines[7]);
+ tmp = vaddq_s16(lines[6], lines[7]);
+ partial5a = vaddq_s16(partial5a, v128_shl_n_byte_neon(tmp, 4));
+ partial5b = vaddq_s16(partial5b, v128_shr_n_byte_neon(tmp, 12));
+ partial7a = vaddq_s16(partial7a, v128_shl_n_byte_neon(tmp, 10));
+ partial7b = vaddq_s16(partial7b, v128_shr_n_byte_neon(tmp, 6));
+ partial6 = vaddq_s16(partial6, tmp);
+
+ uint32x4_t const0 = vreinterpretq_u32_u64(
+ vcombine_u64(vcreate_u64((uint64_t)420 << 32 | 840),
+ vcreate_u64((uint64_t)210 << 32 | 280)));
+ uint32x4_t const1 = vreinterpretq_u32_u64(
+ vcombine_u64(vcreate_u64((uint64_t)140 << 32 | 168),
+ vcreate_u64((uint64_t)105 << 32 | 120)));
+ uint32x4_t const2 = vreinterpretq_u32_u64(
+ vcombine_u64(vcreate_u64(0), vcreate_u64((uint64_t)210 << 32 | 420)));
+ uint32x4_t const3 = vreinterpretq_u32_u64(
+ vcombine_u64(vcreate_u64((uint64_t)105 << 32 | 140),
+ vcreate_u64((uint64_t)105 << 32 | 105)));
+
+ // Compute costs in terms of partial sums.
+ uint32x4_t partial4a_u32 =
+ fold_mul_and_sum_neon(partial4a, partial4b, const0, const1);
+ uint32x4_t partial7a_u32 =
+ fold_mul_and_sum_neon(partial7a, partial7b, const2, const3);
+ uint32x4_t partial5a_u32 =
+ fold_mul_and_sum_neon(partial5a, partial5b, const2, const3);
+ uint32x4_t partial6_u32 = v128_madd_s16_neon(partial6, partial6);
+ partial6_u32 = vmulq_u32(partial6_u32, vdupq_n_u32(105));
+
+ partial4a_u32 =
+ hsum4_neon(partial4a_u32, partial5a_u32, partial6_u32, partial7a_u32);
+ vst1q_u32(cost, partial4a_u32);
+ return partial4a_u32;
+}
+
+static INLINE int64x2_t ziplo_s64(int32x4_t a, int32x4_t b) {
+ return vcombine_s64(vget_low_s64(vreinterpretq_s64_s32(a)),
+ vget_low_s64(vreinterpretq_s64_s32(b)));
+}
+
+static INLINE int64x2_t ziphi_s64(int32x4_t a, int32x4_t b) {
+ return vcombine_s64(vget_high_s64(vreinterpretq_s64_s32(a)),
+ vget_high_s64(vreinterpretq_s64_s32(b)));
+}
+
+// Transpose and reverse the order of the lines -- equivalent to a 90-degree
+// counter-clockwise rotation of the pixels.
+static INLINE void array_reverse_transpose_8x8_neon(int16x8_t *in,
+ int16x8_t *res) {
+ const int32x4_t tr0_0 = vreinterpretq_s32_s16(vzipq_s16(in[0], in[1]).val[0]);
+ const int32x4_t tr0_1 = vreinterpretq_s32_s16(vzipq_s16(in[2], in[3]).val[0]);
+ const int32x4_t tr0_2 = vreinterpretq_s32_s16(vzipq_s16(in[0], in[1]).val[1]);
+ const int32x4_t tr0_3 = vreinterpretq_s32_s16(vzipq_s16(in[2], in[3]).val[1]);
+ const int32x4_t tr0_4 = vreinterpretq_s32_s16(vzipq_s16(in[4], in[5]).val[0]);
+ const int32x4_t tr0_5 = vreinterpretq_s32_s16(vzipq_s16(in[6], in[7]).val[0]);
+ const int32x4_t tr0_6 = vreinterpretq_s32_s16(vzipq_s16(in[4], in[5]).val[1]);
+ const int32x4_t tr0_7 = vreinterpretq_s32_s16(vzipq_s16(in[6], in[7]).val[1]);
+
+ const int32x4_t tr1_0 = vzipq_s32(tr0_0, tr0_1).val[0];
+ const int32x4_t tr1_1 = vzipq_s32(tr0_4, tr0_5).val[0];
+ const int32x4_t tr1_2 = vzipq_s32(tr0_0, tr0_1).val[1];
+ const int32x4_t tr1_3 = vzipq_s32(tr0_4, tr0_5).val[1];
+ const int32x4_t tr1_4 = vzipq_s32(tr0_2, tr0_3).val[0];
+ const int32x4_t tr1_5 = vzipq_s32(tr0_6, tr0_7).val[0];
+ const int32x4_t tr1_6 = vzipq_s32(tr0_2, tr0_3).val[1];
+ const int32x4_t tr1_7 = vzipq_s32(tr0_6, tr0_7).val[1];
+
+ res[7] = vreinterpretq_s16_s64(ziplo_s64(tr1_0, tr1_1));
+ res[6] = vreinterpretq_s16_s64(ziphi_s64(tr1_0, tr1_1));
+ res[5] = vreinterpretq_s16_s64(ziplo_s64(tr1_2, tr1_3));
+ res[4] = vreinterpretq_s16_s64(ziphi_s64(tr1_2, tr1_3));
+ res[3] = vreinterpretq_s16_s64(ziplo_s64(tr1_4, tr1_5));
+ res[2] = vreinterpretq_s16_s64(ziphi_s64(tr1_4, tr1_5));
+ res[1] = vreinterpretq_s16_s64(ziplo_s64(tr1_6, tr1_7));
+ res[0] = vreinterpretq_s16_s64(ziphi_s64(tr1_6, tr1_7));
+}
+
+static INLINE uint32_t compute_best_dir(uint8x16_t a) {
+ uint8x16_t idx =
+ vandq_u8(a, vreinterpretq_u8_u64(vdupq_n_u64(0x8040201008040201ULL)));
+#if AOM_ARCH_AARCH64
+ return vaddv_u8(vget_low_u8(idx)) + (vaddv_u8(vget_high_u8(idx)) << 8);
+#else
+ uint64x2_t m = vpaddlq_u32(vpaddlq_u16(vpaddlq_u8(idx)));
+ uint8x16_t s = vreinterpretq_u8_u64(m);
+ return vget_lane_u32(
+ vreinterpret_u32_u8(vzip_u8(vget_low_u8(s), vget_high_u8(s)).val[0]), 0);
+#endif
+}
+
+int cdef_find_dir_neon(const uint16_t *img, int stride, int32_t *var,
+ int coeff_shift) {
+ uint32_t cost[8];
+ uint32_t best_cost = 0;
+ int best_dir = 0;
+ int16x8_t lines[8];
+ for (int i = 0; i < 8; i++) {
+ uint16x8_t s = vld1q_u16(&img[i * stride]);
+ lines[i] = vreinterpretq_s16_u16(
+ vsubq_u16(vshlq_u16(s, vdupq_n_s16(-coeff_shift)), vdupq_n_u16(128)));
+ }
+
+ // Compute "mostly vertical" directions.
+ uint32x4_t cost47 = compute_directions_neon(lines, cost + 4);
+
+ array_reverse_transpose_8x8_neon(lines, lines);
+
+ // Compute "mostly horizontal" directions.
+ uint32x4_t cost03 = compute_directions_neon(lines, cost);
+
+ uint32x4_t max_cost = vmaxq_u32(cost03, cost47);
+ max_cost = vmaxq_u32(max_cost, vextq_u32(max_cost, max_cost, 2));
+ max_cost = vmaxq_u32(max_cost, vextq_u32(max_cost, max_cost, 1));
+ best_cost = vgetq_lane_u32(max_cost, 0);
+ uint16x8_t idx = vcombine_u16(vqmovn_u32(vceqq_u32(max_cost, cost03)),
+ vqmovn_u32(vceqq_u32(max_cost, cost47)));
+ uint8x16_t idx_u8 = vcombine_u8(vqmovn_u16(idx), vqmovn_u16(idx));
+ best_dir = compute_best_dir(idx_u8);
+ best_dir = get_msb(best_dir ^ (best_dir - 1)); // Count trailing zeros
+
+ // Difference between the optimal variance and the variance along the
+ // orthogonal direction. Again, the sum(x^2) terms cancel out.
+ *var = best_cost - cost[(best_dir + 4) & 7];
+ // We'd normally divide by 840, but dividing by 1024 is close enough
+ // for what we're going to do with this.
+ *var >>= 10;
+ return best_dir;
}
void cdef_find_dir_dual_neon(const uint16_t *img1, const uint16_t *img2,
@@ -38,3 +423,532 @@
// Process second 8x8.
*out_dir_2nd_8x8 = cdef_find_dir(img2, stride, var_out_2nd, coeff_shift);
}
+
+// sign(a-b) * min(abs(a-b), max(0, threshold - (abs(a-b) >> adjdamp)))
+static INLINE int16x8_t constrain16(uint16x8_t a, uint16x8_t b,
+ unsigned int threshold, int adjdamp) {
+ int16x8_t diff = vreinterpretq_s16_u16(vsubq_u16(a, b));
+ const int16x8_t sign = vshrq_n_s16(diff, 15);
+ diff = vabsq_s16(diff);
+ const uint16x8_t s =
+ vqsubq_u16(vdupq_n_u16(threshold),
+ vreinterpretq_u16_s16(vshlq_s16(diff, vdupq_n_s16(-adjdamp))));
+ return veorq_s16(vaddq_s16(sign, vminq_s16(diff, vreinterpretq_s16_u16(s))),
+ sign);
+}
+
+static INLINE uint16x8_t get_max_primary(const int is_lowbd, uint16x8_t *tap,
+ uint16x8_t max,
+ uint16x8_t cdef_large_value_mask) {
+ if (is_lowbd) {
+ uint8x16_t max_u8 = vreinterpretq_u8_u16(tap[0]);
+ max_u8 = vmaxq_u8(max_u8, vreinterpretq_u8_u16(tap[1]));
+ max_u8 = vmaxq_u8(max_u8, vreinterpretq_u8_u16(tap[2]));
+ max_u8 = vmaxq_u8(max_u8, vreinterpretq_u8_u16(tap[3]));
+ /* The source is 16 bits, however, we only really care about the lower
+ 8 bits. The upper 8 bits contain the "large" flag. After the final
+ primary max has been calculated, zero out the upper 8 bits. Use this
+ to find the "16 bit" max. */
+ max = vmaxq_u16(
+ max, vandq_u16(vreinterpretq_u16_u8(max_u8), cdef_large_value_mask));
+ } else {
+ /* Convert CDEF_VERY_LARGE to 0 before calculating max. */
+ max = vmaxq_u16(max, vandq_u16(tap[0], cdef_large_value_mask));
+ max = vmaxq_u16(max, vandq_u16(tap[1], cdef_large_value_mask));
+ max = vmaxq_u16(max, vandq_u16(tap[2], cdef_large_value_mask));
+ max = vmaxq_u16(max, vandq_u16(tap[3], cdef_large_value_mask));
+ }
+ return max;
+}
+
+static INLINE uint16x8_t get_max_secondary(const int is_lowbd, uint16x8_t *tap,
+ uint16x8_t max,
+ uint16x8_t cdef_large_value_mask) {
+ if (is_lowbd) {
+ uint8x16_t max_u8 = vreinterpretq_u8_u16(tap[0]);
+ max_u8 = vmaxq_u8(max_u8, vreinterpretq_u8_u16(tap[1]));
+ max_u8 = vmaxq_u8(max_u8, vreinterpretq_u8_u16(tap[2]));
+ max_u8 = vmaxq_u8(max_u8, vreinterpretq_u8_u16(tap[3]));
+ max_u8 = vmaxq_u8(max_u8, vreinterpretq_u8_u16(tap[4]));
+ max_u8 = vmaxq_u8(max_u8, vreinterpretq_u8_u16(tap[5]));
+ max_u8 = vmaxq_u8(max_u8, vreinterpretq_u8_u16(tap[6]));
+ max_u8 = vmaxq_u8(max_u8, vreinterpretq_u8_u16(tap[7]));
+ /* The source is 16 bits, however, we only really care about the lower
+ 8 bits. The upper 8 bits contain the "large" flag. After the final
+ primary max has been calculated, zero out the upper 8 bits. Use this
+ to find the "16 bit" max. */
+ max = vmaxq_u16(
+ max, vandq_u16(vreinterpretq_u16_u8(max_u8), cdef_large_value_mask));
+ } else {
+ /* Convert CDEF_VERY_LARGE to 0 before calculating max. */
+ max = vmaxq_u16(max, vandq_u16(tap[0], cdef_large_value_mask));
+ max = vmaxq_u16(max, vandq_u16(tap[1], cdef_large_value_mask));
+ max = vmaxq_u16(max, vandq_u16(tap[2], cdef_large_value_mask));
+ max = vmaxq_u16(max, vandq_u16(tap[3], cdef_large_value_mask));
+ max = vmaxq_u16(max, vandq_u16(tap[4], cdef_large_value_mask));
+ max = vmaxq_u16(max, vandq_u16(tap[5], cdef_large_value_mask));
+ max = vmaxq_u16(max, vandq_u16(tap[6], cdef_large_value_mask));
+ max = vmaxq_u16(max, vandq_u16(tap[7], cdef_large_value_mask));
+ }
+ return max;
+}
+
+static INLINE void filter_block_4x4(const int is_lowbd, void *dest, int dstride,
+ const uint16_t *in, int pri_strength,
+ int sec_strength, int dir, int pri_damping,
+ int sec_damping, int coeff_shift,
+ int height, int enable_primary,
+ int enable_secondary) {
+ uint8_t *dst8 = (uint8_t *)dest;
+ uint16_t *dst16 = (uint16_t *)dest;
+ const int clipping_required = enable_primary && enable_secondary;
+ uint16x8_t max, min;
+ const uint16x8_t cdef_large_value_mask =
+ vdupq_n_u16(((uint16_t)~CDEF_VERY_LARGE));
+ const int po1 = cdef_directions[dir][0];
+ const int po2 = cdef_directions[dir][1];
+ const int s1o1 = cdef_directions[dir + 2][0];
+ const int s1o2 = cdef_directions[dir + 2][1];
+ const int s2o1 = cdef_directions[dir - 2][0];
+ const int s2o2 = cdef_directions[dir - 2][1];
+ const int *pri_taps = cdef_pri_taps[(pri_strength >> coeff_shift) & 1];
+ const int *sec_taps = cdef_sec_taps;
+
+ if (enable_primary && pri_strength) {
+ pri_damping = AOMMAX(0, pri_damping - get_msb(pri_strength));
+ }
+ if (enable_secondary && sec_strength) {
+ sec_damping = AOMMAX(0, sec_damping - get_msb(sec_strength));
+ }
+
+ int h = height;
+ do {
+ int16x8_t sum = vdupq_n_s16(0);
+ uint16x8_t s = load_unaligned_u16_4x2(in, CDEF_BSTRIDE);
+ max = min = s;
+
+ if (enable_primary) {
+ uint16x8_t tap[4];
+
+ // Primary near taps
+ tap[0] = load_unaligned_u16_4x2(in + po1, CDEF_BSTRIDE);
+ tap[1] = load_unaligned_u16_4x2(in - po1, CDEF_BSTRIDE);
+ int16x8_t p0 = constrain16(tap[0], s, pri_strength, pri_damping);
+ int16x8_t p1 = constrain16(tap[1], s, pri_strength, pri_damping);
+
+ // sum += pri_taps[0] * (p0 + p1)
+ p0 = vaddq_s16(p0, p1);
+ sum = vmlaq_s16(sum, p0, vdupq_n_s16(pri_taps[0]));
+
+ // Primary far taps
+ tap[2] = load_unaligned_u16_4x2(in + po2, CDEF_BSTRIDE);
+ tap[3] = load_unaligned_u16_4x2(in - po2, CDEF_BSTRIDE);
+ p0 = constrain16(tap[2], s, pri_strength, pri_damping);
+ p1 = constrain16(tap[3], s, pri_strength, pri_damping);
+
+ // sum += pri_taps[1] * (p0 + p1)
+ p0 = vaddq_s16(p0, p1);
+ sum = vmlaq_s16(sum, p0, vdupq_n_s16(pri_taps[1]));
+
+ if (clipping_required) {
+ max = get_max_primary(is_lowbd, tap, max, cdef_large_value_mask);
+
+ min = vminq_u16(min, tap[0]);
+ min = vminq_u16(min, tap[1]);
+ min = vminq_u16(min, tap[2]);
+ min = vminq_u16(min, tap[3]);
+ }
+ }
+
+ if (enable_secondary) {
+ uint16x8_t tap[8];
+
+ // Secondary near taps
+ tap[0] = load_unaligned_u16_4x2(in + s1o1, CDEF_BSTRIDE);
+ tap[1] = load_unaligned_u16_4x2(in - s1o1, CDEF_BSTRIDE);
+ tap[2] = load_unaligned_u16_4x2(in + s2o1, CDEF_BSTRIDE);
+ tap[3] = load_unaligned_u16_4x2(in - s2o1, CDEF_BSTRIDE);
+ int16x8_t p0 = constrain16(tap[0], s, sec_strength, sec_damping);
+ int16x8_t p1 = constrain16(tap[1], s, sec_strength, sec_damping);
+ int16x8_t p2 = constrain16(tap[2], s, sec_strength, sec_damping);
+ int16x8_t p3 = constrain16(tap[3], s, sec_strength, sec_damping);
+
+ // sum += sec_taps[0] * (p0 + p1 + p2 + p3)
+ p0 = vaddq_s16(p0, p1);
+ p2 = vaddq_s16(p2, p3);
+ p0 = vaddq_s16(p0, p2);
+ sum = vmlaq_s16(sum, p0, vdupq_n_s16(sec_taps[0]));
+
+ // Secondary far taps
+ tap[4] = load_unaligned_u16_4x2(in + s1o2, CDEF_BSTRIDE);
+ tap[5] = load_unaligned_u16_4x2(in - s1o2, CDEF_BSTRIDE);
+ tap[6] = load_unaligned_u16_4x2(in + s2o2, CDEF_BSTRIDE);
+ tap[7] = load_unaligned_u16_4x2(in - s2o2, CDEF_BSTRIDE);
+ p0 = constrain16(tap[4], s, sec_strength, sec_damping);
+ p1 = constrain16(tap[5], s, sec_strength, sec_damping);
+ p2 = constrain16(tap[6], s, sec_strength, sec_damping);
+ p3 = constrain16(tap[7], s, sec_strength, sec_damping);
+
+ // sum += sec_taps[1] * (p0 + p1 + p2 + p3)
+ p0 = vaddq_s16(p0, p1);
+ p2 = vaddq_s16(p2, p3);
+ p0 = vaddq_s16(p0, p2);
+ sum = vmlaq_s16(sum, p0, vdupq_n_s16(sec_taps[1]));
+
+ if (clipping_required) {
+ max = get_max_secondary(is_lowbd, tap, max, cdef_large_value_mask);
+
+ min = vminq_u16(min, tap[0]);
+ min = vminq_u16(min, tap[1]);
+ min = vminq_u16(min, tap[2]);
+ min = vminq_u16(min, tap[3]);
+ min = vminq_u16(min, tap[4]);
+ min = vminq_u16(min, tap[5]);
+ min = vminq_u16(min, tap[6]);
+ min = vminq_u16(min, tap[7]);
+ }
+ }
+
+ // res = row + ((sum - (sum < 0) + 8) >> 4)
+ sum = vaddq_s16(sum, vreinterpretq_s16_u16(vcltq_s16(sum, vdupq_n_s16(0))));
+ int16x8_t res = vaddq_s16(sum, vdupq_n_s16(8));
+ res = vshrq_n_s16(res, 4);
+ res = vaddq_s16(vreinterpretq_s16_u16(s), res);
+
+ if (clipping_required) {
+ res = vminq_s16(vmaxq_s16(res, vreinterpretq_s16_u16(min)),
+ vreinterpretq_s16_u16(max));
+ }
+
+ if (is_lowbd) {
+ const uint8x8_t res_128 = vqmovun_s16(res);
+ store_unaligned_u8_4x2(dst8, dstride, res_128);
+ } else {
+ store_unaligned_u16_4x2(dst16, dstride, vreinterpretq_u16_s16(res));
+ }
+
+ in += 2 * CDEF_BSTRIDE;
+ dst8 += 2 * dstride;
+ dst16 += 2 * dstride;
+ h -= 2;
+ } while (h != 0);
+}
+
+static INLINE void filter_block_8x8(const int is_lowbd, void *dest, int dstride,
+ const uint16_t *in, int pri_strength,
+ int sec_strength, int dir, int pri_damping,
+ int sec_damping, int coeff_shift,
+ int height, int enable_primary,
+ int enable_secondary) {
+ uint8_t *dst8 = (uint8_t *)dest;
+ uint16_t *dst16 = (uint16_t *)dest;
+ const int clipping_required = enable_primary && enable_secondary;
+ uint16x8_t max, min;
+ const uint16x8_t cdef_large_value_mask =
+ vdupq_n_u16(((uint16_t)~CDEF_VERY_LARGE));
+ const int po1 = cdef_directions[dir][0];
+ const int po2 = cdef_directions[dir][1];
+ const int s1o1 = cdef_directions[dir + 2][0];
+ const int s1o2 = cdef_directions[dir + 2][1];
+ const int s2o1 = cdef_directions[dir - 2][0];
+ const int s2o2 = cdef_directions[dir - 2][1];
+ const int *pri_taps = cdef_pri_taps[(pri_strength >> coeff_shift) & 1];
+ const int *sec_taps = cdef_sec_taps;
+
+ if (enable_primary && pri_strength) {
+ pri_damping = AOMMAX(0, pri_damping - get_msb(pri_strength));
+ }
+ if (enable_secondary && sec_strength) {
+ sec_damping = AOMMAX(0, sec_damping - get_msb(sec_strength));
+ }
+
+ int h = height;
+ do {
+ int16x8_t sum = vdupq_n_s16(0);
+ uint16x8_t s = vld1q_u16(in);
+ max = min = s;
+
+ if (enable_primary) {
+ uint16x8_t tap[4];
+
+ // Primary near taps
+ tap[0] = vld1q_u16(in + po1);
+ tap[1] = vld1q_u16(in - po1);
+ int16x8_t p0 = constrain16(tap[0], s, pri_strength, pri_damping);
+ int16x8_t p1 = constrain16(tap[1], s, pri_strength, pri_damping);
+
+ // sum += pri_taps[0] * (p0 + p1)
+ p0 = vaddq_s16(p0, p1);
+ sum = vmlaq_s16(sum, p0, vdupq_n_s16(pri_taps[0]));
+
+ // Primary far taps
+ tap[2] = vld1q_u16(in + po2);
+ p0 = constrain16(tap[2], s, pri_strength, pri_damping);
+ tap[3] = vld1q_u16(in - po2);
+ p1 = constrain16(tap[3], s, pri_strength, pri_damping);
+
+ // sum += pri_taps[1] * (p0 + p1)
+ p0 = vaddq_s16(p0, p1);
+ sum = vmlaq_s16(sum, p0, vdupq_n_s16(pri_taps[1]));
+ if (clipping_required) {
+ max = get_max_primary(is_lowbd, tap, max, cdef_large_value_mask);
+
+ min = vminq_u16(min, tap[0]);
+ min = vminq_u16(min, tap[1]);
+ min = vminq_u16(min, tap[2]);
+ min = vminq_u16(min, tap[3]);
+ }
+ }
+
+ if (enable_secondary) {
+ uint16x8_t tap[8];
+
+ // Secondary near taps
+ tap[0] = vld1q_u16(in + s1o1);
+ tap[1] = vld1q_u16(in - s1o1);
+ tap[2] = vld1q_u16(in + s2o1);
+ tap[3] = vld1q_u16(in - s2o1);
+ int16x8_t p0 = constrain16(tap[0], s, sec_strength, sec_damping);
+ int16x8_t p1 = constrain16(tap[1], s, sec_strength, sec_damping);
+ int16x8_t p2 = constrain16(tap[2], s, sec_strength, sec_damping);
+ int16x8_t p3 = constrain16(tap[3], s, sec_strength, sec_damping);
+
+ // sum += sec_taps[0] * (p0 + p1 + p2 + p3)
+ p0 = vaddq_s16(p0, p1);
+ p2 = vaddq_s16(p2, p3);
+ p0 = vaddq_s16(p0, p2);
+ sum = vmlaq_s16(sum, p0, vdupq_n_s16(sec_taps[0]));
+
+ // Secondary far taps
+ tap[4] = vld1q_u16(in + s1o2);
+ tap[5] = vld1q_u16(in - s1o2);
+ tap[6] = vld1q_u16(in + s2o2);
+ tap[7] = vld1q_u16(in - s2o2);
+ p0 = constrain16(tap[4], s, sec_strength, sec_damping);
+ p1 = constrain16(tap[5], s, sec_strength, sec_damping);
+ p2 = constrain16(tap[6], s, sec_strength, sec_damping);
+ p3 = constrain16(tap[7], s, sec_strength, sec_damping);
+
+ // sum += sec_taps[1] * (p0 + p1 + p2 + p3)
+ p0 = vaddq_s16(p0, p1);
+ p2 = vaddq_s16(p2, p3);
+ p0 = vaddq_s16(p0, p2);
+ sum = vmlaq_s16(sum, p0, vdupq_n_s16(sec_taps[1]));
+
+ if (clipping_required) {
+ max = get_max_secondary(is_lowbd, tap, max, cdef_large_value_mask);
+
+ min = vminq_u16(min, tap[0]);
+ min = vminq_u16(min, tap[1]);
+ min = vminq_u16(min, tap[2]);
+ min = vminq_u16(min, tap[3]);
+ min = vminq_u16(min, tap[4]);
+ min = vminq_u16(min, tap[5]);
+ min = vminq_u16(min, tap[6]);
+ min = vminq_u16(min, tap[7]);
+ }
+ }
+
+ // res = row + ((sum - (sum < 0) + 8) >> 4)
+ sum = vaddq_s16(sum, vreinterpretq_s16_u16(vcltq_s16(sum, vdupq_n_s16(0))));
+ int16x8_t res = vaddq_s16(sum, vdupq_n_s16(8));
+ res = vshrq_n_s16(res, 4);
+ res = vaddq_s16(vreinterpretq_s16_u16(s), res);
+ if (clipping_required) {
+ res = vminq_s16(vmaxq_s16(res, vreinterpretq_s16_u16(min)),
+ vreinterpretq_s16_u16(max));
+ }
+
+ if (is_lowbd) {
+ const uint8x8_t res_128 = vqmovun_s16(res);
+ vst1_u8(dst8, res_128);
+ } else {
+ vst1q_u16(dst16, vreinterpretq_u16_s16(res));
+ }
+
+ in += CDEF_BSTRIDE;
+ dst8 += dstride;
+ dst16 += dstride;
+ } while (--h != 0);
+}
+
+static INLINE void copy_block_4xh(const int is_lowbd, void *dest, int dstride,
+ const uint16_t *in, int height) {
+ uint8_t *dst8 = (uint8_t *)dest;
+ uint16_t *dst16 = (uint16_t *)dest;
+
+ int h = height;
+ do {
+ const uint16x8_t row = load_unaligned_u16_4x2(in, CDEF_BSTRIDE);
+ if (is_lowbd) {
+ const uint8x8_t res_128 = vqmovn_u16(row);
+ store_unaligned_u8_4x2(dst8, dstride, res_128);
+ } else {
+ store_unaligned_u16_4x2(dst16, dstride, row);
+ }
+
+ in += 2 * CDEF_BSTRIDE;
+ dst8 += 2 * dstride;
+ dst16 += 2 * dstride;
+ h -= 2;
+ } while (h != 0);
+}
+
+static INLINE void copy_block_8xh(const int is_lowbd, void *dest, int dstride,
+ const uint16_t *in, int height) {
+ uint8_t *dst8 = (uint8_t *)dest;
+ uint16_t *dst16 = (uint16_t *)dest;
+
+ int h = height;
+ do {
+ const uint16x8_t row = vld1q_u16(in);
+ if (is_lowbd) {
+ const uint8x8_t res_128 = vqmovn_u16(row);
+ vst1_u8(dst8, res_128);
+ } else {
+ vst1q_u16(dst16, row);
+ }
+
+ in += CDEF_BSTRIDE;
+ dst8 += dstride;
+ dst16 += dstride;
+ } while (--h != 0);
+}
+
+void cdef_filter_8_0_neon(void *dest, int dstride, const uint16_t *in,
+ int pri_strength, int sec_strength, int dir,
+ int pri_damping, int sec_damping, int coeff_shift,
+ int block_width, int block_height) {
+ if (block_width == 8) {
+ filter_block_8x8(/*is_lowbd=*/1, dest, dstride, in, pri_strength,
+ sec_strength, dir, pri_damping, sec_damping, coeff_shift,
+ block_height, /*enable_primary=*/1,
+ /*enable_secondary=*/1);
+ } else {
+ filter_block_4x4(/*is_lowbd=*/1, dest, dstride, in, pri_strength,
+ sec_strength, dir, pri_damping, sec_damping, coeff_shift,
+ block_height, /*enable_primary=*/1,
+ /*enable_secondary=*/1);
+ }
+}
+
+void cdef_filter_8_1_neon(void *dest, int dstride, const uint16_t *in,
+ int pri_strength, int sec_strength, int dir,
+ int pri_damping, int sec_damping, int coeff_shift,
+ int block_width, int block_height) {
+ if (block_width == 8) {
+ filter_block_8x8(/*is_lowbd=*/1, dest, dstride, in, pri_strength,
+ sec_strength, dir, pri_damping, sec_damping, coeff_shift,
+ block_height, /*enable_primary=*/1,
+ /*enable_secondary=*/0);
+ } else {
+ filter_block_4x4(/*is_lowbd=*/1, dest, dstride, in, pri_strength,
+ sec_strength, dir, pri_damping, sec_damping, coeff_shift,
+ block_height, /*enable_primary=*/1,
+ /*enable_secondary=*/0);
+ }
+}
+
+void cdef_filter_8_2_neon(void *dest, int dstride, const uint16_t *in,
+ int pri_strength, int sec_strength, int dir,
+ int pri_damping, int sec_damping, int coeff_shift,
+ int block_width, int block_height) {
+ if (block_width == 8) {
+ filter_block_8x8(/*is_lowbd=*/1, dest, dstride, in, pri_strength,
+ sec_strength, dir, pri_damping, sec_damping, coeff_shift,
+ block_height, /*enable_primary=*/0,
+ /*enable_secondary=*/1);
+ } else {
+ filter_block_4x4(/*is_lowbd=*/1, dest, dstride, in, pri_strength,
+ sec_strength, dir, pri_damping, sec_damping, coeff_shift,
+ block_height, /*enable_primary=*/0,
+ /*enable_secondary=*/1);
+ }
+}
+
+void cdef_filter_8_3_neon(void *dest, int dstride, const uint16_t *in,
+ int pri_strength, int sec_strength, int dir,
+ int pri_damping, int sec_damping, int coeff_shift,
+ int block_width, int block_height) {
+ (void)pri_strength;
+ (void)sec_strength;
+ (void)dir;
+ (void)pri_damping;
+ (void)sec_damping;
+ (void)coeff_shift;
+ (void)block_width;
+ if (block_width == 8) {
+ copy_block_8xh(/*is_lowbd=*/1, dest, dstride, in, block_height);
+ } else {
+ copy_block_4xh(/*is_lowbd=*/1, dest, dstride, in, block_height);
+ }
+}
+
+void cdef_filter_16_0_neon(void *dest, int dstride, const uint16_t *in,
+ int pri_strength, int sec_strength, int dir,
+ int pri_damping, int sec_damping, int coeff_shift,
+ int block_width, int block_height) {
+ if (block_width == 8) {
+ filter_block_8x8(/*is_lowbd=*/0, dest, dstride, in, pri_strength,
+ sec_strength, dir, pri_damping, sec_damping, coeff_shift,
+ block_height, /*enable_primary=*/1,
+ /*enable_secondary=*/1);
+ } else {
+ filter_block_4x4(/*is_lowbd=*/0, dest, dstride, in, pri_strength,
+ sec_strength, dir, pri_damping, sec_damping, coeff_shift,
+ block_height, /*enable_primary=*/1,
+ /*enable_secondary=*/1);
+ }
+}
+
+void cdef_filter_16_1_neon(void *dest, int dstride, const uint16_t *in,
+ int pri_strength, int sec_strength, int dir,
+ int pri_damping, int sec_damping, int coeff_shift,
+ int block_width, int block_height) {
+ if (block_width == 8) {
+ filter_block_8x8(/*is_lowbd=*/0, dest, dstride, in, pri_strength,
+ sec_strength, dir, pri_damping, sec_damping, coeff_shift,
+ block_height, /*enable_primary=*/1,
+ /*enable_secondary=*/0);
+ } else {
+ filter_block_4x4(/*is_lowbd=*/0, dest, dstride, in, pri_strength,
+ sec_strength, dir, pri_damping, sec_damping, coeff_shift,
+ block_height, /*enable_primary=*/1,
+ /*enable_secondary=*/0);
+ }
+}
+
+void cdef_filter_16_2_neon(void *dest, int dstride, const uint16_t *in,
+ int pri_strength, int sec_strength, int dir,
+ int pri_damping, int sec_damping, int coeff_shift,
+ int block_width, int block_height) {
+ if (block_width == 8) {
+ filter_block_8x8(/*is_lowbd=*/0, dest, dstride, in, pri_strength,
+ sec_strength, dir, pri_damping, sec_damping, coeff_shift,
+ block_height, /*enable_primary=*/0,
+ /*enable_secondary=*/1);
+ } else {
+ filter_block_4x4(/*is_lowbd=*/0, dest, dstride, in, pri_strength,
+ sec_strength, dir, pri_damping, sec_damping, coeff_shift,
+ block_height, /*enable_primary=*/0,
+ /*enable_secondary=*/1);
+ }
+}
+
+void cdef_filter_16_3_neon(void *dest, int dstride, const uint16_t *in,
+ int pri_strength, int sec_strength, int dir,
+ int pri_damping, int sec_damping, int coeff_shift,
+ int block_width, int block_height) {
+ (void)pri_strength;
+ (void)sec_strength;
+ (void)dir;
+ (void)pri_damping;
+ (void)sec_damping;
+ (void)coeff_shift;
+ (void)block_width;
+ if (block_width == 8) {
+ copy_block_8xh(/*is_lowbd=*/0, dest, dstride, in, block_height);
+ } else {
+ copy_block_4xh(/*is_lowbd=*/0, dest, dstride, in, block_height);
+ }
+}
diff --git a/av1/common/arm/compound_convolve_neon.c b/av1/common/arm/compound_convolve_neon.c
new file mode 100644
index 0000000..2e6af68
--- /dev/null
+++ b/av1/common/arm/compound_convolve_neon.c
@@ -0,0 +1,2731 @@
+/*
+ * Copyright (c) 2018, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+#include <assert.h>
+
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_dsp/arm/transpose_neon.h"
+#include "av1/common/arm/compound_convolve_neon.h"
+#include "config/aom_config.h"
+#include "config/av1_rtcd.h"
+
+static INLINE int16x4_t convolve4_4_2d_h(const int16x4_t s0, const int16x4_t s1,
+ const int16x4_t s2, const int16x4_t s3,
+ const int16x4_t x_filter,
+ const int16x4_t horiz_const) {
+ int16x4_t sum = horiz_const;
+ sum = vmla_lane_s16(sum, s0, x_filter, 0);
+ sum = vmla_lane_s16(sum, s1, x_filter, 1);
+ sum = vmla_lane_s16(sum, s2, x_filter, 2);
+ sum = vmla_lane_s16(sum, s3, x_filter, 3);
+
+ // We halved the convolution filter values so -1 from the right shift.
+ return vshr_n_s16(sum, ROUND0_BITS - 1);
+}
+
+static INLINE int16x8_t convolve8_8_2d_h(const int16x8_t s0, const int16x8_t s1,
+ const int16x8_t s2, const int16x8_t s3,
+ const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t s6, const int16x8_t s7,
+ const int16x8_t x_filter,
+ const int16x8_t horiz_const) {
+ const int16x4_t x_filter_0_3 = vget_low_s16(x_filter);
+ const int16x4_t x_filter_4_7 = vget_high_s16(x_filter);
+
+ int16x8_t sum = horiz_const;
+ sum = vmlaq_lane_s16(sum, s0, x_filter_0_3, 0);
+ sum = vmlaq_lane_s16(sum, s1, x_filter_0_3, 1);
+ sum = vmlaq_lane_s16(sum, s2, x_filter_0_3, 2);
+ sum = vmlaq_lane_s16(sum, s3, x_filter_0_3, 3);
+ sum = vmlaq_lane_s16(sum, s4, x_filter_4_7, 0);
+ sum = vmlaq_lane_s16(sum, s5, x_filter_4_7, 1);
+ sum = vmlaq_lane_s16(sum, s6, x_filter_4_7, 2);
+ sum = vmlaq_lane_s16(sum, s7, x_filter_4_7, 3);
+
+ // We halved the convolution filter values so -1 from the right shift.
+ return vshrq_n_s16(sum, ROUND0_BITS - 1);
+}
+
+static INLINE void dist_wtd_convolve_2d_horiz_neon(
+ const uint8_t *src, int src_stride, int16_t *im_block, const int im_stride,
+ const int16_t *x_filter_ptr, const int im_h, int w) {
+ const int bd = 8;
+
+ const uint8_t *src_ptr = src;
+ int16_t *dst_ptr = im_block;
+ int dst_stride = im_stride;
+ int height = im_h;
+
+ if (w == 4) {
+ // A shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-rounding
+ // shifts - which are generally faster than rounding shifts on modern CPUs.
+ // (The extra -1 is needed because we halved the filter values.)
+ const int16x4_t horiz_const = vdup_n_s16((1 << (bd + FILTER_BITS - 2)) +
+ (1 << ((ROUND0_BITS - 1) - 1)));
+ // 4-tap filters are used for blocks having width <= 4.
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int16x4_t x_filter = vshr_n_s16(vld1_s16(x_filter_ptr + 2), 1);
+
+ src_ptr += 2;
+
+ do {
+ uint8x8_t t0 = vld1_u8(src_ptr); // a0 a1 a2 a3 a4 a5 a6 a7
+ int16x4_t s0 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
+ int16x4_t s4 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
+
+ __builtin_prefetch(dst_ptr);
+
+ int16x4_t s1 = vext_s16(s0, s4, 1); // a1 a2 a3 a4
+ int16x4_t s2 = vext_s16(s0, s4, 2); // a2 a3 a4 a5
+ int16x4_t s3 = vext_s16(s0, s4, 3); // a3 a4 a5 a6
+
+ int16x4_t d0 = convolve4_4_2d_h(s0, s1, s2, s3, x_filter, horiz_const);
+
+ vst1_s16(dst_ptr, d0);
+
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ } while (--height != 0);
+ } else {
+ // A shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-rounding
+ // shifts - which are generally faster than rounding shifts on modern CPUs.
+ // (The extra -1 is needed because we halved the filter values.)
+ const int16x8_t horiz_const = vdupq_n_s16((1 << (bd + FILTER_BITS - 2)) +
+ (1 << ((ROUND0_BITS - 1) - 1)));
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int16x8_t x_filter = vshrq_n_s16(vld1q_s16(x_filter_ptr), 1);
+
+#if AOM_ARCH_AARCH64
+ do {
+ const uint8_t *s;
+ int16_t *d = dst_ptr;
+ int width = w;
+
+ __builtin_prefetch(src_ptr + 0 * src_stride);
+ __builtin_prefetch(src_ptr + 1 * src_stride);
+ __builtin_prefetch(src_ptr + 2 * src_stride);
+ __builtin_prefetch(src_ptr + 3 * src_stride);
+ __builtin_prefetch(src_ptr + 4 * src_stride);
+ __builtin_prefetch(src_ptr + 5 * src_stride);
+ __builtin_prefetch(src_ptr + 6 * src_stride);
+ __builtin_prefetch(src_ptr + 7 * src_stride);
+
+ uint8x8_t t0, t1, t2, t3, t4, t5, t6, t7;
+ load_u8_8x8(src_ptr, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
+ transpose_elems_inplace_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
+
+ int16x8_t s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
+ int16x8_t s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
+ int16x8_t s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
+ int16x8_t s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
+ int16x8_t s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
+ int16x8_t s5 = vreinterpretq_s16_u16(vmovl_u8(t5));
+ int16x8_t s6 = vreinterpretq_s16_u16(vmovl_u8(t6));
+
+ s = src_ptr + 7;
+
+ __builtin_prefetch(dst_ptr + 0 * dst_stride);
+ __builtin_prefetch(dst_ptr + 1 * dst_stride);
+ __builtin_prefetch(dst_ptr + 2 * dst_stride);
+ __builtin_prefetch(dst_ptr + 3 * dst_stride);
+ __builtin_prefetch(dst_ptr + 4 * dst_stride);
+ __builtin_prefetch(dst_ptr + 5 * dst_stride);
+ __builtin_prefetch(dst_ptr + 6 * dst_stride);
+ __builtin_prefetch(dst_ptr + 7 * dst_stride);
+
+ do {
+ load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
+ transpose_elems_inplace_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
+
+ int16x8_t s7 = vreinterpretq_s16_u16(vmovl_u8(t0));
+ int16x8_t s8 = vreinterpretq_s16_u16(vmovl_u8(t1));
+ int16x8_t s9 = vreinterpretq_s16_u16(vmovl_u8(t2));
+ int16x8_t s10 = vreinterpretq_s16_u16(vmovl_u8(t3));
+ int16x8_t s11 = vreinterpretq_s16_u16(vmovl_u8(t4));
+ int16x8_t s12 = vreinterpretq_s16_u16(vmovl_u8(t5));
+ int16x8_t s13 = vreinterpretq_s16_u16(vmovl_u8(t6));
+ int16x8_t s14 = vreinterpretq_s16_u16(vmovl_u8(t7));
+
+ int16x8_t d0 = convolve8_8_2d_h(s0, s1, s2, s3, s4, s5, s6, s7,
+ x_filter, horiz_const);
+ int16x8_t d1 = convolve8_8_2d_h(s1, s2, s3, s4, s5, s6, s7, s8,
+ x_filter, horiz_const);
+ int16x8_t d2 = convolve8_8_2d_h(s2, s3, s4, s5, s6, s7, s8, s9,
+ x_filter, horiz_const);
+ int16x8_t d3 = convolve8_8_2d_h(s3, s4, s5, s6, s7, s8, s9, s10,
+ x_filter, horiz_const);
+ int16x8_t d4 = convolve8_8_2d_h(s4, s5, s6, s7, s8, s9, s10, s11,
+ x_filter, horiz_const);
+ int16x8_t d5 = convolve8_8_2d_h(s5, s6, s7, s8, s9, s10, s11, s12,
+ x_filter, horiz_const);
+ int16x8_t d6 = convolve8_8_2d_h(s6, s7, s8, s9, s10, s11, s12, s13,
+ x_filter, horiz_const);
+ int16x8_t d7 = convolve8_8_2d_h(s7, s8, s9, s10, s11, s12, s13, s14,
+ x_filter, horiz_const);
+
+ transpose_elems_inplace_s16_8x8(&d0, &d1, &d2, &d3, &d4, &d5, &d6, &d7);
+ store_s16_8x8(d, dst_stride, d0, d1, d2, d3, d4, d5, d6, d7);
+
+ s0 = s8;
+ s1 = s9;
+ s2 = s10;
+ s3 = s11;
+ s4 = s12;
+ s5 = s13;
+ s6 = s14;
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width > 0);
+ src_ptr += 8 * src_stride;
+ dst_ptr += 8 * dst_stride;
+ height -= 8;
+ } while (height > 8);
+#endif // AOM_ARCH_AARCH64
+
+ do {
+ const uint8_t *s;
+ int16_t *d = dst_ptr;
+ int width = w;
+
+ uint8x8_t t0 = vld1_u8(src_ptr);
+ int16x8_t s0 =
+ vreinterpretq_s16_u16(vmovl_u8(t0)); // a0 a1 a2 a3 a4 a5 a6 a7
+
+ s = src_ptr + 8;
+ __builtin_prefetch(dst_ptr);
+
+ do {
+ t0 = vld1_u8(s); // a8 a9 a10 a11 a12 a13 a14 a15
+ int16x8_t s8 = vreinterpretq_s16_u16(vmovl_u8(t0));
+
+ int16x8_t s1 = vextq_s16(s0, s8, 1); // a1 a2 a3 a4 a5 a6 a7 a8
+ int16x8_t s2 = vextq_s16(s0, s8, 2); // a2 a3 a4 a5 a6 a7 a8 a9
+ int16x8_t s3 = vextq_s16(s0, s8, 3); // a3 a4 a5 a6 a7 a8 a9 a10
+ int16x8_t s4 = vextq_s16(s0, s8, 4); // a4 a5 a6 a7 a8 a9 a10 a11
+ int16x8_t s5 = vextq_s16(s0, s8, 5); // a5 a6 a7 a8 a9 a10 a11 a12
+ int16x8_t s6 = vextq_s16(s0, s8, 6); // a6 a7 a8 a9 a10 a11 a12 a13
+ int16x8_t s7 = vextq_s16(s0, s8, 7); // a7 a8 a9 a10 a11 a12 a13 a14
+
+ int16x8_t d0 = convolve8_8_2d_h(s0, s1, s2, s3, s4, s5, s6, s7,
+ x_filter, horiz_const);
+ vst1q_s16(d, d0);
+
+ s0 = s8;
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width > 0);
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ } while (--height != 0);
+ }
+}
+
+void av1_dist_wtd_convolve_2d_neon(const uint8_t *src, int src_stride,
+ uint8_t *dst8, int dst8_stride, int w, int h,
+ const InterpFilterParams *filter_params_x,
+ const InterpFilterParams *filter_params_y,
+ const int subpel_x_qn, const int subpel_y_qn,
+ ConvolveParams *conv_params) {
+ assert(w % 4 == 0);
+ assert(h % 4 == 0);
+
+ DECLARE_ALIGNED(16, int16_t,
+ im_block[(MAX_SB_SIZE + SUBPEL_TAPS - 1) * MAX_SB_SIZE]);
+
+ const int y_filter_taps = get_filter_tap(filter_params_y, subpel_y_qn);
+ const int clamped_y_taps = y_filter_taps < 6 ? 6 : y_filter_taps;
+
+ const int im_h = h + clamped_y_taps - 1;
+ const int im_stride = MAX_SB_SIZE;
+ const int vert_offset = clamped_y_taps / 2 - 1;
+ const int horiz_offset = filter_params_x->taps / 2 - 1;
+ const uint8_t *src_ptr = src - vert_offset * src_stride - horiz_offset;
+ const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
+ filter_params_x, subpel_x_qn & SUBPEL_MASK);
+ const int16_t *y_filter_ptr = av1_get_interp_filter_subpel_kernel(
+ filter_params_y, subpel_y_qn & SUBPEL_MASK);
+
+ const int16x8_t y_filter = vld1q_s16(y_filter_ptr);
+
+ dist_wtd_convolve_2d_horiz_neon(src_ptr, src_stride, im_block, im_stride,
+ x_filter_ptr, im_h, w);
+
+ if (clamped_y_taps == 6) {
+ if (conv_params->do_average) {
+ if (UNLIKELY(conv_params->use_dist_wtd_comp_avg)) {
+ dist_wtd_convolve_2d_vert_6tap_dist_wtd_avg_neon(
+ im_block, im_stride, dst8, dst8_stride, conv_params, y_filter, h,
+ w);
+ } else {
+ dist_wtd_convolve_2d_vert_6tap_avg_neon(im_block, im_stride, dst8,
+ dst8_stride, conv_params,
+ y_filter, h, w);
+ }
+ } else {
+ dist_wtd_convolve_2d_vert_6tap_neon(im_block, im_stride, conv_params,
+ y_filter, h, w);
+ }
+ } else {
+ if (conv_params->do_average) {
+ if (UNLIKELY(conv_params->use_dist_wtd_comp_avg)) {
+ dist_wtd_convolve_2d_vert_8tap_dist_wtd_avg_neon(
+ im_block, im_stride, dst8, dst8_stride, conv_params, y_filter, h,
+ w);
+ } else {
+ dist_wtd_convolve_2d_vert_8tap_avg_neon(im_block, im_stride, dst8,
+ dst8_stride, conv_params,
+ y_filter, h, w);
+ }
+ } else {
+ dist_wtd_convolve_2d_vert_8tap_neon(im_block, im_stride, conv_params,
+ y_filter, h, w);
+ }
+ }
+}
+
+static INLINE void dist_wtd_convolve_2d_copy_dist_wtd_avg_neon(
+ const uint8_t *src, int src_stride, uint8_t *dst8, int dst8_stride, int w,
+ int h, ConvolveParams *conv_params) {
+ assert(w % 4 == 0);
+ assert(h % 4 == 0);
+
+ const int bd = 8;
+ const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
+ const uint16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
+ (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
+ const uint16x8_t round_offset_vec = vdupq_n_u16(round_offset);
+ const uint8x8_t shift_by_bits = vdup_n_u8(1 << (FILTER_BITS - ROUND0_BITS));
+
+ const uint16_t fwd_offset = conv_params->fwd_offset;
+ const uint16_t bck_offset = conv_params->bck_offset;
+
+ CONV_BUF_TYPE *dst = conv_params->dst;
+ const int dst_stride = conv_params->dst_stride;
+ int height = h;
+
+ if (w == 4) {
+ do {
+ uint8x8_t s0, s1, s2, s3;
+ load_u8_8x4(src, src_stride, &s0, &s1, &s2, &s3);
+
+ uint16x4_t d0 =
+ vget_low_u16(vmlal_u8(round_offset_vec, s0, shift_by_bits));
+ uint16x4_t d1 =
+ vget_low_u16(vmlal_u8(round_offset_vec, s1, shift_by_bits));
+ uint16x4_t d2 =
+ vget_low_u16(vmlal_u8(round_offset_vec, s2, shift_by_bits));
+ uint16x4_t d3 =
+ vget_low_u16(vmlal_u8(round_offset_vec, s3, shift_by_bits));
+
+ uint16x4_t dd0, dd1, dd2, dd3;
+ load_u16_4x4(dst, dst_stride, &dd0, &dd1, &dd2, &dd3);
+
+ uint8x8_t d01, d23;
+ compute_dist_wtd_avg_4x4(
+ dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset, bck_offset,
+ vreinterpretq_s16_u16(round_offset_vec), &d01, &d23);
+
+ store_u8_4x1(dst8 + 0 * dst8_stride, d01, 0);
+ store_u8_4x1(dst8 + 1 * dst8_stride, d01, 1);
+ store_u8_4x1(dst8 + 2 * dst8_stride, d23, 0);
+ store_u8_4x1(dst8 + 3 * dst8_stride, d23, 1);
+
+ src += 4 * src_stride;
+ dst += 4 * dst_stride;
+ dst8 += 4 * dst8_stride;
+ height -= 4;
+ } while (height != 0);
+ } else {
+ do {
+ const uint8_t *s = src;
+ CONV_BUF_TYPE *d = dst;
+ uint8_t *d_u8 = dst8;
+ int width = w;
+
+ do {
+ uint8x8_t s0, s1, s2, s3;
+ load_u8_8x4(s, src_stride, &s0, &s1, &s2, &s3);
+
+ uint16x8_t d0 = vmlal_u8(round_offset_vec, s0, shift_by_bits);
+ uint16x8_t d1 = vmlal_u8(round_offset_vec, s1, shift_by_bits);
+ uint16x8_t d2 = vmlal_u8(round_offset_vec, s2, shift_by_bits);
+ uint16x8_t d3 = vmlal_u8(round_offset_vec, s3, shift_by_bits);
+
+ uint16x8_t dd0, dd1, dd2, dd3;
+ load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
+
+ uint8x8_t d0_u8, d1_u8, d2_u8, d3_u8;
+ compute_dist_wtd_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset,
+ bck_offset,
+ vreinterpretq_s16_u16(round_offset_vec),
+ &d0_u8, &d1_u8, &d2_u8, &d3_u8);
+
+ store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8);
+
+ s += 8;
+ d += 8;
+ d_u8 += 8;
+ width -= 8;
+ } while (width != 0);
+ src += 4 * src_stride;
+ dst += 4 * dst_stride;
+ dst8 += 4 * dst8_stride;
+ height -= 4;
+ } while (height != 0);
+ }
+}
+
+static INLINE void dist_wtd_convolve_2d_copy_avg_neon(
+ const uint8_t *src, int src_stride, uint8_t *dst8, int dst8_stride, int w,
+ int h, ConvolveParams *conv_params) {
+ assert(w % 4 == 0);
+ assert(h % 4 == 0);
+
+ const int bd = 8;
+ const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
+ const uint16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
+ (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
+ const uint16x8_t round_offset_vec = vdupq_n_u16(round_offset);
+ const uint8x8_t shift_by_bits = vdup_n_u8(1 << (FILTER_BITS - ROUND0_BITS));
+
+ CONV_BUF_TYPE *dst = conv_params->dst;
+ const int dst_stride = conv_params->dst_stride;
+ int height = h;
+
+ if (w == 4) {
+ do {
+ uint8x8_t s0, s1, s2, s3;
+ load_u8_8x4(src, src_stride, &s0, &s1, &s2, &s3);
+
+ uint16x4_t d0 =
+ vget_low_u16(vmlal_u8(round_offset_vec, s0, shift_by_bits));
+ uint16x4_t d1 =
+ vget_low_u16(vmlal_u8(round_offset_vec, s1, shift_by_bits));
+ uint16x4_t d2 =
+ vget_low_u16(vmlal_u8(round_offset_vec, s2, shift_by_bits));
+ uint16x4_t d3 =
+ vget_low_u16(vmlal_u8(round_offset_vec, s3, shift_by_bits));
+
+ uint16x4_t dd0, dd1, dd2, dd3;
+ load_u16_4x4(dst, dst_stride, &dd0, &dd1, &dd2, &dd3);
+
+ uint8x8_t d01, d23;
+ compute_basic_avg_4x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3,
+ vreinterpretq_s16_u16(round_offset_vec), &d01,
+ &d23);
+
+ store_u8_4x1(dst8 + 0 * dst8_stride, d01, 0);
+ store_u8_4x1(dst8 + 1 * dst8_stride, d01, 1);
+ store_u8_4x1(dst8 + 2 * dst8_stride, d23, 0);
+ store_u8_4x1(dst8 + 3 * dst8_stride, d23, 1);
+
+ src += 4 * src_stride;
+ dst += 4 * dst_stride;
+ dst8 += 4 * dst8_stride;
+ height -= 4;
+ } while (height != 0);
+ } else {
+ do {
+ const uint8_t *s = src;
+ CONV_BUF_TYPE *d = dst;
+ uint8_t *d_u8 = dst8;
+ int width = w;
+
+ do {
+ uint8x8_t s0, s1, s2, s3;
+ load_u8_8x4(s, src_stride, &s0, &s1, &s2, &s3);
+
+ uint16x8_t d0 = vmlal_u8(round_offset_vec, s0, shift_by_bits);
+ uint16x8_t d1 = vmlal_u8(round_offset_vec, s1, shift_by_bits);
+ uint16x8_t d2 = vmlal_u8(round_offset_vec, s2, shift_by_bits);
+ uint16x8_t d3 = vmlal_u8(round_offset_vec, s3, shift_by_bits);
+
+ uint16x8_t dd0, dd1, dd2, dd3;
+ load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
+
+ uint8x8_t d0_u8, d1_u8, d2_u8, d3_u8;
+ compute_basic_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3,
+ vreinterpretq_s16_u16(round_offset_vec), &d0_u8,
+ &d1_u8, &d2_u8, &d3_u8);
+
+ store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8);
+
+ s += 8;
+ d += 8;
+ d_u8 += 8;
+ width -= 8;
+ } while (width != 0);
+ src += 4 * src_stride;
+ dst += 4 * dst_stride;
+ dst8 += 4 * dst8_stride;
+ height -= 4;
+ } while (height != 0);
+ }
+}
+
+static INLINE void dist_wtd_convolve_2d_copy_neon(const uint8_t *src,
+ int src_stride, int w, int h,
+ ConvolveParams *conv_params) {
+ assert(w % 4 == 0);
+ assert(h % 4 == 0);
+
+ const int bd = 8;
+ const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
+ const uint16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
+ (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
+ const uint16x8_t round_offset_vec = vdupq_n_u16(round_offset);
+ const uint8x8_t shift_by_bits = vdup_n_u8(1 << (FILTER_BITS - ROUND0_BITS));
+
+ CONV_BUF_TYPE *dst = conv_params->dst;
+ const int dst_stride = conv_params->dst_stride;
+ int height = h;
+
+ if (w == 4) {
+ do {
+ uint8x8_t s0, s1, s2, s3;
+ load_u8_8x4(src, src_stride, &s0, &s1, &s2, &s3);
+
+ uint16x4_t d0 =
+ vget_low_u16(vmlal_u8(round_offset_vec, s0, shift_by_bits));
+ uint16x4_t d1 =
+ vget_low_u16(vmlal_u8(round_offset_vec, s1, shift_by_bits));
+ uint16x4_t d2 =
+ vget_low_u16(vmlal_u8(round_offset_vec, s2, shift_by_bits));
+ uint16x4_t d3 =
+ vget_low_u16(vmlal_u8(round_offset_vec, s3, shift_by_bits));
+
+ store_u16_4x4(dst, dst_stride, d0, d1, d2, d3);
+
+ src += 4 * src_stride;
+ dst += 4 * dst_stride;
+ height -= 4;
+ } while (height != 0);
+ } else {
+ do {
+ const uint8_t *s = src;
+ CONV_BUF_TYPE *d = dst;
+ int width = w;
+
+ do {
+ uint8x8_t s0, s1, s2, s3;
+ load_u8_8x4(s, src_stride, &s0, &s1, &s2, &s3);
+
+ uint16x8_t d0 = vmlal_u8(round_offset_vec, s0, shift_by_bits);
+ uint16x8_t d1 = vmlal_u8(round_offset_vec, s1, shift_by_bits);
+ uint16x8_t d2 = vmlal_u8(round_offset_vec, s2, shift_by_bits);
+ uint16x8_t d3 = vmlal_u8(round_offset_vec, s3, shift_by_bits);
+
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src += 4 * src_stride;
+ dst += 4 * dst_stride;
+ height -= 4;
+ } while (height != 0);
+ }
+}
+
+void av1_dist_wtd_convolve_2d_copy_neon(const uint8_t *src, int src_stride,
+ uint8_t *dst8, int dst8_stride, int w,
+ int h, ConvolveParams *conv_params) {
+ if (conv_params->do_average) {
+ if (UNLIKELY(conv_params->use_dist_wtd_comp_avg)) {
+ dist_wtd_convolve_2d_copy_dist_wtd_avg_neon(
+ src, src_stride, dst8, dst8_stride, w, h, conv_params);
+ } else {
+ dist_wtd_convolve_2d_copy_avg_neon(src, src_stride, dst8, dst8_stride, w,
+ h, conv_params);
+ }
+ } else {
+ dist_wtd_convolve_2d_copy_neon(src, src_stride, w, h, conv_params);
+ }
+}
+
+static INLINE uint16x4_t convolve4_4_x(const int16x4_t s0, const int16x4_t s1,
+ const int16x4_t s2, const int16x4_t s3,
+ const int16x4_t x_filter,
+ const int16x4_t round_offset) {
+ int16x4_t sum = vmul_lane_s16(s0, x_filter, 0);
+ sum = vmla_lane_s16(sum, s1, x_filter, 1);
+ sum = vmla_lane_s16(sum, s2, x_filter, 2);
+ sum = vmla_lane_s16(sum, s3, x_filter, 3);
+
+ // We halved the convolution filter values so -1 from the right shift.
+ int16x4_t res = vrsra_n_s16(round_offset, sum, ROUND0_BITS - 1);
+ return vreinterpret_u16_s16(res);
+}
+
+static INLINE uint16x8_t convolve8_8_x(const int16x8_t s0, const int16x8_t s1,
+ const int16x8_t s2, const int16x8_t s3,
+ const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t s6, const int16x8_t s7,
+ const int16x8_t x_filter,
+ const int16x8_t round_offset) {
+ const int16x4_t x_filter_0_3 = vget_low_s16(x_filter);
+ const int16x4_t x_filter_4_7 = vget_high_s16(x_filter);
+
+ int16x8_t sum = vmulq_lane_s16(s0, x_filter_0_3, 0);
+ sum = vmlaq_lane_s16(sum, s1, x_filter_0_3, 1);
+ sum = vmlaq_lane_s16(sum, s2, x_filter_0_3, 2);
+ sum = vmlaq_lane_s16(sum, s3, x_filter_0_3, 3);
+ sum = vmlaq_lane_s16(sum, s4, x_filter_4_7, 0);
+ sum = vmlaq_lane_s16(sum, s5, x_filter_4_7, 1);
+ sum = vmlaq_lane_s16(sum, s6, x_filter_4_7, 2);
+ sum = vmlaq_lane_s16(sum, s7, x_filter_4_7, 3);
+
+ // We halved the convolution filter values so -1 from the right shift.
+ int16x8_t res = vrsraq_n_s16(round_offset, sum, ROUND0_BITS - 1);
+ return vreinterpretq_u16_s16(res);
+}
+
+static INLINE void dist_wtd_convolve_x_dist_wtd_avg_neon(
+ const uint8_t *src, int src_stride, uint8_t *dst8, int dst8_stride, int w,
+ int h, const InterpFilterParams *filter_params_x, const int subpel_x_qn,
+ ConvolveParams *conv_params) {
+ assert(w % 4 == 0);
+ assert(h % 4 == 0);
+
+ const int bd = 8;
+ const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
+ const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
+ (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
+ const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
+
+ const uint16_t fwd_offset = conv_params->fwd_offset;
+ const uint16_t bck_offset = conv_params->bck_offset;
+
+ // Horizontal filter.
+ const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
+ filter_params_x, subpel_x_qn & SUBPEL_MASK);
+
+ const int horiz_offset = filter_params_x->taps / 2 - 1;
+ const uint8_t *src_ptr = src - horiz_offset;
+ CONV_BUF_TYPE *dst_ptr = conv_params->dst;
+ uint8_t *dst8_ptr = dst8;
+ int dst_stride = conv_params->dst_stride;
+ int height = h;
+
+ if (w == 4) {
+ // 4-tap filters are used for blocks having width <= 4.
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int16x4_t x_filter = vshr_n_s16(vld1_s16(x_filter_ptr + 2), 1);
+
+ src_ptr += 2;
+
+ do {
+ uint8x8_t t0 = vld1_u8(src_ptr); // a0 a1 a2 a3 a4 a5 a6 a7
+ int16x4_t s0 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
+ int16x4_t s4 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
+
+ __builtin_prefetch(dst_ptr);
+ __builtin_prefetch(dst8_ptr);
+
+ int16x4_t s1 = vext_s16(s0, s4, 1); // a1 a2 a3 a4
+ int16x4_t s2 = vext_s16(s0, s4, 2); // a2 a3 a4 a5
+ int16x4_t s3 = vext_s16(s0, s4, 3); // a3 a4 a5 a6
+
+ uint16x4_t d0 = convolve4_4_x(s0, s1, s2, s3, x_filter,
+ vget_low_s16(round_offset_vec));
+
+ uint16x4_t dd0 = vld1_u16(dst_ptr);
+
+ uint8x8_t d01;
+ compute_dist_wtd_avg_4x1(dd0, d0, fwd_offset, bck_offset,
+ vget_low_s16(round_offset_vec), &d01);
+
+ store_u8_4x1(dst8_ptr, d01, 0);
+
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ dst8_ptr += dst8_stride;
+ } while (--height != 0);
+ } else {
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int16x8_t x_filter = vshrq_n_s16(vld1q_s16(x_filter_ptr), 1);
+
+#if AOM_ARCH_AARCH64
+ while (height >= 8) {
+ const uint8_t *s = src_ptr;
+ CONV_BUF_TYPE *d = dst_ptr;
+ uint8_t *d_u8 = dst8_ptr;
+ int width = w;
+
+ uint8x8_t t0, t1, t2, t3, t4, t5, t6, t7;
+ load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
+ transpose_elems_inplace_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
+
+ int16x8_t s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
+ int16x8_t s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
+ int16x8_t s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
+ int16x8_t s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
+ int16x8_t s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
+ int16x8_t s5 = vreinterpretq_s16_u16(vmovl_u8(t5));
+ int16x8_t s6 = vreinterpretq_s16_u16(vmovl_u8(t6));
+
+ __builtin_prefetch(d + 0 * dst_stride);
+ __builtin_prefetch(d + 1 * dst_stride);
+ __builtin_prefetch(d + 2 * dst_stride);
+ __builtin_prefetch(d + 3 * dst_stride);
+ __builtin_prefetch(d + 4 * dst_stride);
+ __builtin_prefetch(d + 5 * dst_stride);
+ __builtin_prefetch(d + 6 * dst_stride);
+ __builtin_prefetch(d + 7 * dst_stride);
+
+ s += 7;
+
+ do {
+ load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
+ transpose_elems_inplace_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
+
+ int16x8_t s7 = vreinterpretq_s16_u16(vmovl_u8(t0));
+ int16x8_t s8 = vreinterpretq_s16_u16(vmovl_u8(t1));
+ int16x8_t s9 = vreinterpretq_s16_u16(vmovl_u8(t2));
+ int16x8_t s10 = vreinterpretq_s16_u16(vmovl_u8(t3));
+ int16x8_t s11 = vreinterpretq_s16_u16(vmovl_u8(t4));
+ int16x8_t s12 = vreinterpretq_s16_u16(vmovl_u8(t5));
+ int16x8_t s13 = vreinterpretq_s16_u16(vmovl_u8(t6));
+ int16x8_t s14 = vreinterpretq_s16_u16(vmovl_u8(t7));
+
+ uint16x8_t d0 = convolve8_8_x(s0, s1, s2, s3, s4, s5, s6, s7, x_filter,
+ round_offset_vec);
+ uint16x8_t d1 = convolve8_8_x(s1, s2, s3, s4, s5, s6, s7, s8, x_filter,
+ round_offset_vec);
+ uint16x8_t d2 = convolve8_8_x(s2, s3, s4, s5, s6, s7, s8, s9, x_filter,
+ round_offset_vec);
+ uint16x8_t d3 = convolve8_8_x(s3, s4, s5, s6, s7, s8, s9, s10, x_filter,
+ round_offset_vec);
+ uint16x8_t d4 = convolve8_8_x(s4, s5, s6, s7, s8, s9, s10, s11,
+ x_filter, round_offset_vec);
+ uint16x8_t d5 = convolve8_8_x(s5, s6, s7, s8, s9, s10, s11, s12,
+ x_filter, round_offset_vec);
+ uint16x8_t d6 = convolve8_8_x(s6, s7, s8, s9, s10, s11, s12, s13,
+ x_filter, round_offset_vec);
+ uint16x8_t d7 = convolve8_8_x(s7, s8, s9, s10, s11, s12, s13, s14,
+ x_filter, round_offset_vec);
+
+ transpose_elems_inplace_u16_8x8(&d0, &d1, &d2, &d3, &d4, &d5, &d6, &d7);
+
+ uint16x8_t dd0, dd1, dd2, dd3;
+ load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
+
+ uint8x8_t d0_u8, d1_u8, d2_u8, d3_u8;
+ compute_dist_wtd_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset,
+ bck_offset, round_offset_vec, &d0_u8, &d1_u8,
+ &d2_u8, &d3_u8);
+
+ store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8);
+
+ uint16x8_t dd4, dd5, dd6, dd7;
+ load_u16_8x4(d + 4 * dst_stride, dst_stride, &dd4, &dd5, &dd6, &dd7);
+
+ uint8x8_t d4_u8, d5_u8, d6_u8, d7_u8;
+ compute_dist_wtd_avg_8x4(dd4, dd5, dd6, dd7, d4, d5, d6, d7, fwd_offset,
+ bck_offset, round_offset_vec, &d4_u8, &d5_u8,
+ &d6_u8, &d7_u8);
+
+ store_u8_8x4(d_u8 + 4 * dst8_stride, dst8_stride, d4_u8, d5_u8, d6_u8,
+ d7_u8);
+
+ s0 = s8;
+ s1 = s9;
+ s2 = s10;
+ s3 = s11;
+ s4 = s12;
+ s5 = s13;
+ s6 = s14;
+ s += 8;
+ d += 8;
+ d_u8 += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += 8 * src_stride;
+ dst_ptr += 8 * dst_stride;
+ dst8_ptr += 8 * dst8_stride;
+ height -= 8;
+ }
+#endif // AOM_ARCH_AARCH64
+
+ while (height > 0) {
+ const uint8_t *s = src_ptr;
+ CONV_BUF_TYPE *d = dst_ptr;
+ uint8_t *d_u8 = dst8_ptr;
+ int width = w;
+
+ uint8x8_t t0 = vld1_u8(s); // a0 a1 a2 a3 a4 a5 a6 a7
+ int16x8_t s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
+
+ __builtin_prefetch(d);
+
+ s += 8;
+
+ do {
+ t0 = vld1_u8(s); // a8 a9 a10 a11 a12 a13 a14 a15
+ int16x8_t s8 = vreinterpretq_s16_u16(vmovl_u8(t0));
+
+ int16x8_t s1 = vextq_s16(s0, s8, 1); // a1 a2 a3 a4 a5 a6 a7 a8
+ int16x8_t s2 = vextq_s16(s0, s8, 2); // a2 a3 a4 a5 a6 a7 a8 a9
+ int16x8_t s3 = vextq_s16(s0, s8, 3); // a3 a4 a5 a6 a7 a8 a9 a10
+ int16x8_t s4 = vextq_s16(s0, s8, 4); // a4 a5 a6 a7 a8 a9 a10 a11
+ int16x8_t s5 = vextq_s16(s0, s8, 5); // a5 a6 a7 a8 a9 a10 a11 a12
+ int16x8_t s6 = vextq_s16(s0, s8, 6); // a6 a7 a8 a9 a10 a11 a12 a13
+ int16x8_t s7 = vextq_s16(s0, s8, 7); // a7 a8 a9 a10 a11 a12 a13 a14
+
+ uint16x8_t d0 = convolve8_8_x(s0, s1, s2, s3, s4, s5, s6, s7, x_filter,
+ round_offset_vec);
+
+ uint16x8_t dd0 = vld1q_u16(d);
+
+ uint8x8_t d0_u8;
+ compute_dist_wtd_avg_8x1(dd0, d0, fwd_offset, bck_offset,
+ round_offset_vec, &d0_u8);
+
+ vst1_u8(d_u8, d0_u8);
+
+ s0 = s8;
+ s += 8;
+ d += 8;
+ d_u8 += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ dst8_ptr += dst8_stride;
+ height--;
+ }
+ }
+}
+
+static INLINE void dist_wtd_convolve_x_avg_neon(
+ const uint8_t *src, int src_stride, uint8_t *dst8, int dst8_stride, int w,
+ int h, const InterpFilterParams *filter_params_x, const int subpel_x_qn,
+ ConvolveParams *conv_params) {
+ assert(w % 4 == 0);
+ assert(h % 4 == 0);
+
+ const int bd = 8;
+ const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
+ const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
+ (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
+ const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
+
+ // Horizontal filter.
+ const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
+ filter_params_x, subpel_x_qn & SUBPEL_MASK);
+
+ const int horiz_offset = filter_params_x->taps / 2 - 1;
+ const uint8_t *src_ptr = src - horiz_offset;
+ CONV_BUF_TYPE *dst_ptr = conv_params->dst;
+ uint8_t *dst8_ptr = dst8;
+ int dst_stride = conv_params->dst_stride;
+ int height = h;
+
+ if (w == 4) {
+ // 4-tap filters are used for blocks having width <= 4.
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int16x4_t x_filter = vshr_n_s16(vld1_s16(x_filter_ptr + 2), 1);
+
+ src_ptr += 2;
+
+ do {
+ uint8x8_t t0 = vld1_u8(src_ptr); // a0 a1 a2 a3 a4 a5 a6 a7
+ int16x4_t s0 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
+ int16x4_t s4 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
+
+ __builtin_prefetch(dst_ptr);
+ __builtin_prefetch(dst8_ptr);
+
+ int16x4_t s1 = vext_s16(s0, s4, 1); // a1 a2 a3 a4
+ int16x4_t s2 = vext_s16(s0, s4, 2); // a2 a3 a4 a5
+ int16x4_t s3 = vext_s16(s0, s4, 3); // a3 a4 a5 a6
+
+ uint16x4_t d0 = convolve4_4_x(s0, s1, s2, s3, x_filter,
+ vget_low_s16(round_offset_vec));
+
+ uint16x4_t dd0 = vld1_u16(dst_ptr);
+
+ uint8x8_t d01;
+ compute_basic_avg_4x1(dd0, d0, vget_low_s16(round_offset_vec), &d01);
+
+ store_u8_4x1(dst8_ptr, d01, 0);
+
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ dst8_ptr += dst8_stride;
+ } while (--height != 0);
+ } else {
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int16x8_t x_filter = vshrq_n_s16(vld1q_s16(x_filter_ptr), 1);
+
+#if AOM_ARCH_AARCH64
+ while (height >= 8) {
+ const uint8_t *s = src_ptr;
+ CONV_BUF_TYPE *d = dst_ptr;
+ uint8_t *d_u8 = dst8_ptr;
+ int width = w;
+
+ uint8x8_t t0, t1, t2, t3, t4, t5, t6, t7;
+ load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
+ transpose_elems_inplace_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
+
+ int16x8_t s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
+ int16x8_t s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
+ int16x8_t s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
+ int16x8_t s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
+ int16x8_t s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
+ int16x8_t s5 = vreinterpretq_s16_u16(vmovl_u8(t5));
+ int16x8_t s6 = vreinterpretq_s16_u16(vmovl_u8(t6));
+
+ __builtin_prefetch(d + 0 * dst_stride);
+ __builtin_prefetch(d + 1 * dst_stride);
+ __builtin_prefetch(d + 2 * dst_stride);
+ __builtin_prefetch(d + 3 * dst_stride);
+ __builtin_prefetch(d + 4 * dst_stride);
+ __builtin_prefetch(d + 5 * dst_stride);
+ __builtin_prefetch(d + 6 * dst_stride);
+ __builtin_prefetch(d + 7 * dst_stride);
+
+ s += 7;
+
+ do {
+ load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
+ transpose_elems_inplace_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
+
+ int16x8_t s7 = vreinterpretq_s16_u16(vmovl_u8(t0));
+ int16x8_t s8 = vreinterpretq_s16_u16(vmovl_u8(t1));
+ int16x8_t s9 = vreinterpretq_s16_u16(vmovl_u8(t2));
+ int16x8_t s10 = vreinterpretq_s16_u16(vmovl_u8(t3));
+ int16x8_t s11 = vreinterpretq_s16_u16(vmovl_u8(t4));
+ int16x8_t s12 = vreinterpretq_s16_u16(vmovl_u8(t5));
+ int16x8_t s13 = vreinterpretq_s16_u16(vmovl_u8(t6));
+ int16x8_t s14 = vreinterpretq_s16_u16(vmovl_u8(t7));
+
+ uint16x8_t d0 = convolve8_8_x(s0, s1, s2, s3, s4, s5, s6, s7, x_filter,
+ round_offset_vec);
+ uint16x8_t d1 = convolve8_8_x(s1, s2, s3, s4, s5, s6, s7, s8, x_filter,
+ round_offset_vec);
+ uint16x8_t d2 = convolve8_8_x(s2, s3, s4, s5, s6, s7, s8, s9, x_filter,
+ round_offset_vec);
+ uint16x8_t d3 = convolve8_8_x(s3, s4, s5, s6, s7, s8, s9, s10, x_filter,
+ round_offset_vec);
+ uint16x8_t d4 = convolve8_8_x(s4, s5, s6, s7, s8, s9, s10, s11,
+ x_filter, round_offset_vec);
+ uint16x8_t d5 = convolve8_8_x(s5, s6, s7, s8, s9, s10, s11, s12,
+ x_filter, round_offset_vec);
+ uint16x8_t d6 = convolve8_8_x(s6, s7, s8, s9, s10, s11, s12, s13,
+ x_filter, round_offset_vec);
+ uint16x8_t d7 = convolve8_8_x(s7, s8, s9, s10, s11, s12, s13, s14,
+ x_filter, round_offset_vec);
+
+ transpose_elems_inplace_u16_8x8(&d0, &d1, &d2, &d3, &d4, &d5, &d6, &d7);
+
+ uint16x8_t dd0, dd1, dd2, dd3;
+ load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
+
+ uint8x8_t d0_u8, d1_u8, d2_u8, d3_u8;
+ compute_basic_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3,
+ round_offset_vec, &d0_u8, &d1_u8, &d2_u8, &d3_u8);
+
+ store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8);
+
+ uint16x8_t dd4, dd5, dd6, dd7;
+ load_u16_8x4(d + 4 * dst_stride, dst_stride, &dd4, &dd5, &dd6, &dd7);
+
+ uint8x8_t d4_u8, d5_u8, d6_u8, d7_u8;
+ compute_basic_avg_8x4(dd4, dd5, dd6, dd7, d4, d5, d6, d7,
+ round_offset_vec, &d4_u8, &d5_u8, &d6_u8, &d7_u8);
+
+ store_u8_8x4(d_u8 + 4 * dst8_stride, dst8_stride, d4_u8, d5_u8, d6_u8,
+ d7_u8);
+
+ s0 = s8;
+ s1 = s9;
+ s2 = s10;
+ s3 = s11;
+ s4 = s12;
+ s5 = s13;
+ s6 = s14;
+ s += 8;
+ d += 8;
+ d_u8 += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += 8 * src_stride;
+ dst_ptr += 8 * dst_stride;
+ dst8_ptr += 8 * dst8_stride;
+ height -= 8;
+ }
+#endif // AOM_ARCH_AARCH64
+
+ while (height > 0) {
+ const uint8_t *s = src_ptr;
+ CONV_BUF_TYPE *d = dst_ptr;
+ uint8_t *d_u8 = dst8_ptr;
+ int width = w;
+
+ uint8x8_t t0 = vld1_u8(s); // a0 a1 a2 a3 a4 a5 a6 a7
+ int16x8_t s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
+
+ __builtin_prefetch(d);
+
+ s += 8;
+
+ do {
+ t0 = vld1_u8(s); // a8 a9 a10 a11 a12 a13 a14 a15
+ int16x8_t s8 = vreinterpretq_s16_u16(vmovl_u8(t0));
+
+ int16x8_t s1 = vextq_s16(s0, s8, 1); // a1 a2 a3 a4 a5 a6 a7 a8
+ int16x8_t s2 = vextq_s16(s0, s8, 2); // a2 a3 a4 a5 a6 a7 a8 a9
+ int16x8_t s3 = vextq_s16(s0, s8, 3); // a3 a4 a5 a6 a7 a8 a9 a10
+ int16x8_t s4 = vextq_s16(s0, s8, 4); // a4 a5 a6 a7 a8 a9 a10 a11
+ int16x8_t s5 = vextq_s16(s0, s8, 5); // a5 a6 a7 a8 a9 a10 a11 a12
+ int16x8_t s6 = vextq_s16(s0, s8, 6); // a6 a7 a8 a9 a10 a11 a12 a13
+ int16x8_t s7 = vextq_s16(s0, s8, 7); // a7 a8 a9 a10 a11 a12 a13 a14
+
+ uint16x8_t d0 = convolve8_8_x(s0, s1, s2, s3, s4, s5, s6, s7, x_filter,
+ round_offset_vec);
+
+ uint16x8_t dd0 = vld1q_u16(d);
+
+ uint8x8_t d0_u8;
+ compute_basic_avg_8x1(dd0, d0, round_offset_vec, &d0_u8);
+
+ vst1_u8(d_u8, d0_u8);
+
+ s0 = s8;
+ s += 8;
+ d += 8;
+ d_u8 += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ dst8_ptr += dst8_stride;
+ height--;
+ }
+ }
+}
+
+static INLINE void dist_wtd_convolve_x_neon(
+ const uint8_t *src, int src_stride, int w, int h,
+ const InterpFilterParams *filter_params_x, const int subpel_x_qn,
+ ConvolveParams *conv_params) {
+ assert(w % 4 == 0);
+ assert(h % 4 == 0);
+
+ const int bd = 8;
+ const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
+ const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
+ (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
+ const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
+
+ // Horizontal filter.
+ const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
+ filter_params_x, subpel_x_qn & SUBPEL_MASK);
+
+ const int horiz_offset = filter_params_x->taps / 2 - 1;
+ const uint8_t *src_ptr = src - horiz_offset;
+ CONV_BUF_TYPE *dst_ptr = conv_params->dst;
+ int dst_stride = conv_params->dst_stride;
+ int height = h;
+
+ if (w == 4) {
+ // 4-tap filters are used for blocks having width <= 4.
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int16x4_t x_filter = vshr_n_s16(vld1_s16(x_filter_ptr + 2), 1);
+
+ src_ptr += 2;
+
+ do {
+ uint8x8_t t0 = vld1_u8(src_ptr); // a0 a1 a2 a3 a4 a5 a6 a7
+ int16x4_t s0 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
+ int16x4_t s4 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
+
+ __builtin_prefetch(dst_ptr);
+
+ int16x4_t s1 = vext_s16(s0, s4, 1); // a1 a2 a3 a4
+ int16x4_t s2 = vext_s16(s0, s4, 2); // a2 a3 a4 a5
+ int16x4_t s3 = vext_s16(s0, s4, 3); // a3 a4 a5 a6
+
+ uint16x4_t d0 = convolve4_4_x(s0, s1, s2, s3, x_filter,
+ vget_low_s16(round_offset_vec));
+
+ vst1_u16(dst_ptr, d0);
+
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ } while (--height != 0);
+ } else {
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int16x8_t x_filter = vshrq_n_s16(vld1q_s16(x_filter_ptr), 1);
+
+#if AOM_ARCH_AARCH64
+ while (height >= 8) {
+ const uint8_t *s = src_ptr;
+ CONV_BUF_TYPE *d = dst_ptr;
+ int width = w;
+
+ uint8x8_t t0, t1, t2, t3, t4, t5, t6, t7;
+ load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
+ transpose_elems_inplace_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
+
+ int16x8_t s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
+ int16x8_t s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
+ int16x8_t s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
+ int16x8_t s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
+ int16x8_t s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
+ int16x8_t s5 = vreinterpretq_s16_u16(vmovl_u8(t5));
+ int16x8_t s6 = vreinterpretq_s16_u16(vmovl_u8(t6));
+
+ __builtin_prefetch(d + 0 * dst_stride);
+ __builtin_prefetch(d + 1 * dst_stride);
+ __builtin_prefetch(d + 2 * dst_stride);
+ __builtin_prefetch(d + 3 * dst_stride);
+ __builtin_prefetch(d + 4 * dst_stride);
+ __builtin_prefetch(d + 5 * dst_stride);
+ __builtin_prefetch(d + 6 * dst_stride);
+ __builtin_prefetch(d + 7 * dst_stride);
+
+ s += 7;
+
+ do {
+ load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
+ transpose_elems_inplace_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
+
+ int16x8_t s7 = vreinterpretq_s16_u16(vmovl_u8(t0));
+ int16x8_t s8 = vreinterpretq_s16_u16(vmovl_u8(t1));
+ int16x8_t s9 = vreinterpretq_s16_u16(vmovl_u8(t2));
+ int16x8_t s10 = vreinterpretq_s16_u16(vmovl_u8(t3));
+ int16x8_t s11 = vreinterpretq_s16_u16(vmovl_u8(t4));
+ int16x8_t s12 = vreinterpretq_s16_u16(vmovl_u8(t5));
+ int16x8_t s13 = vreinterpretq_s16_u16(vmovl_u8(t6));
+ int16x8_t s14 = vreinterpretq_s16_u16(vmovl_u8(t7));
+
+ uint16x8_t d0 = convolve8_8_x(s0, s1, s2, s3, s4, s5, s6, s7, x_filter,
+ round_offset_vec);
+ uint16x8_t d1 = convolve8_8_x(s1, s2, s3, s4, s5, s6, s7, s8, x_filter,
+ round_offset_vec);
+ uint16x8_t d2 = convolve8_8_x(s2, s3, s4, s5, s6, s7, s8, s9, x_filter,
+ round_offset_vec);
+ uint16x8_t d3 = convolve8_8_x(s3, s4, s5, s6, s7, s8, s9, s10, x_filter,
+ round_offset_vec);
+ uint16x8_t d4 = convolve8_8_x(s4, s5, s6, s7, s8, s9, s10, s11,
+ x_filter, round_offset_vec);
+ uint16x8_t d5 = convolve8_8_x(s5, s6, s7, s8, s9, s10, s11, s12,
+ x_filter, round_offset_vec);
+ uint16x8_t d6 = convolve8_8_x(s6, s7, s8, s9, s10, s11, s12, s13,
+ x_filter, round_offset_vec);
+ uint16x8_t d7 = convolve8_8_x(s7, s8, s9, s10, s11, s12, s13, s14,
+ x_filter, round_offset_vec);
+
+ transpose_elems_inplace_u16_8x8(&d0, &d1, &d2, &d3, &d4, &d5, &d6, &d7);
+
+ store_u16_8x8(d, dst_stride, d0, d1, d2, d3, d4, d5, d6, d7);
+
+ s0 = s8;
+ s1 = s9;
+ s2 = s10;
+ s3 = s11;
+ s4 = s12;
+ s5 = s13;
+ s6 = s14;
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += 8 * src_stride;
+ dst_ptr += 8 * dst_stride;
+ height -= 8;
+ }
+#endif // AOM_ARCH_AARCH64
+
+ while (height > 0) {
+ const uint8_t *s = src_ptr;
+ CONV_BUF_TYPE *d = dst_ptr;
+ int width = w;
+
+ uint8x8_t t0 = vld1_u8(s); // a0 a1 a2 a3 a4 a5 a6 a7
+ int16x8_t s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
+
+ __builtin_prefetch(d);
+
+ s = src_ptr + 8;
+
+ do {
+ t0 = vld1_u8(s); // a8 a9 a10 a11 a12 a13 a14 a15
+ int16x8_t s8 = vreinterpretq_s16_u16(vmovl_u8(t0));
+
+ int16x8_t s1 = vextq_s16(s0, s8, 1); // a1 a2 a3 a4 a5 a6 a7 a8
+ int16x8_t s2 = vextq_s16(s0, s8, 2); // a2 a3 a4 a5 a6 a7 a8 a9
+ int16x8_t s3 = vextq_s16(s0, s8, 3); // a3 a4 a5 a6 a7 a8 a9 a10
+ int16x8_t s4 = vextq_s16(s0, s8, 4); // a4 a5 a6 a7 a8 a9 a10 a11
+ int16x8_t s5 = vextq_s16(s0, s8, 5); // a5 a6 a7 a8 a9 a10 a11 a12
+ int16x8_t s6 = vextq_s16(s0, s8, 6); // a6 a7 a8 a9 a10 a11 a12 a13
+ int16x8_t s7 = vextq_s16(s0, s8, 7); // a7 a8 a9 a10 a11 a12 a13 a14
+
+ uint16x8_t d0 = convolve8_8_x(s0, s1, s2, s3, s4, s5, s6, s7, x_filter,
+ round_offset_vec);
+
+ vst1q_u16(d, d0);
+
+ s0 = s8;
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ height--;
+ }
+ }
+}
+
+void av1_dist_wtd_convolve_x_neon(const uint8_t *src, int src_stride,
+ uint8_t *dst8, int dst8_stride, int w, int h,
+ const InterpFilterParams *filter_params_x,
+ const int subpel_x_qn,
+ ConvolveParams *conv_params) {
+ if (conv_params->do_average) {
+ if (UNLIKELY(conv_params->use_dist_wtd_comp_avg)) {
+ dist_wtd_convolve_x_dist_wtd_avg_neon(src, src_stride, dst8, dst8_stride,
+ w, h, filter_params_x, subpel_x_qn,
+ conv_params);
+ } else {
+ dist_wtd_convolve_x_avg_neon(src, src_stride, dst8, dst8_stride, w, h,
+ filter_params_x, subpel_x_qn, conv_params);
+ }
+ } else {
+ dist_wtd_convolve_x_neon(src, src_stride, w, h, filter_params_x,
+ subpel_x_qn, conv_params);
+ }
+}
+
+static INLINE uint16x4_t convolve6_4_y(const int16x4_t s0, const int16x4_t s1,
+ const int16x4_t s2, const int16x4_t s3,
+ const int16x4_t s4, const int16x4_t s5,
+ const int16x8_t y_filter,
+ const int16x4_t round_offset) {
+ const int16x4_t y_filter_0_3 = vget_low_s16(y_filter);
+ const int16x4_t y_filter_4_7 = vget_high_s16(y_filter);
+
+ // Filter values at indices 0 and 7 are 0.
+ int16x4_t sum = vmul_lane_s16(s0, y_filter_0_3, 1);
+ sum = vmla_lane_s16(sum, s1, y_filter_0_3, 2);
+ sum = vmla_lane_s16(sum, s2, y_filter_0_3, 3);
+ sum = vmla_lane_s16(sum, s3, y_filter_4_7, 0);
+ sum = vmla_lane_s16(sum, s4, y_filter_4_7, 1);
+ sum = vmla_lane_s16(sum, s5, y_filter_4_7, 2);
+
+ // We halved the convolution filter values so -1 from the right shift.
+ int16x4_t res = vrsra_n_s16(round_offset, sum, ROUND0_BITS - 1);
+ return vreinterpret_u16_s16(res);
+}
+
+static INLINE uint16x8_t convolve6_8_y(const int16x8_t s0, const int16x8_t s1,
+ const int16x8_t s2, const int16x8_t s3,
+ const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t y_filter,
+ const int16x8_t round_offset) {
+ const int16x4_t y_filter_0_3 = vget_low_s16(y_filter);
+ const int16x4_t y_filter_4_7 = vget_high_s16(y_filter);
+
+ // Filter values at indices 0 and 7 are 0.
+ int16x8_t sum = vmulq_lane_s16(s0, y_filter_0_3, 1);
+ sum = vmlaq_lane_s16(sum, s1, y_filter_0_3, 2);
+ sum = vmlaq_lane_s16(sum, s2, y_filter_0_3, 3);
+ sum = vmlaq_lane_s16(sum, s3, y_filter_4_7, 0);
+ sum = vmlaq_lane_s16(sum, s4, y_filter_4_7, 1);
+ sum = vmlaq_lane_s16(sum, s5, y_filter_4_7, 2);
+
+ // We halved the convolution filter values so -1 from the right shift.
+ int16x8_t res = vrsraq_n_s16(round_offset, sum, ROUND0_BITS - 1);
+ return vreinterpretq_u16_s16(res);
+}
+
+static INLINE void dist_wtd_convolve_y_6tap_dist_wtd_avg_neon(
+ const uint8_t *src_ptr, int src_stride, uint8_t *dst8_ptr,
+ const int dst8_stride, int w, int h, const int16x8_t y_filter,
+ ConvolveParams *conv_params) {
+ const int bd = 8;
+ const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
+ const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
+ (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
+ const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
+
+ const uint16_t fwd_offset = conv_params->fwd_offset;
+ const uint16_t bck_offset = conv_params->bck_offset;
+
+ CONV_BUF_TYPE *dst_ptr = conv_params->dst;
+ const int dst_stride = conv_params->dst_stride;
+ int width = w;
+
+ if (w == 4 || h == 4) {
+ do {
+ const uint8_t *s = src_ptr;
+ CONV_BUF_TYPE *d = dst_ptr;
+ uint8_t *d_u8 = dst8_ptr;
+ int height = h;
+
+ uint8x8_t t0 = load_unaligned_u8_4x1(s + 0 * src_stride);
+ uint8x8_t t1 = load_unaligned_u8_4x1(s + 1 * src_stride);
+ uint8x8_t t2 = load_unaligned_u8_4x1(s + 2 * src_stride);
+ uint8x8_t t3 = load_unaligned_u8_4x1(s + 3 * src_stride);
+ uint8x8_t t4 = load_unaligned_u8_4x1(s + 4 * src_stride);
+
+ int16x4_t s0 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
+ int16x4_t s1 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t1)));
+ int16x4_t s2 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t2)));
+ int16x4_t s3 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t3)));
+ int16x4_t s4 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t4)));
+
+ s += 5 * src_stride;
+
+ do {
+#if AOM_ARCH_AARCH64
+ t0 = load_unaligned_u8_4x1(s + 0 * src_stride);
+ t1 = load_unaligned_u8_4x1(s + 1 * src_stride);
+ t2 = load_unaligned_u8_4x1(s + 2 * src_stride);
+ t3 = load_unaligned_u8_4x1(s + 3 * src_stride);
+
+ int16x4_t s5 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
+ int16x4_t s6 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t1)));
+ int16x4_t s7 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t2)));
+ int16x4_t s8 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t3)));
+
+ uint16x4_t d0 = convolve6_4_y(s0, s1, s2, s3, s4, s5, y_filter,
+ vget_low_s16(round_offset_vec));
+ uint16x4_t d1 = convolve6_4_y(s1, s2, s3, s4, s5, s6, y_filter,
+ vget_low_s16(round_offset_vec));
+ uint16x4_t d2 = convolve6_4_y(s2, s3, s4, s5, s6, s7, y_filter,
+ vget_low_s16(round_offset_vec));
+ uint16x4_t d3 = convolve6_4_y(s3, s4, s5, s6, s7, s8, y_filter,
+ vget_low_s16(round_offset_vec));
+
+ uint16x4_t dd0, dd1, dd2, dd3;
+ load_u16_4x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
+
+ uint8x8_t d01, d23;
+ compute_dist_wtd_avg_4x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset,
+ bck_offset, round_offset_vec, &d01, &d23);
+
+ store_u8_4x1(d_u8 + 0 * dst8_stride, d01, 0);
+ store_u8_4x1(d_u8 + 1 * dst8_stride, d01, 1);
+ store_u8_4x1(d_u8 + 2 * dst8_stride, d23, 0);
+ store_u8_4x1(d_u8 + 3 * dst8_stride, d23, 1);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ d_u8 += 4 * dst8_stride;
+ height -= 4;
+#else // !AOM_ARCH_AARCH64
+ t0 = load_unaligned_u8_4x1(s);
+ int16x4_t s5 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
+
+ uint16x4_t d0 = convolve6_4_y(s0, s1, s2, s3, s4, s5, y_filter,
+ vget_low_s16(round_offset_vec));
+
+ uint16x4_t dd0 = vld1_u16(d);
+
+ uint8x8_t d01;
+ compute_dist_wtd_avg_4x1(dd0, d0, fwd_offset, bck_offset,
+ vget_low_s16(round_offset_vec), &d01);
+
+ store_u8_4x1(d_u8, d01, 0);
+
+ s0 = s1;
+ s1 = s2;
+ s2 = s3;
+ s3 = s4;
+ s4 = s5;
+ s += src_stride;
+ d += dst_stride;
+ d_u8 += dst8_stride;
+ height--;
+#endif // AOM_ARCH_AARCH64
+ } while (height != 0);
+ src_ptr += 4;
+ dst_ptr += 4;
+ dst8_ptr += 4;
+ width -= 4;
+ } while (width != 0);
+ } else {
+ do {
+ const uint8_t *s = src_ptr + (5 * src_stride);
+ CONV_BUF_TYPE *d = dst_ptr;
+ uint8_t *d_u8 = dst8_ptr;
+ int height = h;
+
+ uint8x8_t t0, t1, t2, t3, t4;
+ load_u8_8x5(src_ptr, src_stride, &t0, &t1, &t2, &t3, &t4);
+
+ int16x8_t s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
+ int16x8_t s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
+ int16x8_t s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
+ int16x8_t s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
+ int16x8_t s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
+
+ do {
+#if AOM_ARCH_AARCH64
+ uint8x8_t t5, t6, t7;
+ load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
+
+ int16x8_t s5 = vreinterpretq_s16_u16(vmovl_u8(t0));
+ int16x8_t s6 = vreinterpretq_s16_u16(vmovl_u8(t1));
+ int16x8_t s7 = vreinterpretq_s16_u16(vmovl_u8(t2));
+ int16x8_t s8 = vreinterpretq_s16_u16(vmovl_u8(t3));
+ int16x8_t s9 = vreinterpretq_s16_u16(vmovl_u8(t4));
+ int16x8_t s10 = vreinterpretq_s16_u16(vmovl_u8(t5));
+ int16x8_t s11 = vreinterpretq_s16_u16(vmovl_u8(t6));
+ int16x8_t s12 = vreinterpretq_s16_u16(vmovl_u8(t7));
+
+ uint16x8_t d0 =
+ convolve6_8_y(s0, s1, s2, s3, s4, s5, y_filter, round_offset_vec);
+ uint16x8_t d1 =
+ convolve6_8_y(s1, s2, s3, s4, s5, s6, y_filter, round_offset_vec);
+ uint16x8_t d2 =
+ convolve6_8_y(s2, s3, s4, s5, s6, s7, y_filter, round_offset_vec);
+ uint16x8_t d3 =
+ convolve6_8_y(s3, s4, s5, s6, s7, s8, y_filter, round_offset_vec);
+ uint16x8_t d4 =
+ convolve6_8_y(s4, s5, s6, s7, s8, s9, y_filter, round_offset_vec);
+ uint16x8_t d5 =
+ convolve6_8_y(s5, s6, s7, s8, s9, s10, y_filter, round_offset_vec);
+ uint16x8_t d6 =
+ convolve6_8_y(s6, s7, s8, s9, s10, s11, y_filter, round_offset_vec);
+ uint16x8_t d7 = convolve6_8_y(s7, s8, s9, s10, s11, s12, y_filter,
+ round_offset_vec);
+
+ uint16x8_t dd0, dd1, dd2, dd3;
+ load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
+
+ uint8x8_t d0_u8, d1_u8, d2_u8, d3_u8;
+ compute_dist_wtd_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset,
+ bck_offset, round_offset_vec, &d0_u8, &d1_u8,
+ &d2_u8, &d3_u8);
+
+ store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8);
+ d_u8 += 4 * dst8_stride;
+
+ uint16x8_t dd4, dd5, dd6, dd7;
+ load_u16_8x4(d + 4 * dst_stride, dst_stride, &dd4, &dd5, &dd6, &dd7);
+
+ uint8x8_t d4_u8, d5_u8, d6_u8, d7_u8;
+ compute_dist_wtd_avg_8x4(dd4, dd5, dd6, dd7, d4, d5, d6, d7, fwd_offset,
+ bck_offset, round_offset_vec, &d4_u8, &d5_u8,
+ &d6_u8, &d7_u8);
+
+ store_u8_8x4(d_u8, dst8_stride, d4_u8, d5_u8, d6_u8, d7_u8);
+ d_u8 += 4 * dst8_stride;
+
+ s0 = s8;
+ s1 = s9;
+ s2 = s10;
+ s3 = s11;
+ s4 = s12;
+ s += 8 * src_stride;
+ d += 8 * dst_stride;
+ height -= 8;
+#else // !AOM_ARCH_AARCH64
+ int16x8_t s5 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
+
+ uint16x8_t d0 =
+ convolve6_8_y(s0, s1, s2, s3, s4, s5, y_filter, round_offset_vec);
+
+ s0 = s1;
+ s1 = s2;
+ s2 = s3;
+ s3 = s4;
+ s4 = s5;
+
+ uint16x8_t dd0 = vld1q_u16(d);
+
+ uint8x8_t d0_u8;
+ compute_dist_wtd_avg_8x1(dd0, d0, fwd_offset, bck_offset,
+ round_offset_vec, &d0_u8);
+
+ vst1_u8(d_u8, d0_u8);
+ d_u8 += dst8_stride;
+
+ s += src_stride;
+ d += dst_stride;
+ height--;
+#endif // AOM_ARCH_AARCH64
+ } while (height != 0);
+ src_ptr += 8;
+ dst_ptr += 8;
+ dst8_ptr += 8;
+ width -= 8;
+ } while (width != 0);
+ }
+}
+
+static INLINE void dist_wtd_convolve_y_6tap_avg_neon(
+ const uint8_t *src_ptr, int src_stride, uint8_t *dst8_ptr,
+ const int dst8_stride, int w, int h, const int16x8_t y_filter,
+ ConvolveParams *conv_params) {
+ const int bd = 8;
+ const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
+ const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
+ (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
+ const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
+
+ CONV_BUF_TYPE *dst_ptr = conv_params->dst;
+ const int dst_stride = conv_params->dst_stride;
+ int width = w;
+
+ if (w == 4 || h == 4) {
+ do {
+ const uint8_t *s = src_ptr;
+ CONV_BUF_TYPE *d = dst_ptr;
+ uint8_t *d_u8 = dst8_ptr;
+ int height = h;
+
+ uint8x8_t t0 = load_unaligned_u8_4x1(s + 0 * src_stride);
+ uint8x8_t t1 = load_unaligned_u8_4x1(s + 1 * src_stride);
+ uint8x8_t t2 = load_unaligned_u8_4x1(s + 2 * src_stride);
+ uint8x8_t t3 = load_unaligned_u8_4x1(s + 3 * src_stride);
+ uint8x8_t t4 = load_unaligned_u8_4x1(s + 4 * src_stride);
+
+ int16x4_t s0 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
+ int16x4_t s1 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t1)));
+ int16x4_t s2 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t2)));
+ int16x4_t s3 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t3)));
+ int16x4_t s4 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t4)));
+
+ s += 5 * src_stride;
+
+ do {
+#if AOM_ARCH_AARCH64
+ t0 = load_unaligned_u8_4x1(s + 0 * src_stride);
+ t1 = load_unaligned_u8_4x1(s + 1 * src_stride);
+ t2 = load_unaligned_u8_4x1(s + 2 * src_stride);
+ t3 = load_unaligned_u8_4x1(s + 3 * src_stride);
+
+ int16x4_t s5 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
+ int16x4_t s6 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t1)));
+ int16x4_t s7 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t2)));
+ int16x4_t s8 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t3)));
+
+ uint16x4_t d0 = convolve6_4_y(s0, s1, s2, s3, s4, s5, y_filter,
+ vget_low_s16(round_offset_vec));
+ uint16x4_t d1 = convolve6_4_y(s1, s2, s3, s4, s5, s6, y_filter,
+ vget_low_s16(round_offset_vec));
+ uint16x4_t d2 = convolve6_4_y(s2, s3, s4, s5, s6, s7, y_filter,
+ vget_low_s16(round_offset_vec));
+ uint16x4_t d3 = convolve6_4_y(s3, s4, s5, s6, s7, s8, y_filter,
+ vget_low_s16(round_offset_vec));
+
+ uint16x4_t dd0, dd1, dd2, dd3;
+ load_u16_4x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
+
+ uint8x8_t d01, d23;
+ compute_basic_avg_4x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3,
+ round_offset_vec, &d01, &d23);
+
+ store_u8_4x1(d_u8 + 0 * dst8_stride, d01, 0);
+ store_u8_4x1(d_u8 + 1 * dst8_stride, d01, 1);
+ store_u8_4x1(d_u8 + 2 * dst8_stride, d23, 0);
+ store_u8_4x1(d_u8 + 3 * dst8_stride, d23, 1);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ d_u8 += 4 * dst8_stride;
+ height -= 4;
+#else // !AOM_ARCH_AARCH64
+ t0 = load_unaligned_u8_4x1(s);
+ int16x4_t s5 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
+
+ uint16x4_t d0 = convolve6_4_y(s0, s1, s2, s3, s4, s5, y_filter,
+ vget_low_s16(round_offset_vec));
+
+ uint16x4_t dd0 = vld1_u16(d);
+
+ uint8x8_t d01;
+ compute_basic_avg_4x1(dd0, d0, vget_low_s16(round_offset_vec), &d01);
+
+ store_u8_4x1(d_u8, d01, 0);
+
+ s0 = s1;
+ s1 = s2;
+ s2 = s3;
+ s3 = s4;
+ s4 = s5;
+ s += src_stride;
+ d += dst_stride;
+ d_u8 += dst8_stride;
+ height--;
+#endif // AOM_ARCH_AARCH64
+ } while (height != 0);
+ src_ptr += 4;
+ dst_ptr += 4;
+ dst8_ptr += 4;
+ width -= 4;
+ } while (width != 0);
+ } else {
+ do {
+ const uint8_t *s = src_ptr + (5 * src_stride);
+ CONV_BUF_TYPE *d = dst_ptr;
+ uint8_t *d_u8 = dst8_ptr;
+ int height = h;
+
+ uint8x8_t t0, t1, t2, t3, t4;
+ load_u8_8x5(src_ptr, src_stride, &t0, &t1, &t2, &t3, &t4);
+
+ int16x8_t s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
+ int16x8_t s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
+ int16x8_t s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
+ int16x8_t s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
+ int16x8_t s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
+
+ do {
+#if AOM_ARCH_AARCH64
+ uint8x8_t t5, t6, t7;
+ load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
+
+ int16x8_t s5 = vreinterpretq_s16_u16(vmovl_u8(t0));
+ int16x8_t s6 = vreinterpretq_s16_u16(vmovl_u8(t1));
+ int16x8_t s7 = vreinterpretq_s16_u16(vmovl_u8(t2));
+ int16x8_t s8 = vreinterpretq_s16_u16(vmovl_u8(t3));
+ int16x8_t s9 = vreinterpretq_s16_u16(vmovl_u8(t4));
+ int16x8_t s10 = vreinterpretq_s16_u16(vmovl_u8(t5));
+ int16x8_t s11 = vreinterpretq_s16_u16(vmovl_u8(t6));
+ int16x8_t s12 = vreinterpretq_s16_u16(vmovl_u8(t7));
+
+ uint16x8_t d0 =
+ convolve6_8_y(s0, s1, s2, s3, s4, s5, y_filter, round_offset_vec);
+ uint16x8_t d1 =
+ convolve6_8_y(s1, s2, s3, s4, s5, s6, y_filter, round_offset_vec);
+ uint16x8_t d2 =
+ convolve6_8_y(s2, s3, s4, s5, s6, s7, y_filter, round_offset_vec);
+ uint16x8_t d3 =
+ convolve6_8_y(s3, s4, s5, s6, s7, s8, y_filter, round_offset_vec);
+ uint16x8_t d4 =
+ convolve6_8_y(s4, s5, s6, s7, s8, s9, y_filter, round_offset_vec);
+ uint16x8_t d5 =
+ convolve6_8_y(s5, s6, s7, s8, s9, s10, y_filter, round_offset_vec);
+ uint16x8_t d6 =
+ convolve6_8_y(s6, s7, s8, s9, s10, s11, y_filter, round_offset_vec);
+ uint16x8_t d7 = convolve6_8_y(s7, s8, s9, s10, s11, s12, y_filter,
+ round_offset_vec);
+
+ uint16x8_t dd0, dd1, dd2, dd3;
+ load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
+
+ uint8x8_t d0_u8, d1_u8, d2_u8, d3_u8;
+ compute_basic_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3,
+ round_offset_vec, &d0_u8, &d1_u8, &d2_u8, &d3_u8);
+
+ store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8);
+ d_u8 += 4 * dst8_stride;
+
+ uint16x8_t dd4, dd5, dd6, dd7;
+ load_u16_8x4(d + 4 * dst_stride, dst_stride, &dd4, &dd5, &dd6, &dd7);
+
+ uint8x8_t d4_u8, d5_u8, d6_u8, d7_u8;
+ compute_basic_avg_8x4(dd4, dd5, dd6, dd7, d4, d5, d6, d7,
+ round_offset_vec, &d4_u8, &d5_u8, &d6_u8, &d7_u8);
+
+ store_u8_8x4(d_u8, dst8_stride, d4_u8, d5_u8, d6_u8, d7_u8);
+ d_u8 += 4 * dst8_stride;
+
+ s0 = s8;
+ s1 = s9;
+ s2 = s10;
+ s3 = s11;
+ s4 = s12;
+ s += 8 * src_stride;
+ d += 8 * dst_stride;
+ height -= 8;
+#else // !AOM_ARCH_AARCH64
+ int16x8_t s5 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
+
+ uint16x8_t d0 =
+ convolve6_8_y(s0, s1, s2, s3, s4, s5, y_filter, round_offset_vec);
+
+ s0 = s1;
+ s1 = s2;
+ s2 = s3;
+ s3 = s4;
+ s4 = s5;
+
+ uint16x8_t dd0 = vld1q_u16(d);
+
+ uint8x8_t d0_u8;
+ compute_basic_avg_8x1(dd0, d0, round_offset_vec, &d0_u8);
+
+ vst1_u8(d_u8, d0_u8);
+ d_u8 += dst8_stride;
+
+ s += src_stride;
+ d += dst_stride;
+ height--;
+#endif // AOM_ARCH_AARCH64
+ } while (height != 0);
+ src_ptr += 8;
+ dst_ptr += 8;
+ dst8_ptr += 8;
+ width -= 8;
+ } while (width != 0);
+ }
+}
+
+static INLINE void dist_wtd_convolve_y_6tap_neon(const uint8_t *src_ptr,
+ int src_stride, int w, int h,
+ const int16x8_t y_filter,
+ ConvolveParams *conv_params) {
+ const int bd = 8;
+ const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
+ const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
+ (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
+ const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
+
+ CONV_BUF_TYPE *dst_ptr = conv_params->dst;
+ const int dst_stride = conv_params->dst_stride;
+ int width = w;
+
+ if (w == 4 || h == 4) {
+ do {
+ const uint8_t *s = src_ptr;
+ CONV_BUF_TYPE *d = dst_ptr;
+ int height = h;
+
+ uint8x8_t t0 = load_unaligned_u8_4x1(s + 0 * src_stride);
+ uint8x8_t t1 = load_unaligned_u8_4x1(s + 1 * src_stride);
+ uint8x8_t t2 = load_unaligned_u8_4x1(s + 2 * src_stride);
+ uint8x8_t t3 = load_unaligned_u8_4x1(s + 3 * src_stride);
+ uint8x8_t t4 = load_unaligned_u8_4x1(s + 4 * src_stride);
+
+ int16x4_t s0 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
+ int16x4_t s1 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t1)));
+ int16x4_t s2 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t2)));
+ int16x4_t s3 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t3)));
+ int16x4_t s4 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t4)));
+
+ s += 5 * src_stride;
+
+ do {
+#if AOM_ARCH_AARCH64
+ t0 = load_unaligned_u8_4x1(s + 0 * src_stride);
+ t1 = load_unaligned_u8_4x1(s + 1 * src_stride);
+ t2 = load_unaligned_u8_4x1(s + 2 * src_stride);
+ t3 = load_unaligned_u8_4x1(s + 3 * src_stride);
+
+ int16x4_t s5 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
+ int16x4_t s6 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t1)));
+ int16x4_t s7 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t2)));
+ int16x4_t s8 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t3)));
+
+ uint16x4_t d0 = convolve6_4_y(s0, s1, s2, s3, s4, s5, y_filter,
+ vget_low_s16(round_offset_vec));
+ uint16x4_t d1 = convolve6_4_y(s1, s2, s3, s4, s5, s6, y_filter,
+ vget_low_s16(round_offset_vec));
+ uint16x4_t d2 = convolve6_4_y(s2, s3, s4, s5, s6, s7, y_filter,
+ vget_low_s16(round_offset_vec));
+ uint16x4_t d3 = convolve6_4_y(s3, s4, s5, s6, s7, s8, y_filter,
+ vget_low_s16(round_offset_vec));
+
+ store_u16_4x4(d, dst_stride, d0, d1, d2, d3);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ height -= 4;
+#else // !AOM_ARCH_AARCH64
+ t0 = load_unaligned_u8_4x1(s);
+ int16x4_t s5 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
+
+ uint16x4_t d0 = convolve6_4_y(s0, s1, s2, s3, s4, s5, y_filter,
+ vget_low_s16(round_offset_vec));
+
+ vst1_u16(d, d0);
+
+ s0 = s1;
+ s1 = s2;
+ s2 = s3;
+ s3 = s4;
+ s4 = s5;
+ s += src_stride;
+ d += dst_stride;
+ height--;
+#endif // AOM_ARCH_AARCH64
+ } while (height != 0);
+ src_ptr += 4;
+ dst_ptr += 4;
+ width -= 4;
+ } while (width != 0);
+ } else {
+ do {
+ const uint8_t *s = src_ptr + (5 * src_stride);
+ CONV_BUF_TYPE *d = dst_ptr;
+ int height = h;
+
+ uint8x8_t t0, t1, t2, t3, t4;
+ load_u8_8x5(src_ptr, src_stride, &t0, &t1, &t2, &t3, &t4);
+
+ int16x8_t s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
+ int16x8_t s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
+ int16x8_t s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
+ int16x8_t s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
+ int16x8_t s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
+
+ do {
+#if AOM_ARCH_AARCH64
+ uint8x8_t t5, t6, t7;
+ load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
+
+ int16x8_t s5 = vreinterpretq_s16_u16(vmovl_u8(t0));
+ int16x8_t s6 = vreinterpretq_s16_u16(vmovl_u8(t1));
+ int16x8_t s7 = vreinterpretq_s16_u16(vmovl_u8(t2));
+ int16x8_t s8 = vreinterpretq_s16_u16(vmovl_u8(t3));
+ int16x8_t s9 = vreinterpretq_s16_u16(vmovl_u8(t4));
+ int16x8_t s10 = vreinterpretq_s16_u16(vmovl_u8(t5));
+ int16x8_t s11 = vreinterpretq_s16_u16(vmovl_u8(t6));
+ int16x8_t s12 = vreinterpretq_s16_u16(vmovl_u8(t7));
+
+ uint16x8_t d0 =
+ convolve6_8_y(s0, s1, s2, s3, s4, s5, y_filter, round_offset_vec);
+ uint16x8_t d1 =
+ convolve6_8_y(s1, s2, s3, s4, s5, s6, y_filter, round_offset_vec);
+ uint16x8_t d2 =
+ convolve6_8_y(s2, s3, s4, s5, s6, s7, y_filter, round_offset_vec);
+ uint16x8_t d3 =
+ convolve6_8_y(s3, s4, s5, s6, s7, s8, y_filter, round_offset_vec);
+ uint16x8_t d4 =
+ convolve6_8_y(s4, s5, s6, s7, s8, s9, y_filter, round_offset_vec);
+ uint16x8_t d5 =
+ convolve6_8_y(s5, s6, s7, s8, s9, s10, y_filter, round_offset_vec);
+ uint16x8_t d6 =
+ convolve6_8_y(s6, s7, s8, s9, s10, s11, y_filter, round_offset_vec);
+ uint16x8_t d7 = convolve6_8_y(s7, s8, s9, s10, s11, s12, y_filter,
+ round_offset_vec);
+
+ store_u16_8x8(d, dst_stride, d0, d1, d2, d3, d4, d5, d6, d7);
+
+ s0 = s8;
+ s1 = s9;
+ s2 = s10;
+ s3 = s11;
+ s4 = s12;
+ s += 8 * src_stride;
+ d += 8 * dst_stride;
+ height -= 8;
+#else // !AOM_ARCH_AARCH64
+ int16x8_t s5 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
+
+ uint16x8_t d0 =
+ convolve6_8_y(s0, s1, s2, s3, s4, s5, y_filter, round_offset_vec);
+
+ s0 = s1;
+ s1 = s2;
+ s2 = s3;
+ s3 = s4;
+ s4 = s5;
+
+ vst1q_u16(d, d0);
+
+ s += src_stride;
+ d += dst_stride;
+ height--;
+#endif // AOM_ARCH_AARCH64
+ } while (height != 0);
+ src_ptr += 8;
+ dst_ptr += 8;
+ width -= 8;
+ } while (width != 0);
+ }
+}
+
+static INLINE uint16x4_t convolve8_4_y(const int16x4_t s0, const int16x4_t s1,
+ const int16x4_t s2, const int16x4_t s3,
+ const int16x4_t s4, const int16x4_t s5,
+ const int16x4_t s6, const int16x4_t s7,
+ const int16x8_t y_filter,
+ const int16x4_t round_offset) {
+ const int16x4_t y_filter_0_3 = vget_low_s16(y_filter);
+ const int16x4_t y_filter_4_7 = vget_high_s16(y_filter);
+
+ int16x4_t sum = vmul_lane_s16(s0, y_filter_0_3, 0);
+ sum = vmla_lane_s16(sum, s1, y_filter_0_3, 1);
+ sum = vmla_lane_s16(sum, s2, y_filter_0_3, 2);
+ sum = vmla_lane_s16(sum, s3, y_filter_0_3, 3);
+ sum = vmla_lane_s16(sum, s4, y_filter_4_7, 0);
+ sum = vmla_lane_s16(sum, s5, y_filter_4_7, 1);
+ sum = vmla_lane_s16(sum, s6, y_filter_4_7, 2);
+ sum = vmla_lane_s16(sum, s7, y_filter_4_7, 3);
+
+ // We halved the convolution filter values so -1 from the right shift.
+ int16x4_t res = vrsra_n_s16(round_offset, sum, ROUND0_BITS - 1);
+ return vreinterpret_u16_s16(res);
+}
+
+static INLINE uint16x8_t convolve8_8_y(const int16x8_t s0, const int16x8_t s1,
+ const int16x8_t s2, const int16x8_t s3,
+ const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t s6, const int16x8_t s7,
+ const int16x8_t y_filter,
+ const int16x8_t round_offset) {
+ const int16x4_t y_filter_0_3 = vget_low_s16(y_filter);
+ const int16x4_t y_filter_4_7 = vget_high_s16(y_filter);
+
+ int16x8_t sum = vmulq_lane_s16(s0, y_filter_0_3, 0);
+ sum = vmlaq_lane_s16(sum, s1, y_filter_0_3, 1);
+ sum = vmlaq_lane_s16(sum, s2, y_filter_0_3, 2);
+ sum = vmlaq_lane_s16(sum, s3, y_filter_0_3, 3);
+ sum = vmlaq_lane_s16(sum, s4, y_filter_4_7, 0);
+ sum = vmlaq_lane_s16(sum, s5, y_filter_4_7, 1);
+ sum = vmlaq_lane_s16(sum, s6, y_filter_4_7, 2);
+ sum = vmlaq_lane_s16(sum, s7, y_filter_4_7, 3);
+
+ // We halved the convolution filter values so -1 from the right shift.
+ int16x8_t res = vrsraq_n_s16(round_offset, sum, ROUND0_BITS - 1);
+ return vreinterpretq_u16_s16(res);
+}
+
+static INLINE void dist_wtd_convolve_y_8tap_dist_wtd_avg_neon(
+ const uint8_t *src_ptr, int src_stride, uint8_t *dst8_ptr,
+ const int dst8_stride, int w, int h, const int16x8_t y_filter,
+ ConvolveParams *conv_params) {
+ const int bd = 8;
+ const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
+ const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
+ (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
+ const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
+
+ const uint16_t fwd_offset = conv_params->fwd_offset;
+ const uint16_t bck_offset = conv_params->bck_offset;
+
+ CONV_BUF_TYPE *dst_ptr = conv_params->dst;
+ const int dst_stride = conv_params->dst_stride;
+ int width = w;
+
+ if (w == 4 || h == 4) {
+ do {
+ const uint8_t *s = src_ptr;
+ CONV_BUF_TYPE *d = dst_ptr;
+ uint8_t *d_u8 = dst8_ptr;
+ int height = h;
+
+ __builtin_prefetch(s + 0 * src_stride);
+ __builtin_prefetch(s + 1 * src_stride);
+ __builtin_prefetch(s + 2 * src_stride);
+ __builtin_prefetch(s + 3 * src_stride);
+
+ uint8x8_t t0 = load_unaligned_u8_4x1(s + 0 * src_stride);
+ uint8x8_t t1 = load_unaligned_u8_4x1(s + 1 * src_stride);
+ uint8x8_t t2 = load_unaligned_u8_4x1(s + 2 * src_stride);
+ uint8x8_t t3 = load_unaligned_u8_4x1(s + 3 * src_stride);
+ uint8x8_t t4 = load_unaligned_u8_4x1(s + 4 * src_stride);
+ uint8x8_t t5 = load_unaligned_u8_4x1(s + 5 * src_stride);
+ uint8x8_t t6 = load_unaligned_u8_4x1(s + 6 * src_stride);
+
+ int16x4_t s0 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
+ int16x4_t s1 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t1)));
+ int16x4_t s2 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t2)));
+ int16x4_t s3 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t3)));
+ int16x4_t s4 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t4)));
+ int16x4_t s5 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t5)));
+ int16x4_t s6 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t6)));
+
+ __builtin_prefetch(d + 0 * dst_stride);
+ __builtin_prefetch(d + 1 * dst_stride);
+ __builtin_prefetch(d + 2 * dst_stride);
+ __builtin_prefetch(d + 3 * dst_stride);
+
+ s += 7 * src_stride;
+
+ do {
+#if AOM_ARCH_AARCH64
+ t0 = load_unaligned_u8_4x1(s + 0 * src_stride);
+ t1 = load_unaligned_u8_4x1(s + 1 * src_stride);
+ t2 = load_unaligned_u8_4x1(s + 2 * src_stride);
+ t3 = load_unaligned_u8_4x1(s + 3 * src_stride);
+
+ int16x4_t s7 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
+ int16x4_t s8 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t1)));
+ int16x4_t s9 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t2)));
+ int16x4_t s10 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t3)));
+
+ uint16x4_t d0 = convolve8_4_y(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
+ vget_low_s16(round_offset_vec));
+ uint16x4_t d1 = convolve8_4_y(s1, s2, s3, s4, s5, s6, s7, s8, y_filter,
+ vget_low_s16(round_offset_vec));
+ uint16x4_t d2 = convolve8_4_y(s2, s3, s4, s5, s6, s7, s8, s9, y_filter,
+ vget_low_s16(round_offset_vec));
+ uint16x4_t d3 = convolve8_4_y(s3, s4, s5, s6, s7, s8, s9, s10, y_filter,
+ vget_low_s16(round_offset_vec));
+
+ __builtin_prefetch(d + 0 * dst_stride);
+ __builtin_prefetch(d + 1 * dst_stride);
+ __builtin_prefetch(d + 2 * dst_stride);
+ __builtin_prefetch(d + 3 * dst_stride);
+
+ __builtin_prefetch(d_u8 + 0 * dst8_stride);
+ __builtin_prefetch(d_u8 + 1 * dst8_stride);
+ __builtin_prefetch(d_u8 + 2 * dst8_stride);
+ __builtin_prefetch(d_u8 + 3 * dst8_stride);
+
+ uint16x4_t dd0, dd1, dd2, dd3;
+ load_u16_4x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
+
+ uint8x8_t d01, d23;
+ compute_dist_wtd_avg_4x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset,
+ bck_offset, round_offset_vec, &d01, &d23);
+
+ store_u8_4x1(d_u8 + 0 * dst8_stride, d01, 0);
+ store_u8_4x1(d_u8 + 1 * dst8_stride, d01, 1);
+ store_u8_4x1(d_u8 + 2 * dst8_stride, d23, 0);
+ store_u8_4x1(d_u8 + 3 * dst8_stride, d23, 1);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s5 = s9;
+ s6 = s10;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ d_u8 += 4 * dst8_stride;
+ height -= 4;
+#else // !AOM_ARCH_AARCH64
+ t0 = load_unaligned_u8_4x1(s);
+ int16x4_t s7 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
+
+ uint16x4_t d0 = convolve8_4_y(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
+ vget_low_s16(round_offset_vec));
+
+ __builtin_prefetch(d);
+
+ uint16x4_t dd0 = vld1_u16(d);
+
+ uint8x8_t d01;
+ compute_dist_wtd_avg_4x1(dd0, d0, fwd_offset, bck_offset,
+ vget_low_s16(round_offset_vec), &d01);
+
+ store_u8_4x1(d_u8, d01, 0);
+
+ s0 = s1;
+ s1 = s2;
+ s2 = s3;
+ s3 = s4;
+ s4 = s5;
+ s5 = s6;
+ s6 = s7;
+ s += src_stride;
+ d += dst_stride;
+ d_u8 += dst8_stride;
+ height--;
+#endif // AOM_ARCH_AARCH64
+ } while (height != 0);
+ src_ptr += 4;
+ dst_ptr += 4;
+ dst8_ptr += 4;
+ width -= 4;
+ } while (width != 0);
+ } else {
+ do {
+ const uint8_t *s = src_ptr;
+ CONV_BUF_TYPE *d = dst_ptr;
+ uint8_t *d_u8 = dst8_ptr;
+ int height = h;
+
+ __builtin_prefetch(s + 0 * src_stride);
+ __builtin_prefetch(s + 1 * src_stride);
+ __builtin_prefetch(s + 2 * src_stride);
+ __builtin_prefetch(s + 3 * src_stride);
+ __builtin_prefetch(s + 4 * src_stride);
+ __builtin_prefetch(s + 5 * src_stride);
+ __builtin_prefetch(s + 6 * src_stride);
+ __builtin_prefetch(s + 7 * src_stride);
+
+ uint8x8_t t0, t1, t2, t3, t4, t5, t6;
+ load_u8_8x7(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6);
+
+ int16x8_t s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
+ int16x8_t s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
+ int16x8_t s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
+ int16x8_t s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
+ int16x8_t s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
+ int16x8_t s5 = vreinterpretq_s16_u16(vmovl_u8(t5));
+ int16x8_t s6 = vreinterpretq_s16_u16(vmovl_u8(t6));
+
+ s += 7 * src_stride;
+
+ do {
+#if AOM_ARCH_AARCH64
+ uint8x8_t t7;
+ load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
+
+ int16x8_t s7 = vreinterpretq_s16_u16(vmovl_u8(t0));
+ int16x8_t s8 = vreinterpretq_s16_u16(vmovl_u8(t1));
+ int16x8_t s9 = vreinterpretq_s16_u16(vmovl_u8(t2));
+ int16x8_t s10 = vreinterpretq_s16_u16(vmovl_u8(t3));
+ int16x8_t s11 = vreinterpretq_s16_u16(vmovl_u8(t4));
+ int16x8_t s12 = vreinterpretq_s16_u16(vmovl_u8(t5));
+ int16x8_t s13 = vreinterpretq_s16_u16(vmovl_u8(t6));
+ int16x8_t s14 = vreinterpretq_s16_u16(vmovl_u8(t7));
+
+ __builtin_prefetch(dst_ptr + 0 * dst_stride);
+ __builtin_prefetch(dst_ptr + 1 * dst_stride);
+ __builtin_prefetch(dst_ptr + 2 * dst_stride);
+ __builtin_prefetch(dst_ptr + 3 * dst_stride);
+
+ uint16x8_t d0 = convolve8_8_y(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
+ round_offset_vec);
+ uint16x8_t d1 = convolve8_8_y(s1, s2, s3, s4, s5, s6, s7, s8, y_filter,
+ round_offset_vec);
+ uint16x8_t d2 = convolve8_8_y(s2, s3, s4, s5, s6, s7, s8, s9, y_filter,
+ round_offset_vec);
+ uint16x8_t d3 = convolve8_8_y(s3, s4, s5, s6, s7, s8, s9, s10, y_filter,
+ round_offset_vec);
+ uint16x8_t d4 = convolve8_8_y(s4, s5, s6, s7, s8, s9, s10, s11,
+ y_filter, round_offset_vec);
+ uint16x8_t d5 = convolve8_8_y(s5, s6, s7, s8, s9, s10, s11, s12,
+ y_filter, round_offset_vec);
+ uint16x8_t d6 = convolve8_8_y(s6, s7, s8, s9, s10, s11, s12, s13,
+ y_filter, round_offset_vec);
+ uint16x8_t d7 = convolve8_8_y(s7, s8, s9, s10, s11, s12, s13, s14,
+ y_filter, round_offset_vec);
+
+ __builtin_prefetch(d + 0 * dst8_stride);
+ __builtin_prefetch(d + 1 * dst8_stride);
+ __builtin_prefetch(d + 2 * dst8_stride);
+ __builtin_prefetch(d + 3 * dst8_stride);
+
+ uint16x8_t dd0, dd1, dd2, dd3;
+ load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
+
+ uint8x8_t d0_u8, d1_u8, d2_u8, d3_u8;
+ compute_dist_wtd_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset,
+ bck_offset, round_offset_vec, &d0_u8, &d1_u8,
+ &d2_u8, &d3_u8);
+
+ store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8);
+ d_u8 += 4 * dst8_stride;
+
+ uint16x8_t dd4, dd5, dd6, dd7;
+ load_u16_8x4(d + 4 * dst_stride, dst_stride, &dd4, &dd5, &dd6, &dd7);
+
+ uint8x8_t d4_u8, d5_u8, d6_u8, d7_u8;
+ compute_dist_wtd_avg_8x4(dd4, dd5, dd6, dd7, d4, d5, d6, d7, fwd_offset,
+ bck_offset, round_offset_vec, &d4_u8, &d5_u8,
+ &d6_u8, &d7_u8);
+
+ store_u8_8x4(d_u8, dst8_stride, d4_u8, d5_u8, d6_u8, d7_u8);
+ d_u8 += 4 * dst8_stride;
+
+ s0 = s8;
+ s1 = s9;
+ s2 = s10;
+ s3 = s11;
+ s4 = s12;
+ s5 = s13;
+ s6 = s14;
+ s += 8 * src_stride;
+ d += 8 * dst_stride;
+ height -= 8;
+#else // !AOM_ARCH_AARCH64
+ int16x8_t s7 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
+
+ __builtin_prefetch(dst_ptr);
+
+ uint16x8_t d0 = convolve8_8_y(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
+ round_offset_vec);
+
+ s0 = s1;
+ s1 = s2;
+ s2 = s3;
+ s3 = s4;
+ s4 = s5;
+ s5 = s6;
+ s6 = s7;
+
+ __builtin_prefetch(d);
+
+ uint16x8_t dd0 = vld1q_u16(d);
+
+ uint8x8_t d0_u8;
+ compute_dist_wtd_avg_8x1(dd0, d0, fwd_offset, bck_offset,
+ round_offset_vec, &d0_u8);
+
+ vst1_u8(d_u8, d0_u8);
+ d_u8 += dst8_stride;
+
+ s += src_stride;
+ d += dst_stride;
+ height--;
+#endif // AOM_ARCH_AARCH64
+ } while (height != 0);
+ src_ptr += 8;
+ dst_ptr += 8;
+ dst8_ptr += 8;
+ width -= 8;
+ } while (width != 0);
+ }
+}
+
+static INLINE void dist_wtd_convolve_y_8tap_avg_neon(
+ const uint8_t *src_ptr, int src_stride, uint8_t *dst8_ptr,
+ const int dst8_stride, int w, int h, const int16x8_t y_filter,
+ ConvolveParams *conv_params) {
+ const int bd = 8;
+ const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
+ const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
+ (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
+ const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
+
+ CONV_BUF_TYPE *dst_ptr = conv_params->dst;
+ const int dst_stride = conv_params->dst_stride;
+ int width = w;
+
+ if (w == 4 || h == 4) {
+ do {
+ const uint8_t *s = src_ptr;
+ CONV_BUF_TYPE *d = dst_ptr;
+ uint8_t *d_u8 = dst8_ptr;
+ int height = h;
+
+ __builtin_prefetch(s + 0 * src_stride);
+ __builtin_prefetch(s + 1 * src_stride);
+ __builtin_prefetch(s + 2 * src_stride);
+ __builtin_prefetch(s + 3 * src_stride);
+
+ uint8x8_t t0 = load_unaligned_u8_4x1(s + 0 * src_stride);
+ uint8x8_t t1 = load_unaligned_u8_4x1(s + 1 * src_stride);
+ uint8x8_t t2 = load_unaligned_u8_4x1(s + 2 * src_stride);
+ uint8x8_t t3 = load_unaligned_u8_4x1(s + 3 * src_stride);
+ uint8x8_t t4 = load_unaligned_u8_4x1(s + 4 * src_stride);
+ uint8x8_t t5 = load_unaligned_u8_4x1(s + 5 * src_stride);
+ uint8x8_t t6 = load_unaligned_u8_4x1(s + 6 * src_stride);
+
+ int16x4_t s0 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
+ int16x4_t s1 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t1)));
+ int16x4_t s2 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t2)));
+ int16x4_t s3 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t3)));
+ int16x4_t s4 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t4)));
+ int16x4_t s5 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t5)));
+ int16x4_t s6 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t6)));
+
+ __builtin_prefetch(d + 0 * dst_stride);
+ __builtin_prefetch(d + 1 * dst_stride);
+ __builtin_prefetch(d + 2 * dst_stride);
+ __builtin_prefetch(d + 3 * dst_stride);
+
+ s += 7 * src_stride;
+
+ do {
+#if AOM_ARCH_AARCH64
+ t0 = load_unaligned_u8_4x1(s + 0 * src_stride);
+ t1 = load_unaligned_u8_4x1(s + 1 * src_stride);
+ t2 = load_unaligned_u8_4x1(s + 2 * src_stride);
+ t3 = load_unaligned_u8_4x1(s + 3 * src_stride);
+
+ int16x4_t s7 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
+ int16x4_t s8 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t1)));
+ int16x4_t s9 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t2)));
+ int16x4_t s10 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t3)));
+
+ uint16x4_t d0 = convolve8_4_y(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
+ vget_low_s16(round_offset_vec));
+ uint16x4_t d1 = convolve8_4_y(s1, s2, s3, s4, s5, s6, s7, s8, y_filter,
+ vget_low_s16(round_offset_vec));
+ uint16x4_t d2 = convolve8_4_y(s2, s3, s4, s5, s6, s7, s8, s9, y_filter,
+ vget_low_s16(round_offset_vec));
+ uint16x4_t d3 = convolve8_4_y(s3, s4, s5, s6, s7, s8, s9, s10, y_filter,
+ vget_low_s16(round_offset_vec));
+
+ __builtin_prefetch(d + 0 * dst_stride);
+ __builtin_prefetch(d + 1 * dst_stride);
+ __builtin_prefetch(d + 2 * dst_stride);
+ __builtin_prefetch(d + 3 * dst_stride);
+
+ __builtin_prefetch(d_u8 + 0 * dst8_stride);
+ __builtin_prefetch(d_u8 + 1 * dst8_stride);
+ __builtin_prefetch(d_u8 + 2 * dst8_stride);
+ __builtin_prefetch(d_u8 + 3 * dst8_stride);
+
+ uint16x4_t dd0, dd1, dd2, dd3;
+ load_u16_4x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
+
+ uint8x8_t d01, d23;
+ compute_basic_avg_4x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3,
+ round_offset_vec, &d01, &d23);
+
+ store_u8_4x1(d_u8 + 0 * dst8_stride, d01, 0);
+ store_u8_4x1(d_u8 + 1 * dst8_stride, d01, 1);
+ store_u8_4x1(d_u8 + 2 * dst8_stride, d23, 0);
+ store_u8_4x1(d_u8 + 3 * dst8_stride, d23, 1);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s5 = s9;
+ s6 = s10;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ d_u8 += 4 * dst8_stride;
+ height -= 4;
+#else // !AOM_ARCH_AARCH64
+ t0 = load_unaligned_u8_4x1(s);
+ int16x4_t s7 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
+
+ uint16x4_t d0 = convolve8_4_y(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
+ vget_low_s16(round_offset_vec));
+
+ __builtin_prefetch(d);
+
+ uint16x4_t dd0 = vld1_u16(d);
+
+ uint8x8_t d01;
+ compute_basic_avg_4x1(dd0, d0, vget_low_s16(round_offset_vec), &d01);
+
+ store_u8_4x1(d_u8, d01, 0);
+
+ s0 = s1;
+ s1 = s2;
+ s2 = s3;
+ s3 = s4;
+ s4 = s5;
+ s5 = s6;
+ s6 = s7;
+ s += src_stride;
+ d += dst_stride;
+ d_u8 += dst8_stride;
+ height--;
+#endif // AOM_ARCH_AARCH64
+ } while (height != 0);
+ src_ptr += 4;
+ dst_ptr += 4;
+ dst8_ptr += 4;
+ width -= 4;
+ } while (width != 0);
+ } else {
+ do {
+ const uint8_t *s = src_ptr;
+ CONV_BUF_TYPE *d = dst_ptr;
+ uint8_t *d_u8 = dst8_ptr;
+ int height = h;
+
+ __builtin_prefetch(s + 0 * src_stride);
+ __builtin_prefetch(s + 1 * src_stride);
+ __builtin_prefetch(s + 2 * src_stride);
+ __builtin_prefetch(s + 3 * src_stride);
+ __builtin_prefetch(s + 4 * src_stride);
+ __builtin_prefetch(s + 5 * src_stride);
+ __builtin_prefetch(s + 6 * src_stride);
+ __builtin_prefetch(s + 7 * src_stride);
+
+ uint8x8_t t0, t1, t2, t3, t4, t5, t6;
+ load_u8_8x7(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6);
+
+ int16x8_t s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
+ int16x8_t s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
+ int16x8_t s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
+ int16x8_t s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
+ int16x8_t s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
+ int16x8_t s5 = vreinterpretq_s16_u16(vmovl_u8(t5));
+ int16x8_t s6 = vreinterpretq_s16_u16(vmovl_u8(t6));
+
+ s += 7 * src_stride;
+
+ do {
+#if AOM_ARCH_AARCH64
+ uint8x8_t t7;
+ load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
+
+ int16x8_t s7 = vreinterpretq_s16_u16(vmovl_u8(t0));
+ int16x8_t s8 = vreinterpretq_s16_u16(vmovl_u8(t1));
+ int16x8_t s9 = vreinterpretq_s16_u16(vmovl_u8(t2));
+ int16x8_t s10 = vreinterpretq_s16_u16(vmovl_u8(t3));
+ int16x8_t s11 = vreinterpretq_s16_u16(vmovl_u8(t4));
+ int16x8_t s12 = vreinterpretq_s16_u16(vmovl_u8(t5));
+ int16x8_t s13 = vreinterpretq_s16_u16(vmovl_u8(t6));
+ int16x8_t s14 = vreinterpretq_s16_u16(vmovl_u8(t7));
+
+ __builtin_prefetch(dst_ptr + 0 * dst_stride);
+ __builtin_prefetch(dst_ptr + 1 * dst_stride);
+ __builtin_prefetch(dst_ptr + 2 * dst_stride);
+ __builtin_prefetch(dst_ptr + 3 * dst_stride);
+
+ uint16x8_t d0 = convolve8_8_y(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
+ round_offset_vec);
+ uint16x8_t d1 = convolve8_8_y(s1, s2, s3, s4, s5, s6, s7, s8, y_filter,
+ round_offset_vec);
+ uint16x8_t d2 = convolve8_8_y(s2, s3, s4, s5, s6, s7, s8, s9, y_filter,
+ round_offset_vec);
+ uint16x8_t d3 = convolve8_8_y(s3, s4, s5, s6, s7, s8, s9, s10, y_filter,
+ round_offset_vec);
+ uint16x8_t d4 = convolve8_8_y(s4, s5, s6, s7, s8, s9, s10, s11,
+ y_filter, round_offset_vec);
+ uint16x8_t d5 = convolve8_8_y(s5, s6, s7, s8, s9, s10, s11, s12,
+ y_filter, round_offset_vec);
+ uint16x8_t d6 = convolve8_8_y(s6, s7, s8, s9, s10, s11, s12, s13,
+ y_filter, round_offset_vec);
+ uint16x8_t d7 = convolve8_8_y(s7, s8, s9, s10, s11, s12, s13, s14,
+ y_filter, round_offset_vec);
+
+ __builtin_prefetch(d + 0 * dst8_stride);
+ __builtin_prefetch(d + 1 * dst8_stride);
+ __builtin_prefetch(d + 2 * dst8_stride);
+ __builtin_prefetch(d + 3 * dst8_stride);
+
+ uint16x8_t dd0, dd1, dd2, dd3;
+ load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
+
+ uint8x8_t d0_u8, d1_u8, d2_u8, d3_u8;
+ compute_basic_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3,
+ round_offset_vec, &d0_u8, &d1_u8, &d2_u8, &d3_u8);
+
+ store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8);
+ d_u8 += 4 * dst8_stride;
+
+ uint16x8_t dd4, dd5, dd6, dd7;
+ load_u16_8x4(d + 4 * dst_stride, dst_stride, &dd4, &dd5, &dd6, &dd7);
+
+ uint8x8_t d4_u8, d5_u8, d6_u8, d7_u8;
+ compute_basic_avg_8x4(dd4, dd5, dd6, dd7, d4, d5, d6, d7,
+ round_offset_vec, &d4_u8, &d5_u8, &d6_u8, &d7_u8);
+
+ store_u8_8x4(d_u8, dst8_stride, d4_u8, d5_u8, d6_u8, d7_u8);
+ d_u8 += 4 * dst8_stride;
+
+ s0 = s8;
+ s1 = s9;
+ s2 = s10;
+ s3 = s11;
+ s4 = s12;
+ s5 = s13;
+ s6 = s14;
+ s += 8 * src_stride;
+ d += 8 * dst_stride;
+ height -= 8;
+#else // !AOM_ARCH_AARCH64
+ int16x8_t s7 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
+
+ __builtin_prefetch(dst_ptr);
+
+ uint16x8_t d0 = convolve8_8_y(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
+ round_offset_vec);
+
+ s0 = s1;
+ s1 = s2;
+ s2 = s3;
+ s3 = s4;
+ s4 = s5;
+ s5 = s6;
+ s6 = s7;
+
+ __builtin_prefetch(d);
+
+ uint16x8_t dd0 = vld1q_u16(d);
+
+ uint8x8_t d0_u8;
+ compute_basic_avg_8x1(dd0, d0, round_offset_vec, &d0_u8);
+
+ vst1_u8(d_u8, d0_u8);
+ d_u8 += dst8_stride;
+
+ s += src_stride;
+ d += dst_stride;
+ height--;
+#endif // AOM_ARCH_AARCH64
+ } while (height != 0);
+ src_ptr += 8;
+ dst_ptr += 8;
+ dst8_ptr += 8;
+ width -= 8;
+ } while (width != 0);
+ }
+}
+
+static INLINE void dist_wtd_convolve_y_8tap_neon(const uint8_t *src_ptr,
+ int src_stride, int w, int h,
+ const int16x8_t y_filter,
+ ConvolveParams *conv_params) {
+ const int bd = 8;
+ const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
+ const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
+ (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
+ const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
+
+ CONV_BUF_TYPE *dst_ptr = conv_params->dst;
+ const int dst_stride = conv_params->dst_stride;
+ int width = w;
+
+ if (w == 4 || h == 4) {
+ do {
+ const uint8_t *s = src_ptr;
+ CONV_BUF_TYPE *d = dst_ptr;
+ int height = h;
+
+ __builtin_prefetch(s + 0 * src_stride);
+ __builtin_prefetch(s + 1 * src_stride);
+ __builtin_prefetch(s + 2 * src_stride);
+ __builtin_prefetch(s + 3 * src_stride);
+
+ uint8x8_t t0 = load_unaligned_u8_4x1(s + 0 * src_stride);
+ uint8x8_t t1 = load_unaligned_u8_4x1(s + 1 * src_stride);
+ uint8x8_t t2 = load_unaligned_u8_4x1(s + 2 * src_stride);
+ uint8x8_t t3 = load_unaligned_u8_4x1(s + 3 * src_stride);
+ uint8x8_t t4 = load_unaligned_u8_4x1(s + 4 * src_stride);
+ uint8x8_t t5 = load_unaligned_u8_4x1(s + 5 * src_stride);
+ uint8x8_t t6 = load_unaligned_u8_4x1(s + 6 * src_stride);
+
+ int16x4_t s0 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
+ int16x4_t s1 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t1)));
+ int16x4_t s2 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t2)));
+ int16x4_t s3 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t3)));
+ int16x4_t s4 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t4)));
+ int16x4_t s5 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t5)));
+ int16x4_t s6 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t6)));
+
+ __builtin_prefetch(d + 0 * dst_stride);
+ __builtin_prefetch(d + 1 * dst_stride);
+ __builtin_prefetch(d + 2 * dst_stride);
+ __builtin_prefetch(d + 3 * dst_stride);
+
+ s += 7 * src_stride;
+
+ do {
+#if AOM_ARCH_AARCH64
+ t0 = load_unaligned_u8_4x1(s + 0 * src_stride);
+ t1 = load_unaligned_u8_4x1(s + 1 * src_stride);
+ t2 = load_unaligned_u8_4x1(s + 2 * src_stride);
+ t3 = load_unaligned_u8_4x1(s + 3 * src_stride);
+
+ int16x4_t s7 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
+ int16x4_t s8 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t1)));
+ int16x4_t s9 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t2)));
+ int16x4_t s10 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t3)));
+
+ uint16x4_t d0 = convolve8_4_y(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
+ vget_low_s16(round_offset_vec));
+ uint16x4_t d1 = convolve8_4_y(s1, s2, s3, s4, s5, s6, s7, s8, y_filter,
+ vget_low_s16(round_offset_vec));
+ uint16x4_t d2 = convolve8_4_y(s2, s3, s4, s5, s6, s7, s8, s9, y_filter,
+ vget_low_s16(round_offset_vec));
+ uint16x4_t d3 = convolve8_4_y(s3, s4, s5, s6, s7, s8, s9, s10, y_filter,
+ vget_low_s16(round_offset_vec));
+
+ store_u16_4x4(d, dst_stride, d0, d1, d2, d3);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s5 = s9;
+ s6 = s10;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ height -= 4;
+#else // !AOM_ARCH_AARCH64
+ t0 = load_unaligned_u8_4x1(s);
+ int16x4_t s7 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
+
+ uint16x4_t d0 = convolve8_4_y(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
+ vget_low_s16(round_offset_vec));
+
+ vst1_u16(d, d0);
+
+ s0 = s1;
+ s1 = s2;
+ s2 = s3;
+ s3 = s4;
+ s4 = s5;
+ s5 = s6;
+ s6 = s7;
+ s += src_stride;
+ d += dst_stride;
+ height--;
+#endif // AOM_ARCH_AARCH64
+ } while (height != 0);
+ src_ptr += 4;
+ dst_ptr += 4;
+ width -= 4;
+ } while (width != 0);
+ } else {
+ do {
+ const uint8_t *s = src_ptr;
+ CONV_BUF_TYPE *d = dst_ptr;
+ int height = h;
+
+ __builtin_prefetch(s + 0 * src_stride);
+ __builtin_prefetch(s + 1 * src_stride);
+ __builtin_prefetch(s + 2 * src_stride);
+ __builtin_prefetch(s + 3 * src_stride);
+ __builtin_prefetch(s + 4 * src_stride);
+ __builtin_prefetch(s + 5 * src_stride);
+ __builtin_prefetch(s + 6 * src_stride);
+ __builtin_prefetch(s + 7 * src_stride);
+
+ uint8x8_t t0, t1, t2, t3, t4, t5, t6;
+ load_u8_8x7(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6);
+
+ int16x8_t s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
+ int16x8_t s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
+ int16x8_t s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
+ int16x8_t s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
+ int16x8_t s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
+ int16x8_t s5 = vreinterpretq_s16_u16(vmovl_u8(t5));
+ int16x8_t s6 = vreinterpretq_s16_u16(vmovl_u8(t6));
+
+ s += 7 * src_stride;
+
+ do {
+#if AOM_ARCH_AARCH64
+ uint8x8_t t7;
+ load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
+
+ int16x8_t s7 = vreinterpretq_s16_u16(vmovl_u8(t0));
+ int16x8_t s8 = vreinterpretq_s16_u16(vmovl_u8(t1));
+ int16x8_t s9 = vreinterpretq_s16_u16(vmovl_u8(t2));
+ int16x8_t s10 = vreinterpretq_s16_u16(vmovl_u8(t3));
+ int16x8_t s11 = vreinterpretq_s16_u16(vmovl_u8(t4));
+ int16x8_t s12 = vreinterpretq_s16_u16(vmovl_u8(t5));
+ int16x8_t s13 = vreinterpretq_s16_u16(vmovl_u8(t6));
+ int16x8_t s14 = vreinterpretq_s16_u16(vmovl_u8(t7));
+
+ __builtin_prefetch(dst_ptr + 0 * dst_stride);
+ __builtin_prefetch(dst_ptr + 1 * dst_stride);
+ __builtin_prefetch(dst_ptr + 2 * dst_stride);
+ __builtin_prefetch(dst_ptr + 3 * dst_stride);
+
+ uint16x8_t d0 = convolve8_8_y(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
+ round_offset_vec);
+ uint16x8_t d1 = convolve8_8_y(s1, s2, s3, s4, s5, s6, s7, s8, y_filter,
+ round_offset_vec);
+ uint16x8_t d2 = convolve8_8_y(s2, s3, s4, s5, s6, s7, s8, s9, y_filter,
+ round_offset_vec);
+ uint16x8_t d3 = convolve8_8_y(s3, s4, s5, s6, s7, s8, s9, s10, y_filter,
+ round_offset_vec);
+ uint16x8_t d4 = convolve8_8_y(s4, s5, s6, s7, s8, s9, s10, s11,
+ y_filter, round_offset_vec);
+ uint16x8_t d5 = convolve8_8_y(s5, s6, s7, s8, s9, s10, s11, s12,
+ y_filter, round_offset_vec);
+ uint16x8_t d6 = convolve8_8_y(s6, s7, s8, s9, s10, s11, s12, s13,
+ y_filter, round_offset_vec);
+ uint16x8_t d7 = convolve8_8_y(s7, s8, s9, s10, s11, s12, s13, s14,
+ y_filter, round_offset_vec);
+
+ store_u16_8x8(d, dst_stride, d0, d1, d2, d3, d4, d5, d6, d7);
+
+ s0 = s8;
+ s1 = s9;
+ s2 = s10;
+ s3 = s11;
+ s4 = s12;
+ s5 = s13;
+ s6 = s14;
+ s += 8 * src_stride;
+ d += 8 * dst_stride;
+ height -= 8;
+#else // !AOM_ARCH_AARCH64
+ int16x8_t s7 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
+
+ __builtin_prefetch(dst_ptr);
+
+ uint16x8_t d0 = convolve8_8_y(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
+ round_offset_vec);
+
+ s0 = s1;
+ s1 = s2;
+ s2 = s3;
+ s3 = s4;
+ s4 = s5;
+ s5 = s6;
+ s6 = s7;
+
+ vst1q_u16(d, d0);
+
+ s += src_stride;
+ d += dst_stride;
+ height--;
+#endif // AOM_ARCH_AARCH64
+ } while (height != 0);
+ src_ptr += 8;
+ dst_ptr += 8;
+ width -= 8;
+ } while (width != 0);
+ }
+}
+
+void av1_dist_wtd_convolve_y_neon(const uint8_t *src, int src_stride,
+ uint8_t *dst8, int dst8_stride, int w, int h,
+ const InterpFilterParams *filter_params_y,
+ const int subpel_y_qn,
+ ConvolveParams *conv_params) {
+ assert(w % 4 == 0);
+ assert(h % 4 == 0);
+
+ // Vertical filter.
+ const int16_t *y_filter_ptr = av1_get_interp_filter_subpel_kernel(
+ filter_params_y, subpel_y_qn & SUBPEL_MASK);
+ // Filter values are even, so downshift by 1 to reduce intermediate
+ // precision requirements.
+ const int16x8_t y_filter = vshrq_n_s16(vld1q_s16(y_filter_ptr), 1);
+
+ const int vert_offset = filter_params_y->taps / 2 - 1;
+ const uint8_t *src_ptr = src - (vert_offset * src_stride);
+
+ if (get_filter_tap(filter_params_y, subpel_y_qn) <= 6) {
+ if (conv_params->do_average) {
+ if (UNLIKELY(conv_params->use_dist_wtd_comp_avg)) {
+ dist_wtd_convolve_y_6tap_dist_wtd_avg_neon(
+ src_ptr + src_stride, src_stride, dst8, dst8_stride, w, h, y_filter,
+ conv_params);
+ } else {
+ dist_wtd_convolve_y_6tap_avg_neon(src_ptr + src_stride, src_stride,
+ dst8, dst8_stride, w, h, y_filter,
+ conv_params);
+ }
+ } else {
+ dist_wtd_convolve_y_6tap_neon(src_ptr + src_stride, src_stride, w, h,
+ y_filter, conv_params);
+ }
+ } else {
+ if (conv_params->do_average) {
+ if (UNLIKELY(conv_params->use_dist_wtd_comp_avg)) {
+ dist_wtd_convolve_y_8tap_dist_wtd_avg_neon(src_ptr, src_stride, dst8,
+ dst8_stride, w, h, y_filter,
+ conv_params);
+ } else {
+ dist_wtd_convolve_y_8tap_avg_neon(src_ptr, src_stride, dst8,
+ dst8_stride, w, h, y_filter,
+ conv_params);
+ }
+ } else {
+ dist_wtd_convolve_y_8tap_neon(src_ptr, src_stride, w, h, y_filter,
+ conv_params);
+ }
+ }
+}
diff --git a/av1/common/arm/compound_convolve_neon.h b/av1/common/arm/compound_convolve_neon.h
new file mode 100644
index 0000000..cff6838
--- /dev/null
+++ b/av1/common/arm/compound_convolve_neon.h
@@ -0,0 +1,1172 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+#ifndef AOM_AV1_COMMON_ARM_COMPOUND_CONVOLVE_NEON_H_
+#define AOM_AV1_COMMON_ARM_COMPOUND_CONVOLVE_NEON_H_
+
+#include <arm_neon.h>
+
+#include "av1/common/convolve.h"
+#include "av1/common/enums.h"
+#include "av1/common/filter.h"
+
+static INLINE void compute_dist_wtd_avg_4x1(uint16x4_t dd0, uint16x4_t d0,
+ const uint16_t fwd_offset,
+ const uint16_t bck_offset,
+ const int16x4_t round_offset,
+ uint8x8_t *d0_u8) {
+ uint32x4_t blend0 = vmull_n_u16(dd0, fwd_offset);
+ blend0 = vmlal_n_u16(blend0, d0, bck_offset);
+
+ uint16x4_t avg0 = vshrn_n_u32(blend0, DIST_PRECISION_BITS);
+
+ int16x4_t dst0 = vsub_s16(vreinterpret_s16_u16(avg0), round_offset);
+
+ int16x8_t dst0q = vcombine_s16(dst0, vdup_n_s16(0));
+
+ *d0_u8 = vqrshrun_n_s16(dst0q, FILTER_BITS - ROUND0_BITS);
+}
+
+static INLINE void compute_basic_avg_4x1(uint16x4_t dd0, uint16x4_t d0,
+ const int16x4_t round_offset,
+ uint8x8_t *d0_u8) {
+ uint16x4_t avg0 = vhadd_u16(dd0, d0);
+
+ int16x4_t dst0 = vsub_s16(vreinterpret_s16_u16(avg0), round_offset);
+
+ int16x8_t dst0q = vcombine_s16(dst0, vdup_n_s16(0));
+
+ *d0_u8 = vqrshrun_n_s16(dst0q, FILTER_BITS - ROUND0_BITS);
+}
+
+static INLINE void compute_dist_wtd_avg_8x1(uint16x8_t dd0, uint16x8_t d0,
+ const uint16_t fwd_offset,
+ const uint16_t bck_offset,
+ const int16x8_t round_offset,
+ uint8x8_t *d0_u8) {
+ uint32x4_t blend0_lo = vmull_n_u16(vget_low_u16(dd0), fwd_offset);
+ blend0_lo = vmlal_n_u16(blend0_lo, vget_low_u16(d0), bck_offset);
+ uint32x4_t blend0_hi = vmull_n_u16(vget_high_u16(dd0), fwd_offset);
+ blend0_hi = vmlal_n_u16(blend0_hi, vget_high_u16(d0), bck_offset);
+
+ uint16x8_t avg0 = vcombine_u16(vshrn_n_u32(blend0_lo, DIST_PRECISION_BITS),
+ vshrn_n_u32(blend0_hi, DIST_PRECISION_BITS));
+
+ int16x8_t dst0 = vsubq_s16(vreinterpretq_s16_u16(avg0), round_offset);
+
+ *d0_u8 = vqrshrun_n_s16(dst0, FILTER_BITS - ROUND0_BITS);
+}
+
+static INLINE void compute_basic_avg_8x1(uint16x8_t dd0, uint16x8_t d0,
+ const int16x8_t round_offset,
+ uint8x8_t *d0_u8) {
+ uint16x8_t avg0 = vhaddq_u16(dd0, d0);
+
+ int16x8_t dst0 = vsubq_s16(vreinterpretq_s16_u16(avg0), round_offset);
+
+ *d0_u8 = vqrshrun_n_s16(dst0, FILTER_BITS - ROUND0_BITS);
+}
+
+static INLINE void compute_dist_wtd_avg_4x4(
+ uint16x4_t dd0, uint16x4_t dd1, uint16x4_t dd2, uint16x4_t dd3,
+ uint16x4_t d0, uint16x4_t d1, uint16x4_t d2, uint16x4_t d3,
+ const uint16_t fwd_offset, const uint16_t bck_offset,
+ const int16x8_t round_offset, uint8x8_t *d01_u8, uint8x8_t *d23_u8) {
+ uint32x4_t blend0 = vmull_n_u16(dd0, fwd_offset);
+ blend0 = vmlal_n_u16(blend0, d0, bck_offset);
+ uint32x4_t blend1 = vmull_n_u16(dd1, fwd_offset);
+ blend1 = vmlal_n_u16(blend1, d1, bck_offset);
+ uint32x4_t blend2 = vmull_n_u16(dd2, fwd_offset);
+ blend2 = vmlal_n_u16(blend2, d2, bck_offset);
+ uint32x4_t blend3 = vmull_n_u16(dd3, fwd_offset);
+ blend3 = vmlal_n_u16(blend3, d3, bck_offset);
+
+ uint16x4_t avg0 = vshrn_n_u32(blend0, DIST_PRECISION_BITS);
+ uint16x4_t avg1 = vshrn_n_u32(blend1, DIST_PRECISION_BITS);
+ uint16x4_t avg2 = vshrn_n_u32(blend2, DIST_PRECISION_BITS);
+ uint16x4_t avg3 = vshrn_n_u32(blend3, DIST_PRECISION_BITS);
+
+ int16x8_t dst_01 = vreinterpretq_s16_u16(vcombine_u16(avg0, avg1));
+ int16x8_t dst_23 = vreinterpretq_s16_u16(vcombine_u16(avg2, avg3));
+
+ dst_01 = vsubq_s16(dst_01, round_offset);
+ dst_23 = vsubq_s16(dst_23, round_offset);
+
+ *d01_u8 = vqrshrun_n_s16(dst_01, FILTER_BITS - ROUND0_BITS);
+ *d23_u8 = vqrshrun_n_s16(dst_23, FILTER_BITS - ROUND0_BITS);
+}
+
+static INLINE void compute_basic_avg_4x4(uint16x4_t dd0, uint16x4_t dd1,
+ uint16x4_t dd2, uint16x4_t dd3,
+ uint16x4_t d0, uint16x4_t d1,
+ uint16x4_t d2, uint16x4_t d3,
+ const int16x8_t round_offset,
+ uint8x8_t *d01_u8, uint8x8_t *d23_u8) {
+ uint16x4_t avg0 = vhadd_u16(dd0, d0);
+ uint16x4_t avg1 = vhadd_u16(dd1, d1);
+ uint16x4_t avg2 = vhadd_u16(dd2, d2);
+ uint16x4_t avg3 = vhadd_u16(dd3, d3);
+
+ int16x8_t dst_01 = vreinterpretq_s16_u16(vcombine_u16(avg0, avg1));
+ int16x8_t dst_23 = vreinterpretq_s16_u16(vcombine_u16(avg2, avg3));
+
+ dst_01 = vsubq_s16(dst_01, round_offset);
+ dst_23 = vsubq_s16(dst_23, round_offset);
+
+ *d01_u8 = vqrshrun_n_s16(dst_01, FILTER_BITS - ROUND0_BITS);
+ *d23_u8 = vqrshrun_n_s16(dst_23, FILTER_BITS - ROUND0_BITS);
+}
+
+static INLINE void compute_dist_wtd_avg_8x4(
+ uint16x8_t dd0, uint16x8_t dd1, uint16x8_t dd2, uint16x8_t dd3,
+ uint16x8_t d0, uint16x8_t d1, uint16x8_t d2, uint16x8_t d3,
+ const uint16_t fwd_offset, const uint16_t bck_offset,
+ const int16x8_t round_offset, uint8x8_t *d0_u8, uint8x8_t *d1_u8,
+ uint8x8_t *d2_u8, uint8x8_t *d3_u8) {
+ uint32x4_t blend0_lo = vmull_n_u16(vget_low_u16(dd0), fwd_offset);
+ blend0_lo = vmlal_n_u16(blend0_lo, vget_low_u16(d0), bck_offset);
+ uint32x4_t blend0_hi = vmull_n_u16(vget_high_u16(dd0), fwd_offset);
+ blend0_hi = vmlal_n_u16(blend0_hi, vget_high_u16(d0), bck_offset);
+
+ uint32x4_t blend1_lo = vmull_n_u16(vget_low_u16(dd1), fwd_offset);
+ blend1_lo = vmlal_n_u16(blend1_lo, vget_low_u16(d1), bck_offset);
+ uint32x4_t blend1_hi = vmull_n_u16(vget_high_u16(dd1), fwd_offset);
+ blend1_hi = vmlal_n_u16(blend1_hi, vget_high_u16(d1), bck_offset);
+
+ uint32x4_t blend2_lo = vmull_n_u16(vget_low_u16(dd2), fwd_offset);
+ blend2_lo = vmlal_n_u16(blend2_lo, vget_low_u16(d2), bck_offset);
+ uint32x4_t blend2_hi = vmull_n_u16(vget_high_u16(dd2), fwd_offset);
+ blend2_hi = vmlal_n_u16(blend2_hi, vget_high_u16(d2), bck_offset);
+
+ uint32x4_t blend3_lo = vmull_n_u16(vget_low_u16(dd3), fwd_offset);
+ blend3_lo = vmlal_n_u16(blend3_lo, vget_low_u16(d3), bck_offset);
+ uint32x4_t blend3_hi = vmull_n_u16(vget_high_u16(dd3), fwd_offset);
+ blend3_hi = vmlal_n_u16(blend3_hi, vget_high_u16(d3), bck_offset);
+
+ uint16x8_t avg0 = vcombine_u16(vshrn_n_u32(blend0_lo, DIST_PRECISION_BITS),
+ vshrn_n_u32(blend0_hi, DIST_PRECISION_BITS));
+ uint16x8_t avg1 = vcombine_u16(vshrn_n_u32(blend1_lo, DIST_PRECISION_BITS),
+ vshrn_n_u32(blend1_hi, DIST_PRECISION_BITS));
+ uint16x8_t avg2 = vcombine_u16(vshrn_n_u32(blend2_lo, DIST_PRECISION_BITS),
+ vshrn_n_u32(blend2_hi, DIST_PRECISION_BITS));
+ uint16x8_t avg3 = vcombine_u16(vshrn_n_u32(blend3_lo, DIST_PRECISION_BITS),
+ vshrn_n_u32(blend3_hi, DIST_PRECISION_BITS));
+
+ int16x8_t dst0 = vsubq_s16(vreinterpretq_s16_u16(avg0), round_offset);
+ int16x8_t dst1 = vsubq_s16(vreinterpretq_s16_u16(avg1), round_offset);
+ int16x8_t dst2 = vsubq_s16(vreinterpretq_s16_u16(avg2), round_offset);
+ int16x8_t dst3 = vsubq_s16(vreinterpretq_s16_u16(avg3), round_offset);
+
+ *d0_u8 = vqrshrun_n_s16(dst0, FILTER_BITS - ROUND0_BITS);
+ *d1_u8 = vqrshrun_n_s16(dst1, FILTER_BITS - ROUND0_BITS);
+ *d2_u8 = vqrshrun_n_s16(dst2, FILTER_BITS - ROUND0_BITS);
+ *d3_u8 = vqrshrun_n_s16(dst3, FILTER_BITS - ROUND0_BITS);
+}
+
+static INLINE void compute_basic_avg_8x4(uint16x8_t dd0, uint16x8_t dd1,
+ uint16x8_t dd2, uint16x8_t dd3,
+ uint16x8_t d0, uint16x8_t d1,
+ uint16x8_t d2, uint16x8_t d3,
+ const int16x8_t round_offset,
+ uint8x8_t *d0_u8, uint8x8_t *d1_u8,
+ uint8x8_t *d2_u8, uint8x8_t *d3_u8) {
+ uint16x8_t avg0 = vhaddq_u16(dd0, d0);
+ uint16x8_t avg1 = vhaddq_u16(dd1, d1);
+ uint16x8_t avg2 = vhaddq_u16(dd2, d2);
+ uint16x8_t avg3 = vhaddq_u16(dd3, d3);
+
+ int16x8_t dst0 = vsubq_s16(vreinterpretq_s16_u16(avg0), round_offset);
+ int16x8_t dst1 = vsubq_s16(vreinterpretq_s16_u16(avg1), round_offset);
+ int16x8_t dst2 = vsubq_s16(vreinterpretq_s16_u16(avg2), round_offset);
+ int16x8_t dst3 = vsubq_s16(vreinterpretq_s16_u16(avg3), round_offset);
+
+ *d0_u8 = vqrshrun_n_s16(dst0, FILTER_BITS - ROUND0_BITS);
+ *d1_u8 = vqrshrun_n_s16(dst1, FILTER_BITS - ROUND0_BITS);
+ *d2_u8 = vqrshrun_n_s16(dst2, FILTER_BITS - ROUND0_BITS);
+ *d3_u8 = vqrshrun_n_s16(dst3, FILTER_BITS - ROUND0_BITS);
+}
+
+static INLINE uint16x4_t
+convolve6_4_2d_v(const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
+ const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
+ const int16x8_t y_filter, const int32x4_t offset_const) {
+ const int16x4_t y_filter_0_3 = vget_low_s16(y_filter);
+ const int16x4_t y_filter_4_7 = vget_high_s16(y_filter);
+
+ int32x4_t sum = offset_const;
+ // Filter values at indices 0 and 7 are 0.
+ sum = vmlal_lane_s16(sum, s0, y_filter_0_3, 1);
+ sum = vmlal_lane_s16(sum, s1, y_filter_0_3, 2);
+ sum = vmlal_lane_s16(sum, s2, y_filter_0_3, 3);
+ sum = vmlal_lane_s16(sum, s3, y_filter_4_7, 0);
+ sum = vmlal_lane_s16(sum, s4, y_filter_4_7, 1);
+ sum = vmlal_lane_s16(sum, s5, y_filter_4_7, 2);
+
+ return vqrshrun_n_s32(sum, COMPOUND_ROUND1_BITS);
+}
+
+static INLINE uint16x8_t
+convolve6_8_2d_v(const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
+ const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t y_filter, const int32x4_t offset_const) {
+ const int16x4_t y_filter_0_3 = vget_low_s16(y_filter);
+ const int16x4_t y_filter_4_7 = vget_high_s16(y_filter);
+
+ int32x4_t sum0 = offset_const;
+ // Filter values at indices 0 and 7 are 0.
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s0), y_filter_0_3, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s1), y_filter_0_3, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s2), y_filter_0_3, 3);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s3), y_filter_4_7, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s4), y_filter_4_7, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s5), y_filter_4_7, 2);
+
+ int32x4_t sum1 = offset_const;
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s0), y_filter_0_3, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s1), y_filter_0_3, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s2), y_filter_0_3, 3);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s3), y_filter_4_7, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s4), y_filter_4_7, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s5), y_filter_4_7, 2);
+
+ return vcombine_u16(vqrshrun_n_s32(sum0, COMPOUND_ROUND1_BITS),
+ vqrshrun_n_s32(sum1, COMPOUND_ROUND1_BITS));
+}
+
+static INLINE void dist_wtd_convolve_2d_vert_6tap_dist_wtd_avg_neon(
+ int16_t *src_ptr, const int src_stride, uint8_t *dst8_ptr, int dst8_stride,
+ ConvolveParams *conv_params, const int16x8_t y_filter, int h, int w) {
+ const int bd = 8;
+ const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
+ const int32x4_t offset_const = vdupq_n_s32(1 << offset_bits);
+ const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
+ (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
+ const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
+
+ const uint16_t fwd_offset = conv_params->fwd_offset;
+ const uint16_t bck_offset = conv_params->bck_offset;
+
+ CONV_BUF_TYPE *dst_ptr = conv_params->dst;
+ const int dst_stride = conv_params->dst_stride;
+
+ if (w == 4) {
+ int16x4_t s0, s1, s2, s3, s4;
+ load_s16_4x5(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4);
+ src_ptr += 5 * src_stride;
+
+ do {
+#if AOM_ARCH_AARCH64
+ int16x4_t s5, s6, s7, s8;
+ load_s16_4x4(src_ptr, src_stride, &s5, &s6, &s7, &s8);
+
+ uint16x4_t d0 =
+ convolve6_4_2d_v(s0, s1, s2, s3, s4, s5, y_filter, offset_const);
+ uint16x4_t d1 =
+ convolve6_4_2d_v(s1, s2, s3, s4, s5, s6, y_filter, offset_const);
+ uint16x4_t d2 =
+ convolve6_4_2d_v(s2, s3, s4, s5, s6, s7, y_filter, offset_const);
+ uint16x4_t d3 =
+ convolve6_4_2d_v(s3, s4, s5, s6, s7, s8, y_filter, offset_const);
+
+ uint16x4_t dd0, dd1, dd2, dd3;
+ load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3);
+
+ uint8x8_t d01_u8, d23_u8;
+ compute_dist_wtd_avg_4x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset,
+ bck_offset, round_offset_vec, &d01_u8, &d23_u8);
+
+ store_u8_4x1(dst8_ptr + 0 * dst8_stride, d01_u8, 0);
+ store_u8_4x1(dst8_ptr + 1 * dst8_stride, d01_u8, 1);
+ store_u8_4x1(dst8_ptr + 2 * dst8_stride, d23_u8, 0);
+ store_u8_4x1(dst8_ptr + 3 * dst8_stride, d23_u8, 1);
+ dst8_ptr += 4 * dst8_stride;
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ h -= 4;
+#else // !AOM_ARCH_AARCH64
+ int16x4_t s5 = vld1_s16(src_ptr);
+
+ uint16x4_t d0 =
+ convolve6_4_2d_v(s0, s1, s2, s3, s4, s5, y_filter, offset_const);
+
+ uint16x4_t dd0 = vld1_u16(dst_ptr);
+
+ uint8x8_t d01_u8;
+ compute_dist_wtd_avg_4x1(dd0, d0, fwd_offset, bck_offset,
+ vget_low_s16(round_offset_vec), &d01_u8);
+
+ store_u8_4x1(dst8_ptr, d01_u8, 0);
+ dst8_ptr += dst8_stride;
+
+ s0 = s1;
+ s1 = s2;
+ s2 = s3;
+ s3 = s4;
+ s4 = s5;
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ h--;
+#endif // AOM_ARCH_AARCH64
+ } while (h != 0);
+ } else {
+ do {
+ int16_t *s = src_ptr;
+ CONV_BUF_TYPE *d = dst_ptr;
+ uint8_t *d_u8 = dst8_ptr;
+ int height = h;
+
+ int16x8_t s0, s1, s2, s3, s4;
+ load_s16_8x5(s, src_stride, &s0, &s1, &s2, &s3, &s4);
+ s += 5 * src_stride;
+
+ do {
+#if AOM_ARCH_AARCH64
+ int16x8_t s5, s6, s7, s8;
+ load_s16_8x4(s, src_stride, &s5, &s6, &s7, &s8);
+
+ uint16x8_t d0 =
+ convolve6_8_2d_v(s0, s1, s2, s3, s4, s5, y_filter, offset_const);
+ uint16x8_t d1 =
+ convolve6_8_2d_v(s1, s2, s3, s4, s5, s6, y_filter, offset_const);
+ uint16x8_t d2 =
+ convolve6_8_2d_v(s2, s3, s4, s5, s6, s7, y_filter, offset_const);
+ uint16x8_t d3 =
+ convolve6_8_2d_v(s3, s4, s5, s6, s7, s8, y_filter, offset_const);
+
+ uint16x8_t dd0, dd1, dd2, dd3;
+ load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
+
+ uint8x8_t d0_u8, d1_u8, d2_u8, d3_u8;
+ compute_dist_wtd_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset,
+ bck_offset, round_offset_vec, &d0_u8, &d1_u8,
+ &d2_u8, &d3_u8);
+
+ store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8);
+ d_u8 += 4 * dst8_stride;
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ height -= 4;
+#else // !AOM_ARCH_AARCH64
+ int16x8_t s5 = vld1q_s16(s);
+
+ uint16x8_t d0 =
+ convolve6_8_2d_v(s0, s1, s2, s3, s4, s5, y_filter, offset_const);
+
+ uint16x8_t dd0 = vld1q_u16(d);
+
+ uint8x8_t d0_u8;
+ compute_dist_wtd_avg_8x1(dd0, d0, fwd_offset, bck_offset,
+ round_offset_vec, &d0_u8);
+
+ vst1_u8(d_u8, d0_u8);
+ d_u8 += dst8_stride;
+
+ s0 = s1;
+ s1 = s2;
+ s2 = s3;
+ s3 = s4;
+ s4 = s5;
+ s += src_stride;
+ d += dst_stride;
+ height--;
+#endif // AOM_ARCH_AARCH64
+ } while (height != 0);
+ src_ptr += 8;
+ dst_ptr += 8;
+ dst8_ptr += 8;
+ w -= 8;
+ } while (w != 0);
+ }
+}
+
+static INLINE void dist_wtd_convolve_2d_vert_6tap_avg_neon(
+ int16_t *src_ptr, const int src_stride, uint8_t *dst8_ptr, int dst8_stride,
+ ConvolveParams *conv_params, const int16x8_t y_filter, int h, int w) {
+ const int bd = 8;
+ const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
+ const int32x4_t offset_const = vdupq_n_s32(1 << offset_bits);
+ const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
+ (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
+ const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
+
+ CONV_BUF_TYPE *dst_ptr = conv_params->dst;
+ const int dst_stride = conv_params->dst_stride;
+
+ if (w == 4) {
+ int16x4_t s0, s1, s2, s3, s4;
+ load_s16_4x5(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4);
+ src_ptr += 5 * src_stride;
+
+ do {
+#if AOM_ARCH_AARCH64
+ int16x4_t s5, s6, s7, s8;
+ load_s16_4x4(src_ptr, src_stride, &s5, &s6, &s7, &s8);
+
+ uint16x4_t d0 =
+ convolve6_4_2d_v(s0, s1, s2, s3, s4, s5, y_filter, offset_const);
+ uint16x4_t d1 =
+ convolve6_4_2d_v(s1, s2, s3, s4, s5, s6, y_filter, offset_const);
+ uint16x4_t d2 =
+ convolve6_4_2d_v(s2, s3, s4, s5, s6, s7, y_filter, offset_const);
+ uint16x4_t d3 =
+ convolve6_4_2d_v(s3, s4, s5, s6, s7, s8, y_filter, offset_const);
+
+ uint16x4_t dd0, dd1, dd2, dd3;
+ load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3);
+
+ uint8x8_t d01_u8, d23_u8;
+ compute_basic_avg_4x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3,
+ round_offset_vec, &d01_u8, &d23_u8);
+
+ store_u8_4x1(dst8_ptr + 0 * dst8_stride, d01_u8, 0);
+ store_u8_4x1(dst8_ptr + 1 * dst8_stride, d01_u8, 1);
+ store_u8_4x1(dst8_ptr + 2 * dst8_stride, d23_u8, 0);
+ store_u8_4x1(dst8_ptr + 3 * dst8_stride, d23_u8, 1);
+ dst8_ptr += 4 * dst8_stride;
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ h -= 4;
+#else // !AOM_ARCH_AARCH64
+ int16x4_t s5 = vld1_s16(src_ptr);
+
+ uint16x4_t d0 =
+ convolve6_4_2d_v(s0, s1, s2, s3, s4, s5, y_filter, offset_const);
+
+ uint16x4_t dd0 = vld1_u16(dst_ptr);
+
+ uint8x8_t d01_u8;
+ compute_basic_avg_4x1(dd0, d0, vget_low_s16(round_offset_vec), &d01_u8);
+
+ store_u8_4x1(dst8_ptr, d01_u8, 0);
+ dst8_ptr += dst8_stride;
+
+ s0 = s1;
+ s1 = s2;
+ s2 = s3;
+ s3 = s4;
+ s4 = s5;
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ h--;
+#endif // AOM_ARCH_AARCH64
+ } while (h != 0);
+ } else {
+ do {
+ int16_t *s = src_ptr;
+ CONV_BUF_TYPE *d = dst_ptr;
+ uint8_t *d_u8 = dst8_ptr;
+ int height = h;
+
+ int16x8_t s0, s1, s2, s3, s4;
+ load_s16_8x5(s, src_stride, &s0, &s1, &s2, &s3, &s4);
+ s += 5 * src_stride;
+
+ do {
+#if AOM_ARCH_AARCH64
+ int16x8_t s5, s6, s7, s8;
+ load_s16_8x4(s, src_stride, &s5, &s6, &s7, &s8);
+
+ uint16x8_t d0 =
+ convolve6_8_2d_v(s0, s1, s2, s3, s4, s5, y_filter, offset_const);
+ uint16x8_t d1 =
+ convolve6_8_2d_v(s1, s2, s3, s4, s5, s6, y_filter, offset_const);
+ uint16x8_t d2 =
+ convolve6_8_2d_v(s2, s3, s4, s5, s6, s7, y_filter, offset_const);
+ uint16x8_t d3 =
+ convolve6_8_2d_v(s3, s4, s5, s6, s7, s8, y_filter, offset_const);
+
+ uint16x8_t dd0, dd1, dd2, dd3;
+ load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
+
+ uint8x8_t d0_u8, d1_u8, d2_u8, d3_u8;
+ compute_basic_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3,
+ round_offset_vec, &d0_u8, &d1_u8, &d2_u8, &d3_u8);
+
+ store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8);
+ d_u8 += 4 * dst8_stride;
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ height -= 4;
+#else // !AOM_ARCH_AARCH64
+ int16x8_t s5 = vld1q_s16(s);
+
+ uint16x8_t d0 =
+ convolve6_8_2d_v(s0, s1, s2, s3, s4, s5, y_filter, offset_const);
+
+ uint16x8_t dd0 = vld1q_u16(d);
+
+ uint8x8_t d0_u8;
+ compute_basic_avg_8x1(dd0, d0, round_offset_vec, &d0_u8);
+
+ vst1_u8(d_u8, d0_u8);
+ d_u8 += dst8_stride;
+
+ s0 = s1;
+ s1 = s2;
+ s2 = s3;
+ s3 = s4;
+ s4 = s5;
+ s += src_stride;
+ d += dst_stride;
+ height--;
+#endif // AOM_ARCH_AARCH64
+ } while (height != 0);
+ src_ptr += 8;
+ dst_ptr += 8;
+ dst8_ptr += 8;
+ w -= 8;
+ } while (w != 0);
+ }
+}
+
+static INLINE void dist_wtd_convolve_2d_vert_6tap_neon(
+ int16_t *src_ptr, const int src_stride, ConvolveParams *conv_params,
+ const int16x8_t y_filter, int h, int w) {
+ const int bd = 8;
+ const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
+ const int32x4_t offset_const = vdupq_n_s32(1 << offset_bits);
+
+ CONV_BUF_TYPE *dst_ptr = conv_params->dst;
+ const int dst_stride = conv_params->dst_stride;
+
+ if (w == 4) {
+ int16x4_t s0, s1, s2, s3, s4;
+ load_s16_4x5(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4);
+ src_ptr += 5 * src_stride;
+
+ do {
+#if AOM_ARCH_AARCH64
+ int16x4_t s5, s6, s7, s8;
+ load_s16_4x4(src_ptr, src_stride, &s5, &s6, &s7, &s8);
+
+ uint16x4_t d0 =
+ convolve6_4_2d_v(s0, s1, s2, s3, s4, s5, y_filter, offset_const);
+ uint16x4_t d1 =
+ convolve6_4_2d_v(s1, s2, s3, s4, s5, s6, y_filter, offset_const);
+ uint16x4_t d2 =
+ convolve6_4_2d_v(s2, s3, s4, s5, s6, s7, y_filter, offset_const);
+ uint16x4_t d3 =
+ convolve6_4_2d_v(s3, s4, s5, s6, s7, s8, y_filter, offset_const);
+
+ store_u16_4x4(dst_ptr, dst_stride, d0, d1, d2, d3);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ h -= 4;
+#else // !AOM_ARCH_AARCH64
+ int16x4_t s5 = vld1_s16(src_ptr);
+
+ uint16x4_t d0 =
+ convolve6_4_2d_v(s0, s1, s2, s3, s4, s5, y_filter, offset_const);
+
+ vst1_u16(dst_ptr, d0);
+
+ s0 = s1;
+ s1 = s2;
+ s2 = s3;
+ s3 = s4;
+ s4 = s5;
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ h--;
+#endif // AOM_ARCH_AARCH64
+ } while (h != 0);
+ } else {
+ do {
+ int16_t *s = src_ptr;
+ CONV_BUF_TYPE *d = dst_ptr;
+ int height = h;
+
+ int16x8_t s0, s1, s2, s3, s4;
+ load_s16_8x5(s, src_stride, &s0, &s1, &s2, &s3, &s4);
+ s += 5 * src_stride;
+
+ do {
+#if AOM_ARCH_AARCH64
+ int16x8_t s5, s6, s7, s8;
+ load_s16_8x4(s, src_stride, &s5, &s6, &s7, &s8);
+
+ uint16x8_t d0 =
+ convolve6_8_2d_v(s0, s1, s2, s3, s4, s5, y_filter, offset_const);
+ uint16x8_t d1 =
+ convolve6_8_2d_v(s1, s2, s3, s4, s5, s6, y_filter, offset_const);
+ uint16x8_t d2 =
+ convolve6_8_2d_v(s2, s3, s4, s5, s6, s7, y_filter, offset_const);
+ uint16x8_t d3 =
+ convolve6_8_2d_v(s3, s4, s5, s6, s7, s8, y_filter, offset_const);
+
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ height -= 4;
+#else // !AOM_ARCH_AARCH64
+ int16x8_t s5 = vld1q_s16(s);
+
+ uint16x8_t d0 =
+ convolve6_8_2d_v(s0, s1, s2, s3, s4, s5, y_filter, offset_const);
+
+ vst1q_u16(d, d0);
+
+ s0 = s1;
+ s1 = s2;
+ s2 = s3;
+ s3 = s4;
+ s4 = s5;
+ s += src_stride;
+ d += dst_stride;
+ height--;
+#endif // AOM_ARCH_AARCH64
+ } while (height != 0);
+ src_ptr += 8;
+ dst_ptr += 8;
+ w -= 8;
+ } while (w != 0);
+ }
+}
+
+static INLINE uint16x4_t
+convolve8_4_2d_v(const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
+ const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
+ const int16x4_t s6, const int16x4_t s7,
+ const int16x8_t y_filter, const int32x4_t offset_const) {
+ const int16x4_t y_filter_0_3 = vget_low_s16(y_filter);
+ const int16x4_t y_filter_4_7 = vget_high_s16(y_filter);
+
+ int32x4_t sum = offset_const;
+ sum = vmlal_lane_s16(sum, s0, y_filter_0_3, 0);
+ sum = vmlal_lane_s16(sum, s1, y_filter_0_3, 1);
+ sum = vmlal_lane_s16(sum, s2, y_filter_0_3, 2);
+ sum = vmlal_lane_s16(sum, s3, y_filter_0_3, 3);
+ sum = vmlal_lane_s16(sum, s4, y_filter_4_7, 0);
+ sum = vmlal_lane_s16(sum, s5, y_filter_4_7, 1);
+ sum = vmlal_lane_s16(sum, s6, y_filter_4_7, 2);
+ sum = vmlal_lane_s16(sum, s7, y_filter_4_7, 3);
+
+ return vqrshrun_n_s32(sum, COMPOUND_ROUND1_BITS);
+}
+
+static INLINE uint16x8_t
+convolve8_8_2d_v(const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
+ const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t s6, const int16x8_t s7,
+ const int16x8_t y_filter, const int32x4_t offset_const) {
+ const int16x4_t y_filter_0_3 = vget_low_s16(y_filter);
+ const int16x4_t y_filter_4_7 = vget_high_s16(y_filter);
+
+ int32x4_t sum0 = offset_const;
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s0), y_filter_0_3, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s1), y_filter_0_3, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s2), y_filter_0_3, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s3), y_filter_0_3, 3);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s4), y_filter_4_7, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s5), y_filter_4_7, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s6), y_filter_4_7, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s7), y_filter_4_7, 3);
+
+ int32x4_t sum1 = offset_const;
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s0), y_filter_0_3, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s1), y_filter_0_3, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s2), y_filter_0_3, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s3), y_filter_0_3, 3);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s4), y_filter_4_7, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s5), y_filter_4_7, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s6), y_filter_4_7, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s7), y_filter_4_7, 3);
+
+ return vcombine_u16(vqrshrun_n_s32(sum0, COMPOUND_ROUND1_BITS),
+ vqrshrun_n_s32(sum1, COMPOUND_ROUND1_BITS));
+}
+
+static INLINE void dist_wtd_convolve_2d_vert_8tap_dist_wtd_avg_neon(
+ int16_t *src_ptr, const int src_stride, uint8_t *dst8_ptr, int dst8_stride,
+ ConvolveParams *conv_params, const int16x8_t y_filter, int h, int w) {
+ const int bd = 8;
+ const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
+ const int32x4_t offset_const = vdupq_n_s32(1 << offset_bits);
+ const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
+ (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
+ const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
+
+ const uint16_t fwd_offset = conv_params->fwd_offset;
+ const uint16_t bck_offset = conv_params->bck_offset;
+
+ CONV_BUF_TYPE *dst_ptr = conv_params->dst;
+ const int dst_stride = conv_params->dst_stride;
+
+ if (w == 4) {
+ int16x4_t s0, s1, s2, s3, s4, s5, s6;
+ load_s16_4x7(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
+ src_ptr += 7 * src_stride;
+
+ do {
+#if AOM_ARCH_AARCH64
+ int16x4_t s7, s8, s9, s10;
+ load_s16_4x4(src_ptr, src_stride, &s7, &s8, &s9, &s10);
+
+ uint16x4_t d0 = convolve8_4_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
+ offset_const);
+ uint16x4_t d1 = convolve8_4_2d_v(s1, s2, s3, s4, s5, s6, s7, s8, y_filter,
+ offset_const);
+ uint16x4_t d2 = convolve8_4_2d_v(s2, s3, s4, s5, s6, s7, s8, s9, y_filter,
+ offset_const);
+ uint16x4_t d3 = convolve8_4_2d_v(s3, s4, s5, s6, s7, s8, s9, s10,
+ y_filter, offset_const);
+
+ uint16x4_t dd0, dd1, dd2, dd3;
+ load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3);
+
+ uint8x8_t d01_u8, d23_u8;
+ compute_dist_wtd_avg_4x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset,
+ bck_offset, round_offset_vec, &d01_u8, &d23_u8);
+
+ store_u8_4x1(dst8_ptr + 0 * dst8_stride, d01_u8, 0);
+ store_u8_4x1(dst8_ptr + 1 * dst8_stride, d01_u8, 1);
+ store_u8_4x1(dst8_ptr + 2 * dst8_stride, d23_u8, 0);
+ store_u8_4x1(dst8_ptr + 3 * dst8_stride, d23_u8, 1);
+ dst8_ptr += 4 * dst8_stride;
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s5 = s9;
+ s6 = s10;
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ h -= 4;
+#else // !AOM_ARCH_AARCH64
+ int16x4_t s7 = vld1_s16(src_ptr);
+
+ uint16x4_t d0 = convolve8_4_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
+ offset_const);
+
+ uint16x4_t dd0 = vld1_u16(dst_ptr);
+
+ uint8x8_t d01_u8;
+ compute_dist_wtd_avg_4x1(dd0, d0, fwd_offset, bck_offset,
+ vget_low_s16(round_offset_vec), &d01_u8);
+
+ store_u8_4x1(dst8_ptr, d01_u8, 0);
+ dst8_ptr += dst8_stride;
+
+ s0 = s1;
+ s1 = s2;
+ s2 = s3;
+ s3 = s4;
+ s4 = s5;
+ s5 = s6;
+ s6 = s7;
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ h--;
+#endif // AOM_ARCH_AARCH64
+ } while (h != 0);
+ } else {
+ do {
+ int16_t *s = src_ptr;
+ CONV_BUF_TYPE *d = dst_ptr;
+ uint8_t *d_u8 = dst8_ptr;
+ int height = h;
+
+ int16x8_t s0, s1, s2, s3, s4, s5, s6;
+ load_s16_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
+ s += 7 * src_stride;
+
+ do {
+#if AOM_ARCH_AARCH64
+ int16x8_t s7, s8, s9, s10;
+ load_s16_8x4(s, src_stride, &s7, &s8, &s9, &s10);
+
+ uint16x8_t d0 = convolve8_8_2d_v(s0, s1, s2, s3, s4, s5, s6, s7,
+ y_filter, offset_const);
+ uint16x8_t d1 = convolve8_8_2d_v(s1, s2, s3, s4, s5, s6, s7, s8,
+ y_filter, offset_const);
+ uint16x8_t d2 = convolve8_8_2d_v(s2, s3, s4, s5, s6, s7, s8, s9,
+ y_filter, offset_const);
+ uint16x8_t d3 = convolve8_8_2d_v(s3, s4, s5, s6, s7, s8, s9, s10,
+ y_filter, offset_const);
+
+ uint16x8_t dd0, dd1, dd2, dd3;
+ load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
+
+ uint8x8_t d0_u8, d1_u8, d2_u8, d3_u8;
+ compute_dist_wtd_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset,
+ bck_offset, round_offset_vec, &d0_u8, &d1_u8,
+ &d2_u8, &d3_u8);
+
+ store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8);
+ d_u8 += 4 * dst8_stride;
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s5 = s9;
+ s6 = s10;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ height -= 4;
+#else // !AOM_ARCH_AARCH64
+ int16x8_t s7 = vld1q_s16(s);
+
+ uint16x8_t d0 = convolve8_8_2d_v(s0, s1, s2, s3, s4, s5, s6, s7,
+ y_filter, offset_const);
+
+ uint16x8_t dd0 = vld1q_u16(d);
+
+ uint8x8_t d0_u8;
+ compute_dist_wtd_avg_8x1(dd0, d0, fwd_offset, bck_offset,
+ round_offset_vec, &d0_u8);
+
+ vst1_u8(d_u8, d0_u8);
+ d_u8 += dst8_stride;
+
+ s0 = s1;
+ s1 = s2;
+ s2 = s3;
+ s3 = s4;
+ s4 = s5;
+ s5 = s6;
+ s6 = s7;
+ s += src_stride;
+ d += dst_stride;
+ height--;
+#endif // AOM_ARCH_AARCH64
+ } while (height != 0);
+ src_ptr += 8;
+ dst_ptr += 8;
+ dst8_ptr += 8;
+ w -= 8;
+ } while (w != 0);
+ }
+}
+
+static INLINE void dist_wtd_convolve_2d_vert_8tap_avg_neon(
+ int16_t *src_ptr, const int src_stride, uint8_t *dst8_ptr, int dst8_stride,
+ ConvolveParams *conv_params, const int16x8_t y_filter, int h, int w) {
+ const int bd = 8;
+ const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
+ const int32x4_t offset_const = vdupq_n_s32(1 << offset_bits);
+ const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
+ (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
+ const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
+
+ CONV_BUF_TYPE *dst_ptr = conv_params->dst;
+ const int dst_stride = conv_params->dst_stride;
+
+ if (w == 4) {
+ int16x4_t s0, s1, s2, s3, s4, s5, s6;
+ load_s16_4x7(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
+ src_ptr += 7 * src_stride;
+
+ do {
+#if AOM_ARCH_AARCH64
+ int16x4_t s7, s8, s9, s10;
+ load_s16_4x4(src_ptr, src_stride, &s7, &s8, &s9, &s10);
+
+ uint16x4_t d0 = convolve8_4_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
+ offset_const);
+ uint16x4_t d1 = convolve8_4_2d_v(s1, s2, s3, s4, s5, s6, s7, s8, y_filter,
+ offset_const);
+ uint16x4_t d2 = convolve8_4_2d_v(s2, s3, s4, s5, s6, s7, s8, s9, y_filter,
+ offset_const);
+ uint16x4_t d3 = convolve8_4_2d_v(s3, s4, s5, s6, s7, s8, s9, s10,
+ y_filter, offset_const);
+
+ uint16x4_t dd0, dd1, dd2, dd3;
+ load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3);
+
+ uint8x8_t d01_u8, d23_u8;
+ compute_basic_avg_4x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3,
+ round_offset_vec, &d01_u8, &d23_u8);
+
+ store_u8_4x1(dst8_ptr + 0 * dst8_stride, d01_u8, 0);
+ store_u8_4x1(dst8_ptr + 1 * dst8_stride, d01_u8, 1);
+ store_u8_4x1(dst8_ptr + 2 * dst8_stride, d23_u8, 0);
+ store_u8_4x1(dst8_ptr + 3 * dst8_stride, d23_u8, 1);
+ dst8_ptr += 4 * dst8_stride;
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s5 = s9;
+ s6 = s10;
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ h -= 4;
+#else // !AOM_ARCH_AARCH64
+ int16x4_t s7 = vld1_s16(src_ptr);
+
+ uint16x4_t d0 = convolve8_4_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
+ offset_const);
+
+ uint16x4_t dd0 = vld1_u16(dst_ptr);
+
+ uint8x8_t d01_u8;
+ compute_basic_avg_4x1(dd0, d0, vget_low_s16(round_offset_vec), &d01_u8);
+
+ store_u8_4x1(dst8_ptr, d01_u8, 0);
+ dst8_ptr += dst8_stride;
+
+ s0 = s1;
+ s1 = s2;
+ s2 = s3;
+ s3 = s4;
+ s4 = s5;
+ s5 = s6;
+ s6 = s7;
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ h--;
+#endif // AOM_ARCH_AARCH64
+ } while (h != 0);
+ } else {
+ do {
+ int16_t *s = src_ptr;
+ CONV_BUF_TYPE *d = dst_ptr;
+ uint8_t *d_u8 = dst8_ptr;
+ int height = h;
+
+ int16x8_t s0, s1, s2, s3, s4, s5, s6;
+ load_s16_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
+ s += 7 * src_stride;
+
+ do {
+#if AOM_ARCH_AARCH64
+ int16x8_t s7, s8, s9, s10;
+ load_s16_8x4(s, src_stride, &s7, &s8, &s9, &s10);
+
+ uint16x8_t d0 = convolve8_8_2d_v(s0, s1, s2, s3, s4, s5, s6, s7,
+ y_filter, offset_const);
+ uint16x8_t d1 = convolve8_8_2d_v(s1, s2, s3, s4, s5, s6, s7, s8,
+ y_filter, offset_const);
+ uint16x8_t d2 = convolve8_8_2d_v(s2, s3, s4, s5, s6, s7, s8, s9,
+ y_filter, offset_const);
+ uint16x8_t d3 = convolve8_8_2d_v(s3, s4, s5, s6, s7, s8, s9, s10,
+ y_filter, offset_const);
+
+ uint16x8_t dd0, dd1, dd2, dd3;
+ load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
+
+ uint8x8_t d0_u8, d1_u8, d2_u8, d3_u8;
+ compute_basic_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3,
+ round_offset_vec, &d0_u8, &d1_u8, &d2_u8, &d3_u8);
+
+ store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8);
+ d_u8 += 4 * dst8_stride;
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s5 = s9;
+ s6 = s10;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ height -= 4;
+#else // !AOM_ARCH_AARCH64
+ int16x8_t s7 = vld1q_s16(s);
+
+ uint16x8_t d0 = convolve8_8_2d_v(s0, s1, s2, s3, s4, s5, s6, s7,
+ y_filter, offset_const);
+
+ uint16x8_t dd0 = vld1q_u16(d);
+
+ uint8x8_t d0_u8;
+ compute_basic_avg_8x1(dd0, d0, round_offset_vec, &d0_u8);
+
+ vst1_u8(d_u8, d0_u8);
+ d_u8 += dst8_stride;
+
+ s0 = s1;
+ s1 = s2;
+ s2 = s3;
+ s3 = s4;
+ s4 = s5;
+ s5 = s6;
+ s6 = s7;
+ s += src_stride;
+ d += dst_stride;
+ height--;
+#endif // AOM_ARCH_AARCH64
+ } while (height != 0);
+ src_ptr += 8;
+ dst_ptr += 8;
+ dst8_ptr += 8;
+ w -= 8;
+ } while (w != 0);
+ }
+}
+
+static INLINE void dist_wtd_convolve_2d_vert_8tap_neon(
+ int16_t *src_ptr, const int src_stride, ConvolveParams *conv_params,
+ const int16x8_t y_filter, int h, int w) {
+ const int bd = 8;
+ const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
+ const int32x4_t offset_const = vdupq_n_s32(1 << offset_bits);
+
+ CONV_BUF_TYPE *dst_ptr = conv_params->dst;
+ const int dst_stride = conv_params->dst_stride;
+
+ if (w == 4) {
+ int16x4_t s0, s1, s2, s3, s4, s5, s6;
+ load_s16_4x7(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
+ src_ptr += 7 * src_stride;
+
+ do {
+#if AOM_ARCH_AARCH64
+ int16x4_t s7, s8, s9, s10;
+ load_s16_4x4(src_ptr, src_stride, &s7, &s8, &s9, &s10);
+
+ uint16x4_t d0 = convolve8_4_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
+ offset_const);
+ uint16x4_t d1 = convolve8_4_2d_v(s1, s2, s3, s4, s5, s6, s7, s8, y_filter,
+ offset_const);
+ uint16x4_t d2 = convolve8_4_2d_v(s2, s3, s4, s5, s6, s7, s8, s9, y_filter,
+ offset_const);
+ uint16x4_t d3 = convolve8_4_2d_v(s3, s4, s5, s6, s7, s8, s9, s10,
+ y_filter, offset_const);
+
+ store_u16_4x4(dst_ptr, dst_stride, d0, d1, d2, d3);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s5 = s9;
+ s6 = s10;
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ h -= 4;
+#else // !AOM_ARCH_AARCH64
+ int16x4_t s7 = vld1_s16(src_ptr);
+
+ uint16x4_t d0 = convolve8_4_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
+ offset_const);
+
+ vst1_u16(dst_ptr, d0);
+
+ s0 = s1;
+ s1 = s2;
+ s2 = s3;
+ s3 = s4;
+ s4 = s5;
+ s5 = s6;
+ s6 = s7;
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ h--;
+#endif // AOM_ARCH_AARCH64
+ } while (h != 0);
+ } else {
+ do {
+ int16_t *s = src_ptr;
+ CONV_BUF_TYPE *d = dst_ptr;
+ int height = h;
+
+ int16x8_t s0, s1, s2, s3, s4, s5, s6;
+ load_s16_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
+ s += 7 * src_stride;
+
+ do {
+#if AOM_ARCH_AARCH64
+ int16x8_t s7, s8, s9, s10;
+ load_s16_8x4(s, src_stride, &s7, &s8, &s9, &s10);
+
+ uint16x8_t d0 = convolve8_8_2d_v(s0, s1, s2, s3, s4, s5, s6, s7,
+ y_filter, offset_const);
+ uint16x8_t d1 = convolve8_8_2d_v(s1, s2, s3, s4, s5, s6, s7, s8,
+ y_filter, offset_const);
+ uint16x8_t d2 = convolve8_8_2d_v(s2, s3, s4, s5, s6, s7, s8, s9,
+ y_filter, offset_const);
+ uint16x8_t d3 = convolve8_8_2d_v(s3, s4, s5, s6, s7, s8, s9, s10,
+ y_filter, offset_const);
+
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s5 = s9;
+ s6 = s10;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ height -= 4;
+#else // !AOM_ARCH_AARCH64
+ int16x8_t s7 = vld1q_s16(s);
+
+ uint16x8_t d0 = convolve8_8_2d_v(s0, s1, s2, s3, s4, s5, s6, s7,
+ y_filter, offset_const);
+
+ vst1q_u16(d, d0);
+
+ s0 = s1;
+ s1 = s2;
+ s2 = s3;
+ s3 = s4;
+ s4 = s5;
+ s5 = s6;
+ s6 = s7;
+ s += src_stride;
+ d += dst_stride;
+ height--;
+#endif // AOM_ARCH_AARCH64
+ } while (height != 0);
+ src_ptr += 8;
+ dst_ptr += 8;
+ w -= 8;
+ } while (w != 0);
+ }
+}
+
+#endif // AOM_AV1_COMMON_ARM_COMPOUND_CONVOLVE_NEON_H_
diff --git a/av1/common/arm/compound_convolve_neon_dotprod.c b/av1/common/arm/compound_convolve_neon_dotprod.c
new file mode 100644
index 0000000..8ab613d
--- /dev/null
+++ b/av1/common/arm/compound_convolve_neon_dotprod.c
@@ -0,0 +1,679 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+#include <assert.h>
+
+#include "aom_dsp/arm/mem_neon.h"
+#include "av1/common/arm/compound_convolve_neon.h"
+#include "config/aom_config.h"
+#include "config/av1_rtcd.h"
+
+DECLARE_ALIGNED(16, static const uint8_t, dot_prod_permute_tbl[48]) = {
+ 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6,
+ 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10,
+ 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
+};
+
+static INLINE int16x4_t convolve4_4_2d_h(uint8x16_t samples,
+ const int8x8_t x_filter,
+ const int32x4_t correction,
+ const uint8x16_t range_limit,
+ const uint8x16_t permute_tbl) {
+ // Clamp sample range to [-128, 127] for 8-bit signed dot product.
+ int8x16_t clamped_samples =
+ vreinterpretq_s8_u8(vsubq_u8(samples, range_limit));
+
+ // Permute samples ready for dot product.
+ // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
+ int8x16_t permuted_samples = vqtbl1q_s8(clamped_samples, permute_tbl);
+
+ // Accumulate dot product into 'correction' to account for range clamp.
+ int32x4_t sum = vdotq_lane_s32(correction, permuted_samples, x_filter, 0);
+
+ // We halved the convolution filter values so -1 from the right shift.
+ return vshrn_n_s32(sum, ROUND0_BITS - 1);
+}
+
+static INLINE int16x8_t convolve8_8_2d_h(uint8x16_t samples,
+ const int8x8_t x_filter,
+ const int32x4_t correction,
+ const uint8x16_t range_limit,
+ const uint8x16x3_t permute_tbl) {
+ int8x16_t clamped_samples, permuted_samples[3];
+ int32x4_t sum[2];
+
+ // Clamp sample range to [-128, 127] for 8-bit signed dot product.
+ clamped_samples = vreinterpretq_s8_u8(vsubq_u8(samples, range_limit));
+
+ // Permute samples ready for dot product. */
+ // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
+ permuted_samples[0] = vqtbl1q_s8(clamped_samples, permute_tbl.val[0]);
+ // { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 }
+ permuted_samples[1] = vqtbl1q_s8(clamped_samples, permute_tbl.val[1]);
+ // { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 }
+ permuted_samples[2] = vqtbl1q_s8(clamped_samples, permute_tbl.val[2]);
+
+ // Accumulate dot product into 'correction' to account for range clamp.
+ // First 4 output values.
+ sum[0] = vdotq_lane_s32(correction, permuted_samples[0], x_filter, 0);
+ sum[0] = vdotq_lane_s32(sum[0], permuted_samples[1], x_filter, 1);
+ // Second 4 output values.
+ sum[1] = vdotq_lane_s32(correction, permuted_samples[1], x_filter, 0);
+ sum[1] = vdotq_lane_s32(sum[1], permuted_samples[2], x_filter, 1);
+
+ // Narrow and re-pack.
+ // We halved the convolution filter values so -1 from the right shift.
+ return vcombine_s16(vshrn_n_s32(sum[0], ROUND0_BITS - 1),
+ vshrn_n_s32(sum[1], ROUND0_BITS - 1));
+}
+
+static INLINE void dist_wtd_convolve_2d_horiz_neon_dotprod(
+ const uint8_t *src, int src_stride, int16_t *im_block, const int im_stride,
+ const int16_t *x_filter_ptr, const int im_h, int w) {
+ const int bd = 8;
+ const int32_t horiz_const = (1 << (bd + FILTER_BITS - 2));
+ // Dot product constants and other shims.
+ const int16x8_t x_filter_s16 = vld1q_s16(x_filter_ptr);
+ const int32_t correction_s32 =
+ vaddlvq_s16(vshlq_n_s16(x_filter_s16, FILTER_BITS - 1));
+ // Fold horiz_const into the dot-product filter correction constant. The
+ // additional shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-
+ // rounding shifts - which are generally faster than rounding shifts on
+ // modern CPUs. (The extra -1 is needed because we halved the filter values.)
+ const int32x4_t correction = vdupq_n_s32(correction_s32 + horiz_const +
+ (1 << ((ROUND0_BITS - 1) - 1)));
+ const uint8x16_t range_limit = vdupq_n_u8(128);
+
+ const uint8_t *src_ptr = src;
+ int16_t *dst_ptr = im_block;
+ int dst_stride = im_stride;
+ int height = im_h;
+
+ if (w == 4) {
+ const uint8x16_t permute_tbl = vld1q_u8(dot_prod_permute_tbl);
+ // 4-tap filters are used for blocks having width <= 4.
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int8x8_t x_filter =
+ vshrn_n_s16(vcombine_s16(vld1_s16(x_filter_ptr + 2), vdup_n_s16(0)), 1);
+
+ src_ptr += 2;
+
+ do {
+ uint8x16_t s0, s1, s2, s3;
+ load_u8_16x4(src_ptr, src_stride, &s0, &s1, &s2, &s3);
+
+ int16x4_t d0 =
+ convolve4_4_2d_h(s0, x_filter, correction, range_limit, permute_tbl);
+ int16x4_t d1 =
+ convolve4_4_2d_h(s1, x_filter, correction, range_limit, permute_tbl);
+ int16x4_t d2 =
+ convolve4_4_2d_h(s2, x_filter, correction, range_limit, permute_tbl);
+ int16x4_t d3 =
+ convolve4_4_2d_h(s3, x_filter, correction, range_limit, permute_tbl);
+
+ store_s16_4x4(dst_ptr, dst_stride, d0, d1, d2, d3);
+
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ height -= 4;
+ } while (height > 4);
+
+ do {
+ uint8x16_t s0 = vld1q_u8(src_ptr);
+
+ int16x4_t d0 =
+ convolve4_4_2d_h(s0, x_filter, correction, range_limit, permute_tbl);
+
+ vst1_s16(dst_ptr, d0);
+
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ } while (--height != 0);
+ } else {
+ const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int8x8_t x_filter = vshrn_n_s16(x_filter_s16, 1);
+
+ do {
+ const uint8_t *s = src_ptr;
+ int16_t *d = dst_ptr;
+ int width = w;
+
+ do {
+ uint8x16_t s0, s1, s2, s3;
+ load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3);
+
+ int16x8_t d0 = convolve8_8_2d_h(s0, x_filter, correction, range_limit,
+ permute_tbl);
+ int16x8_t d1 = convolve8_8_2d_h(s1, x_filter, correction, range_limit,
+ permute_tbl);
+ int16x8_t d2 = convolve8_8_2d_h(s2, x_filter, correction, range_limit,
+ permute_tbl);
+ int16x8_t d3 = convolve8_8_2d_h(s3, x_filter, correction, range_limit,
+ permute_tbl);
+
+ store_s16_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width > 0);
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ height -= 4;
+ } while (height > 4);
+
+ do {
+ const uint8_t *s = src_ptr;
+ int16_t *d = dst_ptr;
+ int width = w;
+
+ do {
+ uint8x16_t s0 = vld1q_u8(s);
+
+ int16x8_t d0 = convolve8_8_2d_h(s0, x_filter, correction, range_limit,
+ permute_tbl);
+
+ vst1q_s16(d, d0);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width > 0);
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ } while (--height != 0);
+ }
+}
+
+void av1_dist_wtd_convolve_2d_neon_dotprod(
+ const uint8_t *src, int src_stride, uint8_t *dst8, int dst8_stride, int w,
+ int h, const InterpFilterParams *filter_params_x,
+ const InterpFilterParams *filter_params_y, const int subpel_x_qn,
+ const int subpel_y_qn, ConvolveParams *conv_params) {
+ assert(w % 4 == 0);
+ assert(h % 4 == 0);
+
+ DECLARE_ALIGNED(16, int16_t,
+ im_block[(MAX_SB_SIZE + SUBPEL_TAPS - 1) * MAX_SB_SIZE]);
+
+ const int y_filter_taps = get_filter_tap(filter_params_y, subpel_y_qn);
+ const int clamped_y_taps = y_filter_taps < 6 ? 6 : y_filter_taps;
+
+ const int im_h = h + clamped_y_taps - 1;
+ const int im_stride = MAX_SB_SIZE;
+ const int vert_offset = clamped_y_taps / 2 - 1;
+ const int horiz_offset = filter_params_x->taps / 2 - 1;
+ const uint8_t *src_ptr = src - vert_offset * src_stride - horiz_offset;
+ const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
+ filter_params_x, subpel_x_qn & SUBPEL_MASK);
+ const int16_t *y_filter_ptr = av1_get_interp_filter_subpel_kernel(
+ filter_params_y, subpel_y_qn & SUBPEL_MASK);
+
+ const int16x8_t y_filter = vld1q_s16(y_filter_ptr);
+
+ dist_wtd_convolve_2d_horiz_neon_dotprod(src_ptr, src_stride, im_block,
+ im_stride, x_filter_ptr, im_h, w);
+
+ if (clamped_y_taps == 6) {
+ if (conv_params->do_average) {
+ if (UNLIKELY(conv_params->use_dist_wtd_comp_avg)) {
+ dist_wtd_convolve_2d_vert_6tap_dist_wtd_avg_neon(
+ im_block, im_stride, dst8, dst8_stride, conv_params, y_filter, h,
+ w);
+ } else {
+ dist_wtd_convolve_2d_vert_6tap_avg_neon(im_block, im_stride, dst8,
+ dst8_stride, conv_params,
+ y_filter, h, w);
+ }
+ } else {
+ dist_wtd_convolve_2d_vert_6tap_neon(im_block, im_stride, conv_params,
+ y_filter, h, w);
+ }
+ } else {
+ if (conv_params->do_average) {
+ if (UNLIKELY(conv_params->use_dist_wtd_comp_avg)) {
+ dist_wtd_convolve_2d_vert_8tap_dist_wtd_avg_neon(
+ im_block, im_stride, dst8, dst8_stride, conv_params, y_filter, h,
+ w);
+ } else {
+ dist_wtd_convolve_2d_vert_8tap_avg_neon(im_block, im_stride, dst8,
+ dst8_stride, conv_params,
+ y_filter, h, w);
+ }
+ } else {
+ dist_wtd_convolve_2d_vert_8tap_neon(im_block, im_stride, conv_params,
+ y_filter, h, w);
+ }
+ }
+}
+
+static INLINE uint16x4_t convolve4_4_x(uint8x16_t samples,
+ const int8x8_t x_filter,
+ const int32x4_t correction,
+ const uint8x16_t range_limit,
+ const uint8x16_t permute_tbl) {
+ // Clamp sample range to [-128, 127] for 8-bit signed dot product.
+ int8x16_t clamped_samples =
+ vreinterpretq_s8_u8(vsubq_u8(samples, range_limit));
+
+ // Permute samples ready for dot product.
+ // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
+ int8x16_t permuted_samples = vqtbl1q_s8(clamped_samples, permute_tbl);
+
+ // Accumulate dot product into 'correction' to account for range clamp.
+ int32x4_t sum = vdotq_lane_s32(correction, permuted_samples, x_filter, 0);
+
+ // We halved the convolution filter values so -1 from the right shift.
+ return vreinterpret_u16_s16(vshrn_n_s32(sum, ROUND0_BITS - 1));
+}
+
+static INLINE uint16x8_t convolve8_8_x(uint8x16_t samples,
+ const int8x8_t x_filter,
+ const int32x4_t correction,
+ const uint8x16_t range_limit,
+ const uint8x16x3_t permute_tbl) {
+ int8x16_t clamped_samples, permuted_samples[3];
+ int32x4_t sum[2];
+
+ // Clamp sample range to [-128, 127] for 8-bit signed dot product.
+ clamped_samples = vreinterpretq_s8_u8(vsubq_u8(samples, range_limit));
+
+ // Permute samples ready for dot product. */
+ // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
+ permuted_samples[0] = vqtbl1q_s8(clamped_samples, permute_tbl.val[0]);
+ // { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 }
+ permuted_samples[1] = vqtbl1q_s8(clamped_samples, permute_tbl.val[1]);
+ // { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 }
+ permuted_samples[2] = vqtbl1q_s8(clamped_samples, permute_tbl.val[2]);
+
+ // Accumulate dot product into 'correction' to account for range clamp.
+ // First 4 output values.
+ sum[0] = vdotq_lane_s32(correction, permuted_samples[0], x_filter, 0);
+ sum[0] = vdotq_lane_s32(sum[0], permuted_samples[1], x_filter, 1);
+ // Second 4 output values.
+ sum[1] = vdotq_lane_s32(correction, permuted_samples[1], x_filter, 0);
+ sum[1] = vdotq_lane_s32(sum[1], permuted_samples[2], x_filter, 1);
+
+ // Narrow and re-pack.
+ // We halved the convolution filter values so -1 from the right shift.
+ int16x8_t res = vcombine_s16(vshrn_n_s32(sum[0], ROUND0_BITS - 1),
+ vshrn_n_s32(sum[1], ROUND0_BITS - 1));
+ return vreinterpretq_u16_s16(res);
+}
+
+static INLINE void dist_wtd_convolve_x_dist_wtd_avg_neon_dotprod(
+ const uint8_t *src, int src_stride, uint8_t *dst8, int dst8_stride, int w,
+ int h, const InterpFilterParams *filter_params_x, const int subpel_x_qn,
+ ConvolveParams *conv_params) {
+ assert(w % 4 == 0);
+ assert(h % 4 == 0);
+
+ const int bd = 8;
+ const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
+ const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
+ (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
+ const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
+
+ const uint16_t fwd_offset = conv_params->fwd_offset;
+ const uint16_t bck_offset = conv_params->bck_offset;
+
+ // Horizontal filter.
+ const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
+ filter_params_x, subpel_x_qn & SUBPEL_MASK);
+ const int16x8_t x_filter_s16 = vld1q_s16(x_filter_ptr);
+
+ // Dot-product constants and other shims.
+ const uint8x16_t range_limit = vdupq_n_u8(128);
+ const int32_t correction_s32 =
+ vaddlvq_s16(vshlq_n_s16(x_filter_s16, FILTER_BITS - 1));
+ // Fold round_offset into the dot-product filter correction constant. The
+ // additional shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-
+ // rounding shifts - which are generally faster than rounding shifts on
+ // modern CPUs. (The extra -1 is needed because we halved the filter values.)
+ int32x4_t correction =
+ vdupq_n_s32(correction_s32 + (round_offset << (ROUND0_BITS - 1)) +
+ (1 << ((ROUND0_BITS - 1) - 1)));
+
+ const int horiz_offset = filter_params_x->taps / 2 - 1;
+ const uint8_t *src_ptr = src - horiz_offset;
+ CONV_BUF_TYPE *dst_ptr = conv_params->dst;
+ uint8_t *dst8_ptr = dst8;
+ int dst_stride = conv_params->dst_stride;
+ int height = h;
+
+ if (w == 4) {
+ const uint8x16_t permute_tbl = vld1q_u8(dot_prod_permute_tbl);
+ // 4-tap filters are used for blocks having width <= 4.
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int8x8_t x_filter =
+ vshrn_n_s16(vcombine_s16(vld1_s16(x_filter_ptr + 2), vdup_n_s16(0)), 1);
+
+ src_ptr += 2;
+
+ do {
+ uint8x16_t s0, s1, s2, s3;
+ load_u8_16x4(src_ptr, src_stride, &s0, &s1, &s2, &s3);
+
+ uint16x4_t d0 =
+ convolve4_4_x(s0, x_filter, correction, range_limit, permute_tbl);
+ uint16x4_t d1 =
+ convolve4_4_x(s1, x_filter, correction, range_limit, permute_tbl);
+ uint16x4_t d2 =
+ convolve4_4_x(s2, x_filter, correction, range_limit, permute_tbl);
+ uint16x4_t d3 =
+ convolve4_4_x(s3, x_filter, correction, range_limit, permute_tbl);
+
+ uint16x4_t dd0, dd1, dd2, dd3;
+ load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3);
+
+ uint8x8_t d01_u8, d23_u8;
+ compute_dist_wtd_avg_4x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset,
+ bck_offset, round_offset_vec, &d01_u8, &d23_u8);
+
+ store_u8_4x1(dst8_ptr + 0 * dst8_stride, d01_u8, 0);
+ store_u8_4x1(dst8_ptr + 1 * dst8_stride, d01_u8, 1);
+ store_u8_4x1(dst8_ptr + 2 * dst8_stride, d23_u8, 0);
+ store_u8_4x1(dst8_ptr + 3 * dst8_stride, d23_u8, 1);
+
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ dst8_ptr += 4 * dst8_stride;
+ height -= 4;
+ } while (height != 0);
+ } else {
+ const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int8x8_t x_filter = vshrn_n_s16(x_filter_s16, 1);
+
+ do {
+ const uint8_t *s = src_ptr;
+ CONV_BUF_TYPE *d = dst_ptr;
+ uint8_t *d_u8 = dst8_ptr;
+ int width = w;
+
+ do {
+ uint8x16_t s0, s1, s2, s3;
+ load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3);
+
+ uint16x8_t d0 =
+ convolve8_8_x(s0, x_filter, correction, range_limit, permute_tbl);
+ uint16x8_t d1 =
+ convolve8_8_x(s1, x_filter, correction, range_limit, permute_tbl);
+ uint16x8_t d2 =
+ convolve8_8_x(s2, x_filter, correction, range_limit, permute_tbl);
+ uint16x8_t d3 =
+ convolve8_8_x(s3, x_filter, correction, range_limit, permute_tbl);
+
+ uint16x8_t dd0, dd1, dd2, dd3;
+ load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
+
+ uint8x8_t d0_u8, d1_u8, d2_u8, d3_u8;
+ compute_dist_wtd_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset,
+ bck_offset, round_offset_vec, &d0_u8, &d1_u8,
+ &d2_u8, &d3_u8);
+
+ store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8);
+
+ s += 8;
+ d += 8;
+ d_u8 += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ dst8_ptr += 4 * dst8_stride;
+ height -= 4;
+ } while (height != 0);
+ }
+}
+
+static INLINE void dist_wtd_convolve_x_avg_neon_dotprod(
+ const uint8_t *src, int src_stride, uint8_t *dst8, int dst8_stride, int w,
+ int h, const InterpFilterParams *filter_params_x, const int subpel_x_qn,
+ ConvolveParams *conv_params) {
+ assert(w % 4 == 0);
+ assert(h % 4 == 0);
+
+ const int bd = 8;
+ const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
+ const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
+ (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
+ const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
+
+ // Horizontal filter.
+ const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
+ filter_params_x, subpel_x_qn & SUBPEL_MASK);
+ const int16x8_t x_filter_s16 = vld1q_s16(x_filter_ptr);
+
+ // Dot-product constants and other shims.
+ const uint8x16_t range_limit = vdupq_n_u8(128);
+ const int32_t correction_s32 =
+ vaddlvq_s16(vshlq_n_s16(x_filter_s16, FILTER_BITS - 1));
+ // Fold round_offset into the dot-product filter correction constant. The
+ // additional shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-
+ // rounding shifts - which are generally faster than rounding shifts on
+ // modern CPUs. (The extra -1 is needed because we halved the filter values.)
+ int32x4_t correction =
+ vdupq_n_s32(correction_s32 + (round_offset << (ROUND0_BITS - 1)) +
+ (1 << ((ROUND0_BITS - 1) - 1)));
+
+ const int horiz_offset = filter_params_x->taps / 2 - 1;
+ const uint8_t *src_ptr = src - horiz_offset;
+ CONV_BUF_TYPE *dst_ptr = conv_params->dst;
+ uint8_t *dst8_ptr = dst8;
+ int dst_stride = conv_params->dst_stride;
+ int height = h;
+
+ if (w == 4) {
+ const uint8x16_t permute_tbl = vld1q_u8(dot_prod_permute_tbl);
+ // 4-tap filters are used for blocks having width <= 4.
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int8x8_t x_filter =
+ vshrn_n_s16(vcombine_s16(vld1_s16(x_filter_ptr + 2), vdup_n_s16(0)), 1);
+
+ src_ptr += 2;
+
+ do {
+ uint8x16_t s0, s1, s2, s3;
+ load_u8_16x4(src_ptr, src_stride, &s0, &s1, &s2, &s3);
+
+ uint16x4_t d0 =
+ convolve4_4_x(s0, x_filter, correction, range_limit, permute_tbl);
+ uint16x4_t d1 =
+ convolve4_4_x(s1, x_filter, correction, range_limit, permute_tbl);
+ uint16x4_t d2 =
+ convolve4_4_x(s2, x_filter, correction, range_limit, permute_tbl);
+ uint16x4_t d3 =
+ convolve4_4_x(s3, x_filter, correction, range_limit, permute_tbl);
+
+ uint16x4_t dd0, dd1, dd2, dd3;
+ load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3);
+
+ uint8x8_t d01_u8, d23_u8;
+ compute_basic_avg_4x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3,
+ round_offset_vec, &d01_u8, &d23_u8);
+
+ store_u8_4x1(dst8_ptr + 0 * dst8_stride, d01_u8, 0);
+ store_u8_4x1(dst8_ptr + 1 * dst8_stride, d01_u8, 1);
+ store_u8_4x1(dst8_ptr + 2 * dst8_stride, d23_u8, 0);
+ store_u8_4x1(dst8_ptr + 3 * dst8_stride, d23_u8, 1);
+
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ dst8_ptr += 4 * dst8_stride;
+ height -= 4;
+ } while (height != 0);
+ } else {
+ const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int8x8_t x_filter = vshrn_n_s16(x_filter_s16, 1);
+
+ do {
+ const uint8_t *s = src_ptr;
+ CONV_BUF_TYPE *d = dst_ptr;
+ uint8_t *d_u8 = dst8_ptr;
+ int width = w;
+
+ do {
+ uint8x16_t s0, s1, s2, s3;
+ load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3);
+
+ uint16x8_t d0 =
+ convolve8_8_x(s0, x_filter, correction, range_limit, permute_tbl);
+ uint16x8_t d1 =
+ convolve8_8_x(s1, x_filter, correction, range_limit, permute_tbl);
+ uint16x8_t d2 =
+ convolve8_8_x(s2, x_filter, correction, range_limit, permute_tbl);
+ uint16x8_t d3 =
+ convolve8_8_x(s3, x_filter, correction, range_limit, permute_tbl);
+
+ uint16x8_t dd0, dd1, dd2, dd3;
+ load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
+
+ uint8x8_t d0_u8, d1_u8, d2_u8, d3_u8;
+ compute_basic_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3,
+ round_offset_vec, &d0_u8, &d1_u8, &d2_u8, &d3_u8);
+
+ store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8);
+
+ s += 8;
+ d += 8;
+ d_u8 += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ dst8_ptr += 4 * dst8_stride;
+ height -= 4;
+ } while (height != 0);
+ }
+}
+
+static INLINE void dist_wtd_convolve_x_neon_dotprod(
+ const uint8_t *src, int src_stride, int w, int h,
+ const InterpFilterParams *filter_params_x, const int subpel_x_qn,
+ ConvolveParams *conv_params) {
+ assert(w % 4 == 0);
+ assert(h % 4 == 0);
+
+ const int bd = 8;
+ const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
+ const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
+ (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
+
+ // Horizontal filter.
+ const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
+ filter_params_x, subpel_x_qn & SUBPEL_MASK);
+ const int16x8_t x_filter_s16 = vld1q_s16(x_filter_ptr);
+
+ // Dot-product constants and other shims.
+ const uint8x16_t range_limit = vdupq_n_u8(128);
+ const int32_t correction_s32 =
+ vaddlvq_s16(vshlq_n_s16(x_filter_s16, FILTER_BITS - 1));
+ // Fold round_offset into the dot-product filter correction constant. The
+ // additional shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-
+ // rounding shifts - which are generally faster than rounding shifts on
+ // modern CPUs. (The extra -1 is needed because we halved the filter values.)
+ int32x4_t correction =
+ vdupq_n_s32(correction_s32 + (round_offset << (ROUND0_BITS - 1)) +
+ (1 << ((ROUND0_BITS - 1) - 1)));
+
+ const int horiz_offset = filter_params_x->taps / 2 - 1;
+ const uint8_t *src_ptr = src - horiz_offset;
+ CONV_BUF_TYPE *dst_ptr = conv_params->dst;
+ int dst_stride = conv_params->dst_stride;
+ int height = h;
+
+ if (w == 4) {
+ const uint8x16_t permute_tbl = vld1q_u8(dot_prod_permute_tbl);
+ // 4-tap filters are used for blocks having width <= 4.
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int8x8_t x_filter =
+ vshrn_n_s16(vcombine_s16(vld1_s16(x_filter_ptr + 2), vdup_n_s16(0)), 1);
+
+ src_ptr += 2;
+
+ do {
+ uint8x16_t s0, s1, s2, s3;
+ load_u8_16x4(src_ptr, src_stride, &s0, &s1, &s2, &s3);
+
+ uint16x4_t d0 =
+ convolve4_4_x(s0, x_filter, correction, range_limit, permute_tbl);
+ uint16x4_t d1 =
+ convolve4_4_x(s1, x_filter, correction, range_limit, permute_tbl);
+ uint16x4_t d2 =
+ convolve4_4_x(s2, x_filter, correction, range_limit, permute_tbl);
+ uint16x4_t d3 =
+ convolve4_4_x(s3, x_filter, correction, range_limit, permute_tbl);
+
+ store_u16_4x4(dst_ptr, dst_stride, d0, d1, d2, d3);
+
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ height -= 4;
+ } while (height != 0);
+ } else {
+ const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int8x8_t x_filter = vshrn_n_s16(x_filter_s16, 1);
+
+ do {
+ const uint8_t *s = src_ptr;
+ CONV_BUF_TYPE *d = dst_ptr;
+ int width = w;
+
+ do {
+ uint8x16_t s0, s1, s2, s3;
+ load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3);
+
+ uint16x8_t d0 =
+ convolve8_8_x(s0, x_filter, correction, range_limit, permute_tbl);
+ uint16x8_t d1 =
+ convolve8_8_x(s1, x_filter, correction, range_limit, permute_tbl);
+ uint16x8_t d2 =
+ convolve8_8_x(s2, x_filter, correction, range_limit, permute_tbl);
+ uint16x8_t d3 =
+ convolve8_8_x(s3, x_filter, correction, range_limit, permute_tbl);
+
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ height -= 4;
+ } while (height != 0);
+ }
+}
+
+void av1_dist_wtd_convolve_x_neon_dotprod(
+ const uint8_t *src, int src_stride, uint8_t *dst8, int dst8_stride, int w,
+ int h, const InterpFilterParams *filter_params_x, const int subpel_x_qn,
+ ConvolveParams *conv_params) {
+ if (conv_params->do_average) {
+ if (UNLIKELY(conv_params->use_dist_wtd_comp_avg)) {
+ dist_wtd_convolve_x_dist_wtd_avg_neon_dotprod(
+ src, src_stride, dst8, dst8_stride, w, h, filter_params_x,
+ subpel_x_qn, conv_params);
+ } else {
+ dist_wtd_convolve_x_avg_neon_dotprod(src, src_stride, dst8, dst8_stride,
+ w, h, filter_params_x, subpel_x_qn,
+ conv_params);
+ }
+ } else {
+ dist_wtd_convolve_x_neon_dotprod(src, src_stride, w, h, filter_params_x,
+ subpel_x_qn, conv_params);
+ }
+}
diff --git a/av1/common/arm/compound_convolve_neon_i8mm.c b/av1/common/arm/compound_convolve_neon_i8mm.c
new file mode 100644
index 0000000..70d7da9
--- /dev/null
+++ b/av1/common/arm/compound_convolve_neon_i8mm.c
@@ -0,0 +1,618 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+#include <assert.h>
+
+#include "aom_dsp/arm/mem_neon.h"
+#include "av1/common/arm/compound_convolve_neon.h"
+#include "config/aom_config.h"
+#include "config/av1_rtcd.h"
+
+DECLARE_ALIGNED(16, static const uint8_t, dot_prod_permute_tbl[48]) = {
+ 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6,
+ 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10,
+ 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
+};
+
+static INLINE int16x4_t convolve4_4_2d_h(uint8x16_t samples,
+ const int8x8_t x_filter,
+ const uint8x16_t permute_tbl,
+ const int32x4_t horiz_const) {
+ // Permute samples ready for dot product.
+ // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
+ uint8x16_t permuted_samples = vqtbl1q_u8(samples, permute_tbl);
+
+ // First 4 output values.
+ int32x4_t sum = vusdotq_lane_s32(horiz_const, permuted_samples, x_filter, 0);
+
+ // We halved the convolution filter values so -1 from the right shift.
+ return vshrn_n_s32(sum, ROUND0_BITS - 1);
+}
+
+static INLINE int16x8_t convolve8_8_2d_h(uint8x16_t samples,
+ const int8x8_t x_filter,
+ const uint8x16x3_t permute_tbl,
+ const int32x4_t horiz_const) {
+ uint8x16_t permuted_samples[3];
+ int32x4_t sum[2];
+
+ // Permute samples ready for dot product.
+ // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
+ permuted_samples[0] = vqtbl1q_u8(samples, permute_tbl.val[0]);
+ // { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 }
+ permuted_samples[1] = vqtbl1q_u8(samples, permute_tbl.val[1]);
+ // { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 }
+ permuted_samples[2] = vqtbl1q_u8(samples, permute_tbl.val[2]);
+
+ // First 4 output values.
+ sum[0] = vusdotq_lane_s32(horiz_const, permuted_samples[0], x_filter, 0);
+ sum[0] = vusdotq_lane_s32(sum[0], permuted_samples[1], x_filter, 1);
+ // Second 4 output values.
+ sum[1] = vusdotq_lane_s32(horiz_const, permuted_samples[1], x_filter, 0);
+ sum[1] = vusdotq_lane_s32(sum[1], permuted_samples[2], x_filter, 1);
+
+ // Narrow and re-pack.
+ // We halved the convolution filter values so -1 from the right shift.
+ return vcombine_s16(vshrn_n_s32(sum[0], ROUND0_BITS - 1),
+ vshrn_n_s32(sum[1], ROUND0_BITS - 1));
+}
+
+static INLINE void dist_wtd_convolve_2d_horiz_neon_i8mm(
+ const uint8_t *src, int src_stride, int16_t *im_block, const int im_stride,
+ const int16_t *x_filter_ptr, const int im_h, int w) {
+ const int bd = 8;
+ // A shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-rounding
+ // shifts - which are generally faster than rounding shifts on modern CPUs.
+ // (The extra -1 is needed because we halved the filter values.)
+ const int32x4_t horiz_const = vdupq_n_s32((1 << (bd + FILTER_BITS - 2)) +
+ (1 << ((ROUND0_BITS - 1) - 1)));
+
+ const uint8_t *src_ptr = src;
+ int16_t *dst_ptr = im_block;
+ int dst_stride = im_stride;
+ int height = im_h;
+
+ if (w == 4) {
+ const uint8x16_t permute_tbl = vld1q_u8(dot_prod_permute_tbl);
+ // 4-tap filters are used for blocks having width <= 4.
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int8x8_t x_filter =
+ vshrn_n_s16(vcombine_s16(vld1_s16(x_filter_ptr + 2), vdup_n_s16(0)), 1);
+
+ src_ptr += 2;
+
+ do {
+ uint8x16_t s0, s1, s2, s3;
+ load_u8_16x4(src_ptr, src_stride, &s0, &s1, &s2, &s3);
+
+ int16x4_t d0 = convolve4_4_2d_h(s0, x_filter, permute_tbl, horiz_const);
+ int16x4_t d1 = convolve4_4_2d_h(s1, x_filter, permute_tbl, horiz_const);
+ int16x4_t d2 = convolve4_4_2d_h(s2, x_filter, permute_tbl, horiz_const);
+ int16x4_t d3 = convolve4_4_2d_h(s3, x_filter, permute_tbl, horiz_const);
+
+ store_s16_4x4(dst_ptr, dst_stride, d0, d1, d2, d3);
+
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ height -= 4;
+ } while (height > 4);
+
+ do {
+ uint8x16_t s0 = vld1q_u8(src_ptr);
+
+ int16x4_t d0 = convolve4_4_2d_h(s0, x_filter, permute_tbl, horiz_const);
+
+ vst1_s16(dst_ptr, d0);
+
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ } while (--height != 0);
+ } else {
+ const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int8x8_t x_filter = vshrn_n_s16(vld1q_s16(x_filter_ptr), 1);
+
+ do {
+ const uint8_t *s = src_ptr;
+ int16_t *d = dst_ptr;
+ int width = w;
+
+ do {
+ uint8x16_t s0, s1, s2, s3;
+ load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3);
+
+ int16x8_t d0 = convolve8_8_2d_h(s0, x_filter, permute_tbl, horiz_const);
+ int16x8_t d1 = convolve8_8_2d_h(s1, x_filter, permute_tbl, horiz_const);
+ int16x8_t d2 = convolve8_8_2d_h(s2, x_filter, permute_tbl, horiz_const);
+ int16x8_t d3 = convolve8_8_2d_h(s3, x_filter, permute_tbl, horiz_const);
+
+ store_s16_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width > 0);
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ height -= 4;
+ } while (height > 4);
+
+ do {
+ const uint8_t *s = src_ptr;
+ int16_t *d = dst_ptr;
+ int width = w;
+
+ do {
+ uint8x16_t s0 = vld1q_u8(s);
+
+ int16x8_t d0 = convolve8_8_2d_h(s0, x_filter, permute_tbl, horiz_const);
+
+ vst1q_s16(d, d0);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width > 0);
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ } while (--height != 0);
+ }
+}
+
+void av1_dist_wtd_convolve_2d_neon_i8mm(
+ const uint8_t *src, int src_stride, uint8_t *dst8, int dst8_stride, int w,
+ int h, const InterpFilterParams *filter_params_x,
+ const InterpFilterParams *filter_params_y, const int subpel_x_qn,
+ const int subpel_y_qn, ConvolveParams *conv_params) {
+ assert(w % 4 == 0);
+ assert(h % 4 == 0);
+
+ DECLARE_ALIGNED(16, int16_t,
+ im_block[(MAX_SB_SIZE + SUBPEL_TAPS - 1) * MAX_SB_SIZE]);
+
+ const int y_filter_taps = get_filter_tap(filter_params_y, subpel_y_qn);
+ const int clamped_y_taps = y_filter_taps < 6 ? 6 : y_filter_taps;
+
+ const int im_h = h + clamped_y_taps - 1;
+ const int im_stride = MAX_SB_SIZE;
+ const int vert_offset = clamped_y_taps / 2 - 1;
+ const int horiz_offset = filter_params_x->taps / 2 - 1;
+ const uint8_t *src_ptr = src - vert_offset * src_stride - horiz_offset;
+ const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
+ filter_params_x, subpel_x_qn & SUBPEL_MASK);
+ const int16_t *y_filter_ptr = av1_get_interp_filter_subpel_kernel(
+ filter_params_y, subpel_y_qn & SUBPEL_MASK);
+
+ const int16x8_t y_filter = vld1q_s16(y_filter_ptr);
+
+ dist_wtd_convolve_2d_horiz_neon_i8mm(src_ptr, src_stride, im_block, im_stride,
+ x_filter_ptr, im_h, w);
+
+ if (clamped_y_taps == 6) {
+ if (conv_params->do_average) {
+ if (UNLIKELY(conv_params->use_dist_wtd_comp_avg)) {
+ dist_wtd_convolve_2d_vert_6tap_dist_wtd_avg_neon(
+ im_block, im_stride, dst8, dst8_stride, conv_params, y_filter, h,
+ w);
+ } else {
+ dist_wtd_convolve_2d_vert_6tap_avg_neon(im_block, im_stride, dst8,
+ dst8_stride, conv_params,
+ y_filter, h, w);
+ }
+ } else {
+ dist_wtd_convolve_2d_vert_6tap_neon(im_block, im_stride, conv_params,
+ y_filter, h, w);
+ }
+ } else {
+ if (conv_params->do_average) {
+ if (UNLIKELY(conv_params->use_dist_wtd_comp_avg)) {
+ dist_wtd_convolve_2d_vert_8tap_dist_wtd_avg_neon(
+ im_block, im_stride, dst8, dst8_stride, conv_params, y_filter, h,
+ w);
+ } else {
+ dist_wtd_convolve_2d_vert_8tap_avg_neon(im_block, im_stride, dst8,
+ dst8_stride, conv_params,
+ y_filter, h, w);
+ }
+ } else {
+ dist_wtd_convolve_2d_vert_8tap_neon(im_block, im_stride, conv_params,
+ y_filter, h, w);
+ }
+ }
+}
+
+static INLINE uint16x4_t convolve4_4_x(uint8x16_t samples,
+ const int8x8_t x_filter,
+ const uint8x16_t permute_tbl,
+ const int32x4_t round_offset) {
+ // Permute samples ready for dot product.
+ // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
+ uint8x16_t permuted_samples = vqtbl1q_u8(samples, permute_tbl);
+
+ // First 4 output values.
+ int32x4_t sum = vusdotq_lane_s32(round_offset, permuted_samples, x_filter, 0);
+
+ // We halved the convolution filter values so -1 from the right shift.
+ return vreinterpret_u16_s16(vshrn_n_s32(sum, ROUND0_BITS - 1));
+}
+
+static INLINE uint16x8_t convolve8_8_x(uint8x16_t samples,
+ const int8x8_t x_filter,
+ const uint8x16x3_t permute_tbl,
+ const int32x4_t round_offset) {
+ uint8x16_t permuted_samples[3];
+ int32x4_t sum[2];
+
+ // Permute samples ready for dot product.
+ // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
+ permuted_samples[0] = vqtbl1q_u8(samples, permute_tbl.val[0]);
+ // { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 }
+ permuted_samples[1] = vqtbl1q_u8(samples, permute_tbl.val[1]);
+ // { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 }
+ permuted_samples[2] = vqtbl1q_u8(samples, permute_tbl.val[2]);
+
+ // First 4 output values.
+ sum[0] = vusdotq_lane_s32(round_offset, permuted_samples[0], x_filter, 0);
+ sum[0] = vusdotq_lane_s32(sum[0], permuted_samples[1], x_filter, 1);
+ // Second 4 output values.
+ sum[1] = vusdotq_lane_s32(round_offset, permuted_samples[1], x_filter, 0);
+ sum[1] = vusdotq_lane_s32(sum[1], permuted_samples[2], x_filter, 1);
+
+ // Narrow and re-pack.
+ // We halved the convolution filter values so -1 from the right shift.
+ int16x8_t res = vcombine_s16(vshrn_n_s32(sum[0], ROUND0_BITS - 1),
+ vshrn_n_s32(sum[1], ROUND0_BITS - 1));
+ return vreinterpretq_u16_s16(res);
+}
+
+static INLINE void dist_wtd_convolve_x_dist_wtd_avg_neon_i8mm(
+ const uint8_t *src, int src_stride, uint8_t *dst8, int dst8_stride, int w,
+ int h, const InterpFilterParams *filter_params_x, const int subpel_x_qn,
+ ConvolveParams *conv_params) {
+ assert(w % 4 == 0);
+ assert(h % 4 == 0);
+
+ const int bd = 8;
+ const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
+ const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
+ (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
+ const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
+ // A shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-rounding
+ // shifts - which are generally faster than rounding shifts on modern CPUs.
+ // (The extra -1 is needed because we halved the filter values.)
+ const int32x4_t round_offset_shim = vdupq_n_s32(
+ (round_offset << (ROUND0_BITS - 1)) + (1 << ((ROUND0_BITS - 1) - 1)));
+
+ const uint16_t fwd_offset = conv_params->fwd_offset;
+ const uint16_t bck_offset = conv_params->bck_offset;
+
+ // Horizontal filter.
+ const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
+ filter_params_x, subpel_x_qn & SUBPEL_MASK);
+
+ const int horiz_offset = filter_params_x->taps / 2 - 1;
+ const uint8_t *src_ptr = src - horiz_offset;
+ CONV_BUF_TYPE *dst_ptr = conv_params->dst;
+ uint8_t *dst8_ptr = dst8;
+ int dst_stride = conv_params->dst_stride;
+ int height = h;
+
+ if (w == 4) {
+ const uint8x16_t permute_tbl = vld1q_u8(dot_prod_permute_tbl);
+ // 4-tap filters are used for blocks having width <= 4.
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int8x8_t x_filter =
+ vshrn_n_s16(vcombine_s16(vld1_s16(x_filter_ptr + 2), vdup_n_s16(0)), 1);
+
+ src_ptr += 2;
+
+ do {
+ uint8x16_t s0, s1, s2, s3;
+ load_u8_16x4(src_ptr, src_stride, &s0, &s1, &s2, &s3);
+
+ uint16x4_t d0 =
+ convolve4_4_x(s0, x_filter, permute_tbl, round_offset_shim);
+ uint16x4_t d1 =
+ convolve4_4_x(s1, x_filter, permute_tbl, round_offset_shim);
+ uint16x4_t d2 =
+ convolve4_4_x(s2, x_filter, permute_tbl, round_offset_shim);
+ uint16x4_t d3 =
+ convolve4_4_x(s3, x_filter, permute_tbl, round_offset_shim);
+
+ uint16x4_t dd0, dd1, dd2, dd3;
+ load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3);
+
+ uint8x8_t d01_u8, d23_u8;
+ compute_dist_wtd_avg_4x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset,
+ bck_offset, round_offset_vec, &d01_u8, &d23_u8);
+
+ store_u8_4x1(dst8_ptr + 0 * dst8_stride, d01_u8, 0);
+ store_u8_4x1(dst8_ptr + 1 * dst8_stride, d01_u8, 1);
+ store_u8_4x1(dst8_ptr + 2 * dst8_stride, d23_u8, 0);
+ store_u8_4x1(dst8_ptr + 3 * dst8_stride, d23_u8, 1);
+
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ dst8_ptr += 4 * dst8_stride;
+ height -= 4;
+ } while (height != 0);
+ } else {
+ const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int8x8_t x_filter = vshrn_n_s16(vld1q_s16(x_filter_ptr), 1);
+
+ do {
+ const uint8_t *s = src_ptr;
+ CONV_BUF_TYPE *d = dst_ptr;
+ uint8_t *d_u8 = dst8_ptr;
+ int width = w;
+
+ do {
+ uint8x16_t s0, s1, s2, s3;
+ load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3);
+
+ uint16x8_t d0 =
+ convolve8_8_x(s0, x_filter, permute_tbl, round_offset_shim);
+ uint16x8_t d1 =
+ convolve8_8_x(s1, x_filter, permute_tbl, round_offset_shim);
+ uint16x8_t d2 =
+ convolve8_8_x(s2, x_filter, permute_tbl, round_offset_shim);
+ uint16x8_t d3 =
+ convolve8_8_x(s3, x_filter, permute_tbl, round_offset_shim);
+
+ uint16x8_t dd0, dd1, dd2, dd3;
+ load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
+
+ uint8x8_t d0_u8, d1_u8, d2_u8, d3_u8;
+ compute_dist_wtd_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset,
+ bck_offset, round_offset_vec, &d0_u8, &d1_u8,
+ &d2_u8, &d3_u8);
+
+ store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8);
+
+ s += 8;
+ d += 8;
+ d_u8 += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ dst8_ptr += 4 * dst8_stride;
+ height -= 4;
+ } while (height != 0);
+ }
+}
+
+static INLINE void dist_wtd_convolve_x_avg_neon_i8mm(
+ const uint8_t *src, int src_stride, uint8_t *dst8, int dst8_stride, int w,
+ int h, const InterpFilterParams *filter_params_x, const int subpel_x_qn,
+ ConvolveParams *conv_params) {
+ assert(w % 4 == 0);
+ assert(h % 4 == 0);
+
+ const int bd = 8;
+ const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
+ const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
+ (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
+ const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
+ // A shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-rounding
+ // shifts - which are generally faster than rounding shifts on modern CPUs.
+ // (The extra -1 is needed because we halved the filter values.)
+ const int32x4_t round_offset_shim = vdupq_n_s32(
+ (round_offset << (ROUND0_BITS - 1)) + (1 << ((ROUND0_BITS - 1) - 1)));
+
+ // Horizontal filter.
+ const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
+ filter_params_x, subpel_x_qn & SUBPEL_MASK);
+
+ const int horiz_offset = filter_params_x->taps / 2 - 1;
+ const uint8_t *src_ptr = src - horiz_offset;
+ CONV_BUF_TYPE *dst_ptr = conv_params->dst;
+ uint8_t *dst8_ptr = dst8;
+ int dst_stride = conv_params->dst_stride;
+ int height = h;
+
+ if (w == 4) {
+ const uint8x16_t permute_tbl = vld1q_u8(dot_prod_permute_tbl);
+ // 4-tap filters are used for blocks having width <= 4.
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int8x8_t x_filter =
+ vshrn_n_s16(vcombine_s16(vld1_s16(x_filter_ptr + 2), vdup_n_s16(0)), 1);
+
+ src_ptr += 2;
+
+ do {
+ uint8x16_t s0, s1, s2, s3;
+ load_u8_16x4(src_ptr, src_stride, &s0, &s1, &s2, &s3);
+
+ uint16x4_t d0 =
+ convolve4_4_x(s0, x_filter, permute_tbl, round_offset_shim);
+ uint16x4_t d1 =
+ convolve4_4_x(s1, x_filter, permute_tbl, round_offset_shim);
+ uint16x4_t d2 =
+ convolve4_4_x(s2, x_filter, permute_tbl, round_offset_shim);
+ uint16x4_t d3 =
+ convolve4_4_x(s3, x_filter, permute_tbl, round_offset_shim);
+
+ uint16x4_t dd0, dd1, dd2, dd3;
+ load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3);
+
+ uint8x8_t d01_u8, d23_u8;
+ compute_basic_avg_4x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3,
+ round_offset_vec, &d01_u8, &d23_u8);
+
+ store_u8_4x1(dst8_ptr + 0 * dst8_stride, d01_u8, 0);
+ store_u8_4x1(dst8_ptr + 1 * dst8_stride, d01_u8, 1);
+ store_u8_4x1(dst8_ptr + 2 * dst8_stride, d23_u8, 0);
+ store_u8_4x1(dst8_ptr + 3 * dst8_stride, d23_u8, 1);
+
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ dst8_ptr += 4 * dst8_stride;
+ height -= 4;
+ } while (height != 0);
+ } else {
+ const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int8x8_t x_filter = vshrn_n_s16(vld1q_s16(x_filter_ptr), 1);
+
+ do {
+ const uint8_t *s = src_ptr;
+ CONV_BUF_TYPE *d = dst_ptr;
+ uint8_t *d_u8 = dst8_ptr;
+ int width = w;
+
+ do {
+ uint8x16_t s0, s1, s2, s3;
+ load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3);
+
+ uint16x8_t d0 =
+ convolve8_8_x(s0, x_filter, permute_tbl, round_offset_shim);
+ uint16x8_t d1 =
+ convolve8_8_x(s1, x_filter, permute_tbl, round_offset_shim);
+ uint16x8_t d2 =
+ convolve8_8_x(s2, x_filter, permute_tbl, round_offset_shim);
+ uint16x8_t d3 =
+ convolve8_8_x(s3, x_filter, permute_tbl, round_offset_shim);
+
+ uint16x8_t dd0, dd1, dd2, dd3;
+ load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
+
+ uint8x8_t d0_u8, d1_u8, d2_u8, d3_u8;
+ compute_basic_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3,
+ round_offset_vec, &d0_u8, &d1_u8, &d2_u8, &d3_u8);
+
+ store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8);
+
+ s += 8;
+ d += 8;
+ d_u8 += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ dst8_ptr += 4 * dst8_stride;
+ height -= 4;
+ } while (height != 0);
+ }
+}
+
+static INLINE void dist_wtd_convolve_x_neon_i8mm(
+ const uint8_t *src, int src_stride, int w, int h,
+ const InterpFilterParams *filter_params_x, const int subpel_x_qn,
+ ConvolveParams *conv_params) {
+ assert(w % 4 == 0);
+ assert(h % 4 == 0);
+
+ const int bd = 8;
+ const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
+ const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
+ (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
+ // A shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-rounding
+ // shifts - which are generally faster than rounding shifts on modern CPUs.
+ // (The extra -1 is needed because we halved the filter values.)
+ const int32x4_t round_offset_shim = vdupq_n_s32(
+ (round_offset << (ROUND0_BITS - 1)) + (1 << ((ROUND0_BITS - 1) - 1)));
+
+ // Horizontal filter.
+ const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
+ filter_params_x, subpel_x_qn & SUBPEL_MASK);
+
+ const int horiz_offset = filter_params_x->taps / 2 - 1;
+ const uint8_t *src_ptr = src - horiz_offset;
+ CONV_BUF_TYPE *dst_ptr = conv_params->dst;
+ int dst_stride = conv_params->dst_stride;
+ int height = h;
+
+ if (w == 4) {
+ const uint8x16_t permute_tbl = vld1q_u8(dot_prod_permute_tbl);
+ // 4-tap filters are used for blocks having width <= 4.
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int8x8_t x_filter =
+ vshrn_n_s16(vcombine_s16(vld1_s16(x_filter_ptr + 2), vdup_n_s16(0)), 1);
+
+ src_ptr += 2;
+
+ do {
+ uint8x16_t s0, s1, s2, s3;
+ load_u8_16x4(src_ptr, src_stride, &s0, &s1, &s2, &s3);
+
+ uint16x4_t d0 =
+ convolve4_4_x(s0, x_filter, permute_tbl, round_offset_shim);
+ uint16x4_t d1 =
+ convolve4_4_x(s1, x_filter, permute_tbl, round_offset_shim);
+ uint16x4_t d2 =
+ convolve4_4_x(s2, x_filter, permute_tbl, round_offset_shim);
+ uint16x4_t d3 =
+ convolve4_4_x(s3, x_filter, permute_tbl, round_offset_shim);
+
+ store_u16_4x4(dst_ptr, dst_stride, d0, d1, d2, d3);
+
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ height -= 4;
+ } while (height != 0);
+ } else {
+ const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int8x8_t x_filter = vshrn_n_s16(vld1q_s16(x_filter_ptr), 1);
+
+ do {
+ const uint8_t *s = src_ptr;
+ CONV_BUF_TYPE *d = dst_ptr;
+ int width = w;
+
+ do {
+ uint8x16_t s0, s1, s2, s3;
+ load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3);
+
+ uint16x8_t d0 =
+ convolve8_8_x(s0, x_filter, permute_tbl, round_offset_shim);
+ uint16x8_t d1 =
+ convolve8_8_x(s1, x_filter, permute_tbl, round_offset_shim);
+ uint16x8_t d2 =
+ convolve8_8_x(s2, x_filter, permute_tbl, round_offset_shim);
+ uint16x8_t d3 =
+ convolve8_8_x(s3, x_filter, permute_tbl, round_offset_shim);
+
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ height -= 4;
+ } while (height != 0);
+ }
+}
+
+void av1_dist_wtd_convolve_x_neon_i8mm(
+ const uint8_t *src, int src_stride, uint8_t *dst8, int dst8_stride, int w,
+ int h, const InterpFilterParams *filter_params_x, const int subpel_x_qn,
+ ConvolveParams *conv_params) {
+ if (conv_params->do_average) {
+ if (UNLIKELY(conv_params->use_dist_wtd_comp_avg)) {
+ dist_wtd_convolve_x_dist_wtd_avg_neon_i8mm(
+ src, src_stride, dst8, dst8_stride, w, h, filter_params_x,
+ subpel_x_qn, conv_params);
+ } else {
+ dist_wtd_convolve_x_avg_neon_i8mm(src, src_stride, dst8, dst8_stride, w,
+ h, filter_params_x, subpel_x_qn,
+ conv_params);
+ }
+ } else {
+ dist_wtd_convolve_x_neon_i8mm(src, src_stride, w, h, filter_params_x,
+ subpel_x_qn, conv_params);
+ }
+}
diff --git a/av1/common/arm/convolve_neon.c b/av1/common/arm/convolve_neon.c
index 713aaad..fa98922 100644
--- a/av1/common/arm/convolve_neon.c
+++ b/av1/common/arm/convolve_neon.c
@@ -24,848 +24,19 @@
#include "av1/common/filter.h"
#include "av1/common/arm/convolve_neon.h"
-static INLINE int16x4_t convolve8_4x4(const int16x4_t s0, const int16x4_t s1,
- const int16x4_t s2, const int16x4_t s3,
- const int16x4_t s4, const int16x4_t s5,
- const int16x4_t s6, const int16x4_t s7,
- const int16x8_t filter) {
- const int16x4_t filter_lo = vget_low_s16(filter);
- const int16x4_t filter_hi = vget_high_s16(filter);
- int16x4_t sum;
-
- sum = vmul_lane_s16(s0, filter_lo, 0);
- sum = vmla_lane_s16(sum, s1, filter_lo, 1);
- sum = vmla_lane_s16(sum, s2, filter_lo, 2);
- sum = vmla_lane_s16(sum, s3, filter_lo, 3);
- sum = vmla_lane_s16(sum, s4, filter_hi, 0);
- sum = vmla_lane_s16(sum, s5, filter_hi, 1);
- sum = vmla_lane_s16(sum, s6, filter_hi, 2);
- sum = vmla_lane_s16(sum, s7, filter_hi, 3);
-
- return sum;
-}
-
-#if !AOM_ARCH_AARCH64
-static INLINE uint8x8_t convolve8_x_4x1(const int16x4_t s0, const int16x4_t s1,
- const int16x4_t s2, const int16x4_t s3,
- const int16x4_t s4, const int16x4_t s5,
- const int16x4_t s6, const int16x4_t s7,
- const int16x8_t filter,
- const int16x4_t horiz_const) {
- const int16x4_t filter_lo = vget_low_s16(filter);
- const int16x4_t filter_hi = vget_high_s16(filter);
- int16x4_t sum = horiz_const;
-
- sum = vmla_lane_s16(sum, s0, filter_lo, 0);
- sum = vmla_lane_s16(sum, s1, filter_lo, 1);
- sum = vmla_lane_s16(sum, s2, filter_lo, 2);
- sum = vmla_lane_s16(sum, s3, filter_lo, 3);
- sum = vmla_lane_s16(sum, s4, filter_hi, 0);
- sum = vmla_lane_s16(sum, s5, filter_hi, 1);
- sum = vmla_lane_s16(sum, s6, filter_hi, 2);
- sum = vmla_lane_s16(sum, s7, filter_hi, 3);
-
- // We halved the convolution filter values so - 1 from the right shift.
- return vqrshrun_n_s16(vcombine_s16(sum, vdup_n_s16(0)), FILTER_BITS - 1);
-}
-#endif // !AOM_ARCH_AARCH64
-
-#if AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_MATMUL_INT8)
-
-static INLINE int32x4_t convolve12_4_usdot(uint8x16_t samples,
- const int8x16_t filters,
- const uint8x16x3_t permute_tbl,
- const int32x4_t horiz_const) {
- uint8x16_t permuted_samples[3];
- int32x4_t sum;
-
- /* Permute samples ready for dot product. */
- /* { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } */
- permuted_samples[0] = vqtbl1q_u8(samples, permute_tbl.val[0]);
- /* { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } */
- permuted_samples[1] = vqtbl1q_u8(samples, permute_tbl.val[1]);
- /* { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 } */
- permuted_samples[2] = vqtbl1q_u8(samples, permute_tbl.val[2]);
-
- /* First 4 output values. */
- sum = vusdotq_laneq_s32(horiz_const, permuted_samples[0], filters, 0);
- sum = vusdotq_laneq_s32(sum, permuted_samples[1], filters, 1);
- sum = vusdotq_laneq_s32(sum, permuted_samples[2], filters, 2);
-
- return sum;
-}
-
-static INLINE int16x8_t convolve12_8_usdot(uint8x16_t samples0,
- uint8x16_t samples1,
- const int8x16_t filters,
- const uint8x16x3_t permute_tbl,
- const int32x4_t horiz_const) {
- uint8x16_t permuted_samples[4];
- int32x4_t sum[2];
-
- /* Permute samples ready for dot product. */
- /* { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } */
- permuted_samples[0] = vqtbl1q_u8(samples0, permute_tbl.val[0]);
- /* { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } */
- permuted_samples[1] = vqtbl1q_u8(samples0, permute_tbl.val[1]);
- /* { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 } */
- permuted_samples[2] = vqtbl1q_u8(samples0, permute_tbl.val[2]);
- /* {12, 13, 14, 15, 13, 14, 15, 16, 14, 15, 16, 17, 15, 16, 17, 18 } */
- permuted_samples[3] = vqtbl1q_u8(samples1, permute_tbl.val[2]);
-
- /* First 4 output values. */
- sum[0] = vusdotq_laneq_s32(horiz_const, permuted_samples[0], filters, 0);
- sum[0] = vusdotq_laneq_s32(sum[0], permuted_samples[1], filters, 1);
- sum[0] = vusdotq_laneq_s32(sum[0], permuted_samples[2], filters, 2);
- /* Second 4 output values. */
- sum[1] = vusdotq_laneq_s32(horiz_const, permuted_samples[1], filters, 0);
- sum[1] = vusdotq_laneq_s32(sum[1], permuted_samples[2], filters, 1);
- sum[1] = vusdotq_laneq_s32(sum[1], permuted_samples[3], filters, 2);
-
- /* Narrow and re-pack. */
- return vcombine_s16(vqrshrn_n_s32(sum[0], FILTER_BITS),
- vqrshrn_n_s32(sum[1], FILTER_BITS));
-}
-
-#elif AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_DOTPROD)
-
-static INLINE int16x4_t convolve12_horiz_4_sdot(
- uint8x16_t samples, const int8x16_t filters, const int32x4_t correction,
- const uint8x16_t range_limit, const uint8x16x3_t permute_tbl) {
- int8x16_t clamped_samples, permuted_samples[3];
- int32x4_t sum;
-
- /* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
- clamped_samples = vreinterpretq_s8_u8(vsubq_u8(samples, range_limit));
-
- /* Permute samples ready for dot product. */
- /* { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } */
- permuted_samples[0] = vqtbl1q_s8(clamped_samples, permute_tbl.val[0]);
- /* { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } */
- permuted_samples[1] = vqtbl1q_s8(clamped_samples, permute_tbl.val[1]);
- /* { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 } */
- permuted_samples[2] = vqtbl1q_s8(clamped_samples, permute_tbl.val[2]);
-
- /* Accumulate dot product into 'correction' to account for range clamp. */
- /* First 4 output values. */
- sum = vdotq_laneq_s32(correction, permuted_samples[0], filters, 0);
- sum = vdotq_laneq_s32(sum, permuted_samples[1], filters, 1);
- sum = vdotq_laneq_s32(sum, permuted_samples[2], filters, 2);
-
- /* Narrow and re-pack. */
- return vshrn_n_s32(sum, ROUND0_BITS);
-}
-
-static INLINE int16x8_t convolve12_horiz_8_sdot(
- uint8x16_t samples0, uint8x16_t samples1, const int8x16_t filters,
- const int32x4_t correction, const uint8x16_t range_limit,
- const uint8x16x3_t permute_tbl) {
- int8x16_t clamped_samples[2], permuted_samples[4];
- int32x4_t sum[2];
-
- /* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
- clamped_samples[0] = vreinterpretq_s8_u8(vsubq_u8(samples0, range_limit));
- clamped_samples[1] = vreinterpretq_s8_u8(vsubq_u8(samples1, range_limit));
-
- /* Permute samples ready for dot product. */
- /* { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } */
- permuted_samples[0] = vqtbl1q_s8(clamped_samples[0], permute_tbl.val[0]);
- /* { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } */
- permuted_samples[1] = vqtbl1q_s8(clamped_samples[0], permute_tbl.val[1]);
- /* { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 } */
- permuted_samples[2] = vqtbl1q_s8(clamped_samples[0], permute_tbl.val[2]);
- /* {12, 13, 14, 15, 13, 14, 15, 16, 14, 15, 16, 17, 15, 16, 17, 18 } */
- permuted_samples[3] = vqtbl1q_s8(clamped_samples[1], permute_tbl.val[2]);
-
- /* Accumulate dot product into 'correction' to account for range clamp. */
- /* First 4 output values. */
- sum[0] = vdotq_laneq_s32(correction, permuted_samples[0], filters, 0);
- sum[0] = vdotq_laneq_s32(sum[0], permuted_samples[1], filters, 1);
- sum[0] = vdotq_laneq_s32(sum[0], permuted_samples[2], filters, 2);
- /* Second 4 output values. */
- sum[1] = vdotq_laneq_s32(correction, permuted_samples[1], filters, 0);
- sum[1] = vdotq_laneq_s32(sum[1], permuted_samples[2], filters, 1);
- sum[1] = vdotq_laneq_s32(sum[1], permuted_samples[3], filters, 2);
-
- /* Narrow and re-pack. */
- return vcombine_s16(vshrn_n_s32(sum[0], ROUND0_BITS),
- vshrn_n_s32(sum[1], ROUND0_BITS));
-}
-
-static INLINE int32x4_t convolve12_4_sdot(uint8x16_t samples,
- const int8x16_t filters,
- const int32x4_t correction,
- const uint8x16_t range_limit,
- const uint8x16x3_t permute_tbl) {
- int8x16_t clamped_samples, permuted_samples[3];
- int32x4_t sum;
-
- /* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
- clamped_samples = vreinterpretq_s8_u8(vsubq_u8(samples, range_limit));
-
- /* Permute samples ready for dot product. */
- /* { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } */
- permuted_samples[0] = vqtbl1q_s8(clamped_samples, permute_tbl.val[0]);
- /* { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } */
- permuted_samples[1] = vqtbl1q_s8(clamped_samples, permute_tbl.val[1]);
- /* { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 } */
- permuted_samples[2] = vqtbl1q_s8(clamped_samples, permute_tbl.val[2]);
-
- /* Accumulate dot product into 'correction' to account for range clamp. */
- /* First 4 output values. */
- sum = vdotq_laneq_s32(correction, permuted_samples[0], filters, 0);
- sum = vdotq_laneq_s32(sum, permuted_samples[1], filters, 1);
- sum = vdotq_laneq_s32(sum, permuted_samples[2], filters, 2);
-
- return sum;
-}
-
-static INLINE int16x8_t convolve12_8_sdot(uint8x16_t samples0,
- uint8x16_t samples1,
- const int8x16_t filters,
- const int32x4_t correction,
- const uint8x16_t range_limit,
- const uint8x16x3_t permute_tbl) {
- int8x16_t clamped_samples[2], permuted_samples[4];
- int32x4_t sum[2];
-
- /* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
- clamped_samples[0] = vreinterpretq_s8_u8(vsubq_u8(samples0, range_limit));
- clamped_samples[1] = vreinterpretq_s8_u8(vsubq_u8(samples1, range_limit));
-
- /* Permute samples ready for dot product. */
- /* { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } */
- permuted_samples[0] = vqtbl1q_s8(clamped_samples[0], permute_tbl.val[0]);
- /* { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } */
- permuted_samples[1] = vqtbl1q_s8(clamped_samples[0], permute_tbl.val[1]);
- /* { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 } */
- permuted_samples[2] = vqtbl1q_s8(clamped_samples[0], permute_tbl.val[2]);
- /* {12, 13, 14, 15, 13, 14, 15, 16, 14, 15, 16, 17, 15, 16, 17, 18 } */
- permuted_samples[3] = vqtbl1q_s8(clamped_samples[1], permute_tbl.val[2]);
-
- /* Accumulate dot product into 'correction' to account for range clamp. */
- /* First 4 output values. */
- sum[0] = vdotq_laneq_s32(correction, permuted_samples[0], filters, 0);
- sum[0] = vdotq_laneq_s32(sum[0], permuted_samples[1], filters, 1);
- sum[0] = vdotq_laneq_s32(sum[0], permuted_samples[2], filters, 2);
- /* Second 4 output values. */
- sum[1] = vdotq_laneq_s32(correction, permuted_samples[1], filters, 0);
- sum[1] = vdotq_laneq_s32(sum[1], permuted_samples[2], filters, 1);
- sum[1] = vdotq_laneq_s32(sum[1], permuted_samples[3], filters, 2);
-
- /* Narrow and re-pack. */
- return vcombine_s16(vqrshrn_n_s32(sum[0], FILTER_BITS),
- vqrshrn_n_s32(sum[1], FILTER_BITS));
-}
-
-#endif // AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_MATMUL_INT8)
-
-static INLINE uint8x8_t convolve8_vert_8x4(
- const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
- const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
- const int16x8_t s6, const int16x8_t s7, const int16x8_t filter) {
- const int16x4_t filter_lo = vget_low_s16(filter);
- const int16x4_t filter_hi = vget_high_s16(filter);
- int16x8_t sum;
-
- sum = vmulq_lane_s16(s0, filter_lo, 0);
- sum = vmlaq_lane_s16(sum, s1, filter_lo, 1);
- sum = vmlaq_lane_s16(sum, s2, filter_lo, 2);
- sum = vmlaq_lane_s16(sum, s3, filter_lo, 3);
- sum = vmlaq_lane_s16(sum, s4, filter_hi, 0);
- sum = vmlaq_lane_s16(sum, s5, filter_hi, 1);
- sum = vmlaq_lane_s16(sum, s6, filter_hi, 2);
- sum = vmlaq_lane_s16(sum, s7, filter_hi, 3);
-
- return vqrshrun_n_s16(sum, FILTER_BITS - 1);
-}
-
-static INLINE int16x4_t convolve8_vert_4_s32(
- const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
- const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
- const int16x4_t s6, const int16x4_t s7, const int16x8_t y_filter) {
- const int16x4_t y_filter_lo = vget_low_s16(y_filter);
- const int16x4_t y_filter_hi = vget_high_s16(y_filter);
- int32x4_t sum;
-
- sum = vmull_lane_s16(s0, y_filter_lo, 0);
- sum = vmlal_lane_s16(sum, s1, y_filter_lo, 1);
- sum = vmlal_lane_s16(sum, s2, y_filter_lo, 2);
- sum = vmlal_lane_s16(sum, s3, y_filter_lo, 3);
- sum = vmlal_lane_s16(sum, s4, y_filter_hi, 0);
- sum = vmlal_lane_s16(sum, s5, y_filter_hi, 1);
- sum = vmlal_lane_s16(sum, s6, y_filter_hi, 2);
- sum = vmlal_lane_s16(sum, s7, y_filter_hi, 3);
-
- return vqrshrn_n_s32(sum, 2 * FILTER_BITS - ROUND0_BITS);
-}
-
-static INLINE uint8x8_t
-convolve8_vert_8_s32(const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
- const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
- const int16x8_t s6, const int16x8_t s7,
- const int16x8_t y_filter, const int16x8_t sub_const) {
- const int16x4_t y_filter_lo = vget_low_s16(y_filter);
- const int16x4_t y_filter_hi = vget_high_s16(y_filter);
- int32x4_t sum0, sum1;
- int16x8_t res;
-
- sum0 = vmull_lane_s16(vget_low_s16(s0), y_filter_lo, 0);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s1), y_filter_lo, 1);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s2), y_filter_lo, 2);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s3), y_filter_lo, 3);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s4), y_filter_hi, 0);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s5), y_filter_hi, 1);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s6), y_filter_hi, 2);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s7), y_filter_hi, 3);
-
- sum1 = vmull_lane_s16(vget_high_s16(s0), y_filter_lo, 0);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s1), y_filter_lo, 1);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s2), y_filter_lo, 2);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s3), y_filter_lo, 3);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s4), y_filter_hi, 0);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s5), y_filter_hi, 1);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s6), y_filter_hi, 2);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s7), y_filter_hi, 3);
-
- res = vcombine_s16(vqrshrn_n_s32(sum0, 2 * FILTER_BITS - ROUND0_BITS),
- vqrshrn_n_s32(sum1, 2 * FILTER_BITS - ROUND0_BITS));
- res = vsubq_s16(res, sub_const);
-
- return vqmovun_s16(res);
-}
-
-#if AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_MATMUL_INT8)
-
-void convolve_x_sr_12tap_neon(const uint8_t *src, int src_stride, uint8_t *dst,
- int dst_stride, int w, int h,
- const int16_t *x_filter_ptr) {
- const int16x8_t filter_0_7 = vld1q_s16(x_filter_ptr);
- const int16x4_t filter_8_11 = vld1_s16(x_filter_ptr + 8);
- const int16x8_t filter_8_15 = vcombine_s16(filter_8_11, vdup_n_s16(0));
- const int8x16_t filter =
- vcombine_s8(vmovn_s16(filter_0_7), vmovn_s16(filter_8_15));
-
- // Special case the following no-op filter as 128 won't fit into the
- // 8-bit signed dot-product instruction:
- // { 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0 }
- if (vgetq_lane_s16(filter_0_7, 5) == 128) {
- uint8x8_t d0;
-
- // Undo the horizontal offset in the calling function.
- src += 5;
-
- for (int i = 0; i < h; i++) {
- for (int j = 0; j < w; j += 8) {
- d0 = vld1_u8(src + i * src_stride + j);
- if (w == 2) {
- store_u8_2x1(dst + i * dst_stride, d0, 0);
- } else if (w == 4) {
- store_u8_4x1(dst + i * dst_stride, d0, 0);
- } else {
- vst1_u8(dst + i * dst_stride + j, d0);
- }
- }
- }
- } else {
- const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
- // This shim of 1 << (ROUND0_BITS - 1) enables us to use a single rounding
- // right shift by FILTER_BITS - instead of a first rounding right shift by
- // ROUND0_BITS, followed by second rounding right shift by FILTER_BITS -
- // ROUND0_BITS.
- const int32x4_t horiz_const = vdupq_n_s32(1 << (ROUND0_BITS - 1));
-
- if (w <= 4) {
- uint8x16_t s0, s1, s2, s3;
- int32x4_t d0, d1, d2, d3;
- int16x8_t t01, t23;
- uint8x8_t d01, d23;
-
- do {
- load_u8_16x4(src, src_stride, &s0, &s1, &s2, &s3);
-
- d0 = convolve12_4_usdot(s0, filter, permute_tbl, horiz_const);
- d1 = convolve12_4_usdot(s1, filter, permute_tbl, horiz_const);
- d2 = convolve12_4_usdot(s2, filter, permute_tbl, horiz_const);
- d3 = convolve12_4_usdot(s3, filter, permute_tbl, horiz_const);
-
- t01 = vcombine_s16(vqrshrn_n_s32(d0, FILTER_BITS),
- vqrshrn_n_s32(d1, FILTER_BITS));
- t23 = vcombine_s16(vqrshrn_n_s32(d2, FILTER_BITS),
- vqrshrn_n_s32(d3, FILTER_BITS));
-
- d01 = vqmovun_s16(t01);
- d23 = vqmovun_s16(t23);
-
- if (w == 2) {
- store_u8_2x1(dst + 0 * dst_stride, d01, 0);
- store_u8_2x1(dst + 1 * dst_stride, d01, 2);
- if (h != 2) {
- store_u8_2x1(dst + 2 * dst_stride, d23, 0);
- store_u8_2x1(dst + 3 * dst_stride, d23, 2);
- }
- } else {
- store_u8_4x1(dst + 0 * dst_stride, d01, 0);
- store_u8_4x1(dst + 1 * dst_stride, d01, 1);
- if (h != 2) {
- store_u8_4x1(dst + 2 * dst_stride, d23, 0);
- store_u8_4x1(dst + 3 * dst_stride, d23, 1);
- }
- }
-
- dst += 4 * dst_stride;
- src += 4 * src_stride;
- h -= 4;
- } while (h > 0);
- } else {
- uint8x16_t s0, s1, s2, s3, s4, s5, s6, s7;
- int16x8_t d0, d1, d2, d3;
- uint8x8_t dd0, dd1, dd2, dd3;
-
- do {
- const uint8_t *s = src;
- uint8_t *d = dst;
- int width = w;
-
- do {
- load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3);
- load_u8_16x4(s + 4, src_stride, &s4, &s5, &s6, &s7);
-
- d0 = convolve12_8_usdot(s0, s4, filter, permute_tbl, horiz_const);
- d1 = convolve12_8_usdot(s1, s5, filter, permute_tbl, horiz_const);
- d2 = convolve12_8_usdot(s2, s6, filter, permute_tbl, horiz_const);
- d3 = convolve12_8_usdot(s3, s7, filter, permute_tbl, horiz_const);
-
- dd0 = vqmovun_s16(d0);
- dd1 = vqmovun_s16(d1);
- dd2 = vqmovun_s16(d2);
- dd3 = vqmovun_s16(d3);
-
- store_u8_8x2(d + 0 * dst_stride, dst_stride, dd0, dd1);
- if (h != 2) {
- store_u8_8x2(d + 2 * dst_stride, dst_stride, dd2, dd3);
- }
-
- s += 8;
- d += 8;
- width -= 8;
- } while (width > 0);
- src += 4 * src_stride;
- dst += 4 * dst_stride;
- h -= 4;
- } while (h > 0);
- }
- }
-}
-
-void av1_convolve_x_sr_neon(const uint8_t *src, int src_stride, uint8_t *dst,
- int dst_stride, int w, int h,
- const InterpFilterParams *filter_params_x,
- const int subpel_x_qn,
- ConvolveParams *conv_params) {
- (void)conv_params;
- const uint8_t horiz_offset = filter_params_x->taps / 2 - 1;
-
- const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
- filter_params_x, subpel_x_qn & SUBPEL_MASK);
- src -= horiz_offset;
-
- if (filter_params_x->taps > 8) {
- convolve_x_sr_12tap_neon(src, src_stride, dst, dst_stride, w, h,
- x_filter_ptr);
- return;
- }
-
- // Filter values are even, so downshift by 1 to reduce intermediate precision
- // requirements.
- const int8x8_t x_filter = vshrn_n_s16(vld1q_s16(x_filter_ptr), 1);
- // This shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use a single
- // rounding right shift by FILTER_BITS - instead of a first rounding right
- // shift by ROUND0_BITS, followed by second rounding right shift by
- // FILTER_BITS - ROUND0_BITS.
- // The outermost -1 is needed because we halved the filter values.
- const int32x4_t horiz_const = vdupq_n_s32(1 << ((ROUND0_BITS - 1) - 1));
-
- if (w <= 4) {
- const uint8x16x2_t permute_tbl = vld1q_u8_x2(dot_prod_permute_tbl);
- uint8x16_t s0, s1, s2, s3;
- int32x4_t t0, t1, t2, t3;
- int16x8_t t01, t23;
- uint8x8_t d01, d23;
-
- do {
- load_u8_16x4(src, src_stride, &s0, &s1, &s2, &s3);
-
- t0 = convolve8_4_usdot(s0, x_filter, permute_tbl, horiz_const);
- t1 = convolve8_4_usdot(s1, x_filter, permute_tbl, horiz_const);
- t2 = convolve8_4_usdot(s2, x_filter, permute_tbl, horiz_const);
- t3 = convolve8_4_usdot(s3, x_filter, permute_tbl, horiz_const);
-
- t01 = vcombine_s16(vmovn_s32(t0), vmovn_s32(t1));
- t23 = vcombine_s16(vmovn_s32(t2), vmovn_s32(t3));
-
- // We halved the convolution filter values so - 1 from the right shift.
- d01 = vqrshrun_n_s16(t01, FILTER_BITS - 1);
- d23 = vqrshrun_n_s16(t23, FILTER_BITS - 1);
-
- if (w == 2) {
- store_u8_2x1(dst + 0 * dst_stride, d01, 0);
- store_u8_2x1(dst + 1 * dst_stride, d01, 2);
- if (h != 2) {
- store_u8_2x1(dst + 2 * dst_stride, d23, 0);
- store_u8_2x1(dst + 3 * dst_stride, d23, 2);
- }
- } else {
- store_u8_4x1(dst + 0 * dst_stride, d01, 0);
- store_u8_4x1(dst + 1 * dst_stride, d01, 1);
- if (h != 2) {
- store_u8_4x1(dst + 2 * dst_stride, d23, 0);
- store_u8_4x1(dst + 3 * dst_stride, d23, 1);
- }
- }
-
- h -= 4;
- src += 4 * src_stride;
- dst += 4 * dst_stride;
- } while (h > 0);
-
- } else {
- const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
- uint8x16_t s0, s1, s2, s3;
- int16x8_t t0, t1, t2, t3;
- uint8x8_t d0, d1, d2, d3;
-
- do {
- int width = w;
- const uint8_t *s = src;
- uint8_t *d = dst;
-
- do {
- load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3);
-
- t0 = convolve8_x_8_usdot(s0, x_filter, permute_tbl, horiz_const);
- t1 = convolve8_x_8_usdot(s1, x_filter, permute_tbl, horiz_const);
- t2 = convolve8_x_8_usdot(s2, x_filter, permute_tbl, horiz_const);
- t3 = convolve8_x_8_usdot(s3, x_filter, permute_tbl, horiz_const);
-
- // We halved the convolution filter values so - 1 from the right shift.
- d0 = vqrshrun_n_s16(t0, FILTER_BITS - 1);
- d1 = vqrshrun_n_s16(t1, FILTER_BITS - 1);
- d2 = vqrshrun_n_s16(t2, FILTER_BITS - 1);
- d3 = vqrshrun_n_s16(t3, FILTER_BITS - 1);
-
- vst1_u8(d + 0 * dst_stride, d0);
- vst1_u8(d + 1 * dst_stride, d1);
- if (h != 2) {
- vst1_u8(d + 2 * dst_stride, d2);
- vst1_u8(d + 3 * dst_stride, d3);
- }
-
- s += 8;
- d += 8;
- width -= 8;
- } while (width > 0);
-
- src += 4 * src_stride;
- dst += 4 * dst_stride;
- h -= 4;
- } while (h > 0);
- }
-}
-
-#elif AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_DOTPROD)
-
-void convolve_x_sr_12tap_neon(const uint8_t *src, int src_stride, uint8_t *dst,
- int dst_stride, int w, int h,
- const int16_t *x_filter_ptr) {
- const int16x8_t filter_0_7 = vld1q_s16(x_filter_ptr);
- const int16x4_t filter_8_11 = vld1_s16(x_filter_ptr + 8);
- const int16x8_t filter_8_15 = vcombine_s16(filter_8_11, vdup_n_s16(0));
- const int8x16_t filter =
- vcombine_s8(vmovn_s16(filter_0_7), vmovn_s16(filter_8_15));
-
- const int32x4_t correct_tmp =
- vaddq_s32(vpaddlq_s16(vshlq_n_s16(filter_0_7, 7)),
- vpaddlq_s16(vshlq_n_s16(filter_8_15, 7)));
- // This shim of 1 << (ROUND0_BITS - 1) enables us to use a single rounding
- // right shift by FILTER_BITS - instead of a first rounding right shift by
- // ROUND0_BITS, followed by second rounding right shift by FILTER_BITS -
- // ROUND0_BITS.
- int32x4_t correction =
- vdupq_n_s32(vaddvq_s32(correct_tmp) + (1 << (ROUND0_BITS - 1)));
- const uint8x16_t range_limit = vdupq_n_u8(128);
- const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
-
- // Special case the following no-op filter as 128 won't fit into the
- // 8-bit signed dot-product instruction:
- // { 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0 }
- if (vgetq_lane_s16(filter_0_7, 5) == 128) {
- uint8x8_t d0;
-
- // Undo the horizontal offset in the calling function.
- src += 5;
-
- for (int i = 0; i < h; i++) {
- for (int j = 0; j < w; j += 8) {
- d0 = vld1_u8(src + i * src_stride + j);
- if (w == 2) {
- store_u8_2x1(dst + i * dst_stride, d0, 0);
- } else if (w == 4) {
- store_u8_4x1(dst + i * dst_stride, d0, 0);
- } else {
- vst1_u8(dst + i * dst_stride + j, d0);
- }
- }
- }
- } else {
- if (w <= 4) {
- uint8x16_t s0, s1, s2, s3;
- int32x4_t d0, d1, d2, d3;
- int16x8_t t01, t23;
- uint8x8_t d01, d23;
-
- do {
- load_u8_16x4(src, src_stride, &s0, &s1, &s2, &s3);
-
- d0 =
- convolve12_4_sdot(s0, filter, correction, range_limit, permute_tbl);
- d1 =
- convolve12_4_sdot(s1, filter, correction, range_limit, permute_tbl);
- d2 =
- convolve12_4_sdot(s2, filter, correction, range_limit, permute_tbl);
- d3 =
- convolve12_4_sdot(s3, filter, correction, range_limit, permute_tbl);
-
- t01 = vcombine_s16(vqrshrn_n_s32(d0, FILTER_BITS),
- vqrshrn_n_s32(d1, FILTER_BITS));
- t23 = vcombine_s16(vqrshrn_n_s32(d2, FILTER_BITS),
- vqrshrn_n_s32(d3, FILTER_BITS));
-
- d01 = vqmovun_s16(t01);
- d23 = vqmovun_s16(t23);
-
- if (w == 2) {
- store_u8_2x1(dst + 0 * dst_stride, d01, 0);
- store_u8_2x1(dst + 1 * dst_stride, d01, 2);
- if (h != 2) {
- store_u8_2x1(dst + 2 * dst_stride, d23, 0);
- store_u8_2x1(dst + 3 * dst_stride, d23, 2);
- }
- } else {
- store_u8_4x1(dst + 0 * dst_stride, d01, 0);
- store_u8_4x1(dst + 1 * dst_stride, d01, 1);
- if (h != 2) {
- store_u8_4x1(dst + 2 * dst_stride, d23, 0);
- store_u8_4x1(dst + 3 * dst_stride, d23, 1);
- }
- }
-
- dst += 4 * dst_stride;
- src += 4 * src_stride;
- h -= 4;
- } while (h > 0);
- } else {
- uint8x16_t s0, s1, s2, s3, s4, s5, s6, s7;
- int16x8_t d0, d1, d2, d3;
- uint8x8_t dd0, dd1, dd2, dd3;
-
- do {
- const uint8_t *s = src;
- uint8_t *d = dst;
- int width = w;
-
- do {
- load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3);
- load_u8_16x4(s + 4, src_stride, &s4, &s5, &s6, &s7);
-
- d0 = convolve12_8_sdot(s0, s4, filter, correction, range_limit,
- permute_tbl);
- d1 = convolve12_8_sdot(s1, s5, filter, correction, range_limit,
- permute_tbl);
- d2 = convolve12_8_sdot(s2, s6, filter, correction, range_limit,
- permute_tbl);
- d3 = convolve12_8_sdot(s3, s7, filter, correction, range_limit,
- permute_tbl);
-
- dd0 = vqmovun_s16(d0);
- dd1 = vqmovun_s16(d1);
- dd2 = vqmovun_s16(d2);
- dd3 = vqmovun_s16(d3);
-
- store_u8_8x2(d + 0 * dst_stride, dst_stride, dd0, dd1);
- if (h != 2) {
- store_u8_8x2(d + 2 * dst_stride, dst_stride, dd2, dd3);
- }
-
- s += 8;
- d += 8;
- width -= 8;
- } while (width > 0);
- src += 4 * src_stride;
- dst += 4 * dst_stride;
- h -= 4;
- } while (h > 0);
- }
- }
-}
-
-void av1_convolve_x_sr_neon(const uint8_t *src, int src_stride, uint8_t *dst,
- int dst_stride, int w, int h,
- const InterpFilterParams *filter_params_x,
- const int subpel_x_qn,
- ConvolveParams *conv_params) {
- (void)conv_params;
- const uint8_t horiz_offset = filter_params_x->taps / 2 - 1;
-
- const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
- filter_params_x, subpel_x_qn & SUBPEL_MASK);
- src -= horiz_offset;
-
- if (filter_params_x->taps > 8) {
- convolve_x_sr_12tap_neon(src, src_stride, dst, dst_stride, w, h,
- x_filter_ptr);
- return;
- }
-
- // Filter values are even, so downshift by 1 to reduce intermediate precision
- // requirements.
- const int8x8_t x_filter = vshrn_n_s16(vld1q_s16(x_filter_ptr), 1);
- // Dot product constants.
- const int16x8_t correct_tmp = vshll_n_s8(x_filter, 7);
- // This shim of (1 << ((ROUND0_BITS - 1) - 1) enables us to use a single
- // rounding right shift by FILTER_BITS - instead of a first rounding right
- // shift by ROUND0_BITS, followed by second rounding right shift by
- // FILTER_BITS - ROUND0_BITS.
- // The outermost -1 is needed because we halved the filter values.
- const int32x4_t correction =
- vdupq_n_s32(vaddlvq_s16(correct_tmp) + (1 << ((ROUND0_BITS - 1) - 1)));
- const uint8x16_t range_limit = vdupq_n_u8(128);
-
- if (w <= 4) {
- const uint8x16x2_t permute_tbl = vld1q_u8_x2(dot_prod_permute_tbl);
- uint8x16_t s0, s1, s2, s3;
- int32x4_t t0, t1, t2, t3;
- int16x8_t t01, t23;
- uint8x8_t d01, d23;
-
- do {
- load_u8_16x4(src, src_stride, &s0, &s1, &s2, &s3);
-
- t0 = convolve8_4_sdot(s0, x_filter, correction, range_limit, permute_tbl);
- t1 = convolve8_4_sdot(s1, x_filter, correction, range_limit, permute_tbl);
- t2 = convolve8_4_sdot(s2, x_filter, correction, range_limit, permute_tbl);
- t3 = convolve8_4_sdot(s3, x_filter, correction, range_limit, permute_tbl);
-
- t01 = vcombine_s16(vmovn_s32(t0), vmovn_s32(t1));
- t23 = vcombine_s16(vmovn_s32(t2), vmovn_s32(t3));
-
- // We halved the convolution filter values so - 1 from the right shift.
- d01 = vqrshrun_n_s16(t01, FILTER_BITS - 1);
- d23 = vqrshrun_n_s16(t23, FILTER_BITS - 1);
-
- if (w == 2) {
- store_u8_2x1(dst + 0 * dst_stride, d01, 0);
- store_u8_2x1(dst + 1 * dst_stride, d01, 2);
- if (h != 2) {
- store_u8_2x1(dst + 2 * dst_stride, d23, 0);
- store_u8_2x1(dst + 3 * dst_stride, d23, 2);
- }
- } else {
- store_u8_4x1(dst + 0 * dst_stride, d01, 0);
- store_u8_4x1(dst + 1 * dst_stride, d01, 1);
- if (h != 2) {
- store_u8_4x1(dst + 2 * dst_stride, d23, 0);
- store_u8_4x1(dst + 3 * dst_stride, d23, 1);
- }
- }
-
- h -= 4;
- src += 4 * src_stride;
- dst += 4 * dst_stride;
- } while (h > 0);
- } else {
- const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
- uint8x16_t s0, s1, s2, s3;
- int16x8_t t0, t1, t2, t3;
- uint8x8_t d0, d1, d2, d3;
-
- do {
- int width = w;
- const uint8_t *s = src;
- uint8_t *d = dst;
-
- do {
- load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3);
-
- t0 = convolve8_x_8_sdot(s0, x_filter, correction, range_limit,
- permute_tbl);
- t1 = convolve8_x_8_sdot(s1, x_filter, correction, range_limit,
- permute_tbl);
- t2 = convolve8_x_8_sdot(s2, x_filter, correction, range_limit,
- permute_tbl);
- t3 = convolve8_x_8_sdot(s3, x_filter, correction, range_limit,
- permute_tbl);
-
- // We halved the convolution filter values so - 1 from the right shift.
- d0 = vqrshrun_n_s16(t0, FILTER_BITS - 1);
- d1 = vqrshrun_n_s16(t1, FILTER_BITS - 1);
- d2 = vqrshrun_n_s16(t2, FILTER_BITS - 1);
- d3 = vqrshrun_n_s16(t3, FILTER_BITS - 1);
-
- vst1_u8(d + 0 * dst_stride, d0);
- vst1_u8(d + 1 * dst_stride, d1);
- if (h != 2) {
- vst1_u8(d + 2 * dst_stride, d2);
- vst1_u8(d + 3 * dst_stride, d3);
- }
-
- s += 8;
- d += 8;
- width -= 8;
- } while (width > 0);
-
- src += 4 * src_stride;
- dst += 4 * dst_stride;
- h -= 4;
- } while (h > 0);
- }
-}
-
-#else // !(AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_DOTPROD))
-
-static INLINE uint8x8_t
-convolve8_horiz_8x8(const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
- const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
- const int16x8_t s6, const int16x8_t s7,
- const int16x8_t filter, const int16x8_t horiz_const) {
- const int16x4_t filter_lo = vget_low_s16(filter);
- const int16x4_t filter_hi = vget_high_s16(filter);
- int16x8_t sum = horiz_const;
-
- sum = vmlaq_lane_s16(sum, s0, filter_lo, 0);
- sum = vmlaq_lane_s16(sum, s1, filter_lo, 1);
- sum = vmlaq_lane_s16(sum, s2, filter_lo, 2);
- sum = vmlaq_lane_s16(sum, s3, filter_lo, 3);
- sum = vmlaq_lane_s16(sum, s4, filter_hi, 0);
- sum = vmlaq_lane_s16(sum, s5, filter_hi, 1);
- sum = vmlaq_lane_s16(sum, s6, filter_hi, 2);
- sum = vmlaq_lane_s16(sum, s7, filter_hi, 3);
-
- // We halved the convolution filter values so - 1 from the right shift.
- return vqrshrun_n_s16(sum, FILTER_BITS - 1);
-}
-
-static INLINE int16x4_t convolve12_x_4x4_s16(
- const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
- const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
- const int16x4_t s6, const int16x4_t s7, const int16x4_t s8,
- const int16x4_t s9, const int16x4_t s10, const int16x4_t s11,
- const int16x8_t x_filter_0_7, const int16x4_t x_filter_8_11,
- const int32x4_t horiz_const) {
+static INLINE int16x4_t convolve12_4_x(const int16x4_t s0, const int16x4_t s1,
+ const int16x4_t s2, const int16x4_t s3,
+ const int16x4_t s4, const int16x4_t s5,
+ const int16x4_t s6, const int16x4_t s7,
+ const int16x4_t s8, const int16x4_t s9,
+ const int16x4_t s10, const int16x4_t s11,
+ const int16x8_t x_filter_0_7,
+ const int16x4_t x_filter_8_11,
+ const int32x4_t horiz_const) {
const int16x4_t x_filter_0_3 = vget_low_s16(x_filter_0_7);
const int16x4_t x_filter_4_7 = vget_high_s16(x_filter_0_7);
- int32x4_t sum = horiz_const;
+ int32x4_t sum = horiz_const;
sum = vmlal_lane_s16(sum, s0, x_filter_0_3, 0);
sum = vmlal_lane_s16(sum, s1, x_filter_0_3, 1);
sum = vmlal_lane_s16(sum, s2, x_filter_0_3, 2);
@@ -882,69 +53,6 @@
return vqrshrn_n_s32(sum, FILTER_BITS);
}
-// 4 column per iteration filtering for 12-tap convolve_x_sr.
-// Processes one row at a time.
-static INLINE void x_filter_12tap_w4_single_row(
- const uint8_t *src_ptr, int src_stride, uint8_t *dst_ptr,
- const int dst_stride, int w, int h, const int16x8_t x_filter_0_7,
- const int16x4_t x_filter_8_11) {
- // This shim of 1 << (ROUND0_BITS - 1) enables us to use a single
- // rounding right shift by FILTER_BITS - instead of a first rounding right
- // shift by ROUND0_BITS, followed by second rounding right shift by
- // FILTER_BITS - ROUND0_BITS.
- const int32x4_t horiz_const = vdupq_n_s32(1 << (ROUND0_BITS - 1));
-
- do {
- const uint8_t *s = src_ptr;
- uint8_t *d = dst_ptr;
- int width = w;
-
- do {
- uint8x8_t dd0;
- uint8x16_t t0;
- int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, d0;
- int16x8_t tt0, tt1;
-
- t0 = vld1q_u8(s);
- tt0 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(t0)));
- tt1 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(t0)));
-
- s0 = vget_low_s16(tt0);
- s4 = vget_high_s16(tt0);
- s8 = vget_low_s16(tt1);
- s12 = vget_high_s16(tt1);
-
- s1 = vext_s16(s0, s4, 1); // a1 a2 a3 a4
- s2 = vext_s16(s0, s4, 2); // a2 a3 a4 a5
- s3 = vext_s16(s0, s4, 3); // a3 a4 a5 a6
- s5 = vext_s16(s4, s8, 1); // a5 a6 a7 a8
- s6 = vext_s16(s4, s8, 2); // a6 a7 a8 a9
- s7 = vext_s16(s4, s8, 3); // a7 a8 a9 a10
- s9 = vext_s16(s8, s12, 1); // a9 a10 a11 a12
- s10 = vext_s16(s8, s12, 2); // a10 a11 a12 a13
- s11 = vext_s16(s8, s12, 3); // a11 a12 a13 a14
-
- d0 = convolve12_x_4x4_s16(s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10,
- s11, x_filter_0_7, x_filter_8_11, horiz_const);
-
- dd0 = vqmovun_s16(vcombine_s16(d0, vdup_n_s16(0)));
-
- if (w == 2) {
- store_u8_2x1(d, dd0, 0);
- } else {
- store_u8_4x1(d, dd0, 0);
- }
-
- s += 4;
- d += 4;
- width -= 4;
- } while (width > 0);
-
- src_ptr += src_stride;
- dst_ptr += dst_stride;
- } while (--h != 0);
-}
-
static INLINE void convolve_x_sr_12tap_neon(const uint8_t *src_ptr,
int src_stride, uint8_t *dst_ptr,
const int dst_stride, int w, int h,
@@ -952,87 +60,71 @@
const int16x8_t x_filter_0_7 = vld1q_s16(x_filter_ptr);
const int16x4_t x_filter_8_11 = vld1_s16(x_filter_ptr + 8);
-#if AOM_ARCH_AARCH64
- // This shim of 1 << (ROUND0_BITS - 1) enables us to use a single
- // rounding right shift by FILTER_BITS - instead of a first rounding right
- // shift by ROUND0_BITS, followed by second rounding right shift by
- // FILTER_BITS - ROUND0_BITS.
+ // A shim of 1 << (ROUND0_BITS - 1) enables us to use a single rounding right
+ // shift by FILTER_BITS - instead of a first rounding right shift by
+ // ROUND0_BITS, followed by second rounding right shift by FILTER_BITS -
+ // ROUND0_BITS.
const int32x4_t horiz_const = vdupq_n_s32(1 << (ROUND0_BITS - 1));
+#if AOM_ARCH_AARCH64
do {
- int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
- uint8x8_t t0, t1, t2, t3;
-
const uint8_t *s = src_ptr;
uint8_t *d = dst_ptr;
int width = w;
+ uint8x8_t t0, t1, t2, t3;
load_u8_8x4(s, src_stride, &t0, &t1, &t2, &t3);
- transpose_u8_8x4(&t0, &t1, &t2, &t3);
+ transpose_elems_inplace_u8_8x4(&t0, &t1, &t2, &t3);
- s0 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s1 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
- s2 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
- s3 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t3)));
- s4 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s5 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
- s6 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
- s7 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t3)));
+ int16x4_t s0 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
+ int16x4_t s1 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
+ int16x4_t s2 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
+ int16x4_t s3 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t3)));
+ int16x4_t s4 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
+ int16x4_t s5 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
+ int16x4_t s6 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
+ int16x4_t s7 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t3)));
load_u8_8x4(s + 8, src_stride, &t0, &t1, &t2, &t3);
- transpose_u8_8x4(&t0, &t1, &t2, &t3);
+ transpose_elems_inplace_u8_8x4(&t0, &t1, &t2, &t3);
- s8 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s9 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
- s10 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
+ int16x4_t s8 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
+ int16x4_t s9 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
+ int16x4_t s10 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
s += 11;
do {
- int16x4_t s11, s12, s13, s14, d0, d1, d2, d3;
- int16x8_t d01, d23;
- uint8x8_t dd01, dd23;
-
load_u8_8x4(s, src_stride, &t0, &t1, &t2, &t3);
- transpose_u8_8x4(&t0, &t1, &t2, &t3);
+ transpose_elems_inplace_u8_8x4(&t0, &t1, &t2, &t3);
- s11 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s12 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
- s13 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
- s14 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t3)));
+ int16x4_t s11 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
+ int16x4_t s12 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
+ int16x4_t s13 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
+ int16x4_t s14 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t3)));
- d0 = convolve12_x_4x4_s16(s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10,
- s11, x_filter_0_7, x_filter_8_11, horiz_const);
- d1 = convolve12_x_4x4_s16(s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11,
- s12, x_filter_0_7, x_filter_8_11, horiz_const);
- d2 = convolve12_x_4x4_s16(s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12,
- s13, x_filter_0_7, x_filter_8_11, horiz_const);
- d3 = convolve12_x_4x4_s16(s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13,
- s14, x_filter_0_7, x_filter_8_11, horiz_const);
+ int16x4_t d0 =
+ convolve12_4_x(s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11,
+ x_filter_0_7, x_filter_8_11, horiz_const);
+ int16x4_t d1 =
+ convolve12_4_x(s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12,
+ x_filter_0_7, x_filter_8_11, horiz_const);
+ int16x4_t d2 =
+ convolve12_4_x(s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13,
+ x_filter_0_7, x_filter_8_11, horiz_const);
+ int16x4_t d3 =
+ convolve12_4_x(s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14,
+ x_filter_0_7, x_filter_8_11, horiz_const);
- transpose_s16_4x4d(&d0, &d1, &d2, &d3);
+ transpose_elems_inplace_s16_4x4(&d0, &d1, &d2, &d3);
- d01 = vcombine_s16(d0, d1);
- d23 = vcombine_s16(d2, d3);
+ uint8x8_t d01 = vqmovun_s16(vcombine_s16(d0, d1));
+ uint8x8_t d23 = vqmovun_s16(vcombine_s16(d2, d3));
- dd01 = vqmovun_s16(d01);
- dd23 = vqmovun_s16(d23);
-
- if (w == 2) {
- store_u8_2x1(d + 0 * dst_stride, dd01, 0);
- store_u8_2x1(d + 1 * dst_stride, dd01, 2);
- if (h != 2) {
- store_u8_2x1(d + 2 * dst_stride, dd23, 0);
- store_u8_2x1(d + 3 * dst_stride, dd23, 2);
- }
- } else {
- store_u8_4x1(d + 0 * dst_stride, dd01, 0);
- store_u8_4x1(d + 1 * dst_stride, dd01, 1);
- if (h != 2) {
- store_u8_4x1(d + 2 * dst_stride, dd23, 0);
- store_u8_4x1(d + 3 * dst_stride, dd23, 1);
- }
- }
+ store_u8_4x1(d + 0 * dst_stride, d01, 0);
+ store_u8_4x1(d + 1 * dst_stride, d01, 1);
+ store_u8_4x1(d + 2 * dst_stride, d23, 0);
+ store_u8_4x1(d + 3 * dst_stride, d23, 1);
s0 = s4;
s1 = s5;
@@ -1048,34 +140,109 @@
s += 4;
d += 4;
width -= 4;
- } while (width > 0);
-
+ } while (width != 0);
src_ptr += 4 * src_stride;
dst_ptr += 4 * dst_stride;
h -= 4;
- } while (h >= 4);
+ } while (h != 0);
- if (h > 0) {
- x_filter_12tap_w4_single_row(src_ptr, src_stride, dst_ptr, dst_stride, w, h,
- x_filter_0_7, x_filter_8_11);
- }
#else // !AOM_ARCH_AARCH64
- x_filter_12tap_w4_single_row(src_ptr, src_stride, dst_ptr, dst_stride, w, h,
- x_filter_0_7, x_filter_8_11);
+ do {
+ const uint8_t *s = src_ptr;
+ uint8_t *d = dst_ptr;
+ int width = w;
+
+ do {
+ uint8x16_t t0 = vld1q_u8(s);
+ int16x8_t tt0 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(t0)));
+ int16x8_t tt8 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(t0)));
+
+ int16x4_t s0 = vget_low_s16(tt0);
+ int16x4_t s4 = vget_high_s16(tt0);
+ int16x4_t s8 = vget_low_s16(tt8);
+ int16x4_t s12 = vget_high_s16(tt8);
+
+ int16x4_t s1 = vext_s16(s0, s4, 1); // a1 a2 a3 a4
+ int16x4_t s2 = vext_s16(s0, s4, 2); // a2 a3 a4 a5
+ int16x4_t s3 = vext_s16(s0, s4, 3); // a3 a4 a5 a6
+ int16x4_t s5 = vext_s16(s4, s8, 1); // a5 a6 a7 a8
+ int16x4_t s6 = vext_s16(s4, s8, 2); // a6 a7 a8 a9
+ int16x4_t s7 = vext_s16(s4, s8, 3); // a7 a8 a9 a10
+ int16x4_t s9 = vext_s16(s8, s12, 1); // a9 a10 a11 a12
+ int16x4_t s10 = vext_s16(s8, s12, 2); // a10 a11 a12 a13
+ int16x4_t s11 = vext_s16(s8, s12, 3); // a11 a12 a13 a14
+
+ int16x4_t d0 =
+ convolve12_4_x(s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11,
+ x_filter_0_7, x_filter_8_11, horiz_const);
+
+ uint8x8_t dd0 = vqmovun_s16(vcombine_s16(d0, vdup_n_s16(0)));
+
+ store_u8_4x1(d, dd0, 0);
+
+ s += 4;
+ d += 4;
+ width -= 4;
+ } while (width != 0);
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ } while (--h != 0);
#endif // AOM_ARCH_AARCH64
}
+static INLINE uint8x8_t convolve4_4_x(const int16x4_t s0, const int16x4_t s1,
+ const int16x4_t s2, const int16x4_t s3,
+ const int16x4_t filter,
+ const int16x4_t horiz_const) {
+ int16x4_t sum = horiz_const;
+ sum = vmla_lane_s16(sum, s0, filter, 0);
+ sum = vmla_lane_s16(sum, s1, filter, 1);
+ sum = vmla_lane_s16(sum, s2, filter, 2);
+ sum = vmla_lane_s16(sum, s3, filter, 3);
+
+ // We halved the convolution filter values so - 1 from the right shift.
+ return vqrshrun_n_s16(vcombine_s16(sum, vdup_n_s16(0)), FILTER_BITS - 1);
+}
+
+static INLINE uint8x8_t convolve8_8_x(const int16x8_t s0, const int16x8_t s1,
+ const int16x8_t s2, const int16x8_t s3,
+ const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t s6, const int16x8_t s7,
+ const int16x8_t filter,
+ const int16x8_t horiz_const) {
+ const int16x4_t filter_lo = vget_low_s16(filter);
+ const int16x4_t filter_hi = vget_high_s16(filter);
+
+ int16x8_t sum = horiz_const;
+ sum = vmlaq_lane_s16(sum, s0, filter_lo, 0);
+ sum = vmlaq_lane_s16(sum, s1, filter_lo, 1);
+ sum = vmlaq_lane_s16(sum, s2, filter_lo, 2);
+ sum = vmlaq_lane_s16(sum, s3, filter_lo, 3);
+ sum = vmlaq_lane_s16(sum, s4, filter_hi, 0);
+ sum = vmlaq_lane_s16(sum, s5, filter_hi, 1);
+ sum = vmlaq_lane_s16(sum, s6, filter_hi, 2);
+ sum = vmlaq_lane_s16(sum, s7, filter_hi, 3);
+
+ // We halved the convolution filter values so - 1 from the right shift.
+ return vqrshrun_n_s16(sum, FILTER_BITS - 1);
+}
+
void av1_convolve_x_sr_neon(const uint8_t *src, int src_stride, uint8_t *dst,
int dst_stride, int w, int h,
const InterpFilterParams *filter_params_x,
const int subpel_x_qn,
ConvolveParams *conv_params) {
- (void)conv_params;
+ if (w == 2 || h == 2) {
+ av1_convolve_x_sr_c(src, src_stride, dst, dst_stride, w, h, filter_params_x,
+ subpel_x_qn, conv_params);
+ return;
+ }
+
const uint8_t horiz_offset = filter_params_x->taps / 2 - 1;
+ src -= horiz_offset;
const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
filter_params_x, subpel_x_qn & SUBPEL_MASK);
- src -= horiz_offset;
if (filter_params_x->taps > 8) {
convolve_x_sr_12tap_neon(src, src_stride, dst, dst_stride, w, h,
@@ -1083,503 +250,303 @@
return;
}
- uint8x8_t t0;
-#if AOM_ARCH_AARCH64
- uint8x8_t t1, t2, t3;
// This shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use a single
// rounding right shift by FILTER_BITS - instead of a first rounding right
// shift by ROUND0_BITS, followed by second rounding right shift by
// FILTER_BITS - ROUND0_BITS.
- // The outermost -1 is needed because we halved the filter values.
+ // The outermost -1 is needed because we will halve the filter values.
const int16x8_t horiz_const = vdupq_n_s16(1 << ((ROUND0_BITS - 1) - 1));
-#endif // AOM_ARCH_AARCH64
- // Filter values are even so downshift by 1 to reduce precision requirements.
- const int16x8_t x_filter = vshrq_n_s16(vld1q_s16(x_filter_ptr), 1);
-#if AOM_ARCH_AARCH64
- if (h == 4) {
- uint8x8_t d01, d23;
- int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, d0, d1, d2, d3;
- int16x8_t d01_temp, d23_temp;
+ if (w <= 4) {
+ // 4-tap filters are used for blocks having width <= 4.
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int16x4_t x_filter = vshr_n_s16(vld1_s16(x_filter_ptr + 2), 1);
- __builtin_prefetch(src + 0 * src_stride);
- __builtin_prefetch(src + 1 * src_stride);
- __builtin_prefetch(src + 2 * src_stride);
- __builtin_prefetch(src + 3 * src_stride);
-
- load_u8_8x4(src, src_stride, &t0, &t1, &t2, &t3);
- transpose_u8_8x4(&t0, &t1, &t2, &t3);
-
- s0 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s1 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
- s2 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
- s3 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t3)));
- s4 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s5 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
- s6 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
- __builtin_prefetch(dst + 0 * dst_stride);
- __builtin_prefetch(dst + 1 * dst_stride);
- __builtin_prefetch(dst + 2 * dst_stride);
- __builtin_prefetch(dst + 3 * dst_stride);
- src += 7;
+ src += 2;
do {
- load_u8_8x4(src, src_stride, &t0, &t1, &t2, &t3);
- transpose_u8_8x4(&t0, &t1, &t2, &t3);
+ uint8x8_t t0 = vld1_u8(src); // a0 a1 a2 a3 a4 a5 a6 a7
+ int16x4_t s0 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
+ int16x4_t s4 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s7 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s8 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
- s9 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
- s10 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t3)));
+ int16x4_t s1 = vext_s16(s0, s4, 1); // a1 a2 a3 a4
+ int16x4_t s2 = vext_s16(s0, s4, 2); // a2 a3 a4 a5
+ int16x4_t s3 = vext_s16(s0, s4, 3); // a3 a4 a5 a6
- d0 = convolve8_4x4(s0, s1, s2, s3, s4, s5, s6, s7, x_filter);
- d1 = convolve8_4x4(s1, s2, s3, s4, s5, s6, s7, s8, x_filter);
- d2 = convolve8_4x4(s2, s3, s4, s5, s6, s7, s8, s9, x_filter);
- d3 = convolve8_4x4(s3, s4, s5, s6, s7, s8, s9, s10, x_filter);
+ uint8x8_t d0 =
+ convolve4_4_x(s0, s1, s2, s3, x_filter, vget_low_s16(horiz_const));
- d01_temp = vcombine_s16(d0, d1);
- d23_temp = vcombine_s16(d2, d3);
-
- d01_temp = vaddq_s16(d01_temp, horiz_const);
- d23_temp = vaddq_s16(d23_temp, horiz_const);
-
- // We halved the convolution filter values so - 1 from the right shift.
- d01 = vqrshrun_n_s16(d01_temp, FILTER_BITS - 1);
- d23 = vqrshrun_n_s16(d23_temp, FILTER_BITS - 1);
-
- transpose_u8_4x4(&d01, &d23);
-
- if (w == 2) {
- store_u8_2x1(dst + 0 * dst_stride, d01, 0);
- store_u8_2x1(dst + 1 * dst_stride, d23, 0);
- store_u8_2x1(dst + 2 * dst_stride, d01, 2);
- store_u8_2x1(dst + 3 * dst_stride, d23, 2);
- } else {
- store_u8_4x1(dst + 0 * dst_stride, d01, 0);
- store_u8_4x1(dst + 1 * dst_stride, d23, 0);
- store_u8_4x1(dst + 2 * dst_stride, d01, 1);
- store_u8_4x1(dst + 3 * dst_stride, d23, 1);
- }
-
- s0 = s4;
- s1 = s5;
- s2 = s6;
- s3 = s7;
- s4 = s8;
- s5 = s9;
- s6 = s10;
- src += 4;
- dst += 4;
- w -= 4;
- } while (w > 0);
- } else {
-#endif // AOM_ARCH_AARCH64
- int width;
- const uint8_t *s;
- int16x8_t s0, s1, s2, s3, s4, s5, s6, s7;
-
-#if AOM_ARCH_AARCH64
- int16x8_t s8, s9, s10;
- uint8x8_t t4, t5, t6, t7;
-#endif // AOM_ARCH_AARCH64
-
- if (w <= 4) {
-#if AOM_ARCH_AARCH64
- do {
- load_u8_8x8(src, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
- transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
- s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
- s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
- s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
- s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
- s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
- s5 = vreinterpretq_s16_u16(vmovl_u8(t5));
- s6 = vreinterpretq_s16_u16(vmovl_u8(t6));
-
- load_u8_8x8(src + 7, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6,
- &t7);
- src += 8 * src_stride;
- __builtin_prefetch(dst + 0 * dst_stride);
- __builtin_prefetch(dst + 1 * dst_stride);
- __builtin_prefetch(dst + 2 * dst_stride);
- __builtin_prefetch(dst + 3 * dst_stride);
- __builtin_prefetch(dst + 4 * dst_stride);
- __builtin_prefetch(dst + 5 * dst_stride);
- __builtin_prefetch(dst + 6 * dst_stride);
- __builtin_prefetch(dst + 7 * dst_stride);
-
- transpose_u8_4x8(&t0, &t1, &t2, &t3, t4, t5, t6, t7);
-
- s7 = vreinterpretq_s16_u16(vmovl_u8(t0));
- s8 = vreinterpretq_s16_u16(vmovl_u8(t1));
- s9 = vreinterpretq_s16_u16(vmovl_u8(t2));
- s10 = vreinterpretq_s16_u16(vmovl_u8(t3));
-
- __builtin_prefetch(src + 0 * src_stride);
- __builtin_prefetch(src + 1 * src_stride);
- __builtin_prefetch(src + 2 * src_stride);
- __builtin_prefetch(src + 3 * src_stride);
- __builtin_prefetch(src + 4 * src_stride);
- __builtin_prefetch(src + 5 * src_stride);
- __builtin_prefetch(src + 6 * src_stride);
- __builtin_prefetch(src + 7 * src_stride);
- t0 = convolve8_horiz_8x8(s0, s1, s2, s3, s4, s5, s6, s7, x_filter,
- horiz_const);
- t1 = convolve8_horiz_8x8(s1, s2, s3, s4, s5, s6, s7, s8, x_filter,
- horiz_const);
- t2 = convolve8_horiz_8x8(s2, s3, s4, s5, s6, s7, s8, s9, x_filter,
- horiz_const);
- t3 = convolve8_horiz_8x8(s3, s4, s5, s6, s7, s8, s9, s10, x_filter,
- horiz_const);
-
- transpose_u8_8x4(&t0, &t1, &t2, &t3);
-
- if (w == 4) {
- store_u8_4x1(dst + 0 * dst_stride, t0, 0);
- store_u8_4x1(dst + 1 * dst_stride, t1, 0);
- if (h > 4) {
- store_u8_4x1(dst + 2 * dst_stride, t2, 0);
- store_u8_4x1(dst + 3 * dst_stride, t3, 0);
- store_u8_4x1(dst + 4 * dst_stride, t0, 1);
- store_u8_4x1(dst + 5 * dst_stride, t1, 1);
- store_u8_4x1(dst + 6 * dst_stride, t2, 1);
- store_u8_4x1(dst + 7 * dst_stride, t3, 1);
- }
- } else if (w == 2) {
- store_u8_2x1(dst + 0 * dst_stride, t0, 0);
- store_u8_2x1(dst + 1 * dst_stride, t1, 0);
- if (h > 4) {
- store_u8_2x1(dst + 2 * dst_stride, t2, 0);
- store_u8_2x1(dst + 3 * dst_stride, t3, 0);
- store_u8_2x1(dst + 4 * dst_stride, t0, 2);
- store_u8_2x1(dst + 5 * dst_stride, t1, 2);
- store_u8_2x1(dst + 6 * dst_stride, t2, 2);
- store_u8_2x1(dst + 7 * dst_stride, t3, 2);
- }
- }
-
- dst += 8 * dst_stride;
- h -= 8;
- } while (h > 0);
-#else // !AOM_ARCH_AARCH64
- // This shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use a single
- // rounding right shift by FILTER_BITS - instead of a first rounding right
- // shift by ROUND0_BITS, followed by second rounding right shift by
- // FILTER_BITS - ROUND0_BITS.
- // The outermost -1 is needed because we halved the filter values.
- const int16x4_t horiz_const = vdup_n_s16(1 << ((ROUND0_BITS - 1) - 1));
- int16x8_t tt0;
- int16x4_t x0, x1, x2, x3, x4, x5, x6, x7;
-
- do {
- t0 = vld1_u8(src); // a0 a1 a2 a3 a4 a5 a6 a7
- tt0 = vreinterpretq_s16_u16(vmovl_u8(t0));
- x0 = vget_low_s16(tt0); // a0 a1 a2 a3
- x4 = vget_high_s16(tt0); // a4 a5 a6 a7
-
- t0 = vld1_u8(src + 8); // a8 a9 a10 a11 a12 a13 a14 a15
- tt0 = vreinterpretq_s16_u16(vmovl_u8(t0));
- x7 = vget_low_s16(tt0); // a8 a9 a10 a11
-
- x1 = vext_s16(x0, x4, 1); // a1 a2 a3 a4
- x2 = vext_s16(x0, x4, 2); // a2 a3 a4 a5
- x3 = vext_s16(x0, x4, 3); // a3 a4 a5 a6
- x5 = vext_s16(x4, x7, 1); // a5 a6 a7 a8
- x6 = vext_s16(x4, x7, 2); // a6 a7 a8 a9
- x7 = vext_s16(x4, x7, 3); // a7 a8 a9 a10
+ store_u8_4x1(dst, d0, 0);
src += src_stride;
+ dst += dst_stride;
+ } while (--h != 0);
+ } else {
+ // Filter values are even so halve to reduce precision requirements.
+ const int16x8_t x_filter = vshrq_n_s16(vld1q_s16(x_filter_ptr), 1);
- t0 = convolve8_x_4x1(x0, x1, x2, x3, x4, x5, x6, x7, x_filter,
- horiz_const);
-
- if (w == 4) {
- store_u8_4x1(dst, t0, 0);
- dst += dst_stride;
- } else if (w == 2) {
- store_u8_2x1(dst, t0, 0);
- dst += dst_stride;
- }
- h -= 1;
- } while (h > 0);
-#endif // AOM_ARCH_AARCH64
- } else {
- uint8_t *d;
- int16x8_t s11;
#if AOM_ARCH_AARCH64
- int16x8_t s12, s13, s14;
- do {
- __builtin_prefetch(src + 0 * src_stride);
- __builtin_prefetch(src + 1 * src_stride);
- __builtin_prefetch(src + 2 * src_stride);
- __builtin_prefetch(src + 3 * src_stride);
- __builtin_prefetch(src + 4 * src_stride);
- __builtin_prefetch(src + 5 * src_stride);
- __builtin_prefetch(src + 6 * src_stride);
- __builtin_prefetch(src + 7 * src_stride);
- load_u8_8x8(src, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
- transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
- s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
- s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
- s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
- s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
- s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
- s5 = vreinterpretq_s16_u16(vmovl_u8(t5));
- s6 = vreinterpretq_s16_u16(vmovl_u8(t6));
+ while (h >= 8) {
+ uint8x8_t t0, t1, t2, t3, t4, t5, t6, t7;
+ load_u8_8x8(src, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
- width = w;
- s = src + 7;
- d = dst;
- __builtin_prefetch(dst + 0 * dst_stride);
- __builtin_prefetch(dst + 1 * dst_stride);
- __builtin_prefetch(dst + 2 * dst_stride);
- __builtin_prefetch(dst + 3 * dst_stride);
- __builtin_prefetch(dst + 4 * dst_stride);
- __builtin_prefetch(dst + 5 * dst_stride);
- __builtin_prefetch(dst + 6 * dst_stride);
- __builtin_prefetch(dst + 7 * dst_stride);
+ transpose_elems_inplace_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
+ int16x8_t s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
+ int16x8_t s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
+ int16x8_t s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
+ int16x8_t s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
+ int16x8_t s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
+ int16x8_t s5 = vreinterpretq_s16_u16(vmovl_u8(t5));
+ int16x8_t s6 = vreinterpretq_s16_u16(vmovl_u8(t6));
- do {
- load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
- transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
- s7 = vreinterpretq_s16_u16(vmovl_u8(t0));
- s8 = vreinterpretq_s16_u16(vmovl_u8(t1));
- s9 = vreinterpretq_s16_u16(vmovl_u8(t2));
- s10 = vreinterpretq_s16_u16(vmovl_u8(t3));
- s11 = vreinterpretq_s16_u16(vmovl_u8(t4));
- s12 = vreinterpretq_s16_u16(vmovl_u8(t5));
- s13 = vreinterpretq_s16_u16(vmovl_u8(t6));
- s14 = vreinterpretq_s16_u16(vmovl_u8(t7));
+ int width = w;
+ const uint8_t *s = src + 7;
+ uint8_t *d = dst;
- t0 = convolve8_horiz_8x8(s0, s1, s2, s3, s4, s5, s6, s7, x_filter,
- horiz_const);
- t1 = convolve8_horiz_8x8(s1, s2, s3, s4, s5, s6, s7, s8, x_filter,
- horiz_const);
- t2 = convolve8_horiz_8x8(s2, s3, s4, s5, s6, s7, s8, s9, x_filter,
- horiz_const);
- t3 = convolve8_horiz_8x8(s3, s4, s5, s6, s7, s8, s9, s10, x_filter,
- horiz_const);
- t4 = convolve8_horiz_8x8(s4, s5, s6, s7, s8, s9, s10, s11, x_filter,
- horiz_const);
- t5 = convolve8_horiz_8x8(s5, s6, s7, s8, s9, s10, s11, s12, x_filter,
- horiz_const);
- t6 = convolve8_horiz_8x8(s6, s7, s8, s9, s10, s11, s12, s13, x_filter,
- horiz_const);
- t7 = convolve8_horiz_8x8(s7, s8, s9, s10, s11, s12, s13, s14,
- x_filter, horiz_const);
-
- transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
-
- if (h != 2) {
- store_u8_8x8(d, dst_stride, t0, t1, t2, t3, t4, t5, t6, t7);
- } else {
- store_u8_8x2(d, dst_stride, t0, t1);
- }
-
- s0 = s8;
- s1 = s9;
- s2 = s10;
- s3 = s11;
- s4 = s12;
- s5 = s13;
- s6 = s14;
- s += 8;
- d += 8;
- width -= 8;
- } while (width > 0);
- src += 8 * src_stride;
- dst += 8 * dst_stride;
- h -= 8;
- } while (h > 0);
-#else // !AOM_ARCH_AARCH64
- // This shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use a single
- // rounding right shift by FILTER_BITS - instead of a first rounding right
- // shift by ROUND0_BITS, followed by second rounding right shift by
- // FILTER_BITS - ROUND0_BITS.
- // The outermost -1 is needed because we halved the filter values.
- const int16x8_t horiz_const = vdupq_n_s16(1 << ((ROUND0_BITS - 1) - 1));
-
- do {
- t0 = vld1_u8(src); // a0 a1 a2 a3 a4 a5 a6 a7
- s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
-
- width = w;
- s = src + 8;
- d = dst;
- __builtin_prefetch(dst);
+ __builtin_prefetch(d + 0 * dst_stride);
+ __builtin_prefetch(d + 1 * dst_stride);
+ __builtin_prefetch(d + 2 * dst_stride);
+ __builtin_prefetch(d + 3 * dst_stride);
+ __builtin_prefetch(d + 4 * dst_stride);
+ __builtin_prefetch(d + 5 * dst_stride);
+ __builtin_prefetch(d + 6 * dst_stride);
+ __builtin_prefetch(d + 7 * dst_stride);
do {
- t0 = vld1_u8(s); // a8 a9 a10 a11 a12 a13 a14 a15
- s7 = vreinterpretq_s16_u16(vmovl_u8(t0));
- s11 = s0;
- s0 = s7;
+ uint8x8_t t8, t9, t10, t11, t12, t13, t14;
+ load_u8_8x8(s, src_stride, &t7, &t8, &t9, &t10, &t11, &t12, &t13, &t14);
- s1 = vextq_s16(s11, s7, 1); // a1 a2 a3 a4 a5 a6 a7 a8
- s2 = vextq_s16(s11, s7, 2); // a2 a3 a4 a5 a6 a7 a8 a9
- s3 = vextq_s16(s11, s7, 3); // a3 a4 a5 a6 a7 a8 a9 a10
- s4 = vextq_s16(s11, s7, 4); // a4 a5 a6 a7 a8 a9 a10 a11
- s5 = vextq_s16(s11, s7, 5); // a5 a6 a7 a8 a9 a10 a11 a12
- s6 = vextq_s16(s11, s7, 6); // a6 a7 a8 a9 a10 a11 a12 a13
- s7 = vextq_s16(s11, s7, 7); // a7 a8 a9 a10 a11 a12 a13 a14
+ transpose_elems_inplace_u8_8x8(&t7, &t8, &t9, &t10, &t11, &t12, &t13,
+ &t14);
+ int16x8_t s7 = vreinterpretq_s16_u16(vmovl_u8(t7));
+ int16x8_t s8 = vreinterpretq_s16_u16(vmovl_u8(t8));
+ int16x8_t s9 = vreinterpretq_s16_u16(vmovl_u8(t9));
+ int16x8_t s10 = vreinterpretq_s16_u16(vmovl_u8(t10));
+ int16x8_t s11 = vreinterpretq_s16_u16(vmovl_u8(t11));
+ int16x8_t s12 = vreinterpretq_s16_u16(vmovl_u8(t12));
+ int16x8_t s13 = vreinterpretq_s16_u16(vmovl_u8(t13));
+ int16x8_t s14 = vreinterpretq_s16_u16(vmovl_u8(t14));
- t0 = convolve8_horiz_8x8(s11, s1, s2, s3, s4, s5, s6, s7, x_filter,
- horiz_const);
+ uint8x8_t d0 = convolve8_8_x(s0, s1, s2, s3, s4, s5, s6, s7, x_filter,
+ horiz_const);
+ uint8x8_t d1 = convolve8_8_x(s1, s2, s3, s4, s5, s6, s7, s8, x_filter,
+ horiz_const);
+ uint8x8_t d2 = convolve8_8_x(s2, s3, s4, s5, s6, s7, s8, s9, x_filter,
+ horiz_const);
+ uint8x8_t d3 = convolve8_8_x(s3, s4, s5, s6, s7, s8, s9, s10, x_filter,
+ horiz_const);
+ uint8x8_t d4 = convolve8_8_x(s4, s5, s6, s7, s8, s9, s10, s11, x_filter,
+ horiz_const);
+ uint8x8_t d5 = convolve8_8_x(s5, s6, s7, s8, s9, s10, s11, s12,
+ x_filter, horiz_const);
+ uint8x8_t d6 = convolve8_8_x(s6, s7, s8, s9, s10, s11, s12, s13,
+ x_filter, horiz_const);
+ uint8x8_t d7 = convolve8_8_x(s7, s8, s9, s10, s11, s12, s13, s14,
+ x_filter, horiz_const);
- vst1_u8(d, t0);
+ transpose_elems_inplace_u8_8x8(&d0, &d1, &d2, &d3, &d4, &d5, &d6, &d7);
+ store_u8_8x8(d, dst_stride, d0, d1, d2, d3, d4, d5, d6, d7);
+
+ s0 = s8;
+ s1 = s9;
+ s2 = s10;
+ s3 = s11;
+ s4 = s12;
+ s5 = s13;
+ s6 = s14;
s += 8;
d += 8;
width -= 8;
- } while (width > 0);
+ } while (width != 0);
+ src += 8 * src_stride;
+ dst += 8 * dst_stride;
+ h -= 8;
+ }
+#endif // AOM_ARCH_AARCH64
+
+ while (h-- != 0) {
+ uint8x8_t t0 = vld1_u8(src); // a0 a1 a2 a3 a4 a5 a6 a7
+ int16x8_t s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
+
+ int width = w;
+ const uint8_t *s = src + 8;
+ uint8_t *d = dst;
+
+ __builtin_prefetch(d);
+
+ do {
+ uint8x8_t t8 = vld1_u8(s); // a8 a9 a10 a11 a12 a13 a14 a15
+ int16x8_t s8 = vreinterpretq_s16_u16(vmovl_u8(t8));
+
+ int16x8_t s1 = vextq_s16(s0, s8, 1); // a1 a2 a3 a4 a5 a6 a7 a8
+ int16x8_t s2 = vextq_s16(s0, s8, 2); // a2 a3 a4 a5 a6 a7 a8 a9
+ int16x8_t s3 = vextq_s16(s0, s8, 3); // a3 a4 a5 a6 a7 a8 a9 a10
+ int16x8_t s4 = vextq_s16(s0, s8, 4); // a4 a5 a6 a7 a8 a9 a10 a11
+ int16x8_t s5 = vextq_s16(s0, s8, 5); // a5 a6 a7 a8 a9 a10 a11 a12
+ int16x8_t s6 = vextq_s16(s0, s8, 6); // a6 a7 a8 a9 a10 a11 a12 a13
+ int16x8_t s7 = vextq_s16(s0, s8, 7); // a7 a8 a9 a10 a11 a12 a13 a14
+
+ uint8x8_t d0 = convolve8_8_x(s0, s1, s2, s3, s4, s5, s6, s7, x_filter,
+ horiz_const);
+
+ vst1_u8(d, d0);
+
+ s0 = s8;
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
src += src_stride;
dst += dst_stride;
- h -= 1;
- } while (h > 0);
-#endif // AOM_ARCH_AARCH64
}
-#if AOM_ARCH_AARCH64
}
-#endif // AOM_ARCH_AARCH64
}
-#endif // AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_MATMUL_INT8)
+static INLINE int16x4_t convolve6_4_y(const int16x4_t s0, const int16x4_t s1,
+ const int16x4_t s2, const int16x4_t s3,
+ const int16x4_t s4, const int16x4_t s5,
+ const int16x8_t y_filter_0_7) {
+ const int16x4_t y_filter_0_3 = vget_low_s16(y_filter_0_7);
+ const int16x4_t y_filter_4_7 = vget_high_s16(y_filter_0_7);
+
+ // Filter values at indices 0 and 7 are 0.
+ int16x4_t sum = vmul_lane_s16(s0, y_filter_0_3, 1);
+ sum = vmla_lane_s16(sum, s1, y_filter_0_3, 2);
+ sum = vmla_lane_s16(sum, s2, y_filter_0_3, 3);
+ sum = vmla_lane_s16(sum, s3, y_filter_4_7, 0);
+ sum = vmla_lane_s16(sum, s4, y_filter_4_7, 1);
+ sum = vmla_lane_s16(sum, s5, y_filter_4_7, 2);
+
+ return sum;
+}
+
+static INLINE uint8x8_t convolve6_8_y(const int16x8_t s0, const int16x8_t s1,
+ const int16x8_t s2, const int16x8_t s3,
+ const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t y_filters) {
+ const int16x4_t y_filter_lo = vget_low_s16(y_filters);
+ const int16x4_t y_filter_hi = vget_high_s16(y_filters);
+
+ // Filter values at indices 0 and 7 are 0.
+ int16x8_t sum = vmulq_lane_s16(s0, y_filter_lo, 1);
+ sum = vmlaq_lane_s16(sum, s1, y_filter_lo, 2);
+ sum = vmlaq_lane_s16(sum, s2, y_filter_lo, 3);
+ sum = vmlaq_lane_s16(sum, s3, y_filter_hi, 0);
+ sum = vmlaq_lane_s16(sum, s4, y_filter_hi, 1);
+ sum = vmlaq_lane_s16(sum, s5, y_filter_hi, 2);
+ // We halved the convolution filter values so -1 from the right shift.
+ return vqrshrun_n_s16(sum, FILTER_BITS - 1);
+}
static INLINE void convolve_y_sr_6tap_neon(const uint8_t *src_ptr,
int src_stride, uint8_t *dst_ptr,
const int dst_stride, int w, int h,
- const int16x8_t y_filter_0_7) {
+ const int16x8_t y_filter) {
if (w <= 4) {
- uint8x8_t t0, t1, t2, t3, t4, t5;
- int16x4_t s0, s1, s2, s3, s4, s5, d0;
- uint8x8_t d01;
+ uint8x8_t t0 = load_unaligned_u8_4x1(src_ptr + 0 * src_stride);
+ uint8x8_t t1 = load_unaligned_u8_4x1(src_ptr + 1 * src_stride);
+ uint8x8_t t2 = load_unaligned_u8_4x1(src_ptr + 2 * src_stride);
+ uint8x8_t t3 = load_unaligned_u8_4x1(src_ptr + 3 * src_stride);
+ uint8x8_t t4 = load_unaligned_u8_4x1(src_ptr + 4 * src_stride);
-#if AOM_ARCH_AARCH64
- uint8x8_t t6, t7, t8;
- int16x4_t s6, s7, s8, d1, d2, d3;
- uint8x8_t d23;
-#endif // AOM_ARCH_AARCH64
+ int16x4_t s0 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
+ int16x4_t s1 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
+ int16x4_t s2 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
+ int16x4_t s3 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t3)));
+ int16x4_t s4 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t4)));
- const uint8_t *s = src_ptr + src_stride;
- uint8_t *d = dst_ptr;
-
- load_u8_8x5(s, src_stride, &t0, &t1, &t2, &t3, &t4);
- s0 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s1 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
- s2 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
- s3 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t3)));
- s4 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t4)));
- s += 5 * src_stride;
+ src_ptr += 5 * src_stride;
do {
#if AOM_ARCH_AARCH64
- load_u8_8x4(s, src_stride, &t5, &t6, &t7, &t8);
- s5 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t5)));
- s6 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t6)));
- s7 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t7)));
- s8 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t8)));
+ uint8x8_t t5 = load_unaligned_u8_4x1(src_ptr + 0 * src_stride);
+ uint8x8_t t6 = load_unaligned_u8_4x1(src_ptr + 1 * src_stride);
+ uint8x8_t t7 = load_unaligned_u8_4x1(src_ptr + 2 * src_stride);
+ uint8x8_t t8 = load_unaligned_u8_4x1(src_ptr + 3 * src_stride);
- d0 = convolve6_4x4(s0, s1, s2, s3, s4, s5, y_filter_0_7);
- d1 = convolve6_4x4(s1, s2, s3, s4, s5, s6, y_filter_0_7);
- d2 = convolve6_4x4(s2, s3, s4, s5, s6, s7, y_filter_0_7);
- d3 = convolve6_4x4(s3, s4, s5, s6, s7, s8, y_filter_0_7);
+ int16x4_t s5 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t5)));
+ int16x4_t s6 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t6)));
+ int16x4_t s7 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t7)));
+ int16x4_t s8 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t8)));
- d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS - 1);
- d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS - 1);
+ int16x4_t d0 = convolve6_4_y(s0, s1, s2, s3, s4, s5, y_filter);
+ int16x4_t d1 = convolve6_4_y(s1, s2, s3, s4, s5, s6, y_filter);
+ int16x4_t d2 = convolve6_4_y(s2, s3, s4, s5, s6, s7, y_filter);
+ int16x4_t d3 = convolve6_4_y(s3, s4, s5, s6, s7, s8, y_filter);
- if (w == 2) {
- store_u8_2x1(d + 0 * dst_stride, d01, 0);
- store_u8_2x1(d + 1 * dst_stride, d01, 2);
- if (h != 2) {
- store_u8_2x1(d + 2 * dst_stride, d23, 0);
- store_u8_2x1(d + 3 * dst_stride, d23, 2);
- }
- } else {
- store_u8_4x1(d + 0 * dst_stride, d01, 0);
- store_u8_4x1(d + 1 * dst_stride, d01, 1);
- if (h != 2) {
- store_u8_4x1(d + 2 * dst_stride, d23, 0);
- store_u8_4x1(d + 3 * dst_stride, d23, 1);
- }
- }
+ // We halved the convolution filter values so -1 from the right shift.
+ uint8x8_t d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS - 1);
+ uint8x8_t d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS - 1);
+
+ store_u8_4x1(dst_ptr + 0 * dst_stride, d01, 0);
+ store_u8_4x1(dst_ptr + 1 * dst_stride, d01, 1);
+ store_u8_4x1(dst_ptr + 2 * dst_stride, d23, 0);
+ store_u8_4x1(dst_ptr + 3 * dst_stride, d23, 1);
s0 = s4;
s1 = s5;
s2 = s6;
s3 = s7;
s4 = s8;
- s += 4 * src_stride;
- d += 4 * dst_stride;
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
h -= 4;
#else // !AOM_ARCH_AARCH64
- t5 = vld1_u8(s);
- s5 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t5)));
+ uint8x8_t t5 = load_unaligned_u8_4x1(src_ptr);
+ int16x4_t s5 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t5)));
- d0 = convolve6_4x4(s0, s1, s2, s3, s4, s5, y_filter_0_7);
- d01 = vqrshrun_n_s16(vcombine_s16(d0, vdup_n_s16(0)), FILTER_BITS - 1);
+ int16x4_t d0 = convolve6_4_y(s0, s1, s2, s3, s4, s5, y_filter);
+ // We halved the convolution filter values so -1 from the right shift.
+ uint8x8_t d01 =
+ vqrshrun_n_s16(vcombine_s16(d0, vdup_n_s16(0)), FILTER_BITS - 1);
- if (w == 2) {
- store_u8_2x1(d, d01, 0);
- } else {
- store_u8_4x1(d, d01, 0);
- }
+ store_u8_4x1(dst_ptr, d01, 0);
s0 = s1;
s1 = s2;
s2 = s3;
s3 = s4;
s4 = s5;
- s += src_stride;
- d += dst_stride;
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
h--;
#endif // AOM_ARCH_AARCH64
- } while (h > 0);
+ } while (h != 0);
+
} else {
- // if width is a multiple of 8 & height is a multiple of 4
- uint8x8_t t0, t1, t2, t3, t4, t5;
- int16x8_t s0, s1, s2, s3, s4, s5, dd0;
- uint8x8_t d0;
-#if AOM_ARCH_AARCH64
- uint8x8_t t6, t7, t8;
- int16x8_t s6, s7, s8, dd1, dd2, dd3;
- uint8x8_t d1, d2, d3;
-#endif // AOM_ARCH_AARCH64
-
do {
- int height = h;
- const uint8_t *s = src_ptr + src_stride;
+ const uint8_t *s = src_ptr;
uint8_t *d = dst_ptr;
+ int height = h;
+ uint8x8_t t0, t1, t2, t3, t4;
load_u8_8x5(s, src_stride, &t0, &t1, &t2, &t3, &t4);
- s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
- s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
- s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
- s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
- s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
+
+ int16x8_t s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
+ int16x8_t s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
+ int16x8_t s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
+ int16x8_t s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
+ int16x8_t s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
+
s += 5 * src_stride;
do {
#if AOM_ARCH_AARCH64
+ uint8x8_t t5, t6, t7, t8;
load_u8_8x4(s, src_stride, &t5, &t6, &t7, &t8);
- s5 = vreinterpretq_s16_u16(vmovl_u8(t5));
- s6 = vreinterpretq_s16_u16(vmovl_u8(t6));
- s7 = vreinterpretq_s16_u16(vmovl_u8(t7));
- s8 = vreinterpretq_s16_u16(vmovl_u8(t8));
- dd0 = convolve6_8x4(s0, s1, s2, s3, s4, s5, y_filter_0_7);
- dd1 = convolve6_8x4(s1, s2, s3, s4, s5, s6, y_filter_0_7);
- dd2 = convolve6_8x4(s2, s3, s4, s5, s6, s7, y_filter_0_7);
- dd3 = convolve6_8x4(s3, s4, s5, s6, s7, s8, y_filter_0_7);
+ int16x8_t s5 = vreinterpretq_s16_u16(vmovl_u8(t5));
+ int16x8_t s6 = vreinterpretq_s16_u16(vmovl_u8(t6));
+ int16x8_t s7 = vreinterpretq_s16_u16(vmovl_u8(t7));
+ int16x8_t s8 = vreinterpretq_s16_u16(vmovl_u8(t8));
- d0 = vqrshrun_n_s16(dd0, FILTER_BITS - 1);
- d1 = vqrshrun_n_s16(dd1, FILTER_BITS - 1);
- d2 = vqrshrun_n_s16(dd2, FILTER_BITS - 1);
- d3 = vqrshrun_n_s16(dd3, FILTER_BITS - 1);
+ uint8x8_t d0 = convolve6_8_y(s0, s1, s2, s3, s4, s5, y_filter);
+ uint8x8_t d1 = convolve6_8_y(s1, s2, s3, s4, s5, s6, y_filter);
+ uint8x8_t d2 = convolve6_8_y(s2, s3, s4, s5, s6, s7, y_filter);
+ uint8x8_t d3 = convolve6_8_y(s3, s4, s5, s6, s7, s8, y_filter);
- if (h != 2) {
- store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
- } else {
- store_u8_8x2(d, dst_stride, d0, d1);
- }
+ store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
s0 = s4;
s1 = s5;
@@ -1590,11 +557,9 @@
d += 4 * dst_stride;
height -= 4;
#else // !AOM_ARCH_AARCH64
- t5 = vld1_u8(s);
- s5 = vreinterpretq_s16_u16(vmovl_u8(t5));
+ int16x8_t s5 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
- dd0 = convolve6_8x4(s0, s1, s2, s3, s4, s5, y_filter_0_7);
- d0 = vqrshrun_n_s16(dd0, FILTER_BITS - 1);
+ uint8x8_t d0 = convolve6_8_y(s0, s1, s2, s3, s4, s5, y_filter);
vst1_u8(d, d0);
@@ -1607,21 +572,217 @@
d += dst_stride;
height--;
#endif // AOM_ARCH_AARCH64
- } while (height > 0);
-
+ } while (height != 0);
src_ptr += 8;
dst_ptr += 8;
w -= 8;
- } while (w > 0);
+ } while (w != 0);
}
}
-static INLINE int16x4_t convolve12_y_4x4_s32(
- const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
- const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
- const int16x4_t s6, const int16x4_t s7, const int16x4_t s8,
- const int16x4_t s9, const int16x4_t s10, const int16x4_t s11,
- const int16x8_t y_filter_0_7, const int16x4_t y_filter_8_11) {
+static INLINE int16x4_t convolve8_4_y(const int16x4_t s0, const int16x4_t s1,
+ const int16x4_t s2, const int16x4_t s3,
+ const int16x4_t s4, const int16x4_t s5,
+ const int16x4_t s6, const int16x4_t s7,
+ const int16x8_t filter) {
+ const int16x4_t filter_lo = vget_low_s16(filter);
+ const int16x4_t filter_hi = vget_high_s16(filter);
+
+ int16x4_t sum = vmul_lane_s16(s0, filter_lo, 0);
+ sum = vmla_lane_s16(sum, s1, filter_lo, 1);
+ sum = vmla_lane_s16(sum, s2, filter_lo, 2);
+ sum = vmla_lane_s16(sum, s3, filter_lo, 3);
+ sum = vmla_lane_s16(sum, s4, filter_hi, 0);
+ sum = vmla_lane_s16(sum, s5, filter_hi, 1);
+ sum = vmla_lane_s16(sum, s6, filter_hi, 2);
+ sum = vmla_lane_s16(sum, s7, filter_hi, 3);
+
+ return sum;
+}
+
+static INLINE uint8x8_t convolve8_8_y(const int16x8_t s0, const int16x8_t s1,
+ const int16x8_t s2, const int16x8_t s3,
+ const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t s6, const int16x8_t s7,
+ const int16x8_t filter) {
+ const int16x4_t filter_lo = vget_low_s16(filter);
+ const int16x4_t filter_hi = vget_high_s16(filter);
+
+ int16x8_t sum = vmulq_lane_s16(s0, filter_lo, 0);
+ sum = vmlaq_lane_s16(sum, s1, filter_lo, 1);
+ sum = vmlaq_lane_s16(sum, s2, filter_lo, 2);
+ sum = vmlaq_lane_s16(sum, s3, filter_lo, 3);
+ sum = vmlaq_lane_s16(sum, s4, filter_hi, 0);
+ sum = vmlaq_lane_s16(sum, s5, filter_hi, 1);
+ sum = vmlaq_lane_s16(sum, s6, filter_hi, 2);
+ sum = vmlaq_lane_s16(sum, s7, filter_hi, 3);
+
+ // We halved the convolution filter values so -1 from the right shift.
+ return vqrshrun_n_s16(sum, FILTER_BITS - 1);
+}
+
+static INLINE void convolve_y_sr_8tap_neon(const uint8_t *src_ptr,
+ int src_stride, uint8_t *dst_ptr,
+ const int dst_stride, int w, int h,
+ const int16x8_t y_filter) {
+ if (w <= 4) {
+ uint8x8_t t0 = load_unaligned_u8_4x1(src_ptr + 0 * src_stride);
+ uint8x8_t t1 = load_unaligned_u8_4x1(src_ptr + 1 * src_stride);
+ uint8x8_t t2 = load_unaligned_u8_4x1(src_ptr + 2 * src_stride);
+ uint8x8_t t3 = load_unaligned_u8_4x1(src_ptr + 3 * src_stride);
+ uint8x8_t t4 = load_unaligned_u8_4x1(src_ptr + 4 * src_stride);
+ uint8x8_t t5 = load_unaligned_u8_4x1(src_ptr + 5 * src_stride);
+ uint8x8_t t6 = load_unaligned_u8_4x1(src_ptr + 6 * src_stride);
+
+ int16x4_t s0 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
+ int16x4_t s1 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t1)));
+ int16x4_t s2 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t2)));
+ int16x4_t s3 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t3)));
+ int16x4_t s4 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t4)));
+ int16x4_t s5 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t5)));
+ int16x4_t s6 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t6)));
+
+ src_ptr += 7 * src_stride;
+
+ do {
+#if AOM_ARCH_AARCH64
+ uint8x8_t t7 = load_unaligned_u8_4x1(src_ptr + 0 * src_stride);
+ uint8x8_t t8 = load_unaligned_u8_4x1(src_ptr + 1 * src_stride);
+ uint8x8_t t9 = load_unaligned_u8_4x1(src_ptr + 2 * src_stride);
+ uint8x8_t t10 = load_unaligned_u8_4x1(src_ptr + 3 * src_stride);
+
+ int16x4_t s7 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t7)));
+ int16x4_t s8 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t8)));
+ int16x4_t s9 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t9)));
+ int16x4_t s10 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t10)));
+
+ int16x4_t d0 = convolve8_4_y(s0, s1, s2, s3, s4, s5, s6, s7, y_filter);
+ int16x4_t d1 = convolve8_4_y(s1, s2, s3, s4, s5, s6, s7, s8, y_filter);
+ int16x4_t d2 = convolve8_4_y(s2, s3, s4, s5, s6, s7, s8, s9, y_filter);
+ int16x4_t d3 = convolve8_4_y(s3, s4, s5, s6, s7, s8, s9, s10, y_filter);
+
+ // We halved the convolution filter values so -1 from the right shift.
+ uint8x8_t d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS - 1);
+ uint8x8_t d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS - 1);
+
+ store_u8_4x1(dst_ptr + 0 * dst_stride, d01, 0);
+ store_u8_4x1(dst_ptr + 1 * dst_stride, d01, 1);
+ store_u8_4x1(dst_ptr + 2 * dst_stride, d23, 0);
+ store_u8_4x1(dst_ptr + 3 * dst_stride, d23, 1);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s5 = s9;
+ s6 = s10;
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ h -= 4;
+#else // !AOM_ARCH_AARCH64
+ uint8x8_t t7 = load_unaligned_u8_4x1(src_ptr);
+ int16x4_t s7 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t7)));
+
+ int16x4_t d0 = convolve8_4_y(s0, s1, s2, s3, s4, s5, s6, s7, y_filter);
+ // We halved the convolution filter values so -1 from the right shift.
+ uint8x8_t d01 =
+ vqrshrun_n_s16(vcombine_s16(d0, vdup_n_s16(0)), FILTER_BITS - 1);
+
+ store_u8_4x1(dst_ptr, d01, 0);
+
+ s0 = s1;
+ s1 = s2;
+ s2 = s3;
+ s3 = s4;
+ s4 = s5;
+ s5 = s6;
+ s6 = s7;
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ h--;
+#endif // AOM_ARCH_AARCH64
+ } while (h != 0);
+ } else {
+ do {
+ const uint8_t *s = src_ptr;
+ uint8_t *d = dst_ptr;
+ int height = h;
+
+ uint8x8_t t0, t1, t2, t3, t4, t5, t6;
+ load_u8_8x7(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6);
+
+ int16x8_t s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
+ int16x8_t s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
+ int16x8_t s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
+ int16x8_t s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
+ int16x8_t s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
+ int16x8_t s5 = vreinterpretq_s16_u16(vmovl_u8(t5));
+ int16x8_t s6 = vreinterpretq_s16_u16(vmovl_u8(t6));
+
+ s += 7 * src_stride;
+
+ do {
+#if AOM_ARCH_AARCH64
+ uint8x8_t t7, t8, t9, t10;
+ load_u8_8x4(s, src_stride, &t7, &t8, &t9, &t10);
+
+ int16x8_t s7 = vreinterpretq_s16_u16(vmovl_u8(t7));
+ int16x8_t s8 = vreinterpretq_s16_u16(vmovl_u8(t8));
+ int16x8_t s9 = vreinterpretq_s16_u16(vmovl_u8(t9));
+ int16x8_t s10 = vreinterpretq_s16_u16(vmovl_u8(t10));
+
+ uint8x8_t d0 = convolve8_8_y(s0, s1, s2, s3, s4, s5, s6, s7, y_filter);
+ uint8x8_t d1 = convolve8_8_y(s1, s2, s3, s4, s5, s6, s7, s8, y_filter);
+ uint8x8_t d2 = convolve8_8_y(s2, s3, s4, s5, s6, s7, s8, s9, y_filter);
+ uint8x8_t d3 = convolve8_8_y(s3, s4, s5, s6, s7, s8, s9, s10, y_filter);
+
+ store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s5 = s9;
+ s6 = s10;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ height -= 4;
+#else // !AOM_ARCH_AARCH64
+ int16x8_t s7 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
+
+ uint8x8_t d0 = convolve8_8_y(s0, s1, s2, s3, s4, s5, s6, s7, y_filter);
+
+ vst1_u8(d, d0);
+
+ s0 = s1;
+ s1 = s2;
+ s2 = s3;
+ s3 = s4;
+ s4 = s5;
+ s5 = s6;
+ s6 = s7;
+ s += src_stride;
+ d += dst_stride;
+ height--;
+#endif // AOM_ARCH_AARCH64
+ } while (height != 0);
+ src_ptr += 8;
+ dst_ptr += 8;
+ w -= 8;
+ } while (w != 0);
+ }
+}
+
+static INLINE int16x4_t convolve12_4_y(const int16x4_t s0, const int16x4_t s1,
+ const int16x4_t s2, const int16x4_t s3,
+ const int16x4_t s4, const int16x4_t s5,
+ const int16x4_t s6, const int16x4_t s7,
+ const int16x4_t s8, const int16x4_t s9,
+ const int16x4_t s10, const int16x4_t s11,
+ const int16x8_t y_filter_0_7,
+ const int16x4_t y_filter_8_11) {
const int16x4_t y_filter_0_3 = vget_low_s16(y_filter_0_7);
const int16x4_t y_filter_4_7 = vget_high_s16(y_filter_0_7);
int16x4_t sum;
@@ -1638,22 +799,22 @@
sum = vmla_lane_s16(sum, s10, y_filter_8_11, 2);
sum = vmla_lane_s16(sum, s11, y_filter_8_11, 3);
- // Separate out the two filter values in the middle of the kernel that have
- // the largest magnitude and use saturating addition to prevent overflow. This
- // means we can stay at 16-bit elements, rather than having to widen
- // everything to a 32-bit result, requiring twice the number of instructions.
+ // Saturating addition is required for the largest filter taps to avoid
+ // overflow (while staying in 16-bit elements.)
sum = vqadd_s16(sum, vmul_lane_s16(s5, y_filter_4_7, 1));
sum = vqadd_s16(sum, vmul_lane_s16(s6, y_filter_4_7, 2));
return sum;
}
-static INLINE uint8x8_t convolve12_y_8x4_s32(
- const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
- const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
- const int16x8_t s6, const int16x8_t s7, const int16x8_t s8,
- const int16x8_t s9, const int16x8_t s10, const int16x8_t s11,
- const int16x8_t y_filter_0_7, const int16x4_t y_filter_8_11) {
+static INLINE uint8x8_t convolve12_8_y(const int16x8_t s0, const int16x8_t s1,
+ const int16x8_t s2, const int16x8_t s3,
+ const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t s6, const int16x8_t s7,
+ const int16x8_t s8, const int16x8_t s9,
+ const int16x8_t s10, const int16x8_t s11,
+ const int16x8_t y_filter_0_7,
+ const int16x4_t y_filter_8_11) {
const int16x4_t y_filter_0_3 = vget_low_s16(y_filter_0_7);
const int16x4_t y_filter_4_7 = vget_high_s16(y_filter_0_7);
int16x8_t sum;
@@ -1670,10 +831,8 @@
sum = vmlaq_lane_s16(sum, s10, y_filter_8_11, 2);
sum = vmlaq_lane_s16(sum, s11, y_filter_8_11, 3);
- // Separate out the two filter values in the middle of the kernel that have
- // the largest magnitude and use saturating addition to prevent overflow. This
- // means we can stay at 16-bit elements, rather than having to widen
- // everything to a 32-bit result, requiring twice the number of instructions.
+ // Saturating addition is required for the largest filter taps to avoid
+ // overflow (while staying in 16-bit elements.)
sum = vqaddq_s16(sum, vmulq_lane_s16(s5, y_filter_4_7, 1));
sum = vqaddq_s16(sum, vmulq_lane_s16(s6, y_filter_4_7, 2));
@@ -1684,98 +843,52 @@
int src_stride, uint8_t *dst_ptr,
int dst_stride, int w, int h,
const int16_t *y_filter_ptr) {
- // Special case the following no-op filter as 128 won't fit into the
- // 8-bit signed dot-product instruction:
- // { 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0 }
- if (y_filter_ptr[5] == 128) {
- // Undo the horizontal offset in the calling function
- src_ptr += 5 * src_stride;
-
- if (w <= 4) {
- for (int i = 0; i < h; i += 2) {
- uint8x8_t d0 = load_unaligned_u8(src_ptr + i * src_stride, src_stride);
- if (w == 2) {
- store_u8_2x1(dst_ptr + i * dst_stride, d0, 0);
- store_u8_2x1(dst_ptr + (i + 1) * dst_stride, d0, 1);
- } else if (w == 4) {
- store_u8_4x1(dst_ptr + i * dst_stride, d0, 0);
- store_u8_4x1(dst_ptr + (i + 1) * dst_stride, d0, 1);
- }
- }
- } else {
- for (int i = 0; i < h; i++) {
- for (int j = 0; j < w; j += 8) {
- uint8x8_t d0 = vld1_u8(src_ptr + i * src_stride + j);
- vst1_u8(dst_ptr + i * dst_stride + j, d0);
- }
- }
- }
- return;
- }
-
const int16x8_t y_filter_0_7 = vld1q_s16(y_filter_ptr);
const int16x4_t y_filter_8_11 = vld1_s16(y_filter_ptr + 8);
if (w <= 4) {
- uint8x8_t t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, t12, t13, t14;
- int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14;
- int16x4_t d0, d1, d2, d3;
- int16x8_t dd01, dd23;
- uint8x8_t d01, d23;
-
+ uint8x8_t t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10;
load_u8_8x11(src_ptr, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7,
&t8, &t9, &t10);
- s0 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s1 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
- s2 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
- s3 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t3)));
- s4 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t4)));
- s5 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t5)));
- s6 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t6)));
- s7 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t7)));
- s8 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t8)));
- s9 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t9)));
- s10 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t10)));
+ int16x4_t s0 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
+ int16x4_t s1 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
+ int16x4_t s2 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
+ int16x4_t s3 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t3)));
+ int16x4_t s4 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t4)));
+ int16x4_t s5 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t5)));
+ int16x4_t s6 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t6)));
+ int16x4_t s7 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t7)));
+ int16x4_t s8 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t8)));
+ int16x4_t s9 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t9)));
+ int16x4_t s10 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t10)));
src_ptr += 11 * src_stride;
do {
+ uint8x8_t t11, t12, t13, t14;
load_u8_8x4(src_ptr, src_stride, &t11, &t12, &t13, &t14);
- s11 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t11)));
- s12 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t12)));
- s13 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t13)));
- s14 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t14)));
- d0 = convolve12_y_4x4_s32(s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10,
- s11, y_filter_0_7, y_filter_8_11);
- d1 = convolve12_y_4x4_s32(s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11,
- s12, y_filter_0_7, y_filter_8_11);
- d2 = convolve12_y_4x4_s32(s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12,
- s13, y_filter_0_7, y_filter_8_11);
- d3 = convolve12_y_4x4_s32(s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13,
- s14, y_filter_0_7, y_filter_8_11);
+ int16x4_t s11 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t11)));
+ int16x4_t s12 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t12)));
+ int16x4_t s13 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t13)));
+ int16x4_t s14 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t14)));
- dd01 = vcombine_s16(d0, d1);
- dd23 = vcombine_s16(d2, d3);
+ int16x4_t d0 = convolve12_4_y(s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10,
+ s11, y_filter_0_7, y_filter_8_11);
+ int16x4_t d1 = convolve12_4_y(s1, s2, s3, s4, s5, s6, s7, s8, s9, s10,
+ s11, s12, y_filter_0_7, y_filter_8_11);
+ int16x4_t d2 = convolve12_4_y(s2, s3, s4, s5, s6, s7, s8, s9, s10, s11,
+ s12, s13, y_filter_0_7, y_filter_8_11);
+ int16x4_t d3 = convolve12_4_y(s3, s4, s5, s6, s7, s8, s9, s10, s11, s12,
+ s13, s14, y_filter_0_7, y_filter_8_11);
- d01 = vqrshrun_n_s16(dd01, FILTER_BITS);
- d23 = vqrshrun_n_s16(dd23, FILTER_BITS);
+ uint8x8_t d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS);
+ uint8x8_t d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS);
- if (w == 2) {
- store_u8_2x1(dst_ptr + 0 * dst_stride, d01, 0);
- store_u8_2x1(dst_ptr + 1 * dst_stride, d01, 2);
- if (h != 2) {
- store_u8_2x1(dst_ptr + 2 * dst_stride, d23, 0);
- store_u8_2x1(dst_ptr + 3 * dst_stride, d23, 2);
- }
- } else {
- store_u8_4x1(dst_ptr + 0 * dst_stride, d01, 0);
- store_u8_4x1(dst_ptr + 1 * dst_stride, d01, 1);
- if (h != 2) {
- store_u8_4x1(dst_ptr + 2 * dst_stride, d23, 0);
- store_u8_4x1(dst_ptr + 3 * dst_stride, d23, 1);
- }
- }
+ store_u8_4x1(dst_ptr + 0 * dst_stride, d01, 0);
+ store_u8_4x1(dst_ptr + 1 * dst_stride, d01, 1);
+ store_u8_4x1(dst_ptr + 2 * dst_stride, d23, 0);
+ store_u8_4x1(dst_ptr + 3 * dst_stride, d23, 1);
s0 = s4;
s1 = s5;
@@ -1791,54 +904,50 @@
src_ptr += 4 * src_stride;
dst_ptr += 4 * dst_stride;
h -= 4;
- } while (h > 0);
- } else {
- uint8x8_t t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, t12, t13, t14;
- uint8x8_t d0, d1, d2, d3;
- int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14;
+ } while (h != 0);
+ } else {
do {
const uint8_t *s = src_ptr;
uint8_t *d = dst_ptr;
int height = h;
+ uint8x8_t t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10;
load_u8_8x11(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7, &t8,
&t9, &t10);
- s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
- s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
- s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
- s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
- s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
- s5 = vreinterpretq_s16_u16(vmovl_u8(t5));
- s6 = vreinterpretq_s16_u16(vmovl_u8(t6));
- s7 = vreinterpretq_s16_u16(vmovl_u8(t7));
- s8 = vreinterpretq_s16_u16(vmovl_u8(t8));
- s9 = vreinterpretq_s16_u16(vmovl_u8(t9));
- s10 = vreinterpretq_s16_u16(vmovl_u8(t10));
+ int16x8_t s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
+ int16x8_t s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
+ int16x8_t s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
+ int16x8_t s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
+ int16x8_t s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
+ int16x8_t s5 = vreinterpretq_s16_u16(vmovl_u8(t5));
+ int16x8_t s6 = vreinterpretq_s16_u16(vmovl_u8(t6));
+ int16x8_t s7 = vreinterpretq_s16_u16(vmovl_u8(t7));
+ int16x8_t s8 = vreinterpretq_s16_u16(vmovl_u8(t8));
+ int16x8_t s9 = vreinterpretq_s16_u16(vmovl_u8(t9));
+ int16x8_t s10 = vreinterpretq_s16_u16(vmovl_u8(t10));
s += 11 * src_stride;
do {
+ uint8x8_t t11, t12, t13, t14;
load_u8_8x4(s, src_stride, &t11, &t12, &t13, &t14);
- s11 = vreinterpretq_s16_u16(vmovl_u8(t11));
- s12 = vreinterpretq_s16_u16(vmovl_u8(t12));
- s13 = vreinterpretq_s16_u16(vmovl_u8(t13));
- s14 = vreinterpretq_s16_u16(vmovl_u8(t14));
- d0 = convolve12_y_8x4_s32(s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10,
- s11, y_filter_0_7, y_filter_8_11);
- d1 = convolve12_y_8x4_s32(s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11,
- s12, y_filter_0_7, y_filter_8_11);
- d2 = convolve12_y_8x4_s32(s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12,
- s13, y_filter_0_7, y_filter_8_11);
- d3 = convolve12_y_8x4_s32(s3, s4, s5, s6, s7, s8, s9, s10, s11, s12,
- s13, s14, y_filter_0_7, y_filter_8_11);
+ int16x8_t s11 = vreinterpretq_s16_u16(vmovl_u8(t11));
+ int16x8_t s12 = vreinterpretq_s16_u16(vmovl_u8(t12));
+ int16x8_t s13 = vreinterpretq_s16_u16(vmovl_u8(t13));
+ int16x8_t s14 = vreinterpretq_s16_u16(vmovl_u8(t14));
- if (h != 2) {
- store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
- } else {
- store_u8_8x2(d, dst_stride, d0, d1);
- }
+ uint8x8_t d0 = convolve12_8_y(s0, s1, s2, s3, s4, s5, s6, s7, s8, s9,
+ s10, s11, y_filter_0_7, y_filter_8_11);
+ uint8x8_t d1 = convolve12_8_y(s1, s2, s3, s4, s5, s6, s7, s8, s9, s10,
+ s11, s12, y_filter_0_7, y_filter_8_11);
+ uint8x8_t d2 = convolve12_8_y(s2, s3, s4, s5, s6, s7, s8, s9, s10, s11,
+ s12, s13, y_filter_0_7, y_filter_8_11);
+ uint8x8_t d3 = convolve12_8_y(s3, s4, s5, s6, s7, s8, s9, s10, s11, s12,
+ s13, s14, y_filter_0_7, y_filter_8_11);
+
+ store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
s0 = s4;
s1 = s5;
@@ -1854,12 +963,11 @@
s += 4 * src_stride;
d += 4 * dst_stride;
height -= 4;
- } while (height > 0);
-
+ } while (height != 0);
src_ptr += 8;
dst_ptr += 8;
w -= 8;
- } while (w > 0);
+ } while (w != 0);
}
}
@@ -1867,8 +975,15 @@
int dst_stride, int w, int h,
const InterpFilterParams *filter_params_y,
const int subpel_y_qn) {
+ if (w == 2 || h == 2) {
+ av1_convolve_y_sr_c(src, src_stride, dst, dst_stride, w, h, filter_params_y,
+ subpel_y_qn);
+ return;
+ }
+
const int y_filter_taps = get_filter_tap(filter_params_y, subpel_y_qn);
- const int vert_offset = filter_params_y->taps / 2 - 1;
+ const int clamped_y_taps = y_filter_taps < 6 ? 6 : y_filter_taps;
+ const int vert_offset = clamped_y_taps / 2 - 1;
src -= vert_offset * src_stride;
@@ -1881,635 +996,27 @@
return;
}
- // Filter values are even so downshift by 1 to reduce precision requirements.
+ // Filter values are even so halve to reduce precision requirements.
const int16x8_t y_filter = vshrq_n_s16(vld1q_s16(y_filter_ptr), 1);
if (y_filter_taps < 8) {
convolve_y_sr_6tap_neon(src, src_stride, dst, dst_stride, w, h, y_filter);
- return;
- }
-
- if (w <= 4) {
- uint8x8_t d01;
- int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, d0;
-#if AOM_ARCH_AARCH64
- uint8x8_t d23;
- int16x4_t s8, s9, s10, d1, d2, d3;
-#endif // AOM_ARCH_AARCH64
- s0 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src))));
- src += src_stride;
- s1 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src))));
- src += src_stride;
- s2 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src))));
- src += src_stride;
- s3 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src))));
- src += src_stride;
- s4 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src))));
- src += src_stride;
- s5 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src))));
- src += src_stride;
- s6 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src))));
- src += src_stride;
-
- do {
- s7 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src))));
- src += src_stride;
-#if AOM_ARCH_AARCH64
- s8 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src))));
- src += src_stride;
- s9 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src))));
- src += src_stride;
- s10 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src))));
- src += src_stride;
-
- __builtin_prefetch(dst + 0 * dst_stride);
- __builtin_prefetch(dst + 1 * dst_stride);
- __builtin_prefetch(dst + 2 * dst_stride);
- __builtin_prefetch(dst + 3 * dst_stride);
- __builtin_prefetch(src + 0 * src_stride);
- __builtin_prefetch(src + 1 * src_stride);
- __builtin_prefetch(src + 2 * src_stride);
- __builtin_prefetch(src + 3 * src_stride);
- d0 = convolve8_4x4(s0, s1, s2, s3, s4, s5, s6, s7, y_filter);
- d1 = convolve8_4x4(s1, s2, s3, s4, s5, s6, s7, s8, y_filter);
- d2 = convolve8_4x4(s2, s3, s4, s5, s6, s7, s8, s9, y_filter);
- d3 = convolve8_4x4(s3, s4, s5, s6, s7, s8, s9, s10, y_filter);
-
- d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS - 1);
- d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS - 1);
-
- if (w == 2) {
- store_u8_2x1(dst + 0 * dst_stride, d01, 0);
- store_u8_2x1(dst + 1 * dst_stride, d01, 2);
- if (h != 2) {
- store_u8_2x1(dst + 2 * dst_stride, d23, 0);
- store_u8_2x1(dst + 3 * dst_stride, d23, 2);
- }
- } else {
- store_u8_4x1(dst + 0 * dst_stride, d01, 0);
- store_u8_4x1(dst + 1 * dst_stride, d01, 1);
- if (h != 2) {
- store_u8_4x1(dst + 2 * dst_stride, d23, 0);
- store_u8_4x1(dst + 3 * dst_stride, d23, 1);
- }
- }
-
- s0 = s4;
- s1 = s5;
- s2 = s6;
- s3 = s7;
- s4 = s8;
- s5 = s9;
- s6 = s10;
- dst += 4 * dst_stride;
- h -= 4;
-#else // !AOM_ARCH_AARCH64
- __builtin_prefetch(dst + 0 * dst_stride);
- __builtin_prefetch(src + 0 * src_stride);
-
- d0 = convolve8_4x4(s0, s1, s2, s3, s4, s5, s6, s7, y_filter);
-
- d01 = vqrshrun_n_s16(vcombine_s16(d0, d0), FILTER_BITS - 1);
-
- if (w == 4) {
- store_u8_4x1(dst, d01, 0);
- } else if (w == 2) {
- store_u8_2x1(dst, d01, 0);
- }
- s0 = s1;
- s1 = s2;
- s2 = s3;
- s3 = s4;
- s4 = s5;
- s5 = s6;
- s6 = s7;
- dst += dst_stride;
- h -= 1;
-#endif // AOM_ARCH_AARCH64
- } while (h > 0);
} else {
- int height;
- const uint8_t *s;
- uint8_t *d;
- uint8x8_t t0;
- int16x8_t s0, s1, s2, s3, s4, s5, s6, s7;
-#if AOM_ARCH_AARCH64
- uint8x8_t t1, t2, t3;
- int16x8_t s8, s9, s10;
-#endif // AOM_ARCH_AARCH64
- do {
- __builtin_prefetch(src + 0 * src_stride);
- __builtin_prefetch(src + 1 * src_stride);
- __builtin_prefetch(src + 2 * src_stride);
- __builtin_prefetch(src + 3 * src_stride);
- __builtin_prefetch(src + 4 * src_stride);
- __builtin_prefetch(src + 5 * src_stride);
- __builtin_prefetch(src + 6 * src_stride);
- s = src;
- s0 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
- s += src_stride;
- s1 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
- s += src_stride;
- s2 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
- s += src_stride;
- s3 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
- s += src_stride;
- s4 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
- s += src_stride;
- s5 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
- s += src_stride;
- s6 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
- s += src_stride;
- d = dst;
- height = h;
-
- do {
- s7 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
- s += src_stride;
-#if AOM_ARCH_AARCH64
- s8 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
- s += src_stride;
- s9 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
- s += src_stride;
- s10 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
- s += src_stride;
-
- __builtin_prefetch(d + 0 * dst_stride);
- __builtin_prefetch(d + 1 * dst_stride);
- __builtin_prefetch(d + 2 * dst_stride);
- __builtin_prefetch(d + 3 * dst_stride);
- __builtin_prefetch(s + 0 * src_stride);
- __builtin_prefetch(s + 1 * src_stride);
- __builtin_prefetch(s + 2 * src_stride);
- __builtin_prefetch(s + 3 * src_stride);
- t0 = convolve8_vert_8x4(s0, s1, s2, s3, s4, s5, s6, s7, y_filter);
- t1 = convolve8_vert_8x4(s1, s2, s3, s4, s5, s6, s7, s8, y_filter);
- t2 = convolve8_vert_8x4(s2, s3, s4, s5, s6, s7, s8, s9, y_filter);
- t3 = convolve8_vert_8x4(s3, s4, s5, s6, s7, s8, s9, s10, y_filter);
-
- if (h != 2) {
- store_u8_8x4(d, dst_stride, t0, t1, t2, t3);
- } else {
- store_u8_8x2(d, dst_stride, t0, t1);
- }
- s0 = s4;
- s1 = s5;
- s2 = s6;
- s3 = s7;
- s4 = s8;
- s5 = s9;
- s6 = s10;
- d += 4 * dst_stride;
- height -= 4;
-#else // !AOM_ARCH_AARCH64
- __builtin_prefetch(d);
- __builtin_prefetch(s);
-
- t0 = convolve8_vert_8x4(s0, s1, s2, s3, s4, s5, s6, s7, y_filter);
-
- vst1_u8(d, t0);
- d += dst_stride;
-
- s0 = s1;
- s1 = s2;
- s2 = s3;
- s3 = s4;
- s4 = s5;
- s5 = s6;
- s6 = s7;
- height -= 1;
-#endif // AOM_ARCH_AARCH64
- } while (height > 0);
- src += 8;
- dst += 8;
- w -= 8;
- } while (w > 0);
+ convolve_y_sr_8tap_neon(src, src_stride, dst, dst_stride, w, h, y_filter);
}
}
-#if AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_MATMUL_INT8)
-
-static INLINE int16x4_t convolve12_horiz_4_usdot(uint8x16_t samples,
- const int8x16_t filters,
- const uint8x16x3_t permute_tbl,
- int32x4_t horiz_const) {
- uint8x16_t permuted_samples[3];
- int32x4_t sum;
-
- /* Permute samples ready for dot product. */
- /* { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } */
- permuted_samples[0] = vqtbl1q_u8(samples, permute_tbl.val[0]);
- /* { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } */
- permuted_samples[1] = vqtbl1q_u8(samples, permute_tbl.val[1]);
- /* { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 } */
- permuted_samples[2] = vqtbl1q_u8(samples, permute_tbl.val[2]);
-
- /* First 4 output values. */
- sum = vusdotq_laneq_s32(horiz_const, permuted_samples[0], filters, 0);
- sum = vusdotq_laneq_s32(sum, permuted_samples[1], filters, 1);
- sum = vusdotq_laneq_s32(sum, permuted_samples[2], filters, 2);
-
- /* Narrow and re-pack. */
- return vshrn_n_s32(sum, ROUND0_BITS);
-}
-
-static INLINE int16x8_t convolve12_horiz_8_usdot(uint8x16_t samples0,
- uint8x16_t samples1,
- const int8x16_t filters,
- const uint8x16x3_t permute_tbl,
- const int32x4_t horiz_const) {
- uint8x16_t permuted_samples[4];
- int32x4_t sum[2];
-
- /* Permute samples ready for dot product. */
- /* { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } */
- permuted_samples[0] = vqtbl1q_u8(samples0, permute_tbl.val[0]);
- /* { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } */
- permuted_samples[1] = vqtbl1q_u8(samples0, permute_tbl.val[1]);
- /* { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 } */
- permuted_samples[2] = vqtbl1q_u8(samples0, permute_tbl.val[2]);
- /* {12, 13, 14, 15, 13, 14, 15, 16, 14, 15, 16, 17, 15, 16, 17, 18 } */
- permuted_samples[3] = vqtbl1q_u8(samples1, permute_tbl.val[2]);
-
- /* First 4 output values. */
- sum[0] = vusdotq_laneq_s32(horiz_const, permuted_samples[0], filters, 0);
- sum[0] = vusdotq_laneq_s32(sum[0], permuted_samples[1], filters, 1);
- sum[0] = vusdotq_laneq_s32(sum[0], permuted_samples[2], filters, 2);
- /* Second 4 output values. */
- sum[1] = vusdotq_laneq_s32(horiz_const, permuted_samples[1], filters, 0);
- sum[1] = vusdotq_laneq_s32(sum[1], permuted_samples[2], filters, 1);
- sum[1] = vusdotq_laneq_s32(sum[1], permuted_samples[3], filters, 2);
-
- /* Narrow and re-pack. */
- return vcombine_s16(vshrn_n_s32(sum[0], ROUND0_BITS),
- vshrn_n_s32(sum[1], ROUND0_BITS));
-}
-
-static INLINE void convolve_2d_sr_horiz_12tap_neon(
- const uint8_t *src_ptr, int src_stride, int16_t *dst_ptr,
- const int dst_stride, int w, int h, const int16x8_t x_filter_0_7,
- const int16x4_t x_filter_8_11) {
- const int bd = 8;
-
- // Special case the following no-op filter as 128 won't fit into the
- // 8-bit signed dot-product instruction:
- // { 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0 }
- if (vgetq_lane_s16(x_filter_0_7, 5) == 128) {
- const int16x8_t horiz_const = vdupq_n_s16((1 << (bd - 1)));
- // Undo the horizontal offset in the calling function.
- src_ptr += 5;
-
- for (int i = 0; i < h; i++) {
- for (int j = 0; j < w; j += 8) {
- uint8x8_t s0 = vld1_u8(src_ptr + i * src_stride + j);
- uint16x8_t t0 = vaddw_u8(vreinterpretq_u16_s16(horiz_const), s0);
- int16x8_t d0 =
- vshlq_n_s16(vreinterpretq_s16_u16(t0), FILTER_BITS - ROUND0_BITS);
- if (w == 2) {
- store_s16_2x1(dst_ptr + i * dst_stride, vget_low_s16(d0), 0);
- } else if (w == 4) {
- vst1_s16(dst_ptr + i * dst_stride, vget_low_s16(d0));
- } else {
- vst1q_s16(dst_ptr + i * dst_stride + j, d0);
- }
- }
- }
- } else {
- // Narrow filter values to 8-bit.
- const int16x8x2_t x_filter_s16 = {
- { x_filter_0_7, vcombine_s16(x_filter_8_11, vdup_n_s16(0)) }
- };
- const int8x16_t x_filter = vcombine_s8(vmovn_s16(x_filter_s16.val[0]),
- vmovn_s16(x_filter_s16.val[1]));
- // This shim of 1 << (ROUND0_BITS - 1) enables us to use non-rounding shifts
- // - which are generally faster than rounding shifts on modern CPUs.
- const int32x4_t horiz_const =
- vdupq_n_s32((1 << (bd + FILTER_BITS - 1)) + (1 << (ROUND0_BITS - 1)));
- const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
-
- if (w <= 4) {
- do {
- const uint8_t *s = src_ptr;
- int16_t *d = dst_ptr;
- int width = w;
-
- do {
- uint8x16_t s0, s1, s2, s3;
- int16x4_t d0, d1, d2, d3;
-
- load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3);
-
- d0 = convolve12_horiz_4_usdot(s0, x_filter, permute_tbl, horiz_const);
- d1 = convolve12_horiz_4_usdot(s1, x_filter, permute_tbl, horiz_const);
- d2 = convolve12_horiz_4_usdot(s2, x_filter, permute_tbl, horiz_const);
- d3 = convolve12_horiz_4_usdot(s3, x_filter, permute_tbl, horiz_const);
-
- if (w == 2) {
- store_s16_2x1(d + 0 * dst_stride, d0, 0);
- store_s16_2x1(d + 1 * dst_stride, d1, 0);
- store_s16_2x1(d + 2 * dst_stride, d2, 0);
- store_s16_2x1(d + 3 * dst_stride, d3, 0);
- } else {
- store_s16_4x4(d, dst_stride, d0, d1, d2, d3);
- }
-
- s += 4;
- d += 4;
- width -= 4;
- } while (width > 0);
-
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- h -= 4;
- } while (h >= 4);
-
- for (; h > 0; h--) {
- const uint8_t *s = src_ptr;
- int16_t *d = dst_ptr;
- int width = w;
-
- do {
- uint8x16_t s0;
- int16x4_t d0;
-
- s0 = vld1q_u8(s);
-
- d0 = convolve12_horiz_4_usdot(s0, x_filter, permute_tbl, horiz_const);
-
- if (w == 2) {
- store_s16_2x1(d, d0, 0);
- } else {
- vst1_s16(d, d0);
- }
-
- s += 4;
- d += 4;
- width -= 4;
- } while (width > 0);
-
- src_ptr += src_stride;
- dst_ptr += dst_stride;
- }
- } else {
- do {
- const uint8_t *s = src_ptr;
- int16_t *d = dst_ptr;
- int width = w;
-
- do {
- uint8x16_t s0[2], s1[2], s2[2], s3[2];
- int16x8_t d0, d1, d2, d3;
-
- load_u8_16x4(s, src_stride, &s0[0], &s1[0], &s2[0], &s3[0]);
- load_u8_16x4(s + 4, src_stride, &s0[1], &s1[1], &s2[1], &s3[1]);
-
- d0 = convolve12_horiz_8_usdot(s0[0], s0[1], x_filter, permute_tbl,
- horiz_const);
- d1 = convolve12_horiz_8_usdot(s1[0], s1[1], x_filter, permute_tbl,
- horiz_const);
- d2 = convolve12_horiz_8_usdot(s2[0], s2[1], x_filter, permute_tbl,
- horiz_const);
- d3 = convolve12_horiz_8_usdot(s3[0], s3[1], x_filter, permute_tbl,
- horiz_const);
-
- store_s16_8x4(d, dst_stride, d0, d1, d2, d3);
-
- s += 8;
- d += 8;
- width -= 8;
- } while (width > 0);
-
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- h -= 4;
- } while (h >= 4);
-
- for (; h > 0; h--) {
- const uint8_t *s = src_ptr;
- int16_t *d = dst_ptr;
- int width = w;
-
- do {
- uint8x16_t s0[2];
- int16x8_t d0;
-
- s0[0] = vld1q_u8(s);
- s0[1] = vld1q_u8(s + 4);
-
- d0 = convolve12_horiz_8_usdot(s0[0], s0[1], x_filter, permute_tbl,
- horiz_const);
-
- vst1q_s16(d, d0);
-
- s += 8;
- d += 8;
- width -= 8;
- } while (width > 0);
-
- src_ptr += src_stride;
- dst_ptr += dst_stride;
- }
- }
- }
-}
-
-#elif AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_DOTPROD)
-
-static INLINE void convolve_2d_sr_horiz_12tap_neon(
- const uint8_t *src_ptr, int src_stride, int16_t *dst_ptr,
- const int dst_stride, int w, int h, const int16x8_t x_filter_0_7,
- const int16x4_t x_filter_8_11) {
- const int bd = 8;
-
- // Special case the following no-op filter as 128 won't fit into the
- // 8-bit signed dot-product instruction:
- // { 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0 }
- if (vgetq_lane_s16(x_filter_0_7, 5) == 128) {
- const int16x8_t horiz_const = vdupq_n_s16((1 << (bd - 1)));
- // Undo the horizontal offset in the calling function.
- src_ptr += 5;
-
- for (int i = 0; i < h; i++) {
- for (int j = 0; j < w; j += 8) {
- uint8x8_t s0 = vld1_u8(src_ptr + i * src_stride + j);
- uint16x8_t t0 = vaddw_u8(vreinterpretq_u16_s16(horiz_const), s0);
- int16x8_t d0 =
- vshlq_n_s16(vreinterpretq_s16_u16(t0), FILTER_BITS - ROUND0_BITS);
- if (w == 2) {
- store_s16_2x1(dst_ptr + i * dst_stride, vget_low_s16(d0), 0);
- } else if (w == 4) {
- vst1_s16(dst_ptr + i * dst_stride, vget_low_s16(d0));
- } else {
- vst1q_s16(dst_ptr + i * dst_stride + j, d0);
- }
- }
- }
- } else {
- // Narrow filter values to 8-bit.
- const int16x8x2_t x_filter_s16 = {
- { x_filter_0_7, vcombine_s16(x_filter_8_11, vdup_n_s16(0)) }
- };
- const int8x16_t x_filter = vcombine_s8(vmovn_s16(x_filter_s16.val[0]),
- vmovn_s16(x_filter_s16.val[1]));
-
- // This shim of 1 << (ROUND0_BITS - 1) enables us to use non-rounding shifts
- // - which are generally faster than rounding shifts on modern CPUs.
- const int32_t horiz_const =
- ((1 << (bd + FILTER_BITS - 1)) + (1 << (ROUND0_BITS - 1)));
- // Dot product constants.
- const int32x4_t correct_tmp =
- vaddq_s32(vpaddlq_s16(vshlq_n_s16(x_filter_s16.val[0], 7)),
- vpaddlq_s16(vshlq_n_s16(x_filter_s16.val[1], 7)));
- const int32x4_t correction =
- vdupq_n_s32(vaddvq_s32(correct_tmp) + horiz_const);
- const uint8x16_t range_limit = vdupq_n_u8(128);
- const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
-
- if (w <= 4) {
- do {
- const uint8_t *s = src_ptr;
- int16_t *d = dst_ptr;
- int width = w;
-
- do {
- uint8x16_t s0, s1, s2, s3;
- int16x4_t d0, d1, d2, d3;
-
- load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3);
-
- d0 = convolve12_horiz_4_sdot(s0, x_filter, correction, range_limit,
- permute_tbl);
- d1 = convolve12_horiz_4_sdot(s1, x_filter, correction, range_limit,
- permute_tbl);
- d2 = convolve12_horiz_4_sdot(s2, x_filter, correction, range_limit,
- permute_tbl);
- d3 = convolve12_horiz_4_sdot(s3, x_filter, correction, range_limit,
- permute_tbl);
-
- if (w == 2) {
- store_s16_2x1(d + 0 * dst_stride, d0, 0);
- store_s16_2x1(d + 1 * dst_stride, d1, 0);
- store_s16_2x1(d + 2 * dst_stride, d2, 0);
- store_s16_2x1(d + 3 * dst_stride, d3, 0);
- } else {
- store_s16_4x4(d, dst_stride, d0, d1, d2, d3);
- }
-
- s += 4;
- d += 4;
- width -= 4;
- } while (width > 0);
-
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- h -= 4;
- } while (h >= 4);
-
- for (; h > 0; h--) {
- const uint8_t *s = src_ptr;
- int16_t *d = dst_ptr;
- int width = w;
-
- do {
- uint8x16_t s0;
- int16x4_t d0;
-
- s0 = vld1q_u8(s);
-
- d0 = convolve12_horiz_4_sdot(s0, x_filter, correction, range_limit,
- permute_tbl);
-
- if (w == 2) {
- store_s16_2x1(d, d0, 0);
- } else {
- vst1_s16(d, d0);
- }
-
- s += 4;
- d += 4;
- width -= 4;
- } while (width > 0);
-
- src_ptr += src_stride;
- dst_ptr += dst_stride;
- }
- } else {
- do {
- const uint8_t *s = src_ptr;
- int16_t *d = dst_ptr;
- int width = w;
-
- do {
- uint8x16_t s0[2], s1[2], s2[2], s3[2];
- int16x8_t d0, d1, d2, d3;
-
- load_u8_16x4(s, src_stride, &s0[0], &s1[0], &s2[0], &s3[0]);
- load_u8_16x4(s + 4, src_stride, &s0[1], &s1[1], &s2[1], &s3[1]);
-
- d0 = convolve12_horiz_8_sdot(s0[0], s0[1], x_filter, correction,
- range_limit, permute_tbl);
- d1 = convolve12_horiz_8_sdot(s1[0], s1[1], x_filter, correction,
- range_limit, permute_tbl);
- d2 = convolve12_horiz_8_sdot(s2[0], s2[1], x_filter, correction,
- range_limit, permute_tbl);
- d3 = convolve12_horiz_8_sdot(s3[0], s3[1], x_filter, correction,
- range_limit, permute_tbl);
-
- store_s16_8x4(d, dst_stride, d0, d1, d2, d3);
-
- s += 8;
- d += 8;
- width -= 8;
- } while (width > 0);
-
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- h -= 4;
- } while (h >= 4);
-
- for (; h > 0; h--) {
- const uint8_t *s = src_ptr;
- int16_t *d = dst_ptr;
- int width = w;
-
- do {
- uint8x16_t s0[2];
- int16x8_t d0;
-
- s0[0] = vld1q_u8(s);
- s0[1] = vld1q_u8(s + 4);
-
- d0 = convolve12_horiz_8_sdot(s0[0], s0[1], x_filter, correction,
- range_limit, permute_tbl);
-
- vst1q_s16(d, d0);
-
- s += 8;
- d += 8;
- width -= 8;
- } while (width > 0);
-
- src_ptr += src_stride;
- dst_ptr += dst_stride;
- }
- }
- }
-}
-
-#else // !(AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_DOTPROD))
-
-static INLINE int16x4_t convolve12_horiz_4x4_s16(
- const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
- const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
- const int16x4_t s6, const int16x4_t s7, const int16x4_t s8,
- const int16x4_t s9, const int16x4_t s10, const int16x4_t s11,
- const int16x8_t x_filter_0_7, const int16x4_t x_filter_8_11,
- const int32x4_t horiz_const) {
+static INLINE int16x4_t
+convolve12_4_2d_h(const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
+ const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
+ const int16x4_t s6, const int16x4_t s7, const int16x4_t s8,
+ const int16x4_t s9, const int16x4_t s10, const int16x4_t s11,
+ const int16x8_t x_filter_0_7, const int16x4_t x_filter_8_11,
+ const int32x4_t horiz_const) {
const int16x4_t x_filter_0_3 = vget_low_s16(x_filter_0_7);
const int16x4_t x_filter_4_7 = vget_high_s16(x_filter_0_7);
- int32x4_t sum;
- sum = horiz_const;
+ int32x4_t sum = horiz_const;
sum = vmlal_lane_s16(sum, s0, x_filter_0_3, 0);
sum = vmlal_lane_s16(sum, s1, x_filter_0_3, 1);
sum = vmlal_lane_s16(sum, s2, x_filter_0_3, 2);
@@ -2526,136 +1033,68 @@
return vshrn_n_s32(sum, ROUND0_BITS);
}
-// 4 column per iteration horizontal filtering for 12-tap convolve_2d_sr.
-// Processes one row at a time.
-static INLINE void horiz_filter_12tap_w4_single_row(
- const uint8_t *src_ptr, int src_stride, int16_t *dst_ptr,
- const int dst_stride, int w, int h, const int16x8_t x_filter_0_7,
- const int16x4_t x_filter_8_11, const int32x4_t horiz_const) {
- do {
- const uint8_t *s = src_ptr;
- int16_t *d = dst_ptr;
- int width = w;
-
- do {
- int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, d0;
- uint8x16_t t0;
- int16x8_t tt0, tt1;
-
- t0 = vld1q_u8(s);
- tt0 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(t0)));
- tt1 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(t0)));
-
- s0 = vget_low_s16(tt0);
- s4 = vget_high_s16(tt0);
- s8 = vget_low_s16(tt1);
- s12 = vget_high_s16(tt1);
-
- s1 = vext_s16(s0, s4, 1); // a1 a2 a3 a4
- s2 = vext_s16(s0, s4, 2); // a2 a3 a4 a5
- s3 = vext_s16(s0, s4, 3); // a3 a4 a5 a6
- s5 = vext_s16(s4, s8, 1); // a5 a6 a7 a8
- s6 = vext_s16(s4, s8, 2); // a6 a7 a8 a9
- s7 = vext_s16(s4, s8, 3); // a7 a8 a9 a10
- s9 = vext_s16(s8, s12, 1); // a9 a10 a11 a12
- s10 = vext_s16(s8, s12, 2); // a10 a11 a12 a13
- s11 = vext_s16(s8, s12, 3); // a11 a12 a13 a14
-
- d0 = convolve12_horiz_4x4_s16(s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10,
- s11, x_filter_0_7, x_filter_8_11,
- horiz_const);
-
- if (w == 2) {
- store_s16_2x1(d, d0, 0);
- } else {
- vst1_s16(d, d0);
- }
-
- s += 4;
- d += 4;
- width -= 4;
- } while (width > 0);
-
- src_ptr += src_stride;
- dst_ptr += dst_stride;
- h--;
- } while (h > 0);
-}
-
static INLINE void convolve_2d_sr_horiz_12tap_neon(
const uint8_t *src_ptr, int src_stride, int16_t *dst_ptr,
const int dst_stride, int w, int h, const int16x8_t x_filter_0_7,
const int16x4_t x_filter_8_11) {
const int bd = 8;
- // This shim of 1 << (ROUND0_BITS - 1) enables us to use non-rounding shifts -
+ // A shim of 1 << (ROUND0_BITS - 1) enables us to use non-rounding shifts -
// which are generally faster than rounding shifts on modern CPUs.
const int32x4_t horiz_const =
vdupq_n_s32((1 << (bd + FILTER_BITS - 1)) + (1 << (ROUND0_BITS - 1)));
#if AOM_ARCH_AARCH64
do {
- int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
- uint8x8_t t0, t1, t2, t3;
-
const uint8_t *s = src_ptr;
int16_t *d = dst_ptr;
int width = w;
+ uint8x8_t t0, t1, t2, t3;
load_u8_8x4(s, src_stride, &t0, &t1, &t2, &t3);
- transpose_u8_8x4(&t0, &t1, &t2, &t3);
+ transpose_elems_inplace_u8_8x4(&t0, &t1, &t2, &t3);
- s0 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s1 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
- s2 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
- s3 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t3)));
- s4 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s5 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
- s6 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
- s7 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t3)));
+ int16x4_t s0 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
+ int16x4_t s1 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
+ int16x4_t s2 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
+ int16x4_t s3 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t3)));
+ int16x4_t s4 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
+ int16x4_t s5 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
+ int16x4_t s6 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
+ int16x4_t s7 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t3)));
load_u8_8x4(s + 8, src_stride, &t0, &t1, &t2, &t3);
- transpose_u8_8x4(&t0, &t1, &t2, &t3);
+ transpose_elems_inplace_u8_8x4(&t0, &t1, &t2, &t3);
- s8 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s9 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
- s10 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
+ int16x4_t s8 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
+ int16x4_t s9 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
+ int16x4_t s10 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
s += 11;
do {
- int16x4_t s11, s12, s13, s14, d0, d1, d2, d3;
-
load_u8_8x4(s, src_stride, &t0, &t1, &t2, &t3);
- transpose_u8_8x4(&t0, &t1, &t2, &t3);
+ transpose_elems_inplace_u8_8x4(&t0, &t1, &t2, &t3);
- s11 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s12 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
- s13 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
- s14 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t3)));
+ int16x4_t s11 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
+ int16x4_t s12 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
+ int16x4_t s13 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
+ int16x4_t s14 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t3)));
- d0 = convolve12_horiz_4x4_s16(s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10,
- s11, x_filter_0_7, x_filter_8_11,
- horiz_const);
- d1 = convolve12_horiz_4x4_s16(s1, s2, s3, s4, s5, s6, s7, s8, s9, s10,
- s11, s12, x_filter_0_7, x_filter_8_11,
- horiz_const);
- d2 = convolve12_horiz_4x4_s16(s2, s3, s4, s5, s6, s7, s8, s9, s10, s11,
- s12, s13, x_filter_0_7, x_filter_8_11,
- horiz_const);
- d3 = convolve12_horiz_4x4_s16(s3, s4, s5, s6, s7, s8, s9, s10, s11, s12,
- s13, s14, x_filter_0_7, x_filter_8_11,
- horiz_const);
+ int16x4_t d0 =
+ convolve12_4_2d_h(s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11,
+ x_filter_0_7, x_filter_8_11, horiz_const);
+ int16x4_t d1 =
+ convolve12_4_2d_h(s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12,
+ x_filter_0_7, x_filter_8_11, horiz_const);
+ int16x4_t d2 =
+ convolve12_4_2d_h(s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13,
+ x_filter_0_7, x_filter_8_11, horiz_const);
+ int16x4_t d3 =
+ convolve12_4_2d_h(s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14,
+ x_filter_0_7, x_filter_8_11, horiz_const);
- transpose_s16_4x4d(&d0, &d1, &d2, &d3);
-
- if (w == 2) {
- store_s16_2x1(d + 0 * dst_stride, d0, 0);
- store_s16_2x1(d + 1 * dst_stride, d1, 0);
- store_s16_2x1(d + 2 * dst_stride, d2, 0);
- store_s16_2x1(d + 3 * dst_stride, d3, 0);
- } else {
- store_s16_4x4(d, dst_stride, d0, d1, d2, d3);
- }
+ transpose_elems_inplace_s16_4x4(&d0, &d1, &d2, &d3);
+ store_s16_4x4(d, dst_stride, d0, d1, d2, d3);
s0 = s4;
s1 = s5;
@@ -2668,562 +1107,192 @@
s8 = s12;
s9 = s13;
s10 = s14;
+ s += 4;
+ d += 4;
+ width -= 4;
+ } while (width != 0);
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ h -= 4;
+ } while (h > 4);
+#endif // AOM_ARCH_AARCH64
+
+ do {
+ const uint8_t *s = src_ptr;
+ int16_t *d = dst_ptr;
+ int width = w;
+
+ do {
+ uint8x16_t t0 = vld1q_u8(s);
+ int16x8_t tt0 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(t0)));
+ int16x8_t tt1 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(t0)));
+
+ int16x4_t s0 = vget_low_s16(tt0);
+ int16x4_t s4 = vget_high_s16(tt0);
+ int16x4_t s8 = vget_low_s16(tt1);
+ int16x4_t s12 = vget_high_s16(tt1);
+
+ int16x4_t s1 = vext_s16(s0, s4, 1); // a1 a2 a3 a4
+ int16x4_t s2 = vext_s16(s0, s4, 2); // a2 a3 a4 a5
+ int16x4_t s3 = vext_s16(s0, s4, 3); // a3 a4 a5 a6
+ int16x4_t s5 = vext_s16(s4, s8, 1); // a5 a6 a7 a8
+ int16x4_t s6 = vext_s16(s4, s8, 2); // a6 a7 a8 a9
+ int16x4_t s7 = vext_s16(s4, s8, 3); // a7 a8 a9 a10
+ int16x4_t s9 = vext_s16(s8, s12, 1); // a9 a10 a11 a12
+ int16x4_t s10 = vext_s16(s8, s12, 2); // a10 a11 a12 a13
+ int16x4_t s11 = vext_s16(s8, s12, 3); // a11 a12 a13 a14
+
+ int16x4_t d0 =
+ convolve12_4_2d_h(s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11,
+ x_filter_0_7, x_filter_8_11, horiz_const);
+ vst1_s16(d, d0);
s += 4;
d += 4;
width -= 4;
- } while (width > 0);
-
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- h -= 4;
- } while (h >= 4);
-
- if (h) {
- horiz_filter_12tap_w4_single_row(src_ptr, src_stride, dst_ptr, dst_stride,
- w, h, x_filter_0_7, x_filter_8_11,
- horiz_const);
- }
-#else // !AOM_ARCH_AARCH64
- horiz_filter_12tap_w4_single_row(src_ptr, src_stride, dst_ptr, dst_stride, w,
- h, x_filter_0_7, x_filter_8_11, horiz_const);
-#endif // AOM_ARCH_AARCH64
-}
-
-#endif // AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_DOTPROD)
-
-#if AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_MATMUL_INT8)
-
-static INLINE void convolve_2d_sr_horiz_8tap_neon(
- const uint8_t *src, int src_stride, int16_t *im_block, int im_stride, int w,
- int im_h, const int16x8_t x_filter_s16) {
- const int bd = 8;
-
- const uint8_t *src_ptr = src;
- int16_t *dst_ptr = im_block;
- int dst_stride = im_stride;
-
- int height = im_h;
-
- // Filter values are even, so downshift by 1 to reduce intermediate precision
- // requirements.
- const int8x8_t x_filter = vshrn_n_s16(x_filter_s16, 1);
- // This shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-rounding
- // shifts - which are generally faster than rounding shifts on modern CPUs.
- // The outermost -1 is needed because we halved the filter values.
- const int32x4_t horiz_const = vdupq_n_s32((1 << (bd + FILTER_BITS - 2)) +
- (1 << ((ROUND0_BITS - 1) - 1)));
-
- if (w <= 4) {
- const uint8x16x2_t permute_tbl = vld1q_u8_x2(dot_prod_permute_tbl);
- uint8x16_t s0, s1, s2, s3;
- int32x4_t t0, t1, t2, t3;
- int16x4_t d0, d1, d2, d3;
-
- do {
- assert(height >= 4);
-
- load_u8_16x4(src_ptr, src_stride, &s0, &s1, &s2, &s3);
-
- t0 = convolve8_4_usdot(s0, x_filter, permute_tbl, horiz_const);
- t1 = convolve8_4_usdot(s1, x_filter, permute_tbl, horiz_const);
- t2 = convolve8_4_usdot(s2, x_filter, permute_tbl, horiz_const);
- t3 = convolve8_4_usdot(s3, x_filter, permute_tbl, horiz_const);
-
- // We halved the convolution filter values so -1 from the right shift.
- d0 = vshrn_n_s32(t0, ROUND0_BITS - 1);
- d1 = vshrn_n_s32(t1, ROUND0_BITS - 1);
- d2 = vshrn_n_s32(t2, ROUND0_BITS - 1);
- d3 = vshrn_n_s32(t3, ROUND0_BITS - 1);
-
- if (w == 2) {
- store_s16_2x1(dst_ptr + 0 * dst_stride, d0, 0);
- store_s16_2x1(dst_ptr + 1 * dst_stride, d1, 0);
- store_s16_2x1(dst_ptr + 2 * dst_stride, d2, 0);
- store_s16_2x1(dst_ptr + 3 * dst_stride, d3, 0);
- } else {
- store_s16_4x4(dst_ptr, dst_stride, d0, d1, d2, d3);
- }
-
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- height -= 4;
- } while (height >= 4);
-
- if (height) {
- assert(height < 4);
-
- do {
- s0 = vld1q_u8(src_ptr);
- t0 = convolve8_4_usdot(s0, x_filter, permute_tbl, horiz_const);
- // We halved the convolution filter values so -1 from the right shift.
- d0 = vshrn_n_s32(t0, ROUND0_BITS - 1);
-
- if (w == 2) {
- store_s16_2x1(dst_ptr, d0, 0);
- } else {
- vst1_s16(dst_ptr, d0);
- }
-
- src_ptr += src_stride;
- dst_ptr += dst_stride;
- height--;
- } while (height > 0);
- }
- } else {
- const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
- uint8x16_t s0, s1, s2, s3;
- int16x8_t d0, d1, d2, d3;
-
- do {
- assert(height >= 4);
-
- const uint8_t *s = src_ptr;
- int16_t *d = dst_ptr;
- int width = w;
-
- do {
- load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3);
-
- d0 = convolve8_horiz_8_usdot(s0, x_filter, permute_tbl, horiz_const);
- d1 = convolve8_horiz_8_usdot(s1, x_filter, permute_tbl, horiz_const);
- d2 = convolve8_horiz_8_usdot(s2, x_filter, permute_tbl, horiz_const);
- d3 = convolve8_horiz_8_usdot(s3, x_filter, permute_tbl, horiz_const);
-
- store_s16_8x4(d, dst_stride, d0, d1, d2, d3);
-
- s += 8;
- d += 8;
- width -= 8;
- } while (width > 0);
-
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- height -= 4;
- } while (height >= 4);
-
- if (height) {
- assert(height < 4);
-
- do {
- const uint8_t *s = src_ptr;
- int16_t *d = dst_ptr;
- int width = w;
-
- do {
- s0 = vld1q_u8(s);
- d0 = convolve8_horiz_8_usdot(s0, x_filter, permute_tbl, horiz_const);
- vst1q_s16(d, d0);
-
- s += 8;
- d += 8;
- width -= 8;
- } while (width > 0);
-
- src_ptr += src_stride;
- dst_ptr += dst_stride;
- height--;
- } while (height > 0);
- }
- }
-}
-
-#elif AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_DOTPROD)
-
-static INLINE void convolve_2d_sr_horiz_8tap_neon(
- const uint8_t *src, int src_stride, int16_t *im_block, int im_stride, int w,
- int im_h, const int16x8_t x_filter_s16) {
- const int bd = 8;
-
- const uint8_t *src_ptr = src;
- int16_t *dst_ptr = im_block;
- int dst_stride = im_stride;
-
- int height = im_h;
-
- // Filter values are even, so downshift by 1 to reduce intermediate precision
- // requirements.
- const int8x8_t x_filter = vshrn_n_s16(x_filter_s16, 1);
- // This shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-rounding
- // shifts - which are generally faster than rounding shifts on modern CPUs.
- // The outermost -1 is needed because we halved the filter values.
- const int32_t horiz_const =
- ((1 << (bd + FILTER_BITS - 2)) + (1 << ((ROUND0_BITS - 1) - 1)));
- // Dot product constants.
- const int16x8_t correct_tmp = vshlq_n_s16(x_filter_s16, 6);
- int32x4_t correction = vdupq_n_s32(vaddlvq_s16(correct_tmp) + horiz_const);
- const uint8x16_t range_limit = vdupq_n_u8(128);
-
- if (w <= 4) {
- const uint8x16x2_t permute_tbl = vld1q_u8_x2(dot_prod_permute_tbl);
- uint8x16_t s0, s1, s2, s3;
- int32x4_t t0, t1, t2, t3;
- int16x4_t d0, d1, d2, d3;
-
- do {
- assert(height >= 4);
-
- load_u8_16x4(src_ptr, src_stride, &s0, &s1, &s2, &s3);
-
- t0 = convolve8_4_sdot(s0, x_filter, correction, range_limit, permute_tbl);
- t1 = convolve8_4_sdot(s1, x_filter, correction, range_limit, permute_tbl);
- t2 = convolve8_4_sdot(s2, x_filter, correction, range_limit, permute_tbl);
- t3 = convolve8_4_sdot(s3, x_filter, correction, range_limit, permute_tbl);
-
- // We halved the convolution filter values so -1 from the right shift.
- d0 = vshrn_n_s32(t0, ROUND0_BITS - 1);
- d1 = vshrn_n_s32(t1, ROUND0_BITS - 1);
- d2 = vshrn_n_s32(t2, ROUND0_BITS - 1);
- d3 = vshrn_n_s32(t3, ROUND0_BITS - 1);
-
- if (w == 2) {
- store_s16_2x1(dst_ptr + 0 * dst_stride, d0, 0);
- store_s16_2x1(dst_ptr + 1 * dst_stride, d1, 0);
- store_s16_2x1(dst_ptr + 2 * dst_stride, d2, 0);
- store_s16_2x1(dst_ptr + 3 * dst_stride, d3, 0);
- } else {
- store_s16_4x4(dst_ptr, dst_stride, d0, d1, d2, d3);
- }
-
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- height -= 4;
- } while (height >= 4);
-
- if (height) {
- assert(height < 4);
-
- do {
- s0 = vld1q_u8(src_ptr);
- t0 = convolve8_4_sdot(s0, x_filter, correction, range_limit,
- permute_tbl);
- // We halved the convolution filter values so -1 from the right shift.
- d0 = vshrn_n_s32(t0, ROUND0_BITS - 1);
-
- if (w == 2) {
- store_s16_2x1(dst_ptr, d0, 0);
- } else {
- vst1_s16(dst_ptr, d0);
- }
-
- src_ptr += src_stride;
- dst_ptr += dst_stride;
- height--;
- } while (height > 0);
- }
- } else {
- const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
- uint8x16_t s0, s1, s2, s3;
- int16x8_t d0, d1, d2, d3;
-
- do {
- assert(height >= 4);
-
- const uint8_t *s = src_ptr;
- int16_t *d = dst_ptr;
- int width = w;
-
- do {
- load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3);
-
- d0 = convolve8_horiz_8_sdot(s0, x_filter, correction, range_limit,
- permute_tbl);
- d1 = convolve8_horiz_8_sdot(s1, x_filter, correction, range_limit,
- permute_tbl);
- d2 = convolve8_horiz_8_sdot(s2, x_filter, correction, range_limit,
- permute_tbl);
- d3 = convolve8_horiz_8_sdot(s3, x_filter, correction, range_limit,
- permute_tbl);
-
- store_s16_8x4(d, dst_stride, d0, d1, d2, d3);
-
- s += 8;
- d += 8;
- width -= 8;
- } while (width > 0);
-
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- height -= 4;
- } while (height >= 4);
-
- if (height) {
- assert(height < 4);
-
- do {
- const uint8_t *s = src_ptr;
- int16_t *d = dst_ptr;
- int width = w;
-
- do {
- s0 = vld1q_u8(s);
- d0 = convolve8_8_sdot(s0, x_filter, correction, range_limit,
- permute_tbl, vdupq_n_s16(0));
- // We halved the convolution filter values so -1 from the right shift.
- d0 = vshrq_n_s16(d0, ROUND0_BITS - 1);
- vst1q_s16(d, d0);
-
- s += 8;
- d += 8;
- width -= 8;
- } while (width > 0);
-
- src_ptr += src_stride;
- dst_ptr += dst_stride;
- height--;
- } while (height > 0);
- }
- }
-}
-
-#else // !(AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_DOTPROD))
-
-// Horizontal filtering for convolve_2d_sr for width multiple of 8
-// Processes one row at a time
-static INLINE void horiz_filter_w8_single_row(const uint8_t *src_ptr,
- int src_stride, int16_t *dst_ptr,
- const int dst_stride, int width,
- int height,
- const int16x8_t x_filter,
- const int16x8_t horiz_const) {
- int16x8_t s0, s1, s2, s3, s4, s5, s6, s7;
- do {
- uint8x8_t t0 = vld1_u8(src_ptr);
- s0 = vreinterpretq_s16_u16(vmovl_u8(t0)); // a0 a1 a2 a3 a4 a5 a6 a7
-
- int width_tmp = width;
- const uint8_t *s = src_ptr + 8;
- int16_t *dst_tmp = dst_ptr;
-
- __builtin_prefetch(dst_ptr);
-
- do {
- t0 = vld1_u8(s); // a8 a9 a10 a11 a12 a13 a14 a15
- s7 = vreinterpretq_s16_u16(vmovl_u8(t0));
- int16x8_t sum = s0;
- s0 = s7;
-
- s1 = vextq_s16(sum, s7, 1); // a1 a2 a3 a4 a5 a6 a7 a8
- s2 = vextq_s16(sum, s7, 2); // a2 a3 a4 a5 a6 a7 a8 a9
- s3 = vextq_s16(sum, s7, 3); // a3 a4 a5 a6 a7 a8 a9 a10
- s4 = vextq_s16(sum, s7, 4); // a4 a5 a6 a7 a8 a9 a10 a11
- s5 = vextq_s16(sum, s7, 5); // a5 a6 a7 a8 a9 a10 a11 a12
- s6 = vextq_s16(sum, s7, 6); // a6 a7 a8 a9 a10 a11 a12 a13
- s7 = vextq_s16(sum, s7, 7); // a7 a8 a9 a10 a11 a12 a13 a14
-
- int16x8_t res0 = convolve8_horiz_8x8_s16(sum, s1, s2, s3, s4, s5, s6, s7,
- x_filter, horiz_const);
-
- vst1q_s16(dst_tmp, res0);
-
- s += 8;
- dst_tmp += 8;
- width_tmp -= 8;
- } while (width_tmp > 0);
+ } while (width != 0);
src_ptr += src_stride;
dst_ptr += dst_stride;
- height--;
- } while (height > 0);
+ } while (--h != 0);
}
-// Horizontal filtering for convolve_2d_sr for width <= 4
-// Processes one row at a time
-static INLINE void horiz_filter_w4_single_row(const uint8_t *src_ptr,
- int src_stride, int16_t *dst_ptr,
- const int dst_stride, int width,
- int height,
- const int16x8_t x_filter,
- const int16x4_t horiz_const) {
- int16x4_t s0, s1, s2, s3, s4, s5, s6, s7;
- do {
- const uint8_t *s = src_ptr;
+static INLINE int16x4_t convolve4_4_2d_h(const int16x4_t s0, const int16x4_t s1,
+ const int16x4_t s2, const int16x4_t s3,
+ const int16x4_t filter,
+ const int16x4_t horiz_const) {
+ int16x4_t sum = horiz_const;
+ sum = vmla_lane_s16(sum, s0, filter, 0);
+ sum = vmla_lane_s16(sum, s1, filter, 1);
+ sum = vmla_lane_s16(sum, s2, filter, 2);
+ sum = vmla_lane_s16(sum, s3, filter, 3);
- __builtin_prefetch(s);
-
- uint8x8_t t0 = vld1_u8(s); // a0 a1 a2 a3 a4 a5 a6 a7
- int16x8_t tt0 = vreinterpretq_s16_u16(vmovl_u8(t0));
- s0 = vget_low_s16(tt0);
- s4 = vget_high_s16(tt0);
-
- __builtin_prefetch(dst_ptr);
- s += 8;
-
- t0 = vld1_u8(s); // a8 a9 a10 a11 a12 a13 a14 a15
- s7 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
-
- s1 = vext_s16(s0, s4, 1); // a1 a2 a3 a4
- s2 = vext_s16(s0, s4, 2); // a2 a3 a4 a5
- s3 = vext_s16(s0, s4, 3); // a3 a4 a5 a6
- s5 = vext_s16(s4, s7, 1); // a5 a6 a7 a8
- s6 = vext_s16(s4, s7, 2); // a6 a7 a8 a9
- s7 = vext_s16(s4, s7, 3); // a7 a8 a9 a10
-
- int16x4_t d0 = convolve8_horiz_4x4_s16(s0, s1, s2, s3, s4, s5, s6, s7,
- x_filter, horiz_const);
-
- if (width == 2) {
- store_s16_2x1(dst_ptr, d0, 0);
- } else {
- vst1_s16(dst_ptr, d0);
- }
-
- dst_ptr += dst_stride;
- src_ptr += src_stride;
- height--;
- } while (height > 0);
+ // We halved the convolution filter values so -1 from the right shift.
+ return vshr_n_s16(sum, ROUND0_BITS - 1);
}
-static INLINE void convolve_2d_sr_horiz_8tap_neon(
- const uint8_t *src, int src_stride, int16_t *im_block, int im_stride, int w,
- int im_h, const int16x8_t x_filter_s16) {
+static INLINE int16x8_t convolve8_8_2d_h(const int16x8_t s0, const int16x8_t s1,
+ const int16x8_t s2, const int16x8_t s3,
+ const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t s6, const int16x8_t s7,
+ const int16x8_t filter,
+ const int16x8_t horiz_const) {
+ const int16x4_t filter_lo = vget_low_s16(filter);
+ const int16x4_t filter_hi = vget_high_s16(filter);
+
+ int16x8_t sum = horiz_const;
+ sum = vmlaq_lane_s16(sum, s0, filter_lo, 0);
+ sum = vmlaq_lane_s16(sum, s1, filter_lo, 1);
+ sum = vmlaq_lane_s16(sum, s2, filter_lo, 2);
+ sum = vmlaq_lane_s16(sum, s3, filter_lo, 3);
+ sum = vmlaq_lane_s16(sum, s4, filter_hi, 0);
+ sum = vmlaq_lane_s16(sum, s5, filter_hi, 1);
+ sum = vmlaq_lane_s16(sum, s6, filter_hi, 2);
+ sum = vmlaq_lane_s16(sum, s7, filter_hi, 3);
+
+ // We halved the convolution filter values so -1 from the right shift.
+ return vshrq_n_s16(sum, ROUND0_BITS - 1);
+}
+
+static INLINE void convolve_2d_sr_horiz_neon(const uint8_t *src, int src_stride,
+ int16_t *im_block, int im_stride,
+ int w, int im_h,
+ const int16_t *x_filter_ptr) {
const int bd = 8;
const uint8_t *src_ptr = src;
int16_t *dst_ptr = im_block;
int dst_stride = im_stride;
-
int height = im_h;
- // Filter values are even, so downshift by 1 to reduce intermediate precision
- // requirements.
- const int16x8_t x_filter = vshrq_n_s16(x_filter_s16, 1);
-
if (w <= 4) {
- // This shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-rounding
+ // A shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-rounding
// shifts - which are generally faster than rounding shifts on modern CPUs.
- // The outermost -1 is needed because we halved the filter values.
+ // (The extra -1 is needed because we halved the filter values.)
const int16x4_t horiz_const = vdup_n_s16((1 << (bd + FILTER_BITS - 2)) +
(1 << ((ROUND0_BITS - 1) - 1)));
+ // 4-tap filters are used for blocks having width <= 4.
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int16x4_t x_filter = vshr_n_s16(vld1_s16(x_filter_ptr + 2), 1);
-#if AOM_ARCH_AARCH64
+ src_ptr += 2;
+
do {
- int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, d0, d1, d2, d3;
- uint8x8_t t0, t1, t2, t3;
- const uint8_t *s = src_ptr;
+ uint8x8_t t0 = vld1_u8(src_ptr); // a0 a1 a2 a3 a4 a5 a6 a7
+ int16x4_t s0 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
+ int16x4_t s4 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- assert(height >= 4);
+ int16x4_t s1 = vext_s16(s0, s4, 1); // a1 a2 a3 a4
+ int16x4_t s2 = vext_s16(s0, s4, 2); // a2 a3 a4 a5
+ int16x4_t s3 = vext_s16(s0, s4, 3); // a3 a4 a5 a6
- load_u8_8x4(s, src_stride, &t0, &t1, &t2, &t3);
- transpose_u8_8x4(&t0, &t1, &t2, &t3);
+ int16x4_t d0 = convolve4_4_2d_h(s0, s1, s2, s3, x_filter, horiz_const);
- s0 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s1 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
- s2 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
- s3 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t3)));
- s4 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s5 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
- s6 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
+ vst1_s16(dst_ptr, d0);
- s += 7;
-
- load_u8_8x4(s, src_stride, &t0, &t1, &t2, &t3);
- transpose_u8_8x4(&t0, &t1, &t2, &t3);
-
- s7 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s8 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
- s9 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
- s10 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t3)));
-
- d0 = convolve8_horiz_4x4_s16(s0, s1, s2, s3, s4, s5, s6, s7, x_filter,
- horiz_const);
- d1 = convolve8_horiz_4x4_s16(s1, s2, s3, s4, s5, s6, s7, s8, x_filter,
- horiz_const);
- d2 = convolve8_horiz_4x4_s16(s2, s3, s4, s5, s6, s7, s8, s9, x_filter,
- horiz_const);
- d3 = convolve8_horiz_4x4_s16(s3, s4, s5, s6, s7, s8, s9, s10, x_filter,
- horiz_const);
-
- transpose_s16_4x4d(&d0, &d1, &d2, &d3);
-
- if (w == 2) {
- store_s16_2x1(dst_ptr + 0 * dst_stride, d0, 0);
- store_s16_2x1(dst_ptr + 1 * dst_stride, d1, 0);
- store_s16_2x1(dst_ptr + 2 * dst_stride, d2, 0);
- store_s16_2x1(dst_ptr + 3 * dst_stride, d3, 0);
- } else {
- store_s16_4x4(dst_ptr, dst_stride, d0, d1, d2, d3);
- }
-
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- height -= 4;
- } while (height >= 4);
-
- if (height) {
- assert(height < 4);
- horiz_filter_w4_single_row(src_ptr, src_stride, dst_ptr, dst_stride, w,
- height, x_filter, horiz_const);
- }
-
-#else // !AOM_ARCH_AARCH64
- horiz_filter_w4_single_row(src_ptr, src_stride, dst_ptr, dst_stride, w,
- height, x_filter, horiz_const);
-#endif // AOM_ARCH_AARCH64
-
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ } while (--height != 0);
} else {
- // This shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-rounding
+ // A shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-rounding
// shifts - which are generally faster than rounding shifts on modern CPUs.
- // The outermost -1 is needed because we halved the filter values.
+ // (The extra -1 is needed because we halved the filter values.)
const int16x8_t horiz_const = vdupq_n_s16((1 << (bd + FILTER_BITS - 2)) +
(1 << ((ROUND0_BITS - 1) - 1)));
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int16x8_t x_filter = vshrq_n_s16(vld1q_s16(x_filter_ptr), 1);
#if AOM_ARCH_AARCH64
-
- for (; height >= 8; height -= 8) {
- int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14,
- d0, d1, d2, d3, d4, d5, d6, d7;
- uint8x8_t t0, t1, t2, t3, t4, t5, t6, t7;
-
+ while (height > 8) {
const uint8_t *s = src_ptr;
int16_t *d = dst_ptr;
int width = w;
+ uint8x8_t t0, t1, t2, t3, t4, t5, t6, t7;
load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
+ transpose_elems_inplace_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
- transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
-
- s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
- s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
- s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
- s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
- s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
- s5 = vreinterpretq_s16_u16(vmovl_u8(t5));
- s6 = vreinterpretq_s16_u16(vmovl_u8(t6));
+ int16x8_t s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
+ int16x8_t s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
+ int16x8_t s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
+ int16x8_t s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
+ int16x8_t s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
+ int16x8_t s5 = vreinterpretq_s16_u16(vmovl_u8(t5));
+ int16x8_t s6 = vreinterpretq_s16_u16(vmovl_u8(t6));
s += 7;
do {
load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
- transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
+ transpose_elems_inplace_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
- s7 = vreinterpretq_s16_u16(vmovl_u8(t0));
- s8 = vreinterpretq_s16_u16(vmovl_u8(t1));
- s9 = vreinterpretq_s16_u16(vmovl_u8(t2));
- s10 = vreinterpretq_s16_u16(vmovl_u8(t3));
- s11 = vreinterpretq_s16_u16(vmovl_u8(t4));
- s12 = vreinterpretq_s16_u16(vmovl_u8(t5));
- s13 = vreinterpretq_s16_u16(vmovl_u8(t6));
- s14 = vreinterpretq_s16_u16(vmovl_u8(t7));
+ int16x8_t s7 = vreinterpretq_s16_u16(vmovl_u8(t0));
+ int16x8_t s8 = vreinterpretq_s16_u16(vmovl_u8(t1));
+ int16x8_t s9 = vreinterpretq_s16_u16(vmovl_u8(t2));
+ int16x8_t s10 = vreinterpretq_s16_u16(vmovl_u8(t3));
+ int16x8_t s11 = vreinterpretq_s16_u16(vmovl_u8(t4));
+ int16x8_t s12 = vreinterpretq_s16_u16(vmovl_u8(t5));
+ int16x8_t s13 = vreinterpretq_s16_u16(vmovl_u8(t6));
+ int16x8_t s14 = vreinterpretq_s16_u16(vmovl_u8(t7));
- d0 = convolve8_horiz_8x8_s16(s0, s1, s2, s3, s4, s5, s6, s7, x_filter,
- horiz_const);
- d1 = convolve8_horiz_8x8_s16(s1, s2, s3, s4, s5, s6, s7, s8, x_filter,
- horiz_const);
- d2 = convolve8_horiz_8x8_s16(s2, s3, s4, s5, s6, s7, s8, s9, x_filter,
- horiz_const);
- d3 = convolve8_horiz_8x8_s16(s3, s4, s5, s6, s7, s8, s9, s10, x_filter,
- horiz_const);
- d4 = convolve8_horiz_8x8_s16(s4, s5, s6, s7, s8, s9, s10, s11, x_filter,
- horiz_const);
- d5 = convolve8_horiz_8x8_s16(s5, s6, s7, s8, s9, s10, s11, s12,
- x_filter, horiz_const);
- d6 = convolve8_horiz_8x8_s16(s6, s7, s8, s9, s10, s11, s12, s13,
- x_filter, horiz_const);
- d7 = convolve8_horiz_8x8_s16(s7, s8, s9, s10, s11, s12, s13, s14,
- x_filter, horiz_const);
+ int16x8_t d0 = convolve8_8_2d_h(s0, s1, s2, s3, s4, s5, s6, s7,
+ x_filter, horiz_const);
+ int16x8_t d1 = convolve8_8_2d_h(s1, s2, s3, s4, s5, s6, s7, s8,
+ x_filter, horiz_const);
+ int16x8_t d2 = convolve8_8_2d_h(s2, s3, s4, s5, s6, s7, s8, s9,
+ x_filter, horiz_const);
+ int16x8_t d3 = convolve8_8_2d_h(s3, s4, s5, s6, s7, s8, s9, s10,
+ x_filter, horiz_const);
+ int16x8_t d4 = convolve8_8_2d_h(s4, s5, s6, s7, s8, s9, s10, s11,
+ x_filter, horiz_const);
+ int16x8_t d5 = convolve8_8_2d_h(s5, s6, s7, s8, s9, s10, s11, s12,
+ x_filter, horiz_const);
+ int16x8_t d6 = convolve8_8_2d_h(s6, s7, s8, s9, s10, s11, s12, s13,
+ x_filter, horiz_const);
+ int16x8_t d7 = convolve8_8_2d_h(s7, s8, s9, s10, s11, s12, s13, s14,
+ x_filter, horiz_const);
- transpose_s16_8x8(&d0, &d1, &d2, &d3, &d4, &d5, &d6, &d7);
+ transpose_elems_inplace_s16_8x8(&d0, &d1, &d2, &d3, &d4, &d5, &d6, &d7);
store_s16_8x8(d, dst_stride, d0, d1, d2, d3, d4, d5, d6, d7);
@@ -3237,654 +1306,46 @@
s += 8;
d += 8;
width -= 8;
- } while (width > 0);
-
+ } while (width != 0);
src_ptr += 8 * src_stride;
dst_ptr += 8 * dst_stride;
+ height -= 8;
}
+#endif // AOM_ARCH_AARCH64
- for (; height >= 4; height -= 4) {
- int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14,
- dd0, dd1, dd2, dd3, dd4, dd5, dd6, dd7;
- int16x8_t d0, d1, d2, d3;
- uint8x8_t t0, t1, t2, t3;
-
+ do {
const uint8_t *s = src_ptr;
int16_t *d = dst_ptr;
int width = w;
- load_u8_8x4(src_ptr, src_stride, &t0, &t1, &t2, &t3);
- transpose_u8_8x4(&t0, &t1, &t2, &t3);
-
- s0 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s1 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
- s2 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
- s3 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t3)));
- s4 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s5 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
- s6 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
-
- s += 7;
+ uint8x8_t t0 = vld1_u8(s); // a0 a1 a2 a3 a4 a5 a6 a7
+ int16x8_t s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
do {
- load_u8_8x4(s, src_stride, &t0, &t1, &t2, &t3);
- transpose_u8_8x4(&t0, &t1, &t2, &t3);
+ uint8x8_t t1 = vld1_u8(s + 8); // a8 a9 a10 a11 a12 a13 a14 a15
+ int16x8_t s8 = vreinterpretq_s16_u16(vmovl_u8(t1));
- s7 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s8 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
- s9 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
- s10 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t3)));
- s11 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s12 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
- s13 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
- s14 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t3)));
+ int16x8_t s1 = vextq_s16(s0, s8, 1); // a1 a2 a3 a4 a5 a6 a7 a8
+ int16x8_t s2 = vextq_s16(s0, s8, 2); // a2 a3 a4 a5 a6 a7 a8 a9
+ int16x8_t s3 = vextq_s16(s0, s8, 3); // a3 a4 a5 a6 a7 a8 a9 a10
+ int16x8_t s4 = vextq_s16(s0, s8, 4); // a4 a5 a6 a7 a8 a9 a10 a11
+ int16x8_t s5 = vextq_s16(s0, s8, 5); // a5 a6 a7 a8 a9 a10 a11 a12
+ int16x8_t s6 = vextq_s16(s0, s8, 6); // a6 a7 a8 a9 a10 a11 a12 a13
+ int16x8_t s7 = vextq_s16(s0, s8, 7); // a7 a8 a9 a10 a11 a12 a13 a14
- dd0 = convolve8_4x4(s0, s1, s2, s3, s4, s5, s6, s7, x_filter);
- dd1 = convolve8_4x4(s1, s2, s3, s4, s5, s6, s7, s8, x_filter);
- dd2 = convolve8_4x4(s2, s3, s4, s5, s6, s7, s8, s9, x_filter);
- dd3 = convolve8_4x4(s3, s4, s5, s6, s7, s8, s9, s10, x_filter);
- dd4 = convolve8_4x4(s4, s5, s6, s7, s8, s9, s10, s11, x_filter);
- dd5 = convolve8_4x4(s5, s6, s7, s8, s9, s10, s11, s12, x_filter);
- dd6 = convolve8_4x4(s6, s7, s8, s9, s10, s11, s12, s13, x_filter);
- dd7 = convolve8_4x4(s7, s8, s9, s10, s11, s12, s13, s14, x_filter);
+ int16x8_t d0 = convolve8_8_2d_h(s0, s1, s2, s3, s4, s5, s6, s7,
+ x_filter, horiz_const);
- transpose_s16_4x8(&dd0, &dd1, &dd2, &dd3, &dd4, &dd5, &dd6, &dd7, &d0,
- &d1, &d2, &d3);
-
- d0 = vaddq_s16(d0, horiz_const);
- d1 = vaddq_s16(d1, horiz_const);
- d2 = vaddq_s16(d2, horiz_const);
- d3 = vaddq_s16(d3, horiz_const);
-
- // We halved the convolution filter values so -1 from the right shift.
- d0 = vshrq_n_s16(d0, ROUND0_BITS - 1);
- d1 = vshrq_n_s16(d1, ROUND0_BITS - 1);
- d2 = vshrq_n_s16(d2, ROUND0_BITS - 1);
- d3 = vshrq_n_s16(d3, ROUND0_BITS - 1);
-
- store_s16_8x4(d, dst_stride, d0, d1, d2, d3);
+ vst1q_s16(d, d0);
s0 = s8;
- s1 = s9;
- s2 = s10;
- s3 = s11;
- s4 = s12;
- s5 = s13;
- s6 = s14;
s += 8;
d += 8;
width -= 8;
- } while (width > 0);
-
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- }
-
- if (height) {
- assert(height < 4);
- horiz_filter_w8_single_row(src_ptr, src_stride, dst_ptr, dst_stride, w,
- height, x_filter, horiz_const);
- }
-
-#else // !AOM_ARCH_AARCH64
- horiz_filter_w8_single_row(src_ptr, src_stride, dst_ptr, dst_stride, w,
- height, x_filter, horiz_const);
-#endif // AOM_ARCH_AARCH64
- }
-}
-
-#endif // AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_DOTPROD)
-
-static INLINE int32x4_t convolve12_vert_4_s32(
- const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
- const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
- const int16x4_t s6, const int16x4_t s7, const int16x4_t s8,
- const int16x4_t s9, const int16x4_t s10, const int16x4_t s11,
- const int16x8_t y_filter_0_7, const int16x4_t y_filter_8_11) {
- const int16x4_t y_filter_0_3 = vget_low_s16(y_filter_0_7);
- const int16x4_t y_filter_4_7 = vget_high_s16(y_filter_0_7);
- int32x4_t sum;
-
- sum = vmull_lane_s16(s0, y_filter_0_3, 0);
- sum = vmlal_lane_s16(sum, s1, y_filter_0_3, 1);
- sum = vmlal_lane_s16(sum, s2, y_filter_0_3, 2);
- sum = vmlal_lane_s16(sum, s3, y_filter_0_3, 3);
- sum = vmlal_lane_s16(sum, s4, y_filter_4_7, 0);
- sum = vmlal_lane_s16(sum, s5, y_filter_4_7, 1);
- sum = vmlal_lane_s16(sum, s6, y_filter_4_7, 2);
- sum = vmlal_lane_s16(sum, s7, y_filter_4_7, 3);
- sum = vmlal_lane_s16(sum, s8, y_filter_8_11, 0);
- sum = vmlal_lane_s16(sum, s9, y_filter_8_11, 1);
- sum = vmlal_lane_s16(sum, s10, y_filter_8_11, 2);
- sum = vmlal_lane_s16(sum, s11, y_filter_8_11, 3);
-
- return sum;
-}
-
-static INLINE uint8x8_t convolve12_vert_8_s32(
- const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
- const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
- const int16x8_t s6, const int16x8_t s7, const int16x8_t s8,
- const int16x8_t s9, const int16x8_t s10, const int16x8_t s11,
- const int16x8_t y_filter_0_7, const int16x4_t y_filter_8_11,
- const int16x8_t sub_const) {
- const int16x4_t y_filter_0_3 = vget_low_s16(y_filter_0_7);
- const int16x4_t y_filter_4_7 = vget_high_s16(y_filter_0_7);
- int32x4_t sum0, sum1;
- int16x8_t res;
-
- sum0 = vmull_lane_s16(vget_low_s16(s0), y_filter_0_3, 0);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s1), y_filter_0_3, 1);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s2), y_filter_0_3, 2);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s3), y_filter_0_3, 3);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s4), y_filter_4_7, 0);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s5), y_filter_4_7, 1);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s6), y_filter_4_7, 2);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s7), y_filter_4_7, 3);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s8), y_filter_8_11, 0);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s9), y_filter_8_11, 1);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s10), y_filter_8_11, 2);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s11), y_filter_8_11, 3);
-
- sum1 = vmull_lane_s16(vget_high_s16(s0), y_filter_0_3, 0);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s1), y_filter_0_3, 1);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s2), y_filter_0_3, 2);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s3), y_filter_0_3, 3);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s4), y_filter_4_7, 0);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s5), y_filter_4_7, 1);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s6), y_filter_4_7, 2);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s7), y_filter_4_7, 3);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s8), y_filter_8_11, 0);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s9), y_filter_8_11, 1);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s10), y_filter_8_11, 2);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s11), y_filter_8_11, 3);
-
- res = vcombine_s16(vqrshrn_n_s32(sum0, 2 * FILTER_BITS - ROUND0_BITS),
- vqrshrn_n_s32(sum1, 2 * FILTER_BITS - ROUND0_BITS));
- res = vsubq_s16(res, sub_const);
-
- return vqmovun_s16(res);
-}
-
-static INLINE void convolve_2d_sr_vert_12tap_neon(
- int16_t *src_ptr, int src_stride, uint8_t *dst_ptr, int dst_stride, int w,
- int h, const int16x8_t y_filter_0_7, const int16x4_t y_filter_8_11) {
- const int bd = 8;
- const int16x8_t sub_const = vdupq_n_s16(1 << (bd - 1));
-
- if (w <= 4) {
- int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14;
- int32x4_t d0, d1, d2, d3;
- int16x8_t dd01, dd23;
- uint8x8_t d01, d23;
-
- load_s16_4x11(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7,
- &s8, &s9, &s10);
- src_ptr += 11 * src_stride;
-
- do {
- load_s16_4x4(src_ptr, src_stride, &s11, &s12, &s13, &s14);
-
- d0 = convolve12_vert_4_s32(s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10,
- s11, y_filter_0_7, y_filter_8_11);
- d1 = convolve12_vert_4_s32(s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11,
- s12, y_filter_0_7, y_filter_8_11);
- d2 = convolve12_vert_4_s32(s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12,
- s13, y_filter_0_7, y_filter_8_11);
- d3 = convolve12_vert_4_s32(s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13,
- s14, y_filter_0_7, y_filter_8_11);
-
- dd01 = vcombine_s16(vqrshrn_n_s32(d0, 2 * FILTER_BITS - ROUND0_BITS),
- vqrshrn_n_s32(d1, 2 * FILTER_BITS - ROUND0_BITS));
- dd23 = vcombine_s16(vqrshrn_n_s32(d2, 2 * FILTER_BITS - ROUND0_BITS),
- vqrshrn_n_s32(d3, 2 * FILTER_BITS - ROUND0_BITS));
-
- dd01 = vsubq_s16(dd01, sub_const);
- dd23 = vsubq_s16(dd23, sub_const);
-
- d01 = vqmovun_s16(dd01);
- d23 = vqmovun_s16(dd23);
-
- if (w == 2) {
- store_u8_2x1(dst_ptr + 0 * dst_stride, d01, 0);
- store_u8_2x1(dst_ptr + 1 * dst_stride, d01, 2);
- if (h != 2) {
- store_u8_2x1(dst_ptr + 2 * dst_stride, d23, 0);
- store_u8_2x1(dst_ptr + 3 * dst_stride, d23, 2);
- }
- } else {
- store_u8_4x1(dst_ptr + 0 * dst_stride, d01, 0);
- store_u8_4x1(dst_ptr + 1 * dst_stride, d01, 1);
- if (h != 2) {
- store_u8_4x1(dst_ptr + 2 * dst_stride, d23, 0);
- store_u8_4x1(dst_ptr + 3 * dst_stride, d23, 1);
- }
- }
-
- s0 = s4;
- s1 = s5;
- s2 = s6;
- s3 = s7;
- s4 = s8;
- s5 = s9;
- s6 = s10;
- s7 = s11;
- s8 = s12;
- s9 = s13;
- s10 = s14;
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- h -= 4;
- } while (h > 0);
-
- } else {
- do {
- int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14;
- uint8x8_t d0, d1, d2, d3;
-
- int16_t *s = src_ptr;
- uint8_t *d = dst_ptr;
-
- int height = h;
-
- load_s16_8x11(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7, &s8,
- &s9, &s10);
- s += 11 * src_stride;
-
- do {
- load_s16_8x4(s, src_stride, &s11, &s12, &s13, &s14);
-
- d0 = convolve12_vert_8_s32(s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10,
- s11, y_filter_0_7, y_filter_8_11, sub_const);
- d1 = convolve12_vert_8_s32(s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11,
- s12, y_filter_0_7, y_filter_8_11, sub_const);
- d2 =
- convolve12_vert_8_s32(s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12,
- s13, y_filter_0_7, y_filter_8_11, sub_const);
- d3 = convolve12_vert_8_s32(s3, s4, s5, s6, s7, s8, s9, s10, s11, s12,
- s13, s14, y_filter_0_7, y_filter_8_11,
- sub_const);
-
- if (h != 2) {
- store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
- } else {
- store_u8_8x2(d, dst_stride, d0, d1);
- }
-
- s0 = s4;
- s1 = s5;
- s2 = s6;
- s3 = s7;
- s4 = s8;
- s5 = s9;
- s6 = s10;
- s7 = s11;
- s8 = s12;
- s9 = s13;
- s10 = s14;
- s += 4 * src_stride;
- d += 4 * dst_stride;
- height -= 4;
- } while (height > 0);
-
- src_ptr += 8;
- dst_ptr += 8;
- w -= 8;
- } while (w > 0);
- }
-}
-
-static INLINE void convolve_2d_sr_vert_8tap_neon(int16_t *src_ptr,
- int src_stride,
- uint8_t *dst_ptr,
- int dst_stride, int w, int h,
- const int16x8_t y_filter) {
- const int bd = 8;
- const int16x8_t sub_const = vdupq_n_s16(1 << (bd - 1));
-
- if (w <= 4) {
- int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, d0;
- uint8x8_t d01;
-
-#if AOM_ARCH_AARCH64
- int16x4_t s8, s9, s10, d1, d2, d3;
- uint8x8_t d23;
-#endif // AOM_ARCH_AARCH64
-
- int16_t *s = src_ptr;
- uint8_t *d = dst_ptr;
-
- load_s16_4x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
- s += 7 * src_stride;
-
- do {
-#if AOM_ARCH_AARCH64
- load_s16_4x4(s, src_stride, &s7, &s8, &s9, &s10);
-
- d0 = convolve8_vert_4_s32(s0, s1, s2, s3, s4, s5, s6, s7, y_filter);
- d1 = convolve8_vert_4_s32(s1, s2, s3, s4, s5, s6, s7, s8, y_filter);
- d2 = convolve8_vert_4_s32(s2, s3, s4, s5, s6, s7, s8, s9, y_filter);
- d3 = convolve8_vert_4_s32(s3, s4, s5, s6, s7, s8, s9, s10, y_filter);
-
- d01 = vqmovun_s16(vsubq_s16(vcombine_s16(d0, d1), sub_const));
- d23 = vqmovun_s16(vsubq_s16(vcombine_s16(d2, d3), sub_const));
-
- if (w == 2) {
- store_u8_2x1(d + 0 * dst_stride, d01, 0);
- store_u8_2x1(d + 1 * dst_stride, d01, 2);
- if (h != 2) {
- store_u8_2x1(d + 2 * dst_stride, d23, 0);
- store_u8_2x1(d + 3 * dst_stride, d23, 2);
- }
- } else {
- store_u8_4x1(d + 0 * dst_stride, d01, 0);
- store_u8_4x1(d + 1 * dst_stride, d01, 1);
- if (h != 2) {
- store_u8_4x1(d + 2 * dst_stride, d23, 0);
- store_u8_4x1(d + 3 * dst_stride, d23, 1);
- }
- }
-
- s0 = s4;
- s1 = s5;
- s2 = s6;
- s3 = s7;
- s4 = s8;
- s5 = s9;
- s6 = s10;
- s += 4 * src_stride;
- d += 4 * dst_stride;
- h -= 4;
-#else // !AOM_ARCH_AARCH64
- s7 = vld1_s16(s);
- s += src_stride;
-
- d0 = convolve8_vert_4_s32(s0, s1, s2, s3, s4, s5, s6, s7, y_filter);
-
- d01 = vqmovun_s16(vsubq_s16(vcombine_s16(d0, vdup_n_s16(0)), sub_const));
-
- if (w == 2) {
- store_u8_2x1(d, d01, 0);
- } else {
- store_u8_4x1(d, d01, 0);
- }
-
- s0 = s1;
- s1 = s2;
- s2 = s3;
- s3 = s4;
- s4 = s5;
- s5 = s6;
- s6 = s7;
- d += dst_stride;
- h--;
-#endif // AOM_ARCH_AARCH64
- } while (h > 0);
- } else {
- // if width is a multiple of 8 & height is a multiple of 4
- int16x8_t s0, s1, s2, s3, s4, s5, s6, s7;
- uint8x8_t d0;
-#if AOM_ARCH_AARCH64
- int16x8_t s8, s9, s10;
- uint8x8_t d1, d2, d3;
-#endif // AOM_ARCH_AARCH64
-
- do {
- int height = h;
- int16_t *s = src_ptr;
- uint8_t *d = dst_ptr;
-
- load_s16_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
- s += 7 * src_stride;
-
- do {
-#if AOM_ARCH_AARCH64
- load_s16_8x4(s, src_stride, &s7, &s8, &s9, &s10);
-
- d0 = convolve8_vert_8_s32(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
- sub_const);
- d1 = convolve8_vert_8_s32(s1, s2, s3, s4, s5, s6, s7, s8, y_filter,
- sub_const);
- d2 = convolve8_vert_8_s32(s2, s3, s4, s5, s6, s7, s8, s9, y_filter,
- sub_const);
- d3 = convolve8_vert_8_s32(s3, s4, s5, s6, s7, s8, s9, s10, y_filter,
- sub_const);
-
- if (h != 2) {
- store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
- } else {
- store_u8_8x2(d, dst_stride, d0, d1);
- }
-
- s0 = s4;
- s1 = s5;
- s2 = s6;
- s3 = s7;
- s4 = s8;
- s5 = s9;
- s6 = s10;
- s += 4 * src_stride;
- d += 4 * dst_stride;
- height -= 4;
-#else // !AOM_ARCH_AARCH64
- s7 = vld1q_s16(s);
-
- d0 = convolve8_vert_8_s32(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
- sub_const);
-
- vst1_u8(d, d0);
-
- s0 = s1;
- s1 = s2;
- s2 = s3;
- s3 = s4;
- s4 = s5;
- s5 = s6;
- s6 = s7;
- s += src_stride;
- d += dst_stride;
- height--;
-#endif // AOM_ARCH_AARCH64
- } while (height > 0);
-
- src_ptr += 8;
- dst_ptr += 8;
- w -= 8;
- } while (w > 0);
- }
-}
-
-static INLINE int16x4_t
-convolve6_vert_4_s32(const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
- const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
- const int16x8_t y_filter) {
- const int16x4_t y_filter_lo = vget_low_s16(y_filter);
- const int16x4_t y_filter_hi = vget_high_s16(y_filter);
- int32x4_t sum;
-
- sum = vmull_lane_s16(s0, y_filter_lo, 1);
- sum = vmlal_lane_s16(sum, s1, y_filter_lo, 2);
- sum = vmlal_lane_s16(sum, s2, y_filter_lo, 3);
- sum = vmlal_lane_s16(sum, s3, y_filter_hi, 0);
- sum = vmlal_lane_s16(sum, s4, y_filter_hi, 1);
- sum = vmlal_lane_s16(sum, s5, y_filter_hi, 2);
-
- return vqrshrn_n_s32(sum, 2 * FILTER_BITS - ROUND0_BITS);
-}
-
-static INLINE uint8x8_t
-convolve6_vert_8_s32(const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
- const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
- const int16x8_t y_filter, const int16x8_t sub_const) {
- const int16x4_t y_filter_lo = vget_low_s16(y_filter);
- const int16x4_t y_filter_hi = vget_high_s16(y_filter);
- int32x4_t sum0, sum1;
- int16x8_t res;
-
- sum0 = vmull_lane_s16(vget_low_s16(s0), y_filter_lo, 1);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s1), y_filter_lo, 2);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s2), y_filter_lo, 3);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s3), y_filter_hi, 0);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s4), y_filter_hi, 1);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s5), y_filter_hi, 2);
-
- sum1 = vmull_lane_s16(vget_high_s16(s0), y_filter_lo, 1);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s1), y_filter_lo, 2);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s2), y_filter_lo, 3);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s3), y_filter_hi, 0);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s4), y_filter_hi, 1);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s5), y_filter_hi, 2);
-
- res = vcombine_s16(vqrshrn_n_s32(sum0, 2 * FILTER_BITS - ROUND0_BITS),
- vqrshrn_n_s32(sum1, 2 * FILTER_BITS - ROUND0_BITS));
- res = vsubq_s16(res, sub_const);
-
- return vqmovun_s16(res);
-}
-
-static INLINE void convolve_2d_sr_vert_6tap_neon(int16_t *src_ptr,
- int src_stride,
- uint8_t *dst_ptr,
- int dst_stride, int w, int h,
- const int16x8_t y_filter) {
- const int bd = 8;
- const int16x8_t sub_const = vdupq_n_s16(1 << (bd - 1));
-
- if (w <= 4) {
- int16x4_t s0, s1, s2, s3, s4, s5, d0;
- uint8x8_t d01;
-
-#if AOM_ARCH_AARCH64
- int16x4_t s6, s7, s8, d1, d2, d3;
- uint8x8_t d23;
-#endif // AOM_ARCH_AARCH64
-
- int16_t *s = src_ptr;
- uint8_t *d = dst_ptr;
-
- load_s16_4x5(s, src_stride, &s0, &s1, &s2, &s3, &s4);
- s += 5 * src_stride;
-
- do {
-#if AOM_ARCH_AARCH64
- load_s16_4x4(s, src_stride, &s5, &s6, &s7, &s8);
-
- d0 = convolve6_vert_4_s32(s0, s1, s2, s3, s4, s5, y_filter);
- d1 = convolve6_vert_4_s32(s1, s2, s3, s4, s5, s6, y_filter);
- d2 = convolve6_vert_4_s32(s2, s3, s4, s5, s6, s7, y_filter);
- d3 = convolve6_vert_4_s32(s3, s4, s5, s6, s7, s8, y_filter);
-
- d01 = vqmovun_s16(vsubq_s16(vcombine_s16(d0, d1), sub_const));
- d23 = vqmovun_s16(vsubq_s16(vcombine_s16(d2, d3), sub_const));
-
- if (w == 2) {
- store_u8_2x1(d + 0 * dst_stride, d01, 0);
- store_u8_2x1(d + 1 * dst_stride, d01, 2);
- if (h != 2) {
- store_u8_2x1(d + 2 * dst_stride, d23, 0);
- store_u8_2x1(d + 3 * dst_stride, d23, 2);
- }
- } else {
- store_u8_4x1(d + 0 * dst_stride, d01, 0);
- store_u8_4x1(d + 1 * dst_stride, d01, 1);
- if (h != 2) {
- store_u8_4x1(d + 2 * dst_stride, d23, 0);
- store_u8_4x1(d + 3 * dst_stride, d23, 1);
- }
- }
-
- s0 = s4;
- s1 = s5;
- s2 = s6;
- s3 = s7;
- s4 = s8;
- s += 4 * src_stride;
- d += 4 * dst_stride;
- h -= 4;
-#else // !AOM_ARCH_AARCH64
- s5 = vld1_s16(s);
-
- d0 = convolve6_vert_4_s32(s0, s1, s2, s3, s4, s5, y_filter);
- d01 = vqmovun_s16(vsubq_s16(vcombine_s16(d0, vdup_n_s16(0)), sub_const));
-
- if (w == 2) {
- store_u8_2x1(d, d01, 0);
- } else {
- store_u8_4x1(d, d01, 0);
- }
-
- s0 = s1;
- s1 = s2;
- s2 = s3;
- s3 = s4;
- s4 = s5;
- s += src_stride;
- d += dst_stride;
- h--;
-#endif // AOM_ARCH_AARCH64
- } while (h > 0);
- } else {
- // if width is a multiple of 8 & height is a multiple of 4
- int16x8_t s0, s1, s2, s3, s4, s5;
- uint8x8_t d0;
-#if AOM_ARCH_AARCH64
- int16x8_t s6, s7, s8;
- uint8x8_t d1, d2, d3;
-#endif // AOM_ARCH_AARCH64
-
- do {
- int height = h;
- int16_t *s = src_ptr;
- uint8_t *d = dst_ptr;
-
- load_s16_8x5(s, src_stride, &s0, &s1, &s2, &s3, &s4);
- s += 5 * src_stride;
-
- do {
-#if AOM_ARCH_AARCH64
- load_s16_8x4(s, src_stride, &s5, &s6, &s7, &s8);
-
- d0 = convolve6_vert_8_s32(s0, s1, s2, s3, s4, s5, y_filter, sub_const);
- d1 = convolve6_vert_8_s32(s1, s2, s3, s4, s5, s6, y_filter, sub_const);
- d2 = convolve6_vert_8_s32(s2, s3, s4, s5, s6, s7, y_filter, sub_const);
- d3 = convolve6_vert_8_s32(s3, s4, s5, s6, s7, s8, y_filter, sub_const);
-
- if (h != 2) {
- store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
- } else {
- store_u8_8x2(d, dst_stride, d0, d1);
- }
-
- s0 = s4;
- s1 = s5;
- s2 = s6;
- s3 = s7;
- s4 = s8;
- s += 4 * src_stride;
- d += 4 * dst_stride;
- height -= 4;
-#else // !AOM_ARCH_AARCH64
- s5 = vld1q_s16(s);
-
- d0 = convolve6_vert_8_s32(s0, s1, s2, s3, s4, s5, y_filter, sub_const);
-
- vst1_u8(d, d0);
-
- s0 = s1;
- s1 = s2;
- s2 = s3;
- s3 = s4;
- s4 = s5;
- s += src_stride;
- d += dst_stride;
- height--;
-#endif // AOM_ARCH_AARCH64
- } while (height > 0);
-
- src_ptr += 8;
- dst_ptr += 8;
- w -= 8;
- } while (w > 0);
+ } while (width != 0);
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ } while (--height != 0);
}
}
@@ -3894,7 +1355,13 @@
const InterpFilterParams *filter_params_y,
const int subpel_x_qn, const int subpel_y_qn,
ConvolveParams *conv_params) {
- (void)conv_params;
+ if (w == 2 || h == 2) {
+ av1_convolve_2d_sr_c(src, src_stride, dst, dst_stride, w, h,
+ filter_params_x, filter_params_y, subpel_x_qn,
+ subpel_y_qn, conv_params);
+ return;
+ }
+
const int y_filter_taps = get_filter_tap(filter_params_y, subpel_y_qn);
const int clamped_y_taps = y_filter_taps < 6 ? 6 : y_filter_taps;
const int im_h = h + clamped_y_taps - 1;
@@ -3924,14 +1391,13 @@
y_filter_0_7, y_filter_8_11);
} else {
DECLARE_ALIGNED(16, int16_t,
- im_block[(MAX_SB_SIZE + HORIZ_EXTRA_ROWS) * MAX_SB_SIZE]);
+ im_block[(MAX_SB_SIZE + SUBPEL_TAPS - 1) * MAX_SB_SIZE]);
- const int16x8_t x_filter = vld1q_s16(x_filter_ptr);
+ convolve_2d_sr_horiz_neon(src_ptr, src_stride, im_block, im_stride, w, im_h,
+ x_filter_ptr);
+
const int16x8_t y_filter = vld1q_s16(y_filter_ptr);
- convolve_2d_sr_horiz_8tap_neon(src_ptr, src_stride, im_block, im_stride, w,
- im_h, x_filter);
-
if (clamped_y_taps <= 6) {
convolve_2d_sr_vert_6tap_neon(im_block, im_stride, dst, dst_stride, w, h,
y_filter);
@@ -3942,291 +1408,259 @@
}
}
-static INLINE void scaledconvolve_horiz_w4(
- const uint8_t *src, const ptrdiff_t src_stride, uint8_t *dst,
- const ptrdiff_t dst_stride, const InterpKernel *const x_filters,
- const int x0_q4, const int x_step_q4, const int w, const int h) {
- DECLARE_ALIGNED(16, uint8_t, temp[4 * 4]);
- int x, y, z;
+void av1_convolve_x_sr_intrabc_neon(const uint8_t *src, int src_stride,
+ uint8_t *dst, int dst_stride, int w, int h,
+ const InterpFilterParams *filter_params_x,
+ const int subpel_x_qn,
+ ConvolveParams *conv_params) {
+ assert(subpel_x_qn == 8);
+ assert(filter_params_x->taps == 2);
+ assert((conv_params->round_0 + conv_params->round_1) == 2 * FILTER_BITS);
+ (void)filter_params_x;
+ (void)subpel_x_qn;
+ (void)conv_params;
- src -= SUBPEL_TAPS / 2 - 1;
-
- y = h;
- do {
- int x_q4 = x0_q4;
- x = 0;
+ if (w <= 4) {
do {
- // process 4 src_x steps
- for (z = 0; z < 4; ++z) {
- const uint8_t *const src_x = &src[x_q4 >> SUBPEL_BITS];
- if (x_q4 & SUBPEL_MASK) {
- const int16x8_t filters = vld1q_s16(x_filters[x_q4 & SUBPEL_MASK]);
- uint8x8_t s[8], d;
- int16x8_t ss[4];
- int16x4_t t[8], tt;
+ uint8x8_t s0_0 = vld1_u8(src);
+ uint8x8_t s0_1 = vld1_u8(src + 1);
+ uint8x8_t s1_0 = vld1_u8(src + src_stride);
+ uint8x8_t s1_1 = vld1_u8(src + src_stride + 1);
- load_u8_8x4(src_x, src_stride, &s[0], &s[1], &s[2], &s[3]);
- transpose_u8_8x4(&s[0], &s[1], &s[2], &s[3]);
+ uint8x8_t d0 = vrhadd_u8(s0_0, s0_1);
+ uint8x8_t d1 = vrhadd_u8(s1_0, s1_1);
- ss[0] = vreinterpretq_s16_u16(vmovl_u8(s[0]));
- ss[1] = vreinterpretq_s16_u16(vmovl_u8(s[1]));
- ss[2] = vreinterpretq_s16_u16(vmovl_u8(s[2]));
- ss[3] = vreinterpretq_s16_u16(vmovl_u8(s[3]));
- t[0] = vget_low_s16(ss[0]);
- t[1] = vget_low_s16(ss[1]);
- t[2] = vget_low_s16(ss[2]);
- t[3] = vget_low_s16(ss[3]);
- t[4] = vget_high_s16(ss[0]);
- t[5] = vget_high_s16(ss[1]);
- t[6] = vget_high_s16(ss[2]);
- t[7] = vget_high_s16(ss[3]);
-
- tt = convolve8_4(t[0], t[1], t[2], t[3], t[4], t[5], t[6], t[7],
- filters);
- d = vqrshrun_n_s16(vcombine_s16(tt, tt), 7);
- store_u8_4x1(&temp[4 * z], d, 0);
- } else {
- int i;
- for (i = 0; i < 4; ++i) {
- temp[z * 4 + i] = src_x[i * src_stride + 3];
- }
- }
- x_q4 += x_step_q4;
+ if (w == 2) {
+ store_u8_2x1(dst + 0 * dst_stride, d0, 0);
+ store_u8_2x1(dst + 1 * dst_stride, d1, 0);
+ } else {
+ store_u8_4x1(dst + 0 * dst_stride, d0, 0);
+ store_u8_4x1(dst + 1 * dst_stride, d1, 0);
}
- // transpose the 4x4 filters values back to dst
- {
- const uint8x8x4_t d4 = vld4_u8(temp);
- store_u8_4x1(&dst[x + 0 * dst_stride], d4.val[0], 0);
- store_u8_4x1(&dst[x + 1 * dst_stride], d4.val[1], 0);
- store_u8_4x1(&dst[x + 2 * dst_stride], d4.val[2], 0);
- store_u8_4x1(&dst[x + 3 * dst_stride], d4.val[3], 0);
- }
- x += 4;
- } while (x < w);
-
- src += src_stride * 4;
- dst += dst_stride * 4;
- y -= 4;
- } while (y > 0);
-}
-
-static INLINE void scaledconvolve_horiz_w8(
- const uint8_t *src, const ptrdiff_t src_stride, uint8_t *dst,
- const ptrdiff_t dst_stride, const InterpKernel *const x_filters,
- const int x0_q4, const int x_step_q4, const int w, const int h) {
- DECLARE_ALIGNED(16, uint8_t, temp[8 * 8]);
- int x, y, z;
- src -= SUBPEL_TAPS / 2 - 1;
-
- // This function processes 8x8 areas. The intermediate height is not always
- // a multiple of 8, so force it to be a multiple of 8 here.
- y = (h + 7) & ~7;
-
- do {
- int x_q4 = x0_q4;
- x = 0;
+ src += 2 * src_stride;
+ dst += 2 * dst_stride;
+ h -= 2;
+ } while (h != 0);
+ } else if (w == 8) {
do {
- uint8x8_t d[8];
- // process 8 src_x steps
- for (z = 0; z < 8; ++z) {
- const uint8_t *const src_x = &src[x_q4 >> SUBPEL_BITS];
+ uint8x8_t s0_0 = vld1_u8(src);
+ uint8x8_t s0_1 = vld1_u8(src + 1);
+ uint8x8_t s1_0 = vld1_u8(src + src_stride);
+ uint8x8_t s1_1 = vld1_u8(src + src_stride + 1);
- if (x_q4 & SUBPEL_MASK) {
- const int16x8_t filters = vld1q_s16(x_filters[x_q4 & SUBPEL_MASK]);
- uint8x8_t s[8];
- load_u8_8x8(src_x, src_stride, &s[0], &s[1], &s[2], &s[3], &s[4],
- &s[5], &s[6], &s[7]);
- transpose_u8_8x8(&s[0], &s[1], &s[2], &s[3], &s[4], &s[5], &s[6],
- &s[7]);
- d[0] = scale_filter_8(s, filters);
- vst1_u8(&temp[8 * z], d[0]);
- } else {
- int i;
- for (i = 0; i < 8; ++i) {
- temp[z * 8 + i] = src_x[i * src_stride + 3];
- }
- }
- x_q4 += x_step_q4;
- }
+ uint8x8_t d0 = vrhadd_u8(s0_0, s0_1);
+ uint8x8_t d1 = vrhadd_u8(s1_0, s1_1);
- // transpose the 8x8 filters values back to dst
- load_u8_8x8(temp, 8, &d[0], &d[1], &d[2], &d[3], &d[4], &d[5], &d[6],
- &d[7]);
- transpose_u8_8x8(&d[0], &d[1], &d[2], &d[3], &d[4], &d[5], &d[6], &d[7]);
- store_u8_8x8(dst + x, dst_stride, d[0], d[1], d[2], d[3], d[4], d[5],
- d[6], d[7]);
- x += 8;
- } while (x < w);
+ vst1_u8(dst, d0);
+ vst1_u8(dst + dst_stride, d1);
- src += src_stride * 8;
- dst += dst_stride * 8;
- } while (y -= 8);
-}
-
-static INLINE void scaledconvolve_vert_w4(
- const uint8_t *src, const ptrdiff_t src_stride, uint8_t *dst,
- const ptrdiff_t dst_stride, const InterpKernel *const y_filters,
- const int y0_q4, const int y_step_q4, const int w, const int h) {
- int y;
- int y_q4 = y0_q4;
-
- src -= src_stride * (SUBPEL_TAPS / 2 - 1);
- y = h;
- do {
- const unsigned char *src_y = &src[(y_q4 >> SUBPEL_BITS) * src_stride];
-
- if (y_q4 & SUBPEL_MASK) {
- const int16x8_t filters = vld1q_s16(y_filters[y_q4 & SUBPEL_MASK]);
- uint8x8_t s[8], d;
- int16x4_t t[8], tt;
-
- load_u8_8x8(src_y, src_stride, &s[0], &s[1], &s[2], &s[3], &s[4], &s[5],
- &s[6], &s[7]);
- t[0] = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(s[0])));
- t[1] = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(s[1])));
- t[2] = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(s[2])));
- t[3] = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(s[3])));
- t[4] = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(s[4])));
- t[5] = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(s[5])));
- t[6] = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(s[6])));
- t[7] = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(s[7])));
-
- tt = convolve8_4(t[0], t[1], t[2], t[3], t[4], t[5], t[6], t[7], filters);
- d = vqrshrun_n_s16(vcombine_s16(tt, tt), 7);
- store_u8_4x1(dst, d, 0);
- } else {
- memcpy(dst, &src_y[3 * src_stride], w);
- }
-
- dst += dst_stride;
- y_q4 += y_step_q4;
- } while (--y);
-}
-
-static INLINE void scaledconvolve_vert_w8(
- const uint8_t *src, const ptrdiff_t src_stride, uint8_t *dst,
- const ptrdiff_t dst_stride, const InterpKernel *const y_filters,
- const int y0_q4, const int y_step_q4, const int w, const int h) {
- int y;
- int y_q4 = y0_q4;
-
- src -= src_stride * (SUBPEL_TAPS / 2 - 1);
- y = h;
- do {
- const unsigned char *src_y = &src[(y_q4 >> SUBPEL_BITS) * src_stride];
- if (y_q4 & SUBPEL_MASK) {
- const int16x8_t filters = vld1q_s16(y_filters[y_q4 & SUBPEL_MASK]);
- uint8x8_t s[8], d;
- load_u8_8x8(src_y, src_stride, &s[0], &s[1], &s[2], &s[3], &s[4], &s[5],
- &s[6], &s[7]);
- d = scale_filter_8(s, filters);
- vst1_u8(dst, d);
- } else {
- memcpy(dst, &src_y[3 * src_stride], w);
- }
- dst += dst_stride;
- y_q4 += y_step_q4;
- } while (--y);
-}
-
-static INLINE void scaledconvolve_vert_w16(
- const uint8_t *src, const ptrdiff_t src_stride, uint8_t *dst,
- const ptrdiff_t dst_stride, const InterpKernel *const y_filters,
- const int y0_q4, const int y_step_q4, const int w, const int h) {
- int x, y;
- int y_q4 = y0_q4;
-
- src -= src_stride * (SUBPEL_TAPS / 2 - 1);
- y = h;
- do {
- const unsigned char *src_y = &src[(y_q4 >> SUBPEL_BITS) * src_stride];
- if (y_q4 & SUBPEL_MASK) {
- x = 0;
- do {
- const int16x8_t filters = vld1q_s16(y_filters[y_q4 & SUBPEL_MASK]);
- uint8x16_t ss[8];
- uint8x8_t s[8], d[2];
- load_u8_16x8(src_y, src_stride, &ss[0], &ss[1], &ss[2], &ss[3], &ss[4],
- &ss[5], &ss[6], &ss[7]);
- s[0] = vget_low_u8(ss[0]);
- s[1] = vget_low_u8(ss[1]);
- s[2] = vget_low_u8(ss[2]);
- s[3] = vget_low_u8(ss[3]);
- s[4] = vget_low_u8(ss[4]);
- s[5] = vget_low_u8(ss[5]);
- s[6] = vget_low_u8(ss[6]);
- s[7] = vget_low_u8(ss[7]);
- d[0] = scale_filter_8(s, filters);
-
- s[0] = vget_high_u8(ss[0]);
- s[1] = vget_high_u8(ss[1]);
- s[2] = vget_high_u8(ss[2]);
- s[3] = vget_high_u8(ss[3]);
- s[4] = vget_high_u8(ss[4]);
- s[5] = vget_high_u8(ss[5]);
- s[6] = vget_high_u8(ss[6]);
- s[7] = vget_high_u8(ss[7]);
- d[1] = scale_filter_8(s, filters);
- vst1q_u8(&dst[x], vcombine_u8(d[0], d[1]));
- src_y += 16;
- x += 16;
- } while (x < w);
- } else {
- memcpy(dst, &src_y[3 * src_stride], w);
- }
- dst += dst_stride;
- y_q4 += y_step_q4;
- } while (--y);
-}
-
-void aom_scaled_2d_neon(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst,
- ptrdiff_t dst_stride, const InterpKernel *filter,
- int x0_q4, int x_step_q4, int y0_q4, int y_step_q4,
- int w, int h) {
- // Note: Fixed size intermediate buffer, temp, places limits on parameters.
- // 2d filtering proceeds in 2 steps:
- // (1) Interpolate horizontally into an intermediate buffer, temp.
- // (2) Interpolate temp vertically to derive the sub-pixel result.
- // Deriving the maximum number of rows in the temp buffer (135):
- // --Smallest scaling factor is x1/2 ==> y_step_q4 = 32 (Normative).
- // --Largest block size is 64x64 pixels.
- // --64 rows in the downscaled frame span a distance of (64 - 1) * 32 in the
- // original frame (in 1/16th pixel units).
- // --Must round-up because block may be located at sub-pixel position.
- // --Require an additional SUBPEL_TAPS rows for the 8-tap filter tails.
- // --((64 - 1) * 32 + 15) >> 4 + 8 = 135.
- // --Require an additional 8 rows for the horiz_w8 transpose tail.
- // When calling in frame scaling function, the smallest scaling factor is x1/4
- // ==> y_step_q4 = 64. Since w and h are at most 16, the temp buffer is still
- // big enough.
- DECLARE_ALIGNED(16, uint8_t, temp[(135 + 8) * 64]);
- const int intermediate_height =
- (((h - 1) * y_step_q4 + y0_q4) >> SUBPEL_BITS) + SUBPEL_TAPS;
-
- assert(w <= 64);
- assert(h <= 64);
- assert(y_step_q4 <= 32 || (y_step_q4 <= 64 && h <= 32));
- assert(x_step_q4 <= 64);
-
- if (w >= 8) {
- scaledconvolve_horiz_w8(src - src_stride * (SUBPEL_TAPS / 2 - 1),
- src_stride, temp, 64, filter, x0_q4, x_step_q4, w,
- intermediate_height);
+ src += 2 * src_stride;
+ dst += 2 * dst_stride;
+ h -= 2;
+ } while (h != 0);
} else {
- scaledconvolve_horiz_w4(src - src_stride * (SUBPEL_TAPS / 2 - 1),
- src_stride, temp, 64, filter, x0_q4, x_step_q4, w,
- intermediate_height);
+ do {
+ const uint8_t *src_ptr = src;
+ uint8_t *dst_ptr = dst;
+ int width = w;
+
+ do {
+ uint8x16_t s0 = vld1q_u8(src_ptr);
+ uint8x16_t s1 = vld1q_u8(src_ptr + 1);
+
+ uint8x16_t d0 = vrhaddq_u8(s0, s1);
+
+ vst1q_u8(dst_ptr, d0);
+
+ src_ptr += 16;
+ dst_ptr += 16;
+ width -= 16;
+ } while (width != 0);
+ src += src_stride;
+ dst += dst_stride;
+ } while (--h != 0);
+ }
+}
+
+void av1_convolve_y_sr_intrabc_neon(const uint8_t *src, int src_stride,
+ uint8_t *dst, int dst_stride, int w, int h,
+ const InterpFilterParams *filter_params_y,
+ const int subpel_y_qn) {
+ assert(subpel_y_qn == 8);
+ assert(filter_params_y->taps == 2);
+ (void)filter_params_y;
+ (void)subpel_y_qn;
+
+ if (w <= 4) {
+ do {
+ uint8x8_t s0 = load_unaligned_u8_4x1(src);
+ uint8x8_t s1 = load_unaligned_u8_4x1(src + src_stride);
+ uint8x8_t s2 = load_unaligned_u8_4x1(src + 2 * src_stride);
+
+ uint8x8_t d0 = vrhadd_u8(s0, s1);
+ uint8x8_t d1 = vrhadd_u8(s1, s2);
+
+ if (w == 2) {
+ store_u8_2x1(dst + 0 * dst_stride, d0, 0);
+ store_u8_2x1(dst + 1 * dst_stride, d1, 0);
+ } else {
+ store_u8_4x1(dst + 0 * dst_stride, d0, 0);
+ store_u8_4x1(dst + 1 * dst_stride, d1, 0);
+ }
+
+ src += 2 * src_stride;
+ dst += 2 * dst_stride;
+ h -= 2;
+ } while (h != 0);
+ } else if (w == 8) {
+ do {
+ uint8x8_t s0 = vld1_u8(src);
+ uint8x8_t s1 = vld1_u8(src + src_stride);
+ uint8x8_t s2 = vld1_u8(src + 2 * src_stride);
+
+ uint8x8_t d0 = vrhadd_u8(s0, s1);
+ uint8x8_t d1 = vrhadd_u8(s1, s2);
+
+ vst1_u8(dst, d0);
+ vst1_u8(dst + dst_stride, d1);
+
+ src += 2 * src_stride;
+ dst += 2 * dst_stride;
+ h -= 2;
+ } while (h != 0);
+ } else {
+ do {
+ const uint8_t *src_ptr = src;
+ uint8_t *dst_ptr = dst;
+ int height = h;
+
+ do {
+ uint8x16_t s0 = vld1q_u8(src_ptr);
+ uint8x16_t s1 = vld1q_u8(src_ptr + src_stride);
+
+ uint8x16_t d0 = vrhaddq_u8(s0, s1);
+
+ vst1q_u8(dst_ptr, d0);
+
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ } while (--height != 0);
+ src += 16;
+ dst += 16;
+ w -= 16;
+ } while (w != 0);
+ }
+}
+
+void av1_convolve_2d_sr_intrabc_neon(const uint8_t *src, int src_stride,
+ uint8_t *dst, int dst_stride, int w, int h,
+ const InterpFilterParams *filter_params_x,
+ const InterpFilterParams *filter_params_y,
+ const int subpel_x_qn,
+ const int subpel_y_qn,
+ ConvolveParams *conv_params) {
+ assert(subpel_x_qn == 8);
+ assert(subpel_y_qn == 8);
+ assert(filter_params_x->taps == 2 && filter_params_y->taps == 2);
+ assert((conv_params->round_0 + conv_params->round_1) == 2 * FILTER_BITS);
+ (void)filter_params_x;
+ (void)subpel_x_qn;
+ (void)filter_params_y;
+ (void)subpel_y_qn;
+ (void)conv_params;
+
+ uint16_t im_block[(MAX_SB_SIZE + MAX_FILTER_TAP - 1) * MAX_SB_SIZE];
+ int im_h = h + 1;
+ int im_stride = w;
+ assert(w <= MAX_SB_SIZE && h <= MAX_SB_SIZE);
+
+ uint16_t *im = im_block;
+
+ // Horizontal filter.
+ if (w <= 4) {
+ do {
+ uint8x8_t s0 = vld1_u8(src);
+ uint8x8_t s1 = vld1_u8(src + 1);
+
+ uint16x4_t sum = vget_low_u16(vaddl_u8(s0, s1));
+
+ // Safe to store the whole vector, the im buffer is big enough.
+ vst1_u16(im, sum);
+
+ src += src_stride;
+ im += im_stride;
+ } while (--im_h != 0);
+ } else {
+ do {
+ const uint8_t *src_ptr = src;
+ uint16_t *im_ptr = im;
+ int width = w;
+
+ do {
+ uint8x8_t s0 = vld1_u8(src_ptr);
+ uint8x8_t s1 = vld1_u8(src_ptr + 1);
+
+ uint16x8_t sum = vaddl_u8(s0, s1);
+
+ vst1q_u16(im_ptr, sum);
+
+ src_ptr += 8;
+ im_ptr += 8;
+ width -= 8;
+ } while (width != 0);
+ src += src_stride;
+ im += im_stride;
+ } while (--im_h != 0);
}
- if (w >= 16) {
- scaledconvolve_vert_w16(temp + 64 * (SUBPEL_TAPS / 2 - 1), 64, dst,
- dst_stride, filter, y0_q4, y_step_q4, w, h);
- } else if (w == 8) {
- scaledconvolve_vert_w8(temp + 64 * (SUBPEL_TAPS / 2 - 1), 64, dst,
- dst_stride, filter, y0_q4, y_step_q4, w, h);
+ im = im_block;
+
+ // Vertical filter.
+ if (w <= 4) {
+ do {
+ uint16x4_t s0 = vld1_u16(im);
+ uint16x4_t s1 = vld1_u16(im + im_stride);
+ uint16x4_t s2 = vld1_u16(im + 2 * im_stride);
+
+ uint16x4_t sum0 = vadd_u16(s0, s1);
+ uint16x4_t sum1 = vadd_u16(s1, s2);
+
+ uint8x8_t d01 = vqrshrn_n_u16(vcombine_u16(sum0, sum1), 2);
+
+ if (w == 2) {
+ store_u8_2x1(dst + 0 * dst_stride, d01, 0);
+ store_u8_2x1(dst + 1 * dst_stride, d01, 2);
+ } else {
+ store_u8_4x1(dst + 0 * dst_stride, d01, 0);
+ store_u8_4x1(dst + 1 * dst_stride, d01, 1);
+ }
+
+ im += 2 * im_stride;
+ dst += 2 * dst_stride;
+ h -= 2;
+ } while (h != 0);
} else {
- scaledconvolve_vert_w4(temp + 64 * (SUBPEL_TAPS / 2 - 1), 64, dst,
- dst_stride, filter, y0_q4, y_step_q4, w, h);
+ do {
+ uint16_t *im_ptr = im;
+ uint8_t *dst_ptr = dst;
+ int height = h;
+
+ do {
+ uint16x8_t s0 = vld1q_u16(im_ptr);
+ uint16x8_t s1 = vld1q_u16(im_ptr + im_stride);
+
+ uint16x8_t sum = vaddq_u16(s0, s1);
+ uint8x8_t d0 = vqrshrn_n_u16(sum, 2);
+
+ vst1_u8(dst_ptr, d0);
+
+ im_ptr += im_stride;
+ dst_ptr += dst_stride;
+ } while (--height != 0);
+ im += 8;
+ dst += 8;
+ w -= 8;
+ } while (w != 0);
}
}
diff --git a/av1/common/arm/convolve_neon.h b/av1/common/arm/convolve_neon.h
index 14a6ebe..6b8edf8 100644
--- a/av1/common/arm/convolve_neon.h
+++ b/av1/common/arm/convolve_neon.h
@@ -15,549 +15,530 @@
#include "config/aom_config.h"
-#define HORIZ_EXTRA_ROWS ((SUBPEL_TAPS + 7) & ~0x07)
+#include "aom_dsp/arm/mem_neon.h"
+#include "av1/common/convolve.h"
+#include "av1/common/filter.h"
-static INLINE int16x4_t convolve8_4(const int16x4_t s0, const int16x4_t s1,
- const int16x4_t s2, const int16x4_t s3,
- const int16x4_t s4, const int16x4_t s5,
- const int16x4_t s6, const int16x4_t s7,
- const int16x8_t filter) {
- const int16x4_t filter_lo = vget_low_s16(filter);
- const int16x4_t filter_hi = vget_high_s16(filter);
- int16x4_t sum;
-
- sum = vmul_lane_s16(s0, filter_lo, 0);
- sum = vmla_lane_s16(sum, s1, filter_lo, 1);
- sum = vmla_lane_s16(sum, s2, filter_lo, 2);
- sum = vmla_lane_s16(sum, s5, filter_hi, 1);
- sum = vmla_lane_s16(sum, s6, filter_hi, 2);
- sum = vmla_lane_s16(sum, s7, filter_hi, 3);
- sum = vqadd_s16(sum, vmul_lane_s16(s3, filter_lo, 3));
- sum = vqadd_s16(sum, vmul_lane_s16(s4, filter_hi, 0));
- return sum;
-}
-
-static INLINE uint8x8_t convolve8_8(const int16x8_t s0, const int16x8_t s1,
- const int16x8_t s2, const int16x8_t s3,
- const int16x8_t s4, const int16x8_t s5,
- const int16x8_t s6, const int16x8_t s7,
- const int16x8_t filter) {
- const int16x4_t filter_lo = vget_low_s16(filter);
- const int16x4_t filter_hi = vget_high_s16(filter);
- int16x8_t sum;
-
- sum = vmulq_lane_s16(s0, filter_lo, 0);
- sum = vmlaq_lane_s16(sum, s1, filter_lo, 1);
- sum = vmlaq_lane_s16(sum, s2, filter_lo, 2);
- sum = vmlaq_lane_s16(sum, s5, filter_hi, 1);
- sum = vmlaq_lane_s16(sum, s6, filter_hi, 2);
- sum = vmlaq_lane_s16(sum, s7, filter_hi, 3);
- sum = vqaddq_s16(sum, vmulq_lane_s16(s3, filter_lo, 3));
- sum = vqaddq_s16(sum, vmulq_lane_s16(s4, filter_hi, 0));
- return vqrshrun_n_s16(sum, 7);
-}
-
-static INLINE uint8x8_t scale_filter_8(const uint8x8_t *const s,
- const int16x8_t filter) {
- int16x8_t ss[8];
-
- ss[0] = vreinterpretq_s16_u16(vmovl_u8(s[0]));
- ss[1] = vreinterpretq_s16_u16(vmovl_u8(s[1]));
- ss[2] = vreinterpretq_s16_u16(vmovl_u8(s[2]));
- ss[3] = vreinterpretq_s16_u16(vmovl_u8(s[3]));
- ss[4] = vreinterpretq_s16_u16(vmovl_u8(s[4]));
- ss[5] = vreinterpretq_s16_u16(vmovl_u8(s[5]));
- ss[6] = vreinterpretq_s16_u16(vmovl_u8(s[6]));
- ss[7] = vreinterpretq_s16_u16(vmovl_u8(s[7]));
-
- return convolve8_8(ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7],
- filter);
-}
-
-static INLINE uint8x8_t wiener_convolve8_vert_4x8(
- const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
- const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
- const int16x8_t s6, int16_t *filter_y, const int bd,
- const int round1_bits) {
- int16x8_t ss0, ss1, ss2;
- int32x4_t sum0, sum1;
- int16x8_t tmp;
- uint8x8_t res;
-
- const int32_t round_const = (1 << (bd + round1_bits - 1));
- const int32x4_t round_bits = vdupq_n_s32(-round1_bits);
- const int32x4_t round_vec = vdupq_n_s32(round_const);
- const int16x4_t filter = vld1_s16(filter_y);
-
- ss0 = vaddq_s16(s0, s6);
- ss1 = vaddq_s16(s1, s5);
- ss2 = vaddq_s16(s2, s4);
-
- sum0 = vmull_lane_s16(vget_low_s16(ss0), filter, 0);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(ss1), filter, 1);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(ss2), filter, 2);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s3), filter, 3);
-
- sum1 = vmull_lane_s16(vget_high_s16(ss0), filter, 0);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(ss1), filter, 1);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(ss2), filter, 2);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s3), filter, 3);
-
- sum0 = vsubq_s32(sum0, round_vec);
- sum1 = vsubq_s32(sum1, round_vec);
-
- /* right shift & rounding */
- sum0 = vrshlq_s32(sum0, round_bits);
- sum1 = vrshlq_s32(sum1, round_bits);
-
- /* from int32x4_t to uint8x8_t */
- tmp = vcombine_s16(vmovn_s32(sum0), vmovn_s32(sum1));
- res = vqmovun_s16(tmp);
-
- return res;
-}
-
-static INLINE uint16x8_t wiener_convolve8_horiz_8x8(
- const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
- const int16x8_t s3, int16_t *filter_x, const int bd,
- const int round0_bits) {
- int16x8_t sum;
- uint16x8_t res;
- int32x4_t sum_0, sum_1;
- int32x4_t s3_0, s3_1;
- const int32_t round_const_0 = (1 << (bd + FILTER_BITS - 1));
- const int32_t round_const_1 = (1 << (bd + 1 + FILTER_BITS - round0_bits)) - 1;
-
- /* for the purpose of right shift by { conv_params->round_0 } */
- const int32x4_t round_bits = vdupq_n_s32(-round0_bits);
-
- const int32x4_t round_vec_0 = vdupq_n_s32(round_const_0);
- const int32x4_t round_vec_1 = vdupq_n_s32(round_const_1);
- const int16x4_t filter = vld1_s16(filter_x);
-
- sum = vmulq_lane_s16(s0, filter, 0);
- sum = vmlaq_lane_s16(sum, s1, filter, 1);
- sum = vmlaq_lane_s16(sum, s2, filter, 2);
-
- /* sum from 16x8 to 2 32x4 registers */
- sum_0 = vmovl_s16(vget_low_s16(sum));
- sum_1 = vmovl_s16(vget_high_s16(sum));
-
- /* s[3]*128 -- and filter coef max can be 128
- * then max value possible = 128*128*255 exceeding 16 bit
- */
-
- s3_0 = vmull_lane_s16(vget_low_s16(s3), filter, 3);
- s3_1 = vmull_lane_s16(vget_high_s16(s3), filter, 3);
- sum_0 = vaddq_s32(sum_0, s3_0);
- sum_1 = vaddq_s32(sum_1, s3_1);
-
- /* Add the constant value */
- sum_0 = vaddq_s32(sum_0, round_vec_0);
- sum_1 = vaddq_s32(sum_1, round_vec_0);
-
- /* right shift & rounding & saturating */
- sum_0 = vqrshlq_s32(sum_0, round_bits);
- sum_1 = vqrshlq_s32(sum_1, round_bits);
-
- /* Clipping to max value */
- sum_0 = vminq_s32(sum_0, round_vec_1);
- sum_1 = vminq_s32(sum_1, round_vec_1);
-
- res = vcombine_u16(vqmovun_s32(sum_0), vqmovun_s32(sum_1));
- return res;
-}
-
-static INLINE uint16x4_t wiener_convolve8_horiz_4x8(
- const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
- const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
- const int16x4_t s6, int16_t *filter_x, const int bd,
- const int round0_bits) {
- uint16x4_t res;
- int32x4_t sum_0, s3_0;
- int16x4_t sum, temp0, temp1, temp2;
-
- const int32_t round_const_0 = (1 << (bd + FILTER_BITS - 1));
- const int32_t round_const_1 = (1 << (bd + 1 + FILTER_BITS - round0_bits)) - 1;
- const int32x4_t round_bits = vdupq_n_s32(-round0_bits);
- const int32x4_t round_vec_0 = vdupq_n_s32(round_const_0);
- const int32x4_t round_vec_1 = vdupq_n_s32(round_const_1);
- const int16x4_t filter = vld1_s16(filter_x);
-
- temp0 = vadd_s16(s0, s6);
- temp1 = vadd_s16(s1, s5);
- temp2 = vadd_s16(s2, s4);
-
- sum = vmul_lane_s16(temp0, filter, 0);
- sum = vmla_lane_s16(sum, temp1, filter, 1);
- sum = vmla_lane_s16(sum, temp2, filter, 2);
- sum_0 = vmovl_s16(sum);
-
- /* s[3]*128 -- and filter coff max can be 128.
- * then max value possible = 128*128*255 Therefore, 32 bits are required to
- * hold the result.
- */
- s3_0 = vmull_lane_s16(s3, filter, 3);
- sum_0 = vaddq_s32(sum_0, s3_0);
-
- sum_0 = vaddq_s32(sum_0, round_vec_0);
- sum_0 = vrshlq_s32(sum_0, round_bits);
-
- sum_0 = vminq_s32(sum_0, round_vec_1);
- res = vqmovun_s32(sum_0);
- return res;
-}
-
-static INLINE int16x8_t convolve8_8x8_s16(
- const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
- const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
- const int16x8_t s6, const int16x8_t s7, const int16x8_t filter,
- const int16x8_t horiz_const, const int16x8_t shift_round_0) {
- const int16x4_t filter_lo = vget_low_s16(filter);
- const int16x4_t filter_hi = vget_high_s16(filter);
- int16x8_t sum;
-
- sum = horiz_const;
- sum = vmlaq_lane_s16(sum, s0, filter_lo, 0);
- sum = vmlaq_lane_s16(sum, s1, filter_lo, 1);
- sum = vmlaq_lane_s16(sum, s2, filter_lo, 2);
- sum = vmlaq_lane_s16(sum, s3, filter_lo, 3);
- sum = vmlaq_lane_s16(sum, s4, filter_hi, 0);
- sum = vmlaq_lane_s16(sum, s5, filter_hi, 1);
- sum = vmlaq_lane_s16(sum, s6, filter_hi, 2);
- sum = vmlaq_lane_s16(sum, s7, filter_hi, 3);
-
- sum = vqrshlq_s16(sum, shift_round_0);
-
- return sum;
-}
-
-// clang versions < 16 did not include the dotprod feature for Arm architecture
-// versions that should have it by default, e.g., armv8.6-a.
-#if AOM_ARCH_AARCH64 && \
- (defined(__ARM_FEATURE_DOTPROD) || defined(__ARM_FEATURE_MATMUL_INT8))
-
-DECLARE_ALIGNED(16, static const uint8_t, dot_prod_permute_tbl[48]) = {
- 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6,
- 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10,
- 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
-};
-
-#endif // AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_DOTPROD)
-
-#if AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_MATMUL_INT8)
-
-static INLINE int16x8_t convolve8_x_8_usdot(uint8x16_t samples,
- const int8x8_t filters,
- const uint8x16x3_t permute_tbl,
- const int32x4_t horiz_const) {
- uint8x16_t permuted_samples[3];
- int32x4_t sum[2];
-
- /* Permute samples ready for dot product. */
- /* { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } */
- permuted_samples[0] = vqtbl1q_u8(samples, permute_tbl.val[0]);
- /* { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } */
- permuted_samples[1] = vqtbl1q_u8(samples, permute_tbl.val[1]);
- /* { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 } */
- permuted_samples[2] = vqtbl1q_u8(samples, permute_tbl.val[2]);
-
- /* First 4 output values. */
- sum[0] = vusdotq_lane_s32(horiz_const, permuted_samples[0], filters, 0);
- sum[0] = vusdotq_lane_s32(sum[0], permuted_samples[1], filters, 1);
- /* Second 4 output values. */
- sum[1] = vusdotq_lane_s32(horiz_const, permuted_samples[1], filters, 0);
- sum[1] = vusdotq_lane_s32(sum[1], permuted_samples[2], filters, 1);
-
- return vcombine_s16(vmovn_s32(sum[0]), vmovn_s32(sum[1]));
-}
-
-static INLINE int16x8_t convolve8_horiz_8_usdot(uint8x16_t samples,
- const int8x8_t filters,
- const uint8x16x3_t permute_tbl,
- const int32x4_t horiz_const) {
- uint8x16_t permuted_samples[3];
- int32x4_t sum[2];
-
- /* Permute samples ready for dot product. */
- /* { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } */
- permuted_samples[0] = vqtbl1q_u8(samples, permute_tbl.val[0]);
- /* { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } */
- permuted_samples[1] = vqtbl1q_u8(samples, permute_tbl.val[1]);
- /* { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 } */
- permuted_samples[2] = vqtbl1q_u8(samples, permute_tbl.val[2]);
-
- /* First 4 output values. */
- sum[0] = vusdotq_lane_s32(horiz_const, permuted_samples[0], filters, 0);
- sum[0] = vusdotq_lane_s32(sum[0], permuted_samples[1], filters, 1);
- /* Second 4 output values. */
- sum[1] = vusdotq_lane_s32(horiz_const, permuted_samples[1], filters, 0);
- sum[1] = vusdotq_lane_s32(sum[1], permuted_samples[2], filters, 1);
-
- /* Narrow and re-pack. */
- // We halved the convolution filter values so -1 from the right shift.
- return vcombine_s16(vshrn_n_s32(sum[0], ROUND0_BITS - 1),
- vshrn_n_s32(sum[1], ROUND0_BITS - 1));
-}
-
-static INLINE int32x4_t convolve8_4_usdot(uint8x16_t samples,
- const int8x8_t filters,
- const uint8x16x2_t permute_tbl,
- const int32x4_t horiz_const) {
- uint8x16_t permuted_samples[2];
- int32x4_t sum;
-
- /* Permute samples ready for dot product. */
- /* { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } */
- permuted_samples[0] = vqtbl1q_u8(samples, permute_tbl.val[0]);
- /* { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } */
- permuted_samples[1] = vqtbl1q_u8(samples, permute_tbl.val[1]);
-
- /* First 4 output values. */
- sum = vusdotq_lane_s32(horiz_const, permuted_samples[0], filters, 0);
- sum = vusdotq_lane_s32(sum, permuted_samples[1], filters, 1);
-
- /* Narrowing and packing is performed by the caller. */
- return sum;
-}
-
-#elif AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_DOTPROD)
-
-static INLINE int16x8_t convolve8_horiz_8_sdot(uint8x16_t samples,
- const int8x8_t filters,
- const int32x4_t correction,
- const uint8x16_t range_limit,
- const uint8x16x3_t permute_tbl) {
- int8x16_t clamped_samples, permuted_samples[3];
- int32x4_t sum[2];
-
- /* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
- clamped_samples = vreinterpretq_s8_u8(vsubq_u8(samples, range_limit));
-
- /* Permute samples ready for dot product. */
- /* { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } */
- permuted_samples[0] = vqtbl1q_s8(clamped_samples, permute_tbl.val[0]);
- /* { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } */
- permuted_samples[1] = vqtbl1q_s8(clamped_samples, permute_tbl.val[1]);
- /* { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 } */
- permuted_samples[2] = vqtbl1q_s8(clamped_samples, permute_tbl.val[2]);
-
- /* Accumulate dot product into 'correction' to account for range clamp. */
- /* First 4 output values. */
- sum[0] = vdotq_lane_s32(correction, permuted_samples[0], filters, 0);
- sum[0] = vdotq_lane_s32(sum[0], permuted_samples[1], filters, 1);
- /* Second 4 output values. */
- sum[1] = vdotq_lane_s32(correction, permuted_samples[1], filters, 0);
- sum[1] = vdotq_lane_s32(sum[1], permuted_samples[2], filters, 1);
-
- /* Narrow and re-pack. */
- /* We halved the convolution filter values so -1 from the right shift. */
- return vcombine_s16(vshrn_n_s32(sum[0], ROUND0_BITS - 1),
- vshrn_n_s32(sum[1], ROUND0_BITS - 1));
-}
-
-static INLINE int32x4_t convolve8_4_sdot(uint8x16_t samples,
- const int8x8_t filters,
- const int32x4_t correction,
- const uint8x16_t range_limit,
- const uint8x16x2_t permute_tbl) {
- int8x16_t clamped_samples, permuted_samples[2];
- int32x4_t sum;
-
- /* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
- clamped_samples = vreinterpretq_s8_u8(vsubq_u8(samples, range_limit));
-
- /* Permute samples ready for dot product. */
- /* { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } */
- permuted_samples[0] = vqtbl1q_s8(clamped_samples, permute_tbl.val[0]);
- /* { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } */
- permuted_samples[1] = vqtbl1q_s8(clamped_samples, permute_tbl.val[1]);
-
- /* Accumulate dot product into 'correction' to account for range clamp. */
- sum = vdotq_lane_s32(correction, permuted_samples[0], filters, 0);
- sum = vdotq_lane_s32(sum, permuted_samples[1], filters, 1);
-
- /* Narrowing and packing is performed by the caller. */
- return sum;
-}
-
-static INLINE int16x8_t convolve8_8_sdot(uint8x16_t samples,
- const int8x8_t filters,
- const int32x4_t correction,
- const uint8x16_t range_limit,
- const uint8x16x3_t permute_tbl,
- const int16x8_t shift_round_0) {
- int8x16_t clamped_samples, permuted_samples[3];
- int32x4_t sum0, sum1;
- int16x8_t sum;
-
- /* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
- clamped_samples = vreinterpretq_s8_u8(vsubq_u8(samples, range_limit));
-
- /* Permute samples ready for dot product. */
- /* { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } */
- permuted_samples[0] = vqtbl1q_s8(clamped_samples, permute_tbl.val[0]);
- /* { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } */
- permuted_samples[1] = vqtbl1q_s8(clamped_samples, permute_tbl.val[1]);
- /* { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 } */
- permuted_samples[2] = vqtbl1q_s8(clamped_samples, permute_tbl.val[2]);
-
- /* Accumulate dot product into 'correction' to account for range clamp. */
- /* First 4 output values. */
- sum0 = vdotq_lane_s32(correction, permuted_samples[0], filters, 0);
- sum0 = vdotq_lane_s32(sum0, permuted_samples[1], filters, 1);
- /* Second 4 output values. */
- sum1 = vdotq_lane_s32(correction, permuted_samples[1], filters, 0);
- sum1 = vdotq_lane_s32(sum1, permuted_samples[2], filters, 1);
-
- /* Narrow and re-pack. */
- sum = vcombine_s16(vmovn_s32(sum0), vmovn_s32(sum1));
- return vqrshlq_s16(sum, shift_round_0);
-}
-
-static INLINE int16x8_t convolve8_x_8_sdot(uint8x16_t samples,
- const int8x8_t filters,
- const int32x4_t correction,
- const uint8x16_t range_limit,
- const uint8x16x3_t permute_tbl) {
- int8x16_t clamped_samples, permuted_samples[3];
- int32x4_t sum[2];
-
- /* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
- clamped_samples = vreinterpretq_s8_u8(vsubq_u8(samples, range_limit));
-
- /* Permute samples ready for dot product. */
- /* { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } */
- permuted_samples[0] = vqtbl1q_s8(clamped_samples, permute_tbl.val[0]);
- /* { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } */
- permuted_samples[1] = vqtbl1q_s8(clamped_samples, permute_tbl.val[1]);
- /* { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 } */
- permuted_samples[2] = vqtbl1q_s8(clamped_samples, permute_tbl.val[2]);
-
- /* Accumulate dot product into 'correction' to account for range clamp. */
- /* First 4 output values. */
- sum[0] = vdotq_lane_s32(correction, permuted_samples[0], filters, 0);
- sum[0] = vdotq_lane_s32(sum[0], permuted_samples[1], filters, 1);
- /* Second 4 output values. */
- sum[1] = vdotq_lane_s32(correction, permuted_samples[1], filters, 0);
- sum[1] = vdotq_lane_s32(sum[1], permuted_samples[2], filters, 1);
-
- /* Narrow and re-pack. */
- return vcombine_s16(vmovn_s32(sum[0]), vmovn_s32(sum[1]));
-}
-
-#endif // AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_DOTPROD)
-
-static INLINE int16x4_t convolve8_4x4_s16(
- const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
- const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
- const int16x4_t s6, const int16x4_t s7, const int16x8_t filter,
- const int16x4_t horiz_const, const int16x4_t shift_round_0) {
- const int16x4_t filter_lo = vget_low_s16(filter);
- const int16x4_t filter_hi = vget_high_s16(filter);
- int16x4_t sum;
-
- sum = horiz_const;
- sum = vmla_lane_s16(sum, s0, filter_lo, 0);
- sum = vmla_lane_s16(sum, s1, filter_lo, 1);
- sum = vmla_lane_s16(sum, s2, filter_lo, 2);
- sum = vmla_lane_s16(sum, s3, filter_lo, 3);
- sum = vmla_lane_s16(sum, s4, filter_hi, 0);
- sum = vmla_lane_s16(sum, s5, filter_hi, 1);
- sum = vmla_lane_s16(sum, s6, filter_hi, 2);
- sum = vmla_lane_s16(sum, s7, filter_hi, 3);
-
- sum = vqrshl_s16(sum, shift_round_0);
-
- return sum;
-}
-
-static INLINE int16x4_t convolve6_4x4(const int16x4_t s0, const int16x4_t s1,
- const int16x4_t s2, const int16x4_t s3,
- const int16x4_t s4, const int16x4_t s5,
- const int16x8_t y_filter_0_7) {
+static INLINE int32x4_t
+convolve12_4_2d_v(const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
+ const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
+ const int16x4_t s6, const int16x4_t s7, const int16x4_t s8,
+ const int16x4_t s9, const int16x4_t s10, const int16x4_t s11,
+ const int16x8_t y_filter_0_7, const int16x4_t y_filter_8_11) {
const int16x4_t y_filter_0_3 = vget_low_s16(y_filter_0_7);
const int16x4_t y_filter_4_7 = vget_high_s16(y_filter_0_7);
- int16x4_t sum;
- // Filter values at indices 0 and 7 are 0.
- sum = vmul_lane_s16(s0, y_filter_0_3, 1);
- sum = vmla_lane_s16(sum, s1, y_filter_0_3, 2);
- sum = vmla_lane_s16(sum, s2, y_filter_0_3, 3);
- sum = vmla_lane_s16(sum, s3, y_filter_4_7, 0);
- sum = vmla_lane_s16(sum, s4, y_filter_4_7, 1);
- sum = vmla_lane_s16(sum, s5, y_filter_4_7, 2);
+ int32x4_t sum = vmull_lane_s16(s0, y_filter_0_3, 0);
+ sum = vmlal_lane_s16(sum, s1, y_filter_0_3, 1);
+ sum = vmlal_lane_s16(sum, s2, y_filter_0_3, 2);
+ sum = vmlal_lane_s16(sum, s3, y_filter_0_3, 3);
+ sum = vmlal_lane_s16(sum, s4, y_filter_4_7, 0);
+ sum = vmlal_lane_s16(sum, s5, y_filter_4_7, 1);
+ sum = vmlal_lane_s16(sum, s6, y_filter_4_7, 2);
+ sum = vmlal_lane_s16(sum, s7, y_filter_4_7, 3);
+ sum = vmlal_lane_s16(sum, s8, y_filter_8_11, 0);
+ sum = vmlal_lane_s16(sum, s9, y_filter_8_11, 1);
+ sum = vmlal_lane_s16(sum, s10, y_filter_8_11, 2);
+ sum = vmlal_lane_s16(sum, s11, y_filter_8_11, 3);
return sum;
}
-static INLINE int16x8_t convolve6_8x4(const int16x8_t s0, const int16x8_t s1,
- const int16x8_t s2, const int16x8_t s3,
- const int16x8_t s4, const int16x8_t s5,
- const int16x8_t y_filters) {
- const int16x4_t y_filter_lo = vget_low_s16(y_filters);
- const int16x4_t y_filter_hi = vget_high_s16(y_filters);
- int16x8_t sum;
+static INLINE uint8x8_t
+convolve12_8_2d_v(const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
+ const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t s6, const int16x8_t s7, const int16x8_t s8,
+ const int16x8_t s9, const int16x8_t s10, const int16x8_t s11,
+ const int16x8_t y_filter_0_7, const int16x4_t y_filter_8_11,
+ const int16x8_t sub_const) {
+ const int16x4_t y_filter_0_3 = vget_low_s16(y_filter_0_7);
+ const int16x4_t y_filter_4_7 = vget_high_s16(y_filter_0_7);
- // Filter values at indices 0 and 7 are 0.
- sum = vmulq_lane_s16(s0, y_filter_lo, 1);
- sum = vmlaq_lane_s16(sum, s1, y_filter_lo, 2);
- sum = vmlaq_lane_s16(sum, s2, y_filter_lo, 3);
- sum = vmlaq_lane_s16(sum, s3, y_filter_hi, 0);
- sum = vmlaq_lane_s16(sum, s4, y_filter_hi, 1);
- sum = vmlaq_lane_s16(sum, s5, y_filter_hi, 2);
+ int32x4_t sum0 = vmull_lane_s16(vget_low_s16(s0), y_filter_0_3, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s1), y_filter_0_3, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s2), y_filter_0_3, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s3), y_filter_0_3, 3);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s4), y_filter_4_7, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s5), y_filter_4_7, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s6), y_filter_4_7, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s7), y_filter_4_7, 3);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s8), y_filter_8_11, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s9), y_filter_8_11, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s10), y_filter_8_11, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s11), y_filter_8_11, 3);
- return sum;
+ int32x4_t sum1 = vmull_lane_s16(vget_high_s16(s0), y_filter_0_3, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s1), y_filter_0_3, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s2), y_filter_0_3, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s3), y_filter_0_3, 3);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s4), y_filter_4_7, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s5), y_filter_4_7, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s6), y_filter_4_7, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s7), y_filter_4_7, 3);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s8), y_filter_8_11, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s9), y_filter_8_11, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s10), y_filter_8_11, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s11), y_filter_8_11, 3);
+
+ int16x8_t res =
+ vcombine_s16(vqrshrn_n_s32(sum0, 2 * FILTER_BITS - ROUND0_BITS),
+ vqrshrn_n_s32(sum1, 2 * FILTER_BITS - ROUND0_BITS));
+ res = vsubq_s16(res, sub_const);
+
+ return vqmovun_s16(res);
}
-#if !(AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_DOTPROD))
+static INLINE void convolve_2d_sr_vert_12tap_neon(
+ int16_t *src_ptr, int src_stride, uint8_t *dst_ptr, int dst_stride, int w,
+ int h, const int16x8_t y_filter_0_7, const int16x4_t y_filter_8_11) {
+ const int bd = 8;
+ const int16x8_t sub_const = vdupq_n_s16(1 << (bd - 1));
-static INLINE int16x4_t convolve8_horiz_4x4_s16(
- const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
- const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
- const int16x4_t s6, const int16x4_t s7, const int16x8_t filter,
- const int16x4_t horiz_const) {
- const int16x4_t filter_lo = vget_low_s16(filter);
- const int16x4_t filter_hi = vget_high_s16(filter);
- int16x4_t sum;
+ if (w <= 4) {
+ int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
+ load_s16_4x11(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7,
+ &s8, &s9, &s10);
+ src_ptr += 11 * src_stride;
- sum = horiz_const;
- sum = vmla_lane_s16(sum, s0, filter_lo, 0);
- sum = vmla_lane_s16(sum, s1, filter_lo, 1);
- sum = vmla_lane_s16(sum, s2, filter_lo, 2);
- sum = vmla_lane_s16(sum, s3, filter_lo, 3);
- sum = vmla_lane_s16(sum, s4, filter_hi, 0);
- sum = vmla_lane_s16(sum, s5, filter_hi, 1);
- sum = vmla_lane_s16(sum, s6, filter_hi, 2);
- sum = vmla_lane_s16(sum, s7, filter_hi, 3);
+ do {
+ int16x4_t s11, s12, s13, s14;
+ load_s16_4x4(src_ptr, src_stride, &s11, &s12, &s13, &s14);
- // We halved the convolution filter values so -1 from the right shift.
- return vshr_n_s16(sum, ROUND0_BITS - 1);
+ int32x4_t d0 = convolve12_4_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, s8, s9,
+ s10, s11, y_filter_0_7, y_filter_8_11);
+ int32x4_t d1 = convolve12_4_2d_v(s1, s2, s3, s4, s5, s6, s7, s8, s9, s10,
+ s11, s12, y_filter_0_7, y_filter_8_11);
+ int32x4_t d2 = convolve12_4_2d_v(s2, s3, s4, s5, s6, s7, s8, s9, s10, s11,
+ s12, s13, y_filter_0_7, y_filter_8_11);
+ int32x4_t d3 =
+ convolve12_4_2d_v(s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14,
+ y_filter_0_7, y_filter_8_11);
+
+ int16x8_t dd01 =
+ vcombine_s16(vqrshrn_n_s32(d0, 2 * FILTER_BITS - ROUND0_BITS),
+ vqrshrn_n_s32(d1, 2 * FILTER_BITS - ROUND0_BITS));
+ int16x8_t dd23 =
+ vcombine_s16(vqrshrn_n_s32(d2, 2 * FILTER_BITS - ROUND0_BITS),
+ vqrshrn_n_s32(d3, 2 * FILTER_BITS - ROUND0_BITS));
+
+ dd01 = vsubq_s16(dd01, sub_const);
+ dd23 = vsubq_s16(dd23, sub_const);
+
+ uint8x8_t d01 = vqmovun_s16(dd01);
+ uint8x8_t d23 = vqmovun_s16(dd23);
+
+ store_u8_4x1(dst_ptr + 0 * dst_stride, d01, 0);
+ store_u8_4x1(dst_ptr + 1 * dst_stride, d01, 1);
+ store_u8_4x1(dst_ptr + 2 * dst_stride, d23, 0);
+ store_u8_4x1(dst_ptr + 3 * dst_stride, d23, 1);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s5 = s9;
+ s6 = s10;
+ s7 = s11;
+ s8 = s12;
+ s9 = s13;
+ s10 = s14;
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ h -= 4;
+ } while (h != 0);
+
+ } else {
+ do {
+ int height = h;
+ int16_t *s = src_ptr;
+ uint8_t *d = dst_ptr;
+
+ int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
+ load_s16_8x11(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7, &s8,
+ &s9, &s10);
+ s += 11 * src_stride;
+
+ do {
+ int16x8_t s11, s12, s13, s14;
+ load_s16_8x4(s, src_stride, &s11, &s12, &s13, &s14);
+
+ uint8x8_t d0 =
+ convolve12_8_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11,
+ y_filter_0_7, y_filter_8_11, sub_const);
+ uint8x8_t d1 =
+ convolve12_8_2d_v(s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12,
+ y_filter_0_7, y_filter_8_11, sub_const);
+ uint8x8_t d2 =
+ convolve12_8_2d_v(s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12,
+ s13, y_filter_0_7, y_filter_8_11, sub_const);
+ uint8x8_t d3 =
+ convolve12_8_2d_v(s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13,
+ s14, y_filter_0_7, y_filter_8_11, sub_const);
+
+ store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s5 = s9;
+ s6 = s10;
+ s7 = s11;
+ s8 = s12;
+ s9 = s13;
+ s10 = s14;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ height -= 4;
+ } while (height != 0);
+ src_ptr += 8;
+ dst_ptr += 8;
+ w -= 8;
+ } while (w != 0);
+ }
}
-static INLINE int16x8_t convolve8_horiz_8x8_s16(
- const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
- const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
- const int16x8_t s6, const int16x8_t s7, const int16x8_t filter,
- const int16x8_t horiz_const) {
- const int16x4_t filter_lo = vget_low_s16(filter);
- const int16x4_t filter_hi = vget_high_s16(filter);
- int16x8_t sum;
+static INLINE int16x4_t convolve8_4_2d_v(const int16x4_t s0, const int16x4_t s1,
+ const int16x4_t s2, const int16x4_t s3,
+ const int16x4_t s4, const int16x4_t s5,
+ const int16x4_t s6, const int16x4_t s7,
+ const int16x8_t y_filter) {
+ const int16x4_t y_filter_lo = vget_low_s16(y_filter);
+ const int16x4_t y_filter_hi = vget_high_s16(y_filter);
- sum = horiz_const;
- sum = vmlaq_lane_s16(sum, s0, filter_lo, 0);
- sum = vmlaq_lane_s16(sum, s1, filter_lo, 1);
- sum = vmlaq_lane_s16(sum, s2, filter_lo, 2);
- sum = vmlaq_lane_s16(sum, s3, filter_lo, 3);
- sum = vmlaq_lane_s16(sum, s4, filter_hi, 0);
- sum = vmlaq_lane_s16(sum, s5, filter_hi, 1);
- sum = vmlaq_lane_s16(sum, s6, filter_hi, 2);
- sum = vmlaq_lane_s16(sum, s7, filter_hi, 3);
+ int32x4_t sum = vmull_lane_s16(s0, y_filter_lo, 0);
+ sum = vmlal_lane_s16(sum, s1, y_filter_lo, 1);
+ sum = vmlal_lane_s16(sum, s2, y_filter_lo, 2);
+ sum = vmlal_lane_s16(sum, s3, y_filter_lo, 3);
+ sum = vmlal_lane_s16(sum, s4, y_filter_hi, 0);
+ sum = vmlal_lane_s16(sum, s5, y_filter_hi, 1);
+ sum = vmlal_lane_s16(sum, s6, y_filter_hi, 2);
+ sum = vmlal_lane_s16(sum, s7, y_filter_hi, 3);
- // We halved the convolution filter values so -1 from the right shift.
- return vshrq_n_s16(sum, ROUND0_BITS - 1);
+ return vqrshrn_n_s32(sum, 2 * FILTER_BITS - ROUND0_BITS);
}
-#endif // !(AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_DOTPROD))
+static INLINE uint8x8_t convolve8_8_2d_v(const int16x8_t s0, const int16x8_t s1,
+ const int16x8_t s2, const int16x8_t s3,
+ const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t s6, const int16x8_t s7,
+ const int16x8_t y_filter,
+ const int16x8_t sub_const) {
+ const int16x4_t y_filter_lo = vget_low_s16(y_filter);
+ const int16x4_t y_filter_hi = vget_high_s16(y_filter);
+
+ int32x4_t sum0 = vmull_lane_s16(vget_low_s16(s0), y_filter_lo, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s1), y_filter_lo, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s2), y_filter_lo, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s3), y_filter_lo, 3);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s4), y_filter_hi, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s5), y_filter_hi, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s6), y_filter_hi, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s7), y_filter_hi, 3);
+
+ int32x4_t sum1 = vmull_lane_s16(vget_high_s16(s0), y_filter_lo, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s1), y_filter_lo, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s2), y_filter_lo, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s3), y_filter_lo, 3);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s4), y_filter_hi, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s5), y_filter_hi, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s6), y_filter_hi, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s7), y_filter_hi, 3);
+
+ int16x8_t res =
+ vcombine_s16(vqrshrn_n_s32(sum0, 2 * FILTER_BITS - ROUND0_BITS),
+ vqrshrn_n_s32(sum1, 2 * FILTER_BITS - ROUND0_BITS));
+ res = vsubq_s16(res, sub_const);
+
+ return vqmovun_s16(res);
+}
+
+static INLINE void convolve_2d_sr_vert_8tap_neon(int16_t *src_ptr,
+ int src_stride,
+ uint8_t *dst_ptr,
+ int dst_stride, int w, int h,
+ const int16x8_t y_filter) {
+ const int bd = 8;
+ const int16x8_t sub_const = vdupq_n_s16(1 << (bd - 1));
+
+ if (w <= 4) {
+ int16x4_t s0, s1, s2, s3, s4, s5, s6;
+ load_s16_4x7(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
+ src_ptr += 7 * src_stride;
+
+ do {
+#if AOM_ARCH_AARCH64
+ int16x4_t s7, s8, s9, s10;
+ load_s16_4x4(src_ptr, src_stride, &s7, &s8, &s9, &s10);
+
+ int16x4_t d0 = convolve8_4_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, y_filter);
+ int16x4_t d1 = convolve8_4_2d_v(s1, s2, s3, s4, s5, s6, s7, s8, y_filter);
+ int16x4_t d2 = convolve8_4_2d_v(s2, s3, s4, s5, s6, s7, s8, s9, y_filter);
+ int16x4_t d3 =
+ convolve8_4_2d_v(s3, s4, s5, s6, s7, s8, s9, s10, y_filter);
+
+ uint8x8_t d01 = vqmovun_s16(vsubq_s16(vcombine_s16(d0, d1), sub_const));
+ uint8x8_t d23 = vqmovun_s16(vsubq_s16(vcombine_s16(d2, d3), sub_const));
+
+ store_u8_4x1(dst_ptr + 0 * dst_stride, d01, 0);
+ store_u8_4x1(dst_ptr + 1 * dst_stride, d01, 1);
+ store_u8_4x1(dst_ptr + 2 * dst_stride, d23, 0);
+ store_u8_4x1(dst_ptr + 3 * dst_stride, d23, 1);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s5 = s9;
+ s6 = s10;
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ h -= 4;
+#else // !AOM_ARCH_AARCH64
+ int16x4_t s7 = vld1_s16(src_ptr);
+ int16x4_t d0 = convolve8_4_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, y_filter);
+ uint8x8_t d01 =
+ vqmovun_s16(vsubq_s16(vcombine_s16(d0, vdup_n_s16(0)), sub_const));
+
+ store_u8_4x1(dst_ptr, d01, 0);
+
+ s0 = s1;
+ s1 = s2;
+ s2 = s3;
+ s3 = s4;
+ s4 = s5;
+ s5 = s6;
+ s6 = s7;
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ h--;
+#endif // AOM_ARCH_AARCH64
+ } while (h != 0);
+ } else {
+ // Width is a multiple of 8 and height is a multiple of 4.
+ do {
+ int height = h;
+ int16_t *s = src_ptr;
+ uint8_t *d = dst_ptr;
+
+ int16x8_t s0, s1, s2, s3, s4, s5, s6;
+ load_s16_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
+ s += 7 * src_stride;
+
+ do {
+#if AOM_ARCH_AARCH64
+ int16x8_t s7, s8, s9, s10;
+ load_s16_8x4(s, src_stride, &s7, &s8, &s9, &s10);
+
+ uint8x8_t d0 = convolve8_8_2d_v(s0, s1, s2, s3, s4, s5, s6, s7,
+ y_filter, sub_const);
+ uint8x8_t d1 = convolve8_8_2d_v(s1, s2, s3, s4, s5, s6, s7, s8,
+ y_filter, sub_const);
+ uint8x8_t d2 = convolve8_8_2d_v(s2, s3, s4, s5, s6, s7, s8, s9,
+ y_filter, sub_const);
+ uint8x8_t d3 = convolve8_8_2d_v(s3, s4, s5, s6, s7, s8, s9, s10,
+ y_filter, sub_const);
+
+ store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s5 = s9;
+ s6 = s10;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ height -= 4;
+#else // !AOM_ARCH_AARCH64
+ int16x8_t s7 = vld1q_s16(s);
+ uint8x8_t d0 = convolve8_8_2d_v(s0, s1, s2, s3, s4, s5, s6, s7,
+ y_filter, sub_const);
+ vst1_u8(d, d0);
+
+ s0 = s1;
+ s1 = s2;
+ s2 = s3;
+ s3 = s4;
+ s4 = s5;
+ s5 = s6;
+ s6 = s7;
+ s += src_stride;
+ d += dst_stride;
+ height--;
+#endif // AOM_ARCH_AARCH64
+ } while (height != 0);
+ src_ptr += 8;
+ dst_ptr += 8;
+ w -= 8;
+ } while (w != 0);
+ }
+}
+
+static INLINE int16x4_t convolve6_4_2d_v(const int16x4_t s0, const int16x4_t s1,
+ const int16x4_t s2, const int16x4_t s3,
+ const int16x4_t s4, const int16x4_t s5,
+ const int16x8_t y_filter) {
+ const int16x4_t y_filter_lo = vget_low_s16(y_filter);
+ const int16x4_t y_filter_hi = vget_high_s16(y_filter);
+
+ int32x4_t sum = vmull_lane_s16(s0, y_filter_lo, 1);
+ sum = vmlal_lane_s16(sum, s1, y_filter_lo, 2);
+ sum = vmlal_lane_s16(sum, s2, y_filter_lo, 3);
+ sum = vmlal_lane_s16(sum, s3, y_filter_hi, 0);
+ sum = vmlal_lane_s16(sum, s4, y_filter_hi, 1);
+ sum = vmlal_lane_s16(sum, s5, y_filter_hi, 2);
+
+ return vqrshrn_n_s32(sum, 2 * FILTER_BITS - ROUND0_BITS);
+}
+
+static INLINE uint8x8_t convolve6_8_2d_v(const int16x8_t s0, const int16x8_t s1,
+ const int16x8_t s2, const int16x8_t s3,
+ const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t y_filter,
+ const int16x8_t sub_const) {
+ const int16x4_t y_filter_lo = vget_low_s16(y_filter);
+ const int16x4_t y_filter_hi = vget_high_s16(y_filter);
+
+ int32x4_t sum0 = vmull_lane_s16(vget_low_s16(s0), y_filter_lo, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s1), y_filter_lo, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s2), y_filter_lo, 3);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s3), y_filter_hi, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s4), y_filter_hi, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s5), y_filter_hi, 2);
+
+ int32x4_t sum1 = vmull_lane_s16(vget_high_s16(s0), y_filter_lo, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s1), y_filter_lo, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s2), y_filter_lo, 3);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s3), y_filter_hi, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s4), y_filter_hi, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s5), y_filter_hi, 2);
+
+ int16x8_t res =
+ vcombine_s16(vqrshrn_n_s32(sum0, 2 * FILTER_BITS - ROUND0_BITS),
+ vqrshrn_n_s32(sum1, 2 * FILTER_BITS - ROUND0_BITS));
+ res = vsubq_s16(res, sub_const);
+
+ return vqmovun_s16(res);
+}
+
+static INLINE void convolve_2d_sr_vert_6tap_neon(int16_t *src_ptr,
+ int src_stride,
+ uint8_t *dst_ptr,
+ int dst_stride, int w, int h,
+ const int16x8_t y_filter) {
+ const int bd = 8;
+ const int16x8_t sub_const = vdupq_n_s16(1 << (bd - 1));
+
+ if (w <= 4) {
+ int16x4_t s0, s1, s2, s3, s4;
+ load_s16_4x5(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4);
+ src_ptr += 5 * src_stride;
+
+ do {
+#if AOM_ARCH_AARCH64
+ int16x4_t s5, s6, s7, s8;
+ load_s16_4x4(src_ptr, src_stride, &s5, &s6, &s7, &s8);
+
+ int16x4_t d0 = convolve6_4_2d_v(s0, s1, s2, s3, s4, s5, y_filter);
+ int16x4_t d1 = convolve6_4_2d_v(s1, s2, s3, s4, s5, s6, y_filter);
+ int16x4_t d2 = convolve6_4_2d_v(s2, s3, s4, s5, s6, s7, y_filter);
+ int16x4_t d3 = convolve6_4_2d_v(s3, s4, s5, s6, s7, s8, y_filter);
+
+ uint8x8_t d01 = vqmovun_s16(vsubq_s16(vcombine_s16(d0, d1), sub_const));
+ uint8x8_t d23 = vqmovun_s16(vsubq_s16(vcombine_s16(d2, d3), sub_const));
+
+ store_u8_4x1(dst_ptr + 0 * dst_stride, d01, 0);
+ store_u8_4x1(dst_ptr + 1 * dst_stride, d01, 1);
+ store_u8_4x1(dst_ptr + 2 * dst_stride, d23, 0);
+ store_u8_4x1(dst_ptr + 3 * dst_stride, d23, 1);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ h -= 4;
+#else // !AOM_ARCH_AARCH64
+ int16x4_t s5 = vld1_s16(src_ptr);
+ int16x4_t d0 = convolve6_4_2d_v(s0, s1, s2, s3, s4, s5, y_filter);
+ uint8x8_t d01 =
+ vqmovun_s16(vsubq_s16(vcombine_s16(d0, vdup_n_s16(0)), sub_const));
+
+ store_u8_4x1(dst_ptr, d01, 0);
+
+ s0 = s1;
+ s1 = s2;
+ s2 = s3;
+ s3 = s4;
+ s4 = s5;
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ h--;
+#endif // AOM_ARCH_AARCH64
+ } while (h != 0);
+ } else {
+ // Width is a multiple of 8 and height is a multiple of 4.
+ do {
+ int height = h;
+ int16_t *s = src_ptr;
+ uint8_t *d = dst_ptr;
+
+ int16x8_t s0, s1, s2, s3, s4;
+ load_s16_8x5(s, src_stride, &s0, &s1, &s2, &s3, &s4);
+ s += 5 * src_stride;
+
+ do {
+#if AOM_ARCH_AARCH64
+ int16x8_t s5, s6, s7, s8;
+ load_s16_8x4(s, src_stride, &s5, &s6, &s7, &s8);
+
+ uint8x8_t d0 =
+ convolve6_8_2d_v(s0, s1, s2, s3, s4, s5, y_filter, sub_const);
+ uint8x8_t d1 =
+ convolve6_8_2d_v(s1, s2, s3, s4, s5, s6, y_filter, sub_const);
+ uint8x8_t d2 =
+ convolve6_8_2d_v(s2, s3, s4, s5, s6, s7, y_filter, sub_const);
+ uint8x8_t d3 =
+ convolve6_8_2d_v(s3, s4, s5, s6, s7, s8, y_filter, sub_const);
+
+ store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ height -= 4;
+#else // !AOM_ARCH_AARCH64
+ int16x8_t s5 = vld1q_s16(s);
+ uint8x8_t d0 =
+ convolve6_8_2d_v(s0, s1, s2, s3, s4, s5, y_filter, sub_const);
+ vst1_u8(d, d0);
+
+ s0 = s1;
+ s1 = s2;
+ s2 = s3;
+ s3 = s4;
+ s4 = s5;
+ s += src_stride;
+ d += dst_stride;
+ height--;
+#endif // AOM_ARCH_AARCH64
+ } while (height != 0);
+ src_ptr += 8;
+ dst_ptr += 8;
+ w -= 8;
+ } while (w != 0);
+ }
+}
#endif // AOM_AV1_COMMON_ARM_CONVOLVE_NEON_H_
diff --git a/av1/common/arm/convolve_neon_dotprod.c b/av1/common/arm/convolve_neon_dotprod.c
new file mode 100644
index 0000000..ba8f7e7
--- /dev/null
+++ b/av1/common/arm/convolve_neon_dotprod.c
@@ -0,0 +1,797 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+
+#include "config/aom_config.h"
+#include "config/av1_rtcd.h"
+
+#include "aom_dsp/aom_dsp_common.h"
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_ports/mem.h"
+#include "av1/common/arm/convolve_neon.h"
+#include "av1/common/convolve.h"
+#include "av1/common/filter.h"
+
+DECLARE_ALIGNED(16, static const uint8_t, dot_prod_permute_tbl[48]) = {
+ 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6,
+ 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10,
+ 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
+};
+
+static INLINE int16x4_t convolve12_4_x(uint8x16_t samples,
+ const int8x16_t filter,
+ const int32x4_t correction,
+ const uint8x16_t range_limit,
+ const uint8x16x3_t permute_tbl) {
+ int8x16_t clamped_samples, permuted_samples[3];
+ int32x4_t sum;
+
+ // Clamp sample range to [-128, 127] for 8-bit signed dot product.
+ clamped_samples = vreinterpretq_s8_u8(vsubq_u8(samples, range_limit));
+
+ // Permute samples ready for dot product.
+ // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
+ permuted_samples[0] = vqtbl1q_s8(clamped_samples, permute_tbl.val[0]);
+ // { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 }
+ permuted_samples[1] = vqtbl1q_s8(clamped_samples, permute_tbl.val[1]);
+ // { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 }
+ permuted_samples[2] = vqtbl1q_s8(clamped_samples, permute_tbl.val[2]);
+
+ // Accumulate dot product into 'correction' to account for range clamp.
+ // First 4 output values.
+ sum = vdotq_laneq_s32(correction, permuted_samples[0], filter, 0);
+ sum = vdotq_laneq_s32(sum, permuted_samples[1], filter, 1);
+ sum = vdotq_laneq_s32(sum, permuted_samples[2], filter, 2);
+
+ return vqrshrn_n_s32(sum, FILTER_BITS);
+}
+
+static INLINE uint8x8_t convolve12_8_x(uint8x16_t samples[2],
+ const int8x16_t filter,
+ const int32x4_t correction,
+ const uint8x16_t range_limit,
+ const uint8x16x3_t permute_tbl) {
+ int8x16_t clamped_samples[2], permuted_samples[4];
+ int32x4_t sum[2];
+
+ // Clamp sample range to [-128, 127] for 8-bit signed dot product.
+ clamped_samples[0] = vreinterpretq_s8_u8(vsubq_u8(samples[0], range_limit));
+ clamped_samples[1] = vreinterpretq_s8_u8(vsubq_u8(samples[1], range_limit));
+
+ // Permute samples ready for dot product.
+ // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
+ permuted_samples[0] = vqtbl1q_s8(clamped_samples[0], permute_tbl.val[0]);
+ // { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 }
+ permuted_samples[1] = vqtbl1q_s8(clamped_samples[0], permute_tbl.val[1]);
+ // { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 }
+ permuted_samples[2] = vqtbl1q_s8(clamped_samples[0], permute_tbl.val[2]);
+ // {12, 13, 14, 15, 13, 14, 15, 16, 14, 15, 16, 17, 15, 16, 17, 18 }
+ permuted_samples[3] = vqtbl1q_s8(clamped_samples[1], permute_tbl.val[2]);
+
+ // Accumulate dot product into 'correction' to account for range clamp.
+ // First 4 output values.
+ sum[0] = vdotq_laneq_s32(correction, permuted_samples[0], filter, 0);
+ sum[0] = vdotq_laneq_s32(sum[0], permuted_samples[1], filter, 1);
+ sum[0] = vdotq_laneq_s32(sum[0], permuted_samples[2], filter, 2);
+ // Second 4 output values.
+ sum[1] = vdotq_laneq_s32(correction, permuted_samples[1], filter, 0);
+ sum[1] = vdotq_laneq_s32(sum[1], permuted_samples[2], filter, 1);
+ sum[1] = vdotq_laneq_s32(sum[1], permuted_samples[3], filter, 2);
+
+ // Narrow and re-pack.
+ int16x8_t sum_s16 = vcombine_s16(vqrshrn_n_s32(sum[0], FILTER_BITS),
+ vqrshrn_n_s32(sum[1], FILTER_BITS));
+ return vqmovun_s16(sum_s16);
+}
+
+static INLINE void convolve_x_sr_12tap_neon_dotprod(
+ const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w,
+ int h, const int16_t *x_filter_ptr) {
+ const int16x8_t filter_0_7 = vld1q_s16(x_filter_ptr);
+ const int16x4_t filter_8_11 = vld1_s16(x_filter_ptr + 8);
+ const int16x8_t filter_8_15 = vcombine_s16(filter_8_11, vdup_n_s16(0));
+ const int8x16_t filter =
+ vcombine_s8(vmovn_s16(filter_0_7), vmovn_s16(filter_8_15));
+
+ const int32_t correction_s32 =
+ vaddvq_s32(vaddq_s32(vpaddlq_s16(vshlq_n_s16(filter_0_7, FILTER_BITS)),
+ vpaddlq_s16(vshlq_n_s16(filter_8_15, FILTER_BITS))));
+ // A shim of 1 << (ROUND0_BITS - 1) enables us to use a single rounding right
+ // shift by FILTER_BITS - instead of a first rounding right shift by
+ // ROUND0_BITS, followed by second rounding right shift by FILTER_BITS -
+ // ROUND0_BITS.
+ int32x4_t correction = vdupq_n_s32(correction_s32 + (1 << (ROUND0_BITS - 1)));
+ const uint8x16_t range_limit = vdupq_n_u8(128);
+ const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
+
+ // Special case the following no-op filter as 128 won't fit into the
+ // 8-bit signed dot-product instruction:
+ // { 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0 }
+ if (vgetq_lane_s16(filter_0_7, 5) == 128) {
+ // Undo the horizontal offset in the calling function.
+ src += 5;
+
+ do {
+ const uint8_t *s = src;
+ uint8_t *d = dst;
+ int width = w;
+
+ do {
+ uint8x8_t d0 = vld1_u8(s);
+ if (w == 4) {
+ store_u8_4x1(d, d0, 0);
+ } else {
+ vst1_u8(d, d0);
+ }
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width > 0);
+ src += src_stride;
+ dst += dst_stride;
+ } while (--h != 0);
+ } else {
+ if (w <= 4) {
+ do {
+ uint8x16_t s0, s1, s2, s3;
+ load_u8_16x4(src, src_stride, &s0, &s1, &s2, &s3);
+
+ int16x4_t d0 =
+ convolve12_4_x(s0, filter, correction, range_limit, permute_tbl);
+ int16x4_t d1 =
+ convolve12_4_x(s1, filter, correction, range_limit, permute_tbl);
+ int16x4_t d2 =
+ convolve12_4_x(s2, filter, correction, range_limit, permute_tbl);
+ int16x4_t d3 =
+ convolve12_4_x(s3, filter, correction, range_limit, permute_tbl);
+
+ uint8x8_t d01 = vqmovun_s16(vcombine_s16(d0, d1));
+ uint8x8_t d23 = vqmovun_s16(vcombine_s16(d2, d3));
+
+ store_u8_4x1(dst + 0 * dst_stride, d01, 0);
+ store_u8_4x1(dst + 1 * dst_stride, d01, 1);
+ store_u8_4x1(dst + 2 * dst_stride, d23, 0);
+ store_u8_4x1(dst + 3 * dst_stride, d23, 1);
+
+ dst += 4 * dst_stride;
+ src += 4 * src_stride;
+ h -= 4;
+ } while (h != 0);
+ } else {
+ do {
+ const uint8_t *s = src;
+ uint8_t *d = dst;
+ int width = w;
+
+ do {
+ uint8x16_t s0[2], s1[2], s2[2], s3[2];
+ load_u8_16x4(s, src_stride, &s0[0], &s1[0], &s2[0], &s3[0]);
+ load_u8_16x4(s + 4, src_stride, &s0[1], &s1[1], &s2[1], &s3[1]);
+
+ uint8x8_t d0 =
+ convolve12_8_x(s0, filter, correction, range_limit, permute_tbl);
+ uint8x8_t d1 =
+ convolve12_8_x(s1, filter, correction, range_limit, permute_tbl);
+ uint8x8_t d2 =
+ convolve12_8_x(s2, filter, correction, range_limit, permute_tbl);
+ uint8x8_t d3 =
+ convolve12_8_x(s3, filter, correction, range_limit, permute_tbl);
+
+ store_u8_8x4(d + 0 * dst_stride, dst_stride, d0, d1, d2, d3);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src += 4 * src_stride;
+ dst += 4 * dst_stride;
+ h -= 4;
+ } while (h != 0);
+ }
+ }
+}
+
+static INLINE int16x4_t convolve4_4_x(uint8x16_t samples, const int8x8_t filter,
+ const int32x4_t correction,
+ const uint8x16_t range_limit,
+ const uint8x16_t permute_tbl) {
+ // Clamp sample range to [-128, 127] for 8-bit signed dot product.
+ int8x16_t clamped_samples =
+ vreinterpretq_s8_u8(vsubq_u8(samples, range_limit));
+
+ // Permute samples ready for dot product.
+ // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
+ int8x16_t permuted_samples = vqtbl1q_s8(clamped_samples, permute_tbl);
+
+ // Accumulate dot product into 'correction' to account for range clamp.
+ int32x4_t sum = vdotq_lane_s32(correction, permuted_samples, filter, 0);
+
+ // Packing is performed by the caller.
+ return vmovn_s32(sum);
+}
+
+static INLINE uint8x8_t convolve8_8_x(uint8x16_t samples, const int8x8_t filter,
+ const int32x4_t correction,
+ const uint8x16_t range_limit,
+ const uint8x16x3_t permute_tbl) {
+ int8x16_t clamped_samples, permuted_samples[3];
+ int32x4_t sum[2];
+
+ // Clamp sample range to [-128, 127] for 8-bit signed dot product.
+ clamped_samples = vreinterpretq_s8_u8(vsubq_u8(samples, range_limit));
+
+ // Permute samples ready for dot product. */
+ // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
+ permuted_samples[0] = vqtbl1q_s8(clamped_samples, permute_tbl.val[0]);
+ // { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 }
+ permuted_samples[1] = vqtbl1q_s8(clamped_samples, permute_tbl.val[1]);
+ // { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 }
+ permuted_samples[2] = vqtbl1q_s8(clamped_samples, permute_tbl.val[2]);
+
+ // Accumulate dot product into 'correction' to account for range clamp.
+ // First 4 output values.
+ sum[0] = vdotq_lane_s32(correction, permuted_samples[0], filter, 0);
+ sum[0] = vdotq_lane_s32(sum[0], permuted_samples[1], filter, 1);
+ // Second 4 output values.
+ sum[1] = vdotq_lane_s32(correction, permuted_samples[1], filter, 0);
+ sum[1] = vdotq_lane_s32(sum[1], permuted_samples[2], filter, 1);
+
+ // Narrow and re-pack.
+ int16x8_t sum_s16 = vcombine_s16(vmovn_s32(sum[0]), vmovn_s32(sum[1]));
+ // We halved the convolution filter values so - 1 from the right shift.
+ return vqrshrun_n_s16(sum_s16, FILTER_BITS - 1);
+}
+
+void av1_convolve_x_sr_neon_dotprod(const uint8_t *src, int src_stride,
+ uint8_t *dst, int dst_stride, int w, int h,
+ const InterpFilterParams *filter_params_x,
+ const int subpel_x_qn,
+ ConvolveParams *conv_params) {
+ if (w == 2 || h == 2) {
+ av1_convolve_x_sr_c(src, src_stride, dst, dst_stride, w, h, filter_params_x,
+ subpel_x_qn, conv_params);
+ return;
+ }
+
+ const uint8_t horiz_offset = filter_params_x->taps / 2 - 1;
+ src -= horiz_offset;
+
+ const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
+ filter_params_x, subpel_x_qn & SUBPEL_MASK);
+
+ if (filter_params_x->taps > 8) {
+ convolve_x_sr_12tap_neon_dotprod(src, src_stride, dst, dst_stride, w, h,
+ x_filter_ptr);
+ return;
+ }
+
+ const int16x8_t x_filter_s16 = vld1q_s16(x_filter_ptr);
+ // Dot product constants.
+ const int32_t correction_s32 =
+ vaddlvq_s16(vshlq_n_s16(x_filter_s16, FILTER_BITS - 1));
+ // This shim of (1 << ((ROUND0_BITS - 1) - 1) enables us to use a single
+ // rounding right shift by FILTER_BITS - instead of a first rounding right
+ // shift by ROUND0_BITS, followed by second rounding right shift by
+ // FILTER_BITS - ROUND0_BITS.
+ // The outermost -1 is needed because we will halve the filter values.
+ const int32x4_t correction =
+ vdupq_n_s32(correction_s32 + (1 << ((ROUND0_BITS - 1) - 1)));
+ const uint8x16_t range_limit = vdupq_n_u8(128);
+
+ if (w <= 4) {
+ const uint8x16_t permute_tbl = vld1q_u8(dot_prod_permute_tbl);
+ // 4-tap filters are used for blocks having width <= 4.
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int8x8_t x_filter =
+ vshrn_n_s16(vcombine_s16(vld1_s16(x_filter_ptr + 2), vdup_n_s16(0)), 1);
+
+ src += 2;
+
+ do {
+ uint8x16_t s0, s1, s2, s3;
+ load_u8_16x4(src, src_stride, &s0, &s1, &s2, &s3);
+
+ int16x4_t d0 =
+ convolve4_4_x(s0, x_filter, correction, range_limit, permute_tbl);
+ int16x4_t d1 =
+ convolve4_4_x(s1, x_filter, correction, range_limit, permute_tbl);
+ int16x4_t d2 =
+ convolve4_4_x(s2, x_filter, correction, range_limit, permute_tbl);
+ int16x4_t d3 =
+ convolve4_4_x(s3, x_filter, correction, range_limit, permute_tbl);
+
+ // We halved the convolution filter values so - 1 from the right shift.
+ uint8x8_t d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS - 1);
+ uint8x8_t d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS - 1);
+
+ store_u8_4x1(dst + 0 * dst_stride, d01, 0);
+ store_u8_4x1(dst + 1 * dst_stride, d01, 1);
+ store_u8_4x1(dst + 2 * dst_stride, d23, 0);
+ store_u8_4x1(dst + 3 * dst_stride, d23, 1);
+
+ src += 4 * src_stride;
+ dst += 4 * dst_stride;
+ h -= 4;
+ } while (h != 0);
+ } else {
+ const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int8x8_t x_filter = vshrn_n_s16(x_filter_s16, 1);
+
+ do {
+ int width = w;
+ const uint8_t *s = src;
+ uint8_t *d = dst;
+
+ do {
+ uint8x16_t s0, s1, s2, s3;
+ load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3);
+
+ uint8x8_t d0 =
+ convolve8_8_x(s0, x_filter, correction, range_limit, permute_tbl);
+ uint8x8_t d1 =
+ convolve8_8_x(s1, x_filter, correction, range_limit, permute_tbl);
+ uint8x8_t d2 =
+ convolve8_8_x(s2, x_filter, correction, range_limit, permute_tbl);
+ uint8x8_t d3 =
+ convolve8_8_x(s3, x_filter, correction, range_limit, permute_tbl);
+
+ store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src += 4 * src_stride;
+ dst += 4 * dst_stride;
+ h -= 4;
+ } while (h != 0);
+ }
+}
+
+static INLINE int16x4_t convolve12_4_2d_h(uint8x16_t samples,
+ const int8x16_t filters,
+ const int32x4_t correction,
+ const uint8x16_t range_limit,
+ const uint8x16x3_t permute_tbl) {
+ int8x16_t clamped_samples, permuted_samples[3];
+ int32x4_t sum;
+
+ // Clamp sample range to [-128, 127] for 8-bit signed dot product.
+ clamped_samples = vreinterpretq_s8_u8(vsubq_u8(samples, range_limit));
+
+ // Permute samples ready for dot product.
+ // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
+ permuted_samples[0] = vqtbl1q_s8(clamped_samples, permute_tbl.val[0]);
+ // { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 }
+ permuted_samples[1] = vqtbl1q_s8(clamped_samples, permute_tbl.val[1]);
+ // { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 }
+ permuted_samples[2] = vqtbl1q_s8(clamped_samples, permute_tbl.val[2]);
+
+ // Accumulate dot product into 'correction' to account for range clamp.
+ // First 4 output values.
+ sum = vdotq_laneq_s32(correction, permuted_samples[0], filters, 0);
+ sum = vdotq_laneq_s32(sum, permuted_samples[1], filters, 1);
+ sum = vdotq_laneq_s32(sum, permuted_samples[2], filters, 2);
+
+ // Narrow and re-pack.
+ return vshrn_n_s32(sum, ROUND0_BITS);
+}
+
+static INLINE int16x8_t convolve12_8_2d_h(uint8x16_t samples[2],
+ const int8x16_t filters,
+ const int32x4_t correction,
+ const uint8x16_t range_limit,
+ const uint8x16x3_t permute_tbl) {
+ int8x16_t clamped_samples[2], permuted_samples[4];
+ int32x4_t sum[2];
+
+ // Clamp sample range to [-128, 127] for 8-bit signed dot product.
+ clamped_samples[0] = vreinterpretq_s8_u8(vsubq_u8(samples[0], range_limit));
+ clamped_samples[1] = vreinterpretq_s8_u8(vsubq_u8(samples[1], range_limit));
+
+ // Permute samples ready for dot product.
+ // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
+ permuted_samples[0] = vqtbl1q_s8(clamped_samples[0], permute_tbl.val[0]);
+ // { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 }
+ permuted_samples[1] = vqtbl1q_s8(clamped_samples[0], permute_tbl.val[1]);
+ // { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 }
+ permuted_samples[2] = vqtbl1q_s8(clamped_samples[0], permute_tbl.val[2]);
+ // {12, 13, 14, 15, 13, 14, 15, 16, 14, 15, 16, 17, 15, 16, 17, 18 }
+ permuted_samples[3] = vqtbl1q_s8(clamped_samples[1], permute_tbl.val[2]);
+
+ // Accumulate dot product into 'correction' to account for range clamp.
+ // First 4 output values.
+ sum[0] = vdotq_laneq_s32(correction, permuted_samples[0], filters, 0);
+ sum[0] = vdotq_laneq_s32(sum[0], permuted_samples[1], filters, 1);
+ sum[0] = vdotq_laneq_s32(sum[0], permuted_samples[2], filters, 2);
+ // Second 4 output values.
+ sum[1] = vdotq_laneq_s32(correction, permuted_samples[1], filters, 0);
+ sum[1] = vdotq_laneq_s32(sum[1], permuted_samples[2], filters, 1);
+ sum[1] = vdotq_laneq_s32(sum[1], permuted_samples[3], filters, 2);
+
+ // Narrow and re-pack.
+ return vcombine_s16(vshrn_n_s32(sum[0], ROUND0_BITS),
+ vshrn_n_s32(sum[1], ROUND0_BITS));
+}
+
+static INLINE void convolve_2d_sr_horiz_12tap_neon_dotprod(
+ const uint8_t *src_ptr, int src_stride, int16_t *dst_ptr,
+ const int dst_stride, int w, int h, const int16x8_t x_filter_0_7,
+ const int16x4_t x_filter_8_11) {
+ const int bd = 8;
+
+ // Special case the following no-op filter as 128 won't fit into the 8-bit
+ // signed dot-product instruction:
+ // { 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0 }
+ if (vgetq_lane_s16(x_filter_0_7, 5) == 128) {
+ const uint16x8_t horiz_const = vdupq_n_u16((1 << (bd - 1)));
+ // Undo the horizontal offset in the calling function.
+ src_ptr += 5;
+
+ do {
+ const uint8_t *s = src_ptr;
+ int16_t *d = dst_ptr;
+ int width = w;
+
+ do {
+ uint8x8_t s0 = vld1_u8(s);
+ uint16x8_t d0 = vaddw_u8(horiz_const, s0);
+ d0 = vshlq_n_u16(d0, FILTER_BITS - ROUND0_BITS);
+ // Store 8 elements to avoid additional branches. This is safe if the
+ // actual block width is < 8 because the intermediate buffer is large
+ // enough to accommodate 128x128 blocks.
+ vst1q_s16(d, vreinterpretq_s16_u16(d0));
+
+ d += 8;
+ s += 8;
+ width -= 8;
+ } while (width > 0);
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ } while (--h != 0);
+
+ } else {
+ // Narrow filter values to 8-bit.
+ const int16x8x2_t x_filter_s16 = {
+ { x_filter_0_7, vcombine_s16(x_filter_8_11, vdup_n_s16(0)) }
+ };
+ const int8x16_t x_filter = vcombine_s8(vmovn_s16(x_filter_s16.val[0]),
+ vmovn_s16(x_filter_s16.val[1]));
+
+ // This shim of 1 << (ROUND0_BITS - 1) enables us to use non-rounding shifts
+ // - which are generally faster than rounding shifts on modern CPUs.
+ const int32_t horiz_const =
+ ((1 << (bd + FILTER_BITS - 1)) + (1 << (ROUND0_BITS - 1)));
+ // Dot product constants.
+ const int32x4_t correct_tmp =
+ vaddq_s32(vpaddlq_s16(vshlq_n_s16(x_filter_s16.val[0], 7)),
+ vpaddlq_s16(vshlq_n_s16(x_filter_s16.val[1], 7)));
+ const int32x4_t correction =
+ vdupq_n_s32(vaddvq_s32(correct_tmp) + horiz_const);
+ const uint8x16_t range_limit = vdupq_n_u8(128);
+ const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
+
+ if (w <= 4) {
+ do {
+ uint8x16_t s0, s1, s2, s3;
+ load_u8_16x4(src_ptr, src_stride, &s0, &s1, &s2, &s3);
+
+ int16x4_t d0 = convolve12_4_2d_h(s0, x_filter, correction, range_limit,
+ permute_tbl);
+ int16x4_t d1 = convolve12_4_2d_h(s1, x_filter, correction, range_limit,
+ permute_tbl);
+ int16x4_t d2 = convolve12_4_2d_h(s2, x_filter, correction, range_limit,
+ permute_tbl);
+ int16x4_t d3 = convolve12_4_2d_h(s3, x_filter, correction, range_limit,
+ permute_tbl);
+
+ store_s16_4x4(dst_ptr, dst_stride, d0, d1, d2, d3);
+
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ h -= 4;
+ } while (h > 4);
+
+ do {
+ uint8x16_t s0 = vld1q_u8(src_ptr);
+ int16x4_t d0 = convolve12_4_2d_h(s0, x_filter, correction, range_limit,
+ permute_tbl);
+ vst1_s16(dst_ptr, d0);
+
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ } while (--h != 0);
+
+ } else {
+ do {
+ const uint8_t *s = src_ptr;
+ int16_t *d = dst_ptr;
+ int width = w;
+
+ do {
+ uint8x16_t s0[2], s1[2], s2[2], s3[2];
+ load_u8_16x4(s, src_stride, &s0[0], &s1[0], &s2[0], &s3[0]);
+ load_u8_16x4(s + 4, src_stride, &s0[1], &s1[1], &s2[1], &s3[1]);
+
+ int16x8_t d0 = convolve12_8_2d_h(s0, x_filter, correction,
+ range_limit, permute_tbl);
+ int16x8_t d1 = convolve12_8_2d_h(s1, x_filter, correction,
+ range_limit, permute_tbl);
+ int16x8_t d2 = convolve12_8_2d_h(s2, x_filter, correction,
+ range_limit, permute_tbl);
+ int16x8_t d3 = convolve12_8_2d_h(s3, x_filter, correction,
+ range_limit, permute_tbl);
+
+ store_s16_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ h -= 4;
+ } while (h > 4);
+
+ do {
+ const uint8_t *s = src_ptr;
+ int16_t *d = dst_ptr;
+ int width = w;
+
+ do {
+ uint8x16_t s0[2];
+ s0[0] = vld1q_u8(s);
+ s0[1] = vld1q_u8(s + 4);
+ int16x8_t d0 = convolve12_8_2d_h(s0, x_filter, correction,
+ range_limit, permute_tbl);
+ vst1q_s16(d, d0);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ } while (--h != 0);
+ }
+ }
+}
+
+static INLINE int16x4_t convolve4_4_2d_h(uint8x16_t samples,
+ const int8x8_t filters,
+ const int32x4_t correction,
+ const uint8x16_t range_limit,
+ const uint8x16_t permute_tbl) {
+ // Clamp sample range to [-128, 127] for 8-bit signed dot product.
+ int8x16_t clamped_samples =
+ vreinterpretq_s8_u8(vsubq_u8(samples, range_limit));
+
+ // Permute samples ready for dot product.
+ // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
+ int8x16_t permuted_samples = vqtbl1q_s8(clamped_samples, permute_tbl);
+
+ // Accumulate dot product into 'correction' to account for range clamp.
+ int32x4_t sum = vdotq_lane_s32(correction, permuted_samples, filters, 0);
+
+ // We halved the convolution filter values so -1 from the right shift.
+ return vshrn_n_s32(sum, ROUND0_BITS - 1);
+}
+
+static INLINE int16x8_t convolve8_8_2d_h(uint8x16_t samples,
+ const int8x8_t filters,
+ const int32x4_t correction,
+ const uint8x16_t range_limit,
+ const uint8x16x3_t permute_tbl) {
+ int8x16_t clamped_samples, permuted_samples[3];
+ int32x4_t sum[2];
+
+ // Clamp sample range to [-128, 127] for 8-bit signed dot product.
+ clamped_samples = vreinterpretq_s8_u8(vsubq_u8(samples, range_limit));
+
+ // Permute samples ready for dot product.
+ // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
+ permuted_samples[0] = vqtbl1q_s8(clamped_samples, permute_tbl.val[0]);
+ // { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 }
+ permuted_samples[1] = vqtbl1q_s8(clamped_samples, permute_tbl.val[1]);
+ // { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 }
+ permuted_samples[2] = vqtbl1q_s8(clamped_samples, permute_tbl.val[2]);
+
+ // Accumulate dot product into 'correction' to account for range clamp.
+ // First 4 output values.
+ sum[0] = vdotq_lane_s32(correction, permuted_samples[0], filters, 0);
+ sum[0] = vdotq_lane_s32(sum[0], permuted_samples[1], filters, 1);
+ // Second 4 output values.
+ sum[1] = vdotq_lane_s32(correction, permuted_samples[1], filters, 0);
+ sum[1] = vdotq_lane_s32(sum[1], permuted_samples[2], filters, 1);
+
+ // Narrow and re-pack.
+ // We halved the convolution filter values so -1 from the right shift.
+ return vcombine_s16(vshrn_n_s32(sum[0], ROUND0_BITS - 1),
+ vshrn_n_s32(sum[1], ROUND0_BITS - 1));
+}
+
+static INLINE void convolve_2d_sr_horiz_neon_dotprod(
+ const uint8_t *src, int src_stride, int16_t *im_block, int im_stride, int w,
+ int im_h, const int16_t *x_filter_ptr) {
+ const int bd = 8;
+ // This shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-rounding
+ // shifts - which are generally faster than rounding shifts on modern CPUs.
+ // The outermost -1 is needed because we halved the filter values.
+ const int32_t horiz_const =
+ ((1 << (bd + FILTER_BITS - 2)) + (1 << ((ROUND0_BITS - 1) - 1)));
+ // Dot product constants.
+ const int16x8_t x_filter_s16 = vld1q_s16(x_filter_ptr);
+ const int32_t correction_s32 =
+ vaddlvq_s16(vshlq_n_s16(x_filter_s16, FILTER_BITS - 1));
+ const int32x4_t correction = vdupq_n_s32(correction_s32 + horiz_const);
+ const uint8x16_t range_limit = vdupq_n_u8(128);
+
+ const uint8_t *src_ptr = src;
+ int16_t *dst_ptr = im_block;
+ int dst_stride = im_stride;
+ int height = im_h;
+
+ if (w <= 4) {
+ const uint8x16_t permute_tbl = vld1q_u8(dot_prod_permute_tbl);
+ // 4-tap filters are used for blocks having width <= 4.
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int8x8_t x_filter =
+ vshrn_n_s16(vcombine_s16(vld1_s16(x_filter_ptr + 2), vdup_n_s16(0)), 1);
+
+ src_ptr += 2;
+
+ do {
+ uint8x16_t s0, s1, s2, s3;
+ load_u8_16x4(src_ptr, src_stride, &s0, &s1, &s2, &s3);
+
+ int16x4_t d0 =
+ convolve4_4_2d_h(s0, x_filter, correction, range_limit, permute_tbl);
+ int16x4_t d1 =
+ convolve4_4_2d_h(s1, x_filter, correction, range_limit, permute_tbl);
+ int16x4_t d2 =
+ convolve4_4_2d_h(s2, x_filter, correction, range_limit, permute_tbl);
+ int16x4_t d3 =
+ convolve4_4_2d_h(s3, x_filter, correction, range_limit, permute_tbl);
+
+ store_s16_4x4(dst_ptr, dst_stride, d0, d1, d2, d3);
+
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ height -= 4;
+ } while (height > 4);
+
+ do {
+ uint8x16_t s0 = vld1q_u8(src_ptr);
+ int16x4_t d0 =
+ convolve4_4_2d_h(s0, x_filter, correction, range_limit, permute_tbl);
+ vst1_s16(dst_ptr, d0);
+
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ } while (--height != 0);
+ } else {
+ const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int8x8_t x_filter = vshrn_n_s16(x_filter_s16, 1);
+
+ do {
+ const uint8_t *s = src_ptr;
+ int16_t *d = dst_ptr;
+ int width = w;
+
+ do {
+ uint8x16_t s0, s1, s2, s3;
+ load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3);
+
+ int16x8_t d0 = convolve8_8_2d_h(s0, x_filter, correction, range_limit,
+ permute_tbl);
+ int16x8_t d1 = convolve8_8_2d_h(s1, x_filter, correction, range_limit,
+ permute_tbl);
+ int16x8_t d2 = convolve8_8_2d_h(s2, x_filter, correction, range_limit,
+ permute_tbl);
+ int16x8_t d3 = convolve8_8_2d_h(s3, x_filter, correction, range_limit,
+ permute_tbl);
+
+ store_s16_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ height -= 4;
+ } while (height > 4);
+
+ do {
+ const uint8_t *s = src_ptr;
+ int16_t *d = dst_ptr;
+ int width = w;
+
+ do {
+ uint8x16_t s0 = vld1q_u8(s);
+ int16x8_t d0 = convolve8_8_2d_h(s0, x_filter, correction, range_limit,
+ permute_tbl);
+ vst1q_s16(d, d0);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ } while (--height != 0);
+ }
+}
+
+void av1_convolve_2d_sr_neon_dotprod(const uint8_t *src, int src_stride,
+ uint8_t *dst, int dst_stride, int w, int h,
+ const InterpFilterParams *filter_params_x,
+ const InterpFilterParams *filter_params_y,
+ const int subpel_x_qn,
+ const int subpel_y_qn,
+ ConvolveParams *conv_params) {
+ if (w == 2 || h == 2) {
+ av1_convolve_2d_sr_c(src, src_stride, dst, dst_stride, w, h,
+ filter_params_x, filter_params_y, subpel_x_qn,
+ subpel_y_qn, conv_params);
+ return;
+ }
+
+ const int y_filter_taps = get_filter_tap(filter_params_y, subpel_y_qn);
+ const int clamped_y_taps = y_filter_taps < 6 ? 6 : y_filter_taps;
+ const int im_h = h + clamped_y_taps - 1;
+ const int im_stride = MAX_SB_SIZE;
+ const int vert_offset = clamped_y_taps / 2 - 1;
+ const int horiz_offset = filter_params_x->taps / 2 - 1;
+ const uint8_t *src_ptr = src - vert_offset * src_stride - horiz_offset;
+
+ const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
+ filter_params_x, subpel_x_qn & SUBPEL_MASK);
+ const int16_t *y_filter_ptr = av1_get_interp_filter_subpel_kernel(
+ filter_params_y, subpel_y_qn & SUBPEL_MASK);
+
+ if (filter_params_x->taps > 8) {
+ DECLARE_ALIGNED(16, int16_t,
+ im_block[(MAX_SB_SIZE + MAX_FILTER_TAP - 1) * MAX_SB_SIZE]);
+
+ const int16x8_t x_filter_0_7 = vld1q_s16(x_filter_ptr);
+ const int16x4_t x_filter_8_11 = vld1_s16(x_filter_ptr + 8);
+ const int16x8_t y_filter_0_7 = vld1q_s16(y_filter_ptr);
+ const int16x4_t y_filter_8_11 = vld1_s16(y_filter_ptr + 8);
+
+ convolve_2d_sr_horiz_12tap_neon_dotprod(src_ptr, src_stride, im_block,
+ im_stride, w, im_h, x_filter_0_7,
+ x_filter_8_11);
+
+ convolve_2d_sr_vert_12tap_neon(im_block, im_stride, dst, dst_stride, w, h,
+ y_filter_0_7, y_filter_8_11);
+ } else {
+ DECLARE_ALIGNED(16, int16_t,
+ im_block[(MAX_SB_SIZE + SUBPEL_TAPS - 1) * MAX_SB_SIZE]);
+
+ convolve_2d_sr_horiz_neon_dotprod(src_ptr, src_stride, im_block, im_stride,
+ w, im_h, x_filter_ptr);
+
+ const int16x8_t y_filter = vld1q_s16(y_filter_ptr);
+
+ if (clamped_y_taps <= 6) {
+ convolve_2d_sr_vert_6tap_neon(im_block, im_stride, dst, dst_stride, w, h,
+ y_filter);
+ } else {
+ convolve_2d_sr_vert_8tap_neon(im_block, im_stride, dst, dst_stride, w, h,
+ y_filter);
+ }
+ }
+}
diff --git a/av1/common/arm/convolve_neon_i8mm.c b/av1/common/arm/convolve_neon_i8mm.c
new file mode 100644
index 0000000..14140ca
--- /dev/null
+++ b/av1/common/arm/convolve_neon_i8mm.c
@@ -0,0 +1,706 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+
+#include "config/aom_config.h"
+#include "config/av1_rtcd.h"
+
+#include "aom_dsp/aom_dsp_common.h"
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_ports/mem.h"
+#include "av1/common/arm/convolve_neon.h"
+#include "av1/common/convolve.h"
+#include "av1/common/filter.h"
+
+DECLARE_ALIGNED(16, static const uint8_t, dot_prod_permute_tbl[48]) = {
+ 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6,
+ 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10,
+ 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
+};
+
+static INLINE int16x4_t convolve12_4_x(uint8x16_t samples,
+ const int8x16_t filter,
+ const uint8x16x3_t permute_tbl,
+ const int32x4_t horiz_const) {
+ uint8x16_t permuted_samples[3];
+ int32x4_t sum;
+
+ // Permute samples ready for dot product.
+ // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
+ permuted_samples[0] = vqtbl1q_u8(samples, permute_tbl.val[0]);
+ // { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 }
+ permuted_samples[1] = vqtbl1q_u8(samples, permute_tbl.val[1]);
+ // { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 }
+ permuted_samples[2] = vqtbl1q_u8(samples, permute_tbl.val[2]);
+
+ // First 4 output values.
+ sum = vusdotq_laneq_s32(horiz_const, permuted_samples[0], filter, 0);
+ sum = vusdotq_laneq_s32(sum, permuted_samples[1], filter, 1);
+ sum = vusdotq_laneq_s32(sum, permuted_samples[2], filter, 2);
+
+ return vqrshrn_n_s32(sum, FILTER_BITS);
+}
+
+static INLINE uint8x8_t convolve12_8_x(uint8x16_t samples[2],
+ const int8x16_t filter,
+ const uint8x16x3_t permute_tbl,
+ const int32x4_t horiz_const) {
+ uint8x16_t permuted_samples[4];
+ int32x4_t sum[2];
+
+ // Permute samples ready for dot product.
+ // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
+ permuted_samples[0] = vqtbl1q_u8(samples[0], permute_tbl.val[0]);
+ // { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 }
+ permuted_samples[1] = vqtbl1q_u8(samples[0], permute_tbl.val[1]);
+ // { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 }
+ permuted_samples[2] = vqtbl1q_u8(samples[0], permute_tbl.val[2]);
+ // {12, 13, 14, 15, 13, 14, 15, 16, 14, 15, 16, 17, 15, 16, 17, 18 }
+ permuted_samples[3] = vqtbl1q_u8(samples[1], permute_tbl.val[2]);
+
+ // First 4 output values.
+ sum[0] = vusdotq_laneq_s32(horiz_const, permuted_samples[0], filter, 0);
+ sum[0] = vusdotq_laneq_s32(sum[0], permuted_samples[1], filter, 1);
+ sum[0] = vusdotq_laneq_s32(sum[0], permuted_samples[2], filter, 2);
+ // Second 4 output values.
+ sum[1] = vusdotq_laneq_s32(horiz_const, permuted_samples[1], filter, 0);
+ sum[1] = vusdotq_laneq_s32(sum[1], permuted_samples[2], filter, 1);
+ sum[1] = vusdotq_laneq_s32(sum[1], permuted_samples[3], filter, 2);
+
+ // Narrow and re-pack.
+ int16x8_t sum_s16 = vcombine_s16(vqrshrn_n_s32(sum[0], FILTER_BITS),
+ vqrshrn_n_s32(sum[1], FILTER_BITS));
+ return vqmovun_s16(sum_s16);
+}
+
+static INLINE void convolve_x_sr_12tap_neon_i8mm(const uint8_t *src,
+ int src_stride, uint8_t *dst,
+ int dst_stride, int w, int h,
+ const int16_t *x_filter_ptr) {
+ const int16x8_t filter_0_7 = vld1q_s16(x_filter_ptr);
+ const int16x4_t filter_8_11 = vld1_s16(x_filter_ptr + 8);
+ const int16x8_t filter_8_15 = vcombine_s16(filter_8_11, vdup_n_s16(0));
+ const int8x16_t filter =
+ vcombine_s8(vmovn_s16(filter_0_7), vmovn_s16(filter_8_15));
+
+ // Special case the following no-op filter as 128 won't fit into the
+ // 8-bit signed dot-product instruction:
+ // { 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0 }
+ if (vgetq_lane_s16(filter_0_7, 5) == 128) {
+ // Undo the horizontal offset in the calling function.
+ src += 5;
+
+ do {
+ const uint8_t *s = src;
+ uint8_t *d = dst;
+ int width = w;
+
+ do {
+ uint8x8_t d0 = vld1_u8(s);
+ if (w == 4) {
+ store_u8_4x1(d, d0, 0);
+ } else {
+ vst1_u8(d, d0);
+ }
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width > 0);
+ src += src_stride;
+ dst += dst_stride;
+ } while (--h != 0);
+ } else {
+ const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
+ // This shim of 1 << (ROUND0_BITS - 1) enables us to use a single rounding
+ // right shift by FILTER_BITS - instead of a first rounding right shift by
+ // ROUND0_BITS, followed by second rounding right shift by FILTER_BITS -
+ // ROUND0_BITS.
+ const int32x4_t horiz_const = vdupq_n_s32(1 << (ROUND0_BITS - 1));
+
+ if (w <= 4) {
+ do {
+ uint8x16_t s0, s1, s2, s3;
+ load_u8_16x4(src, src_stride, &s0, &s1, &s2, &s3);
+
+ int16x4_t d0 = convolve12_4_x(s0, filter, permute_tbl, horiz_const);
+ int16x4_t d1 = convolve12_4_x(s1, filter, permute_tbl, horiz_const);
+ int16x4_t d2 = convolve12_4_x(s2, filter, permute_tbl, horiz_const);
+ int16x4_t d3 = convolve12_4_x(s3, filter, permute_tbl, horiz_const);
+
+ uint8x8_t d01 = vqmovun_s16(vcombine_s16(d0, d1));
+ uint8x8_t d23 = vqmovun_s16(vcombine_s16(d2, d3));
+
+ store_u8_4x1(dst + 0 * dst_stride, d01, 0);
+ store_u8_4x1(dst + 1 * dst_stride, d01, 1);
+ store_u8_4x1(dst + 2 * dst_stride, d23, 0);
+ store_u8_4x1(dst + 3 * dst_stride, d23, 1);
+
+ dst += 4 * dst_stride;
+ src += 4 * src_stride;
+ h -= 4;
+ } while (h != 0);
+ } else {
+ do {
+ const uint8_t *s = src;
+ uint8_t *d = dst;
+ int width = w;
+
+ do {
+ uint8x16_t s0[2], s1[2], s2[2], s3[2];
+ load_u8_16x4(s, src_stride, &s0[0], &s1[0], &s2[0], &s3[0]);
+ load_u8_16x4(s + 4, src_stride, &s0[1], &s1[1], &s2[1], &s3[1]);
+
+ uint8x8_t d0 = convolve12_8_x(s0, filter, permute_tbl, horiz_const);
+ uint8x8_t d1 = convolve12_8_x(s1, filter, permute_tbl, horiz_const);
+ uint8x8_t d2 = convolve12_8_x(s2, filter, permute_tbl, horiz_const);
+ uint8x8_t d3 = convolve12_8_x(s3, filter, permute_tbl, horiz_const);
+
+ store_u8_8x4(d + 0 * dst_stride, dst_stride, d0, d1, d2, d3);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src += 4 * src_stride;
+ dst += 4 * dst_stride;
+ h -= 4;
+ } while (h != 0);
+ }
+ }
+}
+
+static INLINE int16x4_t convolve4_4_x(uint8x16_t samples, const int8x8_t filter,
+ const uint8x16_t permute_tbl,
+ const int32x4_t horiz_const) {
+ // Permute samples ready for dot product.
+ // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
+ uint8x16_t permuted_samples = vqtbl1q_u8(samples, permute_tbl);
+
+ // First 4 output values.
+ int32x4_t sum = vusdotq_lane_s32(horiz_const, permuted_samples, filter, 0);
+
+ // Packing is performed by the caller.
+ return vmovn_s32(sum);
+}
+
+static INLINE uint8x8_t convolve8_8_x(uint8x16_t samples, const int8x8_t filter,
+ const uint8x16x3_t permute_tbl,
+ const int32x4_t horiz_const) {
+ uint8x16_t permuted_samples[3];
+ int32x4_t sum[2];
+
+ // Permute samples ready for dot product.
+ // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
+ permuted_samples[0] = vqtbl1q_u8(samples, permute_tbl.val[0]);
+ // { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 }
+ permuted_samples[1] = vqtbl1q_u8(samples, permute_tbl.val[1]);
+ // { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 }
+ permuted_samples[2] = vqtbl1q_u8(samples, permute_tbl.val[2]);
+
+ // First 4 output values.
+ sum[0] = vusdotq_lane_s32(horiz_const, permuted_samples[0], filter, 0);
+ sum[0] = vusdotq_lane_s32(sum[0], permuted_samples[1], filter, 1);
+ // Second 4 output values.
+ sum[1] = vusdotq_lane_s32(horiz_const, permuted_samples[1], filter, 0);
+ sum[1] = vusdotq_lane_s32(sum[1], permuted_samples[2], filter, 1);
+
+ int16x8_t sum_s16 = vcombine_s16(vmovn_s32(sum[0]), vmovn_s32(sum[1]));
+ // We halved the convolution filter values so - 1 from the right shift.
+ return vqrshrun_n_s16(sum_s16, FILTER_BITS - 1);
+}
+
+void av1_convolve_x_sr_neon_i8mm(const uint8_t *src, int src_stride,
+ uint8_t *dst, int dst_stride, int w, int h,
+ const InterpFilterParams *filter_params_x,
+ const int subpel_x_qn,
+ ConvolveParams *conv_params) {
+ if (w == 2 || h == 2) {
+ av1_convolve_x_sr_c(src, src_stride, dst, dst_stride, w, h, filter_params_x,
+ subpel_x_qn, conv_params);
+ return;
+ }
+
+ const uint8_t horiz_offset = filter_params_x->taps / 2 - 1;
+ src -= horiz_offset;
+
+ const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
+ filter_params_x, subpel_x_qn & SUBPEL_MASK);
+
+ if (filter_params_x->taps > 8) {
+ convolve_x_sr_12tap_neon_i8mm(src, src_stride, dst, dst_stride, w, h,
+ x_filter_ptr);
+ return;
+ }
+
+ // This shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use a single
+ // rounding right shift by FILTER_BITS - instead of a first rounding right
+ // shift by ROUND0_BITS, followed by second rounding right shift by
+ // FILTER_BITS - ROUND0_BITS.
+ // The outermost -1 is needed because we will halve the filter values.
+ const int32x4_t horiz_const = vdupq_n_s32(1 << ((ROUND0_BITS - 1) - 1));
+
+ if (w <= 4) {
+ const uint8x16_t permute_tbl = vld1q_u8(dot_prod_permute_tbl);
+ // 4-tap filters are used for blocks having width <= 4.
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int8x8_t x_filter =
+ vshrn_n_s16(vcombine_s16(vld1_s16(x_filter_ptr + 2), vdup_n_s16(0)), 1);
+
+ src += 2;
+
+ do {
+ uint8x16_t s0, s1, s2, s3;
+ load_u8_16x4(src, src_stride, &s0, &s1, &s2, &s3);
+
+ int16x4_t d0 = convolve4_4_x(s0, x_filter, permute_tbl, horiz_const);
+ int16x4_t d1 = convolve4_4_x(s1, x_filter, permute_tbl, horiz_const);
+ int16x4_t d2 = convolve4_4_x(s2, x_filter, permute_tbl, horiz_const);
+ int16x4_t d3 = convolve4_4_x(s3, x_filter, permute_tbl, horiz_const);
+
+ // We halved the convolution filter values so - 1 from the right shift.
+ uint8x8_t d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS - 1);
+ uint8x8_t d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS - 1);
+
+ store_u8_4x1(dst + 0 * dst_stride, d01, 0);
+ store_u8_4x1(dst + 1 * dst_stride, d01, 1);
+ store_u8_4x1(dst + 2 * dst_stride, d23, 0);
+ store_u8_4x1(dst + 3 * dst_stride, d23, 1);
+
+ src += 4 * src_stride;
+ dst += 4 * dst_stride;
+ h -= 4;
+ } while (h != 0);
+
+ } else {
+ const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int8x8_t x_filter = vshrn_n_s16(vld1q_s16(x_filter_ptr), 1);
+
+ do {
+ const uint8_t *s = src;
+ uint8_t *d = dst;
+ int width = w;
+
+ do {
+ uint8x16_t s0, s1, s2, s3;
+ load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3);
+
+ uint8x8_t d0 = convolve8_8_x(s0, x_filter, permute_tbl, horiz_const);
+ uint8x8_t d1 = convolve8_8_x(s1, x_filter, permute_tbl, horiz_const);
+ uint8x8_t d2 = convolve8_8_x(s2, x_filter, permute_tbl, horiz_const);
+ uint8x8_t d3 = convolve8_8_x(s3, x_filter, permute_tbl, horiz_const);
+
+ store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src += 4 * src_stride;
+ dst += 4 * dst_stride;
+ h -= 4;
+ } while (h != 0);
+ }
+}
+
+static INLINE int16x4_t convolve12_4_2d_h(uint8x16_t samples,
+ const int8x16_t filters,
+ const uint8x16x3_t permute_tbl,
+ int32x4_t horiz_const) {
+ uint8x16_t permuted_samples[3];
+ int32x4_t sum;
+
+ // Permute samples ready for dot product.
+ // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
+ permuted_samples[0] = vqtbl1q_u8(samples, permute_tbl.val[0]);
+ // { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 }
+ permuted_samples[1] = vqtbl1q_u8(samples, permute_tbl.val[1]);
+ // { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 }
+ permuted_samples[2] = vqtbl1q_u8(samples, permute_tbl.val[2]);
+
+ // First 4 output values.
+ sum = vusdotq_laneq_s32(horiz_const, permuted_samples[0], filters, 0);
+ sum = vusdotq_laneq_s32(sum, permuted_samples[1], filters, 1);
+ sum = vusdotq_laneq_s32(sum, permuted_samples[2], filters, 2);
+
+ // Narrow and re-pack.
+ return vshrn_n_s32(sum, ROUND0_BITS);
+}
+
+static INLINE int16x8_t convolve12_8_2d_h(uint8x16_t samples[2],
+ const int8x16_t filters,
+ const uint8x16x3_t permute_tbl,
+ const int32x4_t horiz_const) {
+ uint8x16_t permuted_samples[4];
+ int32x4_t sum[2];
+
+ // Permute samples ready for dot product.
+ // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
+ permuted_samples[0] = vqtbl1q_u8(samples[0], permute_tbl.val[0]);
+ // { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 }
+ permuted_samples[1] = vqtbl1q_u8(samples[0], permute_tbl.val[1]);
+ // { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 }
+ permuted_samples[2] = vqtbl1q_u8(samples[0], permute_tbl.val[2]);
+ // {12, 13, 14, 15, 13, 14, 15, 16, 14, 15, 16, 17, 15, 16, 17, 18 }
+ permuted_samples[3] = vqtbl1q_u8(samples[1], permute_tbl.val[2]);
+
+ // First 4 output values.
+ sum[0] = vusdotq_laneq_s32(horiz_const, permuted_samples[0], filters, 0);
+ sum[0] = vusdotq_laneq_s32(sum[0], permuted_samples[1], filters, 1);
+ sum[0] = vusdotq_laneq_s32(sum[0], permuted_samples[2], filters, 2);
+ // Second 4 output values.
+ sum[1] = vusdotq_laneq_s32(horiz_const, permuted_samples[1], filters, 0);
+ sum[1] = vusdotq_laneq_s32(sum[1], permuted_samples[2], filters, 1);
+ sum[1] = vusdotq_laneq_s32(sum[1], permuted_samples[3], filters, 2);
+
+ // Narrow and re-pack.
+ return vcombine_s16(vshrn_n_s32(sum[0], ROUND0_BITS),
+ vshrn_n_s32(sum[1], ROUND0_BITS));
+}
+
+static INLINE void convolve_2d_sr_horiz_12tap_neon_i8mm(
+ const uint8_t *src_ptr, int src_stride, int16_t *dst_ptr,
+ const int dst_stride, int w, int h, const int16x8_t x_filter_0_7,
+ const int16x4_t x_filter_8_11) {
+ const int bd = 8;
+
+ // Special case the following no-op filter as 128 won't fit into the
+ // 8-bit signed dot-product instruction:
+ // { 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0 }
+ if (vgetq_lane_s16(x_filter_0_7, 5) == 128) {
+ const uint16x8_t horiz_const = vdupq_n_u16((1 << (bd - 1)));
+ // Undo the horizontal offset in the calling function.
+ src_ptr += 5;
+
+ do {
+ const uint8_t *s = src_ptr;
+ int16_t *d = dst_ptr;
+ int width = w;
+
+ do {
+ uint8x8_t s0 = vld1_u8(s);
+ uint16x8_t d0 = vaddw_u8(horiz_const, s0);
+ d0 = vshlq_n_u16(d0, FILTER_BITS - ROUND0_BITS);
+ // Store 8 elements to avoid additional branches. This is safe if the
+ // actual block width is < 8 because the intermediate buffer is large
+ // enough to accommodate 128x128 blocks.
+ vst1q_s16(d, vreinterpretq_s16_u16(d0));
+
+ d += 8;
+ s += 8;
+ width -= 8;
+ } while (width > 0);
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ } while (--h != 0);
+
+ } else {
+ // Narrow filter values to 8-bit.
+ const int16x8x2_t x_filter_s16 = {
+ { x_filter_0_7, vcombine_s16(x_filter_8_11, vdup_n_s16(0)) }
+ };
+ const int8x16_t x_filter = vcombine_s8(vmovn_s16(x_filter_s16.val[0]),
+ vmovn_s16(x_filter_s16.val[1]));
+ // This shim of 1 << (ROUND0_BITS - 1) enables us to use non-rounding shifts
+ // - which are generally faster than rounding shifts on modern CPUs.
+ const int32x4_t horiz_const =
+ vdupq_n_s32((1 << (bd + FILTER_BITS - 1)) + (1 << (ROUND0_BITS - 1)));
+ const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
+
+ if (w <= 4) {
+ do {
+ uint8x16_t s0, s1, s2, s3;
+ load_u8_16x4(src_ptr, src_stride, &s0, &s1, &s2, &s3);
+
+ int16x4_t d0 =
+ convolve12_4_2d_h(s0, x_filter, permute_tbl, horiz_const);
+ int16x4_t d1 =
+ convolve12_4_2d_h(s1, x_filter, permute_tbl, horiz_const);
+ int16x4_t d2 =
+ convolve12_4_2d_h(s2, x_filter, permute_tbl, horiz_const);
+ int16x4_t d3 =
+ convolve12_4_2d_h(s3, x_filter, permute_tbl, horiz_const);
+
+ store_s16_4x4(dst_ptr, dst_stride, d0, d1, d2, d3);
+
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ h -= 4;
+ } while (h > 4);
+
+ do {
+ uint8x16_t s0 = vld1q_u8(src_ptr);
+ int16x4_t d0 =
+ convolve12_4_2d_h(s0, x_filter, permute_tbl, horiz_const);
+ vst1_s16(dst_ptr, d0);
+
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ } while (--h != 0);
+
+ } else {
+ do {
+ const uint8_t *s = src_ptr;
+ int16_t *d = dst_ptr;
+ int width = w;
+
+ do {
+ uint8x16_t s0[2], s1[2], s2[2], s3[2];
+ load_u8_16x4(s, src_stride, &s0[0], &s1[0], &s2[0], &s3[0]);
+ load_u8_16x4(s + 4, src_stride, &s0[1], &s1[1], &s2[1], &s3[1]);
+
+ int16x8_t d0 =
+ convolve12_8_2d_h(s0, x_filter, permute_tbl, horiz_const);
+ int16x8_t d1 =
+ convolve12_8_2d_h(s1, x_filter, permute_tbl, horiz_const);
+ int16x8_t d2 =
+ convolve12_8_2d_h(s2, x_filter, permute_tbl, horiz_const);
+ int16x8_t d3 =
+ convolve12_8_2d_h(s3, x_filter, permute_tbl, horiz_const);
+
+ store_s16_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ h -= 4;
+ } while (h > 4);
+
+ do {
+ const uint8_t *s = src_ptr;
+ int16_t *d = dst_ptr;
+ int width = w;
+
+ do {
+ uint8x16_t s0[2];
+ s0[0] = vld1q_u8(s);
+ s0[1] = vld1q_u8(s + 4);
+ int16x8_t d0 =
+ convolve12_8_2d_h(s0, x_filter, permute_tbl, horiz_const);
+ vst1q_s16(d, d0);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ } while (--h != 0);
+ }
+ }
+}
+
+static INLINE int16x4_t convolve4_4_2d_h(uint8x16_t samples,
+ const int8x8_t filters,
+ const uint8x16_t permute_tbl,
+ const int32x4_t horiz_const) {
+ // Permute samples ready for dot product.
+ // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
+ uint8x16_t permuted_samples = vqtbl1q_u8(samples, permute_tbl);
+
+ // First 4 output values.
+ int32x4_t sum = vusdotq_lane_s32(horiz_const, permuted_samples, filters, 0);
+
+ // We halved the convolution filter values so -1 from the right shift.
+ return vshrn_n_s32(sum, ROUND0_BITS - 1);
+}
+
+static INLINE int16x8_t convolve8_8_2d_h(uint8x16_t samples,
+ const int8x8_t filters,
+ const uint8x16x3_t permute_tbl,
+ const int32x4_t horiz_const) {
+ uint8x16_t permuted_samples[3];
+ int32x4_t sum[2];
+
+ // Permute samples ready for dot product.
+ // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
+ permuted_samples[0] = vqtbl1q_u8(samples, permute_tbl.val[0]);
+ // { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 }
+ permuted_samples[1] = vqtbl1q_u8(samples, permute_tbl.val[1]);
+ // { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 }
+ permuted_samples[2] = vqtbl1q_u8(samples, permute_tbl.val[2]);
+
+ // First 4 output values.
+ sum[0] = vusdotq_lane_s32(horiz_const, permuted_samples[0], filters, 0);
+ sum[0] = vusdotq_lane_s32(sum[0], permuted_samples[1], filters, 1);
+ // Second 4 output values.
+ sum[1] = vusdotq_lane_s32(horiz_const, permuted_samples[1], filters, 0);
+ sum[1] = vusdotq_lane_s32(sum[1], permuted_samples[2], filters, 1);
+
+ // Narrow and re-pack.
+ // We halved the convolution filter values so -1 from the right shift.
+ return vcombine_s16(vshrn_n_s32(sum[0], ROUND0_BITS - 1),
+ vshrn_n_s32(sum[1], ROUND0_BITS - 1));
+}
+
+static INLINE void convolve_2d_sr_horiz_neon_i8mm(
+ const uint8_t *src, int src_stride, int16_t *im_block, int im_stride, int w,
+ int im_h, const int16_t *x_filter_ptr) {
+ const int bd = 8;
+ // This shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-rounding
+ // shifts - which are generally faster than rounding shifts on modern CPUs.
+ // The outermost -1 is needed because we halved the filter values.
+ const int32x4_t horiz_const = vdupq_n_s32((1 << (bd + FILTER_BITS - 2)) +
+ (1 << ((ROUND0_BITS - 1) - 1)));
+
+ const uint8_t *src_ptr = src;
+ int16_t *dst_ptr = im_block;
+ int dst_stride = im_stride;
+ int height = im_h;
+
+ if (w <= 4) {
+ const uint8x16_t permute_tbl = vld1q_u8(dot_prod_permute_tbl);
+ // 4-tap filters are used for blocks having width <= 4.
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int8x8_t x_filter =
+ vshrn_n_s16(vcombine_s16(vld1_s16(x_filter_ptr + 2), vdup_n_s16(0)), 1);
+
+ src_ptr += 2;
+
+ do {
+ uint8x16_t s0, s1, s2, s3;
+ load_u8_16x4(src_ptr, src_stride, &s0, &s1, &s2, &s3);
+
+ int16x4_t d0 = convolve4_4_2d_h(s0, x_filter, permute_tbl, horiz_const);
+ int16x4_t d1 = convolve4_4_2d_h(s1, x_filter, permute_tbl, horiz_const);
+ int16x4_t d2 = convolve4_4_2d_h(s2, x_filter, permute_tbl, horiz_const);
+ int16x4_t d3 = convolve4_4_2d_h(s3, x_filter, permute_tbl, horiz_const);
+
+ store_s16_4x4(dst_ptr, dst_stride, d0, d1, d2, d3);
+
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ height -= 4;
+ } while (height > 4);
+
+ do {
+ uint8x16_t s0 = vld1q_u8(src_ptr);
+ int16x4_t d0 = convolve4_4_2d_h(s0, x_filter, permute_tbl, horiz_const);
+ vst1_s16(dst_ptr, d0);
+
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ } while (--height != 0);
+ } else {
+ const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
+ // Filter values are even, so halve to reduce intermediate precision reqs.
+ const int8x8_t x_filter = vshrn_n_s16(vld1q_s16(x_filter_ptr), 1);
+
+ do {
+ const uint8_t *s = src_ptr;
+ int16_t *d = dst_ptr;
+ int width = w;
+
+ do {
+ uint8x16_t s0, s1, s2, s3;
+ load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3);
+
+ int16x8_t d0 = convolve8_8_2d_h(s0, x_filter, permute_tbl, horiz_const);
+ int16x8_t d1 = convolve8_8_2d_h(s1, x_filter, permute_tbl, horiz_const);
+ int16x8_t d2 = convolve8_8_2d_h(s2, x_filter, permute_tbl, horiz_const);
+ int16x8_t d3 = convolve8_8_2d_h(s3, x_filter, permute_tbl, horiz_const);
+
+ store_s16_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ height -= 4;
+ } while (height > 4);
+
+ do {
+ const uint8_t *s = src_ptr;
+ int16_t *d = dst_ptr;
+ int width = w;
+
+ do {
+ uint8x16_t s0 = vld1q_u8(s);
+ int16x8_t d0 = convolve8_8_2d_h(s0, x_filter, permute_tbl, horiz_const);
+ vst1q_s16(d, d0);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ } while (--height != 0);
+ }
+}
+
+void av1_convolve_2d_sr_neon_i8mm(const uint8_t *src, int src_stride,
+ uint8_t *dst, int dst_stride, int w, int h,
+ const InterpFilterParams *filter_params_x,
+ const InterpFilterParams *filter_params_y,
+ const int subpel_x_qn, const int subpel_y_qn,
+ ConvolveParams *conv_params) {
+ if (w == 2 || h == 2) {
+ av1_convolve_2d_sr_c(src, src_stride, dst, dst_stride, w, h,
+ filter_params_x, filter_params_y, subpel_x_qn,
+ subpel_y_qn, conv_params);
+ return;
+ }
+
+ const int y_filter_taps = get_filter_tap(filter_params_y, subpel_y_qn);
+ const int clamped_y_taps = y_filter_taps < 6 ? 6 : y_filter_taps;
+ const int im_h = h + clamped_y_taps - 1;
+ const int im_stride = MAX_SB_SIZE;
+ const int vert_offset = clamped_y_taps / 2 - 1;
+ const int horiz_offset = filter_params_x->taps / 2 - 1;
+ const uint8_t *src_ptr = src - vert_offset * src_stride - horiz_offset;
+
+ const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
+ filter_params_x, subpel_x_qn & SUBPEL_MASK);
+ const int16_t *y_filter_ptr = av1_get_interp_filter_subpel_kernel(
+ filter_params_y, subpel_y_qn & SUBPEL_MASK);
+
+ if (filter_params_x->taps > 8) {
+ DECLARE_ALIGNED(16, int16_t,
+ im_block[(MAX_SB_SIZE + MAX_FILTER_TAP - 1) * MAX_SB_SIZE]);
+
+ const int16x8_t x_filter_0_7 = vld1q_s16(x_filter_ptr);
+ const int16x4_t x_filter_8_11 = vld1_s16(x_filter_ptr + 8);
+ const int16x8_t y_filter_0_7 = vld1q_s16(y_filter_ptr);
+ const int16x4_t y_filter_8_11 = vld1_s16(y_filter_ptr + 8);
+
+ convolve_2d_sr_horiz_12tap_neon_i8mm(src_ptr, src_stride, im_block,
+ im_stride, w, im_h, x_filter_0_7,
+ x_filter_8_11);
+
+ convolve_2d_sr_vert_12tap_neon(im_block, im_stride, dst, dst_stride, w, h,
+ y_filter_0_7, y_filter_8_11);
+ } else {
+ DECLARE_ALIGNED(16, int16_t,
+ im_block[(MAX_SB_SIZE + SUBPEL_TAPS - 1) * MAX_SB_SIZE]);
+
+ convolve_2d_sr_horiz_neon_i8mm(src_ptr, src_stride, im_block, im_stride, w,
+ im_h, x_filter_ptr);
+
+ const int16x8_t y_filter = vld1q_s16(y_filter_ptr);
+
+ if (clamped_y_taps <= 6) {
+ convolve_2d_sr_vert_6tap_neon(im_block, im_stride, dst, dst_stride, w, h,
+ y_filter);
+ } else {
+ convolve_2d_sr_vert_8tap_neon(im_block, im_stride, dst, dst_stride, w, h,
+ y_filter);
+ }
+ }
+}
diff --git a/av1/common/arm/highbd_compound_convolve_neon.c b/av1/common/arm/highbd_compound_convolve_neon.c
new file mode 100644
index 0000000..dc3f876
--- /dev/null
+++ b/av1/common/arm/highbd_compound_convolve_neon.c
@@ -0,0 +1,2031 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <assert.h>
+#include <arm_neon.h>
+
+#include "config/aom_config.h"
+#include "config/av1_rtcd.h"
+
+#include "aom_dsp/aom_dsp_common.h"
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_ports/mem.h"
+#include "av1/common/convolve.h"
+#include "av1/common/filter.h"
+#include "av1/common/arm/highbd_convolve_neon.h"
+
+#define ROUND_SHIFT 2 * FILTER_BITS - ROUND0_BITS - COMPOUND_ROUND1_BITS
+
+static INLINE void highbd_12_comp_avg_neon(const uint16_t *src_ptr,
+ int src_stride, uint16_t *dst_ptr,
+ int dst_stride, int w, int h,
+ ConvolveParams *conv_params,
+ const int offset, const int bd) {
+ CONV_BUF_TYPE *ref_ptr = conv_params->dst;
+ const int ref_stride = conv_params->dst_stride;
+ const uint16x4_t offset_vec = vdup_n_u16(offset);
+ const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
+
+ if (w == 4) {
+ do {
+ const uint16x4_t src = vld1_u16(src_ptr);
+ const uint16x4_t ref = vld1_u16(ref_ptr);
+
+ uint16x4_t avg = vhadd_u16(src, ref);
+ int32x4_t d0 = vreinterpretq_s32_u32(vsubl_u16(avg, offset_vec));
+
+ uint16x4_t d0_u16 = vqrshrun_n_s32(d0, ROUND_SHIFT - 2);
+ d0_u16 = vmin_u16(d0_u16, vget_low_u16(max));
+
+ vst1_u16(dst_ptr, d0_u16);
+
+ src_ptr += src_stride;
+ ref_ptr += ref_stride;
+ dst_ptr += dst_stride;
+ } while (--h != 0);
+ } else {
+ do {
+ int width = w;
+ const uint16_t *src = src_ptr;
+ const uint16_t *ref = ref_ptr;
+ uint16_t *dst = dst_ptr;
+ do {
+ const uint16x8_t s = vld1q_u16(src);
+ const uint16x8_t r = vld1q_u16(ref);
+
+ uint16x8_t avg = vhaddq_u16(s, r);
+ int32x4_t d0_lo =
+ vreinterpretq_s32_u32(vsubl_u16(vget_low_u16(avg), offset_vec));
+ int32x4_t d0_hi =
+ vreinterpretq_s32_u32(vsubl_u16(vget_high_u16(avg), offset_vec));
+
+ uint16x8_t d0 = vcombine_u16(vqrshrun_n_s32(d0_lo, ROUND_SHIFT - 2),
+ vqrshrun_n_s32(d0_hi, ROUND_SHIFT - 2));
+ d0 = vminq_u16(d0, max);
+ vst1q_u16(dst, d0);
+
+ src += 8;
+ ref += 8;
+ dst += 8;
+ width -= 8;
+ } while (width != 0);
+
+ src_ptr += src_stride;
+ ref_ptr += ref_stride;
+ dst_ptr += dst_stride;
+ } while (--h != 0);
+ }
+}
+
+static INLINE void highbd_comp_avg_neon(const uint16_t *src_ptr, int src_stride,
+ uint16_t *dst_ptr, int dst_stride,
+ int w, int h,
+ ConvolveParams *conv_params,
+ const int offset, const int bd) {
+ CONV_BUF_TYPE *ref_ptr = conv_params->dst;
+ const int ref_stride = conv_params->dst_stride;
+ const uint16x4_t offset_vec = vdup_n_u16(offset);
+ const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
+
+ if (w == 4) {
+ do {
+ const uint16x4_t src = vld1_u16(src_ptr);
+ const uint16x4_t ref = vld1_u16(ref_ptr);
+
+ uint16x4_t avg = vhadd_u16(src, ref);
+ int32x4_t d0 = vreinterpretq_s32_u32(vsubl_u16(avg, offset_vec));
+
+ uint16x4_t d0_u16 = vqrshrun_n_s32(d0, ROUND_SHIFT);
+ d0_u16 = vmin_u16(d0_u16, vget_low_u16(max));
+
+ vst1_u16(dst_ptr, d0_u16);
+
+ src_ptr += src_stride;
+ ref_ptr += ref_stride;
+ dst_ptr += dst_stride;
+ } while (--h != 0);
+ } else {
+ do {
+ int width = w;
+ const uint16_t *src = src_ptr;
+ const uint16_t *ref = ref_ptr;
+ uint16_t *dst = dst_ptr;
+ do {
+ const uint16x8_t s = vld1q_u16(src);
+ const uint16x8_t r = vld1q_u16(ref);
+
+ uint16x8_t avg = vhaddq_u16(s, r);
+ int32x4_t d0_lo =
+ vreinterpretq_s32_u32(vsubl_u16(vget_low_u16(avg), offset_vec));
+ int32x4_t d0_hi =
+ vreinterpretq_s32_u32(vsubl_u16(vget_high_u16(avg), offset_vec));
+
+ uint16x8_t d0 = vcombine_u16(vqrshrun_n_s32(d0_lo, ROUND_SHIFT),
+ vqrshrun_n_s32(d0_hi, ROUND_SHIFT));
+ d0 = vminq_u16(d0, max);
+ vst1q_u16(dst, d0);
+
+ src += 8;
+ ref += 8;
+ dst += 8;
+ width -= 8;
+ } while (width != 0);
+
+ src_ptr += src_stride;
+ ref_ptr += ref_stride;
+ dst_ptr += dst_stride;
+ } while (--h != 0);
+ }
+}
+
+static INLINE void highbd_12_dist_wtd_comp_avg_neon(
+ const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
+ int w, int h, ConvolveParams *conv_params, const int offset, const int bd) {
+ CONV_BUF_TYPE *ref_ptr = conv_params->dst;
+ const int ref_stride = conv_params->dst_stride;
+ const uint32x4_t offset_vec = vdupq_n_u32(offset);
+ const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
+ uint16x4_t fwd_offset = vdup_n_u16(conv_params->fwd_offset);
+ uint16x4_t bck_offset = vdup_n_u16(conv_params->bck_offset);
+
+ // Weighted averaging
+ if (w == 4) {
+ do {
+ const uint16x4_t src = vld1_u16(src_ptr);
+ const uint16x4_t ref = vld1_u16(ref_ptr);
+
+ uint32x4_t wtd_avg = vmull_u16(ref, fwd_offset);
+ wtd_avg = vmlal_u16(wtd_avg, src, bck_offset);
+ wtd_avg = vshrq_n_u32(wtd_avg, DIST_PRECISION_BITS);
+ int32x4_t d0 = vreinterpretq_s32_u32(vsubq_u32(wtd_avg, offset_vec));
+
+ uint16x4_t d0_u16 = vqrshrun_n_s32(d0, ROUND_SHIFT - 2);
+ d0_u16 = vmin_u16(d0_u16, vget_low_u16(max));
+
+ vst1_u16(dst_ptr, d0_u16);
+
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ ref_ptr += ref_stride;
+ } while (--h != 0);
+ } else {
+ do {
+ int width = w;
+ const uint16_t *src = src_ptr;
+ const uint16_t *ref = ref_ptr;
+ uint16_t *dst = dst_ptr;
+ do {
+ const uint16x8_t s = vld1q_u16(src);
+ const uint16x8_t r = vld1q_u16(ref);
+
+ uint32x4_t wtd_avg0 = vmull_u16(vget_low_u16(r), fwd_offset);
+ wtd_avg0 = vmlal_u16(wtd_avg0, vget_low_u16(s), bck_offset);
+ wtd_avg0 = vshrq_n_u32(wtd_avg0, DIST_PRECISION_BITS);
+ int32x4_t d0 = vreinterpretq_s32_u32(vsubq_u32(wtd_avg0, offset_vec));
+
+ uint32x4_t wtd_avg1 = vmull_u16(vget_high_u16(r), fwd_offset);
+ wtd_avg1 = vmlal_u16(wtd_avg1, vget_high_u16(s), bck_offset);
+ wtd_avg1 = vshrq_n_u32(wtd_avg1, DIST_PRECISION_BITS);
+ int32x4_t d1 = vreinterpretq_s32_u32(vsubq_u32(wtd_avg1, offset_vec));
+
+ uint16x8_t d01 = vcombine_u16(vqrshrun_n_s32(d0, ROUND_SHIFT - 2),
+ vqrshrun_n_s32(d1, ROUND_SHIFT - 2));
+ d01 = vminq_u16(d01, max);
+ vst1q_u16(dst, d01);
+
+ src += 8;
+ ref += 8;
+ dst += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ ref_ptr += ref_stride;
+ } while (--h != 0);
+ }
+}
+
+static INLINE void highbd_dist_wtd_comp_avg_neon(
+ const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
+ int w, int h, ConvolveParams *conv_params, const int offset, const int bd) {
+ CONV_BUF_TYPE *ref_ptr = conv_params->dst;
+ const int ref_stride = conv_params->dst_stride;
+ const uint32x4_t offset_vec = vdupq_n_u32(offset);
+ const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
+ uint16x4_t fwd_offset = vdup_n_u16(conv_params->fwd_offset);
+ uint16x4_t bck_offset = vdup_n_u16(conv_params->bck_offset);
+
+ // Weighted averaging
+ if (w == 4) {
+ do {
+ const uint16x4_t src = vld1_u16(src_ptr);
+ const uint16x4_t ref = vld1_u16(ref_ptr);
+
+ uint32x4_t wtd_avg = vmull_u16(ref, fwd_offset);
+ wtd_avg = vmlal_u16(wtd_avg, src, bck_offset);
+ wtd_avg = vshrq_n_u32(wtd_avg, DIST_PRECISION_BITS);
+ int32x4_t d0 = vreinterpretq_s32_u32(vsubq_u32(wtd_avg, offset_vec));
+
+ uint16x4_t d0_u16 = vqrshrun_n_s32(d0, ROUND_SHIFT);
+ d0_u16 = vmin_u16(d0_u16, vget_low_u16(max));
+
+ vst1_u16(dst_ptr, d0_u16);
+
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ ref_ptr += ref_stride;
+ } while (--h != 0);
+ } else {
+ do {
+ int width = w;
+ const uint16_t *src = src_ptr;
+ const uint16_t *ref = ref_ptr;
+ uint16_t *dst = dst_ptr;
+ do {
+ const uint16x8_t s = vld1q_u16(src);
+ const uint16x8_t r = vld1q_u16(ref);
+
+ uint32x4_t wtd_avg0 = vmull_u16(vget_low_u16(r), fwd_offset);
+ wtd_avg0 = vmlal_u16(wtd_avg0, vget_low_u16(s), bck_offset);
+ wtd_avg0 = vshrq_n_u32(wtd_avg0, DIST_PRECISION_BITS);
+ int32x4_t d0 = vreinterpretq_s32_u32(vsubq_u32(wtd_avg0, offset_vec));
+
+ uint32x4_t wtd_avg1 = vmull_u16(vget_high_u16(r), fwd_offset);
+ wtd_avg1 = vmlal_u16(wtd_avg1, vget_high_u16(s), bck_offset);
+ wtd_avg1 = vshrq_n_u32(wtd_avg1, DIST_PRECISION_BITS);
+ int32x4_t d1 = vreinterpretq_s32_u32(vsubq_u32(wtd_avg1, offset_vec));
+
+ uint16x8_t d01 = vcombine_u16(vqrshrun_n_s32(d0, ROUND_SHIFT),
+ vqrshrun_n_s32(d1, ROUND_SHIFT));
+ d01 = vminq_u16(d01, max);
+ vst1q_u16(dst, d01);
+
+ src += 8;
+ ref += 8;
+ dst += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ ref_ptr += ref_stride;
+ } while (--h != 0);
+ }
+}
+
+static INLINE uint16x4_t highbd_12_convolve6_4(
+ const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
+ const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
+ const int16x8_t filter, const int32x4_t offset) {
+ // Values at indices 0 and 7 of y_filter are zero.
+ const int16x4_t filter_0_3 = vget_low_s16(filter);
+ const int16x4_t filter_4_7 = vget_high_s16(filter);
+
+ int32x4_t sum = vmlal_lane_s16(offset, s0, filter_0_3, 1);
+ sum = vmlal_lane_s16(sum, s1, filter_0_3, 2);
+ sum = vmlal_lane_s16(sum, s2, filter_0_3, 3);
+ sum = vmlal_lane_s16(sum, s3, filter_4_7, 0);
+ sum = vmlal_lane_s16(sum, s4, filter_4_7, 1);
+ sum = vmlal_lane_s16(sum, s5, filter_4_7, 2);
+
+ return vqshrun_n_s32(sum, ROUND0_BITS + 2);
+}
+
+static INLINE uint16x4_t
+highbd_convolve6_4(const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
+ const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
+ const int16x8_t filter, const int32x4_t offset) {
+ // Values at indices 0 and 7 of y_filter are zero.
+ const int16x4_t filter_0_3 = vget_low_s16(filter);
+ const int16x4_t filter_4_7 = vget_high_s16(filter);
+
+ int32x4_t sum = vmlal_lane_s16(offset, s0, filter_0_3, 1);
+ sum = vmlal_lane_s16(sum, s1, filter_0_3, 2);
+ sum = vmlal_lane_s16(sum, s2, filter_0_3, 3);
+ sum = vmlal_lane_s16(sum, s3, filter_4_7, 0);
+ sum = vmlal_lane_s16(sum, s4, filter_4_7, 1);
+ sum = vmlal_lane_s16(sum, s5, filter_4_7, 2);
+
+ return vqshrun_n_s32(sum, ROUND0_BITS);
+}
+
+static INLINE uint16x8_t highbd_12_convolve6_8(
+ const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
+ const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t filter, const int32x4_t offset) {
+ // Values at indices 0 and 7 of y_filter are zero.
+ const int16x4_t filter_0_3 = vget_low_s16(filter);
+ const int16x4_t filter_4_7 = vget_high_s16(filter);
+
+ int32x4_t sum0 = vmlal_lane_s16(offset, vget_low_s16(s0), filter_0_3, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s1), filter_0_3, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s2), filter_0_3, 3);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s3), filter_4_7, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s4), filter_4_7, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s5), filter_4_7, 2);
+
+ int32x4_t sum1 = vmlal_lane_s16(offset, vget_high_s16(s0), filter_0_3, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s1), filter_0_3, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s2), filter_0_3, 3);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s3), filter_4_7, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s4), filter_4_7, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s5), filter_4_7, 2);
+
+ return vcombine_u16(vqshrun_n_s32(sum0, ROUND0_BITS + 2),
+ vqshrun_n_s32(sum1, ROUND0_BITS + 2));
+}
+
+static INLINE uint16x8_t
+highbd_convolve6_8(const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
+ const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t filter, const int32x4_t offset) {
+ // Values at indices 0 and 7 of y_filter are zero.
+ const int16x4_t filter_0_3 = vget_low_s16(filter);
+ const int16x4_t filter_4_7 = vget_high_s16(filter);
+
+ int32x4_t sum0 = vmlal_lane_s16(offset, vget_low_s16(s0), filter_0_3, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s1), filter_0_3, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s2), filter_0_3, 3);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s3), filter_4_7, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s4), filter_4_7, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s5), filter_4_7, 2);
+
+ int32x4_t sum1 = vmlal_lane_s16(offset, vget_high_s16(s0), filter_0_3, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s1), filter_0_3, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s2), filter_0_3, 3);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s3), filter_4_7, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s4), filter_4_7, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s5), filter_4_7, 2);
+
+ return vcombine_u16(vqshrun_n_s32(sum0, 3), vqshrun_n_s32(sum1, ROUND0_BITS));
+}
+
+static INLINE void highbd_12_dist_wtd_convolve_x_6tap_neon(
+ const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
+ int w, int h, const int16_t *x_filter_ptr, const int offset) {
+ const int32x4_t offset_vec = vdupq_n_s32(offset);
+
+ const int16x8_t x_filter = vld1q_s16(x_filter_ptr);
+
+ int height = h;
+
+ do {
+ int width = w;
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ do {
+ int16x8_t s0[6], s1[6], s2[6], s3[6];
+ load_s16_8x6(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3],
+ &s0[4], &s0[5]);
+ load_s16_8x6(s + 1 * src_stride, 1, &s1[0], &s1[1], &s1[2], &s1[3],
+ &s1[4], &s1[5]);
+ load_s16_8x6(s + 2 * src_stride, 1, &s2[0], &s2[1], &s2[2], &s2[3],
+ &s2[4], &s2[5]);
+ load_s16_8x6(s + 3 * src_stride, 1, &s3[0], &s3[1], &s3[2], &s3[3],
+ &s3[4], &s3[5]);
+
+ uint16x8_t d0 = highbd_12_convolve6_8(s0[0], s0[1], s0[2], s0[3], s0[4],
+ s0[5], x_filter, offset_vec);
+ uint16x8_t d1 = highbd_12_convolve6_8(s1[0], s1[1], s1[2], s1[3], s1[4],
+ s1[5], x_filter, offset_vec);
+ uint16x8_t d2 = highbd_12_convolve6_8(s2[0], s2[1], s2[2], s2[3], s2[4],
+ s2[5], x_filter, offset_vec);
+ uint16x8_t d3 = highbd_12_convolve6_8(s3[0], s3[1], s3[2], s3[3], s3[4],
+ s3[5], x_filter, offset_vec);
+
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ height -= 4;
+ } while (height != 0);
+}
+
+static INLINE void highbd_dist_wtd_convolve_x_6tap_neon(
+ const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
+ int w, int h, const int16_t *x_filter_ptr, const int offset) {
+ const int32x4_t offset_vec = vdupq_n_s32(offset);
+
+ const int16x8_t x_filter = vld1q_s16(x_filter_ptr);
+
+ int height = h;
+
+ do {
+ int width = w;
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ do {
+ int16x8_t s0[6], s1[6], s2[6], s3[6];
+ load_s16_8x6(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3],
+ &s0[4], &s0[5]);
+ load_s16_8x6(s + 1 * src_stride, 1, &s1[0], &s1[1], &s1[2], &s1[3],
+ &s1[4], &s1[5]);
+ load_s16_8x6(s + 2 * src_stride, 1, &s2[0], &s2[1], &s2[2], &s2[3],
+ &s2[4], &s2[5]);
+ load_s16_8x6(s + 3 * src_stride, 1, &s3[0], &s3[1], &s3[2], &s3[3],
+ &s3[4], &s3[5]);
+
+ uint16x8_t d0 = highbd_convolve6_8(s0[0], s0[1], s0[2], s0[3], s0[4],
+ s0[5], x_filter, offset_vec);
+ uint16x8_t d1 = highbd_convolve6_8(s1[0], s1[1], s1[2], s1[3], s1[4],
+ s1[5], x_filter, offset_vec);
+ uint16x8_t d2 = highbd_convolve6_8(s2[0], s2[1], s2[2], s2[3], s2[4],
+ s2[5], x_filter, offset_vec);
+ uint16x8_t d3 = highbd_convolve6_8(s3[0], s3[1], s3[2], s3[3], s3[4],
+ s3[5], x_filter, offset_vec);
+
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ height -= 4;
+ } while (height != 0);
+}
+
+static INLINE uint16x4_t highbd_12_convolve8_4(
+ const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
+ const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
+ const int16x4_t s6, const int16x4_t s7, const int16x8_t filter,
+ const int32x4_t offset) {
+ const int16x4_t filter_0_3 = vget_low_s16(filter);
+ const int16x4_t filter_4_7 = vget_high_s16(filter);
+
+ int32x4_t sum = vmlal_lane_s16(offset, s0, filter_0_3, 0);
+ sum = vmlal_lane_s16(sum, s1, filter_0_3, 1);
+ sum = vmlal_lane_s16(sum, s2, filter_0_3, 2);
+ sum = vmlal_lane_s16(sum, s3, filter_0_3, 3);
+ sum = vmlal_lane_s16(sum, s4, filter_4_7, 0);
+ sum = vmlal_lane_s16(sum, s5, filter_4_7, 1);
+ sum = vmlal_lane_s16(sum, s6, filter_4_7, 2);
+ sum = vmlal_lane_s16(sum, s7, filter_4_7, 3);
+
+ return vqshrun_n_s32(sum, ROUND0_BITS + 2);
+}
+
+static INLINE uint16x4_t
+highbd_convolve8_4(const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
+ const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
+ const int16x4_t s6, const int16x4_t s7,
+ const int16x8_t filter, const int32x4_t offset) {
+ const int16x4_t filter_0_3 = vget_low_s16(filter);
+ const int16x4_t filter_4_7 = vget_high_s16(filter);
+
+ int32x4_t sum = vmlal_lane_s16(offset, s0, filter_0_3, 0);
+ sum = vmlal_lane_s16(sum, s1, filter_0_3, 1);
+ sum = vmlal_lane_s16(sum, s2, filter_0_3, 2);
+ sum = vmlal_lane_s16(sum, s3, filter_0_3, 3);
+ sum = vmlal_lane_s16(sum, s4, filter_4_7, 0);
+ sum = vmlal_lane_s16(sum, s5, filter_4_7, 1);
+ sum = vmlal_lane_s16(sum, s6, filter_4_7, 2);
+ sum = vmlal_lane_s16(sum, s7, filter_4_7, 3);
+
+ return vqshrun_n_s32(sum, ROUND0_BITS);
+}
+
+static INLINE uint16x8_t highbd_12_convolve8_8(
+ const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
+ const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t s6, const int16x8_t s7, const int16x8_t filter,
+ const int32x4_t offset) {
+ const int16x4_t filter_0_3 = vget_low_s16(filter);
+ const int16x4_t filter_4_7 = vget_high_s16(filter);
+
+ int32x4_t sum0 = vmlal_lane_s16(offset, vget_low_s16(s0), filter_0_3, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s1), filter_0_3, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s2), filter_0_3, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s3), filter_0_3, 3);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s4), filter_4_7, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s5), filter_4_7, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s6), filter_4_7, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s7), filter_4_7, 3);
+
+ int32x4_t sum1 = vmlal_lane_s16(offset, vget_high_s16(s0), filter_0_3, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s1), filter_0_3, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s2), filter_0_3, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s3), filter_0_3, 3);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s4), filter_4_7, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s5), filter_4_7, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s6), filter_4_7, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s7), filter_4_7, 3);
+
+ return vcombine_u16(vqshrun_n_s32(sum0, ROUND0_BITS + 2),
+ vqshrun_n_s32(sum1, ROUND0_BITS + 2));
+}
+
+static INLINE uint16x8_t
+highbd_convolve8_8(const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
+ const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t s6, const int16x8_t s7,
+ const int16x8_t filter, const int32x4_t offset) {
+ const int16x4_t filter_0_3 = vget_low_s16(filter);
+ const int16x4_t filter_4_7 = vget_high_s16(filter);
+
+ int32x4_t sum0 = vmlal_lane_s16(offset, vget_low_s16(s0), filter_0_3, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s1), filter_0_3, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s2), filter_0_3, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s3), filter_0_3, 3);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s4), filter_4_7, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s5), filter_4_7, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s6), filter_4_7, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s7), filter_4_7, 3);
+
+ int32x4_t sum1 = vmlal_lane_s16(offset, vget_high_s16(s0), filter_0_3, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s1), filter_0_3, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s2), filter_0_3, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s3), filter_0_3, 3);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s4), filter_4_7, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s5), filter_4_7, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s6), filter_4_7, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s7), filter_4_7, 3);
+
+ return vcombine_u16(vqshrun_n_s32(sum0, ROUND0_BITS),
+ vqshrun_n_s32(sum1, ROUND0_BITS));
+}
+
+static INLINE uint16x4_t highbd_12_convolve4_4_x(const int16x4_t s[4],
+ const int16x4_t x_filter,
+ const int32x4_t offset) {
+ int32x4_t sum = vmlal_lane_s16(offset, s[0], x_filter, 0);
+ sum = vmlal_lane_s16(sum, s[1], x_filter, 1);
+ sum = vmlal_lane_s16(sum, s[2], x_filter, 2);
+ sum = vmlal_lane_s16(sum, s[3], x_filter, 3);
+
+ return vqshrun_n_s32(sum, 5);
+}
+
+static INLINE uint16x4_t highbd_convolve4_4_x(const int16x4_t s[4],
+ const int16x4_t x_filter,
+ const int32x4_t offset) {
+ int32x4_t sum = vmlal_lane_s16(offset, s[0], x_filter, 0);
+ sum = vmlal_lane_s16(sum, s[1], x_filter, 1);
+ sum = vmlal_lane_s16(sum, s[2], x_filter, 2);
+ sum = vmlal_lane_s16(sum, s[3], x_filter, 3);
+
+ return vqshrun_n_s32(sum, ROUND0_BITS);
+}
+
+static INLINE void highbd_12_dist_wtd_convolve_x_neon(
+ const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
+ int w, int h, const int16_t *x_filter_ptr, const int offset) {
+ const int32x4_t offset_vec = vdupq_n_s32(offset);
+
+ if (w == 4) {
+ // 4-tap filters are used for blocks having width == 4.
+ const int16x4_t x_filter = vld1_s16(x_filter_ptr + 2);
+ const int16_t *s = (const int16_t *)(src_ptr + 2);
+ uint16_t *d = dst_ptr;
+
+ do {
+ int16x4_t s0[4], s1[4], s2[4], s3[4];
+ load_s16_4x4(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3]);
+ load_s16_4x4(s + 1 * src_stride, 1, &s1[0], &s1[1], &s1[2], &s1[3]);
+ load_s16_4x4(s + 2 * src_stride, 1, &s2[0], &s2[1], &s2[2], &s2[3]);
+ load_s16_4x4(s + 3 * src_stride, 1, &s3[0], &s3[1], &s3[2], &s3[3]);
+
+ uint16x4_t d0 = highbd_12_convolve4_4_x(s0, x_filter, offset_vec);
+ uint16x4_t d1 = highbd_12_convolve4_4_x(s1, x_filter, offset_vec);
+ uint16x4_t d2 = highbd_12_convolve4_4_x(s2, x_filter, offset_vec);
+ uint16x4_t d3 = highbd_12_convolve4_4_x(s3, x_filter, offset_vec);
+
+ store_u16_4x4(d, dst_stride, d0, d1, d2, d3);
+
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ h -= 4;
+ } while (h != 0);
+ } else {
+ const int16x8_t x_filter = vld1q_s16(x_filter_ptr);
+ int height = h;
+
+ do {
+ int width = w;
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ do {
+ int16x8_t s0[8], s1[8], s2[8], s3[8];
+ load_s16_8x8(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3],
+ &s0[4], &s0[5], &s0[6], &s0[7]);
+ load_s16_8x8(s + 1 * src_stride, 1, &s1[0], &s1[1], &s1[2], &s1[3],
+ &s1[4], &s1[5], &s1[6], &s1[7]);
+ load_s16_8x8(s + 2 * src_stride, 1, &s2[0], &s2[1], &s2[2], &s2[3],
+ &s2[4], &s2[5], &s2[6], &s2[7]);
+ load_s16_8x8(s + 3 * src_stride, 1, &s3[0], &s3[1], &s3[2], &s3[3],
+ &s3[4], &s3[5], &s3[6], &s3[7]);
+
+ uint16x8_t d0 =
+ highbd_12_convolve8_8(s0[0], s0[1], s0[2], s0[3], s0[4], s0[5],
+ s0[6], s0[7], x_filter, offset_vec);
+ uint16x8_t d1 =
+ highbd_12_convolve8_8(s1[0], s1[1], s1[2], s1[3], s1[4], s1[5],
+ s1[6], s1[7], x_filter, offset_vec);
+ uint16x8_t d2 =
+ highbd_12_convolve8_8(s2[0], s2[1], s2[2], s2[3], s2[4], s2[5],
+ s2[6], s2[7], x_filter, offset_vec);
+ uint16x8_t d3 =
+ highbd_12_convolve8_8(s3[0], s3[1], s3[2], s3[3], s3[4], s3[5],
+ s3[6], s3[7], x_filter, offset_vec);
+
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ height -= 4;
+ } while (height != 0);
+ }
+}
+
+static INLINE void highbd_dist_wtd_convolve_x_neon(
+ const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
+ int w, int h, const int16_t *x_filter_ptr, const int offset) {
+ const int32x4_t offset_vec = vdupq_n_s32(offset);
+
+ if (w == 4) {
+ // 4-tap filters are used for blocks having width == 4.
+ const int16x4_t x_filter = vld1_s16(x_filter_ptr + 2);
+ const int16_t *s = (const int16_t *)(src_ptr + 2);
+ uint16_t *d = dst_ptr;
+
+ do {
+ int16x4_t s0[4], s1[4], s2[4], s3[4];
+ load_s16_4x4(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3]);
+ load_s16_4x4(s + 1 * src_stride, 1, &s1[0], &s1[1], &s1[2], &s1[3]);
+ load_s16_4x4(s + 2 * src_stride, 1, &s2[0], &s2[1], &s2[2], &s2[3]);
+ load_s16_4x4(s + 3 * src_stride, 1, &s3[0], &s3[1], &s3[2], &s3[3]);
+
+ uint16x4_t d0 = highbd_convolve4_4_x(s0, x_filter, offset_vec);
+ uint16x4_t d1 = highbd_convolve4_4_x(s1, x_filter, offset_vec);
+ uint16x4_t d2 = highbd_convolve4_4_x(s2, x_filter, offset_vec);
+ uint16x4_t d3 = highbd_convolve4_4_x(s3, x_filter, offset_vec);
+
+ store_u16_4x4(d, dst_stride, d0, d1, d2, d3);
+
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ h -= 4;
+ } while (h != 0);
+ } else {
+ const int16x8_t x_filter = vld1q_s16(x_filter_ptr);
+ int height = h;
+
+ do {
+ int width = w;
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ do {
+ int16x8_t s0[8], s1[8], s2[8], s3[8];
+ load_s16_8x8(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3],
+ &s0[4], &s0[5], &s0[6], &s0[7]);
+ load_s16_8x8(s + 1 * src_stride, 1, &s1[0], &s1[1], &s1[2], &s1[3],
+ &s1[4], &s1[5], &s1[6], &s1[7]);
+ load_s16_8x8(s + 2 * src_stride, 1, &s2[0], &s2[1], &s2[2], &s2[3],
+ &s2[4], &s2[5], &s2[6], &s2[7]);
+ load_s16_8x8(s + 3 * src_stride, 1, &s3[0], &s3[1], &s3[2], &s3[3],
+ &s3[4], &s3[5], &s3[6], &s3[7]);
+
+ uint16x8_t d0 =
+ highbd_convolve8_8(s0[0], s0[1], s0[2], s0[3], s0[4], s0[5], s0[6],
+ s0[7], x_filter, offset_vec);
+ uint16x8_t d1 =
+ highbd_convolve8_8(s1[0], s1[1], s1[2], s1[3], s1[4], s1[5], s1[6],
+ s1[7], x_filter, offset_vec);
+ uint16x8_t d2 =
+ highbd_convolve8_8(s2[0], s2[1], s2[2], s2[3], s2[4], s2[5], s2[6],
+ s2[7], x_filter, offset_vec);
+ uint16x8_t d3 =
+ highbd_convolve8_8(s3[0], s3[1], s3[2], s3[3], s3[4], s3[5], s3[6],
+ s3[7], x_filter, offset_vec);
+
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ height -= 4;
+ } while (height != 0);
+ }
+}
+
+void av1_highbd_dist_wtd_convolve_x_neon(
+ const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w,
+ int h, const InterpFilterParams *filter_params_x, const int subpel_x_qn,
+ ConvolveParams *conv_params, int bd) {
+ DECLARE_ALIGNED(16, uint16_t,
+ im_block[(MAX_SB_SIZE + MAX_FILTER_TAP) * MAX_SB_SIZE]);
+ CONV_BUF_TYPE *dst16 = conv_params->dst;
+ const int x_filter_taps = get_filter_tap(filter_params_x, subpel_x_qn);
+ int dst16_stride = conv_params->dst_stride;
+ const int im_stride = MAX_SB_SIZE;
+ const int horiz_offset = filter_params_x->taps / 2 - 1;
+ assert(FILTER_BITS == COMPOUND_ROUND1_BITS);
+ const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0;
+ const int offset_avg = (1 << (offset_bits - conv_params->round_1)) +
+ (1 << (offset_bits - conv_params->round_1 - 1));
+ const int offset_convolve = (1 << (conv_params->round_0 - 1)) +
+ (1 << (bd + FILTER_BITS)) +
+ (1 << (bd + FILTER_BITS - 1));
+
+ const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
+ filter_params_x, subpel_x_qn & SUBPEL_MASK);
+
+ src -= horiz_offset;
+
+ // horizontal filter
+ if (bd == 12) {
+ if (conv_params->do_average) {
+ if (x_filter_taps <= 6 && w != 4) {
+ highbd_12_dist_wtd_convolve_x_6tap_neon(src + 1, src_stride, im_block,
+ im_stride, w, h, x_filter_ptr,
+ offset_convolve);
+ } else {
+ highbd_12_dist_wtd_convolve_x_neon(src, src_stride, im_block, im_stride,
+ w, h, x_filter_ptr, offset_convolve);
+ }
+ if (conv_params->use_dist_wtd_comp_avg) {
+ highbd_12_dist_wtd_comp_avg_neon(im_block, im_stride, dst, dst_stride,
+ w, h, conv_params, offset_avg, bd);
+ } else {
+ highbd_12_comp_avg_neon(im_block, im_stride, dst, dst_stride, w, h,
+ conv_params, offset_avg, bd);
+ }
+ } else {
+ if (x_filter_taps <= 6 && w != 4) {
+ highbd_12_dist_wtd_convolve_x_6tap_neon(src + 1, src_stride, dst16,
+ dst16_stride, w, h,
+ x_filter_ptr, offset_convolve);
+ } else {
+ highbd_12_dist_wtd_convolve_x_neon(src, src_stride, dst16, dst16_stride,
+ w, h, x_filter_ptr, offset_convolve);
+ }
+ }
+ } else {
+ if (conv_params->do_average) {
+ if (x_filter_taps <= 6 && w != 4) {
+ highbd_dist_wtd_convolve_x_6tap_neon(src + 1, src_stride, im_block,
+ im_stride, w, h, x_filter_ptr,
+ offset_convolve);
+ } else {
+ highbd_dist_wtd_convolve_x_neon(src, src_stride, im_block, im_stride, w,
+ h, x_filter_ptr, offset_convolve);
+ }
+ if (conv_params->use_dist_wtd_comp_avg) {
+ highbd_dist_wtd_comp_avg_neon(im_block, im_stride, dst, dst_stride, w,
+ h, conv_params, offset_avg, bd);
+ } else {
+ highbd_comp_avg_neon(im_block, im_stride, dst, dst_stride, w, h,
+ conv_params, offset_avg, bd);
+ }
+ } else {
+ if (x_filter_taps <= 6 && w != 4) {
+ highbd_dist_wtd_convolve_x_6tap_neon(src + 1, src_stride, dst16,
+ dst16_stride, w, h, x_filter_ptr,
+ offset_convolve);
+ } else {
+ highbd_dist_wtd_convolve_x_neon(src, src_stride, dst16, dst16_stride, w,
+ h, x_filter_ptr, offset_convolve);
+ }
+ }
+ }
+}
+
+static INLINE void highbd_12_dist_wtd_convolve_y_6tap_neon(
+ const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
+ int w, int h, const int16_t *y_filter_ptr, const int offset) {
+ const int16x8_t y_filter = vld1q_s16(y_filter_ptr);
+ const int32x4_t offset_vec = vdupq_n_s32(offset);
+
+ if (w == 4) {
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ int16x4_t s0, s1, s2, s3, s4;
+ load_s16_4x5(s, src_stride, &s0, &s1, &s2, &s3, &s4);
+ s += 5 * src_stride;
+
+ do {
+ int16x4_t s5, s6, s7, s8;
+ load_s16_4x4(s, src_stride, &s5, &s6, &s7, &s8);
+
+ uint16x4_t d0 =
+ highbd_12_convolve6_4(s0, s1, s2, s3, s4, s5, y_filter, offset_vec);
+ uint16x4_t d1 =
+ highbd_12_convolve6_4(s1, s2, s3, s4, s5, s6, y_filter, offset_vec);
+ uint16x4_t d2 =
+ highbd_12_convolve6_4(s2, s3, s4, s5, s6, s7, y_filter, offset_vec);
+ uint16x4_t d3 =
+ highbd_12_convolve6_4(s3, s4, s5, s6, s7, s8, y_filter, offset_vec);
+
+ store_u16_4x4(d, dst_stride, d0, d1, d2, d3);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ h -= 4;
+ } while (h != 0);
+ } else {
+ do {
+ int height = h;
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ int16x8_t s0, s1, s2, s3, s4;
+ load_s16_8x5(s, src_stride, &s0, &s1, &s2, &s3, &s4);
+ s += 5 * src_stride;
+
+ do {
+ int16x8_t s5, s6, s7, s8;
+ load_s16_8x4(s, src_stride, &s5, &s6, &s7, &s8);
+
+ uint16x8_t d0 =
+ highbd_12_convolve6_8(s0, s1, s2, s3, s4, s5, y_filter, offset_vec);
+ uint16x8_t d1 =
+ highbd_12_convolve6_8(s1, s2, s3, s4, s5, s6, y_filter, offset_vec);
+ uint16x8_t d2 =
+ highbd_12_convolve6_8(s2, s3, s4, s5, s6, s7, y_filter, offset_vec);
+ uint16x8_t d3 =
+ highbd_12_convolve6_8(s3, s4, s5, s6, s7, s8, y_filter, offset_vec);
+
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ height -= 4;
+ } while (height != 0);
+ src_ptr += 8;
+ dst_ptr += 8;
+ w -= 8;
+ } while (w != 0);
+ }
+}
+
+static INLINE void highbd_dist_wtd_convolve_y_6tap_neon(
+ const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
+ int w, int h, const int16_t *y_filter_ptr, const int offset) {
+ const int16x8_t y_filter = vld1q_s16(y_filter_ptr);
+ const int32x4_t offset_vec = vdupq_n_s32(offset);
+
+ if (w == 4) {
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ int16x4_t s0, s1, s2, s3, s4;
+ load_s16_4x5(s, src_stride, &s0, &s1, &s2, &s3, &s4);
+ s += 5 * src_stride;
+
+ do {
+ int16x4_t s5, s6, s7, s8;
+ load_s16_4x4(s, src_stride, &s5, &s6, &s7, &s8);
+
+ uint16x4_t d0 =
+ highbd_convolve6_4(s0, s1, s2, s3, s4, s5, y_filter, offset_vec);
+ uint16x4_t d1 =
+ highbd_convolve6_4(s1, s2, s3, s4, s5, s6, y_filter, offset_vec);
+ uint16x4_t d2 =
+ highbd_convolve6_4(s2, s3, s4, s5, s6, s7, y_filter, offset_vec);
+ uint16x4_t d3 =
+ highbd_convolve6_4(s3, s4, s5, s6, s7, s8, y_filter, offset_vec);
+
+ store_u16_4x4(d, dst_stride, d0, d1, d2, d3);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ h -= 4;
+ } while (h != 0);
+ } else {
+ do {
+ int height = h;
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ int16x8_t s0, s1, s2, s3, s4;
+ load_s16_8x5(s, src_stride, &s0, &s1, &s2, &s3, &s4);
+ s += 5 * src_stride;
+
+ do {
+ int16x8_t s5, s6, s7, s8;
+ load_s16_8x4(s, src_stride, &s5, &s6, &s7, &s8);
+
+ uint16x8_t d0 =
+ highbd_convolve6_8(s0, s1, s2, s3, s4, s5, y_filter, offset_vec);
+ uint16x8_t d1 =
+ highbd_convolve6_8(s1, s2, s3, s4, s5, s6, y_filter, offset_vec);
+ uint16x8_t d2 =
+ highbd_convolve6_8(s2, s3, s4, s5, s6, s7, y_filter, offset_vec);
+ uint16x8_t d3 =
+ highbd_convolve6_8(s3, s4, s5, s6, s7, s8, y_filter, offset_vec);
+
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ height -= 4;
+ } while (height != 0);
+ src_ptr += 8;
+ dst_ptr += 8;
+ w -= 8;
+ } while (w != 0);
+ }
+}
+
+static INLINE void highbd_12_dist_wtd_convolve_y_8tap_neon(
+ const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
+ int w, int h, const int16_t *y_filter_ptr, const int offset) {
+ const int16x8_t y_filter = vld1q_s16(y_filter_ptr);
+ const int32x4_t offset_vec = vdupq_n_s32(offset);
+
+ if (w == 4) {
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ int16x4_t s0, s1, s2, s3, s4, s5, s6;
+ load_s16_4x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
+ s += 7 * src_stride;
+
+ do {
+ int16x4_t s7, s8, s9, s10;
+ load_s16_4x4(s, src_stride, &s7, &s8, &s9, &s10);
+
+ uint16x4_t d0 = highbd_12_convolve8_4(s0, s1, s2, s3, s4, s5, s6, s7,
+ y_filter, offset_vec);
+ uint16x4_t d1 = highbd_12_convolve8_4(s1, s2, s3, s4, s5, s6, s7, s8,
+ y_filter, offset_vec);
+ uint16x4_t d2 = highbd_12_convolve8_4(s2, s3, s4, s5, s6, s7, s8, s9,
+ y_filter, offset_vec);
+ uint16x4_t d3 = highbd_12_convolve8_4(s3, s4, s5, s6, s7, s8, s9, s10,
+ y_filter, offset_vec);
+
+ store_u16_4x4(d, dst_stride, d0, d1, d2, d3);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s5 = s9;
+ s6 = s10;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ h -= 4;
+ } while (h != 0);
+ } else {
+ do {
+ int height = h;
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ int16x8_t s0, s1, s2, s3, s4, s5, s6;
+ load_s16_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
+ s += 7 * src_stride;
+
+ do {
+ int16x8_t s7, s8, s9, s10;
+ load_s16_8x4(s, src_stride, &s7, &s8, &s9, &s10);
+
+ uint16x8_t d0 = highbd_12_convolve8_8(s0, s1, s2, s3, s4, s5, s6, s7,
+ y_filter, offset_vec);
+ uint16x8_t d1 = highbd_12_convolve8_8(s1, s2, s3, s4, s5, s6, s7, s8,
+ y_filter, offset_vec);
+ uint16x8_t d2 = highbd_12_convolve8_8(s2, s3, s4, s5, s6, s7, s8, s9,
+ y_filter, offset_vec);
+ uint16x8_t d3 = highbd_12_convolve8_8(s3, s4, s5, s6, s7, s8, s9, s10,
+ y_filter, offset_vec);
+
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s5 = s9;
+ s6 = s10;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ height -= 4;
+ } while (height != 0);
+ src_ptr += 8;
+ dst_ptr += 8;
+ w -= 8;
+ } while (w != 0);
+ }
+}
+static INLINE void highbd_dist_wtd_convolve_y_8tap_neon(
+ const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
+ int w, int h, const int16_t *y_filter_ptr, const int offset) {
+ const int16x8_t y_filter = vld1q_s16(y_filter_ptr);
+ const int32x4_t offset_vec = vdupq_n_s32(offset);
+
+ if (w == 4) {
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ int16x4_t s0, s1, s2, s3, s4, s5, s6;
+ load_s16_4x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
+ s += 7 * src_stride;
+
+ do {
+ int16x4_t s7, s8, s9, s10;
+ load_s16_4x4(s, src_stride, &s7, &s8, &s9, &s10);
+
+ uint16x4_t d0 = highbd_convolve8_4(s0, s1, s2, s3, s4, s5, s6, s7,
+ y_filter, offset_vec);
+ uint16x4_t d1 = highbd_convolve8_4(s1, s2, s3, s4, s5, s6, s7, s8,
+ y_filter, offset_vec);
+ uint16x4_t d2 = highbd_convolve8_4(s2, s3, s4, s5, s6, s7, s8, s9,
+ y_filter, offset_vec);
+ uint16x4_t d3 = highbd_convolve8_4(s3, s4, s5, s6, s7, s8, s9, s10,
+ y_filter, offset_vec);
+
+ store_u16_4x4(d, dst_stride, d0, d1, d2, d3);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s5 = s9;
+ s6 = s10;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ h -= 4;
+ } while (h != 0);
+ } else {
+ do {
+ int height = h;
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ int16x8_t s0, s1, s2, s3, s4, s5, s6;
+ load_s16_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
+ s += 7 * src_stride;
+
+ do {
+ int16x8_t s7, s8, s9, s10;
+ load_s16_8x4(s, src_stride, &s7, &s8, &s9, &s10);
+
+ uint16x8_t d0 = highbd_convolve8_8(s0, s1, s2, s3, s4, s5, s6, s7,
+ y_filter, offset_vec);
+ uint16x8_t d1 = highbd_convolve8_8(s1, s2, s3, s4, s5, s6, s7, s8,
+ y_filter, offset_vec);
+ uint16x8_t d2 = highbd_convolve8_8(s2, s3, s4, s5, s6, s7, s8, s9,
+ y_filter, offset_vec);
+ uint16x8_t d3 = highbd_convolve8_8(s3, s4, s5, s6, s7, s8, s9, s10,
+ y_filter, offset_vec);
+
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s5 = s9;
+ s6 = s10;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ height -= 4;
+ } while (height != 0);
+ src_ptr += 8;
+ dst_ptr += 8;
+ w -= 8;
+ } while (w != 0);
+ }
+}
+
+void av1_highbd_dist_wtd_convolve_y_neon(
+ const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w,
+ int h, const InterpFilterParams *filter_params_y, const int subpel_y_qn,
+ ConvolveParams *conv_params, int bd) {
+ DECLARE_ALIGNED(16, uint16_t,
+ im_block[(MAX_SB_SIZE + MAX_FILTER_TAP) * MAX_SB_SIZE]);
+ CONV_BUF_TYPE *dst16 = conv_params->dst;
+ const int y_filter_taps = get_filter_tap(filter_params_y, subpel_y_qn);
+ int dst16_stride = conv_params->dst_stride;
+ const int im_stride = MAX_SB_SIZE;
+ const int vert_offset = filter_params_y->taps / 2 - 1;
+ assert(FILTER_BITS == COMPOUND_ROUND1_BITS);
+ const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0;
+ const int round_offset_avg = (1 << (offset_bits - conv_params->round_1)) +
+ (1 << (offset_bits - conv_params->round_1 - 1));
+ const int round_offset_conv = (1 << (conv_params->round_0 - 1)) +
+ (1 << (bd + FILTER_BITS)) +
+ (1 << (bd + FILTER_BITS - 1));
+
+ const int16_t *y_filter_ptr = av1_get_interp_filter_subpel_kernel(
+ filter_params_y, subpel_y_qn & SUBPEL_MASK);
+
+ src -= vert_offset * src_stride;
+
+ if (bd == 12) {
+ if (conv_params->do_average) {
+ if (y_filter_taps <= 6) {
+ highbd_12_dist_wtd_convolve_y_6tap_neon(
+ src + src_stride, src_stride, im_block, im_stride, w, h,
+ y_filter_ptr, round_offset_conv);
+ } else {
+ highbd_12_dist_wtd_convolve_y_8tap_neon(src, src_stride, im_block,
+ im_stride, w, h, y_filter_ptr,
+ round_offset_conv);
+ }
+ if (conv_params->use_dist_wtd_comp_avg) {
+ highbd_12_dist_wtd_comp_avg_neon(im_block, im_stride, dst, dst_stride,
+ w, h, conv_params, round_offset_avg,
+ bd);
+ } else {
+ highbd_12_comp_avg_neon(im_block, im_stride, dst, dst_stride, w, h,
+ conv_params, round_offset_avg, bd);
+ }
+ } else {
+ if (y_filter_taps <= 6) {
+ highbd_12_dist_wtd_convolve_y_6tap_neon(
+ src + src_stride, src_stride, dst16, dst16_stride, w, h,
+ y_filter_ptr, round_offset_conv);
+ } else {
+ highbd_12_dist_wtd_convolve_y_8tap_neon(
+ src, src_stride, dst16, dst16_stride, w, h, y_filter_ptr,
+ round_offset_conv);
+ }
+ }
+ } else {
+ if (conv_params->do_average) {
+ if (y_filter_taps <= 6) {
+ highbd_dist_wtd_convolve_y_6tap_neon(src + src_stride, src_stride,
+ im_block, im_stride, w, h,
+ y_filter_ptr, round_offset_conv);
+ } else {
+ highbd_dist_wtd_convolve_y_8tap_neon(src, src_stride, im_block,
+ im_stride, w, h, y_filter_ptr,
+ round_offset_conv);
+ }
+ if (conv_params->use_dist_wtd_comp_avg) {
+ highbd_dist_wtd_comp_avg_neon(im_block, im_stride, dst, dst_stride, w,
+ h, conv_params, round_offset_avg, bd);
+ } else {
+ highbd_comp_avg_neon(im_block, im_stride, dst, dst_stride, w, h,
+ conv_params, round_offset_avg, bd);
+ }
+ } else {
+ if (y_filter_taps <= 6) {
+ highbd_dist_wtd_convolve_y_6tap_neon(src + src_stride, src_stride,
+ dst16, dst16_stride, w, h,
+ y_filter_ptr, round_offset_conv);
+ } else {
+ highbd_dist_wtd_convolve_y_8tap_neon(src, src_stride, dst16,
+ dst16_stride, w, h, y_filter_ptr,
+ round_offset_conv);
+ }
+ }
+ }
+}
+
+static INLINE void highbd_2d_copy_neon(const uint16_t *src_ptr, int src_stride,
+ uint16_t *dst_ptr, int dst_stride, int w,
+ int h, const int round_bits,
+ const int offset) {
+ if (w <= 4) {
+ const int16x4_t round_shift_s16 = vdup_n_s16(round_bits);
+ const uint16x4_t offset_u16 = vdup_n_u16(offset);
+
+ for (int y = 0; y < h; ++y) {
+ const uint16x4_t s = vld1_u16(src_ptr + y * src_stride);
+ uint16x4_t d = vshl_u16(s, round_shift_s16);
+ d = vadd_u16(d, offset_u16);
+ if (w == 2) {
+ store_u16_2x1(dst_ptr + y * dst_stride, d, 0);
+ } else {
+ vst1_u16(dst_ptr + y * dst_stride, d);
+ }
+ }
+ } else {
+ const int16x8_t round_shift_s16 = vdupq_n_s16(round_bits);
+ const uint16x8_t offset_u16 = vdupq_n_u16(offset);
+
+ for (int y = 0; y < h; ++y) {
+ for (int x = 0; x < w; x += 8) {
+ const uint16x8_t s = vld1q_u16(src_ptr + y * src_stride + x);
+ uint16x8_t d = vshlq_u16(s, round_shift_s16);
+ d = vaddq_u16(d, offset_u16);
+ vst1q_u16(dst_ptr + y * dst_stride + x, d);
+ }
+ }
+ }
+}
+
+void av1_highbd_dist_wtd_convolve_2d_copy_neon(const uint16_t *src,
+ int src_stride, uint16_t *dst,
+ int dst_stride, int w, int h,
+ ConvolveParams *conv_params,
+ int bd) {
+ DECLARE_ALIGNED(16, uint16_t,
+ im_block[(MAX_SB_SIZE + MAX_FILTER_TAP) * MAX_SB_SIZE]);
+
+ const int im_stride = MAX_SB_SIZE;
+ CONV_BUF_TYPE *dst16 = conv_params->dst;
+ int dst16_stride = conv_params->dst_stride;
+ const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0;
+ const int round_offset = (1 << (offset_bits - conv_params->round_1)) +
+ (1 << (offset_bits - conv_params->round_1 - 1));
+ const int round_bits =
+ 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1;
+ assert(round_bits >= 0);
+
+ if (conv_params->do_average) {
+ highbd_2d_copy_neon(src, src_stride, im_block, im_stride, w, h, round_bits,
+ round_offset);
+ } else {
+ highbd_2d_copy_neon(src, src_stride, dst16, dst16_stride, w, h, round_bits,
+ round_offset);
+ }
+
+ if (conv_params->do_average) {
+ if (conv_params->use_dist_wtd_comp_avg) {
+ if (bd == 12) {
+ highbd_12_dist_wtd_comp_avg_neon(im_block, im_stride, dst, dst_stride,
+ w, h, conv_params, round_offset, bd);
+ } else {
+ highbd_dist_wtd_comp_avg_neon(im_block, im_stride, dst, dst_stride, w,
+ h, conv_params, round_offset, bd);
+ }
+ } else {
+ if (bd == 12) {
+ highbd_12_comp_avg_neon(im_block, im_stride, dst, dst_stride, w, h,
+ conv_params, round_offset, bd);
+ } else {
+ highbd_comp_avg_neon(im_block, im_stride, dst, dst_stride, w, h,
+ conv_params, round_offset, bd);
+ }
+ }
+ }
+}
+
+static INLINE uint16x4_t highbd_convolve6_4_2d_v(
+ const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
+ const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
+ const int16x8_t y_filter, const int32x4_t offset) {
+ // Values at indices 0 and 7 of y_filter are zero.
+ const int16x4_t y_filter_0_3 = vget_low_s16(y_filter);
+ const int16x4_t y_filter_4_7 = vget_high_s16(y_filter);
+
+ int32x4_t sum = vmlal_lane_s16(offset, s0, y_filter_0_3, 1);
+ sum = vmlal_lane_s16(sum, s1, y_filter_0_3, 2);
+ sum = vmlal_lane_s16(sum, s2, y_filter_0_3, 3);
+ sum = vmlal_lane_s16(sum, s3, y_filter_4_7, 0);
+ sum = vmlal_lane_s16(sum, s4, y_filter_4_7, 1);
+ sum = vmlal_lane_s16(sum, s5, y_filter_4_7, 2);
+
+ return vqrshrun_n_s32(sum, COMPOUND_ROUND1_BITS);
+}
+
+static INLINE uint16x8_t highbd_convolve6_8_2d_v(
+ const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
+ const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t y_filter, const int32x4_t offset) {
+ // Values at indices 0 and 7 of y_filter are zero.
+ const int16x4_t y_filter_0_3 = vget_low_s16(y_filter);
+ const int16x4_t y_filter_4_7 = vget_high_s16(y_filter);
+
+ int32x4_t sum0 = vmlal_lane_s16(offset, vget_low_s16(s0), y_filter_0_3, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s1), y_filter_0_3, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s2), y_filter_0_3, 3);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s3), y_filter_4_7, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s4), y_filter_4_7, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s5), y_filter_4_7, 2);
+
+ int32x4_t sum1 = vmlal_lane_s16(offset, vget_high_s16(s0), y_filter_0_3, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s1), y_filter_0_3, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s2), y_filter_0_3, 3);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s3), y_filter_4_7, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s4), y_filter_4_7, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s5), y_filter_4_7, 2);
+
+ return vcombine_u16(vqrshrun_n_s32(sum0, COMPOUND_ROUND1_BITS),
+ vqrshrun_n_s32(sum1, COMPOUND_ROUND1_BITS));
+}
+
+static INLINE void highbd_dist_wtd_convolve_2d_vert_6tap_neon(
+ const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
+ int w, int h, const int16_t *y_filter_ptr, int offset) {
+ const int16x8_t y_filter = vld1q_s16(y_filter_ptr);
+ const int32x4_t offset_vec = vdupq_n_s32(offset);
+
+ if (w == 4) {
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ int16x4_t s0, s1, s2, s3, s4;
+ load_s16_4x5(s, src_stride, &s0, &s1, &s2, &s3, &s4);
+ s += 5 * src_stride;
+
+ do {
+ int16x4_t s5, s6, s7, s8;
+ load_s16_4x4(s, src_stride, &s5, &s6, &s7, &s8);
+
+ uint16x4_t d0 =
+ highbd_convolve6_4_2d_v(s0, s1, s2, s3, s4, s5, y_filter, offset_vec);
+ uint16x4_t d1 =
+ highbd_convolve6_4_2d_v(s1, s2, s3, s4, s5, s6, y_filter, offset_vec);
+ uint16x4_t d2 =
+ highbd_convolve6_4_2d_v(s2, s3, s4, s5, s6, s7, y_filter, offset_vec);
+ uint16x4_t d3 =
+ highbd_convolve6_4_2d_v(s3, s4, s5, s6, s7, s8, y_filter, offset_vec);
+
+ store_u16_4x4(d, dst_stride, d0, d1, d2, d3);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ h -= 4;
+ } while (h != 0);
+ } else {
+ do {
+ int height = h;
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ int16x8_t s0, s1, s2, s3, s4;
+ load_s16_8x5(s, src_stride, &s0, &s1, &s2, &s3, &s4);
+ s += 5 * src_stride;
+
+ do {
+ int16x8_t s5, s6, s7, s8;
+ load_s16_8x4(s, src_stride, &s5, &s6, &s7, &s8);
+
+ uint16x8_t d0 = highbd_convolve6_8_2d_v(s0, s1, s2, s3, s4, s5,
+ y_filter, offset_vec);
+ uint16x8_t d1 = highbd_convolve6_8_2d_v(s1, s2, s3, s4, s5, s6,
+ y_filter, offset_vec);
+ uint16x8_t d2 = highbd_convolve6_8_2d_v(s2, s3, s4, s5, s6, s7,
+ y_filter, offset_vec);
+ uint16x8_t d3 = highbd_convolve6_8_2d_v(s3, s4, s5, s6, s7, s8,
+ y_filter, offset_vec);
+
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ height -= 4;
+ } while (height != 0);
+ src_ptr += 8;
+ dst_ptr += 8;
+ w -= 8;
+ } while (w != 0);
+ }
+}
+
+static INLINE uint16x4_t highbd_convolve8_4_2d_v(
+ const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
+ const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
+ const int16x4_t s6, const int16x4_t s7, const int16x8_t y_filter,
+ const int32x4_t offset) {
+ const int16x4_t y_filter_0_3 = vget_low_s16(y_filter);
+ const int16x4_t y_filter_4_7 = vget_high_s16(y_filter);
+
+ int32x4_t sum = vmlal_lane_s16(offset, s0, y_filter_0_3, 0);
+ sum = vmlal_lane_s16(sum, s1, y_filter_0_3, 1);
+ sum = vmlal_lane_s16(sum, s2, y_filter_0_3, 2);
+ sum = vmlal_lane_s16(sum, s3, y_filter_0_3, 3);
+ sum = vmlal_lane_s16(sum, s4, y_filter_4_7, 0);
+ sum = vmlal_lane_s16(sum, s5, y_filter_4_7, 1);
+ sum = vmlal_lane_s16(sum, s6, y_filter_4_7, 2);
+ sum = vmlal_lane_s16(sum, s7, y_filter_4_7, 3);
+
+ return vqrshrun_n_s32(sum, COMPOUND_ROUND1_BITS);
+}
+
+static INLINE uint16x8_t highbd_convolve8_8_2d_v(
+ const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
+ const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t s6, const int16x8_t s7, const int16x8_t y_filter,
+ const int32x4_t offset) {
+ const int16x4_t y_filter_0_3 = vget_low_s16(y_filter);
+ const int16x4_t y_filter_4_7 = vget_high_s16(y_filter);
+
+ int32x4_t sum0 = vmlal_lane_s16(offset, vget_low_s16(s0), y_filter_0_3, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s1), y_filter_0_3, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s2), y_filter_0_3, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s3), y_filter_0_3, 3);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s4), y_filter_4_7, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s5), y_filter_4_7, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s6), y_filter_4_7, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s7), y_filter_4_7, 3);
+
+ int32x4_t sum1 = vmlal_lane_s16(offset, vget_high_s16(s0), y_filter_0_3, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s1), y_filter_0_3, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s2), y_filter_0_3, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s3), y_filter_0_3, 3);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s4), y_filter_4_7, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s5), y_filter_4_7, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s6), y_filter_4_7, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s7), y_filter_4_7, 3);
+
+ return vcombine_u16(vqrshrun_n_s32(sum0, COMPOUND_ROUND1_BITS),
+ vqrshrun_n_s32(sum1, COMPOUND_ROUND1_BITS));
+}
+
+static INLINE void highbd_dist_wtd_convolve_2d_vert_8tap_neon(
+ const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
+ int w, int h, const int16_t *y_filter_ptr, int offset) {
+ const int16x8_t y_filter = vld1q_s16(y_filter_ptr);
+ const int32x4_t offset_vec = vdupq_n_s32(offset);
+
+ if (w <= 4) {
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ int16x4_t s0, s1, s2, s3, s4, s5, s6;
+ load_s16_4x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
+ s += 7 * src_stride;
+
+ do {
+ int16x4_t s7, s8, s9, s10;
+ load_s16_4x4(s, src_stride, &s7, &s8, &s9, &s10);
+
+ uint16x4_t d0 = highbd_convolve8_4_2d_v(s0, s1, s2, s3, s4, s5, s6, s7,
+ y_filter, offset_vec);
+ uint16x4_t d1 = highbd_convolve8_4_2d_v(s1, s2, s3, s4, s5, s6, s7, s8,
+ y_filter, offset_vec);
+ uint16x4_t d2 = highbd_convolve8_4_2d_v(s2, s3, s4, s5, s6, s7, s8, s9,
+ y_filter, offset_vec);
+ uint16x4_t d3 = highbd_convolve8_4_2d_v(s3, s4, s5, s6, s7, s8, s9, s10,
+ y_filter, offset_vec);
+
+ store_u16_4x4(d, dst_stride, d0, d1, d2, d3);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s5 = s9;
+ s6 = s10;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ h -= 4;
+ } while (h != 0);
+ } else {
+ do {
+ int height = h;
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ int16x8_t s0, s1, s2, s3, s4, s5, s6;
+ load_s16_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
+ s += 7 * src_stride;
+
+ do {
+ int16x8_t s7, s8, s9, s10;
+ load_s16_8x4(s, src_stride, &s7, &s8, &s9, &s10);
+
+ uint16x8_t d0 = highbd_convolve8_8_2d_v(s0, s1, s2, s3, s4, s5, s6, s7,
+ y_filter, offset_vec);
+ uint16x8_t d1 = highbd_convolve8_8_2d_v(s1, s2, s3, s4, s5, s6, s7, s8,
+ y_filter, offset_vec);
+ uint16x8_t d2 = highbd_convolve8_8_2d_v(s2, s3, s4, s5, s6, s7, s8, s9,
+ y_filter, offset_vec);
+ uint16x8_t d3 = highbd_convolve8_8_2d_v(s3, s4, s5, s6, s7, s8, s9, s10,
+ y_filter, offset_vec);
+
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s5 = s9;
+ s6 = s10;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ height -= 4;
+ } while (height != 0);
+ src_ptr += 8;
+ dst_ptr += 8;
+ w -= 8;
+ } while (w != 0);
+ }
+}
+
+static INLINE void highbd_12_dist_wtd_convolve_2d_horiz_6tap_neon(
+ const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
+ int w, int h, const int16_t *x_filter_ptr, const int offset) {
+ // The smallest block height is 4, and the horizontal convolution needs to
+ // process an extra (filter_taps/2 - 1) lines for the vertical convolution.
+ assert(h >= 5);
+ const int32x4_t offset_vec = vdupq_n_s32(offset);
+
+ const int16x8_t x_filter = vld1q_s16(x_filter_ptr);
+
+ int height = h;
+
+ do {
+ int width = w;
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ do {
+ int16x8_t s0[6], s1[6], s2[6], s3[6];
+ load_s16_8x6(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3],
+ &s0[4], &s0[5]);
+ load_s16_8x6(s + 1 * src_stride, 1, &s1[0], &s1[1], &s1[2], &s1[3],
+ &s1[4], &s1[5]);
+ load_s16_8x6(s + 2 * src_stride, 1, &s2[0], &s2[1], &s2[2], &s2[3],
+ &s2[4], &s2[5]);
+ load_s16_8x6(s + 3 * src_stride, 1, &s3[0], &s3[1], &s3[2], &s3[3],
+ &s3[4], &s3[5]);
+
+ uint16x8_t d0 = highbd_12_convolve6_8(s0[0], s0[1], s0[2], s0[3], s0[4],
+ s0[5], x_filter, offset_vec);
+ uint16x8_t d1 = highbd_12_convolve6_8(s1[0], s1[1], s1[2], s1[3], s1[4],
+ s1[5], x_filter, offset_vec);
+ uint16x8_t d2 = highbd_12_convolve6_8(s2[0], s2[1], s2[2], s2[3], s2[4],
+ s2[5], x_filter, offset_vec);
+ uint16x8_t d3 = highbd_12_convolve6_8(s3[0], s3[1], s3[2], s3[3], s3[4],
+ s3[5], x_filter, offset_vec);
+
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ height -= 4;
+ } while (height > 4);
+
+ do {
+ int width = w;
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ do {
+ int16x8_t s0[6];
+ load_s16_8x6(s, 1, &s0[0], &s0[1], &s0[2], &s0[3], &s0[4], &s0[5]);
+
+ uint16x8_t d0 = highbd_12_convolve6_8(s0[0], s0[1], s0[2], s0[3], s0[4],
+ s0[5], x_filter, offset_vec);
+ vst1q_u16(d, d0);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ } while (--height != 0);
+}
+
+static INLINE void highbd_dist_wtd_convolve_2d_horiz_6tap_neon(
+ const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
+ int w, int h, const int16_t *x_filter_ptr, const int offset) {
+ // The smallest block height is 4, and the horizontal convolution needs to
+ // process an extra (filter_taps/2 - 1) lines for the vertical convolution.
+ assert(h >= 5);
+ const int32x4_t offset_vec = vdupq_n_s32(offset);
+
+ const int16x8_t x_filter = vld1q_s16(x_filter_ptr);
+
+ int height = h;
+
+ do {
+ int width = w;
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ do {
+ int16x8_t s0[6], s1[6], s2[6], s3[6];
+ load_s16_8x6(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3],
+ &s0[4], &s0[5]);
+ load_s16_8x6(s + 1 * src_stride, 1, &s1[0], &s1[1], &s1[2], &s1[3],
+ &s1[4], &s1[5]);
+ load_s16_8x6(s + 2 * src_stride, 1, &s2[0], &s2[1], &s2[2], &s2[3],
+ &s2[4], &s2[5]);
+ load_s16_8x6(s + 3 * src_stride, 1, &s3[0], &s3[1], &s3[2], &s3[3],
+ &s3[4], &s3[5]);
+
+ uint16x8_t d0 = highbd_convolve6_8(s0[0], s0[1], s0[2], s0[3], s0[4],
+ s0[5], x_filter, offset_vec);
+ uint16x8_t d1 = highbd_convolve6_8(s1[0], s1[1], s1[2], s1[3], s1[4],
+ s1[5], x_filter, offset_vec);
+ uint16x8_t d2 = highbd_convolve6_8(s2[0], s2[1], s2[2], s2[3], s2[4],
+ s2[5], x_filter, offset_vec);
+ uint16x8_t d3 = highbd_convolve6_8(s3[0], s3[1], s3[2], s3[3], s3[4],
+ s3[5], x_filter, offset_vec);
+
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ height -= 4;
+ } while (height > 4);
+
+ do {
+ int width = w;
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ do {
+ int16x8_t s0[6];
+ load_s16_8x6(s, 1, &s0[0], &s0[1], &s0[2], &s0[3], &s0[4], &s0[5]);
+
+ uint16x8_t d0 = highbd_convolve6_8(s0[0], s0[1], s0[2], s0[3], s0[4],
+ s0[5], x_filter, offset_vec);
+ vst1q_u16(d, d0);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ } while (--height != 0);
+}
+
+static INLINE void highbd_12_dist_wtd_convolve_2d_horiz_neon(
+ const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
+ int w, int h, const int16_t *x_filter_ptr, const int offset) {
+ // The smallest block height is 4, and the horizontal convolution needs to
+ // process an extra (filter_taps/2 - 1) lines for the vertical convolution.
+ assert(h >= 5);
+ const int32x4_t offset_vec = vdupq_n_s32(offset);
+
+ if (w == 4) {
+ // 4-tap filters are used for blocks having width == 4.
+ const int16x4_t x_filter = vld1_s16(x_filter_ptr + 2);
+ const int16_t *s = (const int16_t *)(src_ptr + 1);
+ uint16_t *d = dst_ptr;
+
+ do {
+ int16x4_t s0[4], s1[4], s2[4], s3[4];
+ load_s16_4x4(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3]);
+ load_s16_4x4(s + 1 * src_stride, 1, &s1[0], &s1[1], &s1[2], &s1[3]);
+ load_s16_4x4(s + 2 * src_stride, 1, &s2[0], &s2[1], &s2[2], &s2[3]);
+ load_s16_4x4(s + 3 * src_stride, 1, &s3[0], &s3[1], &s3[2], &s3[3]);
+
+ uint16x4_t d0 = highbd_12_convolve4_4_x(s0, x_filter, offset_vec);
+ uint16x4_t d1 = highbd_12_convolve4_4_x(s1, x_filter, offset_vec);
+ uint16x4_t d2 = highbd_12_convolve4_4_x(s2, x_filter, offset_vec);
+ uint16x4_t d3 = highbd_12_convolve4_4_x(s3, x_filter, offset_vec);
+
+ store_u16_4x4(d, dst_stride, d0, d1, d2, d3);
+
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ h -= 4;
+ } while (h > 4);
+
+ do {
+ int16x4_t s0[4];
+ load_s16_4x4(s, 1, &s0[0], &s0[1], &s0[2], &s0[3]);
+
+ uint16x4_t d0 = highbd_12_convolve4_4_x(s0, x_filter, offset_vec);
+ vst1_u16(d, d0);
+
+ s += src_stride;
+ d += dst_stride;
+ } while (--h != 0);
+ } else {
+ const int16x8_t x_filter = vld1q_s16(x_filter_ptr);
+ int height = h;
+
+ do {
+ int width = w;
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ do {
+ int16x8_t s0[8], s1[8], s2[8], s3[8];
+ load_s16_8x8(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3],
+ &s0[4], &s0[5], &s0[6], &s0[7]);
+ load_s16_8x8(s + 1 * src_stride, 1, &s1[0], &s1[1], &s1[2], &s1[3],
+ &s1[4], &s1[5], &s1[6], &s1[7]);
+ load_s16_8x8(s + 2 * src_stride, 1, &s2[0], &s2[1], &s2[2], &s2[3],
+ &s2[4], &s2[5], &s2[6], &s2[7]);
+ load_s16_8x8(s + 3 * src_stride, 1, &s3[0], &s3[1], &s3[2], &s3[3],
+ &s3[4], &s3[5], &s3[6], &s3[7]);
+
+ uint16x8_t d0 =
+ highbd_12_convolve8_8(s0[0], s0[1], s0[2], s0[3], s0[4], s0[5],
+ s0[6], s0[7], x_filter, offset_vec);
+ uint16x8_t d1 =
+ highbd_12_convolve8_8(s1[0], s1[1], s1[2], s1[3], s1[4], s1[5],
+ s1[6], s1[7], x_filter, offset_vec);
+ uint16x8_t d2 =
+ highbd_12_convolve8_8(s2[0], s2[1], s2[2], s2[3], s2[4], s2[5],
+ s2[6], s2[7], x_filter, offset_vec);
+ uint16x8_t d3 =
+ highbd_12_convolve8_8(s3[0], s3[1], s3[2], s3[3], s3[4], s3[5],
+ s3[6], s3[7], x_filter, offset_vec);
+
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ height -= 4;
+ } while (height > 4);
+
+ do {
+ int width = w;
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ do {
+ int16x8_t s0[8];
+ load_s16_8x8(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3],
+ &s0[4], &s0[5], &s0[6], &s0[7]);
+
+ uint16x8_t d0 =
+ highbd_12_convolve8_8(s0[0], s0[1], s0[2], s0[3], s0[4], s0[5],
+ s0[6], s0[7], x_filter, offset_vec);
+ vst1q_u16(d, d0);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ } while (--height != 0);
+ }
+}
+
+static INLINE void highbd_dist_wtd_convolve_2d_horiz_neon(
+ const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
+ int w, int h, const int16_t *x_filter_ptr, const int offset) {
+ // The smallest block height is 4, and the horizontal convolution needs to
+ // process an extra (filter_taps/2 - 1) lines for the vertical convolution.
+ assert(h >= 5);
+ const int32x4_t offset_vec = vdupq_n_s32(offset);
+
+ if (w == 4) {
+ // 4-tap filters are used for blocks having width == 4.
+ const int16x4_t x_filter = vld1_s16(x_filter_ptr + 2);
+ const int16_t *s = (const int16_t *)(src_ptr + 1);
+ uint16_t *d = dst_ptr;
+
+ do {
+ int16x4_t s0[4], s1[4], s2[4], s3[4];
+ load_s16_4x4(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3]);
+ load_s16_4x4(s + 1 * src_stride, 1, &s1[0], &s1[1], &s1[2], &s1[3]);
+ load_s16_4x4(s + 2 * src_stride, 1, &s2[0], &s2[1], &s2[2], &s2[3]);
+ load_s16_4x4(s + 3 * src_stride, 1, &s3[0], &s3[1], &s3[2], &s3[3]);
+
+ uint16x4_t d0 = highbd_convolve4_4_x(s0, x_filter, offset_vec);
+ uint16x4_t d1 = highbd_convolve4_4_x(s1, x_filter, offset_vec);
+ uint16x4_t d2 = highbd_convolve4_4_x(s2, x_filter, offset_vec);
+ uint16x4_t d3 = highbd_convolve4_4_x(s3, x_filter, offset_vec);
+
+ store_u16_4x4(d, dst_stride, d0, d1, d2, d3);
+
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ h -= 4;
+ } while (h > 4);
+
+ do {
+ int16x4_t s0[4];
+ load_s16_4x4(s, 1, &s0[0], &s0[1], &s0[2], &s0[3]);
+
+ uint16x4_t d0 = highbd_convolve4_4_x(s0, x_filter, offset_vec);
+ vst1_u16(d, d0);
+
+ s += src_stride;
+ d += dst_stride;
+ } while (--h != 0);
+ } else {
+ const int16x8_t x_filter = vld1q_s16(x_filter_ptr);
+ int height = h;
+
+ do {
+ int width = w;
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ do {
+ int16x8_t s0[8], s1[8], s2[8], s3[8];
+ load_s16_8x8(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3],
+ &s0[4], &s0[5], &s0[6], &s0[7]);
+ load_s16_8x8(s + 1 * src_stride, 1, &s1[0], &s1[1], &s1[2], &s1[3],
+ &s1[4], &s1[5], &s1[6], &s1[7]);
+ load_s16_8x8(s + 2 * src_stride, 1, &s2[0], &s2[1], &s2[2], &s2[3],
+ &s2[4], &s2[5], &s2[6], &s2[7]);
+ load_s16_8x8(s + 3 * src_stride, 1, &s3[0], &s3[1], &s3[2], &s3[3],
+ &s3[4], &s3[5], &s3[6], &s3[7]);
+
+ uint16x8_t d0 =
+ highbd_convolve8_8(s0[0], s0[1], s0[2], s0[3], s0[4], s0[5], s0[6],
+ s0[7], x_filter, offset_vec);
+ uint16x8_t d1 =
+ highbd_convolve8_8(s1[0], s1[1], s1[2], s1[3], s1[4], s1[5], s1[6],
+ s1[7], x_filter, offset_vec);
+ uint16x8_t d2 =
+ highbd_convolve8_8(s2[0], s2[1], s2[2], s2[3], s2[4], s2[5], s2[6],
+ s2[7], x_filter, offset_vec);
+ uint16x8_t d3 =
+ highbd_convolve8_8(s3[0], s3[1], s3[2], s3[3], s3[4], s3[5], s3[6],
+ s3[7], x_filter, offset_vec);
+
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ height -= 4;
+ } while (height > 4);
+
+ do {
+ int width = w;
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ do {
+ int16x8_t s0[8];
+ load_s16_8x8(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3],
+ &s0[4], &s0[5], &s0[6], &s0[7]);
+
+ uint16x8_t d0 =
+ highbd_convolve8_8(s0[0], s0[1], s0[2], s0[3], s0[4], s0[5], s0[6],
+ s0[7], x_filter, offset_vec);
+ vst1q_u16(d, d0);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ } while (--height != 0);
+ }
+}
+
+void av1_highbd_dist_wtd_convolve_2d_neon(
+ const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w,
+ int h, const InterpFilterParams *filter_params_x,
+ const InterpFilterParams *filter_params_y, const int subpel_x_qn,
+ const int subpel_y_qn, ConvolveParams *conv_params, int bd) {
+ DECLARE_ALIGNED(16, uint16_t,
+ im_block[(MAX_SB_SIZE + MAX_FILTER_TAP) * MAX_SB_SIZE]);
+ DECLARE_ALIGNED(16, uint16_t,
+ im_block2[(MAX_SB_SIZE + MAX_FILTER_TAP) * MAX_SB_SIZE]);
+
+ CONV_BUF_TYPE *dst16 = conv_params->dst;
+ int dst16_stride = conv_params->dst_stride;
+ const int x_filter_taps = get_filter_tap(filter_params_x, subpel_x_qn);
+ const int clamped_x_taps = x_filter_taps < 6 ? 6 : x_filter_taps;
+ const int y_filter_taps = get_filter_tap(filter_params_y, subpel_y_qn);
+ const int clamped_y_taps = y_filter_taps < 6 ? 6 : y_filter_taps;
+
+ const int im_h = h + clamped_y_taps - 1;
+ const int im_stride = MAX_SB_SIZE;
+ const int vert_offset = clamped_y_taps / 2 - 1;
+ const int horiz_offset = clamped_x_taps / 2 - 1;
+ // The extra shim of (1 << (conv_params->round_0 - 1)) allows us to use a
+ // faster non-rounding non-saturating left shift.
+ const int round_offset_conv_x =
+ (1 << (bd + FILTER_BITS - 1)) + (1 << (conv_params->round_0 - 1));
+ const int y_offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0;
+ const int round_offset_conv_y = (1 << y_offset_bits);
+ const int round_offset_avg =
+ ((1 << (y_offset_bits - conv_params->round_1)) +
+ (1 << (y_offset_bits - conv_params->round_1 - 1)));
+
+ const uint16_t *src_ptr = src - vert_offset * src_stride - horiz_offset;
+
+ const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
+ filter_params_x, subpel_x_qn & SUBPEL_MASK);
+ const int16_t *y_filter_ptr = av1_get_interp_filter_subpel_kernel(
+ filter_params_y, subpel_y_qn & SUBPEL_MASK);
+
+ // horizontal filter
+ if (bd == 12) {
+ if (x_filter_taps <= 6 && w != 4) {
+ highbd_12_dist_wtd_convolve_2d_horiz_6tap_neon(
+ src_ptr, src_stride, im_block, im_stride, w, im_h, x_filter_ptr,
+ round_offset_conv_x);
+ } else {
+ highbd_12_dist_wtd_convolve_2d_horiz_neon(
+ src_ptr, src_stride, im_block, im_stride, w, im_h, x_filter_ptr,
+ round_offset_conv_x);
+ }
+ } else {
+ if (x_filter_taps <= 6 && w != 4) {
+ highbd_dist_wtd_convolve_2d_horiz_6tap_neon(
+ src_ptr, src_stride, im_block, im_stride, w, im_h, x_filter_ptr,
+ round_offset_conv_x);
+ } else {
+ highbd_dist_wtd_convolve_2d_horiz_neon(src_ptr, src_stride, im_block,
+ im_stride, w, im_h, x_filter_ptr,
+ round_offset_conv_x);
+ }
+ }
+
+ // vertical filter
+ if (y_filter_taps <= 6) {
+ if (conv_params->do_average) {
+ highbd_dist_wtd_convolve_2d_vert_6tap_neon(im_block, im_stride, im_block2,
+ im_stride, w, h, y_filter_ptr,
+ round_offset_conv_y);
+ } else {
+ highbd_dist_wtd_convolve_2d_vert_6tap_neon(
+ im_block, im_stride, dst16, dst16_stride, w, h, y_filter_ptr,
+ round_offset_conv_y);
+ }
+ } else {
+ if (conv_params->do_average) {
+ highbd_dist_wtd_convolve_2d_vert_8tap_neon(im_block, im_stride, im_block2,
+ im_stride, w, h, y_filter_ptr,
+ round_offset_conv_y);
+ } else {
+ highbd_dist_wtd_convolve_2d_vert_8tap_neon(
+ im_block, im_stride, dst16, dst16_stride, w, h, y_filter_ptr,
+ round_offset_conv_y);
+ }
+ }
+
+ // Do the compound averaging outside the loop, avoids branching within the
+ // main loop
+ if (conv_params->do_average) {
+ if (conv_params->use_dist_wtd_comp_avg) {
+ if (bd == 12) {
+ highbd_12_dist_wtd_comp_avg_neon(im_block2, im_stride, dst, dst_stride,
+ w, h, conv_params, round_offset_avg,
+ bd);
+ } else {
+ highbd_dist_wtd_comp_avg_neon(im_block2, im_stride, dst, dst_stride, w,
+ h, conv_params, round_offset_avg, bd);
+ }
+ } else {
+ if (bd == 12) {
+ highbd_12_comp_avg_neon(im_block2, im_stride, dst, dst_stride, w, h,
+ conv_params, round_offset_avg, bd);
+ } else {
+ highbd_comp_avg_neon(im_block2, im_stride, dst, dst_stride, w, h,
+ conv_params, round_offset_avg, bd);
+ }
+ }
+ }
+}
diff --git a/av1/common/arm/highbd_convolve_horiz_rs_neon.c b/av1/common/arm/highbd_convolve_horiz_rs_neon.c
new file mode 100644
index 0000000..51da025
--- /dev/null
+++ b/av1/common/arm/highbd_convolve_horiz_rs_neon.c
@@ -0,0 +1,273 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+
+#include "config/aom_config.h"
+#include "config/av1_rtcd.h"
+
+#include "aom_dsp/aom_dsp_common.h"
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_dsp/arm/transpose_neon.h"
+#include "aom_ports/mem.h"
+#include "av1/common/convolve.h"
+#include "av1/common/filter.h"
+#include "av1/common/arm/highbd_convolve_neon.h"
+
+#define UPSCALE_NORMATIVE_TAPS 8
+
+void av1_highbd_convolve_horiz_rs_neon(const uint16_t *src, int src_stride,
+ uint16_t *dst, int dst_stride, int w,
+ int h, const int16_t *x_filters,
+ int x0_qn, int x_step_qn, int bd) {
+ const int horiz_offset = UPSCALE_NORMATIVE_TAPS / 2 - 1;
+
+ static const int32_t kIdx[4] = { 0, 1, 2, 3 };
+ const int32x4_t idx = vld1q_s32(kIdx);
+ const int32x4_t subpel_mask = vdupq_n_s32(RS_SCALE_SUBPEL_MASK);
+ const int32x4_t shift_s32 = vdupq_n_s32(-FILTER_BITS);
+ const int32x4_t offset_s32 = vdupq_n_s32(0);
+ const uint16x4_t max = vdup_n_u16((1 << bd) - 1);
+
+ const uint16_t *src_ptr = src - horiz_offset;
+ uint16_t *dst_ptr = dst;
+
+ if (w <= 4) {
+ int height = h;
+ uint16_t *d = dst_ptr;
+
+ do {
+ int x_qn = x0_qn;
+
+ // Load 4 src vectors at a time, they might be the same, but we have to
+ // calculate the indices anyway. Doing it in SIMD and then storing the
+ // indices is faster than having to calculate the expression
+ // &src_ptr[((x_qn + 0*x_step_qn) >> RS_SCALE_SUBPEL_BITS)] 4 times
+ // Ideally this should be a gather using the indices, but NEON does not
+ // have that, so have to emulate
+ const int32x4_t xqn_idx = vmlaq_n_s32(vdupq_n_s32(x_qn), idx, x_step_qn);
+ // We have to multiply x2 to get the actual pointer as sizeof(uint16_t) =
+ // 2
+ const int32x4_t src_idx =
+ vshlq_n_s32(vshrq_n_s32(xqn_idx, RS_SCALE_SUBPEL_BITS), 1);
+ // Similarly for the filter vector indices, we calculate the filter
+ // indices for 4 columns. First we calculate the indices:
+ // x_qn & RS_SCALE_SUBPEL_MASK) >> RS_SCALE_EXTRA_BITS
+ // Then we calculate the actual pointers, multiplying with
+ // UPSCALE_UPSCALE_NORMATIVE_TAPS
+ // again shift left by 1
+ const int32x4_t x_filter4_idx = vshlq_n_s32(
+ vshrq_n_s32(vandq_s32(xqn_idx, subpel_mask), RS_SCALE_EXTRA_BITS), 1);
+ // Even though pointers are unsigned 32/64-bit ints we do signed
+ // addition The reason for this is that x_qn can be negative, leading to
+ // negative offsets. Argon test
+ // profile0_core/streams/test10573_11003.obu was failing because of
+ // this.
+#if AOM_ARCH_AARCH64
+ uint64x2_t tmp4[2];
+ tmp4[0] = vreinterpretq_u64_s64(vaddw_s32(
+ vdupq_n_s64((const int64_t)src_ptr), vget_low_s32(src_idx)));
+ tmp4[1] = vreinterpretq_u64_s64(vaddw_s32(
+ vdupq_n_s64((const int64_t)src_ptr), vget_high_s32(src_idx)));
+ int16_t *src4_ptr[4];
+ uint64_t *tmp_ptr = (uint64_t *)&src4_ptr;
+ vst1q_u64(tmp_ptr, tmp4[0]);
+ vst1q_u64(tmp_ptr + 2, tmp4[1]);
+
+ // filter vectors
+ tmp4[0] = vreinterpretq_u64_s64(vmlal_s32(
+ vdupq_n_s64((const int64_t)x_filters), vget_low_s32(x_filter4_idx),
+ vdup_n_s32(UPSCALE_NORMATIVE_TAPS)));
+ tmp4[1] = vreinterpretq_u64_s64(vmlal_s32(
+ vdupq_n_s64((const int64_t)x_filters), vget_high_s32(x_filter4_idx),
+ vdup_n_s32(UPSCALE_NORMATIVE_TAPS)));
+
+ const int16_t *x_filter4_ptr[4];
+ tmp_ptr = (uint64_t *)&x_filter4_ptr;
+ vst1q_u64(tmp_ptr, tmp4[0]);
+ vst1q_u64(tmp_ptr + 2, tmp4[1]);
+#else
+ uint32x4_t tmp4;
+ tmp4 = vreinterpretq_u32_s32(
+ vaddq_s32(vdupq_n_s32((const int32_t)src_ptr), src_idx));
+ int16_t *src4_ptr[4];
+ uint32_t *tmp_ptr = (uint32_t *)&src4_ptr;
+ vst1q_u32(tmp_ptr, tmp4);
+
+ // filter vectors
+ tmp4 = vreinterpretq_u32_s32(
+ vmlaq_s32(vdupq_n_s32((const int32_t)x_filters), x_filter4_idx,
+ vdupq_n_s32(UPSCALE_NORMATIVE_TAPS)));
+
+ const int16_t *x_filter4_ptr[4];
+ tmp_ptr = (uint32_t *)&x_filter4_ptr;
+ vst1q_u32(tmp_ptr, tmp4);
+#endif // AOM_ARCH_AARCH64
+ // Load source
+ int16x8_t s0 = vld1q_s16(src4_ptr[0]);
+ int16x8_t s1 = vld1q_s16(src4_ptr[1]);
+ int16x8_t s2 = vld1q_s16(src4_ptr[2]);
+ int16x8_t s3 = vld1q_s16(src4_ptr[3]);
+
+ // Actually load the filters
+ const int16x8_t x_filter0 = vld1q_s16(x_filter4_ptr[0]);
+ const int16x8_t x_filter1 = vld1q_s16(x_filter4_ptr[1]);
+ const int16x8_t x_filter2 = vld1q_s16(x_filter4_ptr[2]);
+ const int16x8_t x_filter3 = vld1q_s16(x_filter4_ptr[3]);
+
+ // Group low and high parts and transpose
+ int16x4_t filters_lo[] = { vget_low_s16(x_filter0),
+ vget_low_s16(x_filter1),
+ vget_low_s16(x_filter2),
+ vget_low_s16(x_filter3) };
+ int16x4_t filters_hi[] = { vget_high_s16(x_filter0),
+ vget_high_s16(x_filter1),
+ vget_high_s16(x_filter2),
+ vget_high_s16(x_filter3) };
+ transpose_array_inplace_u16_4x4((uint16x4_t *)filters_lo);
+ transpose_array_inplace_u16_4x4((uint16x4_t *)filters_hi);
+
+ // Run the 2D Scale convolution
+ uint16x4_t d0 = highbd_convolve8_2d_scale_horiz4x8_s32_s16(
+ s0, s1, s2, s3, filters_lo, filters_hi, shift_s32, offset_s32);
+
+ d0 = vmin_u16(d0, max);
+
+ if (w == 2) {
+ store_u16_2x1(d + 0 * dst_stride, d0, 0);
+ } else {
+ vst1_u16(d + 0 * dst_stride, d0);
+ }
+
+ src_ptr += src_stride;
+ d += dst_stride;
+ height--;
+ } while (height > 0);
+ } else {
+ int height = h;
+
+ do {
+ int width = w;
+ int x_qn = x0_qn;
+ uint16_t *d = dst_ptr;
+ const uint16_t *s = src_ptr;
+
+ do {
+ // Load 4 src vectors at a time, they might be the same, but we have to
+ // calculate the indices anyway. Doing it in SIMD and then storing the
+ // indices is faster than having to calculate the expression
+ // &src_ptr[((x_qn + 0*x_step_qn) >> RS_SCALE_SUBPEL_BITS)] 4 times
+ // Ideally this should be a gather using the indices, but NEON does not
+ // have that, so have to emulate
+ const int32x4_t xqn_idx =
+ vmlaq_n_s32(vdupq_n_s32(x_qn), idx, x_step_qn);
+ // We have to multiply x2 to get the actual pointer as sizeof(uint16_t)
+ // = 2
+ const int32x4_t src_idx =
+ vshlq_n_s32(vshrq_n_s32(xqn_idx, RS_SCALE_SUBPEL_BITS), 1);
+
+ // Similarly for the filter vector indices, we calculate the filter
+ // indices for 4 columns. First we calculate the indices:
+ // x_qn & RS_SCALE_SUBPEL_MASK) >> RS_SCALE_EXTRA_BITS
+ // Then we calculate the actual pointers, multiplying with
+ // UPSCALE_UPSCALE_NORMATIVE_TAPS
+ // again shift left by 1
+ const int32x4_t x_filter4_idx = vshlq_n_s32(
+ vshrq_n_s32(vandq_s32(xqn_idx, subpel_mask), RS_SCALE_EXTRA_BITS),
+ 1);
+ // Even though pointers are unsigned 32/64-bit ints we do signed
+ // addition The reason for this is that x_qn can be negative, leading to
+ // negative offsets. Argon test
+ // profile0_core/streams/test10573_11003.obu was failing because of
+ // this.
+#if AOM_ARCH_AARCH64
+ uint64x2_t tmp4[2];
+ tmp4[0] = vreinterpretq_u64_s64(
+ vaddw_s32(vdupq_n_s64((const int64_t)s), vget_low_s32(src_idx)));
+ tmp4[1] = vreinterpretq_u64_s64(
+ vaddw_s32(vdupq_n_s64((const int64_t)s), vget_high_s32(src_idx)));
+ int16_t *src4_ptr[4];
+ uint64_t *tmp_ptr = (uint64_t *)&src4_ptr;
+ vst1q_u64(tmp_ptr, tmp4[0]);
+ vst1q_u64(tmp_ptr + 2, tmp4[1]);
+
+ // filter vectors
+ tmp4[0] = vreinterpretq_u64_s64(vmlal_s32(
+ vdupq_n_s64((const int64_t)x_filters), vget_low_s32(x_filter4_idx),
+ vdup_n_s32(UPSCALE_NORMATIVE_TAPS)));
+ tmp4[1] = vreinterpretq_u64_s64(vmlal_s32(
+ vdupq_n_s64((const int64_t)x_filters), vget_high_s32(x_filter4_idx),
+ vdup_n_s32(UPSCALE_NORMATIVE_TAPS)));
+
+ const int16_t *x_filter4_ptr[4];
+ tmp_ptr = (uint64_t *)&x_filter4_ptr;
+ vst1q_u64(tmp_ptr, tmp4[0]);
+ vst1q_u64(tmp_ptr + 2, tmp4[1]);
+#else
+ uint32x4_t tmp4;
+ tmp4 = vreinterpretq_u32_s32(
+ vaddq_s32(vdupq_n_s32((const int32_t)s), src_idx));
+ int16_t *src4_ptr[4];
+ uint32_t *tmp_ptr = (uint32_t *)&src4_ptr;
+ vst1q_u32(tmp_ptr, tmp4);
+
+ // filter vectors
+ tmp4 = vreinterpretq_u32_s32(
+ vmlaq_s32(vdupq_n_s32((const int32_t)x_filters), x_filter4_idx,
+ vdupq_n_s32(UPSCALE_NORMATIVE_TAPS)));
+
+ const int16_t *x_filter4_ptr[4];
+ tmp_ptr = (uint32_t *)&x_filter4_ptr;
+ vst1q_u32(tmp_ptr, tmp4);
+#endif // AOM_ARCH_AARCH64
+
+ // Load source
+ int16x8_t s0 = vld1q_s16(src4_ptr[0]);
+ int16x8_t s1 = vld1q_s16(src4_ptr[1]);
+ int16x8_t s2 = vld1q_s16(src4_ptr[2]);
+ int16x8_t s3 = vld1q_s16(src4_ptr[3]);
+
+ // Actually load the filters
+ const int16x8_t x_filter0 = vld1q_s16(x_filter4_ptr[0]);
+ const int16x8_t x_filter1 = vld1q_s16(x_filter4_ptr[1]);
+ const int16x8_t x_filter2 = vld1q_s16(x_filter4_ptr[2]);
+ const int16x8_t x_filter3 = vld1q_s16(x_filter4_ptr[3]);
+
+ // Group low and high parts and transpose
+ int16x4_t filters_lo[] = { vget_low_s16(x_filter0),
+ vget_low_s16(x_filter1),
+ vget_low_s16(x_filter2),
+ vget_low_s16(x_filter3) };
+ int16x4_t filters_hi[] = { vget_high_s16(x_filter0),
+ vget_high_s16(x_filter1),
+ vget_high_s16(x_filter2),
+ vget_high_s16(x_filter3) };
+ transpose_array_inplace_u16_4x4((uint16x4_t *)filters_lo);
+ transpose_array_inplace_u16_4x4((uint16x4_t *)filters_hi);
+
+ // Run the 2D Scale X convolution
+ uint16x4_t d0 = highbd_convolve8_2d_scale_horiz4x8_s32_s16(
+ s0, s1, s2, s3, filters_lo, filters_hi, shift_s32, offset_s32);
+
+ d0 = vmin_u16(d0, max);
+ vst1_u16(d, d0);
+
+ x_qn += 4 * x_step_qn;
+ d += 4;
+ width -= 4;
+ } while (width > 0);
+
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ height--;
+ } while (height > 0);
+ }
+}
diff --git a/av1/common/arm/highbd_convolve_neon.c b/av1/common/arm/highbd_convolve_neon.c
index fb18e28..3f5ff9e 100644
--- a/av1/common/arm/highbd_convolve_neon.c
+++ b/av1/common/arm/highbd_convolve_neon.c
@@ -17,62 +17,87 @@
#include "aom_dsp/aom_dsp_common.h"
#include "aom_dsp/arm/mem_neon.h"
-#include "aom_dsp/arm/transpose_neon.h"
#include "aom_ports/mem.h"
#include "av1/common/convolve.h"
#include "av1/common/filter.h"
-#include "av1/common/arm/highbd_convolve_neon.h"
+
+static INLINE uint16x4_t
+highbd_convolve6_4_y(const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
+ const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
+ const int16x8_t y_filter) {
+ // Values at indices 0 and 7 of y_filter are zero.
+ const int16x4_t y_filter_0_3 = vget_low_s16(y_filter);
+ const int16x4_t y_filter_4_7 = vget_high_s16(y_filter);
+
+ int32x4_t sum = vmull_lane_s16(s0, y_filter_0_3, 1);
+ sum = vmlal_lane_s16(sum, s1, y_filter_0_3, 2);
+ sum = vmlal_lane_s16(sum, s2, y_filter_0_3, 3);
+ sum = vmlal_lane_s16(sum, s3, y_filter_4_7, 0);
+ sum = vmlal_lane_s16(sum, s4, y_filter_4_7, 1);
+ sum = vmlal_lane_s16(sum, s5, y_filter_4_7, 2);
+
+ return vqrshrun_n_s32(sum, COMPOUND_ROUND1_BITS);
+}
+
+static INLINE uint16x8_t
+highbd_convolve6_8_y(const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
+ const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t y_filter) {
+ // Values at indices 0 and 7 of y_filter are zero.
+ const int16x4_t y_filter_0_3 = vget_low_s16(y_filter);
+ const int16x4_t y_filter_4_7 = vget_high_s16(y_filter);
+
+ int32x4_t sum0 = vmull_lane_s16(vget_low_s16(s0), y_filter_0_3, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s1), y_filter_0_3, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s2), y_filter_0_3, 3);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s3), y_filter_4_7, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s4), y_filter_4_7, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s5), y_filter_4_7, 2);
+
+ int32x4_t sum1 = vmull_lane_s16(vget_high_s16(s0), y_filter_0_3, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s1), y_filter_0_3, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s2), y_filter_0_3, 3);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s3), y_filter_4_7, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s4), y_filter_4_7, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s5), y_filter_4_7, 2);
+
+ return vcombine_u16(vqrshrun_n_s32(sum0, COMPOUND_ROUND1_BITS),
+ vqrshrun_n_s32(sum1, COMPOUND_ROUND1_BITS));
+}
static INLINE void highbd_convolve_y_sr_6tap_neon(
const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
int w, int h, const int16_t *y_filter_ptr, const int bd) {
const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
const int16x8_t y_filter_0_7 = vld1q_s16(y_filter_ptr);
- const int32x4_t zero_s32 = vdupq_n_s32(0);
- if (w <= 4) {
- int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8;
- uint16x4_t d0, d1, d2, d3;
- uint16x8_t d01, d23;
+ if (w == 4) {
const int16_t *s = (const int16_t *)(src_ptr + src_stride);
uint16_t *d = dst_ptr;
+ int16x4_t s0, s1, s2, s3, s4;
load_s16_4x5(s, src_stride, &s0, &s1, &s2, &s3, &s4);
s += 5 * src_stride;
do {
+ int16x4_t s5, s6, s7, s8;
load_s16_4x4(s, src_stride, &s5, &s6, &s7, &s8);
- d0 = highbd_convolve6_4_s32_s16(s0, s1, s2, s3, s4, s5, y_filter_0_7,
- zero_s32);
- d1 = highbd_convolve6_4_s32_s16(s1, s2, s3, s4, s5, s6, y_filter_0_7,
- zero_s32);
- d2 = highbd_convolve6_4_s32_s16(s2, s3, s4, s5, s6, s7, y_filter_0_7,
- zero_s32);
- d3 = highbd_convolve6_4_s32_s16(s3, s4, s5, s6, s7, s8, y_filter_0_7,
- zero_s32);
+ uint16x4_t d0 =
+ highbd_convolve6_4_y(s0, s1, s2, s3, s4, s5, y_filter_0_7);
+ uint16x4_t d1 =
+ highbd_convolve6_4_y(s1, s2, s3, s4, s5, s6, y_filter_0_7);
+ uint16x4_t d2 =
+ highbd_convolve6_4_y(s2, s3, s4, s5, s6, s7, y_filter_0_7);
+ uint16x4_t d3 =
+ highbd_convolve6_4_y(s3, s4, s5, s6, s7, s8, y_filter_0_7);
- d01 = vcombine_u16(d0, d1);
- d23 = vcombine_u16(d2, d3);
+ d0 = vmin_u16(d0, vget_low_u16(max));
+ d1 = vmin_u16(d1, vget_low_u16(max));
+ d2 = vmin_u16(d2, vget_low_u16(max));
+ d3 = vmin_u16(d3, vget_low_u16(max));
- d01 = vminq_u16(d01, max);
- d23 = vminq_u16(d23, max);
-
- if (w == 2) {
- store_u16q_2x1(d + 0 * dst_stride, d01, 0);
- store_u16q_2x1(d + 1 * dst_stride, d01, 2);
- if (h != 2) {
- store_u16q_2x1(d + 2 * dst_stride, d23, 0);
- store_u16q_2x1(d + 3 * dst_stride, d23, 2);
- }
- } else {
- vst1_u16(d + 0 * dst_stride, vget_low_u16(d01));
- vst1_u16(d + 1 * dst_stride, vget_high_u16(d01));
- if (h != 2) {
- vst1_u16(d + 2 * dst_stride, vget_low_u16(d23));
- vst1_u16(d + 3 * dst_stride, vget_high_u16(d23));
- }
- }
+ store_u16_4x4(d, dst_stride, d0, d1, d2, d3);
s0 = s4;
s1 = s5;
@@ -82,42 +107,37 @@
s += 4 * src_stride;
d += 4 * dst_stride;
h -= 4;
- } while (h > 0);
+ } while (h != 0);
} else {
- // if width is a multiple of 8 & height is a multiple of 4
- int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8;
- uint16x8_t d0, d1, d2, d3;
-
+ // Width is a multiple of 8 and height is a multiple of 4.
do {
int height = h;
const int16_t *s = (const int16_t *)(src_ptr + src_stride);
uint16_t *d = dst_ptr;
+ int16x8_t s0, s1, s2, s3, s4;
load_s16_8x5(s, src_stride, &s0, &s1, &s2, &s3, &s4);
s += 5 * src_stride;
do {
+ int16x8_t s5, s6, s7, s8;
load_s16_8x4(s, src_stride, &s5, &s6, &s7, &s8);
- d0 = highbd_convolve6_8_s32_s16(s0, s1, s2, s3, s4, s5, y_filter_0_7,
- zero_s32);
- d1 = highbd_convolve6_8_s32_s16(s1, s2, s3, s4, s5, s6, y_filter_0_7,
- zero_s32);
- d2 = highbd_convolve6_8_s32_s16(s2, s3, s4, s5, s6, s7, y_filter_0_7,
- zero_s32);
- d3 = highbd_convolve6_8_s32_s16(s3, s4, s5, s6, s7, s8, y_filter_0_7,
- zero_s32);
+ uint16x8_t d0 =
+ highbd_convolve6_8_y(s0, s1, s2, s3, s4, s5, y_filter_0_7);
+ uint16x8_t d1 =
+ highbd_convolve6_8_y(s1, s2, s3, s4, s5, s6, y_filter_0_7);
+ uint16x8_t d2 =
+ highbd_convolve6_8_y(s2, s3, s4, s5, s6, s7, y_filter_0_7);
+ uint16x8_t d3 =
+ highbd_convolve6_8_y(s3, s4, s5, s6, s7, s8, y_filter_0_7);
d0 = vminq_u16(d0, max);
d1 = vminq_u16(d1, max);
d2 = vminq_u16(d2, max);
d3 = vminq_u16(d3, max);
- if (h == 2) {
- store_u16_8x2(d, dst_stride, d0, d1);
- } else {
- store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
- }
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
s0 = s4;
s1 = s5;
@@ -127,66 +147,96 @@
s += 4 * src_stride;
d += 4 * dst_stride;
height -= 4;
- } while (height > 0);
+ } while (height != 0);
src_ptr += 8;
dst_ptr += 8;
w -= 8;
- } while (w > 0);
+ } while (w != 0);
}
}
+static INLINE uint16x4_t highbd_convolve8_4_y(
+ const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
+ const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
+ const int16x4_t s6, const int16x4_t s7, const int16x8_t y_filter) {
+ const int16x4_t y_filter_0_3 = vget_low_s16(y_filter);
+ const int16x4_t y_filter_4_7 = vget_high_s16(y_filter);
+
+ int32x4_t sum = vmull_lane_s16(s0, y_filter_0_3, 0);
+ sum = vmlal_lane_s16(sum, s1, y_filter_0_3, 1);
+ sum = vmlal_lane_s16(sum, s2, y_filter_0_3, 2);
+ sum = vmlal_lane_s16(sum, s3, y_filter_0_3, 3);
+ sum = vmlal_lane_s16(sum, s4, y_filter_4_7, 0);
+ sum = vmlal_lane_s16(sum, s5, y_filter_4_7, 1);
+ sum = vmlal_lane_s16(sum, s6, y_filter_4_7, 2);
+ sum = vmlal_lane_s16(sum, s7, y_filter_4_7, 3);
+
+ return vqrshrun_n_s32(sum, COMPOUND_ROUND1_BITS);
+}
+
+static INLINE uint16x8_t highbd_convolve8_8_y(
+ const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
+ const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t s6, const int16x8_t s7, const int16x8_t y_filter) {
+ const int16x4_t y_filter_0_3 = vget_low_s16(y_filter);
+ const int16x4_t y_filter_4_7 = vget_high_s16(y_filter);
+
+ int32x4_t sum0 = vmull_lane_s16(vget_low_s16(s0), y_filter_0_3, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s1), y_filter_0_3, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s2), y_filter_0_3, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s3), y_filter_0_3, 3);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s4), y_filter_4_7, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s5), y_filter_4_7, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s6), y_filter_4_7, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s7), y_filter_4_7, 3);
+
+ int32x4_t sum1 = vmull_lane_s16(vget_high_s16(s0), y_filter_0_3, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s1), y_filter_0_3, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s2), y_filter_0_3, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s3), y_filter_0_3, 3);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s4), y_filter_4_7, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s5), y_filter_4_7, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s6), y_filter_4_7, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s7), y_filter_4_7, 3);
+
+ return vcombine_u16(vqrshrun_n_s32(sum0, COMPOUND_ROUND1_BITS),
+ vqrshrun_n_s32(sum1, COMPOUND_ROUND1_BITS));
+}
+
static INLINE void highbd_convolve_y_sr_8tap_neon(
const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
int w, int h, const int16_t *y_filter_ptr, int bd) {
const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
const int16x8_t y_filter = vld1q_s16(y_filter_ptr);
- const int32x4_t zero_s32 = vdupq_n_s32(0);
- if (w <= 4) {
- int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
- uint16x4_t d0, d1, d2, d3;
- uint16x8_t d01, d23;
-
+ if (w == 4) {
const int16_t *s = (const int16_t *)src_ptr;
uint16_t *d = dst_ptr;
+ int16x4_t s0, s1, s2, s3, s4, s5, s6;
load_s16_4x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
s += 7 * src_stride;
do {
+ int16x4_t s7, s8, s9, s10;
load_s16_4x4(s, src_stride, &s7, &s8, &s9, &s10);
- d0 = highbd_convolve8_4_s32_s16(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
- zero_s32);
- d1 = highbd_convolve8_4_s32_s16(s1, s2, s3, s4, s5, s6, s7, s8, y_filter,
- zero_s32);
- d2 = highbd_convolve8_4_s32_s16(s2, s3, s4, s5, s6, s7, s8, s9, y_filter,
- zero_s32);
- d3 = highbd_convolve8_4_s32_s16(s3, s4, s5, s6, s7, s8, s9, s10, y_filter,
- zero_s32);
+ uint16x4_t d0 =
+ highbd_convolve8_4_y(s0, s1, s2, s3, s4, s5, s6, s7, y_filter);
+ uint16x4_t d1 =
+ highbd_convolve8_4_y(s1, s2, s3, s4, s5, s6, s7, s8, y_filter);
+ uint16x4_t d2 =
+ highbd_convolve8_4_y(s2, s3, s4, s5, s6, s7, s8, s9, y_filter);
+ uint16x4_t d3 =
+ highbd_convolve8_4_y(s3, s4, s5, s6, s7, s8, s9, s10, y_filter);
- d01 = vcombine_u16(d0, d1);
- d23 = vcombine_u16(d2, d3);
+ d0 = vmin_u16(d0, vget_low_u16(max));
+ d1 = vmin_u16(d1, vget_low_u16(max));
+ d2 = vmin_u16(d2, vget_low_u16(max));
+ d3 = vmin_u16(d3, vget_low_u16(max));
- d01 = vminq_u16(d01, max);
- d23 = vminq_u16(d23, max);
-
- if (w == 2) {
- store_u16q_2x1(d + 0 * dst_stride, d01, 0);
- store_u16q_2x1(d + 1 * dst_stride, d01, 2);
- if (h != 2) {
- store_u16q_2x1(d + 2 * dst_stride, d23, 0);
- store_u16q_2x1(d + 3 * dst_stride, d23, 2);
- }
- } else {
- vst1_u16(d + 0 * dst_stride, vget_low_u16(d01));
- vst1_u16(d + 1 * dst_stride, vget_high_u16(d01));
- if (h != 2) {
- vst1_u16(d + 2 * dst_stride, vget_low_u16(d23));
- vst1_u16(d + 3 * dst_stride, vget_high_u16(d23));
- }
- }
+ store_u16_4x4(d, dst_stride, d0, d1, d2, d3);
s0 = s4;
s1 = s5;
@@ -198,40 +248,36 @@
s += 4 * src_stride;
d += 4 * dst_stride;
h -= 4;
- } while (h > 0);
+ } while (h != 0);
} else {
- int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
- uint16x8_t d0, d1, d2, d3;
do {
int height = h;
const int16_t *s = (const int16_t *)src_ptr;
uint16_t *d = dst_ptr;
+ int16x8_t s0, s1, s2, s3, s4, s5, s6;
load_s16_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
s += 7 * src_stride;
do {
+ int16x8_t s7, s8, s9, s10;
load_s16_8x4(s, src_stride, &s7, &s8, &s9, &s10);
- d0 = highbd_convolve8_8_s32_s16(s0, s1, s2, s3, s4, s5, s6, s7,
- y_filter, zero_s32);
- d1 = highbd_convolve8_8_s32_s16(s1, s2, s3, s4, s5, s6, s7, s8,
- y_filter, zero_s32);
- d2 = highbd_convolve8_8_s32_s16(s2, s3, s4, s5, s6, s7, s8, s9,
- y_filter, zero_s32);
- d3 = highbd_convolve8_8_s32_s16(s3, s4, s5, s6, s7, s8, s9, s10,
- y_filter, zero_s32);
+ uint16x8_t d0 =
+ highbd_convolve8_8_y(s0, s1, s2, s3, s4, s5, s6, s7, y_filter);
+ uint16x8_t d1 =
+ highbd_convolve8_8_y(s1, s2, s3, s4, s5, s6, s7, s8, y_filter);
+ uint16x8_t d2 =
+ highbd_convolve8_8_y(s2, s3, s4, s5, s6, s7, s8, s9, y_filter);
+ uint16x8_t d3 =
+ highbd_convolve8_8_y(s3, s4, s5, s6, s7, s8, s9, s10, y_filter);
d0 = vminq_u16(d0, max);
d1 = vminq_u16(d1, max);
d2 = vminq_u16(d2, max);
d3 = vminq_u16(d3, max);
- if (h == 2) {
- store_u16_8x2(d, dst_stride, d0, d1);
- } else {
- store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
- }
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
s0 = s4;
s1 = s5;
@@ -243,71 +289,117 @@
s += 4 * src_stride;
d += 4 * dst_stride;
height -= 4;
- } while (height > 0);
+ } while (height != 0);
src_ptr += 8;
dst_ptr += 8;
w -= 8;
- } while (w > 0);
+ } while (w != 0);
}
}
+static INLINE uint16x4_t highbd_convolve12_4_y(
+ const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
+ const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
+ const int16x4_t s6, const int16x4_t s7, const int16x4_t s8,
+ const int16x4_t s9, const int16x4_t s10, const int16x4_t s11,
+ const int16x8_t y_filter_0_7, const int16x4_t y_filter_8_11) {
+ const int16x4_t y_filter_0_3 = vget_low_s16(y_filter_0_7);
+ const int16x4_t y_filter_4_7 = vget_high_s16(y_filter_0_7);
+
+ int32x4_t sum = vmull_lane_s16(s0, y_filter_0_3, 0);
+ sum = vmlal_lane_s16(sum, s1, y_filter_0_3, 1);
+ sum = vmlal_lane_s16(sum, s2, y_filter_0_3, 2);
+ sum = vmlal_lane_s16(sum, s3, y_filter_0_3, 3);
+ sum = vmlal_lane_s16(sum, s4, y_filter_4_7, 0);
+ sum = vmlal_lane_s16(sum, s5, y_filter_4_7, 1);
+ sum = vmlal_lane_s16(sum, s6, y_filter_4_7, 2);
+ sum = vmlal_lane_s16(sum, s7, y_filter_4_7, 3);
+ sum = vmlal_lane_s16(sum, s8, y_filter_8_11, 0);
+ sum = vmlal_lane_s16(sum, s9, y_filter_8_11, 1);
+ sum = vmlal_lane_s16(sum, s10, y_filter_8_11, 2);
+ sum = vmlal_lane_s16(sum, s11, y_filter_8_11, 3);
+
+ return vqrshrun_n_s32(sum, COMPOUND_ROUND1_BITS);
+}
+
+static INLINE uint16x8_t highbd_convolve12_8_y(
+ const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
+ const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t s6, const int16x8_t s7, const int16x8_t s8,
+ const int16x8_t s9, const int16x8_t s10, const int16x8_t s11,
+ const int16x8_t y_filter_0_7, const int16x4_t y_filter_8_11) {
+ const int16x4_t y_filter_0_3 = vget_low_s16(y_filter_0_7);
+ const int16x4_t y_filter_4_7 = vget_high_s16(y_filter_0_7);
+
+ int32x4_t sum0 = vmull_lane_s16(vget_low_s16(s0), y_filter_0_3, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s1), y_filter_0_3, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s2), y_filter_0_3, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s3), y_filter_0_3, 3);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s4), y_filter_4_7, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s5), y_filter_4_7, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s6), y_filter_4_7, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s7), y_filter_4_7, 3);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s8), y_filter_8_11, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s9), y_filter_8_11, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s10), y_filter_8_11, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s11), y_filter_8_11, 3);
+
+ int32x4_t sum1 = vmull_lane_s16(vget_high_s16(s0), y_filter_0_3, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s1), y_filter_0_3, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s2), y_filter_0_3, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s3), y_filter_0_3, 3);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s4), y_filter_4_7, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s5), y_filter_4_7, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s6), y_filter_4_7, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s7), y_filter_4_7, 3);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s8), y_filter_8_11, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s9), y_filter_8_11, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s10), y_filter_8_11, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s11), y_filter_8_11, 3);
+
+ return vcombine_u16(vqrshrun_n_s32(sum0, COMPOUND_ROUND1_BITS),
+ vqrshrun_n_s32(sum1, COMPOUND_ROUND1_BITS));
+}
+
static INLINE void highbd_convolve_y_sr_12tap_neon(
const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
int w, int h, const int16_t *y_filter_ptr, int bd) {
const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
const int16x8_t y_filter_0_7 = vld1q_s16(y_filter_ptr);
const int16x4_t y_filter_8_11 = vld1_s16(y_filter_ptr + 8);
- const int32x4_t zero_s32 = vdupq_n_s32(0);
- if (w <= 4) {
- int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14;
- uint16x4_t d0, d1, d2, d3;
- uint16x8_t d01, d23;
-
+ if (w == 4) {
const int16_t *s = (const int16_t *)src_ptr;
uint16_t *d = dst_ptr;
+ int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
load_s16_4x11(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7, &s8,
&s9, &s10);
s += 11 * src_stride;
do {
+ int16x4_t s11, s12, s13, s14;
load_s16_4x4(s, src_stride, &s11, &s12, &s13, &s14);
- d0 = highbd_convolve12_y_4_s32_s16(s0, s1, s2, s3, s4, s5, s6, s7, s8, s9,
- s10, s11, y_filter_0_7, y_filter_8_11,
- zero_s32);
- d1 = highbd_convolve12_y_4_s32_s16(s1, s2, s3, s4, s5, s6, s7, s8, s9,
- s10, s11, s12, y_filter_0_7,
- y_filter_8_11, zero_s32);
- d2 = highbd_convolve12_y_4_s32_s16(s2, s3, s4, s5, s6, s7, s8, s9, s10,
- s11, s12, s13, y_filter_0_7,
- y_filter_8_11, zero_s32);
- d3 = highbd_convolve12_y_4_s32_s16(s3, s4, s5, s6, s7, s8, s9, s10, s11,
- s12, s13, s14, y_filter_0_7,
- y_filter_8_11, zero_s32);
+ uint16x4_t d0 =
+ highbd_convolve12_4_y(s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10,
+ s11, y_filter_0_7, y_filter_8_11);
+ uint16x4_t d1 =
+ highbd_convolve12_4_y(s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11,
+ s12, y_filter_0_7, y_filter_8_11);
+ uint16x4_t d2 =
+ highbd_convolve12_4_y(s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12,
+ s13, y_filter_0_7, y_filter_8_11);
+ uint16x4_t d3 =
+ highbd_convolve12_4_y(s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13,
+ s14, y_filter_0_7, y_filter_8_11);
- d01 = vcombine_u16(d0, d1);
- d23 = vcombine_u16(d2, d3);
+ d0 = vmin_u16(d0, vget_low_u16(max));
+ d1 = vmin_u16(d1, vget_low_u16(max));
+ d2 = vmin_u16(d2, vget_low_u16(max));
+ d3 = vmin_u16(d3, vget_low_u16(max));
- d01 = vminq_u16(d01, max);
- d23 = vminq_u16(d23, max);
-
- if (w == 2) {
- store_u16q_2x1(d + 0 * dst_stride, d01, 0);
- store_u16q_2x1(d + 1 * dst_stride, d01, 2);
- if (h != 2) {
- store_u16q_2x1(d + 2 * dst_stride, d23, 0);
- store_u16q_2x1(d + 3 * dst_stride, d23, 2);
- }
- } else {
- vst1_u16(d + 0 * dst_stride, vget_low_u16(d01));
- vst1_u16(d + 1 * dst_stride, vget_high_u16(d01));
- if (h != 2) {
- vst1_u16(d + 2 * dst_stride, vget_low_u16(d23));
- vst1_u16(d + 3 * dst_stride, vget_high_u16(d23));
- }
- }
+ store_u16_4x4(d, dst_stride, d0, d1, d2, d3);
s0 = s4;
s1 = s5;
@@ -323,46 +415,41 @@
s += 4 * src_stride;
d += 4 * dst_stride;
h -= 4;
- } while (h > 0);
+ } while (h != 0);
} else {
- uint16x8_t d0, d1, d2, d3;
- int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14;
-
do {
int height = h;
const int16_t *s = (const int16_t *)src_ptr;
uint16_t *d = dst_ptr;
+ int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
load_s16_8x11(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7, &s8,
&s9, &s10);
s += 11 * src_stride;
do {
+ int16x8_t s11, s12, s13, s14;
load_s16_8x4(s, src_stride, &s11, &s12, &s13, &s14);
- d0 = highbd_convolve12_y_8_s32_s16(s0, s1, s2, s3, s4, s5, s6, s7, s8,
- s9, s10, s11, y_filter_0_7,
- y_filter_8_11, zero_s32);
- d1 = highbd_convolve12_y_8_s32_s16(s1, s2, s3, s4, s5, s6, s7, s8, s9,
- s10, s11, s12, y_filter_0_7,
- y_filter_8_11, zero_s32);
- d2 = highbd_convolve12_y_8_s32_s16(s2, s3, s4, s5, s6, s7, s8, s9, s10,
- s11, s12, s13, y_filter_0_7,
- y_filter_8_11, zero_s32);
- d3 = highbd_convolve12_y_8_s32_s16(s3, s4, s5, s6, s7, s8, s9, s10, s11,
- s12, s13, s14, y_filter_0_7,
- y_filter_8_11, zero_s32);
+ uint16x8_t d0 =
+ highbd_convolve12_8_y(s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10,
+ s11, y_filter_0_7, y_filter_8_11);
+ uint16x8_t d1 =
+ highbd_convolve12_8_y(s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11,
+ s12, y_filter_0_7, y_filter_8_11);
+ uint16x8_t d2 =
+ highbd_convolve12_8_y(s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12,
+ s13, y_filter_0_7, y_filter_8_11);
+ uint16x8_t d3 =
+ highbd_convolve12_8_y(s3, s4, s5, s6, s7, s8, s9, s10, s11, s12,
+ s13, s14, y_filter_0_7, y_filter_8_11);
d0 = vminq_u16(d0, max);
d1 = vminq_u16(d1, max);
d2 = vminq_u16(d2, max);
d3 = vminq_u16(d3, max);
- if (h == 2) {
- store_u16_8x2(d, dst_stride, d0, d1);
- } else {
- store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
- }
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
s0 = s4;
s1 = s5;
@@ -378,12 +465,12 @@
s += 4 * src_stride;
d += 4 * dst_stride;
height -= 4;
- } while (height > 0);
+ } while (height != 0);
src_ptr += 8;
dst_ptr += 8;
w -= 8;
- } while (w > 0);
+ } while (w != 0);
}
}
@@ -391,6 +478,11 @@
uint16_t *dst, int dst_stride, int w, int h,
const InterpFilterParams *filter_params_y,
const int subpel_y_qn, int bd) {
+ if (w == 2 || h == 2) {
+ av1_highbd_convolve_y_sr_c(src, src_stride, dst, dst_stride, w, h,
+ filter_params_y, subpel_y_qn, bd);
+ return;
+ }
const int y_filter_taps = get_filter_tap(filter_params_y, subpel_y_qn);
const int vert_offset = filter_params_y->taps / 2 - 1;
const int16_t *y_filter_ptr = av1_get_interp_filter_subpel_kernel(
@@ -413,207 +505,366 @@
y_filter_ptr, bd);
}
-static INLINE void highbd_convolve_x_sr_8tap_neon(
+static INLINE uint16x8_t highbd_convolve6_8_x(const int16x8_t s[6],
+ const int16x8_t x_filter,
+ const int32x4_t offset) {
+ // Values at indices 0 and 7 of y_filter are zero.
+ const int16x4_t x_filter_0_3 = vget_low_s16(x_filter);
+ const int16x4_t x_filter_4_7 = vget_high_s16(x_filter);
+
+ int32x4_t sum0 = offset;
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[0]), x_filter_0_3, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[1]), x_filter_0_3, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[2]), x_filter_0_3, 3);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[3]), x_filter_4_7, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[4]), x_filter_4_7, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[5]), x_filter_4_7, 2);
+
+ int32x4_t sum1 = offset;
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[0]), x_filter_0_3, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[1]), x_filter_0_3, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[2]), x_filter_0_3, 3);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[3]), x_filter_4_7, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[4]), x_filter_4_7, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[5]), x_filter_4_7, 2);
+
+ return vcombine_u16(vqrshrun_n_s32(sum0, FILTER_BITS),
+ vqrshrun_n_s32(sum1, FILTER_BITS));
+}
+
+static INLINE void highbd_convolve_x_sr_6tap_neon(
const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
int w, int h, const int16_t *x_filter_ptr, ConvolveParams *conv_params,
int bd) {
const int16x8_t x_filter = vld1q_s16(x_filter_ptr);
const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
- const int32x4_t shift_s32 = vdupq_n_s32(-conv_params->round_0);
- const int bits = FILTER_BITS - conv_params->round_0;
- const int16x8_t bits_s16 = vdupq_n_s16(-bits);
- const int32x4_t zero_s32 = vdupq_n_s32(0);
+ // This shim allows to do only one rounding shift instead of two.
+ const int32x4_t offset = vdupq_n_s32(1 << (conv_params->round_0 - 1));
- if (w <= 4) {
- int16x8_t s0, s1, s2, s3;
- uint16x4_t d0, d1;
- uint16x8_t d01;
+ int height = h;
+ do {
+ int width = w;
const int16_t *s = (const int16_t *)src_ptr;
uint16_t *d = dst_ptr;
do {
- load_s16_8x2(s, src_stride, &s0, &s2);
- load_s16_8x2(s + 8, src_stride, &s1, &s3);
+ int16x8_t s0[6], s1[6], s2[6], s3[6];
+ load_s16_8x6(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3],
+ &s0[4], &s0[5]);
+ load_s16_8x6(s + 1 * src_stride, 1, &s1[0], &s1[1], &s1[2], &s1[3],
+ &s1[4], &s1[5]);
+ load_s16_8x6(s + 2 * src_stride, 1, &s2[0], &s2[1], &s2[2], &s2[3],
+ &s2[4], &s2[5]);
+ load_s16_8x6(s + 3 * src_stride, 1, &s3[0], &s3[1], &s3[2], &s3[3],
+ &s3[4], &s3[5]);
- d0 = highbd_convolve8_horiz4_s32_s16(s0, s1, x_filter, shift_s32,
- zero_s32);
- d1 = highbd_convolve8_horiz4_s32_s16(s2, s3, x_filter, shift_s32,
- zero_s32);
+ uint16x8_t d0 = highbd_convolve6_8_x(s0, x_filter, offset);
+ uint16x8_t d1 = highbd_convolve6_8_x(s1, x_filter, offset);
+ uint16x8_t d2 = highbd_convolve6_8_x(s2, x_filter, offset);
+ uint16x8_t d3 = highbd_convolve6_8_x(s3, x_filter, offset);
- d01 = vcombine_u16(d0, d1);
- d01 = vqrshlq_u16(d01, bits_s16);
- d01 = vminq_u16(d01, max);
+ d0 = vminq_u16(d0, max);
+ d1 = vminq_u16(d1, max);
+ d2 = vminq_u16(d2, max);
+ d3 = vminq_u16(d3, max);
- if (w == 2) {
- store_u16q_2x1(d + 0 * dst_stride, d01, 0);
- store_u16q_2x1(d + 1 * dst_stride, d01, 2);
- } else {
- vst1_u16(d + 0 * dst_stride, vget_low_u16(d01));
- vst1_u16(d + 1 * dst_stride, vget_high_u16(d01));
- }
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
- s += 2 * src_stride;
- d += 2 * dst_stride;
- h -= 2;
- } while (h > 0);
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ height -= 4;
+ } while (height != 0);
+}
+
+static INLINE uint16x4_t highbd_convolve4_4_x(const int16x4_t s[4],
+ const int16x4_t x_filter,
+ const int32x4_t offset) {
+ int32x4_t sum = offset;
+ sum = vmlal_lane_s16(sum, s[0], x_filter, 0);
+ sum = vmlal_lane_s16(sum, s[1], x_filter, 1);
+ sum = vmlal_lane_s16(sum, s[2], x_filter, 2);
+ sum = vmlal_lane_s16(sum, s[3], x_filter, 3);
+
+ return vqrshrun_n_s32(sum, FILTER_BITS);
+}
+
+static INLINE uint16x8_t highbd_convolve8_8_x(const int16x8_t s[8],
+ const int16x8_t x_filter,
+ const int32x4_t offset) {
+ const int16x4_t x_filter_0_3 = vget_low_s16(x_filter);
+ const int16x4_t x_filter_4_7 = vget_high_s16(x_filter);
+
+ int32x4_t sum0 = offset;
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[0]), x_filter_0_3, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[1]), x_filter_0_3, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[2]), x_filter_0_3, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[3]), x_filter_0_3, 3);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[4]), x_filter_4_7, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[5]), x_filter_4_7, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[6]), x_filter_4_7, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[7]), x_filter_4_7, 3);
+
+ int32x4_t sum1 = offset;
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[0]), x_filter_0_3, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[1]), x_filter_0_3, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[2]), x_filter_0_3, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[3]), x_filter_0_3, 3);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[4]), x_filter_4_7, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[5]), x_filter_4_7, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[6]), x_filter_4_7, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[7]), x_filter_4_7, 3);
+
+ return vcombine_u16(vqrshrun_n_s32(sum0, FILTER_BITS),
+ vqrshrun_n_s32(sum1, FILTER_BITS));
+}
+
+static INLINE void highbd_convolve_x_sr_neon(const uint16_t *src_ptr,
+ int src_stride, uint16_t *dst_ptr,
+ int dst_stride, int w, int h,
+ const int16_t *x_filter_ptr,
+ ConvolveParams *conv_params,
+ int bd) {
+ const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
+ // This shim allows to do only one rounding shift instead of two.
+ const int32x4_t offset = vdupq_n_s32(1 << (conv_params->round_0 - 1));
+
+ if (w == 4) {
+ // 4-tap filters are used for blocks having width == 4.
+ const int16x4_t x_filter = vld1_s16(x_filter_ptr + 2);
+ const int16_t *s = (const int16_t *)(src_ptr + 2);
+ uint16_t *d = dst_ptr;
+
+ do {
+ int16x4_t s0[4], s1[4], s2[4], s3[4];
+ load_s16_4x4(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3]);
+ load_s16_4x4(s + 1 * src_stride, 1, &s1[0], &s1[1], &s1[2], &s1[3]);
+ load_s16_4x4(s + 2 * src_stride, 1, &s2[0], &s2[1], &s2[2], &s2[3]);
+ load_s16_4x4(s + 3 * src_stride, 1, &s3[0], &s3[1], &s3[2], &s3[3]);
+
+ uint16x4_t d0 = highbd_convolve4_4_x(s0, x_filter, offset);
+ uint16x4_t d1 = highbd_convolve4_4_x(s1, x_filter, offset);
+ uint16x4_t d2 = highbd_convolve4_4_x(s2, x_filter, offset);
+ uint16x4_t d3 = highbd_convolve4_4_x(s3, x_filter, offset);
+
+ d0 = vmin_u16(d0, vget_low_u16(max));
+ d1 = vmin_u16(d1, vget_low_u16(max));
+ d2 = vmin_u16(d2, vget_low_u16(max));
+ d3 = vmin_u16(d3, vget_low_u16(max));
+
+ store_u16_4x4(d, dst_stride, d0, d1, d2, d3);
+
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ h -= 4;
+ } while (h != 0);
} else {
+ const int16x8_t x_filter = vld1q_s16(x_filter_ptr);
int height = h;
- int16x8_t s0, s1, s2, s3, s4, s5, s6, s7;
- uint16x8_t d0, d1, d2, d3;
+
do {
int width = w;
const int16_t *s = (const int16_t *)src_ptr;
uint16_t *d = dst_ptr;
- load_s16_8x4(s, src_stride, &s0, &s2, &s4, &s6);
- s += 8;
-
do {
- load_s16_8x4(s, src_stride, &s1, &s3, &s5, &s7);
+ int16x8_t s0[8], s1[8], s2[8], s3[8];
+ load_s16_8x8(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3],
+ &s0[4], &s0[5], &s0[6], &s0[7]);
+ load_s16_8x8(s + 1 * src_stride, 1, &s1[0], &s1[1], &s1[2], &s1[3],
+ &s1[4], &s1[5], &s1[6], &s1[7]);
+ load_s16_8x8(s + 2 * src_stride, 1, &s2[0], &s2[1], &s2[2], &s2[3],
+ &s2[4], &s2[5], &s2[6], &s2[7]);
+ load_s16_8x8(s + 3 * src_stride, 1, &s3[0], &s3[1], &s3[2], &s3[3],
+ &s3[4], &s3[5], &s3[6], &s3[7]);
- d0 = highbd_convolve8_horiz8_s32_s16(s0, s1, x_filter, shift_s32,
- zero_s32);
- d1 = highbd_convolve8_horiz8_s32_s16(s2, s3, x_filter, shift_s32,
- zero_s32);
- d2 = highbd_convolve8_horiz8_s32_s16(s4, s5, x_filter, shift_s32,
- zero_s32);
- d3 = highbd_convolve8_horiz8_s32_s16(s6, s7, x_filter, shift_s32,
- zero_s32);
-
- d0 = vqrshlq_u16(d0, bits_s16);
- d1 = vqrshlq_u16(d1, bits_s16);
- d2 = vqrshlq_u16(d2, bits_s16);
- d3 = vqrshlq_u16(d3, bits_s16);
+ uint16x8_t d0 = highbd_convolve8_8_x(s0, x_filter, offset);
+ uint16x8_t d1 = highbd_convolve8_8_x(s1, x_filter, offset);
+ uint16x8_t d2 = highbd_convolve8_8_x(s2, x_filter, offset);
+ uint16x8_t d3 = highbd_convolve8_8_x(s3, x_filter, offset);
d0 = vminq_u16(d0, max);
d1 = vminq_u16(d1, max);
d2 = vminq_u16(d2, max);
d3 = vminq_u16(d3, max);
- if (h == 2) {
- store_u16_8x2(d, dst_stride, d0, d1);
- } else {
- store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
- }
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
- s0 = s1;
- s2 = s3;
- s4 = s5;
- s6 = s7;
s += 8;
d += 8;
width -= 8;
- } while (width > 0);
+ } while (width != 0);
src_ptr += 4 * src_stride;
dst_ptr += 4 * dst_stride;
height -= 4;
- } while (height > 0);
+ } while (height != 0);
}
}
+static INLINE uint16x4_t highbd_convolve12_4_x(const int16x4_t s[12],
+ const int16x8_t x_filter_0_7,
+ const int16x4_t x_filter_8_11,
+ const int32x4_t offset) {
+ const int16x4_t x_filter_0_3 = vget_low_s16(x_filter_0_7);
+ const int16x4_t x_filter_4_7 = vget_high_s16(x_filter_0_7);
+
+ int32x4_t sum = offset;
+ sum = vmlal_lane_s16(sum, s[0], x_filter_0_3, 0);
+ sum = vmlal_lane_s16(sum, s[1], x_filter_0_3, 1);
+ sum = vmlal_lane_s16(sum, s[2], x_filter_0_3, 2);
+ sum = vmlal_lane_s16(sum, s[3], x_filter_0_3, 3);
+ sum = vmlal_lane_s16(sum, s[4], x_filter_4_7, 0);
+ sum = vmlal_lane_s16(sum, s[5], x_filter_4_7, 1);
+ sum = vmlal_lane_s16(sum, s[6], x_filter_4_7, 2);
+ sum = vmlal_lane_s16(sum, s[7], x_filter_4_7, 3);
+ sum = vmlal_lane_s16(sum, s[8], x_filter_8_11, 0);
+ sum = vmlal_lane_s16(sum, s[9], x_filter_8_11, 1);
+ sum = vmlal_lane_s16(sum, s[10], x_filter_8_11, 2);
+ sum = vmlal_lane_s16(sum, s[11], x_filter_8_11, 3);
+
+ return vqrshrun_n_s32(sum, FILTER_BITS);
+}
+
+static INLINE uint16x8_t highbd_convolve12_8_x(const int16x8_t s[12],
+ const int16x8_t x_filter_0_7,
+ const int16x4_t x_filter_8_11,
+ const int32x4_t offset) {
+ const int16x4_t x_filter_0_3 = vget_low_s16(x_filter_0_7);
+ const int16x4_t x_filter_4_7 = vget_high_s16(x_filter_0_7);
+
+ int32x4_t sum0 = offset;
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[0]), x_filter_0_3, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[1]), x_filter_0_3, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[2]), x_filter_0_3, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[3]), x_filter_0_3, 3);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[4]), x_filter_4_7, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[5]), x_filter_4_7, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[6]), x_filter_4_7, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[7]), x_filter_4_7, 3);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[8]), x_filter_8_11, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[9]), x_filter_8_11, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[10]), x_filter_8_11, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[11]), x_filter_8_11, 3);
+
+ int32x4_t sum1 = offset;
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[0]), x_filter_0_3, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[1]), x_filter_0_3, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[2]), x_filter_0_3, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[3]), x_filter_0_3, 3);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[4]), x_filter_4_7, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[5]), x_filter_4_7, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[6]), x_filter_4_7, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[7]), x_filter_4_7, 3);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[8]), x_filter_8_11, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[9]), x_filter_8_11, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[10]), x_filter_8_11, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[11]), x_filter_8_11, 3);
+
+ return vcombine_u16(vqrshrun_n_s32(sum0, FILTER_BITS),
+ vqrshrun_n_s32(sum1, FILTER_BITS));
+}
+
static INLINE void highbd_convolve_x_sr_12tap_neon(
const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
int w, int h, const int16_t *x_filter_ptr, ConvolveParams *conv_params,
int bd) {
const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
- const int32x4_t shift_s32 = vdupq_n_s32(-conv_params->round_0);
- const int bits = FILTER_BITS - conv_params->round_0;
- const int16x8_t bits_s16 = vdupq_n_s16(-bits);
+ // This shim allows to do only one rounding shift instead of two.
+ const int32x4_t offset = vdupq_n_s32(1 << (conv_params->round_0 - 1));
const int16x8_t x_filter_0_7 = vld1q_s16(x_filter_ptr);
const int16x4_t x_filter_8_11 = vld1_s16(x_filter_ptr + 8);
- const int32x4_t zero_s32 = vdupq_n_s32(0);
- if (w <= 4) {
- int16x8_t s0, s1, s2, s3;
- uint16x4_t d0, d1;
- uint16x8_t d01;
-
+ if (w == 4) {
const int16_t *s = (const int16_t *)src_ptr;
uint16_t *d = dst_ptr;
do {
- load_s16_8x2(s, src_stride, &s0, &s2);
- load_s16_8x2(s + 8, src_stride, &s1, &s3);
+ int16x4_t s0[12], s1[12], s2[12], s3[12];
+ load_s16_4x12(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3],
+ &s0[4], &s0[5], &s0[6], &s0[7], &s0[8], &s0[9], &s0[10],
+ &s0[11]);
+ load_s16_4x12(s + 1 * src_stride, 1, &s1[0], &s1[1], &s1[2], &s1[3],
+ &s1[4], &s1[5], &s1[6], &s1[7], &s1[8], &s1[9], &s1[10],
+ &s1[11]);
+ load_s16_4x12(s + 2 * src_stride, 1, &s2[0], &s2[1], &s2[2], &s2[3],
+ &s2[4], &s2[5], &s2[6], &s2[7], &s2[8], &s2[9], &s2[10],
+ &s2[11]);
+ load_s16_4x12(s + 3 * src_stride, 1, &s3[0], &s3[1], &s3[2], &s3[3],
+ &s3[4], &s3[5], &s3[6], &s3[7], &s3[8], &s3[9], &s3[10],
+ &s3[11]);
- d0 = highbd_convolve12_horiz4_s32_s16(s0, s1, x_filter_0_7, x_filter_8_11,
- shift_s32, zero_s32);
- d1 = highbd_convolve12_horiz4_s32_s16(s2, s3, x_filter_0_7, x_filter_8_11,
- shift_s32, zero_s32);
+ uint16x4_t d0 =
+ highbd_convolve12_4_x(s0, x_filter_0_7, x_filter_8_11, offset);
+ uint16x4_t d1 =
+ highbd_convolve12_4_x(s1, x_filter_0_7, x_filter_8_11, offset);
+ uint16x4_t d2 =
+ highbd_convolve12_4_x(s2, x_filter_0_7, x_filter_8_11, offset);
+ uint16x4_t d3 =
+ highbd_convolve12_4_x(s3, x_filter_0_7, x_filter_8_11, offset);
- d01 = vcombine_u16(d0, d1);
- d01 = vqrshlq_u16(d01, bits_s16);
- d01 = vminq_u16(d01, max);
+ d0 = vmin_u16(d0, vget_low_u16(max));
+ d1 = vmin_u16(d1, vget_low_u16(max));
+ d2 = vmin_u16(d2, vget_low_u16(max));
+ d3 = vmin_u16(d3, vget_low_u16(max));
- if (w == 2) {
- store_u16q_2x1(d + 0 * dst_stride, d01, 0);
- store_u16q_2x1(d + 1 * dst_stride, d01, 2);
- } else {
- vst1_u16(d + 0 * dst_stride, vget_low_u16(d01));
- vst1_u16(d + 1 * dst_stride, vget_high_u16(d01));
- }
+ store_u16_4x4(d, dst_stride, d0, d1, d2, d3);
- s += 2 * src_stride;
- d += 2 * dst_stride;
- h -= 2;
- } while (h > 0);
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ h -= 4;
+ } while (h != 0);
} else {
int height = h;
- int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11;
- uint16x8_t d0, d1, d2, d3;
+
do {
int width = w;
const int16_t *s = (const int16_t *)src_ptr;
uint16_t *d = dst_ptr;
- load_s16_8x4(s, src_stride, &s0, &s3, &s6, &s9);
- s += 8;
-
do {
- load_s16_8x4(s, src_stride, &s1, &s4, &s7, &s10);
- load_s16_8x4(s + 8, src_stride, &s2, &s5, &s8, &s11);
+ int16x8_t s0[12], s1[12], s2[12], s3[12];
+ load_s16_8x12(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3],
+ &s0[4], &s0[5], &s0[6], &s0[7], &s0[8], &s0[9], &s0[10],
+ &s0[11]);
+ load_s16_8x12(s + 1 * src_stride, 1, &s1[0], &s1[1], &s1[2], &s1[3],
+ &s1[4], &s1[5], &s1[6], &s1[7], &s1[8], &s1[9], &s1[10],
+ &s1[11]);
+ load_s16_8x12(s + 2 * src_stride, 1, &s2[0], &s2[1], &s2[2], &s2[3],
+ &s2[4], &s2[5], &s2[6], &s2[7], &s2[8], &s2[9], &s2[10],
+ &s2[11]);
+ load_s16_8x12(s + 3 * src_stride, 1, &s3[0], &s3[1], &s3[2], &s3[3],
+ &s3[4], &s3[5], &s3[6], &s3[7], &s3[8], &s3[9], &s3[10],
+ &s3[11]);
- d0 = highbd_convolve12_horiz8_s32_s16(
- s0, s1, s2, x_filter_0_7, x_filter_8_11, shift_s32, zero_s32);
- d1 = highbd_convolve12_horiz8_s32_s16(
- s3, s4, s5, x_filter_0_7, x_filter_8_11, shift_s32, zero_s32);
- d2 = highbd_convolve12_horiz8_s32_s16(
- s6, s7, s8, x_filter_0_7, x_filter_8_11, shift_s32, zero_s32);
- d3 = highbd_convolve12_horiz8_s32_s16(
- s9, s10, s11, x_filter_0_7, x_filter_8_11, shift_s32, zero_s32);
-
- d0 = vqrshlq_u16(d0, bits_s16);
- d1 = vqrshlq_u16(d1, bits_s16);
- d2 = vqrshlq_u16(d2, bits_s16);
- d3 = vqrshlq_u16(d3, bits_s16);
+ uint16x8_t d0 =
+ highbd_convolve12_8_x(s0, x_filter_0_7, x_filter_8_11, offset);
+ uint16x8_t d1 =
+ highbd_convolve12_8_x(s1, x_filter_0_7, x_filter_8_11, offset);
+ uint16x8_t d2 =
+ highbd_convolve12_8_x(s2, x_filter_0_7, x_filter_8_11, offset);
+ uint16x8_t d3 =
+ highbd_convolve12_8_x(s3, x_filter_0_7, x_filter_8_11, offset);
d0 = vminq_u16(d0, max);
d1 = vminq_u16(d1, max);
d2 = vminq_u16(d2, max);
d3 = vminq_u16(d3, max);
- if (h == 2) {
- store_u16_8x2(d, dst_stride, d0, d1);
- } else {
- store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
- }
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
- s0 = s1;
- s1 = s2;
- s3 = s4;
- s4 = s5;
- s6 = s7;
- s7 = s8;
- s9 = s10;
- s10 = s11;
s += 8;
d += 8;
width -= 8;
- } while (width > 0);
+ } while (width != 0);
src_ptr += 4 * src_stride;
dst_ptr += 4 * dst_stride;
height -= 4;
- } while (height > 0);
+ } while (height != 0);
}
}
@@ -622,6 +873,11 @@
const InterpFilterParams *filter_params_x,
const int subpel_x_qn,
ConvolveParams *conv_params, int bd) {
+ if (w == 2 || h == 2) {
+ av1_highbd_convolve_x_sr_c(src, src_stride, dst, dst_stride, w, h,
+ filter_params_x, subpel_x_qn, conv_params, bd);
+ return;
+ }
const int x_filter_taps = get_filter_tap(filter_params_x, subpel_x_qn);
const int horiz_offset = filter_params_x->taps / 2 - 1;
const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
@@ -634,70 +890,250 @@
x_filter_ptr, conv_params, bd);
return;
}
+ if (x_filter_taps <= 6 && w != 4) {
+ highbd_convolve_x_sr_6tap_neon(src + 1, src_stride, dst, dst_stride, w, h,
+ x_filter_ptr, conv_params, bd);
+ return;
+ }
- highbd_convolve_x_sr_8tap_neon(src, src_stride, dst, dst_stride, w, h,
- x_filter_ptr, conv_params, bd);
+ highbd_convolve_x_sr_neon(src, src_stride, dst, dst_stride, w, h,
+ x_filter_ptr, conv_params, bd);
}
-static INLINE void highbd_convolve_2d_y_sr_8tap_neon(
+static INLINE uint16x4_t highbd_convolve6_4_2d_v(
+ const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
+ const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
+ const int16x8_t y_filter, const int32x4_t round_shift,
+ const int32x4_t offset) {
+ // Values at indices 0 and 7 of y_filter are zero.
+ const int16x4_t y_filter_0_3 = vget_low_s16(y_filter);
+ const int16x4_t y_filter_4_7 = vget_high_s16(y_filter);
+
+ int32x4_t sum = vmlal_lane_s16(offset, s0, y_filter_0_3, 1);
+ sum = vmlal_lane_s16(sum, s1, y_filter_0_3, 2);
+ sum = vmlal_lane_s16(sum, s2, y_filter_0_3, 3);
+ sum = vmlal_lane_s16(sum, s3, y_filter_4_7, 0);
+ sum = vmlal_lane_s16(sum, s4, y_filter_4_7, 1);
+ sum = vmlal_lane_s16(sum, s5, y_filter_4_7, 2);
+
+ sum = vshlq_s32(sum, round_shift);
+ return vqmovun_s32(sum);
+}
+
+static INLINE uint16x8_t highbd_convolve6_8_2d_v(
+ const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
+ const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t y_filter, const int32x4_t round_shift,
+ const int32x4_t offset) {
+ // Values at indices 0 and 7 of y_filter are zero.
+ const int16x4_t y_filter_0_3 = vget_low_s16(y_filter);
+ const int16x4_t y_filter_4_7 = vget_high_s16(y_filter);
+
+ int32x4_t sum0 = vmlal_lane_s16(offset, vget_low_s16(s0), y_filter_0_3, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s1), y_filter_0_3, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s2), y_filter_0_3, 3);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s3), y_filter_4_7, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s4), y_filter_4_7, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s5), y_filter_4_7, 2);
+
+ int32x4_t sum1 = vmlal_lane_s16(offset, vget_high_s16(s0), y_filter_0_3, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s1), y_filter_0_3, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s2), y_filter_0_3, 3);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s3), y_filter_4_7, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s4), y_filter_4_7, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s5), y_filter_4_7, 2);
+
+ sum0 = vshlq_s32(sum0, round_shift);
+ sum1 = vshlq_s32(sum1, round_shift);
+
+ return vcombine_u16(vqmovun_s32(sum0), vqmovun_s32(sum1));
+}
+
+static INLINE void highbd_convolve_2d_sr_vert_6tap_neon(
const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
int w, int h, const int16_t *y_filter_ptr, ConvolveParams *conv_params,
- int bd, const int offset, const int correction) {
+ int bd, const int offset) {
const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
const int16x8_t y_filter = vld1q_s16(y_filter_ptr);
const int32x4_t offset_s32 = vdupq_n_s32(offset);
const int round1_shift = conv_params->round_1;
const int32x4_t round1_shift_s32 = vdupq_n_s32(-round1_shift);
- const int32x4_t correction_s32 = vdupq_n_s32(correction);
- if (w <= 4) {
- int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
- uint16x4_t d0, d1, d2, d3;
- uint16x8_t d01, d23;
+ if (w == 4) {
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+ int16x4_t s0, s1, s2, s3, s4;
+ load_s16_4x5(s, src_stride, &s0, &s1, &s2, &s3, &s4);
+ s += 5 * src_stride;
+ do {
+ int16x4_t s5, s6, s7, s8;
+ load_s16_4x4(s, src_stride, &s5, &s6, &s7, &s8);
+
+ uint16x4_t d0 = highbd_convolve6_4_2d_v(s0, s1, s2, s3, s4, s5, y_filter,
+ round1_shift_s32, offset_s32);
+ uint16x4_t d1 = highbd_convolve6_4_2d_v(s1, s2, s3, s4, s5, s6, y_filter,
+ round1_shift_s32, offset_s32);
+ uint16x4_t d2 = highbd_convolve6_4_2d_v(s2, s3, s4, s5, s6, s7, y_filter,
+ round1_shift_s32, offset_s32);
+ uint16x4_t d3 = highbd_convolve6_4_2d_v(s3, s4, s5, s6, s7, s8, y_filter,
+ round1_shift_s32, offset_s32);
+
+ d0 = vmin_u16(d0, vget_low_u16(max));
+ d1 = vmin_u16(d1, vget_low_u16(max));
+ d2 = vmin_u16(d2, vget_low_u16(max));
+ d3 = vmin_u16(d3, vget_low_u16(max));
+
+ store_u16_4x4(d, dst_stride, d0, d1, d2, d3);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ h -= 4;
+ } while (h != 0);
+ } else {
+ do {
+ int height = h;
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+ int16x8_t s0, s1, s2, s3, s4;
+ load_s16_8x5(s, src_stride, &s0, &s1, &s2, &s3, &s4);
+ s += 5 * src_stride;
+
+ do {
+ int16x8_t s5, s6, s7, s8;
+ load_s16_8x4(s, src_stride, &s5, &s6, &s7, &s8);
+
+ uint16x8_t d0 = highbd_convolve6_8_2d_v(
+ s0, s1, s2, s3, s4, s5, y_filter, round1_shift_s32, offset_s32);
+ uint16x8_t d1 = highbd_convolve6_8_2d_v(
+ s1, s2, s3, s4, s5, s6, y_filter, round1_shift_s32, offset_s32);
+ uint16x8_t d2 = highbd_convolve6_8_2d_v(
+ s2, s3, s4, s5, s6, s7, y_filter, round1_shift_s32, offset_s32);
+ uint16x8_t d3 = highbd_convolve6_8_2d_v(
+ s3, s4, s5, s6, s7, s8, y_filter, round1_shift_s32, offset_s32);
+
+ d0 = vminq_u16(d0, max);
+ d1 = vminq_u16(d1, max);
+ d2 = vminq_u16(d2, max);
+ d3 = vminq_u16(d3, max);
+
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ height -= 4;
+ } while (height != 0);
+ src_ptr += 8;
+ dst_ptr += 8;
+ w -= 8;
+ } while (w != 0);
+ }
+}
+
+static INLINE uint16x4_t highbd_convolve8_4_2d_v(
+ const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
+ const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
+ const int16x4_t s6, const int16x4_t s7, const int16x8_t y_filter,
+ const int32x4_t round_shift, const int32x4_t offset) {
+ const int16x4_t y_filter_lo = vget_low_s16(y_filter);
+ const int16x4_t y_filter_hi = vget_high_s16(y_filter);
+
+ int32x4_t sum = vmlal_lane_s16(offset, s0, y_filter_lo, 0);
+ sum = vmlal_lane_s16(sum, s1, y_filter_lo, 1);
+ sum = vmlal_lane_s16(sum, s2, y_filter_lo, 2);
+ sum = vmlal_lane_s16(sum, s3, y_filter_lo, 3);
+ sum = vmlal_lane_s16(sum, s4, y_filter_hi, 0);
+ sum = vmlal_lane_s16(sum, s5, y_filter_hi, 1);
+ sum = vmlal_lane_s16(sum, s6, y_filter_hi, 2);
+ sum = vmlal_lane_s16(sum, s7, y_filter_hi, 3);
+
+ sum = vshlq_s32(sum, round_shift);
+ return vqmovun_s32(sum);
+}
+
+static INLINE uint16x8_t highbd_convolve8_8_2d_v(
+ const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
+ const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t s6, const int16x8_t s7, const int16x8_t y_filter,
+ const int32x4_t round_shift, const int32x4_t offset) {
+ const int16x4_t y_filter_lo = vget_low_s16(y_filter);
+ const int16x4_t y_filter_hi = vget_high_s16(y_filter);
+
+ int32x4_t sum0 = vmlal_lane_s16(offset, vget_low_s16(s0), y_filter_lo, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s1), y_filter_lo, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s2), y_filter_lo, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s3), y_filter_lo, 3);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s4), y_filter_hi, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s5), y_filter_hi, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s6), y_filter_hi, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s7), y_filter_hi, 3);
+
+ int32x4_t sum1 = vmlal_lane_s16(offset, vget_high_s16(s0), y_filter_lo, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s1), y_filter_lo, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s2), y_filter_lo, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s3), y_filter_lo, 3);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s4), y_filter_hi, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s5), y_filter_hi, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s6), y_filter_hi, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s7), y_filter_hi, 3);
+
+ sum0 = vshlq_s32(sum0, round_shift);
+ sum1 = vshlq_s32(sum1, round_shift);
+
+ return vcombine_u16(vqmovun_s32(sum0), vqmovun_s32(sum1));
+}
+
+static INLINE void highbd_convolve_2d_sr_vert_8tap_neon(
+ const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
+ int w, int h, const int16_t *y_filter_ptr, ConvolveParams *conv_params,
+ int bd, const int offset) {
+ const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
+ const int16x8_t y_filter = vld1q_s16(y_filter_ptr);
+ const int32x4_t offset_s32 = vdupq_n_s32(offset);
+ const int round1_shift = conv_params->round_1;
+ const int32x4_t round1_shift_s32 = vdupq_n_s32(-round1_shift);
+
+ if (w == 4) {
const int16_t *s = (const int16_t *)src_ptr;
uint16_t *d = dst_ptr;
+ int16x4_t s0, s1, s2, s3, s4, s5, s6;
load_s16_4x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
s += 7 * src_stride;
do {
+ int16x4_t s7, s8, s9, s10;
load_s16_4x4(s, src_stride, &s7, &s8, &s9, &s10);
- d0 = highbd_convolve8_4_sr_s32_s16(s0, s1, s2, s3, s4, s5, s6, s7,
- y_filter, round1_shift_s32, offset_s32,
- correction_s32);
- d1 = highbd_convolve8_4_sr_s32_s16(s1, s2, s3, s4, s5, s6, s7, s8,
- y_filter, round1_shift_s32, offset_s32,
- correction_s32);
- d2 = highbd_convolve8_4_sr_s32_s16(s2, s3, s4, s5, s6, s7, s8, s9,
- y_filter, round1_shift_s32, offset_s32,
- correction_s32);
- d3 = highbd_convolve8_4_sr_s32_s16(s3, s4, s5, s6, s7, s8, s9, s10,
- y_filter, round1_shift_s32, offset_s32,
- correction_s32);
+ uint16x4_t d0 =
+ highbd_convolve8_4_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
+ round1_shift_s32, offset_s32);
+ uint16x4_t d1 =
+ highbd_convolve8_4_2d_v(s1, s2, s3, s4, s5, s6, s7, s8, y_filter,
+ round1_shift_s32, offset_s32);
+ uint16x4_t d2 =
+ highbd_convolve8_4_2d_v(s2, s3, s4, s5, s6, s7, s8, s9, y_filter,
+ round1_shift_s32, offset_s32);
+ uint16x4_t d3 =
+ highbd_convolve8_4_2d_v(s3, s4, s5, s6, s7, s8, s9, s10, y_filter,
+ round1_shift_s32, offset_s32);
- d01 = vcombine_u16(d0, d1);
- d23 = vcombine_u16(d2, d3);
+ d0 = vmin_u16(d0, vget_low_u16(max));
+ d1 = vmin_u16(d1, vget_low_u16(max));
+ d2 = vmin_u16(d2, vget_low_u16(max));
+ d3 = vmin_u16(d3, vget_low_u16(max));
- d01 = vminq_u16(d01, max);
- d23 = vminq_u16(d23, max);
-
- if (w == 2) {
- store_u16q_2x1(d + 0 * dst_stride, d01, 0);
- store_u16q_2x1(d + 1 * dst_stride, d01, 2);
- if (h != 2) {
- store_u16q_2x1(d + 2 * dst_stride, d23, 0);
- store_u16q_2x1(d + 3 * dst_stride, d23, 2);
- }
- } else {
- vst1_u16(d + 0 * dst_stride, vget_low_u16(d01));
- vst1_u16(d + 1 * dst_stride, vget_high_u16(d01));
- if (h != 2) {
- vst1_u16(d + 2 * dst_stride, vget_low_u16(d23));
- vst1_u16(d + 3 * dst_stride, vget_high_u16(d23));
- }
- }
+ store_u16_4x4(d, dst_stride, d0, d1, d2, d3);
s0 = s4;
s1 = s5;
@@ -709,44 +1145,40 @@
s += 4 * src_stride;
d += 4 * dst_stride;
h -= 4;
- } while (h > 0);
+ } while (h != 0);
} else {
- int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
- uint16x8_t d0, d1, d2, d3;
do {
int height = h;
const int16_t *s = (const int16_t *)src_ptr;
uint16_t *d = dst_ptr;
+ int16x8_t s0, s1, s2, s3, s4, s5, s6;
load_s16_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
s += 7 * src_stride;
do {
+ int16x8_t s7, s8, s9, s10;
load_s16_8x4(s, src_stride, &s7, &s8, &s9, &s10);
- d0 = highbd_convolve8_8_sr_s32_s16(s0, s1, s2, s3, s4, s5, s6, s7,
- y_filter, round1_shift_s32,
- offset_s32, correction_s32);
- d1 = highbd_convolve8_8_sr_s32_s16(s1, s2, s3, s4, s5, s6, s7, s8,
- y_filter, round1_shift_s32,
- offset_s32, correction_s32);
- d2 = highbd_convolve8_8_sr_s32_s16(s2, s3, s4, s5, s6, s7, s8, s9,
- y_filter, round1_shift_s32,
- offset_s32, correction_s32);
- d3 = highbd_convolve8_8_sr_s32_s16(s3, s4, s5, s6, s7, s8, s9, s10,
- y_filter, round1_shift_s32,
- offset_s32, correction_s32);
+ uint16x8_t d0 =
+ highbd_convolve8_8_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
+ round1_shift_s32, offset_s32);
+ uint16x8_t d1 =
+ highbd_convolve8_8_2d_v(s1, s2, s3, s4, s5, s6, s7, s8, y_filter,
+ round1_shift_s32, offset_s32);
+ uint16x8_t d2 =
+ highbd_convolve8_8_2d_v(s2, s3, s4, s5, s6, s7, s8, s9, y_filter,
+ round1_shift_s32, offset_s32);
+ uint16x8_t d3 =
+ highbd_convolve8_8_2d_v(s3, s4, s5, s6, s7, s8, s9, s10, y_filter,
+ round1_shift_s32, offset_s32);
d0 = vminq_u16(d0, max);
d1 = vminq_u16(d1, max);
d2 = vminq_u16(d2, max);
d3 = vminq_u16(d3, max);
- if (h == 2) {
- store_u16_8x2(d, dst_stride, d0, d1);
- } else {
- store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
- }
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
s0 = s4;
s1 = s5;
@@ -758,75 +1190,126 @@
s += 4 * src_stride;
d += 4 * dst_stride;
height -= 4;
- } while (height > 0);
+ } while (height != 0);
src_ptr += 8;
dst_ptr += 8;
w -= 8;
- } while (w > 0);
+ } while (w != 0);
}
}
-static INLINE void highbd_convolve_2d_y_sr_12tap_neon(
+static INLINE uint16x4_t highbd_convolve12_4_2d_v(
+ const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
+ const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
+ const int16x4_t s6, const int16x4_t s7, const int16x4_t s8,
+ const int16x4_t s9, const int16x4_t s10, const int16x4_t s11,
+ const int16x8_t y_filter_0_7, const int16x4_t y_filter_8_11,
+ const int32x4_t round_shift, const int32x4_t offset) {
+ const int16x4_t y_filter_0_3 = vget_low_s16(y_filter_0_7);
+ const int16x4_t y_filter_4_7 = vget_high_s16(y_filter_0_7);
+
+ int32x4_t sum = vmlal_lane_s16(offset, s0, y_filter_0_3, 0);
+ sum = vmlal_lane_s16(sum, s1, y_filter_0_3, 1);
+ sum = vmlal_lane_s16(sum, s2, y_filter_0_3, 2);
+ sum = vmlal_lane_s16(sum, s3, y_filter_0_3, 3);
+ sum = vmlal_lane_s16(sum, s4, y_filter_4_7, 0);
+ sum = vmlal_lane_s16(sum, s5, y_filter_4_7, 1);
+ sum = vmlal_lane_s16(sum, s6, y_filter_4_7, 2);
+ sum = vmlal_lane_s16(sum, s7, y_filter_4_7, 3);
+ sum = vmlal_lane_s16(sum, s8, y_filter_8_11, 0);
+ sum = vmlal_lane_s16(sum, s9, y_filter_8_11, 1);
+ sum = vmlal_lane_s16(sum, s10, y_filter_8_11, 2);
+ sum = vmlal_lane_s16(sum, s11, y_filter_8_11, 3);
+
+ sum = vshlq_s32(sum, round_shift);
+ return vqmovun_s32(sum);
+}
+
+static INLINE uint16x8_t highbd_convolve12_8_2d_v(
+ const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
+ const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t s6, const int16x8_t s7, const int16x8_t s8,
+ const int16x8_t s9, const int16x8_t s10, const int16x8_t s11,
+ const int16x8_t y_filter_0_7, const int16x4_t y_filter_8_11,
+ const int32x4_t round_shift, const int32x4_t offset) {
+ const int16x4_t y_filter_0_3 = vget_low_s16(y_filter_0_7);
+ const int16x4_t y_filter_4_7 = vget_high_s16(y_filter_0_7);
+
+ int32x4_t sum0 = vmlal_lane_s16(offset, vget_low_s16(s0), y_filter_0_3, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s1), y_filter_0_3, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s2), y_filter_0_3, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s3), y_filter_0_3, 3);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s4), y_filter_4_7, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s5), y_filter_4_7, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s6), y_filter_4_7, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s7), y_filter_4_7, 3);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s8), y_filter_8_11, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s9), y_filter_8_11, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s10), y_filter_8_11, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s11), y_filter_8_11, 3);
+
+ int32x4_t sum1 = vmlal_lane_s16(offset, vget_high_s16(s0), y_filter_0_3, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s1), y_filter_0_3, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s2), y_filter_0_3, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s3), y_filter_0_3, 3);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s4), y_filter_4_7, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s5), y_filter_4_7, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s6), y_filter_4_7, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s7), y_filter_4_7, 3);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s8), y_filter_8_11, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s9), y_filter_8_11, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s10), y_filter_8_11, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s11), y_filter_8_11, 3);
+
+ sum0 = vshlq_s32(sum0, round_shift);
+ sum1 = vshlq_s32(sum1, round_shift);
+
+ return vcombine_u16(vqmovun_s32(sum0), vqmovun_s32(sum1));
+}
+
+static INLINE void highbd_convolve_2d_sr_vert_12tap_neon(
const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
int w, int h, const int16_t *y_filter_ptr, ConvolveParams *conv_params,
- const int bd, const int offset, const int correction) {
+ const int bd, const int offset) {
const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
const int16x8_t y_filter_0_7 = vld1q_s16(y_filter_ptr);
const int16x4_t y_filter_8_11 = vld1_s16(y_filter_ptr + 8);
const int32x4_t offset_s32 = vdupq_n_s32(offset);
const int round1_shift = conv_params->round_1;
const int32x4_t round1_shift_s32 = vdupq_n_s32(-round1_shift);
- const int32x4_t correction_s32 = vdupq_n_s32(correction);
- if (w <= 4) {
- int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14;
- uint16x4_t d0, d1, d2, d3;
- uint16x8_t d01, d23;
-
+ if (w == 4) {
const int16_t *s = (const int16_t *)src_ptr;
uint16_t *d = dst_ptr;
+ int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
load_s16_4x11(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7, &s8,
&s9, &s10);
s += 11 * src_stride;
do {
+ int16x4_t s11, s12, s13, s14;
load_s16_4x4(s, src_stride, &s11, &s12, &s13, &s14);
- d0 = highbd_convolve12_y_4_sr_s32_s16(
+ uint16x4_t d0 = highbd_convolve12_4_2d_v(
s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, y_filter_0_7,
- y_filter_8_11, round1_shift_s32, offset_s32, correction_s32);
- d1 = highbd_convolve12_y_4_sr_s32_s16(
+ y_filter_8_11, round1_shift_s32, offset_s32);
+ uint16x4_t d1 = highbd_convolve12_4_2d_v(
s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, y_filter_0_7,
- y_filter_8_11, round1_shift_s32, offset_s32, correction_s32);
- d2 = highbd_convolve12_y_4_sr_s32_s16(
+ y_filter_8_11, round1_shift_s32, offset_s32);
+ uint16x4_t d2 = highbd_convolve12_4_2d_v(
s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, y_filter_0_7,
- y_filter_8_11, round1_shift_s32, offset_s32, correction_s32);
- d3 = highbd_convolve12_y_4_sr_s32_s16(
+ y_filter_8_11, round1_shift_s32, offset_s32);
+ uint16x4_t d3 = highbd_convolve12_4_2d_v(
s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14, y_filter_0_7,
- y_filter_8_11, round1_shift_s32, offset_s32, correction_s32);
+ y_filter_8_11, round1_shift_s32, offset_s32);
- d01 = vcombine_u16(d0, d1);
- d23 = vcombine_u16(d2, d3);
+ d0 = vmin_u16(d0, vget_low_u16(max));
+ d1 = vmin_u16(d1, vget_low_u16(max));
+ d2 = vmin_u16(d2, vget_low_u16(max));
+ d3 = vmin_u16(d3, vget_low_u16(max));
- d01 = vminq_u16(d01, max);
- d23 = vminq_u16(d23, max);
-
- if (w == 2) {
- store_u16q_2x1(d + 0 * dst_stride, d01, 0);
- store_u16q_2x1(d + 1 * dst_stride, d01, 2);
- if (h != 2) {
- store_u16q_2x1(d + 2 * dst_stride, d23, 0);
- store_u16q_2x1(d + 3 * dst_stride, d23, 2);
- }
- } else {
- vst1_u16(d + 0 * dst_stride, vget_low_u16(d01));
- vst1_u16(d + 1 * dst_stride, vget_high_u16(d01));
- if (h != 2) {
- vst1_u16(d + 2 * dst_stride, vget_low_u16(d23));
- vst1_u16(d + 3 * dst_stride, vget_high_u16(d23));
- }
- }
+ store_u16_4x4(d, dst_stride, d0, d1, d2, d3);
s0 = s4;
s1 = s5;
@@ -842,46 +1325,41 @@
s += 4 * src_stride;
d += 4 * dst_stride;
h -= 4;
- } while (h > 0);
+ } while (h != 0);
} else {
- uint16x8_t d0, d1, d2, d3;
- int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14;
-
do {
int height = h;
const int16_t *s = (const int16_t *)src_ptr;
uint16_t *d = dst_ptr;
+ int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
load_s16_8x11(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7, &s8,
&s9, &s10);
s += 11 * src_stride;
do {
+ int16x8_t s11, s12, s13, s14;
load_s16_8x4(s, src_stride, &s11, &s12, &s13, &s14);
- d0 = highbd_convolve12_y_8_sr_s32_s16(
+ uint16x8_t d0 = highbd_convolve12_8_2d_v(
s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, y_filter_0_7,
- y_filter_8_11, round1_shift_s32, offset_s32, correction_s32);
- d1 = highbd_convolve12_y_8_sr_s32_s16(
+ y_filter_8_11, round1_shift_s32, offset_s32);
+ uint16x8_t d1 = highbd_convolve12_8_2d_v(
s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, y_filter_0_7,
- y_filter_8_11, round1_shift_s32, offset_s32, correction_s32);
- d2 = highbd_convolve12_y_8_sr_s32_s16(
+ y_filter_8_11, round1_shift_s32, offset_s32);
+ uint16x8_t d2 = highbd_convolve12_8_2d_v(
s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, y_filter_0_7,
- y_filter_8_11, round1_shift_s32, offset_s32, correction_s32);
- d3 = highbd_convolve12_y_8_sr_s32_s16(
+ y_filter_8_11, round1_shift_s32, offset_s32);
+ uint16x8_t d3 = highbd_convolve12_8_2d_v(
s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14, y_filter_0_7,
- y_filter_8_11, round1_shift_s32, offset_s32, correction_s32);
+ y_filter_8_11, round1_shift_s32, offset_s32);
d0 = vminq_u16(d0, max);
d1 = vminq_u16(d1, max);
d2 = vminq_u16(d2, max);
d3 = vminq_u16(d3, max);
- if (h == 2) {
- store_u16_8x2(d, dst_stride, d0, d1);
- } else {
- store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
- }
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
s0 = s4;
s1 = s5;
@@ -897,186 +1375,465 @@
s += 4 * src_stride;
d += 4 * dst_stride;
height -= 4;
- } while (height > 0);
+ } while (height != 0);
src_ptr += 8;
dst_ptr += 8;
w -= 8;
- } while (w > 0);
+ } while (w != 0);
}
}
-static INLINE void highbd_convolve_x_8tap_neon(
+static INLINE uint16x8_t highbd_convolve6_8_2d_h(const int16x8_t s[6],
+ const int16x8_t x_filter,
+ const int32x4_t shift_s32,
+ const int32x4_t offset) {
+ // Values at indices 0 and 7 of y_filter are zero.
+ const int16x4_t x_filter_0_3 = vget_low_s16(x_filter);
+ const int16x4_t x_filter_4_7 = vget_high_s16(x_filter);
+
+ int32x4_t sum0 = vmlal_lane_s16(offset, vget_low_s16(s[0]), x_filter_0_3, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[1]), x_filter_0_3, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[2]), x_filter_0_3, 3);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[3]), x_filter_4_7, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[4]), x_filter_4_7, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[5]), x_filter_4_7, 2);
+
+ int32x4_t sum1 = vmlal_lane_s16(offset, vget_high_s16(s[0]), x_filter_0_3, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[1]), x_filter_0_3, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[2]), x_filter_0_3, 3);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[3]), x_filter_4_7, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[4]), x_filter_4_7, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[5]), x_filter_4_7, 2);
+
+ sum0 = vqrshlq_s32(sum0, shift_s32);
+ sum1 = vqrshlq_s32(sum1, shift_s32);
+
+ return vcombine_u16(vqmovun_s32(sum0), vqmovun_s32(sum1));
+}
+
+static INLINE void highbd_convolve_2d_sr_horiz_6tap_neon(
const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
int w, int h, const int16_t *x_filter_ptr, ConvolveParams *conv_params,
const int offset) {
- const int16x8_t x_filter = vld1q_s16(x_filter_ptr);
+ // The smallest block height processed by the SIMD functions is 4, and the
+ // horizontal convolution needs to process an extra (filter_taps/2 - 1) lines
+ // for the vertical convolution.
+ assert(h >= 5);
const int32x4_t shift_s32 = vdupq_n_s32(-conv_params->round_0);
const int32x4_t offset_s32 = vdupq_n_s32(offset);
- if (w <= 4) {
- int16x8_t s0, s1, s2, s3;
- uint16x4_t d0, d1;
- uint16x8_t d01;
+ const int16x8_t x_filter = vld1q_s16(x_filter_ptr);
+ int height = h;
+ do {
+ int width = w;
const int16_t *s = (const int16_t *)src_ptr;
uint16_t *d = dst_ptr;
do {
- load_s16_8x2(s, src_stride, &s0, &s2);
- load_s16_8x2(s + 8, src_stride, &s1, &s3);
+ int16x8_t s0[6], s1[6], s2[6], s3[6];
+ load_s16_8x6(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3],
+ &s0[4], &s0[5]);
+ load_s16_8x6(s + 1 * src_stride, 1, &s1[0], &s1[1], &s1[2], &s1[3],
+ &s1[4], &s1[5]);
+ load_s16_8x6(s + 2 * src_stride, 1, &s2[0], &s2[1], &s2[2], &s2[3],
+ &s2[4], &s2[5]);
+ load_s16_8x6(s + 3 * src_stride, 1, &s3[0], &s3[1], &s3[2], &s3[3],
+ &s3[4], &s3[5]);
- d0 = highbd_convolve8_horiz4_s32_s16(s0, s1, x_filter, shift_s32,
- offset_s32);
- d1 = highbd_convolve8_horiz4_s32_s16(s2, s3, x_filter, shift_s32,
- offset_s32);
+ uint16x8_t d0 =
+ highbd_convolve6_8_2d_h(s0, x_filter, shift_s32, offset_s32);
+ uint16x8_t d1 =
+ highbd_convolve6_8_2d_h(s1, x_filter, shift_s32, offset_s32);
+ uint16x8_t d2 =
+ highbd_convolve6_8_2d_h(s2, x_filter, shift_s32, offset_s32);
+ uint16x8_t d3 =
+ highbd_convolve6_8_2d_h(s3, x_filter, shift_s32, offset_s32);
- d01 = vcombine_u16(d0, d1);
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
- if (w == 2) {
- store_u16q_2x1(d + 0 * dst_stride, d01, 0);
- store_u16q_2x1(d + 1 * dst_stride, d01, 2);
- } else {
- vst1_u16(d + 0 * dst_stride, vget_low_u16(d01));
- vst1_u16(d + 1 * dst_stride, vget_high_u16(d01));
- }
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ height -= 4;
+ } while (height > 4);
+ do {
+ int width = w;
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
- s += 2 * src_stride;
- d += 2 * dst_stride;
- h -= 2;
- } while (h > 0);
+ do {
+ int16x8_t s0[6];
+ load_s16_8x6(s, 1, &s0[0], &s0[1], &s0[2], &s0[3], &s0[4], &s0[5]);
+
+ uint16x8_t d0 =
+ highbd_convolve6_8_2d_h(s0, x_filter, shift_s32, offset_s32);
+ vst1q_u16(d, d0);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ } while (--height != 0);
+}
+
+static INLINE uint16x4_t highbd_convolve4_4_2d_h(const int16x4_t s[4],
+ const int16x4_t x_filter,
+ const int32x4_t shift_s32,
+ const int32x4_t offset) {
+ int32x4_t sum = vmlal_lane_s16(offset, s[0], x_filter, 0);
+ sum = vmlal_lane_s16(sum, s[1], x_filter, 1);
+ sum = vmlal_lane_s16(sum, s[2], x_filter, 2);
+ sum = vmlal_lane_s16(sum, s[3], x_filter, 3);
+
+ sum = vqrshlq_s32(sum, shift_s32);
+ return vqmovun_s32(sum);
+}
+
+static INLINE uint16x8_t highbd_convolve8_8_2d_h(const int16x8_t s[8],
+ const int16x8_t x_filter,
+ const int32x4_t shift_s32,
+ const int32x4_t offset) {
+ const int16x4_t x_filter_0_3 = vget_low_s16(x_filter);
+ const int16x4_t x_filter_4_7 = vget_high_s16(x_filter);
+
+ int32x4_t sum0 = vmlal_lane_s16(offset, vget_low_s16(s[0]), x_filter_0_3, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[1]), x_filter_0_3, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[2]), x_filter_0_3, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[3]), x_filter_0_3, 3);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[4]), x_filter_4_7, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[5]), x_filter_4_7, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[6]), x_filter_4_7, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[7]), x_filter_4_7, 3);
+
+ int32x4_t sum1 = vmlal_lane_s16(offset, vget_high_s16(s[0]), x_filter_0_3, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[1]), x_filter_0_3, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[2]), x_filter_0_3, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[3]), x_filter_0_3, 3);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[4]), x_filter_4_7, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[5]), x_filter_4_7, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[6]), x_filter_4_7, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[7]), x_filter_4_7, 3);
+
+ sum0 = vqrshlq_s32(sum0, shift_s32);
+ sum1 = vqrshlq_s32(sum1, shift_s32);
+
+ return vcombine_u16(vqmovun_s32(sum0), vqmovun_s32(sum1));
+}
+
+static INLINE void highbd_convolve_2d_sr_horiz_neon(
+ const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
+ int w, int h, const int16_t *x_filter_ptr, ConvolveParams *conv_params,
+ const int offset) {
+ // The smallest block height processed by the SIMD functions is 4, and the
+ // horizontal convolution needs to process an extra (filter_taps/2 - 1) lines
+ // for the vertical convolution.
+ assert(h >= 5);
+ const int32x4_t shift_s32 = vdupq_n_s32(-conv_params->round_0);
+ const int32x4_t offset_s32 = vdupq_n_s32(offset);
+
+ if (w == 4) {
+ // 4-tap filters are used for blocks having width <= 4.
+ const int16x4_t x_filter = vld1_s16(x_filter_ptr + 2);
+ const int16_t *s = (const int16_t *)(src_ptr + 1);
+ uint16_t *d = dst_ptr;
+
+ do {
+ int16x4_t s0[4], s1[4], s2[4], s3[4];
+ load_s16_4x4(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3]);
+ load_s16_4x4(s + 1 * src_stride, 1, &s1[0], &s1[1], &s1[2], &s1[3]);
+ load_s16_4x4(s + 2 * src_stride, 1, &s2[0], &s2[1], &s2[2], &s2[3]);
+ load_s16_4x4(s + 3 * src_stride, 1, &s3[0], &s3[1], &s3[2], &s3[3]);
+
+ uint16x4_t d0 =
+ highbd_convolve4_4_2d_h(s0, x_filter, shift_s32, offset_s32);
+ uint16x4_t d1 =
+ highbd_convolve4_4_2d_h(s1, x_filter, shift_s32, offset_s32);
+ uint16x4_t d2 =
+ highbd_convolve4_4_2d_h(s2, x_filter, shift_s32, offset_s32);
+ uint16x4_t d3 =
+ highbd_convolve4_4_2d_h(s3, x_filter, shift_s32, offset_s32);
+
+ store_u16_4x4(d, dst_stride, d0, d1, d2, d3);
+
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ h -= 4;
+ } while (h > 4);
+
+ do {
+ int16x4_t s0[4];
+ load_s16_4x4(s, 1, &s0[0], &s0[1], &s0[2], &s0[3]);
+
+ uint16x4_t d0 =
+ highbd_convolve4_4_2d_h(s0, x_filter, shift_s32, offset_s32);
+
+ vst1_u16(d, d0);
+
+ s += src_stride;
+ d += dst_stride;
+ } while (--h != 0);
} else {
+ const int16x8_t x_filter = vld1q_s16(x_filter_ptr);
int height = h;
- int16x8_t s0, s1, s2, s3, s4, s5, s6, s7;
- uint16x8_t d0, d1, d2, d3;
+
do {
int width = w;
const int16_t *s = (const int16_t *)src_ptr;
uint16_t *d = dst_ptr;
- load_s16_8x4(s, src_stride, &s0, &s2, &s4, &s6);
- s += 8;
-
do {
- load_s16_8x4(s, src_stride, &s1, &s3, &s5, &s7);
+ int16x8_t s0[8], s1[8], s2[8], s3[8];
+ load_s16_8x8(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3],
+ &s0[4], &s0[5], &s0[6], &s0[7]);
+ load_s16_8x8(s + 1 * src_stride, 1, &s1[0], &s1[1], &s1[2], &s1[3],
+ &s1[4], &s1[5], &s1[6], &s1[7]);
+ load_s16_8x8(s + 2 * src_stride, 1, &s2[0], &s2[1], &s2[2], &s2[3],
+ &s2[4], &s2[5], &s2[6], &s2[7]);
+ load_s16_8x8(s + 3 * src_stride, 1, &s3[0], &s3[1], &s3[2], &s3[3],
+ &s3[4], &s3[5], &s3[6], &s3[7]);
- d0 = highbd_convolve8_horiz8_s32_s16(s0, s1, x_filter, shift_s32,
- offset_s32);
- d1 = highbd_convolve8_horiz8_s32_s16(s2, s3, x_filter, shift_s32,
- offset_s32);
- d2 = highbd_convolve8_horiz8_s32_s16(s4, s5, x_filter, shift_s32,
- offset_s32);
- d3 = highbd_convolve8_horiz8_s32_s16(s6, s7, x_filter, shift_s32,
- offset_s32);
+ uint16x8_t d0 =
+ highbd_convolve8_8_2d_h(s0, x_filter, shift_s32, offset_s32);
+ uint16x8_t d1 =
+ highbd_convolve8_8_2d_h(s1, x_filter, shift_s32, offset_s32);
+ uint16x8_t d2 =
+ highbd_convolve8_8_2d_h(s2, x_filter, shift_s32, offset_s32);
+ uint16x8_t d3 =
+ highbd_convolve8_8_2d_h(s3, x_filter, shift_s32, offset_s32);
- if (h == 2) {
- store_u16_8x2(d, dst_stride, d0, d1);
- } else {
- store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
- }
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
- s0 = s1;
- s2 = s3;
- s4 = s5;
- s6 = s7;
s += 8;
d += 8;
width -= 8;
- } while (width > 0);
+ } while (width != 0);
src_ptr += 4 * src_stride;
dst_ptr += 4 * dst_stride;
height -= 4;
- } while (height > 0);
+ } while (height > 4);
+
+ do {
+ int width = w;
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ do {
+ int16x8_t s0[8];
+ load_s16_8x8(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3],
+ &s0[4], &s0[5], &s0[6], &s0[7]);
+
+ uint16x8_t d0 =
+ highbd_convolve8_8_2d_h(s0, x_filter, shift_s32, offset_s32);
+ vst1q_u16(d, d0);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ } while (--height != 0);
}
}
-static INLINE void highbd_convolve_2d_x_sr_12tap_neon(
+static INLINE uint16x4_t highbd_convolve12_4_2d_h(const int16x4_t s[12],
+ const int16x8_t x_filter_0_7,
+ const int16x4_t x_filter_8_11,
+ const int32x4_t shift_s32,
+ const int32x4_t offset) {
+ const int16x4_t x_filter_0_3 = vget_low_s16(x_filter_0_7);
+ const int16x4_t x_filter_4_7 = vget_high_s16(x_filter_0_7);
+
+ int32x4_t sum = vmlal_lane_s16(offset, s[0], x_filter_0_3, 0);
+ sum = vmlal_lane_s16(sum, s[1], x_filter_0_3, 1);
+ sum = vmlal_lane_s16(sum, s[2], x_filter_0_3, 2);
+ sum = vmlal_lane_s16(sum, s[3], x_filter_0_3, 3);
+ sum = vmlal_lane_s16(sum, s[4], x_filter_4_7, 0);
+ sum = vmlal_lane_s16(sum, s[5], x_filter_4_7, 1);
+ sum = vmlal_lane_s16(sum, s[6], x_filter_4_7, 2);
+ sum = vmlal_lane_s16(sum, s[7], x_filter_4_7, 3);
+ sum = vmlal_lane_s16(sum, s[8], x_filter_8_11, 0);
+ sum = vmlal_lane_s16(sum, s[9], x_filter_8_11, 1);
+ sum = vmlal_lane_s16(sum, s[10], x_filter_8_11, 2);
+ sum = vmlal_lane_s16(sum, s[11], x_filter_8_11, 3);
+
+ sum = vqrshlq_s32(sum, shift_s32);
+ return vqmovun_s32(sum);
+}
+
+static INLINE uint16x8_t highbd_convolve12_8_2d_h(const int16x8_t s[12],
+ const int16x8_t x_filter_0_7,
+ const int16x4_t x_filter_8_11,
+ const int32x4_t shift_s32,
+ const int32x4_t offset) {
+ const int16x4_t x_filter_0_3 = vget_low_s16(x_filter_0_7);
+ const int16x4_t x_filter_4_7 = vget_high_s16(x_filter_0_7);
+
+ int32x4_t sum0 = vmlal_lane_s16(offset, vget_low_s16(s[0]), x_filter_0_3, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[1]), x_filter_0_3, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[2]), x_filter_0_3, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[3]), x_filter_0_3, 3);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[4]), x_filter_4_7, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[5]), x_filter_4_7, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[6]), x_filter_4_7, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[7]), x_filter_4_7, 3);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[8]), x_filter_8_11, 0);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[9]), x_filter_8_11, 1);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[10]), x_filter_8_11, 2);
+ sum0 = vmlal_lane_s16(sum0, vget_low_s16(s[11]), x_filter_8_11, 3);
+
+ int32x4_t sum1 = vmlal_lane_s16(offset, vget_high_s16(s[0]), x_filter_0_3, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[1]), x_filter_0_3, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[2]), x_filter_0_3, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[3]), x_filter_0_3, 3);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[4]), x_filter_4_7, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[5]), x_filter_4_7, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[6]), x_filter_4_7, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[7]), x_filter_4_7, 3);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[8]), x_filter_8_11, 0);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[9]), x_filter_8_11, 1);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[10]), x_filter_8_11, 2);
+ sum1 = vmlal_lane_s16(sum1, vget_high_s16(s[11]), x_filter_8_11, 3);
+
+ sum0 = vqrshlq_s32(sum0, shift_s32);
+ sum1 = vqrshlq_s32(sum1, shift_s32);
+
+ return vcombine_u16(vqmovun_s32(sum0), vqmovun_s32(sum1));
+}
+
+static INLINE void highbd_convolve_2d_sr_horiz_12tap_neon(
const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
int w, int h, const int16_t *x_filter_ptr, ConvolveParams *conv_params,
const int offset) {
+ // The smallest block height processed by the SIMD functions is 4, and the
+ // horizontal convolution needs to process an extra (filter_taps/2 - 1) lines
+ // for the vertical convolution.
+ assert(h >= 5);
const int32x4_t shift_s32 = vdupq_n_s32(-conv_params->round_0);
const int16x8_t x_filter_0_7 = vld1q_s16(x_filter_ptr);
const int16x4_t x_filter_8_11 = vld1_s16(x_filter_ptr + 8);
const int32x4_t offset_s32 = vdupq_n_s32(offset);
- if (w <= 4) {
- int16x8_t s0, s1, s2, s3;
- uint16x4_t d0, d1;
- uint16x8_t d01;
-
+ if (w == 4) {
const int16_t *s = (const int16_t *)src_ptr;
uint16_t *d = dst_ptr;
do {
- load_s16_8x2(s, src_stride, &s0, &s2);
- load_s16_8x2(s + 8, src_stride, &s1, &s3);
+ int16x4_t s0[12], s1[12], s2[12], s3[12];
+ load_s16_4x12(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3],
+ &s0[4], &s0[5], &s0[6], &s0[7], &s0[8], &s0[9], &s0[10],
+ &s0[11]);
+ load_s16_4x12(s + 1 * src_stride, 1, &s1[0], &s1[1], &s1[2], &s1[3],
+ &s1[4], &s1[5], &s1[6], &s1[7], &s1[8], &s1[9], &s1[10],
+ &s1[11]);
+ load_s16_4x12(s + 2 * src_stride, 1, &s2[0], &s2[1], &s2[2], &s2[3],
+ &s2[4], &s2[5], &s2[6], &s2[7], &s2[8], &s2[9], &s2[10],
+ &s2[11]);
+ load_s16_4x12(s + 3 * src_stride, 1, &s3[0], &s3[1], &s3[2], &s3[3],
+ &s3[4], &s3[5], &s3[6], &s3[7], &s3[8], &s3[9], &s3[10],
+ &s3[11]);
- d0 = highbd_convolve12_horiz4_s32_s16(s0, s1, x_filter_0_7, x_filter_8_11,
- shift_s32, offset_s32);
- d1 = highbd_convolve12_horiz4_s32_s16(s2, s3, x_filter_0_7, x_filter_8_11,
- shift_s32, offset_s32);
+ uint16x4_t d0 = highbd_convolve12_4_2d_h(s0, x_filter_0_7, x_filter_8_11,
+ shift_s32, offset_s32);
+ uint16x4_t d1 = highbd_convolve12_4_2d_h(s1, x_filter_0_7, x_filter_8_11,
+ shift_s32, offset_s32);
+ uint16x4_t d2 = highbd_convolve12_4_2d_h(s2, x_filter_0_7, x_filter_8_11,
+ shift_s32, offset_s32);
+ uint16x4_t d3 = highbd_convolve12_4_2d_h(s3, x_filter_0_7, x_filter_8_11,
+ shift_s32, offset_s32);
- d01 = vcombine_u16(d0, d1);
+ store_u16_4x4(d, dst_stride, d0, d1, d2, d3);
- if (w == 2) {
- store_u16q_2x1(d + 0 * dst_stride, d01, 0);
- store_u16q_2x1(d + 1 * dst_stride, d01, 2);
- } else {
- vst1_u16(d + 0 * dst_stride, vget_low_u16(d01));
- vst1_u16(d + 1 * dst_stride, vget_high_u16(d01));
- }
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ h -= 4;
+ } while (h > 4);
- s += 2 * src_stride;
- d += 2 * dst_stride;
- h -= 2;
- } while (h > 0);
+ do {
+ int16x4_t s0[12];
+ load_s16_4x12(s, 1, &s0[0], &s0[1], &s0[2], &s0[3], &s0[4], &s0[5],
+ &s0[6], &s0[7], &s0[8], &s0[9], &s0[10], &s0[11]);
+
+ uint16x4_t d0 = highbd_convolve12_4_2d_h(s0, x_filter_0_7, x_filter_8_11,
+ shift_s32, offset_s32);
+
+ vst1_u16(d, d0);
+
+ s += src_stride;
+ d += dst_stride;
+ } while (--h != 0);
} else {
int height = h;
- int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11;
- uint16x8_t d0, d1, d2, d3;
+
do {
int width = w;
const int16_t *s = (const int16_t *)src_ptr;
uint16_t *d = dst_ptr;
- load_s16_8x4(s, src_stride, &s0, &s3, &s6, &s9);
- s += 8;
+ do {
+ int16x8_t s0[12], s1[12], s2[12], s3[12];
+ load_s16_8x12(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3],
+ &s0[4], &s0[5], &s0[6], &s0[7], &s0[8], &s0[9], &s0[10],
+ &s0[11]);
+ load_s16_8x12(s + 1 * src_stride, 1, &s1[0], &s1[1], &s1[2], &s1[3],
+ &s1[4], &s1[5], &s1[6], &s1[7], &s1[8], &s1[9], &s1[10],
+ &s1[11]);
+ load_s16_8x12(s + 2 * src_stride, 1, &s2[0], &s2[1], &s2[2], &s2[3],
+ &s2[4], &s2[5], &s2[6], &s2[7], &s2[8], &s2[9], &s2[10],
+ &s2[11]);
+ load_s16_8x12(s + 3 * src_stride, 1, &s3[0], &s3[1], &s3[2], &s3[3],
+ &s3[4], &s3[5], &s3[6], &s3[7], &s3[8], &s3[9], &s3[10],
+ &s3[11]);
+
+ uint16x8_t d0 = highbd_convolve12_8_2d_h(
+ s0, x_filter_0_7, x_filter_8_11, shift_s32, offset_s32);
+ uint16x8_t d1 = highbd_convolve12_8_2d_h(
+ s1, x_filter_0_7, x_filter_8_11, shift_s32, offset_s32);
+ uint16x8_t d2 = highbd_convolve12_8_2d_h(
+ s2, x_filter_0_7, x_filter_8_11, shift_s32, offset_s32);
+ uint16x8_t d3 = highbd_convolve12_8_2d_h(
+ s3, x_filter_0_7, x_filter_8_11, shift_s32, offset_s32);
+
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ height -= 4;
+ } while (height > 4);
+
+ do {
+ int width = w;
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
do {
- load_s16_8x4(s, src_stride, &s1, &s4, &s7, &s10);
- load_s16_8x4(s + 8, src_stride, &s2, &s5, &s8, &s11);
+ int16x8_t s0[12];
+ load_s16_8x12(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3],
+ &s0[4], &s0[5], &s0[6], &s0[7], &s0[8], &s0[9], &s0[10],
+ &s0[11]);
- d0 = highbd_convolve12_horiz8_s32_s16(
- s0, s1, s2, x_filter_0_7, x_filter_8_11, shift_s32, offset_s32);
- d1 = highbd_convolve12_horiz8_s32_s16(
- s3, s4, s5, x_filter_0_7, x_filter_8_11, shift_s32, offset_s32);
- d2 = highbd_convolve12_horiz8_s32_s16(
- s6, s7, s8, x_filter_0_7, x_filter_8_11, shift_s32, offset_s32);
- d3 = highbd_convolve12_horiz8_s32_s16(
- s9, s10, s11, x_filter_0_7, x_filter_8_11, shift_s32, offset_s32);
+ uint16x8_t d0 = highbd_convolve12_8_2d_h(
+ s0, x_filter_0_7, x_filter_8_11, shift_s32, offset_s32);
+ vst1q_u16(d, d0);
- if (h == 2) {
- store_u16_8x2(d, dst_stride, d0, d1);
- } else {
- store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
- }
-
- s0 = s1;
- s1 = s2;
- s3 = s4;
- s4 = s5;
- s6 = s7;
- s7 = s8;
- s9 = s10;
- s10 = s11;
s += 8;
d += 8;
width -= 8;
} while (width > 0);
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- height -= 4;
- } while (height > 0);
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ } while (--height != 0);
}
}
@@ -1087,18 +1844,29 @@
const int subpel_x_qn,
const int subpel_y_qn,
ConvolveParams *conv_params, int bd) {
+ if (w == 2 || h == 2) {
+ av1_highbd_convolve_2d_sr_c(src, src_stride, dst, dst_stride, w, h,
+ filter_params_x, filter_params_y, subpel_x_qn,
+ subpel_y_qn, conv_params, bd);
+ return;
+ }
DECLARE_ALIGNED(16, uint16_t,
im_block[(MAX_SB_SIZE + MAX_FILTER_TAP) * MAX_SB_SIZE]);
- const int im_h = h + filter_params_y->taps - 1;
+ const int x_filter_taps = get_filter_tap(filter_params_x, subpel_x_qn);
+ const int clamped_x_taps = x_filter_taps < 6 ? 6 : x_filter_taps;
+
+ const int y_filter_taps = get_filter_tap(filter_params_y, subpel_y_qn);
+ const int clamped_y_taps = y_filter_taps < 6 ? 6 : y_filter_taps;
+ const int im_h = h + clamped_y_taps - 1;
const int im_stride = MAX_SB_SIZE;
- const int vert_offset = filter_params_y->taps / 2 - 1;
- const int horiz_offset = filter_params_x->taps / 2 - 1;
+ const int vert_offset = clamped_y_taps / 2 - 1;
+ const int horiz_offset = clamped_x_taps / 2 - 1;
const int x_offset_initial = (1 << (bd + FILTER_BITS - 1));
const int y_offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0;
- const int y_offset_initial = (1 << y_offset_bits);
- const int y_offset_correction =
- ((1 << (y_offset_bits - conv_params->round_1)) +
- (1 << (y_offset_bits - conv_params->round_1 - 1)));
+ // The extra shim of (1 << (conv_params->round_1 - 1)) allows us to do a
+ // simple shift left instead of a rounding saturating shift left.
+ const int y_offset =
+ (1 << (conv_params->round_1 - 1)) - (1 << (y_offset_bits - 1));
const uint16_t *src_ptr = src - vert_offset * src_stride - horiz_offset;
@@ -1107,1275 +1875,246 @@
const int16_t *y_filter_ptr = av1_get_interp_filter_subpel_kernel(
filter_params_y, subpel_y_qn & SUBPEL_MASK);
- if (filter_params_x->taps > 8) {
- highbd_convolve_2d_x_sr_12tap_neon(src_ptr, src_stride, im_block, im_stride,
- w, im_h, x_filter_ptr, conv_params,
- x_offset_initial);
+ if (x_filter_taps > 8) {
+ highbd_convolve_2d_sr_horiz_12tap_neon(src_ptr, src_stride, im_block,
+ im_stride, w, im_h, x_filter_ptr,
+ conv_params, x_offset_initial);
- highbd_convolve_2d_y_sr_12tap_neon(im_block, im_stride, dst, dst_stride, w,
- h, y_filter_ptr, conv_params, bd,
- y_offset_initial, y_offset_correction);
- } else {
- highbd_convolve_x_8tap_neon(src_ptr, src_stride, im_block, im_stride, w,
- im_h, x_filter_ptr, conv_params,
- x_offset_initial);
-
- highbd_convolve_2d_y_sr_8tap_neon(im_block, im_stride, dst, dst_stride, w,
- h, y_filter_ptr, conv_params, bd,
- y_offset_initial, y_offset_correction);
- }
-}
-
-static INLINE void highbd_convolve_2d_x_scale_8tap_neon(
- const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
- int w, int h, const int subpel_x_qn, const int x_step_qn,
- const InterpFilterParams *filter_params, ConvolveParams *conv_params,
- const int offset) {
- const uint32x4_t idx = { 0, 1, 2, 3 };
- const uint32x4_t subpel_mask = vdupq_n_u32(SCALE_SUBPEL_MASK);
- const int32x4_t shift_s32 = vdupq_n_s32(-conv_params->round_0);
- const int32x4_t offset_s32 = vdupq_n_s32(offset);
-
- if (w <= 4) {
- int height = h;
- int16x8_t s0, s1, s2, s3;
- uint16x4_t d0;
-
- uint16_t *d = dst_ptr;
-
- do {
- int x_qn = subpel_x_qn;
-
- // Load 4 src vectors at a time, they might be the same, but we have to
- // calculate the indices anyway. Doing it in SIMD and then storing the
- // indices is faster than having to calculate the expression
- // &src_ptr[((x_qn + 0*x_step_qn) >> SCALE_SUBPEL_BITS)] 4 times
- // Ideally this should be a gather using the indices, but NEON does not
- // have that, so have to emulate
- const uint32x4_t xqn_idx = vmlaq_n_u32(vdupq_n_u32(x_qn), idx, x_step_qn);
- // We have to multiply x2 to get the actual pointer as sizeof(uint16_t) =
- // 2
- const uint32x4_t src_idx_u32 =
- vshlq_n_u32(vshrq_n_u32(xqn_idx, SCALE_SUBPEL_BITS), 1);
-#if AOM_ARCH_AARCH64
- uint64x2_t src4[2];
- src4[0] = vaddw_u32(vdupq_n_u64((const uint64_t)src_ptr),
- vget_low_u32(src_idx_u32));
- src4[1] = vaddw_u32(vdupq_n_u64((const uint64_t)src_ptr),
- vget_high_u32(src_idx_u32));
- int16_t *src4_ptr[4];
- uint64_t *tmp_ptr = (uint64_t *)&src4_ptr;
- vst1q_u64(tmp_ptr, src4[0]);
- vst1q_u64(tmp_ptr + 2, src4[1]);
-#else
- uint32x4_t src4;
- src4 = vaddq_u32(vdupq_n_u32((const uint32_t)src_ptr), src_idx_u32);
- int16_t *src4_ptr[4];
- uint32_t *tmp_ptr = (uint32_t *)&src4_ptr;
- vst1q_u32(tmp_ptr, src4);
-#endif // AOM_ARCH_AARCH64
- // Same for the filter vectors
- const int32x4_t filter_idx_s32 = vreinterpretq_s32_u32(
- vshrq_n_u32(vandq_u32(xqn_idx, subpel_mask), SCALE_EXTRA_BITS));
- int32_t x_filter4_idx[4];
- vst1q_s32(x_filter4_idx, filter_idx_s32);
- const int16_t *x_filter4_ptr[4];
-
- // Load source
- s0 = vld1q_s16(src4_ptr[0]);
- s1 = vld1q_s16(src4_ptr[1]);
- s2 = vld1q_s16(src4_ptr[2]);
- s3 = vld1q_s16(src4_ptr[3]);
-
- // We could easily do this using SIMD as well instead of calling the
- // inline function 4 times.
- x_filter4_ptr[0] =
- av1_get_interp_filter_subpel_kernel(filter_params, x_filter4_idx[0]);
- x_filter4_ptr[1] =
- av1_get_interp_filter_subpel_kernel(filter_params, x_filter4_idx[1]);
- x_filter4_ptr[2] =
- av1_get_interp_filter_subpel_kernel(filter_params, x_filter4_idx[2]);
- x_filter4_ptr[3] =
- av1_get_interp_filter_subpel_kernel(filter_params, x_filter4_idx[3]);
-
- // Actually load the filters
- const int16x8_t x_filter0 = vld1q_s16(x_filter4_ptr[0]);
- const int16x8_t x_filter1 = vld1q_s16(x_filter4_ptr[1]);
- const int16x8_t x_filter2 = vld1q_s16(x_filter4_ptr[2]);
- const int16x8_t x_filter3 = vld1q_s16(x_filter4_ptr[3]);
-
- // Group low and high parts and transpose
- int16x4_t filters_lo[] = { vget_low_s16(x_filter0),
- vget_low_s16(x_filter1),
- vget_low_s16(x_filter2),
- vget_low_s16(x_filter3) };
- int16x4_t filters_hi[] = { vget_high_s16(x_filter0),
- vget_high_s16(x_filter1),
- vget_high_s16(x_filter2),
- vget_high_s16(x_filter3) };
- transpose_u16_4x4((uint16x4_t *)filters_lo);
- transpose_u16_4x4((uint16x4_t *)filters_hi);
-
- // Run the 2D Scale convolution
- d0 = highbd_convolve8_2d_scale_horiz4x8_s32_s16(
- s0, s1, s2, s3, filters_lo, filters_hi, shift_s32, offset_s32);
-
- if (w == 2) {
- store_u16_2x1(d + 0 * dst_stride, d0, 0);
- } else {
- vst1_u16(d + 0 * dst_stride, d0);
- }
-
- src_ptr += src_stride;
- d += dst_stride;
- height--;
- } while (height > 0);
- } else {
- int height = h;
- int16x8_t s0, s1, s2, s3;
- uint16x4_t d0;
-
- do {
- int width = w;
- int x_qn = subpel_x_qn;
- uint16_t *d = dst_ptr;
- const uint16_t *s = src_ptr;
-
- do {
- // Load 4 src vectors at a time, they might be the same, but we have to
- // calculate the indices anyway. Doing it in SIMD and then storing the
- // indices is faster than having to calculate the expression
- // &src_ptr[((x_qn + 0*x_step_qn) >> SCALE_SUBPEL_BITS)] 4 times
- // Ideally this should be a gather using the indices, but NEON does not
- // have that, so have to emulate
- const uint32x4_t xqn_idx =
- vmlaq_n_u32(vdupq_n_u32(x_qn), idx, x_step_qn);
- // We have to multiply x2 to get the actual pointer as sizeof(uint16_t)
- // = 2
- const uint32x4_t src_idx_u32 =
- vshlq_n_u32(vshrq_n_u32(xqn_idx, SCALE_SUBPEL_BITS), 1);
-#if AOM_ARCH_AARCH64
- uint64x2_t src4[2];
- src4[0] = vaddw_u32(vdupq_n_u64((const uint64_t)s),
- vget_low_u32(src_idx_u32));
- src4[1] = vaddw_u32(vdupq_n_u64((const uint64_t)s),
- vget_high_u32(src_idx_u32));
- int16_t *src4_ptr[4];
- uint64_t *tmp_ptr = (uint64_t *)&src4_ptr;
- vst1q_u64(tmp_ptr, src4[0]);
- vst1q_u64(tmp_ptr + 2, src4[1]);
-#else
- uint32x4_t src4;
- src4 = vaddq_u32(vdupq_n_u32((const uint32_t)s), src_idx_u32);
- int16_t *src4_ptr[4];
- uint32_t *tmp_ptr = (uint32_t *)&src4_ptr;
- vst1q_u32(tmp_ptr, src4);
-#endif // AOM_ARCH_AARCH64
- // Same for the filter vectors
- const int32x4_t filter_idx_s32 = vreinterpretq_s32_u32(
- vshrq_n_u32(vandq_u32(xqn_idx, subpel_mask), SCALE_EXTRA_BITS));
- int32_t x_filter4_idx[4];
- vst1q_s32(x_filter4_idx, filter_idx_s32);
- const int16_t *x_filter4_ptr[4];
-
- // Load source
- s0 = vld1q_s16(src4_ptr[0]);
- s1 = vld1q_s16(src4_ptr[1]);
- s2 = vld1q_s16(src4_ptr[2]);
- s3 = vld1q_s16(src4_ptr[3]);
-
- // We could easily do this using SIMD as well instead of calling the
- // inline function 4 times.
- x_filter4_ptr[0] = av1_get_interp_filter_subpel_kernel(
- filter_params, x_filter4_idx[0]);
- x_filter4_ptr[1] = av1_get_interp_filter_subpel_kernel(
- filter_params, x_filter4_idx[1]);
- x_filter4_ptr[2] = av1_get_interp_filter_subpel_kernel(
- filter_params, x_filter4_idx[2]);
- x_filter4_ptr[3] = av1_get_interp_filter_subpel_kernel(
- filter_params, x_filter4_idx[3]);
-
- // Actually load the filters
- const int16x8_t x_filter0 = vld1q_s16(x_filter4_ptr[0]);
- const int16x8_t x_filter1 = vld1q_s16(x_filter4_ptr[1]);
- const int16x8_t x_filter2 = vld1q_s16(x_filter4_ptr[2]);
- const int16x8_t x_filter3 = vld1q_s16(x_filter4_ptr[3]);
-
- // Group low and high parts and transpose
- int16x4_t filters_lo[] = { vget_low_s16(x_filter0),
- vget_low_s16(x_filter1),
- vget_low_s16(x_filter2),
- vget_low_s16(x_filter3) };
- int16x4_t filters_hi[] = { vget_high_s16(x_filter0),
- vget_high_s16(x_filter1),
- vget_high_s16(x_filter2),
- vget_high_s16(x_filter3) };
- transpose_u16_4x4((uint16x4_t *)filters_lo);
- transpose_u16_4x4((uint16x4_t *)filters_hi);
-
- // Run the 2D Scale X convolution
- d0 = highbd_convolve8_2d_scale_horiz4x8_s32_s16(
- s0, s1, s2, s3, filters_lo, filters_hi, shift_s32, offset_s32);
-
- vst1_u16(d, d0);
-
- x_qn += 4 * x_step_qn;
- d += 4;
- width -= 4;
- } while (width > 0);
-
- src_ptr += src_stride;
- dst_ptr += dst_stride;
- height--;
- } while (height > 0);
- }
-}
-
-static INLINE void highbd_convolve_2d_y_scale_8tap_neon(
- const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
- int w, int h, const int subpel_y_qn, const int y_step_qn,
- const InterpFilterParams *filter_params, const int round1_bits,
- const int offset) {
- const int32x4_t offset_s32 = vdupq_n_s32(1 << offset);
-
- const int32x4_t round1_shift_s32 = vdupq_n_s32(-round1_bits);
- if (w <= 4) {
- int height = h;
- int16x4_t s0, s1, s2, s3, s4, s5, s6, s7;
- uint16x4_t d0;
-
- uint16_t *d = dst_ptr;
-
- int y_qn = subpel_y_qn;
- do {
- const int16_t *s =
- (const int16_t *)&src_ptr[(y_qn >> SCALE_SUBPEL_BITS) * src_stride];
-
- load_s16_4x8(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7);
-
- const int y_filter_idx = (y_qn & SCALE_SUBPEL_MASK) >> SCALE_EXTRA_BITS;
- const int16_t *y_filter_ptr =
- av1_get_interp_filter_subpel_kernel(filter_params, y_filter_idx);
- const int16x8_t y_filter = vld1q_s16(y_filter_ptr);
-
- d0 = highbd_convolve8_4_sr_s32_s16(s0, s1, s2, s3, s4, s5, s6, s7,
- y_filter, round1_shift_s32, offset_s32,
- vdupq_n_s32(0));
-
- if (w == 2) {
- store_u16_2x1(d, d0, 0);
- } else {
- vst1_u16(d, d0);
- }
-
- y_qn += y_step_qn;
- d += dst_stride;
- height--;
- } while (height > 0);
- } else {
- int width = w;
- int16x8_t s0, s1, s2, s3, s4, s5, s6, s7;
- uint16x8_t d0;
-
- do {
- int height = h;
- int y_qn = subpel_y_qn;
-
- uint16_t *d = dst_ptr;
-
- do {
- const int16_t *s =
- (const int16_t *)&src_ptr[(y_qn >> SCALE_SUBPEL_BITS) * src_stride];
- load_s16_8x8(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7);
-
- const int y_filter_idx = (y_qn & SCALE_SUBPEL_MASK) >> SCALE_EXTRA_BITS;
- const int16_t *y_filter_ptr =
- av1_get_interp_filter_subpel_kernel(filter_params, y_filter_idx);
- const int16x8_t y_filter = vld1q_s16(y_filter_ptr);
-
- d0 = highbd_convolve8_8_sr_s32_s16(s0, s1, s2, s3, s4, s5, s6, s7,
- y_filter, round1_shift_s32,
- offset_s32, vdupq_n_s32(0));
- vst1q_u16(d, d0);
-
- y_qn += y_step_qn;
- d += dst_stride;
- height--;
- } while (height > 0);
- src_ptr += 8;
- dst_ptr += 8;
- width -= 8;
- } while (width > 0);
- }
-}
-
-static INLINE void highbd_dist_wtd_comp_avg_neon(
- const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
- int w, int h, ConvolveParams *conv_params, const int round_bits,
- const int offset, const int bd) {
- CONV_BUF_TYPE *dst16 = conv_params->dst;
- const int dst16_stride = conv_params->dst_stride;
- const int32x4_t round_shift_s32 = vdupq_n_s32(-round_bits);
- const int16x4_t offset_s16 = vdup_n_s16(offset);
- const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
- uint16x4_t fwd_offset_u16 = vdup_n_u16(conv_params->fwd_offset);
- uint16x4_t bck_offset_u16 = vdup_n_u16(conv_params->bck_offset);
-
- // Weighted averaging
- if (w <= 4) {
- for (int y = 0; y < h; ++y) {
- const uint16x4_t s = vld1_u16(src_ptr + y * src_stride);
- const uint16x4_t d16 = vld1_u16(dst16 + y * dst16_stride);
- // We use vmull_u16/vmlal_u16 instead of of vmull_s16/vmlal_s16
- // because the latter sign-extend and the values are non-negative.
- // However, d0/d1 are signed-integers and we use vqmovun
- // to do saturated narrowing to unsigned.
- int32x4_t d0 = vreinterpretq_s32_u32(vmull_u16(d16, fwd_offset_u16));
- d0 = vreinterpretq_s32_u32(
- vmlal_u16(vreinterpretq_u32_s32(d0), s, bck_offset_u16));
- d0 = vshrq_n_s32(d0, DIST_PRECISION_BITS);
- // Subtract round offset and convolve round
- d0 = vqrshlq_s32(vsubw_s16(d0, offset_s16), round_shift_s32);
- uint16x4_t d = vqmovun_s32(d0);
- d = vmin_u16(d, vget_low_u16(max));
- if (w == 2) {
- store_u16_2x1(dst_ptr + y * dst_stride, d, 0);
- } else {
- vst1_u16(dst_ptr + y * dst_stride, d);
- }
- }
- } else {
- for (int y = 0; y < h; ++y) {
- for (int x = 0; x < w; x += 8) {
- const uint16x8_t s = vld1q_u16(src_ptr + y * src_stride + x);
- const uint16x8_t d16 = vld1q_u16(dst16 + y * dst16_stride + x);
- // We use vmull_u16/vmlal_u16 instead of of vmull_s16/vmlal_s16
- // because the latter sign-extend and the values are non-negative.
- // However, d0/d1 are signed-integers and we use vqmovun
- // to do saturated narrowing to unsigned.
- int32x4_t d0 =
- vreinterpretq_s32_u32(vmull_u16(vget_low_u16(d16), fwd_offset_u16));
- int32x4_t d1 = vreinterpretq_s32_u32(
- vmull_u16(vget_high_u16(d16), fwd_offset_u16));
- d0 = vreinterpretq_s32_u32(vmlal_u16(vreinterpretq_u32_s32(d0),
- vget_low_u16(s), bck_offset_u16));
- d1 = vreinterpretq_s32_u32(vmlal_u16(vreinterpretq_u32_s32(d1),
- vget_high_u16(s), bck_offset_u16));
- d0 = vshrq_n_s32(d0, DIST_PRECISION_BITS);
- d1 = vshrq_n_s32(d1, DIST_PRECISION_BITS);
- d0 = vqrshlq_s32(vsubw_s16(d0, offset_s16), round_shift_s32);
- d1 = vqrshlq_s32(vsubw_s16(d1, offset_s16), round_shift_s32);
- uint16x8_t d01 = vcombine_u16(vqmovun_s32(d0), vqmovun_s32(d1));
- d01 = vminq_u16(d01, max);
- vst1q_u16(dst_ptr + y * dst_stride + x, d01);
- }
- }
- }
-}
-
-static INLINE void highbd_comp_avg_neon(const uint16_t *src_ptr, int src_stride,
- uint16_t *dst_ptr, int dst_stride,
- int w, int h,
- ConvolveParams *conv_params,
- const int round_bits, const int offset,
- const int bd) {
- CONV_BUF_TYPE *dst16 = conv_params->dst;
- const int dst16_stride = conv_params->dst_stride;
- const int32x4_t round_shift_s32 = vdupq_n_s32(-round_bits);
- const int16x4_t offset_s16 = vdup_n_s16(offset);
- const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
-
- if (w <= 4) {
- for (int y = 0; y < h; ++y) {
- const uint16x4_t s = vld1_u16(src_ptr + y * src_stride);
- const uint16x4_t d16 = vld1_u16(dst16 + y * dst16_stride);
- int32x4_t s_s32 = vreinterpretq_s32_u32(vmovl_u16(s));
- int32x4_t d16_s32 = vreinterpretq_s32_u32(vmovl_u16(d16));
- int32x4_t d0 = vhaddq_s32(s_s32, d16_s32);
- d0 = vsubw_s16(d0, offset_s16);
- d0 = vqrshlq_s32(d0, round_shift_s32);
- uint16x4_t d = vqmovun_s32(d0);
- d = vmin_u16(d, vget_low_u16(max));
- if (w == 2) {
- store_u16_2x1(dst_ptr + y * dst_stride, d, 0);
- } else {
- vst1_u16(dst_ptr + y * dst_stride, d);
- }
- }
- } else {
- for (int y = 0; y < h; ++y) {
- for (int x = 0; x < w; x += 8) {
- const uint16x8_t s = vld1q_u16(src_ptr + y * src_stride + x);
- const uint16x8_t d16 = vld1q_u16(dst16 + y * dst16_stride + x);
- int32x4_t s_lo = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(s)));
- int32x4_t s_hi = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(s)));
- int32x4_t d16_lo = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(d16)));
- int32x4_t d16_hi = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(d16)));
- int32x4_t d0 = vhaddq_s32(s_lo, d16_lo);
- int32x4_t d1 = vhaddq_s32(s_hi, d16_hi);
- d0 = vsubw_s16(d0, offset_s16);
- d1 = vsubw_s16(d1, offset_s16);
- d0 = vqrshlq_s32(d0, round_shift_s32);
- d1 = vqrshlq_s32(d1, round_shift_s32);
- uint16x8_t d01 = vcombine_u16(vqmovun_s32(d0), vqmovun_s32(d1));
- d01 = vminq_u16(d01, max);
- vst1q_u16(dst_ptr + y * dst_stride + x, d01);
- }
- }
- }
-}
-
-static INLINE void highbd_convolve_correct_offset_neon(
- const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
- int w, int h, const int round_bits, const int offset, const int bd) {
- const int32x4_t round_shift_s32 = vdupq_n_s32(-round_bits);
- const int16x4_t offset_s16 = vdup_n_s16(offset);
- const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
-
- if (w <= 4) {
- for (int y = 0; y < h; ++y) {
- const int16x4_t s = vld1_s16((const int16_t *)src_ptr + y * src_stride);
- const int32x4_t d0 =
- vqrshlq_s32(vsubl_s16(s, offset_s16), round_shift_s32);
- uint16x4_t d = vqmovun_s32(d0);
- d = vmin_u16(d, vget_low_u16(max));
- if (w == 2) {
- store_u16_2x1(dst_ptr + y * dst_stride, d, 0);
- } else {
- vst1_u16(dst_ptr + y * dst_stride, d);
- }
- }
- } else {
- for (int y = 0; y < h; ++y) {
- for (int x = 0; x < w; x += 8) {
- // Subtract round offset and convolve round
- const int16x8_t s =
- vld1q_s16((const int16_t *)src_ptr + y * src_stride + x);
- const int32x4_t d0 = vqrshlq_s32(vsubl_s16(vget_low_s16(s), offset_s16),
- round_shift_s32);
- const int32x4_t d1 = vqrshlq_s32(
- vsubl_s16(vget_high_s16(s), offset_s16), round_shift_s32);
- uint16x8_t d01 = vcombine_u16(vqmovun_s32(d0), vqmovun_s32(d1));
- d01 = vminq_u16(d01, max);
- vst1q_u16(dst_ptr + y * dst_stride + x, d01);
- }
- }
- }
-}
-
-void av1_highbd_convolve_2d_scale_neon(
- const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w,
- int h, const InterpFilterParams *filter_params_x,
- const InterpFilterParams *filter_params_y, const int subpel_x_qn,
- const int x_step_qn, const int subpel_y_qn, const int y_step_qn,
- ConvolveParams *conv_params, int bd) {
- uint16_t *im_block = (uint16_t *)aom_memalign(
- 16, 2 * sizeof(uint16_t) * MAX_SB_SIZE * (MAX_SB_SIZE + MAX_FILTER_TAP));
- if (!im_block) return;
- uint16_t *im_block2 = (uint16_t *)aom_memalign(
- 16, 2 * sizeof(uint16_t) * MAX_SB_SIZE * (MAX_SB_SIZE + MAX_FILTER_TAP));
- if (!im_block2) {
- aom_free(im_block); // free the first block and return.
+ highbd_convolve_2d_sr_vert_12tap_neon(im_block, im_stride, dst, dst_stride,
+ w, h, y_filter_ptr, conv_params, bd,
+ y_offset);
return;
}
-
- int im_h = (((h - 1) * y_step_qn + subpel_y_qn) >> SCALE_SUBPEL_BITS) +
- filter_params_y->taps;
- const int im_stride = MAX_SB_SIZE;
- const int bits =
- FILTER_BITS * 2 - conv_params->round_0 - conv_params->round_1;
- assert(bits >= 0);
-
- const int vert_offset = filter_params_y->taps / 2 - 1;
- const int horiz_offset = filter_params_x->taps / 2 - 1;
- const int x_offset_bits = (1 << (bd + FILTER_BITS - 1));
- const int y_offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0;
- const int y_offset_correction =
- ((1 << (y_offset_bits - conv_params->round_1)) +
- (1 << (y_offset_bits - conv_params->round_1 - 1)));
-
- CONV_BUF_TYPE *dst16 = conv_params->dst;
- const int dst16_stride = conv_params->dst_stride;
-
- const uint16_t *src_ptr = src - vert_offset * src_stride - horiz_offset;
-
- highbd_convolve_2d_x_scale_8tap_neon(
- src_ptr, src_stride, im_block, im_stride, w, im_h, subpel_x_qn, x_step_qn,
- filter_params_x, conv_params, x_offset_bits);
- if (conv_params->is_compound && !conv_params->do_average) {
- highbd_convolve_2d_y_scale_8tap_neon(
- im_block, im_stride, dst16, dst16_stride, w, h, subpel_y_qn, y_step_qn,
- filter_params_y, conv_params->round_1, y_offset_bits);
+ if (x_filter_taps <= 6 && w != 4) {
+ highbd_convolve_2d_sr_horiz_6tap_neon(src_ptr, src_stride, im_block,
+ im_stride, w, im_h, x_filter_ptr,
+ conv_params, x_offset_initial);
} else {
- highbd_convolve_2d_y_scale_8tap_neon(
- im_block, im_stride, im_block2, im_stride, w, h, subpel_y_qn, y_step_qn,
- filter_params_y, conv_params->round_1, y_offset_bits);
+ highbd_convolve_2d_sr_horiz_neon(src_ptr, src_stride, im_block, im_stride,
+ w, im_h, x_filter_ptr, conv_params,
+ x_offset_initial);
}
- // Do the compound averaging outside the loop, avoids branching within the
- // main loop
- if (conv_params->is_compound) {
- if (conv_params->do_average) {
- if (conv_params->use_dist_wtd_comp_avg) {
- highbd_dist_wtd_comp_avg_neon(im_block2, im_stride, dst, dst_stride, w,
- h, conv_params, bits, y_offset_correction,
- bd);
- } else {
- highbd_comp_avg_neon(im_block2, im_stride, dst, dst_stride, w, h,
- conv_params, bits, y_offset_correction, bd);
- }
- }
+ if (y_filter_taps <= 6) {
+ highbd_convolve_2d_sr_vert_6tap_neon(im_block, im_stride, dst, dst_stride,
+ w, h, y_filter_ptr, conv_params, bd,
+ y_offset);
} else {
- highbd_convolve_correct_offset_neon(im_block2, im_stride, dst, dst_stride,
- w, h, bits, y_offset_correction, bd);
- }
- aom_free(im_block);
- aom_free(im_block2);
-}
-
-static INLINE void highbd_convolve_dist_wtd_x_8tap_neon(
- const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
- int w, int h, const int16_t *x_filter_ptr, ConvolveParams *conv_params,
- const int offset) {
- const int16x8_t x_filter = vld1q_s16(x_filter_ptr);
- const int32x4_t shift_s32 = vdupq_n_s32(-conv_params->round_0);
- const int weight_bits = FILTER_BITS - conv_params->round_1;
- const int32x4_t zero_s32 = vdupq_n_s32(0);
- const int32x4_t weight_s32 = vdupq_n_s32(1 << weight_bits);
- const int32x4_t offset_s32 = vdupq_n_s32(offset);
-
- if (w <= 4) {
- int16x8_t s0, s1, s2, s3;
- uint16x4_t d0, d1;
- uint16x8_t d01;
-
- const int16_t *s = (const int16_t *)src_ptr;
- uint16_t *d = dst_ptr;
-
- do {
- load_s16_8x2(s, src_stride, &s0, &s2);
- load_s16_8x2(s + 8, src_stride, &s1, &s3);
-
- d0 = highbd_convolve8_wtd_horiz4_s32_s16(
- s0, s1, x_filter, shift_s32, zero_s32, weight_s32, offset_s32);
- d1 = highbd_convolve8_wtd_horiz4_s32_s16(
- s2, s3, x_filter, shift_s32, zero_s32, weight_s32, offset_s32);
- d01 = vcombine_u16(d0, d1);
-
- if (w == 2) {
- store_u16q_2x1(d + 0 * dst_stride, d01, 0);
- store_u16q_2x1(d + 1 * dst_stride, d01, 2);
- } else {
- vst1_u16(d + 0 * dst_stride, vget_low_u16(d01));
- vst1_u16(d + 1 * dst_stride, vget_high_u16(d01));
- }
-
- s += 2 * src_stride;
- d += 2 * dst_stride;
- h -= 2;
- } while (h > 0);
- } else {
- int height = h;
- int16x8_t s0, s1, s2, s3;
- uint16x8_t d0, d1;
-
- do {
- int width = w;
- const int16_t *s = (const int16_t *)src_ptr;
- uint16_t *d = dst_ptr;
-
- load_s16_8x2(s, src_stride, &s0, &s2);
- s += 8;
-
- do {
- load_s16_8x2(s, src_stride, &s1, &s3);
-
- d0 = highbd_convolve8_wtd_horiz8_s32_s16(
- s0, s1, x_filter, shift_s32, zero_s32, weight_s32, offset_s32);
- d1 = highbd_convolve8_wtd_horiz8_s32_s16(
- s2, s3, x_filter, shift_s32, zero_s32, weight_s32, offset_s32);
-
- store_u16_8x2(d, dst_stride, d0, d1);
-
- s0 = s1;
- s2 = s3;
- s += 8;
- d += 8;
- width -= 8;
- } while (width > 0);
- src_ptr += 2 * src_stride;
- dst_ptr += 2 * dst_stride;
- height -= 2;
- } while (height > 0);
+ highbd_convolve_2d_sr_vert_8tap_neon(im_block, im_stride, dst, dst_stride,
+ w, h, y_filter_ptr, conv_params, bd,
+ y_offset);
}
}
-static INLINE void highbd_convolve_dist_wtd_y_8tap_neon(
- const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
- int w, int h, const int16_t *y_filter_ptr, ConvolveParams *conv_params,
- const int offset) {
- const int16x8_t y_filter = vld1q_s16(y_filter_ptr);
- const int32x4_t shift_s32 = vdupq_n_s32(-conv_params->round_0);
- const int weight_bits = FILTER_BITS - conv_params->round_1;
- const int32x4_t zero_s32 = vdupq_n_s32(0);
- const int32x4_t weight_s32 = vdupq_n_s32(1 << weight_bits);
- const int32x4_t offset_s32 = vdupq_n_s32(offset);
-
- if (w <= 4) {
- int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
- uint16x4_t d0, d1;
- uint16x8_t d01;
-
- const int16_t *s = (const int16_t *)src_ptr;
- uint16_t *d = dst_ptr;
-
- load_s16_4x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
- s += 7 * src_stride;
-
- do {
- load_s16_4x4(s, src_stride, &s7, &s8, &s9, &s10);
-
- d0 = highbd_convolve8_wtd_4_s32_s16(s0, s1, s2, s3, s4, s5, s6, s7,
- y_filter, shift_s32, zero_s32,
- weight_s32, offset_s32);
- d1 = highbd_convolve8_wtd_4_s32_s16(s1, s2, s3, s4, s5, s6, s7, s8,
- y_filter, shift_s32, zero_s32,
- weight_s32, offset_s32);
- d01 = vcombine_u16(d0, d1);
-
- if (w == 2) {
- store_u16q_2x1(d + 0 * dst_stride, d01, 0);
- store_u16q_2x1(d + 1 * dst_stride, d01, 2);
- } else {
- vst1_u16(d + 0 * dst_stride, vget_low_u16(d01));
- vst1_u16(d + 1 * dst_stride, vget_high_u16(d01));
- }
-
- s0 = s2;
- s1 = s3;
- s2 = s4;
- s3 = s5;
- s4 = s6;
- s5 = s7;
- s6 = s8;
- s += 2 * src_stride;
- d += 2 * dst_stride;
- h -= 2;
- } while (h > 0);
- } else {
- int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8;
- uint16x8_t d0, d1;
-
- do {
- int height = h;
- const int16_t *s = (const int16_t *)src_ptr;
- uint16_t *d = dst_ptr;
-
- load_s16_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
- s += 7 * src_stride;
-
- do {
- load_s16_8x2(s, src_stride, &s7, &s8);
-
- d0 = highbd_convolve8_wtd_8_s32_s16(s0, s1, s2, s3, s4, s5, s6, s7,
- y_filter, shift_s32, zero_s32,
- weight_s32, offset_s32);
- d1 = highbd_convolve8_wtd_8_s32_s16(s1, s2, s3, s4, s5, s6, s7, s8,
- y_filter, shift_s32, zero_s32,
- weight_s32, offset_s32);
-
- store_u16_8x2(d, dst_stride, d0, d1);
-
- s0 = s2;
- s1 = s3;
- s2 = s4;
- s3 = s5;
- s4 = s6;
- s5 = s7;
- s6 = s8;
- s += 2 * src_stride;
- d += 2 * dst_stride;
- height -= 2;
- } while (height > 0);
- src_ptr += 8;
- dst_ptr += 8;
- w -= 8;
- } while (w > 0);
- }
-}
-
-void av1_highbd_dist_wtd_convolve_x_neon(
+// Filter used is [64, 64].
+void av1_highbd_convolve_x_sr_intrabc_neon(
const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w,
int h, const InterpFilterParams *filter_params_x, const int subpel_x_qn,
ConvolveParams *conv_params, int bd) {
- DECLARE_ALIGNED(16, uint16_t,
- im_block[(MAX_SB_SIZE + MAX_FILTER_TAP) * MAX_SB_SIZE]);
- CONV_BUF_TYPE *dst16 = conv_params->dst;
- int dst16_stride = conv_params->dst_stride;
- const int im_stride = MAX_SB_SIZE;
- const int horiz_offset = filter_params_x->taps / 2 - 1;
- const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0;
- const int round_offset = (1 << (offset_bits - conv_params->round_1)) +
- (1 << (offset_bits - conv_params->round_1 - 1));
- const int round_bits =
- 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1;
- assert(round_bits >= 0);
-
- const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
- filter_params_x, subpel_x_qn & SUBPEL_MASK);
-
- src -= horiz_offset;
-
- // horizontal filter
- if (conv_params->do_average) {
- highbd_convolve_dist_wtd_x_8tap_neon(src, src_stride, im_block, im_stride,
- w, h, x_filter_ptr, conv_params,
- round_offset);
- } else {
- highbd_convolve_dist_wtd_x_8tap_neon(src, src_stride, dst16, dst16_stride,
- w, h, x_filter_ptr, conv_params,
- round_offset);
- }
-
- if (conv_params->do_average) {
- if (conv_params->use_dist_wtd_comp_avg) {
- highbd_dist_wtd_comp_avg_neon(im_block, im_stride, dst, dst_stride, w, h,
- conv_params, round_bits, round_offset, bd);
- } else {
- highbd_comp_avg_neon(im_block, im_stride, dst, dst_stride, w, h,
- conv_params, round_bits, round_offset, bd);
- }
- }
-}
-
-void av1_highbd_dist_wtd_convolve_y_neon(
- const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w,
- int h, const InterpFilterParams *filter_params_y, const int subpel_y_qn,
- ConvolveParams *conv_params, int bd) {
- DECLARE_ALIGNED(16, uint16_t,
- im_block[(MAX_SB_SIZE + MAX_FILTER_TAP) * MAX_SB_SIZE]);
- CONV_BUF_TYPE *dst16 = conv_params->dst;
- int dst16_stride = conv_params->dst_stride;
- const int im_stride = MAX_SB_SIZE;
- const int vert_offset = filter_params_y->taps / 2 - 1;
- const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0;
- const int round_offset = (1 << (offset_bits - conv_params->round_1)) +
- (1 << (offset_bits - conv_params->round_1 - 1));
- const int round_bits =
- 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1;
- assert(round_bits >= 0);
-
- const int16_t *y_filter_ptr = av1_get_interp_filter_subpel_kernel(
- filter_params_y, subpel_y_qn & SUBPEL_MASK);
-
- src -= vert_offset * src_stride;
-
- // vertical filter
- if (conv_params->do_average) {
- highbd_convolve_dist_wtd_y_8tap_neon(src, src_stride, im_block, im_stride,
- w, h, y_filter_ptr, conv_params,
- round_offset);
- } else {
- highbd_convolve_dist_wtd_y_8tap_neon(src, src_stride, dst16, dst16_stride,
- w, h, y_filter_ptr, conv_params,
- round_offset);
- }
-
- if (conv_params->do_average) {
- if (conv_params->use_dist_wtd_comp_avg) {
- highbd_dist_wtd_comp_avg_neon(im_block, im_stride, dst, dst_stride, w, h,
- conv_params, round_bits, round_offset, bd);
- } else {
- highbd_comp_avg_neon(im_block, im_stride, dst, dst_stride, w, h,
- conv_params, round_bits, round_offset, bd);
- }
- }
-}
-
-static INLINE void highbd_2d_copy_neon(const uint16_t *src_ptr, int src_stride,
- uint16_t *dst_ptr, int dst_stride, int w,
- int h, const int round_bits,
- const int offset) {
- if (w <= 4) {
- const int16x4_t round_shift_s16 = vdup_n_s16(round_bits);
- const uint16x4_t offset_u16 = vdup_n_u16(offset);
-
- for (int y = 0; y < h; ++y) {
- const uint16x4_t s = vld1_u16(src_ptr + y * src_stride);
- uint16x4_t d = vshl_u16(s, round_shift_s16);
- d = vadd_u16(d, offset_u16);
- if (w == 2) {
- store_u16_2x1(dst_ptr + y * dst_stride, d, 0);
- } else {
- vst1_u16(dst_ptr + y * dst_stride, d);
- }
- }
- } else {
- const int16x8_t round_shift_s16 = vdupq_n_s16(round_bits);
- const uint16x8_t offset_u16 = vdupq_n_u16(offset);
-
- for (int y = 0; y < h; ++y) {
- for (int x = 0; x < w; x += 8) {
- const uint16x8_t s = vld1q_u16(src_ptr + y * src_stride + x);
- uint16x8_t d = vshlq_u16(s, round_shift_s16);
- d = vaddq_u16(d, offset_u16);
- vst1q_u16(dst_ptr + y * dst_stride + x, d);
- }
- }
- }
-}
-
-void av1_highbd_dist_wtd_convolve_2d_copy_neon(const uint16_t *src,
- int src_stride, uint16_t *dst,
- int dst_stride, int w, int h,
- ConvolveParams *conv_params,
- int bd) {
- DECLARE_ALIGNED(16, uint16_t,
- im_block[(MAX_SB_SIZE + MAX_FILTER_TAP) * MAX_SB_SIZE]);
-
- const int im_stride = MAX_SB_SIZE;
- CONV_BUF_TYPE *dst16 = conv_params->dst;
- int dst16_stride = conv_params->dst_stride;
- const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0;
- const int round_offset = (1 << (offset_bits - conv_params->round_1)) +
- (1 << (offset_bits - conv_params->round_1 - 1));
- const int round_bits =
- 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1;
- assert(round_bits >= 0);
-
- if (conv_params->do_average) {
- highbd_2d_copy_neon(src, src_stride, im_block, im_stride, w, h, round_bits,
- round_offset);
- } else {
- highbd_2d_copy_neon(src, src_stride, dst16, dst16_stride, w, h, round_bits,
- round_offset);
- }
-
- if (conv_params->do_average) {
- if (conv_params->use_dist_wtd_comp_avg) {
- highbd_dist_wtd_comp_avg_neon(im_block, im_stride, dst, dst_stride, w, h,
- conv_params, round_bits, round_offset, bd);
- } else {
- highbd_comp_avg_neon(im_block, im_stride, dst, dst_stride, w, h,
- conv_params, round_bits, round_offset, bd);
- }
- }
-}
-
-static INLINE void highbd_convolve_y_8tap_neon(
- const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
- int w, int h, const int16_t *y_filter_ptr, ConvolveParams *conv_params,
- int offset) {
- const int16x8_t y_filter = vld1q_s16(y_filter_ptr);
- const int32x4_t offset_s32 = vdupq_n_s32(offset);
- const int32x4_t shift_s32 = vdupq_n_s32(-conv_params->round_1);
+ assert(subpel_x_qn == 8);
+ assert(filter_params_x->taps == 2);
+ assert((conv_params->round_0 + conv_params->round_1) == 2 * FILTER_BITS);
+ (void)filter_params_x;
+ (void)subpel_x_qn;
+ (void)conv_params;
+ (void)bd;
if (w <= 4) {
- int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
- uint16x4_t d0, d1, d2, d3;
- uint16x8_t d01, d23;
-
- const int16_t *s = (const int16_t *)src_ptr;
- uint16_t *d = dst_ptr;
-
- load_s16_4x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
- s += 7 * src_stride;
-
do {
- load_s16_4x4(s, src_stride, &s7, &s8, &s9, &s10);
+ uint16x4_t s0 = vld1_u16(src);
+ uint16x4_t s1 = vld1_u16(src + 1);
- d0 = highbd_convolve8_sr_4_s32_s16(s0, s1, s2, s3, s4, s5, s6, s7,
- y_filter, shift_s32, offset_s32);
- d1 = highbd_convolve8_sr_4_s32_s16(s1, s2, s3, s4, s5, s6, s7, s8,
- y_filter, shift_s32, offset_s32);
- d2 = highbd_convolve8_sr_4_s32_s16(s2, s3, s4, s5, s6, s7, s8, s9,
- y_filter, shift_s32, offset_s32);
- d3 = highbd_convolve8_sr_4_s32_s16(s3, s4, s5, s6, s7, s8, s9, s10,
- y_filter, shift_s32, offset_s32);
-
- d01 = vcombine_u16(d0, d1);
- d23 = vcombine_u16(d2, d3);
+ uint16x4_t d0 = vrhadd_u16(s0, s1);
if (w == 2) {
- store_u16q_2x1(d + 0 * dst_stride, d01, 0);
- store_u16q_2x1(d + 1 * dst_stride, d01, 2);
- if (h != 2) {
- store_u16q_2x1(d + 2 * dst_stride, d23, 0);
- store_u16q_2x1(d + 3 * dst_stride, d23, 2);
- }
+ store_u16_2x1(dst, d0, 0);
} else {
- vst1_u16(d + 0 * dst_stride, vget_low_u16(d01));
- vst1_u16(d + 1 * dst_stride, vget_high_u16(d01));
- if (h != 2) {
- vst1_u16(d + 2 * dst_stride, vget_low_u16(d23));
- vst1_u16(d + 3 * dst_stride, vget_high_u16(d23));
- }
+ vst1_u16(dst, d0);
}
- s0 = s4;
- s1 = s5;
- s2 = s6;
- s3 = s7;
- s4 = s8;
- s5 = s9;
- s6 = s10;
- s += 4 * src_stride;
- d += 4 * dst_stride;
- h -= 4;
- } while (h > 0);
+ src += src_stride;
+ dst += dst_stride;
+ } while (--h != 0);
} else {
- int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
- uint16x8_t d0, d1, d2, d3;
do {
- int height = h;
- const int16_t *s = (const int16_t *)src_ptr;
- uint16_t *d = dst_ptr;
-
- load_s16_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
- s += 7 * src_stride;
+ const uint16_t *src_ptr = src;
+ uint16_t *dst_ptr = dst;
+ int width = w;
do {
- load_s16_8x4(s, src_stride, &s7, &s8, &s9, &s10);
+ uint16x8_t s0 = vld1q_u16(src_ptr);
+ uint16x8_t s1 = vld1q_u16(src_ptr + 1);
- d0 = highbd_convolve8_8_s32_s16(s0, s1, s2, s3, s4, s5, s6, s7,
- y_filter, offset_s32);
- d1 = highbd_convolve8_8_s32_s16(s1, s2, s3, s4, s5, s6, s7, s8,
- y_filter, offset_s32);
- d2 = highbd_convolve8_8_s32_s16(s2, s3, s4, s5, s6, s7, s8, s9,
- y_filter, offset_s32);
- d3 = highbd_convolve8_8_s32_s16(s3, s4, s5, s6, s7, s8, s9, s10,
- y_filter, offset_s32);
+ uint16x8_t d0 = vrhaddq_u16(s0, s1);
- if (h == 2) {
- store_u16_8x2(d, dst_stride, d0, d1);
- } else {
- store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
- }
+ vst1q_u16(dst_ptr, d0);
- s0 = s4;
- s1 = s5;
- s2 = s6;
- s3 = s7;
- s4 = s8;
- s5 = s9;
- s6 = s10;
- s += 4 * src_stride;
- d += 4 * dst_stride;
- height -= 4;
- } while (height > 0);
- src_ptr += 8;
- dst_ptr += 8;
- w -= 8;
- } while (w > 0);
+ src_ptr += 8;
+ dst_ptr += 8;
+ width -= 8;
+ } while (width != 0);
+ src += src_stride;
+ dst += dst_stride;
+ } while (--h != 0);
}
}
-void av1_highbd_dist_wtd_convolve_2d_neon(
+// Filter used is [64, 64].
+void av1_highbd_convolve_y_sr_intrabc_neon(
+ const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w,
+ int h, const InterpFilterParams *filter_params_y, const int subpel_y_qn,
+ int bd) {
+ assert(subpel_y_qn == 8);
+ assert(filter_params_y->taps == 2);
+ (void)filter_params_y;
+ (void)subpel_y_qn;
+ (void)bd;
+
+ if (w <= 4) {
+ do {
+ uint16x4_t s0 = vld1_u16(src);
+ uint16x4_t s1 = vld1_u16(src + src_stride);
+
+ uint16x4_t d0 = vrhadd_u16(s0, s1);
+
+ if (w == 2) {
+ store_u16_2x1(dst, d0, 0);
+ } else {
+ vst1_u16(dst, d0);
+ }
+
+ src += src_stride;
+ dst += dst_stride;
+ } while (--h != 0);
+ } else {
+ do {
+ const uint16_t *src_ptr = src;
+ uint16_t *dst_ptr = dst;
+ int height = h;
+
+ do {
+ uint16x8_t s0 = vld1q_u16(src_ptr);
+ uint16x8_t s1 = vld1q_u16(src_ptr + src_stride);
+
+ uint16x8_t d0 = vrhaddq_u16(s0, s1);
+
+ vst1q_u16(dst_ptr, d0);
+
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ } while (--height != 0);
+ src += 8;
+ dst += 8;
+ w -= 8;
+ } while (w != 0);
+ }
+}
+
+// Both horizontal and vertical passes use the same 2-tap filter: [64, 64].
+void av1_highbd_convolve_2d_sr_intrabc_neon(
const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w,
int h, const InterpFilterParams *filter_params_x,
const InterpFilterParams *filter_params_y, const int subpel_x_qn,
const int subpel_y_qn, ConvolveParams *conv_params, int bd) {
+ assert(subpel_x_qn == 8);
+ assert(subpel_y_qn == 8);
+ assert(filter_params_x->taps == 2 && filter_params_y->taps == 2);
+ assert((conv_params->round_0 + conv_params->round_1) == 2 * FILTER_BITS);
+ assert(w <= MAX_SB_SIZE && h <= MAX_SB_SIZE);
+ (void)filter_params_x;
+ (void)subpel_x_qn;
+ (void)filter_params_y;
+ (void)subpel_y_qn;
+ (void)conv_params;
+ (void)bd;
+
DECLARE_ALIGNED(16, uint16_t,
- im_block[(MAX_SB_SIZE + MAX_FILTER_TAP) * MAX_SB_SIZE]);
- DECLARE_ALIGNED(16, uint16_t,
- im_block2[(MAX_SB_SIZE + MAX_FILTER_TAP) * MAX_SB_SIZE]);
+ im_block[(MAX_SB_SIZE + MAX_FILTER_TAP - 1) * MAX_SB_SIZE]);
+ int im_h = h + 1;
+ int im_stride = MAX_SB_SIZE;
- CONV_BUF_TYPE *dst16 = conv_params->dst;
- int dst16_stride = conv_params->dst_stride;
+ uint16x8_t vert_offset = vdupq_n_u16(1);
- const int im_h = h + filter_params_y->taps - 1;
- const int im_stride = MAX_SB_SIZE;
- const int vert_offset = filter_params_y->taps / 2 - 1;
- const int horiz_offset = filter_params_x->taps / 2 - 1;
- const int round_bits =
- 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1;
- const int x_offset_initial = (1 << (bd + FILTER_BITS - 1));
- const int y_offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0;
- const int y_offset_initial = (1 << y_offset_bits);
- const int y_offset_correction =
- ((1 << (y_offset_bits - conv_params->round_1)) +
- (1 << (y_offset_bits - conv_params->round_1 - 1)));
+ uint16_t *im = im_block;
- const uint16_t *src_ptr = src - vert_offset * src_stride - horiz_offset;
-
- const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
- filter_params_x, subpel_x_qn & SUBPEL_MASK);
- const int16_t *y_filter_ptr = av1_get_interp_filter_subpel_kernel(
- filter_params_y, subpel_y_qn & SUBPEL_MASK);
-
- // horizontal filter
- highbd_convolve_x_8tap_neon(src_ptr, src_stride, im_block, im_stride, w, im_h,
- x_filter_ptr, conv_params, x_offset_initial);
- // vertical filter
- if (conv_params->do_average) {
- highbd_convolve_y_8tap_neon(im_block, im_stride, im_block2, im_stride, w, h,
- y_filter_ptr, conv_params, y_offset_initial);
- } else {
- highbd_convolve_y_8tap_neon(im_block, im_stride, dst16, dst16_stride, w, h,
- y_filter_ptr, conv_params, y_offset_initial);
- }
-
- // Do the compound averaging outside the loop, avoids branching within the
- // main loop
- if (conv_params->do_average) {
- if (conv_params->use_dist_wtd_comp_avg) {
- highbd_dist_wtd_comp_avg_neon(im_block2, im_stride, dst, dst_stride, w, h,
- conv_params, round_bits,
- y_offset_correction, bd);
- } else {
- highbd_comp_avg_neon(im_block2, im_stride, dst, dst_stride, w, h,
- conv_params, round_bits, y_offset_correction, bd);
- }
- }
-}
-
-#define UPSCALE_NORMATIVE_TAPS 8
-
-void av1_highbd_convolve_horiz_rs_neon(const uint16_t *src, int src_stride,
- uint16_t *dst, int dst_stride, int w,
- int h, const int16_t *x_filters,
- int x0_qn, int x_step_qn, int bd) {
- const int horiz_offset = UPSCALE_NORMATIVE_TAPS / 2 - 1;
-
- const int32x4_t idx = { 0, 1, 2, 3 };
- const int32x4_t subpel_mask = vdupq_n_s32(RS_SCALE_SUBPEL_MASK);
- const int32x4_t shift_s32 = vdupq_n_s32(-FILTER_BITS);
- const int32x4_t offset_s32 = vdupq_n_s32(0);
- const uint16x4_t max = vdup_n_u16((1 << bd) - 1);
-
- const uint16_t *src_ptr = src - horiz_offset;
- uint16_t *dst_ptr = dst;
-
+ // Horizontal filter.
if (w <= 4) {
- int height = h;
- int16x8_t s0, s1, s2, s3;
- uint16x4_t d0;
-
- uint16_t *d = dst_ptr;
do {
- int x_qn = x0_qn;
+ uint16x4_t s0 = vld1_u16(src);
+ uint16x4_t s1 = vld1_u16(src + 1);
- // Load 4 src vectors at a time, they might be the same, but we have to
- // calculate the indices anyway. Doing it in SIMD and then storing the
- // indices is faster than having to calculate the expression
- // &src_ptr[((x_qn + 0*x_step_qn) >> RS_SCALE_SUBPEL_BITS)] 4 times
- // Ideally this should be a gather using the indices, but NEON does not
- // have that, so have to emulate
- const int32x4_t xqn_idx = vmlaq_n_s32(vdupq_n_s32(x_qn), idx, x_step_qn);
- // We have to multiply x2 to get the actual pointer as sizeof(uint16_t) =
- // 2
- const int32x4_t src_idx =
- vshlq_n_s32(vshrq_n_s32(xqn_idx, RS_SCALE_SUBPEL_BITS), 1);
- // Similarly for the filter vector indices, we calculate the filter
- // indices for 4 columns. First we calculate the indices:
- // x_qn & RS_SCALE_SUBPEL_MASK) >> RS_SCALE_EXTRA_BITS
- // Then we calculate the actual pointers, multiplying with
- // UPSCALE_UPSCALE_NORMATIVE_TAPS
- // again shift left by 1
- const int32x4_t x_filter4_idx = vshlq_n_s32(
- vshrq_n_s32(vandq_s32(xqn_idx, subpel_mask), RS_SCALE_EXTRA_BITS), 1);
- // Even though pointers are unsigned 32/64-bit ints we do signed
- // addition The reason for this is that x_qn can be negative, leading to
- // negative offsets. Argon test
- // profile0_core/streams/test10573_11003.obu was failing because of
- // this.
-#if AOM_ARCH_AARCH64
- uint64x2_t tmp4[2];
- tmp4[0] = vreinterpretq_u64_s64(vaddw_s32(
- vdupq_n_s64((const int64_t)src_ptr), vget_low_s32(src_idx)));
- tmp4[1] = vreinterpretq_u64_s64(vaddw_s32(
- vdupq_n_s64((const int64_t)src_ptr), vget_high_s32(src_idx)));
- int16_t *src4_ptr[4];
- uint64_t *tmp_ptr = (uint64_t *)&src4_ptr;
- vst1q_u64(tmp_ptr, tmp4[0]);
- vst1q_u64(tmp_ptr + 2, tmp4[1]);
+ uint16x4_t d0 = vadd_u16(s0, s1);
- // filter vectors
- tmp4[0] = vreinterpretq_u64_s64(vmlal_s32(
- vdupq_n_s64((const int64_t)x_filters), vget_low_s32(x_filter4_idx),
- vdup_n_s32(UPSCALE_NORMATIVE_TAPS)));
- tmp4[1] = vreinterpretq_u64_s64(vmlal_s32(
- vdupq_n_s64((const int64_t)x_filters), vget_high_s32(x_filter4_idx),
- vdup_n_s32(UPSCALE_NORMATIVE_TAPS)));
+ // Safe to store the whole vector, the im buffer is big enough.
+ vst1_u16(im, d0);
- const int16_t *x_filter4_ptr[4];
- tmp_ptr = (uint64_t *)&x_filter4_ptr;
- vst1q_u64(tmp_ptr, tmp4[0]);
- vst1q_u64(tmp_ptr + 2, tmp4[1]);
-#else
- uint32x4_t tmp4;
- tmp4 = vreinterpretq_u32_s32(
- vaddq_s32(vdupq_n_s32((const int32_t)src_ptr), src_idx));
- int16_t *src4_ptr[4];
- uint32_t *tmp_ptr = (uint32_t *)&src4_ptr;
- vst1q_u32(tmp_ptr, tmp4);
- // filter vectors
- tmp4 = vreinterpretq_u32_s32(
- vmlaq_s32(vdupq_n_s32((const int32_t)x_filters), x_filter4_idx,
- vdupq_n_s32(UPSCALE_NORMATIVE_TAPS)));
-
- const int16_t *x_filter4_ptr[4];
- tmp_ptr = (uint32_t *)&x_filter4_ptr;
- vst1q_u32(tmp_ptr, tmp4);
-#endif // AOM_ARCH_AARCH64
- // Load source
- s0 = vld1q_s16(src4_ptr[0]);
- s1 = vld1q_s16(src4_ptr[1]);
- s2 = vld1q_s16(src4_ptr[2]);
- s3 = vld1q_s16(src4_ptr[3]);
-
- // Actually load the filters
- const int16x8_t x_filter0 = vld1q_s16(x_filter4_ptr[0]);
- const int16x8_t x_filter1 = vld1q_s16(x_filter4_ptr[1]);
- const int16x8_t x_filter2 = vld1q_s16(x_filter4_ptr[2]);
- const int16x8_t x_filter3 = vld1q_s16(x_filter4_ptr[3]);
-
- // Group low and high parts and transpose
- int16x4_t filters_lo[] = { vget_low_s16(x_filter0),
- vget_low_s16(x_filter1),
- vget_low_s16(x_filter2),
- vget_low_s16(x_filter3) };
- int16x4_t filters_hi[] = { vget_high_s16(x_filter0),
- vget_high_s16(x_filter1),
- vget_high_s16(x_filter2),
- vget_high_s16(x_filter3) };
- transpose_u16_4x4((uint16x4_t *)filters_lo);
- transpose_u16_4x4((uint16x4_t *)filters_hi);
-
- // Run the 2D Scale convolution
- d0 = highbd_convolve8_2d_scale_horiz4x8_s32_s16(
- s0, s1, s2, s3, filters_lo, filters_hi, shift_s32, offset_s32);
-
- d0 = vmin_u16(d0, max);
-
- if (w == 2) {
- store_u16_2x1(d + 0 * dst_stride, d0, 0);
- } else {
- vst1_u16(d + 0 * dst_stride, d0);
- }
-
- src_ptr += src_stride;
- d += dst_stride;
- height--;
- } while (height > 0);
+ src += src_stride;
+ im += im_stride;
+ } while (--im_h != 0);
} else {
- int height = h;
- int16x8_t s0, s1, s2, s3;
- uint16x4_t d0;
-
do {
+ const uint16_t *src_ptr = src;
+ uint16_t *im_ptr = im;
int width = w;
- int x_qn = x0_qn;
- uint16_t *d = dst_ptr;
- const uint16_t *s = src_ptr;
do {
- // Load 4 src vectors at a time, they might be the same, but we have to
- // calculate the indices anyway. Doing it in SIMD and then storing the
- // indices is faster than having to calculate the expression
- // &src_ptr[((x_qn + 0*x_step_qn) >> RS_SCALE_SUBPEL_BITS)] 4 times
- // Ideally this should be a gather using the indices, but NEON does not
- // have that, so have to emulate
- const int32x4_t xqn_idx =
- vmlaq_n_s32(vdupq_n_s32(x_qn), idx, x_step_qn);
- // We have to multiply x2 to get the actual pointer as sizeof(uint16_t)
- // = 2
- const int32x4_t src_idx =
- vshlq_n_s32(vshrq_n_s32(xqn_idx, RS_SCALE_SUBPEL_BITS), 1);
+ uint16x8_t s0 = vld1q_u16(src_ptr);
+ uint16x8_t s1 = vld1q_u16(src_ptr + 1);
- // Similarly for the filter vector indices, we calculate the filter
- // indices for 4 columns. First we calculate the indices:
- // x_qn & RS_SCALE_SUBPEL_MASK) >> RS_SCALE_EXTRA_BITS
- // Then we calculate the actual pointers, multiplying with
- // UPSCALE_UPSCALE_NORMATIVE_TAPS
- // again shift left by 1
- const int32x4_t x_filter4_idx = vshlq_n_s32(
- vshrq_n_s32(vandq_s32(xqn_idx, subpel_mask), RS_SCALE_EXTRA_BITS),
- 1);
- // Even though pointers are unsigned 32/64-bit ints we do signed
- // addition The reason for this is that x_qn can be negative, leading to
- // negative offsets. Argon test
- // profile0_core/streams/test10573_11003.obu was failing because of
- // this.
-#if AOM_ARCH_AARCH64
- uint64x2_t tmp4[2];
- tmp4[0] = vreinterpretq_u64_s64(
- vaddw_s32(vdupq_n_s64((const int64_t)s), vget_low_s32(src_idx)));
- tmp4[1] = vreinterpretq_u64_s64(
- vaddw_s32(vdupq_n_s64((const int64_t)s), vget_high_s32(src_idx)));
- int16_t *src4_ptr[4];
- uint64_t *tmp_ptr = (uint64_t *)&src4_ptr;
- vst1q_u64(tmp_ptr, tmp4[0]);
- vst1q_u64(tmp_ptr + 2, tmp4[1]);
+ uint16x8_t d0 = vaddq_u16(s0, s1);
- // filter vectors
- tmp4[0] = vreinterpretq_u64_s64(vmlal_s32(
- vdupq_n_s64((const int64_t)x_filters), vget_low_s32(x_filter4_idx),
- vdup_n_s32(UPSCALE_NORMATIVE_TAPS)));
- tmp4[1] = vreinterpretq_u64_s64(vmlal_s32(
- vdupq_n_s64((const int64_t)x_filters), vget_high_s32(x_filter4_idx),
- vdup_n_s32(UPSCALE_NORMATIVE_TAPS)));
+ vst1q_u16(im_ptr, d0);
- const int16_t *x_filter4_ptr[4];
- tmp_ptr = (uint64_t *)&x_filter4_ptr;
- vst1q_u64(tmp_ptr, tmp4[0]);
- vst1q_u64(tmp_ptr + 2, tmp4[1]);
-#else
- uint32x4_t tmp4;
- tmp4 = vreinterpretq_u32_s32(
- vaddq_s32(vdupq_n_s32((const int32_t)s), src_idx));
- int16_t *src4_ptr[4];
- uint32_t *tmp_ptr = (uint32_t *)&src4_ptr;
- vst1q_u32(tmp_ptr, tmp4);
- // filter vectors
- tmp4 = vreinterpretq_u32_s32(
- vmlaq_s32(vdupq_n_s32((const int32_t)x_filters), x_filter4_idx,
- vdupq_n_s32(UPSCALE_NORMATIVE_TAPS)));
+ src_ptr += 8;
+ im_ptr += 8;
+ width -= 8;
+ } while (width != 0);
+ src += src_stride;
+ im += im_stride;
+ } while (--im_h != 0);
+ }
- const int16_t *x_filter4_ptr[4];
- tmp_ptr = (uint32_t *)&x_filter4_ptr;
- vst1q_u32(tmp_ptr, tmp4);
-#endif // AOM_ARCH_AARCH64
+ im = im_block;
- // Load source
- s0 = vld1q_s16(src4_ptr[0]);
- s1 = vld1q_s16(src4_ptr[1]);
- s2 = vld1q_s16(src4_ptr[2]);
- s3 = vld1q_s16(src4_ptr[3]);
+ // Vertical filter.
+ if (w <= 4) {
+ do {
+ uint16x4_t s0 = vld1_u16(im);
+ uint16x4_t s1 = vld1_u16(im + im_stride);
- // Actually load the filters
- const int16x8_t x_filter0 = vld1q_s16(x_filter4_ptr[0]);
- const int16x8_t x_filter1 = vld1q_s16(x_filter4_ptr[1]);
- const int16x8_t x_filter2 = vld1q_s16(x_filter4_ptr[2]);
- const int16x8_t x_filter3 = vld1q_s16(x_filter4_ptr[3]);
+ uint16x4_t d0 = vhadd_u16(s0, s1);
+ d0 = vhadd_u16(d0, vget_low_u16(vert_offset));
- // Group low and high parts and transpose
- int16x4_t filters_lo[] = { vget_low_s16(x_filter0),
- vget_low_s16(x_filter1),
- vget_low_s16(x_filter2),
- vget_low_s16(x_filter3) };
- int16x4_t filters_hi[] = { vget_high_s16(x_filter0),
- vget_high_s16(x_filter1),
- vget_high_s16(x_filter2),
- vget_high_s16(x_filter3) };
- transpose_u16_4x4((uint16x4_t *)filters_lo);
- transpose_u16_4x4((uint16x4_t *)filters_hi);
+ if (w == 2) {
+ store_u16_2x1(dst, d0, 0);
+ } else {
+ vst1_u16(dst, d0);
+ }
- // Run the 2D Scale X convolution
- d0 = highbd_convolve8_2d_scale_horiz4x8_s32_s16(
- s0, s1, s2, s3, filters_lo, filters_hi, shift_s32, offset_s32);
+ im += im_stride;
+ dst += dst_stride;
+ } while (--h != 0);
+ } else {
+ do {
+ uint16_t *im_ptr = im;
+ uint16_t *dst_ptr = dst;
+ int height = h;
- d0 = vmin_u16(d0, max);
- vst1_u16(d, d0);
+ do {
+ uint16x8_t s0 = vld1q_u16(im_ptr);
+ uint16x8_t s1 = vld1q_u16(im_ptr + im_stride);
- x_qn += 4 * x_step_qn;
- d += 4;
- width -= 4;
- } while (width > 0);
+ uint16x8_t d0 = vhaddq_u16(s0, s1);
+ d0 = vhaddq_u16(d0, vert_offset);
- src_ptr += src_stride;
- dst_ptr += dst_stride;
- height--;
- } while (height > 0);
+ vst1q_u16(dst_ptr, d0);
+
+ im_ptr += im_stride;
+ dst_ptr += dst_stride;
+ } while (--height != 0);
+ im += 8;
+ dst += 8;
+ w -= 8;
+ } while (w != 0);
}
}
diff --git a/av1/common/arm/highbd_convolve_neon.h b/av1/common/arm/highbd_convolve_neon.h
index f9d028f..08b2bda 100644
--- a/av1/common/arm/highbd_convolve_neon.h
+++ b/av1/common/arm/highbd_convolve_neon.h
@@ -14,68 +14,9 @@
#include <arm_neon.h>
-static INLINE int32x4_t highbd_convolve6_4_s32(
- const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
- const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
- const int16x8_t y_filter, const int32x4_t offset) {
- const int16x4_t y_filter_lo = vget_low_s16(y_filter);
- const int16x4_t y_filter_hi = vget_high_s16(y_filter);
-
- int32x4_t sum = vmlal_lane_s16(offset, s0, y_filter_lo, 1);
- sum = vmlal_lane_s16(sum, s1, y_filter_lo, 2);
- sum = vmlal_lane_s16(sum, s2, y_filter_lo, 3);
- sum = vmlal_lane_s16(sum, s3, y_filter_hi, 0);
- sum = vmlal_lane_s16(sum, s4, y_filter_hi, 1);
- sum = vmlal_lane_s16(sum, s5, y_filter_hi, 2);
-
- return sum;
-}
-
-static INLINE uint16x4_t highbd_convolve6_4_s32_s16(
- const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
- const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
- const int16x8_t y_filter, const int32x4_t offset) {
- int32x4_t sum =
- highbd_convolve6_4_s32(s0, s1, s2, s3, s4, s5, y_filter, offset);
-
- return vqrshrun_n_s32(sum, COMPOUND_ROUND1_BITS);
-}
-
-static INLINE void highbd_convolve6_8_s32(
- const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
- const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
- const int16x8_t y_filter, const int32x4_t offset, int32x4_t *sum0,
- int32x4_t *sum1) {
- const int16x4_t y_filter_lo = vget_low_s16(y_filter);
- const int16x4_t y_filter_hi = vget_high_s16(y_filter);
-
- *sum0 = vmlal_lane_s16(offset, vget_low_s16(s0), y_filter_lo, 1);
- *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s1), y_filter_lo, 2);
- *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s2), y_filter_lo, 3);
- *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s3), y_filter_hi, 0);
- *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s4), y_filter_hi, 1);
- *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s5), y_filter_hi, 2);
-
- *sum1 = vmlal_lane_s16(offset, vget_high_s16(s0), y_filter_lo, 1);
- *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s1), y_filter_lo, 2);
- *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s2), y_filter_lo, 3);
- *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s3), y_filter_hi, 0);
- *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s4), y_filter_hi, 1);
- *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s5), y_filter_hi, 2);
-}
-
-static INLINE uint16x8_t highbd_convolve6_8_s32_s16(
- const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
- const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
- const int16x8_t y_filter, const int32x4_t offset) {
- int32x4_t sum0;
- int32x4_t sum1;
- highbd_convolve6_8_s32(s0, s1, s2, s3, s4, s5, y_filter, offset, &sum0,
- &sum1);
-
- return vcombine_u16(vqrshrun_n_s32(sum0, COMPOUND_ROUND1_BITS),
- vqrshrun_n_s32(sum1, COMPOUND_ROUND1_BITS));
-}
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_dsp/arm/transpose_neon.h"
+#include "av1/common/convolve.h"
static INLINE int32x4_t highbd_convolve8_4_s32(
const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
@@ -97,18 +38,7 @@
return sum;
}
-static INLINE uint16x4_t highbd_convolve8_4_s32_s16(
- const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
- const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
- const int16x4_t s6, const int16x4_t s7, const int16x8_t y_filter,
- const int32x4_t offset) {
- int32x4_t sum =
- highbd_convolve8_4_s32(s0, s1, s2, s3, s4, s5, s6, s7, y_filter, offset);
-
- return vqrshrun_n_s32(sum, COMPOUND_ROUND1_BITS);
-}
-
-static INLINE uint16x4_t highbd_convolve8_sr_4_s32_s16(
+static INLINE uint16x4_t highbd_convolve8_4_sr_s32_s16(
const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
const int16x4_t s6, const int16x4_t s7, const int16x8_t y_filter,
@@ -120,23 +50,8 @@
return vqmovun_s32(sum);
}
-static INLINE uint16x4_t highbd_convolve8_wtd_4_s32_s16(
- const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
- const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
- const int16x4_t s6, const int16x4_t s7, const int16x8_t y_filter,
- const int32x4_t shift_s32, const int32x4_t offset, const int32x4_t weight,
- const int32x4_t offset2) {
- int32x4_t sum =
- highbd_convolve8_4_s32(s0, s1, s2, s3, s4, s5, s6, s7, y_filter, offset);
-
- sum = vqrshlq_s32(sum, shift_s32);
- sum = vmlaq_s32(offset2, sum, weight);
-
- return vqmovun_s32(sum);
-}
-
// Like above but also perform round shifting and subtract correction term
-static INLINE uint16x4_t highbd_convolve8_4_sr_s32_s16(
+static INLINE uint16x4_t highbd_convolve8_4_srsub_s32_s16(
const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
const int16x4_t s6, const int16x4_t s7, const int16x8_t y_filter,
@@ -176,41 +91,8 @@
*sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s7), y_filter_hi, 3);
}
-static INLINE uint16x8_t highbd_convolve8_8_s32_s16(
- const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
- const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
- const int16x8_t s6, const int16x8_t s7, const int16x8_t y_filter,
- const int32x4_t offset) {
- int32x4_t sum0;
- int32x4_t sum1;
- highbd_convolve8_8_s32(s0, s1, s2, s3, s4, s5, s6, s7, y_filter, offset,
- &sum0, &sum1);
-
- return vcombine_u16(vqrshrun_n_s32(sum0, COMPOUND_ROUND1_BITS),
- vqrshrun_n_s32(sum1, COMPOUND_ROUND1_BITS));
-}
-
-static INLINE uint16x8_t highbd_convolve8_wtd_8_s32_s16(
- const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
- const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
- const int16x8_t s6, const int16x8_t s7, const int16x8_t y_filter,
- const int32x4_t shift_s32, const int32x4_t offset, const int32x4_t weight,
- const int32x4_t offset2) {
- int32x4_t sum0;
- int32x4_t sum1;
- highbd_convolve8_8_s32(s0, s1, s2, s3, s4, s5, s6, s7, y_filter, offset,
- &sum0, &sum1);
-
- sum0 = vqrshlq_s32(sum0, shift_s32);
- sum1 = vqrshlq_s32(sum1, shift_s32);
- sum0 = vmlaq_s32(offset2, sum0, weight);
- sum1 = vmlaq_s32(offset2, sum1, weight);
-
- return vcombine_u16(vqmovun_s32(sum0), vqmovun_s32(sum1));
-}
-
// Like above but also perform round shifting and subtract correction term
-static INLINE uint16x8_t highbd_convolve8_8_sr_s32_s16(
+static INLINE uint16x8_t highbd_convolve8_8_srsub_s32_s16(
const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
const int16x8_t s6, const int16x8_t s7, const int16x8_t y_filter,
@@ -227,290 +109,6 @@
return vcombine_u16(vqmovun_s32(sum0), vqmovun_s32(sum1));
}
-static INLINE int32x4_t highbd_convolve12_y_4_s32(
- const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
- const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
- const int16x4_t s6, const int16x4_t s7, const int16x4_t s8,
- const int16x4_t s9, const int16x4_t s10, const int16x4_t s11,
- const int16x8_t y_filter_0_7, const int16x4_t y_filter_8_11,
- const int32x4_t offset) {
- const int16x4_t y_filter_0_3 = vget_low_s16(y_filter_0_7);
- const int16x4_t y_filter_4_7 = vget_high_s16(y_filter_0_7);
-
- int32x4_t sum = vmlal_lane_s16(offset, s0, y_filter_0_3, 0);
- sum = vmlal_lane_s16(sum, s1, y_filter_0_3, 1);
- sum = vmlal_lane_s16(sum, s2, y_filter_0_3, 2);
- sum = vmlal_lane_s16(sum, s3, y_filter_0_3, 3);
- sum = vmlal_lane_s16(sum, s4, y_filter_4_7, 0);
- sum = vmlal_lane_s16(sum, s5, y_filter_4_7, 1);
- sum = vmlal_lane_s16(sum, s6, y_filter_4_7, 2);
- sum = vmlal_lane_s16(sum, s7, y_filter_4_7, 3);
- sum = vmlal_lane_s16(sum, s8, y_filter_8_11, 0);
- sum = vmlal_lane_s16(sum, s9, y_filter_8_11, 1);
- sum = vmlal_lane_s16(sum, s10, y_filter_8_11, 2);
- sum = vmlal_lane_s16(sum, s11, y_filter_8_11, 3);
-
- return sum;
-}
-
-static INLINE uint16x4_t highbd_convolve12_y_4_s32_s16(
- const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
- const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
- const int16x4_t s6, const int16x4_t s7, const int16x4_t s8,
- const int16x4_t s9, const int16x4_t s10, const int16x4_t s11,
- const int16x8_t y_filter_0_7, const int16x4_t y_filter_8_11,
- const int32x4_t offset) {
- int32x4_t sum =
- highbd_convolve12_y_4_s32(s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10,
- s11, y_filter_0_7, y_filter_8_11, offset);
-
- return vqrshrun_n_s32(sum, COMPOUND_ROUND1_BITS);
-}
-
-// Like above but also perform round shifting and subtract correction term
-static INLINE uint16x4_t highbd_convolve12_y_4_sr_s32_s16(
- const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
- const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
- const int16x4_t s6, const int16x4_t s7, const int16x4_t s8,
- const int16x4_t s9, const int16x4_t s10, const int16x4_t s11,
- const int16x8_t y_filter_0_7, const int16x4_t y_filter_8_11,
- const int32x4_t round_shift, const int32x4_t offset,
- const int32x4_t correction) {
- int32x4_t sum =
- highbd_convolve12_y_4_s32(s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10,
- s11, y_filter_0_7, y_filter_8_11, offset);
-
- sum = vsubq_s32(vqrshlq_s32(sum, round_shift), correction);
- return vqmovun_s32(sum);
-}
-
-static INLINE void highbd_convolve12_y_8_s32(
- const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
- const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
- const int16x8_t s6, const int16x8_t s7, const int16x8_t s8,
- const int16x8_t s9, const int16x8_t s10, const int16x8_t s11,
- const int16x8_t y_filter_0_7, const int16x4_t y_filter_8_11,
- const int32x4_t offset, int32x4_t *sum0, int32x4_t *sum1) {
- const int16x4_t y_filter_0_3 = vget_low_s16(y_filter_0_7);
- const int16x4_t y_filter_4_7 = vget_high_s16(y_filter_0_7);
-
- *sum0 = vmlal_lane_s16(offset, vget_low_s16(s0), y_filter_0_3, 0);
- *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s1), y_filter_0_3, 1);
- *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s2), y_filter_0_3, 2);
- *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s3), y_filter_0_3, 3);
- *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s4), y_filter_4_7, 0);
- *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s5), y_filter_4_7, 1);
- *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s6), y_filter_4_7, 2);
- *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s7), y_filter_4_7, 3);
- *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s8), y_filter_8_11, 0);
- *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s9), y_filter_8_11, 1);
- *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s10), y_filter_8_11, 2);
- *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s11), y_filter_8_11, 3);
-
- *sum1 = vmlal_lane_s16(offset, vget_high_s16(s0), y_filter_0_3, 0);
- *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s1), y_filter_0_3, 1);
- *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s2), y_filter_0_3, 2);
- *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s3), y_filter_0_3, 3);
- *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s4), y_filter_4_7, 0);
- *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s5), y_filter_4_7, 1);
- *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s6), y_filter_4_7, 2);
- *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s7), y_filter_4_7, 3);
- *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s8), y_filter_8_11, 0);
- *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s9), y_filter_8_11, 1);
- *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s10), y_filter_8_11, 2);
- *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s11), y_filter_8_11, 3);
-}
-
-static INLINE uint16x8_t highbd_convolve12_y_8_s32_s16(
- const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
- const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
- const int16x8_t s6, const int16x8_t s7, const int16x8_t s8,
- const int16x8_t s9, const int16x8_t s10, const int16x8_t s11,
- const int16x8_t y_filter_0_7, const int16x4_t y_filter_8_11,
- const int32x4_t offset) {
- int32x4_t sum0;
- int32x4_t sum1;
- highbd_convolve12_y_8_s32(s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11,
- y_filter_0_7, y_filter_8_11, offset, &sum0, &sum1);
-
- return vcombine_u16(vqrshrun_n_s32(sum0, COMPOUND_ROUND1_BITS),
- vqrshrun_n_s32(sum1, COMPOUND_ROUND1_BITS));
-}
-
-// Like above but also perform round shifting and subtract correction term
-static INLINE uint16x8_t highbd_convolve12_y_8_sr_s32_s16(
- const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
- const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
- const int16x8_t s6, const int16x8_t s7, const int16x8_t s8,
- const int16x8_t s9, const int16x8_t s10, const int16x8_t s11,
- const int16x8_t y_filter_0_7, const int16x4_t y_filter_8_11,
- const int32x4_t round_shift, const int32x4_t offset,
- const int32x4_t correction) {
- int32x4_t sum0;
- int32x4_t sum1;
- highbd_convolve12_y_8_s32(s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11,
- y_filter_0_7, y_filter_8_11, offset, &sum0, &sum1);
-
- sum0 = vsubq_s32(vqrshlq_s32(sum0, round_shift), correction);
- sum1 = vsubq_s32(vqrshlq_s32(sum1, round_shift), correction);
-
- return vcombine_u16(vqmovun_s32(sum0), vqmovun_s32(sum1));
-}
-
-static INLINE int32x4_t highbd_convolve8_horiz4_s32(
- const int16x8_t s0, const int16x8_t s1, const int16x8_t x_filter_0_7,
- const int32x4_t offset) {
- const int16x8_t s2 = vextq_s16(s0, s1, 1);
- const int16x8_t s3 = vextq_s16(s0, s1, 2);
- const int16x8_t s4 = vextq_s16(s0, s1, 3);
- const int16x4_t s0_lo = vget_low_s16(s0);
- const int16x4_t s1_lo = vget_low_s16(s2);
- const int16x4_t s2_lo = vget_low_s16(s3);
- const int16x4_t s3_lo = vget_low_s16(s4);
- const int16x4_t s4_lo = vget_high_s16(s0);
- const int16x4_t s5_lo = vget_high_s16(s2);
- const int16x4_t s6_lo = vget_high_s16(s3);
- const int16x4_t s7_lo = vget_high_s16(s4);
-
- return highbd_convolve8_4_s32(s0_lo, s1_lo, s2_lo, s3_lo, s4_lo, s5_lo, s6_lo,
- s7_lo, x_filter_0_7, offset);
-}
-
-static INLINE uint16x4_t highbd_convolve8_horiz4_s32_s16(
- const int16x8_t s0, const int16x8_t s1, const int16x8_t x_filter_0_7,
- const int32x4_t shift_s32, const int32x4_t offset) {
- int32x4_t sum = highbd_convolve8_horiz4_s32(s0, s1, x_filter_0_7, offset);
-
- sum = vqrshlq_s32(sum, shift_s32);
- return vqmovun_s32(sum);
-}
-
-static INLINE uint16x4_t highbd_convolve8_wtd_horiz4_s32_s16(
- const int16x8_t s0, const int16x8_t s1, const int16x8_t x_filter_0_7,
- const int32x4_t shift_s32, const int32x4_t offset, const int32x4_t weight,
- const int32x4_t offset2) {
- int32x4_t sum = highbd_convolve8_horiz4_s32(s0, s1, x_filter_0_7, offset);
-
- sum = vqrshlq_s32(sum, shift_s32);
- sum = vmlaq_s32(offset2, sum, weight);
- return vqmovun_s32(sum);
-}
-
-static INLINE void highbd_convolve8_horiz8_s32(
- const int16x8_t s0, const int16x8_t s0_hi, const int16x8_t x_filter_0_7,
- const int32x4_t offset, int32x4_t *sum0, int32x4_t *sum1) {
- const int16x8_t s1 = vextq_s16(s0, s0_hi, 1);
- const int16x8_t s2 = vextq_s16(s0, s0_hi, 2);
- const int16x8_t s3 = vextq_s16(s0, s0_hi, 3);
- const int16x8_t s4 = vextq_s16(s0, s0_hi, 4);
- const int16x8_t s5 = vextq_s16(s0, s0_hi, 5);
- const int16x8_t s6 = vextq_s16(s0, s0_hi, 6);
- const int16x8_t s7 = vextq_s16(s0, s0_hi, 7);
-
- highbd_convolve8_8_s32(s0, s1, s2, s3, s4, s5, s6, s7, x_filter_0_7, offset,
- sum0, sum1);
-}
-
-static INLINE uint16x8_t highbd_convolve8_horiz8_s32_s16(
- const int16x8_t s0, const int16x8_t s1, const int16x8_t x_filter_0_7,
- const int32x4_t shift_s32, const int32x4_t offset) {
- int32x4_t sum0, sum1;
- highbd_convolve8_horiz8_s32(s0, s1, x_filter_0_7, offset, &sum0, &sum1);
-
- sum0 = vqrshlq_s32(sum0, shift_s32);
- sum1 = vqrshlq_s32(sum1, shift_s32);
-
- return vcombine_u16(vqmovun_s32(sum0), vqmovun_s32(sum1));
-}
-
-static INLINE uint16x8_t highbd_convolve8_wtd_horiz8_s32_s16(
- const int16x8_t s0, const int16x8_t s1, const int16x8_t x_filter_0_7,
- const int32x4_t shift_s32, const int32x4_t offset, const int32x4_t weight,
- const int32x4_t offset2) {
- int32x4_t sum0, sum1;
- highbd_convolve8_horiz8_s32(s0, s1, x_filter_0_7, offset, &sum0, &sum1);
-
- sum0 = vqrshlq_s32(sum0, shift_s32);
- sum1 = vqrshlq_s32(sum1, shift_s32);
- sum0 = vmlaq_s32(offset2, sum0, weight);
- sum1 = vmlaq_s32(offset2, sum1, weight);
-
- return vcombine_u16(vqmovun_s32(sum0), vqmovun_s32(sum1));
-}
-
-static INLINE int32x4_t highbd_convolve12_horiz4_s32(
- const int16x8_t s0, const int16x8_t s1, const int16x8_t x_filter_0_7,
- const int16x4_t x_filter_8_11, const int32x4_t offset) {
- const int16x8_t s2 = vextq_s16(s0, s1, 1);
- const int16x8_t s3 = vextq_s16(s0, s1, 2);
- const int16x8_t s4 = vextq_s16(s0, s1, 3);
- const int16x8_t s5 = vextq_s16(s0, s1, 4);
- const int16x8_t s6 = vextq_s16(s0, s1, 5);
- const int16x8_t s7 = vextq_s16(s0, s1, 6);
- const int16x8_t s8 = vextq_s16(s0, s1, 7);
- const int16x4_t s0_lo = vget_low_s16(s0);
- const int16x4_t s1_lo = vget_low_s16(s2);
- const int16x4_t s2_lo = vget_low_s16(s3);
- const int16x4_t s3_lo = vget_low_s16(s4);
- const int16x4_t s4_lo = vget_high_s16(s0);
- const int16x4_t s5_lo = vget_high_s16(s2);
- const int16x4_t s6_lo = vget_high_s16(s3);
- const int16x4_t s7_lo = vget_high_s16(s4);
- const int16x4_t s8_lo = vget_high_s16(s5);
- const int16x4_t s9_lo = vget_high_s16(s6);
- const int16x4_t s10_lo = vget_high_s16(s7);
- const int16x4_t s11_lo = vget_high_s16(s8);
-
- return highbd_convolve12_y_4_s32(s0_lo, s1_lo, s2_lo, s3_lo, s4_lo, s5_lo,
- s6_lo, s7_lo, s8_lo, s9_lo, s10_lo, s11_lo,
- x_filter_0_7, x_filter_8_11, offset);
-}
-
-static INLINE uint16x4_t highbd_convolve12_horiz4_s32_s16(
- const int16x8_t s0, const int16x8_t s1, const int16x8_t x_filter_0_7,
- const int16x4_t x_filter_8_11, const int32x4_t shift_s32,
- const int32x4_t offset) {
- int32x4_t sum =
- highbd_convolve12_horiz4_s32(s0, s1, x_filter_0_7, x_filter_8_11, offset);
-
- sum = vqrshlq_s32(sum, shift_s32);
- return vqmovun_s32(sum);
-}
-
-static INLINE void highbd_convolve12_horiz8_s32(
- const int16x8_t s0_0, const int16x8_t s0_1, const int16x8_t s0_2,
- const int16x8_t x_filter_0_7, const int16x4_t x_filter_8_11,
- const int32x4_t offset, int32x4_t *sum0, int32x4_t *sum1) {
- const int16x8_t s1 = vextq_s16(s0_0, s0_1, 1);
- const int16x8_t s2 = vextq_s16(s0_0, s0_1, 2);
- const int16x8_t s3 = vextq_s16(s0_0, s0_1, 3);
- const int16x8_t s4 = vextq_s16(s0_0, s0_1, 4);
- const int16x8_t s5 = vextq_s16(s0_0, s0_1, 5);
- const int16x8_t s6 = vextq_s16(s0_0, s0_1, 6);
- const int16x8_t s7 = vextq_s16(s0_0, s0_1, 7);
- const int16x8_t s8 = s0_1;
- const int16x8_t s9 = vextq_s16(s0_1, s0_2, 1);
- const int16x8_t s10 = vextq_s16(s0_1, s0_2, 2);
- const int16x8_t s11 = vextq_s16(s0_1, s0_2, 3);
-
- highbd_convolve12_y_8_s32(s0_0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11,
- x_filter_0_7, x_filter_8_11, offset, sum0, sum1);
-}
-
-static INLINE uint16x8_t highbd_convolve12_horiz8_s32_s16(
- const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
- const int16x8_t x_filter_0_7, const int16x4_t x_filter_8_11,
- const int32x4_t shift_s32, const int32x4_t offset) {
- int32x4_t sum0, sum1;
- highbd_convolve12_horiz8_s32(s0, s1, s2, x_filter_0_7, x_filter_8_11, offset,
- &sum0, &sum1);
-
- sum0 = vqrshlq_s32(sum0, shift_s32);
- sum1 = vqrshlq_s32(sum1, shift_s32);
-
- return vcombine_u16(vqmovun_s32(sum0), vqmovun_s32(sum1));
-}
-
static INLINE int32x4_t highbd_convolve8_2d_scale_horiz4x8_s32(
const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
const int16x8_t s3, const int16x4_t *filters_lo,
@@ -520,8 +118,8 @@
int16x4_t s_hi[] = { vget_high_s16(s0), vget_high_s16(s1), vget_high_s16(s2),
vget_high_s16(s3) };
- transpose_u16_4x4((uint16x4_t *)s_lo);
- transpose_u16_4x4((uint16x4_t *)s_hi);
+ transpose_array_inplace_u16_4x4((uint16x4_t *)s_lo);
+ transpose_array_inplace_u16_4x4((uint16x4_t *)s_hi);
int32x4_t sum = vmlal_s16(offset, s_lo[0], filters_lo[0]);
sum = vmlal_s16(sum, s_lo[1], filters_lo[1]);
diff --git a/av1/common/arm/highbd_convolve_scale_neon.c b/av1/common/arm/highbd_convolve_scale_neon.c
new file mode 100644
index 0000000..eee5a1c
--- /dev/null
+++ b/av1/common/arm/highbd_convolve_scale_neon.c
@@ -0,0 +1,552 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <assert.h>
+#include <arm_neon.h>
+
+#include "config/aom_config.h"
+#include "config/av1_rtcd.h"
+
+#include "aom_dsp/aom_dsp_common.h"
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_dsp/arm/transpose_neon.h"
+#include "aom_ports/mem.h"
+#include "av1/common/convolve.h"
+#include "av1/common/filter.h"
+#include "av1/common/arm/highbd_convolve_neon.h"
+
+static INLINE void highbd_dist_wtd_comp_avg_neon(
+ const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
+ int w, int h, ConvolveParams *conv_params, const int round_bits,
+ const int offset, const int bd) {
+ CONV_BUF_TYPE *ref_ptr = conv_params->dst;
+ const int ref_stride = conv_params->dst_stride;
+ const int32x4_t round_shift = vdupq_n_s32(-round_bits);
+ const uint32x4_t offset_vec = vdupq_n_u32(offset);
+ const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
+ uint16x4_t fwd_offset = vdup_n_u16(conv_params->fwd_offset);
+ uint16x4_t bck_offset = vdup_n_u16(conv_params->bck_offset);
+
+ // Weighted averaging
+ if (w <= 4) {
+ do {
+ const uint16x4_t src = vld1_u16(src_ptr);
+ const uint16x4_t ref = vld1_u16(ref_ptr);
+
+ uint32x4_t wtd_avg = vmull_u16(ref, fwd_offset);
+ wtd_avg = vmlal_u16(wtd_avg, src, bck_offset);
+ wtd_avg = vshrq_n_u32(wtd_avg, DIST_PRECISION_BITS);
+ int32x4_t d0 = vreinterpretq_s32_u32(vsubq_u32(wtd_avg, offset_vec));
+ d0 = vqrshlq_s32(d0, round_shift);
+
+ uint16x4_t d0_u16 = vqmovun_s32(d0);
+ d0_u16 = vmin_u16(d0_u16, vget_low_u16(max));
+
+ if (w == 2) {
+ store_u16_2x1(dst_ptr, d0_u16, 0);
+ } else {
+ vst1_u16(dst_ptr, d0_u16);
+ }
+
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ ref_ptr += ref_stride;
+ } while (--h != 0);
+ } else {
+ do {
+ int width = w;
+ const uint16_t *src = src_ptr;
+ const uint16_t *ref = ref_ptr;
+ uint16_t *dst = dst_ptr;
+ do {
+ const uint16x8_t s = vld1q_u16(src);
+ const uint16x8_t r = vld1q_u16(ref);
+
+ uint32x4_t wtd_avg0 = vmull_u16(vget_low_u16(r), fwd_offset);
+ wtd_avg0 = vmlal_u16(wtd_avg0, vget_low_u16(s), bck_offset);
+ wtd_avg0 = vshrq_n_u32(wtd_avg0, DIST_PRECISION_BITS);
+ int32x4_t d0 = vreinterpretq_s32_u32(vsubq_u32(wtd_avg0, offset_vec));
+ d0 = vqrshlq_s32(d0, round_shift);
+
+ uint32x4_t wtd_avg1 = vmull_u16(vget_high_u16(r), fwd_offset);
+ wtd_avg1 = vmlal_u16(wtd_avg1, vget_high_u16(s), bck_offset);
+ wtd_avg1 = vshrq_n_u32(wtd_avg1, DIST_PRECISION_BITS);
+ int32x4_t d1 = vreinterpretq_s32_u32(vsubq_u32(wtd_avg1, offset_vec));
+ d1 = vqrshlq_s32(d1, round_shift);
+
+ uint16x8_t d01 = vcombine_u16(vqmovun_s32(d0), vqmovun_s32(d1));
+ d01 = vminq_u16(d01, max);
+ vst1q_u16(dst, d01);
+
+ src += 8;
+ ref += 8;
+ dst += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ ref_ptr += ref_stride;
+ } while (--h != 0);
+ }
+}
+
+static INLINE void highbd_comp_avg_neon(const uint16_t *src_ptr, int src_stride,
+ uint16_t *dst_ptr, int dst_stride,
+ int w, int h,
+ ConvolveParams *conv_params,
+ const int round_bits, const int offset,
+ const int bd) {
+ CONV_BUF_TYPE *ref_ptr = conv_params->dst;
+ const int ref_stride = conv_params->dst_stride;
+ const int32x4_t round_shift = vdupq_n_s32(-round_bits);
+ const uint16x4_t offset_vec = vdup_n_u16(offset);
+ const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
+
+ if (w <= 4) {
+ do {
+ const uint16x4_t src = vld1_u16(src_ptr);
+ const uint16x4_t ref = vld1_u16(ref_ptr);
+
+ uint16x4_t avg = vhadd_u16(src, ref);
+ int32x4_t d0 = vreinterpretq_s32_u32(vsubl_u16(avg, offset_vec));
+ d0 = vqrshlq_s32(d0, round_shift);
+
+ uint16x4_t d0_u16 = vqmovun_s32(d0);
+ d0_u16 = vmin_u16(d0_u16, vget_low_u16(max));
+
+ if (w == 2) {
+ store_u16_2x1(dst_ptr, d0_u16, 0);
+ } else {
+ vst1_u16(dst_ptr, d0_u16);
+ }
+
+ src_ptr += src_stride;
+ ref_ptr += ref_stride;
+ dst_ptr += dst_stride;
+ } while (--h != 0);
+ } else {
+ do {
+ int width = w;
+ const uint16_t *src = src_ptr;
+ const uint16_t *ref = ref_ptr;
+ uint16_t *dst = dst_ptr;
+ do {
+ const uint16x8_t s = vld1q_u16(src);
+ const uint16x8_t r = vld1q_u16(ref);
+
+ uint16x8_t avg = vhaddq_u16(s, r);
+ int32x4_t d0_lo =
+ vreinterpretq_s32_u32(vsubl_u16(vget_low_u16(avg), offset_vec));
+ int32x4_t d0_hi =
+ vreinterpretq_s32_u32(vsubl_u16(vget_high_u16(avg), offset_vec));
+ d0_lo = vqrshlq_s32(d0_lo, round_shift);
+ d0_hi = vqrshlq_s32(d0_hi, round_shift);
+
+ uint16x8_t d0 = vcombine_u16(vqmovun_s32(d0_lo), vqmovun_s32(d0_hi));
+ d0 = vminq_u16(d0, max);
+ vst1q_u16(dst, d0);
+
+ src += 8;
+ ref += 8;
+ dst += 8;
+ width -= 8;
+ } while (width != 0);
+
+ src_ptr += src_stride;
+ ref_ptr += ref_stride;
+ dst_ptr += dst_stride;
+ } while (--h != 0);
+ }
+}
+
+static INLINE void highbd_convolve_2d_x_scale_8tap_neon(
+ const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
+ int w, int h, const int subpel_x_qn, const int x_step_qn,
+ const InterpFilterParams *filter_params, ConvolveParams *conv_params,
+ const int offset) {
+ static const uint32_t kIdx[4] = { 0, 1, 2, 3 };
+ const uint32x4_t idx = vld1q_u32(kIdx);
+ const uint32x4_t subpel_mask = vdupq_n_u32(SCALE_SUBPEL_MASK);
+ const int32x4_t shift_s32 = vdupq_n_s32(-conv_params->round_0);
+ const int32x4_t offset_s32 = vdupq_n_s32(offset);
+
+ if (w <= 4) {
+ int height = h;
+ uint16_t *d = dst_ptr;
+
+ do {
+ int x_qn = subpel_x_qn;
+
+ // Load 4 src vectors at a time, they might be the same, but we have to
+ // calculate the indices anyway. Doing it in SIMD and then storing the
+ // indices is faster than having to calculate the expression
+ // &src_ptr[((x_qn + 0*x_step_qn) >> SCALE_SUBPEL_BITS)] 4 times
+ // Ideally this should be a gather using the indices, but NEON does not
+ // have that, so have to emulate
+ const uint32x4_t xqn_idx = vmlaq_n_u32(vdupq_n_u32(x_qn), idx, x_step_qn);
+ // We have to multiply x2 to get the actual pointer as sizeof(uint16_t) =
+ // 2
+ const uint32x4_t src_idx_u32 =
+ vshlq_n_u32(vshrq_n_u32(xqn_idx, SCALE_SUBPEL_BITS), 1);
+#if AOM_ARCH_AARCH64
+ uint64x2_t src4[2];
+ src4[0] = vaddw_u32(vdupq_n_u64((const uint64_t)src_ptr),
+ vget_low_u32(src_idx_u32));
+ src4[1] = vaddw_u32(vdupq_n_u64((const uint64_t)src_ptr),
+ vget_high_u32(src_idx_u32));
+ int16_t *src4_ptr[4];
+ uint64_t *tmp_ptr = (uint64_t *)&src4_ptr;
+ vst1q_u64(tmp_ptr, src4[0]);
+ vst1q_u64(tmp_ptr + 2, src4[1]);
+#else
+ uint32x4_t src4;
+ src4 = vaddq_u32(vdupq_n_u32((const uint32_t)src_ptr), src_idx_u32);
+ int16_t *src4_ptr[4];
+ uint32_t *tmp_ptr = (uint32_t *)&src4_ptr;
+ vst1q_u32(tmp_ptr, src4);
+#endif // AOM_ARCH_AARCH64
+ // Same for the filter vectors
+ const int32x4_t filter_idx_s32 = vreinterpretq_s32_u32(
+ vshrq_n_u32(vandq_u32(xqn_idx, subpel_mask), SCALE_EXTRA_BITS));
+ int32_t x_filter4_idx[4];
+ vst1q_s32(x_filter4_idx, filter_idx_s32);
+ const int16_t *x_filter4_ptr[4];
+
+ // Load source
+ int16x8_t s0 = vld1q_s16(src4_ptr[0]);
+ int16x8_t s1 = vld1q_s16(src4_ptr[1]);
+ int16x8_t s2 = vld1q_s16(src4_ptr[2]);
+ int16x8_t s3 = vld1q_s16(src4_ptr[3]);
+
+ // We could easily do this using SIMD as well instead of calling the
+ // inline function 4 times.
+ x_filter4_ptr[0] =
+ av1_get_interp_filter_subpel_kernel(filter_params, x_filter4_idx[0]);
+ x_filter4_ptr[1] =
+ av1_get_interp_filter_subpel_kernel(filter_params, x_filter4_idx[1]);
+ x_filter4_ptr[2] =
+ av1_get_interp_filter_subpel_kernel(filter_params, x_filter4_idx[2]);
+ x_filter4_ptr[3] =
+ av1_get_interp_filter_subpel_kernel(filter_params, x_filter4_idx[3]);
+
+ // Actually load the filters
+ const int16x8_t x_filter0 = vld1q_s16(x_filter4_ptr[0]);
+ const int16x8_t x_filter1 = vld1q_s16(x_filter4_ptr[1]);
+ const int16x8_t x_filter2 = vld1q_s16(x_filter4_ptr[2]);
+ const int16x8_t x_filter3 = vld1q_s16(x_filter4_ptr[3]);
+
+ // Group low and high parts and transpose
+ int16x4_t filters_lo[] = { vget_low_s16(x_filter0),
+ vget_low_s16(x_filter1),
+ vget_low_s16(x_filter2),
+ vget_low_s16(x_filter3) };
+ int16x4_t filters_hi[] = { vget_high_s16(x_filter0),
+ vget_high_s16(x_filter1),
+ vget_high_s16(x_filter2),
+ vget_high_s16(x_filter3) };
+ transpose_array_inplace_u16_4x4((uint16x4_t *)filters_lo);
+ transpose_array_inplace_u16_4x4((uint16x4_t *)filters_hi);
+
+ // Run the 2D Scale convolution
+ uint16x4_t d0 = highbd_convolve8_2d_scale_horiz4x8_s32_s16(
+ s0, s1, s2, s3, filters_lo, filters_hi, shift_s32, offset_s32);
+
+ if (w == 2) {
+ store_u16_2x1(d + 0 * dst_stride, d0, 0);
+ } else {
+ vst1_u16(d + 0 * dst_stride, d0);
+ }
+
+ src_ptr += src_stride;
+ d += dst_stride;
+ height--;
+ } while (height > 0);
+ } else {
+ int height = h;
+
+ do {
+ int width = w;
+ int x_qn = subpel_x_qn;
+ uint16_t *d = dst_ptr;
+ const uint16_t *s = src_ptr;
+
+ do {
+ // Load 4 src vectors at a time, they might be the same, but we have to
+ // calculate the indices anyway. Doing it in SIMD and then storing the
+ // indices is faster than having to calculate the expression
+ // &src_ptr[((x_qn + 0*x_step_qn) >> SCALE_SUBPEL_BITS)] 4 times
+ // Ideally this should be a gather using the indices, but NEON does not
+ // have that, so have to emulate
+ const uint32x4_t xqn_idx =
+ vmlaq_n_u32(vdupq_n_u32(x_qn), idx, x_step_qn);
+ // We have to multiply x2 to get the actual pointer as sizeof(uint16_t)
+ // = 2
+ const uint32x4_t src_idx_u32 =
+ vshlq_n_u32(vshrq_n_u32(xqn_idx, SCALE_SUBPEL_BITS), 1);
+#if AOM_ARCH_AARCH64
+ uint64x2_t src4[2];
+ src4[0] = vaddw_u32(vdupq_n_u64((const uint64_t)s),
+ vget_low_u32(src_idx_u32));
+ src4[1] = vaddw_u32(vdupq_n_u64((const uint64_t)s),
+ vget_high_u32(src_idx_u32));
+ int16_t *src4_ptr[4];
+ uint64_t *tmp_ptr = (uint64_t *)&src4_ptr;
+ vst1q_u64(tmp_ptr, src4[0]);
+ vst1q_u64(tmp_ptr + 2, src4[1]);
+#else
+ uint32x4_t src4;
+ src4 = vaddq_u32(vdupq_n_u32((const uint32_t)s), src_idx_u32);
+ int16_t *src4_ptr[4];
+ uint32_t *tmp_ptr = (uint32_t *)&src4_ptr;
+ vst1q_u32(tmp_ptr, src4);
+#endif // AOM_ARCH_AARCH64
+ // Same for the filter vectors
+ const int32x4_t filter_idx_s32 = vreinterpretq_s32_u32(
+ vshrq_n_u32(vandq_u32(xqn_idx, subpel_mask), SCALE_EXTRA_BITS));
+ int32_t x_filter4_idx[4];
+ vst1q_s32(x_filter4_idx, filter_idx_s32);
+ const int16_t *x_filter4_ptr[4];
+
+ // Load source
+ int16x8_t s0 = vld1q_s16(src4_ptr[0]);
+ int16x8_t s1 = vld1q_s16(src4_ptr[1]);
+ int16x8_t s2 = vld1q_s16(src4_ptr[2]);
+ int16x8_t s3 = vld1q_s16(src4_ptr[3]);
+
+ // We could easily do this using SIMD as well instead of calling the
+ // inline function 4 times.
+ x_filter4_ptr[0] = av1_get_interp_filter_subpel_kernel(
+ filter_params, x_filter4_idx[0]);
+ x_filter4_ptr[1] = av1_get_interp_filter_subpel_kernel(
+ filter_params, x_filter4_idx[1]);
+ x_filter4_ptr[2] = av1_get_interp_filter_subpel_kernel(
+ filter_params, x_filter4_idx[2]);
+ x_filter4_ptr[3] = av1_get_interp_filter_subpel_kernel(
+ filter_params, x_filter4_idx[3]);
+
+ // Actually load the filters
+ const int16x8_t x_filter0 = vld1q_s16(x_filter4_ptr[0]);
+ const int16x8_t x_filter1 = vld1q_s16(x_filter4_ptr[1]);
+ const int16x8_t x_filter2 = vld1q_s16(x_filter4_ptr[2]);
+ const int16x8_t x_filter3 = vld1q_s16(x_filter4_ptr[3]);
+
+ // Group low and high parts and transpose
+ int16x4_t filters_lo[] = { vget_low_s16(x_filter0),
+ vget_low_s16(x_filter1),
+ vget_low_s16(x_filter2),
+ vget_low_s16(x_filter3) };
+ int16x4_t filters_hi[] = { vget_high_s16(x_filter0),
+ vget_high_s16(x_filter1),
+ vget_high_s16(x_filter2),
+ vget_high_s16(x_filter3) };
+ transpose_array_inplace_u16_4x4((uint16x4_t *)filters_lo);
+ transpose_array_inplace_u16_4x4((uint16x4_t *)filters_hi);
+
+ // Run the 2D Scale X convolution
+ uint16x4_t d0 = highbd_convolve8_2d_scale_horiz4x8_s32_s16(
+ s0, s1, s2, s3, filters_lo, filters_hi, shift_s32, offset_s32);
+
+ vst1_u16(d, d0);
+
+ x_qn += 4 * x_step_qn;
+ d += 4;
+ width -= 4;
+ } while (width > 0);
+
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ height--;
+ } while (height > 0);
+ }
+}
+
+static INLINE void highbd_convolve_2d_y_scale_8tap_neon(
+ const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
+ int w, int h, const int subpel_y_qn, const int y_step_qn,
+ const InterpFilterParams *filter_params, const int round1_bits,
+ const int offset) {
+ const int32x4_t offset_s32 = vdupq_n_s32(1 << offset);
+
+ const int32x4_t round1_shift_s32 = vdupq_n_s32(-round1_bits);
+ if (w <= 4) {
+ int height = h;
+ uint16_t *d = dst_ptr;
+ int y_qn = subpel_y_qn;
+
+ do {
+ const int16_t *s =
+ (const int16_t *)&src_ptr[(y_qn >> SCALE_SUBPEL_BITS) * src_stride];
+
+ int16x4_t s0, s1, s2, s3, s4, s5, s6, s7;
+ load_s16_4x8(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7);
+
+ const int y_filter_idx = (y_qn & SCALE_SUBPEL_MASK) >> SCALE_EXTRA_BITS;
+ const int16_t *y_filter_ptr =
+ av1_get_interp_filter_subpel_kernel(filter_params, y_filter_idx);
+ const int16x8_t y_filter = vld1q_s16(y_filter_ptr);
+
+ uint16x4_t d0 = highbd_convolve8_4_srsub_s32_s16(
+ s0, s1, s2, s3, s4, s5, s6, s7, y_filter, round1_shift_s32,
+ offset_s32, vdupq_n_s32(0));
+
+ if (w == 2) {
+ store_u16_2x1(d, d0, 0);
+ } else {
+ vst1_u16(d, d0);
+ }
+
+ y_qn += y_step_qn;
+ d += dst_stride;
+ height--;
+ } while (height > 0);
+ } else {
+ int width = w;
+
+ do {
+ int height = h;
+ int y_qn = subpel_y_qn;
+
+ uint16_t *d = dst_ptr;
+
+ do {
+ const int16_t *s =
+ (const int16_t *)&src_ptr[(y_qn >> SCALE_SUBPEL_BITS) * src_stride];
+ int16x8_t s0, s1, s2, s3, s4, s5, s6, s7;
+ load_s16_8x8(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7);
+
+ const int y_filter_idx = (y_qn & SCALE_SUBPEL_MASK) >> SCALE_EXTRA_BITS;
+ const int16_t *y_filter_ptr =
+ av1_get_interp_filter_subpel_kernel(filter_params, y_filter_idx);
+ const int16x8_t y_filter = vld1q_s16(y_filter_ptr);
+
+ uint16x8_t d0 = highbd_convolve8_8_srsub_s32_s16(
+ s0, s1, s2, s3, s4, s5, s6, s7, y_filter, round1_shift_s32,
+ offset_s32, vdupq_n_s32(0));
+ vst1q_u16(d, d0);
+
+ y_qn += y_step_qn;
+ d += dst_stride;
+ height--;
+ } while (height > 0);
+ src_ptr += 8;
+ dst_ptr += 8;
+ width -= 8;
+ } while (width > 0);
+ }
+}
+
+static INLINE void highbd_convolve_correct_offset_neon(
+ const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
+ int w, int h, const int round_bits, const int offset, const int bd) {
+ const int32x4_t round_shift_s32 = vdupq_n_s32(-round_bits);
+ const int16x4_t offset_s16 = vdup_n_s16(offset);
+ const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
+
+ if (w <= 4) {
+ for (int y = 0; y < h; ++y) {
+ const int16x4_t s = vld1_s16((const int16_t *)src_ptr + y * src_stride);
+ const int32x4_t d0 =
+ vqrshlq_s32(vsubl_s16(s, offset_s16), round_shift_s32);
+ uint16x4_t d = vqmovun_s32(d0);
+ d = vmin_u16(d, vget_low_u16(max));
+ if (w == 2) {
+ store_u16_2x1(dst_ptr + y * dst_stride, d, 0);
+ } else {
+ vst1_u16(dst_ptr + y * dst_stride, d);
+ }
+ }
+ } else {
+ for (int y = 0; y < h; ++y) {
+ for (int x = 0; x < w; x += 8) {
+ // Subtract round offset and convolve round
+ const int16x8_t s =
+ vld1q_s16((const int16_t *)src_ptr + y * src_stride + x);
+ const int32x4_t d0 = vqrshlq_s32(vsubl_s16(vget_low_s16(s), offset_s16),
+ round_shift_s32);
+ const int32x4_t d1 = vqrshlq_s32(
+ vsubl_s16(vget_high_s16(s), offset_s16), round_shift_s32);
+ uint16x8_t d01 = vcombine_u16(vqmovun_s32(d0), vqmovun_s32(d1));
+ d01 = vminq_u16(d01, max);
+ vst1q_u16(dst_ptr + y * dst_stride + x, d01);
+ }
+ }
+ }
+}
+
+void av1_highbd_convolve_2d_scale_neon(
+ const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w,
+ int h, const InterpFilterParams *filter_params_x,
+ const InterpFilterParams *filter_params_y, const int subpel_x_qn,
+ const int x_step_qn, const int subpel_y_qn, const int y_step_qn,
+ ConvolveParams *conv_params, int bd) {
+ uint16_t *im_block = (uint16_t *)aom_memalign(
+ 16, 2 * sizeof(uint16_t) * MAX_SB_SIZE * (MAX_SB_SIZE + MAX_FILTER_TAP));
+ if (!im_block) return;
+ uint16_t *im_block2 = (uint16_t *)aom_memalign(
+ 16, 2 * sizeof(uint16_t) * MAX_SB_SIZE * (MAX_SB_SIZE + MAX_FILTER_TAP));
+ if (!im_block2) {
+ aom_free(im_block); // free the first block and return.
+ return;
+ }
+
+ int im_h = (((h - 1) * y_step_qn + subpel_y_qn) >> SCALE_SUBPEL_BITS) +
+ filter_params_y->taps;
+ const int im_stride = MAX_SB_SIZE;
+ const int bits =
+ FILTER_BITS * 2 - conv_params->round_0 - conv_params->round_1;
+ assert(bits >= 0);
+
+ const int vert_offset = filter_params_y->taps / 2 - 1;
+ const int horiz_offset = filter_params_x->taps / 2 - 1;
+ const int x_offset_bits = (1 << (bd + FILTER_BITS - 1));
+ const int y_offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0;
+ const int y_offset_correction =
+ ((1 << (y_offset_bits - conv_params->round_1)) +
+ (1 << (y_offset_bits - conv_params->round_1 - 1)));
+
+ CONV_BUF_TYPE *dst16 = conv_params->dst;
+ const int dst16_stride = conv_params->dst_stride;
+
+ const uint16_t *src_ptr = src - vert_offset * src_stride - horiz_offset;
+
+ highbd_convolve_2d_x_scale_8tap_neon(
+ src_ptr, src_stride, im_block, im_stride, w, im_h, subpel_x_qn, x_step_qn,
+ filter_params_x, conv_params, x_offset_bits);
+ if (conv_params->is_compound && !conv_params->do_average) {
+ highbd_convolve_2d_y_scale_8tap_neon(
+ im_block, im_stride, dst16, dst16_stride, w, h, subpel_y_qn, y_step_qn,
+ filter_params_y, conv_params->round_1, y_offset_bits);
+ } else {
+ highbd_convolve_2d_y_scale_8tap_neon(
+ im_block, im_stride, im_block2, im_stride, w, h, subpel_y_qn, y_step_qn,
+ filter_params_y, conv_params->round_1, y_offset_bits);
+ }
+
+ // Do the compound averaging outside the loop, avoids branching within the
+ // main loop
+ if (conv_params->is_compound) {
+ if (conv_params->do_average) {
+ if (conv_params->use_dist_wtd_comp_avg) {
+ highbd_dist_wtd_comp_avg_neon(im_block2, im_stride, dst, dst_stride, w,
+ h, conv_params, bits, y_offset_correction,
+ bd);
+ } else {
+ highbd_comp_avg_neon(im_block2, im_stride, dst, dst_stride, w, h,
+ conv_params, bits, y_offset_correction, bd);
+ }
+ }
+ } else {
+ highbd_convolve_correct_offset_neon(im_block2, im_stride, dst, dst_stride,
+ w, h, bits, y_offset_correction, bd);
+ }
+ aom_free(im_block);
+ aom_free(im_block2);
+}
diff --git a/av1/common/arm/highbd_inv_txfm_neon.c b/av1/common/arm/highbd_inv_txfm_neon.c
index d197fca..84bc8fd 100644
--- a/av1/common/arm/highbd_inv_txfm_neon.c
+++ b/av1/common/arm/highbd_inv_txfm_neon.c
@@ -590,7 +590,7 @@
int bd, int out_shift) {
const int32_t *sinpi = sinpi_arr(bit);
const int32x4_t zero = vdupq_n_s32(0);
- int64x2_t rnding = vdupq_n_s64(1 << (bit + 4 - 1));
+ int64x2_t rnding = vdupq_n_s64(1ll << (bit + 4 - 1));
const int32x2_t mul = vdup_n_s32(1 << 4);
int32x4_t t;
int32x4_t s0, s1, s2, s3, s4, s5, s6, s7;
diff --git a/av1/common/arm/highbd_reconinter_neon.c b/av1/common/arm/highbd_reconinter_neon.c
new file mode 100644
index 0000000..573d3c1
--- /dev/null
+++ b/av1/common/arm/highbd_reconinter_neon.c
@@ -0,0 +1,330 @@
+/*
+ *
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+#include <assert.h>
+#include <stdbool.h>
+
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_dsp/blend.h"
+#include "aom_ports/mem.h"
+#include "config/av1_rtcd.h"
+
+static INLINE void diffwtd_mask_highbd_neon(uint8_t *mask, bool inverse,
+ const uint16_t *src0,
+ int src0_stride,
+ const uint16_t *src1,
+ int src1_stride, int h, int w,
+ const unsigned int bd) {
+ assert(DIFF_FACTOR > 0);
+ uint8x16_t max_alpha = vdupq_n_u8(AOM_BLEND_A64_MAX_ALPHA);
+ uint8x16_t mask_base = vdupq_n_u8(38);
+ uint8x16_t mask_diff = vdupq_n_u8(AOM_BLEND_A64_MAX_ALPHA - 38);
+
+ if (bd == 8) {
+ if (w >= 16) {
+ do {
+ uint8_t *mask_ptr = mask;
+ const uint16_t *src0_ptr = src0;
+ const uint16_t *src1_ptr = src1;
+ int width = w;
+ do {
+ uint16x8_t s0_lo = vld1q_u16(src0_ptr);
+ uint16x8_t s0_hi = vld1q_u16(src0_ptr + 8);
+ uint16x8_t s1_lo = vld1q_u16(src1_ptr);
+ uint16x8_t s1_hi = vld1q_u16(src1_ptr + 8);
+
+ uint16x8_t diff_lo_u16 = vabdq_u16(s0_lo, s1_lo);
+ uint16x8_t diff_hi_u16 = vabdq_u16(s0_hi, s1_hi);
+ uint8x8_t diff_lo_u8 = vshrn_n_u16(diff_lo_u16, DIFF_FACTOR_LOG2);
+ uint8x8_t diff_hi_u8 = vshrn_n_u16(diff_hi_u16, DIFF_FACTOR_LOG2);
+ uint8x16_t diff = vcombine_u8(diff_lo_u8, diff_hi_u8);
+
+ uint8x16_t m;
+ if (inverse) {
+ m = vqsubq_u8(mask_diff, diff);
+ } else {
+ m = vminq_u8(vaddq_u8(diff, mask_base), max_alpha);
+ }
+
+ vst1q_u8(mask_ptr, m);
+
+ src0_ptr += 16;
+ src1_ptr += 16;
+ mask_ptr += 16;
+ width -= 16;
+ } while (width != 0);
+ mask += w;
+ src0 += src0_stride;
+ src1 += src1_stride;
+ } while (--h != 0);
+ } else if (w == 8) {
+ do {
+ uint8_t *mask_ptr = mask;
+ const uint16_t *src0_ptr = src0;
+ const uint16_t *src1_ptr = src1;
+ int width = w;
+ do {
+ uint16x8_t s0 = vld1q_u16(src0_ptr);
+ uint16x8_t s1 = vld1q_u16(src1_ptr);
+
+ uint16x8_t diff_u16 = vabdq_u16(s0, s1);
+ uint8x8_t diff_u8 = vshrn_n_u16(diff_u16, DIFF_FACTOR_LOG2);
+ uint8x8_t m;
+ if (inverse) {
+ m = vqsub_u8(vget_low_u8(mask_diff), diff_u8);
+ } else {
+ m = vmin_u8(vadd_u8(diff_u8, vget_low_u8(mask_base)),
+ vget_low_u8(max_alpha));
+ }
+
+ vst1_u8(mask_ptr, m);
+
+ src0_ptr += 8;
+ src1_ptr += 8;
+ mask_ptr += 8;
+ width -= 8;
+ } while (width != 0);
+ mask += w;
+ src0 += src0_stride;
+ src1 += src1_stride;
+ } while (--h != 0);
+ } else if (w == 4) {
+ do {
+ uint16x8_t s0 = load_unaligned_u16_4x2(src0, src0_stride);
+ uint16x8_t s1 = load_unaligned_u16_4x2(src1, src1_stride);
+
+ uint16x8_t diff_u16 = vabdq_u16(s0, s1);
+ uint8x8_t diff_u8 = vshrn_n_u16(diff_u16, DIFF_FACTOR_LOG2);
+ uint8x8_t m;
+ if (inverse) {
+ m = vqsub_u8(vget_low_u8(mask_diff), diff_u8);
+ } else {
+ m = vmin_u8(vadd_u8(diff_u8, vget_low_u8(mask_base)),
+ vget_low_u8(max_alpha));
+ }
+
+ store_u8_4x1(mask, m, 0);
+ store_u8_4x1(mask + w, m, 1);
+
+ src0 += 2 * src0_stride;
+ src1 += 2 * src1_stride;
+ mask += 2 * w;
+ h -= 2;
+ } while (h != 0);
+ }
+ } else if (bd == 10) {
+ if (w >= 16) {
+ do {
+ uint8_t *mask_ptr = mask;
+ const uint16_t *src0_ptr = src0;
+ const uint16_t *src1_ptr = src1;
+ int width = w;
+ do {
+ uint16x8_t s0_lo = vld1q_u16(src0_ptr);
+ uint16x8_t s0_hi = vld1q_u16(src0_ptr + 8);
+ uint16x8_t s1_lo = vld1q_u16(src1_ptr);
+ uint16x8_t s1_hi = vld1q_u16(src1_ptr + 8);
+
+ uint16x8_t diff_lo_u16 = vabdq_u16(s0_lo, s1_lo);
+ uint16x8_t diff_hi_u16 = vabdq_u16(s0_hi, s1_hi);
+ uint8x8_t diff_lo_u8 = vshrn_n_u16(diff_lo_u16, 2 + DIFF_FACTOR_LOG2);
+ uint8x8_t diff_hi_u8 = vshrn_n_u16(diff_hi_u16, 2 + DIFF_FACTOR_LOG2);
+ uint8x16_t diff = vcombine_u8(diff_lo_u8, diff_hi_u8);
+
+ uint8x16_t m;
+ if (inverse) {
+ m = vqsubq_u8(mask_diff, diff);
+ } else {
+ m = vminq_u8(vaddq_u8(diff, mask_base), max_alpha);
+ }
+
+ vst1q_u8(mask_ptr, m);
+
+ src0_ptr += 16;
+ src1_ptr += 16;
+ mask_ptr += 16;
+ width -= 16;
+ } while (width != 0);
+ mask += w;
+ src0 += src0_stride;
+ src1 += src1_stride;
+ } while (--h != 0);
+ } else if (w == 8) {
+ do {
+ uint8_t *mask_ptr = mask;
+ const uint16_t *src0_ptr = src0;
+ const uint16_t *src1_ptr = src1;
+ int width = w;
+ do {
+ uint16x8_t s0 = vld1q_u16(src0_ptr);
+ uint16x8_t s1 = vld1q_u16(src1_ptr);
+
+ uint16x8_t diff_u16 = vabdq_u16(s0, s1);
+ uint8x8_t diff_u8 = vshrn_n_u16(diff_u16, 2 + DIFF_FACTOR_LOG2);
+ uint8x8_t m;
+ if (inverse) {
+ m = vqsub_u8(vget_low_u8(mask_diff), diff_u8);
+ } else {
+ m = vmin_u8(vadd_u8(diff_u8, vget_low_u8(mask_base)),
+ vget_low_u8(max_alpha));
+ }
+
+ vst1_u8(mask_ptr, m);
+
+ src0_ptr += 8;
+ src1_ptr += 8;
+ mask_ptr += 8;
+ width -= 8;
+ } while (width != 0);
+ mask += w;
+ src0 += src0_stride;
+ src1 += src1_stride;
+ } while (--h != 0);
+ } else if (w == 4) {
+ do {
+ uint16x8_t s0 = load_unaligned_u16_4x2(src0, src0_stride);
+ uint16x8_t s1 = load_unaligned_u16_4x2(src1, src1_stride);
+
+ uint16x8_t diff_u16 = vabdq_u16(s0, s1);
+ uint8x8_t diff_u8 = vshrn_n_u16(diff_u16, 2 + DIFF_FACTOR_LOG2);
+ uint8x8_t m;
+ if (inverse) {
+ m = vqsub_u8(vget_low_u8(mask_diff), diff_u8);
+ } else {
+ m = vmin_u8(vadd_u8(diff_u8, vget_low_u8(mask_base)),
+ vget_low_u8(max_alpha));
+ }
+
+ store_u8_4x1(mask, m, 0);
+ store_u8_4x1(mask + w, m, 1);
+
+ src0 += 2 * src0_stride;
+ src1 += 2 * src1_stride;
+ mask += 2 * w;
+ h -= 2;
+ } while (h != 0);
+ }
+ } else {
+ assert(bd == 12);
+ if (w >= 16) {
+ do {
+ uint8_t *mask_ptr = mask;
+ const uint16_t *src0_ptr = src0;
+ const uint16_t *src1_ptr = src1;
+ int width = w;
+ do {
+ uint16x8_t s0_lo = vld1q_u16(src0_ptr);
+ uint16x8_t s0_hi = vld1q_u16(src0_ptr + 8);
+ uint16x8_t s1_lo = vld1q_u16(src1_ptr);
+ uint16x8_t s1_hi = vld1q_u16(src1_ptr + 8);
+
+ uint16x8_t diff_lo_u16 = vabdq_u16(s0_lo, s1_lo);
+ uint16x8_t diff_hi_u16 = vabdq_u16(s0_hi, s1_hi);
+ uint8x8_t diff_lo_u8 = vshrn_n_u16(diff_lo_u16, 4 + DIFF_FACTOR_LOG2);
+ uint8x8_t diff_hi_u8 = vshrn_n_u16(diff_hi_u16, 4 + DIFF_FACTOR_LOG2);
+ uint8x16_t diff = vcombine_u8(diff_lo_u8, diff_hi_u8);
+
+ uint8x16_t m;
+ if (inverse) {
+ m = vqsubq_u8(mask_diff, diff);
+ } else {
+ m = vminq_u8(vaddq_u8(diff, mask_base), max_alpha);
+ }
+
+ vst1q_u8(mask_ptr, m);
+
+ src0_ptr += 16;
+ src1_ptr += 16;
+ mask_ptr += 16;
+ width -= 16;
+ } while (width != 0);
+ mask += w;
+ src0 += src0_stride;
+ src1 += src1_stride;
+ } while (--h != 0);
+ } else if (w == 8) {
+ do {
+ uint8_t *mask_ptr = mask;
+ const uint16_t *src0_ptr = src0;
+ const uint16_t *src1_ptr = src1;
+ int width = w;
+ do {
+ uint16x8_t s0 = vld1q_u16(src0_ptr);
+ uint16x8_t s1 = vld1q_u16(src1_ptr);
+
+ uint16x8_t diff_u16 = vabdq_u16(s0, s1);
+ uint8x8_t diff_u8 = vshrn_n_u16(diff_u16, 4 + DIFF_FACTOR_LOG2);
+ uint8x8_t m;
+ if (inverse) {
+ m = vqsub_u8(vget_low_u8(mask_diff), diff_u8);
+ } else {
+ m = vmin_u8(vadd_u8(diff_u8, vget_low_u8(mask_base)),
+ vget_low_u8(max_alpha));
+ }
+
+ vst1_u8(mask_ptr, m);
+
+ src0_ptr += 8;
+ src1_ptr += 8;
+ mask_ptr += 8;
+ width -= 8;
+ } while (width != 0);
+ mask += w;
+ src0 += src0_stride;
+ src1 += src1_stride;
+ } while (--h != 0);
+ } else if (w == 4) {
+ do {
+ uint16x8_t s0 = load_unaligned_u16_4x2(src0, src0_stride);
+ uint16x8_t s1 = load_unaligned_u16_4x2(src1, src1_stride);
+
+ uint16x8_t diff_u16 = vabdq_u16(s0, s1);
+ uint8x8_t diff_u8 = vshrn_n_u16(diff_u16, 4 + DIFF_FACTOR_LOG2);
+ uint8x8_t m;
+ if (inverse) {
+ m = vqsub_u8(vget_low_u8(mask_diff), diff_u8);
+ } else {
+ m = vmin_u8(vadd_u8(diff_u8, vget_low_u8(mask_base)),
+ vget_low_u8(max_alpha));
+ }
+
+ store_u8_4x1(mask, m, 0);
+ store_u8_4x1(mask + w, m, 1);
+
+ src0 += 2 * src0_stride;
+ src1 += 2 * src1_stride;
+ mask += 2 * w;
+ h -= 2;
+ } while (h != 0);
+ }
+ }
+}
+
+void av1_build_compound_diffwtd_mask_highbd_neon(
+ uint8_t *mask, DIFFWTD_MASK_TYPE mask_type, const uint8_t *src0,
+ int src0_stride, const uint8_t *src1, int src1_stride, int h, int w,
+ int bd) {
+ assert(h % 4 == 0);
+ assert(w % 4 == 0);
+ assert(mask_type == DIFFWTD_38_INV || mask_type == DIFFWTD_38);
+
+ if (mask_type == DIFFWTD_38) {
+ diffwtd_mask_highbd_neon(mask, /*inverse=*/false, CONVERT_TO_SHORTPTR(src0),
+ src0_stride, CONVERT_TO_SHORTPTR(src1),
+ src1_stride, h, w, bd);
+ } else { // mask_type == DIFFWTD_38_INV
+ diffwtd_mask_highbd_neon(mask, /*inverse=*/true, CONVERT_TO_SHORTPTR(src0),
+ src0_stride, CONVERT_TO_SHORTPTR(src1),
+ src1_stride, h, w, bd);
+ }
+}
diff --git a/av1/common/arm/highbd_reconintra_neon.c b/av1/common/arm/highbd_reconintra_neon.c
new file mode 100644
index 0000000..170491b
--- /dev/null
+++ b/av1/common/arm/highbd_reconintra_neon.c
@@ -0,0 +1,241 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+#include <assert.h>
+
+#include "aom_dsp/arm/sum_neon.h"
+
+#define MAX_UPSAMPLE_SZ 16
+
+void av1_highbd_filter_intra_edge_neon(uint16_t *p, int sz, int strength) {
+ if (!strength) return;
+ assert(sz >= 0 && sz <= 129);
+
+ DECLARE_ALIGNED(16, static const uint16_t,
+ idx[8]) = { 0, 1, 2, 3, 4, 5, 6, 7 };
+ const uint16x8_t index = vld1q_u16(idx);
+
+ uint16_t edge[160]; // Max value of sz + enough padding for vector accesses.
+ memcpy(edge + 1, p, sz * sizeof(*p));
+
+ // Populate extra space appropriately.
+ edge[0] = edge[1];
+ edge[sz + 1] = edge[sz];
+ edge[sz + 2] = edge[sz];
+
+ // Don't overwrite first pixel.
+ uint16_t *dst = p + 1;
+ sz--;
+
+ if (strength == 1) { // Filter: {4, 8, 4}.
+ const uint16_t *src = edge + 1;
+
+ while (sz >= 8) {
+ uint16x8_t s0 = vld1q_u16(src);
+ uint16x8_t s1 = vld1q_u16(src + 1);
+ uint16x8_t s2 = vld1q_u16(src + 2);
+
+ // Make use of the identity:
+ // (4*a + 8*b + 4*c) >> 4 == (a + (b << 1) + c) >> 2
+ uint16x8_t t0 = vaddq_u16(s0, s2);
+ uint16x8_t t1 = vaddq_u16(s1, s1);
+ uint16x8_t sum = vaddq_u16(t0, t1);
+ uint16x8_t res = vrshrq_n_u16(sum, 2);
+
+ vst1q_u16(dst, res);
+
+ src += 8;
+ dst += 8;
+ sz -= 8;
+ }
+
+ if (sz > 0) { // Handle sz < 8 to avoid modifying out-of-bounds values.
+ uint16x8_t s0 = vld1q_u16(src);
+ uint16x8_t s1 = vld1q_u16(src + 1);
+ uint16x8_t s2 = vld1q_u16(src + 2);
+
+ // Make use of the identity:
+ // (4*a + 8*b + 4*c) >> 4 == (a + (b << 1) + c) >> 2
+ uint16x8_t t0 = vaddq_u16(s0, s2);
+ uint16x8_t t1 = vaddq_u16(s1, s1);
+ uint16x8_t sum = vaddq_u16(t0, t1);
+ uint16x8_t res = vrshrq_n_u16(sum, 2);
+
+ // Mask off out-of-bounds indices.
+ uint16x8_t current_dst = vld1q_u16(dst);
+ uint16x8_t mask = vcgtq_u16(vdupq_n_u16(sz), index);
+ res = vbslq_u16(mask, res, current_dst);
+
+ vst1q_u16(dst, res);
+ }
+ } else if (strength == 2) { // Filter: {5, 6, 5}.
+ const uint16_t *src = edge + 1;
+
+ const uint16x8x3_t filter = { { vdupq_n_u16(5), vdupq_n_u16(6),
+ vdupq_n_u16(5) } };
+ while (sz >= 8) {
+ uint16x8_t s0 = vld1q_u16(src);
+ uint16x8_t s1 = vld1q_u16(src + 1);
+ uint16x8_t s2 = vld1q_u16(src + 2);
+
+ uint16x8_t accum = vmulq_u16(s0, filter.val[0]);
+ accum = vmlaq_u16(accum, s1, filter.val[1]);
+ accum = vmlaq_u16(accum, s2, filter.val[2]);
+ uint16x8_t res = vrshrq_n_u16(accum, 4);
+
+ vst1q_u16(dst, res);
+
+ src += 8;
+ dst += 8;
+ sz -= 8;
+ }
+
+ if (sz > 0) { // Handle sz < 8 to avoid modifying out-of-bounds values.
+ uint16x8_t s0 = vld1q_u16(src);
+ uint16x8_t s1 = vld1q_u16(src + 1);
+ uint16x8_t s2 = vld1q_u16(src + 2);
+
+ uint16x8_t accum = vmulq_u16(s0, filter.val[0]);
+ accum = vmlaq_u16(accum, s1, filter.val[1]);
+ accum = vmlaq_u16(accum, s2, filter.val[2]);
+ uint16x8_t res = vrshrq_n_u16(accum, 4);
+
+ // Mask off out-of-bounds indices.
+ uint16x8_t current_dst = vld1q_u16(dst);
+ uint16x8_t mask = vcgtq_u16(vdupq_n_u16(sz), index);
+ res = vbslq_u16(mask, res, current_dst);
+
+ vst1q_u16(dst, res);
+ }
+ } else { // Filter {2, 4, 4, 4, 2}.
+ const uint16_t *src = edge;
+
+ while (sz >= 8) {
+ uint16x8_t s0 = vld1q_u16(src);
+ uint16x8_t s1 = vld1q_u16(src + 1);
+ uint16x8_t s2 = vld1q_u16(src + 2);
+ uint16x8_t s3 = vld1q_u16(src + 3);
+ uint16x8_t s4 = vld1q_u16(src + 4);
+
+ // Make use of the identity:
+ // (2*a + 4*b + 4*c + 4*d + 2*e) >> 4 == (a + ((b + c + d) << 1) + e) >> 3
+ uint16x8_t t0 = vaddq_u16(s0, s4);
+ uint16x8_t t1 = vaddq_u16(s1, s2);
+ t1 = vaddq_u16(t1, s3);
+ t1 = vaddq_u16(t1, t1);
+ uint16x8_t sum = vaddq_u16(t0, t1);
+ uint16x8_t res = vrshrq_n_u16(sum, 3);
+
+ vst1q_u16(dst, res);
+
+ src += 8;
+ dst += 8;
+ sz -= 8;
+ }
+
+ if (sz > 0) { // Handle sz < 8 to avoid modifying out-of-bounds values.
+ uint16x8_t s0 = vld1q_u16(src);
+ uint16x8_t s1 = vld1q_u16(src + 1);
+ uint16x8_t s2 = vld1q_u16(src + 2);
+ uint16x8_t s3 = vld1q_u16(src + 3);
+ uint16x8_t s4 = vld1q_u16(src + 4);
+
+ // Make use of the identity:
+ // (2*a + 4*b + 4*c + 4*d + 2*e) >> 4 == (a + ((b + c + d) << 1) + e) >> 3
+ uint16x8_t t0 = vaddq_u16(s0, s4);
+ uint16x8_t t1 = vaddq_u16(s1, s2);
+ t1 = vaddq_u16(t1, s3);
+ t1 = vaddq_u16(t1, t1);
+ uint16x8_t sum = vaddq_u16(t0, t1);
+ uint16x8_t res = vrshrq_n_u16(sum, 3);
+
+ // Mask off out-of-bounds indices.
+ uint16x8_t current_dst = vld1q_u16(dst);
+ uint16x8_t mask = vcgtq_u16(vdupq_n_u16(sz), index);
+ res = vbslq_u16(mask, res, current_dst);
+
+ vst1q_u16(dst, res);
+ }
+ }
+}
+
+void av1_highbd_upsample_intra_edge_neon(uint16_t *p, int sz, int bd) {
+ if (!sz) return;
+
+ assert(sz <= MAX_UPSAMPLE_SZ);
+
+ uint16_t edge[MAX_UPSAMPLE_SZ + 3];
+ const uint16_t *src = edge;
+
+ // Copy p[-1..(sz-1)] and pad out both ends.
+ edge[0] = p[-1];
+ edge[1] = p[-1];
+ memcpy(edge + 2, p, sz * 2);
+ edge[sz + 2] = p[sz - 1];
+ p[-2] = p[-1];
+
+ uint16x8_t pixel_val_max = vdupq_n_u16((1 << bd) - 1);
+
+ uint16_t *dst = p - 1;
+
+ if (bd == 12) {
+ do {
+ uint16x8_t s0 = vld1q_u16(src);
+ uint16x8_t s1 = vld1q_u16(src + 1);
+ uint16x8_t s2 = vld1q_u16(src + 2);
+ uint16x8_t s3 = vld1q_u16(src + 3);
+
+ uint16x8_t t0 = vaddq_u16(s1, s2);
+ uint16x8_t t1 = vaddq_u16(s0, s3);
+ uint32x4_t acc0 = vmull_n_u16(vget_low_u16(t0), 9);
+ acc0 = vqsubq_u32(acc0, vmovl_u16(vget_low_u16(t1)));
+ uint32x4_t acc1 = vmull_n_u16(vget_high_u16(t0), 9);
+ acc1 = vqsubq_u32(acc1, vmovl_u16(vget_high_u16(t1)));
+
+ uint16x8x2_t res;
+ res.val[0] = vcombine_u16(vrshrn_n_u32(acc0, 4), vrshrn_n_u32(acc1, 4));
+ // Clamp pixel values at bitdepth maximum.
+ res.val[0] = vminq_u16(res.val[0], pixel_val_max);
+ res.val[1] = s2;
+
+ vst2q_u16(dst, res);
+
+ src += 8;
+ dst += 16;
+ sz -= 8;
+ } while (sz > 0);
+ } else { // Bit depth is 8 or 10.
+ do {
+ uint16x8_t s0 = vld1q_u16(src);
+ uint16x8_t s1 = vld1q_u16(src + 1);
+ uint16x8_t s2 = vld1q_u16(src + 2);
+ uint16x8_t s3 = vld1q_u16(src + 3);
+
+ uint16x8_t t0 = vaddq_u16(s0, s3);
+ uint16x8_t t1 = vaddq_u16(s1, s2);
+ t1 = vmulq_n_u16(t1, 9);
+ t1 = vqsubq_u16(t1, t0);
+
+ uint16x8x2_t res;
+ res.val[0] = vrshrq_n_u16(t1, 4);
+ // Clamp pixel values at bitdepth maximum.
+ res.val[0] = vminq_u16(res.val[0], pixel_val_max);
+ res.val[1] = s2;
+
+ vst2q_u16(dst, res);
+
+ src += 8;
+ dst += 16;
+ sz -= 8;
+ } while (sz > 0);
+ }
+}
diff --git a/av1/common/arm/highbd_warp_plane_neon.c b/av1/common/arm/highbd_warp_plane_neon.c
new file mode 100644
index 0000000..0729df6
--- /dev/null
+++ b/av1/common/arm/highbd_warp_plane_neon.c
@@ -0,0 +1,560 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+#include <assert.h>
+#include <stdbool.h>
+
+#include "aom_dsp/aom_dsp_common.h"
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_dsp/arm/sum_neon.h"
+#include "aom_dsp/arm/transpose_neon.h"
+#include "aom_ports/mem.h"
+#include "av1/common/scale.h"
+#include "av1/common/warped_motion.h"
+#include "config/av1_rtcd.h"
+
+static INLINE int16x8_t load_filters_1(int ofs) {
+ const int ofs0 = ROUND_POWER_OF_TWO(ofs, WARPEDDIFF_PREC_BITS);
+
+ const int16_t *base =
+ (int16_t *)av1_warped_filter + WARPEDPIXEL_PREC_SHIFTS * 8;
+ return vld1q_s16(base + ofs0 * 8);
+}
+
+static INLINE void load_filters_4(int16x8_t out[], int ofs, int stride) {
+ const int ofs0 = ROUND_POWER_OF_TWO(ofs + stride * 0, WARPEDDIFF_PREC_BITS);
+ const int ofs1 = ROUND_POWER_OF_TWO(ofs + stride * 1, WARPEDDIFF_PREC_BITS);
+ const int ofs2 = ROUND_POWER_OF_TWO(ofs + stride * 2, WARPEDDIFF_PREC_BITS);
+ const int ofs3 = ROUND_POWER_OF_TWO(ofs + stride * 3, WARPEDDIFF_PREC_BITS);
+
+ const int16_t *base =
+ (int16_t *)av1_warped_filter + WARPEDPIXEL_PREC_SHIFTS * 8;
+ out[0] = vld1q_s16(base + ofs0 * 8);
+ out[1] = vld1q_s16(base + ofs1 * 8);
+ out[2] = vld1q_s16(base + ofs2 * 8);
+ out[3] = vld1q_s16(base + ofs3 * 8);
+}
+
+static INLINE void load_filters_8(int16x8_t out[], int ofs, int stride) {
+ const int ofs0 = ROUND_POWER_OF_TWO(ofs + stride * 0, WARPEDDIFF_PREC_BITS);
+ const int ofs1 = ROUND_POWER_OF_TWO(ofs + stride * 1, WARPEDDIFF_PREC_BITS);
+ const int ofs2 = ROUND_POWER_OF_TWO(ofs + stride * 2, WARPEDDIFF_PREC_BITS);
+ const int ofs3 = ROUND_POWER_OF_TWO(ofs + stride * 3, WARPEDDIFF_PREC_BITS);
+ const int ofs4 = ROUND_POWER_OF_TWO(ofs + stride * 4, WARPEDDIFF_PREC_BITS);
+ const int ofs5 = ROUND_POWER_OF_TWO(ofs + stride * 5, WARPEDDIFF_PREC_BITS);
+ const int ofs6 = ROUND_POWER_OF_TWO(ofs + stride * 6, WARPEDDIFF_PREC_BITS);
+ const int ofs7 = ROUND_POWER_OF_TWO(ofs + stride * 7, WARPEDDIFF_PREC_BITS);
+
+ const int16_t *base =
+ (int16_t *)av1_warped_filter + WARPEDPIXEL_PREC_SHIFTS * 8;
+ out[0] = vld1q_s16(base + ofs0 * 8);
+ out[1] = vld1q_s16(base + ofs1 * 8);
+ out[2] = vld1q_s16(base + ofs2 * 8);
+ out[3] = vld1q_s16(base + ofs3 * 8);
+ out[4] = vld1q_s16(base + ofs4 * 8);
+ out[5] = vld1q_s16(base + ofs5 * 8);
+ out[6] = vld1q_s16(base + ofs6 * 8);
+ out[7] = vld1q_s16(base + ofs7 * 8);
+}
+
+static INLINE int16x8_t warp_affine_horizontal_step_4x1_f4_neon(
+ int bd, int sx, int alpha, uint16x8x2_t in) {
+ int16x8_t f[4];
+ load_filters_4(f, sx, alpha);
+
+ int16x8_t rv0 = vextq_s16(vreinterpretq_s16_u16(in.val[0]),
+ vreinterpretq_s16_u16(in.val[1]), 0);
+ int16x8_t rv1 = vextq_s16(vreinterpretq_s16_u16(in.val[0]),
+ vreinterpretq_s16_u16(in.val[1]), 1);
+ int16x8_t rv2 = vextq_s16(vreinterpretq_s16_u16(in.val[0]),
+ vreinterpretq_s16_u16(in.val[1]), 2);
+ int16x8_t rv3 = vextq_s16(vreinterpretq_s16_u16(in.val[0]),
+ vreinterpretq_s16_u16(in.val[1]), 3);
+
+ int32x4_t m0 = vmull_s16(vget_low_s16(f[0]), vget_low_s16(rv0));
+ m0 = vmlal_s16(m0, vget_high_s16(f[0]), vget_high_s16(rv0));
+ int32x4_t m1 = vmull_s16(vget_low_s16(f[1]), vget_low_s16(rv1));
+ m1 = vmlal_s16(m1, vget_high_s16(f[1]), vget_high_s16(rv1));
+ int32x4_t m2 = vmull_s16(vget_low_s16(f[2]), vget_low_s16(rv2));
+ m2 = vmlal_s16(m2, vget_high_s16(f[2]), vget_high_s16(rv2));
+ int32x4_t m3 = vmull_s16(vget_low_s16(f[3]), vget_low_s16(rv3));
+ m3 = vmlal_s16(m3, vget_high_s16(f[3]), vget_high_s16(rv3));
+
+ int32x4_t m0123[] = { m0, m1, m2, m3 };
+
+ const int round0 = (bd == 12) ? ROUND0_BITS + 2 : ROUND0_BITS;
+ const int offset_bits_horiz = bd + FILTER_BITS - 1;
+
+ int32x4_t res = horizontal_add_4d_s32x4(m0123);
+ res = vaddq_s32(res, vdupq_n_s32(1 << offset_bits_horiz));
+ res = vrshlq_s32(res, vdupq_n_s32(-round0));
+ return vcombine_s16(vmovn_s32(res), vdup_n_s16(0));
+}
+
+static INLINE int16x8_t warp_affine_horizontal_step_8x1_f8_neon(
+ int bd, int sx, int alpha, uint16x8x2_t in) {
+ const int round0 = (bd == 12) ? ROUND0_BITS + 2 : ROUND0_BITS;
+ const int offset_bits_horiz = bd + FILTER_BITS - 1;
+
+ int16x8_t f[8];
+ load_filters_8(f, sx, alpha);
+
+ int16x8_t rv0 = vextq_s16(vreinterpretq_s16_u16(in.val[0]),
+ vreinterpretq_s16_u16(in.val[1]), 0);
+ int16x8_t rv1 = vextq_s16(vreinterpretq_s16_u16(in.val[0]),
+ vreinterpretq_s16_u16(in.val[1]), 1);
+ int16x8_t rv2 = vextq_s16(vreinterpretq_s16_u16(in.val[0]),
+ vreinterpretq_s16_u16(in.val[1]), 2);
+ int16x8_t rv3 = vextq_s16(vreinterpretq_s16_u16(in.val[0]),
+ vreinterpretq_s16_u16(in.val[1]), 3);
+ int16x8_t rv4 = vextq_s16(vreinterpretq_s16_u16(in.val[0]),
+ vreinterpretq_s16_u16(in.val[1]), 4);
+ int16x8_t rv5 = vextq_s16(vreinterpretq_s16_u16(in.val[0]),
+ vreinterpretq_s16_u16(in.val[1]), 5);
+ int16x8_t rv6 = vextq_s16(vreinterpretq_s16_u16(in.val[0]),
+ vreinterpretq_s16_u16(in.val[1]), 6);
+ int16x8_t rv7 = vextq_s16(vreinterpretq_s16_u16(in.val[0]),
+ vreinterpretq_s16_u16(in.val[1]), 7);
+
+ int32x4_t m0 = vmull_s16(vget_low_s16(f[0]), vget_low_s16(rv0));
+ m0 = vmlal_s16(m0, vget_high_s16(f[0]), vget_high_s16(rv0));
+ int32x4_t m1 = vmull_s16(vget_low_s16(f[1]), vget_low_s16(rv1));
+ m1 = vmlal_s16(m1, vget_high_s16(f[1]), vget_high_s16(rv1));
+ int32x4_t m2 = vmull_s16(vget_low_s16(f[2]), vget_low_s16(rv2));
+ m2 = vmlal_s16(m2, vget_high_s16(f[2]), vget_high_s16(rv2));
+ int32x4_t m3 = vmull_s16(vget_low_s16(f[3]), vget_low_s16(rv3));
+ m3 = vmlal_s16(m3, vget_high_s16(f[3]), vget_high_s16(rv3));
+ int32x4_t m4 = vmull_s16(vget_low_s16(f[4]), vget_low_s16(rv4));
+ m4 = vmlal_s16(m4, vget_high_s16(f[4]), vget_high_s16(rv4));
+ int32x4_t m5 = vmull_s16(vget_low_s16(f[5]), vget_low_s16(rv5));
+ m5 = vmlal_s16(m5, vget_high_s16(f[5]), vget_high_s16(rv5));
+ int32x4_t m6 = vmull_s16(vget_low_s16(f[6]), vget_low_s16(rv6));
+ m6 = vmlal_s16(m6, vget_high_s16(f[6]), vget_high_s16(rv6));
+ int32x4_t m7 = vmull_s16(vget_low_s16(f[7]), vget_low_s16(rv7));
+ m7 = vmlal_s16(m7, vget_high_s16(f[7]), vget_high_s16(rv7));
+
+ int32x4_t m0123[] = { m0, m1, m2, m3 };
+ int32x4_t m4567[] = { m4, m5, m6, m7 };
+
+ int32x4_t res0 = horizontal_add_4d_s32x4(m0123);
+ int32x4_t res1 = horizontal_add_4d_s32x4(m4567);
+ res0 = vaddq_s32(res0, vdupq_n_s32(1 << offset_bits_horiz));
+ res1 = vaddq_s32(res1, vdupq_n_s32(1 << offset_bits_horiz));
+ res0 = vrshlq_s32(res0, vdupq_n_s32(-round0));
+ res1 = vrshlq_s32(res1, vdupq_n_s32(-round0));
+ return vcombine_s16(vmovn_s32(res0), vmovn_s32(res1));
+}
+
+static INLINE void warp_affine_horizontal_neon(const uint16_t *ref, int width,
+ int height, int stride,
+ int p_width, int16_t alpha,
+ int16_t beta, int iy4, int sx4,
+ int ix4, int16x8_t tmp[],
+ int bd) {
+ const int round0 = (bd == 12) ? ROUND0_BITS + 2 : ROUND0_BITS;
+
+ if (ix4 <= -7) {
+ for (int k = 0; k < 15; ++k) {
+ int iy = clamp(iy4 + k - 7, 0, height - 1);
+ int32_t dup_val = (1 << (bd + FILTER_BITS - round0 - 1)) +
+ ref[iy * stride] * (1 << (FILTER_BITS - round0));
+ tmp[k] = vdupq_n_s16(dup_val);
+ }
+ return;
+ } else if (ix4 >= width + 6) {
+ for (int k = 0; k < 15; ++k) {
+ int iy = clamp(iy4 + k - 7, 0, height - 1);
+ int32_t dup_val =
+ (1 << (bd + FILTER_BITS - round0 - 1)) +
+ ref[iy * stride + (width - 1)] * (1 << (FILTER_BITS - round0));
+ tmp[k] = vdupq_n_s16(dup_val);
+ }
+ return;
+ }
+
+ for (int k = 0; k < 15; ++k) {
+ const int iy = clamp(iy4 + k - 7, 0, height - 1);
+ uint16x8x2_t in = vld1q_u16_x2(ref + iy * stride + ix4 - 7);
+
+ const int out_of_boundary_left = -(ix4 - 6);
+ const int out_of_boundary_right = (ix4 + 8) - width;
+
+ const uint16_t k0[16] = { 0, 1, 2, 3, 4, 5, 6, 7,
+ 8, 9, 10, 11, 12, 13, 14, 15 };
+ const uint16x8_t indx0 = vld1q_u16(&k0[0]);
+ const uint16x8_t indx1 = vld1q_u16(&k0[8]);
+
+ if (out_of_boundary_left >= 0) {
+ uint16x8_t cmp_vec = vdupq_n_u16(out_of_boundary_left);
+ uint16x8_t vec_dup = vdupq_n_u16(ref[iy * stride]);
+ uint16x8_t mask0 = vcleq_u16(indx0, cmp_vec);
+ uint16x8_t mask1 = vcleq_u16(indx1, cmp_vec);
+ in.val[0] = vbslq_u16(mask0, vec_dup, in.val[0]);
+ in.val[1] = vbslq_u16(mask1, vec_dup, in.val[1]);
+ }
+ if (out_of_boundary_right >= 0) {
+ uint16x8_t cmp_vec = vdupq_n_u16(15 - out_of_boundary_right);
+ uint16x8_t vec_dup = vdupq_n_u16(ref[iy * stride + width - 1]);
+ uint16x8_t mask0 = vcgeq_u16(indx0, cmp_vec);
+ uint16x8_t mask1 = vcgeq_u16(indx1, cmp_vec);
+ in.val[0] = vbslq_u16(mask0, vec_dup, in.val[0]);
+ in.val[1] = vbslq_u16(mask1, vec_dup, in.val[1]);
+ }
+
+ const int sx = sx4 + beta * (k - 3);
+ if (p_width == 4) {
+ tmp[k] = warp_affine_horizontal_step_4x1_f4_neon(bd, sx, alpha, in);
+ } else {
+ tmp[k] = warp_affine_horizontal_step_8x1_f8_neon(bd, sx, alpha, in);
+ }
+ }
+}
+
+static INLINE uint16x4_t clip_pixel_highbd_vec(int32x4_t val, int bd) {
+ const int limit = (1 << bd) - 1;
+ return vqmovun_s32(vminq_s32(val, vdupq_n_s32(limit)));
+}
+
+static INLINE int32x4_t
+warp_affine_vertical_filter_4x1_f1_neon(const int16x8_t *tmp, int sy) {
+ const int16x8_t f = load_filters_1(sy);
+ const int16x4_t f0123 = vget_low_s16(f);
+ const int16x4_t f4567 = vget_high_s16(f);
+
+ int32x4_t m0123 = vmull_lane_s16(vget_low_s16(tmp[0]), f0123, 0);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(tmp[1]), f0123, 1);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(tmp[2]), f0123, 2);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(tmp[3]), f0123, 3);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(tmp[4]), f4567, 0);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(tmp[5]), f4567, 1);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(tmp[6]), f4567, 2);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(tmp[7]), f4567, 3);
+ return m0123;
+}
+
+static INLINE int32x4x2_t
+warp_affine_vertical_filter_8x1_f1_neon(const int16x8_t *tmp, int sy) {
+ const int16x8_t f = load_filters_1(sy);
+ const int16x4_t f0123 = vget_low_s16(f);
+ const int16x4_t f4567 = vget_high_s16(f);
+
+ int32x4_t m0123 = vmull_lane_s16(vget_low_s16(tmp[0]), f0123, 0);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(tmp[1]), f0123, 1);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(tmp[2]), f0123, 2);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(tmp[3]), f0123, 3);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(tmp[4]), f4567, 0);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(tmp[5]), f4567, 1);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(tmp[6]), f4567, 2);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(tmp[7]), f4567, 3);
+
+ int32x4_t m4567 = vmull_lane_s16(vget_high_s16(tmp[0]), f0123, 0);
+ m4567 = vmlal_lane_s16(m4567, vget_high_s16(tmp[1]), f0123, 1);
+ m4567 = vmlal_lane_s16(m4567, vget_high_s16(tmp[2]), f0123, 2);
+ m4567 = vmlal_lane_s16(m4567, vget_high_s16(tmp[3]), f0123, 3);
+ m4567 = vmlal_lane_s16(m4567, vget_high_s16(tmp[4]), f4567, 0);
+ m4567 = vmlal_lane_s16(m4567, vget_high_s16(tmp[5]), f4567, 1);
+ m4567 = vmlal_lane_s16(m4567, vget_high_s16(tmp[6]), f4567, 2);
+ m4567 = vmlal_lane_s16(m4567, vget_high_s16(tmp[7]), f4567, 3);
+ return (int32x4x2_t){ { m0123, m4567 } };
+}
+
+static INLINE int32x4_t warp_affine_vertical_filter_4x1_f4_neon(
+ const int16x8_t *tmp, int sy, int gamma) {
+ int16x8_t s0, s1, s2, s3;
+ transpose_elems_s16_4x8(
+ vget_low_s16(tmp[0]), vget_low_s16(tmp[1]), vget_low_s16(tmp[2]),
+ vget_low_s16(tmp[3]), vget_low_s16(tmp[4]), vget_low_s16(tmp[5]),
+ vget_low_s16(tmp[6]), vget_low_s16(tmp[7]), &s0, &s1, &s2, &s3);
+
+ int16x8_t f[4];
+ load_filters_4(f, sy, gamma);
+
+ int32x4_t m0 = vmull_s16(vget_low_s16(s0), vget_low_s16(f[0]));
+ m0 = vmlal_s16(m0, vget_high_s16(s0), vget_high_s16(f[0]));
+ int32x4_t m1 = vmull_s16(vget_low_s16(s1), vget_low_s16(f[1]));
+ m1 = vmlal_s16(m1, vget_high_s16(s1), vget_high_s16(f[1]));
+ int32x4_t m2 = vmull_s16(vget_low_s16(s2), vget_low_s16(f[2]));
+ m2 = vmlal_s16(m2, vget_high_s16(s2), vget_high_s16(f[2]));
+ int32x4_t m3 = vmull_s16(vget_low_s16(s3), vget_low_s16(f[3]));
+ m3 = vmlal_s16(m3, vget_high_s16(s3), vget_high_s16(f[3]));
+
+ int32x4_t m0123[] = { m0, m1, m2, m3 };
+ return horizontal_add_4d_s32x4(m0123);
+}
+
+static INLINE int32x4x2_t warp_affine_vertical_filter_8x1_f8_neon(
+ const int16x8_t *tmp, int sy, int gamma) {
+ int16x8_t s0 = tmp[0];
+ int16x8_t s1 = tmp[1];
+ int16x8_t s2 = tmp[2];
+ int16x8_t s3 = tmp[3];
+ int16x8_t s4 = tmp[4];
+ int16x8_t s5 = tmp[5];
+ int16x8_t s6 = tmp[6];
+ int16x8_t s7 = tmp[7];
+ transpose_elems_inplace_s16_8x8(&s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7);
+
+ int16x8_t f[8];
+ load_filters_8(f, sy, gamma);
+
+ int32x4_t m0 = vmull_s16(vget_low_s16(s0), vget_low_s16(f[0]));
+ m0 = vmlal_s16(m0, vget_high_s16(s0), vget_high_s16(f[0]));
+ int32x4_t m1 = vmull_s16(vget_low_s16(s1), vget_low_s16(f[1]));
+ m1 = vmlal_s16(m1, vget_high_s16(s1), vget_high_s16(f[1]));
+ int32x4_t m2 = vmull_s16(vget_low_s16(s2), vget_low_s16(f[2]));
+ m2 = vmlal_s16(m2, vget_high_s16(s2), vget_high_s16(f[2]));
+ int32x4_t m3 = vmull_s16(vget_low_s16(s3), vget_low_s16(f[3]));
+ m3 = vmlal_s16(m3, vget_high_s16(s3), vget_high_s16(f[3]));
+ int32x4_t m4 = vmull_s16(vget_low_s16(s4), vget_low_s16(f[4]));
+ m4 = vmlal_s16(m4, vget_high_s16(s4), vget_high_s16(f[4]));
+ int32x4_t m5 = vmull_s16(vget_low_s16(s5), vget_low_s16(f[5]));
+ m5 = vmlal_s16(m5, vget_high_s16(s5), vget_high_s16(f[5]));
+ int32x4_t m6 = vmull_s16(vget_low_s16(s6), vget_low_s16(f[6]));
+ m6 = vmlal_s16(m6, vget_high_s16(s6), vget_high_s16(f[6]));
+ int32x4_t m7 = vmull_s16(vget_low_s16(s7), vget_low_s16(f[7]));
+ m7 = vmlal_s16(m7, vget_high_s16(s7), vget_high_s16(f[7]));
+
+ int32x4_t m0123[] = { m0, m1, m2, m3 };
+ int32x4_t m4567[] = { m4, m5, m6, m7 };
+
+ int32x4x2_t ret;
+ ret.val[0] = horizontal_add_4d_s32x4(m0123);
+ ret.val[1] = horizontal_add_4d_s32x4(m4567);
+ return ret;
+}
+
+static INLINE void warp_affine_vertical_step_4x1_f4_neon(
+ uint16_t *pred, int p_stride, int bd, uint16_t *dst, int dst_stride,
+ bool is_compound, bool do_average, bool use_dist_wtd_comp_avg, int fwd,
+ int bwd, int16_t gamma, const int16x8_t *tmp, int i, int sy, int j) {
+ int32x4_t sum0 =
+ gamma == 0 ? warp_affine_vertical_filter_4x1_f1_neon(tmp, sy)
+ : warp_affine_vertical_filter_4x1_f4_neon(tmp, sy, gamma);
+
+ const int round0 = (bd == 12) ? ROUND0_BITS + 2 : ROUND0_BITS;
+ const int offset_bits_vert = bd + 2 * FILTER_BITS - round0;
+
+ sum0 = vaddq_s32(sum0, vdupq_n_s32(1 << offset_bits_vert));
+
+ uint16_t *dst16 = &pred[i * p_stride + j];
+
+ if (!is_compound) {
+ const int reduce_bits_vert = 2 * FILTER_BITS - round0;
+ sum0 = vrshlq_s32(sum0, vdupq_n_s32(-reduce_bits_vert));
+
+ const int res_sub_const = (1 << (bd - 1)) + (1 << bd);
+ sum0 = vsubq_s32(sum0, vdupq_n_s32(res_sub_const));
+ uint16x4_t res0 = clip_pixel_highbd_vec(sum0, bd);
+ vst1_u16(dst16, res0);
+ return;
+ }
+
+ sum0 = vrshrq_n_s32(sum0, COMPOUND_ROUND1_BITS);
+
+ uint16_t *p = &dst[i * dst_stride + j];
+
+ if (!do_average) {
+ vst1_u16(p, vqmovun_s32(sum0));
+ return;
+ }
+
+ uint16x4_t p0 = vld1_u16(p);
+ int32x4_t p_vec0 = vreinterpretq_s32_u32(vmovl_u16(p0));
+ if (use_dist_wtd_comp_avg) {
+ p_vec0 = vmulq_n_s32(p_vec0, fwd);
+ p_vec0 = vmlaq_n_s32(p_vec0, sum0, bwd);
+ p_vec0 = vshrq_n_s32(p_vec0, DIST_PRECISION_BITS);
+ } else {
+ p_vec0 = vhaddq_s32(p_vec0, sum0);
+ }
+
+ const int offset_bits = bd + 2 * FILTER_BITS - round0;
+ const int round1 = COMPOUND_ROUND1_BITS;
+ const int res_sub_const =
+ (1 << (offset_bits - round1)) + (1 << (offset_bits - round1 - 1));
+ const int round_bits = 2 * FILTER_BITS - round0 - round1;
+
+ p_vec0 = vsubq_s32(p_vec0, vdupq_n_s32(res_sub_const));
+ p_vec0 = vrshlq_s32(p_vec0, vdupq_n_s32(-round_bits));
+ uint16x4_t res0 = clip_pixel_highbd_vec(p_vec0, bd);
+ vst1_u16(dst16, res0);
+}
+
+static INLINE void warp_affine_vertical_step_8x1_f8_neon(
+ uint16_t *pred, int p_stride, int bd, uint16_t *dst, int dst_stride,
+ bool is_compound, bool do_average, bool use_dist_wtd_comp_avg, int fwd,
+ int bwd, int16_t gamma, const int16x8_t *tmp, int i, int sy, int j) {
+ int32x4x2_t sums =
+ gamma == 0 ? warp_affine_vertical_filter_8x1_f1_neon(tmp, sy)
+ : warp_affine_vertical_filter_8x1_f8_neon(tmp, sy, gamma);
+ int32x4_t sum0 = sums.val[0];
+ int32x4_t sum1 = sums.val[1];
+
+ const int round0 = (bd == 12) ? ROUND0_BITS + 2 : ROUND0_BITS;
+ const int offset_bits_vert = bd + 2 * FILTER_BITS - round0;
+
+ sum0 = vaddq_s32(sum0, vdupq_n_s32(1 << offset_bits_vert));
+ sum1 = vaddq_s32(sum1, vdupq_n_s32(1 << offset_bits_vert));
+
+ uint16_t *dst16 = &pred[i * p_stride + j];
+
+ if (!is_compound) {
+ const int reduce_bits_vert = 2 * FILTER_BITS - round0;
+ sum0 = vrshlq_s32(sum0, vdupq_n_s32(-reduce_bits_vert));
+ sum1 = vrshlq_s32(sum1, vdupq_n_s32(-reduce_bits_vert));
+
+ const int res_sub_const = (1 << (bd - 1)) + (1 << bd);
+ sum0 = vsubq_s32(sum0, vdupq_n_s32(res_sub_const));
+ sum1 = vsubq_s32(sum1, vdupq_n_s32(res_sub_const));
+ uint16x4_t res0 = clip_pixel_highbd_vec(sum0, bd);
+ uint16x4_t res1 = clip_pixel_highbd_vec(sum1, bd);
+ vst1_u16(dst16, res0);
+ vst1_u16(dst16 + 4, res1);
+ return;
+ }
+
+ sum0 = vrshrq_n_s32(sum0, COMPOUND_ROUND1_BITS);
+ sum1 = vrshrq_n_s32(sum1, COMPOUND_ROUND1_BITS);
+
+ uint16_t *p = &dst[i * dst_stride + j];
+
+ if (!do_average) {
+ vst1_u16(p, vqmovun_s32(sum0));
+ vst1_u16(p + 4, vqmovun_s32(sum1));
+ return;
+ }
+
+ uint16x8_t p0 = vld1q_u16(p);
+ int32x4_t p_vec0 = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(p0)));
+ int32x4_t p_vec1 = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(p0)));
+ if (use_dist_wtd_comp_avg) {
+ p_vec0 = vmulq_n_s32(p_vec0, fwd);
+ p_vec1 = vmulq_n_s32(p_vec1, fwd);
+ p_vec0 = vmlaq_n_s32(p_vec0, sum0, bwd);
+ p_vec1 = vmlaq_n_s32(p_vec1, sum1, bwd);
+ p_vec0 = vshrq_n_s32(p_vec0, DIST_PRECISION_BITS);
+ p_vec1 = vshrq_n_s32(p_vec1, DIST_PRECISION_BITS);
+ } else {
+ p_vec0 = vhaddq_s32(p_vec0, sum0);
+ p_vec1 = vhaddq_s32(p_vec1, sum1);
+ }
+
+ const int offset_bits = bd + 2 * FILTER_BITS - round0;
+ const int round1 = COMPOUND_ROUND1_BITS;
+ const int res_sub_const =
+ (1 << (offset_bits - round1)) + (1 << (offset_bits - round1 - 1));
+ const int round_bits = 2 * FILTER_BITS - round0 - round1;
+
+ p_vec0 = vsubq_s32(p_vec0, vdupq_n_s32(res_sub_const));
+ p_vec1 = vsubq_s32(p_vec1, vdupq_n_s32(res_sub_const));
+
+ p_vec0 = vrshlq_s32(p_vec0, vdupq_n_s32(-round_bits));
+ p_vec1 = vrshlq_s32(p_vec1, vdupq_n_s32(-round_bits));
+ uint16x4_t res0 = clip_pixel_highbd_vec(p_vec0, bd);
+ uint16x4_t res1 = clip_pixel_highbd_vec(p_vec1, bd);
+ vst1_u16(dst16, res0);
+ vst1_u16(dst16 + 4, res1);
+}
+
+static INLINE void warp_affine_vertical_neon(
+ uint16_t *pred, int p_width, int p_height, int p_stride, int bd,
+ uint16_t *dst, int dst_stride, bool is_compound, bool do_average,
+ bool use_dist_wtd_comp_avg, int fwd, int bwd, int16_t gamma, int16_t delta,
+ const int16x8_t *tmp, int i, int sy4, int j) {
+ int limit_height = p_height > 4 ? 8 : 4;
+
+ if (p_width > 4) {
+ // p_width == 8
+ for (int k = 0; k < limit_height; ++k) {
+ int sy = sy4 + delta * k;
+ warp_affine_vertical_step_8x1_f8_neon(
+ pred, p_stride, bd, dst, dst_stride, is_compound, do_average,
+ use_dist_wtd_comp_avg, fwd, bwd, gamma, tmp + k, i + k, sy, j);
+ }
+ } else {
+ // p_width == 4
+ for (int k = 0; k < limit_height; ++k) {
+ int sy = sy4 + delta * k;
+ warp_affine_vertical_step_4x1_f4_neon(
+ pred, p_stride, bd, dst, dst_stride, is_compound, do_average,
+ use_dist_wtd_comp_avg, fwd, bwd, gamma, tmp + k, i + k, sy, j);
+ }
+ }
+}
+
+void av1_highbd_warp_affine_neon(const int32_t *mat, const uint16_t *ref,
+ int width, int height, int stride,
+ uint16_t *pred, int p_col, int p_row,
+ int p_width, int p_height, int p_stride,
+ int subsampling_x, int subsampling_y, int bd,
+ ConvolveParams *conv_params, int16_t alpha,
+ int16_t beta, int16_t gamma, int16_t delta) {
+ uint16_t *const dst = conv_params->dst;
+ const int dst_stride = conv_params->dst_stride;
+ const bool is_compound = conv_params->is_compound;
+ const bool do_average = conv_params->do_average;
+ const bool use_dist_wtd_comp_avg = conv_params->use_dist_wtd_comp_avg;
+ const int fwd = conv_params->fwd_offset;
+ const int bwd = conv_params->bck_offset;
+
+ assert(IMPLIES(is_compound, dst != NULL));
+
+ for (int i = 0; i < p_height; i += 8) {
+ for (int j = 0; j < p_width; j += 8) {
+ // Calculate the center of this 8x8 block,
+ // project to luma coordinates (if in a subsampled chroma plane),
+ // apply the affine transformation,
+ // then convert back to the original coordinates (if necessary)
+ const int32_t src_x = (j + 4 + p_col) << subsampling_x;
+ const int32_t src_y = (i + 4 + p_row) << subsampling_y;
+ const int64_t dst_x =
+ (int64_t)mat[2] * src_x + (int64_t)mat[3] * src_y + (int64_t)mat[0];
+ const int64_t dst_y =
+ (int64_t)mat[4] * src_x + (int64_t)mat[5] * src_y + (int64_t)mat[1];
+ const int64_t x4 = dst_x >> subsampling_x;
+ const int64_t y4 = dst_y >> subsampling_y;
+
+ const int32_t ix4 = (int32_t)(x4 >> WARPEDMODEL_PREC_BITS);
+ int32_t sx4 = x4 & ((1 << WARPEDMODEL_PREC_BITS) - 1);
+ const int32_t iy4 = (int32_t)(y4 >> WARPEDMODEL_PREC_BITS);
+ int32_t sy4 = y4 & ((1 << WARPEDMODEL_PREC_BITS) - 1);
+
+ sx4 += alpha * (-4) + beta * (-4);
+ sy4 += gamma * (-4) + delta * (-4);
+
+ sx4 &= ~((1 << WARP_PARAM_REDUCE_BITS) - 1);
+ sy4 &= ~((1 << WARP_PARAM_REDUCE_BITS) - 1);
+
+ // Each horizontal filter result is formed by the sum of up to eight
+ // multiplications by filter values and then a shift. Although both the
+ // inputs and filters are loaded as int16, the input data is at most bd
+ // bits and the filters are at most 8 bits each. Additionally since we
+ // know all possible filter values we know that the sum of absolute
+ // filter values will fit in at most 9 bits. With this in mind we can
+ // conclude that the sum of each filter application will fit in bd + 9
+ // bits. The shift following the summation is ROUND0_BITS (which is 3),
+ // +2 for 12-bit, which gives us a final storage of:
+ // bd == 8: ( 8 + 9) - 3 => 14 bits
+ // bd == 10: (10 + 9) - 3 => 16 bits
+ // bd == 12: (12 + 9) - 5 => 16 bits
+ // So it is safe to use int16x8_t as the intermediate storage type here.
+ int16x8_t tmp[15];
+
+ warp_affine_horizontal_neon(ref, width, height, stride, p_width, alpha,
+ beta, iy4, sx4, ix4, tmp, bd);
+ warp_affine_vertical_neon(pred, p_width, p_height, p_stride, bd, dst,
+ dst_stride, is_compound, do_average,
+ use_dist_wtd_comp_avg, fwd, bwd, gamma, delta,
+ tmp, i, sy4, j);
+ }
+ }
+}
diff --git a/av1/common/arm/highbd_wiener_convolve_neon.c b/av1/common/arm/highbd_wiener_convolve_neon.c
new file mode 100644
index 0000000..a6bd6d3
--- /dev/null
+++ b/av1/common/arm/highbd_wiener_convolve_neon.c
@@ -0,0 +1,403 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+#include <assert.h>
+
+#include "aom_dsp/arm/mem_neon.h"
+#include "av1/common/convolve.h"
+#include "config/aom_config.h"
+#include "config/av1_rtcd.h"
+
+#define HBD_WIENER_5TAP_HORIZ(name, shift) \
+ static INLINE uint16x8_t name##_wiener_convolve5_8_2d_h( \
+ const int16x8_t s0, const int16x8_t s1, const int16x8_t s2, \
+ const int16x8_t s3, const int16x8_t s4, const int16x4_t x_filter, \
+ const int32x4_t round_vec, const uint16x8_t im_max_val) { \
+ /* Wiener filter is symmetric so add mirrored source elements. */ \
+ int16x8_t s04 = vaddq_s16(s0, s4); \
+ int16x8_t s13 = vaddq_s16(s1, s3); \
+ \
+ /* x_filter[0] = 0. (5-tap filters are 0-padded to 7 taps.) */ \
+ int32x4_t sum_lo = \
+ vmlal_lane_s16(round_vec, vget_low_s16(s04), x_filter, 1); \
+ sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(s13), x_filter, 2); \
+ sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(s2), x_filter, 3); \
+ \
+ int32x4_t sum_hi = \
+ vmlal_lane_s16(round_vec, vget_high_s16(s04), x_filter, 1); \
+ sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(s13), x_filter, 2); \
+ sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(s2), x_filter, 3); \
+ \
+ uint16x4_t res_lo = vqrshrun_n_s32(sum_lo, shift); \
+ uint16x4_t res_hi = vqrshrun_n_s32(sum_hi, shift); \
+ \
+ return vminq_u16(vcombine_u16(res_lo, res_hi), im_max_val); \
+ } \
+ \
+ static INLINE void name##_convolve_add_src_5tap_horiz( \
+ const uint16_t *src_ptr, ptrdiff_t src_stride, uint16_t *dst_ptr, \
+ ptrdiff_t dst_stride, int w, int h, const int16x4_t x_filter, \
+ const int32x4_t round_vec, const uint16x8_t im_max_val) { \
+ do { \
+ const int16_t *s = (int16_t *)src_ptr; \
+ uint16_t *d = dst_ptr; \
+ int width = w; \
+ \
+ do { \
+ int16x8_t s0, s1, s2, s3, s4; \
+ load_s16_8x5(s, 1, &s0, &s1, &s2, &s3, &s4); \
+ \
+ uint16x8_t d0 = name##_wiener_convolve5_8_2d_h( \
+ s0, s1, s2, s3, s4, x_filter, round_vec, im_max_val); \
+ \
+ vst1q_u16(d, d0); \
+ \
+ s += 8; \
+ d += 8; \
+ width -= 8; \
+ } while (width != 0); \
+ src_ptr += src_stride; \
+ dst_ptr += dst_stride; \
+ } while (--h != 0); \
+ }
+
+HBD_WIENER_5TAP_HORIZ(highbd, WIENER_ROUND0_BITS)
+HBD_WIENER_5TAP_HORIZ(highbd_12, WIENER_ROUND0_BITS + 2)
+
+#undef HBD_WIENER_5TAP_HORIZ
+
+#define HBD_WIENER_7TAP_HORIZ(name, shift) \
+ static INLINE uint16x8_t name##_wiener_convolve7_8_2d_h( \
+ const int16x8_t s0, const int16x8_t s1, const int16x8_t s2, \
+ const int16x8_t s3, const int16x8_t s4, const int16x8_t s5, \
+ const int16x8_t s6, const int16x4_t x_filter, const int32x4_t round_vec, \
+ const uint16x8_t im_max_val) { \
+ /* Wiener filter is symmetric so add mirrored source elements. */ \
+ int16x8_t s06 = vaddq_s16(s0, s6); \
+ int16x8_t s15 = vaddq_s16(s1, s5); \
+ int16x8_t s24 = vaddq_s16(s2, s4); \
+ \
+ int32x4_t sum_lo = \
+ vmlal_lane_s16(round_vec, vget_low_s16(s06), x_filter, 0); \
+ sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(s15), x_filter, 1); \
+ sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(s24), x_filter, 2); \
+ sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(s3), x_filter, 3); \
+ \
+ int32x4_t sum_hi = \
+ vmlal_lane_s16(round_vec, vget_high_s16(s06), x_filter, 0); \
+ sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(s15), x_filter, 1); \
+ sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(s24), x_filter, 2); \
+ sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(s3), x_filter, 3); \
+ \
+ uint16x4_t res_lo = vqrshrun_n_s32(sum_lo, shift); \
+ uint16x4_t res_hi = vqrshrun_n_s32(sum_hi, shift); \
+ \
+ return vminq_u16(vcombine_u16(res_lo, res_hi), im_max_val); \
+ } \
+ \
+ static INLINE void name##_convolve_add_src_7tap_horiz( \
+ const uint16_t *src_ptr, ptrdiff_t src_stride, uint16_t *dst_ptr, \
+ ptrdiff_t dst_stride, int w, int h, const int16x4_t x_filter, \
+ const int32x4_t round_vec, const uint16x8_t im_max_val) { \
+ do { \
+ const int16_t *s = (int16_t *)src_ptr; \
+ uint16_t *d = dst_ptr; \
+ int width = w; \
+ \
+ do { \
+ int16x8_t s0, s1, s2, s3, s4, s5, s6; \
+ load_s16_8x7(s, 1, &s0, &s1, &s2, &s3, &s4, &s5, &s6); \
+ \
+ uint16x8_t d0 = name##_wiener_convolve7_8_2d_h( \
+ s0, s1, s2, s3, s4, s5, s6, x_filter, round_vec, im_max_val); \
+ \
+ vst1q_u16(d, d0); \
+ \
+ s += 8; \
+ d += 8; \
+ width -= 8; \
+ } while (width != 0); \
+ src_ptr += src_stride; \
+ dst_ptr += dst_stride; \
+ } while (--h != 0); \
+ }
+
+HBD_WIENER_7TAP_HORIZ(highbd, WIENER_ROUND0_BITS)
+HBD_WIENER_7TAP_HORIZ(highbd_12, WIENER_ROUND0_BITS + 2)
+
+#undef HBD_WIENER_7TAP_HORIZ
+
+#define HBD_WIENER_5TAP_VERT(name, shift) \
+ static INLINE uint16x8_t name##_wiener_convolve5_8_2d_v( \
+ const int16x8_t s0, const int16x8_t s1, const int16x8_t s2, \
+ const int16x8_t s3, const int16x8_t s4, const int16x4_t y_filter, \
+ const int32x4_t round_vec, const uint16x8_t res_max_val) { \
+ const int32x2_t y_filter_lo = vget_low_s32(vmovl_s16(y_filter)); \
+ const int32x2_t y_filter_hi = vget_high_s32(vmovl_s16(y_filter)); \
+ /* Wiener filter is symmetric so add mirrored source elements. */ \
+ int32x4_t s04_lo = vaddl_s16(vget_low_s16(s0), vget_low_s16(s4)); \
+ int32x4_t s13_lo = vaddl_s16(vget_low_s16(s1), vget_low_s16(s3)); \
+ \
+ /* y_filter[0] = 0. (5-tap filters are 0-padded to 7 taps.) */ \
+ int32x4_t sum_lo = vmlaq_lane_s32(round_vec, s04_lo, y_filter_lo, 1); \
+ sum_lo = vmlaq_lane_s32(sum_lo, s13_lo, y_filter_hi, 0); \
+ sum_lo = \
+ vmlaq_lane_s32(sum_lo, vmovl_s16(vget_low_s16(s2)), y_filter_hi, 1); \
+ \
+ int32x4_t s04_hi = vaddl_s16(vget_high_s16(s0), vget_high_s16(s4)); \
+ int32x4_t s13_hi = vaddl_s16(vget_high_s16(s1), vget_high_s16(s3)); \
+ \
+ int32x4_t sum_hi = vmlaq_lane_s32(round_vec, s04_hi, y_filter_lo, 1); \
+ sum_hi = vmlaq_lane_s32(sum_hi, s13_hi, y_filter_hi, 0); \
+ sum_hi = \
+ vmlaq_lane_s32(sum_hi, vmovl_s16(vget_high_s16(s2)), y_filter_hi, 1); \
+ \
+ uint16x4_t res_lo = vqrshrun_n_s32(sum_lo, shift); \
+ uint16x4_t res_hi = vqrshrun_n_s32(sum_hi, shift); \
+ \
+ return vminq_u16(vcombine_u16(res_lo, res_hi), res_max_val); \
+ } \
+ \
+ static INLINE void name##_convolve_add_src_5tap_vert( \
+ const uint16_t *src_ptr, ptrdiff_t src_stride, uint16_t *dst_ptr, \
+ ptrdiff_t dst_stride, int w, int h, const int16x4_t y_filter, \
+ const int32x4_t round_vec, const uint16x8_t res_max_val) { \
+ do { \
+ const int16_t *s = (int16_t *)src_ptr; \
+ uint16_t *d = dst_ptr; \
+ int height = h; \
+ \
+ while (height > 3) { \
+ int16x8_t s0, s1, s2, s3, s4, s5, s6, s7; \
+ load_s16_8x8(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7); \
+ \
+ uint16x8_t d0 = name##_wiener_convolve5_8_2d_v( \
+ s0, s1, s2, s3, s4, y_filter, round_vec, res_max_val); \
+ uint16x8_t d1 = name##_wiener_convolve5_8_2d_v( \
+ s1, s2, s3, s4, s5, y_filter, round_vec, res_max_val); \
+ uint16x8_t d2 = name##_wiener_convolve5_8_2d_v( \
+ s2, s3, s4, s5, s6, y_filter, round_vec, res_max_val); \
+ uint16x8_t d3 = name##_wiener_convolve5_8_2d_v( \
+ s3, s4, s5, s6, s7, y_filter, round_vec, res_max_val); \
+ \
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3); \
+ \
+ s += 4 * src_stride; \
+ d += 4 * dst_stride; \
+ height -= 4; \
+ } \
+ \
+ while (height-- != 0) { \
+ int16x8_t s0, s1, s2, s3, s4; \
+ load_s16_8x5(s, src_stride, &s0, &s1, &s2, &s3, &s4); \
+ \
+ uint16x8_t d0 = name##_wiener_convolve5_8_2d_v( \
+ s0, s1, s2, s3, s4, y_filter, round_vec, res_max_val); \
+ \
+ vst1q_u16(d, d0); \
+ \
+ s += src_stride; \
+ d += dst_stride; \
+ } \
+ \
+ src_ptr += 8; \
+ dst_ptr += 8; \
+ w -= 8; \
+ } while (w != 0); \
+ }
+
+HBD_WIENER_5TAP_VERT(highbd, 2 * FILTER_BITS - WIENER_ROUND0_BITS)
+HBD_WIENER_5TAP_VERT(highbd_12, 2 * FILTER_BITS - WIENER_ROUND0_BITS - 2)
+
+#undef HBD_WIENER_5TAP_VERT
+
+#define HBD_WIENER_7TAP_VERT(name, shift) \
+ static INLINE uint16x8_t name##_wiener_convolve7_8_2d_v( \
+ const int16x8_t s0, const int16x8_t s1, const int16x8_t s2, \
+ const int16x8_t s3, const int16x8_t s4, const int16x8_t s5, \
+ const int16x8_t s6, const int16x4_t y_filter, const int32x4_t round_vec, \
+ const uint16x8_t res_max_val) { \
+ const int32x2_t y_filter_lo = vget_low_s32(vmovl_s16(y_filter)); \
+ const int32x2_t y_filter_hi = vget_high_s32(vmovl_s16(y_filter)); \
+ /* Wiener filter is symmetric so add mirrored source elements. */ \
+ int32x4_t s06_lo = vaddl_s16(vget_low_s16(s0), vget_low_s16(s6)); \
+ int32x4_t s15_lo = vaddl_s16(vget_low_s16(s1), vget_low_s16(s5)); \
+ int32x4_t s24_lo = vaddl_s16(vget_low_s16(s2), vget_low_s16(s4)); \
+ \
+ int32x4_t sum_lo = vmlaq_lane_s32(round_vec, s06_lo, y_filter_lo, 0); \
+ sum_lo = vmlaq_lane_s32(sum_lo, s15_lo, y_filter_lo, 1); \
+ sum_lo = vmlaq_lane_s32(sum_lo, s24_lo, y_filter_hi, 0); \
+ sum_lo = \
+ vmlaq_lane_s32(sum_lo, vmovl_s16(vget_low_s16(s3)), y_filter_hi, 1); \
+ \
+ int32x4_t s06_hi = vaddl_s16(vget_high_s16(s0), vget_high_s16(s6)); \
+ int32x4_t s15_hi = vaddl_s16(vget_high_s16(s1), vget_high_s16(s5)); \
+ int32x4_t s24_hi = vaddl_s16(vget_high_s16(s2), vget_high_s16(s4)); \
+ \
+ int32x4_t sum_hi = vmlaq_lane_s32(round_vec, s06_hi, y_filter_lo, 0); \
+ sum_hi = vmlaq_lane_s32(sum_hi, s15_hi, y_filter_lo, 1); \
+ sum_hi = vmlaq_lane_s32(sum_hi, s24_hi, y_filter_hi, 0); \
+ sum_hi = \
+ vmlaq_lane_s32(sum_hi, vmovl_s16(vget_high_s16(s3)), y_filter_hi, 1); \
+ \
+ uint16x4_t res_lo = vqrshrun_n_s32(sum_lo, shift); \
+ uint16x4_t res_hi = vqrshrun_n_s32(sum_hi, shift); \
+ \
+ return vminq_u16(vcombine_u16(res_lo, res_hi), res_max_val); \
+ } \
+ \
+ static INLINE void name##_convolve_add_src_7tap_vert( \
+ const uint16_t *src_ptr, ptrdiff_t src_stride, uint16_t *dst_ptr, \
+ ptrdiff_t dst_stride, int w, int h, const int16x4_t y_filter, \
+ const int32x4_t round_vec, const uint16x8_t res_max_val) { \
+ do { \
+ const int16_t *s = (int16_t *)src_ptr; \
+ uint16_t *d = dst_ptr; \
+ int height = h; \
+ \
+ while (height > 3) { \
+ int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9; \
+ load_s16_8x10(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7, \
+ &s8, &s9); \
+ \
+ uint16x8_t d0 = name##_wiener_convolve7_8_2d_v( \
+ s0, s1, s2, s3, s4, s5, s6, y_filter, round_vec, res_max_val); \
+ uint16x8_t d1 = name##_wiener_convolve7_8_2d_v( \
+ s1, s2, s3, s4, s5, s6, s7, y_filter, round_vec, res_max_val); \
+ uint16x8_t d2 = name##_wiener_convolve7_8_2d_v( \
+ s2, s3, s4, s5, s6, s7, s8, y_filter, round_vec, res_max_val); \
+ uint16x8_t d3 = name##_wiener_convolve7_8_2d_v( \
+ s3, s4, s5, s6, s7, s8, s9, y_filter, round_vec, res_max_val); \
+ \
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3); \
+ \
+ s += 4 * src_stride; \
+ d += 4 * dst_stride; \
+ height -= 4; \
+ } \
+ \
+ while (height-- != 0) { \
+ int16x8_t s0, s1, s2, s3, s4, s5, s6; \
+ load_s16_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6); \
+ \
+ uint16x8_t d0 = name##_wiener_convolve7_8_2d_v( \
+ s0, s1, s2, s3, s4, s5, s6, y_filter, round_vec, res_max_val); \
+ \
+ vst1q_u16(d, d0); \
+ \
+ s += src_stride; \
+ d += dst_stride; \
+ } \
+ \
+ src_ptr += 8; \
+ dst_ptr += 8; \
+ w -= 8; \
+ } while (w != 0); \
+ }
+
+HBD_WIENER_7TAP_VERT(highbd, 2 * FILTER_BITS - WIENER_ROUND0_BITS)
+HBD_WIENER_7TAP_VERT(highbd_12, 2 * FILTER_BITS - WIENER_ROUND0_BITS - 2)
+
+#undef HBD_WIENER_7TAP_VERT
+
+static AOM_INLINE int get_wiener_filter_taps(const int16_t *filter) {
+ assert(filter[7] == 0);
+ if (filter[0] == 0 && filter[6] == 0) {
+ return WIENER_WIN_REDUCED;
+ }
+ return WIENER_WIN;
+}
+
+void av1_highbd_wiener_convolve_add_src_neon(
+ const uint8_t *src8, ptrdiff_t src_stride, uint8_t *dst8,
+ ptrdiff_t dst_stride, const int16_t *x_filter, int x_step_q4,
+ const int16_t *y_filter, int y_step_q4, int w, int h,
+ const WienerConvolveParams *conv_params, int bd) {
+ (void)x_step_q4;
+ (void)y_step_q4;
+
+ assert(w % 8 == 0);
+ assert(w <= MAX_SB_SIZE && h <= MAX_SB_SIZE);
+ assert(x_step_q4 == 16 && y_step_q4 == 16);
+ assert(x_filter[7] == 0 && y_filter[7] == 0);
+
+ DECLARE_ALIGNED(16, uint16_t,
+ im_block[(MAX_SB_SIZE + WIENER_WIN - 1) * MAX_SB_SIZE]);
+
+ const int x_filter_taps = get_wiener_filter_taps(x_filter);
+ const int y_filter_taps = get_wiener_filter_taps(y_filter);
+ int16x4_t x_filter_s16 = vld1_s16(x_filter);
+ int16x4_t y_filter_s16 = vld1_s16(y_filter);
+ // Add 128 to tap 3. (Needed for rounding.)
+ x_filter_s16 = vadd_s16(x_filter_s16, vcreate_s16(128ULL << 48));
+ y_filter_s16 = vadd_s16(y_filter_s16, vcreate_s16(128ULL << 48));
+
+ const int im_stride = MAX_SB_SIZE;
+ const int im_h = h + y_filter_taps - 1;
+ const int horiz_offset = x_filter_taps / 2;
+ const int vert_offset = (y_filter_taps / 2) * (int)src_stride;
+
+ const int extraprec_clamp_limit =
+ WIENER_CLAMP_LIMIT(conv_params->round_0, bd);
+ const uint16x8_t im_max_val = vdupq_n_u16(extraprec_clamp_limit - 1);
+ const int32x4_t horiz_round_vec = vdupq_n_s32(1 << (bd + FILTER_BITS - 1));
+
+ const uint16x8_t res_max_val = vdupq_n_u16((1 << bd) - 1);
+ const int32x4_t vert_round_vec =
+ vdupq_n_s32(-(1 << (bd + conv_params->round_1 - 1)));
+
+ uint16_t *src = CONVERT_TO_SHORTPTR(src8);
+ uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
+
+ if (bd == 12) {
+ if (x_filter_taps == WIENER_WIN_REDUCED) {
+ highbd_12_convolve_add_src_5tap_horiz(
+ src - horiz_offset - vert_offset, src_stride, im_block, im_stride, w,
+ im_h, x_filter_s16, horiz_round_vec, im_max_val);
+ } else {
+ highbd_12_convolve_add_src_7tap_horiz(
+ src - horiz_offset - vert_offset, src_stride, im_block, im_stride, w,
+ im_h, x_filter_s16, horiz_round_vec, im_max_val);
+ }
+
+ if (y_filter_taps == WIENER_WIN_REDUCED) {
+ highbd_12_convolve_add_src_5tap_vert(im_block, im_stride, dst, dst_stride,
+ w, h, y_filter_s16, vert_round_vec,
+ res_max_val);
+ } else {
+ highbd_12_convolve_add_src_7tap_vert(im_block, im_stride, dst, dst_stride,
+ w, h, y_filter_s16, vert_round_vec,
+ res_max_val);
+ }
+
+ } else {
+ if (x_filter_taps == WIENER_WIN_REDUCED) {
+ highbd_convolve_add_src_5tap_horiz(
+ src - horiz_offset - vert_offset, src_stride, im_block, im_stride, w,
+ im_h, x_filter_s16, horiz_round_vec, im_max_val);
+ } else {
+ highbd_convolve_add_src_7tap_horiz(
+ src - horiz_offset - vert_offset, src_stride, im_block, im_stride, w,
+ im_h, x_filter_s16, horiz_round_vec, im_max_val);
+ }
+
+ if (y_filter_taps == WIENER_WIN_REDUCED) {
+ highbd_convolve_add_src_5tap_vert(im_block, im_stride, dst, dst_stride, w,
+ h, y_filter_s16, vert_round_vec,
+ res_max_val);
+ } else {
+ highbd_convolve_add_src_7tap_vert(im_block, im_stride, dst, dst_stride, w,
+ h, y_filter_s16, vert_round_vec,
+ res_max_val);
+ }
+ }
+}
diff --git a/av1/common/arm/jnt_convolve_neon.c b/av1/common/arm/jnt_convolve_neon.c
deleted file mode 100644
index 564f7c2..0000000
--- a/av1/common/arm/jnt_convolve_neon.c
+++ /dev/null
@@ -1,5336 +0,0 @@
-/*
- * Copyright (c) 2018, Alliance for Open Media. All rights reserved
- *
- * This source code is subject to the terms of the BSD 2 Clause License and
- * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
- * was not distributed with this source code in the LICENSE file, you can
- * obtain it at www.aomedia.org/license/software. If the Alliance for Open
- * Media Patent License 1.0 was not distributed with this source code in the
- * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
- */
-
-#include <arm_neon.h>
-#include <assert.h>
-
-#include "config/aom_config.h"
-#include "config/av1_rtcd.h"
-
-#include "aom_dsp/txfm_common.h"
-#include "aom_dsp/arm/mem_neon.h"
-#include "aom_dsp/arm/transpose_neon.h"
-#include "aom_ports/mem.h"
-#include "av1/common/common.h"
-#include "av1/common/arm/convolve_neon.h"
-
-#if !AOM_ARCH_AARCH64
-static INLINE void compute_dist_wtd_avg_4x1(uint16x4_t dd0, uint16x4_t d0,
- const uint16_t fwd_offset,
- const uint16_t bck_offset,
- const int16x4_t round_offset,
- uint8x8_t *d0_u8) {
- uint32x4_t blend0 = vmull_n_u16(dd0, fwd_offset);
- blend0 = vmlal_n_u16(blend0, d0, bck_offset);
-
- uint16x4_t avg0 = vshrn_n_u32(blend0, DIST_PRECISION_BITS);
-
- int16x4_t dst0 = vsub_s16(vreinterpret_s16_u16(avg0), round_offset);
-
- int16x8_t dst0q = vcombine_s16(dst0, vdup_n_s16(0));
-
- *d0_u8 = vqrshrun_n_s16(dst0q, FILTER_BITS - ROUND0_BITS);
-}
-
-static INLINE void compute_basic_avg_4x1(uint16x4_t dd0, uint16x4_t d0,
- const int16x4_t round_offset,
- uint8x8_t *d0_u8) {
- uint16x4_t avg0 = vhadd_u16(dd0, d0);
-
- int16x4_t dst0 = vsub_s16(vreinterpret_s16_u16(avg0), round_offset);
-
- int16x8_t dst0q = vcombine_s16(dst0, vdup_n_s16(0));
-
- *d0_u8 = vqrshrun_n_s16(dst0q, FILTER_BITS - ROUND0_BITS);
-}
-
-static INLINE void compute_dist_wtd_avg_8x1(uint16x8_t dd0, uint16x8_t d0,
- const uint16_t fwd_offset,
- const uint16_t bck_offset,
- const int16x8_t round_offset,
- uint8x8_t *d0_u8) {
- uint32x4_t blend0_lo = vmull_n_u16(vget_low_u16(dd0), fwd_offset);
- blend0_lo = vmlal_n_u16(blend0_lo, vget_low_u16(d0), bck_offset);
- uint32x4_t blend0_hi = vmull_n_u16(vget_high_u16(dd0), fwd_offset);
- blend0_hi = vmlal_n_u16(blend0_hi, vget_high_u16(d0), bck_offset);
-
- uint16x8_t avg0 = vcombine_u16(vshrn_n_u32(blend0_lo, DIST_PRECISION_BITS),
- vshrn_n_u32(blend0_hi, DIST_PRECISION_BITS));
-
- int16x8_t dst0 = vsubq_s16(vreinterpretq_s16_u16(avg0), round_offset);
-
- *d0_u8 = vqrshrun_n_s16(dst0, FILTER_BITS - ROUND0_BITS);
-}
-
-static INLINE void compute_basic_avg_8x1(uint16x8_t dd0, uint16x8_t d0,
- const int16x8_t round_offset,
- uint8x8_t *d0_u8) {
- uint16x8_t avg0 = vhaddq_u16(dd0, d0);
-
- int16x8_t dst0 = vsubq_s16(vreinterpretq_s16_u16(avg0), round_offset);
-
- *d0_u8 = vqrshrun_n_s16(dst0, FILTER_BITS - ROUND0_BITS);
-}
-
-#endif // !AOM_ARCH_AARCH64
-
-static INLINE void compute_dist_wtd_avg_4x4(
- uint16x4_t dd0, uint16x4_t dd1, uint16x4_t dd2, uint16x4_t dd3,
- uint16x4_t d0, uint16x4_t d1, uint16x4_t d2, uint16x4_t d3,
- const uint16_t fwd_offset, const uint16_t bck_offset,
- const int16x8_t round_offset, uint8x8_t *d01_u8, uint8x8_t *d23_u8) {
- uint32x4_t blend0 = vmull_n_u16(dd0, fwd_offset);
- blend0 = vmlal_n_u16(blend0, d0, bck_offset);
- uint32x4_t blend1 = vmull_n_u16(dd1, fwd_offset);
- blend1 = vmlal_n_u16(blend1, d1, bck_offset);
- uint32x4_t blend2 = vmull_n_u16(dd2, fwd_offset);
- blend2 = vmlal_n_u16(blend2, d2, bck_offset);
- uint32x4_t blend3 = vmull_n_u16(dd3, fwd_offset);
- blend3 = vmlal_n_u16(blend3, d3, bck_offset);
-
- uint16x4_t avg0 = vshrn_n_u32(blend0, DIST_PRECISION_BITS);
- uint16x4_t avg1 = vshrn_n_u32(blend1, DIST_PRECISION_BITS);
- uint16x4_t avg2 = vshrn_n_u32(blend2, DIST_PRECISION_BITS);
- uint16x4_t avg3 = vshrn_n_u32(blend3, DIST_PRECISION_BITS);
-
- int16x8_t dst_01 = vreinterpretq_s16_u16(vcombine_u16(avg0, avg1));
- int16x8_t dst_23 = vreinterpretq_s16_u16(vcombine_u16(avg2, avg3));
-
- dst_01 = vsubq_s16(dst_01, round_offset);
- dst_23 = vsubq_s16(dst_23, round_offset);
-
- *d01_u8 = vqrshrun_n_s16(dst_01, FILTER_BITS - ROUND0_BITS);
- *d23_u8 = vqrshrun_n_s16(dst_23, FILTER_BITS - ROUND0_BITS);
-}
-
-static INLINE void compute_basic_avg_4x4(uint16x4_t dd0, uint16x4_t dd1,
- uint16x4_t dd2, uint16x4_t dd3,
- uint16x4_t d0, uint16x4_t d1,
- uint16x4_t d2, uint16x4_t d3,
- const int16x8_t round_offset,
- uint8x8_t *d01_u8, uint8x8_t *d23_u8) {
- uint16x4_t avg0 = vhadd_u16(dd0, d0);
- uint16x4_t avg1 = vhadd_u16(dd1, d1);
- uint16x4_t avg2 = vhadd_u16(dd2, d2);
- uint16x4_t avg3 = vhadd_u16(dd3, d3);
-
- int16x8_t dst_01 = vreinterpretq_s16_u16(vcombine_u16(avg0, avg1));
- int16x8_t dst_23 = vreinterpretq_s16_u16(vcombine_u16(avg2, avg3));
-
- dst_01 = vsubq_s16(dst_01, round_offset);
- dst_23 = vsubq_s16(dst_23, round_offset);
-
- *d01_u8 = vqrshrun_n_s16(dst_01, FILTER_BITS - ROUND0_BITS);
- *d23_u8 = vqrshrun_n_s16(dst_23, FILTER_BITS - ROUND0_BITS);
-}
-
-static INLINE void compute_dist_wtd_avg_8x4(
- uint16x8_t dd0, uint16x8_t dd1, uint16x8_t dd2, uint16x8_t dd3,
- uint16x8_t d0, uint16x8_t d1, uint16x8_t d2, uint16x8_t d3,
- const uint16_t fwd_offset, const uint16_t bck_offset,
- const int16x8_t round_offset, uint8x8_t *d0_u8, uint8x8_t *d1_u8,
- uint8x8_t *d2_u8, uint8x8_t *d3_u8) {
- uint32x4_t blend0_lo = vmull_n_u16(vget_low_u16(dd0), fwd_offset);
- blend0_lo = vmlal_n_u16(blend0_lo, vget_low_u16(d0), bck_offset);
- uint32x4_t blend0_hi = vmull_n_u16(vget_high_u16(dd0), fwd_offset);
- blend0_hi = vmlal_n_u16(blend0_hi, vget_high_u16(d0), bck_offset);
-
- uint32x4_t blend1_lo = vmull_n_u16(vget_low_u16(dd1), fwd_offset);
- blend1_lo = vmlal_n_u16(blend1_lo, vget_low_u16(d1), bck_offset);
- uint32x4_t blend1_hi = vmull_n_u16(vget_high_u16(dd1), fwd_offset);
- blend1_hi = vmlal_n_u16(blend1_hi, vget_high_u16(d1), bck_offset);
-
- uint32x4_t blend2_lo = vmull_n_u16(vget_low_u16(dd2), fwd_offset);
- blend2_lo = vmlal_n_u16(blend2_lo, vget_low_u16(d2), bck_offset);
- uint32x4_t blend2_hi = vmull_n_u16(vget_high_u16(dd2), fwd_offset);
- blend2_hi = vmlal_n_u16(blend2_hi, vget_high_u16(d2), bck_offset);
-
- uint32x4_t blend3_lo = vmull_n_u16(vget_low_u16(dd3), fwd_offset);
- blend3_lo = vmlal_n_u16(blend3_lo, vget_low_u16(d3), bck_offset);
- uint32x4_t blend3_hi = vmull_n_u16(vget_high_u16(dd3), fwd_offset);
- blend3_hi = vmlal_n_u16(blend3_hi, vget_high_u16(d3), bck_offset);
-
- uint16x8_t avg0 = vcombine_u16(vshrn_n_u32(blend0_lo, DIST_PRECISION_BITS),
- vshrn_n_u32(blend0_hi, DIST_PRECISION_BITS));
- uint16x8_t avg1 = vcombine_u16(vshrn_n_u32(blend1_lo, DIST_PRECISION_BITS),
- vshrn_n_u32(blend1_hi, DIST_PRECISION_BITS));
- uint16x8_t avg2 = vcombine_u16(vshrn_n_u32(blend2_lo, DIST_PRECISION_BITS),
- vshrn_n_u32(blend2_hi, DIST_PRECISION_BITS));
- uint16x8_t avg3 = vcombine_u16(vshrn_n_u32(blend3_lo, DIST_PRECISION_BITS),
- vshrn_n_u32(blend3_hi, DIST_PRECISION_BITS));
-
- int16x8_t dst0 = vsubq_s16(vreinterpretq_s16_u16(avg0), round_offset);
- int16x8_t dst1 = vsubq_s16(vreinterpretq_s16_u16(avg1), round_offset);
- int16x8_t dst2 = vsubq_s16(vreinterpretq_s16_u16(avg2), round_offset);
- int16x8_t dst3 = vsubq_s16(vreinterpretq_s16_u16(avg3), round_offset);
-
- *d0_u8 = vqrshrun_n_s16(dst0, FILTER_BITS - ROUND0_BITS);
- *d1_u8 = vqrshrun_n_s16(dst1, FILTER_BITS - ROUND0_BITS);
- *d2_u8 = vqrshrun_n_s16(dst2, FILTER_BITS - ROUND0_BITS);
- *d3_u8 = vqrshrun_n_s16(dst3, FILTER_BITS - ROUND0_BITS);
-}
-
-static INLINE void compute_basic_avg_8x4(uint16x8_t dd0, uint16x8_t dd1,
- uint16x8_t dd2, uint16x8_t dd3,
- uint16x8_t d0, uint16x8_t d1,
- uint16x8_t d2, uint16x8_t d3,
- const int16x8_t round_offset,
- uint8x8_t *d0_u8, uint8x8_t *d1_u8,
- uint8x8_t *d2_u8, uint8x8_t *d3_u8) {
- uint16x8_t avg0, avg1, avg2, avg3;
-
- avg0 = vhaddq_u16(dd0, d0);
- avg1 = vhaddq_u16(dd1, d1);
- avg2 = vhaddq_u16(dd2, d2);
- avg3 = vhaddq_u16(dd3, d3);
-
- int16x8_t dst0 = vsubq_s16(vreinterpretq_s16_u16(avg0), round_offset);
- int16x8_t dst1 = vsubq_s16(vreinterpretq_s16_u16(avg1), round_offset);
- int16x8_t dst2 = vsubq_s16(vreinterpretq_s16_u16(avg2), round_offset);
- int16x8_t dst3 = vsubq_s16(vreinterpretq_s16_u16(avg3), round_offset);
-
- *d0_u8 = vqrshrun_n_s16(dst0, FILTER_BITS - ROUND0_BITS);
- *d1_u8 = vqrshrun_n_s16(dst1, FILTER_BITS - ROUND0_BITS);
- *d2_u8 = vqrshrun_n_s16(dst2, FILTER_BITS - ROUND0_BITS);
- *d3_u8 = vqrshrun_n_s16(dst3, FILTER_BITS - ROUND0_BITS);
-}
-
-#if AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_MATMUL_INT8)
-
-static INLINE int16x4_t convolve8_4_2d_h(uint8x16_t samples,
- const int8x8_t x_filter,
- const uint8x16x2_t permute_tbl,
- const int32x4_t horiz_const) {
- uint8x16_t permuted_samples[2];
- int32x4_t sum;
-
- // Permute samples ready for dot product.
- // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
- permuted_samples[0] = vqtbl1q_u8(samples, permute_tbl.val[0]);
- // { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 }
- permuted_samples[1] = vqtbl1q_u8(samples, permute_tbl.val[1]);
-
- // First 4 output values.
- sum = vusdotq_lane_s32(horiz_const, permuted_samples[0], x_filter, 0);
- sum = vusdotq_lane_s32(sum, permuted_samples[1], x_filter, 1);
-
- // We halved the convolution filter values so -1 from the right shift.
- return vshrn_n_s32(sum, ROUND0_BITS - 1);
-}
-
-static INLINE int16x8_t convolve8_8_2d_h(uint8x16_t samples,
- const int8x8_t x_filter,
- const uint8x16x3_t permute_tbl,
- const int32x4_t horiz_const) {
- uint8x16_t permuted_samples[3];
- int32x4_t sum[2];
-
- // Permute samples ready for dot product.
- // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
- permuted_samples[0] = vqtbl1q_u8(samples, permute_tbl.val[0]);
- // { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 }
- permuted_samples[1] = vqtbl1q_u8(samples, permute_tbl.val[1]);
- // { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 }
- permuted_samples[2] = vqtbl1q_u8(samples, permute_tbl.val[2]);
-
- // First 4 output values.
- sum[0] = vusdotq_lane_s32(horiz_const, permuted_samples[0], x_filter, 0);
- sum[0] = vusdotq_lane_s32(sum[0], permuted_samples[1], x_filter, 1);
- // Second 4 output values.
- sum[1] = vusdotq_lane_s32(horiz_const, permuted_samples[1], x_filter, 0);
- sum[1] = vusdotq_lane_s32(sum[1], permuted_samples[2], x_filter, 1);
-
- // Narrow and re-pack.
- // We halved the convolution filter values so -1 from the right shift.
- return vcombine_s16(vshrn_n_s32(sum[0], ROUND0_BITS - 1),
- vshrn_n_s32(sum[1], ROUND0_BITS - 1));
-}
-
-static INLINE void dist_wtd_convolve_2d_horiz_8tap_neon(
- const uint8_t *src, int src_stride, int16_t *im_block, const int im_stride,
- const int16x8_t x_filter_s16, const int im_h, int w) {
- const int bd = 8;
- // A shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-rounding
- // shifts - which are generally faster than rounding shifts on modern CPUs.
- // (The extra -1 is needed because we halved the filter values.)
- const int32x4_t horiz_const = vdupq_n_s32((1 << (bd + FILTER_BITS - 2)) +
- (1 << ((ROUND0_BITS - 1) - 1)));
- // Horizontal filter.
- const int8x8_t x_filter = vmovn_s16(x_filter_s16);
-
- const uint8_t *src_ptr = src;
- int16_t *dst_ptr = im_block;
- int dst_stride = im_stride;
- int height = im_h;
-
- if (w == 4) {
- const uint8x16x2_t permute_tbl = vld1q_u8_x2(dot_prod_permute_tbl);
- uint8x16_t s0, s1, s2, s3;
- int16x4_t d0, d1, d2, d3;
-
- do {
- load_u8_16x4(src_ptr, src_stride, &s0, &s1, &s2, &s3);
-
- d0 = convolve8_4_2d_h(s0, x_filter, permute_tbl, horiz_const);
- d1 = convolve8_4_2d_h(s1, x_filter, permute_tbl, horiz_const);
- d2 = convolve8_4_2d_h(s2, x_filter, permute_tbl, horiz_const);
- d3 = convolve8_4_2d_h(s3, x_filter, permute_tbl, horiz_const);
-
- store_s16_4x4(dst_ptr, dst_stride, d0, d1, d2, d3);
-
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- height -= 4;
- } while (height > 0);
- } else {
- const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
- uint8x16_t s0, s1, s2, s3;
- int16x8_t d0, d1, d2, d3;
-
- do {
- const uint8_t *s = src_ptr;
- int16_t *d = dst_ptr;
- int width = w;
-
- do {
- load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3);
-
- d0 = convolve8_8_2d_h(s0, x_filter, permute_tbl, horiz_const);
- d1 = convolve8_8_2d_h(s1, x_filter, permute_tbl, horiz_const);
- d2 = convolve8_8_2d_h(s2, x_filter, permute_tbl, horiz_const);
- d3 = convolve8_8_2d_h(s3, x_filter, permute_tbl, horiz_const);
-
- store_s16_8x4(d, dst_stride, d0, d1, d2, d3);
-
- s += 8;
- d += 8;
- width -= 8;
- } while (width > 0);
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- height -= 4;
- } while (height > 0);
- }
-}
-
-#elif AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_DOTPROD)
-
-static INLINE int16x4_t convolve8_4_2d_h(uint8x16_t samples,
- const int8x8_t x_filter,
- const int32x4_t correction,
- const uint8x16_t range_limit,
- const uint8x16x2_t permute_tbl) {
- int8x16_t clamped_samples, permuted_samples[2];
- int32x4_t sum;
-
- // Clamp sample range to [-128, 127] for 8-bit signed dot product.
- clamped_samples = vreinterpretq_s8_u8(vsubq_u8(samples, range_limit));
-
- // Permute samples ready for dot product.
- // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
- permuted_samples[0] = vqtbl1q_s8(clamped_samples, permute_tbl.val[0]);
- // { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 }
- permuted_samples[1] = vqtbl1q_s8(clamped_samples, permute_tbl.val[1]);
-
- // Accumulate dot product into 'correction' to account for range clamp.
- sum = vdotq_lane_s32(correction, permuted_samples[0], x_filter, 0);
- sum = vdotq_lane_s32(sum, permuted_samples[1], x_filter, 1);
-
- // We halved the convolution filter values so -1 from the right shift.
- return vshrn_n_s32(sum, ROUND0_BITS - 1);
-}
-
-static INLINE int16x8_t convolve8_8_2d_h(uint8x16_t samples,
- const int8x8_t x_filter,
- const int32x4_t correction,
- const uint8x16_t range_limit,
- const uint8x16x3_t permute_tbl) {
- int8x16_t clamped_samples, permuted_samples[3];
- int32x4_t sum[2];
-
- // Clamp sample range to [-128, 127] for 8-bit signed dot product.
- clamped_samples = vreinterpretq_s8_u8(vsubq_u8(samples, range_limit));
-
- // Permute samples ready for dot product. */
- // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
- permuted_samples[0] = vqtbl1q_s8(clamped_samples, permute_tbl.val[0]);
- // { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 }
- permuted_samples[1] = vqtbl1q_s8(clamped_samples, permute_tbl.val[1]);
- // { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 }
- permuted_samples[2] = vqtbl1q_s8(clamped_samples, permute_tbl.val[2]);
-
- // Accumulate dot product into 'correction' to account for range clamp.
- // First 4 output values.
- sum[0] = vdotq_lane_s32(correction, permuted_samples[0], x_filter, 0);
- sum[0] = vdotq_lane_s32(sum[0], permuted_samples[1], x_filter, 1);
- // Second 4 output values.
- sum[1] = vdotq_lane_s32(correction, permuted_samples[1], x_filter, 0);
- sum[1] = vdotq_lane_s32(sum[1], permuted_samples[2], x_filter, 1);
-
- // Narrow and re-pack.
- // We halved the convolution filter values so -1 from the right shift.
- return vcombine_s16(vshrn_n_s32(sum[0], ROUND0_BITS - 1),
- vshrn_n_s32(sum[1], ROUND0_BITS - 1));
-}
-
-static INLINE void dist_wtd_convolve_2d_horiz_8tap_neon(
- const uint8_t *src, int src_stride, int16_t *im_block, const int im_stride,
- const int16x8_t x_filter_s16, const int im_h, int w) {
- const int bd = 8;
- const int32_t horiz_const = (1 << (bd + FILTER_BITS - 2));
- // Dot product constants and other shims.
- const int32_t correction_s32 = vaddlvq_s16(vshlq_n_s16(x_filter_s16, 7));
- // Fold horiz_const into the dot-product filter correction constant. The
- // additional shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-
- // rounding shifts - which are generally faster than rounding shifts on
- // modern CPUs. (The extra -1 is needed because we halved the filter values.)
- const int32x4_t correction = vdupq_n_s32(correction_s32 + horiz_const +
- (1 << ((ROUND0_BITS - 1) - 1)));
- const uint8x16_t range_limit = vdupq_n_u8(128);
- // Horizontal filter.
- const int8x8_t x_filter = vmovn_s16(x_filter_s16);
-
- const uint8_t *src_ptr = src;
- int16_t *dst_ptr = im_block;
- int dst_stride = im_stride;
- int height = im_h;
-
- if (w == 4) {
- const uint8x16x2_t permute_tbl = vld1q_u8_x2(dot_prod_permute_tbl);
- uint8x16_t s0, s1, s2, s3;
- int16x4_t d0, d1, d2, d3;
-
- do {
- load_u8_16x4(src_ptr, src_stride, &s0, &s1, &s2, &s3);
-
- d0 = convolve8_4_2d_h(s0, x_filter, correction, range_limit, permute_tbl);
- d1 = convolve8_4_2d_h(s1, x_filter, correction, range_limit, permute_tbl);
- d2 = convolve8_4_2d_h(s2, x_filter, correction, range_limit, permute_tbl);
- d3 = convolve8_4_2d_h(s3, x_filter, correction, range_limit, permute_tbl);
-
- store_s16_4x4(dst_ptr, dst_stride, d0, d1, d2, d3);
-
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- height -= 4;
- } while (height > 0);
- } else {
- const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
- uint8x16_t s0, s1, s2, s3;
- int16x8_t d0, d1, d2, d3;
-
- do {
- const uint8_t *s = src_ptr;
- int16_t *d = dst_ptr;
- int width = w;
-
- do {
- load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3);
-
- d0 = convolve8_8_2d_h(s0, x_filter, correction, range_limit,
- permute_tbl);
- d1 = convolve8_8_2d_h(s1, x_filter, correction, range_limit,
- permute_tbl);
- d2 = convolve8_8_2d_h(s2, x_filter, correction, range_limit,
- permute_tbl);
- d3 = convolve8_8_2d_h(s3, x_filter, correction, range_limit,
- permute_tbl);
-
- store_s16_8x4(d, dst_stride, d0, d1, d2, d3);
-
- s += 8;
- d += 8;
- width -= 8;
- } while (width > 0);
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- height -= 4;
- } while (height > 0);
- }
-}
-
-#else // !(AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_DOTPROD))
-
-static INLINE int16x4_t convolve8_4_2d_h(const int16x4_t s0, const int16x4_t s1,
- const int16x4_t s2, const int16x4_t s3,
- const int16x4_t s4, const int16x4_t s5,
- const int16x4_t s6, const int16x4_t s7,
- const int16x8_t x_filter,
- const int16x4_t horiz_const) {
- const int16x4_t x_filter_0_3 = vget_low_s16(x_filter);
- const int16x4_t x_filter_4_7 = vget_high_s16(x_filter);
-
- int16x4_t sum = horiz_const;
- sum = vmla_lane_s16(sum, s0, x_filter_0_3, 0);
- sum = vmla_lane_s16(sum, s1, x_filter_0_3, 1);
- sum = vmla_lane_s16(sum, s2, x_filter_0_3, 2);
- sum = vmla_lane_s16(sum, s3, x_filter_0_3, 3);
- sum = vmla_lane_s16(sum, s4, x_filter_4_7, 0);
- sum = vmla_lane_s16(sum, s5, x_filter_4_7, 1);
- sum = vmla_lane_s16(sum, s6, x_filter_4_7, 2);
- sum = vmla_lane_s16(sum, s7, x_filter_4_7, 3);
-
- // We halved the convolution filter values so -1 from the right shift.
- return vshr_n_s16(sum, ROUND0_BITS - 1);
-}
-
-static INLINE int16x8_t convolve8_8_2d_h(const int16x8_t s0, const int16x8_t s1,
- const int16x8_t s2, const int16x8_t s3,
- const int16x8_t s4, const int16x8_t s5,
- const int16x8_t s6, const int16x8_t s7,
- const int16x8_t x_filter,
- const int16x8_t horiz_const) {
- const int16x4_t x_filter_0_3 = vget_low_s16(x_filter);
- const int16x4_t x_filter_4_7 = vget_high_s16(x_filter);
-
- int16x8_t sum = horiz_const;
- sum = vmlaq_lane_s16(sum, s0, x_filter_0_3, 0);
- sum = vmlaq_lane_s16(sum, s1, x_filter_0_3, 1);
- sum = vmlaq_lane_s16(sum, s2, x_filter_0_3, 2);
- sum = vmlaq_lane_s16(sum, s3, x_filter_0_3, 3);
- sum = vmlaq_lane_s16(sum, s4, x_filter_4_7, 0);
- sum = vmlaq_lane_s16(sum, s5, x_filter_4_7, 1);
- sum = vmlaq_lane_s16(sum, s6, x_filter_4_7, 2);
- sum = vmlaq_lane_s16(sum, s7, x_filter_4_7, 3);
-
- // We halved the convolution filter values so -1 from the right shift.
- return vshrq_n_s16(sum, ROUND0_BITS - 1);
-}
-
-static INLINE void dist_wtd_convolve_2d_horiz_8tap_neon(
- const uint8_t *src, int src_stride, int16_t *im_block, const int im_stride,
- const int16x8_t x_filter, const int im_h, int w) {
- const int bd = 8;
-
- const uint8_t *src_ptr = src;
- int16_t *dst_ptr = im_block;
- int dst_stride = im_stride;
- int height = im_h;
-
- if (w == 4) {
- int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, d0;
- uint8x8_t t0;
-#if AOM_ARCH_AARCH64
- int16x4_t s8, s9, s10, d1, d2, d3;
- uint8x8_t t1, t2, t3;
-#endif // AOM_ARCH_AARCH64
-
- // A shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-rounding
- // shifts - which are generally faster than rounding shifts on modern CPUs.
- // (The extra -1 is needed because we halved the filter values.)
- const int16x4_t horiz_const = vdup_n_s16((1 << (bd + FILTER_BITS - 2)) +
- (1 << ((ROUND0_BITS - 1) - 1)));
- do {
- __builtin_prefetch(src_ptr + 0 * src_stride);
-#if AOM_ARCH_AARCH64
- __builtin_prefetch(src_ptr + 1 * src_stride);
- __builtin_prefetch(src_ptr + 2 * src_stride);
- __builtin_prefetch(src_ptr + 3 * src_stride);
-
- load_u8_8x4(src_ptr, src_stride, &t0, &t1, &t2, &t3);
- transpose_u8_8x4(&t0, &t1, &t2, &t3);
-
- s0 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s1 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
- s2 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
- s3 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t3)));
- s4 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s5 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
- s6 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
-
- __builtin_prefetch(dst_ptr + 0 * dst_stride);
- __builtin_prefetch(dst_ptr + 1 * dst_stride);
- __builtin_prefetch(dst_ptr + 2 * dst_stride);
- __builtin_prefetch(dst_ptr + 3 * dst_stride);
-
- load_u8_8x4(src_ptr + 7, src_stride, &t0, &t1, &t2, &t3);
- transpose_u8_8x4(&t0, &t1, &t2, &t3);
-
- s7 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s8 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
- s9 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
- s10 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t3)));
-
- d0 = convolve8_4_2d_h(s0, s1, s2, s3, s4, s5, s6, s7, x_filter,
- horiz_const);
- d1 = convolve8_4_2d_h(s1, s2, s3, s4, s5, s6, s7, s8, x_filter,
- horiz_const);
- d2 = convolve8_4_2d_h(s2, s3, s4, s5, s6, s7, s8, s9, x_filter,
- horiz_const);
- d3 = convolve8_4_2d_h(s3, s4, s5, s6, s7, s8, s9, s10, x_filter,
- horiz_const);
-
- transpose_s16_4x4d(&d0, &d1, &d2, &d3);
- store_s16_4x4(dst_ptr, dst_stride, d0, d1, d2, d3);
-
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- height -= 4;
-#else // !AOM_ARCH_AARCH64
- t0 = vld1_u8(src_ptr); // a0 a1 a2 a3 a4 a5 a6 a7
- s0 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0))); // a0 a1 a2 a3
- s4 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t0))); // a4 a5 a6 a7
-
- __builtin_prefetch(dst_ptr);
-
- t0 = vld1_u8(src_ptr + 8); // a8 a9 a10 a11
- s7 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
-
- s1 = vext_s16(s0, s4, 1); // a1 a2 a3 a4
- s2 = vext_s16(s0, s4, 2); // a2 a3 a4 a5
- s3 = vext_s16(s0, s4, 3); // a3 a4 a5 a6
- s5 = vext_s16(s4, s7, 1); // a5 a6 a7 a8
- s6 = vext_s16(s4, s7, 2); // a6 a7 a8 a9
- s7 = vext_s16(s4, s7, 3); // a7 a8 a9 a10
-
- d0 = convolve8_4_2d_h(s0, s1, s2, s3, s4, s5, s6, s7, x_filter,
- horiz_const);
- vst1_s16(dst_ptr, d0);
-
- src_ptr += src_stride;
- dst_ptr += dst_stride;
- height--;
-#endif // AOM_ARCH_AARCH64
- } while (height > 0);
- } else {
- int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, d0;
- uint8x8_t t0;
-#if AOM_ARCH_AARCH64
- int16x8_t s9, s10, s11, s12, s13, s14;
- int16x8_t d1, d2, d3, d4, d5, d6, d7;
- uint8x8_t t1, t2, t3, t4, t5, t6, t7;
-#endif // AOM_ARCH_AARCH64
-
- // A shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-rounding
- // shifts - which are generally faster than rounding shifts on modern CPUs.
- // (The extra -1 is needed because we halved the filter values.)
- const int16x8_t horiz_const = vdupq_n_s16((1 << (bd + FILTER_BITS - 2)) +
- (1 << ((ROUND0_BITS - 1) - 1)));
- do {
- const uint8_t *s;
- int16_t *d = dst_ptr;
- int width = w;
-
-#if AOM_ARCH_AARCH64
- __builtin_prefetch(src_ptr + 0 * src_stride);
- __builtin_prefetch(src_ptr + 1 * src_stride);
- __builtin_prefetch(src_ptr + 2 * src_stride);
- __builtin_prefetch(src_ptr + 3 * src_stride);
- __builtin_prefetch(src_ptr + 4 * src_stride);
- __builtin_prefetch(src_ptr + 5 * src_stride);
- __builtin_prefetch(src_ptr + 6 * src_stride);
- __builtin_prefetch(src_ptr + 7 * src_stride);
-
- load_u8_8x8(src_ptr, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
- transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
-
- s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
- s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
- s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
- s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
- s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
- s5 = vreinterpretq_s16_u16(vmovl_u8(t5));
- s6 = vreinterpretq_s16_u16(vmovl_u8(t6));
-
- s = src_ptr + 7;
-
- __builtin_prefetch(dst_ptr + 0 * dst_stride);
- __builtin_prefetch(dst_ptr + 1 * dst_stride);
- __builtin_prefetch(dst_ptr + 2 * dst_stride);
- __builtin_prefetch(dst_ptr + 3 * dst_stride);
- __builtin_prefetch(dst_ptr + 4 * dst_stride);
- __builtin_prefetch(dst_ptr + 5 * dst_stride);
- __builtin_prefetch(dst_ptr + 6 * dst_stride);
- __builtin_prefetch(dst_ptr + 7 * dst_stride);
-
- do {
- load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
- transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
-
- s7 = vreinterpretq_s16_u16(vmovl_u8(t0));
- s8 = vreinterpretq_s16_u16(vmovl_u8(t1));
- s9 = vreinterpretq_s16_u16(vmovl_u8(t2));
- s10 = vreinterpretq_s16_u16(vmovl_u8(t3));
- s11 = vreinterpretq_s16_u16(vmovl_u8(t4));
- s12 = vreinterpretq_s16_u16(vmovl_u8(t5));
- s13 = vreinterpretq_s16_u16(vmovl_u8(t6));
- s14 = vreinterpretq_s16_u16(vmovl_u8(t7));
-
- d0 = convolve8_8_2d_h(s0, s1, s2, s3, s4, s5, s6, s7, x_filter,
- horiz_const);
- d1 = convolve8_8_2d_h(s1, s2, s3, s4, s5, s6, s7, s8, x_filter,
- horiz_const);
- d2 = convolve8_8_2d_h(s2, s3, s4, s5, s6, s7, s8, s9, x_filter,
- horiz_const);
- d3 = convolve8_8_2d_h(s3, s4, s5, s6, s7, s8, s9, s10, x_filter,
- horiz_const);
- d4 = convolve8_8_2d_h(s4, s5, s6, s7, s8, s9, s10, s11, x_filter,
- horiz_const);
- d5 = convolve8_8_2d_h(s5, s6, s7, s8, s9, s10, s11, s12, x_filter,
- horiz_const);
- d6 = convolve8_8_2d_h(s6, s7, s8, s9, s10, s11, s12, s13, x_filter,
- horiz_const);
- d7 = convolve8_8_2d_h(s7, s8, s9, s10, s11, s12, s13, s14, x_filter,
- horiz_const);
-
- transpose_s16_8x8(&d0, &d1, &d2, &d3, &d4, &d5, &d6, &d7);
- store_s16_8x8(d, dst_stride, d0, d1, d2, d3, d4, d5, d6, d7);
-
- s0 = s8;
- s1 = s9;
- s2 = s10;
- s3 = s11;
- s4 = s12;
- s5 = s13;
- s6 = s14;
- s += 8;
- d += 8;
- width -= 8;
- } while (width > 0);
- src_ptr += 8 * src_stride;
- dst_ptr += 8 * dst_stride;
- height -= 8;
-#else // !AOM_ARCH_AARCH64
- t0 = vld1_u8(src_ptr);
- s0 = vreinterpretq_s16_u16(vmovl_u8(t0)); // a0 a1 a2 a3 a4 a5 a6 a7
-
- s = src_ptr + 8;
- __builtin_prefetch(dst_ptr);
-
- do {
- t0 = vld1_u8(s); // a8 a9 a10 a11 a12 a13 a14 a15
- s8 = vreinterpretq_s16_u16(vmovl_u8(t0));
-
- s1 = vextq_s16(s0, s8, 1); // a1 a2 a3 a4 a5 a6 a7 a8
- s2 = vextq_s16(s0, s8, 2); // a2 a3 a4 a5 a6 a7 a8 a9
- s3 = vextq_s16(s0, s8, 3); // a3 a4 a5 a6 a7 a8 a9 a10
- s4 = vextq_s16(s0, s8, 4); // a4 a5 a6 a7 a8 a9 a10 a11
- s5 = vextq_s16(s0, s8, 5); // a5 a6 a7 a8 a9 a10 a11 a12
- s6 = vextq_s16(s0, s8, 6); // a6 a7 a8 a9 a10 a11 a12 a13
- s7 = vextq_s16(s0, s8, 7); // a7 a8 a9 a10 a11 a12 a13 a14
-
- d0 = convolve8_8_2d_h(s0, s1, s2, s3, s4, s5, s6, s7, x_filter,
- horiz_const);
- vst1q_s16(d, d0);
-
- s0 = s8;
- s += 8;
- d += 8;
- width -= 8;
- } while (width > 0);
- src_ptr += src_stride;
- dst_ptr += dst_stride;
- height--;
-#endif // AOM_ARCH_AARCH64
- } while (height > 0);
- }
-}
-
-#endif // AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_DOTPROD)
-
-static INLINE uint16x4_t
-convolve6_4_2d_v(const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
- const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
- const int16x8_t y_filter, const int32x4_t offset_const) {
- const int16x4_t y_filter_0_3 = vget_low_s16(y_filter);
- const int16x4_t y_filter_4_7 = vget_high_s16(y_filter);
-
- int32x4_t sum = offset_const;
- // Filter values at indices 0 and 7 are 0.
- sum = vmlal_lane_s16(sum, s0, y_filter_0_3, 1);
- sum = vmlal_lane_s16(sum, s1, y_filter_0_3, 2);
- sum = vmlal_lane_s16(sum, s2, y_filter_0_3, 3);
- sum = vmlal_lane_s16(sum, s3, y_filter_4_7, 0);
- sum = vmlal_lane_s16(sum, s4, y_filter_4_7, 1);
- sum = vmlal_lane_s16(sum, s5, y_filter_4_7, 2);
-
- return vqrshrun_n_s32(sum, COMPOUND_ROUND1_BITS);
-}
-
-static INLINE uint16x8_t
-convolve6_8_2d_v(const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
- const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
- const int16x8_t y_filter, const int32x4_t offset_const) {
- const int16x4_t y_filter_0_3 = vget_low_s16(y_filter);
- const int16x4_t y_filter_4_7 = vget_high_s16(y_filter);
-
- int32x4_t sum0 = offset_const;
- // Filter values at indices 0 and 7 are 0.
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s0), y_filter_0_3, 1);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s1), y_filter_0_3, 2);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s2), y_filter_0_3, 3);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s3), y_filter_4_7, 0);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s4), y_filter_4_7, 1);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s5), y_filter_4_7, 2);
-
- int32x4_t sum1 = offset_const;
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s0), y_filter_0_3, 1);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s1), y_filter_0_3, 2);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s2), y_filter_0_3, 3);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s3), y_filter_4_7, 0);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s4), y_filter_4_7, 1);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s5), y_filter_4_7, 2);
-
- return vcombine_u16(vqrshrun_n_s32(sum0, COMPOUND_ROUND1_BITS),
- vqrshrun_n_s32(sum1, COMPOUND_ROUND1_BITS));
-}
-
-static INLINE void dist_wtd_convolve_2d_vert_6tap_dist_wtd_avg_neon(
- int16_t *src_ptr, const int src_stride, uint8_t *dst8_ptr, int dst8_stride,
- ConvolveParams *conv_params, const int16x8_t y_filter, int h, int w) {
- const int bd = 8;
- const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
- const int32x4_t offset_const = vdupq_n_s32(1 << offset_bits);
- const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
- (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
- const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
-
- const uint16_t fwd_offset = conv_params->fwd_offset;
- const uint16_t bck_offset = conv_params->bck_offset;
-
- CONV_BUF_TYPE *dst_ptr = conv_params->dst;
- const int dst_stride = conv_params->dst_stride;
-
- if (w == 4) {
- int16x4_t s0, s1, s2, s3, s4, s5;
- uint16x4_t dd0, d0;
- uint8x8_t d01_u8;
-#if AOM_ARCH_AARCH64
- int16x4_t s6, s7, s8;
- uint16x4_t dd1, dd2, dd3, d1, d2, d3;
- uint8x8_t d23_u8;
-#endif // AOM_ARCH_AARCH64
-
- load_s16_4x5(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4);
- src_ptr += 5 * src_stride;
-
- do {
-#if AOM_ARCH_AARCH64
- load_s16_4x4(src_ptr, src_stride, &s5, &s6, &s7, &s8);
-
- d0 = convolve6_4_2d_v(s0, s1, s2, s3, s4, s5, y_filter, offset_const);
- d1 = convolve6_4_2d_v(s1, s2, s3, s4, s5, s6, y_filter, offset_const);
- d2 = convolve6_4_2d_v(s2, s3, s4, s5, s6, s7, y_filter, offset_const);
- d3 = convolve6_4_2d_v(s3, s4, s5, s6, s7, s8, y_filter, offset_const);
-
- load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3);
-
- compute_dist_wtd_avg_4x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset,
- bck_offset, round_offset_vec, &d01_u8, &d23_u8);
-
- store_u8_4x1(dst8_ptr + 0 * dst8_stride, d01_u8, 0);
- store_u8_4x1(dst8_ptr + 1 * dst8_stride, d01_u8, 1);
- store_u8_4x1(dst8_ptr + 2 * dst8_stride, d23_u8, 0);
- store_u8_4x1(dst8_ptr + 3 * dst8_stride, d23_u8, 1);
- dst8_ptr += 4 * dst8_stride;
-
- s0 = s4;
- s1 = s5;
- s2 = s6;
- s3 = s7;
- s4 = s8;
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- h -= 4;
-#else // !AOM_ARCH_AARCH64
- s5 = vld1_s16(src_ptr);
-
- d0 = convolve6_4_2d_v(s0, s1, s2, s3, s4, s5, y_filter, offset_const);
-
- dd0 = vld1_u16(dst_ptr);
-
- compute_dist_wtd_avg_4x1(dd0, d0, fwd_offset, bck_offset,
- vget_low_s16(round_offset_vec), &d01_u8);
-
- store_u8_4x1(dst8_ptr, d01_u8, 0);
- dst8_ptr += dst8_stride;
-
- s0 = s1;
- s1 = s2;
- s2 = s3;
- s3 = s4;
- s4 = s5;
- src_ptr += src_stride;
- dst_ptr += dst_stride;
- h--;
-#endif // AOM_ARCH_AARCH64
- } while (h != 0);
- } else {
- int16x8_t s0, s1, s2, s3, s4, s5;
- uint16x8_t dd0, d0;
- uint8x8_t d0_u8;
-#if AOM_ARCH_AARCH64
- int16x8_t s6, s7, s8;
- uint16x8_t dd1, dd2, dd3, d1, d2, d3;
- uint8x8_t d1_u8, d2_u8, d3_u8;
-#endif // AOM_ARCH_AARCH64
-
- do {
- int16_t *s = src_ptr;
- CONV_BUF_TYPE *d = dst_ptr;
- uint8_t *d_u8 = dst8_ptr;
- int height = h;
-
- load_s16_8x5(s, src_stride, &s0, &s1, &s2, &s3, &s4);
- s += 5 * src_stride;
-
- do {
-#if AOM_ARCH_AARCH64
- load_s16_8x4(s, src_stride, &s5, &s6, &s7, &s8);
-
- d0 = convolve6_8_2d_v(s0, s1, s2, s3, s4, s5, y_filter, offset_const);
- d1 = convolve6_8_2d_v(s1, s2, s3, s4, s5, s6, y_filter, offset_const);
- d2 = convolve6_8_2d_v(s2, s3, s4, s5, s6, s7, y_filter, offset_const);
- d3 = convolve6_8_2d_v(s3, s4, s5, s6, s7, s8, y_filter, offset_const);
-
- load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
-
- compute_dist_wtd_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset,
- bck_offset, round_offset_vec, &d0_u8, &d1_u8,
- &d2_u8, &d3_u8);
-
- store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8);
- d_u8 += 4 * dst8_stride;
-
- s0 = s4;
- s1 = s5;
- s2 = s6;
- s3 = s7;
- s4 = s8;
- s += 4 * src_stride;
- d += 4 * dst_stride;
- height -= 4;
-#else // !AOM_ARCH_AARCH64
- s5 = vld1q_s16(s);
-
- d0 = convolve6_8_2d_v(s0, s1, s2, s3, s4, s5, y_filter, offset_const);
-
- dd0 = vld1q_u16(d);
-
- compute_dist_wtd_avg_8x1(dd0, d0, fwd_offset, bck_offset,
- round_offset_vec, &d0_u8);
-
- vst1_u8(d_u8, d0_u8);
- d_u8 += dst8_stride;
-
- s0 = s1;
- s1 = s2;
- s2 = s3;
- s3 = s4;
- s4 = s5;
- s += src_stride;
- d += dst_stride;
- height--;
-#endif // AOM_ARCH_AARCH64
- } while (height != 0);
- src_ptr += 8;
- dst_ptr += 8;
- dst8_ptr += 8;
- w -= 8;
- } while (w != 0);
- }
-}
-
-static INLINE void dist_wtd_convolve_2d_vert_6tap_avg_neon(
- int16_t *src_ptr, const int src_stride, uint8_t *dst8_ptr, int dst8_stride,
- ConvolveParams *conv_params, const int16x8_t y_filter, int h, int w) {
- const int bd = 8;
- const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
- const int32x4_t offset_const = vdupq_n_s32(1 << offset_bits);
- const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
- (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
- const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
-
- CONV_BUF_TYPE *dst_ptr = conv_params->dst;
- const int dst_stride = conv_params->dst_stride;
-
- if (w == 4) {
- int16x4_t s0, s1, s2, s3, s4, s5;
- uint16x4_t dd0, d0;
- uint8x8_t d01_u8;
-#if AOM_ARCH_AARCH64
- int16x4_t s6, s7, s8;
- uint16x4_t dd1, dd2, dd3, d1, d2, d3;
- uint8x8_t d23_u8;
-#endif // AOM_ARCH_AARCH64
-
- load_s16_4x5(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4);
- src_ptr += 5 * src_stride;
-
- do {
-#if AOM_ARCH_AARCH64
- load_s16_4x4(src_ptr, src_stride, &s5, &s6, &s7, &s8);
-
- d0 = convolve6_4_2d_v(s0, s1, s2, s3, s4, s5, y_filter, offset_const);
- d1 = convolve6_4_2d_v(s1, s2, s3, s4, s5, s6, y_filter, offset_const);
- d2 = convolve6_4_2d_v(s2, s3, s4, s5, s6, s7, y_filter, offset_const);
- d3 = convolve6_4_2d_v(s3, s4, s5, s6, s7, s8, y_filter, offset_const);
-
- load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3);
-
- compute_basic_avg_4x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3,
- round_offset_vec, &d01_u8, &d23_u8);
-
- store_u8_4x1(dst8_ptr + 0 * dst8_stride, d01_u8, 0);
- store_u8_4x1(dst8_ptr + 1 * dst8_stride, d01_u8, 1);
- store_u8_4x1(dst8_ptr + 2 * dst8_stride, d23_u8, 0);
- store_u8_4x1(dst8_ptr + 3 * dst8_stride, d23_u8, 1);
- dst8_ptr += 4 * dst8_stride;
-
- s0 = s4;
- s1 = s5;
- s2 = s6;
- s3 = s7;
- s4 = s8;
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- h -= 4;
-#else // !AOM_ARCH_AARCH64
- s5 = vld1_s16(src_ptr);
-
- d0 = convolve6_4_2d_v(s0, s1, s2, s3, s4, s5, y_filter, offset_const);
-
- dd0 = vld1_u16(dst_ptr);
-
- compute_basic_avg_4x1(dd0, d0, vget_low_s16(round_offset_vec), &d01_u8);
-
- store_u8_4x1(dst8_ptr, d01_u8, 0);
- dst8_ptr += dst8_stride;
-
- s0 = s1;
- s1 = s2;
- s2 = s3;
- s3 = s4;
- s4 = s5;
- src_ptr += src_stride;
- dst_ptr += dst_stride;
- h--;
-#endif // AOM_ARCH_AARCH64
- } while (h != 0);
- } else {
- int16x8_t s0, s1, s2, s3, s4, s5;
- uint16x8_t dd0, d0;
- uint8x8_t d0_u8;
-#if AOM_ARCH_AARCH64
- int16x8_t s6, s7, s8;
- uint16x8_t dd1, dd2, dd3, d1, d2, d3;
- uint8x8_t d1_u8, d2_u8, d3_u8;
-#endif // AOM_ARCH_AARCH64
-
- do {
- int16_t *s = src_ptr;
- CONV_BUF_TYPE *d = dst_ptr;
- uint8_t *d_u8 = dst8_ptr;
- int height = h;
-
- load_s16_8x5(s, src_stride, &s0, &s1, &s2, &s3, &s4);
- s += 5 * src_stride;
-
- do {
-#if AOM_ARCH_AARCH64
- load_s16_8x4(s, src_stride, &s5, &s6, &s7, &s8);
-
- d0 = convolve6_8_2d_v(s0, s1, s2, s3, s4, s5, y_filter, offset_const);
- d1 = convolve6_8_2d_v(s1, s2, s3, s4, s5, s6, y_filter, offset_const);
- d2 = convolve6_8_2d_v(s2, s3, s4, s5, s6, s7, y_filter, offset_const);
- d3 = convolve6_8_2d_v(s3, s4, s5, s6, s7, s8, y_filter, offset_const);
-
- load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
-
- compute_basic_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3,
- round_offset_vec, &d0_u8, &d1_u8, &d2_u8, &d3_u8);
-
- store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8);
- d_u8 += 4 * dst8_stride;
-
- s0 = s4;
- s1 = s5;
- s2 = s6;
- s3 = s7;
- s4 = s8;
- s += 4 * src_stride;
- d += 4 * dst_stride;
- height -= 4;
-#else // !AOM_ARCH_AARCH64
- s5 = vld1q_s16(s);
-
- d0 = convolve6_8_2d_v(s0, s1, s2, s3, s4, s5, y_filter, offset_const);
-
- dd0 = vld1q_u16(d);
-
- compute_basic_avg_8x1(dd0, d0, round_offset_vec, &d0_u8);
-
- vst1_u8(d_u8, d0_u8);
- d_u8 += dst8_stride;
-
- s0 = s1;
- s1 = s2;
- s2 = s3;
- s3 = s4;
- s4 = s5;
- s += src_stride;
- d += dst_stride;
- height--;
-#endif // AOM_ARCH_AARCH64
- } while (height != 0);
- src_ptr += 8;
- dst_ptr += 8;
- dst8_ptr += 8;
- w -= 8;
- } while (w != 0);
- }
-}
-
-static INLINE void dist_wtd_convolve_2d_vert_6tap_neon(
- int16_t *src_ptr, const int src_stride, ConvolveParams *conv_params,
- const int16x8_t y_filter, int h, int w) {
- const int bd = 8;
- const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
- const int32x4_t offset_const = vdupq_n_s32(1 << offset_bits);
-
- CONV_BUF_TYPE *dst_ptr = conv_params->dst;
- const int dst_stride = conv_params->dst_stride;
-
- if (w == 4) {
- int16x4_t s0, s1, s2, s3, s4, s5;
- uint16x4_t d0;
-#if AOM_ARCH_AARCH64
- int16x4_t s6, s7, s8;
- uint16x4_t d1, d2, d3;
-#endif // AOM_ARCH_AARCH64
-
- load_s16_4x5(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4);
- src_ptr += 5 * src_stride;
-
- do {
-#if AOM_ARCH_AARCH64
- load_s16_4x4(src_ptr, src_stride, &s5, &s6, &s7, &s8);
-
- d0 = convolve6_4_2d_v(s0, s1, s2, s3, s4, s5, y_filter, offset_const);
- d1 = convolve6_4_2d_v(s1, s2, s3, s4, s5, s6, y_filter, offset_const);
- d2 = convolve6_4_2d_v(s2, s3, s4, s5, s6, s7, y_filter, offset_const);
- d3 = convolve6_4_2d_v(s3, s4, s5, s6, s7, s8, y_filter, offset_const);
-
- store_u16_4x4(dst_ptr, dst_stride, d0, d1, d2, d3);
-
- s0 = s4;
- s1 = s5;
- s2 = s6;
- s3 = s7;
- s4 = s8;
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- h -= 4;
-#else // !AOM_ARCH_AARCH64
- s5 = vld1_s16(src_ptr);
-
- d0 = convolve6_4_2d_v(s0, s1, s2, s3, s4, s5, y_filter, offset_const);
-
- vst1_u16(dst_ptr, d0);
-
- s0 = s1;
- s1 = s2;
- s2 = s3;
- s3 = s4;
- s4 = s5;
- src_ptr += src_stride;
- dst_ptr += dst_stride;
- h--;
-#endif // AOM_ARCH_AARCH64
- } while (h != 0);
- } else {
- int16x8_t s0, s1, s2, s3, s4, s5;
- uint16x8_t d0;
-#if AOM_ARCH_AARCH64
- int16x8_t s6, s7, s8;
- uint16x8_t d1, d2, d3;
-#endif // AOM_ARCH_AARCH64
-
- do {
- int16_t *s = src_ptr;
- CONV_BUF_TYPE *d = dst_ptr;
- int height = h;
-
- load_s16_8x5(s, src_stride, &s0, &s1, &s2, &s3, &s4);
- s += 5 * src_stride;
-
- do {
-#if AOM_ARCH_AARCH64
- load_s16_8x4(s, src_stride, &s5, &s6, &s7, &s8);
-
- d0 = convolve6_8_2d_v(s0, s1, s2, s3, s4, s5, y_filter, offset_const);
- d1 = convolve6_8_2d_v(s1, s2, s3, s4, s5, s6, y_filter, offset_const);
- d2 = convolve6_8_2d_v(s2, s3, s4, s5, s6, s7, y_filter, offset_const);
- d3 = convolve6_8_2d_v(s3, s4, s5, s6, s7, s8, y_filter, offset_const);
-
- store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
-
- s0 = s4;
- s1 = s5;
- s2 = s6;
- s3 = s7;
- s4 = s8;
- s += 4 * src_stride;
- d += 4 * dst_stride;
- height -= 4;
-#else // !AOM_ARCH_AARCH64
- s5 = vld1q_s16(s);
-
- d0 = convolve6_8_2d_v(s0, s1, s2, s3, s4, s5, y_filter, offset_const);
-
- vst1q_u16(d, d0);
-
- s0 = s1;
- s1 = s2;
- s2 = s3;
- s3 = s4;
- s4 = s5;
- s += src_stride;
- d += dst_stride;
- height--;
-#endif // AOM_ARCH_AARCH64
- } while (height != 0);
- src_ptr += 8;
- dst_ptr += 8;
- w -= 8;
- } while (w != 0);
- }
-}
-
-static INLINE uint16x4_t
-convolve8_4_2d_v(const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
- const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
- const int16x4_t s6, const int16x4_t s7,
- const int16x8_t y_filter, const int32x4_t offset_const) {
- const int16x4_t y_filter_0_3 = vget_low_s16(y_filter);
- const int16x4_t y_filter_4_7 = vget_high_s16(y_filter);
-
- int32x4_t sum = offset_const;
- sum = vmlal_lane_s16(sum, s0, y_filter_0_3, 0);
- sum = vmlal_lane_s16(sum, s1, y_filter_0_3, 1);
- sum = vmlal_lane_s16(sum, s2, y_filter_0_3, 2);
- sum = vmlal_lane_s16(sum, s3, y_filter_0_3, 3);
- sum = vmlal_lane_s16(sum, s4, y_filter_4_7, 0);
- sum = vmlal_lane_s16(sum, s5, y_filter_4_7, 1);
- sum = vmlal_lane_s16(sum, s6, y_filter_4_7, 2);
- sum = vmlal_lane_s16(sum, s7, y_filter_4_7, 3);
-
- return vqrshrun_n_s32(sum, COMPOUND_ROUND1_BITS);
-}
-
-static INLINE uint16x8_t
-convolve8_8_2d_v(const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
- const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
- const int16x8_t s6, const int16x8_t s7,
- const int16x8_t y_filter, const int32x4_t offset_const) {
- const int16x4_t y_filter_0_3 = vget_low_s16(y_filter);
- const int16x4_t y_filter_4_7 = vget_high_s16(y_filter);
-
- int32x4_t sum0 = offset_const;
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s0), y_filter_0_3, 0);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s1), y_filter_0_3, 1);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s2), y_filter_0_3, 2);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s3), y_filter_0_3, 3);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s4), y_filter_4_7, 0);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s5), y_filter_4_7, 1);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s6), y_filter_4_7, 2);
- sum0 = vmlal_lane_s16(sum0, vget_low_s16(s7), y_filter_4_7, 3);
-
- int32x4_t sum1 = offset_const;
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s0), y_filter_0_3, 0);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s1), y_filter_0_3, 1);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s2), y_filter_0_3, 2);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s3), y_filter_0_3, 3);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s4), y_filter_4_7, 0);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s5), y_filter_4_7, 1);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s6), y_filter_4_7, 2);
- sum1 = vmlal_lane_s16(sum1, vget_high_s16(s7), y_filter_4_7, 3);
-
- return vcombine_u16(vqrshrun_n_s32(sum0, COMPOUND_ROUND1_BITS),
- vqrshrun_n_s32(sum1, COMPOUND_ROUND1_BITS));
-}
-
-static INLINE void dist_wtd_convolve_2d_vert_8tap_dist_wtd_avg_neon(
- int16_t *src_ptr, const int src_stride, uint8_t *dst8_ptr, int dst8_stride,
- ConvolveParams *conv_params, const int16x8_t y_filter, int h, int w) {
- const int bd = 8;
- const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
- const int32x4_t offset_const = vdupq_n_s32(1 << offset_bits);
- const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
- (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
- const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
-
- const uint16_t fwd_offset = conv_params->fwd_offset;
- const uint16_t bck_offset = conv_params->bck_offset;
-
- CONV_BUF_TYPE *dst_ptr = conv_params->dst;
- const int dst_stride = conv_params->dst_stride;
-
- if (w == 4) {
- int16x4_t s0, s1, s2, s3, s4, s5, s6, s7;
- uint16x4_t dd0, d0;
- uint8x8_t d01_u8;
-#if AOM_ARCH_AARCH64
- int16x4_t s8, s9, s10;
- uint16x4_t dd1, dd2, dd3, d1, d2, d3;
- uint8x8_t d23_u8;
-#endif // AOM_ARCH_AARCH64
-
- load_s16_4x7(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
- src_ptr += 7 * src_stride;
-
- do {
-#if AOM_ARCH_AARCH64
- load_s16_4x4(src_ptr, src_stride, &s7, &s8, &s9, &s10);
-
- d0 = convolve8_4_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
- offset_const);
- d1 = convolve8_4_2d_v(s1, s2, s3, s4, s5, s6, s7, s8, y_filter,
- offset_const);
- d2 = convolve8_4_2d_v(s2, s3, s4, s5, s6, s7, s8, s9, y_filter,
- offset_const);
- d3 = convolve8_4_2d_v(s3, s4, s5, s6, s7, s8, s9, s10, y_filter,
- offset_const);
-
- load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3);
-
- compute_dist_wtd_avg_4x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset,
- bck_offset, round_offset_vec, &d01_u8, &d23_u8);
-
- store_u8_4x1(dst8_ptr + 0 * dst8_stride, d01_u8, 0);
- store_u8_4x1(dst8_ptr + 1 * dst8_stride, d01_u8, 1);
- store_u8_4x1(dst8_ptr + 2 * dst8_stride, d23_u8, 0);
- store_u8_4x1(dst8_ptr + 3 * dst8_stride, d23_u8, 1);
- dst8_ptr += 4 * dst8_stride;
-
- s0 = s4;
- s1 = s5;
- s2 = s6;
- s3 = s7;
- s4 = s8;
- s5 = s9;
- s6 = s10;
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- h -= 4;
-#else // !AOM_ARCH_AARCH64
- s7 = vld1_s16(src_ptr);
-
- d0 = convolve8_4_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
- offset_const);
-
- dd0 = vld1_u16(dst_ptr);
-
- compute_dist_wtd_avg_4x1(dd0, d0, fwd_offset, bck_offset,
- vget_low_s16(round_offset_vec), &d01_u8);
-
- store_u8_4x1(dst8_ptr, d01_u8, 0);
- dst8_ptr += dst8_stride;
-
- s0 = s1;
- s1 = s2;
- s2 = s3;
- s3 = s4;
- s4 = s5;
- s5 = s6;
- s6 = s7;
- src_ptr += src_stride;
- dst_ptr += dst_stride;
- h--;
-#endif // AOM_ARCH_AARCH64
- } while (h != 0);
- } else {
- int16x8_t s0, s1, s2, s3, s4, s5, s6, s7;
- uint16x8_t dd0, d0;
- uint8x8_t d0_u8;
-#if AOM_ARCH_AARCH64
- int16x8_t s8, s9, s10;
- uint16x8_t dd1, dd2, dd3, d1, d2, d3;
- uint8x8_t d1_u8, d2_u8, d3_u8;
-#endif // AOM_ARCH_AARCH64
-
- do {
- int16_t *s = src_ptr;
- CONV_BUF_TYPE *d = dst_ptr;
- uint8_t *d_u8 = dst8_ptr;
- int height = h;
-
- load_s16_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
- s += 7 * src_stride;
-
- do {
-#if AOM_ARCH_AARCH64
- load_s16_8x4(s, src_stride, &s7, &s8, &s9, &s10);
-
- d0 = convolve8_8_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
- offset_const);
- d1 = convolve8_8_2d_v(s1, s2, s3, s4, s5, s6, s7, s8, y_filter,
- offset_const);
- d2 = convolve8_8_2d_v(s2, s3, s4, s5, s6, s7, s8, s9, y_filter,
- offset_const);
- d3 = convolve8_8_2d_v(s3, s4, s5, s6, s7, s8, s9, s10, y_filter,
- offset_const);
-
- load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
-
- compute_dist_wtd_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset,
- bck_offset, round_offset_vec, &d0_u8, &d1_u8,
- &d2_u8, &d3_u8);
-
- store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8);
- d_u8 += 4 * dst8_stride;
-
- s0 = s4;
- s1 = s5;
- s2 = s6;
- s3 = s7;
- s4 = s8;
- s5 = s9;
- s6 = s10;
- s += 4 * src_stride;
- d += 4 * dst_stride;
- height -= 4;
-#else // !AOM_ARCH_AARCH64
- s7 = vld1q_s16(s);
-
- d0 = convolve8_8_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
- offset_const);
-
- dd0 = vld1q_u16(d);
-
- compute_dist_wtd_avg_8x1(dd0, d0, fwd_offset, bck_offset,
- round_offset_vec, &d0_u8);
-
- vst1_u8(d_u8, d0_u8);
- d_u8 += dst8_stride;
-
- s0 = s1;
- s1 = s2;
- s2 = s3;
- s3 = s4;
- s4 = s5;
- s5 = s6;
- s6 = s7;
- s += src_stride;
- d += dst_stride;
- height--;
-#endif // AOM_ARCH_AARCH64
- } while (height != 0);
- src_ptr += 8;
- dst_ptr += 8;
- dst8_ptr += 8;
- w -= 8;
- } while (w != 0);
- }
-}
-
-static INLINE void dist_wtd_convolve_2d_vert_8tap_avg_neon(
- int16_t *src_ptr, const int src_stride, uint8_t *dst8_ptr, int dst8_stride,
- ConvolveParams *conv_params, const int16x8_t y_filter, int h, int w) {
- const int bd = 8;
- const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
- const int32x4_t offset_const = vdupq_n_s32(1 << offset_bits);
- const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
- (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
- const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
-
- CONV_BUF_TYPE *dst_ptr = conv_params->dst;
- const int dst_stride = conv_params->dst_stride;
-
- if (w == 4) {
- int16x4_t s0, s1, s2, s3, s4, s5, s6, s7;
- uint16x4_t dd0, d0;
- uint8x8_t d01_u8;
-#if AOM_ARCH_AARCH64
- int16x4_t s8, s9, s10;
- uint16x4_t dd1, dd2, dd3, d1, d2, d3;
- uint8x8_t d23_u8;
-#endif // AOM_ARCH_AARCH64
-
- load_s16_4x7(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
- src_ptr += 7 * src_stride;
-
- do {
-#if AOM_ARCH_AARCH64
- load_s16_4x4(src_ptr, src_stride, &s7, &s8, &s9, &s10);
-
- d0 = convolve8_4_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
- offset_const);
- d1 = convolve8_4_2d_v(s1, s2, s3, s4, s5, s6, s7, s8, y_filter,
- offset_const);
- d2 = convolve8_4_2d_v(s2, s3, s4, s5, s6, s7, s8, s9, y_filter,
- offset_const);
- d3 = convolve8_4_2d_v(s3, s4, s5, s6, s7, s8, s9, s10, y_filter,
- offset_const);
-
- load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3);
-
- compute_basic_avg_4x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3,
- round_offset_vec, &d01_u8, &d23_u8);
-
- store_u8_4x1(dst8_ptr + 0 * dst8_stride, d01_u8, 0);
- store_u8_4x1(dst8_ptr + 1 * dst8_stride, d01_u8, 1);
- store_u8_4x1(dst8_ptr + 2 * dst8_stride, d23_u8, 0);
- store_u8_4x1(dst8_ptr + 3 * dst8_stride, d23_u8, 1);
- dst8_ptr += 4 * dst8_stride;
-
- s0 = s4;
- s1 = s5;
- s2 = s6;
- s3 = s7;
- s4 = s8;
- s5 = s9;
- s6 = s10;
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- h -= 4;
-#else // !AOM_ARCH_AARCH64
- s7 = vld1_s16(src_ptr);
-
- d0 = convolve8_4_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
- offset_const);
-
- dd0 = vld1_u16(dst_ptr);
-
- compute_basic_avg_4x1(dd0, d0, vget_low_s16(round_offset_vec), &d01_u8);
-
- store_u8_4x1(dst8_ptr, d01_u8, 0);
- dst8_ptr += dst8_stride;
-
- s0 = s1;
- s1 = s2;
- s2 = s3;
- s3 = s4;
- s4 = s5;
- s5 = s6;
- s6 = s7;
- src_ptr += src_stride;
- dst_ptr += dst_stride;
- h--;
-#endif // AOM_ARCH_AARCH64
- } while (h != 0);
- } else {
- int16x8_t s0, s1, s2, s3, s4, s5, s6, s7;
- uint16x8_t dd0, d0;
- uint8x8_t d0_u8;
-#if AOM_ARCH_AARCH64
- int16x8_t s8, s9, s10;
- uint16x8_t dd1, dd2, dd3, d1, d2, d3;
- uint8x8_t d1_u8, d2_u8, d3_u8;
-#endif // AOM_ARCH_AARCH64
-
- do {
- int16_t *s = src_ptr;
- CONV_BUF_TYPE *d = dst_ptr;
- uint8_t *d_u8 = dst8_ptr;
- int height = h;
-
- load_s16_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
- s += 7 * src_stride;
-
- do {
-#if AOM_ARCH_AARCH64
- load_s16_8x4(s, src_stride, &s7, &s8, &s9, &s10);
-
- d0 = convolve8_8_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
- offset_const);
- d1 = convolve8_8_2d_v(s1, s2, s3, s4, s5, s6, s7, s8, y_filter,
- offset_const);
- d2 = convolve8_8_2d_v(s2, s3, s4, s5, s6, s7, s8, s9, y_filter,
- offset_const);
- d3 = convolve8_8_2d_v(s3, s4, s5, s6, s7, s8, s9, s10, y_filter,
- offset_const);
-
- load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
-
- compute_basic_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3,
- round_offset_vec, &d0_u8, &d1_u8, &d2_u8, &d3_u8);
-
- store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8);
- d_u8 += 4 * dst8_stride;
-
- s0 = s4;
- s1 = s5;
- s2 = s6;
- s3 = s7;
- s4 = s8;
- s5 = s9;
- s6 = s10;
- s += 4 * src_stride;
- d += 4 * dst_stride;
- height -= 4;
-#else // !AOM_ARCH_AARCH64
- s7 = vld1q_s16(s);
-
- d0 = convolve8_8_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
- offset_const);
-
- dd0 = vld1q_u16(d);
-
- compute_basic_avg_8x1(dd0, d0, round_offset_vec, &d0_u8);
-
- vst1_u8(d_u8, d0_u8);
- d_u8 += dst8_stride;
-
- s0 = s1;
- s1 = s2;
- s2 = s3;
- s3 = s4;
- s4 = s5;
- s5 = s6;
- s6 = s7;
- s += src_stride;
- d += dst_stride;
- height--;
-#endif // AOM_ARCH_AARCH64
- } while (height != 0);
- src_ptr += 8;
- dst_ptr += 8;
- dst8_ptr += 8;
- w -= 8;
- } while (w != 0);
- }
-}
-
-static INLINE void dist_wtd_convolve_2d_vert_8tap_neon(
- int16_t *src_ptr, const int src_stride, ConvolveParams *conv_params,
- const int16x8_t y_filter, int h, int w) {
- const int bd = 8;
- const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
- const int32x4_t offset_const = vdupq_n_s32(1 << offset_bits);
-
- CONV_BUF_TYPE *dst_ptr = conv_params->dst;
- const int dst_stride = conv_params->dst_stride;
-
- if (w == 4) {
- int16x4_t s0, s1, s2, s3, s4, s5, s6, s7;
- uint16x4_t d0;
-#if AOM_ARCH_AARCH64
- int16x4_t s8, s9, s10;
- uint16x4_t d1, d2, d3;
-#endif // AOM_ARCH_AARCH64
-
- load_s16_4x7(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
- src_ptr += 7 * src_stride;
-
- do {
-#if AOM_ARCH_AARCH64
- load_s16_4x4(src_ptr, src_stride, &s7, &s8, &s9, &s10);
-
- d0 = convolve8_4_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
- offset_const);
- d1 = convolve8_4_2d_v(s1, s2, s3, s4, s5, s6, s7, s8, y_filter,
- offset_const);
- d2 = convolve8_4_2d_v(s2, s3, s4, s5, s6, s7, s8, s9, y_filter,
- offset_const);
- d3 = convolve8_4_2d_v(s3, s4, s5, s6, s7, s8, s9, s10, y_filter,
- offset_const);
-
- store_u16_4x4(dst_ptr, dst_stride, d0, d1, d2, d3);
-
- s0 = s4;
- s1 = s5;
- s2 = s6;
- s3 = s7;
- s4 = s8;
- s5 = s9;
- s6 = s10;
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- h -= 4;
-#else // !AOM_ARCH_AARCH64
- s7 = vld1_s16(src_ptr);
-
- d0 = convolve8_4_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
- offset_const);
-
- vst1_u16(dst_ptr, d0);
-
- s0 = s1;
- s1 = s2;
- s2 = s3;
- s3 = s4;
- s4 = s5;
- s5 = s6;
- s6 = s7;
- src_ptr += src_stride;
- dst_ptr += dst_stride;
- h--;
-#endif // AOM_ARCH_AARCH64
- } while (h != 0);
- } else {
- int16x8_t s0, s1, s2, s3, s4, s5, s6, s7;
- uint16x8_t d0;
-#if AOM_ARCH_AARCH64
- int16x8_t s8, s9, s10;
- uint16x8_t d1, d2, d3;
-#endif // AOM_ARCH_AARCH64
-
- do {
- int16_t *s = src_ptr;
- CONV_BUF_TYPE *d = dst_ptr;
- int height = h;
-
- load_s16_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
- s += 7 * src_stride;
-
- do {
-#if AOM_ARCH_AARCH64
- load_s16_8x4(s, src_stride, &s7, &s8, &s9, &s10);
-
- d0 = convolve8_8_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
- offset_const);
- d1 = convolve8_8_2d_v(s1, s2, s3, s4, s5, s6, s7, s8, y_filter,
- offset_const);
- d2 = convolve8_8_2d_v(s2, s3, s4, s5, s6, s7, s8, s9, y_filter,
- offset_const);
- d3 = convolve8_8_2d_v(s3, s4, s5, s6, s7, s8, s9, s10, y_filter,
- offset_const);
-
- store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
-
- s0 = s4;
- s1 = s5;
- s2 = s6;
- s3 = s7;
- s4 = s8;
- s5 = s9;
- s6 = s10;
- s += 4 * src_stride;
- d += 4 * dst_stride;
- height -= 4;
-#else // !AOM_ARCH_AARCH64
- s7 = vld1q_s16(s);
-
- d0 = convolve8_8_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
- offset_const);
-
- vst1q_u16(d, d0);
-
- s0 = s1;
- s1 = s2;
- s2 = s3;
- s3 = s4;
- s4 = s5;
- s5 = s6;
- s6 = s7;
- s += src_stride;
- d += dst_stride;
- height--;
-#endif // AOM_ARCH_AARCH64
- } while (height != 0);
- src_ptr += 8;
- dst_ptr += 8;
- w -= 8;
- } while (w != 0);
- }
-}
-
-void av1_dist_wtd_convolve_2d_neon(const uint8_t *src, int src_stride,
- uint8_t *dst8, int dst8_stride, int w, int h,
- const InterpFilterParams *filter_params_x,
- const InterpFilterParams *filter_params_y,
- const int subpel_x_qn, const int subpel_y_qn,
- ConvolveParams *conv_params) {
- assert(w % 4 == 0);
- assert(h % 4 == 0);
-
- DECLARE_ALIGNED(16, int16_t,
- im_block[(MAX_SB_SIZE + HORIZ_EXTRA_ROWS) * MAX_SB_SIZE]);
-
- const int y_filter_taps = get_filter_tap(filter_params_y, subpel_y_qn);
- const int clamped_y_taps = y_filter_taps < 6 ? 6 : y_filter_taps;
-
- const int im_h = h + filter_params_y->taps - 1;
- const int im_stride = MAX_SB_SIZE;
- const int vert_offset = filter_params_y->taps / 2 - 1;
- const int horiz_offset = filter_params_x->taps / 2 - 1;
- const uint8_t *src_ptr = src - vert_offset * src_stride - horiz_offset;
- const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
- filter_params_x, subpel_x_qn & SUBPEL_MASK);
- const int16_t *y_filter_ptr = av1_get_interp_filter_subpel_kernel(
- filter_params_y, subpel_y_qn & SUBPEL_MASK);
-
- // Filter values are even, so downshift by 1 to reduce intermediate precision
- // requirements.
- const int16x8_t x_filter = vshrq_n_s16(vld1q_s16(x_filter_ptr), 1);
- const int16x8_t y_filter = vld1q_s16(y_filter_ptr);
-
- dist_wtd_convolve_2d_horiz_8tap_neon(src_ptr, src_stride, im_block, im_stride,
- x_filter, im_h, w);
-
- if (clamped_y_taps == 6) {
- if (conv_params->do_average) {
- if (UNLIKELY(conv_params->use_dist_wtd_comp_avg)) {
- dist_wtd_convolve_2d_vert_6tap_dist_wtd_avg_neon(
- im_block + im_stride, im_stride, dst8, dst8_stride, conv_params,
- y_filter, h, w);
- } else {
- dist_wtd_convolve_2d_vert_6tap_avg_neon(im_block + im_stride, im_stride,
- dst8, dst8_stride, conv_params,
- y_filter, h, w);
- }
- } else {
- dist_wtd_convolve_2d_vert_6tap_neon(im_block + im_stride, im_stride,
- conv_params, y_filter, h, w);
- }
- } else {
- if (conv_params->do_average) {
- if (UNLIKELY(conv_params->use_dist_wtd_comp_avg)) {
- dist_wtd_convolve_2d_vert_8tap_dist_wtd_avg_neon(
- im_block, im_stride, dst8, dst8_stride, conv_params, y_filter, h,
- w);
- } else {
- dist_wtd_convolve_2d_vert_8tap_avg_neon(im_block, im_stride, dst8,
- dst8_stride, conv_params,
- y_filter, h, w);
- }
- } else {
- dist_wtd_convolve_2d_vert_8tap_neon(im_block, im_stride, conv_params,
- y_filter, h, w);
- }
- }
-}
-
-static INLINE void dist_wtd_convolve_2d_copy_dist_wtd_avg_neon(
- const uint8_t *src, int src_stride, uint8_t *dst8, int dst8_stride, int w,
- int h, ConvolveParams *conv_params) {
- assert(w % 4 == 0);
- assert(h % 4 == 0);
-
- const int bd = 8;
- const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
- const uint16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
- (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
- const uint16x8_t round_offset_vec = vdupq_n_u16(round_offset);
- const uint8x8_t shift_by_bits = vdup_n_u8(1 << (FILTER_BITS - ROUND0_BITS));
-
- const uint16_t fwd_offset = conv_params->fwd_offset;
- const uint16_t bck_offset = conv_params->bck_offset;
-
- CONV_BUF_TYPE *dst = conv_params->dst;
- const int dst_stride = conv_params->dst_stride;
- int height = h;
-
- if (w == 4) {
- uint8x8_t s0, s1, s2, s3, d01, d23;
- uint16x4_t d0, d1, d2, d3, dd0, dd1, dd2, dd3;
-
- do {
- load_u8_8x4(src, src_stride, &s0, &s1, &s2, &s3);
-
- d0 = vget_low_u16(vmlal_u8(round_offset_vec, s0, shift_by_bits));
- d1 = vget_low_u16(vmlal_u8(round_offset_vec, s1, shift_by_bits));
- d2 = vget_low_u16(vmlal_u8(round_offset_vec, s2, shift_by_bits));
- d3 = vget_low_u16(vmlal_u8(round_offset_vec, s3, shift_by_bits));
-
- load_u16_4x4(dst, dst_stride, &dd0, &dd1, &dd2, &dd3);
-
- compute_dist_wtd_avg_4x4(
- dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset, bck_offset,
- vreinterpretq_s16_u16(round_offset_vec), &d01, &d23);
-
- store_u8_4x1(dst8 + 0 * dst8_stride, d01, 0);
- store_u8_4x1(dst8 + 1 * dst8_stride, d01, 1);
- store_u8_4x1(dst8 + 2 * dst8_stride, d23, 0);
- store_u8_4x1(dst8 + 3 * dst8_stride, d23, 1);
-
- src += 4 * src_stride;
- dst += 4 * dst_stride;
- dst8 += 4 * dst8_stride;
- height -= 4;
- } while (height != 0);
- } else {
- uint8x8_t s0, s1, s2, s3, d0_u8, d1_u8, d2_u8, d3_u8;
- uint16x8_t d0, d1, d2, d3, dd0, dd1, dd2, dd3;
-
- do {
- const uint8_t *s = src;
- CONV_BUF_TYPE *d = dst;
- uint8_t *d_u8 = dst8;
- int width = w;
-
- do {
- load_u8_8x4(s, src_stride, &s0, &s1, &s2, &s3);
-
- d0 = vmlal_u8(round_offset_vec, s0, shift_by_bits);
- d1 = vmlal_u8(round_offset_vec, s1, shift_by_bits);
- d2 = vmlal_u8(round_offset_vec, s2, shift_by_bits);
- d3 = vmlal_u8(round_offset_vec, s3, shift_by_bits);
-
- load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
-
- compute_dist_wtd_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset,
- bck_offset,
- vreinterpretq_s16_u16(round_offset_vec),
- &d0_u8, &d1_u8, &d2_u8, &d3_u8);
-
- store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8);
-
- s += 8;
- d += 8;
- d_u8 += 8;
- width -= 8;
- } while (width != 0);
- src += 4 * src_stride;
- dst += 4 * dst_stride;
- dst8 += 4 * dst8_stride;
- height -= 4;
- } while (height != 0);
- }
-}
-
-static INLINE void dist_wtd_convolve_2d_copy_avg_neon(
- const uint8_t *src, int src_stride, uint8_t *dst8, int dst8_stride, int w,
- int h, ConvolveParams *conv_params) {
- assert(w % 4 == 0);
- assert(h % 4 == 0);
-
- const int bd = 8;
- const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
- const uint16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
- (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
- const uint16x8_t round_offset_vec = vdupq_n_u16(round_offset);
- const uint8x8_t shift_by_bits = vdup_n_u8(1 << (FILTER_BITS - ROUND0_BITS));
-
- CONV_BUF_TYPE *dst = conv_params->dst;
- const int dst_stride = conv_params->dst_stride;
- int height = h;
-
- if (w == 4) {
- uint8x8_t s0, s1, s2, s3, d01, d23;
- uint16x4_t d0, d1, d2, d3, dd0, dd1, dd2, dd3;
-
- do {
- load_u8_8x4(src, src_stride, &s0, &s1, &s2, &s3);
-
- d0 = vget_low_u16(vmlal_u8(round_offset_vec, s0, shift_by_bits));
- d1 = vget_low_u16(vmlal_u8(round_offset_vec, s1, shift_by_bits));
- d2 = vget_low_u16(vmlal_u8(round_offset_vec, s2, shift_by_bits));
- d3 = vget_low_u16(vmlal_u8(round_offset_vec, s3, shift_by_bits));
-
- load_u16_4x4(dst, dst_stride, &dd0, &dd1, &dd2, &dd3);
-
- compute_basic_avg_4x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3,
- vreinterpretq_s16_u16(round_offset_vec), &d01,
- &d23);
-
- store_u8_4x1(dst8 + 0 * dst8_stride, d01, 0);
- store_u8_4x1(dst8 + 1 * dst8_stride, d01, 1);
- store_u8_4x1(dst8 + 2 * dst8_stride, d23, 0);
- store_u8_4x1(dst8 + 3 * dst8_stride, d23, 1);
-
- src += 4 * src_stride;
- dst += 4 * dst_stride;
- dst8 += 4 * dst8_stride;
- height -= 4;
- } while (height != 0);
- } else {
- uint8x8_t s0, s1, s2, s3, d0_u8, d1_u8, d2_u8, d3_u8;
- uint16x8_t d0, d1, d2, d3, dd0, dd1, dd2, dd3;
-
- do {
- const uint8_t *s = src;
- CONV_BUF_TYPE *d = dst;
- uint8_t *d_u8 = dst8;
- int width = w;
-
- do {
- load_u8_8x4(s, src_stride, &s0, &s1, &s2, &s3);
-
- d0 = vmlal_u8(round_offset_vec, s0, shift_by_bits);
- d1 = vmlal_u8(round_offset_vec, s1, shift_by_bits);
- d2 = vmlal_u8(round_offset_vec, s2, shift_by_bits);
- d3 = vmlal_u8(round_offset_vec, s3, shift_by_bits);
-
- load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
-
- compute_basic_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3,
- vreinterpretq_s16_u16(round_offset_vec), &d0_u8,
- &d1_u8, &d2_u8, &d3_u8);
-
- store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8);
-
- s += 8;
- d += 8;
- d_u8 += 8;
- width -= 8;
- } while (width != 0);
- src += 4 * src_stride;
- dst += 4 * dst_stride;
- dst8 += 4 * dst8_stride;
- height -= 4;
- } while (height != 0);
- }
-}
-
-static INLINE void dist_wtd_convolve_2d_copy_neon(const uint8_t *src,
- int src_stride, int w, int h,
- ConvolveParams *conv_params) {
- assert(w % 4 == 0);
- assert(h % 4 == 0);
-
- const int bd = 8;
- const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
- const uint16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
- (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
- const uint16x8_t round_offset_vec = vdupq_n_u16(round_offset);
- const uint8x8_t shift_by_bits = vdup_n_u8(1 << (FILTER_BITS - ROUND0_BITS));
-
- CONV_BUF_TYPE *dst = conv_params->dst;
- const int dst_stride = conv_params->dst_stride;
- int height = h;
-
- if (w == 4) {
- uint8x8_t s0, s1, s2, s3;
- uint16x4_t d0, d1, d2, d3;
-
- do {
- load_u8_8x4(src, src_stride, &s0, &s1, &s2, &s3);
-
- d0 = vget_low_u16(vmlal_u8(round_offset_vec, s0, shift_by_bits));
- d1 = vget_low_u16(vmlal_u8(round_offset_vec, s1, shift_by_bits));
- d2 = vget_low_u16(vmlal_u8(round_offset_vec, s2, shift_by_bits));
- d3 = vget_low_u16(vmlal_u8(round_offset_vec, s3, shift_by_bits));
-
- store_u16_4x4(dst, dst_stride, d0, d1, d2, d3);
-
- src += 4 * src_stride;
- dst += 4 * dst_stride;
- height -= 4;
- } while (height != 0);
- } else {
- uint8x8_t s0, s1, s2, s3;
- uint16x8_t d0, d1, d2, d3;
-
- do {
- const uint8_t *s = src;
- CONV_BUF_TYPE *d = dst;
- int width = w;
-
- do {
- load_u8_8x4(s, src_stride, &s0, &s1, &s2, &s3);
-
- d0 = vmlal_u8(round_offset_vec, s0, shift_by_bits);
- d1 = vmlal_u8(round_offset_vec, s1, shift_by_bits);
- d2 = vmlal_u8(round_offset_vec, s2, shift_by_bits);
- d3 = vmlal_u8(round_offset_vec, s3, shift_by_bits);
-
- store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
-
- s += 8;
- d += 8;
- width -= 8;
- } while (width != 0);
- src += 4 * src_stride;
- dst += 4 * dst_stride;
- height -= 4;
- } while (height != 0);
- }
-}
-
-void av1_dist_wtd_convolve_2d_copy_neon(const uint8_t *src, int src_stride,
- uint8_t *dst8, int dst8_stride, int w,
- int h, ConvolveParams *conv_params) {
- if (conv_params->do_average) {
- if (UNLIKELY(conv_params->use_dist_wtd_comp_avg)) {
- dist_wtd_convolve_2d_copy_dist_wtd_avg_neon(
- src, src_stride, dst8, dst8_stride, w, h, conv_params);
- } else {
- dist_wtd_convolve_2d_copy_avg_neon(src, src_stride, dst8, dst8_stride, w,
- h, conv_params);
- }
- } else {
- dist_wtd_convolve_2d_copy_neon(src, src_stride, w, h, conv_params);
- }
-}
-
-#if AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_MATMUL_INT8)
-
-static INLINE uint16x4_t convolve8_4_x(uint8x16_t samples,
- const int8x8_t x_filter,
- const uint8x16x2_t permute_tbl,
- const int32x4_t round_offset) {
- uint8x16_t permuted_samples[2];
- int32x4_t sum;
-
- // Permute samples ready for dot product.
- // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
- permuted_samples[0] = vqtbl1q_u8(samples, permute_tbl.val[0]);
- // { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 }
- permuted_samples[1] = vqtbl1q_u8(samples, permute_tbl.val[1]);
-
- // First 4 output values.
- sum = vusdotq_lane_s32(round_offset, permuted_samples[0], x_filter, 0);
- sum = vusdotq_lane_s32(sum, permuted_samples[1], x_filter, 1);
-
- // We halved the convolution filter values so -1 from the right shift.
- return vreinterpret_u16_s16(vshrn_n_s32(sum, ROUND0_BITS - 1));
-}
-
-static INLINE uint16x8_t convolve8_8_x(uint8x16_t samples,
- const int8x8_t x_filter,
- const uint8x16x3_t permute_tbl,
- const int32x4_t round_offset) {
- uint8x16_t permuted_samples[3];
- int32x4_t sum[2];
-
- // Permute samples ready for dot product.
- // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
- permuted_samples[0] = vqtbl1q_u8(samples, permute_tbl.val[0]);
- // { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 }
- permuted_samples[1] = vqtbl1q_u8(samples, permute_tbl.val[1]);
- // { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 }
- permuted_samples[2] = vqtbl1q_u8(samples, permute_tbl.val[2]);
-
- // First 4 output values.
- sum[0] = vusdotq_lane_s32(round_offset, permuted_samples[0], x_filter, 0);
- sum[0] = vusdotq_lane_s32(sum[0], permuted_samples[1], x_filter, 1);
- // Second 4 output values.
- sum[1] = vusdotq_lane_s32(round_offset, permuted_samples[1], x_filter, 0);
- sum[1] = vusdotq_lane_s32(sum[1], permuted_samples[2], x_filter, 1);
-
- // Narrow and re-pack.
- // We halved the convolution filter values so -1 from the right shift.
- int16x8_t res = vcombine_s16(vshrn_n_s32(sum[0], ROUND0_BITS - 1),
- vshrn_n_s32(sum[1], ROUND0_BITS - 1));
- return vreinterpretq_u16_s16(res);
-}
-
-static INLINE void dist_wtd_convolve_x_dist_wtd_avg_neon(
- const uint8_t *src, int src_stride, uint8_t *dst8, int dst8_stride, int w,
- int h, const InterpFilterParams *filter_params_x, const int subpel_x_qn,
- ConvolveParams *conv_params) {
- assert(w % 4 == 0);
- assert(h % 4 == 0);
-
- const int bd = 8;
- const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
- const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
- (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
- const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
- // A shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-rounding
- // shifts - which are generally faster than rounding shifts on modern CPUs.
- // (The extra -1 is needed because we halved the filter values.)
- const int32x4_t round_offset_shim = vdupq_n_s32(
- (round_offset << (ROUND0_BITS - 1)) + (1 << ((ROUND0_BITS - 1) - 1)));
-
- const uint16_t fwd_offset = conv_params->fwd_offset;
- const uint16_t bck_offset = conv_params->bck_offset;
-
- // Horizontal filter.
- const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
- filter_params_x, subpel_x_qn & SUBPEL_MASK);
- // Filter values are even, so downshift by 1 to reduce intermediate precision
- // requirements.
- const int8x8_t x_filter = vshrn_n_s16(vld1q_s16(x_filter_ptr), 1);
-
- const int horiz_offset = filter_params_x->taps / 2 - 1;
- const uint8_t *src_ptr = src - horiz_offset;
- CONV_BUF_TYPE *dst_ptr = conv_params->dst;
- uint8_t *dst8_ptr = dst8;
- int dst_stride = conv_params->dst_stride;
- int height = h;
-
- if (w == 4) {
- const uint8x16x2_t permute_tbl = vld1q_u8_x2(dot_prod_permute_tbl);
-
- do {
- uint8x16_t s0, s1, s2, s3;
- uint16x4_t d0, d1, d2, d3, dd0, dd1, dd2, dd3;
- uint8x8_t d01_u8, d23_u8;
-
- load_u8_16x4(src_ptr, src_stride, &s0, &s1, &s2, &s3);
-
- d0 = convolve8_4_x(s0, x_filter, permute_tbl, round_offset_shim);
- d1 = convolve8_4_x(s1, x_filter, permute_tbl, round_offset_shim);
- d2 = convolve8_4_x(s2, x_filter, permute_tbl, round_offset_shim);
- d3 = convolve8_4_x(s3, x_filter, permute_tbl, round_offset_shim);
-
- load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3);
-
- compute_dist_wtd_avg_4x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset,
- bck_offset, round_offset_vec, &d01_u8, &d23_u8);
-
- store_u8_4x1(dst8_ptr + 0 * dst8_stride, d01_u8, 0);
- store_u8_4x1(dst8_ptr + 1 * dst8_stride, d01_u8, 1);
- store_u8_4x1(dst8_ptr + 2 * dst8_stride, d23_u8, 0);
- store_u8_4x1(dst8_ptr + 3 * dst8_stride, d23_u8, 1);
-
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- dst8_ptr += 4 * dst8_stride;
- height -= 4;
- } while (height != 0);
- } else {
- const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
-
- do {
- const uint8_t *s = src_ptr;
- CONV_BUF_TYPE *d = dst_ptr;
- uint8_t *d_u8 = dst8_ptr;
- int width = w;
-
- do {
- uint8x16_t s0, s1, s2, s3;
- uint16x8_t d0, d1, d2, d3, dd0, dd1, dd2, dd3;
- uint8x8_t d0_u8, d1_u8, d2_u8, d3_u8;
-
- load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3);
-
- d0 = convolve8_8_x(s0, x_filter, permute_tbl, round_offset_shim);
- d1 = convolve8_8_x(s1, x_filter, permute_tbl, round_offset_shim);
- d2 = convolve8_8_x(s2, x_filter, permute_tbl, round_offset_shim);
- d3 = convolve8_8_x(s3, x_filter, permute_tbl, round_offset_shim);
-
- load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
-
- compute_dist_wtd_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset,
- bck_offset, round_offset_vec, &d0_u8, &d1_u8,
- &d2_u8, &d3_u8);
-
- store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8);
-
- s += 8;
- d += 8;
- d_u8 += 8;
- width -= 8;
- } while (width != 0);
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- dst8_ptr += 4 * dst8_stride;
- height -= 4;
- } while (height != 0);
- }
-}
-
-static INLINE void dist_wtd_convolve_x_avg_neon(
- const uint8_t *src, int src_stride, uint8_t *dst8, int dst8_stride, int w,
- int h, const InterpFilterParams *filter_params_x, const int subpel_x_qn,
- ConvolveParams *conv_params) {
- assert(w % 4 == 0);
- assert(h % 4 == 0);
-
- const int bd = 8;
- const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
- const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
- (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
- const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
- // A shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-rounding
- // shifts - which are generally faster than rounding shifts on modern CPUs.
- // (The extra -1 is needed because we halved the filter values.)
- const int32x4_t round_offset_shim = vdupq_n_s32(
- (round_offset << (ROUND0_BITS - 1)) + (1 << ((ROUND0_BITS - 1) - 1)));
-
- // Horizontal filter.
- const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
- filter_params_x, subpel_x_qn & SUBPEL_MASK);
- // Filter values are even, so downshift by 1 to reduce intermediate precision
- // requirements.
- const int8x8_t x_filter = vshrn_n_s16(vld1q_s16(x_filter_ptr), 1);
-
- const int horiz_offset = filter_params_x->taps / 2 - 1;
- const uint8_t *src_ptr = src - horiz_offset;
- CONV_BUF_TYPE *dst_ptr = conv_params->dst;
- uint8_t *dst8_ptr = dst8;
- int dst_stride = conv_params->dst_stride;
- int height = h;
-
- if (w == 4) {
- const uint8x16x2_t permute_tbl = vld1q_u8_x2(dot_prod_permute_tbl);
-
- do {
- uint8x16_t s0, s1, s2, s3;
- uint16x4_t d0, d1, d2, d3, dd0, dd1, dd2, dd3;
- uint8x8_t d01_u8, d23_u8;
-
- load_u8_16x4(src_ptr, src_stride, &s0, &s1, &s2, &s3);
-
- d0 = convolve8_4_x(s0, x_filter, permute_tbl, round_offset_shim);
- d1 = convolve8_4_x(s1, x_filter, permute_tbl, round_offset_shim);
- d2 = convolve8_4_x(s2, x_filter, permute_tbl, round_offset_shim);
- d3 = convolve8_4_x(s3, x_filter, permute_tbl, round_offset_shim);
-
- load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3);
-
- compute_basic_avg_4x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3,
- round_offset_vec, &d01_u8, &d23_u8);
-
- store_u8_4x1(dst8_ptr + 0 * dst8_stride, d01_u8, 0);
- store_u8_4x1(dst8_ptr + 1 * dst8_stride, d01_u8, 1);
- store_u8_4x1(dst8_ptr + 2 * dst8_stride, d23_u8, 0);
- store_u8_4x1(dst8_ptr + 3 * dst8_stride, d23_u8, 1);
-
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- dst8_ptr += 4 * dst8_stride;
- height -= 4;
- } while (height != 0);
- } else {
- const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
-
- do {
- const uint8_t *s = src_ptr;
- CONV_BUF_TYPE *d = dst_ptr;
- uint8_t *d_u8 = dst8_ptr;
- int width = w;
-
- do {
- uint8x16_t s0, s1, s2, s3;
- uint16x8_t d0, d1, d2, d3, dd0, dd1, dd2, dd3;
- uint8x8_t d0_u8, d1_u8, d2_u8, d3_u8;
-
- load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3);
-
- d0 = convolve8_8_x(s0, x_filter, permute_tbl, round_offset_shim);
- d1 = convolve8_8_x(s1, x_filter, permute_tbl, round_offset_shim);
- d2 = convolve8_8_x(s2, x_filter, permute_tbl, round_offset_shim);
- d3 = convolve8_8_x(s3, x_filter, permute_tbl, round_offset_shim);
-
- load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
-
- compute_basic_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3,
- round_offset_vec, &d0_u8, &d1_u8, &d2_u8, &d3_u8);
-
- store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8);
-
- s += 8;
- d += 8;
- d_u8 += 8;
- width -= 8;
- } while (width != 0);
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- dst8_ptr += 4 * dst8_stride;
- height -= 4;
- } while (height != 0);
- }
-}
-
-static INLINE void dist_wtd_convolve_x_neon(
- const uint8_t *src, int src_stride, int w, int h,
- const InterpFilterParams *filter_params_x, const int subpel_x_qn,
- ConvolveParams *conv_params) {
- assert(w % 4 == 0);
- assert(h % 4 == 0);
-
- const int bd = 8;
- const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
- const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
- (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
- // A shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-rounding
- // shifts - which are generally faster than rounding shifts on modern CPUs.
- // (The extra -1 is needed because we halved the filter values.)
- const int32x4_t round_offset_shim = vdupq_n_s32(
- (round_offset << (ROUND0_BITS - 1)) + (1 << ((ROUND0_BITS - 1) - 1)));
-
- // Horizontal filter.
- const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
- filter_params_x, subpel_x_qn & SUBPEL_MASK);
- // Filter values are even, so downshift by 1 to reduce intermediate precision
- // requirements.
- const int8x8_t x_filter = vshrn_n_s16(vld1q_s16(x_filter_ptr), 1);
-
- const int horiz_offset = filter_params_x->taps / 2 - 1;
- const uint8_t *src_ptr = src - horiz_offset;
- CONV_BUF_TYPE *dst_ptr = conv_params->dst;
- int dst_stride = conv_params->dst_stride;
- int height = h;
-
- if (w == 4) {
- const uint8x16x2_t permute_tbl = vld1q_u8_x2(dot_prod_permute_tbl);
-
- do {
- uint8x16_t s0, s1, s2, s3;
- uint16x4_t d0, d1, d2, d3;
-
- load_u8_16x4(src_ptr, src_stride, &s0, &s1, &s2, &s3);
-
- d0 = convolve8_4_x(s0, x_filter, permute_tbl, round_offset_shim);
- d1 = convolve8_4_x(s1, x_filter, permute_tbl, round_offset_shim);
- d2 = convolve8_4_x(s2, x_filter, permute_tbl, round_offset_shim);
- d3 = convolve8_4_x(s3, x_filter, permute_tbl, round_offset_shim);
-
- store_u16_4x4(dst_ptr, dst_stride, d0, d1, d2, d3);
-
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- height -= 4;
- } while (height != 0);
- } else {
- const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
-
- do {
- const uint8_t *s = src_ptr;
- CONV_BUF_TYPE *d = dst_ptr;
- int width = w;
-
- do {
- uint8x16_t s0, s1, s2, s3;
- uint16x8_t d0, d1, d2, d3;
-
- load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3);
-
- d0 = convolve8_8_x(s0, x_filter, permute_tbl, round_offset_shim);
- d1 = convolve8_8_x(s1, x_filter, permute_tbl, round_offset_shim);
- d2 = convolve8_8_x(s2, x_filter, permute_tbl, round_offset_shim);
- d3 = convolve8_8_x(s3, x_filter, permute_tbl, round_offset_shim);
-
- store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
-
- s += 8;
- d += 8;
- width -= 8;
- } while (width != 0);
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- height -= 4;
- } while (height != 0);
- }
-}
-
-#elif AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_DOTPROD)
-
-static INLINE uint16x4_t convolve8_4_x(uint8x16_t samples,
- const int8x8_t x_filter,
- const int32x4_t correction,
- const uint8x16_t range_limit,
- const uint8x16x2_t permute_tbl) {
- int8x16_t clamped_samples, permuted_samples[2];
- int32x4_t sum;
-
- // Clamp sample range to [-128, 127] for 8-bit signed dot product.
- clamped_samples = vreinterpretq_s8_u8(vsubq_u8(samples, range_limit));
-
- // Permute samples ready for dot product.
- // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
- permuted_samples[0] = vqtbl1q_s8(clamped_samples, permute_tbl.val[0]);
- // { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 }
- permuted_samples[1] = vqtbl1q_s8(clamped_samples, permute_tbl.val[1]);
-
- // Accumulate dot product into 'correction' to account for range clamp.
- sum = vdotq_lane_s32(correction, permuted_samples[0], x_filter, 0);
- sum = vdotq_lane_s32(sum, permuted_samples[1], x_filter, 1);
-
- // We halved the convolution filter values so -1 from the right shift.
- return vreinterpret_u16_s16(vshrn_n_s32(sum, ROUND0_BITS - 1));
-}
-
-static INLINE uint16x8_t convolve8_8_x(uint8x16_t samples,
- const int8x8_t x_filter,
- const int32x4_t correction,
- const uint8x16_t range_limit,
- const uint8x16x3_t permute_tbl) {
- int8x16_t clamped_samples, permuted_samples[3];
- int32x4_t sum[2];
-
- // Clamp sample range to [-128, 127] for 8-bit signed dot product.
- clamped_samples = vreinterpretq_s8_u8(vsubq_u8(samples, range_limit));
-
- // Permute samples ready for dot product. */
- // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
- permuted_samples[0] = vqtbl1q_s8(clamped_samples, permute_tbl.val[0]);
- // { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 }
- permuted_samples[1] = vqtbl1q_s8(clamped_samples, permute_tbl.val[1]);
- // { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 }
- permuted_samples[2] = vqtbl1q_s8(clamped_samples, permute_tbl.val[2]);
-
- // Accumulate dot product into 'correction' to account for range clamp.
- // First 4 output values.
- sum[0] = vdotq_lane_s32(correction, permuted_samples[0], x_filter, 0);
- sum[0] = vdotq_lane_s32(sum[0], permuted_samples[1], x_filter, 1);
- // Second 4 output values.
- sum[1] = vdotq_lane_s32(correction, permuted_samples[1], x_filter, 0);
- sum[1] = vdotq_lane_s32(sum[1], permuted_samples[2], x_filter, 1);
-
- // Narrow and re-pack.
- // We halved the convolution filter values so -1 from the right shift.
- int16x8_t res = vcombine_s16(vshrn_n_s32(sum[0], ROUND0_BITS - 1),
- vshrn_n_s32(sum[1], ROUND0_BITS - 1));
- return vreinterpretq_u16_s16(res);
-}
-
-static INLINE void dist_wtd_convolve_x_dist_wtd_avg_neon(
- const uint8_t *src, int src_stride, uint8_t *dst8, int dst8_stride, int w,
- int h, const InterpFilterParams *filter_params_x, const int subpel_x_qn,
- ConvolveParams *conv_params) {
- assert(w % 4 == 0);
- assert(h % 4 == 0);
-
- const int bd = 8;
- const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
- const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
- (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
- const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
-
- const uint16_t fwd_offset = conv_params->fwd_offset;
- const uint16_t bck_offset = conv_params->bck_offset;
-
- // Horizontal filter.
- const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
- filter_params_x, subpel_x_qn & SUBPEL_MASK);
- // Filter values are even, so downshift by 1 to reduce intermediate precision
- // requirements.
- const int8x8_t x_filter = vshrn_n_s16(vld1q_s16(x_filter_ptr), 1);
-
- // Dot-product constants and other shims.
- const uint8x16_t range_limit = vdupq_n_u8(128);
- const int32_t correction_s32 = vaddlvq_s16(vshll_n_s8(x_filter, 7));
- // Fold round_offset into the dot-product filter correction constant. The
- // additional shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-
- // rounding shifts - which are generally faster than rounding shifts on
- // modern CPUs. (The extra -1 is needed because we halved the filter values.)
- int32x4_t correction =
- vdupq_n_s32(correction_s32 + (round_offset << (ROUND0_BITS - 1)) +
- (1 << ((ROUND0_BITS - 1) - 1)));
-
- const int horiz_offset = filter_params_x->taps / 2 - 1;
- const uint8_t *src_ptr = src - horiz_offset;
- CONV_BUF_TYPE *dst_ptr = conv_params->dst;
- uint8_t *dst8_ptr = dst8;
- int dst_stride = conv_params->dst_stride;
- int height = h;
-
- if (w == 4) {
- const uint8x16x2_t permute_tbl = vld1q_u8_x2(dot_prod_permute_tbl);
-
- do {
- uint8x16_t s0, s1, s2, s3;
- uint16x4_t d0, d1, d2, d3, dd0, dd1, dd2, dd3;
- uint8x8_t d01_u8, d23_u8;
-
- load_u8_16x4(src_ptr, src_stride, &s0, &s1, &s2, &s3);
-
- d0 = convolve8_4_x(s0, x_filter, correction, range_limit, permute_tbl);
- d1 = convolve8_4_x(s1, x_filter, correction, range_limit, permute_tbl);
- d2 = convolve8_4_x(s2, x_filter, correction, range_limit, permute_tbl);
- d3 = convolve8_4_x(s3, x_filter, correction, range_limit, permute_tbl);
-
- load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3);
-
- compute_dist_wtd_avg_4x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset,
- bck_offset, round_offset_vec, &d01_u8, &d23_u8);
-
- store_u8_4x1(dst8_ptr + 0 * dst8_stride, d01_u8, 0);
- store_u8_4x1(dst8_ptr + 1 * dst8_stride, d01_u8, 1);
- store_u8_4x1(dst8_ptr + 2 * dst8_stride, d23_u8, 0);
- store_u8_4x1(dst8_ptr + 3 * dst8_stride, d23_u8, 1);
-
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- dst8_ptr += 4 * dst8_stride;
- height -= 4;
- } while (height != 0);
- } else {
- const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
-
- do {
- const uint8_t *s = src_ptr;
- CONV_BUF_TYPE *d = dst_ptr;
- uint8_t *d_u8 = dst8_ptr;
- int width = w;
-
- do {
- uint8x16_t s0, s1, s2, s3;
- uint16x8_t d0, d1, d2, d3, dd0, dd1, dd2, dd3;
- uint8x8_t d0_u8, d1_u8, d2_u8, d3_u8;
-
- load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3);
-
- d0 = convolve8_8_x(s0, x_filter, correction, range_limit, permute_tbl);
- d1 = convolve8_8_x(s1, x_filter, correction, range_limit, permute_tbl);
- d2 = convolve8_8_x(s2, x_filter, correction, range_limit, permute_tbl);
- d3 = convolve8_8_x(s3, x_filter, correction, range_limit, permute_tbl);
-
- load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
-
- compute_dist_wtd_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset,
- bck_offset, round_offset_vec, &d0_u8, &d1_u8,
- &d2_u8, &d3_u8);
-
- store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8);
-
- s += 8;
- d += 8;
- d_u8 += 8;
- width -= 8;
- } while (width != 0);
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- dst8_ptr += 4 * dst8_stride;
- height -= 4;
- } while (height != 0);
- }
-}
-
-static INLINE void dist_wtd_convolve_x_avg_neon(
- const uint8_t *src, int src_stride, uint8_t *dst8, int dst8_stride, int w,
- int h, const InterpFilterParams *filter_params_x, const int subpel_x_qn,
- ConvolveParams *conv_params) {
- assert(w % 4 == 0);
- assert(h % 4 == 0);
-
- const int bd = 8;
- const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
- const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
- (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
- const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
-
- // Horizontal filter.
- const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
- filter_params_x, subpel_x_qn & SUBPEL_MASK);
- // Filter values are even, so downshift by 1 to reduce intermediate precision
- // requirements.
- const int8x8_t x_filter = vshrn_n_s16(vld1q_s16(x_filter_ptr), 1);
-
- // Dot-product constants and other shims.
- const uint8x16_t range_limit = vdupq_n_u8(128);
- const int32_t correction_s32 = vaddlvq_s16(vshll_n_s8(x_filter, 7));
- // Fold round_offset into the dot-product filter correction constant. The
- // additional shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-
- // rounding shifts - which are generally faster than rounding shifts on
- // modern CPUs. (The extra -1 is needed because we halved the filter values.)
- int32x4_t correction =
- vdupq_n_s32(correction_s32 + (round_offset << (ROUND0_BITS - 1)) +
- (1 << ((ROUND0_BITS - 1) - 1)));
-
- const int horiz_offset = filter_params_x->taps / 2 - 1;
- const uint8_t *src_ptr = src - horiz_offset;
- CONV_BUF_TYPE *dst_ptr = conv_params->dst;
- uint8_t *dst8_ptr = dst8;
- int dst_stride = conv_params->dst_stride;
- int height = h;
-
- if (w == 4) {
- const uint8x16x2_t permute_tbl = vld1q_u8_x2(dot_prod_permute_tbl);
-
- do {
- uint8x16_t s0, s1, s2, s3;
- uint16x4_t d0, d1, d2, d3, dd0, dd1, dd2, dd3;
- uint8x8_t d01_u8, d23_u8;
-
- load_u8_16x4(src_ptr, src_stride, &s0, &s1, &s2, &s3);
-
- d0 = convolve8_4_x(s0, x_filter, correction, range_limit, permute_tbl);
- d1 = convolve8_4_x(s1, x_filter, correction, range_limit, permute_tbl);
- d2 = convolve8_4_x(s2, x_filter, correction, range_limit, permute_tbl);
- d3 = convolve8_4_x(s3, x_filter, correction, range_limit, permute_tbl);
-
- load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3);
-
- compute_basic_avg_4x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3,
- round_offset_vec, &d01_u8, &d23_u8);
-
- store_u8_4x1(dst8_ptr + 0 * dst8_stride, d01_u8, 0);
- store_u8_4x1(dst8_ptr + 1 * dst8_stride, d01_u8, 1);
- store_u8_4x1(dst8_ptr + 2 * dst8_stride, d23_u8, 0);
- store_u8_4x1(dst8_ptr + 3 * dst8_stride, d23_u8, 1);
-
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- dst8_ptr += 4 * dst8_stride;
- height -= 4;
- } while (height != 0);
- } else {
- const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
-
- do {
- const uint8_t *s = src_ptr;
- CONV_BUF_TYPE *d = dst_ptr;
- uint8_t *d_u8 = dst8_ptr;
- int width = w;
-
- do {
- uint8x16_t s0, s1, s2, s3;
- uint16x8_t d0, d1, d2, d3, dd0, dd1, dd2, dd3;
- uint8x8_t d0_u8, d1_u8, d2_u8, d3_u8;
-
- load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3);
-
- d0 = convolve8_8_x(s0, x_filter, correction, range_limit, permute_tbl);
- d1 = convolve8_8_x(s1, x_filter, correction, range_limit, permute_tbl);
- d2 = convolve8_8_x(s2, x_filter, correction, range_limit, permute_tbl);
- d3 = convolve8_8_x(s3, x_filter, correction, range_limit, permute_tbl);
-
- load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
-
- compute_basic_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3,
- round_offset_vec, &d0_u8, &d1_u8, &d2_u8, &d3_u8);
-
- store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8);
-
- s += 8;
- d += 8;
- d_u8 += 8;
- width -= 8;
- } while (width != 0);
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- dst8_ptr += 4 * dst8_stride;
- height -= 4;
- } while (height != 0);
- }
-}
-
-static INLINE void dist_wtd_convolve_x_neon(
- const uint8_t *src, int src_stride, int w, int h,
- const InterpFilterParams *filter_params_x, const int subpel_x_qn,
- ConvolveParams *conv_params) {
- assert(w % 4 == 0);
- assert(h % 4 == 0);
-
- const int bd = 8;
- const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
- const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
- (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
-
- // Horizontal filter.
- const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
- filter_params_x, subpel_x_qn & SUBPEL_MASK);
- // Filter values are even, so downshift by 1 to reduce intermediate precision
- // requirements.
- const int8x8_t x_filter = vshrn_n_s16(vld1q_s16(x_filter_ptr), 1);
-
- // Dot-product constants and other shims.
- const uint8x16_t range_limit = vdupq_n_u8(128);
- const int32_t correction_s32 = vaddlvq_s16(vshll_n_s8(x_filter, 7));
- // Fold round_offset into the dot-product filter correction constant. The
- // additional shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-
- // rounding shifts - which are generally faster than rounding shifts on
- // modern CPUs. (The extra -1 is needed because we halved the filter values.)
- int32x4_t correction =
- vdupq_n_s32(correction_s32 + (round_offset << (ROUND0_BITS - 1)) +
- (1 << ((ROUND0_BITS - 1) - 1)));
-
- const int horiz_offset = filter_params_x->taps / 2 - 1;
- const uint8_t *src_ptr = src - horiz_offset;
- CONV_BUF_TYPE *dst_ptr = conv_params->dst;
- int dst_stride = conv_params->dst_stride;
- int height = h;
-
- if (w == 4) {
- const uint8x16x2_t permute_tbl = vld1q_u8_x2(dot_prod_permute_tbl);
-
- do {
- uint8x16_t s0, s1, s2, s3;
- uint16x4_t d0, d1, d2, d3;
-
- load_u8_16x4(src_ptr, src_stride, &s0, &s1, &s2, &s3);
-
- d0 = convolve8_4_x(s0, x_filter, correction, range_limit, permute_tbl);
- d1 = convolve8_4_x(s1, x_filter, correction, range_limit, permute_tbl);
- d2 = convolve8_4_x(s2, x_filter, correction, range_limit, permute_tbl);
- d3 = convolve8_4_x(s3, x_filter, correction, range_limit, permute_tbl);
-
- store_u16_4x4(dst_ptr, dst_stride, d0, d1, d2, d3);
-
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- height -= 4;
- } while (height != 0);
- } else {
- const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
-
- do {
- const uint8_t *s = src_ptr;
- CONV_BUF_TYPE *d = dst_ptr;
- int width = w;
-
- do {
- uint8x16_t s0, s1, s2, s3;
- uint16x8_t d0, d1, d2, d3;
-
- load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3);
-
- d0 = convolve8_8_x(s0, x_filter, correction, range_limit, permute_tbl);
- d1 = convolve8_8_x(s1, x_filter, correction, range_limit, permute_tbl);
- d2 = convolve8_8_x(s2, x_filter, correction, range_limit, permute_tbl);
- d3 = convolve8_8_x(s3, x_filter, correction, range_limit, permute_tbl);
-
- store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
-
- s += 8;
- d += 8;
- width -= 8;
- } while (width != 0);
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- height -= 4;
- } while (height != 0);
- }
-}
-
-#else // !(AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_DOTPROD))
-
-static INLINE uint16x4_t convolve8_4_x(const int16x4_t s0, const int16x4_t s1,
- const int16x4_t s2, const int16x4_t s3,
- const int16x4_t s4, const int16x4_t s5,
- const int16x4_t s6, const int16x4_t s7,
- const int16x8_t x_filter,
- const int16x4_t round_offset) {
- const int16x4_t x_filter_0_3 = vget_low_s16(x_filter);
- const int16x4_t x_filter_4_7 = vget_high_s16(x_filter);
-
- int16x4_t sum = vmul_lane_s16(s0, x_filter_0_3, 0);
- sum = vmla_lane_s16(sum, s1, x_filter_0_3, 1);
- sum = vmla_lane_s16(sum, s2, x_filter_0_3, 2);
- sum = vmla_lane_s16(sum, s3, x_filter_0_3, 3);
- sum = vmla_lane_s16(sum, s4, x_filter_4_7, 0);
- sum = vmla_lane_s16(sum, s5, x_filter_4_7, 1);
- sum = vmla_lane_s16(sum, s6, x_filter_4_7, 2);
- sum = vmla_lane_s16(sum, s7, x_filter_4_7, 3);
-
- // We halved the convolution filter values so -1 from the right shift.
- int16x4_t res = vrsra_n_s16(round_offset, sum, ROUND0_BITS - 1);
- return vreinterpret_u16_s16(res);
-}
-
-static INLINE uint16x8_t convolve8_8_x(const int16x8_t s0, const int16x8_t s1,
- const int16x8_t s2, const int16x8_t s3,
- const int16x8_t s4, const int16x8_t s5,
- const int16x8_t s6, const int16x8_t s7,
- const int16x8_t x_filter,
- const int16x8_t round_offset) {
- const int16x4_t x_filter_0_3 = vget_low_s16(x_filter);
- const int16x4_t x_filter_4_7 = vget_high_s16(x_filter);
-
- int16x8_t sum = vmulq_lane_s16(s0, x_filter_0_3, 0);
- sum = vmlaq_lane_s16(sum, s1, x_filter_0_3, 1);
- sum = vmlaq_lane_s16(sum, s2, x_filter_0_3, 2);
- sum = vmlaq_lane_s16(sum, s3, x_filter_0_3, 3);
- sum = vmlaq_lane_s16(sum, s4, x_filter_4_7, 0);
- sum = vmlaq_lane_s16(sum, s5, x_filter_4_7, 1);
- sum = vmlaq_lane_s16(sum, s6, x_filter_4_7, 2);
- sum = vmlaq_lane_s16(sum, s7, x_filter_4_7, 3);
-
- // We halved the convolution filter values so -1 from the right shift.
- int16x8_t res = vrsraq_n_s16(round_offset, sum, ROUND0_BITS - 1);
- return vreinterpretq_u16_s16(res);
-}
-
-static INLINE void dist_wtd_convolve_x_dist_wtd_avg_neon(
- const uint8_t *src, int src_stride, uint8_t *dst8, int dst8_stride, int w,
- int h, const InterpFilterParams *filter_params_x, const int subpel_x_qn,
- ConvolveParams *conv_params) {
- assert(w % 4 == 0);
- assert(h % 4 == 0);
-
- const int bd = 8;
- const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
- const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
- (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
- const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
-
- const uint16_t fwd_offset = conv_params->fwd_offset;
- const uint16_t bck_offset = conv_params->bck_offset;
-
- // Horizontal filter.
- const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
- filter_params_x, subpel_x_qn & SUBPEL_MASK);
- // Filter values are even, so downshift by 1 to reduce intermediate precision
- // requirements.
- const int16x8_t x_filter = vshrq_n_s16(vld1q_s16(x_filter_ptr), 1);
-
- const int horiz_offset = filter_params_x->taps / 2 - 1;
- const uint8_t *src_ptr = src - horiz_offset;
- CONV_BUF_TYPE *dst_ptr = conv_params->dst;
- uint8_t *dst8_ptr = dst8;
- int dst_stride = conv_params->dst_stride;
- const uint8_t *s;
- uint8_t *d_u8;
- CONV_BUF_TYPE *d;
- int width;
- int height = h;
-
- uint8x8_t t0;
-#if AOM_ARCH_AARCH64
- uint8x8_t t1, t2, t3, t4, t5, t6, t7;
-#endif // AOM_ARCH_AARCH64
-
- if (w == 4 || h == 4) {
- int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8;
- uint16x4_t d0, dd0;
- uint8x8_t d01;
-#if AOM_ARCH_AARCH64
- int16x4_t s9, s10;
- uint16x4_t d1, d2, d3, dd1, dd2, dd3;
- uint8x8_t d23;
-#endif // AOM_ARCH_AARCH64
-
- do {
- d = dst_ptr;
- d_u8 = dst8_ptr;
- width = w;
-
- __builtin_prefetch(src_ptr + 0 * src_stride);
-#if AOM_ARCH_AARCH64
- __builtin_prefetch(src_ptr + 1 * src_stride);
- __builtin_prefetch(src_ptr + 2 * src_stride);
- __builtin_prefetch(src_ptr + 3 * src_stride);
-
- load_u8_8x4(src_ptr, src_stride, &t0, &t1, &t2, &t3);
- transpose_u8_8x4(&t0, &t1, &t2, &t3);
-
- s0 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s1 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
- s2 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
- s3 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t3)));
- s4 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s5 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
- s6 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
-
- __builtin_prefetch(d + 0 * dst_stride);
- __builtin_prefetch(d + 1 * dst_stride);
- __builtin_prefetch(d + 2 * dst_stride);
- __builtin_prefetch(d + 3 * dst_stride);
-
- s = src_ptr + 7;
-
- do {
- load_unaligned_u8_4x4(s, src_stride, &t0, &t1);
- transpose_u8_4x4(&t0, &t1);
-
- s7 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s8 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
- s9 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s10 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
-
- d0 = convolve8_4_x(s0, s1, s2, s3, s4, s5, s6, s7, x_filter,
- vget_low_s16(round_offset_vec));
- d1 = convolve8_4_x(s1, s2, s3, s4, s5, s6, s7, s8, x_filter,
- vget_low_s16(round_offset_vec));
- d2 = convolve8_4_x(s2, s3, s4, s5, s6, s7, s8, s9, x_filter,
- vget_low_s16(round_offset_vec));
- d3 = convolve8_4_x(s3, s4, s5, s6, s7, s8, s9, s10, x_filter,
- vget_low_s16(round_offset_vec));
-
- transpose_u16_4x4d(&d0, &d1, &d2, &d3);
-
- __builtin_prefetch(d + 0 * dst_stride);
- __builtin_prefetch(d + 1 * dst_stride);
- __builtin_prefetch(d + 2 * dst_stride);
- __builtin_prefetch(d + 3 * dst_stride);
-
- __builtin_prefetch(d_u8 + 0 * dst8_stride);
- __builtin_prefetch(d_u8 + 1 * dst8_stride);
- __builtin_prefetch(d_u8 + 2 * dst8_stride);
- __builtin_prefetch(d_u8 + 3 * dst8_stride);
-
- load_u16_4x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
-
- compute_dist_wtd_avg_4x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset,
- bck_offset, round_offset_vec, &d01, &d23);
-
- store_u8_4x1(d_u8 + 0 * dst8_stride, d01, 0);
- store_u8_4x1(d_u8 + 1 * dst8_stride, d01, 1);
- store_u8_4x1(d_u8 + 2 * dst8_stride, d23, 0);
- store_u8_4x1(d_u8 + 3 * dst8_stride, d23, 1);
-
- s0 = s4;
- s1 = s5;
- s2 = s6;
- s3 = s7;
- s4 = s8;
- s5 = s9;
- s6 = s10;
- s += 4;
- d += 4;
- d_u8 += 4;
- width -= 4;
- } while (width != 0);
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- dst8_ptr += 4 * dst8_stride;
- height -= 4;
-#else // !AOM_ARCH_AARCH64
- t0 = vld1_u8(src_ptr); // a0 a1 a2 a3 a4 a5 a6 a7
- s0 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s4 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
-
- __builtin_prefetch(d);
-
- s = src_ptr + 8;
-
- do {
- t0 = vld1_u8(s); // a8 a9 a10 a11
- s8 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
-
- s1 = vext_s16(s0, s4, 1); // a1 a2 a3 a4
- s2 = vext_s16(s0, s4, 2); // a2 a3 a4 a5
- s3 = vext_s16(s0, s4, 3); // a3 a4 a5 a6
- s5 = vext_s16(s4, s8, 1); // a5 a6 a7 a8
- s6 = vext_s16(s4, s8, 2); // a6 a7 a8 a9
- s7 = vext_s16(s4, s8, 3); // a7 a8 a9 a10
-
- d0 = convolve8_4_x(s0, s1, s2, s3, s4, s5, s6, s7, x_filter,
- vget_low_s16(round_offset_vec));
-
- __builtin_prefetch(d);
- __builtin_prefetch(d_u8);
-
- dd0 = vld1_u16(d);
-
- compute_dist_wtd_avg_4x1(dd0, d0, fwd_offset, bck_offset,
- vget_low_s16(round_offset_vec), &d01);
-
- store_u8_4x1(d_u8, d01, 0);
-
- s0 = s4;
- s4 = s8;
- s += 4;
- d += 4;
- d_u8 += 4;
- width -= 4;
- } while (width != 0);
- src_ptr += src_stride;
- dst_ptr += dst_stride;
- dst8_ptr += dst8_stride;
- height--;
-#endif // AOM_ARCH_AARCH64
- } while (height != 0);
- } else {
- int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8;
- uint16x8_t d0, dd0;
- uint8x8_t d0_u8;
-
- do {
- d = dst_ptr;
- d_u8 = dst8_ptr;
- width = w;
-
-#if AOM_ARCH_AARCH64
- int16x8_t s9, s10, s11, s12, s13, s14;
- uint16x8_t d1, d2, d3, d4, d5, d6, d7, dd1, dd2, dd3, dd4, dd5, dd6, dd7;
- uint8x8_t d1_u8, d2_u8, d3_u8, d4_u8, d5_u8, d6_u8, d7_u8;
-
- __builtin_prefetch(src_ptr + 0 * src_stride);
- __builtin_prefetch(src_ptr + 1 * src_stride);
- __builtin_prefetch(src_ptr + 2 * src_stride);
- __builtin_prefetch(src_ptr + 3 * src_stride);
- __builtin_prefetch(src_ptr + 4 * src_stride);
- __builtin_prefetch(src_ptr + 5 * src_stride);
- __builtin_prefetch(src_ptr + 6 * src_stride);
- __builtin_prefetch(src_ptr + 7 * src_stride);
-
- load_u8_8x8(src_ptr, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
- transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
-
- s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
- s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
- s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
- s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
- s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
- s5 = vreinterpretq_s16_u16(vmovl_u8(t5));
- s6 = vreinterpretq_s16_u16(vmovl_u8(t6));
-
- __builtin_prefetch(dst_ptr + 0 * dst_stride);
- __builtin_prefetch(dst_ptr + 1 * dst_stride);
- __builtin_prefetch(dst_ptr + 2 * dst_stride);
- __builtin_prefetch(dst_ptr + 3 * dst_stride);
- __builtin_prefetch(dst_ptr + 4 * dst_stride);
- __builtin_prefetch(dst_ptr + 5 * dst_stride);
- __builtin_prefetch(dst_ptr + 6 * dst_stride);
- __builtin_prefetch(dst_ptr + 7 * dst_stride);
-
- s = src_ptr + 7;
-
- do {
- load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
- transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
-
- s7 = vreinterpretq_s16_u16(vmovl_u8(t0));
- s8 = vreinterpretq_s16_u16(vmovl_u8(t1));
- s9 = vreinterpretq_s16_u16(vmovl_u8(t2));
- s10 = vreinterpretq_s16_u16(vmovl_u8(t3));
- s11 = vreinterpretq_s16_u16(vmovl_u8(t4));
- s12 = vreinterpretq_s16_u16(vmovl_u8(t5));
- s13 = vreinterpretq_s16_u16(vmovl_u8(t6));
- s14 = vreinterpretq_s16_u16(vmovl_u8(t7));
-
- d0 = convolve8_8_x(s0, s1, s2, s3, s4, s5, s6, s7, x_filter,
- round_offset_vec);
- d1 = convolve8_8_x(s1, s2, s3, s4, s5, s6, s7, s8, x_filter,
- round_offset_vec);
- d2 = convolve8_8_x(s2, s3, s4, s5, s6, s7, s8, s9, x_filter,
- round_offset_vec);
- d3 = convolve8_8_x(s3, s4, s5, s6, s7, s8, s9, s10, x_filter,
- round_offset_vec);
- d4 = convolve8_8_x(s4, s5, s6, s7, s8, s9, s10, s11, x_filter,
- round_offset_vec);
- d5 = convolve8_8_x(s5, s6, s7, s8, s9, s10, s11, s12, x_filter,
- round_offset_vec);
- d6 = convolve8_8_x(s6, s7, s8, s9, s10, s11, s12, s13, x_filter,
- round_offset_vec);
- d7 = convolve8_8_x(s7, s8, s9, s10, s11, s12, s13, s14, x_filter,
- round_offset_vec);
-
- transpose_u16_8x8(&d0, &d1, &d2, &d3, &d4, &d5, &d6, &d7);
-
- load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
-
- compute_dist_wtd_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset,
- bck_offset, round_offset_vec, &d0_u8, &d1_u8,
- &d2_u8, &d3_u8);
-
- store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8);
-
- load_u16_8x4(d + 4 * dst_stride, dst_stride, &dd4, &dd5, &dd6, &dd7);
-
- compute_dist_wtd_avg_8x4(dd4, dd5, dd6, dd7, d4, d5, d6, d7, fwd_offset,
- bck_offset, round_offset_vec, &d4_u8, &d5_u8,
- &d6_u8, &d7_u8);
-
- store_u8_8x4(d_u8 + 4 * dst8_stride, dst8_stride, d4_u8, d5_u8, d6_u8,
- d7_u8);
-
- s0 = s8;
- s1 = s9;
- s2 = s10;
- s3 = s11;
- s4 = s12;
- s5 = s13;
- s6 = s14;
- s += 8;
- d += 8;
- d_u8 += 8;
- width -= 8;
- } while (width != 0);
- src_ptr += 8 * src_stride;
- dst_ptr += 8 * dst_stride;
- dst8_ptr += 8 * dst8_stride;
- height -= 8;
-#else // !AOM_ARCH_AARCH64
- __builtin_prefetch(src_ptr);
-
- t0 = vld1_u8(src_ptr);
- s0 = vreinterpretq_s16_u16(vmovl_u8(t0)); // a0 a1 a2 a3 a4 a5 a6 a7
-
- __builtin_prefetch(dst_ptr);
-
- s = src_ptr + 8;
-
- do {
- t0 = vld1_u8(s); // a8 a9 a10 a11 a12 a13 a14 a15
- s8 = vreinterpretq_s16_u16(vmovl_u8(t0));
-
- s1 = vextq_s16(s0, s8, 1); // a1 a2 a3 a4 a5 a6 a7 a8
- s2 = vextq_s16(s0, s8, 2); // a2 a3 a4 a5 a6 a7 a8 a9
- s3 = vextq_s16(s0, s8, 3); // a3 a4 a5 a6 a7 a8 a9 a10
- s4 = vextq_s16(s0, s8, 4); // a4 a5 a6 a7 a8 a9 a10 a11
- s5 = vextq_s16(s0, s8, 5); // a5 a6 a7 a8 a9 a10 a11 a12
- s6 = vextq_s16(s0, s8, 6); // a6 a7 a8 a9 a10 a11 a12 a13
- s7 = vextq_s16(s0, s8, 7); // a7 a8 a9 a10 a11 a12 a13 a14
-
- d0 = convolve8_8_x(s0, s1, s2, s3, s4, s5, s6, s7, x_filter,
- round_offset_vec);
-
- dd0 = vld1q_u16(d);
-
- compute_dist_wtd_avg_8x1(dd0, d0, fwd_offset, bck_offset,
- round_offset_vec, &d0_u8);
-
- vst1_u8(d_u8, d0_u8);
-
- s0 = s8;
- s += 8;
- d += 8;
- d_u8 += 8;
- width -= 8;
- } while (width != 0);
- src_ptr += src_stride;
- dst_ptr += dst_stride;
- dst8_ptr += dst8_stride;
- height--;
-#endif // AOM_ARCH_AARCH64
- } while (height != 0);
- }
-}
-
-static INLINE void dist_wtd_convolve_x_avg_neon(
- const uint8_t *src, int src_stride, uint8_t *dst8, int dst8_stride, int w,
- int h, const InterpFilterParams *filter_params_x, const int subpel_x_qn,
- ConvolveParams *conv_params) {
- assert(w % 4 == 0);
- assert(h % 4 == 0);
-
- const int bd = 8;
- const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
- const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
- (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
- const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
-
- // Horizontal filter.
- const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
- filter_params_x, subpel_x_qn & SUBPEL_MASK);
- // Filter values are even, so downshift by 1 to reduce intermediate precision
- // requirements.
- const int16x8_t x_filter = vshrq_n_s16(vld1q_s16(x_filter_ptr), 1);
-
- const int horiz_offset = filter_params_x->taps / 2 - 1;
- const uint8_t *src_ptr = src - horiz_offset;
- CONV_BUF_TYPE *dst_ptr = conv_params->dst;
- uint8_t *dst8_ptr = dst8;
- int dst_stride = conv_params->dst_stride;
- const uint8_t *s;
- uint8_t *d_u8;
- CONV_BUF_TYPE *d;
- int width;
- int height = h;
-
- uint8x8_t t0;
-#if AOM_ARCH_AARCH64
- uint8x8_t t1, t2, t3, t4, t5, t6, t7;
-#endif // AOM_ARCH_AARCH64
-
- if (w == 4 || h == 4) {
- int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8;
- uint16x4_t d0, dd0;
- uint8x8_t d01;
-#if AOM_ARCH_AARCH64
- int16x4_t s9, s10;
- uint16x4_t d1, d2, d3, dd1, dd2, dd3;
- uint8x8_t d23;
-#endif // AOM_ARCH_AARCH64
-
- do {
- d = dst_ptr;
- d_u8 = dst8_ptr;
- width = w;
-
- __builtin_prefetch(src_ptr + 0 * src_stride);
-#if AOM_ARCH_AARCH64
- __builtin_prefetch(src_ptr + 1 * src_stride);
- __builtin_prefetch(src_ptr + 2 * src_stride);
- __builtin_prefetch(src_ptr + 3 * src_stride);
-
- load_u8_8x4(src_ptr, src_stride, &t0, &t1, &t2, &t3);
- transpose_u8_8x4(&t0, &t1, &t2, &t3);
-
- s0 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s1 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
- s2 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
- s3 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t3)));
- s4 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s5 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
- s6 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
-
- __builtin_prefetch(d + 0 * dst_stride);
- __builtin_prefetch(d + 1 * dst_stride);
- __builtin_prefetch(d + 2 * dst_stride);
- __builtin_prefetch(d + 3 * dst_stride);
-
- s = src_ptr + 7;
-
- do {
- load_unaligned_u8_4x4(s, src_stride, &t0, &t1);
- transpose_u8_4x4(&t0, &t1);
-
- s7 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s8 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
- s9 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s10 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
-
- d0 = convolve8_4_x(s0, s1, s2, s3, s4, s5, s6, s7, x_filter,
- vget_low_s16(round_offset_vec));
- d1 = convolve8_4_x(s1, s2, s3, s4, s5, s6, s7, s8, x_filter,
- vget_low_s16(round_offset_vec));
- d2 = convolve8_4_x(s2, s3, s4, s5, s6, s7, s8, s9, x_filter,
- vget_low_s16(round_offset_vec));
- d3 = convolve8_4_x(s3, s4, s5, s6, s7, s8, s9, s10, x_filter,
- vget_low_s16(round_offset_vec));
-
- transpose_u16_4x4d(&d0, &d1, &d2, &d3);
-
- __builtin_prefetch(d + 0 * dst_stride);
- __builtin_prefetch(d + 1 * dst_stride);
- __builtin_prefetch(d + 2 * dst_stride);
- __builtin_prefetch(d + 3 * dst_stride);
-
- __builtin_prefetch(d_u8 + 0 * dst8_stride);
- __builtin_prefetch(d_u8 + 1 * dst8_stride);
- __builtin_prefetch(d_u8 + 2 * dst8_stride);
- __builtin_prefetch(d_u8 + 3 * dst8_stride);
-
- load_u16_4x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
-
- compute_basic_avg_4x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3,
- round_offset_vec, &d01, &d23);
-
- store_u8_4x1(d_u8 + 0 * dst8_stride, d01, 0);
- store_u8_4x1(d_u8 + 1 * dst8_stride, d01, 1);
- store_u8_4x1(d_u8 + 2 * dst8_stride, d23, 0);
- store_u8_4x1(d_u8 + 3 * dst8_stride, d23, 1);
-
- s0 = s4;
- s1 = s5;
- s2 = s6;
- s3 = s7;
- s4 = s8;
- s5 = s9;
- s6 = s10;
- s += 4;
- d += 4;
- d_u8 += 4;
- width -= 4;
- } while (width != 0);
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- dst8_ptr += 4 * dst8_stride;
- height -= 4;
-#else // !AOM_ARCH_AARCH64
- t0 = vld1_u8(src_ptr); // a0 a1 a2 a3 a4 a5 a6 a7
- s0 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s4 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
-
- __builtin_prefetch(d);
-
- s = src_ptr + 8;
-
- do {
- t0 = vld1_u8(s); // a8 a9 a10 a11
- s8 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
-
- s1 = vext_s16(s0, s4, 1); // a1 a2 a3 a4
- s2 = vext_s16(s0, s4, 2); // a2 a3 a4 a5
- s3 = vext_s16(s0, s4, 3); // a3 a4 a5 a6
- s5 = vext_s16(s4, s8, 1); // a5 a6 a7 a8
- s6 = vext_s16(s4, s8, 2); // a6 a7 a8 a9
- s7 = vext_s16(s4, s8, 3); // a7 a8 a9 a10
-
- d0 = convolve8_4_x(s0, s1, s2, s3, s4, s5, s6, s7, x_filter,
- vget_low_s16(round_offset_vec));
-
- __builtin_prefetch(d);
- __builtin_prefetch(d_u8);
-
- dd0 = vld1_u16(d);
-
- compute_basic_avg_4x1(dd0, d0, vget_low_s16(round_offset_vec), &d01);
-
- store_u8_4x1(d_u8, d01, 0);
-
- s0 = s4;
- s4 = s8;
- s += 4;
- d += 4;
- d_u8 += 4;
- width -= 4;
- } while (width != 0);
- src_ptr += src_stride;
- dst_ptr += dst_stride;
- dst8_ptr += dst8_stride;
- height--;
-#endif // AOM_ARCH_AARCH64
- } while (height != 0);
- } else {
- int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8;
- uint16x8_t d0, dd0;
- uint8x8_t d0_u8;
-
- do {
- d = dst_ptr;
- d_u8 = dst8_ptr;
- width = w;
-
-#if AOM_ARCH_AARCH64
- int16x8_t s9, s10, s11, s12, s13, s14;
- uint16x8_t d1, d2, d3, d4, d5, d6, d7, dd1, dd2, dd3, dd4, dd5, dd6, dd7;
- uint8x8_t d1_u8, d2_u8, d3_u8, d4_u8, d5_u8, d6_u8, d7_u8;
-
- __builtin_prefetch(src_ptr + 0 * src_stride);
- __builtin_prefetch(src_ptr + 1 * src_stride);
- __builtin_prefetch(src_ptr + 2 * src_stride);
- __builtin_prefetch(src_ptr + 3 * src_stride);
- __builtin_prefetch(src_ptr + 4 * src_stride);
- __builtin_prefetch(src_ptr + 5 * src_stride);
- __builtin_prefetch(src_ptr + 6 * src_stride);
- __builtin_prefetch(src_ptr + 7 * src_stride);
-
- load_u8_8x8(src_ptr, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
- transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
-
- s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
- s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
- s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
- s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
- s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
- s5 = vreinterpretq_s16_u16(vmovl_u8(t5));
- s6 = vreinterpretq_s16_u16(vmovl_u8(t6));
-
- __builtin_prefetch(dst_ptr + 0 * dst_stride);
- __builtin_prefetch(dst_ptr + 1 * dst_stride);
- __builtin_prefetch(dst_ptr + 2 * dst_stride);
- __builtin_prefetch(dst_ptr + 3 * dst_stride);
- __builtin_prefetch(dst_ptr + 4 * dst_stride);
- __builtin_prefetch(dst_ptr + 5 * dst_stride);
- __builtin_prefetch(dst_ptr + 6 * dst_stride);
- __builtin_prefetch(dst_ptr + 7 * dst_stride);
-
- s = src_ptr + 7;
-
- do {
- load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
- transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
-
- s7 = vreinterpretq_s16_u16(vmovl_u8(t0));
- s8 = vreinterpretq_s16_u16(vmovl_u8(t1));
- s9 = vreinterpretq_s16_u16(vmovl_u8(t2));
- s10 = vreinterpretq_s16_u16(vmovl_u8(t3));
- s11 = vreinterpretq_s16_u16(vmovl_u8(t4));
- s12 = vreinterpretq_s16_u16(vmovl_u8(t5));
- s13 = vreinterpretq_s16_u16(vmovl_u8(t6));
- s14 = vreinterpretq_s16_u16(vmovl_u8(t7));
-
- d0 = convolve8_8_x(s0, s1, s2, s3, s4, s5, s6, s7, x_filter,
- round_offset_vec);
- d1 = convolve8_8_x(s1, s2, s3, s4, s5, s6, s7, s8, x_filter,
- round_offset_vec);
- d2 = convolve8_8_x(s2, s3, s4, s5, s6, s7, s8, s9, x_filter,
- round_offset_vec);
- d3 = convolve8_8_x(s3, s4, s5, s6, s7, s8, s9, s10, x_filter,
- round_offset_vec);
- d4 = convolve8_8_x(s4, s5, s6, s7, s8, s9, s10, s11, x_filter,
- round_offset_vec);
- d5 = convolve8_8_x(s5, s6, s7, s8, s9, s10, s11, s12, x_filter,
- round_offset_vec);
- d6 = convolve8_8_x(s6, s7, s8, s9, s10, s11, s12, s13, x_filter,
- round_offset_vec);
- d7 = convolve8_8_x(s7, s8, s9, s10, s11, s12, s13, s14, x_filter,
- round_offset_vec);
-
- transpose_u16_8x8(&d0, &d1, &d2, &d3, &d4, &d5, &d6, &d7);
-
- load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
-
- compute_basic_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3,
- round_offset_vec, &d0_u8, &d1_u8, &d2_u8, &d3_u8);
-
- store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8);
-
- load_u16_8x4(d + 4 * dst_stride, dst_stride, &dd4, &dd5, &dd6, &dd7);
-
- compute_basic_avg_8x4(dd4, dd5, dd6, dd7, d4, d5, d6, d7,
- round_offset_vec, &d4_u8, &d5_u8, &d6_u8, &d7_u8);
-
- store_u8_8x4(d_u8 + 4 * dst8_stride, dst8_stride, d4_u8, d5_u8, d6_u8,
- d7_u8);
-
- s0 = s8;
- s1 = s9;
- s2 = s10;
- s3 = s11;
- s4 = s12;
- s5 = s13;
- s6 = s14;
- s += 8;
- d += 8;
- d_u8 += 8;
- width -= 8;
- } while (width != 0);
- src_ptr += 8 * src_stride;
- dst_ptr += 8 * dst_stride;
- dst8_ptr += 8 * dst8_stride;
- height -= 8;
-#else // !AOM_ARCH_AARCH64
- __builtin_prefetch(src_ptr);
-
- t0 = vld1_u8(src_ptr);
- s0 = vreinterpretq_s16_u16(vmovl_u8(t0)); // a0 a1 a2 a3 a4 a5 a6 a7
-
- __builtin_prefetch(dst_ptr);
-
- s = src_ptr + 8;
-
- do {
- t0 = vld1_u8(s); // a8 a9 a10 a11 a12 a13 a14 a15
- s8 = vreinterpretq_s16_u16(vmovl_u8(t0));
-
- s1 = vextq_s16(s0, s8, 1); // a1 a2 a3 a4 a5 a6 a7 a8
- s2 = vextq_s16(s0, s8, 2); // a2 a3 a4 a5 a6 a7 a8 a9
- s3 = vextq_s16(s0, s8, 3); // a3 a4 a5 a6 a7 a8 a9 a10
- s4 = vextq_s16(s0, s8, 4); // a4 a5 a6 a7 a8 a9 a10 a11
- s5 = vextq_s16(s0, s8, 5); // a5 a6 a7 a8 a9 a10 a11 a12
- s6 = vextq_s16(s0, s8, 6); // a6 a7 a8 a9 a10 a11 a12 a13
- s7 = vextq_s16(s0, s8, 7); // a7 a8 a9 a10 a11 a12 a13 a14
-
- d0 = convolve8_8_x(s0, s1, s2, s3, s4, s5, s6, s7, x_filter,
- round_offset_vec);
-
- dd0 = vld1q_u16(d);
-
- compute_basic_avg_8x1(dd0, d0, round_offset_vec, &d0_u8);
-
- vst1_u8(d_u8, d0_u8);
-
- s0 = s8;
- s += 8;
- d += 8;
- d_u8 += 8;
- width -= 8;
- } while (width != 0);
- src_ptr += src_stride;
- dst_ptr += dst_stride;
- dst8_ptr += dst8_stride;
- height--;
-#endif // AOM_ARCH_AARCH64
- } while (height != 0);
- }
-}
-
-static INLINE void dist_wtd_convolve_x_neon(
- const uint8_t *src, int src_stride, int w, int h,
- const InterpFilterParams *filter_params_x, const int subpel_x_qn,
- ConvolveParams *conv_params) {
- assert(w % 4 == 0);
- assert(h % 4 == 0);
-
- const int bd = 8;
- const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
- const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
- (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
- const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
-
- // Horizontal filter.
- const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
- filter_params_x, subpel_x_qn & SUBPEL_MASK);
- // Filter values are even, so downshift by 1 to reduce intermediate precision
- // requirements.
- const int16x8_t x_filter = vshrq_n_s16(vld1q_s16(x_filter_ptr), 1);
-
- const int horiz_offset = filter_params_x->taps / 2 - 1;
- const uint8_t *src_ptr = src - horiz_offset;
- CONV_BUF_TYPE *dst_ptr = conv_params->dst;
- int dst_stride = conv_params->dst_stride;
- const uint8_t *s;
- CONV_BUF_TYPE *d;
- int width;
- int height = h;
-
- uint8x8_t t0;
-#if AOM_ARCH_AARCH64
- uint8x8_t t1, t2, t3, t4, t5, t6, t7;
-#endif // AOM_ARCH_AARCH64
-
- if (w == 4 || h == 4) {
- int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8;
- uint16x4_t d0;
-#if AOM_ARCH_AARCH64
- int16x4_t s9, s10;
- uint16x4_t d1, d2, d3;
-#endif // AOM_ARCH_AARCH64
-
- do {
- d = dst_ptr;
- width = w;
-
- __builtin_prefetch(src_ptr + 0 * src_stride);
-#if AOM_ARCH_AARCH64
- __builtin_prefetch(src_ptr + 1 * src_stride);
- __builtin_prefetch(src_ptr + 2 * src_stride);
- __builtin_prefetch(src_ptr + 3 * src_stride);
-
- load_u8_8x4(src_ptr, src_stride, &t0, &t1, &t2, &t3);
- transpose_u8_8x4(&t0, &t1, &t2, &t3);
-
- s0 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s1 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
- s2 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
- s3 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t3)));
- s4 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s5 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
- s6 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
-
- __builtin_prefetch(d + 0 * dst_stride);
- __builtin_prefetch(d + 1 * dst_stride);
- __builtin_prefetch(d + 2 * dst_stride);
- __builtin_prefetch(d + 3 * dst_stride);
-
- s = src_ptr + 7;
-
- do {
- load_unaligned_u8_4x4(s, src_stride, &t0, &t1);
- transpose_u8_4x4(&t0, &t1);
-
- s7 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s8 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
- s9 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s10 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
-
- d0 = convolve8_4_x(s0, s1, s2, s3, s4, s5, s6, s7, x_filter,
- vget_low_s16(round_offset_vec));
- d1 = convolve8_4_x(s1, s2, s3, s4, s5, s6, s7, s8, x_filter,
- vget_low_s16(round_offset_vec));
- d2 = convolve8_4_x(s2, s3, s4, s5, s6, s7, s8, s9, x_filter,
- vget_low_s16(round_offset_vec));
- d3 = convolve8_4_x(s3, s4, s5, s6, s7, s8, s9, s10, x_filter,
- vget_low_s16(round_offset_vec));
-
- transpose_u16_4x4d(&d0, &d1, &d2, &d3);
-
- store_u16_4x4(d, dst_stride, d0, d1, d2, d3);
-
- s0 = s4;
- s1 = s5;
- s2 = s6;
- s3 = s7;
- s4 = s8;
- s5 = s9;
- s6 = s10;
- s += 4;
- d += 4;
- width -= 4;
- } while (width != 0);
- src_ptr += 4 * src_stride;
- dst_ptr += 4 * dst_stride;
- height -= 4;
-#else // !AOM_ARCH_AARCH64
- t0 = vld1_u8(src_ptr); // a0 a1 a2 a3 a4 a5 a6 a7
- s0 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
- s4 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
-
- __builtin_prefetch(d);
-
- s = src_ptr + 8;
-
- do {
- t0 = vld1_u8(s); // a8 a9 a10 a11
- s8 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
-
- s1 = vext_s16(s0, s4, 1); // a1 a2 a3 a4
- s2 = vext_s16(s0, s4, 2); // a2 a3 a4 a5
- s3 = vext_s16(s0, s4, 3); // a3 a4 a5 a6
- s5 = vext_s16(s4, s8, 1); // a5 a6 a7 a8
- s6 = vext_s16(s4, s8, 2); // a6 a7 a8 a9
- s7 = vext_s16(s4, s8, 3); // a7 a8 a9 a10
-
- d0 = convolve8_4_x(s0, s1, s2, s3, s4, s5, s6, s7, x_filter,
- vget_low_s16(round_offset_vec));
-
- vst1_u16(d, d0);
-
- s0 = s4;
- s4 = s8;
- s += 4;
- d += 4;
- width -= 4;
- } while (width != 0);
- src_ptr += src_stride;
- dst_ptr += dst_stride;
- height--;
-#endif // AOM_ARCH_AARCH64
- } while (height != 0);
- } else {
- int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8;
- uint16x8_t d0;
-
- do {
- d = dst_ptr;
- width = w;
-
-#if AOM_ARCH_AARCH64
- int16x8_t s9, s10, s11, s12, s13, s14;
- uint16x8_t d1, d2, d3, d4, d5, d6, d7;
-
- __builtin_prefetch(src_ptr + 0 * src_stride);
- __builtin_prefetch(src_ptr + 1 * src_stride);
- __builtin_prefetch(src_ptr + 2 * src_stride);
- __builtin_prefetch(src_ptr + 3 * src_stride);
- __builtin_prefetch(src_ptr + 4 * src_stride);
- __builtin_prefetch(src_ptr + 5 * src_stride);
- __builtin_prefetch(src_ptr + 6 * src_stride);
- __builtin_prefetch(src_ptr + 7 * src_stride);
-
- load_u8_8x8(src_ptr, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
- transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
-
- s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
- s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
- s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
- s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
- s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
- s5 = vreinterpretq_s16_u16(vmovl_u8(t5));
- s6 = vreinterpretq_s16_u16(vmovl_u8(t6));
-
- __builtin_prefetch(dst_ptr + 0 * dst_stride);
- __builtin_prefetch(dst_ptr + 1 * dst_stride);
- __builtin_prefetch(dst_ptr + 2 * dst_stride);
- __builtin_prefetch(dst_ptr + 3 * dst_stride);
- __builtin_prefetch(dst_ptr + 4 * dst_stride);
- __builtin_prefetch(dst_ptr + 5 * dst_stride);
- __builtin_prefetch(dst_ptr + 6 * dst_stride);
- __builtin_prefetch(dst_ptr + 7 * dst_stride);
-
- s = src_ptr + 7;
-
- do {
- load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
- transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
-
- s7 = vreinterpretq_s16_u16(vmovl_u8(t0));
- s8 = vreinterpretq_s16_u16(vmovl_u8(t1));
- s9 = vreinterpretq_s16_u16(vmovl_u8(t2));
- s10 = vreinterpretq_s16_u16(vmovl_u8(t3));
- s11 = vreinterpretq_s16_u16(vmovl_u8(t4));
- s12 = vreinterpretq_s16_u16(vmovl_u8(t5));
- s13 = vreinterpretq_s16_u16(vmovl_u8(t6));
- s14 = vreinterpretq_s16_u16(vmovl_u8(t7));
-
- d0 = convolve8_8_x(s0, s1, s2, s3, s4, s5, s6, s7, x_filter,
- round_offset_vec);
- d1 = convolve8_8_x(s1, s2, s3, s4, s5, s6, s7, s8, x_filter,
- round_offset_vec);
- d2 = convolve8_8_x(s2, s3, s4, s5, s6, s7, s8, s9, x_filter,
- round_offset_vec);
- d3 = convolve8_8_x(s3, s4, s5, s6, s7, s8, s9, s10, x_filter,
- round_offset_vec);
- d4 = convolve8_8_x(s4, s5, s6, s7, s8, s9, s10, s11, x_filter,
- round_offset_vec);
- d5 = convolve8_8_x(s5, s6, s7, s8, s9, s10, s11, s12, x_filter,
- round_offset_vec);
- d6 = convolve8_8_x(s6, s7, s8, s9, s10, s11, s12, s13, x_filter,
- round_offset_vec);
- d7 = convolve8_8_x(s7, s8, s9, s10, s11, s12, s13, s14, x_filter,
- round_offset_vec);
-
- transpose_u16_8x8(&d0, &d1, &d2, &d3, &d4, &d5, &d6, &d7);
-
- store_u16_8x8(d, dst_stride, d0, d1, d2, d3, d4, d5, d6, d7);
-
- s0 = s8;
- s1 = s9;
- s2 = s10;
- s3 = s11;
- s4 = s12;
- s5 = s13;
- s6 = s14;
- s += 8;
- d += 8;
- width -= 8;
- } while (width != 0);
- src_ptr += 8 * src_stride;
- dst_ptr += 8 * dst_stride;
- height -= 8;
-#else // !AOM_ARCH_AARCH64
- __builtin_prefetch(src_ptr);
-
- t0 = vld1_u8(src_ptr);
- s0 = vreinterpretq_s16_u16(vmovl_u8(t0)); // a0 a1 a2 a3 a4 a5 a6 a7
-
- __builtin_prefetch(dst_ptr);
-
- s = src_ptr + 8;
-
- do {
- t0 = vld1_u8(s); // a8 a9 a10 a11 a12 a13 a14 a15
- s8 = vreinterpretq_s16_u16(vmovl_u8(t0));
-
- s1 = vextq_s16(s0, s8, 1); // a1 a2 a3 a4 a5 a6 a7 a8
- s2 = vextq_s16(s0, s8, 2); // a2 a3 a4 a5 a6 a7 a8 a9
- s3 = vextq_s16(s0, s8, 3); // a3 a4 a5 a6 a7 a8 a9 a10
- s4 = vextq_s16(s0, s8, 4); // a4 a5 a6 a7 a8 a9 a10 a11
- s5 = vextq_s16(s0, s8, 5); // a5 a6 a7 a8 a9 a10 a11 a12
- s6 = vextq_s16(s0, s8, 6); // a6 a7 a8 a9 a10 a11 a12 a13
- s7 = vextq_s16(s0, s8, 7); // a7 a8 a9 a10 a11 a12 a13 a14
-
- d0 = convolve8_8_x(s0, s1, s2, s3, s4, s5, s6, s7, x_filter,
- round_offset_vec);
-
- vst1q_u16(d, d0);
-
- s0 = s8;
- s += 8;
- d += 8;
- width -= 8;
- } while (width != 0);
- src_ptr += src_stride;
- dst_ptr += dst_stride;
- height--;
-#endif // AOM_ARCH_AARCH64
- } while (height != 0);
- }
-}
-
-#endif // AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_DOTPROD)
-
-void av1_dist_wtd_convolve_x_neon(const uint8_t *src, int src_stride,
- uint8_t *dst8, int dst8_stride, int w, int h,
- const InterpFilterParams *filter_params_x,
- const int subpel_x_qn,
- ConvolveParams *conv_params) {
- if (conv_params->do_average) {
- if (UNLIKELY(conv_params->use_dist_wtd_comp_avg)) {
- dist_wtd_convolve_x_dist_wtd_avg_neon(src, src_stride, dst8, dst8_stride,
- w, h, filter_params_x, subpel_x_qn,
- conv_params);
- } else {
- dist_wtd_convolve_x_avg_neon(src, src_stride, dst8, dst8_stride, w, h,
- filter_params_x, subpel_x_qn, conv_params);
- }
- } else {
- dist_wtd_convolve_x_neon(src, src_stride, w, h, filter_params_x,
- subpel_x_qn, conv_params);
- }
-}
-
-static INLINE uint16x4_t convolve6_4_y(const int16x4_t s0, const int16x4_t s1,
- const int16x4_t s2, const int16x4_t s3,
- const int16x4_t s4, const int16x4_t s5,
- const int16x8_t y_filter,
- const int16x4_t round_offset) {
- const int16x4_t y_filter_0_3 = vget_low_s16(y_filter);
- const int16x4_t y_filter_4_7 = vget_high_s16(y_filter);
-
- // Filter values at indices 0 and 7 are 0.
- int16x4_t sum = vmul_lane_s16(s0, y_filter_0_3, 1);
- sum = vmla_lane_s16(sum, s1, y_filter_0_3, 2);
- sum = vmla_lane_s16(sum, s2, y_filter_0_3, 3);
- sum = vmla_lane_s16(sum, s3, y_filter_4_7, 0);
- sum = vmla_lane_s16(sum, s4, y_filter_4_7, 1);
- sum = vmla_lane_s16(sum, s5, y_filter_4_7, 2);
-
- // We halved the convolution filter values so -1 from the right shift.
- int16x4_t res = vrsra_n_s16(round_offset, sum, ROUND0_BITS - 1);
- return vreinterpret_u16_s16(res);
-}
-
-static INLINE uint16x8_t convolve6_8_y(const int16x8_t s0, const int16x8_t s1,
- const int16x8_t s2, const int16x8_t s3,
- const int16x8_t s4, const int16x8_t s5,
- const int16x8_t y_filter,
- const int16x8_t round_offset) {
- const int16x4_t y_filter_0_3 = vget_low_s16(y_filter);
- const int16x4_t y_filter_4_7 = vget_high_s16(y_filter);
-
- // Filter values at indices 0 and 7 are 0.
- int16x8_t sum = vmulq_lane_s16(s0, y_filter_0_3, 1);
- sum = vmlaq_lane_s16(sum, s1, y_filter_0_3, 2);
- sum = vmlaq_lane_s16(sum, s2, y_filter_0_3, 3);
- sum = vmlaq_lane_s16(sum, s3, y_filter_4_7, 0);
- sum = vmlaq_lane_s16(sum, s4, y_filter_4_7, 1);
- sum = vmlaq_lane_s16(sum, s5, y_filter_4_7, 2);
-
- // We halved the convolution filter values so -1 from the right shift.
- int16x8_t res = vrsraq_n_s16(round_offset, sum, ROUND0_BITS - 1);
- return vreinterpretq_u16_s16(res);
-}
-
-static INLINE void dist_wtd_convolve_y_6tap_dist_wtd_avg_neon(
- const uint8_t *src_ptr, int src_stride, uint8_t *dst8_ptr,
- const int dst8_stride, int w, int h, const int16x8_t y_filter,
- ConvolveParams *conv_params) {
- const int bd = 8;
- const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
- const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
- (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
- const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
-
- const uint16_t fwd_offset = conv_params->fwd_offset;
- const uint16_t bck_offset = conv_params->bck_offset;
-
- CONV_BUF_TYPE *dst_ptr = conv_params->dst;
- const int dst_stride = conv_params->dst_stride;
- int width = w;
-
- if (w == 4 || h == 4) {
- int16x4_t s0, s1, s2, s3, s4, s5;
- uint16x4_t d0, dd0;
- uint8x8_t t0, t1, t2, t3, t4, d01;
-#if AOM_ARCH_AARCH64
- int16x4_t s6, s7, s8;
- uint16x4_t d1, d2, d3, dd1, dd2, dd3;
- uint8x8_t d23;
-#endif // AOM_ARCH_AARCH64
-
- do {
- const uint8_t *s = src_ptr;
- CONV_BUF_TYPE *d = dst_ptr;
- uint8_t *d_u8 = dst8_ptr;
- int height = h;
-
- t0 = load_unaligned_u8_4x1(s + 0 * src_stride);
- t1 = load_unaligned_u8_4x1(s + 1 * src_stride);
- t2 = load_unaligned_u8_4x1(s + 2 * src_stride);
- t3 = load_unaligned_u8_4x1(s + 3 * src_stride);
- t4 = load_unaligned_u8_4x1(s + 4 * src_stride);
-
- s0 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
- s1 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t1)));
- s2 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t2)));
- s3 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t3)));
- s4 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t4)));
-
- s += 5 * src_stride;
-
- do {
-#if AOM_ARCH_AARCH64
- t0 = load_unaligned_u8_4x1(s + 0 * src_stride);
- t1 = load_unaligned_u8_4x1(s + 1 * src_stride);
- t2 = load_unaligned_u8_4x1(s + 2 * src_stride);
- t3 = load_unaligned_u8_4x1(s + 3 * src_stride);
-
- s5 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
- s6 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t1)));
- s7 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t2)));
- s8 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t3)));
-
- d0 = convolve6_4_y(s0, s1, s2, s3, s4, s5, y_filter,
- vget_low_s16(round_offset_vec));
- d1 = convolve6_4_y(s1, s2, s3, s4, s5, s6, y_filter,
- vget_low_s16(round_offset_vec));
- d2 = convolve6_4_y(s2, s3, s4, s5, s6, s7, y_filter,
- vget_low_s16(round_offset_vec));
- d3 = convolve6_4_y(s3, s4, s5, s6, s7, s8, y_filter,
- vget_low_s16(round_offset_vec));
-
- load_u16_4x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
-
- compute_dist_wtd_avg_4x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset,
- bck_offset, round_offset_vec, &d01, &d23);
-
- store_u8_4x1(d_u8 + 0 * dst8_stride, d01, 0);
- store_u8_4x1(d_u8 + 1 * dst8_stride, d01, 1);
- store_u8_4x1(d_u8 + 2 * dst8_stride, d23, 0);
- store_u8_4x1(d_u8 + 3 * dst8_stride, d23, 1);
-
- s0 = s4;
- s1 = s5;
- s2 = s6;
- s3 = s7;
- s4 = s8;
- s += 4 * src_stride;
- d += 4 * dst_stride;
- d_u8 += 4 * dst8_stride;
- height -= 4;
-#else // !AOM_ARCH_AARCH64
- t0 = load_unaligned_u8_4x1(s);
- s5 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
-
- d0 = convolve6_4_y(s0, s1, s2, s3, s4, s5, y_filter,
- vget_low_s16(round_offset_vec));
-
- dd0 = vld1_u16(d);
-
- compute_dist_wtd_avg_4x1(dd0, d0, fwd_offset, bck_offset,
- vget_low_s16(round_offset_vec), &d01);
-
- store_u8_4x1(d_u8, d01, 0);
-
- s0 = s1;
- s1 = s2;
- s2 = s3;
- s3 = s4;
- s4 = s5;
- s += src_stride;
- d += dst_stride;
- d_u8 += dst8_stride;
- height--;
-#endif // AOM_ARCH_AARCH64
- } while (height != 0);
- src_ptr += 4;
- dst_ptr += 4;
- dst8_ptr += 4;
- width -= 4;
- } while (width != 0);
- } else {
- int16x8_t s0, s1, s2, s3, s4, s5;
- uint16x8_t d0, dd0;
- uint8x8_t d0_u8, t0, t1, t2, t3, t4;
-#if AOM_ARCH_AARCH64
- int16x8_t s6, s7, s8, s9, s10, s11, s12;
- uint16x8_t d1, d2, d3, d4, d5, d6, d7, dd1, dd2, dd3, dd4, dd5, dd6, dd7;
- uint8x8_t d1_u8, d2_u8, d3_u8, d4_u8, d5_u8, d6_u8, d7_u8, t5, t6, t7;
-#endif // AOM_ARCH_AARCH64
-
- do {
- const uint8_t *s = src_ptr + (5 * src_stride);
- CONV_BUF_TYPE *d = dst_ptr;
- uint8_t *d_u8 = dst8_ptr;
- int height = h;
-
- load_u8_8x5(src_ptr, src_stride, &t0, &t1, &t2, &t3, &t4);
-
- s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
- s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
- s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
- s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
- s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
-
- do {
-#if AOM_ARCH_AARCH64
- load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
-
- s5 = vreinterpretq_s16_u16(vmovl_u8(t0));
- s6 = vreinterpretq_s16_u16(vmovl_u8(t1));
- s7 = vreinterpretq_s16_u16(vmovl_u8(t2));
- s8 = vreinterpretq_s16_u16(vmovl_u8(t3));
- s9 = vreinterpretq_s16_u16(vmovl_u8(t4));
- s10 = vreinterpretq_s16_u16(vmovl_u8(t5));
- s11 = vreinterpretq_s16_u16(vmovl_u8(t6));
- s12 = vreinterpretq_s16_u16(vmovl_u8(t7));
-
- d0 = convolve6_8_y(s0, s1, s2, s3, s4, s5, y_filter, round_offset_vec);
- d1 = convolve6_8_y(s1, s2, s3, s4, s5, s6, y_filter, round_offset_vec);
- d2 = convolve6_8_y(s2, s3, s4, s5, s6, s7, y_filter, round_offset_vec);
- d3 = convolve6_8_y(s3, s4, s5, s6, s7, s8, y_filter, round_offset_vec);
- d4 = convolve6_8_y(s4, s5, s6, s7, s8, s9, y_filter, round_offset_vec);
- d5 = convolve6_8_y(s5, s6, s7, s8, s9, s10, y_filter, round_offset_vec);
- d6 =
- convolve6_8_y(s6, s7, s8, s9, s10, s11, y_filter, round_offset_vec);
- d7 = convolve6_8_y(s7, s8, s9, s10, s11, s12, y_filter,
- round_offset_vec);
-
- load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
-
- compute_dist_wtd_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset,
- bck_offset, round_offset_vec, &d0_u8, &d1_u8,
- &d2_u8, &d3_u8);
-
- store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8);
- d_u8 += 4 * dst8_stride;
-
- load_u16_8x4(d + 4 * dst_stride, dst_stride, &dd4, &dd5, &dd6, &dd7);
-
- compute_dist_wtd_avg_8x4(dd4, dd5, dd6, dd7, d4, d5, d6, d7, fwd_offset,
- bck_offset, round_offset_vec, &d4_u8, &d5_u8,
- &d6_u8, &d7_u8);
-
- store_u8_8x4(d_u8, dst8_stride, d4_u8, d5_u8, d6_u8, d7_u8);
- d_u8 += 4 * dst8_stride;
-
- s0 = s8;
- s1 = s9;
- s2 = s10;
- s3 = s11;
- s4 = s12;
- s += 8 * src_stride;
- d += 8 * dst_stride;
- height -= 8;
-#else // !AOM_ARCH_AARCH64
- s5 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
-
- d0 = convolve6_8_y(s0, s1, s2, s3, s4, s5, y_filter, round_offset_vec);
-
- s0 = s1;
- s1 = s2;
- s2 = s3;
- s3 = s4;
- s4 = s5;
-
- dd0 = vld1q_u16(d);
-
- compute_dist_wtd_avg_8x1(dd0, d0, fwd_offset, bck_offset,
- round_offset_vec, &d0_u8);
-
- vst1_u8(d_u8, d0_u8);
- d_u8 += dst8_stride;
-
- s += src_stride;
- d += dst_stride;
- height--;
-#endif // AOM_ARCH_AARCH64
- } while (height != 0);
- src_ptr += 8;
- dst_ptr += 8;
- dst8_ptr += 8;
- width -= 8;
- } while (width != 0);
- }
-}
-
-static INLINE void dist_wtd_convolve_y_6tap_avg_neon(
- const uint8_t *src_ptr, int src_stride, uint8_t *dst8_ptr,
- const int dst8_stride, int w, int h, const int16x8_t y_filter,
- ConvolveParams *conv_params) {
- const int bd = 8;
- const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
- const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
- (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
- const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
-
- CONV_BUF_TYPE *dst_ptr = conv_params->dst;
- const int dst_stride = conv_params->dst_stride;
- int width = w;
-
- if (w == 4 || h == 4) {
- int16x4_t s0, s1, s2, s3, s4, s5;
- uint16x4_t d0, dd0;
- uint8x8_t t0, t1, t2, t3, t4, d01;
-#if AOM_ARCH_AARCH64
- int16x4_t s6, s7, s8;
- uint16x4_t d1, d2, d3, dd1, dd2, dd3;
- uint8x8_t d23;
-#endif // AOM_ARCH_AARCH64
-
- do {
- const uint8_t *s = src_ptr;
- CONV_BUF_TYPE *d = dst_ptr;
- uint8_t *d_u8 = dst8_ptr;
- int height = h;
-
- t0 = load_unaligned_u8_4x1(s + 0 * src_stride);
- t1 = load_unaligned_u8_4x1(s + 1 * src_stride);
- t2 = load_unaligned_u8_4x1(s + 2 * src_stride);
- t3 = load_unaligned_u8_4x1(s + 3 * src_stride);
- t4 = load_unaligned_u8_4x1(s + 4 * src_stride);
-
- s0 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
- s1 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t1)));
- s2 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t2)));
- s3 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t3)));
- s4 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t4)));
-
- s += 5 * src_stride;
-
- do {
-#if AOM_ARCH_AARCH64
- t0 = load_unaligned_u8_4x1(s + 0 * src_stride);
- t1 = load_unaligned_u8_4x1(s + 1 * src_stride);
- t2 = load_unaligned_u8_4x1(s + 2 * src_stride);
- t3 = load_unaligned_u8_4x1(s + 3 * src_stride);
-
- s5 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
- s6 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t1)));
- s7 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t2)));
- s8 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t3)));
-
- d0 = convolve6_4_y(s0, s1, s2, s3, s4, s5, y_filter,
- vget_low_s16(round_offset_vec));
- d1 = convolve6_4_y(s1, s2, s3, s4, s5, s6, y_filter,
- vget_low_s16(round_offset_vec));
- d2 = convolve6_4_y(s2, s3, s4, s5, s6, s7, y_filter,
- vget_low_s16(round_offset_vec));
- d3 = convolve6_4_y(s3, s4, s5, s6, s7, s8, y_filter,
- vget_low_s16(round_offset_vec));
-
- load_u16_4x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
-
- compute_basic_avg_4x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3,
- round_offset_vec, &d01, &d23);
-
- store_u8_4x1(d_u8 + 0 * dst8_stride, d01, 0);
- store_u8_4x1(d_u8 + 1 * dst8_stride, d01, 1);
- store_u8_4x1(d_u8 + 2 * dst8_stride, d23, 0);
- store_u8_4x1(d_u8 + 3 * dst8_stride, d23, 1);
-
- s0 = s4;
- s1 = s5;
- s2 = s6;
- s3 = s7;
- s4 = s8;
- s += 4 * src_stride;
- d += 4 * dst_stride;
- d_u8 += 4 * dst8_stride;
- height -= 4;
-#else // !AOM_ARCH_AARCH64
- t0 = load_unaligned_u8_4x1(s);
- s5 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
-
- d0 = convolve6_4_y(s0, s1, s2, s3, s4, s5, y_filter,
- vget_low_s16(round_offset_vec));
-
- dd0 = vld1_u16(d);
-
- compute_basic_avg_4x1(dd0, d0, vget_low_s16(round_offset_vec), &d01);
-
- store_u8_4x1(d_u8, d01, 0);
-
- s0 = s1;
- s1 = s2;
- s2 = s3;
- s3 = s4;
- s4 = s5;
- s += src_stride;
- d += dst_stride;
- d_u8 += dst8_stride;
- height--;
-#endif // AOM_ARCH_AARCH64
- } while (height != 0);
- src_ptr += 4;
- dst_ptr += 4;
- dst8_ptr += 4;
- width -= 4;
- } while (width != 0);
- } else {
- int16x8_t s0, s1, s2, s3, s4, s5;
- uint16x8_t d0, dd0;
- uint8x8_t d0_u8, t0, t1, t2, t3, t4;
-#if AOM_ARCH_AARCH64
- int16x8_t s6, s7, s8, s9, s10, s11, s12;
- uint16x8_t d1, d2, d3, d4, d5, d6, d7, dd1, dd2, dd3, dd4, dd5, dd6, dd7;
- uint8x8_t d1_u8, d2_u8, d3_u8, d4_u8, d5_u8, d6_u8, d7_u8, t5, t6, t7;
-#endif // AOM_ARCH_AARCH64
-
- do {
- const uint8_t *s = src_ptr + (5 * src_stride);
- CONV_BUF_TYPE *d = dst_ptr;
- uint8_t *d_u8 = dst8_ptr;
- int height = h;
-
- load_u8_8x5(src_ptr, src_stride, &t0, &t1, &t2, &t3, &t4);
-
- s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
- s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
- s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
- s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
- s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
-
- do {
-#if AOM_ARCH_AARCH64
- load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
-
- s5 = vreinterpretq_s16_u16(vmovl_u8(t0));
- s6 = vreinterpretq_s16_u16(vmovl_u8(t1));
- s7 = vreinterpretq_s16_u16(vmovl_u8(t2));
- s8 = vreinterpretq_s16_u16(vmovl_u8(t3));
- s9 = vreinterpretq_s16_u16(vmovl_u8(t4));
- s10 = vreinterpretq_s16_u16(vmovl_u8(t5));
- s11 = vreinterpretq_s16_u16(vmovl_u8(t6));
- s12 = vreinterpretq_s16_u16(vmovl_u8(t7));
-
- d0 = convolve6_8_y(s0, s1, s2, s3, s4, s5, y_filter, round_offset_vec);
- d1 = convolve6_8_y(s1, s2, s3, s4, s5, s6, y_filter, round_offset_vec);
- d2 = convolve6_8_y(s2, s3, s4, s5, s6, s7, y_filter, round_offset_vec);
- d3 = convolve6_8_y(s3, s4, s5, s6, s7, s8, y_filter, round_offset_vec);
- d4 = convolve6_8_y(s4, s5, s6, s7, s8, s9, y_filter, round_offset_vec);
- d5 = convolve6_8_y(s5, s6, s7, s8, s9, s10, y_filter, round_offset_vec);
- d6 =
- convolve6_8_y(s6, s7, s8, s9, s10, s11, y_filter, round_offset_vec);
- d7 = convolve6_8_y(s7, s8, s9, s10, s11, s12, y_filter,
- round_offset_vec);
-
- load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
-
- compute_basic_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3,
- round_offset_vec, &d0_u8, &d1_u8, &d2_u8, &d3_u8);
-
- store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8);
- d_u8 += 4 * dst8_stride;
-
- load_u16_8x4(d + 4 * dst_stride, dst_stride, &dd4, &dd5, &dd6, &dd7);
-
- compute_basic_avg_8x4(dd4, dd5, dd6, dd7, d4, d5, d6, d7,
- round_offset_vec, &d4_u8, &d5_u8, &d6_u8, &d7_u8);
-
- store_u8_8x4(d_u8, dst8_stride, d4_u8, d5_u8, d6_u8, d7_u8);
- d_u8 += 4 * dst8_stride;
-
- s0 = s8;
- s1 = s9;
- s2 = s10;
- s3 = s11;
- s4 = s12;
- s += 8 * src_stride;
- d += 8 * dst_stride;
- height -= 8;
-#else // !AOM_ARCH_AARCH64
- s5 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
-
- d0 = convolve6_8_y(s0, s1, s2, s3, s4, s5, y_filter, round_offset_vec);
-
- s0 = s1;
- s1 = s2;
- s2 = s3;
- s3 = s4;
- s4 = s5;
-
- dd0 = vld1q_u16(d);
-
- compute_basic_avg_8x1(dd0, d0, round_offset_vec, &d0_u8);
-
- vst1_u8(d_u8, d0_u8);
- d_u8 += dst8_stride;
-
- s += src_stride;
- d += dst_stride;
- height--;
-#endif // AOM_ARCH_AARCH64
- } while (height != 0);
- src_ptr += 8;
- dst_ptr += 8;
- dst8_ptr += 8;
- width -= 8;
- } while (width != 0);
- }
-}
-
-static INLINE void dist_wtd_convolve_y_6tap_neon(const uint8_t *src_ptr,
- int src_stride, int w, int h,
- const int16x8_t y_filter,
- ConvolveParams *conv_params) {
- const int bd = 8;
- const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
- const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
- (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
- const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
-
- CONV_BUF_TYPE *dst_ptr = conv_params->dst;
- const int dst_stride = conv_params->dst_stride;
- int width = w;
-
- if (w == 4 || h == 4) {
- int16x4_t s0, s1, s2, s3, s4, s5;
- uint16x4_t d0;
- uint8x8_t t0, t1, t2, t3, t4;
-#if AOM_ARCH_AARCH64
- int16x4_t s6, s7, s8;
- uint16x4_t d1, d2, d3;
-#endif // AOM_ARCH_AARCH64
-
- do {
- const uint8_t *s = src_ptr;
- CONV_BUF_TYPE *d = dst_ptr;
- int height = h;
-
- t0 = load_unaligned_u8_4x1(s + 0 * src_stride);
- t1 = load_unaligned_u8_4x1(s + 1 * src_stride);
- t2 = load_unaligned_u8_4x1(s + 2 * src_stride);
- t3 = load_unaligned_u8_4x1(s + 3 * src_stride);
- t4 = load_unaligned_u8_4x1(s + 4 * src_stride);
-
- s0 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
- s1 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t1)));
- s2 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t2)));
- s3 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t3)));
- s4 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t4)));
-
- s += 5 * src_stride;
-
- do {
-#if AOM_ARCH_AARCH64
- t0 = load_unaligned_u8_4x1(s + 0 * src_stride);
- t1 = load_unaligned_u8_4x1(s + 1 * src_stride);
- t2 = load_unaligned_u8_4x1(s + 2 * src_stride);
- t3 = load_unaligned_u8_4x1(s + 3 * src_stride);
-
- s5 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
- s6 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t1)));
- s7 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t2)));
- s8 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t3)));
-
- d0 = convolve6_4_y(s0, s1, s2, s3, s4, s5, y_filter,
- vget_low_s16(round_offset_vec));
- d1 = convolve6_4_y(s1, s2, s3, s4, s5, s6, y_filter,
- vget_low_s16(round_offset_vec));
- d2 = convolve6_4_y(s2, s3, s4, s5, s6, s7, y_filter,
- vget_low_s16(round_offset_vec));
- d3 = convolve6_4_y(s3, s4, s5, s6, s7, s8, y_filter,
- vget_low_s16(round_offset_vec));
-
- store_u16_4x4(d, dst_stride, d0, d1, d2, d3);
-
- s0 = s4;
- s1 = s5;
- s2 = s6;
- s3 = s7;
- s4 = s8;
- s += 4 * src_stride;
- d += 4 * dst_stride;
- height -= 4;
-#else // !AOM_ARCH_AARCH64
- t0 = load_unaligned_u8_4x1(s);
- s5 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
-
- d0 = convolve6_4_y(s0, s1, s2, s3, s4, s5, y_filter,
- vget_low_s16(round_offset_vec));
-
- vst1_u16(d, d0);
-
- s0 = s1;
- s1 = s2;
- s2 = s3;
- s3 = s4;
- s4 = s5;
- s += src_stride;
- d += dst_stride;
- height--;
-#endif // AOM_ARCH_AARCH64
- } while (height != 0);
- src_ptr += 4;
- dst_ptr += 4;
- width -= 4;
- } while (width != 0);
- } else {
- int16x8_t s0, s1, s2, s3, s4, s5;
- uint16x8_t d0;
- uint8x8_t t0, t1, t2, t3, t4;
-#if AOM_ARCH_AARCH64
- int16x8_t s6, s7, s8, s9, s10, s11, s12;
- uint16x8_t d1, d2, d3, d4, d5, d6, d7;
- uint8x8_t t5, t6, t7;
-#endif // AOM_ARCH_AARCH64
-
- do {
- const uint8_t *s = src_ptr + (5 * src_stride);
- CONV_BUF_TYPE *d = dst_ptr;
- int height = h;
-
- load_u8_8x5(src_ptr, src_stride, &t0, &t1, &t2, &t3, &t4);
-
- s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
- s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
- s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
- s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
- s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
-
- do {
-#if AOM_ARCH_AARCH64
- load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
-
- s5 = vreinterpretq_s16_u16(vmovl_u8(t0));
- s6 = vreinterpretq_s16_u16(vmovl_u8(t1));
- s7 = vreinterpretq_s16_u16(vmovl_u8(t2));
- s8 = vreinterpretq_s16_u16(vmovl_u8(t3));
- s9 = vreinterpretq_s16_u16(vmovl_u8(t4));
- s10 = vreinterpretq_s16_u16(vmovl_u8(t5));
- s11 = vreinterpretq_s16_u16(vmovl_u8(t6));
- s12 = vreinterpretq_s16_u16(vmovl_u8(t7));
-
- d0 = convolve6_8_y(s0, s1, s2, s3, s4, s5, y_filter, round_offset_vec);
- d1 = convolve6_8_y(s1, s2, s3, s4, s5, s6, y_filter, round_offset_vec);
- d2 = convolve6_8_y(s2, s3, s4, s5, s6, s7, y_filter, round_offset_vec);
- d3 = convolve6_8_y(s3, s4, s5, s6, s7, s8, y_filter, round_offset_vec);
- d4 = convolve6_8_y(s4, s5, s6, s7, s8, s9, y_filter, round_offset_vec);
- d5 = convolve6_8_y(s5, s6, s7, s8, s9, s10, y_filter, round_offset_vec);
- d6 =
- convolve6_8_y(s6, s7, s8, s9, s10, s11, y_filter, round_offset_vec);
- d7 = convolve6_8_y(s7, s8, s9, s10, s11, s12, y_filter,
- round_offset_vec);
-
- store_u16_8x8(d, dst_stride, d0, d1, d2, d3, d4, d5, d6, d7);
-
- s0 = s8;
- s1 = s9;
- s2 = s10;
- s3 = s11;
- s4 = s12;
- s += 8 * src_stride;
- d += 8 * dst_stride;
- height -= 8;
-#else // !AOM_ARCH_AARCH64
- s5 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
-
- d0 = convolve6_8_y(s0, s1, s2, s3, s4, s5, y_filter, round_offset_vec);
-
- s0 = s1;
- s1 = s2;
- s2 = s3;
- s3 = s4;
- s4 = s5;
-
- vst1q_u16(d, d0);
-
- s += src_stride;
- d += dst_stride;
- height--;
-#endif // AOM_ARCH_AARCH64
- } while (height != 0);
- src_ptr += 8;
- dst_ptr += 8;
- width -= 8;
- } while (width != 0);
- }
-}
-
-static INLINE uint16x4_t convolve8_4_y(const int16x4_t s0, const int16x4_t s1,
- const int16x4_t s2, const int16x4_t s3,
- const int16x4_t s4, const int16x4_t s5,
- const int16x4_t s6, const int16x4_t s7,
- const int16x8_t y_filter,
- const int16x4_t round_offset) {
- const int16x4_t y_filter_0_3 = vget_low_s16(y_filter);
- const int16x4_t y_filter_4_7 = vget_high_s16(y_filter);
-
- int16x4_t sum = vmul_lane_s16(s0, y_filter_0_3, 0);
- sum = vmla_lane_s16(sum, s1, y_filter_0_3, 1);
- sum = vmla_lane_s16(sum, s2, y_filter_0_3, 2);
- sum = vmla_lane_s16(sum, s3, y_filter_0_3, 3);
- sum = vmla_lane_s16(sum, s4, y_filter_4_7, 0);
- sum = vmla_lane_s16(sum, s5, y_filter_4_7, 1);
- sum = vmla_lane_s16(sum, s6, y_filter_4_7, 2);
- sum = vmla_lane_s16(sum, s7, y_filter_4_7, 3);
-
- // We halved the convolution filter values so -1 from the right shift.
- int16x4_t res = vrsra_n_s16(round_offset, sum, ROUND0_BITS - 1);
- return vreinterpret_u16_s16(res);
-}
-
-static INLINE uint16x8_t convolve8_8_y(const int16x8_t s0, const int16x8_t s1,
- const int16x8_t s2, const int16x8_t s3,
- const int16x8_t s4, const int16x8_t s5,
- const int16x8_t s6, const int16x8_t s7,
- const int16x8_t y_filter,
- const int16x8_t round_offset) {
- const int16x4_t y_filter_0_3 = vget_low_s16(y_filter);
- const int16x4_t y_filter_4_7 = vget_high_s16(y_filter);
-
- int16x8_t sum = vmulq_lane_s16(s0, y_filter_0_3, 0);
- sum = vmlaq_lane_s16(sum, s1, y_filter_0_3, 1);
- sum = vmlaq_lane_s16(sum, s2, y_filter_0_3, 2);
- sum = vmlaq_lane_s16(sum, s3, y_filter_0_3, 3);
- sum = vmlaq_lane_s16(sum, s4, y_filter_4_7, 0);
- sum = vmlaq_lane_s16(sum, s5, y_filter_4_7, 1);
- sum = vmlaq_lane_s16(sum, s6, y_filter_4_7, 2);
- sum = vmlaq_lane_s16(sum, s7, y_filter_4_7, 3);
-
- // We halved the convolution filter values so -1 from the right shift.
- int16x8_t res = vrsraq_n_s16(round_offset, sum, ROUND0_BITS - 1);
- return vreinterpretq_u16_s16(res);
-}
-
-static INLINE void dist_wtd_convolve_y_8tap_dist_wtd_avg_neon(
- const uint8_t *src_ptr, int src_stride, uint8_t *dst8_ptr,
- const int dst8_stride, int w, int h, const int16x8_t y_filter,
- ConvolveParams *conv_params) {
- const int bd = 8;
- const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
- const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
- (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
- const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
-
- const uint16_t fwd_offset = conv_params->fwd_offset;
- const uint16_t bck_offset = conv_params->bck_offset;
-
- CONV_BUF_TYPE *dst_ptr = conv_params->dst;
- const int dst_stride = conv_params->dst_stride;
- int width = w;
-
- if (w == 4 || h == 4) {
- int16x4_t s0, s1, s2, s3, s4, s5, s6, s7;
- uint16x4_t d0, dd0;
- uint8x8_t t0, t1, t2, t3, t4, t5, t6, d01;
-#if AOM_ARCH_AARCH64
- int16x4_t s8, s9, s10;
- uint16x4_t d1, d2, d3, dd1, dd2, dd3;
- uint8x8_t d23;
-#endif // AOM_ARCH_AARCH64
-
- do {
- const uint8_t *s = src_ptr;
- CONV_BUF_TYPE *d = dst_ptr;
- uint8_t *d_u8 = dst8_ptr;
- int height = h;
-
- __builtin_prefetch(s + 0 * src_stride);
- __builtin_prefetch(s + 1 * src_stride);
- __builtin_prefetch(s + 2 * src_stride);
- __builtin_prefetch(s + 3 * src_stride);
-
- t0 = load_unaligned_u8_4x1(s + 0 * src_stride);
- t1 = load_unaligned_u8_4x1(s + 1 * src_stride);
- t2 = load_unaligned_u8_4x1(s + 2 * src_stride);
- t3 = load_unaligned_u8_4x1(s + 3 * src_stride);
- t4 = load_unaligned_u8_4x1(s + 4 * src_stride);
- t5 = load_unaligned_u8_4x1(s + 5 * src_stride);
- t6 = load_unaligned_u8_4x1(s + 6 * src_stride);
-
- s0 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
- s1 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t1)));
- s2 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t2)));
- s3 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t3)));
- s4 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t4)));
- s5 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t5)));
- s6 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t6)));
-
- __builtin_prefetch(d + 0 * dst_stride);
- __builtin_prefetch(d + 1 * dst_stride);
- __builtin_prefetch(d + 2 * dst_stride);
- __builtin_prefetch(d + 3 * dst_stride);
-
- s += 7 * src_stride;
-
- do {
-#if AOM_ARCH_AARCH64
- t0 = load_unaligned_u8_4x1(s + 0 * src_stride);
- t1 = load_unaligned_u8_4x1(s + 1 * src_stride);
- t2 = load_unaligned_u8_4x1(s + 2 * src_stride);
- t3 = load_unaligned_u8_4x1(s + 3 * src_stride);
-
- s7 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
- s8 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t1)));
- s9 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t2)));
- s10 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t3)));
-
- d0 = convolve8_4_y(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
- vget_low_s16(round_offset_vec));
- d1 = convolve8_4_y(s1, s2, s3, s4, s5, s6, s7, s8, y_filter,
- vget_low_s16(round_offset_vec));
- d2 = convolve8_4_y(s2, s3, s4, s5, s6, s7, s8, s9, y_filter,
- vget_low_s16(round_offset_vec));
- d3 = convolve8_4_y(s3, s4, s5, s6, s7, s8, s9, s10, y_filter,
- vget_low_s16(round_offset_vec));
-
- __builtin_prefetch(d + 0 * dst_stride);
- __builtin_prefetch(d + 1 * dst_stride);
- __builtin_prefetch(d + 2 * dst_stride);
- __builtin_prefetch(d + 3 * dst_stride);
-
- __builtin_prefetch(d_u8 + 0 * dst8_stride);
- __builtin_prefetch(d_u8 + 1 * dst8_stride);
- __builtin_prefetch(d_u8 + 2 * dst8_stride);
- __builtin_prefetch(d_u8 + 3 * dst8_stride);
-
- load_u16_4x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
-
- compute_dist_wtd_avg_4x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset,
- bck_offset, round_offset_vec, &d01, &d23);
-
- store_u8_4x1(d_u8 + 0 * dst8_stride, d01, 0);
- store_u8_4x1(d_u8 + 1 * dst8_stride, d01, 1);
- store_u8_4x1(d_u8 + 2 * dst8_stride, d23, 0);
- store_u8_4x1(d_u8 + 3 * dst8_stride, d23, 1);
-
- s0 = s4;
- s1 = s5;
- s2 = s6;
- s3 = s7;
- s4 = s8;
- s5 = s9;
- s6 = s10;
- s += 4 * src_stride;
- d += 4 * dst_stride;
- d_u8 += 4 * dst8_stride;
- height -= 4;
-#else // !AOM_ARCH_AARCH64
- t0 = load_unaligned_u8_4x1(s);
- s7 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
-
- d0 = convolve8_4_y(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
- vget_low_s16(round_offset_vec));
-
- __builtin_prefetch(d);
-
- dd0 = vld1_u16(d);
-
- compute_dist_wtd_avg_4x1(dd0, d0, fwd_offset, bck_offset,
- vget_low_s16(round_offset_vec), &d01);
-
- store_u8_4x1(d_u8, d01, 0);
-
- s0 = s1;
- s1 = s2;
- s2 = s3;
- s3 = s4;
- s4 = s5;
- s5 = s6;
- s6 = s7;
- s += src_stride;
- d += dst_stride;
- d_u8 += dst8_stride;
- height--;
-#endif // AOM_ARCH_AARCH64
- } while (height != 0);
- src_ptr += 4;
- dst_ptr += 4;
- dst8_ptr += 4;
- width -= 4;
- } while (width != 0);
- } else {
- int16x8_t s0, s1, s2, s3, s4, s5, s6, s7;
- uint16x8_t d0, dd0;
- uint8x8_t d0_u8, t0, t1, t2, t3, t4, t5, t6;
-#if AOM_ARCH_AARCH64
- int16x8_t s8, s9, s10, s11, s12, s13, s14;
- uint16x8_t d1, d2, d3, d4, d5, d6, d7, dd1, dd2, dd3, dd4, dd5, dd6, dd7;
- uint8x8_t d1_u8, d2_u8, d3_u8, d4_u8, d5_u8, d6_u8, d7_u8, t7;
-#endif // AOM_ARCH_AARCH64
-
- do {
- const uint8_t *s = src_ptr;
- CONV_BUF_TYPE *d = dst_ptr;
- uint8_t *d_u8 = dst8_ptr;
- int height = h;
-
- __builtin_prefetch(s + 0 * src_stride);
- __builtin_prefetch(s + 1 * src_stride);
- __builtin_prefetch(s + 2 * src_stride);
- __builtin_prefetch(s + 3 * src_stride);
- __builtin_prefetch(s + 4 * src_stride);
- __builtin_prefetch(s + 5 * src_stride);
- __builtin_prefetch(s + 6 * src_stride);
- __builtin_prefetch(s + 7 * src_stride);
- load_u8_8x7(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6);
-
- s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
- s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
- s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
- s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
- s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
- s5 = vreinterpretq_s16_u16(vmovl_u8(t5));
- s6 = vreinterpretq_s16_u16(vmovl_u8(t6));
-
- s += 7 * src_stride;
-
- do {
-#if AOM_ARCH_AARCH64
- load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
-
- s7 = vreinterpretq_s16_u16(vmovl_u8(t0));
- s8 = vreinterpretq_s16_u16(vmovl_u8(t1));
- s9 = vreinterpretq_s16_u16(vmovl_u8(t2));
- s10 = vreinterpretq_s16_u16(vmovl_u8(t3));
- s11 = vreinterpretq_s16_u16(vmovl_u8(t4));
- s12 = vreinterpretq_s16_u16(vmovl_u8(t5));
- s13 = vreinterpretq_s16_u16(vmovl_u8(t6));
- s14 = vreinterpretq_s16_u16(vmovl_u8(t7));
-
- __builtin_prefetch(dst_ptr + 0 * dst_stride);
- __builtin_prefetch(dst_ptr + 1 * dst_stride);
- __builtin_prefetch(dst_ptr + 2 * dst_stride);
- __builtin_prefetch(dst_ptr + 3 * dst_stride);
-
- d0 = convolve8_8_y(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
- round_offset_vec);
- d1 = convolve8_8_y(s1, s2, s3, s4, s5, s6, s7, s8, y_filter,
- round_offset_vec);
- d2 = convolve8_8_y(s2, s3, s4, s5, s6, s7, s8, s9, y_filter,
- round_offset_vec);
- d3 = convolve8_8_y(s3, s4, s5, s6, s7, s8, s9, s10, y_filter,
- round_offset_vec);
- d4 = convolve8_8_y(s4, s5, s6, s7, s8, s9, s10, s11, y_filter,
- round_offset_vec);
- d5 = convolve8_8_y(s5, s6, s7, s8, s9, s10, s11, s12, y_filter,
- round_offset_vec);
- d6 = convolve8_8_y(s6, s7, s8, s9, s10, s11, s12, s13, y_filter,
- round_offset_vec);
- d7 = convolve8_8_y(s7, s8, s9, s10, s11, s12, s13, s14, y_filter,
- round_offset_vec);
-
- __builtin_prefetch(d + 0 * dst8_stride);
- __builtin_prefetch(d + 1 * dst8_stride);
- __builtin_prefetch(d + 2 * dst8_stride);
- __builtin_prefetch(d + 3 * dst8_stride);
-
- load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
-
- compute_dist_wtd_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset,
- bck_offset, round_offset_vec, &d0_u8, &d1_u8,
- &d2_u8, &d3_u8);
-
- store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8);
- d_u8 += 4 * dst8_stride;
-
- load_u16_8x4(d + 4 * dst_stride, dst_stride, &dd4, &dd5, &dd6, &dd7);
-
- compute_dist_wtd_avg_8x4(dd4, dd5, dd6, dd7, d4, d5, d6, d7, fwd_offset,
- bck_offset, round_offset_vec, &d4_u8, &d5_u8,
- &d6_u8, &d7_u8);
-
- store_u8_8x4(d_u8, dst8_stride, d4_u8, d5_u8, d6_u8, d7_u8);
- d_u8 += 4 * dst8_stride;
-
- s0 = s8;
- s1 = s9;
- s2 = s10;
- s3 = s11;
- s4 = s12;
- s5 = s13;
- s6 = s14;
- s += 8 * src_stride;
- d += 8 * dst_stride;
- height -= 8;
-#else // !AOM_ARCH_AARCH64
- s7 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
-
- __builtin_prefetch(dst_ptr);
-
- d0 = convolve8_8_y(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
- round_offset_vec);
-
- s0 = s1;
- s1 = s2;
- s2 = s3;
- s3 = s4;
- s4 = s5;
- s5 = s6;
- s6 = s7;
-
- __builtin_prefetch(d);
-
- dd0 = vld1q_u16(d);
-
- compute_dist_wtd_avg_8x1(dd0, d0, fwd_offset, bck_offset,
- round_offset_vec, &d0_u8);
-
- vst1_u8(d_u8, d0_u8);
- d_u8 += dst8_stride;
-
- s += src_stride;
- d += dst_stride;
- height--;
-#endif // AOM_ARCH_AARCH64
- } while (height != 0);
- src_ptr += 8;
- dst_ptr += 8;
- dst8_ptr += 8;
- width -= 8;
- } while (width != 0);
- }
-}
-
-static INLINE void dist_wtd_convolve_y_8tap_avg_neon(
- const uint8_t *src_ptr, int src_stride, uint8_t *dst8_ptr,
- const int dst8_stride, int w, int h, const int16x8_t y_filter,
- ConvolveParams *conv_params) {
- const int bd = 8;
- const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
- const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
- (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
- const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
-
- CONV_BUF_TYPE *dst_ptr = conv_params->dst;
- const int dst_stride = conv_params->dst_stride;
- int width = w;
-
- if (w == 4 || h == 4) {
- int16x4_t s0, s1, s2, s3, s4, s5, s6, s7;
- uint16x4_t d0, dd0;
- uint8x8_t t0, t1, t2, t3, t4, t5, t6, d01;
-#if AOM_ARCH_AARCH64
- int16x4_t s8, s9, s10;
- uint16x4_t d1, d2, d3, dd1, dd2, dd3;
- uint8x8_t d23;
-#endif // AOM_ARCH_AARCH64
-
- do {
- const uint8_t *s = src_ptr;
- CONV_BUF_TYPE *d = dst_ptr;
- uint8_t *d_u8 = dst8_ptr;
- int height = h;
-
- __builtin_prefetch(s + 0 * src_stride);
- __builtin_prefetch(s + 1 * src_stride);
- __builtin_prefetch(s + 2 * src_stride);
- __builtin_prefetch(s + 3 * src_stride);
-
- t0 = load_unaligned_u8_4x1(s + 0 * src_stride);
- t1 = load_unaligned_u8_4x1(s + 1 * src_stride);
- t2 = load_unaligned_u8_4x1(s + 2 * src_stride);
- t3 = load_unaligned_u8_4x1(s + 3 * src_stride);
- t4 = load_unaligned_u8_4x1(s + 4 * src_stride);
- t5 = load_unaligned_u8_4x1(s + 5 * src_stride);
- t6 = load_unaligned_u8_4x1(s + 6 * src_stride);
-
- s0 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
- s1 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t1)));
- s2 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t2)));
- s3 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t3)));
- s4 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t4)));
- s5 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t5)));
- s6 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t6)));
-
- __builtin_prefetch(d + 0 * dst_stride);
- __builtin_prefetch(d + 1 * dst_stride);
- __builtin_prefetch(d + 2 * dst_stride);
- __builtin_prefetch(d + 3 * dst_stride);
-
- s += 7 * src_stride;
-
- do {
-#if AOM_ARCH_AARCH64
- t0 = load_unaligned_u8_4x1(s + 0 * src_stride);
- t1 = load_unaligned_u8_4x1(s + 1 * src_stride);
- t2 = load_unaligned_u8_4x1(s + 2 * src_stride);
- t3 = load_unaligned_u8_4x1(s + 3 * src_stride);
-
- s7 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
- s8 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t1)));
- s9 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t2)));
- s10 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t3)));
-
- d0 = convolve8_4_y(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
- vget_low_s16(round_offset_vec));
- d1 = convolve8_4_y(s1, s2, s3, s4, s5, s6, s7, s8, y_filter,
- vget_low_s16(round_offset_vec));
- d2 = convolve8_4_y(s2, s3, s4, s5, s6, s7, s8, s9, y_filter,
- vget_low_s16(round_offset_vec));
- d3 = convolve8_4_y(s3, s4, s5, s6, s7, s8, s9, s10, y_filter,
- vget_low_s16(round_offset_vec));
-
- __builtin_prefetch(d + 0 * dst_stride);
- __builtin_prefetch(d + 1 * dst_stride);
- __builtin_prefetch(d + 2 * dst_stride);
- __builtin_prefetch(d + 3 * dst_stride);
-
- __builtin_prefetch(d_u8 + 0 * dst8_stride);
- __builtin_prefetch(d_u8 + 1 * dst8_stride);
- __builtin_prefetch(d_u8 + 2 * dst8_stride);
- __builtin_prefetch(d_u8 + 3 * dst8_stride);
-
- load_u16_4x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
-
- compute_basic_avg_4x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3,
- round_offset_vec, &d01, &d23);
-
- store_u8_4x1(d_u8 + 0 * dst8_stride, d01, 0);
- store_u8_4x1(d_u8 + 1 * dst8_stride, d01, 1);
- store_u8_4x1(d_u8 + 2 * dst8_stride, d23, 0);
- store_u8_4x1(d_u8 + 3 * dst8_stride, d23, 1);
-
- s0 = s4;
- s1 = s5;
- s2 = s6;
- s3 = s7;
- s4 = s8;
- s5 = s9;
- s6 = s10;
- s += 4 * src_stride;
- d += 4 * dst_stride;
- d_u8 += 4 * dst8_stride;
- height -= 4;
-#else // !AOM_ARCH_AARCH64
- t0 = load_unaligned_u8_4x1(s);
- s7 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
-
- d0 = convolve8_4_y(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
- vget_low_s16(round_offset_vec));
-
- __builtin_prefetch(d);
-
- dd0 = vld1_u16(d);
-
- compute_basic_avg_4x1(dd0, d0, vget_low_s16(round_offset_vec), &d01);
-
- store_u8_4x1(d_u8, d01, 0);
-
- s0 = s1;
- s1 = s2;
- s2 = s3;
- s3 = s4;
- s4 = s5;
- s5 = s6;
- s6 = s7;
- s += src_stride;
- d += dst_stride;
- d_u8 += dst8_stride;
- height--;
-#endif // AOM_ARCH_AARCH64
- } while (height != 0);
- src_ptr += 4;
- dst_ptr += 4;
- dst8_ptr += 4;
- width -= 4;
- } while (width != 0);
- } else {
- int16x8_t s0, s1, s2, s3, s4, s5, s6, s7;
- uint16x8_t d0, dd0;
- uint8x8_t d0_u8, t0, t1, t2, t3, t4, t5, t6;
-#if AOM_ARCH_AARCH64
- int16x8_t s8, s9, s10, s11, s12, s13, s14;
- uint16x8_t d1, d2, d3, d4, d5, d6, d7, dd1, dd2, dd3, dd4, dd5, dd6, dd7;
- uint8x8_t d1_u8, d2_u8, d3_u8, d4_u8, d5_u8, d6_u8, d7_u8, t7;
-#endif // AOM_ARCH_AARCH64
-
- do {
- const uint8_t *s = src_ptr;
- CONV_BUF_TYPE *d = dst_ptr;
- uint8_t *d_u8 = dst8_ptr;
- int height = h;
-
- __builtin_prefetch(s + 0 * src_stride);
- __builtin_prefetch(s + 1 * src_stride);
- __builtin_prefetch(s + 2 * src_stride);
- __builtin_prefetch(s + 3 * src_stride);
- __builtin_prefetch(s + 4 * src_stride);
- __builtin_prefetch(s + 5 * src_stride);
- __builtin_prefetch(s + 6 * src_stride);
- __builtin_prefetch(s + 7 * src_stride);
- load_u8_8x7(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6);
-
- s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
- s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
- s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
- s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
- s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
- s5 = vreinterpretq_s16_u16(vmovl_u8(t5));
- s6 = vreinterpretq_s16_u16(vmovl_u8(t6));
-
- s += 7 * src_stride;
-
- do {
-#if AOM_ARCH_AARCH64
- load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
-
- s7 = vreinterpretq_s16_u16(vmovl_u8(t0));
- s8 = vreinterpretq_s16_u16(vmovl_u8(t1));
- s9 = vreinterpretq_s16_u16(vmovl_u8(t2));
- s10 = vreinterpretq_s16_u16(vmovl_u8(t3));
- s11 = vreinterpretq_s16_u16(vmovl_u8(t4));
- s12 = vreinterpretq_s16_u16(vmovl_u8(t5));
- s13 = vreinterpretq_s16_u16(vmovl_u8(t6));
- s14 = vreinterpretq_s16_u16(vmovl_u8(t7));
-
- __builtin_prefetch(dst_ptr + 0 * dst_stride);
- __builtin_prefetch(dst_ptr + 1 * dst_stride);
- __builtin_prefetch(dst_ptr + 2 * dst_stride);
- __builtin_prefetch(dst_ptr + 3 * dst_stride);
-
- d0 = convolve8_8_y(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
- round_offset_vec);
- d1 = convolve8_8_y(s1, s2, s3, s4, s5, s6, s7, s8, y_filter,
- round_offset_vec);
- d2 = convolve8_8_y(s2, s3, s4, s5, s6, s7, s8, s9, y_filter,
- round_offset_vec);
- d3 = convolve8_8_y(s3, s4, s5, s6, s7, s8, s9, s10, y_filter,
- round_offset_vec);
- d4 = convolve8_8_y(s4, s5, s6, s7, s8, s9, s10, s11, y_filter,
- round_offset_vec);
- d5 = convolve8_8_y(s5, s6, s7, s8, s9, s10, s11, s12, y_filter,
- round_offset_vec);
- d6 = convolve8_8_y(s6, s7, s8, s9, s10, s11, s12, s13, y_filter,
- round_offset_vec);
- d7 = convolve8_8_y(s7, s8, s9, s10, s11, s12, s13, s14, y_filter,
- round_offset_vec);
-
- __builtin_prefetch(d + 0 * dst8_stride);
- __builtin_prefetch(d + 1 * dst8_stride);
- __builtin_prefetch(d + 2 * dst8_stride);
- __builtin_prefetch(d + 3 * dst8_stride);
-
- load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3);
-
- compute_basic_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3,
- round_offset_vec, &d0_u8, &d1_u8, &d2_u8, &d3_u8);
-
- store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8);
- d_u8 += 4 * dst8_stride;
-
- load_u16_8x4(d + 4 * dst_stride, dst_stride, &dd4, &dd5, &dd6, &dd7);
-
- compute_basic_avg_8x4(dd4, dd5, dd6, dd7, d4, d5, d6, d7,
- round_offset_vec, &d4_u8, &d5_u8, &d6_u8, &d7_u8);
-
- store_u8_8x4(d_u8, dst8_stride, d4_u8, d5_u8, d6_u8, d7_u8);
- d_u8 += 4 * dst8_stride;
-
- s0 = s8;
- s1 = s9;
- s2 = s10;
- s3 = s11;
- s4 = s12;
- s5 = s13;
- s6 = s14;
- s += 8 * src_stride;
- d += 8 * dst_stride;
- height -= 8;
-#else // !AOM_ARCH_AARCH64
- s7 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
-
- __builtin_prefetch(dst_ptr);
-
- d0 = convolve8_8_y(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
- round_offset_vec);
-
- s0 = s1;
- s1 = s2;
- s2 = s3;
- s3 = s4;
- s4 = s5;
- s5 = s6;
- s6 = s7;
-
- __builtin_prefetch(d);
-
- dd0 = vld1q_u16(d);
-
- compute_basic_avg_8x1(dd0, d0, round_offset_vec, &d0_u8);
-
- vst1_u8(d_u8, d0_u8);
- d_u8 += dst8_stride;
-
- s += src_stride;
- d += dst_stride;
- height--;
-#endif // AOM_ARCH_AARCH64
- } while (height != 0);
- src_ptr += 8;
- dst_ptr += 8;
- dst8_ptr += 8;
- width -= 8;
- } while (width != 0);
- }
-}
-
-static INLINE void dist_wtd_convolve_y_8tap_neon(const uint8_t *src_ptr,
- int src_stride, int w, int h,
- const int16x8_t y_filter,
- ConvolveParams *conv_params) {
- const int bd = 8;
- const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
- const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) +
- (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
- const int16x8_t round_offset_vec = vdupq_n_s16(round_offset);
-
- CONV_BUF_TYPE *dst_ptr = conv_params->dst;
- const int dst_stride = conv_params->dst_stride;
- int width = w;
-
- if (w == 4 || h == 4) {
- int16x4_t s0, s1, s2, s3, s4, s5, s6, s7;
- uint16x4_t d0;
- uint8x8_t t0, t1, t2, t3, t4, t5, t6;
-#if AOM_ARCH_AARCH64
- int16x4_t s8, s9, s10;
- uint16x4_t d1, d2, d3;
-#endif // AOM_ARCH_AARCH64
-
- do {
- const uint8_t *s = src_ptr;
- CONV_BUF_TYPE *d = dst_ptr;
- int height = h;
-
- __builtin_prefetch(s + 0 * src_stride);
- __builtin_prefetch(s + 1 * src_stride);
- __builtin_prefetch(s + 2 * src_stride);
- __builtin_prefetch(s + 3 * src_stride);
-
- t0 = load_unaligned_u8_4x1(s + 0 * src_stride);
- t1 = load_unaligned_u8_4x1(s + 1 * src_stride);
- t2 = load_unaligned_u8_4x1(s + 2 * src_stride);
- t3 = load_unaligned_u8_4x1(s + 3 * src_stride);
- t4 = load_unaligned_u8_4x1(s + 4 * src_stride);
- t5 = load_unaligned_u8_4x1(s + 5 * src_stride);
- t6 = load_unaligned_u8_4x1(s + 6 * src_stride);
-
- s0 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
- s1 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t1)));
- s2 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t2)));
- s3 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t3)));
- s4 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t4)));
- s5 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t5)));
- s6 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t6)));
-
- __builtin_prefetch(d + 0 * dst_stride);
- __builtin_prefetch(d + 1 * dst_stride);
- __builtin_prefetch(d + 2 * dst_stride);
- __builtin_prefetch(d + 3 * dst_stride);
-
- s += 7 * src_stride;
-
- do {
-#if AOM_ARCH_AARCH64
- t0 = load_unaligned_u8_4x1(s + 0 * src_stride);
- t1 = load_unaligned_u8_4x1(s + 1 * src_stride);
- t2 = load_unaligned_u8_4x1(s + 2 * src_stride);
- t3 = load_unaligned_u8_4x1(s + 3 * src_stride);
-
- s7 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
- s8 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t1)));
- s9 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t2)));
- s10 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t3)));
-
- d0 = convolve8_4_y(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
- vget_low_s16(round_offset_vec));
- d1 = convolve8_4_y(s1, s2, s3, s4, s5, s6, s7, s8, y_filter,
- vget_low_s16(round_offset_vec));
- d2 = convolve8_4_y(s2, s3, s4, s5, s6, s7, s8, s9, y_filter,
- vget_low_s16(round_offset_vec));
- d3 = convolve8_4_y(s3, s4, s5, s6, s7, s8, s9, s10, y_filter,
- vget_low_s16(round_offset_vec));
-
- store_u16_4x4(d, dst_stride, d0, d1, d2, d3);
-
- s0 = s4;
- s1 = s5;
- s2 = s6;
- s3 = s7;
- s4 = s8;
- s5 = s9;
- s6 = s10;
- s += 4 * src_stride;
- d += 4 * dst_stride;
- height -= 4;
-#else // !AOM_ARCH_AARCH64
- t0 = load_unaligned_u8_4x1(s);
- s7 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(t0)));
-
- d0 = convolve8_4_y(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
- vget_low_s16(round_offset_vec));
-
- vst1_u16(d, d0);
-
- s0 = s1;
- s1 = s2;
- s2 = s3;
- s3 = s4;
- s4 = s5;
- s5 = s6;
- s6 = s7;
- s += src_stride;
- d += dst_stride;
- height--;
-#endif // AOM_ARCH_AARCH64
- } while (height != 0);
- src_ptr += 4;
- dst_ptr += 4;
- width -= 4;
- } while (width != 0);
- } else {
- int16x8_t s0, s1, s2, s3, s4, s5, s6, s7;
- uint16x8_t d0;
- uint8x8_t t0, t1, t2, t3, t4, t5, t6;
-#if AOM_ARCH_AARCH64
- int16x8_t s8, s9, s10, s11, s12, s13, s14;
- uint16x8_t d1, d2, d3, d4, d5, d6, d7;
- uint8x8_t t7;
-#endif // AOM_ARCH_AARCH64
-
- do {
- const uint8_t *s = src_ptr;
- CONV_BUF_TYPE *d = dst_ptr;
- int height = h;
-
- __builtin_prefetch(s + 0 * src_stride);
- __builtin_prefetch(s + 1 * src_stride);
- __builtin_prefetch(s + 2 * src_stride);
- __builtin_prefetch(s + 3 * src_stride);
- __builtin_prefetch(s + 4 * src_stride);
- __builtin_prefetch(s + 5 * src_stride);
- __builtin_prefetch(s + 6 * src_stride);
- __builtin_prefetch(s + 7 * src_stride);
- load_u8_8x7(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6);
-
- s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
- s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
- s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
- s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
- s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
- s5 = vreinterpretq_s16_u16(vmovl_u8(t5));
- s6 = vreinterpretq_s16_u16(vmovl_u8(t6));
-
- s += 7 * src_stride;
-
- do {
-#if AOM_ARCH_AARCH64
- load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
-
- s7 = vreinterpretq_s16_u16(vmovl_u8(t0));
- s8 = vreinterpretq_s16_u16(vmovl_u8(t1));
- s9 = vreinterpretq_s16_u16(vmovl_u8(t2));
- s10 = vreinterpretq_s16_u16(vmovl_u8(t3));
- s11 = vreinterpretq_s16_u16(vmovl_u8(t4));
- s12 = vreinterpretq_s16_u16(vmovl_u8(t5));
- s13 = vreinterpretq_s16_u16(vmovl_u8(t6));
- s14 = vreinterpretq_s16_u16(vmovl_u8(t7));
-
- __builtin_prefetch(dst_ptr + 0 * dst_stride);
- __builtin_prefetch(dst_ptr + 1 * dst_stride);
- __builtin_prefetch(dst_ptr + 2 * dst_stride);
- __builtin_prefetch(dst_ptr + 3 * dst_stride);
-
- d0 = convolve8_8_y(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
- round_offset_vec);
- d1 = convolve8_8_y(s1, s2, s3, s4, s5, s6, s7, s8, y_filter,
- round_offset_vec);
- d2 = convolve8_8_y(s2, s3, s4, s5, s6, s7, s8, s9, y_filter,
- round_offset_vec);
- d3 = convolve8_8_y(s3, s4, s5, s6, s7, s8, s9, s10, y_filter,
- round_offset_vec);
- d4 = convolve8_8_y(s4, s5, s6, s7, s8, s9, s10, s11, y_filter,
- round_offset_vec);
- d5 = convolve8_8_y(s5, s6, s7, s8, s9, s10, s11, s12, y_filter,
- round_offset_vec);
- d6 = convolve8_8_y(s6, s7, s8, s9, s10, s11, s12, s13, y_filter,
- round_offset_vec);
- d7 = convolve8_8_y(s7, s8, s9, s10, s11, s12, s13, s14, y_filter,
- round_offset_vec);
-
- store_u16_8x8(d, dst_stride, d0, d1, d2, d3, d4, d5, d6, d7);
-
- s0 = s8;
- s1 = s9;
- s2 = s10;
- s3 = s11;
- s4 = s12;
- s5 = s13;
- s6 = s14;
- s += 8 * src_stride;
- d += 8 * dst_stride;
- height -= 8;
-#else // !AOM_ARCH_AARCH64
- s7 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s)));
-
- __builtin_prefetch(dst_ptr);
-
- d0 = convolve8_8_y(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
- round_offset_vec);
-
- s0 = s1;
- s1 = s2;
- s2 = s3;
- s3 = s4;
- s4 = s5;
- s5 = s6;
- s6 = s7;
-
- vst1q_u16(d, d0);
-
- s += src_stride;
- d += dst_stride;
- height--;
-#endif // AOM_ARCH_AARCH64
- } while (height != 0);
- src_ptr += 8;
- dst_ptr += 8;
- width -= 8;
- } while (width != 0);
- }
-}
-
-void av1_dist_wtd_convolve_y_neon(const uint8_t *src, int src_stride,
- uint8_t *dst8, int dst8_stride, int w, int h,
- const InterpFilterParams *filter_params_y,
- const int subpel_y_qn,
- ConvolveParams *conv_params) {
- assert(w % 4 == 0);
- assert(h % 4 == 0);
-
- // Vertical filter.
- const int16_t *y_filter_ptr = av1_get_interp_filter_subpel_kernel(
- filter_params_y, subpel_y_qn & SUBPEL_MASK);
- // Filter values are even, so downshift by 1 to reduce intermediate
- // precision requirements.
- const int16x8_t y_filter = vshrq_n_s16(vld1q_s16(y_filter_ptr), 1);
-
- const int vert_offset = filter_params_y->taps / 2 - 1;
- const uint8_t *src_ptr = src - (vert_offset * src_stride);
-
- if (get_filter_tap(filter_params_y, subpel_y_qn) <= 6) {
- if (conv_params->do_average) {
- if (UNLIKELY(conv_params->use_dist_wtd_comp_avg)) {
- dist_wtd_convolve_y_6tap_dist_wtd_avg_neon(
- src_ptr + src_stride, src_stride, dst8, dst8_stride, w, h, y_filter,
- conv_params);
- } else {
- dist_wtd_convolve_y_6tap_avg_neon(src_ptr + src_stride, src_stride,
- dst8, dst8_stride, w, h, y_filter,
- conv_params);
- }
- } else {
- dist_wtd_convolve_y_6tap_neon(src_ptr + src_stride, src_stride, w, h,
- y_filter, conv_params);
- }
- } else {
- if (conv_params->do_average) {
- if (UNLIKELY(conv_params->use_dist_wtd_comp_avg)) {
- dist_wtd_convolve_y_8tap_dist_wtd_avg_neon(src_ptr, src_stride, dst8,
- dst8_stride, w, h, y_filter,
- conv_params);
- } else {
- dist_wtd_convolve_y_8tap_avg_neon(src_ptr, src_stride, dst8,
- dst8_stride, w, h, y_filter,
- conv_params);
- }
- } else {
- dist_wtd_convolve_y_8tap_neon(src_ptr, src_stride, w, h, y_filter,
- conv_params);
- }
- }
-}
diff --git a/av1/common/arm/reconinter_neon.c b/av1/common/arm/reconinter_neon.c
index 3694763..2b0274c 100644
--- a/av1/common/arm/reconinter_neon.c
+++ b/av1/common/arm/reconinter_neon.c
@@ -12,6 +12,7 @@
#include <arm_neon.h>
#include <assert.h>
+#include <stdbool.h>
#include "aom/aom_integer.h"
#include "aom_dsp/blend.h"
@@ -20,6 +21,93 @@
#include "av1/common/blockd.h"
#include "config/av1_rtcd.h"
+static AOM_INLINE void diffwtd_mask_d16_neon(
+ uint8_t *mask, const bool inverse, const CONV_BUF_TYPE *src0,
+ int src0_stride, const CONV_BUF_TYPE *src1, int src1_stride, int h, int w,
+ ConvolveParams *conv_params, int bd) {
+ const int round =
+ 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1 + (bd - 8);
+ const int16x8_t round_vec = vdupq_n_s16((int16_t)(-round));
+
+ if (w >= 16) {
+ int i = 0;
+ do {
+ int j = 0;
+ do {
+ uint16x8_t s0_lo = vld1q_u16(src0 + j);
+ uint16x8_t s1_lo = vld1q_u16(src1 + j);
+ uint16x8_t s0_hi = vld1q_u16(src0 + j + 8);
+ uint16x8_t s1_hi = vld1q_u16(src1 + j + 8);
+
+ uint16x8_t diff_lo_u16 = vrshlq_u16(vabdq_u16(s0_lo, s1_lo), round_vec);
+ uint16x8_t diff_hi_u16 = vrshlq_u16(vabdq_u16(s0_hi, s1_hi), round_vec);
+ uint8x8_t diff_lo_u8 = vshrn_n_u16(diff_lo_u16, DIFF_FACTOR_LOG2);
+ uint8x8_t diff_hi_u8 = vshrn_n_u16(diff_hi_u16, DIFF_FACTOR_LOG2);
+ uint8x16_t diff = vcombine_u8(diff_lo_u8, diff_hi_u8);
+
+ uint8x16_t m;
+ if (inverse) {
+ m = vqsubq_u8(vdupq_n_u8(64 - 38), diff); // Saturating to 0
+ } else {
+ m = vminq_u8(vaddq_u8(diff, vdupq_n_u8(38)), vdupq_n_u8(64));
+ }
+
+ vst1q_u8(mask, m);
+
+ mask += 16;
+ j += 16;
+ } while (j < w);
+ src0 += src0_stride;
+ src1 += src1_stride;
+ } while (++i < h);
+ } else if (w == 8) {
+ int i = 0;
+ do {
+ uint16x8_t s0 = vld1q_u16(src0);
+ uint16x8_t s1 = vld1q_u16(src1);
+
+ uint16x8_t diff_u16 = vrshlq_u16(vabdq_u16(s0, s1), round_vec);
+ uint8x8_t diff_u8 = vshrn_n_u16(diff_u16, DIFF_FACTOR_LOG2);
+ uint8x8_t m;
+ if (inverse) {
+ m = vqsub_u8(vdup_n_u8(64 - 38), diff_u8); // Saturating to 0
+ } else {
+ m = vmin_u8(vadd_u8(diff_u8, vdup_n_u8(38)), vdup_n_u8(64));
+ }
+
+ vst1_u8(mask, m);
+
+ mask += 8;
+ src0 += src0_stride;
+ src1 += src1_stride;
+ } while (++i < h);
+ } else if (w == 4) {
+ int i = 0;
+ do {
+ uint16x8_t s0 =
+ vcombine_u16(vld1_u16(src0), vld1_u16(src0 + src0_stride));
+ uint16x8_t s1 =
+ vcombine_u16(vld1_u16(src1), vld1_u16(src1 + src1_stride));
+
+ uint16x8_t diff_u16 = vrshlq_u16(vabdq_u16(s0, s1), round_vec);
+ uint8x8_t diff_u8 = vshrn_n_u16(diff_u16, DIFF_FACTOR_LOG2);
+ uint8x8_t m;
+ if (inverse) {
+ m = vqsub_u8(vdup_n_u8(64 - 38), diff_u8); // Saturating to 0
+ } else {
+ m = vmin_u8(vadd_u8(diff_u8, vdup_n_u8(38)), vdup_n_u8(64));
+ }
+
+ vst1_u8(mask, m);
+
+ mask += 8;
+ src0 += 2 * src0_stride;
+ src1 += 2 * src1_stride;
+ i += 2;
+ } while (i < h);
+ }
+}
+
void av1_build_compound_diffwtd_mask_d16_neon(
uint8_t *mask, DIFFWTD_MASK_TYPE mask_type, const CONV_BUF_TYPE *src0,
int src0_stride, const CONV_BUF_TYPE *src1, int src1_stride, int h, int w,
@@ -27,60 +115,103 @@
assert(h >= 4);
assert(w >= 4);
assert((mask_type == DIFFWTD_38_INV) || (mask_type == DIFFWTD_38));
- const int round =
- 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1 + (bd - 8);
- uint16x8_t diff_q, tmp0, tmp1;
- uint8x8_t diff_d, diff_select;
- const CONV_BUF_TYPE *src0_1, *src1_1;
- const int16x8_t dup_round = vdupq_n_s16((int16_t)(-round));
- const uint8x8_t dup_38 = vdup_n_u8(38);
- const uint8x8_t dup_64 = vdup_n_u8(AOM_BLEND_A64_MAX_ALPHA);
+
if (mask_type == DIFFWTD_38) {
- diff_select = vdup_n_u8(255);
- } else {
- diff_select = vdup_n_u8(0);
+ diffwtd_mask_d16_neon(mask, /*inverse=*/false, src0, src0_stride, src1,
+ src1_stride, h, w, conv_params, bd);
+ } else { // mask_type == DIFFWTD_38_INV
+ diffwtd_mask_d16_neon(mask, /*inverse=*/true, src0, src0_stride, src1,
+ src1_stride, h, w, conv_params, bd);
}
- if (w >= 8) {
- for (int i = 0; i < h; ++i) {
- src0_1 = src0;
- src1_1 = src1;
- for (int j = 0; j < w; j += 8) {
- __builtin_prefetch(src0_1);
- __builtin_prefetch(src1_1);
- diff_q = vabdq_u16(vld1q_u16(src0_1), vld1q_u16(src1_1));
- diff_q = vrshlq_u16(diff_q, dup_round);
- diff_d = vshrn_n_u16(diff_q, DIFF_FACTOR_LOG2);
- diff_d = vmin_u8(vadd_u8(diff_d, dup_38), dup_64);
- diff_d = vbsl_u8(diff_select, diff_d, vsub_u8(dup_64, diff_d));
- vst1_u8(mask, diff_d);
- src0_1 += 8;
- src1_1 += 8;
- mask += 8;
- }
+}
+
+static AOM_INLINE void diffwtd_mask_neon(uint8_t *mask, const bool inverse,
+ const uint8_t *src0, int src0_stride,
+ const uint8_t *src1, int src1_stride,
+ int h, int w) {
+ if (w >= 16) {
+ int i = 0;
+ do {
+ int j = 0;
+ do {
+ uint8x16_t s0 = vld1q_u8(src0 + j);
+ uint8x16_t s1 = vld1q_u8(src1 + j);
+
+ uint8x16_t diff = vshrq_n_u8(vabdq_u8(s0, s1), DIFF_FACTOR_LOG2);
+ uint8x16_t m;
+ if (inverse) {
+ m = vqsubq_u8(vdupq_n_u8(64 - 38), diff); // Saturating to 0
+ } else {
+ m = vminq_u8(vaddq_u8(diff, vdupq_n_u8(38)), vdupq_n_u8(64));
+ }
+
+ vst1q_u8(mask, m);
+
+ mask += 16;
+ j += 16;
+ } while (j < w);
src0 += src0_stride;
src1 += src1_stride;
- }
+ } while (++i < h);
+ } else if (w == 8) {
+ int i = 0;
+ do {
+ uint8x16_t s0 = vcombine_u8(vld1_u8(src0), vld1_u8(src0 + src0_stride));
+ uint8x16_t s1 = vcombine_u8(vld1_u8(src1), vld1_u8(src1 + src0_stride));
+
+ uint8x16_t diff = vshrq_n_u8(vabdq_u8(s0, s1), DIFF_FACTOR_LOG2);
+ uint8x16_t m;
+ if (inverse) {
+ m = vqsubq_u8(vdupq_n_u8(64 - 38), diff); // Saturating to 0
+ } else {
+ m = vminq_u8(vaddq_u8(diff, vdupq_n_u8(38)), vdupq_n_u8(64));
+ }
+
+ vst1q_u8(mask, m);
+
+ mask += 16;
+ src0 += 2 * src0_stride;
+ src1 += 2 * src1_stride;
+ i += 2;
+ } while (i < h);
} else if (w == 4) {
- for (int i = 0; i < h; i += 2) {
- src0_1 = src0;
- src1_1 = src1;
- __builtin_prefetch(src0_1 + 0 * src0_stride);
- __builtin_prefetch(src0_1 + 1 * src0_stride);
- __builtin_prefetch(src1_1 + 0 * src1_stride);
- __builtin_prefetch(src1_1 + 1 * src1_stride);
- tmp0 = vcombine_u16(vld1_u16(src0_1 + (0 * src0_stride)),
- vld1_u16(src0_1 + (1 * src0_stride)));
- tmp1 = vcombine_u16(vld1_u16(src1_1 + (0 * src1_stride)),
- vld1_u16(src1_1 + (1 * src1_stride)));
- diff_q = vabdq_u16(tmp0, tmp1);
- diff_q = vrshlq_u16(diff_q, dup_round);
- diff_d = vshrn_n_u16(diff_q, DIFF_FACTOR_LOG2);
- diff_d = vmin_u8(vadd_u8(diff_d, dup_38), dup_64);
- diff_d = vbsl_u8(diff_select, diff_d, vsub_u8(dup_64, diff_d));
- vst1_u8(mask, diff_d);
- src0 += src0_stride * 2;
- src1 += src1_stride * 2;
- mask += w * 2;
- }
+ int i = 0;
+ do {
+ uint8x16_t s0 = load_unaligned_u8q(src0, src0_stride);
+ uint8x16_t s1 = load_unaligned_u8q(src1, src1_stride);
+
+ uint8x16_t diff = vshrq_n_u8(vabdq_u8(s0, s1), DIFF_FACTOR_LOG2);
+ uint8x16_t m;
+ if (inverse) {
+ m = vqsubq_u8(vdupq_n_u8(64 - 38), diff); // Saturating to 0
+ } else {
+ m = vminq_u8(vaddq_u8(diff, vdupq_n_u8(38)), vdupq_n_u8(64));
+ }
+
+ vst1q_u8(mask, m);
+
+ mask += 16;
+ src0 += 4 * src0_stride;
+ src1 += 4 * src1_stride;
+ i += 4;
+ } while (i < h);
+ }
+}
+
+void av1_build_compound_diffwtd_mask_neon(uint8_t *mask,
+ DIFFWTD_MASK_TYPE mask_type,
+ const uint8_t *src0, int src0_stride,
+ const uint8_t *src1, int src1_stride,
+ int h, int w) {
+ assert(h % 4 == 0);
+ assert(w % 4 == 0);
+ assert(mask_type == DIFFWTD_38_INV || mask_type == DIFFWTD_38);
+
+ if (mask_type == DIFFWTD_38) {
+ diffwtd_mask_neon(mask, /*inverse=*/false, src0, src0_stride, src1,
+ src1_stride, h, w);
+ } else { // mask_type == DIFFWTD_38_INV
+ diffwtd_mask_neon(mask, /*inverse=*/true, src0, src0_stride, src1,
+ src1_stride, h, w);
}
}
diff --git a/av1/common/arm/reconintra_neon.c b/av1/common/arm/reconintra_neon.c
index 8d190fb..cf488a9 100644
--- a/av1/common/arm/reconintra_neon.c
+++ b/av1/common/arm/reconintra_neon.c
@@ -17,6 +17,8 @@
#include "aom/aom_integer.h"
#include "aom_dsp/arm/sum_neon.h"
+#define MAX_UPSAMPLE_SZ 16
+
DECLARE_ALIGNED(16, const int8_t,
av1_filter_intra_taps_neon[FILTER_INTRA_MODES][8][8]) = {
{
@@ -153,3 +155,185 @@
dst += stride;
}
}
+
+void av1_filter_intra_edge_neon(uint8_t *p, int sz, int strength) {
+ if (!strength) return;
+ assert(sz >= 0 && sz <= 129);
+
+ uint8_t edge[160]; // Max value of sz + enough padding for vector accesses.
+ memcpy(edge + 1, p, sz * sizeof(*p));
+
+ // Populate extra space appropriately.
+ edge[0] = edge[1];
+ edge[sz + 1] = edge[sz];
+ edge[sz + 2] = edge[sz];
+
+ // Don't overwrite first pixel.
+ uint8_t *dst = p + 1;
+ sz--;
+
+ if (strength == 1) { // Filter: {4, 8, 4}.
+ const uint8_t *src = edge + 1;
+
+ while (sz >= 8) {
+ uint8x8_t s0 = vld1_u8(src);
+ uint8x8_t s1 = vld1_u8(src + 1);
+ uint8x8_t s2 = vld1_u8(src + 2);
+
+ // Make use of the identity:
+ // (4*a + 8*b + 4*c) >> 4 == (a + (b << 1) + c) >> 2
+ uint16x8_t t0 = vaddl_u8(s0, s2);
+ uint16x8_t t1 = vaddl_u8(s1, s1);
+ uint16x8_t sum = vaddq_u16(t0, t1);
+ uint8x8_t res = vrshrn_n_u16(sum, 2);
+
+ vst1_u8(dst, res);
+
+ src += 8;
+ dst += 8;
+ sz -= 8;
+ }
+
+ if (sz > 0) { // Handle sz < 8 to avoid modifying out-of-bounds values.
+ uint8x8_t s0 = vld1_u8(src);
+ uint8x8_t s1 = vld1_u8(src + 1);
+ uint8x8_t s2 = vld1_u8(src + 2);
+
+ uint16x8_t t0 = vaddl_u8(s0, s2);
+ uint16x8_t t1 = vaddl_u8(s1, s1);
+ uint16x8_t sum = vaddq_u16(t0, t1);
+ uint8x8_t res = vrshrn_n_u16(sum, 2);
+
+ // Mask off out-of-bounds indices.
+ uint8x8_t current_dst = vld1_u8(dst);
+ uint8x8_t mask = vcgt_u8(vdup_n_u8(sz), vcreate_u8(0x0706050403020100));
+ res = vbsl_u8(mask, res, current_dst);
+
+ vst1_u8(dst, res);
+ }
+ } else if (strength == 2) { // Filter: {5, 6, 5}.
+ const uint8_t *src = edge + 1;
+
+ const uint8x8x3_t filter = { { vdup_n_u8(5), vdup_n_u8(6), vdup_n_u8(5) } };
+
+ while (sz >= 8) {
+ uint8x8_t s0 = vld1_u8(src);
+ uint8x8_t s1 = vld1_u8(src + 1);
+ uint8x8_t s2 = vld1_u8(src + 2);
+
+ uint16x8_t accum = vmull_u8(s0, filter.val[0]);
+ accum = vmlal_u8(accum, s1, filter.val[1]);
+ accum = vmlal_u8(accum, s2, filter.val[2]);
+ uint8x8_t res = vrshrn_n_u16(accum, 4);
+
+ vst1_u8(dst, res);
+
+ src += 8;
+ dst += 8;
+ sz -= 8;
+ }
+
+ if (sz > 0) { // Handle sz < 8 to avoid modifying out-of-bounds values.
+ uint8x8_t s0 = vld1_u8(src);
+ uint8x8_t s1 = vld1_u8(src + 1);
+ uint8x8_t s2 = vld1_u8(src + 2);
+
+ uint16x8_t accum = vmull_u8(s0, filter.val[0]);
+ accum = vmlal_u8(accum, s1, filter.val[1]);
+ accum = vmlal_u8(accum, s2, filter.val[2]);
+ uint8x8_t res = vrshrn_n_u16(accum, 4);
+
+ // Mask off out-of-bounds indices.
+ uint8x8_t current_dst = vld1_u8(dst);
+ uint8x8_t mask = vcgt_u8(vdup_n_u8(sz), vcreate_u8(0x0706050403020100));
+ res = vbsl_u8(mask, res, current_dst);
+
+ vst1_u8(dst, res);
+ }
+ } else { // Filter {2, 4, 4, 4, 2}.
+ const uint8_t *src = edge;
+
+ while (sz >= 8) {
+ uint8x8_t s0 = vld1_u8(src);
+ uint8x8_t s1 = vld1_u8(src + 1);
+ uint8x8_t s2 = vld1_u8(src + 2);
+ uint8x8_t s3 = vld1_u8(src + 3);
+ uint8x8_t s4 = vld1_u8(src + 4);
+
+ // Make use of the identity:
+ // (2*a + 4*b + 4*c + 4*d + 2*e) >> 4 == (a + ((b + c + d) << 1) + e) >> 3
+ uint16x8_t t0 = vaddl_u8(s0, s4);
+ uint16x8_t t1 = vaddl_u8(s1, s2);
+ t1 = vaddw_u8(t1, s3);
+ t1 = vaddq_u16(t1, t1);
+ uint16x8_t sum = vaddq_u16(t0, t1);
+ uint8x8_t res = vrshrn_n_u16(sum, 3);
+
+ vst1_u8(dst, res);
+
+ src += 8;
+ dst += 8;
+ sz -= 8;
+ }
+
+ if (sz > 0) { // Handle sz < 8 to avoid modifying out-of-bounds values.
+ uint8x8_t s0 = vld1_u8(src);
+ uint8x8_t s1 = vld1_u8(src + 1);
+ uint8x8_t s2 = vld1_u8(src + 2);
+ uint8x8_t s3 = vld1_u8(src + 3);
+ uint8x8_t s4 = vld1_u8(src + 4);
+
+ uint16x8_t t0 = vaddl_u8(s0, s4);
+ uint16x8_t t1 = vaddl_u8(s1, s2);
+ t1 = vaddw_u8(t1, s3);
+ t1 = vaddq_u16(t1, t1);
+ uint16x8_t sum = vaddq_u16(t0, t1);
+ uint8x8_t res = vrshrn_n_u16(sum, 3);
+
+ // Mask off out-of-bounds indices.
+ uint8x8_t current_dst = vld1_u8(dst);
+ uint8x8_t mask = vcgt_u8(vdup_n_u8(sz), vcreate_u8(0x0706050403020100));
+ res = vbsl_u8(mask, res, current_dst);
+
+ vst1_u8(dst, res);
+ }
+ }
+}
+
+void av1_upsample_intra_edge_neon(uint8_t *p, int sz) {
+ if (!sz) return;
+
+ assert(sz <= MAX_UPSAMPLE_SZ);
+
+ uint8_t edge[MAX_UPSAMPLE_SZ + 3];
+ const uint8_t *src = edge;
+
+ // Copy p[-1..(sz-1)] and pad out both ends.
+ edge[0] = p[-1];
+ edge[1] = p[-1];
+ memcpy(edge + 2, p, sz);
+ edge[sz + 2] = p[sz - 1];
+ p[-2] = p[-1];
+
+ uint8_t *dst = p - 1;
+
+ do {
+ uint8x8_t s0 = vld1_u8(src);
+ uint8x8_t s1 = vld1_u8(src + 1);
+ uint8x8_t s2 = vld1_u8(src + 2);
+ uint8x8_t s3 = vld1_u8(src + 3);
+
+ int16x8_t t0 = vreinterpretq_s16_u16(vaddl_u8(s0, s3));
+ int16x8_t t1 = vreinterpretq_s16_u16(vaddl_u8(s1, s2));
+ t1 = vmulq_n_s16(t1, 9);
+ t1 = vsubq_s16(t1, t0);
+
+ uint8x8x2_t res = { { vqrshrun_n_s16(t1, 4), s2 } };
+
+ vst2_u8(dst, res);
+
+ src += 8;
+ dst += 16;
+ sz -= 8;
+ } while (sz > 0);
+}
diff --git a/av1/common/arm/resize_neon.c b/av1/common/arm/resize_neon.c
index 5f6d214..076981b 100644
--- a/av1/common/arm/resize_neon.c
+++ b/av1/common/arm/resize_neon.c
@@ -15,10 +15,61 @@
#include "aom_dsp/arm/mem_neon.h"
#include "aom_dsp/arm/transpose_neon.h"
#include "av1/common/resize.h"
-#include "av1/common/arm/convolve_neon.h"
#include "config/av1_rtcd.h"
#include "config/aom_scale_rtcd.h"
+static INLINE int16x4_t convolve8_4(const int16x4_t s0, const int16x4_t s1,
+ const int16x4_t s2, const int16x4_t s3,
+ const int16x4_t s4, const int16x4_t s5,
+ const int16x4_t s6, const int16x4_t s7,
+ const int16x8_t filter) {
+ const int16x4_t filter_lo = vget_low_s16(filter);
+ const int16x4_t filter_hi = vget_high_s16(filter);
+
+ int16x4_t sum = vmul_lane_s16(s0, filter_lo, 0);
+ sum = vmla_lane_s16(sum, s1, filter_lo, 1);
+ sum = vmla_lane_s16(sum, s2, filter_lo, 2);
+ sum = vmla_lane_s16(sum, s5, filter_hi, 1);
+ sum = vmla_lane_s16(sum, s6, filter_hi, 2);
+ sum = vmla_lane_s16(sum, s7, filter_hi, 3);
+ sum = vqadd_s16(sum, vmul_lane_s16(s3, filter_lo, 3));
+ sum = vqadd_s16(sum, vmul_lane_s16(s4, filter_hi, 0));
+ return sum;
+}
+
+static INLINE uint8x8_t convolve8_8(const int16x8_t s0, const int16x8_t s1,
+ const int16x8_t s2, const int16x8_t s3,
+ const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t s6, const int16x8_t s7,
+ const int16x8_t filter) {
+ const int16x4_t filter_lo = vget_low_s16(filter);
+ const int16x4_t filter_hi = vget_high_s16(filter);
+
+ int16x8_t sum = vmulq_lane_s16(s0, filter_lo, 0);
+ sum = vmlaq_lane_s16(sum, s1, filter_lo, 1);
+ sum = vmlaq_lane_s16(sum, s2, filter_lo, 2);
+ sum = vmlaq_lane_s16(sum, s5, filter_hi, 1);
+ sum = vmlaq_lane_s16(sum, s6, filter_hi, 2);
+ sum = vmlaq_lane_s16(sum, s7, filter_hi, 3);
+ sum = vqaddq_s16(sum, vmulq_lane_s16(s3, filter_lo, 3));
+ sum = vqaddq_s16(sum, vmulq_lane_s16(s4, filter_hi, 0));
+ return vqrshrun_n_s16(sum, 7);
+}
+
+static INLINE uint8x8_t scale_filter_8(const uint8x8_t *const s,
+ const int16x8_t filter) {
+ int16x8_t ss0 = vreinterpretq_s16_u16(vmovl_u8(s[0]));
+ int16x8_t ss1 = vreinterpretq_s16_u16(vmovl_u8(s[1]));
+ int16x8_t ss2 = vreinterpretq_s16_u16(vmovl_u8(s[2]));
+ int16x8_t ss3 = vreinterpretq_s16_u16(vmovl_u8(s[3]));
+ int16x8_t ss4 = vreinterpretq_s16_u16(vmovl_u8(s[4]));
+ int16x8_t ss5 = vreinterpretq_s16_u16(vmovl_u8(s[5]));
+ int16x8_t ss6 = vreinterpretq_s16_u16(vmovl_u8(s[6]));
+ int16x8_t ss7 = vreinterpretq_s16_u16(vmovl_u8(s[7]));
+
+ return convolve8_8(ss0, ss1, ss2, ss3, ss4, ss5, ss6, ss7, filter);
+}
+
static INLINE void scale_plane_2_to_1_phase_0(const uint8_t *src,
const int src_stride,
uint8_t *dst,
@@ -192,15 +243,16 @@
do {
load_u8_8x8(src + 2, src_stride, &s[0], &s[1], &s[2], &s[3], &s[4], &s[5],
&s[6], &s[7]);
- transpose_u8_8x8(&s[0], &s[1], &s[2], &s[3], &s[4], &s[5], &s[6], &s[7]);
+ transpose_elems_inplace_u8_8x8(&s[0], &s[1], &s[2], &s[3], &s[4], &s[5],
+ &s[6], &s[7]);
x = width_hor;
do {
src += 8;
load_u8_8x8(src, src_stride, &s[6], &s[7], &s[8], &s[9], &s[10], &s[11],
&s[12], &s[13]);
- transpose_u8_8x8(&s[6], &s[7], &s[8], &s[9], &s[10], &s[11], &s[12],
- &s[13]);
+ transpose_elems_inplace_u8_8x8(&s[6], &s[7], &s[8], &s[9], &s[10], &s[11],
+ &s[12], &s[13]);
d[0] = scale_filter_8(&s[0], filters); // 00 10 20 30 40 50 60 70
d[1] = scale_filter_8(&s[2], filters); // 01 11 21 31 41 51 61 71
@@ -210,7 +262,7 @@
// 10 11 12 13 50 51 52 53
// 20 21 22 23 60 61 62 63
// 30 31 32 33 70 71 72 73
- transpose_u8_8x4(&d[0], &d[1], &d[2], &d[3]);
+ transpose_elems_inplace_u8_8x4(&d[0], &d[1], &d[2], &d[3]);
vst1_lane_u32((uint32_t *)(t + 0 * width_hor), vreinterpret_u32_u8(d[0]),
0);
vst1_lane_u32((uint32_t *)(t + 1 * width_hor), vreinterpret_u32_u8(d[1]),
@@ -308,7 +360,8 @@
do {
load_u8_8x8(src + 4, src_stride, &s[0], &s[1], &s[2], &s[3], &s[4], &s[5],
&s[6], &s[7]);
- transpose_u8_4x8(&s[0], &s[1], &s[2], &s[3], s[4], s[5], s[6], s[7]);
+ transpose_elems_u8_4x8(s[0], s[1], s[2], s[3], s[4], s[5], s[6], s[7],
+ &s[0], &s[1], &s[2], &s[3]);
x = width_hor;
do {
@@ -316,8 +369,8 @@
src += 8;
load_u8_8x8(src, src_stride, &s[4], &s[5], &s[6], &s[7], &s[8], &s[9],
&s[10], &s[11]);
- transpose_u8_8x8(&s[4], &s[5], &s[6], &s[7], &s[8], &s[9], &s[10],
- &s[11]);
+ transpose_elems_inplace_u8_8x8(&s[4], &s[5], &s[6], &s[7], &s[8], &s[9],
+ &s[10], &s[11]);
d[0] = scale_filter_8(&s[0], filters); // 00 10 20 30 40 50 60 70
d[1] = scale_filter_8(&s[4], filters); // 01 11 21 31 41 51 61 71
@@ -453,14 +506,16 @@
load_u8_8x8(src, src_stride, &s[0], &s[1], &s[2], &s[3], &s[4], &s[5],
&s[6], &s[7]);
src += 1;
- transpose_u8_8x8(&s[0], &s[1], &s[2], &s[3], &s[4], &s[5], &s[6], &s[7]);
+ transpose_elems_inplace_u8_8x8(&s[0], &s[1], &s[2], &s[3], &s[4], &s[5],
+ &s[6], &s[7]);
x = width_hor;
do {
load_u8_8x8(src, src_stride, &s[1], &s[2], &s[3], &s[4], &s[5], &s[6],
&s[7], &s[8]);
src += 8;
- transpose_u8_8x8(&s[1], &s[2], &s[3], &s[4], &s[5], &s[6], &s[7], &s[8]);
+ transpose_elems_inplace_u8_8x8(&s[1], &s[2], &s[3], &s[4], &s[5], &s[6],
+ &s[7], &s[8]);
// 00 10 20 30 40 50 60 70
// 01 11 21 31 41 51 61 71
@@ -487,7 +542,8 @@
// 50 51 52 53 54 55 xx xx
// 60 61 62 63 64 65 xx xx
// 70 71 72 73 74 75 xx xx
- transpose_u8_8x8(&d[0], &d[1], &d[2], &d[3], &d[4], &d[5], &d[6], &d[7]);
+ transpose_elems_inplace_u8_8x8(&d[0], &d[1], &d[2], &d[3], &d[4], &d[5],
+ &d[6], &d[7]);
// store 2 extra pixels
vst1_u8(t + 0 * stride_hor, d[0]);
vst1_u8(t + 1 * stride_hor, d[1]);
@@ -586,15 +642,16 @@
do {
load_u8_8x8(src + 1, src_stride, &s[0], &s[1], &s[2], &s[3], &s[4], &s[5],
&s[6], &s[7]);
- transpose_u8_8x8(&s[0], &s[1], &s[2], &s[3], &s[4], &s[5], &s[6], &s[7]);
+ transpose_elems_inplace_u8_8x8(&s[0], &s[1], &s[2], &s[3], &s[4], &s[5],
+ &s[6], &s[7]);
x = width_hor;
do {
src += 8;
load_u8_8x8(src, src_stride, &s[7], &s[8], &s[9], &s[10], &s[11], &s[12],
&s[13], &s[14]);
- transpose_u8_8x8(&s[7], &s[8], &s[9], &s[10], &s[11], &s[12], &s[13],
- &s[14]);
+ transpose_elems_inplace_u8_8x8(&s[7], &s[8], &s[9], &s[10], &s[11],
+ &s[12], &s[13], &s[14]);
// 00 10 20 30 40 50 60 70
// 01 11 21 31 41 51 61 71
@@ -619,7 +676,8 @@
// 50 51 52 53 54 55 xx xx
// 60 61 62 63 64 65 xx xx
// 70 71 72 73 74 75 xx xx
- transpose_u8_8x8(&d[0], &d[1], &d[2], &d[3], &d[4], &d[5], &d[6], &d[7]);
+ transpose_elems_inplace_u8_8x8(&d[0], &d[1], &d[2], &d[3], &d[4], &d[5],
+ &d[6], &d[7]);
// store 2 extra pixels
vst1_u8(t + 0 * stride_hor, d[0]);
vst1_u8(t + 1 * stride_hor, d[1]);
@@ -828,3 +886,293 @@
aom_extend_frame_borders(dst, num_planes);
}
}
+
+static INLINE void scaledconvolve_horiz_w4(
+ const uint8_t *src, const ptrdiff_t src_stride, uint8_t *dst,
+ const ptrdiff_t dst_stride, const InterpKernel *const x_filters,
+ const int x0_q4, const int x_step_q4, const int w, const int h) {
+ DECLARE_ALIGNED(16, uint8_t, temp[4 * 4]);
+ int x, y, z;
+
+ src -= SUBPEL_TAPS / 2 - 1;
+
+ y = h;
+ do {
+ int x_q4 = x0_q4;
+ x = 0;
+ do {
+ // process 4 src_x steps
+ for (z = 0; z < 4; ++z) {
+ const uint8_t *const src_x = &src[x_q4 >> SUBPEL_BITS];
+ if (x_q4 & SUBPEL_MASK) {
+ const int16x8_t filters = vld1q_s16(x_filters[x_q4 & SUBPEL_MASK]);
+ uint8x8_t s[8], d;
+ int16x8_t ss[4];
+ int16x4_t t[8], tt;
+
+ load_u8_8x4(src_x, src_stride, &s[0], &s[1], &s[2], &s[3]);
+ transpose_elems_inplace_u8_8x4(&s[0], &s[1], &s[2], &s[3]);
+
+ ss[0] = vreinterpretq_s16_u16(vmovl_u8(s[0]));
+ ss[1] = vreinterpretq_s16_u16(vmovl_u8(s[1]));
+ ss[2] = vreinterpretq_s16_u16(vmovl_u8(s[2]));
+ ss[3] = vreinterpretq_s16_u16(vmovl_u8(s[3]));
+ t[0] = vget_low_s16(ss[0]);
+ t[1] = vget_low_s16(ss[1]);
+ t[2] = vget_low_s16(ss[2]);
+ t[3] = vget_low_s16(ss[3]);
+ t[4] = vget_high_s16(ss[0]);
+ t[5] = vget_high_s16(ss[1]);
+ t[6] = vget_high_s16(ss[2]);
+ t[7] = vget_high_s16(ss[3]);
+
+ tt = convolve8_4(t[0], t[1], t[2], t[3], t[4], t[5], t[6], t[7],
+ filters);
+ d = vqrshrun_n_s16(vcombine_s16(tt, tt), 7);
+ store_u8_4x1(&temp[4 * z], d, 0);
+ } else {
+ int i;
+ for (i = 0; i < 4; ++i) {
+ temp[z * 4 + i] = src_x[i * src_stride + 3];
+ }
+ }
+ x_q4 += x_step_q4;
+ }
+
+ // transpose the 4x4 filters values back to dst
+ {
+ const uint8x8x4_t d4 = vld4_u8(temp);
+ store_u8_4x1(&dst[x + 0 * dst_stride], d4.val[0], 0);
+ store_u8_4x1(&dst[x + 1 * dst_stride], d4.val[1], 0);
+ store_u8_4x1(&dst[x + 2 * dst_stride], d4.val[2], 0);
+ store_u8_4x1(&dst[x + 3 * dst_stride], d4.val[3], 0);
+ }
+ x += 4;
+ } while (x < w);
+
+ src += src_stride * 4;
+ dst += dst_stride * 4;
+ y -= 4;
+ } while (y > 0);
+}
+
+static INLINE void scaledconvolve_horiz_w8(
+ const uint8_t *src, const ptrdiff_t src_stride, uint8_t *dst,
+ const ptrdiff_t dst_stride, const InterpKernel *const x_filters,
+ const int x0_q4, const int x_step_q4, const int w, const int h) {
+ DECLARE_ALIGNED(16, uint8_t, temp[8 * 8]);
+ int x, y, z;
+ src -= SUBPEL_TAPS / 2 - 1;
+
+ // This function processes 8x8 areas. The intermediate height is not always
+ // a multiple of 8, so force it to be a multiple of 8 here.
+ y = (h + 7) & ~7;
+
+ do {
+ int x_q4 = x0_q4;
+ x = 0;
+ do {
+ uint8x8_t d[8];
+ // process 8 src_x steps
+ for (z = 0; z < 8; ++z) {
+ const uint8_t *const src_x = &src[x_q4 >> SUBPEL_BITS];
+
+ if (x_q4 & SUBPEL_MASK) {
+ const int16x8_t filters = vld1q_s16(x_filters[x_q4 & SUBPEL_MASK]);
+ uint8x8_t s[8];
+ load_u8_8x8(src_x, src_stride, &s[0], &s[1], &s[2], &s[3], &s[4],
+ &s[5], &s[6], &s[7]);
+ transpose_elems_inplace_u8_8x8(&s[0], &s[1], &s[2], &s[3], &s[4],
+ &s[5], &s[6], &s[7]);
+ d[0] = scale_filter_8(s, filters);
+ vst1_u8(&temp[8 * z], d[0]);
+ } else {
+ int i;
+ for (i = 0; i < 8; ++i) {
+ temp[z * 8 + i] = src_x[i * src_stride + 3];
+ }
+ }
+ x_q4 += x_step_q4;
+ }
+
+ // transpose the 8x8 filters values back to dst
+ load_u8_8x8(temp, 8, &d[0], &d[1], &d[2], &d[3], &d[4], &d[5], &d[6],
+ &d[7]);
+ transpose_elems_inplace_u8_8x8(&d[0], &d[1], &d[2], &d[3], &d[4], &d[5],
+ &d[6], &d[7]);
+ store_u8_8x8(dst + x, dst_stride, d[0], d[1], d[2], d[3], d[4], d[5],
+ d[6], d[7]);
+ x += 8;
+ } while (x < w);
+
+ src += src_stride * 8;
+ dst += dst_stride * 8;
+ } while (y -= 8);
+}
+
+static INLINE void scaledconvolve_vert_w4(
+ const uint8_t *src, const ptrdiff_t src_stride, uint8_t *dst,
+ const ptrdiff_t dst_stride, const InterpKernel *const y_filters,
+ const int y0_q4, const int y_step_q4, const int w, const int h) {
+ int y;
+ int y_q4 = y0_q4;
+
+ src -= src_stride * (SUBPEL_TAPS / 2 - 1);
+ y = h;
+ do {
+ const unsigned char *src_y = &src[(y_q4 >> SUBPEL_BITS) * src_stride];
+
+ if (y_q4 & SUBPEL_MASK) {
+ const int16x8_t filters = vld1q_s16(y_filters[y_q4 & SUBPEL_MASK]);
+ uint8x8_t s[8], d;
+ int16x4_t t[8], tt;
+
+ load_u8_8x8(src_y, src_stride, &s[0], &s[1], &s[2], &s[3], &s[4], &s[5],
+ &s[6], &s[7]);
+ t[0] = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(s[0])));
+ t[1] = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(s[1])));
+ t[2] = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(s[2])));
+ t[3] = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(s[3])));
+ t[4] = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(s[4])));
+ t[5] = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(s[5])));
+ t[6] = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(s[6])));
+ t[7] = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(s[7])));
+
+ tt = convolve8_4(t[0], t[1], t[2], t[3], t[4], t[5], t[6], t[7], filters);
+ d = vqrshrun_n_s16(vcombine_s16(tt, tt), 7);
+ store_u8_4x1(dst, d, 0);
+ } else {
+ memcpy(dst, &src_y[3 * src_stride], w);
+ }
+
+ dst += dst_stride;
+ y_q4 += y_step_q4;
+ } while (--y);
+}
+
+static INLINE void scaledconvolve_vert_w8(
+ const uint8_t *src, const ptrdiff_t src_stride, uint8_t *dst,
+ const ptrdiff_t dst_stride, const InterpKernel *const y_filters,
+ const int y0_q4, const int y_step_q4, const int w, const int h) {
+ int y;
+ int y_q4 = y0_q4;
+
+ src -= src_stride * (SUBPEL_TAPS / 2 - 1);
+ y = h;
+ do {
+ const unsigned char *src_y = &src[(y_q4 >> SUBPEL_BITS) * src_stride];
+ if (y_q4 & SUBPEL_MASK) {
+ const int16x8_t filters = vld1q_s16(y_filters[y_q4 & SUBPEL_MASK]);
+ uint8x8_t s[8], d;
+ load_u8_8x8(src_y, src_stride, &s[0], &s[1], &s[2], &s[3], &s[4], &s[5],
+ &s[6], &s[7]);
+ d = scale_filter_8(s, filters);
+ vst1_u8(dst, d);
+ } else {
+ memcpy(dst, &src_y[3 * src_stride], w);
+ }
+ dst += dst_stride;
+ y_q4 += y_step_q4;
+ } while (--y);
+}
+
+static INLINE void scaledconvolve_vert_w16(
+ const uint8_t *src, const ptrdiff_t src_stride, uint8_t *dst,
+ const ptrdiff_t dst_stride, const InterpKernel *const y_filters,
+ const int y0_q4, const int y_step_q4, const int w, const int h) {
+ int x, y;
+ int y_q4 = y0_q4;
+
+ src -= src_stride * (SUBPEL_TAPS / 2 - 1);
+ y = h;
+ do {
+ const unsigned char *src_y = &src[(y_q4 >> SUBPEL_BITS) * src_stride];
+ if (y_q4 & SUBPEL_MASK) {
+ x = 0;
+ do {
+ const int16x8_t filters = vld1q_s16(y_filters[y_q4 & SUBPEL_MASK]);
+ uint8x16_t ss[8];
+ uint8x8_t s[8], d[2];
+ load_u8_16x8(src_y, src_stride, &ss[0], &ss[1], &ss[2], &ss[3], &ss[4],
+ &ss[5], &ss[6], &ss[7]);
+ s[0] = vget_low_u8(ss[0]);
+ s[1] = vget_low_u8(ss[1]);
+ s[2] = vget_low_u8(ss[2]);
+ s[3] = vget_low_u8(ss[3]);
+ s[4] = vget_low_u8(ss[4]);
+ s[5] = vget_low_u8(ss[5]);
+ s[6] = vget_low_u8(ss[6]);
+ s[7] = vget_low_u8(ss[7]);
+ d[0] = scale_filter_8(s, filters);
+
+ s[0] = vget_high_u8(ss[0]);
+ s[1] = vget_high_u8(ss[1]);
+ s[2] = vget_high_u8(ss[2]);
+ s[3] = vget_high_u8(ss[3]);
+ s[4] = vget_high_u8(ss[4]);
+ s[5] = vget_high_u8(ss[5]);
+ s[6] = vget_high_u8(ss[6]);
+ s[7] = vget_high_u8(ss[7]);
+ d[1] = scale_filter_8(s, filters);
+ vst1q_u8(&dst[x], vcombine_u8(d[0], d[1]));
+ src_y += 16;
+ x += 16;
+ } while (x < w);
+ } else {
+ memcpy(dst, &src_y[3 * src_stride], w);
+ }
+ dst += dst_stride;
+ y_q4 += y_step_q4;
+ } while (--y);
+}
+
+void aom_scaled_2d_neon(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst,
+ ptrdiff_t dst_stride, const InterpKernel *filter,
+ int x0_q4, int x_step_q4, int y0_q4, int y_step_q4,
+ int w, int h) {
+ // Note: Fixed size intermediate buffer, temp, places limits on parameters.
+ // 2d filtering proceeds in 2 steps:
+ // (1) Interpolate horizontally into an intermediate buffer, temp.
+ // (2) Interpolate temp vertically to derive the sub-pixel result.
+ // Deriving the maximum number of rows in the temp buffer (135):
+ // --Smallest scaling factor is x1/2 ==> y_step_q4 = 32 (Normative).
+ // --Largest block size is 64x64 pixels.
+ // --64 rows in the downscaled frame span a distance of (64 - 1) * 32 in the
+ // original frame (in 1/16th pixel units).
+ // --Must round-up because block may be located at sub-pixel position.
+ // --Require an additional SUBPEL_TAPS rows for the 8-tap filter tails.
+ // --((64 - 1) * 32 + 15) >> 4 + 8 = 135.
+ // --Require an additional 8 rows for the horiz_w8 transpose tail.
+ // When calling in frame scaling function, the smallest scaling factor is x1/4
+ // ==> y_step_q4 = 64. Since w and h are at most 16, the temp buffer is still
+ // big enough.
+ DECLARE_ALIGNED(16, uint8_t, temp[(135 + 8) * 64]);
+ const int intermediate_height =
+ (((h - 1) * y_step_q4 + y0_q4) >> SUBPEL_BITS) + SUBPEL_TAPS;
+
+ assert(w <= 64);
+ assert(h <= 64);
+ assert(y_step_q4 <= 32 || (y_step_q4 <= 64 && h <= 32));
+ assert(x_step_q4 <= 64);
+
+ if (w >= 8) {
+ scaledconvolve_horiz_w8(src - src_stride * (SUBPEL_TAPS / 2 - 1),
+ src_stride, temp, 64, filter, x0_q4, x_step_q4, w,
+ intermediate_height);
+ } else {
+ scaledconvolve_horiz_w4(src - src_stride * (SUBPEL_TAPS / 2 - 1),
+ src_stride, temp, 64, filter, x0_q4, x_step_q4, w,
+ intermediate_height);
+ }
+
+ if (w >= 16) {
+ scaledconvolve_vert_w16(temp + 64 * (SUBPEL_TAPS / 2 - 1), 64, dst,
+ dst_stride, filter, y0_q4, y_step_q4, w, h);
+ } else if (w == 8) {
+ scaledconvolve_vert_w8(temp + 64 * (SUBPEL_TAPS / 2 - 1), 64, dst,
+ dst_stride, filter, y0_q4, y_step_q4, w, h);
+ } else {
+ scaledconvolve_vert_w4(temp + 64 * (SUBPEL_TAPS / 2 - 1), 64, dst,
+ dst_stride, filter, y0_q4, y_step_q4, w, h);
+ }
+}
diff --git a/av1/common/arm/selfguided_neon.c b/av1/common/arm/selfguided_neon.c
index d14088e..1d3a3cc 100644
--- a/av1/common/arm/selfguided_neon.c
+++ b/av1/common/arm/selfguided_neon.c
@@ -418,16 +418,16 @@
dst1_32_ptr += 2;
dst2_ptr += 2;
load_s16_4x4(src1_ptr, dst_stride_2, &s1, &s2, &s3, &s4);
- transpose_s16_4x4d(&s1, &s2, &s3, &s4);
+ transpose_elems_inplace_s16_4x4(&s1, &s2, &s3, &s4);
load_s32_4x4(src2_ptr, dst_stride_2, &d1, &d2, &d3, &d4);
- transpose_s32_4x4(&d1, &d2, &d3, &d4);
+ transpose_elems_inplace_s32_4x4(&d1, &d2, &d3, &d4);
do {
src1_ptr += 4;
src2_ptr += 4;
load_s16_4x4(src1_ptr, dst_stride_2, &s5, &s6, &s7, &s8);
- transpose_s16_4x4d(&s5, &s6, &s7, &s8);
+ transpose_elems_inplace_s16_4x4(&s5, &s6, &s7, &s8);
load_s32_4x4(src2_ptr, dst_stride_2, &d5, &d6, &d7, &d8);
- transpose_s32_4x4(&d5, &d6, &d7, &d8);
+ transpose_elems_inplace_s32_4x4(&d5, &d6, &d7, &d8);
q23 = vaddl_s16(s2, s3);
q45 = vaddl_s16(s4, s5);
q67 = vaddl_s16(s6, s7);
@@ -438,7 +438,7 @@
q34567 = vaddq_s32(q4567, vmovl_s16(s3));
q45678 = vaddq_s32(q4567, vmovl_s16(s8));
- transpose_s32_4x4(&q12345, &q23456, &q34567, &q45678);
+ transpose_elems_inplace_s32_4x4(&q12345, &q23456, &q34567, &q45678);
store_s32_4x4(dst1_32_ptr, dst_stride_2, q12345, q23456, q34567,
q45678);
dst1_32_ptr += 4;
@@ -457,7 +457,7 @@
r34567 = vaddq_s32(r4567, d3);
r45678 = vaddq_s32(r4567, d8);
- transpose_s32_4x4(&r12345, &r23456, &r34567, &r45678);
+ transpose_elems_inplace_s32_4x4(&r12345, &r23456, &r34567, &r45678);
store_s32_4x4(dst2_ptr, dst_stride_2, r12345, r23456, r34567, r45678);
dst2_ptr += 4;
d1 = d5;
@@ -844,9 +844,9 @@
w = width;
load_s16_4x4((int16_t *)src1_ptr, dst_stride, &d1, &d2, &d3, &d4);
- transpose_s16_4x4d(&d1, &d2, &d3, &d4);
+ transpose_elems_inplace_s16_4x4(&d1, &d2, &d3, &d4);
load_s32_4x4(src2_ptr, dst_stride, &r1, &r2, &r3, &r4);
- transpose_s32_4x4(&r1, &r2, &r3, &r4);
+ transpose_elems_inplace_s32_4x4(&r1, &r2, &r3, &r4);
src1_ptr += 4;
src2_ptr += 4;
@@ -861,9 +861,9 @@
do {
load_s16_4x4((int16_t *)src1_ptr, dst_stride, &d5, &d6, &d7, &d8);
- transpose_s16_4x4d(&d5, &d6, &d7, &d8);
+ transpose_elems_inplace_s16_4x4(&d5, &d6, &d7, &d8);
load_s32_4x4(src2_ptr, dst_stride, &r5, &r6, &r7, &r8);
- transpose_s32_4x4(&r5, &r6, &r7, &r8);
+ transpose_elems_inplace_s32_4x4(&r5, &r6, &r7, &r8);
src1_ptr += 4;
src2_ptr += 4;
@@ -873,7 +873,7 @@
q567 = vadd_s16(d7, q56);
q78 = vadd_s16(d7, d8);
q678 = vadd_s16(d6, q78);
- transpose_s16_4x4d(&q234, &q345, &q456, &q567);
+ transpose_elems_inplace_s16_4x4(&q234, &q345, &q456, &q567);
store_s16_4x4((int16_t *)dst1_ptr, dst_stride, q234, q345, q456, q567);
dst1_ptr += 4;
@@ -887,7 +887,7 @@
r567 = vaddq_s32(r7, r56);
r78 = vaddq_s32(r7, r8);
r678 = vaddq_s32(r6, r78);
- transpose_s32_4x4(&r234, &r345, &r456, &r567);
+ transpose_elems_inplace_s32_4x4(&r234, &r345, &r456, &r567);
store_s32_4x4(dst2_ptr, dst_stride, r234, r345, r456, r567);
dst2_ptr += 4;
@@ -1449,11 +1449,11 @@
return 0;
}
-void av1_apply_selfguided_restoration_neon(const uint8_t *dat8, int width,
- int height, int stride, int eps,
- const int *xqd, uint8_t *dst8,
- int dst_stride, int32_t *tmpbuf,
- int bit_depth, int highbd) {
+int av1_apply_selfguided_restoration_neon(const uint8_t *dat8, int width,
+ int height, int stride, int eps,
+ const int *xqd, uint8_t *dst8,
+ int dst_stride, int32_t *tmpbuf,
+ int bit_depth, int highbd) {
int32_t *flt0 = tmpbuf;
int32_t *flt1 = flt0 + RESTORATION_UNITPELS_MAX;
assert(width * height <= RESTORATION_UNITPELS_MAX);
@@ -1591,4 +1591,5 @@
h--;
} while (h > 0);
}
+ return 0;
}
diff --git a/av1/common/arm/warp_plane_neon.c b/av1/common/arm/warp_plane_neon.c
index b4d3148..4723154 100644
--- a/av1/common/arm/warp_plane_neon.c
+++ b/av1/common/arm/warp_plane_neon.c
@@ -9,463 +9,259 @@
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
-#include <assert.h>
-#include <arm_neon.h>
-#include <memory.h>
-#include <math.h>
+#include "warp_plane_neon.h"
-#include "aom_dsp/aom_dsp_common.h"
-#include "aom_ports/mem.h"
-#include "config/av1_rtcd.h"
-#include "av1/common/warped_motion.h"
-#include "av1/common/scale.h"
-
-/* This is a modified version of 'av1_warped_filter' from warped_motion.c:
- * Each coefficient is stored in 8 bits instead of 16 bits
- * The coefficients are rearranged in the column order 0, 2, 4, 6, 1, 3, 5, 7
-
- This is done in order to avoid overflow: Since the tap with the largest
- coefficient could be any of taps 2, 3, 4 or 5, we can't use the summation
- order ((0 + 1) + (4 + 5)) + ((2 + 3) + (6 + 7)) used in the regular
- convolve functions.
-
- Instead, we use the summation order
- ((0 + 2) + (4 + 6)) + ((1 + 3) + (5 + 7)).
- The rearrangement of coefficients in this table is so that we can get the
- coefficients into the correct order more quickly.
-*/
-/* clang-format off */
-DECLARE_ALIGNED(8, static const int8_t,
- filter_8bit_neon[WARPEDPIXEL_PREC_SHIFTS * 3 + 1][8]) = {
-#if WARPEDPIXEL_PREC_BITS == 6
- // [-1, 0)
- { 0, 127, 0, 0, 0, 1, 0, 0}, { 0, 127, 0, 0, -1, 2, 0, 0},
- { 1, 127, -1, 0, -3, 4, 0, 0}, { 1, 126, -2, 0, -4, 6, 1, 0},
- { 1, 126, -3, 0, -5, 8, 1, 0}, { 1, 125, -4, 0, -6, 11, 1, 0},
- { 1, 124, -4, 0, -7, 13, 1, 0}, { 2, 123, -5, 0, -8, 15, 1, 0},
- { 2, 122, -6, 0, -9, 18, 1, 0}, { 2, 121, -6, 0, -10, 20, 1, 0},
- { 2, 120, -7, 0, -11, 22, 2, 0}, { 2, 119, -8, 0, -12, 25, 2, 0},
- { 3, 117, -8, 0, -13, 27, 2, 0}, { 3, 116, -9, 0, -13, 29, 2, 0},
- { 3, 114, -10, 0, -14, 32, 3, 0}, { 3, 113, -10, 0, -15, 35, 2, 0},
- { 3, 111, -11, 0, -15, 37, 3, 0}, { 3, 109, -11, 0, -16, 40, 3, 0},
- { 3, 108, -12, 0, -16, 42, 3, 0}, { 4, 106, -13, 0, -17, 45, 3, 0},
- { 4, 104, -13, 0, -17, 47, 3, 0}, { 4, 102, -14, 0, -17, 50, 3, 0},
- { 4, 100, -14, 0, -17, 52, 3, 0}, { 4, 98, -15, 0, -18, 55, 4, 0},
- { 4, 96, -15, 0, -18, 58, 3, 0}, { 4, 94, -16, 0, -18, 60, 4, 0},
- { 4, 91, -16, 0, -18, 63, 4, 0}, { 4, 89, -16, 0, -18, 65, 4, 0},
- { 4, 87, -17, 0, -18, 68, 4, 0}, { 4, 85, -17, 0, -18, 70, 4, 0},
- { 4, 82, -17, 0, -18, 73, 4, 0}, { 4, 80, -17, 0, -18, 75, 4, 0},
- { 4, 78, -18, 0, -18, 78, 4, 0}, { 4, 75, -18, 0, -17, 80, 4, 0},
- { 4, 73, -18, 0, -17, 82, 4, 0}, { 4, 70, -18, 0, -17, 85, 4, 0},
- { 4, 68, -18, 0, -17, 87, 4, 0}, { 4, 65, -18, 0, -16, 89, 4, 0},
- { 4, 63, -18, 0, -16, 91, 4, 0}, { 4, 60, -18, 0, -16, 94, 4, 0},
- { 3, 58, -18, 0, -15, 96, 4, 0}, { 4, 55, -18, 0, -15, 98, 4, 0},
- { 3, 52, -17, 0, -14, 100, 4, 0}, { 3, 50, -17, 0, -14, 102, 4, 0},
- { 3, 47, -17, 0, -13, 104, 4, 0}, { 3, 45, -17, 0, -13, 106, 4, 0},
- { 3, 42, -16, 0, -12, 108, 3, 0}, { 3, 40, -16, 0, -11, 109, 3, 0},
- { 3, 37, -15, 0, -11, 111, 3, 0}, { 2, 35, -15, 0, -10, 113, 3, 0},
- { 3, 32, -14, 0, -10, 114, 3, 0}, { 2, 29, -13, 0, -9, 116, 3, 0},
- { 2, 27, -13, 0, -8, 117, 3, 0}, { 2, 25, -12, 0, -8, 119, 2, 0},
- { 2, 22, -11, 0, -7, 120, 2, 0}, { 1, 20, -10, 0, -6, 121, 2, 0},
- { 1, 18, -9, 0, -6, 122, 2, 0}, { 1, 15, -8, 0, -5, 123, 2, 0},
- { 1, 13, -7, 0, -4, 124, 1, 0}, { 1, 11, -6, 0, -4, 125, 1, 0},
- { 1, 8, -5, 0, -3, 126, 1, 0}, { 1, 6, -4, 0, -2, 126, 1, 0},
- { 0, 4, -3, 0, -1, 127, 1, 0}, { 0, 2, -1, 0, 0, 127, 0, 0},
- // [0, 1)
- { 0, 0, 1, 0, 0, 127, 0, 0}, { 0, -1, 2, 0, 0, 127, 0, 0},
- { 0, -3, 4, 1, 1, 127, -2, 0}, { 0, -5, 6, 1, 1, 127, -2, 0},
- { 0, -6, 8, 1, 2, 126, -3, 0}, {-1, -7, 11, 2, 2, 126, -4, -1},
- {-1, -8, 13, 2, 3, 125, -5, -1}, {-1, -10, 16, 3, 3, 124, -6, -1},
- {-1, -11, 18, 3, 4, 123, -7, -1}, {-1, -12, 20, 3, 4, 122, -7, -1},
- {-1, -13, 23, 3, 4, 121, -8, -1}, {-2, -14, 25, 4, 5, 120, -9, -1},
- {-1, -15, 27, 4, 5, 119, -10, -1}, {-1, -16, 30, 4, 5, 118, -11, -1},
- {-2, -17, 33, 5, 6, 116, -12, -1}, {-2, -17, 35, 5, 6, 114, -12, -1},
- {-2, -18, 38, 5, 6, 113, -13, -1}, {-2, -19, 41, 6, 7, 111, -14, -2},
- {-2, -19, 43, 6, 7, 110, -15, -2}, {-2, -20, 46, 6, 7, 108, -15, -2},
- {-2, -20, 49, 6, 7, 106, -16, -2}, {-2, -21, 51, 7, 7, 104, -16, -2},
- {-2, -21, 54, 7, 7, 102, -17, -2}, {-2, -21, 56, 7, 8, 100, -18, -2},
- {-2, -22, 59, 7, 8, 98, -18, -2}, {-2, -22, 62, 7, 8, 96, -19, -2},
- {-2, -22, 64, 7, 8, 94, -19, -2}, {-2, -22, 67, 8, 8, 91, -20, -2},
- {-2, -22, 69, 8, 8, 89, -20, -2}, {-2, -22, 72, 8, 8, 87, -21, -2},
- {-2, -21, 74, 8, 8, 84, -21, -2}, {-2, -22, 77, 8, 8, 82, -21, -2},
- {-2, -21, 79, 8, 8, 79, -21, -2}, {-2, -21, 82, 8, 8, 77, -22, -2},
- {-2, -21, 84, 8, 8, 74, -21, -2}, {-2, -21, 87, 8, 8, 72, -22, -2},
- {-2, -20, 89, 8, 8, 69, -22, -2}, {-2, -20, 91, 8, 8, 67, -22, -2},
- {-2, -19, 94, 8, 7, 64, -22, -2}, {-2, -19, 96, 8, 7, 62, -22, -2},
- {-2, -18, 98, 8, 7, 59, -22, -2}, {-2, -18, 100, 8, 7, 56, -21, -2},
- {-2, -17, 102, 7, 7, 54, -21, -2}, {-2, -16, 104, 7, 7, 51, -21, -2},
- {-2, -16, 106, 7, 6, 49, -20, -2}, {-2, -15, 108, 7, 6, 46, -20, -2},
- {-2, -15, 110, 7, 6, 43, -19, -2}, {-2, -14, 111, 7, 6, 41, -19, -2},
- {-1, -13, 113, 6, 5, 38, -18, -2}, {-1, -12, 114, 6, 5, 35, -17, -2},
- {-1, -12, 116, 6, 5, 33, -17, -2}, {-1, -11, 118, 5, 4, 30, -16, -1},
- {-1, -10, 119, 5, 4, 27, -15, -1}, {-1, -9, 120, 5, 4, 25, -14, -2},
- {-1, -8, 121, 4, 3, 23, -13, -1}, {-1, -7, 122, 4, 3, 20, -12, -1},
- {-1, -7, 123, 4, 3, 18, -11, -1}, {-1, -6, 124, 3, 3, 16, -10, -1},
- {-1, -5, 125, 3, 2, 13, -8, -1}, {-1, -4, 126, 2, 2, 11, -7, -1},
- { 0, -3, 126, 2, 1, 8, -6, 0}, { 0, -2, 127, 1, 1, 6, -5, 0},
- { 0, -2, 127, 1, 1, 4, -3, 0}, { 0, 0, 127, 0, 0, 2, -1, 0},
- // [1, 2)
- { 0, 0, 127, 0, 0, 1, 0, 0}, { 0, 0, 127, 0, 0, -1, 2, 0},
- { 0, 1, 127, -1, 0, -3, 4, 0}, { 0, 1, 126, -2, 0, -4, 6, 1},
- { 0, 1, 126, -3, 0, -5, 8, 1}, { 0, 1, 125, -4, 0, -6, 11, 1},
- { 0, 1, 124, -4, 0, -7, 13, 1}, { 0, 2, 123, -5, 0, -8, 15, 1},
- { 0, 2, 122, -6, 0, -9, 18, 1}, { 0, 2, 121, -6, 0, -10, 20, 1},
- { 0, 2, 120, -7, 0, -11, 22, 2}, { 0, 2, 119, -8, 0, -12, 25, 2},
- { 0, 3, 117, -8, 0, -13, 27, 2}, { 0, 3, 116, -9, 0, -13, 29, 2},
- { 0, 3, 114, -10, 0, -14, 32, 3}, { 0, 3, 113, -10, 0, -15, 35, 2},
- { 0, 3, 111, -11, 0, -15, 37, 3}, { 0, 3, 109, -11, 0, -16, 40, 3},
- { 0, 3, 108, -12, 0, -16, 42, 3}, { 0, 4, 106, -13, 0, -17, 45, 3},
- { 0, 4, 104, -13, 0, -17, 47, 3}, { 0, 4, 102, -14, 0, -17, 50, 3},
- { 0, 4, 100, -14, 0, -17, 52, 3}, { 0, 4, 98, -15, 0, -18, 55, 4},
- { 0, 4, 96, -15, 0, -18, 58, 3}, { 0, 4, 94, -16, 0, -18, 60, 4},
- { 0, 4, 91, -16, 0, -18, 63, 4}, { 0, 4, 89, -16, 0, -18, 65, 4},
- { 0, 4, 87, -17, 0, -18, 68, 4}, { 0, 4, 85, -17, 0, -18, 70, 4},
- { 0, 4, 82, -17, 0, -18, 73, 4}, { 0, 4, 80, -17, 0, -18, 75, 4},
- { 0, 4, 78, -18, 0, -18, 78, 4}, { 0, 4, 75, -18, 0, -17, 80, 4},
- { 0, 4, 73, -18, 0, -17, 82, 4}, { 0, 4, 70, -18, 0, -17, 85, 4},
- { 0, 4, 68, -18, 0, -17, 87, 4}, { 0, 4, 65, -18, 0, -16, 89, 4},
- { 0, 4, 63, -18, 0, -16, 91, 4}, { 0, 4, 60, -18, 0, -16, 94, 4},
- { 0, 3, 58, -18, 0, -15, 96, 4}, { 0, 4, 55, -18, 0, -15, 98, 4},
- { 0, 3, 52, -17, 0, -14, 100, 4}, { 0, 3, 50, -17, 0, -14, 102, 4},
- { 0, 3, 47, -17, 0, -13, 104, 4}, { 0, 3, 45, -17, 0, -13, 106, 4},
- { 0, 3, 42, -16, 0, -12, 108, 3}, { 0, 3, 40, -16, 0, -11, 109, 3},
- { 0, 3, 37, -15, 0, -11, 111, 3}, { 0, 2, 35, -15, 0, -10, 113, 3},
- { 0, 3, 32, -14, 0, -10, 114, 3}, { 0, 2, 29, -13, 0, -9, 116, 3},
- { 0, 2, 27, -13, 0, -8, 117, 3}, { 0, 2, 25, -12, 0, -8, 119, 2},
- { 0, 2, 22, -11, 0, -7, 120, 2}, { 0, 1, 20, -10, 0, -6, 121, 2},
- { 0, 1, 18, -9, 0, -6, 122, 2}, { 0, 1, 15, -8, 0, -5, 123, 2},
- { 0, 1, 13, -7, 0, -4, 124, 1}, { 0, 1, 11, -6, 0, -4, 125, 1},
- { 0, 1, 8, -5, 0, -3, 126, 1}, { 0, 1, 6, -4, 0, -2, 126, 1},
- { 0, 0, 4, -3, 0, -1, 127, 1}, { 0, 0, 2, -1, 0, 0, 127, 0},
- // dummy (replicate row index 191)
- { 0, 0, 2, -1, 0, 0, 127, 0},
-
-#else
- // [-1, 0)
- { 0, 127, 0, 0, 0, 1, 0, 0}, { 1, 127, -1, 0, -3, 4, 0, 0},
- { 1, 126, -3, 0, -5, 8, 1, 0}, { 1, 124, -4, 0, -7, 13, 1, 0},
- { 2, 122, -6, 0, -9, 18, 1, 0}, { 2, 120, -7, 0, -11, 22, 2, 0},
- { 3, 117, -8, 0, -13, 27, 2, 0}, { 3, 114, -10, 0, -14, 32, 3, 0},
- { 3, 111, -11, 0, -15, 37, 3, 0}, { 3, 108, -12, 0, -16, 42, 3, 0},
- { 4, 104, -13, 0, -17, 47, 3, 0}, { 4, 100, -14, 0, -17, 52, 3, 0},
- { 4, 96, -15, 0, -18, 58, 3, 0}, { 4, 91, -16, 0, -18, 63, 4, 0},
- { 4, 87, -17, 0, -18, 68, 4, 0}, { 4, 82, -17, 0, -18, 73, 4, 0},
- { 4, 78, -18, 0, -18, 78, 4, 0}, { 4, 73, -18, 0, -17, 82, 4, 0},
- { 4, 68, -18, 0, -17, 87, 4, 0}, { 4, 63, -18, 0, -16, 91, 4, 0},
- { 3, 58, -18, 0, -15, 96, 4, 0}, { 3, 52, -17, 0, -14, 100, 4, 0},
- { 3, 47, -17, 0, -13, 104, 4, 0}, { 3, 42, -16, 0, -12, 108, 3, 0},
- { 3, 37, -15, 0, -11, 111, 3, 0}, { 3, 32, -14, 0, -10, 114, 3, 0},
- { 2, 27, -13, 0, -8, 117, 3, 0}, { 2, 22, -11, 0, -7, 120, 2, 0},
- { 1, 18, -9, 0, -6, 122, 2, 0}, { 1, 13, -7, 0, -4, 124, 1, 0},
- { 1, 8, -5, 0, -3, 126, 1, 0}, { 0, 4, -3, 0, -1, 127, 1, 0},
- // [0, 1)
- { 0, 0, 1, 0, 0, 127, 0, 0}, { 0, -3, 4, 1, 1, 127, -2, 0},
- { 0, -6, 8, 1, 2, 126, -3, 0}, {-1, -8, 13, 2, 3, 125, -5, -1},
- {-1, -11, 18, 3, 4, 123, -7, -1}, {-1, -13, 23, 3, 4, 121, -8, -1},
- {-1, -15, 27, 4, 5, 119, -10, -1}, {-2, -17, 33, 5, 6, 116, -12, -1},
- {-2, -18, 38, 5, 6, 113, -13, -1}, {-2, -19, 43, 6, 7, 110, -15, -2},
- {-2, -20, 49, 6, 7, 106, -16, -2}, {-2, -21, 54, 7, 7, 102, -17, -2},
- {-2, -22, 59, 7, 8, 98, -18, -2}, {-2, -22, 64, 7, 8, 94, -19, -2},
- {-2, -22, 69, 8, 8, 89, -20, -2}, {-2, -21, 74, 8, 8, 84, -21, -2},
- {-2, -21, 79, 8, 8, 79, -21, -2}, {-2, -21, 84, 8, 8, 74, -21, -2},
- {-2, -20, 89, 8, 8, 69, -22, -2}, {-2, -19, 94, 8, 7, 64, -22, -2},
- {-2, -18, 98, 8, 7, 59, -22, -2}, {-2, -17, 102, 7, 7, 54, -21, -2},
- {-2, -16, 106, 7, 6, 49, -20, -2}, {-2, -15, 110, 7, 6, 43, -19, -2},
- {-1, -13, 113, 6, 5, 38, -18, -2}, {-1, -12, 116, 6, 5, 33, -17, -2},
- {-1, -10, 119, 5, 4, 27, -15, -1}, {-1, -8, 121, 4, 3, 23, -13, -1},
- {-1, -7, 123, 4, 3, 18, -11, -1}, {-1, -5, 125, 3, 2, 13, -8, -1},
- { 0, -3, 126, 2, 1, 8, -6, 0}, { 0, -2, 127, 1, 1, 4, -3, 0},
- // [1, 2)
- { 0, 0, 127, 0, 0, 1, 0, 0}, { 0, 1, 127, -1, 0, -3, 4, 0},
- { 0, 1, 126, -3, 0, -5, 8, 1}, { 0, 1, 124, -4, 0, -7, 13, 1},
- { 0, 2, 122, -6, 0, -9, 18, 1}, { 0, 2, 120, -7, 0, -11, 22, 2},
- { 0, 3, 117, -8, 0, -13, 27, 2}, { 0, 3, 114, -10, 0, -14, 32, 3},
- { 0, 3, 111, -11, 0, -15, 37, 3}, { 0, 3, 108, -12, 0, -16, 42, 3},
- { 0, 4, 104, -13, 0, -17, 47, 3}, { 0, 4, 100, -14, 0, -17, 52, 3},
- { 0, 4, 96, -15, 0, -18, 58, 3}, { 0, 4, 91, -16, 0, -18, 63, 4},
- { 0, 4, 87, -17, 0, -18, 68, 4}, { 0, 4, 82, -17, 0, -18, 73, 4},
- { 0, 4, 78, -18, 0, -18, 78, 4}, { 0, 4, 73, -18, 0, -17, 82, 4},
- { 0, 4, 68, -18, 0, -17, 87, 4}, { 0, 4, 63, -18, 0, -16, 91, 4},
- { 0, 3, 58, -18, 0, -15, 96, 4}, { 0, 3, 52, -17, 0, -14, 100, 4},
- { 0, 3, 47, -17, 0, -13, 104, 4}, { 0, 3, 42, -16, 0, -12, 108, 3},
- { 0, 3, 37, -15, 0, -11, 111, 3}, { 0, 3, 32, -14, 0, -10, 114, 3},
- { 0, 2, 27, -13, 0, -8, 117, 3}, { 0, 2, 22, -11, 0, -7, 120, 2},
- { 0, 1, 18, -9, 0, -6, 122, 2}, { 0, 1, 13, -7, 0, -4, 124, 1},
- { 0, 1, 8, -5, 0, -3, 126, 1}, { 0, 0, 4, -3, 0, -1, 127, 1},
- // dummy (replicate row index 95)
- { 0, 0, 4, -3, 0, -1, 127, 1},
-#endif // WARPEDPIXEL_PREC_BITS == 6
-};
-/* clang-format on */
-
-static INLINE void convolve(int32x2x2_t x0, int32x2x2_t x1, uint8x8_t src_0,
- uint8x8_t src_1, int16x4_t *res) {
- int16x8_t coeff_0, coeff_1;
- int16x8_t pix_0, pix_1;
-
- coeff_0 = vcombine_s16(vreinterpret_s16_s32(x0.val[0]),
- vreinterpret_s16_s32(x1.val[0]));
- coeff_1 = vcombine_s16(vreinterpret_s16_s32(x0.val[1]),
- vreinterpret_s16_s32(x1.val[1]));
-
- pix_0 = vreinterpretq_s16_u16(vmovl_u8(src_0));
- pix_0 = vmulq_s16(coeff_0, pix_0);
-
- pix_1 = vreinterpretq_s16_u16(vmovl_u8(src_1));
- pix_0 = vmlaq_s16(pix_0, coeff_1, pix_1);
-
- *res = vpadd_s16(vget_low_s16(pix_0), vget_high_s16(pix_0));
-}
-
-static INLINE void horizontal_filter_neon(uint8x16_t src_1, uint8x16_t src_2,
- uint8x16_t src_3, uint8x16_t src_4,
- int16x8_t *tmp_dst, int sx, int alpha,
- int k, const int offset_bits_horiz,
- const int reduce_bits_horiz) {
- const uint8x16_t mask = vreinterpretq_u8_u16(vdupq_n_u16(0x00ff));
- const int32x4_t add_const = vdupq_n_s32((int32_t)(1 << offset_bits_horiz));
- const int16x8_t shift = vdupq_n_s16(-(int16_t)reduce_bits_horiz);
-
- int16x8_t f0, f1, f2, f3, f4, f5, f6, f7;
- int32x2x2_t b0, b1;
- uint8x8_t src_1_low, src_2_low, src_3_low, src_4_low, src_5_low, src_6_low;
- int32x4_t tmp_res_low, tmp_res_high;
- uint16x8_t res;
- int16x4_t res_0246_even, res_0246_odd, res_1357_even, res_1357_odd;
-
- uint8x16_t tmp_0 = vandq_u8(src_1, mask);
- uint8x16_t tmp_1 = vandq_u8(src_2, mask);
- uint8x16_t tmp_2 = vandq_u8(src_3, mask);
- uint8x16_t tmp_3 = vandq_u8(src_4, mask);
-
- tmp_2 = vextq_u8(tmp_0, tmp_0, 1);
- tmp_3 = vextq_u8(tmp_1, tmp_1, 1);
-
- src_1 = vaddq_u8(tmp_0, tmp_2);
- src_2 = vaddq_u8(tmp_1, tmp_3);
-
- src_1_low = vget_low_u8(src_1);
- src_2_low = vget_low_u8(src_2);
- src_3_low = vget_low_u8(vextq_u8(src_1, src_1, 4));
- src_4_low = vget_low_u8(vextq_u8(src_2, src_2, 4));
- src_5_low = vget_low_u8(vextq_u8(src_1, src_1, 2));
- src_6_low = vget_low_u8(vextq_u8(src_1, src_1, 6));
+static INLINE int16x8_t horizontal_filter_4x1_f4(const uint8x16_t in, int sx,
+ int alpha) {
+ const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1));
// Loading the 8 filter taps
- f0 = vmovl_s8(
- vld1_s8(filter_8bit_neon[(sx + 0 * alpha) >> WARPEDDIFF_PREC_BITS]));
- f1 = vmovl_s8(
- vld1_s8(filter_8bit_neon[(sx + 1 * alpha) >> WARPEDDIFF_PREC_BITS]));
- f2 = vmovl_s8(
- vld1_s8(filter_8bit_neon[(sx + 2 * alpha) >> WARPEDDIFF_PREC_BITS]));
- f3 = vmovl_s8(
- vld1_s8(filter_8bit_neon[(sx + 3 * alpha) >> WARPEDDIFF_PREC_BITS]));
- f4 = vmovl_s8(
- vld1_s8(filter_8bit_neon[(sx + 4 * alpha) >> WARPEDDIFF_PREC_BITS]));
- f5 = vmovl_s8(
- vld1_s8(filter_8bit_neon[(sx + 5 * alpha) >> WARPEDDIFF_PREC_BITS]));
- f6 = vmovl_s8(
- vld1_s8(filter_8bit_neon[(sx + 6 * alpha) >> WARPEDDIFF_PREC_BITS]));
- f7 = vmovl_s8(
- vld1_s8(filter_8bit_neon[(sx + 7 * alpha) >> WARPEDDIFF_PREC_BITS]));
+ int16x8_t f[4];
+ load_filters_4(f, sx, alpha);
- b0 = vtrn_s32(vreinterpret_s32_s16(vget_low_s16(f0)),
- vreinterpret_s32_s16(vget_low_s16(f2)));
- b1 = vtrn_s32(vreinterpret_s32_s16(vget_low_s16(f4)),
- vreinterpret_s32_s16(vget_low_s16(f6)));
- convolve(b0, b1, src_1_low, src_3_low, &res_0246_even);
+ int16x8_t in16_lo = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(in)));
+ int16x8_t in16_hi = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(in)));
- b0 = vtrn_s32(vreinterpret_s32_s16(vget_low_s16(f1)),
- vreinterpret_s32_s16(vget_low_s16(f3)));
- b1 = vtrn_s32(vreinterpret_s32_s16(vget_low_s16(f5)),
- vreinterpret_s32_s16(vget_low_s16(f7)));
- convolve(b0, b1, src_2_low, src_4_low, &res_0246_odd);
+ int16x8_t m0 = vmulq_s16(f[0], in16_lo);
+ int16x8_t m1 = vmulq_s16(f[1], vextq_s16(in16_lo, in16_hi, 1));
+ int16x8_t m2 = vmulq_s16(f[2], vextq_s16(in16_lo, in16_hi, 2));
+ int16x8_t m3 = vmulq_s16(f[3], vextq_s16(in16_lo, in16_hi, 3));
- b0 = vtrn_s32(vreinterpret_s32_s16(vget_high_s16(f0)),
- vreinterpret_s32_s16(vget_high_s16(f2)));
- b1 = vtrn_s32(vreinterpret_s32_s16(vget_high_s16(f4)),
- vreinterpret_s32_s16(vget_high_s16(f6)));
- convolve(b0, b1, src_2_low, src_4_low, &res_1357_even);
+ int32x4_t m0123_pairs[] = { vpaddlq_s16(m0), vpaddlq_s16(m1), vpaddlq_s16(m2),
+ vpaddlq_s16(m3) };
- b0 = vtrn_s32(vreinterpret_s32_s16(vget_high_s16(f1)),
- vreinterpret_s32_s16(vget_high_s16(f3)));
- b1 = vtrn_s32(vreinterpret_s32_s16(vget_high_s16(f5)),
- vreinterpret_s32_s16(vget_high_s16(f7)));
- convolve(b0, b1, src_5_low, src_6_low, &res_1357_odd);
+ int32x4_t tmp_res_low = horizontal_add_4d_s32x4(m0123_pairs);
- tmp_res_low = vaddl_s16(res_0246_even, res_1357_even);
- tmp_res_high = vaddl_s16(res_0246_odd, res_1357_odd);
+ tmp_res_low = vaddq_s32(tmp_res_low, add_const);
+
+ uint16x8_t res =
+ vcombine_u16(vqrshrun_n_s32(tmp_res_low, ROUND0_BITS), vdup_n_u16(0));
+ return vreinterpretq_s16_u16(res);
+}
+
+static INLINE int16x8_t horizontal_filter_8x1_f8(const uint8x16_t in, int sx,
+ int alpha) {
+ const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1));
+
+ // Loading the 8 filter taps
+ int16x8_t f[8];
+ load_filters_8(f, sx, alpha);
+
+ int16x8_t in16_lo = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(in)));
+ int16x8_t in16_hi = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(in)));
+
+ int16x8_t m0 = vmulq_s16(f[0], in16_lo);
+ int16x8_t m1 = vmulq_s16(f[1], vextq_s16(in16_lo, in16_hi, 1));
+ int16x8_t m2 = vmulq_s16(f[2], vextq_s16(in16_lo, in16_hi, 2));
+ int16x8_t m3 = vmulq_s16(f[3], vextq_s16(in16_lo, in16_hi, 3));
+ int16x8_t m4 = vmulq_s16(f[4], vextq_s16(in16_lo, in16_hi, 4));
+ int16x8_t m5 = vmulq_s16(f[5], vextq_s16(in16_lo, in16_hi, 5));
+ int16x8_t m6 = vmulq_s16(f[6], vextq_s16(in16_lo, in16_hi, 6));
+ int16x8_t m7 = vmulq_s16(f[7], vextq_s16(in16_lo, in16_hi, 7));
+
+ int32x4_t m0123_pairs[] = { vpaddlq_s16(m0), vpaddlq_s16(m1), vpaddlq_s16(m2),
+ vpaddlq_s16(m3) };
+ int32x4_t m4567_pairs[] = { vpaddlq_s16(m4), vpaddlq_s16(m5), vpaddlq_s16(m6),
+ vpaddlq_s16(m7) };
+
+ int32x4_t tmp_res_low = horizontal_add_4d_s32x4(m0123_pairs);
+ int32x4_t tmp_res_high = horizontal_add_4d_s32x4(m4567_pairs);
tmp_res_low = vaddq_s32(tmp_res_low, add_const);
tmp_res_high = vaddq_s32(tmp_res_high, add_const);
- res = vcombine_u16(vqmovun_s32(tmp_res_low), vqmovun_s32(tmp_res_high));
- res = vqrshlq_u16(res, shift);
-
- tmp_dst[k + 7] = vreinterpretq_s16_u16(res);
+ uint16x8_t res = vcombine_u16(vqrshrun_n_s32(tmp_res_low, ROUND0_BITS),
+ vqrshrun_n_s32(tmp_res_high, ROUND0_BITS));
+ return vreinterpretq_s16_u16(res);
}
-static INLINE void vertical_filter_neon(const int16x8_t *src,
- int32x4_t *res_low, int32x4_t *res_high,
- int sy, int gamma) {
- int16x4_t src_0, src_1, fltr_0, fltr_1;
- int32x4_t res_0, res_1;
- int32x2_t res_0_im, res_1_im;
- int32x4_t res_even, res_odd, im_res_0, im_res_1;
+static INLINE int16x8_t horizontal_filter_4x1_f1(const uint8x16_t in, int sx) {
+ const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1));
- int16x8_t f0, f1, f2, f3, f4, f5, f6, f7;
- int16x8x2_t b0, b1, b2, b3;
- int32x4x2_t c0, c1, c2, c3;
- int32x4x2_t d0, d1, d2, d3;
+ int16x8_t f_s16 =
+ vld1q_s16((int16_t *)(av1_warped_filter + (sx >> WARPEDDIFF_PREC_BITS)));
- b0 = vtrnq_s16(src[0], src[1]);
- b1 = vtrnq_s16(src[2], src[3]);
- b2 = vtrnq_s16(src[4], src[5]);
- b3 = vtrnq_s16(src[6], src[7]);
+ int16x8_t in16_lo = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(in)));
+ int16x8_t in16_hi = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(in)));
- c0 = vtrnq_s32(vreinterpretq_s32_s16(b0.val[0]),
- vreinterpretq_s32_s16(b0.val[1]));
- c1 = vtrnq_s32(vreinterpretq_s32_s16(b1.val[0]),
- vreinterpretq_s32_s16(b1.val[1]));
- c2 = vtrnq_s32(vreinterpretq_s32_s16(b2.val[0]),
- vreinterpretq_s32_s16(b2.val[1]));
- c3 = vtrnq_s32(vreinterpretq_s32_s16(b3.val[0]),
- vreinterpretq_s32_s16(b3.val[1]));
+ int16x8_t m0 = vmulq_s16(f_s16, in16_lo);
+ int16x8_t m1 = vmulq_s16(f_s16, vextq_s16(in16_lo, in16_hi, 1));
+ int16x8_t m2 = vmulq_s16(f_s16, vextq_s16(in16_lo, in16_hi, 2));
+ int16x8_t m3 = vmulq_s16(f_s16, vextq_s16(in16_lo, in16_hi, 3));
- f0 = vld1q_s16((int16_t *)(av1_warped_filter +
- ((sy + 0 * gamma) >> WARPEDDIFF_PREC_BITS)));
- f1 = vld1q_s16((int16_t *)(av1_warped_filter +
- ((sy + 1 * gamma) >> WARPEDDIFF_PREC_BITS)));
- f2 = vld1q_s16((int16_t *)(av1_warped_filter +
- ((sy + 2 * gamma) >> WARPEDDIFF_PREC_BITS)));
- f3 = vld1q_s16((int16_t *)(av1_warped_filter +
- ((sy + 3 * gamma) >> WARPEDDIFF_PREC_BITS)));
- f4 = vld1q_s16((int16_t *)(av1_warped_filter +
- ((sy + 4 * gamma) >> WARPEDDIFF_PREC_BITS)));
- f5 = vld1q_s16((int16_t *)(av1_warped_filter +
- ((sy + 5 * gamma) >> WARPEDDIFF_PREC_BITS)));
- f6 = vld1q_s16((int16_t *)(av1_warped_filter +
- ((sy + 6 * gamma) >> WARPEDDIFF_PREC_BITS)));
- f7 = vld1q_s16((int16_t *)(av1_warped_filter +
- ((sy + 7 * gamma) >> WARPEDDIFF_PREC_BITS)));
+ int32x4_t m0123_pairs[] = { vpaddlq_s16(m0), vpaddlq_s16(m1), vpaddlq_s16(m2),
+ vpaddlq_s16(m3) };
- d0 = vtrnq_s32(vreinterpretq_s32_s16(f0), vreinterpretq_s32_s16(f2));
- d1 = vtrnq_s32(vreinterpretq_s32_s16(f4), vreinterpretq_s32_s16(f6));
- d2 = vtrnq_s32(vreinterpretq_s32_s16(f1), vreinterpretq_s32_s16(f3));
- d3 = vtrnq_s32(vreinterpretq_s32_s16(f5), vreinterpretq_s32_s16(f7));
+ int32x4_t tmp_res_low = horizontal_add_4d_s32x4(m0123_pairs);
- // row:0,1 even_col:0,2
- src_0 = vget_low_s16(vreinterpretq_s16_s32(c0.val[0]));
- fltr_0 = vget_low_s16(vreinterpretq_s16_s32(d0.val[0]));
- res_0 = vmull_s16(src_0, fltr_0);
+ tmp_res_low = vaddq_s32(tmp_res_low, add_const);
- // row:0,1,2,3 even_col:0,2
- src_0 = vget_low_s16(vreinterpretq_s16_s32(c1.val[0]));
- fltr_0 = vget_low_s16(vreinterpretq_s16_s32(d0.val[1]));
- res_0 = vmlal_s16(res_0, src_0, fltr_0);
- res_0_im = vpadd_s32(vget_low_s32(res_0), vget_high_s32(res_0));
+ uint16x8_t res =
+ vcombine_u16(vqrshrun_n_s32(tmp_res_low, ROUND0_BITS), vdup_n_u16(0));
+ return vreinterpretq_s16_u16(res);
+}
- // row:0,1 even_col:4,6
- src_1 = vget_low_s16(vreinterpretq_s16_s32(c0.val[1]));
- fltr_1 = vget_low_s16(vreinterpretq_s16_s32(d1.val[0]));
- res_1 = vmull_s16(src_1, fltr_1);
+static INLINE int16x8_t horizontal_filter_8x1_f1(const uint8x16_t in, int sx) {
+ const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1));
- // row:0,1,2,3 even_col:4,6
- src_1 = vget_low_s16(vreinterpretq_s16_s32(c1.val[1]));
- fltr_1 = vget_low_s16(vreinterpretq_s16_s32(d1.val[1]));
- res_1 = vmlal_s16(res_1, src_1, fltr_1);
- res_1_im = vpadd_s32(vget_low_s32(res_1), vget_high_s32(res_1));
+ int16x8_t f_s16 =
+ vld1q_s16((int16_t *)(av1_warped_filter + (sx >> WARPEDDIFF_PREC_BITS)));
- // row:0,1,2,3 even_col:0,2,4,6
- im_res_0 = vcombine_s32(res_0_im, res_1_im);
+ int16x8_t in16_lo = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(in)));
+ int16x8_t in16_hi = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(in)));
- // row:4,5 even_col:0,2
- src_0 = vget_low_s16(vreinterpretq_s16_s32(c2.val[0]));
- fltr_0 = vget_high_s16(vreinterpretq_s16_s32(d0.val[0]));
- res_0 = vmull_s16(src_0, fltr_0);
+ int16x8_t m0 = vmulq_s16(f_s16, in16_lo);
+ int16x8_t m1 = vmulq_s16(f_s16, vextq_s16(in16_lo, in16_hi, 1));
+ int16x8_t m2 = vmulq_s16(f_s16, vextq_s16(in16_lo, in16_hi, 2));
+ int16x8_t m3 = vmulq_s16(f_s16, vextq_s16(in16_lo, in16_hi, 3));
+ int16x8_t m4 = vmulq_s16(f_s16, vextq_s16(in16_lo, in16_hi, 4));
+ int16x8_t m5 = vmulq_s16(f_s16, vextq_s16(in16_lo, in16_hi, 5));
+ int16x8_t m6 = vmulq_s16(f_s16, vextq_s16(in16_lo, in16_hi, 6));
+ int16x8_t m7 = vmulq_s16(f_s16, vextq_s16(in16_lo, in16_hi, 7));
- // row:4,5,6,7 even_col:0,2
- src_0 = vget_low_s16(vreinterpretq_s16_s32(c3.val[0]));
- fltr_0 = vget_high_s16(vreinterpretq_s16_s32(d0.val[1]));
- res_0 = vmlal_s16(res_0, src_0, fltr_0);
- res_0_im = vpadd_s32(vget_low_s32(res_0), vget_high_s32(res_0));
+ int32x4_t m0123_pairs[] = { vpaddlq_s16(m0), vpaddlq_s16(m1), vpaddlq_s16(m2),
+ vpaddlq_s16(m3) };
+ int32x4_t m4567_pairs[] = { vpaddlq_s16(m4), vpaddlq_s16(m5), vpaddlq_s16(m6),
+ vpaddlq_s16(m7) };
- // row:4,5 even_col:4,6
- src_1 = vget_low_s16(vreinterpretq_s16_s32(c2.val[1]));
- fltr_1 = vget_high_s16(vreinterpretq_s16_s32(d1.val[0]));
- res_1 = vmull_s16(src_1, fltr_1);
+ int32x4_t tmp_res_low = horizontal_add_4d_s32x4(m0123_pairs);
+ int32x4_t tmp_res_high = horizontal_add_4d_s32x4(m4567_pairs);
- // row:4,5,6,7 even_col:4,6
- src_1 = vget_low_s16(vreinterpretq_s16_s32(c3.val[1]));
- fltr_1 = vget_high_s16(vreinterpretq_s16_s32(d1.val[1]));
- res_1 = vmlal_s16(res_1, src_1, fltr_1);
- res_1_im = vpadd_s32(vget_low_s32(res_1), vget_high_s32(res_1));
+ tmp_res_low = vaddq_s32(tmp_res_low, add_const);
+ tmp_res_high = vaddq_s32(tmp_res_high, add_const);
- // row:4,5,6,7 even_col:0,2,4,6
- im_res_1 = vcombine_s32(res_0_im, res_1_im);
+ uint16x8_t res = vcombine_u16(vqrshrun_n_s32(tmp_res_low, ROUND0_BITS),
+ vqrshrun_n_s32(tmp_res_high, ROUND0_BITS));
+ return vreinterpretq_s16_u16(res);
+}
- // row:0-7 even_col:0,2,4,6
- res_even = vaddq_s32(im_res_0, im_res_1);
+static INLINE void vertical_filter_4x1_f1(const int16x8_t *src, int32x4_t *res,
+ int sy) {
+ int16x4_t s0 = vget_low_s16(src[0]);
+ int16x4_t s1 = vget_low_s16(src[1]);
+ int16x4_t s2 = vget_low_s16(src[2]);
+ int16x4_t s3 = vget_low_s16(src[3]);
+ int16x4_t s4 = vget_low_s16(src[4]);
+ int16x4_t s5 = vget_low_s16(src[5]);
+ int16x4_t s6 = vget_low_s16(src[6]);
+ int16x4_t s7 = vget_low_s16(src[7]);
- // row:0,1 odd_col:1,3
- src_0 = vget_high_s16(vreinterpretq_s16_s32(c0.val[0]));
- fltr_0 = vget_low_s16(vreinterpretq_s16_s32(d2.val[0]));
- res_0 = vmull_s16(src_0, fltr_0);
+ int16x8_t f =
+ vld1q_s16((int16_t *)(av1_warped_filter + (sy >> WARPEDDIFF_PREC_BITS)));
- // row:0,1,2,3 odd_col:1,3
- src_0 = vget_high_s16(vreinterpretq_s16_s32(c1.val[0]));
- fltr_0 = vget_low_s16(vreinterpretq_s16_s32(d2.val[1]));
- res_0 = vmlal_s16(res_0, src_0, fltr_0);
- res_0_im = vpadd_s32(vget_low_s32(res_0), vget_high_s32(res_0));
+ int32x4_t m0123 = vmull_lane_s16(s0, vget_low_s16(f), 0);
+ m0123 = vmlal_lane_s16(m0123, s1, vget_low_s16(f), 1);
+ m0123 = vmlal_lane_s16(m0123, s2, vget_low_s16(f), 2);
+ m0123 = vmlal_lane_s16(m0123, s3, vget_low_s16(f), 3);
+ m0123 = vmlal_lane_s16(m0123, s4, vget_high_s16(f), 0);
+ m0123 = vmlal_lane_s16(m0123, s5, vget_high_s16(f), 1);
+ m0123 = vmlal_lane_s16(m0123, s6, vget_high_s16(f), 2);
+ m0123 = vmlal_lane_s16(m0123, s7, vget_high_s16(f), 3);
- // row:0,1 odd_col:5,7
- src_1 = vget_high_s16(vreinterpretq_s16_s32(c0.val[1]));
- fltr_1 = vget_low_s16(vreinterpretq_s16_s32(d3.val[0]));
- res_1 = vmull_s16(src_1, fltr_1);
+ *res = m0123;
+}
- // row:0,1,2,3 odd_col:5,7
- src_1 = vget_high_s16(vreinterpretq_s16_s32(c1.val[1]));
- fltr_1 = vget_low_s16(vreinterpretq_s16_s32(d3.val[1]));
- res_1 = vmlal_s16(res_1, src_1, fltr_1);
- res_1_im = vpadd_s32(vget_low_s32(res_1), vget_high_s32(res_1));
+static INLINE void vertical_filter_4x1_f4(const int16x8_t *src, int32x4_t *res,
+ int sy, int gamma) {
+ int16x8_t s0, s1, s2, s3;
+ transpose_elems_s16_4x8(
+ vget_low_s16(src[0]), vget_low_s16(src[1]), vget_low_s16(src[2]),
+ vget_low_s16(src[3]), vget_low_s16(src[4]), vget_low_s16(src[5]),
+ vget_low_s16(src[6]), vget_low_s16(src[7]), &s0, &s1, &s2, &s3);
- // row:0,1,2,3 odd_col:1,3,5,7
- im_res_0 = vcombine_s32(res_0_im, res_1_im);
+ int16x8_t f[4];
+ load_filters_4(f, sy, gamma);
- // row:4,5 odd_col:1,3
- src_0 = vget_high_s16(vreinterpretq_s16_s32(c2.val[0]));
- fltr_0 = vget_high_s16(vreinterpretq_s16_s32(d2.val[0]));
- res_0 = vmull_s16(src_0, fltr_0);
+ int32x4_t m0 = vmull_s16(vget_low_s16(s0), vget_low_s16(f[0]));
+ m0 = vmlal_s16(m0, vget_high_s16(s0), vget_high_s16(f[0]));
+ int32x4_t m1 = vmull_s16(vget_low_s16(s1), vget_low_s16(f[1]));
+ m1 = vmlal_s16(m1, vget_high_s16(s1), vget_high_s16(f[1]));
+ int32x4_t m2 = vmull_s16(vget_low_s16(s2), vget_low_s16(f[2]));
+ m2 = vmlal_s16(m2, vget_high_s16(s2), vget_high_s16(f[2]));
+ int32x4_t m3 = vmull_s16(vget_low_s16(s3), vget_low_s16(f[3]));
+ m3 = vmlal_s16(m3, vget_high_s16(s3), vget_high_s16(f[3]));
- // row:4,5,6,7 odd_col:1,3
- src_0 = vget_high_s16(vreinterpretq_s16_s32(c3.val[0]));
- fltr_0 = vget_high_s16(vreinterpretq_s16_s32(d2.val[1]));
- res_0 = vmlal_s16(res_0, src_0, fltr_0);
- res_0_im = vpadd_s32(vget_low_s32(res_0), vget_high_s32(res_0));
+ int32x4_t m0123_pairs[] = { m0, m1, m2, m3 };
- // row:4,5 odd_col:5,7
- src_1 = vget_high_s16(vreinterpretq_s16_s32(c2.val[1]));
- fltr_1 = vget_high_s16(vreinterpretq_s16_s32(d3.val[0]));
- res_1 = vmull_s16(src_1, fltr_1);
+ *res = horizontal_add_4d_s32x4(m0123_pairs);
+}
- // row:4,5,6,7 odd_col:5,7
- src_1 = vget_high_s16(vreinterpretq_s16_s32(c3.val[1]));
- fltr_1 = vget_high_s16(vreinterpretq_s16_s32(d3.val[1]));
- res_1 = vmlal_s16(res_1, src_1, fltr_1);
- res_1_im = vpadd_s32(vget_low_s32(res_1), vget_high_s32(res_1));
+static INLINE void vertical_filter_8x1_f1(const int16x8_t *src,
+ int32x4_t *res_low,
+ int32x4_t *res_high, int sy) {
+ int16x8_t s0 = src[0];
+ int16x8_t s1 = src[1];
+ int16x8_t s2 = src[2];
+ int16x8_t s3 = src[3];
+ int16x8_t s4 = src[4];
+ int16x8_t s5 = src[5];
+ int16x8_t s6 = src[6];
+ int16x8_t s7 = src[7];
- // row:4,5,6,7 odd_col:1,3,5,7
- im_res_1 = vcombine_s32(res_0_im, res_1_im);
+ int16x8_t f =
+ vld1q_s16((int16_t *)(av1_warped_filter + (sy >> WARPEDDIFF_PREC_BITS)));
- // row:0-7 odd_col:1,3,5,7
- res_odd = vaddq_s32(im_res_0, im_res_1);
+ int32x4_t m0123 = vmull_lane_s16(vget_low_s16(s0), vget_low_s16(f), 0);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(s1), vget_low_s16(f), 1);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(s2), vget_low_s16(f), 2);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(s3), vget_low_s16(f), 3);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(s4), vget_high_s16(f), 0);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(s5), vget_high_s16(f), 1);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(s6), vget_high_s16(f), 2);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(s7), vget_high_s16(f), 3);
- // reordering as 0 1 2 3 | 4 5 6 7
- c0 = vtrnq_s32(res_even, res_odd);
+ int32x4_t m4567 = vmull_lane_s16(vget_high_s16(s0), vget_low_s16(f), 0);
+ m4567 = vmlal_lane_s16(m4567, vget_high_s16(s1), vget_low_s16(f), 1);
+ m4567 = vmlal_lane_s16(m4567, vget_high_s16(s2), vget_low_s16(f), 2);
+ m4567 = vmlal_lane_s16(m4567, vget_high_s16(s3), vget_low_s16(f), 3);
+ m4567 = vmlal_lane_s16(m4567, vget_high_s16(s4), vget_high_s16(f), 0);
+ m4567 = vmlal_lane_s16(m4567, vget_high_s16(s5), vget_high_s16(f), 1);
+ m4567 = vmlal_lane_s16(m4567, vget_high_s16(s6), vget_high_s16(f), 2);
+ m4567 = vmlal_lane_s16(m4567, vget_high_s16(s7), vget_high_s16(f), 3);
- // Final store
- *res_low = vcombine_s32(vget_low_s32(c0.val[0]), vget_low_s32(c0.val[1]));
- *res_high = vcombine_s32(vget_high_s32(c0.val[0]), vget_high_s32(c0.val[1]));
+ *res_low = m0123;
+ *res_high = m4567;
+}
+
+static INLINE void vertical_filter_8x1_f8(const int16x8_t *src,
+ int32x4_t *res_low,
+ int32x4_t *res_high, int sy,
+ int gamma) {
+ int16x8_t s0 = src[0];
+ int16x8_t s1 = src[1];
+ int16x8_t s2 = src[2];
+ int16x8_t s3 = src[3];
+ int16x8_t s4 = src[4];
+ int16x8_t s5 = src[5];
+ int16x8_t s6 = src[6];
+ int16x8_t s7 = src[7];
+ transpose_elems_inplace_s16_8x8(&s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7);
+
+ int16x8_t f[8];
+ load_filters_8(f, sy, gamma);
+
+ int32x4_t m0 = vmull_s16(vget_low_s16(s0), vget_low_s16(f[0]));
+ m0 = vmlal_s16(m0, vget_high_s16(s0), vget_high_s16(f[0]));
+ int32x4_t m1 = vmull_s16(vget_low_s16(s1), vget_low_s16(f[1]));
+ m1 = vmlal_s16(m1, vget_high_s16(s1), vget_high_s16(f[1]));
+ int32x4_t m2 = vmull_s16(vget_low_s16(s2), vget_low_s16(f[2]));
+ m2 = vmlal_s16(m2, vget_high_s16(s2), vget_high_s16(f[2]));
+ int32x4_t m3 = vmull_s16(vget_low_s16(s3), vget_low_s16(f[3]));
+ m3 = vmlal_s16(m3, vget_high_s16(s3), vget_high_s16(f[3]));
+ int32x4_t m4 = vmull_s16(vget_low_s16(s4), vget_low_s16(f[4]));
+ m4 = vmlal_s16(m4, vget_high_s16(s4), vget_high_s16(f[4]));
+ int32x4_t m5 = vmull_s16(vget_low_s16(s5), vget_low_s16(f[5]));
+ m5 = vmlal_s16(m5, vget_high_s16(s5), vget_high_s16(f[5]));
+ int32x4_t m6 = vmull_s16(vget_low_s16(s6), vget_low_s16(f[6]));
+ m6 = vmlal_s16(m6, vget_high_s16(s6), vget_high_s16(f[6]));
+ int32x4_t m7 = vmull_s16(vget_low_s16(s7), vget_low_s16(f[7]));
+ m7 = vmlal_s16(m7, vget_high_s16(s7), vget_high_s16(f[7]));
+
+ int32x4_t m0123_pairs[] = { m0, m1, m2, m3 };
+ int32x4_t m4567_pairs[] = { m4, m5, m6, m7 };
+
+ *res_low = horizontal_add_4d_s32x4(m0123_pairs);
+ *res_high = horizontal_add_4d_s32x4(m4567_pairs);
}
void av1_warp_affine_neon(const int32_t *mat, const uint8_t *ref, int width,
@@ -474,242 +270,7 @@
int subsampling_x, int subsampling_y,
ConvolveParams *conv_params, int16_t alpha,
int16_t beta, int16_t gamma, int16_t delta) {
- int16x8_t tmp[15];
- const int bd = 8;
- const int w0 = conv_params->fwd_offset;
- const int w1 = conv_params->bck_offset;
- const int32x4_t fwd = vdupq_n_s32((int32_t)w0);
- const int32x4_t bwd = vdupq_n_s32((int32_t)w1);
- const int16x8_t sub_constant = vdupq_n_s16((1 << (bd - 1)) + (1 << bd));
-
- int limit = 0;
- uint8x16_t vec_dup, mask_val;
- int32x4_t res_lo, res_hi;
- int16x8_t result_final;
- uint8x16_t src_1, src_2, src_3, src_4;
- static const uint8_t k0To15[16] = { 0, 1, 2, 3, 4, 5, 6, 7,
- 8, 9, 10, 11, 12, 13, 14, 15 };
- uint8x16_t indx_vec = vld1q_u8(k0To15);
- uint8x16_t cmp_vec;
-
- const int reduce_bits_horiz = conv_params->round_0;
- const int reduce_bits_vert = conv_params->is_compound
- ? conv_params->round_1
- : 2 * FILTER_BITS - reduce_bits_horiz;
- const int32x4_t shift_vert = vdupq_n_s32(-(int32_t)reduce_bits_vert);
- const int offset_bits_horiz = bd + FILTER_BITS - 1;
-
- assert(IMPLIES(conv_params->is_compound, conv_params->dst != NULL));
-
- const int offset_bits_vert = bd + 2 * FILTER_BITS - reduce_bits_horiz;
- int32x4_t add_const_vert = vdupq_n_s32((int32_t)(1 << offset_bits_vert));
- const int round_bits =
- 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1;
- const int16x4_t round_bits_vec = vdup_n_s16(-(int16_t)round_bits);
- const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0;
- const int16x4_t res_sub_const =
- vdup_n_s16(-((1 << (offset_bits - conv_params->round_1)) +
- (1 << (offset_bits - conv_params->round_1 - 1))));
- int k;
-
- assert(IMPLIES(conv_params->do_average, conv_params->is_compound));
-
- for (int i = 0; i < p_height; i += 8) {
- for (int j = 0; j < p_width; j += 8) {
- const int32_t src_x = (p_col + j + 4) << subsampling_x;
- const int32_t src_y = (p_row + i + 4) << subsampling_y;
- const int64_t dst_x =
- (int64_t)mat[2] * src_x + (int64_t)mat[3] * src_y + (int64_t)mat[0];
- const int64_t dst_y =
- (int64_t)mat[4] * src_x + (int64_t)mat[5] * src_y + (int64_t)mat[1];
- const int64_t x4 = dst_x >> subsampling_x;
- const int64_t y4 = dst_y >> subsampling_y;
-
- int32_t ix4 = (int32_t)(x4 >> WARPEDMODEL_PREC_BITS);
- int32_t sx4 = x4 & ((1 << WARPEDMODEL_PREC_BITS) - 1);
- int32_t iy4 = (int32_t)(y4 >> WARPEDMODEL_PREC_BITS);
- int32_t sy4 = y4 & ((1 << WARPEDMODEL_PREC_BITS) - 1);
-
- sx4 += alpha * (-4) + beta * (-4) + (1 << (WARPEDDIFF_PREC_BITS - 1)) +
- (WARPEDPIXEL_PREC_SHIFTS << WARPEDDIFF_PREC_BITS);
- sy4 += gamma * (-4) + delta * (-4) + (1 << (WARPEDDIFF_PREC_BITS - 1)) +
- (WARPEDPIXEL_PREC_SHIFTS << WARPEDDIFF_PREC_BITS);
-
- sx4 &= ~((1 << WARP_PARAM_REDUCE_BITS) - 1);
- sy4 &= ~((1 << WARP_PARAM_REDUCE_BITS) - 1);
- // horizontal
- if (ix4 <= -7) {
- for (k = -7; k < AOMMIN(8, p_height - i); ++k) {
- int iy = iy4 + k;
- if (iy < 0)
- iy = 0;
- else if (iy > height - 1)
- iy = height - 1;
- int16_t dup_val =
- (1 << (bd + FILTER_BITS - reduce_bits_horiz - 1)) +
- ref[iy * stride] * (1 << (FILTER_BITS - reduce_bits_horiz));
-
- tmp[k + 7] = vdupq_n_s16(dup_val);
- }
- } else if (ix4 >= width + 6) {
- for (k = -7; k < AOMMIN(8, p_height - i); ++k) {
- int iy = iy4 + k;
- if (iy < 0)
- iy = 0;
- else if (iy > height - 1)
- iy = height - 1;
- int16_t dup_val = (1 << (bd + FILTER_BITS - reduce_bits_horiz - 1)) +
- ref[iy * stride + (width - 1)] *
- (1 << (FILTER_BITS - reduce_bits_horiz));
- tmp[k + 7] = vdupq_n_s16(dup_val);
- }
- } else if (((ix4 - 7) < 0) || ((ix4 + 9) > width)) {
- const int out_of_boundary_left = -(ix4 - 6);
- const int out_of_boundary_right = (ix4 + 8) - width;
-
- for (k = -7; k < AOMMIN(8, p_height - i); ++k) {
- int iy = iy4 + k;
- if (iy < 0)
- iy = 0;
- else if (iy > height - 1)
- iy = height - 1;
- int sx = sx4 + beta * (k + 4);
-
- const uint8_t *src = ref + iy * stride + ix4 - 7;
- src_1 = vld1q_u8(src);
-
- if (out_of_boundary_left >= 0) {
- limit = out_of_boundary_left + 1;
- cmp_vec = vdupq_n_u8(out_of_boundary_left);
- vec_dup = vdupq_n_u8(*(src + limit));
- mask_val = vcleq_u8(indx_vec, cmp_vec);
- src_1 = vbslq_u8(mask_val, vec_dup, src_1);
- }
- if (out_of_boundary_right >= 0) {
- limit = 15 - (out_of_boundary_right + 1);
- cmp_vec = vdupq_n_u8(15 - out_of_boundary_right);
- vec_dup = vdupq_n_u8(*(src + limit));
- mask_val = vcgeq_u8(indx_vec, cmp_vec);
- src_1 = vbslq_u8(mask_val, vec_dup, src_1);
- }
- src_2 = vextq_u8(src_1, src_1, 1);
- src_3 = vextq_u8(src_2, src_2, 1);
- src_4 = vextq_u8(src_3, src_3, 1);
-
- horizontal_filter_neon(src_1, src_2, src_3, src_4, tmp, sx, alpha, k,
- offset_bits_horiz, reduce_bits_horiz);
- }
- } else {
- for (k = -7; k < AOMMIN(8, p_height - i); ++k) {
- int iy = iy4 + k;
- if (iy < 0)
- iy = 0;
- else if (iy > height - 1)
- iy = height - 1;
- int sx = sx4 + beta * (k + 4);
-
- const uint8_t *src = ref + iy * stride + ix4 - 7;
- src_1 = vld1q_u8(src);
- src_2 = vextq_u8(src_1, src_1, 1);
- src_3 = vextq_u8(src_2, src_2, 1);
- src_4 = vextq_u8(src_3, src_3, 1);
-
- horizontal_filter_neon(src_1, src_2, src_3, src_4, tmp, sx, alpha, k,
- offset_bits_horiz, reduce_bits_horiz);
- }
- }
-
- // vertical
- for (k = -4; k < AOMMIN(4, p_height - i - 4); ++k) {
- int sy = sy4 + delta * (k + 4);
-
- const int16x8_t *v_src = tmp + (k + 4);
-
- vertical_filter_neon(v_src, &res_lo, &res_hi, sy, gamma);
-
- res_lo = vaddq_s32(res_lo, add_const_vert);
- res_hi = vaddq_s32(res_hi, add_const_vert);
-
- if (conv_params->is_compound) {
- uint16_t *const p =
- (uint16_t *)&conv_params
- ->dst[(i + k + 4) * conv_params->dst_stride + j];
-
- res_lo = vrshlq_s32(res_lo, shift_vert);
- if (conv_params->do_average) {
- uint8_t *const dst8 = &pred[(i + k + 4) * p_stride + j];
- uint16x4_t tmp16_lo = vld1_u16(p);
- int32x4_t tmp32_lo = vreinterpretq_s32_u32(vmovl_u16(tmp16_lo));
- int16x4_t tmp16_low;
- if (conv_params->use_dist_wtd_comp_avg) {
- res_lo = vmulq_s32(res_lo, bwd);
- tmp32_lo = vmulq_s32(tmp32_lo, fwd);
- tmp32_lo = vaddq_s32(tmp32_lo, res_lo);
- tmp16_low = vshrn_n_s32(tmp32_lo, DIST_PRECISION_BITS);
- } else {
- tmp32_lo = vaddq_s32(tmp32_lo, res_lo);
- tmp16_low = vshrn_n_s32(tmp32_lo, 1);
- }
- int16x4_t res_low = vadd_s16(tmp16_low, res_sub_const);
- res_low = vqrshl_s16(res_low, round_bits_vec);
- int16x8_t final_res_low = vcombine_s16(res_low, res_low);
- uint8x8_t res_8_low = vqmovun_s16(final_res_low);
-
- vst1_lane_u32((uint32_t *)dst8, vreinterpret_u32_u8(res_8_low), 0);
- } else {
- uint16x4_t res_u16_low = vqmovun_s32(res_lo);
- vst1_u16(p, res_u16_low);
- }
- if (p_width > 4) {
- uint16_t *const p4 =
- (uint16_t *)&conv_params
- ->dst[(i + k + 4) * conv_params->dst_stride + j + 4];
-
- res_hi = vrshlq_s32(res_hi, shift_vert);
- if (conv_params->do_average) {
- uint8_t *const dst8_4 = &pred[(i + k + 4) * p_stride + j + 4];
-
- uint16x4_t tmp16_hi = vld1_u16(p4);
- int32x4_t tmp32_hi = vreinterpretq_s32_u32(vmovl_u16(tmp16_hi));
- int16x4_t tmp16_high;
- if (conv_params->use_dist_wtd_comp_avg) {
- res_hi = vmulq_s32(res_hi, bwd);
- tmp32_hi = vmulq_s32(tmp32_hi, fwd);
- tmp32_hi = vaddq_s32(tmp32_hi, res_hi);
- tmp16_high = vshrn_n_s32(tmp32_hi, DIST_PRECISION_BITS);
- } else {
- tmp32_hi = vaddq_s32(tmp32_hi, res_hi);
- tmp16_high = vshrn_n_s32(tmp32_hi, 1);
- }
- int16x4_t res_high = vadd_s16(tmp16_high, res_sub_const);
- res_high = vqrshl_s16(res_high, round_bits_vec);
- int16x8_t final_res_high = vcombine_s16(res_high, res_high);
- uint8x8_t res_8_high = vqmovun_s16(final_res_high);
-
- vst1_lane_u32((uint32_t *)dst8_4, vreinterpret_u32_u8(res_8_high),
- 0);
- } else {
- uint16x4_t res_u16_high = vqmovun_s32(res_hi);
- vst1_u16(p4, res_u16_high);
- }
- }
- } else {
- res_lo = vrshlq_s32(res_lo, shift_vert);
- res_hi = vrshlq_s32(res_hi, shift_vert);
-
- result_final = vcombine_s16(vmovn_s32(res_lo), vmovn_s32(res_hi));
- result_final = vsubq_s16(result_final, sub_constant);
-
- uint8_t *const p = (uint8_t *)&pred[(i + k + 4) * p_stride + j];
- uint8x8_t val = vqmovun_s16(result_final);
-
- if (p_width == 4) {
- vst1_lane_u32((uint32_t *)p, vreinterpret_u32_u8(val), 0);
- } else {
- vst1_u8(p, val);
- }
- }
- }
- }
- }
+ av1_warp_affine_common(mat, ref, width, height, stride, pred, p_col, p_row,
+ p_width, p_height, p_stride, subsampling_x,
+ subsampling_y, conv_params, alpha, beta, gamma, delta);
}
diff --git a/av1/common/arm/warp_plane_neon.h b/av1/common/arm/warp_plane_neon.h
new file mode 100644
index 0000000..de5e3bd
--- /dev/null
+++ b/av1/common/arm/warp_plane_neon.h
@@ -0,0 +1,378 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+#ifndef AOM_AV1_COMMON_ARM_WARP_PLANE_NEON_H_
+#define AOM_AV1_COMMON_ARM_WARP_PLANE_NEON_H_
+
+#include <assert.h>
+#include <arm_neon.h>
+#include <memory.h>
+#include <math.h>
+
+#include "aom_dsp/aom_dsp_common.h"
+#include "aom_dsp/arm/sum_neon.h"
+#include "aom_dsp/arm/transpose_neon.h"
+#include "aom_ports/mem.h"
+#include "config/av1_rtcd.h"
+#include "av1/common/warped_motion.h"
+#include "av1/common/scale.h"
+
+static INLINE int16x8_t horizontal_filter_4x1_f4(const uint8x16_t in, int sx,
+ int alpha);
+
+static INLINE int16x8_t horizontal_filter_8x1_f8(const uint8x16_t in, int sx,
+ int alpha);
+
+static INLINE int16x8_t horizontal_filter_4x1_f1(const uint8x16_t in, int sx);
+
+static INLINE int16x8_t horizontal_filter_8x1_f1(const uint8x16_t in, int sx);
+
+static INLINE void vertical_filter_4x1_f1(const int16x8_t *src, int32x4_t *res,
+ int sy);
+
+static INLINE void vertical_filter_4x1_f4(const int16x8_t *src, int32x4_t *res,
+ int sy, int gamma);
+
+static INLINE void vertical_filter_8x1_f1(const int16x8_t *src,
+ int32x4_t *res_low,
+ int32x4_t *res_high, int sy);
+
+static INLINE void vertical_filter_8x1_f8(const int16x8_t *src,
+ int32x4_t *res_low,
+ int32x4_t *res_high, int sy,
+ int gamma);
+
+static INLINE void load_filters_4(int16x8_t out[], int offset, int stride) {
+ out[0] = vld1q_s16((int16_t *)(av1_warped_filter + ((offset + 0 * stride) >>
+ WARPEDDIFF_PREC_BITS)));
+ out[1] = vld1q_s16((int16_t *)(av1_warped_filter + ((offset + 1 * stride) >>
+ WARPEDDIFF_PREC_BITS)));
+ out[2] = vld1q_s16((int16_t *)(av1_warped_filter + ((offset + 2 * stride) >>
+ WARPEDDIFF_PREC_BITS)));
+ out[3] = vld1q_s16((int16_t *)(av1_warped_filter + ((offset + 3 * stride) >>
+ WARPEDDIFF_PREC_BITS)));
+}
+
+static INLINE void load_filters_8(int16x8_t out[], int offset, int stride) {
+ out[0] = vld1q_s16((int16_t *)(av1_warped_filter + ((offset + 0 * stride) >>
+ WARPEDDIFF_PREC_BITS)));
+ out[1] = vld1q_s16((int16_t *)(av1_warped_filter + ((offset + 1 * stride) >>
+ WARPEDDIFF_PREC_BITS)));
+ out[2] = vld1q_s16((int16_t *)(av1_warped_filter + ((offset + 2 * stride) >>
+ WARPEDDIFF_PREC_BITS)));
+ out[3] = vld1q_s16((int16_t *)(av1_warped_filter + ((offset + 3 * stride) >>
+ WARPEDDIFF_PREC_BITS)));
+ out[4] = vld1q_s16((int16_t *)(av1_warped_filter + ((offset + 4 * stride) >>
+ WARPEDDIFF_PREC_BITS)));
+ out[5] = vld1q_s16((int16_t *)(av1_warped_filter + ((offset + 5 * stride) >>
+ WARPEDDIFF_PREC_BITS)));
+ out[6] = vld1q_s16((int16_t *)(av1_warped_filter + ((offset + 6 * stride) >>
+ WARPEDDIFF_PREC_BITS)));
+ out[7] = vld1q_s16((int16_t *)(av1_warped_filter + ((offset + 7 * stride) >>
+ WARPEDDIFF_PREC_BITS)));
+}
+
+static INLINE int clamp_iy(int iy, int height) {
+ return clamp(iy, 0, height - 1);
+}
+
+static INLINE void warp_affine_horizontal(
+ const uint8_t *ref, int width, int height, int stride, int p_width,
+ int p_height, int16_t alpha, int16_t beta, const int64_t x4,
+ const int64_t y4, const int i, int16x8_t tmp[], const uint8x16_t indx_vec) {
+ const int bd = 8;
+ const int reduce_bits_horiz = ROUND0_BITS;
+ const int height_limit = AOMMIN(8, p_height - i) + 7;
+
+ int32_t ix4 = (int32_t)(x4 >> WARPEDMODEL_PREC_BITS);
+ int32_t iy4 = (int32_t)(y4 >> WARPEDMODEL_PREC_BITS);
+
+ int32_t sx4 = x4 & ((1 << WARPEDMODEL_PREC_BITS) - 1);
+ sx4 += alpha * (-4) + beta * (-4) + (1 << (WARPEDDIFF_PREC_BITS - 1)) +
+ (WARPEDPIXEL_PREC_SHIFTS << WARPEDDIFF_PREC_BITS);
+ sx4 &= ~((1 << WARP_PARAM_REDUCE_BITS) - 1);
+
+ if (ix4 <= -7) {
+ for (int k = 0; k < height_limit; ++k) {
+ int iy = clamp_iy(iy4 + k - 7, height);
+ int16_t dup_val =
+ (1 << (bd + FILTER_BITS - reduce_bits_horiz - 1)) +
+ ref[iy * stride] * (1 << (FILTER_BITS - reduce_bits_horiz));
+ tmp[k] = vdupq_n_s16(dup_val);
+ }
+ return;
+ } else if (ix4 >= width + 6) {
+ for (int k = 0; k < height_limit; ++k) {
+ int iy = clamp_iy(iy4 + k - 7, height);
+ int16_t dup_val = (1 << (bd + FILTER_BITS - reduce_bits_horiz - 1)) +
+ ref[iy * stride + (width - 1)] *
+ (1 << (FILTER_BITS - reduce_bits_horiz));
+ tmp[k] = vdupq_n_s16(dup_val);
+ }
+ return;
+ }
+
+ uint8x16_t in[15];
+ if (((ix4 - 7) < 0) || ((ix4 + 9) > width)) {
+ const int out_of_boundary_left = -(ix4 - 6);
+ const int out_of_boundary_right = (ix4 + 8) - width;
+
+ for (int k = 0; k < height_limit; ++k) {
+ const int iy = clamp_iy(iy4 + k - 7, height);
+ const uint8_t *src = ref + iy * stride + ix4 - 7;
+ uint8x16_t src_1 = vld1q_u8(src);
+
+ if (out_of_boundary_left >= 0) {
+ int limit = out_of_boundary_left + 1;
+ uint8x16_t cmp_vec = vdupq_n_u8(out_of_boundary_left);
+ uint8x16_t vec_dup = vdupq_n_u8(*(src + limit));
+ uint8x16_t mask_val = vcleq_u8(indx_vec, cmp_vec);
+ src_1 = vbslq_u8(mask_val, vec_dup, src_1);
+ }
+ if (out_of_boundary_right >= 0) {
+ int limit = 15 - (out_of_boundary_right + 1);
+ uint8x16_t cmp_vec = vdupq_n_u8(15 - out_of_boundary_right);
+ uint8x16_t vec_dup = vdupq_n_u8(*(src + limit));
+ uint8x16_t mask_val = vcgeq_u8(indx_vec, cmp_vec);
+ src_1 = vbslq_u8(mask_val, vec_dup, src_1);
+ }
+ in[k] = src_1;
+ }
+ } else {
+ for (int k = 0; k < height_limit; ++k) {
+ const int iy = clamp_iy(iy4 + k - 7, height);
+ const uint8_t *src = ref + iy * stride + ix4 - 7;
+ in[k] = vld1q_u8(src);
+ }
+ }
+
+ if (p_width == 4) {
+ if (beta == 0) {
+ if (alpha == 0) {
+ for (int k = 0; k < height_limit; ++k) {
+ tmp[k] = horizontal_filter_4x1_f1(in[k], sx4);
+ }
+ } else {
+ for (int k = 0; k < height_limit; ++k) {
+ tmp[k] = horizontal_filter_4x1_f4(in[k], sx4, alpha);
+ }
+ }
+ } else {
+ if (alpha == 0) {
+ for (int k = 0; k < height_limit; ++k) {
+ const int sx = sx4 + beta * (k - 3);
+ tmp[k] = horizontal_filter_4x1_f1(in[k], sx);
+ }
+ } else {
+ for (int k = 0; k < height_limit; ++k) {
+ const int sx = sx4 + beta * (k - 3);
+ tmp[k] = horizontal_filter_4x1_f4(in[k], sx, alpha);
+ }
+ }
+ }
+ } else {
+ if (beta == 0) {
+ if (alpha == 0) {
+ for (int k = 0; k < height_limit; ++k) {
+ tmp[k] = horizontal_filter_8x1_f1(in[k], sx4);
+ }
+ } else {
+ for (int k = 0; k < height_limit; ++k) {
+ tmp[k] = horizontal_filter_8x1_f8(in[k], sx4, alpha);
+ }
+ }
+ } else {
+ if (alpha == 0) {
+ for (int k = 0; k < height_limit; ++k) {
+ const int sx = sx4 + beta * (k - 3);
+ tmp[k] = horizontal_filter_8x1_f1(in[k], sx);
+ }
+ } else {
+ for (int k = 0; k < height_limit; ++k) {
+ const int sx = sx4 + beta * (k - 3);
+ tmp[k] = horizontal_filter_8x1_f8(in[k], sx, alpha);
+ }
+ }
+ }
+ }
+}
+
+static INLINE void warp_affine_vertical(
+ uint8_t *pred, int p_width, int p_height, int p_stride, int is_compound,
+ uint16_t *dst, int dst_stride, int do_average, int use_dist_wtd_comp_avg,
+ int16_t gamma, int16_t delta, const int64_t y4, const int i, const int j,
+ int16x8_t tmp[], const int fwd, const int bwd) {
+ const int bd = 8;
+ const int reduce_bits_horiz = ROUND0_BITS;
+ const int offset_bits_vert = bd + 2 * FILTER_BITS - reduce_bits_horiz;
+ int add_const_vert;
+ if (is_compound) {
+ add_const_vert =
+ (1 << offset_bits_vert) + (1 << (COMPOUND_ROUND1_BITS - 1));
+ } else {
+ add_const_vert =
+ (1 << offset_bits_vert) + (1 << (2 * FILTER_BITS - ROUND0_BITS - 1));
+ }
+ const int sub_constant = (1 << (bd - 1)) + (1 << bd);
+
+ const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS;
+ const int res_sub_const =
+ (1 << (2 * FILTER_BITS - ROUND0_BITS - COMPOUND_ROUND1_BITS - 1)) -
+ (1 << (offset_bits - COMPOUND_ROUND1_BITS)) -
+ (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1));
+
+ int32_t sy4 = y4 & ((1 << WARPEDMODEL_PREC_BITS) - 1);
+ sy4 += gamma * (-4) + delta * (-4) + (1 << (WARPEDDIFF_PREC_BITS - 1)) +
+ (WARPEDPIXEL_PREC_SHIFTS << WARPEDDIFF_PREC_BITS);
+ sy4 &= ~((1 << WARP_PARAM_REDUCE_BITS) - 1);
+
+ if (p_width > 4) {
+ for (int k = -4; k < AOMMIN(4, p_height - i - 4); ++k) {
+ int sy = sy4 + delta * (k + 4);
+ const int16x8_t *v_src = tmp + (k + 4);
+
+ int32x4_t res_lo, res_hi;
+ if (gamma == 0) {
+ vertical_filter_8x1_f1(v_src, &res_lo, &res_hi, sy);
+ } else {
+ vertical_filter_8x1_f8(v_src, &res_lo, &res_hi, sy, gamma);
+ }
+
+ res_lo = vaddq_s32(res_lo, vdupq_n_s32(add_const_vert));
+ res_hi = vaddq_s32(res_hi, vdupq_n_s32(add_const_vert));
+
+ if (is_compound) {
+ uint16_t *const p = (uint16_t *)&dst[(i + k + 4) * dst_stride + j];
+ int16x8_t res_s16 =
+ vcombine_s16(vshrn_n_s32(res_lo, COMPOUND_ROUND1_BITS),
+ vshrn_n_s32(res_hi, COMPOUND_ROUND1_BITS));
+ if (do_average) {
+ int16x8_t tmp16 = vreinterpretq_s16_u16(vld1q_u16(p));
+ if (use_dist_wtd_comp_avg) {
+ int32x4_t tmp32_lo = vmull_n_s16(vget_low_s16(tmp16), fwd);
+ int32x4_t tmp32_hi = vmull_n_s16(vget_high_s16(tmp16), fwd);
+ tmp32_lo = vmlal_n_s16(tmp32_lo, vget_low_s16(res_s16), bwd);
+ tmp32_hi = vmlal_n_s16(tmp32_hi, vget_high_s16(res_s16), bwd);
+ tmp16 = vcombine_s16(vshrn_n_s32(tmp32_lo, DIST_PRECISION_BITS),
+ vshrn_n_s32(tmp32_hi, DIST_PRECISION_BITS));
+ } else {
+ tmp16 = vhaddq_s16(tmp16, res_s16);
+ }
+ int16x8_t res = vaddq_s16(tmp16, vdupq_n_s16(res_sub_const));
+ uint8x8_t res8 = vqshrun_n_s16(
+ res, 2 * FILTER_BITS - ROUND0_BITS - COMPOUND_ROUND1_BITS);
+ vst1_u8(&pred[(i + k + 4) * p_stride + j], res8);
+ } else {
+ vst1q_u16(p, vreinterpretq_u16_s16(res_s16));
+ }
+ } else {
+ int16x8_t res16 =
+ vcombine_s16(vshrn_n_s32(res_lo, 2 * FILTER_BITS - ROUND0_BITS),
+ vshrn_n_s32(res_hi, 2 * FILTER_BITS - ROUND0_BITS));
+ res16 = vsubq_s16(res16, vdupq_n_s16(sub_constant));
+
+ uint8_t *const p = (uint8_t *)&pred[(i + k + 4) * p_stride + j];
+ vst1_u8(p, vqmovun_s16(res16));
+ }
+ }
+ } else {
+ // p_width == 4
+ for (int k = -4; k < AOMMIN(4, p_height - i - 4); ++k) {
+ int sy = sy4 + delta * (k + 4);
+ const int16x8_t *v_src = tmp + (k + 4);
+
+ int32x4_t res_lo;
+ if (gamma == 0) {
+ vertical_filter_4x1_f1(v_src, &res_lo, sy);
+ } else {
+ vertical_filter_4x1_f4(v_src, &res_lo, sy, gamma);
+ }
+
+ res_lo = vaddq_s32(res_lo, vdupq_n_s32(add_const_vert));
+
+ if (is_compound) {
+ uint16_t *const p = (uint16_t *)&dst[(i + k + 4) * dst_stride + j];
+
+ int16x4_t res_lo_s16 = vshrn_n_s32(res_lo, COMPOUND_ROUND1_BITS);
+ if (do_average) {
+ uint8_t *const dst8 = &pred[(i + k + 4) * p_stride + j];
+ int16x4_t tmp16_lo = vreinterpret_s16_u16(vld1_u16(p));
+ if (use_dist_wtd_comp_avg) {
+ int32x4_t tmp32_lo = vmull_n_s16(tmp16_lo, fwd);
+ tmp32_lo = vmlal_n_s16(tmp32_lo, res_lo_s16, bwd);
+ tmp16_lo = vshrn_n_s32(tmp32_lo, DIST_PRECISION_BITS);
+ } else {
+ tmp16_lo = vhadd_s16(tmp16_lo, res_lo_s16);
+ }
+ int16x4_t res = vadd_s16(tmp16_lo, vdup_n_s16(res_sub_const));
+ uint8x8_t res8 = vqshrun_n_s16(
+ vcombine_s16(res, vdup_n_s16(0)),
+ 2 * FILTER_BITS - ROUND0_BITS - COMPOUND_ROUND1_BITS);
+ vst1_lane_u32((uint32_t *)dst8, vreinterpret_u32_u8(res8), 0);
+ } else {
+ uint16x4_t res_u16_low = vreinterpret_u16_s16(res_lo_s16);
+ vst1_u16(p, res_u16_low);
+ }
+ } else {
+ int16x4_t res16 = vshrn_n_s32(res_lo, 2 * FILTER_BITS - ROUND0_BITS);
+ res16 = vsub_s16(res16, vdup_n_s16(sub_constant));
+
+ uint8_t *const p = (uint8_t *)&pred[(i + k + 4) * p_stride + j];
+ uint8x8_t val = vqmovun_s16(vcombine_s16(res16, vdup_n_s16(0)));
+ vst1_lane_u32((uint32_t *)p, vreinterpret_u32_u8(val), 0);
+ }
+ }
+ }
+}
+
+static INLINE void av1_warp_affine_common(
+ const int32_t *mat, const uint8_t *ref, int width, int height, int stride,
+ uint8_t *pred, int p_col, int p_row, int p_width, int p_height,
+ int p_stride, int subsampling_x, int subsampling_y,
+ ConvolveParams *conv_params, int16_t alpha, int16_t beta, int16_t gamma,
+ int16_t delta) {
+ const int w0 = conv_params->fwd_offset;
+ const int w1 = conv_params->bck_offset;
+ const int is_compound = conv_params->is_compound;
+ uint16_t *const dst = conv_params->dst;
+ const int dst_stride = conv_params->dst_stride;
+ const int do_average = conv_params->do_average;
+ const int use_dist_wtd_comp_avg = conv_params->use_dist_wtd_comp_avg;
+
+ static const uint8_t k0To15[16] = { 0, 1, 2, 3, 4, 5, 6, 7,
+ 8, 9, 10, 11, 12, 13, 14, 15 };
+ const uint8x16_t indx_vec = vld1q_u8(k0To15);
+
+ assert(IMPLIES(is_compound, dst != NULL));
+ assert(IMPLIES(do_average, is_compound));
+
+ for (int i = 0; i < p_height; i += 8) {
+ for (int j = 0; j < p_width; j += 8) {
+ const int32_t src_x = (p_col + j + 4) << subsampling_x;
+ const int32_t src_y = (p_row + i + 4) << subsampling_y;
+ const int64_t dst_x =
+ (int64_t)mat[2] * src_x + (int64_t)mat[3] * src_y + (int64_t)mat[0];
+ const int64_t dst_y =
+ (int64_t)mat[4] * src_x + (int64_t)mat[5] * src_y + (int64_t)mat[1];
+
+ const int64_t x4 = dst_x >> subsampling_x;
+ const int64_t y4 = dst_y >> subsampling_y;
+
+ int16x8_t tmp[15];
+ warp_affine_horizontal(ref, width, height, stride, p_width, p_height,
+ alpha, beta, x4, y4, i, tmp, indx_vec);
+ warp_affine_vertical(pred, p_width, p_height, p_stride, is_compound, dst,
+ dst_stride, do_average, use_dist_wtd_comp_avg, gamma,
+ delta, y4, i, j, tmp, w0, w1);
+ }
+ }
+}
+
+#endif // AOM_AV1_COMMON_ARM_WARP_PLANE_NEON_H_
diff --git a/av1/common/arm/warp_plane_neon_i8mm.c b/av1/common/arm/warp_plane_neon_i8mm.c
new file mode 100644
index 0000000..39e3ad9
--- /dev/null
+++ b/av1/common/arm/warp_plane_neon_i8mm.c
@@ -0,0 +1,291 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include "warp_plane_neon.h"
+
+DECLARE_ALIGNED(16, static const uint8_t, usdot_permute_idx[48]) = {
+ 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6,
+ 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10,
+ 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
+};
+
+static INLINE int16x8_t horizontal_filter_4x1_f4(const uint8x16_t in, int sx,
+ int alpha) {
+ const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1));
+
+ // Loading the 8 filter taps
+ int16x8_t f[4];
+ load_filters_4(f, sx, alpha);
+
+ int8x16_t f01_u8 = vcombine_s8(vmovn_s16(f[0]), vmovn_s16(f[1]));
+ int8x16_t f23_u8 = vcombine_s8(vmovn_s16(f[2]), vmovn_s16(f[3]));
+
+ uint8x8_t in0 = vget_low_u8(in);
+ uint8x8_t in1 = vget_low_u8(vextq_u8(in, in, 1));
+ uint8x8_t in2 = vget_low_u8(vextq_u8(in, in, 2));
+ uint8x8_t in3 = vget_low_u8(vextq_u8(in, in, 3));
+
+ int32x4_t m01 = vusdotq_s32(vdupq_n_s32(0), vcombine_u8(in0, in1), f01_u8);
+ int32x4_t m23 = vusdotq_s32(vdupq_n_s32(0), vcombine_u8(in2, in3), f23_u8);
+
+ int32x4_t tmp_res_low = vpaddq_s32(m01, m23);
+
+ tmp_res_low = vaddq_s32(tmp_res_low, add_const);
+
+ uint16x8_t res =
+ vcombine_u16(vqrshrun_n_s32(tmp_res_low, ROUND0_BITS), vdup_n_u16(0));
+ return vreinterpretq_s16_u16(res);
+}
+
+static INLINE int16x8_t horizontal_filter_8x1_f8(const uint8x16_t in, int sx,
+ int alpha) {
+ const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1));
+
+ // Loading the 8 filter taps
+ int16x8_t f[8];
+ load_filters_8(f, sx, alpha);
+
+ int8x16_t f01_u8 = vcombine_s8(vmovn_s16(f[0]), vmovn_s16(f[1]));
+ int8x16_t f23_u8 = vcombine_s8(vmovn_s16(f[2]), vmovn_s16(f[3]));
+ int8x16_t f45_u8 = vcombine_s8(vmovn_s16(f[4]), vmovn_s16(f[5]));
+ int8x16_t f67_u8 = vcombine_s8(vmovn_s16(f[6]), vmovn_s16(f[7]));
+
+ uint8x8_t in0 = vget_low_u8(in);
+ uint8x8_t in1 = vget_low_u8(vextq_u8(in, in, 1));
+ uint8x8_t in2 = vget_low_u8(vextq_u8(in, in, 2));
+ uint8x8_t in3 = vget_low_u8(vextq_u8(in, in, 3));
+ uint8x8_t in4 = vget_low_u8(vextq_u8(in, in, 4));
+ uint8x8_t in5 = vget_low_u8(vextq_u8(in, in, 5));
+ uint8x8_t in6 = vget_low_u8(vextq_u8(in, in, 6));
+ uint8x8_t in7 = vget_low_u8(vextq_u8(in, in, 7));
+
+ int32x4_t m01 = vusdotq_s32(vdupq_n_s32(0), vcombine_u8(in0, in1), f01_u8);
+ int32x4_t m23 = vusdotq_s32(vdupq_n_s32(0), vcombine_u8(in2, in3), f23_u8);
+ int32x4_t m45 = vusdotq_s32(vdupq_n_s32(0), vcombine_u8(in4, in5), f45_u8);
+ int32x4_t m67 = vusdotq_s32(vdupq_n_s32(0), vcombine_u8(in6, in7), f67_u8);
+
+ int32x4_t tmp_res_low = vpaddq_s32(m01, m23);
+ int32x4_t tmp_res_high = vpaddq_s32(m45, m67);
+
+ tmp_res_low = vaddq_s32(tmp_res_low, add_const);
+ tmp_res_high = vaddq_s32(tmp_res_high, add_const);
+
+ uint16x8_t res = vcombine_u16(vqrshrun_n_s32(tmp_res_low, ROUND0_BITS),
+ vqrshrun_n_s32(tmp_res_high, ROUND0_BITS));
+ return vreinterpretq_s16_u16(res);
+}
+
+static INLINE int16x8_t horizontal_filter_4x1_f1(const uint8x16_t in, int sx) {
+ const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1));
+
+ int16x8_t f_s16 =
+ vld1q_s16((int16_t *)(av1_warped_filter + (sx >> WARPEDDIFF_PREC_BITS)));
+
+ int8x16_t f_s8 = vcombine_s8(vmovn_s16(f_s16), vmovn_s16(f_s16));
+
+ uint8x16_t perm0 = vld1q_u8(&usdot_permute_idx[0]);
+ uint8x16_t perm1 = vld1q_u8(&usdot_permute_idx[16]);
+
+ // Permute samples ready for dot product.
+ // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
+ // { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 }
+ uint8x16_t in_0123 = vqtbl1q_u8(in, perm0);
+ uint8x16_t in_4567 = vqtbl1q_u8(in, perm1);
+
+ int32x4_t m0123 = vusdotq_laneq_s32(vdupq_n_s32(0), in_0123, f_s8, 0);
+ m0123 = vusdotq_laneq_s32(m0123, in_4567, f_s8, 1);
+
+ int32x4_t tmp_res_low = m0123;
+
+ tmp_res_low = vaddq_s32(tmp_res_low, add_const);
+
+ uint16x8_t res =
+ vcombine_u16(vqrshrun_n_s32(tmp_res_low, ROUND0_BITS), vdup_n_u16(0));
+ return vreinterpretq_s16_u16(res);
+}
+
+static INLINE int16x8_t horizontal_filter_8x1_f1(const uint8x16_t in, int sx) {
+ const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1));
+
+ int16x8_t f_s16 =
+ vld1q_s16((int16_t *)(av1_warped_filter + (sx >> WARPEDDIFF_PREC_BITS)));
+
+ int8x16_t f_s8 = vcombine_s8(vmovn_s16(f_s16), vmovn_s16(f_s16));
+
+ uint8x16_t perm0 = vld1q_u8(&usdot_permute_idx[0]);
+ uint8x16_t perm1 = vld1q_u8(&usdot_permute_idx[16]);
+ uint8x16_t perm2 = vld1q_u8(&usdot_permute_idx[32]);
+
+ // Permute samples ready for dot product.
+ // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
+ // { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 }
+ // { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 }
+ uint8x16_t in_0123 = vqtbl1q_u8(in, perm0);
+ uint8x16_t in_4567 = vqtbl1q_u8(in, perm1);
+ uint8x16_t in_89ab = vqtbl1q_u8(in, perm2);
+
+ int32x4_t m0123 = vusdotq_laneq_s32(vdupq_n_s32(0), in_0123, f_s8, 0);
+ m0123 = vusdotq_laneq_s32(m0123, in_4567, f_s8, 1);
+
+ int32x4_t m4567 = vusdotq_laneq_s32(vdupq_n_s32(0), in_4567, f_s8, 0);
+ m4567 = vusdotq_laneq_s32(m4567, in_89ab, f_s8, 1);
+
+ int32x4_t tmp_res_low = m0123;
+ int32x4_t tmp_res_high = m4567;
+
+ tmp_res_low = vaddq_s32(tmp_res_low, add_const);
+ tmp_res_high = vaddq_s32(tmp_res_high, add_const);
+
+ uint16x8_t res = vcombine_u16(vqrshrun_n_s32(tmp_res_low, ROUND0_BITS),
+ vqrshrun_n_s32(tmp_res_high, ROUND0_BITS));
+ return vreinterpretq_s16_u16(res);
+}
+
+static INLINE void vertical_filter_4x1_f1(const int16x8_t *src, int32x4_t *res,
+ int sy) {
+ int16x4_t s0 = vget_low_s16(src[0]);
+ int16x4_t s1 = vget_low_s16(src[1]);
+ int16x4_t s2 = vget_low_s16(src[2]);
+ int16x4_t s3 = vget_low_s16(src[3]);
+ int16x4_t s4 = vget_low_s16(src[4]);
+ int16x4_t s5 = vget_low_s16(src[5]);
+ int16x4_t s6 = vget_low_s16(src[6]);
+ int16x4_t s7 = vget_low_s16(src[7]);
+
+ int16x8_t f =
+ vld1q_s16((int16_t *)(av1_warped_filter + (sy >> WARPEDDIFF_PREC_BITS)));
+
+ int32x4_t m0123 = vmull_lane_s16(s0, vget_low_s16(f), 0);
+ m0123 = vmlal_lane_s16(m0123, s1, vget_low_s16(f), 1);
+ m0123 = vmlal_lane_s16(m0123, s2, vget_low_s16(f), 2);
+ m0123 = vmlal_lane_s16(m0123, s3, vget_low_s16(f), 3);
+ m0123 = vmlal_lane_s16(m0123, s4, vget_high_s16(f), 0);
+ m0123 = vmlal_lane_s16(m0123, s5, vget_high_s16(f), 1);
+ m0123 = vmlal_lane_s16(m0123, s6, vget_high_s16(f), 2);
+ m0123 = vmlal_lane_s16(m0123, s7, vget_high_s16(f), 3);
+
+ *res = m0123;
+}
+
+static INLINE void vertical_filter_4x1_f4(const int16x8_t *src, int32x4_t *res,
+ int sy, int gamma) {
+ int16x8_t s0, s1, s2, s3;
+ transpose_elems_s16_4x8(
+ vget_low_s16(src[0]), vget_low_s16(src[1]), vget_low_s16(src[2]),
+ vget_low_s16(src[3]), vget_low_s16(src[4]), vget_low_s16(src[5]),
+ vget_low_s16(src[6]), vget_low_s16(src[7]), &s0, &s1, &s2, &s3);
+
+ int16x8_t f[4];
+ load_filters_4(f, sy, gamma);
+
+ int32x4_t m0 = vmull_s16(vget_low_s16(s0), vget_low_s16(f[0]));
+ m0 = vmlal_s16(m0, vget_high_s16(s0), vget_high_s16(f[0]));
+ int32x4_t m1 = vmull_s16(vget_low_s16(s1), vget_low_s16(f[1]));
+ m1 = vmlal_s16(m1, vget_high_s16(s1), vget_high_s16(f[1]));
+ int32x4_t m2 = vmull_s16(vget_low_s16(s2), vget_low_s16(f[2]));
+ m2 = vmlal_s16(m2, vget_high_s16(s2), vget_high_s16(f[2]));
+ int32x4_t m3 = vmull_s16(vget_low_s16(s3), vget_low_s16(f[3]));
+ m3 = vmlal_s16(m3, vget_high_s16(s3), vget_high_s16(f[3]));
+
+ int32x4_t m0123_pairs[] = { m0, m1, m2, m3 };
+
+ *res = horizontal_add_4d_s32x4(m0123_pairs);
+}
+
+static INLINE void vertical_filter_8x1_f1(const int16x8_t *src,
+ int32x4_t *res_low,
+ int32x4_t *res_high, int sy) {
+ int16x8_t s0 = src[0];
+ int16x8_t s1 = src[1];
+ int16x8_t s2 = src[2];
+ int16x8_t s3 = src[3];
+ int16x8_t s4 = src[4];
+ int16x8_t s5 = src[5];
+ int16x8_t s6 = src[6];
+ int16x8_t s7 = src[7];
+
+ int16x8_t f =
+ vld1q_s16((int16_t *)(av1_warped_filter + (sy >> WARPEDDIFF_PREC_BITS)));
+
+ int32x4_t m0123 = vmull_lane_s16(vget_low_s16(s0), vget_low_s16(f), 0);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(s1), vget_low_s16(f), 1);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(s2), vget_low_s16(f), 2);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(s3), vget_low_s16(f), 3);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(s4), vget_high_s16(f), 0);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(s5), vget_high_s16(f), 1);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(s6), vget_high_s16(f), 2);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(s7), vget_high_s16(f), 3);
+
+ int32x4_t m4567 = vmull_lane_s16(vget_high_s16(s0), vget_low_s16(f), 0);
+ m4567 = vmlal_lane_s16(m4567, vget_high_s16(s1), vget_low_s16(f), 1);
+ m4567 = vmlal_lane_s16(m4567, vget_high_s16(s2), vget_low_s16(f), 2);
+ m4567 = vmlal_lane_s16(m4567, vget_high_s16(s3), vget_low_s16(f), 3);
+ m4567 = vmlal_lane_s16(m4567, vget_high_s16(s4), vget_high_s16(f), 0);
+ m4567 = vmlal_lane_s16(m4567, vget_high_s16(s5), vget_high_s16(f), 1);
+ m4567 = vmlal_lane_s16(m4567, vget_high_s16(s6), vget_high_s16(f), 2);
+ m4567 = vmlal_lane_s16(m4567, vget_high_s16(s7), vget_high_s16(f), 3);
+
+ *res_low = m0123;
+ *res_high = m4567;
+}
+
+static INLINE void vertical_filter_8x1_f8(const int16x8_t *src,
+ int32x4_t *res_low,
+ int32x4_t *res_high, int sy,
+ int gamma) {
+ int16x8_t s0 = src[0];
+ int16x8_t s1 = src[1];
+ int16x8_t s2 = src[2];
+ int16x8_t s3 = src[3];
+ int16x8_t s4 = src[4];
+ int16x8_t s5 = src[5];
+ int16x8_t s6 = src[6];
+ int16x8_t s7 = src[7];
+ transpose_elems_inplace_s16_8x8(&s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7);
+
+ int16x8_t f[8];
+ load_filters_8(f, sy, gamma);
+
+ int32x4_t m0 = vmull_s16(vget_low_s16(s0), vget_low_s16(f[0]));
+ m0 = vmlal_s16(m0, vget_high_s16(s0), vget_high_s16(f[0]));
+ int32x4_t m1 = vmull_s16(vget_low_s16(s1), vget_low_s16(f[1]));
+ m1 = vmlal_s16(m1, vget_high_s16(s1), vget_high_s16(f[1]));
+ int32x4_t m2 = vmull_s16(vget_low_s16(s2), vget_low_s16(f[2]));
+ m2 = vmlal_s16(m2, vget_high_s16(s2), vget_high_s16(f[2]));
+ int32x4_t m3 = vmull_s16(vget_low_s16(s3), vget_low_s16(f[3]));
+ m3 = vmlal_s16(m3, vget_high_s16(s3), vget_high_s16(f[3]));
+ int32x4_t m4 = vmull_s16(vget_low_s16(s4), vget_low_s16(f[4]));
+ m4 = vmlal_s16(m4, vget_high_s16(s4), vget_high_s16(f[4]));
+ int32x4_t m5 = vmull_s16(vget_low_s16(s5), vget_low_s16(f[5]));
+ m5 = vmlal_s16(m5, vget_high_s16(s5), vget_high_s16(f[5]));
+ int32x4_t m6 = vmull_s16(vget_low_s16(s6), vget_low_s16(f[6]));
+ m6 = vmlal_s16(m6, vget_high_s16(s6), vget_high_s16(f[6]));
+ int32x4_t m7 = vmull_s16(vget_low_s16(s7), vget_low_s16(f[7]));
+ m7 = vmlal_s16(m7, vget_high_s16(s7), vget_high_s16(f[7]));
+
+ int32x4_t m0123_pairs[] = { m0, m1, m2, m3 };
+ int32x4_t m4567_pairs[] = { m4, m5, m6, m7 };
+
+ *res_low = horizontal_add_4d_s32x4(m0123_pairs);
+ *res_high = horizontal_add_4d_s32x4(m4567_pairs);
+}
+
+void av1_warp_affine_neon_i8mm(const int32_t *mat, const uint8_t *ref,
+ int width, int height, int stride, uint8_t *pred,
+ int p_col, int p_row, int p_width, int p_height,
+ int p_stride, int subsampling_x,
+ int subsampling_y, ConvolveParams *conv_params,
+ int16_t alpha, int16_t beta, int16_t gamma,
+ int16_t delta) {
+ av1_warp_affine_common(mat, ref, width, height, stride, pred, p_col, p_row,
+ p_width, p_height, p_stride, subsampling_x,
+ subsampling_y, conv_params, alpha, beta, gamma, delta);
+}
diff --git a/av1/common/arm/warp_plane_sve.c b/av1/common/arm/warp_plane_sve.c
new file mode 100644
index 0000000..2a48c5e
--- /dev/null
+++ b/av1/common/arm/warp_plane_sve.c
@@ -0,0 +1,297 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include "warp_plane_neon.h"
+
+#include <arm_neon_sve_bridge.h>
+
+DECLARE_ALIGNED(16, static const uint8_t, usdot_permute_idx[48]) = {
+ 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6,
+ 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10,
+ 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
+};
+
+static INLINE int64x2_t aom_sdotq_s16(int64x2_t acc, int16x8_t x, int16x8_t y) {
+ // The 16-bit dot product instructions only exist in SVE and not Neon.
+ // We can get away without rewriting the existing Neon code by making use of
+ // the Neon-SVE bridge intrinsics to reinterpret a Neon vector as a SVE
+ // vector with the high part of the vector being "don't care", and then
+ // operating on that instead.
+ // This is clearly suboptimal in machines with a SVE vector length above
+ // 128-bits as the remainder of the vector is wasted, however this appears to
+ // still be beneficial compared to not using the instruction.
+ return svget_neonq_s64(svdot_s64(svset_neonq_s64(svundef_s64(), acc),
+ svset_neonq_s16(svundef_s16(), x),
+ svset_neonq_s16(svundef_s16(), y)));
+}
+
+static INLINE int16x8_t horizontal_filter_4x1_f4(const uint8x16_t in, int sx,
+ int alpha) {
+ const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1));
+
+ // Loading the 8 filter taps
+ int16x8_t f[4];
+ load_filters_4(f, sx, alpha);
+
+ int8x16_t f01_u8 = vcombine_s8(vmovn_s16(f[0]), vmovn_s16(f[1]));
+ int8x16_t f23_u8 = vcombine_s8(vmovn_s16(f[2]), vmovn_s16(f[3]));
+
+ uint8x8_t in0 = vget_low_u8(in);
+ uint8x8_t in1 = vget_low_u8(vextq_u8(in, in, 1));
+ uint8x8_t in2 = vget_low_u8(vextq_u8(in, in, 2));
+ uint8x8_t in3 = vget_low_u8(vextq_u8(in, in, 3));
+
+ int32x4_t m01 = vusdotq_s32(vdupq_n_s32(0), vcombine_u8(in0, in1), f01_u8);
+ int32x4_t m23 = vusdotq_s32(vdupq_n_s32(0), vcombine_u8(in2, in3), f23_u8);
+
+ int32x4_t tmp_res_low = vpaddq_s32(m01, m23);
+
+ tmp_res_low = vaddq_s32(tmp_res_low, add_const);
+
+ uint16x8_t res =
+ vcombine_u16(vqrshrun_n_s32(tmp_res_low, ROUND0_BITS), vdup_n_u16(0));
+ return vreinterpretq_s16_u16(res);
+}
+
+static INLINE int16x8_t horizontal_filter_8x1_f8(const uint8x16_t in, int sx,
+ int alpha) {
+ const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1));
+
+ // Loading the 8 filter taps
+ int16x8_t f[8];
+ load_filters_8(f, sx, alpha);
+
+ int8x16_t f01_u8 = vcombine_s8(vmovn_s16(f[0]), vmovn_s16(f[1]));
+ int8x16_t f23_u8 = vcombine_s8(vmovn_s16(f[2]), vmovn_s16(f[3]));
+ int8x16_t f45_u8 = vcombine_s8(vmovn_s16(f[4]), vmovn_s16(f[5]));
+ int8x16_t f67_u8 = vcombine_s8(vmovn_s16(f[6]), vmovn_s16(f[7]));
+
+ uint8x8_t in0 = vget_low_u8(in);
+ uint8x8_t in1 = vget_low_u8(vextq_u8(in, in, 1));
+ uint8x8_t in2 = vget_low_u8(vextq_u8(in, in, 2));
+ uint8x8_t in3 = vget_low_u8(vextq_u8(in, in, 3));
+ uint8x8_t in4 = vget_low_u8(vextq_u8(in, in, 4));
+ uint8x8_t in5 = vget_low_u8(vextq_u8(in, in, 5));
+ uint8x8_t in6 = vget_low_u8(vextq_u8(in, in, 6));
+ uint8x8_t in7 = vget_low_u8(vextq_u8(in, in, 7));
+
+ int32x4_t m01 = vusdotq_s32(vdupq_n_s32(0), vcombine_u8(in0, in1), f01_u8);
+ int32x4_t m23 = vusdotq_s32(vdupq_n_s32(0), vcombine_u8(in2, in3), f23_u8);
+ int32x4_t m45 = vusdotq_s32(vdupq_n_s32(0), vcombine_u8(in4, in5), f45_u8);
+ int32x4_t m67 = vusdotq_s32(vdupq_n_s32(0), vcombine_u8(in6, in7), f67_u8);
+
+ int32x4_t tmp_res_low = vpaddq_s32(m01, m23);
+ int32x4_t tmp_res_high = vpaddq_s32(m45, m67);
+
+ tmp_res_low = vaddq_s32(tmp_res_low, add_const);
+ tmp_res_high = vaddq_s32(tmp_res_high, add_const);
+
+ uint16x8_t res = vcombine_u16(vqrshrun_n_s32(tmp_res_low, ROUND0_BITS),
+ vqrshrun_n_s32(tmp_res_high, ROUND0_BITS));
+ return vreinterpretq_s16_u16(res);
+}
+
+static INLINE int16x8_t horizontal_filter_4x1_f1(const uint8x16_t in, int sx) {
+ const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1));
+
+ int16x8_t f_s16 =
+ vld1q_s16((int16_t *)(av1_warped_filter + (sx >> WARPEDDIFF_PREC_BITS)));
+
+ int8x16_t f_s8 = vcombine_s8(vmovn_s16(f_s16), vmovn_s16(f_s16));
+
+ uint8x16_t perm0 = vld1q_u8(&usdot_permute_idx[0]);
+ uint8x16_t perm1 = vld1q_u8(&usdot_permute_idx[16]);
+
+ // Permute samples ready for dot product.
+ // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
+ // { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 }
+ uint8x16_t in_0123 = vqtbl1q_u8(in, perm0);
+ uint8x16_t in_4567 = vqtbl1q_u8(in, perm1);
+
+ int32x4_t m0123 = vusdotq_laneq_s32(vdupq_n_s32(0), in_0123, f_s8, 0);
+ m0123 = vusdotq_laneq_s32(m0123, in_4567, f_s8, 1);
+
+ int32x4_t tmp_res_low = m0123;
+
+ tmp_res_low = vaddq_s32(tmp_res_low, add_const);
+
+ uint16x8_t res =
+ vcombine_u16(vqrshrun_n_s32(tmp_res_low, ROUND0_BITS), vdup_n_u16(0));
+ return vreinterpretq_s16_u16(res);
+}
+
+static INLINE int16x8_t horizontal_filter_8x1_f1(const uint8x16_t in, int sx) {
+ const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1));
+
+ int16x8_t f_s16 =
+ vld1q_s16((int16_t *)(av1_warped_filter + (sx >> WARPEDDIFF_PREC_BITS)));
+
+ int8x16_t f_s8 = vcombine_s8(vmovn_s16(f_s16), vmovn_s16(f_s16));
+
+ uint8x16_t perm0 = vld1q_u8(&usdot_permute_idx[0]);
+ uint8x16_t perm1 = vld1q_u8(&usdot_permute_idx[16]);
+ uint8x16_t perm2 = vld1q_u8(&usdot_permute_idx[32]);
+
+ // Permute samples ready for dot product.
+ // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }
+ // { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 }
+ // { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 }
+ uint8x16_t in_0123 = vqtbl1q_u8(in, perm0);
+ uint8x16_t in_4567 = vqtbl1q_u8(in, perm1);
+ uint8x16_t in_89ab = vqtbl1q_u8(in, perm2);
+
+ int32x4_t m0123 = vusdotq_laneq_s32(vdupq_n_s32(0), in_0123, f_s8, 0);
+ m0123 = vusdotq_laneq_s32(m0123, in_4567, f_s8, 1);
+
+ int32x4_t m4567 = vusdotq_laneq_s32(vdupq_n_s32(0), in_4567, f_s8, 0);
+ m4567 = vusdotq_laneq_s32(m4567, in_89ab, f_s8, 1);
+
+ int32x4_t tmp_res_low = m0123;
+ int32x4_t tmp_res_high = m4567;
+
+ tmp_res_low = vaddq_s32(tmp_res_low, add_const);
+ tmp_res_high = vaddq_s32(tmp_res_high, add_const);
+
+ uint16x8_t res = vcombine_u16(vqrshrun_n_s32(tmp_res_low, ROUND0_BITS),
+ vqrshrun_n_s32(tmp_res_high, ROUND0_BITS));
+ return vreinterpretq_s16_u16(res);
+}
+
+static INLINE void vertical_filter_4x1_f1(const int16x8_t *src, int32x4_t *res,
+ int sy) {
+ int16x4_t s0 = vget_low_s16(src[0]);
+ int16x4_t s1 = vget_low_s16(src[1]);
+ int16x4_t s2 = vget_low_s16(src[2]);
+ int16x4_t s3 = vget_low_s16(src[3]);
+ int16x4_t s4 = vget_low_s16(src[4]);
+ int16x4_t s5 = vget_low_s16(src[5]);
+ int16x4_t s6 = vget_low_s16(src[6]);
+ int16x4_t s7 = vget_low_s16(src[7]);
+
+ int16x8_t f =
+ vld1q_s16((int16_t *)(av1_warped_filter + (sy >> WARPEDDIFF_PREC_BITS)));
+
+ int32x4_t m0123 = vmull_lane_s16(s0, vget_low_s16(f), 0);
+ m0123 = vmlal_lane_s16(m0123, s1, vget_low_s16(f), 1);
+ m0123 = vmlal_lane_s16(m0123, s2, vget_low_s16(f), 2);
+ m0123 = vmlal_lane_s16(m0123, s3, vget_low_s16(f), 3);
+ m0123 = vmlal_lane_s16(m0123, s4, vget_high_s16(f), 0);
+ m0123 = vmlal_lane_s16(m0123, s5, vget_high_s16(f), 1);
+ m0123 = vmlal_lane_s16(m0123, s6, vget_high_s16(f), 2);
+ m0123 = vmlal_lane_s16(m0123, s7, vget_high_s16(f), 3);
+
+ *res = m0123;
+}
+
+static INLINE void vertical_filter_4x1_f4(const int16x8_t *src, int32x4_t *res,
+ int sy, int gamma) {
+ int16x8_t s0, s1, s2, s3;
+ transpose_elems_s16_4x8(
+ vget_low_s16(src[0]), vget_low_s16(src[1]), vget_low_s16(src[2]),
+ vget_low_s16(src[3]), vget_low_s16(src[4]), vget_low_s16(src[5]),
+ vget_low_s16(src[6]), vget_low_s16(src[7]), &s0, &s1, &s2, &s3);
+
+ int16x8_t f[4];
+ load_filters_4(f, sy, gamma);
+
+ int64x2_t m0 = aom_sdotq_s16(vdupq_n_s64(0), s0, f[0]);
+ int64x2_t m1 = aom_sdotq_s16(vdupq_n_s64(0), s1, f[1]);
+ int64x2_t m2 = aom_sdotq_s16(vdupq_n_s64(0), s2, f[2]);
+ int64x2_t m3 = aom_sdotq_s16(vdupq_n_s64(0), s3, f[3]);
+
+ int64x2_t m01 = vpaddq_s64(m0, m1);
+ int64x2_t m23 = vpaddq_s64(m2, m3);
+
+ *res = vcombine_s32(vmovn_s64(m01), vmovn_s64(m23));
+}
+
+static INLINE void vertical_filter_8x1_f1(const int16x8_t *src,
+ int32x4_t *res_low,
+ int32x4_t *res_high, int sy) {
+ int16x8_t s0 = src[0];
+ int16x8_t s1 = src[1];
+ int16x8_t s2 = src[2];
+ int16x8_t s3 = src[3];
+ int16x8_t s4 = src[4];
+ int16x8_t s5 = src[5];
+ int16x8_t s6 = src[6];
+ int16x8_t s7 = src[7];
+
+ int16x8_t f =
+ vld1q_s16((int16_t *)(av1_warped_filter + (sy >> WARPEDDIFF_PREC_BITS)));
+
+ int32x4_t m0123 = vmull_lane_s16(vget_low_s16(s0), vget_low_s16(f), 0);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(s1), vget_low_s16(f), 1);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(s2), vget_low_s16(f), 2);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(s3), vget_low_s16(f), 3);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(s4), vget_high_s16(f), 0);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(s5), vget_high_s16(f), 1);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(s6), vget_high_s16(f), 2);
+ m0123 = vmlal_lane_s16(m0123, vget_low_s16(s7), vget_high_s16(f), 3);
+
+ int32x4_t m4567 = vmull_lane_s16(vget_high_s16(s0), vget_low_s16(f), 0);
+ m4567 = vmlal_lane_s16(m4567, vget_high_s16(s1), vget_low_s16(f), 1);
+ m4567 = vmlal_lane_s16(m4567, vget_high_s16(s2), vget_low_s16(f), 2);
+ m4567 = vmlal_lane_s16(m4567, vget_high_s16(s3), vget_low_s16(f), 3);
+ m4567 = vmlal_lane_s16(m4567, vget_high_s16(s4), vget_high_s16(f), 0);
+ m4567 = vmlal_lane_s16(m4567, vget_high_s16(s5), vget_high_s16(f), 1);
+ m4567 = vmlal_lane_s16(m4567, vget_high_s16(s6), vget_high_s16(f), 2);
+ m4567 = vmlal_lane_s16(m4567, vget_high_s16(s7), vget_high_s16(f), 3);
+
+ *res_low = m0123;
+ *res_high = m4567;
+}
+
+static INLINE void vertical_filter_8x1_f8(const int16x8_t *src,
+ int32x4_t *res_low,
+ int32x4_t *res_high, int sy,
+ int gamma) {
+ int16x8_t s0 = src[0];
+ int16x8_t s1 = src[1];
+ int16x8_t s2 = src[2];
+ int16x8_t s3 = src[3];
+ int16x8_t s4 = src[4];
+ int16x8_t s5 = src[5];
+ int16x8_t s6 = src[6];
+ int16x8_t s7 = src[7];
+ transpose_elems_inplace_s16_8x8(&s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7);
+
+ int16x8_t f[8];
+ load_filters_8(f, sy, gamma);
+
+ int64x2_t m0 = aom_sdotq_s16(vdupq_n_s64(0), s0, f[0]);
+ int64x2_t m1 = aom_sdotq_s16(vdupq_n_s64(0), s1, f[1]);
+ int64x2_t m2 = aom_sdotq_s16(vdupq_n_s64(0), s2, f[2]);
+ int64x2_t m3 = aom_sdotq_s16(vdupq_n_s64(0), s3, f[3]);
+ int64x2_t m4 = aom_sdotq_s16(vdupq_n_s64(0), s4, f[4]);
+ int64x2_t m5 = aom_sdotq_s16(vdupq_n_s64(0), s5, f[5]);
+ int64x2_t m6 = aom_sdotq_s16(vdupq_n_s64(0), s6, f[6]);
+ int64x2_t m7 = aom_sdotq_s16(vdupq_n_s64(0), s7, f[7]);
+
+ int64x2_t m01 = vpaddq_s64(m0, m1);
+ int64x2_t m23 = vpaddq_s64(m2, m3);
+ int64x2_t m45 = vpaddq_s64(m4, m5);
+ int64x2_t m67 = vpaddq_s64(m6, m7);
+
+ *res_low = vcombine_s32(vmovn_s64(m01), vmovn_s64(m23));
+ *res_high = vcombine_s32(vmovn_s64(m45), vmovn_s64(m67));
+}
+
+void av1_warp_affine_sve(const int32_t *mat, const uint8_t *ref, int width,
+ int height, int stride, uint8_t *pred, int p_col,
+ int p_row, int p_width, int p_height, int p_stride,
+ int subsampling_x, int subsampling_y,
+ ConvolveParams *conv_params, int16_t alpha,
+ int16_t beta, int16_t gamma, int16_t delta) {
+ av1_warp_affine_common(mat, ref, width, height, stride, pred, p_col, p_row,
+ p_width, p_height, p_stride, subsampling_x,
+ subsampling_y, conv_params, alpha, beta, gamma, delta);
+}
diff --git a/av1/common/arm/wiener_convolve_neon.c b/av1/common/arm/wiener_convolve_neon.c
index d7f511d..6440c16 100644
--- a/av1/common/arm/wiener_convolve_neon.c
+++ b/av1/common/arm/wiener_convolve_neon.c
@@ -15,318 +15,334 @@
#include "config/aom_config.h"
#include "config/av1_rtcd.h"
-#include "aom_dsp/txfm_common.h"
#include "aom_dsp/arm/mem_neon.h"
#include "aom_dsp/arm/transpose_neon.h"
+#include "aom_dsp/txfm_common.h"
#include "aom_ports/mem.h"
#include "av1/common/common.h"
-#include "av1/common/arm/convolve_neon.h"
+#include "av1/common/restoration.h"
-#define HORZ_FILTERING_CORE(t0, t1, t2, t3, t4, t5, t6, res) \
- res0 = vreinterpretq_s16_u16(vaddl_u8(t0, t1)); \
- res1 = vreinterpretq_s16_u16(vaddl_u8(t2, t3)); \
- res2 = vreinterpretq_s16_u16(vaddl_u8(t4, t5)); \
- res3 = vreinterpretq_s16_u16(vmovl_u8(t6)); \
- res = wiener_convolve8_horiz_8x8(res0, res1, res2, res3, filter_x_tmp, bd, \
- conv_params->round_0);
+static INLINE uint16x8_t wiener_convolve5_8_2d_h(
+ const uint8x8_t t0, const uint8x8_t t1, const uint8x8_t t2,
+ const uint8x8_t t3, const uint8x8_t t4, const int16x4_t x_filter,
+ const int32x4_t round_vec, const uint16x8_t im_max_val) {
+ // Since the Wiener filter is symmetric about the middle tap (tap 2) add
+ // mirrored source elements before multiplying filter coefficients.
+ int16x8_t s04 = vreinterpretq_s16_u16(vaddl_u8(t0, t4));
+ int16x8_t s13 = vreinterpretq_s16_u16(vaddl_u8(t1, t3));
+ int16x8_t s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
-#define PROCESS_ROW_FOR_VERTICAL_FILTER \
- __builtin_prefetch(dst_tmp_ptr + 0 * dst_stride); \
- \
- do { \
- s7 = vld1q_s16(s); \
- s += src_stride; \
- \
- t0 = wiener_convolve8_vert_4x8(s0, s1, s2, s3, s4, s5, s6, filter_y_tmp, \
- bd, conv_params->round_1); \
- vst1_u8(d, t0); \
- d += dst_stride; \
- \
- s0 = s1; \
- s1 = s2; \
- s2 = s3; \
- s3 = s4; \
- s4 = s5; \
- s5 = s6; \
- s6 = s7; \
- height--; \
- } while (height > 0);
+ // x_filter[0] = 0. (5-tap filters are 0-padded to 7 taps.)
+ int32x4_t sum_lo = vmlal_lane_s16(round_vec, vget_low_s16(s04), x_filter, 1);
+ sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(s13), x_filter, 2);
+ sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(s2), x_filter, 3);
-static INLINE void process_row_for_horz_filtering(
- uint16_t *dst_ptr, int16_t *filter_x, const uint8_t *src_ptr,
- ptrdiff_t src_stride, ptrdiff_t dst_stride, int round0_bits, int w,
- int height, int bd) {
+ int32x4_t sum_hi = vmlal_lane_s16(round_vec, vget_high_s16(s04), x_filter, 1);
+ sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(s13), x_filter, 2);
+ sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(s2), x_filter, 3);
+
+ uint16x8_t res = vcombine_u16(vqrshrun_n_s32(sum_lo, WIENER_ROUND0_BITS),
+ vqrshrun_n_s32(sum_hi, WIENER_ROUND0_BITS));
+
+ return vminq_u16(res, im_max_val);
+}
+
+static INLINE void convolve_add_src_horiz_5tap_neon(
+ const uint8_t *src_ptr, ptrdiff_t src_stride, uint16_t *dst_ptr,
+ ptrdiff_t dst_stride, int w, int h, const int16x4_t x_filter,
+ const int32x4_t round_vec, const uint16x8_t im_max_val) {
do {
- __builtin_prefetch(src_ptr);
-
- uint8x8_t tt0 = vld1_u8(src_ptr); // a0 a1 a2 a3 a4 a5 a6 a7
-
- __builtin_prefetch(dst_ptr);
-
- const uint8_t *ss = src_ptr + 8;
- uint16_t *d_tmp = dst_ptr;
+ const uint8_t *s = src_ptr;
+ uint16_t *d = dst_ptr;
int width = w;
do {
- uint8x8_t tt7 = vld1_u8(ss); // a8 a9 a10 a11 a12 a13 a14 a15
- uint8x8_t ttemp_0 = tt0;
- tt0 = tt7;
+ uint8x8_t s0, s1, s2, s3, s4;
+ load_u8_8x5(s, 1, &s0, &s1, &s2, &s3, &s4);
- uint8x8_t tt1 = vext_u8(ttemp_0, tt7, 1); // a1 a2 a3 a4 a5 a6 a7 a8
- uint8x8_t tt2 = vext_u8(ttemp_0, tt7, 2); // a2 a3 a4 a5 a6 a7 a8 a9
- uint8x8_t tt3 = vext_u8(ttemp_0, tt7, 3); // a3 a4 a5 a6 a7 a8 a9 a10
- uint8x8_t tt4 = vext_u8(ttemp_0, tt7, 4); // a4 a5 a6 a7 a8 a9 a10 a11
- uint8x8_t tt5 = vext_u8(ttemp_0, tt7, 5); // a5 a6 a7 a8 a9 a10 a11 a12
- uint8x8_t tt6 = vext_u8(ttemp_0, tt7, 6); // a6 a7 a8 a9 a10 a11 a12 a13
- tt7 = vext_u8(ttemp_0, tt7, 7); // a7 a8 a9 a10 a11 a12 a13 a14
+ uint16x8_t d0 = wiener_convolve5_8_2d_h(s0, s1, s2, s3, s4, x_filter,
+ round_vec, im_max_val);
- int16x8_t ttt0 = vreinterpretq_s16_u16(vaddl_u8(ttemp_0, tt6));
- int16x8_t ttt1 = vreinterpretq_s16_u16(vaddl_u8(tt1, tt5));
- int16x8_t ttt2 = vreinterpretq_s16_u16(vaddl_u8(tt2, tt4));
- int16x8_t ttt3 = vreinterpretq_s16_u16(vmovl_u8(tt3));
- uint16x8_t dd0 = wiener_convolve8_horiz_8x8(ttt0, ttt1, ttt2, ttt3,
- filter_x, bd, round0_bits);
+ vst1q_u16(d, d0);
- vst1q_u16(d_tmp, dd0);
-
- ss += 8;
- d_tmp += 8;
+ s += 8;
+ d += 8;
width -= 8;
- } while (width > 0);
-
+ } while (width != 0);
src_ptr += src_stride;
dst_ptr += dst_stride;
- height--;
- } while (height > 0);
+ } while (--h != 0);
}
-/* Wiener filter 2D
- Apply horizontal filter and store in a temporary buffer. When applying
- vertical filter, overwrite the original pixel values.
-*/
+static INLINE uint16x8_t wiener_convolve7_8_2d_h(
+ const uint8x8_t t0, const uint8x8_t t1, const uint8x8_t t2,
+ const uint8x8_t t3, const uint8x8_t t4, const uint8x8_t t5,
+ const uint8x8_t t6, const int16x4_t x_filter, const int32x4_t round_vec,
+ const uint16x8_t im_max_val) {
+ // Since the Wiener filter is symmetric about the middle tap (tap 3) add
+ // mirrored source elements before multiplying by filter coefficients.
+ int16x8_t s06 = vreinterpretq_s16_u16(vaddl_u8(t0, t6));
+ int16x8_t s15 = vreinterpretq_s16_u16(vaddl_u8(t1, t5));
+ int16x8_t s24 = vreinterpretq_s16_u16(vaddl_u8(t2, t4));
+ int16x8_t s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
+
+ int32x4_t sum_lo = vmlal_lane_s16(round_vec, vget_low_s16(s06), x_filter, 0);
+ sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(s15), x_filter, 1);
+ sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(s24), x_filter, 2);
+ sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(s3), x_filter, 3);
+
+ int32x4_t sum_hi = vmlal_lane_s16(round_vec, vget_high_s16(s06), x_filter, 0);
+ sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(s15), x_filter, 1);
+ sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(s24), x_filter, 2);
+ sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(s3), x_filter, 3);
+
+ uint16x8_t res = vcombine_u16(vqrshrun_n_s32(sum_lo, WIENER_ROUND0_BITS),
+ vqrshrun_n_s32(sum_hi, WIENER_ROUND0_BITS));
+
+ return vminq_u16(res, im_max_val);
+}
+
+static INLINE void convolve_add_src_horiz_7tap_neon(
+ const uint8_t *src_ptr, ptrdiff_t src_stride, uint16_t *dst_ptr,
+ ptrdiff_t dst_stride, int w, int h, const int16x4_t x_filter,
+ const int32x4_t round_vec, const uint16x8_t im_max_val) {
+ do {
+ const uint8_t *s = src_ptr;
+ uint16_t *d = dst_ptr;
+ int width = w;
+
+ do {
+ uint8x8_t s0, s1, s2, s3, s4, s5, s6;
+ load_u8_8x7(s, 1, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
+
+ uint16x8_t d0 = wiener_convolve7_8_2d_h(s0, s1, s2, s3, s4, s5, s6,
+ x_filter, round_vec, im_max_val);
+
+ vst1q_u16(d, d0);
+
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width != 0);
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ } while (--h != 0);
+}
+
+static INLINE uint8x8_t wiener_convolve5_8_2d_v(
+ const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
+ const int16x8_t s3, const int16x8_t s4, const int16x4_t y_filter,
+ const int32x4_t round_vec) {
+ // Since the Wiener filter is symmetric about the middle tap (tap 2) add
+ // mirrored source elements before multiplying by filter coefficients.
+ int16x8_t s04 = vaddq_s16(s0, s4);
+ int16x8_t s13 = vaddq_s16(s1, s3);
+
+ int32x4_t sum_lo = vmlal_lane_s16(round_vec, vget_low_s16(s04), y_filter, 1);
+ sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(s13), y_filter, 2);
+ sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(s2), y_filter, 3);
+
+ int32x4_t sum_hi = vmlal_lane_s16(round_vec, vget_high_s16(s04), y_filter, 1);
+ sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(s13), y_filter, 2);
+ sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(s2), y_filter, 3);
+
+ int16x4_t res_lo = vshrn_n_s32(sum_lo, 2 * FILTER_BITS - WIENER_ROUND0_BITS);
+ int16x4_t res_hi = vshrn_n_s32(sum_hi, 2 * FILTER_BITS - WIENER_ROUND0_BITS);
+
+ return vqmovun_s16(vcombine_s16(res_lo, res_hi));
+}
+
+static INLINE void convolve_add_src_vert_5tap_neon(
+ const uint16_t *src, ptrdiff_t src_stride, uint8_t *dst,
+ ptrdiff_t dst_stride, int w, int h, const int16x4_t y_filter,
+ const int32x4_t round_vec) {
+ do {
+ const int16_t *s = (int16_t *)src;
+ uint8_t *d = dst;
+ int height = h;
+
+ while (height > 3) {
+ int16x8_t s0, s1, s2, s3, s4, s5, s6, s7;
+ load_s16_8x8(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7);
+
+ uint8x8_t d0 =
+ wiener_convolve5_8_2d_v(s0, s1, s2, s3, s4, y_filter, round_vec);
+ uint8x8_t d1 =
+ wiener_convolve5_8_2d_v(s1, s2, s3, s4, s5, y_filter, round_vec);
+ uint8x8_t d2 =
+ wiener_convolve5_8_2d_v(s2, s3, s4, s5, s6, y_filter, round_vec);
+ uint8x8_t d3 =
+ wiener_convolve5_8_2d_v(s3, s4, s5, s6, s7, y_filter, round_vec);
+
+ store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ height -= 4;
+ }
+
+ while (height-- != 0) {
+ int16x8_t s0, s1, s2, s3, s4;
+ load_s16_8x5(s, src_stride, &s0, &s1, &s2, &s3, &s4);
+
+ uint8x8_t d0 =
+ wiener_convolve5_8_2d_v(s0, s1, s2, s3, s4, y_filter, round_vec);
+
+ vst1_u8(d, d0);
+
+ d += dst_stride;
+ s += src_stride;
+ }
+
+ src += 8;
+ dst += 8;
+ w -= 8;
+ } while (w != 0);
+}
+
+static INLINE uint8x8_t wiener_convolve7_8_2d_v(
+ const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
+ const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t s6, const int16x4_t y_filter, const int32x4_t round_vec) {
+ // Since the Wiener filter is symmetric about the middle tap (tap 3) add
+ // mirrored source elements before multiplying by filter coefficients.
+ int16x8_t s06 = vaddq_s16(s0, s6);
+ int16x8_t s15 = vaddq_s16(s1, s5);
+ int16x8_t s24 = vaddq_s16(s2, s4);
+
+ int32x4_t sum_lo = vmlal_lane_s16(round_vec, vget_low_s16(s06), y_filter, 0);
+ sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(s15), y_filter, 1);
+ sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(s24), y_filter, 2);
+ sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(s3), y_filter, 3);
+
+ int32x4_t sum_hi = vmlal_lane_s16(round_vec, vget_high_s16(s06), y_filter, 0);
+ sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(s15), y_filter, 1);
+ sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(s24), y_filter, 2);
+ sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(s3), y_filter, 3);
+
+ int16x4_t res_lo = vshrn_n_s32(sum_lo, 2 * FILTER_BITS - WIENER_ROUND0_BITS);
+ int16x4_t res_hi = vshrn_n_s32(sum_hi, 2 * FILTER_BITS - WIENER_ROUND0_BITS);
+
+ return vqmovun_s16(vcombine_s16(res_lo, res_hi));
+}
+
+static INLINE void convolve_add_src_vert_7tap_neon(
+ const uint16_t *src, ptrdiff_t src_stride, uint8_t *dst,
+ ptrdiff_t dst_stride, int w, int h, const int16x4_t y_filter,
+ const int32x4_t round_vec) {
+ do {
+ const int16_t *s = (int16_t *)src;
+ uint8_t *d = dst;
+ int height = h;
+
+ while (height > 3) {
+ int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9;
+ load_s16_8x10(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7, &s8,
+ &s9);
+
+ uint8x8_t d0 = wiener_convolve7_8_2d_v(s0, s1, s2, s3, s4, s5, s6,
+ y_filter, round_vec);
+ uint8x8_t d1 = wiener_convolve7_8_2d_v(s1, s2, s3, s4, s5, s6, s7,
+ y_filter, round_vec);
+ uint8x8_t d2 = wiener_convolve7_8_2d_v(s2, s3, s4, s5, s6, s7, s8,
+ y_filter, round_vec);
+ uint8x8_t d3 = wiener_convolve7_8_2d_v(s3, s4, s5, s6, s7, s8, s9,
+ y_filter, round_vec);
+
+ store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ height -= 4;
+ }
+
+ while (height-- != 0) {
+ int16x8_t s0, s1, s2, s3, s4, s5, s6;
+ load_s16_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
+
+ uint8x8_t d0 = wiener_convolve7_8_2d_v(s0, s1, s2, s3, s4, s5, s6,
+ y_filter, round_vec);
+
+ vst1_u8(d, d0);
+
+ d += dst_stride;
+ s += src_stride;
+ }
+
+ src += 8;
+ dst += 8;
+ w -= 8;
+ } while (w != 0);
+}
+
+static AOM_INLINE int get_wiener_filter_taps(const int16_t *filter) {
+ assert(filter[7] == 0);
+ if (filter[0] == 0 && filter[6] == 0) {
+ return WIENER_WIN_REDUCED;
+ }
+ return WIENER_WIN;
+}
+
+// Wiener filter 2D
+// Apply horizontal filter and store in a temporary buffer. When applying
+// vertical filter, overwrite the original pixel values.
void av1_wiener_convolve_add_src_neon(const uint8_t *src, ptrdiff_t src_stride,
uint8_t *dst, ptrdiff_t dst_stride,
- const int16_t *filter_x, int x_step_q4,
- const int16_t *filter_y, int y_step_q4,
+ const int16_t *x_filter, int x_step_q4,
+ const int16_t *y_filter, int y_step_q4,
int w, int h,
- const ConvolveParams *conv_params) {
- uint8_t *d;
- const uint8_t *src_ptr, *s_tmp;
- uint16_t *dst_ptr;
+ const WienerConvolveParams *conv_params) {
(void)x_step_q4;
(void)y_step_q4;
+ (void)conv_params;
- int height;
- const int bd = 8;
- // Indicates the height needs to be processed during horizontal filtering.
- const int intermediate_height = h + SUBPEL_TAPS - 1;
- const int center_tap = ((SUBPEL_TAPS - 1) / 2);
- int16_t filter_x_tmp[7], filter_y_tmp[7];
+ assert(w % 8 == 0);
+ assert(w <= MAX_SB_SIZE && h <= MAX_SB_SIZE);
+ assert(x_step_q4 == 16 && y_step_q4 == 16);
+ assert(x_filter[7] == 0 && y_filter[7] == 0);
+ // For bd == 8, assert horizontal filtering output will not exceed 15-bit:
+ assert(8 + 1 + FILTER_BITS - conv_params->round_0 <= 15);
DECLARE_ALIGNED(16, uint16_t,
- temp[(MAX_SB_SIZE + HORIZ_EXTRA_ROWS) * MAX_SB_SIZE]);
+ im_block[(MAX_SB_SIZE + WIENER_WIN - 1) * MAX_SB_SIZE]);
- assert(x_step_q4 == 16 && y_step_q4 == 16);
- assert(!(w % 8));
+ const int x_filter_taps = get_wiener_filter_taps(x_filter);
+ const int y_filter_taps = get_wiener_filter_taps(y_filter);
+ int16x4_t x_filter_s16 = vld1_s16(x_filter);
+ int16x4_t y_filter_s16 = vld1_s16(y_filter);
+ // Add 128 to tap 3. (Needed for rounding.)
+ x_filter_s16 = vadd_s16(x_filter_s16, vcreate_s16(128ULL << 48));
+ y_filter_s16 = vadd_s16(y_filter_s16, vcreate_s16(128ULL << 48));
- assert(w <= MAX_SB_SIZE);
- assert(h <= MAX_SB_SIZE);
+ const int im_stride = MAX_SB_SIZE;
+ const int im_h = h + y_filter_taps - 1;
+ const int horiz_offset = x_filter_taps / 2;
+ const int vert_offset = (y_filter_taps / 2) * (int)src_stride;
- assert(filter_x[7] == 0);
- assert(filter_y[7] == 0);
+ const int bd = 8;
+ const uint16x8_t im_max_val =
+ vdupq_n_u16((1 << (bd + 1 + FILTER_BITS - WIENER_ROUND0_BITS)) - 1);
+ const int32x4_t horiz_round_vec = vdupq_n_s32(1 << (bd + FILTER_BITS - 1));
- /* assumption of horizontal filtering output will not exceed 15 bit.
- ((bd) + 1 + FILTER_BITS - conv_params->round_0) <= 15
- 16 - conv_params->round_0 <= 15 -- (conv_params->round_0) >= 1
- */
- assert((conv_params->round_0) >= 1);
+ const int32x4_t vert_round_vec =
+ vdupq_n_s32((1 << (2 * FILTER_BITS - WIENER_ROUND0_BITS - 1)) -
+ (1 << (bd + (2 * FILTER_BITS - WIENER_ROUND0_BITS) - 1)));
- memcpy(&filter_x_tmp[0], filter_x, sizeof(*filter_x) * FILTER_BITS);
- memcpy(&filter_y_tmp[0], filter_y, sizeof(*filter_y) * FILTER_BITS);
-
- filter_x_tmp[3] += (1 << FILTER_BITS);
- filter_y_tmp[3] += (1 << FILTER_BITS);
-
- s_tmp = src - center_tap * src_stride - center_tap;
- dst_ptr = temp;
- src_ptr = s_tmp;
- height = intermediate_height;
-
- // For aarch_64.
-#if AOM_ARCH_AARCH64
- int processed_height = 0;
- uint16_t *d_tmp;
- int width, remaining_height;
- // Start of horizontal filtering.
- if (intermediate_height > 7) {
- uint16x8_t res4, res5, res6, res7, res8, res9, res10, res11;
- uint8x8_t t0, t1, t2, t3, t4, t5, t6, t7;
- do {
- const uint8_t *s;
-
- __builtin_prefetch(src_ptr + 0 * src_stride);
- __builtin_prefetch(src_ptr + 1 * src_stride);
- __builtin_prefetch(src_ptr + 2 * src_stride);
- __builtin_prefetch(src_ptr + 3 * src_stride);
- __builtin_prefetch(src_ptr + 4 * src_stride);
- __builtin_prefetch(src_ptr + 5 * src_stride);
- __builtin_prefetch(src_ptr + 6 * src_stride);
- __builtin_prefetch(src_ptr + 7 * src_stride);
-
- load_u8_8x8(src_ptr, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
- transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
-
- s = src_ptr + 7;
- d_tmp = dst_ptr;
- width = w;
-
- __builtin_prefetch(dst_ptr + 0 * dst_stride);
- __builtin_prefetch(dst_ptr + 1 * dst_stride);
- __builtin_prefetch(dst_ptr + 2 * dst_stride);
- __builtin_prefetch(dst_ptr + 3 * dst_stride);
- __builtin_prefetch(dst_ptr + 4 * dst_stride);
- __builtin_prefetch(dst_ptr + 5 * dst_stride);
- __builtin_prefetch(dst_ptr + 6 * dst_stride);
- __builtin_prefetch(dst_ptr + 7 * dst_stride);
-
- do {
- int16x8_t res0, res1, res2, res3;
- uint8x8_t t8, t9, t10, t11, t12, t13, t14;
- load_u8_8x8(s, src_stride, &t7, &t8, &t9, &t10, &t11, &t12, &t13, &t14);
- transpose_u8_8x8(&t7, &t8, &t9, &t10, &t11, &t12, &t13, &t14);
-
- HORZ_FILTERING_CORE(t0, t6, t1, t5, t2, t4, t3, res4)
- HORZ_FILTERING_CORE(t1, t7, t2, t6, t3, t5, t4, res5)
- HORZ_FILTERING_CORE(t2, t8, t3, t7, t4, t6, t5, res6)
- HORZ_FILTERING_CORE(t3, t9, t4, t8, t5, t7, t6, res7)
- HORZ_FILTERING_CORE(t4, t10, t5, t9, t6, t8, t7, res8)
- HORZ_FILTERING_CORE(t5, t11, t6, t10, t7, t9, t8, res9)
- HORZ_FILTERING_CORE(t6, t12, t7, t11, t8, t10, t9, res10)
- HORZ_FILTERING_CORE(t7, t13, t8, t12, t9, t11, t10, res11)
-
- transpose_u16_8x8(&res4, &res5, &res6, &res7, &res8, &res9, &res10,
- &res11);
- store_u16_8x8(d_tmp, MAX_SB_SIZE, res4, res5, res6, res7, res8, res9,
- res10, res11);
-
- t0 = t8;
- t1 = t9;
- t2 = t10;
- t3 = t11;
- t4 = t12;
- t5 = t13;
- t6 = t14;
- s += 8;
- d_tmp += 8;
- width -= 8;
- } while (width > 0);
- src_ptr += 8 * src_stride;
- dst_ptr += 8 * MAX_SB_SIZE;
- height -= 8;
- processed_height += 8;
- } while (height > 7);
+ if (x_filter_taps == WIENER_WIN_REDUCED) {
+ convolve_add_src_horiz_5tap_neon(src - horiz_offset - vert_offset,
+ src_stride, im_block, im_stride, w, im_h,
+ x_filter_s16, horiz_round_vec, im_max_val);
+ } else {
+ convolve_add_src_horiz_7tap_neon(src - horiz_offset - vert_offset,
+ src_stride, im_block, im_stride, w, im_h,
+ x_filter_s16, horiz_round_vec, im_max_val);
}
- // Process the remaining rows for horizontal filtering.
- remaining_height = intermediate_height - processed_height;
- if (remaining_height)
- process_row_for_horz_filtering(dst_ptr, filter_x_tmp, src_ptr, src_stride,
- MAX_SB_SIZE, conv_params->round_0, w, height,
- bd);
-
- // Start of vertical filtering.
- {
- int16_t *src_tmp_ptr, *s;
- uint8_t *dst_tmp_ptr;
- height = h;
- width = w;
- src_tmp_ptr = (int16_t *)temp;
- dst_tmp_ptr = dst;
- src_stride = MAX_SB_SIZE;
-
- do {
- int16x8_t s0, s1, s2, s3, s4, s5, s6, s7;
- uint8x8_t t0;
- s = src_tmp_ptr;
- d = dst_tmp_ptr;
-
- load_s16_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
- s += 7 * src_stride;
-
- height = h;
-
- do {
- int16x8_t s8, s9, s10;
- uint8x8_t t1, t2, t3;
- __builtin_prefetch(dst_tmp_ptr + 0 * dst_stride);
- __builtin_prefetch(dst_tmp_ptr + 1 * dst_stride);
- __builtin_prefetch(dst_tmp_ptr + 2 * dst_stride);
- __builtin_prefetch(dst_tmp_ptr + 3 * dst_stride);
-
- load_s16_8x4(s, src_stride, &s7, &s8, &s9, &s10);
-
- t0 = wiener_convolve8_vert_4x8(s0, s1, s2, s3, s4, s5, s6, filter_y_tmp,
- bd, conv_params->round_1);
- t1 = wiener_convolve8_vert_4x8(s1, s2, s3, s4, s5, s6, s7, filter_y_tmp,
- bd, conv_params->round_1);
- t2 = wiener_convolve8_vert_4x8(s2, s3, s4, s5, s6, s7, s8, filter_y_tmp,
- bd, conv_params->round_1);
- t3 = wiener_convolve8_vert_4x8(s3, s4, s5, s6, s7, s8, s9, filter_y_tmp,
- bd, conv_params->round_1);
-
- store_u8_8x4(d, dst_stride, t0, t1, t2, t3);
-
- s0 = s4;
- s1 = s5;
- s2 = s6;
- s3 = s7;
- s4 = s8;
- s5 = s9;
- s6 = s10;
- s += 4 * src_stride;
- d += 4 * dst_stride;
- height -= 4;
- } while (height > 3);
-
- if (height) {
- PROCESS_ROW_FOR_VERTICAL_FILTER
- }
- src_tmp_ptr += 8;
- dst_tmp_ptr += 8;
- w -= 8;
- } while (w > 0);
+ if (y_filter_taps == WIENER_WIN_REDUCED) {
+ convolve_add_src_vert_5tap_neon(im_block, im_stride, dst, dst_stride, w, h,
+ y_filter_s16, vert_round_vec);
+ } else {
+ convolve_add_src_vert_7tap_neon(im_block, im_stride, dst, dst_stride, w, h,
+ y_filter_s16, vert_round_vec);
}
-#else
- // Start of horizontal filtering.
- process_row_for_horz_filtering(dst_ptr, filter_x_tmp, src_ptr, src_stride,
- MAX_SB_SIZE, conv_params->round_0, w, height,
- bd);
-
- // Start of vertical filtering.
- {
- int16_t *src_tmp_ptr, *s;
- uint8_t *dst_tmp_ptr;
- src_tmp_ptr = (int16_t *)temp;
- dst_tmp_ptr = dst;
- src_stride = MAX_SB_SIZE;
-
- do {
- uint8x8_t t0;
- int16x8_t s0, s1, s2, s3, s4, s5, s6, s7;
- s = src_tmp_ptr;
- d = dst_tmp_ptr;
-
- load_s16_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
- s += 7 * src_stride;
-
- height = h;
- PROCESS_ROW_FOR_VERTICAL_FILTER
-
- src_tmp_ptr += 8;
- dst_tmp_ptr += 8;
-
- w -= 8;
- } while (w > 0);
- }
-#endif
}
diff --git a/av1/common/av1_loopfilter.h b/av1/common/av1_loopfilter.h
index 78443c7..c9880cf 100644
--- a/av1/common/av1_loopfilter.h
+++ b/av1/common/av1_loopfilter.h
@@ -14,6 +14,8 @@
#include "config/aom_config.h"
+#include "aom/internal/aom_codec_internal.h"
+
#include "aom_ports/mem.h"
#include "av1/common/blockd.h"
#include "av1/common/seg_common.h"
@@ -87,6 +89,7 @@
AV1_DEBLOCKING_PARAMETERS params_buf[MAX_MIB_SIZE];
TX_SIZE tx_buf[MAX_MIB_SIZE];
+ struct aom_internal_error_info error_info;
} LFWorkerData;
/*!\endcond */
diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl
index 17dcc49..38e1da9 100644
--- a/av1/common/av1_rtcd_defs.pl
+++ b/av1/common/av1_rtcd_defs.pl
@@ -94,11 +94,11 @@
add_proto qw/void av1_highbd_convolve_horiz_rs/, "const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const int16_t *x_filters, int x0_qn, int x_step_qn, int bd";
specialize qw/av1_highbd_convolve_horiz_rs sse4_1 neon/;
- add_proto qw/void av1_highbd_wiener_convolve_add_src/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, const ConvolveParams *conv_params, int bd";
- specialize qw/av1_highbd_wiener_convolve_add_src ssse3 avx2/;
+ add_proto qw/void av1_highbd_wiener_convolve_add_src/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, const WienerConvolveParams *conv_params, int bd";
+ specialize qw/av1_highbd_wiener_convolve_add_src ssse3 avx2 neon/;
}
-add_proto qw/void av1_wiener_convolve_add_src/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, const ConvolveParams *conv_params";
+add_proto qw/void av1_wiener_convolve_add_src/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, const WienerConvolveParams *conv_params";
specialize qw/av1_wiener_convolve_add_src sse2 avx2 neon/;
# directional intra predictor functions
@@ -255,11 +255,11 @@
# build compound seg mask functions
add_proto qw/void av1_build_compound_diffwtd_mask/, "uint8_t *mask, DIFFWTD_MASK_TYPE mask_type, const uint8_t *src0, int src0_stride, const uint8_t *src1, int src1_stride, int h, int w";
-specialize qw/av1_build_compound_diffwtd_mask sse4_1 avx2/;
+specialize qw/av1_build_compound_diffwtd_mask neon sse4_1 avx2/;
if (aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") {
add_proto qw/void av1_build_compound_diffwtd_mask_highbd/, "uint8_t *mask, DIFFWTD_MASK_TYPE mask_type, const uint8_t *src0, int src0_stride, const uint8_t *src1, int src1_stride, int h, int w, int bd";
- specialize qw/av1_build_compound_diffwtd_mask_highbd ssse3 avx2/;
+ specialize qw/av1_build_compound_diffwtd_mask_highbd ssse3 avx2 neon/;
}
add_proto qw/void av1_build_compound_diffwtd_mask_d16/, "uint8_t *mask, DIFFWTD_MASK_TYPE mask_type, const CONV_BUF_TYPE *src0, int src0_stride, const CONV_BUF_TYPE *src1, int src1_stride, int h, int w, ConvolveParams *conv_params, int bd";
@@ -296,24 +296,24 @@
const MV *const mv, uint8_t *comp_pred, const uint8_t *pred, int width,
int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref,
int ref_stride, const DIST_WTD_COMP_PARAMS *jcp_param, int subpel_search";
- specialize qw/aom_dist_wtd_comp_avg_upsampled_pred ssse3/;
+ specialize qw/aom_dist_wtd_comp_avg_upsampled_pred ssse3 neon/;
if (aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") {
add_proto qw/void aom_highbd_upsampled_pred/, "MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col,
const MV *const mv, uint8_t *comp_pred8, int width, int height, int subpel_x_q3,
int subpel_y_q3, const uint8_t *ref8, int ref_stride, int bd, int subpel_search";
- specialize qw/aom_highbd_upsampled_pred sse2/;
+ specialize qw/aom_highbd_upsampled_pred sse2 neon/;
add_proto qw/void aom_highbd_comp_avg_upsampled_pred/, "MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col,
const MV *const mv, uint8_t *comp_pred8, const uint8_t *pred8, int width,
int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref8, int ref_stride, int bd, int subpel_search";
- specialize qw/aom_highbd_comp_avg_upsampled_pred sse2/;
+ specialize qw/aom_highbd_comp_avg_upsampled_pred sse2 neon/;
add_proto qw/void aom_highbd_dist_wtd_comp_avg_upsampled_pred/, "MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col,
const MV *const mv, uint8_t *comp_pred8, const uint8_t *pred8, int width,
int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref8,
int ref_stride, int bd, const DIST_WTD_COMP_PARAMS *jcp_param, int subpel_search";
- specialize qw/aom_highbd_dist_wtd_comp_avg_upsampled_pred sse2/;
+ specialize qw/aom_highbd_dist_wtd_comp_avg_upsampled_pred sse2 neon/;
}
# the transform coefficients are held in 32-bit
@@ -396,15 +396,16 @@
#
if (aom_config("CONFIG_REALTIME_ONLY") ne "yes") {
add_proto qw/void av1_apply_temporal_filter/, "const struct yv12_buffer_config *frame_to_filter, const struct macroblockd *mbd, const BLOCK_SIZE block_size, const int mb_row, const int mb_col, const int num_planes, const double *noise_levels, const MV *subblock_mvs, const int *subblock_mses, const int q_factor, const int filter_strength, int tf_wgt_calc_lvl, const uint8_t *pred, uint32_t *accum, uint16_t *count";
- specialize qw/av1_apply_temporal_filter sse2 avx2 neon/;
+ specialize qw/av1_apply_temporal_filter sse2 avx2 neon neon_dotprod/;
add_proto qw/double av1_estimate_noise_from_single_plane/, "const uint8_t *src, int height, int width, int stride, int edge_thresh";
- specialize qw/av1_estimate_noise_from_single_plane avx2/;
+ specialize qw/av1_estimate_noise_from_single_plane avx2 neon/;
if (aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") {
add_proto qw/void av1_highbd_apply_temporal_filter/, "const struct yv12_buffer_config *frame_to_filter, const struct macroblockd *mbd, const BLOCK_SIZE block_size, const int mb_row, const int mb_col, const int num_planes, const double *noise_levels, const MV *subblock_mvs, const int *subblock_mses, const int q_factor, const int filter_strength, int tf_wgt_calc_lvl, const uint8_t *pred, uint32_t *accum, uint16_t *count";
- specialize qw/av1_highbd_apply_temporal_filter sse2 avx2/;
+ specialize qw/av1_highbd_apply_temporal_filter sse2 avx2 neon/;
add_proto qw/double av1_highbd_estimate_noise_from_single_plane/, "const uint16_t *src, int height, int width, int stride, int bit_depth, int edge_thresh";
+ specialize qw/av1_highbd_estimate_noise_from_single_plane neon/;
}
}
@@ -419,7 +420,7 @@
# ENCODEMB INVOKE
if (aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") {
add_proto qw/int64_t av1_highbd_block_error/, "const tran_low_t *coeff, const tran_low_t *dqcoeff, intptr_t block_size, int64_t *ssz, int bd";
- specialize qw/av1_highbd_block_error sse2 avx2/;
+ specialize qw/av1_highbd_block_error sse2 avx2 neon/;
}
if (aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") {
@@ -438,9 +439,9 @@
add_proto qw/uint64_t av1_wedge_sse_from_residuals/, "const int16_t *r1, const int16_t *d, const uint8_t *m, int N";
specialize qw/av1_wedge_sse_from_residuals sse2 avx2 neon/;
add_proto qw/int8_t av1_wedge_sign_from_residuals/, "const int16_t *ds, const uint8_t *m, int N, int64_t limit";
- specialize qw/av1_wedge_sign_from_residuals sse2 avx2/;
+ specialize qw/av1_wedge_sign_from_residuals sse2 avx2 neon/;
add_proto qw/void av1_wedge_compute_delta_squares/, "int16_t *d, const int16_t *a, const int16_t *b, int N";
- specialize qw/av1_wedge_compute_delta_squares sse2 avx2/;
+ specialize qw/av1_wedge_compute_delta_squares sse2 avx2 neon/;
# hash
add_proto qw/uint32_t av1_get_crc32c_value/, "void *crc_calculator, uint8_t *p, size_t length";
@@ -448,43 +449,43 @@
if (aom_config("CONFIG_REALTIME_ONLY") ne "yes") {
add_proto qw/void av1_compute_stats/, "int wiener_win, const uint8_t *dgd8, const uint8_t *src8, int16_t *dgd_avg, int16_t *src_avg, int h_start, int h_end, int v_start, int v_end, int dgd_stride, int src_stride, int64_t *M, int64_t *H, int use_downsampled_wiener_stats";
- specialize qw/av1_compute_stats sse4_1 avx2/;
- add_proto qw/void av1_calc_proj_params/, " const uint8_t *src8, int width, int height, int src_stride, const uint8_t *dat8, int dat_stride, int32_t *flt0, int flt0_stride, int32_t *flt1, int flt1_stride, int64_t H[2][2], int64_t C[2], const sgr_params_type *params";
- specialize qw/av1_calc_proj_params sse4_1 avx2/;
- add_proto qw/int64_t av1_lowbd_pixel_proj_error/, " const uint8_t *src8, int width, int height, int src_stride, const uint8_t *dat8, int dat_stride, int32_t *flt0, int flt0_stride, int32_t *flt1, int flt1_stride, int xq[2], const sgr_params_type *params";
+ specialize qw/av1_compute_stats sse4_1 avx2 neon/;
+ add_proto qw/void av1_calc_proj_params/, "const uint8_t *src8, int width, int height, int src_stride, const uint8_t *dat8, int dat_stride, int32_t *flt0, int flt0_stride, int32_t *flt1, int flt1_stride, int64_t H[2][2], int64_t C[2], const sgr_params_type *params";
+ specialize qw/av1_calc_proj_params sse4_1 avx2 neon/;
+ add_proto qw/int64_t av1_lowbd_pixel_proj_error/, "const uint8_t *src8, int width, int height, int src_stride, const uint8_t *dat8, int dat_stride, int32_t *flt0, int flt0_stride, int32_t *flt1, int flt1_stride, int xq[2], const sgr_params_type *params";
specialize qw/av1_lowbd_pixel_proj_error sse4_1 avx2 neon/;
if (aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") {
- add_proto qw/void av1_calc_proj_params_high_bd/, " const uint8_t *src8, int width, int height, int src_stride, const uint8_t *dat8, int dat_stride, int32_t *flt0, int flt0_stride, int32_t *flt1, int flt1_stride, int64_t H[2][2], int64_t C[2], const sgr_params_type *params";
- specialize qw/av1_calc_proj_params_high_bd sse4_1 avx2/;
- add_proto qw/int64_t av1_highbd_pixel_proj_error/, " const uint8_t *src8, int width, int height, int src_stride, const uint8_t *dat8, int dat_stride, int32_t *flt0, int flt0_stride, int32_t *flt1, int flt1_stride, int xq[2], const sgr_params_type *params";
+ add_proto qw/void av1_calc_proj_params_high_bd/, "const uint8_t *src8, int width, int height, int src_stride, const uint8_t *dat8, int dat_stride, int32_t *flt0, int flt0_stride, int32_t *flt1, int flt1_stride, int64_t H[2][2], int64_t C[2], const sgr_params_type *params";
+ specialize qw/av1_calc_proj_params_high_bd sse4_1 avx2 neon/;
+ add_proto qw/int64_t av1_highbd_pixel_proj_error/, "const uint8_t *src8, int width, int height, int src_stride, const uint8_t *dat8, int dat_stride, int32_t *flt0, int flt0_stride, int32_t *flt1, int flt1_stride, int xq[2], const sgr_params_type *params";
specialize qw/av1_highbd_pixel_proj_error sse4_1 avx2/;
add_proto qw/void av1_compute_stats_highbd/, "int wiener_win, const uint8_t *dgd8, const uint8_t *src8, int h_start, int h_end, int v_start, int v_end, int dgd_stride, int src_stride, int64_t *M, int64_t *H, aom_bit_depth_t bit_depth";
- specialize qw/av1_compute_stats_highbd sse4_1 avx2/;
+ specialize qw/av1_compute_stats_highbd sse4_1 avx2 neon/;
}
}
- add_proto qw/void av1_get_horver_correlation_full/, " const int16_t *diff, int stride, int w, int h, float *hcorr, float *vcorr";
+ add_proto qw/void av1_get_horver_correlation_full/, "const int16_t *diff, int stride, int w, int h, float *hcorr, float *vcorr";
specialize qw/av1_get_horver_correlation_full sse4_1 avx2 neon/;
- add_proto qw/void av1_nn_predict/, " const float *input_nodes, const NN_CONFIG *const nn_config, int reduce_prec, float *const output";
+ add_proto qw/void av1_nn_predict/, "const float *input_nodes, const NN_CONFIG *const nn_config, int reduce_prec, float *const output";
- add_proto qw/void av1_nn_fast_softmax_16/, " const float *input_nodes, float *output";
+ add_proto qw/void av1_nn_fast_softmax_16/, "const float *input_nodes, float *output";
if (aom_config("CONFIG_EXCLUDE_SIMD_MISMATCH") ne "yes") {
- specialize qw/av1_nn_predict sse3 neon/;
+ specialize qw/av1_nn_predict sse3 avx2 neon/;
specialize qw/av1_nn_fast_softmax_16 sse3/;
}
# CNN functions
if (aom_config("CONFIG_REALTIME_ONLY") ne "yes") {
- add_proto qw/void av1_cnn_activate/, " float **input, int channels, int width, int height, int stride, ACTIVATION layer_activation";
- add_proto qw/void av1_cnn_add/, " float **input, int channels, int width, int height, int stride, const float **add";
- add_proto qw/bool av1_cnn_predict/, " const float **input, int in_width, int in_height, int in_stride, const CNN_CONFIG *cnn_config, const CNN_THREAD_DATA *thread_data, CNN_MULTI_OUT *output_struct";
- add_proto qw/void av1_cnn_convolve_no_maxpool_padding_valid/, " const float **input, int in_width, int in_height, int in_stride, const CNN_LAYER_CONFIG *layer_config, float **output, int out_stride, int start_idx, int cstep, int channel_step";
+ add_proto qw/void av1_cnn_activate/, "float **input, int channels, int width, int height, int stride, ACTIVATION layer_activation";
+ add_proto qw/void av1_cnn_add/, "float **input, int channels, int width, int height, int stride, const float **add";
+ add_proto qw/bool av1_cnn_predict/, "const float **input, int in_width, int in_height, int in_stride, const CNN_CONFIG *cnn_config, const CNN_THREAD_DATA *thread_data, CNN_MULTI_OUT *output_struct";
+ add_proto qw/void av1_cnn_convolve_no_maxpool_padding_valid/, "const float **input, int in_width, int in_height, int in_stride, const CNN_LAYER_CONFIG *layer_config, float **output, int out_stride, int start_idx, int cstep, int channel_step";
if (aom_config("CONFIG_EXCLUDE_SIMD_MISMATCH") ne "yes") {
specialize qw/av1_cnn_convolve_no_maxpool_padding_valid avx2/;
}
- add_proto qw/void av1_cnn_deconvolve/, " const float **input, int in_width, int in_height, int in_stride, const CNN_LAYER_CONFIG *layer_config, float **output, int out_stride";
+ add_proto qw/void av1_cnn_deconvolve/, "const float **input, int in_width, int in_height, int in_stride, const CNN_LAYER_CONFIG *layer_config, float **output, int out_stride";
add_proto qw/void av1_cnn_batchnorm/, "float **image, int channels, int width, int height, int stride, const float *gamma, const float *beta, const float *mean, const float *std";
}
@@ -540,17 +541,14 @@
# WARPED_MOTION / GLOBAL_MOTION functions
if (aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") {
add_proto qw/void av1_highbd_warp_affine/, "const int32_t *mat, const uint16_t *ref, int width, int height, int stride, uint16_t *pred, int p_col, int p_row, int p_width, int p_height, int p_stride, int subsampling_x, int subsampling_y, int bd, ConvolveParams *conv_params, int16_t alpha, int16_t beta, int16_t gamma, int16_t delta";
- specialize qw/av1_highbd_warp_affine sse4_1 avx2/;
+ specialize qw/av1_highbd_warp_affine sse4_1 avx2 neon/;
}
add_proto qw/void av1_warp_affine/, "const int32_t *mat, const uint8_t *ref, int width, int height, int stride, uint8_t *pred, int p_col, int p_row, int p_width, int p_height, int p_stride, int subsampling_x, int subsampling_y, ConvolveParams *conv_params, int16_t alpha, int16_t beta, int16_t gamma, int16_t delta";
-specialize qw/av1_warp_affine sse4_1 avx2 neon/;
-
-add_proto qw/int64_t av1_calc_frame_error/, "const uint8_t *const ref, int stride, const uint8_t *const dst, int p_width, int p_height, int p_stride";
-specialize qw/av1_calc_frame_error sse2 avx2/;
+specialize qw/av1_warp_affine sse4_1 avx2 neon neon_i8mm sve/;
# LOOP_RESTORATION functions
-add_proto qw/void av1_apply_selfguided_restoration/, "const uint8_t *dat, int width, int height, int stride, int eps, const int *xqd, uint8_t *dst, int dst_stride, int32_t *tmpbuf, int bit_depth, int highbd";
+add_proto qw/int av1_apply_selfguided_restoration/, "const uint8_t *dat, int width, int height, int stride, int eps, const int *xqd, uint8_t *dst, int dst_stride, int32_t *tmpbuf, int bit_depth, int highbd";
specialize qw/av1_apply_selfguided_restoration sse4_1 avx2 neon/;
add_proto qw/int av1_selfguided_restoration/, "const uint8_t *dgd8, int width, int height,
@@ -561,16 +559,22 @@
# CONVOLVE_ROUND/COMPOUND_ROUND functions
add_proto qw/void av1_convolve_2d_sr/, "const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_qn, const int subpel_y_qn, ConvolveParams *conv_params";
+add_proto qw/void av1_convolve_2d_sr_intrabc/, "const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_qn, const int subpel_y_qn, ConvolveParams *conv_params";
add_proto qw/void av1_convolve_x_sr/, "const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const int subpel_x_qn, ConvolveParams *conv_params";
+add_proto qw/void av1_convolve_x_sr_intrabc/, "const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const int subpel_x_qn, ConvolveParams *conv_params";
add_proto qw/void av1_convolve_y_sr/, "const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_y, const int subpel_y_qn";
+add_proto qw/void av1_convolve_y_sr_intrabc/, "const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_y, const int subpel_y_qn";
add_proto qw/void av1_dist_wtd_convolve_2d/, "const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_qn, const int subpel_y_qn, ConvolveParams *conv_params";
add_proto qw/void av1_dist_wtd_convolve_2d_copy/, "const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, ConvolveParams *conv_params";
add_proto qw/void av1_dist_wtd_convolve_x/, "const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const int subpel_x_qn, ConvolveParams *conv_params";
add_proto qw/void av1_dist_wtd_convolve_y/, "const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_y, const int subpel_y_qn, ConvolveParams *conv_params";
if(aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") {
add_proto qw/void av1_highbd_convolve_2d_sr/, "const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_qn, const int subpel_y_qn, ConvolveParams *conv_params, int bd";
+ add_proto qw/void av1_highbd_convolve_2d_sr_intrabc/, "const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_qn, const int subpel_y_qn, ConvolveParams *conv_params, int bd";
add_proto qw/void av1_highbd_convolve_x_sr/, "const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const int subpel_x_qn, ConvolveParams *conv_params, int bd";
+ add_proto qw/void av1_highbd_convolve_x_sr_intrabc/, "const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const int subpel_x_qn, ConvolveParams *conv_params, int bd";
add_proto qw/void av1_highbd_convolve_y_sr/, "const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_y, const int subpel_y_qn, int bd";
+ add_proto qw/void av1_highbd_convolve_y_sr_intrabc/, "const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_y, const int subpel_y_qn, int bd";
add_proto qw/void av1_highbd_dist_wtd_convolve_2d/, "const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_qn, const int subpel_y_qn, ConvolveParams *conv_params, int bd";
add_proto qw/void av1_highbd_dist_wtd_convolve_x/, "const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const int subpel_x_qn, ConvolveParams *conv_params, int bd";
add_proto qw/void av1_highbd_dist_wtd_convolve_y/, "const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_y, const int subpel_y_qn, ConvolveParams *conv_params, int bd";
@@ -580,13 +584,16 @@
add_proto qw/void av1_convolve_2d_scale/, "const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_qn, const int x_step_qn, const int subpel_y_qn, const int y_step_qn, ConvolveParams *conv_params";
- specialize qw/av1_convolve_2d_sr sse2 avx2 neon/;
- specialize qw/av1_convolve_x_sr sse2 avx2 neon/;
+ specialize qw/av1_convolve_2d_sr sse2 avx2 neon neon_dotprod neon_i8mm/;
+ specialize qw/av1_convolve_2d_sr_intrabc neon/;
+ specialize qw/av1_convolve_x_sr sse2 avx2 neon neon_dotprod neon_i8mm/;
+ specialize qw/av1_convolve_x_sr_intrabc neon/;
specialize qw/av1_convolve_y_sr sse2 avx2 neon/;
+ specialize qw/av1_convolve_y_sr_intrabc neon/;
specialize qw/av1_convolve_2d_scale sse4_1/;
- specialize qw/av1_dist_wtd_convolve_2d sse2 ssse3 avx2 neon/;
+ specialize qw/av1_dist_wtd_convolve_2d sse2 ssse3 avx2 neon neon_dotprod neon_i8mm/;
specialize qw/av1_dist_wtd_convolve_2d_copy sse2 avx2 neon/;
- specialize qw/av1_dist_wtd_convolve_x sse2 avx2 neon/;
+ specialize qw/av1_dist_wtd_convolve_x sse2 avx2 neon neon_dotprod neon_i8mm/;
specialize qw/av1_dist_wtd_convolve_y sse2 avx2 neon/;
if(aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") {
specialize qw/av1_highbd_dist_wtd_convolve_2d sse4_1 avx2 neon/;
@@ -594,21 +601,26 @@
specialize qw/av1_highbd_dist_wtd_convolve_y sse4_1 avx2 neon/;
specialize qw/av1_highbd_dist_wtd_convolve_2d_copy sse4_1 avx2 neon/;
specialize qw/av1_highbd_convolve_2d_sr ssse3 avx2 neon/;
+ specialize qw/av1_highbd_convolve_2d_sr_intrabc neon/;
specialize qw/av1_highbd_convolve_x_sr ssse3 avx2 neon/;
+ specialize qw/av1_highbd_convolve_x_sr_intrabc neon/;
specialize qw/av1_highbd_convolve_y_sr ssse3 avx2 neon/;
+ specialize qw/av1_highbd_convolve_y_sr_intrabc neon/;
specialize qw/av1_highbd_convolve_2d_scale sse4_1 neon/;
}
# INTRA_EDGE functions
add_proto qw/void av1_filter_intra_edge/, "uint8_t *p, int sz, int strength";
-specialize qw/av1_filter_intra_edge sse4_1/;
+specialize qw/av1_filter_intra_edge sse4_1 neon/;
add_proto qw/void av1_upsample_intra_edge/, "uint8_t *p, int sz";
-specialize qw/av1_upsample_intra_edge sse4_1/;
+specialize qw/av1_upsample_intra_edge sse4_1 neon/;
-add_proto qw/void av1_filter_intra_edge_high/, "uint16_t *p, int sz, int strength";
-specialize qw/av1_filter_intra_edge_high sse4_1/;
-add_proto qw/void av1_upsample_intra_edge_high/, "uint16_t *p, int sz, int bd";
-specialize qw/av1_upsample_intra_edge_high sse4_1/;
+if (aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") {
+ add_proto qw/void av1_highbd_filter_intra_edge/, "uint16_t *p, int sz, int strength";
+ specialize qw/av1_highbd_filter_intra_edge sse4_1 neon/;
+ add_proto qw/void av1_highbd_upsample_intra_edge/, "uint16_t *p, int sz, int bd";
+ specialize qw/av1_highbd_upsample_intra_edge sse4_1 neon/;
+}
# CFL
add_proto qw/cfl_subtract_average_fn cfl_get_subtract_average_fn/, "TX_SIZE tx_size";
diff --git a/av1/common/av1_txfm.c b/av1/common/av1_txfm.c
index ac43402..011403b 100644
--- a/av1/common/av1_txfm.c
+++ b/av1/common/av1_txfm.c
@@ -15,7 +15,7 @@
#include "av1/common/av1_txfm.h"
// av1_cospi_arr[i][j] = (int)round(cos(PI*j/128) * (1<<(cos_bit_min+i)));
-const int32_t av1_cospi_arr_data[7][64] = {
+const int32_t av1_cospi_arr_data[4][64] = {
{ 1024, 1024, 1023, 1021, 1019, 1016, 1013, 1009, 1004, 999, 993, 987, 980,
972, 964, 955, 946, 936, 926, 915, 903, 891, 878, 865, 851, 837,
822, 807, 792, 775, 759, 742, 724, 706, 688, 669, 650, 630, 610,
@@ -38,36 +38,153 @@
7027, 6921, 6811, 6698, 6580, 6458, 6333, 6203, 6070, 5933, 5793,
5649, 5501, 5351, 5197, 5040, 4880, 4717, 4551, 4383, 4212, 4038,
3862, 3683, 3503, 3320, 3135, 2948, 2760, 2570, 2378, 2185, 1990,
- 1795, 1598, 1401, 1202, 1003, 803, 603, 402, 201 },
- { 16384, 16379, 16364, 16340, 16305, 16261, 16207, 16143, 16069, 15986, 15893,
- 15791, 15679, 15557, 15426, 15286, 15137, 14978, 14811, 14635, 14449, 14256,
- 14053, 13842, 13623, 13395, 13160, 12916, 12665, 12406, 12140, 11866, 11585,
- 11297, 11003, 10702, 10394, 10080, 9760, 9434, 9102, 8765, 8423, 8076,
- 7723, 7366, 7005, 6639, 6270, 5897, 5520, 5139, 4756, 4370, 3981,
- 3590, 3196, 2801, 2404, 2006, 1606, 1205, 804, 402 },
- { 32768, 32758, 32729, 32679, 32610, 32522, 32413, 32286, 32138, 31972, 31786,
- 31581, 31357, 31114, 30853, 30572, 30274, 29957, 29622, 29269, 28899, 28511,
- 28106, 27684, 27246, 26791, 26320, 25833, 25330, 24812, 24279, 23732, 23170,
- 22595, 22006, 21403, 20788, 20160, 19520, 18868, 18205, 17531, 16846, 16151,
- 15447, 14733, 14010, 13279, 12540, 11793, 11039, 10279, 9512, 8740, 7962,
- 7180, 6393, 5602, 4808, 4011, 3212, 2411, 1608, 804 },
- { 65536, 65516, 65457, 65358, 65220, 65043, 64827, 64571, 64277, 63944, 63572,
- 63162, 62714, 62228, 61705, 61145, 60547, 59914, 59244, 58538, 57798, 57022,
- 56212, 55368, 54491, 53581, 52639, 51665, 50660, 49624, 48559, 47464, 46341,
- 45190, 44011, 42806, 41576, 40320, 39040, 37736, 36410, 35062, 33692, 32303,
- 30893, 29466, 28020, 26558, 25080, 23586, 22078, 20557, 19024, 17479, 15924,
- 14359, 12785, 11204, 9616, 8022, 6424, 4821, 3216, 1608 }
+ 1795, 1598, 1401, 1202, 1003, 803, 603, 402, 201 }
};
// av1_sinpi_arr_data[i][j] = (int)round((sqrt(2) * sin(j*Pi/9) * 2 / 3) * (1
// << (cos_bit_min + i))) modified so that elements j=1,2 sum to element j=4.
-const int32_t av1_sinpi_arr_data[7][5] = {
- { 0, 330, 621, 836, 951 }, { 0, 660, 1241, 1672, 1901 },
- { 0, 1321, 2482, 3344, 3803 }, { 0, 2642, 4964, 6689, 7606 },
- { 0, 5283, 9929, 13377, 15212 }, { 0, 10566, 19858, 26755, 30424 },
- { 0, 21133, 39716, 53510, 60849 }
+const int32_t av1_sinpi_arr_data[4][5] = { { 0, 330, 621, 836, 951 },
+ { 0, 660, 1241, 1672, 1901 },
+ { 0, 1321, 2482, 3344, 3803 },
+ { 0, 2642, 4964, 6689, 7606 } };
+
+// The reduced bit-width arrays are only used in the Arm Neon implementations
+// in av1_fwd_txfm2d_neon.c for now.
+#if HAVE_NEON
+// Constants are stored in groups of four, where symmetrical constants in the
+// cospi array are stored adjacent in memory, followed immediately by the same
+// constants but negated, i.e.:
+// f(i,j) = (int)round(cos(PI*j/128) * (1<<(cos_bit_min+i))) << (3-i)
+// and then in memory we store 4-tuples of constants together as:
+// f4(i,j) = [ f(i,j), f(i,64-j), -f(i,j), -f(i,64-j) ]
+//
+// Constants are stored in Q2.13 format, see:
+// https://en.wikipedia.org/wiki/Q_(number_format)
+//
+// The order of the constants is such that increasing subdivisions of 64 store
+// f4 tuples contiguously:
+// av1_cospi_arr_q13_data[i] = {
+// f4(i,32), // f(i,32) twice
+// f4(i,16), // f(i,16) and f(i,48), f4(i,32) skipped since present above.
+// f4(i,8), f(i,24), // f4(i,16) and f4(i,32) skipped since present above.
+// f4(i,4), f(i,12), f4(i,20), f4(i,28),
+// f4(i,2), f4(i,6), f4(i,10), f4(i,14), f4(i,18), ...
+// f4(i,1), f4(i,3), f4(i,5), f4(i,7), f4(i,9), f4(i,11), ...
+// }
+const int16_t av1_cospi_arr_q13_data[4][128] = {
+ {
+ 5792, 5792, -5792, -5792, 7568, 3136, -7568, -3136, 8032, 1600,
+ -8032, -1600, 6808, 4552, -6808, -4552, 8152, 800, -8152, -800,
+ 7840, 2376, -7840, -2376, 7224, 3864, -7224, -3864, 6336, 5200,
+ -6336, -5200, 8184, 400, -8184, -400, 8104, 1200, -8104, -1200,
+ 7944, 1992, -7944, -1992, 7712, 2760, -7712, -2760, 7408, 3504,
+ -7408, -3504, 7024, 4208, -7024, -4208, 6576, 4880, -6576, -4880,
+ 6072, 5504, -6072, -5504, 8192, 200, -8192, -200, 8168, 600,
+ -8168, -600, 8128, 1000, -8128, -1000, 8072, 1400, -8072, -1400,
+ 7992, 1792, -7992, -1792, 7896, 2184, -7896, -2184, 7776, 2568,
+ -7776, -2568, 7640, 2952, -7640, -2952, 7488, 3320, -7488, -3320,
+ 7320, 3680, -7320, -3680, 7128, 4040, -7128, -4040, 6920, 4384,
+ -6920, -4384, 6696, 4720, -6696, -4720, 6456, 5040, -6456, -5040,
+ 6200, 5352, -6200, -5352, 5936, 5648, -5936, -5648,
+ },
+ {
+ 5792, 5792, -5792, -5792, 7568, 3136, -7568, -3136, 8036, 1600,
+ -8036, -1600, 6812, 4552, -6812, -4552, 8152, 804, -8152, -804,
+ 7840, 2380, -7840, -2380, 7224, 3860, -7224, -3860, 6332, 5196,
+ -6332, -5196, 8184, 400, -8184, -400, 8104, 1204, -8104, -1204,
+ 7948, 1992, -7948, -1992, 7712, 2760, -7712, -2760, 7404, 3504,
+ -7404, -3504, 7028, 4212, -7028, -4212, 6580, 4880, -6580, -4880,
+ 6068, 5500, -6068, -5500, 8188, 200, -8188, -200, 8168, 604,
+ -8168, -604, 8132, 1004, -8132, -1004, 8072, 1400, -8072, -1400,
+ 7992, 1796, -7992, -1796, 7896, 2184, -7896, -2184, 7780, 2568,
+ -7780, -2568, 7644, 2948, -7644, -2948, 7488, 3320, -7488, -3320,
+ 7316, 3684, -7316, -3684, 7128, 4036, -7128, -4036, 6920, 4384,
+ -6920, -4384, 6696, 4716, -6696, -4716, 6460, 5040, -6460, -5040,
+ 6204, 5352, -6204, -5352, 5932, 5648, -5932, -5648,
+ },
+ {
+ 5792, 5792, -5792, -5792, 7568, 3134, -7568, -3134, 8034, 1598,
+ -8034, -1598, 6812, 4552, -6812, -4552, 8152, 802, -8152, -802,
+ 7840, 2378, -7840, -2378, 7224, 3862, -7224, -3862, 6332, 5196,
+ -6332, -5196, 8182, 402, -8182, -402, 8104, 1202, -8104, -1202,
+ 7946, 1990, -7946, -1990, 7714, 2760, -7714, -2760, 7406, 3502,
+ -7406, -3502, 7026, 4212, -7026, -4212, 6580, 4880, -6580, -4880,
+ 6070, 5502, -6070, -5502, 8190, 202, -8190, -202, 8170, 602,
+ -8170, -602, 8130, 1002, -8130, -1002, 8072, 1400, -8072, -1400,
+ 7992, 1794, -7992, -1794, 7896, 2184, -7896, -2184, 7778, 2570,
+ -7778, -2570, 7644, 2948, -7644, -2948, 7490, 3320, -7490, -3320,
+ 7318, 3684, -7318, -3684, 7128, 4038, -7128, -4038, 6922, 4382,
+ -6922, -4382, 6698, 4718, -6698, -4718, 6458, 5040, -6458, -5040,
+ 6204, 5350, -6204, -5350, 5934, 5648, -5934, -5648,
+ },
+ {
+ 5793, 5793, -5793, -5793, 7568, 3135, -7568, -3135, 8035, 1598,
+ -8035, -1598, 6811, 4551, -6811, -4551, 8153, 803, -8153, -803,
+ 7839, 2378, -7839, -2378, 7225, 3862, -7225, -3862, 6333, 5197,
+ -6333, -5197, 8182, 402, -8182, -402, 8103, 1202, -8103, -1202,
+ 7946, 1990, -7946, -1990, 7713, 2760, -7713, -2760, 7405, 3503,
+ -7405, -3503, 7027, 4212, -7027, -4212, 6580, 4880, -6580, -4880,
+ 6070, 5501, -6070, -5501, 8190, 201, -8190, -201, 8170, 603,
+ -8170, -603, 8130, 1003, -8130, -1003, 8071, 1401, -8071, -1401,
+ 7993, 1795, -7993, -1795, 7895, 2185, -7895, -2185, 7779, 2570,
+ -7779, -2570, 7643, 2948, -7643, -2948, 7489, 3320, -7489, -3320,
+ 7317, 3683, -7317, -3683, 7128, 4038, -7128, -4038, 6921, 4383,
+ -6921, -4383, 6698, 4717, -6698, -4717, 6458, 5040, -6458, -5040,
+ 6203, 5351, -6203, -5351, 5933, 5649, -5933, -5649,
+ }
};
+// av1_sinpi_arr_q13_data[i][j] =
+// round((sqrt2 * sin((j+1)*Pi/9) * 2/3) * (1 << (cos_bit_min + i))) << (3-i)
+// modified so that elements j=0,1 sum to element j=3.
+// See also: https://en.wikipedia.org/wiki/Q_(number_format)
+const int16_t av1_sinpi_arr_q13_data[4][4] = { { 2640, 4968, 6688, 7608 },
+ { 2640, 4964, 6688, 7604 },
+ { 2642, 4964, 6688, 7606 },
+ { 2642, 4964, 6689, 7606 } };
+
+// Constants are stored in pairs, where symmetrical constants in the
+// cospi array are stored adjacent in memory, i.e.:
+// f(i,j) = (int)round(cos(PI*j/128) * (1<<(cos_bit_min+i)))
+// and then in memory we store 4-tuples of constants together as:
+// f2(i,j) = [ f(i,j), f(i,64-j) ]
+const int32_t av1_cospi_arr_s32_data[4][66] = {
+ {
+ 1024, 0, 1024, 25, 1023, 50, 1021, 75, 1019, 100, 1016,
+ 125, 1013, 150, 1009, 175, 1004, 200, 999, 224, 993, 249,
+ 987, 273, 980, 297, 972, 321, 964, 345, 955, 369, 946,
+ 392, 936, 415, 926, 438, 915, 460, 903, 483, 891, 505,
+ 878, 526, 865, 548, 851, 569, 837, 590, 822, 610, 807,
+ 630, 792, 650, 775, 669, 759, 688, 742, 706, 724, 724,
+ },
+ {
+ 2048, 0, 2047, 50, 2046, 100, 2042, 151, 2038, 201, 2033,
+ 251, 2026, 301, 2018, 350, 2009, 400, 1998, 449, 1987, 498,
+ 1974, 546, 1960, 595, 1945, 642, 1928, 690, 1911, 737, 1892,
+ 784, 1872, 830, 1851, 876, 1829, 921, 1806, 965, 1782, 1009,
+ 1757, 1053, 1730, 1096, 1703, 1138, 1674, 1179, 1645, 1220, 1615,
+ 1260, 1583, 1299, 1551, 1338, 1517, 1375, 1483, 1412, 1448, 1448,
+ },
+ {
+ 4096, 0, 4095, 101, 4091, 201, 4085, 301, 4076, 401, 4065,
+ 501, 4052, 601, 4036, 700, 4017, 799, 3996, 897, 3973, 995,
+ 3948, 1092, 3920, 1189, 3889, 1285, 3857, 1380, 3822, 1474, 3784,
+ 1567, 3745, 1660, 3703, 1751, 3659, 1842, 3612, 1931, 3564, 2019,
+ 3513, 2106, 3461, 2191, 3406, 2276, 3349, 2359, 3290, 2440, 3229,
+ 2520, 3166, 2598, 3102, 2675, 3035, 2751, 2967, 2824, 2896, 2896,
+ },
+ {
+ 8192, 0, 8190, 201, 8182, 402, 8170, 603, 8153, 803, 8130,
+ 1003, 8103, 1202, 8071, 1401, 8035, 1598, 7993, 1795, 7946, 1990,
+ 7895, 2185, 7839, 2378, 7779, 2570, 7713, 2760, 7643, 2948, 7568,
+ 3135, 7489, 3320, 7405, 3503, 7317, 3683, 7225, 3862, 7128, 4038,
+ 7027, 4212, 6921, 4383, 6811, 4551, 6698, 4717, 6580, 4880, 6458,
+ 5040, 6333, 5197, 6203, 5351, 6070, 5501, 5933, 5649, 5793, 5793,
+ }
+};
+
+#endif // HAVE_NEON
+
void av1_round_shift_array_c(int32_t *arr, int size, int bit) {
int i;
if (bit == 0) {
diff --git a/av1/common/av1_txfm.h b/av1/common/av1_txfm.h
index be1164f..7ad70af 100644
--- a/av1/common/av1_txfm.h
+++ b/av1/common/av1_txfm.h
@@ -31,13 +31,12 @@
#define DO_RANGE_CHECK_CLAMP 0
#endif
-extern const int32_t av1_cospi_arr_data[7][64];
-extern const int32_t av1_sinpi_arr_data[7][5];
+extern const int32_t av1_cospi_arr_data[4][64];
+extern const int32_t av1_sinpi_arr_data[4][5];
#define MAX_TXFM_STAGE_NUM 12
static const int cos_bit_min = 10;
-static const int cos_bit_max = 16;
#define NewSqrt2Bits ((int32_t)12)
// 2^12 * sqrt(2)
@@ -53,6 +52,29 @@
return av1_sinpi_arr_data[n - cos_bit_min];
}
+// The reduced bit-width and permuted arrays are only used in the Arm Neon
+// implementations in av1_fwd_txfm2d_neon.c and highbd_fwd_txfm_neon.c for now.
+#if HAVE_NEON
+// Store cospi/sinpi costants in Q2.13 format.
+// See: https://en.wikipedia.org/wiki/Q_(number_format)
+extern const int16_t av1_cospi_arr_q13_data[4][128];
+extern const int16_t av1_sinpi_arr_q13_data[4][4];
+
+extern const int32_t av1_cospi_arr_s32_data[4][66];
+
+static INLINE const int16_t *cospi_arr_q13(int n) {
+ return av1_cospi_arr_q13_data[n - cos_bit_min];
+}
+
+static INLINE const int16_t *sinpi_arr_q13(int n) {
+ return av1_sinpi_arr_q13_data[n - cos_bit_min];
+}
+
+static INLINE const int32_t *cospi_arr_s32(int n) {
+ return av1_cospi_arr_s32_data[n - cos_bit_min];
+}
+#endif // HAVE_NEON
+
static INLINE int32_t range_check_value(int32_t value, int8_t bit) {
#if CONFIG_COEFFICIENT_RANGE_CHECKING
const int64_t max_value = (1LL << (bit - 1)) - 1;
diff --git a/av1/common/cdef.c b/av1/common/cdef.c
index 202f9d6..12e9545 100644
--- a/av1/common/cdef.c
+++ b/av1/common/cdef.c
@@ -20,6 +20,7 @@
#include "av1/common/cdef.h"
#include "av1/common/cdef_block.h"
#include "av1/common/reconinter.h"
+#include "av1/common/thread_common.h"
static int is_8x8_block_skip(MB_MODE_INFO **grid, int mi_row, int mi_col,
int mi_stride) {
@@ -413,12 +414,25 @@
uint16_t **const linebuf, uint16_t **const colbuf,
uint16_t *const src, int fbr,
cdef_init_fb_row_t cdef_init_fb_row_fn,
- struct AV1CdefSyncData *const cdef_sync) {
+ struct AV1CdefSyncData *const cdef_sync,
+ struct aom_internal_error_info *error_info) {
+ // TODO(aomedia:3276): Pass error_info to the low-level functions as required
+ // in future to handle error propagation.
+ (void)error_info;
CdefBlockInfo fb_info;
int cdef_left[MAX_MB_PLANE] = { 1, 1, 1 };
const int nhfb = (cm->mi_params.mi_cols + MI_SIZE_64X64 - 1) / MI_SIZE_64X64;
cdef_init_fb_row_fn(cm, xd, &fb_info, linebuf, src, cdef_sync, fbr);
+#if CONFIG_MULTITHREAD
+ if (cdef_sync && cm->cdef_info.allocated_num_workers > 1) {
+ pthread_mutex_lock(cdef_sync->mutex_);
+ const bool cdef_mt_exit = cdef_sync->cdef_mt_exit;
+ pthread_mutex_unlock(cdef_sync->mutex_);
+ // Exit in case any worker has encountered an error.
+ if (cdef_mt_exit) return;
+ }
+#endif
for (int fbc = 0; fbc < nhfb; fbc++) {
fb_info.frame_boundary[LEFT] = (MI_SIZE_64X64 * fbc == 0) ? 1 : 0;
if (fbc != nhfb - 1)
@@ -447,5 +461,6 @@
for (int fbr = 0; fbr < nvfb; fbr++)
av1_cdef_fb_row(cm, xd, cm->cdef_info.linebuf, cm->cdef_info.colbuf,
- cm->cdef_info.srcbuf, fbr, cdef_init_fb_row_fn, NULL);
+ cm->cdef_info.srcbuf, fbr, cdef_init_fb_row_fn, NULL,
+ xd->error_info);
}
diff --git a/av1/common/cdef.h b/av1/common/cdef.h
index e166f4b..a56cd9d 100644
--- a/av1/common/cdef.h
+++ b/av1/common/cdef.h
@@ -98,7 +98,8 @@
uint16_t **const linebuf, uint16_t **const colbuf,
uint16_t *const src, int fbr,
cdef_init_fb_row_t cdef_init_fb_row_fn,
- struct AV1CdefSyncData *const cdef_sync);
+ struct AV1CdefSyncData *const cdef_sync,
+ struct aom_internal_error_info *error_info);
void av1_cdef_init_fb_row(const AV1_COMMON *const cm,
const MACROBLOCKD *const xd,
CdefBlockInfo *const fb_info,
diff --git a/av1/common/cdef_block.h b/av1/common/cdef_block.h
index 455a896..b5e4f12 100644
--- a/av1/common/cdef_block.h
+++ b/av1/common/cdef_block.h
@@ -47,9 +47,6 @@
int coeff_shift, int block_width,
int block_height);
-void copy_cdef_16bit_to_16bit(uint16_t *dst, int dstride, uint16_t *src,
- cdef_list *dlist, int cdef_count, int bsize);
-
void av1_cdef_filter_fb(uint8_t *dst8, uint16_t *dst16, int dstride,
const uint16_t *in, int xdec, int ydec,
int dir[CDEF_NBLOCKS][CDEF_NBLOCKS], int *dirinit,
diff --git a/av1/common/cfl.c b/av1/common/cfl.c
index 6d4221e..0e37d45 100644
--- a/av1/common/cfl.c
+++ b/av1/common/cfl.c
@@ -184,8 +184,8 @@
cfl->are_parameters_computed = 1;
}
-void cfl_predict_block(MACROBLOCKD *const xd, uint8_t *dst, int dst_stride,
- TX_SIZE tx_size, int plane) {
+void av1_cfl_predict_block(MACROBLOCKD *const xd, uint8_t *dst, int dst_stride,
+ TX_SIZE tx_size, int plane) {
CFL_CTX *const cfl = &xd->cfl;
MB_MODE_INFO *mbmi = xd->mi[0];
assert(is_cfl_allowed(xd));
diff --git a/av1/common/cfl.h b/av1/common/cfl.h
index af8b833..dcaa87b 100644
--- a/av1/common/cfl.h
+++ b/av1/common/cfl.h
@@ -72,8 +72,8 @@
cfl->dc_pred_is_cached[CFL_PRED_V] = false;
}
-void cfl_predict_block(MACROBLOCKD *const xd, uint8_t *dst, int dst_stride,
- TX_SIZE tx_size, int plane);
+void av1_cfl_predict_block(MACROBLOCKD *const xd, uint8_t *dst, int dst_stride,
+ TX_SIZE tx_size, int plane);
void cfl_store_block(MACROBLOCKD *const xd, BLOCK_SIZE bsize, TX_SIZE tx_size);
diff --git a/av1/common/convolve.c b/av1/common/convolve.c
index 9bca542..bb72e0c 100644
--- a/av1/common/convolve.c
+++ b/av1/common/convolve.c
@@ -787,6 +787,120 @@
}
}
+// This function is exactly the same as av1_highbd_convolve_2d_sr_c, and is an
+// optimized version for intrabc. Use the following 2-tap filter:
+// DECLARE_ALIGNED(256, static const int16_t,
+// av1_intrabc_bilinear_filter[2 * SUBPEL_SHIFTS]) = {
+// 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+// 64, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+// };
+void av1_highbd_convolve_2d_sr_intrabc_c(
+ const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w,
+ int h, const InterpFilterParams *filter_params_x,
+ const InterpFilterParams *filter_params_y, const int subpel_x_qn,
+ const int subpel_y_qn, ConvolveParams *conv_params, int bd) {
+ const int bits =
+ FILTER_BITS * 2 - conv_params->round_0 - conv_params->round_1;
+ assert(bits >= 0);
+ assert(subpel_x_qn == 8);
+ assert(subpel_y_qn == 8);
+ assert(filter_params_x->taps == 2 && filter_params_y->taps == 2);
+ assert((conv_params->round_0 + conv_params->round_1) == 2 * FILTER_BITS);
+ (void)filter_params_x;
+ (void)subpel_x_qn;
+ (void)filter_params_y;
+ (void)subpel_y_qn;
+ (void)conv_params;
+
+ int16_t im_block[(MAX_SB_SIZE + MAX_FILTER_TAP - 1) * MAX_SB_SIZE];
+ int im_h = h + 1;
+ int im_stride = w;
+ assert(w <= MAX_SB_SIZE && h <= MAX_SB_SIZE);
+
+ // horizontal filter
+ // explicitly operate for subpel_x_qn = 8.
+ int16_t *im = im_block;
+ for (int y = 0; y < im_h; ++y) {
+ for (int x = 0; x < w; ++x) {
+ int32_t sum = (1 << (bd + FILTER_BITS - 1)) + 64 * (src[x] + src[x + 1]);
+ assert(0 <= sum && sum < (1 << (bd + FILTER_BITS + 1)));
+ sum = ROUND_POWER_OF_TWO(sum, conv_params->round_0);
+ im[x] = sum;
+ }
+ src += src_stride;
+ im += im_stride;
+ }
+
+ // vertical filter
+ // explicitly operate for subpel_y_qn = 8.
+ int16_t *src_vert = im_block;
+ const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0;
+ for (int y = 0; y < h; ++y) {
+ for (int x = 0; x < w; ++x) {
+ const int32_t sum =
+ (1 << offset_bits) + 64 * (src_vert[x] + src_vert[im_stride + x]);
+ assert(0 <= sum && sum < (1 << (offset_bits + 2)));
+ const int32_t res = ROUND_POWER_OF_TWO(sum, conv_params->round_1) -
+ ((1 << (offset_bits - conv_params->round_1)) +
+ (1 << (offset_bits - conv_params->round_1 - 1)));
+
+ dst[x] = clip_pixel_highbd(ROUND_POWER_OF_TWO(res, bits), bd);
+ }
+ src_vert += im_stride;
+ dst += dst_stride;
+ }
+}
+
+// This function is exactly the same as av1_highbd_convolve_y_sr_c, and is an
+// optimized version for intrabc.
+void av1_highbd_convolve_y_sr_intrabc_c(
+ const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w,
+ int h, const InterpFilterParams *filter_params_y, const int subpel_y_qn,
+ int bd) {
+ assert(subpel_y_qn == 8);
+ assert(filter_params_y->taps == 2);
+ (void)filter_params_y;
+ (void)subpel_y_qn;
+
+ // vertical filter
+ // explicitly operate for subpel_y_qn = 8.
+ for (int y = 0; y < h; ++y) {
+ for (int x = 0; x < w; ++x) {
+ const int32_t res = src[x] + src[src_stride + x];
+ dst[x] = clip_pixel_highbd(ROUND_POWER_OF_TWO(res, 1), bd);
+ }
+ src += src_stride;
+ dst += dst_stride;
+ }
+}
+
+// This function is exactly the same as av1_highbd_convolve_x_sr_c, and is an
+// optimized version for intrabc.
+void av1_highbd_convolve_x_sr_intrabc_c(
+ const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w,
+ int h, const InterpFilterParams *filter_params_x, const int subpel_x_qn,
+ ConvolveParams *conv_params, int bd) {
+ const int bits = FILTER_BITS - conv_params->round_0;
+ assert(bits >= 0);
+ assert(subpel_x_qn == 8);
+ assert(filter_params_x->taps == 2);
+ assert((conv_params->round_0 + conv_params->round_1) == 2 * FILTER_BITS);
+ (void)filter_params_x;
+ (void)subpel_x_qn;
+
+ // horizontal filter
+ // explicitly operate for subpel_x_qn = 8.
+ for (int y = 0; y < h; ++y) {
+ for (int x = 0; x < w; ++x) {
+ int32_t res = 64 * (src[x] + src[x + 1]);
+ res = ROUND_POWER_OF_TWO(res, conv_params->round_0);
+ dst[x] = clip_pixel_highbd(ROUND_POWER_OF_TWO(res, bits), bd);
+ }
+ src += src_stride;
+ dst += dst_stride;
+ }
+}
+
void av1_highbd_dist_wtd_convolve_2d_c(
const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w,
int h, const InterpFilterParams *filter_params_x,
@@ -1139,14 +1253,31 @@
(void)dst_stride;
const uint16_t *src = CONVERT_TO_SHORTPTR(src8);
- const int need_filter_params_x = (subpel_x_qn != 0) | scaled;
- const int need_filter_params_y = (subpel_y_qn != 0) | scaled;
- const InterpFilterParams *filter_params_x =
- need_filter_params_x ? interp_filters[0] : NULL;
- const InterpFilterParams *filter_params_y =
- need_filter_params_y ? interp_filters[1] : NULL;
+ const InterpFilterParams *filter_params_x = interp_filters[0];
+ const InterpFilterParams *filter_params_y = interp_filters[1];
uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
+ // 2-tap filter indicates that it is for IntraBC.
+ if (filter_params_x->taps == 2 || filter_params_y->taps == 2) {
+ assert(filter_params_x->taps == 2 && filter_params_y->taps == 2);
+ assert(!scaled);
+ if (subpel_x_qn && subpel_y_qn) {
+ av1_highbd_convolve_2d_sr_intrabc_c(
+ src, src_stride, dst, dst_stride, w, h, filter_params_x,
+ filter_params_y, subpel_x_qn, subpel_y_qn, conv_params, bd);
+ return;
+ } else if (subpel_x_qn) {
+ av1_highbd_convolve_x_sr_intrabc_c(src, src_stride, dst, dst_stride, w, h,
+ filter_params_x, subpel_x_qn,
+ conv_params, bd);
+ return;
+ } else if (subpel_y_qn) {
+ av1_highbd_convolve_y_sr_intrabc_c(src, src_stride, dst, dst_stride, w, h,
+ filter_params_y, subpel_y_qn, bd);
+ return;
+ }
+ }
+
if (scaled) {
if (conv_params->is_compound) {
assert(conv_params->dst != NULL);
@@ -1269,7 +1400,7 @@
const int16_t *filter_x, int x_step_q4,
const int16_t *filter_y, int y_step_q4,
int w, int h,
- const ConvolveParams *conv_params) {
+ const WienerConvolveParams *conv_params) {
const InterpKernel *const filters_x = get_filter_base(filter_x);
const int x0_q4 = get_filter_offset(filter_x, filters_x);
@@ -1349,7 +1480,7 @@
const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst,
ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4,
const int16_t *filter_y, int y_step_q4, int w, int h,
- const ConvolveParams *conv_params, int bd) {
+ const WienerConvolveParams *conv_params, int bd) {
const InterpKernel *const filters_x = get_filter_base(filter_x);
const int x0_q4 = get_filter_offset(filter_x, filters_x);
diff --git a/av1/common/convolve.h b/av1/common/convolve.h
index 36c0c84..d6dd876 100644
--- a/av1/common/convolve.h
+++ b/av1/common/convolve.h
@@ -31,6 +31,11 @@
int bck_offset;
} ConvolveParams;
+typedef struct WienerConvolveParams {
+ int round_0;
+ int round_1;
+} WienerConvolveParams;
+
#define ROUND0_BITS 3
#define COMPOUND_ROUND1_BITS 7
#define WIENER_ROUND0_BITS 3
@@ -99,11 +104,8 @@
return get_conv_params_no_round(do_average, plane, NULL, 0, 0, bd);
}
-static INLINE ConvolveParams get_conv_params_wiener(int bd) {
- ConvolveParams conv_params;
- (void)bd;
- conv_params.do_average = 0;
- conv_params.is_compound = 0;
+static INLINE WienerConvolveParams get_conv_params_wiener(int bd) {
+ WienerConvolveParams conv_params;
conv_params.round_0 = WIENER_ROUND0_BITS;
conv_params.round_1 = 2 * FILTER_BITS - conv_params.round_0;
const int intbufrange = bd + FILTER_BITS - conv_params.round_0 + 2;
@@ -112,9 +114,6 @@
conv_params.round_0 += intbufrange - 16;
conv_params.round_1 -= intbufrange - 16;
}
- conv_params.dst = NULL;
- conv_params.dst_stride = 0;
- conv_params.plane = 0;
return conv_params;
}
diff --git a/av1/common/enums.h b/av1/common/enums.h
index fb4d756..b99a138 100644
--- a/av1/common/enums.h
+++ b/av1/common/enums.h
@@ -27,8 +27,6 @@
/*!\cond */
-#undef MAX_SB_SIZE
-
// Max superblock size
#define MAX_SB_SIZE_LOG2 7
#define MAX_SB_SIZE (1 << MAX_SB_SIZE_LOG2)
@@ -199,7 +197,7 @@
#define TX_PAD_END 16
#define TX_PAD_2D ((32 + TX_PAD_HOR) * (32 + TX_PAD_VER) + TX_PAD_END)
-// Number of maxium size transform blocks in the maximum size superblock
+// Number of maximum size transform blocks in the maximum size superblock
#define MAX_TX_BLOCKS_IN_MAX_SB_LOG2 ((MAX_SB_SIZE_LOG2 - MAX_TX_SIZE_LOG2) * 2)
#define MAX_TX_BLOCKS_IN_MAX_SB (1 << MAX_TX_BLOCKS_IN_MAX_SB_LOG2)
diff --git a/av1/common/reconintra.c b/av1/common/reconintra.c
index 3704b8a..67fb13f 100644
--- a/av1/common/reconintra.c
+++ b/av1/common/reconintra.c
@@ -1048,41 +1048,6 @@
p_left[-1] = s;
}
-void av1_filter_intra_edge_high_c(uint16_t *p, int sz, int strength) {
- if (!strength) return;
-
- const int kernel[INTRA_EDGE_FILT][INTRA_EDGE_TAPS] = { { 0, 4, 8, 4, 0 },
- { 0, 5, 6, 5, 0 },
- { 2, 4, 4, 4, 2 } };
- const int filt = strength - 1;
- uint16_t edge[129];
-
- memcpy(edge, p, sz * sizeof(*p));
- for (int i = 1; i < sz; i++) {
- int s = 0;
- for (int j = 0; j < INTRA_EDGE_TAPS; j++) {
- int k = i - 2 + j;
- k = (k < 0) ? 0 : k;
- k = (k > sz - 1) ? sz - 1 : k;
- s += edge[k] * kernel[filt][j];
- }
- s = (s + 8) >> 4;
- p[i] = s;
- }
-}
-
-#if CONFIG_AV1_HIGHBITDEPTH
-static void filter_intra_edge_corner_high(uint16_t *p_above, uint16_t *p_left) {
- const int kernel[3] = { 5, 6, 5 };
-
- int s = (p_left[0] * kernel[0]) + (p_above[-1] * kernel[1]) +
- (p_above[0] * kernel[2]);
- s = (s + 8) >> 4;
- p_above[-1] = s;
- p_left[-1] = s;
-}
-#endif
-
void av1_upsample_intra_edge_c(uint8_t *p, int sz) {
// interpolate half-sample positions
assert(sz <= MAX_UPSAMPLE_SZ);
@@ -1106,206 +1071,6 @@
}
}
-void av1_upsample_intra_edge_high_c(uint16_t *p, int sz, int bd) {
- // interpolate half-sample positions
- assert(sz <= MAX_UPSAMPLE_SZ);
-
- uint16_t in[MAX_UPSAMPLE_SZ + 3];
- // copy p[-1..(sz-1)] and extend first and last samples
- in[0] = p[-1];
- in[1] = p[-1];
- for (int i = 0; i < sz; i++) {
- in[i + 2] = p[i];
- }
- in[sz + 2] = p[sz - 1];
-
- // interpolate half-sample edge positions
- p[-2] = in[0];
- for (int i = 0; i < sz; i++) {
- int s = -in[i] + (9 * in[i + 1]) + (9 * in[i + 2]) - in[i + 3];
- s = (s + 8) >> 4;
- s = clip_pixel_highbd(s, bd);
- p[2 * i - 1] = s;
- p[2 * i] = in[i + 2];
- }
-}
-#if CONFIG_AV1_HIGHBITDEPTH
-static void build_intra_predictors_high(
- const uint8_t *ref8, int ref_stride, uint8_t *dst8, int dst_stride,
- PREDICTION_MODE mode, int p_angle, FILTER_INTRA_MODE filter_intra_mode,
- TX_SIZE tx_size, int disable_edge_filter, int n_top_px, int n_topright_px,
- int n_left_px, int n_bottomleft_px, int intra_edge_filter_type,
- int bit_depth) {
- int i;
- uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
- uint16_t *ref = CONVERT_TO_SHORTPTR(ref8);
- DECLARE_ALIGNED(16, uint16_t, left_data[NUM_INTRA_NEIGHBOUR_PIXELS]);
- DECLARE_ALIGNED(16, uint16_t, above_data[NUM_INTRA_NEIGHBOUR_PIXELS]);
- uint16_t *const above_row = above_data + 16;
- uint16_t *const left_col = left_data + 16;
- const int txwpx = tx_size_wide[tx_size];
- const int txhpx = tx_size_high[tx_size];
- int need_left = extend_modes[mode] & NEED_LEFT;
- int need_above = extend_modes[mode] & NEED_ABOVE;
- int need_above_left = extend_modes[mode] & NEED_ABOVELEFT;
- const uint16_t *above_ref = ref - ref_stride;
- const uint16_t *left_ref = ref - 1;
- const int is_dr_mode = av1_is_directional_mode(mode);
- const int use_filter_intra = filter_intra_mode != FILTER_INTRA_MODES;
- int base = 128 << (bit_depth - 8);
- // The left_data, above_data buffers must be zeroed to fix some intermittent
- // valgrind errors. Uninitialized reads in intra pred modules (e.g. width = 4
- // path in av1_highbd_dr_prediction_z2_avx2()) from left_data, above_data are
- // seen to be the potential reason for this issue.
- aom_memset16(left_data, base + 1, NUM_INTRA_NEIGHBOUR_PIXELS);
- aom_memset16(above_data, base - 1, NUM_INTRA_NEIGHBOUR_PIXELS);
-
- // The default values if ref pixels are not available:
- // base base-1 base-1 .. base-1 base-1 base-1 base-1 base-1 base-1
- // base+1 A B .. Y Z
- // base+1 C D .. W X
- // base+1 E F .. U V
- // base+1 G H .. S T T T T T
-
- if (is_dr_mode) {
- if (p_angle <= 90)
- need_above = 1, need_left = 0, need_above_left = 1;
- else if (p_angle < 180)
- need_above = 1, need_left = 1, need_above_left = 1;
- else
- need_above = 0, need_left = 1, need_above_left = 1;
- }
- if (use_filter_intra) need_left = need_above = need_above_left = 1;
-
- assert(n_top_px >= 0);
- assert(n_topright_px >= -1);
- assert(n_left_px >= 0);
- assert(n_bottomleft_px >= -1);
-
- if ((!need_above && n_left_px == 0) || (!need_left && n_top_px == 0)) {
- int val;
- if (need_left) {
- val = (n_top_px > 0) ? above_ref[0] : base + 1;
- } else {
- val = (n_left_px > 0) ? left_ref[0] : base - 1;
- }
- for (i = 0; i < txhpx; ++i) {
- aom_memset16(dst, val, txwpx);
- dst += dst_stride;
- }
- return;
- }
-
- // NEED_LEFT
- if (need_left) {
- const int num_left_pixels_needed =
- txhpx + (n_bottomleft_px >= 0 ? txwpx : 0);
- i = 0;
- if (n_left_px > 0) {
- for (; i < n_left_px; i++) left_col[i] = left_ref[i * ref_stride];
- if (n_bottomleft_px > 0) {
- assert(i == txhpx);
- for (; i < txhpx + n_bottomleft_px; i++)
- left_col[i] = left_ref[i * ref_stride];
- }
- if (i < num_left_pixels_needed)
- aom_memset16(&left_col[i], left_col[i - 1], num_left_pixels_needed - i);
- } else if (n_top_px > 0) {
- aom_memset16(left_col, above_ref[0], num_left_pixels_needed);
- }
- }
-
- // NEED_ABOVE
- if (need_above) {
- const int num_top_pixels_needed = txwpx + (n_topright_px >= 0 ? txhpx : 0);
- if (n_top_px > 0) {
- memcpy(above_row, above_ref, n_top_px * sizeof(above_ref[0]));
- i = n_top_px;
- if (n_topright_px > 0) {
- assert(n_top_px == txwpx);
- memcpy(above_row + txwpx, above_ref + txwpx,
- n_topright_px * sizeof(above_ref[0]));
- i += n_topright_px;
- }
- if (i < num_top_pixels_needed)
- aom_memset16(&above_row[i], above_row[i - 1],
- num_top_pixels_needed - i);
- } else if (n_left_px > 0) {
- aom_memset16(above_row, left_ref[0], num_top_pixels_needed);
- }
- }
-
- if (need_above_left) {
- if (n_top_px > 0 && n_left_px > 0) {
- above_row[-1] = above_ref[-1];
- } else if (n_top_px > 0) {
- above_row[-1] = above_ref[0];
- } else if (n_left_px > 0) {
- above_row[-1] = left_ref[0];
- } else {
- above_row[-1] = base;
- }
- left_col[-1] = above_row[-1];
- }
-
- if (use_filter_intra) {
- highbd_filter_intra_predictor(dst, dst_stride, tx_size, above_row, left_col,
- filter_intra_mode, bit_depth);
- return;
- }
-
- if (is_dr_mode) {
- int upsample_above = 0;
- int upsample_left = 0;
- if (!disable_edge_filter) {
- const int need_right = p_angle < 90;
- const int need_bottom = p_angle > 180;
- if (p_angle != 90 && p_angle != 180) {
- const int ab_le = need_above_left ? 1 : 0;
- if (need_above && need_left && (txwpx + txhpx >= 24)) {
- filter_intra_edge_corner_high(above_row, left_col);
- }
- if (need_above && n_top_px > 0) {
- const int strength = intra_edge_filter_strength(
- txwpx, txhpx, p_angle - 90, intra_edge_filter_type);
- const int n_px = n_top_px + ab_le + (need_right ? txhpx : 0);
- av1_filter_intra_edge_high(above_row - ab_le, n_px, strength);
- }
- if (need_left && n_left_px > 0) {
- const int strength = intra_edge_filter_strength(
- txhpx, txwpx, p_angle - 180, intra_edge_filter_type);
- const int n_px = n_left_px + ab_le + (need_bottom ? txwpx : 0);
- av1_filter_intra_edge_high(left_col - ab_le, n_px, strength);
- }
- }
- upsample_above = av1_use_intra_edge_upsample(txwpx, txhpx, p_angle - 90,
- intra_edge_filter_type);
- if (need_above && upsample_above) {
- const int n_px = txwpx + (need_right ? txhpx : 0);
- av1_upsample_intra_edge_high(above_row, n_px, bit_depth);
- }
- upsample_left = av1_use_intra_edge_upsample(txhpx, txwpx, p_angle - 180,
- intra_edge_filter_type);
- if (need_left && upsample_left) {
- const int n_px = txhpx + (need_bottom ? txwpx : 0);
- av1_upsample_intra_edge_high(left_col, n_px, bit_depth);
- }
- }
- highbd_dr_predictor(dst, dst_stride, tx_size, above_row, left_col,
- upsample_above, upsample_left, p_angle, bit_depth);
- return;
- }
-
- // predict
- if (mode == DC_PRED) {
- dc_pred_high[n_left_px > 0][n_top_px > 0][tx_size](
- dst, dst_stride, above_row, left_col, bit_depth);
- } else {
- pred_high[mode][tx_size](dst, dst_stride, above_row, left_col, bit_depth);
- }
-}
-#endif // CONFIG_AV1_HIGHBITDEPTH
-
static void build_intra_predictors(
const uint8_t *ref, int ref_stride, uint8_t *dst, int dst_stride,
PREDICTION_MODE mode, int p_angle, FILTER_INTRA_MODE filter_intra_mode,
@@ -1476,6 +1241,241 @@
}
}
+#if CONFIG_AV1_HIGHBITDEPTH
+void av1_highbd_filter_intra_edge_c(uint16_t *p, int sz, int strength) {
+ if (!strength) return;
+
+ const int kernel[INTRA_EDGE_FILT][INTRA_EDGE_TAPS] = { { 0, 4, 8, 4, 0 },
+ { 0, 5, 6, 5, 0 },
+ { 2, 4, 4, 4, 2 } };
+ const int filt = strength - 1;
+ uint16_t edge[129];
+
+ memcpy(edge, p, sz * sizeof(*p));
+ for (int i = 1; i < sz; i++) {
+ int s = 0;
+ for (int j = 0; j < INTRA_EDGE_TAPS; j++) {
+ int k = i - 2 + j;
+ k = (k < 0) ? 0 : k;
+ k = (k > sz - 1) ? sz - 1 : k;
+ s += edge[k] * kernel[filt][j];
+ }
+ s = (s + 8) >> 4;
+ p[i] = s;
+ }
+}
+
+static void highbd_filter_intra_edge_corner(uint16_t *p_above,
+ uint16_t *p_left) {
+ const int kernel[3] = { 5, 6, 5 };
+
+ int s = (p_left[0] * kernel[0]) + (p_above[-1] * kernel[1]) +
+ (p_above[0] * kernel[2]);
+ s = (s + 8) >> 4;
+ p_above[-1] = s;
+ p_left[-1] = s;
+}
+
+void av1_highbd_upsample_intra_edge_c(uint16_t *p, int sz, int bd) {
+ // interpolate half-sample positions
+ assert(sz <= MAX_UPSAMPLE_SZ);
+
+ uint16_t in[MAX_UPSAMPLE_SZ + 3];
+ // copy p[-1..(sz-1)] and extend first and last samples
+ in[0] = p[-1];
+ in[1] = p[-1];
+ for (int i = 0; i < sz; i++) {
+ in[i + 2] = p[i];
+ }
+ in[sz + 2] = p[sz - 1];
+
+ // interpolate half-sample edge positions
+ p[-2] = in[0];
+ for (int i = 0; i < sz; i++) {
+ int s = -in[i] + (9 * in[i + 1]) + (9 * in[i + 2]) - in[i + 3];
+ s = (s + 8) >> 4;
+ s = clip_pixel_highbd(s, bd);
+ p[2 * i - 1] = s;
+ p[2 * i] = in[i + 2];
+ }
+}
+
+static void highbd_build_intra_predictors(
+ const uint8_t *ref8, int ref_stride, uint8_t *dst8, int dst_stride,
+ PREDICTION_MODE mode, int p_angle, FILTER_INTRA_MODE filter_intra_mode,
+ TX_SIZE tx_size, int disable_edge_filter, int n_top_px, int n_topright_px,
+ int n_left_px, int n_bottomleft_px, int intra_edge_filter_type,
+ int bit_depth) {
+ int i;
+ uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
+ uint16_t *ref = CONVERT_TO_SHORTPTR(ref8);
+ DECLARE_ALIGNED(16, uint16_t, left_data[NUM_INTRA_NEIGHBOUR_PIXELS]);
+ DECLARE_ALIGNED(16, uint16_t, above_data[NUM_INTRA_NEIGHBOUR_PIXELS]);
+ uint16_t *const above_row = above_data + 16;
+ uint16_t *const left_col = left_data + 16;
+ const int txwpx = tx_size_wide[tx_size];
+ const int txhpx = tx_size_high[tx_size];
+ int need_left = extend_modes[mode] & NEED_LEFT;
+ int need_above = extend_modes[mode] & NEED_ABOVE;
+ int need_above_left = extend_modes[mode] & NEED_ABOVELEFT;
+ const uint16_t *above_ref = ref - ref_stride;
+ const uint16_t *left_ref = ref - 1;
+ const int is_dr_mode = av1_is_directional_mode(mode);
+ const int use_filter_intra = filter_intra_mode != FILTER_INTRA_MODES;
+ int base = 128 << (bit_depth - 8);
+ // The left_data, above_data buffers must be zeroed to fix some intermittent
+ // valgrind errors. Uninitialized reads in intra pred modules (e.g. width = 4
+ // path in av1_highbd_dr_prediction_z2_avx2()) from left_data, above_data are
+ // seen to be the potential reason for this issue.
+ aom_memset16(left_data, base + 1, NUM_INTRA_NEIGHBOUR_PIXELS);
+ aom_memset16(above_data, base - 1, NUM_INTRA_NEIGHBOUR_PIXELS);
+
+ // The default values if ref pixels are not available:
+ // base base-1 base-1 .. base-1 base-1 base-1 base-1 base-1 base-1
+ // base+1 A B .. Y Z
+ // base+1 C D .. W X
+ // base+1 E F .. U V
+ // base+1 G H .. S T T T T T
+
+ if (is_dr_mode) {
+ if (p_angle <= 90)
+ need_above = 1, need_left = 0, need_above_left = 1;
+ else if (p_angle < 180)
+ need_above = 1, need_left = 1, need_above_left = 1;
+ else
+ need_above = 0, need_left = 1, need_above_left = 1;
+ }
+ if (use_filter_intra) need_left = need_above = need_above_left = 1;
+
+ assert(n_top_px >= 0);
+ assert(n_topright_px >= -1);
+ assert(n_left_px >= 0);
+ assert(n_bottomleft_px >= -1);
+
+ if ((!need_above && n_left_px == 0) || (!need_left && n_top_px == 0)) {
+ int val;
+ if (need_left) {
+ val = (n_top_px > 0) ? above_ref[0] : base + 1;
+ } else {
+ val = (n_left_px > 0) ? left_ref[0] : base - 1;
+ }
+ for (i = 0; i < txhpx; ++i) {
+ aom_memset16(dst, val, txwpx);
+ dst += dst_stride;
+ }
+ return;
+ }
+
+ // NEED_LEFT
+ if (need_left) {
+ const int num_left_pixels_needed =
+ txhpx + (n_bottomleft_px >= 0 ? txwpx : 0);
+ i = 0;
+ if (n_left_px > 0) {
+ for (; i < n_left_px; i++) left_col[i] = left_ref[i * ref_stride];
+ if (n_bottomleft_px > 0) {
+ assert(i == txhpx);
+ for (; i < txhpx + n_bottomleft_px; i++)
+ left_col[i] = left_ref[i * ref_stride];
+ }
+ if (i < num_left_pixels_needed)
+ aom_memset16(&left_col[i], left_col[i - 1], num_left_pixels_needed - i);
+ } else if (n_top_px > 0) {
+ aom_memset16(left_col, above_ref[0], num_left_pixels_needed);
+ }
+ }
+
+ // NEED_ABOVE
+ if (need_above) {
+ const int num_top_pixels_needed = txwpx + (n_topright_px >= 0 ? txhpx : 0);
+ if (n_top_px > 0) {
+ memcpy(above_row, above_ref, n_top_px * sizeof(above_ref[0]));
+ i = n_top_px;
+ if (n_topright_px > 0) {
+ assert(n_top_px == txwpx);
+ memcpy(above_row + txwpx, above_ref + txwpx,
+ n_topright_px * sizeof(above_ref[0]));
+ i += n_topright_px;
+ }
+ if (i < num_top_pixels_needed)
+ aom_memset16(&above_row[i], above_row[i - 1],
+ num_top_pixels_needed - i);
+ } else if (n_left_px > 0) {
+ aom_memset16(above_row, left_ref[0], num_top_pixels_needed);
+ }
+ }
+
+ if (need_above_left) {
+ if (n_top_px > 0 && n_left_px > 0) {
+ above_row[-1] = above_ref[-1];
+ } else if (n_top_px > 0) {
+ above_row[-1] = above_ref[0];
+ } else if (n_left_px > 0) {
+ above_row[-1] = left_ref[0];
+ } else {
+ above_row[-1] = base;
+ }
+ left_col[-1] = above_row[-1];
+ }
+
+ if (use_filter_intra) {
+ highbd_filter_intra_predictor(dst, dst_stride, tx_size, above_row, left_col,
+ filter_intra_mode, bit_depth);
+ return;
+ }
+
+ if (is_dr_mode) {
+ int upsample_above = 0;
+ int upsample_left = 0;
+ if (!disable_edge_filter) {
+ const int need_right = p_angle < 90;
+ const int need_bottom = p_angle > 180;
+ if (p_angle != 90 && p_angle != 180) {
+ const int ab_le = need_above_left ? 1 : 0;
+ if (need_above && need_left && (txwpx + txhpx >= 24)) {
+ highbd_filter_intra_edge_corner(above_row, left_col);
+ }
+ if (need_above && n_top_px > 0) {
+ const int strength = intra_edge_filter_strength(
+ txwpx, txhpx, p_angle - 90, intra_edge_filter_type);
+ const int n_px = n_top_px + ab_le + (need_right ? txhpx : 0);
+ av1_highbd_filter_intra_edge(above_row - ab_le, n_px, strength);
+ }
+ if (need_left && n_left_px > 0) {
+ const int strength = intra_edge_filter_strength(
+ txhpx, txwpx, p_angle - 180, intra_edge_filter_type);
+ const int n_px = n_left_px + ab_le + (need_bottom ? txwpx : 0);
+ av1_highbd_filter_intra_edge(left_col - ab_le, n_px, strength);
+ }
+ }
+ upsample_above = av1_use_intra_edge_upsample(txwpx, txhpx, p_angle - 90,
+ intra_edge_filter_type);
+ if (need_above && upsample_above) {
+ const int n_px = txwpx + (need_right ? txhpx : 0);
+ av1_highbd_upsample_intra_edge(above_row, n_px, bit_depth);
+ }
+ upsample_left = av1_use_intra_edge_upsample(txhpx, txwpx, p_angle - 180,
+ intra_edge_filter_type);
+ if (need_left && upsample_left) {
+ const int n_px = txhpx + (need_bottom ? txwpx : 0);
+ av1_highbd_upsample_intra_edge(left_col, n_px, bit_depth);
+ }
+ }
+ highbd_dr_predictor(dst, dst_stride, tx_size, above_row, left_col,
+ upsample_above, upsample_left, p_angle, bit_depth);
+ return;
+ }
+
+ // predict
+ if (mode == DC_PRED) {
+ dc_pred_high[n_left_px > 0][n_top_px > 0][tx_size](
+ dst, dst_stride, above_row, left_col, bit_depth);
+ } else {
+ pred_high[mode][tx_size](dst, dst_stride, above_row, left_col, bit_depth);
+ }
+}
+#endif // CONFIG_AV1_HIGHBITDEPTH
+
static INLINE BLOCK_SIZE scale_chroma_bsize(BLOCK_SIZE bsize, int subsampling_x,
int subsampling_y) {
assert(subsampling_x >= 0 && subsampling_x < 2);
@@ -1631,7 +1631,7 @@
const int intra_edge_filter_type = get_intra_edge_filter_type(xd, plane);
#if CONFIG_AV1_HIGHBITDEPTH
if (is_cur_buf_hbd(xd)) {
- build_intra_predictors_high(
+ highbd_build_intra_predictors(
ref, ref_stride, dst, dst_stride, mode, p_angle, filter_intra_mode,
tx_size, disable_edge_filter, have_top ? AOMMIN(txwpx, xr + txwpx) : 0,
have_top_right > 0 ? AOMMIN(txwpx, xr) : have_top_right,
@@ -1696,7 +1696,7 @@
} else {
cfl_load_dc_pred(xd, dst, dst_stride, tx_size, pred_plane);
}
- cfl_predict_block(xd, dst, dst_stride, tx_size, plane);
+ av1_cfl_predict_block(xd, dst, dst_stride, tx_size, plane);
return;
}
av1_predict_intra_block(
diff --git a/av1/common/resize.c b/av1/common/resize.c
index f4bfcd0..f89f7ca 100644
--- a/av1/common/resize.c
+++ b/av1/common/resize.c
@@ -644,17 +644,20 @@
}
}
-void av1_resize_plane(const uint8_t *const input, int height, int width,
+bool av1_resize_plane(const uint8_t *const input, int height, int width,
int in_stride, uint8_t *output, int height2, int width2,
int out_stride) {
int i;
+ bool mem_status = true;
uint8_t *intbuf = (uint8_t *)aom_malloc(sizeof(uint8_t) * width2 * height);
uint8_t *tmpbuf =
(uint8_t *)aom_malloc(sizeof(uint8_t) * AOMMAX(width, height));
uint8_t *arrbuf = (uint8_t *)aom_malloc(sizeof(uint8_t) * height);
uint8_t *arrbuf2 = (uint8_t *)aom_malloc(sizeof(uint8_t) * height2);
- if (intbuf == NULL || tmpbuf == NULL || arrbuf == NULL || arrbuf2 == NULL)
+ if (intbuf == NULL || tmpbuf == NULL || arrbuf == NULL || arrbuf2 == NULL) {
+ mem_status = false;
goto Error;
+ }
assert(width > 0);
assert(height > 0);
assert(width2 > 0);
@@ -673,16 +676,21 @@
aom_free(tmpbuf);
aom_free(arrbuf);
aom_free(arrbuf2);
+ return mem_status;
}
-void av1_upscale_plane_double_prec(const double *const input, int height,
+bool av1_upscale_plane_double_prec(const double *const input, int height,
int width, int in_stride, double *output,
int height2, int width2, int out_stride) {
int i;
+ bool mem_status = true;
double *intbuf = (double *)aom_malloc(sizeof(double) * width2 * height);
double *arrbuf = (double *)aom_malloc(sizeof(double) * height);
double *arrbuf2 = (double *)aom_malloc(sizeof(double) * height2);
- if (intbuf == NULL || arrbuf == NULL || arrbuf2 == NULL) goto Error;
+ if (intbuf == NULL || arrbuf == NULL || arrbuf2 == NULL) {
+ mem_status = false;
+ goto Error;
+ }
assert(width > 0);
assert(height > 0);
assert(width2 > 0);
@@ -700,6 +708,7 @@
aom_free(intbuf);
aom_free(arrbuf);
aom_free(arrbuf2);
+ return mem_status;
}
static bool upscale_normative_rect(const uint8_t *const input, int height,
@@ -1128,35 +1137,49 @@
int uv_stride, int height, int width, uint8_t *oy,
int oy_stride, uint8_t *ou, uint8_t *ov,
int ouv_stride, int oheight, int owidth) {
- av1_resize_plane(y, height, width, y_stride, oy, oheight, owidth, oy_stride);
- av1_resize_plane(u, height / 2, width / 2, uv_stride, ou, oheight / 2,
- owidth / 2, ouv_stride);
- av1_resize_plane(v, height / 2, width / 2, uv_stride, ov, oheight / 2,
- owidth / 2, ouv_stride);
+ if (!av1_resize_plane(y, height, width, y_stride, oy, oheight, owidth,
+ oy_stride))
+ abort();
+ if (!av1_resize_plane(u, height / 2, width / 2, uv_stride, ou, oheight / 2,
+ owidth / 2, ouv_stride))
+ abort();
+ if (!av1_resize_plane(v, height / 2, width / 2, uv_stride, ov, oheight / 2,
+ owidth / 2, ouv_stride))
+ abort();
}
-void av1_resize_frame422(const uint8_t *const y, int y_stride,
+bool av1_resize_frame422(const uint8_t *const y, int y_stride,
const uint8_t *const u, const uint8_t *const v,
int uv_stride, int height, int width, uint8_t *oy,
int oy_stride, uint8_t *ou, uint8_t *ov,
int ouv_stride, int oheight, int owidth) {
- av1_resize_plane(y, height, width, y_stride, oy, oheight, owidth, oy_stride);
- av1_resize_plane(u, height, width / 2, uv_stride, ou, oheight, owidth / 2,
- ouv_stride);
- av1_resize_plane(v, height, width / 2, uv_stride, ov, oheight, owidth / 2,
- ouv_stride);
+ if (!av1_resize_plane(y, height, width, y_stride, oy, oheight, owidth,
+ oy_stride))
+ return false;
+ if (!av1_resize_plane(u, height, width / 2, uv_stride, ou, oheight,
+ owidth / 2, ouv_stride))
+ return false;
+ if (!av1_resize_plane(v, height, width / 2, uv_stride, ov, oheight,
+ owidth / 2, ouv_stride))
+ return false;
+ return true;
}
-void av1_resize_frame444(const uint8_t *const y, int y_stride,
+bool av1_resize_frame444(const uint8_t *const y, int y_stride,
const uint8_t *const u, const uint8_t *const v,
int uv_stride, int height, int width, uint8_t *oy,
int oy_stride, uint8_t *ou, uint8_t *ov,
int ouv_stride, int oheight, int owidth) {
- av1_resize_plane(y, height, width, y_stride, oy, oheight, owidth, oy_stride);
- av1_resize_plane(u, height, width, uv_stride, ou, oheight, owidth,
- ouv_stride);
- av1_resize_plane(v, height, width, uv_stride, ov, oheight, owidth,
- ouv_stride);
+ if (!av1_resize_plane(y, height, width, y_stride, oy, oheight, owidth,
+ oy_stride))
+ return false;
+ if (!av1_resize_plane(u, height, width, uv_stride, ou, oheight, owidth,
+ ouv_stride))
+ return false;
+ if (!av1_resize_plane(v, height, width, uv_stride, ov, oheight, owidth,
+ ouv_stride))
+ return false;
+ return true;
}
#if CONFIG_AV1_HIGHBITDEPTH
@@ -1251,7 +1274,7 @@
aom_extend_frame_borders(dst, num_planes);
}
-void av1_resize_and_extend_frame_nonnormative(const YV12_BUFFER_CONFIG *src,
+bool av1_resize_and_extend_frame_nonnormative(const YV12_BUFFER_CONFIG *src,
YV12_BUFFER_CONFIG *dst, int bd,
const int num_planes) {
// TODO(dkovalev): replace YV12_BUFFER_CONFIG with aom_image_t
@@ -1261,25 +1284,29 @@
for (int i = 0; i < AOMMIN(num_planes, MAX_MB_PLANE); ++i) {
const int is_uv = i > 0;
#if CONFIG_AV1_HIGHBITDEPTH
- if (src->flags & YV12_FLAG_HIGHBITDEPTH)
+ if (src->flags & YV12_FLAG_HIGHBITDEPTH) {
av1_highbd_resize_plane(src->buffers[i], src->crop_heights[is_uv],
src->crop_widths[is_uv], src->strides[is_uv],
dst->buffers[i], dst->crop_heights[is_uv],
dst->crop_widths[is_uv], dst->strides[is_uv], bd);
- else
- av1_resize_plane(src->buffers[i], src->crop_heights[is_uv],
- src->crop_widths[is_uv], src->strides[is_uv],
- dst->buffers[i], dst->crop_heights[is_uv],
- dst->crop_widths[is_uv], dst->strides[is_uv]);
+ } else if (!av1_resize_plane(src->buffers[i], src->crop_heights[is_uv],
+ src->crop_widths[is_uv], src->strides[is_uv],
+ dst->buffers[i], dst->crop_heights[is_uv],
+ dst->crop_widths[is_uv],
+ dst->strides[is_uv])) {
+ return false;
+ }
#else
(void)bd;
- av1_resize_plane(src->buffers[i], src->crop_heights[is_uv],
- src->crop_widths[is_uv], src->strides[is_uv],
- dst->buffers[i], dst->crop_heights[is_uv],
- dst->crop_widths[is_uv], dst->strides[is_uv]);
+ if (!av1_resize_plane(src->buffers[i], src->crop_heights[is_uv],
+ src->crop_widths[is_uv], src->strides[is_uv],
+ dst->buffers[i], dst->crop_heights[is_uv],
+ dst->crop_widths[is_uv], dst->strides[is_uv]))
+ return false;
#endif
}
aom_extend_frame_borders(dst, num_planes);
+ return true;
}
void av1_upscale_normative_rows(const AV1_COMMON *cm, const uint8_t *src,
@@ -1410,15 +1437,19 @@
cm->seq_params->bit_depth == AOM_BITS_8) {
av1_resize_and_extend_frame(unscaled, scaled, filter, phase, num_planes);
} else {
- av1_resize_and_extend_frame_nonnormative(
- unscaled, scaled, (int)cm->seq_params->bit_depth, num_planes);
+ if (!av1_resize_and_extend_frame_nonnormative(
+ unscaled, scaled, (int)cm->seq_params->bit_depth, num_planes))
+ aom_internal_error(cm->error, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate buffers during resize");
}
#else
if (use_optimized_scaler && has_optimized_scaler) {
av1_resize_and_extend_frame(unscaled, scaled, filter, phase, num_planes);
} else {
- av1_resize_and_extend_frame_nonnormative(
- unscaled, scaled, (int)cm->seq_params->bit_depth, num_planes);
+ if (!av1_resize_and_extend_frame_nonnormative(
+ unscaled, scaled, (int)cm->seq_params->bit_depth, num_planes))
+ aom_internal_error(cm->error, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate buffers during resize");
}
#endif
return scaled;
diff --git a/av1/common/resize.h b/av1/common/resize.h
index 5927d8e..d1fab82 100644
--- a/av1/common/resize.h
+++ b/av1/common/resize.h
@@ -20,23 +20,25 @@
extern "C" {
#endif
-void av1_resize_plane(const uint8_t *const input, int height, int width,
+bool av1_resize_plane(const uint8_t *const input, int height, int width,
int in_stride, uint8_t *output, int height2, int width2,
int out_stride);
-void av1_upscale_plane_double_prec(const double *const input, int height,
+bool av1_upscale_plane_double_prec(const double *const input, int height,
int width, int in_stride, double *output,
int height2, int width2, int out_stride);
+// TODO(aomedia:3228): In libaom 4.0.0, remove av1_resize_frame420 from
+// av1/exports_com and delete this function.
void av1_resize_frame420(const uint8_t *const y, int y_stride,
const uint8_t *const u, const uint8_t *const v,
int uv_stride, int height, int width, uint8_t *oy,
int oy_stride, uint8_t *ou, uint8_t *ov,
int ouv_stride, int oheight, int owidth);
-void av1_resize_frame422(const uint8_t *const y, int y_stride,
+bool av1_resize_frame422(const uint8_t *const y, int y_stride,
const uint8_t *const u, const uint8_t *const v,
int uv_stride, int height, int width, uint8_t *oy,
int oy_stride, uint8_t *ou, uint8_t *ov,
int ouv_stride, int oheight, int owidth);
-void av1_resize_frame444(const uint8_t *const y, int y_stride,
+bool av1_resize_frame444(const uint8_t *const y, int y_stride,
const uint8_t *const u, const uint8_t *const v,
int uv_stride, int height, int width, uint8_t *oy,
int oy_stride, uint8_t *ou, uint8_t *ov,
@@ -77,7 +79,7 @@
const bool for_psnr, const int border_in_pixels,
const int num_pyramid_levels);
-void av1_resize_and_extend_frame_nonnormative(const YV12_BUFFER_CONFIG *src,
+bool av1_resize_and_extend_frame_nonnormative(const YV12_BUFFER_CONFIG *src,
YV12_BUFFER_CONFIG *dst, int bd,
const int num_planes);
diff --git a/av1/common/restoration.c b/av1/common/restoration.c
index 822f240..a26f329 100644
--- a/av1/common/restoration.c
+++ b/av1/common/restoration.c
@@ -20,6 +20,7 @@
#include "av1/common/av1_common_int.h"
#include "av1/common/resize.h"
#include "av1/common/restoration.h"
+#include "av1/common/thread_common.h"
#include "aom_dsp/aom_dsp_common.h"
#include "aom_mem/aom_mem.h"
@@ -39,63 +40,43 @@
{ { 2, 0 }, { 56, -1 } }, { { 2, 0 }, { 22, -1 } },
};
-PixelRect av1_whole_frame_rect(const AV1_COMMON *cm, int is_uv) {
- PixelRect rect;
-
+void av1_get_upsampled_plane_size(const AV1_COMMON *cm, int is_uv, int *plane_w,
+ int *plane_h) {
int ss_x = is_uv && cm->seq_params->subsampling_x;
int ss_y = is_uv && cm->seq_params->subsampling_y;
-
- rect.top = 0;
- rect.bottom = ROUND_POWER_OF_TWO(cm->height, ss_y);
- rect.left = 0;
- rect.right = ROUND_POWER_OF_TWO(cm->superres_upscaled_width, ss_x);
- return rect;
+ *plane_w = ROUND_POWER_OF_TWO(cm->superres_upscaled_width, ss_x);
+ *plane_h = ROUND_POWER_OF_TWO(cm->height, ss_y);
}
-// Count horizontal or vertical units per tile (use a width or height for
-// tile_size, respectively). We basically want to divide the tile size by the
+// Count horizontal or vertical units in a plane (use a width or height for
+// plane_size, respectively). We basically want to divide the plane size by the
// size of a restoration unit. Rather than rounding up unconditionally as you
// might expect, we round to nearest, which models the way a right or bottom
-// restoration unit can extend to up to 150% its normal width or height. The
-// max with 1 is to deal with tiles that are smaller than half of a restoration
-// unit.
-int av1_lr_count_units_in_tile(int unit_size, int tile_size) {
- return AOMMAX((tile_size + (unit_size >> 1)) / unit_size, 1);
+// restoration unit can extend to up to 150% its normal width or height.
+//
+// The max with 1 is to deal with small frames, which may be smaller than
+// half of an LR unit in size.
+int av1_lr_count_units(int unit_size, int plane_size) {
+ return AOMMAX((plane_size + (unit_size >> 1)) / unit_size, 1);
}
void av1_alloc_restoration_struct(AV1_COMMON *cm, RestorationInfo *rsi,
int is_uv) {
- // We need to allocate enough space for restoration units to cover the
- // largest tile. Without CONFIG_MAX_TILE, this is always the tile at the
- // top-left and we can use av1_get_tile_rect(). With CONFIG_MAX_TILE, we have
- // to do the computation ourselves, iterating over the tiles and keeping
- // track of the largest width and height, then upscaling.
- const PixelRect tile_rect = av1_whole_frame_rect(cm, is_uv);
- const int max_tile_w = tile_rect.right - tile_rect.left;
- const int max_tile_h = tile_rect.bottom - tile_rect.top;
+ int plane_w, plane_h;
+ av1_get_upsampled_plane_size(cm, is_uv, &plane_w, &plane_h);
- // To calculate hpertile and vpertile (horizontal and vertical units per
- // tile), we basically want to divide the largest tile width or height by the
- // size of a restoration unit. Rather than rounding up unconditionally as you
- // might expect, we round to nearest, which models the way a right or bottom
- // restoration unit can extend to up to 150% its normal width or height. The
- // max with 1 is to deal with tiles that are smaller than half of a
- // restoration unit.
const int unit_size = rsi->restoration_unit_size;
- const int hpertile = av1_lr_count_units_in_tile(unit_size, max_tile_w);
- const int vpertile = av1_lr_count_units_in_tile(unit_size, max_tile_h);
+ const int horz_units = av1_lr_count_units(unit_size, plane_w);
+ const int vert_units = av1_lr_count_units(unit_size, plane_h);
- rsi->units_per_tile = hpertile * vpertile;
- rsi->horz_units_per_tile = hpertile;
- rsi->vert_units_per_tile = vpertile;
-
- const int ntiles = 1;
- const int nunits = ntiles * rsi->units_per_tile;
+ rsi->num_rest_units = horz_units * vert_units;
+ rsi->horz_units = horz_units;
+ rsi->vert_units = vert_units;
aom_free(rsi->unit_info);
CHECK_MEM_ERROR(cm, rsi->unit_info,
(RestorationUnitInfo *)aom_memalign(
- 16, sizeof(*rsi->unit_info) * nunits));
+ 16, sizeof(*rsi->unit_info) * rsi->num_rest_units));
}
void av1_free_restoration_struct(RestorationInfo *rst_info) {
@@ -174,8 +155,9 @@
}
}
-static void copy_tile_highbd(int width, int height, const uint16_t *src,
- int src_stride, uint16_t *dst, int dst_stride) {
+static void copy_rest_unit_highbd(int width, int height, const uint16_t *src,
+ int src_stride, uint16_t *dst,
+ int dst_stride) {
for (int i = 0; i < height; ++i)
memcpy(dst + i * dst_stride, src + i * src_stride, width * sizeof(*dst));
}
@@ -194,23 +176,24 @@
extend_frame_lowbd(data, width, height, stride, border_horz, border_vert);
}
-static void copy_tile_lowbd(int width, int height, const uint8_t *src,
- int src_stride, uint8_t *dst, int dst_stride) {
+static void copy_rest_unit_lowbd(int width, int height, const uint8_t *src,
+ int src_stride, uint8_t *dst, int dst_stride) {
for (int i = 0; i < height; ++i)
memcpy(dst + i * dst_stride, src + i * src_stride, width);
}
-static void copy_tile(int width, int height, const uint8_t *src, int src_stride,
- uint8_t *dst, int dst_stride, int highbd) {
+static void copy_rest_unit(int width, int height, const uint8_t *src,
+ int src_stride, uint8_t *dst, int dst_stride,
+ int highbd) {
#if CONFIG_AV1_HIGHBITDEPTH
if (highbd) {
- copy_tile_highbd(width, height, CONVERT_TO_SHORTPTR(src), src_stride,
- CONVERT_TO_SHORTPTR(dst), dst_stride);
+ copy_rest_unit_highbd(width, height, CONVERT_TO_SHORTPTR(src), src_stride,
+ CONVERT_TO_SHORTPTR(dst), dst_stride);
return;
}
#endif
(void)highbd;
- copy_tile_lowbd(width, height, src, src_stride, dst, dst_stride);
+ copy_rest_unit_lowbd(width, height, src, src_stride, dst, dst_stride);
}
#define REAL_PTR(hbd, d) ((hbd) ? (uint8_t *)CONVERT_TO_SHORTPTR(d) : (d))
@@ -218,53 +201,34 @@
// With striped loop restoration, the filtering for each 64-pixel stripe gets
// most of its input from the output of CDEF (stored in data8), but we need to
// fill out a border of 3 pixels above/below the stripe according to the
-// following
-// rules:
+// following rules:
//
-// * At a frame boundary, we copy the outermost row of CDEF pixels three times.
-// This extension is done by a call to av1_extend_frame() at the start of the
-// loop restoration process, so the value of copy_above/copy_below doesn't
-// strictly matter. However, by setting *copy_above = *copy_below = 1 whenever
-// loop filtering across tiles is disabled, we can allow
-// {setup,restore}_processing_stripe_boundary to assume that the top/bottom
-// data has always been copied, simplifying the behaviour at the left and
-// right edges of tiles.
+// * At the top and bottom of the frame, we copy the outermost row of CDEF
+// pixels three times. This extension is done by a call to av1_extend_frame()
+// at the start of the loop restoration process, so the value of
+// copy_above/copy_below doesn't strictly matter.
//
-// * If we're at a tile boundary and loop filtering across tiles is enabled,
-// then there is a logical stripe which is 64 pixels high, but which is split
-// into an 8px high and a 56px high stripe so that the processing (and
-// coefficient set usage) can be aligned to tiles.
-// In this case, we use the 3 rows of CDEF output across the boundary for
-// context; this corresponds to leaving the frame buffer as-is.
-//
-// * If we're at a tile boundary and loop filtering across tiles is disabled,
-// then we take the outermost row of CDEF pixels *within the current tile*
-// and copy it three times. Thus we behave exactly as if the tile were a full
-// frame.
-//
-// * Otherwise, we're at a stripe boundary within a tile. In that case, we
-// take 2 rows of deblocked pixels and extend them to 3 rows of context.
-//
-// The distinction between the latter two cases is handled by the
-// av1_loop_restoration_save_boundary_lines() function, so here we just need
-// to decide if we're overwriting the above/below boundary pixels or not.
+// * All other boundaries are stripe boundaries within the frame. In that case,
+// we take 2 rows of deblocked pixels and extend them to 3 rows of context.
static void get_stripe_boundary_info(const RestorationTileLimits *limits,
- const PixelRect *tile_rect, int ss_y,
+ int plane_w, int plane_h, int ss_y,
int *copy_above, int *copy_below) {
+ (void)plane_w;
+
*copy_above = 1;
*copy_below = 1;
const int full_stripe_height = RESTORATION_PROC_UNIT_SIZE >> ss_y;
const int runit_offset = RESTORATION_UNIT_OFFSET >> ss_y;
- const int first_stripe_in_tile = (limits->v_start == tile_rect->top);
+ const int first_stripe_in_plane = (limits->v_start == 0);
const int this_stripe_height =
- full_stripe_height - (first_stripe_in_tile ? runit_offset : 0);
- const int last_stripe_in_tile =
- (limits->v_start + this_stripe_height >= tile_rect->bottom);
+ full_stripe_height - (first_stripe_in_plane ? runit_offset : 0);
+ const int last_stripe_in_plane =
+ (limits->v_start + this_stripe_height >= plane_h);
- if (first_stripe_in_tile) *copy_above = 0;
- if (last_stripe_in_tile) *copy_below = 0;
+ if (first_stripe_in_plane) *copy_above = 0;
+ if (last_stripe_in_plane) *copy_below = 0;
}
// Overwrite the border pixels around a processing stripe so that the conditions
@@ -276,10 +240,6 @@
// limits gives the rectangular limits of the remaining stripes for the current
// restoration unit. rsb is the stored stripe boundaries (taken from either
// deblock or CDEF output as necessary).
-//
-// tile_rect is the limits of the current tile and tile_stripe0 is the index of
-// the first stripe in this tile (needed to convert the tile-relative stripe
-// index we get from limits into something we can look up in rsb).
static void setup_processing_stripe_boundary(
const RestorationTileLimits *limits, const RestorationStripeBoundaries *rsb,
int rsb_row, int use_highbd, int h, uint8_t *data8, int data_stride,
@@ -300,12 +260,6 @@
// to fill RESTORATION_BORDER=3 lines of above pixels. This is done by
// duplicating the topmost of the 2 lines (see the AOMMAX call when
// calculating src_row, which gets the values 0, 0, 1 for i = -3, -2, -1).
- //
- // Special case: If we're at the top of a tile, which isn't on the topmost
- // tile row, and we're allowed to loop filter across tiles, then we have a
- // logical 64-pixel-high stripe which has been split into an 8-pixel high
- // stripe and a 56-pixel high stripe (the current one). So, in this case,
- // we want to leave the boundary alone!
if (!opt) {
if (copy_above) {
uint8_t *data8_tl = data8 + data_x0 + limits->v_start * data_stride;
@@ -370,19 +324,9 @@
}
}
-// This function restores the boundary lines modified by
-// setup_processing_stripe_boundary.
-//
-// Note: We need to be careful when handling the corners of the processing
-// unit, because (eg.) the top-left corner is considered to be part of
-// both the left and top borders. This means that, depending on the
-// loop_filter_across_tiles_enabled flag, the corner pixels might get
-// overwritten twice, once as part of the "top" border and once as part
-// of the "left" border (or similar for other corners).
-//
-// Everything works out fine as long as we make sure to reverse the order
-// when restoring, ie. we need to restore the left/right borders followed
-// by the top/bottom borders.
+// Once a processing stripe is finished, this function sets the boundary
+// pixels which were overwritten by setup_processing_stripe_boundary()
+// back to their original values
static void restore_processing_stripe_boundary(
const RestorationTileLimits *limits, const RestorationLineBuffers *rlbs,
int use_highbd, int h, uint8_t *data8, int data_stride, int copy_above,
@@ -440,11 +384,13 @@
int stripe_width, int stripe_height,
int procunit_width, const uint8_t *src,
int src_stride, uint8_t *dst, int dst_stride,
- int32_t *tmpbuf, int bit_depth) {
+ int32_t *tmpbuf, int bit_depth,
+ struct aom_internal_error_info *error_info) {
(void)tmpbuf;
(void)bit_depth;
+ (void)error_info;
assert(bit_depth == 8);
- const ConvolveParams conv_params = get_conv_params_wiener(8);
+ const WienerConvolveParams conv_params = get_conv_params_wiener(8);
for (int j = 0; j < stripe_width; j += procunit_width) {
int w = AOMMIN(procunit_width, (stripe_width - j + 15) & ~15);
@@ -908,19 +854,18 @@
return 0;
}
-void av1_apply_selfguided_restoration_c(const uint8_t *dat8, int width,
- int height, int stride, int eps,
- const int *xqd, uint8_t *dst8,
- int dst_stride, int32_t *tmpbuf,
- int bit_depth, int highbd) {
+int av1_apply_selfguided_restoration_c(const uint8_t *dat8, int width,
+ int height, int stride, int eps,
+ const int *xqd, uint8_t *dst8,
+ int dst_stride, int32_t *tmpbuf,
+ int bit_depth, int highbd) {
int32_t *flt0 = tmpbuf;
int32_t *flt1 = flt0 + RESTORATION_UNITPELS_MAX;
assert(width * height <= RESTORATION_UNITPELS_MAX);
const int ret = av1_selfguided_restoration_c(
dat8, width, height, stride, flt0, flt1, width, eps, bit_depth, highbd);
- (void)ret;
- assert(!ret);
+ if (ret != 0) return ret;
const sgr_params_type *const params = &av1_sgr_params[eps];
int xq[2];
av1_decode_xq(xqd, xq, params);
@@ -947,33 +892,40 @@
*dst8ij = (uint8_t)out;
}
}
+ return 0;
}
static void sgrproj_filter_stripe(const RestorationUnitInfo *rui,
int stripe_width, int stripe_height,
int procunit_width, const uint8_t *src,
int src_stride, uint8_t *dst, int dst_stride,
- int32_t *tmpbuf, int bit_depth) {
+ int32_t *tmpbuf, int bit_depth,
+ struct aom_internal_error_info *error_info) {
(void)bit_depth;
assert(bit_depth == 8);
for (int j = 0; j < stripe_width; j += procunit_width) {
int w = AOMMIN(procunit_width, stripe_width - j);
- av1_apply_selfguided_restoration(
- src + j, w, stripe_height, src_stride, rui->sgrproj_info.ep,
- rui->sgrproj_info.xqd, dst + j, dst_stride, tmpbuf, bit_depth, 0);
+ if (av1_apply_selfguided_restoration(
+ src + j, w, stripe_height, src_stride, rui->sgrproj_info.ep,
+ rui->sgrproj_info.xqd, dst + j, dst_stride, tmpbuf, bit_depth,
+ 0) != 0) {
+ aom_internal_error(
+ error_info, AOM_CODEC_MEM_ERROR,
+ "Error allocating buffer in av1_apply_selfguided_restoration");
+ }
}
}
#if CONFIG_AV1_HIGHBITDEPTH
-static void wiener_filter_stripe_highbd(const RestorationUnitInfo *rui,
- int stripe_width, int stripe_height,
- int procunit_width, const uint8_t *src8,
- int src_stride, uint8_t *dst8,
- int dst_stride, int32_t *tmpbuf,
- int bit_depth) {
+static void wiener_filter_stripe_highbd(
+ const RestorationUnitInfo *rui, int stripe_width, int stripe_height,
+ int procunit_width, const uint8_t *src8, int src_stride, uint8_t *dst8,
+ int dst_stride, int32_t *tmpbuf, int bit_depth,
+ struct aom_internal_error_info *error_info) {
(void)tmpbuf;
- const ConvolveParams conv_params = get_conv_params_wiener(bit_depth);
+ (void)error_info;
+ const WienerConvolveParams conv_params = get_conv_params_wiener(bit_depth);
for (int j = 0; j < stripe_width; j += procunit_width) {
int w = AOMMIN(procunit_width, (stripe_width - j + 15) & ~15);
@@ -986,17 +938,21 @@
}
}
-static void sgrproj_filter_stripe_highbd(const RestorationUnitInfo *rui,
- int stripe_width, int stripe_height,
- int procunit_width,
- const uint8_t *src8, int src_stride,
- uint8_t *dst8, int dst_stride,
- int32_t *tmpbuf, int bit_depth) {
+static void sgrproj_filter_stripe_highbd(
+ const RestorationUnitInfo *rui, int stripe_width, int stripe_height,
+ int procunit_width, const uint8_t *src8, int src_stride, uint8_t *dst8,
+ int dst_stride, int32_t *tmpbuf, int bit_depth,
+ struct aom_internal_error_info *error_info) {
for (int j = 0; j < stripe_width; j += procunit_width) {
int w = AOMMIN(procunit_width, stripe_width - j);
- av1_apply_selfguided_restoration(
- src8 + j, w, stripe_height, src_stride, rui->sgrproj_info.ep,
- rui->sgrproj_info.xqd, dst8 + j, dst_stride, tmpbuf, bit_depth, 1);
+ if (av1_apply_selfguided_restoration(
+ src8 + j, w, stripe_height, src_stride, rui->sgrproj_info.ep,
+ rui->sgrproj_info.xqd, dst8 + j, dst_stride, tmpbuf, bit_depth,
+ 1) != 0) {
+ aom_internal_error(
+ error_info, AOM_CODEC_MEM_ERROR,
+ "Error allocating buffer in av1_apply_selfguided_restoration");
+ }
}
}
#endif // CONFIG_AV1_HIGHBITDEPTH
@@ -1005,7 +961,8 @@
int stripe_width, int stripe_height,
int procunit_width, const uint8_t *src,
int src_stride, uint8_t *dst, int dst_stride,
- int32_t *tmpbuf, int bit_depth);
+ int32_t *tmpbuf, int bit_depth,
+ struct aom_internal_error_info *error_info);
#if CONFIG_AV1_HIGHBITDEPTH
#define NUM_STRIPE_FILTERS 4
@@ -1024,9 +981,9 @@
void av1_loop_restoration_filter_unit(
const RestorationTileLimits *limits, const RestorationUnitInfo *rui,
const RestorationStripeBoundaries *rsb, RestorationLineBuffers *rlbs,
- const PixelRect *tile_rect, int tile_stripe0, int ss_x, int ss_y,
- int highbd, int bit_depth, uint8_t *data8, int stride, uint8_t *dst8,
- int dst_stride, int32_t *tmpbuf, int optimized_lr) {
+ int plane_w, int plane_h, int ss_x, int ss_y, int highbd, int bit_depth,
+ uint8_t *data8, int stride, uint8_t *dst8, int dst_stride, int32_t *tmpbuf,
+ int optimized_lr, struct aom_internal_error_info *error_info) {
RestorationType unit_rtype = rui->restoration_type;
int unit_h = limits->v_end - limits->v_start;
@@ -1035,7 +992,8 @@
uint8_t *dst8_tl = dst8 + limits->v_start * dst_stride + limits->h_start;
if (unit_rtype == RESTORE_NONE) {
- copy_tile(unit_w, unit_h, data8_tl, stride, dst8_tl, dst_stride, highbd);
+ copy_rest_unit(unit_w, unit_h, data8_tl, stride, dst8_tl, dst_stride,
+ highbd);
return;
}
@@ -1045,32 +1003,30 @@
const int procunit_width = RESTORATION_PROC_UNIT_SIZE >> ss_x;
- // Convolve the whole tile one stripe at a time
+ // Filter the whole image one stripe at a time
RestorationTileLimits remaining_stripes = *limits;
int i = 0;
while (i < unit_h) {
int copy_above, copy_below;
remaining_stripes.v_start = limits->v_start + i;
- get_stripe_boundary_info(&remaining_stripes, tile_rect, ss_y, ©_above,
- ©_below);
+ get_stripe_boundary_info(&remaining_stripes, plane_w, plane_h, ss_y,
+ ©_above, ©_below);
const int full_stripe_height = RESTORATION_PROC_UNIT_SIZE >> ss_y;
const int runit_offset = RESTORATION_UNIT_OFFSET >> ss_y;
// Work out where this stripe's boundaries are within
// rsb->stripe_boundary_{above,below}
- const int tile_stripe =
- (remaining_stripes.v_start - tile_rect->top + runit_offset) /
- full_stripe_height;
- const int frame_stripe = tile_stripe0 + tile_stripe;
+ const int frame_stripe =
+ (remaining_stripes.v_start + runit_offset) / full_stripe_height;
const int rsb_row = RESTORATION_CTX_VERT * frame_stripe;
// Calculate this stripe's height, based on two rules:
- // * The topmost stripe in each tile is 8 luma pixels shorter than usual.
+ // * The topmost stripe in the frame is 8 luma pixels shorter than usual.
// * We can't extend past the end of the current restoration unit
const int nominal_stripe_height =
- full_stripe_height - ((tile_stripe == 0) ? runit_offset : 0);
+ full_stripe_height - ((frame_stripe == 0) ? runit_offset : 0);
const int h = AOMMIN(nominal_stripe_height,
remaining_stripes.v_end - remaining_stripes.v_start);
@@ -1079,7 +1035,8 @@
copy_below, optimized_lr);
stripe_filter(rui, unit_w, h, procunit_width, data8_tl + i * stride, stride,
- dst8_tl + i * dst_stride, dst_stride, tmpbuf, bit_depth);
+ dst8_tl + i * dst_stride, dst_stride, tmpbuf, bit_depth,
+ error_info);
restore_processing_stripe_boundary(&remaining_stripes, rlbs, highbd, h,
data8, stride, copy_above, copy_below,
@@ -1090,17 +1047,17 @@
}
static void filter_frame_on_unit(const RestorationTileLimits *limits,
- const PixelRect *tile_rect, int rest_unit_idx,
- void *priv, int32_t *tmpbuf,
- RestorationLineBuffers *rlbs) {
+ int rest_unit_idx, void *priv, int32_t *tmpbuf,
+ RestorationLineBuffers *rlbs,
+ struct aom_internal_error_info *error_info) {
FilterFrameCtxt *ctxt = (FilterFrameCtxt *)priv;
const RestorationInfo *rsi = ctxt->rsi;
av1_loop_restoration_filter_unit(
- limits, &rsi->unit_info[rest_unit_idx], &rsi->boundaries, rlbs, tile_rect,
- ctxt->tile_stripe0, ctxt->ss_x, ctxt->ss_y, ctxt->highbd, ctxt->bit_depth,
- ctxt->data8, ctxt->data_stride, ctxt->dst8, ctxt->dst_stride, tmpbuf,
- rsi->optimized_lr);
+ limits, &rsi->unit_info[rest_unit_idx], &rsi->boundaries, rlbs,
+ ctxt->plane_w, ctxt->plane_h, ctxt->ss_x, ctxt->ss_y, ctxt->highbd,
+ ctxt->bit_depth, ctxt->data8, ctxt->data_stride, ctxt->dst8,
+ ctxt->dst_stride, tmpbuf, rsi->optimized_lr, error_info);
}
void av1_loop_restoration_filter_frame_init(AV1LrStruct *lr_ctxt,
@@ -1127,31 +1084,33 @@
RestorationInfo *rsi = &cm->rst_info[plane];
RestorationType rtype = rsi->frame_restoration_type;
rsi->optimized_lr = optimized_lr;
+ lr_ctxt->ctxt[plane].rsi = rsi;
if (rtype == RESTORE_NONE) {
continue;
}
const int is_uv = plane > 0;
- const int plane_width = frame->crop_widths[is_uv];
- const int plane_height = frame->crop_heights[is_uv];
- FilterFrameCtxt *lr_plane_ctxt = &lr_ctxt->ctxt[plane];
+ int plane_w, plane_h;
+ av1_get_upsampled_plane_size(cm, is_uv, &plane_w, &plane_h);
+ assert(plane_w == frame->crop_widths[is_uv]);
+ assert(plane_h == frame->crop_heights[is_uv]);
- av1_extend_frame(frame->buffers[plane], plane_width, plane_height,
+ av1_extend_frame(frame->buffers[plane], plane_w, plane_h,
frame->strides[is_uv], RESTORATION_BORDER,
RESTORATION_BORDER, highbd);
- lr_plane_ctxt->rsi = rsi;
+ FilterFrameCtxt *lr_plane_ctxt = &lr_ctxt->ctxt[plane];
lr_plane_ctxt->ss_x = is_uv && seq_params->subsampling_x;
lr_plane_ctxt->ss_y = is_uv && seq_params->subsampling_y;
+ lr_plane_ctxt->plane_w = plane_w;
+ lr_plane_ctxt->plane_h = plane_h;
lr_plane_ctxt->highbd = highbd;
lr_plane_ctxt->bit_depth = bit_depth;
lr_plane_ctxt->data8 = frame->buffers[plane];
lr_plane_ctxt->dst8 = lr_ctxt->dst->buffers[plane];
lr_plane_ctxt->data_stride = frame->strides[is_uv];
lr_plane_ctxt->dst_stride = lr_ctxt->dst->strides[is_uv];
- lr_plane_ctxt->tile_rect = av1_whole_frame_rect(cm, is_uv);
- lr_plane_ctxt->tile_stripe0 = 0;
}
}
@@ -1166,9 +1125,9 @@
assert(num_planes <= 3);
for (int plane = 0; plane < num_planes; ++plane) {
if (cm->rst_info[plane].frame_restoration_type == RESTORE_NONE) continue;
- PixelRect tile_rect = loop_rest_ctxt->ctxt[plane].tile_rect;
- copy_funs[plane](loop_rest_ctxt->dst, loop_rest_ctxt->frame, tile_rect.left,
- tile_rect.right, tile_rect.top, tile_rect.bottom);
+ FilterFrameCtxt *lr_plane_ctxt = &loop_rest_ctxt->ctxt[plane];
+ copy_funs[plane](loop_rest_ctxt->dst, loop_rest_ctxt->frame, 0,
+ lr_plane_ctxt->plane_w, 0, lr_plane_ctxt->plane_h);
}
}
@@ -1182,8 +1141,7 @@
}
av1_foreach_rest_unit_in_plane(cm, plane, lr_ctxt->on_rest_unit,
- &ctxt[plane], &ctxt[plane].tile_rect,
- cm->rst_tmpbuf, cm->rlbs);
+ &ctxt[plane], cm->rst_tmpbuf, cm->rlbs);
}
}
@@ -1204,24 +1162,23 @@
}
void av1_foreach_rest_unit_in_row(
- RestorationTileLimits *limits, const PixelRect *tile_rect,
+ RestorationTileLimits *limits, int plane_w,
rest_unit_visitor_t on_rest_unit, int row_number, int unit_size,
- int unit_idx0, int hunits_per_tile, int vunits_per_tile, int plane,
- void *priv, int32_t *tmpbuf, RestorationLineBuffers *rlbs,
- sync_read_fn_t on_sync_read, sync_write_fn_t on_sync_write,
- struct AV1LrSyncData *const lr_sync) {
- const int tile_w = tile_rect->right - tile_rect->left;
+ int hnum_rest_units, int vnum_rest_units, int plane, void *priv,
+ int32_t *tmpbuf, RestorationLineBuffers *rlbs, sync_read_fn_t on_sync_read,
+ sync_write_fn_t on_sync_write, struct AV1LrSyncData *const lr_sync,
+ struct aom_internal_error_info *error_info) {
const int ext_size = unit_size * 3 / 2;
int x0 = 0, j = 0;
- while (x0 < tile_w) {
- int remaining_w = tile_w - x0;
+ while (x0 < plane_w) {
+ int remaining_w = plane_w - x0;
int w = (remaining_w < ext_size) ? remaining_w : unit_size;
- limits->h_start = tile_rect->left + x0;
- limits->h_end = tile_rect->left + x0 + w;
- assert(limits->h_end <= tile_rect->right);
+ limits->h_start = x0;
+ limits->h_end = x0 + w;
+ assert(limits->h_end <= plane_w);
- const int unit_idx = unit_idx0 + row_number * hunits_per_tile + j;
+ const int unit_idx = row_number * hnum_rest_units + j;
// No sync for even numbered rows
// For odd numbered rows, Loop Restoration of current block requires the LR
@@ -1229,13 +1186,23 @@
// top-right sync
on_sync_read(lr_sync, row_number, j, plane);
- if ((row_number + 1) < vunits_per_tile)
+ if ((row_number + 1) < vnum_rest_units)
// bottom-right sync
on_sync_read(lr_sync, row_number + 2, j, plane);
- on_rest_unit(limits, tile_rect, unit_idx, priv, tmpbuf, rlbs);
+#if CONFIG_MULTITHREAD
+ if (lr_sync && lr_sync->num_workers > 1) {
+ pthread_mutex_lock(lr_sync->job_mutex);
+ const bool lr_mt_exit = lr_sync->lr_mt_exit;
+ pthread_mutex_unlock(lr_sync->job_mutex);
+ // Exit in case any worker has encountered an error.
+ if (lr_mt_exit) return;
+ }
+#endif
- on_sync_write(lr_sync, row_number, j, hunits_per_tile, plane);
+ on_rest_unit(limits, unit_idx, priv, tmpbuf, rlbs, error_info);
+
+ on_sync_write(lr_sync, row_number, j, hnum_rest_units, plane);
x0 += w;
++j;
@@ -1258,57 +1225,45 @@
(void)plane;
}
-static void foreach_rest_unit_in_tile(
- const PixelRect *tile_rect, int tile_row, int tile_col, int tile_cols,
- int hunits_per_tile, int vunits_per_tile, int units_per_tile, int unit_size,
- int ss_y, int plane, rest_unit_visitor_t on_rest_unit, void *priv,
- int32_t *tmpbuf, RestorationLineBuffers *rlbs) {
- const int tile_h = tile_rect->bottom - tile_rect->top;
- const int ext_size = unit_size * 3 / 2;
+void av1_foreach_rest_unit_in_plane(const struct AV1Common *cm, int plane,
+ rest_unit_visitor_t on_rest_unit,
+ void *priv, int32_t *tmpbuf,
+ RestorationLineBuffers *rlbs) {
+ const RestorationInfo *rsi = &cm->rst_info[plane];
+ const int hnum_rest_units = rsi->horz_units;
+ const int vnum_rest_units = rsi->vert_units;
+ const int unit_size = rsi->restoration_unit_size;
- const int tile_idx = tile_col + tile_row * tile_cols;
- const int unit_idx0 = tile_idx * units_per_tile;
+ const int is_uv = plane > 0;
+ const int ss_y = is_uv && cm->seq_params->subsampling_y;
+ const int ext_size = unit_size * 3 / 2;
+ int plane_w, plane_h;
+ av1_get_upsampled_plane_size(cm, is_uv, &plane_w, &plane_h);
int y0 = 0, i = 0;
- while (y0 < tile_h) {
- int remaining_h = tile_h - y0;
+ while (y0 < plane_h) {
+ int remaining_h = plane_h - y0;
int h = (remaining_h < ext_size) ? remaining_h : unit_size;
RestorationTileLimits limits;
- limits.v_start = tile_rect->top + y0;
- limits.v_end = tile_rect->top + y0 + h;
- assert(limits.v_end <= tile_rect->bottom);
- // Offset the tile upwards to align with the restoration processing stripe
+ limits.v_start = y0;
+ limits.v_end = y0 + h;
+ assert(limits.v_end <= plane_h);
+ // Offset upwards to align with the restoration processing stripe
const int voffset = RESTORATION_UNIT_OFFSET >> ss_y;
- limits.v_start = AOMMAX(tile_rect->top, limits.v_start - voffset);
- if (limits.v_end < tile_rect->bottom) limits.v_end -= voffset;
+ limits.v_start = AOMMAX(0, limits.v_start - voffset);
+ if (limits.v_end < plane_h) limits.v_end -= voffset;
- av1_foreach_rest_unit_in_row(
- &limits, tile_rect, on_rest_unit, i, unit_size, unit_idx0,
- hunits_per_tile, vunits_per_tile, plane, priv, tmpbuf, rlbs,
- av1_lr_sync_read_dummy, av1_lr_sync_write_dummy, NULL);
+ av1_foreach_rest_unit_in_row(&limits, plane_w, on_rest_unit, i, unit_size,
+ hnum_rest_units, vnum_rest_units, plane, priv,
+ tmpbuf, rlbs, av1_lr_sync_read_dummy,
+ av1_lr_sync_write_dummy, NULL, cm->error);
y0 += h;
++i;
}
}
-void av1_foreach_rest_unit_in_plane(const struct AV1Common *cm, int plane,
- rest_unit_visitor_t on_rest_unit,
- void *priv, PixelRect *tile_rect,
- int32_t *tmpbuf,
- RestorationLineBuffers *rlbs) {
- const int is_uv = plane > 0;
- const int ss_y = is_uv && cm->seq_params->subsampling_y;
-
- const RestorationInfo *rsi = &cm->rst_info[plane];
-
- foreach_rest_unit_in_tile(tile_rect, LR_TILE_ROW, LR_TILE_COL, LR_TILE_COLS,
- rsi->horz_units_per_tile, rsi->vert_units_per_tile,
- rsi->units_per_tile, rsi->restoration_unit_size,
- ss_y, plane, on_rest_unit, priv, tmpbuf, rlbs);
-}
-
int av1_loop_restoration_corners_in_sb(const struct AV1Common *cm, int plane,
int mi_row, int mi_col, BLOCK_SIZE bsize,
int *rcol0, int *rcol1, int *rrow0,
@@ -1316,33 +1271,21 @@
assert(rcol0 && rcol1 && rrow0 && rrow1);
if (bsize != cm->seq_params->sb_size) return 0;
- if (cm->rst_info[plane].frame_restoration_type == RESTORE_NONE) return 0;
assert(!cm->features.all_lossless);
const int is_uv = plane > 0;
- const PixelRect tile_rect = av1_whole_frame_rect(cm, is_uv);
- const int tile_w = tile_rect.right - tile_rect.left;
- const int tile_h = tile_rect.bottom - tile_rect.top;
-
- const int mi_top = 0;
- const int mi_left = 0;
-
- // Compute the mi-unit corners of the superblock relative to the top-left of
- // the tile
- const int mi_rel_row0 = mi_row - mi_top;
- const int mi_rel_col0 = mi_col - mi_left;
- const int mi_rel_row1 = mi_rel_row0 + mi_size_high[bsize];
- const int mi_rel_col1 = mi_rel_col0 + mi_size_wide[bsize];
+ // Compute the mi-unit corners of the superblock
+ const int mi_row0 = mi_row;
+ const int mi_col0 = mi_col;
+ const int mi_row1 = mi_row0 + mi_size_high[bsize];
+ const int mi_col1 = mi_col0 + mi_size_wide[bsize];
const RestorationInfo *rsi = &cm->rst_info[plane];
const int size = rsi->restoration_unit_size;
-
- // Calculate the number of restoration units in this tile (which might be
- // strictly less than rsi->horz_units_per_tile and rsi->vert_units_per_tile)
- const int horz_units = av1_lr_count_units_in_tile(size, tile_w);
- const int vert_units = av1_lr_count_units_in_tile(size, tile_h);
+ const int horz_units = rsi->horz_units;
+ const int vert_units = rsi->vert_units;
// The size of an MI-unit on this plane of the image
const int ss_x = is_uv && cm->seq_params->subsampling_x;
@@ -1367,19 +1310,18 @@
const int rnd_x = denom_x - 1;
const int rnd_y = denom_y - 1;
- // rcol0/rrow0 should be the first column/row of restoration units (relative
- // to the top-left of the tile) that doesn't start left/below of
- // mi_col/mi_row. For this calculation, we need to round up the division (if
- // the sb starts at runit column 10.1, the first matching runit has column
- // index 11)
- *rcol0 = (mi_rel_col0 * mi_to_num_x + rnd_x) / denom_x;
- *rrow0 = (mi_rel_row0 * mi_to_num_y + rnd_y) / denom_y;
+ // rcol0/rrow0 should be the first column/row of restoration units that
+ // doesn't start left/below of mi_col/mi_row. For this calculation, we need
+ // to round up the division (if the sb starts at runit column 10.1, the first
+ // matching runit has column index 11)
+ *rcol0 = (mi_col0 * mi_to_num_x + rnd_x) / denom_x;
+ *rrow0 = (mi_row0 * mi_to_num_y + rnd_y) / denom_y;
// rel_col1/rel_row1 is the equivalent calculation, but for the superblock
- // below-right. If we're at the bottom or right of the tile, this restoration
+ // below-right. If we're at the bottom or right of the frame, this restoration
// unit might not exist, in which case we'll clamp accordingly.
- *rcol1 = AOMMIN((mi_rel_col1 * mi_to_num_x + rnd_x) / denom_x, horz_units);
- *rrow1 = AOMMIN((mi_rel_row1 * mi_to_num_y + rnd_y) / denom_y, vert_units);
+ *rcol1 = AOMMIN((mi_col1 * mi_to_num_x + rnd_x) / denom_x, horz_units);
+ *rrow1 = AOMMIN((mi_row1 * mi_to_num_y + rnd_y) / denom_y, vert_units);
return *rcol0 < *rcol1 && *rrow0 < *rrow1;
}
@@ -1480,73 +1422,59 @@
: src_width;
const int line_bytes = upscaled_width << use_highbd;
for (int i = 0; i < RESTORATION_CTX_VERT; i++) {
- // Copy the line at 'row' into both context lines. This is because
- // we want to (effectively) extend the outermost row of CDEF data
- // from this tile to produce a border, rather than using deblocked
- // pixels from the tile above/below.
+ // Copy the line at 'src_rows' into both context lines
memcpy(bdry_rows + i * bdry_stride, src_rows, line_bytes);
}
extend_lines(bdry_rows, upscaled_width, RESTORATION_CTX_VERT, bdry_stride,
RESTORATION_EXTRA_HORZ, use_highbd);
}
-static void save_tile_row_boundary_lines(const YV12_BUFFER_CONFIG *frame,
- int use_highbd, int plane,
- AV1_COMMON *cm, int after_cdef) {
+static void save_boundary_lines(const YV12_BUFFER_CONFIG *frame, int use_highbd,
+ int plane, AV1_COMMON *cm, int after_cdef) {
const int is_uv = plane > 0;
const int ss_y = is_uv && cm->seq_params->subsampling_y;
const int stripe_height = RESTORATION_PROC_UNIT_SIZE >> ss_y;
const int stripe_off = RESTORATION_UNIT_OFFSET >> ss_y;
- // Get the tile rectangle, with height rounded up to the next multiple of 8
- // luma pixels (only relevant for the bottom tile of the frame)
- const PixelRect tile_rect = av1_whole_frame_rect(cm, is_uv);
- const int stripe0 = 0;
+ int plane_w, plane_h;
+ av1_get_upsampled_plane_size(cm, is_uv, &plane_w, &plane_h);
RestorationStripeBoundaries *boundaries = &cm->rst_info[plane].boundaries;
const int plane_height = ROUND_POWER_OF_TWO(cm->height, ss_y);
- int tile_stripe;
- for (tile_stripe = 0;; ++tile_stripe) {
- const int rel_y0 = AOMMAX(0, tile_stripe * stripe_height - stripe_off);
- const int y0 = tile_rect.top + rel_y0;
- if (y0 >= tile_rect.bottom) break;
+ int stripe_idx;
+ for (stripe_idx = 0;; ++stripe_idx) {
+ const int rel_y0 = AOMMAX(0, stripe_idx * stripe_height - stripe_off);
+ const int y0 = rel_y0;
+ if (y0 >= plane_h) break;
- const int rel_y1 = (tile_stripe + 1) * stripe_height - stripe_off;
- const int y1 = AOMMIN(tile_rect.top + rel_y1, tile_rect.bottom);
+ const int rel_y1 = (stripe_idx + 1) * stripe_height - stripe_off;
+ const int y1 = AOMMIN(rel_y1, plane_h);
- const int frame_stripe = stripe0 + tile_stripe;
-
- // In this case, we should only use CDEF pixels at the top
- // and bottom of the frame as a whole; internal tile boundaries
- // can use deblocked pixels from adjacent tiles for context.
- const int use_deblock_above = (frame_stripe > 0);
+ // Extend using CDEF pixels at the top and bottom of the frame,
+ // and deblocked pixels at internal stripe boundaries
+ const int use_deblock_above = (stripe_idx > 0);
const int use_deblock_below = (y1 < plane_height);
if (!after_cdef) {
- // Save deblocked context where needed.
+ // Save deblocked context at internal stripe boundaries
if (use_deblock_above) {
save_deblock_boundary_lines(frame, cm, plane, y0 - RESTORATION_CTX_VERT,
- frame_stripe, use_highbd, 1, boundaries);
+ stripe_idx, use_highbd, 1, boundaries);
}
if (use_deblock_below) {
- save_deblock_boundary_lines(frame, cm, plane, y1, frame_stripe,
+ save_deblock_boundary_lines(frame, cm, plane, y1, stripe_idx,
use_highbd, 0, boundaries);
}
} else {
- // Save CDEF context where needed. Note that we need to save the CDEF
- // context for a particular boundary iff we *didn't* save deblocked
- // context for that boundary.
- //
- // In addition, we need to save copies of the outermost line within
- // the tile, rather than using data from outside the tile.
+ // Save CDEF context at frame boundaries
if (!use_deblock_above) {
- save_cdef_boundary_lines(frame, cm, plane, y0, frame_stripe, use_highbd,
+ save_cdef_boundary_lines(frame, cm, plane, y0, stripe_idx, use_highbd,
1, boundaries);
}
if (!use_deblock_below) {
- save_cdef_boundary_lines(frame, cm, plane, y1 - 1, frame_stripe,
+ save_cdef_boundary_lines(frame, cm, plane, y1 - 1, stripe_idx,
use_highbd, 0, boundaries);
}
}
@@ -1561,6 +1489,6 @@
const int num_planes = av1_num_planes(cm);
const int use_highbd = cm->seq_params->use_highbitdepth;
for (int p = 0; p < num_planes; ++p) {
- save_tile_row_boundary_lines(frame, use_highbd, p, cm, after_cdef);
+ save_boundary_lines(frame, use_highbd, p, cm, after_cdef);
}
}
diff --git a/av1/common/restoration.h b/av1/common/restoration.h
index bf21303..d5da81d 100644
--- a/av1/common/restoration.h
+++ b/av1/common/restoration.h
@@ -33,7 +33,7 @@
#define RESTORATION_PROC_UNIT_SIZE 64
-// Filter tile grid offset upwards compared to the superblock grid
+// Filter stripe grid offset upwards compared to the superblock grid
#define RESTORATION_UNIT_OFFSET 8
#define SGRPROJ_BORDER_VERT 3 // Vertical border used for Sgr
@@ -180,10 +180,6 @@
#error "Wiener filter currently only works if WIENER_FILT_PREC_BITS == 7"
#endif
-#define LR_TILE_ROW 0
-#define LR_TILE_COL 0
-#define LR_TILE_COLS 1
-
typedef struct {
int r[2]; // radii
int s[2]; // sgr parameters for r[0] and r[1], based on GenSgrprojVtable()
@@ -215,12 +211,6 @@
#define RESTORATION_LINEBUFFER_WIDTH \
(RESTORATION_UNITSIZE_MAX * 3 / 2 + 2 * RESTORATION_EXTRA_HORZ)
-// Similarly, the column buffers (used when we're at a vertical tile edge
-// that we can't filter across) need space for one processing unit's worth
-// of pixels, plus the top/bottom border width
-#define RESTORATION_COLBUFFER_HEIGHT \
- (RESTORATION_PROC_UNIT_SIZE + 2 * RESTORATION_BORDER)
-
typedef struct {
// Temporary buffers to save/restore 3 lines above/below the restoration
// stripe.
@@ -266,32 +256,26 @@
/**
* \name Fields allocated and initialised by av1_alloc_restoration_struct.
- * (horz_)units_per_tile give the number of restoration units in
- * (one row of) the largest tile in the frame.
*/
/**@{*/
/*!
- * Number of units per tile for the largest tile in the frame
+ * Total number of restoration units in this plane
*/
- int units_per_tile;
+ int num_rest_units;
/*!
- * Number of vertical units per tile
+ * Number of vertical restoration units in this plane
*/
- int vert_units_per_tile;
+ int vert_units;
/*!
- * Number of horizontal units per tile for the largest tile in the frame
+ * Number of horizontal restoration units in this plane
*/
- int horz_units_per_tile;
+ int horz_units;
/**@}*/
/*!
- * List of info for units in tile.
- * The data in unit_info is laid out with units_per_tile entries for each
- * tile, which have stride horz_units_per_tile.
- * Even if there are tiles of different sizes, the data in unit_info is
- * laid out as if all tiles are of full size.
+ * Parameters for each restoration unit in this plane
*/
RestorationUnitInfo *unit_info;
@@ -332,19 +316,18 @@
} RestorationTileLimits;
typedef void (*rest_unit_visitor_t)(const RestorationTileLimits *limits,
- const PixelRect *tile_rect,
int rest_unit_idx, void *priv,
int32_t *tmpbuf,
- RestorationLineBuffers *rlbs);
+ RestorationLineBuffers *rlbs,
+ struct aom_internal_error_info *error_info);
typedef struct FilterFrameCtxt {
const RestorationInfo *rsi;
- int tile_stripe0;
int ss_x, ss_y;
+ int plane_w, plane_h;
int highbd, bit_depth;
uint8_t *data8, *dst8;
int data_stride, dst_stride;
- PixelRect tile_rect;
} FilterFrameCtxt;
typedef struct AV1LrStruct {
@@ -375,27 +358,29 @@
* This function applies the loop restoration filter to a single
* loop restoration unit.
*
- * \param[in] limits Limits of the unit
- * \param[in] rui The parameters to use for this unit and its
- * coefficients
- * \param[in] rsb Deblocked pixels to use for stripe boundaries
- * \param[in] rlbs Space to use as a scratch buffer
- * \param[in] tile_rect Limits of the tile containing this unit
- * \param[in] tile_stripe0 Index of the first stripe in this tile
- * \param[in] ss_x Horizontal subsampling for plane
- * \param[in] ss_y Vertical subsampling for plane
- * \param[in] highbd Whether high bitdepth pipeline is used
- * \param[in] bit_depth Bit-depth of the video
- * \param[in] data8 Frame data (pointing at the top-left corner of
- * the frame, not the restoration unit).
- * \param[in] stride Stride of \c data8
- * \param[out] dst8 Buffer where the results will be written. Like
- * \c data8, \c dst8 should point at the top-left
- * corner of the frame
- * \param[in] dst_stride Stride of \c dst8
- * \param[in] tmpbuf Scratch buffer used by the sgrproj filter which
- * should be at least SGRPROJ_TMPBUF_SIZE big.
- * \param[in] optimized_lr Whether to use fast optimized Loop Restoration
+ * \param[in] limits Limits of the unit
+ * \param[in] rui The parameters to use for this unit and its
+ * coefficients
+ * \param[in] rsb Deblocked pixels to use for stripe boundaries
+ * \param[in] rlbs Space to use as a scratch buffer
+ * \param[in] ss_x Horizontal subsampling for plane
+ * \param[in] ss_y Vertical subsampling for plane
+ * \param[in] plane_w Width of the current plane
+ * \param[in] plane_h Height of the current plane
+ * \param[in] highbd Whether high bitdepth pipeline is used
+ * \param[in] bit_depth Bit-depth of the video
+ * \param[in] data8 Frame data (pointing at the top-left corner of
+ * the frame, not the restoration unit).
+ * \param[in] stride Stride of \c data8
+ * \param[out] dst8 Buffer where the results will be written. Like
+ * \c data8, \c dst8 should point at the top-left
+ * corner of the frame
+ * \param[in] dst_stride Stride of \c dst8
+ * \param[in] tmpbuf Scratch buffer used by the sgrproj filter
+ * which should be at least SGRPROJ_TMPBUF_SIZE
+ * big.
+ * \param[in] optimized_lr Whether to use fast optimized Loop Restoration
+ * \param[in,out] error_info Error info for reporting errors
*
* \remark Nothing is returned. Instead, the filtered unit is output in
* \c dst8 at the proper restoration unit offset.
@@ -403,17 +388,17 @@
void av1_loop_restoration_filter_unit(
const RestorationTileLimits *limits, const RestorationUnitInfo *rui,
const RestorationStripeBoundaries *rsb, RestorationLineBuffers *rlbs,
- const PixelRect *tile_rect, int tile_stripe0, int ss_x, int ss_y,
- int highbd, int bit_depth, uint8_t *data8, int stride, uint8_t *dst8,
- int dst_stride, int32_t *tmpbuf, int optimized_lr);
+ int plane_w, int plane_h, int ss_x, int ss_y, int highbd, int bit_depth,
+ uint8_t *data8, int stride, uint8_t *dst8, int dst_stride, int32_t *tmpbuf,
+ int optimized_lr, struct aom_internal_error_info *error_info);
/*!\brief Function for applying loop restoration filter to a frame
*
* \ingroup in_loop_restoration
* This function applies the loop restoration filter to a frame.
*
- * \param[in, out] frame Compressed frame buffer
- * \param[in, out] cm Pointer to top level common structure
+ * \param[in,out] frame Compressed frame buffer
+ * \param[in,out] cm Pointer to top level common structure
* \param[in] optimized_lr Whether to use fast optimized Loop Restoration
* \param[in] lr_ctxt Loop restoration context
*
@@ -427,8 +412,6 @@
void av1_loop_restoration_precal();
-typedef void (*rest_tile_start_visitor_t)(int tile_row, int tile_col,
- void *priv);
struct AV1LrSyncData;
typedef void (*sync_read_fn_t)(void *const lr_sync, int r, int c, int plane);
@@ -439,8 +422,7 @@
// Call on_rest_unit for each loop restoration unit in the plane.
void av1_foreach_rest_unit_in_plane(const struct AV1Common *cm, int plane,
rest_unit_visitor_t on_rest_unit,
- void *priv, PixelRect *tile_rect,
- int32_t *tmpbuf,
+ void *priv, int32_t *tmpbuf,
RestorationLineBuffers *rlbs);
// Return 1 iff the block at mi_row, mi_col with size bsize is a
@@ -448,10 +430,9 @@
// loop restoration unit.
//
// If the block is a top-level superblock, the function writes to
-// *rcol0, *rcol1, *rrow0, *rrow1. The rectangle of restoration unit
-// indices given by [*rcol0, *rcol1) x [*rrow0, *rrow1) are relative
-// to the current tile, whose starting index is returned as
-// *tile_tl_idx.
+// *rcol0, *rcol1, *rrow0, *rrow1. This means that the parameters for all
+// restoration units in the rectangle [*rcol0, *rcol1) x [*rrow0, *rrow1)
+// are signaled in this superblock.
int av1_loop_restoration_corners_in_sb(const struct AV1Common *cm, int plane,
int mi_row, int mi_col, BLOCK_SIZE bsize,
int *rcol0, int *rcol1, int *rrow0,
@@ -467,14 +448,16 @@
void av1_loop_restoration_copy_planes(AV1LrStruct *loop_rest_ctxt,
struct AV1Common *cm, int num_planes);
void av1_foreach_rest_unit_in_row(
- RestorationTileLimits *limits, const PixelRect *tile_rect,
+ RestorationTileLimits *limits, int plane_w,
rest_unit_visitor_t on_rest_unit, int row_number, int unit_size,
- int unit_idx0, int hunits_per_tile, int vunits_per_tile, int plane,
- void *priv, int32_t *tmpbuf, RestorationLineBuffers *rlbs,
- sync_read_fn_t on_sync_read, sync_write_fn_t on_sync_write,
- struct AV1LrSyncData *const lr_sync);
-PixelRect av1_whole_frame_rect(const struct AV1Common *cm, int is_uv);
-int av1_lr_count_units_in_tile(int unit_size, int tile_size);
+ int hnum_rest_units, int vnum_rest_units, int plane, void *priv,
+ int32_t *tmpbuf, RestorationLineBuffers *rlbs, sync_read_fn_t on_sync_read,
+ sync_write_fn_t on_sync_write, struct AV1LrSyncData *const lr_sync,
+ struct aom_internal_error_info *error_info);
+
+void av1_get_upsampled_plane_size(const struct AV1Common *cm, int is_uv,
+ int *plane_w, int *plane_h);
+int av1_lr_count_units(int unit_size, int plane_size);
void av1_lr_sync_read_dummy(void *const lr_sync, int r, int c, int plane);
void av1_lr_sync_write_dummy(void *const lr_sync, int r, int c,
const int sb_cols, int plane);
diff --git a/av1/common/thread_common.c b/av1/common/thread_common.c
index 6a4427b..8a6f290 100644
--- a/av1/common/thread_common.c
+++ b/av1/common/thread_common.c
@@ -57,6 +57,7 @@
void av1_loop_filter_alloc(AV1LfSync *lf_sync, AV1_COMMON *cm, int rows,
int width, int num_workers) {
lf_sync->rows = rows;
+ lf_sync->lf_mt_exit = false;
#if CONFIG_MULTITHREAD
{
int i, j;
@@ -252,8 +253,12 @@
const YV12_BUFFER_CONFIG *const frame_buffer, AV1_COMMON *const cm,
struct macroblockd_plane *planes, MACROBLOCKD *xd, int mi_row, int plane,
int dir, int lpf_opt_level, AV1LfSync *const lf_sync,
+ struct aom_internal_error_info *error_info,
AV1_DEBLOCKING_PARAMETERS *params_buf, TX_SIZE *tx_buf,
int num_mis_in_lpf_unit_height_log2) {
+ // TODO(aomedia:3276): Pass error_info to the low-level functions as required
+ // in future to handle error propagation.
+ (void)error_info;
const int sb_cols =
CEIL_POWER_OF_TWO(cm->mi_params.mi_cols, MAX_MIB_SIZE_LOG2);
const int r = mi_row >> num_mis_in_lpf_unit_height_log2;
@@ -300,6 +305,16 @@
sync_read(lf_sync, r + 1, c, plane);
}
+#if CONFIG_MULTITHREAD
+ if (lf_sync && lf_sync->num_workers > 1) {
+ pthread_mutex_lock(lf_sync->job_mutex);
+ const bool lf_mt_exit = lf_sync->lf_mt_exit;
+ pthread_mutex_unlock(lf_sync->job_mutex);
+ // Exit in case any worker has encountered an error.
+ if (lf_mt_exit) return;
+ }
+#endif
+
av1_setup_dst_planes(planes, cm->seq_params->sb_size, frame_buffer,
mi_row, mi_col, plane, plane + num_planes);
if (lpf_opt_level) {
@@ -320,27 +335,93 @@
}
}
+void av1_set_vert_loop_filter_done(AV1_COMMON *cm, AV1LfSync *lf_sync,
+ int num_mis_in_lpf_unit_height_log2) {
+ int plane, sb_row;
+ const int sb_cols =
+ CEIL_POWER_OF_TWO(cm->mi_params.mi_cols, num_mis_in_lpf_unit_height_log2);
+ const int sb_rows =
+ CEIL_POWER_OF_TWO(cm->mi_params.mi_rows, num_mis_in_lpf_unit_height_log2);
+
+ // In case of loopfilter row-multithreading, the worker on an SB row waits for
+ // the vertical edge filtering of the right and top-right SBs. Hence, in case
+ // a thread (main/worker) encounters an error, update that vertical
+ // loopfiltering of every SB row in the frame is complete in order to avoid
+ // dependent workers waiting indefinitely.
+ for (sb_row = 0; sb_row < sb_rows; ++sb_row)
+ for (plane = 0; plane < MAX_MB_PLANE; ++plane)
+ sync_write(lf_sync, sb_row, sb_cols - 1, sb_cols, plane);
+}
+
+static AOM_INLINE void sync_lf_workers(AVxWorker *const workers,
+ AV1_COMMON *const cm, int num_workers) {
+ const AVxWorkerInterface *const winterface = aom_get_worker_interface();
+ int had_error = workers[0].had_error;
+ struct aom_internal_error_info error_info;
+
+ // Read the error_info of main thread.
+ if (had_error) {
+ AVxWorker *const worker = &workers[0];
+ error_info = ((LFWorkerData *)worker->data2)->error_info;
+ }
+
+ // Wait till all rows are finished.
+ for (int i = num_workers - 1; i > 0; --i) {
+ AVxWorker *const worker = &workers[i];
+ if (!winterface->sync(worker)) {
+ had_error = 1;
+ error_info = ((LFWorkerData *)worker->data2)->error_info;
+ }
+ }
+ if (had_error)
+ aom_internal_error(cm->error, error_info.error_code, "%s",
+ error_info.detail);
+}
+
// Row-based multi-threaded loopfilter hook
static int loop_filter_row_worker(void *arg1, void *arg2) {
AV1LfSync *const lf_sync = (AV1LfSync *)arg1;
LFWorkerData *const lf_data = (LFWorkerData *)arg2;
AV1LfMTInfo *cur_job_info;
+
+#if CONFIG_MULTITHREAD
+ pthread_mutex_t *job_mutex_ = lf_sync->job_mutex;
+#endif
+
+ struct aom_internal_error_info *const error_info = &lf_data->error_info;
+
+ // The jmp_buf is valid only for the duration of the function that calls
+ // setjmp(). Therefore, this function must reset the 'setjmp' field to 0
+ // before it returns.
+ if (setjmp(error_info->jmp)) {
+ error_info->setjmp = 0;
+#if CONFIG_MULTITHREAD
+ pthread_mutex_lock(job_mutex_);
+ lf_sync->lf_mt_exit = true;
+ pthread_mutex_unlock(job_mutex_);
+#endif
+ av1_set_vert_loop_filter_done(lf_data->cm, lf_sync, MAX_MIB_SIZE_LOG2);
+ return 0;
+ }
+ error_info->setjmp = 1;
+
while ((cur_job_info = get_lf_job_info(lf_sync)) != NULL) {
const int lpf_opt_level = cur_job_info->lpf_opt_level;
av1_thread_loop_filter_rows(
lf_data->frame_buffer, lf_data->cm, lf_data->planes, lf_data->xd,
cur_job_info->mi_row, cur_job_info->plane, cur_job_info->dir,
- lpf_opt_level, lf_sync, lf_data->params_buf, lf_data->tx_buf,
- MAX_MIB_SIZE_LOG2);
+ lpf_opt_level, lf_sync, error_info, lf_data->params_buf,
+ lf_data->tx_buf, MAX_MIB_SIZE_LOG2);
}
+ error_info->setjmp = 0;
return 1;
}
static void loop_filter_rows_mt(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm,
MACROBLOCKD *xd, int start, int stop,
- const int planes_to_lf[3], AVxWorker *workers,
- int num_workers, AV1LfSync *lf_sync,
- int lpf_opt_level) {
+ const int planes_to_lf[MAX_MB_PLANE],
+ AVxWorker *workers, int num_workers,
+ AV1LfSync *lf_sync, int lpf_opt_level) {
const AVxWorkerInterface *const winterface = aom_get_worker_interface();
int i;
loop_filter_frame_mt_init(cm, start, stop, planes_to_lf, num_workers, lf_sync,
@@ -359,6 +440,7 @@
loop_filter_data_reset(lf_data, frame, cm, xd);
// Start loopfiltering
+ worker->had_error = 0;
if (i == 0) {
winterface->execute(worker);
} else {
@@ -366,15 +448,13 @@
}
}
- // Wait till all rows are finished
- for (i = 1; i < num_workers; ++i) {
- winterface->sync(&workers[i]);
- }
+ sync_lf_workers(workers, cm, num_workers);
}
static void loop_filter_rows(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm,
MACROBLOCKD *xd, int start, int stop,
- const int planes_to_lf[3], int lpf_opt_level) {
+ const int planes_to_lf[MAX_MB_PLANE],
+ int lpf_opt_level) {
// Filter top rows of all planes first, in case the output can be partially
// reconstructed row by row.
int mi_row, plane, dir;
@@ -382,7 +462,7 @@
AV1_DEBLOCKING_PARAMETERS params_buf[MAX_MIB_SIZE];
TX_SIZE tx_buf[MAX_MIB_SIZE];
for (mi_row = start; mi_row < stop; mi_row += MAX_MIB_SIZE) {
- for (plane = 0; plane < 3; ++plane) {
+ for (plane = 0; plane < MAX_MB_PLANE; ++plane) {
if (skip_loop_filter_plane(planes_to_lf, plane, lpf_opt_level)) {
continue;
}
@@ -390,7 +470,8 @@
for (dir = 0; dir < 2; ++dir) {
av1_thread_loop_filter_rows(frame, cm, xd->plane, xd, mi_row, plane,
dir, lpf_opt_level, /*lf_sync=*/NULL,
- params_buf, tx_buf, MAX_MIB_SIZE_LOG2);
+ xd->error_info, params_buf, tx_buf,
+ MAX_MIB_SIZE_LOG2);
}
}
}
@@ -402,7 +483,7 @@
int num_workers, AV1LfSync *lf_sync,
int lpf_opt_level) {
int start_mi_row, end_mi_row, mi_rows_to_filter;
- int planes_to_lf[3];
+ int planes_to_lf[MAX_MB_PLANE];
if (!check_planes_to_loop_filter(&cm->lf, planes_to_lf, plane_start,
plane_end))
@@ -536,6 +617,7 @@
}
lr_sync->num_workers = num_workers;
+ lr_sync->lr_mt_exit = false;
for (int j = 0; j < num_planes; j++) {
CHECK_MEM_ERROR(
@@ -611,7 +693,7 @@
for (int plane = 0; plane < num_planes; plane++) {
if (cm->rst_info[plane].frame_restoration_type == RESTORE_NONE) continue;
num_even_lr_jobs =
- num_even_lr_jobs + ((ctxt[plane].rsi->vert_units_per_tile + 1) >> 1);
+ num_even_lr_jobs + ((ctxt[plane].rsi->vert_units + 1) >> 1);
}
lr_job_counter[0] = 0;
lr_job_counter[1] = num_even_lr_jobs;
@@ -620,26 +702,23 @@
if (cm->rst_info[plane].frame_restoration_type == RESTORE_NONE) continue;
const int is_uv = plane > 0;
const int ss_y = is_uv && cm->seq_params->subsampling_y;
-
- PixelRect tile_rect = ctxt[plane].tile_rect;
const int unit_size = ctxt[plane].rsi->restoration_unit_size;
-
- const int tile_h = tile_rect.bottom - tile_rect.top;
+ const int plane_h = ctxt[plane].plane_h;
const int ext_size = unit_size * 3 / 2;
int y0 = 0, i = 0;
- while (y0 < tile_h) {
- int remaining_h = tile_h - y0;
+ while (y0 < plane_h) {
+ int remaining_h = plane_h - y0;
int h = (remaining_h < ext_size) ? remaining_h : unit_size;
RestorationTileLimits limits;
- limits.v_start = tile_rect.top + y0;
- limits.v_end = tile_rect.top + y0 + h;
- assert(limits.v_end <= tile_rect.bottom);
- // Offset the tile upwards to align with the restoration processing stripe
+ limits.v_start = y0;
+ limits.v_end = y0 + h;
+ assert(limits.v_end <= plane_h);
+ // Offset upwards to align with the restoration processing stripe
const int voffset = RESTORATION_UNIT_OFFSET >> ss_y;
- limits.v_start = AOMMAX(tile_rect.top, limits.v_start - voffset);
- if (limits.v_end < tile_rect.bottom) limits.v_end -= voffset;
+ limits.v_start = AOMMAX(0, limits.v_start - voffset);
+ if (limits.v_end < plane_h) limits.v_end -= voffset;
assert(lr_job_counter[0] <= num_even_lr_jobs);
@@ -654,18 +733,18 @@
lr_job_queue[lr_job_counter[i & 1]].v_copy_end =
limits.v_end - RESTORATION_BORDER;
if (i == 0) {
- assert(limits.v_start == tile_rect.top);
- lr_job_queue[lr_job_counter[i & 1]].v_copy_start = tile_rect.top;
+ assert(limits.v_start == 0);
+ lr_job_queue[lr_job_counter[i & 1]].v_copy_start = 0;
}
- if (i == (ctxt[plane].rsi->vert_units_per_tile - 1)) {
- assert(limits.v_end == tile_rect.bottom);
- lr_job_queue[lr_job_counter[i & 1]].v_copy_end = tile_rect.bottom;
+ if (i == (ctxt[plane].rsi->vert_units - 1)) {
+ assert(limits.v_end == plane_h);
+ lr_job_queue[lr_job_counter[i & 1]].v_copy_end = plane_h;
}
} else {
lr_job_queue[lr_job_counter[i & 1]].v_copy_start =
- AOMMAX(limits.v_start - RESTORATION_BORDER, tile_rect.top);
+ AOMMAX(limits.v_start - RESTORATION_BORDER, 0);
lr_job_queue[lr_job_counter[i & 1]].v_copy_end =
- AOMMIN(limits.v_end + RESTORATION_BORDER, tile_rect.bottom);
+ AOMMIN(limits.v_end + RESTORATION_BORDER, plane_h);
}
lr_job_counter[i & 1]++;
lr_sync->jobs_enqueued++;
@@ -682,7 +761,7 @@
#if CONFIG_MULTITHREAD
pthread_mutex_lock(lr_sync->job_mutex);
- if (lr_sync->jobs_dequeued < lr_sync->jobs_enqueued) {
+ if (!lr_sync->lr_mt_exit && lr_sync->jobs_dequeued < lr_sync->jobs_enqueued) {
cur_job_info = lr_sync->job_queue + lr_sync->jobs_dequeued;
lr_sync->jobs_dequeued++;
}
@@ -695,6 +774,26 @@
return cur_job_info;
}
+static void set_loop_restoration_done(AV1LrSync *const lr_sync,
+ FilterFrameCtxt *const ctxt) {
+ for (int plane = 0; plane < MAX_MB_PLANE; ++plane) {
+ if (ctxt[plane].rsi->frame_restoration_type == RESTORE_NONE) continue;
+ int y0 = 0, row_number = 0;
+ const int unit_size = ctxt[plane].rsi->restoration_unit_size;
+ const int plane_h = ctxt[plane].plane_h;
+ const int ext_size = unit_size * 3 / 2;
+ const int hnum_rest_units = ctxt[plane].rsi->horz_units;
+ while (y0 < plane_h) {
+ const int remaining_h = plane_h - y0;
+ const int h = (remaining_h < ext_size) ? remaining_h : unit_size;
+ lr_sync_write(lr_sync, row_number, hnum_rest_units - 1, hnum_rest_units,
+ plane);
+ y0 += h;
+ ++row_number;
+ }
+ }
+}
+
// Implement row loop restoration for each thread.
static int loop_restoration_row_worker(void *arg1, void *arg2) {
AV1LrSync *const lr_sync = (AV1LrSync *)arg1;
@@ -703,16 +802,39 @@
FilterFrameCtxt *ctxt = lr_ctxt->ctxt;
int lr_unit_row;
int plane;
- const int tile_row = LR_TILE_ROW;
- const int tile_col = LR_TILE_COL;
- const int tile_cols = LR_TILE_COLS;
- const int tile_idx = tile_col + tile_row * tile_cols;
+ int plane_w;
+#if CONFIG_MULTITHREAD
+ pthread_mutex_t *job_mutex_ = lr_sync->job_mutex;
+#endif
+ struct aom_internal_error_info *const error_info = &lrworkerdata->error_info;
+
+ // The jmp_buf is valid only for the duration of the function that calls
+ // setjmp(). Therefore, this function must reset the 'setjmp' field to 0
+ // before it returns.
+ if (setjmp(error_info->jmp)) {
+ error_info->setjmp = 0;
+#if CONFIG_MULTITHREAD
+ pthread_mutex_lock(job_mutex_);
+ lr_sync->lr_mt_exit = true;
+ pthread_mutex_unlock(job_mutex_);
+#endif
+ // In case of loop restoration multithreading, the worker on an even lr
+ // block row waits for the completion of the filtering of the top-right and
+ // bottom-right blocks. Hence, in case a thread (main/worker) encounters an
+ // error, update that filtering of every row in the frame is complete in
+ // order to avoid the dependent workers from waiting indefinitely.
+ set_loop_restoration_done(lr_sync, lr_ctxt->ctxt);
+ return 0;
+ }
+ error_info->setjmp = 1;
+
typedef void (*copy_fun)(const YV12_BUFFER_CONFIG *src_ybc,
YV12_BUFFER_CONFIG *dst_ybc, int hstart, int hend,
int vstart, int vend);
- static const copy_fun copy_funs[3] = { aom_yv12_partial_coloc_copy_y,
- aom_yv12_partial_coloc_copy_u,
- aom_yv12_partial_coloc_copy_v };
+ static const copy_fun copy_funs[MAX_MB_PLANE] = {
+ aom_yv12_partial_coloc_copy_y, aom_yv12_partial_coloc_copy_u,
+ aom_yv12_partial_coloc_copy_v
+ };
while (1) {
AV1LrMTInfo *cur_job_info = get_lr_job_info(lr_sync);
@@ -724,7 +846,7 @@
limits.v_end = cur_job_info->v_end;
lr_unit_row = cur_job_info->lr_unit_row;
plane = cur_job_info->plane;
- const int unit_idx0 = tile_idx * ctxt[plane].rsi->units_per_tile;
+ plane_w = ctxt[plane].plane_w;
// sync_mode == 1 implies only sync read is required in LR Multi-threading
// sync_mode == 0 implies only sync write is required.
@@ -734,16 +856,14 @@
: av1_lr_sync_write_dummy;
av1_foreach_rest_unit_in_row(
- &limits, &(ctxt[plane].tile_rect), lr_ctxt->on_rest_unit, lr_unit_row,
- ctxt[plane].rsi->restoration_unit_size, unit_idx0,
- ctxt[plane].rsi->horz_units_per_tile,
- ctxt[plane].rsi->vert_units_per_tile, plane, &ctxt[plane],
+ &limits, plane_w, lr_ctxt->on_rest_unit, lr_unit_row,
+ ctxt[plane].rsi->restoration_unit_size, ctxt[plane].rsi->horz_units,
+ ctxt[plane].rsi->vert_units, plane, &ctxt[plane],
lrworkerdata->rst_tmpbuf, lrworkerdata->rlbs, on_sync_read,
- on_sync_write, lr_sync);
+ on_sync_write, lr_sync, error_info);
- copy_funs[plane](lr_ctxt->dst, lr_ctxt->frame, ctxt[plane].tile_rect.left,
- ctxt[plane].tile_rect.right, cur_job_info->v_copy_start,
- cur_job_info->v_copy_end);
+ copy_funs[plane](lr_ctxt->dst, lr_ctxt->frame, 0, plane_w,
+ cur_job_info->v_copy_start, cur_job_info->v_copy_end);
if (lrworkerdata->do_extend_border) {
aom_extend_frame_borders_plane_row(lr_ctxt->frame, plane,
@@ -754,11 +874,37 @@
break;
}
}
+ error_info->setjmp = 0;
return 1;
}
+static AOM_INLINE void sync_lr_workers(AVxWorker *const workers,
+ AV1_COMMON *const cm, int num_workers) {
+ const AVxWorkerInterface *const winterface = aom_get_worker_interface();
+ int had_error = workers[0].had_error;
+ struct aom_internal_error_info error_info;
+
+ // Read the error_info of main thread.
+ if (had_error) {
+ AVxWorker *const worker = &workers[0];
+ error_info = ((LRWorkerData *)worker->data2)->error_info;
+ }
+
+ // Wait till all rows are finished.
+ for (int i = num_workers - 1; i > 0; --i) {
+ AVxWorker *const worker = &workers[i];
+ if (!winterface->sync(worker)) {
+ had_error = 1;
+ error_info = ((LRWorkerData *)worker->data2)->error_info;
+ }
+ }
+ if (had_error)
+ aom_internal_error(cm->error, error_info.error_code, "%s",
+ error_info.detail);
+}
+
static void foreach_rest_unit_in_planes_mt(AV1LrStruct *lr_ctxt,
- AVxWorker *workers, int nworkers,
+ AVxWorker *workers, int num_workers,
AV1LrSync *lr_sync, AV1_COMMON *cm,
int do_extend_border) {
FilterFrameCtxt *ctxt = lr_ctxt->ctxt;
@@ -771,16 +917,12 @@
for (int plane = 0; plane < num_planes; plane++) {
if (cm->rst_info[plane].frame_restoration_type == RESTORE_NONE) continue;
- const PixelRect tile_rect = ctxt[plane].tile_rect;
- const int max_tile_h = tile_rect.bottom - tile_rect.top;
-
+ const int plane_h = ctxt[plane].plane_h;
const int unit_size = cm->rst_info[plane].restoration_unit_size;
- num_rows_lr =
- AOMMAX(num_rows_lr, av1_lr_count_units_in_tile(unit_size, max_tile_h));
+ num_rows_lr = AOMMAX(num_rows_lr, av1_lr_count_units(unit_size, plane_h));
}
- const int num_workers = nworkers;
int i;
assert(MAX_MB_PLANE == 3);
@@ -809,6 +951,7 @@
worker->data2 = &lr_sync->lrworkerdata[i];
// Start loop restoration
+ worker->had_error = 0;
if (i == 0) {
winterface->execute(worker);
} else {
@@ -816,10 +959,7 @@
}
}
- // Wait till all rows are finished
- for (i = 1; i < num_workers; ++i) {
- winterface->sync(&workers[i]);
- }
+ sync_lr_workers(workers, cm, num_workers);
}
void av1_loop_restoration_filter_frame_mt(YV12_BUFFER_CONFIG *frame,
@@ -852,6 +992,7 @@
const AVxWorkerInterface *const winterface = aom_get_worker_interface();
for (int i = num_workers - 1; i >= 0; i--) {
AVxWorker *const worker = &workers[i];
+ worker->had_error = 0;
if (i == 0)
winterface->execute(worker);
else
@@ -863,16 +1004,26 @@
AV1_COMMON *const cm,
int num_workers) {
const AVxWorkerInterface *const winterface = aom_get_worker_interface();
- int had_error = 0;
+ int had_error = workers[0].had_error;
+ struct aom_internal_error_info error_info;
- // Wait for completion of Cdef frame.
- for (int i = num_workers - 1; i > 0; i--) {
+ // Read the error_info of main thread.
+ if (had_error) {
+ AVxWorker *const worker = &workers[0];
+ error_info = ((AV1CdefWorkerData *)worker->data2)->error_info;
+ }
+
+ // Wait till all rows are finished.
+ for (int i = num_workers - 1; i > 0; --i) {
AVxWorker *const worker = &workers[i];
- had_error |= !winterface->sync(worker);
+ if (!winterface->sync(worker)) {
+ had_error = 1;
+ error_info = ((AV1CdefWorkerData *)worker->data2)->error_info;
+ }
}
if (had_error)
- aom_internal_error(cm->error, AOM_CODEC_ERROR,
- "Failed to process cdef frame");
+ aom_internal_error(cm->error, error_info.error_code, "%s",
+ error_info.detail);
}
// Updates the row index of the next job to be processed.
@@ -888,14 +1039,15 @@
// Checks if a job is available. If job is available,
// populates next job information and returns 1, else returns 0.
static AOM_INLINE int get_cdef_row_next_job(AV1CdefSync *const cdef_sync,
- int *cur_fbr, const int nvfb) {
+ volatile int *cur_fbr,
+ const int nvfb) {
#if CONFIG_MULTITHREAD
pthread_mutex_lock(cdef_sync->mutex_);
#endif // CONFIG_MULTITHREAD
int do_next_row = 0;
// Populates information needed for current job and update the row
// index of the next row to be processed.
- if (cdef_sync->end_of_frame == 0) {
+ if (!cdef_sync->cdef_mt_exit && cdef_sync->end_of_frame == 0) {
do_next_row = 1;
*cur_fbr = cdef_sync->fbr;
update_cdef_row_next_job_info(cdef_sync, nvfb);
@@ -906,19 +1058,49 @@
return do_next_row;
}
+static void set_cdef_init_fb_row_done(AV1CdefSync *const cdef_sync, int nvfb) {
+ for (int fbr = 0; fbr < nvfb; fbr++) cdef_row_mt_sync_write(cdef_sync, fbr);
+}
+
// Hook function for each thread in CDEF multi-threading.
static int cdef_sb_row_worker_hook(void *arg1, void *arg2) {
AV1CdefSync *const cdef_sync = (AV1CdefSync *)arg1;
AV1CdefWorkerData *const cdef_worker = (AV1CdefWorkerData *)arg2;
AV1_COMMON *cm = cdef_worker->cm;
const int nvfb = (cm->mi_params.mi_rows + MI_SIZE_64X64 - 1) / MI_SIZE_64X64;
- int cur_fbr;
+
+#if CONFIG_MULTITHREAD
+ pthread_mutex_t *job_mutex_ = cdef_sync->mutex_;
+#endif
+ struct aom_internal_error_info *const error_info = &cdef_worker->error_info;
+
+ // The jmp_buf is valid only for the duration of the function that calls
+ // setjmp(). Therefore, this function must reset the 'setjmp' field to 0
+ // before it returns.
+ if (setjmp(error_info->jmp)) {
+ error_info->setjmp = 0;
+#if CONFIG_MULTITHREAD
+ pthread_mutex_lock(job_mutex_);
+ cdef_sync->cdef_mt_exit = true;
+ pthread_mutex_unlock(job_mutex_);
+#endif
+ // In case of cdef row-multithreading, the worker on a filter block row
+ // (fbr) waits for the line buffers (top and bottom) copy of the above row.
+ // Hence, in case a thread (main/worker) encounters an error before copying
+ // of the line buffers, update that line buffer copy is complete in order to
+ // avoid dependent workers waiting indefinitely.
+ set_cdef_init_fb_row_done(cdef_sync, nvfb);
+ return 0;
+ }
+ error_info->setjmp = 1;
+
+ volatile int cur_fbr;
const int num_planes = av1_num_planes(cm);
while (get_cdef_row_next_job(cdef_sync, &cur_fbr, nvfb)) {
MACROBLOCKD *xd = cdef_worker->xd;
av1_cdef_fb_row(cm, xd, cdef_worker->linebuf, cdef_worker->colbuf,
cdef_worker->srcbuf, cur_fbr,
- cdef_worker->cdef_init_fb_row_fn, cdef_sync);
+ cdef_worker->cdef_init_fb_row_fn, cdef_sync, error_info);
if (cdef_worker->do_extend_border) {
for (int plane = 0; plane < num_planes; ++plane) {
const YV12_BUFFER_CONFIG *ybf = &cm->cur_frame->buf;
@@ -932,6 +1114,7 @@
}
}
}
+ error_info->setjmp = 0;
return 1;
}
diff --git a/av1/common/thread_common.h b/av1/common/thread_common.h
index b6485c3..6d695e8 100644
--- a/av1/common/thread_common.h
+++ b/av1/common/thread_common.h
@@ -54,6 +54,10 @@
AV1LfMTInfo *job_queue;
int jobs_enqueued;
int jobs_dequeued;
+
+ // Initialized to false, set to true by the worker thread that encounters an
+ // error in order to abort the processing of other worker threads.
+ bool lf_mt_exit;
} AV1LfSync;
typedef struct AV1LrMTInfo {
@@ -71,6 +75,7 @@
void *rlbs;
void *lr_ctxt;
int do_extend_border;
+ struct aom_internal_error_info error_info;
} LRWorkerData;
// Looprestoration row synchronization
@@ -98,6 +103,9 @@
AV1LrMTInfo *job_queue;
int jobs_enqueued;
int jobs_dequeued;
+ // Initialized to false, set to true by the worker thread that encounters
+ // an error in order to abort the processing of other worker threads.
+ bool lr_mt_exit;
} AV1LrSync;
typedef struct AV1CdefWorker {
@@ -108,6 +116,7 @@
uint16_t *linebuf[MAX_MB_PLANE];
cdef_init_fb_row_t cdef_init_fb_row_fn;
int do_extend_border;
+ struct aom_internal_error_info error_info;
} AV1CdefWorkerData;
typedef struct AV1CdefRowSync {
@@ -132,6 +141,9 @@
int fbr;
// Column index in units of 64x64 block
int fbc;
+ // Initialized to false, set to true by the worker thread that encounters
+ // an error in order to abort the processing of other worker threads.
+ bool cdef_mt_exit;
} AV1CdefSync;
void av1_cdef_frame_mt(AV1_COMMON *const cm, MACROBLOCKD *const xd,
@@ -164,6 +176,9 @@
void av1_loop_filter_alloc(AV1LfSync *lf_sync, AV1_COMMON *cm, int rows,
int width, int num_workers);
+void av1_set_vert_loop_filter_done(AV1_COMMON *cm, AV1LfSync *lf_sync,
+ int num_mis_in_lpf_unit_height_log2);
+
void av1_loop_filter_frame_mt(YV12_BUFFER_CONFIG *frame, struct AV1Common *cm,
struct macroblockd *xd, int plane_start,
int plane_end, int partial_frame,
@@ -185,11 +200,11 @@
const YV12_BUFFER_CONFIG *const frame_buffer, AV1_COMMON *const cm,
struct macroblockd_plane *planes, MACROBLOCKD *xd, int mi_row, int plane,
int dir, int lpf_opt_level, AV1LfSync *const lf_sync,
+ struct aom_internal_error_info *error_info,
AV1_DEBLOCKING_PARAMETERS *params_buf, TX_SIZE *tx_buf, int mib_size_log2);
-static AOM_FORCE_INLINE bool skip_loop_filter_plane(const int planes_to_lf[3],
- int plane,
- int lpf_opt_level) {
+static AOM_FORCE_INLINE bool skip_loop_filter_plane(
+ const int planes_to_lf[MAX_MB_PLANE], int plane, int lpf_opt_level) {
// If LPF_PICK_METHOD is LPF_PICK_FROM_Q, we have the option to filter both
// chroma planes together
if (lpf_opt_level == 2) {
@@ -212,7 +227,7 @@
}
static AOM_INLINE void enqueue_lf_jobs(AV1LfSync *lf_sync, int start, int stop,
- const int planes_to_lf[3],
+ const int planes_to_lf[MAX_MB_PLANE],
int lpf_opt_level,
int num_mis_in_lpf_unit_height) {
int mi_row, plane, dir;
@@ -225,7 +240,7 @@
// partially reconstructed row by row.
for (dir = 0; dir < 2; ++dir) {
for (mi_row = start; mi_row < stop; mi_row += num_mis_in_lpf_unit_height) {
- for (plane = 0; plane < 3; ++plane) {
+ for (plane = 0; plane < MAX_MB_PLANE; ++plane) {
if (skip_loop_filter_plane(planes_to_lf, plane, lpf_opt_level)) {
continue;
}
@@ -242,9 +257,9 @@
}
static AOM_INLINE void loop_filter_frame_mt_init(
- AV1_COMMON *cm, int start_mi_row, int end_mi_row, const int planes_to_lf[3],
- int num_workers, AV1LfSync *lf_sync, int lpf_opt_level,
- int num_mis_in_lpf_unit_height_log2) {
+ AV1_COMMON *cm, int start_mi_row, int end_mi_row,
+ const int planes_to_lf[MAX_MB_PLANE], int num_workers, AV1LfSync *lf_sync,
+ int lpf_opt_level, int num_mis_in_lpf_unit_height_log2) {
// Number of superblock rows
const int sb_rows =
CEIL_POWER_OF_TWO(cm->mi_params.mi_rows, num_mis_in_lpf_unit_height_log2);
@@ -271,7 +286,7 @@
#if CONFIG_MULTITHREAD
pthread_mutex_lock(lf_sync->job_mutex);
- if (lf_sync->jobs_dequeued < lf_sync->jobs_enqueued) {
+ if (!lf_sync->lf_mt_exit && lf_sync->jobs_dequeued < lf_sync->jobs_enqueued) {
cur_job_info = lf_sync->job_queue + lf_sync->jobs_dequeued;
lf_sync->jobs_dequeued++;
}
diff --git a/av1/common/tile_common.c b/av1/common/tile_common.c
index 508fe30..b964f25 100644
--- a/av1/common/tile_common.c
+++ b/av1/common/tile_common.c
@@ -167,12 +167,12 @@
assert(tile->mi_col_end > tile->mi_col_start);
}
-int av1_get_sb_rows_in_tile(AV1_COMMON *cm, const TileInfo *tile) {
+int av1_get_sb_rows_in_tile(const AV1_COMMON *cm, const TileInfo *tile) {
return CEIL_POWER_OF_TWO(tile->mi_row_end - tile->mi_row_start,
cm->seq_params->mib_size_log2);
}
-int av1_get_sb_cols_in_tile(AV1_COMMON *cm, const TileInfo *tile) {
+int av1_get_sb_cols_in_tile(const AV1_COMMON *cm, const TileInfo *tile) {
return CEIL_POWER_OF_TWO(tile->mi_col_end - tile->mi_col_start,
cm->seq_params->mib_size_log2);
}
diff --git a/av1/common/tile_common.h b/av1/common/tile_common.h
index 8615a2c..5383ae9 100644
--- a/av1/common/tile_common.h
+++ b/av1/common/tile_common.h
@@ -40,8 +40,8 @@
void av1_tile_set_row(TileInfo *tile, const struct AV1Common *cm, int row);
void av1_tile_set_col(TileInfo *tile, const struct AV1Common *cm, int col);
-int av1_get_sb_rows_in_tile(struct AV1Common *cm, const TileInfo *tile);
-int av1_get_sb_cols_in_tile(struct AV1Common *cm, const TileInfo *tile);
+int av1_get_sb_rows_in_tile(const struct AV1Common *cm, const TileInfo *tile);
+int av1_get_sb_cols_in_tile(const struct AV1Common *cm, const TileInfo *tile);
// Return the pixel extents of the given tile
PixelRect av1_get_tile_rect(const TileInfo *tile_info,
diff --git a/av1/common/warped_motion.c b/av1/common/warped_motion.c
index 83f410e..4282b92 100644
--- a/av1/common/warped_motion.c
+++ b/av1/common/warped_motion.c
@@ -17,6 +17,7 @@
#include "config/av1_rtcd.h"
+#include "av1/common/av1_common_int.h"
#include "av1/common/warped_motion.h"
#include "av1/common/scale.h"
@@ -214,10 +215,42 @@
return 1;
}
+#ifndef NDEBUG
+// Check that the given warp model satisfies the relevant constraints for
+// its stated model type
+static void check_model_consistency(WarpedMotionParams *wm) {
+ switch (wm->wmtype) {
+ case IDENTITY:
+ assert(wm->wmmat[0] == 0);
+ assert(wm->wmmat[1] == 0);
+ AOM_FALLTHROUGH_INTENDED;
+ case TRANSLATION:
+ assert(wm->wmmat[2] == 1 << WARPEDMODEL_PREC_BITS);
+ assert(wm->wmmat[3] == 0);
+ AOM_FALLTHROUGH_INTENDED;
+ case ROTZOOM:
+ assert(wm->wmmat[4] == -wm->wmmat[3]);
+ assert(wm->wmmat[5] == wm->wmmat[2]);
+ AOM_FALLTHROUGH_INTENDED;
+ case AFFINE: break;
+ default: assert(0 && "Bad wmtype");
+ }
+}
+#endif // NDEBUG
+
// Returns 1 on success or 0 on an invalid affine set
int av1_get_shear_params(WarpedMotionParams *wm) {
+#ifndef NDEBUG
+ // Check that models have been constructed sensibly
+ // This is a good place to check, because this function does not need to
+ // be called until after model construction is complete, but must be called
+ // before the model can be used for prediction.
+ check_model_consistency(wm);
+#endif // NDEBUG
+
const int32_t *mat = wm->wmmat;
if (!is_affine_valid(wm)) return 0;
+
wm->alpha =
clamp(mat[2] - (1 << WARPEDMODEL_PREC_BITS), INT16_MIN, INT16_MAX);
wm->beta = clamp(mat[3], INT16_MIN, INT16_MAX);
@@ -247,17 +280,6 @@
}
#if CONFIG_AV1_HIGHBITDEPTH
-static INLINE int highbd_error_measure(int err, int bd) {
- const int b = bd - 8;
- const int bmask = (1 << b) - 1;
- const int v = (1 << b);
- err = abs(err);
- const int e1 = err >> b;
- const int e2 = err & bmask;
- return error_measure_lut[255 + e1] * (v - e2) +
- error_measure_lut[256 + e1] * e2;
-}
-
/* Note: For an explanation of the warp algorithm, and some notes on bit widths
for hardware implementations, see the comments above av1_warp_affine_c
*/
@@ -269,9 +291,7 @@
ConvolveParams *conv_params, int16_t alpha,
int16_t beta, int16_t gamma, int16_t delta) {
int32_t tmp[15 * 8];
- const int reduce_bits_horiz =
- conv_params->round_0 +
- AOMMAX(bd + FILTER_BITS - conv_params->round_0 - 14, 0);
+ const int reduce_bits_horiz = conv_params->round_0;
const int reduce_bits_vert = conv_params->is_compound
? conv_params->round_1
: 2 * FILTER_BITS - reduce_bits_horiz;
@@ -284,6 +304,10 @@
(void)max_bits_horiz;
assert(IMPLIES(conv_params->is_compound, conv_params->dst != NULL));
+ // Check that, even with 12-bit input, the intermediate values will fit
+ // into an unsigned 16-bit intermediate array.
+ assert(bd + FILTER_BITS + 2 - conv_params->round_0 <= 16);
+
for (int i = p_row; i < p_row + p_height; i += 8) {
for (int j = p_col; j < p_col + p_width; j += 8) {
// Calculate the center of this 8x8 block,
@@ -392,11 +416,6 @@
int p_col, int p_row, int p_width, int p_height,
int p_stride, int subsampling_x, int subsampling_y,
int bd, ConvolveParams *conv_params) {
- assert(wm->wmtype <= AFFINE);
- if (wm->wmtype == ROTZOOM) {
- wm->wmmat[5] = wm->wmmat[2];
- wm->wmmat[4] = -wm->wmmat[3];
- }
const int32_t *const mat = wm->wmmat;
const int16_t alpha = wm->alpha;
const int16_t beta = wm->beta;
@@ -408,46 +427,6 @@
subsampling_y, bd, conv_params, alpha, beta, gamma,
delta);
}
-
-int64_t av1_calc_highbd_frame_error(const uint16_t *const ref, int stride,
- const uint16_t *const dst, int p_width,
- int p_height, int p_stride, int bd) {
- int64_t sum_error = 0;
- for (int i = 0; i < p_height; ++i) {
- for (int j = 0; j < p_width; ++j) {
- sum_error +=
- highbd_error_measure(dst[j + i * p_stride] - ref[j + i * stride], bd);
- }
- }
- return sum_error;
-}
-
-static int64_t highbd_segmented_frame_error(
- const uint16_t *const ref, int stride, const uint16_t *const dst,
- int p_width, int p_height, int p_stride, int bd, uint8_t *segment_map,
- int segment_map_stride) {
- int patch_w, patch_h;
- const int error_bsize_w = AOMMIN(p_width, WARP_ERROR_BLOCK);
- const int error_bsize_h = AOMMIN(p_height, WARP_ERROR_BLOCK);
- int64_t sum_error = 0;
- for (int i = 0; i < p_height; i += WARP_ERROR_BLOCK) {
- for (int j = 0; j < p_width; j += WARP_ERROR_BLOCK) {
- int seg_x = j >> WARP_ERROR_BLOCK_LOG;
- int seg_y = i >> WARP_ERROR_BLOCK_LOG;
- // Only compute the error if this block contains inliers from the motion
- // model
- if (!segment_map[seg_y * segment_map_stride + seg_x]) continue;
-
- // avoid computing error into the frame padding
- patch_w = AOMMIN(error_bsize_w, p_width - j);
- patch_h = AOMMIN(error_bsize_h, p_height - i);
- sum_error += av1_calc_highbd_frame_error(ref + j + i * stride, stride,
- dst + j + i * p_stride, patch_w,
- patch_h, p_stride, bd);
- }
- }
- return sum_error;
-}
#endif // CONFIG_AV1_HIGHBITDEPTH
/* The warp filter for ROTZOOM and AFFINE models works as follows:
@@ -669,11 +648,6 @@
int height, int stride, uint8_t *pred, int p_col, int p_row,
int p_width, int p_height, int p_stride, int subsampling_x,
int subsampling_y, ConvolveParams *conv_params) {
- assert(wm->wmtype <= AFFINE);
- if (wm->wmtype == ROTZOOM) {
- wm->wmmat[5] = wm->wmmat[2];
- wm->wmmat[4] = -wm->wmmat[3];
- }
const int32_t *const mat = wm->wmmat;
const int16_t alpha = wm->alpha;
const int16_t beta = wm->beta;
@@ -684,79 +658,6 @@
alpha, beta, gamma, delta);
}
-int64_t av1_calc_frame_error_c(const uint8_t *const ref, int stride,
- const uint8_t *const dst, int p_width,
- int p_height, int p_stride) {
- int64_t sum_error = 0;
- for (int i = 0; i < p_height; ++i) {
- for (int j = 0; j < p_width; ++j) {
- sum_error +=
- (int64_t)error_measure(dst[j + i * p_stride] - ref[j + i * stride]);
- }
- }
- return sum_error;
-}
-
-static int64_t segmented_frame_error(const uint8_t *const ref, int stride,
- const uint8_t *const dst, int p_width,
- int p_height, int p_stride,
- uint8_t *segment_map,
- int segment_map_stride) {
- int patch_w, patch_h;
- const int error_bsize_w = AOMMIN(p_width, WARP_ERROR_BLOCK);
- const int error_bsize_h = AOMMIN(p_height, WARP_ERROR_BLOCK);
- int64_t sum_error = 0;
- for (int i = 0; i < p_height; i += WARP_ERROR_BLOCK) {
- for (int j = 0; j < p_width; j += WARP_ERROR_BLOCK) {
- int seg_x = j >> WARP_ERROR_BLOCK_LOG;
- int seg_y = i >> WARP_ERROR_BLOCK_LOG;
- // Only compute the error if this block contains inliers from the motion
- // model
- if (!segment_map[seg_y * segment_map_stride + seg_x]) continue;
-
- // avoid computing error into the frame padding
- patch_w = AOMMIN(error_bsize_w, p_width - j);
- patch_h = AOMMIN(error_bsize_h, p_height - i);
- sum_error += av1_calc_frame_error(ref + j + i * stride, stride,
- dst + j + i * p_stride, patch_w,
- patch_h, p_stride);
- }
- }
- return sum_error;
-}
-
-int64_t av1_frame_error(int use_hbd, int bd, const uint8_t *ref, int stride,
- uint8_t *dst, int p_width, int p_height, int p_stride) {
-#if CONFIG_AV1_HIGHBITDEPTH
- if (use_hbd) {
- return av1_calc_highbd_frame_error(CONVERT_TO_SHORTPTR(ref), stride,
- CONVERT_TO_SHORTPTR(dst), p_width,
- p_height, p_stride, bd);
- }
-#endif
- (void)use_hbd;
- (void)bd;
- return av1_calc_frame_error(ref, stride, dst, p_width, p_height, p_stride);
-}
-
-int64_t av1_segmented_frame_error(int use_hbd, int bd, const uint8_t *ref,
- int stride, uint8_t *dst, int p_width,
- int p_height, int p_stride,
- uint8_t *segment_map,
- int segment_map_stride) {
-#if CONFIG_AV1_HIGHBITDEPTH
- if (use_hbd) {
- return highbd_segmented_frame_error(
- CONVERT_TO_SHORTPTR(ref), stride, CONVERT_TO_SHORTPTR(dst), p_width,
- p_height, p_stride, bd, segment_map, segment_map_stride);
- }
-#endif
- (void)use_hbd;
- (void)bd;
- return segmented_frame_error(ref, stride, dst, p_width, p_height, p_stride,
- segment_map, segment_map_stride);
-}
-
void av1_warp_plane(WarpedMotionParams *wm, int use_hbd, int bd,
const uint8_t *ref, int width, int height, int stride,
uint8_t *pred, int p_col, int p_row, int p_width,
diff --git a/av1/common/warped_motion.h b/av1/common/warped_motion.h
index d6fe325..d772df8 100644
--- a/av1/common/warped_motion.h
+++ b/av1/common/warped_motion.h
@@ -38,76 +38,6 @@
DECLARE_ALIGNED(8, extern const int8_t,
av1_filter_8bit[WARPEDPIXEL_PREC_SHIFTS * 3 + 1][8]);
-/* clang-format off */
-static const int error_measure_lut[512] = {
- // pow 0.7
- 16384, 16339, 16294, 16249, 16204, 16158, 16113, 16068,
- 16022, 15977, 15932, 15886, 15840, 15795, 15749, 15703,
- 15657, 15612, 15566, 15520, 15474, 15427, 15381, 15335,
- 15289, 15242, 15196, 15149, 15103, 15056, 15010, 14963,
- 14916, 14869, 14822, 14775, 14728, 14681, 14634, 14587,
- 14539, 14492, 14445, 14397, 14350, 14302, 14254, 14206,
- 14159, 14111, 14063, 14015, 13967, 13918, 13870, 13822,
- 13773, 13725, 13676, 13628, 13579, 13530, 13481, 13432,
- 13383, 13334, 13285, 13236, 13187, 13137, 13088, 13038,
- 12988, 12939, 12889, 12839, 12789, 12739, 12689, 12639,
- 12588, 12538, 12487, 12437, 12386, 12335, 12285, 12234,
- 12183, 12132, 12080, 12029, 11978, 11926, 11875, 11823,
- 11771, 11719, 11667, 11615, 11563, 11511, 11458, 11406,
- 11353, 11301, 11248, 11195, 11142, 11089, 11036, 10982,
- 10929, 10875, 10822, 10768, 10714, 10660, 10606, 10552,
- 10497, 10443, 10388, 10333, 10279, 10224, 10168, 10113,
- 10058, 10002, 9947, 9891, 9835, 9779, 9723, 9666,
- 9610, 9553, 9497, 9440, 9383, 9326, 9268, 9211,
- 9153, 9095, 9037, 8979, 8921, 8862, 8804, 8745,
- 8686, 8627, 8568, 8508, 8449, 8389, 8329, 8269,
- 8208, 8148, 8087, 8026, 7965, 7903, 7842, 7780,
- 7718, 7656, 7593, 7531, 7468, 7405, 7341, 7278,
- 7214, 7150, 7086, 7021, 6956, 6891, 6826, 6760,
- 6695, 6628, 6562, 6495, 6428, 6361, 6293, 6225,
- 6157, 6089, 6020, 5950, 5881, 5811, 5741, 5670,
- 5599, 5527, 5456, 5383, 5311, 5237, 5164, 5090,
- 5015, 4941, 4865, 4789, 4713, 4636, 4558, 4480,
- 4401, 4322, 4242, 4162, 4080, 3998, 3916, 3832,
- 3748, 3663, 3577, 3490, 3402, 3314, 3224, 3133,
- 3041, 2948, 2854, 2758, 2661, 2562, 2461, 2359,
- 2255, 2148, 2040, 1929, 1815, 1698, 1577, 1452,
- 1323, 1187, 1045, 894, 731, 550, 339, 0,
- 339, 550, 731, 894, 1045, 1187, 1323, 1452,
- 1577, 1698, 1815, 1929, 2040, 2148, 2255, 2359,
- 2461, 2562, 2661, 2758, 2854, 2948, 3041, 3133,
- 3224, 3314, 3402, 3490, 3577, 3663, 3748, 3832,
- 3916, 3998, 4080, 4162, 4242, 4322, 4401, 4480,
- 4558, 4636, 4713, 4789, 4865, 4941, 5015, 5090,
- 5164, 5237, 5311, 5383, 5456, 5527, 5599, 5670,
- 5741, 5811, 5881, 5950, 6020, 6089, 6157, 6225,
- 6293, 6361, 6428, 6495, 6562, 6628, 6695, 6760,
- 6826, 6891, 6956, 7021, 7086, 7150, 7214, 7278,
- 7341, 7405, 7468, 7531, 7593, 7656, 7718, 7780,
- 7842, 7903, 7965, 8026, 8087, 8148, 8208, 8269,
- 8329, 8389, 8449, 8508, 8568, 8627, 8686, 8745,
- 8804, 8862, 8921, 8979, 9037, 9095, 9153, 9211,
- 9268, 9326, 9383, 9440, 9497, 9553, 9610, 9666,
- 9723, 9779, 9835, 9891, 9947, 10002, 10058, 10113,
- 10168, 10224, 10279, 10333, 10388, 10443, 10497, 10552,
- 10606, 10660, 10714, 10768, 10822, 10875, 10929, 10982,
- 11036, 11089, 11142, 11195, 11248, 11301, 11353, 11406,
- 11458, 11511, 11563, 11615, 11667, 11719, 11771, 11823,
- 11875, 11926, 11978, 12029, 12080, 12132, 12183, 12234,
- 12285, 12335, 12386, 12437, 12487, 12538, 12588, 12639,
- 12689, 12739, 12789, 12839, 12889, 12939, 12988, 13038,
- 13088, 13137, 13187, 13236, 13285, 13334, 13383, 13432,
- 13481, 13530, 13579, 13628, 13676, 13725, 13773, 13822,
- 13870, 13918, 13967, 14015, 14063, 14111, 14159, 14206,
- 14254, 14302, 14350, 14397, 14445, 14492, 14539, 14587,
- 14634, 14681, 14728, 14775, 14822, 14869, 14916, 14963,
- 15010, 15056, 15103, 15149, 15196, 15242, 15289, 15335,
- 15381, 15427, 15474, 15520, 15566, 15612, 15657, 15703,
- 15749, 15795, 15840, 15886, 15932, 15977, 16022, 16068,
- 16113, 16158, 16204, 16249, 16294, 16339, 16384, 16384,
-};
-/* clang-format on */
-
static const uint8_t warp_pad_left[14][16] = {
{ 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 },
{ 2, 2, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 },
@@ -142,24 +72,6 @@
{ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }
};
-static INLINE int error_measure(int err) {
- return error_measure_lut[255 + err];
-}
-
-// Returns the error between the frame described by 'ref' and the frame
-// described by 'dst'.
-int64_t av1_frame_error(int use_hbd, int bd, const uint8_t *ref, int stride,
- uint8_t *dst, int p_width, int p_height, int p_stride);
-
-int64_t av1_segmented_frame_error(int use_hbd, int bd, const uint8_t *ref,
- int stride, uint8_t *dst, int p_width,
- int p_height, int p_stride,
- uint8_t *segment_map, int segment_map_stride);
-
-int64_t av1_calc_highbd_frame_error(const uint16_t *const ref, int stride,
- const uint16_t *const dst, int p_width,
- int p_height, int p_stride, int bd);
-
void highbd_warp_plane(WarpedMotionParams *wm, const uint16_t *const ref,
int width, int height, int stride, uint16_t *const pred,
int p_col, int p_row, int p_width, int p_height,
diff --git a/av1/common/x86/av1_convolve_scale_sse4.c b/av1/common/x86/av1_convolve_scale_sse4.c
index 67b28bc..8e293b5 100644
--- a/av1/common/x86/av1_convolve_scale_sse4.c
+++ b/av1/common/x86/av1_convolve_scale_sse4.c
@@ -12,7 +12,7 @@
#include <assert.h>
#include <smmintrin.h>
-#include "config/aom_dsp_rtcd.h"
+#include "config/av1_rtcd.h"
#include "aom_dsp/aom_dsp_common.h"
#include "aom_dsp/aom_filter.h"
diff --git a/av1/common/x86/highbd_convolve_2d_avx2.c b/av1/common/x86/highbd_convolve_2d_avx2.c
index de850ee..d65318c 100644
--- a/av1/common/x86/highbd_convolve_2d_avx2.c
+++ b/av1/common/x86/highbd_convolve_2d_avx2.c
@@ -12,7 +12,7 @@
#include <immintrin.h>
#include <assert.h>
-#include "config/aom_dsp_rtcd.h"
+#include "config/av1_rtcd.h"
#include "aom_dsp/x86/convolve_avx2.h"
#include "aom_dsp/x86/synonyms.h"
diff --git a/av1/common/x86/highbd_convolve_2d_sse4.c b/av1/common/x86/highbd_convolve_2d_sse4.c
index b2c39cd..89d7199 100644
--- a/av1/common/x86/highbd_convolve_2d_sse4.c
+++ b/av1/common/x86/highbd_convolve_2d_sse4.c
@@ -13,7 +13,7 @@
#include <smmintrin.h>
#include <assert.h>
-#include "config/aom_dsp_rtcd.h"
+#include "config/av1_rtcd.h"
#include "aom_dsp/aom_dsp_common.h"
#include "aom_dsp/aom_filter.h"
diff --git a/av1/common/x86/highbd_convolve_2d_ssse3.c b/av1/common/x86/highbd_convolve_2d_ssse3.c
index 8324044..88974ba 100644
--- a/av1/common/x86/highbd_convolve_2d_ssse3.c
+++ b/av1/common/x86/highbd_convolve_2d_ssse3.c
@@ -12,7 +12,7 @@
#include <tmmintrin.h>
#include <assert.h>
-#include "config/aom_dsp_rtcd.h"
+#include "config/av1_rtcd.h"
#include "aom_dsp/aom_dsp_common.h"
#include "aom_dsp/aom_filter.h"
diff --git a/av1/common/x86/highbd_jnt_convolve_avx2.c b/av1/common/x86/highbd_jnt_convolve_avx2.c
index da52ecd..6dcac10 100644
--- a/av1/common/x86/highbd_jnt_convolve_avx2.c
+++ b/av1/common/x86/highbd_jnt_convolve_avx2.c
@@ -12,7 +12,7 @@
#include <immintrin.h>
#include <assert.h>
-#include "config/aom_dsp_rtcd.h"
+#include "config/av1_rtcd.h"
#include "aom_dsp/x86/convolve_avx2.h"
#include "aom_dsp/x86/convolve_common_intrin.h"
diff --git a/av1/common/x86/highbd_jnt_convolve_sse4.c b/av1/common/x86/highbd_jnt_convolve_sse4.c
index af45764..5a7fc53 100644
--- a/av1/common/x86/highbd_jnt_convolve_sse4.c
+++ b/av1/common/x86/highbd_jnt_convolve_sse4.c
@@ -12,7 +12,7 @@
#include <smmintrin.h>
#include <assert.h>
-#include "config/aom_dsp_rtcd.h"
+#include "config/av1_rtcd.h"
#include "aom_dsp/x86/convolve_sse2.h"
#include "aom_dsp/x86/convolve_sse4_1.h"
diff --git a/av1/common/x86/highbd_warp_affine_avx2.c b/av1/common/x86/highbd_warp_affine_avx2.c
index 7f6aceb..75108b4 100644
--- a/av1/common/x86/highbd_warp_affine_avx2.c
+++ b/av1/common/x86/highbd_warp_affine_avx2.c
@@ -22,9 +22,7 @@
ConvolveParams *conv_params, int16_t alpha,
int16_t beta, int16_t gamma, int16_t delta) {
__m256i tmp[15];
- const int reduce_bits_horiz =
- conv_params->round_0 +
- AOMMAX(bd + FILTER_BITS - conv_params->round_0 - 14, 0);
+ const int reduce_bits_horiz = conv_params->round_0;
const int reduce_bits_vert = conv_params->is_compound
? conv_params->round_1
: 2 * FILTER_BITS - reduce_bits_horiz;
@@ -37,6 +35,10 @@
(void)max_bits_horiz;
assert(IMPLIES(conv_params->is_compound, conv_params->dst != NULL));
+ // Check that, even with 12-bit input, the intermediate values will fit
+ // into an unsigned 16-bit intermediate array.
+ assert(bd + FILTER_BITS + 2 - conv_params->round_0 <= 16);
+
const __m256i clip_pixel =
_mm256_set1_epi16(bd == 10 ? 1023 : (bd == 12 ? 4095 : 255));
const __m128i reduce_bits_vert_shift = _mm_cvtsi32_si128(reduce_bits_vert);
diff --git a/av1/common/x86/highbd_warp_plane_sse4.c b/av1/common/x86/highbd_warp_plane_sse4.c
index 9df0ddc..96fb4cf 100644
--- a/av1/common/x86/highbd_warp_plane_sse4.c
+++ b/av1/common/x86/highbd_warp_plane_sse4.c
@@ -302,9 +302,7 @@
int16_t beta, int16_t gamma, int16_t delta) {
__m128i tmp[15];
int i, j, k;
- const int reduce_bits_horiz =
- conv_params->round_0 +
- AOMMAX(bd + FILTER_BITS - conv_params->round_0 - 14, 0);
+ const int reduce_bits_horiz = conv_params->round_0;
const int reduce_bits_vert = conv_params->is_compound
? conv_params->round_1
: 2 * FILTER_BITS - reduce_bits_horiz;
@@ -313,6 +311,10 @@
assert(!(bd == 12 && reduce_bits_horiz < 5));
assert(IMPLIES(conv_params->do_average, conv_params->is_compound));
+ // Check that, even with 12-bit input, the intermediate values will fit
+ // into an unsigned 16-bit intermediate array.
+ assert(bd + FILTER_BITS + 2 - conv_params->round_0 <= 16);
+
const int offset_bits_vert = bd + 2 * FILTER_BITS - reduce_bits_horiz;
const __m128i clip_pixel =
_mm_set1_epi16(bd == 10 ? 1023 : (bd == 12 ? 4095 : 255));
diff --git a/av1/common/x86/highbd_wiener_convolve_avx2.c b/av1/common/x86/highbd_wiener_convolve_avx2.c
index 0c8a850..562c623 100644
--- a/av1/common/x86/highbd_wiener_convolve_avx2.c
+++ b/av1/common/x86/highbd_wiener_convolve_avx2.c
@@ -12,7 +12,7 @@
#include <immintrin.h>
#include <assert.h>
-#include "config/aom_dsp_rtcd.h"
+#include "config/av1_rtcd.h"
#include "av1/common/convolve.h"
#include "aom_dsp/aom_dsp_common.h"
@@ -29,7 +29,7 @@
const uint8_t *src8, ptrdiff_t src_stride, uint8_t *dst8,
ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4,
const int16_t *filter_y, int y_step_q4, int w, int h,
- const ConvolveParams *conv_params, int bd) {
+ const WienerConvolveParams *conv_params, int bd) {
assert(x_step_q4 == 16 && y_step_q4 == 16);
assert(!(w & 7));
assert(bd + FILTER_BITS - conv_params->round_0 + 2 <= 16);
diff --git a/av1/common/x86/highbd_wiener_convolve_ssse3.c b/av1/common/x86/highbd_wiener_convolve_ssse3.c
index 818b109..cab37fa 100644
--- a/av1/common/x86/highbd_wiener_convolve_ssse3.c
+++ b/av1/common/x86/highbd_wiener_convolve_ssse3.c
@@ -12,7 +12,7 @@
#include <tmmintrin.h>
#include <assert.h>
-#include "config/aom_dsp_rtcd.h"
+#include "config/av1_rtcd.h"
#include "av1/common/convolve.h"
#include "aom_dsp/aom_dsp_common.h"
@@ -22,7 +22,7 @@
const uint8_t *src8, ptrdiff_t src_stride, uint8_t *dst8,
ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4,
const int16_t *filter_y, int y_step_q4, int w, int h,
- const ConvolveParams *conv_params, int bd) {
+ const WienerConvolveParams *conv_params, int bd) {
assert(x_step_q4 == 16 && y_step_q4 == 16);
assert(!(w & 7));
assert(bd + FILTER_BITS - conv_params->round_0 + 2 <= 16);
diff --git a/av1/common/x86/intra_edge_sse4.c b/av1/common/x86/intra_edge_sse4.c
index f025f79..3eee46f 100644
--- a/av1/common/x86/intra_edge_sse4.c
+++ b/av1/common/x86/intra_edge_sse4.c
@@ -113,7 +113,69 @@
}
}
-void av1_filter_intra_edge_high_sse4_1(uint16_t *p, int sz, int strength) {
+void av1_upsample_intra_edge_sse4_1(uint8_t *p, int sz) {
+ // interpolate half-sample positions
+ assert(sz <= 24);
+
+ DECLARE_ALIGNED(16, static const int8_t, kernel[1][16]) = {
+ { -1, 9, 9, -1, -1, 9, 9, -1, -1, 9, 9, -1, -1, 9, 9, -1 }
+ };
+
+ DECLARE_ALIGNED(
+ 16, static const int8_t,
+ v_const[2][16]) = { { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 },
+ { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } };
+
+ // Extend first/last samples (upper-left p[-1], last p[sz-1])
+ // to support 4-tap filter
+ p[-2] = p[-1];
+ p[sz] = p[sz - 1];
+
+ uint8_t *in = &p[-2];
+ uint8_t *out = &p[-2];
+
+ int n = sz + 1; // Input length including upper-left sample
+
+ __m128i in0 = _mm_lddqu_si128((__m128i *)&in[0]);
+ __m128i in16 = _mm_lddqu_si128((__m128i *)&in[16]);
+
+ __m128i coef0 = _mm_lddqu_si128((__m128i *)kernel[0]);
+ __m128i shuf0 = _mm_lddqu_si128((__m128i *)v_const[0]);
+ __m128i shuf1 = _mm_lddqu_si128((__m128i *)v_const[1]);
+
+ while (n > 0) {
+ __m128i in8 = _mm_alignr_epi8(in16, in0, 8);
+ __m128i d0 = _mm_shuffle_epi8(in0, shuf0);
+ __m128i d1 = _mm_shuffle_epi8(in0, shuf1);
+ __m128i d2 = _mm_shuffle_epi8(in8, shuf0);
+ __m128i d3 = _mm_shuffle_epi8(in8, shuf1);
+ d0 = _mm_maddubs_epi16(d0, coef0);
+ d1 = _mm_maddubs_epi16(d1, coef0);
+ d2 = _mm_maddubs_epi16(d2, coef0);
+ d3 = _mm_maddubs_epi16(d3, coef0);
+ d0 = _mm_hadd_epi16(d0, d1);
+ d2 = _mm_hadd_epi16(d2, d3);
+ __m128i eight = _mm_set1_epi16(8);
+ d0 = _mm_add_epi16(d0, eight);
+ d2 = _mm_add_epi16(d2, eight);
+ d0 = _mm_srai_epi16(d0, 4);
+ d2 = _mm_srai_epi16(d2, 4);
+ d0 = _mm_packus_epi16(d0, d2);
+ __m128i in1 = _mm_alignr_epi8(in16, in0, 1);
+ __m128i out0 = _mm_unpacklo_epi8(in1, d0);
+ __m128i out1 = _mm_unpackhi_epi8(in1, d0);
+ _mm_storeu_si128((__m128i *)&out[0], out0);
+ _mm_storeu_si128((__m128i *)&out[16], out1);
+ in0 = in16;
+ in16 = _mm_setzero_si128();
+ out += 32;
+ n -= 16;
+ }
+}
+
+#if CONFIG_AV1_HIGHBITDEPTH
+
+void av1_highbd_filter_intra_edge_sse4_1(uint16_t *p, int sz, int strength) {
if (!strength) return;
DECLARE_ALIGNED(16, static const int16_t, kern[3][8]) = {
@@ -204,67 +266,7 @@
}
}
-void av1_upsample_intra_edge_sse4_1(uint8_t *p, int sz) {
- // interpolate half-sample positions
- assert(sz <= 24);
-
- DECLARE_ALIGNED(16, static const int8_t, kernel[1][16]) = {
- { -1, 9, 9, -1, -1, 9, 9, -1, -1, 9, 9, -1, -1, 9, 9, -1 }
- };
-
- DECLARE_ALIGNED(
- 16, static const int8_t,
- v_const[2][16]) = { { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 },
- { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } };
-
- // Extend first/last samples (upper-left p[-1], last p[sz-1])
- // to support 4-tap filter
- p[-2] = p[-1];
- p[sz] = p[sz - 1];
-
- uint8_t *in = &p[-2];
- uint8_t *out = &p[-2];
-
- int n = sz + 1; // Input length including upper-left sample
-
- __m128i in0 = _mm_lddqu_si128((__m128i *)&in[0]);
- __m128i in16 = _mm_lddqu_si128((__m128i *)&in[16]);
-
- __m128i coef0 = _mm_lddqu_si128((__m128i *)kernel[0]);
- __m128i shuf0 = _mm_lddqu_si128((__m128i *)v_const[0]);
- __m128i shuf1 = _mm_lddqu_si128((__m128i *)v_const[1]);
-
- while (n > 0) {
- __m128i in8 = _mm_alignr_epi8(in16, in0, 8);
- __m128i d0 = _mm_shuffle_epi8(in0, shuf0);
- __m128i d1 = _mm_shuffle_epi8(in0, shuf1);
- __m128i d2 = _mm_shuffle_epi8(in8, shuf0);
- __m128i d3 = _mm_shuffle_epi8(in8, shuf1);
- d0 = _mm_maddubs_epi16(d0, coef0);
- d1 = _mm_maddubs_epi16(d1, coef0);
- d2 = _mm_maddubs_epi16(d2, coef0);
- d3 = _mm_maddubs_epi16(d3, coef0);
- d0 = _mm_hadd_epi16(d0, d1);
- d2 = _mm_hadd_epi16(d2, d3);
- __m128i eight = _mm_set1_epi16(8);
- d0 = _mm_add_epi16(d0, eight);
- d2 = _mm_add_epi16(d2, eight);
- d0 = _mm_srai_epi16(d0, 4);
- d2 = _mm_srai_epi16(d2, 4);
- d0 = _mm_packus_epi16(d0, d2);
- __m128i in1 = _mm_alignr_epi8(in16, in0, 1);
- __m128i out0 = _mm_unpacklo_epi8(in1, d0);
- __m128i out1 = _mm_unpackhi_epi8(in1, d0);
- _mm_storeu_si128((__m128i *)&out[0], out0);
- _mm_storeu_si128((__m128i *)&out[16], out1);
- in0 = in16;
- in16 = _mm_setzero_si128();
- out += 32;
- n -= 16;
- }
-}
-
-void av1_upsample_intra_edge_high_sse4_1(uint16_t *p, int sz, int bd) {
+void av1_highbd_upsample_intra_edge_sse4_1(uint16_t *p, int sz, int bd) {
// interpolate half-sample positions
assert(sz <= 24);
@@ -316,3 +318,5 @@
n -= 8;
}
}
+
+#endif // CONFIG_AV1_HIGHBITDEPTH
diff --git a/av1/common/x86/jnt_convolve_avx2.c b/av1/common/x86/jnt_convolve_avx2.c
index ae8f88e..9f82ed2 100644
--- a/av1/common/x86/jnt_convolve_avx2.c
+++ b/av1/common/x86/jnt_convolve_avx2.c
@@ -12,7 +12,7 @@
#include <emmintrin.h>
#include <immintrin.h>
-#include "config/aom_dsp_rtcd.h"
+#include "config/av1_rtcd.h"
#include "aom_dsp/aom_dsp_common.h"
#include "aom_dsp/aom_filter.h"
diff --git a/av1/common/x86/jnt_convolve_sse2.c b/av1/common/x86/jnt_convolve_sse2.c
index ab937f9..8c5d9918f 100644
--- a/av1/common/x86/jnt_convolve_sse2.c
+++ b/av1/common/x86/jnt_convolve_sse2.c
@@ -11,7 +11,7 @@
#include <emmintrin.h>
-#include "config/aom_dsp_rtcd.h"
+#include "config/av1_rtcd.h"
#include "aom_dsp/aom_filter.h"
#include "aom_dsp/x86/convolve_sse2.h"
diff --git a/av1/common/x86/jnt_convolve_ssse3.c b/av1/common/x86/jnt_convolve_ssse3.c
index d0cf763..f6bf678 100644
--- a/av1/common/x86/jnt_convolve_ssse3.c
+++ b/av1/common/x86/jnt_convolve_ssse3.c
@@ -11,7 +11,7 @@
#include <tmmintrin.h>
-#include "config/aom_dsp_rtcd.h"
+#include "config/av1_rtcd.h"
#include "aom_dsp/aom_filter.h"
#include "aom_dsp/x86/convolve_sse2.h"
diff --git a/av1/common/x86/reconinter_sse4.c b/av1/common/x86/reconinter_sse4.c
index 95814b4..eb4a4d1 100644
--- a/av1/common/x86/reconinter_sse4.c
+++ b/av1/common/x86/reconinter_sse4.c
@@ -15,6 +15,7 @@
#include "aom/aom_integer.h"
#include "aom_dsp/blend.h"
#include "av1/common/blockd.h"
+#include "config/av1_rtcd.h"
static INLINE __m128i calc_mask(const __m128i mask_base, const __m128i s0,
const __m128i s1) {
diff --git a/av1/common/x86/selfguided_avx2.c b/av1/common/x86/selfguided_avx2.c
index 4ab35e8..5ab6c46 100644
--- a/av1/common/x86/selfguided_avx2.c
+++ b/av1/common/x86/selfguided_avx2.c
@@ -630,18 +630,17 @@
return 0;
}
-void av1_apply_selfguided_restoration_avx2(const uint8_t *dat8, int width,
- int height, int stride, int eps,
- const int *xqd, uint8_t *dst8,
- int dst_stride, int32_t *tmpbuf,
- int bit_depth, int highbd) {
+int av1_apply_selfguided_restoration_avx2(const uint8_t *dat8, int width,
+ int height, int stride, int eps,
+ const int *xqd, uint8_t *dst8,
+ int dst_stride, int32_t *tmpbuf,
+ int bit_depth, int highbd) {
int32_t *flt0 = tmpbuf;
int32_t *flt1 = flt0 + RESTORATION_UNITPELS_MAX;
assert(width * height <= RESTORATION_UNITPELS_MAX);
const int ret = av1_selfguided_restoration_avx2(
dat8, width, height, stride, flt0, flt1, width, eps, bit_depth, highbd);
- (void)ret;
- assert(!ret);
+ if (ret != 0) return ret;
const sgr_params_type *const params = &av1_sgr_params[eps];
int xq[2];
av1_decode_xq(xqd, xq, params);
@@ -721,4 +720,5 @@
}
}
}
+ return 0;
}
diff --git a/av1/common/x86/selfguided_sse4.c b/av1/common/x86/selfguided_sse4.c
index 948bbfb..ac850f5 100644
--- a/av1/common/x86/selfguided_sse4.c
+++ b/av1/common/x86/selfguided_sse4.c
@@ -582,18 +582,17 @@
return 0;
}
-void av1_apply_selfguided_restoration_sse4_1(const uint8_t *dat8, int width,
- int height, int stride, int eps,
- const int *xqd, uint8_t *dst8,
- int dst_stride, int32_t *tmpbuf,
- int bit_depth, int highbd) {
+int av1_apply_selfguided_restoration_sse4_1(const uint8_t *dat8, int width,
+ int height, int stride, int eps,
+ const int *xqd, uint8_t *dst8,
+ int dst_stride, int32_t *tmpbuf,
+ int bit_depth, int highbd) {
int32_t *flt0 = tmpbuf;
int32_t *flt1 = flt0 + RESTORATION_UNITPELS_MAX;
assert(width * height <= RESTORATION_UNITPELS_MAX);
const int ret = av1_selfguided_restoration_sse4_1(
dat8, width, height, stride, flt0, flt1, width, eps, bit_depth, highbd);
- (void)ret;
- assert(!ret);
+ if (ret != 0) return ret;
const sgr_params_type *const params = &av1_sgr_params[eps];
int xq[2];
av1_decode_xq(xqd, xq, params);
@@ -659,4 +658,5 @@
}
}
}
+ return 0;
}
diff --git a/av1/common/x86/warp_plane_avx2.c b/av1/common/x86/warp_plane_avx2.c
index ceb836e..663b8cd 100644
--- a/av1/common/x86/warp_plane_avx2.c
+++ b/av1/common/x86/warp_plane_avx2.c
@@ -1022,116 +1022,6 @@
shuffle_src);
}
-int64_t av1_calc_frame_error_avx2(const uint8_t *const ref, int ref_stride,
- const uint8_t *const dst, int p_width,
- int p_height, int dst_stride) {
- int64_t sum_error = 0;
- int i, j;
- __m256i row_error, col_error;
- __m256i zero = _mm256_setzero_si256();
- __m256i dup_255 = _mm256_set1_epi16(255);
- col_error = zero;
-
- for (i = 0; i < (p_height / 4); i++) {
- row_error = _mm256_setzero_si256();
- for (j = 0; j < (p_width / 16); j++) {
- __m256i ref_1_16 = _mm256_cvtepu8_epi16(_mm_load_si128(
- (__m128i *)(ref + (j * 16) + (((i * 4) + 0) * ref_stride))));
- __m256i dst_1_16 = _mm256_cvtepu8_epi16(_mm_load_si128(
- (__m128i *)(dst + (j * 16) + (((i * 4) + 0) * dst_stride))));
- __m256i ref_2_16 = _mm256_cvtepu8_epi16(_mm_load_si128(
- (__m128i *)(ref + (j * 16) + (((i * 4) + 1) * ref_stride))));
- __m256i dst_2_16 = _mm256_cvtepu8_epi16(_mm_load_si128(
- (__m128i *)(dst + (j * 16) + (((i * 4) + 1) * dst_stride))));
- __m256i ref_3_16 = _mm256_cvtepu8_epi16(_mm_load_si128(
- (__m128i *)(ref + (j * 16) + (((i * 4) + 2) * ref_stride))));
- __m256i dst_3_16 = _mm256_cvtepu8_epi16(_mm_load_si128(
- (__m128i *)(dst + (j * 16) + (((i * 4) + 2) * dst_stride))));
- __m256i ref_4_16 = _mm256_cvtepu8_epi16(_mm_load_si128(
- (__m128i *)(ref + (j * 16) + (((i * 4) + 3) * ref_stride))));
- __m256i dst_4_16 = _mm256_cvtepu8_epi16(_mm_load_si128(
- (__m128i *)(dst + (j * 16) + (((i * 4) + 3) * dst_stride))));
-
- __m256i diff_1 =
- _mm256_add_epi16(_mm256_sub_epi16(dst_1_16, ref_1_16), dup_255);
- __m256i diff_2 =
- _mm256_add_epi16(_mm256_sub_epi16(dst_2_16, ref_2_16), dup_255);
- __m256i diff_3 =
- _mm256_add_epi16(_mm256_sub_epi16(dst_3_16, ref_3_16), dup_255);
- __m256i diff_4 =
- _mm256_add_epi16(_mm256_sub_epi16(dst_4_16, ref_4_16), dup_255);
-
- __m256i diff_1_lo = _mm256_unpacklo_epi16(diff_1, zero);
- __m256i diff_1_hi = _mm256_unpackhi_epi16(diff_1, zero);
- __m256i diff_2_lo = _mm256_unpacklo_epi16(diff_2, zero);
- __m256i diff_2_hi = _mm256_unpackhi_epi16(diff_2, zero);
- __m256i diff_3_lo = _mm256_unpacklo_epi16(diff_3, zero);
- __m256i diff_3_hi = _mm256_unpackhi_epi16(diff_3, zero);
- __m256i diff_4_lo = _mm256_unpacklo_epi16(diff_4, zero);
- __m256i diff_4_hi = _mm256_unpackhi_epi16(diff_4, zero);
-
- __m256i error_1_lo =
- _mm256_i32gather_epi32(error_measure_lut, diff_1_lo, 4);
- __m256i error_1_hi =
- _mm256_i32gather_epi32(error_measure_lut, diff_1_hi, 4);
- __m256i error_2_lo =
- _mm256_i32gather_epi32(error_measure_lut, diff_2_lo, 4);
- __m256i error_2_hi =
- _mm256_i32gather_epi32(error_measure_lut, diff_2_hi, 4);
- __m256i error_3_lo =
- _mm256_i32gather_epi32(error_measure_lut, diff_3_lo, 4);
- __m256i error_3_hi =
- _mm256_i32gather_epi32(error_measure_lut, diff_3_hi, 4);
- __m256i error_4_lo =
- _mm256_i32gather_epi32(error_measure_lut, diff_4_lo, 4);
- __m256i error_4_hi =
- _mm256_i32gather_epi32(error_measure_lut, diff_4_hi, 4);
-
- __m256i error_1 = _mm256_add_epi32(error_1_lo, error_1_hi);
- __m256i error_2 = _mm256_add_epi32(error_2_lo, error_2_hi);
- __m256i error_3 = _mm256_add_epi32(error_3_lo, error_3_hi);
- __m256i error_4 = _mm256_add_epi32(error_4_lo, error_4_hi);
-
- __m256i error_1_2 = _mm256_add_epi32(error_1, error_2);
- __m256i error_3_4 = _mm256_add_epi32(error_3, error_4);
-
- __m256i error_1_2_3_4 = _mm256_add_epi32(error_1_2, error_3_4);
- row_error = _mm256_add_epi32(row_error, error_1_2_3_4);
- }
- __m256i col_error_lo = _mm256_unpacklo_epi32(row_error, zero);
- __m256i col_error_hi = _mm256_unpackhi_epi32(row_error, zero);
- __m256i col_error_temp = _mm256_add_epi64(col_error_lo, col_error_hi);
- col_error = _mm256_add_epi64(col_error, col_error_temp);
- // Error summation for remaining width, which is not multiple of 16
- if (p_width & 0xf) {
- for (int k = 0; k < 4; ++k) {
- for (int l = j * 16; l < p_width; ++l) {
- sum_error +=
- (int64_t)error_measure(dst[l + ((i * 4) + k) * dst_stride] -
- ref[l + ((i * 4) + k) * ref_stride]);
- }
- }
- }
- }
- __m128i sum_error_q_0 = _mm256_castsi256_si128(col_error);
- __m128i sum_error_q_1 = _mm256_extracti128_si256(col_error, 1);
- sum_error_q_0 = _mm_add_epi64(sum_error_q_0, sum_error_q_1);
- int64_t sum_error_d_0, sum_error_d_1;
- xx_storel_64(&sum_error_d_0, sum_error_q_0);
- xx_storel_64(&sum_error_d_1, _mm_srli_si128(sum_error_q_0, 8));
- sum_error = (sum_error + sum_error_d_0 + sum_error_d_1);
- // Error summation for remaining height, which is not multiple of 4
- if (p_height & 0x3) {
- for (int k = i * 4; k < p_height; ++k) {
- for (int l = 0; l < p_width; ++l) {
- sum_error += (int64_t)error_measure(dst[l + k * dst_stride] -
- ref[l + k * ref_stride]);
- }
- }
- }
- return sum_error;
-}
-
void av1_warp_affine_avx2(const int32_t *mat, const uint8_t *ref, int width,
int height, int stride, uint8_t *pred, int p_col,
int p_row, int p_width, int p_height, int p_stride,
diff --git a/av1/common/x86/warp_plane_sse2.c b/av1/common/x86/warp_plane_sse2.c
deleted file mode 100644
index f8fe578..0000000
--- a/av1/common/x86/warp_plane_sse2.c
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Copyright (c) 2019, Alliance for Open Media. All rights reserved
- *
- * This source code is subject to the terms of the BSD 2 Clause License and
- * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
- * was not distributed with this source code in the LICENSE file, you can
- * obtain it at www.aomedia.org/license/software. If the Alliance for Open
- * Media Patent License 1.0 was not distributed with this source code in the
- * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
- */
-
-#include <emmintrin.h>
-
-#include "aom_dsp/x86/synonyms.h"
-#include "av1/common/warped_motion.h"
-#include "config/av1_rtcd.h"
-
-int64_t av1_calc_frame_error_sse2(const uint8_t *const ref, int ref_stride,
- const uint8_t *const dst, int p_width,
- int p_height, int dst_stride) {
- int64_t sum_error = 0;
- int i, j;
- __m128i row_error, col_error;
- __m128i zero = _mm_setzero_si128();
- __m128i dup_255 = _mm_set1_epi16(255);
- col_error = zero;
- for (i = 0; i < (p_height); i++) {
- row_error = zero;
- for (j = 0; j < (p_width / 16); j++) {
- __m128i ref_8 =
- _mm_load_si128((__m128i *)(ref + (j * 16) + (i * ref_stride)));
- __m128i dst_8 =
- _mm_load_si128((__m128i *)(dst + (j * 16) + (i * dst_stride)));
- __m128i ref_16_lo = _mm_unpacklo_epi8(ref_8, zero);
- __m128i ref_16_hi = _mm_unpackhi_epi8(ref_8, zero);
- __m128i dst_16_lo = _mm_unpacklo_epi8(dst_8, zero);
- __m128i dst_16_hi = _mm_unpackhi_epi8(dst_8, zero);
-
- __m128i diff_1 =
- _mm_add_epi16(_mm_sub_epi16(dst_16_lo, ref_16_lo), dup_255);
- __m128i diff_2 =
- _mm_add_epi16(_mm_sub_epi16(dst_16_hi, ref_16_hi), dup_255);
-
- __m128i error_1_lo =
- _mm_set_epi32(error_measure_lut[_mm_extract_epi16(diff_1, 3)],
- error_measure_lut[_mm_extract_epi16(diff_1, 2)],
- error_measure_lut[_mm_extract_epi16(diff_1, 1)],
- error_measure_lut[_mm_extract_epi16(diff_1, 0)]);
- __m128i error_1_hi =
- _mm_set_epi32(error_measure_lut[_mm_extract_epi16(diff_1, 7)],
- error_measure_lut[_mm_extract_epi16(diff_1, 6)],
- error_measure_lut[_mm_extract_epi16(diff_1, 5)],
- error_measure_lut[_mm_extract_epi16(diff_1, 4)]);
- __m128i error_2_lo =
- _mm_set_epi32(error_measure_lut[_mm_extract_epi16(diff_2, 3)],
- error_measure_lut[_mm_extract_epi16(diff_2, 2)],
- error_measure_lut[_mm_extract_epi16(diff_2, 1)],
- error_measure_lut[_mm_extract_epi16(diff_2, 0)]);
- __m128i error_2_hi =
- _mm_set_epi32(error_measure_lut[_mm_extract_epi16(diff_2, 7)],
- error_measure_lut[_mm_extract_epi16(diff_2, 6)],
- error_measure_lut[_mm_extract_epi16(diff_2, 5)],
- error_measure_lut[_mm_extract_epi16(diff_2, 4)]);
-
- __m128i error_1 = _mm_add_epi32(error_1_lo, error_1_hi);
- __m128i error_2 = _mm_add_epi32(error_2_lo, error_2_hi);
- __m128i error_1_2 = _mm_add_epi32(error_1, error_2);
-
- row_error = _mm_add_epi32(row_error, error_1_2);
- }
- __m128i col_error_lo = _mm_unpacklo_epi32(row_error, zero);
- __m128i col_error_hi = _mm_unpackhi_epi32(row_error, zero);
- __m128i col_error_temp = _mm_add_epi64(col_error_lo, col_error_hi);
- col_error = _mm_add_epi64(col_error, col_error_temp);
- // Error summation for remaining width, which is not multiple of 16
- if (p_width & 0xf) {
- for (int l = j * 16; l < p_width; ++l) {
- sum_error += (int64_t)error_measure(dst[l + i * dst_stride] -
- ref[l + i * ref_stride]);
- }
- }
- }
- int64_t sum_error_d_0, sum_error_d_1;
- xx_storel_64(&sum_error_d_0, col_error);
- xx_storel_64(&sum_error_d_1, _mm_srli_si128(col_error, 8));
- sum_error = (sum_error + sum_error_d_0 + sum_error_d_1);
- return sum_error;
-}
diff --git a/av1/common/x86/wiener_convolve_avx2.c b/av1/common/x86/wiener_convolve_avx2.c
index b7ac683..3de630f 100644
--- a/av1/common/x86/wiener_convolve_avx2.c
+++ b/av1/common/x86/wiener_convolve_avx2.c
@@ -45,7 +45,7 @@
const int16_t *filter_x, int x_step_q4,
const int16_t *filter_y, int y_step_q4,
int w, int h,
- const ConvolveParams *conv_params) {
+ const WienerConvolveParams *conv_params) {
const int bd = 8;
assert(x_step_q4 == 16 && y_step_q4 == 16);
assert(!(w & 7));
diff --git a/av1/common/x86/wiener_convolve_sse2.c b/av1/common/x86/wiener_convolve_sse2.c
index f9d00b7..1c039e8 100644
--- a/av1/common/x86/wiener_convolve_sse2.c
+++ b/av1/common/x86/wiener_convolve_sse2.c
@@ -23,7 +23,7 @@
const int16_t *filter_x, int x_step_q4,
const int16_t *filter_y, int y_step_q4,
int w, int h,
- const ConvolveParams *conv_params) {
+ const WienerConvolveParams *conv_params) {
const int bd = 8;
assert(x_step_q4 == 16 && y_step_q4 == 16);
assert(!(w & 7));
diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c
index 5b76de8..e3cce40 100644
--- a/av1/decoder/decodeframe.c
+++ b/av1/decoder/decodeframe.c
@@ -1275,9 +1275,13 @@
const int num_planes = av1_num_planes(cm);
for (int plane = 0; plane < num_planes; ++plane) {
int rcol0, rcol1, rrow0, rrow1;
+
+ // Skip some unnecessary work if loop restoration is disabled
+ if (cm->rst_info[plane].frame_restoration_type == RESTORE_NONE) continue;
+
if (av1_loop_restoration_corners_in_sb(cm, plane, mi_row, mi_col, bsize,
&rcol0, &rcol1, &rrow0, &rrow1)) {
- const int rstride = cm->rst_info[plane].horz_units_per_tile;
+ const int rstride = cm->rst_info[plane].horz_units;
for (int rrow = rrow0; rrow < rrow1; ++rrow) {
for (int rcol = rcol0; rcol < rcol1; ++rcol) {
const int runit_idx = rcol + rrow * rstride;
@@ -4326,7 +4330,6 @@
trans_dec_factor;
}
- assert(params->wmtype <= AFFINE);
int good_shear_params = av1_get_shear_params(params);
if (!good_shear_params) return 0;
@@ -5219,6 +5222,9 @@
cm->rst_info[1].frame_restoration_type != RESTORE_NONE ||
cm->rst_info[2].frame_restoration_type != RESTORE_NONE) {
av1_alloc_restoration_buffers(cm, /*is_sgr_enabled =*/true);
+ for (int p = 0; p < av1_num_planes(cm); p++) {
+ av1_alloc_restoration_struct(cm, &cm->rst_info[p], p > 0);
+ }
}
const int use_highbd = cm->seq_params->use_highbitdepth;
@@ -5238,6 +5244,7 @@
MACROBLOCKD *const xd = &pbi->dcb.xd;
const int tile_count_tg = end_tile - start_tile + 1;
+ xd->error_info = cm->error;
if (initialize_flag) setup_frame_info(pbi);
const int num_planes = av1_num_planes(cm);
diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c
index 5f114f9..bb0ccf5f 100644
--- a/av1/decoder/decodemv.c
+++ b/av1/decoder/decodemv.c
@@ -311,7 +311,7 @@
}
static int read_intra_segment_id(AV1_COMMON *const cm,
- const MACROBLOCKD *const xd, int bsize,
+ const MACROBLOCKD *const xd, BLOCK_SIZE bsize,
aom_reader *r, int skip) {
struct segmentation *const seg = &cm->seg;
if (!seg->enabled) return 0; // Default for disabled segmentation
@@ -825,13 +825,13 @@
if (mbmi->uv_mode == UV_CFL_PRED) {
mbmi->cfl_alpha_idx = read_cfl_alphas(ec_ctx, r, &mbmi->cfl_alpha_signs);
}
+ const PREDICTION_MODE intra_mode = get_uv_mode(mbmi->uv_mode);
mbmi->angle_delta[PLANE_TYPE_UV] =
- (use_angle_delta && av1_is_directional_mode(get_uv_mode(mbmi->uv_mode)))
- ? read_angle_delta(r,
- ec_ctx->angle_delta_cdf[mbmi->uv_mode - V_PRED])
+ (use_angle_delta && av1_is_directional_mode(intra_mode))
+ ? read_angle_delta(r, ec_ctx->angle_delta_cdf[intra_mode - V_PRED])
: 0;
} else {
- // Avoid decoding angle_info if there is is no chroma prediction
+ // Avoid decoding angle_info if there is no chroma prediction
mbmi->uv_mode = UV_DC_PRED;
}
xd->cfl.store_y = store_cfl_required(cm, xd);
@@ -1086,13 +1086,13 @@
mbmi->cfl_alpha_idx =
read_cfl_alphas(xd->tile_ctx, r, &mbmi->cfl_alpha_signs);
}
+ const PREDICTION_MODE intra_mode = get_uv_mode(mbmi->uv_mode);
mbmi->angle_delta[PLANE_TYPE_UV] =
- use_angle_delta && av1_is_directional_mode(get_uv_mode(mbmi->uv_mode))
- ? read_angle_delta(r,
- ec_ctx->angle_delta_cdf[mbmi->uv_mode - V_PRED])
+ use_angle_delta && av1_is_directional_mode(intra_mode)
+ ? read_angle_delta(r, ec_ctx->angle_delta_cdf[intra_mode - V_PRED])
: 0;
} else {
- // Avoid decoding angle_info if there is is no chroma prediction
+ // Avoid decoding angle_info if there is no chroma prediction
mbmi->uv_mode = UV_DC_PRED;
}
xd->cfl.store_y = store_cfl_required(cm, xd);
diff --git a/av1/encoder/allintra_vis.c b/av1/encoder/allintra_vis.c
index 236b296..8dcef5f 100644
--- a/av1/encoder/allintra_vis.c
+++ b/av1/encoder/allintra_vis.c
@@ -29,6 +29,26 @@
#include "av1/encoder/model_rd.h"
#include "av1/encoder/rdopt_utils.h"
+#define MB_WIENER_PRED_BLOCK_SIZE BLOCK_128X128
+#define MB_WIENER_PRED_BUF_STRIDE 128
+
+void av1_alloc_mb_wiener_var_pred_buf(AV1_COMMON *cm, ThreadData *td) {
+ const int is_high_bitdepth = is_cur_buf_hbd(&td->mb.e_mbd);
+ assert(MB_WIENER_PRED_BLOCK_SIZE < BLOCK_SIZES_ALL);
+ const int buf_width = block_size_wide[MB_WIENER_PRED_BLOCK_SIZE];
+ const int buf_height = block_size_high[MB_WIENER_PRED_BLOCK_SIZE];
+ assert(buf_width == MB_WIENER_PRED_BUF_STRIDE);
+ const size_t buf_size =
+ (buf_width * buf_height * sizeof(*td->wiener_tmp_pred_buf))
+ << is_high_bitdepth;
+ CHECK_MEM_ERROR(cm, td->wiener_tmp_pred_buf, aom_memalign(32, buf_size));
+}
+
+void av1_dealloc_mb_wiener_var_pred_buf(ThreadData *td) {
+ aom_free(td->wiener_tmp_pred_buf);
+ td->wiener_tmp_pred_buf = NULL;
+}
+
void av1_init_mb_wiener_var_buffer(AV1_COMP *cpi) {
AV1_COMMON *cm = &cpi->common;
@@ -236,7 +256,7 @@
int16_t *src_diff, tran_low_t *coeff,
tran_low_t *qcoeff, tran_low_t *dqcoeff,
double *sum_rec_distortion,
- double *sum_est_rate) {
+ double *sum_est_rate, uint8_t *pred_buffer) {
AV1_COMMON *const cm = &cpi->common;
uint8_t *buffer = cpi->source->y_buffer;
int buf_stride = cpi->source->y_stride;
@@ -250,27 +270,21 @@
const int coeff_count = block_size * block_size;
const int mb_step = mi_size_wide[bsize];
const BitDepthInfo bd_info = get_bit_depth_info(xd);
- const AV1EncAllIntraMultiThreadInfo *const intra_mt = &cpi->mt_info.intra_mt;
+ const MultiThreadInfo *const mt_info = &cpi->mt_info;
+ const AV1EncAllIntraMultiThreadInfo *const intra_mt = &mt_info->intra_mt;
AV1EncRowMultiThreadSync *const intra_row_mt_sync =
&cpi->ppi->intra_row_mt_sync;
const int mi_cols = cm->mi_params.mi_cols;
const int mt_thread_id = mi_row / mb_step;
// TODO(chengchen): test different unit step size
- const int mt_unit_step = mi_size_wide[BLOCK_64X64];
+ const int mt_unit_step = mi_size_wide[MB_WIENER_MT_UNIT_SIZE];
const int mt_unit_cols = (mi_cols + (mt_unit_step >> 1)) / mt_unit_step;
int mt_unit_col = 0;
const int is_high_bitdepth = is_cur_buf_hbd(xd);
- // We use a scratch buffer to store the prediction.
- // The stride is the max block size (128).
- uint8_t *pred_buffer;
- const int dst_buffer_stride = 128;
- const int buf_width = 128;
- const int buf_height = 128;
- const size_t buf_size = (buf_width * buf_height * sizeof(*pred_buffer))
- << is_high_bitdepth;
- CHECK_MEM_ERROR(cm, pred_buffer, aom_memalign(32, buf_size));
uint8_t *dst_buffer = pred_buffer;
+ const int dst_buffer_stride = MB_WIENER_PRED_BUF_STRIDE;
+
if (is_high_bitdepth) {
uint16_t *pred_buffer_16 = (uint16_t *)pred_buffer;
dst_buffer = CONVERT_TO_BYTEPTR(pred_buffer_16);
@@ -280,6 +294,18 @@
if (mi_col % mt_unit_step == 0) {
intra_mt->intra_sync_read_ptr(intra_row_mt_sync, mt_thread_id,
mt_unit_col);
+#if CONFIG_MULTITHREAD
+ const int num_workers =
+ AOMMIN(mt_info->num_mod_workers[MOD_AI], mt_info->num_workers);
+ if (num_workers > 1) {
+ const AV1EncRowMultiThreadInfo *const enc_row_mt = &mt_info->enc_row_mt;
+ pthread_mutex_lock(enc_row_mt->mutex_);
+ const bool exit = enc_row_mt->mb_wiener_mt_exit;
+ pthread_mutex_unlock(enc_row_mt->mutex_);
+ // Stop further processing in case any worker has encountered an error.
+ if (exit) break;
+ }
+#endif
}
PREDICTION_MODE best_mode = DC_PRED;
@@ -434,7 +460,6 @@
}
// Set the pointer to null since mbmi is only allocated inside this function.
xd->mi = NULL;
- aom_free(pred_buffer);
}
static void calc_mb_wiener_var(AV1_COMP *const cpi, double *sum_rec_distortion,
@@ -449,7 +474,8 @@
DECLARE_ALIGNED(32, tran_low_t, dqcoeff[32 * 32]);
for (int mi_row = 0; mi_row < cpi->frame_info.mi_rows; mi_row += mb_step) {
av1_calc_mb_wiener_var_row(cpi, x, xd, mi_row, src_diff, coeff, qcoeff,
- dqcoeff, sum_rec_distortion, sum_est_rate);
+ dqcoeff, sum_rec_distortion, sum_est_rate,
+ cpi->td.wiener_tmp_pred_buf);
}
}
@@ -565,6 +591,7 @@
NULL, cpi->image_pyramid_levels, 0))
aom_internal_error(cm->error, AOM_CODEC_MEM_ERROR,
"Failed to allocate frame buffer");
+ av1_alloc_mb_wiener_var_pred_buf(&cpi->common, &cpi->td);
cpi->norm_wiener_variance = 0;
MACROBLOCK *x = &cpi->td.mb;
@@ -647,6 +674,7 @@
// Set the pointer to null since mbmi is only allocated inside this function.
xd->mi = NULL;
aom_free_frame_buffer(&cm->cur_frame->buf);
+ av1_dealloc_mb_wiener_var_pred_buf(&cpi->td);
}
static int get_rate_guided_quantizer(AV1_COMP *const cpi, BLOCK_SIZE bsize,
diff --git a/av1/encoder/allintra_vis.h b/av1/encoder/allintra_vis.h
index 9e10566..0d34ce0 100644
--- a/av1/encoder/allintra_vis.h
+++ b/av1/encoder/allintra_vis.h
@@ -20,6 +20,8 @@
#include "av1/encoder/block.h"
#include "av1/encoder/encoder.h"
+#define MB_WIENER_MT_UNIT_SIZE BLOCK_64X64
+
void av1_init_mb_wiener_var_buffer(AV1_COMP *cpi);
void av1_calc_mb_wiener_var_row(AV1_COMP *const cpi, MACROBLOCK *x,
@@ -27,7 +29,7 @@
int16_t *src_diff, tran_low_t *coeff,
tran_low_t *qcoeff, tran_low_t *dqcoeff,
double *sum_rec_distortion,
- double *sum_est_rate);
+ double *sum_est_rate, uint8_t *pred_buffer);
void av1_set_mb_wiener_variance(AV1_COMP *cpi);
diff --git a/av1/encoder/aq_cyclicrefresh.c b/av1/encoder/aq_cyclicrefresh.c
index be51ba1..f48ff11 100644
--- a/av1/encoder/aq_cyclicrefresh.c
+++ b/av1/encoder/aq_cyclicrefresh.c
@@ -48,7 +48,8 @@
// mode, and rate/distortion.
static int candidate_refresh_aq(const CYCLIC_REFRESH *cr,
const MB_MODE_INFO *mbmi, int64_t rate,
- int64_t dist, int bsize, int noise_level) {
+ int64_t dist, BLOCK_SIZE bsize,
+ int noise_level) {
MV mv = mbmi->mv[0].as_mv;
int is_compound = has_second_ref(mbmi);
// Reject the block for lower-qp coding for non-compound mode if
@@ -642,11 +643,15 @@
int av1_cyclic_refresh_disable_lf_cdef(AV1_COMP *const cpi) {
CYCLIC_REFRESH *const cr = cpi->cyclic_refresh;
- // TODO(marpan): Tune these conditons, add QP dependence.
- if (cpi->sf.rt_sf.skip_lf_screen > 1 && !cpi->rc.high_source_sad) return 1;
+ const int qindex = cpi->common.quant_params.base_qindex;
if (cpi->rc.frames_since_key > 30 && cr->percent_refresh > 0 &&
cr->counter_encode_maxq_scene_change > 300 / cr->percent_refresh &&
- cpi->rc.frame_source_sad < 1000)
+ cpi->rc.frame_source_sad < 1000 &&
+ qindex < 7 * (cpi->rc.worst_quality >> 3))
+ return 1;
+ // More aggressive skip.
+ else if (cpi->sf.rt_sf.skip_lf_screen > 1 && !cpi->rc.high_source_sad &&
+ cpi->rc.frame_source_sad < 50000 && qindex < cpi->rc.worst_quality)
return 1;
return 0;
}
diff --git a/av1/encoder/arm/crc32/hash_crc32.c b/av1/encoder/arm/crc32/hash_arm_crc32.c
similarity index 95%
rename from av1/encoder/arm/crc32/hash_crc32.c
rename to av1/encoder/arm/crc32/hash_arm_crc32.c
index 771496c..91fc1e0 100644
--- a/av1/encoder/arm/crc32/hash_crc32.c
+++ b/av1/encoder/arm/crc32/hash_arm_crc32.c
@@ -9,9 +9,14 @@
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
-#include <stdint.h>
-#include <stddef.h>
+#if defined(_MSC_VER) && !defined(__clang__)
+#include <intrin.h>
+#else
#include <arm_acle.h>
+#endif
+
+#include <stddef.h>
+#include <stdint.h>
#include "config/aom_config.h"
diff --git a/av1/encoder/arm/neon/av1_fwd_txfm2d_neon.c b/av1/encoder/arm/neon/av1_fwd_txfm2d_neon.c
index ee8b115..a17a41a 100644
--- a/av1/encoder/arm/neon/av1_fwd_txfm2d_neon.c
+++ b/av1/encoder/arm/neon/av1_fwd_txfm2d_neon.c
@@ -12,792 +12,597 @@
#include <arm_neon.h>
#include <assert.h>
-#include "aom_dsp/txfm_common.h"
#include "aom_dsp/arm/mem_neon.h"
+#include "aom_dsp/arm/transpose_neon.h"
+#include "aom_dsp/txfm_common.h"
#include "aom_ports/mem.h"
#include "av1/common/av1_txfm.h"
#include "av1/encoder/av1_fwd_txfm1d_cfg.h"
#include "config/aom_config.h"
#include "config/av1_rtcd.h"
+#include "shift_neon.h"
+#include "txfm_neon.h"
-#define custom_packs_s32(w0, w1) vcombine_s16(vqmovn_s32(w0), vqmovn_s32(w1))
+#define TXFM_COS_BIT_MAX 13
-static INLINE void transpose_16bit_4x4(const int16x8_t *const in,
- int16x8_t *const out) {
-#if AOM_ARCH_AARCH64
- const int16x8_t a0 = vzip1q_s16(in[0], in[1]);
- const int16x8_t a1 = vzip1q_s16(in[2], in[3]);
-#else
- int16x4x2_t temp;
- temp = vzip_s16(vget_low_s16(in[0]), vget_low_s16(in[1]));
- const int16x8_t a0 = vcombine_s16(temp.val[0], temp.val[1]);
- temp = vzip_s16(vget_low_s16(in[2]), vget_low_s16(in[3]));
- const int16x8_t a1 = vcombine_s16(temp.val[0], temp.val[1]);
-#endif
+// A note on butterfly helper naming:
+//
+// butterfly_[input_ty]_[acc_ty]_[input_num]_[weight_num]_[weight_neg]_neon
+// e.g. butterfly_s32_s32_x4_0231_neon
+// | | | ^ Weights are applied as indices 0, 2, 3, 1
+// | | | (see more detail below)
+// | | ^ (int32)x4 input/output parameters
+// | ^ 32-bit accumulators internally
+// ^ 32-bit input/output parameters
+//
+// Weights are stored as 4-tuples in Q2.13 format as (w0, 1-w0, -w0, w0-1) to
+// avoid needing separate negation instructions. This is represented in the
+// helper naming by referring to the lane index in the loaded tuple that each
+// multiply is performed with:
+//
+// in0 in1
+// /----------
+// out0 | w0 w1 ==> out0 = in0 * w0 + in1 * w1
+// out1 | w2 w3 ==> out1 = in0 * w2 + in1 * w3
+//
+// So for indices 0331 from the earlier example, we end up with:
+//
+// in0 in1
+// /------------------
+// out0 | (lane 0) (lane 2) ==> out0 = in0 * w0 + in1 * -w0
+// out1 | (lane 3) (lane 1) ==> out1 = in0 * (w0-1) + in1 * (1-w0)
- int32x4x2_t a01 =
- vzipq_s32(vreinterpretq_s32_s16(a0), vreinterpretq_s32_s16(a1));
- out[0] = vreinterpretq_s16_s32(a01.val[0]);
- out[1] = vextq_s16(vreinterpretq_s16_s32(a01.val[0]), out[1], 4);
- out[2] = vreinterpretq_s16_s32(a01.val[1]);
- out[3] = vextq_s16(vreinterpretq_s16_s32(a01.val[1]), out[3], 4);
+static AOM_FORCE_INLINE void butterfly_s32_s32_x4_0112_neon(
+ const int16x4_t w0101_s16, const int32x4_t in0, const int32x4_t in1,
+ int32x4_t *out0, int32x4_t *out1) {
+ int32x4_t w0101 = vmovl_s16(w0101_s16);
+ int32x4_t o0 = vmulq_lane_s32(in0, vget_low_s32(w0101), 0);
+ o0 = vmlaq_lane_s32(o0, in1, vget_low_s32(w0101), 1);
+ int32x4_t o1 = vmulq_lane_s32(in0, vget_low_s32(w0101), 1);
+ o1 = vmlaq_lane_s32(o1, in1, vget_high_s32(w0101), 0);
+ *out0 = vrshrq_n_s32(o0, TXFM_COS_BIT_MAX);
+ *out1 = vrshrq_n_s32(o1, TXFM_COS_BIT_MAX);
}
-static INLINE void transpose_16bit_4x8(const int16x8_t *const in,
- int16x8_t *const out) {
-#if AOM_ARCH_AARCH64
- const int16x8_t a0 = vzip1q_s16(in[0], in[1]);
- const int16x8_t a1 = vzip1q_s16(in[2], in[3]);
- const int16x8_t a2 = vzip1q_s16(in[4], in[5]);
- const int16x8_t a3 = vzip1q_s16(in[6], in[7]);
-#else
- int16x4x2_t temp;
- temp = vzip_s16(vget_low_s16(in[0]), vget_low_s16(in[1]));
- const int16x8_t a0 = vcombine_s16(temp.val[0], temp.val[1]);
- temp = vzip_s16(vget_low_s16(in[2]), vget_low_s16(in[3]));
- const int16x8_t a1 = vcombine_s16(temp.val[0], temp.val[1]);
- temp = vzip_s16(vget_low_s16(in[4]), vget_low_s16(in[5]));
- const int16x8_t a2 = vcombine_s16(temp.val[0], temp.val[1]);
- temp = vzip_s16(vget_low_s16(in[6]), vget_low_s16(in[7]));
- const int16x8_t a3 = vcombine_s16(temp.val[0], temp.val[1]);
-#endif
-
- const int32x4x2_t b02 =
- vzipq_s32(vreinterpretq_s32_s16(a0), vreinterpretq_s32_s16(a1));
- const int32x4x2_t b13 =
- vzipq_s32(vreinterpretq_s32_s16(a2), vreinterpretq_s32_s16(a3));
-
-#if AOM_ARCH_AARCH64
- out[0] = vreinterpretq_s16_s64(vzip1q_s64(vreinterpretq_s64_s32(b02.val[0]),
- vreinterpretq_s64_s32(b13.val[0])));
- out[1] = vreinterpretq_s16_s64(vzip2q_s64(vreinterpretq_s64_s32(b02.val[0]),
- vreinterpretq_s64_s32(b13.val[0])));
- out[2] = vreinterpretq_s16_s64(vzip1q_s64(vreinterpretq_s64_s32(b02.val[1]),
- vreinterpretq_s64_s32(b13.val[1])));
- out[3] = vreinterpretq_s16_s64(vzip2q_s64(vreinterpretq_s64_s32(b02.val[1]),
- vreinterpretq_s64_s32(b13.val[1])));
-#else
- out[0] = vreinterpretq_s16_s32(
- vextq_s32(vextq_s32(b02.val[0], b02.val[0], 2), b13.val[0], 2));
- out[2] = vreinterpretq_s16_s32(
- vextq_s32(vextq_s32(b02.val[1], b02.val[1], 2), b13.val[1], 2));
- out[1] = vreinterpretq_s16_s32(
- vextq_s32(b02.val[0], vextq_s32(b13.val[0], b13.val[0], 2), 2));
- out[3] = vreinterpretq_s16_s32(
- vextq_s32(b02.val[1], vextq_s32(b13.val[1], b13.val[1], 2), 2));
-#endif
+static AOM_FORCE_INLINE void butterfly_s32_s32_x4_0332_neon(
+ const int16x4_t w0101_s16, const int32x4_t in0, const int32x4_t in1,
+ int32x4_t *out0, int32x4_t *out1) {
+ int32x4_t w0101 = vmovl_s16(w0101_s16);
+ int32x4_t o0 = vmulq_lane_s32(in0, vget_low_s32(w0101), 0);
+ o0 = vmlaq_lane_s32(o0, in1, vget_high_s32(w0101), 1);
+ int32x4_t o1 = vmulq_lane_s32(in0, vget_high_s32(w0101), 1);
+ o1 = vmlaq_lane_s32(o1, in1, vget_high_s32(w0101), 0);
+ *out0 = vrshrq_n_s32(o0, TXFM_COS_BIT_MAX);
+ *out1 = vrshrq_n_s32(o1, TXFM_COS_BIT_MAX);
}
-static INLINE void transpose_16bit_8x4(const int16x8_t *const in,
- int16x8_t *const out) {
- const int16x8x2_t a04 = vzipq_s16(in[0], in[1]);
- const int16x8x2_t a15 = vzipq_s16(in[2], in[3]);
-
- const int32x4x2_t b01 = vzipq_s32(vreinterpretq_s32_s16(a04.val[0]),
- vreinterpretq_s32_s16(a15.val[0]));
- const int32x4x2_t b45 = vzipq_s32(vreinterpretq_s32_s16(a04.val[1]),
- vreinterpretq_s32_s16(a15.val[1]));
-
- const int32x4_t zeros = vdupq_n_s32(0);
-
-#if AOM_ARCH_AARCH64
- out[0] = vreinterpretq_s16_s64(vzip1q_s64(vreinterpretq_s64_s32(b01.val[0]),
- vreinterpretq_s64_s32(zeros)));
- out[1] = vreinterpretq_s16_s64(vzip2q_s64(vreinterpretq_s64_s32(b01.val[0]),
- vreinterpretq_s64_s32(zeros)));
- out[2] = vreinterpretq_s16_s64(vzip1q_s64(vreinterpretq_s64_s32(b01.val[1]),
- vreinterpretq_s64_s32(zeros)));
- out[3] = vreinterpretq_s16_s64(vzip2q_s64(vreinterpretq_s64_s32(b01.val[1]),
- vreinterpretq_s64_s32(zeros)));
- out[4] = vreinterpretq_s16_s64(vzip1q_s64(vreinterpretq_s64_s32(b45.val[0]),
- vreinterpretq_s64_s32(zeros)));
- out[5] = vreinterpretq_s16_s64(vzip2q_s64(vreinterpretq_s64_s32(b45.val[0]),
- vreinterpretq_s64_s32(zeros)));
- out[6] = vreinterpretq_s16_s64(vzip1q_s64(vreinterpretq_s64_s32(b45.val[1]),
- vreinterpretq_s64_s32(zeros)));
- out[7] = vreinterpretq_s16_s64(vzip2q_s64(vreinterpretq_s64_s32(b45.val[1]),
- vreinterpretq_s64_s32(zeros)));
-#else
- out[0] = vreinterpretq_s16_s32(
- vextq_s32(vextq_s32(b01.val[0], b01.val[0], 2), zeros, 2));
- out[1] = vreinterpretq_s16_s32(vextq_s32(b01.val[0], zeros, 2));
- out[2] = vreinterpretq_s16_s32(
- vextq_s32(vextq_s32(b01.val[1], b01.val[1], 2), zeros, 2));
- out[3] = vreinterpretq_s16_s32(vextq_s32(b01.val[1], zeros, 2));
- out[4] = vreinterpretq_s16_s32(
- vextq_s32(vextq_s32(b45.val[0], b45.val[0], 2), zeros, 2));
- out[5] = vreinterpretq_s16_s32(vextq_s32(b45.val[0], zeros, 2));
- out[6] = vreinterpretq_s16_s32(
- vextq_s32(vextq_s32(b45.val[1], b45.val[1], 2), zeros, 2));
- out[7] = vreinterpretq_s16_s32(vextq_s32(b45.val[1], zeros, 2));
-#endif
+static AOM_FORCE_INLINE void butterfly_s32_s32_x4_1003_neon(
+ const int16x4_t w0101_s16, const int32x4_t in0, const int32x4_t in1,
+ int32x4_t *out0, int32x4_t *out1) {
+ int32x4_t w0101 = vmovl_s16(w0101_s16);
+ int32x4_t o0 = vmulq_lane_s32(in0, vget_low_s32(w0101), 1);
+ o0 = vmlaq_lane_s32(o0, in1, vget_low_s32(w0101), 0);
+ int32x4_t o1 = vmulq_lane_s32(in0, vget_low_s32(w0101), 0);
+ o1 = vmlaq_lane_s32(o1, in1, vget_high_s32(w0101), 1);
+ *out0 = vrshrq_n_s32(o0, TXFM_COS_BIT_MAX);
+ *out1 = vrshrq_n_s32(o1, TXFM_COS_BIT_MAX);
}
-static INLINE void transpose_16bit_8x8(const int16x8_t *const in,
- int16x8_t *const out) {
- const int16x8x2_t a04 = vzipq_s16(in[0], in[1]);
- const int16x8x2_t a15 = vzipq_s16(in[2], in[3]);
- const int16x8x2_t a26 = vzipq_s16(in[4], in[5]);
- const int16x8x2_t a37 = vzipq_s16(in[6], in[7]);
-
- const int32x4x2_t b04 = vzipq_s32(vreinterpretq_s32_s16(a04.val[0]),
- vreinterpretq_s32_s16(a15.val[0]));
- const int32x4x2_t b15 = vzipq_s32(vreinterpretq_s32_s16(a26.val[0]),
- vreinterpretq_s32_s16(a37.val[0]));
- const int32x4x2_t b26 = vzipq_s32(vreinterpretq_s32_s16(a04.val[1]),
- vreinterpretq_s32_s16(a15.val[1]));
- const int32x4x2_t b37 = vzipq_s32(vreinterpretq_s32_s16(a26.val[1]),
- vreinterpretq_s32_s16(a37.val[1]));
-
-#if AOM_ARCH_AARCH64
- out[0] = vreinterpretq_s16_s64(vzip1q_s64(vreinterpretq_s64_s32(b04.val[0]),
- vreinterpretq_s64_s32(b15.val[0])));
- out[1] = vreinterpretq_s16_s64(vzip2q_s64(vreinterpretq_s64_s32(b04.val[0]),
- vreinterpretq_s64_s32(b15.val[0])));
- out[2] = vreinterpretq_s16_s64(vzip1q_s64(vreinterpretq_s64_s32(b04.val[1]),
- vreinterpretq_s64_s32(b15.val[1])));
- out[3] = vreinterpretq_s16_s64(vzip2q_s64(vreinterpretq_s64_s32(b04.val[1]),
- vreinterpretq_s64_s32(b15.val[1])));
- out[4] = vreinterpretq_s16_s64(vzip1q_s64(vreinterpretq_s64_s32(b26.val[0]),
- vreinterpretq_s64_s32(b37.val[0])));
- out[5] = vreinterpretq_s16_s64(vzip2q_s64(vreinterpretq_s64_s32(b26.val[0]),
- vreinterpretq_s64_s32(b37.val[0])));
- out[6] = vreinterpretq_s16_s64(vzip1q_s64(vreinterpretq_s64_s32(b26.val[1]),
- vreinterpretq_s64_s32(b37.val[1])));
- out[7] = vreinterpretq_s16_s64(vzip2q_s64(vreinterpretq_s64_s32(b26.val[1]),
- vreinterpretq_s64_s32(b37.val[1])));
-#else
- out[0] = vreinterpretq_s16_s32(
- vextq_s32(vextq_s32(b04.val[0], b04.val[0], 2), b15.val[0], 2));
- out[1] = vreinterpretq_s16_s32(
- vextq_s32(b04.val[0], vextq_s32(b15.val[0], b15.val[0], 2), 2));
- out[2] = vreinterpretq_s16_s32(
- vextq_s32(vextq_s32(b04.val[1], b04.val[1], 2), b15.val[1], 2));
- out[3] = vreinterpretq_s16_s32(
- vextq_s32(b04.val[1], vextq_s32(b15.val[1], b15.val[1], 2), 2));
- out[4] = vreinterpretq_s16_s32(
- vextq_s32(vextq_s32(b26.val[0], b26.val[0], 2), b37.val[0], 2));
- out[5] = vreinterpretq_s16_s32(
- vextq_s32(b26.val[0], vextq_s32(b37.val[0], b37.val[0], 2), 2));
- out[6] = vreinterpretq_s16_s32(
- vextq_s32(vextq_s32(b26.val[1], b26.val[1], 2), b37.val[1], 2));
- out[7] = vreinterpretq_s16_s32(
- vextq_s32(b26.val[1], vextq_s32(b37.val[1], b37.val[1], 2), 2));
-#endif
+static AOM_FORCE_INLINE void butterfly_s32_s32_x4_1223_neon(
+ const int16x4_t w0101_s16, const int32x4_t in0, const int32x4_t in1,
+ int32x4_t *out0, int32x4_t *out1) {
+ int32x4_t w0101 = vmovl_s16(w0101_s16);
+ int32x4_t o0 = vmulq_lane_s32(in0, vget_low_s32(w0101), 1);
+ o0 = vmlaq_lane_s32(o0, in1, vget_high_s32(w0101), 0);
+ int32x4_t o1 = vmulq_lane_s32(in0, vget_high_s32(w0101), 0);
+ o1 = vmlaq_lane_s32(o1, in1, vget_high_s32(w0101), 1);
+ *out0 = vrshrq_n_s32(o0, TXFM_COS_BIT_MAX);
+ *out1 = vrshrq_n_s32(o1, TXFM_COS_BIT_MAX);
}
-static INLINE void av1_round_shift_rect_array_32_neon(int32x4_t *input,
- int32x4_t *output,
- const int size) {
- int i;
- for (i = 0; i < size; i++) {
- output[i] = vrshrq_n_s32(vmulq_n_s32(vrshrq_n_s32(input[i], 2), NewSqrt2),
- NewSqrt2Bits);
- }
-}
-
-static INLINE void av1_round_shift_array_32_neon(int32x4_t *input,
- int32x4_t *output,
- const int size) {
- int i;
- for (i = 0; i < size; i++) output[i] = vrshrq_n_s32(input[i], 2);
-}
-
-#define btf_32_neon(w0, w1, in0, in1, out0, out1, v_cos_bit) \
- do { \
- out0 = vmulq_n_s32(in0, w0); \
- out0 = vmlaq_n_s32(out0, in1, w1); \
- out0 = vrshlq_s32(out0, v_cos_bit); \
- out1 = vmulq_n_s32(in0, w1); \
- out1 = vmlsq_n_s32(out1, in1, w0); \
- out1 = vrshlq_s32(out1, v_cos_bit); \
+#define butterfly_s16_s32_x4_neon(wvec, lane0, lane1, lane2, lane3, in0, in1, \
+ out0, out1) \
+ do { \
+ int32x4_t u0 = vmull_lane_s16(in0, wvec, lane0); \
+ u0 = vmlal_lane_s16(u0, in1, wvec, lane1); \
+ int32x4_t v0 = vmull_lane_s16(in0, wvec, lane2); \
+ v0 = vmlal_lane_s16(v0, in1, wvec, lane3); \
+ *out0 = vqrshrn_n_s32(u0, TXFM_COS_BIT_MAX); \
+ *out1 = vqrshrn_n_s32(v0, TXFM_COS_BIT_MAX); \
} while (0)
-#define btf_32_type1_neon(w0, w1, in0, in1, out0, out1, v_cos_bit) \
- do { \
- btf_32_neon(w1, w0, in1, in0, out0, out1, v_cos_bit); \
+static AOM_FORCE_INLINE void butterfly_s16_s32_x4_0112_neon(
+ const int16x4_t w0101, const int16x4_t in0, const int16x4_t in1,
+ int16x4_t *out0, int16x4_t *out1) {
+ butterfly_s16_s32_x4_neon(w0101, 0, 1, 1, 2, in0, in1, out0, out1);
+}
+
+static AOM_FORCE_INLINE void butterfly_s16_s32_x4_0332_neon(
+ const int16x4_t w0101, const int16x4_t in0, const int16x4_t in1,
+ int16x4_t *out0, int16x4_t *out1) {
+ butterfly_s16_s32_x4_neon(w0101, 0, 3, 3, 2, in0, in1, out0, out1);
+}
+
+static AOM_FORCE_INLINE void butterfly_s16_s32_x4_1003_neon(
+ const int16x4_t w0101, const int16x4_t in0, const int16x4_t in1,
+ int16x4_t *out0, int16x4_t *out1) {
+ butterfly_s16_s32_x4_neon(w0101, 1, 0, 0, 3, in0, in1, out0, out1);
+}
+
+static AOM_FORCE_INLINE void butterfly_s16_s32_x4_1223_neon(
+ const int16x4_t w0101, const int16x4_t in0, const int16x4_t in1,
+ int16x4_t *out0, int16x4_t *out1) {
+ butterfly_s16_s32_x4_neon(w0101, 1, 2, 2, 3, in0, in1, out0, out1);
+}
+
+#define butterfly_s16_s32_x8_neon(wvec, lane0, lane1, lane2, lane3, in0, in1, \
+ out0, out1) \
+ do { \
+ int32x4_t u0 = vmull_lane_s16(vget_low_s16(in0), wvec, lane0); \
+ u0 = vmlal_lane_s16(u0, vget_low_s16(in1), wvec, lane1); \
+ int32x4_t u1 = vmull_lane_s16(vget_high_s16(in0), wvec, lane0); \
+ u1 = vmlal_lane_s16(u1, vget_high_s16(in1), wvec, lane1); \
+ int32x4_t v0 = vmull_lane_s16(vget_low_s16(in0), wvec, lane2); \
+ v0 = vmlal_lane_s16(v0, vget_low_s16(in1), wvec, lane3); \
+ int32x4_t v1 = vmull_lane_s16(vget_high_s16(in0), wvec, lane2); \
+ v1 = vmlal_lane_s16(v1, vget_high_s16(in1), wvec, lane3); \
+ const int16x4_t c0 = vrshrn_n_s32(u0, TXFM_COS_BIT_MAX); \
+ const int16x4_t c1 = vrshrn_n_s32(u1, TXFM_COS_BIT_MAX); \
+ const int16x4_t d0 = vrshrn_n_s32(v0, TXFM_COS_BIT_MAX); \
+ const int16x4_t d1 = vrshrn_n_s32(v1, TXFM_COS_BIT_MAX); \
+ *out0 = vcombine_s16(c0, c1); \
+ *out1 = vcombine_s16(d0, d1); \
} while (0)
-#define btf_32_neon_mode0(w0, w1, in0, in1, out0, out1, v_cos_bit) \
- do { \
- out0 = vmulq_n_s32(in1, w1); \
- out0 = vmlsq_n_s32(out0, in0, w0); \
- out0 = vrshlq_s32(out0, v_cos_bit); \
- out1 = vmulq_n_s32(in0, w1); \
- out1 = vmlaq_n_s32(out1, in1, w0); \
- out1 = vrshlq_s32(out1, v_cos_bit); \
- } while (0)
+static AOM_FORCE_INLINE void butterfly_s16_s32_x8_0112_neon(
+ const int16x4_t w0101, const int16x8_t in0, const int16x8_t in1,
+ int16x8_t *out0, int16x8_t *out1) {
+ butterfly_s16_s32_x8_neon(w0101, 0, 1, 1, 2, in0, in1, out0, out1);
+}
-#define btf_32_neon_mode01(w0, w1, in0, in1, out0, out1, v_cos_bit) \
- do { \
- out0 = vmulq_n_s32(in1, w1); \
- out0 = vmlaq_n_s32(out0, in0, w0); \
- out0 = vrshlq_s32(vnegq_s32(out0), v_cos_bit); \
- out1 = vmulq_n_s32(in1, w0); \
- out1 = vmlsq_n_s32(out1, in0, w1); \
- out1 = vrshlq_s32(out1, v_cos_bit); \
- } while (0)
+static AOM_FORCE_INLINE void butterfly_s16_s32_x8_0332_neon(
+ const int16x4_t w0101, const int16x8_t in0, const int16x8_t in1,
+ int16x8_t *out0, int16x8_t *out1) {
+ butterfly_s16_s32_x8_neon(w0101, 0, 3, 3, 2, in0, in1, out0, out1);
+}
-static INLINE void flip_buf_neon(int16x8_t *in, int16x8_t *out, int size) {
+static AOM_FORCE_INLINE void butterfly_s16_s32_x8_1003_neon(
+ const int16x4_t w0101, const int16x8_t in0, const int16x8_t in1,
+ int16x8_t *out0, int16x8_t *out1) {
+ butterfly_s16_s32_x8_neon(w0101, 1, 0, 0, 3, in0, in1, out0, out1);
+}
+
+static AOM_FORCE_INLINE void butterfly_s16_s32_x8_1223_neon(
+ const int16x4_t w0101, const int16x8_t in0, const int16x8_t in1,
+ int16x8_t *out0, int16x8_t *out1) {
+ butterfly_s16_s32_x8_neon(w0101, 1, 2, 2, 3, in0, in1, out0, out1);
+}
+
+static AOM_FORCE_INLINE void flip_buf_4_neon(int16x4_t *in, int16x4_t *out,
+ int size) {
for (int i = 0; i < size; ++i) {
out[size - i - 1] = in[i];
}
}
-static INLINE void store_16bit_to_32bit_w4(const int16x8_t a,
- int32_t *const b) {
- vst1q_s32(b, vmovl_s16(vget_low_s16(a)));
+static AOM_FORCE_INLINE void flip_buf_8_neon(int16x8_t *in, int16x8_t *out,
+ int size) {
+ for (int i = 0; i < size; ++i) {
+ out[size - i - 1] = in[i];
+ }
}
-static INLINE void store_16bit_to_32bit(int16x8_t a, int32_t *b) {
- vst1q_s32(b, vmovl_s16(vget_low_s16(a)));
- vst1q_s32((b + 4), vmovl_s16(vget_high_s16(a)));
-}
-
-static INLINE void store_output_32bit_w8(int32_t *const out,
- const int32x4_t *const in1,
- const int32x4_t *const in2,
- const int stride, const int out_size) {
+static AOM_FORCE_INLINE void store_buffer_interleaved_s32_x8(
+ int32_t *const out, const int32x4_t *const in1, const int32x4_t *const in2,
+ const int stride, const int out_size) {
for (int i = 0; i < out_size; ++i) {
vst1q_s32(out + stride * i, in1[i]);
vst1q_s32(out + stride * i + 4, in2[i]);
}
}
-static INLINE void store_rect_16bit_to_32bit_w4(
- const int16x8_t a, int32_t *const b, const int16x4_t *v_newsqrt2,
- const int32x4_t *v_newsqrt2bits) {
- const int32x4_t b_lo =
- vrshlq_s32(vmull_s16(vget_low_s16(a), *v_newsqrt2), *v_newsqrt2bits);
- vst1q_s32(b, b_lo);
-}
-
-static INLINE void store_rect_16bit_to_32bit(const int16x8_t a,
- int32_t *const b,
- const int16x4_t *v_newsqrt2,
- const int32x4_t *v_newsqrt2bits) {
- const int32x4_t b_lo =
- vrshlq_s32(vmull_s16(vget_low_s16(a), *v_newsqrt2), *v_newsqrt2bits);
- const int32x4_t b_hi =
- vrshlq_s32(vmull_s16(vget_high_s16(a), *v_newsqrt2), *v_newsqrt2bits);
- vst1q_s32(b, b_lo);
- vst1q_s32((b + 4), b_hi);
-}
-
-static INLINE void load_buffer_16bit_to_16bit_w4(const int16_t *in,
- const int stride,
- int16x8_t *const out,
- const int out_size) {
+static AOM_FORCE_INLINE void load_buffer_s16_x4(const int16_t *in,
+ const int stride,
+ int16x4_t *const out,
+ const int out_size) {
for (int i = 0; i < out_size; ++i) {
- // vld1q_dup_u64 is used rather than vld1q_lane_u64(lane=0) to avoid
- // -Wmaybe-uninitialized warnings with some versions of gcc. This assumes
- // the upper lane is unused or further modified after this call. The
- // latency should be similar between the two.
- out[i] = vreinterpretq_s16_u64(vld1q_dup_u64((uint64_t *)in));
+ out[i] = vld1_s16(in);
in += stride;
}
}
-static INLINE void load_buffer_16bit_to_16bit_w4_flip(const int16_t *in,
- const int stride,
- int16x8_t *const out,
- const int out_size) {
- for (int i = out_size - 1; i >= 0; --i) {
- // vld1q_dup_u64 is used rather than vld1q_lane_u64(lane=0) to avoid
- // -Wmaybe-uninitialized warnings with some versions of gcc. This assumes
- // the upper lane is unused or further modified after this call. The
- // latency should be similar between the two.
- out[i] = vreinterpretq_s16_u64(vld1q_dup_u64((uint64_t *)in));
- in += stride;
- }
-}
-
-static INLINE void load_buffer_16bit_to_16bit(const int16_t *in, int stride,
- int16x8_t *out, int out_size) {
+static AOM_FORCE_INLINE void load_buffer_s16_x8(const int16_t *in, int stride,
+ int16x8_t *out, int out_size) {
for (int i = 0; i < out_size; ++i) {
out[i] = vld1q_s16(in + i * stride);
}
}
-static INLINE void load_buffer_16bit_to_16bit_flip(const int16_t *in,
- int stride, int16x8_t *out,
- int out_size) {
+static AOM_FORCE_INLINE void store_buffer_s16_x4(const int16x4_t *const in,
+ int32_t *const out,
+ const int stride,
+ const int out_size) {
for (int i = 0; i < out_size; ++i) {
- out[out_size - i - 1] = vld1q_s16(in + i * stride);
+ vst1q_s32(out + i * stride, vmovl_s16(in[i]));
}
}
-static INLINE void store_buffer_16bit_to_32bit_w4(const int16x8_t *const in,
- int32_t *const out,
- const int stride,
- const int out_size) {
+static AOM_FORCE_INLINE void store_buffer_s16_x8(const int16x8_t *const in,
+ int32_t *const out,
+ const int stride,
+ const int out_size) {
for (int i = 0; i < out_size; ++i) {
- store_16bit_to_32bit_w4(in[i], out + i * stride);
+ vst1q_s32(out + i * stride + 0, vmovl_s16(vget_low_s16(in[i])));
+ vst1q_s32(out + i * stride + 4, vmovl_s16(vget_high_s16(in[i])));
}
}
-static INLINE void store_buffer_16bit_to_32bit_w8(const int16x8_t *const in,
- int32_t *const out,
- const int stride,
- const int out_size) {
+// A note on naming:
+// round_shift_[sqrt2]_s16_s32_4x1_neon(...)
+// | | | ^ 1 => a single vector
+// | | | n => an array of vectors
+// | | | ^ input/output vector element count
+// | | ^ output type
+// | ^ input type
+// ^ multiplicand and shift identifier
+
+static AOM_FORCE_INLINE int16x4_t
+round_shift_sqrt2_s16_s16_4x1_neon(int16x4_t a) {
+ return vqrshrn_n_s32(vmull_n_s16(a, NewSqrt2), NewSqrt2Bits);
+}
+
+static AOM_FORCE_INLINE int16x8_t
+round_shift_sqrt2_s16_s16_8x1_neon(int16x8_t a) {
+ return vcombine_s16(round_shift_sqrt2_s16_s16_4x1_neon(vget_low_s16(a)),
+ round_shift_sqrt2_s16_s16_4x1_neon(vget_high_s16(a)));
+}
+
+static AOM_FORCE_INLINE int16x4_t
+round_shift_2sqrt2_s16_s16_4x1_neon(int16x4_t a) {
+ return vqrshrn_n_s32(vmull_n_s16(a, 2 * NewSqrt2), NewSqrt2Bits);
+}
+
+static AOM_FORCE_INLINE int16x8_t
+round_shift_2sqrt2_s16_s16_8x1_neon(int16x8_t a) {
+ return vcombine_s16(round_shift_2sqrt2_s16_s16_4x1_neon(vget_low_s16(a)),
+ round_shift_2sqrt2_s16_s16_4x1_neon(vget_high_s16(a)));
+}
+
+static AOM_FORCE_INLINE int32x4_t
+round_shift_sqrt2_s16_s32_4x1_neon(int16x4_t a) {
+ return vrshrq_n_s32(vmull_n_s16(a, NewSqrt2), NewSqrt2Bits);
+}
+
+static AOM_FORCE_INLINE int32x4_t
+round_shift_sqrt2_s32_s32_4x1_neon(int32x4_t a) {
+ return vrshrq_n_s32(vmulq_n_s32(a, NewSqrt2), NewSqrt2Bits);
+}
+
+#define ROUND_SHIFT_SQRT_LOOP_HELPER(name, type0, type1, fn) \
+ static AOM_FORCE_INLINE void name(const type0 *in, type1 *out, int size) { \
+ for (int i = 0; i < size; ++i) { \
+ out[i] = fn(in[i]); \
+ } \
+ }
+
+ROUND_SHIFT_SQRT_LOOP_HELPER(round_shift_sqrt2_s32_s32_4xn_neon, int32x4_t,
+ int32x4_t, round_shift_sqrt2_s32_s32_4x1_neon)
+ROUND_SHIFT_SQRT_LOOP_HELPER(round_shift_sqrt2_s16_s16_4xn_neon, int16x4_t,
+ int16x4_t, round_shift_sqrt2_s16_s16_4x1_neon)
+ROUND_SHIFT_SQRT_LOOP_HELPER(round_shift_sqrt2_s16_s16_8xn_neon, int16x8_t,
+ int16x8_t, round_shift_sqrt2_s16_s16_8x1_neon)
+ROUND_SHIFT_SQRT_LOOP_HELPER(round_shift_2sqrt2_s16_s16_4xn_neon, int16x4_t,
+ int16x4_t, round_shift_2sqrt2_s16_s16_4x1_neon)
+ROUND_SHIFT_SQRT_LOOP_HELPER(round_shift_2sqrt2_s16_s16_8xn_neon, int16x8_t,
+ int16x8_t, round_shift_2sqrt2_s16_s16_8x1_neon)
+
+static AOM_FORCE_INLINE void store_rect_buffer_s16_x4(const int16x4_t *const in,
+ int32_t *const out,
+ const int stride,
+ const int out_size) {
for (int i = 0; i < out_size; ++i) {
- store_16bit_to_32bit(in[i], out + i * stride);
+ vst1q_s32(out + i * stride, round_shift_sqrt2_s16_s32_4x1_neon(in[i]));
}
}
-static INLINE void store_rect_buffer_16bit_to_32bit_w4(
- const int16x8_t *const in, int32_t *const out, const int stride,
- const int out_size) {
- const int16x4_t v_newsqrt2 = vdup_n_s16(NewSqrt2);
- const int32x4_t v_newsqrt2bits = vdupq_n_s32(-NewSqrt2Bits);
+static AOM_FORCE_INLINE void store_rect_buffer_s16_x8(const int16x8_t *const in,
+ int32_t *const out,
+ const int stride,
+ const int out_size) {
for (int i = 0; i < out_size; ++i) {
- store_rect_16bit_to_32bit_w4(in[i], out + i * stride, &v_newsqrt2,
- &v_newsqrt2bits);
+ vst1q_s32(out + i * stride + 0,
+ round_shift_sqrt2_s16_s32_4x1_neon(vget_low_s16(in[i])));
+ vst1q_s32(out + i * stride + 4,
+ round_shift_sqrt2_s16_s32_4x1_neon(vget_high_s16(in[i])));
}
}
-static INLINE void store_rect_buffer_16bit_to_32bit_w8(
- const int16x8_t *const in, int32_t *const out, const int stride,
- const int out_size) {
- const int16x4_t v_newsqrt2 = vdup_n_s16(NewSqrt2);
- const int32x4_t v_newsqrt2bits = vdupq_n_s32(-NewSqrt2Bits);
- for (int i = 0; i < out_size; ++i) {
- store_rect_16bit_to_32bit(in[i], out + i * stride, &v_newsqrt2,
- &v_newsqrt2bits);
- }
-}
-
-static INLINE void round_shift_16bit(int16x8_t *in, int size, int bit) {
- const int16x8_t vbit = vdupq_n_s16(bit);
- for (int i = 0; i < size; ++i) {
- in[i] = vrshlq_s16(in[i], vbit);
- }
-}
-
-static INLINE void round_shift_16bit_vector(int16x8_t *in, int size,
- const int16x8_t *v_bit) {
- for (int i = 0; i < size; ++i) {
- in[i] = vrshlq_s16(in[i], *v_bit);
- }
-}
-
-void av1_fadst4x4_neon(const int16x8_t *input, int16x8_t *output,
- int8_t cos_bit, const int8_t *stage_range) {
- (void)stage_range;
- const int32_t *sinpi = sinpi_arr(cos_bit);
-
+static AOM_FORCE_INLINE void fadst4x4_neon(const int16x4_t *input,
+ int16x4_t *output, int cos_bit) {
int32x4_t u[6], v[6];
+ const int16x4_t sinpi = vld1_s16(sinpi_arr_q13(cos_bit));
+ const int16x4_t u01 = vqadd_s16(input[0], input[1]);
- u[0] = vmovl_s16(vget_low_s16(input[0]));
- u[1] = vmovl_s16(vget_low_s16(input[1]));
- u[2] = vmovl_s16(vget_low_s16(input[2]));
- u[3] = vmovl_s16(vget_low_s16(input[3]));
- u[4] = vaddq_s32(u[0], u[1]);
- v[5] = vmulq_n_s32(u[2], sinpi[3]);
- v[0] = vmulq_n_s32(u[1], sinpi[2]);
- v[0] = vmlaq_n_s32(v[0], u[0], sinpi[1]);
- v[1] = vmlaq_n_s32(v[5], u[3], sinpi[4]);
- v[2] = vmulq_n_s32(u[4], sinpi[3]);
- v[3] = vmulq_n_s32(u[0], sinpi[4]);
- v[3] = vmlsq_n_s32(v[3], u[1], sinpi[1]);
- v[4] = vmlsq_n_s32(v[5], u[3], sinpi[2]);
+ v[5] = vmull_lane_s16(input[2], sinpi, 2);
+ v[0] = vmull_lane_s16(input[1], sinpi, 1);
+ v[0] = vmlal_lane_s16(v[0], input[0], sinpi, 0);
+ v[1] = vmlal_lane_s16(v[5], input[3], sinpi, 3);
+ v[2] = vmull_lane_s16(u01, sinpi, 2);
+ v[3] = vmull_lane_s16(input[0], sinpi, 3);
+ v[3] = vmlsl_lane_s16(v[3], input[1], sinpi, 0);
+ v[4] = vmlsl_lane_s16(v[5], input[3], sinpi, 1);
u[0] = vaddq_s32(v[0], v[1]);
- u[1] = vmlsq_n_s32(v[2], u[3], sinpi[3]);
+ u[1] = vmlsl_lane_s16(v[2], input[3], sinpi, 2);
u[2] = vsubq_s32(v[3], v[4]);
u[3] = vsubq_s32(u[2], u[0]);
- u[5] = vmlaq_n_s32(u[3], v[5], 3);
+ u[3] = vmlaq_n_s32(u[3], v[5], 3);
- int32x4_t vshift = vdupq_n_s32(-cos_bit);
- u[0] = vrshlq_s32(u[0], vshift);
- u[1] = vrshlq_s32(u[1], vshift);
- u[2] = vrshlq_s32(u[2], vshift);
- u[3] = vrshlq_s32(u[5], vshift);
-
- output[0] = custom_packs_s32(u[0], u[2]);
-
- output[1] = custom_packs_s32(u[1], u[3]);
- output[2] = vextq_s16(output[0], output[0], 4);
- output[3] = vextq_s16(output[1], output[1], 4);
+ output[0] = vrshrn_n_s32(u[0], TXFM_COS_BIT_MAX);
+ output[1] = vrshrn_n_s32(u[1], TXFM_COS_BIT_MAX);
+ output[2] = vrshrn_n_s32(u[2], TXFM_COS_BIT_MAX);
+ output[3] = vrshrn_n_s32(u[3], TXFM_COS_BIT_MAX);
}
-#define btf_16_w4_neon(w0_l, w0_h, w1_l, w1_h, in0, in1, out0, out1, \
- v_cos_bit) \
- do { \
- int32x4_t in0_l = vmovl_s16(vget_low_s16(in0)); \
- int32x4_t in1_l = vmovl_s16(vget_low_s16(in1)); \
- int32x4_t u0 = vmulq_n_s32(in0_l, w0_l); \
- u0 = vmlaq_n_s32(u0, in1_l, w0_h); \
- int32x4_t v0 = vmulq_n_s32(in0_l, w1_l); \
- v0 = vmlaq_n_s32(v0, in1_l, w1_h); \
- int32x4_t c0 = vrshlq_s32(u0, v_cos_bit); \
- int32x4_t d0 = vrshlq_s32(v0, v_cos_bit); \
- const int16x4_t c1 = vqmovn_s32(c0); \
- const int16x4_t d1 = vqmovn_s32(d0); \
- out0 = vcombine_s16(c1, c1); \
- out1 = vcombine_s16(d1, c1); \
- } while (0)
+static AOM_FORCE_INLINE void fadst4x8_neon(const int16x4_t *input,
+ int16x4_t *output, int cos_bit) {
+ const int16_t *cospi = cospi_arr_q13(cos_bit);
-#define btf_16_w4_neon_mode0(w0_l, w0_h, in0, in1, out0, out1, v_cos_bit) \
- do { \
- int32x4_t in0_l = vmovl_s16(vget_low_s16(in0)); \
- int32x4_t in1_l = vmovl_s16(vget_low_s16(in1)); \
- int32x4_t u0 = vmulq_n_s32(in1_l, w0_h); \
- u0 = vmlsq_n_s32(u0, in0_l, w0_l); \
- int32x4_t v0 = vmulq_n_s32(in0_l, w0_h); \
- v0 = vmlaq_n_s32(v0, in1_l, w0_l); \
- int32x4_t c0 = vrshlq_s32(u0, v_cos_bit); \
- int32x4_t d0 = vrshlq_s32(v0, v_cos_bit); \
- const int16x4_t c1 = vqmovn_s32(c0); \
- const int16x4_t d1 = vqmovn_s32(d0); \
- out0 = vcombine_s16(c1, c1); \
- out1 = vcombine_s16(d1, c1); \
- } while (0)
+ const int16x8_t cospi32_16 = vld1q_s16(&cospi[4 * 0]);
+ const int16x8_t cospi4_12 = vld1q_s16(&cospi[4 * 4]);
+ const int16x8_t cospi20_28 = vld1q_s16(&cospi[4 * 6]);
-#define btf_16_w4_neon_mode2(w0_l, w0_h, in0, in1, out0, out1, v_cos_bit) \
- do { \
- int32x4_t in0_l = vmovl_s16(vget_low_s16(in0)); \
- int32x4_t in1_l = vmovl_s16(vget_low_s16(in1)); \
- int32x4_t u0 = vmulq_n_s32(in0_l, w0_l); \
- u0 = vmlaq_n_s32(u0, in1_l, w0_h); \
- int32x4_t v0 = vmulq_n_s32(in1_l, w0_l); \
- v0 = vmlsq_n_s32(v0, in0_l, w0_h); \
- int32x4_t c0 = vrshlq_s32(u0, v_cos_bit); \
- int32x4_t d0 = vrshlq_s32(v0, v_cos_bit); \
- const int16x4_t c1 = vqmovn_s32(c0); \
- const int16x4_t d1 = vqmovn_s32(d0); \
- out0 = vcombine_s16(c1, c1); \
- out1 = vcombine_s16(d1, c1); \
- } while (0)
-
-#define btf_16_w4_neon_mode3(w0_l, w0_h, in0, in1, out0, out1, v_cos_bit) \
- do { \
- int32x4_t in0_l = vmovl_s16(vget_low_s16(in0)); \
- int32x4_t in1_l = vmovl_s16(vget_low_s16(in1)); \
- int32x4_t u0 = vmulq_n_s32(in0_l, w0_l); \
- u0 = vmlaq_n_s32(u0, in1_l, w0_h); \
- int32x4_t v0 = vmulq_n_s32(in0_l, w0_h); \
- v0 = vmlsq_n_s32(v0, in1_l, w0_l); \
- int32x4_t c0 = vrshlq_s32(u0, v_cos_bit); \
- int32x4_t d0 = vrshlq_s32(v0, v_cos_bit); \
- const int16x4_t c1 = vqmovn_s32(c0); \
- const int16x4_t d1 = vqmovn_s32(d0); \
- out0 = vcombine_s16(c1, c1); \
- out1 = vcombine_s16(d1, c1); \
- } while (0)
-
-static void fadst4x8_neon(const int16x8_t *input, int16x8_t *output,
- int8_t cos_bit, const int8_t *stage_range) {
- (void)stage_range;
- const int32_t *cospi = cospi_arr(cos_bit);
- const int32x4_t v_cos_bit = vdupq_n_s32(-cos_bit);
+ const int16x4_t cospi32 = vget_low_s16(cospi32_16);
+ const int16x4_t cospi16 = vget_high_s16(cospi32_16);
+ const int16x4_t cospi4 = vget_low_s16(cospi4_12);
+ const int16x4_t cospi12 = vget_high_s16(cospi4_12);
+ const int16x4_t cospi20 = vget_low_s16(cospi20_28);
+ const int16x4_t cospi28 = vget_high_s16(cospi20_28);
// stage 1-2
- int16x8_t x2[8];
- btf_16_w4_neon_mode3(cospi[32], cospi[32], vqnegq_s16(input[3]), input[4],
- x2[2], x2[3], v_cos_bit);
- btf_16_w4_neon_mode3(cospi[32], cospi[32], input[2], vqnegq_s16(input[5]),
- x2[6], x2[7], v_cos_bit);
+ int16x4_t x2[8];
+ butterfly_s16_s32_x4_0332_neon(cospi32, input[4], input[3], &x2[2], &x2[3]);
+ butterfly_s16_s32_x4_0112_neon(cospi32, input[2], input[5], &x2[7], &x2[6]);
// stage 3
- int16x8_t x3[8];
- x3[0] = vqaddq_s16(input[0], x2[2]);
- x3[2] = vqsubq_s16(input[0], x2[2]);
- x3[1] = vqsubq_s16(x2[3], input[7]);
- x3[3] = vqsubq_s16(vqnegq_s16(input[7]), x2[3]);
- x3[4] = vqaddq_s16(vqnegq_s16(input[1]), x2[6]);
- x3[6] = vqsubq_s16(vqnegq_s16(input[1]), x2[6]);
- x3[5] = vqaddq_s16(input[6], x2[7]);
- x3[7] = vqsubq_s16(input[6], x2[7]);
+ int16x4_t x3[8];
+ x3[0] = vqadd_s16(input[0], x2[2]);
+ x3[1] = vqsub_s16(x2[3], input[7]);
+ x3[2] = vqsub_s16(input[0], x2[2]);
+ x3[3] = vqadd_s16(input[7], x2[3]);
+ x3[4] = vqsub_s16(x2[6], input[1]);
+ x3[5] = vqadd_s16(input[6], x2[7]);
+ x3[6] = vqadd_s16(input[1], x2[6]);
+ x3[7] = vqsub_s16(input[6], x2[7]);
// stage 4
- int16x8_t x4[8];
-
- btf_16_w4_neon_mode3(cospi[16], cospi[48], x3[4], x3[5], x4[4], x4[5],
- v_cos_bit);
- btf_16_w4_neon_mode0(cospi[48], cospi[16], x3[6], x3[7], x4[6], x4[7],
- v_cos_bit);
+ int16x4_t x4[8];
+ butterfly_s16_s32_x4_0112_neon(cospi16, x3[4], x3[5], &x4[4], &x4[5]);
+ butterfly_s16_s32_x4_0112_neon(cospi16, x3[7], x3[6], &x4[6], &x4[7]);
// stage 5
- int16x8_t x5[8];
- x5[0] = vqaddq_s16(x3[0], x4[4]);
- x5[4] = vqsubq_s16(x3[0], x4[4]);
- x5[1] = vqaddq_s16(x3[1], x4[5]);
- x5[5] = vqsubq_s16(x3[1], x4[5]);
- x5[2] = vqaddq_s16(x3[2], x4[6]);
- x5[6] = vqsubq_s16(x3[2], x4[6]);
- x5[3] = vqaddq_s16(x3[3], x4[7]);
- x5[7] = vqsubq_s16(x3[3], x4[7]);
+ int16x4_t x5[8];
+ x5[0] = vqadd_s16(x3[0], x4[4]);
+ x5[1] = vqadd_s16(x3[1], x4[5]);
+ x5[2] = vqadd_s16(x3[2], x4[6]);
+ x5[3] = vqsub_s16(x4[7], x3[3]);
+ x5[4] = vqsub_s16(x3[0], x4[4]);
+ x5[5] = vqsub_s16(x3[1], x4[5]);
+ x5[6] = vqsub_s16(x3[2], x4[6]);
+ x5[7] = vqadd_s16(x3[3], x4[7]);
// stage 6-7
- btf_16_w4_neon_mode3(cospi[4], cospi[60], x5[0], x5[1], output[7], output[0],
- v_cos_bit);
- btf_16_w4_neon_mode3(cospi[20], cospi[44], x5[2], x5[3], output[5], output[2],
- v_cos_bit);
- btf_16_w4_neon_mode3(cospi[36], cospi[28], x5[4], x5[5], output[3], output[4],
- v_cos_bit);
- btf_16_w4_neon_mode3(cospi[52], cospi[12], x5[6], x5[7], output[1], output[6],
- v_cos_bit);
+ butterfly_s16_s32_x4_0112_neon(cospi4, x5[0], x5[1], &output[7], &output[0]);
+ butterfly_s16_s32_x4_0112_neon(cospi20, x5[2], x5[3], &output[5], &output[2]);
+ butterfly_s16_s32_x4_1003_neon(cospi28, x5[4], x5[5], &output[3], &output[4]);
+ butterfly_s16_s32_x4_0112_neon(cospi12, x5[6], x5[7], &output[6], &output[1]);
}
-static void fadst8x4_neon(const int16x8_t *input, int16x8_t *output,
- int8_t cos_bit, const int8_t *stage_range) {
- (void)stage_range;
- const int32_t *sinpi = sinpi_arr(cos_bit);
+static AOM_FORCE_INLINE void fadst8x4_neon(const int16x8_t *input,
+ int16x8_t *output, int cos_bit) {
+ int32x4_t u_lo[4], u_hi[4];
+ const int16x4_t sinpi = vld1_s16(sinpi_arr_q13(cos_bit));
+ const int16x8_t u01 = vqaddq_s16(input[0], input[1]);
- const int16x8_t in7 = vaddq_s16(input[0], input[1]);
- int32x4_t u_lo[8], u_hi[8], v_hi[8];
+ u_lo[0] = vmull_lane_s16(vget_low_s16(input[1]), sinpi, 1);
+ u_hi[0] = vmull_lane_s16(vget_high_s16(input[1]), sinpi, 1);
- int32x4_t in0_l = vmovl_s16(vget_low_s16(input[0]));
- int32x4_t in0_h = vmovl_s16(vget_high_s16(input[0]));
- int32x4_t in1_l = vmovl_s16(vget_low_s16(input[1]));
- int32x4_t in1_h = vmovl_s16(vget_high_s16(input[1]));
- int32x4_t in2_l = vmovl_s16(vget_low_s16(input[2]));
- int32x4_t in2_h = vmovl_s16(vget_high_s16(input[2]));
- int32x4_t in3_l = vmovl_s16(vget_low_s16(input[3]));
- int32x4_t in3_h = vmovl_s16(vget_high_s16(input[3]));
- int32x4_t in7_l = vmovl_s16(vget_low_s16(in7));
- int32x4_t in7_h = vmovl_s16(vget_high_s16(in7));
+ u_lo[0] = vmlal_lane_s16(u_lo[0], vget_low_s16(input[0]), sinpi, 0);
+ u_hi[0] = vmlal_lane_s16(u_hi[0], vget_high_s16(input[0]), sinpi, 0);
- u_lo[0] = vmulq_n_s32(in1_l, sinpi[2]);
- u_lo[0] = vmlaq_n_s32(u_lo[0], in0_l, sinpi[1]);
+ u_lo[0] = vmlal_lane_s16(u_lo[0], vget_low_s16(input[3]), sinpi, 3);
+ u_hi[0] = vmlal_lane_s16(u_hi[0], vget_high_s16(input[3]), sinpi, 3);
- u_hi[0] = vmulq_n_s32(in1_h, sinpi[2]);
- u_hi[0] = vmlaq_n_s32(u_hi[0], in0_h, sinpi[1]);
+ u_lo[0] = vmlal_lane_s16(u_lo[0], vget_low_s16(input[2]), sinpi, 2);
+ u_hi[0] = vmlal_lane_s16(u_hi[0], vget_high_s16(input[2]), sinpi, 2);
- u_lo[0] = vmlaq_n_s32(u_lo[0], in3_l, sinpi[4]);
- u_lo[0] = vmlaq_n_s32(u_lo[0], in2_l, sinpi[3]);
+ u_lo[1] = vmull_lane_s16(vget_low_s16(u01), sinpi, 2);
+ u_hi[1] = vmull_lane_s16(vget_high_s16(u01), sinpi, 2);
- u_hi[0] = vmlaq_n_s32(u_hi[0], in3_h, sinpi[4]);
- u_hi[0] = vmlaq_n_s32(u_hi[0], in2_h, sinpi[3]);
+ u_lo[2] = vmull_lane_s16(vget_low_s16(input[0]), sinpi, 3);
+ u_hi[2] = vmull_lane_s16(vget_high_s16(input[0]), sinpi, 3);
- u_lo[1] = vmulq_n_s32(in7_l, sinpi[3]);
+ u_lo[2] = vmlsl_lane_s16(u_lo[2], vget_low_s16(input[1]), sinpi, 0);
+ u_hi[2] = vmlsl_lane_s16(u_hi[2], vget_high_s16(input[1]), sinpi, 0);
- v_hi[2] = vmulq_n_s32(in7_h, sinpi[3]);
- u_lo[2] = vmulq_n_s32(in0_l, sinpi[4]);
- u_lo[2] = vmlsq_n_s32(u_lo[2], in1_l, sinpi[1]);
+ u_lo[2] = vmlal_lane_s16(u_lo[2], vget_low_s16(input[3]), sinpi, 1);
+ u_hi[2] = vmlal_lane_s16(u_hi[2], vget_high_s16(input[3]), sinpi, 1);
- u_hi[2] = vmulq_n_s32(in0_h, sinpi[4]);
- u_hi[2] = vmlsq_n_s32(u_hi[2], in1_h, sinpi[1]);
+ u_lo[2] = vmlsl_lane_s16(u_lo[2], vget_low_s16(input[2]), sinpi, 2);
+ u_hi[2] = vmlsl_lane_s16(u_hi[2], vget_high_s16(input[2]), sinpi, 2);
- u_lo[2] = vmlaq_n_s32(u_lo[2], in3_l, sinpi[2]);
- u_lo[2] = vmlsq_n_s32(u_lo[2], in2_l, sinpi[3]);
-
- u_hi[2] = vmlaq_n_s32(u_hi[2], in3_h, sinpi[2]);
- u_hi[2] = vmlsq_n_s32(u_hi[2], in2_h, sinpi[3]);
-
- u_lo[1] = vmlsq_n_s32(u_lo[1], in3_l, sinpi[3]);
-
- const int32x4_t v_cos_bit = vdupq_n_s32(-cos_bit);
-
- u_hi[1] = vmlsq_n_s32(v_hi[2], in3_h, sinpi[3]);
+ u_lo[1] = vmlsl_lane_s16(u_lo[1], vget_low_s16(input[3]), sinpi, 2);
+ u_hi[1] = vmlsl_lane_s16(u_hi[1], vget_high_s16(input[3]), sinpi, 2);
u_lo[3] = vsubq_s32(u_lo[2], u_lo[0]);
u_hi[3] = vsubq_s32(u_hi[2], u_hi[0]);
- u_lo[6] = vmlaq_n_s32(u_lo[3], in2_l, sinpi[3] * 3);
- u_hi[6] = vmlaq_n_s32(u_hi[3], in2_h, sinpi[3] * 3);
+ const int16x4_t sinpix3 = vmul_n_s16(sinpi, 3);
+ u_lo[3] = vmlal_lane_s16(u_lo[3], vget_low_s16(input[2]), sinpix3, 2);
+ u_hi[3] = vmlal_lane_s16(u_hi[3], vget_high_s16(input[2]), sinpix3, 2);
- u_lo[0] = vrshlq_s32(u_lo[0], v_cos_bit);
- u_hi[0] = vrshlq_s32(u_hi[0], v_cos_bit);
- u_lo[1] = vrshlq_s32(u_lo[1], v_cos_bit);
- u_hi[1] = vrshlq_s32(u_hi[1], v_cos_bit);
- u_lo[2] = vrshlq_s32(u_lo[2], v_cos_bit);
- u_hi[2] = vrshlq_s32(u_hi[2], v_cos_bit);
- u_lo[3] = vrshlq_s32(u_lo[6], v_cos_bit);
- u_hi[3] = vrshlq_s32(u_hi[6], v_cos_bit);
-
- output[0] = custom_packs_s32(u_lo[0], u_hi[0]);
- output[1] = custom_packs_s32(u_lo[1], u_hi[1]);
- output[2] = custom_packs_s32(u_lo[2], u_hi[2]);
- output[3] = custom_packs_s32(u_lo[3], u_hi[3]);
+ output[0] = vcombine_s16(vrshrn_n_s32(u_lo[0], TXFM_COS_BIT_MAX),
+ vrshrn_n_s32(u_hi[0], TXFM_COS_BIT_MAX));
+ output[1] = vcombine_s16(vrshrn_n_s32(u_lo[1], TXFM_COS_BIT_MAX),
+ vrshrn_n_s32(u_hi[1], TXFM_COS_BIT_MAX));
+ output[2] = vcombine_s16(vrshrn_n_s32(u_lo[2], TXFM_COS_BIT_MAX),
+ vrshrn_n_s32(u_hi[2], TXFM_COS_BIT_MAX));
+ output[3] = vcombine_s16(vrshrn_n_s32(u_lo[3], TXFM_COS_BIT_MAX),
+ vrshrn_n_s32(u_hi[3], TXFM_COS_BIT_MAX));
}
-void av1_fdct4x4_neon(const int16x8_t *input, int16x8_t *output, int8_t cos_bit,
- const int8_t *stage_range) {
- (void)stage_range;
- const int32_t *cospi = cospi_arr(cos_bit);
- const int32x4_t v_cos_bit = vdupq_n_s32(-cos_bit);
+static AOM_FORCE_INLINE void fdct4x4_neon(const int16x4_t *input,
+ int16x4_t *output, int cos_bit) {
+ const int16_t *cospi = cospi_arr_q13(cos_bit);
+ const int16x4_t cospi16 = vld1_s16(&cospi[4 * 1]);
+
+ int16x4_t in12a = vadd_s16(input[1], input[2]);
+ int16x4_t in12s = vsub_s16(input[1], input[2]);
+ int16x4_t in03a = vadd_s16(input[0], input[3]);
+ int16x4_t in03s = vsub_s16(input[0], input[3]);
+
+ int32x4_t u0ad1 = vmull_n_s16(in12a, cospi[4 * 0]);
+ int32x4_t u0ad2 = vmull_n_s16(in03a, cospi[4 * 0]);
int32x4_t u[4];
-
- int32x4_t in12a = vaddl_s16(vget_low_s16(input[1]), vget_low_s16(input[2]));
- int32x4_t in12s = vsubl_s16(vget_low_s16(input[1]), vget_low_s16(input[2]));
- int32x4_t in03a = vaddl_s16(vget_low_s16(input[0]), vget_low_s16(input[3]));
- int32x4_t in03s = vsubl_s16(vget_low_s16(input[0]), vget_low_s16(input[3]));
-
- int32x4_t u0ad1 = vmulq_n_s32(in12a, cospi[32]);
- int32x4_t u0ad2 = vmulq_n_s32(in03a, cospi[32]);
u[0] = vaddq_s32(u0ad1, u0ad2);
u[1] = vsubq_s32(u0ad2, u0ad1);
- u[2] = vmulq_n_s32(in12s, cospi[48]);
- u[2] = vmlaq_n_s32(u[2], in03s, cospi[16]);
+ u[2] = vmull_lane_s16(in12s, cospi16, 1);
+ u[2] = vmlal_lane_s16(u[2], in03s, cospi16, 0);
+ u[3] = vmull_lane_s16(in03s, cospi16, 1);
+ u[3] = vmlsl_lane_s16(u[3], in12s, cospi16, 0);
- u[3] = vmulq_n_s32(in03s, cospi[48]);
- u[3] = vmlsq_n_s32(u[3], in12s, cospi[16]);
-
- u[0] = vrshlq_s32(u[0], v_cos_bit);
- u[1] = vrshlq_s32(u[1], v_cos_bit);
- u[2] = vrshlq_s32(u[2], v_cos_bit);
- u[3] = vrshlq_s32(u[3], v_cos_bit);
-
- output[0] = custom_packs_s32(u[0], u[1]);
- output[1] = custom_packs_s32(u[2], u[3]);
- output[2] = vextq_s16(output[0], output[0], 4);
- output[3] = vextq_s16(output[1], output[1], 4);
+ output[0] = vrshrn_n_s32(u[0], TXFM_COS_BIT_MAX);
+ output[1] = vrshrn_n_s32(u[2], TXFM_COS_BIT_MAX);
+ output[2] = vrshrn_n_s32(u[1], TXFM_COS_BIT_MAX);
+ output[3] = vrshrn_n_s32(u[3], TXFM_COS_BIT_MAX);
}
-#define btf_16_neon(w0_l, w0_h, w1_l, w1_h, in0, in1, out0, out1) \
- do { \
- int32x4_t in_low0 = vmovl_s16(vget_low_s16(in0)); \
- int32x4_t in_high0 = vmovl_s16(vget_high_s16(in0)); \
- int32x4_t in_low1 = vmovl_s16(vget_low_s16(in1)); \
- int32x4_t in_high1 = vmovl_s16(vget_high_s16(in1)); \
- int32x4_t u0 = vmulq_n_s32(in_low1, w0_h); \
- u0 = vmlaq_n_s32(u0, in_low0, w0_l); \
- int32x4_t u1 = vmulq_n_s32(in_high1, w0_h); \
- u1 = vmlaq_n_s32(u1, in_high0, w0_l); \
- int32x4_t v0 = vmulq_n_s32(in_low1, w1_h); \
- v0 = vmlaq_n_s32(v0, in_low0, w1_l); \
- int32x4_t v1 = vmulq_n_s32(in_high1, w1_h); \
- v1 = vmlaq_n_s32(v1, in_high0, w1_l); \
- int32x4_t c0 = vrshlq_s32(u0, v_cos_bit); \
- int32x4_t c1 = vrshlq_s32(u1, v_cos_bit); \
- int32x4_t d0 = vrshlq_s32(v0, v_cos_bit); \
- int32x4_t d1 = vrshlq_s32(v1, v_cos_bit); \
- out0 = custom_packs_s32(c0, c1); \
- out1 = custom_packs_s32(d0, d1); \
- } while (0)
+// Butterfly pre-processing:
+// e.g. n=4:
+// out[0] = in[0] + in[3]
+// out[1] = in[1] + in[2]
+// out[2] = in[1] - in[2]
+// out[3] = in[0] - in[3]
-#define btf_16_neon_mode0(w0_l, w0_h, in0, in1, out0, out1, v_cos_bit) \
- do { \
- int32x4_t in_low0 = vmovl_s16(vget_low_s16(in0)); \
- int32x4_t in_high0 = vmovl_s16(vget_high_s16(in0)); \
- int32x4_t in_low1 = vmovl_s16(vget_low_s16(in1)); \
- int32x4_t in_high1 = vmovl_s16(vget_high_s16(in1)); \
- int32x4_t u0 = vmulq_n_s32(in_low1, w0_h); \
- u0 = vmlsq_n_s32(u0, in_low0, w0_l); \
- int32x4_t u1 = vmulq_n_s32(in_high1, w0_h); \
- u1 = vmlsq_n_s32(u1, in_high0, w0_l); \
- int32x4_t v0 = vmulq_n_s32(in_low1, w0_l); \
- v0 = vmlaq_n_s32(v0, in_low0, w0_h); \
- int32x4_t v1 = vmulq_n_s32(in_high1, w0_l); \
- v1 = vmlaq_n_s32(v1, in_high0, w0_h); \
- int32x4_t c0 = vrshlq_s32(u0, v_cos_bit); \
- int32x4_t c1 = vrshlq_s32(u1, v_cos_bit); \
- int32x4_t d0 = vrshlq_s32(v0, v_cos_bit); \
- int32x4_t d1 = vrshlq_s32(v1, v_cos_bit); \
- out0 = custom_packs_s32(c0, c1); \
- out1 = custom_packs_s32(d0, d1); \
- } while (0)
+static AOM_FORCE_INLINE void butterfly_dct_pre_s16_x4(const int16x4_t *input,
+ int16x4_t *output,
+ int n) {
+ for (int i = 0; i < n / 2; ++i) {
+ output[i] = vqadd_s16(input[i], input[n - i - 1]);
+ }
+ for (int i = 0; i < n / 2; ++i) {
+ output[n / 2 + i] = vqsub_s16(input[n / 2 - i - 1], input[n / 2 + i]);
+ }
+}
-#define btf_16_neon_mode1(w0_l, w0_h, in0, in1, out0, out1, v_cos_bit) \
- do { \
- int32x4_t in_low0 = vmovl_s16(vget_low_s16(in0)); \
- int32x4_t in_high0 = vmovl_s16(vget_high_s16(in0)); \
- int32x4_t in_low1 = vmovl_s16(vget_low_s16(in1)); \
- int32x4_t in_high1 = vmovl_s16(vget_high_s16(in1)); \
- int32x4_t u0 = vmulq_n_s32(in_low0, w0_l); \
- u0 = vmlsq_n_s32(u0, in_low1, w0_h); \
- int32x4_t u1 = vmulq_n_s32(in_high0, w0_l); \
- u1 = vmlsq_n_s32(u1, in_high1, w0_h); \
- int32x4_t v0 = vmulq_n_s32(in_low1, w0_l); \
- v0 = vmlaq_n_s32(v0, in_low0, w0_h); \
- int32x4_t v1 = vmulq_n_s32(in_high1, w0_l); \
- v1 = vmlaq_n_s32(v1, in_high0, w0_h); \
- int32x4_t c0 = vrshlq_s32(u0, v_cos_bit); \
- int32x4_t c1 = vrshlq_s32(u1, v_cos_bit); \
- int32x4_t d0 = vrshlq_s32(v0, v_cos_bit); \
- int32x4_t d1 = vrshlq_s32(v1, v_cos_bit); \
- out0 = custom_packs_s32(c0, c1); \
- out1 = custom_packs_s32(d0, d1); \
- } while (0)
+static AOM_FORCE_INLINE void butterfly_dct_pre_s16_x8(const int16x8_t *input,
+ int16x8_t *output,
+ int n) {
+ for (int i = 0; i < n / 2; ++i) {
+ output[i] = vqaddq_s16(input[i], input[n - i - 1]);
+ }
+ for (int i = 0; i < n / 2; ++i) {
+ output[n / 2 + i] = vqsubq_s16(input[n / 2 - i - 1], input[n / 2 + i]);
+ }
+}
-#define btf_16_neon_mode02(w0_l, w0_h, in0, in1, out0, out1, v_cos_bit) \
- do { \
- int32x4_t in_low0 = vmovl_s16(vget_low_s16(in0)); \
- int32x4_t in_high0 = vmovl_s16(vget_high_s16(in0)); \
- int32x4_t in_low1 = vmovl_s16(vget_low_s16(in1)); \
- int32x4_t in_high1 = vmovl_s16(vget_high_s16(in1)); \
- int32x4_t u0 = vmulq_n_s32(in_low1, -w0_h); \
- u0 = vmlsq_n_s32(u0, in_low0, w0_l); \
- int32x4_t u1 = vmulq_n_s32(in_high1, -w0_h); \
- u1 = vmlsq_n_s32(u1, in_high0, w0_l); \
- int32x4_t v0 = vmulq_n_s32(in_low1, w0_l); \
- v0 = vmlsq_n_s32(v0, in_low0, w0_h); \
- int32x4_t v1 = vmulq_n_s32(in_high1, w0_l); \
- v1 = vmlsq_n_s32(v1, in_high0, w0_h); \
- int32x4_t c0 = vrshlq_s32(u0, v_cos_bit); \
- int32x4_t c1 = vrshlq_s32(u1, v_cos_bit); \
- int32x4_t d0 = vrshlq_s32(v0, v_cos_bit); \
- int32x4_t d1 = vrshlq_s32(v1, v_cos_bit); \
- out0 = custom_packs_s32(c0, c1); \
- out1 = custom_packs_s32(d0, d1); \
- } while (0)
+static AOM_FORCE_INLINE void butterfly_dct_pre_s32_x4(const int32x4_t *input,
+ int32x4_t *output,
+ int n) {
+ for (int i = 0; i < n / 2; ++i) {
+ output[i] = vqaddq_s32(input[i], input[n - i - 1]);
+ }
+ for (int i = 0; i < n / 2; ++i) {
+ output[n / 2 + i] = vqsubq_s32(input[n / 2 - i - 1], input[n / 2 + i]);
+ }
+}
-#define btf_16_neon_mode2(w0_l, w0_h, in0, in1, out0, out1, v_cos_bit) \
- do { \
- int32x4_t in_low0 = vmovl_s16(vget_low_s16(in0)); \
- int32x4_t in_high0 = vmovl_s16(vget_high_s16(in0)); \
- int32x4_t in_low1 = vmovl_s16(vget_low_s16(in1)); \
- int32x4_t in_high1 = vmovl_s16(vget_high_s16(in1)); \
- int32x4_t u0 = vmulq_n_s32(in_low1, w0_h); \
- u0 = vmlaq_n_s32(u0, in_low0, w0_l); \
- int32x4_t u1 = vmulq_n_s32(in_high1, w0_h); \
- u1 = vmlaq_n_s32(u1, in_high0, w0_l); \
- int32x4_t v0 = vmulq_n_s32(in_low1, w0_l); \
- v0 = vmlsq_n_s32(v0, in_low0, w0_h); \
- int32x4_t v1 = vmulq_n_s32(in_high1, w0_l); \
- v1 = vmlsq_n_s32(v1, in_high0, w0_h); \
- int32x4_t c0 = vrshlq_s32(u0, v_cos_bit); \
- int32x4_t c1 = vrshlq_s32(u1, v_cos_bit); \
- int32x4_t d0 = vrshlq_s32(v0, v_cos_bit); \
- int32x4_t d1 = vrshlq_s32(v1, v_cos_bit); \
- out0 = custom_packs_s32(c0, c1); \
- out1 = custom_packs_s32(d0, d1); \
- } while (0)
+// Butterfly post-processing:
+// e.g. n=8:
+// out[0] = in0[0] + in1[3];
+// out[1] = in0[1] + in1[2];
+// out[2] = in0[1] - in1[2];
+// out[3] = in0[0] - in1[3];
+// out[4] = in0[7] - in1[4];
+// out[5] = in0[6] - in1[5];
+// out[6] = in0[6] + in1[5];
+// out[7] = in0[7] + in1[4];
-#define btf_16_neon_mode3(w0_l, w0_h, in0, in1, out0, out1, v_cos_bit) \
- do { \
- int32x4_t in_low0 = vmovl_s16(vget_low_s16(in0)); \
- int32x4_t in_high0 = vmovl_s16(vget_high_s16(in0)); \
- int32x4_t in_low1 = vmovl_s16(vget_low_s16(in1)); \
- int32x4_t in_high1 = vmovl_s16(vget_high_s16(in1)); \
- int32x4_t u0 = vmulq_n_s32(in_low1, w0_h); \
- u0 = vmlaq_n_s32(u0, in_low0, w0_l); \
- int32x4_t u1 = vmulq_n_s32(in_high1, w0_h); \
- u1 = vmlaq_n_s32(u1, in_high0, w0_l); \
- int32x4_t v0 = vmulq_n_s32(in_low0, w0_h); \
- v0 = vmlsq_n_s32(v0, in_low1, w0_l); \
- int32x4_t v1 = vmulq_n_s32(in_high0, w0_h); \
- v1 = vmlsq_n_s32(v1, in_high1, w0_l); \
- int32x4_t c0 = vrshlq_s32(u0, v_cos_bit); \
- int32x4_t c1 = vrshlq_s32(u1, v_cos_bit); \
- int32x4_t d0 = vrshlq_s32(v0, v_cos_bit); \
- int32x4_t d1 = vrshlq_s32(v1, v_cos_bit); \
- out0 = custom_packs_s32(c0, c1); \
- out1 = custom_packs_s32(d0, d1); \
- } while (0)
+static AOM_FORCE_INLINE void butterfly_dct_post_s16_x4(const int16x4_t *in0,
+ const int16x4_t *in1,
+ int16x4_t *output,
+ int n) {
+ for (int i = 0; i < n / 4; ++i) {
+ output[i] = vqadd_s16(in0[i], in1[n / 2 - i - 1]);
+ }
+ for (int i = 0; i < n / 4; ++i) {
+ output[n / 4 + i] = vqsub_s16(in0[n / 4 - i - 1], in1[n / 4 + i]);
+ }
+ for (int i = 0; i < n / 4; ++i) {
+ output[n / 2 + i] = vqsub_s16(in0[n - i - 1], in1[n / 2 + i]);
+ }
+ for (int i = 0; i < n / 4; ++i) {
+ output[(3 * n) / 4 + i] =
+ vqadd_s16(in0[(3 * n) / 4 + i], in1[(3 * n) / 4 - i - 1]);
+ }
+}
-static void fdct8x4_neon(const int16x8_t *input, int16x8_t *output,
- int8_t cos_bit, const int8_t *stage_range) {
- (void)stage_range;
- const int32_t *cospi = cospi_arr(cos_bit);
- const int32x4_t v_cos_bit = vdupq_n_s32(-cos_bit);
+static AOM_FORCE_INLINE void butterfly_dct_post_s16_x8(const int16x8_t *in0,
+ const int16x8_t *in1,
+ int16x8_t *output,
+ int n) {
+ for (int i = 0; i < n / 4; ++i) {
+ output[i] = vqaddq_s16(in0[i], in1[n / 2 - i - 1]);
+ }
+ for (int i = 0; i < n / 4; ++i) {
+ output[n / 4 + i] = vqsubq_s16(in0[n / 4 - i - 1], in1[n / 4 + i]);
+ }
+ for (int i = 0; i < n / 4; ++i) {
+ output[n / 2 + i] = vqsubq_s16(in0[n - i - 1], in1[n / 2 + i]);
+ }
+ for (int i = 0; i < n / 4; ++i) {
+ output[(3 * n) / 4 + i] =
+ vqaddq_s16(in0[(3 * n) / 4 + i], in1[(3 * n) / 4 - i - 1]);
+ }
+}
+
+static AOM_FORCE_INLINE void butterfly_dct_post_s32_x4(const int32x4_t *in0,
+ const int32x4_t *in1,
+ int32x4_t *output,
+ int n) {
+ for (int i = 0; i < n / 4; ++i) {
+ output[i] = vqaddq_s32(in0[i], in1[n / 2 - i - 1]);
+ }
+ for (int i = 0; i < n / 4; ++i) {
+ output[n / 4 + i] = vqsubq_s32(in0[n / 4 - i - 1], in1[n / 4 + i]);
+ }
+ for (int i = 0; i < n / 4; ++i) {
+ output[n / 2 + i] = vqsubq_s32(in0[n - i - 1], in1[n / 2 + i]);
+ }
+ for (int i = 0; i < n / 4; ++i) {
+ output[(3 * n) / 4 + i] =
+ vqaddq_s32(in0[(3 * n) / 4 + i], in1[(3 * n) / 4 - i - 1]);
+ }
+}
+
+static AOM_FORCE_INLINE void fdct8x4_neon(const int16x8_t *input,
+ int16x8_t *output, int cos_bit) {
+ const int16_t *cospi = cospi_arr_q13(cos_bit);
+
+ const int16x8_t cospi32_16 = vld1q_s16(&cospi[4 * 0]);
+
+ const int16x4_t cospi32 = vget_low_s16(cospi32_16);
+ const int16x4_t cospi16 = vget_high_s16(cospi32_16);
// stage 1
int16x8_t x1[4];
- x1[0] = vqaddq_s16(input[0], input[3]);
- x1[3] = vqsubq_s16(input[0], input[3]);
- x1[1] = vqaddq_s16(input[1], input[2]);
- x1[2] = vqsubq_s16(input[1], input[2]);
+ butterfly_dct_pre_s16_x8(input, x1, 4);
// stage 2
int16x8_t x2[4];
- btf_16_neon_mode3(cospi[32], cospi[32], x1[0], x1[1], x2[0], x2[1],
- v_cos_bit);
- btf_16_neon_mode2(cospi[48], cospi[16], x1[2], x1[3], x2[2], x2[3],
- v_cos_bit);
+ butterfly_s16_s32_x8_0112_neon(cospi32, x1[0], x1[1], &x2[0], &x2[1]);
+ butterfly_s16_s32_x8_0112_neon(cospi16, x1[3], x1[2], &x2[2], &x2[3]);
// stage 3
output[0] = x2[0];
@@ -806,988 +611,528 @@
output[3] = x2[3];
}
-static void fdct4x8_neon(const int16x8_t *input, int16x8_t *output,
- int8_t cos_bit, const int8_t *stage_range) {
- (void)stage_range;
- const int32_t *cospi = cospi_arr(cos_bit);
- const int32x4_t v_cos_bit = vdupq_n_s32(-cos_bit);
+static AOM_FORCE_INLINE void fdct4x8_neon(const int16x4_t *input,
+ int16x4_t *output, int cos_bit) {
+ const int16_t *cospi = cospi_arr_q13(cos_bit);
+
+ const int16x8_t cospi32_16 = vld1q_s16(&cospi[4 * 0]);
+ const int16x8_t cospi8_24 = vld1q_s16(&cospi[4 * 2]);
+
+ const int16x4_t cospi32 = vget_low_s16(cospi32_16);
+ const int16x4_t cospi16 = vget_high_s16(cospi32_16);
+ const int16x4_t cospi8 = vget_low_s16(cospi8_24);
+ const int16x4_t cospi24 = vget_high_s16(cospi8_24);
+
+ // stage 1
+ int16x4_t x1[8];
+ butterfly_dct_pre_s16_x4(input, x1, 8);
+
+ // stage 2
+ int16x4_t x2[8];
+ butterfly_dct_pre_s16_x4(x1, x2, 4);
+ butterfly_s16_s32_x4_0112_neon(cospi32, x1[6], x1[5], &x2[6], &x2[5]);
+
+ // stage 3
+ int16x4_t x3[8];
+ butterfly_s16_s32_x4_0112_neon(cospi32, x2[0], x2[1], &output[0], &output[4]);
+ butterfly_s16_s32_x4_0112_neon(cospi16, x2[3], x2[2], &output[2], &output[6]);
+ butterfly_dct_post_s16_x4(x1 + 4, x2 + 4, x3 + 4, 4);
+
+ // stage 4-5
+ butterfly_s16_s32_x4_0112_neon(cospi8, x3[7], x3[4], &output[1], &output[7]);
+ butterfly_s16_s32_x4_1003_neon(cospi24, x3[6], x3[5], &output[5], &output[3]);
+}
+
+static AOM_FORCE_INLINE void fdct8x8_neon(const int16x8_t *input,
+ int16x8_t *output, int cos_bit) {
+ const int16_t *cospi = cospi_arr_q13(cos_bit);
+
+ const int16x8_t cospi32_16 = vld1q_s16(&cospi[4 * 0]);
+ const int16x8_t cospi8_24 = vld1q_s16(&cospi[4 * 2]);
+
+ const int16x4_t cospi32 = vget_low_s16(cospi32_16);
+ const int16x4_t cospi16 = vget_high_s16(cospi32_16);
+ const int16x4_t cospi8 = vget_low_s16(cospi8_24);
+ const int16x4_t cospi24 = vget_high_s16(cospi8_24);
// stage 1
int16x8_t x1[8];
- x1[0] = vqaddq_s16(input[0], input[7]);
- x1[7] = vqsubq_s16(input[0], input[7]);
- x1[1] = vqaddq_s16(input[1], input[6]);
- x1[6] = vqsubq_s16(input[1], input[6]);
- x1[2] = vqaddq_s16(input[2], input[5]);
- x1[5] = vqsubq_s16(input[2], input[5]);
- x1[3] = vqaddq_s16(input[3], input[4]);
- x1[4] = vqsubq_s16(input[3], input[4]);
+ butterfly_dct_pre_s16_x8(input, x1, 8);
// stage 2
int16x8_t x2[8];
- x2[0] = vqaddq_s16(x1[0], x1[3]);
- x2[3] = vqsubq_s16(x1[0], x1[3]);
- x2[1] = vqaddq_s16(x1[1], x1[2]);
- x2[2] = vqsubq_s16(x1[1], x1[2]);
-
- btf_16_w4_neon_mode0(cospi[32], cospi[32], x1[5], x1[6], x2[5], x2[6],
- v_cos_bit);
+ butterfly_dct_pre_s16_x8(x1, x2, 4);
+ butterfly_s16_s32_x8_0112_neon(cospi32, x1[6], x1[5], &x2[6], &x2[5]);
// stage 3
int16x8_t x3[8];
- btf_16_w4_neon_mode3(cospi[32], cospi[32], x2[0], x2[1], output[0], output[4],
- v_cos_bit);
-
- btf_16_w4_neon_mode2(cospi[48], cospi[16], x2[2], x2[3], output[2], output[6],
- v_cos_bit);
- x3[4] = vqaddq_s16(x1[4], x2[5]);
- x3[5] = vqsubq_s16(x1[4], x2[5]);
- x3[6] = vqsubq_s16(x1[7], x2[6]);
- x3[7] = vqaddq_s16(x1[7], x2[6]);
+ butterfly_s16_s32_x8_0112_neon(cospi32, x2[0], x2[1], &output[0], &output[4]);
+ butterfly_s16_s32_x8_0112_neon(cospi16, x2[3], x2[2], &output[2], &output[6]);
+ butterfly_dct_post_s16_x8(x1 + 4, x2 + 4, x3 + 4, 4);
// stage 4-5
- btf_16_w4_neon_mode2(cospi[56], cospi[8], x3[4], x3[7], output[1], output[7],
- v_cos_bit);
- btf_16_w4_neon_mode2(cospi[24], cospi[40], x3[5], x3[6], output[5], output[3],
- v_cos_bit);
+ butterfly_s16_s32_x8_0112_neon(cospi8, x3[7], x3[4], &output[1], &output[7]);
+ butterfly_s16_s32_x8_1003_neon(cospi24, x3[6], x3[5], &output[5], &output[3]);
}
-void fdct8x8_neon(const int16x8_t *input, int16x8_t *output, int8_t cos_bit,
- const int8_t *stage_range) {
- (void)stage_range;
- const int32_t *cospi = cospi_arr(cos_bit);
- const int32x4_t v_cos_bit = vdupq_n_s32(-cos_bit);
+static AOM_FORCE_INLINE void fdct4x16_neon(const int16x4_t *input,
+ int16x4_t *output, int cos_bit) {
+ const int16_t *cospi = cospi_arr_q13(cos_bit);
+
+ const int16x8_t cospi32_16 = vld1q_s16(&cospi[4 * 0]);
+ const int16x8_t cospi8_24 = vld1q_s16(&cospi[4 * 2]);
+ const int16x8_t cospi4_12 = vld1q_s16(&cospi[4 * 4]);
+ const int16x8_t cospi20_28 = vld1q_s16(&cospi[4 * 6]);
+
+ const int16x4_t cospi32 = vget_low_s16(cospi32_16);
+ const int16x4_t cospi16 = vget_high_s16(cospi32_16);
+ const int16x4_t cospi8 = vget_low_s16(cospi8_24);
+ const int16x4_t cospi24 = vget_high_s16(cospi8_24);
+ const int16x4_t cospi4 = vget_low_s16(cospi4_12);
+ const int16x4_t cospi12 = vget_high_s16(cospi4_12);
+ const int16x4_t cospi20 = vget_low_s16(cospi20_28);
+ const int16x4_t cospi28 = vget_high_s16(cospi20_28);
// stage 1
- int16x8_t x1[8];
- x1[0] = vqaddq_s16(input[0], input[7]);
- x1[7] = vqsubq_s16(input[0], input[7]);
- x1[1] = vqaddq_s16(input[1], input[6]);
- x1[6] = vqsubq_s16(input[1], input[6]);
- x1[2] = vqaddq_s16(input[2], input[5]);
- x1[5] = vqsubq_s16(input[2], input[5]);
- x1[3] = vqaddq_s16(input[3], input[4]);
- x1[4] = vqsubq_s16(input[3], input[4]);
+ int16x4_t x1[16];
+ butterfly_dct_pre_s16_x4(input, x1, 16);
// stage 2
- int16x8_t x2[8];
- x2[0] = vqaddq_s16(x1[0], x1[3]);
- x2[3] = vqsubq_s16(x1[0], x1[3]);
- x2[1] = vqaddq_s16(x1[1], x1[2]);
- x2[2] = vqsubq_s16(x1[1], x1[2]);
- btf_16_neon_mode0(cospi[32], cospi[32], x1[5], x1[6], x2[5], x2[6],
- v_cos_bit);
+ int16x4_t x2[16];
+ butterfly_dct_pre_s16_x4(x1, x2, 8);
+ butterfly_s16_s32_x4_0112_neon(cospi32, x1[13], x1[10], &x2[13], &x2[10]);
+ butterfly_s16_s32_x4_0112_neon(cospi32, x1[12], x1[11], &x2[12], &x2[11]);
// stage 3
- int16x8_t x3[8];
- btf_16_neon_mode3(cospi[32], cospi[32], x2[0], x2[1], output[0], output[4],
- v_cos_bit);
- btf_16_neon_mode2(cospi[48], cospi[16], x2[2], x2[3], output[2], output[6],
- v_cos_bit);
- x3[4] = vqaddq_s16(x1[4], x2[5]);
- x3[5] = vqsubq_s16(x1[4], x2[5]);
- x3[6] = vqsubq_s16(x1[7], x2[6]);
- x3[7] = vqaddq_s16(x1[7], x2[6]);
+ int16x4_t x3[16];
+ butterfly_dct_pre_s16_x4(x2, x3, 4);
+ butterfly_s16_s32_x4_0112_neon(cospi32, x2[6], x2[5], &x3[6], &x3[5]);
+ butterfly_dct_post_s16_x4(x1 + 8, x2 + 8, x3 + 8, 8);
- // stage 4-5
- btf_16_neon_mode2(cospi[56], cospi[8], x3[4], x3[7], output[1], output[7],
- v_cos_bit);
- btf_16_neon_mode2(cospi[24], cospi[40], x3[5], x3[6], output[5], output[3],
- v_cos_bit);
+ // stage 4
+ int16x4_t x4[16];
+ butterfly_s16_s32_x4_0112_neon(cospi32, x3[0], x3[1], &output[0], &output[8]);
+ butterfly_s16_s32_x4_0112_neon(cospi16, x3[3], x3[2], &output[4],
+ &output[12]);
+ butterfly_dct_post_s16_x4(x2 + 4, x3 + 4, x4 + 4, 4);
+ butterfly_s16_s32_x4_0112_neon(cospi16, x3[14], x3[9], &x4[14], &x4[9]);
+ butterfly_s16_s32_x4_1223_neon(cospi16, x3[13], x3[10], &x4[13], &x4[10]);
+
+ // stage 5
+ int16x4_t x5[16];
+ butterfly_s16_s32_x4_0112_neon(cospi8, x4[7], x4[4], &output[2], &output[14]);
+ butterfly_s16_s32_x4_1003_neon(cospi24, x4[6], x4[5], &output[10],
+ &output[6]);
+ butterfly_dct_post_s16_x4(x3 + 8, x4 + 8, x5 + 8, 4);
+ butterfly_dct_post_s16_x4(x3 + 12, x4 + 12, x5 + 12, 4);
+
+ // stage 6-7
+ butterfly_s16_s32_x4_0112_neon(cospi4, x5[15], x5[8], &output[1],
+ &output[15]);
+ butterfly_s16_s32_x4_1003_neon(cospi28, x5[14], x5[9], &output[9],
+ &output[7]);
+ butterfly_s16_s32_x4_0112_neon(cospi20, x5[13], x5[10], &output[5],
+ &output[11]);
+ butterfly_s16_s32_x4_1003_neon(cospi12, x5[12], x5[11], &output[13],
+ &output[3]);
}
-static void fdct8x16_neon(const int16x8_t *input, int16x8_t *output,
- int8_t cos_bit, const int8_t *stage_range) {
- (void)stage_range;
- const int32_t *cospi = cospi_arr(cos_bit);
- const int32x4_t v_cos_bit = vdupq_n_s32(-cos_bit);
+static AOM_FORCE_INLINE void fdct8x16_neon(const int16x8_t *input,
+ int16x8_t *output, int cos_bit) {
+ const int16_t *cospi = cospi_arr_q13(cos_bit);
+
+ const int16x8_t cospi32_16 = vld1q_s16(&cospi[4 * 0]);
+ const int16x8_t cospi8_24 = vld1q_s16(&cospi[4 * 2]);
+ const int16x8_t cospi4_12 = vld1q_s16(&cospi[4 * 4]);
+ const int16x8_t cospi20_28 = vld1q_s16(&cospi[4 * 6]);
+
+ const int16x4_t cospi32 = vget_low_s16(cospi32_16);
+ const int16x4_t cospi16 = vget_high_s16(cospi32_16);
+ const int16x4_t cospi8 = vget_low_s16(cospi8_24);
+ const int16x4_t cospi24 = vget_high_s16(cospi8_24);
+ const int16x4_t cospi4 = vget_low_s16(cospi4_12);
+ const int16x4_t cospi12 = vget_high_s16(cospi4_12);
+ const int16x4_t cospi20 = vget_low_s16(cospi20_28);
+ const int16x4_t cospi28 = vget_high_s16(cospi20_28);
// stage 1
int16x8_t x1[16];
- x1[0] = vqaddq_s16(input[0], input[15]);
- x1[15] = vqsubq_s16(input[0], input[15]);
- x1[1] = vqaddq_s16(input[1], input[14]);
- x1[14] = vqsubq_s16(input[1], input[14]);
- x1[2] = vqaddq_s16(input[2], input[13]);
- x1[13] = vqsubq_s16(input[2], input[13]);
- x1[3] = vqaddq_s16(input[3], input[12]);
- x1[12] = vqsubq_s16(input[3], input[12]);
- x1[4] = vqaddq_s16(input[4], input[11]);
- x1[11] = vqsubq_s16(input[4], input[11]);
- x1[5] = vqaddq_s16(input[5], input[10]);
- x1[10] = vqsubq_s16(input[5], input[10]);
- x1[6] = vqaddq_s16(input[6], input[9]);
- x1[9] = vqsubq_s16(input[6], input[9]);
- x1[7] = vqaddq_s16(input[7], input[8]);
- x1[8] = vqsubq_s16(input[7], input[8]);
+ butterfly_dct_pre_s16_x8(input, x1, 16);
// stage 2
int16x8_t x2[16];
- x2[0] = vqaddq_s16(x1[0], x1[7]);
- x2[7] = vqsubq_s16(x1[0], x1[7]);
- x2[1] = vqaddq_s16(x1[1], x1[6]);
- x2[6] = vqsubq_s16(x1[1], x1[6]);
- x2[2] = vqaddq_s16(x1[2], x1[5]);
- x2[5] = vqsubq_s16(x1[2], x1[5]);
- x2[3] = vqaddq_s16(x1[3], x1[4]);
- x2[4] = vqsubq_s16(x1[3], x1[4]);
-
- btf_16_neon_mode0(cospi[32], cospi[32], x1[10], x1[13], x2[10], x2[13],
- v_cos_bit);
- btf_16_neon_mode0(cospi[32], cospi[32], x1[11], x1[12], x2[11], x2[12],
- v_cos_bit);
+ butterfly_dct_pre_s16_x8(x1, x2, 8);
+ butterfly_s16_s32_x8_0112_neon(cospi32, x1[13], x1[10], &x2[13], &x2[10]);
+ butterfly_s16_s32_x8_0112_neon(cospi32, x1[12], x1[11], &x2[12], &x2[11]);
// stage 3
int16x8_t x3[16];
- x3[0] = vqaddq_s16(x2[0], x2[3]);
- x3[3] = vqsubq_s16(x2[0], x2[3]);
- x3[1] = vqaddq_s16(x2[1], x2[2]);
- x3[2] = vqsubq_s16(x2[1], x2[2]);
-
- btf_16_neon_mode0(cospi[32], cospi[32], x2[5], x2[6], x3[5], x3[6],
- v_cos_bit);
-
- x3[8] = vqaddq_s16(x1[8], x2[11]);
- x3[11] = vqsubq_s16(x1[8], x2[11]);
- x3[9] = vqaddq_s16(x1[9], x2[10]);
- x3[10] = vqsubq_s16(x1[9], x2[10]);
- x3[12] = vqsubq_s16(x1[15], x2[12]);
- x3[15] = vqaddq_s16(x1[15], x2[12]);
- x3[13] = vqsubq_s16(x1[14], x2[13]);
- x3[14] = vqaddq_s16(x1[14], x2[13]);
+ butterfly_dct_pre_s16_x8(x2, x3, 4);
+ butterfly_s16_s32_x8_0112_neon(cospi32, x2[6], x2[5], &x3[6], &x3[5]);
+ butterfly_dct_post_s16_x8(x1 + 8, x2 + 8, x3 + 8, 8);
// stage 4
int16x8_t x4[16];
- btf_16_neon(cospi[32], cospi[32], cospi[32], -cospi[32], x3[0], x3[1],
- output[0], output[8]);
- btf_16_neon(cospi[48], cospi[16], -cospi[16], cospi[48], x3[2], x3[3],
- output[4], output[12]);
- x4[4] = vqaddq_s16(x2[4], x3[5]);
- x4[5] = vqsubq_s16(x2[4], x3[5]);
- x4[6] = vqsubq_s16(x2[7], x3[6]);
- x4[7] = vqaddq_s16(x2[7], x3[6]);
- btf_16_neon_mode0(cospi[16], cospi[48], x3[9], x3[14], x4[9], x4[14],
- v_cos_bit);
- btf_16_neon_mode02(cospi[48], cospi[16], x3[10], x3[13], x4[10], x4[13],
- v_cos_bit);
+ butterfly_s16_s32_x8_0112_neon(cospi32, x3[0], x3[1], &output[0], &output[8]);
+ butterfly_s16_s32_x8_0112_neon(cospi16, x3[3], x3[2], &output[4],
+ &output[12]);
+ butterfly_dct_post_s16_x8(x2 + 4, x3 + 4, x4 + 4, 4);
+ butterfly_s16_s32_x8_0112_neon(cospi16, x3[14], x3[9], &x4[14], &x4[9]);
+ butterfly_s16_s32_x8_1223_neon(cospi16, x3[13], x3[10], &x4[13], &x4[10]);
// stage 5
int16x8_t x5[16];
-
- btf_16_neon_mode2(cospi[56], cospi[8], x4[4], x4[7], output[2], output[14],
- v_cos_bit);
- btf_16_neon_mode2(cospi[24], cospi[40], x4[5], x4[6], output[10], output[6],
- v_cos_bit);
- x5[8] = vqaddq_s16(x3[8], x4[9]);
- x5[9] = vqsubq_s16(x3[8], x4[9]);
- x5[10] = vqsubq_s16(x3[11], x4[10]);
- x5[11] = vqaddq_s16(x3[11], x4[10]);
- x5[12] = vqaddq_s16(x3[12], x4[13]);
- x5[13] = vqsubq_s16(x3[12], x4[13]);
- x5[14] = vqsubq_s16(x3[15], x4[14]);
- x5[15] = vqaddq_s16(x3[15], x4[14]);
+ butterfly_s16_s32_x8_0112_neon(cospi8, x4[7], x4[4], &output[2], &output[14]);
+ butterfly_s16_s32_x8_1003_neon(cospi24, x4[6], x4[5], &output[10],
+ &output[6]);
+ butterfly_dct_post_s16_x8(x3 + 8, x4 + 8, x5 + 8, 4);
+ butterfly_dct_post_s16_x8(x3 + 12, x4 + 12, x5 + 12, 4);
// stage 6-7
- btf_16_neon_mode2(cospi[60], cospi[4], x5[8], x5[15], output[1], output[15],
- v_cos_bit);
- btf_16_neon_mode2(cospi[28], cospi[36], x5[9], x5[14], output[9], output[7],
- v_cos_bit);
- btf_16_neon_mode2(cospi[44], cospi[20], x5[10], x5[13], output[5], output[11],
- v_cos_bit);
- btf_16_neon_mode2(cospi[12], cospi[52], x5[11], x5[12], output[13], output[3],
- v_cos_bit);
+ butterfly_s16_s32_x8_0112_neon(cospi4, x5[15], x5[8], &output[1],
+ &output[15]);
+ butterfly_s16_s32_x8_1003_neon(cospi28, x5[14], x5[9], &output[9],
+ &output[7]);
+ butterfly_s16_s32_x8_0112_neon(cospi20, x5[13], x5[10], &output[5],
+ &output[11]);
+ butterfly_s16_s32_x8_1003_neon(cospi12, x5[12], x5[11], &output[13],
+ &output[3]);
}
-void av1_fdct8x32_neon(const int16x8_t *input, int16x8_t *output,
- int8_t cos_bit, const int8_t *stage_range) {
- (void)stage_range;
- const int32_t *cospi = cospi_arr(cos_bit);
- const int32x4_t v_cos_bit = vdupq_n_s32(-cos_bit);
+static AOM_FORCE_INLINE void fdct8x32_neon(const int16x8_t *input,
+ int16x8_t *output, int cos_bit) {
+ const int16_t *cospi = cospi_arr_q13(cos_bit);
+
+ const int16x8_t cospi32_16 = vld1q_s16(&cospi[4 * 0]);
+ const int16x8_t cospi8_24 = vld1q_s16(&cospi[4 * 2]);
+ const int16x8_t cospi4_12 = vld1q_s16(&cospi[4 * 4]);
+ const int16x8_t cospi20_28 = vld1q_s16(&cospi[4 * 6]);
+ const int16x8_t cospi2_6 = vld1q_s16(&cospi[4 * 8]);
+ const int16x8_t cospi10_14 = vld1q_s16(&cospi[4 * 10]);
+ const int16x8_t cospi18_22 = vld1q_s16(&cospi[4 * 12]);
+ const int16x8_t cospi26_30 = vld1q_s16(&cospi[4 * 14]);
+
+ const int16x4_t cospi32 = vget_low_s16(cospi32_16);
+ const int16x4_t cospi16 = vget_high_s16(cospi32_16);
+ const int16x4_t cospi8 = vget_low_s16(cospi8_24);
+ const int16x4_t cospi24 = vget_high_s16(cospi8_24);
+ const int16x4_t cospi4 = vget_low_s16(cospi4_12);
+ const int16x4_t cospi12 = vget_high_s16(cospi4_12);
+ const int16x4_t cospi20 = vget_low_s16(cospi20_28);
+ const int16x4_t cospi28 = vget_high_s16(cospi20_28);
+ const int16x4_t cospi2 = vget_low_s16(cospi2_6);
+ const int16x4_t cospi6 = vget_high_s16(cospi2_6);
+ const int16x4_t cospi10 = vget_low_s16(cospi10_14);
+ const int16x4_t cospi14 = vget_high_s16(cospi10_14);
+ const int16x4_t cospi18 = vget_low_s16(cospi18_22);
+ const int16x4_t cospi22 = vget_high_s16(cospi18_22);
+ const int16x4_t cospi26 = vget_low_s16(cospi26_30);
+ const int16x4_t cospi30 = vget_high_s16(cospi26_30);
// stage 1
int16x8_t x1[32];
- x1[0] = vqaddq_s16(input[0], input[31]);
- x1[31] = vqsubq_s16(input[0], input[31]);
- x1[1] = vqaddq_s16(input[1], input[30]);
- x1[30] = vqsubq_s16(input[1], input[30]);
- x1[2] = vqaddq_s16(input[2], input[29]);
- x1[29] = vqsubq_s16(input[2], input[29]);
- x1[3] = vqaddq_s16(input[3], input[28]);
- x1[28] = vqsubq_s16(input[3], input[28]);
- x1[4] = vqaddq_s16(input[4], input[27]);
- x1[27] = vqsubq_s16(input[4], input[27]);
- x1[5] = vqaddq_s16(input[5], input[26]);
- x1[26] = vqsubq_s16(input[5], input[26]);
- x1[6] = vqaddq_s16(input[6], input[25]);
- x1[25] = vqsubq_s16(input[6], input[25]);
- x1[7] = vqaddq_s16(input[7], input[24]);
- x1[24] = vqsubq_s16(input[7], input[24]);
- x1[8] = vqaddq_s16(input[8], input[23]);
- x1[23] = vqsubq_s16(input[8], input[23]);
- x1[9] = vqaddq_s16(input[9], input[22]);
- x1[22] = vqsubq_s16(input[9], input[22]);
- x1[10] = vqaddq_s16(input[10], input[21]);
- x1[21] = vqsubq_s16(input[10], input[21]);
- x1[11] = vqaddq_s16(input[11], input[20]);
- x1[20] = vqsubq_s16(input[11], input[20]);
- x1[12] = vqaddq_s16(input[12], input[19]);
- x1[19] = vqsubq_s16(input[12], input[19]);
- x1[13] = vqaddq_s16(input[13], input[18]);
- x1[18] = vqsubq_s16(input[13], input[18]);
- x1[14] = vqaddq_s16(input[14], input[17]);
- x1[17] = vqsubq_s16(input[14], input[17]);
- x1[15] = vqaddq_s16(input[15], input[16]);
- x1[16] = vqsubq_s16(input[15], input[16]);
+ butterfly_dct_pre_s16_x8(input, x1, 32);
// stage 2
int16x8_t x2[32];
- x2[0] = vqaddq_s16(x1[0], x1[15]);
- x2[15] = vqsubq_s16(x1[0], x1[15]);
- x2[1] = vqaddq_s16(x1[1], x1[14]);
- x2[14] = vqsubq_s16(x1[1], x1[14]);
- x2[2] = vqaddq_s16(x1[2], x1[13]);
- x2[13] = vqsubq_s16(x1[2], x1[13]);
- x2[3] = vqaddq_s16(x1[3], x1[12]);
- x2[12] = vqsubq_s16(x1[3], x1[12]);
- x2[4] = vqaddq_s16(x1[4], x1[11]);
- x2[11] = vqsubq_s16(x1[4], x1[11]);
- x2[5] = vqaddq_s16(x1[5], x1[10]);
- x2[10] = vqsubq_s16(x1[5], x1[10]);
- x2[6] = vqaddq_s16(x1[6], x1[9]);
- x2[9] = vqsubq_s16(x1[6], x1[9]);
- x2[7] = vqaddq_s16(x1[7], x1[8]);
- x2[8] = vqsubq_s16(x1[7], x1[8]);
-
- btf_16_neon_mode0(cospi[32], cospi[32], x1[20], x1[27], x2[20], x2[27],
- v_cos_bit);
- btf_16_neon_mode0(cospi[32], cospi[32], x1[21], x1[26], x2[21], x2[26],
- v_cos_bit);
- btf_16_neon_mode0(cospi[32], cospi[32], x1[22], x1[25], x2[22], x2[25],
- v_cos_bit);
- btf_16_neon_mode0(cospi[32], cospi[32], x1[23], x1[24], x2[23], x2[24],
- v_cos_bit);
+ butterfly_dct_pre_s16_x8(x1, x2, 16);
+ butterfly_s16_s32_x8_0112_neon(cospi32, x1[27], x1[20], &x2[27], &x2[20]);
+ butterfly_s16_s32_x8_0112_neon(cospi32, x1[26], x1[21], &x2[26], &x2[21]);
+ butterfly_s16_s32_x8_0112_neon(cospi32, x1[25], x1[22], &x2[25], &x2[22]);
+ butterfly_s16_s32_x8_0112_neon(cospi32, x1[24], x1[23], &x2[24], &x2[23]);
// stage 3
int16x8_t x3[32];
- x3[0] = vqaddq_s16(x2[0], x2[7]);
- x3[7] = vqsubq_s16(x2[0], x2[7]);
- x3[1] = vqaddq_s16(x2[1], x2[6]);
- x3[6] = vqsubq_s16(x2[1], x2[6]);
- x3[2] = vqaddq_s16(x2[2], x2[5]);
- x3[5] = vqsubq_s16(x2[2], x2[5]);
- x3[3] = vqaddq_s16(x2[3], x2[4]);
- x3[4] = vqsubq_s16(x2[3], x2[4]);
-
- btf_16_neon_mode0(cospi[32], cospi[32], x2[10], x2[13], x3[10], x3[13],
- v_cos_bit);
- btf_16_neon_mode0(cospi[32], cospi[32], x2[11], x2[12], x3[11], x3[12],
- v_cos_bit);
-
- x3[16] = vqaddq_s16(x1[16], x2[23]);
- x3[23] = vqsubq_s16(x1[16], x2[23]);
- x3[17] = vqaddq_s16(x1[17], x2[22]);
- x3[22] = vqsubq_s16(x1[17], x2[22]);
- x3[18] = vqaddq_s16(x1[18], x2[21]);
- x3[21] = vqsubq_s16(x1[18], x2[21]);
- x3[19] = vqaddq_s16(x1[19], x2[20]);
- x3[20] = vqsubq_s16(x1[19], x2[20]);
- x3[24] = vqsubq_s16(x1[31], x2[24]);
- x3[31] = vqaddq_s16(x1[31], x2[24]);
- x3[25] = vqsubq_s16(x1[30], x2[25]);
- x3[30] = vqaddq_s16(x1[30], x2[25]);
- x3[26] = vqsubq_s16(x1[29], x2[26]);
- x3[29] = vqaddq_s16(x1[29], x2[26]);
- x3[27] = vqsubq_s16(x1[28], x2[27]);
- x3[28] = vqaddq_s16(x1[28], x2[27]);
+ butterfly_dct_pre_s16_x8(x2, x3, 8);
+ butterfly_s16_s32_x8_0112_neon(cospi32, x2[13], x2[10], &x3[13], &x3[10]);
+ butterfly_s16_s32_x8_0112_neon(cospi32, x2[12], x2[11], &x3[12], &x3[11]);
+ butterfly_dct_post_s16_x8(x1 + 16, x2 + 16, x3 + 16, 16);
// stage 4
int16x8_t x4[32];
- x4[0] = vqaddq_s16(x3[0], x3[3]);
- x4[3] = vqsubq_s16(x3[0], x3[3]);
- x4[1] = vqaddq_s16(x3[1], x3[2]);
- x4[2] = vqsubq_s16(x3[1], x3[2]);
- btf_16_neon_mode0(cospi[32], cospi[32], x3[5], x3[6], x4[5], x4[6],
- v_cos_bit);
- x4[8] = vqaddq_s16(x2[8], x3[11]);
- x4[11] = vqsubq_s16(x2[8], x3[11]);
- x4[9] = vqaddq_s16(x2[9], x3[10]);
- x4[10] = vqsubq_s16(x2[9], x3[10]);
- x4[12] = vqsubq_s16(x2[15], x3[12]);
- x4[15] = vqaddq_s16(x2[15], x3[12]);
- x4[13] = vqsubq_s16(x2[14], x3[13]);
- x4[14] = vqaddq_s16(x2[14], x3[13]);
-
- btf_16_neon_mode0(cospi[16], cospi[48], x3[18], x3[29], x4[18], x4[29],
- v_cos_bit);
- btf_16_neon_mode0(cospi[16], cospi[48], x3[19], x3[28], x4[19], x4[28],
- v_cos_bit);
- btf_16_neon_mode02(cospi[48], cospi[16], x3[20], x3[27], x4[20], x4[27],
- v_cos_bit);
- btf_16_neon_mode02(cospi[48], cospi[16], x3[21], x3[26], x4[21], x4[26],
- v_cos_bit);
+ butterfly_dct_pre_s16_x8(x3, x4, 4);
+ butterfly_s16_s32_x8_0112_neon(cospi32, x3[6], x3[5], &x4[6], &x4[5]);
+ butterfly_dct_post_s16_x8(x2 + 8, x3 + 8, x4 + 8, 8);
+ butterfly_s16_s32_x8_0112_neon(cospi16, x3[29], x3[18], &x4[29], &x4[18]);
+ butterfly_s16_s32_x8_0112_neon(cospi16, x3[28], x3[19], &x4[28], &x4[19]);
+ butterfly_s16_s32_x8_1223_neon(cospi16, x3[27], x3[20], &x4[27], &x4[20]);
+ butterfly_s16_s32_x8_1223_neon(cospi16, x3[26], x3[21], &x4[26], &x4[21]);
// stage 5
int16x8_t x5[32];
- btf_16_neon_mode3(cospi[32], cospi[32], x4[0], x4[1], output[0], output[16],
- v_cos_bit);
- btf_16_neon_mode2(cospi[48], cospi[16], x4[2], x4[3], output[8], output[24],
- v_cos_bit);
- x5[4] = vqaddq_s16(x3[4], x4[5]);
- x5[5] = vqsubq_s16(x3[4], x4[5]);
- x5[6] = vqsubq_s16(x3[7], x4[6]);
- x5[7] = vqaddq_s16(x3[7], x4[6]);
-
- btf_16_neon_mode0(cospi[16], cospi[48], x4[9], x4[14], x5[9], x5[14],
- v_cos_bit);
- btf_16_neon_mode02(cospi[48], cospi[16], x4[10], x4[13], x5[10], x5[13],
- v_cos_bit);
-
- x5[16] = vqaddq_s16(x3[16], x4[19]);
- x5[19] = vqsubq_s16(x3[16], x4[19]);
- x5[17] = vqaddq_s16(x3[17], x4[18]);
- x5[18] = vqsubq_s16(x3[17], x4[18]);
- x5[20] = vqsubq_s16(x3[23], x4[20]);
- x5[23] = vqaddq_s16(x3[23], x4[20]);
- x5[21] = vqsubq_s16(x3[22], x4[21]);
- x5[22] = vqaddq_s16(x3[22], x4[21]);
- x5[24] = vqaddq_s16(x3[24], x4[27]);
- x5[27] = vqsubq_s16(x3[24], x4[27]);
- x5[25] = vqaddq_s16(x3[25], x4[26]);
- x5[26] = vqsubq_s16(x3[25], x4[26]);
- x5[28] = vqsubq_s16(x3[31], x4[28]);
- x5[31] = vqaddq_s16(x3[31], x4[28]);
- x5[29] = vqsubq_s16(x3[30], x4[29]);
- x5[30] = vqaddq_s16(x3[30], x4[29]);
+ butterfly_s16_s32_x8_0112_neon(cospi32, x4[0], x4[1], &output[0],
+ &output[16]);
+ butterfly_s16_s32_x8_0112_neon(cospi16, x4[3], x4[2], &output[8],
+ &output[24]);
+ butterfly_dct_post_s16_x8(x3 + 4, x4 + 4, x5 + 4, 4);
+ butterfly_s16_s32_x8_0112_neon(cospi16, x4[14], x4[9], &x5[14], &x5[9]);
+ butterfly_s16_s32_x8_1223_neon(cospi16, x4[13], x4[10], &x5[13], &x5[10]);
+ butterfly_dct_post_s16_x8(x3 + 16, x4 + 16, x5 + 16, 8);
+ butterfly_dct_post_s16_x8(x3 + 24, x4 + 24, x5 + 24, 8);
// stage 6
int16x8_t x6[32];
- btf_16_neon_mode2(cospi[56], cospi[8], x5[4], x5[7], output[4], output[28],
- v_cos_bit);
- btf_16_neon_mode2(cospi[24], cospi[40], x5[5], x5[6], output[20], output[12],
- v_cos_bit);
- x6[8] = vqaddq_s16(x4[8], x5[9]);
- x6[9] = vqsubq_s16(x4[8], x5[9]);
- x6[10] = vqsubq_s16(x4[11], x5[10]);
- x6[11] = vqaddq_s16(x4[11], x5[10]);
- x6[12] = vqaddq_s16(x4[12], x5[13]);
- x6[13] = vqsubq_s16(x4[12], x5[13]);
- x6[14] = vqsubq_s16(x4[15], x5[14]);
- x6[15] = vqaddq_s16(x4[15], x5[14]);
- btf_16_neon_mode0(cospi[8], cospi[56], x5[17], x5[30], x6[17], x6[30],
- v_cos_bit);
- btf_16_neon_mode02(cospi[56], cospi[8], x5[18], x5[29], x6[18], x6[29],
- v_cos_bit);
- btf_16_neon_mode0(cospi[40], cospi[24], x5[21], x5[26], x6[21], x6[26],
- v_cos_bit);
- btf_16_neon_mode02(cospi[24], cospi[40], x5[22], x5[25], x6[22], x6[25],
- v_cos_bit);
+ butterfly_s16_s32_x8_0112_neon(cospi8, x5[7], x5[4], &output[4], &output[28]);
+ butterfly_s16_s32_x8_1003_neon(cospi24, x5[6], x5[5], &output[20],
+ &output[12]);
+ butterfly_dct_post_s16_x8(x4 + 8, x5 + 8, x6 + 8, 4);
+ butterfly_dct_post_s16_x8(x4 + 12, x5 + 12, x6 + 12, 4);
+ butterfly_s16_s32_x8_0112_neon(cospi8, x5[30], x5[17], &x6[30], &x6[17]);
+ butterfly_s16_s32_x8_1223_neon(cospi8, x5[29], x5[18], &x6[29], &x6[18]);
+ butterfly_s16_s32_x8_1003_neon(cospi24, x5[26], x5[21], &x6[26], &x6[21]);
+ butterfly_s16_s32_x8_0332_neon(cospi24, x5[25], x5[22], &x6[25], &x6[22]);
// stage 7
int16x8_t x7[32];
- btf_16_neon_mode2(cospi[60], cospi[4], x6[8], x6[15], output[2], output[30],
- v_cos_bit);
- btf_16_neon_mode2(cospi[28], cospi[36], x6[9], x6[14], output[18], output[14],
- v_cos_bit);
- btf_16_neon_mode2(cospi[44], cospi[20], x6[10], x6[13], output[10],
- output[22], v_cos_bit);
- btf_16_neon_mode2(cospi[12], cospi[52], x6[11], x6[12], output[26], output[6],
- v_cos_bit);
- x7[16] = vqaddq_s16(x5[16], x6[17]);
- x7[17] = vqsubq_s16(x5[16], x6[17]);
- x7[18] = vqsubq_s16(x5[19], x6[18]);
- x7[19] = vqaddq_s16(x5[19], x6[18]);
- x7[20] = vqaddq_s16(x5[20], x6[21]);
- x7[21] = vqsubq_s16(x5[20], x6[21]);
- x7[22] = vqsubq_s16(x5[23], x6[22]);
- x7[23] = vqaddq_s16(x5[23], x6[22]);
- x7[24] = vqaddq_s16(x5[24], x6[25]);
- x7[25] = vqsubq_s16(x5[24], x6[25]);
- x7[26] = vqsubq_s16(x5[27], x6[26]);
- x7[27] = vqaddq_s16(x5[27], x6[26]);
- x7[28] = vqaddq_s16(x5[28], x6[29]);
- x7[29] = vqsubq_s16(x5[28], x6[29]);
- x7[30] = vqsubq_s16(x5[31], x6[30]);
- x7[31] = vqaddq_s16(x5[31], x6[30]);
+ butterfly_s16_s32_x8_0112_neon(cospi4, x6[15], x6[8], &output[2],
+ &output[30]);
+ butterfly_s16_s32_x8_1003_neon(cospi28, x6[14], x6[9], &output[18],
+ &output[14]);
+ butterfly_s16_s32_x8_0112_neon(cospi20, x6[13], x6[10], &output[10],
+ &output[22]);
+ butterfly_s16_s32_x8_1003_neon(cospi12, x6[12], x6[11], &output[26],
+ &output[6]);
+ butterfly_dct_post_s16_x8(x5 + 16, x6 + 16, x7 + 16, 4);
+ butterfly_dct_post_s16_x8(x5 + 20, x6 + 20, x7 + 20, 4);
+ butterfly_dct_post_s16_x8(x5 + 24, x6 + 24, x7 + 24, 4);
+ butterfly_dct_post_s16_x8(x5 + 28, x6 + 28, x7 + 28, 4);
- btf_16_neon_mode2(cospi[62], cospi[2], x7[16], x7[31], output[1], output[31],
- v_cos_bit);
- btf_16_neon_mode2(cospi[30], cospi[34], x7[17], x7[30], output[17],
- output[15], v_cos_bit);
- btf_16_neon_mode2(cospi[46], cospi[18], x7[18], x7[29], output[9], output[23],
- v_cos_bit);
- btf_16_neon_mode2(cospi[14], cospi[50], x7[19], x7[28], output[25], output[7],
- v_cos_bit);
- btf_16_neon_mode2(cospi[54], cospi[10], x7[20], x7[27], output[5], output[27],
- v_cos_bit);
- btf_16_neon_mode2(cospi[22], cospi[42], x7[21], x7[26], output[21],
- output[11], v_cos_bit);
- btf_16_neon_mode2(cospi[38], cospi[26], x7[22], x7[25], output[13],
- output[19], v_cos_bit);
- btf_16_neon_mode2(cospi[6], cospi[58], x7[23], x7[24], output[29], output[3],
- v_cos_bit);
+ butterfly_s16_s32_x8_0112_neon(cospi2, x7[31], x7[16], &output[1],
+ &output[31]);
+ butterfly_s16_s32_x8_1003_neon(cospi30, x7[30], x7[17], &output[17],
+ &output[15]);
+ butterfly_s16_s32_x8_0112_neon(cospi18, x7[29], x7[18], &output[9],
+ &output[23]);
+ butterfly_s16_s32_x8_1003_neon(cospi14, x7[28], x7[19], &output[25],
+ &output[7]);
+ butterfly_s16_s32_x8_0112_neon(cospi10, x7[27], x7[20], &output[5],
+ &output[27]);
+ butterfly_s16_s32_x8_1003_neon(cospi22, x7[26], x7[21], &output[21],
+ &output[11]);
+ butterfly_s16_s32_x8_0112_neon(cospi26, x7[25], x7[22], &output[13],
+ &output[19]);
+ butterfly_s16_s32_x8_1003_neon(cospi6, x7[24], x7[23], &output[29],
+ &output[3]);
}
-void av1_fdct8x64_stage_1234_neon(const int16x8_t *input, int16x8_t *x3,
- int16x8_t *x4, const int32_t *cospi32,
- const int32x4_t *v_cos_bit) {
+static AOM_FORCE_INLINE void fdct8x64_neon(const int16x8_t *input,
+ int16x8_t *output, int cos_bit) {
+ const int16_t *cospi = cospi_arr_q13(cos_bit);
+
+ const int16x8_t cospi32_16 = vld1q_s16(&cospi[4 * 0]);
+ const int16x8_t cospi8_24 = vld1q_s16(&cospi[4 * 2]);
+ const int16x8_t cospi4_12 = vld1q_s16(&cospi[4 * 4]);
+ const int16x8_t cospi20_28 = vld1q_s16(&cospi[4 * 6]);
+ const int16x8_t cospi2_6 = vld1q_s16(&cospi[4 * 8]);
+ const int16x8_t cospi10_14 = vld1q_s16(&cospi[4 * 10]);
+ const int16x8_t cospi18_22 = vld1q_s16(&cospi[4 * 12]);
+ const int16x8_t cospi26_30 = vld1q_s16(&cospi[4 * 14]);
+ const int16x8_t cospi1_3 = vld1q_s16(&cospi[4 * 16]);
+ const int16x8_t cospi5_7 = vld1q_s16(&cospi[4 * 18]);
+ const int16x8_t cospi9_11 = vld1q_s16(&cospi[4 * 20]);
+ const int16x8_t cospi13_15 = vld1q_s16(&cospi[4 * 22]);
+ const int16x8_t cospi17_19 = vld1q_s16(&cospi[4 * 24]);
+ const int16x8_t cospi21_23 = vld1q_s16(&cospi[4 * 26]);
+ const int16x8_t cospi25_27 = vld1q_s16(&cospi[4 * 28]);
+ const int16x8_t cospi29_31 = vld1q_s16(&cospi[4 * 30]);
+
+ const int16x4_t cospi32 = vget_low_s16(cospi32_16);
+ const int16x4_t cospi16 = vget_high_s16(cospi32_16);
+ const int16x4_t cospi8 = vget_low_s16(cospi8_24);
+ const int16x4_t cospi24 = vget_high_s16(cospi8_24);
+ const int16x4_t cospi4 = vget_low_s16(cospi4_12);
+ const int16x4_t cospi12 = vget_high_s16(cospi4_12);
+ const int16x4_t cospi20 = vget_low_s16(cospi20_28);
+ const int16x4_t cospi28 = vget_high_s16(cospi20_28);
+ const int16x4_t cospi2 = vget_low_s16(cospi2_6);
+ const int16x4_t cospi6 = vget_high_s16(cospi2_6);
+ const int16x4_t cospi10 = vget_low_s16(cospi10_14);
+ const int16x4_t cospi14 = vget_high_s16(cospi10_14);
+ const int16x4_t cospi18 = vget_low_s16(cospi18_22);
+ const int16x4_t cospi22 = vget_high_s16(cospi18_22);
+ const int16x4_t cospi26 = vget_low_s16(cospi26_30);
+ const int16x4_t cospi30 = vget_high_s16(cospi26_30);
+ const int16x4_t cospi1 = vget_low_s16(cospi1_3);
+ const int16x4_t cospi3 = vget_high_s16(cospi1_3);
+ const int16x4_t cospi5 = vget_low_s16(cospi5_7);
+ const int16x4_t cospi7 = vget_high_s16(cospi5_7);
+ const int16x4_t cospi9 = vget_low_s16(cospi9_11);
+ const int16x4_t cospi11 = vget_high_s16(cospi9_11);
+ const int16x4_t cospi13 = vget_low_s16(cospi13_15);
+ const int16x4_t cospi15 = vget_high_s16(cospi13_15);
+ const int16x4_t cospi17 = vget_low_s16(cospi17_19);
+ const int16x4_t cospi19 = vget_high_s16(cospi17_19);
+ const int16x4_t cospi21 = vget_low_s16(cospi21_23);
+ const int16x4_t cospi23 = vget_high_s16(cospi21_23);
+ const int16x4_t cospi25 = vget_low_s16(cospi25_27);
+ const int16x4_t cospi27 = vget_high_s16(cospi25_27);
+ const int16x4_t cospi29 = vget_low_s16(cospi29_31);
+ const int16x4_t cospi31 = vget_high_s16(cospi29_31);
+
+ // stage 1
int16x8_t x1[64];
+ butterfly_dct_pre_s16_x8(input, x1, 64);
+
+ // stage 2
int16x8_t x2[64];
- x1[0] = vqaddq_s16(input[0], input[63]);
- x1[63] = vqsubq_s16(input[0], input[63]);
- x1[1] = vqaddq_s16(input[1], input[62]);
- x1[62] = vqsubq_s16(input[1], input[62]);
- x1[2] = vqaddq_s16(input[2], input[61]);
- x1[61] = vqsubq_s16(input[2], input[61]);
- x1[3] = vqaddq_s16(input[3], input[60]);
- x1[60] = vqsubq_s16(input[3], input[60]);
- x1[4] = vqaddq_s16(input[4], input[59]);
- x1[59] = vqsubq_s16(input[4], input[59]);
- x1[5] = vqaddq_s16(input[5], input[58]);
- x1[58] = vqsubq_s16(input[5], input[58]);
- x1[6] = vqaddq_s16(input[6], input[57]);
- x1[57] = vqsubq_s16(input[6], input[57]);
- x1[7] = vqaddq_s16(input[7], input[56]);
- x1[56] = vqsubq_s16(input[7], input[56]);
- x1[8] = vqaddq_s16(input[8], input[55]);
- x1[55] = vqsubq_s16(input[8], input[55]);
- x1[9] = vqaddq_s16(input[9], input[54]);
- x1[54] = vqsubq_s16(input[9], input[54]);
- x1[10] = vqaddq_s16(input[10], input[53]);
- x1[53] = vqsubq_s16(input[10], input[53]);
- x1[11] = vqaddq_s16(input[11], input[52]);
- x1[52] = vqsubq_s16(input[11], input[52]);
- x1[12] = vqaddq_s16(input[12], input[51]);
- x1[51] = vqsubq_s16(input[12], input[51]);
- x1[13] = vqaddq_s16(input[13], input[50]);
- x1[50] = vqsubq_s16(input[13], input[50]);
- x1[14] = vqaddq_s16(input[14], input[49]);
- x1[49] = vqsubq_s16(input[14], input[49]);
- x1[15] = vqaddq_s16(input[15], input[48]);
- x1[48] = vqsubq_s16(input[15], input[48]);
- x1[16] = vqaddq_s16(input[16], input[47]);
- x1[47] = vqsubq_s16(input[16], input[47]);
- x1[17] = vqaddq_s16(input[17], input[46]);
- x1[46] = vqsubq_s16(input[17], input[46]);
- x1[18] = vqaddq_s16(input[18], input[45]);
- x1[45] = vqsubq_s16(input[18], input[45]);
- x1[19] = vqaddq_s16(input[19], input[44]);
- x1[44] = vqsubq_s16(input[19], input[44]);
- x1[20] = vqaddq_s16(input[20], input[43]);
- x1[43] = vqsubq_s16(input[20], input[43]);
- x1[21] = vqaddq_s16(input[21], input[42]);
- x1[42] = vqsubq_s16(input[21], input[42]);
- x1[22] = vqaddq_s16(input[22], input[41]);
- x1[41] = vqsubq_s16(input[22], input[41]);
- x1[23] = vqaddq_s16(input[23], input[40]);
- x1[40] = vqsubq_s16(input[23], input[40]);
- x1[24] = vqaddq_s16(input[24], input[39]);
- x1[39] = vqsubq_s16(input[24], input[39]);
- x1[25] = vqaddq_s16(input[25], input[38]);
- x1[38] = vqsubq_s16(input[25], input[38]);
- x1[26] = vqaddq_s16(input[26], input[37]);
- x1[37] = vqsubq_s16(input[26], input[37]);
- x1[27] = vqaddq_s16(input[27], input[36]);
- x1[36] = vqsubq_s16(input[27], input[36]);
- x1[28] = vqaddq_s16(input[28], input[35]);
- x1[35] = vqsubq_s16(input[28], input[35]);
- x1[29] = vqaddq_s16(input[29], input[34]);
- x1[34] = vqsubq_s16(input[29], input[34]);
- x1[30] = vqaddq_s16(input[30], input[33]);
- x1[33] = vqsubq_s16(input[30], input[33]);
- x1[31] = vqaddq_s16(input[31], input[32]);
- x1[32] = vqsubq_s16(input[31], input[32]);
-
- x2[0] = vqaddq_s16(x1[0], x1[31]);
- x2[31] = vqsubq_s16(x1[0], x1[31]);
- x2[1] = vqaddq_s16(x1[1], x1[30]);
- x2[30] = vqsubq_s16(x1[1], x1[30]);
- x2[2] = vqaddq_s16(x1[2], x1[29]);
- x2[29] = vqsubq_s16(x1[2], x1[29]);
- x2[3] = vqaddq_s16(x1[3], x1[28]);
- x2[28] = vqsubq_s16(x1[3], x1[28]);
- x2[4] = vqaddq_s16(x1[4], x1[27]);
- x2[27] = vqsubq_s16(x1[4], x1[27]);
- x2[5] = vqaddq_s16(x1[5], x1[26]);
- x2[26] = vqsubq_s16(x1[5], x1[26]);
- x2[6] = vqaddq_s16(x1[6], x1[25]);
- x2[25] = vqsubq_s16(x1[6], x1[25]);
- x2[7] = vqaddq_s16(x1[7], x1[24]);
- x2[24] = vqsubq_s16(x1[7], x1[24]);
- x2[8] = vqaddq_s16(x1[8], x1[23]);
- x2[23] = vqsubq_s16(x1[8], x1[23]);
- x2[9] = vqaddq_s16(x1[9], x1[22]);
- x2[22] = vqsubq_s16(x1[9], x1[22]);
- x2[10] = vqaddq_s16(x1[10], x1[21]);
- x2[21] = vqsubq_s16(x1[10], x1[21]);
- x2[11] = vqaddq_s16(x1[11], x1[20]);
- x2[20] = vqsubq_s16(x1[11], x1[20]);
- x2[12] = vqaddq_s16(x1[12], x1[19]);
- x2[19] = vqsubq_s16(x1[12], x1[19]);
- x2[13] = vqaddq_s16(x1[13], x1[18]);
- x2[18] = vqsubq_s16(x1[13], x1[18]);
- x2[14] = vqaddq_s16(x1[14], x1[17]);
- x2[17] = vqsubq_s16(x1[14], x1[17]);
- x2[15] = vqaddq_s16(x1[15], x1[16]);
- x2[16] = vqsubq_s16(x1[15], x1[16]);
-
- btf_16_neon_mode0(*cospi32, *cospi32, x1[40], x1[55], x2[40], x2[55],
- *v_cos_bit);
- btf_16_neon_mode0(*cospi32, *cospi32, x1[41], x1[54], x2[41], x2[54],
- *v_cos_bit);
- btf_16_neon_mode0(*cospi32, *cospi32, x1[42], x1[53], x2[42], x2[53],
- *v_cos_bit);
- btf_16_neon_mode0(*cospi32, *cospi32, x1[43], x1[52], x2[43], x2[52],
- *v_cos_bit);
- btf_16_neon_mode0(*cospi32, *cospi32, x1[44], x1[51], x2[44], x2[51],
- *v_cos_bit);
- btf_16_neon_mode0(*cospi32, *cospi32, x1[45], x1[50], x2[45], x2[50],
- *v_cos_bit);
- btf_16_neon_mode0(*cospi32, *cospi32, x1[46], x1[49], x2[46], x2[49],
- *v_cos_bit);
- btf_16_neon_mode0(*cospi32, *cospi32, x1[47], x1[48], x2[47], x2[48],
- *v_cos_bit);
+ butterfly_dct_pre_s16_x8(x1, x2, 32);
+ butterfly_s16_s32_x8_0112_neon(cospi32, x1[55], x1[40], &x2[55], &x2[40]);
+ butterfly_s16_s32_x8_0112_neon(cospi32, x1[54], x1[41], &x2[54], &x2[41]);
+ butterfly_s16_s32_x8_0112_neon(cospi32, x1[53], x1[42], &x2[53], &x2[42]);
+ butterfly_s16_s32_x8_0112_neon(cospi32, x1[52], x1[43], &x2[52], &x2[43]);
+ butterfly_s16_s32_x8_0112_neon(cospi32, x1[51], x1[44], &x2[51], &x2[44]);
+ butterfly_s16_s32_x8_0112_neon(cospi32, x1[50], x1[45], &x2[50], &x2[45]);
+ butterfly_s16_s32_x8_0112_neon(cospi32, x1[49], x1[46], &x2[49], &x2[46]);
+ butterfly_s16_s32_x8_0112_neon(cospi32, x1[48], x1[47], &x2[48], &x2[47]);
// stage 3
- x3[0] = vqaddq_s16(x2[0], x2[15]);
- x3[15] = vqsubq_s16(x2[0], x2[15]);
- x3[1] = vqaddq_s16(x2[1], x2[14]);
- x3[14] = vqsubq_s16(x2[1], x2[14]);
- x3[2] = vqaddq_s16(x2[2], x2[13]);
- x3[13] = vqsubq_s16(x2[2], x2[13]);
- x3[3] = vqaddq_s16(x2[3], x2[12]);
- x3[12] = vqsubq_s16(x2[3], x2[12]);
- x3[4] = vqaddq_s16(x2[4], x2[11]);
- x3[11] = vqsubq_s16(x2[4], x2[11]);
- x3[5] = vqaddq_s16(x2[5], x2[10]);
- x3[10] = vqsubq_s16(x2[5], x2[10]);
- x3[6] = vqaddq_s16(x2[6], x2[9]);
- x3[9] = vqsubq_s16(x2[6], x2[9]);
- x3[7] = vqaddq_s16(x2[7], x2[8]);
- x3[8] = vqsubq_s16(x2[7], x2[8]);
+ int16x8_t x3[64];
+ butterfly_dct_pre_s16_x8(x2, x3, 16);
x3[16] = x2[16];
x3[17] = x2[17];
x3[18] = x2[18];
x3[19] = x2[19];
- btf_16_neon_mode0(*cospi32, *cospi32, x2[20], x2[27], x3[20], x3[27],
- *v_cos_bit);
- btf_16_neon_mode0(*cospi32, *cospi32, x2[21], x2[26], x3[21], x3[26],
- *v_cos_bit);
- btf_16_neon_mode0(*cospi32, *cospi32, x2[22], x2[25], x3[22], x3[25],
- *v_cos_bit);
- btf_16_neon_mode0(*cospi32, *cospi32, x2[23], x2[24], x3[23], x3[24],
- *v_cos_bit);
+ butterfly_s16_s32_x8_0112_neon(cospi32, x2[27], x2[20], &x3[27], &x3[20]);
+ butterfly_s16_s32_x8_0112_neon(cospi32, x2[26], x2[21], &x3[26], &x3[21]);
+ butterfly_s16_s32_x8_0112_neon(cospi32, x2[25], x2[22], &x3[25], &x3[22]);
+ butterfly_s16_s32_x8_0112_neon(cospi32, x2[24], x2[23], &x3[24], &x3[23]);
x3[28] = x2[28];
x3[29] = x2[29];
x3[30] = x2[30];
x3[31] = x2[31];
- x3[32] = vqaddq_s16(x1[32], x2[47]);
- x3[47] = vqsubq_s16(x1[32], x2[47]);
- x3[33] = vqaddq_s16(x1[33], x2[46]);
- x3[46] = vqsubq_s16(x1[33], x2[46]);
- x3[34] = vqaddq_s16(x1[34], x2[45]);
- x3[45] = vqsubq_s16(x1[34], x2[45]);
- x3[35] = vqaddq_s16(x1[35], x2[44]);
- x3[44] = vqsubq_s16(x1[35], x2[44]);
- x3[36] = vqaddq_s16(x1[36], x2[43]);
- x3[43] = vqsubq_s16(x1[36], x2[43]);
- x3[37] = vqaddq_s16(x1[37], x2[42]);
- x3[42] = vqsubq_s16(x1[37], x2[42]);
- x3[38] = vqaddq_s16(x1[38], x2[41]);
- x3[41] = vqsubq_s16(x1[38], x2[41]);
- x3[39] = vqaddq_s16(x1[39], x2[40]);
- x3[40] = vqsubq_s16(x1[39], x2[40]);
- x3[48] = vqsubq_s16(x1[63], x2[48]);
- x3[63] = vqaddq_s16(x1[63], x2[48]);
- x3[49] = vqsubq_s16(x1[62], x2[49]);
- x3[62] = vqaddq_s16(x1[62], x2[49]);
- x3[50] = vqsubq_s16(x1[61], x2[50]);
- x3[61] = vqaddq_s16(x1[61], x2[50]);
- x3[51] = vqsubq_s16(x1[60], x2[51]);
- x3[60] = vqaddq_s16(x1[60], x2[51]);
- x3[52] = vqsubq_s16(x1[59], x2[52]);
- x3[59] = vqaddq_s16(x1[59], x2[52]);
- x3[53] = vqsubq_s16(x1[58], x2[53]);
- x3[58] = vqaddq_s16(x1[58], x2[53]);
- x3[54] = vqsubq_s16(x1[57], x2[54]);
- x3[57] = vqaddq_s16(x1[57], x2[54]);
- x3[55] = vqsubq_s16(x1[56], x2[55]);
- x3[56] = vqaddq_s16(x1[56], x2[55]);
+ butterfly_dct_post_s16_x8(x1 + 32, x2 + 32, x3 + 32, 32);
// stage 4
- x4[0] = vqaddq_s16(x3[0], x3[7]);
- x4[7] = vqsubq_s16(x3[0], x3[7]);
- x4[1] = vqaddq_s16(x3[1], x3[6]);
- x4[6] = vqsubq_s16(x3[1], x3[6]);
- x4[2] = vqaddq_s16(x3[2], x3[5]);
- x4[5] = vqsubq_s16(x3[2], x3[5]);
- x4[3] = vqaddq_s16(x3[3], x3[4]);
- x4[4] = vqsubq_s16(x3[3], x3[4]);
-
- btf_16_neon_mode0(*cospi32, *cospi32, x3[10], x3[13], x4[10], x4[13],
- *v_cos_bit);
- btf_16_neon_mode0(*cospi32, *cospi32, x3[11], x3[12], x4[11], x4[12],
- *v_cos_bit);
-
- x4[16] = vqaddq_s16(x3[16], x3[23]);
- x4[23] = vqsubq_s16(x3[16], x3[23]);
- x4[17] = vqaddq_s16(x3[17], x3[22]);
- x4[22] = vqsubq_s16(x3[17], x3[22]);
- x4[18] = vqaddq_s16(x3[18], x3[21]);
- x4[21] = vqsubq_s16(x3[18], x3[21]);
- x4[19] = vqaddq_s16(x3[19], x3[20]);
- x4[20] = vqsubq_s16(x3[19], x3[20]);
- x4[24] = vqsubq_s16(x3[31], x3[24]);
- x4[31] = vqaddq_s16(x3[31], x3[24]);
- x4[25] = vqsubq_s16(x3[30], x3[25]);
- x4[30] = vqaddq_s16(x3[30], x3[25]);
- x4[26] = vqsubq_s16(x3[29], x3[26]);
- x4[29] = vqaddq_s16(x3[29], x3[26]);
- x4[27] = vqsubq_s16(x3[28], x3[27]);
- x4[28] = vqaddq_s16(x3[28], x3[27]);
-}
-
-void av1_fdct8x64_neon(const int16x8_t *input, int16x8_t *output,
- int8_t cos_bit, const int8_t *stage_range) {
- (void)stage_range;
- const int32_t *cospi = cospi_arr(cos_bit);
- const int32x4_t v_cos_bit = vdupq_n_s32(-cos_bit);
-
- int16x8_t x3[64];
int16x8_t x4[64];
-
- av1_fdct8x64_stage_1234_neon(input, x3, x4, &cospi[32], &v_cos_bit);
-
- btf_16_neon_mode0(cospi[16], cospi[48], x3[36], x3[59], x4[36], x4[59],
- v_cos_bit);
- btf_16_neon_mode0(cospi[16], cospi[48], x3[37], x3[58], x4[37], x4[58],
- v_cos_bit);
- btf_16_neon_mode0(cospi[16], cospi[48], x3[38], x3[57], x4[38], x4[57],
- v_cos_bit);
- btf_16_neon_mode0(cospi[16], cospi[48], x3[39], x3[56], x4[39], x4[56],
- v_cos_bit);
- btf_16_neon_mode02(cospi[48], cospi[16], x3[40], x3[55], x4[40], x4[55],
- v_cos_bit);
- btf_16_neon_mode02(cospi[48], cospi[16], x3[41], x3[54], x4[41], x4[54],
- v_cos_bit);
- btf_16_neon_mode02(cospi[48], cospi[16], x3[42], x3[53], x4[42], x4[53],
- v_cos_bit);
- btf_16_neon_mode02(cospi[48], cospi[16], x3[43], x3[52], x4[43], x4[52],
- v_cos_bit);
+ butterfly_dct_pre_s16_x8(x3, x4, 8);
+ butterfly_s16_s32_x8_0112_neon(cospi32, x3[13], x3[10], &x4[13], &x4[10]);
+ butterfly_s16_s32_x8_0112_neon(cospi32, x3[12], x3[11], &x4[12], &x4[11]);
+ butterfly_dct_post_s16_x8(x3 + 16, x3 + 16, x4 + 16, 16);
+ butterfly_s16_s32_x8_0112_neon(cospi16, x3[59], x3[36], &x4[59], &x4[36]);
+ butterfly_s16_s32_x8_0112_neon(cospi16, x3[58], x3[37], &x4[58], &x4[37]);
+ butterfly_s16_s32_x8_0112_neon(cospi16, x3[57], x3[38], &x4[57], &x4[38]);
+ butterfly_s16_s32_x8_0112_neon(cospi16, x3[56], x3[39], &x4[56], &x4[39]);
+ butterfly_s16_s32_x8_1223_neon(cospi16, x3[55], x3[40], &x4[55], &x4[40]);
+ butterfly_s16_s32_x8_1223_neon(cospi16, x3[54], x3[41], &x4[54], &x4[41]);
+ butterfly_s16_s32_x8_1223_neon(cospi16, x3[53], x3[42], &x4[53], &x4[42]);
+ butterfly_s16_s32_x8_1223_neon(cospi16, x3[52], x3[43], &x4[52], &x4[43]);
// stage 5
int16x8_t x5[64];
- x5[0] = vqaddq_s16(x4[0], x4[3]);
- x5[3] = vqsubq_s16(x4[0], x4[3]);
- x5[1] = vqaddq_s16(x4[1], x4[2]);
- x5[2] = vqsubq_s16(x4[1], x4[2]);
-
- btf_16_neon_mode0(cospi[32], cospi[32], x4[5], x4[6], x5[5], x5[6],
- v_cos_bit);
-
- x5[8] = vqaddq_s16(x3[8], x4[11]);
- x5[11] = vqsubq_s16(x3[8], x4[11]);
- x5[9] = vqaddq_s16(x3[9], x4[10]);
- x5[10] = vqsubq_s16(x3[9], x4[10]);
- x5[12] = vqsubq_s16(x3[15], x4[12]);
- x5[15] = vqaddq_s16(x3[15], x4[12]);
- x5[13] = vqsubq_s16(x3[14], x4[13]);
- x5[14] = vqaddq_s16(x3[14], x4[13]);
-
- btf_16_neon_mode0(cospi[16], cospi[48], x4[18], x4[29], x5[18], x5[29],
- v_cos_bit);
- btf_16_neon_mode0(cospi[16], cospi[48], x4[19], x4[28], x5[19], x5[28],
- v_cos_bit);
- btf_16_neon_mode02(cospi[48], cospi[16], x4[20], x4[27], x5[20], x5[27],
- v_cos_bit);
- btf_16_neon_mode02(cospi[48], cospi[16], x4[21], x4[26], x5[21], x5[26],
- v_cos_bit);
-
- x5[32] = vqaddq_s16(x3[32], x4[39]);
- x5[39] = vqsubq_s16(x3[32], x4[39]);
- x5[33] = vqaddq_s16(x3[33], x4[38]);
- x5[38] = vqsubq_s16(x3[33], x4[38]);
- x5[34] = vqaddq_s16(x3[34], x4[37]);
- x5[37] = vqsubq_s16(x3[34], x4[37]);
- x5[35] = vqaddq_s16(x3[35], x4[36]);
- x5[36] = vqsubq_s16(x3[35], x4[36]);
- x5[40] = vqsubq_s16(x3[47], x4[40]);
- x5[47] = vqaddq_s16(x3[47], x4[40]);
- x5[41] = vqsubq_s16(x3[46], x4[41]);
- x5[46] = vqaddq_s16(x3[46], x4[41]);
- x5[42] = vqsubq_s16(x3[45], x4[42]);
- x5[45] = vqaddq_s16(x3[45], x4[42]);
- x5[43] = vqsubq_s16(x3[44], x4[43]);
- x5[44] = vqaddq_s16(x3[44], x4[43]);
- x5[48] = vqaddq_s16(x3[48], x4[55]);
- x5[55] = vqsubq_s16(x3[48], x4[55]);
- x5[49] = vqaddq_s16(x3[49], x4[54]);
- x5[54] = vqsubq_s16(x3[49], x4[54]);
- x5[50] = vqaddq_s16(x3[50], x4[53]);
- x5[53] = vqsubq_s16(x3[50], x4[53]);
- x5[51] = vqaddq_s16(x3[51], x4[52]);
- x5[52] = vqsubq_s16(x3[51], x4[52]);
- x5[56] = vqsubq_s16(x3[63], x4[56]);
- x5[63] = vqaddq_s16(x3[63], x4[56]);
- x5[57] = vqsubq_s16(x3[62], x4[57]);
- x5[62] = vqaddq_s16(x3[62], x4[57]);
- x5[58] = vqsubq_s16(x3[61], x4[58]);
- x5[61] = vqaddq_s16(x3[61], x4[58]);
- x5[59] = vqsubq_s16(x3[60], x4[59]);
- x5[60] = vqaddq_s16(x3[60], x4[59]);
+ butterfly_dct_pre_s16_x8(x4, x5, 4);
+ butterfly_s16_s32_x8_0112_neon(cospi32, x4[6], x4[5], &x5[6], &x5[5]);
+ butterfly_dct_post_s16_x8(x3 + 8, x4 + 8, x5 + 8, 8);
+ butterfly_s16_s32_x8_0112_neon(cospi16, x4[29], x4[18], &x5[29], &x5[18]);
+ butterfly_s16_s32_x8_0112_neon(cospi16, x4[28], x4[19], &x5[28], &x5[19]);
+ butterfly_s16_s32_x8_1223_neon(cospi16, x4[27], x4[20], &x5[27], &x5[20]);
+ butterfly_s16_s32_x8_1223_neon(cospi16, x4[26], x4[21], &x5[26], &x5[21]);
+ butterfly_dct_post_s16_x8(x3 + 32, x4 + 32, x5 + 32, 16);
+ butterfly_dct_post_s16_x8(x3 + 48, x4 + 48, x5 + 48, 16);
// stage 6
int16x8_t x6[64];
- btf_16_neon_mode2(cospi[32], cospi[32], x5[0], x5[1], x6[0], x6[1],
- v_cos_bit);
- btf_16_neon_mode2(cospi[48], cospi[16], x5[2], x5[3], x6[2], x6[3],
- v_cos_bit);
- x6[4] = vqaddq_s16(x4[4], x5[5]);
- x6[5] = vqsubq_s16(x4[4], x5[5]);
- x6[6] = vqsubq_s16(x4[7], x5[6]);
- x6[7] = vqaddq_s16(x4[7], x5[6]);
-
- btf_16_neon_mode0(cospi[16], cospi[48], x5[9], x5[14], x6[9], x6[14],
- v_cos_bit);
- btf_16_neon_mode02(cospi[48], cospi[16], x5[10], x5[13], x6[10], x6[13],
- v_cos_bit);
-
- x6[16] = vqaddq_s16(x4[16], x5[19]);
- x6[19] = vqsubq_s16(x4[16], x5[19]);
- x6[17] = vqaddq_s16(x4[17], x5[18]);
- x6[18] = vqsubq_s16(x4[17], x5[18]);
- x6[20] = vqsubq_s16(x4[23], x5[20]);
- x6[23] = vqaddq_s16(x4[23], x5[20]);
- x6[21] = vqsubq_s16(x4[22], x5[21]);
- x6[22] = vqaddq_s16(x4[22], x5[21]);
- x6[24] = vqaddq_s16(x4[24], x5[27]);
- x6[27] = vqsubq_s16(x4[24], x5[27]);
- x6[25] = vqaddq_s16(x4[25], x5[26]);
- x6[26] = vqsubq_s16(x4[25], x5[26]);
- x6[28] = vqsubq_s16(x4[31], x5[28]);
- x6[31] = vqaddq_s16(x4[31], x5[28]);
- x6[29] = vqsubq_s16(x4[30], x5[29]);
- x6[30] = vqaddq_s16(x4[30], x5[29]);
-
- btf_16_neon_mode0(cospi[8], cospi[56], x5[34], x5[61], x6[34], x6[61],
- v_cos_bit);
- btf_16_neon_mode0(cospi[8], cospi[56], x5[35], x5[60], x6[35], x6[60],
- v_cos_bit);
- btf_16_neon_mode02(cospi[56], cospi[8], x5[36], x5[59], x6[36], x6[59],
- v_cos_bit);
- btf_16_neon_mode02(cospi[56], cospi[8], x5[37], x5[58], x6[37], x6[58],
- v_cos_bit);
- btf_16_neon_mode0(cospi[40], cospi[24], x5[42], x5[53], x6[42], x6[53],
- v_cos_bit);
- btf_16_neon_mode0(cospi[40], cospi[24], x5[43], x5[52], x6[43], x6[52],
- v_cos_bit);
- btf_16_neon_mode02(cospi[24], cospi[40], x5[44], x5[51], x6[44], x6[51],
- v_cos_bit);
- btf_16_neon_mode02(cospi[24], cospi[40], x5[45], x5[50], x6[45], x6[50],
- v_cos_bit);
+ butterfly_s16_s32_x8_0112_neon(cospi32, x5[1], x5[0], &x6[0], &x6[1]);
+ butterfly_s16_s32_x8_0112_neon(cospi16, x5[3], x5[2], &x6[2], &x6[3]);
+ butterfly_dct_post_s16_x8(x4 + 4, x5 + 4, x6 + 4, 4);
+ butterfly_s16_s32_x8_0112_neon(cospi16, x5[14], x5[9], &x6[14], &x6[9]);
+ butterfly_s16_s32_x8_1223_neon(cospi16, x5[13], x5[10], &x6[13], &x6[10]);
+ butterfly_dct_post_s16_x8(x4 + 16, x5 + 16, x6 + 16, 8);
+ butterfly_dct_post_s16_x8(x4 + 24, x5 + 24, x6 + 24, 8);
+ butterfly_s16_s32_x8_0112_neon(cospi8, x5[61], x5[34], &x6[61], &x6[34]);
+ butterfly_s16_s32_x8_0112_neon(cospi8, x5[60], x5[35], &x6[60], &x6[35]);
+ butterfly_s16_s32_x8_1223_neon(cospi8, x5[59], x5[36], &x6[59], &x6[36]);
+ butterfly_s16_s32_x8_1223_neon(cospi8, x5[58], x5[37], &x6[58], &x6[37]);
+ butterfly_s16_s32_x8_1003_neon(cospi24, x5[53], x5[42], &x6[53], &x6[42]);
+ butterfly_s16_s32_x8_1003_neon(cospi24, x5[52], x5[43], &x6[52], &x6[43]);
+ butterfly_s16_s32_x8_0332_neon(cospi24, x5[51], x5[44], &x6[51], &x6[44]);
+ butterfly_s16_s32_x8_0332_neon(cospi24, x5[50], x5[45], &x6[50], &x6[45]);
// stage 7
int16x8_t x7[64];
-
- btf_16_neon_mode2(cospi[56], cospi[8], x6[4], x6[7], x7[4], x7[7], v_cos_bit);
- btf_16_neon_mode2(cospi[24], cospi[40], x6[5], x6[6], x7[5], x7[6],
- v_cos_bit);
- x7[8] = vqaddq_s16(x5[8], x6[9]);
- x7[9] = vqsubq_s16(x5[8], x6[9]);
- x7[10] = vqsubq_s16(x5[11], x6[10]);
- x7[11] = vqaddq_s16(x5[11], x6[10]);
- x7[12] = vqaddq_s16(x5[12], x6[13]);
- x7[13] = vqsubq_s16(x5[12], x6[13]);
- x7[14] = vqsubq_s16(x5[15], x6[14]);
- x7[15] = vqaddq_s16(x5[15], x6[14]);
-
- btf_16_neon_mode0(cospi[8], cospi[56], x6[17], x6[30], x7[17], x7[30],
- v_cos_bit);
- btf_16_neon_mode02(cospi[56], cospi[8], x6[18], x6[29], x7[18], x7[29],
- v_cos_bit);
-
- btf_16_neon_mode0(cospi[40], cospi[24], x6[21], x6[26], x7[21], x7[26],
- v_cos_bit);
- btf_16_neon_mode02(cospi[24], cospi[40], x6[22], x6[25], x7[22], x7[25],
- v_cos_bit);
-
- x7[32] = vqaddq_s16(x5[32], x6[35]);
- x7[35] = vqsubq_s16(x5[32], x6[35]);
- x7[33] = vqaddq_s16(x5[33], x6[34]);
- x7[34] = vqsubq_s16(x5[33], x6[34]);
- x7[36] = vqsubq_s16(x5[39], x6[36]);
- x7[39] = vqaddq_s16(x5[39], x6[36]);
- x7[37] = vqsubq_s16(x5[38], x6[37]);
- x7[38] = vqaddq_s16(x5[38], x6[37]);
- x7[40] = vqaddq_s16(x5[40], x6[43]);
- x7[43] = vqsubq_s16(x5[40], x6[43]);
- x7[41] = vqaddq_s16(x5[41], x6[42]);
- x7[42] = vqsubq_s16(x5[41], x6[42]);
- x7[44] = vqsubq_s16(x5[47], x6[44]);
- x7[47] = vqaddq_s16(x5[47], x6[44]);
- x7[45] = vqsubq_s16(x5[46], x6[45]);
- x7[46] = vqaddq_s16(x5[46], x6[45]);
- x7[48] = vqaddq_s16(x5[48], x6[51]);
- x7[51] = vqsubq_s16(x5[48], x6[51]);
- x7[49] = vqaddq_s16(x5[49], x6[50]);
- x7[50] = vqsubq_s16(x5[49], x6[50]);
- x7[52] = vqsubq_s16(x5[55], x6[52]);
- x7[55] = vqaddq_s16(x5[55], x6[52]);
- x7[53] = vqsubq_s16(x5[54], x6[53]);
- x7[54] = vqaddq_s16(x5[54], x6[53]);
- x7[56] = vqaddq_s16(x5[56], x6[59]);
- x7[59] = vqsubq_s16(x5[56], x6[59]);
- x7[57] = vqaddq_s16(x5[57], x6[58]);
- x7[58] = vqsubq_s16(x5[57], x6[58]);
- x7[60] = vqsubq_s16(x5[63], x6[60]);
- x7[63] = vqaddq_s16(x5[63], x6[60]);
- x7[61] = vqsubq_s16(x5[62], x6[61]);
- x7[62] = vqaddq_s16(x5[62], x6[61]);
+ butterfly_s16_s32_x8_0112_neon(cospi8, x6[7], x6[4], &x7[4], &x7[7]);
+ butterfly_s16_s32_x8_1003_neon(cospi24, x6[6], x6[5], &x7[5], &x7[6]);
+ butterfly_dct_post_s16_x8(x5 + 8, x6 + 8, x7 + 8, 4);
+ butterfly_dct_post_s16_x8(x5 + 12, x6 + 12, x7 + 12, 4);
+ butterfly_s16_s32_x8_0112_neon(cospi8, x6[30], x6[17], &x7[30], &x7[17]);
+ butterfly_s16_s32_x8_1223_neon(cospi8, x6[29], x6[18], &x7[29], &x7[18]);
+ butterfly_s16_s32_x8_1003_neon(cospi24, x6[26], x6[21], &x7[26], &x7[21]);
+ butterfly_s16_s32_x8_0332_neon(cospi24, x6[25], x6[22], &x7[25], &x7[22]);
+ butterfly_dct_post_s16_x8(x5 + 32, x6 + 32, x7 + 32, 8);
+ butterfly_dct_post_s16_x8(x5 + 40, x6 + 40, x7 + 40, 8);
+ butterfly_dct_post_s16_x8(x5 + 48, x6 + 48, x7 + 48, 8);
+ butterfly_dct_post_s16_x8(x5 + 56, x6 + 56, x7 + 56, 8);
// stage 8
int16x8_t x8[64];
-
- btf_16_neon_mode2(cospi[60], cospi[4], x7[8], x7[15], x8[8], x8[15],
- v_cos_bit);
- btf_16_neon_mode2(cospi[28], cospi[36], x7[9], x7[14], x8[9], x8[14],
- v_cos_bit);
- btf_16_neon_mode2(cospi[44], cospi[20], x7[10], x7[13], x8[10], x8[13],
- v_cos_bit);
- btf_16_neon_mode2(cospi[12], cospi[52], x7[11], x7[12], x8[11], x8[12],
- v_cos_bit);
- x8[16] = vqaddq_s16(x6[16], x7[17]);
- x8[17] = vqsubq_s16(x6[16], x7[17]);
- x8[18] = vqsubq_s16(x6[19], x7[18]);
- x8[19] = vqaddq_s16(x6[19], x7[18]);
- x8[20] = vqaddq_s16(x6[20], x7[21]);
- x8[21] = vqsubq_s16(x6[20], x7[21]);
- x8[22] = vqsubq_s16(x6[23], x7[22]);
- x8[23] = vqaddq_s16(x6[23], x7[22]);
- x8[24] = vqaddq_s16(x6[24], x7[25]);
- x8[25] = vqsubq_s16(x6[24], x7[25]);
- x8[26] = vqsubq_s16(x6[27], x7[26]);
- x8[27] = vqaddq_s16(x6[27], x7[26]);
- x8[28] = vqaddq_s16(x6[28], x7[29]);
- x8[29] = vqsubq_s16(x6[28], x7[29]);
- x8[30] = vqsubq_s16(x6[31], x7[30]);
- x8[31] = vqaddq_s16(x6[31], x7[30]);
-
- btf_16_neon_mode0(cospi[4], cospi[60], x7[33], x7[62], x8[33], x8[62],
- v_cos_bit);
- btf_16_neon_mode02(cospi[60], cospi[4], x7[34], x7[61], x8[34], x8[61],
- v_cos_bit);
- btf_16_neon_mode0(cospi[36], cospi[28], x7[37], x7[58], x8[37], x8[58],
- v_cos_bit);
- btf_16_neon_mode02(cospi[28], cospi[36], x7[38], x7[57], x8[38], x8[57],
- v_cos_bit);
- btf_16_neon_mode0(cospi[20], cospi[44], x7[41], x7[54], x8[41], x8[54],
- v_cos_bit);
- btf_16_neon_mode02(cospi[44], cospi[20], x7[42], x7[53], x8[42], x8[53],
- v_cos_bit);
- btf_16_neon_mode0(cospi[52], cospi[12], x7[45], x7[50], x8[45], x8[50],
- v_cos_bit);
- btf_16_neon_mode02(cospi[12], cospi[52], x7[46], x7[49], x8[46], x8[49],
- v_cos_bit);
+ butterfly_s16_s32_x8_0112_neon(cospi4, x7[15], x7[8], &x8[8], &x8[15]);
+ butterfly_s16_s32_x8_1003_neon(cospi28, x7[14], x7[9], &x8[9], &x8[14]);
+ butterfly_s16_s32_x8_0112_neon(cospi20, x7[13], x7[10], &x8[10], &x8[13]);
+ butterfly_s16_s32_x8_1003_neon(cospi12, x7[12], x7[11], &x8[11], &x8[12]);
+ butterfly_dct_post_s16_x8(x6 + 16, x7 + 16, x8 + 16, 4);
+ butterfly_dct_post_s16_x8(x6 + 20, x7 + 20, x8 + 20, 4);
+ butterfly_dct_post_s16_x8(x6 + 24, x7 + 24, x8 + 24, 4);
+ butterfly_dct_post_s16_x8(x6 + 28, x7 + 28, x8 + 28, 4);
+ butterfly_s16_s32_x8_0112_neon(cospi4, x7[62], x7[33], &x8[62], &x8[33]);
+ butterfly_s16_s32_x8_1223_neon(cospi4, x7[61], x7[34], &x8[61], &x8[34]);
+ butterfly_s16_s32_x8_1003_neon(cospi28, x7[58], x7[37], &x8[58], &x8[37]);
+ butterfly_s16_s32_x8_0332_neon(cospi28, x7[57], x7[38], &x8[57], &x8[38]);
+ butterfly_s16_s32_x8_0112_neon(cospi20, x7[54], x7[41], &x8[54], &x8[41]);
+ butterfly_s16_s32_x8_1223_neon(cospi20, x7[53], x7[42], &x8[53], &x8[42]);
+ butterfly_s16_s32_x8_1003_neon(cospi12, x7[50], x7[45], &x8[50], &x8[45]);
+ butterfly_s16_s32_x8_0332_neon(cospi12, x7[49], x7[46], &x8[49], &x8[46]);
// stage 9
int16x8_t x9[64];
-
- btf_16_neon_mode2(cospi[62], cospi[2], x8[16], x8[31], x9[16], x9[31],
- v_cos_bit);
- btf_16_neon_mode2(cospi[30], cospi[34], x8[17], x8[30], x9[17], x9[30],
- v_cos_bit);
- btf_16_neon_mode2(cospi[46], cospi[18], x8[18], x8[29], x9[18], x9[29],
- v_cos_bit);
- btf_16_neon_mode2(cospi[14], cospi[50], x8[19], x8[28], x9[19], x9[28],
- v_cos_bit);
- btf_16_neon_mode2(cospi[54], cospi[10], x8[20], x8[27], x9[20], x9[27],
- v_cos_bit);
- btf_16_neon_mode2(cospi[22], cospi[42], x8[21], x8[26], x9[21], x9[26],
- v_cos_bit);
- btf_16_neon_mode2(cospi[38], cospi[26], x8[22], x8[25], x9[22], x9[25],
- v_cos_bit);
- btf_16_neon_mode2(cospi[6], cospi[58], x8[23], x8[24], x9[23], x9[24],
- v_cos_bit);
- x9[32] = vqaddq_s16(x7[32], x8[33]);
- x9[33] = vqsubq_s16(x7[32], x8[33]);
- x9[34] = vqsubq_s16(x7[35], x8[34]);
- x9[35] = vqaddq_s16(x7[35], x8[34]);
- x9[36] = vqaddq_s16(x7[36], x8[37]);
- x9[37] = vqsubq_s16(x7[36], x8[37]);
- x9[38] = vqsubq_s16(x7[39], x8[38]);
- x9[39] = vqaddq_s16(x7[39], x8[38]);
- x9[40] = vqaddq_s16(x7[40], x8[41]);
- x9[41] = vqsubq_s16(x7[40], x8[41]);
- x9[42] = vqsubq_s16(x7[43], x8[42]);
- x9[43] = vqaddq_s16(x7[43], x8[42]);
- x9[44] = vqaddq_s16(x7[44], x8[45]);
- x9[45] = vqsubq_s16(x7[44], x8[45]);
- x9[46] = vqsubq_s16(x7[47], x8[46]);
- x9[47] = vqaddq_s16(x7[47], x8[46]);
- x9[48] = vqaddq_s16(x7[48], x8[49]);
- x9[49] = vqsubq_s16(x7[48], x8[49]);
- x9[50] = vqsubq_s16(x7[51], x8[50]);
- x9[51] = vqaddq_s16(x7[51], x8[50]);
- x9[52] = vqaddq_s16(x7[52], x8[53]);
- x9[53] = vqsubq_s16(x7[52], x8[53]);
- x9[54] = vqsubq_s16(x7[55], x8[54]);
- x9[55] = vqaddq_s16(x7[55], x8[54]);
- x9[56] = vqaddq_s16(x7[56], x8[57]);
- x9[57] = vqsubq_s16(x7[56], x8[57]);
- x9[58] = vqsubq_s16(x7[59], x8[58]);
- x9[59] = vqaddq_s16(x7[59], x8[58]);
- x9[60] = vqaddq_s16(x7[60], x8[61]);
- x9[61] = vqsubq_s16(x7[60], x8[61]);
- x9[62] = vqsubq_s16(x7[63], x8[62]);
- x9[63] = vqaddq_s16(x7[63], x8[62]);
+ butterfly_s16_s32_x8_0112_neon(cospi2, x8[31], x8[16], &x9[16], &x9[31]);
+ butterfly_s16_s32_x8_1003_neon(cospi30, x8[30], x8[17], &x9[17], &x9[30]);
+ butterfly_s16_s32_x8_0112_neon(cospi18, x8[29], x8[18], &x9[18], &x9[29]);
+ butterfly_s16_s32_x8_1003_neon(cospi14, x8[28], x8[19], &x9[19], &x9[28]);
+ butterfly_s16_s32_x8_0112_neon(cospi10, x8[27], x8[20], &x9[20], &x9[27]);
+ butterfly_s16_s32_x8_1003_neon(cospi22, x8[26], x8[21], &x9[21], &x9[26]);
+ butterfly_s16_s32_x8_0112_neon(cospi26, x8[25], x8[22], &x9[22], &x9[25]);
+ butterfly_s16_s32_x8_1003_neon(cospi6, x8[24], x8[23], &x9[23], &x9[24]);
+ butterfly_dct_post_s16_x8(x7 + 32, x8 + 32, x9 + 32, 4);
+ butterfly_dct_post_s16_x8(x7 + 36, x8 + 36, x9 + 36, 4);
+ butterfly_dct_post_s16_x8(x7 + 40, x8 + 40, x9 + 40, 4);
+ butterfly_dct_post_s16_x8(x7 + 44, x8 + 44, x9 + 44, 4);
+ butterfly_dct_post_s16_x8(x7 + 48, x8 + 48, x9 + 48, 4);
+ butterfly_dct_post_s16_x8(x7 + 52, x8 + 52, x9 + 52, 4);
+ butterfly_dct_post_s16_x8(x7 + 56, x8 + 56, x9 + 56, 4);
+ butterfly_dct_post_s16_x8(x7 + 60, x8 + 60, x9 + 60, 4);
// stage 10
- btf_16_neon_mode2(cospi[63], cospi[1], x9[32], x9[63], output[1], output[63],
- v_cos_bit);
-
- btf_16_neon_mode2(cospi[31], cospi[33], x9[33], x9[62], output[33],
- output[31], v_cos_bit);
-
- btf_16_neon_mode2(cospi[47], cospi[17], x9[34], x9[61], output[17],
- output[47], v_cos_bit);
-
- btf_16_neon_mode2(cospi[15], cospi[49], x9[35], x9[60], output[49],
- output[15], v_cos_bit);
-
- btf_16_neon_mode2(cospi[55], cospi[9], x9[36], x9[59], output[9], output[55],
- v_cos_bit);
-
- btf_16_neon_mode2(cospi[23], cospi[41], x9[37], x9[58], output[41],
- output[23], v_cos_bit);
-
- btf_16_neon_mode2(cospi[39], cospi[25], x9[38], x9[57], output[25],
- output[39], v_cos_bit);
-
- btf_16_neon_mode2(cospi[7], cospi[57], x9[39], x9[56], output[57], output[7],
- v_cos_bit);
-
- btf_16_neon_mode2(cospi[59], cospi[5], x9[40], x9[55], output[5], output[59],
- v_cos_bit);
-
- btf_16_neon_mode2(cospi[27], cospi[37], x9[41], x9[54], output[37],
- output[27], v_cos_bit);
-
- btf_16_neon_mode2(cospi[43], cospi[21], x9[42], x9[53], output[21],
- output[43], v_cos_bit);
-
- btf_16_neon_mode2(cospi[11], cospi[53], x9[43], x9[52], output[53],
- output[11], v_cos_bit);
-
- btf_16_neon_mode2(cospi[51], cospi[13], x9[44], x9[51], output[13],
- output[51], v_cos_bit);
-
- btf_16_neon_mode2(cospi[19], cospi[45], x9[45], x9[50], output[45],
- output[19], v_cos_bit);
-
- btf_16_neon_mode2(cospi[35], cospi[29], x9[46], x9[49], output[29],
- output[35], v_cos_bit);
-
- btf_16_neon_mode2(cospi[3], cospi[61], x9[47], x9[48], output[61], output[3],
- v_cos_bit);
+ butterfly_s16_s32_x8_0112_neon(cospi1, x9[63], x9[32], &output[1],
+ &output[63]);
+ butterfly_s16_s32_x8_1003_neon(cospi31, x9[62], x9[33], &output[33],
+ &output[31]);
+ butterfly_s16_s32_x8_0112_neon(cospi17, x9[61], x9[34], &output[17],
+ &output[47]);
+ butterfly_s16_s32_x8_1003_neon(cospi15, x9[60], x9[35], &output[49],
+ &output[15]);
+ butterfly_s16_s32_x8_0112_neon(cospi9, x9[59], x9[36], &output[9],
+ &output[55]);
+ butterfly_s16_s32_x8_1003_neon(cospi23, x9[58], x9[37], &output[41],
+ &output[23]);
+ butterfly_s16_s32_x8_0112_neon(cospi25, x9[57], x9[38], &output[25],
+ &output[39]);
+ butterfly_s16_s32_x8_1003_neon(cospi7, x9[56], x9[39], &output[57],
+ &output[7]);
+ butterfly_s16_s32_x8_0112_neon(cospi5, x9[55], x9[40], &output[5],
+ &output[59]);
+ butterfly_s16_s32_x8_1003_neon(cospi27, x9[54], x9[41], &output[37],
+ &output[27]);
+ butterfly_s16_s32_x8_0112_neon(cospi21, x9[53], x9[42], &output[21],
+ &output[43]);
+ butterfly_s16_s32_x8_1003_neon(cospi11, x9[52], x9[43], &output[53],
+ &output[11]);
+ butterfly_s16_s32_x8_0112_neon(cospi13, x9[51], x9[44], &output[13],
+ &output[51]);
+ butterfly_s16_s32_x8_1003_neon(cospi19, x9[50], x9[45], &output[45],
+ &output[19]);
+ butterfly_s16_s32_x8_0112_neon(cospi29, x9[49], x9[46], &output[29],
+ &output[35]);
+ butterfly_s16_s32_x8_1003_neon(cospi3, x9[48], x9[47], &output[61],
+ &output[3]);
// stage 11
output[0] = x6[0];
@@ -1823,1377 +1168,1297 @@
output[62] = x9[31];
}
-void fadst_8x8_neon(const int16x8_t *input, int16x8_t *output, int8_t cos_bit,
- const int8_t *stage_range) {
- (void)stage_range;
- const int32_t *cospi = cospi_arr(cos_bit);
- const int32x4_t v_cos_bit = vdupq_n_s32(-cos_bit);
+static AOM_FORCE_INLINE void fadst8x8_neon(const int16x8_t *input,
+ int16x8_t *output, int cos_bit) {
+ const int16_t *cospi = cospi_arr_q13(cos_bit);
- // stage 1
- int16x8_t x1[4];
+ const int16x8_t cospi32_16 = vld1q_s16(&cospi[4 * 0]);
+ const int16x8_t cospi4_12 = vld1q_s16(&cospi[4 * 4]);
+ const int16x8_t cospi20_28 = vld1q_s16(&cospi[4 * 6]);
- x1[0] = vqnegq_s16(input[7]);
- x1[1] = vqnegq_s16(input[3]);
- x1[2] = vqnegq_s16(input[1]);
- x1[3] = vqnegq_s16(input[5]);
+ const int16x4_t cospi32 = vget_low_s16(cospi32_16);
+ const int16x4_t cospi16 = vget_high_s16(cospi32_16);
+ const int16x4_t cospi4 = vget_low_s16(cospi4_12);
+ const int16x4_t cospi12 = vget_high_s16(cospi4_12);
+ const int16x4_t cospi20 = vget_low_s16(cospi20_28);
+ const int16x4_t cospi28 = vget_high_s16(cospi20_28);
// stage 2
int16x8_t x2[8];
+ butterfly_s16_s32_x8_0332_neon(cospi32, input[4], input[3], &x2[2], &x2[3]);
+ butterfly_s16_s32_x8_0112_neon(cospi32, input[2], input[5], &x2[7], &x2[6]);
- btf_16_neon_mode3(cospi[32], cospi[32], x1[1], input[4], x2[2], x2[3],
- v_cos_bit);
- btf_16_neon_mode3(cospi[32], cospi[32], input[2], x1[3], x2[6], x2[7],
- v_cos_bit);
// stage 3
int16x8_t x3[8];
x3[0] = vqaddq_s16(input[0], x2[2]);
+ x3[1] = vqsubq_s16(x2[3], input[7]);
x3[2] = vqsubq_s16(input[0], x2[2]);
- x3[1] = vqaddq_s16(x1[0], x2[3]);
- x3[3] = vqsubq_s16(x1[0], x2[3]);
- x3[4] = vqaddq_s16(x1[2], x2[6]);
- x3[6] = vqsubq_s16(x1[2], x2[6]);
+ x3[3] = vqaddq_s16(input[7], x2[3]);
+ x3[4] = vqsubq_s16(x2[6], input[1]);
x3[5] = vqaddq_s16(input[6], x2[7]);
+ x3[6] = vqaddq_s16(input[1], x2[6]);
x3[7] = vqsubq_s16(input[6], x2[7]);
// stage 4
- btf_16_neon_mode3(cospi[16], cospi[48], x3[4], x3[5], x3[4], x3[5],
- v_cos_bit);
- btf_16_neon_mode0(cospi[48], cospi[16], x3[6], x3[7], x3[6], x3[7],
- v_cos_bit);
+ butterfly_s16_s32_x8_0112_neon(cospi16, x3[4], x3[5], &x3[4], &x3[5]);
+ butterfly_s16_s32_x8_0112_neon(cospi16, x3[7], x3[6], &x3[6], &x3[7]);
// stage 5
int16x8_t x5[8];
x5[0] = vqaddq_s16(x3[0], x3[4]);
- x5[4] = vqsubq_s16(x3[0], x3[4]);
x5[1] = vqaddq_s16(x3[1], x3[5]);
- x5[5] = vqsubq_s16(x3[1], x3[5]);
x5[2] = vqaddq_s16(x3[2], x3[6]);
+ x5[3] = vqsubq_s16(x3[7], x3[3]);
+ x5[4] = vqsubq_s16(x3[0], x3[4]);
+ x5[5] = vqsubq_s16(x3[1], x3[5]);
x5[6] = vqsubq_s16(x3[2], x3[6]);
- x5[3] = vqaddq_s16(x3[3], x3[7]);
- x5[7] = vqsubq_s16(x3[3], x3[7]);
+ x5[7] = vqaddq_s16(x3[3], x3[7]);
// stage 6
- btf_16_neon_mode3(cospi[4], cospi[60], x5[0], x5[1], output[7], output[0],
- v_cos_bit);
- btf_16_neon_mode3(cospi[20], cospi[44], x5[2], x5[3], output[5], output[2],
- v_cos_bit);
- btf_16_neon_mode3(cospi[36], cospi[28], x5[4], x5[5], output[3], output[4],
- v_cos_bit);
- btf_16_neon_mode3(cospi[52], cospi[12], x5[6], x5[7], output[1], output[6],
- v_cos_bit);
+ butterfly_s16_s32_x8_0112_neon(cospi4, x5[0], x5[1], &output[7], &output[0]);
+ butterfly_s16_s32_x8_0112_neon(cospi20, x5[2], x5[3], &output[5], &output[2]);
+ butterfly_s16_s32_x8_1003_neon(cospi28, x5[4], x5[5], &output[3], &output[4]);
+ butterfly_s16_s32_x8_0112_neon(cospi12, x5[6], x5[7], &output[6], &output[1]);
}
-static void fadst8x16_neon(const int16x8_t *input, int16x8_t *output,
- int8_t cos_bit, const int8_t *stage_range) {
- (void)stage_range;
- const int32_t *cospi = cospi_arr(cos_bit);
- const int32x4_t v_cos_bit = vdupq_n_s32(-cos_bit);
+static AOM_FORCE_INLINE void fadst4x16_neon(const int16x4_t *input,
+ int16x4_t *output, int cos_bit) {
+ const int16_t *cospi = cospi_arr_q13(cos_bit);
- // stage 1
- int16x8_t x1[12];
- x1[0] = vqnegq_s16(input[15]);
- x1[1] = vqnegq_s16(input[3]);
- x1[2] = vqnegq_s16(input[1]);
- x1[3] = vqnegq_s16(input[13]);
+ const int16x8_t cospi32_16 = vld1q_s16(&cospi[4 * 0]);
+ const int16x8_t cospi8_24 = vld1q_s16(&cospi[4 * 2]);
+ const int16x8_t cospi2_6 = vld1q_s16(&cospi[4 * 8]);
+ const int16x8_t cospi10_14 = vld1q_s16(&cospi[4 * 10]);
+ const int16x8_t cospi18_22 = vld1q_s16(&cospi[4 * 12]);
+ const int16x8_t cospi26_30 = vld1q_s16(&cospi[4 * 14]);
+
+ const int16x4_t cospi32 = vget_low_s16(cospi32_16);
+ const int16x4_t cospi16 = vget_high_s16(cospi32_16);
+ const int16x4_t cospi8 = vget_low_s16(cospi8_24);
+ const int16x4_t cospi24 = vget_high_s16(cospi8_24);
+ const int16x4_t cospi2 = vget_low_s16(cospi2_6);
+ const int16x4_t cospi6 = vget_high_s16(cospi2_6);
+ const int16x4_t cospi10 = vget_low_s16(cospi10_14);
+ const int16x4_t cospi14 = vget_high_s16(cospi10_14);
+ const int16x4_t cospi18 = vget_low_s16(cospi18_22);
+ const int16x4_t cospi22 = vget_high_s16(cospi18_22);
+ const int16x4_t cospi26 = vget_low_s16(cospi26_30);
+ const int16x4_t cospi30 = vget_high_s16(cospi26_30);
// stage 2
- btf_16_neon(-cospi[32], cospi[32], -cospi[32], -cospi[32], input[7], input[8],
- x1[4], x1[5]);
- btf_16_neon_mode1(cospi[32], cospi[32], input[4], input[11], x1[6], x1[7],
- v_cos_bit);
- btf_16_neon_mode1(cospi[32], cospi[32], input[6], input[9], x1[8], x1[9],
- v_cos_bit);
- btf_16_neon(-cospi[32], cospi[32], -cospi[32], -cospi[32], input[5],
- input[10], x1[10], x1[11]);
+ int16x4_t x2[8];
+ butterfly_s16_s32_x4_0332_neon(cospi32, input[8], input[7], &x2[0], &x2[1]);
+ butterfly_s16_s32_x4_0112_neon(cospi32, input[4], input[11], &x2[3], &x2[2]);
+ butterfly_s16_s32_x4_0112_neon(cospi32, input[6], input[9], &x2[5], &x2[4]);
+ butterfly_s16_s32_x4_0332_neon(cospi32, input[10], input[5], &x2[6], &x2[7]);
+
// stage 3
- int16x8_t x3[16];
- x3[0] = vqaddq_s16(input[0], x1[4]);
- x3[2] = vqsubq_s16(input[0], x1[4]);
- x3[1] = vqaddq_s16(x1[0], x1[5]);
- x3[3] = vqsubq_s16(x1[0], x1[5]);
- x3[4] = vqaddq_s16(x1[1], x1[6]);
- x3[6] = vqsubq_s16(x1[1], x1[6]);
- x3[5] = vqaddq_s16(input[12], x1[7]);
- x3[7] = vqsubq_s16(input[12], x1[7]);
- x3[8] = vqaddq_s16(x1[2], x1[8]);
- x3[10] = vqsubq_s16(x1[2], x1[8]);
- x3[9] = vqaddq_s16(input[14], x1[9]);
- x3[11] = vqsubq_s16(input[14], x1[9]);
- x3[12] = vqaddq_s16(input[2], x1[10]);
- x3[14] = vqsubq_s16(input[2], x1[10]);
- x3[13] = vqaddq_s16(x1[3], x1[11]);
- x3[15] = vqsubq_s16(x1[3], x1[11]);
+ int16x4_t x3[16];
+ x3[0] = vqadd_s16(input[0], x2[0]);
+ x3[1] = vqsub_s16(x2[1], input[15]);
+ x3[2] = vqsub_s16(input[0], x2[0]);
+ x3[3] = vqadd_s16(input[15], x2[1]);
+ x3[4] = vqsub_s16(x2[2], input[3]);
+ x3[5] = vqadd_s16(input[12], x2[3]);
+ x3[6] = vqadd_s16(input[3], x2[2]);
+ x3[7] = vqsub_s16(input[12], x2[3]);
+ x3[8] = vqsub_s16(x2[4], input[1]);
+ x3[9] = vqadd_s16(input[14], x2[5]);
+ x3[10] = vqadd_s16(input[1], x2[4]);
+ x3[11] = vqsub_s16(input[14], x2[5]);
+ x3[12] = vqadd_s16(input[2], x2[6]);
+ x3[13] = vqsub_s16(x2[7], input[13]);
+ x3[14] = vqsub_s16(input[2], x2[6]);
+ x3[15] = vqadd_s16(input[13], x2[7]);
// stage 4
- btf_16_neon_mode3(cospi[16], cospi[48], x3[4], x3[5], x3[4], x3[5],
- v_cos_bit);
- btf_16_neon_mode0(cospi[48], cospi[16], x3[6], x3[7], x3[6], x3[7],
- v_cos_bit);
- btf_16_neon_mode3(cospi[16], cospi[48], x3[12], x3[13], x3[12], x3[13],
- v_cos_bit);
- btf_16_neon_mode0(cospi[48], cospi[16], x3[14], x3[15], x3[14], x3[15],
- v_cos_bit);
+ butterfly_s16_s32_x4_0112_neon(cospi16, x3[4], x3[5], &x3[4], &x3[5]);
+ butterfly_s16_s32_x4_0112_neon(cospi16, x3[7], x3[6], &x3[6], &x3[7]);
+ butterfly_s16_s32_x4_0112_neon(cospi16, x3[12], x3[13], &x3[12], &x3[13]);
+ butterfly_s16_s32_x4_0332_neon(cospi16, x3[14], x3[15], &x3[15], &x3[14]);
+
+ // stage 5
+ int16x4_t x5[16];
+ x5[0] = vqadd_s16(x3[0], x3[4]);
+ x5[1] = vqadd_s16(x3[1], x3[5]);
+ x5[2] = vqadd_s16(x3[2], x3[6]);
+ x5[3] = vqsub_s16(x3[7], x3[3]);
+ x5[4] = vqsub_s16(x3[0], x3[4]);
+ x5[5] = vqsub_s16(x3[1], x3[5]);
+ x5[6] = vqsub_s16(x3[2], x3[6]);
+ x5[7] = vqadd_s16(x3[3], x3[7]);
+ x5[8] = vqadd_s16(x3[8], x3[12]);
+ x5[9] = vqadd_s16(x3[9], x3[13]);
+ x5[10] = vqsub_s16(x3[14], x3[10]);
+ x5[11] = vqadd_s16(x3[11], x3[15]);
+ x5[12] = vqsub_s16(x3[8], x3[12]);
+ x5[13] = vqsub_s16(x3[9], x3[13]);
+ x5[14] = vqadd_s16(x3[10], x3[14]);
+ x5[15] = vqsub_s16(x3[11], x3[15]);
+
+ // stage 6
+ butterfly_s16_s32_x4_0112_neon(cospi8, x5[8], x5[9], &x5[8], &x5[9]);
+ butterfly_s16_s32_x4_1003_neon(cospi24, x5[10], x5[11], &x5[10], &x5[11]);
+ butterfly_s16_s32_x4_1003_neon(cospi8, x5[13], x5[12], &x5[13], &x5[12]);
+ butterfly_s16_s32_x4_1003_neon(cospi24, x5[15], x5[14], &x5[14], &x5[15]);
+
+ // stage 7
+ int16x4_t x7[16];
+ x7[0] = vqadd_s16(x5[0], x5[8]);
+ x7[1] = vqadd_s16(x5[1], x5[9]);
+ x7[2] = vqadd_s16(x5[2], x5[10]);
+ x7[3] = vqadd_s16(x5[3], x5[11]);
+ x7[4] = vqadd_s16(x5[4], x5[12]);
+ x7[5] = vqadd_s16(x5[5], x5[13]);
+ x7[6] = vqadd_s16(x5[6], x5[14]);
+ x7[7] = vqsub_s16(x5[15], x5[7]);
+ x7[8] = vqsub_s16(x5[0], x5[8]);
+ x7[9] = vqsub_s16(x5[1], x5[9]);
+ x7[10] = vqsub_s16(x5[2], x5[10]);
+ x7[11] = vqsub_s16(x5[3], x5[11]);
+ x7[12] = vqsub_s16(x5[4], x5[12]);
+ x7[13] = vqsub_s16(x5[5], x5[13]);
+ x7[14] = vqsub_s16(x5[6], x5[14]);
+ x7[15] = vqadd_s16(x5[7], x5[15]);
+
+ // stage 8
+ butterfly_s16_s32_x4_0112_neon(cospi2, x7[0], x7[1], &output[15], &output[0]);
+ butterfly_s16_s32_x4_0112_neon(cospi10, x7[2], x7[3], &output[13],
+ &output[2]);
+ butterfly_s16_s32_x4_0112_neon(cospi18, x7[4], x7[5], &output[11],
+ &output[4]);
+ butterfly_s16_s32_x4_0112_neon(cospi26, x7[6], x7[7], &output[9], &output[6]);
+ butterfly_s16_s32_x4_1003_neon(cospi30, x7[8], x7[9], &output[7], &output[8]);
+ butterfly_s16_s32_x4_1003_neon(cospi22, x7[10], x7[11], &output[5],
+ &output[10]);
+ butterfly_s16_s32_x4_1003_neon(cospi14, x7[12], x7[13], &output[3],
+ &output[12]);
+ butterfly_s16_s32_x4_0112_neon(cospi6, x7[14], x7[15], &output[14],
+ &output[1]);
+}
+
+static AOM_FORCE_INLINE void fadst8x16_neon(const int16x8_t *input,
+ int16x8_t *output, int cos_bit) {
+ const int16_t *cospi = cospi_arr_q13(cos_bit);
+
+ const int16x8_t cospi32_16 = vld1q_s16(&cospi[4 * 0]);
+ const int16x8_t cospi8_24 = vld1q_s16(&cospi[4 * 2]);
+ const int16x8_t cospi2_6 = vld1q_s16(&cospi[4 * 8]);
+ const int16x8_t cospi10_14 = vld1q_s16(&cospi[4 * 10]);
+ const int16x8_t cospi18_22 = vld1q_s16(&cospi[4 * 12]);
+ const int16x8_t cospi26_30 = vld1q_s16(&cospi[4 * 14]);
+
+ const int16x4_t cospi32 = vget_low_s16(cospi32_16);
+ const int16x4_t cospi16 = vget_high_s16(cospi32_16);
+ const int16x4_t cospi8 = vget_low_s16(cospi8_24);
+ const int16x4_t cospi24 = vget_high_s16(cospi8_24);
+ const int16x4_t cospi2 = vget_low_s16(cospi2_6);
+ const int16x4_t cospi6 = vget_high_s16(cospi2_6);
+ const int16x4_t cospi10 = vget_low_s16(cospi10_14);
+ const int16x4_t cospi14 = vget_high_s16(cospi10_14);
+ const int16x4_t cospi18 = vget_low_s16(cospi18_22);
+ const int16x4_t cospi22 = vget_high_s16(cospi18_22);
+ const int16x4_t cospi26 = vget_low_s16(cospi26_30);
+ const int16x4_t cospi30 = vget_high_s16(cospi26_30);
+
+ // stage 2
+ int16x8_t x2[8];
+ butterfly_s16_s32_x8_0332_neon(cospi32, input[8], input[7], &x2[0], &x2[1]);
+ butterfly_s16_s32_x8_0112_neon(cospi32, input[4], input[11], &x2[3], &x2[2]);
+ butterfly_s16_s32_x8_0112_neon(cospi32, input[6], input[9], &x2[5], &x2[4]);
+ butterfly_s16_s32_x8_0332_neon(cospi32, input[10], input[5], &x2[6], &x2[7]);
+
+ // stage 3
+ int16x8_t x3[16];
+ x3[0] = vqaddq_s16(input[0], x2[0]);
+ x3[1] = vqsubq_s16(x2[1], input[15]);
+ x3[2] = vqsubq_s16(input[0], x2[0]);
+ x3[3] = vqaddq_s16(input[15], x2[1]);
+ x3[4] = vqsubq_s16(x2[2], input[3]);
+ x3[5] = vqaddq_s16(input[12], x2[3]);
+ x3[6] = vqaddq_s16(input[3], x2[2]);
+ x3[7] = vqsubq_s16(input[12], x2[3]);
+ x3[8] = vqsubq_s16(x2[4], input[1]);
+ x3[9] = vqaddq_s16(input[14], x2[5]);
+ x3[10] = vqaddq_s16(input[1], x2[4]);
+ x3[11] = vqsubq_s16(input[14], x2[5]);
+ x3[12] = vqaddq_s16(input[2], x2[6]);
+ x3[13] = vqsubq_s16(x2[7], input[13]);
+ x3[14] = vqsubq_s16(input[2], x2[6]);
+ x3[15] = vqaddq_s16(input[13], x2[7]);
+
+ // stage 4
+ butterfly_s16_s32_x8_0112_neon(cospi16, x3[4], x3[5], &x3[4], &x3[5]);
+ butterfly_s16_s32_x8_0112_neon(cospi16, x3[7], x3[6], &x3[6], &x3[7]);
+ butterfly_s16_s32_x8_0112_neon(cospi16, x3[12], x3[13], &x3[12], &x3[13]);
+ butterfly_s16_s32_x8_0332_neon(cospi16, x3[14], x3[15], &x3[15], &x3[14]);
// stage 5
int16x8_t x5[16];
x5[0] = vqaddq_s16(x3[0], x3[4]);
- x5[4] = vqsubq_s16(x3[0], x3[4]);
x5[1] = vqaddq_s16(x3[1], x3[5]);
- x5[5] = vqsubq_s16(x3[1], x3[5]);
x5[2] = vqaddq_s16(x3[2], x3[6]);
+ x5[3] = vqsubq_s16(x3[7], x3[3]);
+ x5[4] = vqsubq_s16(x3[0], x3[4]);
+ x5[5] = vqsubq_s16(x3[1], x3[5]);
x5[6] = vqsubq_s16(x3[2], x3[6]);
- x5[3] = vqaddq_s16(x3[3], x3[7]);
- x5[7] = vqsubq_s16(x3[3], x3[7]);
+ x5[7] = vqaddq_s16(x3[3], x3[7]);
x5[8] = vqaddq_s16(x3[8], x3[12]);
- x5[12] = vqsubq_s16(x3[8], x3[12]);
x5[9] = vqaddq_s16(x3[9], x3[13]);
- x5[13] = vqsubq_s16(x3[9], x3[13]);
- x5[10] = vqaddq_s16(x3[10], x3[14]);
- x5[14] = vqsubq_s16(x3[10], x3[14]);
+ x5[10] = vqsubq_s16(x3[14], x3[10]);
x5[11] = vqaddq_s16(x3[11], x3[15]);
+ x5[12] = vqsubq_s16(x3[8], x3[12]);
+ x5[13] = vqsubq_s16(x3[9], x3[13]);
+ x5[14] = vqaddq_s16(x3[10], x3[14]);
x5[15] = vqsubq_s16(x3[11], x3[15]);
// stage 6
- btf_16_neon_mode3(cospi[8], cospi[56], x5[8], x5[9], x5[8], x5[9], v_cos_bit);
- btf_16_neon_mode3(cospi[40], cospi[24], x5[10], x5[11], x5[10], x5[11],
- v_cos_bit);
- btf_16_neon_mode0(cospi[56], cospi[8], x5[12], x5[13], x5[12], x5[13],
- v_cos_bit);
- btf_16_neon_mode0(cospi[24], cospi[40], x5[14], x5[15], x5[14], x5[15],
- v_cos_bit);
+ butterfly_s16_s32_x8_0112_neon(cospi8, x5[8], x5[9], &x5[8], &x5[9]);
+ butterfly_s16_s32_x8_1003_neon(cospi24, x5[10], x5[11], &x5[10], &x5[11]);
+ butterfly_s16_s32_x8_1003_neon(cospi8, x5[13], x5[12], &x5[13], &x5[12]);
+ butterfly_s16_s32_x8_1003_neon(cospi24, x5[15], x5[14], &x5[14], &x5[15]);
// stage 7
int16x8_t x7[16];
x7[0] = vqaddq_s16(x5[0], x5[8]);
- x7[8] = vqsubq_s16(x5[0], x5[8]);
x7[1] = vqaddq_s16(x5[1], x5[9]);
- x7[9] = vqsubq_s16(x5[1], x5[9]);
x7[2] = vqaddq_s16(x5[2], x5[10]);
- x7[10] = vqsubq_s16(x5[2], x5[10]);
x7[3] = vqaddq_s16(x5[3], x5[11]);
- x7[11] = vqsubq_s16(x5[3], x5[11]);
x7[4] = vqaddq_s16(x5[4], x5[12]);
- x7[12] = vqsubq_s16(x5[4], x5[12]);
x7[5] = vqaddq_s16(x5[5], x5[13]);
- x7[13] = vqsubq_s16(x5[5], x5[13]);
x7[6] = vqaddq_s16(x5[6], x5[14]);
+ x7[7] = vqsubq_s16(x5[15], x5[7]);
+ x7[8] = vqsubq_s16(x5[0], x5[8]);
+ x7[9] = vqsubq_s16(x5[1], x5[9]);
+ x7[10] = vqsubq_s16(x5[2], x5[10]);
+ x7[11] = vqsubq_s16(x5[3], x5[11]);
+ x7[12] = vqsubq_s16(x5[4], x5[12]);
+ x7[13] = vqsubq_s16(x5[5], x5[13]);
x7[14] = vqsubq_s16(x5[6], x5[14]);
- x7[7] = vqaddq_s16(x5[7], x5[15]);
- x7[15] = vqsubq_s16(x5[7], x5[15]);
+ x7[15] = vqaddq_s16(x5[7], x5[15]);
// stage 8
- btf_16_neon_mode3(cospi[2], cospi[62], x7[0], x7[1], output[15], output[0],
- v_cos_bit);
- btf_16_neon_mode3(cospi[10], cospi[54], x7[2], x7[3], output[13], output[2],
- v_cos_bit);
- btf_16_neon_mode3(cospi[18], cospi[46], x7[4], x7[5], output[11], output[4],
- v_cos_bit);
- btf_16_neon_mode3(cospi[26], cospi[38], x7[6], x7[7], output[9], output[6],
- v_cos_bit);
- btf_16_neon_mode3(cospi[34], cospi[30], x7[8], x7[9], output[7], output[8],
- v_cos_bit);
- btf_16_neon_mode3(cospi[42], cospi[22], x7[10], x7[11], output[5], output[10],
- v_cos_bit);
- btf_16_neon_mode3(cospi[50], cospi[14], x7[12], x7[13], output[3], output[12],
- v_cos_bit);
- btf_16_neon_mode3(cospi[58], cospi[6], x7[14], x7[15], output[1], output[14],
- v_cos_bit);
+ butterfly_s16_s32_x8_0112_neon(cospi2, x7[0], x7[1], &output[15], &output[0]);
+ butterfly_s16_s32_x8_0112_neon(cospi10, x7[2], x7[3], &output[13],
+ &output[2]);
+ butterfly_s16_s32_x8_0112_neon(cospi18, x7[4], x7[5], &output[11],
+ &output[4]);
+ butterfly_s16_s32_x8_0112_neon(cospi26, x7[6], x7[7], &output[9], &output[6]);
+ butterfly_s16_s32_x8_1003_neon(cospi30, x7[8], x7[9], &output[7], &output[8]);
+ butterfly_s16_s32_x8_1003_neon(cospi22, x7[10], x7[11], &output[5],
+ &output[10]);
+ butterfly_s16_s32_x8_1003_neon(cospi14, x7[12], x7[13], &output[3],
+ &output[12]);
+ butterfly_s16_s32_x8_0112_neon(cospi6, x7[14], x7[15], &output[14],
+ &output[1]);
}
-void av1_fidentity4x4_neon(const int16x8_t *const input,
- int16x8_t *const output, const int8_t cos_bit,
- const int8_t *stage_range) {
+static AOM_FORCE_INLINE void fidentity4x4_neon(const int16x4_t *const input,
+ int16x4_t *const output,
+ const int cos_bit) {
(void)cos_bit;
- (void)stage_range;
- const int16x4_t v_newsqrt2 = vdup_n_s16(NewSqrt2);
- for (int i = 0; i < 4; ++i) {
- const int16x4_t b = vqrshrn_n_s32(
- vmull_s16(vget_low_s16(input[i]), v_newsqrt2), NewSqrt2Bits);
- output[i] = vcombine_s16(b, b);
+ round_shift_sqrt2_s16_s16_4xn_neon(input, output, 4);
+}
+
+static AOM_FORCE_INLINE void fidentity8x4_neon(const int16x8_t *const input,
+ int16x8_t *const output,
+ const int cos_bit) {
+ (void)cos_bit;
+ round_shift_sqrt2_s16_s16_8xn_neon(input, output, 4);
+}
+
+static AOM_FORCE_INLINE void fidentity4x8_neon(const int16x4_t *input,
+ int16x4_t *output, int cos_bit) {
+ (void)cos_bit;
+ shift_left_1_s16_x4(input, output, 8);
+}
+
+static AOM_FORCE_INLINE void fidentity8x8_neon(const int16x8_t *input,
+ int16x8_t *output, int cos_bit) {
+ (void)cos_bit;
+ shift_left_1_s16_x8(input, output, 8);
+}
+
+static AOM_FORCE_INLINE void fidentity4x16_neon(const int16x4_t *input,
+ int16x4_t *output,
+ int cos_bit) {
+ (void)cos_bit;
+ round_shift_2sqrt2_s16_s16_4xn_neon(input, output, 16);
+}
+
+static AOM_FORCE_INLINE void fidentity8x16_neon(const int16x8_t *input,
+ int16x8_t *output,
+ int cos_bit) {
+ (void)cos_bit;
+ round_shift_2sqrt2_s16_s16_8xn_neon(input, output, 16);
+}
+
+static AOM_FORCE_INLINE void fidentity8x32_neon(const int16x8_t *input,
+ int16x8_t *output,
+ int cos_bit) {
+ (void)cos_bit;
+ shift_left_2_s16_x8(input, output, 32);
+}
+
+#define TRANSFORM_COL(name, tw, n) \
+ static void name##_col_neon(const int16_t *input, int16x##tw##_t *output, \
+ int stride, int cos_bit) { \
+ int16x##tw##_t buf0[n]; \
+ load_buffer_s16_x##tw(input, stride, buf0, n); \
+ shift_left_2_s16_x##tw(buf0, buf0, n); \
+ name##_neon(buf0, output, cos_bit); \
}
-}
-static INLINE void fidentity8x4_neon(const int16x8_t *const input,
- int16x8_t *const output,
- const int8_t cos_bit,
- const int8_t *stage_range) {
- (void)stage_range;
- (void)cos_bit;
- const int16x4_t v_newsqrt2 = vdup_n_s16(NewSqrt2);
- for (int i = 0; i < 4; ++i) {
- const int16x4_t b_lo = vqrshrn_n_s32(
- vmull_s16(vget_low_s16(input[i]), v_newsqrt2), NewSqrt2Bits);
- const int16x4_t b_hi = vqrshrn_n_s32(
- vmull_s16(vget_high_s16(input[i]), v_newsqrt2), NewSqrt2Bits);
- output[i] = vcombine_s16(b_lo, b_hi);
+TRANSFORM_COL(fadst4x4, 4, 4)
+TRANSFORM_COL(fadst4x8, 4, 8)
+TRANSFORM_COL(fadst4x16, 4, 16)
+TRANSFORM_COL(fadst8x4, 8, 4)
+TRANSFORM_COL(fadst8x8, 8, 8)
+TRANSFORM_COL(fadst8x16, 8, 16)
+TRANSFORM_COL(fdct4x4, 4, 4)
+TRANSFORM_COL(fdct4x8, 4, 8)
+TRANSFORM_COL(fdct4x16, 4, 16)
+TRANSFORM_COL(fdct8x4, 8, 4)
+TRANSFORM_COL(fdct8x8, 8, 8)
+TRANSFORM_COL(fdct8x16, 8, 16)
+TRANSFORM_COL(fdct8x32, 8, 32)
+TRANSFORM_COL(fidentity4x4, 4, 4)
+TRANSFORM_COL(fidentity4x8, 4, 8)
+TRANSFORM_COL(fidentity4x16, 4, 16)
+TRANSFORM_COL(fidentity8x4, 8, 4)
+TRANSFORM_COL(fidentity8x8, 8, 8)
+TRANSFORM_COL(fidentity8x16, 8, 16)
+TRANSFORM_COL(fidentity8x32, 8, 32)
+
+#define TRANSFORM_ROW(name, tw, n) \
+ static void name##_row_neon(const int16x##tw##_t *input, int32_t *output, \
+ int stride, int cos_bit) { \
+ int16x##tw##_t buf0[n]; \
+ name##_neon(input, buf0, cos_bit); \
+ store_buffer_s16_x##tw(buf0, output, stride, n); \
}
-}
-void fidentity8x8_neon(const int16x8_t *input, int16x8_t *output,
- int8_t cos_bit, const int8_t *stage_range) {
- (void)cos_bit;
- (void)stage_range;
- int16x8_t one = vdupq_n_s16(1);
- output[0] = vqrshlq_s16(input[0], one);
- output[1] = vqrshlq_s16(input[1], one);
- output[2] = vqrshlq_s16(input[2], one);
- output[3] = vqrshlq_s16(input[3], one);
- output[4] = vqrshlq_s16(input[4], one);
- output[5] = vqrshlq_s16(input[5], one);
- output[6] = vqrshlq_s16(input[6], one);
- output[7] = vqrshlq_s16(input[7], one);
-}
-
-static INLINE void fidentity8x16_neon(const int16x8_t *input, int16x8_t *output,
- int8_t cos_bit,
- const int8_t *stage_range) {
- (void)stage_range;
- (void)cos_bit;
- const int16x4_t v_newsqrt2 = vdup_n_s16(NewSqrt2 * 2);
- for (int i = 0; i < 16; ++i) {
- const int16x4_t b_lo = vqrshrn_n_s32(
- vmull_s16(vget_low_s16(input[i]), v_newsqrt2), NewSqrt2Bits);
- const int16x4_t b_hi = vqrshrn_n_s32(
- vmull_s16(vget_high_s16(input[i]), v_newsqrt2), NewSqrt2Bits);
- output[i] = vcombine_s16(b_lo, b_hi);
+#define TRANSFORM_ROW_RECT(name, tw, n) \
+ static void name##_row_rect_neon(const int16x##tw##_t *input, \
+ int32_t *output, int stride, int cos_bit) { \
+ int16x##tw##_t buf0[n]; \
+ name##_neon(input, buf0, cos_bit); \
+ store_rect_buffer_s16_x##tw(buf0, output, stride, n); \
}
-}
-static INLINE void fidentity8x32_neon(const int16x8_t *input, int16x8_t *output,
- int8_t cos_bit,
- const int8_t *stage_range) {
- (void)stage_range;
- (void)cos_bit;
- for (int i = 0; i < 32; ++i) {
- output[i] = vshlq_n_s16(input[i], 2);
- }
-}
+TRANSFORM_ROW(fadst4x4, 4, 4)
+TRANSFORM_ROW(fadst4x16, 4, 16)
+TRANSFORM_ROW(fadst8x4, 8, 4)
+TRANSFORM_ROW(fadst8x8, 8, 8)
+TRANSFORM_ROW(fadst8x16, 8, 16)
+TRANSFORM_ROW(fdct4x4, 4, 4)
+TRANSFORM_ROW(fdct4x16, 4, 16)
+TRANSFORM_ROW(fdct8x4, 8, 4)
+TRANSFORM_ROW(fdct8x8, 8, 8)
+TRANSFORM_ROW(fdct8x16, 8, 16)
+TRANSFORM_ROW(fdct8x32, 8, 32)
+TRANSFORM_ROW(fidentity4x4, 4, 4)
+TRANSFORM_ROW(fidentity4x16, 4, 16)
+TRANSFORM_ROW(fidentity8x4, 8, 4)
+TRANSFORM_ROW(fidentity8x8, 8, 8)
+TRANSFORM_ROW(fidentity8x16, 8, 16)
+TRANSFORM_ROW(fidentity8x32, 8, 32)
-typedef void (*transform_1d_lbd_neon)(const int16x8_t *input, int16x8_t *output,
- int8_t cos_bit,
- const int8_t *stage_range);
+TRANSFORM_ROW_RECT(fadst4x8, 4, 8)
+TRANSFORM_ROW_RECT(fadst8x4, 8, 4)
+TRANSFORM_ROW_RECT(fadst8x8, 8, 8)
+TRANSFORM_ROW_RECT(fadst8x16, 8, 16)
+TRANSFORM_ROW_RECT(fdct4x8, 4, 8)
+TRANSFORM_ROW_RECT(fdct8x4, 8, 4)
+TRANSFORM_ROW_RECT(fdct8x8, 8, 8)
+TRANSFORM_ROW_RECT(fdct8x16, 8, 16)
+TRANSFORM_ROW_RECT(fdct8x32, 8, 32)
+TRANSFORM_ROW_RECT(fidentity4x8, 4, 8)
+TRANSFORM_ROW_RECT(fidentity8x4, 8, 4)
+TRANSFORM_ROW_RECT(fidentity8x8, 8, 8)
+TRANSFORM_ROW_RECT(fidentity8x16, 8, 16)
+TRANSFORM_ROW_RECT(fidentity8x32, 8, 32)
-static const transform_1d_lbd_neon col_txfm4x4_arr[TX_TYPES] = {
- av1_fdct4x4_neon, // DCT_DCT
- av1_fadst4x4_neon, // ADST_DCT
- av1_fdct4x4_neon, // DCT_ADST
- av1_fadst4x4_neon, // ADST_ADST
- av1_fadst4x4_neon, // FLIPADST_DCT
- av1_fdct4x4_neon, // DCT_FLIPADST
- av1_fadst4x4_neon, // FLIPADST_FLIPADST
- av1_fadst4x4_neon, // ADST_FLIPADST
- av1_fadst4x4_neon, // FLIPADST_ADST
- av1_fidentity4x4_neon, // IDTX
- av1_fdct4x4_neon, // V_DCT
- av1_fidentity4x4_neon, // H_DCT
- av1_fadst4x4_neon, // V_ADST
- av1_fidentity4x4_neon, // H_ADST
- av1_fadst4x4_neon, // V_FLIPADST
- av1_fidentity4x4_neon // H_FLIPADST
+typedef void (*transform_1d_lbd_4_neon)(const int16x4_t *input,
+ int16x4_t *output, int cos_bit);
+typedef void (*transform_1d_lbd_8_neon)(const int16x8_t *input,
+ int16x8_t *output, int cos_bit);
+
+typedef void (*col_transform_1d_lbd_4_neon)(const int16_t *input,
+ int16x4_t *output, int stride,
+ int cos_bit);
+typedef void (*col_transform_1d_lbd_8_neon)(const int16_t *input,
+ int16x8_t *output, int stride,
+ int cos_bit);
+
+typedef void (*row_transform_1d_lbd_4_neon)(const int16x4_t *input,
+ int32_t *output, int stride,
+ int cos_bit);
+typedef void (*row_transform_1d_lbd_8_neon)(const int16x8_t *input,
+ int32_t *output, int stride,
+ int cos_bit);
+
+static const col_transform_1d_lbd_4_neon col_txfm4x4_arr[TX_TYPES] = {
+ fdct4x4_col_neon, // DCT_DCT
+ fadst4x4_col_neon, // ADST_DCT
+ fdct4x4_col_neon, // DCT_ADST
+ fadst4x4_col_neon, // ADST_ADST
+ fadst4x4_col_neon, // FLIPADST_DCT
+ fdct4x4_col_neon, // DCT_FLIPADST
+ fadst4x4_col_neon, // FLIPADST_FLIPADST
+ fadst4x4_col_neon, // ADST_FLIPADST
+ fadst4x4_col_neon, // FLIPADST_ADST
+ fidentity4x4_col_neon, // IDTX
+ fdct4x4_col_neon, // V_DCT
+ fidentity4x4_col_neon, // H_DCT
+ fadst4x4_col_neon, // V_ADST
+ fidentity4x4_col_neon, // H_ADST
+ fadst4x4_col_neon, // V_FLIPADST
+ fidentity4x4_col_neon // H_FLIPADST
};
-static const transform_1d_lbd_neon row_txfm4x4_arr[TX_TYPES] = {
- av1_fdct4x4_neon, // DCT_DCT
- av1_fdct4x4_neon, // ADST_DCT
- av1_fadst4x4_neon, // DCT_ADST
- av1_fadst4x4_neon, // ADST_ADST
- av1_fdct4x4_neon, // FLIPADST_DCT
- av1_fadst4x4_neon, // DCT_FLIPADST
- av1_fadst4x4_neon, // FLIPADST_FLIPADST
- av1_fadst4x4_neon, // ADST_FLIPADST
- av1_fadst4x4_neon, // FLIPADST_ADST
- av1_fidentity4x4_neon, // IDTX
- av1_fidentity4x4_neon, // V_DCT
- av1_fdct4x4_neon, // H_DCT
- av1_fidentity4x4_neon, // V_ADST
- av1_fadst4x4_neon, // H_ADST
- av1_fidentity4x4_neon, // V_FLIPADST
- av1_fadst4x4_neon // H_FLIPADST
+static const row_transform_1d_lbd_4_neon row_txfm4x4_arr[TX_TYPES] = {
+ fdct4x4_row_neon, // DCT_DCT
+ fdct4x4_row_neon, // ADST_DCT
+ fadst4x4_row_neon, // DCT_ADST
+ fadst4x4_row_neon, // ADST_ADST
+ fdct4x4_row_neon, // FLIPADST_DCT
+ fadst4x4_row_neon, // DCT_FLIPADST
+ fadst4x4_row_neon, // FLIPADST_FLIPADST
+ fadst4x4_row_neon, // ADST_FLIPADST
+ fadst4x4_row_neon, // FLIPADST_ADST
+ fidentity4x4_row_neon, // IDTX
+ fidentity4x4_row_neon, // V_DCT
+ fdct4x4_row_neon, // H_DCT
+ fidentity4x4_row_neon, // V_ADST
+ fadst4x4_row_neon, // H_ADST
+ fidentity4x4_row_neon, // V_FLIPADST
+ fadst4x4_row_neon // H_FLIPADST
};
-static const transform_1d_lbd_neon col_txfm4x8_arr[TX_TYPES] = {
- fdct4x8_neon, // DCT_DCT
- fadst4x8_neon, // ADST_DCT
- fdct4x8_neon, // DCT_ADST
- fadst4x8_neon, // ADST_ADST
- fadst4x8_neon, // FLIPADST_DCT
- fdct4x8_neon, // DCT_FLIPADST
- fadst4x8_neon, // FLIPADST_FLIPADST
- fadst4x8_neon, // ADST_FLIPADST
- fadst4x8_neon, // FLIPADST_ADST
- fidentity8x8_neon, // IDTX
- fdct4x8_neon, // V_DCT
- fidentity8x8_neon, // H_DCT
- fadst4x8_neon, // V_ADST
- fidentity8x8_neon, // H_ADST
- fadst4x8_neon, // V_FLIPADST
- fidentity8x8_neon // H_FLIPADST
+static const col_transform_1d_lbd_4_neon col_txfm4x8_arr[TX_TYPES] = {
+ fdct4x8_col_neon, // DCT_DCT
+ fadst4x8_col_neon, // ADST_DCT
+ fdct4x8_col_neon, // DCT_ADST
+ fadst4x8_col_neon, // ADST_ADST
+ fadst4x8_col_neon, // FLIPADST_DCT
+ fdct4x8_col_neon, // DCT_FLIPADST
+ fadst4x8_col_neon, // FLIPADST_FLIPADST
+ fadst4x8_col_neon, // ADST_FLIPADST
+ fadst4x8_col_neon, // FLIPADST_ADST
+ fidentity4x8_col_neon, // IDTX
+ fdct4x8_col_neon, // V_DCT
+ fidentity4x8_col_neon, // H_DCT
+ fadst4x8_col_neon, // V_ADST
+ fidentity4x8_col_neon, // H_ADST
+ fadst4x8_col_neon, // V_FLIPADST
+ fidentity4x8_col_neon // H_FLIPADST
};
-static const transform_1d_lbd_neon row_txfm8x4_arr[TX_TYPES] = {
- fdct8x4_neon, // DCT_DCT
- fdct8x4_neon, // ADST_DCT
- fadst8x4_neon, // DCT_ADST
- fadst8x4_neon, // ADST_ADST
- fdct8x4_neon, // FLIPADST_DCT
- fadst8x4_neon, // DCT_FLIPADST
- fadst8x4_neon, // FLIPADST_FLIPADST
- fadst8x4_neon, // ADST_FLIPADST
- fadst8x4_neon, // FLIPADST_ADST
- fidentity8x4_neon, // IDTX
- fidentity8x4_neon, // V_DCT
- fdct8x4_neon, // H_DCT
- fidentity8x4_neon, // V_ADST
- fadst8x4_neon, // H_ADST
- fidentity8x4_neon, // V_FLIPADST
- fadst8x4_neon // H_FLIPADST
+static const row_transform_1d_lbd_8_neon row_txfm8x4_arr[TX_TYPES] = {
+ fdct8x4_row_neon, // DCT_DCT
+ fdct8x4_row_neon, // ADST_DCT
+ fadst8x4_row_neon, // DCT_ADST
+ fadst8x4_row_neon, // ADST_ADST
+ fdct8x4_row_neon, // FLIPADST_DCT
+ fadst8x4_row_neon, // DCT_FLIPADST
+ fadst8x4_row_neon, // FLIPADST_FLIPADST
+ fadst8x4_row_neon, // ADST_FLIPADST
+ fadst8x4_row_neon, // FLIPADST_ADST
+ fidentity8x4_row_neon, // IDTX
+ fidentity8x4_row_neon, // V_DCT
+ fdct8x4_row_neon, // H_DCT
+ fidentity8x4_row_neon, // V_ADST
+ fadst8x4_row_neon, // H_ADST
+ fidentity8x4_row_neon, // V_FLIPADST
+ fadst8x4_row_neon // H_FLIPADST
};
-static const transform_1d_lbd_neon col_txfm8x4_arr[TX_TYPES] = {
- fdct8x4_neon, // DCT_DCT
- fadst8x4_neon, // ADST_DCT
- fdct8x4_neon, // DCT_ADST
- fadst8x4_neon, // ADST_ADST
- fadst8x4_neon, // FLIPADST_DCT
- fdct8x4_neon, // DCT_FLIPADST
- fadst8x4_neon, // FLIPADST_FLIPADST
- fadst8x4_neon, // ADST_FLIPADST
- fadst8x4_neon, // FLIPADST_ADST
- fidentity8x4_neon, // IDTX
- fdct8x4_neon, // V_DCT
- fidentity8x4_neon, // H_DCT
- fadst8x4_neon, // V_ADST
- fidentity8x4_neon, // H_ADST
- fadst8x4_neon, // V_FLIPADST
- fidentity8x4_neon // H_FLIPADST
+static const row_transform_1d_lbd_8_neon row_rect_txfm8x4_arr[TX_TYPES] = {
+ fdct8x4_row_rect_neon, // DCT_DCT
+ fdct8x4_row_rect_neon, // ADST_DCT
+ fadst8x4_row_rect_neon, // DCT_ADST
+ fadst8x4_row_rect_neon, // ADST_ADST
+ fdct8x4_row_rect_neon, // FLIPADST_DCT
+ fadst8x4_row_rect_neon, // DCT_FLIPADST
+ fadst8x4_row_rect_neon, // FLIPADST_FLIPADST
+ fadst8x4_row_rect_neon, // ADST_FLIPADST
+ fadst8x4_row_rect_neon, // FLIPADST_ADST
+ fidentity8x4_row_rect_neon, // IDTX
+ fidentity8x4_row_rect_neon, // V_DCT
+ fdct8x4_row_rect_neon, // H_DCT
+ fidentity8x4_row_rect_neon, // V_ADST
+ fadst8x4_row_rect_neon, // H_ADST
+ fidentity8x4_row_rect_neon, // V_FLIPADST
+ fadst8x4_row_rect_neon // H_FLIPADST
};
-static const transform_1d_lbd_neon row_txfm4x8_arr[TX_TYPES] = {
- fdct4x8_neon, // DCT_DCT
- fdct4x8_neon, // ADST_DCT
- fadst4x8_neon, // DCT_ADST
- fadst4x8_neon, // ADST_ADST
- fdct4x8_neon, // FLIPADST_DCT
- fadst4x8_neon, // DCT_FLIPADST
- fadst4x8_neon, // FLIPADST_FLIPADST
- fadst4x8_neon, // ADST_FLIPADST
- fadst4x8_neon, // FLIPADST_ADST
- fidentity8x8_neon, // IDTX
- fidentity8x8_neon, // V_DCT
- fdct4x8_neon, // H_DCT
- fidentity8x8_neon, // V_ADST
- fadst4x8_neon, // H_ADST
- fidentity8x8_neon, // V_FLIPADST
- fadst4x8_neon // H_FLIPADST
+static const col_transform_1d_lbd_8_neon col_txfm8x4_arr[TX_TYPES] = {
+ fdct8x4_col_neon, // DCT_DCT
+ fadst8x4_col_neon, // ADST_DCT
+ fdct8x4_col_neon, // DCT_ADST
+ fadst8x4_col_neon, // ADST_ADST
+ fadst8x4_col_neon, // FLIPADST_DCT
+ fdct8x4_col_neon, // DCT_FLIPADST
+ fadst8x4_col_neon, // FLIPADST_FLIPADST
+ fadst8x4_col_neon, // ADST_FLIPADST
+ fadst8x4_col_neon, // FLIPADST_ADST
+ fidentity8x4_col_neon, // IDTX
+ fdct8x4_col_neon, // V_DCT
+ fidentity8x4_col_neon, // H_DCT
+ fadst8x4_col_neon, // V_ADST
+ fidentity8x4_col_neon, // H_ADST
+ fadst8x4_col_neon, // V_FLIPADST
+ fidentity8x4_col_neon // H_FLIPADST
};
-static const transform_1d_lbd_neon col_txfm8x8_arr[TX_TYPES] = {
- fdct8x8_neon, // DCT_DCT
- fadst_8x8_neon, // ADST_DCT
- fdct8x8_neon, // DCT_ADST
- fadst_8x8_neon, // ADST_ADST
- fadst_8x8_neon, // FLIPADST_DCT
- fdct8x8_neon, // DCT_FLIPADST
- fadst_8x8_neon, // FLIPADST_FLIPADST
- fadst_8x8_neon, // ADST_FLIPADST
- fadst_8x8_neon, // FLIPADST_ADST
- fidentity8x8_neon, // IDTX
- fdct8x8_neon, // V_DCT
- fidentity8x8_neon, // H_DCT
- fadst_8x8_neon, // V_ADST
- fidentity8x8_neon, // H_ADST
- fadst_8x8_neon, // V_FLIPADST
- fidentity8x8_neon, // H_FLIPADST
+static const row_transform_1d_lbd_4_neon row_rect_txfm4x8_arr[TX_TYPES] = {
+ fdct4x8_row_rect_neon, // DCT_DCT
+ fdct4x8_row_rect_neon, // ADST_DCT
+ fadst4x8_row_rect_neon, // DCT_ADST
+ fadst4x8_row_rect_neon, // ADST_ADST
+ fdct4x8_row_rect_neon, // FLIPADST_DCT
+ fadst4x8_row_rect_neon, // DCT_FLIPADST
+ fadst4x8_row_rect_neon, // FLIPADST_FLIPADST
+ fadst4x8_row_rect_neon, // ADST_FLIPADST
+ fadst4x8_row_rect_neon, // FLIPADST_ADST
+ fidentity4x8_row_rect_neon, // IDTX
+ fidentity4x8_row_rect_neon, // V_DCT
+ fdct4x8_row_rect_neon, // H_DCT
+ fidentity4x8_row_rect_neon, // V_ADST
+ fadst4x8_row_rect_neon, // H_ADST
+ fidentity4x8_row_rect_neon, // V_FLIPADST
+ fadst4x8_row_rect_neon // H_FLIPADST
};
-static const transform_1d_lbd_neon row_txfm8x8_arr[TX_TYPES] = {
- fdct8x8_neon, // DCT_DCT
- fdct8x8_neon, // ADST_DCT
- fadst_8x8_neon, // DCT_ADST
- fadst_8x8_neon, // ADST_ADST
- fdct8x8_neon, // FLIPADST_DCT
- fadst_8x8_neon, // DCT_FLIPADST
- fadst_8x8_neon, // FLIPADST_FLIPADST
- fadst_8x8_neon, // ADST_FLIPADST
- fadst_8x8_neon, // FLIPADST_ADST
- fidentity8x8_neon, // IDTX
- fidentity8x8_neon, // V_DCT
- fdct8x8_neon, // H_DCT
- fidentity8x8_neon, // V_ADST
- fadst_8x8_neon, // H_ADST
- fidentity8x8_neon, // V_FLIPADST
- fadst_8x8_neon // H_FLIPADST
+static const col_transform_1d_lbd_8_neon col_txfm8x8_arr[TX_TYPES] = {
+ fdct8x8_col_neon, // DCT_DCT
+ fadst8x8_col_neon, // ADST_DCT
+ fdct8x8_col_neon, // DCT_ADST
+ fadst8x8_col_neon, // ADST_ADST
+ fadst8x8_col_neon, // FLIPADST_DCT
+ fdct8x8_col_neon, // DCT_FLIPADST
+ fadst8x8_col_neon, // FLIPADST_FLIPADST
+ fadst8x8_col_neon, // ADST_FLIPADST
+ fadst8x8_col_neon, // FLIPADST_ADST
+ fidentity8x8_col_neon, // IDTX
+ fdct8x8_col_neon, // V_DCT
+ fidentity8x8_col_neon, // H_DCT
+ fadst8x8_col_neon, // V_ADST
+ fidentity8x8_col_neon, // H_ADST
+ fadst8x8_col_neon, // V_FLIPADST
+ fidentity8x8_col_neon, // H_FLIPADST
};
-static const transform_1d_lbd_neon col_txfm8x16_arr[TX_TYPES] = {
- fdct8x16_neon, // DCT_DCT
- fadst8x16_neon, // ADST_DCT
- fdct8x16_neon, // DCT_ADST
- fadst8x16_neon, // ADST_ADST
- fadst8x16_neon, // FLIPADST_DCT
- fdct8x16_neon, // DCT_FLIPADST
- fadst8x16_neon, // FLIPADST_FLIPADST
- fadst8x16_neon, // ADST_FLIPADST
- fadst8x16_neon, // FLIPADST_ADST
- fidentity8x16_neon, // IDTX
- fdct8x16_neon, // V_DCT
- fidentity8x16_neon, // H_DCT
- fadst8x16_neon, // V_ADST
- fidentity8x16_neon, // H_ADST
- fadst8x16_neon, // V_FLIPADST
- fidentity8x16_neon // H_FLIPADST
+static const row_transform_1d_lbd_8_neon row_txfm8x8_arr[TX_TYPES] = {
+ fdct8x8_row_neon, // DCT_DCT
+ fdct8x8_row_neon, // ADST_DCT
+ fadst8x8_row_neon, // DCT_ADST
+ fadst8x8_row_neon, // ADST_ADST
+ fdct8x8_row_neon, // FLIPADST_DCT
+ fadst8x8_row_neon, // DCT_FLIPADST
+ fadst8x8_row_neon, // FLIPADST_FLIPADST
+ fadst8x8_row_neon, // ADST_FLIPADST
+ fadst8x8_row_neon, // FLIPADST_ADST
+ fidentity8x8_row_neon, // IDTX
+ fidentity8x8_row_neon, // V_DCT
+ fdct8x8_row_neon, // H_DCT
+ fidentity8x8_row_neon, // V_ADST
+ fadst8x8_row_neon, // H_ADST
+ fidentity8x8_row_neon, // V_FLIPADST
+ fadst8x8_row_neon // H_FLIPADST
};
-static const transform_1d_lbd_neon row_txfm8x16_arr[TX_TYPES] = {
- fdct8x16_neon, // DCT_DCT
- fdct8x16_neon, // ADST_DCT
- fadst8x16_neon, // DCT_ADST
- fadst8x16_neon, // ADST_ADST
- fdct8x16_neon, // FLIPADST_DCT
- fadst8x16_neon, // DCT_FLIPADST
- fadst8x16_neon, // FLIPADST_FLIPADST
- fadst8x16_neon, // ADST_FLIPADST
- fadst8x16_neon, // FLIPADST_ADST
- fidentity8x16_neon, // IDTX
- fidentity8x16_neon, // V_DCT
- fdct8x16_neon, // H_DCT
- fidentity8x16_neon, // V_ADST
- fadst8x16_neon, // H_ADST
- fidentity8x16_neon, // V_FLIPADST
- fadst8x16_neon // H_FLIPADST
+static const row_transform_1d_lbd_8_neon row_rect_txfm8x8_arr[TX_TYPES] = {
+ fdct8x8_row_rect_neon, // DCT_DCT
+ fdct8x8_row_rect_neon, // ADST_DCT
+ fadst8x8_row_rect_neon, // DCT_ADST
+ fadst8x8_row_rect_neon, // ADST_ADST
+ fdct8x8_row_rect_neon, // FLIPADST_DCT
+ fadst8x8_row_rect_neon, // DCT_FLIPADST
+ fadst8x8_row_rect_neon, // FLIPADST_FLIPADST
+ fadst8x8_row_rect_neon, // ADST_FLIPADST
+ fadst8x8_row_rect_neon, // FLIPADST_ADST
+ fidentity8x8_row_rect_neon, // IDTX
+ fidentity8x8_row_rect_neon, // V_DCT
+ fdct8x8_row_rect_neon, // H_DCT
+ fidentity8x8_row_rect_neon, // V_ADST
+ fadst8x8_row_rect_neon, // H_ADST
+ fidentity8x8_row_rect_neon, // V_FLIPADST
+ fadst8x8_row_rect_neon // H_FLIPADST
};
-static const transform_1d_lbd_neon row_txfm8x32_arr[TX_TYPES] = {
- av1_fdct8x32_neon, // DCT_DCT
- NULL, // ADST_DCT
- NULL, // DCT_ADST
- NULL, // ADST_ADST
- NULL, // FLIPADST_DCT
- NULL, // DCT_FLIPADST
- NULL, // FLIPADST_FLIPADST
- NULL, // ADST_FLIPADST
- NULL, // FLIPADST_ADST
- fidentity8x32_neon, // IDTX
- fidentity8x32_neon, // V_DCT
- av1_fdct8x32_neon, // H_DCT
- NULL, // V_ADST
- NULL, // H_ADST
- NULL, // V_FLIPADST
- NULL // H_FLIPADST
+static const col_transform_1d_lbd_4_neon col_txfm4x16_arr[TX_TYPES] = {
+ fdct4x16_col_neon, // DCT_DCT
+ fadst4x16_col_neon, // ADST_DCT
+ fdct4x16_col_neon, // DCT_ADST
+ fadst4x16_col_neon, // ADST_ADST
+ fadst4x16_col_neon, // FLIPADST_DCT
+ fdct4x16_col_neon, // DCT_FLIPADST
+ fadst4x16_col_neon, // FLIPADST_FLIPADST
+ fadst4x16_col_neon, // ADST_FLIPADST
+ fadst4x16_col_neon, // FLIPADST_ADST
+ fidentity4x16_col_neon, // IDTX
+ fdct4x16_col_neon, // V_DCT
+ fidentity4x16_col_neon, // H_DCT
+ fadst4x16_col_neon, // V_ADST
+ fidentity4x16_col_neon, // H_ADST
+ fadst4x16_col_neon, // V_FLIPADST
+ fidentity4x16_col_neon // H_FLIPADST
};
-static const transform_1d_lbd_neon col_txfm8x32_arr[TX_TYPES] = {
- av1_fdct8x32_neon, // DCT_DCT
- NULL, // ADST_DCT
- NULL, // DCT_ADST
- NULL, // ADST_ADST
- NULL, // FLIPADST_DCT
- NULL, // DCT_FLIPADST
- NULL, // FLIPADST_FLIPADST
- NULL, // ADST_FLIPADST
- NULL, // FLIPADST_ADST
- fidentity8x32_neon, // IDTX
- av1_fdct8x32_neon, // V_DCT
- fidentity8x32_neon, // H_DCT
- NULL, // V_ADST
- NULL, // H_ADST
- NULL, // V_FLIPADST
- NULL // H_FLIPADST
+static const row_transform_1d_lbd_4_neon row_txfm4x16_arr[TX_TYPES] = {
+ fdct4x16_row_neon, // DCT_DCT
+ fdct4x16_row_neon, // ADST_DCT
+ fadst4x16_row_neon, // DCT_ADST
+ fadst4x16_row_neon, // ADST_ADST
+ fdct4x16_row_neon, // FLIPADST_DCT
+ fadst4x16_row_neon, // DCT_FLIPADST
+ fadst4x16_row_neon, // FLIPADST_FLIPADST
+ fadst4x16_row_neon, // ADST_FLIPADST
+ fadst4x16_row_neon, // FLIPADST_ADST
+ fidentity4x16_row_neon, // IDTX
+ fidentity4x16_row_neon, // V_DCT
+ fdct4x16_row_neon, // H_DCT
+ fidentity4x16_row_neon, // V_ADST
+ fadst4x16_row_neon, // H_ADST
+ fidentity4x16_row_neon, // V_FLIPADST
+ fadst4x16_row_neon // H_FLIPADST
};
-void av1_lowbd_fwd_txfm2d_4x4_neon(const int16_t *input, int32_t *output,
- int stride, TX_TYPE tx_type, int bd) {
+static const col_transform_1d_lbd_8_neon col_txfm8x16_arr[TX_TYPES] = {
+ fdct8x16_col_neon, // DCT_DCT
+ fadst8x16_col_neon, // ADST_DCT
+ fdct8x16_col_neon, // DCT_ADST
+ fadst8x16_col_neon, // ADST_ADST
+ fadst8x16_col_neon, // FLIPADST_DCT
+ fdct8x16_col_neon, // DCT_FLIPADST
+ fadst8x16_col_neon, // FLIPADST_FLIPADST
+ fadst8x16_col_neon, // ADST_FLIPADST
+ fadst8x16_col_neon, // FLIPADST_ADST
+ fidentity8x16_col_neon, // IDTX
+ fdct8x16_col_neon, // V_DCT
+ fidentity8x16_col_neon, // H_DCT
+ fadst8x16_col_neon, // V_ADST
+ fidentity8x16_col_neon, // H_ADST
+ fadst8x16_col_neon, // V_FLIPADST
+ fidentity8x16_col_neon // H_FLIPADST
+};
+
+static const row_transform_1d_lbd_8_neon row_txfm8x16_arr[TX_TYPES] = {
+ fdct8x16_row_neon, // DCT_DCT
+ fdct8x16_row_neon, // ADST_DCT
+ fadst8x16_row_neon, // DCT_ADST
+ fadst8x16_row_neon, // ADST_ADST
+ fdct8x16_row_neon, // FLIPADST_DCT
+ fadst8x16_row_neon, // DCT_FLIPADST
+ fadst8x16_row_neon, // FLIPADST_FLIPADST
+ fadst8x16_row_neon, // ADST_FLIPADST
+ fadst8x16_row_neon, // FLIPADST_ADST
+ fidentity8x16_row_neon, // IDTX
+ fidentity8x16_row_neon, // V_DCT
+ fdct8x16_row_neon, // H_DCT
+ fidentity8x16_row_neon, // V_ADST
+ fadst8x16_row_neon, // H_ADST
+ fidentity8x16_row_neon, // V_FLIPADST
+ fadst8x16_row_neon // H_FLIPADST
+};
+
+static const row_transform_1d_lbd_8_neon row_rect_txfm8x16_arr[TX_TYPES] = {
+ fdct8x16_row_rect_neon, // DCT_DCT
+ fdct8x16_row_rect_neon, // ADST_DCT
+ fadst8x16_row_rect_neon, // DCT_ADST
+ fadst8x16_row_rect_neon, // ADST_ADST
+ fdct8x16_row_rect_neon, // FLIPADST_DCT
+ fadst8x16_row_rect_neon, // DCT_FLIPADST
+ fadst8x16_row_rect_neon, // FLIPADST_FLIPADST
+ fadst8x16_row_rect_neon, // ADST_FLIPADST
+ fadst8x16_row_rect_neon, // FLIPADST_ADST
+ fidentity8x16_row_rect_neon, // IDTX
+ fidentity8x16_row_rect_neon, // V_DCT
+ fdct8x16_row_rect_neon, // H_DCT
+ fidentity8x16_row_rect_neon, // V_ADST
+ fadst8x16_row_rect_neon, // H_ADST
+ fidentity8x16_row_rect_neon, // V_FLIPADST
+ fadst8x16_row_rect_neon // H_FLIPADST
+};
+
+static const row_transform_1d_lbd_8_neon row_txfm8x32_arr[TX_TYPES] = {
+ fdct8x32_row_neon, // DCT_DCT
+ NULL, // ADST_DCT
+ NULL, // DCT_ADST
+ NULL, // ADST_ADST
+ NULL, // FLIPADST_DCT
+ NULL, // DCT_FLIPADST
+ NULL, // FLIPADST_FLIPADST
+ NULL, // ADST_FLIPADST
+ NULL, // FLIPADST_ADST
+ fidentity8x32_row_neon, // IDTX
+ fidentity8x32_row_neon, // V_DCT
+ fdct8x32_row_neon, // H_DCT
+ NULL, // V_ADST
+ NULL, // H_ADST
+ NULL, // V_FLIPADST
+ NULL // H_FLIPADST
+};
+
+static const row_transform_1d_lbd_8_neon row_rect_txfm8x32_arr[TX_TYPES] = {
+ fdct8x32_row_rect_neon, // DCT_DCT
+ NULL, // ADST_DCT
+ NULL, // DCT_ADST
+ NULL, // ADST_ADST
+ NULL, // FLIPADST_DCT
+ NULL, // DCT_FLIPADST
+ NULL, // FLIPADST_FLIPADST
+ NULL, // ADST_FLIPADST
+ NULL, // FLIPADST_ADST
+ fidentity8x32_row_rect_neon, // IDTX
+ fidentity8x32_row_rect_neon, // V_DCT
+ fdct8x32_row_rect_neon, // H_DCT
+ NULL, // V_ADST
+ NULL, // H_ADST
+ NULL, // V_FLIPADST
+ NULL // H_FLIPADST
+};
+
+static const col_transform_1d_lbd_8_neon col_txfm8x32_arr[TX_TYPES] = {
+ fdct8x32_col_neon, // DCT_DCT
+ NULL, // ADST_DCT
+ NULL, // DCT_ADST
+ NULL, // ADST_ADST
+ NULL, // FLIPADST_DCT
+ NULL, // DCT_FLIPADST
+ NULL, // FLIPADST_FLIPADST
+ NULL, // ADST_FLIPADST
+ NULL, // FLIPADST_ADST
+ fidentity8x32_col_neon, // IDTX
+ fdct8x32_col_neon, // V_DCT
+ fidentity8x32_col_neon, // H_DCT
+ NULL, // V_ADST
+ NULL, // H_ADST
+ NULL, // V_FLIPADST
+ NULL // H_FLIPADST
+};
+
+static void lowbd_fwd_txfm2d_4x4_neon(const int16_t *input, int32_t *output,
+ int stride, TX_TYPE tx_type, int bd) {
(void)bd;
- int16x8_t buf0[4], buf1[4], *buf;
- const int8_t *shift = av1_fwd_txfm_shift_ls[TX_4X4];
- const int txw_idx = get_txw_idx(TX_4X4);
- const int txh_idx = get_txh_idx(TX_4X4);
- const int cos_bit_col = av1_fwd_cos_bit_col[txw_idx][txh_idx];
- const int cos_bit_row = av1_fwd_cos_bit_row[txw_idx][txh_idx];
- const int width = 4;
- const int height = 4;
- const transform_1d_lbd_neon col_txfm = col_txfm4x4_arr[tx_type];
- const transform_1d_lbd_neon row_txfm = row_txfm4x4_arr[tx_type];
+ int16x4_t buf0[4], buf1[4];
+ const col_transform_1d_lbd_4_neon col_txfm = col_txfm4x4_arr[tx_type];
+ const row_transform_1d_lbd_4_neon row_txfm = row_txfm4x4_arr[tx_type];
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
- const int16x8_t v_shift0 = vdupq_n_s16(shift[0]);
- const int16x8_t v_shift1 = vdupq_n_s16(shift[1]);
- const int16x8_t v_shift2 = vdupq_n_s16(shift[2]);
- if (ud_flip) {
- load_buffer_16bit_to_16bit_w4_flip(input, stride, buf0, height);
- } else {
- load_buffer_16bit_to_16bit_w4(input, stride, buf0, height);
- }
- round_shift_16bit_vector(buf0, height, &v_shift0);
- col_txfm(buf0, buf0, cos_bit_col, NULL);
- round_shift_16bit_vector(buf0, height, &v_shift1);
- transpose_16bit_4x4(buf0, buf1);
+ ud_adjust_input_and_stride(ud_flip, &input, &stride, 4);
+ col_txfm(input, buf0, stride, 13);
+ transpose_arrays_s16_4x4(buf0, buf1);
if (lr_flip) {
- buf = buf0;
- flip_buf_neon(buf1, buf, width);
+ flip_buf_4_neon(buf1, buf0, 4);
+ row_txfm(buf0, output, 4, 13);
} else {
- buf = buf1;
+ row_txfm(buf1, output, 4, 13);
}
- row_txfm(buf, buf, cos_bit_row, NULL);
- round_shift_16bit_vector(buf0, height, &v_shift2);
-
- store_buffer_16bit_to_32bit_w4(buf, output, height, width);
}
-void av1_lowbd_fwd_txfm2d_4x8_neon(const int16_t *input, int32_t *output,
- int stride, TX_TYPE tx_type, int bd) {
- (void)stride;
+static void lowbd_fwd_txfm2d_4x8_neon(const int16_t *input, int32_t *output,
+ int stride, TX_TYPE tx_type, int bd) {
(void)bd;
- int16x8_t buf0[8], buf1[8], *buf;
- const int8_t *shift = av1_fwd_txfm_shift_ls[TX_4X8];
- const int txw_idx = get_txw_idx(TX_4X8);
- const int txh_idx = get_txh_idx(TX_4X8);
- const int cos_bit_col = av1_fwd_cos_bit_col[txw_idx][txh_idx];
- const int cos_bit_row = av1_fwd_cos_bit_row[txw_idx][txh_idx];
- const int width = 4;
- const int height = 8;
- const transform_1d_lbd_neon col_txfm = col_txfm4x8_arr[tx_type];
- const transform_1d_lbd_neon row_txfm = row_txfm8x4_arr[tx_type];
- int ud_flip, lr_flip;
+ int16x4_t buf0[8];
+ int16x8_t buf1[8];
+ const col_transform_1d_lbd_4_neon col_txfm = col_txfm4x8_arr[tx_type];
+ const row_transform_1d_lbd_8_neon row_txfm = row_rect_txfm8x4_arr[tx_type];
+ int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
- const int16x8_t v_shift0 = vdupq_n_s16(shift[0]);
- const int16x8_t v_shift1 = vdupq_n_s16(shift[1]);
- const int16x8_t v_shift2 = vdupq_n_s16(shift[2]);
- if (ud_flip) {
- load_buffer_16bit_to_16bit_w4_flip(input, stride, buf0, height);
- } else {
- load_buffer_16bit_to_16bit_w4(input, stride, buf0, height);
- }
- round_shift_16bit_vector(buf0, height, &v_shift0);
- col_txfm(buf0, buf0, cos_bit_col, NULL);
- round_shift_16bit_vector(buf0, height, &v_shift1);
- transpose_16bit_4x8(buf0, buf1);
+ ud_adjust_input_and_stride(ud_flip, &input, &stride, 8);
+ col_txfm(input, buf0, stride, 13);
+ shift_right_1_round_s16_x4(buf0, buf0, 8);
+ transpose_arrays_s16_4x8(buf0, buf1);
if (lr_flip) {
- buf = buf0;
- flip_buf_neon(buf1, buf, width);
+ int16x8_t buf2[8];
+ flip_buf_8_neon(buf1, buf2, 4);
+ row_txfm(buf2, output, 8, 13);
} else {
- buf = buf1;
+ row_txfm(buf1, output, 8, 13);
}
- row_txfm(buf, buf, cos_bit_row, NULL);
- round_shift_16bit_vector(buf0, height, &v_shift2);
- store_rect_buffer_16bit_to_32bit_w8(buf, output, height, width);
}
-void av1_lowbd_fwd_txfm2d_4x16_neon(const int16_t *input, int32_t *output,
- int stride, TX_TYPE tx_type, int bd) {
+static void lowbd_fwd_txfm2d_4x16_neon(const int16_t *input, int32_t *output,
+ int stride, TX_TYPE tx_type, int bd) {
(void)bd;
- int16x8_t buf0[16], buf1[16];
- const int8_t *shift = av1_fwd_txfm_shift_ls[TX_4X16];
- const int txw_idx = get_txw_idx(TX_4X16);
- const int txh_idx = get_txh_idx(TX_4X16);
- const int cos_bit_col = av1_fwd_cos_bit_col[txw_idx][txh_idx];
- const int cos_bit_row = av1_fwd_cos_bit_row[txw_idx][txh_idx];
- const int width = 4;
- const int height = 16;
- const transform_1d_lbd_neon col_txfm = col_txfm8x16_arr[tx_type];
- const transform_1d_lbd_neon row_txfm = row_txfm8x4_arr[tx_type];
+ int16x4_t buf0[16];
+ int16x8_t buf1[16];
+ const col_transform_1d_lbd_4_neon col_txfm = col_txfm4x16_arr[tx_type];
+ const row_transform_1d_lbd_8_neon row_txfm = row_txfm8x4_arr[tx_type];
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
- const int16x8_t v_shift0 = vdupq_n_s16(shift[0]);
- const int16x8_t v_shift1 = vdupq_n_s16(shift[1]);
- const int16x8_t v_shift2 = vdupq_n_s16(shift[2]);
- if (ud_flip) {
- load_buffer_16bit_to_16bit_w4_flip(input, stride, buf0, height);
- } else {
- load_buffer_16bit_to_16bit_w4(input, stride, buf0, height);
- }
- round_shift_16bit_vector(buf0, height, &v_shift0);
- col_txfm(buf0, buf0, cos_bit_col, NULL);
- round_shift_16bit_vector(buf0, height, &v_shift1);
- transpose_16bit_4x8(buf0, buf1);
- transpose_16bit_4x8(buf0 + 8, buf1 + 8);
+ ud_adjust_input_and_stride(ud_flip, &input, &stride, 16);
+ col_txfm(input, buf0, stride, 13);
+ shift_right_1_round_s16_x4(buf0, buf0, 16);
+ transpose_arrays_s16_4x8(buf0, buf1);
+ transpose_arrays_s16_4x8(buf0 + 8, buf1 + 8);
for (int i = 0; i < 2; i++) {
- int16x8_t *buf;
if (lr_flip) {
- buf = buf0;
- flip_buf_neon(buf1 + 8 * i, buf, width);
+ int16x8_t buf2[16];
+ flip_buf_8_neon(buf1 + 8 * i, buf2, 4);
+ row_txfm(buf2, output + 8 * i, 16, 12);
} else {
- buf = buf1 + 8 * i;
+ int16x8_t *buf = buf1 + 8 * i;
+ row_txfm(buf, output + 8 * i, 16, 12);
}
- row_txfm(buf, buf, cos_bit_row, NULL);
- round_shift_16bit_vector(buf0, height, &v_shift2);
- store_buffer_16bit_to_32bit_w8(buf, output + 8 * i, height, width);
}
}
-void av1_lowbd_fwd_txfm2d_8x4_neon(const int16_t *input, int32_t *output,
- int stride, TX_TYPE tx_type, int bd) {
+static void lowbd_fwd_txfm2d_8x4_neon(const int16_t *input, int32_t *output,
+ int stride, TX_TYPE tx_type, int bd) {
(void)bd;
- int16x8_t buf0[8], buf1[8], *buf;
- const int8_t *shift = av1_fwd_txfm_shift_ls[TX_8X4];
- const int txw_idx = get_txw_idx(TX_8X4);
- const int txh_idx = get_txh_idx(TX_8X4);
- const int cos_bit_col = av1_fwd_cos_bit_col[txw_idx][txh_idx];
- const int cos_bit_row = av1_fwd_cos_bit_row[txw_idx][txh_idx];
- const int width = 8;
- const int height = 4;
- const transform_1d_lbd_neon col_txfm = col_txfm8x4_arr[tx_type];
- const transform_1d_lbd_neon row_txfm = row_txfm4x8_arr[tx_type];
+ int16x8_t buf0[8];
+ int16x4_t buf1[8];
+ const col_transform_1d_lbd_8_neon col_txfm = col_txfm8x4_arr[tx_type];
+ const row_transform_1d_lbd_4_neon row_txfm = row_rect_txfm4x8_arr[tx_type];
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
- const int16x8_t v_shift0 = vdupq_n_s16(shift[0]);
- const int16x8_t v_shift1 = vdupq_n_s16(shift[1]);
- const int16x8_t v_shift2 = vdupq_n_s16(shift[2]);
- if (ud_flip)
- load_buffer_16bit_to_16bit_flip(input, stride, buf0, height);
- else
- load_buffer_16bit_to_16bit(input, stride, buf0, height);
- round_shift_16bit_vector(buf0, height, &v_shift0);
- col_txfm(buf0, buf0, cos_bit_col, NULL);
- round_shift_16bit_vector(buf0, height, &v_shift1);
- transpose_16bit_8x8(buf0, buf1);
+ ud_adjust_input_and_stride(ud_flip, &input, &stride, 4);
+ col_txfm(input, buf0, stride, 13);
+ shift_right_1_round_s16_x8(buf0, buf0, 4);
+ transpose_arrays_s16_8x4(buf0, buf1);
if (lr_flip) {
- buf = buf0;
- flip_buf_neon(buf1, buf, width);
+ int16x4_t buf2[8];
+ flip_buf_4_neon(buf1, buf2, 8);
+ row_txfm(buf2, output, 4, 13);
} else {
- buf = buf1;
+ row_txfm(buf1, output, 4, 13);
}
- row_txfm(buf, buf, cos_bit_row, NULL);
- round_shift_16bit_vector(buf0, height, &v_shift2);
- store_rect_buffer_16bit_to_32bit_w4(buf, output, height, width);
}
-void av1_lowbd_fwd_txfm2d_8x8_neon(const int16_t *input, int32_t *output,
- int stride, TX_TYPE tx_type, int bd) {
+static void lowbd_fwd_txfm2d_8x8_neon(const int16_t *input, int32_t *output,
+ int stride, TX_TYPE tx_type, int bd) {
(void)bd;
- int16x8_t buf0[8], buf1[8], *buf;
- const int8_t *shift = av1_fwd_txfm_shift_ls[TX_8X8];
- const int txw_idx = get_txw_idx(TX_8X8);
- const int txh_idx = get_txh_idx(TX_8X8);
- const int cos_bit_col = av1_fwd_cos_bit_col[txw_idx][txh_idx];
- const int cos_bit_row = av1_fwd_cos_bit_row[txw_idx][txh_idx];
- const int width = 8;
- const int height = 8;
- const transform_1d_lbd_neon col_txfm = col_txfm8x8_arr[tx_type];
- const transform_1d_lbd_neon row_txfm = row_txfm8x8_arr[tx_type];
+ int16x8_t buf0[8], buf1[8];
+ const col_transform_1d_lbd_8_neon col_txfm = col_txfm8x8_arr[tx_type];
+ const row_transform_1d_lbd_8_neon row_txfm = row_txfm8x8_arr[tx_type];
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
- const int16x8_t v_shift0 = vdupq_n_s16(shift[0]);
- const int16x8_t v_shift1 = vdupq_n_s16(shift[1]);
- const int16x8_t v_shift2 = vdupq_n_s16(shift[2]);
- if (ud_flip)
- load_buffer_16bit_to_16bit_flip(input, stride, buf0, height);
- else
- load_buffer_16bit_to_16bit(input, stride, buf0, height);
- round_shift_16bit_vector(buf0, height, &v_shift0);
- col_txfm(buf0, buf0, cos_bit_col, NULL);
- round_shift_16bit_vector(buf0, height, &v_shift1);
- transpose_16bit_8x8(buf0, buf1);
+ ud_adjust_input_and_stride(ud_flip, &input, &stride, 8);
+ col_txfm(input, buf0, stride, 13);
+ shift_right_1_round_s16_x8(buf0, buf0, 8);
+ transpose_arrays_s16_8x8(buf0, buf1);
if (lr_flip) {
- buf = buf0;
- flip_buf_neon(buf1, buf, width);
+ flip_buf_8_neon(buf1, buf0, 8);
+ row_txfm(buf0, output, 8, 13);
} else {
- buf = buf1;
+ row_txfm(buf1, output, 8, 13);
}
- row_txfm(buf, buf, cos_bit_row, NULL);
- round_shift_16bit_vector(buf0, height, &v_shift2);
- store_buffer_16bit_to_32bit_w8(buf, output, height, width);
}
-void av1_lowbd_fwd_txfm2d_8x16_neon(const int16_t *input, int32_t *output,
- int stride, TX_TYPE tx_type, int bd) {
+static void lowbd_fwd_txfm2d_8x16_neon(const int16_t *input, int32_t *output,
+ int stride, TX_TYPE tx_type, int bd) {
(void)bd;
int16x8_t buf0[16], buf1[16];
- const int8_t *shift = av1_fwd_txfm_shift_ls[TX_8X16];
- const int txw_idx = get_txw_idx(TX_8X16);
- const int txh_idx = get_txh_idx(TX_8X16);
- const int cos_bit_col = av1_fwd_cos_bit_col[txw_idx][txh_idx];
- const int cos_bit_row = av1_fwd_cos_bit_row[txw_idx][txh_idx];
- const int width = 8;
- const int height = 16;
- const transform_1d_lbd_neon col_txfm = col_txfm8x16_arr[tx_type];
- const transform_1d_lbd_neon row_txfm = row_txfm8x8_arr[tx_type];
+ const col_transform_1d_lbd_8_neon col_txfm = col_txfm8x16_arr[tx_type];
+ const row_transform_1d_lbd_8_neon row_txfm = row_rect_txfm8x8_arr[tx_type];
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
- const int16x8_t v_shift0 = vdupq_n_s16(shift[0]);
- const int16x8_t v_shift1 = vdupq_n_s16(shift[1]);
- const int16x8_t v_shift2 = vdupq_n_s16(shift[2]);
- if (ud_flip) {
- load_buffer_16bit_to_16bit_flip(input, stride, buf0, height);
- } else {
- load_buffer_16bit_to_16bit(input, stride, buf0, height);
- }
- round_shift_16bit_vector(buf0, height, &v_shift0);
- col_txfm(buf0, buf0, cos_bit_col, NULL);
- round_shift_16bit_vector(buf0, height, &v_shift1);
- transpose_16bit_8x8(buf0, buf1);
- transpose_16bit_8x8(buf0 + 8, buf1 + 8);
+ ud_adjust_input_and_stride(ud_flip, &input, &stride, 16);
+ col_txfm(input, buf0, stride, 13);
+ shift_right_2_round_s16_x8(buf0, buf0, 16);
+ transpose_arrays_s16_8x8(buf0, buf1);
+ transpose_arrays_s16_8x8(buf0 + 8, buf1 + 8);
for (int i = 0; i < 2; i++) {
- int16x8_t *buf;
if (lr_flip) {
- buf = buf0;
- flip_buf_neon(buf1 + width * i, buf, width);
+ flip_buf_8_neon(buf1 + 8 * i, buf0, 8);
+ row_txfm(buf0, output + 8 * i, 16, 13);
} else {
- buf = buf1 + width * i;
+ int16x8_t *buf = buf1 + 8 * i;
+ row_txfm(buf, output + 8 * i, 16, 13);
}
- row_txfm(buf, buf, cos_bit_row, NULL);
- round_shift_16bit_vector(buf0, height, &v_shift2);
- store_rect_buffer_16bit_to_32bit_w8(buf, output + 8 * i, height, 8);
}
}
-void av1_lowbd_fwd_txfm2d_8x32_neon(const int16_t *input, int32_t *output,
- int stride, TX_TYPE tx_type, int bd) {
+static void lowbd_fwd_txfm2d_8x32_neon(const int16_t *input, int32_t *output,
+ int stride, TX_TYPE tx_type, int bd) {
(void)bd;
int16x8_t buf0[32], buf1[32];
- const int8_t *shift = av1_fwd_txfm_shift_ls[TX_8X32];
- const int txw_idx = get_txw_idx(TX_8X32);
- const int txh_idx = get_txh_idx(TX_8X32);
- const int cos_bit_col = av1_fwd_cos_bit_col[txw_idx][txh_idx];
- const int cos_bit_row = av1_fwd_cos_bit_row[txw_idx][txh_idx];
- const int width = 8;
- const int height = 32;
- const transform_1d_lbd_neon col_txfm = col_txfm8x32_arr[tx_type];
- const transform_1d_lbd_neon row_txfm = row_txfm8x8_arr[tx_type];
+ const col_transform_1d_lbd_8_neon col_txfm = col_txfm8x32_arr[tx_type];
+ const row_transform_1d_lbd_8_neon row_txfm = row_txfm8x8_arr[tx_type];
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
- const int16x8_t v_shift0 = vdupq_n_s16(shift[0]);
- const int16x8_t v_shift1 = vdupq_n_s16(shift[1]);
- const int16x8_t v_shift2 = vdupq_n_s16(shift[2]);
- if (ud_flip) {
- load_buffer_16bit_to_16bit_flip(input, stride, buf0, height);
- } else {
- load_buffer_16bit_to_16bit(input, stride, buf0, height);
- }
- round_shift_16bit_vector(buf0, height, &v_shift0);
- col_txfm(buf0, buf0, cos_bit_col, NULL);
- round_shift_16bit_vector(buf0, height, &v_shift1);
- transpose_16bit_8x8(buf0, buf1);
- transpose_16bit_8x8(buf0 + 8, buf1 + 8);
- transpose_16bit_8x8(buf0 + 16, buf1 + 16);
- transpose_16bit_8x8(buf0 + 24, buf1 + 24);
+ ud_adjust_input_and_stride(ud_flip, &input, &stride, 32);
+ col_txfm(input, buf0, stride, 12);
+ shift_right_2_round_s16_x8(buf0, buf0, 32);
+ transpose_arrays_s16_8x8(buf0, buf1);
+ transpose_arrays_s16_8x8(buf0 + 8, buf1 + 8);
+ transpose_arrays_s16_8x8(buf0 + 16, buf1 + 16);
+ transpose_arrays_s16_8x8(buf0 + 24, buf1 + 24);
for (int i = 0; i < 4; i++) {
- int16x8_t *buf;
if (lr_flip) {
- buf = buf0;
- flip_buf_neon(buf1 + width * i, buf, width);
+ flip_buf_8_neon(buf1 + 8 * i, buf0, 8);
+ row_txfm(buf0, output + 8 * i, 32, 12);
} else {
- buf = buf1 + width * i;
+ int16x8_t *buf = buf1 + 8 * i;
+ row_txfm(buf, output + 8 * i, 32, 12);
}
- row_txfm(buf, buf, cos_bit_row, NULL);
- round_shift_16bit_vector(buf0, height, &v_shift2);
- store_buffer_16bit_to_32bit_w8(buf, output + 8 * i, height, width);
}
}
-void av1_lowbd_fwd_txfm2d_16x4_neon(const int16_t *input, int32_t *output,
- int stride, TX_TYPE tx_type, int bd) {
+static void lowbd_fwd_txfm2d_16x4_neon(const int16_t *input, int32_t *output,
+ int stride, TX_TYPE tx_type, int bd) {
(void)bd;
- int16x8_t buf0[16], buf1[16];
- const int8_t *shift = av1_fwd_txfm_shift_ls[TX_16X4];
- const int txw_idx = get_txw_idx(TX_16X4);
- const int txh_idx = get_txh_idx(TX_16X4);
- const int cos_bit_col = av1_fwd_cos_bit_col[txw_idx][txh_idx];
- const int cos_bit_row = av1_fwd_cos_bit_row[txw_idx][txh_idx];
- const int width = 16;
- const int height = 4;
- const transform_1d_lbd_neon col_txfm = col_txfm8x4_arr[tx_type];
- const transform_1d_lbd_neon row_txfm = row_txfm8x16_arr[tx_type];
- int16x8_t *buf;
+ int16x8_t buf0[16];
+ int16x4_t buf1[16];
+ int16x4_t buf2[16];
+ const col_transform_1d_lbd_8_neon col_txfm = col_txfm8x4_arr[tx_type];
+ const row_transform_1d_lbd_4_neon row_txfm = row_txfm4x16_arr[tx_type];
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
- const int16x8_t v_shift0 = vdupq_n_s16(shift[0]);
- const int16x8_t v_shift1 = vdupq_n_s16(shift[1]);
- const int16x8_t v_shift2 = vdupq_n_s16(shift[2]);
+ ud_adjust_input_and_stride(ud_flip, &input, &stride, 4);
for (int i = 0; i < 2; i++) {
- if (ud_flip) {
- load_buffer_16bit_to_16bit_flip(input + 8 * i, stride, buf0, height);
- } else {
- load_buffer_16bit_to_16bit(input + 8 * i, stride, buf0, height);
- }
- round_shift_16bit_vector(buf0, height, &v_shift0);
- col_txfm(buf0, buf0, cos_bit_col, NULL);
- round_shift_16bit_vector(buf0, height, &v_shift1);
- transpose_16bit_8x4(buf0, buf1 + 8 * i);
+ col_txfm(input + 8 * i, buf0, stride, 13);
+ shift_right_1_round_s16_x8(buf0, buf0, 4);
+ transpose_arrays_s16_8x4(buf0, buf1 + 8 * i);
}
if (lr_flip) {
- buf = buf0;
- flip_buf_neon(buf1, buf, width);
+ flip_buf_4_neon(buf1, buf2, 16);
+ row_txfm(buf2, output, 4, 13);
} else {
- buf = buf1;
+ row_txfm(buf1, output, 4, 13);
}
- row_txfm(buf, buf, cos_bit_row, NULL);
- round_shift_16bit_vector(buf0, height, &v_shift2);
- store_buffer_16bit_to_32bit_w4(buf, output, height, width);
}
-void av1_lowbd_fwd_txfm2d_16x8_neon(const int16_t *input, int32_t *output,
- int stride, TX_TYPE tx_type, int bd) {
+static void lowbd_fwd_txfm2d_16x8_neon(const int16_t *input, int32_t *output,
+ int stride, TX_TYPE tx_type, int bd) {
(void)bd;
int16x8_t buf0[16], buf1[16];
- const int8_t *shift = av1_fwd_txfm_shift_ls[TX_16X8];
- const int txw_idx = get_txw_idx(TX_16X8);
- const int txh_idx = get_txh_idx(TX_16X8);
- const int cos_bit_col = av1_fwd_cos_bit_col[txw_idx][txh_idx];
- const int cos_bit_row = av1_fwd_cos_bit_row[txw_idx][txh_idx];
- const int width = 16;
- const int height = 8;
- const transform_1d_lbd_neon col_txfm = col_txfm8x8_arr[tx_type];
- const transform_1d_lbd_neon row_txfm = row_txfm8x16_arr[tx_type];
- int16x8_t *buf;
+ const col_transform_1d_lbd_8_neon col_txfm = col_txfm8x8_arr[tx_type];
+ const row_transform_1d_lbd_8_neon row_txfm = row_rect_txfm8x16_arr[tx_type];
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
- const int16x8_t v_shift0 = vdupq_n_s16(shift[0]);
- const int16x8_t v_shift1 = vdupq_n_s16(shift[1]);
- const int16x8_t v_shift2 = vdupq_n_s16(shift[2]);
+ ud_adjust_input_and_stride(ud_flip, &input, &stride, 8);
for (int i = 0; i < 2; i++) {
- if (ud_flip) {
- load_buffer_16bit_to_16bit_flip(input + 8 * i, stride, buf0, height);
- } else {
- load_buffer_16bit_to_16bit(input + 8 * i, stride, buf0, height);
- }
- round_shift_16bit_vector(buf0, height, &v_shift0);
- col_txfm(buf0, buf0, cos_bit_col, NULL);
- round_shift_16bit_vector(buf0, height, &v_shift1);
- transpose_16bit_8x8(buf0, buf1 + 8 * i);
+ col_txfm(input + 8 * i, buf0, stride, 13);
+ shift_right_2_round_s16_x8(buf0, buf0, 8);
+ transpose_arrays_s16_8x8(buf0, buf1 + 8 * i);
}
if (lr_flip) {
- buf = buf0;
- flip_buf_neon(buf1, buf, width);
+ flip_buf_8_neon(buf1, buf0, 16);
+ row_txfm(buf0, output, 8, 13);
} else {
- buf = buf1;
+ row_txfm(buf1, output, 8, 13);
}
- row_txfm(buf, buf, cos_bit_row, NULL);
- round_shift_16bit_vector(buf0, height, &v_shift2);
- store_rect_buffer_16bit_to_32bit_w8(buf, output, height, width);
}
-void av1_lowbd_fwd_txfm2d_16x16_neon(const int16_t *input, int32_t *output,
- int stride, TX_TYPE tx_type, int bd) {
+static void lowbd_fwd_txfm2d_16x16_neon(const int16_t *input, int32_t *output,
+ int stride, TX_TYPE tx_type, int bd) {
(void)bd;
int16x8_t buf0[16], buf1[32];
- const int8_t *shift = av1_fwd_txfm_shift_ls[TX_16X16];
- const int txw_idx = get_txw_idx(TX_16X16);
- const int txh_idx = get_txh_idx(TX_16X16);
- const int cos_bit_col = av1_fwd_cos_bit_col[txw_idx][txh_idx];
- const int cos_bit_row = av1_fwd_cos_bit_row[txw_idx][txh_idx];
- const int width = 16;
- const int height = 16;
- const transform_1d_lbd_neon col_txfm = col_txfm8x16_arr[tx_type];
- const transform_1d_lbd_neon row_txfm = row_txfm8x16_arr[tx_type];
+ const col_transform_1d_lbd_8_neon col_txfm = col_txfm8x16_arr[tx_type];
+ const row_transform_1d_lbd_8_neon row_txfm = row_txfm8x16_arr[tx_type];
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
- const int16x8_t v_shift0 = vdupq_n_s16(shift[0]);
- const int16x8_t v_shift1 = vdupq_n_s16(shift[1]);
- const int16x8_t v_shift2 = vdupq_n_s16(shift[2]);
-
+ ud_adjust_input_and_stride(ud_flip, &input, &stride, 16);
for (int i = 0; i < 2; i++) {
- if (ud_flip) {
- load_buffer_16bit_to_16bit_flip(input + 8 * i, stride, buf0, height);
- } else {
- load_buffer_16bit_to_16bit(input + 8 * i, stride, buf0, height);
- }
- round_shift_16bit_vector(buf0, height, &v_shift0);
- col_txfm(buf0, buf0, cos_bit_col, NULL);
- round_shift_16bit_vector(buf0, height, &v_shift1);
- transpose_16bit_8x8(buf0, buf1 + 0 * width + 8 * i);
- transpose_16bit_8x8(buf0 + 8, buf1 + 1 * width + 8 * i);
+ col_txfm(input + 8 * i, buf0, stride, 13);
+ shift_right_2_round_s16_x8(buf0, buf0, 16);
+ transpose_arrays_s16_8x8(buf0, buf1 + 0 * 16 + 8 * i);
+ transpose_arrays_s16_8x8(buf0 + 8, buf1 + 1 * 16 + 8 * i);
}
for (int i = 0; i < 2; i++) {
- int16x8_t *buf;
if (lr_flip) {
- buf = buf0;
- flip_buf_neon(buf1 + width * i, buf, width);
+ flip_buf_8_neon(buf1 + 16 * i, buf0, 16);
+ row_txfm(buf0, output + 8 * i, 16, 12);
} else {
- buf = buf1 + width * i;
+ int16x8_t *buf = buf1 + 16 * i;
+ row_txfm(buf, output + 8 * i, 16, 12);
}
- row_txfm(buf, buf, cos_bit_row, NULL);
- round_shift_16bit_vector(buf0, height, &v_shift2);
- store_buffer_16bit_to_32bit_w8(buf, output + 8 * i, height, width);
}
}
-void av1_lowbd_fwd_txfm2d_16x32_neon(const int16_t *input, int32_t *output,
- int stride, TX_TYPE tx_type, int bd) {
+static void lowbd_fwd_txfm2d_16x32_neon(const int16_t *input, int32_t *output,
+ int stride, TX_TYPE tx_type, int bd) {
(void)bd;
int16x8_t buf0[32], buf1[64];
- const int8_t *shift = av1_fwd_txfm_shift_ls[TX_16X32];
- const int txw_idx = get_txw_idx(TX_16X32);
- const int txh_idx = get_txh_idx(TX_16X32);
- const int cos_bit_col = av1_fwd_cos_bit_col[txw_idx][txh_idx];
- const int cos_bit_row = av1_fwd_cos_bit_row[txw_idx][txh_idx];
- const int width = 16;
- const int height = 32;
- const transform_1d_lbd_neon col_txfm = col_txfm8x32_arr[tx_type];
- const transform_1d_lbd_neon row_txfm = row_txfm8x16_arr[tx_type];
+ const col_transform_1d_lbd_8_neon col_txfm = col_txfm8x32_arr[tx_type];
+ const row_transform_1d_lbd_8_neon row_txfm = row_rect_txfm8x16_arr[tx_type];
- if (col_txfm != NULL && row_txfm != NULL) {
- int ud_flip, lr_flip;
- get_flip_cfg(tx_type, &ud_flip, &lr_flip);
- const int16x8_t v_shift0 = vdupq_n_s16(shift[0]);
- const int16x8_t v_shift1 = vdupq_n_s16(shift[1]);
- const int16x8_t v_shift2 = vdupq_n_s16(shift[2]);
-
- for (int i = 0; i < 2; i++) {
- if (ud_flip) {
- load_buffer_16bit_to_16bit_flip(input + 8 * i, stride, buf0, height);
- } else {
- load_buffer_16bit_to_16bit(input + 8 * i, stride, buf0, height);
- }
- round_shift_16bit_vector(buf0, height, &v_shift0);
- col_txfm(buf0, buf0, cos_bit_col, NULL);
- round_shift_16bit_vector(buf0, height, &v_shift1);
- transpose_16bit_8x8(buf0 + 0 * 8, buf1 + 0 * width + 8 * i);
- transpose_16bit_8x8(buf0 + 1 * 8, buf1 + 1 * width + 8 * i);
- transpose_16bit_8x8(buf0 + 2 * 8, buf1 + 2 * width + 8 * i);
- transpose_16bit_8x8(buf0 + 3 * 8, buf1 + 3 * width + 8 * i);
- }
-
- for (int i = 0; i < 4; i++) {
- int16x8_t *buf;
- if (lr_flip) {
- buf = buf0;
- flip_buf_neon(buf1 + width * i, buf, width);
- } else {
- buf = buf1 + width * i;
- }
- row_txfm(buf, buf, cos_bit_row, NULL);
- round_shift_16bit_vector(buf0, height, &v_shift2);
- store_rect_buffer_16bit_to_32bit_w8(buf, output + 8 * i, height, width);
- }
- } else {
+ if (col_txfm == NULL || row_txfm == NULL) {
av1_fwd_txfm2d_16x32_c(input, output, stride, tx_type, bd);
+ return;
+ }
+
+ int ud_flip, lr_flip;
+ get_flip_cfg(tx_type, &ud_flip, &lr_flip);
+ ud_adjust_input_and_stride(ud_flip, &input, &stride, 32);
+ for (int i = 0; i < 2; i++) {
+ col_txfm(input + 8 * i, buf0, stride, 12);
+ shift_right_4_round_s16_x8(buf0, buf0, 32);
+ transpose_arrays_s16_8x8(buf0 + 0 * 8, buf1 + 0 * 16 + 8 * i);
+ transpose_arrays_s16_8x8(buf0 + 1 * 8, buf1 + 1 * 16 + 8 * i);
+ transpose_arrays_s16_8x8(buf0 + 2 * 8, buf1 + 2 * 16 + 8 * i);
+ transpose_arrays_s16_8x8(buf0 + 3 * 8, buf1 + 3 * 16 + 8 * i);
+ }
+
+ for (int i = 0; i < 4; i++) {
+ if (lr_flip) {
+ flip_buf_8_neon(buf1 + 16 * i, buf0, 16);
+ row_txfm(buf0, output + 8 * i, 32, 13);
+ } else {
+ int16x8_t *buf = buf1 + 16 * i;
+ row_txfm(buf, output + 8 * i, 32, 13);
+ }
}
}
-void av1_lowbd_fwd_txfm2d_32x8_neon(const int16_t *input, int32_t *output,
- int stride, TX_TYPE tx_type, int bd) {
+static void lowbd_fwd_txfm2d_32x8_neon(const int16_t *input, int32_t *output,
+ int stride, TX_TYPE tx_type, int bd) {
(void)bd;
int16x8_t buf0[32], buf1[32];
- const int8_t *shift = av1_fwd_txfm_shift_ls[TX_32X8];
- const int txw_idx = get_txw_idx(TX_32X8);
- const int txh_idx = get_txh_idx(TX_32X8);
- const int cos_bit_col = av1_fwd_cos_bit_col[txw_idx][txh_idx];
- const int cos_bit_row = av1_fwd_cos_bit_row[txw_idx][txh_idx];
- const int width = 32;
- const int height = 8;
- const transform_1d_lbd_neon col_txfm = col_txfm8x8_arr[tx_type];
- const transform_1d_lbd_neon row_txfm = row_txfm8x32_arr[tx_type];
+ const col_transform_1d_lbd_8_neon col_txfm = col_txfm8x8_arr[tx_type];
+ const row_transform_1d_lbd_8_neon row_txfm = row_txfm8x32_arr[tx_type];
- if (col_txfm != NULL && row_txfm != NULL) {
- int ud_flip, lr_flip;
- get_flip_cfg(tx_type, &ud_flip, &lr_flip);
- const int16x8_t v_shift0 = vdupq_n_s16(shift[0]);
- const int16x8_t v_shift1 = vdupq_n_s16(shift[1]);
- const int16x8_t v_shift2 = vdupq_n_s16(shift[2]);
-
- for (int i = 0; i < 4; i++) {
- if (ud_flip) {
- load_buffer_16bit_to_16bit_flip(input + 8 * i, stride, buf0, height);
- } else {
- load_buffer_16bit_to_16bit(input + 8 * i, stride, buf0, height);
- }
- round_shift_16bit_vector(buf0, height, &v_shift0);
- col_txfm(buf0, buf0, cos_bit_col, NULL);
- round_shift_16bit_vector(buf0, height, &v_shift1);
- transpose_16bit_8x8(buf0, buf1 + 0 * width + 8 * i);
- }
-
- for (int i = 0; i < 1; i++) {
- int16x8_t *buf;
- if (lr_flip) {
- buf = buf0;
- flip_buf_neon(buf1 + width * i, buf, width);
- } else {
- buf = buf1 + width * i;
- }
- row_txfm(buf, buf, cos_bit_row, NULL);
- round_shift_16bit_vector(buf, width, &v_shift2);
- store_buffer_16bit_to_32bit_w8(buf, output + 8 * i, height, width);
- }
- } else {
+ if (col_txfm == NULL || row_txfm == NULL) {
av1_fwd_txfm2d_32x16_c(input, output, stride, tx_type, bd);
+ return;
+ }
+
+ int ud_flip, lr_flip;
+ get_flip_cfg(tx_type, &ud_flip, &lr_flip);
+ ud_adjust_input_and_stride(ud_flip, &input, &stride, 8);
+ for (int i = 0; i < 4; i++) {
+ col_txfm(input + 8 * i, buf0, stride, 13);
+ shift_right_2_round_s16_x8(buf0, buf0, 8);
+ transpose_arrays_s16_8x8(buf0, buf1 + 0 * 32 + 8 * i);
+ }
+
+ if (lr_flip) {
+ flip_buf_8_neon(buf1, buf0, 32);
+ row_txfm(buf0, output, 8, 12);
+ } else {
+ row_txfm(buf1, output, 8, 12);
}
}
-void av1_lowbd_fwd_txfm2d_32x16_neon(const int16_t *input, int32_t *output,
- int stride, TX_TYPE tx_type, int bd) {
+static void lowbd_fwd_txfm2d_32x16_neon(const int16_t *input, int32_t *output,
+ int stride, TX_TYPE tx_type, int bd) {
(void)bd;
int16x8_t buf0[32], buf1[64];
- const int8_t *shift = av1_fwd_txfm_shift_ls[TX_32X16];
- const int txw_idx = get_txw_idx(TX_32X16);
- const int txh_idx = get_txh_idx(TX_32X16);
- const int cos_bit_col = av1_fwd_cos_bit_col[txw_idx][txh_idx];
- const int cos_bit_row = av1_fwd_cos_bit_row[txw_idx][txh_idx];
- const int width = 32;
- const int height = 16;
- const transform_1d_lbd_neon col_txfm = col_txfm8x16_arr[tx_type];
- const transform_1d_lbd_neon row_txfm = row_txfm8x32_arr[tx_type];
+ const col_transform_1d_lbd_8_neon col_txfm = col_txfm8x16_arr[tx_type];
+ const row_transform_1d_lbd_8_neon row_txfm = row_rect_txfm8x32_arr[tx_type];
- if (col_txfm != NULL && row_txfm != NULL) {
- const int16x8_t v_shift0 = vdupq_n_s16(shift[0]);
- const int16x8_t v_shift1 = vdupq_n_s16(shift[1]);
- const int16x8_t v_shift2 = vdupq_n_s16(shift[2]);
- int ud_flip, lr_flip;
- get_flip_cfg(tx_type, &ud_flip, &lr_flip);
-
- for (int i = 0; i < 4; i++) {
- if (ud_flip) {
- load_buffer_16bit_to_16bit_flip(input + 8 * i, stride, buf0, height);
- } else {
- load_buffer_16bit_to_16bit(input + 8 * i, stride, buf0, height);
- }
- round_shift_16bit_vector(buf0, height, &v_shift0);
- col_txfm(buf0, buf0, cos_bit_col, NULL);
- round_shift_16bit_vector(buf0, height, &v_shift1);
- transpose_16bit_8x8(buf0, buf1 + 0 * width + 8 * i);
- transpose_16bit_8x8(buf0 + 8, buf1 + 1 * width + 8 * i);
- }
-
- for (int i = 0; i < 2; i++) {
- int16x8_t *buf;
- if (lr_flip) {
- buf = buf0;
- flip_buf_neon(buf1 + width * i, buf, width);
- } else {
- buf = buf1 + width * i;
- }
- row_txfm(buf, buf, cos_bit_row, NULL);
- round_shift_16bit_vector(buf, width, &v_shift2);
- store_rect_buffer_16bit_to_32bit_w8(buf, output + 8 * i, height, width);
- }
- } else {
+ if (col_txfm == NULL || row_txfm == NULL) {
av1_fwd_txfm2d_32x16_c(input, output, stride, tx_type, bd);
+ return;
+ }
+
+ int ud_flip, lr_flip;
+ get_flip_cfg(tx_type, &ud_flip, &lr_flip);
+ ud_adjust_input_and_stride(ud_flip, &input, &stride, 16);
+ for (int i = 0; i < 4; i++) {
+ col_txfm(input + 8 * i, buf0, stride, 13);
+ shift_right_4_round_s16_x8(buf0, buf0, 16);
+ transpose_arrays_s16_8x8(buf0, buf1 + 0 * 32 + 8 * i);
+ transpose_arrays_s16_8x8(buf0 + 8, buf1 + 1 * 32 + 8 * i);
+ }
+
+ for (int i = 0; i < 2; i++) {
+ if (lr_flip) {
+ flip_buf_8_neon(buf1 + 32 * i, buf0, 32);
+ row_txfm(buf0, output + 8 * i, 16, 13);
+ } else {
+ int16x8_t *buf = buf1 + 32 * i;
+ row_txfm(buf, output + 8 * i, 16, 13);
+ }
}
}
-void av1_lowbd_fwd_txfm2d_32x32_neon(const int16_t *input, int32_t *output,
- int stride, TX_TYPE tx_type, int bd) {
+static void lowbd_fwd_txfm2d_32x32_neon(const int16_t *input, int32_t *output,
+ int stride, TX_TYPE tx_type, int bd) {
(void)bd;
int16x8_t buf0[32], buf1[128];
- const int8_t *shift = av1_fwd_txfm_shift_ls[TX_32X32];
- const int txw_idx = get_txw_idx(TX_32X32);
- const int txh_idx = get_txh_idx(TX_32X32);
- const int cos_bit_col = av1_fwd_cos_bit_col[txw_idx][txh_idx];
- const int cos_bit_row = av1_fwd_cos_bit_row[txw_idx][txh_idx];
- const int width = 32;
- const int height = 32;
- const transform_1d_lbd_neon col_txfm = col_txfm8x32_arr[tx_type];
- const transform_1d_lbd_neon row_txfm = row_txfm8x32_arr[tx_type];
+ const col_transform_1d_lbd_8_neon col_txfm = col_txfm8x32_arr[tx_type];
+ const row_transform_1d_lbd_8_neon row_txfm = row_txfm8x32_arr[tx_type];
- if (col_txfm != NULL && row_txfm != NULL) {
- int ud_flip, lr_flip;
- get_flip_cfg(tx_type, &ud_flip, &lr_flip);
-
- for (int i = 0; i < 4; i++) {
- if (ud_flip) {
- load_buffer_16bit_to_16bit_flip(input + 8 * i, stride, buf0, height);
- } else {
- load_buffer_16bit_to_16bit(input + 8 * i, stride, buf0, height);
- }
- round_shift_16bit(buf0, height, shift[0]);
- col_txfm(buf0, buf0, cos_bit_col, NULL);
- round_shift_16bit(buf0, height, shift[1]);
- transpose_16bit_8x8(buf0 + 0 * 8, buf1 + 0 * width + 8 * i);
- transpose_16bit_8x8(buf0 + 1 * 8, buf1 + 1 * width + 8 * i);
- transpose_16bit_8x8(buf0 + 2 * 8, buf1 + 2 * width + 8 * i);
- transpose_16bit_8x8(buf0 + 3 * 8, buf1 + 3 * width + 8 * i);
- }
-
- for (int i = 0; i < 4; i++) {
- int16x8_t *buf;
- if (lr_flip) {
- buf = buf0;
- flip_buf_neon(buf1 + width * i, buf, width);
- } else {
- buf = buf1 + width * i;
- }
- row_txfm(buf, buf, cos_bit_row, NULL);
- round_shift_16bit(buf, width, shift[2]);
- store_buffer_16bit_to_32bit_w8(buf, output + 8 * i, height, width);
- }
- } else {
+ if (col_txfm == NULL || row_txfm == NULL) {
av1_fwd_txfm2d_32x32_c(input, output, stride, tx_type, bd);
+ return;
+ }
+
+ int ud_flip, lr_flip;
+ get_flip_cfg(tx_type, &ud_flip, &lr_flip);
+ ud_adjust_input_and_stride(ud_flip, &input, &stride, 32);
+ for (int i = 0; i < 4; i++) {
+ col_txfm(input + 8 * i, buf0, stride, 12);
+ shift_right_4_round_s16_x8(buf0, buf0, 32);
+ transpose_arrays_s16_8x8(buf0 + 0 * 8, buf1 + 0 * 32 + 8 * i);
+ transpose_arrays_s16_8x8(buf0 + 1 * 8, buf1 + 1 * 32 + 8 * i);
+ transpose_arrays_s16_8x8(buf0 + 2 * 8, buf1 + 2 * 32 + 8 * i);
+ transpose_arrays_s16_8x8(buf0 + 3 * 8, buf1 + 3 * 32 + 8 * i);
+ }
+
+ for (int i = 0; i < 4; i++) {
+ if (lr_flip) {
+ flip_buf_8_neon(buf1 + 32 * i, buf0, 32);
+ row_txfm(buf0, output + 8 * i, 32, 12);
+ } else {
+ int16x8_t *buf = buf1 + 32 * i;
+ row_txfm(buf, output + 8 * i, 32, 12);
+ }
}
}
-void av1_lowbd_fwd_txfm2d_64x16_neon(const int16_t *input, int32_t *output,
- int stride, TX_TYPE tx_type, int bd) {
+static void lowbd_fwd_txfm2d_64x16_neon(const int16_t *input, int32_t *output,
+ int stride, TX_TYPE tx_type, int bd) {
(void)bd;
(void)tx_type;
assert(tx_type == DCT_DCT);
- const TX_SIZE tx_size = TX_64X16;
int16x8_t buf0[64], buf1[128];
- const int8_t *shift = av1_fwd_txfm_shift_ls[tx_size];
- const int txw_idx = get_txw_idx(tx_size);
- const int txh_idx = get_txh_idx(tx_size);
- const int cos_bit_col = av1_fwd_cos_bit_col[txw_idx][txh_idx];
- const int cos_bit_row = av1_fwd_cos_bit_row[txw_idx][txh_idx];
- const int width = tx_size_wide[tx_size];
- const int height = tx_size_high[tx_size];
- const transform_1d_lbd_neon col_txfm = fdct8x16_neon;
- const transform_1d_lbd_neon row_txfm = av1_fdct8x64_neon;
- const int width_div8 = (width >> 3);
- const int height_div8 = (height >> 3);
+ const transform_1d_lbd_8_neon col_txfm = fdct8x16_neon;
+ const transform_1d_lbd_8_neon row_txfm = fdct8x64_neon;
- for (int i = 0; i < width_div8; i++) {
- load_buffer_16bit_to_16bit(input + 8 * i, stride, buf0, height);
- round_shift_16bit(buf0, height, shift[0]);
- col_txfm(buf0, buf0, cos_bit_col, NULL);
- round_shift_16bit(buf0, height, shift[1]);
- for (int j = 0; j < height_div8; ++j) {
- transpose_16bit_8x8(buf0 + j * 8, buf1 + j * width + 8 * i);
+ for (int i = 0; i < 8; i++) {
+ load_buffer_s16_x8(input + 8 * i, stride, buf0, 16);
+ shift_left_2_s16_x8(buf0, buf0, 16);
+ col_txfm(buf0, buf0, 13);
+ shift_right_4_round_s16_x8(buf0, buf0, 16);
+ for (int j = 0; j < 2; ++j) {
+ transpose_arrays_s16_8x8(buf0 + j * 8, buf1 + j * 64 + 8 * i);
}
}
- for (int i = 0; i < height_div8; i++) {
- int16x8_t *buf = buf1 + width * i;
- row_txfm(buf, buf, cos_bit_row, NULL);
- round_shift_16bit(buf, width, shift[2]);
- store_buffer_16bit_to_32bit_w8(buf, output + 8 * i, 16, 32);
+ for (int i = 0; i < 2; i++) {
+ int16x8_t *buf = buf1 + 64 * i;
+ row_txfm(buf, buf, 12);
+ store_buffer_s16_x8(buf, output + 8 * i, 16, 32);
}
// Zero out the bottom 16x32 area.
memset(output + 16 * 32, 0, 16 * 32 * sizeof(*output));
}
-void av1_lowbd_fwd_txfm2d_16x64_neon(const int16_t *input, int32_t *output,
- int stride, TX_TYPE tx_type, int bd) {
+static void lowbd_fwd_txfm2d_16x64_neon(const int16_t *input, int32_t *output,
+ int stride, TX_TYPE tx_type, int bd) {
(void)bd;
(void)tx_type;
assert(tx_type == DCT_DCT);
- const TX_SIZE tx_size = TX_16X64;
int16x8_t buf0[64], buf1[128];
- const int8_t *shift = av1_fwd_txfm_shift_ls[tx_size];
- const int txw_idx = get_txw_idx(tx_size);
- const int txh_idx = get_txh_idx(tx_size);
- const int cos_bit_col = av1_fwd_cos_bit_col[txw_idx][txh_idx];
- const int cos_bit_row = av1_fwd_cos_bit_row[txw_idx][txh_idx];
- const int width = tx_size_wide[tx_size];
- const int height = tx_size_high[tx_size];
- const transform_1d_lbd_neon col_txfm = av1_fdct8x64_neon;
- const transform_1d_lbd_neon row_txfm = fdct8x16_neon;
- const int width_div8 = (width >> 3);
- const int height_div8 = (height >> 3);
+ const transform_1d_lbd_8_neon col_txfm = fdct8x64_neon;
+ const transform_1d_lbd_8_neon row_txfm = fdct8x16_neon;
- for (int i = 0; i < width_div8; i++) {
- load_buffer_16bit_to_16bit(input + 8 * i, stride, buf0, height);
- round_shift_16bit(buf0, height, shift[0]);
- col_txfm(buf0, buf0, cos_bit_col, NULL);
- round_shift_16bit(buf0, height, shift[1]);
- for (int j = 0; j < height_div8; ++j) {
- transpose_16bit_8x8(buf0 + j * 8, buf1 + j * width + 8 * i);
+ for (int i = 0; i < 2; i++) {
+ load_buffer_s16_x8(input + 8 * i, stride, buf0, 64);
+ col_txfm(buf0, buf0, 13);
+ shift_right_2_round_s16_x8(buf0, buf0, 64);
+ for (int j = 0; j < 8; ++j) {
+ transpose_arrays_s16_8x8(buf0 + j * 8, buf1 + j * 16 + 8 * i);
}
}
- for (int i = 0; i < AOMMIN(4, height_div8); i++) {
- int16x8_t *buf = buf1 + width * i;
- row_txfm(buf, buf, cos_bit_row, NULL);
- round_shift_16bit(buf, width, shift[2]);
- store_buffer_16bit_to_32bit_w8(buf, output + 8 * i, 32, 16);
+ for (int i = 0; i < 4; i++) {
+ int16x8_t *buf = buf1 + 16 * i;
+ row_txfm(buf, buf, 12);
+ store_buffer_s16_x8(buf, output + 8 * i, 32, 16);
}
}
-#define TRANSPOSE_4X4_L32(x0, x1, x2, x3, y0, y1, y2, y3) \
- do { \
- int32x4x2_t temp01 = vzipq_s32(x0, x1); \
- int32x4x2_t temp23 = vzipq_s32(x2, x3); \
- int32x4x2_t y01 = vzipq_s32(temp01.val[0], temp23.val[0]); \
- int32x4x2_t y23 = vzipq_s32(temp01.val[1], temp23.val[1]); \
- y0 = y01.val[0]; \
- y1 = y01.val[1]; \
- y2 = y23.val[0]; \
- y3 = y23.val[1]; \
- } while (0)
+static void fdct32_new_neon(const int32x4_t *input, int32x4_t *output,
+ int cos_bit) {
+ const int16_t *cospi = cospi_arr_q13(cos_bit);
-static void av1_fdct32_new_neon(int32x4_t *input, int32x4_t *output,
- int cos_bit, const int stride,
- const int8_t *stage_range) {
- (void)stage_range;
+ const int16x8_t cospi32_16 = vld1q_s16(&cospi[4 * 0]);
+ const int16x8_t cospi8_24 = vld1q_s16(&cospi[4 * 2]);
+ const int16x8_t cospi4_12 = vld1q_s16(&cospi[4 * 4]);
+ const int16x8_t cospi20_28 = vld1q_s16(&cospi[4 * 6]);
+ const int16x8_t cospi2_6 = vld1q_s16(&cospi[4 * 8]);
+ const int16x8_t cospi10_14 = vld1q_s16(&cospi[4 * 10]);
+ const int16x8_t cospi18_22 = vld1q_s16(&cospi[4 * 12]);
+ const int16x8_t cospi26_30 = vld1q_s16(&cospi[4 * 14]);
+
+ const int16x4_t cospi32 = vget_low_s16(cospi32_16);
+ const int16x4_t cospi16 = vget_high_s16(cospi32_16);
+ const int16x4_t cospi8 = vget_low_s16(cospi8_24);
+ const int16x4_t cospi24 = vget_high_s16(cospi8_24);
+ const int16x4_t cospi4 = vget_low_s16(cospi4_12);
+ const int16x4_t cospi12 = vget_high_s16(cospi4_12);
+ const int16x4_t cospi20 = vget_low_s16(cospi20_28);
+ const int16x4_t cospi28 = vget_high_s16(cospi20_28);
+ const int16x4_t cospi2 = vget_low_s16(cospi2_6);
+ const int16x4_t cospi6 = vget_high_s16(cospi2_6);
+ const int16x4_t cospi10 = vget_low_s16(cospi10_14);
+ const int16x4_t cospi14 = vget_high_s16(cospi10_14);
+ const int16x4_t cospi18 = vget_low_s16(cospi18_22);
+ const int16x4_t cospi22 = vget_high_s16(cospi18_22);
+ const int16x4_t cospi26 = vget_low_s16(cospi26_30);
+ const int16x4_t cospi30 = vget_high_s16(cospi26_30);
+
int32x4_t buf0[32];
int32x4_t buf1[32];
- const int32_t *cospi;
- cospi = cospi_arr(cos_bit);
- const int32x4_t v_cos_bit = vdupq_n_s32(-cos_bit);
- int startidx = 0 * stride;
- int endidx = 31 * stride;
- // stage 0
// stage 1
- buf1[0] = vaddq_s32(input[startidx], input[endidx]);
- buf1[31] = vsubq_s32(input[startidx], input[endidx]);
- startidx += stride;
- endidx -= stride;
- buf1[1] = vaddq_s32(input[startidx], input[endidx]);
- buf1[30] = vsubq_s32(input[startidx], input[endidx]);
- startidx += stride;
- endidx -= stride;
- buf1[2] = vaddq_s32(input[startidx], input[endidx]);
- buf1[29] = vsubq_s32(input[startidx], input[endidx]);
- startidx += stride;
- endidx -= stride;
- buf1[3] = vaddq_s32(input[startidx], input[endidx]);
- buf1[28] = vsubq_s32(input[startidx], input[endidx]);
- startidx += stride;
- endidx -= stride;
- buf1[4] = vaddq_s32(input[startidx], input[endidx]);
- buf1[27] = vsubq_s32(input[startidx], input[endidx]);
- startidx += stride;
- endidx -= stride;
- buf1[5] = vaddq_s32(input[startidx], input[endidx]);
- buf1[26] = vsubq_s32(input[startidx], input[endidx]);
- startidx += stride;
- endidx -= stride;
- buf1[6] = vaddq_s32(input[startidx], input[endidx]);
- buf1[25] = vsubq_s32(input[startidx], input[endidx]);
- startidx += stride;
- endidx -= stride;
- buf1[7] = vaddq_s32(input[startidx], input[endidx]);
- buf1[24] = vsubq_s32(input[startidx], input[endidx]);
- startidx += stride;
- endidx -= stride;
- buf1[8] = vaddq_s32(input[startidx], input[endidx]);
- buf1[23] = vsubq_s32(input[startidx], input[endidx]);
- startidx += stride;
- endidx -= stride;
- buf1[9] = vaddq_s32(input[startidx], input[endidx]);
- buf1[22] = vsubq_s32(input[startidx], input[endidx]);
- startidx += stride;
- endidx -= stride;
- buf1[10] = vaddq_s32(input[startidx], input[endidx]);
- buf1[21] = vsubq_s32(input[startidx], input[endidx]);
- startidx += stride;
- endidx -= stride;
- buf1[11] = vaddq_s32(input[startidx], input[endidx]);
- buf1[20] = vsubq_s32(input[startidx], input[endidx]);
- startidx += stride;
- endidx -= stride;
- buf1[12] = vaddq_s32(input[startidx], input[endidx]);
- buf1[19] = vsubq_s32(input[startidx], input[endidx]);
- startidx += stride;
- endidx -= stride;
- buf1[13] = vaddq_s32(input[startidx], input[endidx]);
- buf1[18] = vsubq_s32(input[startidx], input[endidx]);
- startidx += stride;
- endidx -= stride;
- buf1[14] = vaddq_s32(input[startidx], input[endidx]);
- buf1[17] = vsubq_s32(input[startidx], input[endidx]);
- startidx += stride;
- endidx -= stride;
- buf1[15] = vaddq_s32(input[startidx], input[endidx]);
- buf1[16] = vsubq_s32(input[startidx], input[endidx]);
+ butterfly_dct_pre_s32_x4(input, buf1, 32);
// stage 2
- buf0[0] = vaddq_s32(buf1[0], buf1[15]);
- buf0[15] = vsubq_s32(buf1[0], buf1[15]);
- buf0[1] = vaddq_s32(buf1[1], buf1[14]);
- buf0[14] = vsubq_s32(buf1[1], buf1[14]);
- buf0[2] = vaddq_s32(buf1[2], buf1[13]);
- buf0[13] = vsubq_s32(buf1[2], buf1[13]);
- buf0[3] = vaddq_s32(buf1[3], buf1[12]);
- buf0[12] = vsubq_s32(buf1[3], buf1[12]);
- buf0[4] = vaddq_s32(buf1[4], buf1[11]);
- buf0[11] = vsubq_s32(buf1[4], buf1[11]);
- buf0[5] = vaddq_s32(buf1[5], buf1[10]);
- buf0[10] = vsubq_s32(buf1[5], buf1[10]);
- buf0[6] = vaddq_s32(buf1[6], buf1[9]);
- buf0[9] = vsubq_s32(buf1[6], buf1[9]);
- buf0[7] = vaddq_s32(buf1[7], buf1[8]);
- buf0[8] = vsubq_s32(buf1[7], buf1[8]);
+ butterfly_dct_pre_s32_x4(buf1, buf0, 16);
buf0[16] = buf1[16];
buf0[17] = buf1[17];
buf0[18] = buf1[18];
buf0[19] = buf1[19];
- btf_32_neon_mode0(cospi[32], cospi[32], buf1[20], buf1[27], buf0[20],
- buf0[27], v_cos_bit);
- btf_32_neon_mode0(cospi[32], cospi[32], buf1[21], buf1[26], buf0[21],
- buf0[26], v_cos_bit);
- btf_32_neon_mode0(cospi[32], cospi[32], buf1[22], buf1[25], buf0[22],
- buf0[25], v_cos_bit);
- btf_32_neon_mode0(cospi[32], cospi[32], buf1[23], buf1[24], buf0[23],
- buf0[24], v_cos_bit);
+ butterfly_s32_s32_x4_0112_neon(cospi32, buf1[27], buf1[20], &buf0[27],
+ &buf0[20]);
+ butterfly_s32_s32_x4_0112_neon(cospi32, buf1[26], buf1[21], &buf0[26],
+ &buf0[21]);
+ butterfly_s32_s32_x4_0112_neon(cospi32, buf1[25], buf1[22], &buf0[25],
+ &buf0[22]);
+ butterfly_s32_s32_x4_0112_neon(cospi32, buf1[24], buf1[23], &buf0[24],
+ &buf0[23]);
buf0[28] = buf1[28];
buf0[29] = buf1[29];
buf0[30] = buf1[30];
buf0[31] = buf1[31];
// stage 3
- cospi = cospi_arr(cos_bit);
- buf1[0] = vaddq_s32(buf0[0], buf0[7]);
- buf1[7] = vsubq_s32(buf0[0], buf0[7]);
- buf1[1] = vaddq_s32(buf0[1], buf0[6]);
- buf1[6] = vsubq_s32(buf0[1], buf0[6]);
- buf1[2] = vaddq_s32(buf0[2], buf0[5]);
- buf1[5] = vsubq_s32(buf0[2], buf0[5]);
- buf1[3] = vaddq_s32(buf0[3], buf0[4]);
- buf1[4] = vsubq_s32(buf0[3], buf0[4]);
+ butterfly_dct_pre_s32_x4(buf0, buf1, 8);
buf1[8] = buf0[8];
buf1[9] = buf0[9];
- btf_32_neon_mode0(cospi[32], cospi[32], buf0[10], buf0[13], buf1[10],
- buf1[13], v_cos_bit);
- btf_32_neon_mode0(cospi[32], cospi[32], buf0[11], buf0[12], buf1[11],
- buf1[12], v_cos_bit);
+ butterfly_s32_s32_x4_0112_neon(cospi32, buf0[13], buf0[10], &buf1[13],
+ &buf1[10]);
+ butterfly_s32_s32_x4_0112_neon(cospi32, buf0[12], buf0[11], &buf1[12],
+ &buf1[11]);
buf1[14] = buf0[14];
buf1[15] = buf0[15];
- buf1[16] = vaddq_s32(buf0[16], buf0[23]);
- buf1[23] = vsubq_s32(buf0[16], buf0[23]);
- buf1[17] = vaddq_s32(buf0[17], buf0[22]);
- buf1[22] = vsubq_s32(buf0[17], buf0[22]);
- buf1[18] = vaddq_s32(buf0[18], buf0[21]);
- buf1[21] = vsubq_s32(buf0[18], buf0[21]);
- buf1[19] = vaddq_s32(buf0[19], buf0[20]);
- buf1[20] = vsubq_s32(buf0[19], buf0[20]);
- buf1[24] = vsubq_s32(buf0[31], buf0[24]);
- buf1[31] = vaddq_s32(buf0[31], buf0[24]);
- buf1[25] = vsubq_s32(buf0[30], buf0[25]);
- buf1[30] = vaddq_s32(buf0[30], buf0[25]);
- buf1[26] = vsubq_s32(buf0[29], buf0[26]);
- buf1[29] = vaddq_s32(buf0[29], buf0[26]);
- buf1[27] = vsubq_s32(buf0[28], buf0[27]);
- buf1[28] = vaddq_s32(buf0[28], buf0[27]);
+ butterfly_dct_post_s32_x4(buf0 + 16, buf0 + 16, buf1 + 16, 16);
// stage 4
- cospi = cospi_arr(cos_bit);
- buf0[0] = vaddq_s32(buf1[0], buf1[3]);
- buf0[3] = vsubq_s32(buf1[0], buf1[3]);
- buf0[1] = vaddq_s32(buf1[1], buf1[2]);
- buf0[2] = vsubq_s32(buf1[1], buf1[2]);
+ butterfly_dct_pre_s32_x4(buf1, buf0, 4);
buf0[4] = buf1[4];
- btf_32_neon_mode0(cospi[32], cospi[32], buf1[5], buf1[6], buf0[5], buf0[6],
- v_cos_bit);
+ butterfly_s32_s32_x4_0112_neon(cospi32, buf1[6], buf1[5], &buf0[6], &buf0[5]);
buf0[7] = buf1[7];
- buf0[8] = vaddq_s32(buf1[8], buf1[11]);
- buf0[11] = vsubq_s32(buf1[8], buf1[11]);
- buf0[9] = vaddq_s32(buf1[9], buf1[10]);
- buf0[10] = vsubq_s32(buf1[9], buf1[10]);
- buf0[12] = vsubq_s32(buf1[15], buf1[12]);
- buf0[15] = vaddq_s32(buf1[15], buf1[12]);
- buf0[13] = vsubq_s32(buf1[14], buf1[13]);
- buf0[14] = vaddq_s32(buf1[14], buf1[13]);
+ butterfly_dct_post_s32_x4(buf1 + 8, buf1 + 8, buf0 + 8, 8);
buf0[16] = buf1[16];
buf0[17] = buf1[17];
- btf_32_neon_mode0(cospi[16], cospi[48], buf1[18], buf1[29], buf0[18],
- buf0[29], v_cos_bit);
- btf_32_neon_mode0(cospi[16], cospi[48], buf1[19], buf1[28], buf0[19],
- buf0[28], v_cos_bit);
- btf_32_neon_mode01(cospi[48], cospi[16], buf1[20], buf1[27], buf0[20],
- buf0[27], v_cos_bit);
- btf_32_neon_mode01(cospi[48], cospi[16], buf1[21], buf1[26], buf0[21],
- buf0[26], v_cos_bit);
+ butterfly_s32_s32_x4_0112_neon(cospi16, buf1[29], buf1[18], &buf0[29],
+ &buf0[18]);
+ butterfly_s32_s32_x4_0112_neon(cospi16, buf1[28], buf1[19], &buf0[28],
+ &buf0[19]);
+ butterfly_s32_s32_x4_1223_neon(cospi16, buf1[27], buf1[20], &buf0[27],
+ &buf0[20]);
+ butterfly_s32_s32_x4_1223_neon(cospi16, buf1[26], buf1[21], &buf0[26],
+ &buf0[21]);
buf0[22] = buf1[22];
buf0[23] = buf1[23];
buf0[24] = buf1[24];
@@ -3202,69 +2467,40 @@
buf0[31] = buf1[31];
// stage 5
- cospi = cospi_arr(cos_bit);
- btf_32_neon(cospi[32], cospi[32], buf0[0], buf0[1], buf1[0], buf1[1],
- v_cos_bit);
- btf_32_type1_neon(cospi[48], cospi[16], buf0[2], buf0[3], buf1[2], buf1[3],
- v_cos_bit);
- buf1[4] = vaddq_s32(buf0[4], buf0[5]);
- buf1[5] = vsubq_s32(buf0[4], buf0[5]);
- buf1[6] = vsubq_s32(buf0[7], buf0[6]);
- buf1[7] = vaddq_s32(buf0[7], buf0[6]);
+ butterfly_s32_s32_x4_0112_neon(cospi32, buf0[0], buf0[1], &buf1[0], &buf1[1]);
+ butterfly_s32_s32_x4_0112_neon(cospi16, buf0[3], buf0[2], &buf1[2], &buf1[3]);
+ butterfly_dct_post_s32_x4(buf0 + 4, buf0 + 4, buf1 + 4, 4);
buf1[8] = buf0[8];
- btf_32_neon_mode0(cospi[16], cospi[48], buf0[9], buf0[14], buf1[9], buf1[14],
- v_cos_bit);
- btf_32_neon_mode01(cospi[48], cospi[16], buf0[10], buf0[13], buf1[10],
- buf1[13], v_cos_bit);
+ butterfly_s32_s32_x4_0112_neon(cospi16, buf0[14], buf0[9], &buf1[14],
+ &buf1[9]);
+ butterfly_s32_s32_x4_1223_neon(cospi16, buf0[13], buf0[10], &buf1[13],
+ &buf1[10]);
buf1[11] = buf0[11];
buf1[12] = buf0[12];
buf1[15] = buf0[15];
- buf1[16] = vaddq_s32(buf0[16], buf0[19]);
- buf1[19] = vsubq_s32(buf0[16], buf0[19]);
- buf1[17] = vaddq_s32(buf0[17], buf0[18]);
- buf1[18] = vsubq_s32(buf0[17], buf0[18]);
- buf1[20] = vsubq_s32(buf0[23], buf0[20]);
- buf1[23] = vaddq_s32(buf0[23], buf0[20]);
- buf1[21] = vsubq_s32(buf0[22], buf0[21]);
- buf1[22] = vaddq_s32(buf0[22], buf0[21]);
- buf1[24] = vaddq_s32(buf0[24], buf0[27]);
- buf1[27] = vsubq_s32(buf0[24], buf0[27]);
- buf1[25] = vaddq_s32(buf0[25], buf0[26]);
- buf1[26] = vsubq_s32(buf0[25], buf0[26]);
- buf1[28] = vsubq_s32(buf0[31], buf0[28]);
- buf1[31] = vaddq_s32(buf0[31], buf0[28]);
- buf1[29] = vsubq_s32(buf0[30], buf0[29]);
- buf1[30] = vaddq_s32(buf0[30], buf0[29]);
+ butterfly_dct_post_s32_x4(buf0 + 16, buf0 + 16, buf1 + 16, 8);
+ butterfly_dct_post_s32_x4(buf0 + 24, buf0 + 24, buf1 + 24, 8);
// stage 6
- cospi = cospi_arr(cos_bit);
buf0[0] = buf1[0];
buf0[1] = buf1[1];
buf0[2] = buf1[2];
buf0[3] = buf1[3];
- btf_32_type1_neon(cospi[56], cospi[8], buf1[4], buf1[7], buf0[4], buf0[7],
- v_cos_bit);
- btf_32_type1_neon(cospi[24], cospi[40], buf1[5], buf1[6], buf0[5], buf0[6],
- v_cos_bit);
- buf0[8] = vaddq_s32(buf1[8], buf1[9]);
- buf0[9] = vsubq_s32(buf1[8], buf1[9]);
- buf0[10] = vsubq_s32(buf1[11], buf1[10]);
- buf0[11] = vaddq_s32(buf1[11], buf1[10]);
- buf0[12] = vaddq_s32(buf1[12], buf1[13]);
- buf0[13] = vsubq_s32(buf1[12], buf1[13]);
- buf0[14] = vsubq_s32(buf1[15], buf1[14]);
- buf0[15] = vaddq_s32(buf1[15], buf1[14]);
+ butterfly_s32_s32_x4_0112_neon(cospi8, buf1[7], buf1[4], &buf0[4], &buf0[7]);
+ butterfly_s32_s32_x4_1003_neon(cospi24, buf1[6], buf1[5], &buf0[5], &buf0[6]);
+ butterfly_dct_post_s32_x4(buf1 + 8, buf1 + 8, buf0 + 8, 4);
+ butterfly_dct_post_s32_x4(buf1 + 12, buf1 + 12, buf0 + 12, 4);
buf0[16] = buf1[16];
- btf_32_neon_mode0(cospi[8], cospi[56], buf1[17], buf1[30], buf0[17], buf0[30],
- v_cos_bit);
- btf_32_neon_mode01(cospi[56], cospi[8], buf1[18], buf1[29], buf0[18],
- buf0[29], v_cos_bit);
+ butterfly_s32_s32_x4_0112_neon(cospi8, buf1[30], buf1[17], &buf0[30],
+ &buf0[17]);
+ butterfly_s32_s32_x4_1223_neon(cospi8, buf1[29], buf1[18], &buf0[29],
+ &buf0[18]);
buf0[19] = buf1[19];
buf0[20] = buf1[20];
- btf_32_neon_mode0(cospi[40], cospi[24], buf1[21], buf1[26], buf0[21],
- buf0[26], v_cos_bit);
- btf_32_neon_mode01(cospi[24], cospi[40], buf1[22], buf1[25], buf0[22],
- buf0[25], v_cos_bit);
+ butterfly_s32_s32_x4_1003_neon(cospi24, buf1[26], buf1[21], &buf0[26],
+ &buf0[21]);
+ butterfly_s32_s32_x4_0332_neon(cospi24, buf1[25], buf1[22], &buf0[25],
+ &buf0[22]);
buf0[23] = buf1[23];
buf0[24] = buf1[24];
buf0[27] = buf1[27];
@@ -3272,7 +2508,6 @@
buf0[31] = buf1[31];
// stage 7
- cospi = cospi_arr(cos_bit);
buf1[0] = buf0[0];
buf1[1] = buf0[1];
buf1[2] = buf0[2];
@@ -3281,34 +2516,20 @@
buf1[5] = buf0[5];
buf1[6] = buf0[6];
buf1[7] = buf0[7];
-
- btf_32_type1_neon(cospi[60], cospi[4], buf0[8], buf0[15], buf1[8], buf1[15],
- v_cos_bit);
- btf_32_type1_neon(cospi[28], cospi[36], buf0[9], buf0[14], buf1[9], buf1[14],
- v_cos_bit);
- btf_32_type1_neon(cospi[44], cospi[20], buf0[10], buf0[13], buf1[10],
- buf1[13], v_cos_bit);
- btf_32_type1_neon(cospi[12], cospi[52], buf0[11], buf0[12], buf1[11],
- buf1[12], v_cos_bit);
- buf1[16] = vaddq_s32(buf0[16], buf0[17]);
- buf1[17] = vsubq_s32(buf0[16], buf0[17]);
- buf1[18] = vsubq_s32(buf0[19], buf0[18]);
- buf1[19] = vaddq_s32(buf0[19], buf0[18]);
- buf1[20] = vaddq_s32(buf0[20], buf0[21]);
- buf1[21] = vsubq_s32(buf0[20], buf0[21]);
- buf1[22] = vsubq_s32(buf0[23], buf0[22]);
- buf1[23] = vaddq_s32(buf0[23], buf0[22]);
- buf1[24] = vaddq_s32(buf0[24], buf0[25]);
- buf1[25] = vsubq_s32(buf0[24], buf0[25]);
- buf1[26] = vsubq_s32(buf0[27], buf0[26]);
- buf1[27] = vaddq_s32(buf0[27], buf0[26]);
- buf1[28] = vaddq_s32(buf0[28], buf0[29]);
- buf1[29] = vsubq_s32(buf0[28], buf0[29]);
- buf1[30] = vsubq_s32(buf0[31], buf0[30]);
- buf1[31] = vaddq_s32(buf0[31], buf0[30]);
+ butterfly_s32_s32_x4_0112_neon(cospi4, buf0[15], buf0[8], &buf1[8],
+ &buf1[15]);
+ butterfly_s32_s32_x4_1003_neon(cospi28, buf0[14], buf0[9], &buf1[9],
+ &buf1[14]);
+ butterfly_s32_s32_x4_0112_neon(cospi20, buf0[13], buf0[10], &buf1[10],
+ &buf1[13]);
+ butterfly_s32_s32_x4_1003_neon(cospi12, buf0[12], buf0[11], &buf1[11],
+ &buf1[12]);
+ butterfly_dct_post_s32_x4(buf0 + 16, buf0 + 16, buf1 + 16, 4);
+ butterfly_dct_post_s32_x4(buf0 + 20, buf0 + 20, buf1 + 20, 4);
+ butterfly_dct_post_s32_x4(buf0 + 24, buf0 + 24, buf1 + 24, 4);
+ butterfly_dct_post_s32_x4(buf0 + 28, buf0 + 28, buf1 + 28, 4);
// stage 8
- cospi = cospi_arr(cos_bit);
buf0[0] = buf1[0];
buf0[1] = buf1[1];
buf0[2] = buf1[2];
@@ -3325,988 +2546,408 @@
buf0[13] = buf1[13];
buf0[14] = buf1[14];
buf0[15] = buf1[15];
+ butterfly_s32_s32_x4_0112_neon(cospi2, buf1[31], buf1[16], &buf0[16],
+ &buf0[31]);
+ butterfly_s32_s32_x4_1003_neon(cospi30, buf1[30], buf1[17], &buf0[17],
+ &buf0[30]);
+ butterfly_s32_s32_x4_0112_neon(cospi18, buf1[29], buf1[18], &buf0[18],
+ &buf0[29]);
+ butterfly_s32_s32_x4_1003_neon(cospi14, buf1[28], buf1[19], &buf0[19],
+ &buf0[28]);
+ butterfly_s32_s32_x4_0112_neon(cospi10, buf1[27], buf1[20], &buf0[20],
+ &buf0[27]);
+ butterfly_s32_s32_x4_1003_neon(cospi22, buf1[26], buf1[21], &buf0[21],
+ &buf0[26]);
+ butterfly_s32_s32_x4_0112_neon(cospi26, buf1[25], buf1[22], &buf0[22],
+ &buf0[25]);
+ butterfly_s32_s32_x4_1003_neon(cospi6, buf1[24], buf1[23], &buf0[23],
+ &buf0[24]);
- btf_32_type1_neon(cospi[62], cospi[2], buf1[16], buf1[31], buf0[16], buf0[31],
- v_cos_bit);
- btf_32_type1_neon(cospi[30], cospi[34], buf1[17], buf1[30], buf0[17],
- buf0[30], v_cos_bit);
- btf_32_type1_neon(cospi[46], cospi[18], buf1[18], buf1[29], buf0[18],
- buf0[29], v_cos_bit);
- btf_32_type1_neon(cospi[14], cospi[50], buf1[19], buf1[28], buf0[19],
- buf0[28], v_cos_bit);
- btf_32_type1_neon(cospi[54], cospi[10], buf1[20], buf1[27], buf0[20],
- buf0[27], v_cos_bit);
- btf_32_type1_neon(cospi[22], cospi[42], buf1[21], buf1[26], buf0[21],
- buf0[26], v_cos_bit);
- btf_32_type1_neon(cospi[38], cospi[26], buf1[22], buf1[25], buf0[22],
- buf0[25], v_cos_bit);
- btf_32_type1_neon(cospi[6], cospi[58], buf1[23], buf1[24], buf0[23], buf0[24],
- v_cos_bit);
-
- startidx = 0 * stride;
- endidx = 31 * stride;
// stage 9
- output[startidx] = buf0[0];
- output[endidx] = buf0[31];
- startidx += stride;
- endidx -= stride;
- output[startidx] = buf0[16];
- output[endidx] = buf0[15];
- startidx += stride;
- endidx -= stride;
- output[startidx] = buf0[8];
- output[endidx] = buf0[23];
- startidx += stride;
- endidx -= stride;
- output[startidx] = buf0[24];
- output[endidx] = buf0[7];
- startidx += stride;
- endidx -= stride;
- output[startidx] = buf0[4];
- output[endidx] = buf0[27];
- startidx += stride;
- endidx -= stride;
- output[startidx] = buf0[20];
- output[endidx] = buf0[11];
- startidx += stride;
- endidx -= stride;
- output[startidx] = buf0[12];
- output[endidx] = buf0[19];
- startidx += stride;
- endidx -= stride;
- output[startidx] = buf0[28];
- output[endidx] = buf0[3];
- startidx += stride;
- endidx -= stride;
- output[startidx] = buf0[2];
- output[endidx] = buf0[29];
- startidx += stride;
- endidx -= stride;
- output[startidx] = buf0[18];
- output[endidx] = buf0[13];
- startidx += stride;
- endidx -= stride;
- output[startidx] = buf0[10];
- output[endidx] = buf0[21];
- startidx += stride;
- endidx -= stride;
- output[startidx] = buf0[26];
- output[endidx] = buf0[5];
- startidx += stride;
- endidx -= stride;
- output[startidx] = buf0[6];
- output[endidx] = buf0[25];
- startidx += stride;
- endidx -= stride;
- output[startidx] = buf0[22];
- output[endidx] = buf0[9];
- startidx += stride;
- endidx -= stride;
- output[startidx] = buf0[14];
- output[endidx] = buf0[17];
- startidx += stride;
- endidx -= stride;
- output[startidx] = buf0[30];
- output[endidx] = buf0[1];
+ output[0] = buf0[0];
+ output[1] = buf0[16];
+ output[2] = buf0[8];
+ output[3] = buf0[24];
+ output[4] = buf0[4];
+ output[5] = buf0[20];
+ output[6] = buf0[12];
+ output[7] = buf0[28];
+ output[8] = buf0[2];
+ output[9] = buf0[18];
+ output[10] = buf0[10];
+ output[11] = buf0[26];
+ output[12] = buf0[6];
+ output[13] = buf0[22];
+ output[14] = buf0[14];
+ output[15] = buf0[30];
+ output[16] = buf0[1];
+ output[17] = buf0[17];
+ output[18] = buf0[9];
+ output[19] = buf0[25];
+ output[20] = buf0[5];
+ output[21] = buf0[21];
+ output[22] = buf0[13];
+ output[23] = buf0[29];
+ output[24] = buf0[3];
+ output[25] = buf0[19];
+ output[26] = buf0[11];
+ output[27] = buf0[27];
+ output[28] = buf0[7];
+ output[29] = buf0[23];
+ output[30] = buf0[15];
+ output[31] = buf0[31];
}
-static void av1_fdct64_new_stage1234_neon(int32x4_t *input, const int instride,
- int32x4_t *x3, int32x4_t *x4,
- const int32_t *cospi,
- const int32x4_t *v_cos_bit,
- int *startidx, int *endidx) {
+static void fdct64_new_neon(const int32x4_t *input, int32x4_t *output,
+ int cos_bit) {
+ const int16_t *cospi = cospi_arr_q13(cos_bit);
+
+ const int16x8_t cospi32_16 = vld1q_s16(&cospi[4 * 0]);
+ const int16x8_t cospi8_24 = vld1q_s16(&cospi[4 * 2]);
+ const int16x8_t cospi4_12 = vld1q_s16(&cospi[4 * 4]);
+ const int16x8_t cospi20_28 = vld1q_s16(&cospi[4 * 6]);
+ const int16x8_t cospi2_6 = vld1q_s16(&cospi[4 * 8]);
+ const int16x8_t cospi10_14 = vld1q_s16(&cospi[4 * 10]);
+ const int16x8_t cospi18_22 = vld1q_s16(&cospi[4 * 12]);
+ const int16x8_t cospi26_30 = vld1q_s16(&cospi[4 * 14]);
+ const int16x8_t cospi1_3 = vld1q_s16(&cospi[4 * 16]);
+ const int16x8_t cospi5_7 = vld1q_s16(&cospi[4 * 18]);
+ const int16x8_t cospi9_11 = vld1q_s16(&cospi[4 * 20]);
+ const int16x8_t cospi13_15 = vld1q_s16(&cospi[4 * 22]);
+ const int16x8_t cospi17_19 = vld1q_s16(&cospi[4 * 24]);
+ const int16x8_t cospi21_23 = vld1q_s16(&cospi[4 * 26]);
+ const int16x8_t cospi25_27 = vld1q_s16(&cospi[4 * 28]);
+ const int16x8_t cospi29_31 = vld1q_s16(&cospi[4 * 30]);
+
+ const int16x4_t cospi32 = vget_low_s16(cospi32_16);
+ const int16x4_t cospi16 = vget_high_s16(cospi32_16);
+ const int16x4_t cospi8 = vget_low_s16(cospi8_24);
+ const int16x4_t cospi24 = vget_high_s16(cospi8_24);
+ const int16x4_t cospi4 = vget_low_s16(cospi4_12);
+ const int16x4_t cospi12 = vget_high_s16(cospi4_12);
+ const int16x4_t cospi20 = vget_low_s16(cospi20_28);
+ const int16x4_t cospi28 = vget_high_s16(cospi20_28);
+ const int16x4_t cospi2 = vget_low_s16(cospi2_6);
+ const int16x4_t cospi6 = vget_high_s16(cospi2_6);
+ const int16x4_t cospi10 = vget_low_s16(cospi10_14);
+ const int16x4_t cospi14 = vget_high_s16(cospi10_14);
+ const int16x4_t cospi18 = vget_low_s16(cospi18_22);
+ const int16x4_t cospi22 = vget_high_s16(cospi18_22);
+ const int16x4_t cospi26 = vget_low_s16(cospi26_30);
+ const int16x4_t cospi30 = vget_high_s16(cospi26_30);
+ const int16x4_t cospi1 = vget_low_s16(cospi1_3);
+ const int16x4_t cospi3 = vget_high_s16(cospi1_3);
+ const int16x4_t cospi5 = vget_low_s16(cospi5_7);
+ const int16x4_t cospi7 = vget_high_s16(cospi5_7);
+ const int16x4_t cospi9 = vget_low_s16(cospi9_11);
+ const int16x4_t cospi11 = vget_high_s16(cospi9_11);
+ const int16x4_t cospi13 = vget_low_s16(cospi13_15);
+ const int16x4_t cospi15 = vget_high_s16(cospi13_15);
+ const int16x4_t cospi17 = vget_low_s16(cospi17_19);
+ const int16x4_t cospi19 = vget_high_s16(cospi17_19);
+ const int16x4_t cospi21 = vget_low_s16(cospi21_23);
+ const int16x4_t cospi23 = vget_high_s16(cospi21_23);
+ const int16x4_t cospi25 = vget_low_s16(cospi25_27);
+ const int16x4_t cospi27 = vget_high_s16(cospi25_27);
+ const int16x4_t cospi29 = vget_low_s16(cospi29_31);
+ const int16x4_t cospi31 = vget_high_s16(cospi29_31);
+
// stage 1
int32x4_t x1[64];
- x1[0] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[63] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[1] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[62] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[2] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[61] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[3] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[60] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[4] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[59] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[5] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[58] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[6] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[57] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[7] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[56] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[8] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[55] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[9] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[54] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[10] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[53] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[11] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[52] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[12] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[51] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[13] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[50] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[14] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[49] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[15] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[48] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[16] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[47] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[17] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[46] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[18] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[45] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[19] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[44] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[20] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[43] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[21] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[42] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[22] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[41] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[23] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[40] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[24] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[39] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[25] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[38] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[26] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[37] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[27] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[36] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[28] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[35] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[29] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[34] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[30] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[33] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[31] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[32] = vsubq_s32(input[*startidx], input[*endidx]);
+ butterfly_dct_pre_s32_x4(input, x1, 64);
// stage 2
int32x4_t x2[64];
- x2[0] = vaddq_s32(x1[0], x1[31]);
- x2[31] = vsubq_s32(x1[0], x1[31]);
- x2[1] = vaddq_s32(x1[1], x1[30]);
- x2[30] = vsubq_s32(x1[1], x1[30]);
- x2[2] = vaddq_s32(x1[2], x1[29]);
- x2[29] = vsubq_s32(x1[2], x1[29]);
- x2[3] = vaddq_s32(x1[3], x1[28]);
- x2[28] = vsubq_s32(x1[3], x1[28]);
- x2[4] = vaddq_s32(x1[4], x1[27]);
- x2[27] = vsubq_s32(x1[4], x1[27]);
- x2[5] = vaddq_s32(x1[5], x1[26]);
- x2[26] = vsubq_s32(x1[5], x1[26]);
- x2[6] = vaddq_s32(x1[6], x1[25]);
- x2[25] = vsubq_s32(x1[6], x1[25]);
- x2[7] = vaddq_s32(x1[7], x1[24]);
- x2[24] = vsubq_s32(x1[7], x1[24]);
- x2[8] = vaddq_s32(x1[8], x1[23]);
- x2[23] = vsubq_s32(x1[8], x1[23]);
- x2[9] = vaddq_s32(x1[9], x1[22]);
- x2[22] = vsubq_s32(x1[9], x1[22]);
- x2[10] = vaddq_s32(x1[10], x1[21]);
- x2[21] = vsubq_s32(x1[10], x1[21]);
- x2[11] = vaddq_s32(x1[11], x1[20]);
- x2[20] = vsubq_s32(x1[11], x1[20]);
- x2[12] = vaddq_s32(x1[12], x1[19]);
- x2[19] = vsubq_s32(x1[12], x1[19]);
- x2[13] = vaddq_s32(x1[13], x1[18]);
- x2[18] = vsubq_s32(x1[13], x1[18]);
- x2[14] = vaddq_s32(x1[14], x1[17]);
- x2[17] = vsubq_s32(x1[14], x1[17]);
- x2[15] = vaddq_s32(x1[15], x1[16]);
- x2[16] = vsubq_s32(x1[15], x1[16]);
-
- btf_32_neon_mode0(cospi[32], cospi[32], x1[40], x1[55], x2[40], x2[55],
- *v_cos_bit);
- btf_32_neon_mode0(cospi[32], cospi[32], x1[41], x1[54], x2[41], x2[54],
- *v_cos_bit);
- btf_32_neon_mode0(cospi[32], cospi[32], x1[42], x1[53], x2[42], x2[53],
- *v_cos_bit);
- btf_32_neon_mode0(cospi[32], cospi[32], x1[43], x1[52], x2[43], x2[52],
- *v_cos_bit);
- btf_32_neon_mode0(cospi[32], cospi[32], x1[44], x1[51], x2[44], x2[51],
- *v_cos_bit);
- btf_32_neon_mode0(cospi[32], cospi[32], x1[45], x1[50], x2[45], x2[50],
- *v_cos_bit);
- btf_32_neon_mode0(cospi[32], cospi[32], x1[46], x1[49], x2[46], x2[49],
- *v_cos_bit);
- btf_32_neon_mode0(cospi[32], cospi[32], x1[47], x1[48], x2[47], x2[48],
- *v_cos_bit);
+ butterfly_dct_pre_s32_x4(x1, x2, 32);
+ butterfly_s32_s32_x4_0112_neon(cospi32, x1[55], x1[40], &x2[55], &x2[40]);
+ butterfly_s32_s32_x4_0112_neon(cospi32, x1[54], x1[41], &x2[54], &x2[41]);
+ butterfly_s32_s32_x4_0112_neon(cospi32, x1[53], x1[42], &x2[53], &x2[42]);
+ butterfly_s32_s32_x4_0112_neon(cospi32, x1[52], x1[43], &x2[52], &x2[43]);
+ butterfly_s32_s32_x4_0112_neon(cospi32, x1[51], x1[44], &x2[51], &x2[44]);
+ butterfly_s32_s32_x4_0112_neon(cospi32, x1[50], x1[45], &x2[50], &x2[45]);
+ butterfly_s32_s32_x4_0112_neon(cospi32, x1[49], x1[46], &x2[49], &x2[46]);
+ butterfly_s32_s32_x4_0112_neon(cospi32, x1[48], x1[47], &x2[48], &x2[47]);
// stage 3
- x3[0] = vaddq_s32(x2[0], x2[15]);
- x3[15] = vsubq_s32(x2[0], x2[15]);
- x3[1] = vaddq_s32(x2[1], x2[14]);
- x3[14] = vsubq_s32(x2[1], x2[14]);
- x3[2] = vaddq_s32(x2[2], x2[13]);
- x3[13] = vsubq_s32(x2[2], x2[13]);
- x3[3] = vaddq_s32(x2[3], x2[12]);
- x3[12] = vsubq_s32(x2[3], x2[12]);
- x3[4] = vaddq_s32(x2[4], x2[11]);
- x3[11] = vsubq_s32(x2[4], x2[11]);
- x3[5] = vaddq_s32(x2[5], x2[10]);
- x3[10] = vsubq_s32(x2[5], x2[10]);
- x3[6] = vaddq_s32(x2[6], x2[9]);
- x3[9] = vsubq_s32(x2[6], x2[9]);
- x3[7] = vaddq_s32(x2[7], x2[8]);
- x3[8] = vsubq_s32(x2[7], x2[8]);
-
- btf_32_neon_mode0(cospi[32], cospi[32], x2[20], x2[27], x3[20], x3[27],
- *v_cos_bit);
- btf_32_neon_mode0(cospi[32], cospi[32], x2[21], x2[26], x3[21], x3[26],
- *v_cos_bit);
- btf_32_neon_mode0(cospi[32], cospi[32], x2[22], x2[25], x3[22], x3[25],
- *v_cos_bit);
- btf_32_neon_mode0(cospi[32], cospi[32], x2[23], x2[24], x3[23], x3[24],
- *v_cos_bit);
-
- x3[32] = vaddq_s32(x1[32], x2[47]);
- x3[47] = vsubq_s32(x1[32], x2[47]);
- x3[33] = vaddq_s32(x1[33], x2[46]);
- x3[46] = vsubq_s32(x1[33], x2[46]);
- x3[34] = vaddq_s32(x1[34], x2[45]);
- x3[45] = vsubq_s32(x1[34], x2[45]);
- x3[35] = vaddq_s32(x1[35], x2[44]);
- x3[44] = vsubq_s32(x1[35], x2[44]);
- x3[36] = vaddq_s32(x1[36], x2[43]);
- x3[43] = vsubq_s32(x1[36], x2[43]);
- x3[37] = vaddq_s32(x1[37], x2[42]);
- x3[42] = vsubq_s32(x1[37], x2[42]);
- x3[38] = vaddq_s32(x1[38], x2[41]);
- x3[41] = vsubq_s32(x1[38], x2[41]);
- x3[39] = vaddq_s32(x1[39], x2[40]);
- x3[40] = vsubq_s32(x1[39], x2[40]);
- x3[48] = vsubq_s32(x1[63], x2[48]);
- x3[63] = vaddq_s32(x1[63], x2[48]);
- x3[49] = vsubq_s32(x1[62], x2[49]);
- x3[62] = vaddq_s32(x1[62], x2[49]);
- x3[50] = vsubq_s32(x1[61], x2[50]);
- x3[61] = vaddq_s32(x1[61], x2[50]);
- x3[51] = vsubq_s32(x1[60], x2[51]);
- x3[60] = vaddq_s32(x1[60], x2[51]);
- x3[52] = vsubq_s32(x1[59], x2[52]);
- x3[59] = vaddq_s32(x1[59], x2[52]);
- x3[53] = vsubq_s32(x1[58], x2[53]);
- x3[58] = vaddq_s32(x1[58], x2[53]);
- x3[54] = vsubq_s32(x1[57], x2[54]);
- x3[57] = vaddq_s32(x1[57], x2[54]);
- x3[55] = vsubq_s32(x1[56], x2[55]);
- x3[56] = vaddq_s32(x1[56], x2[55]);
+ int32x4_t x3[64];
+ butterfly_dct_pre_s32_x4(x2, x3, 16);
+ butterfly_s32_s32_x4_0112_neon(cospi32, x2[27], x2[20], &x3[27], &x3[20]);
+ butterfly_s32_s32_x4_0112_neon(cospi32, x2[26], x2[21], &x3[26], &x3[21]);
+ butterfly_s32_s32_x4_0112_neon(cospi32, x2[25], x2[22], &x3[25], &x3[22]);
+ butterfly_s32_s32_x4_0112_neon(cospi32, x2[24], x2[23], &x3[24], &x3[23]);
+ butterfly_dct_post_s32_x4(x1 + 32, x2 + 32, x3 + 32, 32);
// stage 4
- x4[0] = vaddq_s32(x3[0], x3[7]);
- x4[7] = vsubq_s32(x3[0], x3[7]);
- x4[1] = vaddq_s32(x3[1], x3[6]);
- x4[6] = vsubq_s32(x3[1], x3[6]);
- x4[2] = vaddq_s32(x3[2], x3[5]);
- x4[5] = vsubq_s32(x3[2], x3[5]);
- x4[3] = vaddq_s32(x3[3], x3[4]);
- x4[4] = vsubq_s32(x3[3], x3[4]);
-
- btf_32_neon_mode0(cospi[32], cospi[32], x3[10], x3[13], x4[10], x4[13],
- *v_cos_bit);
- btf_32_neon_mode0(cospi[32], cospi[32], x3[11], x3[12], x4[11], x4[12],
- *v_cos_bit);
-
- x4[16] = vaddq_s32(x2[16], x3[23]);
- x4[23] = vsubq_s32(x2[16], x3[23]);
- x4[17] = vaddq_s32(x2[17], x3[22]);
- x4[22] = vsubq_s32(x2[17], x3[22]);
- x4[18] = vaddq_s32(x2[18], x3[21]);
- x4[21] = vsubq_s32(x2[18], x3[21]);
- x4[19] = vaddq_s32(x2[19], x3[20]);
- x4[20] = vsubq_s32(x2[19], x3[20]);
- x4[24] = vsubq_s32(x2[31], x3[24]);
- x4[31] = vaddq_s32(x2[31], x3[24]);
- x4[25] = vsubq_s32(x2[30], x3[25]);
- x4[30] = vaddq_s32(x2[30], x3[25]);
- x4[26] = vsubq_s32(x2[29], x3[26]);
- x4[29] = vaddq_s32(x2[29], x3[26]);
- x4[27] = vsubq_s32(x2[28], x3[27]);
- x4[28] = vaddq_s32(x2[28], x3[27]);
-
- btf_32_neon_mode0(cospi[16], cospi[48], x3[36], x3[59], x4[36], x4[59],
- *v_cos_bit);
- btf_32_neon_mode0(cospi[16], cospi[48], x3[37], x3[58], x4[37], x4[58],
- *v_cos_bit);
- btf_32_neon_mode0(cospi[16], cospi[48], x3[38], x3[57], x4[38], x4[57],
- *v_cos_bit);
- btf_32_neon_mode0(cospi[16], cospi[48], x3[39], x3[56], x4[39], x4[56],
- *v_cos_bit);
- btf_32_neon_mode01(cospi[48], cospi[16], x3[40], x3[55], x4[40], x4[55],
- *v_cos_bit);
- btf_32_neon_mode01(cospi[48], cospi[16], x3[41], x3[54], x4[41], x4[54],
- *v_cos_bit);
- btf_32_neon_mode01(cospi[48], cospi[16], x3[42], x3[53], x4[42], x4[53],
- *v_cos_bit);
- btf_32_neon_mode01(cospi[48], cospi[16], x3[43], x3[52], x4[43], x4[52],
- *v_cos_bit);
-}
-
-static void av1_fdct64_new_neon(int32x4_t *input, int32x4_t *output,
- int8_t cos_bit, const int instride,
- const int outstride,
- const int8_t *stage_range) {
- (void)stage_range;
- const int32_t *cospi = cospi_arr(cos_bit);
- const int32x4_t v_cos_bit = vdupq_n_s32(-cos_bit);
-
- int startidx = 0 * instride;
- int endidx = 63 * instride;
-
- // stage 1-2-3-4
- int32x4_t x3[64], x4[64];
- av1_fdct64_new_stage1234_neon(input, instride, x3, x4, cospi, &v_cos_bit,
- &startidx, &endidx);
+ int32x4_t x4[64];
+ butterfly_dct_pre_s32_x4(x3, x4, 8);
+ butterfly_s32_s32_x4_0112_neon(cospi32, x3[13], x3[10], &x4[13], &x4[10]);
+ butterfly_s32_s32_x4_0112_neon(cospi32, x3[12], x3[11], &x4[12], &x4[11]);
+ butterfly_dct_post_s32_x4(x2 + 16, x3 + 16, x4 + 16, 16);
+ butterfly_s32_s32_x4_0112_neon(cospi16, x3[59], x3[36], &x4[59], &x4[36]);
+ butterfly_s32_s32_x4_0112_neon(cospi16, x3[58], x3[37], &x4[58], &x4[37]);
+ butterfly_s32_s32_x4_0112_neon(cospi16, x3[57], x3[38], &x4[57], &x4[38]);
+ butterfly_s32_s32_x4_0112_neon(cospi16, x3[56], x3[39], &x4[56], &x4[39]);
+ butterfly_s32_s32_x4_1223_neon(cospi16, x3[55], x3[40], &x4[55], &x4[40]);
+ butterfly_s32_s32_x4_1223_neon(cospi16, x3[54], x3[41], &x4[54], &x4[41]);
+ butterfly_s32_s32_x4_1223_neon(cospi16, x3[53], x3[42], &x4[53], &x4[42]);
+ butterfly_s32_s32_x4_1223_neon(cospi16, x3[52], x3[43], &x4[52], &x4[43]);
// stage 5
int32x4_t x5[64];
- x5[0] = vaddq_s32(x4[0], x4[3]);
- x5[3] = vsubq_s32(x4[0], x4[3]);
- x5[1] = vaddq_s32(x4[1], x4[2]);
- x5[2] = vsubq_s32(x4[1], x4[2]);
-
- btf_32_neon_mode0(cospi[32], cospi[32], x4[5], x4[6], x5[5], x5[6],
- v_cos_bit);
-
- x5[8] = vaddq_s32(x3[8], x4[11]);
- x5[11] = vsubq_s32(x3[8], x4[11]);
- x5[9] = vaddq_s32(x3[9], x4[10]);
- x5[10] = vsubq_s32(x3[9], x4[10]);
- x5[12] = vsubq_s32(x3[15], x4[12]);
- x5[15] = vaddq_s32(x3[15], x4[12]);
- x5[13] = vsubq_s32(x3[14], x4[13]);
- x5[14] = vaddq_s32(x3[14], x4[13]);
-
- btf_32_neon_mode0(cospi[16], cospi[48], x4[18], x4[29], x5[18], x5[29],
- v_cos_bit);
- btf_32_neon_mode0(cospi[16], cospi[48], x4[19], x4[28], x5[19], x5[28],
- v_cos_bit);
- btf_32_neon_mode01(cospi[48], cospi[16], x4[20], x4[27], x5[20], x5[27],
- v_cos_bit);
- btf_32_neon_mode01(cospi[48], cospi[16], x4[21], x4[26], x5[21], x5[26],
- v_cos_bit);
-
- x5[32] = vaddq_s32(x3[32], x4[39]);
- x5[39] = vsubq_s32(x3[32], x4[39]);
- x5[33] = vaddq_s32(x3[33], x4[38]);
- x5[38] = vsubq_s32(x3[33], x4[38]);
- x5[34] = vaddq_s32(x3[34], x4[37]);
- x5[37] = vsubq_s32(x3[34], x4[37]);
- x5[35] = vaddq_s32(x3[35], x4[36]);
- x5[36] = vsubq_s32(x3[35], x4[36]);
- x5[40] = vsubq_s32(x3[47], x4[40]);
- x5[47] = vaddq_s32(x3[47], x4[40]);
- x5[41] = vsubq_s32(x3[46], x4[41]);
- x5[46] = vaddq_s32(x3[46], x4[41]);
- x5[42] = vsubq_s32(x3[45], x4[42]);
- x5[45] = vaddq_s32(x3[45], x4[42]);
- x5[43] = vsubq_s32(x3[44], x4[43]);
- x5[44] = vaddq_s32(x3[44], x4[43]);
- x5[48] = vaddq_s32(x3[48], x4[55]);
- x5[55] = vsubq_s32(x3[48], x4[55]);
- x5[49] = vaddq_s32(x3[49], x4[54]);
- x5[54] = vsubq_s32(x3[49], x4[54]);
- x5[50] = vaddq_s32(x3[50], x4[53]);
- x5[53] = vsubq_s32(x3[50], x4[53]);
- x5[51] = vaddq_s32(x3[51], x4[52]);
- x5[52] = vsubq_s32(x3[51], x4[52]);
- x5[56] = vsubq_s32(x3[63], x4[56]);
- x5[63] = vaddq_s32(x3[63], x4[56]);
- x5[57] = vsubq_s32(x3[62], x4[57]);
- x5[62] = vaddq_s32(x3[62], x4[57]);
- x5[58] = vsubq_s32(x3[61], x4[58]);
- x5[61] = vaddq_s32(x3[61], x4[58]);
- x5[59] = vsubq_s32(x3[60], x4[59]);
- x5[60] = vaddq_s32(x3[60], x4[59]);
+ butterfly_dct_pre_s32_x4(x4, x5, 4);
+ butterfly_s32_s32_x4_0112_neon(cospi32, x4[6], x4[5], &x5[6], &x5[5]);
+ butterfly_dct_post_s32_x4(x3 + 8, x4 + 8, x5 + 8, 8);
+ butterfly_s32_s32_x4_0112_neon(cospi16, x4[29], x4[18], &x5[29], &x5[18]);
+ butterfly_s32_s32_x4_0112_neon(cospi16, x4[28], x4[19], &x5[28], &x5[19]);
+ butterfly_s32_s32_x4_1223_neon(cospi16, x4[27], x4[20], &x5[27], &x5[20]);
+ butterfly_s32_s32_x4_1223_neon(cospi16, x4[26], x4[21], &x5[26], &x5[21]);
+ butterfly_dct_post_s32_x4(x3 + 32, x4 + 32, x5 + 32, 16);
+ butterfly_dct_post_s32_x4(x3 + 48, x4 + 48, x5 + 48, 16);
// stage 6
int32x4_t x6[64];
- btf_32_neon(cospi[32], cospi[32], x5[0], x5[1], x6[0], x6[1], v_cos_bit);
- btf_32_type1_neon(cospi[48], cospi[16], x5[2], x5[3], x6[2], x6[3],
- v_cos_bit);
- x6[4] = vaddq_s32(x4[4], x5[5]);
- x6[5] = vsubq_s32(x4[4], x5[5]);
- x6[6] = vsubq_s32(x4[7], x5[6]);
- x6[7] = vaddq_s32(x4[7], x5[6]);
- btf_32_neon_mode0(cospi[16], cospi[48], x5[9], x5[14], x6[9], x6[14],
- v_cos_bit);
- btf_32_neon_mode01(cospi[48], cospi[16], x5[10], x5[13], x6[10], x6[13],
- v_cos_bit);
-
- x6[16] = vaddq_s32(x4[16], x5[19]);
- x6[19] = vsubq_s32(x4[16], x5[19]);
- x6[17] = vaddq_s32(x4[17], x5[18]);
- x6[18] = vsubq_s32(x4[17], x5[18]);
- x6[20] = vsubq_s32(x4[23], x5[20]);
- x6[23] = vaddq_s32(x4[23], x5[20]);
- x6[21] = vsubq_s32(x4[22], x5[21]);
- x6[22] = vaddq_s32(x4[22], x5[21]);
- x6[24] = vaddq_s32(x4[24], x5[27]);
- x6[27] = vsubq_s32(x4[24], x5[27]);
- x6[25] = vaddq_s32(x4[25], x5[26]);
- x6[26] = vsubq_s32(x4[25], x5[26]);
- x6[28] = vsubq_s32(x4[31], x5[28]);
- x6[31] = vaddq_s32(x4[31], x5[28]);
- x6[29] = vsubq_s32(x4[30], x5[29]);
- x6[30] = vaddq_s32(x4[30], x5[29]);
-
- btf_32_neon_mode0(cospi[8], cospi[56], x5[34], x5[61], x6[34], x6[61],
- v_cos_bit);
- btf_32_neon_mode0(cospi[8], cospi[56], x5[35], x5[60], x6[35], x6[60],
- v_cos_bit);
- btf_32_neon_mode01(cospi[56], cospi[8], x5[36], x5[59], x6[36], x6[59],
- v_cos_bit);
- btf_32_neon_mode01(cospi[56], cospi[8], x5[37], x5[58], x6[37], x6[58],
- v_cos_bit);
- btf_32_neon_mode0(cospi[40], cospi[24], x5[42], x5[53], x6[42], x6[53],
- v_cos_bit);
- btf_32_neon_mode0(cospi[40], cospi[24], x5[43], x5[52], x6[43], x6[52],
- v_cos_bit);
- btf_32_neon_mode01(cospi[24], cospi[40], x5[44], x5[51], x6[44], x6[51],
- v_cos_bit);
- btf_32_neon_mode01(cospi[24], cospi[40], x5[45], x5[50], x6[45], x6[50],
- v_cos_bit);
+ butterfly_s32_s32_x4_0112_neon(cospi32, x5[0], x5[1], &x6[0], &x6[1]);
+ butterfly_s32_s32_x4_0112_neon(cospi16, x5[3], x5[2], &x6[2], &x6[3]);
+ butterfly_dct_post_s32_x4(x4 + 4, x5 + 4, x6 + 4, 4);
+ butterfly_s32_s32_x4_0112_neon(cospi16, x5[14], x5[9], &x6[14], &x6[9]);
+ butterfly_s32_s32_x4_1223_neon(cospi16, x5[13], x5[10], &x6[13], &x6[10]);
+ butterfly_dct_post_s32_x4(x4 + 16, x5 + 16, x6 + 16, 8);
+ butterfly_dct_post_s32_x4(x4 + 24, x5 + 24, x6 + 24, 8);
+ butterfly_s32_s32_x4_0112_neon(cospi8, x5[61], x5[34], &x6[61], &x6[34]);
+ butterfly_s32_s32_x4_0112_neon(cospi8, x5[60], x5[35], &x6[60], &x6[35]);
+ butterfly_s32_s32_x4_1223_neon(cospi8, x5[59], x5[36], &x6[59], &x6[36]);
+ butterfly_s32_s32_x4_1223_neon(cospi8, x5[58], x5[37], &x6[58], &x6[37]);
+ butterfly_s32_s32_x4_1003_neon(cospi24, x5[53], x5[42], &x6[53], &x6[42]);
+ butterfly_s32_s32_x4_1003_neon(cospi24, x5[52], x5[43], &x6[52], &x6[43]);
+ butterfly_s32_s32_x4_0332_neon(cospi24, x5[51], x5[44], &x6[51], &x6[44]);
+ butterfly_s32_s32_x4_0332_neon(cospi24, x5[50], x5[45], &x6[50], &x6[45]);
// stage 7
int32x4_t x7[64];
-
- btf_32_type1_neon(cospi[56], cospi[8], x6[4], x6[7], x7[4], x7[7], v_cos_bit);
- btf_32_type1_neon(cospi[24], cospi[40], x6[5], x6[6], x7[5], x7[6],
- v_cos_bit);
- x7[8] = vaddq_s32(x5[8], x6[9]);
- x7[9] = vsubq_s32(x5[8], x6[9]);
- x7[10] = vsubq_s32(x5[11], x6[10]);
- x7[11] = vaddq_s32(x5[11], x6[10]);
- x7[12] = vaddq_s32(x5[12], x6[13]);
- x7[13] = vsubq_s32(x5[12], x6[13]);
- x7[14] = vsubq_s32(x5[15], x6[14]);
- x7[15] = vaddq_s32(x5[15], x6[14]);
-
- btf_32_neon_mode0(cospi[8], cospi[56], x6[17], x6[30], x7[17], x7[30],
- v_cos_bit);
- btf_32_neon_mode01(cospi[56], cospi[8], x6[18], x6[29], x7[18], x7[29],
- v_cos_bit);
-
- btf_32_neon_mode0(cospi[40], cospi[24], x6[21], x6[26], x7[21], x7[26],
- v_cos_bit);
- btf_32_neon_mode01(cospi[24], cospi[40], x6[22], x6[25], x7[22], x7[25],
- v_cos_bit);
-
- x7[32] = vaddq_s32(x5[32], x6[35]);
- x7[35] = vsubq_s32(x5[32], x6[35]);
- x7[33] = vaddq_s32(x5[33], x6[34]);
- x7[34] = vsubq_s32(x5[33], x6[34]);
- x7[36] = vsubq_s32(x5[39], x6[36]);
- x7[39] = vaddq_s32(x5[39], x6[36]);
- x7[37] = vsubq_s32(x5[38], x6[37]);
- x7[38] = vaddq_s32(x5[38], x6[37]);
- x7[40] = vaddq_s32(x5[40], x6[43]);
- x7[43] = vsubq_s32(x5[40], x6[43]);
- x7[41] = vaddq_s32(x5[41], x6[42]);
- x7[42] = vsubq_s32(x5[41], x6[42]);
- x7[44] = vsubq_s32(x5[47], x6[44]);
- x7[47] = vaddq_s32(x5[47], x6[44]);
- x7[45] = vsubq_s32(x5[46], x6[45]);
- x7[46] = vaddq_s32(x5[46], x6[45]);
- x7[48] = vaddq_s32(x5[48], x6[51]);
- x7[51] = vsubq_s32(x5[48], x6[51]);
- x7[49] = vaddq_s32(x5[49], x6[50]);
- x7[50] = vsubq_s32(x5[49], x6[50]);
- x7[52] = vsubq_s32(x5[55], x6[52]);
- x7[55] = vaddq_s32(x5[55], x6[52]);
- x7[53] = vsubq_s32(x5[54], x6[53]);
- x7[54] = vaddq_s32(x5[54], x6[53]);
- x7[56] = vaddq_s32(x5[56], x6[59]);
- x7[59] = vsubq_s32(x5[56], x6[59]);
- x7[57] = vaddq_s32(x5[57], x6[58]);
- x7[58] = vsubq_s32(x5[57], x6[58]);
- x7[60] = vsubq_s32(x5[63], x6[60]);
- x7[63] = vaddq_s32(x5[63], x6[60]);
- x7[61] = vsubq_s32(x5[62], x6[61]);
- x7[62] = vaddq_s32(x5[62], x6[61]);
+ butterfly_s32_s32_x4_0112_neon(cospi8, x6[7], x6[4], &x7[4], &x7[7]);
+ butterfly_s32_s32_x4_1003_neon(cospi24, x6[6], x6[5], &x7[5], &x7[6]);
+ butterfly_dct_post_s32_x4(x5 + 8, x6 + 8, x7 + 8, 4);
+ butterfly_dct_post_s32_x4(x5 + 12, x6 + 12, x7 + 12, 4);
+ butterfly_s32_s32_x4_0112_neon(cospi8, x6[30], x6[17], &x7[30], &x7[17]);
+ butterfly_s32_s32_x4_1223_neon(cospi8, x6[29], x6[18], &x7[29], &x7[18]);
+ butterfly_s32_s32_x4_1003_neon(cospi24, x6[26], x6[21], &x7[26], &x7[21]);
+ butterfly_s32_s32_x4_0332_neon(cospi24, x6[25], x6[22], &x7[25], &x7[22]);
+ butterfly_dct_post_s32_x4(x5 + 32, x6 + 32, x7 + 32, 8);
+ butterfly_dct_post_s32_x4(x5 + 40, x6 + 40, x7 + 40, 8);
+ butterfly_dct_post_s32_x4(x5 + 48, x6 + 48, x7 + 48, 8);
+ butterfly_dct_post_s32_x4(x5 + 56, x6 + 56, x7 + 56, 8);
// stage 8
int32x4_t x8[64];
-
- btf_32_type1_neon(cospi[60], cospi[4], x7[8], x7[15], x8[8], x8[15],
- v_cos_bit);
- btf_32_type1_neon(cospi[28], cospi[36], x7[9], x7[14], x8[9], x8[14],
- v_cos_bit);
- btf_32_type1_neon(cospi[44], cospi[20], x7[10], x7[13], x8[10], x8[13],
- v_cos_bit);
- btf_32_type1_neon(cospi[12], cospi[52], x7[11], x7[12], x8[11], x8[12],
- v_cos_bit);
- x8[16] = vaddq_s32(x6[16], x7[17]);
- x8[17] = vsubq_s32(x6[16], x7[17]);
- x8[18] = vsubq_s32(x6[19], x7[18]);
- x8[19] = vaddq_s32(x6[19], x7[18]);
- x8[20] = vaddq_s32(x6[20], x7[21]);
- x8[21] = vsubq_s32(x6[20], x7[21]);
- x8[22] = vsubq_s32(x6[23], x7[22]);
- x8[23] = vaddq_s32(x6[23], x7[22]);
- x8[24] = vaddq_s32(x6[24], x7[25]);
- x8[25] = vsubq_s32(x6[24], x7[25]);
- x8[26] = vsubq_s32(x6[27], x7[26]);
- x8[27] = vaddq_s32(x6[27], x7[26]);
- x8[28] = vaddq_s32(x6[28], x7[29]);
- x8[29] = vsubq_s32(x6[28], x7[29]);
- x8[30] = vsubq_s32(x6[31], x7[30]);
- x8[31] = vaddq_s32(x6[31], x7[30]);
-
- btf_32_neon_mode0(cospi[4], cospi[60], x7[33], x7[62], x8[33], x8[62],
- v_cos_bit);
- btf_32_neon_mode01(cospi[60], cospi[4], x7[34], x7[61], x8[34], x8[61],
- v_cos_bit);
- btf_32_neon_mode0(cospi[36], cospi[28], x7[37], x7[58], x8[37], x8[58],
- v_cos_bit);
- btf_32_neon_mode01(cospi[28], cospi[36], x7[38], x7[57], x8[38], x8[57],
- v_cos_bit);
- btf_32_neon_mode0(cospi[20], cospi[44], x7[41], x7[54], x8[41], x8[54],
- v_cos_bit);
- btf_32_neon_mode01(cospi[44], cospi[20], x7[42], x7[53], x8[42], x8[53],
- v_cos_bit);
- btf_32_neon_mode0(cospi[52], cospi[12], x7[45], x7[50], x8[45], x8[50],
- v_cos_bit);
- btf_32_neon_mode01(cospi[12], cospi[52], x7[46], x7[49], x8[46], x8[49],
- v_cos_bit);
+ butterfly_s32_s32_x4_0112_neon(cospi4, x7[15], x7[8], &x8[8], &x8[15]);
+ butterfly_s32_s32_x4_1003_neon(cospi28, x7[14], x7[9], &x8[9], &x8[14]);
+ butterfly_s32_s32_x4_0112_neon(cospi20, x7[13], x7[10], &x8[10], &x8[13]);
+ butterfly_s32_s32_x4_1003_neon(cospi12, x7[12], x7[11], &x8[11], &x8[12]);
+ butterfly_dct_post_s32_x4(x6 + 16, x7 + 16, x8 + 16, 4);
+ butterfly_dct_post_s32_x4(x6 + 20, x7 + 20, x8 + 20, 4);
+ butterfly_dct_post_s32_x4(x6 + 24, x7 + 24, x8 + 24, 4);
+ butterfly_dct_post_s32_x4(x6 + 28, x7 + 28, x8 + 28, 4);
+ butterfly_s32_s32_x4_0112_neon(cospi4, x7[62], x7[33], &x8[62], &x8[33]);
+ butterfly_s32_s32_x4_1223_neon(cospi4, x7[61], x7[34], &x8[61], &x8[34]);
+ butterfly_s32_s32_x4_1003_neon(cospi28, x7[58], x7[37], &x8[58], &x8[37]);
+ butterfly_s32_s32_x4_0332_neon(cospi28, x7[57], x7[38], &x8[57], &x8[38]);
+ butterfly_s32_s32_x4_0112_neon(cospi20, x7[54], x7[41], &x8[54], &x8[41]);
+ butterfly_s32_s32_x4_1223_neon(cospi20, x7[53], x7[42], &x8[53], &x8[42]);
+ butterfly_s32_s32_x4_1003_neon(cospi12, x7[50], x7[45], &x8[50], &x8[45]);
+ butterfly_s32_s32_x4_0332_neon(cospi12, x7[49], x7[46], &x8[49], &x8[46]);
// stage 9
int32x4_t x9[64];
-
- btf_32_type1_neon(cospi[62], cospi[2], x8[16], x8[31], x9[16], x9[31],
- v_cos_bit);
- btf_32_type1_neon(cospi[30], cospi[34], x8[17], x8[30], x9[17], x9[30],
- v_cos_bit);
- btf_32_type1_neon(cospi[46], cospi[18], x8[18], x8[29], x9[18], x9[29],
- v_cos_bit);
- btf_32_type1_neon(cospi[14], cospi[50], x8[19], x8[28], x9[19], x9[28],
- v_cos_bit);
- btf_32_type1_neon(cospi[54], cospi[10], x8[20], x8[27], x9[20], x9[27],
- v_cos_bit);
- btf_32_type1_neon(cospi[22], cospi[42], x8[21], x8[26], x9[21], x9[26],
- v_cos_bit);
- btf_32_type1_neon(cospi[38], cospi[26], x8[22], x8[25], x9[22], x9[25],
- v_cos_bit);
- btf_32_type1_neon(cospi[6], cospi[58], x8[23], x8[24], x9[23], x9[24],
- v_cos_bit);
- x9[32] = vaddq_s32(x7[32], x8[33]);
- x9[33] = vsubq_s32(x7[32], x8[33]);
- x9[34] = vsubq_s32(x7[35], x8[34]);
- x9[35] = vaddq_s32(x7[35], x8[34]);
- x9[36] = vaddq_s32(x7[36], x8[37]);
- x9[37] = vsubq_s32(x7[36], x8[37]);
- x9[38] = vsubq_s32(x7[39], x8[38]);
- x9[39] = vaddq_s32(x7[39], x8[38]);
- x9[40] = vaddq_s32(x7[40], x8[41]);
- x9[41] = vsubq_s32(x7[40], x8[41]);
- x9[42] = vsubq_s32(x7[43], x8[42]);
- x9[43] = vaddq_s32(x7[43], x8[42]);
- x9[44] = vaddq_s32(x7[44], x8[45]);
- x9[45] = vsubq_s32(x7[44], x8[45]);
- x9[46] = vsubq_s32(x7[47], x8[46]);
- x9[47] = vaddq_s32(x7[47], x8[46]);
- x9[48] = vaddq_s32(x7[48], x8[49]);
- x9[49] = vsubq_s32(x7[48], x8[49]);
- x9[50] = vsubq_s32(x7[51], x8[50]);
- x9[51] = vaddq_s32(x7[51], x8[50]);
- x9[52] = vaddq_s32(x7[52], x8[53]);
- x9[53] = vsubq_s32(x7[52], x8[53]);
- x9[54] = vsubq_s32(x7[55], x8[54]);
- x9[55] = vaddq_s32(x7[55], x8[54]);
- x9[56] = vaddq_s32(x7[56], x8[57]);
- x9[57] = vsubq_s32(x7[56], x8[57]);
- x9[58] = vsubq_s32(x7[59], x8[58]);
- x9[59] = vaddq_s32(x7[59], x8[58]);
- x9[60] = vaddq_s32(x7[60], x8[61]);
- x9[61] = vsubq_s32(x7[60], x8[61]);
- x9[62] = vsubq_s32(x7[63], x8[62]);
- x9[63] = vaddq_s32(x7[63], x8[62]);
+ butterfly_s32_s32_x4_0112_neon(cospi2, x8[31], x8[16], &x9[16], &x9[31]);
+ butterfly_s32_s32_x4_1003_neon(cospi30, x8[30], x8[17], &x9[17], &x9[30]);
+ butterfly_s32_s32_x4_0112_neon(cospi18, x8[29], x8[18], &x9[18], &x9[29]);
+ butterfly_s32_s32_x4_1003_neon(cospi14, x8[28], x8[19], &x9[19], &x9[28]);
+ butterfly_s32_s32_x4_0112_neon(cospi10, x8[27], x8[20], &x9[20], &x9[27]);
+ butterfly_s32_s32_x4_1003_neon(cospi22, x8[26], x8[21], &x9[21], &x9[26]);
+ butterfly_s32_s32_x4_0112_neon(cospi26, x8[25], x8[22], &x9[22], &x9[25]);
+ butterfly_s32_s32_x4_1003_neon(cospi6, x8[24], x8[23], &x9[23], &x9[24]);
+ butterfly_dct_post_s32_x4(x7 + 32, x8 + 32, x9 + 32, 4);
+ butterfly_dct_post_s32_x4(x7 + 36, x8 + 36, x9 + 36, 4);
+ butterfly_dct_post_s32_x4(x7 + 40, x8 + 40, x9 + 40, 4);
+ butterfly_dct_post_s32_x4(x7 + 44, x8 + 44, x9 + 44, 4);
+ butterfly_dct_post_s32_x4(x7 + 48, x8 + 48, x9 + 48, 4);
+ butterfly_dct_post_s32_x4(x7 + 52, x8 + 52, x9 + 52, 4);
+ butterfly_dct_post_s32_x4(x7 + 56, x8 + 56, x9 + 56, 4);
+ butterfly_dct_post_s32_x4(x7 + 60, x8 + 60, x9 + 60, 4);
// stage 10
int32x4_t x10[64];
+ butterfly_s32_s32_x4_0112_neon(cospi1, x9[63], x9[32], &x10[32], &x10[63]);
+ butterfly_s32_s32_x4_1003_neon(cospi31, x9[62], x9[33], &x10[33], &x10[62]);
+ butterfly_s32_s32_x4_0112_neon(cospi17, x9[61], x9[34], &x10[34], &x10[61]);
+ butterfly_s32_s32_x4_1003_neon(cospi15, x9[60], x9[35], &x10[35], &x10[60]);
+ butterfly_s32_s32_x4_0112_neon(cospi9, x9[59], x9[36], &x10[36], &x10[59]);
+ butterfly_s32_s32_x4_1003_neon(cospi23, x9[58], x9[37], &x10[37], &x10[58]);
+ butterfly_s32_s32_x4_0112_neon(cospi25, x9[57], x9[38], &x10[38], &x10[57]);
+ butterfly_s32_s32_x4_1003_neon(cospi7, x9[56], x9[39], &x10[39], &x10[56]);
+ butterfly_s32_s32_x4_0112_neon(cospi5, x9[55], x9[40], &x10[40], &x10[55]);
+ butterfly_s32_s32_x4_1003_neon(cospi27, x9[54], x9[41], &x10[41], &x10[54]);
+ butterfly_s32_s32_x4_0112_neon(cospi21, x9[53], x9[42], &x10[42], &x10[53]);
+ butterfly_s32_s32_x4_1003_neon(cospi11, x9[52], x9[43], &x10[43], &x10[52]);
+ butterfly_s32_s32_x4_0112_neon(cospi13, x9[51], x9[44], &x10[44], &x10[51]);
+ butterfly_s32_s32_x4_1003_neon(cospi19, x9[50], x9[45], &x10[45], &x10[50]);
+ butterfly_s32_s32_x4_0112_neon(cospi29, x9[49], x9[46], &x10[46], &x10[49]);
+ butterfly_s32_s32_x4_1003_neon(cospi3, x9[48], x9[47], &x10[47], &x10[48]);
- btf_32_type1_neon(cospi[63], cospi[1], x9[32], x9[63], x10[32], x10[63],
- v_cos_bit);
- btf_32_type1_neon(cospi[31], cospi[33], x9[33], x9[62], x10[33], x10[62],
- v_cos_bit);
- btf_32_type1_neon(cospi[47], cospi[17], x9[34], x9[61], x10[34], x10[61],
- v_cos_bit);
- btf_32_type1_neon(cospi[15], cospi[49], x9[35], x9[60], x10[35], x10[60],
- v_cos_bit);
- btf_32_type1_neon(cospi[55], cospi[9], x9[36], x9[59], x10[36], x10[59],
- v_cos_bit);
- btf_32_type1_neon(cospi[23], cospi[41], x9[37], x9[58], x10[37], x10[58],
- v_cos_bit);
- btf_32_type1_neon(cospi[39], cospi[25], x9[38], x9[57], x10[38], x10[57],
- v_cos_bit);
- btf_32_type1_neon(cospi[7], cospi[57], x9[39], x9[56], x10[39], x10[56],
- v_cos_bit);
- btf_32_type1_neon(cospi[59], cospi[5], x9[40], x9[55], x10[40], x10[55],
- v_cos_bit);
- btf_32_type1_neon(cospi[27], cospi[37], x9[41], x9[54], x10[41], x10[54],
- v_cos_bit);
- btf_32_type1_neon(cospi[43], cospi[21], x9[42], x9[53], x10[42], x10[53],
- v_cos_bit);
- btf_32_type1_neon(cospi[11], cospi[53], x9[43], x9[52], x10[43], x10[52],
- v_cos_bit);
- btf_32_type1_neon(cospi[51], cospi[13], x9[44], x9[51], x10[44], x10[51],
- v_cos_bit);
- btf_32_type1_neon(cospi[19], cospi[45], x9[45], x9[50], x10[45], x10[50],
- v_cos_bit);
- btf_32_type1_neon(cospi[35], cospi[29], x9[46], x9[49], x10[46], x10[49],
- v_cos_bit);
- btf_32_type1_neon(cospi[3], cospi[61], x9[47], x9[48], x10[47], x10[48],
- v_cos_bit);
-
- startidx = 0 * outstride;
- endidx = 63 * outstride;
- // stage 11
- output[startidx] = x6[0];
- output[endidx] = x10[63];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[32];
- output[endidx] = x9[31];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x9[16];
- output[endidx] = x10[47];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[48];
- output[endidx] = x8[15];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x8[8];
- output[endidx] = x10[55];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[40];
- output[endidx] = x9[23];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x9[24];
- output[endidx] = x10[39];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[56];
- output[endidx] = x7[7];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x7[4];
- output[endidx] = x10[59];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[36];
- output[endidx] = x9[27];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x9[20];
- output[endidx] = x10[43];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[52];
- output[endidx] = x8[11];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x8[12];
- output[endidx] = x10[51];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[44];
- output[endidx] = x9[19];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x9[28];
- output[endidx] = x10[35];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[60];
- output[endidx] = x6[3];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x6[2];
- output[endidx] = x10[61];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[34];
- output[endidx] = x9[29];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x9[18];
- output[endidx] = x10[45];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[50];
- output[endidx] = x8[13];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x8[10];
- output[endidx] = x10[53];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[42];
- output[endidx] = x9[21];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x9[26];
- output[endidx] = x10[37];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[58];
- output[endidx] = x7[5];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x7[6];
- output[endidx] = x10[57];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[38];
- output[endidx] = x9[25];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x9[22];
- output[endidx] = x10[41];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[54];
- output[endidx] = x8[9];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x8[14];
- output[endidx] = x10[49];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[46];
- output[endidx] = x9[17];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x9[30];
- output[endidx] = x10[33];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[62];
- output[endidx] = x6[1];
+ // stage 11, only store into the low 32 output indices.
+ output[0] = x6[0];
+ output[1] = x10[32];
+ output[2] = x9[16];
+ output[3] = x10[48];
+ output[4] = x8[8];
+ output[5] = x10[40];
+ output[6] = x9[24];
+ output[7] = x10[56];
+ output[8] = x7[4];
+ output[9] = x10[36];
+ output[10] = x9[20];
+ output[11] = x10[52];
+ output[12] = x8[12];
+ output[13] = x10[44];
+ output[14] = x9[28];
+ output[15] = x10[60];
+ output[16] = x6[2];
+ output[17] = x10[34];
+ output[18] = x9[18];
+ output[19] = x10[50];
+ output[20] = x8[10];
+ output[21] = x10[42];
+ output[22] = x9[26];
+ output[23] = x10[58];
+ output[24] = x7[6];
+ output[25] = x10[38];
+ output[26] = x9[22];
+ output[27] = x10[54];
+ output[28] = x8[14];
+ output[29] = x10[46];
+ output[30] = x9[30];
+ output[31] = x10[62];
}
-static void av1_lowbd_fwd_txfm2d_64x64_neon(const int16_t *input,
- int32_t *output, int stride,
- TX_TYPE tx_type, int bd) {
+static void lowbd_fwd_txfm2d_64x64_neon(const int16_t *input, int32_t *output,
+ int stride, TX_TYPE tx_type, int bd) {
(void)bd;
(void)tx_type;
assert(tx_type == DCT_DCT);
- const TX_SIZE tx_size = TX_64X64;
int16x8_t buf0[64], buf1[512];
- const int8_t *shift = av1_fwd_txfm_shift_ls[tx_size];
- const int txw_idx = get_txw_idx(tx_size);
- const int txh_idx = get_txh_idx(tx_size);
- const int cos_bit_col = av1_fwd_cos_bit_col[txw_idx][txh_idx];
- const int cos_bit_row = av1_fwd_cos_bit_row[txw_idx][txh_idx];
- const int width = tx_size_wide[tx_size];
- const int height = tx_size_high[tx_size];
- const transform_1d_lbd_neon col_txfm = av1_fdct8x64_neon;
- const int width_div8 = (width >> 3);
- const int height_div8 = (height >> 3);
+ const transform_1d_lbd_8_neon col_txfm = fdct8x64_neon;
- for (int i = 0; i < width_div8; i++) {
- load_buffer_16bit_to_16bit(input + 8 * i, stride, buf0, height);
- round_shift_16bit(buf0, height, shift[0]);
- col_txfm(buf0, buf0, cos_bit_col, NULL);
- round_shift_16bit(buf0, height, shift[1]);
- for (int j = 0; j < AOMMIN(4, height_div8); ++j) {
- transpose_16bit_8x8(buf0 + j * 8, buf1 + j * width + 8 * i);
+ for (int i = 0; i < 8; i++) {
+ load_buffer_s16_x8(input + 8 * i, stride, buf0, 64);
+ col_txfm(buf0, buf0, 13);
+ shift_right_2_round_s16_x8(buf0, buf0, 64);
+ for (int j = 0; j < 4; ++j) {
+ transpose_arrays_s16_8x8(buf0 + j * 8, buf1 + j * 64 + 8 * i);
}
}
- for (int i = 0; i < AOMMIN(4, height_div8); i++) {
+ for (int i = 0; i < 4; i++) {
int32x4_t bufA[64];
int32x4_t bufB[64];
- int16x8_t *buf = buf1 + width * i;
- for (int j = 0; j < width; ++j) {
+ int16x8_t *buf = buf1 + 64 * i;
+ for (int j = 0; j < 64; ++j) {
bufA[j] = vmovl_s16(vget_low_s16(buf[j]));
bufB[j] = vmovl_s16(vget_high_s16(buf[j]));
}
- av1_fdct64_new_neon(bufA, bufA, cos_bit_row, 1, 1, NULL);
- av1_fdct64_new_neon(bufB, bufB, cos_bit_row, 1, 1, NULL);
- av1_round_shift_array_32_neon(bufA, bufA, 32);
- av1_round_shift_array_32_neon(bufB, bufB, 32);
-
- store_output_32bit_w8(output + i * 8, bufA, bufB, 32, 32);
+ fdct64_new_neon(bufA, bufA, 10);
+ fdct64_new_neon(bufB, bufB, 10);
+ shift_right_2_round_s32_x4(bufA, bufA, 32);
+ shift_right_2_round_s32_x4(bufB, bufB, 32);
+ store_buffer_interleaved_s32_x8(output + i * 8, bufA, bufB, 32, 32);
}
}
-static void av1_lowbd_fwd_txfm2d_64x32_neon(const int16_t *input,
- int32_t *output, int stride,
- TX_TYPE tx_type, int bd) {
- (void)bd;
- const TX_SIZE tx_size = TX_64X32;
- int16x8_t buf0[64], buf1[256];
- const int8_t *shift = av1_fwd_txfm_shift_ls[tx_size];
- const int txw_idx = get_txw_idx(tx_size);
- const int txh_idx = get_txh_idx(tx_size);
- const int cos_bit_col = av1_fwd_cos_bit_col[txw_idx][txh_idx];
- const int cos_bit_row = av1_fwd_cos_bit_row[txw_idx][txh_idx];
- const int width = tx_size_wide[tx_size];
- const int height = tx_size_high[tx_size];
- const transform_1d_lbd_neon col_txfm = col_txfm8x32_arr[tx_type];
- const int width_div8 = (width >> 3);
- const int height_div8 = (height >> 3);
- for (int i = 0; i < width_div8; i++) {
- load_buffer_16bit_to_16bit(input + 8 * i, stride, buf0, height);
- round_shift_16bit(buf0, height, shift[0]);
- col_txfm(buf0, buf0, cos_bit_col, NULL);
- round_shift_16bit(buf0, height, shift[1]);
- for (int j = 0; j < AOMMIN(4, height_div8); ++j) {
- transpose_16bit_8x8(buf0 + j * 8, buf1 + j * width + 8 * i);
+static void lowbd_fwd_txfm2d_64x32_neon(const int16_t *input, int32_t *output,
+ int stride, TX_TYPE tx_type, int bd) {
+ (void)bd;
+ int16x8_t buf0[64], buf1[256];
+ const col_transform_1d_lbd_8_neon col_txfm = col_txfm8x32_arr[tx_type];
+
+ for (int i = 0; i < 8; i++) {
+ col_txfm(input + 8 * i, buf0, stride, 12);
+ shift_right_4_round_s16_x8(buf0, buf0, 32);
+ for (int j = 0; j < 4; ++j) {
+ transpose_arrays_s16_8x8(buf0 + j * 8, buf1 + j * 64 + 8 * i);
}
}
assert(tx_type == DCT_DCT);
- for (int i = 0; i < AOMMIN(4, height_div8); i++) {
+ for (int i = 0; i < 4; i++) {
int32x4_t bufA[64];
int32x4_t bufB[64];
- int16x8_t *buf = buf1 + width * i;
- for (int j = 0; j < width; ++j) {
+ int16x8_t *buf = buf1 + 64 * i;
+ for (int j = 0; j < 64; ++j) {
bufA[j] = vmovl_s16(vget_low_s16(buf[j]));
bufB[j] = vmovl_s16(vget_high_s16(buf[j]));
}
- av1_fdct64_new_neon(bufA, bufA, cos_bit_row, 1, 1, NULL);
- av1_fdct64_new_neon(bufB, bufB, cos_bit_row, 1, 1, NULL);
- av1_round_shift_rect_array_32_neon(bufA, bufA, 32);
- av1_round_shift_rect_array_32_neon(bufB, bufB, 32);
-
- store_output_32bit_w8(output + i * 8, bufA, bufB, 32, 32);
+ fdct64_new_neon(bufA, bufA, 11);
+ fdct64_new_neon(bufB, bufB, 11);
+ shift_right_2_round_s32_x4(bufA, bufA, 32);
+ shift_right_2_round_s32_x4(bufB, bufB, 32);
+ round_shift_sqrt2_s32_s32_4xn_neon(bufA, bufA, 32);
+ round_shift_sqrt2_s32_s32_4xn_neon(bufB, bufB, 32);
+ store_buffer_interleaved_s32_x8(output + i * 8, bufA, bufB, 32, 32);
}
}
-static void av1_lowbd_fwd_txfm2d_32x64_neon(const int16_t *input,
- int32_t *output, int stride,
- TX_TYPE tx_type, int bd) {
+static void lowbd_fwd_txfm2d_32x64_neon(const int16_t *input, int32_t *output,
+ int stride, TX_TYPE tx_type, int bd) {
(void)bd;
(void)tx_type;
assert(tx_type == DCT_DCT);
- const TX_SIZE tx_size = TX_32X64;
int16x8_t buf0[64], buf1[256];
- const int8_t *shift = av1_fwd_txfm_shift_ls[tx_size];
- const int txw_idx = get_txw_idx(tx_size);
- const int txh_idx = get_txh_idx(tx_size);
- const int cos_bit_col = av1_fwd_cos_bit_col[txw_idx][txh_idx];
- const int cos_bit_row = av1_fwd_cos_bit_row[txw_idx][txh_idx];
- const int width = tx_size_wide[tx_size];
- const int height = tx_size_high[tx_size];
- const transform_1d_lbd_neon col_txfm = av1_fdct8x64_neon;
- const int width_div8 = (width >> 3);
- const int height_div8 = (height >> 3);
+ const transform_1d_lbd_8_neon col_txfm = fdct8x64_neon;
- for (int i = 0; i < width_div8; i++) {
- load_buffer_16bit_to_16bit(input + 8 * i, stride, buf0, height);
- round_shift_16bit(buf0, height, shift[0]);
- col_txfm(buf0, buf0, cos_bit_col, NULL);
- round_shift_16bit(buf0, height, shift[1]);
- for (int j = 0; j < AOMMIN(4, height_div8); ++j) {
- transpose_16bit_8x8(buf0 + j * 8, buf1 + j * width + 8 * i);
+ for (int i = 0; i < 4; i++) {
+ load_buffer_s16_x8(input + 8 * i, stride, buf0, 64);
+ col_txfm(buf0, buf0, 13);
+ shift_right_2_round_s16_x8(buf0, buf0, 64);
+ for (int j = 0; j < 4; ++j) {
+ transpose_arrays_s16_8x8(buf0 + j * 8, buf1 + j * 32 + 8 * i);
}
}
- for (int i = 0; i < AOMMIN(4, height_div8); i++) {
+ for (int i = 0; i < 4; i++) {
int32x4_t bufA[32];
int32x4_t bufB[32];
- int16x8_t *buf = buf1 + width * i;
- for (int j = 0; j < width; ++j) {
+ int16x8_t *buf = buf1 + 32 * i;
+ for (int j = 0; j < 32; ++j) {
bufA[j] = vmovl_s16(vget_low_s16(buf[j]));
bufB[j] = vmovl_s16(vget_high_s16(buf[j]));
}
- av1_fdct32_new_neon(bufA, bufA, cos_bit_row, 1, NULL);
- av1_fdct32_new_neon(bufB, bufB, cos_bit_row, 1, NULL);
- av1_round_shift_rect_array_32_neon(bufA, bufA, 32);
- av1_round_shift_rect_array_32_neon(bufB, bufB, 32);
-
- store_output_32bit_w8(output + i * 8, bufA, bufB, 32, 32);
+ fdct32_new_neon(bufA, bufA, 11);
+ fdct32_new_neon(bufB, bufB, 11);
+ shift_right_2_round_s32_x4(bufA, bufA, 32);
+ shift_right_2_round_s32_x4(bufB, bufB, 32);
+ round_shift_sqrt2_s32_s32_4xn_neon(bufA, bufA, 32);
+ round_shift_sqrt2_s32_s32_4xn_neon(bufB, bufB, 32);
+ store_buffer_interleaved_s32_x8(output + i * 8, bufA, bufB, 32, 32);
}
}
static FwdTxfm2dFunc lowbd_fwd_txfm_func_ls[TX_SIZES_ALL] = {
- av1_lowbd_fwd_txfm2d_4x4_neon, // 4x4 transform
- av1_lowbd_fwd_txfm2d_8x8_neon, // 8x8 transform
- av1_lowbd_fwd_txfm2d_16x16_neon, // 16x16 transform
- av1_lowbd_fwd_txfm2d_32x32_neon, // 32x32 transform
- av1_lowbd_fwd_txfm2d_64x64_neon, // 64x64 transform
- av1_lowbd_fwd_txfm2d_4x8_neon, // 4x8 transform
- av1_lowbd_fwd_txfm2d_8x4_neon, // 8x4 transform
- av1_lowbd_fwd_txfm2d_8x16_neon, // 8x16 transform
- av1_lowbd_fwd_txfm2d_16x8_neon, // 16x8 transform
- av1_lowbd_fwd_txfm2d_16x32_neon, // 16x32 transform
- av1_lowbd_fwd_txfm2d_32x16_neon, // 32x16 transform
- av1_lowbd_fwd_txfm2d_32x64_neon, // 32x64 transform
- av1_lowbd_fwd_txfm2d_64x32_neon, // 64x32 transform
- av1_lowbd_fwd_txfm2d_4x16_neon, // 4x16 transform
- av1_lowbd_fwd_txfm2d_16x4_neon, // 16x4 transform
- av1_lowbd_fwd_txfm2d_8x32_neon, // 8x32 transform
- av1_lowbd_fwd_txfm2d_32x8_neon, // 32x8 transform
- av1_lowbd_fwd_txfm2d_16x64_neon, // 16x64 transform
- av1_lowbd_fwd_txfm2d_64x16_neon, // 64x16 transform
+ lowbd_fwd_txfm2d_4x4_neon, // 4x4 transform
+ lowbd_fwd_txfm2d_8x8_neon, // 8x8 transform
+ lowbd_fwd_txfm2d_16x16_neon, // 16x16 transform
+ lowbd_fwd_txfm2d_32x32_neon, // 32x32 transform
+ lowbd_fwd_txfm2d_64x64_neon, // 64x64 transform
+ lowbd_fwd_txfm2d_4x8_neon, // 4x8 transform
+ lowbd_fwd_txfm2d_8x4_neon, // 8x4 transform
+ lowbd_fwd_txfm2d_8x16_neon, // 8x16 transform
+ lowbd_fwd_txfm2d_16x8_neon, // 16x8 transform
+ lowbd_fwd_txfm2d_16x32_neon, // 16x32 transform
+ lowbd_fwd_txfm2d_32x16_neon, // 32x16 transform
+ lowbd_fwd_txfm2d_32x64_neon, // 32x64 transform
+ lowbd_fwd_txfm2d_64x32_neon, // 64x32 transform
+ lowbd_fwd_txfm2d_4x16_neon, // 4x16 transform
+ lowbd_fwd_txfm2d_16x4_neon, // 16x4 transform
+ lowbd_fwd_txfm2d_8x32_neon, // 8x32 transform
+ lowbd_fwd_txfm2d_32x8_neon, // 32x8 transform
+ lowbd_fwd_txfm2d_16x64_neon, // 16x64 transform
+ lowbd_fwd_txfm2d_64x16_neon, // 64x16 transform
};
void av1_lowbd_fwd_txfm_neon(const int16_t *src_diff, tran_low_t *coeff,
diff --git a/av1/encoder/arm/neon/encodetxb_neon.c b/av1/encoder/arm/neon/encodetxb_neon.c
index ee93608..582863a 100644
--- a/av1/encoder/arm/neon/encodetxb_neon.c
+++ b/av1/encoder/arm/neon/encodetxb_neon.c
@@ -57,10 +57,7 @@
} while (i < width);
} else if (height == 8) {
do {
- const int32x4_t coeffA = vld1q_s32(cf);
- const int32x4_t coeffB = vld1q_s32(cf + 4);
- const int16x8_t coeffAB =
- vcombine_s16(vqmovn_s32(coeffA), vqmovn_s32(coeffB));
+ const int16x8_t coeffAB = load_tran_low_to_s16q(cf);
const int16x8_t absAB = vqabsq_s16(coeffAB);
const uint8x16_t absAB8 = vreinterpretq_u8_s8(vcombine_s8(
vqmovn_s16(absAB), vreinterpret_s8_s32(vget_low_s32(zeros))));
@@ -73,14 +70,8 @@
do {
int j = 0;
do {
- const int32x4_t coeffA = vld1q_s32(cf);
- const int32x4_t coeffB = vld1q_s32(cf + 4);
- const int32x4_t coeffC = vld1q_s32(cf + 8);
- const int32x4_t coeffD = vld1q_s32(cf + 12);
- const int16x8_t coeffAB =
- vcombine_s16(vqmovn_s32(coeffA), vqmovn_s32(coeffB));
- const int16x8_t coeffCD =
- vcombine_s16(vqmovn_s32(coeffC), vqmovn_s32(coeffD));
+ const int16x8_t coeffAB = load_tran_low_to_s16q(cf);
+ const int16x8_t coeffCD = load_tran_low_to_s16q(cf + 8);
const int16x8_t absAB = vqabsq_s16(coeffAB);
const int16x8_t absCD = vqabsq_s16(coeffCD);
const uint8x16_t absABCD = vreinterpretq_u8_s8(
@@ -282,7 +273,7 @@
const uint8x16_t pos_to_offset_large = vdupq_n_u8(21);
uint8x16_t pos_to_offset =
- vld1q_u8((width == 4) ? c_4_po_2d[0] : c_4_po_2d[1]);
+ (width == 4) ? vld1q_u8(c_4_po_2d[0]) : vld1q_u8(c_4_po_2d[1]);
uint8x16_t count;
uint8x16_t level[5];
diff --git a/av1/encoder/arm/neon/highbd_fwd_txfm_neon.c b/av1/encoder/arm/neon/highbd_fwd_txfm_neon.c
index 15d375a..aa64a38 100644
--- a/av1/encoder/arm/neon/highbd_fwd_txfm_neon.c
+++ b/av1/encoder/arm/neon/highbd_fwd_txfm_neon.c
@@ -12,2112 +12,1586 @@
#include <arm_neon.h>
#include <assert.h>
-#include "av1/common/av1_txfm.h"
-#include "av1/encoder/av1_fwd_txfm1d_cfg.h"
+#include "aom_dsp/arm/transpose_neon.h"
#include "aom_dsp/txfm_common.h"
#include "aom_ports/mem.h"
-#include "config/av1_rtcd.h"
+#include "av1/common/av1_txfm.h"
+#include "av1/encoder/av1_fwd_txfm1d_cfg.h"
#include "config/aom_config.h"
+#include "config/av1_rtcd.h"
+#include "shift_neon.h"
+#include "txfm_neon.h"
-static INLINE void store_output_w4(int32_t *const out,
- const int32x4_t *const in, const int stride,
- const int out_size) {
- for (int i = 0; i < out_size; ++i) {
- vst1q_s32(out + i * stride, in[i]);
+static AOM_FORCE_INLINE void transpose_arrays_s32_64x64(const int32x4_t *in,
+ int32x4_t *out) {
+ // This is not quite the same as the other transposes defined in
+ // transpose_neon.h: We only write the low 64x32 sub-matrix since the rest is
+ // unused by the following row transform.
+ for (int j = 0; j < 8; ++j) {
+ for (int i = 0; i < 16; ++i) {
+ transpose_arrays_s32_4x4(in + 64 * i + 4 * j, out + 64 * j + 4 * i);
+ }
}
}
-static INLINE int32x4_t half_btf_neon(const int32_t *w0, const int32x4_t *n0,
- const int32_t *w1, const int32x4_t *n1,
- const int32x4_t v_bit) {
- int32x4_t x;
- x = vmulq_n_s32(*n0, *w0);
- x = vmlaq_n_s32(x, *n1, *w1);
- x = vrshlq_s32(x, v_bit);
- return x;
+// A note on butterfly helper naming:
+//
+// butterfly_[weight_indices]_neon
+// e.g. butterfly_0312_neon
+// ^ Weights are applied as indices 0, 3, 2, 1
+// (see more detail below)
+//
+// Weight indices are treated as an index into the 4-tuple of the weight
+// itself, plus related and negated constants: w=(w0, 1-w0, -w0, w0-1).
+// This is then represented in the helper naming by referring to the lane index
+// in the loaded tuple that each multiply is performed with:
+//
+// in0 in1
+// /------------
+// out0 | w[0] w[1] ==> out0 = in0 * w[0] + in1 * w[1]
+// out1 | w[2] w[3] ==> out1 = in0 * w[2] + in1 * w[3]
+//
+// So for indices 0321 from the earlier example, we end up with:
+//
+// in0 in1
+// /------------------
+// out0 | (lane 0) (lane 3) ==> out0 = in0 * w0 + in1 * (w0-1)
+// out1 | (lane 2) (lane 1) ==> out1 = in0 * -w0 + in1 * (1-w0)
+
+#define butterfly_half_neon(wvec, lane0, lane1, in0, in1, out, v_bit) \
+ do { \
+ int32x2x2_t wvecs = { { wvec, vneg_s32(wvec) } }; \
+ int32x4_t x = vmulq_lane_s32(in0, wvecs.val[lane0 / 2], lane0 % 2); \
+ x = vmlaq_lane_s32(x, in1, wvecs.val[lane1 / 2], lane1 % 2); \
+ *out = vrshlq_s32(x, v_bit); \
+ } while (false)
+
+static AOM_FORCE_INLINE void butterfly_0112_neon(
+ const int32_t *cospi, const int widx0, const int32x4_t n0,
+ const int32x4_t n1, int32x4_t *out0, int32x4_t *out1,
+ const int32x4_t v_bit) {
+ int32x2_t w01 = vld1_s32(cospi + 2 * widx0);
+ butterfly_half_neon(w01, 0, 1, n0, n1, out0, v_bit);
+ butterfly_half_neon(w01, 1, 2, n0, n1, out1, v_bit);
}
-static INLINE int32x4_t half_btf_neon_m(const int32_t *w0, const int32x4_t *n0,
- const int32_t *w1, const int32x4_t *n1,
- const int32x4_t v_bit) {
- int32x4_t x;
- x = vmulq_n_s32(*n0, *w0);
- x = vmlsq_n_s32(x, *n1, *w1);
- x = vrshlq_s32(x, v_bit);
- return x;
+static AOM_FORCE_INLINE void butterfly_2312_neon(
+ const int32_t *cospi, const int widx0, const int32x4_t n0,
+ const int32x4_t n1, int32x4_t *out0, int32x4_t *out1,
+ const int32x4_t v_bit) {
+ int32x2_t w01 = vld1_s32(cospi + 2 * widx0);
+ butterfly_half_neon(w01, 2, 3, n0, n1, out0, v_bit);
+ butterfly_half_neon(w01, 1, 2, n0, n1, out1, v_bit);
}
-#if AOM_ARCH_AARCH64
-#define TRANSPOSE_4X4(x0, x1, x2, x3, y0, y1, y2, y3) \
- do { \
- int32x4x2_t swap_low = vtrnq_s32(x0, x1); \
- int32x4x2_t swap_high = vtrnq_s32(x2, x3); \
- y0 = vreinterpretq_s32_s64( \
- vzip1q_s64(vreinterpretq_s64_s32(swap_low.val[0]), \
- vreinterpretq_s64_s32(swap_high.val[0]))); \
- y1 = vreinterpretq_s32_s64( \
- vzip1q_s64(vreinterpretq_s64_s32(swap_low.val[1]), \
- vreinterpretq_s64_s32(swap_high.val[1]))); \
- y2 = vreinterpretq_s32_s64( \
- vzip2q_s64(vreinterpretq_s64_s32(swap_low.val[0]), \
- vreinterpretq_s64_s32(swap_high.val[0]))); \
- y3 = vreinterpretq_s32_s64( \
- vzip2q_s64(vreinterpretq_s64_s32(swap_low.val[1]), \
- vreinterpretq_s64_s32(swap_high.val[1]))); \
- } while (0)
-#else
-#define TRANSPOSE_4X4(x0, x1, x2, x3, y0, y1, y2, y3) \
- do { \
- int32x4x2_t swap_low = vtrnq_s32(x0, x1); \
- int32x4x2_t swap_high = vtrnq_s32(x2, x3); \
- y0 = vextq_s32(vextq_s32(swap_low.val[0], swap_low.val[0], 2), \
- swap_high.val[0], 2); \
- y1 = vextq_s32(vextq_s32(swap_low.val[1], swap_low.val[1], 2), \
- swap_high.val[1], 2); \
- y2 = vextq_s32(swap_low.val[0], \
- vextq_s32(swap_high.val[0], swap_high.val[0], 2), 2); \
- y3 = vextq_s32(swap_low.val[1], \
- vextq_s32(swap_high.val[1], swap_high.val[1], 2), 2); \
- } while (0)
-#endif // AOM_ARCH_AARCH64
-
-static INLINE void transpose_4x4(const int32x4_t *in, int32x4_t *out) {
- TRANSPOSE_4X4(in[0], in[1], in[2], in[3], out[0], out[1], out[2], out[3]);
+static AOM_FORCE_INLINE void butterfly_0332_neon(
+ const int32_t *cospi, const int widx0, const int32x4_t n0,
+ const int32x4_t n1, int32x4_t *out0, int32x4_t *out1,
+ const int32x4_t v_bit) {
+ int32x2_t w01 = vld1_s32(cospi + 2 * widx0);
+ butterfly_half_neon(w01, 0, 3, n0, n1, out0, v_bit);
+ butterfly_half_neon(w01, 3, 2, n0, n1, out1, v_bit);
}
-static INLINE void transpose_8x8(const int32x4_t *in, int32x4_t *out) {
- TRANSPOSE_4X4(in[0], in[2], in[4], in[6], out[0], out[2], out[4], out[6]);
- TRANSPOSE_4X4(in[1], in[3], in[5], in[7], out[8], out[10], out[12], out[14]);
- TRANSPOSE_4X4(in[8], in[10], in[12], in[14], out[1], out[3], out[5], out[7]);
- TRANSPOSE_4X4(in[9], in[11], in[13], in[15], out[9], out[11], out[13],
- out[15]);
+static AOM_FORCE_INLINE void butterfly_0130_neon(
+ const int32_t *cospi, const int widx0, const int32x4_t n0,
+ const int32x4_t n1, int32x4_t *out0, int32x4_t *out1,
+ const int32x4_t v_bit) {
+ int32x2_t w01 = vld1_s32(cospi + 2 * widx0);
+ butterfly_half_neon(w01, 0, 1, n0, n1, out0, v_bit);
+ butterfly_half_neon(w01, 3, 0, n0, n1, out1, v_bit);
}
-static INLINE void transpose_16x16(const int32x4_t *in, int32x4_t *out) {
- // Upper left 8x8
- TRANSPOSE_4X4(in[0], in[4], in[8], in[12], out[0], out[4], out[8], out[12]);
- TRANSPOSE_4X4(in[1], in[5], in[9], in[13], out[16], out[20], out[24],
- out[28]);
- TRANSPOSE_4X4(in[16], in[20], in[24], in[28], out[1], out[5], out[9],
- out[13]);
- TRANSPOSE_4X4(in[17], in[21], in[25], in[29], out[17], out[21], out[25],
- out[29]);
-
- // Upper right 8x8
- TRANSPOSE_4X4(in[2], in[6], in[10], in[14], out[32], out[36], out[40],
- out[44]);
- TRANSPOSE_4X4(in[3], in[7], in[11], in[15], out[48], out[52], out[56],
- out[60]);
- TRANSPOSE_4X4(in[18], in[22], in[26], in[30], out[33], out[37], out[41],
- out[45]);
- TRANSPOSE_4X4(in[19], in[23], in[27], in[31], out[49], out[53], out[57],
- out[61]);
-
- // Lower left 8x8
- TRANSPOSE_4X4(in[32], in[36], in[40], in[44], out[2], out[6], out[10],
- out[14]);
- TRANSPOSE_4X4(in[33], in[37], in[41], in[45], out[18], out[22], out[26],
- out[30]);
- TRANSPOSE_4X4(in[48], in[52], in[56], in[60], out[3], out[7], out[11],
- out[15]);
- TRANSPOSE_4X4(in[49], in[53], in[57], in[61], out[19], out[23], out[27],
- out[31]);
- // Lower right 8x8
- TRANSPOSE_4X4(in[34], in[38], in[42], in[46], out[34], out[38], out[42],
- out[46]);
- TRANSPOSE_4X4(in[35], in[39], in[43], in[47], out[50], out[54], out[58],
- out[62]);
- TRANSPOSE_4X4(in[50], in[54], in[58], in[62], out[35], out[39], out[43],
- out[47]);
- TRANSPOSE_4X4(in[51], in[55], in[59], in[63], out[51], out[55], out[59],
- out[63]);
+static AOM_FORCE_INLINE void butterfly_cospi32_0002_neon(
+ const int32_t *cospi, const int32x4_t n0, const int32x4_t n1,
+ int32x4_t *out0, int32x4_t *out1, const int32x4_t v_bit) {
+ int32x2_t w01 = vld1_s32(cospi + 2 * 32);
+ butterfly_half_neon(w01, 0, 0, n0, n1, out0, v_bit);
+ butterfly_half_neon(w01, 0, 2, n0, n1, out1, v_bit);
}
-static INLINE void av1_round_shift_rect_array_32_neon(int32x4_t *input,
- int32x4_t *output,
- const int size,
- const int bit,
- const int val) {
- const int32x4_t sqrt2 = vdupq_n_s32(val);
- const int32x4_t v_bit = vdupq_n_s32(-bit);
- int i;
- for (i = 0; i < size; i++) {
- const int32x4_t r0 = vrshlq_s32(input[i], v_bit);
- const int32x4_t r1 = vmulq_s32(sqrt2, r0);
+static AOM_FORCE_INLINE void butterfly_cospi32_0222_neon(
+ const int32_t *cospi, const int32x4_t n0, const int32x4_t n1,
+ int32x4_t *out0, int32x4_t *out1, const int32x4_t v_bit) {
+ int32x2_t w01 = vld1_s32(cospi + 2 * 32);
+ butterfly_half_neon(w01, 0, 2, n0, n1, out0, v_bit);
+ butterfly_half_neon(w01, 2, 2, n0, n1, out1, v_bit);
+}
+
+static AOM_FORCE_INLINE void round_rect_array_s32_neon(const int32x4_t *input,
+ int32x4_t *output,
+ const int size) {
+ const int32x4_t sqrt2 = vdupq_n_s32(NewSqrt2);
+ int i = 0;
+ do {
+ const int32x4_t r1 = vmulq_s32(input[i], sqrt2);
output[i] = vrshrq_n_s32(r1, NewSqrt2Bits);
- }
+ } while (++i < size);
}
-#define btf_32_neon_type0(w0, w1, in0, in1, out0, out1, v_cos_bit) \
- do { \
- out0 = vmulq_n_s32(in0, w0); \
- out0 = vmlaq_n_s32(out0, in1, w1); \
- out0 = vrshlq_s32(out0, v_cos_bit); \
- out1 = vmulq_n_s32(in0, w1); \
- out1 = vmlsq_n_s32(out1, in1, w0); \
- out1 = vrshlq_s32(out1, v_cos_bit); \
- } while (0)
+static AOM_FORCE_INLINE void round_shift2_rect_array_s32_neon(
+ const int32x4_t *input, int32x4_t *output, const int size) {
+ const int32x4_t sqrt2 = vdupq_n_s32(NewSqrt2);
+ int i = 0;
+ do {
+ const int32x4_t r0 = vrshrq_n_s32(input[i], 2);
+ const int32x4_t r1 = vmulq_s32(r0, sqrt2);
+ output[i] = vrshrq_n_s32(r1, NewSqrt2Bits);
+ } while (++i < size);
+}
-#define btf_32_neon_type1(w0, w1, in0, in1, out0, out1, bit) \
- do { \
- btf_32_neon_type0(w1, w0, in1, in0, out0, out1, bit); \
- } while (0)
-
-static INLINE void load_buffer_4x4(const int16_t *input, int32x4_t *in,
- int stride, int flipud, int fliplr,
- const int32x4_t *v_shift) {
- int16x4_t v0, v1, v2, v3;
-
- if (!flipud) {
- v0 = vld1_s16(input + 0 * stride);
- v1 = vld1_s16(input + 1 * stride);
- v2 = vld1_s16(input + 2 * stride);
- v3 = vld1_s16(input + 3 * stride);
- } else {
- v0 = vld1_s16(input + 3 * stride);
- v1 = vld1_s16(input + 2 * stride);
- v2 = vld1_s16(input + 1 * stride);
- v3 = vld1_s16(input + 0 * stride);
+#define LOAD_BUFFER_4XH(h) \
+ static AOM_FORCE_INLINE void load_buffer_4x##h( \
+ const int16_t *input, int32x4_t *in, int stride, int fliplr) { \
+ if (fliplr) { \
+ for (int i = 0; i < (h); ++i) { \
+ int16x4_t a = vld1_s16(input + i * stride); \
+ a = vrev64_s16(a); \
+ in[i] = vshll_n_s16(a, 2); \
+ } \
+ } else { \
+ for (int i = 0; i < (h); ++i) { \
+ int16x4_t a = vld1_s16(input + i * stride); \
+ in[i] = vshll_n_s16(a, 2); \
+ } \
+ } \
}
- if (fliplr) {
- v0 = vrev64_s16(v0);
- v1 = vrev64_s16(v1);
- v2 = vrev64_s16(v2);
- v3 = vrev64_s16(v3);
+// AArch32 does not permit the argument to vshll_n_s16 to be zero, so need to
+// avoid the expression even though the compiler can prove that the code path
+// is never taken if `shift == 0`.
+#define shift_left_long_s16(a, shift) \
+ ((shift) == 0 ? vmovl_s16(a) : vshll_n_s16((a), (shift) == 0 ? 1 : (shift)))
+
+#define LOAD_BUFFER_WXH(w, h, shift) \
+ static AOM_FORCE_INLINE void load_buffer_##w##x##h( \
+ const int16_t *input, int32x4_t *in, int stride, int fliplr) { \
+ assert(w >= 8); \
+ if (fliplr) { \
+ for (int i = 0; i < (h); ++i) { \
+ for (int j = 0; j < (w) / 8; ++j) { \
+ int16x8_t a = vld1q_s16(input + i * stride + j * 8); \
+ a = vrev64q_s16(a); \
+ int j2 = (w) / 8 - j - 1; \
+ in[i + (h) * (2 * j2 + 0)] = \
+ shift_left_long_s16(vget_high_s16(a), (shift)); \
+ in[i + (h) * (2 * j2 + 1)] = \
+ shift_left_long_s16(vget_low_s16(a), (shift)); \
+ } \
+ } \
+ } else { \
+ for (int i = 0; i < (h); ++i) { \
+ for (int j = 0; j < (w) / 8; ++j) { \
+ int16x8_t a = vld1q_s16(input + i * stride + j * 8); \
+ in[i + (h) * (2 * j + 0)] = \
+ shift_left_long_s16(vget_low_s16(a), (shift)); \
+ in[i + (h) * (2 * j + 1)] = \
+ shift_left_long_s16(vget_high_s16(a), (shift)); \
+ } \
+ } \
+ } \
}
- in[0] = vshlq_s32(vmovl_s16(v0), *v_shift);
- in[1] = vshlq_s32(vmovl_s16(v1), *v_shift);
- in[2] = vshlq_s32(vmovl_s16(v2), *v_shift);
- in[3] = vshlq_s32(vmovl_s16(v3), *v_shift);
-}
-static void fdct4x4_neon(int32x4_t *in, int32x4_t *out, int bit,
- const int num_col) {
- const int32_t *cospi = cospi_arr(bit);
- const int32x4_t cospi32 = vdupq_n_s32(cospi[32]);
- const int32x4_t cospi48 = vdupq_n_s32(cospi[48]);
- const int32x4_t cospi16 = vdupq_n_s32(cospi[16]);
- int32x4_t s0, s1, s2, s3;
- int32x4_t u0, u1, u2, u3;
- int32x4_t v0, v2;
+LOAD_BUFFER_4XH(4)
+LOAD_BUFFER_4XH(8)
+LOAD_BUFFER_4XH(16)
+LOAD_BUFFER_4XH(32)
+LOAD_BUFFER_WXH(8, 8, 2)
+LOAD_BUFFER_WXH(16, 16, 2)
+LOAD_BUFFER_WXH(32, 64, 0)
+LOAD_BUFFER_WXH(64, 32, 2)
+LOAD_BUFFER_WXH(64, 64, 0)
- int endidx = 3 * num_col;
- s0 = vaddq_s32(in[0], in[endidx]);
- s3 = vsubq_s32(in[0], in[endidx]);
- endidx -= num_col;
- s1 = vaddq_s32(in[num_col], in[endidx]);
- s2 = vsubq_s32(in[num_col], in[endidx]);
+#if !CONFIG_REALTIME_ONLY
+LOAD_BUFFER_WXH(16, 64, 0)
+LOAD_BUFFER_WXH(64, 16, 2)
+#endif // !CONFIG_REALTIME_ONLY
- u0 = vmulq_s32(s0, cospi32);
- u1 = vmulq_s32(s1, cospi32);
- u2 = vaddq_s32(u0, u1);
- v0 = vsubq_s32(u0, u1);
- const int32x4_t v_bit = vdupq_n_s32(-bit);
- u0 = vrshlq_s32(u2, v_bit);
- u2 = vrshlq_s32(v0, v_bit);
+#define STORE_BUFFER_WXH(w, h) \
+ static AOM_FORCE_INLINE void store_buffer_##w##x##h( \
+ const int32x4_t *in, int32_t *out, int stride) { \
+ for (int i = 0; i < (w); ++i) { \
+ for (int j = 0; j < (h) / 4; ++j) { \
+ vst1q_s32(&out[i * stride + j * 4], in[i + j * (w)]); \
+ } \
+ } \
+ }
- v0 = vmulq_s32(s2, cospi48);
- v2 = vmlaq_s32(v0, s3, cospi16);
+STORE_BUFFER_WXH(4, 4)
+STORE_BUFFER_WXH(8, 4)
+STORE_BUFFER_WXH(8, 8)
+STORE_BUFFER_WXH(16, 4)
+STORE_BUFFER_WXH(16, 16)
+STORE_BUFFER_WXH(32, 4)
+STORE_BUFFER_WXH(32, 32)
+STORE_BUFFER_WXH(64, 32)
- u1 = vrshlq_s32(v2, v_bit);
+#if !CONFIG_REALTIME_ONLY
+STORE_BUFFER_WXH(16, 32)
+STORE_BUFFER_WXH(64, 16)
+#endif // !CONFIG_REALTIME_ONLY
- v0 = vmulq_s32(s3, cospi48);
- v2 = vmlsq_s32(v0, s2, cospi16);
+static AOM_FORCE_INLINE void highbd_fdct4_x4_neon(const int32x4_t *in,
+ int32x4_t *out, int bit) {
+ const int32_t *const cospi = cospi_arr_s32(bit);
+ const int32x4_t cospi32 = vdupq_n_s32(cospi[2 * 32]);
+ const int32x2_t cospi16_48 = vld1_s32(&cospi[2 * 16]);
- u3 = vrshlq_s32(v2, v_bit);
+ const int32x4_t a0 = vaddq_s32(in[0], in[3]);
+ const int32x4_t a1 = vsubq_s32(in[0], in[3]);
+ const int32x4_t a2 = vaddq_s32(in[1], in[2]);
+ const int32x4_t a3 = vsubq_s32(in[1], in[2]);
- out[0] = u0;
- out[1] = u1;
- out[2] = u2;
- out[3] = u3;
-}
+ const int32x4_t b0 = vmulq_s32(a0, cospi32);
+ const int32x4_t b1 = vmulq_lane_s32(a1, cospi16_48, 1);
+ const int32x4_t b2 = vmulq_s32(a2, cospi32);
+ const int32x4_t b3 = vmulq_lane_s32(a3, cospi16_48, 1);
-static INLINE void write_buffer_4x4(int32x4_t *res, int32_t *output) {
- vst1q_s32((output + 0 * 4), res[0]);
- vst1q_s32((output + 1 * 4), res[1]);
- vst1q_s32((output + 2 * 4), res[2]);
- vst1q_s32((output + 3 * 4), res[3]);
-}
-
-static void fadst4x4_neon(int32x4_t *in, int32x4_t *out, int bit,
- const int num_col) {
- const int32_t *sinpi = sinpi_arr(bit);
- const int32x4_t sinpi4x = vld1q_s32(&sinpi[1]);
-
- const int32x4_t sinpi1 = vdupq_lane_s32(vget_low_s32(sinpi4x), 0);
- const int32x4_t sinpi2 = vdupq_lane_s32(vget_low_s32(sinpi4x), 1);
- const int32x4_t sinpi3 = vdupq_lane_s32(vget_high_s32(sinpi4x), 0);
- const int32x4_t sinpi4 = vdupq_lane_s32(vget_high_s32(sinpi4x), 1);
- int32x4_t t;
- int32x4_t s0, s1, s2, s3, s7;
- int32x4_t x0, x1, x2, x3;
-
- int idx = 0 * num_col;
- s0 = vmulq_s32(in[idx], sinpi1);
- s1 = vmulq_s32(in[idx], sinpi4);
- t = vaddq_s32(in[idx], in[idx + num_col]);
- idx += 2 * num_col;
- x3 = vmulq_s32(in[idx], sinpi3);
- idx += num_col;
- s7 = vsubq_s32(t, in[idx]);
-
- t = vmlaq_s32(s0, in[idx - 2 * num_col], sinpi2);
- x0 = vmlaq_s32(t, in[idx], sinpi4);
- x1 = vmulq_s32(s7, sinpi3);
- t = vmlsq_s32(s1, in[idx - 2 * num_col], sinpi1);
- x2 = vmlaq_s32(t, in[idx], sinpi2);
-
- s0 = vaddq_s32(x0, x3);
- s1 = x1;
- s2 = vsubq_s32(x2, x3);
- t = vsubq_s32(x2, x0);
- s3 = vaddq_s32(t, x3);
+ const int32x4_t c0 = vaddq_s32(b0, b2);
+ const int32x4_t c1 = vsubq_s32(b0, b2);
+ const int32x4_t c2 = vmlaq_lane_s32(b3, a1, cospi16_48, 0);
+ const int32x4_t c3 = vmlsq_lane_s32(b1, a3, cospi16_48, 0);
const int32x4_t v_bit = vdupq_n_s32(-bit);
- out[0] = vrshlq_s32(s0, v_bit);
- out[1] = vrshlq_s32(s1, v_bit);
- out[2] = vrshlq_s32(s2, v_bit);
- out[3] = vrshlq_s32(s3, v_bit);
+ const int32x4_t d0 = vrshlq_s32(c0, v_bit);
+ const int32x4_t d1 = vrshlq_s32(c1, v_bit);
+ const int32x4_t d2 = vrshlq_s32(c2, v_bit);
+ const int32x4_t d3 = vrshlq_s32(c3, v_bit);
+
+ out[0] = d0;
+ out[1] = d2;
+ out[2] = d1;
+ out[3] = d3;
}
-static void idtx4x4_neon(int32x4_t *in, int32x4_t *out, int bit, int col_num) {
+
+static AOM_FORCE_INLINE void highbd_fadst4_x4_neon(const int32x4_t *in,
+ int32x4_t *out, int bit) {
+ const int32x4_t sinpi = vld1q_s32(sinpi_arr(bit) + 1);
+
+ const int32x4_t a0 = vaddq_s32(in[0], in[1]);
+ const int32x4_t a1 = vmulq_lane_s32(in[0], vget_low_s32(sinpi), 0);
+ const int32x4_t a2 = vmulq_lane_s32(in[0], vget_high_s32(sinpi), 1);
+ const int32x4_t a3 = vmulq_lane_s32(in[2], vget_high_s32(sinpi), 0);
+
+ const int32x4_t b0 = vmlaq_lane_s32(a1, in[1], vget_low_s32(sinpi), 1);
+ const int32x4_t b1 = vmlsq_lane_s32(a2, in[1], vget_low_s32(sinpi), 0);
+ const int32x4_t b2 = vsubq_s32(a0, in[3]);
+
+ const int32x4_t c0 = vmlaq_lane_s32(b0, in[3], vget_high_s32(sinpi), 1);
+ const int32x4_t c1 = vmlaq_lane_s32(b1, in[3], vget_low_s32(sinpi), 1);
+ const int32x4_t c2 = vmulq_lane_s32(b2, vget_high_s32(sinpi), 0);
+
+ const int32x4_t d0 = vaddq_s32(c0, a3);
+ const int32x4_t d1 = vsubq_s32(c1, a3);
+ const int32x4_t d2 = vsubq_s32(c1, c0);
+
+ const int32x4_t e0 = vaddq_s32(d2, a3);
+
+ const int32x4_t v_bit = vdupq_n_s32(-bit);
+ out[0] = vrshlq_s32(d0, v_bit);
+ out[1] = vrshlq_s32(c2, v_bit);
+ out[2] = vrshlq_s32(d1, v_bit);
+ out[3] = vrshlq_s32(e0, v_bit);
+}
+
+static AOM_FORCE_INLINE void highbd_fidentity4_x4_neon(const int32x4_t *in,
+ int32x4_t *out,
+ int bit) {
(void)bit;
int32x4_t fact = vdupq_n_s32(NewSqrt2);
- int32x4_t a_low;
- int i;
- for (i = 0; i < 4; i++) {
- a_low = vmulq_s32(in[i * col_num], fact);
+ for (int i = 0; i < 4; i++) {
+ const int32x4_t a_low = vmulq_s32(in[i], fact);
out[i] = vrshrq_n_s32(a_low, NewSqrt2Bits);
}
}
+
void av1_fwd_txfm2d_4x4_neon(const int16_t *input, int32_t *coeff,
int input_stride, TX_TYPE tx_type, int bd) {
- int32x4_t in[4];
- const int8_t *shift = av1_fwd_txfm_shift_ls[TX_4X4];
- const int txw_idx = get_txw_idx(TX_4X4);
- const int txh_idx = get_txh_idx(TX_4X4);
- int32x4_t v_shift0 = vdupq_n_s32(shift[0]);
+ (void)bd;
+
+ int ud_flip, lr_flip;
+ get_flip_cfg(tx_type, &ud_flip, &lr_flip);
+ ud_adjust_input_and_stride(ud_flip, &input, &input_stride, 4);
+
+ // Workspace for column/row-wise transforms.
+ int32x4_t buf[4];
+
switch (tx_type) {
case DCT_DCT:
- load_buffer_4x4(input, in, input_stride, 0, 0, &v_shift0);
- fdct4x4_neon(in, in, av1_fwd_cos_bit_col[txw_idx][txh_idx], 1);
- transpose_4x4(in, in);
- fdct4x4_neon(in, in, av1_fwd_cos_bit_row[txw_idx][txh_idx], 1);
- write_buffer_4x4(in, coeff);
+ load_buffer_4x4(input, buf, input_stride, 0);
+ highbd_fdct4_x4_neon(buf, buf, av1_fwd_cos_bit_col[0][0]);
+ transpose_arrays_s32_4x4(buf, buf);
+ highbd_fdct4_x4_neon(buf, buf, av1_fwd_cos_bit_row[0][0]);
+ store_buffer_4x4(buf, coeff, /*stride=*/4);
break;
case ADST_DCT:
- load_buffer_4x4(input, in, input_stride, 0, 0, &v_shift0);
- fadst4x4_neon(in, in, av1_fwd_cos_bit_col[txw_idx][txh_idx], 1);
- transpose_4x4(in, in);
- fdct4x4_neon(in, in, av1_fwd_cos_bit_row[txw_idx][txh_idx], 1);
- write_buffer_4x4(in, coeff);
+ load_buffer_4x4(input, buf, input_stride, 0);
+ highbd_fadst4_x4_neon(buf, buf, av1_fwd_cos_bit_col[0][0]);
+ transpose_arrays_s32_4x4(buf, buf);
+ highbd_fdct4_x4_neon(buf, buf, av1_fwd_cos_bit_row[0][0]);
+ store_buffer_4x4(buf, coeff, /*stride=*/4);
break;
case DCT_ADST:
- load_buffer_4x4(input, in, input_stride, 0, 0, &v_shift0);
- fdct4x4_neon(in, in, av1_fwd_cos_bit_col[txw_idx][txh_idx], 1);
- transpose_4x4(in, in);
- fadst4x4_neon(in, in, av1_fwd_cos_bit_row[txw_idx][txh_idx], 1);
- write_buffer_4x4(in, coeff);
+ load_buffer_4x4(input, buf, input_stride, 0);
+ highbd_fdct4_x4_neon(buf, buf, av1_fwd_cos_bit_col[0][0]);
+ transpose_arrays_s32_4x4(buf, buf);
+ highbd_fadst4_x4_neon(buf, buf, av1_fwd_cos_bit_row[0][0]);
+ store_buffer_4x4(buf, coeff, /*stride=*/4);
break;
case ADST_ADST:
- load_buffer_4x4(input, in, input_stride, 0, 0, &v_shift0);
- fadst4x4_neon(in, in, av1_fwd_cos_bit_col[txw_idx][txh_idx], 1);
- transpose_4x4(in, in);
- fadst4x4_neon(in, in, av1_fwd_cos_bit_row[txw_idx][txh_idx], 1);
- write_buffer_4x4(in, coeff);
+ load_buffer_4x4(input, buf, input_stride, 0);
+ highbd_fadst4_x4_neon(buf, buf, av1_fwd_cos_bit_col[0][0]);
+ transpose_arrays_s32_4x4(buf, buf);
+ highbd_fadst4_x4_neon(buf, buf, av1_fwd_cos_bit_row[0][0]);
+ store_buffer_4x4(buf, coeff, /*stride=*/4);
break;
case FLIPADST_DCT:
- load_buffer_4x4(input, in, input_stride, 1, 0, &v_shift0);
- fadst4x4_neon(in, in, av1_fwd_cos_bit_col[txw_idx][txh_idx], 1);
- transpose_4x4(in, in);
- fdct4x4_neon(in, in, av1_fwd_cos_bit_row[txw_idx][txh_idx], 1);
- write_buffer_4x4(in, coeff);
+ load_buffer_4x4(input, buf, input_stride, 0);
+ highbd_fadst4_x4_neon(buf, buf, av1_fwd_cos_bit_col[0][0]);
+ transpose_arrays_s32_4x4(buf, buf);
+ highbd_fdct4_x4_neon(buf, buf, av1_fwd_cos_bit_row[0][0]);
+ store_buffer_4x4(buf, coeff, /*stride=*/4);
break;
case DCT_FLIPADST:
- load_buffer_4x4(input, in, input_stride, 0, 1, &v_shift0);
- fdct4x4_neon(in, in, av1_fwd_cos_bit_col[txw_idx][txh_idx], 1);
- transpose_4x4(in, in);
- fadst4x4_neon(in, in, av1_fwd_cos_bit_row[txw_idx][txh_idx], 1);
- write_buffer_4x4(in, coeff);
+ load_buffer_4x4(input, buf, input_stride, 1);
+ highbd_fdct4_x4_neon(buf, buf, av1_fwd_cos_bit_col[0][0]);
+ transpose_arrays_s32_4x4(buf, buf);
+ highbd_fadst4_x4_neon(buf, buf, av1_fwd_cos_bit_row[0][0]);
+ store_buffer_4x4(buf, coeff, /*stride=*/4);
break;
case FLIPADST_FLIPADST:
- load_buffer_4x4(input, in, input_stride, 1, 1, &v_shift0);
- fadst4x4_neon(in, in, av1_fwd_cos_bit_col[txw_idx][txh_idx], 1);
- transpose_4x4(in, in);
- fadst4x4_neon(in, in, av1_fwd_cos_bit_row[txw_idx][txh_idx], 1);
- write_buffer_4x4(in, coeff);
+ load_buffer_4x4(input, buf, input_stride, 1);
+ highbd_fadst4_x4_neon(buf, buf, av1_fwd_cos_bit_col[0][0]);
+ transpose_arrays_s32_4x4(buf, buf);
+ highbd_fadst4_x4_neon(buf, buf, av1_fwd_cos_bit_row[0][0]);
+ store_buffer_4x4(buf, coeff, /*stride=*/4);
break;
case ADST_FLIPADST:
- load_buffer_4x4(input, in, input_stride, 0, 1, &v_shift0);
- fadst4x4_neon(in, in, av1_fwd_cos_bit_col[txw_idx][txh_idx], 1);
- transpose_4x4(in, in);
- fadst4x4_neon(in, in, av1_fwd_cos_bit_row[txw_idx][txh_idx], 1);
- write_buffer_4x4(in, coeff);
+ load_buffer_4x4(input, buf, input_stride, 1);
+ highbd_fadst4_x4_neon(buf, buf, av1_fwd_cos_bit_col[0][0]);
+ transpose_arrays_s32_4x4(buf, buf);
+ highbd_fadst4_x4_neon(buf, buf, av1_fwd_cos_bit_row[0][0]);
+ store_buffer_4x4(buf, coeff, /*stride=*/4);
break;
case FLIPADST_ADST:
- load_buffer_4x4(input, in, input_stride, 1, 0, &v_shift0);
- fadst4x4_neon(in, in, av1_fwd_cos_bit_col[txw_idx][txh_idx], 1);
- transpose_4x4(in, in);
- fadst4x4_neon(in, in, av1_fwd_cos_bit_row[txw_idx][txh_idx], 1);
- write_buffer_4x4(in, coeff);
+ load_buffer_4x4(input, buf, input_stride, 0);
+ highbd_fadst4_x4_neon(buf, buf, av1_fwd_cos_bit_col[0][0]);
+ transpose_arrays_s32_4x4(buf, buf);
+ highbd_fadst4_x4_neon(buf, buf, av1_fwd_cos_bit_row[0][0]);
+ store_buffer_4x4(buf, coeff, /*stride=*/4);
break;
case IDTX:
- load_buffer_4x4(input, in, input_stride, 0, 0, &v_shift0);
- idtx4x4_neon(in, in, av1_fwd_cos_bit_col[txw_idx][txh_idx], 1);
- transpose_4x4(in, in);
- idtx4x4_neon(in, in, av1_fwd_cos_bit_row[txw_idx][txh_idx], 1);
- write_buffer_4x4(in, coeff);
+ load_buffer_4x4(input, buf, input_stride, 0);
+ highbd_fidentity4_x4_neon(buf, buf, av1_fwd_cos_bit_col[0][0]);
+ transpose_arrays_s32_4x4(buf, buf);
+ highbd_fidentity4_x4_neon(buf, buf, av1_fwd_cos_bit_row[0][0]);
+ store_buffer_4x4(buf, coeff, /*stride=*/4);
break;
case V_DCT:
- load_buffer_4x4(input, in, input_stride, 0, 0, &v_shift0);
- fdct4x4_neon(in, in, av1_fwd_cos_bit_col[txw_idx][txh_idx], 1);
- transpose_4x4(in, in);
- idtx4x4_neon(in, in, av1_fwd_cos_bit_row[txw_idx][txh_idx], 1);
- write_buffer_4x4(in, coeff);
+ load_buffer_4x4(input, buf, input_stride, 0);
+ highbd_fdct4_x4_neon(buf, buf, av1_fwd_cos_bit_col[0][0]);
+ transpose_arrays_s32_4x4(buf, buf);
+ highbd_fidentity4_x4_neon(buf, buf, av1_fwd_cos_bit_row[0][0]);
+ store_buffer_4x4(buf, coeff, /*stride=*/4);
break;
case H_DCT:
- load_buffer_4x4(input, in, input_stride, 0, 0, &v_shift0);
- idtx4x4_neon(in, in, av1_fwd_cos_bit_row[txw_idx][txh_idx], 1);
- transpose_4x4(in, in);
- fdct4x4_neon(in, in, av1_fwd_cos_bit_col[txw_idx][txh_idx], 1);
- write_buffer_4x4(in, coeff);
+ load_buffer_4x4(input, buf, input_stride, 0);
+ highbd_fidentity4_x4_neon(buf, buf, av1_fwd_cos_bit_row[0][0]);
+ transpose_arrays_s32_4x4(buf, buf);
+ highbd_fdct4_x4_neon(buf, buf, av1_fwd_cos_bit_col[0][0]);
+ store_buffer_4x4(buf, coeff, /*stride=*/4);
break;
case V_ADST:
- load_buffer_4x4(input, in, input_stride, 0, 0, &v_shift0);
- fadst4x4_neon(in, in, av1_fwd_cos_bit_col[txw_idx][txh_idx], 1);
- transpose_4x4(in, in);
- idtx4x4_neon(in, in, av1_fwd_cos_bit_row[txw_idx][txh_idx], 1);
- write_buffer_4x4(in, coeff);
+ load_buffer_4x4(input, buf, input_stride, 0);
+ highbd_fadst4_x4_neon(buf, buf, av1_fwd_cos_bit_col[0][0]);
+ transpose_arrays_s32_4x4(buf, buf);
+ highbd_fidentity4_x4_neon(buf, buf, av1_fwd_cos_bit_row[0][0]);
+ store_buffer_4x4(buf, coeff, /*stride=*/4);
break;
case H_ADST:
- load_buffer_4x4(input, in, input_stride, 0, 0, &v_shift0);
- idtx4x4_neon(in, in, av1_fwd_cos_bit_row[txw_idx][txh_idx], 1);
- transpose_4x4(in, in);
- fadst4x4_neon(in, in, av1_fwd_cos_bit_col[txw_idx][txh_idx], 1);
- write_buffer_4x4(in, coeff);
+ load_buffer_4x4(input, buf, input_stride, 0);
+ highbd_fidentity4_x4_neon(buf, buf, av1_fwd_cos_bit_row[0][0]);
+ transpose_arrays_s32_4x4(buf, buf);
+ highbd_fadst4_x4_neon(buf, buf, av1_fwd_cos_bit_col[0][0]);
+ store_buffer_4x4(buf, coeff, /*stride=*/4);
break;
case V_FLIPADST:
- load_buffer_4x4(input, in, input_stride, 1, 0, &v_shift0);
- fadst4x4_neon(in, in, av1_fwd_cos_bit_row[txw_idx][txh_idx], 1);
- transpose_4x4(in, in);
- idtx4x4_neon(in, in, av1_fwd_cos_bit_row[txw_idx][txh_idx], 1);
- write_buffer_4x4(in, coeff);
+ load_buffer_4x4(input, buf, input_stride, 0);
+ highbd_fadst4_x4_neon(buf, buf, av1_fwd_cos_bit_row[0][0]);
+ transpose_arrays_s32_4x4(buf, buf);
+ highbd_fidentity4_x4_neon(buf, buf, av1_fwd_cos_bit_row[0][0]);
+ store_buffer_4x4(buf, coeff, /*stride=*/4);
break;
case H_FLIPADST:
- load_buffer_4x4(input, in, input_stride, 0, 1, &v_shift0);
- idtx4x4_neon(in, in, av1_fwd_cos_bit_row[txw_idx][txh_idx], 1);
- transpose_4x4(in, in);
- fadst4x4_neon(in, in, av1_fwd_cos_bit_row[txw_idx][txh_idx], 1);
- write_buffer_4x4(in, coeff);
+ load_buffer_4x4(input, buf, input_stride, 1);
+ highbd_fidentity4_x4_neon(buf, buf, av1_fwd_cos_bit_row[0][0]);
+ transpose_arrays_s32_4x4(buf, buf);
+ highbd_fadst4_x4_neon(buf, buf, av1_fwd_cos_bit_row[0][0]);
+ store_buffer_4x4(buf, coeff, /*stride=*/4);
break;
default: assert(0);
}
- (void)bd;
}
-static INLINE void load_buffer_8x8(const int16_t *input, int32x4_t *in,
- int stride, int flipud, int fliplr,
- const int shift) {
- if (!flipud) {
- in[0] = vreinterpretq_s32_s16(vld1q_s16((input + 0 * stride)));
- in[1] = vreinterpretq_s32_s16(vld1q_s16((input + 1 * stride)));
- in[2] = vreinterpretq_s32_s16(vld1q_s16((input + 2 * stride)));
- in[3] = vreinterpretq_s32_s16(vld1q_s16((input + 3 * stride)));
- in[4] = vreinterpretq_s32_s16(vld1q_s16((input + 4 * stride)));
- in[5] = vreinterpretq_s32_s16(vld1q_s16((input + 5 * stride)));
- in[6] = vreinterpretq_s32_s16(vld1q_s16((input + 6 * stride)));
- in[7] = vreinterpretq_s32_s16(vld1q_s16((input + 7 * stride)));
- } else {
- in[0] = vreinterpretq_s32_s16(vld1q_s16((input + 7 * stride)));
- in[1] = vreinterpretq_s32_s16(vld1q_s16((input + 6 * stride)));
- in[2] = vreinterpretq_s32_s16(vld1q_s16((input + 5 * stride)));
- in[3] = vreinterpretq_s32_s16(vld1q_s16((input + 4 * stride)));
- in[4] = vreinterpretq_s32_s16(vld1q_s16((input + 3 * stride)));
- in[5] = vreinterpretq_s32_s16(vld1q_s16((input + 2 * stride)));
- in[6] = vreinterpretq_s32_s16(vld1q_s16((input + 1 * stride)));
- in[7] = vreinterpretq_s32_s16(vld1q_s16((input + 0 * stride)));
+// Butterfly pre-processing:
+// e.g. n=4:
+// out[0] = in[0] + in[3]
+// out[1] = in[1] + in[2]
+// out[2] = in[1] - in[2]
+// out[3] = in[0] - in[3]
+
+static AOM_FORCE_INLINE void butterfly_dct_pre(const int32x4_t *input,
+ int32x4_t *output, int n) {
+ for (int i = 0; i < n / 2; ++i) {
+ output[i] = vaddq_s32(input[i], input[n - i - 1]);
}
-
- if (fliplr) {
- in[0] = vreinterpretq_s32_s16(vrev64q_s16(vreinterpretq_s16_s32(in[0])));
- in[0] = vextq_s32(in[0], in[0], 2);
- in[1] = vreinterpretq_s32_s16(vrev64q_s16(vreinterpretq_s16_s32(in[1])));
- in[1] = vextq_s32(in[1], in[1], 2);
- in[2] = vreinterpretq_s32_s16(vrev64q_s16(vreinterpretq_s16_s32(in[2])));
- in[2] = vextq_s32(in[2], in[2], 2);
- in[3] = vreinterpretq_s32_s16(vrev64q_s16(vreinterpretq_s16_s32(in[3])));
- in[3] = vextq_s32(in[3], in[3], 2);
- in[4] = vreinterpretq_s32_s16(vrev64q_s16(vreinterpretq_s16_s32(in[4])));
- in[4] = vextq_s32(in[4], in[4], 2);
- in[5] = vreinterpretq_s32_s16(vrev64q_s16(vreinterpretq_s16_s32(in[5])));
- in[5] = vextq_s32(in[5], in[5], 2);
- in[6] = vreinterpretq_s32_s16(vrev64q_s16(vreinterpretq_s16_s32(in[6])));
- in[6] = vextq_s32(in[6], in[6], 2);
- in[7] = vreinterpretq_s32_s16(vrev64q_s16(vreinterpretq_s16_s32(in[7])));
- in[7] = vextq_s32(in[7], in[7], 2);
+ for (int i = 0; i < n / 2; ++i) {
+ output[n / 2 + i] = vsubq_s32(input[n / 2 - i - 1], input[n / 2 + i]);
}
-
- int16x4_t u = vget_high_s16(vreinterpretq_s16_s32(in[4]));
- in[8] = vmovl_s16(vget_low_s16(vreinterpretq_s16_s32(in[4])));
- in[9] = vmovl_s16(u);
-
- u = vget_high_s16(vreinterpretq_s16_s32(in[5]));
- in[10] = vmovl_s16(vget_low_s16(vreinterpretq_s16_s32(in[5])));
- in[11] = vmovl_s16(u);
-
- u = vget_high_s16(vreinterpretq_s16_s32(in[6]));
- in[12] = vmovl_s16(vget_low_s16(vreinterpretq_s16_s32(in[6])));
- in[13] = vmovl_s16(u);
-
- u = vget_high_s16(vreinterpretq_s16_s32(in[7]));
- in[14] = vmovl_s16(vget_low_s16(vreinterpretq_s16_s32(in[7])));
- in[15] = vmovl_s16(u);
-
- u = vget_high_s16(vreinterpretq_s16_s32(in[3]));
- in[6] = vmovl_s16(vget_low_s16(vreinterpretq_s16_s32(in[3])));
- in[7] = vmovl_s16(u);
-
- u = vget_high_s16(vreinterpretq_s16_s32(in[2]));
- in[4] = vmovl_s16(vget_low_s16(vreinterpretq_s16_s32(in[2])));
- in[5] = vmovl_s16(u);
-
- u = vget_high_s16(vreinterpretq_s16_s32(in[1]));
- in[2] = vmovl_s16(vget_low_s16(vreinterpretq_s16_s32(in[1])));
- in[3] = vmovl_s16(u);
-
- u = vget_high_s16(vreinterpretq_s16_s32(in[0]));
- in[0] = vmovl_s16(vget_low_s16(vreinterpretq_s16_s32(in[0])));
- in[1] = vmovl_s16(u);
-
- const int32x4_t v_shift = vdupq_n_s32(shift);
-
- in[0] = vshlq_s32(in[0], v_shift);
- in[1] = vshlq_s32(in[1], v_shift);
- in[2] = vshlq_s32(in[2], v_shift);
- in[3] = vshlq_s32(in[3], v_shift);
- in[4] = vshlq_s32(in[4], v_shift);
- in[5] = vshlq_s32(in[5], v_shift);
- in[6] = vshlq_s32(in[6], v_shift);
- in[7] = vshlq_s32(in[7], v_shift);
-
- in[8] = vshlq_s32(in[8], v_shift);
- in[9] = vshlq_s32(in[9], v_shift);
- in[10] = vshlq_s32(in[10], v_shift);
- in[11] = vshlq_s32(in[11], v_shift);
- in[12] = vshlq_s32(in[12], v_shift);
- in[13] = vshlq_s32(in[13], v_shift);
- in[14] = vshlq_s32(in[14], v_shift);
- in[15] = vshlq_s32(in[15], v_shift);
}
-static INLINE void col_txfm_8x8_rounding(int32x4_t *in,
- const int32x4_t *v_shift) {
- in[0] = vrshlq_s32(in[0], *v_shift);
- in[1] = vrshlq_s32(in[1], *v_shift);
- in[2] = vrshlq_s32(in[2], *v_shift);
- in[3] = vrshlq_s32(in[3], *v_shift);
- in[4] = vrshlq_s32(in[4], *v_shift);
- in[5] = vrshlq_s32(in[5], *v_shift);
- in[6] = vrshlq_s32(in[6], *v_shift);
- in[7] = vrshlq_s32(in[7], *v_shift);
- in[8] = vrshlq_s32(in[8], *v_shift);
- in[9] = vrshlq_s32(in[9], *v_shift);
- in[10] = vrshlq_s32(in[10], *v_shift);
- in[11] = vrshlq_s32(in[11], *v_shift);
- in[12] = vrshlq_s32(in[12], *v_shift);
- in[13] = vrshlq_s32(in[13], *v_shift);
- in[14] = vrshlq_s32(in[14], *v_shift);
- in[15] = vrshlq_s32(in[15], *v_shift);
+// Butterfly post-processing:
+// e.g. n=8:
+// out[0] = in0[0] + in1[3];
+// out[1] = in0[1] + in1[2];
+// out[2] = in0[1] - in1[2];
+// out[3] = in0[0] - in1[3];
+// out[4] = in0[7] - in1[4];
+// out[5] = in0[6] - in1[5];
+// out[6] = in0[6] + in1[5];
+// out[7] = in0[7] + in1[4];
+
+static AOM_FORCE_INLINE void butterfly_dct_post(const int32x4_t *in0,
+ const int32x4_t *in1,
+ int32x4_t *output, int n) {
+ for (int i = 0; i < n / 4; ++i) {
+ output[i] = vaddq_s32(in0[i], in1[n / 2 - i - 1]);
+ }
+ for (int i = 0; i < n / 4; ++i) {
+ output[n / 4 + i] = vsubq_s32(in0[n / 4 - i - 1], in1[n / 4 + i]);
+ }
+ for (int i = 0; i < n / 4; ++i) {
+ output[n / 2 + i] = vsubq_s32(in0[n - i - 1], in1[n / 2 + i]);
+ }
+ for (int i = 0; i < n / 4; ++i) {
+ output[(3 * n) / 4 + i] =
+ vaddq_s32(in0[(3 * n) / 4 + i], in1[(3 * n) / 4 - i - 1]);
+ }
}
-static INLINE void col_txfm_4x8_rounding(int32x4_t *in,
- const int32x4_t *v_shift) {
- in[0] = vrshlq_s32(in[0], *v_shift);
- in[1] = vrshlq_s32(in[1], *v_shift);
- in[2] = vrshlq_s32(in[2], *v_shift);
- in[3] = vrshlq_s32(in[3], *v_shift);
- in[4] = vrshlq_s32(in[4], *v_shift);
- in[5] = vrshlq_s32(in[5], *v_shift);
- in[6] = vrshlq_s32(in[6], *v_shift);
- in[7] = vrshlq_s32(in[7], *v_shift);
-}
-
-static INLINE void write_buffer_8x8(const int32x4_t *res, int32_t *output) {
- vst1q_s32(output + 0 * 4, res[0]);
- vst1q_s32(output + 1 * 4, res[1]);
- vst1q_s32(output + 2 * 4, res[2]);
- vst1q_s32(output + 3 * 4, res[3]);
-
- vst1q_s32(output + 4 * 4, res[4]);
- vst1q_s32(output + 5 * 4, res[5]);
- vst1q_s32(output + 6 * 4, res[6]);
- vst1q_s32(output + 7 * 4, res[7]);
-
- vst1q_s32(output + 8 * 4, res[8]);
- vst1q_s32(output + 9 * 4, res[9]);
- vst1q_s32(output + 10 * 4, res[10]);
- vst1q_s32(output + 11 * 4, res[11]);
-
- vst1q_s32(output + 12 * 4, res[12]);
- vst1q_s32(output + 13 * 4, res[13]);
- vst1q_s32(output + 14 * 4, res[14]);
- vst1q_s32(output + 15 * 4, res[15]);
-}
-
-static INLINE void write_buffer_16x8(const int32x4_t *res, int32_t *output,
- const int stride) {
- vst1q_s32(output, res[0]);
- vst1q_s32(output + 4, res[1]);
- vst1q_s32(output + stride, res[2]);
- vst1q_s32(output + stride + 4, res[3]);
-
- vst1q_s32(output + (stride * 2), res[4]);
- vst1q_s32(output + (stride * 2) + 4, res[5]);
- vst1q_s32(output + (stride * 3), res[6]);
- vst1q_s32(output + (stride * 3) + 4, res[7]);
-
- vst1q_s32(output + (stride * 4), res[8]);
- vst1q_s32(output + (stride * 4) + 4, res[9]);
- vst1q_s32(output + (stride * 5), res[10]);
- vst1q_s32(output + (stride * 5) + 4, res[11]);
-
- vst1q_s32(output + (stride * 6), res[12]);
- vst1q_s32(output + (stride * 6) + 4, res[13]);
- vst1q_s32(output + (stride * 7), res[14]);
- vst1q_s32(output + (stride * 7) + 4, res[15]);
-}
-
-static void fdct4x8_neon(int32x4_t *in, int32x4_t *out, int bit,
- const int col_num) {
- const int32_t *cospi = cospi_arr(bit);
+static AOM_FORCE_INLINE void highbd_fdct8_x4_neon(const int32x4_t *in,
+ int32x4_t *out, int bit) {
+ const int32_t *const cospi = cospi_arr_s32(bit);
const int32x4_t v_bit = vdupq_n_s32(-bit);
- int32x4_t u[8], v[8];
- int startidx = 0 * col_num;
- int endidx = 7 * col_num;
- // stage 0-1
- u[0] = vaddq_s32(in[startidx], in[endidx]);
- v[7] = vsubq_s32(in[startidx], in[endidx]);
- startidx += col_num;
- endidx -= col_num;
- u[1] = vaddq_s32(in[startidx], in[endidx]);
- u[6] = vsubq_s32(in[startidx], in[endidx]);
- startidx += col_num;
- endidx -= col_num;
- u[2] = vaddq_s32(in[startidx], in[endidx]);
- u[5] = vsubq_s32(in[startidx], in[endidx]);
- startidx += col_num;
- endidx -= col_num;
- u[3] = vaddq_s32(in[startidx], in[endidx]);
- v[4] = vsubq_s32(in[startidx], in[endidx]);
+ // stage 1
+ int32x4_t a[8];
+ butterfly_dct_pre(in, a, 8);
// stage 2
- v[0] = vaddq_s32(u[0], u[3]);
- v[3] = vsubq_s32(u[0], u[3]);
- v[1] = vaddq_s32(u[1], u[2]);
- v[2] = vsubq_s32(u[1], u[2]);
-
- v[5] = vmulq_n_s32(u[6], cospi[32]);
- v[5] = vmlsq_n_s32(v[5], u[5], cospi[32]);
- v[5] = vrshlq_s32(v[5], v_bit);
-
- u[0] = vmulq_n_s32(u[5], cospi[32]);
- v[6] = vmlaq_n_s32(u[0], u[6], cospi[32]);
- v[6] = vrshlq_s32(v[6], v_bit);
+ int32x4_t b[8];
+ butterfly_dct_pre(a, b, 4);
+ butterfly_0130_neon(cospi, 32, a[5], a[6], &b[6], &b[5], v_bit);
// stage 3
- // type 0
- v[0] = vmulq_n_s32(v[0], cospi[32]);
- v[1] = vmulq_n_s32(v[1], cospi[32]);
- u[0] = vaddq_s32(v[0], v[1]);
- u[0] = vrshlq_s32(u[0], v_bit);
-
- u[1] = vsubq_s32(v[0], v[1]);
- u[1] = vrshlq_s32(u[1], v_bit);
-
- // type 1
- v[0] = vmulq_n_s32(v[2], cospi[48]);
- u[2] = vmlaq_n_s32(v[0], v[3], cospi[16]);
- u[2] = vrshlq_s32(u[2], v_bit);
-
- v[1] = vmulq_n_s32(v[3], cospi[48]);
- u[3] = vmlsq_n_s32(v[1], v[2], cospi[16]);
- u[3] = vrshlq_s32(u[3], v_bit);
-
- u[4] = vaddq_s32(v[4], v[5]);
- u[5] = vsubq_s32(v[4], v[5]);
- u[6] = vsubq_s32(v[7], v[6]);
- u[7] = vaddq_s32(v[7], v[6]);
+ int32x4_t c[8];
+ butterfly_0130_neon(cospi, 32, b[1], b[0], &c[0], &c[1], v_bit);
+ butterfly_0112_neon(cospi, 16, b[3], b[2], &c[2], &c[3], v_bit);
+ butterfly_dct_post(a + 4, b + 4, c + 4, 4);
// stage 4-5
- v[0] = vmulq_n_s32(u[4], cospi[56]);
- v[0] = vmlaq_n_s32(v[0], u[7], cospi[8]);
- out[1 * col_num] = vrshlq_s32(v[0], v_bit);
+ butterfly_0112_neon(cospi, 8, c[7], c[4], &out[1], &out[7], v_bit);
+ butterfly_0130_neon(cospi, 24, c[5], c[6], &out[5], &out[3], v_bit);
- v[1] = vmulq_n_s32(u[7], cospi[56]);
- v[0] = vmlsq_n_s32(v[1], u[4], cospi[8]);
- out[7 * col_num] = vrshlq_s32(v[0], v_bit);
-
- v[0] = vmulq_n_s32(u[5], cospi[24]);
- v[0] = vmlaq_n_s32(v[0], u[6], cospi[40]);
- out[5 * col_num] = vrshlq_s32(v[0], v_bit);
-
- v[1] = vmulq_n_s32(u[6], cospi[24]);
- v[0] = vmlsq_n_s32(v[1], u[5], cospi[40]);
- out[3 * col_num] = vrshlq_s32(v[0], v_bit);
-
- out[0 * col_num] = u[0];
- out[4 * col_num] = u[1];
- out[2 * col_num] = u[2];
- out[6 * col_num] = u[3];
+ out[0] = c[0];
+ out[2] = c[2];
+ out[4] = c[1];
+ out[6] = c[3];
}
-static void fdct8x8_neon(int32x4_t *in, int32x4_t *out, int bit,
- const int col_num) {
- fdct4x8_neon(in, out, bit, col_num);
- fdct4x8_neon(in + 1, out + 1, bit, col_num);
-}
-
-static void fadst8x8_neon(int32x4_t *in, int32x4_t *out, int bit,
- const int col_num) {
- const int32_t *cospi = cospi_arr(bit);
-
+static AOM_FORCE_INLINE void highbd_fadst8_x4_neon(const int32x4_t *in,
+ int32x4_t *out, int bit) {
+ const int32_t *const cospi = cospi_arr_s32(bit);
const int32x4_t v_bit = vdupq_n_s32(-bit);
+
int32x4_t u0, u1, u2, u3, u4, u5, u6, u7;
int32x4_t v0, v1, v2, v3, v4, v5, v6, v7;
- int32x4_t x, y;
- int col;
- for (col = 0; col < col_num; ++col) {
- // stage 0-1
- u0 = in[col_num * 0 + col];
- u1 = vnegq_s32(in[col_num * 7 + col]);
- u2 = vnegq_s32(in[col_num * 3 + col]);
- u3 = in[col_num * 4 + col];
- u4 = vnegq_s32(in[col_num * 1 + col]);
- u5 = in[col_num * 6 + col];
- u6 = in[col_num * 2 + col];
- u7 = vnegq_s32(in[col_num * 5 + col]);
-
- // stage 2
- v0 = u0;
- v1 = u1;
-
- x = vmulq_n_s32(u2, cospi[32]);
- y = vmulq_n_s32(u3, cospi[32]);
- v2 = vaddq_s32(x, y);
- v2 = vrshlq_s32(v2, v_bit);
-
- v3 = vsubq_s32(x, y);
- v3 = vrshlq_s32(v3, v_bit);
-
- v4 = u4;
- v5 = u5;
-
- x = vmulq_n_s32(u6, cospi[32]);
- y = vmulq_n_s32(u7, cospi[32]);
- v6 = vaddq_s32(x, y);
- v6 = vrshlq_s32(v6, v_bit);
-
- v7 = vsubq_s32(x, y);
- v7 = vrshlq_s32(v7, v_bit);
-
- // stage 3
- u0 = vaddq_s32(v0, v2);
- u1 = vaddq_s32(v1, v3);
- u2 = vsubq_s32(v0, v2);
- u3 = vsubq_s32(v1, v3);
- u4 = vaddq_s32(v4, v6);
- u5 = vaddq_s32(v5, v7);
- u6 = vsubq_s32(v4, v6);
- u7 = vsubq_s32(v5, v7);
-
- // stage 4
- v0 = u0;
- v1 = u1;
- v2 = u2;
- v3 = u3;
-
- v4 = vmulq_n_s32(u4, cospi[16]);
- v4 = vmlaq_n_s32(v4, u5, cospi[48]);
- v4 = vrshlq_s32(v4, v_bit);
-
- v5 = vmulq_n_s32(u4, cospi[48]);
- v5 = vmlsq_n_s32(v5, u5, cospi[16]);
- v5 = vrshlq_s32(v5, v_bit);
-
- v6 = vmulq_n_s32(u7, cospi[16]);
- v6 = vmlsq_n_s32(v6, u6, cospi[48]);
- v6 = vrshlq_s32(v6, v_bit);
-
- v7 = vmulq_n_s32(u6, cospi[16]);
- v7 = vmlaq_n_s32(v7, u7, cospi[48]);
- v7 = vrshlq_s32(v7, v_bit);
-
- // stage 5
- u0 = vaddq_s32(v0, v4);
- u1 = vaddq_s32(v1, v5);
- u2 = vaddq_s32(v2, v6);
- u3 = vaddq_s32(v3, v7);
- u4 = vsubq_s32(v0, v4);
- u5 = vsubq_s32(v1, v5);
- u6 = vsubq_s32(v2, v6);
- u7 = vsubq_s32(v3, v7);
-
- // stage 6
- v0 = vmulq_n_s32(u0, cospi[4]);
- v0 = vmlaq_n_s32(v0, u1, cospi[60]);
- v0 = vrshlq_s32(v0, v_bit);
-
- v1 = vmulq_n_s32(u0, cospi[60]);
- v1 = vmlsq_n_s32(v1, u1, cospi[4]);
- v1 = vrshlq_s32(v1, v_bit);
-
- v2 = vmulq_n_s32(u2, cospi[20]);
- v2 = vmlaq_n_s32(v2, u3, cospi[44]);
- v2 = vrshlq_s32(v2, v_bit);
-
- v3 = vmulq_n_s32(u2, cospi[44]);
- v3 = vmlsq_n_s32(v3, u3, cospi[20]);
- v3 = vrshlq_s32(v3, v_bit);
-
- v4 = vmulq_n_s32(u4, cospi[36]);
- v4 = vmlaq_n_s32(v4, u5, cospi[28]);
- v4 = vrshlq_s32(v4, v_bit);
-
- v5 = vmulq_n_s32(u4, cospi[28]);
- v5 = vmlsq_n_s32(v5, u5, cospi[36]);
- v5 = vrshlq_s32(v5, v_bit);
-
- x = vmulq_n_s32(u6, cospi[52]);
- v6 = vmlaq_n_s32(x, u7, cospi[12]);
- v6 = vrshlq_s32(v6, v_bit);
-
- v7 = vmulq_n_s32(u6, cospi[12]);
- v7 = vmlsq_n_s32(v7, u7, cospi[52]);
- v7 = vrshlq_s32(v7, v_bit);
-
- // stage 7
- out[col_num * 0 + col] = v1;
- out[col_num * 1 + col] = v6;
- out[col_num * 2 + col] = v3;
- out[col_num * 3 + col] = v4;
- out[col_num * 4 + col] = v5;
- out[col_num * 5 + col] = v2;
- out[col_num * 6 + col] = v7;
- out[col_num * 7 + col] = v0;
- }
-}
-static void idtx8x8_neon(int32x4_t *in, int32x4_t *out, int bit, int col_num) {
- (void)bit;
-
- for (int i = 0; i < col_num; i += 1) {
- out[0 + 8 * i] = vshlq_n_s32(in[0 + 8 * i], 1);
- out[1 + 8 * i] = vshlq_n_s32(in[1 + 8 * i], 1);
- out[2 + 8 * i] = vshlq_n_s32(in[2 + 8 * i], 1);
- out[3 + 8 * i] = vshlq_n_s32(in[3 + 8 * i], 1);
- out[4 + 8 * i] = vshlq_n_s32(in[4 + 8 * i], 1);
- out[5 + 8 * i] = vshlq_n_s32(in[5 + 8 * i], 1);
- out[6 + 8 * i] = vshlq_n_s32(in[6 + 8 * i], 1);
- out[7 + 8 * i] = vshlq_n_s32(in[7 + 8 * i], 1);
- }
-}
-#if !CONFIG_REALTIME_ONLY
-static void idtx32x8_neon(int32x4_t *in, int32x4_t *out, int bit, int col_num) {
- (void)bit;
- (void)col_num;
- for (int j = 0; j < 2; j++) {
- out[j + 8 * 0] = vshlq_n_s32(in[j + 8 * 0], 1);
- out[j + 8 * 1] = vshlq_n_s32(in[j + 8 * 1], 1);
- out[j + 8 * 2] = vshlq_n_s32(in[j + 8 * 2], 1);
- out[j + 8 * 3] = vshlq_n_s32(in[j + 8 * 3], 1);
- out[j + 8 * 4] = vshlq_n_s32(in[j + 8 * 4], 1);
- out[j + 8 * 5] = vshlq_n_s32(in[j + 8 * 5], 1);
- out[j + 8 * 6] = vshlq_n_s32(in[j + 8 * 6], 1);
- out[j + 8 * 7] = vshlq_n_s32(in[j + 8 * 7], 1);
- }
-}
-#endif
-void av1_fwd_txfm2d_8x8_neon(const int16_t *input, int32_t *coeff, int stride,
- TX_TYPE tx_type, int bd) {
- int32x4_t in[16], out[16];
- const int8_t *shift = av1_fwd_txfm_shift_ls[TX_8X8];
- const int txw_idx = get_txw_idx(TX_8X8);
- const int txh_idx = get_txh_idx(TX_8X8);
- const int32x4_t v_shift1 = vdupq_n_s32(shift[1]);
- switch (tx_type) {
- case DCT_DCT:
- load_buffer_8x8(input, in, stride, 0, 0, shift[0]);
- fdct8x8_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], 2);
- col_txfm_8x8_rounding(out, &v_shift1);
- transpose_8x8(out, in);
- fdct8x8_neon(in, out, av1_fwd_cos_bit_row[txw_idx][txh_idx], 2);
- write_buffer_8x8(out, coeff);
- break;
- case ADST_DCT:
- load_buffer_8x8(input, in, stride, 0, 0, shift[0]);
- fadst8x8_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], 2);
- col_txfm_8x8_rounding(out, &v_shift1);
- transpose_8x8(out, in);
- fdct8x8_neon(in, out, av1_fwd_cos_bit_row[txw_idx][txh_idx], 2);
- write_buffer_8x8(out, coeff);
- break;
- case DCT_ADST:
- load_buffer_8x8(input, in, stride, 0, 0, shift[0]);
- fdct8x8_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], 2);
- col_txfm_8x8_rounding(out, &v_shift1);
- transpose_8x8(out, in);
- fadst8x8_neon(in, out, av1_fwd_cos_bit_row[txw_idx][txh_idx], 2);
- write_buffer_8x8(out, coeff);
- break;
- case ADST_ADST:
- load_buffer_8x8(input, in, stride, 0, 0, shift[0]);
- fadst8x8_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], 2);
- col_txfm_8x8_rounding(out, &v_shift1);
- transpose_8x8(out, in);
- fadst8x8_neon(in, out, av1_fwd_cos_bit_row[txw_idx][txh_idx], 2);
- write_buffer_8x8(out, coeff);
- break;
- case FLIPADST_DCT:
- load_buffer_8x8(input, in, stride, 1, 0, shift[0]);
- fadst8x8_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], 2);
- col_txfm_8x8_rounding(out, &v_shift1);
- transpose_8x8(out, in);
- fdct8x8_neon(in, out, av1_fwd_cos_bit_row[txw_idx][txh_idx], 2);
- write_buffer_8x8(out, coeff);
- break;
- case DCT_FLIPADST:
- load_buffer_8x8(input, in, stride, 0, 1, shift[0]);
- fdct8x8_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], 2);
- col_txfm_8x8_rounding(out, &v_shift1);
- transpose_8x8(out, in);
- fadst8x8_neon(in, out, av1_fwd_cos_bit_row[txw_idx][txh_idx], 2);
- write_buffer_8x8(out, coeff);
- break;
- case FLIPADST_FLIPADST:
- load_buffer_8x8(input, in, stride, 1, 1, shift[0]);
- fadst8x8_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], 2);
- col_txfm_8x8_rounding(out, &v_shift1);
- transpose_8x8(out, in);
- fadst8x8_neon(in, out, av1_fwd_cos_bit_row[txw_idx][txh_idx], 2);
- write_buffer_8x8(out, coeff);
- break;
- case ADST_FLIPADST:
- load_buffer_8x8(input, in, stride, 0, 1, shift[0]);
- fadst8x8_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], 2);
- col_txfm_8x8_rounding(out, &v_shift1);
- transpose_8x8(out, in);
- fadst8x8_neon(in, out, av1_fwd_cos_bit_row[txw_idx][txh_idx], 2);
- write_buffer_8x8(out, coeff);
- break;
- case FLIPADST_ADST:
- load_buffer_8x8(input, in, stride, 1, 0, shift[0]);
- fadst8x8_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], 2);
- col_txfm_8x8_rounding(out, &v_shift1);
- transpose_8x8(out, in);
- fadst8x8_neon(in, out, av1_fwd_cos_bit_row[txw_idx][txh_idx], 2);
- write_buffer_8x8(out, coeff);
- break;
- case IDTX:
- load_buffer_8x8(input, in, stride, 0, 0, shift[0]);
- idtx8x8_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], 2);
- col_txfm_8x8_rounding(out, &v_shift1);
- transpose_8x8(out, in);
- idtx8x8_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], 2);
- write_buffer_8x8(out, coeff);
- break;
- case V_DCT:
- load_buffer_8x8(input, in, stride, 0, 0, shift[0]);
- fdct8x8_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], 2);
- col_txfm_8x8_rounding(out, &v_shift1);
- transpose_8x8(out, in);
- idtx8x8_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], 2);
- write_buffer_8x8(out, coeff);
- break;
- case H_DCT:
- load_buffer_8x8(input, in, stride, 0, 0, shift[0]);
- idtx8x8_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], 2);
- col_txfm_8x8_rounding(out, &v_shift1);
- transpose_8x8(out, in);
- fdct8x8_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], 2);
- write_buffer_8x8(out, coeff);
- break;
- case V_ADST:
- load_buffer_8x8(input, in, stride, 0, 0, shift[0]);
- fadst8x8_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], 2);
- col_txfm_8x8_rounding(out, &v_shift1);
- transpose_8x8(out, in);
- idtx8x8_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], 2);
- write_buffer_8x8(out, coeff);
- break;
- case H_ADST:
- load_buffer_8x8(input, in, stride, 0, 0, shift[0]);
- idtx8x8_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], 2);
- col_txfm_8x8_rounding(out, &v_shift1);
- transpose_8x8(out, in);
- fadst8x8_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], 2);
- write_buffer_8x8(out, coeff);
- break;
- case V_FLIPADST:
- load_buffer_8x8(input, in, stride, 1, 0, shift[0]);
- fadst8x8_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], 2);
- col_txfm_8x8_rounding(out, &v_shift1);
- transpose_8x8(out, in);
- idtx8x8_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], 2);
- write_buffer_8x8(out, coeff);
- break;
- case H_FLIPADST:
- load_buffer_8x8(input, in, stride, 0, 1, shift[0]);
- idtx8x8_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], 2);
- col_txfm_8x8_rounding(out, &v_shift1);
- transpose_8x8(out, in);
- fadst8x8_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], 2);
- write_buffer_8x8(out, coeff);
- break;
- default: assert(0);
- }
- (void)bd;
-}
-
-// Hybrid Transform 16x16
-
-static INLINE void convert_8x8_to_16x16(const int32x4_t *in, int32x4_t *out) {
- int row_index = 0;
- int dst_index = 0;
- int src_index = 0;
-
- // row 0, 1, .., 7
- do {
- out[dst_index] = in[src_index];
- out[dst_index + 1] = in[src_index + 1];
- out[dst_index + 2] = in[src_index + 16];
- out[dst_index + 3] = in[src_index + 17];
- dst_index += 4;
- src_index += 2;
- row_index += 1;
- } while (row_index < 8);
-
- // row 8, 9, ..., 15
- src_index += 16;
- do {
- out[dst_index] = in[src_index];
- out[dst_index + 1] = in[src_index + 1];
- out[dst_index + 2] = in[src_index + 16];
- out[dst_index + 3] = in[src_index + 17];
- dst_index += 4;
- src_index += 2;
- row_index += 1;
- } while (row_index < 16);
-}
-
-static INLINE void load_buffer_16x16(const int16_t *input, int32x4_t *out,
- int stride, int flipud, int fliplr,
- int shift) {
- int32x4_t in[64];
- // Load 4 8x8 blocks
- const int16_t *topL = input;
- const int16_t *topR = input + 8;
- const int16_t *botL = input + 8 * stride;
- const int16_t *botR = input + 8 * stride + 8;
-
- const int16_t *tmp;
-
- if (flipud) {
- // Swap left columns
- tmp = topL;
- topL = botL;
- botL = tmp;
- // Swap right columns
- tmp = topR;
- topR = botR;
- botR = tmp;
- }
-
- if (fliplr) {
- // Swap top rows
- tmp = topL;
- topL = topR;
- topR = tmp;
- // Swap bottom rows
- tmp = botL;
- botL = botR;
- botR = tmp;
- }
-
- // load first 8 columns
- load_buffer_8x8(topL, &in[0], stride, flipud, fliplr, shift);
- load_buffer_8x8(botL, &in[32], stride, flipud, fliplr, shift);
-
- // load second 8 columns
- load_buffer_8x8(topR, &in[16], stride, flipud, fliplr, shift);
- load_buffer_8x8(botR, &in[48], stride, flipud, fliplr, shift);
-
- convert_8x8_to_16x16(in, out);
-}
-
-static INLINE void load_buffer_8x16(const int16_t *input, int32x4_t *out,
- int stride, int flipud, int fliplr,
- int shift) {
- const int16_t *topL = input;
- const int16_t *botL = input + 8 * stride;
-
- const int16_t *tmp;
-
- if (flipud) {
- tmp = topL;
- topL = botL;
- botL = tmp;
- }
-
- load_buffer_8x8(topL, out, stride, flipud, fliplr, shift);
- load_buffer_8x8(botL, out + 16, stride, flipud, fliplr, shift);
-}
-
-static INLINE void load_buffer_8x4(const int16_t *input, int32x4_t *out,
- int stride, int flipud, int fliplr,
- const int32x4_t *v_shift) {
- const int16_t *topL = input;
- const int16_t *topR = input + 4;
-
- const int16_t *tmp;
-
- if (fliplr) {
- tmp = topL;
- topL = topR;
- topR = tmp;
- }
- load_buffer_4x4(topL, out, stride, flipud, fliplr, v_shift);
- load_buffer_4x4(topR, out + 4, stride, flipud, fliplr, v_shift);
-}
-
-static INLINE void load_buffer_16x4(const int16_t *input, int32x4_t *out,
- int stride, int flipud, int fliplr,
- const int32x4_t *v_shift) {
- const int16_t *topL = input;
- const int16_t *topR = input + 8;
-
- const int16_t *tmp;
-
- if (fliplr) {
- tmp = topL;
- topL = topR;
- topR = tmp;
- }
-
- load_buffer_8x4(topL, out, stride, flipud, fliplr, v_shift);
- load_buffer_8x4(topR, out + 8, stride, flipud, fliplr, v_shift);
-}
-
-static INLINE void load_buffer_4x8(const int16_t *input, int32x4_t *out,
- int stride, int flipud, int fliplr,
- const int32x4_t *v_shift) {
- const int16_t *topL = input;
- const int16_t *botL = input + 4 * stride;
-
- const int16_t *tmp;
-
- if (flipud) {
- tmp = topL;
- topL = botL;
- botL = tmp;
- }
-
- load_buffer_4x4(topL, out, stride, flipud, fliplr, v_shift);
- load_buffer_4x4(botL, out + 4, stride, flipud, fliplr, v_shift);
-}
-
-#if !CONFIG_REALTIME_ONLY
-static INLINE void load_buffer_4x16(const int16_t *input, int32x4_t *out,
- const int stride, const int flipud,
- const int fliplr,
- const int32x4_t *v_shift) {
- const int16_t *topL = input;
- const int16_t *botL = input + 8 * stride;
-
- const int16_t *tmp;
-
- if (flipud) {
- tmp = topL;
- topL = botL;
- botL = tmp;
- }
- load_buffer_4x8(topL, out, stride, flipud, fliplr, v_shift);
- load_buffer_4x8(botL, out + 8, stride, flipud, fliplr, v_shift);
-}
-#endif
-
-static INLINE void load_buffer_32x8n(const int16_t *input, int32x4_t *out,
- int stride, int flipud, int fliplr,
- int shift, const int height) {
- const int16_t *in = input;
- int32x4_t *output = out;
- for (int col = 0; col < height; col++) {
- in = input + col * stride;
- output = out + col * 8;
- int32x4_t v_shift = vdupq_n_s32(shift);
- load_buffer_4x4(in, output, 4, flipud, fliplr, &v_shift);
- load_buffer_4x4((in + 16), (output + 4), 4, flipud, fliplr, &v_shift);
- }
-}
-
-static void fdct16x16_neon(int32x4_t *in, int32x4_t *out, int bit,
- const int col_num) {
- const int32_t *cospi = cospi_arr(bit);
- const int32x4_t v_bit = vdupq_n_s32(-bit);
- int32x4_t u[16], v[16];
- int col;
-
- // Calculate the column 0, 1, 2, 3
- for (col = 0; col < col_num; ++col) {
- // stage 0
- // stage 1
- u[0] = vaddq_s32(in[0 * col_num + col], in[15 * col_num + col]);
- u[15] = vsubq_s32(in[0 * col_num + col], in[15 * col_num + col]);
- u[1] = vaddq_s32(in[1 * col_num + col], in[14 * col_num + col]);
- u[14] = vsubq_s32(in[1 * col_num + col], in[14 * col_num + col]);
- u[2] = vaddq_s32(in[2 * col_num + col], in[13 * col_num + col]);
- u[13] = vsubq_s32(in[2 * col_num + col], in[13 * col_num + col]);
- u[3] = vaddq_s32(in[3 * col_num + col], in[12 * col_num + col]);
- u[12] = vsubq_s32(in[3 * col_num + col], in[12 * col_num + col]);
- u[4] = vaddq_s32(in[4 * col_num + col], in[11 * col_num + col]);
- u[11] = vsubq_s32(in[4 * col_num + col], in[11 * col_num + col]);
- u[5] = vaddq_s32(in[5 * col_num + col], in[10 * col_num + col]);
- u[10] = vsubq_s32(in[5 * col_num + col], in[10 * col_num + col]);
- u[6] = vaddq_s32(in[6 * col_num + col], in[9 * col_num + col]);
- u[9] = vsubq_s32(in[6 * col_num + col], in[9 * col_num + col]);
- u[7] = vaddq_s32(in[7 * col_num + col], in[8 * col_num + col]);
- u[8] = vsubq_s32(in[7 * col_num + col], in[8 * col_num + col]);
-
- // stage 2
- v[0] = vaddq_s32(u[0], u[7]);
- v[7] = vsubq_s32(u[0], u[7]);
- v[1] = vaddq_s32(u[1], u[6]);
- v[6] = vsubq_s32(u[1], u[6]);
- v[2] = vaddq_s32(u[2], u[5]);
- v[5] = vsubq_s32(u[2], u[5]);
- v[3] = vaddq_s32(u[3], u[4]);
- v[4] = vsubq_s32(u[3], u[4]);
- v[8] = u[8];
- v[9] = u[9];
-
- v[10] = vmulq_n_s32(u[13], cospi[32]);
- v[10] = vmlsq_n_s32(v[10], u[10], cospi[32]);
- v[10] = vrshlq_s32(v[10], v_bit);
-
- v[13] = vmulq_n_s32(u[10], cospi[32]);
- v[13] = vmlaq_n_s32(v[13], u[13], cospi[32]);
- v[13] = vrshlq_s32(v[13], v_bit);
-
- v[11] = vmulq_n_s32(u[12], cospi[32]);
- v[11] = vmlsq_n_s32(v[11], u[11], cospi[32]);
- v[11] = vrshlq_s32(v[11], v_bit);
-
- v[12] = vmulq_n_s32(u[11], cospi[32]);
- v[12] = vmlaq_n_s32(v[12], u[12], cospi[32]);
- v[12] = vrshlq_s32(v[12], v_bit);
- v[14] = u[14];
- v[15] = u[15];
-
- // stage 3
- u[0] = vaddq_s32(v[0], v[3]);
- u[3] = vsubq_s32(v[0], v[3]);
- u[1] = vaddq_s32(v[1], v[2]);
- u[2] = vsubq_s32(v[1], v[2]);
- u[4] = v[4];
-
- u[5] = vmulq_n_s32(v[6], cospi[32]);
- u[5] = vmlsq_n_s32(u[5], v[5], cospi[32]);
- u[5] = vrshlq_s32(u[5], v_bit);
-
- u[6] = vmulq_n_s32(v[5], cospi[32]);
- u[6] = vmlaq_n_s32(u[6], v[6], cospi[32]);
- u[6] = vrshlq_s32(u[6], v_bit);
-
- u[7] = v[7];
- u[8] = vaddq_s32(v[8], v[11]);
- u[11] = vsubq_s32(v[8], v[11]);
- u[9] = vaddq_s32(v[9], v[10]);
- u[10] = vsubq_s32(v[9], v[10]);
- u[12] = vsubq_s32(v[15], v[12]);
- u[15] = vaddq_s32(v[15], v[12]);
- u[13] = vsubq_s32(v[14], v[13]);
- u[14] = vaddq_s32(v[14], v[13]);
-
- // stage 4
- u[0] = vmulq_n_s32(u[0], cospi[32]);
- u[1] = vmulq_n_s32(u[1], cospi[32]);
- v[0] = vaddq_s32(u[0], u[1]);
- v[0] = vrshlq_s32(v[0], v_bit);
-
- v[1] = vsubq_s32(u[0], u[1]);
- v[1] = vrshlq_s32(v[1], v_bit);
-
- v[2] = vmulq_n_s32(u[2], cospi[48]);
- v[2] = vmlaq_n_s32(v[2], u[3], cospi[16]);
- v[2] = vrshlq_s32(v[2], v_bit);
-
- v[3] = vmulq_n_s32(u[3], cospi[48]);
- v[3] = vmlsq_n_s32(v[3], u[2], cospi[16]);
- v[3] = vrshlq_s32(v[3], v_bit);
-
- v[4] = vaddq_s32(u[4], u[5]);
- v[5] = vsubq_s32(u[4], u[5]);
- v[6] = vsubq_s32(u[7], u[6]);
- v[7] = vaddq_s32(u[7], u[6]);
- v[8] = u[8];
-
- v[9] = vmulq_n_s32(u[14], cospi[48]);
- v[9] = vmlsq_n_s32(v[9], u[9], cospi[16]);
- v[9] = vrshlq_s32(v[9], v_bit);
-
- v[14] = vmulq_n_s32(u[9], cospi[48]);
- v[14] = vmlaq_n_s32(v[14], u[14], cospi[16]);
- v[14] = vrshlq_s32(v[14], v_bit);
-
- v[10] = vmulq_n_s32(u[13], -cospi[16]);
- v[10] = vmlsq_n_s32(v[10], u[10], cospi[48]);
- v[10] = vrshlq_s32(v[10], v_bit);
-
- v[13] = vmulq_n_s32(u[10], -cospi[16]);
- v[13] = vmlaq_n_s32(v[13], u[13], cospi[48]);
- v[13] = vrshlq_s32(v[13], v_bit);
-
- v[11] = u[11];
- v[12] = u[12];
- v[15] = u[15];
-
- // stage 5
- u[0] = v[0];
- u[1] = v[1];
- u[2] = v[2];
- u[3] = v[3];
-
- u[4] = vmulq_n_s32(v[4], cospi[56]);
- u[4] = vmlaq_n_s32(u[4], v[7], cospi[8]);
- u[4] = vrshlq_s32(u[4], v_bit);
-
- u[7] = vmulq_n_s32(v[7], cospi[56]);
- u[7] = vmlsq_n_s32(u[7], v[4], cospi[8]);
- u[7] = vrshlq_s32(u[7], v_bit);
-
- u[5] = vmulq_n_s32(v[5], cospi[24]);
- u[5] = vmlaq_n_s32(u[5], v[6], cospi[40]);
- u[5] = vrshlq_s32(u[5], v_bit);
-
- u[6] = vmulq_n_s32(v[6], cospi[24]);
- u[6] = vmlsq_n_s32(u[6], v[5], cospi[40]);
- u[6] = vrshlq_s32(u[6], v_bit);
-
- u[8] = vaddq_s32(v[8], v[9]);
- u[9] = vsubq_s32(v[8], v[9]);
- u[10] = vsubq_s32(v[11], v[10]);
- u[11] = vaddq_s32(v[11], v[10]);
- u[12] = vaddq_s32(v[12], v[13]);
- u[13] = vsubq_s32(v[12], v[13]);
- u[14] = vsubq_s32(v[15], v[14]);
- u[15] = vaddq_s32(v[15], v[14]);
-
- // stage 6
- v[0] = u[0];
- v[1] = u[1];
- v[2] = u[2];
- v[3] = u[3];
- v[4] = u[4];
- v[5] = u[5];
- v[6] = u[6];
- v[7] = u[7];
-
- v[8] = vmulq_n_s32(u[8], cospi[60]);
- v[8] = vmlaq_n_s32(v[8], u[15], cospi[4]);
- v[8] = vrshlq_s32(v[8], v_bit);
-
- v[15] = vmulq_n_s32(u[15], cospi[60]);
- v[15] = vmlsq_n_s32(v[15], u[8], cospi[4]);
- v[15] = vrshlq_s32(v[15], v_bit);
-
- v[9] = vmulq_n_s32(u[9], cospi[28]);
- v[9] = vmlaq_n_s32(v[9], u[14], cospi[36]);
- v[9] = vrshlq_s32(v[9], v_bit);
-
- v[14] = vmulq_n_s32(u[14], cospi[28]);
- v[14] = vmlsq_n_s32(v[14], u[9], cospi[36]);
- v[14] = vrshlq_s32(v[14], v_bit);
-
- v[10] = vmulq_n_s32(u[10], cospi[44]);
- v[10] = vmlaq_n_s32(v[10], u[13], cospi[20]);
- v[10] = vrshlq_s32(v[10], v_bit);
-
- v[13] = vmulq_n_s32(u[13], cospi[44]);
- v[13] = vmlsq_n_s32(v[13], u[10], cospi[20]);
- v[13] = vrshlq_s32(v[13], v_bit);
-
- v[11] = vmulq_n_s32(u[11], cospi[12]);
- v[11] = vmlaq_n_s32(v[11], u[12], cospi[52]);
- v[11] = vrshlq_s32(v[11], v_bit);
-
- v[12] = vmulq_n_s32(u[12], cospi[12]);
- v[12] = vmlsq_n_s32(v[12], u[11], cospi[52]);
- v[12] = vrshlq_s32(v[12], v_bit);
-
- out[0 * col_num + col] = v[0];
- out[1 * col_num + col] = v[8];
- out[2 * col_num + col] = v[4];
- out[3 * col_num + col] = v[12];
- out[4 * col_num + col] = v[2];
- out[5 * col_num + col] = v[10];
- out[6 * col_num + col] = v[6];
- out[7 * col_num + col] = v[14];
- out[8 * col_num + col] = v[1];
- out[9 * col_num + col] = v[9];
- out[10 * col_num + col] = v[5];
- out[11 * col_num + col] = v[13];
- out[12 * col_num + col] = v[3];
- out[13 * col_num + col] = v[11];
- out[14 * col_num + col] = v[7];
- out[15 * col_num + col] = v[15];
- }
-}
-
-static void fadst16x16_neon(int32x4_t *in, int32x4_t *out, int bit,
- const int num_cols) {
- const int32_t *cospi = cospi_arr(bit);
-
- const int32x4_t v_bit = vdupq_n_s32(-bit);
-
- int32x4_t u[16], v[16], x, y;
- int col;
-
- for (col = 0; col < num_cols; ++col) {
- // stage 0-1
- u[0] = in[0 * num_cols + col];
- u[1] = vnegq_s32(in[15 * num_cols + col]);
- u[2] = vnegq_s32(in[7 * num_cols + col]);
- u[3] = in[8 * num_cols + col];
- u[4] = vnegq_s32(in[3 * num_cols + col]);
- u[5] = in[12 * num_cols + col];
- u[6] = in[4 * num_cols + col];
- u[7] = vnegq_s32(in[11 * num_cols + col]);
- u[8] = vnegq_s32(in[1 * num_cols + col]);
- u[9] = in[14 * num_cols + col];
- u[10] = in[6 * num_cols + col];
- u[11] = vnegq_s32(in[9 * num_cols + col]);
- u[12] = in[2 * num_cols + col];
- u[13] = vnegq_s32(in[13 * num_cols + col]);
- u[14] = vnegq_s32(in[5 * num_cols + col]);
- u[15] = in[10 * num_cols + col];
-
- // stage 2
- v[0] = u[0];
- v[1] = u[1];
-
- x = vmulq_n_s32(u[2], cospi[32]);
- y = vmulq_n_s32(u[3], cospi[32]);
- v[2] = vaddq_s32(x, y);
- v[2] = vrshlq_s32(v[2], v_bit);
-
- v[3] = vsubq_s32(x, y);
- v[3] = vrshlq_s32(v[3], v_bit);
-
- v[4] = u[4];
- v[5] = u[5];
-
- x = vmulq_n_s32(u[6], cospi[32]);
- y = vmulq_n_s32(u[7], cospi[32]);
- v[6] = vaddq_s32(x, y);
- v[6] = vrshlq_s32(v[6], v_bit);
-
- v[7] = vsubq_s32(x, y);
- v[7] = vrshlq_s32(v[7], v_bit);
-
- v[8] = u[8];
- v[9] = u[9];
-
- x = vmulq_n_s32(u[10], cospi[32]);
- y = vmulq_n_s32(u[11], cospi[32]);
- v[10] = vaddq_s32(x, y);
- v[10] = vrshlq_s32(v[10], v_bit);
-
- v[11] = vsubq_s32(x, y);
- v[11] = vrshlq_s32(v[11], v_bit);
-
- v[12] = u[12];
- v[13] = u[13];
-
- x = vmulq_n_s32(u[14], cospi[32]);
- y = vmulq_n_s32(u[15], cospi[32]);
- v[14] = vaddq_s32(x, y);
- v[14] = vrshlq_s32(v[14], v_bit);
-
- v[15] = vsubq_s32(x, y);
- v[15] = vrshlq_s32(v[15], v_bit);
-
- // stage 3
- u[0] = vaddq_s32(v[0], v[2]);
- u[1] = vaddq_s32(v[1], v[3]);
- u[2] = vsubq_s32(v[0], v[2]);
- u[3] = vsubq_s32(v[1], v[3]);
- u[4] = vaddq_s32(v[4], v[6]);
- u[5] = vaddq_s32(v[5], v[7]);
- u[6] = vsubq_s32(v[4], v[6]);
- u[7] = vsubq_s32(v[5], v[7]);
- u[8] = vaddq_s32(v[8], v[10]);
- u[9] = vaddq_s32(v[9], v[11]);
- u[10] = vsubq_s32(v[8], v[10]);
- u[11] = vsubq_s32(v[9], v[11]);
- u[12] = vaddq_s32(v[12], v[14]);
- u[13] = vaddq_s32(v[13], v[15]);
- u[14] = vsubq_s32(v[12], v[14]);
- u[15] = vsubq_s32(v[13], v[15]);
-
- // stage 4
- v[0] = u[0];
- v[1] = u[1];
- v[2] = u[2];
- v[3] = u[3];
- v[4] = half_btf_neon(&cospi[16], &u[4], &cospi[48], &u[5], v_bit);
- v[7] = half_btf_neon(&cospi[16], &u[6], &cospi[48], &u[7], v_bit);
- v[5] = half_btf_neon_m(&cospi[48], &u[4], &cospi[16], &u[5], v_bit);
- v[6] = half_btf_neon_m(&cospi[16], &u[7], &cospi[48], &u[6], v_bit);
-
- v[8] = u[8];
- v[9] = u[9];
- v[10] = u[10];
- v[11] = u[11];
-
- v[12] = half_btf_neon(&cospi[16], &u[12], &cospi[48], &u[13], v_bit);
- v[15] = half_btf_neon(&cospi[16], &u[14], &cospi[48], &u[15], v_bit);
- v[13] = half_btf_neon_m(&cospi[48], &u[12], &cospi[16], &u[13], v_bit);
- v[14] = half_btf_neon_m(&cospi[16], &u[15], &cospi[48], &u[14], v_bit);
-
- // stage 5
- u[0] = vaddq_s32(v[0], v[4]);
- u[1] = vaddq_s32(v[1], v[5]);
- u[2] = vaddq_s32(v[2], v[6]);
- u[3] = vaddq_s32(v[3], v[7]);
- u[4] = vsubq_s32(v[0], v[4]);
- u[5] = vsubq_s32(v[1], v[5]);
- u[6] = vsubq_s32(v[2], v[6]);
- u[7] = vsubq_s32(v[3], v[7]);
- u[8] = vaddq_s32(v[8], v[12]);
- u[9] = vaddq_s32(v[9], v[13]);
- u[10] = vaddq_s32(v[10], v[14]);
- u[11] = vaddq_s32(v[11], v[15]);
- u[12] = vsubq_s32(v[8], v[12]);
- u[13] = vsubq_s32(v[9], v[13]);
- u[14] = vsubq_s32(v[10], v[14]);
- u[15] = vsubq_s32(v[11], v[15]);
-
- // stage 6
- v[0] = u[0];
- v[1] = u[1];
- v[2] = u[2];
- v[3] = u[3];
- v[4] = u[4];
- v[5] = u[5];
- v[6] = u[6];
- v[7] = u[7];
-
- v[8] = half_btf_neon(&cospi[8], &u[8], &cospi[56], &u[9], v_bit);
- v[13] = half_btf_neon(&cospi[8], &u[12], &cospi[56], &u[13], v_bit);
- v[9] = half_btf_neon_m(&cospi[56], &u[8], &cospi[8], &u[9], v_bit);
- v[12] = half_btf_neon_m(&cospi[8], &u[13], &cospi[56], &u[12], v_bit);
-
- v[10] = half_btf_neon(&cospi[40], &u[10], &cospi[24], &u[11], v_bit);
- v[15] = half_btf_neon(&cospi[40], &u[14], &cospi[24], &u[15], v_bit);
- v[11] = half_btf_neon_m(&cospi[24], &u[10], &cospi[40], &u[11], v_bit);
- v[14] = half_btf_neon_m(&cospi[40], &u[15], &cospi[24], &u[14], v_bit);
-
- // stage 7
- u[0] = vaddq_s32(v[0], v[8]);
- u[1] = vaddq_s32(v[1], v[9]);
- u[2] = vaddq_s32(v[2], v[10]);
- u[3] = vaddq_s32(v[3], v[11]);
- u[4] = vaddq_s32(v[4], v[12]);
- u[5] = vaddq_s32(v[5], v[13]);
- u[6] = vaddq_s32(v[6], v[14]);
- u[7] = vaddq_s32(v[7], v[15]);
- u[8] = vsubq_s32(v[0], v[8]);
- u[9] = vsubq_s32(v[1], v[9]);
- u[10] = vsubq_s32(v[2], v[10]);
- u[11] = vsubq_s32(v[3], v[11]);
- u[12] = vsubq_s32(v[4], v[12]);
- u[13] = vsubq_s32(v[5], v[13]);
- u[14] = vsubq_s32(v[6], v[14]);
- u[15] = vsubq_s32(v[7], v[15]);
-
- // stage 8
- v[0] = half_btf_neon(&cospi[2], &u[0], &cospi[62], &u[1], v_bit);
- v[1] = half_btf_neon_m(&cospi[62], &u[0], &cospi[2], &u[1], v_bit);
- v[2] = half_btf_neon(&cospi[10], &u[2], &cospi[54], &u[3], v_bit);
- v[3] = half_btf_neon_m(&cospi[54], &u[2], &cospi[10], &u[3], v_bit);
- v[4] = half_btf_neon(&cospi[18], &u[4], &cospi[46], &u[5], v_bit);
- v[5] = half_btf_neon_m(&cospi[46], &u[4], &cospi[18], &u[5], v_bit);
- v[6] = half_btf_neon(&cospi[26], &u[6], &cospi[38], &u[7], v_bit);
- v[7] = half_btf_neon_m(&cospi[38], &u[6], &cospi[26], &u[7], v_bit);
- v[8] = half_btf_neon(&cospi[34], &u[8], &cospi[30], &u[9], v_bit);
- v[9] = half_btf_neon_m(&cospi[30], &u[8], &cospi[34], &u[9], v_bit);
- v[10] = half_btf_neon(&cospi[42], &u[10], &cospi[22], &u[11], v_bit);
- v[11] = half_btf_neon_m(&cospi[22], &u[10], &cospi[42], &u[11], v_bit);
- v[12] = half_btf_neon(&cospi[50], &u[12], &cospi[14], &u[13], v_bit);
- v[13] = half_btf_neon_m(&cospi[14], &u[12], &cospi[50], &u[13], v_bit);
- v[14] = half_btf_neon(&cospi[58], &u[14], &cospi[6], &u[15], v_bit);
- v[15] = half_btf_neon_m(&cospi[6], &u[14], &cospi[58], &u[15], v_bit);
-
- // stage 9
- out[0 * num_cols + col] = v[1];
- out[1 * num_cols + col] = v[14];
- out[2 * num_cols + col] = v[3];
- out[3 * num_cols + col] = v[12];
- out[4 * num_cols + col] = v[5];
- out[5 * num_cols + col] = v[10];
- out[6 * num_cols + col] = v[7];
- out[7 * num_cols + col] = v[8];
- out[8 * num_cols + col] = v[9];
- out[9 * num_cols + col] = v[6];
- out[10 * num_cols + col] = v[11];
- out[11 * num_cols + col] = v[4];
- out[12 * num_cols + col] = v[13];
- out[13 * num_cols + col] = v[2];
- out[14 * num_cols + col] = v[15];
- out[15 * num_cols + col] = v[0];
- }
-}
-
-static void col_txfm_16x16_rounding(int32x4_t *in, const int32x4_t *v_shift) {
- // Note:
- // We split 16x16 rounding into 4 sections of 8x8 rounding,
- // instead of 4 columns
- col_txfm_8x8_rounding(&in[0], v_shift);
- col_txfm_8x8_rounding(&in[16], v_shift);
- col_txfm_8x8_rounding(&in[32], v_shift);
- col_txfm_8x8_rounding(&in[48], v_shift);
-}
-
-static void col_txfm_8x16_rounding(int32x4_t *in, const int32x4_t *v_shift) {
- col_txfm_8x8_rounding(&in[0], v_shift);
- col_txfm_8x8_rounding(&in[16], v_shift);
-}
-
-static void write_buffer_16x16(const int32x4_t *in, int32_t *output) {
- const int size_8x8 = 16 * 4;
- write_buffer_8x8(&in[0], output);
- output += size_8x8;
- write_buffer_8x8(&in[16], output);
- output += size_8x8;
- write_buffer_8x8(&in[32], output);
- output += size_8x8;
- write_buffer_8x8(&in[48], output);
-}
-static void idtx16x16_neon(int32x4_t *in, int32x4_t *out, int bit,
- int col_num) {
- (void)bit;
- int32x4_t fact = vdupq_n_s32(2 * NewSqrt2);
- int32x4_t offset = vdupq_n_s32(1 << (NewSqrt2Bits - 1));
- int32x4_t a_low;
-
- int num_iters = 16 * col_num;
- for (int i = 0; i < num_iters; i++) {
- a_low = vmulq_s32(in[i], fact);
- a_low = vaddq_s32(a_low, offset);
- out[i] = vshrq_n_s32(a_low, NewSqrt2Bits);
- }
-}
-void av1_fwd_txfm2d_16x16_neon(const int16_t *input, int32_t *coeff, int stride,
- TX_TYPE tx_type, int bd) {
- int32x4_t in[64], out[64];
- const int8_t *shift = av1_fwd_txfm_shift_ls[TX_16X16];
- const int txw_idx = get_txw_idx(TX_16X16);
- const int txh_idx = get_txh_idx(TX_16X16);
- const int col_num = 4;
- const int32x4_t v_shift = vdupq_n_s32(shift[1]);
- switch (tx_type) {
- case DCT_DCT:
- load_buffer_16x16(input, in, stride, 0, 0, shift[0]);
- fdct16x16_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], col_num);
- col_txfm_16x16_rounding(out, &v_shift);
- transpose_16x16(out, in);
- fdct16x16_neon(in, out, av1_fwd_cos_bit_row[txw_idx][txh_idx], col_num);
- write_buffer_16x16(out, coeff);
- break;
- case ADST_DCT:
- load_buffer_16x16(input, in, stride, 0, 0, shift[0]);
- fadst16x16_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], col_num);
- col_txfm_16x16_rounding(out, &v_shift);
- transpose_16x16(out, in);
- fdct16x16_neon(in, out, av1_fwd_cos_bit_row[txw_idx][txh_idx], col_num);
- write_buffer_16x16(out, coeff);
- break;
- case DCT_ADST:
- load_buffer_16x16(input, in, stride, 0, 0, shift[0]);
- fdct16x16_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], col_num);
- col_txfm_16x16_rounding(out, &v_shift);
- transpose_16x16(out, in);
- fadst16x16_neon(in, out, av1_fwd_cos_bit_row[txw_idx][txh_idx], col_num);
- write_buffer_16x16(out, coeff);
- break;
- case ADST_ADST:
- load_buffer_16x16(input, in, stride, 0, 0, shift[0]);
- fadst16x16_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], col_num);
- col_txfm_16x16_rounding(out, &v_shift);
- transpose_16x16(out, in);
- fadst16x16_neon(in, out, av1_fwd_cos_bit_row[txw_idx][txh_idx], col_num);
- write_buffer_16x16(out, coeff);
- break;
- case FLIPADST_DCT:
- load_buffer_16x16(input, in, stride, 1, 0, shift[0]);
- fadst16x16_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], col_num);
- col_txfm_16x16_rounding(out, &v_shift);
- transpose_16x16(out, in);
- fdct16x16_neon(in, out, av1_fwd_cos_bit_row[txw_idx][txh_idx], col_num);
- write_buffer_16x16(out, coeff);
- break;
- case DCT_FLIPADST:
- load_buffer_16x16(input, in, stride, 0, 1, shift[0]);
- fdct16x16_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], col_num);
- col_txfm_16x16_rounding(out, &v_shift);
- transpose_16x16(out, in);
- fadst16x16_neon(in, out, av1_fwd_cos_bit_row[txw_idx][txh_idx], col_num);
- write_buffer_16x16(out, coeff);
- break;
- case FLIPADST_FLIPADST:
- load_buffer_16x16(input, in, stride, 1, 1, shift[0]);
- fadst16x16_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], col_num);
- col_txfm_16x16_rounding(out, &v_shift);
- transpose_16x16(out, in);
- fadst16x16_neon(in, out, av1_fwd_cos_bit_row[txw_idx][txh_idx], col_num);
- write_buffer_16x16(out, coeff);
- break;
- case ADST_FLIPADST:
- load_buffer_16x16(input, in, stride, 0, 1, shift[0]);
- fadst16x16_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], col_num);
- col_txfm_16x16_rounding(out, &v_shift);
- transpose_16x16(out, in);
- fadst16x16_neon(in, out, av1_fwd_cos_bit_row[txw_idx][txh_idx], col_num);
- write_buffer_16x16(out, coeff);
- break;
- case FLIPADST_ADST:
- load_buffer_16x16(input, in, stride, 1, 0, shift[0]);
- fadst16x16_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], col_num);
- col_txfm_16x16_rounding(out, &v_shift);
- transpose_16x16(out, in);
- fadst16x16_neon(in, out, av1_fwd_cos_bit_row[txw_idx][txh_idx], col_num);
- write_buffer_16x16(out, coeff);
- break;
- case IDTX:
- load_buffer_16x16(input, in, stride, 0, 0, shift[0]);
- idtx16x16_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], col_num);
- col_txfm_16x16_rounding(out, &v_shift);
- transpose_16x16(out, in);
- idtx16x16_neon(in, out, av1_fwd_cos_bit_row[txw_idx][txh_idx], col_num);
- write_buffer_16x16(out, coeff);
- break;
- case V_DCT:
- load_buffer_16x16(input, in, stride, 0, 0, shift[0]);
- fdct16x16_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], col_num);
- col_txfm_16x16_rounding(out, &v_shift);
- transpose_16x16(out, in);
- idtx16x16_neon(in, out, av1_fwd_cos_bit_row[txw_idx][txh_idx], col_num);
- write_buffer_16x16(out, coeff);
- break;
- case H_DCT:
- load_buffer_16x16(input, in, stride, 0, 0, shift[0]);
- idtx16x16_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], col_num);
- col_txfm_16x16_rounding(out, &v_shift);
- transpose_16x16(out, in);
- fdct16x16_neon(in, out, av1_fwd_cos_bit_row[txw_idx][txh_idx], col_num);
- write_buffer_16x16(out, coeff);
- break;
- case V_ADST:
- load_buffer_16x16(input, in, stride, 0, 0, shift[0]);
- fadst16x16_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], col_num);
- col_txfm_16x16_rounding(out, &v_shift);
- transpose_16x16(out, in);
- idtx16x16_neon(in, out, av1_fwd_cos_bit_row[txw_idx][txh_idx], col_num);
- write_buffer_16x16(out, coeff);
- break;
- case H_ADST:
- load_buffer_16x16(input, in, stride, 0, 0, shift[0]);
- idtx16x16_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], col_num);
- col_txfm_16x16_rounding(out, &v_shift);
- transpose_16x16(out, in);
- fadst16x16_neon(in, out, av1_fwd_cos_bit_row[txw_idx][txh_idx], col_num);
- write_buffer_16x16(out, coeff);
- break;
- case V_FLIPADST:
- load_buffer_16x16(input, in, stride, 1, 0, shift[0]);
- fadst16x16_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], col_num);
- col_txfm_16x16_rounding(out, &v_shift);
- transpose_16x16(out, in);
- idtx16x16_neon(in, out, av1_fwd_cos_bit_row[txw_idx][txh_idx], col_num);
- write_buffer_16x16(out, coeff);
- break;
- case H_FLIPADST:
- load_buffer_16x16(input, in, stride, 0, 1, shift[0]);
- idtx16x16_neon(in, out, av1_fwd_cos_bit_col[txw_idx][txh_idx], col_num);
- col_txfm_16x16_rounding(out, &v_shift);
- transpose_16x16(out, in);
- fadst16x16_neon(in, out, av1_fwd_cos_bit_row[txw_idx][txh_idx], col_num);
- write_buffer_16x16(out, coeff);
- break;
- default: assert(0);
- }
- (void)bd;
-}
-
-static INLINE void flip_buf_neon(int32x4_t *in, int32x4_t *out, int size) {
- for (int i = 0; i < size; i += 2) in[30 - i] = out[i];
- for (int i = 1; i < size; i += 2) in[size - i] = out[i];
-}
-
-typedef void (*fwd_transform_1d_neon)(int32x4_t *in, int32x4_t *out, int bit,
- const int num_cols);
-
-static const fwd_transform_1d_neon col_highbd_txfm8x8_arr[TX_TYPES] = {
- fdct8x8_neon, // DCT_DCT
- fadst8x8_neon, // ADST_DCT
- fdct8x8_neon, // DCT_ADST
- fadst8x8_neon, // ADST_ADST
- fadst8x8_neon, // FLIPADST_DCT
- fdct8x8_neon, // DCT_FLIPADST
- fadst8x8_neon, // FLIPADST_FLIPADST
- fadst8x8_neon, // ADST_FLIPADST
- fadst8x8_neon, // FLIPADST_ADST
- idtx8x8_neon, // IDTX
- fdct8x8_neon, // V_DCT
- idtx8x8_neon, // H_DCT
- fadst8x8_neon, // V_ADST
- idtx8x8_neon, // H_ADST
- fadst8x8_neon, // V_FLIPADST
- idtx8x8_neon // H_FLIPADST
-};
-#if !CONFIG_REALTIME_ONLY
-static const fwd_transform_1d_neon row_highbd_txfm32x8_arr[TX_TYPES] = {
- fdct8x8_neon, // DCT_DCT
- NULL, // ADST_DCT
- NULL, // DCT_ADST
- NULL, // ADST_ADST
- NULL, // FLIPADST_DCT
- NULL, // DCT_FLIPADST
- NULL, // FLIPADST_FLIPADST
- NULL, // ADST_FLIPADST
- NULL, // FLIPADST-ADST
- idtx32x8_neon, // IDTX
- NULL, // V_DCT
- NULL, // H_DCT
- NULL, // V_ADST
- NULL, // H_ADST
- NULL, // V_FLIPADST
- NULL, // H_FLIPADST
-};
-#endif
-static const fwd_transform_1d_neon col_highbd_txfm4x8_arr[TX_TYPES] = {
- fdct4x8_neon, // DCT_DCT
- fadst8x8_neon, // ADST_DCT
- fdct4x8_neon, // DCT_ADST
- fadst8x8_neon, // ADST_ADST
- fadst8x8_neon, // FLIPADST_DCT
- fdct4x8_neon, // DCT_FLIPADST
- fadst8x8_neon, // FLIPADST_FLIPADST
- fadst8x8_neon, // ADST_FLIPADST
- fadst8x8_neon, // FLIPADST_ADST
- idtx8x8_neon, // IDTX
- fdct4x8_neon, // V_DCT
- idtx8x8_neon, // H_DCT
- fadst8x8_neon, // V_ADST
- idtx8x8_neon, // H_ADST
- fadst8x8_neon, // V_FLIPADST
- idtx8x8_neon // H_FLIPADST
-};
-
-static const fwd_transform_1d_neon row_highbd_txfm8x16_arr[TX_TYPES] = {
- fdct16x16_neon, // DCT_DCT
- fdct16x16_neon, // ADST_DCT
- fadst16x16_neon, // DCT_ADST
- fadst16x16_neon, // ADST_ADST
- fdct16x16_neon, // FLIPADST_DCT
- fadst16x16_neon, // DCT_FLIPADST
- fadst16x16_neon, // FLIPADST_FLIPADST
- fadst16x16_neon, // ADST_FLIPADST
- fadst16x16_neon, // FLIPADST_ADST
- idtx16x16_neon, // IDTX
- idtx16x16_neon, // V_DCT
- fdct16x16_neon, // H_DCT
- idtx16x16_neon, // V_ADST
- fadst16x16_neon, // H_ADST
- idtx16x16_neon, // V_FLIPADST
- fadst16x16_neon // H_FLIPADST
-};
-
-static const fwd_transform_1d_neon col_highbd_txfm8x16_arr[TX_TYPES] = {
- fdct16x16_neon, // DCT_DCT
- fadst16x16_neon, // ADST_DCT
- fdct16x16_neon, // DCT_ADST
- fadst16x16_neon, // ADST_ADST
- fadst16x16_neon, // FLIPADST_DCT
- fdct16x16_neon, // DCT_FLIPADST
- fadst16x16_neon, // FLIPADST_FLIPADST
- fadst16x16_neon, // ADST_FLIPADST
- fadst16x16_neon, // FLIPADST_ADST
- idtx16x16_neon, // IDTX
- fdct16x16_neon, // V_DCT
- idtx16x16_neon, // H_DCT
- fadst16x16_neon, // V_ADST
- idtx16x16_neon, // H_ADST
- fadst16x16_neon, // V_FLIPADST
- idtx16x16_neon // H_FLIPADST
-};
-static const fwd_transform_1d_neon row_highbd_txfm8x8_arr[TX_TYPES] = {
- fdct8x8_neon, // DCT_DCT
- fdct8x8_neon, // ADST_DCT
- fadst8x8_neon, // DCT_ADST
- fadst8x8_neon, // ADST_ADST
- fdct8x8_neon, // FLIPADST_DCT
- fadst8x8_neon, // DCT_FLIPADST
- fadst8x8_neon, // FLIPADST_FLIPADST
- fadst8x8_neon, // ADST_FLIPADST
- fadst8x8_neon, // FLIPADST_ADST
- idtx8x8_neon, // IDTX
- idtx8x8_neon, // V_DCT
- fdct8x8_neon, // H_DCT
- idtx8x8_neon, // V_ADST
- fadst8x8_neon, // H_ADST
- idtx8x8_neon, // V_FLIPADST
- fadst8x8_neon // H_FLIPADST
-};
-
-static const fwd_transform_1d_neon row_highbd_txfm4x8_arr[TX_TYPES] = {
- fdct4x8_neon, // DCT_DCT
- fdct4x8_neon, // ADST_DCT
- fadst8x8_neon, // DCT_ADST
- fadst8x8_neon, // ADST_ADST
- fdct4x8_neon, // FLIPADST_DCT
- fadst8x8_neon, // DCT_FLIPADST
- fadst8x8_neon, // FLIPADST_FLIPADST
- fadst8x8_neon, // ADST_FLIPADST
- fadst8x8_neon, // FLIPADST_ADST
- idtx8x8_neon, // IDTX
- idtx8x8_neon, // V_DCT
- fdct4x8_neon, // H_DCT
- idtx8x8_neon, // V_ADST
- fadst8x8_neon, // H_ADST
- idtx8x8_neon, // V_FLIPADST
- fadst8x8_neon // H_FLIPADST
-};
-
-static const fwd_transform_1d_neon row_highbd_txfm4x4_arr[TX_TYPES] = {
- fdct4x4_neon, // DCT_DCT
- fdct4x4_neon, // ADST_DCT
- fadst4x4_neon, // DCT_ADST
- fadst4x4_neon, // ADST_ADST
- fdct4x4_neon, // FLIPADST_DCT
- fadst4x4_neon, // DCT_FLIPADST
- fadst4x4_neon, // FLIPADST_FLIPADST
- fadst4x4_neon, // ADST_FLIPADST
- fadst4x4_neon, // FLIPADST_ADST
- idtx4x4_neon, // IDTX
- idtx4x4_neon, // V_DCT
- fdct4x4_neon, // H_DCT
- idtx4x4_neon, // V_ADST
- fadst4x4_neon, // H_ADST
- idtx4x4_neon, // V_FLIPADST
- fadst4x4_neon // H_FLIPADST
-};
-
-static const fwd_transform_1d_neon col_highbd_txfm4x4_arr[TX_TYPES] = {
- fdct4x4_neon, // DCT_DCT
- fadst4x4_neon, // ADST_DCT
- fdct4x4_neon, // DCT_ADST
- fadst4x4_neon, // ADST_ADST
- fadst4x4_neon, // FLIPADST_DCT
- fdct4x4_neon, // DCT_FLIPADST
- fadst4x4_neon, // FLIPADST_FLIPADST
- fadst4x4_neon, // ADST_FLIPADST
- fadst4x4_neon, // FLIPADST_ADST
- idtx4x4_neon, // IDTX
- fdct4x4_neon, // V_DCT
- idtx4x4_neon, // H_DCT
- fadst4x4_neon, // V_ADST
- idtx4x4_neon, // H_ADST
- fadst4x4_neon, // V_FLIPADST
- idtx4x4_neon // H_FLIPADST
-};
-
-void av1_fdct32_new_neon(int32x4_t *input, int32x4_t *output, int cos_bit,
- const int stride) {
- int32x4_t buf0[32];
- int32x4_t buf1[32];
- const int32_t *cospi;
- const int32x4_t v_cos_bit = vdupq_n_s32(-cos_bit);
-
- int startidx = 0 * stride;
- int endidx = 31 * stride;
- // stage 0
- // stage 1
- buf1[0] = vaddq_s32(input[startidx], input[endidx]);
- buf1[31] = vsubq_s32(input[startidx], input[endidx]);
- startidx += stride;
- endidx -= stride;
- buf1[1] = vaddq_s32(input[startidx], input[endidx]);
- buf1[30] = vsubq_s32(input[startidx], input[endidx]);
- startidx += stride;
- endidx -= stride;
- buf1[2] = vaddq_s32(input[startidx], input[endidx]);
- buf1[29] = vsubq_s32(input[startidx], input[endidx]);
- startidx += stride;
- endidx -= stride;
- buf1[3] = vaddq_s32(input[startidx], input[endidx]);
- buf1[28] = vsubq_s32(input[startidx], input[endidx]);
- startidx += stride;
- endidx -= stride;
- buf1[4] = vaddq_s32(input[startidx], input[endidx]);
- buf1[27] = vsubq_s32(input[startidx], input[endidx]);
- startidx += stride;
- endidx -= stride;
- buf1[5] = vaddq_s32(input[startidx], input[endidx]);
- buf1[26] = vsubq_s32(input[startidx], input[endidx]);
- startidx += stride;
- endidx -= stride;
- buf1[6] = vaddq_s32(input[startidx], input[endidx]);
- buf1[25] = vsubq_s32(input[startidx], input[endidx]);
- startidx += stride;
- endidx -= stride;
- buf1[7] = vaddq_s32(input[startidx], input[endidx]);
- buf1[24] = vsubq_s32(input[startidx], input[endidx]);
- startidx += stride;
- endidx -= stride;
- buf1[8] = vaddq_s32(input[startidx], input[endidx]);
- buf1[23] = vsubq_s32(input[startidx], input[endidx]);
- startidx += stride;
- endidx -= stride;
- buf1[9] = vaddq_s32(input[startidx], input[endidx]);
- buf1[22] = vsubq_s32(input[startidx], input[endidx]);
- startidx += stride;
- endidx -= stride;
- buf1[10] = vaddq_s32(input[startidx], input[endidx]);
- buf1[21] = vsubq_s32(input[startidx], input[endidx]);
- startidx += stride;
- endidx -= stride;
- buf1[11] = vaddq_s32(input[startidx], input[endidx]);
- buf1[20] = vsubq_s32(input[startidx], input[endidx]);
- startidx += stride;
- endidx -= stride;
- buf1[12] = vaddq_s32(input[startidx], input[endidx]);
- buf1[19] = vsubq_s32(input[startidx], input[endidx]);
- startidx += stride;
- endidx -= stride;
- buf1[13] = vaddq_s32(input[startidx], input[endidx]);
- buf1[18] = vsubq_s32(input[startidx], input[endidx]);
- startidx += stride;
- endidx -= stride;
- buf1[14] = vaddq_s32(input[startidx], input[endidx]);
- buf1[17] = vsubq_s32(input[startidx], input[endidx]);
- startidx += stride;
- endidx -= stride;
- buf1[15] = vaddq_s32(input[startidx], input[endidx]);
- buf1[16] = vsubq_s32(input[startidx], input[endidx]);
+ // stage 0-1
+ u0 = in[0];
+ u1 = in[7];
+ u2 = in[3];
+ u3 = in[4];
+ u4 = in[1];
+ u5 = in[6];
+ u6 = in[2];
+ u7 = in[5];
// stage 2
- cospi = cospi_arr(cos_bit);
- buf0[0] = vaddq_s32(buf1[0], buf1[15]);
- buf0[15] = vsubq_s32(buf1[0], buf1[15]);
- buf0[1] = vaddq_s32(buf1[1], buf1[14]);
- buf0[14] = vsubq_s32(buf1[1], buf1[14]);
- buf0[2] = vaddq_s32(buf1[2], buf1[13]);
- buf0[13] = vsubq_s32(buf1[2], buf1[13]);
- buf0[3] = vaddq_s32(buf1[3], buf1[12]);
- buf0[12] = vsubq_s32(buf1[3], buf1[12]);
- buf0[4] = vaddq_s32(buf1[4], buf1[11]);
- buf0[11] = vsubq_s32(buf1[4], buf1[11]);
- buf0[5] = vaddq_s32(buf1[5], buf1[10]);
- buf0[10] = vsubq_s32(buf1[5], buf1[10]);
- buf0[6] = vaddq_s32(buf1[6], buf1[9]);
- buf0[9] = vsubq_s32(buf1[6], buf1[9]);
- buf0[7] = vaddq_s32(buf1[7], buf1[8]);
- buf0[8] = vsubq_s32(buf1[7], buf1[8]);
+ v0 = u0;
+ v1 = u1;
+ butterfly_cospi32_0222_neon(cospi, u3, u2, &v2, &v3, v_bit);
+ v4 = u4;
+ v5 = u5;
+ butterfly_cospi32_0002_neon(cospi, u6, u7, &v7, &v6, v_bit);
+
+ // stage 3
+ u0 = vaddq_s32(v0, v2);
+ u1 = vsubq_s32(v3, v1);
+ u2 = vsubq_s32(v0, v2);
+ u3 = vaddq_s32(v1, v3);
+ u4 = vsubq_s32(v6, v4);
+ u5 = vaddq_s32(v5, v7);
+ u6 = vaddq_s32(v4, v6);
+ u7 = vsubq_s32(v5, v7);
+
+ // stage 4
+ v0 = u0;
+ v1 = u1;
+ v2 = u2;
+ v3 = u3;
+
+ butterfly_0112_neon(cospi, 16, u4, u5, &v4, &v5, v_bit);
+ butterfly_0112_neon(cospi, 16, u7, u6, &v6, &v7, v_bit);
+
+ // stage 5
+ u0 = vaddq_s32(v0, v4);
+ u1 = vaddq_s32(v1, v5);
+ u2 = vaddq_s32(v2, v6);
+ u3 = vsubq_s32(v7, v3);
+ u4 = vsubq_s32(v0, v4);
+ u5 = vsubq_s32(v1, v5);
+ u6 = vsubq_s32(v2, v6);
+ u7 = vaddq_s32(v3, v7);
+
+ // stage 6
+ butterfly_0112_neon(cospi, 4, u0, u1, &v0, &v1, v_bit);
+ butterfly_0112_neon(cospi, 20, u2, u3, &v2, &v3, v_bit);
+ butterfly_0130_neon(cospi, 28, u5, u4, &v4, &v5, v_bit);
+ butterfly_0112_neon(cospi, 12, u6, u7, &v7, &v6, v_bit);
+
+ // stage 7
+ out[0] = v1;
+ out[1] = v6;
+ out[2] = v3;
+ out[3] = v4;
+ out[4] = v5;
+ out[5] = v2;
+ out[6] = v7;
+ out[7] = v0;
+}
+
+static AOM_FORCE_INLINE void highbd_fidentity8_x4_neon(const int32x4_t *in,
+ int32x4_t *out,
+ int bit) {
+ (void)bit;
+ out[0] = vshlq_n_s32(in[0], 1);
+ out[1] = vshlq_n_s32(in[1], 1);
+ out[2] = vshlq_n_s32(in[2], 1);
+ out[3] = vshlq_n_s32(in[3], 1);
+ out[4] = vshlq_n_s32(in[4], 1);
+ out[5] = vshlq_n_s32(in[5], 1);
+ out[6] = vshlq_n_s32(in[6], 1);
+ out[7] = vshlq_n_s32(in[7], 1);
+}
+
+static AOM_FORCE_INLINE void highbd_fdct8_xn_neon(const int32x4_t *in,
+ int32x4_t *out, int bit,
+ int howmany) {
+ const int stride = 8;
+ int i = 0;
+ do {
+ highbd_fdct8_x4_neon(in + i * stride, out + i * stride, bit);
+ } while (++i < howmany);
+}
+
+static AOM_FORCE_INLINE void highbd_fadst8_xn_neon(const int32x4_t *in,
+ int32x4_t *out, int bit,
+ int howmany) {
+ const int stride = 8;
+ int i = 0;
+ do {
+ highbd_fadst8_x4_neon(in + i * stride, out + i * stride, bit);
+ } while (++i < howmany);
+}
+
+static AOM_FORCE_INLINE void highbd_fidentity8_xn_neon(const int32x4_t *in,
+ int32x4_t *out, int bit,
+ int howmany) {
+ (void)bit;
+ const int stride = 8;
+ int i = 0;
+ do {
+ highbd_fidentity8_x4_neon(in + i * stride, out + i * stride, bit);
+ } while (++i < howmany);
+}
+
+void av1_fwd_txfm2d_8x8_neon(const int16_t *input, int32_t *coeff, int stride,
+ TX_TYPE tx_type, int bd) {
+ (void)bd;
+
+ int ud_flip, lr_flip;
+ get_flip_cfg(tx_type, &ud_flip, &lr_flip);
+ ud_adjust_input_and_stride(ud_flip, &input, &stride, 8);
+
+ // Workspaces for column/row-wise transforms.
+ int32x4_t buf0[16], buf1[16];
+
+ switch (tx_type) {
+ case DCT_DCT:
+ load_buffer_8x8(input, buf0, stride, 0);
+ highbd_fdct8_xn_neon(buf0, buf0, av1_fwd_cos_bit_col[1][1], 2);
+ shift_right_1_round_s32_x4(buf0, buf0, 16);
+ transpose_arrays_s32_8x8(buf0, buf1);
+ highbd_fdct8_xn_neon(buf1, buf1, av1_fwd_cos_bit_row[1][1], 2);
+ store_buffer_8x8(buf1, coeff, /*stride=*/8);
+ break;
+ case ADST_DCT:
+ load_buffer_8x8(input, buf0, stride, 0);
+ highbd_fadst8_xn_neon(buf0, buf0, av1_fwd_cos_bit_col[1][1], 2);
+ shift_right_1_round_s32_x4(buf0, buf0, 16);
+ transpose_arrays_s32_8x8(buf0, buf1);
+ highbd_fdct8_xn_neon(buf1, buf1, av1_fwd_cos_bit_row[1][1], 2);
+ store_buffer_8x8(buf1, coeff, /*stride=*/8);
+ break;
+ case DCT_ADST:
+ load_buffer_8x8(input, buf0, stride, 0);
+ highbd_fdct8_xn_neon(buf0, buf0, av1_fwd_cos_bit_col[1][1], 2);
+ shift_right_1_round_s32_x4(buf0, buf0, 16);
+ transpose_arrays_s32_8x8(buf0, buf1);
+ highbd_fadst8_xn_neon(buf1, buf1, av1_fwd_cos_bit_row[1][1], 2);
+ store_buffer_8x8(buf1, coeff, /*stride=*/8);
+ break;
+ case ADST_ADST:
+ load_buffer_8x8(input, buf0, stride, 0);
+ highbd_fadst8_xn_neon(buf0, buf0, av1_fwd_cos_bit_col[1][1], 2);
+ shift_right_1_round_s32_x4(buf0, buf0, 16);
+ transpose_arrays_s32_8x8(buf0, buf1);
+ highbd_fadst8_xn_neon(buf1, buf1, av1_fwd_cos_bit_row[1][1], 2);
+ store_buffer_8x8(buf1, coeff, /*stride=*/8);
+ break;
+ case FLIPADST_DCT:
+ load_buffer_8x8(input, buf0, stride, 0);
+ highbd_fadst8_xn_neon(buf0, buf0, av1_fwd_cos_bit_col[1][1], 2);
+ shift_right_1_round_s32_x4(buf0, buf0, 16);
+ transpose_arrays_s32_8x8(buf0, buf1);
+ highbd_fdct8_xn_neon(buf1, buf1, av1_fwd_cos_bit_row[1][1], 2);
+ store_buffer_8x8(buf1, coeff, /*stride=*/8);
+ break;
+ case DCT_FLIPADST:
+ load_buffer_8x8(input, buf0, stride, 1);
+ highbd_fdct8_xn_neon(buf0, buf0, av1_fwd_cos_bit_col[1][1], 2);
+ shift_right_1_round_s32_x4(buf0, buf0, 16);
+ transpose_arrays_s32_8x8(buf0, buf1);
+ highbd_fadst8_xn_neon(buf1, buf1, av1_fwd_cos_bit_row[1][1], 2);
+ store_buffer_8x8(buf1, coeff, /*stride=*/8);
+ break;
+ case FLIPADST_FLIPADST:
+ load_buffer_8x8(input, buf0, stride, 1);
+ highbd_fadst8_xn_neon(buf0, buf0, av1_fwd_cos_bit_col[1][1], 2);
+ shift_right_1_round_s32_x4(buf0, buf0, 16);
+ transpose_arrays_s32_8x8(buf0, buf1);
+ highbd_fadst8_xn_neon(buf1, buf1, av1_fwd_cos_bit_row[1][1], 2);
+ store_buffer_8x8(buf1, coeff, /*stride=*/8);
+ break;
+ case ADST_FLIPADST:
+ load_buffer_8x8(input, buf0, stride, 1);
+ highbd_fadst8_xn_neon(buf0, buf0, av1_fwd_cos_bit_col[1][1], 2);
+ shift_right_1_round_s32_x4(buf0, buf0, 16);
+ transpose_arrays_s32_8x8(buf0, buf1);
+ highbd_fadst8_xn_neon(buf1, buf1, av1_fwd_cos_bit_row[1][1], 2);
+ store_buffer_8x8(buf1, coeff, /*stride=*/8);
+ break;
+ case FLIPADST_ADST:
+ load_buffer_8x8(input, buf0, stride, 0);
+ highbd_fadst8_xn_neon(buf0, buf0, av1_fwd_cos_bit_col[1][1], 2);
+ shift_right_1_round_s32_x4(buf0, buf0, 16);
+ transpose_arrays_s32_8x8(buf0, buf1);
+ highbd_fadst8_xn_neon(buf1, buf1, av1_fwd_cos_bit_row[1][1], 2);
+ store_buffer_8x8(buf1, coeff, /*stride=*/8);
+ break;
+ case IDTX:
+ load_buffer_8x8(input, buf0, stride, 0);
+ highbd_fidentity8_xn_neon(buf0, buf0, av1_fwd_cos_bit_col[1][1], 2);
+ shift_right_1_round_s32_x4(buf0, buf0, 16);
+ transpose_arrays_s32_8x8(buf0, buf1);
+ highbd_fidentity8_xn_neon(buf1, buf1, av1_fwd_cos_bit_col[1][1], 2);
+ store_buffer_8x8(buf1, coeff, /*stride=*/8);
+ break;
+ case V_DCT:
+ load_buffer_8x8(input, buf0, stride, 0);
+ highbd_fdct8_xn_neon(buf0, buf0, av1_fwd_cos_bit_col[1][1], 2);
+ shift_right_1_round_s32_x4(buf0, buf0, 16);
+ transpose_arrays_s32_8x8(buf0, buf1);
+ highbd_fidentity8_xn_neon(buf1, buf1, av1_fwd_cos_bit_col[1][1], 2);
+ store_buffer_8x8(buf1, coeff, /*stride=*/8);
+ break;
+ case H_DCT:
+ load_buffer_8x8(input, buf0, stride, 0);
+ highbd_fidentity8_xn_neon(buf0, buf0, av1_fwd_cos_bit_col[1][1], 2);
+ shift_right_1_round_s32_x4(buf0, buf0, 16);
+ transpose_arrays_s32_8x8(buf0, buf1);
+ highbd_fdct8_xn_neon(buf1, buf1, av1_fwd_cos_bit_col[1][1], 2);
+ store_buffer_8x8(buf1, coeff, /*stride=*/8);
+ break;
+ case V_ADST:
+ load_buffer_8x8(input, buf0, stride, 0);
+ highbd_fadst8_xn_neon(buf0, buf0, av1_fwd_cos_bit_col[1][1], 2);
+ shift_right_1_round_s32_x4(buf0, buf0, 16);
+ transpose_arrays_s32_8x8(buf0, buf1);
+ highbd_fidentity8_xn_neon(buf1, buf1, av1_fwd_cos_bit_col[1][1], 2);
+ store_buffer_8x8(buf1, coeff, /*stride=*/8);
+ break;
+ case H_ADST:
+ load_buffer_8x8(input, buf0, stride, 0);
+ highbd_fidentity8_xn_neon(buf0, buf0, av1_fwd_cos_bit_col[1][1], 2);
+ shift_right_1_round_s32_x4(buf0, buf0, 16);
+ transpose_arrays_s32_8x8(buf0, buf1);
+ highbd_fadst8_xn_neon(buf1, buf1, av1_fwd_cos_bit_col[1][1], 2);
+ store_buffer_8x8(buf1, coeff, /*stride=*/8);
+ break;
+ case V_FLIPADST:
+ load_buffer_8x8(input, buf0, stride, 0);
+ highbd_fadst8_xn_neon(buf0, buf0, av1_fwd_cos_bit_col[1][1], 2);
+ shift_right_1_round_s32_x4(buf0, buf0, 16);
+ transpose_arrays_s32_8x8(buf0, buf1);
+ highbd_fidentity8_xn_neon(buf1, buf1, av1_fwd_cos_bit_col[1][1], 2);
+ store_buffer_8x8(buf1, coeff, /*stride=*/8);
+ break;
+ case H_FLIPADST:
+ load_buffer_8x8(input, buf0, stride, 1);
+ highbd_fidentity8_xn_neon(buf0, buf0, av1_fwd_cos_bit_col[1][1], 2);
+ shift_right_1_round_s32_x4(buf0, buf0, 16);
+ transpose_arrays_s32_8x8(buf0, buf1);
+ highbd_fadst8_xn_neon(buf1, buf1, av1_fwd_cos_bit_col[1][1], 2);
+ store_buffer_8x8(buf1, coeff, /*stride=*/8);
+ break;
+ default: assert(0);
+ }
+}
+
+static void highbd_fdct16_x4_neon(const int32x4_t *in, int32x4_t *out,
+ int bit) {
+ const int32_t *const cospi = cospi_arr_s32(bit);
+ const int32x4_t v_bit = vdupq_n_s32(-bit);
+
+ int32x4_t u[16], v[16];
+
+ // stage 1
+ butterfly_dct_pre(in, u, 16);
+
+ // stage 2
+ butterfly_dct_pre(u, v, 8);
+ v[8] = u[8];
+ v[9] = u[9];
+ butterfly_cospi32_0002_neon(cospi, u[13], u[10], &v[13], &v[10], v_bit);
+ butterfly_cospi32_0002_neon(cospi, u[12], u[11], &v[12], &v[11], v_bit);
+ v[14] = u[14];
+ v[15] = u[15];
+
+ // stage 3
+ butterfly_dct_pre(v, u, 4);
+ u[4] = v[4];
+ butterfly_cospi32_0002_neon(cospi, v[6], v[5], &u[6], &u[5], v_bit);
+ u[7] = v[7];
+ butterfly_dct_post(v + 8, v + 8, u + 8, 8);
+
+ // stage 4
+ butterfly_cospi32_0002_neon(cospi, u[0], u[1], &v[0], &v[1], v_bit);
+ butterfly_0112_neon(cospi, 16, u[3], u[2], &v[2], &v[3], v_bit);
+ butterfly_dct_post(u + 4, u + 4, v + 4, 4);
+ v[8] = u[8];
+ butterfly_0112_neon(cospi, 16, u[14], u[9], &v[14], &v[9], v_bit);
+ butterfly_2312_neon(cospi, 16, u[13], u[10], &v[10], &v[13], v_bit);
+ v[11] = u[11];
+ v[12] = u[12];
+ v[15] = u[15];
+
+ // stage 5
+ u[0] = v[0];
+ u[1] = v[1];
+ u[2] = v[2];
+ u[3] = v[3];
+ butterfly_0112_neon(cospi, 8, v[7], v[4], &u[4], &u[7], v_bit);
+ butterfly_0130_neon(cospi, 24, v[5], v[6], &u[5], &u[6], v_bit);
+ butterfly_dct_post(v + 8, v + 8, u + 8, 4);
+ butterfly_dct_post(v + 12, v + 12, u + 12, 4);
+
+ // stage 6
+ v[0] = u[0];
+ v[1] = u[1];
+ v[2] = u[2];
+ v[3] = u[3];
+ v[4] = u[4];
+ v[5] = u[5];
+ v[6] = u[6];
+ v[7] = u[7];
+ butterfly_0112_neon(cospi, 4, u[15], u[8], &v[8], &v[15], v_bit);
+ butterfly_0130_neon(cospi, 28, u[9], u[14], &v[9], &v[14], v_bit);
+ butterfly_0112_neon(cospi, 20, u[13], u[10], &v[10], &v[13], v_bit);
+ butterfly_0130_neon(cospi, 12, u[11], u[12], &v[11], &v[12], v_bit);
+
+ out[0] = v[0];
+ out[1] = v[8];
+ out[2] = v[4];
+ out[3] = v[12];
+ out[4] = v[2];
+ out[5] = v[10];
+ out[6] = v[6];
+ out[7] = v[14];
+ out[8] = v[1];
+ out[9] = v[9];
+ out[10] = v[5];
+ out[11] = v[13];
+ out[12] = v[3];
+ out[13] = v[11];
+ out[14] = v[7];
+ out[15] = v[15];
+}
+
+static void highbd_fadst16_x4_neon(const int32x4_t *in, int32x4_t *out,
+ int bit) {
+ const int32_t *const cospi = cospi_arr_s32(bit);
+ const int32x4_t v_bit = vdupq_n_s32(-bit);
+
+ int32x4_t u[16], v[16];
+
+ // stage 0-1
+ u[0] = in[0];
+ u[1] = in[15];
+ u[2] = in[7];
+ u[3] = in[8];
+ u[4] = in[3];
+ u[5] = in[12];
+ u[6] = in[4];
+ u[7] = in[11];
+ u[8] = in[1];
+ u[9] = in[14];
+ u[10] = in[6];
+ u[11] = in[9];
+ u[12] = in[2];
+ u[13] = in[13];
+ u[14] = in[5];
+ u[15] = in[10];
+
+ // stage 2
+ v[0] = u[0];
+ v[1] = u[1];
+ butterfly_cospi32_0222_neon(cospi, u[3], u[2], &v[2], &v[3], v_bit);
+ v[4] = u[4];
+ v[5] = u[5];
+ butterfly_cospi32_0002_neon(cospi, u[6], u[7], &v[7], &v[6], v_bit);
+ v[8] = u[8];
+ v[9] = u[9];
+ butterfly_cospi32_0002_neon(cospi, u[10], u[11], &v[11], &v[10], v_bit);
+ v[12] = u[12];
+ v[13] = u[13];
+ butterfly_cospi32_0222_neon(cospi, u[15], u[14], &v[14], &v[15], v_bit);
+
+ // stage 3
+ u[0] = vaddq_s32(v[0], v[2]);
+ u[1] = vsubq_s32(v[3], v[1]);
+ u[2] = vsubq_s32(v[0], v[2]);
+ u[3] = vaddq_s32(v[1], v[3]);
+ u[4] = vsubq_s32(v[6], v[4]);
+ u[5] = vaddq_s32(v[5], v[7]);
+ u[6] = vaddq_s32(v[4], v[6]);
+ u[7] = vsubq_s32(v[5], v[7]);
+ u[8] = vsubq_s32(v[10], v[8]);
+ u[9] = vaddq_s32(v[9], v[11]);
+ u[10] = vaddq_s32(v[8], v[10]);
+ u[11] = vsubq_s32(v[9], v[11]);
+ u[12] = vaddq_s32(v[12], v[14]);
+ u[13] = vsubq_s32(v[15], v[13]);
+ u[14] = vsubq_s32(v[12], v[14]);
+ u[15] = vaddq_s32(v[13], v[15]);
+
+ // stage 4
+ v[0] = u[0];
+ v[1] = u[1];
+ v[2] = u[2];
+ v[3] = u[3];
+ butterfly_0112_neon(cospi, 16, u[4], u[5], &v[4], &v[5], v_bit);
+ butterfly_0112_neon(cospi, 16, u[7], u[6], &v[6], &v[7], v_bit);
+
+ v[8] = u[8];
+ v[9] = u[9];
+ v[10] = u[10];
+ v[11] = u[11];
+
+ butterfly_0112_neon(cospi, 16, u[12], u[13], &v[12], &v[13], v_bit);
+ butterfly_0332_neon(cospi, 16, u[14], u[15], &v[15], &v[14], v_bit);
+
+ // stage 5
+ u[0] = vaddq_s32(v[0], v[4]);
+ u[1] = vaddq_s32(v[1], v[5]);
+ u[2] = vaddq_s32(v[2], v[6]);
+ u[3] = vsubq_s32(v[7], v[3]);
+ u[4] = vsubq_s32(v[0], v[4]);
+ u[5] = vsubq_s32(v[1], v[5]);
+ u[6] = vsubq_s32(v[2], v[6]);
+ u[7] = vaddq_s32(v[3], v[7]);
+ u[8] = vaddq_s32(v[8], v[12]);
+ u[9] = vaddq_s32(v[9], v[13]);
+ u[10] = vsubq_s32(v[14], v[10]);
+ u[11] = vaddq_s32(v[11], v[15]);
+ u[12] = vsubq_s32(v[8], v[12]);
+ u[13] = vsubq_s32(v[9], v[13]);
+ u[14] = vaddq_s32(v[10], v[14]);
+ u[15] = vsubq_s32(v[11], v[15]);
+
+ // stage 6
+ v[0] = u[0];
+ v[1] = u[1];
+ v[2] = u[2];
+ v[3] = u[3];
+ v[4] = u[4];
+ v[5] = u[5];
+ v[6] = u[6];
+ v[7] = u[7];
+
+ butterfly_0112_neon(cospi, 8, u[8], u[9], &v[8], &v[9], v_bit);
+ butterfly_0130_neon(cospi, 8, u[12], u[13], &v[13], &v[12], v_bit);
+ butterfly_0130_neon(cospi, 24, u[11], u[10], &v[10], &v[11], v_bit);
+ butterfly_0130_neon(cospi, 24, u[14], u[15], &v[14], &v[15], v_bit);
+
+ // stage 7
+ u[0] = vaddq_s32(v[0], v[8]);
+ u[1] = vaddq_s32(v[1], v[9]);
+ u[2] = vaddq_s32(v[2], v[10]);
+ u[3] = vaddq_s32(v[3], v[11]);
+ u[4] = vaddq_s32(v[4], v[12]);
+ u[5] = vaddq_s32(v[5], v[13]);
+ u[6] = vaddq_s32(v[6], v[14]);
+ u[7] = vsubq_s32(v[15], v[7]);
+ u[8] = vsubq_s32(v[0], v[8]);
+ u[9] = vsubq_s32(v[1], v[9]);
+ u[10] = vsubq_s32(v[2], v[10]);
+ u[11] = vsubq_s32(v[3], v[11]);
+ u[12] = vsubq_s32(v[4], v[12]);
+ u[13] = vsubq_s32(v[5], v[13]);
+ u[14] = vsubq_s32(v[6], v[14]);
+ u[15] = vaddq_s32(v[7], v[15]);
+
+ // stage 8
+ butterfly_0112_neon(cospi, 2, u[0], u[1], &v[0], &v[1], v_bit);
+ butterfly_0112_neon(cospi, 10, u[2], u[3], &v[2], &v[3], v_bit);
+ butterfly_0112_neon(cospi, 18, u[4], u[5], &v[4], &v[5], v_bit);
+ butterfly_0112_neon(cospi, 26, u[6], u[7], &v[6], &v[7], v_bit);
+ butterfly_0130_neon(cospi, 30, u[9], u[8], &v[8], &v[9], v_bit);
+ butterfly_0130_neon(cospi, 22, u[11], u[10], &v[10], &v[11], v_bit);
+ butterfly_0130_neon(cospi, 14, u[13], u[12], &v[12], &v[13], v_bit);
+ butterfly_0112_neon(cospi, 6, u[14], u[15], &v[15], &v[14], v_bit);
+
+ // stage 9
+ out[0] = v[1];
+ out[1] = v[14];
+ out[2] = v[3];
+ out[3] = v[12];
+ out[4] = v[5];
+ out[5] = v[10];
+ out[6] = v[7];
+ out[7] = v[8];
+ out[8] = v[9];
+ out[9] = v[6];
+ out[10] = v[11];
+ out[11] = v[4];
+ out[12] = v[13];
+ out[13] = v[2];
+ out[14] = v[15];
+ out[15] = v[0];
+}
+
+static void highbd_fidentity16_x4_neon(const int32x4_t *in, int32x4_t *out,
+ int bit) {
+ (void)bit;
+ const int32x4_t fact = vdupq_n_s32(2 * NewSqrt2);
+ const int32x4_t offset = vdupq_n_s32(1 << (NewSqrt2Bits - 1));
+
+ for (int i = 0; i < 16; i++) {
+ int32x4_t a = vmulq_s32(in[i], fact);
+ a = vaddq_s32(a, offset);
+ out[i] = vshrq_n_s32(a, NewSqrt2Bits);
+ }
+}
+
+static void highbd_fdct16_xn_neon(const int32x4_t *in, int32x4_t *out, int bit,
+ const int howmany) {
+ const int stride = 16;
+ int i = 0;
+ do {
+ highbd_fdct16_x4_neon(in + i * stride, out + i * stride, bit);
+ } while (++i < howmany);
+}
+
+static void highbd_fadst16_xn_neon(const int32x4_t *in, int32x4_t *out, int bit,
+ int howmany) {
+ const int stride = 16;
+ int i = 0;
+ do {
+ highbd_fadst16_x4_neon(in + i * stride, out + i * stride, bit);
+ } while (++i < howmany);
+}
+
+static void highbd_fidentity16_xn_neon(const int32x4_t *in, int32x4_t *out,
+ int bit, int howmany) {
+ const int stride = 16;
+ int i = 0;
+ do {
+ highbd_fidentity16_x4_neon(in + i * stride, out + i * stride, bit);
+ } while (++i < howmany);
+}
+
+void av1_fwd_txfm2d_16x16_neon(const int16_t *input, int32_t *coeff, int stride,
+ TX_TYPE tx_type, int bd) {
+ (void)bd;
+ int ud_flip, lr_flip;
+ get_flip_cfg(tx_type, &ud_flip, &lr_flip);
+ ud_adjust_input_and_stride(ud_flip, &input, &stride, 16);
+
+ // Workspaces for column/row-wise transforms.
+ int32x4_t buf0[64], buf1[64];
+
+ switch (tx_type) {
+ case DCT_DCT:
+ load_buffer_16x16(input, buf0, stride, 0);
+ highbd_fdct16_xn_neon(buf0, buf0, av1_fwd_cos_bit_col[2][2], 4);
+ shift_right_2_round_s32_x4(buf0, buf0, 64);
+ transpose_arrays_s32_16x16(buf0, buf1);
+ highbd_fdct16_xn_neon(buf1, buf1, av1_fwd_cos_bit_row[2][2], 4);
+ store_buffer_16x16(buf1, coeff, /*stride=*/16);
+ break;
+ case ADST_DCT:
+ load_buffer_16x16(input, buf0, stride, 0);
+ highbd_fadst16_xn_neon(buf0, buf0, av1_fwd_cos_bit_col[2][2], 4);
+ shift_right_2_round_s32_x4(buf0, buf0, 64);
+ transpose_arrays_s32_16x16(buf0, buf1);
+ highbd_fdct16_xn_neon(buf1, buf1, av1_fwd_cos_bit_row[2][2], 4);
+ store_buffer_16x16(buf1, coeff, /*stride=*/16);
+ break;
+ case DCT_ADST:
+ load_buffer_16x16(input, buf0, stride, 0);
+ highbd_fdct16_xn_neon(buf0, buf0, av1_fwd_cos_bit_col[2][2], 4);
+ shift_right_2_round_s32_x4(buf0, buf0, 64);
+ transpose_arrays_s32_16x16(buf0, buf1);
+ highbd_fadst16_xn_neon(buf1, buf1, av1_fwd_cos_bit_row[2][2], 4);
+ store_buffer_16x16(buf1, coeff, /*stride=*/16);
+ break;
+ case ADST_ADST:
+ load_buffer_16x16(input, buf0, stride, 0);
+ highbd_fadst16_xn_neon(buf0, buf0, av1_fwd_cos_bit_col[2][2], 4);
+ shift_right_2_round_s32_x4(buf0, buf0, 64);
+ transpose_arrays_s32_16x16(buf0, buf1);
+ highbd_fadst16_xn_neon(buf1, buf1, av1_fwd_cos_bit_row[2][2], 4);
+ store_buffer_16x16(buf1, coeff, /*stride=*/16);
+ break;
+ case FLIPADST_DCT:
+ load_buffer_16x16(input, buf0, stride, 0);
+ highbd_fadst16_xn_neon(buf0, buf0, av1_fwd_cos_bit_col[2][2], 4);
+ shift_right_2_round_s32_x4(buf0, buf0, 64);
+ transpose_arrays_s32_16x16(buf0, buf1);
+ highbd_fdct16_xn_neon(buf1, buf1, av1_fwd_cos_bit_row[2][2], 4);
+ store_buffer_16x16(buf1, coeff, /*stride=*/16);
+ break;
+ case DCT_FLIPADST:
+ load_buffer_16x16(input, buf0, stride, 1);
+ highbd_fdct16_xn_neon(buf0, buf0, av1_fwd_cos_bit_col[2][2], 4);
+ shift_right_2_round_s32_x4(buf0, buf0, 64);
+ transpose_arrays_s32_16x16(buf0, buf1);
+ highbd_fadst16_xn_neon(buf1, buf1, av1_fwd_cos_bit_row[2][2], 4);
+ store_buffer_16x16(buf1, coeff, /*stride=*/16);
+ break;
+ case FLIPADST_FLIPADST:
+ load_buffer_16x16(input, buf0, stride, 1);
+ highbd_fadst16_xn_neon(buf0, buf0, av1_fwd_cos_bit_col[2][2], 4);
+ shift_right_2_round_s32_x4(buf0, buf0, 64);
+ transpose_arrays_s32_16x16(buf0, buf1);
+ highbd_fadst16_xn_neon(buf1, buf1, av1_fwd_cos_bit_row[2][2], 4);
+ store_buffer_16x16(buf1, coeff, /*stride=*/16);
+ break;
+ case ADST_FLIPADST:
+ load_buffer_16x16(input, buf0, stride, 1);
+ highbd_fadst16_xn_neon(buf0, buf0, av1_fwd_cos_bit_col[2][2], 4);
+ shift_right_2_round_s32_x4(buf0, buf0, 64);
+ transpose_arrays_s32_16x16(buf0, buf1);
+ highbd_fadst16_xn_neon(buf1, buf1, av1_fwd_cos_bit_row[2][2], 4);
+ store_buffer_16x16(buf1, coeff, /*stride=*/16);
+ break;
+ case FLIPADST_ADST:
+ load_buffer_16x16(input, buf0, stride, 0);
+ highbd_fadst16_xn_neon(buf0, buf0, av1_fwd_cos_bit_col[2][2], 4);
+ shift_right_2_round_s32_x4(buf0, buf0, 64);
+ transpose_arrays_s32_16x16(buf0, buf1);
+ highbd_fadst16_xn_neon(buf1, buf1, av1_fwd_cos_bit_row[2][2], 4);
+ store_buffer_16x16(buf1, coeff, /*stride=*/16);
+ break;
+ case IDTX:
+ load_buffer_16x16(input, buf0, stride, 0);
+ highbd_fidentity16_xn_neon(buf0, buf0, av1_fwd_cos_bit_col[2][2], 4);
+ shift_right_2_round_s32_x4(buf0, buf0, 64);
+ transpose_arrays_s32_16x16(buf0, buf1);
+ highbd_fidentity16_xn_neon(buf1, buf1, av1_fwd_cos_bit_row[2][2], 4);
+ store_buffer_16x16(buf1, coeff, /*stride=*/16);
+ break;
+ case V_DCT:
+ load_buffer_16x16(input, buf0, stride, 0);
+ highbd_fdct16_xn_neon(buf0, buf0, av1_fwd_cos_bit_col[2][2], 4);
+ shift_right_2_round_s32_x4(buf0, buf0, 64);
+ transpose_arrays_s32_16x16(buf0, buf1);
+ highbd_fidentity16_xn_neon(buf1, buf1, av1_fwd_cos_bit_row[2][2], 4);
+ store_buffer_16x16(buf1, coeff, /*stride=*/16);
+ break;
+ case H_DCT:
+ load_buffer_16x16(input, buf0, stride, 0);
+ highbd_fidentity16_xn_neon(buf0, buf0, av1_fwd_cos_bit_col[2][2], 4);
+ shift_right_2_round_s32_x4(buf0, buf0, 64);
+ transpose_arrays_s32_16x16(buf0, buf1);
+ highbd_fdct16_xn_neon(buf1, buf1, av1_fwd_cos_bit_row[2][2], 4);
+ store_buffer_16x16(buf1, coeff, /*stride=*/16);
+ break;
+ case V_ADST:
+ load_buffer_16x16(input, buf0, stride, 0);
+ highbd_fadst16_xn_neon(buf0, buf0, av1_fwd_cos_bit_col[2][2], 4);
+ shift_right_2_round_s32_x4(buf0, buf0, 64);
+ transpose_arrays_s32_16x16(buf0, buf1);
+ highbd_fidentity16_xn_neon(buf1, buf1, av1_fwd_cos_bit_row[2][2], 4);
+ store_buffer_16x16(buf1, coeff, /*stride=*/16);
+ break;
+ case H_ADST:
+ load_buffer_16x16(input, buf0, stride, 0);
+ highbd_fidentity16_xn_neon(buf0, buf0, av1_fwd_cos_bit_col[2][2], 4);
+ shift_right_2_round_s32_x4(buf0, buf0, 64);
+ transpose_arrays_s32_16x16(buf0, buf1);
+ highbd_fadst16_xn_neon(buf1, buf1, av1_fwd_cos_bit_row[2][2], 4);
+ store_buffer_16x16(buf1, coeff, /*stride=*/16);
+ break;
+ case V_FLIPADST:
+ load_buffer_16x16(input, buf0, stride, 0);
+ highbd_fadst16_xn_neon(buf0, buf0, av1_fwd_cos_bit_col[2][2], 4);
+ shift_right_2_round_s32_x4(buf0, buf0, 64);
+ transpose_arrays_s32_16x16(buf0, buf1);
+ highbd_fidentity16_xn_neon(buf1, buf1, av1_fwd_cos_bit_row[2][2], 4);
+ store_buffer_16x16(buf1, coeff, /*stride=*/16);
+ break;
+ case H_FLIPADST:
+ load_buffer_16x16(input, buf0, stride, 1);
+ highbd_fidentity16_xn_neon(buf0, buf0, av1_fwd_cos_bit_col[2][2], 4);
+ shift_right_2_round_s32_x4(buf0, buf0, 64);
+ transpose_arrays_s32_16x16(buf0, buf1);
+ highbd_fadst16_xn_neon(buf1, buf1, av1_fwd_cos_bit_row[2][2], 4);
+ store_buffer_16x16(buf1, coeff, /*stride=*/16);
+ break;
+ default: assert(0);
+ }
+}
+
+typedef void (*fwd_transform_1d_col_neon)(const int16_t *in, int32x4_t *out,
+ int stride, int bit, int lr_flip);
+typedef void (*fwd_transform_1d_col_many_neon)(const int16_t *in,
+ int32x4_t *out, int stride,
+ int bit, int lr_flip,
+ int howmany, int hm_stride);
+
+typedef void (*fwd_transform_1d_row_neon)(const int32x4_t *in, int32_t *out,
+ int bit, int stride);
+typedef void (*fwd_transform_1d_row_many_neon)(const int32x4_t *in,
+ int32_t *out, int bit,
+ int howmany, int hm_stride,
+ int stride);
+
+// Construct component kernels that include the load_buffer and store_buffer
+// stages to avoid the need to spill loaded data to the stack between these and
+// the txfm kernel calls.
+// The TRANSFORM_*_ONE cases are only ever called in situations where the
+// howmany parameter would be one, so no need for the loop at all in these
+// cases.
+
+#define TRANSFORM_COL_ONE(name, n) \
+ static void highbd_##name##_col_neon(const int16_t *input, \
+ int32x4_t *output, int stride, \
+ int cos_bit, int lr_flip) { \
+ int32x4_t buf0[n]; \
+ load_buffer_4x##n(input, buf0, stride, lr_flip); \
+ highbd_##name##_x4_neon(buf0, output, cos_bit); \
+ }
+
+#define TRANSFORM_COL_MANY(name, n) \
+ static void highbd_##name##_col_many_neon( \
+ const int16_t *input, int32x4_t *output, int stride, int cos_bit, \
+ int lr_flip, int howmany, int hm_stride) { \
+ int i = 0; \
+ do { \
+ int32x4_t buf0[n]; \
+ load_buffer_4x##n(input + 4 * i, buf0, stride, lr_flip); \
+ highbd_##name##_x4_neon(buf0, output + i * hm_stride, cos_bit); \
+ } while (++i < howmany); \
+ }
+
+#define TRANSFORM_ROW_ONE(name, n) \
+ static void highbd_##name##_row_neon( \
+ const int32x4_t *input, int32_t *output, int cos_bit, int stride) { \
+ int32x4_t buf0[n]; \
+ highbd_##name##_x4_neon(input, buf0, cos_bit); \
+ store_buffer_##n##x4(buf0, output, stride); \
+ }
+
+#define TRANSFORM_ROW_RECT_ONE(name, n) \
+ static void highbd_##name##_row_rect_neon( \
+ const int32x4_t *input, int32_t *output, int cos_bit, int stride) { \
+ int32x4_t buf0[n]; \
+ highbd_##name##_x4_neon(input, buf0, cos_bit); \
+ round_rect_array_s32_neon(buf0, buf0, (n)); \
+ store_buffer_##n##x4(buf0, output, stride); \
+ }
+
+#define TRANSFORM_ROW_MANY(name, n) \
+ static void highbd_##name##_row_many_neon( \
+ const int32x4_t *input, int32_t *output, int cos_bit, int howmany, \
+ int hm_stride, int stride) { \
+ int i = 0; \
+ do { \
+ int32x4_t buf0[n]; \
+ highbd_##name##_x4_neon(input + hm_stride * i, buf0, cos_bit); \
+ store_buffer_##n##x4(buf0, output + 4 * i, stride); \
+ } while (++i < howmany); \
+ }
+
+#define TRANSFORM_ROW_RECT_MANY(name, n) \
+ static void highbd_##name##_row_rect_many_neon( \
+ const int32x4_t *input, int32_t *output, int cos_bit, int howmany, \
+ int hm_stride, int stride) { \
+ int i = 0; \
+ do { \
+ int32x4_t buf0[n]; \
+ highbd_##name##_x4_neon(input + hm_stride * i, buf0, cos_bit); \
+ round_rect_array_s32_neon(buf0, buf0, (n)); \
+ store_buffer_##n##x4(buf0, output + 4 * i, stride); \
+ } while (++i < howmany); \
+ }
+
+TRANSFORM_COL_ONE(fdct8, 8)
+TRANSFORM_COL_ONE(fadst8, 8)
+TRANSFORM_COL_ONE(fidentity8, 8)
+
+TRANSFORM_COL_MANY(fdct4, 4)
+TRANSFORM_COL_MANY(fdct8, 8)
+TRANSFORM_COL_MANY(fdct16, 16)
+TRANSFORM_COL_MANY(fadst4, 4)
+TRANSFORM_COL_MANY(fadst8, 8)
+TRANSFORM_COL_MANY(fadst16, 16)
+TRANSFORM_COL_MANY(fidentity4, 4)
+TRANSFORM_COL_MANY(fidentity8, 8)
+TRANSFORM_COL_MANY(fidentity16, 16)
+
+TRANSFORM_ROW_ONE(fdct16, 16)
+TRANSFORM_ROW_ONE(fadst16, 16)
+TRANSFORM_ROW_ONE(fidentity16, 16)
+
+TRANSFORM_ROW_RECT_ONE(fdct8, 8)
+TRANSFORM_ROW_RECT_ONE(fadst8, 8)
+TRANSFORM_ROW_RECT_ONE(fidentity8, 8)
+
+#if !CONFIG_REALTIME_ONLY
+TRANSFORM_ROW_MANY(fdct4, 4)
+TRANSFORM_ROW_MANY(fdct8, 8)
+TRANSFORM_ROW_MANY(fadst4, 4)
+TRANSFORM_ROW_MANY(fadst8, 8)
+TRANSFORM_ROW_MANY(fidentity4, 4)
+TRANSFORM_ROW_MANY(fidentity8, 8)
+#endif
+
+TRANSFORM_ROW_RECT_MANY(fdct4, 4)
+TRANSFORM_ROW_RECT_MANY(fdct8, 8)
+TRANSFORM_ROW_RECT_MANY(fdct16, 16)
+TRANSFORM_ROW_RECT_MANY(fadst4, 4)
+TRANSFORM_ROW_RECT_MANY(fadst8, 8)
+TRANSFORM_ROW_RECT_MANY(fadst16, 16)
+TRANSFORM_ROW_RECT_MANY(fidentity4, 4)
+TRANSFORM_ROW_RECT_MANY(fidentity8, 8)
+TRANSFORM_ROW_RECT_MANY(fidentity16, 16)
+
+static const fwd_transform_1d_col_many_neon
+ col_highbd_txfm8_xn_arr[TX_TYPES] = {
+ highbd_fdct8_col_many_neon, // DCT_DCT
+ highbd_fadst8_col_many_neon, // ADST_DCT
+ highbd_fdct8_col_many_neon, // DCT_ADST
+ highbd_fadst8_col_many_neon, // ADST_ADST
+ highbd_fadst8_col_many_neon, // FLIPADST_DCT
+ highbd_fdct8_col_many_neon, // DCT_FLIPADST
+ highbd_fadst8_col_many_neon, // FLIPADST_FLIPADST
+ highbd_fadst8_col_many_neon, // ADST_FLIPADST
+ highbd_fadst8_col_many_neon, // FLIPADST_ADST
+ highbd_fidentity8_col_many_neon, // IDTX
+ highbd_fdct8_col_many_neon, // V_DCT
+ highbd_fidentity8_col_many_neon, // H_DCT
+ highbd_fadst8_col_many_neon, // V_ADST
+ highbd_fidentity8_col_many_neon, // H_ADST
+ highbd_fadst8_col_many_neon, // V_FLIPADST
+ highbd_fidentity8_col_many_neon // H_FLIPADST
+ };
+
+static const fwd_transform_1d_col_neon col_highbd_txfm8_x4_arr[TX_TYPES] = {
+ highbd_fdct8_col_neon, // DCT_DCT
+ highbd_fadst8_col_neon, // ADST_DCT
+ highbd_fdct8_col_neon, // DCT_ADST
+ highbd_fadst8_col_neon, // ADST_ADST
+ highbd_fadst8_col_neon, // FLIPADST_DCT
+ highbd_fdct8_col_neon, // DCT_FLIPADST
+ highbd_fadst8_col_neon, // FLIPADST_FLIPADST
+ highbd_fadst8_col_neon, // ADST_FLIPADST
+ highbd_fadst8_col_neon, // FLIPADST_ADST
+ highbd_fidentity8_col_neon, // IDTX
+ highbd_fdct8_col_neon, // V_DCT
+ highbd_fidentity8_col_neon, // H_DCT
+ highbd_fadst8_col_neon, // V_ADST
+ highbd_fidentity8_col_neon, // H_ADST
+ highbd_fadst8_col_neon, // V_FLIPADST
+ highbd_fidentity8_col_neon // H_FLIPADST
+};
+
+static const fwd_transform_1d_col_many_neon
+ col_highbd_txfm16_xn_arr[TX_TYPES] = {
+ highbd_fdct16_col_many_neon, // DCT_DCT
+ highbd_fadst16_col_many_neon, // ADST_DCT
+ highbd_fdct16_col_many_neon, // DCT_ADST
+ highbd_fadst16_col_many_neon, // ADST_ADST
+ highbd_fadst16_col_many_neon, // FLIPADST_DCT
+ highbd_fdct16_col_many_neon, // DCT_FLIPADST
+ highbd_fadst16_col_many_neon, // FLIPADST_FLIPADST
+ highbd_fadst16_col_many_neon, // ADST_FLIPADST
+ highbd_fadst16_col_many_neon, // FLIPADST_ADST
+ highbd_fidentity16_col_many_neon, // IDTX
+ highbd_fdct16_col_many_neon, // V_DCT
+ highbd_fidentity16_col_many_neon, // H_DCT
+ highbd_fadst16_col_many_neon, // V_ADST
+ highbd_fidentity16_col_many_neon, // H_ADST
+ highbd_fadst16_col_many_neon, // V_FLIPADST
+ highbd_fidentity16_col_many_neon // H_FLIPADST
+ };
+
+static const fwd_transform_1d_col_many_neon
+ col_highbd_txfm4_xn_arr[TX_TYPES] = {
+ highbd_fdct4_col_many_neon, // DCT_DCT
+ highbd_fadst4_col_many_neon, // ADST_DCT
+ highbd_fdct4_col_many_neon, // DCT_ADST
+ highbd_fadst4_col_many_neon, // ADST_ADST
+ highbd_fadst4_col_many_neon, // FLIPADST_DCT
+ highbd_fdct4_col_many_neon, // DCT_FLIPADST
+ highbd_fadst4_col_many_neon, // FLIPADST_FLIPADST
+ highbd_fadst4_col_many_neon, // ADST_FLIPADST
+ highbd_fadst4_col_many_neon, // FLIPADST_ADST
+ highbd_fidentity4_col_many_neon, // IDTX
+ highbd_fdct4_col_many_neon, // V_DCT
+ highbd_fidentity4_col_many_neon, // H_DCT
+ highbd_fadst4_col_many_neon, // V_ADST
+ highbd_fidentity4_col_many_neon, // H_ADST
+ highbd_fadst4_col_many_neon, // V_FLIPADST
+ highbd_fidentity4_col_many_neon // H_FLIPADST
+ };
+
+static const fwd_transform_1d_row_neon row_highbd_txfm16_xn_arr[TX_TYPES] = {
+ highbd_fdct16_row_neon, // DCT_DCT
+ highbd_fdct16_row_neon, // ADST_DCT
+ highbd_fadst16_row_neon, // DCT_ADST
+ highbd_fadst16_row_neon, // ADST_ADST
+ highbd_fdct16_row_neon, // FLIPADST_DCT
+ highbd_fadst16_row_neon, // DCT_FLIPADST
+ highbd_fadst16_row_neon, // FLIPADST_FLIPADST
+ highbd_fadst16_row_neon, // ADST_FLIPADST
+ highbd_fadst16_row_neon, // FLIPADST_ADST
+ highbd_fidentity16_row_neon, // IDTX
+ highbd_fidentity16_row_neon, // V_DCT
+ highbd_fdct16_row_neon, // H_DCT
+ highbd_fidentity16_row_neon, // V_ADST
+ highbd_fadst16_row_neon, // H_ADST
+ highbd_fidentity16_row_neon, // V_FLIPADST
+ highbd_fadst16_row_neon // H_FLIPADST
+};
+
+static const fwd_transform_1d_row_many_neon
+ row_rect_highbd_txfm16_xn_arr[TX_TYPES] = {
+ highbd_fdct16_row_rect_many_neon, // DCT_DCT
+ highbd_fdct16_row_rect_many_neon, // ADST_DCT
+ highbd_fadst16_row_rect_many_neon, // DCT_ADST
+ highbd_fadst16_row_rect_many_neon, // ADST_ADST
+ highbd_fdct16_row_rect_many_neon, // FLIPADST_DCT
+ highbd_fadst16_row_rect_many_neon, // DCT_FLIPADST
+ highbd_fadst16_row_rect_many_neon, // FLIPADST_FLIPADST
+ highbd_fadst16_row_rect_many_neon, // ADST_FLIPADST
+ highbd_fadst16_row_rect_many_neon, // FLIPADST_ADST
+ highbd_fidentity16_row_rect_many_neon, // IDTX
+ highbd_fidentity16_row_rect_many_neon, // V_DCT
+ highbd_fdct16_row_rect_many_neon, // H_DCT
+ highbd_fidentity16_row_rect_many_neon, // V_ADST
+ highbd_fadst16_row_rect_many_neon, // H_ADST
+ highbd_fidentity16_row_rect_many_neon, // V_FLIPADST
+ highbd_fadst16_row_rect_many_neon // H_FLIPADST
+ };
+
+#if !CONFIG_REALTIME_ONLY
+static const fwd_transform_1d_row_many_neon
+ row_highbd_txfm8_xn_arr[TX_TYPES] = {
+ highbd_fdct8_row_many_neon, // DCT_DCT
+ highbd_fdct8_row_many_neon, // ADST_DCT
+ highbd_fadst8_row_many_neon, // DCT_ADST
+ highbd_fadst8_row_many_neon, // ADST_ADST
+ highbd_fdct8_row_many_neon, // FLIPADST_DCT
+ highbd_fadst8_row_many_neon, // DCT_FLIPADST
+ highbd_fadst8_row_many_neon, // FLIPADST_FLIPADST
+ highbd_fadst8_row_many_neon, // ADST_FLIPADST
+ highbd_fadst8_row_many_neon, // FLIPADST_ADST
+ highbd_fidentity8_row_many_neon, // IDTX
+ highbd_fidentity8_row_many_neon, // V_DCT
+ highbd_fdct8_row_many_neon, // H_DCT
+ highbd_fidentity8_row_many_neon, // V_ADST
+ highbd_fadst8_row_many_neon, // H_ADST
+ highbd_fidentity8_row_many_neon, // V_FLIPADST
+ highbd_fadst8_row_many_neon // H_FLIPADST
+ };
+#endif
+
+static const fwd_transform_1d_row_many_neon
+ row_rect_highbd_txfm8_xn_arr[TX_TYPES] = {
+ highbd_fdct8_row_rect_many_neon, // DCT_DCT
+ highbd_fdct8_row_rect_many_neon, // ADST_DCT
+ highbd_fadst8_row_rect_many_neon, // DCT_ADST
+ highbd_fadst8_row_rect_many_neon, // ADST_ADST
+ highbd_fdct8_row_rect_many_neon, // FLIPADST_DCT
+ highbd_fadst8_row_rect_many_neon, // DCT_FLIPADST
+ highbd_fadst8_row_rect_many_neon, // FLIPADST_FLIPADST
+ highbd_fadst8_row_rect_many_neon, // ADST_FLIPADST
+ highbd_fadst8_row_rect_many_neon, // FLIPADST_ADST
+ highbd_fidentity8_row_rect_many_neon, // IDTX
+ highbd_fidentity8_row_rect_many_neon, // V_DCT
+ highbd_fdct8_row_rect_many_neon, // H_DCT
+ highbd_fidentity8_row_rect_many_neon, // V_ADST
+ highbd_fadst8_row_rect_many_neon, // H_ADST
+ highbd_fidentity8_row_rect_many_neon, // V_FLIPADST
+ highbd_fadst8_row_rect_many_neon // H_FLIPADST
+ };
+
+static const fwd_transform_1d_row_neon row_highbd_txfm8_x4_arr[TX_TYPES] = {
+ highbd_fdct8_row_rect_neon, // DCT_DCT
+ highbd_fdct8_row_rect_neon, // ADST_DCT
+ highbd_fadst8_row_rect_neon, // DCT_ADST
+ highbd_fadst8_row_rect_neon, // ADST_ADST
+ highbd_fdct8_row_rect_neon, // FLIPADST_DCT
+ highbd_fadst8_row_rect_neon, // DCT_FLIPADST
+ highbd_fadst8_row_rect_neon, // FLIPADST_FLIPADST
+ highbd_fadst8_row_rect_neon, // ADST_FLIPADST
+ highbd_fadst8_row_rect_neon, // FLIPADST_ADST
+ highbd_fidentity8_row_rect_neon, // IDTX
+ highbd_fidentity8_row_rect_neon, // V_DCT
+ highbd_fdct8_row_rect_neon, // H_DCT
+ highbd_fidentity8_row_rect_neon, // V_ADST
+ highbd_fadst8_row_rect_neon, // H_ADST
+ highbd_fidentity8_row_rect_neon, // V_FLIPADST
+ highbd_fadst8_row_rect_neon // H_FLIPADST
+};
+
+#if !CONFIG_REALTIME_ONLY
+static const fwd_transform_1d_row_many_neon
+ row_highbd_txfm4_xn_arr[TX_TYPES] = {
+ highbd_fdct4_row_many_neon, // DCT_DCT
+ highbd_fdct4_row_many_neon, // ADST_DCT
+ highbd_fadst4_row_many_neon, // DCT_ADST
+ highbd_fadst4_row_many_neon, // ADST_ADST
+ highbd_fdct4_row_many_neon, // FLIPADST_DCT
+ highbd_fadst4_row_many_neon, // DCT_FLIPADST
+ highbd_fadst4_row_many_neon, // FLIPADST_FLIPADST
+ highbd_fadst4_row_many_neon, // ADST_FLIPADST
+ highbd_fadst4_row_many_neon, // FLIPADST_ADST
+ highbd_fidentity4_row_many_neon, // IDTX
+ highbd_fidentity4_row_many_neon, // V_DCT
+ highbd_fdct4_row_many_neon, // H_DCT
+ highbd_fidentity4_row_many_neon, // V_ADST
+ highbd_fadst4_row_many_neon, // H_ADST
+ highbd_fidentity4_row_many_neon, // V_FLIPADST
+ highbd_fadst4_row_many_neon // H_FLIPADST
+ };
+#endif
+
+static const fwd_transform_1d_row_many_neon
+ row_rect_highbd_txfm4_xn_arr[TX_TYPES] = {
+ highbd_fdct4_row_rect_many_neon, // DCT_DCT
+ highbd_fdct4_row_rect_many_neon, // ADST_DCT
+ highbd_fadst4_row_rect_many_neon, // DCT_ADST
+ highbd_fadst4_row_rect_many_neon, // ADST_ADST
+ highbd_fdct4_row_rect_many_neon, // FLIPADST_DCT
+ highbd_fadst4_row_rect_many_neon, // DCT_FLIPADST
+ highbd_fadst4_row_rect_many_neon, // FLIPADST_FLIPADST
+ highbd_fadst4_row_rect_many_neon, // ADST_FLIPADST
+ highbd_fadst4_row_rect_many_neon, // FLIPADST_ADST
+ highbd_fidentity4_row_rect_many_neon, // IDTX
+ highbd_fidentity4_row_rect_many_neon, // V_DCT
+ highbd_fdct4_row_rect_many_neon, // H_DCT
+ highbd_fidentity4_row_rect_many_neon, // V_ADST
+ highbd_fadst4_row_rect_many_neon, // H_ADST
+ highbd_fidentity4_row_rect_many_neon, // V_FLIPADST
+ highbd_fadst4_row_rect_many_neon // H_FLIPADST
+ };
+
+static void highbd_fdct32_x4_neon(const int32x4_t *input, int32x4_t *output,
+ int cos_bit) {
+ const int32_t *const cospi = cospi_arr_s32(cos_bit);
+ const int32x4_t v_cos_bit = vdupq_n_s32(-cos_bit);
+
+ // Workspaces for intermediate transform steps.
+ int32x4_t buf0[32];
+ int32x4_t buf1[32];
+
+ // stage 1
+ butterfly_dct_pre(input, buf1, 32);
+
+ // stage 2
+ butterfly_dct_pre(buf1, buf0, 16);
buf0[16] = buf1[16];
buf0[17] = buf1[17];
buf0[18] = buf1[18];
buf0[19] = buf1[19];
- btf_32_neon_type0(-cospi[32], cospi[32], buf1[20], buf1[27], buf0[20],
- buf0[27], v_cos_bit);
- btf_32_neon_type0(-cospi[32], cospi[32], buf1[21], buf1[26], buf0[21],
- buf0[26], v_cos_bit);
- btf_32_neon_type0(-cospi[32], cospi[32], buf1[22], buf1[25], buf0[22],
- buf0[25], v_cos_bit);
- btf_32_neon_type0(-cospi[32], cospi[32], buf1[23], buf1[24], buf0[23],
- buf0[24], v_cos_bit);
+ butterfly_0112_neon(cospi, 32, buf1[27], buf1[20], &buf0[27], &buf0[20],
+ v_cos_bit);
+ butterfly_0112_neon(cospi, 32, buf1[26], buf1[21], &buf0[26], &buf0[21],
+ v_cos_bit);
+ butterfly_0112_neon(cospi, 32, buf1[25], buf1[22], &buf0[25], &buf0[22],
+ v_cos_bit);
+ butterfly_0112_neon(cospi, 32, buf1[24], buf1[23], &buf0[24], &buf0[23],
+ v_cos_bit);
buf0[28] = buf1[28];
buf0[29] = buf1[29];
buf0[30] = buf1[30];
buf0[31] = buf1[31];
// stage 3
- cospi = cospi_arr(cos_bit);
- buf1[0] = vaddq_s32(buf0[0], buf0[7]);
- buf1[7] = vsubq_s32(buf0[0], buf0[7]);
- buf1[1] = vaddq_s32(buf0[1], buf0[6]);
- buf1[6] = vsubq_s32(buf0[1], buf0[6]);
- buf1[2] = vaddq_s32(buf0[2], buf0[5]);
- buf1[5] = vsubq_s32(buf0[2], buf0[5]);
- buf1[3] = vaddq_s32(buf0[3], buf0[4]);
- buf1[4] = vsubq_s32(buf0[3], buf0[4]);
+ butterfly_dct_pre(buf0, buf1, 8);
buf1[8] = buf0[8];
buf1[9] = buf0[9];
- btf_32_neon_type0(-cospi[32], cospi[32], buf0[10], buf0[13], buf1[10],
- buf1[13], v_cos_bit);
- btf_32_neon_type0(-cospi[32], cospi[32], buf0[11], buf0[12], buf1[11],
- buf1[12], v_cos_bit);
+ butterfly_0112_neon(cospi, 32, buf0[13], buf0[10], &buf1[13], &buf1[10],
+ v_cos_bit);
+ butterfly_0112_neon(cospi, 32, buf0[12], buf0[11], &buf1[12], &buf1[11],
+ v_cos_bit);
buf1[14] = buf0[14];
buf1[15] = buf0[15];
- buf1[16] = vaddq_s32(buf0[16], buf0[23]);
- buf1[23] = vsubq_s32(buf0[16], buf0[23]);
- buf1[17] = vaddq_s32(buf0[17], buf0[22]);
- buf1[22] = vsubq_s32(buf0[17], buf0[22]);
- buf1[18] = vaddq_s32(buf0[18], buf0[21]);
- buf1[21] = vsubq_s32(buf0[18], buf0[21]);
- buf1[19] = vaddq_s32(buf0[19], buf0[20]);
- buf1[20] = vsubq_s32(buf0[19], buf0[20]);
- buf1[24] = vsubq_s32(buf0[31], buf0[24]);
- buf1[31] = vaddq_s32(buf0[31], buf0[24]);
- buf1[25] = vsubq_s32(buf0[30], buf0[25]);
- buf1[30] = vaddq_s32(buf0[30], buf0[25]);
- buf1[26] = vsubq_s32(buf0[29], buf0[26]);
- buf1[29] = vaddq_s32(buf0[29], buf0[26]);
- buf1[27] = vsubq_s32(buf0[28], buf0[27]);
- buf1[28] = vaddq_s32(buf0[28], buf0[27]);
+ butterfly_dct_post(buf0 + 16, buf0 + 16, buf1 + 16, 16);
// stage 4
- cospi = cospi_arr(cos_bit);
- buf0[0] = vaddq_s32(buf1[0], buf1[3]);
- buf0[3] = vsubq_s32(buf1[0], buf1[3]);
- buf0[1] = vaddq_s32(buf1[1], buf1[2]);
- buf0[2] = vsubq_s32(buf1[1], buf1[2]);
+ butterfly_dct_pre(buf1, buf0, 4);
buf0[4] = buf1[4];
- btf_32_neon_type0(-cospi[32], cospi[32], buf1[5], buf1[6], buf0[5], buf0[6],
- v_cos_bit);
+ butterfly_0112_neon(cospi, 32, buf1[6], buf1[5], &buf0[6], &buf0[5],
+ v_cos_bit);
buf0[7] = buf1[7];
- buf0[8] = vaddq_s32(buf1[8], buf1[11]);
- buf0[11] = vsubq_s32(buf1[8], buf1[11]);
- buf0[9] = vaddq_s32(buf1[9], buf1[10]);
- buf0[10] = vsubq_s32(buf1[9], buf1[10]);
- buf0[12] = vsubq_s32(buf1[15], buf1[12]);
- buf0[15] = vaddq_s32(buf1[15], buf1[12]);
- buf0[13] = vsubq_s32(buf1[14], buf1[13]);
- buf0[14] = vaddq_s32(buf1[14], buf1[13]);
+ butterfly_dct_post(buf1 + 8, buf1 + 8, buf0 + 8, 8);
buf0[16] = buf1[16];
buf0[17] = buf1[17];
-
- btf_32_neon_type0(-cospi[16], cospi[48], buf1[18], buf1[29], buf0[18],
- buf0[29], v_cos_bit);
- btf_32_neon_type0(-cospi[16], cospi[48], buf1[19], buf1[28], buf0[19],
- buf0[28], v_cos_bit);
-
- btf_32_neon_type0(-cospi[48], -cospi[16], buf1[20], buf1[27], buf0[20],
- buf0[27], v_cos_bit);
- btf_32_neon_type0(-cospi[48], -cospi[16], buf1[21], buf1[26], buf0[21],
- buf0[26], v_cos_bit);
-
+ butterfly_0112_neon(cospi, 16, buf1[29], buf1[18], &buf0[29], &buf0[18],
+ v_cos_bit);
+ butterfly_0112_neon(cospi, 16, buf1[28], buf1[19], &buf0[28], &buf0[19],
+ v_cos_bit);
+ butterfly_2312_neon(cospi, 16, buf1[27], buf1[20], &buf0[20], &buf0[27],
+ v_cos_bit);
+ butterfly_2312_neon(cospi, 16, buf1[26], buf1[21], &buf0[21], &buf0[26],
+ v_cos_bit);
buf0[22] = buf1[22];
buf0[23] = buf1[23];
buf0[24] = buf1[24];
@@ -2126,72 +1600,46 @@
buf0[31] = buf1[31];
// stage 5
- btf_32_neon_type0(cospi[32], cospi[32], buf0[0], buf0[1], buf1[0], buf1[1],
- v_cos_bit);
-
- btf_32_neon_type1(cospi[48], cospi[16], buf0[2], buf0[3], buf1[2], buf1[3],
- v_cos_bit);
- buf1[4] = vaddq_s32(buf0[4], buf0[5]);
- buf1[5] = vsubq_s32(buf0[4], buf0[5]);
- buf1[6] = vsubq_s32(buf0[7], buf0[6]);
- buf1[7] = vaddq_s32(buf0[7], buf0[6]);
+ butterfly_0112_neon(cospi, 32, buf0[0], buf0[1], &buf1[0], &buf1[1],
+ v_cos_bit);
+ butterfly_0112_neon(cospi, 16, buf0[3], buf0[2], &buf1[2], &buf1[3],
+ v_cos_bit);
+ butterfly_dct_post(buf0 + 4, buf0 + 4, buf1 + 4, 4);
buf1[8] = buf0[8];
- btf_32_neon_type0(-cospi[16], cospi[48], buf0[9], buf0[14], buf1[9], buf1[14],
- v_cos_bit);
- btf_32_neon_type0(-cospi[48], -cospi[16], buf0[10], buf0[13], buf1[10],
- buf1[13], v_cos_bit);
+ butterfly_0112_neon(cospi, 16, buf0[14], buf0[9], &buf1[14], &buf1[9],
+ v_cos_bit);
+ butterfly_2312_neon(cospi, 16, buf0[13], buf0[10], &buf1[10], &buf1[13],
+ v_cos_bit);
buf1[11] = buf0[11];
buf1[12] = buf0[12];
buf1[15] = buf0[15];
- buf1[16] = vaddq_s32(buf0[16], buf0[19]);
- buf1[19] = vsubq_s32(buf0[16], buf0[19]);
- buf1[17] = vaddq_s32(buf0[17], buf0[18]);
- buf1[18] = vsubq_s32(buf0[17], buf0[18]);
- buf1[20] = vsubq_s32(buf0[23], buf0[20]);
- buf1[23] = vaddq_s32(buf0[23], buf0[20]);
- buf1[21] = vsubq_s32(buf0[22], buf0[21]);
- buf1[22] = vaddq_s32(buf0[22], buf0[21]);
- buf1[24] = vaddq_s32(buf0[24], buf0[27]);
- buf1[27] = vsubq_s32(buf0[24], buf0[27]);
- buf1[25] = vaddq_s32(buf0[25], buf0[26]);
- buf1[26] = vsubq_s32(buf0[25], buf0[26]);
- buf1[28] = vsubq_s32(buf0[31], buf0[28]);
- buf1[31] = vaddq_s32(buf0[31], buf0[28]);
- buf1[29] = vsubq_s32(buf0[30], buf0[29]);
- buf1[30] = vaddq_s32(buf0[30], buf0[29]);
+ butterfly_dct_post(buf0 + 16, buf0 + 16, buf1 + 16, 8);
+ butterfly_dct_post(buf0 + 24, buf0 + 24, buf1 + 24, 8);
// stage 6
- cospi = cospi_arr(cos_bit);
buf0[0] = buf1[0];
buf0[1] = buf1[1];
buf0[2] = buf1[2];
buf0[3] = buf1[3];
- btf_32_neon_type1(cospi[56], cospi[8], buf1[4], buf1[7], buf0[4], buf0[7],
- v_cos_bit);
- btf_32_neon_type0(-cospi[8], cospi[56], buf1[17], buf1[30], buf0[17],
- buf0[30], v_cos_bit);
- btf_32_neon_type0(-cospi[56], -cospi[8], buf1[18], buf1[29], buf0[18],
- buf0[29], v_cos_bit);
-
- buf0[8] = vaddq_s32(buf1[8], buf1[9]);
- buf0[9] = vsubq_s32(buf1[8], buf1[9]);
- buf0[10] = vsubq_s32(buf1[11], buf1[10]);
- buf0[11] = vaddq_s32(buf1[11], buf1[10]);
- buf0[12] = vaddq_s32(buf1[12], buf1[13]);
- buf0[13] = vsubq_s32(buf1[12], buf1[13]);
- buf0[14] = vsubq_s32(buf1[15], buf1[14]);
- buf0[15] = vaddq_s32(buf1[15], buf1[14]);
+ butterfly_0112_neon(cospi, 8, buf1[7], buf1[4], &buf0[4], &buf0[7],
+ v_cos_bit);
+ butterfly_0112_neon(cospi, 8, buf1[30], buf1[17], &buf0[30], &buf0[17],
+ v_cos_bit);
+ butterfly_2312_neon(cospi, 8, buf1[29], buf1[18], &buf0[18], &buf0[29],
+ v_cos_bit);
+ butterfly_dct_post(buf1 + 8, buf1 + 8, buf0 + 8, 4);
+ butterfly_dct_post(buf1 + 12, buf1 + 12, buf0 + 12, 4);
buf0[16] = buf1[16];
buf0[19] = buf1[19];
buf0[20] = buf1[20];
- btf_32_neon_type1(cospi[24], cospi[40], buf1[5], buf1[6], buf0[5], buf0[6],
- v_cos_bit);
- btf_32_neon_type0(-cospi[40], cospi[24], buf1[21], buf1[26], buf0[21],
- buf0[26], v_cos_bit);
- btf_32_neon_type0(-cospi[24], -cospi[40], buf1[22], buf1[25], buf0[22],
- buf0[25], v_cos_bit);
+ butterfly_0130_neon(cospi, 24, buf1[5], buf1[6], &buf0[5], &buf0[6],
+ v_cos_bit);
+ butterfly_0130_neon(cospi, 24, buf1[21], buf1[26], &buf0[26], &buf0[21],
+ v_cos_bit);
+ butterfly_0332_neon(cospi, 24, buf1[25], buf1[22], &buf0[25], &buf0[22],
+ v_cos_bit);
buf0[23] = buf1[23];
buf0[24] = buf1[24];
@@ -2200,7 +1648,6 @@
buf0[31] = buf1[31];
// stage 7
- cospi = cospi_arr(cos_bit);
buf1[0] = buf0[0];
buf1[1] = buf0[1];
buf1[2] = buf0[2];
@@ -2209,33 +1656,20 @@
buf1[5] = buf0[5];
buf1[6] = buf0[6];
buf1[7] = buf0[7];
- btf_32_neon_type1(cospi[60], cospi[4], buf0[8], buf0[15], buf1[8], buf1[15],
- v_cos_bit);
- btf_32_neon_type1(cospi[28], cospi[36], buf0[9], buf0[14], buf1[9], buf1[14],
- v_cos_bit);
- btf_32_neon_type1(cospi[44], cospi[20], buf0[10], buf0[13], buf1[10],
- buf1[13], v_cos_bit);
- btf_32_neon_type1(cospi[12], cospi[52], buf0[11], buf0[12], buf1[11],
- buf1[12], v_cos_bit);
- buf1[16] = vaddq_s32(buf0[16], buf0[17]);
- buf1[17] = vsubq_s32(buf0[16], buf0[17]);
- buf1[18] = vsubq_s32(buf0[19], buf0[18]);
- buf1[19] = vaddq_s32(buf0[19], buf0[18]);
- buf1[20] = vaddq_s32(buf0[20], buf0[21]);
- buf1[21] = vsubq_s32(buf0[20], buf0[21]);
- buf1[22] = vsubq_s32(buf0[23], buf0[22]);
- buf1[23] = vaddq_s32(buf0[23], buf0[22]);
- buf1[24] = vaddq_s32(buf0[24], buf0[25]);
- buf1[25] = vsubq_s32(buf0[24], buf0[25]);
- buf1[26] = vsubq_s32(buf0[27], buf0[26]);
- buf1[27] = vaddq_s32(buf0[27], buf0[26]);
- buf1[28] = vaddq_s32(buf0[28], buf0[29]);
- buf1[29] = vsubq_s32(buf0[28], buf0[29]);
- buf1[30] = vsubq_s32(buf0[31], buf0[30]);
- buf1[31] = vaddq_s32(buf0[31], buf0[30]);
+ butterfly_0112_neon(cospi, 4, buf0[15], buf0[8], &buf1[8], &buf1[15],
+ v_cos_bit);
+ butterfly_0130_neon(cospi, 28, buf0[9], buf0[14], &buf1[9], &buf1[14],
+ v_cos_bit);
+ butterfly_0112_neon(cospi, 20, buf0[13], buf0[10], &buf1[10], &buf1[13],
+ v_cos_bit);
+ butterfly_0130_neon(cospi, 12, buf0[11], buf0[12], &buf1[11], &buf1[12],
+ v_cos_bit);
+ butterfly_dct_post(buf0 + 16, buf0 + 16, buf1 + 16, 4);
+ butterfly_dct_post(buf0 + 20, buf0 + 20, buf1 + 20, 4);
+ butterfly_dct_post(buf0 + 24, buf0 + 24, buf1 + 24, 4);
+ butterfly_dct_post(buf0 + 28, buf0 + 28, buf1 + 28, 4);
// stage 8
- cospi = cospi_arr(cos_bit);
buf0[0] = buf1[0];
buf0[1] = buf1[1];
buf0[2] = buf1[2];
@@ -2252,313 +1686,70 @@
buf0[13] = buf1[13];
buf0[14] = buf1[14];
buf0[15] = buf1[15];
- btf_32_neon_type1(cospi[62], cospi[2], buf1[16], buf1[31], buf0[16], buf0[31],
- v_cos_bit);
- btf_32_neon_type1(cospi[30], cospi[34], buf1[17], buf1[30], buf0[17],
- buf0[30], v_cos_bit);
- btf_32_neon_type1(cospi[46], cospi[18], buf1[18], buf1[29], buf0[18],
- buf0[29], v_cos_bit);
- btf_32_neon_type1(cospi[14], cospi[50], buf1[19], buf1[28], buf0[19],
- buf0[28], v_cos_bit);
- btf_32_neon_type1(cospi[54], cospi[10], buf1[20], buf1[27], buf0[20],
- buf0[27], v_cos_bit);
- btf_32_neon_type1(cospi[22], cospi[42], buf1[21], buf1[26], buf0[21],
- buf0[26], v_cos_bit);
- btf_32_neon_type1(cospi[38], cospi[26], buf1[22], buf1[25], buf0[22],
- buf0[25], v_cos_bit);
- btf_32_neon_type1(cospi[6], cospi[58], buf1[23], buf1[24], buf0[23], buf0[24],
- v_cos_bit);
+ butterfly_0112_neon(cospi, 2, buf1[31], buf1[16], &buf0[16], &buf0[31],
+ v_cos_bit);
+ butterfly_0130_neon(cospi, 30, buf1[17], buf1[30], &buf0[17], &buf0[30],
+ v_cos_bit);
+ butterfly_0112_neon(cospi, 18, buf1[29], buf1[18], &buf0[18], &buf0[29],
+ v_cos_bit);
+ butterfly_0130_neon(cospi, 14, buf1[19], buf1[28], &buf0[19], &buf0[28],
+ v_cos_bit);
+ butterfly_0112_neon(cospi, 10, buf1[27], buf1[20], &buf0[20], &buf0[27],
+ v_cos_bit);
+ butterfly_0130_neon(cospi, 22, buf1[21], buf1[26], &buf0[21], &buf0[26],
+ v_cos_bit);
+ butterfly_0112_neon(cospi, 26, buf1[25], buf1[22], &buf0[22], &buf0[25],
+ v_cos_bit);
+ butterfly_0130_neon(cospi, 6, buf1[23], buf1[24], &buf0[23], &buf0[24],
+ v_cos_bit);
- startidx = 0 * stride;
- endidx = 31 * stride;
// stage 9
- output[startidx] = buf0[0];
- output[endidx] = buf0[31];
- startidx += stride;
- endidx -= stride;
- output[startidx] = buf0[16];
- output[endidx] = buf0[15];
- startidx += stride;
- endidx -= stride;
- output[startidx] = buf0[8];
- output[endidx] = buf0[23];
- startidx += stride;
- endidx -= stride;
- output[startidx] = buf0[24];
- output[endidx] = buf0[7];
- startidx += stride;
- endidx -= stride;
- output[startidx] = buf0[4];
- output[endidx] = buf0[27];
- startidx += stride;
- endidx -= stride;
- output[startidx] = buf0[20];
- output[endidx] = buf0[11];
- startidx += stride;
- endidx -= stride;
- output[startidx] = buf0[12];
- output[endidx] = buf0[19];
- startidx += stride;
- endidx -= stride;
- output[startidx] = buf0[28];
- output[endidx] = buf0[3];
- startidx += stride;
- endidx -= stride;
- output[startidx] = buf0[2];
- output[endidx] = buf0[29];
- startidx += stride;
- endidx -= stride;
- output[startidx] = buf0[18];
- output[endidx] = buf0[13];
- startidx += stride;
- endidx -= stride;
- output[startidx] = buf0[10];
- output[endidx] = buf0[21];
- startidx += stride;
- endidx -= stride;
- output[startidx] = buf0[26];
- output[endidx] = buf0[5];
- startidx += stride;
- endidx -= stride;
- output[startidx] = buf0[6];
- output[endidx] = buf0[25];
- startidx += stride;
- endidx -= stride;
- output[startidx] = buf0[22];
- output[endidx] = buf0[9];
- startidx += stride;
- endidx -= stride;
- output[startidx] = buf0[14];
- output[endidx] = buf0[17];
- startidx += stride;
- endidx -= stride;
- output[startidx] = buf0[30];
- output[endidx] = buf0[1];
+ output[0] = buf0[0];
+ output[1] = buf0[16];
+ output[2] = buf0[8];
+ output[3] = buf0[24];
+ output[4] = buf0[4];
+ output[5] = buf0[20];
+ output[6] = buf0[12];
+ output[7] = buf0[28];
+ output[8] = buf0[2];
+ output[9] = buf0[18];
+ output[10] = buf0[10];
+ output[11] = buf0[26];
+ output[12] = buf0[6];
+ output[13] = buf0[22];
+ output[14] = buf0[14];
+ output[15] = buf0[30];
+ output[16] = buf0[1];
+ output[17] = buf0[17];
+ output[18] = buf0[9];
+ output[19] = buf0[25];
+ output[20] = buf0[5];
+ output[21] = buf0[21];
+ output[22] = buf0[13];
+ output[23] = buf0[29];
+ output[24] = buf0[3];
+ output[25] = buf0[19];
+ output[26] = buf0[11];
+ output[27] = buf0[27];
+ output[28] = buf0[7];
+ output[29] = buf0[23];
+ output[30] = buf0[15];
+ output[31] = buf0[31];
}
-void av1_fadst4_new_neon(const int32x4_t *input, int32x4_t *output,
- const int8_t cos_bit, const int8_t *stage_range) {
- const int txfm_size = 4;
- const int num_per_128 = 4;
- const int32_t *cospi;
- int32x4_t v_cos_bit = vdupq_n_s32(-cos_bit);
- int32x4_t buf0[4];
- int32x4_t buf1[4];
- int col_num = txfm_size / num_per_128;
- int col;
- (void)stage_range;
- cospi = cospi_arr(cos_bit);
- for (col = 0; col < col_num; col++) {
- // stage 0;
- int j;
- for (j = 0; j < 4; ++j) {
- buf0[j] = input[j * col_num + col];
- }
+static void highbd_fdct64_x4_neon(const int32x4_t *input, int32x4_t *output,
+ int8_t cos_bit) {
+ const int32_t *const cospi = cospi_arr_s32(cos_bit);
+ const int32x4_t v_cos_bit = vdupq_n_s32(-cos_bit);
- // stage 1
- buf1[0] = buf0[3];
- buf1[1] = buf0[0];
- buf1[2] = buf0[1];
- buf1[3] = buf0[2];
-
- // stage 2
- btf_32_neon_type0(cospi[8], cospi[56], buf1[0], buf1[1], buf0[0], buf0[1],
- v_cos_bit);
- btf_32_neon_type0(cospi[40], cospi[24], buf1[2], buf1[3], buf0[2], buf0[3],
- v_cos_bit);
-
- // stage 3
- buf1[0] = vaddq_s32(buf0[0], buf0[2]);
- buf1[2] = vsubq_s32(buf0[0], buf0[2]);
- buf1[1] = vaddq_s32(buf0[1], buf0[3]);
- buf1[3] = vsubq_s32(buf0[1], buf0[3]);
-
- // stage 4
- cospi = cospi_arr(cos_bit);
- buf0[0] = buf1[0];
- buf0[1] = buf1[1];
-
- btf_32_neon_type0(cospi[32], cospi[32], buf1[2], buf1[3], buf0[2], buf0[3],
- v_cos_bit);
-
- // stage 5
- buf1[0] = buf0[0];
- buf1[1] = vnegq_s32(buf0[2]);
- buf1[2] = buf0[3];
- buf1[3] = vnegq_s32(buf0[1]);
-
- for (j = 0; j < 4; ++j) {
- output[j * col_num + col] = buf1[j];
- }
- }
-}
-
-static void av1_fdct64_new_stage12345_neon(int32x4_t *input, const int instride,
- int32x4_t *x5, const int32_t *cospi,
- const int32x4_t *v_cos_bit,
- int *startidx, int *endidx) {
+ // stage 1
int32x4_t x1[64];
- x1[0] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[63] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[1] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[62] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[2] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[61] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[3] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[60] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[4] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[59] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[5] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[58] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[6] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[57] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[7] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[56] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[8] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[55] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[9] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[54] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[10] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[53] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[11] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[52] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[12] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[51] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[13] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[50] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[14] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[49] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[15] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[48] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[16] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[47] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[17] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[46] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[18] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[45] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[19] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[44] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[20] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[43] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[21] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[42] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[22] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[41] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[23] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[40] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[24] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[39] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[25] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[38] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[26] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[37] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[27] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[36] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[28] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[35] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[29] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[34] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[30] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[33] = vsubq_s32(input[*startidx], input[*endidx]);
- *startidx += instride;
- *endidx -= instride;
- x1[31] = vaddq_s32(input[*startidx], input[*endidx]);
- x1[32] = vsubq_s32(input[*startidx], input[*endidx]);
+ butterfly_dct_pre(input, x1, 64);
// stage 2
int32x4_t x2[64];
- x2[0] = vaddq_s32(x1[0], x1[31]);
- x2[31] = vsubq_s32(x1[0], x1[31]);
- x2[1] = vaddq_s32(x1[1], x1[30]);
- x2[30] = vsubq_s32(x1[1], x1[30]);
- x2[2] = vaddq_s32(x1[2], x1[29]);
- x2[29] = vsubq_s32(x1[2], x1[29]);
- x2[3] = vaddq_s32(x1[3], x1[28]);
- x2[28] = vsubq_s32(x1[3], x1[28]);
- x2[4] = vaddq_s32(x1[4], x1[27]);
- x2[27] = vsubq_s32(x1[4], x1[27]);
- x2[5] = vaddq_s32(x1[5], x1[26]);
- x2[26] = vsubq_s32(x1[5], x1[26]);
- x2[6] = vaddq_s32(x1[6], x1[25]);
- x2[25] = vsubq_s32(x1[6], x1[25]);
- x2[7] = vaddq_s32(x1[7], x1[24]);
- x2[24] = vsubq_s32(x1[7], x1[24]);
- x2[8] = vaddq_s32(x1[8], x1[23]);
- x2[23] = vsubq_s32(x1[8], x1[23]);
- x2[9] = vaddq_s32(x1[9], x1[22]);
- x2[22] = vsubq_s32(x1[9], x1[22]);
- x2[10] = vaddq_s32(x1[10], x1[21]);
- x2[21] = vsubq_s32(x1[10], x1[21]);
- x2[11] = vaddq_s32(x1[11], x1[20]);
- x2[20] = vsubq_s32(x1[11], x1[20]);
- x2[12] = vaddq_s32(x1[12], x1[19]);
- x2[19] = vsubq_s32(x1[12], x1[19]);
- x2[13] = vaddq_s32(x1[13], x1[18]);
- x2[18] = vsubq_s32(x1[13], x1[18]);
- x2[14] = vaddq_s32(x1[14], x1[17]);
- x2[17] = vsubq_s32(x1[14], x1[17]);
- x2[15] = vaddq_s32(x1[15], x1[16]);
- x2[16] = vsubq_s32(x1[15], x1[16]);
+ butterfly_dct_pre(x1, x2, 32);
x2[32] = x1[32];
x2[33] = x1[33];
x2[34] = x1[34];
@@ -2567,23 +1758,14 @@
x2[37] = x1[37];
x2[38] = x1[38];
x2[39] = x1[39];
-
- btf_32_neon_type0(-cospi[32], cospi[32], x1[40], x1[55], x2[40], x2[55],
- *v_cos_bit);
- btf_32_neon_type0(-cospi[32], cospi[32], x1[41], x1[54], x2[41], x2[54],
- *v_cos_bit);
- btf_32_neon_type0(-cospi[32], cospi[32], x1[42], x1[53], x2[42], x2[53],
- *v_cos_bit);
- btf_32_neon_type0(-cospi[32], cospi[32], x1[43], x1[52], x2[43], x2[52],
- *v_cos_bit);
- btf_32_neon_type0(-cospi[32], cospi[32], x1[44], x1[51], x2[44], x2[51],
- *v_cos_bit);
- btf_32_neon_type0(-cospi[32], cospi[32], x1[45], x1[50], x2[45], x2[50],
- *v_cos_bit);
- btf_32_neon_type0(-cospi[32], cospi[32], x1[46], x1[49], x2[46], x2[49],
- *v_cos_bit);
- btf_32_neon_type0(-cospi[32], cospi[32], x1[47], x1[48], x2[47], x2[48],
- *v_cos_bit);
+ butterfly_0112_neon(cospi, 32, x1[55], x1[40], &x2[55], &x2[40], v_cos_bit);
+ butterfly_0112_neon(cospi, 32, x1[54], x1[41], &x2[54], &x2[41], v_cos_bit);
+ butterfly_0112_neon(cospi, 32, x1[53], x1[42], &x2[53], &x2[42], v_cos_bit);
+ butterfly_0112_neon(cospi, 32, x1[52], x1[43], &x2[52], &x2[43], v_cos_bit);
+ butterfly_0112_neon(cospi, 32, x1[51], x1[44], &x2[51], &x2[44], v_cos_bit);
+ butterfly_0112_neon(cospi, 32, x1[50], x1[45], &x2[50], &x2[45], v_cos_bit);
+ butterfly_0112_neon(cospi, 32, x1[49], x1[46], &x2[49], &x2[46], v_cos_bit);
+ butterfly_0112_neon(cospi, 32, x1[48], x1[47], &x2[48], &x2[47], v_cos_bit);
x2[56] = x1[56];
x2[57] = x1[57];
x2[58] = x1[58];
@@ -2595,126 +1777,43 @@
// stage 3
int32x4_t x3[64];
- x3[0] = vaddq_s32(x2[0], x2[15]);
- x3[15] = vsubq_s32(x2[0], x2[15]);
- x3[1] = vaddq_s32(x2[1], x2[14]);
- x3[14] = vsubq_s32(x2[1], x2[14]);
- x3[2] = vaddq_s32(x2[2], x2[13]);
- x3[13] = vsubq_s32(x2[2], x2[13]);
- x3[3] = vaddq_s32(x2[3], x2[12]);
- x3[12] = vsubq_s32(x2[3], x2[12]);
- x3[4] = vaddq_s32(x2[4], x2[11]);
- x3[11] = vsubq_s32(x2[4], x2[11]);
- x3[5] = vaddq_s32(x2[5], x2[10]);
- x3[10] = vsubq_s32(x2[5], x2[10]);
- x3[6] = vaddq_s32(x2[6], x2[9]);
- x3[9] = vsubq_s32(x2[6], x2[9]);
- x3[7] = vaddq_s32(x2[7], x2[8]);
- x3[8] = vsubq_s32(x2[7], x2[8]);
+ butterfly_dct_pre(x2, x3, 16);
x3[16] = x2[16];
x3[17] = x2[17];
x3[18] = x2[18];
x3[19] = x2[19];
- btf_32_neon_type0(-cospi[32], cospi[32], x2[20], x2[27], x3[20], x3[27],
- *v_cos_bit);
- btf_32_neon_type0(-cospi[32], cospi[32], x2[21], x2[26], x3[21], x3[26],
- *v_cos_bit);
- btf_32_neon_type0(-cospi[32], cospi[32], x2[22], x2[25], x3[22], x3[25],
- *v_cos_bit);
- btf_32_neon_type0(-cospi[32], cospi[32], x2[23], x2[24], x3[23], x3[24],
- *v_cos_bit);
+ butterfly_0112_neon(cospi, 32, x2[27], x2[20], &x3[27], &x3[20], v_cos_bit);
+ butterfly_0112_neon(cospi, 32, x2[26], x2[21], &x3[26], &x3[21], v_cos_bit);
+ butterfly_0112_neon(cospi, 32, x2[25], x2[22], &x3[25], &x3[22], v_cos_bit);
+ butterfly_0112_neon(cospi, 32, x2[24], x2[23], &x3[24], &x3[23], v_cos_bit);
x3[28] = x2[28];
x3[29] = x2[29];
x3[30] = x2[30];
x3[31] = x2[31];
- x3[32] = vaddq_s32(x2[32], x2[47]);
- x3[47] = vsubq_s32(x2[32], x2[47]);
- x3[33] = vaddq_s32(x2[33], x2[46]);
- x3[46] = vsubq_s32(x2[33], x2[46]);
- x3[34] = vaddq_s32(x2[34], x2[45]);
- x3[45] = vsubq_s32(x2[34], x2[45]);
- x3[35] = vaddq_s32(x2[35], x2[44]);
- x3[44] = vsubq_s32(x2[35], x2[44]);
- x3[36] = vaddq_s32(x2[36], x2[43]);
- x3[43] = vsubq_s32(x2[36], x2[43]);
- x3[37] = vaddq_s32(x2[37], x2[42]);
- x3[42] = vsubq_s32(x2[37], x2[42]);
- x3[38] = vaddq_s32(x2[38], x2[41]);
- x3[41] = vsubq_s32(x2[38], x2[41]);
- x3[39] = vaddq_s32(x2[39], x2[40]);
- x3[40] = vsubq_s32(x2[39], x2[40]);
- x3[48] = vsubq_s32(x2[63], x2[48]);
- x3[63] = vaddq_s32(x2[63], x2[48]);
- x3[49] = vsubq_s32(x2[62], x2[49]);
- x3[62] = vaddq_s32(x2[62], x2[49]);
- x3[50] = vsubq_s32(x2[61], x2[50]);
- x3[61] = vaddq_s32(x2[61], x2[50]);
- x3[51] = vsubq_s32(x2[60], x2[51]);
- x3[60] = vaddq_s32(x2[60], x2[51]);
- x3[52] = vsubq_s32(x2[59], x2[52]);
- x3[59] = vaddq_s32(x2[59], x2[52]);
- x3[53] = vsubq_s32(x2[58], x2[53]);
- x3[58] = vaddq_s32(x2[58], x2[53]);
- x3[54] = vsubq_s32(x2[57], x2[54]);
- x3[57] = vaddq_s32(x2[57], x2[54]);
- x3[55] = vsubq_s32(x2[56], x2[55]);
- x3[56] = vaddq_s32(x2[56], x2[55]);
+ butterfly_dct_post(x2 + 32, x2 + 32, x3 + 32, 32);
// stage 4
int32x4_t x4[64];
- x4[0] = vaddq_s32(x3[0], x3[7]);
- x4[7] = vsubq_s32(x3[0], x3[7]);
- x4[1] = vaddq_s32(x3[1], x3[6]);
- x4[6] = vsubq_s32(x3[1], x3[6]);
- x4[2] = vaddq_s32(x3[2], x3[5]);
- x4[5] = vsubq_s32(x3[2], x3[5]);
- x4[3] = vaddq_s32(x3[3], x3[4]);
- x4[4] = vsubq_s32(x3[3], x3[4]);
+ butterfly_dct_pre(x3, x4, 8);
x4[8] = x3[8];
x4[9] = x3[9];
- btf_32_neon_type0(-cospi[32], cospi[32], x3[10], x3[13], x4[10], x4[13],
- *v_cos_bit);
- btf_32_neon_type0(-cospi[32], cospi[32], x3[11], x3[12], x4[11], x4[12],
- *v_cos_bit);
+ butterfly_0112_neon(cospi, 32, x3[13], x3[10], &x4[13], &x4[10], v_cos_bit);
+ butterfly_0112_neon(cospi, 32, x3[12], x3[11], &x4[12], &x4[11], v_cos_bit);
x4[14] = x3[14];
x4[15] = x3[15];
- x4[16] = vaddq_s32(x3[16], x3[23]);
- x4[23] = vsubq_s32(x3[16], x3[23]);
- x4[17] = vaddq_s32(x3[17], x3[22]);
- x4[22] = vsubq_s32(x3[17], x3[22]);
- x4[18] = vaddq_s32(x3[18], x3[21]);
- x4[21] = vsubq_s32(x3[18], x3[21]);
- x4[19] = vaddq_s32(x3[19], x3[20]);
- x4[20] = vsubq_s32(x3[19], x3[20]);
- x4[24] = vsubq_s32(x3[31], x3[24]);
- x4[31] = vaddq_s32(x3[31], x3[24]);
- x4[25] = vsubq_s32(x3[30], x3[25]);
- x4[30] = vaddq_s32(x3[30], x3[25]);
- x4[26] = vsubq_s32(x3[29], x3[26]);
- x4[29] = vaddq_s32(x3[29], x3[26]);
- x4[27] = vsubq_s32(x3[28], x3[27]);
- x4[28] = vaddq_s32(x3[28], x3[27]);
+ butterfly_dct_post(x3 + 16, x3 + 16, x4 + 16, 16);
x4[32] = x3[32];
x4[33] = x3[33];
x4[34] = x3[34];
x4[35] = x3[35];
-
- btf_32_neon_type0(-cospi[16], cospi[48], x3[36], x3[59], x4[36], x4[59],
- *v_cos_bit);
- btf_32_neon_type0(-cospi[16], cospi[48], x3[37], x3[58], x4[37], x4[58],
- *v_cos_bit);
- btf_32_neon_type0(-cospi[16], cospi[48], x3[38], x3[57], x4[38], x4[57],
- *v_cos_bit);
- btf_32_neon_type0(-cospi[16], cospi[48], x3[39], x3[56], x4[39], x4[56],
- *v_cos_bit);
- btf_32_neon_type0(-cospi[48], -cospi[16], x3[40], x3[55], x4[40], x4[55],
- *v_cos_bit);
- btf_32_neon_type0(-cospi[48], -cospi[16], x3[41], x3[54], x4[41], x4[54],
- *v_cos_bit);
- btf_32_neon_type0(-cospi[48], -cospi[16], x3[42], x3[53], x4[42], x4[53],
- *v_cos_bit);
- btf_32_neon_type0(-cospi[48], -cospi[16], x3[43], x3[52], x4[43], x4[52],
- *v_cos_bit);
+ butterfly_0112_neon(cospi, 16, x3[59], x3[36], &x4[59], &x4[36], v_cos_bit);
+ butterfly_0112_neon(cospi, 16, x3[58], x3[37], &x4[58], &x4[37], v_cos_bit);
+ butterfly_0112_neon(cospi, 16, x3[57], x3[38], &x4[57], &x4[38], v_cos_bit);
+ butterfly_0112_neon(cospi, 16, x3[56], x3[39], &x4[56], &x4[39], v_cos_bit);
+ butterfly_2312_neon(cospi, 16, x3[55], x3[40], &x4[40], &x4[55], v_cos_bit);
+ butterfly_2312_neon(cospi, 16, x3[54], x3[41], &x4[41], &x4[54], v_cos_bit);
+ butterfly_2312_neon(cospi, 16, x3[53], x3[42], &x4[42], &x4[53], v_cos_bit);
+ butterfly_2312_neon(cospi, 16, x3[52], x3[43], &x4[43], &x4[52], v_cos_bit);
x4[44] = x3[44];
x4[45] = x3[45];
x4[46] = x3[46];
@@ -2729,134 +1828,54 @@
x4[63] = x3[63];
// stage 5
- x5[0] = vaddq_s32(x4[0], x4[3]);
- x5[3] = vsubq_s32(x4[0], x4[3]);
- x5[1] = vaddq_s32(x4[1], x4[2]);
- x5[2] = vsubq_s32(x4[1], x4[2]);
+ int32x4_t x5[64];
+ butterfly_dct_pre(x4, x5, 4);
x5[4] = x4[4];
-
- btf_32_neon_type0(-cospi[32], cospi[32], x4[5], x4[6], x5[5], x5[6],
- *v_cos_bit);
+ butterfly_0112_neon(cospi, 32, x4[6], x4[5], &x5[6], &x5[5], v_cos_bit);
x5[7] = x4[7];
- x5[8] = vaddq_s32(x4[8], x4[11]);
- x5[11] = vsubq_s32(x4[8], x4[11]);
- x5[9] = vaddq_s32(x4[9], x4[10]);
- x5[10] = vsubq_s32(x4[9], x4[10]);
- x5[12] = vsubq_s32(x4[15], x4[12]);
- x5[15] = vaddq_s32(x4[15], x4[12]);
- x5[13] = vsubq_s32(x4[14], x4[13]);
- x5[14] = vaddq_s32(x4[14], x4[13]);
+ butterfly_dct_post(x4 + 8, x4 + 8, x5 + 8, 8);
x5[16] = x4[16];
x5[17] = x4[17];
-
- btf_32_neon_type0(-cospi[16], cospi[48], x4[18], x4[29], x5[18], x5[29],
- *v_cos_bit);
- btf_32_neon_type0(-cospi[16], cospi[48], x4[19], x4[28], x5[19], x5[28],
- *v_cos_bit);
- btf_32_neon_type0(-cospi[48], -cospi[16], x4[20], x4[27], x5[20], x5[27],
- *v_cos_bit);
- btf_32_neon_type0(-cospi[48], -cospi[16], x4[21], x4[26], x5[21], x5[26],
- *v_cos_bit);
+ butterfly_0112_neon(cospi, 16, x4[29], x4[18], &x5[29], &x5[18], v_cos_bit);
+ butterfly_0112_neon(cospi, 16, x4[28], x4[19], &x5[28], &x5[19], v_cos_bit);
+ butterfly_2312_neon(cospi, 16, x4[27], x4[20], &x5[20], &x5[27], v_cos_bit);
+ butterfly_2312_neon(cospi, 16, x4[26], x4[21], &x5[21], &x5[26], v_cos_bit);
x5[22] = x4[22];
x5[23] = x4[23];
x5[24] = x4[24];
x5[25] = x4[25];
x5[30] = x4[30];
x5[31] = x4[31];
- x5[32] = vaddq_s32(x4[32], x4[39]);
- x5[39] = vsubq_s32(x4[32], x4[39]);
- x5[33] = vaddq_s32(x4[33], x4[38]);
- x5[38] = vsubq_s32(x4[33], x4[38]);
- x5[34] = vaddq_s32(x4[34], x4[37]);
- x5[37] = vsubq_s32(x4[34], x4[37]);
- x5[35] = vaddq_s32(x4[35], x4[36]);
- x5[36] = vsubq_s32(x4[35], x4[36]);
- x5[40] = vsubq_s32(x4[47], x4[40]);
- x5[47] = vaddq_s32(x4[47], x4[40]);
- x5[41] = vsubq_s32(x4[46], x4[41]);
- x5[46] = vaddq_s32(x4[46], x4[41]);
- x5[42] = vsubq_s32(x4[45], x4[42]);
- x5[45] = vaddq_s32(x4[45], x4[42]);
- x5[43] = vsubq_s32(x4[44], x4[43]);
- x5[44] = vaddq_s32(x4[44], x4[43]);
- x5[48] = vaddq_s32(x4[48], x4[55]);
- x5[55] = vsubq_s32(x4[48], x4[55]);
- x5[49] = vaddq_s32(x4[49], x4[54]);
- x5[54] = vsubq_s32(x4[49], x4[54]);
- x5[50] = vaddq_s32(x4[50], x4[53]);
- x5[53] = vsubq_s32(x4[50], x4[53]);
- x5[51] = vaddq_s32(x4[51], x4[52]);
- x5[52] = vsubq_s32(x4[51], x4[52]);
- x5[56] = vsubq_s32(x4[63], x4[56]);
- x5[63] = vaddq_s32(x4[63], x4[56]);
- x5[57] = vsubq_s32(x4[62], x4[57]);
- x5[62] = vaddq_s32(x4[62], x4[57]);
- x5[58] = vsubq_s32(x4[61], x4[58]);
- x5[61] = vaddq_s32(x4[61], x4[58]);
- x5[59] = vsubq_s32(x4[60], x4[59]);
- x5[60] = vaddq_s32(x4[60], x4[59]);
-}
-
-static void av1_fdct64_new_neon(int32x4_t *input, int32x4_t *output,
- int8_t cos_bit, const int instride,
- const int outstride) {
- const int32_t *cospi = cospi_arr(cos_bit);
- const int32x4_t v_cos_bit = vdupq_n_s32(-cos_bit);
-
- int startidx = 0 * instride;
- int endidx = 63 * instride;
-
- // stage 1-2-3-4-5
- int32x4_t x5[64];
- av1_fdct64_new_stage12345_neon(input, instride, x5, cospi, &v_cos_bit,
- &startidx, &endidx);
+ butterfly_dct_post(x4 + 32, x4 + 32, x5 + 32, 16);
+ butterfly_dct_post(x4 + 48, x4 + 48, x5 + 48, 16);
// stage 6
int32x4_t x6[64];
- btf_32_neon_type0(cospi[32], cospi[32], x5[0], x5[1], x6[0], x6[1],
- v_cos_bit);
- btf_32_neon_type1(cospi[48], cospi[16], x5[2], x5[3], x6[2], x6[3],
- v_cos_bit);
- x6[4] = vaddq_s32(x5[4], x5[5]);
- x6[5] = vsubq_s32(x5[4], x5[5]);
- x6[6] = vsubq_s32(x5[7], x5[6]);
- x6[7] = vaddq_s32(x5[7], x5[6]);
+ butterfly_0112_neon(cospi, 32, x5[0], x5[1], &x6[0], &x6[1], v_cos_bit);
+ butterfly_0112_neon(cospi, 16, x5[3], x5[2], &x6[2], &x6[3], v_cos_bit);
+ butterfly_dct_post(x5 + 4, x5 + 4, x6 + 4, 4);
x6[8] = x5[8];
- btf_32_neon_type0(-cospi[16], cospi[48], x5[9], x5[14], x6[9], x6[14],
- v_cos_bit);
- btf_32_neon_type0(-cospi[48], -cospi[16], x5[10], x5[13], x6[10], x6[13],
- v_cos_bit);
+ butterfly_0112_neon(cospi, 16, x5[14], x5[9], &x6[14], &x6[9], v_cos_bit);
+ butterfly_2312_neon(cospi, 16, x5[13], x5[10], &x6[10], &x6[13], v_cos_bit);
x6[11] = x5[11];
x6[12] = x5[12];
x6[15] = x5[15];
- x6[16] = vaddq_s32(x5[16], x5[19]);
- x6[19] = vsubq_s32(x5[16], x5[19]);
- x6[17] = vaddq_s32(x5[17], x5[18]);
- x6[18] = vsubq_s32(x5[17], x5[18]);
- x6[20] = vsubq_s32(x5[23], x5[20]);
- x6[23] = vaddq_s32(x5[23], x5[20]);
- x6[21] = vsubq_s32(x5[22], x5[21]);
- x6[22] = vaddq_s32(x5[22], x5[21]);
- x6[24] = vaddq_s32(x5[24], x5[27]);
- x6[27] = vsubq_s32(x5[24], x5[27]);
- x6[25] = vaddq_s32(x5[25], x5[26]);
- x6[26] = vsubq_s32(x5[25], x5[26]);
- x6[28] = vsubq_s32(x5[31], x5[28]);
- x6[31] = vaddq_s32(x5[31], x5[28]);
- x6[29] = vsubq_s32(x5[30], x5[29]);
- x6[30] = vaddq_s32(x5[30], x5[29]);
+ butterfly_dct_post(x5 + 16, x5 + 16, x6 + 16, 8);
+ butterfly_dct_post(x5 + 24, x5 + 24, x6 + 24, 8);
x6[32] = x5[32];
x6[33] = x5[33];
-
- btf_32_neon_type0(-cospi[40], cospi[24], x5[42], x5[53], x6[42], x6[53],
- v_cos_bit);
- btf_32_neon_type0(-cospi[40], cospi[24], x5[43], x5[52], x6[43], x6[52],
- v_cos_bit);
- btf_32_neon_type0(-cospi[24], -cospi[40], x5[44], x5[51], x6[44], x6[51],
- v_cos_bit);
- btf_32_neon_type0(-cospi[24], -cospi[40], x5[45], x5[50], x6[45], x6[50],
- v_cos_bit);
-
+ butterfly_0112_neon(cospi, 8, x5[61], x5[34], &x6[61], &x6[34], v_cos_bit);
+ butterfly_0112_neon(cospi, 8, x5[60], x5[35], &x6[60], &x6[35], v_cos_bit);
+ butterfly_2312_neon(cospi, 8, x5[59], x5[36], &x6[36], &x6[59], v_cos_bit);
+ butterfly_2312_neon(cospi, 8, x5[58], x5[37], &x6[37], &x6[58], v_cos_bit);
+ x6[38] = x5[38];
+ x6[39] = x5[39];
+ x6[40] = x5[40];
+ x6[41] = x5[41];
+ butterfly_0130_neon(cospi, 24, x5[42], x5[53], &x6[53], &x6[42], v_cos_bit);
+ butterfly_0130_neon(cospi, 24, x5[43], x5[52], &x6[52], &x6[43], v_cos_bit);
+ butterfly_0332_neon(cospi, 24, x5[51], x5[44], &x6[51], &x6[44], v_cos_bit);
+ butterfly_0332_neon(cospi, 24, x5[50], x5[45], &x6[50], &x6[45], v_cos_bit);
x6[46] = x5[46];
x6[47] = x5[47];
x6[48] = x5[48];
@@ -2874,82 +1893,26 @@
x7[1] = x6[1];
x7[2] = x6[2];
x7[3] = x6[3];
- btf_32_neon_type1(cospi[24], cospi[40], x6[5], x6[6], x7[5], x7[6],
- v_cos_bit);
-
- x7[8] = vaddq_s32(x6[8], x6[9]);
- x7[9] = vsubq_s32(x6[8], x6[9]);
- x7[10] = vsubq_s32(x6[11], x6[10]);
- x7[11] = vaddq_s32(x6[11], x6[10]);
- x7[12] = vaddq_s32(x6[12], x6[13]);
- x7[13] = vsubq_s32(x6[12], x6[13]);
- x7[14] = vsubq_s32(x6[15], x6[14]);
- x7[15] = vaddq_s32(x6[15], x6[14]);
+ butterfly_0112_neon(cospi, 8, x6[7], x6[4], &x7[4], &x7[7], v_cos_bit);
+ butterfly_0130_neon(cospi, 24, x6[5], x6[6], &x7[5], &x7[6], v_cos_bit);
+ butterfly_dct_post(x6 + 8, x6 + 8, x7 + 8, 4);
+ butterfly_dct_post(x6 + 12, x6 + 12, x7 + 12, 4);
x7[16] = x6[16];
-
- btf_32_neon_type0(-cospi[40], cospi[24], x6[21], x6[26], x7[21], x7[26],
- v_cos_bit);
- btf_32_neon_type0(-cospi[24], -cospi[40], x6[22], x6[25], x7[22], x7[25],
- v_cos_bit);
+ butterfly_0112_neon(cospi, 8, x6[30], x6[17], &x7[30], &x7[17], v_cos_bit);
+ butterfly_2312_neon(cospi, 8, x6[29], x6[18], &x7[18], &x7[29], v_cos_bit);
+ x7[19] = x6[19];
+ x7[20] = x6[20];
+ butterfly_0130_neon(cospi, 24, x6[21], x6[26], &x7[26], &x7[21], v_cos_bit);
+ butterfly_0332_neon(cospi, 24, x6[25], x6[22], &x7[25], &x7[22], v_cos_bit);
x7[23] = x6[23];
x7[24] = x6[24];
x7[27] = x6[27];
x7[28] = x6[28];
x7[31] = x6[31];
-
- btf_32_neon_type0(-cospi[8], cospi[56], x5[34], x5[61], x6[34], x6[61],
- v_cos_bit);
- btf_32_neon_type0(-cospi[8], cospi[56], x5[35], x5[60], x6[35], x6[60],
- v_cos_bit);
- btf_32_neon_type0(-cospi[56], -cospi[8], x5[36], x5[59], x6[36], x6[59],
- v_cos_bit);
- btf_32_neon_type0(-cospi[56], -cospi[8], x5[37], x5[58], x6[37], x6[58],
- v_cos_bit);
- x6[38] = x5[38];
- x6[39] = x5[39];
- x6[40] = x5[40];
- x6[41] = x5[41];
-
- btf_32_neon_type1(cospi[56], cospi[8], x6[4], x6[7], x7[4], x7[7], v_cos_bit);
- btf_32_neon_type0(-cospi[8], cospi[56], x6[17], x6[30], x7[17], x7[30],
- v_cos_bit);
- btf_32_neon_type0(-cospi[56], -cospi[8], x6[18], x6[29], x7[18], x7[29],
- v_cos_bit);
- x7[19] = x6[19];
- x7[20] = x6[20];
-
- x7[32] = vaddq_s32(x6[32], x6[35]);
- x7[35] = vsubq_s32(x6[32], x6[35]);
- x7[33] = vaddq_s32(x6[33], x6[34]);
- x7[34] = vsubq_s32(x6[33], x6[34]);
- x7[36] = vsubq_s32(x6[39], x6[36]);
- x7[39] = vaddq_s32(x6[39], x6[36]);
- x7[37] = vsubq_s32(x6[38], x6[37]);
- x7[38] = vaddq_s32(x6[38], x6[37]);
- x7[40] = vaddq_s32(x6[40], x6[43]);
- x7[43] = vsubq_s32(x6[40], x6[43]);
- x7[41] = vaddq_s32(x6[41], x6[42]);
- x7[42] = vsubq_s32(x6[41], x6[42]);
- x7[44] = vsubq_s32(x6[47], x6[44]);
- x7[47] = vaddq_s32(x6[47], x6[44]);
- x7[45] = vsubq_s32(x6[46], x6[45]);
- x7[46] = vaddq_s32(x6[46], x6[45]);
- x7[48] = vaddq_s32(x6[48], x6[51]);
- x7[51] = vsubq_s32(x6[48], x6[51]);
- x7[49] = vaddq_s32(x6[49], x6[50]);
- x7[50] = vsubq_s32(x6[49], x6[50]);
- x7[52] = vsubq_s32(x6[55], x6[52]);
- x7[55] = vaddq_s32(x6[55], x6[52]);
- x7[53] = vsubq_s32(x6[54], x6[53]);
- x7[54] = vaddq_s32(x6[54], x6[53]);
- x7[56] = vaddq_s32(x6[56], x6[59]);
- x7[59] = vsubq_s32(x6[56], x6[59]);
- x7[57] = vaddq_s32(x6[57], x6[58]);
- x7[58] = vsubq_s32(x6[57], x6[58]);
- x7[60] = vsubq_s32(x6[63], x6[60]);
- x7[63] = vaddq_s32(x6[63], x6[60]);
- x7[61] = vsubq_s32(x6[62], x6[61]);
- x7[62] = vaddq_s32(x6[62], x6[61]);
+ butterfly_dct_post(x6 + 32, x6 + 32, x7 + 32, 8);
+ butterfly_dct_post(x6 + 40, x6 + 40, x7 + 40, 8);
+ butterfly_dct_post(x6 + 48, x6 + 48, x7 + 48, 8);
+ butterfly_dct_post(x6 + 56, x6 + 56, x7 + 56, 8);
// stage 8
int32x4_t x8[64];
@@ -2962,54 +1925,29 @@
x8[6] = x7[6];
x8[7] = x7[7];
- btf_32_neon_type1(cospi[60], cospi[4], x7[8], x7[15], x8[8], x8[15],
- v_cos_bit);
- btf_32_neon_type1(cospi[28], cospi[36], x7[9], x7[14], x8[9], x8[14],
- v_cos_bit);
- btf_32_neon_type1(cospi[44], cospi[20], x7[10], x7[13], x8[10], x8[13],
- v_cos_bit);
- btf_32_neon_type1(cospi[12], cospi[52], x7[11], x7[12], x8[11], x8[12],
- v_cos_bit);
- x8[16] = vaddq_s32(x7[16], x7[17]);
- x8[17] = vsubq_s32(x7[16], x7[17]);
- x8[18] = vsubq_s32(x7[19], x7[18]);
- x8[19] = vaddq_s32(x7[19], x7[18]);
- x8[20] = vaddq_s32(x7[20], x7[21]);
- x8[21] = vsubq_s32(x7[20], x7[21]);
- x8[22] = vsubq_s32(x7[23], x7[22]);
- x8[23] = vaddq_s32(x7[23], x7[22]);
- x8[24] = vaddq_s32(x7[24], x7[25]);
- x8[25] = vsubq_s32(x7[24], x7[25]);
- x8[26] = vsubq_s32(x7[27], x7[26]);
- x8[27] = vaddq_s32(x7[27], x7[26]);
- x8[28] = vaddq_s32(x7[28], x7[29]);
- x8[29] = vsubq_s32(x7[28], x7[29]);
- x8[30] = vsubq_s32(x7[31], x7[30]);
- x8[31] = vaddq_s32(x7[31], x7[30]);
+ butterfly_0112_neon(cospi, 4, x7[15], x7[8], &x8[8], &x8[15], v_cos_bit);
+ butterfly_0130_neon(cospi, 28, x7[9], x7[14], &x8[9], &x8[14], v_cos_bit);
+ butterfly_0112_neon(cospi, 20, x7[13], x7[10], &x8[10], &x8[13], v_cos_bit);
+ butterfly_0130_neon(cospi, 12, x7[11], x7[12], &x8[11], &x8[12], v_cos_bit);
+ butterfly_dct_post(x7 + 16, x7 + 16, x8 + 16, 4);
+ butterfly_dct_post(x7 + 20, x7 + 20, x8 + 20, 4);
+ butterfly_dct_post(x7 + 24, x7 + 24, x8 + 24, 4);
+ butterfly_dct_post(x7 + 28, x7 + 28, x8 + 28, 4);
x8[32] = x7[32];
-
- btf_32_neon_type0(-cospi[4], cospi[60], x7[33], x7[62], x8[33], x8[62],
- v_cos_bit);
- btf_32_neon_type0(-cospi[60], -cospi[4], x7[34], x7[61], x8[34], x8[61],
- v_cos_bit);
+ butterfly_0112_neon(cospi, 4, x7[62], x7[33], &x8[62], &x8[33], v_cos_bit);
+ butterfly_2312_neon(cospi, 4, x7[61], x7[34], &x8[34], &x8[61], v_cos_bit);
x8[35] = x7[35];
x8[36] = x7[36];
- btf_32_neon_type0(-cospi[36], cospi[28], x7[37], x7[58], x8[37], x8[58],
- v_cos_bit);
- btf_32_neon_type0(-cospi[28], -cospi[36], x7[38], x7[57], x8[38], x8[57],
- v_cos_bit);
+ butterfly_0130_neon(cospi, 28, x7[37], x7[58], &x8[58], &x8[37], v_cos_bit);
+ butterfly_0332_neon(cospi, 28, x7[57], x7[38], &x8[57], &x8[38], v_cos_bit);
x8[39] = x7[39];
x8[40] = x7[40];
- btf_32_neon_type0(-cospi[20], cospi[44], x7[41], x7[54], x8[41], x8[54],
- v_cos_bit);
- btf_32_neon_type0(-cospi[44], -cospi[20], x7[42], x7[53], x8[42], x8[53],
- v_cos_bit);
+ butterfly_0112_neon(cospi, 20, x7[54], x7[41], &x8[54], &x8[41], v_cos_bit);
+ butterfly_2312_neon(cospi, 20, x7[53], x7[42], &x8[42], &x8[53], v_cos_bit);
x8[43] = x7[43];
x8[44] = x7[44];
- btf_32_neon_type0(-cospi[52], cospi[12], x7[45], x7[50], x8[45], x8[50],
- v_cos_bit);
- btf_32_neon_type0(-cospi[12], -cospi[52], x7[46], x7[49], x8[46], x8[49],
- v_cos_bit);
+ butterfly_0130_neon(cospi, 12, x7[45], x7[50], &x8[50], &x8[45], v_cos_bit);
+ butterfly_0332_neon(cospi, 12, x7[49], x7[46], &x8[49], &x8[46], v_cos_bit);
x8[47] = x7[47];
x8[48] = x7[48];
x8[51] = x7[51];
@@ -3038,56 +1976,22 @@
x9[13] = x8[13];
x9[14] = x8[14];
x9[15] = x8[15];
-
- btf_32_neon_type1(cospi[62], cospi[2], x8[16], x8[31], x9[16], x9[31],
- v_cos_bit);
- btf_32_neon_type1(cospi[30], cospi[34], x8[17], x8[30], x9[17], x9[30],
- v_cos_bit);
- btf_32_neon_type1(cospi[46], cospi[18], x8[18], x8[29], x9[18], x9[29],
- v_cos_bit);
- btf_32_neon_type1(cospi[14], cospi[50], x8[19], x8[28], x9[19], x9[28],
- v_cos_bit);
- btf_32_neon_type1(cospi[54], cospi[10], x8[20], x8[27], x9[20], x9[27],
- v_cos_bit);
- btf_32_neon_type1(cospi[22], cospi[42], x8[21], x8[26], x9[21], x9[26],
- v_cos_bit);
- btf_32_neon_type1(cospi[38], cospi[26], x8[22], x8[25], x9[22], x9[25],
- v_cos_bit);
- btf_32_neon_type1(cospi[6], cospi[58], x8[23], x8[24], x9[23], x9[24],
- v_cos_bit);
-
- x9[32] = vaddq_s32(x8[32], x8[33]);
- x9[33] = vsubq_s32(x8[32], x8[33]);
- x9[34] = vsubq_s32(x8[35], x8[34]);
- x9[35] = vaddq_s32(x8[35], x8[34]);
- x9[36] = vaddq_s32(x8[36], x8[37]);
- x9[37] = vsubq_s32(x8[36], x8[37]);
- x9[38] = vsubq_s32(x8[39], x8[38]);
- x9[39] = vaddq_s32(x8[39], x8[38]);
- x9[40] = vaddq_s32(x8[40], x8[41]);
- x9[41] = vsubq_s32(x8[40], x8[41]);
- x9[42] = vsubq_s32(x8[43], x8[42]);
- x9[43] = vaddq_s32(x8[43], x8[42]);
- x9[44] = vaddq_s32(x8[44], x8[45]);
- x9[45] = vsubq_s32(x8[44], x8[45]);
- x9[46] = vsubq_s32(x8[47], x8[46]);
- x9[47] = vaddq_s32(x8[47], x8[46]);
- x9[48] = vaddq_s32(x8[48], x8[49]);
- x9[49] = vsubq_s32(x8[48], x8[49]);
- x9[50] = vsubq_s32(x8[51], x8[50]);
- x9[51] = vaddq_s32(x8[51], x8[50]);
- x9[52] = vaddq_s32(x8[52], x8[53]);
- x9[53] = vsubq_s32(x8[52], x8[53]);
- x9[54] = vsubq_s32(x8[55], x8[54]);
- x9[55] = vaddq_s32(x8[55], x8[54]);
- x9[56] = vaddq_s32(x8[56], x8[57]);
- x9[57] = vsubq_s32(x8[56], x8[57]);
- x9[58] = vsubq_s32(x8[59], x8[58]);
- x9[59] = vaddq_s32(x8[59], x8[58]);
- x9[60] = vaddq_s32(x8[60], x8[61]);
- x9[61] = vsubq_s32(x8[60], x8[61]);
- x9[62] = vsubq_s32(x8[63], x8[62]);
- x9[63] = vaddq_s32(x8[63], x8[62]);
+ butterfly_0112_neon(cospi, 2, x8[31], x8[16], &x9[16], &x9[31], v_cos_bit);
+ butterfly_0130_neon(cospi, 30, x8[17], x8[30], &x9[17], &x9[30], v_cos_bit);
+ butterfly_0112_neon(cospi, 18, x8[29], x8[18], &x9[18], &x9[29], v_cos_bit);
+ butterfly_0130_neon(cospi, 14, x8[19], x8[28], &x9[19], &x9[28], v_cos_bit);
+ butterfly_0112_neon(cospi, 10, x8[27], x8[20], &x9[20], &x9[27], v_cos_bit);
+ butterfly_0130_neon(cospi, 22, x8[21], x8[26], &x9[21], &x9[26], v_cos_bit);
+ butterfly_0112_neon(cospi, 26, x8[25], x8[22], &x9[22], &x9[25], v_cos_bit);
+ butterfly_0130_neon(cospi, 6, x8[23], x8[24], &x9[23], &x9[24], v_cos_bit);
+ butterfly_dct_post(x8 + 32, x8 + 32, x9 + 32, 4);
+ butterfly_dct_post(x8 + 36, x8 + 36, x9 + 36, 4);
+ butterfly_dct_post(x8 + 40, x8 + 40, x9 + 40, 4);
+ butterfly_dct_post(x8 + 44, x8 + 44, x9 + 44, 4);
+ butterfly_dct_post(x8 + 48, x8 + 48, x9 + 48, 4);
+ butterfly_dct_post(x8 + 52, x8 + 52, x9 + 52, 4);
+ butterfly_dct_post(x8 + 56, x8 + 56, x9 + 56, 4);
+ butterfly_dct_post(x8 + 60, x8 + 60, x9 + 60, 4);
// stage 10
int32x4_t x10[64];
@@ -3123,903 +2027,593 @@
x10[29] = x9[29];
x10[30] = x9[30];
x10[31] = x9[31];
- btf_32_neon_type1(cospi[63], cospi[1], x9[32], x9[63], x10[32], x10[63],
- v_cos_bit);
- btf_32_neon_type1(cospi[31], cospi[33], x9[33], x9[62], x10[33], x10[62],
- v_cos_bit);
- btf_32_neon_type1(cospi[47], cospi[17], x9[34], x9[61], x10[34], x10[61],
- v_cos_bit);
- btf_32_neon_type1(cospi[15], cospi[49], x9[35], x9[60], x10[35], x10[60],
- v_cos_bit);
- btf_32_neon_type1(cospi[55], cospi[9], x9[36], x9[59], x10[36], x10[59],
- v_cos_bit);
- btf_32_neon_type1(cospi[23], cospi[41], x9[37], x9[58], x10[37], x10[58],
- v_cos_bit);
- btf_32_neon_type1(cospi[39], cospi[25], x9[38], x9[57], x10[38], x10[57],
- v_cos_bit);
- btf_32_neon_type1(cospi[7], cospi[57], x9[39], x9[56], x10[39], x10[56],
- v_cos_bit);
- btf_32_neon_type1(cospi[59], cospi[5], x9[40], x9[55], x10[40], x10[55],
- v_cos_bit);
- btf_32_neon_type1(cospi[27], cospi[37], x9[41], x9[54], x10[41], x10[54],
- v_cos_bit);
- btf_32_neon_type1(cospi[43], cospi[21], x9[42], x9[53], x10[42], x10[53],
- v_cos_bit);
- btf_32_neon_type1(cospi[11], cospi[53], x9[43], x9[52], x10[43], x10[52],
- v_cos_bit);
- btf_32_neon_type1(cospi[51], cospi[13], x9[44], x9[51], x10[44], x10[51],
- v_cos_bit);
- btf_32_neon_type1(cospi[19], cospi[45], x9[45], x9[50], x10[45], x10[50],
- v_cos_bit);
- btf_32_neon_type1(cospi[35], cospi[29], x9[46], x9[49], x10[46], x10[49],
- v_cos_bit);
- btf_32_neon_type1(cospi[3], cospi[61], x9[47], x9[48], x10[47], x10[48],
- v_cos_bit);
+ butterfly_0112_neon(cospi, 1, x9[63], x9[32], &x10[32], &x10[63], v_cos_bit);
+ butterfly_0130_neon(cospi, 31, x9[33], x9[62], &x10[33], &x10[62], v_cos_bit);
+ butterfly_0112_neon(cospi, 17, x9[61], x9[34], &x10[34], &x10[61], v_cos_bit);
+ butterfly_0130_neon(cospi, 15, x9[35], x9[60], &x10[35], &x10[60], v_cos_bit);
+ butterfly_0112_neon(cospi, 9, x9[59], x9[36], &x10[36], &x10[59], v_cos_bit);
+ butterfly_0130_neon(cospi, 23, x9[37], x9[58], &x10[37], &x10[58], v_cos_bit);
+ butterfly_0112_neon(cospi, 25, x9[57], x9[38], &x10[38], &x10[57], v_cos_bit);
+ butterfly_0130_neon(cospi, 7, x9[39], x9[56], &x10[39], &x10[56], v_cos_bit);
+ butterfly_0112_neon(cospi, 5, x9[55], x9[40], &x10[40], &x10[55], v_cos_bit);
+ butterfly_0130_neon(cospi, 27, x9[41], x9[54], &x10[41], &x10[54], v_cos_bit);
+ butterfly_0112_neon(cospi, 21, x9[53], x9[42], &x10[42], &x10[53], v_cos_bit);
+ butterfly_0130_neon(cospi, 11, x9[43], x9[52], &x10[43], &x10[52], v_cos_bit);
+ butterfly_0112_neon(cospi, 13, x9[51], x9[44], &x10[44], &x10[51], v_cos_bit);
+ butterfly_0130_neon(cospi, 19, x9[45], x9[50], &x10[45], &x10[50], v_cos_bit);
+ butterfly_0112_neon(cospi, 29, x9[49], x9[46], &x10[46], &x10[49], v_cos_bit);
+ butterfly_0130_neon(cospi, 3, x9[47], x9[48], &x10[47], &x10[48], v_cos_bit);
- startidx = 0 * outstride;
- endidx = 63 * outstride;
// stage 11
- output[startidx] = x10[0];
- output[endidx] = x10[63];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[32];
- output[endidx] = x10[31];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[16];
- output[endidx] = x10[47];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[48];
- output[endidx] = x10[15];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[8];
- output[endidx] = x10[55];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[40];
- output[endidx] = x10[23];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[24];
- output[endidx] = x10[39];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[56];
- output[endidx] = x10[7];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[4];
- output[endidx] = x10[59];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[36];
- output[endidx] = x10[27];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[20];
- output[endidx] = x10[43];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[52];
- output[endidx] = x10[11];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[12];
- output[endidx] = x10[51];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[44];
- output[endidx] = x10[19];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[28];
- output[endidx] = x10[35];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[60];
- output[endidx] = x10[3];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[2];
- output[endidx] = x10[61];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[34];
- output[endidx] = x10[29];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[18];
- output[endidx] = x10[45];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[50];
- output[endidx] = x10[13];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[10];
- output[endidx] = x10[53];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[42];
- output[endidx] = x10[21];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[26];
- output[endidx] = x10[37];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[58];
- output[endidx] = x10[5];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[6];
- output[endidx] = x10[57];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[38];
- output[endidx] = x10[25];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[22];
- output[endidx] = x10[41];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[54];
- output[endidx] = x10[9];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[14];
- output[endidx] = x10[49];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[46];
- output[endidx] = x10[17];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[30];
- output[endidx] = x10[33];
- startidx += outstride;
- endidx -= outstride;
- output[startidx] = x10[62];
- output[endidx] = x10[1];
+ output[0] = x10[0];
+ output[1] = x10[32];
+ output[2] = x10[16];
+ output[3] = x10[48];
+ output[4] = x10[8];
+ output[5] = x10[40];
+ output[6] = x10[24];
+ output[7] = x10[56];
+ output[8] = x10[4];
+ output[9] = x10[36];
+ output[10] = x10[20];
+ output[11] = x10[52];
+ output[12] = x10[12];
+ output[13] = x10[44];
+ output[14] = x10[28];
+ output[15] = x10[60];
+ output[16] = x10[2];
+ output[17] = x10[34];
+ output[18] = x10[18];
+ output[19] = x10[50];
+ output[20] = x10[10];
+ output[21] = x10[42];
+ output[22] = x10[26];
+ output[23] = x10[58];
+ output[24] = x10[6];
+ output[25] = x10[38];
+ output[26] = x10[22];
+ output[27] = x10[54];
+ output[28] = x10[14];
+ output[29] = x10[46];
+ output[30] = x10[30];
+ output[31] = x10[62];
+ output[32] = x10[1];
+ output[33] = x10[33];
+ output[34] = x10[17];
+ output[35] = x10[49];
+ output[36] = x10[9];
+ output[37] = x10[41];
+ output[38] = x10[25];
+ output[39] = x10[57];
+ output[40] = x10[5];
+ output[41] = x10[37];
+ output[42] = x10[21];
+ output[43] = x10[53];
+ output[44] = x10[13];
+ output[45] = x10[45];
+ output[46] = x10[29];
+ output[47] = x10[61];
+ output[48] = x10[3];
+ output[49] = x10[35];
+ output[50] = x10[19];
+ output[51] = x10[51];
+ output[52] = x10[11];
+ output[53] = x10[43];
+ output[54] = x10[27];
+ output[55] = x10[59];
+ output[56] = x10[7];
+ output[57] = x10[39];
+ output[58] = x10[23];
+ output[59] = x10[55];
+ output[60] = x10[15];
+ output[61] = x10[47];
+ output[62] = x10[31];
+ output[63] = x10[63];
}
-void av1_idtx32_new_neon(int32x4_t *input, int32x4_t *output, int cos_bit,
- const int col_num) {
+static void highbd_fidentity32_x4_neon(const int32x4_t *input,
+ int32x4_t *output, int cos_bit) {
(void)cos_bit;
for (int i = 0; i < 32; i++) {
- output[i * col_num] = vshlq_n_s32(input[i * col_num], 2);
+ output[i] = vshlq_n_s32(input[i], 2);
}
}
-static const fwd_transform_1d_neon col_highbd_txfm8x32_arr[TX_TYPES] = {
- av1_fdct32_new_neon, // DCT_DCT
- NULL, // ADST_DCT
- NULL, // DCT_ADST
- NULL, // ADST_ADST
- NULL, // FLIPADST_DCT
- NULL, // DCT_FLIPADST
- NULL, // FLIPADST_FLIPADST
- NULL, // ADST_FLIPADST
- NULL, // FLIPADST_ADST
- av1_idtx32_new_neon, // IDTX
- NULL, // V_DCT
- NULL, // H_DCT
- NULL, // V_ADST
- NULL, // H_ADST
- NULL, // V_FLIPADST
- NULL // H_FLIPADST
-};
+TRANSFORM_COL_MANY(fdct32, 32)
+TRANSFORM_COL_MANY(fidentity32, 32)
-static const fwd_transform_1d_neon row_highbd_txfm8x32_arr[TX_TYPES] = {
- fdct16x16_neon, // DCT_DCT
- NULL, // ADST_DCT
- NULL, // DCT_ADST
- NULL, // ADST_ADST
- NULL, // FLIPADST_DCT
- NULL, // DCT_FLIPADST
- NULL, // FLIPADST_FLIPADST
- NULL, // ADST_FLIPADST
- NULL, // FLIPADST_ADST
- idtx16x16_neon, // IDTX
- NULL, // V_DCT
- NULL, // H_DCT
- NULL, // V_ADST
- NULL, // H_ADST
- NULL, // V_FLIPADST
- NULL // H_FLIPADST
-};
+static const fwd_transform_1d_col_many_neon
+ col_highbd_txfm32_x4_arr[TX_TYPES] = {
+ highbd_fdct32_col_many_neon, // DCT_DCT
+ NULL, // ADST_DCT
+ NULL, // DCT_ADST
+ NULL, // ADST_ADST
+ NULL, // FLIPADST_DCT
+ NULL, // DCT_FLIPADST
+ NULL, // FLIPADST_FLIPADST
+ NULL, // ADST_FLIPADST
+ NULL, // FLIPADST_ADST
+ highbd_fidentity32_col_many_neon, // IDTX
+ NULL, // V_DCT
+ NULL, // H_DCT
+ NULL, // V_ADST
+ NULL, // H_ADST
+ NULL, // V_FLIPADST
+ NULL // H_FLIPADST
+ };
+
+TRANSFORM_ROW_MANY(fdct32, 32)
+TRANSFORM_ROW_MANY(fidentity32, 32)
+
+static const fwd_transform_1d_row_many_neon
+ row_highbd_txfm32_x4_arr[TX_TYPES] = {
+ highbd_fdct32_row_many_neon, // DCT_DCT
+ NULL, // ADST_DCT
+ NULL, // DCT_ADST
+ NULL, // ADST_ADST
+ NULL, // FLIPADST_DCT
+ NULL, // DCT_FLIPADST
+ NULL, // FLIPADST_FLIPADST
+ NULL, // ADST_FLIPADST
+ NULL, // FLIPADST_ADST
+ highbd_fidentity32_row_many_neon, // IDTX
+ NULL, // V_DCT
+ NULL, // H_DCT
+ NULL, // V_ADST
+ NULL, // H_ADST
+ NULL, // V_FLIPADST
+ NULL // H_FLIPADST
+ };
+
+TRANSFORM_ROW_RECT_MANY(fdct32, 32)
+TRANSFORM_ROW_RECT_MANY(fidentity32, 32)
+
+static const fwd_transform_1d_row_many_neon
+ row_rect_highbd_txfm32_x4_arr[TX_TYPES] = {
+ highbd_fdct32_row_rect_many_neon, // DCT_DCT
+ NULL, // ADST_DCT
+ NULL, // DCT_ADST
+ NULL, // ADST_ADST
+ NULL, // FLIPADST_DCT
+ NULL, // DCT_FLIPADST
+ NULL, // FLIPADST_FLIPADST
+ NULL, // ADST_FLIPADST
+ NULL, // FLIPADST_ADST
+ highbd_fidentity32_row_rect_many_neon, // IDTX
+ NULL, // V_DCT
+ NULL, // H_DCT
+ NULL, // V_ADST
+ NULL, // H_ADST
+ NULL, // V_FLIPADST
+ NULL // H_FLIPADST
+ };
void av1_fwd_txfm2d_16x8_neon(const int16_t *input, int32_t *coeff, int stride,
TX_TYPE tx_type, int bd) {
(void)bd;
- int32x4_t in[32], out[32];
- const int8_t *shift = av1_fwd_txfm_shift_ls[TX_16X8];
- const int txw_idx = get_txw_idx(TX_16X8);
- const int txh_idx = get_txh_idx(TX_16X8);
- const fwd_transform_1d_neon col_txfm = col_highbd_txfm8x8_arr[tx_type];
- const fwd_transform_1d_neon row_txfm = row_highbd_txfm8x16_arr[tx_type];
- int bit = av1_fwd_cos_bit_col[txw_idx][txh_idx];
+ const fwd_transform_1d_col_many_neon col_txfm =
+ col_highbd_txfm8_xn_arr[tx_type];
+ const fwd_transform_1d_row_many_neon row_txfm =
+ row_rect_highbd_txfm16_xn_arr[tx_type];
+ int bit = av1_fwd_cos_bit_col[2][1];
+
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
- const int32x4_t v_shift1 = vdupq_n_s32(shift[1]);
- for (int i = 0; i < 2; i++) {
- load_buffer_8x8(input + i * 8, in, stride, ud_flip, 0, shift[0]);
- col_txfm(in, in, bit, 2);
- col_txfm_8x8_rounding(in, &v_shift1);
- transpose_8x8(in, out + i * 16);
- }
+ ud_adjust_input_and_stride(ud_flip, &input, &stride, 8);
+ // Column-wise transform.
+ int32x4_t buf0[32];
if (lr_flip) {
- flip_buf_neon(in, out, 32);
- row_txfm(in, out, bit, 2);
+ col_txfm(input, buf0 + 3 * 8, stride, bit, /*lr_flip=*/1, /*howmany=*/4,
+ /*hm_stride=*/-8);
} else {
- row_txfm(out, out, bit, 2);
+ col_txfm(input, buf0, stride, bit, /*lr_flip=*/0, /*howmany=*/4,
+ /*hm_stride=*/8);
}
+ shift_right_2_round_s32_x4(buf0, buf0, 32);
- for (int i = 0; i < 2; i++) {
- av1_round_shift_rect_array_32_neon(out + i * 16, in, 16, -shift[2],
- NewSqrt2);
- write_buffer_8x8(in, coeff + i * 64);
- }
+ int32x4_t buf1[32];
+ transpose_arrays_s32_16x8(buf0, buf1);
+
+ // Row-wise transform.
+ row_txfm(buf1, coeff, bit, /*howmany=*/2, /*hm_stride=*/16, /*stride=*/8);
}
void av1_fwd_txfm2d_8x16_neon(const int16_t *input, int32_t *coeff, int stride,
TX_TYPE tx_type, int bd) {
(void)bd;
- int32x4_t in[32], out[32];
- const int8_t *shift = av1_fwd_txfm_shift_ls[TX_8X16];
- const int txw_idx = get_txw_idx(TX_8X16);
- const int txh_idx = get_txh_idx(TX_8X16);
- const fwd_transform_1d_neon col_txfm = col_highbd_txfm8x16_arr[tx_type];
- const fwd_transform_1d_neon row_txfm = row_highbd_txfm8x8_arr[tx_type];
- int bit = av1_fwd_cos_bit_col[txw_idx][txh_idx];
+ const fwd_transform_1d_col_many_neon col_txfm =
+ col_highbd_txfm16_xn_arr[tx_type];
+ const fwd_transform_1d_row_many_neon row_txfm =
+ row_rect_highbd_txfm8_xn_arr[tx_type];
+ int bit = av1_fwd_cos_bit_col[1][2];
+
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
+ ud_adjust_input_and_stride(ud_flip, &input, &stride, 16);
- load_buffer_8x16(input, in, stride, ud_flip, lr_flip, shift[0]);
- col_txfm(in, in, bit, 2);
- const int32x4_t v_shift1 = vdupq_n_s32(shift[1]);
- col_txfm_8x16_rounding(in, &v_shift1);
- transpose_8x8(in, out);
- transpose_8x8(in + 16, out + 16);
-
- for (int i = 0; i < 2; i++) {
- row_txfm(out + i * 16, out, bit, 2);
- av1_round_shift_rect_array_32_neon(out, out, 16, -shift[2], NewSqrt2);
- write_buffer_16x8(out, coeff + i * 8, 16);
+ // Column-wise transform.
+ int32x4_t buf0[32];
+ if (lr_flip) {
+ col_txfm(input, buf0 + 16, stride, bit, /*lr_flip=*/1, /*howmany=*/2,
+ /*hm_stride=*/-16);
+ } else {
+ col_txfm(input, buf0, stride, bit, /*lr_flip=*/0, /*howmany=*/2,
+ /*hm_stride=*/16);
}
-}
+ shift_right_2_round_s32_x4(buf0, buf0, 32);
-static INLINE void transpose_8nx8n(const int32x4_t *input, int32x4_t *output,
- const int width, const int height) {
- const int numcol = height >> 2;
- const int numrow = width >> 2;
- for (int j = 0; j < numrow; j++) {
- for (int i = 0; i < numcol; i++) {
- TRANSPOSE_4X4(input[i * width + j + (numrow * 0)],
- input[i * width + j + (numrow * 1)],
- input[i * width + j + (numrow * 2)],
- input[i * width + j + (numrow * 3)],
- output[j * height + i + (numcol * 0)],
- output[j * height + i + (numcol * 1)],
- output[j * height + i + (numcol * 2)],
- output[j * height + i + (numcol * 3)]);
- }
- }
+ int32x4_t buf1[32];
+ transpose_arrays_s32_8x16(buf0, buf1);
+
+ // Row-wise transform.
+ row_txfm(buf1, coeff, bit, /*howmany=*/4, /*hm_stride=*/8, /*stride=*/16);
}
#if !CONFIG_REALTIME_ONLY
void av1_fwd_txfm2d_4x16_neon(const int16_t *input, int32_t *coeff, int stride,
TX_TYPE tx_type, int bd) {
(void)bd;
-
- int32x4_t in[16];
- int32x4_t *outcoeff128 = (int32x4_t *)coeff;
- const int8_t *shift = av1_fwd_txfm_shift_ls[TX_4X16];
- const int txw_idx = get_txw_idx(TX_4X16);
- const int txh_idx = get_txh_idx(TX_4X16);
- const int txfm_size_col = tx_size_wide[TX_4X16];
- const int txfm_size_row = tx_size_high[TX_4X16];
- int bitcol = av1_fwd_cos_bit_col[txw_idx][txh_idx];
- int bitrow = av1_fwd_cos_bit_row[txw_idx][txh_idx];
- const fwd_transform_1d_neon col_txfm = col_highbd_txfm8x16_arr[tx_type];
- const fwd_transform_1d_neon row_txfm = row_highbd_txfm4x4_arr[tx_type];
+ int bitcol = av1_fwd_cos_bit_col[0][2];
+ int bitrow = av1_fwd_cos_bit_row[0][2];
+ const fwd_transform_1d_col_many_neon col_txfm =
+ col_highbd_txfm16_xn_arr[tx_type];
+ const fwd_transform_1d_row_many_neon row_txfm =
+ row_highbd_txfm4_xn_arr[tx_type];
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
- // col transform
- int32x4_t v_shift0 = vdupq_n_s32(shift[0]);
- load_buffer_4x16(input, in, stride, ud_flip, lr_flip, &v_shift0);
- col_txfm(in, outcoeff128, bitcol, 1);
- const int32x4_t v_shift1 = vdupq_n_s32(shift[1]);
- col_txfm_8x8_rounding(outcoeff128, &v_shift1);
- transpose_8nx8n(outcoeff128, in, txfm_size_col, txfm_size_row);
+ ud_adjust_input_and_stride(ud_flip, &input, &stride, 16);
- // row transform
- for (int i = 0; i < txfm_size_col; i++) {
- int32x4_t tmp[4];
- row_txfm(in + i, tmp, bitrow, txfm_size_row >> 2);
- store_output_w4(coeff + i * 4, tmp, txfm_size_row, txfm_size_col);
+ // Column-wise transform.
+ int32x4_t buf0[16];
+ if (lr_flip) {
+ col_txfm(input, buf0, stride, bitcol, /*lr_flip=*/1, /*howmany=*/1,
+ /*hm_stride=*/0);
+ } else {
+ col_txfm(input, buf0, stride, bitcol, /*lr_flip=*/0, /*howmany=*/1,
+ /*hm_stride=*/0);
}
+ shift_right_1_round_s32_x4(buf0, buf0, 16);
+
+ int32x4_t buf1[16];
+ transpose_arrays_s32_4x16(buf0, buf1);
+
+ // Row-wise transform.
+ row_txfm(buf1, coeff, bitrow, /*howmany=*/4, /*hm_stride=*/4, /*stride=*/16);
}
#endif
void av1_fwd_txfm2d_16x4_neon(const int16_t *input, int32_t *coeff, int stride,
TX_TYPE tx_type, int bd) {
(void)bd;
+ int bitcol = av1_fwd_cos_bit_col[2][0];
+ int bitrow = av1_fwd_cos_bit_row[2][0];
+ const fwd_transform_1d_col_many_neon col_txfm =
+ col_highbd_txfm4_xn_arr[tx_type];
+ const fwd_transform_1d_row_neon row_txfm = row_highbd_txfm16_xn_arr[tx_type];
- int32x4_t in[16];
- int32x4_t *outcoeff128 = (int32x4_t *)coeff;
- const int8_t *shift = av1_fwd_txfm_shift_ls[TX_16X4];
- const int txw_idx = get_txw_idx(TX_16X4);
- const int txh_idx = get_txh_idx(TX_16X4);
- const int txfm_size_col = tx_size_wide[TX_16X4];
- const int txfm_size_row = tx_size_high[TX_16X4];
- int bitcol = av1_fwd_cos_bit_col[txw_idx][txh_idx];
- int bitrow = av1_fwd_cos_bit_row[txw_idx][txh_idx];
- const fwd_transform_1d_neon col_txfm = col_highbd_txfm4x4_arr[tx_type];
- const fwd_transform_1d_neon row_txfm = row_highbd_txfm8x16_arr[tx_type];
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
+ ud_adjust_input_and_stride(ud_flip, &input, &stride, 4);
- // col transform
- const int32x4_t v_shift0 = vdupq_n_s32(shift[0]);
- load_buffer_16x4(input, in, stride, ud_flip, lr_flip, &v_shift0);
-
- for (int i = 0; i < (txfm_size_col >> 2); i++) {
- int32x4_t *cur_in = &in[i * txfm_size_row];
- col_txfm(cur_in, cur_in, bitcol, 1);
- transpose_4x4(cur_in, cur_in);
+ // Column-wise transform.
+ int32x4_t buf0[16];
+ if (lr_flip) {
+ col_txfm(input, buf0 + 3 * 4, stride, bitcol, /*lr_flip=*/1, /*howmany=*/4,
+ /*hm_stride=*/-4);
+ } else {
+ col_txfm(input, buf0, stride, bitcol, /*lr_flip=*/0, /*howmany=*/4,
+ /*hm_stride=*/4);
}
- const int32x4_t v_shift1 = vdupq_n_s32(shift[1]);
- col_txfm_8x8_rounding(in, &v_shift1);
- // row transform
- row_txfm(in, outcoeff128, bitrow, 1);
+ shift_right_1_round_s32_x4(buf0, buf0, 16);
+ transpose_arrays_s32_4x16(buf0, buf0);
+
+ // Row-wise transform.
+ row_txfm(buf0, coeff, bitrow, /*stride=*/4);
}
void av1_fwd_txfm2d_16x32_neon(const int16_t *input, int32_t *coeff, int stride,
TX_TYPE tx_type, int bd) {
(void)bd;
+ const fwd_transform_1d_col_many_neon col_txfm =
+ col_highbd_txfm32_x4_arr[tx_type];
+ const fwd_transform_1d_row_many_neon row_txfm =
+ row_rect_highbd_txfm16_xn_arr[tx_type];
+ int bitcol = av1_fwd_cos_bit_col[2][3];
+ int bitrow = av1_fwd_cos_bit_row[2][3];
- int32x4_t in[128];
- int32x4_t *outcoef128 = (int32x4_t *)coeff;
- const int8_t *shift = av1_fwd_txfm_shift_ls[TX_16X32];
- const int txw_idx = get_txw_idx(TX_16X32);
- const int txh_idx = get_txh_idx(TX_16X32);
- const fwd_transform_1d_neon col_txfm = col_highbd_txfm8x32_arr[tx_type];
- const fwd_transform_1d_neon row_txfm = row_highbd_txfm8x32_arr[tx_type];
- int bitcol = av1_fwd_cos_bit_col[txw_idx][txh_idx];
- int bitrow = av1_fwd_cos_bit_row[txw_idx][txh_idx];
+ // Column-wise transform.
+ int32x4_t buf0[128];
+ col_txfm(input, buf0, stride, bitcol, /*lr_flip=*/0, /*howmany=*/4,
+ /*hm_stride=*/32);
+ shift_right_4_round_s32_x4(buf0, buf0, 128);
- // column transform
- load_buffer_16x16(input, in, stride, 0, 0, shift[0]);
- load_buffer_16x16(input + 16 * stride, in + 64, stride, 0, 0, shift[0]);
+ int32x4_t buf1[128];
+ transpose_arrays_s32_16x32(buf0, buf1);
- for (int i = 0; i < 4; i++) {
- col_txfm((in + i), (in + i), bitcol, 4);
- }
-
- const int32x4_t v_shift = vdupq_n_s32(shift[1]);
- col_txfm_16x16_rounding(&in[0], &v_shift);
- col_txfm_16x16_rounding(&in[64], &v_shift);
- transpose_8nx8n(in, outcoef128, 16, 32);
-
- // row transform
- row_txfm(outcoef128, in, bitrow, 8);
- av1_round_shift_rect_array_32_neon(in, outcoef128, 128, -shift[2], NewSqrt2);
+ // Row-wise transform.
+ row_txfm(buf1, coeff, bitrow, /*howmany=*/8, /*hm_stride=*/16, /*stride=*/32);
}
void av1_fwd_txfm2d_32x64_neon(const int16_t *input, int32_t *coeff, int stride,
TX_TYPE tx_type, int bd) {
- (void)tx_type;
(void)bd;
+ (void)tx_type;
+ int bitcol = av1_fwd_cos_bit_col[3][4];
+ int bitrow = av1_fwd_cos_bit_row[3][4];
- int32x4_t in[512];
- int32x4_t *outcoef128 = (int32x4_t *)coeff;
- const int8_t *shift = av1_fwd_txfm_shift_ls[TX_32X64];
- const int txw_idx = get_txw_idx(TX_32X64);
- const int txh_idx = get_txh_idx(TX_32X64);
- const int txfm_size_col = tx_size_wide[TX_32X64];
- const int txfm_size_row = tx_size_high[TX_32X64];
- int bitcol = av1_fwd_cos_bit_col[txw_idx][txh_idx];
- int bitrow = av1_fwd_cos_bit_row[txw_idx][txh_idx];
- const int num_row = txfm_size_row >> 2;
- const int num_col = txfm_size_col >> 2;
+ // Column-wise transform.
+ int32x4_t buf0[512];
+ load_buffer_32x64(input, buf0, stride, 0);
+ for (int i = 0; i < 8; i++) {
+ highbd_fdct64_x4_neon(buf0 + i * 64, buf0 + i * 64, bitcol);
+ }
+ shift_right_2_round_s32_x4(buf0, buf0, 512);
- // column transform
- load_buffer_32x8n(input, in, stride, 0, 0, shift[0], txfm_size_row);
- for (int i = 0; i < num_col; i++) {
- av1_fdct64_new_neon((in + i), (in + i), bitcol, num_col, num_col);
- }
+ int32x4_t buf1[512];
+ transpose_arrays_s32_32x64(buf0, buf1);
- const int32x4_t v_shift = vdupq_n_s32(shift[1]);
- for (int i = 0; i < num_col; i++) {
- col_txfm_16x16_rounding((in + i * txfm_size_row), &v_shift);
+ // Row-wise transform.
+ for (int i = 0; i < 16; i++) {
+ highbd_fdct32_x4_neon(buf1 + i * 32, buf1 + i * 32, bitrow);
}
- transpose_8nx8n(in, outcoef128, txfm_size_col, txfm_size_row);
-
- // row transform
- for (int i = 0; i < num_row; i++) {
- av1_fdct32_new_neon((outcoef128 + i), (in + i), bitrow, num_row);
- }
- for (int i = 0; i < txfm_size_col; i++) {
- av1_round_shift_rect_array_32_neon(in + i * 16, outcoef128 + i * 8, 8,
- -shift[2], NewSqrt2);
- }
+ round_shift2_rect_array_s32_neon(buf1, buf1, 512);
+ store_buffer_32x32(buf1, coeff, /*stride=*/32);
}
void av1_fwd_txfm2d_64x32_neon(const int16_t *input, int32_t *coeff, int stride,
TX_TYPE tx_type, int bd) {
- (void)tx_type;
- int32x4_t in[512];
- int32x4_t *outcoef128 = (int32x4_t *)coeff;
- const int8_t *shift = av1_fwd_txfm_shift_ls[TX_64X32];
- const int txw_idx = get_txw_idx(TX_64X32);
- const int txh_idx = get_txh_idx(TX_64X32);
- const int txfm_size_col = tx_size_wide[TX_64X32];
- const int txfm_size_row = tx_size_high[TX_64X32];
- int bitcol = av1_fwd_cos_bit_col[txw_idx][txh_idx];
- int bitrow = av1_fwd_cos_bit_row[txw_idx][txh_idx];
- const int num_row = txfm_size_row >> 2;
- const int num_col = txfm_size_col >> 2;
-
- // column transform
- const int32x4_t v_shift0 = vdupq_n_s32(shift[0]);
- for (int i = 0; i < 32; i++) {
- load_buffer_4x4(input + 0 + i * stride, in + 0 + i * 16, 4, 0, 0,
- &v_shift0);
- load_buffer_4x4(input + 16 + i * stride, in + 4 + i * 16, 4, 0, 0,
- &v_shift0);
- load_buffer_4x4(input + 32 + i * stride, in + 8 + i * 16, 4, 0, 0,
- &v_shift0);
- load_buffer_4x4(input + 48 + i * stride, in + 12 + i * 16, 4, 0, 0,
- &v_shift0);
- }
-
- for (int i = 0; i < num_col; i++) {
- av1_fdct32_new_neon((in + i), (in + i), bitcol, num_col);
- }
-
- const int32x4_t v_shift1 = vdupq_n_s32(shift[1]);
- for (int i = 0; i < num_row; i++) {
- col_txfm_16x16_rounding((in + i * txfm_size_col), &v_shift1);
- }
- transpose_8nx8n(in, outcoef128, txfm_size_col, txfm_size_row);
-
- // row transform
- for (int i = 0; i < num_row; i++) {
- av1_fdct64_new_neon((outcoef128 + i), (in + i), bitrow, num_row, num_row);
- }
- av1_round_shift_rect_array_32_neon(in, outcoef128, 512, -shift[2], NewSqrt2);
(void)bd;
+ (void)tx_type;
+ int bitcol = av1_fwd_cos_bit_col[4][3];
+ int bitrow = av1_fwd_cos_bit_row[4][3];
+
+ // Column-wise transform.
+ int32x4_t buf0[512];
+ load_buffer_64x32(input, buf0, stride, 0);
+ for (int i = 0; i < 16; i++) {
+ highbd_fdct32_x4_neon(buf0 + i * 32, buf0 + i * 32, bitcol);
+ }
+ shift_right_4_round_s32_x4(buf0, buf0, 512);
+
+ int32x4_t buf1[512];
+ transpose_arrays_s32_64x32(buf0, buf1);
+
+ // Row-wise transform.
+ for (int i = 0; i < 8; i++) {
+ highbd_fdct64_x4_neon(buf1 + i * 64, buf1 + i * 64, bitrow);
+ }
+ round_shift2_rect_array_s32_neon(buf1, buf1, 512);
+ store_buffer_64x32(buf1, coeff, /*stride=*/32);
}
void av1_fwd_txfm2d_32x16_neon(const int16_t *input, int32_t *coeff, int stride,
TX_TYPE tx_type, int bd) {
- int32x4_t in[128];
- int32x4_t *outcoef128 = (int32x4_t *)coeff;
- const int8_t *shift = av1_fwd_txfm_shift_ls[TX_32X16];
- const int txw_idx = get_txw_idx(TX_32X16);
- const int txh_idx = get_txh_idx(TX_32X16);
- const fwd_transform_1d_neon col_txfm = row_highbd_txfm8x32_arr[tx_type];
- const fwd_transform_1d_neon row_txfm = col_highbd_txfm8x32_arr[tx_type];
- int bitcol = av1_fwd_cos_bit_col[txw_idx][txh_idx];
- int bitrow = av1_fwd_cos_bit_row[txw_idx][txh_idx];
-
- // column transform
- load_buffer_32x8n(input, in, stride, 0, 0, shift[0], 16);
- col_txfm(in, in, bitcol, 8);
- const int32x4_t v_shift = vdupq_n_s32(shift[1]);
- col_txfm_16x16_rounding(&in[0], &v_shift);
- col_txfm_16x16_rounding(&in[64], &v_shift);
- transpose_8nx8n(in, outcoef128, 32, 16);
-
- // row transform
- for (int i = 0; i < 4; i++) {
- row_txfm((outcoef128 + i), (in + i), bitrow, 4);
- }
- av1_round_shift_rect_array_32_neon(in, outcoef128, 128, -shift[2], NewSqrt2);
(void)bd;
+ const fwd_transform_1d_col_many_neon col_txfm =
+ col_highbd_txfm16_xn_arr[tx_type];
+ const fwd_transform_1d_row_many_neon row_txfm =
+ row_rect_highbd_txfm32_x4_arr[tx_type];
+ int bitcol = av1_fwd_cos_bit_col[3][2];
+ int bitrow = av1_fwd_cos_bit_row[3][2];
+
+ // Column-wise transform.
+ int32x4_t buf0[128];
+ col_txfm(input, buf0, stride, bitcol, /*lr_flip=*/0, /*howmany=*/8,
+ /*hm_stride=*/16);
+ shift_right_4_round_s32_x4(buf0, buf0, 128);
+
+ int32x4_t buf1[128];
+ transpose_arrays_s32_32x16(buf0, buf1);
+
+ // Row-wise transform.
+ row_txfm(buf1, coeff, bitrow, /*howmany=*/4, /*hm_stride=*/32, /*stride=*/16);
}
#if !CONFIG_REALTIME_ONLY
void av1_fwd_txfm2d_8x32_neon(const int16_t *input, int32_t *coeff, int stride,
TX_TYPE tx_type, int bd) {
- int32x4_t in[64];
- int32x4_t *outcoef128 = (int32x4_t *)coeff;
- const int8_t *shift = av1_fwd_txfm_shift_ls[TX_8X32];
- const int txw_idx = get_txw_idx(TX_8X32);
- const int txh_idx = get_txh_idx(TX_8X32);
- const fwd_transform_1d_neon col_txfm = col_highbd_txfm8x32_arr[tx_type];
- const fwd_transform_1d_neon row_txfm = row_highbd_txfm32x8_arr[tx_type];
- int bitcol = av1_fwd_cos_bit_col[txw_idx][txh_idx];
- int bitrow = av1_fwd_cos_bit_row[txw_idx][txh_idx];
-
- const int txfm_size_col = tx_size_wide[TX_8X32];
- const int txfm_size_row = tx_size_high[TX_8X32];
- const int num_col = txfm_size_col >> 2;
-
- // column transform
- load_buffer_8x16(input, in, stride, 0, 0, shift[0]);
- load_buffer_8x16(input + (txfm_size_row >> 1) * stride, in + txfm_size_row,
- stride, 0, 0, shift[0]);
-
- for (int i = 0; i < num_col; i++) {
- col_txfm((in + i), (in + i), bitcol, num_col);
- }
-
- const int32x4_t v_shift = vdupq_n_s32(shift[1]);
- col_txfm_16x16_rounding(in, &v_shift);
- transpose_8nx8n(in, outcoef128, txfm_size_col, txfm_size_row);
-
- // row transform
- for (int i = 0; i < txfm_size_col; i += 2) {
- row_txfm((outcoef128 + i), (outcoef128 + i), bitrow, txfm_size_col);
- }
(void)bd;
+ const fwd_transform_1d_col_many_neon col_txfm =
+ col_highbd_txfm32_x4_arr[tx_type];
+ const fwd_transform_1d_row_many_neon row_txfm =
+ row_highbd_txfm8_xn_arr[tx_type];
+ int bitcol = av1_fwd_cos_bit_col[1][3];
+ int bitrow = av1_fwd_cos_bit_row[1][3];
+
+ // Column-wise transform.
+ int32x4_t buf0[64];
+ col_txfm(input, buf0, stride, bitcol, /*lr_flip=*/0, /*howmany=*/2,
+ /*hm_stride=*/32);
+ shift_right_2_round_s32_x4(buf0, buf0, 64);
+
+ int32x4_t buf1[64];
+ transpose_arrays_s32_8x32(buf0, buf1);
+
+ // Row-wise transform.
+ row_txfm(buf1, coeff, bitrow, /*howmany=*/8, /*hm_stride=*/8, /*stride=*/32);
}
void av1_fwd_txfm2d_32x8_neon(const int16_t *input, int32_t *coeff, int stride,
TX_TYPE tx_type, int bd) {
- int32x4_t in[64];
- int32x4_t *outcoef128 = (int32x4_t *)coeff;
- const int8_t *shift = av1_fwd_txfm_shift_ls[TX_32X8];
- const int txw_idx = get_txw_idx(TX_32X8);
- const int txh_idx = get_txh_idx(TX_32X8);
- const fwd_transform_1d_neon col_txfm = row_highbd_txfm32x8_arr[tx_type];
- const fwd_transform_1d_neon row_txfm = col_highbd_txfm8x32_arr[tx_type];
- int bitcol = av1_fwd_cos_bit_col[txw_idx][txh_idx];
- int bitrow = av1_fwd_cos_bit_row[txw_idx][txh_idx];
-
- const int txfm_size_col = tx_size_wide[TX_32X8];
- const int txfm_size_row = tx_size_high[TX_32X8];
- const int num_col = txfm_size_row >> 2;
-
- // column transform
- load_buffer_32x8n(input, in, stride, 0, 0, shift[0], 8);
- for (int i = 0; i < txfm_size_row; i += 2) {
- col_txfm((in + i), (in + i), bitcol, txfm_size_row);
- }
-
- const int32x4_t v_shift = vdupq_n_s32(shift[1]);
- col_txfm_16x16_rounding(&in[0], &v_shift);
- transpose_8nx8n(in, outcoef128, txfm_size_col, txfm_size_row);
-
- // row transform
- for (int i = 0; i < num_col; i++) {
- row_txfm((outcoef128 + i), (outcoef128 + i), bitrow, num_col);
- }
(void)bd;
+ const fwd_transform_1d_col_many_neon col_txfm =
+ col_highbd_txfm8_xn_arr[tx_type];
+ const fwd_transform_1d_row_many_neon row_txfm =
+ row_highbd_txfm32_x4_arr[tx_type];
+ int bitcol = av1_fwd_cos_bit_col[3][1];
+ int bitrow = av1_fwd_cos_bit_row[3][1];
+
+ // Column-wise transform.
+ int32x4_t buf0[64];
+ col_txfm(input, buf0, stride, bitcol, /*lr_flip=*/0, /*howmany=*/8,
+ /*hm_stride=*/8);
+ shift_right_2_round_s32_x4(buf0, buf0, 64);
+
+ int32x4_t buf1[64];
+ transpose_arrays_s32_32x8(buf0, buf1);
+
+ // Row-wise transform.
+ row_txfm(buf1, coeff, bitrow, /*howmany=*/2, /*hm_stride=*/32, /*stride=*/8);
}
#endif
void av1_fwd_txfm2d_4x8_neon(const int16_t *input, int32_t *coeff, int stride,
TX_TYPE tx_type, int bd) {
- int32x4_t in[8];
- const int8_t *shift = av1_fwd_txfm_shift_ls[TX_4X8];
- const int txw_idx = get_txw_idx(TX_4X8);
- const int txh_idx = get_txh_idx(TX_4X8);
- const int txfm_size_col = tx_size_wide[TX_4X8];
- const int txfm_size_row = tx_size_high[TX_4X8];
- int bitcol = av1_fwd_cos_bit_col[txw_idx][txh_idx];
- int bitrow = av1_fwd_cos_bit_row[txw_idx][txh_idx];
- const fwd_transform_1d_neon col_txfm = col_highbd_txfm4x8_arr[tx_type];
- const fwd_transform_1d_neon row_txfm = row_highbd_txfm4x4_arr[tx_type];
+ (void)bd;
+ int bitcol = av1_fwd_cos_bit_col[0][1];
+ int bitrow = av1_fwd_cos_bit_row[0][1];
+ const fwd_transform_1d_col_neon col_txfm = col_highbd_txfm8_x4_arr[tx_type];
+ const fwd_transform_1d_row_many_neon row_txfm =
+ row_rect_highbd_txfm4_xn_arr[tx_type];
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
- const int32x4_t v_shift0 = vdupq_n_s32(shift[0]);
- load_buffer_4x8(input, in, stride, ud_flip, lr_flip, &v_shift0);
- col_txfm(in, in, bitcol, 1);
- int32x4_t v_shift1 = vdupq_n_s32(shift[1]);
- col_txfm_4x8_rounding(in, &v_shift1);
+ ud_adjust_input_and_stride(ud_flip, &input, &stride, 8);
- for (int i = 0; i < 2; i++) {
- int32x4_t *cur_in = &in[i * 4];
- transpose_4x4(cur_in, cur_in);
- row_txfm(cur_in, cur_in, bitrow, 1);
- av1_round_shift_rect_array_32_neon(cur_in, cur_in, txfm_size_col, -shift[2],
- NewSqrt2);
- store_output_w4(coeff + i * 4, cur_in, txfm_size_row, 4);
- }
- (void)bd;
+ // Column-wise transform.
+ int32x4_t buf0[8];
+ col_txfm(input, buf0, stride, bitcol, lr_flip);
+ shift_right_1_round_s32_x4(buf0, buf0, 8);
+
+ int32x4_t buf1[8];
+ transpose_arrays_s32_4x8(buf0, buf1);
+
+ // Row-wise transform.
+ row_txfm(buf1, coeff, bitrow, /*howmany=*/2, /*hm_stride=*/4, /*stride=*/8);
}
void av1_fwd_txfm2d_8x4_neon(const int16_t *input, int32_t *coeff, int stride,
TX_TYPE tx_type, int bd) {
- int32x4_t in[8];
- int32x4_t *outcoeff128 = (int32x4_t *)coeff;
- const int8_t *shift = av1_fwd_txfm_shift_ls[TX_8X4];
- const int txw_idx = get_txw_idx(TX_8X4);
- const int txh_idx = get_txh_idx(TX_8X4);
- const int txfm_size_col = tx_size_wide[TX_8X4];
- const int txfm_size_row = tx_size_high[TX_8X4];
- int bitcol = av1_fwd_cos_bit_col[txw_idx][txh_idx];
- int bitrow = av1_fwd_cos_bit_row[txw_idx][txh_idx];
- const fwd_transform_1d_neon col_txfm = col_highbd_txfm4x4_arr[tx_type];
- const fwd_transform_1d_neon row_txfm = row_highbd_txfm4x8_arr[tx_type];
+ (void)bd;
+ const int bitcol = av1_fwd_cos_bit_col[1][0];
+ const int bitrow = av1_fwd_cos_bit_row[1][0];
+ const fwd_transform_1d_col_many_neon col_txfm =
+ col_highbd_txfm4_xn_arr[tx_type];
+ const fwd_transform_1d_row_neon row_txfm = row_highbd_txfm8_x4_arr[tx_type];
+
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
- // col tranform
- int32x4_t v_shift0 = vdupq_n_s32(shift[0]);
- load_buffer_8x4(input, in, stride, ud_flip, lr_flip, &v_shift0);
- for (int i = 0; i < 2; i++) {
- int32x4_t *cur_in = &in[i * txfm_size_row];
- col_txfm(cur_in, cur_in, bitcol, 1);
- transpose_4x4(cur_in, cur_in);
- }
- int32x4_t v_shift1 = vdupq_n_s32(shift[1]);
- col_txfm_4x8_rounding(in, &v_shift1);
+ ud_adjust_input_and_stride(ud_flip, &input, &stride, 4);
- // row tranform
- row_txfm(in, outcoeff128, bitrow, 1);
- av1_round_shift_rect_array_32_neon(outcoeff128, outcoeff128, txfm_size_col,
- -shift[2], NewSqrt2);
- (void)bd;
+ // Column-wise transform.
+ int32x4_t buf0[8];
+ if (lr_flip) {
+ col_txfm(input, buf0 + 4, stride, bitcol, /*lr_flip=*/1, /*howmany=*/2,
+ /*hm_stride=*/-4);
+ } else {
+ col_txfm(input, buf0, stride, bitcol, /*lr_flip=*/0, /*howmany=*/2,
+ /*hm_stride=*/4);
+ }
+
+ shift_right_1_round_s32_x4(buf0, buf0, 8);
+
+ int32x4_t buf1[8];
+ transpose_arrays_s32_8x4(buf0, buf1);
+
+ // Row-wise transform.
+ row_txfm(buf1, coeff, bitrow, /*stride=*/4);
}
#if !CONFIG_REALTIME_ONLY
void av1_fwd_txfm2d_16x64_neon(const int16_t *input, int32_t *coeff, int stride,
TX_TYPE tx_type, int bd) {
- int32x4_t in[256];
- int32x4_t *outcoeff128 = (int32x4_t *)coeff;
- const int8_t *shift = av1_fwd_txfm_shift_ls[TX_16X64];
- const int txw_idx = get_txw_idx(TX_16X64);
- const int txh_idx = get_txh_idx(TX_16X64);
- const int txfm_size_col = tx_size_wide[TX_16X64];
- const int txfm_size_row = tx_size_high[TX_16X64];
- int bitcol = av1_fwd_cos_bit_col[txw_idx][txh_idx];
- int bitrow = av1_fwd_cos_bit_row[txw_idx][txh_idx];
+ (void)bd;
+ const int bitcol = av1_fwd_cos_bit_col[2][4];
+ const int bitrow = av1_fwd_cos_bit_row[2][4];
+
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
- const int num_col = txfm_size_col >> 2;
- // col tranform
- const int32x4_t v_shift0 = vdupq_n_s32(shift[0]);
- for (int i = 0; i < txfm_size_row; i += num_col) {
- load_buffer_4x4(input + (i + 0) * stride, in + (i + 0) * num_col, num_col,
- ud_flip, lr_flip, &v_shift0);
- load_buffer_4x4(input + (i + 1) * stride, in + (i + 1) * num_col, num_col,
- ud_flip, lr_flip, &v_shift0);
- load_buffer_4x4(input + (i + 2) * stride, in + (i + 2) * num_col, num_col,
- ud_flip, lr_flip, &v_shift0);
- load_buffer_4x4(input + (i + 3) * stride, in + (i + 3) * num_col, num_col,
- ud_flip, lr_flip, &v_shift0);
+ ud_adjust_input_and_stride(ud_flip, &input, &stride, 64);
+
+ // Column-wise transform.
+ int32x4_t buf0[256];
+ load_buffer_16x64(input, buf0, stride, lr_flip);
+ for (int i = 0; i < 4; i++) {
+ highbd_fdct64_x4_neon(buf0 + i * 64, buf0 + i * 64, bitcol);
}
+ shift_right_2_round_s32_x4(buf0, buf0, 256);
- for (int i = 0; i < num_col; i++) {
- av1_fdct64_new_neon(in + i, outcoeff128 + i, bitcol, num_col, num_col);
- }
+ int32x4_t buf1[256];
+ transpose_arrays_s32_16x64(buf0, buf1);
- const int32x4_t v_shift = vdupq_n_s32(shift[1]);
- col_txfm_16x16_rounding(outcoeff128, &v_shift);
- col_txfm_16x16_rounding(outcoeff128 + 64, &v_shift);
- col_txfm_16x16_rounding(outcoeff128 + 128, &v_shift);
- col_txfm_16x16_rounding(outcoeff128 + 192, &v_shift);
-
- transpose_8nx8n(outcoeff128, in, txfm_size_col, 32);
- fdct16x16_neon(in, outcoeff128, bitrow, 8);
- (void)bd;
+ // Row-wise transform.
+ highbd_fdct16_xn_neon(buf1, buf1, bitrow, 8);
+ store_buffer_16x32(buf1, coeff, /*stride=*/32);
}
void av1_fwd_txfm2d_64x16_neon(const int16_t *input, int32_t *coeff, int stride,
TX_TYPE tx_type, int bd) {
- int32x4_t in[256];
- int32x4_t *outcoeff128 = (int32x4_t *)coeff;
- const int8_t *shift = av1_fwd_txfm_shift_ls[TX_64X16];
- const int txw_idx = get_txw_idx(TX_64X16);
- const int txh_idx = get_txh_idx(TX_64X16);
- const int txfm_size_col = tx_size_wide[TX_64X16];
- const int txfm_size_row = tx_size_high[TX_64X16];
- int bitcol = av1_fwd_cos_bit_col[txw_idx][txh_idx];
- int bitrow = av1_fwd_cos_bit_row[txw_idx][txh_idx];
+ (void)bd;
+ const int bitcol = av1_fwd_cos_bit_col[4][2];
+ const int bitrow = av1_fwd_cos_bit_row[4][2];
+
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
- // col tranform
- const int32x4_t v_shift0 = vdupq_n_s32(shift[0]);
- for (int i = 0; i < txfm_size_row; i++) {
- load_buffer_4x4(input + 0 + i * stride, in + 0 + i * txfm_size_row, 4,
- ud_flip, lr_flip, &v_shift0);
- load_buffer_4x4(input + 16 + i * stride, in + 4 + i * txfm_size_row, 4,
- ud_flip, lr_flip, &v_shift0);
- load_buffer_4x4(input + 32 + i * stride, in + 8 + i * txfm_size_row, 4,
- ud_flip, lr_flip, &v_shift0);
- load_buffer_4x4(input + 48 + i * stride, in + 12 + i * txfm_size_row, 4,
- ud_flip, lr_flip, &v_shift0);
- }
+ ud_adjust_input_and_stride(ud_flip, &input, &stride, 16);
- fdct16x16_neon(in, outcoeff128, bitcol, txfm_size_row);
- const int32x4_t v_shift = vdupq_n_s32(shift[1]);
- col_txfm_16x16_rounding(outcoeff128, &v_shift);
- col_txfm_16x16_rounding(outcoeff128 + 64, &v_shift);
- col_txfm_16x16_rounding(outcoeff128 + 128, &v_shift);
- col_txfm_16x16_rounding(outcoeff128 + 192, &v_shift);
+ // Column-wise transform.
+ int32x4_t buf0[256];
+ load_buffer_64x16(input, buf0, stride, lr_flip);
+ highbd_fdct16_xn_neon(buf0, buf0, bitcol, 16);
+ shift_right_4_round_s32_x4(buf0, buf0, 256);
- transpose_8nx8n(outcoeff128, in, txfm_size_col, txfm_size_row);
+ int32x4_t buf1[256];
+ transpose_arrays_s32_64x16(buf0, buf1);
+
+ // Row-wise transform.
for (int i = 0; i < 4; i++) {
- av1_fdct64_new_neon(in + i, outcoeff128 + i, bitrow, 4, 4);
+ highbd_fdct64_x4_neon(buf1 + i * 64, buf1 + i * 64, bitrow);
}
- memset(coeff + txfm_size_row * 32, 0, txfm_size_row * 32 * sizeof(*coeff));
- (void)bd;
+ store_buffer_64x16(buf1, coeff, /*stride=*/16);
+ memset(coeff + 16 * 32, 0, 16 * 32 * sizeof(*coeff));
}
#endif
-static void fdct64_new_neon(int32x4_t *input, int32x4_t *output,
- const int8_t cos_bit, const int8_t *stage_range) {
- const int txfm_size = 64;
- const int num_per_128 = 4;
- int col_num = txfm_size / num_per_128;
- (void)stage_range;
- for (int col = 0; col < col_num; col++) {
- av1_fdct64_new_neon((input + col), (output + col), cos_bit, col_num,
- col_num);
- }
-}
-
-static void fdct32_new_neon(int32x4_t *input, int32x4_t *output,
- const int8_t cos_bit, const int8_t *stage_range) {
- const int txfm_size = 32;
- const int num_per_128 = 4;
- int col_num = txfm_size / num_per_128;
- int col;
- (void)stage_range;
- for (col = 0; col < col_num; col++) {
- av1_fdct32_new_neon((input + col), (output + col), cos_bit, col_num);
- }
-}
-
-static void idtx32x32_neon(int32x4_t *input, int32x4_t *output,
- const int8_t cos_bit, const int8_t *stage_range) {
- (void)stage_range;
-
- for (int i = 0; i < 8; i++) {
- av1_idtx32_new_neon(&input[i * 32], &output[i * 32], cos_bit, 1);
- }
-}
-
-typedef void (*TxfmFuncNEON)(int32x4_t *input, int32x4_t *output,
- const int8_t cos_bit, const int8_t *stage_range);
-
-static INLINE TxfmFuncNEON fwd_txfm_type_to_func(TXFM_TYPE txfm_type) {
- switch (txfm_type) {
- case TXFM_TYPE_DCT32: return fdct32_new_neon;
- case TXFM_TYPE_DCT64: return fdct64_new_neon;
- case TXFM_TYPE_IDENTITY32: return idtx32x32_neon;
- default: assert(0);
- }
- return NULL;
-}
-
-static INLINE void int16_array_with_stride_to_int32_array_without_stride(
- const int16_t *input, int stride, int32_t *output, int txfm1d_size) {
- int r, c;
- for (r = 0; r < txfm1d_size; r++) {
- for (c = 0; c < txfm1d_size; c++) {
- output[r * txfm1d_size + c] = (int32_t)input[r * stride + c];
- }
- }
-}
-
-static INLINE void av1_round_shift_array_32_neon(int32x4_t *input,
- int32x4_t *output,
- const int size,
- const int bit) {
- const int32x4_t v_bit = vdupq_n_s32(-bit);
- for (int i = 0; i < size; i++) output[i] = vrshlq_s32(input[i], v_bit);
-}
-
-static INLINE void transpose_32_4x4(int stride, const int32x4_t *input,
- int32x4_t *output) {
- int32x4x2_t temp01 = vzipq_s32(input[0 * stride], input[2 * stride]);
- int32x4x2_t temp23 = vzipq_s32(input[1 * stride], input[3 * stride]);
-
- const int32x4x2_t output01 = vzipq_s32(temp01.val[0], temp23.val[0]);
- const int32x4x2_t output23 = vzipq_s32(temp01.val[1], temp23.val[1]);
-
- output[0 * stride] = output01.val[0];
- output[1 * stride] = output01.val[1];
- output[2 * stride] = output23.val[0];
- output[3 * stride] = output23.val[1];
-}
-
-static INLINE void transpose_32(int txfm_size, const int32x4_t *input,
- int32x4_t *output) {
- const int num_per_128 = 4;
- const int row_size = txfm_size;
- const int col_size = txfm_size / num_per_128;
- int r, c;
-
- // transpose each 4x4 block internally
- for (r = 0; r < row_size; r += 4) {
- for (c = 0; c < col_size; c++) {
- transpose_32_4x4(col_size, &input[r * col_size + c],
- &output[c * 4 * col_size + r / 4]);
- }
- }
-}
-
-static INLINE void fwd_txfm2d_64x64_neon(const int16_t *input, int32_t *output,
- const int stride,
- const TXFM_2D_FLIP_CFG *cfg,
- int32_t *txfm_buf) {
- assert(cfg->tx_size < TX_SIZES);
- const int txfm_size = tx_size_wide[cfg->tx_size];
- const int8_t *shift = cfg->shift;
- const int8_t *stage_range_col = cfg->stage_range_col;
- const int8_t cos_bit_col = cfg->cos_bit_col;
- const int8_t cos_bit_row = cfg->cos_bit_row;
- const TxfmFuncNEON txfm_func_col = fwd_txfm_type_to_func(cfg->txfm_type_col);
- int32x4_t *buf_128 = (int32x4_t *)txfm_buf;
- int32x4_t *out_128 = (int32x4_t *)output;
-
- const int num_per_128 = 4;
- int txfm2d_size_128 = txfm_size * txfm_size / num_per_128;
- int col_num = txfm_size / num_per_128;
-
- int16_array_with_stride_to_int32_array_without_stride(input, stride, output,
- txfm_size);
- /*col wise transform*/
- txfm_func_col(out_128, buf_128, cos_bit_col, stage_range_col);
- av1_round_shift_array_32_neon(buf_128, out_128, txfm2d_size_128, -shift[1]);
- transpose_32(txfm_size, out_128, buf_128);
-
- /*row wise transform*/
- for (int col = 0; col < (col_num >> 1); col++) {
- av1_fdct64_new_neon((buf_128 + col), (out_128 + col), cos_bit_row, col_num,
- (col_num >> 1));
- }
-
- txfm2d_size_128 = (col_num >> 1) * (txfm_size >> 1);
- av1_round_shift_array_32_neon(out_128, out_128, txfm2d_size_128, -shift[2]);
-}
-
-static INLINE void fwd_txfm2d_neon(const int16_t *input, int32_t *output,
- const int stride,
- const TXFM_2D_FLIP_CFG *cfg,
- int32_t *txfm_buf) {
- assert(cfg->tx_size < TX_SIZES);
- const int txfm_size = tx_size_wide[cfg->tx_size];
- const int8_t *shift = cfg->shift;
- const int8_t *stage_range_col = cfg->stage_range_col;
- const int8_t *stage_range_row = cfg->stage_range_row;
- const int8_t cos_bit_col = cfg->cos_bit_col;
- const int8_t cos_bit_row = cfg->cos_bit_row;
- const TxfmFuncNEON txfm_func_col = fwd_txfm_type_to_func(cfg->txfm_type_col);
- const TxfmFuncNEON txfm_func_row = fwd_txfm_type_to_func(cfg->txfm_type_row);
-
- int32x4_t *buf_128 = (int32x4_t *)txfm_buf;
- int32x4_t *out_128 = (int32x4_t *)output;
- int num_per_128 = 4;
- int txfm2d_size_128 = txfm_size * txfm_size / num_per_128;
-
- int16_array_with_stride_to_int32_array_without_stride(input, stride, txfm_buf,
- txfm_size);
- av1_round_shift_array_32_neon(buf_128, out_128, txfm2d_size_128, -shift[0]);
- txfm_func_col(out_128, buf_128, cos_bit_col, stage_range_col);
- av1_round_shift_array_32_neon(buf_128, out_128, txfm2d_size_128, -shift[1]);
- transpose_32(txfm_size, out_128, buf_128);
- txfm_func_row(buf_128, out_128, cos_bit_row, stage_range_row);
- av1_round_shift_array_32_neon(out_128, out_128, txfm2d_size_128, -shift[2]);
-}
-
void av1_fwd_txfm2d_32x32_neon(const int16_t *input, int32_t *output,
int stride, TX_TYPE tx_type, int bd) {
- DECLARE_ALIGNED(16, int32_t, txfm_buf[1024]);
- TXFM_2D_FLIP_CFG cfg;
- av1_get_fwd_txfm_cfg(tx_type, TX_32X32, &cfg);
(void)bd;
- fwd_txfm2d_neon(input, output, stride, &cfg, txfm_buf);
+ const fwd_transform_1d_col_many_neon col_txfm =
+ col_highbd_txfm32_x4_arr[tx_type];
+ const fwd_transform_1d_row_many_neon row_txfm =
+ row_highbd_txfm32_x4_arr[tx_type];
+
+ // Column-wise transform.
+ int32x4_t buf0[256];
+ col_txfm(input, buf0, stride, /*cos_bit=*/12, /*lr_flip=*/0, /*howmany=*/8,
+ /*hm_stride=*/32);
+ shift_right_4_round_s32_x4(buf0, buf0, 256);
+
+ int32x4_t buf1[256];
+ transpose_arrays_s32_32x32(buf0, buf1);
+
+ // Row-wise transform.
+ row_txfm(buf1, output, /*cos_bit=*/12, /*howmany=*/8, /*hm_stride=*/32,
+ /*stride=*/32);
}
void av1_fwd_txfm2d_64x64_neon(const int16_t *input, int32_t *output,
int stride, TX_TYPE tx_type, int bd) {
- DECLARE_ALIGNED(16, int32_t, txfm_buf[4096]);
- TXFM_2D_FLIP_CFG cfg;
- av1_get_fwd_txfm_cfg(tx_type, TX_64X64, &cfg);
(void)bd;
- fwd_txfm2d_64x64_neon(input, output, stride, &cfg, txfm_buf);
+ (void)tx_type;
+
+ // Column-wise transform.
+ int32x4_t buf0[1024];
+ load_buffer_64x64(input, buf0, stride, 0);
+ for (int col = 0; col < 16; col++) {
+ highbd_fdct64_x4_neon(buf0 + col * 64, buf0 + col * 64, 13);
+ }
+ shift_right_2_round_s32_x4(buf0, buf0, 1024);
+
+ int32x4_t buf1[1024];
+ transpose_arrays_s32_64x64(buf0, buf1);
+
+ // Row-wise transform.
+ for (int col = 0; col < 8; col++) {
+ highbd_fdct64_x4_neon(buf1 + col * 64, buf1 + col * 64, 10);
+ }
+ shift_right_2_round_s32_x4(buf1, buf1, 512);
+ store_buffer_64x32(buf1, output, /*stride=*/32);
}
diff --git a/av1/encoder/arm/neon/highbd_pickrst_neon.c b/av1/encoder/arm/neon/highbd_pickrst_neon.c
new file mode 100644
index 0000000..76e0344
--- /dev/null
+++ b/av1/encoder/arm/neon/highbd_pickrst_neon.c
@@ -0,0 +1,741 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+#include <assert.h>
+#include <stdint.h>
+
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_dsp/arm/sum_neon.h"
+#include "aom_dsp/arm/transpose_neon.h"
+#include "av1/encoder/arm/neon/pickrst_neon.h"
+#include "av1/encoder/pickrst.h"
+
+static INLINE void highbd_calc_proj_params_r0_r1_neon(
+ const uint8_t *src8, int width, int height, int src_stride,
+ const uint8_t *dat8, int dat_stride, int32_t *flt0, int flt0_stride,
+ int32_t *flt1, int flt1_stride, int64_t H[2][2], int64_t C[2]) {
+ assert(width % 8 == 0);
+ const int size = width * height;
+ const uint16_t *src = CONVERT_TO_SHORTPTR(src8);
+ const uint16_t *dat = CONVERT_TO_SHORTPTR(dat8);
+
+ int64x2_t h00_lo = vdupq_n_s64(0);
+ int64x2_t h00_hi = vdupq_n_s64(0);
+ int64x2_t h11_lo = vdupq_n_s64(0);
+ int64x2_t h11_hi = vdupq_n_s64(0);
+ int64x2_t h01_lo = vdupq_n_s64(0);
+ int64x2_t h01_hi = vdupq_n_s64(0);
+ int64x2_t c0_lo = vdupq_n_s64(0);
+ int64x2_t c0_hi = vdupq_n_s64(0);
+ int64x2_t c1_lo = vdupq_n_s64(0);
+ int64x2_t c1_hi = vdupq_n_s64(0);
+
+ do {
+ const uint16_t *src_ptr = src;
+ const uint16_t *dat_ptr = dat;
+ int32_t *flt0_ptr = flt0;
+ int32_t *flt1_ptr = flt1;
+ int w = width;
+
+ do {
+ uint16x8_t s = vld1q_u16(src_ptr);
+ uint16x8_t d = vld1q_u16(dat_ptr);
+ int32x4_t f0_lo = vld1q_s32(flt0_ptr);
+ int32x4_t f0_hi = vld1q_s32(flt0_ptr + 4);
+ int32x4_t f1_lo = vld1q_s32(flt1_ptr);
+ int32x4_t f1_hi = vld1q_s32(flt1_ptr + 4);
+
+ int32x4_t u_lo =
+ vreinterpretq_s32_u32(vshll_n_u16(vget_low_u16(d), SGRPROJ_RST_BITS));
+ int32x4_t u_hi = vreinterpretq_s32_u32(
+ vshll_n_u16(vget_high_u16(d), SGRPROJ_RST_BITS));
+ int32x4_t s_lo =
+ vreinterpretq_s32_u32(vshll_n_u16(vget_low_u16(s), SGRPROJ_RST_BITS));
+ int32x4_t s_hi = vreinterpretq_s32_u32(
+ vshll_n_u16(vget_high_u16(s), SGRPROJ_RST_BITS));
+ s_lo = vsubq_s32(s_lo, u_lo);
+ s_hi = vsubq_s32(s_hi, u_hi);
+
+ f0_lo = vsubq_s32(f0_lo, u_lo);
+ f0_hi = vsubq_s32(f0_hi, u_hi);
+ f1_lo = vsubq_s32(f1_lo, u_lo);
+ f1_hi = vsubq_s32(f1_hi, u_hi);
+
+ h00_lo = vmlal_s32(h00_lo, vget_low_s32(f0_lo), vget_low_s32(f0_lo));
+ h00_lo = vmlal_s32(h00_lo, vget_high_s32(f0_lo), vget_high_s32(f0_lo));
+ h00_hi = vmlal_s32(h00_hi, vget_low_s32(f0_hi), vget_low_s32(f0_hi));
+ h00_hi = vmlal_s32(h00_hi, vget_high_s32(f0_hi), vget_high_s32(f0_hi));
+
+ h11_lo = vmlal_s32(h11_lo, vget_low_s32(f1_lo), vget_low_s32(f1_lo));
+ h11_lo = vmlal_s32(h11_lo, vget_high_s32(f1_lo), vget_high_s32(f1_lo));
+ h11_hi = vmlal_s32(h11_hi, vget_low_s32(f1_hi), vget_low_s32(f1_hi));
+ h11_hi = vmlal_s32(h11_hi, vget_high_s32(f1_hi), vget_high_s32(f1_hi));
+
+ h01_lo = vmlal_s32(h01_lo, vget_low_s32(f0_lo), vget_low_s32(f1_lo));
+ h01_lo = vmlal_s32(h01_lo, vget_high_s32(f0_lo), vget_high_s32(f1_lo));
+ h01_hi = vmlal_s32(h01_hi, vget_low_s32(f0_hi), vget_low_s32(f1_hi));
+ h01_hi = vmlal_s32(h01_hi, vget_high_s32(f0_hi), vget_high_s32(f1_hi));
+
+ c0_lo = vmlal_s32(c0_lo, vget_low_s32(f0_lo), vget_low_s32(s_lo));
+ c0_lo = vmlal_s32(c0_lo, vget_high_s32(f0_lo), vget_high_s32(s_lo));
+ c0_hi = vmlal_s32(c0_hi, vget_low_s32(f0_hi), vget_low_s32(s_hi));
+ c0_hi = vmlal_s32(c0_hi, vget_high_s32(f0_hi), vget_high_s32(s_hi));
+
+ c1_lo = vmlal_s32(c1_lo, vget_low_s32(f1_lo), vget_low_s32(s_lo));
+ c1_lo = vmlal_s32(c1_lo, vget_high_s32(f1_lo), vget_high_s32(s_lo));
+ c1_hi = vmlal_s32(c1_hi, vget_low_s32(f1_hi), vget_low_s32(s_hi));
+ c1_hi = vmlal_s32(c1_hi, vget_high_s32(f1_hi), vget_high_s32(s_hi));
+
+ src_ptr += 8;
+ dat_ptr += 8;
+ flt0_ptr += 8;
+ flt1_ptr += 8;
+ w -= 8;
+ } while (w != 0);
+
+ src += src_stride;
+ dat += dat_stride;
+ flt0 += flt0_stride;
+ flt1 += flt1_stride;
+ } while (--height != 0);
+
+ H[0][0] = horizontal_add_s64x2(vaddq_s64(h00_lo, h00_hi)) / size;
+ H[0][1] = horizontal_add_s64x2(vaddq_s64(h01_lo, h01_hi)) / size;
+ H[1][1] = horizontal_add_s64x2(vaddq_s64(h11_lo, h11_hi)) / size;
+ H[1][0] = H[0][1];
+ C[0] = horizontal_add_s64x2(vaddq_s64(c0_lo, c0_hi)) / size;
+ C[1] = horizontal_add_s64x2(vaddq_s64(c1_lo, c1_hi)) / size;
+}
+
+static INLINE void highbd_calc_proj_params_r0_neon(
+ const uint8_t *src8, int width, int height, int src_stride,
+ const uint8_t *dat8, int dat_stride, int32_t *flt0, int flt0_stride,
+ int64_t H[2][2], int64_t C[2]) {
+ assert(width % 8 == 0);
+ const int size = width * height;
+ const uint16_t *src = CONVERT_TO_SHORTPTR(src8);
+ const uint16_t *dat = CONVERT_TO_SHORTPTR(dat8);
+
+ int64x2_t h00_lo = vdupq_n_s64(0);
+ int64x2_t h00_hi = vdupq_n_s64(0);
+ int64x2_t c0_lo = vdupq_n_s64(0);
+ int64x2_t c0_hi = vdupq_n_s64(0);
+
+ do {
+ const uint16_t *src_ptr = src;
+ const uint16_t *dat_ptr = dat;
+ int32_t *flt0_ptr = flt0;
+ int w = width;
+
+ do {
+ uint16x8_t s = vld1q_u16(src_ptr);
+ uint16x8_t d = vld1q_u16(dat_ptr);
+ int32x4_t f0_lo = vld1q_s32(flt0_ptr);
+ int32x4_t f0_hi = vld1q_s32(flt0_ptr + 4);
+
+ int32x4_t u_lo =
+ vreinterpretq_s32_u32(vshll_n_u16(vget_low_u16(d), SGRPROJ_RST_BITS));
+ int32x4_t u_hi = vreinterpretq_s32_u32(
+ vshll_n_u16(vget_high_u16(d), SGRPROJ_RST_BITS));
+ int32x4_t s_lo =
+ vreinterpretq_s32_u32(vshll_n_u16(vget_low_u16(s), SGRPROJ_RST_BITS));
+ int32x4_t s_hi = vreinterpretq_s32_u32(
+ vshll_n_u16(vget_high_u16(s), SGRPROJ_RST_BITS));
+ s_lo = vsubq_s32(s_lo, u_lo);
+ s_hi = vsubq_s32(s_hi, u_hi);
+
+ f0_lo = vsubq_s32(f0_lo, u_lo);
+ f0_hi = vsubq_s32(f0_hi, u_hi);
+
+ h00_lo = vmlal_s32(h00_lo, vget_low_s32(f0_lo), vget_low_s32(f0_lo));
+ h00_lo = vmlal_s32(h00_lo, vget_high_s32(f0_lo), vget_high_s32(f0_lo));
+ h00_hi = vmlal_s32(h00_hi, vget_low_s32(f0_hi), vget_low_s32(f0_hi));
+ h00_hi = vmlal_s32(h00_hi, vget_high_s32(f0_hi), vget_high_s32(f0_hi));
+
+ c0_lo = vmlal_s32(c0_lo, vget_low_s32(f0_lo), vget_low_s32(s_lo));
+ c0_lo = vmlal_s32(c0_lo, vget_high_s32(f0_lo), vget_high_s32(s_lo));
+ c0_hi = vmlal_s32(c0_hi, vget_low_s32(f0_hi), vget_low_s32(s_hi));
+ c0_hi = vmlal_s32(c0_hi, vget_high_s32(f0_hi), vget_high_s32(s_hi));
+
+ src_ptr += 8;
+ dat_ptr += 8;
+ flt0_ptr += 8;
+ w -= 8;
+ } while (w != 0);
+
+ src += src_stride;
+ dat += dat_stride;
+ flt0 += flt0_stride;
+ } while (--height != 0);
+
+ H[0][0] = horizontal_add_s64x2(vaddq_s64(h00_lo, h00_hi)) / size;
+ C[0] = horizontal_add_s64x2(vaddq_s64(c0_lo, c0_hi)) / size;
+}
+
+static INLINE void highbd_calc_proj_params_r1_neon(
+ const uint8_t *src8, int width, int height, int src_stride,
+ const uint8_t *dat8, int dat_stride, int32_t *flt1, int flt1_stride,
+ int64_t H[2][2], int64_t C[2]) {
+ assert(width % 8 == 0);
+ const int size = width * height;
+ const uint16_t *src = CONVERT_TO_SHORTPTR(src8);
+ const uint16_t *dat = CONVERT_TO_SHORTPTR(dat8);
+
+ int64x2_t h11_lo = vdupq_n_s64(0);
+ int64x2_t h11_hi = vdupq_n_s64(0);
+ int64x2_t c1_lo = vdupq_n_s64(0);
+ int64x2_t c1_hi = vdupq_n_s64(0);
+
+ do {
+ const uint16_t *src_ptr = src;
+ const uint16_t *dat_ptr = dat;
+ int32_t *flt1_ptr = flt1;
+ int w = width;
+
+ do {
+ uint16x8_t s = vld1q_u16(src_ptr);
+ uint16x8_t d = vld1q_u16(dat_ptr);
+ int32x4_t f1_lo = vld1q_s32(flt1_ptr);
+ int32x4_t f1_hi = vld1q_s32(flt1_ptr + 4);
+
+ int32x4_t u_lo =
+ vreinterpretq_s32_u32(vshll_n_u16(vget_low_u16(d), SGRPROJ_RST_BITS));
+ int32x4_t u_hi = vreinterpretq_s32_u32(
+ vshll_n_u16(vget_high_u16(d), SGRPROJ_RST_BITS));
+ int32x4_t s_lo =
+ vreinterpretq_s32_u32(vshll_n_u16(vget_low_u16(s), SGRPROJ_RST_BITS));
+ int32x4_t s_hi = vreinterpretq_s32_u32(
+ vshll_n_u16(vget_high_u16(s), SGRPROJ_RST_BITS));
+ s_lo = vsubq_s32(s_lo, u_lo);
+ s_hi = vsubq_s32(s_hi, u_hi);
+
+ f1_lo = vsubq_s32(f1_lo, u_lo);
+ f1_hi = vsubq_s32(f1_hi, u_hi);
+
+ h11_lo = vmlal_s32(h11_lo, vget_low_s32(f1_lo), vget_low_s32(f1_lo));
+ h11_lo = vmlal_s32(h11_lo, vget_high_s32(f1_lo), vget_high_s32(f1_lo));
+ h11_hi = vmlal_s32(h11_hi, vget_low_s32(f1_hi), vget_low_s32(f1_hi));
+ h11_hi = vmlal_s32(h11_hi, vget_high_s32(f1_hi), vget_high_s32(f1_hi));
+
+ c1_lo = vmlal_s32(c1_lo, vget_low_s32(f1_lo), vget_low_s32(s_lo));
+ c1_lo = vmlal_s32(c1_lo, vget_high_s32(f1_lo), vget_high_s32(s_lo));
+ c1_hi = vmlal_s32(c1_hi, vget_low_s32(f1_hi), vget_low_s32(s_hi));
+ c1_hi = vmlal_s32(c1_hi, vget_high_s32(f1_hi), vget_high_s32(s_hi));
+
+ src_ptr += 8;
+ dat_ptr += 8;
+ flt1_ptr += 8;
+ w -= 8;
+ } while (w != 0);
+
+ src += src_stride;
+ dat += dat_stride;
+ flt1 += flt1_stride;
+ } while (--height != 0);
+
+ H[1][1] = horizontal_add_s64x2(vaddq_s64(h11_lo, h11_hi)) / size;
+ C[1] = horizontal_add_s64x2(vaddq_s64(c1_lo, c1_hi)) / size;
+}
+
+// The function calls 3 subfunctions for the following cases :
+// 1) When params->r[0] > 0 and params->r[1] > 0. In this case all elements
+// of C and H need to be computed.
+// 2) When only params->r[0] > 0. In this case only H[0][0] and C[0] are
+// non-zero and need to be computed.
+// 3) When only params->r[1] > 0. In this case only H[1][1] and C[1] are
+// non-zero and need to be computed.
+void av1_calc_proj_params_high_bd_neon(const uint8_t *src8, int width,
+ int height, int src_stride,
+ const uint8_t *dat8, int dat_stride,
+ int32_t *flt0, int flt0_stride,
+ int32_t *flt1, int flt1_stride,
+ int64_t H[2][2], int64_t C[2],
+ const sgr_params_type *params) {
+ if ((params->r[0] > 0) && (params->r[1] > 0)) {
+ highbd_calc_proj_params_r0_r1_neon(src8, width, height, src_stride, dat8,
+ dat_stride, flt0, flt0_stride, flt1,
+ flt1_stride, H, C);
+ } else if (params->r[0] > 0) {
+ highbd_calc_proj_params_r0_neon(src8, width, height, src_stride, dat8,
+ dat_stride, flt0, flt0_stride, H, C);
+ } else if (params->r[1] > 0) {
+ highbd_calc_proj_params_r1_neon(src8, width, height, src_stride, dat8,
+ dat_stride, flt1, flt1_stride, H, C);
+ }
+}
+
+static int16_t highbd_find_average_neon(const int16_t *src, int src_stride,
+ int width, int height) {
+ assert(width > 0);
+ assert(height > 0);
+
+ int64x2_t sum_s64 = vdupq_n_s64(0);
+ int64_t sum = 0;
+
+ int h = height;
+ do {
+ int32x4_t sum_s32[2] = { vdupq_n_s32(0), vdupq_n_s32(0) };
+
+ int w = width;
+ const int16_t *row = src;
+ while (w >= 32) {
+ int16x8_t s0 = vld1q_s16(row + 0);
+ int16x8_t s1 = vld1q_s16(row + 8);
+ int16x8_t s2 = vld1q_s16(row + 16);
+ int16x8_t s3 = vld1q_s16(row + 24);
+
+ s0 = vaddq_s16(s0, s1);
+ s2 = vaddq_s16(s2, s3);
+ sum_s32[0] = vpadalq_s16(sum_s32[0], s0);
+ sum_s32[1] = vpadalq_s16(sum_s32[1], s2);
+
+ row += 32;
+ w -= 32;
+ }
+
+ if (w >= 16) {
+ int16x8_t s0 = vld1q_s16(row + 0);
+ int16x8_t s1 = vld1q_s16(row + 8);
+
+ s0 = vaddq_s16(s0, s1);
+ sum_s32[0] = vpadalq_s16(sum_s32[0], s0);
+
+ row += 16;
+ w -= 16;
+ }
+
+ if (w >= 8) {
+ int16x8_t s0 = vld1q_s16(row);
+ sum_s32[1] = vpadalq_s16(sum_s32[1], s0);
+
+ row += 8;
+ w -= 8;
+ }
+
+ if (w >= 4) {
+ int16x8_t s0 = vcombine_s16(vld1_s16(row), vdup_n_s16(0));
+ sum_s32[0] = vpadalq_s16(sum_s32[0], s0);
+
+ row += 4;
+ w -= 4;
+ }
+
+ while (w-- > 0) {
+ sum += *row++;
+ }
+
+ sum_s64 = vpadalq_s32(sum_s64, vaddq_s32(sum_s32[0], sum_s32[1]));
+
+ src += src_stride;
+ } while (--h != 0);
+ return (int16_t)((horizontal_add_s64x2(sum_s64) + sum) / (height * width));
+}
+
+static INLINE void compute_H_one_col(int16x8_t *dgd, int col, int64_t *H,
+ const int wiener_win,
+ const int wiener_win2) {
+ for (int row0 = 0; row0 < wiener_win; row0++) {
+ for (int row1 = row0; row1 < wiener_win; row1++) {
+ int auto_cov_idx =
+ (col * wiener_win + row0) * wiener_win2 + (col * wiener_win) + row1;
+
+ int32x4_t auto_cov =
+ vmull_s16(vget_low_s16(dgd[row0]), vget_low_s16(dgd[row1]));
+ auto_cov = vmlal_s16(auto_cov, vget_high_s16(dgd[row0]),
+ vget_high_s16(dgd[row1]));
+
+ H[auto_cov_idx] += horizontal_long_add_s32x4(auto_cov);
+ }
+ }
+}
+
+// This function computes two matrices: the cross-correlation between the src
+// buffer and dgd buffer (M), and the auto-covariance of the dgd buffer (H).
+//
+// M is of size 7 * 7. It needs to be filled such that multiplying one element
+// from src with each element of a row of the wiener window will fill one
+// column of M. However this is not very convenient in terms of memory
+// accesses, as it means we do contiguous loads of dgd but strided stores to M.
+// As a result, we use an intermediate matrix M_trn which is instead filled
+// such that one row of the wiener window gives one row of M_trn. Once fully
+// computed, M_trn is then transposed to return M.
+//
+// H is of size 49 * 49. It is filled by multiplying every pair of elements of
+// the wiener window together. Since it is a symmetric matrix, we only compute
+// the upper triangle, and then copy it down to the lower one. Here we fill it
+// by taking each different pair of columns, and multiplying all the elements of
+// the first one with all the elements of the second one, with a special case
+// when multiplying a column by itself.
+static INLINE void highbd_compute_stats_win7_neon(
+ const int16_t *dgd, int dgd_stride, const int16_t *src, int src_stride,
+ int width, int height, int64_t *M, int64_t *H, int16_t avg, int bit_depth) {
+ const int wiener_win = 7;
+ const int wiener_win2 = wiener_win * wiener_win;
+ const int16x8_t mask = vld1q_s16(&av1_neon_mask_16bit[8] - (width % 8));
+
+ // We use an intermediate matrix that will be transposed to get M.
+ int64_t M_trn[49];
+ memset(M_trn, 0, sizeof(M_trn));
+
+ int16x8_t vavg = vdupq_n_s16(avg);
+ do {
+ // Cross-correlation (M).
+ for (int row = 0; row < wiener_win; row++) {
+ int16x8_t dgd0 = vsubq_s16(vld1q_s16(dgd + row * dgd_stride), vavg);
+ int j = 0;
+ while (j <= width - 8) {
+ int16x8_t dgd1 =
+ vsubq_s16(vld1q_s16(dgd + row * dgd_stride + j + 8), vavg);
+ int16x8_t s = vsubq_s16(vld1q_s16(src + j), vavg);
+
+ // Compute all the elements of one row of M.
+ compute_M_one_row_win7(s, dgd0, dgd1, M_trn, wiener_win, row);
+
+ dgd0 = dgd1;
+ j += 8;
+ }
+ // Process remaining elements without Neon.
+ while (j < width) {
+ int16_t s = src[j] - avg;
+ int16_t d0 = dgd[row * dgd_stride + 0 + j] - avg;
+ int16_t d1 = dgd[row * dgd_stride + 1 + j] - avg;
+ int16_t d2 = dgd[row * dgd_stride + 2 + j] - avg;
+ int16_t d3 = dgd[row * dgd_stride + 3 + j] - avg;
+ int16_t d4 = dgd[row * dgd_stride + 4 + j] - avg;
+ int16_t d5 = dgd[row * dgd_stride + 5 + j] - avg;
+ int16_t d6 = dgd[row * dgd_stride + 6 + j] - avg;
+
+ M_trn[row * wiener_win + 0] += d0 * s;
+ M_trn[row * wiener_win + 1] += d1 * s;
+ M_trn[row * wiener_win + 2] += d2 * s;
+ M_trn[row * wiener_win + 3] += d3 * s;
+ M_trn[row * wiener_win + 4] += d4 * s;
+ M_trn[row * wiener_win + 5] += d5 * s;
+ M_trn[row * wiener_win + 6] += d6 * s;
+
+ j++;
+ }
+ }
+
+ // Auto-covariance (H).
+ int j = 0;
+ while (j <= width - 8) {
+ for (int col0 = 0; col0 < wiener_win; col0++) {
+ int16x8_t dgd0[7];
+ dgd0[0] = vsubq_s16(vld1q_s16(dgd + 0 * dgd_stride + j + col0), vavg);
+ dgd0[1] = vsubq_s16(vld1q_s16(dgd + 1 * dgd_stride + j + col0), vavg);
+ dgd0[2] = vsubq_s16(vld1q_s16(dgd + 2 * dgd_stride + j + col0), vavg);
+ dgd0[3] = vsubq_s16(vld1q_s16(dgd + 3 * dgd_stride + j + col0), vavg);
+ dgd0[4] = vsubq_s16(vld1q_s16(dgd + 4 * dgd_stride + j + col0), vavg);
+ dgd0[5] = vsubq_s16(vld1q_s16(dgd + 5 * dgd_stride + j + col0), vavg);
+ dgd0[6] = vsubq_s16(vld1q_s16(dgd + 6 * dgd_stride + j + col0), vavg);
+
+ // Perform computation of the first column with itself (28 elements).
+ // For the first column this will fill the upper triangle of the 7x7
+ // matrix at the top left of the H matrix. For the next columns this
+ // will fill the upper triangle of the other 7x7 matrices around H's
+ // diagonal.
+ compute_H_one_col(dgd0, col0, H, wiener_win, wiener_win2);
+
+ // All computation next to the matrix diagonal has already been done.
+ for (int col1 = col0 + 1; col1 < wiener_win; col1++) {
+ // Load second column.
+ int16x8_t dgd1[7];
+ dgd1[0] = vsubq_s16(vld1q_s16(dgd + 0 * dgd_stride + j + col1), vavg);
+ dgd1[1] = vsubq_s16(vld1q_s16(dgd + 1 * dgd_stride + j + col1), vavg);
+ dgd1[2] = vsubq_s16(vld1q_s16(dgd + 2 * dgd_stride + j + col1), vavg);
+ dgd1[3] = vsubq_s16(vld1q_s16(dgd + 3 * dgd_stride + j + col1), vavg);
+ dgd1[4] = vsubq_s16(vld1q_s16(dgd + 4 * dgd_stride + j + col1), vavg);
+ dgd1[5] = vsubq_s16(vld1q_s16(dgd + 5 * dgd_stride + j + col1), vavg);
+ dgd1[6] = vsubq_s16(vld1q_s16(dgd + 6 * dgd_stride + j + col1), vavg);
+
+ // Compute all elements from the combination of both columns (49
+ // elements).
+ compute_H_two_cols(dgd0, dgd1, col0, col1, H, wiener_win,
+ wiener_win2);
+ }
+ }
+ j += 8;
+ }
+
+ if (j < width) {
+ // Process remaining columns using a mask to discard excess elements.
+ for (int col0 = 0; col0 < wiener_win; col0++) {
+ // Load first column.
+ int16x8_t dgd0[7];
+ dgd0[0] = vsubq_s16(vld1q_s16(dgd + 0 * dgd_stride + j + col0), vavg);
+ dgd0[0] = vandq_s16(dgd0[0], mask);
+ dgd0[1] = vsubq_s16(vld1q_s16(dgd + 1 * dgd_stride + j + col0), vavg);
+ dgd0[1] = vandq_s16(dgd0[1], mask);
+ dgd0[2] = vsubq_s16(vld1q_s16(dgd + 2 * dgd_stride + j + col0), vavg);
+ dgd0[2] = vandq_s16(dgd0[2], mask);
+ dgd0[3] = vsubq_s16(vld1q_s16(dgd + 3 * dgd_stride + j + col0), vavg);
+ dgd0[3] = vandq_s16(dgd0[3], mask);
+ dgd0[4] = vsubq_s16(vld1q_s16(dgd + 4 * dgd_stride + j + col0), vavg);
+ dgd0[4] = vandq_s16(dgd0[4], mask);
+ dgd0[5] = vsubq_s16(vld1q_s16(dgd + 5 * dgd_stride + j + col0), vavg);
+ dgd0[5] = vandq_s16(dgd0[5], mask);
+ dgd0[6] = vsubq_s16(vld1q_s16(dgd + 6 * dgd_stride + j + col0), vavg);
+ dgd0[6] = vandq_s16(dgd0[6], mask);
+
+ // Perform computation of the first column with itself (28 elements).
+ // For the first column this will fill the upper triangle of the 7x7
+ // matrix at the top left of the H matrix. For the next columns this
+ // will fill the upper triangle of the other 7x7 matrices around H's
+ // diagonal.
+ compute_H_one_col(dgd0, col0, H, wiener_win, wiener_win2);
+
+ // All computation next to the matrix diagonal has already been done.
+ for (int col1 = col0 + 1; col1 < wiener_win; col1++) {
+ // Load second column.
+ int16x8_t dgd1[7];
+ dgd1[0] = vsubq_s16(vld1q_s16(dgd + 0 * dgd_stride + j + col1), vavg);
+ dgd1[1] = vsubq_s16(vld1q_s16(dgd + 1 * dgd_stride + j + col1), vavg);
+ dgd1[2] = vsubq_s16(vld1q_s16(dgd + 2 * dgd_stride + j + col1), vavg);
+ dgd1[3] = vsubq_s16(vld1q_s16(dgd + 3 * dgd_stride + j + col1), vavg);
+ dgd1[4] = vsubq_s16(vld1q_s16(dgd + 4 * dgd_stride + j + col1), vavg);
+ dgd1[5] = vsubq_s16(vld1q_s16(dgd + 5 * dgd_stride + j + col1), vavg);
+ dgd1[6] = vsubq_s16(vld1q_s16(dgd + 6 * dgd_stride + j + col1), vavg);
+
+ // Compute all elements from the combination of both columns (49
+ // elements).
+ compute_H_two_cols(dgd0, dgd1, col0, col1, H, wiener_win,
+ wiener_win2);
+ }
+ }
+ }
+ dgd += dgd_stride;
+ src += src_stride;
+ } while (--height != 0);
+
+ // Transpose M_trn.
+ transpose_M_win7(M, M_trn, 7);
+
+ // Copy upper triangle of H in the lower one.
+ copy_upper_triangle(H, wiener_win2);
+
+ // Scaling the results.
+ uint8_t bit_depth_divider = 1;
+ if (bit_depth == AOM_BITS_12) {
+ bit_depth_divider = 16;
+ } else if (bit_depth == AOM_BITS_10) {
+ bit_depth_divider = 4;
+ }
+
+ for (int i = 0; i < wiener_win2; ++i) {
+ M[i] /= bit_depth_divider;
+ for (int j = 0; j < wiener_win2; ++j) {
+ H[i * wiener_win2 + j] /= bit_depth_divider;
+ }
+ }
+}
+
+// This function computes two matrices: the cross-correlation between the src
+// buffer and dgd buffer (M), and the auto-covariance of the dgd buffer (H).
+//
+// M is of size 5 * 5. It needs to be filled such that multiplying one element
+// from src with each element of a row of the wiener window will fill one
+// column of M. However this is not very convenient in terms of memory
+// accesses, as it means we do contiguous loads of dgd but strided stores to M.
+// As a result, we use an intermediate matrix M_trn which is instead filled
+// such that one row of the wiener window gives one row of M_trn. Once fully
+// computed, M_trn is then transposed to return M.
+//
+// H is of size 25 * 25. It is filled by multiplying every pair of elements of
+// the wiener window together. Since it is a symmetric matrix, we only compute
+// the upper triangle, and then copy it down to the lower one. Here we fill it
+// by taking each different pair of columns, and multiplying all the elements of
+// the first one with all the elements of the second one, with a special case
+// when multiplying a column by itself.
+static INLINE void highbd_compute_stats_win5_neon(
+ const int16_t *dgd, int dgd_stride, const int16_t *src, int src_stride,
+ int width, int height, int64_t *M, int64_t *H, int16_t avg, int bit_depth) {
+ const int wiener_win = 5;
+ const int wiener_win2 = wiener_win * wiener_win;
+ const int16x8_t mask = vld1q_s16(&av1_neon_mask_16bit[8] - (width % 8));
+
+ // We use an intermediate matrix that will be transposed to get M.
+ int64_t M_trn[25];
+ memset(M_trn, 0, sizeof(M_trn));
+
+ int16x8_t vavg = vdupq_n_s16(avg);
+ do {
+ // Cross-correlation (M).
+ for (int row = 0; row < wiener_win; row++) {
+ int16x8_t dgd0 = vsubq_s16(vld1q_s16(dgd + row * dgd_stride), vavg);
+ int j = 0;
+ while (j <= width - 8) {
+ int16x8_t dgd1 =
+ vsubq_s16(vld1q_s16(dgd + row * dgd_stride + j + 8), vavg);
+ int16x8_t s = vsubq_s16(vld1q_s16(src + j), vavg);
+
+ // Compute all the elements of one row of M.
+ compute_M_one_row_win5(s, dgd0, dgd1, M_trn, wiener_win, row);
+
+ dgd0 = dgd1;
+ j += 8;
+ }
+ // Process remaining elements without Neon.
+ while (j < width) {
+ int16_t s = src[j] - avg;
+ int16_t d0 = dgd[row * dgd_stride + 0 + j] - avg;
+ int16_t d1 = dgd[row * dgd_stride + 1 + j] - avg;
+ int16_t d2 = dgd[row * dgd_stride + 2 + j] - avg;
+ int16_t d3 = dgd[row * dgd_stride + 3 + j] - avg;
+ int16_t d4 = dgd[row * dgd_stride + 4 + j] - avg;
+
+ M_trn[row * wiener_win + 0] += d0 * s;
+ M_trn[row * wiener_win + 1] += d1 * s;
+ M_trn[row * wiener_win + 2] += d2 * s;
+ M_trn[row * wiener_win + 3] += d3 * s;
+ M_trn[row * wiener_win + 4] += d4 * s;
+
+ j++;
+ }
+ }
+
+ // Auto-covariance (H).
+ int j = 0;
+ while (j <= width - 8) {
+ for (int col0 = 0; col0 < wiener_win; col0++) {
+ // Load first column.
+ int16x8_t dgd0[5];
+ dgd0[0] = vsubq_s16(vld1q_s16(dgd + 0 * dgd_stride + j + col0), vavg);
+ dgd0[1] = vsubq_s16(vld1q_s16(dgd + 1 * dgd_stride + j + col0), vavg);
+ dgd0[2] = vsubq_s16(vld1q_s16(dgd + 2 * dgd_stride + j + col0), vavg);
+ dgd0[3] = vsubq_s16(vld1q_s16(dgd + 3 * dgd_stride + j + col0), vavg);
+ dgd0[4] = vsubq_s16(vld1q_s16(dgd + 4 * dgd_stride + j + col0), vavg);
+
+ // Perform computation of the first column with itself (15 elements).
+ // For the first column this will fill the upper triangle of the 5x5
+ // matrix at the top left of the H matrix. For the next columns this
+ // will fill the upper triangle of the other 5x5 matrices around H's
+ // diagonal.
+ compute_H_one_col(dgd0, col0, H, wiener_win, wiener_win2);
+
+ // All computation next to the matrix diagonal has already been done.
+ for (int col1 = col0 + 1; col1 < wiener_win; col1++) {
+ // Load second column.
+ int16x8_t dgd1[5];
+ dgd1[0] = vsubq_s16(vld1q_s16(dgd + 0 * dgd_stride + j + col1), vavg);
+ dgd1[1] = vsubq_s16(vld1q_s16(dgd + 1 * dgd_stride + j + col1), vavg);
+ dgd1[2] = vsubq_s16(vld1q_s16(dgd + 2 * dgd_stride + j + col1), vavg);
+ dgd1[3] = vsubq_s16(vld1q_s16(dgd + 3 * dgd_stride + j + col1), vavg);
+ dgd1[4] = vsubq_s16(vld1q_s16(dgd + 4 * dgd_stride + j + col1), vavg);
+
+ // Compute all elements from the combination of both columns (25
+ // elements).
+ compute_H_two_cols(dgd0, dgd1, col0, col1, H, wiener_win,
+ wiener_win2);
+ }
+ }
+ j += 8;
+ }
+
+ if (j < width) {
+ // Process remaining columns using a mask to discard excess elements.
+ for (int col0 = 0; col0 < wiener_win; col0++) {
+ // Load first column.
+ int16x8_t dgd0[5];
+ dgd0[0] = vsubq_s16(vld1q_s16(dgd + 0 * dgd_stride + j + col0), vavg);
+ dgd0[0] = vandq_s16(dgd0[0], mask);
+ dgd0[1] = vsubq_s16(vld1q_s16(dgd + 1 * dgd_stride + j + col0), vavg);
+ dgd0[1] = vandq_s16(dgd0[1], mask);
+ dgd0[2] = vsubq_s16(vld1q_s16(dgd + 2 * dgd_stride + j + col0), vavg);
+ dgd0[2] = vandq_s16(dgd0[2], mask);
+ dgd0[3] = vsubq_s16(vld1q_s16(dgd + 3 * dgd_stride + j + col0), vavg);
+ dgd0[3] = vandq_s16(dgd0[3], mask);
+ dgd0[4] = vsubq_s16(vld1q_s16(dgd + 4 * dgd_stride + j + col0), vavg);
+ dgd0[4] = vandq_s16(dgd0[4], mask);
+
+ // Perform computation of the first column with itself (15 elements).
+ // For the first column this will fill the upper triangle of the 5x5
+ // matrix at the top left of the H matrix. For the next columns this
+ // will fill the upper triangle of the other 5x5 matrices around H's
+ // diagonal.
+ compute_H_one_col(dgd0, col0, H, wiener_win, wiener_win2);
+
+ // All computation next to the matrix diagonal has already been done.
+ for (int col1 = col0 + 1; col1 < wiener_win; col1++) {
+ // Load second column.
+ int16x8_t dgd1[5];
+ dgd1[0] = vsubq_s16(vld1q_s16(dgd + 0 * dgd_stride + j + col1), vavg);
+ dgd1[1] = vsubq_s16(vld1q_s16(dgd + 1 * dgd_stride + j + col1), vavg);
+ dgd1[2] = vsubq_s16(vld1q_s16(dgd + 2 * dgd_stride + j + col1), vavg);
+ dgd1[3] = vsubq_s16(vld1q_s16(dgd + 3 * dgd_stride + j + col1), vavg);
+ dgd1[4] = vsubq_s16(vld1q_s16(dgd + 4 * dgd_stride + j + col1), vavg);
+
+ // Compute all elements from the combination of both columns (25
+ // elements).
+ compute_H_two_cols(dgd0, dgd1, col0, col1, H, wiener_win,
+ wiener_win2);
+ }
+ }
+ }
+ dgd += dgd_stride;
+ src += src_stride;
+ } while (--height != 0);
+
+ // Transpose M_trn.
+ transpose_M_win5(M, M_trn, 5);
+
+ // Copy upper triangle of H in the lower one.
+ copy_upper_triangle(H, wiener_win2);
+
+ // Scaling the results.
+ uint8_t bit_depth_divider = 1;
+ if (bit_depth == AOM_BITS_12) {
+ bit_depth_divider = 16;
+ } else if (bit_depth == AOM_BITS_10) {
+ bit_depth_divider = 4;
+ }
+
+ for (int i = 0; i < wiener_win2; ++i) {
+ M[i] /= bit_depth_divider;
+ for (int j = 0; j < wiener_win2; ++j) {
+ H[i * wiener_win2 + j] /= bit_depth_divider;
+ }
+ }
+}
+
+void av1_compute_stats_highbd_neon(int wiener_win, const uint8_t *dgd8,
+ const uint8_t *src8, int h_start, int h_end,
+ int v_start, int v_end, int dgd_stride,
+ int src_stride, int64_t *M, int64_t *H,
+ aom_bit_depth_t bit_depth) {
+ assert(wiener_win == WIENER_WIN || wiener_win == WIENER_WIN_REDUCED);
+
+ const int wiener_halfwin = wiener_win >> 1;
+ const int wiener_win2 = wiener_win * wiener_win;
+ memset(H, 0, sizeof(*H) * wiener_win2 * wiener_win2);
+
+ const int16_t *src = (const int16_t *)CONVERT_TO_SHORTPTR(src8);
+ const int16_t *dgd = (const int16_t *)CONVERT_TO_SHORTPTR(dgd8);
+ const int height = v_end - v_start;
+ const int width = h_end - h_start;
+ const int vert_offset = v_start - wiener_halfwin;
+ const int horiz_offset = h_start - wiener_halfwin;
+
+ int16_t avg = highbd_find_average_neon(dgd + v_start * dgd_stride + h_start,
+ dgd_stride, width, height);
+
+ src += v_start * src_stride + h_start;
+ dgd += vert_offset * dgd_stride + horiz_offset;
+
+ if (wiener_win == WIENER_WIN) {
+ highbd_compute_stats_win7_neon(dgd, dgd_stride, src, src_stride, width,
+ height, M, H, avg, bit_depth);
+ } else {
+ highbd_compute_stats_win5_neon(dgd, dgd_stride, src, src_stride, width,
+ height, M, H, avg, bit_depth);
+ }
+}
diff --git a/av1/encoder/arm/neon/highbd_rdopt_neon.c b/av1/encoder/arm/neon/highbd_rdopt_neon.c
new file mode 100644
index 0000000..4bf7ae6
--- /dev/null
+++ b/av1/encoder/arm/neon/highbd_rdopt_neon.c
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <assert.h>
+#include <arm_neon.h>
+
+#include "config/av1_rtcd.h"
+#include "aom_dsp/arm/sum_neon.h"
+
+int64_t av1_highbd_block_error_neon(const tran_low_t *coeff,
+ const tran_low_t *dqcoeff,
+ intptr_t block_size, int64_t *ssz, int bd) {
+ uint64x2_t err_u64 = vdupq_n_u64(0);
+ int64x2_t ssz_s64 = vdupq_n_s64(0);
+
+ const int shift = 2 * (bd - 8);
+ const int rounding = shift > 0 ? 1 << (shift - 1) : 0;
+
+ assert(block_size >= 16);
+ assert((block_size % 16) == 0);
+
+ do {
+ const int32x4_t c = vld1q_s32(coeff);
+ const int32x4_t d = vld1q_s32(dqcoeff);
+
+ const uint32x4_t diff = vreinterpretq_u32_s32(vabdq_s32(c, d));
+
+ err_u64 = vmlal_u32(err_u64, vget_low_u32(diff), vget_low_u32(diff));
+ err_u64 = vmlal_u32(err_u64, vget_high_u32(diff), vget_high_u32(diff));
+
+ ssz_s64 = vmlal_s32(ssz_s64, vget_low_s32(c), vget_low_s32(c));
+ ssz_s64 = vmlal_s32(ssz_s64, vget_high_s32(c), vget_high_s32(c));
+
+ coeff += 4;
+ dqcoeff += 4;
+ block_size -= 4;
+ } while (block_size != 0);
+
+ *ssz = (horizontal_add_s64x2(ssz_s64) + rounding) >> shift;
+ return ((int64_t)horizontal_add_u64x2(err_u64) + rounding) >> shift;
+}
diff --git a/av1/encoder/arm/neon/highbd_temporal_filter_neon.c b/av1/encoder/arm/neon/highbd_temporal_filter_neon.c
new file mode 100644
index 0000000..88e176f
--- /dev/null
+++ b/av1/encoder/arm/neon/highbd_temporal_filter_neon.c
@@ -0,0 +1,562 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+
+#include "config/aom_config.h"
+#include "config/av1_rtcd.h"
+#include "av1/encoder/encoder.h"
+#include "av1/encoder/temporal_filter.h"
+#include "aom_dsp/mathutils.h"
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_dsp/arm/sum_neon.h"
+
+static INLINE void get_squared_error(
+ const uint16_t *frame1, const uint32_t stride1, const uint16_t *frame2,
+ const uint32_t stride2, const uint32_t block_width,
+ const uint32_t block_height, uint32_t *frame_sse,
+ const unsigned int dst_stride) {
+ uint32_t *dst = frame_sse;
+
+ uint32_t i = 0;
+ do {
+ uint32_t j = 0;
+ do {
+ uint16x8_t s = vld1q_u16(frame1 + i * stride1 + j);
+ uint16x8_t r = vld1q_u16(frame2 + i * stride2 + j);
+
+ uint16x8_t abs_diff = vabdq_u16(s, r);
+ uint32x4_t sse_lo =
+ vmull_u16(vget_low_u16(abs_diff), vget_low_u16(abs_diff));
+ uint32x4_t sse_hi =
+ vmull_u16(vget_high_u16(abs_diff), vget_high_u16(abs_diff));
+
+ vst1q_u32(dst + j, sse_lo);
+ vst1q_u32(dst + j + 4, sse_hi);
+
+ j += 8;
+ } while (j < block_width);
+
+ dst += dst_stride;
+ i++;
+ } while (i < block_height);
+}
+
+static uint32_t sum_kernel5x5_mask_single(const uint32x4_t vsrc[5][2],
+ const uint32x4_t mask_single) {
+ uint32x4_t vsums = vmulq_u32(vsrc[0][0], mask_single);
+ vsums = vmlaq_u32(vsums, vsrc[1][0], mask_single);
+ vsums = vmlaq_u32(vsums, vsrc[2][0], mask_single);
+ vsums = vmlaq_u32(vsums, vsrc[3][0], mask_single);
+ vsums = vmlaq_u32(vsums, vsrc[4][0], mask_single);
+ return horizontal_add_u32x4(vsums);
+}
+
+static uint32x4_t sum_kernel5x5_mask_double(const uint32x4_t vsrc[5][2],
+ const uint32x4_t mask1,
+ const uint32x4_t mask2) {
+ uint32x4_t vsums = vmulq_u32(vsrc[0][0], mask1);
+ vsums = vmlaq_u32(vsums, vsrc[1][0], mask1);
+ vsums = vmlaq_u32(vsums, vsrc[2][0], mask1);
+ vsums = vmlaq_u32(vsums, vsrc[3][0], mask1);
+ vsums = vmlaq_u32(vsums, vsrc[4][0], mask1);
+ vsums = vmlaq_u32(vsums, vsrc[0][1], mask2);
+ vsums = vmlaq_u32(vsums, vsrc[1][1], mask2);
+ vsums = vmlaq_u32(vsums, vsrc[2][1], mask2);
+ vsums = vmlaq_u32(vsums, vsrc[3][1], mask2);
+ vsums = vmlaq_u32(vsums, vsrc[4][1], mask2);
+ return vsums;
+}
+
+static void highbd_apply_temporal_filter(
+ const uint16_t *frame, const unsigned int stride,
+ const uint32_t block_width, const uint32_t block_height,
+ const int *subblock_mses, unsigned int *accumulator, uint16_t *count,
+ const uint32_t *frame_sse, const uint32_t frame_sse_stride,
+ const uint32_t *luma_sse_sum, const double inv_num_ref_pixels,
+ const double decay_factor, const double inv_factor,
+ const double weight_factor, const double *d_factor, int tf_wgt_calc_lvl,
+ int bd) {
+ assert(((block_width == 16) || (block_width == 32)) &&
+ ((block_height == 16) || (block_height == 32)));
+
+ uint32_t acc_5x5_neon[BH][BW] = { 0 };
+ const int half_window = TF_WINDOW_LENGTH >> 1;
+
+ uint32x4_t vsrc[5][2] = { 0 };
+ const uint32x4_t k0000 = vdupq_n_u32(0);
+ const uint32x4_t k1111 = vdupq_n_u32(1);
+ const uint32_t k3110_u32[4] = { 0, 1, 1, 3 };
+ const uint32_t k2111_u32[4] = { 1, 1, 1, 2 };
+ const uint32_t k1112_u32[4] = { 2, 1, 1, 1 };
+ const uint32_t k0113_u32[4] = { 3, 1, 1, 0 };
+ const uint32x4_t k3110 = vld1q_u32(k3110_u32);
+ const uint32x4_t k2111 = vld1q_u32(k2111_u32);
+ const uint32x4_t k1112 = vld1q_u32(k1112_u32);
+ const uint32x4_t k0113 = vld1q_u32(k0113_u32);
+
+ uint32x4_t vmask1[4], vmask2[4];
+ vmask1[0] = k1111;
+ vmask2[0] = vextq_u32(k1111, k0000, 3);
+ vmask1[1] = vextq_u32(k0000, k1111, 3);
+ vmask2[1] = vextq_u32(k1111, k0000, 2);
+ vmask1[2] = vextq_u32(k0000, k1111, 2);
+ vmask2[2] = vextq_u32(k1111, k0000, 1);
+ vmask1[3] = vextq_u32(k0000, k1111, 1);
+ vmask2[3] = k1111;
+
+ uint32_t row = 0;
+ do {
+ uint32_t col = 0;
+ const uint32_t *src = frame_sse + row * frame_sse_stride;
+ if (row == 0) {
+ vsrc[2][0] = vld1q_u32(src);
+ vsrc[3][0] = vld1q_u32(src + frame_sse_stride);
+ vsrc[4][0] = vld1q_u32(src + 2 * frame_sse_stride);
+
+ // First 2 rows of the 5x5 matrix are padded from the 1st.
+ vsrc[0][0] = vsrc[2][0];
+ vsrc[1][0] = vsrc[2][0];
+ } else if (row == 1) {
+ vsrc[1][0] = vld1q_u32(src - frame_sse_stride);
+ vsrc[2][0] = vld1q_u32(src);
+ vsrc[3][0] = vld1q_u32(src + frame_sse_stride);
+ vsrc[4][0] = vld1q_u32(src + 2 * frame_sse_stride);
+
+ // First row of the 5x5 matrix are padded from the 1st.
+ vsrc[0][0] = vsrc[1][0];
+ } else if (row == block_height - 2) {
+ vsrc[0][0] = vld1q_u32(src - 2 * frame_sse_stride);
+ vsrc[1][0] = vld1q_u32(src - frame_sse_stride);
+ vsrc[2][0] = vld1q_u32(src);
+ vsrc[3][0] = vld1q_u32(src + frame_sse_stride);
+
+ // Last row of the 5x5 matrix are padded from the one before.
+ vsrc[4][0] = vsrc[3][0];
+ } else if (row == block_height - 1) {
+ vsrc[0][0] = vld1q_u32(src - 2 * frame_sse_stride);
+ vsrc[1][0] = vld1q_u32(src - frame_sse_stride);
+ vsrc[2][0] = vld1q_u32(src);
+
+ // Last 2 rows of the 5x5 matrix are padded from the 3rd.
+ vsrc[3][0] = vsrc[2][0];
+ vsrc[4][0] = vsrc[2][0];
+ } else {
+ vsrc[0][0] = vld1q_u32(src - 2 * frame_sse_stride);
+ vsrc[1][0] = vld1q_u32(src - frame_sse_stride);
+ vsrc[2][0] = vld1q_u32(src);
+ vsrc[3][0] = vld1q_u32(src + frame_sse_stride);
+ vsrc[4][0] = vld1q_u32(src + 2 * frame_sse_stride);
+ }
+
+ acc_5x5_neon[row][0] = sum_kernel5x5_mask_single(vsrc, k0113);
+ acc_5x5_neon[row][1] = sum_kernel5x5_mask_single(vsrc, k1112);
+
+ col += 4;
+ src += 4;
+ // Traverse 4 columns at a time
+ do {
+ if (row == 0) {
+ vsrc[2][1] = vld1q_u32(src);
+ vsrc[3][1] = vld1q_u32(src + frame_sse_stride);
+ vsrc[4][1] = vld1q_u32(src + 2 * frame_sse_stride);
+
+ // First 2 rows of the 5x5 matrix are padded from the 1st.
+ vsrc[0][1] = vsrc[2][1];
+ vsrc[1][1] = vsrc[2][1];
+ } else if (row == 1) {
+ vsrc[1][1] = vld1q_u32(src - frame_sse_stride);
+ vsrc[2][1] = vld1q_u32(src);
+ vsrc[3][1] = vld1q_u32(src + frame_sse_stride);
+ vsrc[4][1] = vld1q_u32(src + 2 * frame_sse_stride);
+
+ // First row of the 5x5 matrix are padded from the 1st.
+ vsrc[0][1] = vsrc[1][1];
+ } else if (row == block_height - 2) {
+ vsrc[0][1] = vld1q_u32(src - 2 * frame_sse_stride);
+ vsrc[1][1] = vld1q_u32(src - frame_sse_stride);
+ vsrc[2][1] = vld1q_u32(src);
+ vsrc[3][1] = vld1q_u32(src + frame_sse_stride);
+
+ // Last row of the 5x5 matrix are padded from the one before.
+ vsrc[4][1] = vsrc[3][1];
+ } else if (row == block_height - 1) {
+ vsrc[0][1] = vld1q_u32(src - 2 * frame_sse_stride);
+ vsrc[1][1] = vld1q_u32(src - frame_sse_stride);
+ vsrc[2][1] = vld1q_u32(src);
+
+ // Last 2 rows of the 5x5 matrix are padded from the 3rd.
+ vsrc[3][1] = vsrc[2][1];
+ vsrc[4][1] = vsrc[2][1];
+ } else {
+ vsrc[0][1] = vld1q_u32(src - 2 * frame_sse_stride);
+ vsrc[1][1] = vld1q_u32(src - frame_sse_stride);
+ vsrc[2][1] = vld1q_u32(src);
+ vsrc[3][1] = vld1q_u32(src + frame_sse_stride);
+ vsrc[4][1] = vld1q_u32(src + 2 * frame_sse_stride);
+ }
+
+ uint32x4_t sums[4];
+ sums[0] = sum_kernel5x5_mask_double(vsrc, vmask1[0], vmask2[0]);
+ sums[1] = sum_kernel5x5_mask_double(vsrc, vmask1[1], vmask2[1]);
+ sums[2] = sum_kernel5x5_mask_double(vsrc, vmask1[2], vmask2[2]);
+ sums[3] = sum_kernel5x5_mask_double(vsrc, vmask1[3], vmask2[3]);
+ vst1q_u32(&acc_5x5_neon[row][col - half_window],
+ horizontal_add_4d_u32x4(sums));
+
+ vsrc[0][0] = vsrc[0][1];
+ vsrc[1][0] = vsrc[1][1];
+ vsrc[2][0] = vsrc[2][1];
+ vsrc[3][0] = vsrc[3][1];
+ vsrc[4][0] = vsrc[4][1];
+
+ src += 4;
+ col += 4;
+ } while (col <= block_width - 4);
+
+ acc_5x5_neon[row][col - half_window] =
+ sum_kernel5x5_mask_single(vsrc, k2111);
+ acc_5x5_neon[row][col - half_window + 1] =
+ sum_kernel5x5_mask_single(vsrc, k3110);
+
+ row++;
+ } while (row < block_height);
+
+ // Perform filtering.
+ if (tf_wgt_calc_lvl == 0) {
+ for (unsigned int i = 0, k = 0; i < block_height; i++) {
+ for (unsigned int j = 0; j < block_width; j++, k++) {
+ const int pixel_value = frame[i * stride + j];
+ // Scale down the difference for high bit depth input.
+ const uint32_t diff_sse =
+ (acc_5x5_neon[i][j] + luma_sse_sum[i * BW + j]) >> ((bd - 8) * 2);
+
+ const double window_error = diff_sse * inv_num_ref_pixels;
+ const int subblock_idx =
+ (i >= block_height / 2) * 2 + (j >= block_width / 2);
+ const double block_error = (double)subblock_mses[subblock_idx];
+ const double combined_error =
+ weight_factor * window_error + block_error * inv_factor;
+ // Compute filter weight.
+ double scaled_error =
+ combined_error * d_factor[subblock_idx] * decay_factor;
+ scaled_error = AOMMIN(scaled_error, 7);
+ const int weight = (int)(exp(-scaled_error) * TF_WEIGHT_SCALE);
+ accumulator[k] += weight * pixel_value;
+ count[k] += weight;
+ }
+ }
+ } else {
+ for (unsigned int i = 0, k = 0; i < block_height; i++) {
+ for (unsigned int j = 0; j < block_width; j++, k++) {
+ const int pixel_value = frame[i * stride + j];
+ // Scale down the difference for high bit depth input.
+ const uint32_t diff_sse =
+ (acc_5x5_neon[i][j] + luma_sse_sum[i * BW + j]) >> ((bd - 8) * 2);
+
+ const double window_error = diff_sse * inv_num_ref_pixels;
+ const int subblock_idx =
+ (i >= block_height / 2) * 2 + (j >= block_width / 2);
+ const double block_error = (double)subblock_mses[subblock_idx];
+ const double combined_error =
+ weight_factor * window_error + block_error * inv_factor;
+ // Compute filter weight.
+ double scaled_error =
+ combined_error * d_factor[subblock_idx] * decay_factor;
+ scaled_error = AOMMIN(scaled_error, 7);
+ const float fweight =
+ approx_exp((float)-scaled_error) * TF_WEIGHT_SCALE;
+ const int weight = iroundpf(fweight);
+ accumulator[k] += weight * pixel_value;
+ count[k] += weight;
+ }
+ }
+ }
+}
+
+void av1_highbd_apply_temporal_filter_neon(
+ const YV12_BUFFER_CONFIG *frame_to_filter, const MACROBLOCKD *mbd,
+ const BLOCK_SIZE block_size, const int mb_row, const int mb_col,
+ const int num_planes, const double *noise_levels, const MV *subblock_mvs,
+ const int *subblock_mses, const int q_factor, const int filter_strength,
+ int tf_wgt_calc_lvl, const uint8_t *pred8, uint32_t *accum,
+ uint16_t *count) {
+ const int is_high_bitdepth = frame_to_filter->flags & YV12_FLAG_HIGHBITDEPTH;
+ assert(TF_WINDOW_LENGTH == 5 && "Only support window length 5 with Neon!");
+ assert(num_planes >= 1 && num_planes <= MAX_MB_PLANE);
+ (void)is_high_bitdepth;
+ assert(is_high_bitdepth);
+
+ // Block information.
+ const int mb_height = block_size_high[block_size];
+ const int mb_width = block_size_wide[block_size];
+ // Frame information.
+ const int frame_height = frame_to_filter->y_crop_height;
+ const int frame_width = frame_to_filter->y_crop_width;
+ const int min_frame_size = AOMMIN(frame_height, frame_width);
+ // Variables to simplify combined error calculation.
+ const double inv_factor = 1.0 / ((TF_WINDOW_BLOCK_BALANCE_WEIGHT + 1) *
+ TF_SEARCH_ERROR_NORM_WEIGHT);
+ const double weight_factor =
+ (double)TF_WINDOW_BLOCK_BALANCE_WEIGHT * inv_factor;
+ // Adjust filtering based on q.
+ // Larger q -> stronger filtering -> larger weight.
+ // Smaller q -> weaker filtering -> smaller weight.
+ double q_decay = pow((double)q_factor / TF_Q_DECAY_THRESHOLD, 2);
+ q_decay = CLIP(q_decay, 1e-5, 1);
+ if (q_factor >= TF_QINDEX_CUTOFF) {
+ // Max q_factor is 255, therefore the upper bound of q_decay is 8.
+ // We do not need a clip here.
+ q_decay = 0.5 * pow((double)q_factor / 64, 2);
+ }
+ // Smaller strength -> smaller filtering weight.
+ double s_decay = pow((double)filter_strength / TF_STRENGTH_THRESHOLD, 2);
+ s_decay = CLIP(s_decay, 1e-5, 1);
+ double d_factor[4] = { 0 };
+ uint32_t frame_sse[BW * BH] = { 0 };
+ uint32_t luma_sse_sum[BW * BH] = { 0 };
+ uint16_t *pred = CONVERT_TO_SHORTPTR(pred8);
+
+ for (int subblock_idx = 0; subblock_idx < 4; subblock_idx++) {
+ // Larger motion vector -> smaller filtering weight.
+ const MV mv = subblock_mvs[subblock_idx];
+ const double distance = sqrt(pow(mv.row, 2) + pow(mv.col, 2));
+ double distance_threshold = min_frame_size * TF_SEARCH_DISTANCE_THRESHOLD;
+ distance_threshold = AOMMAX(distance_threshold, 1);
+ d_factor[subblock_idx] = distance / distance_threshold;
+ d_factor[subblock_idx] = AOMMAX(d_factor[subblock_idx], 1);
+ }
+
+ // Handle planes in sequence.
+ int plane_offset = 0;
+ for (int plane = 0; plane < num_planes; ++plane) {
+ const uint32_t plane_h = mb_height >> mbd->plane[plane].subsampling_y;
+ const uint32_t plane_w = mb_width >> mbd->plane[plane].subsampling_x;
+ const uint32_t frame_stride =
+ frame_to_filter->strides[plane == AOM_PLANE_Y ? 0 : 1];
+ const uint32_t frame_sse_stride = plane_w;
+ const int frame_offset = mb_row * plane_h * frame_stride + mb_col * plane_w;
+
+ const uint16_t *ref =
+ CONVERT_TO_SHORTPTR(frame_to_filter->buffers[plane]) + frame_offset;
+ const int ss_x_shift =
+ mbd->plane[plane].subsampling_x - mbd->plane[AOM_PLANE_Y].subsampling_x;
+ const int ss_y_shift =
+ mbd->plane[plane].subsampling_y - mbd->plane[AOM_PLANE_Y].subsampling_y;
+ const int num_ref_pixels = TF_WINDOW_LENGTH * TF_WINDOW_LENGTH +
+ ((plane) ? (1 << (ss_x_shift + ss_y_shift)) : 0);
+ const double inv_num_ref_pixels = 1.0 / num_ref_pixels;
+ // Larger noise -> larger filtering weight.
+ const double n_decay = 0.5 + log(2 * noise_levels[plane] + 5.0);
+ // Decay factors for non-local mean approach.
+ const double decay_factor = 1 / (n_decay * q_decay * s_decay);
+
+ // Filter U-plane and V-plane using Y-plane. This is because motion
+ // search is only done on Y-plane, so the information from Y-plane
+ // will be more accurate. The luma sse sum is reused in both chroma
+ // planes.
+ if (plane == AOM_PLANE_U) {
+ for (unsigned int i = 0; i < plane_h; i++) {
+ for (unsigned int j = 0; j < plane_w; j++) {
+ for (int ii = 0; ii < (1 << ss_y_shift); ++ii) {
+ for (int jj = 0; jj < (1 << ss_x_shift); ++jj) {
+ const int yy = (i << ss_y_shift) + ii; // Y-coord on Y-plane.
+ const int xx = (j << ss_x_shift) + jj; // X-coord on Y-plane.
+ const int ww = frame_sse_stride
+ << ss_x_shift; // Width of Y-plane.
+ luma_sse_sum[i * BW + j] += frame_sse[yy * ww + xx];
+ }
+ }
+ }
+ }
+ }
+ get_squared_error(ref, frame_stride, pred + plane_offset, plane_w, plane_w,
+ plane_h, frame_sse, frame_sse_stride);
+
+ highbd_apply_temporal_filter(
+ pred + plane_offset, plane_w, plane_w, plane_h, subblock_mses,
+ accum + plane_offset, count + plane_offset, frame_sse, frame_sse_stride,
+ luma_sse_sum, inv_num_ref_pixels, decay_factor, inv_factor,
+ weight_factor, d_factor, tf_wgt_calc_lvl, mbd->bd);
+
+ plane_offset += plane_h * plane_w;
+ }
+}
+
+double av1_highbd_estimate_noise_from_single_plane_neon(const uint16_t *src,
+ int height, int width,
+ int stride,
+ int bitdepth,
+ int edge_thresh) {
+ uint16x8_t thresh = vdupq_n_u16(edge_thresh);
+ uint64x2_t acc = vdupq_n_u64(0);
+ // Count is in theory positive as it counts the number of times we're under
+ // the threshold, but it will be counted negatively in order to make best use
+ // of the vclt instruction, which sets every bit of a lane to 1 when the
+ // condition is true.
+ int32x4_t count = vdupq_n_s32(0);
+ int final_count = 0;
+ uint64_t final_acc = 0;
+ const uint16_t *src_start = src + stride + 1;
+ int h = 1;
+
+ do {
+ int w = 1;
+ const uint16_t *src_ptr = src_start;
+
+ while (w <= (width - 1) - 8) {
+ uint16x8_t mat[3][3];
+ mat[0][0] = vld1q_u16(src_ptr - stride - 1);
+ mat[0][1] = vld1q_u16(src_ptr - stride);
+ mat[0][2] = vld1q_u16(src_ptr - stride + 1);
+ mat[1][0] = vld1q_u16(src_ptr - 1);
+ mat[1][1] = vld1q_u16(src_ptr);
+ mat[1][2] = vld1q_u16(src_ptr + 1);
+ mat[2][0] = vld1q_u16(src_ptr + stride - 1);
+ mat[2][1] = vld1q_u16(src_ptr + stride);
+ mat[2][2] = vld1q_u16(src_ptr + stride + 1);
+
+ // Compute Sobel gradients.
+ uint16x8_t gxa = vaddq_u16(mat[0][0], mat[2][0]);
+ uint16x8_t gxb = vaddq_u16(mat[0][2], mat[2][2]);
+ gxa = vaddq_u16(gxa, vaddq_u16(mat[1][0], mat[1][0]));
+ gxb = vaddq_u16(gxb, vaddq_u16(mat[1][2], mat[1][2]));
+
+ uint16x8_t gya = vaddq_u16(mat[0][0], mat[0][2]);
+ uint16x8_t gyb = vaddq_u16(mat[2][0], mat[2][2]);
+ gya = vaddq_u16(gya, vaddq_u16(mat[0][1], mat[0][1]));
+ gyb = vaddq_u16(gyb, vaddq_u16(mat[2][1], mat[2][1]));
+
+ uint16x8_t ga = vabaq_u16(vabdq_u16(gxa, gxb), gya, gyb);
+ ga = vrshlq_u16(ga, vdupq_n_s16(8 - bitdepth));
+
+ // Check which vector elements are under the threshold. The Laplacian is
+ // then unconditionnally computed and we accumulate zeros if we're not
+ // under the threshold. This is much faster than using an if statement.
+ uint16x8_t thresh_u16 = vcltq_u16(ga, thresh);
+
+ uint16x8_t center = vshlq_n_u16(mat[1][1], 2);
+
+ uint16x8_t adj0 = vaddq_u16(mat[0][1], mat[2][1]);
+ uint16x8_t adj1 = vaddq_u16(mat[1][0], mat[1][2]);
+ uint16x8_t adj = vaddq_u16(adj0, adj1);
+ adj = vaddq_u16(adj, adj);
+
+ uint16x8_t diag0 = vaddq_u16(mat[0][0], mat[0][2]);
+ uint16x8_t diag1 = vaddq_u16(mat[2][0], mat[2][2]);
+ uint16x8_t diag = vaddq_u16(diag0, diag1);
+
+ uint16x8_t v = vabdq_u16(vaddq_u16(center, diag), adj);
+ v = vandq_u16(vrshlq_u16(v, vdupq_n_s16(8 - bitdepth)), thresh_u16);
+ uint32x4_t v_u32 = vpaddlq_u16(v);
+
+ acc = vpadalq_u32(acc, v_u32);
+ // Add -1 for each lane where the gradient is under the threshold.
+ count = vpadalq_s16(count, vreinterpretq_s16_u16(thresh_u16));
+
+ w += 8;
+ src_ptr += 8;
+ }
+
+ if (w <= (width - 1) - 4) {
+ uint16x4_t mat[3][3];
+ mat[0][0] = vld1_u16(src_ptr - stride - 1);
+ mat[0][1] = vld1_u16(src_ptr - stride);
+ mat[0][2] = vld1_u16(src_ptr - stride + 1);
+ mat[1][0] = vld1_u16(src_ptr - 1);
+ mat[1][1] = vld1_u16(src_ptr);
+ mat[1][2] = vld1_u16(src_ptr + 1);
+ mat[2][0] = vld1_u16(src_ptr + stride - 1);
+ mat[2][1] = vld1_u16(src_ptr + stride);
+ mat[2][2] = vld1_u16(src_ptr + stride + 1);
+
+ // Compute Sobel gradients.
+ uint16x4_t gxa = vadd_u16(mat[0][0], mat[2][0]);
+ uint16x4_t gxb = vadd_u16(mat[0][2], mat[2][2]);
+ gxa = vadd_u16(gxa, vadd_u16(mat[1][0], mat[1][0]));
+ gxb = vadd_u16(gxb, vadd_u16(mat[1][2], mat[1][2]));
+
+ uint16x4_t gya = vadd_u16(mat[0][0], mat[0][2]);
+ uint16x4_t gyb = vadd_u16(mat[2][0], mat[2][2]);
+ gya = vadd_u16(gya, vadd_u16(mat[0][1], mat[0][1]));
+ gyb = vadd_u16(gyb, vadd_u16(mat[2][1], mat[2][1]));
+
+ uint16x4_t ga = vaba_u16(vabd_u16(gxa, gxb), gya, gyb);
+ ga = vrshl_u16(ga, vdup_n_s16(8 - bitdepth));
+
+ // Check which vector elements are under the threshold. The Laplacian is
+ // then unconditionnally computed and we accumulate zeros if we're not
+ // under the threshold. This is much faster than using an if statement.
+ uint16x4_t thresh_u16 = vclt_u16(ga, vget_low_u16(thresh));
+
+ uint16x4_t center = vshl_n_u16(mat[1][1], 2);
+
+ uint16x4_t adj0 = vadd_u16(mat[0][1], mat[2][1]);
+ uint16x4_t adj1 = vadd_u16(mat[1][0], mat[1][2]);
+ uint16x4_t adj = vadd_u16(adj0, adj1);
+ adj = vadd_u16(adj, adj);
+
+ uint16x4_t diag0 = vadd_u16(mat[0][0], mat[0][2]);
+ uint16x4_t diag1 = vadd_u16(mat[2][0], mat[2][2]);
+ uint16x4_t diag = vadd_u16(diag0, diag1);
+
+ uint16x4_t v = vabd_u16(vadd_u16(center, diag), adj);
+ v = vand_u16(v, thresh_u16);
+ uint32x4_t v_u32 = vmovl_u16(vrshl_u16(v, vdup_n_s16(8 - bitdepth)));
+
+ acc = vpadalq_u32(acc, v_u32);
+ // Add -1 for each lane where the gradient is under the threshold.
+ count = vaddw_s16(count, vreinterpret_s16_u16(thresh_u16));
+
+ w += 4;
+ src_ptr += 4;
+ }
+
+ while (w < width - 1) {
+ int mat[3][3];
+ mat[0][0] = *(src_ptr - stride - 1);
+ mat[0][1] = *(src_ptr - stride);
+ mat[0][2] = *(src_ptr - stride + 1);
+ mat[1][0] = *(src_ptr - 1);
+ mat[1][1] = *(src_ptr);
+ mat[1][2] = *(src_ptr + 1);
+ mat[2][0] = *(src_ptr + stride - 1);
+ mat[2][1] = *(src_ptr + stride);
+ mat[2][2] = *(src_ptr + stride + 1);
+
+ // Compute Sobel gradients.
+ const int gx = (mat[0][0] - mat[0][2]) + (mat[2][0] - mat[2][2]) +
+ 2 * (mat[1][0] - mat[1][2]);
+ const int gy = (mat[0][0] - mat[2][0]) + (mat[0][2] - mat[2][2]) +
+ 2 * (mat[0][1] - mat[2][1]);
+ const int ga = ROUND_POWER_OF_TWO(abs(gx) + abs(gy), bitdepth - 8);
+
+ // Accumulate Laplacian.
+ const int is_under = ga < edge_thresh;
+ const int v = 4 * mat[1][1] -
+ 2 * (mat[0][1] + mat[2][1] + mat[1][0] + mat[1][2]) +
+ (mat[0][0] + mat[0][2] + mat[2][0] + mat[2][2]);
+ final_acc += ROUND_POWER_OF_TWO(abs(v), bitdepth - 8) * is_under;
+ final_count += is_under;
+
+ src_ptr++;
+ w++;
+ }
+ src_start += stride;
+ } while (++h < height - 1);
+
+ // We counted negatively, so subtract to get the final value.
+ final_count -= horizontal_add_s32x4(count);
+ final_acc += horizontal_add_u64x2(acc);
+ return (final_count < 16)
+ ? -1.0
+ : (double)final_acc / (6 * final_count) * SQRT_PI_BY_2;
+}
diff --git a/av1/encoder/arm/neon/pickrst_neon.c b/av1/encoder/arm/neon/pickrst_neon.c
new file mode 100644
index 0000000..6227028
--- /dev/null
+++ b/av1/encoder/arm/neon/pickrst_neon.c
@@ -0,0 +1,1261 @@
+/*
+ * Copyright (c) 2020, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+
+#include "config/aom_config.h"
+#include "config/av1_rtcd.h"
+
+#include "aom_dsp/arm/sum_neon.h"
+#include "aom_dsp/arm/transpose_neon.h"
+#include "av1/common/restoration.h"
+#include "av1/encoder/arm/neon/pickrst_neon.h"
+#include "av1/encoder/pickrst.h"
+
+int64_t av1_lowbd_pixel_proj_error_neon(
+ const uint8_t *src8, int width, int height, int src_stride,
+ const uint8_t *dat8, int dat_stride, int32_t *flt0, int flt0_stride,
+ int32_t *flt1, int flt1_stride, int xq[2], const sgr_params_type *params) {
+ int i, j, k;
+ const int32_t shift = SGRPROJ_RST_BITS + SGRPROJ_PRJ_BITS;
+ const int32x4_t zero = vdupq_n_s32(0);
+ uint64x2_t sum64 = vreinterpretq_u64_s32(zero);
+ const uint8_t *src = src8;
+ const uint8_t *dat = dat8;
+
+ int64_t err = 0;
+ if (params->r[0] > 0 && params->r[1] > 0) {
+ for (i = 0; i < height; ++i) {
+ int32x4_t err0 = zero;
+ for (j = 0; j <= width - 8; j += 8) {
+ const uint8x8_t d0 = vld1_u8(&dat[j]);
+ const uint8x8_t s0 = vld1_u8(&src[j]);
+ const int16x8_t flt0_16b =
+ vcombine_s16(vqmovn_s32(vld1q_s32(&flt0[j])),
+ vqmovn_s32(vld1q_s32(&flt0[j + 4])));
+ const int16x8_t flt1_16b =
+ vcombine_s16(vqmovn_s32(vld1q_s32(&flt1[j])),
+ vqmovn_s32(vld1q_s32(&flt1[j + 4])));
+ const int16x8_t u0 =
+ vreinterpretq_s16_u16(vshll_n_u8(d0, SGRPROJ_RST_BITS));
+ const int16x8_t flt0_0_sub_u = vsubq_s16(flt0_16b, u0);
+ const int16x8_t flt1_0_sub_u = vsubq_s16(flt1_16b, u0);
+ const int16x4_t flt0_16b_sub_u_lo = vget_low_s16(flt0_0_sub_u);
+ const int16x4_t flt0_16b_sub_u_hi = vget_high_s16(flt0_0_sub_u);
+ const int16x4_t flt1_16b_sub_u_lo = vget_low_s16(flt1_0_sub_u);
+ const int16x4_t flt1_16b_sub_u_hi = vget_high_s16(flt1_0_sub_u);
+
+ int32x4_t v0 = vmull_n_s16(flt0_16b_sub_u_lo, (int16_t)xq[0]);
+ v0 = vmlal_n_s16(v0, flt1_16b_sub_u_lo, (int16_t)xq[1]);
+ int32x4_t v1 = vmull_n_s16(flt0_16b_sub_u_hi, (int16_t)xq[0]);
+ v1 = vmlal_n_s16(v1, flt1_16b_sub_u_hi, (int16_t)xq[1]);
+ const int16x4_t vr0 = vqrshrn_n_s32(v0, 11);
+ const int16x4_t vr1 = vqrshrn_n_s32(v1, 11);
+ const int16x8_t e0 = vaddq_s16(vcombine_s16(vr0, vr1),
+ vreinterpretq_s16_u16(vsubl_u8(d0, s0)));
+ const int16x4_t e0_lo = vget_low_s16(e0);
+ const int16x4_t e0_hi = vget_high_s16(e0);
+ err0 = vmlal_s16(err0, e0_lo, e0_lo);
+ err0 = vmlal_s16(err0, e0_hi, e0_hi);
+ }
+ for (k = j; k < width; ++k) {
+ const int32_t u = dat[k] << SGRPROJ_RST_BITS;
+ int32_t v = xq[0] * (flt0[k] - u) + xq[1] * (flt1[k] - u);
+ const int32_t e = ROUND_POWER_OF_TWO(v, 11) + dat[k] - src[k];
+ err += e * e;
+ }
+ dat += dat_stride;
+ src += src_stride;
+ flt0 += flt0_stride;
+ flt1 += flt1_stride;
+ sum64 = vpadalq_u32(sum64, vreinterpretq_u32_s32(err0));
+ }
+
+ } else if (params->r[0] > 0 || params->r[1] > 0) {
+ const int xq_active = (params->r[0] > 0) ? xq[0] : xq[1];
+ const int32_t *flt = (params->r[0] > 0) ? flt0 : flt1;
+ const int flt_stride = (params->r[0] > 0) ? flt0_stride : flt1_stride;
+ for (i = 0; i < height; ++i) {
+ int32x4_t err0 = zero;
+ for (j = 0; j <= width - 8; j += 8) {
+ const uint8x8_t d0 = vld1_u8(&dat[j]);
+ const uint8x8_t s0 = vld1_u8(&src[j]);
+ const uint16x8_t d0s0 = vsubl_u8(d0, s0);
+ const uint16x8x2_t d0w =
+ vzipq_u16(vmovl_u8(d0), vreinterpretq_u16_s32(zero));
+
+ const int32x4_t flt_16b_lo = vld1q_s32(&flt[j]);
+ const int32x4_t flt_16b_hi = vld1q_s32(&flt[j + 4]);
+
+ int32x4_t v0 = vmulq_n_s32(flt_16b_lo, xq_active);
+ v0 = vmlsq_n_s32(v0, vreinterpretq_s32_u16(d0w.val[0]),
+ xq_active * (1 << SGRPROJ_RST_BITS));
+ int32x4_t v1 = vmulq_n_s32(flt_16b_hi, xq_active);
+ v1 = vmlsq_n_s32(v1, vreinterpretq_s32_u16(d0w.val[1]),
+ xq_active * (1 << SGRPROJ_RST_BITS));
+ const int16x4_t vr0 = vqrshrn_n_s32(v0, 11);
+ const int16x4_t vr1 = vqrshrn_n_s32(v1, 11);
+ const int16x8_t e0 =
+ vaddq_s16(vcombine_s16(vr0, vr1), vreinterpretq_s16_u16(d0s0));
+ const int16x4_t e0_lo = vget_low_s16(e0);
+ const int16x4_t e0_hi = vget_high_s16(e0);
+ err0 = vmlal_s16(err0, e0_lo, e0_lo);
+ err0 = vmlal_s16(err0, e0_hi, e0_hi);
+ }
+ for (k = j; k < width; ++k) {
+ const int32_t u = dat[k] << SGRPROJ_RST_BITS;
+ int32_t v = xq_active * (flt[k] - u);
+ const int32_t e = ROUND_POWER_OF_TWO(v, shift) + dat[k] - src[k];
+ err += e * e;
+ }
+ dat += dat_stride;
+ src += src_stride;
+ flt += flt_stride;
+ sum64 = vpadalq_u32(sum64, vreinterpretq_u32_s32(err0));
+ }
+ } else {
+ uint32x4_t err0 = vreinterpretq_u32_s32(zero);
+ for (i = 0; i < height; ++i) {
+ for (j = 0; j <= width - 16; j += 16) {
+ const uint8x16_t d = vld1q_u8(&dat[j]);
+ const uint8x16_t s = vld1q_u8(&src[j]);
+ const uint8x16_t diff = vabdq_u8(d, s);
+ const uint8x8_t diff0 = vget_low_u8(diff);
+ const uint8x8_t diff1 = vget_high_u8(diff);
+ err0 = vpadalq_u16(err0, vmull_u8(diff0, diff0));
+ err0 = vpadalq_u16(err0, vmull_u8(diff1, diff1));
+ }
+ for (k = j; k < width; ++k) {
+ const int32_t e = dat[k] - src[k];
+ err += e * e;
+ }
+ dat += dat_stride;
+ src += src_stride;
+ }
+ sum64 = vpaddlq_u32(err0);
+ }
+#if AOM_ARCH_AARCH64
+ err += vaddvq_u64(sum64);
+#else
+ err += vget_lane_u64(vadd_u64(vget_low_u64(sum64), vget_high_u64(sum64)), 0);
+#endif // AOM_ARCH_AARCH64
+ return err;
+}
+
+static INLINE uint8_t find_average_neon(const uint8_t *src, int src_stride,
+ int width, int height) {
+ uint64_t sum = 0;
+
+ if (width >= 16) {
+ int h = 0;
+ // We can accumulate up to 257 8-bit values in a 16-bit value, given
+ // that each 16-bit vector has 8 elements, that means we can process up to
+ // int(257*8/width) rows before we need to widen to 32-bit vector
+ // elements.
+ int h_overflow = 257 * 8 / width;
+ int h_limit = height > h_overflow ? h_overflow : height;
+ uint32x4_t avg_u32 = vdupq_n_u32(0);
+ do {
+ uint16x8_t avg_u16 = vdupq_n_u16(0);
+ do {
+ int j = width;
+ const uint8_t *src_ptr = src;
+ do {
+ uint8x16_t s = vld1q_u8(src_ptr);
+ avg_u16 = vpadalq_u8(avg_u16, s);
+ j -= 16;
+ src_ptr += 16;
+ } while (j >= 16);
+ if (j >= 8) {
+ uint8x8_t s = vld1_u8(src_ptr);
+ avg_u16 = vaddw_u8(avg_u16, s);
+ j -= 8;
+ src_ptr += 8;
+ }
+ // Scalar tail case.
+ while (j > 0) {
+ sum += src[width - j];
+ j--;
+ }
+ src += src_stride;
+ } while (++h < h_limit);
+ avg_u32 = vpadalq_u16(avg_u32, avg_u16);
+
+ h_limit += h_overflow;
+ h_limit = height > h_overflow ? h_overflow : height;
+ } while (h < height);
+ return (uint8_t)((horizontal_long_add_u32x4(avg_u32) + sum) /
+ (width * height));
+ }
+ if (width >= 8) {
+ int h = 0;
+ // We can accumulate up to 257 8-bit values in a 16-bit value, given
+ // that each 16-bit vector has 4 elements, that means we can process up to
+ // int(257*4/width) rows before we need to widen to 32-bit vector
+ // elements.
+ int h_overflow = 257 * 4 / width;
+ int h_limit = height > h_overflow ? h_overflow : height;
+ uint32x2_t avg_u32 = vdup_n_u32(0);
+ do {
+ uint16x4_t avg_u16 = vdup_n_u16(0);
+ do {
+ int j = width;
+ const uint8_t *src_ptr = src;
+ uint8x8_t s = vld1_u8(src_ptr);
+ avg_u16 = vpadal_u8(avg_u16, s);
+ j -= 8;
+ src_ptr += 8;
+ // Scalar tail case.
+ while (j > 0) {
+ sum += src[width - j];
+ j--;
+ }
+ src += src_stride;
+ } while (++h < h_limit);
+ avg_u32 = vpadal_u16(avg_u32, avg_u16);
+
+ h_limit += h_overflow;
+ h_limit = height > h_overflow ? h_overflow : height;
+ } while (h < height);
+ return (uint8_t)((horizontal_long_add_u32x2(avg_u32) + sum) /
+ (width * height));
+ }
+ int i = height;
+ do {
+ int j = 0;
+ do {
+ sum += src[j];
+ } while (++j < width);
+ src += src_stride;
+ } while (--i != 0);
+ return (uint8_t)(sum / (width * height));
+}
+
+static INLINE void compute_sub_avg(const uint8_t *buf, int buf_stride, int avg,
+ int16_t *buf_avg, int buf_avg_stride,
+ int width, int height,
+ int downsample_factor) {
+ uint8x8_t avg_u8 = vdup_n_u8(avg);
+
+ if (width > 8) {
+ int i = 0;
+ do {
+ int j = width;
+ const uint8_t *buf_ptr = buf;
+ int16_t *buf_avg_ptr = buf_avg;
+ do {
+ uint8x8_t d = vld1_u8(buf_ptr);
+ vst1q_s16(buf_avg_ptr, vreinterpretq_s16_u16(vsubl_u8(d, avg_u8)));
+
+ j -= 8;
+ buf_ptr += 8;
+ buf_avg_ptr += 8;
+ } while (j >= 8);
+ while (j > 0) {
+ *buf_avg_ptr = (int16_t)buf[width - j] - (int16_t)avg;
+ buf_avg_ptr++;
+ j--;
+ }
+ buf += buf_stride;
+ buf_avg += buf_avg_stride;
+ i += downsample_factor;
+ } while (i < height);
+ } else {
+ // For width < 8, don't use Neon.
+ for (int i = 0; i < height; i = i + downsample_factor) {
+ for (int j = 0; j < width; j++) {
+ buf_avg[j] = (int16_t)buf[j] - (int16_t)avg;
+ }
+ buf += buf_stride;
+ buf_avg += buf_avg_stride;
+ }
+ }
+}
+
+static INLINE void compute_H_one_col(int16x8_t *dgd, int col, int64_t *H,
+ const int wiener_win,
+ const int wiener_win2, int32x4_t df_s32) {
+ for (int row0 = 0; row0 < wiener_win; row0++) {
+ for (int row1 = row0; row1 < wiener_win; row1++) {
+ int auto_cov_idx =
+ (col * wiener_win + row0) * wiener_win2 + (col * wiener_win) + row1;
+
+ int32x4_t auto_cov =
+ vmull_s16(vget_low_s16(dgd[row0]), vget_low_s16(dgd[row1]));
+ auto_cov = vmlal_s16(auto_cov, vget_high_s16(dgd[row0]),
+ vget_high_s16(dgd[row1]));
+ auto_cov = vshlq_s32(auto_cov, df_s32);
+
+ H[auto_cov_idx] += horizontal_long_add_s32x4(auto_cov);
+ }
+ }
+}
+
+static INLINE void compute_H_one_col_last_row(int16x8_t *dgd, int col,
+ int64_t *H, const int wiener_win,
+ const int wiener_win2,
+ int last_row_df) {
+ for (int row0 = 0; row0 < wiener_win; row0++) {
+ for (int row1 = row0; row1 < wiener_win; row1++) {
+ int auto_cov_idx =
+ (col * wiener_win + row0) * wiener_win2 + (col * wiener_win) + row1;
+
+ int32x4_t auto_cov =
+ vmull_s16(vget_low_s16(dgd[row0]), vget_low_s16(dgd[row1]));
+ auto_cov = vmlal_s16(auto_cov, vget_high_s16(dgd[row0]),
+ vget_high_s16(dgd[row1]));
+ auto_cov = vmulq_n_s32(auto_cov, last_row_df);
+
+ H[auto_cov_idx] += horizontal_long_add_s32x4(auto_cov);
+ }
+ }
+}
+
+// When we load 8 values of int16_t type and need less than 8 values for
+// processing, the below mask is used to make the extra values zero.
+const int16_t av1_neon_mask_16bit[16] = {
+ -1, -1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0, 0,
+};
+
+// This function computes two matrices: the cross-correlation between the src
+// buffer and dgd buffer (M), and the auto-covariance of the dgd buffer (H).
+//
+// M is of size 7 * 7. It needs to be filled such that multiplying one element
+// from src with each element of a row of the wiener window will fill one
+// column of M. However this is not very convenient in terms of memory
+// accesses, as it means we do contiguous loads of dgd but strided stores to M.
+// As a result, we use an intermediate matrix M_trn which is instead filled
+// such that one row of the wiener window gives one row of M_trn. Once fully
+// computed, M_trn is then transposed to return M.
+//
+// H is of size 49 * 49. It is filled by multiplying every pair of elements of
+// the wiener window together. Since it is a symmetric matrix, we only compute
+// the upper triangle, and then copy it down to the lower one. Here we fill it
+// by taking each different pair of columns, and multiplying all the elements of
+// the first one with all the elements of the second one, with a special case
+// when multiplying a column by itself.
+static INLINE void compute_stats_win7_neon(int16_t *dgd_avg, int dgd_avg_stride,
+ int16_t *src_avg, int src_avg_stride,
+ int width, int v_start, int v_end,
+ int64_t *M, int64_t *H,
+ int downsample_factor,
+ int last_row_downsample_factor) {
+ const int wiener_win = 7;
+ const int wiener_win2 = wiener_win * wiener_win;
+ // The downsample factor can be either 1 or 4, so instead of multiplying the
+ // values by 1 or 4, we can left shift by 0 or 2 respectively, which is
+ // faster. (This doesn't apply to the last row where we can scale the values
+ // by 1, 2 or 3, so we keep the multiplication).
+ const int downsample_shift = downsample_factor >> 1;
+ const int16x8_t df_s16 = vdupq_n_s16(downsample_shift);
+ const int32x4_t df_s32 = vdupq_n_s32(downsample_shift);
+ const int16x8_t mask = vld1q_s16(&av1_neon_mask_16bit[8] - (width % 8));
+
+ // We use an intermediate matrix that will be transposed to get M.
+ int64_t M_trn[49];
+ memset(M_trn, 0, sizeof(M_trn));
+
+ int h = v_start;
+ do {
+ // Cross-correlation (M).
+ for (int row = 0; row < wiener_win; row++) {
+ int16x8_t dgd0 = vld1q_s16(dgd_avg + row * dgd_avg_stride);
+ int j = 0;
+ while (j <= width - 8) {
+ int16x8_t dgd1 = vld1q_s16(dgd_avg + row * dgd_avg_stride + j + 8);
+ // Load src and scale based on downsampling factor.
+ int16x8_t s = vshlq_s16(vld1q_s16(src_avg + j), df_s16);
+
+ // Compute all the elements of one row of M.
+ compute_M_one_row_win7(s, dgd0, dgd1, M_trn, wiener_win, row);
+
+ dgd0 = dgd1;
+ j += 8;
+ }
+ // Process remaining elements without Neon.
+ while (j < width) {
+ int16_t s = src_avg[j] * downsample_factor;
+ int16_t d0 = dgd_avg[row * dgd_avg_stride + 0 + j];
+ int16_t d1 = dgd_avg[row * dgd_avg_stride + 1 + j];
+ int16_t d2 = dgd_avg[row * dgd_avg_stride + 2 + j];
+ int16_t d3 = dgd_avg[row * dgd_avg_stride + 3 + j];
+ int16_t d4 = dgd_avg[row * dgd_avg_stride + 4 + j];
+ int16_t d5 = dgd_avg[row * dgd_avg_stride + 5 + j];
+ int16_t d6 = dgd_avg[row * dgd_avg_stride + 6 + j];
+
+ M_trn[row * wiener_win + 0] += d0 * s;
+ M_trn[row * wiener_win + 1] += d1 * s;
+ M_trn[row * wiener_win + 2] += d2 * s;
+ M_trn[row * wiener_win + 3] += d3 * s;
+ M_trn[row * wiener_win + 4] += d4 * s;
+ M_trn[row * wiener_win + 5] += d5 * s;
+ M_trn[row * wiener_win + 6] += d6 * s;
+
+ j++;
+ }
+ }
+
+ // Auto-covariance (H).
+ int j = 0;
+ while (j <= width - 8) {
+ for (int col0 = 0; col0 < wiener_win; col0++) {
+ int16x8_t dgd0[7];
+ dgd0[0] = vld1q_s16(dgd_avg + 0 * dgd_avg_stride + j + col0);
+ dgd0[1] = vld1q_s16(dgd_avg + 1 * dgd_avg_stride + j + col0);
+ dgd0[2] = vld1q_s16(dgd_avg + 2 * dgd_avg_stride + j + col0);
+ dgd0[3] = vld1q_s16(dgd_avg + 3 * dgd_avg_stride + j + col0);
+ dgd0[4] = vld1q_s16(dgd_avg + 4 * dgd_avg_stride + j + col0);
+ dgd0[5] = vld1q_s16(dgd_avg + 5 * dgd_avg_stride + j + col0);
+ dgd0[6] = vld1q_s16(dgd_avg + 6 * dgd_avg_stride + j + col0);
+
+ // Perform computation of the first column with itself (28 elements).
+ // For the first column this will fill the upper triangle of the 7x7
+ // matrix at the top left of the H matrix. For the next columns this
+ // will fill the upper triangle of the other 7x7 matrices around H's
+ // diagonal.
+ compute_H_one_col(dgd0, col0, H, wiener_win, wiener_win2, df_s32);
+
+ // All computation next to the matrix diagonal has already been done.
+ for (int col1 = col0 + 1; col1 < wiener_win; col1++) {
+ // Load second column and scale based on downsampling factor.
+ int16x8_t dgd1[7];
+ dgd1[0] = vld1q_s16(dgd_avg + 0 * dgd_avg_stride + j + col1);
+ dgd1[0] = vshlq_s16(dgd1[0], df_s16);
+ dgd1[1] = vld1q_s16(dgd_avg + 1 * dgd_avg_stride + j + col1);
+ dgd1[1] = vshlq_s16(dgd1[1], df_s16);
+ dgd1[2] = vld1q_s16(dgd_avg + 2 * dgd_avg_stride + j + col1);
+ dgd1[2] = vshlq_s16(dgd1[2], df_s16);
+ dgd1[3] = vld1q_s16(dgd_avg + 3 * dgd_avg_stride + j + col1);
+ dgd1[3] = vshlq_s16(dgd1[3], df_s16);
+ dgd1[4] = vld1q_s16(dgd_avg + 4 * dgd_avg_stride + j + col1);
+ dgd1[4] = vshlq_s16(dgd1[4], df_s16);
+ dgd1[5] = vld1q_s16(dgd_avg + 5 * dgd_avg_stride + j + col1);
+ dgd1[5] = vshlq_s16(dgd1[5], df_s16);
+ dgd1[6] = vld1q_s16(dgd_avg + 6 * dgd_avg_stride + j + col1);
+ dgd1[6] = vshlq_s16(dgd1[6], df_s16);
+
+ // Compute all elements from the combination of both columns (49
+ // elements).
+ compute_H_two_cols(dgd0, dgd1, col0, col1, H, wiener_win,
+ wiener_win2);
+ }
+ }
+ j += 8;
+ }
+
+ if (j < width) {
+ // Process remaining columns using a mask to discard excess elements.
+ for (int col0 = 0; col0 < wiener_win; col0++) {
+ // Load first column.
+ int16x8_t dgd0[7];
+ dgd0[0] = vld1q_s16(dgd_avg + 0 * dgd_avg_stride + j + col0);
+ dgd0[0] = vandq_s16(dgd0[0], mask);
+ dgd0[1] = vld1q_s16(dgd_avg + 1 * dgd_avg_stride + j + col0);
+ dgd0[1] = vandq_s16(dgd0[1], mask);
+ dgd0[2] = vld1q_s16(dgd_avg + 2 * dgd_avg_stride + j + col0);
+ dgd0[2] = vandq_s16(dgd0[2], mask);
+ dgd0[3] = vld1q_s16(dgd_avg + 3 * dgd_avg_stride + j + col0);
+ dgd0[3] = vandq_s16(dgd0[3], mask);
+ dgd0[4] = vld1q_s16(dgd_avg + 4 * dgd_avg_stride + j + col0);
+ dgd0[4] = vandq_s16(dgd0[4], mask);
+ dgd0[5] = vld1q_s16(dgd_avg + 5 * dgd_avg_stride + j + col0);
+ dgd0[5] = vandq_s16(dgd0[5], mask);
+ dgd0[6] = vld1q_s16(dgd_avg + 6 * dgd_avg_stride + j + col0);
+ dgd0[6] = vandq_s16(dgd0[6], mask);
+
+ // Perform computation of the first column with itself (28 elements).
+ // For the first column this will fill the upper triangle of the 7x7
+ // matrix at the top left of the H matrix. For the next columns this
+ // will fill the upper triangle of the other 7x7 matrices around H's
+ // diagonal.
+ compute_H_one_col(dgd0, col0, H, wiener_win, wiener_win2, df_s32);
+
+ // All computation next to the matrix diagonal has already been done.
+ for (int col1 = col0 + 1; col1 < wiener_win; col1++) {
+ // Load second column and scale based on downsampling factor.
+ int16x8_t dgd1[7];
+ dgd1[0] = vld1q_s16(dgd_avg + 0 * dgd_avg_stride + j + col1);
+ dgd1[0] = vshlq_s16(dgd1[0], df_s16);
+ dgd1[1] = vld1q_s16(dgd_avg + 1 * dgd_avg_stride + j + col1);
+ dgd1[1] = vshlq_s16(dgd1[1], df_s16);
+ dgd1[2] = vld1q_s16(dgd_avg + 2 * dgd_avg_stride + j + col1);
+ dgd1[2] = vshlq_s16(dgd1[2], df_s16);
+ dgd1[3] = vld1q_s16(dgd_avg + 3 * dgd_avg_stride + j + col1);
+ dgd1[3] = vshlq_s16(dgd1[3], df_s16);
+ dgd1[4] = vld1q_s16(dgd_avg + 4 * dgd_avg_stride + j + col1);
+ dgd1[4] = vshlq_s16(dgd1[4], df_s16);
+ dgd1[5] = vld1q_s16(dgd_avg + 5 * dgd_avg_stride + j + col1);
+ dgd1[5] = vshlq_s16(dgd1[5], df_s16);
+ dgd1[6] = vld1q_s16(dgd_avg + 6 * dgd_avg_stride + j + col1);
+ dgd1[6] = vshlq_s16(dgd1[6], df_s16);
+
+ // Compute all elements from the combination of both columns (49
+ // elements).
+ compute_H_two_cols(dgd0, dgd1, col0, col1, H, wiener_win,
+ wiener_win2);
+ }
+ }
+ }
+ dgd_avg += downsample_factor * dgd_avg_stride;
+ src_avg += src_avg_stride;
+ h += downsample_factor;
+ } while (h <= v_end - downsample_factor);
+
+ if (h < v_end) {
+ // The last row is scaled by a different downsample factor, so process
+ // separately.
+
+ // Cross-correlation (M).
+ for (int row = 0; row < 7; row++) {
+ int16x8_t dgd0 = vld1q_s16(dgd_avg + row * dgd_avg_stride);
+ int j = 0;
+ while (j <= width - 8) {
+ int16x8_t dgd1 = vld1q_s16(dgd_avg + row * dgd_avg_stride + j + 8);
+ // Load src vector and scale based on downsampling factor.
+ int16x8_t s =
+ vmulq_n_s16(vld1q_s16(src_avg + j), last_row_downsample_factor);
+
+ // Compute all the elements of one row of M.
+ compute_M_one_row_win7(s, dgd0, dgd1, M_trn, wiener_win, row);
+
+ dgd0 = dgd1;
+ j += 8;
+ }
+ // Process remaining elements without Neon.
+ while (j < width) {
+ int16_t s = src_avg[j];
+ int16_t d0 = dgd_avg[row * dgd_avg_stride + 0 + j];
+ int16_t d1 = dgd_avg[row * dgd_avg_stride + 1 + j];
+ int16_t d2 = dgd_avg[row * dgd_avg_stride + 2 + j];
+ int16_t d3 = dgd_avg[row * dgd_avg_stride + 3 + j];
+ int16_t d4 = dgd_avg[row * dgd_avg_stride + 4 + j];
+ int16_t d5 = dgd_avg[row * dgd_avg_stride + 5 + j];
+ int16_t d6 = dgd_avg[row * dgd_avg_stride + 6 + j];
+
+ M_trn[row * wiener_win + 0] += d0 * s * last_row_downsample_factor;
+ M_trn[row * wiener_win + 1] += d1 * s * last_row_downsample_factor;
+ M_trn[row * wiener_win + 2] += d2 * s * last_row_downsample_factor;
+ M_trn[row * wiener_win + 3] += d3 * s * last_row_downsample_factor;
+ M_trn[row * wiener_win + 4] += d4 * s * last_row_downsample_factor;
+ M_trn[row * wiener_win + 5] += d5 * s * last_row_downsample_factor;
+ M_trn[row * wiener_win + 6] += d6 * s * last_row_downsample_factor;
+
+ j++;
+ }
+ }
+
+ // Auto-covariance (H).
+ int j = 0;
+ while (j <= width - 8) {
+ int col0 = 0;
+ do {
+ // Load first column.
+ int16x8_t dgd0[7];
+ dgd0[0] = vld1q_s16(dgd_avg + 0 * dgd_avg_stride + j + col0);
+ dgd0[1] = vld1q_s16(dgd_avg + 1 * dgd_avg_stride + j + col0);
+ dgd0[2] = vld1q_s16(dgd_avg + 2 * dgd_avg_stride + j + col0);
+ dgd0[3] = vld1q_s16(dgd_avg + 3 * dgd_avg_stride + j + col0);
+ dgd0[4] = vld1q_s16(dgd_avg + 4 * dgd_avg_stride + j + col0);
+ dgd0[5] = vld1q_s16(dgd_avg + 5 * dgd_avg_stride + j + col0);
+ dgd0[6] = vld1q_s16(dgd_avg + 6 * dgd_avg_stride + j + col0);
+
+ // Perform computation of the first column with itself (28 elements).
+ // For the first column this will fill the upper triangle of the 7x7
+ // matrix at the top left of the H matrix. For the next columns this
+ // will fill the upper triangle of the other 7x7 matrices around H's
+ // diagonal.
+ compute_H_one_col_last_row(dgd0, col0, H, wiener_win, wiener_win2,
+ last_row_downsample_factor);
+
+ // All computation next to the matrix diagonal has already been done.
+ for (int col1 = col0 + 1; col1 < wiener_win; col1++) {
+ // Load second column and scale based on downsampling factor.
+ int16x8_t dgd1[7];
+ dgd1[0] = vld1q_s16(dgd_avg + 0 * dgd_avg_stride + j + col1);
+ dgd1[0] = vmulq_n_s16(dgd1[0], last_row_downsample_factor);
+ dgd1[1] = vld1q_s16(dgd_avg + 1 * dgd_avg_stride + j + col1);
+ dgd1[1] = vmulq_n_s16(dgd1[1], last_row_downsample_factor);
+ dgd1[2] = vld1q_s16(dgd_avg + 2 * dgd_avg_stride + j + col1);
+ dgd1[2] = vmulq_n_s16(dgd1[2], last_row_downsample_factor);
+ dgd1[3] = vld1q_s16(dgd_avg + 3 * dgd_avg_stride + j + col1);
+ dgd1[3] = vmulq_n_s16(dgd1[3], last_row_downsample_factor);
+ dgd1[4] = vld1q_s16(dgd_avg + 4 * dgd_avg_stride + j + col1);
+ dgd1[4] = vmulq_n_s16(dgd1[4], last_row_downsample_factor);
+ dgd1[5] = vld1q_s16(dgd_avg + 5 * dgd_avg_stride + j + col1);
+ dgd1[5] = vmulq_n_s16(dgd1[5], last_row_downsample_factor);
+ dgd1[6] = vld1q_s16(dgd_avg + 6 * dgd_avg_stride + j + col1);
+ dgd1[6] = vmulq_n_s16(dgd1[6], last_row_downsample_factor);
+
+ // Compute all elements from the combination of both columns (49
+ // elements).
+ compute_H_two_cols(dgd0, dgd1, col0, col1, H, wiener_win,
+ wiener_win2);
+ }
+ } while (++col0 < wiener_win);
+ j += 8;
+ }
+
+ // Process remaining columns using a mask to discard excess elements.
+ if (j < width) {
+ int col0 = 0;
+ do {
+ // Load first column.
+ int16x8_t dgd0[7];
+ dgd0[0] = vld1q_s16(dgd_avg + 0 * dgd_avg_stride + j + col0);
+ dgd0[0] = vandq_s16(dgd0[0], mask);
+ dgd0[1] = vld1q_s16(dgd_avg + 1 * dgd_avg_stride + j + col0);
+ dgd0[1] = vandq_s16(dgd0[1], mask);
+ dgd0[2] = vld1q_s16(dgd_avg + 2 * dgd_avg_stride + j + col0);
+ dgd0[2] = vandq_s16(dgd0[2], mask);
+ dgd0[3] = vld1q_s16(dgd_avg + 3 * dgd_avg_stride + j + col0);
+ dgd0[3] = vandq_s16(dgd0[3], mask);
+ dgd0[4] = vld1q_s16(dgd_avg + 4 * dgd_avg_stride + j + col0);
+ dgd0[4] = vandq_s16(dgd0[4], mask);
+ dgd0[5] = vld1q_s16(dgd_avg + 5 * dgd_avg_stride + j + col0);
+ dgd0[5] = vandq_s16(dgd0[5], mask);
+ dgd0[6] = vld1q_s16(dgd_avg + 6 * dgd_avg_stride + j + col0);
+ dgd0[6] = vandq_s16(dgd0[6], mask);
+
+ // Perform computation of the first column with itself (15 elements).
+ // For the first column this will fill the upper triangle of the 7x7
+ // matrix at the top left of the H matrix. For the next columns this
+ // will fill the upper triangle of the other 7x7 matrices around H's
+ // diagonal.
+ compute_H_one_col_last_row(dgd0, col0, H, wiener_win, wiener_win2,
+ last_row_downsample_factor);
+
+ // All computation next to the matrix diagonal has already been done.
+ for (int col1 = col0 + 1; col1 < wiener_win; col1++) {
+ // Load second column and scale based on downsampling factor.
+ int16x8_t dgd1[7];
+ dgd1[0] = vld1q_s16(dgd_avg + 0 * dgd_avg_stride + j + col1);
+ dgd1[0] = vmulq_n_s16(dgd1[0], last_row_downsample_factor);
+ dgd1[1] = vld1q_s16(dgd_avg + 1 * dgd_avg_stride + j + col1);
+ dgd1[1] = vmulq_n_s16(dgd1[1], last_row_downsample_factor);
+ dgd1[2] = vld1q_s16(dgd_avg + 2 * dgd_avg_stride + j + col1);
+ dgd1[2] = vmulq_n_s16(dgd1[2], last_row_downsample_factor);
+ dgd1[3] = vld1q_s16(dgd_avg + 3 * dgd_avg_stride + j + col1);
+ dgd1[3] = vmulq_n_s16(dgd1[3], last_row_downsample_factor);
+ dgd1[4] = vld1q_s16(dgd_avg + 4 * dgd_avg_stride + j + col1);
+ dgd1[4] = vmulq_n_s16(dgd1[4], last_row_downsample_factor);
+ dgd1[5] = vld1q_s16(dgd_avg + 5 * dgd_avg_stride + j + col1);
+ dgd1[5] = vmulq_n_s16(dgd1[5], last_row_downsample_factor);
+ dgd1[6] = vld1q_s16(dgd_avg + 6 * dgd_avg_stride + j + col1);
+ dgd1[6] = vmulq_n_s16(dgd1[6], last_row_downsample_factor);
+
+ // Compute all elements from the combination of both columns (49
+ // elements).
+ compute_H_two_cols(dgd0, dgd1, col0, col1, H, wiener_win,
+ wiener_win2);
+ }
+ } while (++col0 < wiener_win);
+ }
+ }
+
+ // Transpose M_trn.
+ transpose_M_win7(M, M_trn, 7);
+
+ // Copy upper triangle of H in the lower one.
+ copy_upper_triangle(H, wiener_win2);
+}
+
+// This function computes two matrices: the cross-correlation between the src
+// buffer and dgd buffer (M), and the auto-covariance of the dgd buffer (H).
+//
+// M is of size 5 * 5. It needs to be filled such that multiplying one element
+// from src with each element of a row of the wiener window will fill one
+// column of M. However this is not very convenient in terms of memory
+// accesses, as it means we do contiguous loads of dgd but strided stores to M.
+// As a result, we use an intermediate matrix M_trn which is instead filled
+// such that one row of the wiener window gives one row of M_trn. Once fully
+// computed, M_trn is then transposed to return M.
+//
+// H is of size 25 * 25. It is filled by multiplying every pair of elements of
+// the wiener window together. Since it is a symmetric matrix, we only compute
+// the upper triangle, and then copy it down to the lower one. Here we fill it
+// by taking each different pair of columns, and multiplying all the elements of
+// the first one with all the elements of the second one, with a special case
+// when multiplying a column by itself.
+static INLINE void compute_stats_win5_neon(int16_t *dgd_avg, int dgd_avg_stride,
+ int16_t *src_avg, int src_avg_stride,
+ int width, int v_start, int v_end,
+ int64_t *M, int64_t *H,
+ int downsample_factor,
+ int last_row_downsample_factor) {
+ const int wiener_win = 5;
+ const int wiener_win2 = wiener_win * wiener_win;
+ // The downsample factor can be either 1 or 4, so instead of multiplying the
+ // values by 1 or 4, we can left shift by 0 or 2 respectively, which is
+ // faster. (This doesn't apply to the last row where we can scale the values
+ // by 1, 2 or 3, so we keep the multiplication).
+ const int downsample_shift = downsample_factor >> 1;
+ const int16x8_t df_s16 = vdupq_n_s16(downsample_shift);
+ const int32x4_t df_s32 = vdupq_n_s32(downsample_shift);
+ const int16x8_t mask = vld1q_s16(&av1_neon_mask_16bit[8] - (width % 8));
+
+ // We use an intermediate matrix that will be transposed to get M.
+ int64_t M_trn[25];
+ memset(M_trn, 0, sizeof(M_trn));
+
+ int h = v_start;
+ do {
+ // Cross-correlation (M).
+ for (int row = 0; row < wiener_win; row++) {
+ int16x8_t dgd0 = vld1q_s16(dgd_avg + row * dgd_avg_stride);
+ int j = 0;
+ while (j <= width - 8) {
+ int16x8_t dgd1 = vld1q_s16(dgd_avg + row * dgd_avg_stride + j + 8);
+ // Load src vector and scale based on downsampling factor.
+ int16x8_t s = vshlq_s16(vld1q_s16(src_avg + j), df_s16);
+
+ // Compute all the elements of one row of M.
+ compute_M_one_row_win5(s, dgd0, dgd1, M_trn, wiener_win, row);
+
+ dgd0 = dgd1;
+ j += 8;
+ }
+
+ // Process remaining elements without Neon.
+ while (j < width) {
+ int16_t s = src_avg[j];
+ int16_t d0 = dgd_avg[row * dgd_avg_stride + 0 + j];
+ int16_t d1 = dgd_avg[row * dgd_avg_stride + 1 + j];
+ int16_t d2 = dgd_avg[row * dgd_avg_stride + 2 + j];
+ int16_t d3 = dgd_avg[row * dgd_avg_stride + 3 + j];
+ int16_t d4 = dgd_avg[row * dgd_avg_stride + 4 + j];
+
+ M_trn[row * wiener_win + 0] += d0 * s * downsample_factor;
+ M_trn[row * wiener_win + 1] += d1 * s * downsample_factor;
+ M_trn[row * wiener_win + 2] += d2 * s * downsample_factor;
+ M_trn[row * wiener_win + 3] += d3 * s * downsample_factor;
+ M_trn[row * wiener_win + 4] += d4 * s * downsample_factor;
+
+ j++;
+ }
+ }
+
+ // Auto-covariance (H).
+ int j = 0;
+ while (j <= width - 8) {
+ for (int col0 = 0; col0 < wiener_win; col0++) {
+ // Load first column.
+ int16x8_t dgd0[5];
+ dgd0[0] = vld1q_s16(dgd_avg + 0 * dgd_avg_stride + j + col0);
+ dgd0[1] = vld1q_s16(dgd_avg + 1 * dgd_avg_stride + j + col0);
+ dgd0[2] = vld1q_s16(dgd_avg + 2 * dgd_avg_stride + j + col0);
+ dgd0[3] = vld1q_s16(dgd_avg + 3 * dgd_avg_stride + j + col0);
+ dgd0[4] = vld1q_s16(dgd_avg + 4 * dgd_avg_stride + j + col0);
+
+ // Perform computation of the first column with itself (15 elements).
+ // For the first column this will fill the upper triangle of the 5x5
+ // matrix at the top left of the H matrix. For the next columns this
+ // will fill the upper triangle of the other 5x5 matrices around H's
+ // diagonal.
+ compute_H_one_col(dgd0, col0, H, wiener_win, wiener_win2, df_s32);
+
+ // All computation next to the matrix diagonal has already been done.
+ for (int col1 = col0 + 1; col1 < wiener_win; col1++) {
+ // Load second column and scale based on downsampling factor.
+ int16x8_t dgd1[5];
+ dgd1[0] = vld1q_s16(dgd_avg + 0 * dgd_avg_stride + j + col1);
+ dgd1[0] = vshlq_s16(dgd1[0], df_s16);
+ dgd1[1] = vld1q_s16(dgd_avg + 1 * dgd_avg_stride + j + col1);
+ dgd1[1] = vshlq_s16(dgd1[1], df_s16);
+ dgd1[2] = vld1q_s16(dgd_avg + 2 * dgd_avg_stride + j + col1);
+ dgd1[2] = vshlq_s16(dgd1[2], df_s16);
+ dgd1[3] = vld1q_s16(dgd_avg + 3 * dgd_avg_stride + j + col1);
+ dgd1[3] = vshlq_s16(dgd1[3], df_s16);
+ dgd1[4] = vld1q_s16(dgd_avg + 4 * dgd_avg_stride + j + col1);
+ dgd1[4] = vshlq_s16(dgd1[4], df_s16);
+
+ // Compute all elements from the combination of both columns (25
+ // elements).
+ compute_H_two_cols(dgd0, dgd1, col0, col1, H, wiener_win,
+ wiener_win2);
+ }
+ }
+ j += 8;
+ }
+
+ // Process remaining columns using a mask to discard excess elements.
+ if (j < width) {
+ for (int col0 = 0; col0 < wiener_win; col0++) {
+ // Load first column.
+ int16x8_t dgd0[5];
+ dgd0[0] = vld1q_s16(dgd_avg + 0 * dgd_avg_stride + j + col0);
+ dgd0[0] = vandq_s16(dgd0[0], mask);
+ dgd0[1] = vld1q_s16(dgd_avg + 1 * dgd_avg_stride + j + col0);
+ dgd0[1] = vandq_s16(dgd0[1], mask);
+ dgd0[2] = vld1q_s16(dgd_avg + 2 * dgd_avg_stride + j + col0);
+ dgd0[2] = vandq_s16(dgd0[2], mask);
+ dgd0[3] = vld1q_s16(dgd_avg + 3 * dgd_avg_stride + j + col0);
+ dgd0[3] = vandq_s16(dgd0[3], mask);
+ dgd0[4] = vld1q_s16(dgd_avg + 4 * dgd_avg_stride + j + col0);
+ dgd0[4] = vandq_s16(dgd0[4], mask);
+
+ // Perform computation of the first column with itself (15 elements).
+ // For the first column this will fill the upper triangle of the 5x5
+ // matrix at the top left of the H matrix. For the next columns this
+ // will fill the upper triangle of the other 5x5 matrices around H's
+ // diagonal.
+ compute_H_one_col(dgd0, col0, H, wiener_win, wiener_win2, df_s32);
+
+ // All computation next to the matrix diagonal has already been done.
+ for (int col1 = col0 + 1; col1 < wiener_win; col1++) {
+ // Load second column and scale based on downsampling factor.
+ int16x8_t dgd1[5];
+ dgd1[0] = vld1q_s16(dgd_avg + 0 * dgd_avg_stride + j + col1);
+ dgd1[0] = vshlq_s16(dgd1[0], df_s16);
+ dgd1[1] = vld1q_s16(dgd_avg + 1 * dgd_avg_stride + j + col1);
+ dgd1[1] = vshlq_s16(dgd1[1], df_s16);
+ dgd1[2] = vld1q_s16(dgd_avg + 2 * dgd_avg_stride + j + col1);
+ dgd1[2] = vshlq_s16(dgd1[2], df_s16);
+ dgd1[3] = vld1q_s16(dgd_avg + 3 * dgd_avg_stride + j + col1);
+ dgd1[3] = vshlq_s16(dgd1[3], df_s16);
+ dgd1[4] = vld1q_s16(dgd_avg + 4 * dgd_avg_stride + j + col1);
+ dgd1[4] = vshlq_s16(dgd1[4], df_s16);
+
+ // Compute all elements from the combination of both columns (25
+ // elements).
+ compute_H_two_cols(dgd0, dgd1, col0, col1, H, wiener_win,
+ wiener_win2);
+ }
+ }
+ }
+ dgd_avg += downsample_factor * dgd_avg_stride;
+ src_avg += src_avg_stride;
+ h += downsample_factor;
+ } while (h <= v_end - downsample_factor);
+
+ if (h < v_end) {
+ // The last row is scaled by a different downsample factor, so process
+ // separately.
+
+ // Cross-correlation (M).
+ for (int row = 0; row < wiener_win; row++) {
+ int16x8_t dgd0 = vld1q_s16(dgd_avg + row * dgd_avg_stride);
+ int j = 0;
+ while (j <= width - 8) {
+ int16x8_t dgd1 = vld1q_s16(dgd_avg + row * dgd_avg_stride + j + 8);
+ // Load src vector and scale based on downsampling factor.
+ int16x8_t s =
+ vmulq_n_s16(vld1q_s16(src_avg + j), last_row_downsample_factor);
+
+ // Compute all the elements of one row of M.
+ compute_M_one_row_win5(s, dgd0, dgd1, M_trn, wiener_win, row);
+
+ dgd0 = dgd1;
+ j += 8;
+ }
+
+ // Process remaining elements without Neon.
+ while (j < width) {
+ int16_t s = src_avg[j];
+ int16_t d0 = dgd_avg[row * dgd_avg_stride + 0 + j];
+ int16_t d1 = dgd_avg[row * dgd_avg_stride + 1 + j];
+ int16_t d2 = dgd_avg[row * dgd_avg_stride + 2 + j];
+ int16_t d3 = dgd_avg[row * dgd_avg_stride + 3 + j];
+ int16_t d4 = dgd_avg[row * dgd_avg_stride + 4 + j];
+
+ M_trn[row * wiener_win + 0] += d0 * s * last_row_downsample_factor;
+ M_trn[row * wiener_win + 1] += d1 * s * last_row_downsample_factor;
+ M_trn[row * wiener_win + 2] += d2 * s * last_row_downsample_factor;
+ M_trn[row * wiener_win + 3] += d3 * s * last_row_downsample_factor;
+ M_trn[row * wiener_win + 4] += d4 * s * last_row_downsample_factor;
+
+ j++;
+ }
+ }
+
+ // Auto-covariance (H).
+ int j = 0;
+ while (j <= width - 8) {
+ for (int col0 = 0; col0 < wiener_win; col0++) {
+ // Load first column.
+ int16x8_t dgd0[5];
+ dgd0[0] = vld1q_s16(dgd_avg + 0 * dgd_avg_stride + j + col0);
+ dgd0[1] = vld1q_s16(dgd_avg + 1 * dgd_avg_stride + j + col0);
+ dgd0[2] = vld1q_s16(dgd_avg + 2 * dgd_avg_stride + j + col0);
+ dgd0[3] = vld1q_s16(dgd_avg + 3 * dgd_avg_stride + j + col0);
+ dgd0[4] = vld1q_s16(dgd_avg + 4 * dgd_avg_stride + j + col0);
+
+ // Perform computation of the first column with itself (15 elements).
+ // For the first column this will fill the upper triangle of the 5x5
+ // matrix at the top left of the H matrix. For the next columns this
+ // will fill the upper triangle of the other 5x5 matrices around H's
+ // diagonal.
+ compute_H_one_col_last_row(dgd0, col0, H, wiener_win, wiener_win2,
+ last_row_downsample_factor);
+
+ // All computation next to the matrix diagonal has already been done.
+ for (int col1 = col0 + 1; col1 < wiener_win; col1++) {
+ // Load second column and scale based on downsampling factor.
+ int16x8_t dgd1[5];
+ dgd1[0] = vld1q_s16(dgd_avg + 0 * dgd_avg_stride + j + col1);
+ dgd1[0] = vmulq_n_s16(dgd1[0], last_row_downsample_factor);
+ dgd1[1] = vld1q_s16(dgd_avg + 1 * dgd_avg_stride + j + col1);
+ dgd1[1] = vmulq_n_s16(dgd1[1], last_row_downsample_factor);
+ dgd1[2] = vld1q_s16(dgd_avg + 2 * dgd_avg_stride + j + col1);
+ dgd1[2] = vmulq_n_s16(dgd1[2], last_row_downsample_factor);
+ dgd1[3] = vld1q_s16(dgd_avg + 3 * dgd_avg_stride + j + col1);
+ dgd1[3] = vmulq_n_s16(dgd1[3], last_row_downsample_factor);
+ dgd1[4] = vld1q_s16(dgd_avg + 4 * dgd_avg_stride + j + col1);
+ dgd1[4] = vmulq_n_s16(dgd1[4], last_row_downsample_factor);
+
+ // Compute all elements from the combination of both columns (25
+ // elements).
+ compute_H_two_cols(dgd0, dgd1, col0, col1, H, wiener_win,
+ wiener_win2);
+ }
+ }
+ j += 8;
+ }
+
+ // Process remaining columns using a mask to discard excess elements.
+ if (j < width) {
+ for (int col0 = 0; col0 < wiener_win; col0++) {
+ // Load first column.
+ int16x8_t dgd0[5];
+ dgd0[0] = vld1q_s16(dgd_avg + 0 * dgd_avg_stride + j + col0);
+ dgd0[0] = vandq_s16(dgd0[0], mask);
+ dgd0[1] = vld1q_s16(dgd_avg + 1 * dgd_avg_stride + j + col0);
+ dgd0[1] = vandq_s16(dgd0[1], mask);
+ dgd0[2] = vld1q_s16(dgd_avg + 2 * dgd_avg_stride + j + col0);
+ dgd0[2] = vandq_s16(dgd0[2], mask);
+ dgd0[3] = vld1q_s16(dgd_avg + 3 * dgd_avg_stride + j + col0);
+ dgd0[3] = vandq_s16(dgd0[3], mask);
+ dgd0[4] = vld1q_s16(dgd_avg + 4 * dgd_avg_stride + j + col0);
+ dgd0[4] = vandq_s16(dgd0[4], mask);
+
+ // Perform computation of the first column with itself (15 elements).
+ // For the first column this will fill the upper triangle of the 5x5
+ // matrix at the top left of the H matrix. For the next columns this
+ // will fill the upper triangle of the other 5x5 matrices around H's
+ // diagonal.
+ compute_H_one_col_last_row(dgd0, col0, H, wiener_win, wiener_win2,
+ last_row_downsample_factor);
+
+ // All computation next to the matrix diagonal has already been done.
+ for (int col1 = col0 + 1; col1 < wiener_win; col1++) {
+ // Load second column and scale based on downsampling factor.
+ int16x8_t dgd1[5];
+ dgd1[0] = vld1q_s16(dgd_avg + 0 * dgd_avg_stride + j + col1);
+ dgd1[0] = vmulq_n_s16(dgd1[0], last_row_downsample_factor);
+ dgd1[1] = vld1q_s16(dgd_avg + 1 * dgd_avg_stride + j + col1);
+ dgd1[1] = vmulq_n_s16(dgd1[1], last_row_downsample_factor);
+ dgd1[2] = vld1q_s16(dgd_avg + 2 * dgd_avg_stride + j + col1);
+ dgd1[2] = vmulq_n_s16(dgd1[2], last_row_downsample_factor);
+ dgd1[3] = vld1q_s16(dgd_avg + 3 * dgd_avg_stride + j + col1);
+ dgd1[3] = vmulq_n_s16(dgd1[3], last_row_downsample_factor);
+ dgd1[4] = vld1q_s16(dgd_avg + 4 * dgd_avg_stride + j + col1);
+ dgd1[4] = vmulq_n_s16(dgd1[4], last_row_downsample_factor);
+
+ // Compute all elements from the combination of both columns (25
+ // elements).
+ compute_H_two_cols(dgd0, dgd1, col0, col1, H, wiener_win,
+ wiener_win2);
+ }
+ }
+ }
+ }
+
+ // Transpose M_trn.
+ transpose_M_win5(M, M_trn, 5);
+
+ // Copy upper triangle of H in the lower one.
+ copy_upper_triangle(H, wiener_win2);
+}
+
+void av1_compute_stats_neon(int wiener_win, const uint8_t *dgd,
+ const uint8_t *src, int16_t *dgd_avg,
+ int16_t *src_avg, int h_start, int h_end,
+ int v_start, int v_end, int dgd_stride,
+ int src_stride, int64_t *M, int64_t *H,
+ int use_downsampled_wiener_stats) {
+ assert(wiener_win == WIENER_WIN || wiener_win == WIENER_WIN_CHROMA);
+
+ const int wiener_win2 = wiener_win * wiener_win;
+ const int wiener_halfwin = wiener_win >> 1;
+ const int32_t width = h_end - h_start;
+ const int32_t height = v_end - v_start;
+ const uint8_t *dgd_start = &dgd[v_start * dgd_stride + h_start];
+ memset(H, 0, sizeof(*H) * wiener_win2 * wiener_win2);
+
+ uint8_t avg = find_average_neon(dgd_start, dgd_stride, width, height);
+ assert(WIENER_STATS_DOWNSAMPLE_FACTOR == 4);
+ int downsample_factor =
+ use_downsampled_wiener_stats ? WIENER_STATS_DOWNSAMPLE_FACTOR : 1;
+
+ int dgd_avg_stride = width + 2 * wiener_halfwin;
+ int src_avg_stride = width;
+
+ // Compute (dgd - avg) and store it in dgd_avg.
+ // The wiener window will slide along the dgd frame, centered on each pixel.
+ // For the top left pixel and all the pixels on the side of the frame this
+ // means half of the window will be outside of the frame. As such the actual
+ // buffer that we need to subtract the avg from will be 2 * wiener_halfwin
+ // wider and 2 * wiener_halfwin higher than the original dgd buffer.
+ const int vert_offset = v_start - wiener_halfwin;
+ const int horiz_offset = h_start - wiener_halfwin;
+ const uint8_t *dgd_win = dgd + horiz_offset + vert_offset * dgd_stride;
+ compute_sub_avg(dgd_win, dgd_stride, avg, dgd_avg, dgd_avg_stride,
+ width + 2 * wiener_halfwin, height + 2 * wiener_halfwin, 1);
+
+ // Compute (src - avg), downsample if necessary and store in src-avg.
+ const uint8_t *src_start = src + h_start + v_start * src_stride;
+ compute_sub_avg(src_start, src_stride * downsample_factor, avg, src_avg,
+ src_avg_stride, width, height, downsample_factor);
+
+ // Since the height is not necessarily a multiple of the downsample factor,
+ // the last line of src will be scaled according to how many rows remain.
+ int last_row_downsample_factor =
+ use_downsampled_wiener_stats ? height % downsample_factor : 1;
+
+ if (wiener_win == WIENER_WIN) {
+ compute_stats_win7_neon(dgd_avg, dgd_avg_stride, src_avg, src_avg_stride,
+ width, v_start, v_end, M, H, downsample_factor,
+ last_row_downsample_factor);
+ } else {
+ compute_stats_win5_neon(dgd_avg, dgd_avg_stride, src_avg, src_avg_stride,
+ width, v_start, v_end, M, H, downsample_factor,
+ last_row_downsample_factor);
+ }
+}
+
+static INLINE void calc_proj_params_r0_r1_neon(
+ const uint8_t *src8, int width, int height, int src_stride,
+ const uint8_t *dat8, int dat_stride, int32_t *flt0, int flt0_stride,
+ int32_t *flt1, int flt1_stride, int64_t H[2][2], int64_t C[2]) {
+ assert(width % 8 == 0);
+ const int size = width * height;
+
+ int64x2_t h00_lo = vdupq_n_s64(0);
+ int64x2_t h00_hi = vdupq_n_s64(0);
+ int64x2_t h11_lo = vdupq_n_s64(0);
+ int64x2_t h11_hi = vdupq_n_s64(0);
+ int64x2_t h01_lo = vdupq_n_s64(0);
+ int64x2_t h01_hi = vdupq_n_s64(0);
+ int64x2_t c0_lo = vdupq_n_s64(0);
+ int64x2_t c0_hi = vdupq_n_s64(0);
+ int64x2_t c1_lo = vdupq_n_s64(0);
+ int64x2_t c1_hi = vdupq_n_s64(0);
+
+ do {
+ const uint8_t *src_ptr = src8;
+ const uint8_t *dat_ptr = dat8;
+ int32_t *flt0_ptr = flt0;
+ int32_t *flt1_ptr = flt1;
+ int w = width;
+
+ do {
+ uint8x8_t s = vld1_u8(src_ptr);
+ uint8x8_t d = vld1_u8(dat_ptr);
+ int32x4_t f0_lo = vld1q_s32(flt0_ptr);
+ int32x4_t f0_hi = vld1q_s32(flt0_ptr + 4);
+ int32x4_t f1_lo = vld1q_s32(flt1_ptr);
+ int32x4_t f1_hi = vld1q_s32(flt1_ptr + 4);
+
+ int16x8_t u = vreinterpretq_s16_u16(vshll_n_u8(d, SGRPROJ_RST_BITS));
+ int16x8_t s_s16 = vreinterpretq_s16_u16(vshll_n_u8(s, SGRPROJ_RST_BITS));
+
+ int32x4_t s_lo = vsubl_s16(vget_low_s16(s_s16), vget_low_s16(u));
+ int32x4_t s_hi = vsubl_s16(vget_high_s16(s_s16), vget_high_s16(u));
+ f0_lo = vsubw_s16(f0_lo, vget_low_s16(u));
+ f0_hi = vsubw_s16(f0_hi, vget_high_s16(u));
+ f1_lo = vsubw_s16(f1_lo, vget_low_s16(u));
+ f1_hi = vsubw_s16(f1_hi, vget_high_s16(u));
+
+ h00_lo = vmlal_s32(h00_lo, vget_low_s32(f0_lo), vget_low_s32(f0_lo));
+ h00_lo = vmlal_s32(h00_lo, vget_high_s32(f0_lo), vget_high_s32(f0_lo));
+ h00_hi = vmlal_s32(h00_hi, vget_low_s32(f0_hi), vget_low_s32(f0_hi));
+ h00_hi = vmlal_s32(h00_hi, vget_high_s32(f0_hi), vget_high_s32(f0_hi));
+
+ h11_lo = vmlal_s32(h11_lo, vget_low_s32(f1_lo), vget_low_s32(f1_lo));
+ h11_lo = vmlal_s32(h11_lo, vget_high_s32(f1_lo), vget_high_s32(f1_lo));
+ h11_hi = vmlal_s32(h11_hi, vget_low_s32(f1_hi), vget_low_s32(f1_hi));
+ h11_hi = vmlal_s32(h11_hi, vget_high_s32(f1_hi), vget_high_s32(f1_hi));
+
+ h01_lo = vmlal_s32(h01_lo, vget_low_s32(f0_lo), vget_low_s32(f1_lo));
+ h01_lo = vmlal_s32(h01_lo, vget_high_s32(f0_lo), vget_high_s32(f1_lo));
+ h01_hi = vmlal_s32(h01_hi, vget_low_s32(f0_hi), vget_low_s32(f1_hi));
+ h01_hi = vmlal_s32(h01_hi, vget_high_s32(f0_hi), vget_high_s32(f1_hi));
+
+ c0_lo = vmlal_s32(c0_lo, vget_low_s32(f0_lo), vget_low_s32(s_lo));
+ c0_lo = vmlal_s32(c0_lo, vget_high_s32(f0_lo), vget_high_s32(s_lo));
+ c0_hi = vmlal_s32(c0_hi, vget_low_s32(f0_hi), vget_low_s32(s_hi));
+ c0_hi = vmlal_s32(c0_hi, vget_high_s32(f0_hi), vget_high_s32(s_hi));
+
+ c1_lo = vmlal_s32(c1_lo, vget_low_s32(f1_lo), vget_low_s32(s_lo));
+ c1_lo = vmlal_s32(c1_lo, vget_high_s32(f1_lo), vget_high_s32(s_lo));
+ c1_hi = vmlal_s32(c1_hi, vget_low_s32(f1_hi), vget_low_s32(s_hi));
+ c1_hi = vmlal_s32(c1_hi, vget_high_s32(f1_hi), vget_high_s32(s_hi));
+
+ src_ptr += 8;
+ dat_ptr += 8;
+ flt0_ptr += 8;
+ flt1_ptr += 8;
+ w -= 8;
+ } while (w != 0);
+
+ src8 += src_stride;
+ dat8 += dat_stride;
+ flt0 += flt0_stride;
+ flt1 += flt1_stride;
+ } while (--height != 0);
+
+ H[0][0] = horizontal_add_s64x2(vaddq_s64(h00_lo, h00_hi)) / size;
+ H[0][1] = horizontal_add_s64x2(vaddq_s64(h01_lo, h01_hi)) / size;
+ H[1][1] = horizontal_add_s64x2(vaddq_s64(h11_lo, h11_hi)) / size;
+ H[1][0] = H[0][1];
+ C[0] = horizontal_add_s64x2(vaddq_s64(c0_lo, c0_hi)) / size;
+ C[1] = horizontal_add_s64x2(vaddq_s64(c1_lo, c1_hi)) / size;
+}
+
+static INLINE void calc_proj_params_r0_neon(const uint8_t *src8, int width,
+ int height, int src_stride,
+ const uint8_t *dat8, int dat_stride,
+ int32_t *flt0, int flt0_stride,
+ int64_t H[2][2], int64_t C[2]) {
+ assert(width % 8 == 0);
+ const int size = width * height;
+
+ int64x2_t h00_lo = vdupq_n_s64(0);
+ int64x2_t h00_hi = vdupq_n_s64(0);
+ int64x2_t c0_lo = vdupq_n_s64(0);
+ int64x2_t c0_hi = vdupq_n_s64(0);
+
+ do {
+ const uint8_t *src_ptr = src8;
+ const uint8_t *dat_ptr = dat8;
+ int32_t *flt0_ptr = flt0;
+ int w = width;
+
+ do {
+ uint8x8_t s = vld1_u8(src_ptr);
+ uint8x8_t d = vld1_u8(dat_ptr);
+ int32x4_t f0_lo = vld1q_s32(flt0_ptr);
+ int32x4_t f0_hi = vld1q_s32(flt0_ptr + 4);
+
+ int16x8_t u = vreinterpretq_s16_u16(vshll_n_u8(d, SGRPROJ_RST_BITS));
+ int16x8_t s_s16 = vreinterpretq_s16_u16(vshll_n_u8(s, SGRPROJ_RST_BITS));
+
+ int32x4_t s_lo = vsubl_s16(vget_low_s16(s_s16), vget_low_s16(u));
+ int32x4_t s_hi = vsubl_s16(vget_high_s16(s_s16), vget_high_s16(u));
+ f0_lo = vsubw_s16(f0_lo, vget_low_s16(u));
+ f0_hi = vsubw_s16(f0_hi, vget_high_s16(u));
+
+ h00_lo = vmlal_s32(h00_lo, vget_low_s32(f0_lo), vget_low_s32(f0_lo));
+ h00_lo = vmlal_s32(h00_lo, vget_high_s32(f0_lo), vget_high_s32(f0_lo));
+ h00_hi = vmlal_s32(h00_hi, vget_low_s32(f0_hi), vget_low_s32(f0_hi));
+ h00_hi = vmlal_s32(h00_hi, vget_high_s32(f0_hi), vget_high_s32(f0_hi));
+
+ c0_lo = vmlal_s32(c0_lo, vget_low_s32(f0_lo), vget_low_s32(s_lo));
+ c0_lo = vmlal_s32(c0_lo, vget_high_s32(f0_lo), vget_high_s32(s_lo));
+ c0_hi = vmlal_s32(c0_hi, vget_low_s32(f0_hi), vget_low_s32(s_hi));
+ c0_hi = vmlal_s32(c0_hi, vget_high_s32(f0_hi), vget_high_s32(s_hi));
+
+ src_ptr += 8;
+ dat_ptr += 8;
+ flt0_ptr += 8;
+ w -= 8;
+ } while (w != 0);
+
+ src8 += src_stride;
+ dat8 += dat_stride;
+ flt0 += flt0_stride;
+ } while (--height != 0);
+
+ H[0][0] = horizontal_add_s64x2(vaddq_s64(h00_lo, h00_hi)) / size;
+ C[0] = horizontal_add_s64x2(vaddq_s64(c0_lo, c0_hi)) / size;
+}
+
+static INLINE void calc_proj_params_r1_neon(const uint8_t *src8, int width,
+ int height, int src_stride,
+ const uint8_t *dat8, int dat_stride,
+ int32_t *flt1, int flt1_stride,
+ int64_t H[2][2], int64_t C[2]) {
+ assert(width % 8 == 0);
+ const int size = width * height;
+
+ int64x2_t h11_lo = vdupq_n_s64(0);
+ int64x2_t h11_hi = vdupq_n_s64(0);
+ int64x2_t c1_lo = vdupq_n_s64(0);
+ int64x2_t c1_hi = vdupq_n_s64(0);
+
+ do {
+ const uint8_t *src_ptr = src8;
+ const uint8_t *dat_ptr = dat8;
+ int32_t *flt1_ptr = flt1;
+ int w = width;
+
+ do {
+ uint8x8_t s = vld1_u8(src_ptr);
+ uint8x8_t d = vld1_u8(dat_ptr);
+ int32x4_t f1_lo = vld1q_s32(flt1_ptr);
+ int32x4_t f1_hi = vld1q_s32(flt1_ptr + 4);
+
+ int16x8_t u = vreinterpretq_s16_u16(vshll_n_u8(d, SGRPROJ_RST_BITS));
+ int16x8_t s_s16 = vreinterpretq_s16_u16(vshll_n_u8(s, SGRPROJ_RST_BITS));
+
+ int32x4_t s_lo = vsubl_s16(vget_low_s16(s_s16), vget_low_s16(u));
+ int32x4_t s_hi = vsubl_s16(vget_high_s16(s_s16), vget_high_s16(u));
+ f1_lo = vsubw_s16(f1_lo, vget_low_s16(u));
+ f1_hi = vsubw_s16(f1_hi, vget_high_s16(u));
+
+ h11_lo = vmlal_s32(h11_lo, vget_low_s32(f1_lo), vget_low_s32(f1_lo));
+ h11_lo = vmlal_s32(h11_lo, vget_high_s32(f1_lo), vget_high_s32(f1_lo));
+ h11_hi = vmlal_s32(h11_hi, vget_low_s32(f1_hi), vget_low_s32(f1_hi));
+ h11_hi = vmlal_s32(h11_hi, vget_high_s32(f1_hi), vget_high_s32(f1_hi));
+
+ c1_lo = vmlal_s32(c1_lo, vget_low_s32(f1_lo), vget_low_s32(s_lo));
+ c1_lo = vmlal_s32(c1_lo, vget_high_s32(f1_lo), vget_high_s32(s_lo));
+ c1_hi = vmlal_s32(c1_hi, vget_low_s32(f1_hi), vget_low_s32(s_hi));
+ c1_hi = vmlal_s32(c1_hi, vget_high_s32(f1_hi), vget_high_s32(s_hi));
+
+ src_ptr += 8;
+ dat_ptr += 8;
+ flt1_ptr += 8;
+ w -= 8;
+ } while (w != 0);
+
+ src8 += src_stride;
+ dat8 += dat_stride;
+ flt1 += flt1_stride;
+ } while (--height != 0);
+
+ H[1][1] = horizontal_add_s64x2(vaddq_s64(h11_lo, h11_hi)) / size;
+ C[1] = horizontal_add_s64x2(vaddq_s64(c1_lo, c1_hi)) / size;
+}
+
+// The function calls 3 subfunctions for the following cases :
+// 1) When params->r[0] > 0 and params->r[1] > 0. In this case all elements
+// of C and H need to be computed.
+// 2) When only params->r[0] > 0. In this case only H[0][0] and C[0] are
+// non-zero and need to be computed.
+// 3) When only params->r[1] > 0. In this case only H[1][1] and C[1] are
+// non-zero and need to be computed.
+void av1_calc_proj_params_neon(const uint8_t *src8, int width, int height,
+ int src_stride, const uint8_t *dat8,
+ int dat_stride, int32_t *flt0, int flt0_stride,
+ int32_t *flt1, int flt1_stride, int64_t H[2][2],
+ int64_t C[2], const sgr_params_type *params) {
+ if ((params->r[0] > 0) && (params->r[1] > 0)) {
+ calc_proj_params_r0_r1_neon(src8, width, height, src_stride, dat8,
+ dat_stride, flt0, flt0_stride, flt1,
+ flt1_stride, H, C);
+ } else if (params->r[0] > 0) {
+ calc_proj_params_r0_neon(src8, width, height, src_stride, dat8, dat_stride,
+ flt0, flt0_stride, H, C);
+ } else if (params->r[1] > 0) {
+ calc_proj_params_r1_neon(src8, width, height, src_stride, dat8, dat_stride,
+ flt1, flt1_stride, H, C);
+ }
+}
diff --git a/av1/encoder/arm/neon/pickrst_neon.h b/av1/encoder/arm/neon/pickrst_neon.h
new file mode 100644
index 0000000..d9a9ad4
--- /dev/null
+++ b/av1/encoder/arm/neon/pickrst_neon.h
@@ -0,0 +1,281 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#ifndef AOM_AV1_ENCODER_ARM_NEON_PICKRST_NEON_H_
+#define AOM_AV1_ENCODER_ARM_NEON_PICKRST_NEON_H_
+
+#include <arm_neon.h>
+
+#include "aom_dsp/arm/sum_neon.h"
+#include "aom_dsp/arm/transpose_neon.h"
+
+// When we load 8 values of int16_t type and need less than 8 values for
+// processing, the below mask is used to make the extra values zero.
+extern const int16_t av1_neon_mask_16bit[16];
+
+static INLINE void copy_upper_triangle(int64_t *H, const int wiener_win2) {
+ for (int i = 0; i < wiener_win2 - 2; i = i + 2) {
+ // Transpose the first 2x2 square. It needs a special case as the element
+ // of the bottom left is on the diagonal.
+ int64x2_t row0 = vld1q_s64(H + i * wiener_win2 + i + 1);
+ int64x2_t row1 = vld1q_s64(H + (i + 1) * wiener_win2 + i + 1);
+
+ int64x2_t tr_row = aom_vtrn2q_s64(row0, row1);
+
+ vst1_s64(H + (i + 1) * wiener_win2 + i, vget_low_s64(row0));
+ vst1q_s64(H + (i + 2) * wiener_win2 + i, tr_row);
+
+ // Transpose and store all the remaining 2x2 squares of the line.
+ for (int j = i + 3; j < wiener_win2; j = j + 2) {
+ row0 = vld1q_s64(H + i * wiener_win2 + j);
+ row1 = vld1q_s64(H + (i + 1) * wiener_win2 + j);
+
+ int64x2_t tr_row0 = aom_vtrn1q_s64(row0, row1);
+ int64x2_t tr_row1 = aom_vtrn2q_s64(row0, row1);
+
+ vst1q_s64(H + j * wiener_win2 + i, tr_row0);
+ vst1q_s64(H + (j + 1) * wiener_win2 + i, tr_row1);
+ }
+ }
+}
+
+static INLINE void transpose_M_win5(int64_t *M, int64_t *M_trn,
+ const int wiener_win) {
+ // 1st and 2nd rows.
+ int64x2_t row00 = vld1q_s64(M_trn);
+ int64x2_t row10 = vld1q_s64(M_trn + wiener_win);
+ vst1q_s64(M, aom_vtrn1q_s64(row00, row10));
+ vst1q_s64(M + wiener_win, aom_vtrn2q_s64(row00, row10));
+
+ int64x2_t row02 = vld1q_s64(M_trn + 2);
+ int64x2_t row12 = vld1q_s64(M_trn + wiener_win + 2);
+ vst1q_s64(M + 2 * wiener_win, aom_vtrn1q_s64(row02, row12));
+ vst1q_s64(M + 3 * wiener_win, aom_vtrn2q_s64(row02, row12));
+
+ // Last column only needs trn2.
+ int64x2_t row03 = vld1q_s64(M_trn + 3);
+ int64x2_t row13 = vld1q_s64(M_trn + wiener_win + 3);
+ vst1q_s64(M + 4 * wiener_win, aom_vtrn2q_s64(row03, row13));
+
+ // 3rd and 4th rows.
+ int64x2_t row20 = vld1q_s64(M_trn + 2 * wiener_win);
+ int64x2_t row30 = vld1q_s64(M_trn + 3 * wiener_win);
+ vst1q_s64(M + 2, aom_vtrn1q_s64(row20, row30));
+ vst1q_s64(M + wiener_win + 2, aom_vtrn2q_s64(row20, row30));
+
+ int64x2_t row22 = vld1q_s64(M_trn + 2 * wiener_win + 2);
+ int64x2_t row32 = vld1q_s64(M_trn + 3 * wiener_win + 2);
+ vst1q_s64(M + 2 * wiener_win + 2, aom_vtrn1q_s64(row22, row32));
+ vst1q_s64(M + 3 * wiener_win + 2, aom_vtrn2q_s64(row22, row32));
+
+ // Last column only needs trn2.
+ int64x2_t row23 = vld1q_s64(M_trn + 2 * wiener_win + 3);
+ int64x2_t row33 = vld1q_s64(M_trn + 3 * wiener_win + 3);
+ vst1q_s64(M + 4 * wiener_win + 2, aom_vtrn2q_s64(row23, row33));
+
+ // Last row.
+ int64x2_t row40 = vld1q_s64(M_trn + 4 * wiener_win);
+ vst1_s64(M + 4, vget_low_s64(row40));
+ vst1_s64(M + 1 * wiener_win + 4, vget_high_s64(row40));
+
+ int64x2_t row42 = vld1q_s64(M_trn + 4 * wiener_win + 2);
+ vst1_s64(M + 2 * wiener_win + 4, vget_low_s64(row42));
+ vst1_s64(M + 3 * wiener_win + 4, vget_high_s64(row42));
+
+ // Element on the bottom right of M_trn is copied as is.
+ vst1_s64(M + 4 * wiener_win + 4, vld1_s64(M_trn + 4 * wiener_win + 4));
+}
+
+static INLINE void transpose_M_win7(int64_t *M, int64_t *M_trn,
+ const int wiener_win) {
+ // 1st and 2nd rows.
+ int64x2_t row00 = vld1q_s64(M_trn);
+ int64x2_t row10 = vld1q_s64(M_trn + wiener_win);
+ vst1q_s64(M, aom_vtrn1q_s64(row00, row10));
+ vst1q_s64(M + wiener_win, aom_vtrn2q_s64(row00, row10));
+
+ int64x2_t row02 = vld1q_s64(M_trn + 2);
+ int64x2_t row12 = vld1q_s64(M_trn + wiener_win + 2);
+ vst1q_s64(M + 2 * wiener_win, aom_vtrn1q_s64(row02, row12));
+ vst1q_s64(M + 3 * wiener_win, aom_vtrn2q_s64(row02, row12));
+
+ int64x2_t row04 = vld1q_s64(M_trn + 4);
+ int64x2_t row14 = vld1q_s64(M_trn + wiener_win + 4);
+ vst1q_s64(M + 4 * wiener_win, aom_vtrn1q_s64(row04, row14));
+ vst1q_s64(M + 5 * wiener_win, aom_vtrn2q_s64(row04, row14));
+
+ // Last column only needs trn2.
+ int64x2_t row05 = vld1q_s64(M_trn + 5);
+ int64x2_t row15 = vld1q_s64(M_trn + wiener_win + 5);
+ vst1q_s64(M + 6 * wiener_win, aom_vtrn2q_s64(row05, row15));
+
+ // 3rd and 4th rows.
+ int64x2_t row20 = vld1q_s64(M_trn + 2 * wiener_win);
+ int64x2_t row30 = vld1q_s64(M_trn + 3 * wiener_win);
+ vst1q_s64(M + 2, aom_vtrn1q_s64(row20, row30));
+ vst1q_s64(M + wiener_win + 2, aom_vtrn2q_s64(row20, row30));
+
+ int64x2_t row22 = vld1q_s64(M_trn + 2 * wiener_win + 2);
+ int64x2_t row32 = vld1q_s64(M_trn + 3 * wiener_win + 2);
+ vst1q_s64(M + 2 * wiener_win + 2, aom_vtrn1q_s64(row22, row32));
+ vst1q_s64(M + 3 * wiener_win + 2, aom_vtrn2q_s64(row22, row32));
+
+ int64x2_t row24 = vld1q_s64(M_trn + 2 * wiener_win + 4);
+ int64x2_t row34 = vld1q_s64(M_trn + 3 * wiener_win + 4);
+ vst1q_s64(M + 4 * wiener_win + 2, aom_vtrn1q_s64(row24, row34));
+ vst1q_s64(M + 5 * wiener_win + 2, aom_vtrn2q_s64(row24, row34));
+
+ // Last column only needs trn2.
+ int64x2_t row25 = vld1q_s64(M_trn + 2 * wiener_win + 5);
+ int64x2_t row35 = vld1q_s64(M_trn + 3 * wiener_win + 5);
+ vst1q_s64(M + 6 * wiener_win + 2, aom_vtrn2q_s64(row25, row35));
+
+ // 5th and 6th rows.
+ int64x2_t row40 = vld1q_s64(M_trn + 4 * wiener_win);
+ int64x2_t row50 = vld1q_s64(M_trn + 5 * wiener_win);
+ vst1q_s64(M + 4, aom_vtrn1q_s64(row40, row50));
+ vst1q_s64(M + wiener_win + 4, aom_vtrn2q_s64(row40, row50));
+
+ int64x2_t row42 = vld1q_s64(M_trn + 4 * wiener_win + 2);
+ int64x2_t row52 = vld1q_s64(M_trn + 5 * wiener_win + 2);
+ vst1q_s64(M + 2 * wiener_win + 4, aom_vtrn1q_s64(row42, row52));
+ vst1q_s64(M + 3 * wiener_win + 4, aom_vtrn2q_s64(row42, row52));
+
+ int64x2_t row44 = vld1q_s64(M_trn + 4 * wiener_win + 4);
+ int64x2_t row54 = vld1q_s64(M_trn + 5 * wiener_win + 4);
+ vst1q_s64(M + 4 * wiener_win + 4, aom_vtrn1q_s64(row44, row54));
+ vst1q_s64(M + 5 * wiener_win + 4, aom_vtrn2q_s64(row44, row54));
+
+ // Last column only needs trn2.
+ int64x2_t row45 = vld1q_s64(M_trn + 4 * wiener_win + 5);
+ int64x2_t row55 = vld1q_s64(M_trn + 5 * wiener_win + 5);
+ vst1q_s64(M + 6 * wiener_win + 4, aom_vtrn2q_s64(row45, row55));
+
+ // Last row.
+ int64x2_t row60 = vld1q_s64(M_trn + 6 * wiener_win);
+ vst1_s64(M + 6, vget_low_s64(row60));
+ vst1_s64(M + 1 * wiener_win + 6, vget_high_s64(row60));
+
+ int64x2_t row62 = vld1q_s64(M_trn + 6 * wiener_win + 2);
+ vst1_s64(M + 2 * wiener_win + 6, vget_low_s64(row62));
+ vst1_s64(M + 3 * wiener_win + 6, vget_high_s64(row62));
+
+ int64x2_t row64 = vld1q_s64(M_trn + 6 * wiener_win + 4);
+ vst1_s64(M + 4 * wiener_win + 6, vget_low_s64(row64));
+ vst1_s64(M + 5 * wiener_win + 6, vget_high_s64(row64));
+
+ // Element on the bottom right of M_trn is copied as is.
+ vst1_s64(M + 6 * wiener_win + 6, vld1_s64(M_trn + 6 * wiener_win + 6));
+}
+
+static INLINE void compute_M_one_row_win5(int16x8_t src, int16x8_t dgd0,
+ int16x8_t dgd1, int64_t *M,
+ const int wiener_win, int row) {
+ int64x2_t m_01 = vld1q_s64(M + row * wiener_win + 0);
+ int64x2_t m_23 = vld1q_s64(M + row * wiener_win + 2);
+
+ int32x4_t m0 = vmull_s16(vget_low_s16(src), vget_low_s16(dgd0));
+ m0 = vmlal_s16(m0, vget_high_s16(src), vget_high_s16(dgd0));
+
+ int16x8_t dgd01 = vextq_s16(dgd0, dgd1, 1);
+ int32x4_t m1 = vmull_s16(vget_low_s16(src), vget_low_s16(dgd01));
+ m1 = vmlal_s16(m1, vget_high_s16(src), vget_high_s16(dgd01));
+
+ m0 = horizontal_add_2d_s32(m0, m1);
+ m_01 = vpadalq_s32(m_01, m0);
+ vst1q_s64(M + row * wiener_win + 0, m_01);
+
+ int16x8_t dgd02 = vextq_s16(dgd0, dgd1, 2);
+ int32x4_t m2 = vmull_s16(vget_low_s16(src), vget_low_s16(dgd02));
+ m2 = vmlal_s16(m2, vget_high_s16(src), vget_high_s16(dgd02));
+
+ int16x8_t dgd03 = vextq_s16(dgd0, dgd1, 3);
+ int32x4_t m3 = vmull_s16(vget_low_s16(src), vget_low_s16(dgd03));
+ m3 = vmlal_s16(m3, vget_high_s16(src), vget_high_s16(dgd03));
+
+ m2 = horizontal_add_2d_s32(m2, m3);
+ m_23 = vpadalq_s32(m_23, m2);
+ vst1q_s64(M + row * wiener_win + 2, m_23);
+
+ int16x8_t dgd04 = vextq_s16(dgd0, dgd1, 4);
+ int32x4_t m4 = vmull_s16(vget_low_s16(src), vget_low_s16(dgd04));
+ m4 = vmlal_s16(m4, vget_high_s16(src), vget_high_s16(dgd04));
+ M[row * wiener_win + 4] += horizontal_long_add_s32x4(m4);
+}
+
+static INLINE void compute_M_one_row_win7(int16x8_t src, int16x8_t dgd0,
+ int16x8_t dgd1, int64_t *M,
+ const int wiener_win, int row) {
+ int64x2_t m_01 = vld1q_s64(M + row * wiener_win + 0);
+ int64x2_t m_23 = vld1q_s64(M + row * wiener_win + 2);
+ int64x2_t m_45 = vld1q_s64(M + row * wiener_win + 4);
+
+ int32x4_t m0 = vmull_s16(vget_low_s16(src), vget_low_s16(dgd0));
+ m0 = vmlal_s16(m0, vget_high_s16(src), vget_high_s16(dgd0));
+
+ int16x8_t dgd01 = vextq_s16(dgd0, dgd1, 1);
+ int32x4_t m1 = vmull_s16(vget_low_s16(src), vget_low_s16(dgd01));
+ m1 = vmlal_s16(m1, vget_high_s16(src), vget_high_s16(dgd01));
+
+ m0 = horizontal_add_2d_s32(m0, m1);
+ m_01 = vpadalq_s32(m_01, m0);
+ vst1q_s64(M + row * wiener_win + 0, m_01);
+
+ int16x8_t dgd02 = vextq_s16(dgd0, dgd1, 2);
+ int32x4_t m2 = vmull_s16(vget_low_s16(src), vget_low_s16(dgd02));
+ m2 = vmlal_s16(m2, vget_high_s16(src), vget_high_s16(dgd02));
+
+ int16x8_t dgd03 = vextq_s16(dgd0, dgd1, 3);
+ int32x4_t m3 = vmull_s16(vget_low_s16(src), vget_low_s16(dgd03));
+ m3 = vmlal_s16(m3, vget_high_s16(src), vget_high_s16(dgd03));
+
+ m2 = horizontal_add_2d_s32(m2, m3);
+ m_23 = vpadalq_s32(m_23, m2);
+ vst1q_s64(M + row * wiener_win + 2, m_23);
+
+ int16x8_t dgd04 = vextq_s16(dgd0, dgd1, 4);
+ int32x4_t m4 = vmull_s16(vget_low_s16(src), vget_low_s16(dgd04));
+ m4 = vmlal_s16(m4, vget_high_s16(src), vget_high_s16(dgd04));
+
+ int16x8_t dgd05 = vextq_s16(dgd0, dgd1, 5);
+ int32x4_t m5 = vmull_s16(vget_low_s16(src), vget_low_s16(dgd05));
+ m5 = vmlal_s16(m5, vget_high_s16(src), vget_high_s16(dgd05));
+
+ m4 = horizontal_add_2d_s32(m4, m5);
+ m_45 = vpadalq_s32(m_45, m4);
+ vst1q_s64(M + row * wiener_win + 4, m_45);
+
+ int16x8_t dgd06 = vextq_s16(dgd0, dgd1, 6);
+ int32x4_t m6 = vmull_s16(vget_low_s16(src), vget_low_s16(dgd06));
+ m6 = vmlal_s16(m6, vget_high_s16(src), vget_high_s16(dgd06));
+ M[row * wiener_win + 6] += horizontal_long_add_s32x4(m6);
+}
+
+static INLINE void compute_H_two_cols(int16x8_t *dgd0, int16x8_t *dgd1,
+ int col0, int col1, int64_t *H,
+ const int wiener_win,
+ const int wiener_win2) {
+ for (int row0 = 0; row0 < wiener_win; row0++) {
+ for (int row1 = 0; row1 < wiener_win; row1++) {
+ int auto_cov_idx =
+ (col0 * wiener_win + row0) * wiener_win2 + (col1 * wiener_win) + row1;
+
+ int32x4_t auto_cov =
+ vmull_s16(vget_low_s16(dgd0[row0]), vget_low_s16(dgd1[row1]));
+ auto_cov = vmlal_s16(auto_cov, vget_high_s16(dgd0[row0]),
+ vget_high_s16(dgd1[row1]));
+
+ H[auto_cov_idx] += horizontal_long_add_s32x4(auto_cov);
+ }
+ }
+}
+
+#endif // AOM_AV1_ENCODER_ARM_NEON_PICKRST_NEON_H_
diff --git a/av1/encoder/arm/neon/picksrt_neon.c b/av1/encoder/arm/neon/picksrt_neon.c
deleted file mode 100644
index 1346d6b..0000000
--- a/av1/encoder/arm/neon/picksrt_neon.c
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Copyright (c) 2020, Alliance for Open Media. All rights reserved
- *
- * This source code is subject to the terms of the BSD 2 Clause License and
- * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
- * was not distributed with this source code in the LICENSE file, you can
- * obtain it at www.aomedia.org/license/software. If the Alliance for Open
- * Media Patent License 1.0 was not distributed with this source code in the
- * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
- */
-
-#include <arm_neon.h>
-#include <math.h>
-
-#include "aom/aom_integer.h"
-#include "aom_mem/aom_mem.h"
-#include "aom_ports/mem.h"
-#include "av1/common/restoration.h"
-#include "config/aom_config.h"
-#include "config/aom_dsp_rtcd.h"
-
-int64_t av1_lowbd_pixel_proj_error_neon(
- const uint8_t *src8, int width, int height, int src_stride,
- const uint8_t *dat8, int dat_stride, int32_t *flt0, int flt0_stride,
- int32_t *flt1, int flt1_stride, int xq[2], const sgr_params_type *params) {
- int i, j, k;
- const int32_t shift = SGRPROJ_RST_BITS + SGRPROJ_PRJ_BITS;
- const int32x4_t zero = vdupq_n_s32(0);
- uint64x2_t sum64 = vreinterpretq_u64_s32(zero);
- const uint8_t *src = src8;
- const uint8_t *dat = dat8;
-
- int64_t err = 0;
- if (params->r[0] > 0 && params->r[1] > 0) {
- for (i = 0; i < height; ++i) {
- int32x4_t err0 = zero;
- for (j = 0; j <= width - 8; j += 8) {
- const uint8x8_t d0 = vld1_u8(&dat[j]);
- const uint8x8_t s0 = vld1_u8(&src[j]);
- const int16x8_t flt0_16b =
- vcombine_s16(vqmovn_s32(vld1q_s32(&flt0[j])),
- vqmovn_s32(vld1q_s32(&flt0[j + 4])));
- const int16x8_t flt1_16b =
- vcombine_s16(vqmovn_s32(vld1q_s32(&flt1[j])),
- vqmovn_s32(vld1q_s32(&flt1[j + 4])));
- const int16x8_t u0 =
- vreinterpretq_s16_u16(vshll_n_u8(d0, SGRPROJ_RST_BITS));
- const int16x8_t flt0_0_sub_u = vsubq_s16(flt0_16b, u0);
- const int16x8_t flt1_0_sub_u = vsubq_s16(flt1_16b, u0);
- const int16x4_t flt0_16b_sub_u_lo = vget_low_s16(flt0_0_sub_u);
- const int16x4_t flt0_16b_sub_u_hi = vget_high_s16(flt0_0_sub_u);
- const int16x4_t flt1_16b_sub_u_lo = vget_low_s16(flt1_0_sub_u);
- const int16x4_t flt1_16b_sub_u_hi = vget_high_s16(flt1_0_sub_u);
-
- int32x4_t v0 = vmull_n_s16(flt0_16b_sub_u_lo, (int16_t)xq[0]);
- v0 = vmlal_n_s16(v0, flt1_16b_sub_u_lo, (int16_t)xq[1]);
- int32x4_t v1 = vmull_n_s16(flt0_16b_sub_u_hi, (int16_t)xq[0]);
- v1 = vmlal_n_s16(v1, flt1_16b_sub_u_hi, (int16_t)xq[1]);
- const int16x4_t vr0 = vqrshrn_n_s32(v0, 11);
- const int16x4_t vr1 = vqrshrn_n_s32(v1, 11);
- const int16x8_t e0 = vaddq_s16(vcombine_s16(vr0, vr1),
- vreinterpretq_s16_u16(vsubl_u8(d0, s0)));
- const int16x4_t e0_lo = vget_low_s16(e0);
- const int16x4_t e0_hi = vget_high_s16(e0);
- err0 = vmlal_s16(err0, e0_lo, e0_lo);
- err0 = vmlal_s16(err0, e0_hi, e0_hi);
- }
- for (k = j; k < width; ++k) {
- const int32_t u = dat[k] << SGRPROJ_RST_BITS;
- int32_t v = xq[0] * (flt0[k] - u) + xq[1] * (flt1[k] - u);
- const int32_t e = ROUND_POWER_OF_TWO(v, 11) + dat[k] - src[k];
- err += e * e;
- }
- dat += dat_stride;
- src += src_stride;
- flt0 += flt0_stride;
- flt1 += flt1_stride;
- sum64 = vpadalq_u32(sum64, vreinterpretq_u32_s32(err0));
- }
-
- } else if (params->r[0] > 0 || params->r[1] > 0) {
- const int xq_active = (params->r[0] > 0) ? xq[0] : xq[1];
- const int32_t *flt = (params->r[0] > 0) ? flt0 : flt1;
- const int flt_stride = (params->r[0] > 0) ? flt0_stride : flt1_stride;
- for (i = 0; i < height; ++i) {
- int32x4_t err0 = zero;
- for (j = 0; j <= width - 8; j += 8) {
- const uint8x8_t d0 = vld1_u8(&dat[j]);
- const uint8x8_t s0 = vld1_u8(&src[j]);
- const uint16x8_t d0s0 = vsubl_u8(d0, s0);
- const uint16x8x2_t d0w =
- vzipq_u16(vmovl_u8(d0), vreinterpretq_u16_s32(zero));
-
- const int32x4_t flt_16b_lo = vld1q_s32(&flt[j]);
- const int32x4_t flt_16b_hi = vld1q_s32(&flt[j + 4]);
-
- int32x4_t v0 = vmulq_n_s32(flt_16b_lo, xq_active);
- v0 = vmlsq_n_s32(v0, vreinterpretq_s32_u16(d0w.val[0]),
- xq_active * (1 << SGRPROJ_RST_BITS));
- int32x4_t v1 = vmulq_n_s32(flt_16b_hi, xq_active);
- v1 = vmlsq_n_s32(v1, vreinterpretq_s32_u16(d0w.val[1]),
- xq_active * (1 << SGRPROJ_RST_BITS));
- const int16x4_t vr0 = vqrshrn_n_s32(v0, 11);
- const int16x4_t vr1 = vqrshrn_n_s32(v1, 11);
- const int16x8_t e0 =
- vaddq_s16(vcombine_s16(vr0, vr1), vreinterpretq_s16_u16(d0s0));
- const int16x4_t e0_lo = vget_low_s16(e0);
- const int16x4_t e0_hi = vget_high_s16(e0);
- err0 = vmlal_s16(err0, e0_lo, e0_lo);
- err0 = vmlal_s16(err0, e0_hi, e0_hi);
- }
- for (k = j; k < width; ++k) {
- const int32_t u = dat[k] << SGRPROJ_RST_BITS;
- int32_t v = xq_active * (flt[k] - u);
- const int32_t e = ROUND_POWER_OF_TWO(v, shift) + dat[k] - src[k];
- err += e * e;
- }
- dat += dat_stride;
- src += src_stride;
- flt += flt_stride;
- sum64 = vpadalq_u32(sum64, vreinterpretq_u32_s32(err0));
- }
- } else {
- uint32x4_t err0 = vreinterpretq_u32_s32(zero);
- for (i = 0; i < height; ++i) {
- for (j = 0; j <= width - 16; j += 16) {
- const uint8x16_t d = vld1q_u8(&dat[j]);
- const uint8x16_t s = vld1q_u8(&src[j]);
- const uint8x16_t diff = vabdq_u8(d, s);
- const uint8x8_t diff0 = vget_low_u8(diff);
- const uint8x8_t diff1 = vget_high_u8(diff);
- err0 = vpadalq_u16(err0, vmull_u8(diff0, diff0));
- err0 = vpadalq_u16(err0, vmull_u8(diff1, diff1));
- }
- for (k = j; k < width; ++k) {
- const int32_t e = dat[k] - src[k];
- err += e * e;
- }
- dat += dat_stride;
- src += src_stride;
- }
- sum64 = vpaddlq_u32(err0);
- }
-#if AOM_ARCH_AARCH64
- err += vaddvq_u64(sum64);
-#else
- err += vget_lane_u64(vadd_u64(vget_low_u64(sum64), vget_high_u64(sum64)), 0);
-#endif // AOM_ARCH_AARCH64
- return err;
-}
diff --git a/av1/encoder/arm/neon/reconinter_enc_neon.c b/av1/encoder/arm/neon/reconinter_enc_neon.c
index e5975b0..03afa30 100644
--- a/av1/encoder/arm/neon/reconinter_enc_neon.c
+++ b/av1/encoder/arm/neon/reconinter_enc_neon.c
@@ -94,13 +94,13 @@
} else if (!subpel_y_q3) {
const int16_t *const filter_x =
av1_get_interp_filter_subpel_kernel(filter_params, subpel_x_q3 << 1);
- aom_convolve8_horiz_neon(ref, ref_stride, comp_pred, width, filter_x, 16,
- NULL, -1, width, height);
+ aom_convolve8_horiz(ref, ref_stride, comp_pred, width, filter_x, 16, NULL,
+ -1, width, height);
} else if (!subpel_x_q3) {
const int16_t *const filter_y =
av1_get_interp_filter_subpel_kernel(filter_params, subpel_y_q3 << 1);
- aom_convolve8_vert_neon(ref, ref_stride, comp_pred, width, NULL, -1,
- filter_y, 16, width, height);
+ aom_convolve8_vert(ref, ref_stride, comp_pred, width, NULL, -1, filter_y,
+ 16, width, height);
} else {
DECLARE_ALIGNED(16, uint8_t,
im_block[((MAX_SB_SIZE * 2 + 16) + 16) * MAX_SB_SIZE]);
@@ -117,11 +117,10 @@
const int im_vert_offset = im_stride * ((filter_params->taps >> 1) - 1);
assert(im_height <= (MAX_SB_SIZE * 2 + 16) + 16);
- aom_convolve8_horiz_neon(ref - ref_vert_offset, ref_stride, im_block,
- MAX_SB_SIZE, filter_x, 16, NULL, -1, width,
- im_height);
- aom_convolve8_vert_neon(im_block + im_vert_offset, MAX_SB_SIZE, comp_pred,
- width, NULL, -1, filter_y, 16, width, height);
+ aom_convolve8_horiz(ref - ref_vert_offset, ref_stride, im_block,
+ MAX_SB_SIZE, filter_x, 16, NULL, -1, width, im_height);
+ aom_convolve8_vert(im_block + im_vert_offset, MAX_SB_SIZE, comp_pred, width,
+ NULL, -1, filter_y, 16, width, height);
}
}
@@ -138,3 +137,153 @@
aom_comp_avg_pred_neon(comp_pred, pred, width, height, comp_pred, width);
}
+
+void aom_dist_wtd_comp_avg_upsampled_pred_neon(
+ MACROBLOCKD *xd, const AV1_COMMON *const cm, int mi_row, int mi_col,
+ const MV *const mv, uint8_t *comp_pred, const uint8_t *pred, int width,
+ int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref,
+ int ref_stride, const DIST_WTD_COMP_PARAMS *jcp_param, int subpel_search) {
+ aom_upsampled_pred_neon(xd, cm, mi_row, mi_col, mv, comp_pred, width, height,
+ subpel_x_q3, subpel_y_q3, ref, ref_stride,
+ subpel_search);
+
+ aom_dist_wtd_comp_avg_pred_neon(comp_pred, pred, width, height, comp_pred,
+ width, jcp_param);
+}
+
+#if CONFIG_AV1_HIGHBITDEPTH
+void aom_highbd_upsampled_pred_neon(MACROBLOCKD *xd,
+ const struct AV1Common *const cm,
+ int mi_row, int mi_col, const MV *const mv,
+ uint8_t *comp_pred8, int width, int height,
+ int subpel_x_q3, int subpel_y_q3,
+ const uint8_t *ref8, int ref_stride, int bd,
+ int subpel_search) {
+ // expect xd == NULL only in tests
+ if (xd != NULL) {
+ const MB_MODE_INFO *mi = xd->mi[0];
+ const int ref_num = 0;
+ const int is_intrabc = is_intrabc_block(mi);
+ const struct scale_factors *const sf =
+ is_intrabc ? &cm->sf_identity : xd->block_ref_scale_factors[ref_num];
+ const int is_scaled = av1_is_scaled(sf);
+
+ if (is_scaled) {
+ int plane = 0;
+ const int mi_x = mi_col * MI_SIZE;
+ const int mi_y = mi_row * MI_SIZE;
+ const struct macroblockd_plane *const pd = &xd->plane[plane];
+ const struct buf_2d *const dst_buf = &pd->dst;
+ const struct buf_2d *const pre_buf =
+ is_intrabc ? dst_buf : &pd->pre[ref_num];
+
+ InterPredParams inter_pred_params;
+ inter_pred_params.conv_params = get_conv_params(0, plane, xd->bd);
+ const int_interpfilters filters =
+ av1_broadcast_interp_filter(EIGHTTAP_REGULAR);
+ av1_init_inter_params(
+ &inter_pred_params, width, height, mi_y >> pd->subsampling_y,
+ mi_x >> pd->subsampling_x, pd->subsampling_x, pd->subsampling_y,
+ xd->bd, is_cur_buf_hbd(xd), is_intrabc, sf, pre_buf, filters);
+ av1_enc_build_one_inter_predictor(comp_pred8, width, mv,
+ &inter_pred_params);
+ return;
+ }
+ }
+
+ const InterpFilterParams *filter = av1_get_filter(subpel_search);
+
+ if (!subpel_x_q3 && !subpel_y_q3) {
+ const uint16_t *ref = CONVERT_TO_SHORTPTR(ref8);
+ uint16_t *comp_pred = CONVERT_TO_SHORTPTR(comp_pred8);
+ if (width > 4) {
+ assert(width % 8 == 0);
+ int i = height;
+ do {
+ int j = 0;
+ do {
+ uint16x8_t r = vld1q_u16(ref + j);
+ vst1q_u16(comp_pred + j, r);
+ j += 8;
+ } while (j < width);
+ ref += ref_stride;
+ comp_pred += width;
+ } while (--i != 0);
+ } else if (width == 4) {
+ int i = height;
+ do {
+ uint16x4_t r = vld1_u16(ref);
+ vst1_u16(comp_pred, r);
+ ref += ref_stride;
+ comp_pred += width;
+ } while (--i != 0);
+ } else {
+ assert(width == 2);
+ int i = height / 2;
+ do {
+ uint16x4_t r = load_u16_2x2(ref, ref_stride);
+ store_u16_2x1(comp_pred + 0 * width, r, 0);
+ store_u16_2x1(comp_pred + 1 * width, r, 1);
+ ref += 2 * ref_stride;
+ comp_pred += 2 * width;
+ } while (--i != 0);
+ }
+ } else if (!subpel_y_q3) {
+ const int16_t *const kernel =
+ av1_get_interp_filter_subpel_kernel(filter, subpel_x_q3 << 1);
+ aom_highbd_convolve8_horiz_neon(ref8, ref_stride, comp_pred8, width, kernel,
+ 16, NULL, -1, width, height, bd);
+ } else if (!subpel_x_q3) {
+ const int16_t *const kernel =
+ av1_get_interp_filter_subpel_kernel(filter, subpel_y_q3 << 1);
+ aom_highbd_convolve8_vert_neon(ref8, ref_stride, comp_pred8, width, NULL,
+ -1, kernel, 16, width, height, bd);
+ } else {
+ DECLARE_ALIGNED(16, uint16_t,
+ temp[((MAX_SB_SIZE + 16) + 16) * MAX_SB_SIZE]);
+ const int16_t *const kernel_x =
+ av1_get_interp_filter_subpel_kernel(filter, subpel_x_q3 << 1);
+ const int16_t *const kernel_y =
+ av1_get_interp_filter_subpel_kernel(filter, subpel_y_q3 << 1);
+ const int intermediate_height =
+ (((height - 1) * 8 + subpel_y_q3) >> 3) + filter->taps;
+ assert(intermediate_height <= (MAX_SB_SIZE * 2 + 16) + 16);
+ aom_highbd_convolve8_horiz_neon(
+ ref8 - ref_stride * ((filter->taps >> 1) - 1), ref_stride,
+ CONVERT_TO_BYTEPTR(temp), MAX_SB_SIZE, kernel_x, 16, NULL, -1, width,
+ intermediate_height, bd);
+ aom_highbd_convolve8_vert_neon(
+ CONVERT_TO_BYTEPTR(temp + MAX_SB_SIZE * ((filter->taps >> 1) - 1)),
+ MAX_SB_SIZE, comp_pred8, width, NULL, -1, kernel_y, 16, width, height,
+ bd);
+ }
+}
+
+void aom_highbd_comp_avg_upsampled_pred_neon(
+ MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col,
+ const MV *const mv, uint8_t *comp_pred8, const uint8_t *pred8, int width,
+ int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref8,
+ int ref_stride, int bd, int subpel_search) {
+ aom_highbd_upsampled_pred_neon(xd, cm, mi_row, mi_col, mv, comp_pred8, width,
+ height, subpel_x_q3, subpel_y_q3, ref8,
+ ref_stride, bd, subpel_search);
+
+ aom_highbd_comp_avg_pred_neon(comp_pred8, pred8, width, height, comp_pred8,
+ width);
+}
+
+void aom_highbd_dist_wtd_comp_avg_upsampled_pred_neon(
+ MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col,
+ const MV *const mv, uint8_t *comp_pred8, const uint8_t *pred8, int width,
+ int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref8,
+ int ref_stride, int bd, const DIST_WTD_COMP_PARAMS *jcp_param,
+ int subpel_search) {
+ aom_highbd_upsampled_pred_neon(xd, cm, mi_row, mi_col, mv, comp_pred8, width,
+ height, subpel_x_q3, subpel_y_q3, ref8,
+ ref_stride, bd, subpel_search);
+
+ aom_highbd_dist_wtd_comp_avg_pred_neon(comp_pred8, pred8, width, height,
+ comp_pred8, width, jcp_param);
+}
+
+#endif // CONFIG_AV1_HIGHBITDEPTH
diff --git a/av1/encoder/arm/neon/shift_neon.h b/av1/encoder/arm/neon/shift_neon.h
new file mode 100644
index 0000000..d73aef2
--- /dev/null
+++ b/av1/encoder/arm/neon/shift_neon.h
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#ifndef AOM_AV1_ENCODER_ARM_NEON_SHIFT_NEON_H_
+#define AOM_AV1_ENCODER_ARM_NEON_SHIFT_NEON_H_
+
+#include <arm_neon.h>
+
+#include "aom/aom_integer.h" // For AOM_INLINE.
+
+#define SHIFT_LOOP_HELPER(name, type, intrinsic, arg) \
+ static AOM_INLINE void name(const type *in, type *out, int size) { \
+ int i = 0; \
+ do { \
+ out[i] = intrinsic(in[i], arg); \
+ } while (++i < size); \
+ }
+
+SHIFT_LOOP_HELPER(shift_left_2_s16_x4, int16x4_t, vshl_n_s16, 2)
+SHIFT_LOOP_HELPER(shift_left_2_s16_x8, int16x8_t, vshlq_n_s16, 2)
+SHIFT_LOOP_HELPER(shift_left_2_s32_x4, int32x4_t, vshlq_n_s32, 2)
+SHIFT_LOOP_HELPER(shift_right_2_round_s16_x8, int16x8_t, vrshrq_n_s16, 2)
+SHIFT_LOOP_HELPER(shift_right_2_round_s32_x4, int32x4_t, vrshrq_n_s32, 2)
+SHIFT_LOOP_HELPER(shift_right_4_round_s16_x8, int16x8_t, vrshrq_n_s16, 4)
+SHIFT_LOOP_HELPER(shift_right_4_round_s32_x4, int32x4_t, vrshrq_n_s32, 4)
+
+// Addition instructions have slightly better performance compared to shift
+// instructions on some micro-architectures, so use these for shifts by one.
+
+SHIFT_LOOP_HELPER(shift_left_1_s16_x4, int16x4_t, vadd_s16, in[i])
+SHIFT_LOOP_HELPER(shift_left_1_s16_x8, int16x8_t, vaddq_s16, in[i])
+SHIFT_LOOP_HELPER(shift_right_1_round_s16_x4, int16x4_t, vrhadd_s16,
+ vdup_n_s16(0))
+SHIFT_LOOP_HELPER(shift_right_1_round_s16_x8, int16x8_t, vrhaddq_s16,
+ vdupq_n_s16(0))
+SHIFT_LOOP_HELPER(shift_right_1_round_s32_x4, int32x4_t, vrhaddq_s32,
+ vdupq_n_s32(0))
+
+#undef SHIFT_LOOP_HELPER
+
+#endif // AOM_AV1_ENCODER_ARM_NEON_SHIFT_NEON_H_
diff --git a/av1/encoder/arm/neon/temporal_filter_neon.c b/av1/encoder/arm/neon/temporal_filter_neon.c
index 163768b..986f143 100644
--- a/av1/encoder/arm/neon/temporal_filter_neon.c
+++ b/av1/encoder/arm/neon/temporal_filter_neon.c
@@ -22,179 +22,6 @@
// For the squared error buffer, add padding for 4 samples.
#define SSE_STRIDE (BW + 4)
-#if AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_DOTPROD)
-
-// clang-format off
-
-DECLARE_ALIGNED(16, static const uint8_t, kSlidingWindowMask[]) = {
- 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00,
- 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00,
- 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00,
- 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF
-};
-
-// clang-format on
-
-static INLINE void get_abs_diff(const uint8_t *frame1, const uint32_t stride1,
- const uint8_t *frame2, const uint32_t stride2,
- const uint32_t block_width,
- const uint32_t block_height,
- uint8_t *frame_abs_diff,
- const unsigned int dst_stride) {
- uint8_t *dst = frame_abs_diff;
-
- uint32_t i = 0;
- do {
- uint32_t j = 0;
- do {
- uint8x16_t s = vld1q_u8(frame1 + i * stride1 + j);
- uint8x16_t r = vld1q_u8(frame2 + i * stride2 + j);
- uint8x16_t abs_diff = vabdq_u8(s, r);
- vst1q_u8(dst + j + 2, abs_diff);
- j += 16;
- } while (j < block_width);
-
- dst += dst_stride;
- i++;
- } while (i < block_height);
-}
-
-static INLINE uint8x16_t load_and_pad(const uint8_t *src, const uint32_t col,
- const uint32_t block_width) {
- uint8x8_t s = vld1_u8(src);
-
- if (col == 0) {
- const uint8_t lane2 = vget_lane_u8(s, 2);
- s = vset_lane_u8(lane2, s, 0);
- s = vset_lane_u8(lane2, s, 1);
- } else if (col >= block_width - 4) {
- const uint8_t lane5 = vget_lane_u8(s, 5);
- s = vset_lane_u8(lane5, s, 6);
- s = vset_lane_u8(lane5, s, 7);
- }
- return vcombine_u8(s, s);
-}
-
-static void apply_temporal_filter(
- const uint8_t *frame, const unsigned int stride, const uint32_t block_width,
- const uint32_t block_height, const int *subblock_mses,
- unsigned int *accumulator, uint16_t *count, const uint8_t *frame_abs_diff,
- const uint32_t *luma_sse_sum, const double inv_num_ref_pixels,
- const double decay_factor, const double inv_factor,
- const double weight_factor, const double *d_factor, int tf_wgt_calc_lvl) {
- assert(((block_width == 16) || (block_width == 32)) &&
- ((block_height == 16) || (block_height == 32)));
-
- uint32_t acc_5x5_neon[BH][BW];
- const uint8x16x2_t vmask = vld1q_u8_x2(kSlidingWindowMask);
-
- // Traverse 4 columns at a time - first and last two columns need padding.
- for (uint32_t col = 0; col < block_width; col += 4) {
- uint8x16_t vsrc[5][2];
- const uint8_t *src = frame_abs_diff + col;
-
- // Load, pad (for first and last two columns) and mask 3 rows from the top.
- for (int i = 2; i < 5; i++) {
- const uint8x16_t s = load_and_pad(src, col, block_width);
- vsrc[i][0] = vandq_u8(s, vmask.val[0]);
- vsrc[i][1] = vandq_u8(s, vmask.val[1]);
- src += SSE_STRIDE;
- }
-
- // Pad the top 2 rows.
- vsrc[0][0] = vsrc[2][0];
- vsrc[0][1] = vsrc[2][1];
- vsrc[1][0] = vsrc[2][0];
- vsrc[1][1] = vsrc[2][1];
-
- for (unsigned int row = 0; row < block_height; row++) {
- uint32x4_t sum_01 = vdupq_n_u32(0);
- uint32x4_t sum_23 = vdupq_n_u32(0);
-
- sum_01 = vdotq_u32(sum_01, vsrc[0][0], vsrc[0][0]);
- sum_01 = vdotq_u32(sum_01, vsrc[1][0], vsrc[1][0]);
- sum_01 = vdotq_u32(sum_01, vsrc[2][0], vsrc[2][0]);
- sum_01 = vdotq_u32(sum_01, vsrc[3][0], vsrc[3][0]);
- sum_01 = vdotq_u32(sum_01, vsrc[4][0], vsrc[4][0]);
-
- sum_23 = vdotq_u32(sum_23, vsrc[0][1], vsrc[0][1]);
- sum_23 = vdotq_u32(sum_23, vsrc[1][1], vsrc[1][1]);
- sum_23 = vdotq_u32(sum_23, vsrc[2][1], vsrc[2][1]);
- sum_23 = vdotq_u32(sum_23, vsrc[3][1], vsrc[3][1]);
- sum_23 = vdotq_u32(sum_23, vsrc[4][1], vsrc[4][1]);
-
- vst1q_u32(&acc_5x5_neon[row][col], vpaddq_u32(sum_01, sum_23));
-
- // Push all rows in the sliding window up one.
- for (int i = 0; i < 4; i++) {
- vsrc[i][0] = vsrc[i + 1][0];
- vsrc[i][1] = vsrc[i + 1][1];
- }
-
- if (row <= block_height - 4) {
- // Load next row into the bottom of the sliding window.
- uint8x16_t s = load_and_pad(src, col, block_width);
- vsrc[4][0] = vandq_u8(s, vmask.val[0]);
- vsrc[4][1] = vandq_u8(s, vmask.val[1]);
- src += SSE_STRIDE;
- } else {
- // Pad the bottom 2 rows.
- vsrc[4][0] = vsrc[3][0];
- vsrc[4][1] = vsrc[3][1];
- }
- }
- }
-
- // Perform filtering.
- if (tf_wgt_calc_lvl == 0) {
- for (unsigned int i = 0, k = 0; i < block_height; i++) {
- for (unsigned int j = 0; j < block_width; j++, k++) {
- const int pixel_value = frame[i * stride + j];
- const uint32_t diff_sse = acc_5x5_neon[i][j] + luma_sse_sum[i * BW + j];
-
- const double window_error = diff_sse * inv_num_ref_pixels;
- const int subblock_idx =
- (i >= block_height / 2) * 2 + (j >= block_width / 2);
- const double block_error = (double)subblock_mses[subblock_idx];
- const double combined_error =
- weight_factor * window_error + block_error * inv_factor;
- // Compute filter weight.
- double scaled_error =
- combined_error * d_factor[subblock_idx] * decay_factor;
- scaled_error = AOMMIN(scaled_error, 7);
- const int weight = (int)(exp(-scaled_error) * TF_WEIGHT_SCALE);
- accumulator[k] += weight * pixel_value;
- count[k] += weight;
- }
- }
- } else {
- for (unsigned int i = 0, k = 0; i < block_height; i++) {
- for (unsigned int j = 0; j < block_width; j++, k++) {
- const int pixel_value = frame[i * stride + j];
- const uint32_t diff_sse = acc_5x5_neon[i][j] + luma_sse_sum[i * BW + j];
-
- const double window_error = diff_sse * inv_num_ref_pixels;
- const int subblock_idx =
- (i >= block_height / 2) * 2 + (j >= block_width / 2);
- const double block_error = (double)subblock_mses[subblock_idx];
- const double combined_error =
- weight_factor * window_error + block_error * inv_factor;
- // Compute filter weight.
- double scaled_error =
- combined_error * d_factor[subblock_idx] * decay_factor;
- scaled_error = AOMMIN(scaled_error, 7);
- const float fweight =
- approx_exp((float)-scaled_error) * TF_WEIGHT_SCALE;
- const int weight = iroundpf(fweight);
- accumulator[k] += weight * pixel_value;
- count[k] += weight;
- }
- }
- }
-}
-
-#else // !(AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_DOTPROD))
-
// When using vld1q_u16_x4 compilers may insert an alignment hint of 256 bits.
DECLARE_ALIGNED(32, static const uint16_t, kSlidingWindowMask[]) = {
0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0x0000, 0x0000, 0x0000,
@@ -230,8 +57,7 @@
} while (j < block_width);
dst += dst_stride;
- i++;
- } while (i < block_height);
+ } while (++i < block_height);
}
static INLINE uint16x8_t load_and_pad(const uint16_t *src, const uint32_t col,
@@ -351,8 +177,6 @@
}
}
-#endif // AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_DOTPROD)
-
void av1_apply_temporal_filter_neon(
const YV12_BUFFER_CONFIG *frame_to_filter, const MACROBLOCKD *mbd,
const BLOCK_SIZE block_size, const int mb_row, const int mb_col,
@@ -393,11 +217,7 @@
double s_decay = pow((double)filter_strength / TF_STRENGTH_THRESHOLD, 2);
s_decay = CLIP(s_decay, 1e-5, 1);
double d_factor[4] = { 0 };
-#if AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_DOTPROD)
- uint8_t frame_abs_diff[SSE_STRIDE * BH] = { 0 };
-#else // !(AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_DOTPROD))
uint16_t frame_sse[SSE_STRIDE * BH] = { 0 };
-#endif // AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_DOTPROD)
uint32_t luma_sse_sum[BW * BH] = { 0 };
for (int subblock_idx = 0; subblock_idx < 4; subblock_idx++) {
@@ -436,32 +256,6 @@
// search is only done on Y-plane, so the information from Y-plane
// will be more accurate. The luma sse sum is reused in both chroma
// planes.
-#if AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_DOTPROD)
- if (plane == AOM_PLANE_U) {
- for (unsigned int i = 0; i < plane_h; i++) {
- for (unsigned int j = 0; j < plane_w; j++) {
- for (int ii = 0; ii < (1 << ss_y_shift); ++ii) {
- for (int jj = 0; jj < (1 << ss_x_shift); ++jj) {
- const int yy = (i << ss_y_shift) + ii; // Y-coord on Y-plane.
- const int xx = (j << ss_x_shift) + jj; // X-coord on Y-plane.
- luma_sse_sum[i * BW + j] +=
- (frame_abs_diff[yy * SSE_STRIDE + xx + 2] *
- frame_abs_diff[yy * SSE_STRIDE + xx + 2]);
- }
- }
- }
- }
- }
-
- get_abs_diff(ref, frame_stride, pred + plane_offset, plane_w, plane_w,
- plane_h, frame_abs_diff, SSE_STRIDE);
-
- apply_temporal_filter(pred + plane_offset, plane_w, plane_w, plane_h,
- subblock_mses, accum + plane_offset,
- count + plane_offset, frame_abs_diff, luma_sse_sum,
- inv_num_ref_pixels, decay_factor, inv_factor,
- weight_factor, d_factor, tf_wgt_calc_lvl);
-#else // !(AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_DOTPROD))
if (plane == AOM_PLANE_U) {
for (unsigned int i = 0; i < plane_h; i++) {
for (unsigned int j = 0; j < plane_w; j++) {
@@ -484,8 +278,271 @@
count + plane_offset, frame_sse, luma_sse_sum,
inv_num_ref_pixels, decay_factor, inv_factor,
weight_factor, d_factor, tf_wgt_calc_lvl);
-#endif // AOM_ARCH_AARCH64 && defined(__ARM_FEATURE_DOTPROD)
plane_offset += plane_h * plane_w;
}
}
+
+double av1_estimate_noise_from_single_plane_neon(const uint8_t *src, int height,
+ int width, int stride,
+ int edge_thresh) {
+ uint16x8_t thresh = vdupq_n_u16(edge_thresh);
+ uint32x4_t acc = vdupq_n_u32(0);
+ // Count is in theory positive as it counts the number of times we're under
+ // the threshold, but it will be counted negatively in order to make best use
+ // of the vclt instruction, which sets every bit of a lane to 1 when the
+ // condition is true.
+ int32x4_t count = vdupq_n_s32(0);
+ int final_count = 0;
+ int64_t final_acc = 0;
+ const uint8_t *src_start = src + stride + 1;
+ int h = 1;
+
+ do {
+ int w = 1;
+ const uint8_t *src_ptr = src_start;
+
+ while (w <= (width - 1) - 16) {
+ uint8x16_t mat[3][3];
+ mat[0][0] = vld1q_u8(src_ptr - stride - 1);
+ mat[0][1] = vld1q_u8(src_ptr - stride);
+ mat[0][2] = vld1q_u8(src_ptr - stride + 1);
+ mat[1][0] = vld1q_u8(src_ptr - 1);
+ mat[1][1] = vld1q_u8(src_ptr);
+ mat[1][2] = vld1q_u8(src_ptr + 1);
+ mat[2][0] = vld1q_u8(src_ptr + stride - 1);
+ mat[2][1] = vld1q_u8(src_ptr + stride);
+ mat[2][2] = vld1q_u8(src_ptr + stride + 1);
+
+ // Compute Sobel gradients.
+ uint16x8_t gxa_lo =
+ vaddl_u8(vget_low_u8(mat[0][0]), vget_low_u8(mat[2][0]));
+ uint16x8_t gxa_hi =
+ vaddl_u8(vget_high_u8(mat[0][0]), vget_high_u8(mat[2][0]));
+ uint16x8_t gxb_lo =
+ vaddl_u8(vget_low_u8(mat[0][2]), vget_low_u8(mat[2][2]));
+ uint16x8_t gxb_hi =
+ vaddl_u8(vget_high_u8(mat[0][2]), vget_high_u8(mat[2][2]));
+ gxa_lo = vaddq_u16(
+ gxa_lo, vaddl_u8(vget_low_u8(mat[1][0]), vget_low_u8(mat[1][0])));
+ gxa_hi = vaddq_u16(
+ gxa_hi, vaddl_u8(vget_high_u8(mat[1][0]), vget_high_u8(mat[1][0])));
+ gxb_lo = vaddq_u16(
+ gxb_lo, vaddl_u8(vget_low_u8(mat[1][2]), vget_low_u8(mat[1][2])));
+ gxb_hi = vaddq_u16(
+ gxb_hi, vaddl_u8(vget_high_u8(mat[1][2]), vget_high_u8(mat[1][2])));
+
+ uint16x8_t gya_lo =
+ vaddl_u8(vget_low_u8(mat[0][0]), vget_low_u8(mat[0][2]));
+ uint16x8_t gya_hi =
+ vaddl_u8(vget_high_u8(mat[0][0]), vget_high_u8(mat[0][2]));
+ uint16x8_t gyb_lo =
+ vaddl_u8(vget_low_u8(mat[2][0]), vget_low_u8(mat[2][2]));
+ uint16x8_t gyb_hi =
+ vaddl_u8(vget_high_u8(mat[2][0]), vget_high_u8(mat[2][2]));
+ gya_lo = vaddq_u16(
+ gya_lo, vaddl_u8(vget_low_u8(mat[0][1]), vget_low_u8(mat[0][1])));
+ gya_hi = vaddq_u16(
+ gya_hi, vaddl_u8(vget_high_u8(mat[0][1]), vget_high_u8(mat[0][1])));
+ gyb_lo = vaddq_u16(
+ gyb_lo, vaddl_u8(vget_low_u8(mat[2][1]), vget_low_u8(mat[2][1])));
+ gyb_hi = vaddq_u16(
+ gyb_hi, vaddl_u8(vget_high_u8(mat[2][1]), vget_high_u8(mat[2][1])));
+
+ uint16x8_t ga_lo = vabaq_u16(vabdq_u16(gxa_lo, gxb_lo), gya_lo, gyb_lo);
+ uint16x8_t ga_hi = vabaq_u16(vabdq_u16(gxa_hi, gxb_hi), gya_hi, gyb_hi);
+
+ // Check which vector elements are under the threshold. The Laplacian is
+ // then unconditionally computed and we accumulate zeros if we're not
+ // under the threshold. This is much faster than using an if statement.
+ uint16x8_t thresh_u16_lo = vcltq_u16(ga_lo, thresh);
+ uint16x8_t thresh_u16_hi = vcltq_u16(ga_hi, thresh);
+
+ uint16x8_t center_lo = vshll_n_u8(vget_low_u8(mat[1][1]), 2);
+ uint16x8_t center_hi = vshll_n_u8(vget_high_u8(mat[1][1]), 2);
+
+ uint16x8_t adj0_lo =
+ vaddl_u8(vget_low_u8(mat[0][1]), vget_low_u8(mat[2][1]));
+ uint16x8_t adj0_hi =
+ vaddl_u8(vget_high_u8(mat[0][1]), vget_high_u8(mat[2][1]));
+ uint16x8_t adj1_lo =
+ vaddl_u8(vget_low_u8(mat[1][0]), vget_low_u8(mat[1][2]));
+ uint16x8_t adj1_hi =
+ vaddl_u8(vget_high_u8(mat[1][0]), vget_high_u8(mat[1][2]));
+ uint16x8_t adj_lo = vaddq_u16(adj0_lo, adj1_lo);
+ adj_lo = vaddq_u16(adj_lo, adj_lo);
+ uint16x8_t adj_hi = vaddq_u16(adj0_hi, adj1_hi);
+ adj_hi = vaddq_u16(adj_hi, adj_hi);
+
+ uint16x8_t diag0_lo =
+ vaddl_u8(vget_low_u8(mat[0][0]), vget_low_u8(mat[0][2]));
+ uint16x8_t diag0_hi =
+ vaddl_u8(vget_high_u8(mat[0][0]), vget_high_u8(mat[0][2]));
+ uint16x8_t diag1_lo =
+ vaddl_u8(vget_low_u8(mat[2][0]), vget_low_u8(mat[2][2]));
+ uint16x8_t diag1_hi =
+ vaddl_u8(vget_high_u8(mat[2][0]), vget_high_u8(mat[2][2]));
+ uint16x8_t diag_lo = vaddq_u16(diag0_lo, diag1_lo);
+ uint16x8_t diag_hi = vaddq_u16(diag0_hi, diag1_hi);
+
+ uint16x8_t v_lo = vaddq_u16(center_lo, diag_lo);
+ v_lo = vabdq_u16(v_lo, adj_lo);
+ uint16x8_t v_hi = vaddq_u16(center_hi, diag_hi);
+ v_hi = vabdq_u16(v_hi, adj_hi);
+
+ acc = vpadalq_u16(acc, vandq_u16(v_lo, thresh_u16_lo));
+ acc = vpadalq_u16(acc, vandq_u16(v_hi, thresh_u16_hi));
+
+ // Add -1 for each lane where the gradient is under the threshold.
+ count = vpadalq_s16(count, vreinterpretq_s16_u16(thresh_u16_lo));
+ count = vpadalq_s16(count, vreinterpretq_s16_u16(thresh_u16_hi));
+
+ w += 16;
+ src_ptr += 16;
+ }
+
+ if (w <= (width - 1) - 8) {
+ uint8x8_t mat[3][3];
+ mat[0][0] = vld1_u8(src_ptr - stride - 1);
+ mat[0][1] = vld1_u8(src_ptr - stride);
+ mat[0][2] = vld1_u8(src_ptr - stride + 1);
+ mat[1][0] = vld1_u8(src_ptr - 1);
+ mat[1][1] = vld1_u8(src_ptr);
+ mat[1][2] = vld1_u8(src_ptr + 1);
+ mat[2][0] = vld1_u8(src_ptr + stride - 1);
+ mat[2][1] = vld1_u8(src_ptr + stride);
+ mat[2][2] = vld1_u8(src_ptr + stride + 1);
+
+ // Compute Sobel gradients.
+ uint16x8_t gxa = vaddl_u8(mat[0][0], mat[2][0]);
+ uint16x8_t gxb = vaddl_u8(mat[0][2], mat[2][2]);
+ gxa = vaddq_u16(gxa, vaddl_u8(mat[1][0], mat[1][0]));
+ gxb = vaddq_u16(gxb, vaddl_u8(mat[1][2], mat[1][2]));
+
+ uint16x8_t gya = vaddl_u8(mat[0][0], mat[0][2]);
+ uint16x8_t gyb = vaddl_u8(mat[2][0], mat[2][2]);
+ gya = vaddq_u16(gya, vaddl_u8(mat[0][1], mat[0][1]));
+ gyb = vaddq_u16(gyb, vaddl_u8(mat[2][1], mat[2][1]));
+
+ uint16x8_t ga = vabaq_u16(vabdq_u16(gxa, gxb), gya, gyb);
+
+ // Check which vector elements are under the threshold. The Laplacian is
+ // then unconditionally computed and we accumulate zeros if we're not
+ // under the threshold. This is much faster than using an if statement.
+ uint16x8_t thresh_u16 = vcltq_u16(ga, thresh);
+
+ uint16x8_t center = vshll_n_u8(mat[1][1], 2);
+
+ uint16x8_t adj0 = vaddl_u8(mat[0][1], mat[2][1]);
+ uint16x8_t adj1 = vaddl_u8(mat[1][0], mat[1][2]);
+ uint16x8_t adj = vaddq_u16(adj0, adj1);
+ adj = vaddq_u16(adj, adj);
+
+ uint16x8_t diag0 = vaddl_u8(mat[0][0], mat[0][2]);
+ uint16x8_t diag1 = vaddl_u8(mat[2][0], mat[2][2]);
+ uint16x8_t diag = vaddq_u16(diag0, diag1);
+
+ uint16x8_t v = vaddq_u16(center, diag);
+ v = vabdq_u16(v, adj);
+
+ acc = vpadalq_u16(acc, vandq_u16(v, thresh_u16));
+ // Add -1 for each lane where the gradient is under the threshold.
+ count = vpadalq_s16(count, vreinterpretq_s16_u16(thresh_u16));
+
+ w += 8;
+ src_ptr += 8;
+ }
+
+ if (w <= (width - 1) - 4) {
+ uint16x8_t mask = vcombine_u16(vdup_n_u16(65535), vdup_n_u16(0));
+ uint8x8_t mat[3][3];
+ mat[0][0] = load_u8_4x1(src_ptr - stride - 1);
+ mat[0][1] = load_u8_4x1(src_ptr - stride);
+ mat[0][2] = load_u8_4x1(src_ptr - stride + 1);
+ mat[1][0] = load_u8_4x1(src_ptr - 1);
+ mat[1][1] = load_u8_4x1(src_ptr);
+ mat[1][2] = load_u8_4x1(src_ptr + 1);
+ mat[2][0] = load_u8_4x1(src_ptr + stride - 1);
+ mat[2][1] = load_u8_4x1(src_ptr + stride);
+ mat[2][2] = load_u8_4x1(src_ptr + stride + 1);
+
+ // Compute Sobel gradients.
+ uint16x8_t gxa = vaddl_u8(mat[0][0], mat[2][0]);
+ uint16x8_t gxb = vaddl_u8(mat[0][2], mat[2][2]);
+ gxa = vaddq_u16(gxa, vaddl_u8(mat[1][0], mat[1][0]));
+ gxb = vaddq_u16(gxb, vaddl_u8(mat[1][2], mat[1][2]));
+
+ uint16x8_t gya = vaddl_u8(mat[0][0], mat[0][2]);
+ uint16x8_t gyb = vaddl_u8(mat[2][0], mat[2][2]);
+ gya = vaddq_u16(gya, vaddl_u8(mat[0][1], mat[0][1]));
+ gyb = vaddq_u16(gyb, vaddl_u8(mat[2][1], mat[2][1]));
+
+ uint16x8_t ga = vabaq_u16(vabdq_u16(gxa, gxb), gya, gyb);
+
+ // Check which vector elements are under the threshold. The Laplacian is
+ // then unconditionally computed and we accumulate zeros if we're not
+ // under the threshold. This is much faster than using an if statement.
+ uint16x8_t thresh_u16 = vandq_u16(vcltq_u16(ga, thresh), mask);
+
+ uint16x8_t center = vshll_n_u8(mat[1][1], 2);
+
+ uint16x8_t adj0 = vaddl_u8(mat[0][1], mat[2][1]);
+ uint16x8_t adj1 = vaddl_u8(mat[1][0], mat[1][2]);
+ uint16x8_t adj = vaddq_u16(adj0, adj1);
+ adj = vaddq_u16(adj, adj);
+
+ uint16x8_t diag0 = vaddl_u8(mat[0][0], mat[0][2]);
+ uint16x8_t diag1 = vaddl_u8(mat[2][0], mat[2][2]);
+ uint16x8_t diag = vaddq_u16(diag0, diag1);
+
+ uint16x8_t v = vaddq_u16(center, diag);
+ v = vabdq_u16(v, adj);
+
+ acc = vpadalq_u16(acc, vandq_u16(v, thresh_u16));
+ // Add -1 for each lane where the gradient is under the threshold.
+ count = vpadalq_s16(count, vreinterpretq_s16_u16(thresh_u16));
+
+ w += 4;
+ src_ptr += 4;
+ }
+
+ while (w < width - 1) {
+ int mat[3][3];
+ mat[0][0] = *(src_ptr - stride - 1);
+ mat[0][1] = *(src_ptr - stride);
+ mat[0][2] = *(src_ptr - stride + 1);
+ mat[1][0] = *(src_ptr - 1);
+ mat[1][1] = *(src_ptr);
+ mat[1][2] = *(src_ptr + 1);
+ mat[2][0] = *(src_ptr + stride - 1);
+ mat[2][1] = *(src_ptr + stride);
+ mat[2][2] = *(src_ptr + stride + 1);
+
+ // Compute Sobel gradients.
+ const int gx = (mat[0][0] - mat[0][2]) + (mat[2][0] - mat[2][2]) +
+ 2 * (mat[1][0] - mat[1][2]);
+ const int gy = (mat[0][0] - mat[2][0]) + (mat[0][2] - mat[2][2]) +
+ 2 * (mat[0][1] - mat[2][1]);
+ const int ga = abs(gx) + abs(gy);
+
+ // Accumulate Laplacian.
+ const int is_under = ga < edge_thresh;
+ const int v = 4 * mat[1][1] -
+ 2 * (mat[0][1] + mat[2][1] + mat[1][0] + mat[1][2]) +
+ (mat[0][0] + mat[0][2] + mat[2][0] + mat[2][2]);
+ final_acc += abs(v) * is_under;
+ final_count += is_under;
+
+ src_ptr++;
+ w++;
+ }
+ src_start += stride;
+ } while (++h < height - 1);
+
+ // We counted negatively, so subtract to get the final value.
+ final_count -= horizontal_add_s32x4(count);
+ final_acc += horizontal_long_add_u32x4(acc);
+ return (final_count < 16)
+ ? -1.0
+ : (double)final_acc / (6 * final_count) * SQRT_PI_BY_2;
+}
diff --git a/av1/encoder/arm/neon/temporal_filter_neon_dotprod.c b/av1/encoder/arm/neon/temporal_filter_neon_dotprod.c
new file mode 100644
index 0000000..5a52e70
--- /dev/null
+++ b/av1/encoder/arm/neon/temporal_filter_neon_dotprod.c
@@ -0,0 +1,299 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+
+#include "config/aom_config.h"
+#include "config/av1_rtcd.h"
+#include "av1/encoder/encoder.h"
+#include "av1/encoder/temporal_filter.h"
+#include "aom_dsp/mathutils.h"
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_dsp/arm/sum_neon.h"
+
+// For the squared error buffer, add padding for 4 samples.
+#define SSE_STRIDE (BW + 4)
+
+// clang-format off
+
+DECLARE_ALIGNED(16, static const uint8_t, kSlidingWindowMask[]) = {
+ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00,
+ 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00,
+ 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00,
+ 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF
+};
+
+// clang-format on
+
+static INLINE void get_abs_diff(const uint8_t *frame1, const uint32_t stride1,
+ const uint8_t *frame2, const uint32_t stride2,
+ const uint32_t block_width,
+ const uint32_t block_height,
+ uint8_t *frame_abs_diff,
+ const unsigned int dst_stride) {
+ uint8_t *dst = frame_abs_diff;
+
+ uint32_t i = 0;
+ do {
+ uint32_t j = 0;
+ do {
+ uint8x16_t s = vld1q_u8(frame1 + i * stride1 + j);
+ uint8x16_t r = vld1q_u8(frame2 + i * stride2 + j);
+ uint8x16_t abs_diff = vabdq_u8(s, r);
+ vst1q_u8(dst + j + 2, abs_diff);
+ j += 16;
+ } while (j < block_width);
+
+ dst += dst_stride;
+ } while (++i < block_height);
+}
+
+static INLINE uint8x16_t load_and_pad(const uint8_t *src, const uint32_t col,
+ const uint32_t block_width) {
+ uint8x8_t s = vld1_u8(src);
+
+ if (col == 0) {
+ const uint8_t lane2 = vget_lane_u8(s, 2);
+ s = vset_lane_u8(lane2, s, 0);
+ s = vset_lane_u8(lane2, s, 1);
+ } else if (col >= block_width - 4) {
+ const uint8_t lane5 = vget_lane_u8(s, 5);
+ s = vset_lane_u8(lane5, s, 6);
+ s = vset_lane_u8(lane5, s, 7);
+ }
+ return vcombine_u8(s, s);
+}
+
+static void apply_temporal_filter(
+ const uint8_t *frame, const unsigned int stride, const uint32_t block_width,
+ const uint32_t block_height, const int *subblock_mses,
+ unsigned int *accumulator, uint16_t *count, const uint8_t *frame_abs_diff,
+ const uint32_t *luma_sse_sum, const double inv_num_ref_pixels,
+ const double decay_factor, const double inv_factor,
+ const double weight_factor, const double *d_factor, int tf_wgt_calc_lvl) {
+ assert(((block_width == 16) || (block_width == 32)) &&
+ ((block_height == 16) || (block_height == 32)));
+
+ uint32_t acc_5x5_neon[BH][BW];
+ const uint8x16x2_t vmask = vld1q_u8_x2(kSlidingWindowMask);
+
+ // Traverse 4 columns at a time - first and last two columns need padding.
+ for (uint32_t col = 0; col < block_width; col += 4) {
+ uint8x16_t vsrc[5][2];
+ const uint8_t *src = frame_abs_diff + col;
+
+ // Load, pad (for first and last two columns) and mask 3 rows from the top.
+ for (int i = 2; i < 5; i++) {
+ const uint8x16_t s = load_and_pad(src, col, block_width);
+ vsrc[i][0] = vandq_u8(s, vmask.val[0]);
+ vsrc[i][1] = vandq_u8(s, vmask.val[1]);
+ src += SSE_STRIDE;
+ }
+
+ // Pad the top 2 rows.
+ vsrc[0][0] = vsrc[2][0];
+ vsrc[0][1] = vsrc[2][1];
+ vsrc[1][0] = vsrc[2][0];
+ vsrc[1][1] = vsrc[2][1];
+
+ for (unsigned int row = 0; row < block_height; row++) {
+ uint32x4_t sum_01 = vdupq_n_u32(0);
+ uint32x4_t sum_23 = vdupq_n_u32(0);
+
+ sum_01 = vdotq_u32(sum_01, vsrc[0][0], vsrc[0][0]);
+ sum_01 = vdotq_u32(sum_01, vsrc[1][0], vsrc[1][0]);
+ sum_01 = vdotq_u32(sum_01, vsrc[2][0], vsrc[2][0]);
+ sum_01 = vdotq_u32(sum_01, vsrc[3][0], vsrc[3][0]);
+ sum_01 = vdotq_u32(sum_01, vsrc[4][0], vsrc[4][0]);
+
+ sum_23 = vdotq_u32(sum_23, vsrc[0][1], vsrc[0][1]);
+ sum_23 = vdotq_u32(sum_23, vsrc[1][1], vsrc[1][1]);
+ sum_23 = vdotq_u32(sum_23, vsrc[2][1], vsrc[2][1]);
+ sum_23 = vdotq_u32(sum_23, vsrc[3][1], vsrc[3][1]);
+ sum_23 = vdotq_u32(sum_23, vsrc[4][1], vsrc[4][1]);
+
+ vst1q_u32(&acc_5x5_neon[row][col], vpaddq_u32(sum_01, sum_23));
+
+ // Push all rows in the sliding window up one.
+ for (int i = 0; i < 4; i++) {
+ vsrc[i][0] = vsrc[i + 1][0];
+ vsrc[i][1] = vsrc[i + 1][1];
+ }
+
+ if (row <= block_height - 4) {
+ // Load next row into the bottom of the sliding window.
+ uint8x16_t s = load_and_pad(src, col, block_width);
+ vsrc[4][0] = vandq_u8(s, vmask.val[0]);
+ vsrc[4][1] = vandq_u8(s, vmask.val[1]);
+ src += SSE_STRIDE;
+ } else {
+ // Pad the bottom 2 rows.
+ vsrc[4][0] = vsrc[3][0];
+ vsrc[4][1] = vsrc[3][1];
+ }
+ }
+ }
+
+ // Perform filtering.
+ if (tf_wgt_calc_lvl == 0) {
+ for (unsigned int i = 0, k = 0; i < block_height; i++) {
+ for (unsigned int j = 0; j < block_width; j++, k++) {
+ const int pixel_value = frame[i * stride + j];
+ const uint32_t diff_sse = acc_5x5_neon[i][j] + luma_sse_sum[i * BW + j];
+
+ const double window_error = diff_sse * inv_num_ref_pixels;
+ const int subblock_idx =
+ (i >= block_height / 2) * 2 + (j >= block_width / 2);
+ const double block_error = (double)subblock_mses[subblock_idx];
+ const double combined_error =
+ weight_factor * window_error + block_error * inv_factor;
+ // Compute filter weight.
+ double scaled_error =
+ combined_error * d_factor[subblock_idx] * decay_factor;
+ scaled_error = AOMMIN(scaled_error, 7);
+ const int weight = (int)(exp(-scaled_error) * TF_WEIGHT_SCALE);
+ accumulator[k] += weight * pixel_value;
+ count[k] += weight;
+ }
+ }
+ } else {
+ for (unsigned int i = 0, k = 0; i < block_height; i++) {
+ for (unsigned int j = 0; j < block_width; j++, k++) {
+ const int pixel_value = frame[i * stride + j];
+ const uint32_t diff_sse = acc_5x5_neon[i][j] + luma_sse_sum[i * BW + j];
+
+ const double window_error = diff_sse * inv_num_ref_pixels;
+ const int subblock_idx =
+ (i >= block_height / 2) * 2 + (j >= block_width / 2);
+ const double block_error = (double)subblock_mses[subblock_idx];
+ const double combined_error =
+ weight_factor * window_error + block_error * inv_factor;
+ // Compute filter weight.
+ double scaled_error =
+ combined_error * d_factor[subblock_idx] * decay_factor;
+ scaled_error = AOMMIN(scaled_error, 7);
+ const float fweight =
+ approx_exp((float)-scaled_error) * TF_WEIGHT_SCALE;
+ const int weight = iroundpf(fweight);
+ accumulator[k] += weight * pixel_value;
+ count[k] += weight;
+ }
+ }
+ }
+}
+
+void av1_apply_temporal_filter_neon_dotprod(
+ const YV12_BUFFER_CONFIG *frame_to_filter, const MACROBLOCKD *mbd,
+ const BLOCK_SIZE block_size, const int mb_row, const int mb_col,
+ const int num_planes, const double *noise_levels, const MV *subblock_mvs,
+ const int *subblock_mses, const int q_factor, const int filter_strength,
+ int tf_wgt_calc_lvl, const uint8_t *pred, uint32_t *accum,
+ uint16_t *count) {
+ const int is_high_bitdepth = frame_to_filter->flags & YV12_FLAG_HIGHBITDEPTH;
+ assert(block_size == BLOCK_32X32 && "Only support 32x32 block with Neon!");
+ assert(TF_WINDOW_LENGTH == 5 && "Only support window length 5 with Neon!");
+ assert(!is_high_bitdepth && "Only support low bit-depth with Neon!");
+ assert(num_planes >= 1 && num_planes <= MAX_MB_PLANE);
+ (void)is_high_bitdepth;
+
+ // Block information.
+ const int mb_height = block_size_high[block_size];
+ const int mb_width = block_size_wide[block_size];
+ // Frame information.
+ const int frame_height = frame_to_filter->y_crop_height;
+ const int frame_width = frame_to_filter->y_crop_width;
+ const int min_frame_size = AOMMIN(frame_height, frame_width);
+ // Variables to simplify combined error calculation.
+ const double inv_factor = 1.0 / ((TF_WINDOW_BLOCK_BALANCE_WEIGHT + 1) *
+ TF_SEARCH_ERROR_NORM_WEIGHT);
+ const double weight_factor =
+ (double)TF_WINDOW_BLOCK_BALANCE_WEIGHT * inv_factor;
+ // Adjust filtering based on q.
+ // Larger q -> stronger filtering -> larger weight.
+ // Smaller q -> weaker filtering -> smaller weight.
+ double q_decay = pow((double)q_factor / TF_Q_DECAY_THRESHOLD, 2);
+ q_decay = CLIP(q_decay, 1e-5, 1);
+ if (q_factor >= TF_QINDEX_CUTOFF) {
+ // Max q_factor is 255, therefore the upper bound of q_decay is 8.
+ // We do not need a clip here.
+ q_decay = 0.5 * pow((double)q_factor / 64, 2);
+ }
+ // Smaller strength -> smaller filtering weight.
+ double s_decay = pow((double)filter_strength / TF_STRENGTH_THRESHOLD, 2);
+ s_decay = CLIP(s_decay, 1e-5, 1);
+ double d_factor[4] = { 0 };
+ uint8_t frame_abs_diff[SSE_STRIDE * BH] = { 0 };
+ uint32_t luma_sse_sum[BW * BH] = { 0 };
+
+ for (int subblock_idx = 0; subblock_idx < 4; subblock_idx++) {
+ // Larger motion vector -> smaller filtering weight.
+ const MV mv = subblock_mvs[subblock_idx];
+ const double distance = sqrt(pow(mv.row, 2) + pow(mv.col, 2));
+ double distance_threshold = min_frame_size * TF_SEARCH_DISTANCE_THRESHOLD;
+ distance_threshold = AOMMAX(distance_threshold, 1);
+ d_factor[subblock_idx] = distance / distance_threshold;
+ d_factor[subblock_idx] = AOMMAX(d_factor[subblock_idx], 1);
+ }
+
+ // Handle planes in sequence.
+ int plane_offset = 0;
+ for (int plane = 0; plane < num_planes; ++plane) {
+ const uint32_t plane_h = mb_height >> mbd->plane[plane].subsampling_y;
+ const uint32_t plane_w = mb_width >> mbd->plane[plane].subsampling_x;
+ const uint32_t frame_stride =
+ frame_to_filter->strides[plane == AOM_PLANE_Y ? 0 : 1];
+ const int frame_offset = mb_row * plane_h * frame_stride + mb_col * plane_w;
+
+ const uint8_t *ref = frame_to_filter->buffers[plane] + frame_offset;
+ const int ss_x_shift =
+ mbd->plane[plane].subsampling_x - mbd->plane[AOM_PLANE_Y].subsampling_x;
+ const int ss_y_shift =
+ mbd->plane[plane].subsampling_y - mbd->plane[AOM_PLANE_Y].subsampling_y;
+ const int num_ref_pixels = TF_WINDOW_LENGTH * TF_WINDOW_LENGTH +
+ ((plane) ? (1 << (ss_x_shift + ss_y_shift)) : 0);
+ const double inv_num_ref_pixels = 1.0 / num_ref_pixels;
+ // Larger noise -> larger filtering weight.
+ const double n_decay = 0.5 + log(2 * noise_levels[plane] + 5.0);
+ // Decay factors for non-local mean approach.
+ const double decay_factor = 1 / (n_decay * q_decay * s_decay);
+
+ // Filter U-plane and V-plane using Y-plane. This is because motion
+ // search is only done on Y-plane, so the information from Y-plane
+ // will be more accurate. The luma sse sum is reused in both chroma
+ // planes.
+ if (plane == AOM_PLANE_U) {
+ for (unsigned int i = 0; i < plane_h; i++) {
+ for (unsigned int j = 0; j < plane_w; j++) {
+ for (int ii = 0; ii < (1 << ss_y_shift); ++ii) {
+ for (int jj = 0; jj < (1 << ss_x_shift); ++jj) {
+ const int yy = (i << ss_y_shift) + ii; // Y-coord on Y-plane.
+ const int xx = (j << ss_x_shift) + jj; // X-coord on Y-plane.
+ luma_sse_sum[i * BW + j] +=
+ (frame_abs_diff[yy * SSE_STRIDE + xx + 2] *
+ frame_abs_diff[yy * SSE_STRIDE + xx + 2]);
+ }
+ }
+ }
+ }
+ }
+
+ get_abs_diff(ref, frame_stride, pred + plane_offset, plane_w, plane_w,
+ plane_h, frame_abs_diff, SSE_STRIDE);
+
+ apply_temporal_filter(pred + plane_offset, plane_w, plane_w, plane_h,
+ subblock_mses, accum + plane_offset,
+ count + plane_offset, frame_abs_diff, luma_sse_sum,
+ inv_num_ref_pixels, decay_factor, inv_factor,
+ weight_factor, d_factor, tf_wgt_calc_lvl);
+
+ plane_offset += plane_h * plane_w;
+ }
+}
diff --git a/av1/encoder/arm/neon/txfm_neon.h b/av1/encoder/arm/neon/txfm_neon.h
new file mode 100644
index 0000000..635364f
--- /dev/null
+++ b/av1/encoder/arm/neon/txfm_neon.h
@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#ifndef AOM_AV1_ENCODER_ARM_NEON_TXFM_NEON_H_
+#define AOM_AV1_ENCODER_ARM_NEON_TXFM_NEON_H_
+
+#include "aom/aom_integer.h" // For AOM_INLINE.
+
+static AOM_INLINE void ud_adjust_input_and_stride(int ud_flip,
+ const int16_t **input,
+ int *stride, int out_size) {
+ if (ud_flip) {
+ *input = *input + (out_size - 1) * *stride;
+ *stride = -*stride;
+ }
+}
+
+#endif // AOM_AV1_ENCODER_ARM_NEON_TXFM_NEON_H_
diff --git a/av1/encoder/arm/neon/wedge_utils_neon.c b/av1/encoder/arm/neon/wedge_utils_neon.c
index 54d8d19..1b35269 100644
--- a/av1/encoder/arm/neon/wedge_utils_neon.c
+++ b/av1/encoder/arm/neon/wedge_utils_neon.c
@@ -75,3 +75,57 @@
uint64_t csse = horizontal_add_u64x2(vaddq_u64(v_csse[0], v_csse[1]));
return ROUND_POWER_OF_TWO(csse, 2 * WEDGE_WEIGHT_BITS);
}
+
+int8_t av1_wedge_sign_from_residuals_neon(const int16_t *ds, const uint8_t *m,
+ int N, int64_t limit) {
+ int32x4_t acc[4] = { vdupq_n_s32(0), vdupq_n_s32(0), vdupq_n_s32(0),
+ vdupq_n_s32(0) };
+
+ do {
+ int16x8_t ds_l = vld1q_s16(ds);
+ int16x8_t ds_h = vld1q_s16(ds + 8);
+
+ int8x16_t m_s8 = vreinterpretq_s8_u8(vld1q_u8(m));
+ int16x8_t m_l = vmovl_s8(vget_low_s8(m_s8));
+ int16x8_t m_h = vmovl_s8(vget_high_s8(m_s8));
+
+ acc[0] = vmlal_s16(acc[0], vget_low_s16(ds_l), vget_low_s16(m_l));
+ acc[1] = vmlal_s16(acc[1], vget_high_s16(ds_l), vget_high_s16(m_l));
+ acc[2] = vmlal_s16(acc[2], vget_low_s16(ds_h), vget_low_s16(m_h));
+ acc[3] = vmlal_s16(acc[3], vget_high_s16(ds_h), vget_high_s16(m_h));
+
+ ds += 16;
+ m += 16;
+ N -= 16;
+ } while (N != 0);
+
+ int64x2_t sum = vpaddlq_s32(acc[0]);
+ sum = vpadalq_s32(sum, acc[1]);
+ sum = vpadalq_s32(sum, acc[2]);
+ sum = vpadalq_s32(sum, acc[3]);
+
+ return (horizontal_add_s64x2(sum) > limit);
+}
+
+void av1_wedge_compute_delta_squares_neon(int16_t *d_ptr, const int16_t *a_ptr,
+ const int16_t *b_ptr, int N) {
+ do {
+ int16x8_t a = vld1q_s16(a_ptr);
+ int16x8_t b = vld1q_s16(b_ptr);
+
+ int32x4_t sq_lo = vmull_s16(vget_low_s16(a), vget_low_s16(a));
+ int32x4_t sq_hi = vmull_s16(vget_high_s16(a), vget_high_s16(a));
+
+ sq_lo = vmlsl_s16(sq_lo, vget_low_s16(b), vget_low_s16(b));
+ sq_hi = vmlsl_s16(sq_hi, vget_high_s16(b), vget_high_s16(b));
+
+ int16x8_t res = vcombine_s16(vqmovn_s32(sq_lo), vqmovn_s32(sq_hi));
+
+ vst1q_s16(d_ptr, res);
+
+ d_ptr += 8;
+ a_ptr += 8;
+ b_ptr += 8;
+ N -= 8;
+ } while (N != 0);
+}
diff --git a/av1/encoder/av1_noise_estimate.c b/av1/encoder/av1_noise_estimate.c
index 4419085..25007bb 100644
--- a/av1/encoder/av1_noise_estimate.c
+++ b/av1/encoder/av1_noise_estimate.c
@@ -34,18 +34,19 @@
#endif
void av1_noise_estimate_init(NOISE_ESTIMATE *const ne, int width, int height) {
+ const int64_t area = (int64_t)width * height;
ne->enabled = 0;
- ne->level = (width * height < 1280 * 720) ? kLowLow : kLow;
+ ne->level = (area < 1280 * 720) ? kLowLow : kLow;
ne->value = 0;
ne->count = 0;
ne->thresh = 90;
ne->last_w = 0;
ne->last_h = 0;
- if (width * height >= 1920 * 1080) {
+ if (area >= 1920 * 1080) {
ne->thresh = 200;
- } else if (width * height >= 1280 * 720) {
+ } else if (area >= 1280 * 720) {
ne->thresh = 140;
- } else if (width * height >= 640 * 360) {
+ } else if (area >= 640 * 360) {
ne->thresh = 115;
}
ne->num_frames_estimate = 15;
@@ -171,7 +172,7 @@
unsigned int max_bin = 0;
unsigned int max_bin_count = 0;
unsigned int bin_cnt;
- int bsize = BLOCK_16X16;
+ BLOCK_SIZE bsize = BLOCK_16X16;
// Loop over sub-sample of 16x16 blocks of frame, and for blocks that have
// been encoded as zero/small mv at least x consecutive frames, compute
// the variance to update estimate of noise in the source.
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 39aa027..a9e7978 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -43,6 +43,7 @@
#include "av1/encoder/ethread.h"
#include "av1/encoder/mcomp.h"
#include "av1/encoder/palette.h"
+#include "av1/encoder/pickrst.h"
#include "av1/encoder/segmentation.h"
#include "av1/encoder/tokenize.h"
@@ -64,7 +65,7 @@
#if !CONFIG_REALTIME_ONLY
static AOM_INLINE void loop_restoration_write_sb_coeffs(
- const AV1_COMMON *const cm, MACROBLOCKD *xd, const RestorationUnitInfo *rui,
+ const AV1_COMMON *const cm, MACROBLOCKD *xd, int runit_idx,
aom_writer *const w, int plane, FRAME_COUNTS *counts);
#endif
@@ -1027,9 +1028,10 @@
write_intra_uv_mode(ec_ctx, uv_mode, mode, is_cfl_allowed(xd), w);
if (uv_mode == UV_CFL_PRED)
write_cfl_alphas(ec_ctx, mbmi->cfl_alpha_idx, mbmi->cfl_alpha_signs, w);
- if (use_angle_delta && av1_is_directional_mode(get_uv_mode(uv_mode))) {
+ const PREDICTION_MODE intra_mode = get_uv_mode(uv_mode);
+ if (use_angle_delta && av1_is_directional_mode(intra_mode)) {
write_angle_delta(w, mbmi->angle_delta[PLANE_TYPE_UV],
- ec_ctx->angle_delta_cdf[uv_mode - V_PRED]);
+ ec_ctx->angle_delta_cdf[intra_mode - V_PRED]);
}
}
@@ -1621,15 +1623,18 @@
const int num_planes = av1_num_planes(cm);
for (int plane = 0; plane < num_planes; ++plane) {
int rcol0, rcol1, rrow0, rrow1;
+
+ // Skip some unnecessary work if loop restoration is disabled
+ if (cm->rst_info[plane].frame_restoration_type == RESTORE_NONE) continue;
+
if (av1_loop_restoration_corners_in_sb(cm, plane, mi_row, mi_col, bsize,
&rcol0, &rcol1, &rrow0, &rrow1)) {
- const int rstride = cm->rst_info[plane].horz_units_per_tile;
+ const int rstride = cm->rst_info[plane].horz_units;
for (int rrow = rrow0; rrow < rrow1; ++rrow) {
for (int rcol = rcol0; rcol < rcol1; ++rcol) {
const int runit_idx = rcol + rrow * rstride;
- const RestorationUnitInfo *rui =
- &cm->rst_info[plane].unit_info[runit_idx];
- loop_restoration_write_sb_coeffs(cm, xd, rui, w, plane, td->counts);
+ loop_restoration_write_sb_coeffs(cm, xd, runit_idx, w, plane,
+ td->counts);
}
}
}
@@ -1913,8 +1918,9 @@
}
static AOM_INLINE void loop_restoration_write_sb_coeffs(
- const AV1_COMMON *const cm, MACROBLOCKD *xd, const RestorationUnitInfo *rui,
+ const AV1_COMMON *const cm, MACROBLOCKD *xd, int runit_idx,
aom_writer *const w, int plane, FRAME_COUNTS *counts) {
+ const RestorationUnitInfo *rui = &cm->rst_info[plane].unit_info[runit_idx];
const RestorationInfo *rsi = cm->rst_info + plane;
RestorationType frame_rtype = rsi->frame_restoration_type;
assert(frame_rtype != RESTORE_NONE);
@@ -1935,9 +1941,21 @@
#endif
switch (unit_rtype) {
case RESTORE_WIENER:
+#if DEBUG_LR_COSTING
+ assert(!memcmp(
+ ref_wiener_info,
+ &lr_ref_params[RESTORE_SWITCHABLE][plane][runit_idx].wiener_info,
+ sizeof(*ref_wiener_info)));
+#endif
write_wiener_filter(wiener_win, &rui->wiener_info, ref_wiener_info, w);
break;
case RESTORE_SGRPROJ:
+#if DEBUG_LR_COSTING
+ assert(!memcmp(&ref_sgrproj_info->xqd,
+ &lr_ref_params[RESTORE_SWITCHABLE][plane][runit_idx]
+ .sgrproj_info.xqd,
+ sizeof(ref_sgrproj_info->xqd)));
+#endif
write_sgrproj_filter(&rui->sgrproj_info, ref_sgrproj_info, w);
break;
default: assert(unit_rtype == RESTORE_NONE); break;
@@ -1949,6 +1967,12 @@
++counts->wiener_restore[unit_rtype != RESTORE_NONE];
#endif
if (unit_rtype != RESTORE_NONE) {
+#if DEBUG_LR_COSTING
+ assert(
+ !memcmp(ref_wiener_info,
+ &lr_ref_params[RESTORE_WIENER][plane][runit_idx].wiener_info,
+ sizeof(*ref_wiener_info)));
+#endif
write_wiener_filter(wiener_win, &rui->wiener_info, ref_wiener_info, w);
}
} else if (frame_rtype == RESTORE_SGRPROJ) {
@@ -1958,6 +1982,12 @@
++counts->sgrproj_restore[unit_rtype != RESTORE_NONE];
#endif
if (unit_rtype != RESTORE_NONE) {
+#if DEBUG_LR_COSTING
+ assert(!memcmp(
+ &ref_sgrproj_info->xqd,
+ &lr_ref_params[RESTORE_SGRPROJ][plane][runit_idx].sgrproj_info.xqd,
+ sizeof(ref_sgrproj_info->xqd)));
+#endif
write_sgrproj_filter(&rui->sgrproj_info, ref_sgrproj_info, w);
}
}
@@ -3335,7 +3365,7 @@
aom_wb_write_literal(&wb, 0, 1); // forbidden bit.
aom_wb_write_literal(&wb, (int)obu_type, 4);
aom_wb_write_literal(&wb, obu_extension ? 1 : 0, 1);
- aom_wb_write_literal(&wb, 1, 1); // obu_has_payload_length_field
+ aom_wb_write_literal(&wb, 1, 1); // obu_has_size_field
aom_wb_write_literal(&wb, 0, 1); // reserved
if (obu_extension) {
diff --git a/av1/encoder/bitstream.h b/av1/encoder/bitstream.h
index 5999f9e..12e8a63 100644
--- a/av1/encoder/bitstream.h
+++ b/av1/encoder/bitstream.h
@@ -74,6 +74,9 @@
// Index of next job to be processed.
int next_job_idx;
+ // Initialized to false, set to true by the worker thread that encounters an
+ // error in order to abort the processing of other worker threads.
+ bool pack_bs_mt_exit;
} AV1EncPackBSSync;
/*!\endcond */
diff --git a/av1/encoder/block.h b/av1/encoder/block.h
index 360b9d4..33d2d8c 100644
--- a/av1/encoder/block.h
+++ b/av1/encoder/block.h
@@ -1322,6 +1322,12 @@
uint8_t color_sensitivity_sb_alt[MAX_MB_PLANE - 1];
//! Color sensitivity flag for the coding block.
uint8_t color_sensitivity[MAX_MB_PLANE - 1];
+ //! Coding block distortion value for uv/color, minimum over the inter modes.
+ int64_t min_dist_inter_uv;
+
+ //! The buffer used by search_tx_type() to swap dqcoeff in macroblockd_plane
+ // so we can keep dqcoeff of the best tx_type.
+ tran_low_t *dqcoeff_buf;
/**@}*/
/*****************************************************************************
@@ -1330,6 +1336,18 @@
/**@{*/
//! Variance of the source frame.
unsigned int source_variance;
+ //! Flag to indicate coding block is zero sad.
+ int block_is_zero_sad;
+ //! Flag to indicate superblock ME in variance partition is determined to be
+ // good/reliable, and so the superblock MV will be tested in the
+ // nonrd_pickmode. This is only used for LAST_FRAME.
+ int sb_me_partition;
+ //! Flag to indicate to test the superblock MV for the coding block in the
+ // nonrd_pickmode.
+ int sb_me_block;
+ //! Motion vector from superblock MV derived from int_pro_motion() in
+ // the variance_partitioning.
+ int_mv sb_me_mv;
//! SSE of the current predictor.
unsigned int pred_sse[REF_FRAMES];
//! Prediction for ML based partition.
@@ -1366,6 +1384,23 @@
* fast encoding stage for screen content tool detemination.
*/
int palette_pixels;
+
+ /*!\brief Pointer to the structure which stores the statistics used by
+ * sb-level multi-pass encoding.
+ */
+ struct SB_FIRST_PASS_STATS *sb_stats_cache;
+
+ /*!\brief Pointer to the structure which stores the statistics used by
+ * first-pass when superblock is searched twice consecutively.
+ */
+ struct SB_FIRST_PASS_STATS *sb_fp_stats;
+
+#if CONFIG_PARTITION_SEARCH_ORDER
+ /*!\brief Pointer to RD_STATS structure to be used in
+ * av1_rd_partition_search().
+ */
+ RD_STATS *rdcost;
+#endif // CONFIG_PARTITION_SEARCH_ORDER
} MACROBLOCK;
#undef SINGLE_REF_MODES
diff --git a/av1/encoder/compound_type.c b/av1/encoder/compound_type.c
index 1992f23..3b0ee88 100644
--- a/av1/encoder/compound_type.c
+++ b/av1/encoder/compound_type.c
@@ -1024,8 +1024,9 @@
int64_t ref_skip_rd, int mode_rate) {
int eval_txfm = 1;
const int txfm_rd_gate_level =
- get_txfm_rd_gate_level(cpi->sf.inter_sf.txfm_rd_gate_level, bsize,
- TX_SEARCH_DEFAULT, /*eval_motion_mode=*/0);
+ get_txfm_rd_gate_level(cpi->common.seq_params->enable_masked_compound,
+ cpi->sf.inter_sf.txfm_rd_gate_level, bsize,
+ TX_SEARCH_COMP_TYPE_MODE, /*eval_motion_mode=*/0);
// Check if the mode is good enough based on skip rd
if (txfm_rd_gate_level) {
int64_t sse_y = compute_sse_plane(x, xd, PLANE_TYPE_Y, bsize);
@@ -1108,8 +1109,9 @@
// TODO(nithya): Handle wedge_newmv_search if extending for lower speed
// setting
const int txfm_rd_gate_level =
- get_txfm_rd_gate_level(cpi->sf.inter_sf.txfm_rd_gate_level, bsize,
- TX_SEARCH_DEFAULT, /*eval_motion_mode=*/0);
+ get_txfm_rd_gate_level(cm->seq_params->enable_masked_compound,
+ cpi->sf.inter_sf.txfm_rd_gate_level, bsize,
+ TX_SEARCH_COMP_TYPE_MODE, /*eval_motion_mode=*/0);
if (txfm_rd_gate_level) {
int eval_txfm = check_txfm_eval(x, bsize, ref_skip_rd, skip_rd_cur,
txfm_rd_gate_level, 1);
diff --git a/av1/encoder/context_tree.c b/av1/encoder/context_tree.c
index 2bd2d7f..7b8240d 100644
--- a/av1/encoder/context_tree.c
+++ b/av1/encoder/context_tree.c
@@ -304,8 +304,6 @@
}
void av1_free_sms_tree(ThreadData *td) {
- if (td->sms_tree != NULL) {
- aom_free(td->sms_tree);
- td->sms_tree = NULL;
- }
+ aom_free(td->sms_tree);
+ td->sms_tree = NULL;
}
diff --git a/av1/encoder/encode_strategy.c b/av1/encoder/encode_strategy.c
index 90279b0..878cec5 100644
--- a/av1/encoder/encode_strategy.c
+++ b/av1/encoder/encode_strategy.c
@@ -724,6 +724,7 @@
#endif
const AV1EncoderConfig *const oxcf = &cpi->oxcf;
AV1_COMMON *const cm = &cpi->common;
+
GF_GROUP *const gf_group = &cpi->ppi->gf_group;
FRAME_UPDATE_TYPE update_type =
get_frame_update_type(&cpi->ppi->gf_group, cpi->gf_frame_index);
@@ -1658,6 +1659,15 @@
cm->quant_params.using_qmatrix = oxcf->q_cfg.using_qm;
}
+ const int is_intra_frame = frame_params.frame_type == KEY_FRAME ||
+ frame_params.frame_type == INTRA_ONLY_FRAME;
+ FeatureFlags *const features = &cm->features;
+ if (!is_stat_generation_stage(cpi) &&
+ (oxcf->pass == AOM_RC_ONE_PASS || oxcf->pass >= AOM_RC_SECOND_PASS) &&
+ is_intra_frame) {
+ av1_set_screen_content_options(cpi, features);
+ }
+
#if CONFIG_REALTIME_ONLY
if (av1_encode(cpi, dest, &frame_input, &frame_params, &frame_results) !=
AOM_CODEC_OK) {
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 50f046d..2c6e49f 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -523,7 +523,7 @@
MB_MODE_INFO **mi = cm->mi_params.mi_grid_base +
get_mi_grid_idx(&cm->mi_params, mi_row, mi_col);
const BLOCK_SIZE sb_size = cm->seq_params->sb_size;
- PC_TREE *const pc_root = td->rt_pc_root;
+ PC_TREE *const pc_root = td->pc_root;
#if CONFIG_RT_ML_PARTITIONING
if (sf->part_sf.partition_search_type == ML_BASED_PARTITION) {
@@ -731,9 +731,12 @@
av1_restore_sb_state(sb_org_stats, cpi, td, tile_data, mi_row, mi_col);
cm->mi_params.mi_alloc[alloc_mi_idx].current_qindex = backup_current_qindex;
- PC_TREE *const pc_root = av1_alloc_pc_tree_node(bsize);
+ td->pc_root = av1_alloc_pc_tree_node(bsize);
+ if (!td->pc_root)
+ aom_internal_error(x->e_mbd.error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PC_TREE");
av1_rd_pick_partition(cpi, td, tile_data, tp, mi_row, mi_col, bsize,
- &cur_rdc, cur_rdc, pc_root, sms_tree, NULL,
+ &cur_rdc, cur_rdc, td->pc_root, sms_tree, NULL,
SB_DRY_PASS, NULL);
if ((rdc_winner.rdcost > cur_rdc.rdcost) ||
@@ -760,6 +763,7 @@
const int seg_skip) {
AV1_COMMON *const cm = &cpi->common;
MACROBLOCK *const x = &td->mb;
+ MACROBLOCKD *const xd = &x->e_mbd;
const SPEED_FEATURES *const sf = &cpi->sf;
const TileInfo *const tile_info = &tile_data->tile_info;
MB_MODE_INFO **mi = cm->mi_params.mi_grid_base +
@@ -787,11 +791,15 @@
#if CONFIG_COLLECT_COMPONENT_TIMING
start_timing(cpi, rd_use_partition_time);
#endif
- PC_TREE *const pc_root = av1_alloc_pc_tree_node(sb_size);
+ td->pc_root = av1_alloc_pc_tree_node(sb_size);
+ if (!td->pc_root)
+ aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PC_TREE");
av1_rd_use_partition(cpi, td, tile_data, mi, tp, mi_row, mi_col, sb_size,
- &dummy_rate, &dummy_dist, 1, pc_root);
- av1_free_pc_tree_recursive(pc_root, num_planes, 0, 0,
+ &dummy_rate, &dummy_dist, 1, td->pc_root);
+ av1_free_pc_tree_recursive(td->pc_root, num_planes, 0, 0,
sf->part_sf.partition_search_type);
+ td->pc_root = NULL;
#if CONFIG_COLLECT_COMPONENT_TIMING
end_timing(cpi, rd_use_partition_time);
#endif
@@ -803,20 +811,16 @@
const BLOCK_SIZE bsize =
seg_skip ? sb_size : sf->part_sf.fixed_partition_size;
av1_set_fixed_partitioning(cpi, tile_info, mi, mi_row, mi_col, bsize);
- PC_TREE *const pc_root = av1_alloc_pc_tree_node(sb_size);
+ td->pc_root = av1_alloc_pc_tree_node(sb_size);
+ if (!td->pc_root)
+ aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PC_TREE");
av1_rd_use_partition(cpi, td, tile_data, mi, tp, mi_row, mi_col, sb_size,
- &dummy_rate, &dummy_dist, 1, pc_root);
- av1_free_pc_tree_recursive(pc_root, num_planes, 0, 0,
+ &dummy_rate, &dummy_dist, 1, td->pc_root);
+ av1_free_pc_tree_recursive(td->pc_root, num_planes, 0, 0,
sf->part_sf.partition_search_type);
+ td->pc_root = NULL;
} else {
- SB_FIRST_PASS_STATS *sb_org_stats = NULL;
-
- if (cpi->oxcf.sb_qp_sweep) {
- CHECK_MEM_ERROR(
- cm, sb_org_stats,
- (SB_FIRST_PASS_STATS *)aom_malloc(sizeof(SB_FIRST_PASS_STATS)));
- av1_backup_sb_state(sb_org_stats, cpi, td, tile_data, mi_row, mi_col);
- }
// The most exhaustive recursive partition search
SuperBlockEnc *sb_enc = &x->sb_enc;
// No stats for overlay frames. Exclude key frame.
@@ -843,12 +847,16 @@
!(has_no_stats_stage(cpi) && cpi->oxcf.mode == REALTIME &&
cpi->oxcf.gf_cfg.lag_in_frames == 0) &&
cm->delta_q_info.delta_q_present_flag) {
+ AOM_CHECK_MEM_ERROR(
+ x->e_mbd.error_info, td->mb.sb_stats_cache,
+ (SB_FIRST_PASS_STATS *)aom_malloc(sizeof(*td->mb.sb_stats_cache)));
+ av1_backup_sb_state(td->mb.sb_stats_cache, cpi, td, tile_data, mi_row,
+ mi_col);
assert(x->rdmult_delta_qindex == x->delta_qindex);
- assert(sb_org_stats);
const int best_qp_diff =
sb_qp_sweep(cpi, td, tile_data, tp, mi_row, mi_col, sb_size, sms_root,
- sb_org_stats) -
+ td->mb.sb_stats_cache) -
x->rdmult_delta_qindex;
sb_qp_sweep_init_quantizers(cpi, td, tile_data, sms_root, &dummy_rdc,
@@ -859,10 +867,13 @@
cm->mi_params.mi_alloc[alloc_mi_idx].current_qindex;
av1_reset_mbmi(&cm->mi_params, sb_size, mi_row, mi_col);
- av1_restore_sb_state(sb_org_stats, cpi, td, tile_data, mi_row, mi_col);
+ av1_restore_sb_state(td->mb.sb_stats_cache, cpi, td, tile_data, mi_row,
+ mi_col);
cm->mi_params.mi_alloc[alloc_mi_idx].current_qindex =
backup_current_qindex;
+ aom_free(td->mb.sb_stats_cache);
+ td->mb.sb_stats_cache = NULL;
}
if (num_passes == 1) {
#if CONFIG_PARTITION_SEARCH_ORDER
@@ -873,24 +884,36 @@
av1_rd_partition_search(cpi, td, tile_data, tp, sms_root, mi_row,
mi_col, sb_size, &this_rdc);
} else {
- PC_TREE *const pc_root = av1_alloc_pc_tree_node(sb_size);
+ td->pc_root = av1_alloc_pc_tree_node(sb_size);
+ if (!td->pc_root)
+ aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PC_TREE");
av1_rd_pick_partition(cpi, td, tile_data, tp, mi_row, mi_col, sb_size,
- &dummy_rdc, dummy_rdc, pc_root, sms_root, NULL,
- SB_SINGLE_PASS, NULL);
+ &dummy_rdc, dummy_rdc, td->pc_root, sms_root,
+ NULL, SB_SINGLE_PASS, NULL);
}
#else
- PC_TREE *const pc_root = av1_alloc_pc_tree_node(sb_size);
+ td->pc_root = av1_alloc_pc_tree_node(sb_size);
+ if (!td->pc_root)
+ aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PC_TREE");
av1_rd_pick_partition(cpi, td, tile_data, tp, mi_row, mi_col, sb_size,
- &dummy_rdc, dummy_rdc, pc_root, sms_root, NULL,
+ &dummy_rdc, dummy_rdc, td->pc_root, sms_root, NULL,
SB_SINGLE_PASS, NULL);
#endif // CONFIG_PARTITION_SEARCH_ORDER
} else {
// First pass
- SB_FIRST_PASS_STATS sb_fp_stats;
- av1_backup_sb_state(&sb_fp_stats, cpi, td, tile_data, mi_row, mi_col);
- PC_TREE *const pc_root_p0 = av1_alloc_pc_tree_node(sb_size);
+ AOM_CHECK_MEM_ERROR(
+ x->e_mbd.error_info, td->mb.sb_fp_stats,
+ (SB_FIRST_PASS_STATS *)aom_malloc(sizeof(*td->mb.sb_fp_stats)));
+ av1_backup_sb_state(td->mb.sb_fp_stats, cpi, td, tile_data, mi_row,
+ mi_col);
+ td->pc_root = av1_alloc_pc_tree_node(sb_size);
+ if (!td->pc_root)
+ aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PC_TREE");
av1_rd_pick_partition(cpi, td, tile_data, tp, mi_row, mi_col, sb_size,
- &dummy_rdc, dummy_rdc, pc_root_p0, sms_root, NULL,
+ &dummy_rdc, dummy_rdc, td->pc_root, sms_root, NULL,
SB_DRY_PASS, NULL);
// Second pass
@@ -899,14 +922,19 @@
av1_reset_mbmi(&cm->mi_params, sb_size, mi_row, mi_col);
av1_reset_simple_motion_tree_partition(sms_root, sb_size);
- av1_restore_sb_state(&sb_fp_stats, cpi, td, tile_data, mi_row, mi_col);
+ av1_restore_sb_state(td->mb.sb_fp_stats, cpi, td, tile_data, mi_row,
+ mi_col);
- PC_TREE *const pc_root_p1 = av1_alloc_pc_tree_node(sb_size);
+ td->pc_root = av1_alloc_pc_tree_node(sb_size);
+ if (!td->pc_root)
+ aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PC_TREE");
av1_rd_pick_partition(cpi, td, tile_data, tp, mi_row, mi_col, sb_size,
- &dummy_rdc, dummy_rdc, pc_root_p1, sms_root, NULL,
+ &dummy_rdc, dummy_rdc, td->pc_root, sms_root, NULL,
SB_WET_PASS, NULL);
+ aom_free(td->mb.sb_fp_stats);
+ td->mb.sb_fp_stats = NULL;
}
- aom_free(sb_org_stats);
// Reset to 0 so that it wouldn't be used elsewhere mistakenly.
sb_enc->tpl_data_count = 0;
@@ -1124,6 +1152,17 @@
// top-right superblock to finish encoding.
enc_row_mt->sync_read_ptr(
row_mt_sync, sb_row, sb_col_in_tile - delay_wait_for_top_right_sb(cpi));
+
+#if CONFIG_MULTITHREAD
+ if (row_mt_enabled) {
+ pthread_mutex_lock(enc_row_mt->mutex_);
+ const bool row_mt_exit = enc_row_mt->row_mt_exit;
+ pthread_mutex_unlock(enc_row_mt->mutex_);
+ // Exit in case any worker has encountered an error.
+ if (row_mt_exit) return;
+ }
+#endif
+
const int update_cdf = tile_data->allow_update_cdf && row_mt_enabled;
if (update_cdf && (tile_info->mi_row_start != mi_row)) {
if ((tile_info->mi_col_start == mi_col)) {
@@ -1155,6 +1194,9 @@
x->content_state_sb.lighting_change = 0;
x->content_state_sb.low_sumdiff = 0;
x->force_zeromv_skip_for_sb = 0;
+ x->sb_me_block = 0;
+ x->sb_me_partition = 0;
+ x->sb_me_mv.as_int = 0;
if (cpi->oxcf.mode == ALLINTRA) {
x->intra_sb_rdmult_modifier = 128;
@@ -1230,7 +1272,7 @@
av1_row_mt_mem_dealloc(cpi);
- if (cpi->tile_data != NULL) aom_free(cpi->tile_data);
+ aom_free(cpi->tile_data);
CHECK_MEM_ERROR(
cm, cpi->tile_data,
aom_memalign(32, tile_cols * tile_rows * sizeof(*cpi->tile_data)));
@@ -1441,7 +1483,7 @@
}
}
- av1_dealloc_mb_data(cm, mb);
+ av1_dealloc_mb_data(mb, av1_num_planes(cm));
}
// Set the relative distance of a reference frame w.r.t. current frame
@@ -1670,6 +1712,19 @@
}
}
+static void free_block_hash_buffers(uint32_t *block_hash_values[2][2],
+ int8_t *is_block_same[2][3]) {
+ for (int k = 0; k < 2; ++k) {
+ for (int j = 0; j < 2; ++j) {
+ aom_free(block_hash_values[k][j]);
+ }
+
+ for (int j = 0; j < 3; ++j) {
+ aom_free(is_block_same[k][j]);
+ }
+ }
+}
+
/*!\brief Encoder setup(only for the current frame), encoding, and recontruction
* for a single frame
*
@@ -1740,26 +1795,34 @@
// add to hash table
const int pic_width = cpi->source->y_crop_width;
const int pic_height = cpi->source->y_crop_height;
- uint32_t *block_hash_values[2][2];
- int8_t *is_block_same[2][3];
+ uint32_t *block_hash_values[2][2] = { { NULL } };
+ int8_t *is_block_same[2][3] = { { NULL } };
int k, j;
+ bool error = false;
- for (k = 0; k < 2; k++) {
- for (j = 0; j < 2; j++) {
- CHECK_MEM_ERROR(cm, block_hash_values[k][j],
- aom_malloc(sizeof(uint32_t) * pic_width * pic_height));
+ for (k = 0; k < 2 && !error; ++k) {
+ for (j = 0; j < 2; ++j) {
+ block_hash_values[k][j] = (uint32_t *)aom_malloc(
+ sizeof(*block_hash_values[0][0]) * pic_width * pic_height);
+ if (!block_hash_values[k][j]) {
+ error = true;
+ break;
+ }
}
- for (j = 0; j < 3; j++) {
- CHECK_MEM_ERROR(cm, is_block_same[k][j],
- aom_malloc(sizeof(int8_t) * pic_width * pic_height));
+ for (j = 0; j < 3 && !error; ++j) {
+ is_block_same[k][j] = (int8_t *)aom_malloc(
+ sizeof(*is_block_same[0][0]) * pic_width * pic_height);
+ if (!is_block_same[k][j]) error = true;
}
}
av1_hash_table_init(intrabc_hash_info);
- if (!av1_hash_table_create(&intrabc_hash_info->intrabc_hash_table)) {
+ if (error ||
+ !av1_hash_table_create(&intrabc_hash_info->intrabc_hash_table)) {
+ free_block_hash_buffers(block_hash_values, is_block_same);
aom_internal_error(cm->error, AOM_CODEC_MEM_ERROR,
- "Error allocating intrabc_hash_table");
+ "Error allocating intrabc_hash_table and buffers");
}
hash_table_created = 1;
av1_generate_block_2x2_hash_value(intrabc_hash_info, cpi->source,
@@ -1769,7 +1832,6 @@
const int max_sb_size =
(1 << (cm->seq_params->mib_size_log2 + MI_SIZE_LOG2));
int src_idx = 0;
- bool error = false;
for (int size = 4; size <= max_sb_size; size *= 2, src_idx = !src_idx) {
const int dst_idx = !src_idx;
av1_generate_block_hash_value(
@@ -1787,15 +1849,7 @@
}
}
- for (k = 0; k < 2; k++) {
- for (j = 0; j < 2; j++) {
- aom_free(block_hash_values[k][j]);
- }
-
- for (j = 0; j < 3; j++) {
- aom_free(is_block_same[k][j]);
- }
- }
+ free_block_hash_buffers(block_hash_values, is_block_same);
if (error) {
aom_internal_error(cm->error, AOM_CODEC_MEM_ERROR,
@@ -1957,12 +2011,19 @@
// Preallocate the pc_tree for realtime coding to reduce the cost of
// memory allocation.
const int use_nonrd_mode = cpi->sf.rt_sf.use_nonrd_pick_mode;
- td->rt_pc_root = use_nonrd_mode
- ? av1_alloc_pc_tree_node(cm->seq_params->sb_size)
- : NULL;
+ if (use_nonrd_mode) {
+ td->pc_root = av1_alloc_pc_tree_node(cm->seq_params->sb_size);
+ if (!td->pc_root)
+ aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PC_TREE");
+ } else {
+ td->pc_root = NULL;
+ }
+
encode_tiles(cpi);
- av1_free_pc_tree_recursive(td->rt_pc_root, av1_num_planes(cm), 0, 0,
+ av1_free_pc_tree_recursive(td->pc_root, av1_num_planes(cm), 0, 0,
cpi->sf.part_sf.partition_search_type);
+ td->pc_root = NULL;
}
}
diff --git a/av1/encoder/encodeframe_utils.c b/av1/encoder/encodeframe_utils.c
index 29d7fe4..94298c8 100644
--- a/av1/encoder/encodeframe_utils.c
+++ b/av1/encoder/encodeframe_utils.c
@@ -22,7 +22,7 @@
const int mi_col, int *const rdmult) {
const AV1_COMMON *const cm = &cpi->common;
- const int bsize_base = BLOCK_16X16;
+ const BLOCK_SIZE bsize_base = BLOCK_16X16;
const int num_mi_w = mi_size_wide[bsize_base];
const int num_mi_h = mi_size_high[bsize_base];
const int num_cols = (cm->mi_params.mi_cols + num_mi_w - 1) / num_mi_w;
@@ -177,7 +177,7 @@
const int block_mi_width_sr =
coded_to_superres_mi(mi_size_wide[bsize], cm->superres_scale_denominator);
- const int bsize_base = BLOCK_16X16;
+ const BLOCK_SIZE bsize_base = BLOCK_16X16;
const int num_mi_w = mi_size_wide[bsize_base];
const int num_mi_h = mi_size_high[bsize_base];
const int num_cols = (mi_cols_sr + num_mi_w - 1) / num_mi_w;
@@ -588,13 +588,13 @@
update_cdf(cdf_v, CFL_IDX_V(idx), CFL_ALPHABET_SIZE);
}
}
- if (av1_is_directional_mode(get_uv_mode(uv_mode)) &&
- av1_use_angle_delta(bsize)) {
+ const PREDICTION_MODE intra_mode = get_uv_mode(uv_mode);
+ if (av1_is_directional_mode(intra_mode) && av1_use_angle_delta(bsize)) {
#if CONFIG_ENTROPY_STATS
- ++counts->angle_delta[uv_mode - UV_V_PRED]
+ ++counts->angle_delta[intra_mode - V_PRED]
[mbmi->angle_delta[PLANE_TYPE_UV] + MAX_ANGLE_DELTA];
#endif
- update_cdf(fc->angle_delta_cdf[uv_mode - UV_V_PRED],
+ update_cdf(fc->angle_delta_cdf[intra_mode - V_PRED],
mbmi->angle_delta[PLANE_TYPE_UV] + MAX_ANGLE_DELTA,
2 * MAX_ANGLE_DELTA + 1);
}
@@ -1743,3 +1743,23 @@
default: assert(0);
}
}
+
+void av1_dealloc_src_diff_buf(struct macroblock *mb, int num_planes) {
+ for (int plane = 0; plane < num_planes; ++plane) {
+ aom_free(mb->plane[plane].src_diff);
+ mb->plane[plane].src_diff = NULL;
+ }
+}
+
+void av1_alloc_src_diff_buf(const struct AV1Common *cm, struct macroblock *mb) {
+ const int num_planes = av1_num_planes(cm);
+ for (int plane = 0; plane < num_planes; ++plane) {
+ const int subsampling_xy =
+ plane ? cm->seq_params->subsampling_x + cm->seq_params->subsampling_y
+ : 0;
+ const int sb_size = MAX_SB_SQUARE >> subsampling_xy;
+ CHECK_MEM_ERROR(cm, mb->plane[plane].src_diff,
+ (int16_t *)aom_memalign(
+ 32, sizeof(*mb->plane[plane].src_diff) * sb_size));
+ }
+}
diff --git a/av1/encoder/encodeframe_utils.h b/av1/encoder/encodeframe_utils.h
index 24a36c5..14c71b8 100644
--- a/av1/encoder/encodeframe_utils.h
+++ b/av1/encoder/encodeframe_utils.h
@@ -430,25 +430,26 @@
const TileInfo *const tile_info, const int mi_row,
const int mi_col);
-static AOM_INLINE void av1_dealloc_mb_data(struct AV1Common *cm,
- struct macroblock *mb) {
+void av1_dealloc_src_diff_buf(struct macroblock *mb, int num_planes);
+
+static AOM_INLINE void av1_dealloc_mb_data(struct macroblock *mb,
+ int num_planes) {
aom_free(mb->txfm_search_info.mb_rd_record);
mb->txfm_search_info.mb_rd_record = NULL;
aom_free(mb->inter_modes_info);
mb->inter_modes_info = NULL;
- const int num_planes = av1_num_planes(cm);
- for (int plane = 0; plane < num_planes; plane++) {
- aom_free(mb->plane[plane].src_diff);
- mb->plane[plane].src_diff = NULL;
- }
+ av1_dealloc_src_diff_buf(mb, num_planes);
aom_free(mb->e_mbd.seg_mask);
mb->e_mbd.seg_mask = NULL;
aom_free(mb->winner_mode_stats);
mb->winner_mode_stats = NULL;
+
+ aom_free(mb->dqcoeff_buf);
+ mb->dqcoeff_buf = NULL;
}
static AOM_INLINE void allocate_winner_mode_stats(const AV1_COMP *cpi,
@@ -468,6 +469,8 @@
winner_mode_count * sizeof(mb->winner_mode_stats[0])));
}
+void av1_alloc_src_diff_buf(const struct AV1Common *cm, struct macroblock *mb);
+
static AOM_INLINE void av1_alloc_mb_data(const AV1_COMP *cpi,
struct macroblock *mb) {
const AV1_COMMON *cm = &cpi->common;
@@ -483,21 +486,20 @@
cm, mb->inter_modes_info,
(InterModesInfo *)aom_malloc(sizeof(*mb->inter_modes_info)));
}
- const int num_planes = av1_num_planes(cm);
- for (int plane = 0; plane < num_planes; plane++) {
- const int subsampling_xy =
- plane ? cm->seq_params->subsampling_x + cm->seq_params->subsampling_y
- : 0;
- const int sb_size = MAX_SB_SQUARE >> subsampling_xy;
- CHECK_MEM_ERROR(cm, mb->plane[plane].src_diff,
- (int16_t *)aom_memalign(
- 32, sizeof(*mb->plane[plane].src_diff) * sb_size));
- }
+
+ av1_alloc_src_diff_buf(cm, mb);
+
CHECK_MEM_ERROR(cm, mb->e_mbd.seg_mask,
(uint8_t *)aom_memalign(
16, 2 * MAX_SB_SQUARE * sizeof(mb->e_mbd.seg_mask[0])));
allocate_winner_mode_stats(cpi, mb);
+
+ const int max_sb_square_y = 1
+ << num_pels_log2_lookup[cm->seq_params->sb_size];
+ CHECK_MEM_ERROR(
+ cm, mb->dqcoeff_buf,
+ (tran_low_t *)aom_memalign(32, max_sb_square_y * sizeof(tran_low_t)));
}
// This function will compute the number of reference frames to be disabled
diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c
index 78efa0c..c78761d 100644
--- a/av1/encoder/encodemb.c
+++ b/av1/encoder/encodemb.c
@@ -449,10 +449,16 @@
av1_set_txb_context(x, plane, block, tx_size, a, l);
if (p->eobs[block]) {
- *(args->skip) = 0;
+ // As long as any YUV plane has non-zero quantized transform coefficients,
+ // mbmi->skip_txfm flag is set to 0.
+ mbmi->skip_txfm = 0;
av1_inverse_transform_block(xd, dqcoeff, plane, tx_type, tx_size, dst,
pd->dst.stride, p->eobs[block],
cm->features.reduced_tx_set_used);
+ } else {
+ // Only when YUV planes all have zero quantized transform coefficients,
+ // mbmi->skip_txfm flag is set to 1.
+ mbmi->skip_txfm &= 1;
}
// TODO(debargha, jingning): Temporarily disable txk_type check for eob=0
@@ -650,13 +656,19 @@
assert(bsize < BLOCK_SIZES_ALL);
MACROBLOCKD *const xd = &x->e_mbd;
MB_MODE_INFO *mbmi = xd->mi[0];
+ // In the current encoder implementation, for inter blocks,
+ // only when YUV planes all have zero quantized transform coefficients,
+ // mbmi->skip_txfm flag is set to 1.
+ // For intra blocks, this flag is set to 0 since skipped blocks are so rare
+ // that transmitting skip_txfm = 1 is very expensive.
+ // mbmi->skip_txfm is init to 1, and will be modified in encode_block() based
+ // on transform, quantization, and (if exists) trellis optimization.
mbmi->skip_txfm = 1;
if (x->txfm_search_info.skip_txfm) return;
struct optimize_ctx ctx;
struct encode_b_args arg = {
- cpi, x, &ctx, &mbmi->skip_txfm,
- NULL, NULL, dry_run, cpi->optimize_seg_arr[mbmi->segment_id]
+ cpi, x, &ctx, NULL, NULL, dry_run, cpi->optimize_seg_arr[mbmi->segment_id]
};
const AV1_COMMON *const cm = &cpi->common;
const int num_planes = av1_num_planes(cm);
@@ -727,6 +739,7 @@
const AV1_COMMON *const cm = &cpi->common;
MACROBLOCK *const x = args->x;
MACROBLOCKD *const xd = &x->e_mbd;
+ MB_MODE_INFO *mbmi = xd->mi[0];
struct macroblock_plane *const p = &x->plane[plane];
struct macroblockd_plane *const pd = &xd->plane[plane];
tran_low_t *dqcoeff = p->dqcoeff + BLOCK_OFFSET(block);
@@ -820,9 +833,9 @@
update_txk_array(xd, blk_row, blk_col, tx_size, DCT_DCT);
}
- // For intra mode, skipped blocks are so rare that transmitting skip=1 is
- // very expensive.
- *(args->skip) = 0;
+ // For intra mode, skipped blocks are so rare that transmitting
+ // skip_txfm = 1 is very expensive.
+ mbmi->skip_txfm = 0;
if (plane == AOM_PLANE_Y && xd->cfl.store_y) {
cfl_store_tx(xd, blk_row, blk_col, tx_size, plane_bsize);
@@ -841,8 +854,9 @@
const int ss_y = pd->subsampling_y;
ENTROPY_CONTEXT ta[MAX_MIB_SIZE] = { 0 };
ENTROPY_CONTEXT tl[MAX_MIB_SIZE] = { 0 };
- struct encode_b_args arg = { cpi, x, NULL, &(xd->mi[0]->skip_txfm),
- ta, tl, dry_run, enable_optimize_b };
+ struct encode_b_args arg = {
+ cpi, x, NULL, ta, tl, dry_run, enable_optimize_b
+ };
const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, ss_x, ss_y);
if (enable_optimize_b) {
av1_get_entropy_contexts(plane_bsize, pd, ta, tl);
diff --git a/av1/encoder/encodemb.h b/av1/encoder/encodemb.h
index b819e82..f97bf8f 100644
--- a/av1/encoder/encodemb.h
+++ b/av1/encoder/encodemb.h
@@ -56,7 +56,6 @@
const struct AV1_COMP *cpi;
MACROBLOCK *x;
struct optimize_ctx *ctx;
- uint8_t *skip;
ENTROPY_CONTEXT *ta;
ENTROPY_CONTEXT *tl;
RUN_TYPE dry_run;
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index e1cb49b..550504f 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -236,16 +236,16 @@
size_t encoded_frame_size) {
const int upscaled_width = cm->superres_upscaled_width;
const int height = cm->height;
- const int luma_pic_size = upscaled_width * height;
+ const int64_t luma_pic_size = (int64_t)upscaled_width * height;
const SequenceHeader *const seq_params = cm->seq_params;
const BITSTREAM_PROFILE profile = seq_params->profile;
const int pic_size_profile_factor =
profile == PROFILE_0 ? 15 : (profile == PROFILE_1 ? 30 : 36);
encoded_frame_size =
(encoded_frame_size > 129 ? encoded_frame_size - 128 : 1);
- const size_t uncompressed_frame_size =
+ const int64_t uncompressed_frame_size =
(luma_pic_size * pic_size_profile_factor) >> 3;
- return uncompressed_frame_size / (double)encoded_frame_size;
+ return (double)uncompressed_frame_size / encoded_frame_size;
}
static void auto_tile_size_balancing(AV1_COMMON *const cm, int num_sbs,
@@ -362,9 +362,9 @@
static INLINE int does_level_match(int width, int height, double fps,
int lvl_width, int lvl_height,
double lvl_fps, int lvl_dim_mult) {
- const int64_t lvl_luma_pels = lvl_width * lvl_height;
+ const int64_t lvl_luma_pels = (int64_t)lvl_width * lvl_height;
const double lvl_display_sample_rate = lvl_luma_pels * lvl_fps;
- const int64_t luma_pels = width * height;
+ const int64_t luma_pels = (int64_t)width * height;
const double display_sample_rate = luma_pels * fps;
return luma_pels <= lvl_luma_pels &&
display_sample_rate <= lvl_display_sample_rate &&
@@ -1278,7 +1278,7 @@
enc_set_mb_mi(&mi_params, oxcf->frm_dim_cfg.width, oxcf->frm_dim_cfg.height,
BLOCK_4X4);
- const int bsize = BLOCK_16X16;
+ const BLOCK_SIZE bsize = BLOCK_16X16;
const int w = mi_size_wide[bsize];
const int h = mi_size_high[bsize];
const int num_cols = (mi_params.mi_cols + w - 1) / w;
@@ -1410,6 +1410,8 @@
cm->current_frame.frame_number = 0;
cpi->rc.frame_number_encoded = 0;
cpi->rc.prev_frame_is_dropped = 0;
+ cpi->rc.max_consec_drop = INT_MAX;
+ cpi->rc.drop_count_consec = 0;
cm->current_frame_id = -1;
cpi->tile_data = NULL;
cpi->last_show_frame_buf = NULL;
@@ -1493,9 +1495,10 @@
cpi->mb_weber_stats = NULL;
cpi->mb_delta_q = NULL;
cpi->palette_pixel_num = 0;
+ cpi->scaled_last_source_available = 0;
{
- const int bsize = BLOCK_16X16;
+ const BLOCK_SIZE bsize = BLOCK_16X16;
const int w = mi_size_wide[bsize];
const int h = mi_size_high[bsize];
const int num_cols = (max_mi_cols + w - 1) / w;
@@ -1510,7 +1513,7 @@
#if CONFIG_TUNE_VMAF
{
- const int bsize = BLOCK_64X64;
+ const BLOCK_SIZE bsize = BLOCK_64X64;
const int w = mi_size_wide[bsize];
const int h = mi_size_high[bsize];
const int num_cols = (mi_params->mi_cols + w - 1) / w;
@@ -1699,6 +1702,7 @@
pthread_mutex_t *const enc_row_mt_mutex_ = mt_info->enc_row_mt.mutex_;
pthread_cond_t *const enc_row_mt_cond_ = mt_info->enc_row_mt.cond_;
pthread_mutex_t *const gm_mt_mutex_ = mt_info->gm_sync.mutex_;
+ pthread_mutex_t *const tpl_error_mutex_ = mt_info->tpl_row_mt.mutex_;
pthread_mutex_t *const pack_bs_mt_mutex_ = mt_info->pack_bs_sync.mutex_;
if (enc_row_mt_mutex_ != NULL) {
pthread_mutex_destroy(enc_row_mt_mutex_);
@@ -1712,6 +1716,10 @@
pthread_mutex_destroy(gm_mt_mutex_);
aom_free(gm_mt_mutex_);
}
+ if (tpl_error_mutex_ != NULL) {
+ pthread_mutex_destroy(tpl_error_mutex_);
+ aom_free(tpl_error_mutex_);
+ }
if (pack_bs_mt_mutex_ != NULL) {
pthread_mutex_destroy(pack_bs_mt_mutex_);
aom_free(pack_bs_mt_mutex_);
@@ -1720,11 +1728,11 @@
av1_row_mt_mem_dealloc(cpi);
if (mt_info->num_workers > 1) {
+ av1_row_mt_sync_mem_dealloc(&cpi->ppi->intra_row_mt_sync);
av1_loop_filter_dealloc(&mt_info->lf_row_sync);
av1_cdef_mt_dealloc(&mt_info->cdef_sync);
#if !CONFIG_REALTIME_ONLY
av1_loop_restoration_dealloc(&mt_info->lr_row_sync);
- av1_gm_dealloc(&mt_info->gm_sync);
av1_tf_mt_dealloc(&mt_info->tf_sync);
#endif
}
@@ -1950,6 +1958,7 @@
const int stride = cpi->unfiltered_source->y_stride;
const int width = cpi->unfiltered_source->y_width;
const int height = cpi->unfiltered_source->y_height;
+ const int64_t area = (int64_t)width * height;
const int bd = cm->seq_params->bit_depth;
const int blk_w = 16;
const int blk_h = 16;
@@ -1957,10 +1966,10 @@
const int color_thresh = 4;
const unsigned int var_thresh = 0;
// Counts of blocks with no more than color_thresh colors.
- int counts_1 = 0;
+ int64_t counts_1 = 0;
// Counts of blocks with no more than color_thresh colors and variance larger
// than var_thresh.
- int counts_2 = 0;
+ int64_t counts_2 = 0;
for (int r = 0; r + blk_h <= height; r += blk_h) {
for (int c = 0; c + blk_w <= width; c += blk_w) {
@@ -1985,17 +1994,15 @@
}
// The threshold values are selected experimentally.
- features->allow_screen_content_tools =
- counts_1 * blk_h * blk_w * 10 > width * height;
+ features->allow_screen_content_tools = counts_1 * blk_h * blk_w * 10 > area;
// IntraBC would force loop filters off, so we use more strict rules that also
// requires that the block has high variance.
features->allow_intrabc = features->allow_screen_content_tools &&
- counts_2 * blk_h * blk_w * 12 > width * height;
+ counts_2 * blk_h * blk_w * 12 > area;
cpi->use_screen_content_tools = features->allow_screen_content_tools;
cpi->is_screen_content_type =
- features->allow_intrabc ||
- (counts_1 * blk_h * blk_w * 10 > width * height * 4 &&
- counts_2 * blk_h * blk_w * 30 > width * height);
+ features->allow_intrabc || (counts_1 * blk_h * blk_w * 10 > area * 4 &&
+ counts_2 * blk_h * blk_w * 30 > area);
}
static void init_motion_estimation(AV1_COMP *cpi) {
@@ -2044,29 +2051,6 @@
}
}
-#if !CONFIG_REALTIME_ONLY
-#define COUPLED_CHROMA_FROM_LUMA_RESTORATION 0
-static void set_restoration_unit_size(int width, int height, int sx, int sy,
- RestorationInfo *rst) {
- (void)width;
- (void)height;
- (void)sx;
- (void)sy;
-#if COUPLED_CHROMA_FROM_LUMA_RESTORATION
- int s = AOMMIN(sx, sy);
-#else
- int s = 0;
-#endif // !COUPLED_CHROMA_FROM_LUMA_RESTORATION
-
- if (width * height > 352 * 288)
- rst[0].restoration_unit_size = RESTORATION_UNITSIZE_MAX;
- else
- rst[0].restoration_unit_size = (RESTORATION_UNITSIZE_MAX >> 1);
- rst[1].restoration_unit_size = rst[0].restoration_unit_size >> s;
- rst[2].restoration_unit_size = rst[1].restoration_unit_size;
-}
-#endif
-
static void init_ref_frame_bufs(AV1_COMP *cpi) {
AV1_COMMON *const cm = &cpi->common;
int i;
@@ -2232,11 +2216,6 @@
#if !CONFIG_REALTIME_ONLY
if (is_restoration_used(cm)) {
- const int frame_width = cm->superres_upscaled_width;
- const int frame_height = cm->superres_upscaled_height;
- set_restoration_unit_size(frame_width, frame_height,
- seq_params->subsampling_x,
- seq_params->subsampling_y, cm->rst_info);
for (int i = 0; i < num_planes; ++i)
cm->rst_info[i].frame_restoration_type = RESTORE_NONE;
@@ -2310,18 +2289,8 @@
start_timing(cpi, cdef_time);
#endif
const int num_workers = cpi->mt_info.num_mod_workers[MOD_CDEF];
- const int use_screen_content_model =
- cm->quant_params.base_qindex >
- AOMMAX(cpi->sf.rt_sf.screen_content_cdef_filter_qindex_thresh,
- cpi->rc.best_quality + 5) &&
- cpi->oxcf.tune_cfg.content == AOM_CONTENT_SCREEN;
// Find CDEF parameters
- av1_cdef_search(&cpi->mt_info, &cm->cur_frame->buf, cpi->source, cm, xd,
- cpi->sf.lpf_sf.cdef_pick_method, cpi->td.mb.rdmult,
- cpi->sf.rt_sf.skip_cdef_sb, cpi->oxcf.tool_cfg.cdef_control,
- use_screen_content_model,
- cpi->ppi->rtc_ref.non_reference_frame,
- cpi->rc.rtc_external_ratectrl);
+ av1_cdef_search(cpi);
// Apply the filter
if ((skip_apply_postproc_filters & SKIP_APPLY_CDEF) == 0) {
@@ -2499,7 +2468,6 @@
const QuantizationCfg *const q_cfg = &cpi->oxcf.q_cfg;
SVC *const svc = &cpi->svc;
const int resize_pending = is_frame_resize_pending(cpi);
-
int top_index = 0, bottom_index = 0, q = 0;
YV12_BUFFER_CONFIG *unscaled = cpi->unscaled_source;
InterpFilter filter_scaler =
@@ -2588,7 +2556,10 @@
memset(cpi->consec_zero_mv, 0, current_size * sizeof(*cpi->consec_zero_mv));
}
- if (cpi->unscaled_last_source != NULL) {
+ if (cpi->scaled_last_source_available) {
+ cpi->last_source = &cpi->scaled_last_source;
+ cpi->scaled_last_source_available = 0;
+ } else if (cpi->unscaled_last_source != NULL) {
cpi->last_source = av1_realloc_and_scale_if_required(
cm, cpi->unscaled_last_source, &cpi->scaled_last_source, filter_scaler,
phase_scaler, true, false, cpi->oxcf.border_in_pixels,
@@ -2610,12 +2581,15 @@
// av1_scale_references. Note GOLDEN is forced to update on the (first/tigger)
// resized frame and ALTREF will be refreshed ~4 frames later, so both
// references become available again after few frames.
+ // For superres: don't disable golden reference.
if (svc->number_spatial_layers == 1) {
- if (cpi->ref_frame_flags & av1_ref_frame_flag_list[GOLDEN_FRAME]) {
- const YV12_BUFFER_CONFIG *const ref =
- get_ref_frame_yv12_buf(cm, GOLDEN_FRAME);
- if (ref->y_crop_width != cm->width || ref->y_crop_height != cm->height)
- cpi->ref_frame_flags ^= AOM_GOLD_FLAG;
+ if (!cpi->oxcf.superres_cfg.enable_superres) {
+ if (cpi->ref_frame_flags & av1_ref_frame_flag_list[GOLDEN_FRAME]) {
+ const YV12_BUFFER_CONFIG *const ref =
+ get_ref_frame_yv12_buf(cm, GOLDEN_FRAME);
+ if (ref->y_crop_width != cm->width || ref->y_crop_height != cm->height)
+ cpi->ref_frame_flags ^= AOM_GOLD_FLAG;
+ }
}
if (cpi->ref_frame_flags & av1_ref_frame_flag_list[ALTREF_FRAME]) {
const YV12_BUFFER_CONFIG *const ref =
@@ -2632,16 +2606,9 @@
#endif // CONFIG_FPMT_TEST
if (scale_references ||
cpi->ppi->gf_group.frame_parallel_level[cpi->gf_frame_index] == 0) {
- // For SVC the inter-layer/spatial prediction is not done for newmv
- // (zero_mode is forced), and since the scaled references are only
- // use for newmv search, we can avoid scaling here when
- // force_zero_mode_spatial_ref is set for SVC mode.
- // Also add condition for dynamic_resize: for dynamic_resize we always
- // check for scaling references for now.
- if (!frame_is_intra_only(cm) &&
- (!cpi->ppi->use_svc || !cpi->svc.force_zero_mode_spatial_ref ||
- cpi->oxcf.resize_cfg.resize_mode == RESIZE_DYNAMIC))
+ if (!frame_is_intra_only(cm)) {
av1_scale_references(cpi, filter_scaler, phase_scaler, 1);
+ }
}
av1_set_quantizer(cm, q_cfg->qm_minlevel, q_cfg->qm_maxlevel, q,
@@ -2733,6 +2700,22 @@
sf->rt_sf.gf_refresh_based_on_qp)
av1_adjust_gf_refresh_qp_one_pass_rt(cpi);
+ // For non-svc: if scaling is required, copy scaled_source
+ // into scaled_last_source.
+ if (cm->current_frame.frame_number > 1 && !cpi->ppi->use_svc &&
+ cpi->scaled_source.y_buffer != NULL &&
+ cpi->scaled_last_source.y_buffer != NULL &&
+ cpi->scaled_source.y_crop_width == cpi->scaled_last_source.y_crop_width &&
+ cpi->scaled_source.y_crop_height ==
+ cpi->scaled_last_source.y_crop_height &&
+ (cm->width != cpi->unscaled_source->y_crop_width ||
+ cm->height != cpi->unscaled_source->y_crop_height)) {
+ cpi->scaled_last_source_available = 1;
+ aom_yv12_copy_y(&cpi->scaled_source, &cpi->scaled_last_source);
+ aom_yv12_copy_u(&cpi->scaled_source, &cpi->scaled_last_source);
+ aom_yv12_copy_v(&cpi->scaled_source, &cpi->scaled_last_source);
+ }
+
#if CONFIG_COLLECT_COMPONENT_TIMING
end_timing(cpi, av1_encode_frame_time);
#endif
@@ -3051,7 +3034,7 @@
rc->projected_frame_size, psnr.sse[0]);
++rd_command->frame_index;
if (rd_command->frame_index == rd_command->frame_count) {
- exit(0);
+ return AOM_CODEC_ERROR;
}
#endif // CONFIG_RD_COMMAND
@@ -3585,10 +3568,6 @@
start_timing(cpi, encode_frame_to_data_rate_time);
#endif
- if (frame_is_intra_only(cm)) {
- av1_set_screen_content_options(cpi, features);
- }
-
#if !CONFIG_REALTIME_ONLY
calculate_frame_avg_haar_energy(cpi);
#endif
@@ -3736,19 +3715,14 @@
cpi->num_tg = DEFAULT_MAX_NUM_TG;
}
- // For 1 pass CBR, check if we are dropping this frame.
- // Never drop on key frame, or for frame whose base layer is key.
- if (has_no_stats_stage(cpi) && oxcf->rc_cfg.mode == AOM_CBR &&
- current_frame->frame_type != KEY_FRAME &&
- !(cpi->ppi->use_svc &&
- cpi->svc.layer_context[cpi->svc.temporal_layer_id].is_key_frame)) {
- FRAME_UPDATE_TYPE update_type =
- cpi->ppi->gf_group.update_type[cpi->gf_frame_index];
- (void)update_type;
- assert(
- IMPLIES(cpi->is_dropped_frame, (update_type == OVERLAY_UPDATE ||
- update_type == INTNL_OVERLAY_UPDATE)));
- if (av1_rc_drop_frame(cpi)) {
+ // For 1 pass CBR mode: check if we are dropping this frame.
+ if (has_no_stats_stage(cpi) && oxcf->rc_cfg.mode == AOM_CBR) {
+ // Always drop for spatial enhancement layer if layer bandwidth is 0.
+ // Otherwise check for frame-dropping based on buffer level in
+ // av1_rc_drop_frame().
+ if ((cpi->svc.spatial_layer_id > 0 &&
+ cpi->oxcf.rc_cfg.target_bandwidth == 0) ||
+ av1_rc_drop_frame(cpi)) {
cpi->is_dropped_frame = true;
}
if (cpi->is_dropped_frame) {
@@ -4438,10 +4412,8 @@
fclose(f);
- if (ppi->ssim_vars != NULL) {
- aom_free(ppi->ssim_vars);
- ppi->ssim_vars = NULL;
- }
+ aom_free(ppi->ssim_vars);
+ ppi->ssim_vars = NULL;
}
}
#endif // CONFIG_INTERNAL_STATS
@@ -4481,7 +4453,16 @@
static AOM_INLINE void update_gf_group_index(AV1_COMP *cpi) {
// Increment the gf group index ready for the next frame.
- ++cpi->gf_frame_index;
+ if (is_one_pass_rt_params(cpi) &&
+ cpi->svc.spatial_layer_id == cpi->svc.number_spatial_layers - 1) {
+ ++cpi->gf_frame_index;
+ // Reset gf_frame_index in case it reaches MAX_STATIC_GF_GROUP_LENGTH
+ // for real time encoding.
+ if (cpi->gf_frame_index == MAX_STATIC_GF_GROUP_LENGTH)
+ cpi->gf_frame_index = 0;
+ } else {
+ ++cpi->gf_frame_index;
+ }
}
static void update_fb_of_context_type(const AV1_COMP *const cpi,
diff --git a/av1/encoder/encoder.h b/av1/encoder/encoder.h
index 9b2b4ae..9d02993 100644
--- a/av1/encoder/encoder.h
+++ b/av1/encoder/encoder.h
@@ -1442,6 +1442,8 @@
typedef struct ThreadData {
MACROBLOCK mb;
+ MvCosts *mv_costs_alloc;
+ IntraBCMVCosts *dv_costs_alloc;
RD_COUNTS rd_counts;
FRAME_COUNTS *counts;
PC_TREE_SHARED_BUFFERS shared_coeff_buf;
@@ -1454,6 +1456,7 @@
CONV_BUF_TYPE *tmp_conv_dst;
uint64_t abs_sum_level;
uint8_t *tmp_pred_bufs[2];
+ uint8_t *wiener_tmp_pred_buf;
int intrabc_used;
int deltaq_used;
int coefficient_size;
@@ -1464,7 +1467,9 @@
int32_t num_64x64_blocks;
PICK_MODE_CONTEXT *firstpass_ctx;
TemporalFilterData tf_data;
+ TplBuffers tpl_tmp_buffers;
TplTxfmStats tpl_txfm_stats;
+ GlobalMotionData gm_data;
// Pointer to the array of structures to store gradient information of each
// pixel in a superblock. The buffer constitutes of MAX_SB_SQUARE pixel level
// structures for each of the plane types (PLANE_TYPE_Y and PLANE_TYPE_UV).
@@ -1474,8 +1479,8 @@
// store source variance and log of source variance of each 4x4 sub-block
// for subsequent retrieval.
Block4x4VarInfo *src_var_info_of_4x4_sub_blocks;
- // The pc tree root for RTC non-rd case.
- PC_TREE *rt_pc_root;
+ // Pointer to pc tree root.
+ PC_TREE *pc_root;
} ThreadData;
struct EncWorkerData;
@@ -1526,6 +1531,26 @@
*/
int allocated_sb_rows;
+ /*!
+ * Initialized to false, set to true by the worker thread that encounters an
+ * error in order to abort the processing of other worker threads.
+ */
+ bool row_mt_exit;
+
+ /*!
+ * Initialized to false, set to true during first pass encoding by the worker
+ * thread that encounters an error in order to abort the processing of other
+ * worker threads.
+ */
+ bool firstpass_mt_exit;
+
+ /*!
+ * Initialized to false, set to true in cal_mb_wiener_var_hook() by the worker
+ * thread that encounters an error in order to abort the processing of other
+ * worker threads.
+ */
+ bool mb_wiener_mt_exit;
+
#if CONFIG_MULTITHREAD
/*!
* Mutex lock used while dispatching jobs.
@@ -1619,6 +1644,45 @@
} RestoreStateBuffers;
/*!
+ * \brief Parameters related to restoration types.
+ */
+typedef struct {
+ /*!
+ * Stores the best coefficients for Wiener restoration.
+ */
+ WienerInfo wiener;
+
+ /*!
+ * Stores the best coefficients for Sgrproj restoration.
+ */
+ SgrprojInfo sgrproj;
+
+ /*!
+ * The rtype to use for this unit given a frame rtype as index. Indices:
+ * WIENER, SGRPROJ, SWITCHABLE.
+ */
+ RestorationType best_rtype[RESTORE_TYPES - 1];
+} RestUnitSearchInfo;
+
+/*!
+ * \brief Structure to hold search parameter per restoration unit and
+ * intermediate buffer of Wiener filter used in pick filter stage of Loop
+ * restoration.
+ */
+typedef struct {
+ /*!
+ * Array of pointers to 'RestUnitSearchInfo' which holds data related to
+ * restoration types.
+ */
+ RestUnitSearchInfo *rusi[MAX_MB_PLANE];
+
+ /*!
+ * Buffer used to hold dgd-avg data during SIMD call of Wiener filter.
+ */
+ int16_t *dgd_avg;
+} AV1LrPickStruct;
+
+/*!
* \brief Primary Encoder parameters related to multi-threading.
*/
typedef struct PrimaryMultiThreadInfo {
@@ -2938,6 +3002,11 @@
TemporalFilterCtx tf_ctx;
/*!
+ * Pointer to CDEF search context.
+ */
+ CdefSearchCtx *cdef_search_ctx;
+
+ /*!
* Variables related to forcing integer mv decisions for the current frame.
*/
ForceIntegerMVInfo force_intpel_info;
@@ -3220,6 +3289,11 @@
AV1LrStruct lr_ctxt;
/*!
+ * Loop Restoration context used during pick stage.
+ */
+ AV1LrPickStruct pick_lr_ctxt;
+
+ /*!
* Pointer to list of tables with film grain parameters.
*/
aom_film_grain_table_t *film_grain_table;
@@ -3564,6 +3638,12 @@
* fast encoding pass in av1_determine_sc_tools_with_encoding().
*/
int palette_pixel_num;
+
+ /*!
+ * Flag to indicate scaled_last_source is available,
+ * so scaling is not needed for last_source.
+ */
+ int scaled_last_source_available;
} AV1_COMP;
/*!
@@ -3782,6 +3862,10 @@
int av1_convert_sect5obus_to_annexb(uint8_t *buffer, size_t *input_size);
+void av1_alloc_mb_wiener_var_pred_buf(AV1_COMMON *cm, ThreadData *td);
+
+void av1_dealloc_mb_wiener_var_pred_buf(ThreadData *td);
+
// Set screen content options.
// This function estimates whether to use screen content tools, by counting
// the portion of blocks that have few luma colors.
diff --git a/av1/encoder/encoder_alloc.h b/av1/encoder/encoder_alloc.h
index 9a1d60f..d0fd782 100644
--- a/av1/encoder/encoder_alloc.h
+++ b/av1/encoder/encoder_alloc.h
@@ -13,10 +13,13 @@
#define AOM_AV1_ENCODER_ENCODER_ALLOC_H_
#include "av1/encoder/block.h"
+#include "av1/encoder/encodeframe_utils.h"
#include "av1/encoder/encoder.h"
#include "av1/encoder/encodetxb.h"
#include "av1/encoder/ethread.h"
+#include "av1/encoder/global_motion_facade.h"
#include "av1/encoder/intra_mode_search_utils.h"
+#include "av1/encoder/pickcdef.h"
#ifdef __cplusplus
extern "C" {
@@ -24,11 +27,9 @@
static AOM_INLINE void dealloc_context_buffers_ext(
MBMIExtFrameBufferInfo *mbmi_ext_info) {
- if (mbmi_ext_info->frame_base) {
- aom_free(mbmi_ext_info->frame_base);
- mbmi_ext_info->frame_base = NULL;
- mbmi_ext_info->alloc_size = 0;
- }
+ aom_free(mbmi_ext_info->frame_base);
+ mbmi_ext_info->frame_base = NULL;
+ mbmi_ext_info->alloc_size = 0;
}
static AOM_INLINE void alloc_context_buffers_ext(
@@ -64,14 +65,14 @@
if (!is_stat_generation_stage(cpi)) av1_alloc_txb_buf(cpi);
- if (cpi->td.mb.mv_costs) {
- aom_free(cpi->td.mb.mv_costs);
- cpi->td.mb.mv_costs = NULL;
- }
- // Avoid the memory allocation of 'mv_costs' for allintra encoding mode.
+ aom_free(cpi->td.mv_costs_alloc);
+ cpi->td.mv_costs_alloc = NULL;
+ // Avoid the memory allocation of 'mv_costs_alloc' for allintra encoding
+ // mode.
if (cpi->oxcf.kf_cfg.key_freq_max != 0) {
- CHECK_MEM_ERROR(cm, cpi->td.mb.mv_costs,
- (MvCosts *)aom_calloc(1, sizeof(MvCosts)));
+ CHECK_MEM_ERROR(cm, cpi->td.mv_costs_alloc,
+ (MvCosts *)aom_calloc(1, sizeof(*cpi->td.mv_costs_alloc)));
+ cpi->td.mb.mv_costs = cpi->td.mv_costs_alloc;
}
av1_setup_shared_coeff_buffer(cm->seq_params, &cpi->td.shared_coeff_buf,
@@ -79,6 +80,9 @@
av1_setup_sms_tree(cpi, &cpi->td);
cpi->td.firstpass_ctx =
av1_alloc_pmc(cpi, BLOCK_16X16, &cpi->td.shared_coeff_buf);
+ if (!cpi->td.firstpass_ctx)
+ aom_internal_error(cm->error, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PICK_MODE_CONTEXT");
}
// Allocate mbmi buffers which are used to store mode information at block
@@ -178,7 +182,7 @@
static AOM_INLINE void dealloc_compressor_data(AV1_COMP *cpi) {
AV1_COMMON *const cm = &cpi->common;
TokenInfo *token_info = &cpi->token_info;
-
+ const int num_planes = av1_num_planes(cm);
dealloc_context_buffers_ext(&cpi->mbmi_ext_info);
aom_free(cpi->tile_data);
@@ -220,15 +224,25 @@
release_obmc_buffers(&cpi->td.mb.obmc_buffer);
- if (cpi->td.mb.mv_costs) {
- aom_free(cpi->td.mb.mv_costs);
- cpi->td.mb.mv_costs = NULL;
- }
+ aom_free(cpi->td.mv_costs_alloc);
+ cpi->td.mv_costs_alloc = NULL;
+ aom_free(cpi->td.dv_costs_alloc);
+ cpi->td.dv_costs_alloc = NULL;
- if (cpi->td.mb.dv_costs) {
- aom_free(cpi->td.mb.dv_costs);
- cpi->td.mb.dv_costs = NULL;
- }
+ aom_free(cpi->td.mb.sb_stats_cache);
+ cpi->td.mb.sb_stats_cache = NULL;
+
+ aom_free(cpi->td.mb.sb_fp_stats);
+ cpi->td.mb.sb_fp_stats = NULL;
+
+#if CONFIG_PARTITION_SEARCH_ORDER
+ aom_free(cpi->td.mb.rdcost);
+ cpi->td.mb.rdcost = NULL;
+#endif
+
+ av1_free_pc_tree_recursive(cpi->td.pc_root, num_planes, 0, 0,
+ cpi->sf.part_sf.partition_search_type);
+ cpi->td.pc_root = NULL;
for (int i = 0; i < 2; i++)
for (int j = 0; j < 2; j++) {
@@ -236,33 +250,55 @@
cpi->td.mb.intrabc_hash_info.hash_value_buffer[i][j] = NULL;
}
+ av1_hash_table_destroy(&cpi->td.mb.intrabc_hash_info.intrabc_hash_table);
+
aom_free(cm->tpl_mvs);
cm->tpl_mvs = NULL;
- if (cpi->td.pixel_gradient_info) {
- aom_free(cpi->td.pixel_gradient_info);
- cpi->td.pixel_gradient_info = NULL;
- }
+ aom_free(cpi->td.pixel_gradient_info);
+ cpi->td.pixel_gradient_info = NULL;
- if (cpi->td.src_var_info_of_4x4_sub_blocks) {
- aom_free(cpi->td.src_var_info_of_4x4_sub_blocks);
- cpi->td.src_var_info_of_4x4_sub_blocks = NULL;
- }
+ aom_free(cpi->td.src_var_info_of_4x4_sub_blocks);
+ cpi->td.src_var_info_of_4x4_sub_blocks = NULL;
- if (cpi->td.vt64x64) {
- aom_free(cpi->td.vt64x64);
- cpi->td.vt64x64 = NULL;
- }
+ aom_free(cpi->td.vt64x64);
+ cpi->td.vt64x64 = NULL;
- av1_free_pmc(cpi->td.firstpass_ctx, av1_num_planes(cm));
+ av1_free_pmc(cpi->td.firstpass_ctx, num_planes);
cpi->td.firstpass_ctx = NULL;
+ const int is_highbitdepth = cpi->tf_ctx.is_highbitdepth;
+ // This call ensures that the buffers allocated by tf_alloc_and_reset_data()
+ // in av1_temporal_filter() for single-threaded encode are freed in case an
+ // error is encountered during temporal filtering (due to early termination
+ // tf_dealloc_data() in av1_temporal_filter() would not be invoked).
+ tf_dealloc_data(&cpi->td.tf_data, is_highbitdepth);
+
+ // This call ensures that tpl_tmp_buffers for single-threaded encode are freed
+ // in case of an error during tpl.
+ tpl_dealloc_temp_buffers(&cpi->td.tpl_tmp_buffers);
+
+ // This call ensures that the global motion (gm) data buffers for
+ // single-threaded encode are freed in case of an error during gm.
+ gm_dealloc_data(&cpi->td.gm_data);
+
+ // This call ensures that CDEF search context buffers are deallocated in case
+ // of an error during cdef search.
+ av1_cdef_dealloc_data(cpi->cdef_search_ctx);
+ aom_free(cpi->cdef_search_ctx);
+ cpi->cdef_search_ctx = NULL;
+
+ av1_dealloc_mb_data(&cpi->td.mb, num_planes);
+
+ av1_dealloc_mb_wiener_var_pred_buf(&cpi->td);
+
av1_free_txb_buf(cpi);
av1_free_context_buffers(cm);
aom_free_frame_buffer(&cpi->last_frame_uf);
#if !CONFIG_REALTIME_ONLY
av1_free_restoration_buffers(cm);
+ av1_free_firstpass_data(&cpi->firstpass_data);
#endif
if (!is_stat_generation_stage(cpi)) {
@@ -270,6 +306,13 @@
&cpi->mt_info.cdef_sync);
}
+ for (int plane = 0; plane < num_planes; plane++) {
+ aom_free(cpi->pick_lr_ctxt.rusi[plane]);
+ cpi->pick_lr_ctxt.rusi[plane] = NULL;
+ }
+ aom_free(cpi->pick_lr_ctxt.dgd_avg);
+ cpi->pick_lr_ctxt.dgd_avg = NULL;
+
aom_free_frame_buffer(&cpi->trial_frame_rst);
aom_free_frame_buffer(&cpi->scaled_source);
aom_free_frame_buffer(&cpi->scaled_last_source);
@@ -304,16 +347,12 @@
aom_free(cpi->svc.layer_context);
cpi->svc.layer_context = NULL;
- if (cpi->consec_zero_mv) {
- aom_free(cpi->consec_zero_mv);
- cpi->consec_zero_mv = NULL;
- cpi->consec_zero_mv_alloc_size = 0;
- }
+ aom_free(cpi->consec_zero_mv);
+ cpi->consec_zero_mv = NULL;
+ cpi->consec_zero_mv_alloc_size = 0;
- if (cpi->src_sad_blk_64x64) {
- aom_free(cpi->src_sad_blk_64x64);
- cpi->src_sad_blk_64x64 = NULL;
- }
+ aom_free(cpi->src_sad_blk_64x64);
+ cpi->src_sad_blk_64x64 = NULL;
aom_free(cpi->mb_weber_stats);
cpi->mb_weber_stats = NULL;
@@ -399,15 +438,23 @@
"Failed to reallocate scaled source buffer");
assert(cpi->scaled_source.y_crop_width == scaled_width);
assert(cpi->scaled_source.y_crop_height == scaled_height);
- av1_resize_and_extend_frame_nonnormative(
- cpi->unscaled_source, &cpi->scaled_source, (int)cm->seq_params->bit_depth,
- num_planes);
+ if (!av1_resize_and_extend_frame_nonnormative(
+ cpi->unscaled_source, &cpi->scaled_source,
+ (int)cm->seq_params->bit_depth, num_planes))
+ aom_internal_error(cm->error, AOM_CODEC_MEM_ERROR,
+ "Failed to reallocate buffers during resize");
return &cpi->scaled_source;
}
// Deallocate allocated thread_data.
static AOM_INLINE void free_thread_data(AV1_PRIMARY *ppi) {
PrimaryMultiThreadInfo *const p_mt_info = &ppi->p_mt_info;
+ const int num_tf_workers =
+ AOMMIN(p_mt_info->num_mod_workers[MOD_TF], p_mt_info->num_workers);
+ const int num_tpl_workers =
+ AOMMIN(p_mt_info->num_mod_workers[MOD_TPL], p_mt_info->num_workers);
+ const int is_highbitdepth = ppi->seq_params.use_highbitdepth;
+ const int num_planes = ppi->seq_params.monochrome ? 1 : MAX_MB_PLANE;
for (int t = 1; t < p_mt_info->num_workers; ++t) {
EncWorkerData *const thread_data = &p_mt_info->tile_thr_data[t];
thread_data->td = thread_data->original_td;
@@ -429,12 +476,42 @@
thread_data->td->hash_value_buffer[x][y] = NULL;
}
}
+ aom_free(thread_data->td->mv_costs_alloc);
+ thread_data->td->mv_costs_alloc = NULL;
+ aom_free(thread_data->td->dv_costs_alloc);
+ thread_data->td->dv_costs_alloc = NULL;
aom_free(thread_data->td->counts);
- av1_free_pmc(thread_data->td->firstpass_ctx,
- ppi->seq_params.monochrome ? 1 : MAX_MB_PLANE);
+ av1_free_pmc(thread_data->td->firstpass_ctx, num_planes);
thread_data->td->firstpass_ctx = NULL;
av1_free_shared_coeff_buffer(&thread_data->td->shared_coeff_buf);
av1_free_sms_tree(thread_data->td);
+ // This call ensures that the buffers allocated by tf_alloc_and_reset_data()
+ // in prepare_tf_workers() for MT encode are freed in case an error is
+ // encountered during temporal filtering (due to early termination
+ // tf_dealloc_thread_data() in av1_tf_do_filtering_mt() would not be
+ // invoked).
+ if (t < num_tf_workers)
+ tf_dealloc_data(&thread_data->td->tf_data, is_highbitdepth);
+ // This call ensures that tpl_tmp_buffers for MT encode are freed in case of
+ // an error during tpl.
+ if (t < num_tpl_workers)
+ tpl_dealloc_temp_buffers(&thread_data->td->tpl_tmp_buffers);
+ // This call ensures that the buffers in gm_data for MT encode are freed in
+ // case of an error during gm.
+ gm_dealloc_data(&thread_data->td->gm_data);
+ av1_dealloc_mb_data(&thread_data->td->mb, num_planes);
+ aom_free(thread_data->td->mb.sb_stats_cache);
+ thread_data->td->mb.sb_stats_cache = NULL;
+ aom_free(thread_data->td->mb.sb_fp_stats);
+ thread_data->td->mb.sb_fp_stats = NULL;
+#if CONFIG_PARTITION_SEARCH_ORDER
+ aom_free(thread_data->td->mb.rdcost);
+ thread_data->td->mb.rdcost = NULL;
+#endif
+ av1_free_pc_tree_recursive(thread_data->td->pc_root, num_planes, 0, 0,
+ SEARCH_PARTITION);
+ thread_data->td->pc_root = NULL;
+ av1_dealloc_mb_wiener_var_pred_buf(thread_data->td);
aom_free(thread_data->td);
}
}
diff --git a/av1/encoder/encoder_utils.c b/av1/encoder/encoder_utils.c
index bc136b1..c35873d 100644
--- a/av1/encoder/encoder_utils.c
+++ b/av1/encoder/encoder_utils.c
@@ -521,7 +521,12 @@
cpi->ppi->p_rc.gfu_boost, gfu_boost,
cpi->ppi->p_rc.num_stats_used_for_gfu_boost);
} else {
- const int gfu_boost = (int)(200.0 / cpi->rd.r0);
+ // TPL may only look at a subset of frame in the gf group when the
+ // speed feature 'reduce_num_frames' is on, which affects the r0
+ // calcuation. Thus, to compensate for TPL not using all frames a
+ // factor to adjust r0 is used.
+ const int gfu_boost =
+ (int)(200.0 * cpi->ppi->tpl_data.r0_adjust_factor / cpi->rd.r0);
cpi->ppi->p_rc.gfu_boost = combine_prior_with_tpl_boost(
MIN_BOOST_COMBINE_FACTOR, MAX_BOOST_COMBINE_FACTOR,
cpi->ppi->p_rc.gfu_boost, gfu_boost, cpi->rc.frames_to_key);
@@ -691,6 +696,25 @@
continue;
}
+ // For RTC-SVC: if force_zero_mode_spatial_ref is enabled, check if the
+ // motion search can be skipped for the references: last, golden, altref.
+ // If so, we can skip scaling that reference.
+ if (cpi->ppi->use_svc && cpi->svc.force_zero_mode_spatial_ref &&
+ cpi->ppi->rtc_ref.set_ref_frame_config) {
+ if (ref_frame == LAST_FRAME && cpi->svc.skip_mvsearch_last) continue;
+ if (ref_frame == GOLDEN_FRAME && cpi->svc.skip_mvsearch_gf) continue;
+ if (ref_frame == ALTREF_FRAME && cpi->svc.skip_mvsearch_altref)
+ continue;
+ }
+ // For RTC with superres on: golden reference only needs to be scaled
+ // if it was refreshed in previous frame.
+ if (is_one_pass_rt_params(cpi) &&
+ cpi->oxcf.superres_cfg.enable_superres && ref_frame == GOLDEN_FRAME &&
+ cpi->rc.frame_num_last_gf_refresh <
+ (int)cm->current_frame.frame_number - 1) {
+ continue;
+ }
+
if (ref->y_crop_width != cm->width || ref->y_crop_height != cm->height) {
// Replace the reference buffer with a copy having a thicker border,
// if the reference buffer is higher resolution than the current
@@ -745,19 +769,25 @@
}
#if CONFIG_AV1_HIGHBITDEPTH
if (use_optimized_scaler && has_optimized_scaler &&
- cm->seq_params->bit_depth == AOM_BITS_8)
+ cm->seq_params->bit_depth == AOM_BITS_8) {
av1_resize_and_extend_frame(ref, &new_fb->buf, filter, phase,
num_planes);
- else
- av1_resize_and_extend_frame_nonnormative(
- ref, &new_fb->buf, (int)cm->seq_params->bit_depth, num_planes);
+ } else if (!av1_resize_and_extend_frame_nonnormative(
+ ref, &new_fb->buf, (int)cm->seq_params->bit_depth,
+ num_planes)) {
+ aom_internal_error(cm->error, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate buffer during resize");
+ }
#else
- if (use_optimized_scaler && has_optimized_scaler)
+ if (use_optimized_scaler && has_optimized_scaler) {
av1_resize_and_extend_frame(ref, &new_fb->buf, filter, phase,
num_planes);
- else
- av1_resize_and_extend_frame_nonnormative(
- ref, &new_fb->buf, (int)cm->seq_params->bit_depth, num_planes);
+ } else if (!av1_resize_and_extend_frame_nonnormative(
+ ref, &new_fb->buf, (int)cm->seq_params->bit_depth,
+ num_planes)) {
+ aom_internal_error(cm->error, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate buffer during resize");
+ }
#endif
cpi->scaled_ref_buf[ref_frame - 1] = new_fb;
alloc_frame_mvs(cm, new_fb);
diff --git a/av1/encoder/encoder_utils.h b/av1/encoder/encoder_utils.h
index 92e69da..10ff92d 100644
--- a/av1/encoder/encoder_utils.h
+++ b/av1/encoder/encoder_utils.h
@@ -510,8 +510,8 @@
#define HIGHBD_OBFP_WRAPPER_8(WIDTH, HEIGHT) \
HIGHBD_OBFP(BLOCK_##WIDTH##X##HEIGHT, \
aom_highbd_obmc_sad##WIDTH##x##HEIGHT##_bits8, \
- aom_highbd_obmc_variance##WIDTH##x##HEIGHT, \
- aom_highbd_obmc_sub_pixel_variance##WIDTH##x##HEIGHT)
+ aom_highbd_8_obmc_variance##WIDTH##x##HEIGHT, \
+ aom_highbd_8_obmc_sub_pixel_variance##WIDTH##x##HEIGHT)
#define HIGHBD_OBFP(BT, OSDF, OVF, OSVF) \
ppi->fn_ptr[BT].osdf = OSDF; \
@@ -1013,10 +1013,23 @@
}
static AOM_INLINE void release_scaled_references(AV1_COMP *cpi) {
- // TODO(isbs): only refresh the necessary frames, rather than all of them
+ // Scaled references should only need to be released under certain conditions:
+ // if the reference will be updated, or if the scaled reference has same
+ // resolution. For now only apply this to Golden for non-svc RTC mode.
+ AV1_COMMON *const cm = &cpi->common;
+ const bool refresh_golden = (cpi->refresh_frame.golden_frame) ? 1 : 0;
+ bool release_golden = true;
for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) {
RefCntBuffer *const buf = cpi->scaled_ref_buf[i];
- if (buf != NULL) {
+ const int golden_ref = (i == GOLDEN_FRAME - 1);
+ if (golden_ref && is_one_pass_rt_params(cpi) && !cpi->ppi->use_svc &&
+ buf != NULL) {
+ const RefCntBuffer *const ref = get_ref_frame_buf(cm, GOLDEN_FRAME);
+ const bool same_resoln = buf->buf.y_crop_width == ref->buf.y_crop_width &&
+ buf->buf.y_crop_height == ref->buf.y_crop_height;
+ release_golden = refresh_golden || same_resoln;
+ }
+ if (buf != NULL && (!golden_ref || (golden_ref && release_golden))) {
--buf->ref_count;
cpi->scaled_ref_buf[i] = NULL;
}
diff --git a/av1/encoder/ethread.c b/av1/encoder/ethread.c
index 430c6ae..24d47f3 100644
--- a/av1/encoder/ethread.c
+++ b/av1/encoder/ethread.c
@@ -9,15 +9,17 @@
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
+#include <assert.h>
+
#include "av1/common/warped_motion.h"
#include "av1/common/thread_common.h"
#include "av1/encoder/allintra_vis.h"
#include "av1/encoder/bitstream.h"
#include "av1/encoder/encodeframe.h"
+#include "av1/encoder/encodeframe_utils.h"
#include "av1/encoder/encoder.h"
#include "av1/encoder/encoder_alloc.h"
-#include "av1/encoder/encodeframe_utils.h"
#include "av1/encoder/ethread.h"
#if !CONFIG_REALTIME_ONLY
#include "av1/encoder/firstpass.h"
@@ -149,7 +151,13 @@
if (sig) {
pthread_mutex_lock(&row_mt_sync->mutex_[r]);
- row_mt_sync->num_finished_cols[r] = cur;
+ // When a thread encounters an error, num_finished_cols[r] is set to maximum
+ // column number. In this case, the AOMMAX operation here ensures that
+ // num_finished_cols[r] is not overwritten with a smaller value thus
+ // preventing the infinite waiting of threads in the relevant sync_read()
+ // function.
+ row_mt_sync->num_finished_cols[r] =
+ AOMMAX(row_mt_sync->num_finished_cols[r], cur);
pthread_cond_signal(&row_mt_sync->cond_[r]);
pthread_mutex_unlock(&row_mt_sync->mutex_[r]);
@@ -194,7 +202,7 @@
}
// Deallocate row based multi-threading synchronization related mutex and data
-static void row_mt_sync_mem_dealloc(AV1EncRowMultiThreadSync *row_mt_sync) {
+void av1_row_mt_sync_mem_dealloc(AV1EncRowMultiThreadSync *row_mt_sync) {
if (row_mt_sync != NULL) {
#if CONFIG_MULTITHREAD
int i;
@@ -264,6 +272,8 @@
enc_row_mt->allocated_rows = max_rows;
enc_row_mt->allocated_cols = max_cols - 1;
enc_row_mt->allocated_sb_rows = sb_rows;
+ enc_row_mt->row_mt_exit = false;
+ enc_row_mt->firstpass_mt_exit = false;
}
void av1_row_mt_mem_dealloc(AV1_COMP *cpi) {
@@ -278,7 +288,7 @@
int tile_index = tile_row * tile_cols + tile_col;
TileDataEnc *const this_tile = &cpi->tile_data[tile_index];
- row_mt_sync_mem_dealloc(&this_tile->row_mt_sync);
+ av1_row_mt_sync_mem_dealloc(&this_tile->row_mt_sync);
if (cpi->oxcf.algo_cfg.cdf_update_mode) aom_free(this_tile->row_ctx);
}
@@ -391,18 +401,68 @@
}
#if !CONFIG_REALTIME_ONLY
+static void set_firstpass_encode_done(AV1_COMP *cpi) {
+ AV1_COMMON *const cm = &cpi->common;
+ AV1EncRowMultiThreadInfo *const enc_row_mt = &cpi->mt_info.enc_row_mt;
+ const int tile_cols = cm->tiles.cols;
+ const int tile_rows = cm->tiles.rows;
+ const BLOCK_SIZE fp_block_size = cpi->fp_block_size;
+ const int unit_height = mi_size_high[fp_block_size];
+
+ // In case of multithreading of firstpass encode, due to top-right
+ // dependency, the worker on a firstpass row waits for the completion of the
+ // firstpass processing of the top and top-right fp_blocks. Hence, in case a
+ // thread (main/worker) encounters an error, update the firstpass processing
+ // of every row in the frame to indicate that it is complete in order to avoid
+ // dependent workers waiting indefinitely.
+ for (int tile_row = 0; tile_row < tile_rows; ++tile_row) {
+ for (int tile_col = 0; tile_col < tile_cols; ++tile_col) {
+ TileDataEnc *const tile_data =
+ &cpi->tile_data[tile_row * tile_cols + tile_col];
+ TileInfo *tile = &tile_data->tile_info;
+ AV1EncRowMultiThreadSync *const row_mt_sync = &tile_data->row_mt_sync;
+ const int unit_cols_in_tile =
+ av1_get_unit_cols_in_tile(tile, fp_block_size);
+ for (int mi_row = tile->mi_row_start, unit_row_in_tile = 0;
+ mi_row < tile->mi_row_end;
+ mi_row += unit_height, unit_row_in_tile++) {
+ enc_row_mt->sync_write_ptr(row_mt_sync, unit_row_in_tile,
+ unit_cols_in_tile - 1, unit_cols_in_tile);
+ }
+ }
+ }
+}
+
static int fp_enc_row_mt_worker_hook(void *arg1, void *unused) {
EncWorkerData *const thread_data = (EncWorkerData *)arg1;
AV1_COMP *const cpi = thread_data->cpi;
- AV1_COMMON *const cm = &cpi->common;
int thread_id = thread_data->thread_id;
AV1EncRowMultiThreadInfo *const enc_row_mt = &cpi->mt_info.enc_row_mt;
- int cur_tile_id = enc_row_mt->thread_id_to_tile_id[thread_id];
#if CONFIG_MULTITHREAD
pthread_mutex_t *enc_row_mt_mutex_ = enc_row_mt->mutex_;
#endif
(void)unused;
+ struct aom_internal_error_info *const error_info = &thread_data->error_info;
+ MACROBLOCKD *const xd = &thread_data->td->mb.e_mbd;
+ xd->error_info = error_info;
+ // The jmp_buf is valid only for the duration of the function that calls
+ // setjmp(). Therefore, this function must reset the 'setjmp' field to 0
+ // before it returns.
+ if (setjmp(error_info->jmp)) {
+ error_info->setjmp = 0;
+#if CONFIG_MULTITHREAD
+ pthread_mutex_lock(enc_row_mt_mutex_);
+ enc_row_mt->firstpass_mt_exit = true;
+ pthread_mutex_unlock(enc_row_mt_mutex_);
+#endif
+ set_firstpass_encode_done(cpi);
+ return 0;
+ }
+ error_info->setjmp = 1;
+
+ AV1_COMMON *const cm = &cpi->common;
+ int cur_tile_id = enc_row_mt->thread_id_to_tile_id[thread_id];
assert(cur_tile_id != -1);
const BLOCK_SIZE fp_block_size = cpi->fp_block_size;
@@ -413,8 +473,9 @@
#if CONFIG_MULTITHREAD
pthread_mutex_lock(enc_row_mt_mutex_);
#endif
- if (!get_next_job(&cpi->tile_data[cur_tile_id], ¤t_mi_row,
- unit_height)) {
+ bool firstpass_mt_exit = enc_row_mt->firstpass_mt_exit;
+ if (!firstpass_mt_exit && !get_next_job(&cpi->tile_data[cur_tile_id],
+ ¤t_mi_row, unit_height)) {
// No jobs are available for the current tile. Query for the status of
// other tiles and get the next job if available
switch_tile_and_get_next_job(cm, cpi->tile_data, &cur_tile_id,
@@ -424,7 +485,9 @@
#if CONFIG_MULTITHREAD
pthread_mutex_unlock(enc_row_mt_mutex_);
#endif
- if (end_of_frame == 1) break;
+ // When firstpass_mt_exit is set to true, other workers need not pursue any
+ // further jobs.
+ if (firstpass_mt_exit || end_of_frame) break;
TileDataEnc *const this_tile = &cpi->tile_data[cur_tile_id];
AV1EncRowMultiThreadSync *const row_mt_sync = &this_tile->row_mt_sync;
@@ -444,7 +507,7 @@
pthread_mutex_unlock(enc_row_mt_mutex_);
#endif
}
-
+ error_info->setjmp = 0;
return 1;
}
#endif
@@ -455,6 +518,7 @@
AV1LfSync *const lf_sync = (AV1LfSync *)thread_data->lf_sync;
const int sb_rows = get_sb_rows_in_frame(cm);
AV1LfMTInfo *cur_job_info;
+ bool row_mt_exit = false;
(void)enc_row_mt;
#if CONFIG_MULTITHREAD
pthread_mutex_t *enc_row_mt_mutex_ = enc_row_mt->mutex_;
@@ -469,43 +533,116 @@
const int next_sb_row = AOMMIN(sb_rows - 1, cur_sb_row + 1);
// Wait for current and next superblock row to finish encoding.
pthread_mutex_lock(enc_row_mt_mutex_);
- while (enc_row_mt->num_tile_cols_done[cur_sb_row] < cm->tiles.cols ||
- enc_row_mt->num_tile_cols_done[next_sb_row] < cm->tiles.cols) {
+ while (!enc_row_mt->row_mt_exit &&
+ (enc_row_mt->num_tile_cols_done[cur_sb_row] < cm->tiles.cols ||
+ enc_row_mt->num_tile_cols_done[next_sb_row] < cm->tiles.cols)) {
pthread_cond_wait(enc_row_mt->cond_, enc_row_mt_mutex_);
}
+ row_mt_exit = enc_row_mt->row_mt_exit;
pthread_mutex_unlock(enc_row_mt_mutex_);
#endif
+ if (row_mt_exit) return;
+
av1_thread_loop_filter_rows(
lf_data->frame_buffer, lf_data->cm, lf_data->planes, lf_data->xd,
cur_job_info->mi_row, cur_job_info->plane, cur_job_info->dir,
- lpf_opt_level, lf_sync, lf_data->params_buf, lf_data->tx_buf,
- mib_size_log2);
+ lpf_opt_level, lf_sync, &thread_data->error_info, lf_data->params_buf,
+ lf_data->tx_buf, mib_size_log2);
+ }
+}
+
+static void set_encoding_done(AV1_COMP *cpi) {
+ AV1_COMMON *const cm = &cpi->common;
+ const int tile_cols = cm->tiles.cols;
+ const int tile_rows = cm->tiles.rows;
+ AV1EncRowMultiThreadInfo *const enc_row_mt = &cpi->mt_info.enc_row_mt;
+ const int mib_size = cm->seq_params->mib_size;
+
+ // In case of row-multithreading, due to top-right dependency, the worker on
+ // an SB row waits for the completion of the encode of the top and top-right
+ // SBs. Hence, in case a thread (main/worker) encounters an error, update that
+ // encoding of every SB row in the frame is complete in order to avoid the
+ // dependent workers of every tile from waiting indefinitely.
+ for (int tile_row = 0; tile_row < tile_rows; tile_row++) {
+ for (int tile_col = 0; tile_col < tile_cols; tile_col++) {
+ TileDataEnc *const this_tile =
+ &cpi->tile_data[tile_row * tile_cols + tile_col];
+ const TileInfo *const tile_info = &this_tile->tile_info;
+ AV1EncRowMultiThreadSync *const row_mt_sync = &this_tile->row_mt_sync;
+ const int sb_cols_in_tile = av1_get_sb_cols_in_tile(cm, tile_info);
+ for (int mi_row = tile_info->mi_row_start, sb_row_in_tile = 0;
+ mi_row < tile_info->mi_row_end;
+ mi_row += mib_size, sb_row_in_tile++) {
+ enc_row_mt->sync_write_ptr(row_mt_sync, sb_row_in_tile,
+ sb_cols_in_tile - 1, sb_cols_in_tile);
+ }
+ }
}
}
static int enc_row_mt_worker_hook(void *arg1, void *unused) {
EncWorkerData *const thread_data = (EncWorkerData *)arg1;
AV1_COMP *const cpi = thread_data->cpi;
- AV1_COMMON *const cm = &cpi->common;
int thread_id = thread_data->thread_id;
AV1EncRowMultiThreadInfo *const enc_row_mt = &cpi->mt_info.enc_row_mt;
- int cur_tile_id = enc_row_mt->thread_id_to_tile_id[thread_id];
- const int mib_size_log2 = cm->seq_params->mib_size_log2;
#if CONFIG_MULTITHREAD
pthread_mutex_t *enc_row_mt_mutex_ = enc_row_mt->mutex_;
#endif
(void)unused;
+
+ struct aom_internal_error_info *const error_info = &thread_data->error_info;
+ AV1LfSync *const lf_sync = thread_data->lf_sync;
+ MACROBLOCKD *const xd = &thread_data->td->mb.e_mbd;
+ xd->error_info = error_info;
+
+ // The jmp_buf is valid only for the duration of the function that calls
+ // setjmp(). Therefore, this function must reset the 'setjmp' field to 0
+ // before it returns.
+ if (setjmp(error_info->jmp)) {
+ error_info->setjmp = 0;
+#if CONFIG_MULTITHREAD
+ pthread_mutex_lock(enc_row_mt_mutex_);
+ enc_row_mt->row_mt_exit = true;
+ // Wake up all the workers waiting in launch_loop_filter_rows() to exit in
+ // case of an error.
+ pthread_cond_broadcast(enc_row_mt->cond_);
+ pthread_mutex_unlock(enc_row_mt_mutex_);
+#endif
+ set_encoding_done(cpi);
+
+ if (cpi->mt_info.pipeline_lpf_mt_with_enc) {
+#if CONFIG_MULTITHREAD
+ pthread_mutex_lock(lf_sync->job_mutex);
+ lf_sync->lf_mt_exit = true;
+ pthread_mutex_unlock(lf_sync->job_mutex);
+#endif
+ av1_set_vert_loop_filter_done(&cpi->common, lf_sync,
+ cpi->common.seq_params->mib_size_log2);
+ }
+ return 0;
+ }
+ error_info->setjmp = 1;
+
+ AV1_COMMON *const cm = &cpi->common;
+ const int mib_size_log2 = cm->seq_params->mib_size_log2;
+ int cur_tile_id = enc_row_mt->thread_id_to_tile_id[thread_id];
+
// Preallocate the pc_tree for realtime coding to reduce the cost of memory
// allocation.
- thread_data->td->rt_pc_root =
- cpi->sf.rt_sf.use_nonrd_pick_mode
- ? av1_alloc_pc_tree_node(cm->seq_params->sb_size)
- : NULL;
+ if (cpi->sf.rt_sf.use_nonrd_pick_mode) {
+ thread_data->td->pc_root = av1_alloc_pc_tree_node(cm->seq_params->sb_size);
+ if (!thread_data->td->pc_root)
+ aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PC_TREE");
+ } else {
+ thread_data->td->pc_root = NULL;
+ }
assert(cur_tile_id != -1);
const BLOCK_SIZE fp_block_size = cpi->fp_block_size;
int end_of_frame = 0;
+ bool row_mt_exit = false;
// When master thread does not have a valid job to process, xd->tile_ctx
// is not set and it contains NULL pointer. This can result in NULL pointer
@@ -518,7 +655,12 @@
#if CONFIG_MULTITHREAD
pthread_mutex_lock(enc_row_mt_mutex_);
#endif
- if (!get_next_job(&cpi->tile_data[cur_tile_id], ¤t_mi_row,
+ row_mt_exit = enc_row_mt->row_mt_exit;
+ // row_mt_exit check here can be avoided as it is checked after
+ // sync_read_ptr() in encode_sb_row(). However, checking row_mt_exit here,
+ // tries to return before calling the function get_next_job().
+ if (!row_mt_exit &&
+ !get_next_job(&cpi->tile_data[cur_tile_id], ¤t_mi_row,
cm->seq_params->mib_size)) {
// No jobs are available for the current tile. Query for the status of
// other tiles and get the next job if available
@@ -529,7 +671,14 @@
#if CONFIG_MULTITHREAD
pthread_mutex_unlock(enc_row_mt_mutex_);
#endif
- if (end_of_frame == 1) break;
+ // When row_mt_exit is set to true, other workers need not pursue any
+ // further jobs.
+ if (row_mt_exit) {
+ error_info->setjmp = 0;
+ return 1;
+ }
+
+ if (end_of_frame) break;
TileDataEnc *const this_tile = &cpi->tile_data[cur_tile_id];
AV1EncRowMultiThreadSync *const row_mt_sync = &this_tile->row_mt_sync;
@@ -583,26 +732,46 @@
// encoding and loop filter stage.
launch_loop_filter_rows(cm, thread_data, enc_row_mt, mib_size_log2);
}
- av1_free_pc_tree_recursive(thread_data->td->rt_pc_root, av1_num_planes(cm), 0,
- 0, cpi->sf.part_sf.partition_search_type);
+ av1_free_pc_tree_recursive(thread_data->td->pc_root, av1_num_planes(cm), 0, 0,
+ cpi->sf.part_sf.partition_search_type);
+ thread_data->td->pc_root = NULL;
+ error_info->setjmp = 0;
return 1;
}
static int enc_worker_hook(void *arg1, void *unused) {
EncWorkerData *const thread_data = (EncWorkerData *)arg1;
AV1_COMP *const cpi = thread_data->cpi;
+ MACROBLOCKD *const xd = &thread_data->td->mb.e_mbd;
+ struct aom_internal_error_info *const error_info = &thread_data->error_info;
const AV1_COMMON *const cm = &cpi->common;
const int tile_cols = cm->tiles.cols;
const int tile_rows = cm->tiles.rows;
int t;
(void)unused;
+
+ xd->error_info = error_info;
+
+ // The jmp_buf is valid only for the duration of the function that calls
+ // setjmp(). Therefore, this function must reset the 'setjmp' field to 0
+ // before it returns.
+ if (setjmp(error_info->jmp)) {
+ error_info->setjmp = 0;
+ return 0;
+ }
+ error_info->setjmp = 1;
+
// Preallocate the pc_tree for realtime coding to reduce the cost of memory
// allocation.
- thread_data->td->rt_pc_root =
- cpi->sf.rt_sf.use_nonrd_pick_mode
- ? av1_alloc_pc_tree_node(cm->seq_params->sb_size)
- : NULL;
+ if (cpi->sf.rt_sf.use_nonrd_pick_mode) {
+ thread_data->td->pc_root = av1_alloc_pc_tree_node(cm->seq_params->sb_size);
+ if (!thread_data->td->pc_root)
+ aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PC_TREE");
+ } else {
+ thread_data->td->pc_root = NULL;
+ }
for (t = thread_data->start; t < tile_rows * tile_cols;
t += cpi->mt_info.num_workers) {
@@ -616,9 +785,10 @@
av1_encode_tile(cpi, thread_data->td, tile_row, tile_col);
}
- av1_free_pc_tree_recursive(thread_data->td->rt_pc_root, av1_num_planes(cm), 0,
- 0, cpi->sf.part_sf.partition_search_type);
-
+ av1_free_pc_tree_recursive(thread_data->td->pc_root, av1_num_planes(cm), 0, 0,
+ cpi->sf.part_sf.partition_search_type);
+ thread_data->td->pc_root = NULL;
+ error_info->setjmp = 0;
return 1;
}
@@ -651,10 +821,11 @@
AV1_COMMON *const cm = &cpi->common;
AV1LrSync *lr_sync = &cpi->mt_info.lr_row_sync;
if (lr_sync->sync_range) {
- int num_lr_workers =
- av1_get_num_mod_workers_for_alloc(&cpi->ppi->p_mt_info, MOD_LR);
if (cpi->ppi->gf_group.frame_parallel_level[cpi->gf_frame_index] > 0)
return;
+ int num_lr_workers =
+ av1_get_num_mod_workers_for_alloc(&cpi->ppi->p_mt_info, MOD_LR);
+ assert(num_lr_workers <= lr_sync->num_workers);
lr_sync->lrworkerdata[num_lr_workers - 1].rst_tmpbuf = cm->rst_tmpbuf;
lr_sync->lrworkerdata[num_lr_workers - 1].rlbs = cm->rlbs;
}
@@ -720,16 +891,21 @@
av1_loop_filter_alloc(lf_sync, cm, sb_rows, cm->width, num_lf_workers);
}
+ // Initialize tpl MT object.
+ AV1TplRowMultiThreadInfo *tpl_row_mt = &mt_info->tpl_row_mt;
+ if (tpl_row_mt->mutex_ == NULL) {
+ CHECK_MEM_ERROR(cm, tpl_row_mt->mutex_,
+ aom_malloc(sizeof(*(tpl_row_mt->mutex_))));
+ if (tpl_row_mt->mutex_) pthread_mutex_init(tpl_row_mt->mutex_, NULL);
+ }
+ tpl_row_mt->tpl_mt_exit = false;
+
#if !CONFIG_REALTIME_ONLY
if (is_restoration_used(cm)) {
// Initialize loop restoration MT object.
AV1LrSync *lr_sync = &mt_info->lr_row_sync;
- int rst_unit_size;
- if (cm->width * cm->height > 352 * 288)
- rst_unit_size = RESTORATION_UNITSIZE_MAX;
- else
- rst_unit_size = (RESTORATION_UNITSIZE_MAX >> 1);
- int num_rows_lr = av1_lr_count_units_in_tile(rst_unit_size, cm->height);
+ int rst_unit_size = cpi->sf.lpf_sf.min_lr_unit_size;
+ int num_rows_lr = av1_lr_count_units(rst_unit_size, cm->height);
int num_lr_workers = av1_get_num_mod_workers_for_alloc(p_mt_info, MOD_LR);
if (!lr_sync->sync_range || num_rows_lr > lr_sync->rows ||
num_lr_workers > lr_sync->num_workers ||
@@ -754,7 +930,7 @@
// Computes the number of workers to be considered while allocating memory for a
// multi-threaded module under FPMT.
-int av1_get_num_mod_workers_for_alloc(PrimaryMultiThreadInfo *const p_mt_info,
+int av1_get_num_mod_workers_for_alloc(const PrimaryMultiThreadInfo *p_mt_info,
MULTI_THREADED_MODULES mod_name) {
int num_mod_workers = p_mt_info->num_mod_workers[mod_name];
if (p_mt_info->num_mod_workers[MOD_FRAME_ENC] > 1) {
@@ -798,6 +974,9 @@
// Set up firstpass PICK_MODE_CONTEXT.
thread_data->td->firstpass_ctx = av1_alloc_pmc(
ppi->cpi, BLOCK_16X16, &thread_data->td->shared_coeff_buf);
+ if (!thread_data->td->firstpass_ctx)
+ aom_internal_error(&ppi->error, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PICK_MODE_CONTEXT");
}
if (!is_first_pass && i < num_enc_workers) {
@@ -1111,6 +1290,7 @@
if (is_restoration_used(cm)) {
// Back up the original LR buffers before update.
int idx = i + mt_info->num_workers - 1;
+ assert(idx < mt_info->lr_row_sync.num_workers);
mt_info->restore_state_buf.rst_tmpbuf =
mt_info->lr_row_sync.lrworkerdata[idx].rst_tmpbuf;
mt_info->restore_state_buf.rlbs =
@@ -1153,27 +1333,6 @@
}
}
-// Synchronize level 1 workers.
-static AOM_INLINE void sync_fpmt_workers(AV1_PRIMARY *ppi) {
- const AVxWorkerInterface *const winterface = aom_get_worker_interface();
- int num_workers = ppi->p_mt_info.p_num_workers;
- int had_error = 0;
- // Points to error in the earliest display order frame in the parallel set.
- const struct aom_internal_error_info *error;
-
- // Encoding ends.
- for (int i = num_workers - 1; i >= 0; i--) {
- AVxWorker *const worker = ppi->p_mt_info.p_workers[i];
- if (!winterface->sync(worker)) {
- had_error = 1;
- error = ((AV1_COMP *)worker->data1)->common.error;
- }
- }
-
- if (had_error)
- aom_internal_error(&ppi->error, error->error_code, "%s", error->detail);
-}
-
// Restore worker states after parallel encode.
static AOM_INLINE void restore_workers_after_fpmt(AV1_PRIMARY *ppi,
int parallel_frame_count) {
@@ -1203,6 +1362,7 @@
if (is_restoration_used(cm)) {
// Restore the original LR buffers.
int idx = i + mt_info->num_workers - 1;
+ assert(idx < mt_info->lr_row_sync.num_workers);
mt_info->lr_row_sync.lrworkerdata[idx].rst_tmpbuf =
mt_info->restore_state_buf.rst_tmpbuf;
mt_info->lr_row_sync.lrworkerdata[idx].rlbs =
@@ -1215,6 +1375,30 @@
}
}
+// Synchronize level 1 workers.
+static AOM_INLINE void sync_fpmt_workers(AV1_PRIMARY *ppi,
+ int frames_in_parallel_set) {
+ const AVxWorkerInterface *const winterface = aom_get_worker_interface();
+ int num_workers = ppi->p_mt_info.p_num_workers;
+ int had_error = 0;
+ // Points to error in the earliest display order frame in the parallel set.
+ const struct aom_internal_error_info *error;
+
+ // Encoding ends.
+ for (int i = num_workers - 1; i >= 0; --i) {
+ AVxWorker *const worker = ppi->p_mt_info.p_workers[i];
+ if (!winterface->sync(worker)) {
+ had_error = 1;
+ error = ppi->parallel_cpi[i]->common.error;
+ }
+ }
+
+ restore_workers_after_fpmt(ppi, frames_in_parallel_set);
+
+ if (had_error)
+ aom_internal_error(&ppi->error, error->error_code, "%s", error->detail);
+}
+
static int get_compressed_data_hook(void *arg1, void *arg2) {
AV1_COMP *cpi = (AV1_COMP *)arg1;
AV1_COMP_DATA *cpi_data = (AV1_COMP_DATA *)arg2;
@@ -1236,8 +1420,7 @@
prepare_fpmt_workers(ppi, first_cpi_data, get_compressed_data_hook,
frames_in_parallel_set);
launch_fpmt_workers(ppi);
- sync_fpmt_workers(ppi);
- restore_workers_after_fpmt(ppi, frames_in_parallel_set);
+ sync_fpmt_workers(ppi, frames_in_parallel_set);
// Release cpi->scaled_ref_buf corresponding to frames in the current parallel
// encode set.
@@ -1254,6 +1437,7 @@
const AVxWorkerInterface *const winterface = aom_get_worker_interface();
for (int i = num_workers - 1; i >= 0; i--) {
AVxWorker *const worker = &mt_info->workers[i];
+ worker->had_error = 0;
if (i == 0)
winterface->execute(worker);
else
@@ -1264,17 +1448,33 @@
static AOM_INLINE void sync_enc_workers(MultiThreadInfo *const mt_info,
AV1_COMMON *const cm, int num_workers) {
const AVxWorkerInterface *const winterface = aom_get_worker_interface();
- int had_error = mt_info->workers[0].had_error;
+ const AVxWorker *const worker_main = &mt_info->workers[0];
+ int had_error = worker_main->had_error;
+ struct aom_internal_error_info error_info;
+
+ // Read the error_info of main thread.
+ if (had_error) {
+ error_info = ((EncWorkerData *)worker_main->data1)->error_info;
+ }
// Encoding ends.
for (int i = num_workers - 1; i > 0; i--) {
AVxWorker *const worker = &mt_info->workers[i];
- had_error |= !winterface->sync(worker);
+ if (!winterface->sync(worker)) {
+ had_error = 1;
+ error_info = ((EncWorkerData *)worker->data1)->error_info;
+ }
}
if (had_error)
- aom_internal_error(cm->error, AOM_CODEC_ERROR,
- "Failed to encode tile data");
+ aom_internal_error(cm->error, error_info.error_code, "%s",
+ error_info.detail);
+
+ // Restore xd->error_info of the main thread back to cm->error so that the
+ // multithreaded code, when executed using a single thread, has a valid
+ // xd->error_info.
+ MACROBLOCKD *const xd = &((EncWorkerData *)worker_main->data1)->td->mb.e_mbd;
+ xd->error_info = cm->error;
}
static AOM_INLINE void accumulate_counters_enc_workers(AV1_COMP *cpi,
@@ -1292,13 +1492,15 @@
// Keep these conditional expressions in sync with the corresponding ones
// in prepare_enc_workers().
if (cpi->sf.inter_sf.mv_cost_upd_level != INTERNAL_COST_UPD_OFF) {
- aom_free(thread_data->td->mb.mv_costs);
+ aom_free(thread_data->td->mv_costs_alloc);
+ thread_data->td->mv_costs_alloc = NULL;
}
if (cpi->sf.intra_sf.dv_cost_upd_level != INTERNAL_COST_UPD_OFF) {
- aom_free(thread_data->td->mb.dv_costs);
+ aom_free(thread_data->td->dv_costs_alloc);
+ thread_data->td->dv_costs_alloc = NULL;
}
}
- av1_dealloc_mb_data(&cpi->common, &thread_data->td->mb);
+ av1_dealloc_mb_data(&thread_data->td->mb, av1_num_planes(&cpi->common));
// Accumulate counters.
if (i > 0) {
@@ -1362,8 +1564,10 @@
// Keep these conditional expressions in sync with the corresponding ones
// in accumulate_counters_enc_workers().
if (cpi->sf.inter_sf.mv_cost_upd_level != INTERNAL_COST_UPD_OFF) {
- CHECK_MEM_ERROR(cm, thread_data->td->mb.mv_costs,
- (MvCosts *)aom_malloc(sizeof(MvCosts)));
+ CHECK_MEM_ERROR(
+ cm, thread_data->td->mv_costs_alloc,
+ (MvCosts *)aom_malloc(sizeof(*thread_data->td->mv_costs_alloc)));
+ thread_data->td->mb.mv_costs = thread_data->td->mv_costs_alloc;
memcpy(thread_data->td->mb.mv_costs, cpi->td.mb.mv_costs,
sizeof(MvCosts));
}
@@ -1373,8 +1577,10 @@
// aom_free() call for the same.
thread_data->td->mb.dv_costs = NULL;
if (av1_need_dv_costs(cpi)) {
- CHECK_MEM_ERROR(cm, thread_data->td->mb.dv_costs,
- (IntraBCMVCosts *)aom_malloc(sizeof(IntraBCMVCosts)));
+ CHECK_MEM_ERROR(cm, thread_data->td->dv_costs_alloc,
+ (IntraBCMVCosts *)aom_malloc(
+ sizeof(*thread_data->td->dv_costs_alloc)));
+ thread_data->td->mb.dv_costs = thread_data->td->dv_costs_alloc;
memcpy(thread_data->td->mb.dv_costs, cpi->td.mb.dv_costs,
sizeof(IntraBCMVCosts));
}
@@ -1438,26 +1644,17 @@
thread_data->td = thread_data->original_td;
}
- // Before encoding a frame, copy the thread data from cpi.
if (thread_data->td != &cpi->td) {
+ // Before encoding a frame, copy the thread data from cpi.
thread_data->td->mb = cpi->td.mb;
- // Keep this conditional expression in sync with the corresponding one
- // in av1_fp_encode_tiles_row_mt().
- if (cpi->sf.inter_sf.mv_cost_upd_level != INTERNAL_COST_UPD_OFF) {
- CHECK_MEM_ERROR(cm, thread_data->td->mb.mv_costs,
- (MvCosts *)aom_malloc(sizeof(MvCosts)));
- memcpy(thread_data->td->mb.mv_costs, cpi->td.mb.mv_costs,
- sizeof(MvCosts));
- }
+ av1_alloc_src_diff_buf(cm, &thread_data->td->mb);
}
-
- av1_alloc_mb_data(cpi, &thread_data->td->mb);
}
}
#endif
// Computes the number of workers for row multi-threading of encoding stage
-static AOM_INLINE int compute_num_enc_row_mt_workers(AV1_COMMON *const cm,
+static AOM_INLINE int compute_num_enc_row_mt_workers(const AV1_COMMON *cm,
int max_threads) {
TileInfo tile_info;
const int tile_cols = cm->tiles.cols;
@@ -1476,7 +1673,7 @@
}
// Computes the number of workers for tile multi-threading of encoding stage
-static AOM_INLINE int compute_num_enc_tile_mt_workers(AV1_COMMON *const cm,
+static AOM_INLINE int compute_num_enc_tile_mt_workers(const AV1_COMMON *cm,
int max_threads) {
const int tile_cols = cm->tiles.cols;
const int tile_rows = cm->tiles.rows;
@@ -1494,7 +1691,7 @@
}
// Computes the number of workers for encoding stage (row/tile multi-threading)
-int av1_compute_num_enc_workers(AV1_COMP *cpi, int max_workers) {
+int av1_compute_num_enc_workers(const AV1_COMP *cpi, int max_workers) {
if (max_workers <= 1) return 1;
if (cpi->oxcf.row_mt)
return compute_num_enc_row_mt_workers(&cpi->common, max_workers);
@@ -1752,6 +1949,15 @@
}
#if !CONFIG_REALTIME_ONLY
+static void dealloc_thread_data_src_diff_buf(AV1_COMP *cpi, int num_workers) {
+ for (int i = num_workers - 1; i >= 0; --i) {
+ EncWorkerData *const thread_data = &cpi->mt_info.tile_thr_data[i];
+ if (thread_data->td != &cpi->td)
+ av1_dealloc_src_diff_buf(&thread_data->td->mb,
+ av1_num_planes(&cpi->common));
+ }
+}
+
void av1_fp_encode_tiles_row_mt(AV1_COMP *cpi) {
AV1_COMMON *const cm = &cpi->common;
MultiThreadInfo *const mt_info = &cpi->mt_info;
@@ -1814,18 +2020,7 @@
fp_prepare_enc_workers(cpi, fp_enc_row_mt_worker_hook, num_workers);
launch_workers(&cpi->mt_info, num_workers);
sync_enc_workers(&cpi->mt_info, cm, num_workers);
- for (int i = num_workers - 1; i >= 0; i--) {
- EncWorkerData *const thread_data = &cpi->mt_info.tile_thr_data[i];
- if (thread_data->td != &cpi->td) {
- // Keep this conditional expression in sync with the corresponding one
- // in fp_prepare_enc_workers().
- if (cpi->sf.inter_sf.mv_cost_upd_level != INTERNAL_COST_UPD_OFF) {
- aom_free(thread_data->td->mb.mv_costs);
- }
- assert(!thread_data->td->mb.dv_costs);
- }
- av1_dealloc_mb_data(cm, &thread_data->td->mb);
- }
+ dealloc_thread_data_src_diff_buf(cpi, num_workers);
}
void av1_tpl_row_mt_sync_read_dummy(AV1TplRowMultiThreadSync *tpl_mt_sync,
@@ -1894,6 +2089,27 @@
#endif // CONFIG_MULTITHREAD
}
+static AOM_INLINE void set_mode_estimation_done(AV1_COMP *cpi) {
+ const CommonModeInfoParams *const mi_params = &cpi->common.mi_params;
+ TplParams *const tpl_data = &cpi->ppi->tpl_data;
+ const BLOCK_SIZE bsize =
+ convert_length_to_bsize(cpi->ppi->tpl_data.tpl_bsize_1d);
+ const int mi_height = mi_size_high[bsize];
+ AV1TplRowMultiThreadInfo *const tpl_row_mt = &cpi->mt_info.tpl_row_mt;
+ const int tplb_cols_in_tile =
+ ROUND_POWER_OF_TWO(mi_params->mi_cols, mi_size_wide_log2[bsize]);
+ // In case of tpl row-multithreading, due to top-right dependency, the worker
+ // on an mb_row waits for the completion of the tpl processing of the top and
+ // top-right blocks. Hence, in case a thread (main/worker) encounters an
+ // error, update that the tpl processing of every mb_row in the frame is
+ // complete in order to avoid dependent workers waiting indefinitely.
+ for (int mi_row = 0, tplb_row = 0; mi_row < mi_params->mi_rows;
+ mi_row += mi_height, tplb_row++) {
+ (*tpl_row_mt->sync_write_ptr)(&tpl_data->tpl_mt_sync, tplb_row,
+ tplb_cols_in_tile - 1, tplb_cols_in_tile);
+ }
+}
+
// Each worker calls tpl_worker_hook() and computes the tpl data.
static int tpl_worker_hook(void *arg1, void *unused) {
(void)unused;
@@ -1903,11 +2119,36 @@
MACROBLOCK *x = &thread_data->td->mb;
MACROBLOCKD *xd = &x->e_mbd;
TplTxfmStats *tpl_txfm_stats = &thread_data->td->tpl_txfm_stats;
+ TplBuffers *tpl_tmp_buffers = &thread_data->td->tpl_tmp_buffers;
CommonModeInfoParams *mi_params = &cm->mi_params;
+ int num_active_workers = cpi->ppi->tpl_data.tpl_mt_sync.num_threads_working;
+
+ struct aom_internal_error_info *const error_info = &thread_data->error_info;
+ xd->error_info = error_info;
+ AV1TplRowMultiThreadInfo *const tpl_row_mt = &cpi->mt_info.tpl_row_mt;
+ (void)tpl_row_mt;
+#if CONFIG_MULTITHREAD
+ pthread_mutex_t *tpl_error_mutex_ = tpl_row_mt->mutex_;
+#endif
+
+ // The jmp_buf is valid only for the duration of the function that calls
+ // setjmp(). Therefore, this function must reset the 'setjmp' field to 0
+ // before it returns.
+ if (setjmp(error_info->jmp)) {
+ error_info->setjmp = 0;
+#if CONFIG_MULTITHREAD
+ pthread_mutex_lock(tpl_error_mutex_);
+ tpl_row_mt->tpl_mt_exit = true;
+ pthread_mutex_unlock(tpl_error_mutex_);
+#endif
+ set_mode_estimation_done(cpi);
+ return 0;
+ }
+ error_info->setjmp = 1;
+
BLOCK_SIZE bsize = convert_length_to_bsize(cpi->ppi->tpl_data.tpl_bsize_1d);
TX_SIZE tx_size = max_txsize_lookup[bsize];
int mi_height = mi_size_high[bsize];
- int num_active_workers = cpi->ppi->tpl_data.tpl_mt_sync.num_threads_working;
av1_init_tpl_txfm_stats(tpl_txfm_stats);
@@ -1919,8 +2160,10 @@
xd->mb_to_top_edge = -GET_MV_SUBPEL(mi_row * MI_SIZE);
xd->mb_to_bottom_edge =
GET_MV_SUBPEL((mi_params->mi_rows - mi_height - mi_row) * MI_SIZE);
- av1_mc_flow_dispenser_row(cpi, tpl_txfm_stats, x, mi_row, bsize, tx_size);
+ av1_mc_flow_dispenser_row(cpi, tpl_txfm_stats, tpl_tmp_buffers, x, mi_row,
+ bsize, tx_size);
}
+ error_info->setjmp = 0;
return 1;
}
@@ -2005,6 +2248,11 @@
// OBMC buffers are used only to init MS params and remain unused when
// called from tpl, hence set the buffers to defaults.
av1_init_obmc_buffer(&thread_data->td->mb.obmc_buffer);
+ if (!tpl_alloc_temp_buffers(&thread_data->td->tpl_tmp_buffers,
+ cpi->ppi->tpl_data.tpl_bsize_1d)) {
+ aom_internal_error(cpi->common.error, AOM_CODEC_MEM_ERROR,
+ "Error allocating tpl data");
+ }
thread_data->td->mb.tmp_conv_dst = thread_data->td->tmp_conv_dst;
thread_data->td->mb.e_mbd.tmp_conv_dst = thread_data->td->mb.tmp_conv_dst;
}
@@ -2056,6 +2304,11 @@
#if CONFIG_BITRATE_ACCURACY
tpl_accumulate_txfm_stats(&cpi->td, &cpi->mt_info, num_workers);
#endif // CONFIG_BITRATE_ACCURACY
+ for (int i = num_workers - 1; i >= 0; i--) {
+ EncWorkerData *thread_data = &mt_info->tile_thr_data[i];
+ ThreadData *td = thread_data->td;
+ if (td != &cpi->td) tpl_dealloc_temp_buffers(&td->tpl_tmp_buffers);
+ }
}
// Deallocate memory for temporal filter multi-thread synchronization.
@@ -2079,7 +2332,7 @@
pthread_mutex_t *tf_mutex_ = tf_mt_sync->mutex_;
pthread_mutex_lock(tf_mutex_);
#endif
- if (tf_mt_sync->next_tf_row < mb_rows) {
+ if (!tf_mt_sync->tf_mt_exit && tf_mt_sync->next_tf_row < mb_rows) {
*current_mb_row = tf_mt_sync->next_tf_row;
tf_mt_sync->next_tf_row++;
do_next_row = 1;
@@ -2099,6 +2352,28 @@
TemporalFilterCtx *tf_ctx = &cpi->tf_ctx;
AV1TemporalFilterSync *tf_sync = &cpi->mt_info.tf_sync;
const struct scale_factors *scale = &cpi->tf_ctx.sf;
+
+#if CONFIG_MULTITHREAD
+ pthread_mutex_t *tf_mutex_ = tf_sync->mutex_;
+#endif
+ MACROBLOCKD *const xd = &thread_data->td->mb.e_mbd;
+ struct aom_internal_error_info *const error_info = &thread_data->error_info;
+ xd->error_info = error_info;
+
+ // The jmp_buf is valid only for the duration of the function that calls
+ // setjmp(). Therefore, this function must reset the 'setjmp' field to 0
+ // before it returns.
+ if (setjmp(error_info->jmp)) {
+ error_info->setjmp = 0;
+#if CONFIG_MULTITHREAD
+ pthread_mutex_lock(tf_mutex_);
+ tf_sync->tf_mt_exit = true;
+ pthread_mutex_unlock(tf_mutex_);
+#endif
+ return 0;
+ }
+ error_info->setjmp = 1;
+
const int num_planes = av1_num_planes(&cpi->common);
assert(num_planes >= 1 && num_planes <= MAX_MB_PLANE);
@@ -2115,6 +2390,7 @@
tf_restore_state(mbd, input_mb_mode_info, input_buffer, num_planes);
+ error_info->setjmp = 0;
return 1;
}
@@ -2123,6 +2399,7 @@
int num_workers, int is_highbitdepth) {
MultiThreadInfo *mt_info = &cpi->mt_info;
mt_info->tf_sync.next_tf_row = 0;
+ mt_info->tf_sync.tf_mt_exit = false;
for (int i = num_workers - 1; i >= 0; i--) {
AVxWorker *worker = &mt_info->workers[i];
EncWorkerData *thread_data = &mt_info->tile_thr_data[i];
@@ -2227,19 +2504,6 @@
get_next_gm_job(cpi, frame_idx, *(cur_dir));
}
-// Initializes inliers, num_inliers and segment_map.
-static AOM_INLINE void init_gm_thread_data(
- const GlobalMotionInfo *gm_info, GlobalMotionThreadData *thread_data) {
- for (int m = 0; m < RANSAC_NUM_MOTIONS; m++) {
- MotionModel motion_params = thread_data->motion_models[m];
- av1_zero(motion_params.params);
- motion_params.num_inliers = 0;
- }
-
- av1_zero_array(thread_data->segment_map,
- gm_info->segment_map_w * gm_info->segment_map_h);
-}
-
// Hook function for each thread in global motion multi-threading.
static int gm_mt_worker_hook(void *arg1, void *unused) {
(void)unused;
@@ -2247,16 +2511,34 @@
EncWorkerData *thread_data = (EncWorkerData *)arg1;
AV1_COMP *cpi = thread_data->cpi;
GlobalMotionInfo *gm_info = &cpi->gm_info;
- MultiThreadInfo *mt_info = &cpi->mt_info;
- JobInfo *job_info = &mt_info->gm_sync.job_info;
+ AV1GlobalMotionSync *gm_sync = &cpi->mt_info.gm_sync;
+ JobInfo *job_info = &gm_sync->job_info;
int thread_id = thread_data->thread_id;
- GlobalMotionThreadData *gm_thread_data =
- &mt_info->gm_sync.thread_data[thread_id];
- int cur_dir = job_info->thread_id_to_dir[thread_id];
+ GlobalMotionData *gm_thread_data = &thread_data->td->gm_data;
#if CONFIG_MULTITHREAD
- pthread_mutex_t *gm_mt_mutex_ = mt_info->gm_sync.mutex_;
+ pthread_mutex_t *gm_mt_mutex_ = gm_sync->mutex_;
#endif
+ MACROBLOCKD *const xd = &thread_data->td->mb.e_mbd;
+ struct aom_internal_error_info *const error_info = &thread_data->error_info;
+ xd->error_info = error_info;
+
+ // The jmp_buf is valid only for the duration of the function that calls
+ // setjmp(). Therefore, this function must reset the 'setjmp' field to 0
+ // before it returns.
+ if (setjmp(error_info->jmp)) {
+ error_info->setjmp = 0;
+#if CONFIG_MULTITHREAD
+ pthread_mutex_lock(gm_mt_mutex_);
+ gm_sync->gm_mt_exit = true;
+ pthread_mutex_unlock(gm_mt_mutex_);
+#endif
+ return 0;
+ }
+ error_info->setjmp = 1;
+
+ int cur_dir = job_info->thread_id_to_dir[thread_id];
+ bool gm_mt_exit = false;
while (1) {
int ref_buf_idx = -1;
@@ -2264,9 +2546,10 @@
pthread_mutex_lock(gm_mt_mutex_);
#endif
+ gm_mt_exit = gm_sync->gm_mt_exit;
// Populates ref_buf_idx(the reference frame type) for which global motion
// estimation will be done.
- if (!get_next_gm_job(cpi, &ref_buf_idx, cur_dir)) {
+ if (!gm_mt_exit && !get_next_gm_job(cpi, &ref_buf_idx, cur_dir)) {
// No jobs are available for the current direction. Switch
// to other direction and get the next job, if available.
switch_direction(cpi, &ref_buf_idx, &cur_dir);
@@ -2276,15 +2559,15 @@
pthread_mutex_unlock(gm_mt_mutex_);
#endif
- if (ref_buf_idx == -1) break;
-
- init_gm_thread_data(gm_info, gm_thread_data);
+ // When gm_mt_exit is set to true, other workers need not pursue any
+ // further jobs.
+ if (gm_mt_exit || ref_buf_idx == -1) break;
// Compute global motion for the given ref_buf_idx.
av1_compute_gm_for_valid_ref_frames(
- cpi, gm_info->ref_buf, ref_buf_idx, gm_thread_data->motion_models,
- gm_thread_data->segment_map, gm_info->segment_map_w,
- gm_info->segment_map_h);
+ cpi, error_info, gm_info->ref_buf, ref_buf_idx,
+ gm_thread_data->motion_models, gm_thread_data->segment_map,
+ gm_info->segment_map_w, gm_info->segment_map_h);
#if CONFIG_MULTITHREAD
pthread_mutex_lock(gm_mt_mutex_);
@@ -2300,6 +2583,7 @@
pthread_mutex_unlock(gm_mt_mutex_);
#endif
}
+ error_info->setjmp = 0;
return 1;
}
@@ -2307,6 +2591,7 @@
static AOM_INLINE void prepare_gm_workers(AV1_COMP *cpi, AVxWorkerHook hook,
int num_workers) {
MultiThreadInfo *mt_info = &cpi->mt_info;
+ mt_info->gm_sync.gm_mt_exit = false;
for (int i = num_workers - 1; i >= 0; i--) {
AVxWorker *worker = &mt_info->workers[i];
EncWorkerData *thread_data = &mt_info->tile_thr_data[i];
@@ -2325,6 +2610,9 @@
} else {
thread_data->td = thread_data->original_td;
}
+
+ if (thread_data->td != &cpi->td)
+ gm_alloc_data(cpi, &thread_data->td->gm_data);
}
}
@@ -2351,69 +2639,28 @@
}
// Frees the memory allocated for each worker in global motion multi-threading.
-void av1_gm_dealloc(AV1GlobalMotionSync *gm_sync_data) {
- if (gm_sync_data->thread_data != NULL) {
- for (int j = 0; j < gm_sync_data->allocated_workers; j++) {
- GlobalMotionThreadData *thread_data = &gm_sync_data->thread_data[j];
- aom_free(thread_data->segment_map);
-
- for (int m = 0; m < RANSAC_NUM_MOTIONS; m++)
- aom_free(thread_data->motion_models[m].inliers);
- }
- aom_free(gm_sync_data->thread_data);
- }
-}
-
-// Allocates memory for inliers and segment_map for each worker in global motion
-// multi-threading.
-static AOM_INLINE void gm_alloc(AV1_COMP *cpi, int num_workers) {
- AV1_COMMON *cm = &cpi->common;
- AV1GlobalMotionSync *gm_sync = &cpi->mt_info.gm_sync;
- GlobalMotionInfo *gm_info = &cpi->gm_info;
-
- gm_sync->allocated_workers = num_workers;
- gm_sync->allocated_width = cpi->source->y_width;
- gm_sync->allocated_height = cpi->source->y_height;
-
- CHECK_MEM_ERROR(cm, gm_sync->thread_data,
- aom_malloc(sizeof(*gm_sync->thread_data) * num_workers));
-
- for (int i = 0; i < num_workers; i++) {
- GlobalMotionThreadData *thread_data = &gm_sync->thread_data[i];
- CHECK_MEM_ERROR(
- cm, thread_data->segment_map,
- aom_malloc(sizeof(*thread_data->segment_map) * gm_info->segment_map_w *
- gm_info->segment_map_h));
-
- for (int m = 0; m < RANSAC_NUM_MOTIONS; m++) {
- CHECK_MEM_ERROR(
- cm, thread_data->motion_models[m].inliers,
- aom_malloc(sizeof(*thread_data->motion_models[m].inliers) * 2 *
- MAX_CORNERS));
- }
+static AOM_INLINE void gm_dealloc_thread_data(AV1_COMP *cpi, int num_workers) {
+ MultiThreadInfo *mt_info = &cpi->mt_info;
+ for (int j = 0; j < num_workers; j++) {
+ EncWorkerData *thread_data = &mt_info->tile_thr_data[j];
+ ThreadData *td = thread_data->td;
+ if (td != &cpi->td) gm_dealloc_data(&td->gm_data);
}
}
// Implements multi-threading for global motion.
void av1_global_motion_estimation_mt(AV1_COMP *cpi) {
- AV1GlobalMotionSync *gm_sync = &cpi->mt_info.gm_sync;
- JobInfo *job_info = &gm_sync->job_info;
+ JobInfo *job_info = &cpi->mt_info.gm_sync.job_info;
av1_zero(*job_info);
int num_workers = compute_gm_workers(cpi);
- if (num_workers > gm_sync->allocated_workers ||
- cpi->source->y_width != gm_sync->allocated_width ||
- cpi->source->y_height != gm_sync->allocated_height) {
- av1_gm_dealloc(gm_sync);
- gm_alloc(cpi, num_workers);
- }
-
assign_thread_to_dir(job_info->thread_id_to_dir, num_workers);
prepare_gm_workers(cpi, gm_mt_worker_hook, num_workers);
launch_workers(&cpi->mt_info, num_workers);
sync_enc_workers(&cpi->mt_info, &cpi->common, num_workers);
+ gm_dealloc_thread_data(cpi, num_workers);
}
#endif // !CONFIG_REALTIME_ONLY
@@ -2455,10 +2702,33 @@
if (thread_data->td != &cpi->td) {
thread_data->td->mb = cpi->td.mb;
+ av1_alloc_mb_wiener_var_pred_buf(&cpi->common, thread_data->td);
}
}
}
+static void set_mb_wiener_var_calc_done(AV1_COMP *const cpi) {
+ const CommonModeInfoParams *const mi_params = &cpi->common.mi_params;
+ const BLOCK_SIZE bsize = cpi->weber_bsize;
+ const int mb_step = mi_size_wide[bsize];
+ assert(MB_WIENER_MT_UNIT_SIZE < BLOCK_SIZES_ALL);
+ const int mt_unit_step = mi_size_wide[MB_WIENER_MT_UNIT_SIZE];
+ const int mt_unit_cols =
+ (mi_params->mi_cols + (mt_unit_step >> 1)) / mt_unit_step;
+ const AV1EncAllIntraMultiThreadInfo *const intra_mt = &cpi->mt_info.intra_mt;
+ AV1EncRowMultiThreadSync *const intra_row_mt_sync =
+ &cpi->ppi->intra_row_mt_sync;
+
+ // Update the wiener variance computation of every row in the frame to
+ // indicate that it is complete in order to avoid dependent workers waiting
+ // indefinitely.
+ for (int mi_row = 0, mt_thread_id = 0; mi_row < mi_params->mi_rows;
+ mi_row += mb_step, ++mt_thread_id) {
+ intra_mt->intra_sync_write_ptr(intra_row_mt_sync, mt_thread_id,
+ mt_unit_cols - 1, mt_unit_cols);
+ }
+}
+
static int cal_mb_wiener_var_hook(void *arg1, void *unused) {
(void)unused;
EncWorkerData *const thread_data = (EncWorkerData *)arg1;
@@ -2472,42 +2742,74 @@
AV1EncRowMultiThreadInfo *const enc_row_mt = &cpi->mt_info.enc_row_mt;
(void)enc_row_mt;
#if CONFIG_MULTITHREAD
- pthread_mutex_t *enc_row_mt_mutex_ = enc_row_mt->mutex_;
+ pthread_mutex_t *enc_row_mt_mutex = enc_row_mt->mutex_;
#endif
+
+ struct aom_internal_error_info *const error_info = &thread_data->error_info;
+ xd->error_info = error_info;
+
+ // The jmp_buf is valid only for the duration of the function that calls
+ // setjmp(). Therefore, this function must reset the 'setjmp' field to 0
+ // before it returns.
+ if (setjmp(error_info->jmp)) {
+ error_info->setjmp = 0;
+#if CONFIG_MULTITHREAD
+ pthread_mutex_lock(enc_row_mt_mutex);
+ enc_row_mt->mb_wiener_mt_exit = true;
+ pthread_mutex_unlock(enc_row_mt_mutex);
+#endif
+ set_mb_wiener_var_calc_done(cpi);
+ return 0;
+ }
+ error_info->setjmp = 1;
DECLARE_ALIGNED(32, int16_t, src_diff[32 * 32]);
DECLARE_ALIGNED(32, tran_low_t, coeff[32 * 32]);
DECLARE_ALIGNED(32, tran_low_t, qcoeff[32 * 32]);
DECLARE_ALIGNED(32, tran_low_t, dqcoeff[32 * 32]);
double sum_rec_distortion = 0;
double sum_est_rate = 0;
- int has_jobs = 1;
- while (has_jobs) {
+ while (1) {
int current_mi_row = -1;
#if CONFIG_MULTITHREAD
- pthread_mutex_lock(enc_row_mt_mutex_);
+ pthread_mutex_lock(enc_row_mt_mutex);
#endif
- has_jobs =
- get_next_job_allintra(intra_row_mt_sync, cpi->common.mi_params.mi_rows,
- ¤t_mi_row, mb_step);
+ int has_jobs = enc_row_mt->mb_wiener_mt_exit
+ ? 0
+ : get_next_job_allintra(intra_row_mt_sync,
+ cpi->common.mi_params.mi_rows,
+ ¤t_mi_row, mb_step);
#if CONFIG_MULTITHREAD
- pthread_mutex_unlock(enc_row_mt_mutex_);
+ pthread_mutex_unlock(enc_row_mt_mutex);
#endif
if (!has_jobs) break;
// TODO(chengchen): properly accumulate the distortion and rate.
av1_calc_mb_wiener_var_row(cpi, x, xd, current_mi_row, src_diff, coeff,
qcoeff, dqcoeff, &sum_rec_distortion,
- &sum_est_rate);
+ &sum_est_rate,
+ thread_data->td->wiener_tmp_pred_buf);
#if CONFIG_MULTITHREAD
- pthread_mutex_lock(enc_row_mt_mutex_);
+ pthread_mutex_lock(enc_row_mt_mutex);
#endif
intra_row_mt_sync->num_threads_working--;
#if CONFIG_MULTITHREAD
- pthread_mutex_unlock(enc_row_mt_mutex_);
+ pthread_mutex_unlock(enc_row_mt_mutex);
#endif
}
+ error_info->setjmp = 0;
return 1;
}
+static void dealloc_mb_wiener_var_mt_data(AV1_COMP *cpi, int num_workers) {
+ av1_row_mt_sync_mem_dealloc(&cpi->ppi->intra_row_mt_sync);
+
+ MultiThreadInfo *mt_info = &cpi->mt_info;
+ for (int j = 0; j < num_workers; ++j) {
+ EncWorkerData *thread_data = &mt_info->tile_thr_data[j];
+ ThreadData *td = thread_data->td;
+ if (td != &cpi->td) av1_dealloc_mb_wiener_var_pred_buf(td);
+ }
+}
+
// This function is the multi-threading version of computing the wiener
// variance.
// Note that the wiener variance is used for allintra mode (1 pass) and its
@@ -2533,12 +2835,12 @@
intra_row_mt_sync->next_mi_row = 0;
memset(intra_row_mt_sync->num_finished_cols, -1,
sizeof(*intra_row_mt_sync->num_finished_cols) * mi_rows);
+ mt_info->enc_row_mt.mb_wiener_mt_exit = false;
prepare_wiener_var_workers(cpi, cal_mb_wiener_var_hook, num_workers);
launch_workers(mt_info, num_workers);
sync_enc_workers(mt_info, cm, num_workers);
-
- row_mt_sync_mem_dealloc(intra_row_mt_sync);
+ dealloc_mb_wiener_var_mt_data(cpi, num_workers);
}
// Compare and order tiles based on absolute sum of tx coeffs.
@@ -2713,14 +3015,40 @@
const CommonTileParams *const tiles = &cm->tiles;
const int num_tiles = tiles->cols * tiles->rows;
+#if CONFIG_MULTITHREAD
+ pthread_mutex_t *const pack_bs_mutex = pack_bs_sync->mutex_;
+#endif
+ MACROBLOCKD *const xd = &thread_data->td->mb.e_mbd;
+ struct aom_internal_error_info *const error_info = &thread_data->error_info;
+ xd->error_info = error_info;
+
+ // The jmp_buf is valid only for the duration of the function that calls
+ // setjmp(). Therefore, this function must reset the 'setjmp' field to 0
+ // before it returns.
+ if (setjmp(error_info->jmp)) {
+ error_info->setjmp = 0;
+#if CONFIG_MULTITHREAD
+ pthread_mutex_lock(pack_bs_mutex);
+ pack_bs_sync->pack_bs_mt_exit = true;
+ pthread_mutex_unlock(pack_bs_mutex);
+#endif
+ return 0;
+ }
+ error_info->setjmp = 1;
+
while (1) {
#if CONFIG_MULTITHREAD
- pthread_mutex_lock(pack_bs_sync->mutex_);
+ pthread_mutex_lock(pack_bs_mutex);
#endif
- const int tile_idx = get_next_pack_bs_tile_idx(pack_bs_sync, num_tiles);
+ const int tile_idx =
+ pack_bs_sync->pack_bs_mt_exit
+ ? -1
+ : get_next_pack_bs_tile_idx(pack_bs_sync, num_tiles);
#if CONFIG_MULTITHREAD
- pthread_mutex_unlock(pack_bs_sync->mutex_);
+ pthread_mutex_unlock(pack_bs_mutex);
#endif
+ // When pack_bs_mt_exit is set to true, other workers need not pursue any
+ // further jobs.
if (tile_idx == -1) break;
TileDataEnc *this_tile = &cpi->tile_data[tile_idx];
thread_data->td->mb.e_mbd.tile_ctx = &this_tile->tctx;
@@ -2728,6 +3056,7 @@
av1_pack_tile_info(cpi, thread_data->td, &pack_bs_params[tile_idx]);
}
+ error_info->setjmp = 0;
return 1;
}
@@ -2902,8 +3231,9 @@
// populates next job information and returns 1, else returns 0.
static AOM_INLINE int cdef_get_next_job(AV1CdefSync *cdef_sync,
CdefSearchCtx *cdef_search_ctx,
- int *cur_fbr, int *cur_fbc,
- int *sb_count) {
+ volatile int *cur_fbr,
+ volatile int *cur_fbc,
+ volatile int *sb_count) {
#if CONFIG_MULTITHREAD
pthread_mutex_lock(cdef_sync->mutex_);
#endif // CONFIG_MULTITHREAD
@@ -2913,15 +3243,15 @@
// If a block is skip, do not process the block and
// check the skip condition for the next block.
- while ((!cdef_sync->end_of_frame) &&
- (cdef_sb_skip(cdef_search_ctx->mi_params, cdef_sync->fbr,
- cdef_sync->fbc))) {
+ while (!cdef_sync->cdef_mt_exit && !cdef_sync->end_of_frame &&
+ cdef_sb_skip(cdef_search_ctx->mi_params, cdef_sync->fbr,
+ cdef_sync->fbc)) {
update_next_job_info(cdef_sync, nvfb, nhfb);
}
// Populates information needed for current job and update the row,
// column indices of the next block to be processed.
- if (cdef_sync->end_of_frame == 0) {
+ if (!cdef_sync->cdef_mt_exit && cdef_sync->end_of_frame == 0) {
do_next_block = 1;
*cur_fbr = cdef_sync->fbr;
*cur_fbc = cdef_sync->fbc;
@@ -2937,43 +3267,68 @@
// Hook function for each thread in CDEF search multi-threading.
static int cdef_filter_block_worker_hook(void *arg1, void *arg2) {
- AV1CdefSync *const cdef_sync = (AV1CdefSync *)arg1;
- CdefSearchCtx *cdef_search_ctx = (CdefSearchCtx *)arg2;
- int cur_fbr, cur_fbc, sb_count;
+ EncWorkerData *thread_data = (EncWorkerData *)arg1;
+ AV1CdefSync *const cdef_sync = (AV1CdefSync *)arg2;
+
+#if CONFIG_MULTITHREAD
+ pthread_mutex_t *cdef_mutex_ = cdef_sync->mutex_;
+#endif
+ struct aom_internal_error_info *const error_info = &thread_data->error_info;
+ CdefSearchCtx *cdef_search_ctx = thread_data->cpi->cdef_search_ctx;
+
+ // The jmp_buf is valid only for the duration of the function that calls
+ // setjmp(). Therefore, this function must reset the 'setjmp' field to 0
+ // before it returns.
+ if (setjmp(error_info->jmp)) {
+ error_info->setjmp = 0;
+#if CONFIG_MULTITHREAD
+ pthread_mutex_lock(cdef_mutex_);
+ cdef_sync->cdef_mt_exit = true;
+ pthread_mutex_unlock(cdef_mutex_);
+#endif
+ return 0;
+ }
+ error_info->setjmp = 1;
+
+ volatile int cur_fbr, cur_fbc, sb_count;
while (cdef_get_next_job(cdef_sync, cdef_search_ctx, &cur_fbr, &cur_fbc,
&sb_count)) {
- av1_cdef_mse_calc_block(cdef_search_ctx, cur_fbr, cur_fbc, sb_count);
+ av1_cdef_mse_calc_block(cdef_search_ctx, error_info, cur_fbr, cur_fbc,
+ sb_count);
}
+ error_info->setjmp = 0;
return 1;
}
// Assigns CDEF search hook function and thread data to each worker.
-static void prepare_cdef_workers(MultiThreadInfo *mt_info,
- CdefSearchCtx *cdef_search_ctx,
- AVxWorkerHook hook, int num_workers) {
+static void prepare_cdef_workers(AV1_COMP *cpi, AVxWorkerHook hook,
+ int num_workers) {
+ MultiThreadInfo *mt_info = &cpi->mt_info;
for (int i = num_workers - 1; i >= 0; i--) {
AVxWorker *worker = &mt_info->workers[i];
+ EncWorkerData *thread_data = &mt_info->tile_thr_data[i];
+
+ thread_data->cpi = cpi;
worker->hook = hook;
- worker->data1 = &mt_info->cdef_sync;
- worker->data2 = cdef_search_ctx;
+ worker->data1 = thread_data;
+ worker->data2 = &mt_info->cdef_sync;
}
}
// Implements multi-threading for CDEF search.
-void av1_cdef_mse_calc_frame_mt(AV1_COMMON *cm, MultiThreadInfo *mt_info,
- CdefSearchCtx *cdef_search_ctx) {
+void av1_cdef_mse_calc_frame_mt(AV1_COMP *cpi) {
+ MultiThreadInfo *mt_info = &cpi->mt_info;
AV1CdefSync *cdef_sync = &mt_info->cdef_sync;
const int num_workers = mt_info->num_mod_workers[MOD_CDEF_SEARCH];
cdef_reset_job_info(cdef_sync);
- prepare_cdef_workers(mt_info, cdef_search_ctx, cdef_filter_block_worker_hook,
- num_workers);
+ prepare_cdef_workers(cpi, cdef_filter_block_worker_hook, num_workers);
launch_workers(mt_info, num_workers);
- sync_enc_workers(mt_info, cm, num_workers);
+ sync_enc_workers(mt_info, &cpi->common, num_workers);
}
// Computes num_workers for temporal filter multi-threading.
-static AOM_INLINE int compute_num_tf_workers(AV1_COMP *cpi) {
+static AOM_INLINE int compute_num_tf_workers(const AV1_COMP *cpi) {
// For single-pass encode, using no. of workers as per tf block size was not
// found to improve speed. Hence the thread assignment for single-pass encode
// is kept based on compute_num_enc_workers().
@@ -3058,11 +3413,10 @@
case MOD_AI:
if (cpi->oxcf.pass == AOM_RC_ONE_PASS) {
num_mod_workers = compute_num_ai_workers(cpi);
- break;
} else {
num_mod_workers = 0;
- break;
}
+ break;
default: assert(0); break;
}
return (num_mod_workers);
diff --git a/av1/encoder/ethread.h b/av1/encoder/ethread.h
index 6c4bce4..4f4c232 100644
--- a/av1/encoder/ethread.h
+++ b/av1/encoder/ethread.h
@@ -23,6 +23,7 @@
struct AV1_COMP *cpi;
struct ThreadData *td;
struct ThreadData *original_td;
+ struct aom_internal_error_info error_info;
AV1LfSync *lf_sync;
LFWorkerData *lf_data;
int start;
@@ -52,9 +53,9 @@
void av1_row_mt_mem_dealloc(AV1_COMP *cpi);
-void av1_global_motion_estimation_mt(AV1_COMP *cpi);
+void av1_row_mt_sync_mem_dealloc(AV1EncRowMultiThreadSync *row_mt_sync);
-void av1_gm_dealloc(AV1GlobalMotionSync *gm_sync_data);
+void av1_global_motion_estimation_mt(AV1_COMP *cpi);
#if !CONFIG_REALTIME_ONLY
void av1_tpl_row_mt_sync_read_dummy(AV1TplRowMultiThreadSync *tpl_mt_sync,
@@ -99,13 +100,12 @@
void av1_init_mt_sync(AV1_COMP *cpi, int is_first_pass);
#endif // CONFIG_MULTITHREAD
-int av1_get_num_mod_workers_for_alloc(PrimaryMultiThreadInfo *const p_mt_info,
+int av1_get_num_mod_workers_for_alloc(const PrimaryMultiThreadInfo *p_mt_info,
MULTI_THREADED_MODULES mod_name);
void av1_init_tile_thread_data(AV1_PRIMARY *ppi, int is_first_pass);
-void av1_cdef_mse_calc_frame_mt(AV1_COMMON *cm, MultiThreadInfo *mt_info,
- CdefSearchCtx *cdef_search_ctx);
+void av1_cdef_mse_calc_frame_mt(AV1_COMP *cpi);
void av1_cdef_mt_dealloc(AV1CdefSync *cdef_sync);
@@ -116,7 +116,7 @@
unsigned int *max_tile_size, uint32_t *const obu_header_size,
uint8_t **tile_data_start, const int num_workers);
-int av1_compute_num_enc_workers(AV1_COMP *cpi, int max_workers);
+int av1_compute_num_enc_workers(const AV1_COMP *cpi, int max_workers);
int av1_compute_num_fp_contexts(AV1_PRIMARY *ppi, AV1EncoderConfig *oxcf);
diff --git a/av1/encoder/firstpass.c b/av1/encoder/firstpass.c
index 1fad149..7ddc3e3 100644
--- a/av1/encoder/firstpass.c
+++ b/av1/encoder/firstpass.c
@@ -33,6 +33,7 @@
#include "av1/encoder/block.h"
#include "av1/encoder/dwt.h"
#include "av1/encoder/encodeframe.h"
+#include "av1/encoder/encodeframe_utils.h"
#include "av1/encoder/encodemb.h"
#include "av1/encoder/encodemv.h"
#include "av1/encoder/encoder.h"
@@ -259,6 +260,35 @@
return sr;
}
+static AOM_INLINE const search_site_config *
+av1_get_first_pass_search_site_config(const AV1_COMP *cpi, MACROBLOCK *x,
+ SEARCH_METHODS search_method) {
+ const int ref_stride = x->e_mbd.plane[0].pre[0].stride;
+
+ // For AVIF applications, even the source frames can have changing resolution,
+ // so we need to manually check for the strides :(
+ // AV1_COMP::mv_search_params.search_site_config is a compressor level cache
+ // that's shared by multiple threads. In most cases where all frames have the
+ // same resolution, the cache contains the search site config that we need.
+ const MotionVectorSearchParams *mv_search_params = &cpi->mv_search_params;
+ if (ref_stride == mv_search_params->search_site_cfg[SS_CFG_FPF]->stride) {
+ return mv_search_params->search_site_cfg[SS_CFG_FPF];
+ }
+
+ // If the cache does not contain the correct stride, then we will need to rely
+ // on the thread level config MACROBLOCK::search_site_cfg_buf. If even the
+ // thread level config doesn't match, then we need to update it.
+ search_method = search_method_lookup[search_method];
+ assert(search_method_lookup[search_method] == search_method &&
+ "The search_method_lookup table should be idempotent.");
+ if (ref_stride != x->search_site_cfg_buf[search_method].stride) {
+ av1_refresh_search_site_config(x->search_site_cfg_buf, search_method,
+ ref_stride);
+ }
+
+ return x->search_site_cfg_buf;
+}
+
static AOM_INLINE void first_pass_motion_search(AV1_COMP *cpi, MACROBLOCK *x,
const MV *ref_mv,
FULLPEL_MV *best_mv,
@@ -272,18 +302,18 @@
const int step_param = cpi->sf.fp_sf.reduce_mv_step_param + sr;
const search_site_config *first_pass_search_sites =
- cpi->mv_search_params.search_site_cfg[SS_CFG_FPF];
+ av1_get_first_pass_search_site_config(cpi, x, NSTEP);
const int fine_search_interval =
cpi->is_screen_content_type && cpi->common.features.allow_intrabc;
FULLPEL_MOTION_SEARCH_PARAMS ms_params;
av1_make_default_fullpel_ms_params(&ms_params, cpi, x, bsize, ref_mv,
- start_mv, first_pass_search_sites,
+ start_mv, first_pass_search_sites, NSTEP,
fine_search_interval);
- av1_set_mv_search_method(&ms_params, first_pass_search_sites, NSTEP);
FULLPEL_MV this_best_mv;
+ FULLPEL_MV_STATS best_mv_stats;
tmp_err = av1_full_pixel_search(start_mv, &ms_params, step_param, NULL,
- &this_best_mv, NULL);
+ &this_best_mv, &best_mv_stats, NULL);
if (tmp_err < INT_MAX) {
aom_variance_fn_ptr_t v_fn_ptr = cpi->ppi->fn_ptr[bsize];
@@ -744,9 +774,8 @@
if ((current_frame->frame_number > 1) && golden_frame != NULL) {
FULLPEL_MV tmp_mv = kZeroFullMv;
// Assume 0,0 motion with no mv overhead.
- xd->plane[0].pre[0].buf = golden_frame->y_buffer + recon_yoffset;
- xd->plane[0].pre[0].stride = golden_frame->y_stride;
- xd->plane[0].pre[0].width = golden_frame->y_width;
+ av1_setup_pre_planes(xd, 0, golden_frame, 0, 0, NULL, 1);
+ xd->plane[0].pre[0].buf += recon_yoffset;
gf_motion_error =
get_prediction_error_bitdepth(is_high_bitdepth, bitdepth, bsize,
&x->plane[0].src, &xd->plane[0].pre[0]);
@@ -1032,9 +1061,11 @@
}
}
-static void free_firstpass_data(FirstPassData *firstpass_data) {
+void av1_free_firstpass_data(FirstPassData *firstpass_data) {
aom_free(firstpass_data->raw_motion_err_list);
+ firstpass_data->raw_motion_err_list = NULL;
aom_free(firstpass_data->mb_stats);
+ firstpass_data->mb_stats = NULL;
}
int av1_get_unit_rows_in_tile(const TileInfo *tile,
@@ -1073,17 +1104,7 @@
AV1_COMMON *const cm = &cpi->common;
const int tile_cols = cm->tiles.cols;
const int tile_rows = cm->tiles.rows;
- const int num_planes = av1_num_planes(&cpi->common);
- for (int plane = 0; plane < num_planes; plane++) {
- const int subsampling_xy =
- plane ? cm->seq_params->subsampling_x + cm->seq_params->subsampling_y
- : 0;
- const int sb_size = MAX_SB_SQUARE >> subsampling_xy;
- CHECK_MEM_ERROR(
- cm, cpi->td.mb.plane[plane].src_diff,
- (int16_t *)aom_memalign(
- 32, sizeof(*cpi->td.mb.plane[plane].src_diff) * sb_size));
- }
+
for (int tile_row = 0; tile_row < tile_rows; ++tile_row) {
for (int tile_col = 0; tile_col < tile_cols; ++tile_col) {
TileDataEnc *const tile_data =
@@ -1091,12 +1112,6 @@
first_pass_tile(cpi, &cpi->td, tile_data, fp_block_size);
}
}
- for (int plane = 0; plane < num_planes; plane++) {
- if (cpi->td.mb.plane[plane].src_diff) {
- aom_free(cpi->td.mb.plane[plane].src_diff);
- cpi->td.mb.plane[plane].src_diff = NULL;
- }
- }
}
void av1_first_pass_row(AV1_COMP *cpi, ThreadData *td, TileDataEnc *tile_data,
@@ -1187,6 +1202,16 @@
enc_row_mt->sync_read_ptr(row_mt_sync, unit_row_in_tile, unit_col_in_tile);
+#if CONFIG_MULTITHREAD
+ if (cpi->ppi->p_mt_info.num_workers > 1) {
+ pthread_mutex_lock(enc_row_mt->mutex_);
+ bool firstpass_mt_exit = enc_row_mt->firstpass_mt_exit;
+ pthread_mutex_unlock(enc_row_mt->mutex_);
+ // Exit in case any worker has encountered an error.
+ if (firstpass_mt_exit) return;
+ }
+#endif
+
if (unit_col_in_tile == 0) {
last_mv = *first_top_mv;
}
@@ -1246,7 +1271,7 @@
setup_firstpass_data(cm, &cpi->firstpass_data, unit_rows, unit_cols);
FRAME_STATS *mb_stats = cpi->firstpass_data.mb_stats;
FRAME_STATS stats = accumulate_frame_stats(mb_stats, unit_rows, unit_cols);
- free_firstpass_data(&cpi->firstpass_data);
+ av1_free_firstpass_data(&cpi->firstpass_data);
update_firstpass_stats(cpi, &stats, 1.0, current_frame->frame_number,
ts_duration, BLOCK_16X16);
}
@@ -1365,6 +1390,7 @@
av1_init_mode_probs(cm->fc);
av1_init_mv_probs(cm);
av1_initialize_rd_consts(cpi);
+ av1_alloc_src_diff_buf(cm, &cpi->td.mb);
enc_row_mt->sync_read_ptr = av1_row_mt_sync_read_dummy;
enc_row_mt->sync_write_ptr = av1_row_mt_sync_write_dummy;
@@ -1382,7 +1408,8 @@
frame_is_intra_only(cm) ? 0 : unit_rows * unit_cols;
const double raw_err_stdev =
raw_motion_error_stdev(raw_motion_err_list, total_raw_motion_err_count);
- free_firstpass_data(&cpi->firstpass_data);
+ av1_free_firstpass_data(&cpi->firstpass_data);
+ av1_dealloc_src_diff_buf(&cpi->td.mb, av1_num_planes(cm));
// Clamp the image start to rows/2. This number of rows is discarded top
// and bottom as dead data so rows / 2 means the frame is blank.
diff --git a/av1/encoder/firstpass.h b/av1/encoder/firstpass.h
index e18e9e4..d01363a 100644
--- a/av1/encoder/firstpass.h
+++ b/av1/encoder/firstpass.h
@@ -568,6 +568,8 @@
const BLOCK_SIZE fp_block_size);
void av1_end_first_pass(struct AV1_COMP *cpi);
+void av1_free_firstpass_data(FirstPassData *firstpass_data);
+
void av1_twopass_zero_stats(FIRSTPASS_STATS *section);
void av1_accumulate_stats(FIRSTPASS_STATS *section,
const FIRSTPASS_STATS *frame);
diff --git a/av1/encoder/global_motion.c b/av1/encoder/global_motion.c
index bc5e186..73910de 100644
--- a/av1/encoder/global_motion.c
+++ b/av1/encoder/global_motion.c
@@ -30,6 +30,83 @@
// Border over which to compute the global motion
#define ERRORADV_BORDER 0
+/* clang-format off */
+// Error metric used for global motion evaluation.
+// For 8-bit input, the pixel error used to index this table will always
+// be between -255 and +255. But for 10- and 12-bit input, we use interpolation
+// which means that we need to support indices of -256 and +256 as well.
+// Therefore, the table is offset so that logical index 0 corresponds to
+// error_measure_lut[256].
+const int error_measure_lut[513] = {
+ // pow 0.7
+ 16384, 16384, 16339, 16294, 16249, 16204, 16158, 16113,
+ 16068, 16022, 15977, 15932, 15886, 15840, 15795, 15749,
+ 15703, 15657, 15612, 15566, 15520, 15474, 15427, 15381,
+ 15335, 15289, 15242, 15196, 15149, 15103, 15056, 15010,
+ 14963, 14916, 14869, 14822, 14775, 14728, 14681, 14634,
+ 14587, 14539, 14492, 14445, 14397, 14350, 14302, 14254,
+ 14206, 14159, 14111, 14063, 14015, 13967, 13918, 13870,
+ 13822, 13773, 13725, 13676, 13628, 13579, 13530, 13481,
+ 13432, 13383, 13334, 13285, 13236, 13187, 13137, 13088,
+ 13038, 12988, 12939, 12889, 12839, 12789, 12739, 12689,
+ 12639, 12588, 12538, 12487, 12437, 12386, 12335, 12285,
+ 12234, 12183, 12132, 12080, 12029, 11978, 11926, 11875,
+ 11823, 11771, 11719, 11667, 11615, 11563, 11511, 11458,
+ 11406, 11353, 11301, 11248, 11195, 11142, 11089, 11036,
+ 10982, 10929, 10875, 10822, 10768, 10714, 10660, 10606,
+ 10552, 10497, 10443, 10388, 10333, 10279, 10224, 10168,
+ 10113, 10058, 10002, 9947, 9891, 9835, 9779, 9723,
+ 9666, 9610, 9553, 9497, 9440, 9383, 9326, 9268,
+ 9211, 9153, 9095, 9037, 8979, 8921, 8862, 8804,
+ 8745, 8686, 8627, 8568, 8508, 8449, 8389, 8329,
+ 8269, 8208, 8148, 8087, 8026, 7965, 7903, 7842,
+ 7780, 7718, 7656, 7593, 7531, 7468, 7405, 7341,
+ 7278, 7214, 7150, 7086, 7021, 6956, 6891, 6826,
+ 6760, 6695, 6628, 6562, 6495, 6428, 6361, 6293,
+ 6225, 6157, 6089, 6020, 5950, 5881, 5811, 5741,
+ 5670, 5599, 5527, 5456, 5383, 5311, 5237, 5164,
+ 5090, 5015, 4941, 4865, 4789, 4713, 4636, 4558,
+ 4480, 4401, 4322, 4242, 4162, 4080, 3998, 3916,
+ 3832, 3748, 3663, 3577, 3490, 3402, 3314, 3224,
+ 3133, 3041, 2948, 2854, 2758, 2661, 2562, 2461,
+ 2359, 2255, 2148, 2040, 1929, 1815, 1698, 1577,
+ 1452, 1323, 1187, 1045, 894, 731, 550, 339,
+ 0, 339, 550, 731, 894, 1045, 1187, 1323,
+ 1452, 1577, 1698, 1815, 1929, 2040, 2148, 2255,
+ 2359, 2461, 2562, 2661, 2758, 2854, 2948, 3041,
+ 3133, 3224, 3314, 3402, 3490, 3577, 3663, 3748,
+ 3832, 3916, 3998, 4080, 4162, 4242, 4322, 4401,
+ 4480, 4558, 4636, 4713, 4789, 4865, 4941, 5015,
+ 5090, 5164, 5237, 5311, 5383, 5456, 5527, 5599,
+ 5670, 5741, 5811, 5881, 5950, 6020, 6089, 6157,
+ 6225, 6293, 6361, 6428, 6495, 6562, 6628, 6695,
+ 6760, 6826, 6891, 6956, 7021, 7086, 7150, 7214,
+ 7278, 7341, 7405, 7468, 7531, 7593, 7656, 7718,
+ 7780, 7842, 7903, 7965, 8026, 8087, 8148, 8208,
+ 8269, 8329, 8389, 8449, 8508, 8568, 8627, 8686,
+ 8745, 8804, 8862, 8921, 8979, 9037, 9095, 9153,
+ 9211, 9268, 9326, 9383, 9440, 9497, 9553, 9610,
+ 9666, 9723, 9779, 9835, 9891, 9947, 10002, 10058,
+ 10113, 10168, 10224, 10279, 10333, 10388, 10443, 10497,
+ 10552, 10606, 10660, 10714, 10768, 10822, 10875, 10929,
+ 10982, 11036, 11089, 11142, 11195, 11248, 11301, 11353,
+ 11406, 11458, 11511, 11563, 11615, 11667, 11719, 11771,
+ 11823, 11875, 11926, 11978, 12029, 12080, 12132, 12183,
+ 12234, 12285, 12335, 12386, 12437, 12487, 12538, 12588,
+ 12639, 12689, 12739, 12789, 12839, 12889, 12939, 12988,
+ 13038, 13088, 13137, 13187, 13236, 13285, 13334, 13383,
+ 13432, 13481, 13530, 13579, 13628, 13676, 13725, 13773,
+ 13822, 13870, 13918, 13967, 14015, 14063, 14111, 14159,
+ 14206, 14254, 14302, 14350, 14397, 14445, 14492, 14539,
+ 14587, 14634, 14681, 14728, 14775, 14822, 14869, 14916,
+ 14963, 15010, 15056, 15103, 15149, 15196, 15242, 15289,
+ 15335, 15381, 15427, 15474, 15520, 15566, 15612, 15657,
+ 15703, 15749, 15795, 15840, 15886, 15932, 15977, 16022,
+ 16068, 16113, 16158, 16204, 16249, 16294, 16339, 16384,
+ 16384,
+};
+/* clang-format on */
+
int av1_is_enough_erroradvantage(double best_erroradvantage, int params_cost) {
return best_erroradvantage < erroradv_tr &&
best_erroradvantage * params_cost < erroradv_prod_tr;
@@ -110,15 +187,76 @@
}
#if CONFIG_AV1_HIGHBITDEPTH
-static int64_t highbd_warp_error(
- WarpedMotionParams *wm, const uint16_t *const ref, int width, int height,
- int stride, const uint16_t *const dst, int p_col, int p_row, int p_width,
- int p_height, int p_stride, int subsampling_x, int subsampling_y, int bd,
- int64_t best_error, uint8_t *segment_map, int segment_map_stride) {
+static INLINE int generic_sad_highbd(const uint16_t *const ref, int ref_stride,
+ const uint16_t *const dst, int dst_stride,
+ int p_width, int p_height) {
+ // This function should only be called for patches smaller than
+ // WARP_ERROR_BLOCK x WARP_ERROR_BLOCK. This keeps the number of pixels
+ // small enough that we don't need a 64-bit accumulator
+ assert(p_width <= WARP_ERROR_BLOCK && p_height <= WARP_ERROR_BLOCK);
+
+ int sad = 0;
+ for (int i = 0; i < p_height; ++i) {
+ for (int j = 0; j < p_width; ++j) {
+ sad += abs(dst[j + i * dst_stride] - ref[j + i * ref_stride]);
+ }
+ }
+ return sad;
+}
+
+#if WARP_ERROR_BLOCK != 32
+#error "Need to change SAD call size in highbd_segmented_frame_error"
+#endif // WARP_ERROR_BLOCK != 32
+static int64_t highbd_segmented_frame_error(
+ const uint16_t *const ref, int ref_stride, const uint16_t *const dst,
+ int dst_stride, int p_width, int p_height, int bd, uint8_t *segment_map,
+ int segment_map_stride) {
+ (void)bd;
+ int patch_w, patch_h;
+ const int error_bsize_w = AOMMIN(p_width, WARP_ERROR_BLOCK);
+ const int error_bsize_h = AOMMIN(p_height, WARP_ERROR_BLOCK);
+ int64_t sum_error = 0;
+ for (int i = 0; i < p_height; i += WARP_ERROR_BLOCK) {
+ for (int j = 0; j < p_width; j += WARP_ERROR_BLOCK) {
+ int seg_x = j >> WARP_ERROR_BLOCK_LOG;
+ int seg_y = i >> WARP_ERROR_BLOCK_LOG;
+ // Only compute the error if this block contains inliers from the motion
+ // model
+ if (!segment_map[seg_y * segment_map_stride + seg_x]) continue;
+
+ // avoid computing error into the frame padding
+ patch_w = AOMMIN(error_bsize_w, p_width - j);
+ patch_h = AOMMIN(error_bsize_h, p_height - i);
+
+ if (patch_w == WARP_ERROR_BLOCK && patch_h == WARP_ERROR_BLOCK) {
+ sum_error += aom_highbd_sad32x32(
+ CONVERT_TO_BYTEPTR(ref + j + i * ref_stride), ref_stride,
+ CONVERT_TO_BYTEPTR(dst + j + i * dst_stride), dst_stride);
+ } else {
+ sum_error += generic_sad_highbd(ref + j + i * ref_stride, ref_stride,
+ dst + j + i * dst_stride, dst_stride,
+ patch_w, patch_h);
+ }
+ }
+ }
+ return sum_error;
+}
+
+#if WARP_ERROR_BLOCK != 32
+#error "Need to change SAD call size in highbd_warp_error"
+#endif // WARP_ERROR_BLOCK != 32
+static int64_t highbd_warp_error(WarpedMotionParams *wm,
+ const uint16_t *const ref, int ref_width,
+ int ref_height, int ref_stride,
+ const uint16_t *const dst, int dst_stride,
+ int p_col, int p_row, int p_width,
+ int p_height, int subsampling_x,
+ int subsampling_y, int bd, int64_t best_error,
+ uint8_t *segment_map, int segment_map_stride) {
int64_t gm_sumerr = 0;
const int error_bsize_w = AOMMIN(p_width, WARP_ERROR_BLOCK);
const int error_bsize_h = AOMMIN(p_height, WARP_ERROR_BLOCK);
- uint16_t tmp[WARP_ERROR_BLOCK * WARP_ERROR_BLOCK];
+ DECLARE_ALIGNED(32, uint16_t, tmp[WARP_ERROR_BLOCK * WARP_ERROR_BLOCK]);
ConvolveParams conv_params = get_conv_params(0, 0, bd);
conv_params.use_dist_wtd_comp_avg = 0;
@@ -131,14 +269,22 @@
if (!segment_map[seg_y * segment_map_stride + seg_x]) continue;
// avoid warping extra 8x8 blocks in the padded region of the frame
// when p_width and p_height are not multiples of WARP_ERROR_BLOCK
- const int warp_w = AOMMIN(error_bsize_w, p_col + p_width - j);
- const int warp_h = AOMMIN(error_bsize_h, p_row + p_height - i);
- highbd_warp_plane(wm, ref, width, height, stride, tmp, j, i, warp_w,
- warp_h, WARP_ERROR_BLOCK, subsampling_x, subsampling_y,
- bd, &conv_params);
- gm_sumerr += av1_calc_highbd_frame_error(tmp, WARP_ERROR_BLOCK,
- dst + j + i * p_stride, warp_w,
- warp_h, p_stride, bd);
+ const int warp_w = AOMMIN(error_bsize_w, p_col + ref_width - j);
+ const int warp_h = AOMMIN(error_bsize_h, p_row + ref_height - i);
+ highbd_warp_plane(wm, ref, ref_width, ref_height, ref_stride, tmp, j, i,
+ warp_w, warp_h, WARP_ERROR_BLOCK, subsampling_x,
+ subsampling_y, bd, &conv_params);
+
+ if (warp_w == WARP_ERROR_BLOCK && warp_h == WARP_ERROR_BLOCK) {
+ gm_sumerr += aom_highbd_sad32x32(
+ CONVERT_TO_BYTEPTR(tmp), WARP_ERROR_BLOCK,
+ CONVERT_TO_BYTEPTR(dst + j + i * dst_stride), dst_stride);
+ } else {
+ gm_sumerr +=
+ generic_sad_highbd(tmp, WARP_ERROR_BLOCK, dst + j + i * dst_stride,
+ dst_stride, warp_w, warp_h);
+ }
+
if (gm_sumerr > best_error) return INT64_MAX;
}
}
@@ -146,10 +292,67 @@
}
#endif
+static INLINE int generic_sad(const uint8_t *const ref, int ref_stride,
+ const uint8_t *const dst, int dst_stride,
+ int p_width, int p_height) {
+ // This function should only be called for patches smaller than
+ // WARP_ERROR_BLOCK x WARP_ERROR_BLOCK. This keeps the number of pixels
+ // small enough that we don't need a 64-bit accumulator
+ assert(p_width <= WARP_ERROR_BLOCK && p_height <= WARP_ERROR_BLOCK);
+
+ int sad = 0;
+ for (int i = 0; i < p_height; ++i) {
+ for (int j = 0; j < p_width; ++j) {
+ sad += abs(dst[j + i * dst_stride] - ref[j + i * ref_stride]);
+ }
+ }
+ return sad;
+}
+
+#if WARP_ERROR_BLOCK != 32
+#error "Need to change SAD call size in segmented_warp_error"
+#endif // WARP_ERROR_BLOCK != 32
+static int64_t segmented_frame_error(const uint8_t *const ref, int ref_stride,
+ const uint8_t *const dst, int dst_stride,
+ int p_width, int p_height,
+ uint8_t *segment_map,
+ int segment_map_stride) {
+ int patch_w, patch_h;
+ const int error_bsize_w = AOMMIN(p_width, WARP_ERROR_BLOCK);
+ const int error_bsize_h = AOMMIN(p_height, WARP_ERROR_BLOCK);
+ int64_t sum_error = 0;
+ for (int i = 0; i < p_height; i += WARP_ERROR_BLOCK) {
+ for (int j = 0; j < p_width; j += WARP_ERROR_BLOCK) {
+ int seg_x = j >> WARP_ERROR_BLOCK_LOG;
+ int seg_y = i >> WARP_ERROR_BLOCK_LOG;
+ // Only compute the error if this block contains inliers from the motion
+ // model
+ if (!segment_map[seg_y * segment_map_stride + seg_x]) continue;
+
+ // avoid computing error into the frame padding
+ patch_w = AOMMIN(error_bsize_w, p_width - j);
+ patch_h = AOMMIN(error_bsize_h, p_height - i);
+
+ if (patch_w == WARP_ERROR_BLOCK && patch_h == WARP_ERROR_BLOCK) {
+ sum_error += aom_sad32x32(ref + j + i * ref_stride, ref_stride,
+ dst + j + i * dst_stride, dst_stride);
+ } else {
+ sum_error +=
+ generic_sad(ref + j + i * ref_stride, ref_stride,
+ dst + j + i * dst_stride, dst_stride, patch_w, patch_h);
+ }
+ }
+ }
+ return sum_error;
+}
+
+#if WARP_ERROR_BLOCK != 32
+#error "Need to change SAD call size in warp_error"
+#endif // WARP_ERROR_BLOCK != 32
static int64_t warp_error(WarpedMotionParams *wm, const uint8_t *const ref,
- int width, int height, int stride,
- const uint8_t *const dst, int p_col, int p_row,
- int p_width, int p_height, int p_stride,
+ int ref_width, int ref_height, int ref_stride,
+ const uint8_t *const dst, int dst_stride, int p_col,
+ int p_row, int p_width, int p_height,
int subsampling_x, int subsampling_y,
int64_t best_error, uint8_t *segment_map,
int segment_map_stride) {
@@ -170,62 +373,72 @@
if (!segment_map[seg_y * segment_map_stride + seg_x]) continue;
// avoid warping extra 8x8 blocks in the padded region of the frame
// when p_width and p_height are not multiples of WARP_ERROR_BLOCK
- warp_w = AOMMIN(error_bsize_w, p_col + p_width - j);
- warp_h = AOMMIN(error_bsize_h, p_row + p_height - i);
- warp_plane(wm, ref, width, height, stride, tmp, j, i, warp_w, warp_h,
- WARP_ERROR_BLOCK, subsampling_x, subsampling_y, &conv_params);
+ warp_w = AOMMIN(error_bsize_w, p_col + ref_width - j);
+ warp_h = AOMMIN(error_bsize_h, p_row + ref_height - i);
+ warp_plane(wm, ref, ref_width, ref_height, ref_stride, tmp, j, i, warp_w,
+ warp_h, WARP_ERROR_BLOCK, subsampling_x, subsampling_y,
+ &conv_params);
- gm_sumerr +=
- av1_calc_frame_error(tmp, WARP_ERROR_BLOCK, dst + j + i * p_stride,
- warp_w, warp_h, p_stride);
+ if (warp_w == WARP_ERROR_BLOCK && warp_h == WARP_ERROR_BLOCK) {
+ gm_sumerr += aom_sad32x32(tmp, WARP_ERROR_BLOCK,
+ dst + j + i * dst_stride, dst_stride);
+ } else {
+ gm_sumerr +=
+ generic_sad(tmp, WARP_ERROR_BLOCK, dst + j + i * dst_stride,
+ dst_stride, warp_w, warp_h);
+ }
+
if (gm_sumerr > best_error) return INT64_MAX;
}
}
return gm_sumerr;
}
-int64_t av1_warp_error(WarpedMotionParams *wm, int use_hbd, int bd,
- const uint8_t *ref, int width, int height, int stride,
- uint8_t *dst, int p_col, int p_row, int p_width,
- int p_height, int p_stride, int subsampling_x,
- int subsampling_y, int64_t best_error,
- uint8_t *segment_map, int segment_map_stride) {
- force_wmtype(wm, wm->wmtype);
- assert(wm->wmtype <= AFFINE);
- if (!av1_get_shear_params(wm)) return INT64_MAX;
+int64_t av1_segmented_frame_error(int use_hbd, int bd, const uint8_t *ref,
+ int ref_stride, uint8_t *dst, int dst_stride,
+ int p_width, int p_height,
+ uint8_t *segment_map,
+ int segment_map_stride) {
#if CONFIG_AV1_HIGHBITDEPTH
- if (use_hbd)
- return highbd_warp_error(wm, CONVERT_TO_SHORTPTR(ref), width, height,
- stride, CONVERT_TO_SHORTPTR(dst), p_col, p_row,
- p_width, p_height, p_stride, subsampling_x,
- subsampling_y, bd, best_error, segment_map,
- segment_map_stride);
+ if (use_hbd) {
+ return highbd_segmented_frame_error(
+ CONVERT_TO_SHORTPTR(ref), ref_stride, CONVERT_TO_SHORTPTR(dst),
+ dst_stride, p_width, p_height, bd, segment_map, segment_map_stride);
+ }
#endif
(void)use_hbd;
(void)bd;
- return warp_error(wm, ref, width, height, stride, dst, p_col, p_row, p_width,
- p_height, p_stride, subsampling_x, subsampling_y,
- best_error, segment_map, segment_map_stride);
+ return segmented_frame_error(ref, ref_stride, dst, dst_stride, p_width,
+ p_height, segment_map, segment_map_stride);
}
-// Factors used to calculate the thresholds for av1_warp_error
-static double thresh_factors[GM_MAX_REFINEMENT_STEPS] = { 1.25, 1.20, 1.15,
- 1.10, 1.05 };
-
-static INLINE int64_t calc_approx_erroradv_threshold(
- double scaling_factor, int64_t erroradv_threshold) {
- return erroradv_threshold <
- (int64_t)(((double)INT64_MAX / scaling_factor) + 0.5)
- ? (int64_t)(scaling_factor * erroradv_threshold + 0.5)
- : INT64_MAX;
+int64_t av1_warp_error(WarpedMotionParams *wm, int use_hbd, int bd,
+ const uint8_t *ref, int ref_width, int ref_height,
+ int ref_stride, uint8_t *dst, int dst_stride, int p_col,
+ int p_row, int p_width, int p_height, int subsampling_x,
+ int subsampling_y, int64_t best_error,
+ uint8_t *segment_map, int segment_map_stride) {
+ if (!av1_get_shear_params(wm)) return INT64_MAX;
+#if CONFIG_AV1_HIGHBITDEPTH
+ if (use_hbd)
+ return highbd_warp_error(wm, CONVERT_TO_SHORTPTR(ref), ref_width,
+ ref_height, ref_stride, CONVERT_TO_SHORTPTR(dst),
+ dst_stride, p_col, p_row, p_width, p_height,
+ subsampling_x, subsampling_y, bd, best_error,
+ segment_map, segment_map_stride);
+#endif
+ (void)use_hbd;
+ (void)bd;
+ return warp_error(wm, ref, ref_width, ref_height, ref_stride, dst, dst_stride,
+ p_col, p_row, p_width, p_height, subsampling_x,
+ subsampling_y, best_error, segment_map, segment_map_stride);
}
int64_t av1_refine_integerized_param(
WarpedMotionParams *wm, TransformationType wmtype, int use_hbd, int bd,
uint8_t *ref, int r_width, int r_height, int r_stride, uint8_t *dst,
int d_width, int d_height, int d_stride, int n_refinements,
- int64_t best_frame_error, uint8_t *segment_map, int segment_map_stride,
- int64_t erroradv_threshold) {
+ int64_t ref_frame_error, uint8_t *segment_map, int segment_map_stride) {
static const int max_trans_model_params[TRANS_TYPES] = { 0, 2, 4, 6 };
const int border = ERRORADV_BORDER;
int i = 0, p;
@@ -238,36 +451,51 @@
int32_t best_param;
force_wmtype(wm, wmtype);
- best_error =
- av1_warp_error(wm, use_hbd, bd, ref, r_width, r_height, r_stride,
- dst + border * d_stride + border, border, border,
- d_width - 2 * border, d_height - 2 * border, d_stride, 0,
- 0, best_frame_error, segment_map, segment_map_stride);
+ wm->wmtype = get_wmtype(wm);
if (n_refinements == 0) {
- wm->wmtype = get_wmtype(wm);
- return best_error;
+ // Compute the maximum error value that will be accepted, so that
+ // av1_warp_error can terminate early if it proves the model will not
+ // be accepted.
+ int64_t selection_threshold = (int64_t)lrint(ref_frame_error * erroradv_tr);
+ return av1_warp_error(wm, use_hbd, bd, ref, r_width, r_height, r_stride,
+ dst + border * d_stride + border, d_stride, border,
+ border, d_width - 2 * border, d_height - 2 * border,
+ 0, 0, selection_threshold, segment_map,
+ segment_map_stride);
}
- best_error = AOMMIN(best_error, best_frame_error);
+ // When refining, use a slightly higher threshold for the initial error
+ // calculation - see comment above erroradv_early_tr for why.
+ int64_t selection_threshold =
+ (int64_t)lrint(ref_frame_error * erroradv_early_tr);
+ best_error =
+ av1_warp_error(wm, use_hbd, bd, ref, r_width, r_height, r_stride,
+ dst + border * d_stride + border, d_stride, border, border,
+ d_width - 2 * border, d_height - 2 * border, 0, 0,
+ selection_threshold, segment_map, segment_map_stride);
+
+ if (best_error > selection_threshold) {
+ return INT64_MAX;
+ }
+
step = 1 << (n_refinements - 1);
for (i = 0; i < n_refinements; i++, step >>= 1) {
- int64_t error_adv_thresh =
- calc_approx_erroradv_threshold(thresh_factors[i], erroradv_threshold);
for (p = 0; p < n_params; ++p) {
int step_dir = 0;
- // Skip searches for parameters that are forced to be 0
param = param_mat + p;
curr_param = *param;
best_param = curr_param;
// look to the left
+ // Note: We have to use force_wmtype() to keep the proper symmetry for
+ // ROTZOOM type models
*param = add_param_offset(p, curr_param, -step);
+ force_wmtype(wm, wmtype);
step_error =
av1_warp_error(wm, use_hbd, bd, ref, r_width, r_height, r_stride,
- dst + border * d_stride + border, border, border,
- d_width - 2 * border, d_height - 2 * border, d_stride,
- 0, 0, AOMMIN(best_error, error_adv_thresh),
- segment_map, segment_map_stride);
+ dst + border * d_stride + border, d_stride, border,
+ border, d_width - 2 * border, d_height - 2 * border, 0,
+ 0, best_error, segment_map, segment_map_stride);
if (step_error < best_error) {
best_error = step_error;
best_param = *param;
@@ -276,40 +504,42 @@
// look to the right
*param = add_param_offset(p, curr_param, step);
+ force_wmtype(wm, wmtype);
step_error =
av1_warp_error(wm, use_hbd, bd, ref, r_width, r_height, r_stride,
- dst + border * d_stride + border, border, border,
- d_width - 2 * border, d_height - 2 * border, d_stride,
- 0, 0, AOMMIN(best_error, error_adv_thresh),
- segment_map, segment_map_stride);
+ dst + border * d_stride + border, d_stride, border,
+ border, d_width - 2 * border, d_height - 2 * border, 0,
+ 0, best_error, segment_map, segment_map_stride);
if (step_error < best_error) {
best_error = step_error;
best_param = *param;
step_dir = 1;
}
- *param = best_param;
// look to the direction chosen above repeatedly until error increases
// for the biggest step size
while (step_dir) {
*param = add_param_offset(p, best_param, step * step_dir);
+ force_wmtype(wm, wmtype);
step_error =
av1_warp_error(wm, use_hbd, bd, ref, r_width, r_height, r_stride,
- dst + border * d_stride + border, border, border,
- d_width - 2 * border, d_height - 2 * border,
- d_stride, 0, 0, AOMMIN(best_error, error_adv_thresh),
- segment_map, segment_map_stride);
+ dst + border * d_stride + border, d_stride, border,
+ border, d_width - 2 * border, d_height - 2 * border,
+ 0, 0, best_error, segment_map, segment_map_stride);
if (step_error < best_error) {
best_error = step_error;
best_param = *param;
} else {
- *param = best_param;
step_dir = 0;
}
}
+
+ // Restore best parameter value so far
+ *param = best_param;
+ force_wmtype(wm, wmtype);
}
}
- force_wmtype(wm, wmtype);
+
wm->wmtype = get_wmtype(wm);
return best_error;
}
diff --git a/av1/encoder/global_motion.h b/av1/encoder/global_motion.h
index cf1d0fd..8c9c60f 100644
--- a/av1/encoder/global_motion.h
+++ b/av1/encoder/global_motion.h
@@ -40,7 +40,7 @@
// Pointer to hold inliers from motion model.
uint8_t *segment_map;
-} GlobalMotionThreadData;
+} GlobalMotionData;
typedef struct {
// Holds the mapping of each thread to past/future direction.
@@ -63,43 +63,82 @@
// Data related to assigning jobs for global motion multi-threading.
JobInfo job_info;
- // Data specific to each worker in global motion multi-threading.
- // thread_data[i] stores the thread specific data for worker 'i'.
- GlobalMotionThreadData *thread_data;
-
#if CONFIG_MULTITHREAD
// Mutex lock used while dispatching jobs.
pthread_mutex_t *mutex_;
#endif
- // Width and height for which segment_map is allocated for each thread.
- int allocated_width;
- int allocated_height;
-
- // Number of workers for which thread_data is allocated.
- int8_t allocated_workers;
+ // Initialized to false, set to true by the worker thread that encounters an
+ // error in order to abort the processing of other worker threads.
+ bool gm_mt_exit;
} AV1GlobalMotionSync;
void av1_convert_model_to_params(const double *params,
WarpedMotionParams *model);
-// TODO(sarahparker) These need to be retuned for speed 0 and 1 to
-// maximize gains from segmented error metric
+// Criteria for accepting a global motion model
static const double erroradv_tr = 0.65;
static const double erroradv_prod_tr = 20000;
+// Early exit threshold for global motion refinement
+// This is set slightly higher than erroradv_tr, as a compromise between
+// two factors:
+//
+// 1) By rejecting un-promising models early, we can reduce the encode time
+// spent trying to refine them
+//
+// 2) When we refine a model, its error may decrease to below the acceptance
+// threshold even if the model is initially above the threshold
+static const double erroradv_early_tr = 0.70;
+
int av1_is_enough_erroradvantage(double best_erroradvantage, int params_cost);
void av1_compute_feature_segmentation_map(uint8_t *segment_map, int width,
int height, int *inliers,
int num_inliers);
+extern const int error_measure_lut[513];
+
+static INLINE int error_measure(int err) {
+ return error_measure_lut[256 + err];
+}
+
+#if CONFIG_AV1_HIGHBITDEPTH
+static INLINE int highbd_error_measure(int err, int bd) {
+ const int b = bd - 8;
+ const int bmask = (1 << b) - 1;
+ const int v = (1 << b);
+
+ // Split error into two parts and do an interpolated table lookup
+ // To compute the table index and interpolation value, we want to calculate
+ // the quotient and remainder of err / 2^b. But it is very important that
+ // the division must round down, and the remainder must be positive,
+ // ie. in the range [0, 2^b).
+ //
+ // In C, the >> and & operators do what we want, but the / and % operators
+ // give the wrong results for negative inputs. So we must use >> and & here.
+ //
+ // For example, if bd == 10 and err == -5, compare the results:
+ // (-5) >> 2 = -2, (-5) & 3 = 3
+ // vs. (-5) / 4 = -1, (-5) % 4 = -1
+ const int e1 = err >> b;
+ const int e2 = err & bmask;
+ return error_measure_lut[256 + e1] * (v - e2) +
+ error_measure_lut[257 + e1] * e2;
+}
+#endif // CONFIG_AV1_HIGHBITDEPTH
+
+int64_t av1_segmented_frame_error(int use_hbd, int bd, const uint8_t *ref,
+ int ref_stride, uint8_t *dst, int dst_stride,
+ int p_width, int p_height,
+ uint8_t *segment_map, int segment_map_stride);
+
// Returns the error between the result of applying motion 'wm' to the frame
// described by 'ref' and the frame described by 'dst'.
int64_t av1_warp_error(WarpedMotionParams *wm, int use_hbd, int bd,
- const uint8_t *ref, int width, int height, int stride,
- uint8_t *dst, int p_col, int p_row, int p_width,
- int p_height, int p_stride, int subsampling_x,
+ const uint8_t *ref, int ref_width, int ref_height,
+ int ref_stride, uint8_t *dst, int dst_stride, int p_col,
+ int p_row, int p_width, int p_height, int subsampling_x,
int subsampling_y, int64_t best_error,
uint8_t *segment_map, int segment_map_stride);
@@ -110,8 +149,7 @@
WarpedMotionParams *wm, TransformationType wmtype, int use_hbd, int bd,
uint8_t *ref, int r_width, int r_height, int r_stride, uint8_t *dst,
int d_width, int d_height, int d_stride, int n_refinements,
- int64_t best_frame_error, uint8_t *segment_map, int segment_map_stride,
- int64_t erroradv_threshold);
+ int64_t ref_frame_error, uint8_t *segment_map, int segment_map_stride);
#ifdef __cplusplus
} // extern "C"
diff --git a/av1/encoder/global_motion_facade.c b/av1/encoder/global_motion_facade.c
index 1a00cbb..02a4e70 100644
--- a/av1/encoder/global_motion_facade.c
+++ b/av1/encoder/global_motion_facade.c
@@ -20,8 +20,9 @@
#include "av1/encoder/rdopt.h"
#include "av1/encoder/global_motion_facade.h"
-// Highest motion model to search.
-#define GLOBAL_TRANS_TYPES_ENC 3
+// Range of model types to search
+#define FIRST_GLOBAL_TRANS_TYPE ROTZOOM
+#define LAST_GLOBAL_TRANS_TYPE ROTZOOM
// Computes the cost for the warp parameters.
static int gm_get_params_cost(const WarpedMotionParams *gm,
@@ -73,47 +74,46 @@
return (params_cost << AV1_PROB_COST_SHIFT);
}
-// Calculates the threshold to be used for warp error computation.
-static AOM_INLINE int64_t calc_erroradv_threshold(int64_t ref_frame_error) {
- return (int64_t)(ref_frame_error * erroradv_tr + 0.5);
-}
-
// For the given reference frame, computes the global motion parameters for
// different motion models and finds the best.
static AOM_INLINE void compute_global_motion_for_ref_frame(
- AV1_COMP *cpi, YV12_BUFFER_CONFIG *ref_buf[REF_FRAMES], int frame,
+ AV1_COMP *cpi, struct aom_internal_error_info *error_info,
+ YV12_BUFFER_CONFIG *ref_buf[REF_FRAMES], int frame,
MotionModel *motion_models, uint8_t *segment_map, const int segment_map_w,
const int segment_map_h, const WarpedMotionParams *ref_params) {
- ThreadData *const td = &cpi->td;
- MACROBLOCK *const x = &td->mb;
AV1_COMMON *const cm = &cpi->common;
- MACROBLOCKD *const xd = &x->e_mbd;
- int i;
+ MACROBLOCKD *const xd = &cpi->td.mb.e_mbd;
int src_width = cpi->source->y_crop_width;
int src_height = cpi->source->y_crop_height;
int src_stride = cpi->source->y_stride;
- WarpedMotionParams tmp_wm_params;
- const double *params_this_motion;
assert(ref_buf[frame] != NULL);
- TransformationType model;
int bit_depth = cpi->common.seq_params->bit_depth;
GlobalMotionMethod global_motion_method = default_global_motion_method;
int num_refinements = cpi->sf.gm_sf.num_refinement_steps;
+ bool mem_alloc_failed = false;
- for (model = ROTZOOM; model < GLOBAL_TRANS_TYPES_ENC; ++model) {
- if (!aom_compute_global_motion(model, cpi->source, ref_buf[frame],
- bit_depth, global_motion_method,
- motion_models, RANSAC_NUM_MOTIONS)) {
+ // Select the best model based on fractional error reduction.
+ // By initializing this to erroradv_tr, the same logic which is used to
+ // select the best model will automatically filter out any model which
+ // doesn't meet the required quality threshold
+ double best_erroradv = erroradv_tr;
+ for (TransformationType model = FIRST_GLOBAL_TRANS_TYPE;
+ model <= LAST_GLOBAL_TRANS_TYPE; ++model) {
+ if (!aom_compute_global_motion(
+ model, cpi->source, ref_buf[frame], bit_depth, global_motion_method,
+ motion_models, RANSAC_NUM_MOTIONS, &mem_alloc_failed)) {
+ if (mem_alloc_failed) {
+ aom_internal_error(error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate global motion buffers");
+ }
continue;
}
- int64_t best_ref_frame_error = 0;
- int64_t best_warp_error = INT64_MAX;
- for (i = 0; i < RANSAC_NUM_MOTIONS; ++i) {
+ for (int i = 0; i < RANSAC_NUM_MOTIONS; ++i) {
if (motion_models[i].num_inliers == 0) continue;
- params_this_motion = motion_models[i].params;
- av1_convert_model_to_params(params_this_motion, &tmp_wm_params);
+ WarpedMotionParams tmp_wm_params;
+ av1_convert_model_to_params(motion_models[i].params, &tmp_wm_params);
// Skip models that we won't use (IDENTITY or TRANSLATION)
//
@@ -133,29 +133,26 @@
int64_t ref_frame_error = av1_segmented_frame_error(
is_cur_buf_hbd(xd), xd->bd, ref_buf[frame]->y_buffer,
- ref_buf[frame]->y_stride, cpi->source->y_buffer, src_width,
- src_height, src_stride, segment_map, segment_map_w);
+ ref_buf[frame]->y_stride, cpi->source->y_buffer, src_stride,
+ src_width, src_height, segment_map, segment_map_w);
if (ref_frame_error == 0) continue;
- const int64_t erroradv_threshold =
- calc_erroradv_threshold(ref_frame_error);
-
const int64_t warp_error = av1_refine_integerized_param(
&tmp_wm_params, tmp_wm_params.wmtype, is_cur_buf_hbd(xd), xd->bd,
ref_buf[frame]->y_buffer, ref_buf[frame]->y_crop_width,
ref_buf[frame]->y_crop_height, ref_buf[frame]->y_stride,
cpi->source->y_buffer, src_width, src_height, src_stride,
- num_refinements, best_warp_error, segment_map, segment_map_w,
- erroradv_threshold);
+ num_refinements, ref_frame_error, segment_map, segment_map_w);
// av1_refine_integerized_param() can return a simpler model type than
// its input, so re-check model type here
if (tmp_wm_params.wmtype <= TRANSLATION) continue;
- if (warp_error < best_warp_error) {
- best_ref_frame_error = ref_frame_error;
- best_warp_error = warp_error;
+ double erroradvantage = (double)warp_error / ref_frame_error;
+
+ if (erroradvantage < best_erroradv) {
+ best_erroradv = erroradvantage;
// Save the wm_params modified by
// av1_refine_integerized_param() rather than motion index to
// avoid rerunning refine() below.
@@ -163,47 +160,41 @@
sizeof(WarpedMotionParams));
}
}
- assert(cm->global_motion[frame].wmtype <= AFFINE);
- if (!av1_get_shear_params(&cm->global_motion[frame]))
- cm->global_motion[frame] = default_warp_params;
+ }
+
+ if (!av1_get_shear_params(&cm->global_motion[frame]))
+ cm->global_motion[frame] = default_warp_params;
#if 0
- // We never choose translational models, so this code is disabled
- if (cm->global_motion[frame].wmtype == TRANSLATION) {
- cm->global_motion[frame].wmmat[0] =
- convert_to_trans_prec(cm->features.allow_high_precision_mv,
- cm->global_motion[frame].wmmat[0]) *
- GM_TRANS_ONLY_DECODE_FACTOR;
- cm->global_motion[frame].wmmat[1] =
- convert_to_trans_prec(cm->features.allow_high_precision_mv,
- cm->global_motion[frame].wmmat[1]) *
- GM_TRANS_ONLY_DECODE_FACTOR;
- }
+ // We never choose translational models, so this code is disabled
+ if (cm->global_motion[frame].wmtype == TRANSLATION) {
+ cm->global_motion[frame].wmmat[0] =
+ convert_to_trans_prec(cm->features.allow_high_precision_mv,
+ cm->global_motion[frame].wmmat[0]) *
+ GM_TRANS_ONLY_DECODE_FACTOR;
+ cm->global_motion[frame].wmmat[1] =
+ convert_to_trans_prec(cm->features.allow_high_precision_mv,
+ cm->global_motion[frame].wmmat[1]) *
+ GM_TRANS_ONLY_DECODE_FACTOR;
+ }
#endif
- if (cm->global_motion[frame].wmtype == IDENTITY) continue;
+ if (cm->global_motion[frame].wmtype == IDENTITY) return;
- // Once we get here, best_ref_frame_error must be > 0. This is because
- // of the logic above, which skips over any models which have
- // ref_frame_error == 0
- assert(best_ref_frame_error > 0);
-
- // If the best error advantage found doesn't meet the threshold for
- // this motion type, revert to IDENTITY.
- if (!av1_is_enough_erroradvantage(
- (double)best_warp_error / best_ref_frame_error,
- gm_get_params_cost(&cm->global_motion[frame], ref_params,
- cm->features.allow_high_precision_mv))) {
- cm->global_motion[frame] = default_warp_params;
- }
-
- if (cm->global_motion[frame].wmtype != IDENTITY) break;
+ // If the best error advantage found doesn't meet the threshold for
+ // this motion type, revert to IDENTITY.
+ if (!av1_is_enough_erroradvantage(
+ best_erroradv,
+ gm_get_params_cost(&cm->global_motion[frame], ref_params,
+ cm->features.allow_high_precision_mv))) {
+ cm->global_motion[frame] = default_warp_params;
}
}
// Computes global motion for the given reference frame.
void av1_compute_gm_for_valid_ref_frames(
- AV1_COMP *cpi, YV12_BUFFER_CONFIG *ref_buf[REF_FRAMES], int frame,
+ AV1_COMP *cpi, struct aom_internal_error_info *error_info,
+ YV12_BUFFER_CONFIG *ref_buf[REF_FRAMES], int frame,
MotionModel *motion_models, uint8_t *segment_map, int segment_map_w,
int segment_map_h) {
AV1_COMMON *const cm = &cpi->common;
@@ -211,9 +202,9 @@
cm->prev_frame ? &cm->prev_frame->global_motion[frame]
: &default_warp_params;
- compute_global_motion_for_ref_frame(cpi, ref_buf, frame, motion_models,
- segment_map, segment_map_w, segment_map_h,
- ref_params);
+ compute_global_motion_for_ref_frame(cpi, error_info, ref_buf, frame,
+ motion_models, segment_map, segment_map_w,
+ segment_map_h, ref_params);
}
// Loops over valid reference frames and computes global motion estimation.
@@ -223,13 +214,15 @@
MotionModel *motion_models, uint8_t *segment_map, const int segment_map_w,
const int segment_map_h) {
AV1_COMMON *const cm = &cpi->common;
+ struct aom_internal_error_info *const error_info =
+ cpi->td.mb.e_mbd.error_info;
// Compute global motion w.r.t. reference frames starting from the nearest ref
// frame in a given direction.
for (int frame = 0; frame < num_ref_frames; frame++) {
int ref_frame = reference_frame[frame].frame;
- av1_compute_gm_for_valid_ref_frames(cpi, ref_buf, ref_frame, motion_models,
- segment_map, segment_map_w,
- segment_map_h);
+ av1_compute_gm_for_valid_ref_frames(cpi, error_info, ref_buf, ref_frame,
+ motion_models, segment_map,
+ segment_map_w, segment_map_h);
// If global motion w.r.t. current ref frame is
// INVALID/TRANSLATION/IDENTITY, skip the evaluation of global motion w.r.t
// the remaining ref frames in that direction.
@@ -361,40 +354,6 @@
}
}
-// Deallocates segment_map and inliers.
-static AOM_INLINE void dealloc_global_motion_data(MotionModel *motion_models,
- uint8_t *segment_map) {
- aom_free(segment_map);
-
- for (int m = 0; m < RANSAC_NUM_MOTIONS; m++) {
- aom_free(motion_models[m].inliers);
- }
-}
-
-// Allocates and initializes memory for segment_map and MotionModel.
-static AOM_INLINE bool alloc_global_motion_data(MotionModel *motion_models,
- uint8_t **segment_map,
- const int segment_map_w,
- const int segment_map_h) {
- av1_zero_array(motion_models, RANSAC_NUM_MOTIONS);
- for (int m = 0; m < RANSAC_NUM_MOTIONS; m++) {
- motion_models[m].inliers =
- aom_malloc(sizeof(*(motion_models[m].inliers)) * 2 * MAX_CORNERS);
- if (!motion_models[m].inliers) {
- dealloc_global_motion_data(motion_models, NULL);
- return false;
- }
- }
-
- *segment_map = (uint8_t *)aom_calloc(segment_map_w * segment_map_h,
- sizeof(*segment_map));
- if (!*segment_map) {
- dealloc_global_motion_data(motion_models, NULL);
- return false;
- }
- return true;
-}
-
// Initializes parameters used for computing global motion.
static AOM_INLINE void setup_global_motion_info_params(AV1_COMP *cpi) {
GlobalMotionInfo *const gm_info = &cpi->gm_info;
@@ -439,11 +398,7 @@
// Computes global motion w.r.t. valid reference frames.
static AOM_INLINE void global_motion_estimation(AV1_COMP *cpi) {
GlobalMotionInfo *const gm_info = &cpi->gm_info;
- MotionModel motion_models[RANSAC_NUM_MOTIONS];
- uint8_t *segment_map = NULL;
-
- alloc_global_motion_data(motion_models, &segment_map, gm_info->segment_map_w,
- gm_info->segment_map_h);
+ GlobalMotionData *gm_data = &cpi->td.gm_data;
// Compute global motion w.r.t. past reference frames and future reference
// frames
@@ -451,11 +406,9 @@
if (gm_info->num_ref_frames[dir] > 0)
compute_global_motion_for_references(
cpi, gm_info->ref_buf, gm_info->reference_frames[dir],
- gm_info->num_ref_frames[dir], motion_models, segment_map,
- gm_info->segment_map_w, gm_info->segment_map_h);
+ gm_info->num_ref_frames[dir], gm_data->motion_models,
+ gm_data->segment_map, gm_info->segment_map_w, gm_info->segment_map_h);
}
-
- dealloc_global_motion_data(motion_models, segment_map);
}
// Global motion estimation for the current frame is computed.This computation
@@ -478,13 +431,19 @@
}
if (cpi->common.current_frame.frame_type == INTER_FRAME && cpi->source &&
- cpi->oxcf.tool_cfg.enable_global_motion && !gm_info->search_done) {
+ cpi->oxcf.tool_cfg.enable_global_motion && !gm_info->search_done &&
+ cpi->sf.gm_sf.gm_search_type != GM_DISABLE_SEARCH) {
setup_global_motion_info_params(cpi);
- if (cpi->mt_info.num_workers > 1)
- av1_global_motion_estimation_mt(cpi);
- else
- global_motion_estimation(cpi);
- gm_info->search_done = 1;
+ // Terminate early if the total number of reference frames is zero.
+ if (cpi->gm_info.num_ref_frames[0] || cpi->gm_info.num_ref_frames[1]) {
+ gm_alloc_data(cpi, &cpi->td.gm_data);
+ if (cpi->mt_info.num_workers > 1)
+ av1_global_motion_estimation_mt(cpi);
+ else
+ global_motion_estimation(cpi);
+ gm_dealloc_data(&cpi->td.gm_data);
+ gm_info->search_done = 1;
+ }
}
memcpy(cm->cur_frame->global_motion, cm->global_motion,
sizeof(cm->cur_frame->global_motion));
diff --git a/av1/encoder/global_motion_facade.h b/av1/encoder/global_motion_facade.h
index dfdedf7..f13989aa 100644
--- a/av1/encoder/global_motion_facade.h
+++ b/av1/encoder/global_motion_facade.h
@@ -18,8 +18,36 @@
struct yv12_buffer_config;
struct AV1_COMP;
+// Allocates memory for members of GlobalMotionData.
+static AOM_INLINE void gm_alloc_data(AV1_COMP *cpi, GlobalMotionData *gm_data) {
+ AV1_COMMON *cm = &cpi->common;
+ GlobalMotionInfo *gm_info = &cpi->gm_info;
+
+ CHECK_MEM_ERROR(cm, gm_data->segment_map,
+ aom_malloc(sizeof(*gm_data->segment_map) *
+ gm_info->segment_map_w * gm_info->segment_map_h));
+
+ av1_zero_array(gm_data->motion_models, RANSAC_NUM_MOTIONS);
+ for (int m = 0; m < RANSAC_NUM_MOTIONS; m++) {
+ CHECK_MEM_ERROR(cm, gm_data->motion_models[m].inliers,
+ aom_malloc(sizeof(*gm_data->motion_models[m].inliers) * 2 *
+ MAX_CORNERS));
+ }
+}
+
+// Deallocates the memory allocated for members of GlobalMotionData.
+static AOM_INLINE void gm_dealloc_data(GlobalMotionData *gm_data) {
+ aom_free(gm_data->segment_map);
+ gm_data->segment_map = NULL;
+ for (int m = 0; m < RANSAC_NUM_MOTIONS; m++) {
+ aom_free(gm_data->motion_models[m].inliers);
+ gm_data->motion_models[m].inliers = NULL;
+ }
+}
+
void av1_compute_gm_for_valid_ref_frames(
- AV1_COMP *cpi, YV12_BUFFER_CONFIG *ref_buf[REF_FRAMES], int frame,
+ AV1_COMP *cpi, struct aom_internal_error_info *error_info,
+ YV12_BUFFER_CONFIG *ref_buf[REF_FRAMES], int frame,
MotionModel *motion_models, uint8_t *segment_map, int segment_map_w,
int segment_map_h);
void av1_compute_global_motion_facade(struct AV1_COMP *cpi);
diff --git a/av1/encoder/hash.c b/av1/encoder/hash.c
index 3091037..8037b59 100644
--- a/av1/encoder/hash.c
+++ b/av1/encoder/hash.c
@@ -10,6 +10,7 @@
*/
#include "av1/encoder/hash.h"
+#include "config/av1_rtcd.h"
static void crc_calculator_process_data(CRC_CALCULATOR *p_crc_calculator,
uint8_t *pData, uint32_t dataLength) {
diff --git a/av1/encoder/hash_motion.c b/av1/encoder/hash_motion.c
index 164aa09..8b04e22 100644
--- a/av1/encoder/hash_motion.c
+++ b/av1/encoder/hash_motion.c
@@ -128,7 +128,7 @@
}
p_hash_table->p_lookup_table =
(Vector **)aom_calloc(kMaxAddr, sizeof(p_hash_table->p_lookup_table[0]));
- if (!p_hash_table) return false;
+ if (!p_hash_table->p_lookup_table) return false;
return true;
}
@@ -141,13 +141,16 @@
if (p_hash_table->p_lookup_table[hash_value] == NULL) {
return false;
}
- aom_vector_setup(p_hash_table->p_lookup_table[hash_value], 10,
- sizeof(curr_block_hash[0]));
- aom_vector_push_back(p_hash_table->p_lookup_table[hash_value],
- curr_block_hash);
+ if (aom_vector_setup(p_hash_table->p_lookup_table[hash_value], 10,
+ sizeof(curr_block_hash[0])) == VECTOR_ERROR)
+ return false;
+ if (aom_vector_push_back(p_hash_table->p_lookup_table[hash_value],
+ curr_block_hash) == VECTOR_ERROR)
+ return false;
} else {
- aom_vector_push_back(p_hash_table->p_lookup_table[hash_value],
- curr_block_hash);
+ if (aom_vector_push_back(p_hash_table->p_lookup_table[hash_value],
+ curr_block_hash) == VECTOR_ERROR)
+ return false;
}
return true;
}
diff --git a/av1/encoder/hybrid_fwd_txfm.c b/av1/encoder/hybrid_fwd_txfm.c
index 4c2f8d0..a108e81 100644
--- a/av1/encoder/hybrid_fwd_txfm.c
+++ b/av1/encoder/hybrid_fwd_txfm.c
@@ -312,17 +312,51 @@
}
}
+#if CONFIG_AV1_HIGHBITDEPTH
+static INLINE void highbd_wht_fwd_txfm(TX_SIZE tx_size, const int16_t *src_diff,
+ ptrdiff_t src_stride,
+ tran_low_t *coeff) {
+ switch (tx_size) {
+ // As the output transform co-efficients of 4x4 Hadamard transform can be
+ // represented using 15 bits (for 12-bit clip) use lowbd variant of
+ // hadamard_4x4.
+ case TX_4X4: aom_hadamard_4x4(src_diff, src_stride, coeff); break;
+ case TX_8X8: aom_highbd_hadamard_8x8(src_diff, src_stride, coeff); break;
+ case TX_16X16:
+ aom_highbd_hadamard_16x16(src_diff, src_stride, coeff);
+ break;
+ case TX_32X32:
+ aom_highbd_hadamard_32x32(src_diff, src_stride, coeff);
+ break;
+ default: assert(0);
+ }
+}
+#endif // CONFIG_AV1_HIGHBITDEPTH
+
+static INLINE void wht_fwd_txfm(TX_SIZE tx_size, const int16_t *src_diff,
+ ptrdiff_t src_stride, tran_low_t *coeff) {
+ switch (tx_size) {
+ case TX_4X4: aom_hadamard_4x4(src_diff, src_stride, coeff); break;
+ case TX_8X8: aom_hadamard_8x8(src_diff, src_stride, coeff); break;
+ case TX_16X16: aom_hadamard_16x16(src_diff, src_stride, coeff); break;
+ case TX_32X32: aom_hadamard_32x32(src_diff, src_stride, coeff); break;
+ default: assert(0);
+ }
+}
+
void av1_quick_txfm(int use_hadamard, TX_SIZE tx_size, BitDepthInfo bd_info,
const int16_t *src_diff, int src_stride,
tran_low_t *coeff) {
if (use_hadamard) {
- switch (tx_size) {
- case TX_4X4: aom_hadamard_4x4(src_diff, src_stride, coeff); break;
- case TX_8X8: aom_hadamard_8x8(src_diff, src_stride, coeff); break;
- case TX_16X16: aom_hadamard_16x16(src_diff, src_stride, coeff); break;
- case TX_32X32: aom_hadamard_32x32(src_diff, src_stride, coeff); break;
- default: assert(0);
+#if CONFIG_AV1_HIGHBITDEPTH
+ if (bd_info.use_highbitdepth_buf) {
+ highbd_wht_fwd_txfm(tx_size, src_diff, src_stride, coeff);
+ } else {
+ wht_fwd_txfm(tx_size, src_diff, src_stride, coeff);
}
+#else
+ wht_fwd_txfm(tx_size, src_diff, src_stride, coeff);
+#endif // CONFIG_AV1_HIGHBITDEPTH
} else {
TxfmParam txfm_param;
txfm_param.tx_type = DCT_DCT;
diff --git a/av1/encoder/interp_search.c b/av1/encoder/interp_search.c
index 247fa3e..2723530 100644
--- a/av1/encoder/interp_search.c
+++ b/av1/encoder/interp_search.c
@@ -662,8 +662,7 @@
const int num_planes = av1_num_planes(cm);
MACROBLOCKD *const xd = &x->e_mbd;
MB_MODE_INFO *const mbmi = xd->mi[0];
- const int need_search =
- av1_is_interp_needed(xd) && !cpi->sf.rt_sf.skip_interp_filter_search;
+ const int need_search = av1_is_interp_needed(xd);
const int ref_frame = xd->mi[0]->ref_frame[0];
RD_STATS rd_stats_luma, rd_stats;
diff --git a/av1/encoder/interp_search.h b/av1/encoder/interp_search.h
index bce494e..9815e0b 100644
--- a/av1/encoder/interp_search.h
+++ b/av1/encoder/interp_search.h
@@ -109,6 +109,11 @@
*/
int skip_motion_mode;
/*!
+ * Initialized to false. If true, skips interpolation filter search and uses
+ * the default EIGHTTAP_REGULAR.
+ */
+ bool skip_ifs;
+ /*!
* A pointer to the first element in an array of INTERINTRA_MODE types. This
* contains the best inter_intra mode for each reference frame.
*/
diff --git a/av1/encoder/intra_mode_search.c b/av1/encoder/intra_mode_search.c
index 3b5dd75..99b0af2 100644
--- a/av1/encoder/intra_mode_search.c
+++ b/av1/encoder/intra_mode_search.c
@@ -874,16 +874,17 @@
for (int mode_idx = 0; mode_idx < UV_INTRA_MODES; ++mode_idx) {
int this_rate;
RD_STATS tokenonly_rd_stats;
- UV_PREDICTION_MODE mode = uv_rd_search_mode_order[mode_idx];
+ UV_PREDICTION_MODE uv_mode = uv_rd_search_mode_order[mode_idx];
// Skip the current mode evaluation if the RD cost derived using the mode
// signaling rate exceeds the best_rd so far.
const int mode_rate =
- mode_costs->intra_uv_mode_cost[cfl_allowed][mbmi->mode][mode];
+ mode_costs->intra_uv_mode_cost[cfl_allowed][mbmi->mode][uv_mode];
if (RDCOST(x->rdmult, mode_rate, 0) > best_rd) continue;
- const int is_diagonal_mode = av1_is_diagonal_mode(get_uv_mode(mode));
- const int is_directional_mode = av1_is_directional_mode(get_uv_mode(mode));
+ PREDICTION_MODE intra_mode = get_uv_mode(uv_mode);
+ const int is_diagonal_mode = av1_is_diagonal_mode(intra_mode);
+ const int is_directional_mode = av1_is_directional_mode(intra_mode);
if (is_diagonal_mode && !cpi->oxcf.intra_mode_cfg.enable_diagonal_intra)
continue;
@@ -892,25 +893,26 @@
continue;
if (!(cpi->sf.intra_sf.intra_uv_mode_mask[txsize_sqr_up_map[max_tx_size]] &
- (1 << mode)))
+ (1 << uv_mode)))
continue;
- if (!intra_mode_cfg->enable_smooth_intra && mode >= UV_SMOOTH_PRED &&
- mode <= UV_SMOOTH_H_PRED)
+ if (!intra_mode_cfg->enable_smooth_intra && uv_mode >= UV_SMOOTH_PRED &&
+ uv_mode <= UV_SMOOTH_H_PRED)
continue;
- if (!intra_mode_cfg->enable_paeth_intra && mode == UV_PAETH_PRED) continue;
+ if (!intra_mode_cfg->enable_paeth_intra && uv_mode == UV_PAETH_PRED)
+ continue;
assert(mbmi->mode < INTRA_MODES);
if (cpi->sf.intra_sf.prune_chroma_modes_using_luma_winner &&
- !(av1_derived_chroma_intra_mode_used_flag[mbmi->mode] & (1 << mode)))
+ !(av1_derived_chroma_intra_mode_used_flag[mbmi->mode] & (1 << uv_mode)))
continue;
- mbmi->uv_mode = mode;
+ mbmi->uv_mode = uv_mode;
// Init variables for cfl and angle delta
const SPEED_FEATURES *sf = &cpi->sf;
mbmi->angle_delta[PLANE_TYPE_UV] = 0;
- if (mode == UV_CFL_PRED) {
+ if (uv_mode == UV_CFL_PRED) {
if (!cfl_allowed || !intra_mode_cfg->enable_cfl_intra) continue;
assert(!is_directional_mode);
const TX_SIZE uv_tx_size = av1_get_tx_size(AOM_PLANE_U, xd);
@@ -936,18 +938,18 @@
intra_search_state.directional_mode_skip_mask, is_chroma);
intra_search_state.dir_mode_skip_mask_ready = 1;
}
- if (intra_search_state.directional_mode_skip_mask[mode]) {
+ if (intra_search_state.directional_mode_skip_mask[uv_mode]) {
continue;
}
// Search through angle delta
const int rate_overhead =
- mode_costs->intra_uv_mode_cost[cfl_allowed][mbmi->mode][mode];
+ mode_costs->intra_uv_mode_cost[cfl_allowed][mbmi->mode][uv_mode];
if (!rd_pick_intra_angle_sbuv(cpi, x, bsize, rate_overhead, best_rd,
&this_rate, &tokenonly_rd_stats))
continue;
} else {
- if (mode == UV_SMOOTH_PRED &&
+ if (uv_mode == UV_SMOOTH_PRED &&
should_prune_chroma_smooth_pred_based_on_source_variance(cpi, x,
bsize))
continue;
@@ -958,7 +960,7 @@
}
}
const int mode_cost =
- mode_costs->intra_uv_mode_cost[cfl_allowed][mbmi->mode][mode];
+ mode_costs->intra_uv_mode_cost[cfl_allowed][mbmi->mode][uv_mode];
this_rate = tokenonly_rd_stats.rate +
intra_mode_info_cost_uv(cpi, x, mbmi, bsize, mode_cost);
this_rd = RDCOST(x->rdmult, this_rate, tokenonly_rd_stats.dist);
diff --git a/av1/encoder/intra_mode_search_utils.h b/av1/encoder/intra_mode_search_utils.h
index 4519e46..107c223 100644
--- a/av1/encoder/intra_mode_search_utils.h
+++ b/av1/encoder/intra_mode_search_utils.h
@@ -576,13 +576,13 @@
int total_rate = mode_cost;
const ModeCosts *mode_costs = &x->mode_costs;
const int use_palette = mbmi->palette_mode_info.palette_size[1] > 0;
- const UV_PREDICTION_MODE mode = mbmi->uv_mode;
+ const UV_PREDICTION_MODE uv_mode = mbmi->uv_mode;
// Can only activate one mode.
- assert(((mode != UV_DC_PRED) + use_palette + mbmi->use_intrabc) <= 1);
+ assert(((uv_mode != UV_DC_PRED) + use_palette + mbmi->use_intrabc) <= 1);
const int try_palette = av1_allow_palette(
cpi->common.features.allow_screen_content_tools, mbmi->bsize);
- if (try_palette && mode == UV_DC_PRED) {
+ if (try_palette && uv_mode == UV_DC_PRED) {
const PALETTE_MODE_INFO *pmi = &mbmi->palette_mode_info;
total_rate +=
mode_costs->palette_uv_mode_cost[pmi->palette_size[0] > 0][use_palette];
@@ -604,10 +604,11 @@
total_rate += palette_mode_cost;
}
}
- if (av1_is_directional_mode(get_uv_mode(mode))) {
+ const PREDICTION_MODE intra_mode = get_uv_mode(uv_mode);
+ if (av1_is_directional_mode(intra_mode)) {
if (av1_use_angle_delta(bsize)) {
total_rate +=
- mode_costs->angle_delta_cost[mode - V_PRED]
+ mode_costs->angle_delta_cost[intra_mode - V_PRED]
[mbmi->angle_delta[PLANE_TYPE_UV] +
MAX_ANGLE_DELTA];
}
diff --git a/av1/encoder/mcomp.c b/av1/encoder/mcomp.c
index cc39c81..a8b0d10 100644
--- a/av1/encoder/mcomp.c
+++ b/av1/encoder/mcomp.c
@@ -61,30 +61,6 @@
ms_buffers->obmc_mask = x->obmc_buffer.mask;
}
-static AOM_INLINE SEARCH_METHODS
-get_faster_search_method(SEARCH_METHODS search_method) {
- // Note on search method's accuracy:
- // 1. NSTEP
- // 2. DIAMOND
- // 3. BIGDIA \approx SQUARE
- // 4. HEX.
- // 5. FAST_HEX \approx FAST_DIAMOND
- switch (search_method) {
- case NSTEP: return DIAMOND;
- case NSTEP_8PT: return DIAMOND;
- case DIAMOND: return BIGDIA;
- case CLAMPED_DIAMOND: return BIGDIA;
- case BIGDIA: return HEX;
- case SQUARE: return HEX;
- case HEX: return FAST_HEX;
- case FAST_HEX: return FAST_HEX;
- case FAST_DIAMOND: return VFAST_DIAMOND;
- case FAST_BIGDIA: return FAST_BIGDIA;
- case VFAST_DIAMOND: return VFAST_DIAMOND;
- default: assert(0 && "Invalid search method!"); return DIAMOND;
- }
-}
-
void av1_init_obmc_buffer(OBMCBuffer *obmc_buffer) {
obmc_buffer->wsrc = NULL;
obmc_buffer->mask = NULL;
@@ -96,7 +72,7 @@
FULLPEL_MOTION_SEARCH_PARAMS *ms_params, const struct AV1_COMP *cpi,
MACROBLOCK *x, BLOCK_SIZE bsize, const MV *ref_mv, FULLPEL_MV start_mv,
const search_site_config search_sites[NUM_DISTINCT_SEARCH_METHODS],
- int fine_search_interval) {
+ SEARCH_METHODS search_method, int fine_search_interval) {
const MV_SPEED_FEATURES *mv_sf = &cpi->sf.mv_sf;
const int is_key_frame =
cpi->ppi->gf_group.update_type[cpi->gf_frame_index] == KF_UPDATE;
@@ -107,28 +83,6 @@
init_ms_buffers(&ms_params->ms_buffers, x);
- SEARCH_METHODS search_method = mv_sf->search_method;
- const int sf_blk_search_method = mv_sf->use_bsize_dependent_search_method;
- const int min_dim = AOMMIN(block_size_wide[bsize], block_size_high[bsize]);
- const int qband = x->qindex >> (QINDEX_BITS - 2);
- const bool use_faster_search_method =
- (sf_blk_search_method == 1 && min_dim >= 32) ||
- (sf_blk_search_method >= 2 && min_dim >= 16 &&
- x->content_state_sb.source_sad_nonrd <= kMedSad && qband < 3);
-
- if (use_faster_search_method) {
- search_method = get_faster_search_method(search_method);
-
- // We might need to update the search site config since search_method
- // is changed here.
- const int ref_stride = ms_params->ms_buffers.ref->stride;
- if (ref_stride != search_sites[search_method].stride) {
- av1_refresh_search_site_config(x->search_site_cfg_buf, search_method,
- ref_stride);
- search_sites = x->search_site_cfg_buf;
- }
- }
-
av1_set_mv_search_method(ms_params, search_sites, search_method);
ms_params->mesh_patterns[0] = mv_sf->mesh_patterns;
@@ -700,7 +654,8 @@
}
static INLINE int get_mvpred_var_cost(
- const FULLPEL_MOTION_SEARCH_PARAMS *ms_params, const FULLPEL_MV *this_mv) {
+ const FULLPEL_MOTION_SEARCH_PARAMS *ms_params, const FULLPEL_MV *this_mv,
+ FULLPEL_MV_STATS *mv_stats) {
const aom_variance_fn_ptr_t *vfp = ms_params->vfp;
const MV sub_this_mv = get_mv_from_fullmv(this_mv);
const struct buf_2d *const src = ms_params->ms_buffers.src;
@@ -709,13 +664,14 @@
const int src_stride = src->stride;
const int ref_stride = ref->stride;
- unsigned unused;
int bestsme;
bestsme = vfp->vf(src_buf, src_stride, get_buf_from_fullmv(ref, this_mv),
- ref_stride, &unused);
+ ref_stride, &mv_stats->sse);
+ mv_stats->distortion = bestsme;
- bestsme += mv_err_cost_(&sub_this_mv, &ms_params->mv_cost_params);
+ mv_stats->err_cost = mv_err_cost_(&sub_this_mv, &ms_params->mv_cost_params);
+ bestsme += mv_stats->err_cost;
return bestsme;
}
@@ -731,7 +687,8 @@
}
static INLINE int get_mvpred_compound_var_cost(
- const FULLPEL_MOTION_SEARCH_PARAMS *ms_params, const FULLPEL_MV *this_mv) {
+ const FULLPEL_MOTION_SEARCH_PARAMS *ms_params, const FULLPEL_MV *this_mv,
+ FULLPEL_MV_STATS *mv_stats) {
const aom_variance_fn_ptr_t *vfp = ms_params->vfp;
const struct buf_2d *const src = ms_params->ms_buffers.src;
const struct buf_2d *const ref = ms_params->ms_buffers.ref;
@@ -743,23 +700,24 @@
const uint8_t *second_pred = ms_params->ms_buffers.second_pred;
const int mask_stride = ms_params->ms_buffers.mask_stride;
const int invert_mask = ms_params->ms_buffers.inv_mask;
- unsigned unused;
int bestsme;
if (mask) {
bestsme = vfp->msvf(get_buf_from_fullmv(ref, this_mv), ref_stride, 0, 0,
src_buf, src_stride, second_pred, mask, mask_stride,
- invert_mask, &unused);
+ invert_mask, &mv_stats->sse);
} else if (second_pred) {
bestsme = vfp->svaf(get_buf_from_fullmv(ref, this_mv), ref_stride, 0, 0,
- src_buf, src_stride, &unused, second_pred);
+ src_buf, src_stride, &mv_stats->sse, second_pred);
} else {
bestsme = vfp->vf(src_buf, src_stride, get_buf_from_fullmv(ref, this_mv),
- ref_stride, &unused);
+ ref_stride, &mv_stats->sse);
}
+ mv_stats->distortion = bestsme;
const MV sub_this_mv = get_mv_from_fullmv(this_mv);
- bestsme += mv_err_cost_(&sub_this_mv, &ms_params->mv_cost_params);
+ mv_stats->err_cost = mv_err_cost_(&sub_this_mv, &ms_params->mv_cost_params);
+ bestsme += mv_stats->err_cost;
return bestsme;
}
@@ -803,13 +761,15 @@
const int br = best_mv.row;
const int bc = best_mv.col;
- cost_list[0] = get_mvpred_var_cost(ms_params, &best_mv);
+ FULLPEL_MV_STATS mv_stats;
+ cost_list[0] = get_mvpred_var_cost(ms_params, &best_mv, &mv_stats);
if (check_bounds(&ms_params->mv_limits, br, bc, 1)) {
for (int i = 0; i < 4; i++) {
const FULLPEL_MV neighbor_mv = { br + neighbors[i].row,
bc + neighbors[i].col };
- cost_list[i + 1] = get_mvpred_var_cost(ms_params, &neighbor_mv);
+ cost_list[i + 1] =
+ get_mvpred_var_cost(ms_params, &neighbor_mv, &mv_stats);
}
} else {
for (int i = 0; i < 4; i++) {
@@ -818,7 +778,8 @@
if (!av1_is_fullmv_in_range(&ms_params->mv_limits, neighbor_mv)) {
cost_list[i + 1] = INT_MAX;
} else {
- cost_list[i + 1] = get_mvpred_var_cost(ms_params, &neighbor_mv);
+ cost_list[i + 1] =
+ get_mvpred_var_cost(ms_params, &neighbor_mv, &mv_stats);
}
}
}
@@ -1055,7 +1016,8 @@
static int pattern_search(FULLPEL_MV start_mv,
const FULLPEL_MOTION_SEARCH_PARAMS *ms_params,
int search_step, const int do_init_search,
- int *cost_list, FULLPEL_MV *best_mv) {
+ int *cost_list, FULLPEL_MV *best_mv,
+ FULLPEL_MV_STATS *best_mv_stats) {
static const int search_steps[MAX_MVSEARCH_STEPS] = {
10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0,
};
@@ -1278,7 +1240,7 @@
}
}
- const int var_cost = get_mvpred_var_cost(ms_params, best_mv);
+ const int var_cost = get_mvpred_var_cost(ms_params, best_mv, best_mv_stats);
return var_cost;
}
@@ -1296,61 +1258,68 @@
static int hex_search(const FULLPEL_MV start_mv,
const FULLPEL_MOTION_SEARCH_PARAMS *ms_params,
const int search_step, const int do_init_search,
- int *cost_list, FULLPEL_MV *best_mv) {
+ int *cost_list, FULLPEL_MV *best_mv,
+ FULLPEL_MV_STATS *best_mv_stats) {
return pattern_search(start_mv, ms_params, search_step, do_init_search,
- cost_list, best_mv);
+ cost_list, best_mv, best_mv_stats);
}
static int bigdia_search(const FULLPEL_MV start_mv,
const FULLPEL_MOTION_SEARCH_PARAMS *ms_params,
const int search_step, const int do_init_search,
- int *cost_list, FULLPEL_MV *best_mv) {
+ int *cost_list, FULLPEL_MV *best_mv,
+ FULLPEL_MV_STATS *best_mv_stats) {
return pattern_search(start_mv, ms_params, search_step, do_init_search,
- cost_list, best_mv);
+ cost_list, best_mv, best_mv_stats);
}
static int square_search(const FULLPEL_MV start_mv,
const FULLPEL_MOTION_SEARCH_PARAMS *ms_params,
const int search_step, const int do_init_search,
- int *cost_list, FULLPEL_MV *best_mv) {
+ int *cost_list, FULLPEL_MV *best_mv,
+ FULLPEL_MV_STATS *best_mv_stats) {
return pattern_search(start_mv, ms_params, search_step, do_init_search,
- cost_list, best_mv);
+ cost_list, best_mv, best_mv_stats);
}
static int fast_hex_search(const FULLPEL_MV start_mv,
const FULLPEL_MOTION_SEARCH_PARAMS *ms_params,
const int search_step, const int do_init_search,
- int *cost_list, FULLPEL_MV *best_mv) {
+ int *cost_list, FULLPEL_MV *best_mv,
+ FULLPEL_MV_STATS *best_mv_stats) {
return hex_search(start_mv, ms_params,
AOMMAX(MAX_MVSEARCH_STEPS - 2, search_step), do_init_search,
- cost_list, best_mv);
+ cost_list, best_mv, best_mv_stats);
}
static int vfast_dia_search(const FULLPEL_MV start_mv,
const FULLPEL_MOTION_SEARCH_PARAMS *ms_params,
const int search_step, const int do_init_search,
- int *cost_list, FULLPEL_MV *best_mv) {
+ int *cost_list, FULLPEL_MV *best_mv,
+ FULLPEL_MV_STATS *best_mv_stats) {
return bigdia_search(start_mv, ms_params,
AOMMAX(MAX_MVSEARCH_STEPS - 1, search_step),
- do_init_search, cost_list, best_mv);
+ do_init_search, cost_list, best_mv, best_mv_stats);
}
static int fast_dia_search(const FULLPEL_MV start_mv,
const FULLPEL_MOTION_SEARCH_PARAMS *ms_params,
const int search_step, const int do_init_search,
- int *cost_list, FULLPEL_MV *best_mv) {
+ int *cost_list, FULLPEL_MV *best_mv,
+ FULLPEL_MV_STATS *best_mv_stats) {
return bigdia_search(start_mv, ms_params,
AOMMAX(MAX_MVSEARCH_STEPS - 2, search_step),
- do_init_search, cost_list, best_mv);
+ do_init_search, cost_list, best_mv, best_mv_stats);
}
static int fast_bigdia_search(const FULLPEL_MV start_mv,
const FULLPEL_MOTION_SEARCH_PARAMS *ms_params,
const int search_step, const int do_init_search,
- int *cost_list, FULLPEL_MV *best_mv) {
+ int *cost_list, FULLPEL_MV *best_mv,
+ FULLPEL_MV_STATS *best_mv_stats) {
return bigdia_search(start_mv, ms_params,
AOMMAX(MAX_MVSEARCH_STEPS - 3, search_step),
- do_init_search, cost_list, best_mv);
+ do_init_search, cost_list, best_mv, best_mv_stats);
}
static int diamond_search_sad(FULLPEL_MV start_mv, unsigned int start_mv_sad,
@@ -1528,7 +1497,9 @@
static int full_pixel_diamond(FULLPEL_MV start_mv,
const FULLPEL_MOTION_SEARCH_PARAMS *ms_params,
const int step_param, int *cost_list,
- FULLPEL_MV *best_mv, FULLPEL_MV *second_best_mv) {
+ FULLPEL_MV *best_mv,
+ FULLPEL_MV_STATS *best_mv_stats,
+ FULLPEL_MV *second_best_mv) {
const search_site_config *cfg = ms_params->search_sites;
int thissme, n, num00 = 0;
@@ -1539,7 +1510,7 @@
diamond_search_sad(start_mv, start_mv_sad, ms_params, step_param, &n, best_mv,
second_best_mv);
- int bestsme = get_mvpred_compound_var_cost(ms_params, best_mv);
+ int bestsme = get_mvpred_compound_var_cost(ms_params, best_mv, best_mv_stats);
// If there won't be more n-step search, check to see if refining search is
// needed.
@@ -1550,14 +1521,17 @@
// TODO(chiyotsai@google.com): There is another bug here where the second
// best mv gets incorrectly overwritten. Fix it later.
FULLPEL_MV tmp_best_mv;
+ FULLPEL_MV_STATS tmp_best_mv_stats;
diamond_search_sad(start_mv, start_mv_sad, ms_params, step_param + n,
&num00, &tmp_best_mv, second_best_mv);
- thissme = get_mvpred_compound_var_cost(ms_params, &tmp_best_mv);
+ thissme = get_mvpred_compound_var_cost(ms_params, &tmp_best_mv,
+ &tmp_best_mv_stats);
if (thissme < bestsme) {
bestsme = thissme;
*best_mv = tmp_best_mv;
+ *best_mv_stats = tmp_best_mv_stats;
}
if (num00) {
@@ -1658,6 +1632,7 @@
const FULLPEL_MOTION_SEARCH_PARAMS *ms_params,
const struct MESH_PATTERN *const mesh_patterns,
int *cost_list, FULLPEL_MV *best_mv,
+ FULLPEL_MV_STATS *mv_stats,
FULLPEL_MV *second_best_mv) {
const int kMinRange = 7;
const int kMaxRange = 256;
@@ -1717,7 +1692,7 @@
}
if (bestsme < INT_MAX) {
- bestsme = get_mvpred_var_cost(ms_params, best_mv);
+ bestsme = get_mvpred_var_cost(ms_params, best_mv, mv_stats);
}
// Return cost list.
@@ -1809,7 +1784,8 @@
int av1_full_pixel_search(const FULLPEL_MV start_mv,
const FULLPEL_MOTION_SEARCH_PARAMS *ms_params,
const int step_param, int *cost_list,
- FULLPEL_MV *best_mv, FULLPEL_MV *second_best_mv) {
+ FULLPEL_MV *best_mv, FULLPEL_MV_STATS *best_mv_stats,
+ FULLPEL_MV *second_best_mv) {
const BLOCK_SIZE bsize = ms_params->bsize;
const SEARCH_METHODS search_method = ms_params->search_method;
@@ -1831,41 +1807,43 @@
}
assert(ms_params->ms_buffers.ref->stride == ms_params->search_sites->stride);
+ assert(ms_params->ms_buffers.ref->width == ms_params->ms_buffers.src->width);
switch (search_method) {
case FAST_BIGDIA:
var = fast_bigdia_search(start_mv, ms_params, step_param, 0, cost_list,
- best_mv);
+ best_mv, best_mv_stats);
break;
case VFAST_DIAMOND:
var = vfast_dia_search(start_mv, ms_params, step_param, 0, cost_list,
- best_mv);
+ best_mv, best_mv_stats);
break;
case FAST_DIAMOND:
var = fast_dia_search(start_mv, ms_params, step_param, 0, cost_list,
- best_mv);
+ best_mv, best_mv_stats);
break;
case FAST_HEX:
var = fast_hex_search(start_mv, ms_params, step_param, 0, cost_list,
- best_mv);
+ best_mv, best_mv_stats);
break;
case HEX:
- var = hex_search(start_mv, ms_params, step_param, 1, cost_list, best_mv);
+ var = hex_search(start_mv, ms_params, step_param, 1, cost_list, best_mv,
+ best_mv_stats);
break;
case SQUARE:
- var =
- square_search(start_mv, ms_params, step_param, 1, cost_list, best_mv);
+ var = square_search(start_mv, ms_params, step_param, 1, cost_list,
+ best_mv, best_mv_stats);
break;
case BIGDIA:
- var =
- bigdia_search(start_mv, ms_params, step_param, 1, cost_list, best_mv);
+ var = bigdia_search(start_mv, ms_params, step_param, 1, cost_list,
+ best_mv, best_mv_stats);
break;
case NSTEP:
case NSTEP_8PT:
case DIAMOND:
case CLAMPED_DIAMOND:
var = full_pixel_diamond(start_mv, ms_params, step_param, cost_list,
- best_mv, second_best_mv);
+ best_mv, best_mv_stats, second_best_mv);
break;
default: assert(0 && "Invalid search method.");
}
@@ -1922,13 +1900,15 @@
new_ms_params.sdx3df = new_ms_params.vfp->sdx3df;
return av1_full_pixel_search(start_mv, &new_ms_params, step_param,
- cost_list, best_mv, second_best_mv);
+ cost_list, best_mv, best_mv_stats,
+ second_best_mv);
}
}
if (run_mesh_search) {
int var_ex;
FULLPEL_MV tmp_mv_ex;
+ FULLPEL_MV_STATS tmp_mv_stats;
// Pick the mesh pattern for exhaustive search based on the toolset (intraBC
// or non-intraBC)
// TODO(chiyotsai@google.com): There is a bug here where the second best mv
@@ -1937,10 +1917,12 @@
ms_params->mesh_patterns[is_intra_mode];
// TODO(chiyotsai@google.com): the second best mv is not set correctly by
// full_pixel_exhaustive, which can incorrectly override it.
- var_ex = full_pixel_exhaustive(*best_mv, ms_params, mesh_patterns,
- cost_list, &tmp_mv_ex, second_best_mv);
+ var_ex =
+ full_pixel_exhaustive(*best_mv, ms_params, mesh_patterns, cost_list,
+ &tmp_mv_ex, &tmp_mv_stats, second_best_mv);
if (var_ex < var) {
var = var_ex;
+ *best_mv_stats = tmp_mv_stats;
*best_mv = tmp_mv_ex;
}
}
@@ -2000,7 +1982,8 @@
hash_mv.col = ref_block_hash.x - x_pos;
hash_mv.row = ref_block_hash.y - y_pos;
if (!av1_is_fullmv_in_range(mv_limits, hash_mv)) continue;
- const int refCost = get_mvpred_var_cost(ms_params, &hash_mv);
+ FULLPEL_MV_STATS mv_stats;
+ const int refCost = get_mvpred_var_cost(ms_params, &hash_mv, &mv_stats);
if (refCost < best_hash_cost) {
best_hash_cost = refCost;
*best_mv = hash_mv;
@@ -2011,12 +1994,27 @@
return best_hash_cost;
}
-static int vector_match(int16_t *ref, int16_t *src, int bwl) {
+static int vector_match(int16_t *ref, int16_t *src, int bwl, int search_size,
+ int full_search, int *sad) {
int best_sad = INT_MAX;
int this_sad;
int d;
int center, offset = 0;
- int bw = 4 << bwl; // redundant variable, to be changed in the experiments.
+ int bw = search_size << 1;
+
+ if (full_search) {
+ for (d = 0; d <= bw; d++) {
+ this_sad = aom_vector_var(&ref[d], src, bwl);
+ if (this_sad < best_sad) {
+ best_sad = this_sad;
+ offset = d;
+ }
+ }
+ center = offset;
+ *sad = best_sad;
+ return (center - (bw >> 1));
+ }
+
for (d = 0; d <= bw; d += 16) {
this_sad = aom_vector_var(&ref[d], src, bwl);
if (this_sad < best_sad) {
@@ -2072,31 +2070,47 @@
center = this_pos;
}
}
-
+ *sad = best_sad;
return (center - (bw >> 1));
}
-// A special fast version of motion search used in rt mode
+// A special fast version of motion search used in rt mode.
+// The search window along columns and row is given by:
+// +/- me_search_size_col/row.
unsigned int av1_int_pro_motion_estimation(const AV1_COMP *cpi, MACROBLOCK *x,
BLOCK_SIZE bsize, int mi_row,
- int mi_col, const MV *ref_mv) {
+ int mi_col, const MV *ref_mv,
+ unsigned int *y_sad_zero,
+ int me_search_size_col,
+ int me_search_size_row) {
+ const AV1_COMMON *const cm = &cpi->common;
MACROBLOCKD *xd = &x->e_mbd;
MB_MODE_INFO *mi = xd->mi[0];
struct buf_2d backup_yv12[MAX_MB_PLANE] = { { 0, 0, 0, 0, 0 } };
- DECLARE_ALIGNED(16, int16_t, hbuf[256]);
- DECLARE_ALIGNED(16, int16_t, vbuf[256]);
- DECLARE_ALIGNED(16, int16_t, src_hbuf[128]);
- DECLARE_ALIGNED(16, int16_t, src_vbuf[128]);
int idx;
- const int bw = 4 << mi_size_wide_log2[bsize];
- const int bh = 4 << mi_size_high_log2[bsize];
- const int search_width = bw << 1;
- const int search_height = bh << 1;
+ const int bw = block_size_wide[bsize];
+ const int bh = block_size_high[bsize];
+ const int is_screen = cpi->oxcf.tune_cfg.content == AOM_CONTENT_SCREEN;
+ const int full_search = is_screen;
+ const bool screen_scroll_superblock =
+ is_screen && bsize == cm->seq_params->sb_size;
+ // Keep border a multiple of 16.
+ const int border = (cpi->oxcf.border_in_pixels >> 4) << 4;
+ int search_size_width = me_search_size_col;
+ int search_size_height = me_search_size_row;
+ // Adjust based on boundary.
+ if (((mi_col << 2) - search_size_width < -border) ||
+ ((mi_col << 2) + search_size_width > cm->width + border))
+ search_size_width = border;
+ if (((mi_row << 2) - search_size_height < -border) ||
+ ((mi_row << 2) + search_size_height > cm->height + border))
+ search_size_height = border;
const int src_stride = x->plane[0].src.stride;
const int ref_stride = xd->plane[0].pre[0].stride;
uint8_t const *ref_buf, *src_buf;
int_mv *best_int_mv = &xd->mi[0]->mv[0];
unsigned int best_sad, tmp_sad, this_sad[4];
+ int best_sad_col, best_sad_row;
const int row_norm_factor = mi_size_high_log2[bsize] + 1;
const int col_norm_factor = 3 + (bw >> 5);
const YV12_BUFFER_CONFIG *scaled_ref_frame =
@@ -2129,13 +2143,29 @@
}
return best_sad;
}
+ const int width_ref_buf = (search_size_width << 1) + bw;
+ const int height_ref_buf = (search_size_height << 1) + bh;
+ int16_t *hbuf = (int16_t *)aom_malloc(width_ref_buf * sizeof(*hbuf));
+ int16_t *vbuf = (int16_t *)aom_malloc(height_ref_buf * sizeof(*vbuf));
+ int16_t *src_hbuf = (int16_t *)aom_malloc(bw * sizeof(*src_hbuf));
+ int16_t *src_vbuf = (int16_t *)aom_malloc(bh * sizeof(*src_vbuf));
+ if (!hbuf || !vbuf || !src_hbuf || !src_vbuf) {
+ aom_free(hbuf);
+ aom_free(vbuf);
+ aom_free(src_hbuf);
+ aom_free(src_vbuf);
+ aom_internal_error(cm->error, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate hbuf, vbuf, src_hbuf, or src_vbuf");
+ }
- // Set up prediction 1-D reference set
- ref_buf = xd->plane[0].pre[0].buf - (bw >> 1);
- aom_int_pro_row(hbuf, ref_buf, ref_stride, search_width, bh, row_norm_factor);
+ // Set up prediction 1-D reference set for rows.
+ ref_buf = xd->plane[0].pre[0].buf - search_size_width;
+ aom_int_pro_row(hbuf, ref_buf, ref_stride, width_ref_buf, bh,
+ row_norm_factor);
- ref_buf = xd->plane[0].pre[0].buf - (bh >> 1) * ref_stride;
- aom_int_pro_col(vbuf, ref_buf, ref_stride, bw, search_height,
+ // Set up prediction 1-D reference set for cols
+ ref_buf = xd->plane[0].pre[0].buf - search_size_height * ref_stride;
+ aom_int_pro_col(vbuf, ref_buf, ref_stride, bw, height_ref_buf,
col_norm_factor);
// Set up src 1-D reference set
@@ -2145,9 +2175,19 @@
// Find the best match per 1-D search
best_int_mv->as_fullmv.col =
- vector_match(hbuf, src_hbuf, mi_size_wide_log2[bsize]);
+ vector_match(hbuf, src_hbuf, mi_size_wide_log2[bsize], search_size_width,
+ full_search, &best_sad_col);
best_int_mv->as_fullmv.row =
- vector_match(vbuf, src_vbuf, mi_size_high_log2[bsize]);
+ vector_match(vbuf, src_vbuf, mi_size_high_log2[bsize], search_size_height,
+ full_search, &best_sad_row);
+
+ // For screen: select between horiz or vert motion.
+ if (is_screen) {
+ if (best_sad_col < best_sad_row)
+ best_int_mv->as_fullmv.row = 0;
+ else
+ best_int_mv->as_fullmv.col = 0;
+ }
FULLPEL_MV this_mv = best_int_mv->as_fullmv;
src_buf = x->plane[0].src.buf;
@@ -2159,16 +2199,18 @@
if (best_int_mv->as_int != 0) {
tmp_sad = cpi->ppi->fn_ptr[bsize].sdf(x->plane[0].src.buf, src_stride,
xd->plane[0].pre[0].buf, ref_stride);
-
+ *y_sad_zero = tmp_sad;
if (tmp_sad < best_sad) {
best_int_mv->as_fullmv = kZeroFullMv;
this_mv = best_int_mv->as_fullmv;
ref_buf = xd->plane[0].pre[0].buf;
best_sad = tmp_sad;
}
+ } else {
+ *y_sad_zero = best_sad;
}
- {
+ if (!screen_scroll_superblock) {
const uint8_t *const pos[4] = {
ref_buf - ref_stride,
ref_buf - 1,
@@ -2178,33 +2220,33 @@
cpi->ppi->fn_ptr[bsize].sdx4df(src_buf, src_stride, pos, ref_stride,
this_sad);
- }
- for (idx = 0; idx < 4; ++idx) {
- if (this_sad[idx] < best_sad) {
- best_sad = this_sad[idx];
- best_int_mv->as_fullmv.row = search_pos[idx].row + this_mv.row;
- best_int_mv->as_fullmv.col = search_pos[idx].col + this_mv.col;
+ for (idx = 0; idx < 4; ++idx) {
+ if (this_sad[idx] < best_sad) {
+ best_sad = this_sad[idx];
+ best_int_mv->as_fullmv.row = search_pos[idx].row + this_mv.row;
+ best_int_mv->as_fullmv.col = search_pos[idx].col + this_mv.col;
+ }
}
- }
- if (this_sad[0] < this_sad[3])
- this_mv.row -= 1;
- else
- this_mv.row += 1;
+ if (this_sad[0] < this_sad[3])
+ this_mv.row -= 1;
+ else
+ this_mv.row += 1;
- if (this_sad[1] < this_sad[2])
- this_mv.col -= 1;
- else
- this_mv.col += 1;
+ if (this_sad[1] < this_sad[2])
+ this_mv.col -= 1;
+ else
+ this_mv.col += 1;
- ref_buf = get_buf_from_fullmv(&xd->plane[0].pre[0], &this_mv);
+ ref_buf = get_buf_from_fullmv(&xd->plane[0].pre[0], &this_mv);
- tmp_sad =
- cpi->ppi->fn_ptr[bsize].sdf(src_buf, src_stride, ref_buf, ref_stride);
- if (best_sad > tmp_sad) {
- best_int_mv->as_fullmv = this_mv;
- best_sad = tmp_sad;
+ tmp_sad =
+ cpi->ppi->fn_ptr[bsize].sdf(src_buf, src_stride, ref_buf, ref_stride);
+ if (best_sad > tmp_sad) {
+ best_int_mv->as_fullmv = this_mv;
+ best_sad = tmp_sad;
+ }
}
FullMvLimits mv_limits = x->mv_limits;
@@ -2218,6 +2260,10 @@
for (i = 0; i < MAX_MB_PLANE; i++) xd->plane[i].pre[0] = backup_yv12[i];
}
+ aom_free(hbuf);
+ aom_free(vbuf);
+ aom_free(src_hbuf);
+ aom_free(src_vbuf);
return best_sad;
}
@@ -2960,8 +3006,9 @@
int av1_find_best_sub_pixel_tree_pruned_more(
MACROBLOCKD *xd, const AV1_COMMON *const cm,
- const SUBPEL_MOTION_SEARCH_PARAMS *ms_params, MV start_mv, MV *bestmv,
- int *distortion, unsigned int *sse1, int_mv *last_mv_search_list) {
+ const SUBPEL_MOTION_SEARCH_PARAMS *ms_params, MV start_mv,
+ const FULLPEL_MV_STATS *start_mv_stats, MV *bestmv, int *distortion,
+ unsigned int *sse1, int_mv *last_mv_search_list) {
(void)cm;
const int allow_hp = ms_params->allow_hp;
const int forced_stop = ms_params->forced_stop;
@@ -2982,8 +3029,16 @@
? &cm->sf_identity
: xd->block_ref_scale_factors[0];
const int is_scaled = av1_is_scaled(sf);
- besterr = setup_center_error_facade(
- xd, cm, bestmv, var_params, mv_cost_params, sse1, distortion, is_scaled);
+
+ if (start_mv_stats != NULL && !is_scaled) {
+ besterr = start_mv_stats->distortion + start_mv_stats->err_cost;
+ *distortion = start_mv_stats->distortion;
+ *sse1 = start_mv_stats->sse;
+ } else {
+ besterr =
+ setup_center_error_facade(xd, cm, bestmv, var_params, mv_cost_params,
+ sse1, distortion, is_scaled);
+ }
// If forced_stop is FULL_PEL, return.
if (forced_stop == FULL_PEL) return besterr;
@@ -3045,9 +3100,11 @@
int av1_find_best_sub_pixel_tree_pruned(
MACROBLOCKD *xd, const AV1_COMMON *const cm,
- const SUBPEL_MOTION_SEARCH_PARAMS *ms_params, MV start_mv, MV *bestmv,
- int *distortion, unsigned int *sse1, int_mv *last_mv_search_list) {
+ const SUBPEL_MOTION_SEARCH_PARAMS *ms_params, MV start_mv,
+ const FULLPEL_MV_STATS *start_mv_stats, MV *bestmv, int *distortion,
+ unsigned int *sse1, int_mv *last_mv_search_list) {
(void)cm;
+ (void)start_mv_stats;
const int allow_hp = ms_params->allow_hp;
const int forced_stop = ms_params->forced_stop;
const int iters_per_step = ms_params->iters_per_step;
@@ -3067,8 +3124,16 @@
? &cm->sf_identity
: xd->block_ref_scale_factors[0];
const int is_scaled = av1_is_scaled(sf);
- besterr = setup_center_error_facade(
- xd, cm, bestmv, var_params, mv_cost_params, sse1, distortion, is_scaled);
+
+ if (start_mv_stats != NULL && !is_scaled) {
+ besterr = start_mv_stats->distortion + start_mv_stats->err_cost;
+ *distortion = start_mv_stats->distortion;
+ *sse1 = start_mv_stats->sse;
+ } else {
+ besterr =
+ setup_center_error_facade(xd, cm, bestmv, var_params, mv_cost_params,
+ sse1, distortion, is_scaled);
+ }
// If forced_stop is FULL_PEL, return.
if (forced_stop == FULL_PEL) return besterr;
@@ -3181,9 +3246,12 @@
int av1_find_best_sub_pixel_tree(MACROBLOCKD *xd, const AV1_COMMON *const cm,
const SUBPEL_MOTION_SEARCH_PARAMS *ms_params,
- MV start_mv, MV *bestmv, int *distortion,
+ MV start_mv,
+ const FULLPEL_MV_STATS *start_mv_stats,
+ MV *bestmv, int *distortion,
unsigned int *sse1,
int_mv *last_mv_search_list) {
+ (void)start_mv_stats;
const int allow_hp = ms_params->allow_hp;
const int forced_stop = ms_params->forced_stop;
const int iters_per_step = ms_params->iters_per_step;
@@ -3207,12 +3275,18 @@
: xd->block_ref_scale_factors[0];
const int is_scaled = av1_is_scaled(sf);
- if (subpel_search_type != USE_2_TAPS_ORIG) {
- besterr = upsampled_setup_center_error(xd, cm, bestmv, var_params,
- mv_cost_params, sse1, distortion);
+ if (start_mv_stats != NULL && !is_scaled) {
+ besterr = start_mv_stats->distortion + start_mv_stats->err_cost;
+ *distortion = start_mv_stats->distortion;
+ *sse1 = start_mv_stats->sse;
} else {
- besterr = setup_center_error(xd, bestmv, var_params, mv_cost_params, sse1,
- distortion);
+ if (subpel_search_type != USE_2_TAPS_ORIG) {
+ besterr = upsampled_setup_center_error(xd, cm, bestmv, var_params,
+ mv_cost_params, sse1, distortion);
+ } else {
+ besterr = setup_center_error(xd, bestmv, var_params, mv_cost_params, sse1,
+ distortion);
+ }
}
// If forced_stop is FULL_PEL, return.
@@ -3255,12 +3329,14 @@
// Returns the maximum MV.
int av1_return_max_sub_pixel_mv(MACROBLOCKD *xd, const AV1_COMMON *const cm,
const SUBPEL_MOTION_SEARCH_PARAMS *ms_params,
- MV start_mv, MV *bestmv, int *distortion,
- unsigned int *sse1,
+ MV start_mv,
+ const FULLPEL_MV_STATS *start_mv_stats,
+ MV *bestmv, int *distortion, unsigned int *sse1,
int_mv *last_mv_search_list) {
(void)xd;
(void)cm;
(void)start_mv;
+ (void)start_mv_stats;
(void)sse1;
(void)distortion;
(void)last_mv_search_list;
@@ -3282,12 +3358,14 @@
// Returns the minimum MV.
int av1_return_min_sub_pixel_mv(MACROBLOCKD *xd, const AV1_COMMON *const cm,
const SUBPEL_MOTION_SEARCH_PARAMS *ms_params,
- MV start_mv, MV *bestmv, int *distortion,
- unsigned int *sse1,
+ MV start_mv,
+ const FULLPEL_MV_STATS *start_mv_stats,
+ MV *bestmv, int *distortion, unsigned int *sse1,
int_mv *last_mv_search_list) {
(void)xd;
(void)cm;
(void)start_mv;
+ (void)start_mv_stats;
(void)sse1;
(void)distortion;
(void)last_mv_search_list;
@@ -3814,9 +3892,11 @@
int av1_find_best_obmc_sub_pixel_tree_up(
MACROBLOCKD *xd, const AV1_COMMON *const cm,
- const SUBPEL_MOTION_SEARCH_PARAMS *ms_params, MV start_mv, MV *bestmv,
- int *distortion, unsigned int *sse1, int_mv *last_mv_search_list) {
+ const SUBPEL_MOTION_SEARCH_PARAMS *ms_params, MV start_mv,
+ const FULLPEL_MV_STATS *start_mv_stats, MV *bestmv, int *distortion,
+ unsigned int *sse1, int_mv *last_mv_search_list) {
(void)last_mv_search_list;
+ (void)start_mv_stats;
const int allow_hp = ms_params->allow_hp;
const int forced_stop = ms_params->forced_stop;
const int iters_per_step = ms_params->iters_per_step;
diff --git a/av1/encoder/mcomp.h b/av1/encoder/mcomp.h
index 6b9af07..87b9309 100644
--- a/av1/encoder/mcomp.h
+++ b/av1/encoder/mcomp.h
@@ -140,13 +140,19 @@
aom_sad_multi_d_fn_t sdx3df;
} FULLPEL_MOTION_SEARCH_PARAMS;
+typedef struct {
+ int err_cost;
+ unsigned int distortion;
+ unsigned int sse;
+} FULLPEL_MV_STATS;
+
void av1_init_obmc_buffer(OBMCBuffer *obmc_buffer);
void av1_make_default_fullpel_ms_params(
FULLPEL_MOTION_SEARCH_PARAMS *ms_params, const struct AV1_COMP *cpi,
MACROBLOCK *x, BLOCK_SIZE bsize, const MV *ref_mv, FULLPEL_MV start_mv,
const search_site_config search_sites[NUM_DISTINCT_SEARCH_METHODS],
- int fine_search_interval);
+ SEARCH_METHODS search_method, int fine_search_interval);
/*! Sets the \ref FULLPEL_MOTION_SEARCH_PARAMS to intra mode. */
void av1_set_ms_to_intra_mode(FULLPEL_MOTION_SEARCH_PARAMS *ms_params,
@@ -256,10 +262,10 @@
int av1_init_search_range(int size);
-unsigned int av1_int_pro_motion_estimation(const struct AV1_COMP *cpi,
- MACROBLOCK *x, BLOCK_SIZE bsize,
- int mi_row, int mi_col,
- const MV *ref_mv);
+unsigned int av1_int_pro_motion_estimation(
+ const struct AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bsize, int mi_row,
+ int mi_col, const MV *ref_mv, unsigned int *y_sad_zero,
+ int me_search_size_col, int me_search_size_row);
int av1_refining_search_8p_c(const FULLPEL_MOTION_SEARCH_PARAMS *ms_params,
const FULLPEL_MV start_mv, FULLPEL_MV *best_mv);
@@ -267,7 +273,8 @@
int av1_full_pixel_search(const FULLPEL_MV start_mv,
const FULLPEL_MOTION_SEARCH_PARAMS *ms_params,
const int step_param, int *cost_list,
- FULLPEL_MV *best_mv, FULLPEL_MV *second_best_mv);
+ FULLPEL_MV *best_mv, FULLPEL_MV_STATS *best_mv_stats,
+ FULLPEL_MV *second_best_mv);
int av1_intrabc_hash_search(const struct AV1_COMP *cpi, const MACROBLOCKD *xd,
const FULLPEL_MOTION_SEARCH_PARAMS *ms_params,
@@ -325,7 +332,9 @@
typedef int(fractional_mv_step_fp)(MACROBLOCKD *xd, const AV1_COMMON *const cm,
const SUBPEL_MOTION_SEARCH_PARAMS *ms_params,
- MV start_mv, MV *bestmv, int *distortion,
+ MV start_mv,
+ const FULLPEL_MV_STATS *start_mv_stats,
+ MV *bestmv, int *distortion,
unsigned int *sse1,
int_mv *last_mv_search_list);
diff --git a/av1/encoder/motion_search_facade.c b/av1/encoder/motion_search_facade.c
index b8c000b..e7eec29 100644
--- a/av1/encoder/motion_search_facade.c
+++ b/av1/encoder/motion_search_facade.c
@@ -239,14 +239,16 @@
// the stride of the reference frame can be different from indicated by
// MotionVectorSearchParams::search_site_cfg. When this happens, we need to
// readjust the stride.
- const SEARCH_METHODS search_method = cpi->sf.mv_sf.search_method;
+ const MV_SPEED_FEATURES *mv_sf = &cpi->sf.mv_sf;
+ const SEARCH_METHODS search_method =
+ av1_get_default_mv_search_method(x, mv_sf, bsize);
const search_site_config *src_search_site_cfg =
av1_get_search_site_config(cpi, x, search_method);
// Further reduce the search range.
if (search_range < INT_MAX) {
const search_site_config *search_site_cfg =
- &src_search_site_cfg[search_method_lookup[cpi->sf.mv_sf.search_method]];
+ &src_search_site_cfg[search_method_lookup[search_method]];
// Max step_param is search_site_cfg->num_search_steps.
if (search_range < 1) {
step_param = search_site_cfg->num_search_steps;
@@ -259,6 +261,7 @@
}
int cost_list[5];
+ FULLPEL_MV_STATS best_mv_stats;
int_mv second_best_mv;
best_mv->as_int = second_best_mv.as_int = INVALID_MV;
@@ -273,21 +276,23 @@
for (int m = 0; m < cand_cnt; m++) {
int_mv smv = cand[m].fmv;
FULLPEL_MV this_best_mv, this_second_best_mv;
+ FULLPEL_MV_STATS this_mv_stats;
if (smv.as_int == INVALID_MV) continue;
av1_make_default_fullpel_ms_params(
&full_ms_params, cpi, x, bsize, &ref_mv, smv.as_fullmv,
- src_search_site_cfg, fine_search_interval);
+ src_search_site_cfg, search_method, fine_search_interval);
const int thissme =
av1_full_pixel_search(smv.as_fullmv, &full_ms_params, step_param,
cond_cost_list(cpi, cost_list), &this_best_mv,
- &this_second_best_mv);
+ &this_mv_stats, &this_second_best_mv);
if (thissme < bestsme) {
bestsme = thissme;
best_mv->as_fullmv = this_best_mv;
+ best_mv_stats = this_mv_stats;
second_best_mv.as_fullmv = this_second_best_mv;
}
@@ -298,7 +303,7 @@
case OBMC_CAUSAL:
av1_make_default_fullpel_ms_params(&full_ms_params, cpi, x, bsize,
&ref_mv, start_mv, src_search_site_cfg,
- fine_search_interval);
+ search_method, fine_search_interval);
bestsme = av1_obmc_full_pixel_search(start_mv, &full_ms_params,
step_param, &best_mv->as_fullmv);
@@ -385,13 +390,13 @@
switch (mbmi->motion_mode) {
case SIMPLE_TRANSLATION:
- if (cpi->sf.mv_sf.use_accurate_subpel_search) {
+ if (mv_sf->use_accurate_subpel_search) {
const int try_second = second_best_mv.as_int != INVALID_MV &&
second_best_mv.as_int != best_mv->as_int &&
- (cpi->sf.mv_sf.disable_second_mv <= 1);
+ (mv_sf->disable_second_mv <= 1);
const int best_mv_var = mv_search_params->find_fractional_mv_step(
- xd, cm, &ms_params, subpel_start_mv, &best_mv->as_mv, &dis,
- &x->pred_sse[ref], fractional_ms_list);
+ xd, cm, &ms_params, subpel_start_mv, &best_mv_stats,
+ &best_mv->as_mv, &dis, &x->pred_sse[ref], fractional_ms_list);
if (try_second) {
struct macroblockd_plane *p = xd->plane;
@@ -400,7 +405,7 @@
{ p[0].dst.stride, p[1].dst.stride, p[2].dst.stride },
};
int64_t rd = INT64_MAX;
- if (!cpi->sf.mv_sf.disable_second_mv) {
+ if (!mv_sf->disable_second_mv) {
// Calculate actual rd cost.
mbmi->mv[0].as_mv = best_mv->as_mv;
av1_enc_build_inter_predictor(cm, xd, mi_row, mi_col, &orig_dst,
@@ -423,10 +428,10 @@
subpel_start_mv)) {
unsigned int sse;
const int this_var = mv_search_params->find_fractional_mv_step(
- xd, cm, &ms_params, subpel_start_mv, &this_best_mv, &dis,
- &sse, fractional_ms_list);
+ xd, cm, &ms_params, subpel_start_mv, NULL, &this_best_mv,
+ &dis, &sse, fractional_ms_list);
- if (!cpi->sf.mv_sf.disable_second_mv) {
+ if (!mv_sf->disable_second_mv) {
// If cpi->sf.mv_sf.disable_second_mv is 0, use actual rd cost
// to choose the better MV.
mbmi->mv[0].as_mv = this_best_mv;
@@ -459,14 +464,14 @@
}
} else {
mv_search_params->find_fractional_mv_step(
- xd, cm, &ms_params, subpel_start_mv, &best_mv->as_mv, &dis,
- &x->pred_sse[ref], NULL);
+ xd, cm, &ms_params, subpel_start_mv, &best_mv_stats,
+ &best_mv->as_mv, &dis, &x->pred_sse[ref], NULL);
}
break;
case OBMC_CAUSAL:
- av1_find_best_obmc_sub_pixel_tree_up(xd, cm, &ms_params,
- subpel_start_mv, &best_mv->as_mv,
- &dis, &x->pred_sse[ref], NULL);
+ av1_find_best_obmc_sub_pixel_tree_up(
+ xd, cm, &ms_params, subpel_start_mv, NULL, &best_mv->as_mv, &dis,
+ &x->pred_sse[ref], NULL);
break;
default: assert(0 && "Invalid motion mode!\n");
}
@@ -621,25 +626,28 @@
// Make motion search params
FULLPEL_MOTION_SEARCH_PARAMS full_ms_params;
- const SEARCH_METHODS search_method = cpi->sf.mv_sf.search_method;
+ FULLPEL_MV_STATS best_mv_stats;
+ const MV_SPEED_FEATURES *mv_sf = &cpi->sf.mv_sf;
+ const SEARCH_METHODS search_method =
+ av1_get_default_mv_search_method(x, mv_sf, bsize);
const search_site_config *src_search_sites =
av1_get_search_site_config(cpi, x, search_method);
// Use the mv result from the single mode as mv predictor.
const FULLPEL_MV start_fullmv = get_fullmv_from_mv(&cur_mv[id].as_mv);
av1_make_default_fullpel_ms_params(&full_ms_params, cpi, x, bsize,
&ref_mv[id].as_mv, start_fullmv,
- src_search_sites,
+ src_search_sites, search_method,
/*fine_search_interval=*/0);
av1_set_ms_compound_refs(&full_ms_params.ms_buffers, second_pred, mask,
mask_stride, id);
// Small-range full-pixel motion search.
- if (!cpi->sf.mv_sf.disable_extensive_joint_motion_search &&
+ if (!mv_sf->disable_extensive_joint_motion_search &&
mbmi->interinter_comp.type != COMPOUND_WEDGE) {
- bestsme =
- av1_full_pixel_search(start_fullmv, &full_ms_params, 5, NULL,
- &best_mv.as_fullmv, &second_best_mv.as_fullmv);
+ bestsme = av1_full_pixel_search(start_fullmv, &full_ms_params, 5, NULL,
+ &best_mv.as_fullmv, &best_mv_stats,
+ &second_best_mv.as_fullmv);
} else {
bestsme = av1_refining_search_8p_c(&full_ms_params, start_fullmv,
&best_mv.as_fullmv);
@@ -683,15 +691,15 @@
MV start_mv = get_mv_from_fullmv(&best_mv.as_fullmv);
assert(av1_is_subpelmv_in_range(&ms_params.mv_limits, start_mv));
bestsme = cpi->mv_search_params.find_fractional_mv_step(
- xd, cm, &ms_params, start_mv, &best_mv.as_mv, &dis, &sse, NULL);
+ xd, cm, &ms_params, start_mv, NULL, &best_mv.as_mv, &dis, &sse, NULL);
if (try_second) {
MV this_best_mv;
MV subpel_start_mv = get_mv_from_fullmv(&second_best_mv.as_fullmv);
if (av1_is_subpelmv_in_range(&ms_params.mv_limits, subpel_start_mv)) {
const int thissme = cpi->mv_search_params.find_fractional_mv_step(
- xd, cm, &ms_params, subpel_start_mv, &this_best_mv, &dis, &sse,
- NULL);
+ xd, cm, &ms_params, subpel_start_mv, NULL, &this_best_mv, &dis,
+ &sse, NULL);
if (thissme < bestsme) {
best_mv.as_mv = this_best_mv;
bestsme = thissme;
@@ -775,14 +783,16 @@
// Make motion search params
FULLPEL_MOTION_SEARCH_PARAMS full_ms_params;
- const SEARCH_METHODS search_method = cpi->sf.mv_sf.search_method;
+ FULLPEL_MV_STATS best_mv_stats;
+ const SEARCH_METHODS search_method =
+ av1_get_default_mv_search_method(x, &cpi->sf.mv_sf, bsize);
const search_site_config *src_search_sites =
av1_get_search_site_config(cpi, x, search_method);
// Use the mv result from the single mode as mv predictor.
const FULLPEL_MV start_fullmv = get_fullmv_from_mv(this_mv);
av1_make_default_fullpel_ms_params(&full_ms_params, cpi, x, bsize,
&ref_mv.as_mv, start_fullmv,
- src_search_sites,
+ src_search_sites, search_method,
/*fine_search_interval=*/0);
av1_set_ms_compound_refs(&full_ms_params.ms_buffers, second_pred, mask,
@@ -790,7 +800,7 @@
// Small-range full-pixel motion search.
bestsme = av1_full_pixel_search(start_fullmv, &full_ms_params, 5, NULL,
- &best_mv.as_fullmv, NULL);
+ &best_mv.as_fullmv, &best_mv_stats, NULL);
if (scaled_ref_frame) {
// Swap back the original buffers for subpel motion search for the 0th slot.
@@ -816,7 +826,8 @@
MV start_mv = get_mv_from_fullmv(&best_mv.as_fullmv);
assert(av1_is_subpelmv_in_range(&ms_params.mv_limits, start_mv));
bestsme = cpi->mv_search_params.find_fractional_mv_step(
- xd, cm, &ms_params, start_mv, &best_mv.as_mv, &dis, &sse, NULL);
+ xd, cm, &ms_params, start_mv, &best_mv_stats, &best_mv.as_mv, &dis,
+ &sse, NULL);
}
// Restore the pointer to the first unscaled prediction buffer.
@@ -954,10 +965,12 @@
return tmp_rate_mv;
}
-int_mv av1_simple_motion_search(AV1_COMP *const cpi, MACROBLOCK *x, int mi_row,
- int mi_col, BLOCK_SIZE bsize, int ref,
- FULLPEL_MV start_mv, int num_planes,
- int use_subpixel) {
+int_mv av1_simple_motion_search_sse_var(AV1_COMP *const cpi, MACROBLOCK *x,
+ int mi_row, int mi_col,
+ BLOCK_SIZE bsize, int ref,
+ FULLPEL_MV start_mv, int num_planes,
+ int use_subpixel, unsigned int *sse,
+ unsigned int *var) {
assert(num_planes == 1 &&
"Currently simple_motion_search only supports luma plane");
assert(!frame_is_intra_only(&cpi->common) &&
@@ -986,8 +999,9 @@
MAX_MVSEARCH_STEPS - 2);
int cost_list[5];
const int ref_idx = 0;
- int var;
+ int bestsme;
int_mv best_mv;
+ FULLPEL_MV_STATS best_mv_stats;
av1_setup_pre_planes(xd, ref_idx, yv12, mi_row, mi_col,
get_ref_scale_factors(cm, ref), num_planes);
@@ -1001,20 +1015,23 @@
// Allow more mesh searches for screen content type on the ARF.
const int fine_search_interval = use_fine_search_interval(cpi);
FULLPEL_MOTION_SEARCH_PARAMS full_ms_params;
- const SEARCH_METHODS search_method = cpi->sf.mv_sf.search_method;
+ const MV_SPEED_FEATURES *mv_sf = &cpi->sf.mv_sf;
+ const SEARCH_METHODS search_method =
+ av1_get_default_mv_search_method(x, mv_sf, bsize);
const search_site_config *src_search_sites =
av1_get_search_site_config(cpi, x, search_method);
av1_make_default_fullpel_ms_params(&full_ms_params, cpi, x, bsize, &ref_mv,
- start_mv, src_search_sites,
+ start_mv, src_search_sites, search_method,
fine_search_interval);
- var = av1_full_pixel_search(start_mv, &full_ms_params, step_param,
- cond_cost_list(cpi, cost_list),
- &best_mv.as_fullmv, NULL);
+ bestsme = av1_full_pixel_search(start_mv, &full_ms_params, step_param,
+ cond_cost_list(cpi, cost_list),
+ &best_mv.as_fullmv, &best_mv_stats, NULL);
const int use_subpel_search =
- var < INT_MAX && !cpi->common.features.cur_frame_force_integer_mv &&
- use_subpixel;
+ bestsme < INT_MAX && !cpi->common.features.cur_frame_force_integer_mv &&
+ use_subpixel &&
+ (cpi->sf.mv_sf.simple_motion_subpel_force_stop != FULL_PEL);
if (scaled_ref_frame) {
xd->plane[AOM_PLANE_Y].pre[ref_idx] = backup_yv12;
}
@@ -1025,50 +1042,30 @@
av1_make_default_subpel_ms_params(&ms_params, cpi, x, bsize, &ref_mv,
cost_list);
// TODO(yunqing): integrate this into av1_make_default_subpel_ms_params().
- ms_params.forced_stop = cpi->sf.mv_sf.simple_motion_subpel_force_stop;
+ ms_params.forced_stop = mv_sf->simple_motion_subpel_force_stop;
MV subpel_start_mv = get_mv_from_fullmv(&best_mv.as_fullmv);
assert(av1_is_subpelmv_in_range(&ms_params.mv_limits, subpel_start_mv));
cpi->mv_search_params.find_fractional_mv_step(
- xd, cm, &ms_params, subpel_start_mv, &best_mv.as_mv, ¬_used,
- &x->pred_sse[ref], NULL);
+ xd, cm, &ms_params, subpel_start_mv, &best_mv_stats, &best_mv.as_mv,
+ ¬_used, &x->pred_sse[ref], NULL);
+
+ mbmi->mv[0] = best_mv;
+
+ // Get a copy of the prediction output
+ av1_enc_build_inter_predictor(cm, xd, mi_row, mi_col, NULL, bsize,
+ AOM_PLANE_Y, AOM_PLANE_Y);
+ *var = cpi->ppi->fn_ptr[bsize].vf(
+ x->plane[0].src.buf, x->plane[0].src.stride, xd->plane[0].dst.buf,
+ xd->plane[0].dst.stride, sse);
} else {
// Manually convert from units of pixel to 1/8-pixels if we are not doing
// subpel search
convert_fullmv_to_mv(&best_mv);
+ *var = best_mv_stats.distortion;
+ *sse = best_mv_stats.sse;
}
- mbmi->mv[0] = best_mv;
-
- // Get a copy of the prediction output
- av1_enc_build_inter_predictor(cm, xd, mi_row, mi_col, NULL, bsize,
- AOM_PLANE_Y, AOM_PLANE_Y);
-
- if (scaled_ref_frame) {
- xd->plane[AOM_PLANE_Y].pre[ref_idx] = backup_yv12;
- }
-
- return best_mv;
-}
-
-int_mv av1_simple_motion_sse_var(AV1_COMP *cpi, MACROBLOCK *x, int mi_row,
- int mi_col, BLOCK_SIZE bsize,
- const FULLPEL_MV start_mv, int use_subpixel,
- unsigned int *sse, unsigned int *var) {
- MACROBLOCKD *xd = &x->e_mbd;
- const MV_REFERENCE_FRAME ref =
- cpi->rc.is_src_frame_alt_ref ? ALTREF_FRAME : LAST_FRAME;
-
- int_mv best_mv = av1_simple_motion_search(cpi, x, mi_row, mi_col, bsize, ref,
- start_mv, 1, use_subpixel);
-
- const uint8_t *src = x->plane[0].src.buf;
- const int src_stride = x->plane[0].src.stride;
- const uint8_t *dst = xd->plane[0].dst.buf;
- const int dst_stride = xd->plane[0].dst.stride;
-
- *var = cpi->ppi->fn_ptr[bsize].vf(src, src_stride, dst, dst_stride, sse);
-
return best_mv;
}
diff --git a/av1/encoder/motion_search_facade.h b/av1/encoder/motion_search_facade.h
index d2996bc..d1fa915 100644
--- a/av1/encoder/motion_search_facade.h
+++ b/av1/encoder/motion_search_facade.h
@@ -59,18 +59,14 @@
int *rate_mv, int ref_idx);
// Performs a motion search in SIMPLE_TRANSLATION mode using reference frame
-// ref. Note that this sets the offset of mbmi, so we will need to reset it
-// after calling this function.
-int_mv av1_simple_motion_search(struct AV1_COMP *const cpi, MACROBLOCK *x,
- int mi_row, int mi_col, BLOCK_SIZE bsize,
- int ref, FULLPEL_MV start_mv, int num_planes,
- int use_subpixel);
-
-// Performs a simple motion search to calculate the sse and var of the residue
-int_mv av1_simple_motion_sse_var(struct AV1_COMP *cpi, MACROBLOCK *x,
- int mi_row, int mi_col, BLOCK_SIZE bsize,
- const FULLPEL_MV start_mv, int use_subpixel,
- unsigned int *sse, unsigned int *var);
+// ref and calculates the sse and var of the residue. Note that this sets the
+// offset of mbmi, so we will need to reset it after calling this function.
+int_mv av1_simple_motion_search_sse_var(struct AV1_COMP *cpi, MACROBLOCK *x,
+ int mi_row, int mi_col,
+ BLOCK_SIZE bsize, int ref,
+ const FULLPEL_MV start_mv,
+ int num_planes, int use_subpixel,
+ unsigned int *sse, unsigned int *var);
static AOM_INLINE const search_site_config *av1_get_search_site_config(
const AV1_COMP *cpi, MACROBLOCK *x, SEARCH_METHODS search_method) {
@@ -101,6 +97,47 @@
return x->search_site_cfg_buf;
}
+static AOM_INLINE SEARCH_METHODS
+av1_get_faster_search_method(SEARCH_METHODS search_method) {
+ // Note on search method's accuracy:
+ // 1. NSTEP
+ // 2. DIAMOND
+ // 3. BIGDIA \approx SQUARE
+ // 4. HEX.
+ // 5. FAST_HEX \approx FAST_DIAMOND
+ switch (search_method) {
+ case NSTEP: return DIAMOND;
+ case NSTEP_8PT: return DIAMOND;
+ case DIAMOND: return BIGDIA;
+ case CLAMPED_DIAMOND: return BIGDIA;
+ case BIGDIA: return HEX;
+ case SQUARE: return HEX;
+ case HEX: return FAST_HEX;
+ case FAST_HEX: return FAST_HEX;
+ case FAST_DIAMOND: return VFAST_DIAMOND;
+ case FAST_BIGDIA: return FAST_BIGDIA;
+ case VFAST_DIAMOND: return VFAST_DIAMOND;
+ default: assert(0 && "Invalid search method!"); return DIAMOND;
+ }
+}
+
+static AOM_INLINE SEARCH_METHODS av1_get_default_mv_search_method(
+ const MACROBLOCK *x, const MV_SPEED_FEATURES *mv_sf, BLOCK_SIZE bsize) {
+ SEARCH_METHODS search_method = mv_sf->search_method;
+ const int sf_blk_search_method = mv_sf->use_bsize_dependent_search_method;
+ const int min_dim = AOMMIN(block_size_wide[bsize], block_size_high[bsize]);
+ const int qband = x->qindex >> (QINDEX_BITS - 2);
+ const bool use_faster_search_method =
+ (sf_blk_search_method == 1 && min_dim >= 32) ||
+ (sf_blk_search_method >= 2 && min_dim >= 16 &&
+ x->content_state_sb.source_sad_nonrd <= kMedSad && qband < 3);
+
+ if (use_faster_search_method) {
+ search_method = av1_get_faster_search_method(search_method);
+ }
+ return search_method;
+}
+
#ifdef __cplusplus
} // extern "C"
#endif
diff --git a/av1/encoder/nonrd_opt.h b/av1/encoder/nonrd_opt.h
index 36a0c23..4fd88a7 100644
--- a/av1/encoder/nonrd_opt.h
+++ b/av1/encoder/nonrd_opt.h
@@ -103,6 +103,8 @@
int use_ref_frame_mask[REF_FRAMES];
//! Array to hold flags of evaluated modes for each reference frame
uint8_t mode_checked[MB_MODE_COUNT][REF_FRAMES];
+ //! Array to hold flag indicating if scaled reference frame is used.
+ bool use_scaled_ref_frame[REF_FRAMES];
} InterModeSearchStateNonrd;
static const uint8_t b_width_log2_lookup[BLOCK_SIZES] = { 0, 0, 1, 1, 1, 2,
@@ -412,30 +414,36 @@
* data for the current macroblock
* \param[in] ref_frame Reference frame for which to find
* ref MVs
- * \param[in] frame_mv Predicted MVs for a block
+ * \param[out] frame_mv Predicted MVs for a block
* \param[in] yv12_mb Buffer to hold predicted block
* \param[in] bsize Current block size
* \param[in] force_skip_low_temp_var Flag indicating possible mode search
* prune for low temporal variance block
* \param[in] skip_pred_mv Flag indicating to skip av1_mv_pred
+ * \param[out] use_scaled_ref_frame Flag to indicate if scaled reference
+ * frame is used.
*
* \remark Nothing is returned. Instead, predicted MVs are placed into
- * \c frame_mv array
+ * \c frame_mv array, and use_scaled_ref_frame is set.
*/
-static INLINE void find_predictors(AV1_COMP *cpi, MACROBLOCK *x,
- MV_REFERENCE_FRAME ref_frame,
- int_mv frame_mv[MB_MODE_COUNT][REF_FRAMES],
- struct buf_2d yv12_mb[8][MAX_MB_PLANE],
- BLOCK_SIZE bsize,
- int force_skip_low_temp_var,
- int skip_pred_mv) {
+static INLINE void find_predictors(
+ AV1_COMP *cpi, MACROBLOCK *x, MV_REFERENCE_FRAME ref_frame,
+ int_mv frame_mv[MB_MODE_COUNT][REF_FRAMES],
+ struct buf_2d yv12_mb[8][MAX_MB_PLANE], BLOCK_SIZE bsize,
+ int force_skip_low_temp_var, int skip_pred_mv, bool *use_scaled_ref_frame) {
AV1_COMMON *const cm = &cpi->common;
MACROBLOCKD *const xd = &x->e_mbd;
MB_MODE_INFO *const mbmi = xd->mi[0];
MB_MODE_INFO_EXT *const mbmi_ext = &x->mbmi_ext;
- const YV12_BUFFER_CONFIG *yv12 = get_ref_frame_yv12_buf(cm, ref_frame);
+ int base_is_key_frame = 0;
+ const YV12_BUFFER_CONFIG *ref = get_ref_frame_yv12_buf(cm, ref_frame);
+ const bool ref_is_scaled =
+ ref->y_crop_height != cm->height || ref->y_crop_width != cm->width;
+ const YV12_BUFFER_CONFIG *scaled_ref =
+ av1_get_scaled_ref_frame(cpi, ref_frame);
+ const YV12_BUFFER_CONFIG *yv12 =
+ ref_is_scaled && scaled_ref ? scaled_ref : ref;
const int num_planes = av1_num_planes(cm);
-
x->pred_mv_sad[ref_frame] = INT_MAX;
x->pred_mv0_sad[ref_frame] = INT_MAX;
x->pred_mv1_sad[ref_frame] = INT_MAX;
@@ -443,8 +451,15 @@
// TODO(kyslov) this needs various further optimizations. to be continued..
assert(yv12 != NULL);
if (yv12 != NULL) {
- const struct scale_factors *const sf =
- get_ref_scale_factors_const(cm, ref_frame);
+ if (cpi->ppi->use_svc) {
+ SVC *const svc = &cpi->svc;
+ const int layer =
+ LAYER_IDS_TO_IDX(svc->spatial_layer_id, svc->temporal_layer_id,
+ svc->number_temporal_layers);
+ base_is_key_frame = svc->layer_context[layer].is_key_frame;
+ }
+ struct scale_factors *const sf =
+ scaled_ref ? NULL : get_ref_scale_factors(cm, ref_frame);
av1_setup_pred_block(xd, yv12_mb[ref_frame], yv12, sf, sf, num_planes);
av1_find_mv_refs(cm, xd, mbmi, ref_frame, mbmi_ext->ref_mv_count,
xd->ref_mv_stack, xd->weight, NULL, mbmi_ext->global_mvs,
@@ -456,9 +471,11 @@
cm->features.allow_high_precision_mv, mbmi_ext, ref_frame,
&frame_mv[NEARESTMV][ref_frame], &frame_mv[NEARMV][ref_frame], 0);
frame_mv[GLOBALMV][ref_frame] = mbmi_ext->global_mvs[ref_frame];
- // Early exit for non-LAST frame if force_skip_low_temp_var is set.
- if (!av1_is_scaled(sf) && bsize >= BLOCK_8X8 && !skip_pred_mv &&
- !(force_skip_low_temp_var && ref_frame != LAST_FRAME)) {
+ // Early exit for non-LAST frame if force_skip_low_temp_var or
+ // ref_is_scaled is set.
+ if (bsize >= BLOCK_8X8 && !skip_pred_mv && !base_is_key_frame &&
+ !((ref_is_scaled || force_skip_low_temp_var) &&
+ ref_frame != LAST_FRAME)) {
av1_mv_pred(cpi, x, yv12_mb[ref_frame][0].buf, yv12->y_stride, ref_frame,
bsize);
}
@@ -467,6 +484,7 @@
av1_count_overlappable_neighbors(cm, xd);
}
mbmi->num_proj_ref = 1;
+ *use_scaled_ref_frame = ref_is_scaled && scaled_ref;
}
static INLINE void init_mbmi_nonrd(MB_MODE_INFO *mbmi,
diff --git a/av1/encoder/nonrd_pickmode.c b/av1/encoder/nonrd_pickmode.c
index 24a5264..1836869 100644
--- a/av1/encoder/nonrd_pickmode.c
+++ b/av1/encoder/nonrd_pickmode.c
@@ -179,8 +179,6 @@
* \param[in] x Pointer to structure holding all the
* data for the current macroblock
* \param[in] bsize Current block size
- * \param[in] mi_row Row index in 4x4 units
- * \param[in] mi_col Column index in 4x4 units
* \param[in] tmp_mv Pointer to best found New MV
* \param[in] rate_mv Pointer to Rate of the best new MV
* \param[in] best_rd_sofar RD Cost of the best mode found so far
@@ -192,15 +190,13 @@
* Rate estimation for this vector is placed to \c rate_mv
*/
static int combined_motion_search(AV1_COMP *cpi, MACROBLOCK *x,
- BLOCK_SIZE bsize, int mi_row, int mi_col,
- int_mv *tmp_mv, int *rate_mv,
- int64_t best_rd_sofar, int use_base_mv) {
+ BLOCK_SIZE bsize, int_mv *tmp_mv,
+ int *rate_mv, int64_t best_rd_sofar,
+ int use_base_mv) {
MACROBLOCKD *xd = &x->e_mbd;
const AV1_COMMON *cm = &cpi->common;
- const int num_planes = av1_num_planes(cm);
const SPEED_FEATURES *sf = &cpi->sf;
MB_MODE_INFO *mi = xd->mi[0];
- struct buf_2d backup_yv12[MAX_MB_PLANE] = { { 0, 0, 0, 0, 0 } };
int step_param = (sf->rt_sf.fullpel_search_step_param)
? sf->rt_sf.fullpel_search_step_param
: cpi->mv_search_params.mv_step_param;
@@ -212,19 +208,6 @@
int rv = 0;
int cost_list[5];
int search_subpel = 1;
- const YV12_BUFFER_CONFIG *scaled_ref_frame =
- av1_get_scaled_ref_frame(cpi, ref);
-
- if (scaled_ref_frame) {
- int plane;
- // Swap out the reference frame for a version that's been scaled to
- // match the resolution of the current frame, allowing the existing
- // motion search code to be used without additional modifications.
- for (plane = 0; plane < MAX_MB_PLANE; plane++)
- backup_yv12[plane] = xd->plane[plane].pre[0];
- av1_setup_pre_planes(xd, 0, scaled_ref_frame, mi_row, mi_col, NULL,
- num_planes);
- }
start_mv = get_fullmv_from_mv(&ref_mv);
@@ -233,17 +216,19 @@
else
center_mv = tmp_mv->as_mv;
- const SEARCH_METHODS search_method = sf->mv_sf.search_method;
+ const SEARCH_METHODS search_method =
+ av1_get_default_mv_search_method(x, &cpi->sf.mv_sf, bsize);
const search_site_config *src_search_sites =
av1_get_search_site_config(cpi, x, search_method);
FULLPEL_MOTION_SEARCH_PARAMS full_ms_params;
+ FULLPEL_MV_STATS best_mv_stats;
av1_make_default_fullpel_ms_params(&full_ms_params, cpi, x, bsize, ¢er_mv,
- start_mv, src_search_sites,
+ start_mv, src_search_sites, search_method,
/*fine_search_interval=*/0);
const unsigned int full_var_rd = av1_full_pixel_search(
start_mv, &full_ms_params, step_param, cond_cost_list(cpi, cost_list),
- &tmp_mv->as_fullmv, NULL);
+ &tmp_mv->as_fullmv, &best_mv_stats, NULL);
// calculate the bit cost on motion vector
MV mvp_full = get_mv_from_fullmv(&tmp_mv->as_fullmv);
@@ -272,22 +257,17 @@
// adaptively downgrade subpel search method based on block properties
if (use_aggressive_subpel_search_method(
x, sf->rt_sf.use_adaptive_subpel_search, fullpel_performed_well))
- av1_find_best_sub_pixel_tree_pruned_more(xd, cm, &ms_params,
- subpel_start_mv, &tmp_mv->as_mv,
- &dis, &x->pred_sse[ref], NULL);
+ av1_find_best_sub_pixel_tree_pruned_more(
+ xd, cm, &ms_params, subpel_start_mv, &best_mv_stats, &tmp_mv->as_mv,
+ &dis, &x->pred_sse[ref], NULL);
else
cpi->mv_search_params.find_fractional_mv_step(
- xd, cm, &ms_params, subpel_start_mv, &tmp_mv->as_mv, &dis,
- &x->pred_sse[ref], NULL);
+ xd, cm, &ms_params, subpel_start_mv, &best_mv_stats, &tmp_mv->as_mv,
+ &dis, &x->pred_sse[ref], NULL);
*rate_mv =
av1_mv_bit_cost(&tmp_mv->as_mv, &ref_mv, x->mv_costs->nmv_joint_cost,
x->mv_costs->mv_cost_stack, MV_COST_WEIGHT);
}
-
- if (scaled_ref_frame) {
- for (int plane = 0; plane < MAX_MB_PLANE; plane++)
- xd->plane[plane].pre[0] = backup_yv12[plane];
- }
// The final MV can not be equal to the reference MV as this will trigger an
// assert later. This can happen if both NEAREST and NEAR modes were skipped.
rv = (tmp_mv->as_mv.col != ref_mv.col || tmp_mv->as_mv.row != ref_mv.row);
@@ -331,6 +311,7 @@
MB_MODE_INFO *const mi = xd->mi[0];
AV1_COMMON *cm = &cpi->common;
int_mv *this_ref_frm_newmv = &frame_mv[NEWMV][ref_frame];
+ unsigned int y_sad_zero;
if (ref_frame > LAST_FRAME && cpi->oxcf.rc_cfg.mode == AOM_CBR &&
gf_temporal_ref) {
int tmp_sad;
@@ -338,9 +319,12 @@
if (bsize < BLOCK_16X16) return -1;
+ int me_search_size_col = block_size_wide[bsize] >> 1;
+ int me_search_size_row = block_size_high[bsize] >> 1;
tmp_sad = av1_int_pro_motion_estimation(
cpi, x, bsize, mi_row, mi_col,
- &x->mbmi_ext.ref_mv_stack[ref_frame][0].this_mv.as_mv);
+ &x->mbmi_ext.ref_mv_stack[ref_frame][0].this_mv.as_mv, &y_sad_zero,
+ me_search_size_col, me_search_size_row);
if (tmp_sad > x->pred_mv_sad[LAST_FRAME]) return -1;
@@ -363,7 +347,7 @@
MV start_mv = get_mv_from_fullmv(&best_mv.as_fullmv);
assert(av1_is_subpelmv_in_range(&ms_params.mv_limits, start_mv));
cpi->mv_search_params.find_fractional_mv_step(
- xd, cm, &ms_params, start_mv, &best_mv.as_mv, &dis,
+ xd, cm, &ms_params, start_mv, NULL, &best_mv.as_mv, &dis,
&x->pred_sse[ref_frame], NULL);
this_ref_frm_newmv->as_int = best_mv.as_int;
@@ -378,9 +362,8 @@
*rate_mv = av1_mv_bit_cost(&this_ref_frm_newmv->as_mv, &ref_mv,
x->mv_costs->nmv_joint_cost,
x->mv_costs->mv_cost_stack, MV_COST_WEIGHT);
- } else if (!combined_motion_search(cpi, x, bsize, mi_row, mi_col,
- &frame_mv[NEWMV][ref_frame], rate_mv,
- best_rdc->rdcost, 0)) {
+ } else if (!combined_motion_search(cpi, x, bsize, &frame_mv[NEWMV][ref_frame],
+ rate_mv, best_rdc->rdcost, 0)) {
return -1;
}
@@ -1689,7 +1672,7 @@
static AOM_INLINE void get_ref_frame_use_mask(AV1_COMP *cpi, MACROBLOCK *x,
MB_MODE_INFO *mi, int mi_row,
- int mi_col, int bsize,
+ int mi_col, BLOCK_SIZE bsize,
int gf_temporal_ref,
int use_ref_frame[],
int *force_skip_low_temp_var) {
@@ -1779,7 +1762,7 @@
x->nonrd_prune_ref_frame_search > 2 &&
x->color_sensitivity_sb_g[COLOR_SENS_IDX(AOM_PLANE_U)] == 0 &&
x->color_sensitivity_sb_g[COLOR_SENS_IDX(AOM_PLANE_V)] == 0) {
- int thr = (cm->width * cm->height >= 640 * 360) ? 100 : 150;
+ int thr = (cm->width * cm->height > RESOLUTION_288P) ? 100 : 150;
int pred = x->pred_mv_sad[LAST_FRAME] >>
(b_width_log2_lookup[bsize] + b_height_log2_lookup[bsize]);
if (pred > thr) use_golden_ref_frame = 1;
@@ -1804,10 +1787,10 @@
use_ref_frame[ALTREF_FRAME] = use_alt_ref_frame;
use_ref_frame[GOLDEN_FRAME] = use_golden_ref_frame;
use_ref_frame[LAST_FRAME] = use_last_ref_frame;
- // For now keep this assert on, but we should remove it for svc mode,
- // as the user may want to generate an intra-only frame (no inter-modes).
- // Remove this assert in subsequent CL when nonrd_pickmode is tested for the
- // case of intra-only frame (no references enabled).
+ // Keep this assert on, as only 3 references are used in nonrd_pickmode
+ // (LAST, GOLDEN, ALTREF), and if all 3 are not set by user then this
+ // frame must be an intra-only frame and hence should never enter the
+ // pickmode here for inter frames.
assert(use_last_ref_frame || use_golden_ref_frame || use_alt_ref_frame);
}
@@ -1918,6 +1901,14 @@
return 0;
}
+static void set_block_source_sad(AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bsize,
+ struct buf_2d *yv12_mb) {
+ struct macroblock_plane *const p = &x->plane[0];
+ const int y_sad = cpi->ppi->fn_ptr[bsize].sdf(p->src.buf, p->src.stride,
+ yv12_mb->buf, yv12_mb->stride);
+ if (y_sad == 0) x->block_is_zero_sad = 1;
+}
+
static void set_color_sensitivity(AV1_COMP *cpi, MACROBLOCK *x,
BLOCK_SIZE bsize, int y_sad,
unsigned int source_variance,
@@ -1925,14 +1916,33 @@
const int subsampling_x = cpi->common.seq_params->subsampling_x;
const int subsampling_y = cpi->common.seq_params->subsampling_y;
const int source_sad_nonrd = x->content_state_sb.source_sad_nonrd;
+ const int high_res = cpi->common.width * cpi->common.height >= 640 * 360;
+ if (bsize == cpi->common.seq_params->sb_size) {
+ // At superblock level color_sensitivity is already set to 0, 1, or 2.
+ // 2 is middle/uncertain level. To avoid additional sad
+ // computations when bsize = sb_size force level 2 to 1 (certain color)
+ // for motion areas.
+ if (x->color_sensitivity[COLOR_SENS_IDX(AOM_PLANE_U)] == 2) {
+ x->color_sensitivity[COLOR_SENS_IDX(AOM_PLANE_U)] =
+ source_sad_nonrd >= kMedSad ? 1 : 0;
+ }
+ if (x->color_sensitivity[COLOR_SENS_IDX(AOM_PLANE_V)] == 2) {
+ x->color_sensitivity[COLOR_SENS_IDX(AOM_PLANE_V)] =
+ source_sad_nonrd >= kMedSad ? 1 : 0;
+ }
+ return;
+ }
int shift = 3;
- if (source_sad_nonrd >= kMedSad &&
- cpi->oxcf.tune_cfg.content != AOM_CONTENT_SCREEN &&
- cpi->common.width * cpi->common.height >= 640 * 360)
+ unsigned int source_var_thr = 50;
+ int uv_sad_thr = 100;
+ if (source_sad_nonrd >= kMedSad && x->source_variance > 0 && high_res)
shift = 4;
- if (cpi->oxcf.tune_cfg.content == AOM_CONTENT_SCREEN &&
- cpi->rc.high_source_sad) {
- shift = 6;
+ if (cpi->oxcf.tune_cfg.content == AOM_CONTENT_SCREEN) {
+ if (cpi->rc.high_source_sad) shift = 6;
+ if (source_sad_nonrd > kMedSad) {
+ source_var_thr = 1200;
+ uv_sad_thr = 10;
+ }
}
NOISE_LEVEL noise_level = kLow;
int norm_sad =
@@ -1953,8 +1963,12 @@
const int num_planes = av1_num_planes(&cpi->common);
for (int plane = AOM_PLANE_U; plane < num_planes; ++plane) {
+ // Always check if level = 2. If level = 0 check again for
+ // motion areas for higher resolns, where color artifacts
+ // are more noticeable.
if (x->color_sensitivity[COLOR_SENS_IDX(plane)] == 2 ||
- source_variance < 50) {
+ (x->color_sensitivity[COLOR_SENS_IDX(plane)] == 0 &&
+ source_sad_nonrd >= kMedSad && high_res)) {
struct macroblock_plane *const p = &x->plane[plane];
const BLOCK_SIZE bs =
get_plane_block_size(bsize, subsampling_x, subsampling_y);
@@ -1966,7 +1980,7 @@
uv_sad >> (b_width_log2_lookup[bs] + b_height_log2_lookup[bs]);
x->color_sensitivity[COLOR_SENS_IDX(plane)] =
uv_sad > (y_sad >> shift) && norm_uv_sad > 40;
- if (source_variance < 50 && norm_uv_sad > 100)
+ if (source_variance < source_var_thr && norm_uv_sad > uv_sad_thr)
x->color_sensitivity[COLOR_SENS_IDX(plane)] = 1;
}
}
@@ -2213,9 +2227,8 @@
// Function to setup parameters used for inter mode evaluation in non-rd.
static AOM_FORCE_INLINE void set_params_nonrd_pick_inter_mode(
AV1_COMP *cpi, MACROBLOCK *x, InterModeSearchStateNonrd *search_state,
- RD_STATS *rd_cost, int *force_skip_low_temp_var, int *skip_pred_mv,
- int mi_row, int mi_col, int gf_temporal_ref, unsigned char segment_id,
- BLOCK_SIZE bsize
+ RD_STATS *rd_cost, int *force_skip_low_temp_var, int mi_row, int mi_col,
+ int gf_temporal_ref, unsigned char segment_id, BLOCK_SIZE bsize
#if CONFIG_AV1_TEMPORAL_DENOISING
,
PICK_MODE_CONTEXT *ctx, int denoise_svc_pickmode
@@ -2226,6 +2239,7 @@
TxfmSearchInfo *txfm_info = &x->txfm_search_info;
MB_MODE_INFO *const mi = xd->mi[0];
const ModeCosts *mode_costs = &x->mode_costs;
+ int skip_pred_mv = 0;
// Initialize variance and distortion (chroma) for all modes and reference
// frames
@@ -2272,20 +2286,21 @@
#endif
// Populate predicated motion vectors for LAST_FRAME
- if (cpi->ref_frame_flags & AOM_LAST_FLAG)
+ if (cpi->ref_frame_flags & AOM_LAST_FLAG) {
find_predictors(cpi, x, LAST_FRAME, search_state->frame_mv,
search_state->yv12_mb, bsize, *force_skip_low_temp_var,
- x->force_zeromv_skip_for_blk);
-
+ x->force_zeromv_skip_for_blk,
+ &search_state->use_scaled_ref_frame[LAST_FRAME]);
+ }
// Update mask to use all reference frame
get_ref_frame_use_mask(cpi, x, mi, mi_row, mi_col, bsize, gf_temporal_ref,
search_state->use_ref_frame_mask,
force_skip_low_temp_var);
- *skip_pred_mv = x->force_zeromv_skip_for_blk ||
- (x->nonrd_prune_ref_frame_search > 2 &&
- x->color_sensitivity[COLOR_SENS_IDX(AOM_PLANE_U)] != 2 &&
- x->color_sensitivity[COLOR_SENS_IDX(AOM_PLANE_V)] != 2);
+ skip_pred_mv = x->force_zeromv_skip_for_blk ||
+ (x->nonrd_prune_ref_frame_search > 2 &&
+ x->color_sensitivity[COLOR_SENS_IDX(AOM_PLANE_U)] != 2 &&
+ x->color_sensitivity[COLOR_SENS_IDX(AOM_PLANE_V)] != 2);
// Populate predicated motion vectors for other single reference frame
// Start at LAST_FRAME + 1.
@@ -2294,7 +2309,8 @@
if (search_state->use_ref_frame_mask[ref_frame_iter]) {
find_predictors(cpi, x, ref_frame_iter, search_state->frame_mv,
search_state->yv12_mb, bsize, *force_skip_low_temp_var,
- *skip_pred_mv);
+ skip_pred_mv,
+ &search_state->use_scaled_ref_frame[ref_frame_iter]);
}
}
}
@@ -2334,6 +2350,27 @@
*ref_frame2 = NONE_FRAME;
}
+ if (x->sb_me_block && *ref_frame == LAST_FRAME) {
+ // We want to make sure to test the superblock MV:
+ // so don't skip (return false) for NEAREST_LAST or NEAR_LAST if they
+ // have this sb MV. And don't skip NEWMV_LAST: this will be set to
+ // sb MV in handle_inter_mode_nonrd(), in case NEAREST or NEAR don't
+ // have it.
+ if (*this_mode == NEARESTMV &&
+ search_state->frame_mv[NEARESTMV][LAST_FRAME].as_int ==
+ x->sb_me_mv.as_int) {
+ return false;
+ }
+ if (*this_mode == NEARMV &&
+ search_state->frame_mv[NEARMV][LAST_FRAME].as_int ==
+ x->sb_me_mv.as_int) {
+ return false;
+ }
+ if (*this_mode == NEWMV) {
+ return false;
+ }
+ }
+
// Skip the single reference mode for which mode check flag is set.
if (*is_single_pred && search_state->mode_checked[*this_mode][*ref_frame]) {
return true;
@@ -2397,13 +2434,12 @@
get_segdata(seg, segment_id, SEG_LVL_REF_FRAME) != (int)(*ref_frame))
return true;
- // For screen content: for base spatial layer only for now.
- if (cpi->oxcf.tune_cfg.content == AOM_CONTENT_SCREEN &&
- cpi->svc.spatial_layer_id == 0) {
+ // For screen content: skip mode testing based on source_sad.
+ if (cpi->oxcf.tune_cfg.content == AOM_CONTENT_SCREEN) {
// If source_sad is computed: skip non-zero motion
// check for stationary (super)blocks. Otherwise if superblock
- // has motion skip the modes with zero motion for flat blocks,
- // and color is not set.
+ // has motion skip the modes with zero motion on last reference
+ // for flat blocks, and color is not set.
// For the latter condition: the same condition should apply
// to newmv if (0, 0), so this latter condition is repeated
// below after search_new_mv.
@@ -2411,9 +2447,9 @@
if ((search_state->frame_mv[*this_mode][*ref_frame].as_int != 0 &&
x->content_state_sb.source_sad_nonrd == kZeroSad) ||
(search_state->frame_mv[*this_mode][*ref_frame].as_int == 0 &&
- x->content_state_sb.source_sad_nonrd != kZeroSad &&
- ((x->color_sensitivity[COLOR_SENS_IDX(AOM_PLANE_U)] == 0 &&
- x->color_sensitivity[COLOR_SENS_IDX(AOM_PLANE_V)] == 0) ||
+ x->block_is_zero_sad == 0 && *ref_frame == LAST_FRAME &&
+ ((x->color_sensitivity_sb[COLOR_SENS_IDX(AOM_PLANE_U)] == 0 &&
+ x->color_sensitivity_sb[COLOR_SENS_IDX(AOM_PLANE_V)] == 0) ||
cpi->rc.high_source_sad) &&
x->source_variance == 0))
return true;
@@ -2479,10 +2515,11 @@
#if CONFIG_AV1_TEMPORAL_DENOISING
int64_t *zero_last_cost_orig, int denoise_svc_pickmode,
#endif
- int idx, int force_mv_inter_layer, int is_single_pred, int skip_pred_mv,
- int gf_temporal_ref, int use_model_yrd_large, int filter_search_enabled_blk,
- BLOCK_SIZE bsize, PREDICTION_MODE this_mode, InterpFilter filt_select,
- int cb_pred_filter_search, int reuse_inter_pred) {
+ int idx, int force_mv_inter_layer, int is_single_pred, int gf_temporal_ref,
+ int use_model_yrd_large, int filter_search_enabled_blk, BLOCK_SIZE bsize,
+ PREDICTION_MODE this_mode, InterpFilter filt_select,
+ int cb_pred_filter_search, int reuse_inter_pred,
+ int *sb_me_has_been_tested) {
AV1_COMMON *const cm = &cpi->common;
MACROBLOCKD *const xd = &x->e_mbd;
MB_MODE_INFO *const mi = xd->mi[0];
@@ -2511,7 +2548,10 @@
RD_STATS nonskip_rdc;
av1_invalid_rd_stats(&nonskip_rdc);
- if (this_mode == NEWMV && !force_mv_inter_layer) {
+ if (x->sb_me_block && this_mode == NEWMV && ref_frame == LAST_FRAME) {
+ // Set the NEWMV_LAST to the sb MV.
+ search_state->frame_mv[NEWMV][LAST_FRAME].as_int = x->sb_me_mv.as_int;
+ } else if (this_mode == NEWMV && !force_mv_inter_layer) {
#if COLLECT_NONRD_PICK_MODE_STAT
aom_usec_timer_start(&x->ms_stat_nonrd.timer2);
#endif
@@ -2552,13 +2592,13 @@
if (skip_this_mv && is_single_pred) return true;
// For screen: for spatially flat blocks with non-zero motion,
- // skip newmv if the motion vector is (0, 0), and color is not set.
+ // skip newmv if the motion vector is (0, 0)-LAST, and color is not set.
if (this_mode == NEWMV && cpi->oxcf.tune_cfg.content == AOM_CONTENT_SCREEN &&
cpi->svc.spatial_layer_id == 0 && rt_sf->source_metrics_sb_nonrd) {
- if (this_mv->as_int == 0 &&
- x->content_state_sb.source_sad_nonrd != kZeroSad &&
- ((x->color_sensitivity[COLOR_SENS_IDX(AOM_PLANE_U)] == 0 &&
- x->color_sensitivity[COLOR_SENS_IDX(AOM_PLANE_V)] == 0) ||
+ if (this_mv->as_int == 0 && ref_frame == LAST_FRAME &&
+ x->block_is_zero_sad == 0 &&
+ ((x->color_sensitivity_sb[COLOR_SENS_IDX(AOM_PLANE_U)] == 0 &&
+ x->color_sensitivity_sb[COLOR_SENS_IDX(AOM_PLANE_V)] == 0) ||
cpi->rc.high_source_sad) &&
x->source_variance == 0)
return true;
@@ -2581,32 +2621,6 @@
}
}
- if (idx == 0 && !skip_pred_mv) {
- // Set color sensitivity on first tested mode only.
- // Use y-sad already computed in find_predictors: take the sad with motion
- // vector closest to 0; the uv-sad computed below in set_color_sensitivity
- // is for zeromv.
- // For screen: first check if golden reference is being used, if so,
- // force color_sensitivity on if the color sensitivity for sb_g is on.
- if (cpi->oxcf.tune_cfg.content == AOM_CONTENT_SCREEN &&
- search_state->use_ref_frame_mask[GOLDEN_FRAME]) {
- if (x->color_sensitivity_sb_g[COLOR_SENS_IDX(AOM_PLANE_U)] == 1)
- x->color_sensitivity[COLOR_SENS_IDX(AOM_PLANE_U)] = 1;
- if (x->color_sensitivity_sb_g[COLOR_SENS_IDX(AOM_PLANE_V)] == 1)
- x->color_sensitivity[COLOR_SENS_IDX(AOM_PLANE_V)] = 1;
- } else {
- int y_sad = x->pred_mv0_sad[LAST_FRAME];
- if (x->pred_mv1_sad[LAST_FRAME] != INT_MAX &&
- (abs(search_state->frame_mv[NEARMV][LAST_FRAME].as_mv.col) +
- abs(search_state->frame_mv[NEARMV][LAST_FRAME].as_mv.row)) <
- (abs(search_state->frame_mv[NEARESTMV][LAST_FRAME].as_mv.col) +
- abs(search_state->frame_mv[NEARESTMV][LAST_FRAME].as_mv.row)))
- y_sad = x->pred_mv1_sad[LAST_FRAME];
- set_color_sensitivity(cpi, x, bsize, y_sad, x->source_variance,
- search_state->yv12_mb[LAST_FRAME]);
- }
- }
-
mi->motion_mode = SIMPLE_TRANSLATION;
#if !CONFIG_REALTIME_ONLY
if (cpi->oxcf.motion_mode_cfg.allow_warped_motion) {
@@ -2786,6 +2800,8 @@
// Compute sse for chroma planes.
const int64_t sse_uv = av1_model_rd_for_sb_uv(
cpi, uv_bsize, x, xd, &rdc_uv, AOM_PLANE_U, AOM_PLANE_V);
+ if (rdc_uv.dist < x->min_dist_inter_uv)
+ x->min_dist_inter_uv = rdc_uv.dist;
search_state->this_rdc.sse += sse_uv;
// Restore Y rdc if UV rdc disallows txfm skip
if (search_state->this_rdc.skip_txfm && !rdc_uv.skip_txfm &&
@@ -2875,6 +2891,11 @@
aom_usec_timer_elapsed(&x->ms_stat_nonrd.timer1);
#endif
+ if (x->sb_me_block && ref_frame == LAST_FRAME &&
+ search_state->frame_mv[this_best_mode][ref_frame].as_int ==
+ x->sb_me_mv.as_int)
+ *sb_me_has_been_tested = 1;
+
// Copy best mode params to search state
if (search_state->this_rdc.rdcost < search_state->best_rdc.rdcost) {
search_state->best_rdc = search_state->this_rdc;
@@ -2900,7 +2921,7 @@
if (*best_early_term && (idx > 0 || rt_sf->nonrd_aggressive_skip)) {
txfm_info->skip_txfm = 1;
- return false;
+ if (!x->sb_me_block || *sb_me_has_been_tested) return false;
}
return true;
}
@@ -2960,6 +2981,8 @@
}
av1_model_rd_for_sb_uv(cpi, uv_bsize, x, xd, &rdc_uv, AOM_PLANE_U,
AOM_PLANE_V);
+ if (rdc_uv.dist < x->min_dist_inter_uv)
+ x->min_dist_inter_uv = rdc_uv.dist;
idtx_rdc.rate += rdc_uv.rate;
idtx_rdc.dist += rdc_uv.dist;
idtx_rdc.skip_txfm = idtx_rdc.skip_txfm && rdc_uv.skip_txfm;
@@ -3071,7 +3094,6 @@
int best_early_term = 0;
int force_skip_low_temp_var = 0;
unsigned int sse_zeromv_norm = UINT_MAX;
- int skip_pred_mv = 0;
const int num_inter_modes = NUM_INTER_MODES;
const REAL_TIME_SPEED_FEATURES *const rt_sf = &cpi->sf.rt_sf;
bool check_globalmv = rt_sf->check_globalmv_on_single_ref;
@@ -3082,6 +3104,7 @@
rt_sf->reuse_inter_pred_nonrd && cm->seq_params->bit_depth == AOM_BITS_8;
InterModeSearchStateNonrd search_state;
av1_zero(search_state.use_ref_frame_mask);
+ av1_zero(search_state.use_scaled_ref_frame);
BEST_PICKMODE *const best_pickmode = &search_state.best_pickmode;
(void)tile_data;
@@ -3111,7 +3134,9 @@
const int resize_pending = is_frame_resize_pending(cpi);
#endif
const ModeCosts *mode_costs = &x->mode_costs;
-
+ struct scale_factors sf_no_scale;
+ av1_setup_scale_factors_for_frame(&sf_no_scale, cm->width, cm->height,
+ cm->width, cm->height);
if (reuse_inter_pred) {
for (int buf_idx = 0; buf_idx < 3; buf_idx++) {
tmp_buffer[buf_idx].data = &pred_buf[pixels_in_block * buf_idx];
@@ -3130,7 +3155,9 @@
// to source, so use subpel motion vector to compensate. The nonzero motion
// is half pixel shifted to left and top, so (-4, -4). This has more effect
// on higher resolutions, so condition it on that for now.
+ // Exclude quality layers, which have the same resolution and hence no shift.
if (cpi->ppi->use_svc && svc->spatial_layer_id > 0 &&
+ !svc->has_lower_quality_layer &&
svc->downsample_filter_phase[svc->spatial_layer_id - 1] == 8 &&
cm->width * cm->height > 640 * 480) {
svc_mv.as_mv.row = -4;
@@ -3138,12 +3165,12 @@
}
// Setup parameters used for inter mode evaluation.
- set_params_nonrd_pick_inter_mode(
- cpi, x, &search_state, rd_cost, &force_skip_low_temp_var, &skip_pred_mv,
- mi_row, mi_col, gf_temporal_ref, segment_id, bsize
+ set_params_nonrd_pick_inter_mode(cpi, x, &search_state, rd_cost,
+ &force_skip_low_temp_var, mi_row, mi_col,
+ gf_temporal_ref, segment_id, bsize
#if CONFIG_AV1_TEMPORAL_DENOISING
- ,
- ctx, denoise_svc_pickmode
+ ,
+ ctx, denoise_svc_pickmode
#endif
);
@@ -3207,6 +3234,28 @@
inter_pred_params_sr.conv_params =
get_conv_params(/*do_average=*/0, AOM_PLANE_Y, xd->bd);
+ x->block_is_zero_sad = x->content_state_sb.source_sad_nonrd == kZeroSad;
+ if (cpi->oxcf.tune_cfg.content == AOM_CONTENT_SCREEN &&
+ !x->force_zeromv_skip_for_blk &&
+ x->content_state_sb.source_sad_nonrd != kZeroSad &&
+ x->source_variance == 0 && bsize < cm->seq_params->sb_size &&
+ search_state.yv12_mb[LAST_FRAME][0].width == cm->width &&
+ search_state.yv12_mb[LAST_FRAME][0].height == cm->height) {
+ set_block_source_sad(cpi, x, bsize, &search_state.yv12_mb[LAST_FRAME][0]);
+ }
+
+ int sb_me_has_been_tested = 0;
+ x->sb_me_block = x->sb_me_partition;
+ // Only use this feature (force testing of superblock motion) if coding
+ // block size is large.
+ if (x->sb_me_block) {
+ if (cm->seq_params->sb_size == BLOCK_128X128 && bsize < BLOCK_64X64)
+ x->sb_me_block = 0;
+ else if (cm->seq_params->sb_size == BLOCK_64X64 && bsize < BLOCK_32X32)
+ x->sb_me_block = 0;
+ }
+
+ x->min_dist_inter_uv = INT64_MAX;
for (int idx = 0; idx < num_inter_modes + tot_num_comp_modes; ++idx) {
// If we are at the first compound mode, and the single modes already
// perform well, then end the search.
@@ -3218,6 +3267,36 @@
int is_single_pred = 1;
PREDICTION_MODE this_mode;
+ if (idx == 0 && !x->force_zeromv_skip_for_blk) {
+ // Set color sensitivity on first tested mode only.
+ // Use y-sad already computed in find_predictors: take the sad with motion
+ // vector closest to 0; the uv-sad computed below in set_color_sensitivity
+ // is for zeromv.
+ // For screen: first check if golden reference is being used, if so,
+ // force color_sensitivity on (=1) if the color sensitivity for sb_g is 1.
+ // The check in set_color_sensitivity() will then follow and check for
+ // setting the flag if the level is still 2 or 0.
+ if (cpi->oxcf.tune_cfg.content == AOM_CONTENT_SCREEN &&
+ search_state.use_ref_frame_mask[GOLDEN_FRAME]) {
+ if (x->color_sensitivity_sb_g[COLOR_SENS_IDX(AOM_PLANE_U)] == 1)
+ x->color_sensitivity[COLOR_SENS_IDX(AOM_PLANE_U)] = 1;
+ if (x->color_sensitivity_sb_g[COLOR_SENS_IDX(AOM_PLANE_V)] == 1)
+ x->color_sensitivity[COLOR_SENS_IDX(AOM_PLANE_V)] = 1;
+ }
+ if (search_state.use_ref_frame_mask[LAST_FRAME] &&
+ x->pred_mv0_sad[LAST_FRAME] != INT_MAX) {
+ int y_sad = x->pred_mv0_sad[LAST_FRAME];
+ if (x->pred_mv1_sad[LAST_FRAME] != INT_MAX &&
+ (abs(search_state.frame_mv[NEARMV][LAST_FRAME].as_mv.col) +
+ abs(search_state.frame_mv[NEARMV][LAST_FRAME].as_mv.row)) <
+ (abs(search_state.frame_mv[NEARESTMV][LAST_FRAME].as_mv.col) +
+ abs(search_state.frame_mv[NEARESTMV][LAST_FRAME].as_mv.row)))
+ y_sad = x->pred_mv1_sad[LAST_FRAME];
+ set_color_sensitivity(cpi, x, bsize, y_sad, x->source_variance,
+ search_state.yv12_mb[LAST_FRAME]);
+ }
+ }
+
// Check the inter mode can be skipped based on mode statistics and speed
// features settings.
if (skip_inter_mode_nonrd(cpi, x, &search_state, &thresh_sad_pred,
@@ -3239,6 +3318,16 @@
mi->ref_frame[1] = ref_frame2;
set_ref_ptrs(cm, xd, ref_frame, ref_frame2);
+ // Check if the scaled reference frame should be used. This is set in the
+ // find_predictors() for each usable reference. If so, set the
+ // block_ref_scale_factors[] to no reference scaling.
+ if (search_state.use_scaled_ref_frame[ref_frame]) {
+ xd->block_ref_scale_factors[0] = &sf_no_scale;
+ }
+ if (!is_single_pred && search_state.use_scaled_ref_frame[ref_frame2]) {
+ xd->block_ref_scale_factors[1] = &sf_no_scale;
+ }
+
// Perform inter mode evaluation for non-rd
if (!handle_inter_mode_nonrd(
cpi, x, &search_state, ctx, &this_mode_pred, tmp_buffer,
@@ -3247,10 +3336,10 @@
#if CONFIG_AV1_TEMPORAL_DENOISING
&zero_last_cost_orig, denoise_svc_pickmode,
#endif
- idx, force_mv_inter_layer, is_single_pred, skip_pred_mv,
- gf_temporal_ref, use_model_yrd_large, filter_search_enabled_blk,
- bsize, this_mode, filt_select, cb_pred_filter_search,
- reuse_inter_pred)) {
+ idx, force_mv_inter_layer, is_single_pred, gf_temporal_ref,
+ use_model_yrd_large, filter_search_enabled_blk, bsize, this_mode,
+ filt_select, cb_pred_filter_search, reuse_inter_pred,
+ &sb_me_has_been_tested)) {
break;
}
}
@@ -3292,8 +3381,8 @@
if (cpi->oxcf.tune_cfg.content == AOM_CONTENT_SCREEN &&
x->content_state_sb.source_sad_nonrd != kZeroSad &&
bsize <= BLOCK_16X16) {
- unsigned int thresh_sse = cpi->rc.high_source_sad ? 15000 : 250000;
- unsigned int thresh_source_var = cpi->rc.high_source_sad ? 50 : 1000;
+ unsigned int thresh_sse = cpi->rc.high_source_sad ? 15000 : 200000;
+ unsigned int thresh_source_var = cpi->rc.high_source_sad ? 50 : 200;
unsigned int best_sse_inter_motion =
(unsigned int)(search_state.best_rdc.sse >>
(b_width_log2_lookup[bsize] +
@@ -3324,7 +3413,7 @@
try_palette &&
(is_mode_intra(best_pickmode->best_mode) || force_palette_test) &&
x->source_variance > 0 && !x->force_zeromv_skip_for_blk &&
- (cpi->rc.high_source_sad || x->source_variance > 500);
+ (cpi->rc.high_source_sad || x->source_variance > 300);
if (rt_sf->prune_palette_nonrd && bsize > BLOCK_16X16) try_palette = 0;
@@ -3360,6 +3449,14 @@
if (!is_inter_block(mi)) {
mi->interp_filters = av1_broadcast_interp_filter(SWITCHABLE_FILTERS);
+ } else {
+ // If inter mode is selected and ref_frame was one that uses the
+ // scaled reference frame, then we can't use reuse_inter_pred.
+ if (search_state.use_scaled_ref_frame[best_pickmode->best_ref_frame] ||
+ (has_second_ref(mi) &&
+ search_state
+ .use_scaled_ref_frame[best_pickmode->best_second_ref_frame]))
+ x->reuse_inter_pred = 0;
}
// Restore the predicted samples of best mode to final buffer
@@ -3425,4 +3522,9 @@
#endif // COLLECT_NONRD_PICK_MODE_STAT
*rd_cost = search_state.best_rdc;
+
+ // Reset the xd->block_ref_scale_factors[i], as they may have
+ // been set to pointer &sf_no_scale, which becomes invalid afer
+ // this function.
+ set_ref_ptrs(cm, xd, mi->ref_frame[0], mi->ref_frame[1]);
}
diff --git a/av1/encoder/palette.c b/av1/encoder/palette.c
index b1a73e4..7f79e95 100644
--- a/av1/encoder/palette.c
+++ b/av1/encoder/palette.c
@@ -564,7 +564,24 @@
}
uint8_t *const color_map = xd->plane[0].color_index_map;
- if (colors_threshold > 1 && colors_threshold <= 64) {
+ int color_thresh_palette = 64;
+ // Allow for larger color_threshold for palette search, based on color,
+ // scene_change, and block source variance.
+ // Since palette is Y based, only allow larger threshold if block
+ // color_dist is below threshold.
+ if (cpi->sf.rt_sf.use_nonrd_pick_mode &&
+ cpi->sf.rt_sf.increase_color_thresh_palette && cpi->rc.high_source_sad &&
+ x->source_variance > 50) {
+ int64_t norm_color_dist = 0;
+ if (x->color_sensitivity[0] || x->color_sensitivity[1]) {
+ norm_color_dist = x->min_dist_inter_uv >>
+ (mi_size_wide_log2[bsize] + mi_size_high_log2[bsize]);
+ if (x->color_sensitivity[0] && x->color_sensitivity[1])
+ norm_color_dist = norm_color_dist >> 1;
+ }
+ if (norm_color_dist < 8000) color_thresh_palette += 20;
+ }
+ if (colors_threshold > 1 && colors_threshold <= color_thresh_palette) {
int16_t *const data = x->palette_buffer->kmeans_data_buf;
int16_t centroids[PALETTE_MAX_SIZE];
int lower_bound, upper_bound;
diff --git a/av1/encoder/partition_search.c b/av1/encoder/partition_search.c
index 96567dd..1e3d980 100644
--- a/av1/encoder/partition_search.c
+++ b/av1/encoder/partition_search.c
@@ -40,7 +40,6 @@
#endif
#define COLLECT_MOTION_SEARCH_FEATURE_SB 0
-#define ML_PARTITION_WHOLE_TREE_DECISION 0
void av1_reset_part_sf(PARTITION_SPEED_FEATURES *part_sf) {
part_sf->partition_search_type = SEARCH_PARTITION;
@@ -73,6 +72,7 @@
part_sf->intra_cnn_based_part_prune_level = 0;
part_sf->ext_partition_eval_thresh = BLOCK_8X8;
part_sf->rect_partition_eval_thresh = BLOCK_128X128;
+ part_sf->ext_part_eval_based_on_cur_best = 0;
part_sf->prune_ext_part_using_split_info = 0;
part_sf->prune_rectangular_split_based_on_qidx = 0;
part_sf->early_term_after_none_split = 0;
@@ -1772,6 +1772,9 @@
if (pc_tree->none == NULL) {
pc_tree->none = av1_alloc_pmc(cpi, bsize, &td->shared_coeff_buf);
+ if (!pc_tree->none)
+ aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PICK_MODE_CONTEXT");
}
PICK_MODE_CONTEXT *ctx_none = pc_tree->none;
@@ -1832,6 +1835,9 @@
for (int i = 0; i < SUB_PARTITIONS_SPLIT; ++i) {
pc_tree->split[i] = av1_alloc_pc_tree_node(subsize);
+ if (!pc_tree->split[i])
+ aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PC_TREE");
pc_tree->split[i]->index = i;
}
switch (partition) {
@@ -1848,6 +1854,9 @@
for (int i = 0; i < SUB_PARTITIONS_RECT; ++i) {
pc_tree->horizontal[i] =
av1_alloc_pmc(cpi, subsize, &td->shared_coeff_buf);
+ if (!pc_tree->horizontal[i])
+ aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PICK_MODE_CONTEXT");
}
pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &last_part_rdc,
PARTITION_HORZ, subsize, pc_tree->horizontal[0],
@@ -1881,6 +1890,9 @@
for (int i = 0; i < SUB_PARTITIONS_RECT; ++i) {
pc_tree->vertical[i] =
av1_alloc_pmc(cpi, subsize, &td->shared_coeff_buf);
+ if (!pc_tree->vertical[i])
+ aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PICK_MODE_CONTEXT");
}
pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &last_part_rdc,
PARTITION_VERT, subsize, pc_tree->vertical[0], invalid_rdc);
@@ -1981,6 +1993,9 @@
if (pc_tree->split[i]->none == NULL)
pc_tree->split[i]->none =
av1_alloc_pmc(cpi, split_subsize, &td->shared_coeff_buf);
+ if (!pc_tree->split[i]->none)
+ aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PICK_MODE_CONTEXT");
pick_sb_modes(cpi, tile_data, x, mi_row + y_idx, mi_col + x_idx, &tmp_rdc,
PARTITION_SPLIT, split_subsize, pc_tree->split[i]->none,
invalid_rdc);
@@ -2264,10 +2279,13 @@
x->force_zeromv_skip_for_blk =
get_force_zeromv_skip_flag_for_blk(cpi, x, bsize);
- if (!x->force_zeromv_skip_for_blk) {
+ // Source variance may be already compute at superblock level, so no need
+ // to recompute, unless bsize < sb_size or source_variance is not yet set.
+ if (!x->force_zeromv_skip_for_blk &&
+ (x->source_variance == UINT_MAX || bsize < cm->seq_params->sb_size))
x->source_variance = av1_get_perpixel_variance_facade(
cpi, xd, &x->plane[0].src, bsize, AOM_PLANE_Y);
- }
+
// Save rdmult before it might be changed, so it can be restored later.
const int orig_rdmult = x->rdmult;
setup_block_rdmult(cpi, x, mi_row, mi_col, bsize, aq_mode, mbmi);
@@ -2396,6 +2414,9 @@
av1_set_offsets(cpi, tile_info, x, mi_row, mi_col, bsize);
if (!pc_tree->none) {
pc_tree->none = av1_alloc_pmc(cpi, bsize, &td->shared_coeff_buf);
+ if (!pc_tree->none)
+ aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PICK_MODE_CONTEXT");
} else {
av1_reset_pmc(pc_tree->none);
}
@@ -2416,6 +2437,9 @@
for (int i = 0; i < SUB_PARTITIONS_SPLIT; ++i) {
if (!pc_tree->split[i]) {
pc_tree->split[i] = av1_alloc_pc_tree_node(subsize);
+ if (!pc_tree->split[i])
+ aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PC_TREE");
}
pc_tree->split[i]->index = i;
}
@@ -2434,6 +2458,9 @@
if (!pc_tree->split[i]->none) {
pc_tree->split[i]->none =
av1_alloc_pmc(cpi, subsize, &td->shared_coeff_buf);
+ if (!pc_tree->split[i]->none)
+ aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PICK_MODE_CONTEXT");
} else {
av1_reset_pmc(pc_tree->split[i]->none);
}
@@ -2568,6 +2595,9 @@
pc_tree->partitioning = PARTITION_NONE;
if (!pc_tree->none) {
pc_tree->none = av1_alloc_pmc(cpi, bsize, &td->shared_coeff_buf);
+ if (!pc_tree->none)
+ aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PICK_MODE_CONTEXT");
} else {
av1_reset_pmc(pc_tree->none);
}
@@ -2599,6 +2629,9 @@
if (!pc_tree->split[i]->none) {
pc_tree->split[i]->none =
av1_alloc_pmc(cpi, subsize, &td->shared_coeff_buf);
+ if (!pc_tree->split[i]->none)
+ aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PICK_MODE_CONTEXT");
} else {
av1_reset_pmc(pc_tree->split[i]->none);
}
@@ -2659,6 +2692,9 @@
if (!pc_tree->split[i]->none) {
pc_tree->split[i]->none =
av1_alloc_pmc(cpi, subsize, &td->shared_coeff_buf);
+ if (!pc_tree->split[i]->none)
+ aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PICK_MODE_CONTEXT");
}
encode_b_nonrd(cpi, tile_data, td, tp, mi_row + y_idx, mi_col + x_idx, 0,
subsize, PARTITION_NONE, pc_tree->split[i]->none, NULL);
@@ -2762,6 +2798,7 @@
struct buf_2d yv12_mb[REF_FRAMES][MAX_MB_PLANE];
int force_skip_low_temp_var = 0;
int skip_pred_mv = 0;
+ bool use_scaled_ref;
for (int i = 0; i < MB_MODE_COUNT; ++i) {
for (int j = 0; j < REF_FRAMES; ++j) {
@@ -2774,7 +2811,7 @@
x->color_sensitivity[COLOR_SENS_IDX(AOM_PLANE_V)] != 2);
find_predictors(cpi, x, ref_frame, frame_mv, yv12_mb, bsize,
- force_skip_low_temp_var, skip_pred_mv);
+ force_skip_low_temp_var, skip_pred_mv, &use_scaled_ref);
int continue_merging = 1;
if (frame_mv[NEARESTMV][ref_frame].as_mv.row != b0[0]->mv[0].as_mv.row ||
@@ -2790,7 +2827,7 @@
av1_set_offsets_without_segment_id(cpi, &tile_data->tile_info, x, mi_row,
mi_col, this_mi[0]->bsize);
find_predictors(cpi, x, ref_frame, frame_mv, yv12_mb, this_mi[0]->bsize,
- force_skip_low_temp_var, skip_pred_mv);
+ force_skip_low_temp_var, skip_pred_mv, &use_scaled_ref);
} else {
struct scale_factors *sf = get_ref_scale_factors(cm, ref_frame);
const int is_scaled = av1_is_scaled(sf);
@@ -2945,6 +2982,9 @@
case PARTITION_NONE:
if (!pc_tree->none) {
pc_tree->none = av1_alloc_pmc(cpi, bsize, &td->shared_coeff_buf);
+ if (!pc_tree->none)
+ aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PICK_MODE_CONTEXT");
} else {
av1_reset_pmc(pc_tree->none);
}
@@ -2958,6 +2998,9 @@
if (!pc_tree->vertical[i]) {
pc_tree->vertical[i] =
av1_alloc_pmc(cpi, subsize, &td->shared_coeff_buf);
+ if (!pc_tree->vertical[i])
+ aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PICK_MODE_CONTEXT");
} else {
av1_reset_pmc(pc_tree->vertical[i]);
}
@@ -2978,6 +3021,9 @@
if (!pc_tree->horizontal[i]) {
pc_tree->horizontal[i] =
av1_alloc_pmc(cpi, subsize, &td->shared_coeff_buf);
+ if (!pc_tree->horizontal[i])
+ aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PICK_MODE_CONTEXT");
} else {
av1_reset_pmc(pc_tree->horizontal[i]);
}
@@ -2998,6 +3044,9 @@
for (int i = 0; i < SUB_PARTITIONS_SPLIT; ++i) {
if (!pc_tree->split[i]) {
pc_tree->split[i] = av1_alloc_pc_tree_node(subsize);
+ if (!pc_tree->split[i])
+ aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PC_TREE");
}
pc_tree->split[i]->index = i;
}
@@ -3494,6 +3543,9 @@
if (cur_ctx[i][j][0] == NULL) {
cur_ctx[i][j][0] =
av1_alloc_pmc(cpi, blk_params.subsize, &td->shared_coeff_buf);
+ if (!cur_ctx[i][j][0])
+ aom_internal_error(x->e_mbd.error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PICK_MODE_CONTEXT");
}
}
sum_rdc->rate = part_search_state->partition_cost[partition_type];
@@ -3573,7 +3625,7 @@
PartitionBlkParams blk_params = part_search_state->part_blk_params;
const int mi_row = blk_params.mi_row;
const int mi_col = blk_params.mi_col;
- const int bsize = blk_params.bsize;
+ const BLOCK_SIZE bsize = blk_params.bsize;
int64_t this_rdcost = 0;
#if CONFIG_COLLECT_PARTITION_STATS
@@ -3683,7 +3735,7 @@
PartitionBlkParams blk_params = part_search_state->part_blk_params;
const int mi_row = blk_params.mi_row;
const int mi_col = blk_params.mi_col;
- const int bsize = blk_params.bsize;
+ const BLOCK_SIZE bsize = blk_params.bsize;
if (part_search_state->terminate_partition_search) {
return;
@@ -3759,6 +3811,9 @@
// Set AB partition context.
cur_part_ctxs[ab_part_type][i] = av1_alloc_pmc(
cpi, ab_subsize[ab_part_type][i], &td->shared_coeff_buf);
+ if (!cur_part_ctxs[ab_part_type][i])
+ aom_internal_error(x->e_mbd.error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PICK_MODE_CONTEXT");
// Set mode as not ready.
cur_part_ctxs[ab_part_type][i]->rd_mode_is_ready = 0;
}
@@ -3818,8 +3873,12 @@
part_search_state->partition_cost[partition_type];
part_search_state->sum_rdc.rdcost =
RDCOST(x->rdmult, part_search_state->sum_rdc.rate, 0);
- for (PART4_TYPES i = 0; i < SUB_PARTITIONS_PART4; ++i)
+ for (PART4_TYPES i = 0; i < SUB_PARTITIONS_PART4; ++i) {
cur_part_ctx[i] = av1_alloc_pmc(cpi, subsize, &td->shared_coeff_buf);
+ if (!cur_part_ctx[i])
+ aom_internal_error(x->e_mbd.error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PICK_MODE_CONTEXT");
+ }
}
// Partition search of HORZ4 / VERT4 partition types.
@@ -3882,6 +3941,50 @@
blk_params.bsize, av1_num_planes(cm));
}
+// Do not evaluate extended partitions if NONE partition is skippable.
+static INLINE int prune_ext_part_none_skippable(
+ PICK_MODE_CONTEXT *part_none, int must_find_valid_partition,
+ int skip_non_sq_part_based_on_none, BLOCK_SIZE bsize) {
+ if ((skip_non_sq_part_based_on_none >= 1) && (part_none != NULL)) {
+ if (part_none->skippable && !must_find_valid_partition &&
+ bsize >= BLOCK_16X16) {
+ return 1;
+ }
+ }
+ return 0;
+}
+
+// Allow ab partition search
+static int allow_ab_partition_search(PartitionSearchState *part_search_state,
+ PARTITION_SPEED_FEATURES *part_sf,
+ PARTITION_TYPE curr_best_part,
+ int must_find_valid_partition,
+ int prune_ext_part_state,
+ int64_t best_rdcost) {
+ const PartitionBlkParams blk_params = part_search_state->part_blk_params;
+ const BLOCK_SIZE bsize = blk_params.bsize;
+
+ // Do not prune if there is no valid partition
+ if (best_rdcost == INT64_MAX) return 1;
+
+ // Determine bsize threshold to evaluate ab partitions
+ BLOCK_SIZE ab_bsize_thresh = part_sf->ext_partition_eval_thresh;
+ if (part_sf->ext_part_eval_based_on_cur_best && !must_find_valid_partition &&
+ !(curr_best_part == PARTITION_HORZ || curr_best_part == PARTITION_VERT))
+ ab_bsize_thresh = BLOCK_128X128;
+
+ // ab partitions are only allowed for square block sizes BLOCK_16X16 or
+ // higher, so ab_bsize_thresh must be large enough to exclude BLOCK_4X4 and
+ // BLOCK_8X8.
+ assert(ab_bsize_thresh >= BLOCK_8X8);
+
+ int ab_partition_allowed =
+ part_search_state->do_rectangular_split && bsize > ab_bsize_thresh &&
+ av1_blk_has_rows_and_cols(&blk_params) && !prune_ext_part_state;
+
+ return ab_partition_allowed;
+}
+
// Prune 4-way partitions based on the number of horz/vert wins
// in the current block and sub-blocks in PARTITION_SPLIT.
static void prune_4_partition_using_split_info(
@@ -3915,9 +4018,28 @@
static void prune_4_way_partition_search(
AV1_COMP *const cpi, MACROBLOCK *x, PC_TREE *pc_tree,
PartitionSearchState *part_search_state, RD_STATS *best_rdc,
- int pb_source_variance, int ext_partition_allowed,
+ int pb_source_variance, int prune_ext_part_state,
int part4_search_allowed[NUM_PART4_TYPES]) {
- PartitionBlkParams blk_params = part_search_state->part_blk_params;
+ const PartitionBlkParams blk_params = part_search_state->part_blk_params;
+ const BLOCK_SIZE bsize = blk_params.bsize;
+
+ // Do not prune if there is no valid partition
+ if (best_rdc->rdcost == INT64_MAX) return;
+
+ // Determine bsize threshold to evaluate 4-way partitions
+ BLOCK_SIZE part4_bsize_thresh = cpi->sf.part_sf.ext_partition_eval_thresh;
+ if (cpi->sf.part_sf.ext_part_eval_based_on_cur_best &&
+ !x->must_find_valid_partition && pc_tree->partitioning == PARTITION_NONE)
+ part4_bsize_thresh = BLOCK_128X128;
+
+ // 4-way partitions are only allowed for BLOCK_16X16, BLOCK_32X32, and
+ // BLOCK_64X64, so part4_bsize_thresh must be large enough to exclude
+ // BLOCK_4X4 and BLOCK_8X8.
+ assert(part4_bsize_thresh >= BLOCK_8X8);
+
+ bool partition4_allowed =
+ part_search_state->do_rectangular_split && bsize > part4_bsize_thresh &&
+ av1_blk_has_rows_and_cols(&blk_params) && !prune_ext_part_state;
// Disable 4-way partition search flags for width less than a multiple of the
// minimum partition width.
@@ -3928,17 +4050,15 @@
return;
}
- const int bsize = blk_params.bsize;
PARTITION_TYPE cur_part[NUM_PART4_TYPES] = { PARTITION_HORZ_4,
PARTITION_VERT_4 };
const PartitionCfg *const part_cfg = &cpi->oxcf.part_cfg;
// partition4_allowed is 1 if we can use a PARTITION_HORZ_4 or
// PARTITION_VERT_4 for this block. This is almost the same as
- // ext_partition_allowed, except that we don't allow 128x32 or 32x128
+ // partition4_allowed, except that we don't allow 128x32 or 32x128
// blocks, so we require that bsize is not BLOCK_128X128.
- const int partition4_allowed = part_cfg->enable_1to4_partitions &&
- ext_partition_allowed &&
- bsize != BLOCK_128X128;
+ partition4_allowed &=
+ part_cfg->enable_1to4_partitions && bsize != BLOCK_128X128;
for (PART4_TYPES i = HORZ4; i < NUM_PART4_TYPES; i++) {
part4_search_allowed[i] =
@@ -3988,6 +4108,9 @@
// Set PARTITION_NONE context.
if (pc_tree->none == NULL)
pc_tree->none = av1_alloc_pmc(cpi, blk_params.bsize, &td->shared_coeff_buf);
+ if (!pc_tree->none)
+ aom_internal_error(x->e_mbd.error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PICK_MODE_CONTEXT");
// Set PARTITION_NONE type cost.
if (part_search_state->partition_none_allowed) {
@@ -4215,7 +4338,7 @@
const CommonModeInfoParams *const mi_params = &cm->mi_params;
const int mi_row = blk_params.mi_row;
const int mi_col = blk_params.mi_col;
- const int bsize = blk_params.bsize;
+ const BLOCK_SIZE bsize = blk_params.bsize;
assert(bsize < BLOCK_SIZES_ALL);
RD_STATS sum_rdc = part_search_state->sum_rdc;
const BLOCK_SIZE subsize = get_partition_subsize(bsize, PARTITION_SPLIT);
@@ -4228,6 +4351,9 @@
for (int i = 0; i < SUB_PARTITIONS_SPLIT; ++i) {
if (pc_tree->split[i] == NULL)
pc_tree->split[i] = av1_alloc_pc_tree_node(subsize);
+ if (!pc_tree->split[i])
+ aom_internal_error(x->e_mbd.error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PC_TREE");
pc_tree->split[i]->index = i;
}
@@ -4400,6 +4526,7 @@
fclose(pfile);
}
+#if CONFIG_PARTITION_SEARCH_ORDER
static void verify_write_partition_tree(const AV1_COMP *const cpi,
const PC_TREE *const pc_tree,
const BLOCK_SIZE bsize,
@@ -4463,6 +4590,7 @@
}
static int read_partition_tree(AV1_COMP *const cpi, PC_TREE *const pc_tree,
+ struct aom_internal_error_info *error_info,
const int config_id) {
const AV1_COMMON *const cm = &cpi->common;
const char *path = cpi->oxcf.partition_info_path;
@@ -4502,6 +4630,9 @@
for (int i = 0; i < 4; ++i) {
if (node != NULL) { // Suppress warning
node->split[i] = av1_alloc_pc_tree_node(subsize);
+ if (!node->split[i])
+ aom_internal_error(error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PC_TREE");
node->split[i]->index = i;
tree_node_queue[last_idx] = node->split[i];
++last_idx;
@@ -4665,7 +4796,8 @@
static void build_pc_tree_from_part_decision(
const aom_partition_decision_t *partition_decision,
- const BLOCK_SIZE this_bsize, PC_TREE *pc_tree) {
+ const BLOCK_SIZE this_bsize, PC_TREE *pc_tree,
+ struct aom_internal_error_info *error_info) {
BLOCK_SIZE bsize = this_bsize;
int num_nodes = partition_decision->num_nodes;
PC_TREE *tree_node_queue[NUM_NODES] = { NULL };
@@ -4686,6 +4818,9 @@
for (int i = 0; i < 4; ++i) {
if (node != NULL) { // Suppress warning
node->split[i] = av1_alloc_pc_tree_node(subsize);
+ if (!node->split[i])
+ aom_internal_error(error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PC_TREE");
node->split[i]->index = i;
tree_node_queue[last_idx] = node->split[i];
++last_idx;
@@ -4707,6 +4842,7 @@
AV1_COMMON *const cm = &cpi->common;
MACROBLOCK *const x = &td->mb;
ExtPartController *const ext_part_controller = &cpi->ext_part_controller;
+ struct aom_internal_error_info *error_info = x->e_mbd.error_info;
aom_partition_features_t features;
prepare_sb_features_before_search(cpi, td, tile_data, mi_row, mi_col, bsize,
&features);
@@ -4716,7 +4852,6 @@
features.frame_height = cpi->frame_info.frame_height;
features.block_size = bsize;
av1_ext_part_send_features(ext_part_controller, &features);
- PC_TREE *pc_tree;
// rd mode search (dry run) for a valid partition decision from the ml model.
aom_partition_decision_t partition_decision;
@@ -4728,26 +4863,32 @@
// First, let's take the easy approach.
// We require that the ml model has to provide partition decisions for the
// whole superblock.
- pc_tree = av1_alloc_pc_tree_node(bsize);
- build_pc_tree_from_part_decision(&partition_decision, bsize, pc_tree);
+ td->pc_root = av1_alloc_pc_tree_node(bsize);
+ if (!td->pc_root)
+ aom_internal_error(error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PC_TREE");
+ build_pc_tree_from_part_decision(&partition_decision, bsize, td->pc_root,
+ error_info);
const RD_STATS this_rdcost = rd_search_for_fixed_partition(
- cpi, td, tile_data, tp, sms_root, mi_row, mi_col, bsize, pc_tree);
+ cpi, td, tile_data, tp, sms_root, mi_row, mi_col, bsize, td->pc_root);
aom_partition_stats_t stats;
update_partition_stats(&this_rdcost, &stats);
av1_ext_part_send_partition_stats(ext_part_controller, &stats);
if (!partition_decision.is_final_decision) {
- av1_free_pc_tree_recursive(pc_tree, av1_num_planes(cm), 0, 0,
+ av1_free_pc_tree_recursive(td->pc_root, av1_num_planes(cm), 0, 0,
cpi->sf.part_sf.partition_search_type);
+ td->pc_root = NULL;
}
} while (!partition_decision.is_final_decision);
// Encode with the selected mode and partition.
set_cb_offsets(x->cb_offset, 0, 0);
encode_sb(cpi, td, tile_data, tp, mi_row, mi_col, OUTPUT_ENABLED, bsize,
- pc_tree, NULL);
- av1_free_pc_tree_recursive(pc_tree, av1_num_planes(cm), 0, 0,
+ td->pc_root, NULL);
+ av1_free_pc_tree_recursive(td->pc_root, av1_num_planes(cm), 0, 0,
cpi->sf.part_sf.partition_search_type);
+ td->pc_root = NULL;
return true;
}
@@ -4978,6 +5119,9 @@
av1_init_rd_stats(&split_rdc[i]);
if (pc_tree->split[i] == NULL)
pc_tree->split[i] = av1_alloc_pc_tree_node(subsize);
+ if (!pc_tree->split[i])
+ aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PC_TREE");
pc_tree->split[i]->index = i;
}
const int orig_rdmult_tmp = x->rdmult;
@@ -5050,12 +5194,14 @@
features.frame_height = cpi->frame_info.frame_height;
features.block_size = bsize;
av1_ext_part_send_features(ext_part_controller, &features);
- PC_TREE *pc_tree;
- pc_tree = av1_alloc_pc_tree_node(bsize);
+ td->pc_root = av1_alloc_pc_tree_node(bsize);
+ if (!td->pc_root)
+ aom_internal_error(x->e_mbd.error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PC_TREE");
RD_STATS rdcost;
const bool valid_partition =
- recursive_partition(cpi, td, tile_data, tp, sms_root, pc_tree, mi_row,
+ recursive_partition(cpi, td, tile_data, tp, sms_root, td->pc_root, mi_row,
mi_col, bsize, &rdcost);
if (!valid_partition) {
return false;
@@ -5064,9 +5210,10 @@
// Encode with the selected mode and partition.
set_cb_offsets(x->cb_offset, 0, 0);
encode_sb(cpi, td, tile_data, tp, mi_row, mi_col, OUTPUT_ENABLED, bsize,
- pc_tree, NULL);
- av1_free_pc_tree_recursive(pc_tree, av1_num_planes(cm), 0, 0,
+ td->pc_root, NULL);
+ av1_free_pc_tree_recursive(td->pc_root, av1_num_planes(cm), 0, 0,
cpi->sf.part_sf.partition_search_type);
+ td->pc_root = NULL;
return true;
}
@@ -5100,54 +5247,65 @@
}
MACROBLOCK *const x = &td->mb;
+ MACROBLOCKD *const xd = &x->e_mbd;
int best_idx = 0;
int64_t min_rdcost = INT64_MAX;
int num_configs;
- RD_STATS *rdcost = NULL;
int i = 0;
do {
- PC_TREE *const pc_tree = av1_alloc_pc_tree_node(bsize);
- num_configs = read_partition_tree(cpi, pc_tree, i);
- if (i == 0) {
- CHECK_MEM_ERROR(cm, rdcost, aom_calloc(num_configs, sizeof(*rdcost)));
- }
+ td->pc_root = av1_alloc_pc_tree_node(bsize);
+ if (!td->pc_root)
+ aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PC_TREE");
+ num_configs = read_partition_tree(cpi, td->pc_root, xd->error_info, i);
if (num_configs <= 0) {
- av1_free_pc_tree_recursive(pc_tree, av1_num_planes(cm), 0, 0,
+ av1_free_pc_tree_recursive(td->pc_root, av1_num_planes(cm), 0, 0,
cpi->sf.part_sf.partition_search_type);
- if (rdcost != NULL) aom_free(rdcost);
- aom_internal_error(cm->error, AOM_CODEC_ERROR, "Invalid configs.");
+ td->pc_root = NULL;
+ aom_internal_error(xd->error_info, AOM_CODEC_ERROR, "Invalid configs.");
}
- verify_write_partition_tree(cpi, pc_tree, bsize, i, mi_row, mi_col);
+ verify_write_partition_tree(cpi, td->pc_root, bsize, i, mi_row, mi_col);
+ if (i == 0) {
+ AOM_CHECK_MEM_ERROR(xd->error_info, x->rdcost,
+ aom_calloc(num_configs, sizeof(*x->rdcost)));
+ }
// Encode the block with the given partition tree. Get rdcost and encoding
// time.
- rdcost[i] = rd_search_for_fixed_partition(cpi, td, tile_data, tp, sms_root,
- mi_row, mi_col, bsize, pc_tree);
+ x->rdcost[i] = rd_search_for_fixed_partition(
+ cpi, td, tile_data, tp, sms_root, mi_row, mi_col, bsize, td->pc_root);
- if (rdcost[i].rdcost < min_rdcost) {
- min_rdcost = rdcost[i].rdcost;
+ if (x->rdcost[i].rdcost < min_rdcost) {
+ min_rdcost = x->rdcost[i].rdcost;
best_idx = i;
- *best_rd_cost = rdcost[i];
+ *best_rd_cost = x->rdcost[i];
}
- av1_free_pc_tree_recursive(pc_tree, av1_num_planes(cm), 0, 0,
+ av1_free_pc_tree_recursive(td->pc_root, av1_num_planes(cm), 0, 0,
cpi->sf.part_sf.partition_search_type);
+ td->pc_root = NULL;
++i;
} while (i < num_configs);
+ aom_free(x->rdcost);
+ x->rdcost = NULL;
// Encode with the partition configuration with the smallest rdcost.
- PC_TREE *const pc_tree = av1_alloc_pc_tree_node(bsize);
- read_partition_tree(cpi, pc_tree, best_idx);
+ td->pc_root = av1_alloc_pc_tree_node(bsize);
+ if (!td->pc_root)
+ aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PC_TREE");
+ read_partition_tree(cpi, td->pc_root, xd->error_info, best_idx);
rd_search_for_fixed_partition(cpi, td, tile_data, tp, sms_root, mi_row,
- mi_col, bsize, pc_tree);
+ mi_col, bsize, td->pc_root);
set_cb_offsets(x->cb_offset, 0, 0);
encode_sb(cpi, td, tile_data, tp, mi_row, mi_col, OUTPUT_ENABLED, bsize,
- pc_tree, NULL);
- av1_free_pc_tree_recursive(pc_tree, av1_num_planes(cm), 0, 0,
+ td->pc_root, NULL);
+ av1_free_pc_tree_recursive(td->pc_root, av1_num_planes(cm), 0, 0,
cpi->sf.part_sf.partition_search_type);
- aom_free(rdcost);
+ td->pc_root = NULL;
++cpi->sb_counter;
return true;
}
+#endif // CONFIG_PARTITION_SEARCH_ORDER
static AOM_INLINE bool should_do_dry_run_encode_for_current_block(
BLOCK_SIZE sb_size, BLOCK_SIZE max_partition_size, int curr_block_index,
@@ -5493,25 +5651,21 @@
assert(IMPLIES(!cpi->oxcf.part_cfg.enable_rect_partitions,
!part_search_state.do_rectangular_split));
- int ext_partition_allowed =
- part_search_state.do_rectangular_split &&
- bsize > cpi->sf.part_sf.ext_partition_eval_thresh &&
- av1_blk_has_rows_and_cols(&blk_params);
+ const int prune_ext_part_state = prune_ext_part_none_skippable(
+ pc_tree->none, x->must_find_valid_partition,
+ cpi->sf.part_sf.skip_non_sq_part_based_on_none, bsize);
- // Do not evaluate extended partitions if NONE partition is skippable.
- if ((cpi->sf.part_sf.skip_non_sq_part_based_on_none >= 1) &&
- (pc_tree->none != NULL)) {
- if (pc_tree->none->skippable && !x->must_find_valid_partition &&
- bsize >= BLOCK_16X16)
- ext_partition_allowed = 0;
- }
+ const int ab_partition_allowed = allow_ab_partition_search(
+ &part_search_state, &cpi->sf.part_sf, pc_tree->partitioning,
+ x->must_find_valid_partition, prune_ext_part_state, best_rdc.rdcost);
+
#if CONFIG_COLLECT_COMPONENT_TIMING
start_timing(cpi, ab_partitions_search_time);
#endif
// AB partitions search stage.
ab_partitions_search(cpi, td, tile_data, tp, x, &x_ctx, pc_tree,
&part_search_state, &best_rdc, rect_part_win_info,
- pb_source_variance, ext_partition_allowed, HORZ_A,
+ pb_source_variance, ab_partition_allowed, HORZ_A,
VERT_B);
#if CONFIG_COLLECT_COMPONENT_TIMING
end_timing(cpi, ab_partitions_search_time);
@@ -5521,7 +5675,7 @@
int part4_search_allowed[NUM_PART4_TYPES] = { 1, 1 };
// Prune 4-way partition search.
prune_4_way_partition_search(cpi, x, pc_tree, &part_search_state, &best_rdc,
- pb_source_variance, ext_partition_allowed,
+ pb_source_variance, prune_ext_part_state,
part4_search_allowed);
#if CONFIG_COLLECT_COMPONENT_TIMING
@@ -5618,9 +5772,12 @@
set_cb_offsets(x->cb_offset, 0, 0);
encode_sb(cpi, td, tile_data, tp, mi_row, mi_col, run_type, bsize,
pc_tree, NULL);
+ assert(pc_tree == td->pc_root);
// Dealloc the whole PC_TREE after a superblock is done.
av1_free_pc_tree_recursive(pc_tree, num_planes, 0, 0,
cpi->sf.part_sf.partition_search_type);
+ pc_tree = NULL;
+ td->pc_root = NULL;
pc_tree_dealloc = 1;
} else if (should_do_dry_run_encode_for_current_block(
cm->seq_params->sb_size, x->sb_enc.max_partition_size,
@@ -5941,6 +6098,9 @@
// PARTITION_NONE
if (partition_none_allowed) {
pc_tree->none = av1_alloc_pmc(cpi, bsize, &td->shared_coeff_buf);
+ if (!pc_tree->none)
+ aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PICK_MODE_CONTEXT");
PICK_MODE_CONTEXT *ctx = pc_tree->none;
// Flip for RDO based pick mode
@@ -5974,6 +6134,9 @@
for (int i = 0; i < SUB_PARTITIONS_SPLIT; ++i) {
pc_tree->split[i] = av1_alloc_pc_tree_node(subsize);
+ if (!pc_tree->split[i])
+ aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate PC_TREE");
pc_tree->split[i]->index = i;
}
diff --git a/av1/encoder/partition_search.h b/av1/encoder/partition_search.h
index 2577e79..1b5d71b 100644
--- a/av1/encoder/partition_search.h
+++ b/av1/encoder/partition_search.h
@@ -42,11 +42,14 @@
void av1_reset_part_sf(PARTITION_SPEED_FEATURES *part_sf);
void av1_reset_sf_for_ext_part(AV1_COMP *const cpi);
+#if CONFIG_PARTITION_SEARCH_ORDER
bool av1_rd_partition_search(AV1_COMP *const cpi, ThreadData *td,
TileDataEnc *tile_data, TokenExtra **tp,
SIMPLE_MOTION_DATA_TREE *sms_root, int mi_row,
int mi_col, BLOCK_SIZE bsize,
RD_STATS *best_rd_cost);
+#endif
+
bool av1_rd_pick_partition(AV1_COMP *const cpi, ThreadData *td,
TileDataEnc *tile_data, TokenExtra **tp, int mi_row,
int mi_col, BLOCK_SIZE bsize, RD_STATS *rd_cost,
diff --git a/av1/encoder/partition_strategy.c b/av1/encoder/partition_strategy.c
index 080587b..ce06313 100644
--- a/av1/encoder/partition_strategy.c
+++ b/av1/encoder/partition_strategy.c
@@ -108,7 +108,7 @@
const bool is_test_mode,
const float *features,
const int feature_size, const int id,
- const int bsize, const int mi_row,
+ const BLOCK_SIZE bsize, const int mi_row,
const int mi_col) {
if (!WRITE_FEATURE_TO_FILE && !is_test_mode) return;
@@ -118,7 +118,8 @@
FILE *pfile = fopen(filename, "a");
if (pfile == NULL) return;
if (!is_test_mode) {
- fprintf(pfile, "%d,%d,%d,%d,%d\n", id, bsize, mi_row, mi_col, feature_size);
+ fprintf(pfile, "%d,%d,%d,%d,%d\n", id, (int)bsize, mi_row, mi_col,
+ feature_size);
}
for (int i = 0; i < feature_size; ++i) {
fprintf(pfile, "%.6f", features[i]);
@@ -203,7 +204,7 @@
if (!av1_cnn_predict_img_multi_out_highbd(image, width, height, stride,
cnn_config, &thread_data,
bit_depth, &output)) {
- aom_internal_error(cm->error, AOM_CODEC_MEM_ERROR,
+ aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR,
"Error allocating CNN data");
return;
}
@@ -212,7 +213,7 @@
if (!av1_cnn_predict_img_multi_out(image, width, height, stride,
cnn_config, &thread_data, &output)) {
- aom_internal_error(cm->error, AOM_CODEC_MEM_ERROR,
+ aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR,
"Error allocating CNN data");
return;
}
@@ -471,8 +472,6 @@
// Otherwise do loop through the reference frames and find the one with the
// minimum SSE
- const MACROBLOCKD *xd = &x->e_mbd;
-
const int num_planes = 1;
*best_sse = INT_MAX;
@@ -483,12 +482,9 @@
if (cpi->ref_frame_flags & av1_ref_frame_flag_list[ref]) {
const FULLPEL_MV *start_mvs = sms_tree->start_mvs;
unsigned int curr_sse = 0, curr_var = 0;
- int_mv best_mv =
- av1_simple_motion_search(cpi, x, mi_row, mi_col, bsize, ref,
- start_mvs[ref], num_planes, use_subpixel);
- curr_var = cpi->ppi->fn_ptr[bsize].vf(
- x->plane[0].src.buf, x->plane[0].src.stride, xd->plane[0].dst.buf,
- xd->plane[0].dst.stride, &curr_sse);
+ const int_mv best_mv = av1_simple_motion_search_sse_var(
+ cpi, x, mi_row, mi_col, bsize, ref, start_mvs[ref], num_planes,
+ use_subpixel, &curr_sse, &curr_var);
if (curr_sse < *best_sse) {
*best_sse = curr_sse;
*best_var = curr_var;
@@ -840,8 +836,11 @@
unsigned int sse = 0;
unsigned int var = 0;
const FULLPEL_MV start_mv = kZeroFullMv;
- int_mv best_mv = av1_simple_motion_sse_var(
- cpi, x, this_mi_row, this_mi_col, mb_size, start_mv, 0, &sse, &var);
+ const MV_REFERENCE_FRAME ref =
+ cpi->rc.is_src_frame_alt_ref ? ALTREF_FRAME : LAST_FRAME;
+ const int_mv best_mv = av1_simple_motion_search_sse_var(
+ cpi, x, this_mi_row, this_mi_col, mb_size, ref, start_mv, 1, 0, &sse,
+ &var);
const float mv_row = (float)(best_mv.as_mv.row / 8);
const float mv_col = (float)(best_mv.as_mv.col / 8);
@@ -1214,7 +1213,7 @@
const PartitionBlkParams blk_params = part_state->part_blk_params;
const int mi_row = blk_params.mi_row;
const int mi_col = blk_params.mi_col;
- const int bsize = blk_params.bsize;
+ const BLOCK_SIZE bsize = blk_params.bsize;
if (bsize < BLOCK_8X8 || best_rd >= 1000000000) return;
const NN_CONFIG *nn_config = NULL;
@@ -1317,7 +1316,7 @@
const PartitionBlkParams blk_params = part_state->part_blk_params;
const int mi_row = blk_params.mi_row;
const int mi_col = blk_params.mi_col;
- const int bsize = blk_params.bsize;
+ const BLOCK_SIZE bsize = blk_params.bsize;
int64_t(*rect_part_rd)[SUB_PARTITIONS_RECT] = part_state->rect_part_rd;
int64_t *split_rd = part_state->split_rd;
@@ -1331,6 +1330,7 @@
int64_t *horz_rd = rect_part_rd[HORZ4];
int64_t *vert_rd = rect_part_rd[VERT4];
const NN_CONFIG *nn_config = NULL;
+ // 4-way partitions are only allowed for these three square block sizes.
switch (bsize) {
case BLOCK_16X16: nn_config = &av1_4_partition_nnconfig_16; break;
case BLOCK_32X32: nn_config = &av1_4_partition_nnconfig_32; break;
@@ -1377,6 +1377,10 @@
{
BLOCK_SIZE horz_4_bs = get_partition_subsize(bsize, PARTITION_HORZ_4);
BLOCK_SIZE vert_4_bs = get_partition_subsize(bsize, PARTITION_VERT_4);
+
+ assert(horz_4_bs != BLOCK_INVALID);
+ assert(vert_4_bs != BLOCK_INVALID);
+
av1_setup_src_planes(x, cpi->source, mi_row, mi_col,
av1_num_planes(&cpi->common), bsize);
const int src_stride = x->plane[0].src.stride;
@@ -1650,17 +1654,15 @@
if (cpi->sf.part_sf.prune_sub_8x8_partition_level && (bsize == BLOCK_8X8)) {
const MACROBLOCKD *const xd = &x->e_mbd;
- int prune_sub_8x8 = 1;
- if (cpi->sf.part_sf.prune_sub_8x8_partition_level == 1) {
- int num_neighbors_lt_8x8 = 0;
- if (xd->left_available)
- num_neighbors_lt_8x8 += (xd->left_mbmi->bsize <= BLOCK_8X8);
- if (xd->up_available)
- num_neighbors_lt_8x8 += (xd->above_mbmi->bsize <= BLOCK_8X8);
- // Evaluate only if both left and above blocks are of size <= BLOCK_8X8.
- if (num_neighbors_lt_8x8 == 2) {
- prune_sub_8x8 = 0;
- }
+ int prune_sub_8x8;
+ if (cpi->sf.part_sf.prune_sub_8x8_partition_level == 2) {
+ prune_sub_8x8 = 1;
+ } else {
+ assert(cpi->sf.part_sf.prune_sub_8x8_partition_level == 1);
+ // Prune if both neighbors are available and either is > BLOCK_8X8
+ prune_sub_8x8 = xd->left_available && xd->up_available &&
+ (xd->left_mbmi->bsize > BLOCK_8X8 ||
+ xd->above_mbmi->bsize > BLOCK_8X8);
}
if (prune_sub_8x8) {
av1_disable_all_splits(part_state);
@@ -1962,12 +1964,19 @@
features->after_part_ab.f[feature_index++] = rd_ratio;
}
+ // 4-way partitions are only allowed for these three square block sizes.
+ assert(bsize == BLOCK_16X16 || bsize == BLOCK_32X32 || bsize == BLOCK_64X64);
+
// Get variance of the 1:4 and 4:1 sub-blocks.
unsigned int horz_4_source_var[SUB_PARTITIONS_PART4] = { 0 };
unsigned int vert_4_source_var[SUB_PARTITIONS_PART4] = { 0 };
{
BLOCK_SIZE horz_4_bs = get_partition_subsize(bsize, PARTITION_HORZ_4);
BLOCK_SIZE vert_4_bs = get_partition_subsize(bsize, PARTITION_VERT_4);
+
+ assert(horz_4_bs != BLOCK_INVALID);
+ assert(vert_4_bs != BLOCK_INVALID);
+
av1_setup_src_planes(x, cpi->source, mi_row, mi_col,
av1_num_planes(&cpi->common), bsize);
const int src_stride = x->plane[0].src.stride;
diff --git a/av1/encoder/pass2_strategy.c b/av1/encoder/pass2_strategy.c
index 46bc6b0..3641c8b 100644
--- a/av1/encoder/pass2_strategy.c
+++ b/av1/encoder/pass2_strategy.c
@@ -181,7 +181,7 @@
// Based on recent history adjust expectations of bits per macroblock.
double damp_fac = AOMMAX(5.0, rate_err_tol / 10.0);
double rate_err_factor = 1.0;
- const double adj_limit = AOMMAX(0.20, (double)(100 - rate_err_tol) / 200.0);
+ const double adj_limit = AOMMAX(0.2, (double)(100 - rate_err_tol) / 200.0);
const double min_fac = 1.0 - adj_limit;
const double max_fac = 1.0 + adj_limit;
@@ -255,7 +255,6 @@
rate_err_factor = 1.0 - ((double)(bits_off_target) /
AOMMAX(total_actual_bits, bits_left));
}
- rate_err_factor = AOMMAX(min_fac, AOMMIN(max_fac, rate_err_factor));
// Adjustment is damped if this is 1 pass with look ahead processing
// (as there are only ever a few frames of data) and for all but the first
@@ -263,6 +262,7 @@
if ((twopass->bpm_factor != 1.0) || cpi->ppi->lap_enabled) {
rate_err_factor = 1.0 + ((rate_err_factor - 1.0) / damp_fac);
}
+ rate_err_factor = AOMMAX(min_fac, AOMMIN(max_fac, rate_err_factor));
}
// Is the rate control trending in the right direction. Only make
@@ -270,7 +270,12 @@
if ((rate_err_factor < 1.0 && err_estimate >= 0) ||
(rate_err_factor > 1.0 && err_estimate <= 0)) {
twopass->bpm_factor *= rate_err_factor;
- twopass->bpm_factor = AOMMAX(min_fac, AOMMIN(max_fac, twopass->bpm_factor));
+ if (rate_err_tol >= 100) {
+ twopass->bpm_factor =
+ AOMMAX(min_fac, AOMMIN(max_fac, twopass->bpm_factor));
+ } else {
+ twopass->bpm_factor = AOMMAX(0.1, AOMMIN(10.0, twopass->bpm_factor));
+ }
}
}
@@ -1748,22 +1753,42 @@
cleanup_regions(regions, num_regions);
}
-void av1_identify_regions(const FIRSTPASS_STATS *const stats_start,
- int total_frames, int offset, REGIONS *regions,
- int *total_regions) {
+static void free_firstpass_stats_buffers(REGIONS *temp_regions,
+ double *filt_intra_err,
+ double *filt_coded_err,
+ double *grad_coded) {
+ aom_free(temp_regions);
+ aom_free(filt_intra_err);
+ aom_free(filt_coded_err);
+ aom_free(grad_coded);
+}
+
+// Identify stable and unstable regions from first pass stats.
+// stats_start points to the first frame to analyze.
+// |offset| is the offset from the current frame to the frame stats_start is
+// pointing to.
+// Returns 0 on success, -1 on memory allocation failure.
+static int identify_regions(const FIRSTPASS_STATS *const stats_start,
+ int total_frames, int offset, REGIONS *regions,
+ int *total_regions) {
int k;
- if (total_frames <= 1) return;
+ if (total_frames <= 1) return 0;
// store the initial decisions
REGIONS *temp_regions =
(REGIONS *)aom_malloc(total_frames * sizeof(temp_regions[0]));
- av1_zero_array(temp_regions, total_frames);
// buffers for filtered stats
double *filt_intra_err =
(double *)aom_calloc(total_frames, sizeof(*filt_intra_err));
double *filt_coded_err =
(double *)aom_calloc(total_frames, sizeof(*filt_coded_err));
double *grad_coded = (double *)aom_calloc(total_frames, sizeof(*grad_coded));
+ if (!(temp_regions && filt_intra_err && filt_coded_err && grad_coded)) {
+ free_firstpass_stats_buffers(temp_regions, filt_intra_err, filt_coded_err,
+ grad_coded);
+ return -1;
+ }
+ av1_zero_array(temp_regions, total_frames);
int cur_region = 0, this_start = 0, this_last;
@@ -1853,10 +1878,9 @@
regions[k].last += offset;
}
- aom_free(temp_regions);
- aom_free(filt_coded_err);
- aom_free(filt_intra_err);
- aom_free(grad_coded);
+ free_firstpass_stats_buffers(temp_regions, filt_intra_err, filt_coded_err,
+ grad_coded);
+ return 0;
}
static int find_regions_index(const REGIONS *regions, int num_regions,
@@ -3794,6 +3818,7 @@
(rc->frames_since_key == 0)));
p_rc->frames_till_regions_update = rest_frames;
+ int ret;
if (cpi->ppi->lap_enabled) {
av1_mark_flashes(twopass->stats_buf_ctx->stats_in_start,
twopass->stats_buf_ctx->stats_in_end);
@@ -3801,14 +3826,18 @@
twopass->stats_buf_ctx->stats_in_end);
av1_estimate_coeff(twopass->stats_buf_ctx->stats_in_start,
twopass->stats_buf_ctx->stats_in_end);
- av1_identify_regions(cpi->twopass_frame.stats_in, rest_frames,
- (rc->frames_since_key == 0), p_rc->regions,
- &p_rc->num_regions);
+ ret = identify_regions(cpi->twopass_frame.stats_in, rest_frames,
+ (rc->frames_since_key == 0), p_rc->regions,
+ &p_rc->num_regions);
} else {
- av1_identify_regions(
+ ret = identify_regions(
cpi->twopass_frame.stats_in - (rc->frames_since_key == 0),
rest_frames, 0, p_rc->regions, &p_rc->num_regions);
}
+ if (ret == -1) {
+ aom_internal_error(cpi->common.error, AOM_CODEC_MEM_ERROR,
+ "Error allocating buffers in identify_regions");
+ }
}
int cur_region_idx =
diff --git a/av1/encoder/pass2_strategy.h b/av1/encoder/pass2_strategy.h
index e34454e..ff1591c 100644
--- a/av1/encoder/pass2_strategy.h
+++ b/av1/encoder/pass2_strategy.h
@@ -134,14 +134,6 @@
int *num_fpstats_used, int *num_fpstats_required,
int project_gfu_boost);
-// Identify stable and unstable regions from first pass stats.
-// stats_start points to the first frame to analyze.
-// |offset| is the offset from the current frame to the frame stats_start is
-// pointing to.
-void av1_identify_regions(const FIRSTPASS_STATS *const stats_start,
- int total_frames, int offset, REGIONS *regions,
- int *total_regions);
-
void av1_mark_flashes(FIRSTPASS_STATS *first_stats,
FIRSTPASS_STATS *last_stats);
void av1_estimate_noise(FIRSTPASS_STATS *first_stats,
diff --git a/av1/encoder/pickcdef.c b/av1/encoder/pickcdef.c
index 293dafa..232a2f9 100644
--- a/av1/encoder/pickcdef.c
+++ b/av1/encoder/pickcdef.c
@@ -515,8 +515,12 @@
// fbc: Column index in units of 64x64 block
// Returns:
// Nothing will be returned. Contents of cdef_search_ctx will be modified.
-void av1_cdef_mse_calc_block(CdefSearchCtx *cdef_search_ctx, int fbr, int fbc,
- int sb_count) {
+void av1_cdef_mse_calc_block(CdefSearchCtx *cdef_search_ctx,
+ struct aom_internal_error_info *error_info,
+ int fbr, int fbc, int sb_count) {
+ // TODO(aomedia:3276): Pass error_info to the low-level functions as required
+ // in future to handle error propagation.
+ (void)error_info;
const CommonModeInfoParams *const mi_params = cdef_search_ctx->mi_params;
const YV12_BUFFER_CONFIG *ref = cdef_search_ctx->ref;
const int coeff_shift = cdef_search_ctx->coeff_shift;
@@ -614,14 +618,15 @@
// CDEF search context.
// Returns:
// Nothing will be returned. Contents of cdef_search_ctx will be modified.
-static void cdef_mse_calc_frame(CdefSearchCtx *cdef_search_ctx) {
+static void cdef_mse_calc_frame(CdefSearchCtx *cdef_search_ctx,
+ struct aom_internal_error_info *error_info) {
// Loop over each sb.
for (int fbr = 0; fbr < cdef_search_ctx->nvfb; ++fbr) {
for (int fbc = 0; fbc < cdef_search_ctx->nhfb; ++fbc) {
// Checks if cdef processing can be skipped for particular sb.
if (cdef_sb_skip(cdef_search_ctx->mi_params, fbr, fbc)) continue;
// Calculate mse for each sb and store the relevant sb index.
- av1_cdef_mse_calc_block(cdef_search_ctx, fbr, fbc,
+ av1_cdef_mse_calc_block(cdef_search_ctx, error_info, fbr, fbc,
cdef_search_ctx->sb_count);
cdef_search_ctx->sb_count++;
}
@@ -634,24 +639,17 @@
// related to CDEF search context.
// Returns:
// Nothing will be returned. Contents of cdef_search_ctx will be modified.
-static AOM_INLINE bool cdef_alloc_data(CdefSearchCtx *cdef_search_ctx) {
+static void cdef_alloc_data(AV1_COMMON *cm, CdefSearchCtx *cdef_search_ctx) {
const int nvfb = cdef_search_ctx->nvfb;
const int nhfb = cdef_search_ctx->nhfb;
- cdef_search_ctx->sb_index =
- aom_malloc(nvfb * nhfb * sizeof(cdef_search_ctx->sb_index[0]));
+ CHECK_MEM_ERROR(
+ cm, cdef_search_ctx->sb_index,
+ aom_malloc(nvfb * nhfb * sizeof(cdef_search_ctx->sb_index[0])));
cdef_search_ctx->sb_count = 0;
- cdef_search_ctx->mse[0] =
- aom_malloc(sizeof(**cdef_search_ctx->mse) * nvfb * nhfb);
- cdef_search_ctx->mse[1] =
- aom_malloc(sizeof(**cdef_search_ctx->mse) * nvfb * nhfb);
- if (!(cdef_search_ctx->sb_index && cdef_search_ctx->mse[0] &&
- cdef_search_ctx->mse[1])) {
- aom_free(cdef_search_ctx->sb_index);
- aom_free(cdef_search_ctx->mse[0]);
- aom_free(cdef_search_ctx->mse[1]);
- return false;
- }
- return true;
+ CHECK_MEM_ERROR(cm, cdef_search_ctx->mse[0],
+ aom_malloc(sizeof(**cdef_search_ctx->mse) * nvfb * nhfb));
+ CHECK_MEM_ERROR(cm, cdef_search_ctx->mse[1],
+ aom_malloc(sizeof(**cdef_search_ctx->mse) * nvfb * nhfb));
}
// Deallocates the memory allocated for members of CdefSearchCtx.
@@ -660,10 +658,15 @@
// related to CDEF search context.
// Returns:
// Nothing will be returned.
-static AOM_INLINE void cdef_dealloc_data(CdefSearchCtx *cdef_search_ctx) {
- aom_free(cdef_search_ctx->mse[0]);
- aom_free(cdef_search_ctx->mse[1]);
- aom_free(cdef_search_ctx->sb_index);
+void av1_cdef_dealloc_data(CdefSearchCtx *cdef_search_ctx) {
+ if (cdef_search_ctx) {
+ aom_free(cdef_search_ctx->mse[0]);
+ cdef_search_ctx->mse[0] = NULL;
+ aom_free(cdef_search_ctx->mse[1]);
+ cdef_search_ctx->mse[1] = NULL;
+ aom_free(cdef_search_ctx->sb_index);
+ cdef_search_ctx->sb_index = NULL;
+ }
}
// Initialize the parameters related to CDEF search context.
@@ -818,14 +821,12 @@
}
}
-void av1_cdef_search(MultiThreadInfo *mt_info, const YV12_BUFFER_CONFIG *frame,
- const YV12_BUFFER_CONFIG *ref, AV1_COMMON *cm,
- MACROBLOCKD *xd, CDEF_PICK_METHOD pick_method, int rdmult,
- int skip_cdef_feature, CDEF_CONTROL cdef_control,
- const int is_screen_content, int non_reference_frame,
- int rtc_ext_rc) {
+void av1_cdef_search(AV1_COMP *cpi) {
+ AV1_COMMON *cm = &cpi->common;
+ CDEF_CONTROL cdef_control = cpi->oxcf.tool_cfg.cdef_control;
+
assert(cdef_control != CDEF_NONE);
- if (cdef_control == CDEF_REFERENCE && non_reference_frame) {
+ if (cdef_control == CDEF_REFERENCE && cpi->ppi->rtc_ref.non_reference_frame) {
CdefInfo *const cdef_info = &cm->cdef_info;
cdef_info->nb_cdef_strengths = 1;
cdef_info->cdef_bits = 0;
@@ -834,12 +835,21 @@
return;
}
+ // Indicate if external RC is used for testing
+ const int rtc_ext_rc = cpi->rc.rtc_external_ratectrl;
if (rtc_ext_rc) {
av1_pick_cdef_from_qp(cm, 0, 0);
return;
}
+ CDEF_PICK_METHOD pick_method = cpi->sf.lpf_sf.cdef_pick_method;
if (pick_method == CDEF_PICK_FROM_Q) {
- av1_pick_cdef_from_qp(cm, skip_cdef_feature, is_screen_content);
+ const int use_screen_content_model =
+ cm->quant_params.base_qindex >
+ AOMMAX(cpi->sf.rt_sf.screen_content_cdef_filter_qindex_thresh,
+ cpi->rc.best_quality + 5) &&
+ cpi->oxcf.tune_cfg.content == AOM_CONTENT_SCREEN;
+ av1_pick_cdef_from_qp(cm, cpi->sf.rt_sf.skip_cdef_sb,
+ use_screen_content_model);
return;
}
const CommonModeInfoParams *const mi_params = &cm->mi_params;
@@ -847,33 +857,33 @@
const int fast = (pick_method >= CDEF_FAST_SEARCH_LVL1 &&
pick_method <= CDEF_FAST_SEARCH_LVL5);
const int num_planes = av1_num_planes(cm);
- CdefSearchCtx cdef_search_ctx;
+ MACROBLOCKD *xd = &cpi->td.mb.e_mbd;
+
+ if (!cpi->cdef_search_ctx)
+ CHECK_MEM_ERROR(cm, cpi->cdef_search_ctx,
+ aom_malloc(sizeof(*cpi->cdef_search_ctx)));
+ CdefSearchCtx *cdef_search_ctx = cpi->cdef_search_ctx;
+
// Initialize parameters related to CDEF search context.
- cdef_params_init(frame, ref, cm, xd, &cdef_search_ctx, pick_method);
+ cdef_params_init(&cm->cur_frame->buf, cpi->source, cm, xd, cdef_search_ctx,
+ pick_method);
// Allocate CDEF search context buffers.
- if (!cdef_alloc_data(&cdef_search_ctx)) {
- CdefInfo *const cdef_info = &cm->cdef_info;
- cdef_info->nb_cdef_strengths = 0;
- cdef_info->cdef_bits = 0;
- cdef_info->cdef_strengths[0] = 0;
- cdef_info->cdef_uv_strengths[0] = 0;
- return;
- }
+ cdef_alloc_data(cm, cdef_search_ctx);
// Frame level mse calculation.
- if (mt_info->num_workers > 1) {
- av1_cdef_mse_calc_frame_mt(cm, mt_info, &cdef_search_ctx);
+ if (cpi->mt_info.num_workers > 1) {
+ av1_cdef_mse_calc_frame_mt(cpi);
} else {
- cdef_mse_calc_frame(&cdef_search_ctx);
+ cdef_mse_calc_frame(cdef_search_ctx, cm->error);
}
/* Search for different number of signaling bits. */
int nb_strength_bits = 0;
uint64_t best_rd = UINT64_MAX;
CdefInfo *const cdef_info = &cm->cdef_info;
- int sb_count = cdef_search_ctx.sb_count;
+ int sb_count = cdef_search_ctx->sb_count;
uint64_t(*mse[2])[TOTAL_STRENGTHS];
- mse[0] = cdef_search_ctx.mse[0];
- mse[1] = cdef_search_ctx.mse[1];
+ mse[0] = cdef_search_ctx->mse[0];
+ mse[1] = cdef_search_ctx->mse[1];
/* Calculate the maximum number of bits required to signal CDEF strengths at
* block level */
const int total_strengths = nb_cdef_strengths[pick_method];
@@ -881,6 +891,7 @@
num_planes > 1 ? total_strengths * total_strengths : total_strengths;
const int max_signaling_bits =
joint_strengths == 1 ? 0 : get_msb(joint_strengths - 1) + 1;
+ int rdmult = cpi->td.mb.rdmult;
for (int i = 0; i <= 3; i++) {
if (i > max_signaling_bits) break;
int best_lev0[CDEF_MAX_STRENGTHS];
@@ -925,7 +936,7 @@
best_mse = curr;
}
}
- mi_params->mi_grid_base[cdef_search_ctx.sb_index[i]]->cdef_strength =
+ mi_params->mi_grid_base[cdef_search_ctx->sb_index[i]]->cdef_strength =
best_gi;
}
if (fast) {
@@ -943,5 +954,5 @@
cdef_info->cdef_damping = damping;
// Deallocate CDEF search context buffers.
- cdef_dealloc_data(&cdef_search_ctx);
+ av1_cdef_dealloc_data(cdef_search_ctx);
}
diff --git a/av1/encoder/pickcdef.h b/av1/encoder/pickcdef.h
index bdd8233..192e734 100644
--- a/av1/encoder/pickcdef.h
+++ b/av1/encoder/pickcdef.h
@@ -213,8 +213,11 @@
return 0;
}
-void av1_cdef_mse_calc_block(CdefSearchCtx *cdef_search_ctx, int fbr, int fbc,
- int sb_count);
+void av1_cdef_dealloc_data(CdefSearchCtx *cdef_search_ctx);
+
+void av1_cdef_mse_calc_block(CdefSearchCtx *cdef_search_ctx,
+ struct aom_internal_error_info *error_info,
+ int fbr, int fbc, int sb_count);
/*!\endcond */
/*!\brief AV1 CDEF parameter search
@@ -223,19 +226,7 @@
*
* Searches for optimal CDEF parameters for frame
*
- * \param[in] mt_info Pointer to multi-threading parameters
- * \param[in] frame Compressed frame buffer
- * \param[in] ref Source frame buffer
- * \param[in,out] cm Pointer to top level common structure
- * \param[in] xd Pointer to common current coding block structure
- * \param[in] pick_method The method used to select params
- * \param[in] rdmult rd multiplier to use in making param choices
- * \param[in] skip_cdef_feature Speed feature to skip cdef
- * \param[in] cdef_control Parameter that controls CDEF application
- * \param[in] is_screen_content Whether it is screen content type
- * \param[in] non_reference_frame Indicates if current frame is
- * non-reference
- * \param[in] rtc_ext_rc Indicate if external RC is used for testing
+ * \param[in,out] cpi Top level encoder structure
*
* \remark Nothing is returned. Instead, optimal CDEF parameters are stored
* in the \c cdef_info structure of type \ref CdefInfo inside \c cm:
@@ -248,13 +239,7 @@
* \arg \c damping_factor: CDEF damping factor.
*
*/
-void av1_cdef_search(struct MultiThreadInfo *mt_info,
- const YV12_BUFFER_CONFIG *frame,
- const YV12_BUFFER_CONFIG *ref, AV1_COMMON *cm,
- MACROBLOCKD *xd, CDEF_PICK_METHOD pick_method, int rdmult,
- int skip_cdef_feature, CDEF_CONTROL cdef_control,
- const int is_screen_content, int non_reference_frame,
- int rtc_ext_rc);
+void av1_cdef_search(struct AV1_COMP *cpi);
/*!\brief AV1 CDEF level from QP
*
diff --git a/av1/encoder/pickrst.c b/av1/encoder/pickrst.c
index 7212469..6429064 100644
--- a/av1/encoder/pickrst.c
+++ b/av1/encoder/pickrst.c
@@ -52,6 +52,11 @@
{ 14, 14, 14, 14, 14, 14, 14, 15, 15, 15, 15, 15, 15, 15 }
};
+#if DEBUG_LR_COSTING
+RestorationUnitInfo lr_ref_params[RESTORE_TYPES][MAX_MB_PLANE]
+ [MAX_LR_UNITS_W * MAX_LR_UNITS_H];
+#endif // DEBUG_LR_COSTING
+
typedef int64_t (*sse_extractor_type)(const YV12_BUFFER_CONFIG *a,
const YV12_BUFFER_CONFIG *b);
typedef int64_t (*sse_part_extractor_type)(const YV12_BUFFER_CONFIG *a,
@@ -100,31 +105,14 @@
}
typedef struct {
- // The best coefficients for Wiener or Sgrproj restoration
- WienerInfo wiener;
- SgrprojInfo sgrproj;
-
- // The sum of squared errors for this rtype.
- int64_t sse[RESTORE_SWITCHABLE_TYPES];
-
- // The rtype to use for this unit given a frame rtype as
- // index. Indices: WIENER, SGRPROJ, SWITCHABLE.
- RestorationType best_rtype[RESTORE_TYPES - 1];
-
- // This flag will be set based on the speed feature
- // 'prune_sgr_based_on_wiener'. 0 implies no pruning and 1 implies pruning.
- uint8_t skip_sgr_eval;
-} RestUnitSearchInfo;
-
-typedef struct {
const YV12_BUFFER_CONFIG *src;
YV12_BUFFER_CONFIG *dst;
const AV1_COMMON *cm;
const MACROBLOCK *x;
int plane;
- int plane_width;
- int plane_height;
+ int plane_w;
+ int plane_h;
RestUnitSearchInfo *rusi;
// Speed features
@@ -135,16 +123,32 @@
const uint8_t *src_buffer;
int src_stride;
- // sse and bits are initialised by reset_rsc in search_rest_type
- int64_t sse;
- int64_t bits;
- int tile_y0, tile_stripe0;
+ // SSE values for each restoration mode for the current RU
+ // These are saved by each search function for use in search_switchable()
+ int64_t sse[RESTORE_SWITCHABLE_TYPES];
- // sgrproj and wiener are initialised by rsc_on_tile when starting the first
- // tile in the frame.
- SgrprojInfo sgrproj;
- WienerInfo wiener;
- PixelRect tile_rect;
+ // This flag will be set based on the speed feature
+ // 'prune_sgr_based_on_wiener'. 0 implies no pruning and 1 implies pruning.
+ uint8_t skip_sgr_eval;
+
+ // Total rate and distortion so far for each restoration type
+ // These are initialised by reset_rsc in search_rest_type
+ int64_t total_sse[RESTORE_TYPES];
+ int64_t total_bits[RESTORE_TYPES];
+
+ // Reference parameters for delta-coding
+ //
+ // For each restoration type, we need to store the latest parameter set which
+ // has been used, so that we can properly cost up the next parameter set.
+ // Note that we have two sets of these - one for the single-restoration-mode
+ // search (ie, frame_restoration_type = RESTORE_WIENER or RESTORE_SGRPROJ)
+ // and one for the switchable mode. This is because these two cases can lead
+ // to different sets of parameters being signaled, but we don't know which
+ // we will pick for sure until the end of the search process.
+ WienerInfo ref_wiener;
+ SgrprojInfo ref_sgrproj;
+ WienerInfo switchable_ref_wiener;
+ SgrprojInfo switchable_ref_sgrproj;
// Buffers used to hold dgd-avg and src-avg data respectively during SIMD
// call of Wiener filter.
@@ -154,14 +158,15 @@
static AOM_INLINE void rsc_on_tile(void *priv) {
RestSearchCtxt *rsc = (RestSearchCtxt *)priv;
- set_default_sgrproj(&rsc->sgrproj);
- set_default_wiener(&rsc->wiener);
- rsc->tile_stripe0 = 0;
+ set_default_wiener(&rsc->ref_wiener);
+ set_default_sgrproj(&rsc->ref_sgrproj);
+ set_default_wiener(&rsc->switchable_ref_wiener);
+ set_default_sgrproj(&rsc->switchable_ref_sgrproj);
}
static AOM_INLINE void reset_rsc(RestSearchCtxt *rsc) {
- rsc->sse = 0;
- rsc->bits = 0;
+ memset(rsc->total_sse, 0, sizeof(rsc->total_sse));
+ memset(rsc->total_bits, 0, sizeof(rsc->total_bits));
}
static AOM_INLINE void init_rsc(const YV12_BUFFER_CONFIG *src,
@@ -179,20 +184,23 @@
const YV12_BUFFER_CONFIG *dgd = &cm->cur_frame->buf;
const int is_uv = plane != AOM_PLANE_Y;
- rsc->plane_width = src->crop_widths[is_uv];
- rsc->plane_height = src->crop_heights[is_uv];
+ int plane_w, plane_h;
+ av1_get_upsampled_plane_size(cm, is_uv, &plane_w, &plane_h);
+ assert(plane_w == src->crop_widths[is_uv]);
+ assert(plane_h == src->crop_heights[is_uv]);
+ assert(src->crop_widths[is_uv] == dgd->crop_widths[is_uv]);
+ assert(src->crop_heights[is_uv] == dgd->crop_heights[is_uv]);
+
+ rsc->plane_w = plane_w;
+ rsc->plane_h = plane_h;
rsc->src_buffer = src->buffers[plane];
rsc->src_stride = src->strides[is_uv];
rsc->dgd_buffer = dgd->buffers[plane];
rsc->dgd_stride = dgd->strides[is_uv];
- rsc->tile_rect = av1_whole_frame_rect(cm, is_uv);
- assert(src->crop_widths[is_uv] == dgd->crop_widths[is_uv]);
- assert(src->crop_heights[is_uv] == dgd->crop_heights[is_uv]);
}
static int64_t try_restoration_unit(const RestSearchCtxt *rsc,
const RestorationTileLimits *limits,
- const PixelRect *tile_rect,
const RestorationUnitInfo *rui) {
const AV1_COMMON *const cm = rsc->cm;
const int plane = rsc->plane;
@@ -208,11 +216,11 @@
const int optimized_lr = 0;
av1_loop_restoration_filter_unit(
- limits, rui, &rsi->boundaries, &rlbs, tile_rect, rsc->tile_stripe0,
+ limits, rui, &rsi->boundaries, &rlbs, rsc->plane_w, rsc->plane_h,
is_uv && cm->seq_params->subsampling_x,
is_uv && cm->seq_params->subsampling_y, highbd, bit_depth,
fts->buffers[plane], fts->strides[is_uv], rsc->dst->buffers[plane],
- rsc->dst->strides[is_uv], cm->rst_tmpbuf, optimized_lr);
+ rsc->dst->strides[is_uv], cm->rst_tmpbuf, optimized_lr, cm->error);
return sse_restoration_unit(limits, rsc->src, rsc->dst, plane, highbd);
}
@@ -746,7 +754,8 @@
int width, int height, int dat_stride,
int use_highbd, int bit_depth, int pu_width,
int pu_height, int32_t *flt0, int32_t *flt1,
- int flt_stride) {
+ int flt_stride,
+ struct aom_internal_error_info *error_info) {
for (int i = 0; i < height; i += pu_height) {
const int h = AOMMIN(pu_height, height - i);
int32_t *flt0_row = flt0 + i * flt_stride;
@@ -756,11 +765,13 @@
// Iterate over the stripe in blocks of width pu_width
for (int j = 0; j < width; j += pu_width) {
const int w = AOMMIN(pu_width, width - j);
- const int ret = av1_selfguided_restoration(
- dat8_row + j, w, h, dat_stride, flt0_row + j, flt1_row + j,
- flt_stride, sgr_params_idx, bit_depth, use_highbd);
- (void)ret;
- assert(!ret);
+ if (av1_selfguided_restoration(
+ dat8_row + j, w, h, dat_stride, flt0_row + j, flt1_row + j,
+ flt_stride, sgr_params_idx, bit_depth, use_highbd) != 0) {
+ aom_internal_error(
+ error_info, AOM_CODEC_MEM_ERROR,
+ "Error allocating buffer in av1_selfguided_restoration");
+ }
}
}
}
@@ -770,10 +781,11 @@
const int dat_stride, const uint8_t *src8, const int src_stride,
const int use_highbitdepth, const int bit_depth, const int pu_width,
const int pu_height, const int ep, int32_t *flt0, int32_t *flt1,
- const int flt_stride, int *exqd, int64_t *err) {
+ const int flt_stride, int *exqd, int64_t *err,
+ struct aom_internal_error_info *error_info) {
int exq[2];
apply_sgr(ep, dat8, width, height, dat_stride, use_highbitdepth, bit_depth,
- pu_width, pu_height, flt0, flt1, flt_stride);
+ pu_width, pu_height, flt0, flt1, flt_stride, error_info);
const sgr_params_type *const params = &av1_sgr_params[ep];
get_proj_subspace(src8, width, height, src_stride, dat8, dat_stride,
use_highbitdepth, flt0, flt_stride, flt1, flt_stride, exq,
@@ -798,7 +810,8 @@
static SgrprojInfo search_selfguided_restoration(
const uint8_t *dat8, int width, int height, int dat_stride,
const uint8_t *src8, int src_stride, int use_highbitdepth, int bit_depth,
- int pu_width, int pu_height, int32_t *rstbuf, int enable_sgr_ep_pruning) {
+ int pu_width, int pu_height, int32_t *rstbuf, int enable_sgr_ep_pruning,
+ struct aom_internal_error_info *error_info) {
int32_t *flt0 = rstbuf;
int32_t *flt1 = flt0 + RESTORATION_UNITPELS_MAX;
int ep, idx, bestep = 0;
@@ -814,7 +827,7 @@
int64_t err;
compute_sgrproj_err(dat8, width, height, dat_stride, src8, src_stride,
use_highbitdepth, bit_depth, pu_width, pu_height, ep,
- flt0, flt1, flt_stride, exqd, &err);
+ flt0, flt1, flt_stride, exqd, &err, error_info);
get_best_error(&besterr, err, exqd, bestxqd, &bestep, ep);
}
} else {
@@ -824,7 +837,7 @@
int64_t err;
compute_sgrproj_err(dat8, width, height, dat_stride, src8, src_stride,
use_highbitdepth, bit_depth, pu_width, pu_height, ep,
- flt0, flt1, flt_stride, exqd, &err);
+ flt0, flt1, flt_stride, exqd, &err, error_info);
get_best_error(&besterr, err, exqd, bestxqd, &bestep, ep);
}
// evaluate left and right ep of winner in seed ep
@@ -835,7 +848,7 @@
int64_t err;
compute_sgrproj_err(dat8, width, height, dat_stride, src8, src_stride,
use_highbitdepth, bit_depth, pu_width, pu_height, ep,
- flt0, flt1, flt_stride, exqd, &err);
+ flt0, flt1, flt_stride, exqd, &err, error_info);
get_best_error(&besterr, err, exqd, bestxqd, &bestep, ep);
}
// evaluate last two group
@@ -844,7 +857,7 @@
int64_t err;
compute_sgrproj_err(dat8, width, height, dat_stride, src8, src_stride,
use_highbitdepth, bit_depth, pu_width, pu_height, ep,
- flt0, flt1, flt_stride, exqd, &err);
+ flt0, flt1, flt_stride, exqd, &err, error_info);
get_best_error(&besterr, err, exqd, bestxqd, &bestep, ep);
}
}
@@ -873,10 +886,10 @@
return bits;
}
-static AOM_INLINE void search_sgrproj(const RestorationTileLimits *limits,
- const PixelRect *tile, int rest_unit_idx,
- void *priv, int32_t *tmpbuf,
- RestorationLineBuffers *rlbs) {
+static AOM_INLINE void search_sgrproj(
+ const RestorationTileLimits *limits, int rest_unit_idx, void *priv,
+ int32_t *tmpbuf, RestorationLineBuffers *rlbs,
+ struct aom_internal_error_info *error_info) {
(void)rlbs;
RestSearchCtxt *rsc = (RestSearchCtxt *)priv;
RestUnitSearchInfo *rusi = &rsc->rusi[rest_unit_idx];
@@ -888,11 +901,11 @@
const int64_t bits_none = x->mode_costs.sgrproj_restore_cost[0];
// Prune evaluation of RESTORE_SGRPROJ if 'skip_sgr_eval' is set
- if (rusi->skip_sgr_eval) {
- rsc->bits += bits_none;
- rsc->sse += rusi->sse[RESTORE_NONE];
+ if (rsc->skip_sgr_eval) {
+ rsc->total_bits[RESTORE_SGRPROJ] += bits_none;
+ rsc->total_sse[RESTORE_SGRPROJ] += rsc->sse[RESTORE_NONE];
rusi->best_rtype[RESTORE_SGRPROJ - 1] = RESTORE_NONE;
- rusi->sse[RESTORE_SGRPROJ] = INT64_MAX;
+ rsc->sse[RESTORE_SGRPROJ] = INT64_MAX;
return;
}
@@ -911,21 +924,22 @@
dgd_start, limits->h_end - limits->h_start,
limits->v_end - limits->v_start, rsc->dgd_stride, src_start,
rsc->src_stride, highbd, bit_depth, procunit_width, procunit_height,
- tmpbuf, rsc->lpf_sf->enable_sgr_ep_pruning);
+ tmpbuf, rsc->lpf_sf->enable_sgr_ep_pruning, error_info);
RestorationUnitInfo rui;
rui.restoration_type = RESTORE_SGRPROJ;
rui.sgrproj_info = rusi->sgrproj;
- rusi->sse[RESTORE_SGRPROJ] = try_restoration_unit(rsc, limits, tile, &rui);
+ rsc->sse[RESTORE_SGRPROJ] = try_restoration_unit(rsc, limits, &rui);
- const int64_t bits_sgr = x->mode_costs.sgrproj_restore_cost[1] +
- (count_sgrproj_bits(&rusi->sgrproj, &rsc->sgrproj)
- << AV1_PROB_COST_SHIFT);
+ const int64_t bits_sgr =
+ x->mode_costs.sgrproj_restore_cost[1] +
+ (count_sgrproj_bits(&rusi->sgrproj, &rsc->ref_sgrproj)
+ << AV1_PROB_COST_SHIFT);
double cost_none = RDCOST_DBL_WITH_NATIVE_BD_DIST(
- x->rdmult, bits_none >> 4, rusi->sse[RESTORE_NONE], bit_depth);
+ x->rdmult, bits_none >> 4, rsc->sse[RESTORE_NONE], bit_depth);
double cost_sgr = RDCOST_DBL_WITH_NATIVE_BD_DIST(
- x->rdmult, bits_sgr >> 4, rusi->sse[RESTORE_SGRPROJ], bit_depth);
+ x->rdmult, bits_sgr >> 4, rsc->sse[RESTORE_SGRPROJ], bit_depth);
if (rusi->sgrproj.ep < 10)
cost_sgr *=
(1 + DUAL_SGR_PENALTY_MULT * rsc->lpf_sf->dual_sgr_penalty_level);
@@ -934,9 +948,16 @@
(cost_sgr < cost_none) ? RESTORE_SGRPROJ : RESTORE_NONE;
rusi->best_rtype[RESTORE_SGRPROJ - 1] = rtype;
- rsc->sse += rusi->sse[rtype];
- rsc->bits += (cost_sgr < cost_none) ? bits_sgr : bits_none;
- if (cost_sgr < cost_none) rsc->sgrproj = rusi->sgrproj;
+#if DEBUG_LR_COSTING
+ // Store ref params for later checking
+ lr_ref_params[RESTORE_SGRPROJ][rsc->plane][rest_unit_idx].sgrproj_info =
+ rsc->ref_sgrproj;
+#endif // DEBUG_LR_COSTING
+
+ rsc->total_sse[RESTORE_SGRPROJ] += rsc->sse[rtype];
+ rsc->total_bits[RESTORE_SGRPROJ] +=
+ (cost_sgr < cost_none) ? bits_sgr : bits_none;
+ if (cost_sgr < cost_none) rsc->ref_sgrproj = rusi->sgrproj;
}
static void acc_stat_one_line(const uint8_t *dgd, const uint8_t *src,
@@ -1455,13 +1476,11 @@
return bits;
}
-static int64_t finer_tile_search_wiener(const RestSearchCtxt *rsc,
- const RestorationTileLimits *limits,
- const PixelRect *tile,
- RestorationUnitInfo *rui,
- int wiener_win) {
+static int64_t finer_search_wiener(const RestSearchCtxt *rsc,
+ const RestorationTileLimits *limits,
+ RestorationUnitInfo *rui, int wiener_win) {
const int plane_off = (WIENER_WIN - wiener_win) >> 1;
- int64_t err = try_restoration_unit(rsc, limits, tile, rui);
+ int64_t err = try_restoration_unit(rsc, limits, rui);
if (rsc->lpf_sf->disable_wiener_coeff_refine_search) return err;
@@ -1484,7 +1503,7 @@
plane_wiener->hfilter[p] -= s;
plane_wiener->hfilter[WIENER_WIN - p - 1] -= s;
plane_wiener->hfilter[WIENER_HALFWIN] += 2 * s;
- err2 = try_restoration_unit(rsc, limits, tile, rui);
+ err2 = try_restoration_unit(rsc, limits, rui);
if (err2 > err) {
plane_wiener->hfilter[p] += s;
plane_wiener->hfilter[WIENER_WIN - p - 1] += s;
@@ -1504,7 +1523,7 @@
plane_wiener->hfilter[p] += s;
plane_wiener->hfilter[WIENER_WIN - p - 1] += s;
plane_wiener->hfilter[WIENER_HALFWIN] -= 2 * s;
- err2 = try_restoration_unit(rsc, limits, tile, rui);
+ err2 = try_restoration_unit(rsc, limits, rui);
if (err2 > err) {
plane_wiener->hfilter[p] -= s;
plane_wiener->hfilter[WIENER_WIN - p - 1] -= s;
@@ -1525,7 +1544,7 @@
plane_wiener->vfilter[p] -= s;
plane_wiener->vfilter[WIENER_WIN - p - 1] -= s;
plane_wiener->vfilter[WIENER_HALFWIN] += 2 * s;
- err2 = try_restoration_unit(rsc, limits, tile, rui);
+ err2 = try_restoration_unit(rsc, limits, rui);
if (err2 > err) {
plane_wiener->vfilter[p] += s;
plane_wiener->vfilter[WIENER_WIN - p - 1] += s;
@@ -1545,7 +1564,7 @@
plane_wiener->vfilter[p] += s;
plane_wiener->vfilter[WIENER_WIN - p - 1] += s;
plane_wiener->vfilter[WIENER_HALFWIN] -= 2 * s;
- err2 = try_restoration_unit(rsc, limits, tile, rui);
+ err2 = try_restoration_unit(rsc, limits, rui);
if (err2 > err) {
plane_wiener->vfilter[p] -= s;
plane_wiener->vfilter[WIENER_WIN - p - 1] -= s;
@@ -1564,13 +1583,13 @@
return err;
}
-static AOM_INLINE void search_wiener(const RestorationTileLimits *limits,
- const PixelRect *tile_rect,
- int rest_unit_idx, void *priv,
- int32_t *tmpbuf,
- RestorationLineBuffers *rlbs) {
+static AOM_INLINE void search_wiener(
+ const RestorationTileLimits *limits, int rest_unit_idx, void *priv,
+ int32_t *tmpbuf, RestorationLineBuffers *rlbs,
+ struct aom_internal_error_info *error_info) {
(void)tmpbuf;
(void)rlbs;
+ (void)error_info;
RestSearchCtxt *rsc = (RestSearchCtxt *)priv;
RestUnitSearchInfo *rusi = &rsc->rusi[rest_unit_idx];
@@ -1592,13 +1611,13 @@
var_restoration_unit(limits, rsc->src, rsc->plane, highbd);
// Do not perform Wiener search if source variance is lower than threshold
// or if the reconstruction error is zero
- int prune_wiener = (src_var < thresh) || (rusi->sse[RESTORE_NONE] == 0);
+ int prune_wiener = (src_var < thresh) || (rsc->sse[RESTORE_NONE] == 0);
if (prune_wiener) {
- rsc->bits += bits_none;
- rsc->sse += rusi->sse[RESTORE_NONE];
+ rsc->total_bits[RESTORE_WIENER] += bits_none;
+ rsc->total_sse[RESTORE_WIENER] += rsc->sse[RESTORE_NONE];
rusi->best_rtype[RESTORE_WIENER - 1] = RESTORE_NONE;
- rusi->sse[RESTORE_WIENER] = INT64_MAX;
- if (rsc->lpf_sf->prune_sgr_based_on_wiener == 2) rusi->skip_sgr_eval = 1;
+ rsc->sse[RESTORE_WIENER] = INT64_MAX;
+ if (rsc->lpf_sf->prune_sgr_based_on_wiener == 2) rsc->skip_sgr_eval = 1;
return;
}
}
@@ -1654,16 +1673,16 @@
// reduction in the function, the filter is reverted back to identity
if (compute_score(reduced_wiener_win, M, H, rui.wiener_info.vfilter,
rui.wiener_info.hfilter) > 0) {
- rsc->bits += bits_none;
- rsc->sse += rusi->sse[RESTORE_NONE];
+ rsc->total_bits[RESTORE_WIENER] += bits_none;
+ rsc->total_sse[RESTORE_WIENER] += rsc->sse[RESTORE_NONE];
rusi->best_rtype[RESTORE_WIENER - 1] = RESTORE_NONE;
- rusi->sse[RESTORE_WIENER] = INT64_MAX;
- if (rsc->lpf_sf->prune_sgr_based_on_wiener == 2) rusi->skip_sgr_eval = 1;
+ rsc->sse[RESTORE_WIENER] = INT64_MAX;
+ if (rsc->lpf_sf->prune_sgr_based_on_wiener == 2) rsc->skip_sgr_eval = 1;
return;
}
- rusi->sse[RESTORE_WIENER] = finer_tile_search_wiener(
- rsc, limits, tile_rect, &rui, reduced_wiener_win);
+ rsc->sse[RESTORE_WIENER] =
+ finer_search_wiener(rsc, limits, &rui, reduced_wiener_win);
rusi->wiener = rui.wiener_info;
if (reduced_wiener_win != WIENER_WIN) {
@@ -1675,14 +1694,14 @@
const int64_t bits_wiener =
x->mode_costs.wiener_restore_cost[1] +
- (count_wiener_bits(wiener_win, &rusi->wiener, &rsc->wiener)
+ (count_wiener_bits(wiener_win, &rusi->wiener, &rsc->ref_wiener)
<< AV1_PROB_COST_SHIFT);
double cost_none = RDCOST_DBL_WITH_NATIVE_BD_DIST(
- x->rdmult, bits_none >> 4, rusi->sse[RESTORE_NONE],
+ x->rdmult, bits_none >> 4, rsc->sse[RESTORE_NONE],
rsc->cm->seq_params->bit_depth);
double cost_wiener = RDCOST_DBL_WITH_NATIVE_BD_DIST(
- x->rdmult, bits_wiener >> 4, rusi->sse[RESTORE_WIENER],
+ x->rdmult, bits_wiener >> 4, rsc->sse[RESTORE_WIENER],
rsc->cm->seq_params->bit_depth);
RestorationType rtype =
@@ -1692,44 +1711,49 @@
// Set 'skip_sgr_eval' based on rdcost ratio of RESTORE_WIENER and
// RESTORE_NONE or based on best_rtype
if (rsc->lpf_sf->prune_sgr_based_on_wiener == 1) {
- rusi->skip_sgr_eval = cost_wiener > (1.01 * cost_none);
+ rsc->skip_sgr_eval = cost_wiener > (1.01 * cost_none);
} else if (rsc->lpf_sf->prune_sgr_based_on_wiener == 2) {
- rusi->skip_sgr_eval = rusi->best_rtype[RESTORE_WIENER - 1] == RESTORE_NONE;
+ rsc->skip_sgr_eval = rusi->best_rtype[RESTORE_WIENER - 1] == RESTORE_NONE;
}
- rsc->sse += rusi->sse[rtype];
- rsc->bits += (cost_wiener < cost_none) ? bits_wiener : bits_none;
- if (cost_wiener < cost_none) rsc->wiener = rusi->wiener;
+#if DEBUG_LR_COSTING
+ // Store ref params for later checking
+ lr_ref_params[RESTORE_WIENER][rsc->plane][rest_unit_idx].wiener_info =
+ rsc->ref_wiener;
+#endif // DEBUG_LR_COSTING
+
+ rsc->total_sse[RESTORE_WIENER] += rsc->sse[rtype];
+ rsc->total_bits[RESTORE_WIENER] +=
+ (cost_wiener < cost_none) ? bits_wiener : bits_none;
+ if (cost_wiener < cost_none) rsc->ref_wiener = rusi->wiener;
}
-static AOM_INLINE void search_norestore(const RestorationTileLimits *limits,
- const PixelRect *tile_rect,
- int rest_unit_idx, void *priv,
- int32_t *tmpbuf,
- RestorationLineBuffers *rlbs) {
- (void)tile_rect;
+static AOM_INLINE void search_norestore(
+ const RestorationTileLimits *limits, int rest_unit_idx, void *priv,
+ int32_t *tmpbuf, RestorationLineBuffers *rlbs,
+ struct aom_internal_error_info *error_info) {
+ (void)rest_unit_idx;
(void)tmpbuf;
(void)rlbs;
+ (void)error_info;
RestSearchCtxt *rsc = (RestSearchCtxt *)priv;
- RestUnitSearchInfo *rusi = &rsc->rusi[rest_unit_idx];
const int highbd = rsc->cm->seq_params->use_highbitdepth;
- rusi->sse[RESTORE_NONE] = sse_restoration_unit(
+ rsc->sse[RESTORE_NONE] = sse_restoration_unit(
limits, rsc->src, &rsc->cm->cur_frame->buf, rsc->plane, highbd);
- rsc->sse += rusi->sse[RESTORE_NONE];
+ rsc->total_sse[RESTORE_NONE] += rsc->sse[RESTORE_NONE];
}
-static AOM_INLINE void search_switchable(const RestorationTileLimits *limits,
- const PixelRect *tile_rect,
- int rest_unit_idx, void *priv,
- int32_t *tmpbuf,
- RestorationLineBuffers *rlbs) {
+static AOM_INLINE void search_switchable(
+ const RestorationTileLimits *limits, int rest_unit_idx, void *priv,
+ int32_t *tmpbuf, RestorationLineBuffers *rlbs,
+ struct aom_internal_error_info *error_info) {
(void)limits;
- (void)tile_rect;
(void)tmpbuf;
(void)rlbs;
+ (void)error_info;
RestSearchCtxt *rsc = (RestSearchCtxt *)priv;
RestUnitSearchInfo *rusi = &rsc->rusi[rest_unit_idx];
@@ -1743,24 +1767,32 @@
RestorationType best_rtype = RESTORE_NONE;
for (RestorationType r = 0; r < RESTORE_SWITCHABLE_TYPES; ++r) {
- // Check for the condition that wiener or sgrproj search could not
- // find a solution or the solution was worse than RESTORE_NONE.
- // In either case the best_rtype will be set as RESTORE_NONE. These
- // should be skipped from the test below.
+ // If this restoration mode was skipped, or could not find a solution
+ // that was better than RESTORE_NONE, then we can't select it here either.
+ //
+ // Note: It is possible for the restoration search functions to find a
+ // filter which is better than RESTORE_NONE when looking purely at SSE, but
+ // for it to be rejected overall due to its rate cost. In this case, there
+ // is a chance that it may be have a lower rate cost when looking at
+ // RESTORE_SWITCHABLE, and so it might be acceptable here.
+ //
+ // Therefore we prune based on SSE, rather than on whether or not the
+ // previous search function selected this mode.
if (r > RESTORE_NONE) {
- if (rusi->best_rtype[r - 1] == RESTORE_NONE) continue;
+ if (rsc->sse[r] > rsc->sse[RESTORE_NONE]) continue;
}
- const int64_t sse = rusi->sse[r];
+ const int64_t sse = rsc->sse[r];
int64_t coeff_pcost = 0;
switch (r) {
case RESTORE_NONE: coeff_pcost = 0; break;
case RESTORE_WIENER:
- coeff_pcost =
- count_wiener_bits(wiener_win, &rusi->wiener, &rsc->wiener);
+ coeff_pcost = count_wiener_bits(wiener_win, &rusi->wiener,
+ &rsc->switchable_ref_wiener);
break;
case RESTORE_SGRPROJ:
- coeff_pcost = count_sgrproj_bits(&rusi->sgrproj, &rsc->sgrproj);
+ coeff_pcost =
+ count_sgrproj_bits(&rusi->sgrproj, &rsc->switchable_ref_sgrproj);
break;
default: assert(0); break;
}
@@ -1779,10 +1811,19 @@
rusi->best_rtype[RESTORE_SWITCHABLE - 1] = best_rtype;
- rsc->sse += rusi->sse[best_rtype];
- rsc->bits += best_bits;
- if (best_rtype == RESTORE_WIENER) rsc->wiener = rusi->wiener;
- if (best_rtype == RESTORE_SGRPROJ) rsc->sgrproj = rusi->sgrproj;
+#if DEBUG_LR_COSTING
+ // Store ref params for later checking
+ lr_ref_params[RESTORE_SWITCHABLE][rsc->plane][rest_unit_idx].wiener_info =
+ rsc->switchable_ref_wiener;
+ lr_ref_params[RESTORE_SWITCHABLE][rsc->plane][rest_unit_idx].sgrproj_info =
+ rsc->switchable_ref_sgrproj;
+#endif // DEBUG_LR_COSTING
+
+ rsc->total_sse[RESTORE_SWITCHABLE] += rsc->sse[best_rtype];
+ rsc->total_bits[RESTORE_SWITCHABLE] += best_bits;
+ if (best_rtype == RESTORE_WIENER) rsc->switchable_ref_wiener = rusi->wiener;
+ if (best_rtype == RESTORE_SGRPROJ)
+ rsc->switchable_ref_sgrproj = rusi->sgrproj;
}
static AOM_INLINE void copy_unit_info(RestorationType frame_rtype,
@@ -1796,23 +1837,96 @@
rui->sgrproj_info = rusi->sgrproj;
}
-static double search_rest_type(RestSearchCtxt *rsc, RestorationType rtype) {
+static void restoration_search(AV1_COMMON *cm, int plane, RestSearchCtxt *rsc,
+ bool *disable_lr_filter) {
+ const BLOCK_SIZE sb_size = cm->seq_params->sb_size;
+ const int mib_size_log2 = cm->seq_params->mib_size_log2;
+ const CommonTileParams *tiles = &cm->tiles;
+ const int is_uv = plane > 0;
+ const int ss_y = is_uv && cm->seq_params->subsampling_y;
+ RestorationInfo *rsi = &cm->rst_info[plane];
+ const int ru_size = rsi->restoration_unit_size;
+ const int ext_size = ru_size * 3 / 2;
+
+ int plane_w, plane_h;
+ av1_get_upsampled_plane_size(cm, is_uv, &plane_w, &plane_h);
+
static const rest_unit_visitor_t funs[RESTORE_TYPES] = {
search_norestore, search_wiener, search_sgrproj, search_switchable
};
+ const int plane_num_units = rsi->num_rest_units;
+ const RestorationType num_rtypes =
+ (plane_num_units > 1) ? RESTORE_TYPES : RESTORE_SWITCHABLE_TYPES;
+
reset_rsc(rsc);
- rsc_on_tile(rsc);
- av1_foreach_rest_unit_in_plane(rsc->cm, rsc->plane, funs[rtype], rsc,
- &rsc->tile_rect, rsc->cm->rst_tmpbuf, NULL);
- return RDCOST_DBL_WITH_NATIVE_BD_DIST(
- rsc->x->rdmult, rsc->bits >> 4, rsc->sse, rsc->cm->seq_params->bit_depth);
-}
+ // Iterate over restoration units in encoding order, so that each RU gets
+ // the correct reference parameters when we cost it up. This is effectively
+ // a nested iteration over:
+ // * Each tile, order does not matter
+ // * Each superblock within that tile, in raster order
+ // * Each LR unit which is coded within that superblock, in raster order
+ for (int tile_row = 0; tile_row < tiles->rows; tile_row++) {
+ int sb_row_start = tiles->row_start_sb[tile_row];
+ int sb_row_end = tiles->row_start_sb[tile_row + 1];
+ for (int tile_col = 0; tile_col < tiles->cols; tile_col++) {
+ int sb_col_start = tiles->col_start_sb[tile_col];
+ int sb_col_end = tiles->col_start_sb[tile_col + 1];
-static int rest_tiles_in_plane(const AV1_COMMON *cm, int plane) {
- const RestorationInfo *rsi = &cm->rst_info[plane];
- return rsi->units_per_tile;
+ // Reset reference parameters for delta-coding at the start of each tile
+ rsc_on_tile(rsc);
+
+ for (int sb_row = sb_row_start; sb_row < sb_row_end; sb_row++) {
+ int mi_row = sb_row << mib_size_log2;
+ for (int sb_col = sb_col_start; sb_col < sb_col_end; sb_col++) {
+ int mi_col = sb_col << mib_size_log2;
+
+ int rcol0, rcol1, rrow0, rrow1;
+ int has_lr_info = av1_loop_restoration_corners_in_sb(
+ cm, plane, mi_row, mi_col, sb_size, &rcol0, &rcol1, &rrow0,
+ &rrow1);
+
+ if (!has_lr_info) continue;
+
+ RestorationTileLimits limits;
+ for (int rrow = rrow0; rrow < rrow1; rrow++) {
+ int y0 = rrow * ru_size;
+ int remaining_h = plane_h - y0;
+ int h = (remaining_h < ext_size) ? remaining_h : ru_size;
+
+ limits.v_start = y0;
+ limits.v_end = y0 + h;
+ assert(limits.v_end <= plane_h);
+ // Offset upwards to align with the restoration processing stripe
+ const int voffset = RESTORATION_UNIT_OFFSET >> ss_y;
+ limits.v_start = AOMMAX(0, limits.v_start - voffset);
+ if (limits.v_end < plane_h) limits.v_end -= voffset;
+
+ for (int rcol = rcol0; rcol < rcol1; rcol++) {
+ int x0 = rcol * ru_size;
+ int remaining_w = plane_w - x0;
+ int w = (remaining_w < ext_size) ? remaining_w : ru_size;
+
+ limits.h_start = x0;
+ limits.h_end = x0 + w;
+ assert(limits.h_end <= plane_w);
+
+ const int unit_idx = rrow * rsi->horz_units + rcol;
+
+ rsc->skip_sgr_eval = 0;
+ for (RestorationType r = RESTORE_NONE; r < num_rtypes; r++) {
+ if (disable_lr_filter[r]) continue;
+
+ funs[r](&limits, unit_idx, rsc, rsc->cm->rst_tmpbuf, NULL,
+ cm->error);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
}
static INLINE void av1_derive_flags_for_lr_processing(
@@ -1833,31 +1947,101 @@
(is_wiener_disabled || is_sgr_disabled);
}
-void av1_pick_filter_restoration(const YV12_BUFFER_CONFIG *src, AV1_COMP *cpi) {
- AV1_COMMON *const cm = &cpi->common;
- MACROBLOCK *const x = &cpi->td.mb;
- const SequenceHeader *const seq_params = cm->seq_params;
- const int num_planes = av1_num_planes(cm);
- assert(!cm->features.all_lossless);
+#define COUPLED_CHROMA_FROM_LUMA_RESTORATION 0
+// Allocate both decoder-side and encoder-side info structs for a single plane.
+// The unit size passed in should be the minimum size which we are going to
+// search; before each search, set_restoration_unit_size() must be called to
+// configure the actual size.
+static RestUnitSearchInfo *allocate_search_structs(AV1_COMMON *cm,
+ RestorationInfo *rsi,
+ int is_uv,
+ int min_luma_unit_size) {
+#if COUPLED_CHROMA_FROM_LUMA_RESTORATION
+ int sx = cm->seq_params.subsampling_x;
+ int sy = cm->seq_params.subsampling_y;
+ int s = (p > 0) ? AOMMIN(sx, sy) : 0;
+#else
+ int s = 0;
+#endif // !COUPLED_CHROMA_FROM_LUMA_RESTORATION
+ int min_unit_size = min_luma_unit_size >> s;
- av1_fill_lr_rates(&x->mode_costs, x->e_mbd.tile_ctx);
+ int plane_w, plane_h;
+ av1_get_upsampled_plane_size(cm, is_uv, &plane_w, &plane_h);
- int ntiles[2];
- for (int is_uv = 0; is_uv < 2; ++is_uv)
- ntiles[is_uv] = rest_tiles_in_plane(cm, is_uv);
+ const int max_horz_units = av1_lr_count_units(min_unit_size, plane_w);
+ const int max_vert_units = av1_lr_count_units(min_unit_size, plane_h);
+ const int max_num_units = max_horz_units * max_vert_units;
- assert(ntiles[1] <= ntiles[0]);
+ aom_free(rsi->unit_info);
+ CHECK_MEM_ERROR(cm, rsi->unit_info,
+ (RestorationUnitInfo *)aom_memalign(
+ 16, sizeof(*rsi->unit_info) * max_num_units));
+
RestUnitSearchInfo *rusi;
CHECK_MEM_ERROR(
cm, rusi,
- (RestUnitSearchInfo *)aom_memalign(16, sizeof(*rusi) * ntiles[0]));
+ (RestUnitSearchInfo *)aom_memalign(16, sizeof(*rusi) * max_num_units));
// If the restoration unit dimensions are not multiples of
// rsi->restoration_unit_size then some elements of the rusi array may be
// left uninitialised when we reach copy_unit_info(...). This is not a
// problem, as these elements are ignored later, but in order to quiet
// Valgrind's warnings we initialise the array below.
- memset(rusi, 0, sizeof(*rusi) * ntiles[0]);
+ memset(rusi, 0, sizeof(*rusi) * max_num_units);
+
+ return rusi;
+}
+
+static void set_restoration_unit_size(AV1_COMMON *cm, RestorationInfo *rsi,
+ int is_uv, int luma_unit_size) {
+#if COUPLED_CHROMA_FROM_LUMA_RESTORATION
+ int sx = cm->seq_params.subsampling_x;
+ int sy = cm->seq_params.subsampling_y;
+ int s = (p > 0) ? AOMMIN(sx, sy) : 0;
+#else
+ int s = 0;
+#endif // !COUPLED_CHROMA_FROM_LUMA_RESTORATION
+ int unit_size = luma_unit_size >> s;
+
+ int plane_w, plane_h;
+ av1_get_upsampled_plane_size(cm, is_uv, &plane_w, &plane_h);
+
+ const int horz_units = av1_lr_count_units(unit_size, plane_w);
+ const int vert_units = av1_lr_count_units(unit_size, plane_h);
+
+ rsi->restoration_unit_size = unit_size;
+ rsi->num_rest_units = horz_units * vert_units;
+ rsi->horz_units = horz_units;
+ rsi->vert_units = vert_units;
+}
+
+void av1_pick_filter_restoration(const YV12_BUFFER_CONFIG *src, AV1_COMP *cpi) {
+ AV1_COMMON *const cm = &cpi->common;
+ MACROBLOCK *const x = &cpi->td.mb;
+ const SequenceHeader *const seq_params = cm->seq_params;
+ const LOOP_FILTER_SPEED_FEATURES *lpf_sf = &cpi->sf.lpf_sf;
+ const int num_planes = av1_num_planes(cm);
+ const int highbd = cm->seq_params->use_highbitdepth;
+ assert(!cm->features.all_lossless);
+
+ av1_fill_lr_rates(&x->mode_costs, x->e_mbd.tile_ctx);
+
+ // Select unit size based on speed feature settings, and allocate
+ // rui structs based on this size
+ int min_lr_unit_size = cpi->sf.lpf_sf.min_lr_unit_size;
+ int max_lr_unit_size = cpi->sf.lpf_sf.max_lr_unit_size;
+
+ // The minimum allowed unit size at a syntax level is 1 superblock.
+ // Apply this constraint here so that the speed features code which sets
+ // cpi->sf.lpf_sf.min_lr_unit_size does not need to know the superblock size
+ min_lr_unit_size =
+ AOMMAX(min_lr_unit_size, block_size_wide[cm->seq_params->sb_size]);
+
+ for (int plane = 0; plane < num_planes; ++plane) {
+ cpi->pick_lr_ctxt.rusi[plane] = allocate_search_structs(
+ cm, &cm->rst_info[plane], plane > 0, min_lr_unit_size);
+ }
+
x->rdmult = cpi->rd.RDMULT;
// Allocate the frame buffer trial_frame_rst, which is used to temporarily
@@ -1865,33 +2049,32 @@
if (aom_realloc_frame_buffer(
&cpi->trial_frame_rst, cm->superres_upscaled_width,
cm->superres_upscaled_height, seq_params->subsampling_x,
- seq_params->subsampling_y, seq_params->use_highbitdepth,
- AOM_RESTORATION_FRAME_BORDER, cm->features.byte_alignment, NULL, NULL,
- NULL, 0, 0))
+ seq_params->subsampling_y, highbd, AOM_RESTORATION_FRAME_BORDER,
+ cm->features.byte_alignment, NULL, NULL, NULL, 0, 0))
aom_internal_error(cm->error, AOM_CODEC_MEM_ERROR,
"Failed to allocate trial restored frame buffer");
RestSearchCtxt rsc;
// The buffers 'src_avg' and 'dgd_avg' are used to compute H and M buffers.
- // These buffers are required for AVX2 SIMD purpose only. Hence, allocated the
- // same if AVX2 variant of SIMD for av1_compute_stats() is enabled. The buffer
- // size required is calculated based on maximum width and height of the LRU
- // (i.e., from foreach_rest_unit_in_tile() 1.5 times the
- // RESTORATION_UNITSIZE_MAX) allowed for Wiener filtering. The width and
- // height aligned to multiple of 16 is considered for intrinsic purpose.
+ // These buffers are only required for the AVX2 and NEON implementations of
+ // av1_compute_stats. The buffer size required is calculated based on maximum
+ // width and height of the LRU (i.e., from foreach_rest_unit_in_plane() 1.5
+ // times the RESTORATION_UNITSIZE_MAX) allowed for Wiener filtering. The width
+ // and height aligned to multiple of 16 is considered for intrinsic purpose.
rsc.dgd_avg = NULL;
rsc.src_avg = NULL;
-#if HAVE_AVX2
+#if HAVE_AVX2 || HAVE_NEON
// The buffers allocated below are used during Wiener filter processing of low
// bitdepth path. Hence, allocate the same when Wiener filter is enabled in
// low bitdepth path.
- if (!cpi->sf.lpf_sf.disable_wiener_filter &&
- !cm->seq_params->use_highbitdepth) {
- const int buf_size = sizeof(*rsc.dgd_avg) * 6 * RESTORATION_UNITSIZE_MAX *
- RESTORATION_UNITSIZE_MAX;
- CHECK_MEM_ERROR(cm, rsc.dgd_avg, (int16_t *)aom_memalign(32, buf_size));
+ if (!cpi->sf.lpf_sf.disable_wiener_filter && !highbd) {
+ const int buf_size = sizeof(*cpi->pick_lr_ctxt.dgd_avg) * 6 *
+ RESTORATION_UNITSIZE_MAX * RESTORATION_UNITSIZE_MAX;
+ CHECK_MEM_ERROR(cm, cpi->pick_lr_ctxt.dgd_avg,
+ (int16_t *)aom_memalign(32, buf_size));
+ rsc.dgd_avg = cpi->pick_lr_ctxt.dgd_avg;
// When LRU width isn't multiple of 16, the 256 bits load instruction used
// in AVX2 intrinsic can read data beyond valid LRU. Hence, in order to
// silence Valgrind warning this buffer is initialized with zero. Overhead
@@ -1904,59 +2087,131 @@
}
#endif
- const int plane_start = AOM_PLANE_Y;
- const int plane_end = num_planes > 1 ? AOM_PLANE_V : AOM_PLANE_Y;
+ // Initialize all planes, so that any planes we skip searching will still have
+ // valid data
+ for (int plane = 0; plane < num_planes; plane++) {
+ cm->rst_info[plane].frame_restoration_type = RESTORE_NONE;
+ }
+
+ // Decide which planes to search
+ int plane_start, plane_end;
+
+ if (lpf_sf->disable_loop_restoration_luma) {
+ plane_start = AOM_PLANE_U;
+ } else {
+ plane_start = AOM_PLANE_Y;
+ }
+
+ if (num_planes == 1 || lpf_sf->disable_loop_restoration_chroma) {
+ plane_end = AOM_PLANE_Y;
+ } else {
+ plane_end = AOM_PLANE_V;
+ }
// Derive the flags to enable/disable Loop restoration filters based on the
// speed features 'disable_wiener_filter' and 'disable_sgr_filter'.
bool disable_lr_filter[RESTORE_TYPES] = { false };
- const LOOP_FILTER_SPEED_FEATURES *lpf_sf = &cpi->sf.lpf_sf;
av1_derive_flags_for_lr_processing(lpf_sf, disable_lr_filter);
- for (int plane = plane_start; plane <= plane_end; ++plane) {
- init_rsc(src, &cpi->common, x, lpf_sf, plane, rusi, &cpi->trial_frame_rst,
- &rsc);
+ for (int plane = plane_start; plane <= plane_end; plane++) {
+ const YV12_BUFFER_CONFIG *dgd = &cm->cur_frame->buf;
+ const int is_uv = plane != AOM_PLANE_Y;
+ int plane_w, plane_h;
+ av1_get_upsampled_plane_size(cm, is_uv, &plane_w, &plane_h);
+ av1_extend_frame(dgd->buffers[plane], plane_w, plane_h, dgd->strides[is_uv],
+ RESTORATION_BORDER, RESTORATION_BORDER, highbd);
+ }
- const int plane_ntiles = ntiles[plane > 0];
- const RestorationType num_rtypes =
- (plane_ntiles > 1) ? RESTORE_TYPES : RESTORE_SWITCHABLE_TYPES;
+ double best_cost = DBL_MAX;
+ int best_luma_unit_size = max_lr_unit_size;
+ for (int luma_unit_size = max_lr_unit_size;
+ luma_unit_size >= min_lr_unit_size; luma_unit_size >>= 1) {
+ int64_t bits_this_size = 0;
+ int64_t sse_this_size = 0;
+ RestorationType best_rtype[MAX_MB_PLANE] = { RESTORE_NONE, RESTORE_NONE,
+ RESTORE_NONE };
+ for (int plane = plane_start; plane <= plane_end; ++plane) {
+ set_restoration_unit_size(cm, &cm->rst_info[plane], plane > 0,
+ luma_unit_size);
+ init_rsc(src, &cpi->common, x, lpf_sf, plane,
+ cpi->pick_lr_ctxt.rusi[plane], &cpi->trial_frame_rst, &rsc);
- double best_cost = 0;
- RestorationType best_rtype = RESTORE_NONE;
+ restoration_search(cm, plane, &rsc, disable_lr_filter);
- const int highbd = rsc.cm->seq_params->use_highbitdepth;
- if ((plane && !lpf_sf->disable_loop_restoration_chroma) ||
- (!plane && !lpf_sf->disable_loop_restoration_luma)) {
- av1_extend_frame(rsc.dgd_buffer, rsc.plane_width, rsc.plane_height,
- rsc.dgd_stride, RESTORATION_BORDER, RESTORATION_BORDER,
- highbd);
-
+ const int plane_num_units = cm->rst_info[plane].num_rest_units;
+ const RestorationType num_rtypes =
+ (plane_num_units > 1) ? RESTORE_TYPES : RESTORE_SWITCHABLE_TYPES;
+ double best_cost_this_plane = DBL_MAX;
for (RestorationType r = 0; r < num_rtypes; ++r) {
// Disable Loop restoration filter based on the flags set using speed
// feature 'disable_wiener_filter' and 'disable_sgr_filter'.
if (disable_lr_filter[r]) continue;
- double cost = search_rest_type(&rsc, r);
+ double cost_this_plane = RDCOST_DBL_WITH_NATIVE_BD_DIST(
+ x->rdmult, rsc.total_bits[r] >> 4, rsc.total_sse[r],
+ cm->seq_params->bit_depth);
- if (r == 0 || cost < best_cost) {
- best_cost = cost;
- best_rtype = r;
+ if (cost_this_plane < best_cost_this_plane) {
+ best_cost_this_plane = cost_this_plane;
+ best_rtype[plane] = r;
}
}
+
+ bits_this_size += rsc.total_bits[best_rtype[plane]];
+ sse_this_size += rsc.total_sse[best_rtype[plane]];
}
- cm->rst_info[plane].frame_restoration_type = best_rtype;
- if (best_rtype != RESTORE_NONE) {
- for (int u = 0; u < plane_ntiles; ++u) {
- copy_unit_info(best_rtype, &rusi[u], &cm->rst_info[plane].unit_info[u]);
+ double cost_this_size = RDCOST_DBL_WITH_NATIVE_BD_DIST(
+ x->rdmult, bits_this_size >> 4, sse_this_size,
+ cm->seq_params->bit_depth);
+
+ if (cost_this_size < best_cost) {
+ best_cost = cost_this_size;
+ best_luma_unit_size = luma_unit_size;
+ // Copy parameters out of rusi struct, before we overwrite it at
+ // the start of the next iteration
+ bool all_none = true;
+ for (int plane = plane_start; plane <= plane_end; ++plane) {
+ cm->rst_info[plane].frame_restoration_type = best_rtype[plane];
+ if (best_rtype[plane] != RESTORE_NONE) {
+ all_none = false;
+ const int plane_num_units = cm->rst_info[plane].num_rest_units;
+ for (int u = 0; u < plane_num_units; ++u) {
+ copy_unit_info(best_rtype[plane], &cpi->pick_lr_ctxt.rusi[plane][u],
+ &cm->rst_info[plane].unit_info[u]);
+ }
+ }
}
+ // Heuristic: If all best_rtype entries are RESTORE_NONE, this means we
+ // couldn't find any good filters at this size. So we likely won't find
+ // any good filters at a smaller size either, so skip
+ if (all_none) {
+ break;
+ }
+ } else {
+ // Heuristic: If this size is worse than the previous (larger) size, then
+ // the next size down will likely be even worse, so skip
+ break;
}
}
-#if HAVE_AVX2
- if (!cpi->sf.lpf_sf.disable_wiener_filter &&
- !cm->seq_params->use_highbitdepth) {
- aom_free(rsc.dgd_avg);
+
+ // Final fixup to set the correct unit size
+ // We set this for all planes, even ones we have skipped searching,
+ // so that other code does not need to care which planes were and weren't
+ // searched
+ for (int plane = 0; plane < num_planes; ++plane) {
+ set_restoration_unit_size(cm, &cm->rst_info[plane], plane > 0,
+ best_luma_unit_size);
+ }
+
+#if HAVE_AVX || HAVE_NEON
+ if (!cpi->sf.lpf_sf.disable_wiener_filter && !highbd) {
+ aom_free(cpi->pick_lr_ctxt.dgd_avg);
+ cpi->pick_lr_ctxt.dgd_avg = NULL;
}
#endif
- aom_free(rusi);
+ for (int plane = 0; plane < num_planes; plane++) {
+ aom_free(cpi->pick_lr_ctxt.rusi[plane]);
+ cpi->pick_lr_ctxt.rusi[plane] = NULL;
+ }
}
diff --git a/av1/encoder/pickrst.h b/av1/encoder/pickrst.h
index 94a6932..d1d0b0c 100644
--- a/av1/encoder/pickrst.h
+++ b/av1/encoder/pickrst.h
@@ -20,6 +20,34 @@
struct yv12_buffer_config;
struct AV1_COMP;
+// Enable extra debugging for loop restoration costing?
+//
+// If this is set to 1, then we record not just the selected LR parameters, but
+// also the values which the search process thinks they should be delta-coded
+// against. Then, when writing out the bitstream, we verify this information,
+// to help ensure that the search code is costing things properly
+#define DEBUG_LR_COSTING 0
+
+#if DEBUG_LR_COSTING
+#define MAX_LR_UNITS_W 64
+#define MAX_LR_UNITS_H 64
+
+// Storage for reference parameters.
+//
+// The storage size is determined by:
+// * This is always written and then checked within the same frame encode pass,
+// so we do not need to buffer multiple frames of data
+// * The parameters can be different per plane within one frame
+// * The relevant set of ref parameters can differ between the search where
+// we set the frame restoration mode to RESTORE_WIENER, and the search where
+// we set it to RESTORE_SWITCHABLE.
+// So we need to store at least two sets of Wiener params and two sets of
+// SGR params, and the easiest way to do this is to index by
+// frame_restoration_type
+extern RestorationUnitInfo lr_ref_params[RESTORE_TYPES][MAX_MB_PLANE]
+ [MAX_LR_UNITS_W * MAX_LR_UNITS_H];
+#endif // DEBUG_LR_COSTING
+
static const uint8_t g_shuffle_stats_data[16] = {
0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8,
};
diff --git a/av1/encoder/ratectrl.c b/av1/encoder/ratectrl.c
index fdf1495..de0f596 100644
--- a/av1/encoder/ratectrl.c
+++ b/av1/encoder/ratectrl.c
@@ -334,7 +334,7 @@
double framerate) {
// Assume we do not need any constraint lower than 4K 20 fps
static const double factor_safe = 3840 * 2160 * 20.0;
- const double factor = width * height * framerate;
+ const double factor = (double)width * height * framerate;
const int default_interval =
clamp((int)(framerate * 0.125), MIN_GF_INTERVAL, MAX_GF_INTERVAL);
@@ -453,12 +453,19 @@
#else
int64_t buffer_level = p_rc->buffer_level;
#endif
-
- if (!oxcf->rc_cfg.drop_frames_water_mark) {
+ // Never drop on key frame, or for frame whose base layer is key.
+ // If drop_count_consec hits or exceeds max_consec_drop then don't drop.
+ if (cpi->common.current_frame.frame_type == KEY_FRAME ||
+ (cpi->ppi->use_svc &&
+ cpi->svc.layer_context[cpi->svc.temporal_layer_id].is_key_frame) ||
+ !oxcf->rc_cfg.drop_frames_water_mark ||
+ (rc->max_consec_drop > 0 &&
+ rc->drop_count_consec >= rc->max_consec_drop)) {
return 0;
} else {
if (buffer_level < 0) {
// Always drop if buffer is below 0.
+ rc->drop_count_consec++;
return 1;
} else {
// If buffer is below drop_mark, for now just drop every other frame
@@ -473,6 +480,7 @@
if (rc->decimation_factor > 0) {
if (rc->decimation_count > 0) {
--rc->decimation_count;
+ rc->drop_count_consec++;
return 1;
} else {
rc->decimation_count = rc->decimation_factor;
@@ -493,8 +501,16 @@
const AV1_COMMON *const cm = &cpi->common;
const SVC *const svc = &cpi->svc;
const RefreshFrameInfo *const refresh_frame = &cpi->refresh_frame;
+ // Flag to indicate previous frame has overshoot, and buffer level
+ // for current frame is low (less than ~half of optimal). For such
+ // (inter) frames, if the source_sad is non-zero, relax the max_delta_up
+ // and clamp applied below.
+ const bool overshoot_buffer_low =
+ cpi->rc.rc_1_frame == -1 && rc->frame_source_sad > 1000 &&
+ p_rc->buffer_level < (p_rc->optimal_buffer_level >> 1) &&
+ rc->frames_since_key > 4;
int max_delta_down;
- int max_delta_up = 20;
+ int max_delta_up = overshoot_buffer_low ? 60 : 20;
const int change_avg_frame_bandwidth =
abs(rc->avg_frame_bandwidth - rc->prev_avg_frame_bandwidth) >
0.1 * (rc->avg_frame_bandwidth);
@@ -543,7 +559,7 @@
// not been set due to dropped frames.
if (rc->rc_1_frame * rc->rc_2_frame == -1 &&
rc->q_1_frame != rc->q_2_frame && rc->q_1_frame > 0 &&
- rc->q_2_frame > 0) {
+ rc->q_2_frame > 0 && !overshoot_buffer_low) {
int qclamp = clamp(q, AOMMIN(rc->q_1_frame, rc->q_2_frame),
AOMMAX(rc->q_1_frame, rc->q_2_frame));
// If the previous frame had overshoot and the current q needs to
@@ -2343,6 +2359,7 @@
rc->prev_coded_height = cm->height;
rc->frame_number_encoded++;
rc->prev_frame_is_dropped = 0;
+ rc->drop_count_consec = 0;
// if (current_frame->frame_number == 1 && cm->show_frame)
/*
rc->this_frame_target =
@@ -2957,10 +2974,8 @@
}
if (width != cm->render_width || height != cm->render_height ||
unscaled_src == NULL || unscaled_last_src == NULL) {
- if (cpi->src_sad_blk_64x64) {
- aom_free(cpi->src_sad_blk_64x64);
- cpi->src_sad_blk_64x64 = NULL;
- }
+ aom_free(cpi->src_sad_blk_64x64);
+ cpi->src_sad_blk_64x64 = NULL;
}
if (unscaled_src == NULL || unscaled_last_src == NULL) return;
src_y = unscaled_src->y_buffer;
@@ -2972,10 +2987,8 @@
last_src_width = unscaled_last_src->y_width;
last_src_height = unscaled_last_src->y_height;
if (src_width != last_src_width || src_height != last_src_height) {
- if (cpi->src_sad_blk_64x64) {
- aom_free(cpi->src_sad_blk_64x64);
- cpi->src_sad_blk_64x64 = NULL;
- }
+ aom_free(cpi->src_sad_blk_64x64);
+ cpi->src_sad_blk_64x64 = NULL;
return;
}
rc->high_source_sad = 0;
@@ -2990,13 +3003,18 @@
}
int num_zero_temp_sad = 0;
uint32_t min_thresh = 10000;
- if (cpi->oxcf.tune_cfg.content != AOM_CONTENT_SCREEN) min_thresh = 100000;
+ if (cpi->oxcf.tune_cfg.content != AOM_CONTENT_SCREEN) {
+ min_thresh = cm->width * cm->height <= 320 * 240 && cpi->framerate < 10.0
+ ? 50000
+ : 100000;
+ }
const BLOCK_SIZE bsize = BLOCK_64X64;
// Loop over sub-sample of frame, compute average sad over 64x64 blocks.
uint64_t avg_sad = 0;
uint64_t tmp_sad = 0;
int num_samples = 0;
- const int thresh = 6;
+ const int thresh =
+ cm->width * cm->height <= 320 * 240 && cpi->framerate < 10.0 ? 5 : 6;
// SAD is computed on 64x64 blocks
const int sb_size_by_mb = (cm->seq_params->sb_size == BLOCK_128X128)
? (cm->seq_params->mib_size >> 1)
@@ -3127,6 +3145,10 @@
int qindex;
double tot_scale_change = (double)(resize_width * resize_height) /
(double)(prev_width * prev_height);
+ // Disable the skip mv search for svc on resize frame.
+ svc->skip_mvsearch_last = 0;
+ svc->skip_mvsearch_gf = 0;
+ svc->skip_mvsearch_altref = 0;
// Reset buffer level to optimal, update target size.
p_rc->buffer_level = p_rc->optimal_buffer_level;
p_rc->bits_off_target = p_rc->optimal_buffer_level;
@@ -3386,7 +3408,7 @@
if (rc->prev_coded_width == cm->width &&
rc->prev_coded_height == cm->height) {
rc_scene_detection_onepass_rt(cpi, frame_input);
- } else if (cpi->src_sad_blk_64x64) {
+ } else {
aom_free(cpi->src_sad_blk_64x64);
cpi->src_sad_blk_64x64 = NULL;
}
diff --git a/av1/encoder/ratectrl.h b/av1/encoder/ratectrl.h
index 4fb1179..6802ad4 100644
--- a/av1/encoder/ratectrl.h
+++ b/av1/encoder/ratectrl.h
@@ -205,6 +205,8 @@
int decimation_factor;
int decimation_count;
int prev_frame_is_dropped;
+ int drop_count_consec;
+ int max_consec_drop;
/*!
* Frame number for encoded frames (non-dropped).
diff --git a/av1/encoder/rd.c b/av1/encoder/rd.c
index 8bc7d1b..c2d76e7 100644
--- a/av1/encoder/rd.c
+++ b/av1/encoder/rd.c
@@ -809,10 +809,11 @@
// Frame level dv cost update
if (av1_need_dv_costs(cpi)) {
- if (cpi->td.mb.dv_costs == NULL) {
+ if (cpi->td.dv_costs_alloc == NULL) {
CHECK_MEM_ERROR(
- cm, cpi->td.mb.dv_costs,
- (IntraBCMVCosts *)aom_malloc(sizeof(*cpi->td.mb.dv_costs)));
+ cm, cpi->td.dv_costs_alloc,
+ (IntraBCMVCosts *)aom_malloc(sizeof(*cpi->td.dv_costs_alloc)));
+ cpi->td.mb.dv_costs = cpi->td.dv_costs_alloc;
}
av1_fill_dv_costs(&cm->fc->ndvc, x->dv_costs);
}
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 8620087..c17fbcc 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -1322,7 +1322,8 @@
const int mi_col = xd->mi_col;
int mode_index_start, mode_index_end;
const int txfm_rd_gate_level =
- get_txfm_rd_gate_level(cpi->sf.inter_sf.txfm_rd_gate_level, bsize,
+ get_txfm_rd_gate_level(cm->seq_params->enable_masked_compound,
+ cpi->sf.inter_sf.txfm_rd_gate_level, bsize,
TX_SEARCH_MOTION_MODE, eval_motion_mode);
// Modify the start and end index according to speed features. For example,
@@ -1656,16 +1657,16 @@
// Call av1_enc_build_inter_predictor() for one plane at a time.
av1_enc_build_inter_predictor(cm, xd, mi_row, mi_col, orig_dst, bsize,
plane, plane);
- const struct macroblock_plane *const p = &x->plane[plane];
const struct macroblockd_plane *const pd = &xd->plane[plane];
const BLOCK_SIZE plane_bsize =
get_plane_block_size(bsize, pd->subsampling_x, pd->subsampling_y);
- const int bw = block_size_wide[plane_bsize];
- const int bh = block_size_high[plane_bsize];
av1_subtract_plane(x, plane_bsize, plane);
- int64_t sse = aom_sum_squares_2d_i16(p->src_diff, bw, bw, bh) << 4;
- sse >>= ((cpi->frame_info.bit_depth - 8) * 2);
+
+ int64_t sse =
+ av1_pixel_diff_dist(x, plane, 0, 0, plane_bsize, plane_bsize, NULL);
+ if (is_cur_buf_hbd(xd)) sse = ROUND_POWER_OF_TWO(sse, (xd->bd - 8) * 2);
+ sse <<= 4;
total_sse += sse;
// When current rd cost is more than the best rd, skip evaluation of
// remaining planes.
@@ -2055,6 +2056,9 @@
HandleInterModeArgs *const args,
int64_t ref_best_rd, BLOCK_SIZE bsize,
const int ref_set) {
+ // If the number of ref mv count is equal to 1, do not prune the same. It
+ // is better to evaluate the same than to prune it.
+ if (ref_set == 1) return 1;
AV1_COMMON *const cm = &cpi->common;
const MACROBLOCKD *const xd = &x->e_mbd;
const MB_MODE_INFO *const mbmi = xd->mi[0];
@@ -2833,17 +2837,6 @@
const int base_rate =
args->ref_frame_cost + args->single_comp_cost + ref_mv_cost;
- // As per the experiments, in real-time preset impact of model rd based
- // breakouts is less on encoding time if the following conditions are true.
- // (1) compound mode is disabled
- // (2) interpolation filter search is disabled
- // TODO(any): Check the impact of model rd based breakouts in other presets
- const int skip_interp_search_modelrd_calc =
- cpi->oxcf.mode == REALTIME &&
- cm->current_frame.reference_mode == SINGLE_REFERENCE &&
- (cpi->sf.rt_sf.skip_interp_filter_search ||
- cpi->sf.winner_mode_sf.winner_mode_ifs);
-
for (i = 0; i < MAX_REF_MV_SEARCH - 1; ++i) {
save_mv[i][0].as_int = INVALID_MV;
save_mv[i][1].as_int = INVALID_MV;
@@ -2993,7 +2986,7 @@
if (not_best_mode) continue;
}
- if (!skip_interp_search_modelrd_calc) {
+ if (!args->skip_ifs) {
#if CONFIG_COLLECT_COMPONENT_TIMING
start_timing(cpi, interpolation_filter_search_time);
#endif
@@ -3125,7 +3118,7 @@
int64_t best_rd) {
const AV1_COMMON *const cm = &cpi->common;
if (!av1_allow_intrabc(cm) || !cpi->oxcf.kf_cfg.enable_intrabc ||
- cpi->sf.rt_sf.use_nonrd_pick_mode)
+ !cpi->sf.mv_sf.use_intrabc || cpi->sf.rt_sf.use_nonrd_pick_mode)
return INT64_MAX;
const int num_planes = av1_num_planes(cm);
@@ -3188,12 +3181,14 @@
av1_copy_array(best_tx_type_map, xd->tx_type_map, ctx->num_4x4_blk);
FULLPEL_MOTION_SEARCH_PARAMS fullms_params;
+ const SEARCH_METHODS search_method =
+ av1_get_default_mv_search_method(x, &cpi->sf.mv_sf, bsize);
const search_site_config *lookahead_search_sites =
cpi->mv_search_params.search_site_cfg[SS_CFG_LOOKAHEAD];
const FULLPEL_MV start_mv = get_fullmv_from_mv(&dv_ref.as_mv);
av1_make_default_fullpel_ms_params(&fullms_params, cpi, x, bsize,
&dv_ref.as_mv, start_mv,
- lookahead_search_sites,
+ lookahead_search_sites, search_method,
/*fine_search_interval=*/0);
const IntraBCMVCosts *const dv_costs = x->dv_costs;
av1_set_ms_to_intra_mode(&fullms_params, dv_costs);
@@ -3242,9 +3237,11 @@
const int step_param = cpi->mv_search_params.mv_step_param;
IntraBCHashInfo *intrabc_hash_info = &x->intrabc_hash_info;
int_mv best_mv, best_hash_mv;
+ FULLPEL_MV_STATS best_mv_stats;
- int bestsme = av1_full_pixel_search(start_mv, &fullms_params, step_param,
- NULL, &best_mv.as_fullmv, NULL);
+ int bestsme =
+ av1_full_pixel_search(start_mv, &fullms_params, step_param, NULL,
+ &best_mv.as_fullmv, &best_mv_stats, NULL);
const int hashsme = av1_intrabc_hash_search(
cpi, xd, &fullms_params, intrabc_hash_info, &best_hash_mv.as_fullmv);
if (hashsme < bestsme) {
@@ -3780,6 +3777,7 @@
MB_MODE_INFO *const mbmi = xd->mi[0];
unsigned char segment_id = mbmi->segment_id;
const SPEED_FEATURES *const sf = &cpi->sf;
+ const INTER_MODE_SPEED_FEATURES *const inter_sf = &sf->inter_sf;
REF_SET ref_set = REF_SET_FULL;
if (sf->rt_sf.use_real_time_ref_set)
@@ -3852,7 +3850,7 @@
}
if (cpi->rc.is_src_frame_alt_ref) {
- if (sf->inter_sf.alt_ref_search_fp &&
+ if (inter_sf->alt_ref_search_fp &&
(cpi->ref_frame_flags & av1_ref_frame_flag_list[ALTREF_FRAME])) {
mask->pred_modes[ALTREF_FRAME] = 0;
disable_inter_references_except_altref(mask->ref_combo);
@@ -3860,19 +3858,19 @@
}
}
- if (sf->inter_sf.alt_ref_search_fp) {
+ if (inter_sf->alt_ref_search_fp) {
if (!cm->show_frame && x->best_pred_mv_sad[0] < INT_MAX) {
int sad_thresh = x->best_pred_mv_sad[0] + (x->best_pred_mv_sad[0] >> 3);
// Conservatively skip the modes w.r.t. BWDREF, ALTREF2 and ALTREF, if
// those are past frames
MV_REFERENCE_FRAME start_frame =
- sf->inter_sf.alt_ref_search_fp == 1 ? ALTREF2_FRAME : BWDREF_FRAME;
+ inter_sf->alt_ref_search_fp == 1 ? ALTREF2_FRAME : BWDREF_FRAME;
for (ref_frame = start_frame; ref_frame <= ALTREF_FRAME; ref_frame++) {
if (cpi->ref_frame_dist_info.ref_relative_dist[ref_frame - LAST_FRAME] <
0) {
// Prune inter modes when relative dist of ALTREF2 and ALTREF is close
// to the relative dist of LAST_FRAME.
- if (sf->inter_sf.alt_ref_search_fp == 1 &&
+ if (inter_sf->alt_ref_search_fp == 1 &&
(abs(cpi->ref_frame_dist_info
.ref_relative_dist[ref_frame - LAST_FRAME]) >
1.5 * abs(cpi->ref_frame_dist_info
@@ -3913,6 +3911,33 @@
mask->pred_modes[INTRA_FRAME] |=
~(uint32_t)sf->intra_sf.intra_y_mode_mask[max_txsize_lookup[bsize]];
+
+ // Prune reference frames which are not the closest to the current
+ // frame and with large pred_mv_sad.
+ if (inter_sf->prune_single_ref) {
+ assert(inter_sf->prune_single_ref > 0 && inter_sf->prune_single_ref < 3);
+ const double prune_threshes[2] = { 1.20, 1.05 };
+
+ for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
+ const RefFrameDistanceInfo *const ref_frame_dist_info =
+ &cpi->ref_frame_dist_info;
+ const int is_closest_ref =
+ (ref_frame == ref_frame_dist_info->nearest_past_ref) ||
+ (ref_frame == ref_frame_dist_info->nearest_future_ref);
+
+ if (!is_closest_ref) {
+ const int dir =
+ (ref_frame_dist_info->ref_relative_dist[ref_frame - LAST_FRAME] < 0)
+ ? 0
+ : 1;
+ if (x->best_pred_mv_sad[dir] < INT_MAX &&
+ x->pred_mv_sad[ref_frame] >
+ prune_threshes[inter_sf->prune_single_ref - 1] *
+ x->best_pred_mv_sad[dir])
+ mask->pred_modes[ref_frame] |= INTER_SINGLE_ALL;
+ }
+ }
+ }
}
static AOM_INLINE void init_neighbor_pred_buf(
@@ -4025,6 +4050,7 @@
setup_buffer_ref_mvs_inter(cpi, x, ref_frame, bsize, yv12_mb);
}
if (cpi->sf.inter_sf.alt_ref_search_fp ||
+ cpi->sf.inter_sf.prune_single_ref ||
cpi->sf.rt_sf.prune_inter_modes_wrt_gf_arf_based_on_sad) {
// Store the best pred_mv_sad across all past frames
if (cpi->ref_frame_dist_info.ref_relative_dist[ref_frame - LAST_FRAME] <
@@ -4995,8 +5021,16 @@
*(args->skip_motion_mode) = (ret == 2);
// We've reached the first compound prediction mode, get stats from the
- // single reference predictors to help with pruning
- if (sf->inter_sf.prune_comp_search_by_single_result > 0 && comp_pred &&
+ // single reference predictors to help with pruning.
+ // Disable this pruning logic if interpolation filter search was skipped for
+ // single prediction modes as it can result in aggressive pruning of compound
+ // prediction modes due to the absence of modelled_rd populated by
+ // av1_interpolation_filter_search().
+ // TODO(Remya): Check the impact of the sf
+ // 'prune_comp_search_by_single_result' if compound prediction modes are
+ // enabled in future for REALTIME encode.
+ if (!sf->interp_sf.skip_interp_filter_search &&
+ sf->inter_sf.prune_comp_search_by_single_result > 0 && comp_pred &&
args->reach_first_comp_mode == 0) {
analyze_single_states(cpi, args->search_state);
args->reach_first_comp_mode = 1;
@@ -5014,7 +5048,8 @@
// Skip this compound mode based on the RD results from the single prediction
// modes
- if (sf->inter_sf.prune_comp_search_by_single_result > 0 && comp_pred) {
+ if (!sf->interp_sf.skip_interp_filter_search &&
+ sf->inter_sf.prune_comp_search_by_single_result > 0 && comp_pred) {
if (compound_skip_by_single_states(cpi, args->search_state, this_mode,
ref_frame, second_ref_frame, x))
return 1;
@@ -5203,6 +5238,7 @@
const int mode_rate = inter_modes_info->mode_rate_arr[data_idx];
int64_t skip_rd = INT64_MAX;
const int txfm_rd_gate_level = get_txfm_rd_gate_level(
+ cm->seq_params->enable_masked_compound,
cpi->sf.inter_sf.txfm_rd_gate_level, bsize, TX_SEARCH_DEFAULT,
/*eval_motion_mode=*/0);
if (txfm_rd_gate_level) {
@@ -5665,6 +5701,20 @@
}
}
+static AOM_INLINE bool skip_interp_filter_search(const AV1_COMP *cpi,
+ int is_single_pred) {
+ const MODE encoding_mode = cpi->oxcf.mode;
+ if (encoding_mode == REALTIME) {
+ return (cpi->common.current_frame.reference_mode == SINGLE_REFERENCE &&
+ (cpi->sf.interp_sf.skip_interp_filter_search ||
+ cpi->sf.winner_mode_sf.winner_mode_ifs));
+ } else if (encoding_mode == GOOD) {
+ // Skip interpolation filter search for single prediction modes.
+ return (cpi->sf.interp_sf.skip_interp_filter_search && is_single_pred);
+ }
+ return false;
+}
+
static AOM_INLINE int get_block_temp_var(const AV1_COMP *cpi,
const MACROBLOCK *x,
BLOCK_SIZE bsize) {
@@ -5727,6 +5777,7 @@
INT_MAX,
search_state.simple_rd,
0,
+ false,
interintra_modes,
{ { { 0 }, { { 0 } }, { 0 }, 0, 0, 0, 0 } },
{ { 0, 0 } },
@@ -5960,6 +6011,7 @@
args.single_comp_cost = real_compmode_cost;
args.ref_frame_cost = ref_frame_cost;
args.best_pred_sse = search_state.best_pred_sse;
+ args.skip_ifs = skip_interp_filter_search(cpi, is_single_pred);
int64_t skip_rd[2] = { search_state.best_skip_rd[0],
search_state.best_skip_rd[1] };
@@ -5976,7 +6028,8 @@
end_timing(cpi, handle_inter_mode_time);
#endif
if (current_frame->reference_mode != SINGLE_REFERENCE) {
- if (sf->inter_sf.prune_comp_search_by_single_result > 0 &&
+ if (!args.skip_ifs &&
+ sf->inter_sf.prune_comp_search_by_single_result > 0 &&
is_inter_singleref_mode(this_mode)) {
collect_single_states(x, &search_state, mbmi);
}
@@ -6007,6 +6060,10 @@
update_search_state(&search_state, rd_cost, ctx, &rd_stats, &rd_stats_y,
&rd_stats_uv, mode_enum, x, do_tx_search);
if (do_tx_search) search_state.best_skip_rd[0] = skip_rd[0];
+ // skip_rd[0] is the best total rd for a skip mode so far.
+ // skip_rd[1] is the best total rd for a skip mode so far in luma.
+ // When do_tx_search = 1, both skip_rd[0] and skip_rd[1] are updated.
+ // When do_tx_search = 0, skip_rd[1] is updated.
search_state.best_skip_rd[1] = skip_rd[1];
}
if (sf->winner_mode_sf.motion_mode_for_winner_cand) {
diff --git a/av1/encoder/rdopt_utils.h b/av1/encoder/rdopt_utils.h
index 1c5b3db..b6bc492 100644
--- a/av1/encoder/rdopt_utils.h
+++ b/av1/encoder/rdopt_utils.h
@@ -774,12 +774,18 @@
// Get transform rd gate level for the given transform search case.
static INLINE int get_txfm_rd_gate_level(
+ const int is_masked_compound_enabled,
const int txfm_rd_gate_level[TX_SEARCH_CASES], BLOCK_SIZE bsize,
TX_SEARCH_CASE tx_search_case, int eval_motion_mode) {
assert(tx_search_case < TX_SEARCH_CASES);
if (tx_search_case == TX_SEARCH_MOTION_MODE && !eval_motion_mode &&
num_pels_log2_lookup[bsize] > 8)
return txfm_rd_gate_level[TX_SEARCH_MOTION_MODE];
+ // Enable aggressive gating of transform search only when masked compound type
+ // is enabled.
+ else if (tx_search_case == TX_SEARCH_COMP_TYPE_MODE &&
+ is_masked_compound_enabled)
+ return txfm_rd_gate_level[TX_SEARCH_COMP_TYPE_MODE];
return txfm_rd_gate_level[TX_SEARCH_DEFAULT];
}
diff --git a/av1/encoder/saliency_map.c b/av1/encoder/saliency_map.c
index 3376846..30019bb 100644
--- a/av1/encoder/saliency_map.c
+++ b/av1/encoder/saliency_map.c
@@ -1261,7 +1261,7 @@
return 0;
}
- const int bsize = cm->seq_params->sb_size;
+ const BLOCK_SIZE bsize = cm->seq_params->sb_size;
const int num_mi_w = mi_size_wide[bsize];
const int num_mi_h = mi_size_high[bsize];
const int block_width = block_size_wide[bsize];
diff --git a/av1/encoder/speed_features.c b/av1/encoder/speed_features.c
index c432e42..830d2c6 100644
--- a/av1/encoder/speed_features.c
+++ b/av1/encoder/speed_features.c
@@ -847,6 +847,14 @@
} else {
sf->inter_sf.prune_nearmv_using_neighbors = PRUNE_NEARMV_LEVEL2;
}
+
+ if (is_720p_or_larger)
+ sf->part_sf.ext_part_eval_based_on_cur_best =
+ (allow_screen_content_tools || frame_is_intra_only(cm)) ? 0 : 1;
+
+ if (is_480p_or_larger) {
+ sf->tpl_sf.reduce_num_frames = 1;
+ }
}
if (speed >= 6) {
@@ -863,6 +871,10 @@
sf->part_sf.auto_max_partition_based_on_simple_motion = DIRECT_PRED;
}
+ if (is_480p_or_larger) {
+ sf->hl_sf.allow_sub_blk_me_in_tf = 1;
+ }
+
if (is_1080p_or_larger) {
sf->part_sf.default_min_partition_size = BLOCK_8X8;
}
@@ -972,6 +984,9 @@
sf->hl_sf.superres_auto_search_type = SUPERRES_AUTO_DUAL;
if (speed >= 1) {
+ sf->hl_sf.adjust_num_frames_for_arf_filtering =
+ allow_screen_content_tools ? 0 : 1;
+
sf->part_sf.intra_cnn_based_part_prune_level =
allow_screen_content_tools ? 0 : 2;
sf->part_sf.simple_motion_search_early_term_none = 1;
@@ -1100,6 +1115,7 @@
sf->mv_sf.search_method = DIAMOND;
sf->mv_sf.disable_second_mv = 2;
sf->mv_sf.prune_mesh_search = PRUNE_MESH_SEARCH_LVL_1;
+ sf->mv_sf.use_intrabc = 0;
sf->inter_sf.disable_interinter_wedge_newmv_search = boosted ? 0 : 1;
sf->inter_sf.mv_cost_upd_level = INTERNAL_COST_UPD_SBROW;
@@ -1173,6 +1189,7 @@
sf->inter_sf.alt_ref_search_fp = 2;
sf->inter_sf.txfm_rd_gate_level[TX_SEARCH_DEFAULT] = boosted ? 0 : 3;
sf->inter_sf.txfm_rd_gate_level[TX_SEARCH_MOTION_MODE] = boosted ? 0 : 5;
+ sf->inter_sf.txfm_rd_gate_level[TX_SEARCH_COMP_TYPE_MODE] = boosted ? 0 : 3;
sf->inter_sf.prune_inter_modes_based_on_tpl = boosted ? 0 : 2;
sf->inter_sf.prune_ext_comp_using_neighbors = 2;
@@ -1197,6 +1214,7 @@
sf->tpl_sf.subpel_force_stop = HALF_PEL;
sf->tpl_sf.search_method = FAST_BIGDIA;
+ sf->tpl_sf.use_sad_for_mode_decision = 1;
sf->tx_sf.tx_type_search.fast_intra_tx_type_search = 1;
@@ -1213,6 +1231,8 @@
if (speed >= 5) {
sf->hl_sf.weight_calc_level_in_tf = 1;
+ sf->hl_sf.adjust_num_frames_for_arf_filtering =
+ allow_screen_content_tools ? 0 : 2;
sf->fp_sf.reduce_mv_step_param = 4;
@@ -1223,15 +1243,18 @@
sf->part_sf.ext_partition_eval_thresh =
allow_screen_content_tools ? BLOCK_8X8 : BLOCK_16X16;
sf->part_sf.prune_sub_8x8_partition_level =
- (allow_screen_content_tools || frame_is_intra_only(&cpi->common)) ? 0
- : 2;
+ allow_screen_content_tools ? 1 : 2;
sf->mv_sf.warp_search_method = WARP_SEARCH_DIAMOND;
sf->inter_sf.prune_inter_modes_if_skippable = 1;
+ sf->inter_sf.prune_single_ref = is_boosted_arf2_bwd_type ? 0 : 1;
sf->inter_sf.txfm_rd_gate_level[TX_SEARCH_DEFAULT] = boosted ? 0 : 4;
+ sf->inter_sf.txfm_rd_gate_level[TX_SEARCH_COMP_TYPE_MODE] = boosted ? 0 : 5;
sf->inter_sf.enable_fast_compound_mode_search = 2;
+ sf->interp_sf.skip_interp_filter_search = boosted ? 0 : 1;
+
sf->intra_sf.chroma_intra_pruning_with_hog = 3;
// TODO(any): Extend multi-winner mode processing support for inter frames
@@ -1247,6 +1270,7 @@
sf->tpl_sf.use_y_only_rate_distortion = 1;
sf->tpl_sf.subpel_force_stop = FULL_PEL;
sf->tpl_sf.gop_length_decision_method = 2;
+ sf->tpl_sf.use_sad_for_mode_decision = 2;
sf->winner_mode_sf.dc_blk_pred_level = 2;
@@ -1256,11 +1280,10 @@
if (speed >= 6) {
sf->hl_sf.disable_extra_sc_testing = 1;
sf->hl_sf.second_alt_ref_filtering = 0;
- sf->hl_sf.adjust_num_frames_for_arf_filtering =
- allow_screen_content_tools ? 0 : 1;
sf->inter_sf.prune_inter_modes_based_on_tpl = boosted ? 0 : 3;
sf->inter_sf.selective_ref_frame = 6;
+ sf->inter_sf.prune_single_ref = is_boosted_arf2_bwd_type ? 0 : 2;
sf->inter_sf.prune_ext_comp_using_neighbors = 3;
sf->intra_sf.chroma_intra_pruning_with_hog = 4;
@@ -1273,10 +1296,7 @@
sf->part_sf.prune_rectangular_split_based_on_qidx =
boosted || allow_screen_content_tools ? 0 : 2;
- sf->part_sf.prune_sub_8x8_partition_level =
- allow_screen_content_tools ? 0
- : frame_is_intra_only(&cpi->common) ? 1
- : 2;
+
sf->part_sf.prune_part4_search = 3;
sf->mv_sf.simple_motion_subpel_force_stop = FULL_PEL;
@@ -1488,11 +1508,7 @@
cpi->svc.number_temporal_layers > 1)
sf->hl_sf.accurate_bit_estimate = 0;
- // TODO(yunqingwang@google.com): test to see if
- // estimate_motion_for_var_based_partition == 2 helps here.
- if (sf->rt_sf.estimate_motion_for_var_based_partition == 2)
- sf->rt_sf.estimate_motion_for_var_based_partition = 1;
- if (speed >= 9) sf->rt_sf.estimate_motion_for_var_based_partition = 0;
+ sf->rt_sf.estimate_motion_for_var_based_partition = 1;
// For single layers RPS: bias/adjustment for recovery frame.
if (cpi->ppi->rtc_ref.bias_recovery_frame) {
@@ -1509,6 +1525,8 @@
sf->rt_sf.reduce_mv_pel_precision_highmotion = 1;
sf->mv_sf.use_bsize_dependent_search_method = 0;
sf->rt_sf.skip_cdef_sb = 1;
+ sf->rt_sf.increase_color_thresh_palette = 1;
+ if (!frame_is_intra_only(cm)) sf->rt_sf.dct_only_palette_nonrd = 1;
}
if (speed >= 8) {
sf->rt_sf.nonrd_check_partition_merge_mode = 3;
@@ -1542,6 +1560,7 @@
sf->rt_sf.part_early_exit_zeromv = 2;
sf->rt_sf.prune_palette_nonrd = 1;
sf->rt_sf.set_zeromv_skip_based_on_source_sad = 2;
+ sf->rt_sf.increase_color_thresh_palette = 0;
}
sf->rt_sf.use_nonrd_altref_frame = 0;
sf->rt_sf.use_rtc_tf = 0;
@@ -1565,12 +1584,11 @@
}
sf->rt_sf.partition_direct_merging = 0;
sf->hl_sf.accurate_bit_estimate = 0;
-
- // "sf->rt_sf.estimate_motion_for_var_based_partition = 2" doesn't work well
- // for screen contents.
- if (sf->rt_sf.estimate_motion_for_var_based_partition == 2)
+ // This feature is for nonrd_pickmode and non-svc for now.
+ if (sf->rt_sf.use_nonrd_pick_mode && !cpi->ppi->use_svc)
sf->rt_sf.estimate_motion_for_var_based_partition = 1;
- if (speed >= 9) sf->rt_sf.estimate_motion_for_var_based_partition = 0;
+ else
+ sf->rt_sf.estimate_motion_for_var_based_partition = 0;
}
if (is_lossless_requested(&cpi->oxcf.rc_cfg)) {
sf->rt_sf.use_rtc_tf = 0;
@@ -1608,6 +1626,7 @@
sf->inter_sf.mv_cost_upd_level = INTERNAL_COST_UPD_SBROW;
sf->inter_sf.disable_interinter_wedge_var_thresh = 100;
sf->interp_sf.cb_pred_filter_search = 0;
+ sf->interp_sf.skip_interp_filter_search = 1;
sf->part_sf.ml_prune_partition = 1;
sf->part_sf.reuse_prev_rd_results_for_part_ab = 1;
sf->part_sf.prune_ext_partition_types_search_level = 2;
@@ -1619,7 +1638,6 @@
// Disable Wiener and Self-guided Loop restoration filters.
sf->lpf_sf.disable_wiener_filter = true;
sf->lpf_sf.disable_sgr_filter = true;
- sf->rt_sf.skip_interp_filter_search = 1;
sf->intra_sf.prune_palette_search_level = 2;
sf->intra_sf.prune_luma_palette_size_search_level = 2;
sf->intra_sf.early_term_chroma_palette_size_search = 1;
@@ -1785,6 +1803,8 @@
// This sf is not applicable in non-rd path.
sf->inter_sf.skip_newmv_in_drl = 0;
+ sf->interp_sf.skip_interp_filter_search = 0;
+
// Disable intra_y_mode_mask pruning since the performance at speed 7 isn't
// good. May need more study.
for (int i = 0; i < TX_SIZES; ++i) {
@@ -1810,7 +1830,6 @@
sf->rt_sf.reuse_inter_pred_nonrd = (cpi->oxcf.noise_sensitivity == 0);
#endif
sf->rt_sf.short_circuit_low_temp_var = 0;
- sf->rt_sf.skip_interp_filter_search = 0;
// For spatial layers, only LAST and GOLDEN are currently used in the SVC
// for nonrd. The flag use_nonrd_altref_frame can disable GOLDEN in the
// get_ref_frame_flags() for some patterns, so disable it here for
@@ -1872,6 +1891,10 @@
sf->rt_sf.var_part_split_threshold_shift = 10;
sf->mv_sf.subpel_search_method = SUBPEL_TREE_PRUNED_MORE;
}
+ if (speed >= 11 && !frame_is_intra_only(cm) &&
+ cpi->oxcf.tune_cfg.content == AOM_CONTENT_SCREEN) {
+ sf->winner_mode_sf.dc_blk_pred_level = 3;
+ }
}
static AOM_INLINE void init_hl_sf(HIGH_LEVEL_SPEED_FEATURES *hl_sf) {
@@ -1887,6 +1910,7 @@
hl_sf->adjust_num_frames_for_arf_filtering = 0;
hl_sf->accurate_bit_estimate = 0;
hl_sf->weight_calc_level_in_tf = 0;
+ hl_sf->allow_sub_blk_me_in_tf = 0;
}
static AOM_INLINE void init_fp_sf(FIRST_PASS_SPEED_FEATURES *fp_sf) {
@@ -1907,6 +1931,8 @@
tpl_sf->prune_ref_frames_in_tpl = 0;
tpl_sf->allow_compound_pred = 1;
tpl_sf->use_y_only_rate_distortion = 0;
+ tpl_sf->use_sad_for_mode_decision = 0;
+ tpl_sf->reduce_num_frames = 0;
}
static AOM_INLINE void init_gm_sf(GLOBAL_MOTION_SPEED_FEATURES *gm_sf) {
@@ -1948,6 +1974,7 @@
part_sf->intra_cnn_based_part_prune_level = 0;
part_sf->ext_partition_eval_thresh = BLOCK_8X8;
part_sf->rect_partition_eval_thresh = BLOCK_128X128;
+ part_sf->ext_part_eval_based_on_cur_best = 0;
part_sf->prune_ext_part_using_split_info = 0;
part_sf->prune_rectangular_split_based_on_qidx = 0;
part_sf->prune_rect_part_using_4x4_var_deviation = false;
@@ -1983,6 +2010,7 @@
mv_sf->skip_fullpel_search_using_startmv = 0;
mv_sf->warp_search_method = WARP_SEARCH_SQUARE;
mv_sf->warp_search_iters = 8;
+ mv_sf->use_intrabc = 1;
}
static AOM_INLINE void init_inter_sf(INTER_MODE_SPEED_FEATURES *inter_sf) {
@@ -1990,6 +2018,7 @@
inter_sf->model_based_post_interp_filter_breakout = 0;
inter_sf->reduce_inter_modes = 0;
inter_sf->alt_ref_search_fp = 0;
+ inter_sf->prune_single_ref = 0;
inter_sf->prune_comp_ref_frames = 0;
inter_sf->selective_ref_frame = 0;
inter_sf->prune_ref_frame_for_rect_partitions = 0;
@@ -2042,6 +2071,7 @@
interp_sf->skip_sharp_interp_filter_search = 0;
interp_sf->use_fast_interpolation_filter_search = 0;
interp_sf->use_interp_filter = 0;
+ interp_sf->skip_interp_filter_search = 0;
}
static AOM_INLINE void init_intra_sf(INTRA_MODE_SPEED_FEATURES *intra_sf) {
@@ -2139,6 +2169,8 @@
static AOM_INLINE void init_lpf_sf(LOOP_FILTER_SPEED_FEATURES *lpf_sf) {
lpf_sf->disable_loop_restoration_chroma = 0;
lpf_sf->disable_loop_restoration_luma = 0;
+ lpf_sf->min_lr_unit_size = RESTORATION_PROC_UNIT_SIZE;
+ lpf_sf->max_lr_unit_size = RESTORATION_UNITSIZE_MAX;
lpf_sf->prune_wiener_based_on_src_var = 0;
lpf_sf->prune_sgr_based_on_wiener = 0;
lpf_sf->enable_sgr_ep_pruning = 0;
@@ -2172,7 +2204,6 @@
rt_sf->num_inter_modes_for_tx_search = INT_MAX;
rt_sf->use_nonrd_filter_search = 0;
rt_sf->use_simple_rd_model = 0;
- rt_sf->skip_interp_filter_search = 0;
rt_sf->hybrid_intra_pickmode = 0;
rt_sf->source_metrics_sb_nonrd = 0;
rt_sf->overshoot_detection_cbr = NO_DETECTION;
@@ -2200,6 +2231,7 @@
rt_sf->use_rtc_tf = 0;
rt_sf->prune_idtx_nonrd = 0;
rt_sf->prune_palette_nonrd = 0;
+ rt_sf->dct_only_palette_nonrd = 0;
rt_sf->part_early_exit_zeromv = 0;
rt_sf->sse_early_term_inter_search = EARLY_TERM_DISABLED;
rt_sf->skip_lf_screen = 0;
@@ -2222,6 +2254,7 @@
rt_sf->screen_content_cdef_filter_qindex_thresh = 0;
rt_sf->enable_ref_short_signaling = false;
rt_sf->check_globalmv_on_single_ref = true;
+ rt_sf->increase_color_thresh_palette = false;
}
static fractional_mv_step_fp
@@ -2454,9 +2487,11 @@
SPEED_FEATURES *const sf = &cpi->sf;
WinnerModeParams *const winner_mode_params = &cpi->winner_mode_params;
const int boosted = frame_is_boosted(cpi);
+ const int is_480p_or_lesser = AOMMIN(cm->width, cm->height) <= 480;
const int is_480p_or_larger = AOMMIN(cm->width, cm->height) >= 480;
const int is_720p_or_larger = AOMMIN(cm->width, cm->height) >= 720;
const int is_1080p_or_larger = AOMMIN(cm->width, cm->height) >= 1080;
+ const int is_1440p_or_larger = AOMMIN(cm->width, cm->height) >= 1440;
const int is_arf2_bwd_type =
cpi->ppi->gf_group.update_type[cpi->gf_frame_index] == INTNL_ARF_UPDATE;
@@ -2508,23 +2543,36 @@
if (speed >= 2) {
// Disable extended partitions for lower quantizers
- const int aggr = AOMMIN(3, speed - 2);
+ const int aggr = AOMMIN(4, speed - 2);
const int qindex_thresh1[4] = { 50, 50, 80, 100 };
const int qindex_thresh2[4] = { 80, 100, 120, 160 };
int qindex_thresh;
- int disable_ext_part;
if (aggr <= 1) {
const int qthresh2 =
(!aggr && !is_480p_or_larger) ? 70 : qindex_thresh2[aggr];
qindex_thresh = cm->features.allow_screen_content_tools
? qindex_thresh1[aggr]
: qthresh2;
- disable_ext_part = !boosted;
- } else {
+ if (cm->quant_params.base_qindex <= qindex_thresh && !boosted)
+ sf->part_sf.ext_partition_eval_thresh = BLOCK_128X128;
+ } else if (aggr <= 2) {
qindex_thresh = boosted ? qindex_thresh1[aggr] : qindex_thresh2[aggr];
- disable_ext_part = !frame_is_intra_only(cm);
- }
- if (cm->quant_params.base_qindex <= qindex_thresh && disable_ext_part) {
+ if (cm->quant_params.base_qindex <= qindex_thresh &&
+ !frame_is_intra_only(cm))
+ sf->part_sf.ext_partition_eval_thresh = BLOCK_128X128;
+ } else if (aggr <= 3) {
+ if (!is_480p_or_larger) {
+ sf->part_sf.ext_partition_eval_thresh = BLOCK_128X128;
+ } else if (!is_720p_or_larger && !frame_is_intra_only(cm) &&
+ !cm->features.allow_screen_content_tools) {
+ sf->part_sf.ext_partition_eval_thresh = BLOCK_128X128;
+ } else {
+ qindex_thresh = boosted ? qindex_thresh1[aggr] : qindex_thresh2[aggr];
+ if (cm->quant_params.base_qindex <= qindex_thresh &&
+ !frame_is_intra_only(cm))
+ sf->part_sf.ext_partition_eval_thresh = BLOCK_128X128;
+ }
+ } else {
sf->part_sf.ext_partition_eval_thresh = BLOCK_128X128;
}
}
@@ -2593,6 +2641,45 @@
}
}
+ if (speed >= 5) {
+ // Disable the sf for low quantizers in case of low resolution screen
+ // contents.
+ if (cm->features.allow_screen_content_tools &&
+ cm->quant_params.base_qindex < 128 && is_480p_or_lesser) {
+ sf->part_sf.prune_sub_8x8_partition_level = 0;
+ }
+ }
+
+ // Loop restoration size search
+ // At speed 0, always search all available sizes for the maximum possible gain
+ sf->lpf_sf.min_lr_unit_size = RESTORATION_PROC_UNIT_SIZE;
+ sf->lpf_sf.max_lr_unit_size = RESTORATION_UNITSIZE_MAX;
+
+ if (speed >= 1) {
+ // For large frames, small restoration units are almost never useful,
+ // so prune them away
+ if (is_1440p_or_larger) {
+ sf->lpf_sf.min_lr_unit_size = RESTORATION_UNITSIZE_MAX;
+ } else if (is_720p_or_larger) {
+ sf->lpf_sf.min_lr_unit_size = RESTORATION_UNITSIZE_MAX >> 1;
+ }
+ }
+
+ if (speed >= 3 || (cpi->oxcf.mode == ALLINTRA && speed >= 1)) {
+ // At this speed, a full search is too expensive. Instead, pick a single
+ // size based on size and qindex. Note that, in general, higher quantizers
+ // (== lower quality) and larger frames generally want to use larger
+ // restoration units.
+ int qindex_thresh = 96;
+ if (cm->quant_params.base_qindex <= qindex_thresh && !is_1440p_or_larger) {
+ sf->lpf_sf.min_lr_unit_size = RESTORATION_UNITSIZE_MAX >> 1;
+ sf->lpf_sf.max_lr_unit_size = RESTORATION_UNITSIZE_MAX >> 1;
+ } else {
+ sf->lpf_sf.min_lr_unit_size = RESTORATION_UNITSIZE_MAX;
+ sf->lpf_sf.max_lr_unit_size = RESTORATION_UNITSIZE_MAX;
+ }
+ }
+
set_subpel_search_method(&cpi->mv_search_params,
cpi->oxcf.unit_test_cfg.motion_vector_unit_test,
sf->mv_sf.subpel_search_method);
diff --git a/av1/encoder/speed_features.h b/av1/encoder/speed_features.h
index 27a07c5..14cd874 100644
--- a/av1/encoder/speed_features.h
+++ b/av1/encoder/speed_features.h
@@ -92,6 +92,8 @@
(1 << NEAREST_NEWMV) | (1 << NEW_NEARESTMV) |
(1 << NEW_NEARMV) | (1 << NEAR_NEWMV) |
(1 << NEAR_NEARMV),
+ INTER_SINGLE_ALL =
+ (1 << NEARESTMV) | (1 << NEARMV) | (1 << GLOBALMV) | (1 << NEWMV),
};
enum {
@@ -240,11 +242,14 @@
} UENUM1BYTE(PRUNE_NEARMV_LEVEL);
enum {
- // Default Transform search case - used in evaluation of compound type mode
- // and best inter candidates
+ // Default transform search used in evaluation of best inter candidates
+ // (MODE_EVAL stage) and motion mode winner processing (WINNER_MODE_EVAL
+ // stage).
TX_SEARCH_DEFAULT = 0,
- // Transform search in motion mode rd
+ // Transform search in motion mode rd during MODE_EVAL stage.
TX_SEARCH_MOTION_MODE,
+ // Transform search in compound type mode rd during MODE_EVAL stage.
+ TX_SEARCH_COMP_TYPE_MODE,
// All transform search cases
TX_SEARCH_CASES
} UENUM1BYTE(TX_SEARCH_CASE);
@@ -448,7 +453,11 @@
/*!
* The number of frames to be used during temporal filtering of an ARF frame
- * is adjusted based on noise level of the current frame.
+ * is adjusted based on noise level of the current frame. The sf has three
+ * levels to decide number of frames to be considered for filtering:
+ * 0 : Use default number of frames
+ * 1 and 2 : Reduce the number of frames based on noise level with varied
+ * aggressiveness
*/
int adjust_num_frames_for_arf_filtering;
@@ -465,6 +474,14 @@
* 1: Calculate weight using a lookup table that approximates exp().
*/
int weight_calc_level_in_tf;
+
+ /*!
+ * Decide whether to perform motion estimation at split block (i.e. 16x16)
+ * level or not.
+ * 0: Always allow motion estimation.
+ * 1: Conditionally allow motion estimation based on 4x4 sub-blocks variance.
+ */
+ int allow_sub_blk_me_in_tf;
} HIGH_LEVEL_SPEED_FEATURES;
/*!
@@ -536,6 +553,19 @@
// Calculate rate and distortion based on Y plane only.
int use_y_only_rate_distortion;
+
+ // Use SAD instead of SATD during intra/inter mode search.
+ // If set to 0, use SATD always.
+ // If set to 1, use SAD during intra/inter mode search for frames in the
+ // higher temporal layers of the hierarchical prediction structure.
+ // If set to 2, use SAD during intra/inter mode search for all frames.
+ // This sf is disabled for the first GF group of the key-frame interval,
+ // i.e., SATD is used during intra/inter mode search of the first GF group.
+ int use_sad_for_mode_decision;
+
+ // Skip tpl processing for frames of type LF_UPDATE.
+ // This sf is disabled for the first GF group of the key-frame interval.
+ int reduce_num_frames;
} TPL_SPEED_FEATURES;
typedef struct GLOBAL_MOTION_SPEED_FEATURES {
@@ -567,9 +597,10 @@
// Used if partition_search_type = FIXED_PARTITION
BLOCK_SIZE fixed_partition_size;
- // Prune extended partition types search
- // Can take values 0 - 2, 0 referring to no pruning, and 1 - 2 increasing
- // aggressiveness of pruning in order.
+ // Prune extended partition types search based on the current best partition
+ // and the combined rdcost of the subblocks estimated from previous
+ // partitions. Can take values 0 - 2, 0 referring to no pruning, and 1 - 2
+ // increasing aggressiveness of pruning in order.
int prune_ext_partition_types_search_level;
// Prune part4 based on block size
@@ -654,13 +685,18 @@
// 2: Prune none, split and rectangular partitions
int intra_cnn_based_part_prune_level;
- // Disable extended partition search for lower block sizes.
- int ext_partition_eval_thresh;
+ // Disable extended partition search if the current bsize is greater than the
+ // threshold. Must be a square block size BLOCK_8X8 or higher.
+ BLOCK_SIZE ext_partition_eval_thresh;
+
+ // Use best partition decision so far to tune 'ext_partition_eval_thresh'
+ int ext_part_eval_based_on_cur_best;
// Disable rectangular partitions for larger block sizes.
int rect_partition_eval_thresh;
- // prune extended partition search
+ // Prune extended partition search based on whether the split/rect partitions
+ // provided an improvement in the previous search.
// 0 : no pruning
// 1 : prune 1:4 partition search using winner info from split partitions
// 2 : prune 1:4 and AB partition search using split and HORZ/VERT info
@@ -822,6 +858,9 @@
// Accurate full pixel motion search based on TPL stats.
int full_pixel_search_level;
+ // Allow intrabc motion search
+ int use_intrabc;
+
// Whether to downsample the rows in sad calculation during motion search.
// This is only active when there are at least 16 rows. When this sf is
// active, if there is a large discrepancy in the SAD values for the final
@@ -900,6 +939,12 @@
// 2 prune inter modes w.r.t BWDREF, ALTREF2 and ALTREF reference frames
int alt_ref_search_fp;
+ // Prune reference frames for single prediction modes based on temporal
+ // distance and pred MV SAD. Feasible values are 0, 1, 2. The feature is
+ // disabled for 0. An increasing value indicates more aggressive pruning
+ // threshold.
+ int prune_single_ref;
+
// Prune compound reference frames
// 0 no pruning
// 1 prune compound references which do not satisfy the two conditions:
@@ -1123,6 +1168,10 @@
// adaptive interp_filter search to allow skip of certain filter types.
int adaptive_interp_filter_search;
+
+ // Forces interpolation filter to EIGHTTAP_REGULAR and skips interpolation
+ // filter search.
+ int skip_interp_filter_search;
} INTERP_FILTER_SPEED_FEATURES;
typedef struct INTRA_MODE_SPEED_FEATURES {
@@ -1441,6 +1490,13 @@
// Disable loop restoration for luma plane
int disable_loop_restoration_luma;
+ // Range of loop restoration unit sizes to search
+ // The minimum size is clamped against the superblock size in
+ // av1_pick_filter_restoration, so that the code which sets this value does
+ // not need to know the superblock size ahead of time.
+ int min_lr_unit_size;
+ int max_lr_unit_size;
+
// Prune RESTORE_WIENER evaluation based on source variance
// 0 : no pruning
// 1 : conservative pruning
@@ -1541,9 +1597,6 @@
// Use simplified RD model for interpolation search and Intra
int use_simple_rd_model;
- // If set forces interpolation filter to EIGHTTAP_REGULAR
- int skip_interp_filter_search;
-
// For nonrd mode: use hybrid intra mode search for intra only frames based on
// block properties.
// 0 : use nonrd pick intra for all blocks
@@ -1671,6 +1724,9 @@
// Prune the use of paletter mode in nonrd pickmode.
int prune_palette_nonrd;
+ // Force to only use dct for palette search in nonrd pickmode.
+ int dct_only_palette_nonrd;
+
// Skip loopfilter, for static content after slide change
// or key frame, once quality has ramped up.
// 0: disabled
@@ -1798,6 +1854,11 @@
// A flag that controls if we check or bypass GLOBALMV in rtc single ref frame
// case.
bool check_globalmv_on_single_ref;
+
+ // Allows for increasing the color_threshold for palette prediction.
+ // This generally leads to better coding efficiency but with some speed loss.
+ // Only used for screen content and for nonrd_pickmode.
+ bool increase_color_thresh_palette;
} REAL_TIME_SPEED_FEATURES;
/*!\endcond */
diff --git a/av1/encoder/svc_layercontext.c b/av1/encoder/svc_layercontext.c
index 85678dc..ae0c276 100644
--- a/av1/encoder/svc_layercontext.c
+++ b/av1/encoder/svc_layercontext.c
@@ -33,6 +33,7 @@
svc->force_zero_mode_spatial_ref = 1;
svc->num_encoded_top_layer = 0;
svc->use_flexible_mode = 0;
+ svc->has_lower_quality_layer = 0;
for (int sl = 0; sl < svc->number_spatial_layers; ++sl) {
for (int tl = 0; tl < svc->number_temporal_layers; ++tl) {
@@ -69,7 +70,7 @@
lc->actual_num_seg1_blocks = 0;
lc->actual_num_seg2_blocks = 0;
lc->counter_encode_maxq_scene_change = 0;
- if (lc->map) aom_free(lc->map);
+ aom_free(lc->map);
CHECK_MEM_ERROR(cm, lc->map,
aom_calloc(mi_rows * mi_cols, sizeof(*lc->map)));
}
@@ -155,7 +156,7 @@
lc->actual_num_seg1_blocks = 0;
lc->actual_num_seg2_blocks = 0;
lc->counter_encode_maxq_scene_change = 0;
- if (lc->map) aom_free(lc->map);
+ aom_free(lc->map);
CHECK_MEM_ERROR(cm, lc->map,
aom_calloc(mi_rows * mi_cols, sizeof(*lc->map)));
}
@@ -215,6 +216,7 @@
LAYER_CONTEXT *const lc = get_layer_context(cpi);
const int old_frame_since_key = cpi->rc.frames_since_key;
const int old_frame_to_key = cpi->rc.frames_to_key;
+ const int max_consec_drop = cpi->rc.max_consec_drop;
// Restore layer rate control.
cpi->rc = lc->rc;
cpi->ppi->p_rc = lc->p_rc;
@@ -227,6 +229,8 @@
// before the layer restore. Keep these defined for the stream (not layer).
cpi->rc.frames_since_key = old_frame_since_key;
cpi->rc.frames_to_key = old_frame_to_key;
+ // Reset to value before the layer restore.
+ cpi->rc.max_consec_drop = max_consec_drop;
// For spatial-svc, allow cyclic-refresh to be applied on the spatial layers,
// for the base temporal layer.
if (cpi->oxcf.q_cfg.aq_mode == CYCLIC_REFRESH_AQ &&
@@ -245,7 +249,8 @@
// This is to skip searching mv for that reference if it was last
// refreshed (i.e., buffer slot holding that reference was refreshed) on the
// previous spatial layer(s) at the same time (current_superframe).
- if (rtc_ref->set_ref_frame_config && svc->force_zero_mode_spatial_ref) {
+ if (rtc_ref->set_ref_frame_config && svc->force_zero_mode_spatial_ref &&
+ cpi->sf.rt_sf.use_nonrd_pick_mode) {
if (check_ref_is_low_spatial_res_super_frame(LAST_FRAME, svc, rtc_ref)) {
svc->skip_mvsearch_last = 1;
}
@@ -357,7 +362,8 @@
for (int tl = 0; tl < svc->number_temporal_layers; ++tl) {
int layer = LAYER_IDS_TO_IDX(sl, tl, svc->number_temporal_layers);
LAYER_CONTEXT *const lc = &svc->layer_context[layer];
- if (lc->map) aom_free(lc->map);
+ aom_free(lc->map);
+ lc->map = NULL;
}
}
}
@@ -395,6 +401,16 @@
int width = 0, height = 0;
lc = &svc->layer_context[svc->spatial_layer_id * svc->number_temporal_layers +
svc->temporal_layer_id];
+ // Set the lower quality layer flag.
+ svc->has_lower_quality_layer = 0;
+ if (cpi->svc.spatial_layer_id > 0) {
+ const LAYER_CONTEXT *lc_prev =
+ &svc->layer_context[(svc->spatial_layer_id - 1) *
+ svc->number_temporal_layers +
+ svc->temporal_layer_id];
+ if (lc_prev->scaling_factor_den == 1 && lc_prev->scaling_factor_num == 1)
+ svc->has_lower_quality_layer = 1;
+ }
av1_get_layer_resolution(cpi->oxcf.frm_dim_cfg.width,
cpi->oxcf.frm_dim_cfg.height, lc->scaling_factor_num,
lc->scaling_factor_den, &width, &height);
@@ -499,7 +515,8 @@
// Set all buffer_idx to 0.
// Set GOLDEN to slot 5 and update slot 5.
for (i = 0; i < INTER_REFS_PER_FRAME; i++) rtc_ref->ref_idx[i] = 0;
- if (svc->temporal_layer_id < svc->number_temporal_layers - 1) {
+ if (svc->temporal_layer_id < svc->number_temporal_layers - 1 ||
+ svc->spatial_layer_id < svc->number_spatial_layers - 1) {
rtc_ref->ref_idx[SVC_GOLDEN_FRAME] = 5;
rtc_ref->refresh[5] = 1;
}
@@ -509,7 +526,8 @@
// Set LAST3 to slot 6 and update slot 6.
for (i = 0; i < INTER_REFS_PER_FRAME; i++) rtc_ref->ref_idx[i] = 5;
rtc_ref->ref_idx[SVC_LAST_FRAME] = 1;
- if (svc->temporal_layer_id < svc->number_temporal_layers - 1) {
+ if (svc->temporal_layer_id < svc->number_temporal_layers - 1 ||
+ svc->spatial_layer_id < svc->number_spatial_layers - 1) {
rtc_ref->ref_idx[SVC_LAST3_FRAME] = 6;
rtc_ref->refresh[6] = 1;
}
diff --git a/av1/encoder/svc_layercontext.h b/av1/encoder/svc_layercontext.h
index 3a6e0fc..bfde33d 100644
--- a/av1/encoder/svc_layercontext.h
+++ b/av1/encoder/svc_layercontext.h
@@ -139,6 +139,14 @@
* Force zero-mv in mode search for the spatial/inter-layer reference.
*/
int force_zero_mode_spatial_ref;
+
+ /*!
+ * Flag to indicate that current spatial layer has a lower quality layer
+ * (at the same timestamp) that can be used as a reference.
+ * Lower quality layer refers to the same resolution but encoded at
+ * different/lower bitrate.
+ */
+ int has_lower_quality_layer;
} SVC;
struct AV1_COMP;
diff --git a/av1/encoder/temporal_filter.c b/av1/encoder/temporal_filter.c
index 91a0c78..d6ae667 100644
--- a/av1/encoder/temporal_filter.c
+++ b/av1/encoder/temporal_filter.c
@@ -9,6 +9,7 @@
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
+#include <float.h>
#include <math.h>
#include <limits.h>
@@ -33,6 +34,7 @@
#include "av1/encoder/extend.h"
#include "av1/encoder/firstpass.h"
#include "av1/encoder/gop_structure.h"
+#include "av1/encoder/intra_mode_search_utils.h"
#include "av1/encoder/mcomp.h"
#include "av1/encoder/motion_search_facade.h"
#include "av1/encoder/pass2_strategy.h"
@@ -49,6 +51,39 @@
static void tf_determine_block_partition(const MV block_mv, const int block_mse,
MV *subblock_mvs, int *subblock_mses);
+// This function returns the minimum and maximum log variances for 4x4 sub
+// blocks in the current block.
+static INLINE void get_log_var_4x4sub_blk(
+ AV1_COMP *cpi, const YV12_BUFFER_CONFIG *const frame_to_filter, int mb_row,
+ int mb_col, BLOCK_SIZE block_size, double *blk_4x4_var_min,
+ double *blk_4x4_var_max, int is_hbd) {
+ const int mb_height = block_size_high[block_size];
+ const int mb_width = block_size_wide[block_size];
+ int var_min = INT_MAX;
+ int var_max = 0;
+
+ // Derive the source buffer.
+ const int src_stride = frame_to_filter->y_stride;
+ const int y_offset = mb_row * mb_height * src_stride + mb_col * mb_width;
+ const uint8_t *src_buf = frame_to_filter->y_buffer + y_offset;
+
+ for (int i = 0; i < mb_height; i += MI_SIZE) {
+ for (int j = 0; j < mb_width; j += MI_SIZE) {
+ // Calculate the 4x4 sub-block variance.
+ const int var = av1_calc_normalized_variance(
+ cpi->ppi->fn_ptr[BLOCK_4X4].vf, src_buf + (i * src_stride) + j,
+ src_stride, is_hbd);
+
+ // Record min and max for over-arching block
+ var_min = AOMMIN(var_min, var);
+ var_max = AOMMAX(var_max, var);
+ }
+ }
+
+ *blk_4x4_var_min = log1p(var_min / 16.0);
+ *blk_4x4_var_max = log1p(var_max / 16.0);
+}
+
/*!\endcond */
/*!\brief Does motion search for blocks in temporal filtering. This is
* the first step for temporal filtering. More specifically, given a frame to
@@ -68,19 +103,22 @@
* the entire block.
*
* \ingroup src_frame_proc
- * \param[in] cpi Top level encoder instance structure
- * \param[in] mb Pointer to macroblock
- * \param[in] frame_to_filter Pointer to the frame to be filtered
- * \param[in] ref_frame Pointer to the reference frame
- * \param[in] block_size Block size used for motion search
- * \param[in] mb_row Row index of the block in the frame
- * \param[in] mb_col Column index of the block in the frame
- * \param[in] ref_mv Reference motion vector, which is commonly
- * inherited from the motion search result of
- * previous frame.
- * \param[out] subblock_mvs Pointer to the motion vectors for 4 sub-blocks
- * \param[out] subblock_mses Pointer to the search errors (MSE) for 4
- * sub-blocks
+ * \param[in] cpi Top level encoder instance structure
+ * \param[in] mb Pointer to macroblock
+ * \param[in] frame_to_filter Pointer to the frame to be filtered
+ * \param[in] ref_frame Pointer to the reference frame
+ * \param[in] block_size Block size used for motion search
+ * \param[in] mb_row Row index of the block in the frame
+ * \param[in] mb_col Column index of the block in the frame
+ * \param[in] ref_mv Reference motion vector, which is commonly
+ * inherited from the motion search result of
+ * previous frame.
+ * \param[in] allow_me_for_sub_blks Flag to indicate whether motion search at
+ * 16x16 sub-block level is needed or not.
+ * \param[out] subblock_mvs Pointer to the motion vectors for
+ * 4 sub-blocks
+ * \param[out] subblock_mses Pointer to the search errors (MSE) for
+ * 4 sub-blocks
*
* \remark Nothing will be returned. Results are saved in subblock_mvs and
* subblock_mses
@@ -89,7 +127,8 @@
const YV12_BUFFER_CONFIG *frame_to_filter,
const YV12_BUFFER_CONFIG *ref_frame,
const BLOCK_SIZE block_size, const int mb_row,
- const int mb_col, MV *ref_mv, MV *subblock_mvs,
+ const int mb_col, MV *ref_mv,
+ bool allow_me_for_sub_blks, MV *subblock_mvs,
int *subblock_mses) {
// Frame information
const int min_frame_size = AOMMIN(cpi->common.width, cpi->common.height);
@@ -99,7 +138,10 @@
const int mb_width = block_size_wide[block_size];
const int mb_pels = mb_height * mb_width;
const int y_stride = frame_to_filter->y_stride;
+ const int src_width = frame_to_filter->y_width;
+ const int ref_width = ref_frame->y_width;
assert(y_stride == ref_frame->y_stride);
+ assert(src_width == ref_width);
const int y_offset = mb_row * mb_height * y_stride + mb_col * mb_width;
// Save input state.
@@ -127,8 +169,10 @@
// Setup.
mb->plane[0].src.buf = frame_to_filter->y_buffer + y_offset;
mb->plane[0].src.stride = y_stride;
+ mb->plane[0].src.width = src_width;
mbd->plane[0].pre[0].buf = ref_frame->y_buffer + y_offset;
mbd->plane[0].pre[0].stride = y_stride;
+ mbd->plane[0].pre[0].width = ref_width;
const SEARCH_METHODS search_method = NSTEP;
const search_site_config *search_site_cfg =
@@ -141,14 +185,15 @@
// Do motion search.
int_mv best_mv; // Searched motion vector.
+ FULLPEL_MV_STATS best_mv_stats;
int block_mse = INT_MAX;
MV block_mv = kZeroMv;
const int q = av1_get_q(cpi);
av1_make_default_fullpel_ms_params(&full_ms_params, cpi, mb, block_size,
&baseline_mv, start_mv, search_site_cfg,
+ search_method,
/*fine_search_interval=*/0);
- av1_set_mv_search_method(&full_ms_params, search_site_cfg, search_method);
full_ms_params.run_mesh_search = 1;
full_ms_params.mv_cost_params.mv_cost_type = mv_cost_type;
@@ -160,7 +205,7 @@
av1_full_pixel_search(start_mv, &full_ms_params, step_param,
cond_cost_list(cpi, cost_list), &best_mv.as_fullmv,
- NULL);
+ &best_mv_stats, NULL);
if (force_integer_mv == 1) { // Only do full search on the entire block.
const int mv_row = best_mv.as_mv.row;
@@ -181,63 +226,66 @@
// Since we are merely refining the result from full pixel search, we don't
// need regularization for subpel search
ms_params.mv_cost_params.mv_cost_type = MV_COST_NONE;
+ best_mv_stats.err_cost = 0;
MV subpel_start_mv = get_mv_from_fullmv(&best_mv.as_fullmv);
assert(av1_is_subpelmv_in_range(&ms_params.mv_limits, subpel_start_mv));
error = cpi->mv_search_params.find_fractional_mv_step(
- &mb->e_mbd, &cpi->common, &ms_params, subpel_start_mv, &best_mv.as_mv,
- &distortion, &sse, NULL);
+ &mb->e_mbd, &cpi->common, &ms_params, subpel_start_mv, &best_mv_stats,
+ &best_mv.as_mv, &distortion, &sse, NULL);
block_mse = DIVIDE_AND_ROUND(error, mb_pels);
block_mv = best_mv.as_mv;
*ref_mv = best_mv.as_mv;
- // On 4 sub-blocks.
- const BLOCK_SIZE subblock_size = av1_ss_size_lookup[block_size][1][1];
- const int subblock_height = block_size_high[subblock_size];
- const int subblock_width = block_size_wide[subblock_size];
- const int subblock_pels = subblock_height * subblock_width;
- start_mv = get_fullmv_from_mv(ref_mv);
- int subblock_idx = 0;
- for (int i = 0; i < mb_height; i += subblock_height) {
- for (int j = 0; j < mb_width; j += subblock_width) {
- const int offset = i * y_stride + j;
- mb->plane[0].src.buf = frame_to_filter->y_buffer + y_offset + offset;
- mbd->plane[0].pre[0].buf = ref_frame->y_buffer + y_offset + offset;
- av1_make_default_fullpel_ms_params(&full_ms_params, cpi, mb,
- subblock_size, &baseline_mv,
- start_mv, search_site_cfg,
- /*fine_search_interval=*/0);
- av1_set_mv_search_method(&full_ms_params, search_site_cfg,
- search_method);
- full_ms_params.run_mesh_search = 1;
- full_ms_params.mv_cost_params.mv_cost_type = mv_cost_type;
+ if (allow_me_for_sub_blks) {
+ // On 4 sub-blocks.
+ const BLOCK_SIZE subblock_size = av1_ss_size_lookup[block_size][1][1];
+ const int subblock_height = block_size_high[subblock_size];
+ const int subblock_width = block_size_wide[subblock_size];
+ const int subblock_pels = subblock_height * subblock_width;
+ start_mv = get_fullmv_from_mv(ref_mv);
- if (cpi->sf.mv_sf.prune_mesh_search == PRUNE_MESH_SEARCH_LVL_1) {
- // Enable prune_mesh_search based on q for PRUNE_MESH_SEARCH_LVL_1.
- full_ms_params.prune_mesh_search = (q <= 20) ? 0 : 1;
- full_ms_params.mesh_search_mv_diff_threshold = 2;
+ int subblock_idx = 0;
+ for (int i = 0; i < mb_height; i += subblock_height) {
+ for (int j = 0; j < mb_width; j += subblock_width) {
+ const int offset = i * y_stride + j;
+ mb->plane[0].src.buf = frame_to_filter->y_buffer + y_offset + offset;
+ mbd->plane[0].pre[0].buf = ref_frame->y_buffer + y_offset + offset;
+ av1_make_default_fullpel_ms_params(
+ &full_ms_params, cpi, mb, subblock_size, &baseline_mv, start_mv,
+ search_site_cfg, search_method,
+ /*fine_search_interval=*/0);
+ full_ms_params.run_mesh_search = 1;
+ full_ms_params.mv_cost_params.mv_cost_type = mv_cost_type;
+
+ if (cpi->sf.mv_sf.prune_mesh_search == PRUNE_MESH_SEARCH_LVL_1) {
+ // Enable prune_mesh_search based on q for PRUNE_MESH_SEARCH_LVL_1.
+ full_ms_params.prune_mesh_search = (q <= 20) ? 0 : 1;
+ full_ms_params.mesh_search_mv_diff_threshold = 2;
+ }
+ av1_full_pixel_search(start_mv, &full_ms_params, step_param,
+ cond_cost_list(cpi, cost_list),
+ &best_mv.as_fullmv, &best_mv_stats, NULL);
+
+ av1_make_default_subpel_ms_params(&ms_params, cpi, mb, subblock_size,
+ &baseline_mv, cost_list);
+ ms_params.forced_stop = EIGHTH_PEL;
+ ms_params.var_params.subpel_search_type = subpel_search_type;
+ // Since we are merely refining the result from full pixel search, we
+ // don't need regularization for subpel search
+ ms_params.mv_cost_params.mv_cost_type = MV_COST_NONE;
+ best_mv_stats.err_cost = 0;
+
+ subpel_start_mv = get_mv_from_fullmv(&best_mv.as_fullmv);
+ assert(
+ av1_is_subpelmv_in_range(&ms_params.mv_limits, subpel_start_mv));
+ error = cpi->mv_search_params.find_fractional_mv_step(
+ &mb->e_mbd, &cpi->common, &ms_params, subpel_start_mv,
+ &best_mv_stats, &best_mv.as_mv, &distortion, &sse, NULL);
+ subblock_mses[subblock_idx] = DIVIDE_AND_ROUND(error, subblock_pels);
+ subblock_mvs[subblock_idx] = best_mv.as_mv;
+ ++subblock_idx;
}
-
- av1_full_pixel_search(start_mv, &full_ms_params, step_param,
- cond_cost_list(cpi, cost_list),
- &best_mv.as_fullmv, NULL);
-
- av1_make_default_subpel_ms_params(&ms_params, cpi, mb, subblock_size,
- &baseline_mv, cost_list);
- ms_params.forced_stop = EIGHTH_PEL;
- ms_params.var_params.subpel_search_type = subpel_search_type;
- // Since we are merely refining the result from full pixel search, we
- // don't need regularization for subpel search
- ms_params.mv_cost_params.mv_cost_type = MV_COST_NONE;
-
- subpel_start_mv = get_mv_from_fullmv(&best_mv.as_fullmv);
- assert(av1_is_subpelmv_in_range(&ms_params.mv_limits, subpel_start_mv));
- error = cpi->mv_search_params.find_fractional_mv_step(
- &mb->e_mbd, &cpi->common, &ms_params, subpel_start_mv,
- &best_mv.as_mv, &distortion, &sse, NULL);
- subblock_mses[subblock_idx] = DIVIDE_AND_ROUND(error, subblock_pels);
- subblock_mvs[subblock_idx] = best_mv.as_mv;
- ++subblock_idx;
}
}
}
@@ -247,9 +295,16 @@
mbd->plane[0].pre[0] = ori_pre_buf;
// Make partition decision.
- tf_determine_block_partition(block_mv, block_mse, subblock_mvs,
- subblock_mses);
-
+ if (allow_me_for_sub_blks) {
+ tf_determine_block_partition(block_mv, block_mse, subblock_mvs,
+ subblock_mses);
+ } else {
+ // Copy 32X32 block mv and mse values to sub blocks
+ for (int i = 0; i < 4; ++i) {
+ subblock_mvs[i] = block_mv;
+ subblock_mses[i] = block_mse;
+ }
+ }
// Do not pass down the reference motion vector if error is too large.
const int thresh = (min_frame_size >= 720) ? 12 : 3;
if (block_mse > (thresh << (mbd->bd - 8))) {
@@ -842,6 +897,26 @@
memset(count, 0, num_pels * sizeof(count[0]));
MV ref_mv = kZeroMv; // Reference motion vector passed down along frames.
// Perform temporal filtering frame by frame.
+
+ // Decide whether to perform motion search at 16x16 sub-block level or not
+ // based on 4x4 sub-blocks source variance. Allow motion search for split
+ // partition only if the difference between max and min source variance of
+ // 4x4 blocks is greater than a threshold (which is derived empirically).
+ bool allow_me_for_sub_blks = true;
+ if (cpi->sf.hl_sf.allow_sub_blk_me_in_tf) {
+ const int is_hbd = is_frame_high_bitdepth(frame_to_filter);
+ // Initialize minimum variance to a large value and maximum variance to 0.
+ double blk_4x4_var_min = DBL_MAX;
+ double blk_4x4_var_max = 0;
+ get_log_var_4x4sub_blk(cpi, frame_to_filter, mb_row, mb_col,
+ TF_BLOCK_SIZE, &blk_4x4_var_min, &blk_4x4_var_max,
+ is_hbd);
+ // TODO(sanampudi.venkatarao@ittiam.com): Experiment and adjust the
+ // threshold for high bit depth.
+ if ((blk_4x4_var_max - blk_4x4_var_min) <= 4.0)
+ allow_me_for_sub_blks = false;
+ }
+
for (int frame = 0; frame < num_frames; frame++) {
if (frames[frame] == NULL) continue;
@@ -855,7 +930,8 @@
ref_mv.col *= -1;
} else { // Other reference frames.
tf_motion_search(cpi, mb, frame_to_filter, frames[frame], block_size,
- mb_row, mb_col, &ref_mv, subblock_mvs, subblock_mses);
+ mb_row, mb_col, &ref_mv, allow_me_for_sub_blks,
+ subblock_mvs, subblock_mses);
}
// Perform weighted averaging.
@@ -887,8 +963,9 @@
filter_strength, weight_calc_level_in_tf, pred, accum, count);
#if CONFIG_AV1_HIGHBITDEPTH
}
-#endif // CONFIG_AV1_HIGHBITDEPTH
- } else { // for 8-bit
+#endif // CONFIG_AV1_HIGHBITDEPTH
+ } else {
+ // for 8-bit
if (TF_BLOCK_SIZE == BLOCK_32X32 && TF_WINDOW_LENGTH == 5) {
av1_apply_temporal_filter(
frame_to_filter, mbd, block_size, mb_row, mb_col, num_planes,
@@ -1047,22 +1124,32 @@
// change the number of frames for key frame filtering, which is to avoid
// visual quality drop.
int adjust_num = 6;
+ const int adjust_num_frames_for_arf_filtering =
+ cpi->sf.hl_sf.adjust_num_frames_for_arf_filtering;
if (num_frames == 1) { // `arnr_max_frames = 1` is used to disable filtering.
adjust_num = 0;
} else if ((update_type == KF_UPDATE) && q <= 10) {
adjust_num = 0;
- } else if (cpi->sf.hl_sf.adjust_num_frames_for_arf_filtering &&
- update_type != KF_UPDATE) {
+ } else if (adjust_num_frames_for_arf_filtering > 0 &&
+ update_type != KF_UPDATE && (cpi->rc.frames_since_key > 0)) {
+ // Since screen content detection happens after temporal filtering,
+ // 'frames_since_key' check is added to ensure the sf is disabled for the
+ // first alt-ref frame.
// Adjust number of frames to be considered for filtering based on noise
// level of the current frame. For low-noise frame, use more frames to
// filter such that the filtered frame can provide better predictions for
// subsequent frames and vice versa.
+ const uint8_t av1_adjust_num_using_noise_lvl[2][3] = { { 6, 4, 2 },
+ { 4, 2, 0 } };
+ const uint8_t *adjust_num_frames =
+ av1_adjust_num_using_noise_lvl[adjust_num_frames_for_arf_filtering - 1];
+
if (noise_levels[AOM_PLANE_Y] < 0.5)
- adjust_num = 4;
+ adjust_num = adjust_num_frames[0];
else if (noise_levels[AOM_PLANE_Y] < 1.0)
- adjust_num = 2;
+ adjust_num = adjust_num_frames[1];
else
- adjust_num = 0;
+ adjust_num = adjust_num_frames[2];
}
num_frames = AOMMIN(num_frames + adjust_num, lookahead_depth);
@@ -1166,11 +1253,11 @@
}
#if CONFIG_AV1_HIGHBITDEPTH
-double av1_highbd_estimate_noise_from_single_plane(const uint16_t *src16,
- int height, int width,
- const int stride,
- int bit_depth,
- int edge_thresh) {
+double av1_highbd_estimate_noise_from_single_plane_c(const uint16_t *src16,
+ int height, int width,
+ const int stride,
+ int bit_depth,
+ int edge_thresh) {
int64_t accum = 0;
int count = 0;
for (int i = 1; i < height - 1; ++i) {
diff --git a/av1/encoder/temporal_filter.h b/av1/encoder/temporal_filter.h
index 8aa4731..0b00c88 100644
--- a/av1/encoder/temporal_filter.h
+++ b/av1/encoder/temporal_filter.h
@@ -21,9 +21,9 @@
struct AV1_COMP;
struct AV1EncoderConfig;
struct ThreadData;
-// TODO(any): These two variables are only used in avx2, sse2, sse4
-// implementations, where the block size is still hard coded. This should be
-// fixed to align with the c implementation.
+// TODO(wtc): These two variables are only used in avx2, sse2, neon
+// implementations, where the block size is still hard coded to TF_BLOCK_SIZE.
+// This should be fixed to align with the c implementation.
#define BH 32
#define BW 32
@@ -261,6 +261,9 @@
#endif // CONFIG_MULTITHREAD
// Next temporal filter block row to be filtered.
int next_tf_row;
+ // Initialized to false, set to true by the worker thread that encounters an
+ // error in order to abort the processing of other worker threads.
+ bool tf_mt_exit;
} AV1TemporalFilterSync;
// Estimates noise level from a given frame using a single plane (Y, U, or V).
@@ -353,29 +356,26 @@
// num_pels: Number of pixels in the block across all planes.
// is_high_bitdepth: Whether the frame is high-bitdepth or not.
// Returns:
-// Nothing will be returned. But the contents of tf_data will be modified.
+// True if allocation is successful and false otherwise.
static AOM_INLINE bool tf_alloc_and_reset_data(TemporalFilterData *tf_data,
int num_pels,
int is_high_bitdepth) {
- tf_data->tmp_mbmi = (MB_MODE_INFO *)malloc(sizeof(*tf_data->tmp_mbmi));
- memset(tf_data->tmp_mbmi, 0, sizeof(*tf_data->tmp_mbmi));
+ tf_data->tmp_mbmi = (MB_MODE_INFO *)aom_calloc(1, sizeof(*tf_data->tmp_mbmi));
tf_data->accum =
(uint32_t *)aom_memalign(16, num_pels * sizeof(*tf_data->accum));
tf_data->count =
(uint16_t *)aom_memalign(16, num_pels * sizeof(*tf_data->count));
- memset(&tf_data->diff, 0, sizeof(tf_data->diff));
if (is_high_bitdepth)
tf_data->pred = CONVERT_TO_BYTEPTR(
aom_memalign(32, num_pels * 2 * sizeof(*tf_data->pred)));
else
tf_data->pred =
(uint8_t *)aom_memalign(32, num_pels * sizeof(*tf_data->pred));
- if (!(tf_data->accum && tf_data->count && tf_data->pred)) {
- aom_free(tf_data->accum);
- aom_free(tf_data->count);
- aom_free(tf_data->pred);
+ // In case of an allocation failure, other successfully allocated buffers will
+ // be freed by the tf_dealloc_data() call in encoder_destroy().
+ if (!(tf_data->tmp_mbmi && tf_data->accum && tf_data->count && tf_data->pred))
return false;
- }
+ memset(&tf_data->diff, 0, sizeof(tf_data->diff));
return true;
}
@@ -405,10 +405,14 @@
int is_high_bitdepth) {
if (is_high_bitdepth)
tf_data->pred = (uint8_t *)CONVERT_TO_SHORTPTR(tf_data->pred);
- free(tf_data->tmp_mbmi);
+ aom_free(tf_data->tmp_mbmi);
+ tf_data->tmp_mbmi = NULL;
aom_free(tf_data->accum);
+ tf_data->accum = NULL;
aom_free(tf_data->count);
+ tf_data->count = NULL;
aom_free(tf_data->pred);
+ tf_data->pred = NULL;
}
// Saves the state prior to temporal filter process.
diff --git a/av1/encoder/tpl_model.c b/av1/encoder/tpl_model.c
index 3aeb511..ca60e49 100644
--- a/av1/encoder/tpl_model.c
+++ b/av1/encoder/tpl_model.c
@@ -252,13 +252,15 @@
static uint32_t motion_estimation(AV1_COMP *cpi, MACROBLOCK *x,
uint8_t *cur_frame_buf,
uint8_t *ref_frame_buf, int stride,
- int stride_ref, BLOCK_SIZE bsize,
- MV center_mv, int_mv *best_mv) {
+ int ref_stride, int width, int ref_width,
+ BLOCK_SIZE bsize, MV center_mv,
+ int_mv *best_mv) {
AV1_COMMON *cm = &cpi->common;
MACROBLOCKD *const xd = &x->e_mbd;
TPL_SPEED_FEATURES *tpl_sf = &cpi->sf.tpl_sf;
int step_param;
uint32_t bestsme = UINT_MAX;
+ FULLPEL_MV_STATS best_mv_stats;
int distortion;
uint32_t sse;
int cost_list[5];
@@ -267,28 +269,29 @@
// Setup frame pointers
x->plane[0].src.buf = cur_frame_buf;
x->plane[0].src.stride = stride;
+ x->plane[0].src.width = width;
xd->plane[0].pre[0].buf = ref_frame_buf;
- xd->plane[0].pre[0].stride = stride_ref;
+ xd->plane[0].pre[0].stride = ref_stride;
+ xd->plane[0].pre[0].width = ref_width;
step_param = tpl_sf->reduce_first_step_size;
step_param = AOMMIN(step_param, MAX_MVSEARCH_STEPS - 2);
const search_site_config *search_site_cfg =
cpi->mv_search_params.search_site_cfg[SS_CFG_SRC];
- if (search_site_cfg->stride != stride_ref)
+ if (search_site_cfg->stride != ref_stride)
search_site_cfg = cpi->mv_search_params.search_site_cfg[SS_CFG_LOOKAHEAD];
- assert(search_site_cfg->stride == stride_ref);
+ assert(search_site_cfg->stride == ref_stride);
FULLPEL_MOTION_SEARCH_PARAMS full_ms_params;
av1_make_default_fullpel_ms_params(&full_ms_params, cpi, x, bsize, ¢er_mv,
start_mv, search_site_cfg,
+ tpl_sf->search_method,
/*fine_search_interval=*/0);
- av1_set_mv_search_method(&full_ms_params, search_site_cfg,
- tpl_sf->search_method);
bestsme = av1_full_pixel_search(start_mv, &full_ms_params, step_param,
cond_cost_list(cpi, cost_list),
- &best_mv->as_fullmv, NULL);
+ &best_mv->as_fullmv, &best_mv_stats, NULL);
// When sub-pel motion search is skipped, populate sub-pel precision MV and
// return.
@@ -303,11 +306,12 @@
ms_params.forced_stop = tpl_sf->subpel_force_stop;
ms_params.var_params.subpel_search_type = USE_2_TAPS;
ms_params.mv_cost_params.mv_cost_type = MV_COST_NONE;
+ best_mv_stats.err_cost = 0;
MV subpel_start_mv = get_mv_from_fullmv(&best_mv->as_fullmv);
assert(av1_is_subpelmv_in_range(&ms_params.mv_limits, subpel_start_mv));
bestsme = cpi->mv_search_params.find_fractional_mv_step(
- xd, cm, &ms_params, subpel_start_mv, &best_mv->as_mv, &distortion, &sse,
- NULL);
+ xd, cm, &ms_params, subpel_start_mv, &best_mv_stats, &best_mv->as_mv,
+ &distortion, &sse, NULL);
return bestsme;
}
@@ -451,8 +455,74 @@
}
}
+static AOM_INLINE int32_t get_inter_cost(const AV1_COMP *cpi, MACROBLOCKD *xd,
+ const uint8_t *src_mb_buffer,
+ int src_stride,
+ TplBuffers *tpl_tmp_buffers,
+ BLOCK_SIZE bsize, TX_SIZE tx_size,
+ int mi_row, int mi_col, int rf_idx,
+ MV *rfidx_mv, int use_pred_sad) {
+ const BitDepthInfo bd_info = get_bit_depth_info(xd);
+ TplParams *tpl_data = &cpi->ppi->tpl_data;
+ const YV12_BUFFER_CONFIG *const ref_frame_ptr =
+ tpl_data->src_ref_frame[rf_idx];
+ int16_t *src_diff = tpl_tmp_buffers->src_diff;
+ tran_low_t *coeff = tpl_tmp_buffers->coeff;
+ const int bw = 4 << mi_size_wide_log2[bsize];
+ const int bh = 4 << mi_size_high_log2[bsize];
+ int32_t inter_cost;
+
+ if (cpi->sf.tpl_sf.subpel_force_stop != FULL_PEL) {
+ const int_interpfilters kernel =
+ av1_broadcast_interp_filter(EIGHTTAP_REGULAR);
+ uint8_t *predictor8 = tpl_tmp_buffers->predictor8;
+ uint8_t *predictor =
+ is_cur_buf_hbd(xd) ? CONVERT_TO_BYTEPTR(predictor8) : predictor8;
+ struct buf_2d ref_buf = { NULL, ref_frame_ptr->y_buffer,
+ ref_frame_ptr->y_width, ref_frame_ptr->y_height,
+ ref_frame_ptr->y_stride };
+ InterPredParams inter_pred_params;
+ av1_init_inter_params(&inter_pred_params, bw, bh, mi_row * MI_SIZE,
+ mi_col * MI_SIZE, 0, 0, xd->bd, is_cur_buf_hbd(xd), 0,
+ &tpl_data->sf, &ref_buf, kernel);
+ inter_pred_params.conv_params = get_conv_params(0, 0, xd->bd);
+
+ av1_enc_build_one_inter_predictor(predictor, bw, rfidx_mv,
+ &inter_pred_params);
+
+ if (use_pred_sad) {
+ inter_cost = (int)cpi->ppi->fn_ptr[bsize].sdf(src_mb_buffer, src_stride,
+ predictor, bw);
+ } else {
+ inter_cost =
+ tpl_get_satd_cost(bd_info, src_diff, bw, src_mb_buffer, src_stride,
+ predictor, bw, coeff, bw, bh, tx_size);
+ }
+ } else {
+ int ref_mb_offset =
+ mi_row * MI_SIZE * ref_frame_ptr->y_stride + mi_col * MI_SIZE;
+ uint8_t *ref_mb = ref_frame_ptr->y_buffer + ref_mb_offset;
+ int ref_stride = ref_frame_ptr->y_stride;
+ const FULLPEL_MV fullmv = get_fullmv_from_mv(rfidx_mv);
+ // Since sub-pel motion search is not performed, use the prediction pixels
+ // directly from the reference block ref_mb
+ if (use_pred_sad) {
+ inter_cost = (int)cpi->ppi->fn_ptr[bsize].sdf(
+ src_mb_buffer, src_stride,
+ &ref_mb[fullmv.row * ref_stride + fullmv.col], ref_stride);
+ } else {
+ inter_cost =
+ tpl_get_satd_cost(bd_info, src_diff, bw, src_mb_buffer, src_stride,
+ &ref_mb[fullmv.row * ref_stride + fullmv.col],
+ ref_stride, coeff, bw, bh, tx_size);
+ }
+ }
+ return inter_cost;
+}
+
static AOM_INLINE void mode_estimation(AV1_COMP *cpi,
TplTxfmStats *tpl_txfm_stats,
+ TplBuffers *tpl_tmp_buffers,
MACROBLOCK *x, int mi_row, int mi_col,
BLOCK_SIZE bsize, TX_SIZE tx_size,
TplDepStats *tpl_stats) {
@@ -470,8 +540,6 @@
const int bw = 4 << mi_size_wide_log2[bsize];
const int bh = 4 << mi_size_high_log2[bsize];
- const int_interpfilters kernel =
- av1_broadcast_interp_filter(EIGHTTAP_REGULAR);
int frame_offset = tpl_data->frame_idx - cpi->gf_frame_index;
@@ -479,9 +547,11 @@
int32_t intra_cost;
PREDICTION_MODE best_mode = DC_PRED;
- int mb_y_offset = mi_row * MI_SIZE * xd->cur_buf->y_stride + mi_col * MI_SIZE;
+ const int mb_y_offset =
+ mi_row * MI_SIZE * xd->cur_buf->y_stride + mi_col * MI_SIZE;
uint8_t *src_mb_buffer = xd->cur_buf->y_buffer + mb_y_offset;
- int src_stride = xd->cur_buf->y_stride;
+ const int src_stride = xd->cur_buf->y_stride;
+ const int src_width = xd->cur_buf->y_width;
int dst_mb_offset =
mi_row * MI_SIZE * tpl_frame->rec_picture->y_stride + mi_col * MI_SIZE;
@@ -507,29 +577,16 @@
pd->subsampling_y = xd->cur_buf->subsampling_y;
}
- // Number of pixels in a tpl block
- const int tpl_block_pels = tpl_data->tpl_bsize_1d * tpl_data->tpl_bsize_1d;
- // Allocate temporary buffers used in motion estimation.
- uint8_t *predictor8 = aom_memalign(32, tpl_block_pels * 2 * sizeof(uint8_t));
- int16_t *src_diff = aom_memalign(32, tpl_block_pels * sizeof(int16_t));
- tran_low_t *coeff = aom_memalign(32, tpl_block_pels * sizeof(tran_low_t));
- tran_low_t *qcoeff = aom_memalign(32, tpl_block_pels * sizeof(tran_low_t));
- tran_low_t *dqcoeff = aom_memalign(32, tpl_block_pels * sizeof(tran_low_t));
+ uint8_t *predictor8 = tpl_tmp_buffers->predictor8;
+ int16_t *src_diff = tpl_tmp_buffers->src_diff;
+ tran_low_t *coeff = tpl_tmp_buffers->coeff;
+ tran_low_t *qcoeff = tpl_tmp_buffers->qcoeff;
+ tran_low_t *dqcoeff = tpl_tmp_buffers->dqcoeff;
uint8_t *predictor =
is_cur_buf_hbd(xd) ? CONVERT_TO_BYTEPTR(predictor8) : predictor8;
int64_t recon_error = 1;
int64_t pred_error = 1;
- if (!(predictor8 && src_diff && coeff && qcoeff && dqcoeff)) {
- aom_free(predictor8);
- aom_free(src_diff);
- aom_free(coeff);
- aom_free(qcoeff);
- aom_free(dqcoeff);
- aom_internal_error(cm->error, AOM_CODEC_MEM_ERROR,
- "Error allocating tpl data");
- }
-
memset(tpl_stats, 0, sizeof(*tpl_stats));
tpl_stats->ref_frame_index[0] = -1;
tpl_stats->ref_frame_index[1] = -1;
@@ -576,15 +633,32 @@
tx_size, mode, 0, 0, FILTER_INTRA_MODES, dst_buffer,
dst_buffer_stride, predictor, bw, 0, 0, 0);
- intra_cost =
- tpl_get_satd_cost(bd_info, src_diff, bw, src_mb_buffer, src_stride,
- predictor, bw, coeff, bw, bh, tx_size);
+ if (tpl_frame->use_pred_sad) {
+ intra_cost = (int32_t)cpi->ppi->fn_ptr[bsize].sdf(
+ src_mb_buffer, src_stride, predictor, bw);
+ } else {
+ intra_cost =
+ tpl_get_satd_cost(bd_info, src_diff, bw, src_mb_buffer, src_stride,
+ predictor, bw, coeff, bw, bh, tx_size);
+ }
if (intra_cost < best_intra_cost) {
best_intra_cost = intra_cost;
best_mode = mode;
}
}
+ // Calculate SATD of the best intra mode if SAD was used for mode decision
+ // as best_intra_cost is used in ML model to skip intra mode evaluation.
+ if (tpl_frame->use_pred_sad) {
+ av1_predict_intra_block(
+ xd, seq_params->sb_size, seq_params->enable_intra_edge_filter,
+ block_size_wide[bsize], block_size_high[bsize], tx_size, best_mode, 0,
+ 0, FILTER_INTRA_MODES, dst_buffer, dst_buffer_stride, predictor, bw, 0,
+ 0, 0);
+ best_intra_cost =
+ tpl_get_satd_cost(bd_info, src_diff, bw, src_mb_buffer, src_stride,
+ predictor, bw, coeff, bw, bh, tx_size);
+ }
int rate_cost = 1;
@@ -653,10 +727,11 @@
}
const YV12_BUFFER_CONFIG *ref_frame_ptr = tpl_data->src_ref_frame[rf_idx];
- int ref_mb_offset =
+ const int ref_mb_offset =
mi_row * MI_SIZE * ref_frame_ptr->y_stride + mi_col * MI_SIZE;
uint8_t *ref_mb = ref_frame_ptr->y_buffer + ref_mb_offset;
- int ref_stride = ref_frame_ptr->y_stride;
+ const int ref_stride = ref_frame_ptr->y_stride;
+ const int ref_width = ref_frame_ptr->y_width;
int_mv best_rfidx_mv = { 0 };
uint32_t bestsme = UINT32_MAX;
@@ -743,9 +818,9 @@
for (idx = 0; idx < refmv_count; ++idx) {
int_mv this_mv;
- uint32_t thissme = motion_estimation(cpi, x, src_mb_buffer, ref_mb,
- src_stride, ref_stride, bsize,
- center_mvs[idx].mv.as_mv, &this_mv);
+ uint32_t thissme = motion_estimation(
+ cpi, x, src_mb_buffer, ref_mb, src_stride, ref_stride, src_width,
+ ref_width, bsize, center_mvs[idx].mv.as_mv, &this_mv);
if (thissme < bestsme) {
bestsme = thissme;
@@ -756,32 +831,10 @@
tpl_stats->mv[rf_idx].as_int = best_rfidx_mv.as_int;
single_mv[rf_idx] = best_rfidx_mv;
- if (tpl_sf->subpel_force_stop != FULL_PEL) {
- struct buf_2d ref_buf = { NULL, ref_frame_ptr->y_buffer,
- ref_frame_ptr->y_width, ref_frame_ptr->y_height,
- ref_frame_ptr->y_stride };
- InterPredParams inter_pred_params;
- av1_init_inter_params(&inter_pred_params, bw, bh, mi_row * MI_SIZE,
- mi_col * MI_SIZE, 0, 0, xd->bd, is_cur_buf_hbd(xd),
- 0, &tpl_data->sf, &ref_buf, kernel);
- inter_pred_params.conv_params = get_conv_params(0, 0, xd->bd);
-
- av1_enc_build_one_inter_predictor(predictor, bw, &best_rfidx_mv.as_mv,
- &inter_pred_params);
-
- inter_cost =
- tpl_get_satd_cost(bd_info, src_diff, bw, src_mb_buffer, src_stride,
- predictor, bw, coeff, bw, bh, tx_size);
- } else {
- const FULLPEL_MV best_fullmv = get_fullmv_from_mv(&best_rfidx_mv.as_mv);
- // Since sub-pel motion search is not performed, use the prediction pixels
- // directly from the reference block ref_mb
- inter_cost = tpl_get_satd_cost(
- bd_info, src_diff, bw, src_mb_buffer, src_stride,
- &ref_mb[best_fullmv.row * ref_stride + best_fullmv.col], ref_stride,
- coeff, bw, bh, tx_size);
- }
- // Store inter cost for each ref frame
+ inter_cost = get_inter_cost(
+ cpi, xd, src_mb_buffer, src_stride, tpl_tmp_buffers, bsize, tx_size,
+ mi_row, mi_col, rf_idx, &best_rfidx_mv.as_mv, tpl_frame->use_pred_sad);
+ // Store inter cost for each ref frame. This is used to prune inter modes.
tpl_stats->pred_error[rf_idx] = AOMMAX(1, inter_cost);
if (inter_cost < best_inter_cost) {
@@ -791,6 +844,14 @@
best_mv[0].as_int = best_rfidx_mv.as_int;
}
}
+ // Calculate SATD of the best inter mode if SAD was used for mode decision
+ // as best_inter_cost is used in ML model to skip intra mode evaluation.
+ if (best_inter_cost < INT32_MAX && tpl_frame->use_pred_sad) {
+ assert(best_rf_idx != -1);
+ best_inter_cost = get_inter_cost(
+ cpi, xd, src_mb_buffer, src_stride, tpl_tmp_buffers, bsize, tx_size,
+ mi_row, mi_col, best_rf_idx, &best_mv[0].as_mv, 0 /* use_pred_sad */);
+ }
if (best_rf_idx != -1 && best_inter_cost < best_intra_cost) {
best_mode = NEWMV;
@@ -841,6 +902,8 @@
xd->mi_row = mi_row;
xd->mi_col = mi_col;
int best_cmp_rf_idx = -1;
+ const int_interpfilters kernel =
+ av1_broadcast_interp_filter(EIGHTTAP_REGULAR);
for (int cmp_rf_idx = start_rf; cmp_rf_idx < end_rf; ++cmp_rf_idx) {
int rf_idx0 = comp_ref_frames[cmp_rf_idx][0];
int rf_idx1 = comp_ref_frames[cmp_rf_idx][1];
@@ -1039,13 +1102,6 @@
}
}
}
-
- // Free temporary buffers.
- aom_free(predictor8);
- aom_free(src_diff);
- aom_free(coeff);
- aom_free(qcoeff);
- aom_free(dqcoeff);
}
static int round_floor(int ref_pos, int bsize_pix) {
@@ -1231,9 +1287,10 @@
const YV12_BUFFER_CONFIG *ref_frames_ordered[INTER_REFS_PER_FRAME];
uint32_t ref_frame_display_indices[INTER_REFS_PER_FRAME];
const GF_GROUP *gf_group = &cpi->ppi->gf_group;
+ TPL_SPEED_FEATURES *tpl_sf = &cpi->sf.tpl_sf;
int ref_pruning_enabled = is_frame_eligible_for_ref_pruning(
gf_group, cpi->sf.inter_sf.selective_ref_frame,
- cpi->sf.tpl_sf.prune_ref_frames_in_tpl, frame_idx);
+ tpl_sf->prune_ref_frames_in_tpl, frame_idx);
int gop_length = get_gop_length(gf_group);
int ref_frame_flags;
AV1_COMMON *cm = &cpi->common;
@@ -1341,18 +1398,27 @@
av1_init_tpl_txfm_stats(tpl_txfm_stats);
// Initialize x->mbmi_ext when compound predictions are enabled.
- if (cpi->sf.tpl_sf.allow_compound_pred) av1_zero(x->mbmi_ext);
+ if (tpl_sf->allow_compound_pred) av1_zero(x->mbmi_ext);
// Set the pointer to null since mbmi is only allocated inside this function.
assert(xd->mi == &mbmi_ptr);
xd->mi = NULL;
+
+ // Tpl module is called before the setting of speed features at frame level.
+ // Thus, turning off this speed feature for key frame is done here and not
+ // integrated into the speed feature setting itself.
+ const int layer_depth_th = (tpl_sf->use_sad_for_mode_decision == 1) ? 5 : 0;
+ tpl_frame->use_pred_sad =
+ tpl_sf->use_sad_for_mode_decision &&
+ gf_group->update_type[cpi->gf_frame_index] != KF_UPDATE &&
+ gf_group->layer_depth[frame_idx] >= layer_depth_th;
}
// This function stores the motion estimation dependencies of all the blocks in
// a row
void av1_mc_flow_dispenser_row(AV1_COMP *cpi, TplTxfmStats *tpl_txfm_stats,
- MACROBLOCK *x, int mi_row, BLOCK_SIZE bsize,
- TX_SIZE tx_size) {
+ TplBuffers *tpl_tmp_buffers, MACROBLOCK *x,
+ int mi_row, BLOCK_SIZE bsize, TX_SIZE tx_size) {
AV1_COMMON *const cm = &cpi->common;
MultiThreadInfo *const mt_info = &cpi->mt_info;
AV1TplRowMultiThreadInfo *const tpl_row_mt = &mt_info->tpl_row_mt;
@@ -1372,6 +1438,17 @@
mi_col += mi_width, tplb_col_in_tile++) {
(*tpl_row_mt->sync_read_ptr)(&tpl_data->tpl_mt_sync, tplb_row,
tplb_col_in_tile);
+
+#if CONFIG_MULTITHREAD
+ if (mt_info->num_workers > 1) {
+ pthread_mutex_lock(tpl_row_mt->mutex_);
+ const bool tpl_mt_exit = tpl_row_mt->tpl_mt_exit;
+ pthread_mutex_unlock(tpl_row_mt->mutex_);
+ // Exit in case any worker has encountered an error.
+ if (tpl_mt_exit) return;
+ }
+#endif
+
TplDepStats tpl_stats;
// Motion estimation column boundary
@@ -1380,8 +1457,8 @@
xd->mb_to_left_edge = -GET_MV_SUBPEL(mi_col * MI_SIZE);
xd->mb_to_right_edge =
GET_MV_SUBPEL(mi_params->mi_cols - mi_width - mi_col);
- mode_estimation(cpi, tpl_txfm_stats, x, mi_row, mi_col, bsize, tx_size,
- &tpl_stats);
+ mode_estimation(cpi, tpl_txfm_stats, tpl_tmp_buffers, x, mi_row, mi_col,
+ bsize, tx_size, &tpl_stats);
// Motion flow dependency dispenser.
tpl_model_store(tpl_frame->tpl_stats_ptr, mi_row, mi_col, tpl_frame->stride,
@@ -1408,8 +1485,8 @@
xd->mb_to_top_edge = -GET_MV_SUBPEL(mi_row * MI_SIZE);
xd->mb_to_bottom_edge =
GET_MV_SUBPEL((mi_params->mi_rows - mi_height - mi_row) * MI_SIZE);
- av1_mc_flow_dispenser_row(cpi, &td->tpl_txfm_stats, x, mi_row, bsize,
- tx_size);
+ av1_mc_flow_dispenser_row(cpi, &td->tpl_txfm_stats, &td->tpl_tmp_buffers, x,
+ mi_row, bsize, tx_size);
}
}
@@ -1703,6 +1780,34 @@
}
}
+static AOM_INLINE int skip_tpl_for_frame(const GF_GROUP *gf_group,
+ int frame_idx, int gop_eval,
+ int approx_gop_eval,
+ int reduce_num_frames) {
+ // When gop_eval is set to 2, tpl stats calculation is done for ARFs from base
+ // layer, (base+1) layer and (base+2) layer. When gop_eval is set to 3,
+ // tpl stats calculation is limited to ARFs from base layer and (base+1)
+ // layer.
+ const int num_arf_layers = (gop_eval == 2) ? 3 : 2;
+ const int gop_length = get_gop_length(gf_group);
+
+ if (gf_group->update_type[frame_idx] == INTNL_OVERLAY_UPDATE ||
+ gf_group->update_type[frame_idx] == OVERLAY_UPDATE)
+ return 1;
+
+ // When approx_gop_eval = 1, skip tpl stats calculation for higher layer
+ // frames and for frames beyond gop length.
+ if (approx_gop_eval && (gf_group->layer_depth[frame_idx] > num_arf_layers ||
+ frame_idx >= gop_length))
+ return 1;
+
+ if (reduce_num_frames && gf_group->update_type[frame_idx] == LF_UPDATE &&
+ frame_idx < gop_length)
+ return 1;
+
+ return 0;
+}
+
int av1_tpl_setup_stats(AV1_COMP *cpi, int gop_eval,
const EncodeFrameParams *const frame_params) {
#if CONFIG_COLLECT_COMPONENT_TIMING
@@ -1716,13 +1821,6 @@
EncodeFrameParams this_frame_params = *frame_params;
TplParams *const tpl_data = &cpi->ppi->tpl_data;
int approx_gop_eval = (gop_eval > 1);
- int num_arf_layers = MAX_ARF_LAYERS;
-
- // When gop_eval is set to 2, tpl stats calculation is done for ARFs from base
- // layer, (base+1) layer and (base+2) layer. When gop_eval is set to 3,
- // tpl stats calculation is limited to ARFs from base layer and (base+1)
- // layer.
- if (approx_gop_eval) num_arf_layers = (gop_eval == 2) ? 3 : 2;
if (cpi->superres_mode != AOM_SUPERRES_NONE) {
assert(cpi->superres_mode != AOM_SUPERRES_AUTO);
@@ -1751,6 +1849,12 @@
av1_init_tpl_stats(tpl_data);
+ TplBuffers *tpl_tmp_buffers = &cpi->td.tpl_tmp_buffers;
+ if (!tpl_alloc_temp_buffers(tpl_tmp_buffers, tpl_data->tpl_bsize_1d)) {
+ aom_internal_error(cpi->common.error, AOM_CODEC_MEM_ERROR,
+ "Error allocating tpl data");
+ }
+
tpl_row_mt->sync_read_ptr = av1_tpl_row_mt_sync_read_dummy;
tpl_row_mt->sync_write_ptr = av1_tpl_row_mt_sync_write_dummy;
@@ -1763,20 +1867,26 @@
av1_fill_mv_costs(&cm->fc->nmvc, cm->features.cur_frame_force_integer_mv,
cm->features.allow_high_precision_mv, cpi->td.mb.mv_costs);
- const int gop_length = get_gop_length(gf_group);
const int num_planes =
cpi->sf.tpl_sf.use_y_only_rate_distortion ? 1 : av1_num_planes(cm);
+ // As tpl module is called before the setting of speed features at frame
+ // level, turning off this speed feature for the first GF group of the
+ // key-frame interval is done here.
+ int reduce_num_frames =
+ cpi->sf.tpl_sf.reduce_num_frames &&
+ gf_group->update_type[cpi->gf_frame_index] != KF_UPDATE &&
+ gf_group->max_layer_depth > 2;
+ // TPL processing is skipped for frames of type LF_UPDATE when
+ // 'reduce_num_frames' is 1, which affects the r0 calcuation. Thus, a factor
+ // to adjust r0 is used. The value of 1.6 corresponds to using ~60% of the
+ // frames in the gf group on an average.
+ tpl_data->r0_adjust_factor = reduce_num_frames ? 1.6 : 1.0;
+
// Backward propagation from tpl_group_frames to 1.
for (int frame_idx = cpi->gf_frame_index; frame_idx < tpl_gf_group_frames;
++frame_idx) {
- if (gf_group->update_type[frame_idx] == INTNL_OVERLAY_UPDATE ||
- gf_group->update_type[frame_idx] == OVERLAY_UPDATE)
- continue;
-
- // When approx_gop_eval = 1, skip tpl stats calculation for higher layer
- // frames and for frames beyond gop length.
- if (approx_gop_eval && (gf_group->layer_depth[frame_idx] > num_arf_layers ||
- frame_idx >= gop_length))
+ if (skip_tpl_for_frame(gf_group, frame_idx, gop_eval, approx_gop_eval,
+ reduce_num_frames))
continue;
init_mc_flow_dispenser(cpi, frame_idx, pframe_qindex);
@@ -1806,12 +1916,8 @@
for (int frame_idx = tpl_gf_group_frames - 1;
frame_idx >= cpi->gf_frame_index; --frame_idx) {
- if (gf_group->update_type[frame_idx] == INTNL_OVERLAY_UPDATE ||
- gf_group->update_type[frame_idx] == OVERLAY_UPDATE)
- continue;
-
- if (approx_gop_eval && (gf_group->layer_depth[frame_idx] > num_arf_layers ||
- frame_idx >= gop_length))
+ if (skip_tpl_for_frame(gf_group, frame_idx, gop_eval, approx_gop_eval,
+ reduce_num_frames))
continue;
mc_flow_synthesizer(tpl_data, frame_idx, cm->mi_params.mi_rows,
@@ -1831,6 +1937,8 @@
end_timing(cpi, av1_tpl_setup_stats_time);
#endif
+ tpl_dealloc_temp_buffers(tpl_tmp_buffers);
+
if (!approx_gop_eval) {
tpl_data->ready = 1;
}
diff --git a/av1/encoder/tpl_model.h b/av1/encoder/tpl_model.h
index 36c3ae0..bcd5821 100644
--- a/av1/encoder/tpl_model.h
+++ b/av1/encoder/tpl_model.h
@@ -24,6 +24,7 @@
struct EncodeFrameParams;
struct EncodeFrameInput;
struct GF_GROUP;
+struct ThreadData;
struct TPL_INFO;
#include "config/aom_config.h"
@@ -70,6 +71,13 @@
} AV1TplRowMultiThreadSync;
typedef struct AV1TplRowMultiThreadInfo {
+ // Initialized to false, set to true by the worker thread that encounters an
+ // error in order to abort the processing of other worker threads.
+ bool tpl_mt_exit;
+#if CONFIG_MULTITHREAD
+ // Mutex lock object used for error handling.
+ pthread_mutex_t *mutex_;
+#endif
// Row synchronization related function pointers.
void (*sync_read_ptr)(AV1TplRowMultiThreadSync *tpl_mt_sync, int r, int c);
void (*sync_write_ptr)(AV1TplRowMultiThreadSync *tpl_mt_sync, int r, int c,
@@ -103,6 +111,14 @@
int coeff_num;
} TplTxfmStats;
+typedef struct {
+ uint8_t *predictor8;
+ int16_t *src_diff;
+ tran_low_t *coeff;
+ tran_low_t *qcoeff;
+ tran_low_t *dqcoeff;
+} TplBuffers;
+
typedef struct TplDepStats {
int64_t srcrf_sse;
int64_t srcrf_dist;
@@ -137,6 +153,8 @@
int mi_cols;
int base_rdmult;
uint32_t frame_display_index;
+ // When set, SAD metric is used for intra and inter mode decision.
+ int use_pred_sad;
} TplDepFrame;
/*!\endcond */
@@ -227,6 +245,10 @@
*/
int border_in_pixels;
+ /*!
+ * Factor to adjust r0 if TPL uses a subset of frames in the gf group.
+ */
+ double r0_adjust_factor;
} TplParams;
#if CONFIG_BITRATE_ACCURACY || CONFIG_RATECTRL_LOG
@@ -393,6 +415,45 @@
CommonModeInfoParams *const mi_params, int width,
int height, int byte_alignment, int lag_in_frames);
+static AOM_INLINE void tpl_dealloc_temp_buffers(TplBuffers *tpl_tmp_buffers) {
+ aom_free(tpl_tmp_buffers->predictor8);
+ tpl_tmp_buffers->predictor8 = NULL;
+ aom_free(tpl_tmp_buffers->src_diff);
+ tpl_tmp_buffers->src_diff = NULL;
+ aom_free(tpl_tmp_buffers->coeff);
+ tpl_tmp_buffers->coeff = NULL;
+ aom_free(tpl_tmp_buffers->qcoeff);
+ tpl_tmp_buffers->qcoeff = NULL;
+ aom_free(tpl_tmp_buffers->dqcoeff);
+ tpl_tmp_buffers->dqcoeff = NULL;
+}
+
+static AOM_INLINE bool tpl_alloc_temp_buffers(TplBuffers *tpl_tmp_buffers,
+ uint8_t tpl_bsize_1d) {
+ // Number of pixels in a tpl block
+ const int tpl_block_pels = tpl_bsize_1d * tpl_bsize_1d;
+
+ // Allocate temporary buffers used in mode estimation.
+ tpl_tmp_buffers->predictor8 = (uint8_t *)aom_memalign(
+ 32, tpl_block_pels * 2 * sizeof(*tpl_tmp_buffers->predictor8));
+ tpl_tmp_buffers->src_diff = (int16_t *)aom_memalign(
+ 32, tpl_block_pels * sizeof(*tpl_tmp_buffers->src_diff));
+ tpl_tmp_buffers->coeff = (tran_low_t *)aom_memalign(
+ 32, tpl_block_pels * sizeof(*tpl_tmp_buffers->coeff));
+ tpl_tmp_buffers->qcoeff = (tran_low_t *)aom_memalign(
+ 32, tpl_block_pels * sizeof(*tpl_tmp_buffers->qcoeff));
+ tpl_tmp_buffers->dqcoeff = (tran_low_t *)aom_memalign(
+ 32, tpl_block_pels * sizeof(*tpl_tmp_buffers->dqcoeff));
+
+ if (!(tpl_tmp_buffers->predictor8 && tpl_tmp_buffers->src_diff &&
+ tpl_tmp_buffers->coeff && tpl_tmp_buffers->qcoeff &&
+ tpl_tmp_buffers->dqcoeff)) {
+ tpl_dealloc_temp_buffers(tpl_tmp_buffers);
+ return false;
+ }
+ return true;
+}
+
/*!\brief Implements temporal dependency modelling for a GOP (GF/ARF
* group) and selects between 16 and 32 frame GOP structure.
*
@@ -424,7 +485,8 @@
BLOCK_SIZE sb_size, int mi_row, int mi_col);
void av1_mc_flow_dispenser_row(struct AV1_COMP *cpi,
- TplTxfmStats *tpl_txfm_stats, MACROBLOCK *x,
+ TplTxfmStats *tpl_txfm_stats,
+ TplBuffers *tpl_tmp_buffers, MACROBLOCK *x,
int mi_row, BLOCK_SIZE bsize, TX_SIZE tx_size);
/*!\brief Compute the entropy of an exponential probability distribution
diff --git a/av1/encoder/tune_butteraugli.c b/av1/encoder/tune_butteraugli.c
index 8f59373..92fc4b2 100644
--- a/av1/encoder/tune_butteraugli.c
+++ b/av1/encoder/tune_butteraugli.c
@@ -220,8 +220,11 @@
cm->seq_params->use_highbitdepth, cpi->oxcf.border_in_pixels,
cm->features.byte_alignment, 0, 0);
}
- av1_resize_and_extend_frame_nonnormative(cpi->source, resized_dst, bit_depth,
- av1_num_planes(cm));
+ if (!av1_resize_and_extend_frame_nonnormative(
+ cpi->source, resized_dst, bit_depth, av1_num_planes(cm))) {
+ aom_internal_error(cm->error, AOM_CODEC_MEM_ERROR,
+ "Error allocating buffers during resize");
+ }
zero_img(cpi->source);
copy_img(resized_dst, cpi->source, width / resize_factor,
diff --git a/av1/encoder/tune_vmaf.c b/av1/encoder/tune_vmaf.c
index 9c7c112..4e5ffa3 100644
--- a/av1/encoder/tune_vmaf.c
+++ b/av1/encoder/tune_vmaf.c
@@ -42,6 +42,7 @@
// Parameters used for motion search.
FULLPEL_MOTION_SEARCH_PARAMS full_ms_params;
+ FULLPEL_MV_STATS best_mv_stats;
const SEARCH_METHODS search_method = NSTEP;
const search_site_config *search_site_cfg =
cpi->mv_search_params.search_site_cfg[SS_CFG_FPF];
@@ -64,10 +65,11 @@
// Only do full search on the entire block.
av1_make_default_fullpel_ms_params(&full_ms_params, cpi, mb, block_size,
&baseline_mv, *ref_mv, search_site_cfg,
+ search_method,
/*fine_search_interval=*/0);
- av1_set_mv_search_method(&full_ms_params, search_site_cfg, search_method);
av1_full_pixel_search(*ref_mv, &full_ms_params, step_param,
- cond_cost_list(cpi, cost_list), ref_mv, NULL);
+ cond_cost_list(cpi, cost_list), ref_mv, &best_mv_stats,
+ NULL);
// Restore input state.
mb->plane[0].src = ori_src_buf;
@@ -621,8 +623,11 @@
&resized_source, y_width / resize_factor, y_height / resize_factor, ss_x,
ss_y, cm->seq_params->use_highbitdepth, cpi->oxcf.border_in_pixels,
cm->features.byte_alignment, 0, 0);
- av1_resize_and_extend_frame_nonnormative(cpi->source, &resized_source,
- bit_depth, av1_num_planes(cm));
+ if (!av1_resize_and_extend_frame_nonnormative(
+ cpi->source, &resized_source, bit_depth, av1_num_planes(cm))) {
+ aom_internal_error(cm->error, AOM_CODEC_MEM_ERROR,
+ "Error allocating buffers during resize");
+ }
const int resized_y_width = resized_source.y_width;
const int resized_y_height = resized_source.y_height;
diff --git a/av1/encoder/tx_search.c b/av1/encoder/tx_search.c
index d6217b7..7292c01 100644
--- a/av1/encoder/tx_search.c
+++ b/av1/encoder/tx_search.c
@@ -119,13 +119,10 @@
*rd_stats = mb_rd_info->rd_stats;
}
-// Compute the pixel domain distortion from diff on all visible 4x4s in the
-// transform block.
-static INLINE int64_t pixel_diff_dist(const MACROBLOCK *x, int plane,
- int blk_row, int blk_col,
- const BLOCK_SIZE plane_bsize,
- const BLOCK_SIZE tx_bsize,
- unsigned int *block_mse_q8) {
+int64_t av1_pixel_diff_dist(const MACROBLOCK *x, int plane, int blk_row,
+ int blk_col, const BLOCK_SIZE plane_bsize,
+ const BLOCK_SIZE tx_bsize,
+ unsigned int *block_mse_q8) {
int visible_rows, visible_cols;
const MACROBLOCKD *xd = &x->e_mbd;
get_txb_dimensions(xd, plane, plane_bsize, blk_row, blk_col, tx_bsize, NULL,
@@ -188,7 +185,7 @@
const MACROBLOCKD *xd = &x->e_mbd;
const int16_t dc_q = av1_dc_quant_QTX(x->qindex, 0, xd->bd);
- *dist = pixel_diff_dist(x, 0, 0, 0, bsize, bsize, NULL);
+ *dist = av1_pixel_diff_dist(x, 0, 0, 0, bsize, bsize, NULL);
const int64_t mse = *dist / bw / bh;
// Normalized quantizer takes the transform upscaling factor (8 for tx size
@@ -243,7 +240,7 @@
// Used to set proper context for early termination with skip = 1.
static AOM_INLINE void set_skip_txfm(MACROBLOCK *x, RD_STATS *rd_stats,
- int bsize, int64_t dist) {
+ BLOCK_SIZE bsize, int64_t dist) {
MACROBLOCKD *const xd = &x->e_mbd;
MB_MODE_INFO *const mbmi = xd->mi[0];
const int n4 = bsize_to_num_blk(bsize);
@@ -644,7 +641,7 @@
get_plane_block_size(mbmi->bsize, pd->subsampling_x, pd->subsampling_y);
unsigned int sse;
- if (x->skip_chroma_rd && plane) continue;
+ if (plane) continue;
cpi->ppi->fn_ptr[bs].vf(p->src.buf, p->src.stride, pd->dst.buf,
pd->dst.stride, &sse);
@@ -2030,12 +2027,9 @@
uint16_t best_eob = 0;
TX_TYPE best_tx_type = DCT_DCT;
int rate_cost = 0;
- // The buffer used to swap dqcoeff in macroblockd_plane so we can keep dqcoeff
- // of the best tx_type
- DECLARE_ALIGNED(32, tran_low_t, this_dqcoeff[MAX_SB_SQUARE]);
struct macroblock_plane *const p = &x->plane[plane];
tran_low_t *orig_dqcoeff = p->dqcoeff;
- tran_low_t *best_dqcoeff = this_dqcoeff;
+ tran_low_t *best_dqcoeff = x->dqcoeff_buf;
const int tx_type_map_idx =
plane ? 0 : blk_row * xd->tx_type_map_stride + blk_col;
av1_invalid_rd_stats(best_rd_stats);
@@ -2071,8 +2065,8 @@
return;
}
} else {
- block_sse = pixel_diff_dist(x, plane, blk_row, blk_col, plane_bsize,
- txsize_to_bsize[tx_size], &block_mse_q8);
+ block_sse = av1_pixel_diff_dist(x, plane, blk_row, blk_col, plane_bsize,
+ txsize_to_bsize[tx_size], &block_mse_q8);
assert(block_mse_q8 != UINT_MAX);
}
@@ -2080,7 +2074,7 @@
uint16_t tx_mask;
// Use DCT_DCT transform for DC only block.
- if (dc_only_blk)
+ if (dc_only_blk || cpi->sf.rt_sf.dct_only_palette_nonrd == 1)
tx_mask = 1 << DCT_DCT;
else
tx_mask = get_tx_mask(cpi, x, plane, block, blk_row, blk_col, plane_bsize,
diff --git a/av1/encoder/tx_search.h b/av1/encoder/tx_search.h
index b3689cf..ed95c1c 100644
--- a/av1/encoder/tx_search.h
+++ b/av1/encoder/tx_search.h
@@ -47,6 +47,27 @@
return x->mode_costs.tx_size_cost[tx_size_cat][tx_size_ctx][depth];
}
+/*!\brief Compute the pixel domain distortion.
+ *
+ * \ingroup transform_search
+ * Compute the pixel domain distortion from diff on all visible 4x4s in the
+ * transform block.
+ *
+ * \param[in] x Pointer to structure holding the data for the
+ current encoding macroblock
+ * \param[in] plane Plane index
+ * \param[in] blk_row Block row index
+ * \param[in] blk_col Block col index
+ * \param[in] plane_bsize Current plane block size
+ * \param[in] tx_bsize Transform size
+ * \param[in] block_mse_q8 Block mse
+ * \return An int64_t value that is the block sse.
+ */
+int64_t av1_pixel_diff_dist(const MACROBLOCK *x, int plane, int blk_row,
+ int blk_col, const BLOCK_SIZE plane_bsize,
+ const BLOCK_SIZE tx_bsize,
+ unsigned int *block_mse_q8);
+
int64_t av1_estimate_txfm_yrd(const AV1_COMP *const cpi, MACROBLOCK *x,
RD_STATS *rd_stats, int64_t ref_best_rd,
BLOCK_SIZE bs, TX_SIZE tx_size);
diff --git a/av1/encoder/var_based_part.c b/av1/encoder/var_based_part.c
index 5b8f598..d545600 100644
--- a/av1/encoder/var_based_part.c
+++ b/av1/encoder/var_based_part.c
@@ -29,6 +29,7 @@
#include "av1/encoder/encodeframe.h"
#include "av1/encoder/var_based_part.h"
#include "av1/encoder/reconinter_enc.h"
+#include "av1/encoder/rdopt_utils.h"
// Possible values for the force_split variable while evaluating variance based
// partitioning.
@@ -1021,8 +1022,7 @@
if (cpi->oxcf.tune_cfg.content == AOM_CONTENT_SCREEN &&
cpi->rc.high_source_sad) {
shift_lower_limit = 7;
- } else if (source_sad_nonrd >= kMedSad &&
- cpi->oxcf.tune_cfg.content != AOM_CONTENT_SCREEN &&
+ } else if (source_sad_nonrd >= kMedSad && x->source_variance > 500 &&
cpi->common.width * cpi->common.height >= 640 * 360) {
shift_upper_limit = 2;
shift_lower_limit = source_sad_nonrd > kMedSad ? 5 : 4;
@@ -1243,6 +1243,7 @@
*y_sad = *y_sad_g;
*ref_frame_partition = GOLDEN_FRAME;
x->nonrd_prune_ref_frame_search = 0;
+ x->sb_me_partition = 0;
} else if (is_set_altref_ref_frame) {
av1_setup_pre_planes(xd, 0, yv12_alt, mi_row, mi_col,
get_ref_scale_factors(cm, ALTREF_FRAME), num_planes);
@@ -1251,6 +1252,7 @@
*y_sad = *y_sad_alt;
*ref_frame_partition = ALTREF_FRAME;
x->nonrd_prune_ref_frame_search = 0;
+ x->sb_me_partition = 0;
} else {
*ref_frame_partition = LAST_FRAME;
x->nonrd_prune_ref_frame_search =
@@ -1339,11 +1341,14 @@
static void setup_planes(AV1_COMP *cpi, MACROBLOCK *x, unsigned int *y_sad,
unsigned int *y_sad_g, unsigned int *y_sad_alt,
unsigned int *y_sad_last,
- MV_REFERENCE_FRAME *ref_frame_partition, int mi_row,
+ MV_REFERENCE_FRAME *ref_frame_partition,
+ struct scale_factors *sf_no_scale, int mi_row,
int mi_col, bool is_small_sb, bool scaled_ref_last) {
AV1_COMMON *const cm = &cpi->common;
MACROBLOCKD *xd = &x->e_mbd;
const int num_planes = av1_num_planes(cm);
+ bool scaled_ref_golden = false;
+ bool scaled_ref_alt = false;
BLOCK_SIZE bsize = is_small_sb ? BLOCK_64X64 : BLOCK_128X128;
MB_MODE_INFO *mi = xd->mi[0];
const YV12_BUFFER_CONFIG *yv12 =
@@ -1361,21 +1366,22 @@
cpi->sf.rt_sf.use_nonrd_altref_frame ||
(cpi->sf.rt_sf.use_comp_ref_nonrd &&
cpi->sf.rt_sf.ref_frame_comp_nonrd[2] == 1);
- // On a resized frame (reference has different scale) only use
- // LAST as reference for partitioning for now.
- if (scaled_ref_last) {
- use_golden_ref = 0;
- use_alt_ref = 0;
- }
// For 1 spatial layer: GOLDEN is another temporal reference.
// Check if it should be used as reference for partitioning.
if (cpi->svc.number_spatial_layers == 1 && use_golden_ref &&
(x->content_state_sb.source_sad_nonrd != kZeroSad || !use_last_ref)) {
yv12_g = get_ref_frame_yv12_buf(cm, GOLDEN_FRAME);
+ if (yv12_g && (yv12_g->y_crop_height != cm->height ||
+ yv12_g->y_crop_width != cm->width)) {
+ yv12_g = av1_get_scaled_ref_frame(cpi, GOLDEN_FRAME);
+ scaled_ref_golden = true;
+ }
if (yv12_g && yv12_g != yv12) {
- av1_setup_pre_planes(xd, 0, yv12_g, mi_row, mi_col,
- get_ref_scale_factors(cm, GOLDEN_FRAME), num_planes);
+ av1_setup_pre_planes(
+ xd, 0, yv12_g, mi_row, mi_col,
+ scaled_ref_golden ? NULL : get_ref_scale_factors(cm, GOLDEN_FRAME),
+ num_planes);
*y_sad_g = cpi->ppi->fn_ptr[bsize].sdf(
x->plane[AOM_PLANE_Y].src.buf, x->plane[AOM_PLANE_Y].src.stride,
xd->plane[AOM_PLANE_Y].pre[0].buf,
@@ -1389,9 +1395,16 @@
(cpi->ref_frame_flags & AOM_ALT_FLAG) &&
(x->content_state_sb.source_sad_nonrd != kZeroSad || !use_last_ref)) {
yv12_alt = get_ref_frame_yv12_buf(cm, ALTREF_FRAME);
+ if (yv12_alt && (yv12_alt->y_crop_height != cm->height ||
+ yv12_alt->y_crop_width != cm->width)) {
+ yv12_alt = av1_get_scaled_ref_frame(cpi, ALTREF_FRAME);
+ scaled_ref_alt = true;
+ }
if (yv12_alt && yv12_alt != yv12) {
- av1_setup_pre_planes(xd, 0, yv12_alt, mi_row, mi_col,
- get_ref_scale_factors(cm, ALTREF_FRAME), num_planes);
+ av1_setup_pre_planes(
+ xd, 0, yv12_alt, mi_row, mi_col,
+ scaled_ref_alt ? NULL : get_ref_scale_factors(cm, ALTREF_FRAME),
+ num_planes);
*y_sad_alt = cpi->ppi->fn_ptr[bsize].sdf(
x->plane[AOM_PLANE_Y].src.buf, x->plane[AOM_PLANE_Y].src.stride,
xd->plane[AOM_PLANE_Y].pre[0].buf,
@@ -1420,9 +1433,37 @@
if (est_motion == 1 || est_motion == 2) {
if (xd->mb_to_right_edge >= 0 && xd->mb_to_bottom_edge >= 0) {
- const MV dummy_mv = { 0, 0 };
- *y_sad = av1_int_pro_motion_estimation(cpi, x, cm->seq_params->sb_size,
- mi_row, mi_col, &dummy_mv);
+ // For screen only do int_pro_motion for spatial variance above
+ // threshold and motion level above LowSad.
+ if (x->source_variance > 100 && source_sad_nonrd > kLowSad) {
+ int is_screen = cpi->oxcf.tune_cfg.content == AOM_CONTENT_SCREEN;
+ int me_search_size_col =
+ is_screen ? 96 : block_size_wide[cm->seq_params->sb_size] >> 1;
+ // For screen use larger search size row motion to capture
+ // vertical scroll, which can be larger motion.
+ int me_search_size_row =
+ is_screen ? 192 : block_size_high[cm->seq_params->sb_size] >> 1;
+ unsigned int y_sad_zero;
+ *y_sad = av1_int_pro_motion_estimation(
+ cpi, x, cm->seq_params->sb_size, mi_row, mi_col, &kZeroMv,
+ &y_sad_zero, me_search_size_col, me_search_size_row);
+ // The logic below selects whether the motion estimated in the
+ // int_pro_motion() will be used in nonrd_pickmode. Only do this
+ // for screen for now.
+ if (is_screen) {
+ unsigned int thresh_sad =
+ (cm->seq_params->sb_size == BLOCK_128X128) ? 50000 : 20000;
+ if (*y_sad < (y_sad_zero >> 1) && *y_sad < thresh_sad) {
+ x->sb_me_partition = 1;
+ x->sb_me_mv.as_int = mi->mv[0].as_int;
+ } else {
+ x->sb_me_partition = 0;
+ // Fall back to using zero motion.
+ *y_sad = y_sad_zero;
+ mi->mv[0].as_int = 0;
+ }
+ }
+ }
}
}
@@ -1450,7 +1491,12 @@
// Only calculate the predictor for non-zero MV.
if (mi->mv[0].as_int != 0) {
- set_ref_ptrs(cm, xd, mi->ref_frame[0], mi->ref_frame[1]);
+ if (!scaled_ref_last) {
+ set_ref_ptrs(cm, xd, mi->ref_frame[0], mi->ref_frame[1]);
+ } else {
+ xd->block_ref_scale_factors[0] = sf_no_scale;
+ xd->block_ref_scale_factors[1] = sf_no_scale;
+ }
av1_enc_build_inter_predictor(cm, xd, mi_row, mi_col, NULL,
cm->seq_params->sb_size, AOM_PLANE_Y,
num_planes - 1);
@@ -1517,8 +1563,8 @@
uv_sad[0] < thresh_exit_part_uv && uv_sad[1] < thresh_exit_part_uv) {
set_block_size(cpi, mi_row, mi_col, bsize);
x->force_zeromv_skip_for_sb = 1;
- if (vt2) aom_free(vt2);
- if (vt) aom_free(vt);
+ aom_free(vt2);
+ aom_free(vt);
// Partition shape is set here at SB level.
// Exit needs to happen from av1_choose_var_based_partitioning().
return true;
@@ -1558,6 +1604,9 @@
NOISE_LEVEL noise_level = kLow;
bool is_zero_motion = true;
bool scaled_ref_last = false;
+ struct scale_factors sf_no_scale;
+ av1_setup_scale_factors_for_frame(&sf_no_scale, cm->width, cm->height,
+ cm->width, cm->height);
bool is_key_frame =
(frame_is_intra_only(cm) ||
@@ -1578,7 +1627,7 @@
// Ref frame used in partitioning.
MV_REFERENCE_FRAME ref_frame_partition = LAST_FRAME;
- CHECK_MEM_ERROR(cm, vt, aom_malloc(sizeof(*vt)));
+ AOM_CHECK_MEM_ERROR(xd->error_info, vt, aom_malloc(sizeof(*vt)));
vt->split = td->vt64x64;
@@ -1645,10 +1694,19 @@
}
}
+ x->source_variance = UINT_MAX;
+ // For nord_pickmode: compute source_variance, only for superblocks with
+ // some motion for now. This input can then be used to bias the partitioning
+ // or the chroma_check.
+ if (cpi->sf.rt_sf.use_nonrd_pick_mode &&
+ x->content_state_sb.source_sad_nonrd > kLowSad)
+ x->source_variance = av1_get_perpixel_variance_facade(
+ cpi, xd, &x->plane[0].src, cm->seq_params->sb_size, AOM_PLANE_Y);
+
if (!is_key_frame) {
setup_planes(cpi, x, &y_sad, &y_sad_g, &y_sad_alt, &y_sad_last,
- &ref_frame_partition, mi_row, mi_col, is_small_sb,
- scaled_ref_last);
+ &ref_frame_partition, &sf_no_scale, mi_row, mi_col,
+ is_small_sb, scaled_ref_last);
MB_MODE_INFO *mi = xd->mi[0];
// Use reference SB directly for zero mv.
@@ -1690,8 +1748,14 @@
if (cpi->noise_estimate.enabled)
noise_level = av1_noise_estimate_extract_level(&cpi->noise_estimate);
- if (low_res && threshold_4x4avg < INT64_MAX)
- CHECK_MEM_ERROR(cm, vt2, aom_malloc(sizeof(*vt2)));
+ if (low_res && threshold_4x4avg < INT64_MAX) {
+ vt2 = aom_malloc(sizeof(*vt2));
+ if (!vt2) {
+ aom_free(vt);
+ aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR,
+ "Error allocating partition buffer vt2");
+ }
+ }
// Fill in the entire tree of 8x8 (or 4x4 under some conditions) variances
// for splits.
fill_variance_tree_leaves(cpi, x, vt, force_split, avg_16x16, maxvar_16x16,
@@ -1869,8 +1933,8 @@
ref_frame_partition, mi_col, mi_row, is_small_sb);
}
- if (vt2) aom_free(vt2);
- if (vt) aom_free(vt);
+ aom_free(vt2);
+ aom_free(vt);
#if CONFIG_COLLECT_COMPONENT_TIMING
end_timing(cpi, choose_var_based_partitioning_time);
#endif
diff --git a/av1/encoder/x86/av1_k_means_avx2.c b/av1/encoder/x86/av1_k_means_avx2.c
index ad0b374..52ddc66 100644
--- a/av1/encoder/x86/av1_k_means_avx2.c
+++ b/av1/encoder/x86/av1_k_means_avx2.c
@@ -10,7 +10,7 @@
*/
#include <immintrin.h> // AVX2
-#include "config/aom_dsp_rtcd.h"
+#include "config/av1_rtcd.h"
#include "aom_dsp/x86/synonyms.h"
static int64_t k_means_horizontal_sum_avx2(__m256i a) {
diff --git a/av1/encoder/x86/av1_k_means_sse2.c b/av1/encoder/x86/av1_k_means_sse2.c
index 4338bf7..6c75822 100644
--- a/av1/encoder/x86/av1_k_means_sse2.c
+++ b/av1/encoder/x86/av1_k_means_sse2.c
@@ -11,7 +11,7 @@
#include <emmintrin.h> // SSE2
-#include "config/aom_dsp_rtcd.h"
+#include "config/av1_rtcd.h"
#include "aom_dsp/x86/synonyms.h"
static int64_t k_means_horizontal_sum_sse2(__m128i a) {
diff --git a/av1/encoder/x86/hash_sse42.c b/av1/encoder/x86/hash_sse42.c
index 9e06ebe..ebe7531 100644
--- a/av1/encoder/x86/hash_sse42.c
+++ b/av1/encoder/x86/hash_sse42.c
@@ -12,6 +12,8 @@
#include <stdint.h>
#include <smmintrin.h>
+#include "config/av1_rtcd.h"
+
// Byte-boundary alignment issues
#define ALIGN_SIZE 8
#define ALIGN_MASK (ALIGN_SIZE - 1)
diff --git a/av1/encoder/x86/highbd_block_error_intrin_avx2.c b/av1/encoder/x86/highbd_block_error_intrin_avx2.c
index ee3714d..340307c 100644
--- a/av1/encoder/x86/highbd_block_error_intrin_avx2.c
+++ b/av1/encoder/x86/highbd_block_error_intrin_avx2.c
@@ -13,6 +13,7 @@
#include <stdio.h>
#include "aom/aom_integer.h"
#include "av1/common/common.h"
+#include "config/av1_rtcd.h"
int64_t av1_highbd_block_error_avx2(const tran_low_t *coeff,
const tran_low_t *dqcoeff,
diff --git a/av1/encoder/x86/highbd_block_error_intrin_sse2.c b/av1/encoder/x86/highbd_block_error_intrin_sse2.c
index 0287f01..b0b2757 100644
--- a/av1/encoder/x86/highbd_block_error_intrin_sse2.c
+++ b/av1/encoder/x86/highbd_block_error_intrin_sse2.c
@@ -13,6 +13,7 @@
#include <stdio.h>
#include "av1/common/common.h"
+#include "config/av1_rtcd.h"
int64_t av1_highbd_block_error_sse2(const tran_low_t *coeff,
const tran_low_t *dqcoeff,
diff --git a/av1/encoder/x86/ml_avx2.c b/av1/encoder/x86/ml_avx2.c
new file mode 100644
index 0000000..6432708
--- /dev/null
+++ b/av1/encoder/x86/ml_avx2.c
@@ -0,0 +1,240 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <stdbool.h>
+#include <assert.h>
+#include <immintrin.h>
+
+#include "config/av1_rtcd.h"
+#include "av1/encoder/ml.h"
+#include "av1/encoder/x86/ml_sse3.h"
+
+#define CALC_OUTPUT_FOR_2ROWS \
+ const int index = weight_idx + (2 * i * tot_num_inputs); \
+ const __m256 weight0 = _mm256_loadu_ps(&weights[index]); \
+ const __m256 weight1 = _mm256_loadu_ps(&weights[index + tot_num_inputs]); \
+ const __m256 mul0 = _mm256_mul_ps(inputs256, weight0); \
+ const __m256 mul1 = _mm256_mul_ps(inputs256, weight1); \
+ hadd[i] = _mm256_hadd_ps(mul0, mul1);
+
+static INLINE void nn_propagate_8to1(
+ const float *const inputs, const float *const weights,
+ const float *const bias, int num_inputs_to_process, int tot_num_inputs,
+ int num_outputs, float *const output_nodes, int is_clip_required) {
+ // Process one output row at a time.
+ for (int out = 0; out < num_outputs; out++) {
+ __m256 in_result = _mm256_setzero_ps();
+ float bias_val = bias[out];
+ for (int in = 0; in < num_inputs_to_process; in += 8) {
+ const __m256 inputs256 = _mm256_loadu_ps(&inputs[in]);
+ const int weight_idx = in + (out * tot_num_inputs);
+ const __m256 weight0 = _mm256_loadu_ps(&weights[weight_idx]);
+ const __m256 mul0 = _mm256_mul_ps(inputs256, weight0);
+ in_result = _mm256_add_ps(in_result, mul0);
+ }
+ const __m128 low_128 = _mm256_castps256_ps128(in_result);
+ const __m128 high_128 = _mm256_extractf128_ps(in_result, 1);
+ const __m128 sum_par_0 = _mm_add_ps(low_128, high_128);
+ const __m128 sum_par_1 = _mm_hadd_ps(sum_par_0, sum_par_0);
+ const __m128 sum_tot =
+ _mm_add_ps(_mm_shuffle_ps(sum_par_1, sum_par_1, 0x99), sum_par_1);
+
+ bias_val += _mm_cvtss_f32(sum_tot);
+ if (is_clip_required) bias_val = AOMMAX(bias_val, 0);
+ output_nodes[out] = bias_val;
+ }
+}
+
+static INLINE void nn_propagate_8to4(
+ const float *const inputs, const float *const weights,
+ const float *const bias, int num_inputs_to_process, int tot_num_inputs,
+ int num_outputs, float *const output_nodes, int is_clip_required) {
+ __m256 hadd[2];
+ for (int out = 0; out < num_outputs; out += 4) {
+ __m128 bias_reg = _mm_loadu_ps(&bias[out]);
+ __m128 in_result = _mm_setzero_ps();
+ for (int in = 0; in < num_inputs_to_process; in += 8) {
+ const __m256 inputs256 = _mm256_loadu_ps(&inputs[in]);
+ const int weight_idx = in + (out * tot_num_inputs);
+ // Process two output row at a time.
+ for (int i = 0; i < 2; i++) {
+ CALC_OUTPUT_FOR_2ROWS
+ }
+
+ const __m256 sum_par = _mm256_hadd_ps(hadd[0], hadd[1]);
+ const __m128 low_128 = _mm256_castps256_ps128(sum_par);
+ const __m128 high_128 = _mm256_extractf128_ps(sum_par, 1);
+ const __m128 result = _mm_add_ps(low_128, high_128);
+
+ in_result = _mm_add_ps(in_result, result);
+ }
+
+ in_result = _mm_add_ps(in_result, bias_reg);
+ if (is_clip_required) in_result = _mm_max_ps(in_result, _mm_setzero_ps());
+ _mm_storeu_ps(&output_nodes[out], in_result);
+ }
+}
+
+static INLINE void nn_propagate_8to8(
+ const float *const inputs, const float *const weights,
+ const float *const bias, int num_inputs_to_process, int tot_num_inputs,
+ int num_outputs, float *const output_nodes, int is_clip_required) {
+ __m256 hadd[4];
+ for (int out = 0; out < num_outputs; out += 8) {
+ __m256 bias_reg = _mm256_loadu_ps(&bias[out]);
+ __m256 in_result = _mm256_setzero_ps();
+ for (int in = 0; in < num_inputs_to_process; in += 8) {
+ const __m256 inputs256 = _mm256_loadu_ps(&inputs[in]);
+ const int weight_idx = in + (out * tot_num_inputs);
+ // Process two output rows at a time.
+ for (int i = 0; i < 4; i++) {
+ CALC_OUTPUT_FOR_2ROWS
+ }
+ const __m256 hh0 = _mm256_hadd_ps(hadd[0], hadd[1]);
+ const __m256 hh1 = _mm256_hadd_ps(hadd[2], hadd[3]);
+
+ __m256 ht_0 = _mm256_permute2f128_ps(hh0, hh1, 0x20);
+ __m256 ht_1 = _mm256_permute2f128_ps(hh0, hh1, 0x31);
+
+ __m256 result = _mm256_add_ps(ht_0, ht_1);
+ in_result = _mm256_add_ps(in_result, result);
+ }
+ in_result = _mm256_add_ps(in_result, bias_reg);
+ if (is_clip_required)
+ in_result = _mm256_max_ps(in_result, _mm256_setzero_ps());
+ _mm256_storeu_ps(&output_nodes[out], in_result);
+ }
+}
+
+static INLINE void nn_propagate_input_multiple_of_8(
+ const float *const inputs, const float *const weights,
+ const float *const bias, int num_inputs_to_process, int tot_num_inputs,
+ bool is_output_layer, int num_outputs, float *const output_nodes) {
+ // The saturation of output is considered for hidden layer which is not equal
+ // to final hidden layer.
+ const int is_clip_required =
+ !is_output_layer && num_inputs_to_process == tot_num_inputs;
+ if (num_outputs % 8 == 0) {
+ nn_propagate_8to8(inputs, weights, bias, num_inputs_to_process,
+ tot_num_inputs, num_outputs, output_nodes,
+ is_clip_required);
+ } else if (num_outputs % 4 == 0) {
+ nn_propagate_8to4(inputs, weights, bias, num_inputs_to_process,
+ tot_num_inputs, num_outputs, output_nodes,
+ is_clip_required);
+ } else {
+ nn_propagate_8to1(inputs, weights, bias, num_inputs_to_process,
+ tot_num_inputs, num_outputs, output_nodes,
+ is_clip_required);
+ }
+}
+
+void av1_nn_predict_avx2(const float *input_nodes,
+ const NN_CONFIG *const nn_config, int reduce_prec,
+ float *const output) {
+ float buf[2][NN_MAX_NODES_PER_LAYER];
+ int buf_index = 0;
+ int num_inputs = nn_config->num_inputs;
+ assert(num_inputs > 0 && num_inputs <= NN_MAX_NODES_PER_LAYER);
+
+ for (int layer = 0; layer <= nn_config->num_hidden_layers; layer++) {
+ const float *layer_weights = nn_config->weights[layer];
+ const float *layer_bias = nn_config->bias[layer];
+ bool is_output_layer = layer == nn_config->num_hidden_layers;
+ float *const output_nodes = is_output_layer ? output : &buf[buf_index][0];
+ const int num_outputs = is_output_layer
+ ? nn_config->num_outputs
+ : nn_config->num_hidden_nodes[layer];
+ assert(num_outputs > 0 && num_outputs <= NN_MAX_NODES_PER_LAYER);
+
+ // Process input multiple of 8 using AVX2 intrinsic.
+ if (num_inputs % 8 == 0) {
+ nn_propagate_input_multiple_of_8(input_nodes, layer_weights, layer_bias,
+ num_inputs, num_inputs, is_output_layer,
+ num_outputs, output_nodes);
+ } else {
+ // When number of inputs is not multiple of 8, use hybrid approach of AVX2
+ // and SSE3 based on the need.
+ const int in_mul_8 = num_inputs / 8;
+ const int num_inputs_to_process = in_mul_8 * 8;
+ int bias_is_considered = 0;
+ if (in_mul_8) {
+ nn_propagate_input_multiple_of_8(
+ input_nodes, layer_weights, layer_bias, num_inputs_to_process,
+ num_inputs, is_output_layer, num_outputs, output_nodes);
+ bias_is_considered = 1;
+ }
+
+ const float *out_temp = bias_is_considered ? output_nodes : layer_bias;
+ const int input_remaining = num_inputs % 8;
+ if (input_remaining % 4 == 0 && num_outputs % 8 == 0) {
+ for (int out = 0; out < num_outputs; out += 8) {
+ __m128 out_h = _mm_loadu_ps(&out_temp[out + 4]);
+ __m128 out_l = _mm_loadu_ps(&out_temp[out]);
+ for (int in = in_mul_8 * 8; in < num_inputs; in += 4) {
+ av1_nn_propagate_4to8_sse3(&input_nodes[in],
+ &layer_weights[out * num_inputs + in],
+ &out_h, &out_l, num_inputs);
+ }
+ if (!is_output_layer) {
+ const __m128 zero = _mm_setzero_ps();
+ out_h = _mm_max_ps(out_h, zero);
+ out_l = _mm_max_ps(out_l, zero);
+ }
+ _mm_storeu_ps(&output_nodes[out + 4], out_h);
+ _mm_storeu_ps(&output_nodes[out], out_l);
+ }
+ } else if (input_remaining % 4 == 0 && num_outputs % 4 == 0) {
+ for (int out = 0; out < num_outputs; out += 4) {
+ __m128 outputs = _mm_loadu_ps(&out_temp[out]);
+ for (int in = in_mul_8 * 8; in < num_inputs; in += 4) {
+ av1_nn_propagate_4to4_sse3(&input_nodes[in],
+ &layer_weights[out * num_inputs + in],
+ &outputs, num_inputs);
+ }
+ if (!is_output_layer) outputs = _mm_max_ps(outputs, _mm_setzero_ps());
+ _mm_storeu_ps(&output_nodes[out], outputs);
+ }
+ } else if (input_remaining % 4 == 0) {
+ for (int out = 0; out < num_outputs; out++) {
+ __m128 outputs = _mm_load1_ps(&out_temp[out]);
+ for (int in = in_mul_8 * 8; in < num_inputs; in += 4) {
+ av1_nn_propagate_4to1_sse3(&input_nodes[in],
+ &layer_weights[out * num_inputs + in],
+ &outputs);
+ }
+ if (!is_output_layer) outputs = _mm_max_ps(outputs, _mm_setzero_ps());
+ output_nodes[out] = _mm_cvtss_f32(outputs);
+ }
+ } else {
+ // Use SSE instructions for scalar operations to avoid the latency
+ // of swapping between SIMD and FPU modes.
+ for (int out = 0; out < num_outputs; out++) {
+ __m128 outputs = _mm_load1_ps(&out_temp[out]);
+ for (int in_node = in_mul_8 * 8; in_node < num_inputs; in_node++) {
+ __m128 input = _mm_load1_ps(&input_nodes[in_node]);
+ __m128 weight =
+ _mm_load1_ps(&layer_weights[num_inputs * out + in_node]);
+ outputs = _mm_add_ps(outputs, _mm_mul_ps(input, weight));
+ }
+ if (!is_output_layer) outputs = _mm_max_ps(outputs, _mm_setzero_ps());
+ output_nodes[out] = _mm_cvtss_f32(outputs);
+ }
+ }
+ }
+ // Before processing the next layer, treat the output of current layer as
+ // input to next layer.
+ input_nodes = output_nodes;
+ num_inputs = num_outputs;
+ buf_index = 1 - buf_index;
+ }
+ if (reduce_prec) av1_nn_output_prec_reduce(output, nn_config->num_outputs);
+}
diff --git a/av1/encoder/x86/ml_sse3.c b/av1/encoder/x86/ml_sse3.c
index ab69088..4748a68 100644
--- a/av1/encoder/x86/ml_sse3.c
+++ b/av1/encoder/x86/ml_sse3.c
@@ -11,10 +11,10 @@
#include <stdbool.h>
#include <assert.h>
-#include <pmmintrin.h>
#include "config/av1_rtcd.h"
#include "av1/encoder/ml.h"
+#include "av1/encoder/x86/ml_sse3.h"
// In order to avoid the high-latency of swapping between FPU and SIMD
// operations, we keep the result in a 128-bit register even though we only
@@ -41,9 +41,9 @@
*output = _mm_add_ps(*output, hadd2);
}
-static void nn_propagate_4to1(const float *const inputs,
- const float *const weights,
- __m128 *const output) {
+void av1_nn_propagate_4to1_sse3(const float *const inputs,
+ const float *const weights,
+ __m128 *const output) {
const __m128 inputs128 = _mm_loadu_ps(inputs);
const __m128 weights128 = _mm_loadu_ps(weights);
@@ -58,9 +58,9 @@
*output = _mm_add_ps(*output, hadd2);
}
-static void nn_propagate_4to4(const float *const inputs,
- const float *const weights, __m128 *const outputs,
- const int num_inputs) {
+void av1_nn_propagate_4to4_sse3(const float *const inputs,
+ const float *const weights,
+ __m128 *const outputs, const int num_inputs) {
const __m128 inputs128 = _mm_loadu_ps(inputs);
__m128 hadd[2];
@@ -80,9 +80,9 @@
*outputs = _mm_add_ps(*outputs, hh);
}
-static void nn_propagate_4to8(const float *const inputs,
- const float *const weights, __m128 *const out_h,
- __m128 *const out_l, const int num_inputs) {
+void av1_nn_propagate_4to8_sse3(const float *const inputs,
+ const float *const weights, __m128 *const out_h,
+ __m128 *const out_l, const int num_inputs) {
const __m128 inputs128 = _mm_loadu_ps(inputs);
__m128 hadd[4];
@@ -171,9 +171,9 @@
__m128 out_h = _mm_loadu_ps(&layer_bias[out + 4]);
__m128 out_l = _mm_loadu_ps(&layer_bias[out]);
for (int in = 0; in < num_inputs; in += 4) {
- nn_propagate_4to8(&input_nodes[in],
- &layer_weights[out * num_inputs + in], &out_h,
- &out_l, num_inputs);
+ av1_nn_propagate_4to8_sse3(&input_nodes[in],
+ &layer_weights[out * num_inputs + in],
+ &out_h, &out_l, num_inputs);
}
if (!output_layer) nn_activate8(&out_h, &out_l);
_mm_storeu_ps(&output_nodes[out + 4], out_h);
@@ -194,9 +194,9 @@
for (int out = 0; out < num_outputs; out += 4) {
__m128 outputs = _mm_loadu_ps(&layer_bias[out]);
for (int in = 0; in < num_inputs; in += 4) {
- nn_propagate_4to4(&input_nodes[in],
- &layer_weights[out * num_inputs + in], &outputs,
- num_inputs);
+ av1_nn_propagate_4to4_sse3(&input_nodes[in],
+ &layer_weights[out * num_inputs + in],
+ &outputs, num_inputs);
}
if (!output_layer) nn_activate4(&outputs);
_mm_storeu_ps(&output_nodes[out], outputs);
@@ -215,8 +215,8 @@
for (int out = 0; out < num_outputs; out++) {
__m128 total = _mm_load1_ps(&layer_bias[out]);
for (int in = 0; in < num_inputs; in += 4) {
- nn_propagate_4to1(&input_nodes[in],
- &layer_weights[out * num_inputs + in], &total);
+ av1_nn_propagate_4to1_sse3(
+ &input_nodes[in], &layer_weights[out * num_inputs + in], &total);
}
if (!output_layer) nn_activate4(&total);
output_nodes[out] = _mm_cvtss_f32(total);
diff --git a/av1/encoder/x86/ml_sse3.h b/av1/encoder/x86/ml_sse3.h
new file mode 100644
index 0000000..f41a247
--- /dev/null
+++ b/av1/encoder/x86/ml_sse3.h
@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#ifndef AOM_AV1_ENCODER_X86_ML_SSE3_H_
+#define AOM_AV1_ENCODER_X86_ML_SSE3_H_
+
+#include <pmmintrin.h>
+
+void av1_nn_propagate_4to1_sse3(const float *const inputs,
+ const float *const weights,
+ __m128 *const output);
+
+void av1_nn_propagate_4to4_sse3(const float *const inputs,
+ const float *const weights,
+ __m128 *const outputs, const int num_inputs);
+
+void av1_nn_propagate_4to8_sse3(const float *const inputs,
+ const float *const weights, __m128 *const out_h,
+ __m128 *const out_l, const int num_inputs);
+
+#endif // AOM_AV1_ENCODER_X86_ML_SSE3_H_
diff --git a/av1/ratectrl_rtc.cc b/av1/ratectrl_rtc.cc
index a3ec6f6..62d6e74 100644
--- a/av1/ratectrl_rtc.cc
+++ b/av1/ratectrl_rtc.cc
@@ -39,6 +39,8 @@
undershoot_pct = overshoot_pct = 50;
max_intra_bitrate_pct = 50;
max_inter_bitrate_pct = 0;
+ frame_drop_thresh = 0;
+ max_consec_drop = 0;
framerate = 30.0;
ss_number_layers = 1;
ts_number_layers = 1;
@@ -124,7 +126,8 @@
oxcf->pass = AOM_RC_ONE_PASS;
oxcf->q_cfg.aq_mode = rc_cfg.aq_mode ? CYCLIC_REFRESH_AQ : NO_AQ;
oxcf->tune_cfg.content = AOM_CONTENT_DEFAULT;
- oxcf->rc_cfg.drop_frames_water_mark = 0;
+ oxcf->rc_cfg.drop_frames_water_mark = rc_cfg.frame_drop_thresh;
+ rc->max_consec_drop = rc_cfg.max_consec_drop;
oxcf->tool_cfg.bit_depth = AOM_BITS_8;
oxcf->tool_cfg.superblock_size = AOM_SUPERBLOCK_SIZE_DYNAMIC;
oxcf->algo_cfg.loopfilter_control = LOOPFILTER_ALL;
@@ -185,9 +188,15 @@
oxcf->rc_cfg.maximum_buffer_size_ms = rc_cfg.buf_sz;
oxcf->rc_cfg.under_shoot_pct = rc_cfg.undershoot_pct;
oxcf->rc_cfg.over_shoot_pct = rc_cfg.overshoot_pct;
+ oxcf->rc_cfg.drop_frames_water_mark = rc_cfg.frame_drop_thresh;
+ rc->max_consec_drop = rc_cfg.max_consec_drop;
oxcf->rc_cfg.max_intra_bitrate_pct = rc_cfg.max_intra_bitrate_pct;
oxcf->rc_cfg.max_inter_bitrate_pct = rc_cfg.max_inter_bitrate_pct;
cpi_->framerate = rc_cfg.framerate;
+ if (rc_cfg.is_screen) {
+ cpi_->oxcf.tune_cfg.content = AOM_CONTENT_SCREEN;
+ cpi_->is_screen_content_type = 1;
+ }
cpi_->svc.number_spatial_layers = rc_cfg.ss_number_layers;
cpi_->svc.number_temporal_layers = rc_cfg.ts_number_layers;
set_primary_rc_buffer_sizes(oxcf, cpi_->ppi);
@@ -226,7 +235,8 @@
return true;
}
-void AV1RateControlRTC::ComputeQP(const AV1FrameParamsRTC &frame_params) {
+FrameDropDecision AV1RateControlRTC::ComputeQP(
+ const AV1FrameParamsRTC &frame_params) {
AV1_COMMON *const cm = &cpi_->common;
int width, height;
GF_GROUP *const gf_group = &cpi_->ppi->gf_group;
@@ -292,14 +302,25 @@
}
}
av1_rc_set_frame_target(cpi_, target, cm->width, cm->height);
-
- int bottom_index, top_index;
+ // Always drop for spatial enhancement layer if layer bandwidth is 0.
+ // Otherwise check for frame-dropping based on buffer level in
+ // av1_rc_drop_frame().
+ if ((cpi_->svc.spatial_layer_id > 0 &&
+ cpi_->oxcf.rc_cfg.target_bandwidth == 0) ||
+ av1_rc_drop_frame(cpi_)) {
+ cpi_->is_dropped_frame = true;
+ av1_rc_postencode_update_drop_frame(cpi_);
+ cpi_->frame_index_set.show_frame_count++;
+ cpi_->common.current_frame.frame_number++;
+ return FrameDropDecision::kDrop;
+ }
+ int bottom_index = 0, top_index = 0;
cpi_->common.quant_params.base_qindex =
av1_rc_pick_q_and_bounds(cpi_, cm->width, cm->height,
cpi_->gf_frame_index, &bottom_index, &top_index);
-
if (cpi_->oxcf.q_cfg.aq_mode == CYCLIC_REFRESH_AQ)
av1_cyclic_refresh_setup(cpi_);
+ return FrameDropDecision::kOk;
}
int AV1RateControlRTC::GetQP() const {
@@ -327,12 +348,17 @@
return cdef_level;
}
-signed char *AV1RateControlRTC::GetCyclicRefreshMap() const {
- return cpi_->cyclic_refresh->map;
-}
-
-int *AV1RateControlRTC::GetDeltaQ() const {
- return cpi_->cyclic_refresh->qindex_delta;
+bool AV1RateControlRTC::GetSegmentationData(
+ AV1SegmentationData *segmentation_data) const {
+ if (cpi_->oxcf.q_cfg.aq_mode == 0) {
+ return false;
+ }
+ segmentation_data->segmentation_map = cpi_->enc_seg.map;
+ segmentation_data->segmentation_map_size =
+ cpi_->common.mi_params.mi_rows * cpi_->common.mi_params.mi_cols;
+ segmentation_data->delta_q = cpi_->cyclic_refresh->qindex_delta;
+ segmentation_data->delta_q_size = 3u;
+ return true;
}
void AV1RateControlRTC::PostEncodeUpdate(uint64_t encoded_frame_size) {
diff --git a/av1/ratectrl_rtc.h b/av1/ratectrl_rtc.h
index e96e210..1894469 100644
--- a/av1/ratectrl_rtc.h
+++ b/av1/ratectrl_rtc.h
@@ -33,7 +33,7 @@
int width;
int height;
// Flag indicating if the content is screen or not.
- bool is_screen;
+ bool is_screen = false;
// 0-63
int max_quantizer;
int min_quantizer;
@@ -45,6 +45,8 @@
int overshoot_pct;
int max_intra_bitrate_pct;
int max_inter_bitrate_pct;
+ int frame_drop_thresh;
+ int max_consec_drop;
double framerate;
int layer_target_bitrate[kAV1MaxLayers];
int ts_rate_decimator[kAV1MaxTemporalLayers];
@@ -77,6 +79,18 @@
int damping;
};
+struct AV1SegmentationData {
+ const uint8_t *segmentation_map;
+ size_t segmentation_map_size;
+ const int *delta_q;
+ size_t delta_q_size;
+};
+
+enum class FrameDropDecision {
+ kOk, // Frame is encoded.
+ kDrop, // Frame is dropped.
+};
+
class AV1RateControlRTC {
public:
static std::unique_ptr<AV1RateControlRTC> Create(
@@ -90,9 +104,13 @@
AV1LoopfilterLevel GetLoopfilterLevel() const;
// GetCdefInfo() needs to be called after ComputeQP()
AV1CdefInfo GetCdefInfo() const;
- signed char *GetCyclicRefreshMap() const;
- int *GetDeltaQ() const;
- void ComputeQP(const AV1FrameParamsRTC &frame_params);
+ // Returns the segmentation map used for cyclic refresh, based on 4x4 blocks.
+ bool GetSegmentationData(AV1SegmentationData *segmentation_data) const;
+ // ComputeQP returns the QP if the frame is not dropped (kOk return),
+ // otherwise it returns kDrop and subsequent GetQP and PostEncodeUpdate
+ // are not to be called (av1_rc_postencode_update_drop_frame is already
+ // called via ComputeQP if drop is decided).
+ FrameDropDecision ComputeQP(const AV1FrameParamsRTC &frame_params);
// Feedback to rate control with the size of current encoded frame
void PostEncodeUpdate(uint64_t encoded_frame_size);
diff --git a/build/cmake/aom_config_defaults.cmake b/build/cmake/aom_config_defaults.cmake
index 5058022..85390d5 100644
--- a/build/cmake/aom_config_defaults.cmake
+++ b/build/cmake/aom_config_defaults.cmake
@@ -29,9 +29,14 @@
set_aom_detect_var(AOM_ARCH_X86 0 "Enables X86 architecture.")
set_aom_detect_var(AOM_ARCH_X86_64 0 "Enables X86_64 architecture.")
-# ARM feature flags.
-set_aom_detect_var(HAVE_NEON 0 "Enables NEON intrinsics optimizations.")
+# Arm/AArch64 feature flags.
+set_aom_detect_var(HAVE_NEON 0 "Enables Neon intrinsics optimizations.")
set_aom_detect_var(HAVE_ARM_CRC32 0 "Enables Arm CRC32 optimizations.")
+set_aom_detect_var(HAVE_NEON_DOTPROD 0
+ "Enables Armv8.2-A Neon dotprod intrinsics optimizations.")
+set_aom_detect_var(HAVE_NEON_I8MM 0
+ "Enables Armv8.2-A Neon i8mm intrinsics optimizations.")
+set_aom_detect_var(HAVE_SVE 0 "Enables Armv8.2-A SVE intrinsics optimizations.")
# PPC feature flags.
set_aom_detect_var(HAVE_VSX 0 "Enables VSX optimizations.")
@@ -189,8 +194,18 @@
set_aom_option_var(ENABLE_WERROR "Converts warnings to errors at compile time."
OFF)
-# ARM assembly/intrinsics flags.
-set_aom_option_var(ENABLE_NEON "Enables NEON optimizations on ARM targets." ON)
+# Arm/AArch64 assembly/intrinsics flags.
+set_aom_option_var(ENABLE_NEON
+ "Enables Neon optimizations on Arm/AArch64 targets." ON)
+set_aom_option_var(ENABLE_ARM_CRC32 "Enables Arm CRC32 optimizations." ON)
+set_aom_option_var(
+ ENABLE_NEON_DOTPROD
+ "Enables Armv8.2-A Neon dotprod optimizations on AArch64 targets." ON)
+set_aom_option_var(
+ ENABLE_NEON_I8MM
+ "Enables Armv8.2-A Neon i8mm optimizations on AArch64 targets." ON)
+set_aom_option_var(ENABLE_SVE
+ "Enables Armv8.2-A SVE optimizations on AArch64 targets." ON)
# VSX intrinsics flags.
set_aom_option_var(ENABLE_VSX "Enables VSX optimizations on PowerPC targets."
diff --git a/build/cmake/aom_configure.cmake b/build/cmake/aom_configure.cmake
index aaef2c3..6c932e8 100644
--- a/build/cmake/aom_configure.cmake
+++ b/build/cmake/aom_configure.cmake
@@ -13,8 +13,6 @@
endif() # AOM_BUILD_CMAKE_AOM_CONFIGURE_CMAKE_
set(AOM_BUILD_CMAKE_AOM_CONFIGURE_CMAKE_ 1)
-include(FindGit)
-include(FindPerl)
include(FindThreads)
include("${AOM_ROOT}/build/cmake/aom_config_defaults.cmake")
@@ -214,7 +212,6 @@
endif()
if(CONFIG_ANALYZER)
- include(FindwxWidgets)
find_package(wxWidgets REQUIRED adv base core)
include(${wxWidgets_USE_FILE})
endif()
@@ -361,7 +358,7 @@
# This combination has more stack overhead, so we account for it by
# providing higher stack limit than usual.
- add_c_flag_if_supported("-Wstack-usage=170000")
+ add_c_flag_if_supported("-Wstack-usage=285000")
add_cxx_flag_if_supported("-Wstack-usage=270000")
elseif(CONFIG_RD_DEBUG) # Another case where higher stack usage is expected.
add_c_flag_if_supported("-Wstack-usage=135000")
diff --git a/build/cmake/aom_install.cmake b/build/cmake/aom_install.cmake
index b02c7b9..2c263e9 100644
--- a/build/cmake/aom_install.cmake
+++ b/build/cmake/aom_install.cmake
@@ -46,12 +46,12 @@
-DCMAKE_INSTALL_INCLUDEDIR=${CMAKE_INSTALL_INCLUDEDIR}
-DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR}
-DCMAKE_PROJECT_NAME=${CMAKE_PROJECT_NAME}
+ -DCMAKE_THREAD_LIBS_INIT=${CMAKE_THREAD_LIBS_INIT}
-DCONFIG_MULTITHREAD=${CONFIG_MULTITHREAD}
-DCONFIG_TUNE_VMAF=${CONFIG_TUNE_VMAF}
-DCONFIG_TUNE_BUTTERAUGLI=${CONFIG_TUNE_BUTTERAUGLI}
-DCONFIG_SALIENCY_MAP=${CONFIG_SALIENCY_MAP}
-DCONFIG_TFLITE=${CONFIG_TFLITE}
- -DHAVE_PTHREAD_H=${HAVE_PTHREAD_H}
-P
"${AOM_ROOT}/build/cmake/pkg_config.cmake"
COMMENT "Writing aom.pc"
diff --git a/build/cmake/aom_optimization.cmake b/build/cmake/aom_optimization.cmake
index 6b0c55a..0f93228 100644
--- a/build/cmake/aom_optimization.cmake
+++ b/build/cmake/aom_optimization.cmake
@@ -270,7 +270,7 @@
--arch=${AOM_TARGET_CPU}
--sym=${symbol} ${AOM_RTCD_FLAGS}
--config=${AOM_CONFIG_DIR}/config/aom_config.h ${config} > ${output}
- DEPENDS ${config}
+ DEPENDS "${AOM_ROOT}/build/cmake/rtcd.pl" ${config}
COMMENT "Generating ${output}"
WORKING_DIRECTORY ${AOM_CONFIG_DIR}
VERBATIM)
diff --git a/build/cmake/cpu.cmake b/build/cmake/cpu.cmake
index 799a313..a9b7a67 100644
--- a/build/cmake/cpu.cmake
+++ b/build/cmake/cpu.cmake
@@ -9,11 +9,60 @@
# can obtain it at www.aomedia.org/license/patent.
#
-if("${AOM_TARGET_CPU}" MATCHES "^arm")
+if("${AOM_TARGET_CPU}" STREQUAL "arm64")
set(AOM_ARCH_ARM 1)
- if("${AOM_TARGET_CPU}" STREQUAL "arm64")
- set(AOM_ARCH_AARCH64 1)
+ set(AOM_ARCH_AARCH64 1)
+ set(RTCD_ARCH_ARM "yes")
+
+ set(ARM64_FLAVORS "NEON;ARM_CRC32;NEON_DOTPROD;NEON_I8MM;SVE")
+ set(AOM_ARM_CRC32_DEFAULT_FLAG "-march=armv8-a+crc")
+ set(AOM_NEON_DOTPROD_DEFAULT_FLAG "-march=armv8.2-a+dotprod")
+ set(AOM_NEON_I8MM_DEFAULT_FLAG "-march=armv8.2-a+dotprod+i8mm")
+ set(AOM_SVE_DEFAULT_FLAG "-march=armv8.2-a+dotprod+i8mm+sve")
+
+ # Check that the compiler flag to enable each flavor is supported by the
+ # compiler. This may not be the case for new architecture features on old
+ # compiler versions.
+ foreach(flavor ${ARM64_FLAVORS})
+ if(ENABLE_${flavor} AND NOT DEFINED AOM_${flavor}_FLAG)
+ set(AOM_${flavor}_FLAG "${AOM_${flavor}_DEFAULT_FLAG}")
+ unset(FLAG_SUPPORTED)
+ check_c_compiler_flag("${AOM_${flavor}_FLAG}" FLAG_SUPPORTED)
+ if(NOT ${FLAG_SUPPORTED})
+ set(ENABLE_${flavor} 0)
+ endif()
+ endif()
+ endforeach()
+
+ # SVE requires that the Neon-SVE bridge header is also available.
+ if(ENABLE_SVE)
+ set(OLD_CMAKE_REQURED_FLAGS ${CMAKE_REQUIRED_FLAGS})
+ set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${AOM_SVE_FLAG}")
+ aom_check_source_compiles("arm_neon_sve_bridge_available" "
+#ifndef __ARM_NEON_SVE_BRIDGE
+#error 1
+#endif
+#include <arm_sve.h>
+#include <arm_neon_sve_bridge.h>" HAVE_SVE_HEADERS)
+ set(CMAKE_REQUIRED_FLAGS ${OLD_CMAKE_REQURED_FLAGS})
+ if(HAVE_SVE_HEADERS EQUAL 0)
+ set(ENABLE_SVE 0)
+ endif()
endif()
+
+ foreach(flavor ${ARM64_FLAVORS})
+ if(ENABLE_${flavor})
+ set(HAVE_${flavor} 1)
+ set(RTCD_HAVE_${flavor} "yes")
+ else()
+ set(HAVE_${flavor} 0)
+ string(TOLOWER ${flavor} flavor)
+ set(AOM_RTCD_FLAGS ${AOM_RTCD_FLAGS} --disable-${flavor})
+ endif()
+ endforeach()
+
+elseif("${AOM_TARGET_CPU}" MATCHES "^arm")
+ set(AOM_ARCH_ARM 1)
set(RTCD_ARCH_ARM "yes")
if(ENABLE_NEON)
@@ -24,18 +73,6 @@
set(AOM_RTCD_FLAGS ${AOM_RTCD_FLAGS} --disable-neon)
endif()
- check_c_source_compiles("
- #if !defined(__ARM_FEATURE_CRC32) || __ARM_FEATURE_CRC32 != 1
- #error \"CRC32 is unavailable.\"
- #endif
- int main(void) { return 0; }" HAVE_CRC32)
- if(HAVE_CRC32)
- set(HAVE_ARM_CRC32 1)
- else()
- set(HAVE_ARM_CRC32 0)
- set(AOM_RTCD_FLAGS ${AOM_RTCD_FLAGS} --disable-arm_crc32)
- endif()
-
elseif("${AOM_TARGET_CPU}" MATCHES "ppc")
set(AOM_ARCH_PPC 1)
set(RTCD_ARCH_PPC "yes")
diff --git a/build/cmake/pkg_config.cmake b/build/cmake/pkg_config.cmake
index e8fff2e..c4f9480 100644
--- a/build/cmake/pkg_config.cmake
+++ b/build/cmake/pkg_config.cmake
@@ -13,7 +13,7 @@
set(REQUIRED_ARGS "AOM_ROOT" "AOM_CONFIG_DIR" "CMAKE_INSTALL_PREFIX"
"CMAKE_INSTALL_BINDIR" "CMAKE_INSTALL_INCLUDEDIR"
"CMAKE_INSTALL_LIBDIR" "CMAKE_PROJECT_NAME"
- "CONFIG_MULTITHREAD" "HAVE_PTHREAD_H")
+ "CONFIG_MULTITHREAD")
foreach(arg ${REQUIRED_ARGS})
if("${${arg}}" STREQUAL "")
@@ -60,8 +60,9 @@
endif()
file(APPEND "${pkgconfig_file}" "\nConflicts:\n")
file(APPEND "${pkgconfig_file}" "Libs: -L\${libdir} -l${pkg_name}\n")
-if(CONFIG_MULTITHREAD AND HAVE_PTHREAD_H)
- file(APPEND "${pkgconfig_file}" "Libs.private: -lm -lpthread\n")
+if(CONFIG_MULTITHREAD AND CMAKE_THREAD_LIBS_INIT)
+ file(APPEND "${pkgconfig_file}"
+ "Libs.private: -lm ${CMAKE_THREAD_LIBS_INIT}\n")
else()
file(APPEND "${pkgconfig_file}" "Libs.private: -lm\n")
endif()
diff --git a/build/cmake/rtcd.pl b/build/cmake/rtcd.pl
index bd3b9d5..1cf52f0 100755
--- a/build/cmake/rtcd.pl
+++ b/build/cmake/rtcd.pl
@@ -392,8 +392,9 @@
@ALL_ARCHS = filter(qw/neon/);
arm;
} elsif ($opts{arch} eq 'arm64' ) {
- @ALL_ARCHS = filter(qw/neon arm_crc32/);
- &require(@ALL_ARCHS);
+ @ALL_ARCHS = filter(qw/neon arm_crc32 neon_dotprod neon_i8mm sve/);
+ @REQUIRES = filter(qw/neon/);
+ &require(@REQUIRES);
arm;
} elsif ($opts{arch} eq 'ppc') {
@ALL_ARCHS = filter(qw/vsx/);
diff --git a/build/cmake/toolchains/android.cmake b/build/cmake/toolchains/android.cmake
index 4d38c9a..fb08685 100644
--- a/build/cmake/toolchains/android.cmake
+++ b/build/cmake/toolchains/android.cmake
@@ -46,8 +46,6 @@
if(ANDROID_ABI MATCHES "^arm")
set(CMAKE_ASM_COMPILER as)
- # No runtime cpu detect for arm targets.
- set(CONFIG_RUNTIME_CPU_DETECT 0 CACHE STRING "")
elseif(ANDROID_ABI MATCHES "^x86")
set(CMAKE_ASM_NASM_COMPILER yasm)
endif()
diff --git a/build/cmake/toolchains/arm-ios-common.cmake b/build/cmake/toolchains/arm-ios-common.cmake
index 62ca115..2c433be 100644
--- a/build/cmake/toolchains/arm-ios-common.cmake
+++ b/build/cmake/toolchains/arm-ios-common.cmake
@@ -21,7 +21,4 @@
set(CMAKE_CXX_FLAGS_INIT "-arch ${CMAKE_SYSTEM_PROCESSOR}")
set(CMAKE_EXE_LINKER_FLAGS_INIT "-arch ${CMAKE_SYSTEM_PROCESSOR}")
-# No runtime cpu detect for arm*-ios targets.
-set(CONFIG_RUNTIME_CPU_DETECT 0 CACHE STRING "")
-
# TODO(tomfinegan): Handle bit code embedding.
diff --git a/build/cmake/toolchains/arm64-linux-clang.cmake b/build/cmake/toolchains/arm64-linux-clang.cmake
new file mode 100644
index 0000000..b4645cc
--- /dev/null
+++ b/build/cmake/toolchains/arm64-linux-clang.cmake
@@ -0,0 +1,30 @@
+#
+# Copyright (c) 2023, Alliance for Open Media. All rights reserved
+#
+# This source code is subject to the terms of the BSD 2 Clause License and the
+# Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License was
+# not distributed with this source code in the LICENSE file, you can obtain it
+# at www.aomedia.org/license/software. If the Alliance for Open Media Patent
+# License 1.0 was not distributed with this source code in the PATENTS file, you
+# can obtain it at www.aomedia.org/license/patent.
+#
+
+if(AOM_BUILD_CMAKE_TOOLCHAINS_ARM64_LINUX_CLANG_CMAKE_)
+ return()
+endif() # AOM_BUILD_CMAKE_TOOLCHAINS_ARM64_LINUX_CLANG_CMAKE_
+set(AOM_BUILD_CMAKE_TOOLCHAINS_ARM64_LINUX_CLANG_CMAKE_ 1)
+
+set(CMAKE_SYSTEM_NAME "Linux")
+
+set(TRIPLE aarch64-linux-gnu)
+
+set(CMAKE_C_COMPILER clang)
+set(CMAKE_C_COMPILER_TARGET ${TRIPLE})
+
+set(CMAKE_CXX_COMPILER clang++)
+set(CMAKE_CXX_COMPILER_TARGET ${TRIPLE})
+
+set(CMAKE_ASM_COMPILER clang)
+set(CMAKE_ASM_COMPILER_TARGET ${TRIPLE})
+
+set(CMAKE_SYSTEM_PROCESSOR "arm64")
diff --git a/build/cmake/toolchains/arm64-linux-gcc.cmake b/build/cmake/toolchains/arm64-linux-gcc.cmake
index 133a96a..3d0dff0 100644
--- a/build/cmake/toolchains/arm64-linux-gcc.cmake
+++ b/build/cmake/toolchains/arm64-linux-gcc.cmake
@@ -38,6 +38,3 @@
# No intrinsics flag required for arm64-linux-gcc.
set(AOM_NEON_INTRIN_FLAG "")
-
-# No runtime cpu detect for arm64-linux-gcc.
-set(CONFIG_RUNTIME_CPU_DETECT 0 CACHE STRING "")
diff --git a/build/cmake/toolchains/arm64-mingw-gcc.cmake b/build/cmake/toolchains/arm64-mingw-gcc.cmake
index 7400423..95b26d3 100644
--- a/build/cmake/toolchains/arm64-mingw-gcc.cmake
+++ b/build/cmake/toolchains/arm64-mingw-gcc.cmake
@@ -34,6 +34,3 @@
if(NOT CMAKE_RANLIB)
set(CMAKE_RANLIB ${CROSS}ranlib CACHE FILEPATH Indexer)
endif()
-
-# No runtime cpu detect for arm64-mingw-gcc.
-set(CONFIG_RUNTIME_CPU_DETECT 0 CACHE STRING "")
diff --git a/build/cmake/toolchains/armv7-linux-gcc.cmake b/build/cmake/toolchains/armv7-linux-gcc.cmake
index 366e198..aa05505 100644
--- a/build/cmake/toolchains/armv7-linux-gcc.cmake
+++ b/build/cmake/toolchains/armv7-linux-gcc.cmake
@@ -44,6 +44,3 @@
set(CMAKE_SYSTEM_PROCESSOR "armv7")
set(AOM_NEON_INTRIN_FLAG "-mfpu=neon ${AOM_EXTRA_TOOLCHAIN_FLAGS}")
-
-# No runtime cpu detect for armv7-linux-gcc.
-set(CONFIG_RUNTIME_CPU_DETECT 0 CACHE STRING "")
diff --git a/build/cmake/toolchains/i686-linux-gcc.cmake b/build/cmake/toolchains/i686-linux-gcc.cmake
new file mode 100644
index 0000000..c4f6ab9
--- /dev/null
+++ b/build/cmake/toolchains/i686-linux-gcc.cmake
@@ -0,0 +1,34 @@
+#
+# Copyright (c) 2023, Alliance for Open Media. All rights reserved
+#
+# This source code is subject to the terms of the BSD 2 Clause License and the
+# Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License was
+# not distributed with this source code in the LICENSE file, you can obtain it
+# at www.aomedia.org/license/software. If the Alliance for Open Media Patent
+# License 1.0 was not distributed with this source code in the PATENTS file, you
+# can obtain it at www.aomedia.org/license/patent.
+#
+if(AOM_BUILD_CMAKE_TOOLCHAINS_I686_LINUX_GCC_CMAKE_)
+ return()
+endif() # AOM_BUILD_CMAKE_TOOLCHAINS_I686_LINUX_GCC_CMAKE_
+set(AOM_BUILD_CMAKE_TOOLCHAINS_I686_LINUX_GCC_CMAKE_ 1)
+
+set(CMAKE_SYSTEM_NAME "Linux")
+set(CMAKE_SYSTEM_PROCESSOR "x86")
+
+if("${CROSS}" STREQUAL "")
+
+ # Default the cross compiler prefix to one used by Debian and other package
+ # management systems.
+ set(CROSS i686-linux-gnu-)
+endif()
+
+if(NOT CMAKE_C_COMPILER)
+ set(CMAKE_C_COMPILER ${CROSS}gcc)
+endif()
+if(NOT CMAKE_CXX_COMPILER)
+ set(CMAKE_CXX_COMPILER ${CROSS}g++)
+endif()
+if(NOT CMAKE_ASM_COMPILER)
+ set(CMAKE_ASM_COMPILER ${CROSS}as)
+endif()
diff --git a/common/tools_common.c b/common/tools_common.c
index afe4619..4d77a1b 100644
--- a/common/tools_common.c
+++ b/common/tools_common.c
@@ -65,8 +65,8 @@
void die_codec(aom_codec_ctx_t *ctx, const char *s) {
const char *detail = aom_codec_error_detail(ctx);
- printf("%s: %s\n", s, aom_codec_error(ctx));
- if (detail) printf(" %s\n", detail);
+ fprintf(stderr, "%s: %s\n", s, aom_codec_error(ctx));
+ if (detail) fprintf(stderr, " %s\n", detail);
exit(EXIT_FAILURE);
}
diff --git a/docs.cmake b/docs.cmake
index 0d8db92..0d7b4cf 100644
--- a/docs.cmake
+++ b/docs.cmake
@@ -223,7 +223,7 @@
list(LENGTH AOM_DOXYGEN_EXAMPLE_SOURCES num_sources)
list(LENGTH AOM_DOXYGEN_EXAMPLE_DESCRIPTIONS num_descs)
if(NOT ${num_sources} EQUAL ${num_descs})
- message(FATAL_ERROR "Unqeual example and description totals.")
+ message(FATAL_ERROR "Unequal example and description totals.")
endif()
# Take the list of examples and produce example_basename.dox for each file in
diff --git a/examples/lightfield_tile_list_decoder.c b/examples/lightfield_tile_list_decoder.c
index 5b15ae0..d71ff5b 100644
--- a/examples/lightfield_tile_list_decoder.c
+++ b/examples/lightfield_tile_list_decoder.c
@@ -170,7 +170,7 @@
if (!aom_img_alloc_with_border(&reference_images[j], ref_fmt,
frame_res[0], frame_res[1], 32, 8,
border)) {
- die("Failed to allocate references.");
+ fatal("Failed to allocate references.");
}
}
}
diff --git a/examples/resize_util.c b/examples/resize_util.c
deleted file mode 100644
index 45a1db2..0000000
--- a/examples/resize_util.c
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Copyright (c) 2016, Alliance for Open Media. All rights reserved
- *
- * This source code is subject to the terms of the BSD 2 Clause License and
- * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
- * was not distributed with this source code in the LICENSE file, you can
- * obtain it at www.aomedia.org/license/software. If the Alliance for Open
- * Media Patent License 1.0 was not distributed with this source code in the
- * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
- */
-
-#include <assert.h>
-#include <limits.h>
-#include <math.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include "av1/common/resize.h"
-#include "common/tools_common.h"
-
-static const char *exec_name = NULL;
-
-static void usage() {
- printf("Usage:\n");
- printf("%s <input_yuv> <width>x<height> <target_width>x<target_height> ",
- exec_name);
- printf("<output_yuv> [<frames>]\n");
-}
-
-void usage_exit(void) {
- usage();
- exit(EXIT_FAILURE);
-}
-
-static int parse_dim(char *v, int *width, int *height) {
- char *x = strchr(v, 'x');
- if (x == NULL) x = strchr(v, 'X');
- if (x == NULL) return 0;
- *width = atoi(v);
- *height = atoi(&x[1]);
- if (*width <= 0 || *height <= 0)
- return 0;
- else
- return 1;
-}
-
-int main(int argc, char *argv[]) {
- char *fin, *fout;
- FILE *fpin, *fpout;
- uint8_t *inbuf, *outbuf;
- uint8_t *inbuf_u, *outbuf_u;
- uint8_t *inbuf_v, *outbuf_v;
- int f, frames;
- int width, height, target_width, target_height;
- int failed = 0;
-
- exec_name = argv[0];
-
- if (argc < 5) {
- printf("Incorrect parameters:\n");
- usage();
- return 1;
- }
-
- fin = argv[1];
- fout = argv[4];
- if (!parse_dim(argv[2], &width, &height)) {
- printf("Incorrect parameters: %s\n", argv[2]);
- usage();
- return 1;
- }
- if (!parse_dim(argv[3], &target_width, &target_height)) {
- printf("Incorrect parameters: %s\n", argv[3]);
- usage();
- return 1;
- }
-
- fpin = fopen(fin, "rb");
- if (fpin == NULL) {
- printf("Can't open file %s to read\n", fin);
- usage();
- return 1;
- }
- fpout = fopen(fout, "wb");
- if (fpout == NULL) {
- fclose(fpin);
- printf("Can't open file %s to write\n", fout);
- usage();
- return 1;
- }
- if (argc >= 6)
- frames = atoi(argv[5]);
- else
- frames = INT_MAX;
-
- printf("Input size: %dx%d\n", width, height);
- printf("Target size: %dx%d, Frames: ", target_width, target_height);
- if (frames == INT_MAX)
- printf("All\n");
- else
- printf("%d\n", frames);
-
- inbuf = (uint8_t *)malloc(width * height * 3 / 2);
- outbuf = (uint8_t *)malloc(target_width * target_height * 3 / 2);
- if (!(inbuf && outbuf)) {
- printf("Failed to allocate buffers.\n");
- failed = 1;
- goto Error;
- }
- inbuf_u = inbuf + width * height;
- inbuf_v = inbuf_u + width * height / 4;
- outbuf_u = outbuf + target_width * target_height;
- outbuf_v = outbuf_u + target_width * target_height / 4;
- f = 0;
- while (f < frames) {
- if (fread(inbuf, width * height * 3 / 2, 1, fpin) != 1) break;
- av1_resize_frame420(inbuf, width, inbuf_u, inbuf_v, width / 2, height,
- width, outbuf, target_width, outbuf_u, outbuf_v,
- target_width / 2, target_height, target_width);
- fwrite(outbuf, target_width * target_height * 3 / 2, 1, fpout);
- f++;
- }
- printf("%d frames processed\n", f);
-Error:
- fclose(fpin);
- fclose(fpout);
-
- free(inbuf);
- free(outbuf);
- return failed;
-}
diff --git a/examples/svc_encoder_rtc.cc b/examples/svc_encoder_rtc.cc
index 1730f89..c37df79 100644
--- a/examples/svc_encoder_rtc.cc
+++ b/examples/svc_encoder_rtc.cc
@@ -18,6 +18,8 @@
#include <stdlib.h>
#include <string.h>
+#include <memory>
+
#include "config/aom_config.h"
#if CONFIG_AV1_DECODER
@@ -30,6 +32,7 @@
#include "common/video_writer.h"
#include "examples/encoder_util.h"
#include "aom_ports/aom_timer.h"
+#include "av1/ratectrl_rtc.h"
#define OPTION_BUFFER_SIZE 1024
@@ -44,6 +47,7 @@
int decode;
int tune_content;
int show_psnr;
+ bool use_external_rc;
} AppInput;
typedef enum {
@@ -99,6 +103,8 @@
"Attempt to test decoding the output when set to 1. Default is 1.");
static const arg_def_t psnr_arg =
ARG_DEF(NULL, "psnr", -1, "Show PSNR in status line.");
+static const arg_def_t ext_rc_arg =
+ ARG_DEF(NULL, "use-ext-rc", 0, "Use external rate control.");
static const struct arg_enum_list tune_content_enum[] = {
{ "default", AOM_CONTENT_DEFAULT },
{ "screen", AOM_CONTENT_SCREEN },
@@ -372,6 +378,8 @@
printf("tune content %d\n", app_input->tune_content);
} else if (arg_match(&arg, &psnr_arg, argi)) {
app_input->show_psnr = 1;
+ } else if (arg_match(&arg, &ext_rc_arg, argi)) {
+ app_input->use_external_rc = true;
} else {
++argj;
}
@@ -429,10 +437,12 @@
enc_cfg->rc_target_bitrate, enc_cfg->kf_max_dist);
}
-static int mode_to_num_temporal_layers[11] = {
- 1, 2, 3, 3, 2, 1, 1, 3, 3, 3, 3
+static int mode_to_num_temporal_layers[12] = {
+ 1, 2, 3, 3, 2, 1, 1, 3, 3, 3, 3, 3,
};
-static int mode_to_num_spatial_layers[11] = { 1, 1, 1, 1, 1, 2, 3, 2, 3, 3, 3 };
+static int mode_to_num_spatial_layers[12] = {
+ 1, 1, 1, 1, 1, 2, 3, 2, 3, 3, 3, 3,
+};
// For rate control encoding stats.
struct RateControlMetrics {
@@ -607,6 +617,7 @@
int i;
int enable_longterm_temporal_ref = 1;
int shift = (layering_mode == 8) ? 2 : 0;
+ int simulcast_mode = (layering_mode == 11);
*use_svc_control = 1;
layer_id->spatial_layer_id = spatial_layer_id;
int lag_index = 0;
@@ -1102,7 +1113,173 @@
ref_frame_config->ref_idx[SVC_GOLDEN_FRAME] = 4;
}
}
- if (layer_id->spatial_layer_id > 0) {
+ break;
+ case 11:
+ // Simulcast mode for 3 spatial and 3 temporal layers.
+ // No inter-layer predicton, only prediction is temporal and single
+ // reference (LAST).
+ // No overlap in buffer slots between spatial layers. So for example,
+ // SL0 only uses slots 0 and 1.
+ // SL1 only uses slots 2 and 3.
+ // SL2 only uses slots 4 and 5.
+ // All 7 references for each inter-frame must only access buffer slots
+ // for that spatial layer.
+ // On key (super)frames: SL1 and SL2 must have no references set
+ // and must refresh all the slots for that layer only (so 2 and 3
+ // for SL1, 4 and 5 for SL2). The base SL0 will be labelled internally
+ // as a Key frame (refresh all slots). SL1/SL2 will be labelled
+ // internally as Intra-only frames that allow that stream to be decoded.
+ // These conditions will allow for each spatial stream to be
+ // independently decodeable.
+
+ // Initialize all references to 0 (don't use reference).
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->reference[i] = 0;
+ // Initialize as no refresh/update for all slots.
+ for (i = 0; i < REF_FRAMES; i++) ref_frame_config->refresh[i] = 0;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 0;
+
+ if (is_key_frame) {
+ if (layer_id->spatial_layer_id == 0) {
+ // Assign LAST/GOLDEN to slot 0/1.
+ // Refesh slots 0 and 1 for SL0.
+ // SL0: this will get set to KEY frame internally.
+ ref_frame_config->ref_idx[SVC_LAST_FRAME] = 0;
+ ref_frame_config->ref_idx[SVC_GOLDEN_FRAME] = 1;
+ ref_frame_config->refresh[0] = 1;
+ ref_frame_config->refresh[1] = 1;
+ } else if (layer_id->spatial_layer_id == 1) {
+ // Assign LAST/GOLDEN to slot 2/3.
+ // Refesh slots 2 and 3 for SL1.
+ // This will get set to Intra-only frame internally.
+ ref_frame_config->ref_idx[SVC_LAST_FRAME] = 2;
+ ref_frame_config->ref_idx[SVC_GOLDEN_FRAME] = 3;
+ ref_frame_config->refresh[2] = 1;
+ ref_frame_config->refresh[3] = 1;
+ } else if (layer_id->spatial_layer_id == 2) {
+ // Assign LAST/GOLDEN to slot 4/5.
+ // Refresh slots 4 and 5 for SL2.
+ // This will get set to Intra-only frame internally.
+ ref_frame_config->ref_idx[SVC_LAST_FRAME] = 4;
+ ref_frame_config->ref_idx[SVC_GOLDEN_FRAME] = 5;
+ ref_frame_config->refresh[4] = 1;
+ ref_frame_config->refresh[5] = 1;
+ }
+ } else if (superframe_cnt % 4 == 0) {
+ // Base temporal layer: TL0
+ layer_id->temporal_layer_id = 0;
+ if (layer_id->spatial_layer_id == 0) { // SL0
+ // Reference LAST. Assign all references to either slot
+ // 0 or 1. Here we assign LAST to slot 0, all others to 1.
+ // Update slot 0 (LAST).
+ ref_frame_config->reference[SVC_LAST_FRAME] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 1;
+ ref_frame_config->ref_idx[SVC_LAST_FRAME] = 0;
+ ref_frame_config->refresh[0] = 1;
+ } else if (layer_id->spatial_layer_id == 1) { // SL1
+ // Reference LAST. Assign all references to either slot
+ // 2 or 3. Here we assign LAST to slot 2, all others to 3.
+ // Update slot 2 (LAST).
+ ref_frame_config->reference[SVC_LAST_FRAME] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 3;
+ ref_frame_config->ref_idx[SVC_LAST_FRAME] = 2;
+ ref_frame_config->refresh[2] = 1;
+ } else if (layer_id->spatial_layer_id == 2) { // SL2
+ // Reference LAST. Assign all references to either slot
+ // 4 or 5. Here we assign LAST to slot 4, all others to 5.
+ // Update slot 4 (LAST).
+ ref_frame_config->reference[SVC_LAST_FRAME] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 5;
+ ref_frame_config->ref_idx[SVC_LAST_FRAME] = 4;
+ ref_frame_config->refresh[4] = 1;
+ }
+ } else if ((superframe_cnt - 1) % 4 == 0) {
+ // First top temporal enhancement layer: TL2
+ layer_id->temporal_layer_id = 2;
+ if (layer_id->spatial_layer_id == 0) { // SL0
+ // Reference LAST (slot 0). Assign other references to slot 1.
+ // No update/refresh on any slots.
+ ref_frame_config->reference[SVC_LAST_FRAME] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 1;
+ ref_frame_config->ref_idx[SVC_LAST_FRAME] = 0;
+ } else if (layer_id->spatial_layer_id == 1) { // SL1
+ // Reference LAST (slot 2). Assign other references to slot 3.
+ // No update/refresh on any slots.
+ ref_frame_config->reference[SVC_LAST_FRAME] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 3;
+ ref_frame_config->ref_idx[SVC_LAST_FRAME] = 2;
+ } else if (layer_id->spatial_layer_id == 2) { // SL2
+ // Reference LAST (slot 4). Assign other references to slot 4.
+ // No update/refresh on any slots.
+ ref_frame_config->reference[SVC_LAST_FRAME] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 5;
+ ref_frame_config->ref_idx[SVC_LAST_FRAME] = 4;
+ }
+ } else if ((superframe_cnt - 2) % 4 == 0) {
+ // Middle temporal enhancement layer: TL1
+ layer_id->temporal_layer_id = 1;
+ if (layer_id->spatial_layer_id == 0) { // SL0
+ // Reference LAST (slot 0).
+ // Set GOLDEN to slot 1 and update slot 1.
+ // This will be used as reference for next TL2.
+ ref_frame_config->reference[SVC_LAST_FRAME] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 1;
+ ref_frame_config->ref_idx[SVC_LAST_FRAME] = 0;
+ ref_frame_config->refresh[1] = 1;
+ } else if (layer_id->spatial_layer_id == 1) { // SL1
+ // Reference LAST (slot 2).
+ // Set GOLDEN to slot 3 and update slot 3.
+ // This will be used as reference for next TL2.
+ ref_frame_config->reference[SVC_LAST_FRAME] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 3;
+ ref_frame_config->ref_idx[SVC_LAST_FRAME] = 2;
+ ref_frame_config->refresh[3] = 1;
+ } else if (layer_id->spatial_layer_id == 2) { // SL2
+ // Reference LAST (slot 4).
+ // Set GOLDEN to slot 5 and update slot 5.
+ // This will be used as reference for next TL2.
+ ref_frame_config->reference[SVC_LAST_FRAME] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 5;
+ ref_frame_config->ref_idx[SVC_LAST_FRAME] = 4;
+ ref_frame_config->refresh[5] = 1;
+ }
+ } else if ((superframe_cnt - 3) % 4 == 0) {
+ // Second top temporal enhancement layer: TL2
+ layer_id->temporal_layer_id = 2;
+ if (layer_id->spatial_layer_id == 0) { // SL0
+ // Reference LAST (slot 1). Assign other references to slot 0.
+ // No update/refresh on any slots.
+ ref_frame_config->reference[SVC_LAST_FRAME] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 0;
+ ref_frame_config->ref_idx[SVC_LAST_FRAME] = 1;
+ } else if (layer_id->spatial_layer_id == 1) { // SL1
+ // Reference LAST (slot 3). Assign other references to slot 2.
+ // No update/refresh on any slots.
+ ref_frame_config->reference[SVC_LAST_FRAME] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 2;
+ ref_frame_config->ref_idx[SVC_LAST_FRAME] = 3;
+ } else if (layer_id->spatial_layer_id == 2) { // SL2
+ // Reference LAST (slot 5). Assign other references to slot 4.
+ // No update/refresh on any slots.
+ ref_frame_config->reference[SVC_LAST_FRAME] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 4;
+ ref_frame_config->ref_idx[SVC_LAST_FRAME] = 5;
+ }
+ }
+ if (!simulcast_mode && layer_id->spatial_layer_id > 0) {
// Always reference GOLDEN (inter-layer prediction).
ref_frame_config->reference[SVC_GOLDEN_FRAME] = 1;
if (ksvc_mode) {
@@ -1120,8 +1297,8 @@
// allow for top spatial layer to use additional temporal reference.
// Additional reference is only updated on base temporal layer, every
// 10 TL0 frames here.
- if (enable_longterm_temporal_ref && layer_id->spatial_layer_id == 2 &&
- layering_mode == 8) {
+ if (!simulcast_mode && enable_longterm_temporal_ref &&
+ layer_id->spatial_layer_id == 2 && layering_mode == 8) {
ref_frame_config->ref_idx[SVC_ALTREF_FRAME] = REF_FRAMES - 1;
if (!is_key_frame) ref_frame_config->reference[SVC_ALTREF_FRAME] = 1;
if (base_count % 10 == 0 && layer_id->temporal_layer_id == 0)
@@ -1220,6 +1397,51 @@
fprintf(stderr, "\n");
}
+static aom::AV1RateControlRtcConfig create_rtc_rc_config(
+ const aom_codec_enc_cfg_t &cfg, const AppInput &app_input) {
+ aom::AV1RateControlRtcConfig rc_cfg;
+ rc_cfg.width = cfg.g_w;
+ rc_cfg.height = cfg.g_h;
+ rc_cfg.max_quantizer = cfg.rc_max_quantizer;
+ rc_cfg.min_quantizer = cfg.rc_min_quantizer;
+ rc_cfg.target_bandwidth = cfg.rc_target_bitrate;
+ rc_cfg.buf_initial_sz = cfg.rc_buf_initial_sz;
+ rc_cfg.buf_optimal_sz = cfg.rc_buf_optimal_sz;
+ rc_cfg.buf_sz = cfg.rc_buf_sz;
+ rc_cfg.overshoot_pct = cfg.rc_overshoot_pct;
+ rc_cfg.undershoot_pct = cfg.rc_undershoot_pct;
+ // This is hardcoded as AOME_SET_MAX_INTRA_BITRATE_PCT
+ rc_cfg.max_intra_bitrate_pct = 300;
+ rc_cfg.framerate = cfg.g_timebase.den;
+ // TODO(jianj): Add suppor for SVC.
+ rc_cfg.ss_number_layers = 1;
+ rc_cfg.ts_number_layers = 1;
+ rc_cfg.scaling_factor_num[0] = 1;
+ rc_cfg.scaling_factor_den[0] = 1;
+ rc_cfg.layer_target_bitrate[0] = static_cast<int>(rc_cfg.target_bandwidth);
+ rc_cfg.max_quantizers[0] = rc_cfg.max_quantizer;
+ rc_cfg.min_quantizers[0] = rc_cfg.min_quantizer;
+ rc_cfg.aq_mode = app_input.aq_mode;
+
+ return rc_cfg;
+}
+
+static int qindex_to_quantizer(int qindex) {
+ // Table that converts 0-63 range Q values passed in outside to the 0-255
+ // range Qindex used internally.
+ static const int quantizer_to_qindex[] = {
+ 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48,
+ 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92, 96, 100,
+ 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 148, 152,
+ 156, 160, 164, 168, 172, 176, 180, 184, 188, 192, 196, 200, 204,
+ 208, 212, 216, 220, 224, 228, 232, 236, 240, 244, 249, 255,
+ };
+ for (int quantizer = 0; quantizer < 64; ++quantizer)
+ if (quantizer_to_qindex[quantizer] >= qindex) return quantizer;
+
+ return 63;
+}
+
int main(int argc, const char **argv) {
AppInput app_input;
AvxVideoWriter *outfile[AOM_MAX_LAYERS] = { NULL };
@@ -1447,6 +1669,12 @@
aom_codec_control(&codec, AV1E_SET_ENABLE_INTRABC, 0);
}
+ if (app_input.use_external_rc) {
+ aom_codec_control(&codec, AV1E_SET_RTC_EXTERNAL_RC, 1);
+ }
+
+ aom_codec_control(&codec, AV1E_SET_MAX_CONSEC_FRAME_DROP_CBR, INT_MAX);
+
svc_params.number_spatial_layers = ss_number_layers;
svc_params.number_temporal_layers = ts_number_layers;
for (i = 0; i < ss_number_layers * ts_number_layers; ++i) {
@@ -1483,6 +1711,13 @@
frame_cnt_layer[lx] = 0;
}
+ std::unique_ptr<aom::AV1RateControlRTC> rc_api;
+ if (app_input.use_external_rc) {
+ const aom::AV1RateControlRtcConfig rc_cfg =
+ create_rtc_rc_config(cfg, app_input);
+ rc_api = aom::AV1RateControlRTC::Create(rc_cfg);
+ }
+
frame_avail = 1;
struct psnr_stats psnr_stream;
memset(&psnr_stream, 0, sizeof(psnr_stream));
@@ -1621,6 +1856,21 @@
die_codec(&codec, "Failed to SET_BITRATE_ONE_PASS_CBR");
}
+ if (rc_api) {
+ aom::AV1FrameParamsRTC frame_params;
+ // TODO(jianj): Add support for SVC.
+ frame_params.spatial_layer_id = 0;
+ frame_params.temporal_layer_id = 0;
+ frame_params.frame_type =
+ is_key_frame ? aom::kKeyFrame : aom::kInterFrame;
+ rc_api->ComputeQP(frame_params);
+ const int current_qp = rc_api->GetQP();
+ if (aom_codec_control(&codec, AV1E_SET_QUANTIZER_ONE_PASS,
+ qindex_to_quantizer(current_qp))) {
+ die_codec(&codec, "Failed to SET_QUANTIZER_ONE_PASS");
+ }
+ }
+
// Do the layer encode.
aom_usec_timer_start(&timer);
if (aom_codec_encode(&codec, frame_avail ? &raw : NULL, pts, 1, flags))
@@ -1631,10 +1881,14 @@
frame_cnt_layer[layer] += 1;
got_data = 0;
+ // For simulcast (mode 11): write out each spatial layer to the file.
+ int ss_layers_write = (app_input.layering_mode == 11)
+ ? layer_id.spatial_layer_id + 1
+ : ss_number_layers;
while ((pkt = aom_codec_get_cx_data(&codec, &iter))) {
switch (pkt->kind) {
case AOM_CODEC_CX_FRAME_PKT:
- for (int sl = layer_id.spatial_layer_id; sl < ss_number_layers;
+ for (int sl = layer_id.spatial_layer_id; sl < ss_layers_write;
++sl) {
for (int tl = layer_id.temporal_layer_id; tl < ts_number_layers;
++tl) {
@@ -1675,6 +1929,9 @@
if (slx == 0) ++rc.layer_enc_frames[layer_id.temporal_layer_id];
}
+ if (rc_api) {
+ rc_api->PostEncodeUpdate(pkt->data.frame.sz);
+ }
// Update for short-time encoding bitrate states, for moving window
// of size rc->window, shifted by rc->window / 2.
// Ignore first window segment, due to key frame.
diff --git a/test/active_map_test.cc b/test/active_map_test.cc
index a9f7f85..979ee6b 100644
--- a/test/active_map_test.cc
+++ b/test/active_map_test.cc
@@ -27,15 +27,15 @@
static const int kHeight = 144;
ActiveMapTest() : EncoderTest(GET_PARAM(0)) {}
- virtual ~ActiveMapTest() {}
+ ~ActiveMapTest() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(GET_PARAM(1));
cpu_used_ = GET_PARAM(2);
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AOME_SET_CPUUSED, cpu_used_);
encoder->Control(AV1E_SET_ALLOW_WARPED_MOTION, 0);
diff --git a/test/allintra_end_to_end_test.cc b/test/allintra_end_to_end_test.cc
index 98a7973..8ec24aa 100644
--- a/test/allintra_end_to_end_test.cc
+++ b/test/allintra_end_to_end_test.cc
@@ -56,25 +56,25 @@
deltaq_mode_(GET_PARAM(3)), threads_(GET_PARAM(4)),
tile_columns_(GET_PARAM(5)), enable_tx_size_search_(GET_PARAM(6)) {}
- virtual ~AllIntraEndToEndTest() {}
+ ~AllIntraEndToEndTest() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(::libaom_test::kAllIntra);
cfg_.g_threads = threads_;
}
- virtual void BeginPassHook(unsigned int) {
+ void BeginPassHook(unsigned int) override {
psnr_ = 0.0;
nframes_ = 0;
}
- virtual void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) {
+ void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) override {
psnr_ += pkt->data.psnr.psnr[0];
nframes_++;
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AV1E_SET_ROW_MT, 1);
encoder->Control(AV1E_SET_TUNE_CONTENT, AOM_CONTENT_DEFAULT);
diff --git a/test/altref_test.cc b/test/altref_test.cc
index 002a206..081123c 100644
--- a/test/altref_test.cc
+++ b/test/altref_test.cc
@@ -58,9 +58,9 @@
rc_end_usage_(GET_PARAM(2)) {
is_arf_frame_present_ = 0;
}
- virtual ~AltRefFramePresenceTestLarge() {}
+ ~AltRefFramePresenceTestLarge() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(altref_test_params_.encoding_mode);
const aom_rational timebase = { 1, 30 };
cfg_.g_timebase = timebase;
@@ -71,10 +71,10 @@
cfg_.g_lag_in_frames = altref_test_params_.lag_in_frames;
}
- virtual bool DoDecode() const { return 1; }
+ bool DoDecode() const override { return true; }
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AOME_SET_CPUUSED, 5);
encoder->Control(AOME_SET_ENABLEAUTOALTREF, 1);
@@ -85,8 +85,8 @@
}
}
- virtual bool HandleDecodeResult(const aom_codec_err_t res_dec,
- libaom_test::Decoder *decoder) {
+ bool HandleDecodeResult(const aom_codec_err_t res_dec,
+ libaom_test::Decoder *decoder) override {
EXPECT_EQ(AOM_CODEC_OK, res_dec) << decoder->DecodeError();
if (is_arf_frame_present_ != 1 && AOM_CODEC_OK == res_dec) {
aom_codec_ctx_t *ctx_dec = decoder->GetDecoder();
@@ -149,9 +149,9 @@
limit_ = 60;
frame_num_ = 0;
}
- virtual ~GoldenFrameIntervalTestLarge() {}
+ ~GoldenFrameIntervalTestLarge() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(gf_interval_param_.encoding_mode);
const aom_rational timebase = { 1, 30 };
cfg_.g_timebase = timebase;
@@ -166,10 +166,10 @@
cfg_.rc_target_bitrate = 1000;
}
- virtual bool DoDecode() const { return 1; }
+ bool DoDecode() const override { return true; }
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AOME_SET_CPUUSED, 5);
encoder->Control(AOME_SET_ENABLEAUTOALTREF, 1);
@@ -189,7 +189,7 @@
}
}
- virtual void FramePktHook(const aom_codec_cx_pkt_t *pkt) {
+ void FramePktHook(const aom_codec_cx_pkt_t *pkt) override {
(void)pkt;
++frame_num_;
}
diff --git a/test/aq_segment_test.cc b/test/aq_segment_test.cc
index b4a8b61..674a883 100644
--- a/test/aq_segment_test.cc
+++ b/test/aq_segment_test.cc
@@ -32,16 +32,16 @@
public ::libaom_test::EncoderTest {
protected:
AqSegmentTest() : EncoderTest(GET_PARAM(0)) {}
- virtual ~AqSegmentTest() {}
+ ~AqSegmentTest() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(GET_PARAM(1));
set_cpu_used_ = GET_PARAM(2);
aq_mode_ = 0;
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AOME_SET_CPUUSED, set_cpu_used_);
encoder->Control(AV1E_SET_AQ_MODE, aq_mode_);
diff --git a/test/arf_freq_test.cc b/test/arf_freq_test.cc
index 63ccdfc..f51444d 100644
--- a/test/arf_freq_test.cc
+++ b/test/arf_freq_test.cc
@@ -80,9 +80,9 @@
: EncoderTest(GET_PARAM(0)), test_video_param_(GET_PARAM(1)),
test_encode_param_(GET_PARAM(2)), min_arf_requested_(GET_PARAM(3)) {}
- virtual ~ArfFreqTestLarge() {}
+ ~ArfFreqTestLarge() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(test_encode_param_.mode);
if (test_encode_param_.mode != ::libaom_test::kRealTime) {
cfg_.g_lag_in_frames = 25;
@@ -93,7 +93,7 @@
}
}
- virtual void BeginPassHook(unsigned int) {
+ void BeginPassHook(unsigned int) override {
min_run_ = ARF_NOT_SEEN;
run_of_visible_frames_ = 0;
}
@@ -115,7 +115,7 @@
return frames;
}
- virtual void FramePktHook(const aom_codec_cx_pkt_t *pkt) {
+ void FramePktHook(const aom_codec_cx_pkt_t *pkt) override {
if (pkt->kind != AOM_CODEC_CX_FRAME_PKT) return;
const int frames = GetNumFramesInPkt(pkt);
if (frames == 1) {
@@ -134,8 +134,8 @@
}
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AV1E_SET_FRAME_PARALLEL_DECODING, 1);
encoder->Control(AV1E_SET_TILE_COLUMNS, 4);
diff --git a/test/av1_c_vs_simd_encode.sh b/test/av1_c_vs_simd_encode.sh
new file mode 100755
index 0000000..29a2f99
--- /dev/null
+++ b/test/av1_c_vs_simd_encode.sh
@@ -0,0 +1,559 @@
+#!/bin/sh
+## Copyright (c) 2023, Alliance for Open Media. All rights reserved
+##
+## This source code is subject to the terms of the BSD 2 Clause License and
+## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+## was not distributed with this source code in the LICENSE file, you can
+## obtain it at www.aomedia.org/license/software. If the Alliance for Open
+## Media Patent License 1.0 was not distributed with this source code in the
+## PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+##
+## This script checks the bit exactness between C and SIMD
+## implementations of AV1 encoder.
+##
+. $(dirname $0)/tools_common.sh
+
+PRESETS="good rt"
+LOWBD_CIF_CLIP="yuv_raw_input"
+LOWBD_480p_CLIP="yuv_480p_raw_input"
+LOWBD_720p_CLIP="y4m_720p_input"
+HIGHBD_CLIP="y4m_360p_10bit_input"
+SC_CLIP="y4m_screen_input"
+OUT_FILE_SUFFIX=".ivf"
+SCRIPT_DIR=$(dirname "$0")
+LIBAOM_SOURCE_DIR=$(cd ${SCRIPT_DIR}/..; pwd)
+
+# Clips used in test.
+YUV_RAW_INPUT="${LIBAOM_TEST_DATA_PATH}/hantro_collage_w352h288.yuv"
+YUV_480P_RAW_INPUT="${LIBAOM_TEST_DATA_PATH}/niklas_640_480_30.yuv"
+Y4M_360P_10BIT_INPUT="${LIBAOM_TEST_DATA_PATH}/crowd_run_360p_10_150f.y4m"
+Y4M_720P_INPUT="${LIBAOM_TEST_DATA_PATH}/niklas_1280_720_30.y4m"
+Y4M_SCREEN_INPUT="${LIBAOM_TEST_DATA_PATH}/wikipedia_420_360p_60f.y4m"
+
+# Number of frames to test.
+AV1_ENCODE_C_VS_SIMD_TEST_FRAME_LIMIT=35
+
+# Create a temporary directory for output files.
+if [ -n "${TMPDIR}" ]; then
+ AOM_TEST_TEMP_ROOT="${TMPDIR}"
+elif [ -n "${TEMPDIR}" ]; then
+ AOM_TEST_TEMP_ROOT="${TEMPDIR}"
+else
+ AOM_TEST_TEMP_ROOT=/tmp
+fi
+
+AOM_TEST_OUTPUT_DIR="${AOM_TEST_TEMP_ROOT}/av1_test_$$"
+
+if ! mkdir -p "${AOM_TEST_OUTPUT_DIR}" || \
+ [ ! -d "${AOM_TEST_OUTPUT_DIR}" ]; then
+ echo "${0##*/}: Cannot create output directory, giving up."
+ echo "${0##*/}: AOM_TEST_OUTPUT_DIR=${AOM_TEST_OUTPUT_DIR}"
+ exit 1
+fi
+
+elog() {
+ echo "$@" 1>&2
+}
+
+# Echoes path to $1 when it's executable and exists in ${AOM_TEST_OUTPUT_DIR},
+# or an empty string. Caller is responsible for testing the string once the
+# function returns.
+av1_enc_tool_path() {
+ local target="$1"
+ local preset="$2"
+ local tool_path="${AOM_TEST_OUTPUT_DIR}/build_target_${target}/aomenc_${preset}"
+
+ if [ ! -x "${tool_path}" ]; then
+ tool_path=""
+ fi
+ echo "${tool_path}"
+}
+
+# Environment check: Make sure input and source directories are available.
+av1_c_vs_simd_enc_verify_environment () {
+ if [ ! -e "${YUV_RAW_INPUT}" ]; then
+ elog "libaom test data must exist in LIBAOM_TEST_DATA_PATH."
+ return 1
+ fi
+ if [ ! -e "${Y4M_360P_10BIT_INPUT}" ]; then
+ elog "libaom test data must exist in LIBAOM_TEST_DATA_PATH."
+ return 1
+ fi
+ if [ ! -e "${YUV_480P_RAW_INPUT}" ]; then
+ elog "libaom test data must exist in LIBAOM_TEST_DATA_PATH."
+ return 1
+ fi
+ if [ ! -e "${Y4M_720P_INPUT}" ]; then
+ elog "libaom test data must exist in LIBAOM_TEST_DATA_PATH."
+ return 1
+ fi
+ if [ ! -e "${Y4M_SCREEN_INPUT}" ]; then
+ elog "libaom test data must exist in LIBAOM_TEST_DATA_PATH."
+ return 1
+ fi
+ if [ ! -d "$LIBAOM_SOURCE_DIR" ]; then
+ elog "LIBAOM_SOURCE_DIR does not exist."
+ return 1
+ fi
+}
+
+# This is not needed since tools_common.sh does the same cleanup.
+# Keep the code here for our reference.
+# cleanup() {
+# rm -rf ${AOM_TEST_OUTPUT_DIR}
+# }
+
+# Echo AOM_SIMD_CAPS_MASK for different instruction set architecture.
+avx512f() {
+ echo "0x1FF"
+}
+
+avx2() {
+ echo "0x0FF"
+}
+
+avx() {
+ echo "0x07F"
+}
+
+sse4_1() {
+ echo "0x03F"
+}
+
+ssse3() {
+ echo "0x01F"
+}
+
+sse3() {
+ echo "0x00F"
+}
+
+sse2() {
+ echo "0x007"
+}
+
+get_bitrates() {
+ local content=$1
+ local preset=$2
+
+ # Bit-rates:
+ local bitrate_lowres_good="300"
+ local bitrate_480p_good="500"
+ local bitrate_720p_good="1000"
+ local bitrate_scc_360p_good="500"
+ local bitrate_lowres_rt="200"
+ local bitrate_480p_rt="300"
+ local bitrate_720p_rt="600"
+ local bitrate_scc_360p_rt="300"
+ local bitrate_hbd_360p="500"
+
+ if [ "${preset}" = "good" ]; then
+ if [ "${content}" = "yuv_raw_input" ]; then
+ echo "${bitrate_lowres_good}"
+ elif [ "${content}" = "yuv_480p_raw_input" ]; then
+ echo "${bitrate_480p_good}"
+ elif [ "${content}" = "y4m_720p_input" ]; then
+ echo "${bitrate_720p_good}"
+ elif [ "${content}" = "y4m_screen_input" ]; then
+ echo "${bitrate_scc_360p_good}"
+ elif [ "${content}" = "y4m_360p_10bit_input" ]; then
+ echo "${bitrate_hbd_360p}"
+ else
+ elog "Invalid content"
+ fi
+ elif [ "${preset}" = "rt" ]; then
+ if [ "${content}" = "yuv_raw_input" ]; then
+ echo "${bitrate_lowres_rt}"
+ elif [ "${content}" = "yuv_480p_raw_input" ]; then
+ echo "${bitrate_480p_rt}"
+ elif [ "${content}" = "y4m_720p_input" ]; then
+ echo "${bitrate_720p_rt}"
+ elif [ "${content}" = "y4m_screen_input" ]; then
+ echo "${bitrate_scc_360p_rt}"
+ elif [ "${content}" = "y4m_360p_10bit_input" ]; then
+ echo "${bitrate_hbd_360p}"
+ else
+ elog "Invalid content"
+ fi
+ else
+ elog "invalid preset"
+ fi
+}
+
+# Echo clip details to be used as input to aomenc.
+yuv_raw_input() {
+ echo ""${YUV_RAW_INPUT}"
+ --width=352
+ --height=288
+ --bit-depth=8"
+}
+
+y4m_360p_10bit_input() {
+ echo ""${Y4M_360P_10BIT_INPUT}"
+ --bit-depth=10"
+}
+
+yuv_480p_raw_input() {
+ echo ""${YUV_480P_RAW_INPUT}"
+ --width=640
+ --height=480
+ --bit-depth=8"
+}
+
+y4m_720p_input() {
+ echo ""${Y4M_720P_INPUT}"
+ --bit-depth=8"
+}
+
+y4m_screen_input() {
+ echo ""${Y4M_SCREEN_INPUT}"
+ --tune-content=screen
+ --enable-palette=1
+ --bit-depth=8"
+}
+
+has_x86_isa_extn() {
+ instruction_set=$1
+ if ! grep -q "$instruction_set" /proc/cpuinfo; then
+ # This instruction set is not supported.
+ return 1
+ fi
+}
+
+# Echo good encode params for use with AV1 encoder.
+av1_encode_good_params() {
+ echo "--good \
+ --ivf \
+ --profile=0 \
+ --static-thresh=0 \
+ --threads=1 \
+ --tile-columns=0 \
+ --tile-rows=0 \
+ --verbose \
+ --end-usage=vbr \
+ --kf-max-dist=160 \
+ --kf-min-dist=0 \
+ --max-q=63 \
+ --min-q=0 \
+ --overshoot-pct=100 \
+ --undershoot-pct=100 \
+ --passes=2 \
+ --arnr-maxframes=7 \
+ --arnr-strength=5 \
+ --auto-alt-ref=1 \
+ --drop-frame=0 \
+ --frame-parallel=0 \
+ --lag-in-frames=35 \
+ --maxsection-pct=2000 \
+ --minsection-pct=0 \
+ --sharpness=0"
+}
+
+# Echo realtime encode params for use with AV1 encoder.
+av1_encode_rt_params() {
+ echo "--rt \
+ --ivf \
+ --profile=0 \
+ --static-thresh=0 \
+ --threads=1 \
+ --tile-columns=0 \
+ --tile-rows=0 \
+ --verbose \
+ --end-usage=cbr \
+ --kf-max-dist=90000 \
+ --max-q=58 \
+ --min-q=2 \
+ --overshoot-pct=50 \
+ --undershoot-pct=50 \
+ --passes=1 \
+ --aq-mode=3 \
+ --buf-initial-sz=500 \
+ --buf-optimal-sz=600 \
+ --buf-sz=1000 \
+ --coeff-cost-upd-freq=3 \
+ --dv-cost-upd-freq=3 \
+ --mode-cost-upd-freq=3 \
+ --mv-cost-upd-freq=3 \
+ --deltaq-mode=0 \
+ --enable-global-motion=0 \
+ --enable-obmc=0 \
+ --enable-order-hint=0 \
+ --enable-ref-frame-mvs=0 \
+ --enable-tpl-model=0 \
+ --enable-warped-motion=0 \
+ --lag-in-frames=0 \
+ --max-intra-rate=300 \
+ --noise-sensitivity=0"
+}
+
+# Configures for the given target in AOM_TEST_OUTPUT_DIR/build_target_${target}
+# directory.
+av1_enc_build() {
+ local target="$1"
+ local cmake_command="$2"
+ local tmp_build_dir=${AOM_TEST_OUTPUT_DIR}/build_target_${target}
+ if [ -d "$tmp_build_dir" ]; then
+ rm -rf $tmp_build_dir
+ fi
+
+ mkdir -p $tmp_build_dir
+ cd $tmp_build_dir
+
+ local cmake_common_args="-DCONFIG_EXCLUDE_SIMD_MISMATCH=1 \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DENABLE_CCACHE=1 \
+ '-DCMAKE_C_FLAGS_RELEASE=-O3 -g' \
+ '-DCMAKE_CXX_FLAGS_RELEASE=-O3 -g'"
+
+ for preset in $PRESETS; do
+ echo "Building target[${preset} encoding]: ${target}"
+ if [ "${preset}" = "good" ]; then
+ local cmake_extra_args="-DCONFIG_AV1_HIGHBITDEPTH=1"
+ elif [ "${preset}" = "rt" ]; then
+ local cmake_extra_args="-DCONFIG_REALTIME_ONLY=1 -DCONFIG_AV1_HIGHBITDEPTH=0"
+ else
+ elog "Invalid preset"
+ return 1
+ fi
+ eval "$cmake_command" "${cmake_common_args}" "${cmake_extra_args}" ${devnull}
+ eval make -j$(nproc) ${devnull}
+ mv aomenc aomenc_${preset}
+ done
+ echo "Done building target: ${target}"
+}
+
+compare_enc_output() {
+ local target=$1
+ local cpu=$2
+ local clip=$3
+ local bitrate=$4
+ local preset=$5
+ if ! diff -q ${AOM_TEST_OUTPUT_DIR}/Out-generic-"${clip}"-${preset}-${bitrate}kbps-cpu${cpu}${OUT_FILE_SUFFIX} \
+ ${AOM_TEST_OUTPUT_DIR}/Out-${target}-"${clip}"-${preset}-${bitrate}kbps-cpu${cpu}${OUT_FILE_SUFFIX}; then
+ elog "C vs ${target} encode mismatches for ${clip}, at ${bitrate} kbps, speed ${cpu}, ${preset} preset"
+ return 1
+ fi
+}
+
+av1_enc_test() {
+ local encoder="$1"
+ local arch="$2"
+ local target="$3"
+ local preset="$4"
+ if [ -z "$(av1_enc_tool_path "${target}" "${preset}")" ]; then
+ elog "aomenc_{preset} not found. It must exist in ${AOM_TEST_OUTPUT_DIR}/build_target_${target} path"
+ return 1
+ fi
+
+ if [ "${preset}" = "good" ]; then
+ if [ "${arch}" = "x86_64" ]; then
+ local min_cpu_used=0
+ local max_cpu_used=6
+ elif [ "${arch}" = "x86" ]; then
+ local min_cpu_used=2
+ local max_cpu_used=3
+ fi
+ local test_params=av1_encode_good_params
+ elif [ "${preset}" = "rt" ]; then
+ local min_cpu_used=5
+ local max_cpu_used=11
+ local test_params=av1_encode_rt_params
+ else
+ elog "Invalid preset"
+ return 1
+ fi
+
+ for cpu in $(seq $min_cpu_used $max_cpu_used); do
+ if [ "${preset}" = "good" ]; then
+ if [ "${arch}" = "x86_64" ]; then
+ if [ "${cpu}" -lt 2 ]; then
+ local test_clips="${LOWBD_CIF_CLIP} ${HIGHBD_CLIP}"
+ elif [ "${cpu}" -lt 5 ]; then
+ local test_clips="${LOWBD_480p_CLIP} ${HIGHBD_CLIP}"
+ else
+ local test_clips="${LOWBD_720p_CLIP} ${HIGHBD_CLIP}"
+ fi
+ elif [ "${arch}" = "x86" ]; then
+ local test_clips="${LOWBD_CIF_CLIP} ${HIGHBD_CLIP}"
+ elif [ "${arch}" = "arm64" ]; then
+ # TODO(BUG=aomedia:3474): Enable testing of high bit-depth clips after
+ # fixing C vs SIMD mismatches.
+ local test_clips="${LOWBD_CIF_CLIP}"
+ fi
+ elif [ "${preset}" = "rt" ]; then
+ if [ "${cpu}" -lt 8 ]; then
+ local test_clips="${LOWBD_CIF_CLIP} ${SC_CLIP}"
+ else
+ local test_clips="${LOWBD_480p_CLIP} ${SC_CLIP}"
+ fi
+ else
+ elog "Invalid preset"
+ return 1
+ fi
+
+ for clip in ${test_clips}; do
+ local test_bitrates=$(get_bitrates ${clip} ${preset})
+ for bitrate in ${test_bitrates}; do
+ eval "${encoder}" $($clip) $($test_params) \
+ "--limit=${AV1_ENCODE_C_VS_SIMD_TEST_FRAME_LIMIT}" \
+ "--cpu-used=${cpu}" "--target-bitrate=${bitrate}" "-o" \
+ ${AOM_TEST_OUTPUT_DIR}/Out-${target}-"${clip}"-${preset}-${bitrate}kbps-cpu${cpu}${OUT_FILE_SUFFIX} \
+ ${devnull}
+
+ if [ "${target}" != "generic" ]; then
+ if ! compare_enc_output ${target} $cpu ${clip} $bitrate ${preset}; then
+ # Found a mismatch
+ return 1
+ fi
+ fi
+ done
+ done
+ done
+}
+
+av1_test_generic() {
+ local arch=$1
+ local target="generic"
+ if [ $arch = "x86_64" ]; then
+ local cmake_command="cmake $LIBAOM_SOURCE_DIR -DAOM_TARGET_CPU=${target}"
+ elif [ $arch = "x86" ]; then
+ # As AV1 encode output differs for x86 32-bit and 64-bit platforms
+ # (BUG=aomedia:3479), the x86 32-bit C-only build is generated separately.
+ # The cmake command line option -DENABLE_MMX=0 flag disables all SIMD
+ # optimizations, and generates a C-only binary.
+ local cmake_command="cmake $LIBAOM_SOURCE_DIR -DENABLE_MMX=0 \
+ -DCMAKE_TOOLCHAIN_FILE=${LIBAOM_SOURCE_DIR}/build/cmake/toolchains/i686-linux-gcc.cmake"
+ fi
+
+ echo "Build for: Generic ${arch}"
+ av1_enc_build "${target}" "${cmake_command}"
+
+ for preset in $PRESETS; do
+ local encoder="$(av1_enc_tool_path "${target}" "${preset}")"
+ av1_enc_test $encoder "${arch}" "${target}" "${preset}"
+ done
+}
+
+# This function encodes AV1 bitstream by enabling SSE2, SSE3, SSSE3, SSE4_1, AVX, AVX2 as there are
+# no functions with MMX, SSE and AVX512 specialization.
+# The value of environment variable 'AOM_SIMD_CAPS_MASK' controls enabling of different instruction
+# set extension optimizations. The value of the flag 'AOM_SIMD_CAPS_MASK' and the corresponding
+# instruction set extension optimization enabled are as follows:
+# AVX512 AVX2 AVX SSE4_1 SSSE3 SSE3 SSE2 SSE MMX
+# 1 1 1 1 1 1 1 1 1 -> 0x1FF -> Enable AVX512 and lower variants
+# 0 1 1 1 1 1 1 1 1 -> 0x0FF -> Enable AVX2 and lower variants
+# 0 0 1 1 1 1 1 1 1 -> 0x07F -> Enable AVX and lower variants
+# 0 0 0 1 1 1 1 1 1 -> 0x03F -> Enable SSE4_1 and lower variants
+# 0 0 0 0 1 1 1 1 1 -> 0x01F -> Enable SSSE3 and lower variants
+# 0 0 0 0 0 1 1 1 1 -> 0x00F -> Enable SSE3 and lower variants
+# 0 0 0 0 0 0 1 1 1 -> 0x007 -> Enable SSE2 and lower variants
+# 0 0 0 0 0 0 0 1 1 -> 0x003 -> Enable SSE and lower variants
+# 0 0 0 0 0 0 0 0 1 -> 0x001 -> Enable MMX
+## NOTE: In x86_64 platform, it is not possible to enable sse/mmx/c using "AOM_SIMD_CAPS_MASK" as
+# all x86_64 platforms implement sse2.
+av1_test_x86() {
+ local arch=$1
+
+ if ! uname -m | grep -q "x86"; then
+ elog "Machine architecture is not x86 or x86_64"
+ return 0
+ fi
+
+ if [ $arch = "x86" ]; then
+ local target="x86-linux"
+ local cmake_command="cmake \
+ $LIBAOM_SOURCE_DIR \
+ -DCMAKE_TOOLCHAIN_FILE=${LIBAOM_SOURCE_DIR}/build/cmake/toolchains/i686-linux-gcc.cmake"
+ elif [ $arch = "x86_64" ]; then
+ local target="x86_64-linux"
+ local cmake_command="cmake $LIBAOM_SOURCE_DIR"
+ fi
+
+ # Available x86 isa variants: "avx2 avx sse4_1 ssse3 sse3 sse2"
+ local x86_isa_variants="avx2 sse4_1 sse2"
+
+ echo "Build for x86: ${target}"
+ av1_enc_build "${target}" "${cmake_command}"
+
+ for preset in $PRESETS; do
+ local encoder="$(av1_enc_tool_path "${target}" "${preset}")"
+ for isa in $x86_isa_variants; do
+ # Note that if has_x86_isa_extn returns 1, it is false, and vice versa.
+ if ! has_x86_isa_extn $isa; then
+ echo "${isa} is not supported in this machine"
+ continue
+ fi
+ export AOM_SIMD_CAPS_MASK=$($isa)
+ if ! av1_enc_test $encoder "${arch}" "${target}" "${preset}"; then
+ # Found a mismatch
+ return 1
+ fi
+ unset AOM_SIMD_CAPS_MASK
+ done
+ done
+}
+
+av1_test_arm() {
+ local arch="arm64"
+ local target="arm64-linux-gcc"
+ local cmake_command="cmake $LIBAOM_SOURCE_DIR \
+ -DCMAKE_TOOLCHAIN_FILE=$LIBAOM_SOURCE_DIR/build/cmake/toolchains/${target}.cmake \
+ -DCMAKE_C_FLAGS=-Wno-maybe-uninitialized"
+ echo "Build for arm64: ${target}"
+ av1_enc_build "${target}" "${cmake_command}"
+
+ for preset in $PRESETS; do
+ # Enable armv8 test for real-time only
+ # TODO(BUG=aomedia:3486, BUG=aomedia:3474): Enable testing for 'good' preset
+ # after fixing C vs NEON mismatches.
+ if [ "${preset}" = "good" ]; then
+ continue
+ fi
+ local encoder="$(av1_enc_tool_path "${target}" "${preset}")"
+ if ! av1_enc_test "qemu-aarch64 -L /usr/aarch64-linux-gnu ${encoder}" "${arch}" "${target}" "${preset}"; then
+ # Found a mismatch
+ return 1
+ fi
+ done
+}
+
+av1_c_vs_simd_enc_test () {
+ # Test x86 (32 bit)
+ # x86 requires the i686-linux-gnu toolchain:
+ # $ sudo apt-get install g++-i686-linux-gnu
+ echo "av1 test for x86 (32 bit): Started."
+ # Encode 'C' only
+ av1_test_generic "x86"
+ # Encode with SIMD optimizations enabled
+ if ! av1_test_x86 "x86"; then
+ echo "av1 test for x86 (32 bit): Done, test failed."
+ return 1
+ else
+ echo "av1 test for x86 (32 bit): Done, all tests passed."
+ fi
+
+ # Test x86_64 (64 bit)
+ if [ "$(eval uname -m)" = "x86_64" ]; then
+ echo "av1 test for x86_64 (64 bit): Started."
+ # Encode 'C' only
+ av1_test_generic "x86_64"
+ # Encode with SIMD optimizations enabled
+ if ! av1_test_x86 "x86_64"; then
+ echo "av1 test for x86_64 (64 bit): Done, test failed."
+ return 1
+ else
+ echo "av1 test for x86_64 (64 bit): Done, all tests passed."
+ fi
+ fi
+
+ # Test ARM
+ echo "av1_test_arm: Started."
+ if ! av1_test_arm; then
+ echo "av1 test for arm: Done, test failed."
+ return 1
+ else
+ echo "av1 test for arm: Done, all tests passed."
+ fi
+}
+
+run_tests av1_c_vs_simd_enc_verify_environment av1_c_vs_simd_enc_test
diff --git a/test/av1_convolve_scale_test.cc b/test/av1_convolve_scale_test.cc
index c321de2..76cf77a 100644
--- a/test/av1_convolve_scale_test.cc
+++ b/test/av1_convolve_scale_test.cc
@@ -258,14 +258,13 @@
class ConvolveScaleTestBase : public ::testing::Test {
public:
ConvolveScaleTestBase() : image_(nullptr) {}
- virtual ~ConvolveScaleTestBase() { delete image_; }
- virtual void TearDown() {}
+ ~ConvolveScaleTestBase() override { delete image_; }
// Implemented by subclasses (SetUp depends on the parameters passed
// in and RunOne depends on the function to be tested. These can't
// be templated for low/high bit depths because they have different
// numbers of parameters)
- virtual void SetUp() = 0;
+ void SetUp() override = 0;
virtual void RunOne(bool ref) = 0;
protected:
@@ -407,9 +406,9 @@
: public ConvolveScaleTestBase<uint8_t>,
public ::testing::WithParamInterface<LowBDParams> {
public:
- virtual ~LowBDConvolveScaleTest() {}
+ ~LowBDConvolveScaleTest() override = default;
- void SetUp() {
+ void SetUp() override {
tst_fun_ = GET_PARAM(0);
const BlockDimension &block = GET_PARAM(1);
@@ -421,7 +420,7 @@
SetParams(BaseParams(block, ntaps_x, ntaps_y, avg), bd);
}
- void RunOne(bool ref) {
+ void RunOne(bool ref) override {
const uint8_t *src = image_->GetSrcData(ref, false);
uint8_t *dst = image_->GetDstData(ref, false);
convolve_params_.dst = image_->GetDst16Data(ref, false);
@@ -490,9 +489,9 @@
: public ConvolveScaleTestBase<uint16_t>,
public ::testing::WithParamInterface<HighBDParams> {
public:
- virtual ~HighBDConvolveScaleTest() {}
+ ~HighBDConvolveScaleTest() override = default;
- void SetUp() {
+ void SetUp() override {
tst_fun_ = GET_PARAM(0);
const BlockDimension &block = GET_PARAM(1);
@@ -504,7 +503,7 @@
SetParams(BaseParams(block, ntaps_x, ntaps_y, avg), bd);
}
- void RunOne(bool ref) {
+ void RunOne(bool ref) override {
const uint16_t *src = image_->GetSrcData(ref, false);
uint16_t *dst = image_->GetDstData(ref, false);
convolve_params_.dst = image_->GetDst16Data(ref, false);
diff --git a/test/av1_convolve_test.cc b/test/av1_convolve_test.cc
index 873960d..5bbac21 100644
--- a/test/av1_convolve_test.cc
+++ b/test/av1_convolve_test.cc
@@ -183,14 +183,12 @@
template <typename T>
class AV1ConvolveTest : public ::testing::TestWithParam<TestParam<T>> {
public:
- ~AV1ConvolveTest() override { TearDown(); }
+ ~AV1ConvolveTest() override = default;
void SetUp() override {
rnd_.Reset(libaom_test::ACMRandom::DeterministicSeed());
}
- void TearDown() override {}
-
// Randomizes the 8-bit input buffer and returns a pointer to it. Note that
// the pointer is safe to use with an 8-tap filter. The stride can range
// from width to (width + kPadding). Also note that the pointer is to the
@@ -427,6 +425,99 @@
BuildLowbdParams(av1_convolve_x_sr_neon));
#endif
+#if HAVE_NEON_DOTPROD
+INSTANTIATE_TEST_SUITE_P(NEON_DOTPROD, AV1ConvolveXTest,
+ BuildLowbdParams(av1_convolve_x_sr_neon_dotprod));
+#endif
+
+#if HAVE_NEON_I8MM
+INSTANTIATE_TEST_SUITE_P(NEON_I8MM, AV1ConvolveXTest,
+ BuildLowbdParams(av1_convolve_x_sr_neon_i8mm));
+#endif
+
+////////////////////////////////////////////////////////////////
+// Single reference convolve-x IntraBC functions (low bit-depth)
+////////////////////////////////////////////////////////////////
+
+class AV1ConvolveXIntraBCTest : public AV1ConvolveTest<convolve_x_func> {
+ public:
+ void RunTest() {
+ // IntraBC functions only operate for subpel_x_qn = 8.
+ constexpr int kSubX = 8;
+ const int width = GetParam().Block().Width();
+ const int height = GetParam().Block().Height();
+ const InterpFilterParams *filter_params_x = &av1_intrabc_filter_params;
+ const uint8_t *input = FirstRandomInput8(GetParam());
+
+ ConvolveParams conv_params1 =
+ get_conv_params_no_round(0, 0, nullptr, 0, 0, 8);
+ DECLARE_ALIGNED(32, uint8_t, reference[MAX_SB_SQUARE]);
+ // Use a stride different from width to avoid potential storing errors that
+ // would go undetected. The input buffer is filled using a padding of 12, so
+ // the stride can be anywhere between width and width + 12.
+ av1_convolve_x_sr_intrabc_c(input, width + 2, reference, kOutputStride,
+ width, height, filter_params_x, kSubX,
+ &conv_params1);
+
+ ConvolveParams conv_params2 =
+ get_conv_params_no_round(0, 0, nullptr, 0, 0, 8);
+ convolve_x_func test_func = GetParam().TestFunction();
+ DECLARE_ALIGNED(32, uint8_t, test[MAX_SB_SQUARE]);
+ test_func(input, width + 2, test, kOutputStride, width, height,
+ filter_params_x, kSubX, &conv_params2);
+
+ AssertOutputBufferEq(reference, test, width, height);
+ }
+
+ void SpeedTest() {
+ constexpr int kNumIters = 10000;
+ const InterpFilter filter = static_cast<InterpFilter>(BILINEAR);
+ const int width = GetParam().Block().Width();
+ const int height = GetParam().Block().Height();
+ const InterpFilterParams *filter_params_x = &av1_intrabc_filter_params;
+ const uint8_t *input = FirstRandomInput8(GetParam());
+
+ ConvolveParams conv_params1 =
+ get_conv_params_no_round(0, 0, nullptr, 0, 0, 8);
+ DECLARE_ALIGNED(32, uint8_t, reference[MAX_SB_SQUARE]);
+ aom_usec_timer timer;
+ aom_usec_timer_start(&timer);
+ for (int i = 0; i < kNumIters; ++i) {
+ av1_convolve_x_sr_intrabc_c(input, width, reference, kOutputStride, width,
+ height, filter_params_x, 0, &conv_params1);
+ }
+ aom_usec_timer_mark(&timer);
+ const double time1 = static_cast<double>(aom_usec_timer_elapsed(&timer));
+
+ ConvolveParams conv_params2 =
+ get_conv_params_no_round(0, 0, nullptr, 0, 0, 8);
+ convolve_x_func test_func = GetParam().TestFunction();
+ DECLARE_ALIGNED(32, uint8_t, test[MAX_SB_SQUARE]);
+ aom_usec_timer_start(&timer);
+ for (int i = 0; i < kNumIters; ++i) {
+ test_func(input, width, test, kOutputStride, width, height,
+ filter_params_x, 0, &conv_params2);
+ }
+ aom_usec_timer_mark(&timer);
+ const double time2 = static_cast<double>(aom_usec_timer_elapsed(&timer));
+
+ printf("%d %3dx%-3d:%7.2f/%7.2fns (%3.2f)\n", filter, width, height, time1,
+ time2, time1 / time2);
+ }
+};
+
+TEST_P(AV1ConvolveXIntraBCTest, RunTest) { RunTest(); }
+
+TEST_P(AV1ConvolveXIntraBCTest, DISABLED_SpeedTest) { SpeedTest(); }
+
+INSTANTIATE_TEST_SUITE_P(C, AV1ConvolveXIntraBCTest,
+ BuildLowbdParams(av1_convolve_x_sr_intrabc_c));
+
+#if HAVE_NEON
+INSTANTIATE_TEST_SUITE_P(NEON, AV1ConvolveXIntraBCTest,
+ BuildLowbdParams(av1_convolve_x_sr_intrabc_neon));
+#endif
+
#if CONFIG_AV1_HIGHBITDEPTH
/////////////////////////////////////////////////////////
// Single reference convolve-x functions (high bit-depth)
@@ -540,6 +631,94 @@
BuildHighbdParams(av1_highbd_convolve_x_sr_neon));
#endif
+/////////////////////////////////////////////////////////////////
+// Single reference convolve-x IntraBC functions (high bit-depth)
+/////////////////////////////////////////////////////////////////
+
+class AV1ConvolveXHighbdIntraBCTest
+ : public AV1ConvolveTest<highbd_convolve_x_func> {
+ public:
+ void RunTest() {
+ // IntraBC functions only operate for subpel_x_qn = 8.
+ constexpr int kSubX = 8;
+ const int width = GetParam().Block().Width();
+ const int height = GetParam().Block().Height();
+ const int bit_depth = GetParam().BitDepth();
+ const InterpFilterParams *filter_params_x = &av1_intrabc_filter_params;
+ const uint16_t *input = FirstRandomInput16(GetParam());
+
+ ConvolveParams conv_params1 =
+ get_conv_params_no_round(0, 0, nullptr, 0, 0, bit_depth);
+ DECLARE_ALIGNED(32, uint16_t, reference[MAX_SB_SQUARE]);
+ // Use a stride different from width to avoid potential storing errors that
+ // would go undetected. The input buffer is filled using a padding of 12, so
+ // the stride can be anywhere between width and width + 12.
+ av1_highbd_convolve_x_sr_intrabc_c(
+ input, width + 2, reference, kOutputStride, width, height,
+ filter_params_x, kSubX, &conv_params1, bit_depth);
+
+ ConvolveParams conv_params2 =
+ get_conv_params_no_round(0, 0, nullptr, 0, 0, bit_depth);
+ DECLARE_ALIGNED(32, uint16_t, test[MAX_SB_SQUARE]);
+ GetParam().TestFunction()(input, width + 2, test, kOutputStride, width,
+ height, filter_params_x, kSubX, &conv_params2,
+ bit_depth);
+
+ AssertOutputBufferEq(reference, test, width, height);
+ }
+
+ void SpeedTest() {
+ constexpr int kNumIters = 10000;
+ const InterpFilter filter = static_cast<InterpFilter>(BILINEAR);
+ const int width = GetParam().Block().Width();
+ const int height = GetParam().Block().Height();
+ const int bit_depth = GetParam().BitDepth();
+ const InterpFilterParams *filter_params_x = &av1_intrabc_filter_params;
+ const uint16_t *input = FirstRandomInput16(GetParam());
+
+ ConvolveParams conv_params1 =
+ get_conv_params_no_round(0, 0, nullptr, 0, 0, 8);
+ DECLARE_ALIGNED(32, uint16_t, reference[MAX_SB_SQUARE]);
+ aom_usec_timer timer;
+ aom_usec_timer_start(&timer);
+ for (int i = 0; i < kNumIters; ++i) {
+ av1_highbd_convolve_x_sr_intrabc_c(input, width, reference, kOutputStride,
+ width, height, filter_params_x, 0,
+ &conv_params1, bit_depth);
+ }
+ aom_usec_timer_mark(&timer);
+ const double time1 = static_cast<double>(aom_usec_timer_elapsed(&timer));
+
+ ConvolveParams conv_params2 =
+ get_conv_params_no_round(0, 0, nullptr, 0, 0, 8);
+ highbd_convolve_x_func test_func = GetParam().TestFunction();
+ DECLARE_ALIGNED(32, uint16_t, test[MAX_SB_SQUARE]);
+ aom_usec_timer_start(&timer);
+ for (int i = 0; i < kNumIters; ++i) {
+ test_func(input, width, test, kOutputStride, width, height,
+ filter_params_x, 0, &conv_params2, bit_depth);
+ }
+ aom_usec_timer_mark(&timer);
+ const double time2 = static_cast<double>(aom_usec_timer_elapsed(&timer));
+
+ printf("%d %3dx%-3d:%7.2f/%7.2fns (%3.2f)\n", filter, width, height, time1,
+ time2, time1 / time2);
+ }
+};
+
+TEST_P(AV1ConvolveXHighbdIntraBCTest, RunTest) { RunTest(); }
+
+TEST_P(AV1ConvolveXHighbdIntraBCTest, DISABLED_SpeedTest) { SpeedTest(); }
+
+INSTANTIATE_TEST_SUITE_P(C, AV1ConvolveXHighbdIntraBCTest,
+ BuildHighbdParams(av1_highbd_convolve_x_sr_intrabc_c));
+
+#if HAVE_NEON
+INSTANTIATE_TEST_SUITE_P(
+ NEON, AV1ConvolveXHighbdIntraBCTest,
+ BuildHighbdParams(av1_highbd_convolve_x_sr_intrabc_neon));
+#endif
+
#endif // CONFIG_AV1_HIGHBITDEPTH
////////////////////////////////////////////////////////
@@ -643,6 +822,80 @@
BuildLowbdParams(av1_convolve_y_sr_neon));
#endif
+////////////////////////////////////////////////////////////////
+// Single reference convolve-y IntraBC functions (low bit-depth)
+////////////////////////////////////////////////////////////////
+
+class AV1ConvolveYIntraBCTest : public AV1ConvolveTest<convolve_y_func> {
+ public:
+ void RunTest() {
+ // IntraBC functions only operate for subpel_y_qn = 8.
+ constexpr int kSubY = 8;
+ const int width = GetParam().Block().Width();
+ const int height = GetParam().Block().Height();
+ const InterpFilterParams *filter_params_y = &av1_intrabc_filter_params;
+ const uint8_t *input = FirstRandomInput8(GetParam());
+
+ DECLARE_ALIGNED(32, uint8_t, reference[MAX_SB_SQUARE]);
+ // Use a stride different from width to avoid potential storing errors that
+ // would go undetected. The input buffer is filled using a padding of 12, so
+ // the stride can be anywhere between width and width + 12.
+ av1_convolve_y_sr_intrabc_c(input, width + 2, reference, kOutputStride,
+ width, height, filter_params_y, kSubY);
+
+ DECLARE_ALIGNED(32, uint8_t, test[MAX_SB_SQUARE]);
+ GetParam().TestFunction()(input, width + 2, test, kOutputStride, width,
+ height, filter_params_y, kSubY);
+
+ AssertOutputBufferEq(reference, test, width, height);
+ }
+
+ void SpeedTest() {
+ constexpr int kNumIters = 10000;
+ const InterpFilter filter = static_cast<InterpFilter>(BILINEAR);
+ const int width = GetParam().Block().Width();
+ const int height = GetParam().Block().Height();
+
+ const InterpFilterParams *filter_params_y = &av1_intrabc_filter_params;
+ const uint8_t *input = FirstRandomInput8(GetParam());
+ DECLARE_ALIGNED(32, uint8_t, reference[MAX_SB_SQUARE]);
+
+ aom_usec_timer timer;
+ aom_usec_timer_start(&timer);
+ for (int i = 0; i < kNumIters; ++i) {
+ av1_convolve_y_sr_intrabc_c(input, width, reference, kOutputStride, width,
+ height, filter_params_y, 0);
+ }
+ aom_usec_timer_mark(&timer);
+ const double time1 = static_cast<double>(aom_usec_timer_elapsed(&timer));
+
+ DECLARE_ALIGNED(32, uint8_t, test[MAX_SB_SQUARE]);
+ convolve_y_func test_func = GetParam().TestFunction();
+ aom_usec_timer_start(&timer);
+ for (int i = 0; i < kNumIters; ++i) {
+ test_func(input, width, test, kOutputStride, width, height,
+ filter_params_y, 0);
+ }
+ aom_usec_timer_mark(&timer);
+ const double time2 = static_cast<double>(aom_usec_timer_elapsed(&timer));
+
+ printf("%d %3dx%-3d:%7.2f/%7.2fns (%3.2f)\n", filter, width, height, time1,
+ time2, time1 / time2);
+ }
+};
+
+TEST_P(AV1ConvolveYIntraBCTest, RunTest) { RunTest(); }
+
+TEST_P(AV1ConvolveYIntraBCTest, DISABLED_SpeedTest) { SpeedTest(); }
+
+INSTANTIATE_TEST_SUITE_P(C, AV1ConvolveYIntraBCTest,
+ BuildLowbdParams(av1_convolve_y_sr_intrabc_c));
+
+#if HAVE_NEON
+INSTANTIATE_TEST_SUITE_P(NEON, AV1ConvolveYIntraBCTest,
+ BuildLowbdParams(av1_convolve_y_sr_intrabc_neon));
+#endif
+
#if CONFIG_AV1_HIGHBITDEPTH
/////////////////////////////////////////////////////////
// Single reference convolve-y functions (high bit-depth)
@@ -745,6 +998,86 @@
BuildHighbdParams(av1_highbd_convolve_y_sr_neon));
#endif
+/////////////////////////////////////////////////////////////////
+// Single reference convolve-y IntraBC functions (high bit-depth)
+/////////////////////////////////////////////////////////////////
+
+class AV1ConvolveYHighbdIntraBCTest
+ : public AV1ConvolveTest<highbd_convolve_y_func> {
+ public:
+ void RunTest() {
+ // IntraBC functions only operate for subpel_y_qn = 8.
+ constexpr int kSubY = 8;
+ const int width = GetParam().Block().Width();
+ const int height = GetParam().Block().Height();
+ const int bit_depth = GetParam().BitDepth();
+ const InterpFilterParams *filter_params_y = &av1_intrabc_filter_params;
+ const uint16_t *input = FirstRandomInput16(GetParam());
+
+ DECLARE_ALIGNED(32, uint16_t, reference[MAX_SB_SQUARE]);
+ // Use a stride different from width to avoid potential storing errors that
+ // would go undetected. The input buffer is filled using a padding of 12, so
+ // the stride can be anywhere between width and width + 12.
+ av1_highbd_convolve_y_sr_intrabc_c(input, width + 2, reference,
+ kOutputStride, width, height,
+ filter_params_y, kSubY, bit_depth);
+
+ DECLARE_ALIGNED(32, uint16_t, test[MAX_SB_SQUARE]);
+ GetParam().TestFunction()(input, width + 2, test, kOutputStride, width,
+ height, filter_params_y, kSubY, bit_depth);
+
+ AssertOutputBufferEq(reference, test, width, height);
+ }
+
+ void SpeedTest() {
+ constexpr int kNumIters = 10000;
+ const InterpFilter filter = static_cast<InterpFilter>(BILINEAR);
+ const int width = GetParam().Block().Width();
+ const int height = GetParam().Block().Height();
+ const int bit_depth = GetParam().BitDepth();
+ const InterpFilterParams *filter_params_y =
+ av1_get_interp_filter_params_with_block_size(filter, width);
+ const uint16_t *input = FirstRandomInput16(GetParam());
+
+ DECLARE_ALIGNED(32, uint16_t, reference[MAX_SB_SQUARE]);
+ aom_usec_timer timer;
+ aom_usec_timer_start(&timer);
+ for (int i = 0; i < kNumIters; ++i) {
+ av1_highbd_convolve_y_sr_intrabc_c(input, width, reference, kOutputStride,
+ width, height, filter_params_y, 0,
+ bit_depth);
+ }
+ aom_usec_timer_mark(&timer);
+ const double time1 = static_cast<double>(aom_usec_timer_elapsed(&timer));
+
+ highbd_convolve_y_func test_func = GetParam().TestFunction();
+ DECLARE_ALIGNED(32, uint16_t, test[MAX_SB_SQUARE]);
+ aom_usec_timer_start(&timer);
+ for (int i = 0; i < kNumIters; ++i) {
+ test_func(input, width, test, kOutputStride, width, height,
+ filter_params_y, 0, bit_depth);
+ }
+ aom_usec_timer_mark(&timer);
+ const double time2 = static_cast<double>(aom_usec_timer_elapsed(&timer));
+
+ printf("%d %3dx%-3d:%7.2f/%7.2fns (%3.2f)\n", filter, width, height, time1,
+ time2, time1 / time2);
+ }
+};
+
+TEST_P(AV1ConvolveYHighbdIntraBCTest, RunTest) { RunTest(); }
+
+TEST_P(AV1ConvolveYHighbdIntraBCTest, DISABLED_SpeedTest) { SpeedTest(); }
+
+INSTANTIATE_TEST_SUITE_P(C, AV1ConvolveYHighbdIntraBCTest,
+ BuildHighbdParams(av1_highbd_convolve_y_sr_intrabc_c));
+
+#if HAVE_NEON
+INSTANTIATE_TEST_SUITE_P(
+ NEON, AV1ConvolveYHighbdIntraBCTest,
+ BuildHighbdParams(av1_highbd_convolve_y_sr_intrabc_neon));
+#endif
+
#endif // CONFIG_AV1_HIGHBITDEPTH
//////////////////////////////////////////////////////////////
@@ -830,6 +1163,11 @@
BuildHighbdParams(aom_highbd_convolve_copy_avx2));
#endif
+#if HAVE_NEON
+INSTANTIATE_TEST_SUITE_P(NEON, AV1ConvolveCopyHighbdTest,
+ BuildHighbdParams(aom_highbd_convolve_copy_neon));
+#endif
+
#endif // CONFIG_AV1_HIGHBITDEPTH
/////////////////////////////////////////////////////////
@@ -958,6 +1296,104 @@
BuildLowbdParams(av1_convolve_2d_sr_neon));
#endif
+#if HAVE_NEON_DOTPROD
+INSTANTIATE_TEST_SUITE_P(NEON_DOTPROD, AV1Convolve2DTest,
+ BuildLowbdParams(av1_convolve_2d_sr_neon_dotprod));
+#endif
+
+#if HAVE_NEON_I8MM
+INSTANTIATE_TEST_SUITE_P(NEON_I8MM, AV1Convolve2DTest,
+ BuildLowbdParams(av1_convolve_2d_sr_neon_i8mm));
+#endif
+
+/////////////////////////////////////////////////////////////////
+// Single reference convolve-2D IntraBC functions (low bit-depth)
+/////////////////////////////////////////////////////////////////
+
+class AV1Convolve2DIntraBCTest : public AV1ConvolveTest<convolve_2d_func> {
+ public:
+ void RunTest() {
+ // IntraBC functions only operate for subpel_x_qn = 8 and subpel_y_qn = 8.
+ constexpr int kSubX = 8;
+ constexpr int kSubY = 8;
+ const int width = GetParam().Block().Width();
+ const int height = GetParam().Block().Height();
+ const InterpFilterParams *filter_params_x = &av1_intrabc_filter_params;
+ const InterpFilterParams *filter_params_y = &av1_intrabc_filter_params;
+ const uint8_t *input = FirstRandomInput8(GetParam());
+
+ DECLARE_ALIGNED(32, uint8_t, reference[MAX_SB_SQUARE]);
+ ConvolveParams conv_params1 =
+ get_conv_params_no_round(0, 0, nullptr, 0, 0, 8);
+ // Use a stride different from width to avoid potential storing errors that
+ // would go undetected. The input buffer is filled using a padding of 12, so
+ // the stride can be anywhere between width and width + 12.
+ av1_convolve_2d_sr_intrabc_c(input, width + 2, reference, kOutputStride,
+ width, height, filter_params_x,
+ filter_params_y, kSubX, kSubY, &conv_params1);
+
+ DECLARE_ALIGNED(32, uint8_t, test[MAX_SB_SQUARE]);
+ ConvolveParams conv_params2 =
+ get_conv_params_no_round(0, 0, nullptr, 0, 0, 8);
+ GetParam().TestFunction()(input, width + 2, test, kOutputStride, width,
+ height, filter_params_x, filter_params_y, kSubX,
+ kSubY, &conv_params2);
+
+ AssertOutputBufferEq(reference, test, width, height);
+ }
+
+ void SpeedTest() {
+ constexpr int kNumIters = 10000;
+ const InterpFilter h_f = static_cast<InterpFilter>(BILINEAR);
+ const InterpFilter v_f = static_cast<InterpFilter>(BILINEAR);
+ const int width = GetParam().Block().Width();
+ const int height = GetParam().Block().Height();
+ const InterpFilterParams *filter_params_x = &av1_intrabc_filter_params;
+ const InterpFilterParams *filter_params_y = &av1_intrabc_filter_params;
+ const uint8_t *input = FirstRandomInput8(GetParam());
+
+ DECLARE_ALIGNED(32, uint8_t, reference[MAX_SB_SQUARE]);
+ ConvolveParams conv_params1 =
+ get_conv_params_no_round(0, 0, nullptr, 0, 0, 8);
+ aom_usec_timer timer;
+ aom_usec_timer_start(&timer);
+ for (int i = 0; i < kNumIters; ++i) {
+ av1_convolve_2d_sr_intrabc_c(input, width, reference, kOutputStride,
+ width, height, filter_params_x,
+ filter_params_y, 8, 8, &conv_params1);
+ }
+ aom_usec_timer_mark(&timer);
+ const double time1 = static_cast<double>(aom_usec_timer_elapsed(&timer));
+
+ convolve_2d_func test_func = GetParam().TestFunction();
+ DECLARE_ALIGNED(32, uint8_t, test[MAX_SB_SQUARE]);
+ ConvolveParams conv_params2 =
+ get_conv_params_no_round(0, 0, nullptr, 0, 0, 8);
+ aom_usec_timer_start(&timer);
+ for (int i = 0; i < kNumIters; ++i) {
+ test_func(input, width, test, kOutputStride, width, height,
+ filter_params_x, filter_params_y, 8, 8, &conv_params2);
+ }
+ aom_usec_timer_mark(&timer);
+ const double time2 = static_cast<double>(aom_usec_timer_elapsed(&timer));
+
+ printf("%d - %d %3dx%-3d:%7.2f/%7.2fns (%3.2f)\n", h_f, v_f, width, height,
+ time1, time2, time1 / time2);
+ }
+};
+
+TEST_P(AV1Convolve2DIntraBCTest, RunTest) { RunTest(); }
+
+TEST_P(AV1Convolve2DIntraBCTest, DISABLED_SpeedTest) { SpeedTest(); }
+
+INSTANTIATE_TEST_SUITE_P(C, AV1Convolve2DIntraBCTest,
+ BuildLowbdParams(av1_convolve_2d_sr_intrabc_c));
+
+#if HAVE_NEON
+INSTANTIATE_TEST_SUITE_P(NEON, AV1Convolve2DIntraBCTest,
+ BuildLowbdParams(av1_convolve_2d_sr_intrabc_neon));
+#endif
+
#if CONFIG_AV1_HIGHBITDEPTH
//////////////////////////////////////////////////////////
// Single reference convolve-2d functions (high bit-depth)
@@ -1087,6 +1523,103 @@
BuildHighbdParams(av1_highbd_convolve_2d_sr_neon));
#endif
+//////////////////////////////////////////////////////////////////
+// Single reference convolve-2d IntraBC functions (high bit-depth)
+//////////////////////////////////////////////////////////////////
+
+class AV1Convolve2DHighbdIntraBCTest
+ : public AV1ConvolveTest<highbd_convolve_2d_func> {
+ public:
+ void RunTest() {
+ // IntraBC functions only operate for subpel_x_qn = 8 and subpel_y_qn = 8.
+ constexpr int kSubX = 8;
+ constexpr int kSubY = 8;
+ const int width = GetParam().Block().Width();
+ const int height = GetParam().Block().Height();
+ const int bit_depth = GetParam().BitDepth();
+ const InterpFilterParams *filter_params_x = &av1_intrabc_filter_params;
+ const InterpFilterParams *filter_params_y = &av1_intrabc_filter_params;
+ const uint16_t *input = FirstRandomInput16(GetParam());
+
+ DECLARE_ALIGNED(32, uint16_t, reference[MAX_SB_SQUARE]);
+ ConvolveParams conv_params1 =
+ get_conv_params_no_round(0, 0, nullptr, 0, 0, bit_depth);
+ // Use a stride different from width to avoid potential storing errors that
+ // would go undetected. The input buffer is filled using a padding of 12, so
+ // the stride can be anywhere between width and width + 12.
+ av1_highbd_convolve_2d_sr_intrabc_c(input, width + 2, reference,
+ kOutputStride, width, height,
+ filter_params_x, filter_params_y, kSubX,
+ kSubY, &conv_params1, bit_depth);
+
+ DECLARE_ALIGNED(32, uint16_t, test[MAX_SB_SQUARE]);
+ ConvolveParams conv_params2 =
+ get_conv_params_no_round(0, 0, nullptr, 0, 0, bit_depth);
+ GetParam().TestFunction()(input, width + 2, test, kOutputStride, width,
+ height, filter_params_x, filter_params_y, kSubX,
+ kSubY, &conv_params2, bit_depth);
+
+ AssertOutputBufferEq(reference, test, width, height);
+ }
+
+ void SpeedTest() {
+ constexpr int kNumIters = 10000;
+ const InterpFilter h_f = static_cast<InterpFilter>(BILINEAR);
+ const InterpFilter v_f = static_cast<InterpFilter>(BILINEAR);
+ const int width = GetParam().Block().Width();
+ const int height = GetParam().Block().Height();
+ const int bit_depth = GetParam().BitDepth();
+ const InterpFilterParams *filter_params_x =
+ av1_get_interp_filter_params_with_block_size(h_f, width);
+ const InterpFilterParams *filter_params_y =
+ av1_get_interp_filter_params_with_block_size(v_f, height);
+ const uint16_t *input = FirstRandomInput16(GetParam());
+
+ DECLARE_ALIGNED(32, uint16_t, reference[MAX_SB_SQUARE]);
+ ConvolveParams conv_params1 =
+ get_conv_params_no_round(0, 0, nullptr, 0, 0, 8);
+ aom_usec_timer timer;
+ aom_usec_timer_start(&timer);
+ for (int i = 0; i < kNumIters; ++i) {
+ av1_highbd_convolve_2d_sr_intrabc_c(
+ input, width, reference, kOutputStride, width, height,
+ filter_params_x, filter_params_y, 0, 0, &conv_params1, bit_depth);
+ }
+ aom_usec_timer_mark(&timer);
+ const double time1 = static_cast<double>(aom_usec_timer_elapsed(&timer));
+
+ DECLARE_ALIGNED(32, uint16_t, test[MAX_SB_SQUARE]);
+ highbd_convolve_2d_func test_func = GetParam().TestFunction();
+ ConvolveParams conv_params2 =
+ get_conv_params_no_round(0, 0, nullptr, 0, 0, 8);
+ aom_usec_timer_start(&timer);
+ for (int i = 0; i < kNumIters; ++i) {
+ test_func(input, width, test, kOutputStride, width, height,
+ filter_params_x, filter_params_y, 0, 0, &conv_params2,
+ bit_depth);
+ }
+ aom_usec_timer_mark(&timer);
+ const double time2 = static_cast<double>(aom_usec_timer_elapsed(&timer));
+
+ printf("%d - %d %3dx%-3d:%7.2f/%7.2fns (%3.2f)\n", h_f, v_f, width, height,
+ time1, time2, time1 / time2);
+ }
+};
+
+TEST_P(AV1Convolve2DHighbdIntraBCTest, RunTest) { RunTest(); }
+
+TEST_P(AV1Convolve2DHighbdIntraBCTest, DISABLED_SpeedTest) { SpeedTest(); }
+
+INSTANTIATE_TEST_SUITE_P(
+ C, AV1Convolve2DHighbdIntraBCTest,
+ BuildHighbdParams(av1_highbd_convolve_2d_sr_intrabc_c));
+
+#if HAVE_NEON
+INSTANTIATE_TEST_SUITE_P(
+ NEON, AV1Convolve2DHighbdIntraBCTest,
+ BuildHighbdParams(av1_highbd_convolve_2d_sr_intrabc_neon));
+#endif
+
#endif // CONFIG_AV1_HIGHBITDEPTH
//////////////////////////
@@ -1304,6 +1837,18 @@
BuildLowbdLumaParams(av1_dist_wtd_convolve_x_neon));
#endif
+#if HAVE_NEON_DOTPROD
+INSTANTIATE_TEST_SUITE_P(
+ NEON_DOTPROD, AV1ConvolveXCompoundTest,
+ BuildLowbdLumaParams(av1_dist_wtd_convolve_x_neon_dotprod));
+#endif
+
+#if HAVE_NEON_I8MM
+INSTANTIATE_TEST_SUITE_P(
+ NEON_I8MM, AV1ConvolveXCompoundTest,
+ BuildLowbdLumaParams(av1_dist_wtd_convolve_x_neon_i8mm));
+#endif
+
#if CONFIG_AV1_HIGHBITDEPTH
/////////////////////////////////////////////////
// Compound convolve-x functions (high bit-depth)
@@ -1787,6 +2332,18 @@
BuildLowbdLumaParams(av1_dist_wtd_convolve_2d_neon));
#endif
+#if HAVE_NEON_DOTPROD
+INSTANTIATE_TEST_SUITE_P(
+ NEON_DOTPROD, AV1Convolve2DCompoundTest,
+ BuildLowbdLumaParams(av1_dist_wtd_convolve_2d_neon_dotprod));
+#endif
+
+#if HAVE_NEON_I8MM
+INSTANTIATE_TEST_SUITE_P(
+ NEON_I8MM, AV1Convolve2DCompoundTest,
+ BuildLowbdLumaParams(av1_dist_wtd_convolve_2d_neon_i8mm));
+#endif
+
#if CONFIG_AV1_HIGHBITDEPTH
//////////////////////////////////////////////////
// Compound convolve-2d functions (high bit-depth)
diff --git a/test/av1_encoder_parms_get_to_decoder.cc b/test/av1_encoder_parms_get_to_decoder.cc
index e81ad87..402e70c 100644
--- a/test/av1_encoder_parms_get_to_decoder.cc
+++ b/test/av1_encoder_parms_get_to_decoder.cc
@@ -85,17 +85,17 @@
AVxEncoderParmsGetToDecoder()
: EncoderTest(GET_PARAM(0)), encode_parms(GET_PARAM(1)) {}
- virtual ~AVxEncoderParmsGetToDecoder() {}
+ ~AVxEncoderParmsGetToDecoder() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(::libaom_test::kTwoPassGood);
cfg_.g_lag_in_frames = 25;
test_video_ = kAV1ParamPassingTestVector;
cfg_.rc_target_bitrate = test_video_.bitrate;
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AOME_SET_CPUUSED, 3);
encoder->Control(AV1E_SET_COLOR_PRIMARIES, encode_parms.color_primaries);
@@ -113,8 +113,8 @@
}
}
- virtual void DecompressedFrameHook(const aom_image_t &img,
- aom_codec_pts_t pts) {
+ void DecompressedFrameHook(const aom_image_t &img,
+ aom_codec_pts_t pts) override {
(void)pts;
if (encode_parms.render_size[0] > 0 && encode_parms.render_size[1] > 0) {
EXPECT_EQ(encode_parms.render_size[0], (int)img.r_w);
@@ -127,14 +127,14 @@
EXPECT_EQ(encode_parms.chroma_sample_position, img.csp);
}
- virtual void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) {
+ void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) override {
if (encode_parms.lossless) {
EXPECT_EQ(kMaxPsnr, pkt->data.psnr.psnr[0]);
}
}
- virtual bool HandleDecodeResult(const aom_codec_err_t res_dec,
- libaom_test::Decoder *decoder) {
+ bool HandleDecodeResult(const aom_codec_err_t res_dec,
+ libaom_test::Decoder *decoder) override {
EXPECT_EQ(AOM_CODEC_OK, res_dec) << decoder->DecodeError();
return AOM_CODEC_OK == res_dec;
}
diff --git a/test/av1_ext_tile_test.cc b/test/av1_ext_tile_test.cc
index 5eaf382..59c44ca 100644
--- a/test/av1_ext_tile_test.cc
+++ b/test/av1_ext_tile_test.cc
@@ -58,12 +58,12 @@
tile_md5_.clear();
}
- virtual ~AV1ExtTileTest() {
+ ~AV1ExtTileTest() override {
aom_img_free(&tile_img_);
delete decoder_;
}
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(encoding_mode_);
cfg_.g_lag_in_frames = 0;
@@ -74,8 +74,8 @@
cfg_.rc_min_quantizer = 0;
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
// Encode setting
encoder->Control(AOME_SET_CPUUSED, set_cpu_used_);
@@ -96,8 +96,8 @@
}
}
- virtual void DecompressedFrameHook(const aom_image_t &img,
- aom_codec_pts_t pts) {
+ void DecompressedFrameHook(const aom_image_t &img,
+ aom_codec_pts_t pts) override {
// Skip 1 already decoded frame to be consistent with the decoder in this
// test.
if (pts == (aom_codec_pts_t)kSkip) return;
@@ -108,7 +108,7 @@
md5_.push_back(md5_res.Get());
}
- virtual void FramePktHook(const aom_codec_cx_pkt_t *pkt) {
+ void FramePktHook(const aom_codec_cx_pkt_t *pkt) override {
// Skip decoding 1 frame.
if (pkt->data.frame.pts == (aom_codec_pts_t)kSkip) return;
diff --git a/test/av1_external_partition_test.cc b/test/av1_external_partition_test.cc
index 41fc96c..88f6216 100644
--- a/test/av1_external_partition_test.cc
+++ b/test/av1_external_partition_test.cc
@@ -247,9 +247,9 @@
ExternalPartitionTestAPI()
: EncoderTest(GET_PARAM(0)), encoding_mode_(GET_PARAM(1)),
cpu_used_(GET_PARAM(2)), psnr_(0.0), nframes_(0) {}
- virtual ~ExternalPartitionTestAPI() {}
+ ~ExternalPartitionTestAPI() override {}
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(encoding_mode_);
const aom_rational timebase = { 1, 30 };
cfg_.g_timebase = timebase;
@@ -260,14 +260,14 @@
init_flags_ = AOM_CODEC_USE_PSNR;
}
- virtual bool DoDecode() const { return false; }
+ bool DoDecode() const override { return false; }
- virtual void BeginPassHook(unsigned int) {
+ void BeginPassHook(unsigned int) override {
psnr_ = 0.0;
nframes_ = 0;
}
- virtual void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) {
+ void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) override {
psnr_ += pkt->data.psnr.psnr[0];
nframes_++;
}
@@ -287,8 +287,8 @@
decision_mode_ = mode;
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
if (decision_mode_ == AOM_EXT_PART_WHOLE_TREE) {
aom_ext_part_funcs_t ext_part_funcs;
@@ -559,9 +559,9 @@
ExternalPartitionTestDfsAPI()
: EncoderTest(GET_PARAM(0)), encoding_mode_(GET_PARAM(1)),
cpu_used_(GET_PARAM(2)), psnr_(0.0), nframes_(0) {}
- virtual ~ExternalPartitionTestDfsAPI() {}
+ ~ExternalPartitionTestDfsAPI() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(encoding_mode_);
const aom_rational timebase = { 1, 30 };
cfg_.g_timebase = timebase;
@@ -572,14 +572,14 @@
init_flags_ = AOM_CODEC_USE_PSNR;
}
- virtual bool DoDecode() const { return false; }
+ bool DoDecode() const override { return false; }
- virtual void BeginPassHook(unsigned int) {
+ void BeginPassHook(unsigned int) override {
psnr_ = 0.0;
nframes_ = 0;
}
- virtual void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) {
+ void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) override {
psnr_ += pkt->data.psnr.psnr[0];
nframes_++;
}
@@ -597,8 +597,8 @@
test_send_features_ = test_send_features;
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
aom_ext_part_funcs_t ext_part_funcs;
ext_part_funcs.priv = reinterpret_cast<void *>(&test_data_);
diff --git a/test/av1_fwd_txfm1d_test.cc b/test/av1_fwd_txfm1d_test.cc
index 885a6db..6bae9f8 100644
--- a/test/av1_fwd_txfm1d_test.cc
+++ b/test/av1_fwd_txfm1d_test.cc
@@ -41,7 +41,7 @@
};
// the maximum stage number of fwd/inv 1d dct/adst txfm is 12
-const int8_t cos_bit = 14;
+const int8_t cos_bit = 13;
const int8_t range_bit[12] = { 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20 };
TEST(av1_fwd_txfm1d, round_shift) {
@@ -56,7 +56,7 @@
}
TEST(av1_fwd_txfm1d, av1_cospi_arr_data) {
- for (int i = 0; i < 7; i++) {
+ for (int i = 0; i < 4; i++) {
for (int j = 0; j < 64; j++) {
EXPECT_EQ(av1_cospi_arr_data[i][j],
(int32_t)round(cos(PI * j / 128) * (1 << (cos_bit_min + i))));
diff --git a/test/av1_fwd_txfm2d_test.cc b/test/av1_fwd_txfm2d_test.cc
index 7b84eb9..2ed5d94 100644
--- a/test/av1_fwd_txfm2d_test.cc
+++ b/test/av1_fwd_txfm2d_test.cc
@@ -38,7 +38,7 @@
class AV1FwdTxfm2d : public ::testing::TestWithParam<AV1FwdTxfm2dParam> {
public:
- virtual void SetUp() {
+ void SetUp() override {
tx_type_ = GET_PARAM(0);
tx_size_ = GET_PARAM(1);
max_error_ = GET_PARAM(2);
@@ -116,7 +116,7 @@
<< "tx_size = " << tx_size_ << ", tx_type = " << tx_type_;
}
- virtual void TearDown() {
+ void TearDown() override {
aom_free(input_);
aom_free(output_);
aom_free(ref_input_);
diff --git a/test/av1_highbd_iht_test.cc b/test/av1_highbd_iht_test.cc
index dae53ea..2c57362 100644
--- a/test/av1_highbd_iht_test.cc
+++ b/test/av1_highbd_iht_test.cc
@@ -63,9 +63,9 @@
class AV1HighbdInvHTNxN : public ::testing::TestWithParam<IHbdHtParam> {
public:
- virtual ~AV1HighbdInvHTNxN() {}
+ ~AV1HighbdInvHTNxN() override = default;
- virtual void SetUp() {
+ void SetUp() override {
txfm_ref_ = GET_PARAM(0);
inv_txfm_ = GET_PARAM(1);
inv_txfm_ref_ = GET_PARAM(2);
@@ -92,7 +92,7 @@
ASSERT_NE(output_ref_, nullptr);
}
- virtual void TearDown() {
+ void TearDown() override {
aom_free(input_);
aom_free(coeffs_);
aom_free(output_);
@@ -200,7 +200,7 @@
class AV1HighbdInvTxfm2d
: public ::testing::TestWithParam<AV1HighbdInvTxfm2dParam> {
public:
- virtual void SetUp() { target_func_ = GET_PARAM(0); }
+ void SetUp() override { target_func_ = GET_PARAM(0); }
void RunAV1InvTxfm2dTest(TX_TYPE tx_type, TX_SIZE tx_size, int run_times,
int bit_depth, int gt_int16 = 0);
diff --git a/test/av1_horz_only_frame_superres_test.cc b/test/av1_horz_only_frame_superres_test.cc
index 28ee534..e9cf02e 100644
--- a/test/av1_horz_only_frame_superres_test.cc
+++ b/test/av1_horz_only_frame_superres_test.cc
@@ -162,14 +162,13 @@
class ConvolveHorizRSTestBase : public ::testing::Test {
public:
ConvolveHorizRSTestBase() : image_(nullptr) {}
- virtual ~ConvolveHorizRSTestBase() {}
- virtual void TearDown() {}
+ ~ConvolveHorizRSTestBase() override = default;
// Implemented by subclasses (SetUp depends on the parameters passed
// in and RunOne depends on the function to be tested. These can't
// be templated for low/high bit depths because they have different
// numbers of parameters)
- virtual void SetUp() = 0;
+ void SetUp() override = 0;
virtual void RunOne(bool ref) = 0;
protected:
@@ -261,15 +260,15 @@
: public ConvolveHorizRSTestBase<uint8_t>,
public ::testing::WithParamInterface<LowBDParams> {
public:
- virtual ~LowBDConvolveHorizRSTest() {}
+ ~LowBDConvolveHorizRSTest() override = default;
- void SetUp() {
+ void SetUp() override {
tst_fun_ = GET_PARAM(0);
const int bd = 8;
SetBitDepth(bd);
}
- void RunOne(bool ref) {
+ void RunOne(bool ref) override {
const uint8_t *src = image_->GetSrcData(ref, false);
uint8_t *dst = image_->GetDstData(ref, false);
const int src_stride = image_->src_stride();
@@ -322,15 +321,15 @@
: public ConvolveHorizRSTestBase<uint16_t>,
public ::testing::WithParamInterface<HighBDParams> {
public:
- virtual ~HighBDConvolveHorizRSTest() {}
+ ~HighBDConvolveHorizRSTest() override = default;
- void SetUp() {
+ void SetUp() override {
tst_fun_ = GET_PARAM(0);
const int bd = GET_PARAM(1);
SetBitDepth(bd);
}
- void RunOne(bool ref) {
+ void RunOne(bool ref) override {
const uint16_t *src = image_->GetSrcData(ref, false);
uint16_t *dst = image_->GetDstData(ref, false);
const int src_stride = image_->src_stride();
diff --git a/test/av1_inv_txfm2d_test.cc b/test/av1_inv_txfm2d_test.cc
index dfa0481..35a87a4 100644
--- a/test/av1_inv_txfm2d_test.cc
+++ b/test/av1_inv_txfm2d_test.cc
@@ -49,7 +49,7 @@
class AV1InvTxfm2d : public ::testing::TestWithParam<AV1InvTxfm2dParam> {
public:
- virtual void SetUp() {
+ void SetUp() override {
tx_type_ = GET_PARAM(0);
tx_size_ = GET_PARAM(1);
max_error_ = GET_PARAM(2);
@@ -249,7 +249,7 @@
typedef std::tuple<const LbdInvTxfm2dFunc> AV1LbdInvTxfm2dParam;
class AV1LbdInvTxfm2d : public ::testing::TestWithParam<AV1LbdInvTxfm2dParam> {
public:
- virtual void SetUp() { target_func_ = GET_PARAM(0); }
+ void SetUp() override { target_func_ = GET_PARAM(0); }
void RunAV1InvTxfm2dTest(TxType tx_type, TxSize tx_size, int run_times,
int gt_int16 = 0);
@@ -393,8 +393,6 @@
::testing::Values(av1_lowbd_inv_txfm2d_add_avx2));
#endif // HAVE_AVX2
-// TODO(yunqing): Re-enable this unit test for NEON version after the functions
-// are fixed.
#if HAVE_NEON
extern "C" void av1_lowbd_inv_txfm2d_add_neon(const int32_t *input,
uint8_t *output, int stride,
diff --git a/test/av1_k_means_test.cc b/test/av1_k_means_test.cc
index 99f0fba..7e66a8e 100644
--- a/test/av1_k_means_test.cc
+++ b/test/av1_k_means_test.cc
@@ -46,10 +46,8 @@
class AV1KmeansTest1
: public ::testing::TestWithParam<av1_calc_indices_dim1Param> {
public:
- ~AV1KmeansTest1();
- void SetUp();
-
- void TearDown();
+ ~AV1KmeansTest1() override;
+ void SetUp() override;
protected:
void RunCheckOutput(av1_calc_indices_dim1_func test_impl, BLOCK_SIZE bsize,
@@ -75,7 +73,7 @@
};
GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(AV1KmeansTest1);
-AV1KmeansTest1::~AV1KmeansTest1() {}
+AV1KmeansTest1::~AV1KmeansTest1() = default;
void AV1KmeansTest1::SetUp() {
rnd_.Reset(libaom_test::ACMRandom::DeterministicSeed());
@@ -87,8 +85,6 @@
}
}
-void AV1KmeansTest1::TearDown() {}
-
void AV1KmeansTest1::RunCheckOutput(av1_calc_indices_dim1_func test_impl,
BLOCK_SIZE bsize, int k) {
const int w = block_size_wide[bsize];
@@ -152,10 +148,8 @@
class AV1KmeansTest2
: public ::testing::TestWithParam<av1_calc_indices_dim2Param> {
public:
- ~AV1KmeansTest2();
- void SetUp();
-
- void TearDown();
+ ~AV1KmeansTest2() override;
+ void SetUp() override;
protected:
void RunCheckOutput(av1_calc_indices_dim2_func test_impl, BLOCK_SIZE bsize,
@@ -185,7 +179,7 @@
};
GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(AV1KmeansTest2);
-AV1KmeansTest2::~AV1KmeansTest2() {}
+AV1KmeansTest2::~AV1KmeansTest2() = default;
void AV1KmeansTest2::SetUp() {
rnd_.Reset(libaom_test::ACMRandom::DeterministicSeed());
@@ -197,8 +191,6 @@
}
}
-void AV1KmeansTest2::TearDown() {}
-
void AV1KmeansTest2::RunCheckOutput(av1_calc_indices_dim2_func test_impl,
BLOCK_SIZE bsize, int k) {
const int w = block_size_wide[bsize];
diff --git a/test/av1_nn_predict_test.cc b/test/av1_nn_predict_test.cc
index 48504c8..4201ea6 100644
--- a/test/av1_nn_predict_test.cc
+++ b/test/av1_nn_predict_test.cc
@@ -34,7 +34,7 @@
class NnPredictTest : public ::testing::TestWithParam<NnPredictTestParam> {
public:
- virtual void SetUp() {
+ void SetUp() override {
const int MAX_NODES2 = NN_MAX_NODES_PER_LAYER * NN_MAX_NODES_PER_LAYER;
// Allocate two massive buffers on the heap for edge weights and node bias
// Then set-up the double-dimension arrays pointing into the big buffers
@@ -51,7 +51,7 @@
}
target_func_ = GET_PARAM(0);
}
- virtual void TearDown() {
+ void TearDown() override {
aom_free(weights_buf);
aom_free(bias_buf);
}
@@ -65,8 +65,8 @@
private:
NnPredict_Func target_func_;
libaom_test::ACMRandom rng_;
- float *weights[NN_MAX_HIDDEN_LAYERS + 1] = { 0 };
- float *bias[NN_MAX_HIDDEN_LAYERS + 1] = { 0 };
+ float *weights[NN_MAX_HIDDEN_LAYERS + 1] = {};
+ float *bias[NN_MAX_HIDDEN_LAYERS + 1] = {};
float *weights_buf = nullptr, *bias_buf = nullptr;
};
GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(NnPredictTest);
@@ -176,13 +176,15 @@
// runs of the encoder. It also conveniently covers all the kernels
// implemented.
static const NN_CONFIG kShapes[] = {
- { 10, 16, 1, { 64 }, { 0 }, { 0 } }, { 12, 1, 1, { 12 }, { 0 }, { 0 } },
- { 12, 1, 1, { 24 }, { 0 }, { 0 } }, { 12, 1, 1, { 32 }, { 0 }, { 0 } },
- { 18, 4, 1, { 24 }, { 0 }, { 0 } }, { 18, 4, 1, { 32 }, { 0 }, { 0 } },
- { 4, 1, 1, { 16 }, { 0 }, { 0 } }, { 8, 1, 1, { 16 }, { 0 }, { 0 } },
- { 8, 4, 1, { 16 }, { 0 }, { 0 } }, { 8, 1, 1, { 24 }, { 0 }, { 0 } },
- { 8, 1, 1, { 32 }, { 0 }, { 0 } }, { 8, 1, 1, { 64 }, { 0 }, { 0 } },
- { 9, 3, 1, { 32 }, { 0 }, { 0 } }, { 4, 4, 1, { 8 }, { 0 }, { 0 } },
+ { 37, 1, 2, { 16, 24 }, {}, {} }, { 24, 24, 1, { 12 }, {}, {} },
+ { 10, 16, 1, { 64 }, {}, {} }, { 12, 1, 1, { 12 }, {}, {} },
+ { 12, 1, 1, { 24 }, {}, {} }, { 12, 1, 1, { 32 }, {}, {} },
+ { 18, 4, 1, { 24 }, {}, {} }, { 18, 4, 1, { 32 }, {}, {} },
+ { 4, 1, 1, { 16 }, {}, {} }, { 8, 1, 0, { 0 }, {}, {} },
+ { 8, 4, 1, { 16 }, {}, {} }, { 8, 1, 1, { 32 }, {}, {} },
+ { 9, 3, 1, { 32 }, {}, {} }, { 8, 4, 0, { 0 }, {}, {} },
+ { 8, 8, 0, { 0 }, {}, {} }, { 4, 4, 1, { 8 }, {}, {} },
+ { 4, 3, 0, { 64 }, {}, {} },
};
void NnPredictTest::RunNnPredictTest_all(const NN_CONFIG *const shapes,
@@ -206,14 +208,21 @@
10000000);
}
-#if HAVE_SSE3 && !CONFIG_EXCLUDE_SIMD_MISMATCH
+#if !CONFIG_EXCLUDE_SIMD_MISMATCH
+#if HAVE_SSE3
INSTANTIATE_TEST_SUITE_P(SSE3, NnPredictTest,
::testing::Values(av1_nn_predict_sse3));
#endif
+#if HAVE_AVX2
+INSTANTIATE_TEST_SUITE_P(AVX2, NnPredictTest,
+ ::testing::Values(av1_nn_predict_avx2));
+#endif
+
#if HAVE_NEON
INSTANTIATE_TEST_SUITE_P(NEON, NnPredictTest,
::testing::Values(av1_nn_predict_neon));
#endif
+#endif // !CONFIG_EXCLUDE_SIMD_MISMATCH
} // namespace
diff --git a/test/av1_quantize_test.cc b/test/av1_quantize_test.cc
index 5823647..c8af14a 100644
--- a/test/av1_quantize_test.cc
+++ b/test/av1_quantize_test.cc
@@ -183,11 +183,9 @@
}
}
- virtual void SetUp() { params_ = GetParam(); }
+ void SetUp() override { params_ = GetParam(); }
- virtual void TearDown() {}
-
- virtual ~AV1QuantizeTest() {}
+ ~AV1QuantizeTest() override = default;
private:
TX_SIZE getTxSize(int count) {
diff --git a/test/av1_round_shift_array_test.cc b/test/av1_round_shift_array_test.cc
index facb84b..937e864 100644
--- a/test/av1_round_shift_array_test.cc
+++ b/test/av1_round_shift_array_test.cc
@@ -39,10 +39,11 @@
class AV1CompRoundShiftTest
: public ::testing::TestWithParam<CompRoundShiftParam> {
public:
- ~AV1CompRoundShiftTest();
+ ~AV1CompRoundShiftTest() override;
- void SetUp() { rnd_.Reset(libaom_test::ACMRandom::DeterministicSeed()); }
- void TearDown() {}
+ void SetUp() override {
+ rnd_.Reset(libaom_test::ACMRandom::DeterministicSeed());
+ }
protected:
void RunCheckOutput(comp_round_shift_array_func test_impl, BLOCK_SIZE bsize,
@@ -54,7 +55,7 @@
};
GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(AV1CompRoundShiftTest);
-AV1CompRoundShiftTest::~AV1CompRoundShiftTest() {}
+AV1CompRoundShiftTest::~AV1CompRoundShiftTest() = default;
void AV1CompRoundShiftTest::RunCheckOutput(
comp_round_shift_array_func test_impl, BLOCK_SIZE bsize, int bit) {
diff --git a/test/av1_softmax_test.cc b/test/av1_softmax_test.cc
index 60c7b6f..2b04af1 100644
--- a/test/av1_softmax_test.cc
+++ b/test/av1_softmax_test.cc
@@ -35,7 +35,7 @@
class FastSoftmaxTest : public ::testing::TestWithParam<FastSoftmaxTestParams> {
public:
FastSoftmaxTest() : target_fn_(GET_PARAM(0)), num_classes_(GET_PARAM(1)) {}
- virtual void SetUp() {
+ void SetUp() override {
ref_buf_.reset(new (std::nothrow) float[num_classes_]());
ASSERT_NE(ref_buf_, nullptr);
dst_buf_.reset(new (std::nothrow) float[num_classes_]());
diff --git a/test/av1_temporal_denoiser_test.cc b/test/av1_temporal_denoiser_test.cc
index 571fd92..7aa8fb6 100644
--- a/test/av1_temporal_denoiser_test.cc
+++ b/test/av1_temporal_denoiser_test.cc
@@ -43,11 +43,9 @@
: public ::testing::Test,
public ::testing::WithParamInterface<AV1DenoiserTestParam> {
public:
- virtual ~AV1DenoiserTest() {}
+ ~AV1DenoiserTest() override = default;
- virtual void SetUp() { bs_ = GET_PARAM(1); }
-
- virtual void TearDown() {}
+ void SetUp() override { bs_ = GET_PARAM(1); }
protected:
BLOCK_SIZE bs_;
diff --git a/test/av1_wedge_utils_test.cc b/test/av1_wedge_utils_test.cc
index 46f6d92..1055ff3 100644
--- a/test/av1_wedge_utils_test.cc
+++ b/test/av1_wedge_utils_test.cc
@@ -379,6 +379,16 @@
NEON, WedgeUtilsSSEOptTest,
::testing::Values(TestFuncsFSSE(av1_wedge_sse_from_residuals_c,
av1_wedge_sse_from_residuals_neon)));
+
+INSTANTIATE_TEST_SUITE_P(
+ NEON, WedgeUtilsSignOptTest,
+ ::testing::Values(TestFuncsFSign(av1_wedge_sign_from_residuals_c,
+ av1_wedge_sign_from_residuals_neon)));
+
+INSTANTIATE_TEST_SUITE_P(
+ NEON, WedgeUtilsDeltaSquaresOptTest,
+ ::testing::Values(TestFuncsFDS(av1_wedge_compute_delta_squares_c,
+ av1_wedge_compute_delta_squares_neon)));
#endif // HAVE_NEON
#if HAVE_AVX2
diff --git a/test/avg_test.cc b/test/avg_test.cc
index 8865915..d7817a8 100644
--- a/test/avg_test.cc
+++ b/test/avg_test.cc
@@ -35,7 +35,7 @@
: width_(width), height_(height), source_data_(nullptr),
source_stride_(0), bit_depth_(bit_depth) {}
- virtual void TearDown() {
+ void TearDown() override {
aom_free(source_data_);
source_data_ = nullptr;
}
@@ -47,7 +47,7 @@
static const int kDataBlockHeight = 128;
static const int kDataBlockSize = kDataBlockWidth * kDataBlockHeight;
- virtual void SetUp() {
+ void SetUp() override {
const testing::TestInfo *const test_info =
testing::UnitTest::GetInstance()->current_test_info();
// Skip the speed test for C code as the baseline uses the same function.
@@ -378,7 +378,7 @@
}
protected:
- virtual void SetUp() {
+ void SetUp() override {
source_data_ = static_cast<uint8_t *>(
aom_memalign(kDataAlignment, kDataBlockSize * sizeof(source_data_[0])));
ASSERT_NE(source_data_, nullptr);
@@ -391,7 +391,7 @@
ASSERT_NE(hbuf_c_, nullptr);
}
- virtual void TearDown() {
+ void TearDown() override {
aom_free(source_data_);
source_data_ = nullptr;
aom_free(hbuf_c_);
@@ -469,7 +469,7 @@
}
protected:
- virtual void SetUp() {
+ void SetUp() override {
source_data_ = static_cast<uint8_t *>(
aom_memalign(kDataAlignment, kDataBlockSize * sizeof(source_data_[0])));
ASSERT_NE(source_data_, nullptr);
@@ -482,7 +482,7 @@
ASSERT_NE(vbuf_c_, nullptr);
}
- virtual void TearDown() {
+ void TearDown() override {
aom_free(source_data_);
source_data_ = nullptr;
aom_free(vbuf_c_);
@@ -582,13 +582,13 @@
class VectorVarTestBase : public ::testing::Test {
public:
explicit VectorVarTestBase(int bwl) { m_bwl = bwl; }
- VectorVarTestBase() {}
- ~VectorVarTestBase() {}
+ VectorVarTestBase() = default;
+ ~VectorVarTestBase() override = default;
protected:
static const int kDataAlignment = 16;
- virtual void SetUp() {
+ void SetUp() override {
width = 4 << m_bwl;
ref_vector = static_cast<int16_t *>(
@@ -600,7 +600,7 @@
rnd_.Reset(ACMRandom::DeterministicSeed());
}
- virtual void TearDown() {
+ void TearDown() override {
aom_free(ref_vector);
ref_vector = nullptr;
aom_free(src_vector);
@@ -883,13 +883,13 @@
satd_func_ref_ = func_param.func_ref;
satd_func_simd_ = func_param.func_simd;
}
- virtual void SetUp() {
+ void SetUp() override {
rnd_.Reset(ACMRandom::DeterministicSeed());
src_ = reinterpret_cast<CoeffType *>(
aom_memalign(32, sizeof(*src_) * satd_size_));
ASSERT_NE(src_, nullptr);
}
- virtual void TearDown() { aom_free(src_); }
+ void TearDown() override { aom_free(src_); }
void FillConstant(const CoeffType val) {
for (int i = 0; i < satd_size_; ++i) src_[i] = val;
}
@@ -963,13 +963,13 @@
};
TEST_P(SatdTest, MinValue) {
- const int kMin = -32640;
+ const int kMin = -524287;
const int expected = -kMin * satd_size_;
FillConstant(kMin);
Check(expected);
}
TEST_P(SatdTest, MaxValue) {
- const int kMax = 32640;
+ const int kMax = 524287;
const int expected = kMax * satd_size_;
FillConstant(kMax);
Check(expected);
diff --git a/test/blend_a64_mask_1d_test.cc b/test/blend_a64_mask_1d_test.cc
index 9a95987..f9549bc 100644
--- a/test/blend_a64_mask_1d_test.cc
+++ b/test/blend_a64_mask_1d_test.cc
@@ -41,13 +41,13 @@
static const int kMaxMaskWidth = 2 * MAX_SB_SIZE;
static const int kMaxMaskSize = kMaxMaskWidth;
- virtual ~BlendA64Mask1DTest() {}
+ ~BlendA64Mask1DTest() override = default;
virtual void Execute(const T *p_src0, const T *p_src1) = 0;
- void Common() {
- w_ = 2 << this->rng_(MAX_SB_SIZE_LOG2);
- h_ = 2 << this->rng_(MAX_SB_SIZE_LOG2);
+ void Common(int block_size) {
+ w_ = block_size_wide[block_size];
+ h_ = block_size_high[block_size];
dst_offset_ = this->rng_(33);
dst_stride_ = this->rng_(kMaxWidth + 1 - w_) + w_;
@@ -121,7 +121,7 @@
class BlendA64Mask1DTest8B : public BlendA64Mask1DTest<F8B, uint8_t> {
protected:
- void Execute(const uint8_t *p_src0, const uint8_t *p_src1) {
+ void Execute(const uint8_t *p_src0, const uint8_t *p_src1) override {
params_.ref_func(dst_ref_ + dst_offset_, dst_stride_, p_src0 + src0_offset_,
src0_stride_, p_src1 + src1_offset_, src1_stride_, mask_,
w_, h_);
@@ -132,7 +132,7 @@
};
TEST_P(BlendA64Mask1DTest8B, RandomValues) {
- for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) {
+ for (int bsize = 0; bsize < BLOCK_SIZES_ALL; ++bsize) {
for (int i = 0; i < kBufSize; ++i) {
dst_ref_[i] = rng_.Rand8();
dst_tst_[i] = rng_.Rand8();
@@ -144,23 +144,23 @@
for (int i = 0; i < kMaxMaskSize; ++i)
mask_[i] = rng_(AOM_BLEND_A64_MAX_ALPHA + 1);
- Common();
+ Common(bsize);
}
}
TEST_P(BlendA64Mask1DTest8B, ExtremeValues) {
- for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) {
- for (int i = 0; i < kBufSize; ++i) {
- dst_ref_[i] = rng_(2) + 254;
- dst_tst_[i] = rng_(2) + 254;
- src0_[i] = rng_(2) + 254;
- src1_[i] = rng_(2) + 254;
- }
+ for (int i = 0; i < kBufSize; ++i) {
+ dst_ref_[i] = rng_(2) + 254;
+ dst_tst_[i] = rng_(2) + 254;
+ src0_[i] = rng_(2) + 254;
+ src1_[i] = rng_(2) + 254;
+ }
- for (int i = 0; i < kMaxMaskSize; ++i)
- mask_[i] = rng_(2) + AOM_BLEND_A64_MAX_ALPHA - 1;
+ for (int i = 0; i < kMaxMaskSize; ++i)
+ mask_[i] = rng_(2) + AOM_BLEND_A64_MAX_ALPHA - 1;
- Common();
+ for (int bsize = 0; bsize < BLOCK_SIZES_ALL; ++bsize) {
+ Common(bsize);
}
}
@@ -227,7 +227,7 @@
class BlendA64Mask1DTestHBD : public BlendA64Mask1DTest<FHBD, uint16_t> {
protected:
- void Execute(const uint16_t *p_src0, const uint16_t *p_src1) {
+ void Execute(const uint16_t *p_src0, const uint16_t *p_src1) override {
params_.ref_func(CONVERT_TO_BYTEPTR(dst_ref_ + dst_offset_), dst_stride_,
CONVERT_TO_BYTEPTR(p_src0 + src0_offset_), src0_stride_,
CONVERT_TO_BYTEPTR(p_src1 + src1_offset_), src1_stride_,
@@ -243,37 +243,27 @@
};
TEST_P(BlendA64Mask1DTestHBD, RandomValues) {
- for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) {
- switch (rng_(3)) {
- case 0: bit_depth_ = 8; break;
- case 1: bit_depth_ = 10; break;
- default: bit_depth_ = 12; break;
+ for (bit_depth_ = 8; bit_depth_ <= 12; bit_depth_ += 2) {
+ for (int bsize = 0; bsize < BLOCK_SIZES_ALL; ++bsize) {
+ const int hi = 1 << bit_depth_;
+
+ for (int i = 0; i < kBufSize; ++i) {
+ dst_ref_[i] = rng_(hi);
+ dst_tst_[i] = rng_(hi);
+ src0_[i] = rng_(hi);
+ src1_[i] = rng_(hi);
+ }
+
+ for (int i = 0; i < kMaxMaskSize; ++i)
+ mask_[i] = rng_(AOM_BLEND_A64_MAX_ALPHA + 1);
+
+ Common(bsize);
}
-
- const int hi = 1 << bit_depth_;
-
- for (int i = 0; i < kBufSize; ++i) {
- dst_ref_[i] = rng_(hi);
- dst_tst_[i] = rng_(hi);
- src0_[i] = rng_(hi);
- src1_[i] = rng_(hi);
- }
-
- for (int i = 0; i < kMaxMaskSize; ++i)
- mask_[i] = rng_(AOM_BLEND_A64_MAX_ALPHA + 1);
-
- Common();
}
}
TEST_P(BlendA64Mask1DTestHBD, ExtremeValues) {
- for (int iter = 0; iter < 1000 && !HasFatalFailure(); ++iter) {
- switch (rng_(3)) {
- case 0: bit_depth_ = 8; break;
- case 1: bit_depth_ = 10; break;
- default: bit_depth_ = 12; break;
- }
-
+ for (bit_depth_ = 8; bit_depth_ <= 12; bit_depth_ += 2) {
const int hi = 1 << bit_depth_;
const int lo = hi - 2;
@@ -287,7 +277,9 @@
for (int i = 0; i < kMaxMaskSize; ++i)
mask_[i] = rng_(2) + AOM_BLEND_A64_MAX_ALPHA - 1;
- Common();
+ for (int bsize = 0; bsize < BLOCK_SIZES_ALL; ++bsize) {
+ Common(bsize);
+ }
}
}
@@ -336,5 +328,15 @@
TestFuncsHBD(highbd_blend_a64_vmask_ref,
aom_highbd_blend_a64_vmask_sse4_1)));
#endif // HAVE_SSE4_1
+
+#if HAVE_NEON
+INSTANTIATE_TEST_SUITE_P(
+ NEON, BlendA64Mask1DTestHBD,
+ ::testing::Values(TestFuncsHBD(highbd_blend_a64_hmask_ref,
+ aom_highbd_blend_a64_hmask_neon),
+ TestFuncsHBD(highbd_blend_a64_vmask_ref,
+ aom_highbd_blend_a64_vmask_neon)));
+#endif // HAVE_NEON
+
#endif // CONFIG_AV1_HIGHBITDEPTH
} // namespace
diff --git a/test/blend_a64_mask_test.cc b/test/blend_a64_mask_test.cc
index 9dece57..fafc7f0 100644
--- a/test/blend_a64_mask_test.cc
+++ b/test/blend_a64_mask_test.cc
@@ -41,7 +41,7 @@
static const int kMaxMaskWidth = 2 * MAX_SB_SIZE;
static const int kMaxMaskSize = kMaxMaskWidth * kMaxMaskWidth;
- virtual ~BlendA64MaskTest() {}
+ ~BlendA64MaskTest() override = default;
virtual void Execute(const SrcPixel *p_src0, const SrcPixel *p_src1,
int run_times) = 0;
@@ -123,9 +123,11 @@
}
void RunTest(int block_size, int run_times) {
- subx_ = Rand1();
- suby_ = Rand1();
- RunOneTest(block_size, subx_, suby_, run_times);
+ for (subx_ = 0; subx_ <= 1; subx_++) {
+ for (suby_ = 0; suby_ <= 1; suby_++) {
+ RunOneTest(block_size, subx_, suby_, run_times);
+ }
+ }
}
DstPixel dst_ref_[kBufSize];
@@ -163,7 +165,8 @@
class BlendA64MaskTest8B : public BlendA64MaskTest<F8B, uint8_t, uint8_t> {
protected:
- void Execute(const uint8_t *p_src0, const uint8_t *p_src1, int run_times) {
+ void Execute(const uint8_t *p_src0, const uint8_t *p_src1,
+ int run_times) override {
aom_usec_timer timer;
aom_usec_timer_start(&timer);
for (int i = 0; i < run_times; ++i) {
@@ -193,8 +196,7 @@
GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(BlendA64MaskTest8B);
TEST_P(BlendA64MaskTest8B, RandomValues) {
- for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) {
- int bsize = rng_.Rand8() % BLOCK_SIZES_ALL;
+ for (int bsize = 0; bsize < BLOCK_SIZES_ALL && !HasFatalFailure(); ++bsize) {
for (int i = 0; i < kBufSize; ++i) {
dst_ref_[i] = rng_.Rand8();
dst_tst_[i] = rng_.Rand8();
@@ -211,21 +213,20 @@
}
TEST_P(BlendA64MaskTest8B, ExtremeValues) {
- for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) {
- int bsize = rng_.Rand8() % BLOCK_SIZES_ALL;
- for (int i = 0; i < kBufSize; ++i) {
- dst_ref_[i] = rng_(2) + 254;
- dst_tst_[i] = rng_(2) + 254;
- src0_[i] = rng_(2) + 254;
- src1_[i] = rng_(2) + 254;
- }
-
- for (int i = 0; i < kMaxMaskSize; ++i)
- mask_[i] = rng_(2) + AOM_BLEND_A64_MAX_ALPHA - 1;
-
- RunTest(bsize, 1);
+ for (int i = 0; i < kBufSize; ++i) {
+ dst_ref_[i] = rng_(2) + 254;
+ dst_tst_[i] = rng_(2) + 254;
+ src0_[i] = rng_(2) + 254;
+ src1_[i] = rng_(2) + 254;
}
+
+ for (int i = 0; i < kMaxMaskSize; ++i)
+ mask_[i] = rng_(2) + AOM_BLEND_A64_MAX_ALPHA - 1;
+
+ for (int bsize = 0; bsize < BLOCK_SIZES_ALL && !HasFatalFailure(); ++bsize)
+ RunTest(bsize, 1);
}
+
TEST_P(BlendA64MaskTest8B, DISABLED_Speed) {
const int kRunTimes = 10000000;
for (int bsize = 0; bsize < BLOCK_SIZES_ALL; ++bsize) {
@@ -240,10 +241,7 @@
for (int i = 0; i < kMaxMaskSize; ++i)
mask_[i] = rng_(AOM_BLEND_A64_MAX_ALPHA + 1);
- RunOneTest(bsize, 1, 1, kRunTimes);
- RunOneTest(bsize, 1, 0, kRunTimes);
- RunOneTest(bsize, 0, 1, kRunTimes);
- RunOneTest(bsize, 0, 0, kRunTimes);
+ RunTest(bsize, kRunTimes);
}
}
#if HAVE_SSE4_1
@@ -258,6 +256,12 @@
aom_blend_a64_mask_avx2)));
#endif // HAVE_AVX2
+#if HAVE_NEON
+INSTANTIATE_TEST_SUITE_P(NEON, BlendA64MaskTest8B,
+ ::testing::Values(TestFuncs(aom_blend_a64_mask_c,
+ aom_blend_a64_mask_neon)));
+#endif // HAVE_NEON
+
//////////////////////////////////////////////////////////////////////////////
// 8 bit _d16 version
//////////////////////////////////////////////////////////////////////////////
@@ -275,7 +279,8 @@
// max number of bits used by the source
static const int kSrcMaxBitsMask = 0x3fff;
- void Execute(const uint16_t *p_src0, const uint16_t *p_src1, int run_times) {
+ void Execute(const uint16_t *p_src0, const uint16_t *p_src1,
+ int run_times) override {
ConvolveParams conv_params;
conv_params.round_0 = ROUND0_BITS;
conv_params.round_1 = COMPOUND_ROUND1_BITS;
@@ -308,8 +313,7 @@
GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(BlendA64MaskTest8B_d16);
TEST_P(BlendA64MaskTest8B_d16, RandomValues) {
- for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) {
- int bsize = rng_.Rand8() % BLOCK_SIZES_ALL;
+ for (int bsize = 0; bsize < BLOCK_SIZES_ALL && !HasFatalFailure(); ++bsize) {
for (int i = 0; i < kBufSize; ++i) {
dst_ref_[i] = rng_.Rand8();
dst_tst_[i] = rng_.Rand8();
@@ -326,20 +330,35 @@
}
TEST_P(BlendA64MaskTest8B_d16, ExtremeValues) {
- for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) {
- int bsize = rng_.Rand8() % BLOCK_SIZES_ALL;
- for (int i = 0; i < kBufSize; ++i) {
- dst_ref_[i] = 255;
- dst_tst_[i] = 255;
+ for (int i = 0; i < kBufSize; ++i) {
+ dst_ref_[i] = 255;
+ dst_tst_[i] = 255;
- src0_[i] = kSrcMaxBitsMask;
- src1_[i] = kSrcMaxBitsMask;
+ src0_[i] = kSrcMaxBitsMask;
+ src1_[i] = kSrcMaxBitsMask;
+ }
+
+ for (int i = 0; i < kMaxMaskSize; ++i) mask_[i] = AOM_BLEND_A64_MAX_ALPHA - 1;
+
+ for (int bsize = 0; bsize < BLOCK_SIZES_ALL && !HasFatalFailure(); ++bsize)
+ RunTest(bsize, 1);
+}
+
+TEST_P(BlendA64MaskTest8B_d16, DISABLED_Speed) {
+ const int kRunTimes = 10000000;
+ for (int bsize = 0; bsize < BLOCK_SIZES_ALL; ++bsize) {
+ for (int i = 0; i < kBufSize; ++i) {
+ dst_ref_[i] = rng_.Rand8();
+ dst_tst_[i] = rng_.Rand8();
+
+ src0_[i] = rng_.Rand16() & kSrcMaxBitsMask;
+ src1_[i] = rng_.Rand16() & kSrcMaxBitsMask;
}
for (int i = 0; i < kMaxMaskSize; ++i)
- mask_[i] = AOM_BLEND_A64_MAX_ALPHA - 1;
+ mask_[i] = rng_(AOM_BLEND_A64_MAX_ALPHA + 1);
- RunTest(bsize, 1);
+ RunTest(bsize, kRunTimes);
}
}
@@ -377,7 +396,8 @@
class BlendA64MaskTestHBD : public BlendA64MaskTest<FHBD, uint16_t, uint16_t> {
protected:
- void Execute(const uint16_t *p_src0, const uint16_t *p_src1, int run_times) {
+ void Execute(const uint16_t *p_src0, const uint16_t *p_src1,
+ int run_times) override {
aom_usec_timer timer;
aom_usec_timer_start(&timer);
for (int i = 0; i < run_times; ++i) {
@@ -409,53 +429,46 @@
GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(BlendA64MaskTestHBD);
TEST_P(BlendA64MaskTestHBD, RandomValues) {
- for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) {
- int bsize = rng_.Rand8() % BLOCK_SIZES_ALL;
- switch (rng_(3)) {
- case 0: bit_depth_ = 8; break;
- case 1: bit_depth_ = 10; break;
- default: bit_depth_ = 12; break;
- }
-
+ for (bit_depth_ = 8; bit_depth_ <= 12 && !HasFatalFailure();
+ bit_depth_ += 2) {
const int hi = 1 << bit_depth_;
- for (int i = 0; i < kBufSize; ++i) {
- dst_ref_[i] = rng_(hi);
- dst_tst_[i] = rng_(hi);
- src0_[i] = rng_(hi);
- src1_[i] = rng_(hi);
+ for (int bsize = 0; bsize < BLOCK_SIZES_ALL; ++bsize) {
+ for (int i = 0; i < kBufSize; ++i) {
+ dst_ref_[i] = rng_(hi);
+ dst_tst_[i] = rng_(hi);
+ src0_[i] = rng_(hi);
+ src1_[i] = rng_(hi);
+ }
+
+ for (int i = 0; i < kMaxMaskSize; ++i)
+ mask_[i] = rng_(AOM_BLEND_A64_MAX_ALPHA + 1);
+
+ RunTest(bsize, 1);
}
-
- for (int i = 0; i < kMaxMaskSize; ++i)
- mask_[i] = rng_(AOM_BLEND_A64_MAX_ALPHA + 1);
-
- RunTest(bsize, 1);
}
}
TEST_P(BlendA64MaskTestHBD, ExtremeValues) {
- for (int iter = 0; iter < 1000 && !HasFatalFailure(); ++iter) {
- int bsize = rng_.Rand8() % BLOCK_SIZES_ALL;
- switch (rng_(3)) {
- case 0: bit_depth_ = 8; break;
- case 1: bit_depth_ = 10; break;
- default: bit_depth_ = 12; break;
- }
-
+ for (bit_depth_ = 8; bit_depth_ <= 12 && !HasFatalFailure();
+ bit_depth_ += 2) {
const int hi = 1 << bit_depth_;
const int lo = hi - 2;
- for (int i = 0; i < kBufSize; ++i) {
- dst_ref_[i] = rng_(hi - lo) + lo;
- dst_tst_[i] = rng_(hi - lo) + lo;
- src0_[i] = rng_(hi - lo) + lo;
- src1_[i] = rng_(hi - lo) + lo;
+ for (int bsize = 0; bsize < BLOCK_SIZES_ALL && !HasFatalFailure();
+ ++bsize) {
+ for (int i = 0; i < kBufSize; ++i) {
+ dst_ref_[i] = rng_(hi - lo) + lo;
+ dst_tst_[i] = rng_(hi - lo) + lo;
+ src0_[i] = rng_(hi - lo) + lo;
+ src1_[i] = rng_(hi - lo) + lo;
+ }
+
+ for (int i = 0; i < kMaxMaskSize; ++i)
+ mask_[i] = rng_(2) + AOM_BLEND_A64_MAX_ALPHA - 1;
+
+ RunTest(bsize, 1);
}
-
- for (int i = 0; i < kMaxMaskSize; ++i)
- mask_[i] = rng_(2) + AOM_BLEND_A64_MAX_ALPHA - 1;
-
- RunTest(bsize, 1);
}
}
@@ -466,6 +479,13 @@
aom_highbd_blend_a64_mask_sse4_1)));
#endif // HAVE_SSE4_1
+#if HAVE_NEON
+INSTANTIATE_TEST_SUITE_P(
+ NEON, BlendA64MaskTestHBD,
+ ::testing::Values(TestFuncsHBD(aom_highbd_blend_a64_mask_c,
+ aom_highbd_blend_a64_mask_neon)));
+#endif // HAVE_NEON
+
//////////////////////////////////////////////////////////////////////////////
// HBD _d16 version
//////////////////////////////////////////////////////////////////////////////
@@ -485,7 +505,8 @@
static const int kSrcMaxBitsMask = (1 << 14) - 1;
static const int kSrcMaxBitsMaskHBD = (1 << 16) - 1;
- void Execute(const uint16_t *p_src0, const uint16_t *p_src1, int run_times) {
+ void Execute(const uint16_t *p_src0, const uint16_t *p_src1,
+ int run_times) override {
ASSERT_GT(run_times, 0) << "Cannot run 0 iterations of the test.";
ConvolveParams conv_params;
conv_params.round_0 = (bit_depth_ == 12) ? ROUND0_BITS + 2 : ROUND0_BITS;
@@ -526,51 +547,49 @@
TEST_P(BlendA64MaskTestHBD_d16, RandomValues) {
if (params_.tst_func == nullptr) return;
- for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) {
- int bsize = rng_.Rand8() % BLOCK_SIZES_ALL;
- switch (rng_(3)) {
- case 0: bit_depth_ = 8; break;
- case 1: bit_depth_ = 10; break;
- default: bit_depth_ = 12; break;
- }
+ for (bit_depth_ = 8; bit_depth_ <= 12 && !HasFatalFailure();
+ bit_depth_ += 2) {
src_max_bits_mask_ =
(bit_depth_ == 8) ? kSrcMaxBitsMask : kSrcMaxBitsMaskHBD;
- for (int i = 0; i < kBufSize; ++i) {
- dst_ref_[i] = rng_.Rand8();
- dst_tst_[i] = rng_.Rand8();
-
- src0_[i] = rng_.Rand16() & src_max_bits_mask_;
- src1_[i] = rng_.Rand16() & src_max_bits_mask_;
- }
-
- for (int i = 0; i < kMaxMaskSize; ++i)
- mask_[i] = rng_(AOM_BLEND_A64_MAX_ALPHA + 1);
-
- RunTest(bsize, 1);
- }
-}
-// TODO (Scott LaVarnway), fix this test
-TEST_P(BlendA64MaskTestHBD_d16, DISABLED_SaturatedValues) {
- for (int bsize = 0; bsize < BLOCK_SIZES_ALL; ++bsize) {
- for (bit_depth_ = 8; bit_depth_ <= 12; bit_depth_ += 2) {
- src_max_bits_mask_ =
- (bit_depth_ == 8) ? kSrcMaxBitsMask : kSrcMaxBitsMaskHBD;
-
+ for (int bsize = 0; bsize < BLOCK_SIZES_ALL && !HasFatalFailure();
+ ++bsize) {
for (int i = 0; i < kBufSize; ++i) {
- dst_ref_[i] = 0;
- dst_tst_[i] = (1 << bit_depth_) - 1;
+ dst_ref_[i] = rng_.Rand8();
+ dst_tst_[i] = rng_.Rand8();
- src0_[i] = src_max_bits_mask_;
- src1_[i] = src_max_bits_mask_;
+ src0_[i] = rng_.Rand16() & src_max_bits_mask_;
+ src1_[i] = rng_.Rand16() & src_max_bits_mask_;
}
- for (int i = 0; i < kMaxMaskSize; ++i) mask_[i] = AOM_BLEND_A64_MAX_ALPHA;
+ for (int i = 0; i < kMaxMaskSize; ++i)
+ mask_[i] = rng_(AOM_BLEND_A64_MAX_ALPHA + 1);
RunTest(bsize, 1);
}
}
}
+
+TEST_P(BlendA64MaskTestHBD_d16, ExtremeValues) {
+ for (bit_depth_ = 8; bit_depth_ <= 12; bit_depth_ += 2) {
+ src_max_bits_mask_ =
+ (bit_depth_ == 8) ? kSrcMaxBitsMask : kSrcMaxBitsMaskHBD;
+
+ for (int i = 0; i < kBufSize; ++i) {
+ dst_ref_[i] = 0;
+ dst_tst_[i] = (1 << bit_depth_) - 1;
+
+ src0_[i] = src_max_bits_mask_;
+ src1_[i] = src_max_bits_mask_;
+ }
+
+ for (int i = 0; i < kMaxMaskSize; ++i) mask_[i] = AOM_BLEND_A64_MAX_ALPHA;
+ for (int bsize = 0; bsize < BLOCK_SIZES_ALL; ++bsize) {
+ RunTest(bsize, 1);
+ }
+ }
+}
+
TEST_P(BlendA64MaskTestHBD_d16, DISABLED_Speed) {
const int kRunTimes = 10000000;
for (int bsize = 0; bsize < BLOCK_SIZES_ALL; ++bsize) {
@@ -586,15 +605,15 @@
for (int i = 0; i < kMaxMaskSize; ++i)
mask_[i] = rng_(AOM_BLEND_A64_MAX_ALPHA + 1);
- RunOneTest(bsize, 1, 1, kRunTimes);
- RunOneTest(bsize, 0, 0, kRunTimes);
+ RunTest(bsize, kRunTimes);
}
}
}
-INSTANTIATE_TEST_SUITE_P(C, BlendA64MaskTestHBD_d16,
- ::testing::Values(TestFuncsHBD_d16(
- aom_highbd_blend_a64_d16_mask_c, nullptr)));
+INSTANTIATE_TEST_SUITE_P(
+ C, BlendA64MaskTestHBD_d16,
+ ::testing::Values(TestFuncsHBD_d16(aom_highbd_blend_a64_d16_mask_c,
+ aom_highbd_blend_a64_d16_mask_c)));
#if HAVE_SSE4_1
INSTANTIATE_TEST_SUITE_P(
@@ -610,6 +629,13 @@
aom_highbd_blend_a64_d16_mask_avx2)));
#endif // HAVE_AVX2
+#if HAVE_NEON
+INSTANTIATE_TEST_SUITE_P(
+ NEON, BlendA64MaskTestHBD_d16,
+ ::testing::Values(TestFuncsHBD_d16(aom_highbd_blend_a64_d16_mask_c,
+ aom_highbd_blend_a64_d16_mask_neon)));
+#endif // HAVE_NEON
+
// TODO(slavarnway): Enable the following in the avx2 commit. (56501)
#if 0
#if HAVE_AVX2
diff --git a/test/block_test.cc b/test/block_test.cc
index 74deee3..686180c 100644
--- a/test/block_test.cc
+++ b/test/block_test.cc
@@ -140,9 +140,9 @@
superblock_size_(GET_PARAM(2)), rc_end_usage_(GET_PARAM(3)) {
sb_size_violated_ = false;
}
- virtual ~SuperBlockSizeTestLarge() {}
+ ~SuperBlockSizeTestLarge() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(encoding_mode_);
const aom_rational timebase = { 1, 30 };
cfg_.g_timebase = timebase;
@@ -152,10 +152,10 @@
cfg_.rc_target_bitrate = 1000;
}
- virtual bool DoDecode() const { return 1; }
+ bool DoDecode() const override { return true; }
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AOME_SET_CPUUSED, 5);
encoder->Control(AOME_SET_ENABLEAUTOALTREF, 1);
@@ -163,8 +163,8 @@
}
}
- virtual bool HandleDecodeResult(const aom_codec_err_t res_dec,
- libaom_test::Decoder *decoder) {
+ bool HandleDecodeResult(const aom_codec_err_t res_dec,
+ libaom_test::Decoder *decoder) override {
EXPECT_EQ(AOM_CODEC_OK, res_dec) << decoder->DecodeError();
if (AOM_CODEC_OK == res_dec &&
superblock_size_ != AOM_SUPERBLOCK_SIZE_DYNAMIC) {
diff --git a/test/borders_test.cc b/test/borders_test.cc
index bf9cc8b..594c3e8 100644
--- a/test/borders_test.cc
+++ b/test/borders_test.cc
@@ -24,12 +24,12 @@
public ::libaom_test::EncoderTest {
protected:
BordersTestLarge() : EncoderTest(GET_PARAM(0)) {}
- virtual ~BordersTestLarge() {}
+ ~BordersTestLarge() override = default;
- virtual void SetUp() { InitializeConfig(GET_PARAM(1)); }
+ void SetUp() override { InitializeConfig(GET_PARAM(1)); }
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AOME_SET_CPUUSED, 1);
encoder->Control(AOME_SET_ENABLEAUTOALTREF, 1);
@@ -38,7 +38,7 @@
}
}
- virtual void FramePktHook(const aom_codec_cx_pkt_t *pkt) {
+ void FramePktHook(const aom_codec_cx_pkt_t *pkt) override {
if (pkt->data.frame.flags & AOM_FRAME_IS_KEY) {
}
}
diff --git a/test/cdef_test.cc b/test/cdef_test.cc
index 3f971be..ad54407 100644
--- a/test/cdef_test.cc
+++ b/test/cdef_test.cc
@@ -38,8 +38,8 @@
class CDEFBlockTest : public ::testing::TestWithParam<cdef_dir_param_t> {
public:
- virtual ~CDEFBlockTest() {}
- virtual void SetUp() {
+ ~CDEFBlockTest() override = default;
+ void SetUp() override {
cdef = GET_PARAM(0);
ref_cdef = GET_PARAM(1);
bsize = GET_PARAM(2);
@@ -47,10 +47,8 @@
depth = GET_PARAM(4);
}
- virtual void TearDown() {}
-
protected:
- int bsize;
+ BLOCK_SIZE bsize;
int boundary;
int depth;
CdefFilterBlockFunctions cdef;
@@ -67,7 +65,8 @@
typedef CDEFBlockTest CDEFSpeedHighbdTest;
GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(CDEFSpeedHighbdTest);
-int64_t test_cdef(int bsize, int iterations, CdefFilterBlockFunctions cdef,
+int64_t test_cdef(BLOCK_SIZE bsize, int iterations,
+ CdefFilterBlockFunctions cdef,
CdefFilterBlockFunctions ref_cdef, int boundary, int depth) {
aom_usec_timer ref_timer;
int64_t ref_elapsed_time = 0;
@@ -188,7 +187,8 @@
return ref_elapsed_time;
}
-void test_cdef_speed(int bsize, int iterations, CdefFilterBlockFunctions cdef,
+void test_cdef_speed(BLOCK_SIZE bsize, int iterations,
+ CdefFilterBlockFunctions cdef,
CdefFilterBlockFunctions ref_cdef, int boundary,
int depth) {
int64_t ref_elapsed_time =
@@ -213,14 +213,12 @@
class CDEFFindDirTest : public ::testing::TestWithParam<find_dir_param_t> {
public:
- virtual ~CDEFFindDirTest() {}
- virtual void SetUp() {
+ ~CDEFFindDirTest() override = default;
+ void SetUp() override {
finddir = GET_PARAM(0);
ref_finddir = GET_PARAM(1);
}
- virtual void TearDown() {}
-
protected:
find_dir_t finddir;
find_dir_t ref_finddir;
@@ -304,14 +302,12 @@
class CDEFFindDirDualTest
: public ::testing::TestWithParam<find_dir_dual_param_t> {
public:
- virtual ~CDEFFindDirDualTest() {}
- virtual void SetUp() {
+ ~CDEFFindDirDualTest() override = default;
+ void SetUp() override {
finddir = GET_PARAM(0);
ref_finddir = GET_PARAM(1);
}
- virtual void TearDown() {}
-
protected:
find_dir_dual_t finddir;
find_dir_dual_t ref_finddir;
@@ -405,6 +401,177 @@
ref_elapsed_time, elapsed_time, ref_elapsed_time / elapsed_time);
}
+#define MAX_CDEF_BLOCK 256
+
+constexpr int kIterations = 100;
+
+using CDEFCopyRect8To16 = void (*)(uint16_t *dst, int dstride,
+ const uint8_t *src, int sstride, int width,
+ int height);
+
+using CDEFCopyRect8To16Param = std::tuple<CDEFCopyRect8To16, CDEFCopyRect8To16>;
+
+class CDEFCopyRect8to16Test
+ : public ::testing::TestWithParam<CDEFCopyRect8To16Param> {
+ public:
+ CDEFCopyRect8to16Test()
+ : rnd_(libaom_test::ACMRandom::DeterministicSeed()),
+ test_func_(GET_PARAM(0)), ref_func_(GET_PARAM(1)) {}
+ ~CDEFCopyRect8to16Test() override = default;
+ void SetUp() override {
+ src_ = reinterpret_cast<uint8_t *>(
+ aom_memalign(8, sizeof(uint8_t) * MAX_CDEF_BLOCK * MAX_CDEF_BLOCK));
+ ASSERT_NE(src_, nullptr);
+ ref_dst_ = reinterpret_cast<uint16_t *>(
+ aom_memalign(16, sizeof(uint16_t) * MAX_CDEF_BLOCK * MAX_CDEF_BLOCK));
+ ASSERT_NE(ref_dst_, nullptr);
+ test_dst_ = reinterpret_cast<uint16_t *>(
+ aom_memalign(16, sizeof(uint16_t) * MAX_CDEF_BLOCK * MAX_CDEF_BLOCK));
+ ASSERT_NE(test_dst_, nullptr);
+ }
+
+ void TearDown() override {
+ aom_free(src_);
+ aom_free(ref_dst_);
+ aom_free(test_dst_);
+ }
+
+ void test_copy_rect_8_to_16(CDEFCopyRect8To16 test_func,
+ CDEFCopyRect8To16 ref_func) {
+ constexpr int stride = MAX_CDEF_BLOCK;
+ int error = 0;
+ for (int k = 0; k < kIterations && !error; k++) {
+ // This function operates on values of width that are either 4 or a
+ // multiple of 8. For height, generate a random value between 1 and 256,
+ // making sure it is even.
+ const int width = k == 0 ? 4 : (rnd_.Rand8() % 32 + 1) * 8;
+ const int height = k == 0 ? 4 : (rnd_.Rand8() % 128 + 1) * 2;
+ for (int i = 0; i < height; i++) {
+ for (int j = 0; j < width; j++) {
+ src_[i * stride + j] = rnd_.Rand8();
+ }
+ }
+
+ ref_func(ref_dst_, stride, src_, stride, width, height);
+ test_func(test_dst_, stride, src_, stride, width, height);
+
+ int i, j;
+ for (i = 0; i < height; i++) {
+ for (j = 0; j < width; j++) {
+ if (test_dst_[i * stride + j] != ref_dst_[i * stride + j]) {
+ error = 1;
+ break;
+ }
+ }
+ if (error) {
+ break;
+ }
+ }
+ EXPECT_EQ(0, error)
+ << "Error: CDEFCopyRect8to16Test, SIMD and C mismatch." << std::endl
+ << "First error at " << i << "," << j << " ("
+ << ref_dst_[i * stride + j] << " : " << test_dst_[i * stride + j]
+ << ") " << std::endl
+ << "width: " << width << std::endl
+ << "height: " << height << std::endl
+ << std::endl;
+ }
+ }
+
+ protected:
+ libaom_test::ACMRandom rnd_;
+ uint8_t *src_;
+ uint16_t *ref_dst_;
+ uint16_t *test_dst_;
+ CDEFCopyRect8To16 test_func_;
+ CDEFCopyRect8To16 ref_func_;
+};
+GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(CDEFCopyRect8to16Test);
+
+using CDEFCopyRect16To16 = void (*)(uint16_t *dst, int dstride,
+ const uint16_t *src, int sstride, int width,
+ int height);
+
+using CDEFCopyRect16To16Param =
+ std::tuple<CDEFCopyRect16To16, CDEFCopyRect16To16>;
+
+class CDEFCopyRect16to16Test
+ : public ::testing::TestWithParam<CDEFCopyRect16To16Param> {
+ public:
+ CDEFCopyRect16to16Test()
+ : rnd_(libaom_test::ACMRandom::DeterministicSeed()),
+ test_func_(GET_PARAM(0)), ref_func_(GET_PARAM(1)) {}
+ ~CDEFCopyRect16to16Test() override = default;
+ void SetUp() override {
+ src_ = reinterpret_cast<uint16_t *>(
+ aom_memalign(16, sizeof(uint16_t) * MAX_CDEF_BLOCK * MAX_CDEF_BLOCK));
+ ASSERT_NE(src_, nullptr);
+ ref_dst_ = reinterpret_cast<uint16_t *>(
+ aom_memalign(16, sizeof(uint16_t) * MAX_CDEF_BLOCK * MAX_CDEF_BLOCK));
+ ASSERT_NE(ref_dst_, nullptr);
+ test_dst_ = reinterpret_cast<uint16_t *>(
+ aom_memalign(16, sizeof(uint16_t) * MAX_CDEF_BLOCK * MAX_CDEF_BLOCK));
+ ASSERT_NE(test_dst_, nullptr);
+ }
+
+ void TearDown() override {
+ aom_free(src_);
+ aom_free(ref_dst_);
+ aom_free(test_dst_);
+ }
+
+ void test_copy_rect_16_to_16(CDEFCopyRect16To16 test_func,
+ CDEFCopyRect16To16 ref_func) {
+ constexpr int stride = MAX_CDEF_BLOCK;
+ int error = 0;
+ for (int k = 0; k < kIterations && !error; k++) {
+ // This function operates on values of width that are either 4 or a
+ // multiple of 8. For height, generate a random value between 1 and 256,
+ // making sure it is even.
+ const int width = k == 0 ? 4 : (rnd_.Rand8() % 32 + 1) * 8;
+ const int height = k == 0 ? 4 : (rnd_.Rand8() % 128 + 1) * 2;
+ for (int i = 0; i < height; i++) {
+ for (int j = 0; j < width; j++) {
+ src_[i * stride + j] = rnd_.Rand16();
+ }
+ }
+
+ ref_func(ref_dst_, stride, src_, stride, width, height);
+ test_func(test_dst_, stride, src_, stride, width, height);
+
+ int i, j;
+ for (i = 0; i < height; i++) {
+ for (j = 0; j < width; j++) {
+ if (test_dst_[i * stride + j] != ref_dst_[i * stride + j]) {
+ error = 1;
+ break;
+ }
+ }
+ if (error) {
+ break;
+ }
+ }
+ EXPECT_EQ(0, error)
+ << "Error: CDEFCopyRect16to16Test, SIMD and C mismatch." << std::endl
+ << "First error at " << i << "," << j << " ("
+ << ref_dst_[i * stride + j] << " : " << test_dst_[i * stride + j]
+ << ") " << std::endl
+ << "width: " << width << std::endl
+ << "height: " << height << std::endl
+ << std::endl;
+ }
+ }
+
+ protected:
+ libaom_test::ACMRandom rnd_;
+ uint16_t *src_;
+ uint16_t *ref_dst_;
+ uint16_t *test_dst_;
+ CDEFCopyRect16To16 test_func_;
+ CDEFCopyRect16To16 ref_func_;
+};
+GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(CDEFCopyRect16to16Test);
+
TEST_P(CDEFBlockTest, TestSIMDNoMismatch) {
test_cdef(bsize, 1, cdef, ref_cdef, boundary, depth);
}
@@ -437,6 +604,14 @@
test_finddir_dual_speed(finddir, ref_finddir);
}
+TEST_P(CDEFCopyRect8to16Test, TestSIMDNoMismatch) {
+ test_copy_rect_8_to_16(test_func_, ref_func_);
+}
+
+TEST_P(CDEFCopyRect16to16Test, TestSIMDNoMismatch) {
+ test_copy_rect_16_to_16(test_func_, ref_func_);
+}
+
using std::make_tuple;
#if (HAVE_SSE2 || HAVE_SSSE3 || HAVE_SSE4_1 || HAVE_AVX2 || HAVE_NEON)
@@ -482,6 +657,16 @@
INSTANTIATE_TEST_SUITE_P(SSE2, CDEFFindDirDualTest,
::testing::Values(make_tuple(&cdef_find_dir_dual_sse2,
&cdef_find_dir_dual_c)));
+
+INSTANTIATE_TEST_SUITE_P(
+ SSE2, CDEFCopyRect8to16Test,
+ ::testing::Values(make_tuple(&cdef_copy_rect8_8bit_to_16bit_c,
+ &cdef_copy_rect8_8bit_to_16bit_sse2)));
+
+INSTANTIATE_TEST_SUITE_P(
+ SSE2, CDEFCopyRect16to16Test,
+ ::testing::Values(make_tuple(&cdef_copy_rect8_16bit_to_16bit_c,
+ &cdef_copy_rect8_16bit_to_16bit_sse2)));
#endif
#if HAVE_SSSE3
@@ -515,6 +700,16 @@
INSTANTIATE_TEST_SUITE_P(SSSE3, CDEFFindDirDualTest,
::testing::Values(make_tuple(&cdef_find_dir_dual_ssse3,
&cdef_find_dir_dual_c)));
+
+INSTANTIATE_TEST_SUITE_P(
+ SSSE3, CDEFCopyRect8to16Test,
+ ::testing::Values(make_tuple(&cdef_copy_rect8_8bit_to_16bit_c,
+ &cdef_copy_rect8_8bit_to_16bit_ssse3)));
+
+INSTANTIATE_TEST_SUITE_P(
+ SSSE3, CDEFCopyRect16to16Test,
+ ::testing::Values(make_tuple(&cdef_copy_rect8_16bit_to_16bit_c,
+ &cdef_copy_rect8_16bit_to_16bit_ssse3)));
#endif
#if HAVE_SSE4_1
@@ -549,6 +744,16 @@
SSE4_1, CDEFFindDirDualTest,
::testing::Values(make_tuple(&cdef_find_dir_dual_sse4_1,
&cdef_find_dir_dual_c)));
+
+INSTANTIATE_TEST_SUITE_P(
+ SSE4_1, CDEFCopyRect8to16Test,
+ ::testing::Values(make_tuple(&cdef_copy_rect8_8bit_to_16bit_c,
+ &cdef_copy_rect8_8bit_to_16bit_sse4_1)));
+
+INSTANTIATE_TEST_SUITE_P(
+ SSE4_1, CDEFCopyRect16to16Test,
+ ::testing::Values(make_tuple(&cdef_copy_rect8_16bit_to_16bit_c,
+ &cdef_copy_rect8_16bit_to_16bit_sse4_1)));
#endif
#if HAVE_AVX2
@@ -582,6 +787,16 @@
INSTANTIATE_TEST_SUITE_P(AVX2, CDEFFindDirDualTest,
::testing::Values(make_tuple(&cdef_find_dir_dual_avx2,
&cdef_find_dir_dual_c)));
+
+INSTANTIATE_TEST_SUITE_P(
+ AVX2, CDEFCopyRect8to16Test,
+ ::testing::Values(make_tuple(&cdef_copy_rect8_8bit_to_16bit_c,
+ &cdef_copy_rect8_8bit_to_16bit_avx2)));
+
+INSTANTIATE_TEST_SUITE_P(
+ AVX2, CDEFCopyRect16to16Test,
+ ::testing::Values(make_tuple(&cdef_copy_rect8_16bit_to_16bit_c,
+ &cdef_copy_rect8_16bit_to_16bit_avx2)));
#endif
#if HAVE_NEON
@@ -615,6 +830,16 @@
INSTANTIATE_TEST_SUITE_P(NEON, CDEFFindDirDualTest,
::testing::Values(make_tuple(&cdef_find_dir_dual_neon,
&cdef_find_dir_dual_c)));
+
+INSTANTIATE_TEST_SUITE_P(
+ NEON, CDEFCopyRect8to16Test,
+ ::testing::Values(make_tuple(&cdef_copy_rect8_8bit_to_16bit_c,
+ &cdef_copy_rect8_8bit_to_16bit_neon)));
+
+INSTANTIATE_TEST_SUITE_P(
+ NEON, CDEFCopyRect16to16Test,
+ ::testing::Values(make_tuple(&cdef_copy_rect8_16bit_to_16bit_c,
+ &cdef_copy_rect8_16bit_to_16bit_neon)));
#endif
// Test speed for all supported architectures
diff --git a/test/cfl_test.cc b/test/cfl_test.cc
index 97533da..7fdea04 100644
--- a/test/cfl_test.cc
+++ b/test/cfl_test.cc
@@ -88,7 +88,7 @@
class CFLTest {
public:
- virtual ~CFLTest() {}
+ virtual ~CFLTest() = default;
void init(TX_SIZE tx) {
tx_size = tx;
width = tx_size_wide[tx_size];
@@ -106,7 +106,7 @@
template <typename I>
class CFLTestWithData : public CFLTest {
public:
- virtual ~CFLTestWithData() {}
+ ~CFLTestWithData() override = default;
protected:
I data[CFL_BUF_SQUARE];
@@ -125,7 +125,7 @@
template <typename I>
class CFLTestWithAlignedData : public CFLTest {
public:
- ~CFLTestWithAlignedData() {
+ ~CFLTestWithAlignedData() override {
aom_free(chroma_pels_ref);
aom_free(sub_luma_pels_ref);
aom_free(chroma_pels);
@@ -177,12 +177,12 @@
class CFLSubAvgTest : public ::testing::TestWithParam<sub_avg_param>,
public CFLTestWithData<int16_t> {
public:
- virtual void SetUp() {
+ void SetUp() override {
CFLTest::init(std::get<0>(this->GetParam()));
sub_avg = std::get<1>(this->GetParam())(tx_size);
sub_avg_ref = cfl_get_subtract_average_fn_c(tx_size);
}
- virtual ~CFLSubAvgTest() {}
+ ~CFLSubAvgTest() override = default;
protected:
cfl_subtract_average_fn sub_avg;
@@ -223,7 +223,7 @@
class CFLSubsampleTest : public ::testing::TestWithParam<S>,
public CFLTestWithData<I> {
public:
- virtual void SetUp() {
+ void SetUp() override {
CFLTest::init(std::get<0>(this->GetParam()));
fun_420 = std::get<1>(this->GetParam())(this->tx_size);
fun_422 = std::get<2>(this->GetParam())(this->tx_size);
@@ -284,8 +284,8 @@
: public CFLSubsampleTest<subsample_lbd_param, cfl_subsample_lbd_fn,
uint8_t> {
public:
- virtual ~CFLSubsampleLBDTest() {}
- virtual void SetUp() {
+ ~CFLSubsampleLBDTest() override = default;
+ void SetUp() override {
CFLSubsampleTest::SetUp();
fun_420_ref = cfl_get_luma_subsampling_420_lbd_c(tx_size);
fun_422_ref = cfl_get_luma_subsampling_422_lbd_c(tx_size);
@@ -328,8 +328,8 @@
: public CFLSubsampleTest<subsample_hbd_param, cfl_subsample_hbd_fn,
uint16_t> {
public:
- virtual ~CFLSubsampleHBDTest() {}
- virtual void SetUp() {
+ ~CFLSubsampleHBDTest() override = default;
+ void SetUp() override {
CFLSubsampleTest::SetUp();
fun_420_ref = cfl_get_luma_subsampling_420_hbd_c(tx_size);
fun_422_ref = cfl_get_luma_subsampling_422_hbd_c(tx_size);
@@ -369,13 +369,13 @@
class CFLPredictTest : public ::testing::TestWithParam<predict_param>,
public CFLTestWithAlignedData<uint8_t> {
public:
- virtual void SetUp() {
+ void SetUp() override {
CFLTest::init(std::get<0>(this->GetParam()));
CFLTestWithAlignedData::init();
predict = std::get<1>(this->GetParam())(tx_size);
predict_ref = cfl_get_predict_lbd_fn_c(tx_size);
}
- virtual ~CFLPredictTest() {}
+ ~CFLPredictTest() override = default;
protected:
cfl_predict_lbd_fn predict;
@@ -418,13 +418,13 @@
class CFLPredictHBDTest : public ::testing::TestWithParam<predict_param_hbd>,
public CFLTestWithAlignedData<uint16_t> {
public:
- virtual void SetUp() {
+ void SetUp() override {
CFLTest::init(std::get<0>(this->GetParam()));
CFLTestWithAlignedData::init();
predict = std::get<1>(this->GetParam())(tx_size);
predict_ref = cfl_get_predict_hbd_fn_c(tx_size);
}
- virtual ~CFLPredictHBDTest() {}
+ ~CFLPredictHBDTest() override = default;
protected:
cfl_predict_hbd_fn predict;
diff --git a/test/cnn_test.cc b/test/cnn_test.cc
index 77d8d55..127ed3d 100644
--- a/test/cnn_test.cc
+++ b/test/cnn_test.cc
@@ -2520,7 +2520,7 @@
class CNNConvolveTest : public ::testing::TestWithParam<CNNConvolveTestFuncs> {
protected:
- virtual void SetUp() { params_ = GetParam(); }
+ void SetUp() override { params_ = GetParam(); }
void RunCNNConvolveSetup(int run_times) {
int in_width = 65;
diff --git a/test/codec_factory.h b/test/codec_factory.h
index d768d2e..7ffc465 100644
--- a/test/codec_factory.h
+++ b/test/codec_factory.h
@@ -32,9 +32,9 @@
class CodecFactory {
public:
- CodecFactory() {}
+ CodecFactory() = default;
- virtual ~CodecFactory() {}
+ virtual ~CodecFactory() = default;
virtual Decoder *CreateDecoder(aom_codec_dec_cfg_t cfg) const = 0;
@@ -95,7 +95,7 @@
: Decoder(cfg, flag) {}
protected:
- virtual aom_codec_iface_t *CodecInterface() const {
+ aom_codec_iface_t *CodecInterface() const override {
#if CONFIG_AV1_DECODER
return aom_codec_av1_dx();
#else
@@ -111,7 +111,7 @@
: Encoder(cfg, init_flags, stats) {}
protected:
- virtual aom_codec_iface_t *CodecInterface() const {
+ aom_codec_iface_t *CodecInterface() const override {
#if CONFIG_AV1_ENCODER
return aom_codec_av1_cx();
#else
@@ -124,12 +124,12 @@
public:
AV1CodecFactory() : CodecFactory() {}
- virtual Decoder *CreateDecoder(aom_codec_dec_cfg_t cfg) const {
+ Decoder *CreateDecoder(aom_codec_dec_cfg_t cfg) const override {
return CreateDecoder(cfg, 0);
}
- virtual Decoder *CreateDecoder(aom_codec_dec_cfg_t cfg,
- const aom_codec_flags_t flags) const {
+ Decoder *CreateDecoder(aom_codec_dec_cfg_t cfg,
+ const aom_codec_flags_t flags) const override {
#if CONFIG_AV1_DECODER
return new AV1Decoder(cfg, flags);
#else
@@ -139,9 +139,9 @@
#endif
}
- virtual Encoder *CreateEncoder(aom_codec_enc_cfg_t cfg,
- const aom_codec_flags_t init_flags,
- TwopassStatsStore *stats) const {
+ Encoder *CreateEncoder(aom_codec_enc_cfg_t cfg,
+ const aom_codec_flags_t init_flags,
+ TwopassStatsStore *stats) const override {
#if CONFIG_AV1_ENCODER
return new AV1Encoder(cfg, init_flags, stats);
#else
@@ -152,8 +152,8 @@
#endif
}
- virtual aom_codec_err_t DefaultEncoderConfig(aom_codec_enc_cfg_t *cfg,
- unsigned int usage) const {
+ aom_codec_err_t DefaultEncoderConfig(aom_codec_enc_cfg_t *cfg,
+ unsigned int usage) const override {
#if CONFIG_AV1_ENCODER
return aom_codec_enc_config_default(aom_codec_av1_cx(), cfg, usage);
#else
diff --git a/test/coding_path_sync.cc b/test/coding_path_sync.cc
index c3e51fd..f7b7eac 100644
--- a/test/coding_path_sync.cc
+++ b/test/coding_path_sync.cc
@@ -120,9 +120,9 @@
int width_, height_;
};
-// lowers an aom_image_t to a easily comparable/printable form
-std::vector<int16_t> Serialize(const aom_image_t *img) {
- std::vector<int16_t> bytes;
+// lowers an aom_image_t to an easily comparable/printable form
+std::vector<uint16_t> Serialize(const aom_image_t *img) {
+ std::vector<uint16_t> bytes;
bytes.reserve(img->d_w * img->d_h * 3);
for (int plane = 0; plane < 3; ++plane) {
const int w = aom_img_plane_width(img, plane);
@@ -130,11 +130,13 @@
for (int r = 0; r < h; ++r) {
for (int c = 0; c < w; ++c) {
- unsigned char *row = img->planes[plane] + r * img->stride[plane];
- if (img->fmt & AOM_IMG_FMT_HIGHBITDEPTH)
- bytes.push_back(row[c * 2]);
- else
+ const unsigned char *row = img->planes[plane] + r * img->stride[plane];
+ if (img->fmt & AOM_IMG_FMT_HIGHBITDEPTH) {
+ const uint16_t *row16 = reinterpret_cast<const uint16_t *>(row);
+ bytes.push_back(row16[c]);
+ } else {
bytes.push_back(row[c]);
+ }
}
}
}
@@ -155,7 +157,7 @@
~Decoder() { aom_codec_destroy(&dec_); }
- std::vector<int16_t> decode(const aom_codec_cx_pkt_t *pkt) {
+ std::vector<uint16_t> decode(const aom_codec_cx_pkt_t *pkt) {
aom_codec_decode(&dec_, static_cast<uint8_t *>(pkt->data.frame.buf),
pkt->data.frame.sz, nullptr);
@@ -179,8 +181,8 @@
for (int k = 0; k < 3; ++k) {
const aom_codec_cx_pkt_t *frame = enc.ReadFrame();
- std::vector<int16_t> lbd_yuv = dec_lbd.decode(frame);
- std::vector<int16_t> hbd_yuv = dec_hbd.decode(frame);
+ std::vector<uint16_t> lbd_yuv = dec_lbd.decode(frame);
+ std::vector<uint16_t> hbd_yuv = dec_hbd.decode(frame);
ASSERT_EQ(lbd_yuv, hbd_yuv);
}
@@ -199,8 +201,8 @@
for (int k = 0; k < 5; ++k) {
const aom_codec_cx_pkt_t *frame = enc.ReadFrame();
- std::vector<int16_t> lbd_yuv = dec_lbd.decode(frame);
- std::vector<int16_t> hbd_yuv = dec_hbd.decode(frame);
+ std::vector<uint16_t> lbd_yuv = dec_lbd.decode(frame);
+ std::vector<uint16_t> hbd_yuv = dec_hbd.decode(frame);
ASSERT_EQ(lbd_yuv, hbd_yuv);
}
diff --git a/test/comp_avg_pred_test.cc b/test/comp_avg_pred_test.cc
index 4218ac3..2f81d7e 100644
--- a/test/comp_avg_pred_test.cc
+++ b/test/comp_avg_pred_test.cc
@@ -13,9 +13,12 @@
using libaom_test::ACMRandom;
using libaom_test::AV1DISTWTDCOMPAVG::AV1DISTWTDCOMPAVGTest;
+using libaom_test::AV1DISTWTDCOMPAVG::DistWtdCompAvgParam;
GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(AV1DISTWTDCOMPAVGTest);
using libaom_test::AV1DISTWTDCOMPAVG::AV1DISTWTDCOMPAVGUPSAMPLEDTest;
GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(AV1DISTWTDCOMPAVGUPSAMPLEDTest);
+using libaom_test::AV1DISTWTDCOMPAVG::DistWtdCompAvgTest;
+GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(DistWtdCompAvgTest);
#if CONFIG_AV1_HIGHBITDEPTH
using libaom_test::AV1DISTWTDCOMPAVG::AV1HighBDDISTWTDCOMPAVGTest;
GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(AV1HighBDDISTWTDCOMPAVGTest);
@@ -26,6 +29,19 @@
using std::make_tuple;
using std::tuple;
+uint8_t *DistWtdCompAvgTest::reference_data_ = nullptr;
+uint8_t *DistWtdCompAvgTest::second_pred_ = nullptr;
+uint8_t *DistWtdCompAvgTest::comp_pred_ = nullptr;
+uint8_t *DistWtdCompAvgTest::comp_pred_test_ = nullptr;
+uint8_t *DistWtdCompAvgTest::reference_data8_ = nullptr;
+uint8_t *DistWtdCompAvgTest::second_pred8_ = nullptr;
+uint8_t *DistWtdCompAvgTest::comp_pred8_ = nullptr;
+uint8_t *DistWtdCompAvgTest::comp_pred8_test_ = nullptr;
+uint16_t *DistWtdCompAvgTest::reference_data16_ = nullptr;
+uint16_t *DistWtdCompAvgTest::second_pred16_ = nullptr;
+uint16_t *DistWtdCompAvgTest::comp_pred16_ = nullptr;
+uint16_t *DistWtdCompAvgTest::comp_pred16_test_ = nullptr;
+
namespace {
TEST_P(AV1DISTWTDCOMPAVGTest, DISABLED_Speed) { RunSpeedTest(GET_PARAM(0)); }
@@ -52,6 +68,141 @@
aom_dist_wtd_comp_avg_upsampled_pred_ssse3));
#endif
+#if HAVE_NEON
+INSTANTIATE_TEST_SUITE_P(NEON, AV1DISTWTDCOMPAVGUPSAMPLEDTest,
+ libaom_test::AV1DISTWTDCOMPAVG::BuildParams(
+ aom_dist_wtd_comp_avg_upsampled_pred_neon));
+#endif // HAVE_NEON
+
+TEST_P(DistWtdCompAvgTest, MaxRef) {
+ FillConstant(reference_data_, reference_stride_, mask_);
+ FillConstant(second_pred_, width_, 0);
+ CheckCompAvg();
+}
+
+TEST_P(DistWtdCompAvgTest, MaxSecondPred) {
+ FillConstant(reference_data_, reference_stride_, 0);
+ FillConstant(second_pred_, width_, mask_);
+ CheckCompAvg();
+}
+
+TEST_P(DistWtdCompAvgTest, ShortRef) {
+ const int tmp_stride = reference_stride_;
+ reference_stride_ >>= 1;
+ FillRandom(reference_data_, reference_stride_);
+ FillRandom(second_pred_, width_);
+ CheckCompAvg();
+ reference_stride_ = tmp_stride;
+}
+
+TEST_P(DistWtdCompAvgTest, UnalignedRef) {
+ // The reference frame, but not the source frame, may be unaligned for
+ // certain types of searches.
+ const int tmp_stride = reference_stride_;
+ reference_stride_ -= 1;
+ FillRandom(reference_data_, reference_stride_);
+ FillRandom(second_pred_, width_);
+ CheckCompAvg();
+ reference_stride_ = tmp_stride;
+}
+
+// TODO(chengchen): add highbd tests
+const DistWtdCompAvgParam dist_wtd_comp_avg_c_tests[] = {
+ make_tuple(128, 128, &aom_dist_wtd_comp_avg_pred_c, -1),
+ make_tuple(128, 64, &aom_dist_wtd_comp_avg_pred_c, -1),
+ make_tuple(64, 128, &aom_dist_wtd_comp_avg_pred_c, -1),
+ make_tuple(64, 64, &aom_dist_wtd_comp_avg_pred_c, -1),
+ make_tuple(64, 32, &aom_dist_wtd_comp_avg_pred_c, -1),
+ make_tuple(32, 64, &aom_dist_wtd_comp_avg_pred_c, -1),
+ make_tuple(32, 32, &aom_dist_wtd_comp_avg_pred_c, -1),
+ make_tuple(32, 16, &aom_dist_wtd_comp_avg_pred_c, -1),
+ make_tuple(16, 32, &aom_dist_wtd_comp_avg_pred_c, -1),
+ make_tuple(16, 16, &aom_dist_wtd_comp_avg_pred_c, -1),
+ make_tuple(16, 8, &aom_dist_wtd_comp_avg_pred_c, -1),
+ make_tuple(8, 16, &aom_dist_wtd_comp_avg_pred_c, -1),
+ make_tuple(8, 8, &aom_dist_wtd_comp_avg_pred_c, -1),
+ make_tuple(8, 4, &aom_dist_wtd_comp_avg_pred_c, -1),
+ make_tuple(4, 8, &aom_dist_wtd_comp_avg_pred_c, -1),
+ make_tuple(4, 4, &aom_dist_wtd_comp_avg_pred_c, -1),
+
+#if !CONFIG_REALTIME_ONLY
+ make_tuple(64, 16, &aom_dist_wtd_comp_avg_pred_c, -1),
+ make_tuple(16, 64, &aom_dist_wtd_comp_avg_pred_c, -1),
+ make_tuple(32, 8, &aom_dist_wtd_comp_avg_pred_c, -1),
+ make_tuple(8, 32, &aom_dist_wtd_comp_avg_pred_c, -1),
+ make_tuple(16, 4, &aom_dist_wtd_comp_avg_pred_c, -1),
+ make_tuple(4, 16, &aom_dist_wtd_comp_avg_pred_c, -1),
+#endif
+};
+
+INSTANTIATE_TEST_SUITE_P(C, DistWtdCompAvgTest,
+ ::testing::ValuesIn(dist_wtd_comp_avg_c_tests));
+
+#if HAVE_SSSE3
+const DistWtdCompAvgParam dist_wtd_comp_avg_ssse3_tests[] = {
+ make_tuple(128, 128, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
+ make_tuple(128, 64, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
+ make_tuple(64, 128, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
+ make_tuple(64, 64, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
+ make_tuple(64, 32, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
+ make_tuple(32, 64, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
+ make_tuple(32, 32, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
+ make_tuple(32, 16, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
+ make_tuple(16, 32, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
+ make_tuple(16, 16, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
+ make_tuple(16, 8, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
+ make_tuple(8, 16, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
+ make_tuple(8, 8, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
+ make_tuple(8, 4, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
+ make_tuple(4, 8, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
+ make_tuple(4, 4, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
+ make_tuple(16, 16, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
+#if !CONFIG_REALTIME_ONLY
+ make_tuple(64, 16, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
+ make_tuple(16, 64, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
+ make_tuple(32, 8, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
+ make_tuple(8, 32, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
+ make_tuple(16, 4, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
+ make_tuple(4, 16, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
+#endif
+};
+
+INSTANTIATE_TEST_SUITE_P(SSSE3, DistWtdCompAvgTest,
+ ::testing::ValuesIn(dist_wtd_comp_avg_ssse3_tests));
+#endif // HAVE_SSSE3
+
+#if HAVE_NEON
+const DistWtdCompAvgParam dist_wtd_comp_avg_neon_tests[] = {
+ make_tuple(128, 128, &aom_dist_wtd_comp_avg_pred_neon, -1),
+ make_tuple(128, 64, &aom_dist_wtd_comp_avg_pred_neon, -1),
+ make_tuple(64, 128, &aom_dist_wtd_comp_avg_pred_neon, -1),
+ make_tuple(64, 64, &aom_dist_wtd_comp_avg_pred_neon, -1),
+ make_tuple(64, 32, &aom_dist_wtd_comp_avg_pred_neon, -1),
+ make_tuple(32, 64, &aom_dist_wtd_comp_avg_pred_neon, -1),
+ make_tuple(32, 32, &aom_dist_wtd_comp_avg_pred_neon, -1),
+ make_tuple(32, 16, &aom_dist_wtd_comp_avg_pred_neon, -1),
+ make_tuple(16, 32, &aom_dist_wtd_comp_avg_pred_neon, -1),
+ make_tuple(16, 16, &aom_dist_wtd_comp_avg_pred_neon, -1),
+ make_tuple(16, 8, &aom_dist_wtd_comp_avg_pred_neon, -1),
+ make_tuple(8, 16, &aom_dist_wtd_comp_avg_pred_neon, -1),
+ make_tuple(8, 8, &aom_dist_wtd_comp_avg_pred_neon, -1),
+ make_tuple(8, 4, &aom_dist_wtd_comp_avg_pred_neon, -1),
+ make_tuple(4, 8, &aom_dist_wtd_comp_avg_pred_neon, -1),
+ make_tuple(4, 4, &aom_dist_wtd_comp_avg_pred_neon, -1),
+#if !CONFIG_REALTIME_ONLY
+ make_tuple(64, 16, &aom_dist_wtd_comp_avg_pred_neon, -1),
+ make_tuple(16, 64, &aom_dist_wtd_comp_avg_pred_neon, -1),
+ make_tuple(32, 8, &aom_dist_wtd_comp_avg_pred_neon, -1),
+ make_tuple(8, 32, &aom_dist_wtd_comp_avg_pred_neon, -1),
+ make_tuple(16, 4, &aom_dist_wtd_comp_avg_pred_neon, -1),
+ make_tuple(4, 16, &aom_dist_wtd_comp_avg_pred_neon, -1),
+#endif // !CONFIG_REALTIME_ONLY
+};
+
+INSTANTIATE_TEST_SUITE_P(NEON, DistWtdCompAvgTest,
+ ::testing::ValuesIn(dist_wtd_comp_avg_neon_tests));
+#endif // HAVE_NEON
+
#if CONFIG_AV1_HIGHBITDEPTH
TEST_P(AV1HighBDDISTWTDCOMPAVGTest, DISABLED_Speed) {
RunSpeedTest(GET_PARAM(1));
@@ -67,6 +218,12 @@
aom_highbd_dist_wtd_comp_avg_pred_sse2, 1));
#endif
+#if HAVE_NEON
+INSTANTIATE_TEST_SUITE_P(NEON, AV1HighBDDISTWTDCOMPAVGTest,
+ libaom_test::AV1DISTWTDCOMPAVG::BuildParams(
+ aom_highbd_dist_wtd_comp_avg_pred_neon, 1));
+#endif
+
TEST_P(AV1HighBDDISTWTDCOMPAVGUPSAMPLEDTest, DISABLED_Speed) {
RunSpeedTest(GET_PARAM(1));
}
@@ -80,6 +237,13 @@
libaom_test::AV1DISTWTDCOMPAVG::BuildParams(
aom_highbd_dist_wtd_comp_avg_upsampled_pred_sse2));
#endif
+
+#if HAVE_NEON
+INSTANTIATE_TEST_SUITE_P(NEON, AV1HighBDDISTWTDCOMPAVGUPSAMPLEDTest,
+ libaom_test::AV1DISTWTDCOMPAVG::BuildParams(
+ aom_highbd_dist_wtd_comp_avg_upsampled_pred_neon));
+#endif
+
#endif // CONFIG_AV1_HIGHBITDEPTH
} // namespace
diff --git a/test/comp_avg_pred_test.h b/test/comp_avg_pred_test.h
index c1526d8..396df2e 100644
--- a/test/comp_avg_pred_test.h
+++ b/test/comp_avg_pred_test.h
@@ -40,11 +40,18 @@
int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref,
int ref_stride, const DIST_WTD_COMP_PARAMS *jcp_param, int subpel_search);
+typedef void (*DistWtdCompAvgFunc)(uint8_t *comp_pred, const uint8_t *pred,
+ int width, int height, const uint8_t *ref,
+ int ref_stride,
+ const DIST_WTD_COMP_PARAMS *jcp_param);
+
typedef std::tuple<distwtdcompavg_func, BLOCK_SIZE> DISTWTDCOMPAVGParam;
typedef std::tuple<distwtdcompavgupsampled_func, BLOCK_SIZE>
DISTWTDCOMPAVGUPSAMPLEDParam;
+typedef std::tuple<int, int, DistWtdCompAvgFunc, int> DistWtdCompAvgParam;
+
#if CONFIG_AV1_HIGHBITDEPTH
typedef void (*highbddistwtdcompavgupsampled_func)(
MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col,
@@ -90,8 +97,8 @@
class AV1DISTWTDCOMPAVGTest
: public ::testing::TestWithParam<DISTWTDCOMPAVGParam> {
public:
- ~AV1DISTWTDCOMPAVGTest() {}
- void SetUp() { rnd_.Reset(ACMRandom::DeterministicSeed()); }
+ ~AV1DISTWTDCOMPAVGTest() override = default;
+ void SetUp() override { rnd_.Reset(ACMRandom::DeterministicSeed()); }
protected:
void RunCheckOutput(distwtdcompavg_func test_impl) {
@@ -193,8 +200,8 @@
class AV1DISTWTDCOMPAVGUPSAMPLEDTest
: public ::testing::TestWithParam<DISTWTDCOMPAVGUPSAMPLEDParam> {
public:
- ~AV1DISTWTDCOMPAVGUPSAMPLEDTest() {}
- void SetUp() { rnd_.Reset(ACMRandom::DeterministicSeed()); }
+ ~AV1DISTWTDCOMPAVGUPSAMPLEDTest() override = default;
+ void SetUp() override { rnd_.Reset(ACMRandom::DeterministicSeed()); }
protected:
void RunCheckOutput(distwtdcompavgupsampled_func test_impl) {
@@ -317,12 +324,198 @@
libaom_test::ACMRandom rnd_;
}; // class AV1DISTWTDCOMPAVGUPSAMPLEDTest
+class DistWtdCompAvgTest
+ : public ::testing::WithParamInterface<DistWtdCompAvgParam>,
+ public ::testing::Test {
+ public:
+ DistWtdCompAvgTest()
+ : width_(GET_PARAM(0)), height_(GET_PARAM(1)), bd_(GET_PARAM(3)) {}
+
+ static void SetUpTestSuite() {
+ reference_data8_ = reinterpret_cast<uint8_t *>(
+ aom_memalign(kDataAlignment, kDataBufferSize));
+ ASSERT_NE(reference_data8_, nullptr);
+ second_pred8_ =
+ reinterpret_cast<uint8_t *>(aom_memalign(kDataAlignment, 128 * 128));
+ ASSERT_NE(second_pred8_, nullptr);
+ comp_pred8_ =
+ reinterpret_cast<uint8_t *>(aom_memalign(kDataAlignment, 128 * 128));
+ ASSERT_NE(comp_pred8_, nullptr);
+ comp_pred8_test_ =
+ reinterpret_cast<uint8_t *>(aom_memalign(kDataAlignment, 128 * 128));
+ ASSERT_NE(comp_pred8_test_, nullptr);
+ reference_data16_ = reinterpret_cast<uint16_t *>(
+ aom_memalign(kDataAlignment, kDataBufferSize * sizeof(uint16_t)));
+ ASSERT_NE(reference_data16_, nullptr);
+ second_pred16_ = reinterpret_cast<uint16_t *>(
+ aom_memalign(kDataAlignment, 128 * 128 * sizeof(uint16_t)));
+ ASSERT_NE(second_pred16_, nullptr);
+ comp_pred16_ = reinterpret_cast<uint16_t *>(
+ aom_memalign(kDataAlignment, 128 * 128 * sizeof(uint16_t)));
+ ASSERT_NE(comp_pred16_, nullptr);
+ comp_pred16_test_ = reinterpret_cast<uint16_t *>(
+ aom_memalign(kDataAlignment, 128 * 128 * sizeof(uint16_t)));
+ ASSERT_NE(comp_pred16_test_, nullptr);
+ }
+
+ static void TearDownTestSuite() {
+ aom_free(reference_data8_);
+ reference_data8_ = nullptr;
+ aom_free(second_pred8_);
+ second_pred8_ = nullptr;
+ aom_free(comp_pred8_);
+ comp_pred8_ = nullptr;
+ aom_free(comp_pred8_test_);
+ comp_pred8_test_ = nullptr;
+ aom_free(reference_data16_);
+ reference_data16_ = nullptr;
+ aom_free(second_pred16_);
+ second_pred16_ = nullptr;
+ aom_free(comp_pred16_);
+ comp_pred16_ = nullptr;
+ aom_free(comp_pred16_test_);
+ comp_pred16_test_ = nullptr;
+ }
+
+ protected:
+ // Handle up to 4 128x128 blocks, with stride up to 256
+ static const int kDataAlignment = 16;
+ static const int kDataBlockSize = 128 * 256;
+ static const int kDataBufferSize = 4 * kDataBlockSize;
+
+ void SetUp() override {
+ if (bd_ == -1) {
+ use_high_bit_depth_ = false;
+ bit_depth_ = AOM_BITS_8;
+ reference_data_ = reference_data8_;
+ second_pred_ = second_pred8_;
+ comp_pred_ = comp_pred8_;
+ comp_pred_test_ = comp_pred8_test_;
+ } else {
+ use_high_bit_depth_ = true;
+ bit_depth_ = static_cast<aom_bit_depth_t>(bd_);
+ reference_data_ = CONVERT_TO_BYTEPTR(reference_data16_);
+ second_pred_ = CONVERT_TO_BYTEPTR(second_pred16_);
+ comp_pred_ = CONVERT_TO_BYTEPTR(comp_pred16_);
+ comp_pred_test_ = CONVERT_TO_BYTEPTR(comp_pred16_test_);
+ }
+ mask_ = (1 << bit_depth_) - 1;
+ reference_stride_ = width_ * 2;
+ rnd_.Reset(ACMRandom::DeterministicSeed());
+ }
+
+ virtual uint8_t *GetReference(int block_idx) {
+ if (use_high_bit_depth_)
+ return CONVERT_TO_BYTEPTR(CONVERT_TO_SHORTPTR(reference_data_) +
+ block_idx * kDataBlockSize);
+ return reference_data_ + block_idx * kDataBlockSize;
+ }
+
+ void ReferenceDistWtdCompAvg(int block_idx) {
+ const uint8_t *const reference8 = GetReference(block_idx);
+ const uint8_t *const second_pred8 = second_pred_;
+ uint8_t *const comp_pred8 = comp_pred_;
+ const uint16_t *const reference16 =
+ CONVERT_TO_SHORTPTR(GetReference(block_idx));
+ const uint16_t *const second_pred16 = CONVERT_TO_SHORTPTR(second_pred_);
+ uint16_t *const comp_pred16 = CONVERT_TO_SHORTPTR(comp_pred_);
+ for (int h = 0; h < height_; ++h) {
+ for (int w = 0; w < width_; ++w) {
+ if (!use_high_bit_depth_) {
+ const int tmp =
+ second_pred8[h * width_ + w] * jcp_param_.bck_offset +
+ reference8[h * reference_stride_ + w] * jcp_param_.fwd_offset;
+ comp_pred8[h * width_ + w] = ROUND_POWER_OF_TWO(tmp, 4);
+ } else {
+ const int tmp =
+ second_pred16[h * width_ + w] * jcp_param_.bck_offset +
+ reference16[h * reference_stride_ + w] * jcp_param_.fwd_offset;
+ comp_pred16[h * width_ + w] = ROUND_POWER_OF_TWO(tmp, 4);
+ }
+ }
+ }
+ }
+
+ void FillConstant(uint8_t *data, int stride, uint16_t fill_constant) {
+ uint8_t *data8 = data;
+ uint16_t *data16 = CONVERT_TO_SHORTPTR(data);
+ for (int h = 0; h < height_; ++h) {
+ for (int w = 0; w < width_; ++w) {
+ if (!use_high_bit_depth_) {
+ data8[h * stride + w] = static_cast<uint8_t>(fill_constant);
+ } else {
+ data16[h * stride + w] = fill_constant;
+ }
+ }
+ }
+ }
+
+ void FillRandom(uint8_t *data, int stride) {
+ uint8_t *data8 = data;
+ uint16_t *data16 = CONVERT_TO_SHORTPTR(data);
+ for (int h = 0; h < height_; ++h) {
+ for (int w = 0; w < width_; ++w) {
+ if (!use_high_bit_depth_) {
+ data8[h * stride + w] = rnd_.Rand8();
+ } else {
+ data16[h * stride + w] = rnd_.Rand16() & mask_;
+ }
+ }
+ }
+ }
+
+ void dist_wtd_comp_avg(int block_idx) {
+ const uint8_t *const reference = GetReference(block_idx);
+
+ API_REGISTER_STATE_CHECK(GET_PARAM(2)(comp_pred_test_, second_pred_, width_,
+ height_, reference, reference_stride_,
+ &jcp_param_));
+ }
+
+ void CheckCompAvg() {
+ for (int j = 0; j < 2; ++j) {
+ for (int i = 0; i < 4; ++i) {
+ jcp_param_.fwd_offset = quant_dist_lookup_table[i][j];
+ jcp_param_.bck_offset = quant_dist_lookup_table[i][1 - j];
+
+ ReferenceDistWtdCompAvg(0);
+ dist_wtd_comp_avg(0);
+
+ for (int y = 0; y < height_; ++y)
+ for (int x = 0; x < width_; ++x)
+ ASSERT_EQ(comp_pred_[y * width_ + x],
+ comp_pred_test_[y * width_ + x]);
+ }
+ }
+ }
+
+ int width_, height_, mask_, bd_;
+ aom_bit_depth_t bit_depth_;
+ static uint8_t *reference_data_;
+ static uint8_t *second_pred_;
+ bool use_high_bit_depth_;
+ static uint8_t *reference_data8_;
+ static uint8_t *second_pred8_;
+ static uint16_t *reference_data16_;
+ static uint16_t *second_pred16_;
+ int reference_stride_;
+ static uint8_t *comp_pred_;
+ static uint8_t *comp_pred8_;
+ static uint16_t *comp_pred16_;
+ static uint8_t *comp_pred_test_;
+ static uint8_t *comp_pred8_test_;
+ static uint16_t *comp_pred16_test_;
+ DIST_WTD_COMP_PARAMS jcp_param_;
+
+ ACMRandom rnd_;
+};
+
#if CONFIG_AV1_HIGHBITDEPTH
class AV1HighBDDISTWTDCOMPAVGTest
: public ::testing::TestWithParam<HighbdDISTWTDCOMPAVGParam> {
public:
- ~AV1HighBDDISTWTDCOMPAVGTest() {}
- void SetUp() { rnd_.Reset(ACMRandom::DeterministicSeed()); }
+ ~AV1HighBDDISTWTDCOMPAVGTest() override = default;
+ void SetUp() override { rnd_.Reset(ACMRandom::DeterministicSeed()); }
protected:
void RunCheckOutput(distwtdcompavg_func test_impl) {
@@ -430,8 +623,8 @@
class AV1HighBDDISTWTDCOMPAVGUPSAMPLEDTest
: public ::testing::TestWithParam<HighbdDISTWTDCOMPAVGUPSAMPLEDParam> {
public:
- ~AV1HighBDDISTWTDCOMPAVGUPSAMPLEDTest() {}
- void SetUp() { rnd_.Reset(ACMRandom::DeterministicSeed()); }
+ ~AV1HighBDDISTWTDCOMPAVGUPSAMPLEDTest() override = default;
+ void SetUp() override { rnd_.Reset(ACMRandom::DeterministicSeed()); }
protected:
void RunCheckOutput(highbddistwtdcompavgupsampled_func test_impl) {
diff --git a/test/comp_mask_pred_test.cc b/test/comp_mask_pred_test.cc
index 06c3192..b65730a 100644
--- a/test/comp_mask_pred_test.cc
+++ b/test/comp_mask_pred_test.cc
@@ -48,10 +48,10 @@
class AV1CompMaskPredBase : public ::testing::Test {
public:
- ~AV1CompMaskPredBase();
- void SetUp();
+ ~AV1CompMaskPredBase() override;
+ void SetUp() override;
- void TearDown();
+ void TearDown() override;
protected:
bool CheckResult(int width, int height) {
@@ -76,7 +76,7 @@
uint8_t *ref_;
};
-AV1CompMaskPredBase::~AV1CompMaskPredBase() {}
+AV1CompMaskPredBase::~AV1CompMaskPredBase() = default;
void AV1CompMaskPredBase::SetUp() {
rnd_.Reset(libaom_test::ACMRandom::DeterministicSeed());
@@ -303,10 +303,10 @@
class AV1CompAvgPredTest : public ::testing::TestWithParam<CompAvgPredParam> {
public:
- ~AV1CompAvgPredTest();
- void SetUp();
+ ~AV1CompAvgPredTest() override;
+ void SetUp() override;
- void TearDown();
+ void TearDown() override;
protected:
void RunCheckOutput(comp_avg_pred_func test_impl, BLOCK_SIZE bsize);
@@ -333,7 +333,7 @@
};
GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(AV1CompAvgPredTest);
-AV1CompAvgPredTest::~AV1CompAvgPredTest() {}
+AV1CompAvgPredTest::~AV1CompAvgPredTest() = default;
void AV1CompAvgPredTest::SetUp() {
rnd_.Reset(libaom_test::ACMRandom::DeterministicSeed());
@@ -390,7 +390,7 @@
const double time = static_cast<double>(aom_usec_timer_elapsed(&timer));
elapsed_time[i] = 1000.0 * time;
}
- printf("compMask %3dx%-3d: %7.2f/%7.2fns", w, h, elapsed_time[0],
+ printf("CompAvgPred %3dx%-3d: %7.2f/%7.2fns", w, h, elapsed_time[0],
elapsed_time[1]);
printf("(%3.2f)\n", elapsed_time[0] / elapsed_time[1]);
}
@@ -420,10 +420,10 @@
#if CONFIG_AV1_HIGHBITDEPTH
class AV1HighbdCompMaskPredTestBase : public ::testing::Test {
public:
- ~AV1HighbdCompMaskPredTestBase();
- void SetUp();
+ ~AV1HighbdCompMaskPredTestBase() override;
+ void SetUp() override;
- void TearDown();
+ void TearDown() override;
protected:
bool CheckResult(int width, int height) {
@@ -448,7 +448,7 @@
uint16_t *ref_;
};
-AV1HighbdCompMaskPredTestBase::~AV1HighbdCompMaskPredTestBase() {}
+AV1HighbdCompMaskPredTestBase::~AV1HighbdCompMaskPredTestBase() = default;
void AV1HighbdCompMaskPredTestBase::SetUp() {
rnd_.Reset(libaom_test::ACMRandom::DeterministicSeed());
@@ -494,14 +494,14 @@
: public AV1HighbdCompMaskPredTestBase,
public ::testing::WithParamInterface<HighbdCompMaskPredParam> {
public:
- ~AV1HighbdCompMaskPredTest();
+ ~AV1HighbdCompMaskPredTest() override;
protected:
void RunCheckOutput(comp_mask_pred_func test_impl, BLOCK_SIZE bsize, int inv);
void RunSpeedTest(comp_mask_pred_func test_impl, BLOCK_SIZE bsize);
};
-AV1HighbdCompMaskPredTest::~AV1HighbdCompMaskPredTest() {}
+AV1HighbdCompMaskPredTest::~AV1HighbdCompMaskPredTest() = default;
void AV1HighbdCompMaskPredTest::RunCheckOutput(
highbd_comp_mask_pred_func test_impl, BLOCK_SIZE bsize, int inv) {
@@ -583,6 +583,14 @@
RunSpeedTest(GET_PARAM(0), GET_PARAM(1));
}
+#if HAVE_NEON
+INSTANTIATE_TEST_SUITE_P(
+ NEON, AV1HighbdCompMaskPredTest,
+ ::testing::Combine(::testing::Values(&aom_highbd_comp_mask_pred_neon),
+ ::testing::ValuesIn(kCompMaskPredParams),
+ ::testing::Range(8, 13, 2)));
+#endif
+
#if HAVE_AVX2
INSTANTIATE_TEST_SUITE_P(
AVX2, AV1HighbdCompMaskPredTest,
@@ -612,7 +620,7 @@
: public AV1HighbdCompMaskPredTestBase,
public ::testing::WithParamInterface<HighbdUpsampledPredParam> {
public:
- ~AV1HighbdUpsampledPredTest();
+ ~AV1HighbdUpsampledPredTest() override;
protected:
void RunCheckOutput(highbd_upsampled_pred_func test_impl, BLOCK_SIZE bsize);
@@ -620,7 +628,7 @@
int havSub);
};
-AV1HighbdUpsampledPredTest::~AV1HighbdUpsampledPredTest() {}
+AV1HighbdUpsampledPredTest::~AV1HighbdUpsampledPredTest() = default;
void AV1HighbdUpsampledPredTest::RunCheckOutput(
highbd_upsampled_pred_func test_impl, BLOCK_SIZE bsize) {
@@ -712,5 +720,137 @@
::testing::Range(8, 13, 2)));
#endif
+#if HAVE_NEON
+INSTANTIATE_TEST_SUITE_P(
+ NEON, AV1HighbdUpsampledPredTest,
+ ::testing::Combine(::testing::Values(&aom_highbd_upsampled_pred_neon),
+ ::testing::ValuesIn(kValidBlockSize),
+ ::testing::Range(8, 13, 2)));
+#endif
+
+typedef void (*highbd_comp_avg_pred_func)(uint8_t *comp_pred,
+ const uint8_t *pred, int width,
+ int height, const uint8_t *ref,
+ int ref_stride);
+
+typedef std::tuple<highbd_comp_avg_pred_func, BLOCK_SIZE, int>
+ HighbdCompAvgPredParam;
+
+class AV1HighbdCompAvgPredTest
+ : public ::testing::TestWithParam<HighbdCompAvgPredParam> {
+ public:
+ ~AV1HighbdCompAvgPredTest() override;
+ void SetUp() override;
+
+ protected:
+ void RunCheckOutput(highbd_comp_avg_pred_func test_impl, BLOCK_SIZE bsize);
+ void RunSpeedTest(highbd_comp_avg_pred_func test_impl, BLOCK_SIZE bsize);
+ bool CheckResult(int width, int height) const {
+ for (int y = 0; y < height; ++y) {
+ for (int x = 0; x < width; ++x) {
+ const int idx = y * width + x;
+ if (comp_pred1_[idx] != comp_pred2_[idx]) {
+ printf("%dx%d mismatch @%d(%d,%d) ", width, height, idx, x, y);
+ printf("%d != %d ", comp_pred1_[idx], comp_pred2_[idx]);
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ libaom_test::ACMRandom rnd_;
+ uint16_t *comp_pred1_;
+ uint16_t *comp_pred2_;
+ uint16_t *pred_;
+ uint16_t *ref_;
+};
+
+AV1HighbdCompAvgPredTest::~AV1HighbdCompAvgPredTest() {
+ aom_free(comp_pred1_);
+ aom_free(comp_pred2_);
+ aom_free(pred_);
+ aom_free(ref_);
+}
+
+void AV1HighbdCompAvgPredTest::SetUp() {
+ int bd_ = GET_PARAM(2);
+ rnd_.Reset(libaom_test::ACMRandom::DeterministicSeed());
+
+ comp_pred1_ =
+ (uint16_t *)aom_memalign(16, MAX_SB_SQUARE * sizeof(*comp_pred1_));
+ ASSERT_NE(comp_pred1_, nullptr);
+ comp_pred2_ =
+ (uint16_t *)aom_memalign(16, MAX_SB_SQUARE * sizeof(*comp_pred2_));
+ ASSERT_NE(comp_pred2_, nullptr);
+ pred_ = (uint16_t *)aom_memalign(16, MAX_SB_SQUARE * sizeof(*pred_));
+ ASSERT_NE(pred_, nullptr);
+ ref_ = (uint16_t *)aom_memalign(16, MAX_SB_SQUARE * sizeof(*ref_));
+ ASSERT_NE(ref_, nullptr);
+ for (int i = 0; i < MAX_SB_SQUARE; ++i) {
+ pred_[i] = rnd_.Rand16() & ((1 << bd_) - 1);
+ }
+ for (int i = 0; i < MAX_SB_SQUARE; ++i) {
+ ref_[i] = rnd_.Rand16() & ((1 << bd_) - 1);
+ }
+}
+
+void AV1HighbdCompAvgPredTest::RunCheckOutput(
+ highbd_comp_avg_pred_func test_impl, BLOCK_SIZE bsize) {
+ const int w = block_size_wide[bsize];
+ const int h = block_size_high[bsize];
+ aom_highbd_comp_avg_pred_c(CONVERT_TO_BYTEPTR(comp_pred1_),
+ CONVERT_TO_BYTEPTR(pred_), w, h,
+ CONVERT_TO_BYTEPTR(ref_), MAX_SB_SIZE);
+ test_impl(CONVERT_TO_BYTEPTR(comp_pred2_), CONVERT_TO_BYTEPTR(pred_), w, h,
+ CONVERT_TO_BYTEPTR(ref_), MAX_SB_SIZE);
+
+ ASSERT_EQ(CheckResult(w, h), true);
+}
+
+void AV1HighbdCompAvgPredTest::RunSpeedTest(highbd_comp_avg_pred_func test_impl,
+ BLOCK_SIZE bsize) {
+ const int w = block_size_wide[bsize];
+ const int h = block_size_high[bsize];
+ const int num_loops = 1000000000 / (w + h);
+
+ highbd_comp_avg_pred_func functions[2] = { aom_highbd_comp_avg_pred_c,
+ test_impl };
+ double elapsed_time[2] = { 0.0 };
+ for (int i = 0; i < 2; ++i) {
+ aom_usec_timer timer;
+ aom_usec_timer_start(&timer);
+ highbd_comp_avg_pred_func func = functions[i];
+ for (int j = 0; j < num_loops; ++j) {
+ func(CONVERT_TO_BYTEPTR(comp_pred1_), CONVERT_TO_BYTEPTR(pred_), w, h,
+ CONVERT_TO_BYTEPTR(ref_), MAX_SB_SIZE);
+ }
+ aom_usec_timer_mark(&timer);
+ const double time = static_cast<double>(aom_usec_timer_elapsed(&timer));
+ elapsed_time[i] = 1000.0 * time;
+ }
+ printf("HighbdCompAvg %3dx%-3d: %7.2f/%7.2fns", w, h, elapsed_time[0],
+ elapsed_time[1]);
+ printf("(%3.2f)\n", elapsed_time[0] / elapsed_time[1]);
+}
+
+GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(AV1HighbdCompAvgPredTest);
+
+TEST_P(AV1HighbdCompAvgPredTest, CheckOutput) {
+ RunCheckOutput(GET_PARAM(0), GET_PARAM(1));
+}
+
+TEST_P(AV1HighbdCompAvgPredTest, DISABLED_Speed) {
+ RunSpeedTest(GET_PARAM(0), GET_PARAM(1));
+}
+
+#if HAVE_NEON
+INSTANTIATE_TEST_SUITE_P(
+ NEON, AV1HighbdCompAvgPredTest,
+ ::testing::Combine(::testing::Values(&aom_highbd_comp_avg_pred_neon),
+ ::testing::ValuesIn(kValidBlockSize),
+ ::testing::Range(8, 13, 2)));
+#endif
+
#endif // CONFIG_AV1_HIGHBITDEPTH
} // namespace
diff --git a/test/convolve_round_test.cc b/test/convolve_round_test.cc
deleted file mode 100644
index 0580744..0000000
--- a/test/convolve_round_test.cc
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- * Copyright (c) 2017, Alliance for Open Media. All rights reserved
- *
- * This source code is subject to the terms of the BSD 2 Clause License and
- * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
- * was not distributed with this source code in the LICENSE file, you can
- * obtain it at www.aomedia.org/license/software. If the Alliance for Open
- * Media Patent License 1.0 was not distributed with this source code in the
- * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
- */
-
-#include <assert.h>
-#include <tuple>
-
-#include "config/av1_rtcd.h"
-
-#include "aom/aom_integer.h"
-#include "aom_ports/aom_timer.h"
-#include "test/acm_random.h"
-#include "test/register_state_check.h"
-#include "test/util.h"
-#include "third_party/googletest/src/googletest/include/gtest/gtest.h"
-
-using libaom_test::ACMRandom;
-
-namespace {
-#define CONVOLVE_ROUNDING_PARAM \
- const int32_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, \
- int h, int bits
-
-typedef void (*ConvolveRoundFunc)(CONVOLVE_ROUNDING_PARAM);
-
-typedef void (*ConvolveRoundFuncHbd)(CONVOLVE_ROUNDING_PARAM, int bd);
-
-template <ConvolveRoundFuncHbd fn>
-void highbd_convolve_rounding_8(CONVOLVE_ROUNDING_PARAM) {
- const int bd = 8;
- fn(src, src_stride, dst, dst_stride, w, h, bits, bd);
-}
-
-template <ConvolveRoundFuncHbd fn>
-void highbd_convolve_rounding_10(CONVOLVE_ROUNDING_PARAM) {
- const int bd = 10;
- fn(src, src_stride, dst, dst_stride, w, h, bits, bd);
-}
-
-template <ConvolveRoundFuncHbd fn>
-void highbd_convolve_rounding_12(CONVOLVE_ROUNDING_PARAM) {
- const int bd = 12;
- fn(src, src_stride, dst, dst_stride, w, h, bits, bd);
-}
-
-typedef enum { LOWBITDEPTH_TEST, HIGHBITDEPTH_TEST } DataPathType;
-
-using std::tuple;
-
-typedef tuple<ConvolveRoundFunc, ConvolveRoundFunc, DataPathType>
- ConvolveRoundParam;
-
-const int kTestNum = 5000;
-
-class ConvolveRoundTest : public ::testing::TestWithParam<ConvolveRoundParam> {
- protected:
- ConvolveRoundTest()
- : func_ref_(GET_PARAM(0)), func_(GET_PARAM(1)), data_path_(GET_PARAM(2)) {
- }
- virtual ~ConvolveRoundTest() {}
-
- virtual void SetUp() {
- const size_t block_size = 128 * 128;
- src_ = reinterpret_cast<int32_t *>(
- aom_memalign(16, block_size * sizeof(*src_)));
- ASSERT_NE(src_, nullptr);
- dst_ref_ = reinterpret_cast<uint16_t *>(
- aom_memalign(16, block_size * sizeof(*dst_ref_)));
- ASSERT_NE(dst_ref_, nullptr);
- dst_ = reinterpret_cast<uint16_t *>(
- aom_memalign(16, block_size * sizeof(*dst_)));
- ASSERT_NE(dst_, nullptr);
- }
-
- virtual void TearDown() {
- aom_free(src_);
- aom_free(dst_ref_);
- aom_free(dst_);
- }
-
- void ConvolveRoundingRun() {
- int test_num = 0;
- const int src_stride = 128;
- const int dst_stride = 128;
- int bits = 13;
- uint8_t *dst = 0;
- uint8_t *dst_ref = 0;
-
- if (data_path_ == LOWBITDEPTH_TEST) {
- dst = reinterpret_cast<uint8_t *>(dst_);
- dst_ref = reinterpret_cast<uint8_t *>(dst_ref_);
- } else if (data_path_ == HIGHBITDEPTH_TEST) {
- dst = CONVERT_TO_BYTEPTR(dst_);
- dst_ref = CONVERT_TO_BYTEPTR(dst_ref_);
- } else {
- assert(0);
- }
-
- while (test_num < kTestNum) {
- int block_size = test_num % BLOCK_SIZES_ALL;
- int w = block_size_wide[block_size];
- int h = block_size_high[block_size];
-
- if (test_num % 2 == 0)
- bits -= 1;
- else
- bits += 1;
-
- GenerateBufferWithRandom(src_, src_stride, bits, w, h);
-
- func_ref_(src_, src_stride, dst_ref, dst_stride, w, h, bits);
- API_REGISTER_STATE_CHECK(
- func_(src_, src_stride, dst, dst_stride, w, h, bits));
-
- if (data_path_ == LOWBITDEPTH_TEST) {
- for (int r = 0; r < h; ++r) {
- for (int c = 0; c < w; ++c) {
- ASSERT_EQ(dst_ref[r * dst_stride + c], dst[r * dst_stride + c])
- << "Mismatch at r: " << r << " c: " << c << " w: " << w
- << " h: " << h << " test: " << test_num;
- }
- }
- } else {
- for (int r = 0; r < h; ++r) {
- for (int c = 0; c < w; ++c) {
- ASSERT_EQ(dst_ref_[r * dst_stride + c], dst_[r * dst_stride + c])
- << "Mismatch at r: " << r << " c: " << c << " w: " << w
- << " h: " << h << " test: " << test_num;
- }
- }
- }
-
- test_num++;
- }
- }
-
- void GenerateBufferWithRandom(int32_t *src, int src_stride, int bits, int w,
- int h) {
- int32_t number;
- for (int r = 0; r < h; ++r) {
- for (int c = 0; c < w; ++c) {
- number = static_cast<int32_t>(rand_.Rand31());
- number %= 1 << (bits + 9);
- src[r * src_stride + c] = number;
- }
- }
- }
-
- ACMRandom rand_;
- int32_t *src_;
- uint16_t *dst_ref_;
- uint16_t *dst_;
-
- ConvolveRoundFunc func_ref_;
- ConvolveRoundFunc func_;
- DataPathType data_path_;
-};
-
-TEST_P(ConvolveRoundTest, BitExactCheck) { ConvolveRoundingRun(); }
-
-using std::make_tuple;
-#if HAVE_AVX2
-const ConvolveRoundParam kConvRndParamArray[] = {
- make_tuple(&av1_convolve_rounding_c, &av1_convolve_rounding_avx2,
- LOWBITDEPTH_TEST),
- make_tuple(&highbd_convolve_rounding_8<av1_highbd_convolve_rounding_c>,
- &highbd_convolve_rounding_8<av1_highbd_convolve_rounding_avx2>,
- HIGHBITDEPTH_TEST),
- make_tuple(&highbd_convolve_rounding_10<av1_highbd_convolve_rounding_c>,
- &highbd_convolve_rounding_10<av1_highbd_convolve_rounding_avx2>,
- HIGHBITDEPTH_TEST),
- make_tuple(&highbd_convolve_rounding_12<av1_highbd_convolve_rounding_c>,
- &highbd_convolve_rounding_12<av1_highbd_convolve_rounding_avx2>,
- HIGHBITDEPTH_TEST)
-};
-INSTANTIATE_TEST_SUITE_P(AVX2, ConvolveRoundTest,
- ::testing::ValuesIn(kConvRndParamArray));
-#endif // HAVE_AVX2
-} // namespace
diff --git a/test/convolve_test.cc b/test/convolve_test.cc
index 8aed171..c97f814 100644
--- a/test/convolve_test.cc
+++ b/test/convolve_test.cc
@@ -301,8 +301,6 @@
ASSERT_NE(output16_ref_, nullptr);
}
- virtual void TearDown() {}
-
static void TearDownTestSuite() {
aom_free(input_ - 1);
input_ = nullptr;
@@ -345,7 +343,7 @@
i % kOuterBlockSize >= (BorderLeft() + Width()));
}
- virtual void SetUp() {
+ void SetUp() override {
UUT_ = GET_PARAM(2);
if (UUT_->use_highbd_ != 0)
mask_ = (1 << UUT_->use_highbd_) - 1;
@@ -764,6 +762,17 @@
WRAP(convolve8_horiz_avx2, 12)
WRAP(convolve8_vert_avx2, 12)
#endif // HAVE_AVX2
+
+#if HAVE_NEON
+WRAP(convolve8_horiz_neon, 8)
+WRAP(convolve8_vert_neon, 8)
+
+WRAP(convolve8_horiz_neon, 10)
+WRAP(convolve8_vert_neon, 10)
+
+WRAP(convolve8_horiz_neon, 12)
+WRAP(convolve8_vert_neon, 12)
+#endif // HAVE_NEON
#endif // CONFIG_AV1_HIGHBITDEPTH
#undef WRAP
@@ -866,6 +875,21 @@
#endif // HAVE_AVX2
#if HAVE_NEON
+#if CONFIG_AV1_HIGHBITDEPTH
+const ConvolveFunctions wrap_convolve8_neon(wrap_convolve8_horiz_neon_8,
+ wrap_convolve8_vert_neon_8, 8);
+const ConvolveFunctions wrap_convolve10_neon(wrap_convolve8_horiz_neon_10,
+ wrap_convolve8_vert_neon_10, 10);
+const ConvolveFunctions wrap_convolve12_neon(wrap_convolve8_horiz_neon_12,
+ wrap_convolve8_vert_neon_12, 12);
+const ConvolveParam kArray_HighbdConvolve8_neon[] = {
+ ALL_SIZES_64(wrap_convolve8_neon), ALL_SIZES_64(wrap_convolve10_neon),
+ ALL_SIZES_64(wrap_convolve12_neon)
+};
+
+INSTANTIATE_TEST_SUITE_P(NEON, HighbdConvolveTest,
+ ::testing::ValuesIn(kArray_HighbdConvolve8_neon));
+#endif
const ConvolveFunctions convolve8_neon(aom_convolve8_horiz_neon,
aom_convolve8_vert_neon, 0);
const ConvolveParam kArray_Convolve8_neon[] = { ALL_SIZES(convolve8_neon) };
@@ -874,4 +898,25 @@
::testing::ValuesIn(kArray_Convolve8_neon));
#endif // HAVE_NEON
+#if HAVE_NEON_DOTPROD
+const ConvolveFunctions convolve8_neon_dotprod(aom_convolve8_horiz_neon_dotprod,
+ aom_convolve8_vert_neon_dotprod,
+ 0);
+const ConvolveParam kArray_Convolve8_neon_dotprod[] = { ALL_SIZES(
+ convolve8_neon_dotprod) };
+
+INSTANTIATE_TEST_SUITE_P(NEON_DOTPROD, LowbdConvolveTest,
+ ::testing::ValuesIn(kArray_Convolve8_neon_dotprod));
+#endif // HAVE_NEON_DOTPROD
+
+#if HAVE_NEON_I8MM
+const ConvolveFunctions convolve8_neon_i8mm(aom_convolve8_horiz_neon_i8mm,
+ aom_convolve8_vert_neon_i8mm, 0);
+const ConvolveParam kArray_Convolve8_neon_i8mm[] = { ALL_SIZES(
+ convolve8_neon_i8mm) };
+
+INSTANTIATE_TEST_SUITE_P(NEON_I8MM, LowbdConvolveTest,
+ ::testing::ValuesIn(kArray_Convolve8_neon_i8mm));
+#endif // HAVE_NEON_I8MM
+
} // namespace
diff --git a/test/corner_match_test.cc b/test/corner_match_test.cc
index 93ca8ec..9733732 100644
--- a/test/corner_match_test.cc
+++ b/test/corner_match_test.cc
@@ -37,10 +37,8 @@
class AV1CornerMatchTest : public ::testing::TestWithParam<CornerMatchParam> {
public:
- virtual ~AV1CornerMatchTest();
- virtual void SetUp();
-
- virtual void TearDown();
+ ~AV1CornerMatchTest() override;
+ void SetUp() override;
protected:
void RunCheckOutput(int run_times);
@@ -50,12 +48,11 @@
};
GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(AV1CornerMatchTest);
-AV1CornerMatchTest::~AV1CornerMatchTest() {}
+AV1CornerMatchTest::~AV1CornerMatchTest() = default;
void AV1CornerMatchTest::SetUp() {
rnd_.Reset(ACMRandom::DeterministicSeed());
target_func = GET_PARAM(1);
}
-void AV1CornerMatchTest::TearDown() {}
void AV1CornerMatchTest::RunCheckOutput(int run_times) {
const int w = 128, h = 128;
diff --git a/test/cpu_speed_test.cc b/test/cpu_speed_test.cc
index 5396bec..b5f5d29 100644
--- a/test/cpu_speed_test.cc
+++ b/test/cpu_speed_test.cc
@@ -28,19 +28,19 @@
: EncoderTest(GET_PARAM(0)), encoding_mode_(GET_PARAM(1)),
set_cpu_used_(GET_PARAM(2)), min_psnr_(kMaxPSNR),
tune_content_(AOM_CONTENT_DEFAULT) {}
- virtual ~CpuSpeedTest() {}
+ ~CpuSpeedTest() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(encoding_mode_);
if (encoding_mode_ != ::libaom_test::kRealTime) {
cfg_.g_lag_in_frames = 25;
}
}
- virtual void BeginPassHook(unsigned int /*pass*/) { min_psnr_ = kMaxPSNR; }
+ void BeginPassHook(unsigned int /*pass*/) override { min_psnr_ = kMaxPSNR; }
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AOME_SET_CPUUSED, set_cpu_used_);
encoder->Control(AV1E_SET_TUNE_CONTENT, tune_content_);
@@ -52,7 +52,7 @@
}
}
- virtual void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) {
+ void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) override {
if (pkt->data.psnr.psnr[0] < min_psnr_) min_psnr_ = pkt->data.psnr.psnr[0];
}
diff --git a/test/cpu_used_firstpass_test.cc b/test/cpu_used_firstpass_test.cc
index cfffcd7a..53db8b0 100644
--- a/test/cpu_used_firstpass_test.cc
+++ b/test/cpu_used_firstpass_test.cc
@@ -27,9 +27,9 @@
protected:
CpuUsedFirstpassTest()
: EncoderTest(GET_PARAM(0)), second_pass_cpu_used_(GET_PARAM(2)) {}
- virtual ~CpuUsedFirstpassTest() {}
+ ~CpuUsedFirstpassTest() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(::libaom_test::kTwoPassGood);
const aom_rational timebase = { 1, 30 };
cfg_.g_timebase = timebase;
@@ -40,7 +40,7 @@
init_flags_ = AOM_CODEC_USE_PSNR;
}
- virtual void BeginPassHook(unsigned int pass) {
+ void BeginPassHook(unsigned int pass) override {
psnr_ = 0.0;
nframes_ = 0;
@@ -50,13 +50,13 @@
cpu_used_ = second_pass_cpu_used_;
}
- virtual void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) {
+ void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) override {
psnr_ += pkt->data.psnr.psnr[0];
nframes_++;
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AOME_SET_CPUUSED, cpu_used_);
encoder->Control(AOME_SET_ENABLEAUTOALTREF, 1);
diff --git a/test/datarate_test.cc b/test/datarate_test.cc
index 21b40d9..a75a72f 100644
--- a/test/datarate_test.cc
+++ b/test/datarate_test.cc
@@ -36,9 +36,9 @@
}
protected:
- virtual ~DatarateTestLarge() {}
+ ~DatarateTestLarge() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(GET_PARAM(1));
ResetModel();
}
@@ -298,6 +298,72 @@
<< " The datarate for the file missed the target!"
<< cfg_.rc_target_bitrate << " " << effective_datarate_;
}
+
+ virtual void BasicRateTargetingSuperresCBR() {
+ ::libaom_test::I420VideoSource video("desktopqvga2.320_240.yuv", 320, 240,
+ 30, 1, 0, 800);
+
+ cfg_.g_profile = 0;
+ cfg_.g_timebase = video.timebase();
+
+ cfg_.rc_buf_initial_sz = 500;
+ cfg_.rc_buf_optimal_sz = 500;
+ cfg_.rc_buf_sz = 1000;
+ cfg_.rc_dropframe_thresh = 1;
+ cfg_.rc_min_quantizer = 0;
+ cfg_.rc_max_quantizer = 63;
+ cfg_.rc_end_usage = AOM_CBR;
+
+ cfg_.rc_superres_mode = AOM_SUPERRES_FIXED;
+ cfg_.rc_superres_denominator = 16;
+ cfg_.rc_superres_kf_denominator = 16;
+
+ const int bitrate_array[2] = { 250, 650 };
+ cfg_.rc_target_bitrate = bitrate_array[GET_PARAM(4)];
+ ResetModel();
+ ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
+ ASSERT_GE(static_cast<double>(cfg_.rc_target_bitrate),
+ effective_datarate_ * 0.85)
+ << " The datarate for the file exceeds the target by too much!";
+ ASSERT_LE(static_cast<double>(cfg_.rc_target_bitrate),
+ effective_datarate_ * 1.15)
+ << " The datarate for the file missed the target!"
+ << cfg_.rc_target_bitrate << " " << effective_datarate_;
+ }
+
+ virtual void BasicRateTargetingSuperresCBRMultiThreads() {
+ ::libaom_test::I420VideoSource video("niklas_640_480_30.yuv", 640, 480, 30,
+ 1, 0, 400);
+
+ cfg_.g_profile = 0;
+ cfg_.g_timebase = video.timebase();
+
+ cfg_.rc_buf_initial_sz = 500;
+ cfg_.rc_buf_optimal_sz = 500;
+ cfg_.rc_buf_sz = 1000;
+ cfg_.rc_dropframe_thresh = 1;
+ cfg_.rc_min_quantizer = 0;
+ cfg_.rc_max_quantizer = 63;
+ cfg_.rc_end_usage = AOM_CBR;
+ cfg_.g_threads = 2;
+
+ cfg_.rc_superres_mode = AOM_SUPERRES_FIXED;
+ cfg_.rc_superres_denominator = 16;
+ cfg_.rc_superres_kf_denominator = 16;
+
+ const int bitrate_array[2] = { 250, 650 };
+ cfg_.rc_target_bitrate = bitrate_array[GET_PARAM(4)];
+ ResetModel();
+ tile_column_ = 1;
+ ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
+ ASSERT_GE(static_cast<double>(cfg_.rc_target_bitrate),
+ effective_datarate_ * 0.85)
+ << " The datarate for the file exceeds the target by too much!";
+ ASSERT_LE(static_cast<double>(cfg_.rc_target_bitrate),
+ effective_datarate_ * 1.15)
+ << " The datarate for the file missed the target!"
+ << cfg_.rc_target_bitrate << " " << effective_datarate_;
+ }
};
// Params: test mode, speed, aq mode.
@@ -312,9 +378,9 @@
}
protected:
- virtual ~DatarateTestFrameDropLarge() {}
+ ~DatarateTestFrameDropLarge() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(GET_PARAM(1));
ResetModel();
}
@@ -405,6 +471,16 @@
BasicRateTargeting444CBRScreenTest();
}
+// Check basic rate targeting for Superres mode with CBR.
+TEST_P(DatarateTestLarge, BasicRateTargetingSuperresCBR) {
+ BasicRateTargetingSuperresCBR();
+}
+
+// Check basic rate targeting for Superres mode with CBR and multi-threads.
+TEST_P(DatarateTestLarge, BasicRateTargetingSuperresCBRMultiThreads) {
+ BasicRateTargetingSuperresCBRMultiThreads();
+}
+
// Check that (1) the first dropped frame gets earlier and earlier
// as the drop frame threshold is increased, and (2) that the total number of
// frame drops does not decrease as we increase frame drop threshold.
@@ -433,9 +509,9 @@
}
protected:
- virtual ~DatarateTestSpeedChangeRealtime() {}
+ ~DatarateTestSpeedChangeRealtime() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(GET_PARAM(1));
ResetModel();
}
@@ -521,6 +597,16 @@
BasicRateTargeting444CBRScreenTest();
}
+// Check basic rate targeting for Superres mode with CBR.
+TEST_P(DatarateTestRealtime, BasicRateTargetingSuperresCBR) {
+ BasicRateTargetingSuperresCBR();
+}
+
+// Check basic rate targeting for Superres mode with CBR and multi-threads.
+TEST_P(DatarateTestRealtime, BasicRateTargetingSuperresCBRMultiThreads) {
+ BasicRateTargetingSuperresCBRMultiThreads();
+}
+
// Check that (1) the first dropped frame gets earlier and earlier
// as the drop frame threshold is increased, and (2) that the total number of
// frame drops does not decrease as we increase frame drop threshold.
@@ -540,15 +626,15 @@
DatarateTestSetFrameQpRealtime() : DatarateTest(GetParam()), frame_(0) {}
protected:
- virtual ~DatarateTestSetFrameQpRealtime() {}
+ ~DatarateTestSetFrameQpRealtime() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(libaom_test::kRealTime);
ResetModel();
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
set_cpu_used_ = 7;
DatarateTest::PreEncodeFrameHook(video, encoder);
frame_qp_ = rnd_.PseudoUniform(63);
@@ -556,7 +642,7 @@
frame_++;
}
- virtual void PostEncodeFrameHook(::libaom_test::Encoder *encoder) {
+ void PostEncodeFrameHook(::libaom_test::Encoder *encoder) override {
if (frame_ >= total_frames_) return;
int qp = 0;
encoder->Control(AOME_GET_LAST_QUANTIZER_64, &qp);
diff --git a/test/datarate_test.h b/test/datarate_test.h
index 4b74c65..accc1ad 100644
--- a/test/datarate_test.h
+++ b/test/datarate_test.h
@@ -28,7 +28,7 @@
speed_change_test_(false) {}
protected:
- virtual ~DatarateTest() {}
+ ~DatarateTest() override = default;
virtual void ResetModel() {
last_pts_ = 0;
@@ -57,8 +57,8 @@
}
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AOME_SET_CPUUSED, set_cpu_used_);
encoder->Control(AV1E_SET_AQ_MODE, aq_mode_);
@@ -122,7 +122,7 @@
duration_ = 0;
}
- virtual void FramePktHook(const aom_codec_cx_pkt_t *pkt) {
+ void FramePktHook(const aom_codec_cx_pkt_t *pkt) override {
// Time since last timestamp = duration.
aom_codec_pts_t duration = pkt->data.frame.pts - last_pts_;
@@ -176,7 +176,7 @@
}
}
- virtual void EndPassHook() {
+ void EndPassHook() override {
duration_ = (last_pts_ + 1) * timebase_;
// Effective file datarate:
effective_datarate_ = (bits_total_ / 1000.0) / duration_;
diff --git a/test/decode_multithreaded_test.cc b/test/decode_multithreaded_test.cc
index 5a13f75..4e06f1a 100644
--- a/test/decode_multithreaded_test.cc
+++ b/test/decode_multithreaded_test.cc
@@ -63,16 +63,16 @@
}
}
- virtual ~AV1DecodeMultiThreadedTest() {
+ ~AV1DecodeMultiThreadedTest() override {
delete single_thread_dec_;
for (int i = 0; i < kNumMultiThreadDecoders; ++i)
delete multi_thread_dec_[i];
}
- virtual void SetUp() { InitializeConfig(libaom_test::kTwoPassGood); }
+ void SetUp() override { InitializeConfig(libaom_test::kTwoPassGood); }
- virtual void PreEncodeFrameHook(libaom_test::VideoSource *video,
- libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(libaom_test::VideoSource *video,
+ libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AV1E_SET_TILE_COLUMNS, n_tile_cols_);
encoder->Control(AV1E_SET_TILE_ROWS, n_tile_rows_);
@@ -93,7 +93,7 @@
md5->Add(img);
}
- virtual void FramePktHook(const aom_codec_cx_pkt_t *pkt) {
+ void FramePktHook(const aom_codec_cx_pkt_t *pkt) override {
UpdateMD5(single_thread_dec_, pkt, &md5_single_thread_);
for (int i = 0; i < kNumMultiThreadDecoders; ++i)
diff --git a/test/decode_perf_test.cc b/test/decode_perf_test.cc
index 900cb67..0300354 100644
--- a/test/decode_perf_test.cc
+++ b/test/decode_perf_test.cc
@@ -101,13 +101,12 @@
protected:
AV1NewEncodeDecodePerfTest()
: EncoderTest(GET_PARAM(0)), encoding_mode_(GET_PARAM(1)), speed_(0),
- outfile_(0), out_frames_(0) {}
+ outfile_(nullptr), out_frames_(0) {}
- virtual ~AV1NewEncodeDecodePerfTest() {}
+ ~AV1NewEncodeDecodePerfTest() override = default;
- virtual void SetUp() {
- InitializeConfig();
- SetMode(encoding_mode_);
+ void SetUp() override {
+ InitializeConfig(encoding_mode_);
cfg_.g_lag_in_frames = 25;
cfg_.rc_min_quantizer = 2;
@@ -121,8 +120,8 @@
cfg_.rc_end_usage = AOM_VBR;
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AOME_SET_CPUUSED, speed_);
encoder->Control(AV1E_SET_FRAME_PARALLEL_DECODING, 1);
@@ -130,7 +129,7 @@
}
}
- virtual void BeginPassHook(unsigned int /*pass*/) {
+ void BeginPassHook(unsigned int /*pass*/) override {
const char *const env = getenv("LIBAOM_TEST_DATA_PATH");
const std::string data_path(env ? env : ".");
const std::string path_to_source = data_path + "/" + kNewEncodeOutputFile;
@@ -138,7 +137,7 @@
ASSERT_NE(outfile_, nullptr);
}
- virtual void EndPassHook() {
+ void EndPassHook() override {
if (outfile_ != nullptr) {
if (!fseek(outfile_, 0, SEEK_SET))
ivf_write_file_header(outfile_, &cfg_, AV1_FOURCC, out_frames_);
@@ -147,7 +146,7 @@
}
}
- virtual void FramePktHook(const aom_codec_cx_pkt_t *pkt) {
+ void FramePktHook(const aom_codec_cx_pkt_t *pkt) override {
++out_frames_;
// Write initial file header if first frame.
@@ -160,7 +159,7 @@
pkt->data.frame.sz);
}
- virtual bool DoDecode() const { return false; }
+ bool DoDecode() const override { return false; }
void set_speed(unsigned int speed) { speed_ = speed; }
diff --git a/test/decode_scalability_test.cc b/test/decode_scalability_test.cc
index c04d58b..d66c8ec 100644
--- a/test/decode_scalability_test.cc
+++ b/test/decode_scalability_test.cc
@@ -43,7 +43,7 @@
: DecoderTest(GET_PARAM(0)), headers_(GET_PARAM(1).headers),
num_headers_(GET_PARAM(1).num_headers) {}
- ~DecodeScalabilityTest() override {}
+ ~DecodeScalabilityTest() override = default;
void PreDecodeFrameHook(const libaom_test::CompressedVideoSource &video,
libaom_test::Decoder *decoder) override {
diff --git a/test/decode_test_driver.h b/test/decode_test_driver.h
index 9678f72..311898e 100644
--- a/test/decode_test_driver.h
+++ b/test/decode_test_driver.h
@@ -153,7 +153,7 @@
explicit DecoderTest(const CodecFactory *codec)
: codec_(codec), cfg_(), flags_(0) {}
- virtual ~DecoderTest() {}
+ virtual ~DecoderTest() = default;
const CodecFactory *codec_;
aom_codec_dec_cfg_t cfg_;
diff --git a/test/disflow_test.cc b/test/disflow_test.cc
new file mode 100644
index 0000000..124c9a9
--- /dev/null
+++ b/test/disflow_test.cc
@@ -0,0 +1,122 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include "aom_dsp/flow_estimation/disflow.h"
+
+#include "third_party/googletest/src/googletest/include/gtest/gtest.h"
+
+#include "config/aom_dsp_rtcd.h"
+#include "test/acm_random.h"
+#include "test/register_state_check.h"
+#include "test/util.h"
+#include "test/yuv_video_source.h"
+
+namespace {
+
+using ComputeFlowAtPointFunc = void (*)(const uint8_t *src, const uint8_t *ref,
+ int x, int y, int width, int height,
+ int stride, double *u, double *v);
+
+class ComputeFlowTest
+ : public ::testing::TestWithParam<ComputeFlowAtPointFunc> {
+ public:
+ ComputeFlowTest()
+ : target_func_(GetParam()),
+ rnd_(libaom_test::ACMRandom::DeterministicSeed()) {}
+
+ protected:
+ void RunCheckOutput(int run_times);
+ ComputeFlowAtPointFunc target_func_;
+
+ libaom_test::ACMRandom rnd_;
+};
+GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(ComputeFlowTest);
+
+void ComputeFlowTest::RunCheckOutput(int run_times) {
+ constexpr int kWidth = 352;
+ constexpr int kHeight = 288;
+
+ ::libaom_test::YUVVideoSource video("bus_352x288_420_f20_b8.yuv",
+ AOM_IMG_FMT_I420, kWidth, kHeight, 30, 1,
+ 0, 2);
+ // Use Y (Luminance) plane.
+ video.Begin();
+ uint8_t *src = video.img()->planes[0];
+ ASSERT_NE(src, nullptr);
+ video.Next();
+ uint8_t *ref = video.img()->planes[0];
+ ASSERT_NE(ref, nullptr);
+
+ // Pick a random value between -5 and 5. The range was chosen arbitrarily as
+ // u and v can take any kind of value in practise, but it shouldn't change the
+ // outcome of the tests.
+ const double u_rand = (static_cast<double>(rnd_.Rand8()) / 255) * 10 - 5;
+ double u_ref = u_rand;
+ double u_test = u_rand;
+
+ const double v_rand = (static_cast<double>(rnd_.Rand8()) / 255) * 10 - 5;
+ double v_ref = v_rand;
+ double v_test = v_rand;
+
+ // Pick a random point in the frame. If the frame is 352x288, that means we
+ // can call the function on all values of x comprised between 8 and 344, and
+ // all values of y comprised between 8 and 280.
+ const int x = rnd_((kWidth - 8) - 8 + 1) + 8;
+ const int y = rnd_((kHeight - 8) - 8 + 1) + 8;
+
+ aom_usec_timer ref_timer, test_timer;
+
+ aom_compute_flow_at_point_c(src, ref, x, y, kWidth, kHeight, kWidth, &u_ref,
+ &v_ref);
+
+ target_func_(src, ref, x, y, kWidth, kHeight, kWidth, &u_test, &v_test);
+
+ if (run_times > 1) {
+ aom_usec_timer_start(&ref_timer);
+ for (int i = 0; i < run_times; ++i) {
+ aom_compute_flow_at_point_c(src, ref, x, y, kWidth, kHeight, kWidth,
+ &u_ref, &v_ref);
+ }
+ aom_usec_timer_mark(&ref_timer);
+ const double elapsed_time_c =
+ static_cast<double>(aom_usec_timer_elapsed(&ref_timer));
+
+ aom_usec_timer_start(&test_timer);
+ for (int i = 0; i < run_times; ++i) {
+ target_func_(src, ref, x, y, kWidth, kHeight, kWidth, &u_test, &v_test);
+ }
+ aom_usec_timer_mark(&test_timer);
+ const double elapsed_time_simd =
+ static_cast<double>(aom_usec_timer_elapsed(&test_timer));
+
+ printf("c_time=%fns \t simd_time=%fns \t speedup=%.2f\n", elapsed_time_c,
+ elapsed_time_simd, (elapsed_time_c / elapsed_time_simd));
+ } else {
+ ASSERT_EQ(u_ref, u_test);
+ ASSERT_EQ(v_ref, v_test);
+ }
+}
+
+TEST_P(ComputeFlowTest, CheckOutput) { RunCheckOutput(1); }
+
+TEST_P(ComputeFlowTest, DISABLED_Speed) { RunCheckOutput(10000000); }
+
+#if HAVE_SSE4_1
+INSTANTIATE_TEST_SUITE_P(SSE4_1, ComputeFlowTest,
+ ::testing::Values(aom_compute_flow_at_point_sse4_1));
+#endif
+
+#if HAVE_NEON
+INSTANTIATE_TEST_SUITE_P(NEON, ComputeFlowTest,
+ ::testing::Values(aom_compute_flow_at_point_neon));
+#endif
+
+} // namespace
diff --git a/test/dr_prediction_test.cc b/test/dr_prediction_test.cc
index cbb3153..3135d2a 100644
--- a/test/dr_prediction_test.cc
+++ b/test/dr_prediction_test.cc
@@ -178,7 +178,7 @@
}
}
- virtual ~DrPredTest() {}
+ ~DrPredTest() override = default;
void Predict(bool speedtest, int tx) {
const int kNumTests = speedtest ? kMaxNumTests : 1;
diff --git a/test/dropframe_encode_test.cc b/test/dropframe_encode_test.cc
index c7a801b..4a54c0b 100644
--- a/test/dropframe_encode_test.cc
+++ b/test/dropframe_encode_test.cc
@@ -25,10 +25,10 @@
DropFrameEncodeTestLarge()
: EncoderTest(GET_PARAM(0)), frame_number_(0), threads_(GET_PARAM(2)) {}
- virtual void SetUp() { InitializeConfig(GET_PARAM(1)); }
+ void SetUp() override { InitializeConfig(GET_PARAM(1)); }
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
frame_number_ = video->frame();
if (frame_number_ == 0) {
encoder->Control(AOME_SET_CPUUSED, 1);
diff --git a/test/encode_api_test.cc b/test/encode_api_test.cc
index 2a8c072..e6ef4c2 100644
--- a/test/encode_api_test.cc
+++ b/test/encode_api_test.cc
@@ -24,9 +24,9 @@
namespace {
#if CONFIG_REALTIME_ONLY
-const int kUsage = AOM_USAGE_REALTIME;
+const unsigned int kUsage = AOM_USAGE_REALTIME;
#else
-const int kUsage = AOM_USAGE_GOOD_QUALITY;
+const unsigned int kUsage = AOM_USAGE_GOOD_QUALITY;
#endif
static void *Memset16(void *dest, int val, size_t length) {
@@ -66,6 +66,22 @@
EXPECT_EQ(AOM_CODEC_INVALID_PARAM,
aom_codec_enc_config_default(iface, &cfg, 3));
EXPECT_EQ(AOM_CODEC_OK, aom_codec_enc_config_default(iface, &cfg, kUsage));
+ cfg.g_w = 1 << 16;
+ cfg.g_h = (1 << 14) + 1;
+ EXPECT_EQ(AOM_CODEC_INVALID_PARAM, aom_codec_enc_init(&enc, iface, &cfg, 0));
+ EXPECT_EQ(AOM_CODEC_OK, aom_codec_enc_config_default(iface, &cfg, kUsage));
+ cfg.g_w = (1 << 14) + 1;
+ cfg.g_h = 1 << 16;
+ EXPECT_EQ(AOM_CODEC_INVALID_PARAM, aom_codec_enc_init(&enc, iface, &cfg, 0));
+ EXPECT_EQ(AOM_CODEC_OK, aom_codec_enc_config_default(iface, &cfg, kUsage));
+ cfg.g_forced_max_frame_width = 1 << 16;
+ cfg.g_forced_max_frame_height = (1 << 14) + 1;
+ EXPECT_EQ(AOM_CODEC_INVALID_PARAM, aom_codec_enc_init(&enc, iface, &cfg, 0));
+ EXPECT_EQ(AOM_CODEC_OK, aom_codec_enc_config_default(iface, &cfg, kUsage));
+ cfg.g_forced_max_frame_width = (1 << 14) + 1;
+ cfg.g_forced_max_frame_height = 1 << 16;
+ EXPECT_EQ(AOM_CODEC_INVALID_PARAM, aom_codec_enc_init(&enc, iface, &cfg, 0));
+ EXPECT_EQ(AOM_CODEC_OK, aom_codec_enc_config_default(iface, &cfg, kUsage));
EXPECT_EQ(AOM_CODEC_OK, aom_codec_enc_init(&enc, iface, &cfg, 0));
EXPECT_EQ(nullptr, aom_codec_get_global_headers(nullptr));
@@ -90,13 +106,12 @@
EXPECT_EQ(AOM_CODEC_OK, aom_codec_destroy(&enc));
}
-TEST(EncodeAPI, SetSFrameOnFirstFrame) {
+void EncodeSetSFrameOnFirstFrame(aom_img_fmt fmt, aom_codec_flags_t flag) {
constexpr int kWidth = 2;
constexpr int kHeight = 128;
unsigned char kBuffer[kWidth * kHeight * 3] = { 0 };
aom_image_t img;
- ASSERT_EQ(aom_img_wrap(&img, AOM_IMG_FMT_I420, kWidth, kHeight, 1, kBuffer),
- &img);
+ ASSERT_EQ(aom_img_wrap(&img, fmt, kWidth, kHeight, 1, kBuffer), &img);
aom_codec_iface_t *iface = aom_codec_av1_cx();
aom_codec_enc_cfg_t cfg;
@@ -105,15 +120,25 @@
cfg.g_h = kHeight;
aom_codec_ctx_t enc;
- ASSERT_EQ(aom_codec_enc_init(&enc, iface, &cfg, 0), AOM_CODEC_OK);
+ ASSERT_EQ(aom_codec_enc_init(&enc, iface, &cfg, flag), AOM_CODEC_OK);
// One of these aom_codec_encode() calls should fail.
if (aom_codec_encode(&enc, &img, 0, 1, AOM_EFLAG_SET_S_FRAME) ==
AOM_CODEC_OK) {
- EXPECT_NE(aom_codec_encode(&enc, NULL, 0, 0, 0), AOM_CODEC_OK);
+ EXPECT_NE(aom_codec_encode(&enc, nullptr, 0, 0, 0), AOM_CODEC_OK);
}
EXPECT_EQ(aom_codec_destroy(&enc), AOM_CODEC_OK);
}
+TEST(EncodeAPI, SetSFrameOnFirstFrame) {
+ EncodeSetSFrameOnFirstFrame(AOM_IMG_FMT_I420, 0);
+}
+
+#if CONFIG_AV1_HIGHBITDEPTH
+TEST(EncodeAPI, SetSFrameOnFirstFrameHighbd) {
+ EncodeSetSFrameOnFirstFrame(AOM_IMG_FMT_I42016, AOM_CODEC_USE_HIGHBITDEPTH);
+}
+#endif // CONFIG_AV1_HIGHBITDEPTH
+
TEST(EncodeAPI, MonochromeInProfiles) {
aom_codec_iface_t *iface = aom_codec_av1_cx();
aom_codec_enc_cfg_t cfg;
@@ -147,7 +172,7 @@
ASSERT_EQ(aom_codec_enc_init(&enc, iface, &cfg, 0), AOM_CODEC_OK);
aom_image_t *image =
- aom_img_alloc(NULL, AOM_IMG_FMT_I420, cfg.g_w, cfg.g_h, 0);
+ aom_img_alloc(nullptr, AOM_IMG_FMT_I420, cfg.g_w, cfg.g_h, 0);
ASSERT_NE(image, nullptr);
// Set the image to two colors so that av1_set_screen_content_options() will
@@ -184,7 +209,7 @@
ASSERT_EQ(init_status, AOM_CODEC_OK);
aom_image_t *image =
- aom_img_alloc(NULL, AOM_IMG_FMT_I42016, cfg.g_w, cfg.g_h, 0);
+ aom_img_alloc(nullptr, AOM_IMG_FMT_I42016, cfg.g_w, cfg.g_h, 0);
ASSERT_NE(image, nullptr);
// Set the image to two colors so that av1_set_screen_content_options() will
@@ -222,7 +247,7 @@
ASSERT_EQ(init_status, AOM_CODEC_OK);
aom_image_t *image =
- aom_img_alloc(NULL, AOM_IMG_FMT_I420, cfg.g_w, cfg.g_h, 0);
+ aom_img_alloc(nullptr, AOM_IMG_FMT_I420, cfg.g_w, cfg.g_h, 0);
ASSERT_NE(image, nullptr);
// Set the image to two colors so that av1_set_screen_content_options() will
@@ -255,7 +280,7 @@
ASSERT_EQ(aom_codec_enc_init(&enc, iface, &cfg, 0), AOM_CODEC_OK);
aom_image_t *image =
- aom_img_alloc(NULL, AOM_IMG_FMT_I42016, cfg.g_w, cfg.g_h, 0);
+ aom_img_alloc(nullptr, AOM_IMG_FMT_I42016, cfg.g_w, cfg.g_h, 0);
ASSERT_NE(image, nullptr);
// Set the image to two colors so that av1_set_screen_content_options() will
@@ -279,13 +304,13 @@
}
class EncodeAPIParameterized
- : public testing::TestWithParam<
- std::tuple</*usage=*/int, /*speed=*/int, /*aq_mode=*/int>> {};
+ : public testing::TestWithParam<std::tuple<
+ /*usage=*/unsigned int, /*speed=*/int, /*aq_mode=*/unsigned int>> {};
// Encodes two frames at a given usage, speed, and aq_mode setting.
// Reproduces b/303023614
TEST_P(EncodeAPIParameterized, HighBDEncoderHighBDFrames) {
- const int usage = std::get<0>(GetParam());
+ const unsigned int usage = std::get<0>(GetParam());
int speed = std::get<1>(GetParam());
if (speed == 10 && usage != AOM_USAGE_REALTIME) {
@@ -304,15 +329,15 @@
#if !CONFIG_AV1_HIGHBITDEPTH
ASSERT_EQ(init_status, AOM_CODEC_INCAPABLE);
#else
- const int aq_mode = std::get<2>(GetParam());
-
ASSERT_EQ(init_status, AOM_CODEC_OK);
+ const unsigned int aq_mode = std::get<2>(GetParam());
+
ASSERT_EQ(aom_codec_control(&enc, AOME_SET_CPUUSED, speed), AOM_CODEC_OK);
ASSERT_EQ(aom_codec_control(&enc, AV1E_SET_AQ_MODE, aq_mode), AOM_CODEC_OK);
aom_image_t *image =
- aom_img_alloc(NULL, AOM_IMG_FMT_I42016, cfg.g_w, cfg.g_h, 0);
+ aom_img_alloc(nullptr, AOM_IMG_FMT_I42016, cfg.g_w, cfg.g_h, 0);
ASSERT_NE(image, nullptr);
for (unsigned int i = 0; i < image->d_h; ++i) {
@@ -338,7 +363,7 @@
#endif
}
-const int kUsages[] = {
+const unsigned int kUsages[] = {
AOM_USAGE_REALTIME,
#if !CONFIG_REALTIME_ONLY
AOM_USAGE_GOOD_QUALITY,
diff --git a/test/encode_perf_test.cc b/test/encode_perf_test.cc
index b626acd..b52cf33 100644
--- a/test/encode_perf_test.cc
+++ b/test/encode_perf_test.cc
@@ -63,11 +63,10 @@
: EncoderTest(GET_PARAM(0)), min_psnr_(kMaxPsnr), nframes_(0),
encoding_mode_(GET_PARAM(1)), speed_(0), threads_(1) {}
- virtual ~AV1EncodePerfTest() {}
+ ~AV1EncodePerfTest() override = default;
- virtual void SetUp() {
- InitializeConfig();
- SetMode(encoding_mode_);
+ void SetUp() override {
+ InitializeConfig(encoding_mode_);
cfg_.g_lag_in_frames = 0;
cfg_.rc_min_quantizer = 2;
@@ -83,8 +82,8 @@
cfg_.g_threads = threads_;
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
const int log2_tile_columns = 3;
encoder->Control(AOME_SET_CPUUSED, speed_);
@@ -94,19 +93,19 @@
}
}
- virtual void BeginPassHook(unsigned int /*pass*/) {
+ void BeginPassHook(unsigned int /*pass*/) override {
min_psnr_ = kMaxPsnr;
nframes_ = 0;
}
- virtual void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) {
+ void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) override {
if (pkt->data.psnr.psnr[0] < min_psnr_) {
min_psnr_ = pkt->data.psnr.psnr[0];
}
}
// for performance reasons don't decode
- virtual bool DoDecode() { return 0; }
+ bool DoDecode() const override { return false; }
double min_psnr() const { return min_psnr_; }
diff --git a/test/encode_small_width_height_test.cc b/test/encode_small_width_height_test.cc
index 3d00327..22f6939 100644
--- a/test/encode_small_width_height_test.cc
+++ b/test/encode_small_width_height_test.cc
@@ -25,21 +25,21 @@
namespace {
// Dummy buffer of zero samples.
-constexpr unsigned char kBuffer[256 * 512 + 2 * 128 * 256] = { 0 };
+constexpr unsigned char kBuffer[2 * (256 * 512 + 2 * 128 * 256)] = { 0 };
#if CONFIG_REALTIME_ONLY
const int kUsage = 1;
#else
const int kUsage = 0;
#endif
-TEST(EncodeSmallWidthHeight, SmallWidthMultiThreaded) {
+void EncodeSmallWidthMultiThreaded(aom_img_fmt fmt, aom_codec_flags_t flag) {
// The image has only one tile and the tile is two AV1 superblocks wide.
// For speed >= 1, superblock size is 64x64 (see av1_select_sb_size()).
constexpr int kWidth = 128;
constexpr int kHeight = 512;
aom_image_t img;
- EXPECT_EQ(&img, aom_img_wrap(&img, AOM_IMG_FMT_I420, kWidth, kHeight, 1,
+ EXPECT_EQ(&img, aom_img_wrap(&img, fmt, kWidth, kHeight, 1,
const_cast<unsigned char *>(kBuffer)));
aom_codec_iface_t *iface = aom_codec_av1_cx();
@@ -49,22 +49,33 @@
cfg.g_w = kWidth;
cfg.g_h = kHeight;
aom_codec_ctx_t enc;
- EXPECT_EQ(AOM_CODEC_OK, aom_codec_enc_init(&enc, iface, &cfg, 0));
+ EXPECT_EQ(AOM_CODEC_OK, aom_codec_enc_init(&enc, iface, &cfg, flag));
EXPECT_EQ(AOM_CODEC_OK, aom_codec_control(&enc, AOME_SET_CPUUSED, 5));
EXPECT_EQ(AOM_CODEC_OK, aom_codec_encode(&enc, &img, 0, 1, 0));
EXPECT_EQ(AOM_CODEC_OK, aom_codec_encode(&enc, nullptr, 0, 0, 0));
EXPECT_EQ(AOM_CODEC_OK, aom_codec_destroy(&enc));
}
+TEST(EncodeSmallWidthHeight, SmallWidthMultiThreaded) {
+ EncodeSmallWidthMultiThreaded(AOM_IMG_FMT_I420, 0);
+}
+
+#if CONFIG_AV1_HIGHBITDEPTH
+TEST(HighbdEncodeSmallWidthHeight, SmallWidthMultiThreaded) {
+ EncodeSmallWidthMultiThreaded(AOM_IMG_FMT_I42016, AOM_CODEC_USE_HIGHBITDEPTH);
+}
+#endif // CONFIG_AV1_HIGHBITDEPTH
+
#if !CONFIG_REALTIME_ONLY
-TEST(EncodeSmallWidthHeight, SmallWidthMultiThreadedSpeed0) {
+void EncodeSmallWidthMultiThreadedSpeed0(aom_img_fmt fmt,
+ aom_codec_flags_t flag) {
// The image has only one tile and the tile is two AV1 superblocks wide.
// For speed 0, superblock size is 128x128 (see av1_select_sb_size()).
constexpr int kWidth = 256;
constexpr int kHeight = 512;
aom_image_t img;
- EXPECT_EQ(&img, aom_img_wrap(&img, AOM_IMG_FMT_I420, kWidth, kHeight, 1,
+ EXPECT_EQ(&img, aom_img_wrap(&img, fmt, kWidth, kHeight, 1,
const_cast<unsigned char *>(kBuffer)));
aom_codec_iface_t *iface = aom_codec_av1_cx();
@@ -74,22 +85,34 @@
cfg.g_w = kWidth;
cfg.g_h = kHeight;
aom_codec_ctx_t enc;
- EXPECT_EQ(AOM_CODEC_OK, aom_codec_enc_init(&enc, iface, &cfg, 0));
+ EXPECT_EQ(AOM_CODEC_OK, aom_codec_enc_init(&enc, iface, &cfg, flag));
EXPECT_EQ(AOM_CODEC_OK, aom_codec_control(&enc, AOME_SET_CPUUSED, 0));
EXPECT_EQ(AOM_CODEC_OK, aom_codec_encode(&enc, &img, 0, 1, 0));
EXPECT_EQ(AOM_CODEC_OK, aom_codec_encode(&enc, nullptr, 0, 0, 0));
EXPECT_EQ(AOM_CODEC_OK, aom_codec_destroy(&enc));
}
+
+TEST(EncodeSmallWidthHeight, SmallWidthMultiThreadedSpeed0) {
+ EncodeSmallWidthMultiThreadedSpeed0(AOM_IMG_FMT_I420, 0);
+}
+
+#if CONFIG_AV1_HIGHBITDEPTH
+TEST(HighbdEncodeSmallWidthHeight, SmallWidthMultiThreadedSpeed0) {
+ EncodeSmallWidthMultiThreadedSpeed0(AOM_IMG_FMT_I42016,
+ AOM_CODEC_USE_HIGHBITDEPTH);
+}
+#endif // CONFIG_AV1_HIGHBITDEPTH
+
#endif
-TEST(EncodeSmallWidthHeight, SmallHeightMultiThreaded) {
+void EncodeSmallHeightMultiThreaded(aom_img_fmt fmt, aom_codec_flags_t flag) {
// The image has only one tile and the tile is one AV1 superblock tall.
// For speed >= 1, superblock size is 64x64 (see av1_select_sb_size()).
constexpr int kWidth = 512;
constexpr int kHeight = 64;
aom_image_t img;
- EXPECT_EQ(&img, aom_img_wrap(&img, AOM_IMG_FMT_I420, kWidth, kHeight, 1,
+ EXPECT_EQ(&img, aom_img_wrap(&img, fmt, kWidth, kHeight, 1,
const_cast<unsigned char *>(kBuffer)));
aom_codec_iface_t *iface = aom_codec_av1_cx();
@@ -99,22 +122,34 @@
cfg.g_w = kWidth;
cfg.g_h = kHeight;
aom_codec_ctx_t enc;
- EXPECT_EQ(AOM_CODEC_OK, aom_codec_enc_init(&enc, iface, &cfg, 0));
+ EXPECT_EQ(AOM_CODEC_OK, aom_codec_enc_init(&enc, iface, &cfg, flag));
EXPECT_EQ(AOM_CODEC_OK, aom_codec_control(&enc, AOME_SET_CPUUSED, 5));
EXPECT_EQ(AOM_CODEC_OK, aom_codec_encode(&enc, &img, 0, 1, 0));
EXPECT_EQ(AOM_CODEC_OK, aom_codec_encode(&enc, nullptr, 0, 0, 0));
EXPECT_EQ(AOM_CODEC_OK, aom_codec_destroy(&enc));
}
+TEST(EncodeSmallWidthHeight, SmallHeightMultiThreaded) {
+ EncodeSmallHeightMultiThreaded(AOM_IMG_FMT_I420, 0);
+}
+
+#if CONFIG_AV1_HIGHBITDEPTH
+TEST(HighbdEncodeSmallWidthHeight, SmallHeightMultiThreaded) {
+ EncodeSmallHeightMultiThreaded(AOM_IMG_FMT_I42016,
+ AOM_CODEC_USE_HIGHBITDEPTH);
+}
+#endif // CONFIG_AV1_HIGHBITDEPTH
+
#if !CONFIG_REALTIME_ONLY
-TEST(EncodeSmallWidthHeight, SmallHeightMultiThreadedSpeed0) {
+void EncodeSmallHeightMultiThreadedSpeed0(aom_img_fmt fmt,
+ aom_codec_flags_t flag) {
// The image has only one tile and the tile is one AV1 superblock tall.
// For speed 0, superblock size is 128x128 (see av1_select_sb_size()).
constexpr int kWidth = 512;
constexpr int kHeight = 128;
aom_image_t img;
- EXPECT_EQ(&img, aom_img_wrap(&img, AOM_IMG_FMT_I420, kWidth, kHeight, 1,
+ EXPECT_EQ(&img, aom_img_wrap(&img, fmt, kWidth, kHeight, 1,
const_cast<unsigned char *>(kBuffer)));
aom_codec_iface_t *iface = aom_codec_av1_cx();
@@ -124,17 +159,28 @@
cfg.g_w = kWidth;
cfg.g_h = kHeight;
aom_codec_ctx_t enc;
- EXPECT_EQ(AOM_CODEC_OK, aom_codec_enc_init(&enc, iface, &cfg, 0));
+ EXPECT_EQ(AOM_CODEC_OK, aom_codec_enc_init(&enc, iface, &cfg, flag));
EXPECT_EQ(AOM_CODEC_OK, aom_codec_control(&enc, AOME_SET_CPUUSED, 0));
EXPECT_EQ(AOM_CODEC_OK, aom_codec_encode(&enc, &img, 0, 1, 0));
EXPECT_EQ(AOM_CODEC_OK, aom_codec_encode(&enc, nullptr, 0, 0, 0));
EXPECT_EQ(AOM_CODEC_OK, aom_codec_destroy(&enc));
}
+
+TEST(EncodeSmallWidthHeight, SmallHeightMultiThreadedSpeed0) {
+ EncodeSmallHeightMultiThreadedSpeed0(AOM_IMG_FMT_I420, 0);
+}
+
+#if CONFIG_AV1_HIGHBITDEPTH
+TEST(HighbdEncodeSmallWidthHeight, SmallHeightMultiThreadedSpeed0) {
+ EncodeSmallHeightMultiThreadedSpeed0(AOM_IMG_FMT_I42016,
+ AOM_CODEC_USE_HIGHBITDEPTH);
+}
+#endif // CONFIG_AV1_HIGHBITDEPTH
#endif
// A reproducer test for aomedia:3113. The test should complete without any
// memory errors.
-TEST(EncodeSmallWidthHeight, 1x1) {
+void Encode1x1(aom_img_fmt fmt, int bitdepth, aom_codec_flags_t flags) {
constexpr int kWidth = 1;
constexpr int kHeight = 1;
@@ -144,8 +190,8 @@
// set up img manually.
aom_image_t img;
memset(&img, 0, sizeof(img));
- img.fmt = AOM_IMG_FMT_I420;
- img.bit_depth = 8;
+ img.fmt = fmt;
+ img.bit_depth = bitdepth;
img.w = kWidth;
img.h = kHeight;
img.d_w = kWidth;
@@ -153,10 +199,14 @@
img.x_chroma_shift = 1;
img.y_chroma_shift = 1;
img.bps = 12;
- int y_stride = kWidth;
- int uv_stride = (kWidth + 1) >> 1;
+ const int y_stride = kWidth;
+ const int uv_stride = (kWidth + 1) >> 1;
int y_height = kHeight;
int uv_height = (kHeight + 1) >> 1;
+ if (bitdepth > 8) {
+ y_height <<= 1;
+ uv_height <<= 1;
+ }
img.stride[AOM_PLANE_Y] = y_stride;
img.stride[AOM_PLANE_U] = img.stride[AOM_PLANE_V] = uv_stride;
std::unique_ptr<unsigned char[]> y_plane(
@@ -178,11 +228,19 @@
cfg.g_w = kWidth;
cfg.g_h = kHeight;
aom_codec_ctx_t enc;
- EXPECT_EQ(AOM_CODEC_OK, aom_codec_enc_init(&enc, iface, &cfg, 0));
+ EXPECT_EQ(AOM_CODEC_OK, aom_codec_enc_init(&enc, iface, &cfg, flags));
EXPECT_EQ(AOM_CODEC_OK, aom_codec_control(&enc, AOME_SET_CPUUSED, 5));
EXPECT_EQ(AOM_CODEC_OK, aom_codec_encode(&enc, &img, 0, 1, 0));
EXPECT_EQ(AOM_CODEC_OK, aom_codec_encode(&enc, nullptr, 0, 0, 0));
EXPECT_EQ(AOM_CODEC_OK, aom_codec_destroy(&enc));
}
+TEST(EncodeSmallWidthHeight, 1x1) { Encode1x1(AOM_IMG_FMT_I420, 8, 0); }
+
+#if CONFIG_AV1_HIGHBITDEPTH
+TEST(HighbdEncodeSmallWidthHeight, 1x1) {
+ Encode1x1(AOM_IMG_FMT_I42016, 12, AOM_CODEC_USE_HIGHBITDEPTH);
+}
+#endif // CONFIG_AV1_HIGHBITDEPTH
+
} // namespace
diff --git a/test/encode_test_driver.h b/test/encode_test_driver.h
index 80be8ed..d1e6615 100644
--- a/test/encode_test_driver.h
+++ b/test/encode_test_driver.h
@@ -199,7 +199,7 @@
cfg_.g_threads = 1;
}
- virtual ~EncoderTest() {}
+ virtual ~EncoderTest() = default;
// Initialize the cfg_ member with the default configuration for the
// TestMode enum and maps the TestMode enum to the passes_ variable.
diff --git a/test/encodemb_test.cc b/test/encodemb_test.cc
index 4c725c7..6165fc3 100644
--- a/test/encodemb_test.cc
+++ b/test/encodemb_test.cc
@@ -82,7 +82,7 @@
while (new_eob > 0 && qcoeff_scan[new_eob - 1] == 0) --new_eob;
EXPECT_EQ(new_eob, mb.plane[kPlane].eobs[0]);
- // Check qqcoeff is still valid.
+ // Check dqcoeff is still valid.
for (int i = 0; i < max_eob; ++i) {
EXPECT_EQ(qcoeff[i] * kDequantFactor, dqcoeff[i]);
}
diff --git a/test/encodetxb_test.cc b/test/encodetxb_test.cc
index 0a58737..49b0fba 100644
--- a/test/encodetxb_test.cc
+++ b/test/encodetxb_test.cc
@@ -42,9 +42,9 @@
public:
EncodeTxbTest() : get_nz_map_contexts_func_(GetParam()) {}
- virtual ~EncodeTxbTest() {}
+ ~EncodeTxbTest() override = default;
- virtual void SetUp() {
+ void SetUp() override {
coeff_contexts_ref_ = reinterpret_cast<int8_t *>(
aom_memalign(16, sizeof(*coeff_contexts_ref_) * MAX_TX_SQUARE));
ASSERT_NE(coeff_contexts_ref_, nullptr);
@@ -53,7 +53,7 @@
ASSERT_NE(coeff_contexts_, nullptr);
}
- virtual void TearDown() {
+ void TearDown() override {
aom_free(coeff_contexts_ref_);
aom_free(coeff_contexts_);
}
@@ -211,8 +211,7 @@
class EncodeTxbInitLevelTest
: public ::testing::TestWithParam<TxbInitLevelParam> {
public:
- virtual ~EncodeTxbInitLevelTest() {}
- virtual void TearDown() {}
+ ~EncodeTxbInitLevelTest() override = default;
void RunTest(av1_txb_init_levels_func test_func, int tx_size, int is_speed);
};
GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(EncodeTxbInitLevelTest);
diff --git a/test/end_to_end_psnr_test.cc b/test/end_to_end_psnr_test.cc
index 0396438..687308d 100644
--- a/test/end_to_end_psnr_test.cc
+++ b/test/end_to_end_psnr_test.cc
@@ -86,9 +86,9 @@
cpu_used_(GET_PARAM(3)), psnr_(0.0), nframes_(0),
encoding_mode_(GET_PARAM(1)) {}
- virtual ~EndToEndTest() {}
+ ~EndToEndTest() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(encoding_mode_);
if (encoding_mode_ == ::libaom_test::kOnePassGood ||
encoding_mode_ == ::libaom_test::kTwoPassGood) {
@@ -100,18 +100,18 @@
}
}
- virtual void BeginPassHook(unsigned int) {
+ void BeginPassHook(unsigned int) override {
psnr_ = 0.0;
nframes_ = 0;
}
- virtual void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) {
+ void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) override {
psnr_ += pkt->data.psnr.psnr[0];
nframes_++;
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AV1E_SET_FRAME_PARALLEL_DECODING, 1);
encoder->Control(AV1E_SET_TILE_COLUMNS, 4);
diff --git a/test/end_to_end_qmpsnr_test.cc b/test/end_to_end_qmpsnr_test.cc
index de183ad..7a755a7 100644
--- a/test/end_to_end_qmpsnr_test.cc
+++ b/test/end_to_end_qmpsnr_test.cc
@@ -69,7 +69,7 @@
test_video_param_(GET_PARAM(2)), cpu_used_(GET_PARAM(3)), nframes_(0),
ssim_(0.0) {}
- ~EndToEndQMPSNRTest() override {}
+ ~EndToEndQMPSNRTest() override = default;
void SetUp() override { InitializeConfig(encoding_mode_); }
diff --git a/test/end_to_end_ssim_test.cc b/test/end_to_end_ssim_test.cc
index 2e40c94..f1b0cae 100644
--- a/test/end_to_end_ssim_test.cc
+++ b/test/end_to_end_ssim_test.cc
@@ -66,7 +66,7 @@
test_video_param_(GET_PARAM(2)), cpu_used_(GET_PARAM(3)), nframes_(0),
ssim_(0.0) {}
- ~EndToEndSSIMTest() override {}
+ ~EndToEndSSIMTest() override = default;
void SetUp() override { InitializeConfig(encoding_mode_); }
diff --git a/test/error_block_test.cc b/test/error_block_test.cc
index aadbb44..176efdf 100644
--- a/test/error_block_test.cc
+++ b/test/error_block_test.cc
@@ -65,15 +65,13 @@
class ErrorBlockTest : public ::testing::TestWithParam<ErrorBlockParam> {
public:
- virtual ~ErrorBlockTest() {}
- virtual void SetUp() {
+ ~ErrorBlockTest() override = default;
+ void SetUp() override {
error_block_op_ = GET_PARAM(0);
ref_error_block_op_ = GET_PARAM(1);
bit_depth_ = GET_PARAM(2);
}
- virtual void TearDown() {}
-
protected:
aom_bit_depth_t bit_depth_;
ErrorBlockFunc error_block_op_;
@@ -289,6 +287,14 @@
#if (HAVE_NEON)
const ErrorBlockParam kErrorBlockTestParamsNeon[] = {
+#if CONFIG_AV1_HIGHBITDEPTH
+ make_tuple(&av1_highbd_block_error_neon, &av1_highbd_block_error_c,
+ AOM_BITS_10),
+ make_tuple(&av1_highbd_block_error_neon, &av1_highbd_block_error_c,
+ AOM_BITS_12),
+ make_tuple(&av1_highbd_block_error_neon, &av1_highbd_block_error_c,
+ AOM_BITS_8),
+#endif
make_tuple(&BlockError8BitWrapper<av1_block_error_neon>,
&BlockError8BitWrapper<av1_block_error_c>, AOM_BITS_8),
make_tuple(&BlockErrorLpWrapper<av1_block_error_lp_neon>,
diff --git a/test/error_resilience_test.cc b/test/error_resilience_test.cc
index 84330d6..d41884d 100644
--- a/test/error_resilience_test.cc
+++ b/test/error_resilience_test.cc
@@ -37,7 +37,7 @@
Reset();
}
- virtual ~ErrorResilienceTestLarge() {}
+ ~ErrorResilienceTestLarge() override = default;
void Reset() {
error_nframes_ = 0;
@@ -58,9 +58,9 @@
init_flags_ = AOM_CODEC_USE_PSNR;
}
- virtual void SetUp() { InitializeConfig(encoding_mode_); }
+ void SetUp() override { InitializeConfig(encoding_mode_); }
- virtual void BeginPassHook(unsigned int /*pass*/) {
+ void BeginPassHook(unsigned int /*pass*/) override {
psnr_ = 0.0;
nframes_ = 0;
decoded_nframes_ = 0;
@@ -68,13 +68,13 @@
mismatch_nframes_ = 0;
}
- virtual void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) {
+ void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) override {
psnr_ += pkt->data.psnr.psnr[0];
nframes_++;
}
- virtual void PreEncodeFrameHook(libaom_test::VideoSource *video,
- libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(libaom_test::VideoSource *video,
+ libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AOME_SET_CPUUSED, kCpuUsed);
encoder->Control(AOME_SET_ENABLEAUTOALTREF, enable_altref_);
@@ -146,7 +146,7 @@
}
}
- virtual void FramePktHook(const aom_codec_cx_pkt_t *pkt) {
+ void FramePktHook(const aom_codec_cx_pkt_t *pkt) override {
// Check that the encode frame flags are correctly reflected
// in the output frame flags.
const int encode_flags = pkt->data.frame.flags >> 16;
@@ -176,21 +176,21 @@
return 0.0;
}
- virtual bool DoDecode() const {
+ bool DoDecode() const override {
if (error_nframes_ > 0 &&
(cfg_.g_pass == AOM_RC_LAST_PASS || cfg_.g_pass == AOM_RC_ONE_PASS)) {
for (unsigned int i = 0; i < error_nframes_; ++i) {
if (error_frames_[i] == nframes_ - 1) {
std::cout << " Skipping decoding frame: "
<< error_frames_[i] << "\n";
- return 0;
+ return false;
}
}
}
- return 1;
+ return true;
}
- virtual bool DoDecodeInvisible() const {
+ bool DoDecodeInvisible() const override {
if (invisible_error_nframes_ > 0 &&
(cfg_.g_pass == AOM_RC_LAST_PASS || cfg_.g_pass == AOM_RC_ONE_PASS)) {
for (unsigned int i = 0; i < invisible_error_nframes_; ++i) {
@@ -198,14 +198,14 @@
std::cout << " Skipping decoding all invisible frames in "
"frame pkt: "
<< invisible_error_frames_[i] << "\n";
- return 0;
+ return false;
}
}
}
- return 1;
+ return true;
}
- virtual void MismatchHook(const aom_image_t *img1, const aom_image_t *img2) {
+ void MismatchHook(const aom_image_t *img1, const aom_image_t *img2) override {
if (allow_mismatch_) {
double mismatch_psnr = compute_psnr(img1, img2);
mismatch_psnr_ += mismatch_psnr;
@@ -216,8 +216,8 @@
}
}
- virtual void DecompressedFrameHook(const aom_image_t &img,
- aom_codec_pts_t pts) {
+ void DecompressedFrameHook(const aom_image_t &img,
+ aom_codec_pts_t pts) override {
(void)img;
(void)pts;
++decoded_nframes_;
diff --git a/test/ethread_test.cc b/test/ethread_test.cc
index 6b7fcce..ce45394 100644
--- a/test/ethread_test.cc
+++ b/test/ethread_test.cc
@@ -40,9 +40,9 @@
firstpass_stats_.buf = nullptr;
firstpass_stats_.sz = 0;
}
- virtual ~AVxFirstPassEncoderThreadTest() { free(firstpass_stats_.buf); }
+ ~AVxFirstPassEncoderThreadTest() override { free(firstpass_stats_.buf); }
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(encoding_mode_);
cfg_.g_lag_in_frames = 35;
@@ -53,18 +53,18 @@
cfg_.rc_min_quantizer = 0;
}
- virtual void BeginPassHook(unsigned int /*pass*/) {
+ void BeginPassHook(unsigned int /*pass*/) override {
encoder_initialized_ = false;
abort_ = false;
}
- virtual void EndPassHook() {
+ void EndPassHook() override {
// For first pass stats test, only run first pass encoder.
if (cfg_.g_pass == AOM_RC_FIRST_PASS) abort_ = true;
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource * /*video*/,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource * /*video*/,
+ ::libaom_test::Encoder *encoder) override {
if (!encoder_initialized_) {
// Encode in 2-pass mode.
SetTileSize(encoder);
@@ -84,7 +84,7 @@
encoder->Control(AV1E_SET_TILE_ROWS, tile_rows_);
}
- virtual void StatsPktHook(const aom_codec_cx_pkt_t *pkt) {
+ void StatsPktHook(const aom_codec_cx_pkt_t *pkt) override {
const uint8_t *const pkt_buf =
reinterpret_cast<uint8_t *>(pkt->data.twopass_stats.buf);
const size_t pkt_size = pkt->data.twopass_stats.sz;
@@ -227,9 +227,9 @@
md5_dec_.clear();
md5_enc_.clear();
}
- virtual ~AVxEncoderThreadTest() { delete decoder_; }
+ ~AVxEncoderThreadTest() override { delete decoder_; }
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(encoding_mode_);
if (encoding_mode_ == ::libaom_test::kOnePassGood ||
@@ -244,12 +244,12 @@
cfg_.rc_min_quantizer = 0;
}
- virtual void BeginPassHook(unsigned int /*pass*/) {
+ void BeginPassHook(unsigned int /*pass*/) override {
encoder_initialized_ = false;
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource * /*video*/,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource * /*video*/,
+ ::libaom_test::Encoder *encoder) override {
if (!encoder_initialized_) {
SetTileSize(encoder);
encoder->Control(AOME_SET_CPUUSED, set_cpu_used_);
@@ -290,7 +290,7 @@
encoder->Control(AV1E_SET_TILE_ROWS, tile_rows_);
}
- virtual void FramePktHook(const aom_codec_cx_pkt_t *pkt) {
+ void FramePktHook(const aom_codec_cx_pkt_t *pkt) override {
size_enc_.push_back(pkt->data.frame.sz);
::libaom_test::MD5 md5_enc;
@@ -531,15 +531,15 @@
#endif // !CONFIG_REALTIME_ONLY
class AVxEncoderThreadLSTest : public AVxEncoderThreadTest {
- virtual void SetTileSize(libaom_test::Encoder *encoder) {
+ void SetTileSize(libaom_test::Encoder *encoder) override {
encoder->Control(AV1E_SET_TILE_COLUMNS, tile_cols_);
encoder->Control(AV1E_SET_TILE_ROWS, tile_rows_);
}
- virtual void DoTestMaxThreads(::libaom_test::YUVVideoSource *video,
- const std::vector<size_t> ref_size_enc,
- const std::vector<std::string> ref_md5_enc,
- const std::vector<std::string> ref_md5_dec) {
+ void DoTestMaxThreads(::libaom_test::YUVVideoSource *video,
+ const std::vector<size_t> ref_size_enc,
+ const std::vector<std::string> ref_md5_enc,
+ const std::vector<std::string> ref_md5_dec) override {
(void)video;
(void)ref_size_enc;
(void)ref_md5_enc;
diff --git a/test/external_frame_buffer_test.cc b/test/external_frame_buffer_test.cc
index ea7ed50..8f16c4e 100644
--- a/test/external_frame_buffer_test.cc
+++ b/test/external_frame_buffer_test.cc
@@ -214,13 +214,12 @@
: DecoderTest(GET_PARAM(::libaom_test::kCodecFactoryParam)),
md5_file_(nullptr), num_buffers_(0) {}
- virtual ~ExternalFrameBufferMD5Test() {
+ ~ExternalFrameBufferMD5Test() override {
if (md5_file_ != nullptr) fclose(md5_file_);
}
- virtual void PreDecodeFrameHook(
- const libaom_test::CompressedVideoSource &video,
- libaom_test::Decoder *decoder) {
+ void PreDecodeFrameHook(const libaom_test::CompressedVideoSource &video,
+ libaom_test::Decoder *decoder) override {
if (num_buffers_ > 0 && video.frame_number() == 0) {
// Have libaom use frame buffers we create.
ASSERT_TRUE(fb_list_.CreateBufferList(num_buffers_));
@@ -236,8 +235,8 @@
<< "Md5 file open failed. Filename: " << md5_file_name_;
}
- virtual void DecompressedFrameHook(const aom_image_t &img,
- const unsigned int frame_number) {
+ void DecompressedFrameHook(const aom_image_t &img,
+ const unsigned int frame_number) override {
ASSERT_NE(md5_file_, nullptr);
char expected_md5[33];
char junk[128];
@@ -315,7 +314,7 @@
ExternalFrameBufferTest()
: video_(nullptr), decoder_(nullptr), num_buffers_(0) {}
- virtual void SetUp() {
+ void SetUp() override {
video_ = new libaom_test::WebMVideoSource(kAV1TestFile);
ASSERT_NE(video_, nullptr);
video_->Init();
@@ -327,7 +326,7 @@
ASSERT_NE(decoder_, nullptr);
}
- virtual void TearDown() {
+ void TearDown() override {
delete decoder_;
decoder_ = nullptr;
delete video_;
@@ -383,7 +382,7 @@
class ExternalFrameBufferNonRefTest : public ExternalFrameBufferTest {
protected:
- virtual void SetUp() {
+ void SetUp() override {
video_ = new libaom_test::IVFVideoSource(kAV1NonRefTestFile);
ASSERT_NE(video_, nullptr);
video_->Init();
diff --git a/test/fdct4x4_test.cc b/test/fdct4x4_test.cc
index 046d810..9cbf208 100644
--- a/test/fdct4x4_test.cc
+++ b/test/fdct4x4_test.cc
@@ -59,10 +59,10 @@
class Trans4x4FDCT : public libaom_test::TransformTestBase<OutputType>,
public ::testing::TestWithParam<Fdct4x4Param<OutputType>> {
public:
- virtual ~Trans4x4FDCT() {}
+ ~Trans4x4FDCT() override = default;
using TxfmBaseOutType = libaom_test::TransformTestBase<OutputType>;
- virtual void SetUp() {
+ void SetUp() override {
fwd_txfm_ = std::get<0>(this->GetParam());
TxfmBaseOutType::pitch_ = 4;
TxfmBaseOutType::height_ = 4;
@@ -71,14 +71,13 @@
TxfmBaseOutType::mask_ = (1 << TxfmBaseOutType::bit_depth_) - 1;
TxfmBaseOutType::num_coeffs_ = std::get<3>(this->GetParam());
}
- virtual void TearDown() {}
protected:
- void RunFwdTxfm(const int16_t *in, OutputType *out, int stride) {
+ void RunFwdTxfm(const int16_t *in, OutputType *out, int stride) override {
fwd_txfm_(in, out, stride);
}
- void RunInvTxfm(const OutputType *out, uint8_t *dst, int stride) {
+ void RunInvTxfm(const OutputType *out, uint8_t *dst, int stride) override {
(void)out;
(void)dst;
(void)stride;
diff --git a/test/fft_test.cc b/test/fft_test.cc
index 5443c99..06a17a3 100644
--- a/test/fft_test.cc
+++ b/test/fft_test.cc
@@ -88,7 +88,7 @@
class FFT2DTest : public ::testing::TestWithParam<FFTTestArg> {
protected:
- void SetUp() {
+ void SetUp() override {
int n = GetParam().n;
input_ = (float *)aom_memalign(32, sizeof(*input_) * n * n);
temp_ = (float *)aom_memalign(32, sizeof(*temp_) * n * n);
@@ -100,7 +100,7 @@
memset(temp_, 0, sizeof(*temp_) * n * n);
memset(output_, 0, sizeof(*output_) * n * n * 2);
}
- void TearDown() {
+ void TearDown() override {
aom_free(input_);
aom_free(temp_);
aom_free(output_);
@@ -178,7 +178,7 @@
class IFFT2DTest : public ::testing::TestWithParam<IFFTTestArg> {
protected:
- void SetUp() {
+ void SetUp() override {
int n = GetParam().n;
input_ = (float *)aom_memalign(32, sizeof(*input_) * n * n * 2);
temp_ = (float *)aom_memalign(32, sizeof(*temp_) * n * n * 2);
@@ -190,7 +190,7 @@
memset(temp_, 0, sizeof(*temp_) * n * n * 2);
memset(output_, 0, sizeof(*output_) * n * n);
}
- void TearDown() {
+ void TearDown() override {
aom_free(input_);
aom_free(temp_);
aom_free(output_);
diff --git a/test/film_grain_table_test.cc b/test/film_grain_table_test.cc
index f8937f1..808d966 100644
--- a/test/film_grain_table_test.cc
+++ b/test/film_grain_table_test.cc
@@ -91,7 +91,7 @@
// Extend the existing segment
aom_film_grain_table_append(&table, 2000, 3000, film_grain_test_vectors + 0);
- EXPECT_EQ(0, table.head->next);
+ EXPECT_EQ(nullptr, table.head->next);
// Lookup and remove and check that the entry is no longer there
EXPECT_TRUE(aom_film_grain_table_lookup(&table, 1000, 2000, true, &grain));
@@ -100,8 +100,8 @@
EXPECT_TRUE(aom_film_grain_table_lookup(&table, 2000, 3000, true, &grain));
EXPECT_FALSE(aom_film_grain_table_lookup(&table, 2000, 3000, false, &grain));
- EXPECT_EQ(0, table.head);
- EXPECT_EQ(0, table.tail);
+ EXPECT_EQ(nullptr, table.head);
+ EXPECT_EQ(nullptr, table.tail);
aom_film_grain_table_free(&table);
}
@@ -114,8 +114,8 @@
aom_film_grain_table_append(&table, 0, 1000, film_grain_test_vectors + 0);
EXPECT_TRUE(aom_film_grain_table_lookup(&table, 0, 1100, true, &grain));
- EXPECT_EQ(0, table.head);
- EXPECT_EQ(0, table.tail);
+ EXPECT_EQ(nullptr, table.head);
+ EXPECT_EQ(nullptr, table.tail);
aom_film_grain_table_free(&table);
}
@@ -180,7 +180,7 @@
class FilmGrainTableIOTest : public ::testing::Test {
protected:
- void SetUp() { memset(&error_, 0, sizeof(error_)); }
+ void SetUp() override { memset(&error_, 0, sizeof(error_)); }
struct aom_internal_error_info error_;
};
@@ -280,32 +280,35 @@
};
class FilmGrainEncodeTest
- : public ::libaom_test::CodecTestWith2Params<bool, ::libaom_test::TestMode>,
+ : public ::libaom_test::CodecTestWith3Params<int, int,
+ ::libaom_test::TestMode>,
public ::libaom_test::EncoderTest {
protected:
FilmGrainEncodeTest()
: EncoderTest(GET_PARAM(0)), test_monochrome_(GET_PARAM(1)),
- test_mode_(GET_PARAM(2)) {}
+ key_frame_dist_(GET_PARAM(2)), test_mode_(GET_PARAM(3)) {}
~FilmGrainEncodeTest() override = default;
void SetUp() override {
InitializeConfig(test_mode_);
- cfg_.monochrome = test_monochrome_;
+ cfg_.monochrome = test_monochrome_ == 1;
cfg_.rc_target_bitrate = 300;
- cfg_.kf_max_dist = 0;
+ cfg_.kf_max_dist = key_frame_dist_;
+ cfg_.g_lag_in_frames = 0;
}
void PreEncodeFrameHook(::libaom_test::VideoSource *video,
::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
- encoder->Control(AOME_SET_CPUUSED, 5);
+ encoder->Control(AOME_SET_CPUUSED,
+ test_mode_ == ::libaom_test::kRealTime ? 7 : 5);
encoder->Control(AV1E_SET_TUNE_CONTENT, AOM_CONTENT_FILM);
encoder->Control(AV1E_SET_DENOISE_NOISE_LEVEL, 1);
} else if (video->frame() == 1) {
- cfg_.monochrome = 0;
+ cfg_.monochrome = (test_monochrome_ == 1 || test_monochrome_ == 2);
encoder->Config(&cfg_);
} else {
- cfg_.monochrome = test_monochrome_;
+ cfg_.monochrome = test_monochrome_ == 1;
encoder->Config(&cfg_);
}
}
@@ -313,11 +316,6 @@
bool DoDecode() const override { return false; }
void DoTest() {
- if (test_monochrome_ && test_mode_ == ::libaom_test::kRealTime) {
- // TODO(bohanli): Running real time mode with monochrome will cause the
- // encoder to crash. Check if this is intended or there is a bug.
- GTEST_SKIP();
- }
::libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352,
288, 30, 1, 0, 3);
cfg_.g_w = video.img()->d_w;
@@ -326,11 +324,58 @@
}
private:
- bool test_monochrome_;
+ // 0: monochroome always off.
+ // 1: monochrome always on.
+ // 2: monochrome changes from 0, 1, 0, for encoded frames 0, 1, 2.
+ // The case where monochrome changes from 1 to 0 (i.e., encoder initialized
+ // with monochrome = 1 and then subsequently encoded with monochrome = 0)
+ // will fail. The test InitMonochrome1_EncodeMonochrome0 below verifies this.
+ int test_monochrome_;
+ int key_frame_dist_;
::libaom_test::TestMode test_mode_;
};
TEST_P(FilmGrainEncodeTest, Test) { DoTest(); }
-AV1_INSTANTIATE_TEST_SUITE(FilmGrainEncodeTest, ::testing::Bool(),
+AV1_INSTANTIATE_TEST_SUITE(FilmGrainEncodeTest, ::testing::Range(0, 3),
+ ::testing::Values(0, 10),
::testing::ValuesIn(kFilmGrainEncodeTestModes));
+
+// Initialize encoder with monochrome = 1, and then encode frame with
+// monochrome = 0. This will result in an error: see the following check
+// in encoder_set_config() in av1/av1_cx_iface.c.
+// TODO(marpan): Consider moving this test to another file, as the failure
+// has nothing to do with film grain mode.
+TEST(FilmGrainEncodeTest, InitMonochrome1EncodeMonochrome0) {
+ const int kWidth = 352;
+ const int kHeight = 288;
+ const int usage = AOM_USAGE_REALTIME;
+ aom_codec_iface_t *iface = aom_codec_av1_cx();
+ aom_codec_enc_cfg_t cfg;
+ ASSERT_EQ(aom_codec_enc_config_default(iface, &cfg, usage), AOM_CODEC_OK);
+ aom_codec_ctx_t enc;
+ cfg.g_w = kWidth;
+ cfg.g_h = kHeight;
+ // Initialize encoder, with monochrome = 0.
+ cfg.monochrome = 1;
+ aom_codec_err_t init_status = aom_codec_enc_init(&enc, iface, &cfg, 0);
+ ASSERT_EQ(init_status, AOM_CODEC_OK);
+ ASSERT_EQ(aom_codec_control(&enc, AOME_SET_CPUUSED, 7), AOM_CODEC_OK);
+ ASSERT_EQ(aom_codec_control(&enc, AV1E_SET_TUNE_CONTENT, AOM_CONTENT_FILM),
+ AOM_CODEC_OK);
+ ASSERT_EQ(aom_codec_control(&enc, AV1E_SET_DENOISE_NOISE_LEVEL, 1),
+ AOM_CODEC_OK);
+ // Set image with zero values.
+ constexpr size_t kBufferSize =
+ kWidth * kHeight + 2 * (kWidth + 1) / 2 * (kHeight + 1) / 2;
+ std::vector<unsigned char> buffer(kBufferSize);
+ aom_image_t img;
+ EXPECT_EQ(&img, aom_img_wrap(&img, AOM_IMG_FMT_I420, kWidth, kHeight, 1,
+ buffer.data()));
+ // Encode first frame.
+ ASSERT_EQ(aom_codec_encode(&enc, &img, 0, 1, 0), AOM_CODEC_OK);
+ // Second frame: update config with monochrome = 1.
+ cfg.monochrome = 0;
+ ASSERT_EQ(aom_codec_enc_config_set(&enc, &cfg), AOM_CODEC_INVALID_PARAM);
+ ASSERT_EQ(aom_codec_destroy(&enc), AOM_CODEC_OK);
+}
diff --git a/test/filterintra_test.cc b/test/filterintra_test.cc
index c54bec5..0a0ab11 100644
--- a/test/filterintra_test.cc
+++ b/test/filterintra_test.cc
@@ -41,8 +41,8 @@
class AV1FilterIntraPredTest : public ::testing::TestWithParam<PredParams> {
public:
- virtual ~AV1FilterIntraPredTest() {}
- virtual void SetUp() {
+ ~AV1FilterIntraPredTest() override = default;
+ void SetUp() override {
PredFuncMode funcMode = GET_PARAM(0);
predFuncRef_ = std::get<0>(funcMode);
predFunc_ = std::get<1>(funcMode);
@@ -57,7 +57,7 @@
ASSERT_NE(pred_, nullptr);
}
- virtual void TearDown() {
+ void TearDown() override {
delete[] alloc_;
delete[] predRef_;
delete[] pred_;
diff --git a/test/forced_max_frame_width_height_test.cc b/test/forced_max_frame_width_height_test.cc
index 2e019b6..3347713 100644
--- a/test/forced_max_frame_width_height_test.cc
+++ b/test/forced_max_frame_width_height_test.cc
@@ -114,10 +114,7 @@
}
}
-// A test that reproduces bug aomedia:3348: Assertion
-// `ms_params->ms_buffers.ref->stride == ms_params->search_sites->stride'
-// failed.
-TEST(EncodeForcedMaxFrameWidthHeight, DISABLED_DimensionDecreasing) {
+TEST(EncodeForcedMaxFrameWidthHeight, DimensionDecreasing) {
constexpr int kWidth = 128;
constexpr int kHeight = 128;
constexpr size_t kBufferSize = 3 * kWidth * kHeight;
diff --git a/test/frame_error_test.cc b/test/frame_error_test.cc
deleted file mode 100644
index c355efc..0000000
--- a/test/frame_error_test.cc
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Copyright (c) 2019, Alliance for Open Media. All rights reserved
- *
- * This source code is subject to the terms of the BSD 2 Clause License and
- * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
- * was not distributed with this source code in the LICENSE file, you can
- * obtain it at www.aomedia.org/license/software. If the Alliance for Open
- * Media Patent License 1.0 was not distributed with this source code in the
- * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
- */
-
-#include <math.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <tuple>
-
-#include "config/av1_rtcd.h"
-
-#include "aom_mem/aom_mem.h"
-#include "aom_ports/aom_timer.h"
-#include "aom_ports/mem.h"
-#include "test/acm_random.h"
-#include "test/util.h"
-#include "third_party/googletest/src/googletest/include/gtest/gtest.h"
-
-namespace {
-typedef int64_t (*frame_error_func)(const uint8_t *const ref, int stride,
- const uint8_t *const dst, int p_width,
- int p_height, int p_stride);
-#if HAVE_AVX2 || HAVE_SSE2
-const int kBlockWidth[] = {
- 832, 834, 640, 1280, 1920,
-};
-const int kBlockHeight[] = {
- 480, 482, 360, 720, 1080,
-};
-#endif
-typedef std::tuple<frame_error_func, int, int> FrameErrorParam;
-
-class AV1FrameErrorTest : public ::testing::TestWithParam<FrameErrorParam> {
- public:
- virtual ~AV1FrameErrorTest() {}
- virtual void SetUp() {
- rnd_.Reset(libaom_test::ACMRandom::DeterministicSeed());
- }
- virtual void TearDown() {}
-
- protected:
- void RandomValues(frame_error_func test_impl, int width, int height);
- void ExtremeValues(frame_error_func test_impl, int width, int height);
- void RunSpeedTest(frame_error_func test_impl, int width, int height);
- libaom_test::ACMRandom rnd_;
-};
-GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(AV1FrameErrorTest);
-
-void AV1FrameErrorTest::RandomValues(frame_error_func test_impl, int width,
- int height) {
- const int stride = (((width * 3) / 2) + 15) & ~15;
- const int max_blk_size = stride * height;
- uint8_t *const dst =
- static_cast<uint8_t *>(aom_memalign(16, max_blk_size * sizeof(*dst)));
- uint8_t *const ref =
- static_cast<uint8_t *>(aom_memalign(16, max_blk_size * sizeof(*ref)));
- ASSERT_NE(dst, nullptr);
- ASSERT_NE(ref, nullptr);
- for (int i = 0; i < max_blk_size; ++i) {
- dst[i] = rnd_.Rand8();
- ref[i] = rnd_.Rand8();
- }
- const int64_t ref_error =
- av1_calc_frame_error_c(ref, stride, dst, width, height, stride);
- const int64_t test_error = test_impl(ref, stride, dst, width, height, stride);
- ASSERT_EQ(test_error, ref_error) << width << "x" << height;
- aom_free(dst);
- aom_free(ref);
-}
-
-void AV1FrameErrorTest::ExtremeValues(frame_error_func test_impl, int width,
- int height) {
- const int stride = (((width * 3) / 2) + 15) & ~15;
- const int max_blk_size = stride * height;
- uint8_t *const dst =
- static_cast<uint8_t *>(aom_memalign(16, max_blk_size * sizeof(*dst)));
- uint8_t *const ref =
- static_cast<uint8_t *>(aom_memalign(16, max_blk_size * sizeof(*ref)));
- ASSERT_NE(dst, nullptr);
- ASSERT_NE(ref, nullptr);
- for (int r = 0; r < 2; r++) {
- if (r == 0) {
- memset(dst, 0, max_blk_size);
- memset(ref, 255, max_blk_size);
- } else if (r == 1) {
- memset(dst, 255, max_blk_size);
- memset(ref, 0, max_blk_size);
- }
- const int64_t ref_error =
- av1_calc_frame_error_c(ref, stride, dst, width, height, stride);
- const int64_t test_error =
- test_impl(ref, stride, dst, width, height, stride);
- ASSERT_EQ(test_error, ref_error) << width << "x" << height;
- }
- aom_free(dst);
- aom_free(ref);
-}
-
-void AV1FrameErrorTest::RunSpeedTest(frame_error_func test_impl, int width,
- int height) {
- const int stride = (((width * 3) / 2) + 15) & ~15;
- const int max_blk_size = stride * height;
- uint8_t *const dst =
- static_cast<uint8_t *>(aom_memalign(16, max_blk_size * sizeof(*dst)));
- uint8_t *const ref =
- static_cast<uint8_t *>(aom_memalign(16, max_blk_size * sizeof(*ref)));
- ASSERT_NE(dst, nullptr);
- ASSERT_NE(ref, nullptr);
- for (int i = 0; i < max_blk_size; ++i) {
- dst[i] = ref[i] = rnd_.Rand8();
- }
- const int num_loops = 10000000 / (width + height);
- frame_error_func funcs[2] = { av1_calc_frame_error_c, test_impl };
- double elapsed_time[2] = { 0 };
- for (int i = 0; i < 2; ++i) {
- aom_usec_timer timer;
- aom_usec_timer_start(&timer);
- frame_error_func func = funcs[i];
- for (int j = 0; j < num_loops; ++j) {
- func(ref, stride, dst, width, height, stride);
- }
- aom_usec_timer_mark(&timer);
- double time = static_cast<double>(aom_usec_timer_elapsed(&timer));
- elapsed_time[i] = 1000.0 * time / num_loops;
- }
- aom_free(dst);
- aom_free(ref);
- printf("av1_calc_frame_error %3dx%-3d: %7.2f/%7.2fns", width, height,
- elapsed_time[0], elapsed_time[1]);
- printf("(%3.2f)\n", elapsed_time[0] / elapsed_time[1]);
-}
-
-TEST_P(AV1FrameErrorTest, CheckOutput) {
- RandomValues(GET_PARAM(0), GET_PARAM(1), GET_PARAM(2));
- ExtremeValues(GET_PARAM(0), GET_PARAM(1), GET_PARAM(2));
-}
-
-TEST_P(AV1FrameErrorTest, DISABLED_Speed) {
- RunSpeedTest(GET_PARAM(0), GET_PARAM(1), GET_PARAM(2));
-}
-
-#if HAVE_SSE2
-INSTANTIATE_TEST_SUITE_P(
- SSE2, AV1FrameErrorTest,
- ::testing::Combine(::testing::Values(&av1_calc_frame_error_sse2),
- ::testing::ValuesIn(kBlockWidth),
- ::testing::ValuesIn(kBlockHeight)));
-#endif
-
-#if HAVE_AVX2
-INSTANTIATE_TEST_SUITE_P(
- AVX2, AV1FrameErrorTest,
- ::testing::Combine(::testing::Values(&av1_calc_frame_error_avx2),
- ::testing::ValuesIn(kBlockWidth),
- ::testing::ValuesIn(kBlockHeight)));
-#endif
-} // namespace
diff --git a/test/frame_parallel_enc_test.cc b/test/frame_parallel_enc_test.cc
index 7508eb7..86d5ddb 100644
--- a/test/frame_parallel_enc_test.cc
+++ b/test/frame_parallel_enc_test.cc
@@ -36,9 +36,9 @@
cfg.allow_lowbitdepth = 1;
decoder_ = codec_->CreateDecoder(cfg, 0);
}
- virtual ~AVxFrameParallelThreadEncodeTest() { delete decoder_; }
+ ~AVxFrameParallelThreadEncodeTest() override { delete decoder_; }
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(::libaom_test::kTwoPassGood);
cfg_.rc_end_usage = AOM_VBR;
cfg_.g_lag_in_frames = 35;
@@ -49,12 +49,12 @@
cfg_.g_threads = 16;
}
- virtual void BeginPassHook(unsigned int /*pass*/) {
+ void BeginPassHook(unsigned int /*pass*/) override {
encoder_initialized_ = false;
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource * /*video*/,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource * /*video*/,
+ ::libaom_test::Encoder *encoder) override {
if (encoder_initialized_) return;
SetTileSize(encoder);
encoder->Control(AOME_SET_CPUUSED, set_cpu_used_);
@@ -73,7 +73,7 @@
encoder->Control(AV1E_SET_TILE_ROWS, tile_rows_);
}
- virtual void FramePktHook(const aom_codec_cx_pkt_t *pkt) {
+ void FramePktHook(const aom_codec_cx_pkt_t *pkt) override {
size_enc_.push_back(pkt->data.frame.sz);
::libaom_test::MD5 md5_enc;
diff --git a/test/frame_size_tests.cc b/test/frame_size_tests.cc
index 3b35db8..ea8cf47 100644
--- a/test/frame_size_tests.cc
+++ b/test/frame_size_tests.cc
@@ -24,18 +24,18 @@
protected:
AV1FrameSizeTests()
: EncoderTest(&::libaom_test::kAV1), expected_res_(AOM_CODEC_OK) {}
- virtual ~AV1FrameSizeTests() {}
+ ~AV1FrameSizeTests() override = default;
- virtual void SetUp() { InitializeConfig(::libaom_test::kRealTime); }
+ void SetUp() override { InitializeConfig(::libaom_test::kRealTime); }
- virtual bool HandleDecodeResult(const aom_codec_err_t res_dec,
- libaom_test::Decoder *decoder) {
+ bool HandleDecodeResult(const aom_codec_err_t res_dec,
+ libaom_test::Decoder *decoder) override {
EXPECT_EQ(expected_res_, res_dec) << decoder->DecodeError();
return !::testing::Test::HasFailure();
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AOME_SET_CPUUSED, 7);
encoder->Control(AOME_SET_ENABLEAUTOALTREF, 1);
@@ -312,15 +312,13 @@
::testing::Range(6, 11)));
#if !CONFIG_REALTIME_ONLY
-// TODO(https://crbug.com/aomedia/3348): Modes that use av1_full_pixel_search()
-// will cause an assert.
INSTANTIATE_TEST_SUITE_P(
- DISABLED_GoodQuality, AV1ResolutionChange,
+ GoodQuality, AV1ResolutionChange,
::testing::Combine(::testing::Values(AOM_USAGE_GOOD_QUALITY),
::testing::Values(AOM_VBR, AOM_CBR, AOM_CQ, AOM_Q),
::testing::Range(2, 6)));
INSTANTIATE_TEST_SUITE_P(
- DISABLED_GoodQualityLarge, AV1ResolutionChange,
+ GoodQualityLarge, AV1ResolutionChange,
::testing::Combine(::testing::Values(AOM_USAGE_GOOD_QUALITY),
::testing::Values(AOM_VBR, AOM_CBR, AOM_CQ, AOM_Q),
::testing::Range(0, 2)));
@@ -350,18 +348,18 @@
AV1LosslessFrameSizeTests()
: EncoderTest(GET_PARAM(0)), frame_size_param_(GET_PARAM(1)),
encoding_mode_(GET_PARAM(2)) {}
- virtual ~AV1LosslessFrameSizeTests() {}
+ ~AV1LosslessFrameSizeTests() override = default;
- virtual void SetUp() { InitializeConfig(encoding_mode_); }
+ void SetUp() override { InitializeConfig(encoding_mode_); }
- virtual bool HandleDecodeResult(const aom_codec_err_t res_dec,
- libaom_test::Decoder *decoder) {
+ bool HandleDecodeResult(const aom_codec_err_t res_dec,
+ libaom_test::Decoder *decoder) override {
EXPECT_EQ(expected_res_, res_dec) << decoder->DecodeError();
return !::testing::Test::HasFailure();
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AOME_SET_CPUUSED, 6);
encoder->Control(AV1E_SET_LOSSLESS, 1);
diff --git a/test/function_equivalence_test.h b/test/function_equivalence_test.h
index fc2a769..2268b9f 100644
--- a/test/function_equivalence_test.h
+++ b/test/function_equivalence_test.h
@@ -55,11 +55,9 @@
public:
FunctionEquivalenceTest() : rng_(ACMRandom::DeterministicSeed()) {}
- virtual ~FunctionEquivalenceTest() {}
+ ~FunctionEquivalenceTest() override = default;
- virtual void SetUp() { params_ = this->GetParam(); }
-
- virtual void TearDown() {}
+ void SetUp() override { params_ = this->GetParam(); }
protected:
ACMRandom rng_;
diff --git a/test/fwht4x4_test.cc b/test/fwht4x4_test.cc
index 9d27db8..bb9e218 100644
--- a/test/fwht4x4_test.cc
+++ b/test/fwht4x4_test.cc
@@ -67,9 +67,9 @@
class Trans4x4WHT : public libaom_test::TransformTestBase<tran_low_t>,
public ::testing::TestWithParam<Dct4x4Param> {
public:
- virtual ~Trans4x4WHT() {}
+ ~Trans4x4WHT() override = default;
- virtual void SetUp() {
+ void SetUp() override {
fwd_txfm_ = GET_PARAM(0);
inv_txfm_ = GET_PARAM(1);
pitch_ = 4;
@@ -80,13 +80,12 @@
num_coeffs_ = GET_PARAM(4);
fwd_txfm_c_ = GET_PARAM(5);
}
- virtual void TearDown() {}
protected:
- void RunFwdTxfm(const int16_t *in, tran_low_t *out, int stride) {
+ void RunFwdTxfm(const int16_t *in, tran_low_t *out, int stride) override {
fwd_txfm_(in, out, stride);
}
- void RunInvTxfm(const tran_low_t *out, uint8_t *dst, int stride) {
+ void RunInvTxfm(const tran_low_t *out, uint8_t *dst, int stride) override {
inv_txfm_(out, dst, stride);
}
void RunSpeedTest() {
diff --git a/test/gf_pyr_height_test.cc b/test/gf_pyr_height_test.cc
index a2d1a8f..0996d80 100644
--- a/test/gf_pyr_height_test.cc
+++ b/test/gf_pyr_height_test.cc
@@ -79,9 +79,9 @@
gf_max_pyr_height_ = GET_PARAM(3).gf_max_pyr_height;
psnr_threshold_ = GET_PARAM(3).psnr_thresh;
}
- virtual ~GFPyrHeightTest() {}
+ ~GFPyrHeightTest() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(encoding_mode_);
const aom_rational timebase = { 1, 30 };
cfg_.g_timebase = timebase;
@@ -95,18 +95,18 @@
init_flags_ = AOM_CODEC_USE_PSNR;
}
- virtual void BeginPassHook(unsigned int) {
+ void BeginPassHook(unsigned int) override {
psnr_ = 0.0;
nframes_ = 0;
}
- virtual void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) {
+ void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) override {
psnr_ += pkt->data.psnr.psnr[0];
nframes_++;
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AOME_SET_CPUUSED, cpu_used_);
if (rc_mode_ == AOM_Q) {
diff --git a/test/hadamard_test.cc b/test/hadamard_test.cc
index fc306e6..b01e78f 100644
--- a/test/hadamard_test.cc
+++ b/test/hadamard_test.cc
@@ -240,7 +240,7 @@
shift_ = do_shift;
}
- virtual void SetUp() { rnd_.Reset(ACMRandom::DeterministicSeed()); }
+ void SetUp() override { rnd_.Reset(ACMRandom::DeterministicSeed()); }
// The Rand() function generates values in the range [-((1 << BitDepth) - 1),
// (1 << BitDepth) - 1]. This is because the input to the Hadamard transform
@@ -252,7 +252,7 @@
void CompareReferenceRandom() {
const int kMaxBlockSize = 32 * 32;
- const int block_size_ = bw_ * bh_;
+ const int block_size = bw_ * bh_;
DECLARE_ALIGNED(16, int16_t, a[kMaxBlockSize]);
DECLARE_ALIGNED(16, OutputType, b[kMaxBlockSize]);
@@ -262,13 +262,13 @@
OutputType b_ref[kMaxBlockSize];
memset(b_ref, 0, sizeof(b_ref));
- for (int i = 0; i < block_size_; ++i) a[i] = Rand();
+ for (int i = 0; i < block_size; ++i) a[i] = Rand();
ReferenceHadamard(a, bw_, b_ref, bw_, bh_, shift_);
API_REGISTER_STATE_CHECK(h_func_(a, bw_, b));
// The order of the output is not important. Sort before checking.
- std::sort(b, b + block_size_);
- std::sort(b_ref, b_ref + block_size_);
+ std::sort(b, b + block_size);
+ std::sort(b_ref, b_ref + block_size);
EXPECT_EQ(memcmp(b, b_ref, sizeof(b)), 0);
}
@@ -298,12 +298,12 @@
void VaryStride() {
const int kMaxBlockSize = 32 * 32;
- const int block_size_ = bw_ * bh_;
+ const int block_size = bw_ * bh_;
DECLARE_ALIGNED(16, int16_t, a[kMaxBlockSize * 8]);
DECLARE_ALIGNED(16, OutputType, b[kMaxBlockSize]);
memset(a, 0, sizeof(a));
- for (int i = 0; i < block_size_ * 8; ++i) a[i] = Rand();
+ for (int i = 0; i < block_size * 8; ++i) a[i] = Rand();
OutputType b_ref[kMaxBlockSize];
for (int i = 8; i < 64; i += 8) {
@@ -314,8 +314,8 @@
API_REGISTER_STATE_CHECK(h_func_(a, i, b));
// The order of the output is not important. Sort before checking.
- std::sort(b, b + block_size_);
- std::sort(b_ref, b_ref + block_size_);
+ std::sort(b, b + block_size);
+ std::sort(b_ref, b_ref + block_size);
EXPECT_EQ(0, memcmp(b, b_ref, sizeof(b)));
}
}
@@ -338,6 +338,7 @@
printf("Hadamard%dx%d[%12d runs]: %d us\n", bw_, bh_, times, elapsed_time);
}
+ protected:
ACMRandom rnd_;
private:
@@ -351,7 +352,7 @@
public:
HadamardLowbdTest() : HadamardTestBase(GetParam(), /*do_shift=*/true) {}
// Use values between -255 (0xFF01) and 255 (0x00FF)
- virtual int16_t Rand() {
+ int16_t Rand() override {
int16_t src = rnd_.Rand8();
int16_t pred = rnd_.Rand8();
return src - pred;
@@ -407,7 +408,7 @@
protected:
HadamardHighbdTest() : HadamardTestBase(GetParam(), /*do_shift=*/true) {}
// Use values between -4095 (0xF001) and 4095 (0x0FFF)
- virtual int16_t Rand() {
+ int16_t Rand() override {
int16_t src = rnd_.Rand12();
int16_t pred = rnd_.Rand12();
return src - pred;
@@ -431,6 +432,15 @@
HadamardFuncWithSize(&aom_highbd_hadamard_16x16_c, 16, 16),
HadamardFuncWithSize(&aom_highbd_hadamard_32x32_c, 32, 32)));
+#if HAVE_AVX2
+INSTANTIATE_TEST_SUITE_P(
+ AVX2, HadamardHighbdTest,
+ ::testing::Values(
+ HadamardFuncWithSize(&aom_highbd_hadamard_8x8_avx2, 8, 8),
+ HadamardFuncWithSize(&aom_highbd_hadamard_16x16_avx2, 16, 16),
+ HadamardFuncWithSize(&aom_highbd_hadamard_32x32_avx2, 32, 32)));
+#endif // HAVE_AVX2
+
#if HAVE_NEON
INSTANTIATE_TEST_SUITE_P(
NEON, HadamardHighbdTest,
@@ -447,7 +457,7 @@
public:
HadamardLowbdLPTest() : HadamardTestBase(GetParam(), /*do_shift=*/false) {}
// Use values between -255 (0xFF01) and 255 (0x00FF)
- virtual int16_t Rand() {
+ int16_t Rand() override {
int16_t src = rnd_.Rand8();
int16_t pred = rnd_.Rand8();
return src - pred;
@@ -497,7 +507,7 @@
HadamardLowbdLP8x8DualTest()
: HadamardTestBase(GetParam(), /*do_shift=*/false) {}
// Use values between -255 (0xFF01) and 255 (0x00FF)
- virtual int16_t Rand() {
+ int16_t Rand() override {
int16_t src = rnd_.Rand8();
int16_t pred = rnd_.Rand8();
return src - pred;
diff --git a/test/hash_test.cc b/test/hash_test.cc
index 61e0b51..a1de932 100644
--- a/test/hash_test.cc
+++ b/test/hash_test.cc
@@ -31,10 +31,10 @@
class AV1Crc32cHashTest : public ::testing::TestWithParam<HashParam> {
public:
- ~AV1Crc32cHashTest();
- void SetUp();
+ ~AV1Crc32cHashTest() override;
+ void SetUp() override;
- void TearDown();
+ void TearDown() override;
protected:
void RunCheckOutput(get_crc32c_value_func test_impl);
@@ -49,7 +49,7 @@
size_t length_;
};
-AV1Crc32cHashTest::~AV1Crc32cHashTest() {}
+AV1Crc32cHashTest::~AV1Crc32cHashTest() = default;
void AV1Crc32cHashTest::SetUp() {
rnd_.Reset(libaom_test::ACMRandom::DeterministicSeed());
diff --git a/test/hbd_metrics_test.cc b/test/hbd_metrics_test.cc
index 074213a..303d580 100644
--- a/test/hbd_metrics_test.cc
+++ b/test/hbd_metrics_test.cc
@@ -94,7 +94,7 @@
class HBDMetricsTestBase {
public:
- virtual ~HBDMetricsTestBase() {}
+ virtual ~HBDMetricsTestBase() = default;
protected:
void RunAccuracyCheck() {
@@ -179,14 +179,13 @@
class HBDMetricsTest : public HBDMetricsTestBase,
public ::testing::TestWithParam<MetricTestTParam> {
public:
- virtual void SetUp() {
+ void SetUp() override {
lbd_metric_ = GET_PARAM(0);
hbd_metric_ = GET_PARAM(1);
input_bit_depth_ = GET_PARAM(2);
bit_depth_ = GET_PARAM(3);
threshold_ = GET_PARAM(4);
}
- virtual void TearDown() {}
};
TEST_P(HBDMetricsTest, RunAccuracyCheck) { RunAccuracyCheck(); }
diff --git a/test/hiprec_convolve_test.cc b/test/hiprec_convolve_test.cc
index 3e93a06..78883cc 100644
--- a/test/hiprec_convolve_test.cc
+++ b/test/hiprec_convolve_test.cc
@@ -47,7 +47,7 @@
#endif
#if CONFIG_AV1_HIGHBITDEPTH
-#if HAVE_SSSE3 || HAVE_AVX2
+#if HAVE_SSSE3 || HAVE_AVX2 || HAVE_NEON
TEST_P(AV1HighbdHiprecConvolveTest, CheckOutput) {
RunCheckOutput(GET_PARAM(4));
}
@@ -64,6 +64,12 @@
libaom_test::AV1HighbdHiprecConvolve::BuildParams(
av1_highbd_wiener_convolve_add_src_avx2));
#endif
+
+#if HAVE_NEON
+INSTANTIATE_TEST_SUITE_P(NEON, AV1HighbdHiprecConvolveTest,
+ libaom_test::AV1HighbdHiprecConvolve::BuildParams(
+ av1_highbd_wiener_convolve_add_src_neon));
+#endif
#endif
#endif // CONFIG_AV1_HIGHBITDEPTH
diff --git a/test/hiprec_convolve_test_util.cc b/test/hiprec_convolve_test_util.cc
index e2496b3..6d7902f 100644
--- a/test/hiprec_convolve_test_util.cc
+++ b/test/hiprec_convolve_test_util.cc
@@ -26,21 +26,21 @@
static void generate_kernels(ACMRandom *rnd, InterpKernel hkernel,
InterpKernel vkernel, int kernel_type = 2) {
if (kernel_type == 0) {
- // Low possible values for filter coefficients
+ // Low possible values for filter coefficients, 7-tap kernel
hkernel[0] = hkernel[6] = vkernel[0] = vkernel[6] = WIENER_FILT_TAP0_MINV;
hkernel[1] = hkernel[5] = vkernel[1] = vkernel[5] = WIENER_FILT_TAP1_MINV;
hkernel[2] = hkernel[4] = vkernel[2] = vkernel[4] = WIENER_FILT_TAP2_MINV;
hkernel[3] = vkernel[3] = -2 * (hkernel[0] + hkernel[1] + hkernel[2]);
hkernel[7] = vkernel[7] = 0;
} else if (kernel_type == 1) {
- // Max possible values for filter coefficients
+ // Max possible values for filter coefficients, 7-tap kernel
hkernel[0] = hkernel[6] = vkernel[0] = vkernel[6] = WIENER_FILT_TAP0_MAXV;
hkernel[1] = hkernel[5] = vkernel[1] = vkernel[5] = WIENER_FILT_TAP1_MAXV;
hkernel[2] = hkernel[4] = vkernel[2] = vkernel[4] = WIENER_FILT_TAP2_MAXV;
hkernel[3] = vkernel[3] = -2 * (hkernel[0] + hkernel[1] + hkernel[2]);
hkernel[7] = vkernel[7] = 0;
- } else {
- // Randomly generated values for filter coefficients
+ } else if (kernel_type == 2) {
+ // Randomly generated values for filter coefficients, 7-tap kernel
hkernel[0] = hkernel[6] =
WIENER_FILT_TAP0_MINV +
rnd->PseudoUniform(WIENER_FILT_TAP0_MAXV + 1 - WIENER_FILT_TAP0_MINV);
@@ -64,6 +64,41 @@
rnd->PseudoUniform(WIENER_FILT_TAP2_MAXV + 2 - WIENER_FILT_TAP2_MINV);
vkernel[3] = -2 * (vkernel[0] + vkernel[1] + vkernel[2]);
vkernel[7] = 0;
+ } else if (kernel_type == 3) {
+ // Low possible values for filter coefficients, 5-tap kernel
+ hkernel[0] = hkernel[6] = vkernel[0] = vkernel[6] = 0;
+ hkernel[1] = hkernel[5] = vkernel[1] = vkernel[5] = WIENER_FILT_TAP1_MINV;
+ hkernel[2] = hkernel[4] = vkernel[2] = vkernel[4] = WIENER_FILT_TAP2_MINV;
+ hkernel[3] = vkernel[3] = -2 * (hkernel[0] + hkernel[1] + hkernel[2]);
+ hkernel[7] = vkernel[7] = 0;
+ } else if (kernel_type == 4) {
+ // Max possible values for filter coefficients, 5-tap kernel
+ hkernel[0] = hkernel[6] = vkernel[0] = vkernel[6] = 0;
+ hkernel[1] = hkernel[5] = vkernel[1] = vkernel[5] = WIENER_FILT_TAP1_MAXV;
+ hkernel[2] = hkernel[4] = vkernel[2] = vkernel[4] = WIENER_FILT_TAP2_MAXV;
+ hkernel[3] = vkernel[3] = -2 * (hkernel[0] + hkernel[1] + hkernel[2]);
+ hkernel[7] = vkernel[7] = 0;
+ } else {
+ // Randomly generated values for filter coefficients, 5-tap kernel
+ hkernel[0] = hkernel[6] = 0;
+ hkernel[1] = hkernel[5] =
+ WIENER_FILT_TAP1_MINV +
+ rnd->PseudoUniform(WIENER_FILT_TAP1_MAXV + 1 - WIENER_FILT_TAP1_MINV);
+ hkernel[2] = hkernel[4] =
+ WIENER_FILT_TAP2_MINV +
+ rnd->PseudoUniform(WIENER_FILT_TAP2_MAXV + 1 - WIENER_FILT_TAP2_MINV);
+ hkernel[3] = -2 * (hkernel[0] + hkernel[1] + hkernel[2]);
+ hkernel[7] = 0;
+
+ vkernel[0] = vkernel[6] = 0;
+ vkernel[1] = vkernel[5] =
+ WIENER_FILT_TAP1_MINV +
+ rnd->PseudoUniform(WIENER_FILT_TAP1_MAXV + 2 - WIENER_FILT_TAP1_MINV);
+ vkernel[2] = vkernel[4] =
+ WIENER_FILT_TAP2_MINV +
+ rnd->PseudoUniform(WIENER_FILT_TAP2_MAXV + 2 - WIENER_FILT_TAP2_MINV);
+ vkernel[3] = -2 * (vkernel[0] + vkernel[1] + vkernel[2]);
+ vkernel[7] = 0;
}
}
@@ -83,19 +118,17 @@
return ::testing::ValuesIn(params);
}
-AV1HiprecConvolveTest::~AV1HiprecConvolveTest() {}
+AV1HiprecConvolveTest::~AV1HiprecConvolveTest() = default;
void AV1HiprecConvolveTest::SetUp() {
rnd_.Reset(ACMRandom::DeterministicSeed());
}
-void AV1HiprecConvolveTest::TearDown() {}
-
void AV1HiprecConvolveTest::RunCheckOutput(hiprec_convolve_func test_impl) {
const int w = 128, h = 128;
const int out_w = GET_PARAM(0), out_h = GET_PARAM(1);
const int num_iters = GET_PARAM(2);
int i, j, k, m;
- const ConvolveParams conv_params = get_conv_params_wiener(8);
+ const WienerConvolveParams conv_params = get_conv_params_wiener(8);
std::unique_ptr<uint8_t[]> input_(new (std::nothrow) uint8_t[h * w]);
ASSERT_NE(input_, nullptr);
@@ -114,7 +147,7 @@
DECLARE_ALIGNED(16, InterpKernel, hkernel);
DECLARE_ALIGNED(16, InterpKernel, vkernel);
- for (int kernel_type = 0; kernel_type < 3; kernel_type++) {
+ for (int kernel_type = 0; kernel_type < 6; kernel_type++) {
generate_kernels(&rnd_, hkernel, vkernel, kernel_type);
for (i = 0; i < num_iters; ++i) {
for (k = 0; k < h; ++k)
@@ -141,7 +174,7 @@
const int out_w = GET_PARAM(0), out_h = GET_PARAM(1);
const int num_iters = GET_PARAM(2) / 500;
int i, j, k;
- const ConvolveParams conv_params = get_conv_params_wiener(8);
+ const WienerConvolveParams conv_params = get_conv_params_wiener(8);
std::unique_ptr<uint8_t[]> input_(new (std::nothrow) uint8_t[h * w]);
ASSERT_NE(input_, nullptr);
@@ -217,13 +250,11 @@
return ::testing::ValuesIn(params);
}
-AV1HighbdHiprecConvolveTest::~AV1HighbdHiprecConvolveTest() {}
+AV1HighbdHiprecConvolveTest::~AV1HighbdHiprecConvolveTest() = default;
void AV1HighbdHiprecConvolveTest::SetUp() {
rnd_.Reset(ACMRandom::DeterministicSeed());
}
-void AV1HighbdHiprecConvolveTest::TearDown() {}
-
void AV1HighbdHiprecConvolveTest::RunCheckOutput(
highbd_hiprec_convolve_func test_impl) {
const int w = 128, h = 128;
@@ -231,7 +262,7 @@
const int num_iters = GET_PARAM(2);
const int bd = GET_PARAM(3);
int i, j;
- const ConvolveParams conv_params = get_conv_params_wiener(bd);
+ const WienerConvolveParams conv_params = get_conv_params_wiener(bd);
std::unique_ptr<uint16_t[]> input(new (std::nothrow) uint16_t[h * w]);
ASSERT_NE(input, nullptr);
@@ -255,7 +286,7 @@
uint8_t *input_ptr = CONVERT_TO_BYTEPTR(input.get());
uint8_t *output_ptr = CONVERT_TO_BYTEPTR(output.get());
uint8_t *output2_ptr = CONVERT_TO_BYTEPTR(output2.get());
- for (int kernel_type = 0; kernel_type < 3; kernel_type++) {
+ for (int kernel_type = 0; kernel_type < 6; kernel_type++) {
generate_kernels(&rnd_, hkernel, vkernel, kernel_type);
for (i = 0; i < num_iters; ++i) {
// Choose random locations within the source block
@@ -282,7 +313,7 @@
const int num_iters = GET_PARAM(2) / 500;
const int bd = GET_PARAM(3);
int i, j, k;
- const ConvolveParams conv_params = get_conv_params_wiener(bd);
+ const WienerConvolveParams conv_params = get_conv_params_wiener(bd);
std::unique_ptr<uint16_t[]> input(new (std::nothrow) uint16_t[h * w]);
ASSERT_NE(input, nullptr);
diff --git a/test/hiprec_convolve_test_util.h b/test/hiprec_convolve_test_util.h
index e064ba6..beae5c7 100644
--- a/test/hiprec_convolve_test_util.h
+++ b/test/hiprec_convolve_test_util.h
@@ -34,7 +34,7 @@
const int16_t *filter_x, int x_step_q4,
const int16_t *filter_y, int y_step_q4,
int w, int h,
- const ConvolveParams *conv_params);
+ const WienerConvolveParams *conv_params);
typedef std::tuple<int, int, int, hiprec_convolve_func> HiprecConvolveParam;
@@ -44,10 +44,8 @@
class AV1HiprecConvolveTest
: public ::testing::TestWithParam<HiprecConvolveParam> {
public:
- virtual ~AV1HiprecConvolveTest();
- virtual void SetUp();
-
- virtual void TearDown();
+ ~AV1HiprecConvolveTest() override;
+ void SetUp() override;
protected:
void RunCheckOutput(hiprec_convolve_func test_impl);
@@ -64,7 +62,7 @@
const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst,
ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4,
const int16_t *filter_y, int y_step_q4, int w, int h,
- const ConvolveParams *conv_params, int bps);
+ const WienerConvolveParams *conv_params, int bps);
typedef std::tuple<int, int, int, int, highbd_hiprec_convolve_func>
HighbdHiprecConvolveParam;
@@ -75,10 +73,8 @@
class AV1HighbdHiprecConvolveTest
: public ::testing::TestWithParam<HighbdHiprecConvolveParam> {
public:
- virtual ~AV1HighbdHiprecConvolveTest();
- virtual void SetUp();
-
- virtual void TearDown();
+ ~AV1HighbdHiprecConvolveTest() override;
+ void SetUp() override;
protected:
void RunCheckOutput(highbd_hiprec_convolve_func test_impl);
diff --git a/test/horver_correlation_test.cc b/test/horver_correlation_test.cc
index 2873490..5e397ff 100644
--- a/test/horver_correlation_test.cc
+++ b/test/horver_correlation_test.cc
@@ -33,12 +33,12 @@
class HorverTest : public ::testing::TestWithParam<HorverTestParam> {
public:
- virtual void SetUp() {
+ void SetUp() override {
data_buf_ = (int16_t *)aom_malloc(MAX_SB_SQUARE * sizeof(int16_t));
ASSERT_NE(data_buf_, nullptr);
target_func_ = GET_PARAM(0);
}
- virtual void TearDown() { aom_free(data_buf_); }
+ void TearDown() override { aom_free(data_buf_); }
void RunHorverTest();
void RunHorverTest_ExtremeValues();
void RunHorverSpeedTest(int run_times);
diff --git a/test/horz_superres_test.cc b/test/horz_superres_test.cc
index cba29e9..595ed54 100644
--- a/test/horz_superres_test.cc
+++ b/test/horz_superres_test.cc
@@ -100,9 +100,9 @@
: EncoderTest(GET_PARAM(0)), test_video_param_(GET_PARAM(1)),
superres_mode_(GET_PARAM(2)), psnr_(0.0), frame_count_(0) {}
- virtual ~HorzSuperresEndToEndTest() {}
+ ~HorzSuperresEndToEndTest() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(::libaom_test::kTwoPassGood);
cfg_.g_lag_in_frames = 5;
cfg_.rc_end_usage = AOM_Q;
@@ -118,18 +118,18 @@
cfg_.rc_superres_mode = superres_mode_;
}
- virtual void BeginPassHook(unsigned int) {
+ void BeginPassHook(unsigned int) override {
psnr_ = 0.0;
frame_count_ = 0;
}
- virtual void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) {
+ void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) override {
psnr_ += pkt->data.psnr.psnr[0];
frame_count_++;
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AV1E_SET_FRAME_PARALLEL_DECODING, 1);
encoder->Control(AV1E_SET_TILE_COLUMNS, 4);
@@ -203,9 +203,9 @@
superres_kf_denom_ = std::get<1>(denoms);
}
- virtual ~HorzSuperresFixedEndToEndTest() {}
+ ~HorzSuperresFixedEndToEndTest() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(::libaom_test::kTwoPassGood);
cfg_.g_lag_in_frames = 5;
cfg_.rc_end_usage = AOM_VBR;
@@ -223,18 +223,18 @@
cfg_.rc_superres_kf_denominator = superres_kf_denom_;
}
- virtual void BeginPassHook(unsigned int) {
+ void BeginPassHook(unsigned int) override {
psnr_ = 0.0;
frame_count_ = 0;
}
- virtual void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) {
+ void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) override {
psnr_ += pkt->data.psnr.psnr[0];
frame_count_++;
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AV1E_SET_FRAME_PARALLEL_DECODING, 1);
encoder->Control(AV1E_SET_TILE_COLUMNS, 4);
@@ -313,9 +313,9 @@
superres_kf_qthresh_ = std::get<1>(qthresholds);
}
- virtual ~HorzSuperresQThreshEndToEndTest() {}
+ ~HorzSuperresQThreshEndToEndTest() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(::libaom_test::kTwoPassGood);
cfg_.g_lag_in_frames = 5;
cfg_.rc_end_usage = AOM_VBR;
@@ -333,18 +333,18 @@
cfg_.rc_superres_kf_qthresh = superres_kf_qthresh_;
}
- virtual void BeginPassHook(unsigned int) {
+ void BeginPassHook(unsigned int) override {
psnr_ = 0.0;
frame_count_ = 0;
}
- virtual void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) {
+ void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) override {
psnr_ += pkt->data.psnr.psnr[0];
frame_count_++;
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AV1E_SET_FRAME_PARALLEL_DECODING, 1);
encoder->Control(AV1E_SET_TILE_COLUMNS, 0);
diff --git a/test/intra_edge_test.cc b/test/intra_edge_test.cc
index 84e712d..96ee654 100644
--- a/test/intra_edge_test.cc
+++ b/test/intra_edge_test.cc
@@ -37,7 +37,7 @@
static const int kBufSize = 2 * 64 + 32;
static const int kOffset = 16;
- virtual ~UpsampleTest() {}
+ ~UpsampleTest() override = default;
virtual void Execute(T *edge_tst) = 0;
@@ -62,16 +62,12 @@
int size_;
};
-//////////////////////////////////////////////////////////////////////////////
-// 8 bit version
-//////////////////////////////////////////////////////////////////////////////
-
typedef void (*UP8B)(uint8_t *p, int size);
typedef libaom_test::FuncParam<UP8B> TestFuncs;
class UpsampleTest8B : public UpsampleTest<UP8B, uint8_t> {
protected:
- void Execute(uint8_t *edge_tst) {
+ void Execute(uint8_t *edge_tst) override {
params_.ref_func(edge_ref_, size_);
API_REGISTER_STATE_CHECK(params_.tst_func(edge_tst, size_));
}
@@ -99,6 +95,18 @@
}
}
+TEST_P(UpsampleTest8B, DISABLED_Speed) {
+ const int test_count = 10000000;
+ size_ = kMaxEdge;
+ for (int i = 0; i < kOffset + size_; ++i) {
+ edge_tst_data_[i] = rng_.Rand8();
+ }
+ edge_tst_ = &edge_tst_data_[kOffset];
+ for (int iter = 0; iter < test_count; ++iter) {
+ API_REGISTER_STATE_CHECK(params_.tst_func(edge_tst_, size_));
+ }
+}
+
#if HAVE_SSE4_1
INSTANTIATE_TEST_SUITE_P(
SSE4_1, UpsampleTest8B,
@@ -106,16 +114,110 @@
av1_upsample_intra_edge_sse4_1)));
#endif // HAVE_SSE4_1
-//////////////////////////////////////////////////////////////////////////////
-// High bit-depth version
-//////////////////////////////////////////////////////////////////////////////
+#if HAVE_NEON
+INSTANTIATE_TEST_SUITE_P(
+ NEON, UpsampleTest8B,
+ ::testing::Values(TestFuncs(av1_upsample_intra_edge_c,
+ av1_upsample_intra_edge_neon)));
+#endif // HAVE_NEON
+
+template <typename F, typename T>
+class FilterEdgeTest : public FunctionEquivalenceTest<F> {
+ protected:
+ static const int kIterations = 1000000;
+ static const int kMaxEdge = 2 * 64;
+ static const int kBufSize = kMaxEdge + 32;
+ static const int kOffset = 15;
+
+ ~FilterEdgeTest() override = default;
+
+ virtual void Execute(T *edge_tst) = 0;
+
+ void Common() {
+ edge_ref_ = &edge_ref_data_[kOffset];
+ edge_tst_ = &edge_tst_data_[kOffset];
+
+ Execute(edge_tst_);
+
+ for (int r = 0; r < size_; ++r) {
+ ASSERT_EQ(edge_ref_[r], edge_tst_[r]);
+ }
+ }
+
+ T edge_ref_data_[kBufSize];
+ T edge_tst_data_[kBufSize];
+
+ T *edge_ref_;
+ T *edge_tst_;
+
+ int size_;
+ int strength_;
+};
+
+typedef void (*FE8B)(uint8_t *p, int size, int strength);
+typedef libaom_test::FuncParam<FE8B> FilterEdgeTestFuncs;
+
+class FilterEdgeTest8B : public FilterEdgeTest<FE8B, uint8_t> {
+ protected:
+ void Execute(uint8_t *edge_tst) override {
+ params_.ref_func(edge_ref_, size_, strength_);
+ API_REGISTER_STATE_CHECK(params_.tst_func(edge_tst, size_, strength_));
+ }
+};
+
+TEST_P(FilterEdgeTest8B, RandomValues) {
+ for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) {
+ strength_ = this->rng_(4);
+ size_ = 4 * (this->rng_(128 / 4) + 1) + 1;
+
+ int i, pix = 0;
+ for (i = 0; i < kOffset + size_; ++i) {
+ pix = rng_.Rand8();
+ edge_ref_data_[i] = pix;
+ edge_tst_data_[i] = pix;
+ }
+
+ Common();
+ }
+}
+
+TEST_P(FilterEdgeTest8B, DISABLED_Speed) {
+ const int test_count = 10000000;
+ size_ = kMaxEdge;
+ strength_ = 1;
+ for (int i = 0; i < kOffset + size_; ++i) {
+ edge_tst_data_[i] = rng_.Rand8();
+ }
+ edge_tst_ = &edge_tst_data_[kOffset];
+ for (int iter = 0; iter < test_count; ++iter) {
+ API_REGISTER_STATE_CHECK(params_.tst_func(edge_tst_, size_, strength_));
+ // iterate over filter strengths (1,2,3)
+ strength_ = strength_ == 3 ? 1 : strength_ + 1;
+ }
+}
+
+#if HAVE_SSE4_1
+INSTANTIATE_TEST_SUITE_P(
+ SSE4_1, FilterEdgeTest8B,
+ ::testing::Values(FilterEdgeTestFuncs(av1_filter_intra_edge_c,
+ av1_filter_intra_edge_sse4_1)));
+#endif // HAVE_SSE4_1
+
+#if HAVE_NEON
+INSTANTIATE_TEST_SUITE_P(
+ NEON, FilterEdgeTest8B,
+ ::testing::Values(FilterEdgeTestFuncs(av1_filter_intra_edge_c,
+ av1_filter_intra_edge_neon)));
+#endif // HAVE_NEON
+
+#if CONFIG_AV1_HIGHBITDEPTH
typedef void (*UPHB)(uint16_t *p, int size, int bd);
typedef libaom_test::FuncParam<UPHB> TestFuncsHBD;
class UpsampleTestHB : public UpsampleTest<UPHB, uint16_t> {
protected:
- void Execute(uint16_t *edge_tst) {
+ void Execute(uint16_t *edge_tst) override {
params_.ref_func(edge_ref_, size_, bit_depth_);
API_REGISTER_STATE_CHECK(params_.tst_func(edge_tst, size_, bit_depth_));
}
@@ -151,94 +253,40 @@
}
}
-#if HAVE_SSE4_1
-INSTANTIATE_TEST_SUITE_P(
- SSE4_1, UpsampleTestHB,
- ::testing::Values(TestFuncsHBD(av1_upsample_intra_edge_high_c,
- av1_upsample_intra_edge_high_sse4_1)));
-#endif // HAVE_SSE4_1
-
-template <typename F, typename T>
-class FilterEdgeTest : public FunctionEquivalenceTest<F> {
- protected:
- static const int kIterations = 1000000;
- static const int kMaxEdge = 2 * 64;
- static const int kBufSize = kMaxEdge + 32;
- static const int kOffset = 15;
-
- virtual ~FilterEdgeTest() {}
-
- virtual void Execute(T *edge_tst) = 0;
-
- void Common() {
- edge_ref_ = &edge_ref_data_[kOffset];
- edge_tst_ = &edge_tst_data_[kOffset];
-
- Execute(edge_tst_);
-
- for (int r = 0; r < size_; ++r) {
- ASSERT_EQ(edge_ref_[r], edge_tst_[r]);
- }
+TEST_P(UpsampleTestHB, DISABLED_Speed) {
+ const int test_count = 10000000;
+ size_ = kMaxEdge;
+ bit_depth_ = 12;
+ const int hi = 1 << bit_depth_;
+ for (int i = 0; i < kOffset + size_; ++i) {
+ edge_tst_data_[i] = rng_(hi);
}
-
- T edge_ref_data_[kBufSize];
- T edge_tst_data_[kBufSize];
-
- T *edge_ref_;
- T *edge_tst_;
-
- int size_;
- int strength_;
-};
-
-//////////////////////////////////////////////////////////////////////////////
-// 8 bit version
-//////////////////////////////////////////////////////////////////////////////
-
-typedef void (*FE8B)(uint8_t *p, int size, int strength);
-typedef libaom_test::FuncParam<FE8B> FilterEdgeTestFuncs;
-
-class FilterEdgeTest8B : public FilterEdgeTest<FE8B, uint8_t> {
- protected:
- void Execute(uint8_t *edge_tst) {
- params_.ref_func(edge_ref_, size_, strength_);
- API_REGISTER_STATE_CHECK(params_.tst_func(edge_tst, size_, strength_));
- }
-};
-
-TEST_P(FilterEdgeTest8B, RandomValues) {
- for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) {
- strength_ = this->rng_(4);
- size_ = 4 * (this->rng_(128 / 4) + 1) + 1;
-
- int i, pix = 0;
- for (i = 0; i < kOffset + size_; ++i) {
- pix = rng_.Rand8();
- edge_ref_data_[i] = pix;
- edge_tst_data_[i] = pix;
- }
-
- Common();
+ edge_tst_ = &edge_tst_data_[kOffset];
+ for (int iter = 0; iter < test_count; ++iter) {
+ API_REGISTER_STATE_CHECK(params_.tst_func(edge_tst_, size_, bit_depth_));
}
}
#if HAVE_SSE4_1
INSTANTIATE_TEST_SUITE_P(
- SSE4_1, FilterEdgeTest8B,
- ::testing::Values(FilterEdgeTestFuncs(av1_filter_intra_edge_c,
- av1_filter_intra_edge_sse4_1)));
+ SSE4_1, UpsampleTestHB,
+ ::testing::Values(TestFuncsHBD(av1_highbd_upsample_intra_edge_c,
+ av1_highbd_upsample_intra_edge_sse4_1)));
#endif // HAVE_SSE4_1
-//////////////////////////////////////////////////////////////////////////////
-// High bit-depth version
-//////////////////////////////////////////////////////////////////////////////
+#if HAVE_NEON
+INSTANTIATE_TEST_SUITE_P(
+ NEON, UpsampleTestHB,
+ ::testing::Values(TestFuncsHBD(av1_highbd_upsample_intra_edge_c,
+ av1_highbd_upsample_intra_edge_neon)));
+#endif // HAVE_NEON
typedef void (*FEHB)(uint16_t *p, int size, int strength);
typedef libaom_test::FuncParam<FEHB> FilterEdgeTestFuncsHBD;
class FilterEdgeTestHB : public FilterEdgeTest<FEHB, uint16_t> {
protected:
- void Execute(uint16_t *edge_tst) {
+ void Execute(uint16_t *edge_tst) override {
params_.ref_func(edge_ref_, size_, strength_);
API_REGISTER_STATE_CHECK(params_.tst_func(edge_tst, size_, strength_));
}
@@ -267,56 +315,6 @@
}
}
-#if HAVE_SSE4_1
-INSTANTIATE_TEST_SUITE_P(SSE4_1, FilterEdgeTestHB,
- ::testing::Values(FilterEdgeTestFuncsHBD(
- av1_filter_intra_edge_high_c,
- av1_filter_intra_edge_high_sse4_1)));
-#endif // HAVE_SSE4_1
-
-// Speed tests
-
-TEST_P(UpsampleTest8B, DISABLED_Speed) {
- const int test_count = 10000000;
- size_ = kMaxEdge;
- for (int i = 0; i < kOffset + size_; ++i) {
- edge_tst_data_[i] = rng_.Rand8();
- }
- edge_tst_ = &edge_tst_data_[kOffset];
- for (int iter = 0; iter < test_count; ++iter) {
- API_REGISTER_STATE_CHECK(params_.tst_func(edge_tst_, size_));
- }
-}
-
-TEST_P(UpsampleTestHB, DISABLED_Speed) {
- const int test_count = 10000000;
- size_ = kMaxEdge;
- bit_depth_ = 12;
- const int hi = 1 << bit_depth_;
- for (int i = 0; i < kOffset + size_; ++i) {
- edge_tst_data_[i] = rng_(hi);
- }
- edge_tst_ = &edge_tst_data_[kOffset];
- for (int iter = 0; iter < test_count; ++iter) {
- API_REGISTER_STATE_CHECK(params_.tst_func(edge_tst_, size_, bit_depth_));
- }
-}
-
-TEST_P(FilterEdgeTest8B, DISABLED_Speed) {
- const int test_count = 10000000;
- size_ = kMaxEdge;
- strength_ = 1;
- for (int i = 0; i < kOffset + size_; ++i) {
- edge_tst_data_[i] = rng_.Rand8();
- }
- edge_tst_ = &edge_tst_data_[kOffset];
- for (int iter = 0; iter < test_count; ++iter) {
- API_REGISTER_STATE_CHECK(params_.tst_func(edge_tst_, size_, strength_));
- // iterate over filter strengths (1,2,3)
- strength_ = (strength_ == 3) ? 1 : strength_ + 1;
- }
-}
-
TEST_P(FilterEdgeTestHB, DISABLED_Speed) {
const int test_count = 10000000;
size_ = kMaxEdge;
@@ -330,8 +328,24 @@
for (int iter = 0; iter < test_count; ++iter) {
API_REGISTER_STATE_CHECK(params_.tst_func(edge_tst_, size_, strength_));
// iterate over filter strengths (1,2,3)
- strength_ = (strength_ == 3) ? 1 : strength_ + 1;
+ strength_ = strength_ == 3 ? 1 : strength_ + 1;
}
}
+#if HAVE_SSE4_1
+INSTANTIATE_TEST_SUITE_P(SSE4_1, FilterEdgeTestHB,
+ ::testing::Values(FilterEdgeTestFuncsHBD(
+ av1_highbd_filter_intra_edge_c,
+ av1_highbd_filter_intra_edge_sse4_1)));
+#endif // HAVE_SSE4_1
+
+#if HAVE_NEON
+INSTANTIATE_TEST_SUITE_P(NEON, FilterEdgeTestHB,
+ ::testing::Values(FilterEdgeTestFuncsHBD(
+ av1_highbd_filter_intra_edge_c,
+ av1_highbd_filter_intra_edge_neon)));
+#endif // HAVE_NEON
+
+#endif // CONFIG_AV1_HIGHBITDEPTH
+
} // namespace
diff --git a/test/intrapred_test.cc b/test/intrapred_test.cc
index aced593..8796e8b 100644
--- a/test/intrapred_test.cc
+++ b/test/intrapred_test.cc
@@ -155,7 +155,7 @@
}
protected:
- virtual void SetUp() {
+ void SetUp() override {
params_ = this->GetParam();
stride_ = params_.block_width * 3;
mask_ = (1 << params_.bit_depth) - 1;
@@ -195,19 +195,19 @@
#if CONFIG_AV1_HIGHBITDEPTH
class HighbdIntraPredTest : public AV1IntraPredTest<HighbdIntraPred, uint16_t> {
protected:
- void Predict() {
+ void Predict() override {
const int bit_depth = params_.bit_depth;
params_.ref_fn(ref_dst_, stride_, above_row_, left_col_, bit_depth);
API_REGISTER_STATE_CHECK(
params_.pred_fn(dst_, stride_, above_row_, left_col_, bit_depth));
}
- void PredictRefSpeedTest(int num) {
+ void PredictRefSpeedTest(int num) override {
const int bit_depth = params_.bit_depth;
for (int i = 0; i < num; i++) {
params_.ref_fn(ref_dst_, stride_, above_row_, left_col_, bit_depth);
}
}
- void PredictFncSpeedTest(int num) {
+ void PredictFncSpeedTest(int num) override {
const int bit_depth = params_.bit_depth;
for (int i = 0; i < num; i++) {
params_.pred_fn(dst_, stride_, above_row_, left_col_, bit_depth);
@@ -220,17 +220,17 @@
class LowbdIntraPredTest : public AV1IntraPredTest<IntraPred, uint8_t> {
protected:
- void Predict() {
+ void Predict() override {
params_.ref_fn(ref_dst_, stride_, above_row_, left_col_);
API_REGISTER_STATE_CHECK(
params_.pred_fn(dst_, stride_, above_row_, left_col_));
}
- void PredictRefSpeedTest(int num) {
+ void PredictRefSpeedTest(int num) override {
for (int i = 0; i < num; i++) {
params_.ref_fn(ref_dst_, stride_, above_row_, left_col_);
}
}
- void PredictFncSpeedTest(int num) {
+ void PredictFncSpeedTest(int num) override {
for (int i = 0; i < num; i++) {
params_.pred_fn(dst_, stride_, above_row_, left_col_);
}
diff --git a/test/invalid_file_test.cc b/test/invalid_file_test.cc
index 63e15ca..791cdb8 100644
--- a/test/invalid_file_test.cc
+++ b/test/invalid_file_test.cc
@@ -45,7 +45,7 @@
protected:
InvalidFileTest() : DecoderTest(GET_PARAM(0)), res_file_(nullptr) {}
- virtual ~InvalidFileTest() {
+ ~InvalidFileTest() override {
if (res_file_ != nullptr) fclose(res_file_);
}
@@ -55,15 +55,14 @@
<< "Result file open failed. Filename: " << res_file_name;
}
- virtual void DecompressedFrameHook(const aom_image_t &img,
- const unsigned int /*frame_number*/) {
+ void DecompressedFrameHook(const aom_image_t &img,
+ const unsigned int /*frame_number*/) override {
EXPECT_NE(img.fb_priv, nullptr);
}
- virtual bool HandleDecodeResult(
- const aom_codec_err_t res_dec,
- const libaom_test::CompressedVideoSource &video,
- libaom_test::Decoder *decoder) {
+ bool HandleDecodeResult(const aom_codec_err_t res_dec,
+ const libaom_test::CompressedVideoSource &video,
+ libaom_test::Decoder *decoder) override {
EXPECT_NE(res_file_, nullptr);
int expected_res_dec = -1;
@@ -95,9 +94,9 @@
return !HasFailure();
}
- virtual void HandlePeekResult(libaom_test::Decoder *const /*decoder*/,
- libaom_test::CompressedVideoSource * /*video*/,
- const aom_codec_err_t /*res_peek*/) {}
+ void HandlePeekResult(libaom_test::Decoder *const /*decoder*/,
+ libaom_test::CompressedVideoSource * /*video*/,
+ const aom_codec_err_t /*res_peek*/) override {}
void RunTest() {
const DecodeParam input = GET_PARAM(1);
diff --git a/test/ivf_video_source.h b/test/ivf_video_source.h
index 45828b5..85731f5 100644
--- a/test/ivf_video_source.h
+++ b/test/ivf_video_source.h
@@ -37,20 +37,20 @@
compressed_frame_buf_(nullptr), frame_sz_(0), frame_(0),
end_of_file_(false) {}
- virtual ~IVFVideoSource() {
+ ~IVFVideoSource() override {
delete[] compressed_frame_buf_;
if (input_file_) fclose(input_file_);
}
- virtual void Init() {
+ void Init() override {
// Allocate a buffer for read in the compressed video frame.
compressed_frame_buf_ = new uint8_t[kCodeBufferSize];
ASSERT_NE(compressed_frame_buf_, nullptr) << "Allocate frame buffer failed";
ASAN_POISON_MEMORY_REGION(compressed_frame_buf_, kCodeBufferSize);
}
- virtual void Begin() {
+ void Begin() override {
input_file_ = OpenTestDataFile(file_name_);
ASSERT_NE(input_file_, nullptr)
<< "Input file open failed. Filename: " << file_name_;
@@ -67,7 +67,7 @@
FillFrame();
}
- virtual void Next() {
+ void Next() override {
++frame_;
FillFrame();
}
@@ -94,11 +94,11 @@
}
}
- virtual const uint8_t *cxdata() const {
+ const uint8_t *cxdata() const override {
return end_of_file_ ? nullptr : compressed_frame_buf_;
}
- virtual size_t frame_size() const { return frame_sz_; }
- virtual unsigned int frame_number() const { return frame_; }
+ size_t frame_size() const override { return frame_sz_; }
+ unsigned int frame_number() const override { return frame_; }
protected:
std::string file_name_;
diff --git a/test/kf_test.cc b/test/kf_test.cc
index 5daf600..bc475fd 100644
--- a/test/kf_test.cc
+++ b/test/kf_test.cc
@@ -47,9 +47,9 @@
kf_dist_ = -1;
is_kf_interval_violated_ = false;
}
- virtual ~KeyFrameIntervalTestLarge() {}
+ ~KeyFrameIntervalTestLarge() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(encoding_mode_);
const aom_rational timebase = { 1, 30 };
cfg_.g_timebase = timebase;
@@ -60,18 +60,18 @@
cfg_.g_lag_in_frames = 19;
}
- virtual bool DoDecode() const { return 1; }
+ bool DoDecode() const override { return true; }
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AOME_SET_CPUUSED, 5);
encoder->Control(AOME_SET_ENABLEAUTOALTREF, 1);
}
}
- virtual bool HandleDecodeResult(const aom_codec_err_t res_dec,
- libaom_test::Decoder *decoder) {
+ bool HandleDecodeResult(const aom_codec_err_t res_dec,
+ libaom_test::Decoder *decoder) override {
EXPECT_EQ(AOM_CODEC_OK, res_dec) << decoder->DecodeError();
if (AOM_CODEC_OK == res_dec) {
aom_codec_ctx_t *ctx_dec = decoder->GetDecoder();
@@ -149,9 +149,9 @@
frame_num_ = 0;
is_kf_placement_violated_ = false;
}
- virtual ~ForcedKeyTestLarge() {}
+ ~ForcedKeyTestLarge() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(encoding_mode_);
cfg_.rc_end_usage = rc_end_usage_;
cfg_.g_threads = 0;
@@ -160,8 +160,8 @@
cfg_.fwd_kf_enabled = fwd_kf_enabled_;
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AOME_SET_CPUUSED, cpu_used_);
encoder->Control(AOME_SET_ENABLEAUTOALTREF, auto_alt_ref_);
@@ -176,8 +176,8 @@
((int)video->frame() == forced_kf_frame_num_) ? AOM_EFLAG_FORCE_KF : 0;
}
- virtual bool HandleDecodeResult(const aom_codec_err_t res_dec,
- libaom_test::Decoder *decoder) {
+ bool HandleDecodeResult(const aom_codec_err_t res_dec,
+ libaom_test::Decoder *decoder) override {
EXPECT_EQ(AOM_CODEC_OK, res_dec) << decoder->DecodeError();
if (AOM_CODEC_OK == res_dec) {
if ((int)frame_num_ == forced_kf_frame_num_) {
diff --git a/test/level_test.cc b/test/level_test.cc
index cc79926..a7c26d2 100644
--- a/test/level_test.cc
+++ b/test/level_test.cc
@@ -40,9 +40,9 @@
: EncoderTest(GET_PARAM(0)), encoding_mode_(GET_PARAM(1)),
cpu_used_(GET_PARAM(2)), target_level_(31) {}
- virtual ~LevelTest() {}
+ ~LevelTest() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(encoding_mode_);
if (encoding_mode_ != ::libaom_test::kRealTime) {
cfg_.g_lag_in_frames = 5;
@@ -53,8 +53,8 @@
}
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AOME_SET_CPUUSED, cpu_used_);
encoder->Control(AV1E_SET_TARGET_SEQ_LEVEL_IDX, target_level_);
diff --git a/test/loopfilter_control_test.cc b/test/loopfilter_control_test.cc
index 5f01340..9c00235 100644
--- a/test/loopfilter_control_test.cc
+++ b/test/loopfilter_control_test.cc
@@ -79,9 +79,9 @@
aq_mode_(GET_PARAM(3)), threads_(GET_PARAM(4)),
tile_columns_(GET_PARAM(5)) {}
- virtual ~LFControlEndToEndTest() {}
+ ~LFControlEndToEndTest() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(::libaom_test::kRealTime);
cfg_.g_threads = threads_;
@@ -92,18 +92,18 @@
cfg_.kf_min_dist = 9999;
}
- virtual void BeginPassHook(unsigned int) {
+ void BeginPassHook(unsigned int) override {
psnr_ = 0.0;
nframes_ = 0;
}
- virtual void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) {
+ void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) override {
psnr_ += pkt->data.psnr.psnr[0];
nframes_++;
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AV1E_SET_ENABLE_RESTORATION, 0);
encoder->Control(AV1E_SET_ENABLE_OBMC, 0);
diff --git a/test/lossless_test.cc b/test/lossless_test.cc
index ef4e19f..756ad05 100644
--- a/test/lossless_test.cc
+++ b/test/lossless_test.cc
@@ -33,15 +33,15 @@
encoding_mode_(GET_PARAM(1)), rc_end_usage_(GET_PARAM(2)),
cpu_used_(GET_PARAM(3)) {}
- virtual ~LosslessTestLarge() {}
+ ~LosslessTestLarge() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(encoding_mode_);
cfg_.rc_end_usage = rc_end_usage_;
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
// Only call Control if quantizer > 0 to verify that using quantizer
// alone will activate lossless
@@ -52,19 +52,19 @@
}
}
- virtual void BeginPassHook(unsigned int /*pass*/) {
+ void BeginPassHook(unsigned int /*pass*/) override {
psnr_ = kMaxPsnr;
nframes_ = 0;
}
- virtual void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) {
+ void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) override {
if (pkt->data.psnr.psnr[0] < psnr_) psnr_ = pkt->data.psnr.psnr[0];
}
double GetMinPsnr() const { return psnr_; }
- virtual bool HandleDecodeResult(const aom_codec_err_t res_dec,
- libaom_test::Decoder *decoder) {
+ bool HandleDecodeResult(const aom_codec_err_t res_dec,
+ libaom_test::Decoder *decoder) override {
EXPECT_EQ(AOM_CODEC_OK, res_dec) << decoder->DecodeError();
if (AOM_CODEC_OK == res_dec) {
aom_codec_ctx_t *ctx_dec = decoder->GetDecoder();
diff --git a/test/lpf_test.cc b/test/lpf_test.cc
index 421fdef..04b1c86 100644
--- a/test/lpf_test.cc
+++ b/test/lpf_test.cc
@@ -127,16 +127,14 @@
template <typename func_type_t, typename params_t>
class LoopTestParam : public ::testing::TestWithParam<params_t> {
public:
- virtual ~LoopTestParam() {}
- virtual void SetUp() {
+ ~LoopTestParam() override = default;
+ void SetUp() override {
loopfilter_op_ = std::get<0>(this->GetParam());
ref_loopfilter_op_ = std::get<1>(this->GetParam());
bit_depth_ = std::get<2>(this->GetParam());
mask_ = (1 << bit_depth_) - 1;
}
- virtual void TearDown() {}
-
protected:
int bit_depth_;
int mask_;
diff --git a/test/masked_sad_test.cc b/test/masked_sad_test.cc
index 2ef3e4d..bb03746 100644
--- a/test/masked_sad_test.cc
+++ b/test/masked_sad_test.cc
@@ -45,8 +45,8 @@
class MaskedSADTestBase : public ::testing::Test {
public:
- virtual ~MaskedSADTestBase() {}
- virtual void SetUp() = 0;
+ ~MaskedSADTestBase() override = default;
+ void SetUp() override = 0;
virtual void runRef(const uint8_t *src_ptr, int src_stride,
const uint8_t *ref_ptr[], int ref_stride,
const uint8_t *second_pred, const uint8_t *msk,
@@ -58,28 +58,26 @@
int msk_stride, int inv_mask, unsigned sads[],
int times) = 0;
- virtual void TearDown() {}
void runMaskedSADTest(int run_times);
};
class MaskedSADTest : public MaskedSADTestBase,
public ::testing::WithParamInterface<MaskedSADParam> {
public:
- virtual ~MaskedSADTest() {}
- virtual void SetUp() {
+ ~MaskedSADTest() override = default;
+ void SetUp() override {
maskedSAD_op_ = GET_PARAM(0);
ref_maskedSAD_op_ = GET_PARAM(1);
}
- virtual void runRef(const uint8_t *src_ptr, int src_stride,
- const uint8_t *ref_ptr[], int ref_stride,
- const uint8_t *second_pred, const uint8_t *msk,
- int msk_stride, int inv_mask, unsigned sads[], int times);
- virtual void runTest(const uint8_t *src_ptr, int src_stride,
- const uint8_t *ref_ptr[], int ref_stride,
- const uint8_t *second_pred, const uint8_t *msk,
- int msk_stride, int inv_mask, unsigned sads[],
- int times);
+ void runRef(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr[],
+ int ref_stride, const uint8_t *second_pred, const uint8_t *msk,
+ int msk_stride, int inv_mask, unsigned sads[],
+ int times) override;
+ void runTest(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr[],
+ int ref_stride, const uint8_t *second_pred, const uint8_t *msk,
+ int msk_stride, int inv_mask, unsigned sads[],
+ int times) override;
protected:
MaskedSADFunc maskedSAD_op_;
@@ -90,20 +88,19 @@
class MaskedSADx4Test : public MaskedSADTestBase,
public ::testing::WithParamInterface<MaskedSADx4Param> {
public:
- virtual ~MaskedSADx4Test() {}
- virtual void SetUp() {
+ ~MaskedSADx4Test() override = default;
+ void SetUp() override {
maskedSAD_op_ = GET_PARAM(0);
ref_maskedSAD_op_ = GET_PARAM(1);
}
- virtual void runRef(const uint8_t *src_ptr, int src_stride,
- const uint8_t *ref_ptr[], int ref_stride,
- const uint8_t *second_pred, const uint8_t *msk,
- int msk_stride, int inv_mask, unsigned sads[], int times);
- virtual void runTest(const uint8_t *src_ptr, int src_stride,
- const uint8_t *ref_ptr[], int ref_stride,
- const uint8_t *second_pred, const uint8_t *msk,
- int msk_stride, int inv_mask, unsigned sads[],
- int times);
+ void runRef(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr[],
+ int ref_stride, const uint8_t *second_pred, const uint8_t *msk,
+ int msk_stride, int inv_mask, unsigned sads[],
+ int times) override;
+ void runTest(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr[],
+ int ref_stride, const uint8_t *second_pred, const uint8_t *msk,
+ int msk_stride, int inv_mask, unsigned sads[],
+ int times) override;
protected:
MaskedSADx4Func maskedSAD_op_;
@@ -264,13 +261,12 @@
class HighbdMaskedSADTest
: public ::testing::TestWithParam<HighbdMaskedSADParam> {
public:
- virtual ~HighbdMaskedSADTest() {}
- virtual void SetUp() {
+ ~HighbdMaskedSADTest() override = default;
+ void SetUp() override {
maskedSAD_op_ = GET_PARAM(0);
ref_maskedSAD_op_ = GET_PARAM(1);
}
- virtual void TearDown() {}
void runHighbdMaskedSADTest(int run_times);
protected:
@@ -581,6 +577,41 @@
INSTANTIATE_TEST_SUITE_P(NEON, MaskedSADx4Test,
::testing::ValuesIn(msadx4_test));
+
+#if CONFIG_AV1_HIGHBITDEPTH
+const MaskedSADParam hbd_msad_neon_test[] = {
+ make_tuple(&aom_highbd_masked_sad4x4_neon, &aom_highbd_masked_sad4x4_c),
+ make_tuple(&aom_highbd_masked_sad4x8_neon, &aom_highbd_masked_sad4x8_c),
+ make_tuple(&aom_highbd_masked_sad8x4_neon, &aom_highbd_masked_sad8x4_c),
+ make_tuple(&aom_highbd_masked_sad8x8_neon, &aom_highbd_masked_sad8x8_c),
+ make_tuple(&aom_highbd_masked_sad8x16_neon, &aom_highbd_masked_sad8x16_c),
+ make_tuple(&aom_highbd_masked_sad16x8_neon, &aom_highbd_masked_sad16x8_c),
+ make_tuple(&aom_highbd_masked_sad16x16_neon, &aom_highbd_masked_sad16x16_c),
+ make_tuple(&aom_highbd_masked_sad16x32_neon, &aom_highbd_masked_sad16x32_c),
+ make_tuple(&aom_highbd_masked_sad32x16_neon, &aom_highbd_masked_sad32x16_c),
+ make_tuple(&aom_highbd_masked_sad32x32_neon, &aom_highbd_masked_sad32x32_c),
+ make_tuple(&aom_highbd_masked_sad32x64_neon, &aom_highbd_masked_sad32x64_c),
+ make_tuple(&aom_highbd_masked_sad64x32_neon, &aom_highbd_masked_sad64x32_c),
+ make_tuple(&aom_highbd_masked_sad64x64_neon, &aom_highbd_masked_sad64x64_c),
+ make_tuple(&aom_highbd_masked_sad64x128_neon, &aom_highbd_masked_sad64x128_c),
+ make_tuple(&aom_highbd_masked_sad128x64_neon, &aom_highbd_masked_sad128x64_c),
+ make_tuple(&aom_highbd_masked_sad128x128_neon,
+ &aom_highbd_masked_sad128x128_c),
+#if !CONFIG_REALTIME_ONLY
+ make_tuple(&aom_highbd_masked_sad4x16_neon, &aom_highbd_masked_sad4x16_c),
+ make_tuple(&aom_highbd_masked_sad16x4_neon, &aom_highbd_masked_sad16x4_c),
+ make_tuple(&aom_highbd_masked_sad8x32_neon, &aom_highbd_masked_sad8x32_c),
+ make_tuple(&aom_highbd_masked_sad32x8_neon, &aom_highbd_masked_sad32x8_c),
+ make_tuple(&aom_highbd_masked_sad16x64_neon, &aom_highbd_masked_sad16x64_c),
+ make_tuple(&aom_highbd_masked_sad64x16_neon, &aom_highbd_masked_sad64x16_c),
+#endif // !CONFIG_REALTIME_ONLY
+};
+
+INSTANTIATE_TEST_SUITE_P(NEON, HighbdMaskedSADTest,
+ ::testing::ValuesIn(hbd_msad_neon_test));
+
+#endif // CONFIG_AV1_HIGHBITDEPTH
+
#endif // HAVE_NEON
} // namespace
diff --git a/test/masked_variance_test.cc b/test/masked_variance_test.cc
index e76403e..8482a12 100644
--- a/test/masked_variance_test.cc
+++ b/test/masked_variance_test.cc
@@ -43,14 +43,12 @@
class MaskedSubPixelVarianceTest
: public ::testing::TestWithParam<MaskedSubPixelVarianceParam> {
public:
- virtual ~MaskedSubPixelVarianceTest() {}
- virtual void SetUp() {
+ ~MaskedSubPixelVarianceTest() override = default;
+ void SetUp() override {
opt_func_ = GET_PARAM(0);
ref_func_ = GET_PARAM(1);
}
- virtual void TearDown() {}
-
protected:
MaskedSubPixelVarianceFunc opt_func_;
MaskedSubPixelVarianceFunc ref_func_;
@@ -179,15 +177,13 @@
class HighbdMaskedSubPixelVarianceTest
: public ::testing::TestWithParam<HighbdMaskedSubPixelVarianceParam> {
public:
- virtual ~HighbdMaskedSubPixelVarianceTest() {}
- virtual void SetUp() {
+ ~HighbdMaskedSubPixelVarianceTest() override = default;
+ void SetUp() override {
opt_func_ = GET_PARAM(0);
ref_func_ = GET_PARAM(1);
bit_depth_ = GET_PARAM(2);
}
- virtual void TearDown() {}
-
protected:
MaskedSubPixelVarianceFunc opt_func_;
MaskedSubPixelVarianceFunc ref_func_;
@@ -568,5 +564,149 @@
INSTANTIATE_TEST_SUITE_P(NEON_C_COMPARE, MaskedSubPixelVarianceTest,
::testing::ValuesIn(sub_pel_var_test));
+
+#if CONFIG_AV1_HIGHBITDEPTH
+const HighbdMaskedSubPixelVarianceParam hbd_sub_pel_var_test_neon[] = {
+ make_tuple(&aom_highbd_8_masked_sub_pixel_variance128x128_neon,
+ &aom_highbd_8_masked_sub_pixel_variance128x128_c, AOM_BITS_8),
+ make_tuple(&aom_highbd_8_masked_sub_pixel_variance128x64_neon,
+ &aom_highbd_8_masked_sub_pixel_variance128x64_c, AOM_BITS_8),
+ make_tuple(&aom_highbd_8_masked_sub_pixel_variance64x128_neon,
+ &aom_highbd_8_masked_sub_pixel_variance64x128_c, AOM_BITS_8),
+ make_tuple(&aom_highbd_8_masked_sub_pixel_variance64x64_neon,
+ &aom_highbd_8_masked_sub_pixel_variance64x64_c, AOM_BITS_8),
+ make_tuple(&aom_highbd_8_masked_sub_pixel_variance64x32_neon,
+ &aom_highbd_8_masked_sub_pixel_variance64x32_c, AOM_BITS_8),
+ make_tuple(&aom_highbd_8_masked_sub_pixel_variance32x64_neon,
+ &aom_highbd_8_masked_sub_pixel_variance32x64_c, AOM_BITS_8),
+ make_tuple(&aom_highbd_8_masked_sub_pixel_variance32x32_neon,
+ &aom_highbd_8_masked_sub_pixel_variance32x32_c, AOM_BITS_8),
+ make_tuple(&aom_highbd_8_masked_sub_pixel_variance32x16_neon,
+ &aom_highbd_8_masked_sub_pixel_variance32x16_c, AOM_BITS_8),
+ make_tuple(&aom_highbd_8_masked_sub_pixel_variance16x32_neon,
+ &aom_highbd_8_masked_sub_pixel_variance16x32_c, AOM_BITS_8),
+ make_tuple(&aom_highbd_8_masked_sub_pixel_variance16x16_neon,
+ &aom_highbd_8_masked_sub_pixel_variance16x16_c, AOM_BITS_8),
+ make_tuple(&aom_highbd_8_masked_sub_pixel_variance16x8_neon,
+ &aom_highbd_8_masked_sub_pixel_variance16x8_c, AOM_BITS_8),
+ make_tuple(&aom_highbd_8_masked_sub_pixel_variance8x16_neon,
+ &aom_highbd_8_masked_sub_pixel_variance8x16_c, AOM_BITS_8),
+ make_tuple(&aom_highbd_8_masked_sub_pixel_variance8x8_neon,
+ &aom_highbd_8_masked_sub_pixel_variance8x8_c, AOM_BITS_8),
+ make_tuple(&aom_highbd_8_masked_sub_pixel_variance8x4_neon,
+ &aom_highbd_8_masked_sub_pixel_variance8x4_c, AOM_BITS_8),
+ make_tuple(&aom_highbd_8_masked_sub_pixel_variance4x8_neon,
+ &aom_highbd_8_masked_sub_pixel_variance4x8_c, AOM_BITS_8),
+ make_tuple(&aom_highbd_8_masked_sub_pixel_variance4x4_neon,
+ &aom_highbd_8_masked_sub_pixel_variance4x4_c, AOM_BITS_8),
+ make_tuple(&aom_highbd_10_masked_sub_pixel_variance128x128_neon,
+ &aom_highbd_10_masked_sub_pixel_variance128x128_c, AOM_BITS_10),
+ make_tuple(&aom_highbd_10_masked_sub_pixel_variance128x64_neon,
+ &aom_highbd_10_masked_sub_pixel_variance128x64_c, AOM_BITS_10),
+ make_tuple(&aom_highbd_10_masked_sub_pixel_variance64x128_neon,
+ &aom_highbd_10_masked_sub_pixel_variance64x128_c, AOM_BITS_10),
+ make_tuple(&aom_highbd_10_masked_sub_pixel_variance64x64_neon,
+ &aom_highbd_10_masked_sub_pixel_variance64x64_c, AOM_BITS_10),
+ make_tuple(&aom_highbd_10_masked_sub_pixel_variance64x32_neon,
+ &aom_highbd_10_masked_sub_pixel_variance64x32_c, AOM_BITS_10),
+ make_tuple(&aom_highbd_10_masked_sub_pixel_variance32x64_neon,
+ &aom_highbd_10_masked_sub_pixel_variance32x64_c, AOM_BITS_10),
+ make_tuple(&aom_highbd_10_masked_sub_pixel_variance32x32_neon,
+ &aom_highbd_10_masked_sub_pixel_variance32x32_c, AOM_BITS_10),
+ make_tuple(&aom_highbd_10_masked_sub_pixel_variance32x16_neon,
+ &aom_highbd_10_masked_sub_pixel_variance32x16_c, AOM_BITS_10),
+ make_tuple(&aom_highbd_10_masked_sub_pixel_variance16x32_neon,
+ &aom_highbd_10_masked_sub_pixel_variance16x32_c, AOM_BITS_10),
+ make_tuple(&aom_highbd_10_masked_sub_pixel_variance16x16_neon,
+ &aom_highbd_10_masked_sub_pixel_variance16x16_c, AOM_BITS_10),
+ make_tuple(&aom_highbd_10_masked_sub_pixel_variance16x8_neon,
+ &aom_highbd_10_masked_sub_pixel_variance16x8_c, AOM_BITS_10),
+ make_tuple(&aom_highbd_10_masked_sub_pixel_variance8x16_neon,
+ &aom_highbd_10_masked_sub_pixel_variance8x16_c, AOM_BITS_10),
+ make_tuple(&aom_highbd_10_masked_sub_pixel_variance8x8_neon,
+ &aom_highbd_10_masked_sub_pixel_variance8x8_c, AOM_BITS_10),
+ make_tuple(&aom_highbd_10_masked_sub_pixel_variance8x4_neon,
+ &aom_highbd_10_masked_sub_pixel_variance8x4_c, AOM_BITS_10),
+ make_tuple(&aom_highbd_10_masked_sub_pixel_variance4x8_neon,
+ &aom_highbd_10_masked_sub_pixel_variance4x8_c, AOM_BITS_10),
+ make_tuple(&aom_highbd_10_masked_sub_pixel_variance4x4_neon,
+ &aom_highbd_10_masked_sub_pixel_variance4x4_c, AOM_BITS_10),
+ make_tuple(&aom_highbd_12_masked_sub_pixel_variance128x128_neon,
+ &aom_highbd_12_masked_sub_pixel_variance128x128_c, AOM_BITS_12),
+ make_tuple(&aom_highbd_12_masked_sub_pixel_variance128x64_neon,
+ &aom_highbd_12_masked_sub_pixel_variance128x64_c, AOM_BITS_12),
+ make_tuple(&aom_highbd_12_masked_sub_pixel_variance64x128_neon,
+ &aom_highbd_12_masked_sub_pixel_variance64x128_c, AOM_BITS_12),
+ make_tuple(&aom_highbd_12_masked_sub_pixel_variance64x64_neon,
+ &aom_highbd_12_masked_sub_pixel_variance64x64_c, AOM_BITS_12),
+ make_tuple(&aom_highbd_12_masked_sub_pixel_variance64x32_neon,
+ &aom_highbd_12_masked_sub_pixel_variance64x32_c, AOM_BITS_12),
+ make_tuple(&aom_highbd_12_masked_sub_pixel_variance32x64_neon,
+ &aom_highbd_12_masked_sub_pixel_variance32x64_c, AOM_BITS_12),
+ make_tuple(&aom_highbd_12_masked_sub_pixel_variance32x32_neon,
+ &aom_highbd_12_masked_sub_pixel_variance32x32_c, AOM_BITS_12),
+ make_tuple(&aom_highbd_12_masked_sub_pixel_variance32x16_neon,
+ &aom_highbd_12_masked_sub_pixel_variance32x16_c, AOM_BITS_12),
+ make_tuple(&aom_highbd_12_masked_sub_pixel_variance16x32_neon,
+ &aom_highbd_12_masked_sub_pixel_variance16x32_c, AOM_BITS_12),
+ make_tuple(&aom_highbd_12_masked_sub_pixel_variance16x16_neon,
+ &aom_highbd_12_masked_sub_pixel_variance16x16_c, AOM_BITS_12),
+ make_tuple(&aom_highbd_12_masked_sub_pixel_variance16x8_neon,
+ &aom_highbd_12_masked_sub_pixel_variance16x8_c, AOM_BITS_12),
+ make_tuple(&aom_highbd_12_masked_sub_pixel_variance8x16_neon,
+ &aom_highbd_12_masked_sub_pixel_variance8x16_c, AOM_BITS_12),
+ make_tuple(&aom_highbd_12_masked_sub_pixel_variance8x8_neon,
+ &aom_highbd_12_masked_sub_pixel_variance8x8_c, AOM_BITS_12),
+ make_tuple(&aom_highbd_12_masked_sub_pixel_variance8x4_neon,
+ &aom_highbd_12_masked_sub_pixel_variance8x4_c, AOM_BITS_12),
+ make_tuple(&aom_highbd_12_masked_sub_pixel_variance4x8_neon,
+ &aom_highbd_12_masked_sub_pixel_variance4x8_c, AOM_BITS_12),
+ make_tuple(&aom_highbd_12_masked_sub_pixel_variance4x4_neon,
+ &aom_highbd_12_masked_sub_pixel_variance4x4_c, AOM_BITS_12),
+#if !CONFIG_REALTIME_ONLY
+ make_tuple(&aom_highbd_8_masked_sub_pixel_variance64x16_neon,
+ &aom_highbd_8_masked_sub_pixel_variance64x16_c, AOM_BITS_8),
+ make_tuple(&aom_highbd_8_masked_sub_pixel_variance16x64_neon,
+ &aom_highbd_8_masked_sub_pixel_variance16x64_c, AOM_BITS_8),
+ make_tuple(&aom_highbd_8_masked_sub_pixel_variance32x8_neon,
+ &aom_highbd_8_masked_sub_pixel_variance32x8_c, AOM_BITS_8),
+ make_tuple(&aom_highbd_8_masked_sub_pixel_variance8x32_neon,
+ &aom_highbd_8_masked_sub_pixel_variance8x32_c, AOM_BITS_8),
+ make_tuple(&aom_highbd_8_masked_sub_pixel_variance16x4_neon,
+ &aom_highbd_8_masked_sub_pixel_variance16x4_c, AOM_BITS_8),
+ make_tuple(&aom_highbd_8_masked_sub_pixel_variance4x16_neon,
+ &aom_highbd_8_masked_sub_pixel_variance4x16_c, AOM_BITS_8),
+ make_tuple(&aom_highbd_10_masked_sub_pixel_variance64x16_neon,
+ &aom_highbd_10_masked_sub_pixel_variance64x16_c, AOM_BITS_10),
+ make_tuple(&aom_highbd_10_masked_sub_pixel_variance16x64_neon,
+ &aom_highbd_10_masked_sub_pixel_variance16x64_c, AOM_BITS_10),
+ make_tuple(&aom_highbd_10_masked_sub_pixel_variance32x8_neon,
+ &aom_highbd_10_masked_sub_pixel_variance32x8_c, AOM_BITS_10),
+ make_tuple(&aom_highbd_10_masked_sub_pixel_variance8x32_neon,
+ &aom_highbd_10_masked_sub_pixel_variance8x32_c, AOM_BITS_10),
+ make_tuple(&aom_highbd_10_masked_sub_pixel_variance16x4_neon,
+ &aom_highbd_10_masked_sub_pixel_variance16x4_c, AOM_BITS_10),
+ make_tuple(&aom_highbd_10_masked_sub_pixel_variance4x16_neon,
+ &aom_highbd_10_masked_sub_pixel_variance4x16_c, AOM_BITS_10),
+ make_tuple(&aom_highbd_12_masked_sub_pixel_variance64x16_neon,
+ &aom_highbd_12_masked_sub_pixel_variance64x16_c, AOM_BITS_12),
+ make_tuple(&aom_highbd_12_masked_sub_pixel_variance16x64_neon,
+ &aom_highbd_12_masked_sub_pixel_variance16x64_c, AOM_BITS_12),
+ make_tuple(&aom_highbd_12_masked_sub_pixel_variance32x8_neon,
+ &aom_highbd_12_masked_sub_pixel_variance32x8_c, AOM_BITS_12),
+ make_tuple(&aom_highbd_12_masked_sub_pixel_variance8x32_neon,
+ &aom_highbd_12_masked_sub_pixel_variance8x32_c, AOM_BITS_12),
+ make_tuple(&aom_highbd_12_masked_sub_pixel_variance16x4_neon,
+ &aom_highbd_12_masked_sub_pixel_variance16x4_c, AOM_BITS_12),
+ make_tuple(&aom_highbd_12_masked_sub_pixel_variance4x16_neon,
+ &aom_highbd_12_masked_sub_pixel_variance4x16_c, AOM_BITS_12),
+#endif
+};
+
+INSTANTIATE_TEST_SUITE_P(NEON_C_COMPARE, HighbdMaskedSubPixelVarianceTest,
+ ::testing::ValuesIn(hbd_sub_pel_var_test_neon));
+
+#endif // CONFIG_AV1_HIGHBITDEPTH
+
#endif // HAVE_NEON
} // namespace
diff --git a/test/metadata_test.cc b/test/metadata_test.cc
index e77e5d1..9467c29 100644
--- a/test/metadata_test.cc
+++ b/test/metadata_test.cc
@@ -56,12 +56,12 @@
protected:
MetadataEncodeTest() : EncoderTest(GET_PARAM(0)) {}
- virtual ~MetadataEncodeTest() {}
+ ~MetadataEncodeTest() override = default;
- virtual void SetUp() { InitializeConfig(GET_PARAM(1)); }
+ void SetUp() override { InitializeConfig(GET_PARAM(1)); }
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder * /*encoder*/) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder * /*encoder*/) override {
aom_image_t *current_frame = video->img();
if (current_frame) {
if (current_frame->metadata) aom_img_remove_metadata(current_frame);
@@ -95,7 +95,7 @@
}
}
- virtual void FramePktHook(const aom_codec_cx_pkt_t *pkt) {
+ void FramePktHook(const aom_codec_cx_pkt_t *pkt) override {
if (pkt->kind == AOM_CODEC_CX_FRAME_PKT) {
const size_t bitstream_size = pkt->data.frame.sz;
const uint8_t *bitstream =
@@ -138,8 +138,8 @@
}
}
- virtual void DecompressedFrameHook(const aom_image_t &img,
- aom_codec_pts_t /*pts*/) {
+ void DecompressedFrameHook(const aom_image_t &img,
+ aom_codec_pts_t /*pts*/) override {
ASSERT_NE(img.metadata, nullptr);
ASSERT_EQ(img.metadata->sz, 3u);
diff --git a/test/minmax_test.cc b/test/minmax_test.cc
index cf67b7b..33be4ff 100644
--- a/test/minmax_test.cc
+++ b/test/minmax_test.cc
@@ -31,7 +31,7 @@
class MinMaxTest : public ::testing::TestWithParam<MinMaxFunc> {
public:
- virtual void SetUp() {
+ void SetUp() override {
mm_func_ = GetParam();
rnd_.Reset(ACMRandom::DeterministicSeed());
}
diff --git a/test/monochrome_test.cc b/test/monochrome_test.cc
index c5229fc..f22b5fe 100644
--- a/test/monochrome_test.cc
+++ b/test/monochrome_test.cc
@@ -42,12 +42,12 @@
: EncoderTest(GET_PARAM(0)), lossless_(GET_PARAM(2)),
frame0_psnr_y_(0.0) {}
- virtual ~MonochromeTest() {}
+ ~MonochromeTest() override = default;
- virtual void SetUp() { InitializeConfig(GET_PARAM(1)); }
+ void SetUp() override { InitializeConfig(GET_PARAM(1)); }
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AOME_SET_CPUUSED, GET_PARAM(3));
if (mode_ == ::libaom_test::kAllIntra) {
@@ -59,8 +59,8 @@
}
}
- virtual void DecompressedFrameHook(const aom_image_t &img,
- aom_codec_pts_t pts) {
+ void DecompressedFrameHook(const aom_image_t &img,
+ aom_codec_pts_t pts) override {
(void)pts;
// Get value of top-left corner pixel of U plane
@@ -96,7 +96,7 @@
return true;
}
- virtual void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) {
+ void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) override {
// Check average PSNR value is >= 100 db in case of lossless encoding.
if (lossless_) {
EXPECT_GE(pkt->data.psnr.psnr[0], kMaxPsnr);
diff --git a/test/motion_vector_test.cc b/test/motion_vector_test.cc
index bf10ede..4fc8d53 100644
--- a/test/motion_vector_test.cc
+++ b/test/motion_vector_test.cc
@@ -43,9 +43,9 @@
: EncoderTest(GET_PARAM(0)), encoding_mode_(GET_PARAM(1)),
cpu_used_(GET_PARAM(2)), mv_test_mode_(GET_PARAM(3)) {}
- virtual ~MotionVectorTestLarge() {}
+ ~MotionVectorTestLarge() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(encoding_mode_);
if (encoding_mode_ != ::libaom_test::kRealTime) {
cfg_.g_lag_in_frames = 3;
@@ -56,8 +56,8 @@
}
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AOME_SET_CPUUSED, cpu_used_);
encoder->Control(AV1E_ENABLE_MOTION_VECTOR_UNIT_TEST, mv_test_mode_);
diff --git a/test/mv_cost_test.cc b/test/mv_cost_test.cc
index 86e310c..73d5666 100644
--- a/test/mv_cost_test.cc
+++ b/test/mv_cost_test.cc
@@ -23,18 +23,19 @@
int bits_cost[MV_OFFSET_BITS][2];
int class0_fp_cost[CLASS0_SIZE][MV_FP_SIZE], fp_cost[MV_FP_SIZE];
int class0_hp_cost[2], hp_cost[2];
- av1_cost_tokens_from_cdf(sign_cost, mvcomp->sign_cdf, NULL);
- av1_cost_tokens_from_cdf(class_cost, mvcomp->classes_cdf, NULL);
- av1_cost_tokens_from_cdf(class0_cost, mvcomp->class0_cdf, NULL);
+ av1_cost_tokens_from_cdf(sign_cost, mvcomp->sign_cdf, nullptr);
+ av1_cost_tokens_from_cdf(class_cost, mvcomp->classes_cdf, nullptr);
+ av1_cost_tokens_from_cdf(class0_cost, mvcomp->class0_cdf, nullptr);
for (i = 0; i < MV_OFFSET_BITS; ++i) {
- av1_cost_tokens_from_cdf(bits_cost[i], mvcomp->bits_cdf[i], NULL);
+ av1_cost_tokens_from_cdf(bits_cost[i], mvcomp->bits_cdf[i], nullptr);
}
for (i = 0; i < CLASS0_SIZE; ++i)
- av1_cost_tokens_from_cdf(class0_fp_cost[i], mvcomp->class0_fp_cdf[i], NULL);
- av1_cost_tokens_from_cdf(fp_cost, mvcomp->fp_cdf, NULL);
+ av1_cost_tokens_from_cdf(class0_fp_cost[i], mvcomp->class0_fp_cdf[i],
+ nullptr);
+ av1_cost_tokens_from_cdf(fp_cost, mvcomp->fp_cdf, nullptr);
if (precision > MV_SUBPEL_LOW_PRECISION) {
- av1_cost_tokens_from_cdf(class0_hp_cost, mvcomp->class0_hp_cdf, NULL);
- av1_cost_tokens_from_cdf(hp_cost, mvcomp->hp_cdf, NULL);
+ av1_cost_tokens_from_cdf(class0_hp_cost, mvcomp->class0_hp_cdf, nullptr);
+ av1_cost_tokens_from_cdf(hp_cost, mvcomp->hp_cdf, nullptr);
}
mvcost[0] = 0;
for (v = 1; v <= MV_MAX; ++v) {
diff --git a/test/noise_model_test.cc b/test/noise_model_test.cc
index 650af79..b3edcc2 100644
--- a/test/noise_model_test.cc
+++ b/test/noise_model_test.cc
@@ -24,7 +24,7 @@
// Return normally distrbuted values with standard deviation of sigma.
double randn(libaom_test::ACMRandom *random, double sigma) {
- while (1) {
+ while (true) {
const double u = 2.0 * ((double)random->Rand31() /
testing::internal::Random::kMaxRange) -
1.0;
@@ -367,7 +367,7 @@
template <typename T>
class FlatBlockEstimatorTest : public ::testing::Test, public T {
public:
- virtual void SetUp() { random_.Reset(171); }
+ void SetUp() override { random_.Reset(171); }
typedef std::vector<typename T::data_type_t> VecType;
VecType data_;
libaom_test::ACMRandom random_;
@@ -544,7 +544,7 @@
static const int kNumBlocksX = kWidth / kBlockSize;
static const int kNumBlocksY = kHeight / kBlockSize;
- virtual void SetUp() {
+ void SetUp() override {
const aom_noise_model_params_t params = { AOM_NOISE_SHAPE_SQUARE, 3,
T::kBitDepth, T::kUseHighBD };
ASSERT_TRUE(aom_noise_model_init(&model_, params));
@@ -576,7 +576,7 @@
&flat_blocks_[0], block_size);
}
- void TearDown() { aom_noise_model_free(&model_); }
+ void TearDown() override { aom_noise_model_free(&model_); }
protected:
aom_noise_model_t model_;
@@ -1186,7 +1186,7 @@
static void SetUpTestSuite() { aom_dsp_rtcd(); }
protected:
- void SetUp() {
+ void SetUp() override {
static const float kNoiseLevel = 5.f;
static const float kStd = 4.0;
static const double kMaxValue = (1 << T::kBitDepth) - 1;
diff --git a/test/obmc_sad_test.cc b/test/obmc_sad_test.cc
index 8d13ac1..967b677 100644
--- a/test/obmc_sad_test.cc
+++ b/test/obmc_sad_test.cc
@@ -236,6 +236,38 @@
}
}
+#if HAVE_NEON
+ObmcSadHBDTest::ParamType neon_functions_hbd[] = {
+ TestFuncs(aom_highbd_obmc_sad128x128_c, aom_highbd_obmc_sad128x128_neon),
+ TestFuncs(aom_highbd_obmc_sad128x64_c, aom_highbd_obmc_sad128x64_neon),
+ TestFuncs(aom_highbd_obmc_sad64x128_c, aom_highbd_obmc_sad64x128_neon),
+ TestFuncs(aom_highbd_obmc_sad64x64_c, aom_highbd_obmc_sad64x64_neon),
+ TestFuncs(aom_highbd_obmc_sad64x32_c, aom_highbd_obmc_sad64x32_neon),
+ TestFuncs(aom_highbd_obmc_sad32x64_c, aom_highbd_obmc_sad32x64_neon),
+ TestFuncs(aom_highbd_obmc_sad32x32_c, aom_highbd_obmc_sad32x32_neon),
+ TestFuncs(aom_highbd_obmc_sad32x16_c, aom_highbd_obmc_sad32x16_neon),
+ TestFuncs(aom_highbd_obmc_sad16x32_c, aom_highbd_obmc_sad16x32_neon),
+ TestFuncs(aom_highbd_obmc_sad16x16_c, aom_highbd_obmc_sad16x16_neon),
+ TestFuncs(aom_highbd_obmc_sad16x8_c, aom_highbd_obmc_sad16x8_neon),
+ TestFuncs(aom_highbd_obmc_sad8x16_c, aom_highbd_obmc_sad8x16_neon),
+ TestFuncs(aom_highbd_obmc_sad8x8_c, aom_highbd_obmc_sad8x8_neon),
+ TestFuncs(aom_highbd_obmc_sad8x4_c, aom_highbd_obmc_sad8x4_neon),
+ TestFuncs(aom_highbd_obmc_sad4x8_c, aom_highbd_obmc_sad4x8_neon),
+ TestFuncs(aom_highbd_obmc_sad4x4_c, aom_highbd_obmc_sad4x4_neon),
+#if !CONFIG_REALTIME_ONLY
+ TestFuncs(aom_highbd_obmc_sad64x16_c, aom_highbd_obmc_sad64x16_neon),
+ TestFuncs(aom_highbd_obmc_sad16x64_c, aom_highbd_obmc_sad16x64_neon),
+ TestFuncs(aom_highbd_obmc_sad32x8_c, aom_highbd_obmc_sad32x8_neon),
+ TestFuncs(aom_highbd_obmc_sad8x32_c, aom_highbd_obmc_sad8x32_neon),
+ TestFuncs(aom_highbd_obmc_sad16x4_c, aom_highbd_obmc_sad16x4_neon),
+ TestFuncs(aom_highbd_obmc_sad4x16_c, aom_highbd_obmc_sad4x16_neon),
+#endif // !CONFIG_REALTIME_ONLY
+};
+
+INSTANTIATE_TEST_SUITE_P(NEON, ObmcSadHBDTest,
+ ::testing::ValuesIn(neon_functions_hbd));
+#endif // HAVE_NEON
+
#if HAVE_SSE4_1
ObmcSadHBDTest::ParamType sse4_functions_hbd[] = {
TestFuncs(aom_highbd_obmc_sad128x128_c, aom_highbd_obmc_sad128x128_sse4_1),
diff --git a/test/obmc_variance_test.cc b/test/obmc_variance_test.cc
index b2bf42a..5f21a8a 100644
--- a/test/obmc_variance_test.cc
+++ b/test/obmc_variance_test.cc
@@ -228,7 +228,7 @@
////////////////////////////////////////////////////////////////////////////////
// High bit-depth
////////////////////////////////////////////////////////////////////////////////
-#if CONFIG_AV1_HIGHBITDEPTH
+#if CONFIG_AV1_HIGHBITDEPTH && !CONFIG_REALTIME_ONLY
class ObmcVarianceHBDTest : public FunctionEquivalenceTest<ObmcVarF> {};
GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(ObmcVarianceHBDTest);
@@ -287,40 +287,180 @@
}
}
+#if HAVE_NEON
+ObmcVarianceHBDTest::ParamType neon_functions_hbd[] = {
+ TestFuncs(aom_highbd_8_obmc_variance128x128_c,
+ aom_highbd_8_obmc_variance128x128_neon, 8),
+ TestFuncs(aom_highbd_8_obmc_variance128x64_c,
+ aom_highbd_8_obmc_variance128x64_neon, 8),
+ TestFuncs(aom_highbd_8_obmc_variance64x128_c,
+ aom_highbd_8_obmc_variance64x128_neon, 8),
+ TestFuncs(aom_highbd_8_obmc_variance64x64_c,
+ aom_highbd_8_obmc_variance64x64_neon, 8),
+ TestFuncs(aom_highbd_8_obmc_variance64x32_c,
+ aom_highbd_8_obmc_variance64x32_neon, 8),
+ TestFuncs(aom_highbd_8_obmc_variance32x64_c,
+ aom_highbd_8_obmc_variance32x64_neon, 8),
+ TestFuncs(aom_highbd_8_obmc_variance32x32_c,
+ aom_highbd_8_obmc_variance32x32_neon, 8),
+ TestFuncs(aom_highbd_8_obmc_variance32x16_c,
+ aom_highbd_8_obmc_variance32x16_neon, 8),
+ TestFuncs(aom_highbd_8_obmc_variance16x32_c,
+ aom_highbd_8_obmc_variance16x32_neon, 8),
+ TestFuncs(aom_highbd_8_obmc_variance16x16_c,
+ aom_highbd_8_obmc_variance16x16_neon, 8),
+ TestFuncs(aom_highbd_8_obmc_variance16x8_c,
+ aom_highbd_8_obmc_variance16x8_neon, 8),
+ TestFuncs(aom_highbd_8_obmc_variance8x16_c,
+ aom_highbd_8_obmc_variance8x16_neon, 8),
+ TestFuncs(aom_highbd_8_obmc_variance8x8_c, aom_highbd_8_obmc_variance8x8_neon,
+ 8),
+ TestFuncs(aom_highbd_8_obmc_variance8x4_c, aom_highbd_8_obmc_variance8x4_neon,
+ 8),
+ TestFuncs(aom_highbd_8_obmc_variance4x8_c, aom_highbd_8_obmc_variance4x8_neon,
+ 8),
+ TestFuncs(aom_highbd_8_obmc_variance4x4_c, aom_highbd_8_obmc_variance4x4_neon,
+ 8),
+ TestFuncs(aom_highbd_10_obmc_variance128x128_c,
+ aom_highbd_10_obmc_variance128x128_neon, 10),
+ TestFuncs(aom_highbd_10_obmc_variance128x64_c,
+ aom_highbd_10_obmc_variance128x64_neon, 10),
+ TestFuncs(aom_highbd_10_obmc_variance64x128_c,
+ aom_highbd_10_obmc_variance64x128_neon, 10),
+ TestFuncs(aom_highbd_10_obmc_variance64x64_c,
+ aom_highbd_10_obmc_variance64x64_neon, 10),
+ TestFuncs(aom_highbd_10_obmc_variance64x32_c,
+ aom_highbd_10_obmc_variance64x32_neon, 10),
+ TestFuncs(aom_highbd_10_obmc_variance32x64_c,
+ aom_highbd_10_obmc_variance32x64_neon, 10),
+ TestFuncs(aom_highbd_10_obmc_variance32x32_c,
+ aom_highbd_10_obmc_variance32x32_neon, 10),
+ TestFuncs(aom_highbd_10_obmc_variance32x16_c,
+ aom_highbd_10_obmc_variance32x16_neon, 10),
+ TestFuncs(aom_highbd_10_obmc_variance16x32_c,
+ aom_highbd_10_obmc_variance16x32_neon, 10),
+ TestFuncs(aom_highbd_10_obmc_variance16x16_c,
+ aom_highbd_10_obmc_variance16x16_neon, 10),
+ TestFuncs(aom_highbd_10_obmc_variance16x8_c,
+ aom_highbd_10_obmc_variance16x8_neon, 10),
+ TestFuncs(aom_highbd_10_obmc_variance8x16_c,
+ aom_highbd_10_obmc_variance8x16_neon, 10),
+ TestFuncs(aom_highbd_10_obmc_variance8x8_c,
+ aom_highbd_10_obmc_variance8x8_neon, 10),
+ TestFuncs(aom_highbd_10_obmc_variance8x4_c,
+ aom_highbd_10_obmc_variance8x4_neon, 10),
+ TestFuncs(aom_highbd_10_obmc_variance4x8_c,
+ aom_highbd_10_obmc_variance4x8_neon, 10),
+ TestFuncs(aom_highbd_10_obmc_variance4x4_c,
+ aom_highbd_10_obmc_variance4x4_neon, 10),
+ TestFuncs(aom_highbd_12_obmc_variance128x128_c,
+ aom_highbd_12_obmc_variance128x128_neon, 12),
+ TestFuncs(aom_highbd_12_obmc_variance128x64_c,
+ aom_highbd_12_obmc_variance128x64_neon, 12),
+ TestFuncs(aom_highbd_12_obmc_variance64x128_c,
+ aom_highbd_12_obmc_variance64x128_neon, 12),
+ TestFuncs(aom_highbd_12_obmc_variance64x64_c,
+ aom_highbd_12_obmc_variance64x64_neon, 12),
+ TestFuncs(aom_highbd_12_obmc_variance64x32_c,
+ aom_highbd_12_obmc_variance64x32_neon, 12),
+ TestFuncs(aom_highbd_12_obmc_variance32x64_c,
+ aom_highbd_12_obmc_variance32x64_neon, 12),
+ TestFuncs(aom_highbd_12_obmc_variance32x32_c,
+ aom_highbd_12_obmc_variance32x32_neon, 12),
+ TestFuncs(aom_highbd_12_obmc_variance32x16_c,
+ aom_highbd_12_obmc_variance32x16_neon, 12),
+ TestFuncs(aom_highbd_12_obmc_variance16x32_c,
+ aom_highbd_12_obmc_variance16x32_neon, 12),
+ TestFuncs(aom_highbd_12_obmc_variance16x16_c,
+ aom_highbd_12_obmc_variance16x16_neon, 12),
+ TestFuncs(aom_highbd_12_obmc_variance16x8_c,
+ aom_highbd_12_obmc_variance16x8_neon, 12),
+ TestFuncs(aom_highbd_12_obmc_variance8x16_c,
+ aom_highbd_12_obmc_variance8x16_neon, 12),
+ TestFuncs(aom_highbd_12_obmc_variance8x8_c,
+ aom_highbd_12_obmc_variance8x8_neon, 12),
+ TestFuncs(aom_highbd_12_obmc_variance8x4_c,
+ aom_highbd_12_obmc_variance8x4_neon, 12),
+ TestFuncs(aom_highbd_12_obmc_variance4x8_c,
+ aom_highbd_12_obmc_variance4x8_neon, 12),
+ TestFuncs(aom_highbd_12_obmc_variance4x4_c,
+ aom_highbd_12_obmc_variance4x4_neon, 12),
+ TestFuncs(aom_highbd_8_obmc_variance64x16_c,
+ aom_highbd_8_obmc_variance64x16_neon, 8),
+ TestFuncs(aom_highbd_8_obmc_variance16x64_c,
+ aom_highbd_8_obmc_variance16x64_neon, 8),
+ TestFuncs(aom_highbd_8_obmc_variance32x8_c,
+ aom_highbd_8_obmc_variance32x8_neon, 8),
+ TestFuncs(aom_highbd_8_obmc_variance8x32_c,
+ aom_highbd_8_obmc_variance8x32_neon, 8),
+ TestFuncs(aom_highbd_8_obmc_variance16x4_c,
+ aom_highbd_8_obmc_variance16x4_neon, 8),
+ TestFuncs(aom_highbd_8_obmc_variance4x16_c,
+ aom_highbd_8_obmc_variance4x16_neon, 8),
+ TestFuncs(aom_highbd_10_obmc_variance64x16_c,
+ aom_highbd_10_obmc_variance64x16_neon, 10),
+ TestFuncs(aom_highbd_10_obmc_variance16x64_c,
+ aom_highbd_10_obmc_variance16x64_neon, 10),
+ TestFuncs(aom_highbd_10_obmc_variance32x8_c,
+ aom_highbd_10_obmc_variance32x8_neon, 10),
+ TestFuncs(aom_highbd_10_obmc_variance8x32_c,
+ aom_highbd_10_obmc_variance8x32_neon, 10),
+ TestFuncs(aom_highbd_10_obmc_variance16x4_c,
+ aom_highbd_10_obmc_variance16x4_neon, 10),
+ TestFuncs(aom_highbd_10_obmc_variance4x16_c,
+ aom_highbd_10_obmc_variance4x16_neon, 10),
+ TestFuncs(aom_highbd_12_obmc_variance64x16_c,
+ aom_highbd_12_obmc_variance64x16_neon, 12),
+ TestFuncs(aom_highbd_12_obmc_variance16x64_c,
+ aom_highbd_12_obmc_variance16x64_neon, 12),
+ TestFuncs(aom_highbd_12_obmc_variance32x8_c,
+ aom_highbd_12_obmc_variance32x8_neon, 12),
+ TestFuncs(aom_highbd_12_obmc_variance8x32_c,
+ aom_highbd_12_obmc_variance8x32_neon, 12),
+ TestFuncs(aom_highbd_12_obmc_variance16x4_c,
+ aom_highbd_12_obmc_variance16x4_neon, 12),
+ TestFuncs(aom_highbd_12_obmc_variance4x16_c,
+ aom_highbd_12_obmc_variance4x16_neon, 12),
+};
+
+INSTANTIATE_TEST_SUITE_P(NEON, ObmcVarianceHBDTest,
+ ::testing::ValuesIn(neon_functions_hbd));
+#endif // HAVE_NEON
+
#if HAVE_SSE4_1
ObmcVarianceHBDTest::ParamType sse4_functions_hbd[] = {
- TestFuncs(aom_highbd_obmc_variance128x128_c,
- aom_highbd_obmc_variance128x128_sse4_1, 8),
- TestFuncs(aom_highbd_obmc_variance128x64_c,
- aom_highbd_obmc_variance128x64_sse4_1, 8),
- TestFuncs(aom_highbd_obmc_variance64x128_c,
- aom_highbd_obmc_variance64x128_sse4_1, 8),
- TestFuncs(aom_highbd_obmc_variance64x64_c,
- aom_highbd_obmc_variance64x64_sse4_1, 8),
- TestFuncs(aom_highbd_obmc_variance64x32_c,
- aom_highbd_obmc_variance64x32_sse4_1, 8),
- TestFuncs(aom_highbd_obmc_variance32x64_c,
- aom_highbd_obmc_variance32x64_sse4_1, 8),
- TestFuncs(aom_highbd_obmc_variance32x32_c,
- aom_highbd_obmc_variance32x32_sse4_1, 8),
- TestFuncs(aom_highbd_obmc_variance32x16_c,
- aom_highbd_obmc_variance32x16_sse4_1, 8),
- TestFuncs(aom_highbd_obmc_variance16x32_c,
- aom_highbd_obmc_variance16x32_sse4_1, 8),
- TestFuncs(aom_highbd_obmc_variance16x16_c,
- aom_highbd_obmc_variance16x16_sse4_1, 8),
- TestFuncs(aom_highbd_obmc_variance16x8_c, aom_highbd_obmc_variance16x8_sse4_1,
- 8),
- TestFuncs(aom_highbd_obmc_variance8x16_c, aom_highbd_obmc_variance8x16_sse4_1,
- 8),
- TestFuncs(aom_highbd_obmc_variance8x8_c, aom_highbd_obmc_variance8x8_sse4_1,
- 8),
- TestFuncs(aom_highbd_obmc_variance8x4_c, aom_highbd_obmc_variance8x4_sse4_1,
- 8),
- TestFuncs(aom_highbd_obmc_variance4x8_c, aom_highbd_obmc_variance4x8_sse4_1,
- 8),
- TestFuncs(aom_highbd_obmc_variance4x4_c, aom_highbd_obmc_variance4x4_sse4_1,
- 8),
+ TestFuncs(aom_highbd_8_obmc_variance128x128_c,
+ aom_highbd_8_obmc_variance128x128_sse4_1, 8),
+ TestFuncs(aom_highbd_8_obmc_variance128x64_c,
+ aom_highbd_8_obmc_variance128x64_sse4_1, 8),
+ TestFuncs(aom_highbd_8_obmc_variance64x128_c,
+ aom_highbd_8_obmc_variance64x128_sse4_1, 8),
+ TestFuncs(aom_highbd_8_obmc_variance64x64_c,
+ aom_highbd_8_obmc_variance64x64_sse4_1, 8),
+ TestFuncs(aom_highbd_8_obmc_variance64x32_c,
+ aom_highbd_8_obmc_variance64x32_sse4_1, 8),
+ TestFuncs(aom_highbd_8_obmc_variance32x64_c,
+ aom_highbd_8_obmc_variance32x64_sse4_1, 8),
+ TestFuncs(aom_highbd_8_obmc_variance32x32_c,
+ aom_highbd_8_obmc_variance32x32_sse4_1, 8),
+ TestFuncs(aom_highbd_8_obmc_variance32x16_c,
+ aom_highbd_8_obmc_variance32x16_sse4_1, 8),
+ TestFuncs(aom_highbd_8_obmc_variance16x32_c,
+ aom_highbd_8_obmc_variance16x32_sse4_1, 8),
+ TestFuncs(aom_highbd_8_obmc_variance16x16_c,
+ aom_highbd_8_obmc_variance16x16_sse4_1, 8),
+ TestFuncs(aom_highbd_8_obmc_variance16x8_c,
+ aom_highbd_8_obmc_variance16x8_sse4_1, 8),
+ TestFuncs(aom_highbd_8_obmc_variance8x16_c,
+ aom_highbd_8_obmc_variance8x16_sse4_1, 8),
+ TestFuncs(aom_highbd_8_obmc_variance8x8_c,
+ aom_highbd_8_obmc_variance8x8_sse4_1, 8),
+ TestFuncs(aom_highbd_8_obmc_variance8x4_c,
+ aom_highbd_8_obmc_variance8x4_sse4_1, 8),
+ TestFuncs(aom_highbd_8_obmc_variance4x8_c,
+ aom_highbd_8_obmc_variance4x8_sse4_1, 8),
+ TestFuncs(aom_highbd_8_obmc_variance4x4_c,
+ aom_highbd_8_obmc_variance4x4_sse4_1, 8),
TestFuncs(aom_highbd_10_obmc_variance128x128_c,
aom_highbd_10_obmc_variance128x128_sse4_1, 10),
TestFuncs(aom_highbd_10_obmc_variance128x64_c,
@@ -386,18 +526,18 @@
TestFuncs(aom_highbd_12_obmc_variance4x4_c,
aom_highbd_12_obmc_variance4x4_sse4_1, 12),
- TestFuncs(aom_highbd_obmc_variance64x16_c,
- aom_highbd_obmc_variance64x16_sse4_1, 8),
- TestFuncs(aom_highbd_obmc_variance16x64_c,
- aom_highbd_obmc_variance16x64_sse4_1, 8),
- TestFuncs(aom_highbd_obmc_variance32x8_c, aom_highbd_obmc_variance32x8_sse4_1,
- 8),
- TestFuncs(aom_highbd_obmc_variance8x32_c, aom_highbd_obmc_variance8x32_sse4_1,
- 8),
- TestFuncs(aom_highbd_obmc_variance16x4_c, aom_highbd_obmc_variance16x4_sse4_1,
- 8),
- TestFuncs(aom_highbd_obmc_variance4x16_c, aom_highbd_obmc_variance4x16_sse4_1,
- 8),
+ TestFuncs(aom_highbd_8_obmc_variance64x16_c,
+ aom_highbd_8_obmc_variance64x16_sse4_1, 8),
+ TestFuncs(aom_highbd_8_obmc_variance16x64_c,
+ aom_highbd_8_obmc_variance16x64_sse4_1, 8),
+ TestFuncs(aom_highbd_8_obmc_variance32x8_c,
+ aom_highbd_8_obmc_variance32x8_sse4_1, 8),
+ TestFuncs(aom_highbd_8_obmc_variance8x32_c,
+ aom_highbd_8_obmc_variance8x32_sse4_1, 8),
+ TestFuncs(aom_highbd_8_obmc_variance16x4_c,
+ aom_highbd_8_obmc_variance16x4_sse4_1, 8),
+ TestFuncs(aom_highbd_8_obmc_variance4x16_c,
+ aom_highbd_8_obmc_variance4x16_sse4_1, 8),
TestFuncs(aom_highbd_10_obmc_variance64x16_c,
aom_highbd_10_obmc_variance64x16_sse4_1, 10),
TestFuncs(aom_highbd_10_obmc_variance16x64_c,
@@ -427,5 +567,5 @@
INSTANTIATE_TEST_SUITE_P(SSE4_1, ObmcVarianceHBDTest,
::testing::ValuesIn(sse4_functions_hbd));
#endif // HAVE_SSE4_1
-#endif // CONFIG_AV1_HIGHBITDEPTH
+#endif // CONFIG_AV1_HIGHBITDEPTH && !CONFIG_REALTIME_ONLY
} // namespace
diff --git a/test/pickrst_test.cc b/test/pickrst_test.cc
index 131e1dd..534d9b1 100644
--- a/test/pickrst_test.cc
+++ b/test/pickrst_test.cc
@@ -43,7 +43,7 @@
class PixelProjErrorTest
: public ::testing::TestWithParam<PixelProjErrorTestParam> {
public:
- virtual void SetUp() {
+ void SetUp() override {
target_func_ = GET_PARAM(0);
src_ = (uint8_t *)(aom_malloc(MAX_DATA_BLOCK * MAX_DATA_BLOCK *
sizeof(*src_)));
@@ -58,7 +58,7 @@
sizeof(*flt1_)));
ASSERT_NE(flt1_, nullptr);
}
- virtual void TearDown() {
+ void TearDown() override {
aom_free(src_);
aom_free(dgd_);
aom_free(flt0_);
@@ -215,7 +215,7 @@
class PixelProjHighbdErrorTest
: public ::testing::TestWithParam<PixelProjErrorTestParam> {
public:
- virtual void SetUp() {
+ void SetUp() override {
target_func_ = GET_PARAM(0);
src_ =
(uint16_t *)aom_malloc(MAX_DATA_BLOCK * MAX_DATA_BLOCK * sizeof(*src_));
@@ -230,7 +230,7 @@
(int32_t *)aom_malloc(MAX_DATA_BLOCK * MAX_DATA_BLOCK * sizeof(*flt1_));
ASSERT_NE(flt1_, nullptr);
}
- virtual void TearDown() {
+ void TearDown() override {
aom_free(src_);
aom_free(dgd_);
aom_free(flt0_);
@@ -386,7 +386,7 @@
class GetProjSubspaceTest
: public ::testing::TestWithParam<GetProjSubspaceTestParam> {
public:
- virtual void SetUp() {
+ void SetUp() override {
target_func_ = GET_PARAM(0);
src_ = (uint8_t *)(aom_malloc(MAX_DATA_BLOCK * MAX_DATA_BLOCK *
sizeof(*src_)));
@@ -401,7 +401,7 @@
sizeof(*flt1_)));
ASSERT_NE(flt1_, nullptr);
}
- virtual void TearDown() {
+ void TearDown() override {
aom_free(src_);
aom_free(dgd_);
aom_free(flt0_);
@@ -432,7 +432,9 @@
const int flt0_stride = MAX_DATA_BLOCK;
const int flt1_stride = MAX_DATA_BLOCK;
sgr_params_type params;
- const int iters = run_times == 1 ? kIterations : 4;
+ const int iters = run_times == 1 ? kIterations : 3;
+ static constexpr int kR0[3] = { 1, 1, 0 };
+ static constexpr int kR1[3] = { 1, 0, 1 };
for (int iter = 0; iter < iters && !HasFatalFailure(); ++iter) {
int64_t C_ref[2] = { 0 }, C_test[2] = { 0 };
int64_t H_ref[2][2] = { { 0, 0 }, { 0, 0 } };
@@ -444,10 +446,8 @@
flt1_[i] = rng_.Rand15Signed();
}
- params.r[0] = run_times == 1 ? (rng_.Rand8() % MAX_RADIUS) : 1;
- params.r[1] = run_times == 1 ? (rng_.Rand8() % MAX_RADIUS) : 1;
- params.s[0] = run_times == 1 ? (rng_.Rand8() % MAX_RADIUS) : (iter % 2);
- params.s[1] = run_times == 1 ? (rng_.Rand8() % MAX_RADIUS) : (iter / 2);
+ params.r[0] = run_times == 1 ? (rng_.Rand8() % MAX_RADIUS) : kR0[iter];
+ params.r[1] = run_times == 1 ? (rng_.Rand8() % MAX_RADIUS) : kR1[iter];
uint8_t *dgd = dgd_;
uint8_t *src = src_;
@@ -492,6 +492,8 @@
const int flt1_stride = MAX_DATA_BLOCK;
sgr_params_type params;
const int iters = kIterations;
+ static constexpr int kR0[3] = { 1, 1, 0 };
+ static constexpr int kR1[3] = { 1, 0, 1 };
for (int iter = 0; iter < iters && !HasFatalFailure(); ++iter) {
int64_t C_ref[2] = { 0 }, C_test[2] = { 0 };
int64_t H_ref[2][2] = { { 0, 0 }, { 0, 0 } };
@@ -502,10 +504,8 @@
flt0_[i] = rng_.Rand15Signed();
flt1_[i] = rng_.Rand15Signed();
}
- params.r[0] = 1;
- params.r[1] = 1;
- params.s[0] = rng_.Rand8() % MAX_RADIUS;
- params.s[1] = rng_.Rand8() % MAX_RADIUS;
+ params.r[0] = kR0[iter % 3];
+ params.r[1] = kR1[iter % 3];
uint8_t *dgd = dgd_;
uint8_t *src = src_;
@@ -546,6 +546,12 @@
::testing::Values(av1_calc_proj_params_avx2));
#endif // HAVE_AVX2
+#if HAVE_NEON
+
+INSTANTIATE_TEST_SUITE_P(NEON, GetProjSubspaceTest,
+ ::testing::Values(av1_calc_proj_params_neon));
+#endif // HAVE_NEON
+
} // namespace get_proj_subspace_test_lowbd
#if CONFIG_AV1_HIGHBITDEPTH
@@ -565,7 +571,7 @@
class GetProjSubspaceTestHBD
: public ::testing::TestWithParam<GetProjSubspaceHBDTestParam> {
public:
- virtual void SetUp() {
+ void SetUp() override {
target_func_ = GET_PARAM(0);
src_ = (uint16_t *)(aom_malloc(MAX_DATA_BLOCK * MAX_DATA_BLOCK *
sizeof(*src_)));
@@ -580,7 +586,7 @@
sizeof(*flt1_)));
ASSERT_NE(flt1_, nullptr);
}
- virtual void TearDown() {
+ void TearDown() override {
aom_free(src_);
aom_free(dgd_);
aom_free(flt0_);
@@ -611,7 +617,9 @@
const int flt0_stride = MAX_DATA_BLOCK;
const int flt1_stride = MAX_DATA_BLOCK;
sgr_params_type params;
- const int iters = run_times == 1 ? kIterations : 4;
+ const int iters = run_times == 1 ? kIterations : 3;
+ static constexpr int kR0[3] = { 1, 1, 0 };
+ static constexpr int kR1[3] = { 1, 0, 1 };
for (int iter = 0; iter < iters && !HasFatalFailure(); ++iter) {
int64_t C_ref[2] = { 0 }, C_test[2] = { 0 };
int64_t H_ref[2][2] = { { 0, 0 }, { 0, 0 } };
@@ -623,10 +631,8 @@
flt1_[i] = rng_.Rand15Signed();
}
- params.r[0] = run_times == 1 ? (rng_.Rand8() % MAX_RADIUS) : 1;
- params.r[1] = run_times == 1 ? (rng_.Rand8() % MAX_RADIUS) : 1;
- params.s[0] = run_times == 1 ? (rng_.Rand8() % MAX_RADIUS) : (iter % 2);
- params.s[1] = run_times == 1 ? (rng_.Rand8() % MAX_RADIUS) : (iter / 2);
+ params.r[0] = run_times == 1 ? (rng_.Rand8() % MAX_RADIUS) : kR0[iter];
+ params.r[1] = run_times == 1 ? (rng_.Rand8() % MAX_RADIUS) : kR1[iter];
uint8_t *dgd = CONVERT_TO_BYTEPTR(dgd_);
uint8_t *src = CONVERT_TO_BYTEPTR(src_);
@@ -671,6 +677,8 @@
const int flt1_stride = MAX_DATA_BLOCK;
sgr_params_type params;
const int iters = kIterations;
+ static constexpr int kR0[3] = { 1, 1, 0 };
+ static constexpr int kR1[3] = { 1, 0, 1 };
for (int iter = 0; iter < iters && !HasFatalFailure(); ++iter) {
int64_t C_ref[2] = { 0 }, C_test[2] = { 0 };
int64_t H_ref[2][2] = { { 0, 0 }, { 0, 0 } };
@@ -681,10 +689,8 @@
flt0_[i] = rng_.Rand15Signed();
flt1_[i] = rng_.Rand15Signed();
}
- params.r[0] = 1;
- params.r[1] = 1;
- params.s[0] = rng_.Rand8() % MAX_RADIUS;
- params.s[1] = rng_.Rand8() % MAX_RADIUS;
+ params.r[0] = kR0[iter % 3];
+ params.r[1] = kR1[iter % 3];
uint8_t *dgd = CONVERT_TO_BYTEPTR(dgd_);
uint8_t *src = CONVERT_TO_BYTEPTR(src_);
@@ -728,6 +734,11 @@
::testing::Values(av1_calc_proj_params_high_bd_avx2));
#endif // HAVE_AVX2
+#if HAVE_NEON
+
+INSTANTIATE_TEST_SUITE_P(NEON, GetProjSubspaceTestHBD,
+ ::testing::Values(av1_calc_proj_params_high_bd_neon));
+#endif // HAVE_NEON
} // namespace get_proj_subspace_test_hbd
#endif // CONFIG_AV1_HIGHBITDEPTH
diff --git a/test/postproc_filters_test.cc b/test/postproc_filters_test.cc
index 37de5d2..9584dd8 100644
--- a/test/postproc_filters_test.cc
+++ b/test/postproc_filters_test.cc
@@ -30,13 +30,13 @@
: EncoderTest(GET_PARAM(0)), set_skip_postproc_filtering_(false),
frame_number_(0), cpu_used_(GET_PARAM(1)), bd_(GET_PARAM(2)) {}
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(::libaom_test::kAllIntra);
cfg_.g_input_bit_depth = bd_;
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
frame_number_ = video->frame();
if (frame_number_ == 0) {
encoder->Control(AOME_SET_CPUUSED, cpu_used_);
@@ -53,14 +53,14 @@
}
}
- virtual void FramePktHook(const aom_codec_cx_pkt_t *pkt) {
+ void FramePktHook(const aom_codec_cx_pkt_t *pkt) override {
::libaom_test::MD5 md5_enc;
md5_enc.Add(reinterpret_cast<uint8_t *>(pkt->data.frame.buf),
pkt->data.frame.sz);
md5_enc_.push_back(md5_enc.Get());
}
- virtual void PostEncodeFrameHook(::libaom_test::Encoder *encoder) {
+ void PostEncodeFrameHook(::libaom_test::Encoder *encoder) override {
const aom_image_t *img_enc = encoder->GetPreviewFrame();
if (!set_skip_postproc_filtering_) {
ASSERT_NE(img_enc, nullptr);
diff --git a/test/quant_test.cc b/test/quant_test.cc
index a042af1..afbabb3 100644
--- a/test/quant_test.cc
+++ b/test/quant_test.cc
@@ -32,15 +32,15 @@
public ::libaom_test::EncoderTest {
protected:
QMTest() : EncoderTest(GET_PARAM(0)) {}
- virtual ~QMTest() {}
+ ~QMTest() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(GET_PARAM(1));
set_cpu_used_ = GET_PARAM(2);
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AOME_SET_CPUUSED, set_cpu_used_);
encoder->Control(AV1E_SET_ENABLE_QM, 1);
@@ -119,9 +119,9 @@
quant_param_(GET_PARAM(2)), rc_end_usage_(GET_PARAM(3)) {
quant_bound_violated_ = false;
}
- virtual ~QuantizerBoundsCheckTestLarge() {}
+ ~QuantizerBoundsCheckTestLarge() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(encoding_mode_);
const aom_rational timebase = { 1, 30 };
cfg_.g_timebase = timebase;
@@ -135,17 +135,17 @@
}
}
- virtual bool DoDecode() const { return 1; }
+ bool DoDecode() const override { return true; }
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AOME_SET_CPUUSED, 5);
}
}
- virtual bool HandleDecodeResult(const aom_codec_err_t res_dec,
- libaom_test::Decoder *decoder) {
+ bool HandleDecodeResult(const aom_codec_err_t res_dec,
+ libaom_test::Decoder *decoder) override {
EXPECT_EQ(AOM_CODEC_OK, res_dec) << decoder->DecodeError();
if (AOM_CODEC_OK == res_dec) {
aom_codec_ctx_t *ctx_dec = decoder->GetDecoder();
diff --git a/test/quantize_func_test.cc b/test/quantize_func_test.cc
index 04e8306..328d5b1 100644
--- a/test/quantize_func_test.cc
+++ b/test/quantize_func_test.cc
@@ -100,9 +100,9 @@
tx_size_(GET_TEMPLATE_PARAM(2)), type_(GET_TEMPLATE_PARAM(3)),
bd_(GET_TEMPLATE_PARAM(4)) {}
- virtual ~QuantizeTestBase() {}
+ ~QuantizeTestBase() override = default;
- virtual void SetUp() {
+ void SetUp() override {
qtab_ = reinterpret_cast<QuanTable *>(aom_memalign(32, sizeof(*qtab_)));
ASSERT_NE(qtab_, nullptr);
const int n_coeffs = coeff_num();
@@ -112,7 +112,7 @@
InitQuantizer();
}
- virtual void TearDown() {
+ void TearDown() override {
aom_free(qtab_);
qtab_ = nullptr;
aom_free(coeff_);
@@ -149,8 +149,8 @@
// Testing uses luminance quantization table
const int16_t *zbin = qtab_->quant.y_zbin[q];
- const int16_t *round = 0;
- const int16_t *quant = 0;
+ const int16_t *round = nullptr;
+ const int16_t *quant = nullptr;
if (type_ == TYPE_B) {
round = qtab_->quant.y_round[q];
quant = qtab_->quant.y_quant[q];
diff --git a/test/ratectrl_rtc_test.cc b/test/ratectrl_rtc_test.cc
index 0d8d48f..cc054b6 100644
--- a/test/ratectrl_rtc_test.cc
+++ b/test/ratectrl_rtc_test.cc
@@ -36,12 +36,14 @@
RcInterfaceTest()
: EncoderTest(GET_PARAM(0)), aq_mode_(GET_PARAM(1)), key_interval_(3000),
encoder_exit_(false), layer_frame_cnt_(0), superframe_cnt_(0),
- dynamic_temporal_layers_(false), dynamic_spatial_layers_(false) {
+ frame_cnt_(0), dynamic_temporal_layers_(false),
+ dynamic_spatial_layers_(false), num_drops_(0), max_consec_drop_(0),
+ frame_drop_thresh_(0) {
memset(&svc_params_, 0, sizeof(svc_params_));
memset(&layer_id_, 0, sizeof(layer_id_));
}
- ~RcInterfaceTest() override {}
+ ~RcInterfaceTest() override = default;
protected:
void SetUp() override { InitializeConfig(::libaom_test::kRealTime); }
@@ -57,10 +59,15 @@
if (video->frame() == 0 && layer_frame_cnt_ == 0) {
encoder->Control(AOME_SET_CPUUSED, 7);
encoder->Control(AV1E_SET_AQ_MODE, aq_mode_);
- encoder->Control(AV1E_SET_TUNE_CONTENT, AOM_CONTENT_DEFAULT);
+ if (rc_cfg_.is_screen) {
+ encoder->Control(AV1E_SET_TUNE_CONTENT, AOM_CONTENT_SCREEN);
+ } else {
+ encoder->Control(AV1E_SET_TUNE_CONTENT, AOM_CONTENT_DEFAULT);
+ }
encoder->Control(AOME_SET_MAX_INTRA_BITRATE_PCT,
rc_cfg_.max_intra_bitrate_pct);
if (use_svc) encoder->Control(AV1E_SET_SVC_PARAMS, &svc_params_);
+ encoder->Control(AV1E_SET_MAX_CONSEC_FRAME_DROP_CBR, max_consec_drop_);
}
// SVC specific settings
if (use_svc) {
@@ -140,20 +147,24 @@
return;
}
layer_frame_cnt_++;
+ frame_cnt_++;
if (layer_id_.spatial_layer_id == rc_cfg_.ss_number_layers - 1)
superframe_cnt_++;
int qp;
encoder->Control(AOME_GET_LAST_QUANTIZER, &qp);
- rc_api_->ComputeQP(frame_params_);
- ASSERT_EQ(rc_api_->GetQP(), qp);
- int encoder_lpf_level;
- encoder->Control(AOME_GET_LOOPFILTER_LEVEL, &encoder_lpf_level);
- aom::AV1LoopfilterLevel loopfilter_level = rc_api_->GetLoopfilterLevel();
- ASSERT_EQ(loopfilter_level.filter_level[0], encoder_lpf_level);
- aom::AV1CdefInfo cdef_level = rc_api_->GetCdefInfo();
- int cdef_y_strengths[16];
- encoder->Control(AV1E_GET_LUMA_CDEF_STRENGTH, cdef_y_strengths);
- ASSERT_EQ(cdef_level.cdef_strength_y, cdef_y_strengths[0]);
+ if (rc_api_->ComputeQP(frame_params_) == aom::FrameDropDecision::kOk) {
+ ASSERT_EQ(rc_api_->GetQP(), qp) << "at frame " << frame_cnt_ - 1;
+ int encoder_lpf_level;
+ encoder->Control(AOME_GET_LOOPFILTER_LEVEL, &encoder_lpf_level);
+ aom::AV1LoopfilterLevel loopfilter_level = rc_api_->GetLoopfilterLevel();
+ ASSERT_EQ(loopfilter_level.filter_level[0], encoder_lpf_level);
+ aom::AV1CdefInfo cdef_level = rc_api_->GetCdefInfo();
+ int cdef_y_strengths[16];
+ encoder->Control(AV1E_GET_LUMA_CDEF_STRENGTH, cdef_y_strengths);
+ ASSERT_EQ(cdef_level.cdef_strength_y, cdef_y_strengths[0]);
+ } else {
+ num_drops_++;
+ }
}
void FramePktHook(const aom_codec_cx_pkt_t *pkt) override {
@@ -181,6 +192,43 @@
ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
}
+ void RunOneLayerScreen() {
+ key_interval_ = 10000;
+ SetConfig();
+ rc_cfg_.is_screen = true;
+ rc_cfg_.width = 352;
+ rc_cfg_.height = 288;
+ rc_api_ = aom::AV1RateControlRTC::Create(rc_cfg_);
+ frame_params_.spatial_layer_id = 0;
+ frame_params_.temporal_layer_id = 0;
+
+ ::libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352,
+ 288, 30, 1, 0, 140);
+
+ ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
+ }
+
+ void RunOneLayerDropFramesCBR() {
+ key_interval_ = 10000;
+ max_consec_drop_ = 8;
+ frame_drop_thresh_ = 30;
+ SetConfig();
+ rc_cfg_.target_bandwidth = 100;
+ cfg_.rc_target_bitrate = 100;
+ rc_cfg_.max_quantizer = 50;
+ cfg_.rc_max_quantizer = 50;
+ rc_api_ = aom::AV1RateControlRTC::Create(rc_cfg_);
+ frame_params_.spatial_layer_id = 0;
+ frame_params_.temporal_layer_id = 0;
+
+ ::libaom_test::I420VideoSource video("niklas_640_480_30.yuv", 640, 480, 30,
+ 1, 0, kNumFrames);
+
+ ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
+ // Check that some frames were dropped, otherwise test has no value.
+ ASSERT_GE(num_drops_, 1);
+ }
+
void RunOneLayerPeriodicKey() {
key_interval_ = 100;
SetConfig();
@@ -270,6 +318,8 @@
rc_cfg_.max_quantizers[0] = 52;
rc_cfg_.min_quantizers[0] = 2;
rc_cfg_.aq_mode = aq_mode_;
+ rc_cfg_.frame_drop_thresh = frame_drop_thresh_;
+ rc_cfg_.max_consec_drop = max_consec_drop_;
// Encoder settings for ground truth.
cfg_.g_w = 640;
@@ -288,6 +338,7 @@
cfg_.rc_target_bitrate = 1000;
cfg_.kf_min_dist = key_interval_;
cfg_.kf_max_dist = key_interval_;
+ cfg_.rc_dropframe_thresh = frame_drop_thresh_;
}
void SetConfigSvc(int number_spatial_layers, int number_temporal_layers) {
@@ -425,14 +476,22 @@
aom_svc_layer_id_t layer_id_;
int layer_frame_cnt_;
int superframe_cnt_;
+ int frame_cnt_;
bool dynamic_temporal_layers_;
bool dynamic_spatial_layers_;
+ int num_drops_;
+ int max_consec_drop_;
+ int frame_drop_thresh_;
};
TEST_P(RcInterfaceTest, OneLayer) { RunOneLayer(); }
+TEST_P(RcInterfaceTest, OneLayerDropFramesCBR) { RunOneLayerDropFramesCBR(); }
+
TEST_P(RcInterfaceTest, OneLayerPeriodicKey) { RunOneLayerPeriodicKey(); }
+TEST_P(RcInterfaceTest, OneLayerScreen) { RunOneLayerScreen(); }
+
TEST_P(RcInterfaceTest, Svc) { RunSvc(); }
TEST_P(RcInterfaceTest, SvcPeriodicKey) { RunSvcPeriodicKey(); }
diff --git a/test/reconinter_test.cc b/test/reconinter_test.cc
index b45b7bb..ee1a989 100644
--- a/test/reconinter_test.cc
+++ b/test/reconinter_test.cc
@@ -28,18 +28,18 @@
namespace {
using libaom_test::ACMRandom;
-typedef void (*buildcompdiffwtdmaskd_func)(uint8_t *mask,
- DIFFWTD_MASK_TYPE mask_type,
- const uint8_t *src0, int src0_stride,
- const uint8_t *src1, int src1_stride,
- int h, int w);
+using BuildCompDiffWtdMaskFunc = void (*)(uint8_t *mask,
+ DIFFWTD_MASK_TYPE mask_type,
+ const uint8_t *src0, int src0_stride,
+ const uint8_t *src1, int src1_stride,
+ int h, int w);
-typedef std::tuple<BLOCK_SIZE, buildcompdiffwtdmaskd_func>
- BuildCompDiffwtdMaskDParam;
+using BuildCompDiffwtdMaskDParam =
+ std::tuple<BLOCK_SIZE, BuildCompDiffWtdMaskFunc>;
-#if HAVE_SSE4_1
+#if HAVE_SSE4_1 || HAVE_AVX2 || HAVE_NEON
::testing::internal::ParamGenerator<BuildCompDiffwtdMaskDParam> BuildParams(
- buildcompdiffwtdmaskd_func filter) {
+ BuildCompDiffWtdMaskFunc filter) {
return ::testing::Combine(::testing::Range(BLOCK_4X4, BLOCK_SIZES_ALL),
::testing::Values(filter));
}
@@ -48,175 +48,53 @@
class BuildCompDiffwtdMaskTest
: public ::testing::TestWithParam<BuildCompDiffwtdMaskDParam> {
public:
- virtual ~BuildCompDiffwtdMaskTest() {}
+ BuildCompDiffwtdMaskTest() : rnd_(ACMRandom::DeterministicSeed()) {}
+ ~BuildCompDiffwtdMaskTest() override = default;
- virtual void TearDown() {}
- void RunTest(buildcompdiffwtdmaskd_func test_impl, const int is_speed,
- const DIFFWTD_MASK_TYPE type);
+ protected:
+ void RunTest(BuildCompDiffWtdMaskFunc test_impl, bool is_speed,
+ const DIFFWTD_MASK_TYPE type) {
+ const int sb_type = GET_PARAM(0);
+ const int width = block_size_wide[sb_type];
+ const int height = block_size_high[sb_type];
+ DECLARE_ALIGNED(16, uint8_t, mask_ref[MAX_SB_SQUARE]);
+ DECLARE_ALIGNED(16, uint8_t, mask_test[MAX_SB_SQUARE]);
+ DECLARE_ALIGNED(16, uint8_t, src0[MAX_SB_SQUARE]);
+ DECLARE_ALIGNED(16, uint8_t, src1[MAX_SB_SQUARE]);
+ for (int i = 0; i < width * height; i++) {
+ src0[i] = rnd_.Rand8();
+ src1[i] = rnd_.Rand8();
+ }
+ const int run_times = is_speed ? (10000000 / (width + height)) : 1;
+ aom_usec_timer timer;
+ aom_usec_timer_start(&timer);
+ for (int i = 0; i < run_times; ++i) {
+ av1_build_compound_diffwtd_mask_c(mask_ref, type, src0, width, src1,
+ width, height, width);
+ }
+ const double t1 = get_time_mark(&timer);
+ aom_usec_timer_start(&timer);
+ for (int i = 0; i < run_times; ++i) {
+ test_impl(mask_test, type, src0, width, src1, width, height, width);
+ }
+ const double t2 = get_time_mark(&timer);
+ if (is_speed) {
+ printf("mask %d %3dx%-3d:%7.2f/%7.2fns", type, width, height, t1, t2);
+ printf("(%3.2f)\n", t1 / t2);
+ }
+ for (int r = 0; r < height; ++r) {
+ for (int c = 0; c < width; ++c) {
+ ASSERT_EQ(mask_ref[c + r * width], mask_test[c + r * width])
+ << "[" << r << "," << c << "] " << run_times << " @ " << width
+ << "x" << height << " inv " << type;
+ }
+ }
+ }
private:
ACMRandom rnd_;
};
-
-typedef void (*buildcompdiffwtdmaskd16_func)(
- uint8_t *mask, DIFFWTD_MASK_TYPE mask_type, const CONV_BUF_TYPE *src0,
- int src0_stride, const CONV_BUF_TYPE *src1, int src1_stride, int h, int w,
- ConvolveParams *conv_params, int bd);
-
-typedef std::tuple<int, buildcompdiffwtdmaskd16_func, BLOCK_SIZE>
- BuildCompDiffwtdMaskD16Param;
-
-#if HAVE_SSE4_1 || HAVE_NEON
-::testing::internal::ParamGenerator<BuildCompDiffwtdMaskD16Param> BuildParams(
- buildcompdiffwtdmaskd16_func filter) {
- return ::testing::Combine(::testing::Range(8, 13, 2),
- ::testing::Values(filter),
- ::testing::Range(BLOCK_4X4, BLOCK_SIZES_ALL));
-}
-#endif
-class BuildCompDiffwtdMaskD16Test
- : public ::testing::TestWithParam<BuildCompDiffwtdMaskD16Param> {
- public:
- ~BuildCompDiffwtdMaskD16Test() {}
- virtual void TearDown() {}
- void SetUp() { rnd_.Reset(ACMRandom::DeterministicSeed()); }
-
- protected:
- void RunCheckOutput(buildcompdiffwtdmaskd16_func test_impl);
- void RunSpeedTest(buildcompdiffwtdmaskd16_func test_impl,
- DIFFWTD_MASK_TYPE mask_type);
- libaom_test::ACMRandom rnd_;
-}; // class BuildCompDiffwtdMaskD16Test
-GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(BuildCompDiffwtdMaskD16Test);
-
-void BuildCompDiffwtdMaskD16Test::RunCheckOutput(
- buildcompdiffwtdmaskd16_func test_impl) {
- const int block_idx = GET_PARAM(2);
- const int bd = GET_PARAM(0);
- const int width = block_size_wide[block_idx];
- const int height = block_size_high[block_idx];
- DECLARE_ALIGNED(16, uint8_t, mask_ref[2 * MAX_SB_SQUARE]);
- DECLARE_ALIGNED(16, uint8_t, mask_test[2 * MAX_SB_SQUARE]);
- DECLARE_ALIGNED(32, uint16_t, src0[MAX_SB_SQUARE]);
- DECLARE_ALIGNED(32, uint16_t, src1[MAX_SB_SQUARE]);
-
- ConvolveParams conv_params =
- get_conv_params_no_round(0, 0, nullptr, 0, 1, bd);
-
- int in_precision =
- bd + 2 * FILTER_BITS - conv_params.round_0 - conv_params.round_1 + 2;
-
- for (int i = 0; i < MAX_SB_SQUARE; i++) {
- src0[i] = rnd_.Rand16() & ((1 << in_precision) - 1);
- src1[i] = rnd_.Rand16() & ((1 << in_precision) - 1);
- }
-
- for (int mask_type = 0; mask_type < DIFFWTD_MASK_TYPES; mask_type++) {
- av1_build_compound_diffwtd_mask_d16_c(
- mask_ref, (DIFFWTD_MASK_TYPE)mask_type, src0, width, src1, width,
- height, width, &conv_params, bd);
-
- test_impl(mask_test, (DIFFWTD_MASK_TYPE)mask_type, src0, width, src1, width,
- height, width, &conv_params, bd);
-
- for (int r = 0; r < height; ++r) {
- for (int c = 0; c < width; ++c) {
- ASSERT_EQ(mask_ref[c + r * width], mask_test[c + r * width])
- << "Mismatch at unit tests for BuildCompDiffwtdMaskD16Test\n"
- << " Pixel mismatch at index "
- << "[" << r << "," << c << "] "
- << " @ " << width << "x" << height << " inv " << mask_type;
- }
- }
- }
-}
-
-void BuildCompDiffwtdMaskD16Test::RunSpeedTest(
- buildcompdiffwtdmaskd16_func test_impl, DIFFWTD_MASK_TYPE mask_type) {
- const int block_idx = GET_PARAM(2);
- const int bd = GET_PARAM(0);
- const int width = block_size_wide[block_idx];
- const int height = block_size_high[block_idx];
- DECLARE_ALIGNED(16, uint8_t, mask[MAX_SB_SQUARE]);
- DECLARE_ALIGNED(32, uint16_t, src0[MAX_SB_SQUARE]);
- DECLARE_ALIGNED(32, uint16_t, src1[MAX_SB_SQUARE]);
-
- ConvolveParams conv_params =
- get_conv_params_no_round(0, 0, nullptr, 0, 1, bd);
-
- int in_precision =
- bd + 2 * FILTER_BITS - conv_params.round_0 - conv_params.round_1 + 2;
-
- for (int i = 0; i < MAX_SB_SQUARE; i++) {
- src0[i] = rnd_.Rand16() & ((1 << in_precision) - 1);
- src1[i] = rnd_.Rand16() & ((1 << in_precision) - 1);
- }
-
- const int num_loops = 10000000 / (width + height);
- aom_usec_timer timer;
- aom_usec_timer_start(&timer);
-
- for (int i = 0; i < num_loops; ++i)
- av1_build_compound_diffwtd_mask_d16_c(mask, mask_type, src0, width, src1,
- width, height, width, &conv_params,
- bd);
-
- aom_usec_timer_mark(&timer);
- const int elapsed_time = static_cast<int>(aom_usec_timer_elapsed(&timer));
-
- aom_usec_timer timer1;
- aom_usec_timer_start(&timer1);
-
- for (int i = 0; i < num_loops; ++i)
- test_impl(mask, mask_type, src0, width, src1, width, height, width,
- &conv_params, bd);
-
- aom_usec_timer_mark(&timer1);
- const int elapsed_time1 = static_cast<int>(aom_usec_timer_elapsed(&timer1));
- printf("av1_build_compound_diffwtd_mask_d16 %3dx%-3d: %7.2f \n", width,
- height, elapsed_time / double(elapsed_time1));
-}
-#if HAVE_SSE4_1
-void BuildCompDiffwtdMaskTest::RunTest(buildcompdiffwtdmaskd_func test_impl,
- const int is_speed,
- const DIFFWTD_MASK_TYPE type) {
- const int sb_type = GET_PARAM(0);
- const int width = block_size_wide[sb_type];
- const int height = block_size_high[sb_type];
- DECLARE_ALIGNED(16, uint8_t, mask_ref[MAX_SB_SQUARE]);
- DECLARE_ALIGNED(16, uint8_t, mask_test[MAX_SB_SQUARE]);
- DECLARE_ALIGNED(16, uint8_t, src0[MAX_SB_SQUARE]);
- DECLARE_ALIGNED(16, uint8_t, src1[MAX_SB_SQUARE]);
- ACMRandom rnd(ACMRandom::DeterministicSeed());
- for (int i = 0; i < width * height; i++) {
- src0[i] = rnd.Rand8();
- src1[i] = rnd.Rand8();
- }
- const int run_times = is_speed ? (10000000 / (width + height)) : 1;
- aom_usec_timer timer;
- aom_usec_timer_start(&timer);
- for (int i = 0; i < run_times; ++i) {
- av1_build_compound_diffwtd_mask_c(mask_ref, type, src0, width, src1, width,
- height, width);
- }
- const double t1 = get_time_mark(&timer);
- aom_usec_timer_start(&timer);
- for (int i = 0; i < run_times; ++i) {
- test_impl(mask_test, type, src0, width, src1, width, height, width);
- }
- const double t2 = get_time_mark(&timer);
- if (is_speed) {
- printf("mask %d %3dx%-3d:%7.2f/%7.2fns", type, width, height, t1, t2);
- printf("(%3.2f)\n", t1 / t2);
- }
- for (int r = 0; r < height; ++r) {
- for (int c = 0; c < width; ++c) {
- ASSERT_EQ(mask_ref[c + r * width], mask_test[c + r * width])
- << "[" << r << "," << c << "] " << run_times << " @ " << width << "x"
- << height << " inv " << type;
- }
- }
-}
+GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(BuildCompDiffwtdMaskTest);
TEST_P(BuildCompDiffwtdMaskTest, match) {
RunTest(GET_PARAM(1), 0, DIFFWTD_38);
@@ -226,7 +104,246 @@
RunTest(GET_PARAM(1), 1, DIFFWTD_38);
RunTest(GET_PARAM(1), 1, DIFFWTD_38_INV);
}
+
+#if HAVE_SSE4_1
+INSTANTIATE_TEST_SUITE_P(SSE4_1, BuildCompDiffwtdMaskTest,
+ BuildParams(av1_build_compound_diffwtd_mask_sse4_1));
#endif
+
+#if HAVE_AVX2
+INSTANTIATE_TEST_SUITE_P(AVX2, BuildCompDiffwtdMaskTest,
+ BuildParams(av1_build_compound_diffwtd_mask_avx2));
+#endif
+
+#if HAVE_NEON
+INSTANTIATE_TEST_SUITE_P(NEON, BuildCompDiffwtdMaskTest,
+ BuildParams(av1_build_compound_diffwtd_mask_neon));
+#endif
+
+#if CONFIG_AV1_HIGHBITDEPTH
+
+using BuildCompDiffWtdMaskHighbdFunc =
+ void (*)(uint8_t *mask, DIFFWTD_MASK_TYPE mask_type, const uint8_t *src0,
+ int src0_stride, const uint8_t *src1, int src1_stride, int h,
+ int w, int bd);
+
+using BuildCompDiffwtdMaskHighbdParam =
+ std::tuple<BLOCK_SIZE, int, BuildCompDiffWtdMaskHighbdFunc>;
+
+#if HAVE_SSSE3 || HAVE_AVX2 || HAVE_NEON
+::testing::internal::ParamGenerator<BuildCompDiffwtdMaskHighbdParam>
+BuildParamsHighbd(BuildCompDiffWtdMaskHighbdFunc filter) {
+ return ::testing::Combine(::testing::Range(BLOCK_4X4, BLOCK_SIZES_ALL),
+ ::testing::Values(8, 10, 12),
+ ::testing::Values(filter));
+}
+#endif
+
+class BuildCompDiffwtdMaskHighbdTest
+ : public ::testing::TestWithParam<BuildCompDiffwtdMaskHighbdParam> {
+ public:
+ BuildCompDiffwtdMaskHighbdTest() : rnd_(ACMRandom::DeterministicSeed()) {}
+ ~BuildCompDiffwtdMaskHighbdTest() override = default;
+
+ protected:
+ void RunTest(BuildCompDiffWtdMaskHighbdFunc test_impl, bool is_speed,
+ const DIFFWTD_MASK_TYPE type) {
+ const int sb_type = GET_PARAM(0);
+ const int bd = GET_PARAM(1);
+ const int width = block_size_wide[sb_type];
+ const int height = block_size_high[sb_type];
+ const int mask = (1 << bd) - 1;
+ DECLARE_ALIGNED(16, uint8_t, mask_ref[MAX_SB_SQUARE]);
+ DECLARE_ALIGNED(16, uint8_t, mask_test[MAX_SB_SQUARE]);
+ DECLARE_ALIGNED(16, uint16_t, src0[MAX_SB_SQUARE]);
+ DECLARE_ALIGNED(16, uint16_t, src1[MAX_SB_SQUARE]);
+ for (int i = 0; i < width * height; i++) {
+ src0[i] = rnd_.Rand16() & mask;
+ src1[i] = rnd_.Rand16() & mask;
+ }
+ const int run_times = is_speed ? (10000000 / (width + height)) : 1;
+ aom_usec_timer timer;
+
+ aom_usec_timer_start(&timer);
+ for (int i = 0; i < run_times; ++i) {
+ uint8_t *src0_8 = CONVERT_TO_BYTEPTR(src0);
+ uint8_t *src1_8 = CONVERT_TO_BYTEPTR(src1);
+ av1_build_compound_diffwtd_mask_highbd_c(
+ mask_ref, type, src0_8, width, src1_8, width, height, width, bd);
+ }
+ const double t1 = get_time_mark(&timer);
+
+ aom_usec_timer_start(&timer);
+ for (int i = 0; i < run_times; ++i) {
+ uint8_t *src0_8 = CONVERT_TO_BYTEPTR(src0);
+ uint8_t *src1_8 = CONVERT_TO_BYTEPTR(src1);
+ test_impl(mask_test, type, src0_8, width, src1_8, width, height, width,
+ bd);
+ }
+ const double t2 = get_time_mark(&timer);
+
+ if (is_speed) {
+ printf("mask %d %3dx%-3d:%7.2f/%7.2fns", type, width, height, t1, t2);
+ printf("(%3.2f)\n", t1 / t2);
+ }
+ for (int r = 0; r < height; ++r) {
+ for (int c = 0; c < width; ++c) {
+ ASSERT_EQ(mask_ref[c + r * width], mask_test[c + r * width])
+ << "[" << r << "," << c << "] " << run_times << " @ " << width
+ << "x" << height << " inv " << type;
+ }
+ }
+ }
+
+ private:
+ ACMRandom rnd_;
+};
+GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(BuildCompDiffwtdMaskHighbdTest);
+
+TEST_P(BuildCompDiffwtdMaskHighbdTest, match) {
+ RunTest(GET_PARAM(2), 0, DIFFWTD_38);
+ RunTest(GET_PARAM(2), 0, DIFFWTD_38_INV);
+}
+TEST_P(BuildCompDiffwtdMaskHighbdTest, DISABLED_Speed) {
+ RunTest(GET_PARAM(2), 1, DIFFWTD_38);
+ RunTest(GET_PARAM(2), 1, DIFFWTD_38_INV);
+}
+
+#if HAVE_SSSE3
+INSTANTIATE_TEST_SUITE_P(
+ SSSE3, BuildCompDiffwtdMaskHighbdTest,
+ BuildParamsHighbd(av1_build_compound_diffwtd_mask_highbd_ssse3));
+#endif
+
+#if HAVE_AVX2
+INSTANTIATE_TEST_SUITE_P(
+ AVX2, BuildCompDiffwtdMaskHighbdTest,
+ BuildParamsHighbd(av1_build_compound_diffwtd_mask_highbd_avx2));
+#endif
+
+#if HAVE_NEON
+INSTANTIATE_TEST_SUITE_P(
+ NEON, BuildCompDiffwtdMaskHighbdTest,
+ BuildParamsHighbd(av1_build_compound_diffwtd_mask_highbd_neon));
+#endif
+#endif // CONFIG_AV1_HIGHBITDEPTH
+
+using BuildCompDiffWtdMaskD16Func = void (*)(
+ uint8_t *mask, DIFFWTD_MASK_TYPE mask_type, const CONV_BUF_TYPE *src0,
+ int src0_stride, const CONV_BUF_TYPE *src1, int src1_stride, int h, int w,
+ ConvolveParams *conv_params, int bd);
+
+using BuildCompDiffwtdMaskD16Param =
+ std::tuple<int, BuildCompDiffWtdMaskD16Func, BLOCK_SIZE>;
+
+#if HAVE_SSE4_1 || HAVE_AVX2 || HAVE_NEON
+::testing::internal::ParamGenerator<BuildCompDiffwtdMaskD16Param> BuildParams(
+ BuildCompDiffWtdMaskD16Func filter) {
+ return ::testing::Combine(::testing::Range(8, 13, 2),
+ ::testing::Values(filter),
+ ::testing::Range(BLOCK_4X4, BLOCK_SIZES_ALL));
+}
+#endif
+
+class BuildCompDiffwtdMaskD16Test
+ : public ::testing::TestWithParam<BuildCompDiffwtdMaskD16Param> {
+ public:
+ BuildCompDiffwtdMaskD16Test() : rnd_(ACMRandom::DeterministicSeed()) {}
+ ~BuildCompDiffwtdMaskD16Test() override = default;
+
+ protected:
+ void RunCheckOutput(BuildCompDiffWtdMaskD16Func test_impl) {
+ const int block_idx = GET_PARAM(2);
+ const int bd = GET_PARAM(0);
+ const int width = block_size_wide[block_idx];
+ const int height = block_size_high[block_idx];
+ DECLARE_ALIGNED(16, uint8_t, mask_ref[2 * MAX_SB_SQUARE]);
+ DECLARE_ALIGNED(16, uint8_t, mask_test[2 * MAX_SB_SQUARE]);
+ DECLARE_ALIGNED(32, uint16_t, src0[MAX_SB_SQUARE]);
+ DECLARE_ALIGNED(32, uint16_t, src1[MAX_SB_SQUARE]);
+
+ ConvolveParams conv_params =
+ get_conv_params_no_round(0, 0, nullptr, 0, 1, bd);
+
+ const int in_precision =
+ bd + 2 * FILTER_BITS - conv_params.round_0 - conv_params.round_1 + 2;
+
+ for (int i = 0; i < MAX_SB_SQUARE; i++) {
+ src0[i] = rnd_.Rand16() & ((1 << in_precision) - 1);
+ src1[i] = rnd_.Rand16() & ((1 << in_precision) - 1);
+ }
+
+ for (int mask_type = 0; mask_type < DIFFWTD_MASK_TYPES; mask_type++) {
+ av1_build_compound_diffwtd_mask_d16_c(
+ mask_ref, (DIFFWTD_MASK_TYPE)mask_type, src0, width, src1, width,
+ height, width, &conv_params, bd);
+
+ test_impl(mask_test, (DIFFWTD_MASK_TYPE)mask_type, src0, width, src1,
+ width, height, width, &conv_params, bd);
+
+ for (int r = 0; r < height; ++r) {
+ for (int c = 0; c < width; ++c) {
+ ASSERT_EQ(mask_ref[c + r * width], mask_test[c + r * width])
+ << "Mismatch at unit tests for BuildCompDiffwtdMaskD16Test\n"
+ << " Pixel mismatch at index "
+ << "[" << r << "," << c << "] "
+ << " @ " << width << "x" << height << " inv " << mask_type;
+ }
+ }
+ }
+ }
+
+ void RunSpeedTest(BuildCompDiffWtdMaskD16Func test_impl,
+ DIFFWTD_MASK_TYPE mask_type) {
+ const int block_idx = GET_PARAM(2);
+ const int bd = GET_PARAM(0);
+ const int width = block_size_wide[block_idx];
+ const int height = block_size_high[block_idx];
+ DECLARE_ALIGNED(16, uint8_t, mask[MAX_SB_SQUARE]);
+ DECLARE_ALIGNED(32, uint16_t, src0[MAX_SB_SQUARE]);
+ DECLARE_ALIGNED(32, uint16_t, src1[MAX_SB_SQUARE]);
+
+ ConvolveParams conv_params =
+ get_conv_params_no_round(0, 0, nullptr, 0, 1, bd);
+
+ const int in_precision =
+ bd + 2 * FILTER_BITS - conv_params.round_0 - conv_params.round_1 + 2;
+
+ for (int i = 0; i < MAX_SB_SQUARE; i++) {
+ src0[i] = rnd_.Rand16() & ((1 << in_precision) - 1);
+ src1[i] = rnd_.Rand16() & ((1 << in_precision) - 1);
+ }
+
+ const int num_loops = 10000000 / (width + height);
+ aom_usec_timer timer;
+ aom_usec_timer_start(&timer);
+
+ for (int i = 0; i < num_loops; ++i)
+ av1_build_compound_diffwtd_mask_d16_c(mask, mask_type, src0, width, src1,
+ width, height, width, &conv_params,
+ bd);
+
+ aom_usec_timer_mark(&timer);
+ const int elapsed_time = static_cast<int>(aom_usec_timer_elapsed(&timer));
+
+ aom_usec_timer timer1;
+ aom_usec_timer_start(&timer1);
+
+ for (int i = 0; i < num_loops; ++i)
+ test_impl(mask, mask_type, src0, width, src1, width, height, width,
+ &conv_params, bd);
+
+ aom_usec_timer_mark(&timer1);
+ const int elapsed_time1 = static_cast<int>(aom_usec_timer_elapsed(&timer1));
+ printf("av1_build_compound_diffwtd_mask_d16 %3dx%-3d: %7.2f \n", width,
+ height, elapsed_time / double(elapsed_time1));
+ }
+
+ private:
+ ACMRandom rnd_;
+}; // class BuildCompDiffwtdMaskD16Test
+GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(BuildCompDiffwtdMaskD16Test);
+
TEST_P(BuildCompDiffwtdMaskD16Test, CheckOutput) {
RunCheckOutput(GET_PARAM(1));
}
@@ -237,18 +354,12 @@
}
#if HAVE_SSE4_1
-INSTANTIATE_TEST_SUITE_P(SSE4_1, BuildCompDiffwtdMaskTest,
- BuildParams(av1_build_compound_diffwtd_mask_sse4_1));
-
INSTANTIATE_TEST_SUITE_P(
SSE4_1, BuildCompDiffwtdMaskD16Test,
BuildParams(av1_build_compound_diffwtd_mask_d16_sse4_1));
#endif
#if HAVE_AVX2
-INSTANTIATE_TEST_SUITE_P(AVX2, BuildCompDiffwtdMaskTest,
- BuildParams(av1_build_compound_diffwtd_mask_avx2));
-
INSTANTIATE_TEST_SUITE_P(AVX2, BuildCompDiffwtdMaskD16Test,
BuildParams(av1_build_compound_diffwtd_mask_d16_avx2));
#endif
diff --git a/test/resize_test.cc b/test/resize_test.cc
index 437b8e0..7bad453 100644
--- a/test/resize_test.cc
+++ b/test/resize_test.cc
@@ -186,7 +186,7 @@
}
int flag_codec_;
bool change_start_resln_;
- virtual ~ResizingVideoSource() {}
+ ~ResizingVideoSource() override = default;
protected:
void Begin() override {
@@ -215,12 +215,12 @@
protected:
ResizeTest() : EncoderTest(GET_PARAM(0)) {}
- virtual ~ResizeTest() {}
+ ~ResizeTest() override = default;
- virtual void SetUp() { InitializeConfig(GET_PARAM(1)); }
+ void SetUp() override { InitializeConfig(GET_PARAM(1)); }
- virtual void PreEncodeFrameHook(libaom_test::VideoSource *video,
- libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(libaom_test::VideoSource *video,
+ libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
if (GET_PARAM(1) == ::libaom_test::kRealTime) {
encoder->Control(AV1E_SET_AQ_MODE, 3);
@@ -230,8 +230,8 @@
}
}
- virtual void DecompressedFrameHook(const aom_image_t &img,
- aom_codec_pts_t pts) {
+ void DecompressedFrameHook(const aom_image_t &img,
+ aom_codec_pts_t pts) override {
frame_info_list_.push_back(FrameInfo(pts, img.d_w, img.d_h));
}
@@ -279,15 +279,15 @@
ResizeInternalTestLarge() : ResizeTest(), frame0_psnr_(0.0) {}
#endif
- virtual ~ResizeInternalTestLarge() {}
+ ~ResizeInternalTestLarge() override = default;
- virtual void BeginPassHook(unsigned int /*pass*/) {
+ void BeginPassHook(unsigned int /*pass*/) override {
#if WRITE_COMPRESSED_STREAM
outfile_ = fopen("av10-2-05-resize.ivf", "wb");
#endif
}
- virtual void EndPassHook() {
+ void EndPassHook() override {
#if WRITE_COMPRESSED_STREAM
if (outfile_) {
if (!fseek(outfile_, 0, SEEK_SET))
@@ -298,8 +298,8 @@
#endif
}
- virtual void PreEncodeFrameHook(libaom_test::VideoSource *video,
- libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(libaom_test::VideoSource *video,
+ libaom_test::Encoder *encoder) override {
if (change_config_) {
int new_q = 60;
if (video->frame() == 0) {
@@ -323,13 +323,13 @@
}
}
- virtual void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) {
+ void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) override {
if (frame0_psnr_ == 0.) frame0_psnr_ = pkt->data.psnr.psnr[0];
EXPECT_NEAR(pkt->data.psnr.psnr[0], frame0_psnr_, 4.1);
}
#if WRITE_COMPRESSED_STREAM
- virtual void FramePktHook(const aom_codec_cx_pkt_t *pkt) {
+ void FramePktHook(const aom_codec_cx_pkt_t *pkt) override {
++out_frames_;
// Write initial file header if first frame.
@@ -402,11 +402,12 @@
protected:
ResizeRealtimeTest()
: EncoderTest(GET_PARAM(0)), num_threads_(GET_PARAM(3)),
- set_scale_mode_(false), set_scale_mode2_(false) {}
- virtual ~ResizeRealtimeTest() {}
+ set_scale_mode_(false), set_scale_mode2_(false),
+ set_scale_mode3_(false) {}
+ ~ResizeRealtimeTest() override = default;
- virtual void PreEncodeFrameHook(libaom_test::VideoSource *video,
- libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(libaom_test::VideoSource *video,
+ libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AV1E_SET_AQ_MODE, 3);
encoder->Control(AV1E_SET_ALLOW_WARPED_MOTION, 0);
@@ -433,6 +434,13 @@
else if (video->frame() > 40)
mode = { AOME_THREEFOUR, AOME_THREEFOUR };
encoder->Control(AOME_SET_SCALEMODE, &mode);
+ } else if (set_scale_mode3_) {
+ struct aom_scaling_mode mode;
+ if (video->frame() <= 30)
+ mode = { AOME_ONETWO, AOME_NORMAL };
+ else
+ mode = { AOME_NORMAL, AOME_NORMAL };
+ encoder->Control(AOME_SET_SCALEMODE, &mode);
}
if (change_bitrate_ && video->frame() == frame_change_bitrate_) {
@@ -442,17 +450,17 @@
}
}
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(GET_PARAM(1));
set_cpu_used_ = GET_PARAM(2);
}
- virtual void DecompressedFrameHook(const aom_image_t &img,
- aom_codec_pts_t pts) {
+ void DecompressedFrameHook(const aom_image_t &img,
+ aom_codec_pts_t pts) override {
frame_info_list_.push_back(FrameInfo(pts, img.d_w, img.d_h));
}
- virtual void MismatchHook(const aom_image_t *img1, const aom_image_t *img2) {
+ void MismatchHook(const aom_image_t *img1, const aom_image_t *img2) override {
double mismatch_psnr = compute_psnr(img1, img2);
mismatch_psnr_ += mismatch_psnr;
++mismatch_nframes_;
@@ -483,7 +491,7 @@
// the width and height of the frame are swapped
cfg_.g_forced_max_frame_width = cfg_.g_forced_max_frame_height =
AOMMAX(kInitialWidth, kInitialHeight);
- if (set_scale_mode_ || set_scale_mode2_) {
+ if (set_scale_mode_ || set_scale_mode2_ || set_scale_mode3_) {
cfg_.rc_dropframe_thresh = 0;
cfg_.g_forced_max_frame_width = 1280;
cfg_.g_forced_max_frame_height = 1280;
@@ -499,6 +507,7 @@
int mismatch_nframes_;
bool set_scale_mode_;
bool set_scale_mode2_;
+ bool set_scale_mode3_;
};
// Check the AOME_SET_SCALEMODE control by downsizing to
@@ -509,6 +518,7 @@
cfg_.g_h = 720;
set_scale_mode_ = true;
set_scale_mode2_ = false;
+ set_scale_mode3_ = false;
DefaultConfig();
change_bitrate_ = false;
mismatch_nframes_ = 0;
@@ -544,6 +554,7 @@
cfg_.g_h = 180;
set_scale_mode_ = true;
set_scale_mode2_ = false;
+ set_scale_mode3_ = false;
DefaultConfig();
change_bitrate_ = false;
mismatch_nframes_ = 0;
@@ -578,6 +589,7 @@
cfg_.g_h = 720;
set_scale_mode_ = false;
set_scale_mode2_ = true;
+ set_scale_mode3_ = false;
DefaultConfig();
change_bitrate_ = false;
mismatch_nframes_ = 0;
@@ -604,12 +616,45 @@
}
}
+// Check the AOME_SET_SCALEMODE control by downsizing to
+// 1/2 horizontally only and then back up to original.
+TEST_P(ResizeRealtimeTest, TestInternalResizeSetScaleMode3) {
+ ::libaom_test::Y4mVideoSource video("niklas_1280_720_30.y4m", 0, 60);
+ cfg_.g_w = 1280;
+ cfg_.g_h = 720;
+ set_scale_mode_ = false;
+ set_scale_mode2_ = false;
+ set_scale_mode3_ = true;
+ DefaultConfig();
+ change_bitrate_ = false;
+ mismatch_nframes_ = 0;
+ ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
+ // Check we decoded the same number of frames as we attempted to encode
+ ASSERT_EQ(frame_info_list_.size(), video.limit());
+ for (std::vector<FrameInfo>::const_iterator info = frame_info_list_.begin();
+ info != frame_info_list_.end(); ++info) {
+ const auto frame = static_cast<unsigned>(info->pts);
+ unsigned int expected_w = 640;
+ unsigned int expected_h = 720;
+ if (frame > 30) {
+ expected_w = 1280;
+ expected_h = 720;
+ }
+ EXPECT_EQ(expected_w, info->w)
+ << "Frame " << frame << " had unexpected width";
+ EXPECT_EQ(expected_h, info->h)
+ << "Frame " << frame << " had unexpected height";
+ EXPECT_EQ(static_cast<unsigned int>(0), GetMismatchFrames());
+ }
+}
+
TEST_P(ResizeRealtimeTest, TestExternalResizeWorks) {
ResizingVideoSource video;
video.flag_codec_ = 1;
change_bitrate_ = false;
set_scale_mode_ = false;
set_scale_mode2_ = false;
+ set_scale_mode3_ = false;
mismatch_psnr_ = 0.0;
mismatch_nframes_ = 0;
DefaultConfig();
@@ -651,6 +696,7 @@
change_bitrate_ = false;
set_scale_mode_ = false;
set_scale_mode2_ = false;
+ set_scale_mode3_ = false;
mismatch_psnr_ = 0.0;
mismatch_nframes_ = 0;
DefaultConfig();
@@ -700,6 +746,7 @@
frame_change_bitrate_ = 120;
set_scale_mode_ = false;
set_scale_mode2_ = false;
+ set_scale_mode3_ = false;
mismatch_psnr_ = 0.0;
mismatch_nframes_ = 0;
DefaultConfig();
@@ -757,15 +804,15 @@
ResizeCspTest() : ResizeTest(), frame0_psnr_(0.0) {}
#endif
- virtual ~ResizeCspTest() {}
+ ~ResizeCspTest() override = default;
- virtual void BeginPassHook(unsigned int /*pass*/) {
+ void BeginPassHook(unsigned int /*pass*/) override {
#if WRITE_COMPRESSED_STREAM
outfile_ = fopen("av11-2-05-cspchape.ivf", "wb");
#endif
}
- virtual void EndPassHook() {
+ void EndPassHook() override {
#if WRITE_COMPRESSED_STREAM
if (outfile_) {
if (!fseek(outfile_, 0, SEEK_SET))
@@ -776,13 +823,13 @@
#endif
}
- virtual void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) {
+ void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) override {
if (frame0_psnr_ == 0.) frame0_psnr_ = pkt->data.psnr.psnr[0];
EXPECT_NEAR(pkt->data.psnr.psnr[0], frame0_psnr_, 2.0);
}
#if WRITE_COMPRESSED_STREAM
- virtual void FramePktHook(const aom_codec_cx_pkt_t *pkt) {
+ void FramePktHook(const aom_codec_cx_pkt_t *pkt) override {
++out_frames_;
// Write initial file header if first frame.
@@ -809,7 +856,7 @@
limit_ = 30;
}
- virtual ~ResizingCspVideoSource() {}
+ ~ResizingCspVideoSource() override = default;
};
#if (defined(DISABLE_TRELLISQ_SEARCH) && DISABLE_TRELLISQ_SEARCH) || \
@@ -845,9 +892,9 @@
: EncoderTest(GET_PARAM(0)), encoding_mode_(GET_PARAM(1)),
resize_mode_(GET_PARAM(2)), resize_denominator_(GET_PARAM(3)),
resize_kf_denominator_(GET_PARAM(4)), cpu_used_(GET_PARAM(5)) {}
- virtual ~ResizeModeTestLarge() {}
+ ~ResizeModeTestLarge() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(encoding_mode_);
const aom_rational timebase = { 1, 30 };
cfg_.g_timebase = timebase;
@@ -861,8 +908,8 @@
init_flags_ = AOM_CODEC_USE_PSNR;
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AOME_SET_CPUUSED, cpu_used_);
encoder->Control(AOME_SET_ENABLEAUTOALTREF, 1);
diff --git a/test/rt_end_to_end_test.cc b/test/rt_end_to_end_test.cc
index 735d799..f1f9e01 100644
--- a/test/rt_end_to_end_test.cc
+++ b/test/rt_end_to_end_test.cc
@@ -94,9 +94,9 @@
aq_mode_(GET_PARAM(3)), threads_(GET_PARAM(4)),
tile_columns_(GET_PARAM(5)), tile_rows_(GET_PARAM(6)) {}
- virtual ~RTEndToEndTest() {}
+ ~RTEndToEndTest() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(::libaom_test::kRealTime);
cfg_.g_threads = threads_;
@@ -107,18 +107,18 @@
cfg_.kf_min_dist = 9999;
}
- virtual void BeginPassHook(unsigned int) {
+ void BeginPassHook(unsigned int) override {
psnr_ = 0.0;
nframes_ = 0;
}
- virtual void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) {
+ void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) override {
psnr_ += pkt->data.psnr.psnr[0];
nframes_++;
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AV1E_SET_ENABLE_RESTORATION, 0);
encoder->Control(AV1E_SET_ENABLE_OBMC, 0);
diff --git a/test/sad_test.cc b/test/sad_test.cc
index 0a39ca6..5212748 100644
--- a/test/sad_test.cc
+++ b/test/sad_test.cc
@@ -39,12 +39,6 @@
const uint8_t *second_pred);
typedef std::tuple<int, int, SadMxNAvgFunc, int> SadMxNAvgParam;
-typedef void (*DistWtdCompAvgFunc)(uint8_t *comp_pred, const uint8_t *pred,
- int width, int height, const uint8_t *ref,
- int ref_stride,
- const DIST_WTD_COMP_PARAMS *jcp_param);
-typedef std::tuple<int, int, DistWtdCompAvgFunc, int> DistWtdCompAvgParam;
-
typedef unsigned int (*DistWtdSadMxhFunc)(const uint8_t *src_ptr,
int src_stride,
const uint8_t *ref_ptr,
@@ -138,15 +132,13 @@
comp_pred16_test_ = nullptr;
}
- virtual void TearDown() {}
-
protected:
// Handle up to 4 128x128 blocks, with stride up to 256
static const int kDataAlignment = 16;
static const int kDataBlockSize = 128 * 256;
static const int kDataBufferSize = 4 * kDataBlockSize;
- virtual void SetUp() {
+ void SetUp() override {
if (bd_ == -1) {
use_high_bit_depth_ = false;
bit_depth_ = AOM_BITS_8;
@@ -255,31 +247,6 @@
return sad;
}
- void ReferenceDistWtdCompAvg(int block_idx) {
- const uint8_t *const reference8 = GetReference(block_idx);
- const uint8_t *const second_pred8 = second_pred_;
- uint8_t *const comp_pred8 = comp_pred_;
- const uint16_t *const reference16 =
- CONVERT_TO_SHORTPTR(GetReference(block_idx));
- const uint16_t *const second_pred16 = CONVERT_TO_SHORTPTR(second_pred_);
- uint16_t *const comp_pred16 = CONVERT_TO_SHORTPTR(comp_pred_);
- for (int h = 0; h < height_; ++h) {
- for (int w = 0; w < width_; ++w) {
- if (!use_high_bit_depth_) {
- const int tmp =
- second_pred8[h * width_ + w] * jcp_param_.bck_offset +
- reference8[h * reference_stride_ + w] * jcp_param_.fwd_offset;
- comp_pred8[h * width_ + w] = ROUND_POWER_OF_TWO(tmp, 4);
- } else {
- const int tmp =
- second_pred16[h * width_ + w] * jcp_param_.bck_offset +
- reference16[h * reference_stride_ + w] * jcp_param_.fwd_offset;
- comp_pred16[h * width_ + w] = ROUND_POWER_OF_TWO(tmp, 4);
- }
- }
- }
- }
-
unsigned int ReferenceDistWtdSADavg(int block_idx) {
unsigned int sad = 0;
const uint8_t *const reference8 = GetReference(block_idx);
@@ -401,7 +368,7 @@
}
void SADForSpeedTest(unsigned int *results,
- const uint8_t *const *references) {
+ const uint8_t *const *references) override {
GET_PARAM(2)
(source_data_, source_stride_, references, reference_stride_, results);
}
@@ -432,7 +399,7 @@
}
void SADForSpeedTest(unsigned int *results,
- const uint8_t *const *references) {
+ const uint8_t *const *references) override {
GET_PARAM(2)
(source_data_, source_stride_, references, reference_stride_, results);
}
@@ -475,7 +442,7 @@
}
void SADForSpeedTest(unsigned int *results,
- const uint8_t *const *references) {
+ const uint8_t *const *references) override {
GET_PARAM(2)
(source_data_, source_stride_, references, reference_stride_, results);
}
@@ -504,7 +471,7 @@
}
void SADForSpeedTest(unsigned int *results,
- const uint8_t *const *references) {
+ const uint8_t *const *references) override {
GET_PARAM(2)
(source_data_, source_stride_, references[0], reference_stride_);
(void)results;
@@ -534,7 +501,7 @@
}
void SADForSpeedTest(unsigned int *results,
- const uint8_t *const *references) {
+ const uint8_t *const *references) override {
GET_PARAM(2)
(source_data_, source_stride_, references[0], reference_stride_);
(void)results;
@@ -565,40 +532,6 @@
}
};
-class DistWtdCompAvgTest
- : public ::testing::WithParamInterface<DistWtdCompAvgParam>,
- public SADTestBase {
- public:
- DistWtdCompAvgTest()
- : SADTestBase(GET_PARAM(0), GET_PARAM(1), GET_PARAM(3)) {}
-
- protected:
- void dist_wtd_comp_avg(int block_idx) {
- const uint8_t *const reference = GetReference(block_idx);
-
- API_REGISTER_STATE_CHECK(GET_PARAM(2)(comp_pred_test_, second_pred_, width_,
- height_, reference, reference_stride_,
- &jcp_param_));
- }
-
- void CheckCompAvg() {
- for (int j = 0; j < 2; ++j) {
- for (int i = 0; i < 4; ++i) {
- jcp_param_.fwd_offset = quant_dist_lookup_table[i][j];
- jcp_param_.bck_offset = quant_dist_lookup_table[i][1 - j];
-
- ReferenceDistWtdCompAvg(0);
- dist_wtd_comp_avg(0);
-
- for (int y = 0; y < height_; ++y)
- for (int x = 0; x < width_; ++x)
- ASSERT_EQ(comp_pred_[y * width_ + x],
- comp_pred_test_[y * width_ + x]);
- }
- }
- }
-};
-
class DistWtdSADavgTest
: public ::testing::WithParamInterface<DistWtdSadMxNAvgParam>,
public SADTestBase {
@@ -807,38 +740,6 @@
source_stride_ = tmp_stride;
}
-TEST_P(DistWtdCompAvgTest, MaxRef) {
- FillConstant(reference_data_, reference_stride_, mask_);
- FillConstant(second_pred_, width_, 0);
- CheckCompAvg();
-}
-
-TEST_P(DistWtdCompAvgTest, MaxSecondPred) {
- FillConstant(reference_data_, reference_stride_, 0);
- FillConstant(second_pred_, width_, mask_);
- CheckCompAvg();
-}
-
-TEST_P(DistWtdCompAvgTest, ShortRef) {
- const int tmp_stride = reference_stride_;
- reference_stride_ >>= 1;
- FillRandom(reference_data_, reference_stride_);
- FillRandom(second_pred_, width_);
- CheckCompAvg();
- reference_stride_ = tmp_stride;
-}
-
-TEST_P(DistWtdCompAvgTest, UnalignedRef) {
- // The reference frame, but not the source frame, may be unaligned for
- // certain types of searches.
- const int tmp_stride = reference_stride_;
- reference_stride_ -= 1;
- FillRandom(reference_data_, reference_stride_);
- FillRandom(second_pred_, width_);
- CheckCompAvg();
- reference_stride_ = tmp_stride;
-}
-
TEST_P(DistWtdSADavgTest, MaxRef) {
FillConstant(source_data_, source_stride_, 0);
FillConstant(reference_data_, reference_stride_, mask_);
@@ -1445,38 +1346,6 @@
};
INSTANTIATE_TEST_SUITE_P(C, SADavgTest, ::testing::ValuesIn(avg_c_tests));
-// TODO(chengchen): add highbd tests
-const DistWtdCompAvgParam dist_wtd_comp_avg_c_tests[] = {
- make_tuple(128, 128, &aom_dist_wtd_comp_avg_pred_c, -1),
- make_tuple(128, 64, &aom_dist_wtd_comp_avg_pred_c, -1),
- make_tuple(64, 128, &aom_dist_wtd_comp_avg_pred_c, -1),
- make_tuple(64, 64, &aom_dist_wtd_comp_avg_pred_c, -1),
- make_tuple(64, 32, &aom_dist_wtd_comp_avg_pred_c, -1),
- make_tuple(32, 64, &aom_dist_wtd_comp_avg_pred_c, -1),
- make_tuple(32, 32, &aom_dist_wtd_comp_avg_pred_c, -1),
- make_tuple(32, 16, &aom_dist_wtd_comp_avg_pred_c, -1),
- make_tuple(16, 32, &aom_dist_wtd_comp_avg_pred_c, -1),
- make_tuple(16, 16, &aom_dist_wtd_comp_avg_pred_c, -1),
- make_tuple(16, 8, &aom_dist_wtd_comp_avg_pred_c, -1),
- make_tuple(8, 16, &aom_dist_wtd_comp_avg_pred_c, -1),
- make_tuple(8, 8, &aom_dist_wtd_comp_avg_pred_c, -1),
- make_tuple(8, 4, &aom_dist_wtd_comp_avg_pred_c, -1),
- make_tuple(4, 8, &aom_dist_wtd_comp_avg_pred_c, -1),
- make_tuple(4, 4, &aom_dist_wtd_comp_avg_pred_c, -1),
-
-#if !CONFIG_REALTIME_ONLY
- make_tuple(64, 16, &aom_dist_wtd_comp_avg_pred_c, -1),
- make_tuple(16, 64, &aom_dist_wtd_comp_avg_pred_c, -1),
- make_tuple(32, 8, &aom_dist_wtd_comp_avg_pred_c, -1),
- make_tuple(8, 32, &aom_dist_wtd_comp_avg_pred_c, -1),
- make_tuple(16, 4, &aom_dist_wtd_comp_avg_pred_c, -1),
- make_tuple(4, 16, &aom_dist_wtd_comp_avg_pred_c, -1),
-#endif
-};
-
-INSTANTIATE_TEST_SUITE_P(C, DistWtdCompAvgTest,
- ::testing::ValuesIn(dist_wtd_comp_avg_c_tests));
-
const DistWtdSadMxNAvgParam dist_wtd_avg_c_tests[] = {
make_tuple(128, 128, &aom_dist_wtd_sad128x128_avg_c, -1),
make_tuple(128, 64, &aom_dist_wtd_sad128x64_avg_c, -1),
@@ -2227,6 +2096,56 @@
make_tuple(8, 4, &aom_sad8x4_avg_neon, -1),
make_tuple(4, 8, &aom_sad4x8_avg_neon, -1),
make_tuple(4, 4, &aom_sad4x4_avg_neon, -1),
+#if CONFIG_AV1_HIGHBITDEPTH
+ make_tuple(128, 128, &aom_highbd_sad128x128_avg_neon, 8),
+ make_tuple(128, 64, &aom_highbd_sad128x64_avg_neon, 8),
+ make_tuple(64, 128, &aom_highbd_sad64x128_avg_neon, 8),
+ make_tuple(64, 64, &aom_highbd_sad64x64_avg_neon, 8),
+ make_tuple(64, 32, &aom_highbd_sad64x32_avg_neon, 8),
+ make_tuple(32, 64, &aom_highbd_sad32x64_avg_neon, 8),
+ make_tuple(32, 32, &aom_highbd_sad32x32_avg_neon, 8),
+ make_tuple(32, 16, &aom_highbd_sad32x16_avg_neon, 8),
+ make_tuple(16, 32, &aom_highbd_sad16x32_avg_neon, 8),
+ make_tuple(16, 16, &aom_highbd_sad16x16_avg_neon, 8),
+ make_tuple(16, 8, &aom_highbd_sad16x8_avg_neon, 8),
+ make_tuple(8, 16, &aom_highbd_sad8x16_avg_neon, 8),
+ make_tuple(8, 8, &aom_highbd_sad8x8_avg_neon, 8),
+ make_tuple(8, 4, &aom_highbd_sad8x4_avg_neon, 8),
+ make_tuple(4, 8, &aom_highbd_sad4x8_avg_neon, 8),
+ make_tuple(4, 4, &aom_highbd_sad4x4_avg_neon, 8),
+ make_tuple(128, 128, &aom_highbd_sad128x128_avg_neon, 10),
+ make_tuple(128, 64, &aom_highbd_sad128x64_avg_neon, 10),
+ make_tuple(64, 128, &aom_highbd_sad64x128_avg_neon, 10),
+ make_tuple(64, 64, &aom_highbd_sad64x64_avg_neon, 10),
+ make_tuple(64, 32, &aom_highbd_sad64x32_avg_neon, 10),
+ make_tuple(32, 64, &aom_highbd_sad32x64_avg_neon, 10),
+ make_tuple(32, 32, &aom_highbd_sad32x32_avg_neon, 10),
+ make_tuple(32, 16, &aom_highbd_sad32x16_avg_neon, 10),
+ make_tuple(16, 32, &aom_highbd_sad16x32_avg_neon, 10),
+ make_tuple(16, 16, &aom_highbd_sad16x16_avg_neon, 10),
+ make_tuple(16, 8, &aom_highbd_sad16x8_avg_neon, 10),
+ make_tuple(8, 16, &aom_highbd_sad8x16_avg_neon, 10),
+ make_tuple(8, 8, &aom_highbd_sad8x8_avg_neon, 10),
+ make_tuple(8, 4, &aom_highbd_sad8x4_avg_neon, 10),
+ make_tuple(4, 8, &aom_highbd_sad4x8_avg_neon, 10),
+ make_tuple(4, 4, &aom_highbd_sad4x4_avg_neon, 10),
+ make_tuple(128, 128, &aom_highbd_sad128x128_avg_neon, 12),
+ make_tuple(128, 64, &aom_highbd_sad128x64_avg_neon, 12),
+ make_tuple(64, 128, &aom_highbd_sad64x128_avg_neon, 12),
+ make_tuple(64, 64, &aom_highbd_sad64x64_avg_neon, 12),
+ make_tuple(64, 32, &aom_highbd_sad64x32_avg_neon, 12),
+ make_tuple(32, 64, &aom_highbd_sad32x64_avg_neon, 12),
+ make_tuple(32, 32, &aom_highbd_sad32x32_avg_neon, 12),
+ make_tuple(32, 16, &aom_highbd_sad32x16_avg_neon, 12),
+ make_tuple(16, 32, &aom_highbd_sad16x32_avg_neon, 12),
+ make_tuple(16, 16, &aom_highbd_sad16x16_avg_neon, 12),
+ make_tuple(16, 8, &aom_highbd_sad16x8_avg_neon, 12),
+ make_tuple(8, 16, &aom_highbd_sad8x16_avg_neon, 12),
+ make_tuple(8, 8, &aom_highbd_sad8x8_avg_neon, 12),
+ make_tuple(8, 4, &aom_highbd_sad8x4_avg_neon, 12),
+ make_tuple(4, 8, &aom_highbd_sad4x8_avg_neon, 12),
+ make_tuple(4, 4, &aom_highbd_sad4x4_avg_neon, 12),
+#endif // CONFIG_AV1_HIGHBITDEPTH
#if !CONFIG_REALTIME_ONLY
make_tuple(64, 16, &aom_sad64x16_avg_neon, -1),
make_tuple(32, 8, &aom_sad32x8_avg_neon, -1),
@@ -2234,10 +2153,61 @@
make_tuple(16, 4, &aom_sad16x4_avg_neon, -1),
make_tuple(8, 32, &aom_sad8x32_avg_neon, -1),
make_tuple(4, 16, &aom_sad4x16_avg_neon, -1),
-#endif
+#if CONFIG_AV1_HIGHBITDEPTH
+ make_tuple(64, 16, &aom_highbd_sad64x16_avg_neon, 8),
+ make_tuple(16, 64, &aom_highbd_sad16x64_avg_neon, 8),
+ make_tuple(32, 8, &aom_highbd_sad32x8_avg_neon, 8),
+ make_tuple(8, 32, &aom_highbd_sad8x32_avg_neon, 8),
+ make_tuple(16, 4, &aom_highbd_sad16x4_avg_neon, 8),
+ make_tuple(4, 16, &aom_highbd_sad4x16_avg_neon, 8),
+ make_tuple(64, 16, &aom_highbd_sad64x16_avg_neon, 10),
+ make_tuple(16, 64, &aom_highbd_sad16x64_avg_neon, 10),
+ make_tuple(32, 8, &aom_highbd_sad32x8_avg_neon, 10),
+ make_tuple(8, 32, &aom_highbd_sad8x32_avg_neon, 10),
+ make_tuple(16, 4, &aom_highbd_sad16x4_avg_neon, 10),
+ make_tuple(4, 16, &aom_highbd_sad4x16_avg_neon, 10),
+ make_tuple(64, 16, &aom_highbd_sad64x16_avg_neon, 12),
+ make_tuple(16, 64, &aom_highbd_sad16x64_avg_neon, 12),
+ make_tuple(32, 8, &aom_highbd_sad32x8_avg_neon, 12),
+ make_tuple(8, 32, &aom_highbd_sad8x32_avg_neon, 12),
+ make_tuple(16, 4, &aom_highbd_sad16x4_avg_neon, 12),
+ make_tuple(4, 16, &aom_highbd_sad4x16_avg_neon, 12),
+#endif // CONFIG_AV1_HIGHBITDEPTH
+#endif // !CONFIG_REALTIME_ONLY
};
INSTANTIATE_TEST_SUITE_P(NEON, SADavgTest, ::testing::ValuesIn(avg_neon_tests));
+const DistWtdSadMxNAvgParam dist_wtd_avg_neon_tests[] = {
+ make_tuple(128, 128, &aom_dist_wtd_sad128x128_avg_neon, -1),
+ make_tuple(128, 64, &aom_dist_wtd_sad128x64_avg_neon, -1),
+ make_tuple(64, 128, &aom_dist_wtd_sad64x128_avg_neon, -1),
+ make_tuple(64, 64, &aom_dist_wtd_sad64x64_avg_neon, -1),
+ make_tuple(64, 32, &aom_dist_wtd_sad64x32_avg_neon, -1),
+ make_tuple(32, 64, &aom_dist_wtd_sad32x64_avg_neon, -1),
+ make_tuple(32, 32, &aom_dist_wtd_sad32x32_avg_neon, -1),
+ make_tuple(32, 16, &aom_dist_wtd_sad32x16_avg_neon, -1),
+ make_tuple(16, 32, &aom_dist_wtd_sad16x32_avg_neon, -1),
+ make_tuple(16, 16, &aom_dist_wtd_sad16x16_avg_neon, -1),
+ make_tuple(16, 8, &aom_dist_wtd_sad16x8_avg_neon, -1),
+ make_tuple(8, 16, &aom_dist_wtd_sad8x16_avg_neon, -1),
+ make_tuple(8, 8, &aom_dist_wtd_sad8x8_avg_neon, -1),
+ make_tuple(8, 4, &aom_dist_wtd_sad8x4_avg_neon, -1),
+ make_tuple(4, 8, &aom_dist_wtd_sad4x8_avg_neon, -1),
+ make_tuple(4, 4, &aom_dist_wtd_sad4x4_avg_neon, -1),
+
+#if !CONFIG_REALTIME_ONLY
+ make_tuple(64, 16, &aom_dist_wtd_sad64x16_avg_neon, -1),
+ make_tuple(16, 64, &aom_dist_wtd_sad16x64_avg_neon, -1),
+ make_tuple(32, 8, &aom_dist_wtd_sad32x8_avg_neon, -1),
+ make_tuple(8, 32, &aom_dist_wtd_sad8x32_avg_neon, -1),
+ make_tuple(16, 4, &aom_dist_wtd_sad16x4_avg_neon, -1),
+ make_tuple(4, 16, &aom_dist_wtd_sad4x16_avg_neon, -1),
+#endif // !CONFIG_REALTIME_ONLY
+};
+
+INSTANTIATE_TEST_SUITE_P(NEON, DistWtdSADavgTest,
+ ::testing::ValuesIn(dist_wtd_avg_neon_tests));
+
const SadMxNx4Param x3d_neon_tests[] = {
make_tuple(128, 128, &aom_sad128x128x3d_neon, -1),
make_tuple(128, 64, &aom_sad128x64x3d_neon, -1),
@@ -2255,6 +2225,56 @@
make_tuple(8, 4, &aom_sad8x4x3d_neon, -1),
make_tuple(4, 8, &aom_sad4x8x3d_neon, -1),
make_tuple(4, 4, &aom_sad4x4x3d_neon, -1),
+#if CONFIG_AV1_HIGHBITDEPTH
+ make_tuple(128, 128, &aom_highbd_sad128x128x3d_neon, 8),
+ make_tuple(128, 64, &aom_highbd_sad128x64x3d_neon, 8),
+ make_tuple(64, 128, &aom_highbd_sad64x128x3d_neon, 8),
+ make_tuple(64, 64, &aom_highbd_sad64x64x3d_neon, 8),
+ make_tuple(64, 32, &aom_highbd_sad64x32x3d_neon, 8),
+ make_tuple(32, 64, &aom_highbd_sad32x64x3d_neon, 8),
+ make_tuple(32, 32, &aom_highbd_sad32x32x3d_neon, 8),
+ make_tuple(32, 16, &aom_highbd_sad32x16x3d_neon, 8),
+ make_tuple(16, 32, &aom_highbd_sad16x32x3d_neon, 8),
+ make_tuple(16, 16, &aom_highbd_sad16x16x3d_neon, 8),
+ make_tuple(16, 8, &aom_highbd_sad16x8x3d_neon, 8),
+ make_tuple(8, 16, &aom_highbd_sad8x16x3d_neon, 8),
+ make_tuple(8, 8, &aom_highbd_sad8x8x3d_neon, 8),
+ make_tuple(8, 4, &aom_highbd_sad8x4x3d_neon, 8),
+ make_tuple(4, 8, &aom_highbd_sad4x8x3d_neon, 8),
+ make_tuple(4, 4, &aom_highbd_sad4x4x3d_neon, 8),
+ make_tuple(128, 128, &aom_highbd_sad128x128x3d_neon, 10),
+ make_tuple(128, 64, &aom_highbd_sad128x64x3d_neon, 10),
+ make_tuple(64, 128, &aom_highbd_sad64x128x3d_neon, 10),
+ make_tuple(64, 64, &aom_highbd_sad64x64x3d_neon, 10),
+ make_tuple(64, 32, &aom_highbd_sad64x32x3d_neon, 10),
+ make_tuple(32, 64, &aom_highbd_sad32x64x3d_neon, 10),
+ make_tuple(32, 32, &aom_highbd_sad32x32x3d_neon, 10),
+ make_tuple(32, 16, &aom_highbd_sad32x16x3d_neon, 10),
+ make_tuple(16, 32, &aom_highbd_sad16x32x3d_neon, 10),
+ make_tuple(16, 16, &aom_highbd_sad16x16x3d_neon, 10),
+ make_tuple(16, 8, &aom_highbd_sad16x8x3d_neon, 10),
+ make_tuple(8, 16, &aom_highbd_sad8x16x3d_neon, 10),
+ make_tuple(8, 8, &aom_highbd_sad8x8x3d_neon, 10),
+ make_tuple(8, 4, &aom_highbd_sad8x4x3d_neon, 10),
+ make_tuple(4, 8, &aom_highbd_sad4x8x3d_neon, 10),
+ make_tuple(4, 4, &aom_highbd_sad4x4x3d_neon, 10),
+ make_tuple(128, 128, &aom_highbd_sad128x128x3d_neon, 12),
+ make_tuple(128, 64, &aom_highbd_sad128x64x3d_neon, 12),
+ make_tuple(64, 128, &aom_highbd_sad64x128x3d_neon, 12),
+ make_tuple(64, 64, &aom_highbd_sad64x64x3d_neon, 12),
+ make_tuple(64, 32, &aom_highbd_sad64x32x3d_neon, 12),
+ make_tuple(32, 64, &aom_highbd_sad32x64x3d_neon, 12),
+ make_tuple(32, 32, &aom_highbd_sad32x32x3d_neon, 12),
+ make_tuple(32, 16, &aom_highbd_sad32x16x3d_neon, 12),
+ make_tuple(16, 32, &aom_highbd_sad16x32x3d_neon, 12),
+ make_tuple(16, 16, &aom_highbd_sad16x16x3d_neon, 12),
+ make_tuple(16, 8, &aom_highbd_sad16x8x3d_neon, 12),
+ make_tuple(8, 16, &aom_highbd_sad8x16x3d_neon, 12),
+ make_tuple(8, 8, &aom_highbd_sad8x8x3d_neon, 12),
+ make_tuple(8, 4, &aom_highbd_sad8x4x3d_neon, 12),
+ make_tuple(4, 8, &aom_highbd_sad4x8x3d_neon, 12),
+ make_tuple(4, 4, &aom_highbd_sad4x4x3d_neon, 12),
+#endif // CONFIG_AV1_HIGHBITDEPTH
#if !CONFIG_REALTIME_ONLY
make_tuple(64, 16, &aom_sad64x16x3d_neon, -1),
make_tuple(32, 8, &aom_sad32x8x3d_neon, -1),
@@ -2262,12 +2282,189 @@
make_tuple(16, 4, &aom_sad16x4x3d_neon, -1),
make_tuple(8, 32, &aom_sad8x32x3d_neon, -1),
make_tuple(4, 16, &aom_sad4x16x3d_neon, -1),
+#if CONFIG_AV1_HIGHBITDEPTH
+ make_tuple(64, 16, &aom_highbd_sad64x16x3d_neon, 8),
+ make_tuple(16, 64, &aom_highbd_sad16x64x3d_neon, 8),
+ make_tuple(32, 8, &aom_highbd_sad32x8x3d_neon, 8),
+ make_tuple(8, 32, &aom_highbd_sad8x32x3d_neon, 8),
+ make_tuple(16, 4, &aom_highbd_sad16x4x3d_neon, 8),
+ make_tuple(4, 16, &aom_highbd_sad4x16x3d_neon, 8),
+ make_tuple(64, 16, &aom_highbd_sad64x16x3d_neon, 10),
+ make_tuple(16, 64, &aom_highbd_sad16x64x3d_neon, 10),
+ make_tuple(32, 8, &aom_highbd_sad32x8x3d_neon, 10),
+ make_tuple(8, 32, &aom_highbd_sad8x32x3d_neon, 10),
+ make_tuple(16, 4, &aom_highbd_sad16x4x3d_neon, 10),
+ make_tuple(4, 16, &aom_highbd_sad4x16x3d_neon, 10),
+ make_tuple(64, 16, &aom_highbd_sad64x16x3d_neon, 12),
+ make_tuple(16, 64, &aom_highbd_sad16x64x3d_neon, 12),
+ make_tuple(32, 8, &aom_highbd_sad32x8x3d_neon, 12),
+ make_tuple(8, 32, &aom_highbd_sad8x32x3d_neon, 12),
+ make_tuple(16, 4, &aom_highbd_sad16x4x3d_neon, 12),
+ make_tuple(4, 16, &aom_highbd_sad4x16x3d_neon, 12),
+#endif // CONFIG_AV1_HIGHBITDEPTH
#endif // !CONFIG_REALTIME_ONLY
};
INSTANTIATE_TEST_SUITE_P(NEON, SADx3Test, ::testing::ValuesIn(x3d_neon_tests));
#endif // HAVE_NEON
+#if HAVE_NEON_DOTPROD
+const SadMxNParam neon_dotprod_tests[] = {
+ make_tuple(128, 128, &aom_sad128x128_neon_dotprod, -1),
+ make_tuple(128, 64, &aom_sad128x64_neon_dotprod, -1),
+ make_tuple(64, 128, &aom_sad64x128_neon_dotprod, -1),
+ make_tuple(64, 64, &aom_sad64x64_neon_dotprod, -1),
+ make_tuple(64, 32, &aom_sad64x32_neon_dotprod, -1),
+ make_tuple(32, 64, &aom_sad32x64_neon_dotprod, -1),
+ make_tuple(32, 32, &aom_sad32x32_neon_dotprod, -1),
+ make_tuple(32, 16, &aom_sad32x16_neon_dotprod, -1),
+ make_tuple(16, 32, &aom_sad16x32_neon_dotprod, -1),
+ make_tuple(16, 16, &aom_sad16x16_neon_dotprod, -1),
+ make_tuple(16, 8, &aom_sad16x8_neon_dotprod, -1),
+#if !CONFIG_REALTIME_ONLY
+ make_tuple(64, 16, &aom_sad64x16_neon_dotprod, -1),
+ make_tuple(32, 8, &aom_sad32x8_neon_dotprod, -1),
+ make_tuple(16, 64, &aom_sad16x64_neon_dotprod, -1),
+ make_tuple(16, 4, &aom_sad16x4_neon_dotprod, -1),
+#endif // !CONFIG_REALTIME_ONLY
+};
+INSTANTIATE_TEST_SUITE_P(NEON_DOTPROD, SADTest,
+ ::testing::ValuesIn(neon_dotprod_tests));
+
+const SadMxNParam skip_neon_dotprod_tests[] = {
+ make_tuple(128, 128, &aom_sad_skip_128x128_neon_dotprod, -1),
+ make_tuple(128, 64, &aom_sad_skip_128x64_neon_dotprod, -1),
+ make_tuple(64, 128, &aom_sad_skip_64x128_neon_dotprod, -1),
+ make_tuple(64, 64, &aom_sad_skip_64x64_neon_dotprod, -1),
+ make_tuple(64, 32, &aom_sad_skip_64x32_neon_dotprod, -1),
+ make_tuple(32, 64, &aom_sad_skip_32x64_neon_dotprod, -1),
+ make_tuple(32, 32, &aom_sad_skip_32x32_neon_dotprod, -1),
+ make_tuple(32, 16, &aom_sad_skip_32x16_neon_dotprod, -1),
+ make_tuple(16, 32, &aom_sad_skip_16x32_neon_dotprod, -1),
+ make_tuple(16, 16, &aom_sad_skip_16x16_neon_dotprod, -1),
+ make_tuple(16, 8, &aom_sad_skip_16x8_neon_dotprod, -1),
+#if !CONFIG_REALTIME_ONLY
+ make_tuple(64, 16, &aom_sad_skip_64x16_neon_dotprod, -1),
+ make_tuple(32, 8, &aom_sad_skip_32x8_neon_dotprod, -1),
+ make_tuple(16, 64, &aom_sad_skip_16x64_neon_dotprod, -1),
+ make_tuple(16, 4, &aom_sad_skip_16x4_neon_dotprod, -1),
+#endif // !CONFIG_REALTIME_ONLY
+};
+INSTANTIATE_TEST_SUITE_P(NEON_DOTPROD, SADSkipTest,
+ ::testing::ValuesIn(skip_neon_dotprod_tests));
+
+const SadMxNAvgParam avg_neon_dotprod_tests[] = {
+ make_tuple(128, 128, &aom_sad128x128_avg_neon_dotprod, -1),
+ make_tuple(128, 64, &aom_sad128x64_avg_neon_dotprod, -1),
+ make_tuple(64, 128, &aom_sad64x128_avg_neon_dotprod, -1),
+ make_tuple(64, 64, &aom_sad64x64_avg_neon_dotprod, -1),
+ make_tuple(64, 32, &aom_sad64x32_avg_neon_dotprod, -1),
+ make_tuple(32, 64, &aom_sad32x64_avg_neon_dotprod, -1),
+ make_tuple(32, 32, &aom_sad32x32_avg_neon_dotprod, -1),
+ make_tuple(32, 16, &aom_sad32x16_avg_neon_dotprod, -1),
+ make_tuple(16, 32, &aom_sad16x32_avg_neon_dotprod, -1),
+ make_tuple(16, 16, &aom_sad16x16_avg_neon_dotprod, -1),
+ make_tuple(16, 8, &aom_sad16x8_avg_neon_dotprod, -1),
+#if !CONFIG_REALTIME_ONLY
+ make_tuple(64, 16, &aom_sad64x16_avg_neon_dotprod, -1),
+ make_tuple(32, 8, &aom_sad32x8_avg_neon_dotprod, -1),
+ make_tuple(16, 64, &aom_sad16x64_avg_neon_dotprod, -1),
+ make_tuple(16, 4, &aom_sad16x4_avg_neon_dotprod, -1),
+#endif // !CONFIG_REALTIME_ONLY
+};
+INSTANTIATE_TEST_SUITE_P(NEON_DOTPROD, SADavgTest,
+ ::testing::ValuesIn(avg_neon_dotprod_tests));
+
+const DistWtdSadMxNAvgParam dist_wtd_avg_neon_dotprod_tests[] = {
+ make_tuple(128, 128, &aom_dist_wtd_sad128x128_avg_neon_dotprod, -1),
+ make_tuple(128, 64, &aom_dist_wtd_sad128x64_avg_neon_dotprod, -1),
+ make_tuple(64, 128, &aom_dist_wtd_sad64x128_avg_neon_dotprod, -1),
+ make_tuple(64, 64, &aom_dist_wtd_sad64x64_avg_neon_dotprod, -1),
+ make_tuple(64, 32, &aom_dist_wtd_sad64x32_avg_neon_dotprod, -1),
+ make_tuple(32, 64, &aom_dist_wtd_sad32x64_avg_neon_dotprod, -1),
+ make_tuple(32, 32, &aom_dist_wtd_sad32x32_avg_neon_dotprod, -1),
+ make_tuple(32, 16, &aom_dist_wtd_sad32x16_avg_neon_dotprod, -1),
+ make_tuple(16, 32, &aom_dist_wtd_sad16x32_avg_neon_dotprod, -1),
+ make_tuple(16, 16, &aom_dist_wtd_sad16x16_avg_neon_dotprod, -1),
+ make_tuple(16, 8, &aom_dist_wtd_sad16x8_avg_neon_dotprod, -1),
+#if !CONFIG_REALTIME_ONLY
+ make_tuple(64, 16, &aom_dist_wtd_sad64x16_avg_neon_dotprod, -1),
+ make_tuple(16, 64, &aom_dist_wtd_sad16x64_avg_neon_dotprod, -1),
+ make_tuple(32, 8, &aom_dist_wtd_sad32x8_avg_neon_dotprod, -1),
+ make_tuple(16, 4, &aom_dist_wtd_sad16x4_avg_neon_dotprod, -1),
+#endif // !CONFIG_REALTIME_ONLY
+};
+
+INSTANTIATE_TEST_SUITE_P(NEON_DOTPROD, DistWtdSADavgTest,
+ ::testing::ValuesIn(dist_wtd_avg_neon_dotprod_tests));
+
+const SadMxNx4Param x3d_neon_dotprod_tests[] = {
+ make_tuple(128, 128, &aom_sad128x128x3d_neon_dotprod, -1),
+ make_tuple(128, 64, &aom_sad128x64x3d_neon_dotprod, -1),
+ make_tuple(64, 128, &aom_sad64x128x3d_neon_dotprod, -1),
+ make_tuple(64, 64, &aom_sad64x64x3d_neon_dotprod, -1),
+ make_tuple(64, 32, &aom_sad64x32x3d_neon_dotprod, -1),
+ make_tuple(32, 64, &aom_sad32x64x3d_neon_dotprod, -1),
+ make_tuple(32, 32, &aom_sad32x32x3d_neon_dotprod, -1),
+ make_tuple(32, 16, &aom_sad32x16x3d_neon_dotprod, -1),
+ make_tuple(16, 32, &aom_sad16x32x3d_neon_dotprod, -1),
+ make_tuple(16, 16, &aom_sad16x16x3d_neon_dotprod, -1),
+ make_tuple(16, 8, &aom_sad16x8x3d_neon_dotprod, -1),
+#if !CONFIG_REALTIME_ONLY
+ make_tuple(64, 16, &aom_sad64x16x3d_neon_dotprod, -1),
+ make_tuple(32, 8, &aom_sad32x8x3d_neon_dotprod, -1),
+ make_tuple(16, 64, &aom_sad16x64x3d_neon_dotprod, -1),
+ make_tuple(16, 4, &aom_sad16x4x3d_neon_dotprod, -1),
+#endif // !CONFIG_REALTIME_ONLY
+};
+INSTANTIATE_TEST_SUITE_P(NEON_DOTPROD, SADx3Test,
+ ::testing::ValuesIn(x3d_neon_dotprod_tests));
+
+const SadMxNx4Param x4d_neon_dotprod_tests[] = {
+ make_tuple(128, 128, &aom_sad128x128x4d_neon_dotprod, -1),
+ make_tuple(128, 64, &aom_sad128x64x4d_neon_dotprod, -1),
+ make_tuple(64, 128, &aom_sad64x128x4d_neon_dotprod, -1),
+ make_tuple(64, 64, &aom_sad64x64x4d_neon_dotprod, -1),
+ make_tuple(64, 32, &aom_sad64x32x4d_neon_dotprod, -1),
+ make_tuple(32, 64, &aom_sad32x64x4d_neon_dotprod, -1),
+ make_tuple(32, 32, &aom_sad32x32x4d_neon_dotprod, -1),
+ make_tuple(32, 16, &aom_sad32x16x4d_neon_dotprod, -1),
+ make_tuple(16, 32, &aom_sad16x32x4d_neon_dotprod, -1),
+ make_tuple(16, 16, &aom_sad16x16x4d_neon_dotprod, -1),
+ make_tuple(16, 8, &aom_sad16x8x4d_neon_dotprod, -1),
+#if !CONFIG_REALTIME_ONLY
+ make_tuple(64, 16, &aom_sad64x16x4d_neon_dotprod, -1),
+ make_tuple(32, 8, &aom_sad32x8x4d_neon_dotprod, -1),
+ make_tuple(16, 64, &aom_sad16x64x4d_neon_dotprod, -1),
+ make_tuple(16, 4, &aom_sad16x4x4d_neon_dotprod, -1),
+#endif // !CONFIG_REALTIME_ONLY
+};
+INSTANTIATE_TEST_SUITE_P(NEON_DOTPROD, SADx4Test,
+ ::testing::ValuesIn(x4d_neon_dotprod_tests));
+
+const SadSkipMxNx4Param skip_x4d_neon_dotprod_tests[] = {
+ make_tuple(128, 128, &aom_sad_skip_128x128x4d_neon_dotprod, -1),
+ make_tuple(128, 64, &aom_sad_skip_128x64x4d_neon_dotprod, -1),
+ make_tuple(64, 128, &aom_sad_skip_64x128x4d_neon_dotprod, -1),
+ make_tuple(64, 64, &aom_sad_skip_64x64x4d_neon_dotprod, -1),
+ make_tuple(64, 32, &aom_sad_skip_64x32x4d_neon_dotprod, -1),
+ make_tuple(32, 64, &aom_sad_skip_32x64x4d_neon_dotprod, -1),
+ make_tuple(32, 32, &aom_sad_skip_32x32x4d_neon_dotprod, -1),
+ make_tuple(32, 16, &aom_sad_skip_32x16x4d_neon_dotprod, -1),
+ make_tuple(16, 32, &aom_sad_skip_16x32x4d_neon_dotprod, -1),
+ make_tuple(16, 16, &aom_sad_skip_16x16x4d_neon_dotprod, -1),
+ make_tuple(16, 8, &aom_sad_skip_16x8x4d_neon_dotprod, -1),
+#if !CONFIG_REALTIME_ONLY
+ make_tuple(64, 16, &aom_sad_skip_64x16x4d_neon_dotprod, -1),
+ make_tuple(32, 8, &aom_sad_skip_32x8x4d_neon_dotprod, -1),
+ make_tuple(16, 64, &aom_sad_skip_16x64x4d_neon_dotprod, -1),
+ make_tuple(16, 4, &aom_sad_skip_16x4x4d_neon_dotprod, -1),
+#endif // !CONFIG_REALTIME_ONLY
+};
+INSTANTIATE_TEST_SUITE_P(NEON_DOTPROD, SADSkipx4Test,
+ ::testing::ValuesIn(skip_x4d_neon_dotprod_tests));
+#endif // HAVE_NEON_DOTPROD
+
//------------------------------------------------------------------------------
// x86 functions
#if HAVE_SSE2
@@ -2750,39 +2947,6 @@
// Only functions are x3, which do not have tests.
#endif // HAVE_SSE3
-#if HAVE_SSSE3
-const DistWtdCompAvgParam dist_wtd_comp_avg_ssse3_tests[] = {
- make_tuple(128, 128, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
- make_tuple(128, 64, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
- make_tuple(64, 128, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
- make_tuple(64, 64, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
- make_tuple(64, 32, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
- make_tuple(32, 64, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
- make_tuple(32, 32, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
- make_tuple(32, 16, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
- make_tuple(16, 32, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
- make_tuple(16, 16, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
- make_tuple(16, 8, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
- make_tuple(8, 16, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
- make_tuple(8, 8, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
- make_tuple(8, 4, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
- make_tuple(4, 8, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
- make_tuple(4, 4, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
- make_tuple(16, 16, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
-#if !CONFIG_REALTIME_ONLY
- make_tuple(64, 16, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
- make_tuple(16, 64, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
- make_tuple(32, 8, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
- make_tuple(8, 32, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
- make_tuple(16, 4, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
- make_tuple(4, 16, &aom_dist_wtd_comp_avg_pred_ssse3, -1),
-#endif
-};
-
-INSTANTIATE_TEST_SUITE_P(SSSE3, DistWtdCompAvgTest,
- ::testing::ValuesIn(dist_wtd_comp_avg_ssse3_tests));
-#endif // HAVE_SSSE3
-
#if HAVE_SSE4_1
// Only functions are x8, which do not have tests.
#endif // HAVE_SSE4_1
diff --git a/test/sb_multipass_test.cc b/test/sb_multipass_test.cc
index 8ddc002..e27a2c6 100644
--- a/test/sb_multipass_test.cc
+++ b/test/sb_multipass_test.cc
@@ -42,9 +42,9 @@
md5_dec_.clear();
md5_enc_.clear();
}
- virtual ~AV1SBMultipassTest() { delete decoder_; }
+ ~AV1SBMultipassTest() override { delete decoder_; }
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(::libaom_test::kTwoPassGood);
cfg_.g_lag_in_frames = 5;
@@ -56,8 +56,8 @@
cfg_.rc_min_quantizer = 0;
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
SetTileSize(encoder);
encoder->Control(AOME_SET_CPUUSED, set_cpu_used_);
@@ -75,7 +75,7 @@
encoder->Control(AV1E_SET_TILE_ROWS, 1);
}
- virtual void FramePktHook(const aom_codec_cx_pkt_t *pkt) {
+ void FramePktHook(const aom_codec_cx_pkt_t *pkt) override {
size_enc_.push_back(pkt->data.frame.sz);
::libaom_test::MD5 md5_enc;
diff --git a/test/scalability_test.cc b/test/scalability_test.cc
index 9ea8256..12cb03c 100644
--- a/test/scalability_test.cc
+++ b/test/scalability_test.cc
@@ -26,15 +26,15 @@
public ::libaom_test::EncoderTest {
protected:
ScalabilityTest() : EncoderTest(GET_PARAM(0)) {}
- virtual ~ScalabilityTest() {}
+ ~ScalabilityTest() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(GET_PARAM(1));
num_spatial_layers_ = 2;
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AOME_SET_CPUUSED, kCpuUsed);
encoder->Control(AOME_SET_NUMBER_SPATIAL_LAYERS, num_spatial_layers_);
diff --git a/test/screen_content_test.cc b/test/screen_content_test.cc
index 4d3e09a..974c50b 100644
--- a/test/screen_content_test.cc
+++ b/test/screen_content_test.cc
@@ -29,9 +29,9 @@
is_screen_content_violated_ = true;
tune_content_ = AOM_CONTENT_DEFAULT;
}
- virtual ~ScreenContentToolsTestLarge() {}
+ ~ScreenContentToolsTestLarge() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(encoding_mode_);
const aom_rational timebase = { 1, 30 };
cfg_.g_timebase = timebase;
@@ -42,10 +42,10 @@
cfg_.g_profile = 0;
}
- virtual bool DoDecode() const { return 1; }
+ bool DoDecode() const override { return true; }
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AOME_SET_CPUUSED, 5);
encoder->Control(AOME_SET_ENABLEAUTOALTREF, 1);
@@ -53,8 +53,8 @@
}
}
- virtual bool HandleDecodeResult(const aom_codec_err_t res_dec,
- libaom_test::Decoder *decoder) {
+ bool HandleDecodeResult(const aom_codec_err_t res_dec,
+ libaom_test::Decoder *decoder) override {
EXPECT_EQ(AOM_CODEC_OK, res_dec) << decoder->DecodeError();
if (AOM_CODEC_OK == res_dec) {
aom_codec_ctx_t *ctx_dec = decoder->GetDecoder();
diff --git a/test/selfguided_filter_test.cc b/test/selfguided_filter_test.cc
index a8461b5..3dd513b 100644
--- a/test/selfguided_filter_test.cc
+++ b/test/selfguided_filter_test.cc
@@ -30,9 +30,9 @@
using std::make_tuple;
using std::tuple;
-typedef void (*SgrFunc)(const uint8_t *dat8, int width, int height, int stride,
- int eps, const int *xqd, uint8_t *dst8, int dst_stride,
- int32_t *tmpbuf, int bit_depth, int highbd);
+typedef int (*SgrFunc)(const uint8_t *dat8, int width, int height, int stride,
+ int eps, const int *xqd, uint8_t *dst8, int dst_stride,
+ int32_t *tmpbuf, int bit_depth, int highbd);
// Test parameter list:
// <tst_fun_>
@@ -41,10 +41,8 @@
class AV1SelfguidedFilterTest
: public ::testing::TestWithParam<FilterTestParam> {
public:
- virtual ~AV1SelfguidedFilterTest() {}
- virtual void SetUp() {}
-
- virtual void TearDown() {}
+ ~AV1SelfguidedFilterTest() override = default;
+ void SetUp() override {}
protected:
void RunSpeedTest() {
@@ -91,9 +89,10 @@
int h = AOMMIN(pu_height, height - k);
uint8_t *input_p = input + k * stride + j;
uint8_t *output_p = output + k * out_stride + j;
- av1_apply_selfguided_restoration_c(input_p, w, h, stride, eps, xqd,
- output_p, out_stride, tmpbuf, 8,
- 0);
+ const int ret_c = av1_apply_selfguided_restoration_c(
+ input_p, w, h, stride, eps, xqd, output_p, out_stride, tmpbuf, 8,
+ 0);
+ ASSERT_EQ(ret_c, 0);
}
}
aom_usec_timer_mark(&ref_timer);
@@ -108,8 +107,9 @@
int h = AOMMIN(pu_height, height - k);
uint8_t *input_p = input + k * stride + j;
uint8_t *output_p = output + k * out_stride + j;
- tst_fun_(input_p, w, h, stride, eps, xqd, output_p, out_stride,
- tmpbuf, 8, 0);
+ const int ret_tst = tst_fun_(input_p, w, h, stride, eps, xqd,
+ output_p, out_stride, tmpbuf, 8, 0);
+ ASSERT_EQ(ret_tst, 0);
}
}
aom_usec_timer_mark(&tst_timer);
@@ -181,11 +181,13 @@
uint8_t *input_p = input + k * stride + j;
uint8_t *output_p = output + k * out_stride + j;
uint8_t *output2_p = output2 + k * out_stride + j;
- tst_fun_(input_p, w, h, stride, eps, xqd, output_p, out_stride,
- tmpbuf, 8, 0);
- av1_apply_selfguided_restoration_c(input_p, w, h, stride, eps, xqd,
- output2_p, out_stride, tmpbuf, 8,
- 0);
+ const int ret_tst = tst_fun_(input_p, w, h, stride, eps, xqd,
+ output_p, out_stride, tmpbuf, 8, 0);
+ ASSERT_EQ(ret_tst, 0);
+ const int ret_c = av1_apply_selfguided_restoration_c(
+ input_p, w, h, stride, eps, xqd, output2_p, out_stride, tmpbuf, 8,
+ 0);
+ ASSERT_EQ(ret_c, 0);
}
for (j = 0; j < test_h; ++j)
@@ -234,10 +236,8 @@
class AV1HighbdSelfguidedFilterTest
: public ::testing::TestWithParam<HighbdFilterTestParam> {
public:
- virtual ~AV1HighbdSelfguidedFilterTest() {}
- virtual void SetUp() {}
-
- virtual void TearDown() {}
+ ~AV1HighbdSelfguidedFilterTest() override = default;
+ void SetUp() override {}
protected:
void RunSpeedTest() {
diff --git a/test/sharpness_test.cc b/test/sharpness_test.cc
index 49c5804..64465c8 100644
--- a/test/sharpness_test.cc
+++ b/test/sharpness_test.cc
@@ -48,7 +48,7 @@
cpu_used_(GET_PARAM(2)), sharpness_level_(GET_PARAM(3)), psnr_(0.0),
nframes_(0) {}
- ~SharpnessTest() override {}
+ ~SharpnessTest() override = default;
void SetUp() override {
InitializeConfig(encoding_mode_);
diff --git a/test/simd_cmp_neon.cc b/test/simd_cmp_neon.cc
deleted file mode 100644
index 53c1e2a..0000000
--- a/test/simd_cmp_neon.cc
+++ /dev/null
@@ -1,17 +0,0 @@
-/*
- * Copyright (c) 2016, Alliance for Open Media. All rights reserved
- *
- * This source code is subject to the terms of the BSD 2 Clause License and
- * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
- * was not distributed with this source code in the LICENSE file, you can
- * obtain it at www.aomedia.org/license/software. If the Alliance for Open
- * Media Patent License 1.0 was not distributed with this source code in the
- * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
- */
-
-#if defined(__OPTIMIZE__) && __OPTIMIZE__
-#define ARCH NEON
-#define ARCH_POSTFIX(name) name##_neon
-#define SIMD_NAMESPACE simd_test_neon
-#include "test/simd_cmp_impl.h"
-#endif
diff --git a/test/simd_impl.h b/test/simd_impl.h
index 8535e37..b564a7f 100644
--- a/test/simd_impl.h
+++ b/test/simd_impl.h
@@ -22,15 +22,13 @@
template <typename param_signature>
class TestIntrinsic : public ::testing::TestWithParam<param_signature> {
public:
- virtual ~TestIntrinsic() {}
- virtual void SetUp() {
+ ~TestIntrinsic() override = default;
+ void SetUp() override {
mask = std::get<0>(this->GetParam());
maskwidth = std::get<1>(this->GetParam());
name = std::get<2>(this->GetParam());
}
- virtual void TearDown() {}
-
protected:
uint32_t mask, maskwidth;
const char *name;
diff --git a/test/simd_neon_test.cc b/test/simd_neon_test.cc
deleted file mode 100644
index b67b188..0000000
--- a/test/simd_neon_test.cc
+++ /dev/null
@@ -1,17 +0,0 @@
-/*
- * Copyright (c) 2016, Alliance for Open Media. All rights reserved
- *
- * This source code is subject to the terms of the BSD 2 Clause License and
- * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
- * was not distributed with this source code in the LICENSE file, you can
- * obtain it at www.aomedia.org/license/software. If the Alliance for Open
- * Media Patent License 1.0 was not distributed with this source code in the
- * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
- */
-
-#if defined(__OPTIMIZE__) && __OPTIMIZE__
-#define ARCH NEON
-#define ARCH_POSTFIX(name) name##_neon
-#define SIMD_NAMESPACE simd_test_neon
-#include "test/simd_impl.h"
-#endif
diff --git a/test/sse_sum_test.cc b/test/sse_sum_test.cc
index 68355ec..70d8da5 100644
--- a/test/sse_sum_test.cc
+++ b/test/sse_sum_test.cc
@@ -41,15 +41,15 @@
class SumSSETest : public ::testing::TestWithParam<TestFuncs> {
public:
- virtual ~SumSSETest() {}
- virtual void SetUp() {
+ ~SumSSETest() override = default;
+ void SetUp() override {
params_ = this->GetParam();
rnd_.Reset(ACMRandom::DeterministicSeed());
src_ = reinterpret_cast<int16_t *>(aom_memalign(16, 256 * 256 * 2));
ASSERT_NE(src_, nullptr);
}
- virtual void TearDown() { aom_free(src_); }
+ void TearDown() override { aom_free(src_); }
void RunTest(int isRandom);
void RunSpeedTest();
diff --git a/test/still_picture_test.cc b/test/still_picture_test.cc
index e2eef94..3dfb1c8 100644
--- a/test/still_picture_test.cc
+++ b/test/still_picture_test.cc
@@ -27,9 +27,9 @@
enable_full_header_(GET_PARAM(2)) {
still_picture_coding_violated_ = false;
}
- virtual ~StillPicturePresenceTest() {}
+ ~StillPicturePresenceTest() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(encoding_mode_);
const aom_rational timebase = { 1, 30 };
cfg_.g_timebase = timebase;
@@ -39,18 +39,18 @@
cfg_.g_limit = 1;
}
- virtual bool DoDecode() const { return 1; }
+ bool DoDecode() const override { return true; }
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AOME_SET_CPUUSED, 5);
encoder->Control(AV1E_SET_FORCE_VIDEO_MODE, 0);
}
}
- virtual bool HandleDecodeResult(const aom_codec_err_t res_dec,
- libaom_test::Decoder *decoder) {
+ bool HandleDecodeResult(const aom_codec_err_t res_dec,
+ libaom_test::Decoder *decoder) override {
EXPECT_EQ(AOM_CODEC_OK, res_dec) << decoder->DecodeError();
if (AOM_CODEC_OK == res_dec) {
aom_codec_ctx_t *ctx_dec = decoder->GetDecoder();
diff --git a/test/subtract_test.cc b/test/subtract_test.cc
index 4003e51..e591e65 100644
--- a/test/subtract_test.cc
+++ b/test/subtract_test.cc
@@ -49,15 +49,15 @@
func_ = func;
ref_func_ = ref_func;
if (bit_depth == -1) {
- hbd_ = 0;
+ hbd_ = false;
bit_depth_ = AOM_BITS_8;
} else {
- hbd_ = 1;
+ hbd_ = true;
bit_depth_ = static_cast<aom_bit_depth_t>(bit_depth);
}
}
- virtual void SetUp() {
+ void SetUp() override {
rnd_.Reset(ACMRandom::DeterministicSeed());
const size_t max_width = 128;
@@ -82,7 +82,7 @@
ASSERT_NE(diff_, nullptr);
}
- virtual void TearDown() {
+ void TearDown() override {
if (hbd_) {
aom_free(CONVERT_TO_SHORTPTR(src_));
aom_free(CONVERT_TO_SHORTPTR(pred_));
diff --git a/test/sum_squares_test.cc b/test/sum_squares_test.cc
index 91f172d..cba33b7 100644
--- a/test/sum_squares_test.cc
+++ b/test/sum_squares_test.cc
@@ -20,6 +20,7 @@
#include "config/aom_dsp_rtcd.h"
#include "aom_ports/mem.h"
+#include "av1/common/common_data.h"
#include "test/acm_random.h"
#include "test/register_state_check.h"
#include "test/util.h"
@@ -43,16 +44,16 @@
class SumSquaresTest : public ::testing::TestWithParam<TestFuncs> {
public:
- virtual ~SumSquaresTest() {}
- virtual void SetUp() {
+ ~SumSquaresTest() override = default;
+ void SetUp() override {
params_ = this->GetParam();
rnd_.Reset(ACMRandom::DeterministicSeed());
src_ = reinterpret_cast<int16_t *>(aom_memalign(16, 256 * 256 * 2));
ASSERT_NE(src_, nullptr);
}
- virtual void TearDown() { aom_free(src_); }
- void RunTest(int isRandom);
+ void TearDown() override { aom_free(src_); }
+ void RunTest(bool is_random);
void RunSpeedTest();
void GenRandomData(int width, int height, int stride) {
@@ -83,7 +84,7 @@
};
GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(SumSquaresTest);
-void SumSquaresTest::RunTest(int isRandom) {
+void SumSquaresTest::RunTest(bool is_random) {
int failed = 0;
for (int k = 0; k < kNumIterations; k++) {
const int width = 4 * (rnd_(31) + 1); // Up to 128x128
@@ -92,7 +93,7 @@
while (stride < width) { // Make sure it's valid
stride = 4 << rnd_(7);
}
- if (isRandom) {
+ if (is_random) {
GenRandomData(width, height, stride);
} else {
GenExtremeData(width, height, stride);
@@ -144,11 +145,11 @@
}
TEST_P(SumSquaresTest, OperationCheck) {
- RunTest(1); // GenRandomData
+ RunTest(true); // GenRandomData
}
TEST_P(SumSquaresTest, ExtremeValues) {
- RunTest(0); // GenExtremeData
+ RunTest(false); // GenExtremeData
}
TEST_P(SumSquaresTest, DISABLED_Speed) { RunSpeedTest(); }
@@ -182,7 +183,7 @@
// 1D version
//////////////////////////////////////////////////////////////////////////////
-typedef uint64_t (*F1D)(const int16_t *src, uint32_t N);
+typedef uint64_t (*F1D)(const int16_t *src, uint32_t n);
typedef libaom_test::FuncParam<F1D> TestFuncs1D;
class SumSquares1DTest : public FunctionEquivalenceTest<F1D> {
@@ -199,12 +200,12 @@
for (int i = 0; i < kMaxSize * kMaxSize; ++i)
src[i] = rng_(kInt13Max * 2 + 1) - kInt13Max;
- const int N = rng_(2) ? rng_(kMaxSize * kMaxSize + 1 - kMaxSize) + kMaxSize
+ const int n = rng_(2) ? rng_(kMaxSize * kMaxSize + 1 - kMaxSize) + kMaxSize
: rng_(kMaxSize) + 1;
- const uint64_t ref_res = params_.ref_func(src, N);
+ const uint64_t ref_res = params_.ref_func(src, n);
uint64_t tst_res;
- API_REGISTER_STATE_CHECK(tst_res = params_.tst_func(src, N));
+ API_REGISTER_STATE_CHECK(tst_res = params_.tst_func(src, n));
ASSERT_EQ(ref_res, tst_res);
}
@@ -220,12 +221,12 @@
for (int i = 0; i < kMaxSize * kMaxSize; ++i) src[i] = -kInt13Max;
}
- const int N = rng_(2) ? rng_(kMaxSize * kMaxSize + 1 - kMaxSize) + kMaxSize
+ const int n = rng_(2) ? rng_(kMaxSize * kMaxSize + 1 - kMaxSize) + kMaxSize
: rng_(kMaxSize) + 1;
- const uint64_t ref_res = params_.ref_func(src, N);
+ const uint64_t ref_res = params_.ref_func(src, n);
uint64_t tst_res;
- API_REGISTER_STATE_CHECK(tst_res = params_.tst_func(src, N));
+ API_REGISTER_STATE_CHECK(tst_res = params_.tst_func(src, n));
ASSERT_EQ(ref_res, tst_res);
}
@@ -245,23 +246,23 @@
#endif // HAVE_NEON
-typedef int64_t (*sse_func)(const uint8_t *a, int a_stride, const uint8_t *b,
- int b_stride, int width, int height);
-typedef libaom_test::FuncParam<sse_func> TestSSEFuncs;
+typedef int64_t (*SSEFunc)(const uint8_t *a, int a_stride, const uint8_t *b,
+ int b_stride, int width, int height);
+typedef libaom_test::FuncParam<SSEFunc> TestSSEFuncs;
typedef std::tuple<TestSSEFuncs, int> SSETestParam;
class SSETest : public ::testing::TestWithParam<SSETestParam> {
public:
- virtual ~SSETest() {}
- virtual void SetUp() {
+ ~SSETest() override = default;
+ void SetUp() override {
params_ = GET_PARAM(0);
width_ = GET_PARAM(1);
- isHbd_ =
+ is_hbd_ =
#if CONFIG_AV1_HIGHBITDEPTH
params_.ref_func == aom_highbd_sse_c;
#else
- 0;
+ false;
#endif
rnd_.Reset(ACMRandom::DeterministicSeed());
src_ = reinterpret_cast<uint8_t *>(aom_memalign(32, 256 * 256 * 2));
@@ -270,25 +271,25 @@
ASSERT_NE(ref_, nullptr);
}
- virtual void TearDown() {
+ void TearDown() override {
aom_free(src_);
aom_free(ref_);
}
- void RunTest(int isRandom, int width, int height, int run_times);
+ void RunTest(bool is_random, int width, int height, int run_times);
void GenRandomData(int width, int height, int stride) {
- uint16_t *pSrc = (uint16_t *)src_;
- uint16_t *pRef = (uint16_t *)ref_;
+ uint16_t *src16 = reinterpret_cast<uint16_t *>(src_);
+ uint16_t *ref16 = reinterpret_cast<uint16_t *>(ref_);
const int msb = 11; // Up to 12 bit input
const int limit = 1 << (msb + 1);
for (int ii = 0; ii < height; ii++) {
for (int jj = 0; jj < width; jj++) {
- if (!isHbd_) {
+ if (!is_hbd_) {
src_[ii * stride + jj] = rnd_.Rand8();
ref_[ii * stride + jj] = rnd_.Rand8();
} else {
- pSrc[ii * stride + jj] = rnd_(limit);
- pRef[ii * stride + jj] = rnd_(limit);
+ src16[ii * stride + jj] = rnd_(limit);
+ ref16[ii * stride + jj] = rnd_(limit);
}
}
}
@@ -296,20 +297,20 @@
void GenExtremeData(int width, int height, int stride, uint8_t *data,
int16_t val) {
- uint16_t *pData = (uint16_t *)data;
+ uint16_t *data16 = reinterpret_cast<uint16_t *>(data);
for (int ii = 0; ii < height; ii++) {
for (int jj = 0; jj < width; jj++) {
- if (!isHbd_) {
- data[ii * stride + jj] = (uint8_t)val;
+ if (!is_hbd_) {
+ data[ii * stride + jj] = static_cast<uint8_t>(val);
} else {
- pData[ii * stride + jj] = val;
+ data16[ii * stride + jj] = val;
}
}
}
}
protected:
- int isHbd_;
+ bool is_hbd_;
int width_;
TestSSEFuncs params_;
uint8_t *src_;
@@ -318,7 +319,7 @@
};
GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(SSETest);
-void SSETest::RunTest(int isRandom, int width, int height, int run_times) {
+void SSETest::RunTest(bool is_random, int width, int height, int run_times) {
int failed = 0;
aom_usec_timer ref_timer, test_timer;
for (int k = 0; k < 3; k++) {
@@ -326,10 +327,10 @@
while (stride < width) { // Make sure it's valid
stride = 4 << rnd_(7);
}
- if (isRandom) {
+ if (is_random) {
GenRandomData(width, height, stride);
} else {
- const int msb = isHbd_ ? 12 : 8; // Up to 12 bit input
+ const int msb = is_hbd_ ? 12 : 8; // Up to 12 bit input
const int limit = (1 << msb) - 1;
if (k == 0) {
GenExtremeData(width, height, stride, src_, 0);
@@ -340,18 +341,18 @@
}
}
int64_t res_ref, res_tst;
- uint8_t *pSrc = src_;
- uint8_t *pRef = ref_;
- if (isHbd_) {
- pSrc = CONVERT_TO_BYTEPTR(src_);
- pRef = CONVERT_TO_BYTEPTR(ref_);
+ uint8_t *src = src_;
+ uint8_t *ref = ref_;
+ if (is_hbd_) {
+ src = CONVERT_TO_BYTEPTR(src_);
+ ref = CONVERT_TO_BYTEPTR(ref_);
}
- res_ref = params_.ref_func(pSrc, stride, pRef, stride, width, height);
- res_tst = params_.tst_func(pSrc, stride, pRef, stride, width, height);
+ res_ref = params_.ref_func(src, stride, ref, stride, width, height);
+ res_tst = params_.tst_func(src, stride, ref, stride, width, height);
if (run_times > 1) {
aom_usec_timer_start(&ref_timer);
for (int j = 0; j < run_times; j++) {
- params_.ref_func(pSrc, stride, pRef, stride, width, height);
+ params_.ref_func(src, stride, ref, stride, width, height);
}
aom_usec_timer_mark(&ref_timer);
const int elapsed_time_c =
@@ -359,7 +360,7 @@
aom_usec_timer_start(&test_timer);
for (int j = 0; j < run_times; j++) {
- params_.tst_func(pSrc, stride, pRef, stride, width, height);
+ params_.tst_func(src, stride, ref, stride, width, height);
}
aom_usec_timer_mark(&test_timer);
const int elapsed_time_simd =
@@ -374,7 +375,7 @@
if (!failed) {
failed = res_ref != res_tst;
EXPECT_EQ(res_ref, res_tst)
- << "Error:" << (isHbd_ ? "hbd " : " ") << k << " SSE Test ["
+ << "Error:" << (is_hbd_ ? "hbd " : " ") << k << " SSE Test ["
<< width << "x" << height
<< "] C output does not match optimized output.";
}
@@ -384,19 +385,19 @@
TEST_P(SSETest, OperationCheck) {
for (int height = 4; height <= 128; height += 4) {
- RunTest(1, width_, height, 1); // GenRandomData
+ RunTest(true, width_, height, 1); // GenRandomData
}
}
TEST_P(SSETest, ExtremeValues) {
for (int height = 4; height <= 128; height += 4) {
- RunTest(0, width_, height, 1);
+ RunTest(false, width_, height, 1);
}
}
TEST_P(SSETest, DISABLED_Speed) {
for (int height = 4; height <= 128; height += 4) {
- RunTest(1, width_, height, 100);
+ RunTest(true, width_, height, 100);
}
}
@@ -411,6 +412,14 @@
Combine(ValuesIn(sse_neon), Range(4, 129, 4)));
#endif // HAVE_NEON
+#if HAVE_NEON_DOTPROD
+TestSSEFuncs sse_neon_dotprod[] = {
+ TestSSEFuncs(&aom_sse_c, &aom_sse_neon_dotprod),
+};
+INSTANTIATE_TEST_SUITE_P(NEON_DOTPROD, SSETest,
+ Combine(ValuesIn(sse_neon_dotprod), Range(4, 129, 4)));
+#endif // HAVE_NEON_DOTPROD
+
#if HAVE_SSE4_1
TestSSEFuncs sse_sse4[] = {
TestSSEFuncs(&aom_sse_c, &aom_sse_sse4_1),
@@ -442,21 +451,20 @@
int *x_sum, int64_t *x2_sum);
typedef libaom_test::FuncParam<sse_sum_func> TestSSE_SumFuncs;
-typedef std::tuple<TestSSE_SumFuncs, int> SSE_SumTestParam;
+typedef std::tuple<TestSSE_SumFuncs, TX_SIZE> SSE_SumTestParam;
class SSE_Sum_Test : public ::testing::TestWithParam<SSE_SumTestParam> {
public:
- virtual ~SSE_Sum_Test() {}
- virtual void SetUp() {
+ ~SSE_Sum_Test() override = default;
+ void SetUp() override {
params_ = GET_PARAM(0);
- width_ = GET_PARAM(1);
rnd_.Reset(ACMRandom::DeterministicSeed());
src_ = reinterpret_cast<int16_t *>(aom_memalign(32, 256 * 256 * 2));
ASSERT_NE(src_, nullptr);
}
- virtual void TearDown() { aom_free(src_); }
- void RunTest(int isRandom, int width, int height, int run_times);
+ void TearDown() override { aom_free(src_); }
+ void RunTest(bool is_random, int tx_size, int run_times);
void GenRandomData(int width, int height, int stride) {
const int msb = 11; // Up to 12 bit input
@@ -478,21 +486,22 @@
}
protected:
- int width_;
TestSSE_SumFuncs params_;
int16_t *src_;
ACMRandom rnd_;
};
GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(SSE_Sum_Test);
-void SSE_Sum_Test::RunTest(int isRandom, int width, int height, int run_times) {
+void SSE_Sum_Test::RunTest(bool is_random, int tx_size, int run_times) {
aom_usec_timer ref_timer, test_timer;
+ int width = tx_size_wide[tx_size];
+ int height = tx_size_high[tx_size];
for (int k = 0; k < 3; k++) {
int stride = 4 << rnd_(7); // Up to 256 stride
while (stride < width) { // Make sure it's valid
stride = 4 << rnd_(7);
}
- if (isRandom) {
+ if (is_random) {
GenRandomData(width, height, stride);
} else {
const int msb = 12; // Up to 12 bit input
@@ -547,37 +556,45 @@
}
TEST_P(SSE_Sum_Test, OperationCheck) {
- for (int height = 4; height <= 64; height = height * 2) {
- RunTest(1, width_, height, 1); // GenRandomData
- }
+ RunTest(true, GET_PARAM(1), 1); // GenRandomData
}
-TEST_P(SSE_Sum_Test, ExtremeValues) {
- for (int height = 4; height <= 64; height = height * 2) {
- RunTest(0, width_, height, 1);
- }
-}
+TEST_P(SSE_Sum_Test, ExtremeValues) { RunTest(false, GET_PARAM(1), 1); }
-TEST_P(SSE_Sum_Test, DISABLED_Speed) {
- for (int height = 4; height <= 64; height = height * 2) {
- RunTest(1, width_, height, 10000);
- }
-}
+TEST_P(SSE_Sum_Test, DISABLED_Speed) { RunTest(true, GET_PARAM(1), 10000); }
+
+#if HAVE_SSE2 || HAVE_AVX2 || HAVE_NEON
+const TX_SIZE kValidBlockSize[] = { TX_4X4, TX_8X8, TX_16X16, TX_32X32,
+ TX_64X64, TX_4X8, TX_8X4, TX_8X16,
+ TX_16X8, TX_16X32, TX_32X16, TX_64X32,
+ TX_32X64, TX_4X16, TX_16X4, TX_8X32,
+ TX_32X8, TX_16X64, TX_64X16 };
+#endif
#if HAVE_SSE2
TestSSE_SumFuncs sse_sum_sse2[] = { TestSSE_SumFuncs(
&aom_get_blk_sse_sum_c, &aom_get_blk_sse_sum_sse2) };
INSTANTIATE_TEST_SUITE_P(SSE2, SSE_Sum_Test,
- Combine(ValuesIn(sse_sum_sse2), Range(4, 65, 4)));
+ Combine(ValuesIn(sse_sum_sse2),
+ ValuesIn(kValidBlockSize)));
#endif // HAVE_SSE2
#if HAVE_AVX2
TestSSE_SumFuncs sse_sum_avx2[] = { TestSSE_SumFuncs(
&aom_get_blk_sse_sum_c, &aom_get_blk_sse_sum_avx2) };
INSTANTIATE_TEST_SUITE_P(AVX2, SSE_Sum_Test,
- Combine(ValuesIn(sse_sum_avx2), Range(4, 65, 4)));
+ Combine(ValuesIn(sse_sum_avx2),
+ ValuesIn(kValidBlockSize)));
#endif // HAVE_AVX2
+#if HAVE_NEON
+TestSSE_SumFuncs sse_sum_neon[] = { TestSSE_SumFuncs(
+ &aom_get_blk_sse_sum_c, &aom_get_blk_sse_sum_neon) };
+INSTANTIATE_TEST_SUITE_P(NEON, SSE_Sum_Test,
+ Combine(ValuesIn(sse_sum_neon),
+ ValuesIn(kValidBlockSize)));
+#endif // HAVE_NEON
+
//////////////////////////////////////////////////////////////////////////////
// 2D Variance test functions
//////////////////////////////////////////////////////////////////////////////
@@ -589,8 +606,8 @@
class Lowbd2dVarTest : public ::testing::TestWithParam<TestFuncVar2D> {
public:
- virtual ~Lowbd2dVarTest() {}
- virtual void SetUp() {
+ ~Lowbd2dVarTest() override = default;
+ void SetUp() override {
params_ = this->GetParam();
rnd_.Reset(ACMRandom::DeterministicSeed());
src_ = reinterpret_cast<uint8_t *>(
@@ -598,8 +615,8 @@
ASSERT_NE(src_, nullptr);
}
- virtual void TearDown() { aom_free(src_); }
- void RunTest(int isRandom);
+ void TearDown() override { aom_free(src_); }
+ void RunTest(bool is_random);
void RunSpeedTest();
void GenRandomData(int width, int height, int stride) {
@@ -630,7 +647,7 @@
};
GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(Lowbd2dVarTest);
-void Lowbd2dVarTest::RunTest(int isRandom) {
+void Lowbd2dVarTest::RunTest(bool is_random) {
int failed = 0;
for (int k = 0; k < kNumIterations; k++) {
const int width = 4 * (rnd_(63) + 1); // Up to 256x256
@@ -639,7 +656,7 @@
while (stride < width) { // Make sure it's valid
stride = 4 << rnd_(8);
}
- if (isRandom) {
+ if (is_random) {
GenRandomData(width, height, stride);
} else {
GenExtremeData(width, height, stride);
@@ -690,11 +707,11 @@
}
TEST_P(Lowbd2dVarTest, OperationCheck) {
- RunTest(1); // GenRandomData
+ RunTest(true); // GenRandomData
}
TEST_P(Lowbd2dVarTest, ExtremeValues) {
- RunTest(0); // GenExtremeData
+ RunTest(false); // GenExtremeData
}
TEST_P(Lowbd2dVarTest, DISABLED_Speed) { RunSpeedTest(); }
@@ -723,10 +740,18 @@
#endif // HAVE_NEON
+#if HAVE_NEON_DOTPROD
+
+INSTANTIATE_TEST_SUITE_P(NEON_DOTPROD, Lowbd2dVarTest,
+ ::testing::Values(TestFuncVar2D(
+ &aom_var_2d_u8_c, &aom_var_2d_u8_neon_dotprod)));
+
+#endif // HAVE_NEON_DOTPROD
+
class Highbd2dVarTest : public ::testing::TestWithParam<TestFuncVar2D> {
public:
- virtual ~Highbd2dVarTest() {}
- virtual void SetUp() {
+ ~Highbd2dVarTest() override = default;
+ void SetUp() override {
params_ = this->GetParam();
rnd_.Reset(ACMRandom::DeterministicSeed());
src_ = reinterpret_cast<uint16_t *>(
@@ -734,8 +759,8 @@
ASSERT_NE(src_, nullptr);
}
- virtual void TearDown() { aom_free(src_); }
- void RunTest(int isRandom);
+ void TearDown() override { aom_free(src_); }
+ void RunTest(bool is_random);
void RunSpeedTest();
void GenRandomData(int width, int height, int stride) {
@@ -766,7 +791,7 @@
};
GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(Highbd2dVarTest);
-void Highbd2dVarTest::RunTest(int isRandom) {
+void Highbd2dVarTest::RunTest(bool is_random) {
int failed = 0;
for (int k = 0; k < kNumIterations; k++) {
const int width = 4 * (rnd_(63) + 1); // Up to 256x256
@@ -775,7 +800,7 @@
while (stride < width) { // Make sure it's valid
stride = 4 << rnd_(8);
}
- if (isRandom) {
+ if (is_random) {
GenRandomData(width, height, stride);
} else {
GenExtremeData(width, height, stride);
@@ -828,11 +853,11 @@
}
TEST_P(Highbd2dVarTest, OperationCheck) {
- RunTest(1); // GenRandomData
+ RunTest(true); // GenRandomData
}
TEST_P(Highbd2dVarTest, ExtremeValues) {
- RunTest(0); // GenExtremeData
+ RunTest(false); // GenExtremeData
}
TEST_P(Highbd2dVarTest, DISABLED_Speed) { RunSpeedTest(); }
diff --git a/test/svc_datarate_test.cc b/test/svc_datarate_test.cc
index d99d6a3..cc3fb67 100644
--- a/test/svc_datarate_test.cc
+++ b/test/svc_datarate_test.cc
@@ -9,6 +9,7 @@
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
+#include <climits>
#include <vector>
#include "config/aom_config.h"
#include "third_party/googletest/src/googletest/include/gtest/gtest.h"
@@ -45,22 +46,22 @@
}
protected:
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(GET_PARAM(1));
ResetModel();
}
- virtual void DecompressedFrameHook(const aom_image_t &img,
- aom_codec_pts_t pts) {
+ void DecompressedFrameHook(const aom_image_t &img,
+ aom_codec_pts_t pts) override {
frame_info_list_.push_back(FrameInfo(pts, img.d_w, img.d_h));
++decoded_nframes_;
}
std::vector<FrameInfo> frame_info_list_;
- virtual int GetNumSpatialLayers() { return number_spatial_layers_; }
+ int GetNumSpatialLayers() override { return number_spatial_layers_; }
- virtual void ResetModel() {
+ void ResetModel() override {
DatarateTest::ResetModel();
layer_frame_cnt_ = 0;
superframe_cnt_ = 0;
@@ -94,10 +95,11 @@
rps_recovery_frame_ = 0;
user_define_frame_qp_ = 0;
set_speed_per_layer_ = false;
+ simulcast_mode_ = false;
}
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
int spatial_layer_id = 0;
current_video_frame_ = video->frame();
// video->frame() is called every superframe, so we should condition
@@ -144,7 +146,7 @@
video->frame(), &layer_id_, &ref_frame_config_, &ref_frame_comp_pred_,
spatial_layer_id, multi_ref_, comp_pred_,
(video->frame() % cfg_.kf_max_dist) == 0, dynamic_enable_disable_mode_,
- rps_mode_, rps_recovery_frame_);
+ rps_mode_, rps_recovery_frame_, simulcast_mode_);
if (intra_only_ == 1 && frame_sync_ > 0) {
// Set an Intra-only frame on SL0 at frame_sync_.
// In order to allow decoding to start on SL0 in mid-sequence we need to
@@ -227,7 +229,7 @@
}
}
- virtual void PostEncodeFrameHook(::libaom_test::Encoder *encoder) {
+ void PostEncodeFrameHook(::libaom_test::Encoder *encoder) override {
int num_operating_points;
encoder->Control(AV1E_GET_NUM_OPERATING_POINTS, &num_operating_points);
ASSERT_EQ(num_operating_points,
@@ -242,7 +244,7 @@
}
}
- virtual void FramePktHook(const aom_codec_cx_pkt_t *pkt) {
+ void FramePktHook(const aom_codec_cx_pkt_t *pkt) override {
const size_t frame_size_in_bits = pkt->data.frame.sz * 8;
// Update the layer cumulative bitrate.
for (int i = layer_id_.temporal_layer_id; i < number_temporal_layers_;
@@ -254,36 +256,50 @@
last_pts_ = pkt->data.frame.pts;
superframe_cnt_++;
}
+ // For simulcast mode: verify that for first frame to start decoding,
+ // for SL > 0, are Intra-only frames (not Key), whereas SL0 is Key.
+ if (simulcast_mode_ && superframe_cnt_ == (int)frame_to_start_decoding_) {
+ if (layer_id_.spatial_layer_id > 0) {
+ EXPECT_NE(pkt->data.frame.flags & AOM_FRAME_IS_KEY, AOM_FRAME_IS_KEY);
+ } else if (layer_id_.spatial_layer_id == 0) {
+ EXPECT_EQ(pkt->data.frame.flags & AOM_FRAME_IS_KEY, AOM_FRAME_IS_KEY);
+ }
+ }
}
- virtual void EndPassHook() {
+ void EndPassHook() override {
duration_ = ((last_pts_ + 1) * timebase_);
for (int i = 0; i < number_temporal_layers_ * number_spatial_layers_; i++) {
effective_datarate_tl[i] = (effective_datarate_tl[i] / 1000) / duration_;
}
}
- virtual bool DoDecode() const {
+ bool DoDecode() const override {
if (drop_frames_ > 0) {
for (unsigned int i = 0; i < drop_frames_; ++i) {
if (drop_frames_list_[i] == (unsigned int)superframe_cnt_) {
std::cout << " Skipping decoding frame: "
<< drop_frames_list_[i] << "\n";
- return 0;
+ return false;
}
}
} else if (intra_only_ == 1) {
// Only start decoding at frames_to_start_decoding_.
- if (current_video_frame_ < frame_to_start_decoding_) return 0;
+ if (current_video_frame_ < frame_to_start_decoding_) return false;
// Only decode base layer for 3SL, for layer_to_decode_ = 0.
if (layer_to_decode_ == 0 && frame_sync_ > 0 &&
(layer_frame_cnt_ - 1) % 3 != 0)
- return 0;
+ return false;
+ } else if (simulcast_mode_) {
+ // Only start decoding at frames_to_start_decoding_ and only
+ // for top spatial layer SL2 (layer_to_decode_).
+ if (current_video_frame_ < frame_to_start_decoding_) return false;
+ if (layer_id_.spatial_layer_id < (int)layer_to_decode_) return false;
}
- return 1;
+ return true;
}
- virtual void MismatchHook(const aom_image_t *img1, const aom_image_t *img2) {
+ void MismatchHook(const aom_image_t *img1, const aom_image_t *img2) override {
double mismatch_psnr = compute_psnr(img1, img2);
mismatch_psnr_ += mismatch_psnr;
++mismatch_nframes_;
@@ -345,13 +361,301 @@
}
}
+ // Simulcast mode for 3 spatial and 3 temporal layers.
+ // No inter-layer predicton, only prediction is temporal and single
+ // reference (LAST).
+ // No overlap in buffer slots between spatial layers. So for example,
+ // SL0 only uses slots 0 and 1.
+ // SL1 only uses slots 2 and 3.
+ // SL2 only uses slots 4 and 5.
+ // All 7 references for each inter-frame must only access buffer slots
+ // for that spatial layer.
+ // On key (super)frames: SL1 and SL2 must have no references set
+ // and must refresh all the slots for that layer only (so 2 and 3
+ // for SL1, 4 and 5 for SL2). The base SL0 will be labelled internally
+ // as a Key frame (refresh all slots). SL1/SL2 will be labelled
+ // internally as Intra-only frames that allow that stream to be decoded.
+ // These conditions will allow for each spatial stream to be
+ // independently decodeable.
+ static void ref_config_simulcast3SL3TL(
+ aom_svc_ref_frame_config_t *ref_frame_config,
+ aom_svc_layer_id_t *layer_id, int is_key_frame, int superframe_cnt) {
+ int i;
+ // Initialize all references to 0 (don't use reference).
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->reference[i] = 0;
+ // Initialize as no refresh/update for all slots.
+ for (i = 0; i < REF_FRAMES; i++) ref_frame_config->refresh[i] = 0;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++) ref_frame_config->ref_idx[i] = 0;
+
+ if (is_key_frame) {
+ if (layer_id->spatial_layer_id == 0) {
+ // Assign LAST/GOLDEN to slot 0/1.
+ // Refesh slots 0 and 1 for SL0.
+ // SL0: this will get set to KEY frame internally.
+ ref_frame_config->ref_idx[0] = 0;
+ ref_frame_config->ref_idx[3] = 1;
+ ref_frame_config->refresh[0] = 1;
+ ref_frame_config->refresh[1] = 1;
+ } else if (layer_id->spatial_layer_id == 1) {
+ // Assign LAST/GOLDEN to slot 2/3.
+ // Refesh slots 2 and 3 for SL1.
+ // This will get set to Intra-only frame internally.
+ ref_frame_config->ref_idx[0] = 2;
+ ref_frame_config->ref_idx[3] = 3;
+ ref_frame_config->refresh[2] = 1;
+ ref_frame_config->refresh[3] = 1;
+ } else if (layer_id->spatial_layer_id == 2) {
+ // Assign LAST/GOLDEN to slot 4/5.
+ // Refresh slots 4 and 5 for SL2.
+ // This will get set to Intra-only frame internally.
+ ref_frame_config->ref_idx[0] = 4;
+ ref_frame_config->ref_idx[3] = 5;
+ ref_frame_config->refresh[4] = 1;
+ ref_frame_config->refresh[5] = 1;
+ }
+ } else if (superframe_cnt % 4 == 0) {
+ // Base temporal layer: TL0
+ layer_id->temporal_layer_id = 0;
+ if (layer_id->spatial_layer_id == 0) { // SL0
+ // Reference LAST. Assign all references to either slot
+ // 0 or 1. Here we assign LAST to slot 0, all others to 1.
+ // Update slot 0 (LAST).
+ ref_frame_config->reference[0] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 1;
+ ref_frame_config->ref_idx[0] = 0;
+ ref_frame_config->refresh[0] = 1;
+ } else if (layer_id->spatial_layer_id == 1) { // SL1
+ // Reference LAST. Assign all references to either slot
+ // 2 or 3. Here we assign LAST to slot 2, all others to 3.
+ // Update slot 2 (LAST).
+ ref_frame_config->reference[0] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 3;
+ ref_frame_config->ref_idx[0] = 2;
+ ref_frame_config->refresh[2] = 1;
+ } else if (layer_id->spatial_layer_id == 2) { // SL2
+ // Reference LAST. Assign all references to either slot
+ // 4 or 5. Here we assign LAST to slot 4, all others to 5.
+ // Update slot 4 (LAST).
+ ref_frame_config->reference[0] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 5;
+ ref_frame_config->ref_idx[0] = 4;
+ ref_frame_config->refresh[4] = 1;
+ }
+ } else if ((superframe_cnt - 1) % 4 == 0) {
+ // First top temporal enhancement layer: TL2
+ layer_id->temporal_layer_id = 2;
+ if (layer_id->spatial_layer_id == 0) { // SL0
+ // Reference LAST (slot 0). Assign other references to slot 1.
+ // No update/refresh on any slots.
+ ref_frame_config->reference[0] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 1;
+ ref_frame_config->ref_idx[0] = 0;
+ } else if (layer_id->spatial_layer_id == 1) { // SL1
+ // Reference LAST (slot 2). Assign other references to slot 3.
+ // No update/refresh on any slots.
+ ref_frame_config->reference[0] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 3;
+ ref_frame_config->ref_idx[0] = 2;
+ } else if (layer_id->spatial_layer_id == 2) { // SL2
+ // Reference LAST (slot 4). Assign other references to slot 4.
+ // No update/refresh on any slots.
+ ref_frame_config->reference[0] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 5;
+ ref_frame_config->ref_idx[0] = 4;
+ }
+ } else if ((superframe_cnt - 2) % 4 == 0) {
+ // Middle temporal enhancement layer: TL1
+ layer_id->temporal_layer_id = 1;
+ if (layer_id->spatial_layer_id == 0) { // SL0
+ // Reference LAST (slot 0).
+ // Set GOLDEN to slot 1 and update slot 1.
+ // This will be used as reference for next TL2.
+ ref_frame_config->reference[0] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 1;
+ ref_frame_config->ref_idx[0] = 0;
+ ref_frame_config->refresh[1] = 1;
+ } else if (layer_id->spatial_layer_id == 1) { // SL1
+ // Reference LAST (slot 2).
+ // Set GOLDEN to slot 3 and update slot 3.
+ // This will be used as reference for next TL2.
+ ref_frame_config->reference[0] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 3;
+ ref_frame_config->ref_idx[0] = 2;
+ ref_frame_config->refresh[3] = 1;
+ } else if (layer_id->spatial_layer_id == 2) { // SL2
+ // Reference LAST (slot 4).
+ // Set GOLDEN to slot 5 and update slot 5.
+ // This will be used as reference for next TL2.
+ ref_frame_config->reference[0] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 5;
+ ref_frame_config->ref_idx[0] = 4;
+ ref_frame_config->refresh[5] = 1;
+ }
+ } else if ((superframe_cnt - 3) % 4 == 0) {
+ // Second top temporal enhancement layer: TL2
+ layer_id->temporal_layer_id = 2;
+ if (layer_id->spatial_layer_id == 0) { // SL0
+ // Reference LAST (slot 1). Assign other references to slot 0.
+ // No update/refresh on any slots.
+ ref_frame_config->reference[0] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 0;
+ ref_frame_config->ref_idx[0] = 1;
+ } else if (layer_id->spatial_layer_id == 1) { // SL1
+ // Reference LAST (slot 3). Assign other references to slot 2.
+ // No update/refresh on any slots.
+ ref_frame_config->reference[0] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 2;
+ ref_frame_config->ref_idx[0] = 3;
+ } else if (layer_id->spatial_layer_id == 2) { // SL2
+ // Reference LAST (slot 5). Assign other references to slot 4.
+ // No update/refresh on any slots.
+ ref_frame_config->reference[0] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 4;
+ ref_frame_config->ref_idx[0] = 5;
+ }
+ }
+ }
+
+ // 3 spatial and 3 temporal layer.
+ // Overlap in the buffer slot updates: the slots 3 and 4 updated by
+ // first TL2 are reused for update in TL1 superframe.
+ static void ref_config_3SL3TL(aom_svc_ref_frame_config_t *ref_frame_config,
+ aom_svc_layer_id_t *layer_id, int is_key_frame,
+ int superframe_cnt) {
+ if (superframe_cnt % 4 == 0) {
+ // Base temporal layer.
+ layer_id->temporal_layer_id = 0;
+ if (layer_id->spatial_layer_id == 0) {
+ // Reference LAST, update LAST.
+ // Set all buffer_idx to 0.
+ for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 0;
+ ref_frame_config->refresh[0] = 1;
+ } else if (layer_id->spatial_layer_id == 1) {
+ // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 1,
+ // GOLDEN (and all other refs) to slot 0.
+ // Update slot 1 (LAST).
+ for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 0;
+ ref_frame_config->ref_idx[0] = 1;
+ ref_frame_config->refresh[1] = 1;
+ } else if (layer_id->spatial_layer_id == 2) {
+ // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 2,
+ // GOLDEN (and all other refs) to slot 1.
+ // Update slot 2 (LAST).
+ for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 1;
+ ref_frame_config->ref_idx[0] = 2;
+ ref_frame_config->refresh[2] = 1;
+ }
+ } else if ((superframe_cnt - 1) % 4 == 0) {
+ // First top temporal enhancement layer.
+ layer_id->temporal_layer_id = 2;
+ if (layer_id->spatial_layer_id == 0) {
+ // Reference LAST (slot 0).
+ // Set GOLDEN to slot 3 and update slot 3.
+ // Set all other buffer_idx to slot 0.
+ for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 0;
+ ref_frame_config->ref_idx[3] = 3;
+ ref_frame_config->refresh[3] = 1;
+ } else if (layer_id->spatial_layer_id == 1) {
+ // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 1,
+ // GOLDEN (and all other refs) to slot 3.
+ // Set LAST2 to slot 4 and Update slot 4.
+ for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 3;
+ ref_frame_config->ref_idx[0] = 1;
+ ref_frame_config->ref_idx[1] = 4;
+ ref_frame_config->refresh[4] = 1;
+ } else if (layer_id->spatial_layer_id == 2) {
+ // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 2,
+ // GOLDEN (and all other refs) to slot 4.
+ // No update.
+ for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 4;
+ ref_frame_config->ref_idx[0] = 2;
+ }
+ } else if ((superframe_cnt - 2) % 4 == 0) {
+ // Middle temporal enhancement layer.
+ layer_id->temporal_layer_id = 1;
+ if (layer_id->spatial_layer_id == 0) {
+ // Reference LAST.
+ // Set all buffer_idx to 0.
+ // Set GOLDEN to slot 3 and update slot 3.
+ for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 0;
+ ref_frame_config->ref_idx[3] = 3;
+ ref_frame_config->refresh[3] = 1;
+ } else if (layer_id->spatial_layer_id == 1) {
+ // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 1,
+ // GOLDEN (and all other refs) to slot 3.
+ // Set LAST2 to slot 4 and update slot 4.
+ for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 3;
+ ref_frame_config->ref_idx[0] = 1;
+ ref_frame_config->ref_idx[2] = 4;
+ ref_frame_config->refresh[4] = 1;
+ } else if (layer_id->spatial_layer_id == 2) {
+ // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 2,
+ // GOLDEN (and all other refs) to slot 4.
+ // Set LAST2 to slot 5 and update slot 5.
+ for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 4;
+ ref_frame_config->ref_idx[0] = 2;
+ ref_frame_config->ref_idx[2] = 5;
+ ref_frame_config->refresh[5] = 1;
+ }
+ } else if ((superframe_cnt - 3) % 4 == 0) {
+ // Second top temporal enhancement layer.
+ layer_id->temporal_layer_id = 2;
+ if (layer_id->spatial_layer_id == 0) {
+ // Set LAST to slot 3 and reference LAST.
+ // Set GOLDEN to slot 3 and update slot 3.
+ // Set all other buffer_idx to 0.
+ for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 0;
+ ref_frame_config->ref_idx[0] = 3;
+ ref_frame_config->ref_idx[3] = 3;
+ ref_frame_config->refresh[3] = 1;
+ } else if (layer_id->spatial_layer_id == 1) {
+ // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 4,
+ // GOLDEN to slot 3. Set LAST2 to slot 4 and update slot 4.
+ for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 0;
+ ref_frame_config->ref_idx[0] = 4;
+ ref_frame_config->ref_idx[3] = 3;
+ ref_frame_config->ref_idx[1] = 4;
+ ref_frame_config->refresh[4] = 1;
+ } else if (layer_id->spatial_layer_id == 2) {
+ // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 5,
+ // GOLDEN to slot 4. No update.
+ for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 0;
+ ref_frame_config->ref_idx[0] = 5;
+ ref_frame_config->ref_idx[3] = 4;
+ }
+ }
+ if (layer_id->spatial_layer_id > 0) {
+ // Always reference GOLDEN (inter-layer prediction).
+ ref_frame_config->reference[3] = 1;
+ if (is_key_frame && layer_id->spatial_layer_id > 0) {
+ // On superframes whose base is key: remove LAST since GOLDEN
+ // is used as reference.
+ ref_frame_config->reference[0] = 0;
+ }
+ }
+ }
+
// Layer pattern configuration.
virtual int set_layer_pattern(
int frame_cnt, aom_svc_layer_id_t *layer_id,
aom_svc_ref_frame_config_t *ref_frame_config,
aom_svc_ref_frame_comp_pred_t *ref_frame_comp_pred, int spatial_layer,
int multi_ref, int comp_pred, int is_key_frame,
- int dynamic_enable_disable_mode, int rps_mode, int rps_recovery_frame) {
+ int dynamic_enable_disable_mode, int rps_mode, int rps_recovery_frame,
+ int simulcast_mode) {
int lag_index = 0;
int base_count = frame_cnt >> 2;
layer_id->spatial_layer_id = spatial_layer;
@@ -506,128 +810,21 @@
// Reference GOLDEN.
if (layer_id->spatial_layer_id > 0) ref_frame_config->reference[3] = 1;
} else if (number_temporal_layers_ == 3 && number_spatial_layers_ == 3) {
- // 3 spatial and 3 temporal layer.
- // Overlap in the buffer slot updates: the slots 3 and 4 updated by
- // first TL2 are reused for update in TL1 superframe.
- if (superframe_cnt_ % 4 == 0) {
- // Base temporal layer.
- layer_id->temporal_layer_id = 0;
- if (layer_id->spatial_layer_id == 0) {
- // Reference LAST, update LAST.
- // Set all buffer_idx to 0.
- for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 0;
- ref_frame_config->refresh[0] = 1;
- } else if (layer_id->spatial_layer_id == 1) {
- // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 1,
- // GOLDEN (and all other refs) to slot 0.
- // Update slot 1 (LAST).
- for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 0;
- ref_frame_config->ref_idx[0] = 1;
- ref_frame_config->refresh[1] = 1;
- } else if (layer_id->spatial_layer_id == 2) {
- // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 2,
- // GOLDEN (and all other refs) to slot 1.
- // Update slot 2 (LAST).
- for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 1;
- ref_frame_config->ref_idx[0] = 2;
- ref_frame_config->refresh[2] = 1;
+ if (simulcast_mode) {
+ ref_config_simulcast3SL3TL(ref_frame_config, layer_id, is_key_frame,
+ superframe_cnt_);
+ } else {
+ ref_config_3SL3TL(ref_frame_config, layer_id, is_key_frame,
+ superframe_cnt_);
+ // Allow for top spatial layer to use additional temporal reference.
+ // Additional reference is only updated on base temporal layer, every
+ // 10 TL0 frames here.
+ if (multi_ref && layer_id->spatial_layer_id == 2) {
+ ref_frame_config->ref_idx[6] = 7;
+ if (!is_key_frame) ref_frame_config->reference[6] = 1;
+ if (base_count % 10 == 0 && layer_id->temporal_layer_id == 0)
+ ref_frame_config->refresh[7] = 1;
}
- } else if ((superframe_cnt_ - 1) % 4 == 0) {
- // First top temporal enhancement layer.
- layer_id->temporal_layer_id = 2;
- if (layer_id->spatial_layer_id == 0) {
- // Reference LAST (slot 0).
- // Set GOLDEN to slot 3 and update slot 3.
- // Set all other buffer_idx to slot 0.
- for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 0;
- ref_frame_config->ref_idx[3] = 3;
- ref_frame_config->refresh[3] = 1;
- } else if (layer_id->spatial_layer_id == 1) {
- // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 1,
- // GOLDEN (and all other refs) to slot 3.
- // Set LAST2 to slot 4 and Update slot 4.
- for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 3;
- ref_frame_config->ref_idx[0] = 1;
- ref_frame_config->ref_idx[1] = 4;
- ref_frame_config->refresh[4] = 1;
- } else if (layer_id->spatial_layer_id == 2) {
- // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 2,
- // GOLDEN (and all other refs) to slot 4.
- // No update.
- for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 4;
- ref_frame_config->ref_idx[0] = 2;
- }
- } else if ((superframe_cnt_ - 2) % 4 == 0) {
- // Middle temporal enhancement layer.
- layer_id->temporal_layer_id = 1;
- if (layer_id->spatial_layer_id == 0) {
- // Reference LAST.
- // Set all buffer_idx to 0.
- // Set GOLDEN to slot 3 and update slot 3.
- for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 0;
- ref_frame_config->ref_idx[3] = 3;
- ref_frame_config->refresh[3] = 1;
- } else if (layer_id->spatial_layer_id == 1) {
- // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 1,
- // GOLDEN (and all other refs) to slot 3.
- // Set LAST2 to slot 4 and update slot 4.
- for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 3;
- ref_frame_config->ref_idx[0] = 1;
- ref_frame_config->ref_idx[2] = 4;
- ref_frame_config->refresh[4] = 1;
- } else if (layer_id->spatial_layer_id == 2) {
- // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 2,
- // GOLDEN (and all other refs) to slot 4.
- // Set LAST2 to slot 5 and update slot 5.
- for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 4;
- ref_frame_config->ref_idx[0] = 2;
- ref_frame_config->ref_idx[2] = 5;
- ref_frame_config->refresh[5] = 1;
- }
- } else if ((superframe_cnt_ - 3) % 4 == 0) {
- // Second top temporal enhancement layer.
- layer_id->temporal_layer_id = 2;
- if (layer_id->spatial_layer_id == 0) {
- // Set LAST to slot 3 and reference LAST.
- // Set GOLDEN to slot 3 and update slot 3.
- // Set all other buffer_idx to 0.
- for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 0;
- ref_frame_config->ref_idx[0] = 3;
- ref_frame_config->ref_idx[3] = 3;
- ref_frame_config->refresh[3] = 1;
- } else if (layer_id->spatial_layer_id == 1) {
- // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 4,
- // GOLDEN to slot 3. Set LAST2 to slot 4 and update slot 4.
- for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 0;
- ref_frame_config->ref_idx[0] = 4;
- ref_frame_config->ref_idx[3] = 3;
- ref_frame_config->ref_idx[1] = 4;
- ref_frame_config->refresh[4] = 1;
- } else if (layer_id->spatial_layer_id == 2) {
- // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 5,
- // GOLDEN to slot 4. No update.
- for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 0;
- ref_frame_config->ref_idx[0] = 5;
- ref_frame_config->ref_idx[3] = 4;
- }
- }
- if (layer_id->spatial_layer_id > 0) {
- // Always reference GOLDEN (inter-layer prediction).
- ref_frame_config->reference[3] = 1;
- if (is_key_frame && layer_id->spatial_layer_id > 0) {
- // On superframes whose base is key: remove LAST since GOLDEN
- // is used as reference.
- ref_frame_config->reference[0] = 0;
- }
- }
- // Allow for top spatial layer to use additional temporal reference.
- // Additional reference is only updated on base temporal layer, every
- // 10 TL0 frames here.
- if (multi_ref && layer_id->spatial_layer_id == 2) {
- ref_frame_config->ref_idx[6] = 7;
- if (!is_key_frame) ref_frame_config->reference[6] = 1;
- if (base_count % 10 == 0 && layer_id->temporal_layer_id == 0)
- ref_frame_config->refresh[7] = 1;
}
}
// If the top spatial layer is first-time encoded in mid-sequence
@@ -1090,6 +1287,60 @@
EXPECT_EQ((int)GetMismatchFrames(), 150);
}
+ virtual void BasicRateTargetingSVC3TL3SLSimulcast() {
+ cfg_.rc_buf_initial_sz = 500;
+ cfg_.rc_buf_optimal_sz = 500;
+ cfg_.rc_buf_sz = 1000;
+ cfg_.rc_dropframe_thresh = 0;
+ cfg_.rc_min_quantizer = 0;
+ cfg_.rc_max_quantizer = 56;
+ cfg_.rc_end_usage = AOM_CBR;
+ cfg_.g_lag_in_frames = 0;
+ cfg_.g_error_resilient = 0;
+ cfg_.kf_max_dist = 150;
+ cfg_.kf_min_dist = 150;
+ int num_frames = 300;
+ ::libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352,
+ 288, 30, 1, 0, num_frames);
+ const int bitrate_array[2] = { 500, 1000 };
+ cfg_.rc_target_bitrate = bitrate_array[GET_PARAM(4)];
+ ResetModel();
+ simulcast_mode_ = 1;
+ frame_to_start_decoding_ = cfg_.kf_max_dist;
+ layer_to_decode_ = 2; // SL2
+ number_temporal_layers_ = 3;
+ number_spatial_layers_ = 3;
+ // SL0
+ const int bitrate_sl0 = 1 * cfg_.rc_target_bitrate / 8;
+ target_layer_bitrate_[0] = 50 * bitrate_sl0 / 100;
+ target_layer_bitrate_[1] = 70 * bitrate_sl0 / 100;
+ target_layer_bitrate_[2] = bitrate_sl0;
+ // SL1
+ const int bitrate_sl1 = 3 * cfg_.rc_target_bitrate / 8;
+ target_layer_bitrate_[3] = 50 * bitrate_sl1 / 100;
+ target_layer_bitrate_[4] = 70 * bitrate_sl1 / 100;
+ target_layer_bitrate_[5] = bitrate_sl1;
+ // SL2
+ const int bitrate_sl2 = 4 * cfg_.rc_target_bitrate / 8;
+ target_layer_bitrate_[6] = 50 * bitrate_sl2 / 100;
+ target_layer_bitrate_[7] = 70 * bitrate_sl2 / 100;
+ target_layer_bitrate_[8] = bitrate_sl2;
+ ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
+ // Only SL2 layer is decoded.
+ for (int tl = 0; tl < number_temporal_layers_; tl++) {
+ int i = layer_to_decode_ * number_temporal_layers_ + tl;
+ ASSERT_GE(effective_datarate_tl[i], target_layer_bitrate_[i] * 0.6)
+ << " The datarate for the file is lower than target by too much!";
+ ASSERT_LE(effective_datarate_tl[i], target_layer_bitrate_[i] * 1.7)
+ << " The datarate for the file is greater than target by too much!";
+ }
+ // Only top spatial layer (SL2) is decoded, starting at frame 150
+ // (frame_to_start_decoding_), so there (300 - 150) / 2 = 75
+ // non-reference frames, so mismatch is 75.
+ int num_mismatch = (num_frames - frame_to_start_decoding_) / 2;
+ EXPECT_EQ((int)GetMismatchFrames(), num_mismatch);
+ }
+
virtual void BasicRateTargetingSVC1TL2SLIntraOnlyTest() {
cfg_.rc_buf_initial_sz = 500;
cfg_.rc_buf_optimal_sz = 500;
@@ -2133,6 +2384,7 @@
int screen_mode_;
int rps_mode_;
int rps_recovery_frame_;
+ int simulcast_mode_;
int user_define_frame_qp_;
int frame_qp_;
@@ -2199,6 +2451,14 @@
BasicRateTargetingSVC3TL3SLIntraMidSeqDecodeAll();
}
+// Check simulcast mode for 3 spatial layers, 3 temporal,
+// Key frame is inserted on base SLO in mid-stream, and verify that the
+// top spatial layer (SL2) case be decoded, starting with an Intra-only frame.
+// Verify that we can decode all frames for SL2 with no mismatch.
+TEST_P(DatarateTestSVC, BasicRateTargetingSVC3TL3SLSimulcast) {
+ BasicRateTargetingSVC3TL3SLSimulcast();
+}
+
// Check basic rate targeting for CBR, for 2 spatial layers, 1 temporal,
// with Intra-only frame inserted in the stream.
TEST_P(DatarateTestSVC, BasicRateTargetingSVC1TL2SLIntraOnly) {
@@ -2373,6 +2633,39 @@
BasicRateTargetingRPS1TL1SLDropFramesTest();
}
+TEST(SvcParams, BitrateOverflow) {
+ uint8_t buf[6] = { 0 };
+ aom_image_t img;
+ aom_codec_ctx_t enc;
+ aom_codec_enc_cfg_t cfg;
+
+ EXPECT_EQ(&img, aom_img_wrap(&img, AOM_IMG_FMT_I420, 1, 1, 1, buf));
+
+ aom_codec_iface_t *const iface = aom_codec_av1_cx();
+ EXPECT_EQ(aom_codec_enc_config_default(iface, &cfg, AOM_USAGE_REALTIME),
+ AOM_CODEC_OK);
+ cfg.g_w = 1;
+ cfg.g_h = 1;
+ ASSERT_EQ(aom_codec_enc_init(&enc, iface, &cfg, 0), AOM_CODEC_OK);
+
+ aom_svc_params_t svc_params = {};
+ svc_params.framerate_factor[0] = 1;
+ svc_params.framerate_factor[1] = 2;
+ svc_params.number_spatial_layers = 1;
+ svc_params.number_temporal_layers = 2;
+ svc_params.layer_target_bitrate[0] = INT_MAX;
+ svc_params.layer_target_bitrate[1] = INT_MAX;
+ EXPECT_EQ(aom_codec_control(&enc, AV1E_SET_SVC_PARAMS, &svc_params),
+ AOM_CODEC_OK);
+ EXPECT_EQ(
+ aom_codec_encode(&enc, &img, /*pts=*/0, /*duration=*/1, /*flags=*/0),
+ AOM_CODEC_OK);
+ EXPECT_EQ(aom_codec_encode(&enc, /*img=*/nullptr, /*pts=*/0, /*duration=*/0,
+ /*flags=*/0),
+ AOM_CODEC_OK);
+ EXPECT_EQ(aom_codec_destroy(&enc), AOM_CODEC_OK);
+}
+
AV1_INSTANTIATE_TEST_SUITE(DatarateTestSVC,
::testing::Values(::libaom_test::kRealTime),
::testing::Range(7, 12), ::testing::Values(0, 3),
diff --git a/test/temporal_filter_test.cc b/test/temporal_filter_test.cc
index e689cd3..85f68b8 100644
--- a/test/temporal_filter_test.cc
+++ b/test/temporal_filter_test.cc
@@ -57,8 +57,8 @@
class TemporalFilterTest
: public ::testing::TestWithParam<TemporalFilterWithParam> {
public:
- virtual ~TemporalFilterTest() {}
- virtual void SetUp() {
+ ~TemporalFilterTest() override = default;
+ void SetUp() override {
params_ = GET_PARAM(0);
tf_wgt_calc_lvl_ = GET_PARAM(1);
rnd_.Reset(ACMRandom::DeterministicSeed());
@@ -71,7 +71,7 @@
ASSERT_NE(src2_, nullptr);
}
- virtual void TearDown() {
+ void TearDown() override {
aom_free(src1_);
aom_free(src2_);
}
@@ -308,6 +308,23 @@
Values(0, 1)));
#endif // HAVE_NEON
+#if HAVE_NEON_DOTPROD
+TemporalFilterFuncParam temporal_filter_test_neon_dotprod[] = {
+ TemporalFilterFuncParam(&av1_apply_temporal_filter_c,
+ &av1_apply_temporal_filter_neon_dotprod)
+};
+INSTANTIATE_TEST_SUITE_P(NEON_DOTPROD, TemporalFilterTest,
+ Combine(ValuesIn(temporal_filter_test_neon_dotprod),
+ Values(0, 1)));
+#endif // HAVE_NEON_DOTPROD
+
+#if HAVE_AVX2 || HAVE_NEON
+// Width and height for which av1_estimate_noise_from_single_plane() will be
+// tested.
+const int kWidths[] = { 3840, 1920, 1280, 800, 640, 360, 357 };
+const int kHeights[] = { 2160, 1080, 720, 600, 480, 240, 237 };
+#endif // HAVE_AVX2 || HAVE_NEON
+
typedef double (*EstimateNoiseFunc)(const uint8_t *src, int height, int width,
int stride, int edge_thresh);
@@ -317,8 +334,8 @@
class EstimateNoiseTest
: public ::testing::TestWithParam<EstimateNoiseWithParam> {
public:
- virtual ~EstimateNoiseTest() {}
- virtual void SetUp() {
+ ~EstimateNoiseTest() override = default;
+ void SetUp() override {
ref_func = GET_PARAM(0);
tst_func = GET_PARAM(1);
width_ = GET_PARAM(2);
@@ -330,7 +347,7 @@
ASSERT_NE(src1_, nullptr);
}
- virtual void TearDown() { aom_free(src1_); }
+ void TearDown() override { aom_free(src1_); }
void RunTest(int run_times) {
stride_ = width_;
@@ -387,11 +404,6 @@
TEST_P(EstimateNoiseTest, DISABLED_Speed) { SpeedTest(2000); }
#if HAVE_AVX2
-// Width and height for which av1_estimate_noise_from_single_plane() will be
-// tested.
-const int kWidths[] = { 3840, 1920, 1280, 800, 640, 360, 357 };
-const int kHeights[] = { 2160, 1080, 720, 600, 480, 240, 237 };
-
INSTANTIATE_TEST_SUITE_P(
AVX2, EstimateNoiseTest,
::testing::Combine(
@@ -400,6 +412,15 @@
::testing::ValuesIn(kWidths), ::testing::ValuesIn(kHeights)));
#endif // HAVE_AVX2
+#if HAVE_NEON
+INSTANTIATE_TEST_SUITE_P(
+ NEON, EstimateNoiseTest,
+ ::testing::Combine(
+ ::testing::Values(av1_estimate_noise_from_single_plane_c),
+ ::testing::Values(av1_estimate_noise_from_single_plane_neon),
+ ::testing::ValuesIn(kWidths), ::testing::ValuesIn(kHeights)));
+#endif // HAVE_NEON
+
#if CONFIG_AV1_HIGHBITDEPTH
typedef void (*HBDTemporalFilterFunc)(
@@ -416,8 +437,8 @@
class HBDTemporalFilterTest
: public ::testing::TestWithParam<HBDTemporalFilterWithParam> {
public:
- virtual ~HBDTemporalFilterTest() {}
- virtual void SetUp() {
+ ~HBDTemporalFilterTest() override = default;
+ void SetUp() override {
params_ = GET_PARAM(0);
tf_wgt_calc_lvl_ = GET_PARAM(1);
rnd_.Reset(ACMRandom::DeterministicSeed());
@@ -430,7 +451,7 @@
ASSERT_NE(src2_, nullptr);
}
- virtual void TearDown() {
+ void TearDown() override {
aom_free(src1_);
aom_free(src2_);
}
@@ -664,6 +685,104 @@
Combine(ValuesIn(HBDtemporal_filter_test_avx2),
Values(0, 1)));
#endif // HAVE_AVX2
+
+#if HAVE_NEON
+HBDTemporalFilterFuncParam HBDtemporal_filter_test_neon[] = {
+ HBDTemporalFilterFuncParam(&av1_highbd_apply_temporal_filter_c,
+ &av1_highbd_apply_temporal_filter_neon)
+};
+INSTANTIATE_TEST_SUITE_P(NEON, HBDTemporalFilterTest,
+ Combine(ValuesIn(HBDtemporal_filter_test_neon),
+ Values(0, 1)));
+#endif // HAVE_NEON
+
+using HBDEstimateNoiseFunc = double (*)(const uint16_t *src, int height,
+ int width, int stride, int bit_depth,
+ int edge_thresh);
+
+using HBDEstimateNoiseWithParam =
+ std::tuple<HBDEstimateNoiseFunc, HBDEstimateNoiseFunc, int, int, int>;
+
+class HBDEstimateNoiseTest
+ : public ::testing::TestWithParam<HBDEstimateNoiseWithParam> {
+ public:
+ HBDEstimateNoiseTest()
+ : ref_func_(GET_PARAM(0)), tst_func_(GET_PARAM(1)),
+ rnd_(libaom_test::ACMRandom::DeterministicSeed()), width_(GET_PARAM(2)),
+ height_(GET_PARAM(3)), bitdepth_(GET_PARAM(4)) {}
+ ~HBDEstimateNoiseTest() override = default;
+ void SetUp() override {
+ src1_ = reinterpret_cast<uint16_t *>(
+ aom_memalign(16, sizeof(uint16_t) * width_ * height_));
+ ASSERT_NE(src1_, nullptr);
+ GenRandomData(width_ * height_);
+ }
+
+ void TearDown() override { aom_free(src1_); }
+
+ void RunTest() {
+ stride_ = width_;
+
+ double ref_out = ref_func_(src1_, height_, width_, stride_, bitdepth_,
+ NOISE_ESTIMATION_EDGE_THRESHOLD);
+
+ double tst_out = tst_func_(src1_, height_, width_, stride_, bitdepth_,
+ NOISE_ESTIMATION_EDGE_THRESHOLD);
+
+ EXPECT_EQ(ref_out, tst_out);
+ }
+
+ void SpeedTest(int run_times) {
+ stride_ = width_;
+ aom_usec_timer timer;
+ aom_usec_timer_start(&timer);
+ for (int i = 0; i < run_times; i++) {
+ ref_func_(src1_, height_, width_, stride_, bitdepth_,
+ NOISE_ESTIMATION_EDGE_THRESHOLD);
+ }
+ aom_usec_timer_mark(&timer);
+ const double time1 = static_cast<double>(aom_usec_timer_elapsed(&timer));
+ aom_usec_timer_start(&timer);
+ for (int i = 0; i < run_times; i++) {
+ tst_func_(src1_, height_, width_, stride_, bitdepth_,
+ NOISE_ESTIMATION_EDGE_THRESHOLD);
+ }
+ aom_usec_timer_mark(&timer);
+ const double time2 = static_cast<double>(aom_usec_timer_elapsed(&timer));
+
+ printf("%d %dx%d :%7.2f/%7.2f (%3.2f)\n", bitdepth_, width_, height_, time1,
+ time2, time1 / time2);
+ }
+
+ void GenRandomData(int size) {
+ for (int ii = 0; ii < size; ii++) src1_[ii] = rnd_.Rand12();
+ }
+
+ private:
+ HBDEstimateNoiseFunc ref_func_;
+ HBDEstimateNoiseFunc tst_func_;
+ ACMRandom rnd_;
+ uint16_t *src1_;
+ int width_;
+ int height_;
+ int stride_;
+ int bitdepth_;
+};
+GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(HBDEstimateNoiseTest);
+
+TEST_P(HBDEstimateNoiseTest, RandomValues) { RunTest(); }
+
+TEST_P(HBDEstimateNoiseTest, DISABLED_Speed) { SpeedTest(2000); }
+
+#if HAVE_NEON
+INSTANTIATE_TEST_SUITE_P(
+ NEON, HBDEstimateNoiseTest,
+ ::testing::Combine(
+ ::testing::Values(av1_highbd_estimate_noise_from_single_plane_c),
+ ::testing::Values(av1_highbd_estimate_noise_from_single_plane_neon),
+ ::testing::ValuesIn(kWidths), ::testing::ValuesIn(kHeights),
+ ::testing::ValuesIn({ 8, 10, 12 })));
+#endif // HAVE_NEON
#endif // CONFIG_AV1_HIGHBITDEPTH
} // namespace
#endif
diff --git a/test/test-data.sha1 b/test/test-data.sha1
index 4bd0ddc..4b4a96d 100644
--- a/test/test-data.sha1
+++ b/test/test-data.sha1
@@ -571,3 +571,5 @@
36a4fcf07e645ed522cde5845dd9c6ab2b2d1502 *av1-1-b8-16-intra_only-intrabc-extreme-dv.ivf.md5
9f935d391fdf4a6f7c320355d45770d2e7d6095c *desktopqvga2.320_240.yuv
4d1ad6d3070268ccb000d7fc3ae0f5a9447bfe82 *test_input_w1h1.yuv
+ad9942a073e245585c93f764ea299382a65939a7 *crowd_run_360p_10_150f.y4m
+9c2aa2d0f63f706f775bf661dfa81e8bb3089d8b *wikipedia_420_360p_60f.y4m
diff --git a/test/test.cmake b/test/test.cmake
index 672edb3..ce94a5a 100644
--- a/test/test.cmake
+++ b/test/test.cmake
@@ -157,21 +157,29 @@
"${AOM_ROOT}/test/simd_cmp_impl.h"
"${AOM_ROOT}/test/simd_impl.h")
- list(APPEND AOM_UNIT_TEST_COMMON_INTRIN_NEON
- "${AOM_ROOT}/test/simd_cmp_neon.cc")
- add_to_libaom_test_srcs(AOM_UNIT_TEST_COMMON_INTRIN_NEON)
+ if(HAVE_SSE2)
+ list(APPEND AOM_UNIT_TEST_COMMON_INTRIN_SSE2
+ "${AOM_ROOT}/test/simd_cmp_sse2.cc")
+ add_to_libaom_test_srcs(AOM_UNIT_TEST_COMMON_INTRIN_SSE2)
+ endif()
- list(APPEND AOM_UNIT_TEST_COMMON_INTRIN_SSE2
- "${AOM_ROOT}/test/simd_cmp_sse2.cc")
- add_to_libaom_test_srcs(AOM_UNIT_TEST_COMMON_INTRIN_SSE2)
+ if(HAVE_SSSE3)
+ list(APPEND AOM_UNIT_TEST_COMMON_INTRIN_SSSE3
+ "${AOM_ROOT}/test/simd_cmp_ssse3.cc")
+ add_to_libaom_test_srcs(AOM_UNIT_TEST_COMMON_INTRIN_SSSE3)
+ endif()
- list(APPEND AOM_UNIT_TEST_COMMON_INTRIN_SSSE3
- "${AOM_ROOT}/test/simd_cmp_ssse3.cc")
- add_to_libaom_test_srcs(AOM_UNIT_TEST_COMMON_INTRIN_SSSE3)
+ if(HAVE_SSE4_1)
+ list(APPEND AOM_UNIT_TEST_COMMON_INTRIN_SSE4_1
+ "${AOM_ROOT}/test/simd_cmp_sse4.cc")
+ add_to_libaom_test_srcs(AOM_UNIT_TEST_COMMON_INTRIN_SSE4_1)
+ endif()
- list(APPEND AOM_UNIT_TEST_COMMON_INTRIN_AVX2
- "${AOM_ROOT}/test/simd_cmp_avx2.cc")
- add_to_libaom_test_srcs(AOM_UNIT_TEST_COMMON_INTRIN_AVX2)
+ if(HAVE_AVX2)
+ list(APPEND AOM_UNIT_TEST_COMMON_INTRIN_AVX2
+ "${AOM_ROOT}/test/simd_cmp_avx2.cc")
+ add_to_libaom_test_srcs(AOM_UNIT_TEST_COMMON_INTRIN_AVX2)
+ endif()
list(APPEND AOM_UNIT_TEST_ENCODER_SOURCES
"${AOM_ROOT}/test/arf_freq_test.cc"
@@ -180,6 +188,7 @@
"${AOM_ROOT}/test/av1_fwd_txfm2d_test.cc"
"${AOM_ROOT}/test/av1_inv_txfm1d_test.cc"
"${AOM_ROOT}/test/av1_inv_txfm2d_test.cc"
+ "${AOM_ROOT}/test/av1_k_means_test.cc"
"${AOM_ROOT}/test/av1_nn_predict_test.cc"
"${AOM_ROOT}/test/av1_round_shift_array_test.cc"
"${AOM_ROOT}/test/av1_softmax_test.cc"
@@ -192,15 +201,16 @@
"${AOM_ROOT}/test/comp_avg_pred_test.cc"
"${AOM_ROOT}/test/comp_avg_pred_test.h"
"${AOM_ROOT}/test/comp_mask_pred_test.cc"
+ "${AOM_ROOT}/test/disflow_test.cc"
"${AOM_ROOT}/test/encodemb_test.cc"
"${AOM_ROOT}/test/encodetxb_test.cc"
"${AOM_ROOT}/test/end_to_end_qmpsnr_test.cc"
"${AOM_ROOT}/test/end_to_end_ssim_test.cc"
"${AOM_ROOT}/test/error_block_test.cc"
+ "${AOM_ROOT}/test/fdct4x4_test.cc"
"${AOM_ROOT}/test/fft_test.cc"
"${AOM_ROOT}/test/firstpass_test.cc"
"${AOM_ROOT}/test/fwht4x4_test.cc"
- "${AOM_ROOT}/test/fdct4x4_test.cc"
"${AOM_ROOT}/test/hadamard_test.cc"
"${AOM_ROOT}/test/horver_correlation_test.cc"
"${AOM_ROOT}/test/masked_sad_test.cc"
@@ -212,23 +222,17 @@
"${AOM_ROOT}/test/obmc_sad_test.cc"
"${AOM_ROOT}/test/obmc_variance_test.cc"
"${AOM_ROOT}/test/pickrst_test.cc"
+ "${AOM_ROOT}/test/reconinter_test.cc"
"${AOM_ROOT}/test/sad_test.cc"
"${AOM_ROOT}/test/subtract_test.cc"
- "${AOM_ROOT}/test/reconinter_test.cc"
"${AOM_ROOT}/test/sum_squares_test.cc"
"${AOM_ROOT}/test/sse_sum_test.cc"
"${AOM_ROOT}/test/variance_test.cc"
- "${AOM_ROOT}/test/wiener_test.cc"
- "${AOM_ROOT}/test/frame_error_test.cc"
"${AOM_ROOT}/test/warp_filter_test.cc"
"${AOM_ROOT}/test/warp_filter_test_util.cc"
"${AOM_ROOT}/test/warp_filter_test_util.h"
"${AOM_ROOT}/test/webmenc_test.cc"
- "${AOM_ROOT}/test/av1_k_means_test.cc")
-
- list(APPEND AOM_UNIT_TEST_ENCODER_INTRIN_SSE4_1
- "${AOM_ROOT}/test/simd_cmp_sse4.cc")
- add_to_libaom_test_srcs(AOM_UNIT_TEST_ENCODER_INTRIN_SSE4_1)
+ "${AOM_ROOT}/test/wiener_test.cc")
if(NOT CONFIG_REALTIME_ONLY)
list(APPEND AOM_UNIT_TEST_ENCODER_INTRIN_SSE4_1
@@ -293,11 +297,6 @@
endif()
endif()
- if(HAVE_NEON)
- list(APPEND AOM_UNIT_TEST_COMMON_SOURCES
- "${AOM_ROOT}/test/simd_neon_test.cc")
- endif()
-
if(CONFIG_FPMT_TEST AND (NOT CONFIG_REALTIME_ONLY))
list(APPEND AOM_UNIT_TEST_COMMON_SOURCES
"${AOM_ROOT}/test/frame_parallel_enc_test.cc")
@@ -313,7 +312,7 @@
"${AOM_ROOT}/test/simd_ssse3_test.cc")
endif()
- if(HAVE_SSE4)
+ if(HAVE_SSE4_1)
list(APPEND AOM_UNIT_TEST_COMMON_SOURCES
"${AOM_ROOT}/test/simd_sse4_test.cc")
endif()
@@ -351,13 +350,13 @@
"${AOM_ROOT}/test/av1_convolve_scale_test.cc"
"${AOM_ROOT}/test/av1_horz_only_frame_superres_test.cc"
"${AOM_ROOT}/test/intra_edge_test.cc")
-
endif()
if(HAVE_NEON)
list(APPEND AOM_UNIT_TEST_ENCODER_SOURCES
"${AOM_ROOT}/test/av1_convolve_scale_test.cc"
- "${AOM_ROOT}/test/av1_horz_only_frame_superres_test.cc")
+ "${AOM_ROOT}/test/av1_horz_only_frame_superres_test.cc"
+ "${AOM_ROOT}/test/intra_edge_test.cc")
endif()
if(HAVE_SSE4_2 OR HAVE_ARM_CRC32)
@@ -366,10 +365,10 @@
if(CONFIG_REALTIME_ONLY)
list(REMOVE_ITEM AOM_UNIT_TEST_ENCODER_SOURCES
+ "${AOM_ROOT}/test/disflow_test.cc"
"${AOM_ROOT}/test/end_to_end_qmpsnr_test.cc"
"${AOM_ROOT}/test/end_to_end_ssim_test.cc"
"${AOM_ROOT}/test/firstpass_test.cc"
- "${AOM_ROOT}/test/frame_error_test.cc"
"${AOM_ROOT}/test/motion_vector_test.cc"
"${AOM_ROOT}/test/obmc_sad_test.cc"
"${AOM_ROOT}/test/obmc_variance_test.cc"
@@ -498,9 +497,6 @@
target_link_libraries(test_libaom ${AOM_LIB_LINK_TYPE} aom aom_gtest)
- if(CONFIG_LIBYUV)
- target_sources(test_libaom PRIVATE $<TARGET_OBJECTS:yuv>)
- endif()
if(CONFIG_WEBM_IO)
target_sources(test_libaom PRIVATE $<TARGET_OBJECTS:webm>)
endif()
@@ -530,10 +526,6 @@
add_intrinsics_source_to_target("${AOM_NEON_INTRIN_FLAG}" "test_libaom"
"AOM_UNIT_TEST_COMMON_INTRIN_NEON")
endif()
- if(HAVE_ARM_CRC32)
- add_intrinsics_source_to_target("${AOM_ARM_CRC32_FLAG}" "test_libaom"
- "AOM_UNIT_TEST_COMMON_INTRIN_CRC32")
- endif()
if(ENABLE_TESTDATA)
make_test_data_lists("${AOM_UNIT_TEST_DATA_LIST_FILE}" test_files
diff --git a/test/test_data_util.cmake b/test/test_data_util.cmake
index de7d153..069e1ad 100644
--- a/test/test_data_util.cmake
+++ b/test/test_data_util.cmake
@@ -39,7 +39,9 @@
"vase10x10.yuv"
"vase10x10_tiles.txt"
"bus_352x288_420_f20_b8.yuv"
- "test_input_w1h1.yuv")
+ "test_input_w1h1.yuv"
+ "crowd_run_360p_10_150f.y4m"
+ "wikipedia_420_360p_60f.y4m")
if(ENABLE_DECODE_PERF_TESTS AND CONFIG_AV1_ENCODER)
list(APPEND AOM_TEST_DATA_FILE_NAMES "niklas_1280_720_30.yuv")
diff --git a/test/test_libaom.cc b/test/test_libaom.cc
index 6ffbbc5..fbd7f2e 100644
--- a/test/test_libaom.cc
+++ b/test/test_libaom.cc
@@ -9,24 +9,29 @@
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
-#include <string.h>
-
-#include <string>
-
#include "third_party/googletest/src/googletest/include/gtest/gtest.h"
#include "config/aom_config.h"
+#if !CONFIG_SHARED
+#include <string.h>
+
+#include <string>
+
+#if AOM_ARCH_ARM
+#include "aom_ports/arm.h"
+#endif
#if AOM_ARCH_X86 || AOM_ARCH_X86_64
#include "aom_ports/x86.h"
#endif
+
extern "C" {
extern void av1_rtcd();
extern void aom_dsp_rtcd();
extern void aom_scale_rtcd();
}
-#if AOM_ARCH_X86 || AOM_ARCH_X86_64
+#if AOM_ARCH_ARM || AOM_ARCH_X86 || AOM_ARCH_X86_64
static void append_negative_gtest_filter(const char *str) {
std::string flag_value = GTEST_FLAG_GET(filter);
// Negative patterns begin with one '-' followed by a ':' separated list.
@@ -44,11 +49,24 @@
}
GTEST_FLAG_SET(filter, flag_value);
}
-#endif // AOM_ARCH_X86 || AOM_ARCH_X86_64
+#endif // AOM_ARCH_ARM || AOM_ARCH_X86 || AOM_ARCH_X86_64
+#endif // !CONFIG_SHARED
int main(int argc, char **argv) {
::testing::InitGoogleTest(&argc, argv);
+#if !CONFIG_SHARED
+#if AOM_ARCH_AARCH64
+ const int caps = aom_arm_cpu_caps();
+ if (!(caps & HAS_ARM_CRC32)) append_negative_gtest_filter("ARM_CRC32");
+ if (!(caps & HAS_NEON_DOTPROD)) append_negative_gtest_filter("NEON_DOTPROD");
+ if (!(caps & HAS_NEON_I8MM)) append_negative_gtest_filter("NEON_I8MM");
+ if (!(caps & HAS_SVE)) append_negative_gtest_filter("SVE");
+#elif AOM_ARCH_ARM
+ const int caps = aom_arm_cpu_caps();
+ if (!(caps & HAS_NEON)) append_negative_gtest_filter("NEON");
+#endif // AOM_ARCH_ARM
+
#if AOM_ARCH_X86 || AOM_ARCH_X86_64
const int simd_caps = x86_simd_caps();
if (!(simd_caps & HAS_MMX)) append_negative_gtest_filter("MMX");
@@ -62,9 +80,8 @@
if (!(simd_caps & HAS_AVX2)) append_negative_gtest_filter("AVX2");
#endif // AOM_ARCH_X86 || AOM_ARCH_X86_64
-// Shared library builds don't support whitebox tests that exercise internal
-// symbols.
-#if !CONFIG_SHARED
+ // Shared library builds don't support whitebox tests that exercise internal
+ // symbols.
av1_rtcd();
aom_dsp_rtcd();
aom_scale_rtcd();
diff --git a/test/test_vector_test.cc b/test/test_vector_test.cc
index 87155c7..39414e3 100644
--- a/test/test_vector_test.cc
+++ b/test/test_vector_test.cc
@@ -41,7 +41,7 @@
protected:
TestVectorTest() : DecoderTest(GET_PARAM(0)), md5_file_(nullptr) {}
- virtual ~TestVectorTest() {
+ ~TestVectorTest() override {
if (md5_file_) fclose(md5_file_);
}
@@ -51,14 +51,13 @@
<< "Md5 file open failed. Filename: " << md5_file_name_;
}
- virtual void PreDecodeFrameHook(
- const libaom_test::CompressedVideoSource &video,
- libaom_test::Decoder *decoder) {
+ void PreDecodeFrameHook(const libaom_test::CompressedVideoSource &video,
+ libaom_test::Decoder *decoder) override {
if (video.frame_number() == 0) decoder->Control(AV1D_SET_ROW_MT, row_mt_);
}
- virtual void DecompressedFrameHook(const aom_image_t &img,
- const unsigned int frame_number) {
+ void DecompressedFrameHook(const aom_image_t &img,
+ const unsigned int frame_number) override {
ASSERT_NE(md5_file_, nullptr);
char expected_md5[33];
char junk[128];
diff --git a/test/tile_config_test.cc b/test/tile_config_test.cc
index 517d54b..e2ac592 100644
--- a/test/tile_config_test.cc
+++ b/test/tile_config_test.cc
@@ -82,9 +82,9 @@
max_tile_cols_log2_ = tile_log2(1, AOM_MAX_TILE_COLS);
max_tile_rows_log2_ = tile_log2(1, AOM_MAX_TILE_ROWS);
}
- virtual ~UniformTileConfigTestLarge() {}
+ ~UniformTileConfigTestLarge() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(encoding_mode_);
const aom_rational timebase = { 1, 30 };
cfg_.g_timebase = timebase;
@@ -93,10 +93,10 @@
cfg_.g_lag_in_frames = 19;
}
- virtual bool DoDecode() const { return 1; }
+ bool DoDecode() const override { return true; }
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AV1E_SET_TILE_COLUMNS, tile_config_param_.tile_cols);
encoder->Control(AV1E_SET_TILE_ROWS, tile_config_param_.tile_rows);
@@ -109,8 +109,8 @@
}
}
- virtual bool HandleDecodeResult(const aom_codec_err_t res_dec,
- libaom_test::Decoder *decoder) {
+ bool HandleDecodeResult(const aom_codec_err_t res_dec,
+ libaom_test::Decoder *decoder) override {
EXPECT_EQ(AOM_CODEC_OK, res_dec) << decoder->DecodeError();
if (AOM_CODEC_OK == res_dec) {
aom_codec_ctx_t *ctx_dec = decoder->GetDecoder();
@@ -148,9 +148,9 @@
tile_config_param_(GET_PARAM(2)), rc_end_usage_(GET_PARAM(3)) {
tile_config_violated_ = false;
}
- virtual ~NonUniformTileConfigTestLarge() {}
+ ~NonUniformTileConfigTestLarge() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(encoding_mode_);
const aom_rational timebase = { 1, 30 };
cfg_.g_timebase = timebase;
@@ -168,10 +168,10 @@
tile_config_param_.tile_height_count);
}
- virtual bool DoDecode() const { return 1; }
+ bool DoDecode() const override { return true; }
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AOME_SET_CPUUSED, 5);
encoder->Control(AOME_SET_ENABLEAUTOALTREF, 1);
@@ -182,8 +182,8 @@
}
}
- virtual bool HandleDecodeResult(const aom_codec_err_t res_dec,
- libaom_test::Decoder *decoder) {
+ bool HandleDecodeResult(const aom_codec_err_t res_dec,
+ libaom_test::Decoder *decoder) override {
EXPECT_EQ(AOM_CODEC_OK, res_dec) << decoder->DecodeError();
if (AOM_CODEC_OK == res_dec) {
aom_codec_ctx_t *ctx_dec = decoder->GetDecoder();
@@ -302,9 +302,9 @@
tile_group_config_params_(GET_PARAM(2)) {
tile_group_config_violated_ = false;
}
- virtual ~TileGroupTestLarge() {}
+ ~TileGroupTestLarge() override = default;
- virtual void SetUp() {
+ void SetUp() override {
InitializeConfig(encoding_mode_);
const aom_rational timebase = { 1, 30 };
cfg_.g_timebase = timebase;
@@ -312,10 +312,10 @@
cfg_.g_threads = 1;
}
- virtual bool DoDecode() const { return 1; }
+ bool DoDecode() const override { return true; }
- virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
- ::libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AOME_SET_CPUUSED, 5);
encoder->Control(AV1E_SET_NUM_TG, tile_group_config_params_.num_tg);
@@ -326,8 +326,8 @@
}
}
- virtual bool HandleDecodeResult(const aom_codec_err_t res_dec,
- libaom_test::Decoder *decoder) {
+ bool HandleDecodeResult(const aom_codec_err_t res_dec,
+ libaom_test::Decoder *decoder) override {
EXPECT_EQ(AOM_CODEC_OK, res_dec) << decoder->DecodeError();
if (AOM_CODEC_OK == res_dec) {
aom_tile_info tile_info;
diff --git a/test/tile_independence_test.cc b/test/tile_independence_test.cc
index 888c3ab..84406dd 100644
--- a/test/tile_independence_test.cc
+++ b/test/tile_independence_test.cc
@@ -47,15 +47,15 @@
}
}
- virtual ~TileIndependenceTest() {
+ ~TileIndependenceTest() override {
delete fw_dec_;
delete inv_dec_;
}
- virtual void SetUp() { InitializeConfig(libaom_test::kTwoPassGood); }
+ void SetUp() override { InitializeConfig(libaom_test::kTwoPassGood); }
- virtual void PreEncodeFrameHook(libaom_test::VideoSource *video,
- libaom_test::Encoder *encoder) {
+ void PreEncodeFrameHook(libaom_test::VideoSource *video,
+ libaom_test::Encoder *encoder) override {
if (video->frame() == 0) {
encoder->Control(AV1E_SET_TILE_COLUMNS, n_tile_cols_);
encoder->Control(AV1E_SET_TILE_ROWS, n_tile_rows_);
@@ -82,7 +82,7 @@
md5->Add(img);
}
- virtual void FramePktHook(const aom_codec_cx_pkt_t *pkt) {
+ void FramePktHook(const aom_codec_cx_pkt_t *pkt) override {
UpdateMD5(fw_dec_, pkt, &md5_fw_order_);
UpdateMD5(inv_dec_, pkt, &md5_inv_order_);
}
@@ -123,7 +123,7 @@
}
class TileIndependenceTestLarge : public TileIndependenceTest {
- virtual void SetCpuUsed(libaom_test::Encoder *encoder) {
+ void SetCpuUsed(libaom_test::Encoder *encoder) override {
static const int kCpuUsed = 0;
encoder->Control(AOME_SET_CPUUSED, kCpuUsed);
}
diff --git a/test/time_stamp_test.cc b/test/time_stamp_test.cc
index baa0dc0..5de98b7 100644
--- a/test/time_stamp_test.cc
+++ b/test/time_stamp_test.cc
@@ -47,16 +47,16 @@
(static_cast<double>(framerate_numerator_) / framerate_denominator_);
}
- virtual aom_codec_pts_t pts() const {
+ aom_codec_pts_t pts() const override {
return static_cast<aom_codec_pts_t>(frame_ * FrameDuration() +
starting_pts_ + 0.5);
}
- virtual unsigned long duration() const {
+ unsigned long duration() const override {
return static_cast<unsigned long>(FrameDuration() + 0.5);
}
- virtual aom_rational_t timebase() const { return timebase_; }
+ aom_rational_t timebase() const override { return timebase_; }
void set_starting_pts(int64_t starting_pts) { starting_pts_ = starting_pts; }
@@ -72,9 +72,9 @@
public ::libaom_test::CodecTestWithParam<libaom_test::TestMode> {
protected:
TimestampTest() : EncoderTest(GET_PARAM(0)) {}
- virtual ~TimestampTest() {}
+ ~TimestampTest() override = default;
- virtual void SetUp() { InitializeConfig(GET_PARAM(1)); }
+ void SetUp() override { InitializeConfig(GET_PARAM(1)); }
};
// Tests encoding in millisecond timebase.
diff --git a/test/tools_common.sh b/test/tools_common.sh
index f2d1802..cb9eba1 100755
--- a/test/tools_common.sh
+++ b/test/tools_common.sh
@@ -312,7 +312,11 @@
# Combine environment and actual tests.
local tests_to_run="${env_tests} ${tests_to_filter}"
- check_version_strings
+ # av1_c_vs_simd_encode is a standalone test, and it doesn't need to check the
+ # version string.
+ if [ "${test_name}" != "av1_c_vs_simd_encode" ]; then
+ check_version_strings
+ fi
# Run tests.
for test in ${tests_to_run}; do
@@ -464,6 +468,8 @@
AOM_TEST_PRESERVE_OUTPUT=${AOM_TEST_PRESERVE_OUTPUT:-no}
+# This checking requires config/aom_config.c that is available in Jenkins
+# testing.
if [ "$(is_windows_target)" = "yes" ]; then
AOM_TEST_EXE_SUFFIX=".exe"
fi
diff --git a/test/transform_test_base.h b/test/transform_test_base.h
index 260f4ff..55e78fe 100644
--- a/test/transform_test_base.h
+++ b/test/transform_test_base.h
@@ -12,11 +12,12 @@
#ifndef AOM_TEST_TRANSFORM_TEST_BASE_H_
#define AOM_TEST_TRANSFORM_TEST_BASE_H_
-#include "config/aom_config.h"
+#include "third_party/googletest/src/googletest/include/gtest/gtest.h"
-#include "aom_mem/aom_mem.h"
#include "aom/aom_codec.h"
#include "aom_dsp/txfm_common.h"
+#include "aom_mem/aom_mem.h"
+#include "test/acm_random.h"
namespace libaom_test {
@@ -40,7 +41,7 @@
template <typename OutType>
class TransformTestBase {
public:
- virtual ~TransformTestBase() {}
+ virtual ~TransformTestBase() = default;
protected:
virtual void RunFwdTxfm(const int16_t *in, OutType *out, int stride) = 0;
diff --git a/test/variance_test.cc b/test/variance_test.cc
index 2863aea..adca1b1 100644
--- a/test/variance_test.cc
+++ b/test/variance_test.cc
@@ -348,7 +348,7 @@
public:
SumOfSquaresTest() : func_(GetParam()) {}
- virtual ~SumOfSquaresTest() {}
+ ~SumOfSquaresTest() override = default;
protected:
void ConstTest();
@@ -427,7 +427,7 @@
class MseWxHTestClass
: public ::testing::TestWithParam<TestParams<FunctionType> > {
public:
- virtual void SetUp() {
+ void SetUp() override {
params_ = this->GetParam();
rnd_.Reset(ACMRandom::DeterministicSeed());
@@ -439,7 +439,7 @@
ASSERT_NE(dst_, nullptr);
}
- virtual void TearDown() {
+ void TearDown() override {
aom_free(src_);
aom_free(dst_);
src_ = nullptr;
@@ -528,7 +528,7 @@
// Memory required to compute mse of two 8x8 and four 4x4 blocks assigned for
// maximum width 16 and maximum height 8.
int mem_size = 16 * 8;
- virtual void SetUp() {
+ void SetUp() override {
params_ = this->GetParam();
rnd_.Reset(ACMRandom::DeterministicSeed());
src_ = reinterpret_cast<uint16_t *>(
@@ -539,7 +539,7 @@
ASSERT_NE(dst_, nullptr);
}
- virtual void TearDown() {
+ void TearDown() override {
aom_free(src_);
aom_free(dst_);
src_ = nullptr;
@@ -659,7 +659,7 @@
class MainTestClass
: public ::testing::TestWithParam<TestParams<FunctionType> > {
public:
- virtual void SetUp() {
+ void SetUp() override {
params_ = this->GetParam();
rnd_.Reset(ACMRandom::DeterministicSeed());
@@ -678,7 +678,7 @@
}
}
- virtual void TearDown() {
+ void TearDown() override {
if (use_high_bit_depth()) {
// TODO(skal): remove!
src_ = reinterpret_cast<uint8_t *>(CONVERT_TO_SHORTPTR(src_));
@@ -1286,7 +1286,7 @@
class SubpelVarianceTest
: public ::testing::TestWithParam<TestParams<FunctionType> > {
public:
- virtual void SetUp() {
+ void SetUp() override {
params_ = this->GetParam();
rnd_.Reset(ACMRandom::DeterministicSeed());
@@ -1308,7 +1308,7 @@
ASSERT_NE(ref_, nullptr);
}
- virtual void TearDown() {
+ void TearDown() override {
if (!use_high_bit_depth()) {
aom_free(src_);
aom_free(ref_);
@@ -1544,7 +1544,7 @@
class ObmcVarianceTest
: public ::testing::TestWithParam<TestParams<FunctionType> > {
public:
- virtual void SetUp() {
+ void SetUp() override {
params_ = this->GetParam();
rnd_.Reset(ACMRandom::DeterministicSeed());
@@ -1553,7 +1553,7 @@
aom_memalign(32, block_size() + width() + height() + 1));
} else {
pre_ = CONVERT_TO_BYTEPTR(reinterpret_cast<uint16_t *>(aom_memalign(
- 32, block_size() + width() + height() + 1 * sizeof(uint16_t))));
+ 32, (block_size() + width() + height() + 1) * sizeof(uint16_t))));
}
wsrc_ = reinterpret_cast<int32_t *>(
aom_memalign(32, block_size() * sizeof(uint32_t)));
@@ -1564,7 +1564,7 @@
ASSERT_NE(mask_, nullptr);
}
- virtual void TearDown() {
+ void TearDown() override {
if (!use_high_bit_depth()) {
aom_free(pre_);
} else {
@@ -1635,7 +1635,8 @@
memset(pre_ + half, 0, half + width() + height() + 1);
} else {
aom_memset16(CONVERT_TO_SHORTPTR(pre_), bd_mask(), half);
- aom_memset16(CONVERT_TO_SHORTPTR(pre_) + half, 0, half);
+ aom_memset16(CONVERT_TO_SHORTPTR(pre_) + half, 0,
+ half + width() + height() + 1);
}
for (int j = 0; j < half; j++) {
wsrc_[j] = bd_mask() * kMaskMax * kMaskMax;
@@ -1962,7 +1963,7 @@
class MseHBDWxHTestClass
: public ::testing::TestWithParam<TestParams<FunctionType> > {
public:
- virtual void SetUp() {
+ void SetUp() override {
params_ = this->GetParam();
rnd_.Reset(ACMRandom::DeterministicSeed());
@@ -1974,7 +1975,7 @@
ASSERT_NE(dst_, nullptr);
}
- virtual void TearDown() {
+ void TearDown() override {
aom_free(src_);
aom_free(dst_);
src_ = nullptr;
@@ -2062,6 +2063,8 @@
typedef MainTestClass<VarianceMxNFunc> AvxHBDVarianceTest;
typedef SubpelVarianceTest<SubpixVarMxNFunc> AvxHBDSubpelVarianceTest;
typedef SubpelVarianceTest<SubpixAvgVarMxNFunc> AvxHBDSubpelAvgVarianceTest;
+typedef SubpelVarianceTest<DistWtdSubpixAvgVarMxNFunc>
+ AvxHBDDistWtdSubpelAvgVarianceTest;
#if !CONFIG_REALTIME_ONLY
typedef ObmcVarianceTest<ObmcSubpelVarFunc> AvxHBDObmcSubpelVarianceTest;
#endif
@@ -2081,6 +2084,12 @@
TEST_P(AvxHBDSubpelVarianceTest, ExtremeRef) { ExtremeRefTest(); }
TEST_P(AvxHBDSubpelVarianceTest, DISABLED_Speed) { SpeedTest(); }
TEST_P(AvxHBDSubpelAvgVarianceTest, Ref) { RefTest(); }
+TEST_P(AvxHBDDistWtdSubpelAvgVarianceTest, Ref) { RefTest(); }
+#if !CONFIG_REALTIME_ONLY
+TEST_P(AvxHBDObmcSubpelVarianceTest, Ref) { RefTest(); }
+TEST_P(AvxHBDObmcSubpelVarianceTest, ExtremeRef) { ExtremeRefTest(); }
+TEST_P(AvxHBDObmcSubpelVarianceTest, DISABLED_Speed) { SpeedTest(); }
+#endif
INSTANTIATE_TEST_SUITE_P(
C, MseHBDWxHTest,
@@ -2106,6 +2115,14 @@
#if HAVE_NEON
INSTANTIATE_TEST_SUITE_P(
+ NEON, MseHBDWxHTest,
+ ::testing::Values(MseHBDWxHParams(3, 3, &aom_mse_wxh_16bit_highbd_neon, 10),
+ MseHBDWxHParams(3, 2, &aom_mse_wxh_16bit_highbd_neon, 10),
+ MseHBDWxHParams(2, 3, &aom_mse_wxh_16bit_highbd_neon, 10),
+ MseHBDWxHParams(2, 2, &aom_mse_wxh_16bit_highbd_neon,
+ 10)));
+
+INSTANTIATE_TEST_SUITE_P(
NEON, AvxHBDMseTest,
::testing::Values(MseParams(4, 4, &aom_highbd_12_mse16x16_neon, 12),
MseParams(4, 3, &aom_highbd_12_mse16x8_neon, 12),
@@ -2121,6 +2138,15 @@
MseParams(3, 3, &aom_highbd_8_mse8x8_neon, 8)));
#endif // HAVE_NEON
+#if HAVE_NEON_DOTPROD
+INSTANTIATE_TEST_SUITE_P(
+ NEON_DOTPROD, AvxHBDMseTest,
+ ::testing::Values(MseParams(4, 4, &aom_highbd_8_mse16x16_neon_dotprod, 8),
+ MseParams(4, 3, &aom_highbd_8_mse16x8_neon_dotprod, 8),
+ MseParams(3, 4, &aom_highbd_8_mse8x16_neon_dotprod, 8),
+ MseParams(3, 3, &aom_highbd_8_mse8x8_neon_dotprod, 8)));
+#endif // HAVE_NEON_DOTPROD
+
const VarianceParams kArrayHBDVariance_c[] = {
VarianceParams(7, 7, &aom_highbd_12_variance128x128_c, 12),
VarianceParams(7, 6, &aom_highbd_12_variance128x64_c, 12),
@@ -2389,27 +2415,177 @@
INSTANTIATE_TEST_SUITE_P(C, AvxHBDSubpelAvgVarianceTest,
::testing::ValuesIn(kArrayHBDSubpelAvgVariance_c));
+const DistWtdSubpelAvgVarianceParams kArrayHBDDistWtdSubpelAvgVariance_c[] = {
+ DistWtdSubpelAvgVarianceParams(
+ 7, 7, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance128x128_c, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 7, 6, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance128x64_c, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 6, 7, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance64x128_c, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 6, 6, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance64x64_c, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 6, 5, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance64x32_c, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 5, 6, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance32x64_c, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 5, 5, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance32x32_c, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 5, 4, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance32x16_c, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 5, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance16x32_c, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 4, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance16x16_c, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 3, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance16x8_c, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 3, 4, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance8x16_c, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 3, 3, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance8x8_c, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 3, 2, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance8x4_c, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 2, 3, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance4x8_c, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 2, 2, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance4x4_c, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 7, 7, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance128x128_c, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 7, 6, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance128x64_c, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 6, 7, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance64x128_c, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 6, 6, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance64x64_c, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 6, 5, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance64x32_c, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 5, 6, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance32x64_c, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 5, 5, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance32x32_c, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 5, 4, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance32x16_c, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 5, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance16x32_c, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 4, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance16x16_c, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 3, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance16x8_c, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 3, 4, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance8x16_c, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 3, 3, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance8x8_c, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 3, 2, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance8x4_c, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 2, 3, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance4x8_c, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 2, 2, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance4x4_c, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 7, 7, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance128x128_c, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 7, 6, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance128x64_c, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 6, 7, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance64x128_c, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 6, 6, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance64x64_c, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 6, 5, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance64x32_c, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 5, 6, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance32x64_c, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 5, 5, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance32x32_c, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 5, 4, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance32x16_c, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 5, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance16x32_c, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 4, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance16x16_c, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 3, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance16x8_c, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 3, 4, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance8x16_c, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 3, 3, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance8x8_c, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 3, 2, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance8x4_c, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 2, 3, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance4x8_c, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 2, 2, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance4x4_c, 12),
+
+#if !CONFIG_REALTIME_ONLY
+ DistWtdSubpelAvgVarianceParams(
+ 6, 4, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance64x16_c, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 6, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance16x64_c, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 5, 3, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance32x8_c, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 3, 5, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance8x32_c, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 2, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance16x4_c, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 2, 4, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance4x16_c, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 6, 4, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance64x16_c, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 6, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance16x64_c, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 5, 3, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance32x8_c, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 3, 5, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance8x32_c, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 2, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance16x4_c, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 2, 4, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance4x16_c, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 6, 4, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance64x16_c, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 6, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance16x64_c, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 5, 3, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance32x8_c, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 3, 5, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance8x32_c, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 2, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance16x4_c, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 2, 4, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance4x16_c, 12),
+#endif
+};
+INSTANTIATE_TEST_SUITE_P(
+ C, AvxHBDDistWtdSubpelAvgVarianceTest,
+ ::testing::ValuesIn(kArrayHBDDistWtdSubpelAvgVariance_c));
+
#if !CONFIG_REALTIME_ONLY
const ObmcSubpelVarianceParams kArrayHBDObmcSubpelVariance_c[] = {
- ObmcSubpelVarianceParams(7, 7, &aom_highbd_obmc_sub_pixel_variance128x128_c,
+ ObmcSubpelVarianceParams(7, 7, &aom_highbd_8_obmc_sub_pixel_variance128x128_c,
8),
- ObmcSubpelVarianceParams(7, 6, &aom_highbd_obmc_sub_pixel_variance128x64_c,
+ ObmcSubpelVarianceParams(7, 6, &aom_highbd_8_obmc_sub_pixel_variance128x64_c,
8),
- ObmcSubpelVarianceParams(6, 7, &aom_highbd_obmc_sub_pixel_variance64x128_c,
+ ObmcSubpelVarianceParams(6, 7, &aom_highbd_8_obmc_sub_pixel_variance64x128_c,
8),
- ObmcSubpelVarianceParams(6, 6, &aom_highbd_obmc_sub_pixel_variance64x64_c, 8),
- ObmcSubpelVarianceParams(6, 5, &aom_highbd_obmc_sub_pixel_variance64x32_c, 8),
- ObmcSubpelVarianceParams(5, 6, &aom_highbd_obmc_sub_pixel_variance32x64_c, 8),
- ObmcSubpelVarianceParams(5, 5, &aom_highbd_obmc_sub_pixel_variance32x32_c, 8),
- ObmcSubpelVarianceParams(5, 4, &aom_highbd_obmc_sub_pixel_variance32x16_c, 8),
- ObmcSubpelVarianceParams(4, 5, &aom_highbd_obmc_sub_pixel_variance16x32_c, 8),
- ObmcSubpelVarianceParams(4, 4, &aom_highbd_obmc_sub_pixel_variance16x16_c, 8),
- ObmcSubpelVarianceParams(4, 3, &aom_highbd_obmc_sub_pixel_variance16x8_c, 8),
- ObmcSubpelVarianceParams(3, 4, &aom_highbd_obmc_sub_pixel_variance8x16_c, 8),
- ObmcSubpelVarianceParams(3, 3, &aom_highbd_obmc_sub_pixel_variance8x8_c, 8),
- ObmcSubpelVarianceParams(3, 2, &aom_highbd_obmc_sub_pixel_variance8x4_c, 8),
- ObmcSubpelVarianceParams(2, 3, &aom_highbd_obmc_sub_pixel_variance4x8_c, 8),
- ObmcSubpelVarianceParams(2, 2, &aom_highbd_obmc_sub_pixel_variance4x4_c, 8),
+ ObmcSubpelVarianceParams(6, 6, &aom_highbd_8_obmc_sub_pixel_variance64x64_c,
+ 8),
+ ObmcSubpelVarianceParams(6, 5, &aom_highbd_8_obmc_sub_pixel_variance64x32_c,
+ 8),
+ ObmcSubpelVarianceParams(5, 6, &aom_highbd_8_obmc_sub_pixel_variance32x64_c,
+ 8),
+ ObmcSubpelVarianceParams(5, 5, &aom_highbd_8_obmc_sub_pixel_variance32x32_c,
+ 8),
+ ObmcSubpelVarianceParams(5, 4, &aom_highbd_8_obmc_sub_pixel_variance32x16_c,
+ 8),
+ ObmcSubpelVarianceParams(4, 5, &aom_highbd_8_obmc_sub_pixel_variance16x32_c,
+ 8),
+ ObmcSubpelVarianceParams(4, 4, &aom_highbd_8_obmc_sub_pixel_variance16x16_c,
+ 8),
+ ObmcSubpelVarianceParams(4, 3, &aom_highbd_8_obmc_sub_pixel_variance16x8_c,
+ 8),
+ ObmcSubpelVarianceParams(3, 4, &aom_highbd_8_obmc_sub_pixel_variance8x16_c,
+ 8),
+ ObmcSubpelVarianceParams(3, 3, &aom_highbd_8_obmc_sub_pixel_variance8x8_c, 8),
+ ObmcSubpelVarianceParams(3, 2, &aom_highbd_8_obmc_sub_pixel_variance8x4_c, 8),
+ ObmcSubpelVarianceParams(2, 3, &aom_highbd_8_obmc_sub_pixel_variance4x8_c, 8),
+ ObmcSubpelVarianceParams(2, 2, &aom_highbd_8_obmc_sub_pixel_variance4x4_c, 8),
ObmcSubpelVarianceParams(7, 7,
&aom_highbd_10_obmc_sub_pixel_variance128x128_c, 10),
ObmcSubpelVarianceParams(7, 6, &aom_highbd_10_obmc_sub_pixel_variance128x64_c,
@@ -2475,12 +2651,18 @@
ObmcSubpelVarianceParams(2, 2, &aom_highbd_12_obmc_sub_pixel_variance4x4_c,
12),
- ObmcSubpelVarianceParams(6, 4, &aom_highbd_obmc_sub_pixel_variance64x16_c, 8),
- ObmcSubpelVarianceParams(4, 6, &aom_highbd_obmc_sub_pixel_variance16x64_c, 8),
- ObmcSubpelVarianceParams(5, 3, &aom_highbd_obmc_sub_pixel_variance32x8_c, 8),
- ObmcSubpelVarianceParams(3, 5, &aom_highbd_obmc_sub_pixel_variance8x32_c, 8),
- ObmcSubpelVarianceParams(4, 2, &aom_highbd_obmc_sub_pixel_variance16x4_c, 8),
- ObmcSubpelVarianceParams(2, 4, &aom_highbd_obmc_sub_pixel_variance4x16_c, 8),
+ ObmcSubpelVarianceParams(6, 4, &aom_highbd_8_obmc_sub_pixel_variance64x16_c,
+ 8),
+ ObmcSubpelVarianceParams(4, 6, &aom_highbd_8_obmc_sub_pixel_variance16x64_c,
+ 8),
+ ObmcSubpelVarianceParams(5, 3, &aom_highbd_8_obmc_sub_pixel_variance32x8_c,
+ 8),
+ ObmcSubpelVarianceParams(3, 5, &aom_highbd_8_obmc_sub_pixel_variance8x32_c,
+ 8),
+ ObmcSubpelVarianceParams(4, 2, &aom_highbd_8_obmc_sub_pixel_variance16x4_c,
+ 8),
+ ObmcSubpelVarianceParams(2, 4, &aom_highbd_8_obmc_sub_pixel_variance4x16_c,
+ 8),
ObmcSubpelVarianceParams(6, 4, &aom_highbd_10_obmc_sub_pixel_variance64x16_c,
10),
ObmcSubpelVarianceParams(4, 6, &aom_highbd_10_obmc_sub_pixel_variance16x64_c,
@@ -2772,6 +2954,12 @@
VarianceParams(4, 3, &aom_highbd_10_variance16x8_avx2, 10),
VarianceParams(3, 4, &aom_highbd_10_variance8x16_avx2, 10),
VarianceParams(3, 3, &aom_highbd_10_variance8x8_avx2, 10),
+#if !CONFIG_REALTIME_ONLY
+ VarianceParams(6, 4, &aom_highbd_10_variance64x16_avx2, 10),
+ VarianceParams(5, 3, &aom_highbd_10_variance32x8_avx2, 10),
+ VarianceParams(4, 6, &aom_highbd_10_variance16x64_avx2, 10),
+ VarianceParams(3, 5, &aom_highbd_10_variance8x32_avx2, 10),
+#endif
};
INSTANTIATE_TEST_SUITE_P(AVX2, AvxHBDVarianceTest,
@@ -3247,6 +3435,16 @@
MseWxHParams(2, 3, &aom_mse_wxh_16bit_neon, 8),
MseWxHParams(2, 2, &aom_mse_wxh_16bit_neon, 8)));
+INSTANTIATE_TEST_SUITE_P(
+ NEON, Mse16xHTest,
+ ::testing::Values(Mse16xHParams(3, 3, &aom_mse_16xh_16bit_neon, 8),
+ Mse16xHParams(3, 2, &aom_mse_16xh_16bit_neon, 8),
+ Mse16xHParams(2, 3, &aom_mse_16xh_16bit_neon, 8),
+ Mse16xHParams(2, 2, &aom_mse_16xh_16bit_neon, 8)));
+
+INSTANTIATE_TEST_SUITE_P(NEON, SumOfSquaresTest,
+ ::testing::Values(aom_get_mb_ss_neon));
+
INSTANTIATE_TEST_SUITE_P(NEON, AvxMseTest,
::testing::Values(MseParams(3, 3, &aom_mse8x8_neon),
MseParams(3, 4, &aom_mse8x16_neon),
@@ -3342,6 +3540,52 @@
INSTANTIATE_TEST_SUITE_P(NEON, AvxSubpelAvgVarianceTest,
::testing::ValuesIn(kArraySubpelAvgVariance_neon));
+const DistWtdSubpelAvgVarianceParams kArrayDistWtdSubpelAvgVariance_neon[] = {
+ DistWtdSubpelAvgVarianceParams(
+ 6, 6, &aom_dist_wtd_sub_pixel_avg_variance64x64_neon, 0),
+ DistWtdSubpelAvgVarianceParams(
+ 6, 5, &aom_dist_wtd_sub_pixel_avg_variance64x32_neon, 0),
+ DistWtdSubpelAvgVarianceParams(
+ 5, 6, &aom_dist_wtd_sub_pixel_avg_variance32x64_neon, 0),
+ DistWtdSubpelAvgVarianceParams(
+ 5, 5, &aom_dist_wtd_sub_pixel_avg_variance32x32_neon, 0),
+ DistWtdSubpelAvgVarianceParams(
+ 5, 4, &aom_dist_wtd_sub_pixel_avg_variance32x16_neon, 0),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 5, &aom_dist_wtd_sub_pixel_avg_variance16x32_neon, 0),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 4, &aom_dist_wtd_sub_pixel_avg_variance16x16_neon, 0),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 3, &aom_dist_wtd_sub_pixel_avg_variance16x8_neon, 0),
+ DistWtdSubpelAvgVarianceParams(
+ 3, 4, &aom_dist_wtd_sub_pixel_avg_variance8x16_neon, 0),
+ DistWtdSubpelAvgVarianceParams(
+ 3, 3, &aom_dist_wtd_sub_pixel_avg_variance8x8_neon, 0),
+ DistWtdSubpelAvgVarianceParams(
+ 3, 2, &aom_dist_wtd_sub_pixel_avg_variance8x4_neon, 0),
+ DistWtdSubpelAvgVarianceParams(
+ 2, 3, &aom_dist_wtd_sub_pixel_avg_variance4x8_neon, 0),
+ DistWtdSubpelAvgVarianceParams(
+ 2, 2, &aom_dist_wtd_sub_pixel_avg_variance4x4_neon, 0),
+#if !CONFIG_REALTIME_ONLY
+ DistWtdSubpelAvgVarianceParams(
+ 6, 4, &aom_dist_wtd_sub_pixel_avg_variance64x16_neon, 0),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 6, &aom_dist_wtd_sub_pixel_avg_variance16x64_neon, 0),
+ DistWtdSubpelAvgVarianceParams(
+ 5, 3, &aom_dist_wtd_sub_pixel_avg_variance32x8_neon, 0),
+ DistWtdSubpelAvgVarianceParams(
+ 3, 5, &aom_dist_wtd_sub_pixel_avg_variance8x32_neon, 0),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 2, &aom_dist_wtd_sub_pixel_avg_variance16x4_neon, 0),
+ DistWtdSubpelAvgVarianceParams(
+ 2, 4, &aom_dist_wtd_sub_pixel_avg_variance4x16_neon, 0),
+#endif // !CONFIG_REALTIME_ONLY
+};
+INSTANTIATE_TEST_SUITE_P(
+ NEON, AvxDistWtdSubpelAvgVarianceTest,
+ ::testing::ValuesIn(kArrayDistWtdSubpelAvgVariance_neon));
+
#if !CONFIG_REALTIME_ONLY
const ObmcSubpelVarianceParams kArrayObmcSubpelVariance_neon[] = {
ObmcSubpelVarianceParams(7, 7, &aom_obmc_sub_pixel_variance128x128_neon, 0),
@@ -3463,8 +3707,559 @@
INSTANTIATE_TEST_SUITE_P(NEON, AvxHBDVarianceTest,
::testing::ValuesIn(kArrayHBDVariance_neon));
+
+const SubpelVarianceParams kArrayHBDSubpelVariance_neon[] = {
+ SubpelVarianceParams(6, 6, &aom_highbd_12_sub_pixel_variance64x64_neon, 12),
+ SubpelVarianceParams(6, 5, &aom_highbd_12_sub_pixel_variance64x32_neon, 12),
+ SubpelVarianceParams(5, 6, &aom_highbd_12_sub_pixel_variance32x64_neon, 12),
+ SubpelVarianceParams(5, 5, &aom_highbd_12_sub_pixel_variance32x32_neon, 12),
+ SubpelVarianceParams(5, 4, &aom_highbd_12_sub_pixel_variance32x16_neon, 12),
+ SubpelVarianceParams(4, 5, &aom_highbd_12_sub_pixel_variance16x32_neon, 12),
+ SubpelVarianceParams(4, 4, &aom_highbd_12_sub_pixel_variance16x16_neon, 12),
+ SubpelVarianceParams(4, 3, &aom_highbd_12_sub_pixel_variance16x8_neon, 12),
+ SubpelVarianceParams(3, 4, &aom_highbd_12_sub_pixel_variance8x16_neon, 12),
+ SubpelVarianceParams(3, 3, &aom_highbd_12_sub_pixel_variance8x8_neon, 12),
+ SubpelVarianceParams(3, 2, &aom_highbd_12_sub_pixel_variance8x4_neon, 12),
+ SubpelVarianceParams(2, 3, &aom_highbd_12_sub_pixel_variance4x8_neon, 12),
+ SubpelVarianceParams(2, 2, &aom_highbd_12_sub_pixel_variance4x4_neon, 12),
+ SubpelVarianceParams(6, 6, &aom_highbd_10_sub_pixel_variance64x64_neon, 10),
+ SubpelVarianceParams(6, 5, &aom_highbd_10_sub_pixel_variance64x32_neon, 10),
+ SubpelVarianceParams(5, 6, &aom_highbd_10_sub_pixel_variance32x64_neon, 10),
+ SubpelVarianceParams(5, 5, &aom_highbd_10_sub_pixel_variance32x32_neon, 10),
+ SubpelVarianceParams(5, 4, &aom_highbd_10_sub_pixel_variance32x16_neon, 10),
+ SubpelVarianceParams(4, 5, &aom_highbd_10_sub_pixel_variance16x32_neon, 10),
+ SubpelVarianceParams(4, 4, &aom_highbd_10_sub_pixel_variance16x16_neon, 10),
+ SubpelVarianceParams(4, 3, &aom_highbd_10_sub_pixel_variance16x8_neon, 10),
+ SubpelVarianceParams(3, 4, &aom_highbd_10_sub_pixel_variance8x16_neon, 10),
+ SubpelVarianceParams(3, 3, &aom_highbd_10_sub_pixel_variance8x8_neon, 10),
+ SubpelVarianceParams(3, 2, &aom_highbd_10_sub_pixel_variance8x4_neon, 10),
+ SubpelVarianceParams(2, 3, &aom_highbd_10_sub_pixel_variance4x8_neon, 10),
+ SubpelVarianceParams(2, 2, &aom_highbd_10_sub_pixel_variance4x4_neon, 10),
+ SubpelVarianceParams(6, 6, &aom_highbd_8_sub_pixel_variance64x64_neon, 8),
+ SubpelVarianceParams(6, 5, &aom_highbd_8_sub_pixel_variance64x32_neon, 8),
+ SubpelVarianceParams(5, 6, &aom_highbd_8_sub_pixel_variance32x64_neon, 8),
+ SubpelVarianceParams(5, 5, &aom_highbd_8_sub_pixel_variance32x32_neon, 8),
+ SubpelVarianceParams(5, 4, &aom_highbd_8_sub_pixel_variance32x16_neon, 8),
+ SubpelVarianceParams(4, 5, &aom_highbd_8_sub_pixel_variance16x32_neon, 8),
+ SubpelVarianceParams(4, 4, &aom_highbd_8_sub_pixel_variance16x16_neon, 8),
+ SubpelVarianceParams(4, 3, &aom_highbd_8_sub_pixel_variance16x8_neon, 8),
+ SubpelVarianceParams(3, 4, &aom_highbd_8_sub_pixel_variance8x16_neon, 8),
+ SubpelVarianceParams(3, 3, &aom_highbd_8_sub_pixel_variance8x8_neon, 8),
+ SubpelVarianceParams(3, 2, &aom_highbd_8_sub_pixel_variance8x4_neon, 8),
+ SubpelVarianceParams(2, 3, &aom_highbd_8_sub_pixel_variance4x8_neon, 8),
+ SubpelVarianceParams(2, 2, &aom_highbd_8_sub_pixel_variance4x4_neon, 8),
+#if !CONFIG_REALTIME_ONLY
+ SubpelVarianceParams(6, 4, &aom_highbd_8_sub_pixel_variance64x16_neon, 8),
+ SubpelVarianceParams(4, 6, &aom_highbd_8_sub_pixel_variance16x64_neon, 8),
+ SubpelVarianceParams(5, 3, &aom_highbd_8_sub_pixel_variance32x8_neon, 8),
+ SubpelVarianceParams(3, 5, &aom_highbd_8_sub_pixel_variance8x32_neon, 8),
+ SubpelVarianceParams(4, 2, &aom_highbd_8_sub_pixel_variance16x4_neon, 8),
+ SubpelVarianceParams(2, 4, &aom_highbd_8_sub_pixel_variance4x16_neon, 8),
+ SubpelVarianceParams(6, 4, &aom_highbd_10_sub_pixel_variance64x16_neon, 10),
+ SubpelVarianceParams(4, 6, &aom_highbd_10_sub_pixel_variance16x64_neon, 10),
+ SubpelVarianceParams(5, 3, &aom_highbd_10_sub_pixel_variance32x8_neon, 10),
+ SubpelVarianceParams(3, 5, &aom_highbd_10_sub_pixel_variance8x32_neon, 10),
+ SubpelVarianceParams(4, 2, &aom_highbd_10_sub_pixel_variance16x4_neon, 10),
+ SubpelVarianceParams(2, 4, &aom_highbd_10_sub_pixel_variance4x16_neon, 10),
+ SubpelVarianceParams(6, 4, &aom_highbd_12_sub_pixel_variance64x16_neon, 12),
+ SubpelVarianceParams(4, 6, &aom_highbd_12_sub_pixel_variance16x64_neon, 12),
+ SubpelVarianceParams(5, 3, &aom_highbd_12_sub_pixel_variance32x8_neon, 12),
+ SubpelVarianceParams(3, 5, &aom_highbd_12_sub_pixel_variance8x32_neon, 12),
+ SubpelVarianceParams(4, 2, &aom_highbd_12_sub_pixel_variance16x4_neon, 12),
+ SubpelVarianceParams(2, 4, &aom_highbd_12_sub_pixel_variance4x16_neon, 12),
+#endif //! CONFIG_REALTIME_ONLY
+};
+
+INSTANTIATE_TEST_SUITE_P(NEON, AvxHBDSubpelVarianceTest,
+ ::testing::ValuesIn(kArrayHBDSubpelVariance_neon));
+
+const SubpelAvgVarianceParams kArrayHBDSubpelAvgVariance_neon[] = {
+ SubpelAvgVarianceParams(7, 7,
+ &aom_highbd_8_sub_pixel_avg_variance128x128_neon, 8),
+ SubpelAvgVarianceParams(7, 6, &aom_highbd_8_sub_pixel_avg_variance128x64_neon,
+ 8),
+ SubpelAvgVarianceParams(6, 7, &aom_highbd_8_sub_pixel_avg_variance64x128_neon,
+ 8),
+ SubpelAvgVarianceParams(6, 6, &aom_highbd_8_sub_pixel_avg_variance64x64_neon,
+ 8),
+ SubpelAvgVarianceParams(6, 5, &aom_highbd_8_sub_pixel_avg_variance64x32_neon,
+ 8),
+ SubpelAvgVarianceParams(5, 6, &aom_highbd_8_sub_pixel_avg_variance32x64_neon,
+ 8),
+ SubpelAvgVarianceParams(5, 5, &aom_highbd_8_sub_pixel_avg_variance32x32_neon,
+ 8),
+ SubpelAvgVarianceParams(5, 4, &aom_highbd_8_sub_pixel_avg_variance32x16_neon,
+ 8),
+ SubpelAvgVarianceParams(4, 5, &aom_highbd_8_sub_pixel_avg_variance16x32_neon,
+ 8),
+ SubpelAvgVarianceParams(4, 4, &aom_highbd_8_sub_pixel_avg_variance16x16_neon,
+ 8),
+ SubpelAvgVarianceParams(4, 3, &aom_highbd_8_sub_pixel_avg_variance16x8_neon,
+ 8),
+ SubpelAvgVarianceParams(3, 4, &aom_highbd_8_sub_pixel_avg_variance8x16_neon,
+ 8),
+ SubpelAvgVarianceParams(3, 3, &aom_highbd_8_sub_pixel_avg_variance8x8_neon,
+ 8),
+ SubpelAvgVarianceParams(3, 2, &aom_highbd_8_sub_pixel_avg_variance8x4_neon,
+ 8),
+ SubpelAvgVarianceParams(2, 3, &aom_highbd_8_sub_pixel_avg_variance4x8_neon,
+ 8),
+ SubpelAvgVarianceParams(2, 2, &aom_highbd_8_sub_pixel_avg_variance4x4_neon,
+ 8),
+ SubpelAvgVarianceParams(
+ 7, 7, &aom_highbd_10_sub_pixel_avg_variance128x128_neon, 10),
+ SubpelAvgVarianceParams(7, 6,
+ &aom_highbd_10_sub_pixel_avg_variance128x64_neon, 10),
+ SubpelAvgVarianceParams(6, 7,
+ &aom_highbd_10_sub_pixel_avg_variance64x128_neon, 10),
+ SubpelAvgVarianceParams(6, 6, &aom_highbd_10_sub_pixel_avg_variance64x64_neon,
+ 10),
+ SubpelAvgVarianceParams(6, 5, &aom_highbd_10_sub_pixel_avg_variance64x32_neon,
+ 10),
+ SubpelAvgVarianceParams(5, 6, &aom_highbd_10_sub_pixel_avg_variance32x64_neon,
+ 10),
+ SubpelAvgVarianceParams(5, 5, &aom_highbd_10_sub_pixel_avg_variance32x32_neon,
+ 10),
+ SubpelAvgVarianceParams(5, 4, &aom_highbd_10_sub_pixel_avg_variance32x16_neon,
+ 10),
+ SubpelAvgVarianceParams(4, 5, &aom_highbd_10_sub_pixel_avg_variance16x32_neon,
+ 10),
+ SubpelAvgVarianceParams(4, 4, &aom_highbd_10_sub_pixel_avg_variance16x16_neon,
+ 10),
+ SubpelAvgVarianceParams(4, 3, &aom_highbd_10_sub_pixel_avg_variance16x8_neon,
+ 10),
+ SubpelAvgVarianceParams(3, 4, &aom_highbd_10_sub_pixel_avg_variance8x16_neon,
+ 10),
+ SubpelAvgVarianceParams(3, 3, &aom_highbd_10_sub_pixel_avg_variance8x8_neon,
+ 10),
+ SubpelAvgVarianceParams(3, 2, &aom_highbd_10_sub_pixel_avg_variance8x4_neon,
+ 10),
+ SubpelAvgVarianceParams(2, 3, &aom_highbd_10_sub_pixel_avg_variance4x8_neon,
+ 10),
+ SubpelAvgVarianceParams(2, 2, &aom_highbd_10_sub_pixel_avg_variance4x4_neon,
+ 10),
+ SubpelAvgVarianceParams(
+ 7, 7, &aom_highbd_12_sub_pixel_avg_variance128x128_neon, 12),
+ SubpelAvgVarianceParams(7, 6,
+ &aom_highbd_12_sub_pixel_avg_variance128x64_neon, 12),
+ SubpelAvgVarianceParams(6, 7,
+ &aom_highbd_12_sub_pixel_avg_variance64x128_neon, 12),
+ SubpelAvgVarianceParams(6, 6, &aom_highbd_12_sub_pixel_avg_variance64x64_neon,
+ 12),
+ SubpelAvgVarianceParams(6, 5, &aom_highbd_12_sub_pixel_avg_variance64x32_neon,
+ 12),
+ SubpelAvgVarianceParams(5, 6, &aom_highbd_12_sub_pixel_avg_variance32x64_neon,
+ 12),
+ SubpelAvgVarianceParams(5, 5, &aom_highbd_12_sub_pixel_avg_variance32x32_neon,
+ 12),
+ SubpelAvgVarianceParams(5, 4, &aom_highbd_12_sub_pixel_avg_variance32x16_neon,
+ 12),
+ SubpelAvgVarianceParams(4, 5, &aom_highbd_12_sub_pixel_avg_variance16x32_neon,
+ 12),
+ SubpelAvgVarianceParams(4, 4, &aom_highbd_12_sub_pixel_avg_variance16x16_neon,
+ 12),
+ SubpelAvgVarianceParams(4, 3, &aom_highbd_12_sub_pixel_avg_variance16x8_neon,
+ 12),
+ SubpelAvgVarianceParams(3, 4, &aom_highbd_12_sub_pixel_avg_variance8x16_neon,
+ 12),
+ SubpelAvgVarianceParams(3, 3, &aom_highbd_12_sub_pixel_avg_variance8x8_neon,
+ 12),
+ SubpelAvgVarianceParams(3, 2, &aom_highbd_12_sub_pixel_avg_variance8x4_neon,
+ 12),
+ SubpelAvgVarianceParams(2, 3, &aom_highbd_12_sub_pixel_avg_variance4x8_neon,
+ 12),
+ SubpelAvgVarianceParams(2, 2, &aom_highbd_12_sub_pixel_avg_variance4x4_neon,
+ 12),
+
+#if !CONFIG_REALTIME_ONLY
+ SubpelAvgVarianceParams(6, 4, &aom_highbd_8_sub_pixel_avg_variance64x16_neon,
+ 8),
+ SubpelAvgVarianceParams(4, 6, &aom_highbd_8_sub_pixel_avg_variance16x64_neon,
+ 8),
+ SubpelAvgVarianceParams(5, 3, &aom_highbd_8_sub_pixel_avg_variance32x8_neon,
+ 8),
+ SubpelAvgVarianceParams(3, 5, &aom_highbd_8_sub_pixel_avg_variance8x32_neon,
+ 8),
+ SubpelAvgVarianceParams(4, 2, &aom_highbd_8_sub_pixel_avg_variance16x4_neon,
+ 8),
+ SubpelAvgVarianceParams(2, 4, &aom_highbd_8_sub_pixel_avg_variance4x16_neon,
+ 8),
+ SubpelAvgVarianceParams(6, 4, &aom_highbd_10_sub_pixel_avg_variance64x16_neon,
+ 10),
+ SubpelAvgVarianceParams(4, 6, &aom_highbd_10_sub_pixel_avg_variance16x64_neon,
+ 10),
+ SubpelAvgVarianceParams(5, 3, &aom_highbd_10_sub_pixel_avg_variance32x8_neon,
+ 10),
+ SubpelAvgVarianceParams(3, 5, &aom_highbd_10_sub_pixel_avg_variance8x32_neon,
+ 10),
+ SubpelAvgVarianceParams(4, 2, &aom_highbd_10_sub_pixel_avg_variance16x4_neon,
+ 10),
+ SubpelAvgVarianceParams(2, 4, &aom_highbd_10_sub_pixel_avg_variance4x16_neon,
+ 10),
+ SubpelAvgVarianceParams(6, 4, &aom_highbd_12_sub_pixel_avg_variance64x16_neon,
+ 12),
+ SubpelAvgVarianceParams(4, 6, &aom_highbd_12_sub_pixel_avg_variance16x64_neon,
+ 12),
+ SubpelAvgVarianceParams(5, 3, &aom_highbd_12_sub_pixel_avg_variance32x8_neon,
+ 12),
+ SubpelAvgVarianceParams(3, 5, &aom_highbd_12_sub_pixel_avg_variance8x32_neon,
+ 12),
+ SubpelAvgVarianceParams(4, 2, &aom_highbd_12_sub_pixel_avg_variance16x4_neon,
+ 12),
+ SubpelAvgVarianceParams(2, 4, &aom_highbd_12_sub_pixel_avg_variance4x16_neon,
+ 12),
+#endif
+};
+
+INSTANTIATE_TEST_SUITE_P(NEON, AvxHBDSubpelAvgVarianceTest,
+ ::testing::ValuesIn(kArrayHBDSubpelAvgVariance_neon));
+
+const DistWtdSubpelAvgVarianceParams
+ kArrayHBDDistWtdSubpelAvgVariance_neon[] = {
+ DistWtdSubpelAvgVarianceParams(
+ 7, 7, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance128x128_neon, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 7, 6, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance128x64_neon, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 6, 7, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance64x128_neon, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 6, 6, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance64x64_neon, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 6, 5, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance64x32_neon, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 5, 6, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance32x64_neon, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 5, 5, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance32x32_neon, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 5, 4, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance32x16_neon, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 5, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance16x32_neon, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 4, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance16x16_neon, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 3, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance16x8_neon, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 3, 4, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance8x16_neon, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 3, 3, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance8x8_neon, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 3, 2, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance8x4_neon, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 2, 3, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance4x8_neon, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 2, 2, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance4x4_neon, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 7, 7, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance128x128_neon, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 7, 6, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance128x64_neon, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 6, 7, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance64x128_neon, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 6, 6, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance64x64_neon, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 6, 5, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance64x32_neon, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 5, 6, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance32x64_neon, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 5, 5, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance32x32_neon, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 5, 4, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance32x16_neon, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 5, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance16x32_neon, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 4, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance16x16_neon, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 3, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance16x8_neon, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 3, 4, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance8x16_neon, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 3, 3, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance8x8_neon, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 3, 2, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance8x4_neon, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 2, 3, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance4x8_neon, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 2, 2, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance4x4_neon, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 7, 7, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance128x128_neon, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 7, 6, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance128x64_neon, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 6, 7, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance64x128_neon, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 6, 6, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance64x64_neon, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 6, 5, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance64x32_neon, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 5, 6, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance32x64_neon, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 5, 5, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance32x32_neon, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 5, 4, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance32x16_neon, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 5, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance16x32_neon, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 4, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance16x16_neon, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 3, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance16x8_neon, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 3, 4, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance8x16_neon, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 3, 3, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance8x8_neon, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 3, 2, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance8x4_neon, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 2, 3, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance4x8_neon, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 2, 2, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance4x4_neon, 12),
+
+#if !CONFIG_REALTIME_ONLY
+ DistWtdSubpelAvgVarianceParams(
+ 6, 4, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance64x16_neon, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 6, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance16x64_neon, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 5, 3, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance32x8_neon, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 3, 5, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance8x32_neon, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 2, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance16x4_neon, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 2, 4, &aom_highbd_8_dist_wtd_sub_pixel_avg_variance4x16_neon, 8),
+ DistWtdSubpelAvgVarianceParams(
+ 6, 4, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance64x16_neon, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 6, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance16x64_neon, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 5, 3, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance32x8_neon, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 3, 5, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance8x32_neon, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 2, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance16x4_neon, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 2, 4, &aom_highbd_10_dist_wtd_sub_pixel_avg_variance4x16_neon, 10),
+ DistWtdSubpelAvgVarianceParams(
+ 6, 4, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance64x16_neon, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 6, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance16x64_neon, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 5, 3, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance32x8_neon, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 3, 5, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance8x32_neon, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 4, 2, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance16x4_neon, 12),
+ DistWtdSubpelAvgVarianceParams(
+ 2, 4, &aom_highbd_12_dist_wtd_sub_pixel_avg_variance4x16_neon, 12),
+#endif // !CONFIG_REALTIME_ONLY
+ };
+INSTANTIATE_TEST_SUITE_P(
+ NEON, AvxHBDDistWtdSubpelAvgVarianceTest,
+ ::testing::ValuesIn(kArrayHBDDistWtdSubpelAvgVariance_neon));
+
+#if !CONFIG_REALTIME_ONLY
+const ObmcSubpelVarianceParams kArrayHBDObmcSubpelVariance_neon[] = {
+ ObmcSubpelVarianceParams(
+ 7, 7, &aom_highbd_12_obmc_sub_pixel_variance128x128_neon, 12),
+ ObmcSubpelVarianceParams(
+ 7, 6, &aom_highbd_12_obmc_sub_pixel_variance128x64_neon, 12),
+ ObmcSubpelVarianceParams(
+ 6, 7, &aom_highbd_12_obmc_sub_pixel_variance64x128_neon, 12),
+ ObmcSubpelVarianceParams(
+ 6, 6, &aom_highbd_12_obmc_sub_pixel_variance64x64_neon, 12),
+ ObmcSubpelVarianceParams(
+ 6, 5, &aom_highbd_12_obmc_sub_pixel_variance64x32_neon, 12),
+ ObmcSubpelVarianceParams(
+ 5, 6, &aom_highbd_12_obmc_sub_pixel_variance32x64_neon, 12),
+ ObmcSubpelVarianceParams(
+ 5, 5, &aom_highbd_12_obmc_sub_pixel_variance32x32_neon, 12),
+ ObmcSubpelVarianceParams(
+ 5, 4, &aom_highbd_12_obmc_sub_pixel_variance32x16_neon, 12),
+ ObmcSubpelVarianceParams(
+ 4, 5, &aom_highbd_12_obmc_sub_pixel_variance16x32_neon, 12),
+ ObmcSubpelVarianceParams(
+ 4, 4, &aom_highbd_12_obmc_sub_pixel_variance16x16_neon, 12),
+ ObmcSubpelVarianceParams(4, 3,
+ &aom_highbd_12_obmc_sub_pixel_variance16x8_neon, 12),
+ ObmcSubpelVarianceParams(3, 4,
+ &aom_highbd_12_obmc_sub_pixel_variance8x16_neon, 12),
+ ObmcSubpelVarianceParams(3, 3, &aom_highbd_12_obmc_sub_pixel_variance8x8_neon,
+ 12),
+ ObmcSubpelVarianceParams(3, 2, &aom_highbd_12_obmc_sub_pixel_variance8x4_neon,
+ 12),
+ ObmcSubpelVarianceParams(2, 3, &aom_highbd_12_obmc_sub_pixel_variance4x8_neon,
+ 12),
+ ObmcSubpelVarianceParams(2, 2, &aom_highbd_12_obmc_sub_pixel_variance4x4_neon,
+ 12),
+ ObmcSubpelVarianceParams(
+ 6, 4, &aom_highbd_12_obmc_sub_pixel_variance64x16_neon, 12),
+ ObmcSubpelVarianceParams(
+ 4, 6, &aom_highbd_12_obmc_sub_pixel_variance16x64_neon, 12),
+ ObmcSubpelVarianceParams(5, 3,
+ &aom_highbd_12_obmc_sub_pixel_variance32x8_neon, 12),
+ ObmcSubpelVarianceParams(3, 5,
+ &aom_highbd_12_obmc_sub_pixel_variance8x32_neon, 12),
+ ObmcSubpelVarianceParams(4, 2,
+ &aom_highbd_12_obmc_sub_pixel_variance16x4_neon, 12),
+ ObmcSubpelVarianceParams(2, 4,
+ &aom_highbd_12_obmc_sub_pixel_variance4x16_neon, 12),
+ ObmcSubpelVarianceParams(
+ 7, 7, &aom_highbd_10_obmc_sub_pixel_variance128x128_neon, 10),
+ ObmcSubpelVarianceParams(
+ 7, 6, &aom_highbd_10_obmc_sub_pixel_variance128x64_neon, 10),
+ ObmcSubpelVarianceParams(
+ 6, 7, &aom_highbd_10_obmc_sub_pixel_variance64x128_neon, 10),
+ ObmcSubpelVarianceParams(
+ 6, 6, &aom_highbd_10_obmc_sub_pixel_variance64x64_neon, 10),
+ ObmcSubpelVarianceParams(
+ 6, 5, &aom_highbd_10_obmc_sub_pixel_variance64x32_neon, 10),
+ ObmcSubpelVarianceParams(
+ 5, 6, &aom_highbd_10_obmc_sub_pixel_variance32x64_neon, 10),
+ ObmcSubpelVarianceParams(
+ 5, 5, &aom_highbd_10_obmc_sub_pixel_variance32x32_neon, 10),
+ ObmcSubpelVarianceParams(
+ 5, 4, &aom_highbd_10_obmc_sub_pixel_variance32x16_neon, 10),
+ ObmcSubpelVarianceParams(
+ 4, 5, &aom_highbd_10_obmc_sub_pixel_variance16x32_neon, 10),
+ ObmcSubpelVarianceParams(
+ 4, 4, &aom_highbd_10_obmc_sub_pixel_variance16x16_neon, 10),
+ ObmcSubpelVarianceParams(4, 3,
+ &aom_highbd_10_obmc_sub_pixel_variance16x8_neon, 10),
+ ObmcSubpelVarianceParams(3, 4,
+ &aom_highbd_10_obmc_sub_pixel_variance8x16_neon, 10),
+ ObmcSubpelVarianceParams(3, 3, &aom_highbd_10_obmc_sub_pixel_variance8x8_neon,
+ 10),
+ ObmcSubpelVarianceParams(3, 2, &aom_highbd_10_obmc_sub_pixel_variance8x4_neon,
+ 10),
+ ObmcSubpelVarianceParams(2, 3, &aom_highbd_10_obmc_sub_pixel_variance4x8_neon,
+ 10),
+ ObmcSubpelVarianceParams(2, 2, &aom_highbd_10_obmc_sub_pixel_variance4x4_neon,
+ 10),
+ ObmcSubpelVarianceParams(
+ 6, 4, &aom_highbd_10_obmc_sub_pixel_variance64x16_neon, 10),
+ ObmcSubpelVarianceParams(
+ 4, 6, &aom_highbd_10_obmc_sub_pixel_variance16x64_neon, 10),
+ ObmcSubpelVarianceParams(5, 3,
+ &aom_highbd_10_obmc_sub_pixel_variance32x8_neon, 10),
+ ObmcSubpelVarianceParams(3, 5,
+ &aom_highbd_10_obmc_sub_pixel_variance8x32_neon, 10),
+ ObmcSubpelVarianceParams(4, 2,
+ &aom_highbd_10_obmc_sub_pixel_variance16x4_neon, 10),
+ ObmcSubpelVarianceParams(2, 4,
+ &aom_highbd_10_obmc_sub_pixel_variance4x16_neon, 10),
+ ObmcSubpelVarianceParams(
+ 7, 7, &aom_highbd_8_obmc_sub_pixel_variance128x128_neon, 8),
+ ObmcSubpelVarianceParams(7, 6,
+ &aom_highbd_8_obmc_sub_pixel_variance128x64_neon, 8),
+ ObmcSubpelVarianceParams(6, 7,
+ &aom_highbd_8_obmc_sub_pixel_variance64x128_neon, 8),
+ ObmcSubpelVarianceParams(6, 6,
+ &aom_highbd_8_obmc_sub_pixel_variance64x64_neon, 8),
+ ObmcSubpelVarianceParams(6, 5,
+ &aom_highbd_8_obmc_sub_pixel_variance64x32_neon, 8),
+ ObmcSubpelVarianceParams(5, 6,
+ &aom_highbd_8_obmc_sub_pixel_variance32x64_neon, 8),
+ ObmcSubpelVarianceParams(5, 5,
+ &aom_highbd_8_obmc_sub_pixel_variance32x32_neon, 8),
+ ObmcSubpelVarianceParams(5, 4,
+ &aom_highbd_8_obmc_sub_pixel_variance32x16_neon, 8),
+ ObmcSubpelVarianceParams(4, 5,
+ &aom_highbd_8_obmc_sub_pixel_variance16x32_neon, 8),
+ ObmcSubpelVarianceParams(4, 4,
+ &aom_highbd_8_obmc_sub_pixel_variance16x16_neon, 8),
+ ObmcSubpelVarianceParams(4, 3, &aom_highbd_8_obmc_sub_pixel_variance16x8_neon,
+ 8),
+ ObmcSubpelVarianceParams(3, 4, &aom_highbd_8_obmc_sub_pixel_variance8x16_neon,
+ 8),
+ ObmcSubpelVarianceParams(3, 3, &aom_highbd_8_obmc_sub_pixel_variance8x8_neon,
+ 8),
+ ObmcSubpelVarianceParams(3, 2, &aom_highbd_8_obmc_sub_pixel_variance8x4_neon,
+ 8),
+ ObmcSubpelVarianceParams(2, 3, &aom_highbd_8_obmc_sub_pixel_variance4x8_neon,
+ 8),
+ ObmcSubpelVarianceParams(2, 2, &aom_highbd_8_obmc_sub_pixel_variance4x4_neon,
+ 8),
+ ObmcSubpelVarianceParams(6, 4,
+ &aom_highbd_8_obmc_sub_pixel_variance64x16_neon, 8),
+ ObmcSubpelVarianceParams(4, 6,
+ &aom_highbd_8_obmc_sub_pixel_variance16x64_neon, 8),
+ ObmcSubpelVarianceParams(5, 3, &aom_highbd_8_obmc_sub_pixel_variance32x8_neon,
+ 8),
+ ObmcSubpelVarianceParams(3, 5, &aom_highbd_8_obmc_sub_pixel_variance8x32_neon,
+ 8),
+ ObmcSubpelVarianceParams(4, 2, &aom_highbd_8_obmc_sub_pixel_variance16x4_neon,
+ 8),
+ ObmcSubpelVarianceParams(2, 4, &aom_highbd_8_obmc_sub_pixel_variance4x16_neon,
+ 8),
+};
+
+INSTANTIATE_TEST_SUITE_P(NEON, AvxHBDObmcSubpelVarianceTest,
+ ::testing::ValuesIn(kArrayHBDObmcSubpelVariance_neon));
+#endif // !CONFIG_REALTIME_ONLY
+
#endif // CONFIG_AV1_HIGHBITDEPTH
#endif // HAVE_NEON
+#if HAVE_NEON_DOTPROD
+
+const VarianceParams kArrayVariance_neon_dotprod[] = {
+ VarianceParams(7, 7, &aom_variance128x128_neon_dotprod),
+ VarianceParams(6, 6, &aom_variance64x64_neon_dotprod),
+ VarianceParams(7, 6, &aom_variance128x64_neon_dotprod),
+ VarianceParams(6, 7, &aom_variance64x128_neon_dotprod),
+ VarianceParams(6, 6, &aom_variance64x64_neon_dotprod),
+ VarianceParams(6, 5, &aom_variance64x32_neon_dotprod),
+ VarianceParams(5, 6, &aom_variance32x64_neon_dotprod),
+ VarianceParams(5, 5, &aom_variance32x32_neon_dotprod),
+ VarianceParams(5, 4, &aom_variance32x16_neon_dotprod),
+ VarianceParams(4, 5, &aom_variance16x32_neon_dotprod),
+ VarianceParams(4, 4, &aom_variance16x16_neon_dotprod),
+ VarianceParams(4, 3, &aom_variance16x8_neon_dotprod),
+ VarianceParams(3, 4, &aom_variance8x16_neon_dotprod),
+ VarianceParams(3, 3, &aom_variance8x8_neon_dotprod),
+ VarianceParams(3, 2, &aom_variance8x4_neon_dotprod),
+ VarianceParams(2, 3, &aom_variance4x8_neon_dotprod),
+ VarianceParams(2, 2, &aom_variance4x4_neon_dotprod),
+#if !CONFIG_REALTIME_ONLY
+ VarianceParams(2, 4, &aom_variance4x16_neon_dotprod),
+ VarianceParams(4, 2, &aom_variance16x4_neon_dotprod),
+ VarianceParams(3, 5, &aom_variance8x32_neon_dotprod),
+ VarianceParams(5, 3, &aom_variance32x8_neon_dotprod),
+ VarianceParams(4, 6, &aom_variance16x64_neon_dotprod),
+ VarianceParams(6, 4, &aom_variance64x16_neon_dotprod),
+#endif
+};
+
+INSTANTIATE_TEST_SUITE_P(NEON_DOTPROD, AvxVarianceTest,
+ ::testing::ValuesIn(kArrayVariance_neon_dotprod));
+
+const GetSseSumParams kArrayGetSseSum8x8Quad_neon_dotprod[] = {
+ GetSseSumParams(7, 7, &aom_get_var_sse_sum_8x8_quad_neon_dotprod, 0),
+ GetSseSumParams(6, 6, &aom_get_var_sse_sum_8x8_quad_neon_dotprod, 0),
+ GetSseSumParams(5, 5, &aom_get_var_sse_sum_8x8_quad_neon_dotprod, 0),
+ GetSseSumParams(5, 4, &aom_get_var_sse_sum_8x8_quad_neon_dotprod, 0)
+};
+INSTANTIATE_TEST_SUITE_P(
+ NEON_DOTPROD, GetSseSum8x8QuadTest,
+ ::testing::ValuesIn(kArrayGetSseSum8x8Quad_neon_dotprod));
+
+const GetSseSumParamsDual kArrayGetSseSum16x16Dual_neon_dotprod[] = {
+ GetSseSumParamsDual(7, 7, &aom_get_var_sse_sum_16x16_dual_neon_dotprod, 0),
+ GetSseSumParamsDual(6, 6, &aom_get_var_sse_sum_16x16_dual_neon_dotprod, 0),
+ GetSseSumParamsDual(5, 5, &aom_get_var_sse_sum_16x16_dual_neon_dotprod, 0),
+ GetSseSumParamsDual(5, 4, &aom_get_var_sse_sum_16x16_dual_neon_dotprod, 0)
+};
+INSTANTIATE_TEST_SUITE_P(
+ NEON_DOTPROD, GetSseSum16x16DualTest,
+ ::testing::ValuesIn(kArrayGetSseSum16x16Dual_neon_dotprod));
+
+INSTANTIATE_TEST_SUITE_P(
+ NEON_DOTPROD, AvxMseTest,
+ ::testing::Values(MseParams(3, 3, &aom_mse8x8_neon_dotprod),
+ MseParams(3, 4, &aom_mse8x16_neon_dotprod),
+ MseParams(4, 4, &aom_mse16x16_neon_dotprod),
+ MseParams(4, 3, &aom_mse16x8_neon_dotprod)));
+
+#endif // HAVE_NEON_DOTPROD
+
} // namespace
diff --git a/test/video_source.h b/test/video_source.h
index f7a8b98..9d73d7b 100644
--- a/test/video_source.h
+++ b/test/video_source.h
@@ -125,7 +125,7 @@
// aom_image_t images with associated timestamps and duration.
class VideoSource {
public:
- virtual ~VideoSource() {}
+ virtual ~VideoSource() = default;
// Prepare the stream for reading, rewind/open as necessary.
virtual void Begin() = 0;
@@ -160,35 +160,35 @@
ReallocImage();
}
- virtual ~DummyVideoSource() { aom_img_free(img_); }
+ ~DummyVideoSource() override { aom_img_free(img_); }
- virtual void Begin() {
+ void Begin() override {
frame_ = 0;
FillFrame();
}
- virtual void Next() {
+ void Next() override {
++frame_;
FillFrame();
}
- virtual aom_image_t *img() const {
+ aom_image_t *img() const override {
return (frame_ < limit_) ? img_ : nullptr;
}
// Models a stream where Timebase = 1/FPS, so pts == frame.
- virtual aom_codec_pts_t pts() const { return frame_; }
+ aom_codec_pts_t pts() const override { return frame_; }
- virtual unsigned long duration() const { return 1; }
+ unsigned long duration() const override { return 1; }
- virtual aom_rational_t timebase() const {
+ aom_rational_t timebase() const override {
const aom_rational_t t = { 1, 30 };
return t;
}
- virtual unsigned int frame() const { return frame_; }
+ unsigned int frame() const override { return frame_; }
- virtual unsigned int limit() const { return limit_; }
+ unsigned int limit() const override { return limit_; }
void set_limit(unsigned int limit) { limit_ = limit; }
@@ -234,7 +234,7 @@
: rnd_(seed), seed_(seed) {}
// Reset the RNG to get a matching stream for the second pass
- virtual void Begin() {
+ void Begin() override {
frame_ = 0;
rnd_.Reset(seed_);
FillFrame();
@@ -243,7 +243,7 @@
protected:
// 15 frames of noise, followed by 15 static frames. Reset to 0 rather
// than holding previous frames to encourage keyframes to be thrown.
- virtual void FillFrame() {
+ void FillFrame() override {
if (img_) {
if (frame_ % 30 < 15)
for (size_t i = 0; i < raw_sz_; ++i) img_->img_data[i] = rnd_.Rand8();
@@ -260,7 +260,7 @@
// decompressed images to the decoder.
class CompressedVideoSource {
public:
- virtual ~CompressedVideoSource() {}
+ virtual ~CompressedVideoSource() = default;
virtual void Init() = 0;
diff --git a/test/warp_filter_test.cc b/test/warp_filter_test.cc
index 1d9dd45..f0be7d2 100644
--- a/test/warp_filter_test.cc
+++ b/test/warp_filter_test.cc
@@ -33,19 +33,21 @@
C, AV1WarpFilterTest,
libaom_test::AV1WarpFilter::BuildParams(av1_warp_affine_c));
-#if HAVE_SSE4_1
-INSTANTIATE_TEST_SUITE_P(
- SSE4_1, AV1WarpFilterTest,
- libaom_test::AV1WarpFilter::BuildParams(av1_warp_affine_sse4_1));
-
-#if CONFIG_AV1_HIGHBITDEPTH
+#if CONFIG_AV1_HIGHBITDEPTH && (HAVE_SSE4_1 || HAVE_NEON)
TEST_P(AV1HighbdWarpFilterTest, CheckOutput) {
RunCheckOutput(std::get<4>(GET_PARAM(0)));
}
TEST_P(AV1HighbdWarpFilterTest, DISABLED_Speed) {
RunSpeedTest(std::get<4>(GET_PARAM(0)));
}
+#endif // CONFIG_AV1_HIGHBITDEPTH && (HAVE_SSE4_1 || HAVE_NEON)
+#if HAVE_SSE4_1
+INSTANTIATE_TEST_SUITE_P(
+ SSE4_1, AV1WarpFilterTest,
+ libaom_test::AV1WarpFilter::BuildParams(av1_warp_affine_sse4_1));
+
+#if CONFIG_AV1_HIGHBITDEPTH
INSTANTIATE_TEST_SUITE_P(SSE4_1, AV1HighbdWarpFilterTest,
libaom_test::AV1HighbdWarpFilter::BuildParams(
av1_highbd_warp_affine_sse4_1));
@@ -58,7 +60,6 @@
libaom_test::AV1WarpFilter::BuildParams(av1_warp_affine_avx2));
#if CONFIG_AV1_HIGHBITDEPTH
-
INSTANTIATE_TEST_SUITE_P(
AVX2, AV1HighbdWarpFilterTest,
libaom_test::AV1HighbdWarpFilter::BuildParams(av1_highbd_warp_affine_avx2));
@@ -69,6 +70,24 @@
INSTANTIATE_TEST_SUITE_P(
NEON, AV1WarpFilterTest,
libaom_test::AV1WarpFilter::BuildParams(av1_warp_affine_neon));
+
+#if CONFIG_AV1_HIGHBITDEPTH
+INSTANTIATE_TEST_SUITE_P(
+ NEON, AV1HighbdWarpFilterTest,
+ libaom_test::AV1HighbdWarpFilter::BuildParams(av1_highbd_warp_affine_neon));
+#endif // CONFIG_AV1_HIGHBITDEPTH
#endif // HAVE_NEON
+#if HAVE_NEON_I8MM
+INSTANTIATE_TEST_SUITE_P(
+ NEON_I8MM, AV1WarpFilterTest,
+ libaom_test::AV1WarpFilter::BuildParams(av1_warp_affine_neon_i8mm));
+#endif // HAVE_NEON_I8MM
+
+#if HAVE_SVE
+INSTANTIATE_TEST_SUITE_P(
+ SVE, AV1WarpFilterTest,
+ libaom_test::AV1WarpFilter::BuildParams(av1_warp_affine_sve));
+#endif // HAVE_SVE
+
} // namespace
diff --git a/test/warp_filter_test_util.cc b/test/warp_filter_test_util.cc
index e42671e..470c980 100644
--- a/test/warp_filter_test_util.cc
+++ b/test/warp_filter_test_util.cc
@@ -19,9 +19,14 @@
namespace libaom_test {
-int32_t random_warped_param(libaom_test::ACMRandom *rnd, int bits) {
- // 1 in 8 chance of generating zero (arbitrarily chosen)
- if (((rnd->Rand8()) & 7) == 0) return 0;
+int32_t random_warped_param(libaom_test::ACMRandom *rnd, int bits,
+ int rnd_gen_zeros) {
+ // Avoid accidentally generating a zero in speed tests, they are set by the
+ // is_*_zero parameters instead.
+ if (rnd_gen_zeros) {
+ // 1 in 8 chance of generating zero (arbitrarily chosen)
+ if (((rnd->Rand8()) & 7) == 0) return 0;
+ }
// Otherwise, enerate uniform values in the range
// [-(1 << bits), 1] U [1, 1<<bits]
int32_t v = 1 + (rnd->Rand16() & ((1 << bits) - 1));
@@ -33,34 +38,47 @@
int16_t *alpha, int16_t *beta, int16_t *gamma,
int16_t *delta, const int is_alpha_zero,
const int is_beta_zero, const int is_gamma_zero,
- const int is_delta_zero) {
- while (1) {
+ const int is_delta_zero, const int rnd_gen_zeros) {
+ while (true) {
int rnd8 = rnd->Rand8() & 3;
- mat[0] = random_warped_param(rnd, WARPEDMODEL_PREC_BITS + 6);
- mat[1] = random_warped_param(rnd, WARPEDMODEL_PREC_BITS + 6);
- mat[2] = (random_warped_param(rnd, WARPEDMODEL_PREC_BITS - 3)) +
- (1 << WARPEDMODEL_PREC_BITS);
- mat[3] = random_warped_param(rnd, WARPEDMODEL_PREC_BITS - 3);
+ mat[0] = random_warped_param(rnd, WARPEDMODEL_PREC_BITS + 6, rnd_gen_zeros);
+ mat[1] = random_warped_param(rnd, WARPEDMODEL_PREC_BITS + 6, rnd_gen_zeros);
+ mat[2] =
+ (random_warped_param(rnd, WARPEDMODEL_PREC_BITS - 3, rnd_gen_zeros)) +
+ (1 << WARPEDMODEL_PREC_BITS);
+ mat[3] = random_warped_param(rnd, WARPEDMODEL_PREC_BITS - 3, rnd_gen_zeros);
if (rnd8 <= 1) {
// AFFINE
- mat[4] = random_warped_param(rnd, WARPEDMODEL_PREC_BITS - 3);
- mat[5] = (random_warped_param(rnd, WARPEDMODEL_PREC_BITS - 3)) +
- (1 << WARPEDMODEL_PREC_BITS);
+ mat[4] =
+ random_warped_param(rnd, WARPEDMODEL_PREC_BITS - 3, rnd_gen_zeros);
+ mat[5] =
+ (random_warped_param(rnd, WARPEDMODEL_PREC_BITS - 3, rnd_gen_zeros)) +
+ (1 << WARPEDMODEL_PREC_BITS);
} else if (rnd8 == 2) {
mat[4] = -mat[3];
mat[5] = mat[2];
} else {
- mat[4] = random_warped_param(rnd, WARPEDMODEL_PREC_BITS - 3);
- mat[5] = (random_warped_param(rnd, WARPEDMODEL_PREC_BITS - 3)) +
- (1 << WARPEDMODEL_PREC_BITS);
- if (is_alpha_zero == 1) mat[2] = 1 << WARPEDMODEL_PREC_BITS;
- if (is_beta_zero == 1) mat[3] = 0;
- if (is_gamma_zero == 1) mat[4] = 0;
- if (is_delta_zero == 1)
- mat[5] = static_cast<int32_t>(
- ((static_cast<int64_t>(mat[3]) * mat[4] + (mat[2] / 2)) / mat[2]) +
- (1 << WARPEDMODEL_PREC_BITS));
+ mat[4] =
+ random_warped_param(rnd, WARPEDMODEL_PREC_BITS - 3, rnd_gen_zeros);
+ mat[5] =
+ (random_warped_param(rnd, WARPEDMODEL_PREC_BITS - 3, rnd_gen_zeros)) +
+ (1 << WARPEDMODEL_PREC_BITS);
+ }
+
+ if (is_alpha_zero == 1) {
+ mat[2] = 1 << WARPEDMODEL_PREC_BITS;
+ }
+ if (is_beta_zero == 1) {
+ mat[3] = 0;
+ }
+ if (is_gamma_zero == 1) {
+ mat[4] = 0;
+ }
+ if (is_delta_zero == 1) {
+ mat[5] = static_cast<int32_t>(
+ ((static_cast<int64_t>(mat[3]) * mat[4] + (mat[2] / 2)) / mat[2]) +
+ (1 << WARPEDMODEL_PREC_BITS));
}
// Calculate the derived parameters and check that they are suitable
@@ -109,11 +127,9 @@
::testing::Values(0, 1), ::testing::Values(0, 1));
}
-AV1WarpFilterTest::~AV1WarpFilterTest() {}
+AV1WarpFilterTest::~AV1WarpFilterTest() = default;
void AV1WarpFilterTest::SetUp() { rnd_.Reset(ACMRandom::DeterministicSeed()); }
-void AV1WarpFilterTest::TearDown() {}
-
void AV1WarpFilterTest::RunSpeedTest(warp_affine_func test_impl) {
const int w = 128, h = 128;
const int border = 16;
@@ -144,7 +160,7 @@
ASSERT_NE(dsta, nullptr);
generate_warped_model(&rnd_, mat, &alpha, &beta, &gamma, &delta,
is_alpha_zero, is_beta_zero, is_gamma_zero,
- is_delta_zero);
+ is_delta_zero, 0);
for (int r = 0; r < h; ++r)
for (int c = 0; c < w; ++c) input[r * stride + c] = rnd_.Rand8();
@@ -170,8 +186,8 @@
aom_usec_timer_mark(&timer);
const int elapsed_time = static_cast<int>(aom_usec_timer_elapsed(&timer));
- printf("warp %3dx%-3d: %7.2f ns\n", out_w, out_h,
- 1000.0 * elapsed_time / num_loops);
+ printf("warp %3dx%-3d alpha=%d beta=%d gamma=%d delta=%d: %7.2f ns \n", out_w,
+ out_h, alpha, beta, gamma, delta, 1000.0 * elapsed_time / num_loops);
}
void AV1WarpFilterTest::RunCheckOutput(warp_affine_func test_impl) {
@@ -221,7 +237,7 @@
for (int sub_y = 0; sub_y < 2; ++sub_y) {
generate_warped_model(&rnd_, mat, &alpha, &beta, &gamma, &delta,
is_alpha_zero, is_beta_zero, is_gamma_zero,
- is_delta_zero);
+ is_delta_zero, 1);
for (int ii = 0; ii < 2; ++ii) {
for (int jj = 0; jj < 5; ++jj) {
@@ -301,13 +317,11 @@
::testing::Values(0, 1), ::testing::Values(0, 1));
}
-AV1HighbdWarpFilterTest::~AV1HighbdWarpFilterTest() {}
+AV1HighbdWarpFilterTest::~AV1HighbdWarpFilterTest() = default;
void AV1HighbdWarpFilterTest::SetUp() {
rnd_.Reset(ACMRandom::DeterministicSeed());
}
-void AV1HighbdWarpFilterTest::TearDown() {}
-
void AV1HighbdWarpFilterTest::RunSpeedTest(highbd_warp_affine_func test_impl) {
const int w = 128, h = 128;
const int border = 16;
@@ -339,7 +353,7 @@
generate_warped_model(&rnd_, mat, &alpha, &beta, &gamma, &delta,
is_alpha_zero, is_beta_zero, is_gamma_zero,
- is_delta_zero);
+ is_delta_zero, 0);
// Generate an input block and extend its borders horizontally
for (int r = 0; r < h; ++r)
for (int c = 0; c < w; ++c) input[r * stride + c] = rnd_.Rand16() & mask;
@@ -367,7 +381,8 @@
aom_usec_timer_mark(&timer);
const int elapsed_time = static_cast<int>(aom_usec_timer_elapsed(&timer));
- printf("highbd warp %3dx%-3d: %7.2f ns\n", out_w, out_h,
+ printf("highbd warp %3dx%-3d alpha=%d beta=%d gamma=%d delta=%d: %7.2f ns \n",
+ out_w, out_h, alpha, beta, gamma, delta,
1000.0 * elapsed_time / num_loops);
}
@@ -422,7 +437,7 @@
for (int sub_y = 0; sub_y < 2; ++sub_y) {
generate_warped_model(&rnd_, mat, &alpha, &beta, &gamma, &delta,
is_alpha_zero, is_beta_zero, is_gamma_zero,
- is_delta_zero);
+ is_delta_zero, 1);
for (int ii = 0; ii < 2; ++ii) {
for (int jj = 0; jj < 5; ++jj) {
for (int do_average = 0; do_average <= 1; ++do_average) {
diff --git a/test/warp_filter_test_util.h b/test/warp_filter_test_util.h
index 583f312..364368a 100644
--- a/test/warp_filter_test_util.h
+++ b/test/warp_filter_test_util.h
@@ -50,10 +50,8 @@
class AV1WarpFilterTest : public ::testing::TestWithParam<WarpTestParams> {
public:
- virtual ~AV1WarpFilterTest();
- virtual void SetUp();
-
- virtual void TearDown();
+ ~AV1WarpFilterTest() override;
+ void SetUp() override;
protected:
void RunCheckOutput(warp_affine_func test_impl);
@@ -86,10 +84,8 @@
class AV1HighbdWarpFilterTest
: public ::testing::TestWithParam<HighbdWarpTestParams> {
public:
- virtual ~AV1HighbdWarpFilterTest();
- virtual void SetUp();
-
- virtual void TearDown();
+ ~AV1HighbdWarpFilterTest() override;
+ void SetUp() override;
protected:
void RunCheckOutput(highbd_warp_affine_func test_impl);
diff --git a/test/webm_video_source.h b/test/webm_video_source.h
index 706e596..845abd6 100644
--- a/test/webm_video_source.h
+++ b/test/webm_video_source.h
@@ -30,19 +30,19 @@
webm_ctx_(new WebmInputContext()), buf_(nullptr), buf_sz_(0),
frame_sz_(0), frame_number_(0), end_of_file_(false) {}
- virtual ~WebMVideoSource() {
+ ~WebMVideoSource() override {
if (aom_ctx_->file != nullptr) fclose(aom_ctx_->file);
webm_free(webm_ctx_);
delete aom_ctx_;
delete webm_ctx_;
}
- virtual void Init() {
+ void Init() override {
ASSERT_NE(aom_ctx_, nullptr);
ASSERT_NE(webm_ctx_, nullptr);
}
- virtual void Begin() {
+ void Begin() override {
ASSERT_NE(aom_ctx_, nullptr);
ASSERT_NE(webm_ctx_, nullptr);
aom_ctx_->file = OpenTestDataFile(file_name_);
@@ -54,7 +54,7 @@
FillFrame();
}
- virtual void Next() {
+ void Next() override {
++frame_number_;
FillFrame();
}
@@ -85,11 +85,11 @@
} while (!webm_ctx_->is_key_frame && !end_of_file_);
}
- virtual const uint8_t *cxdata() const {
+ const uint8_t *cxdata() const override {
return end_of_file_ ? nullptr : buf_;
}
- virtual size_t frame_size() const { return frame_sz_; }
- virtual unsigned int frame_number() const { return frame_number_; }
+ size_t frame_size() const override { return frame_sz_; }
+ unsigned int frame_number() const override { return frame_number_; }
protected:
std::string file_name_;
diff --git a/test/wiener_test.cc b/test/wiener_test.cc
index 8be6a64..7eb6372 100644
--- a/test/wiener_test.cc
+++ b/test/wiener_test.cc
@@ -190,7 +190,7 @@
class WienerTest : public ::testing::TestWithParam<WienerTestParam> {
public:
- virtual void SetUp() {
+ void SetUp() override {
src_buf = (uint8_t *)aom_memalign(
32, MAX_DATA_BLOCK * MAX_DATA_BLOCK * sizeof(*src_buf));
ASSERT_NE(src_buf, nullptr);
@@ -204,7 +204,7 @@
memset(buf, 0, buf_size);
target_func_ = GET_PARAM(0);
}
- virtual void TearDown() {
+ void TearDown() override {
aom_free(src_buf);
aom_free(dgd_buf);
aom_free(buf);
@@ -322,9 +322,11 @@
buf + (3 * RESTORATION_UNITSIZE_MAX * RESTORATION_UNITSIZE_MAX);
for (int iter = 0; iter < iters && !HasFatalFailure(); ++iter) {
+ // Fill with alternating extreme values to maximize difference with
+ // the average.
for (int i = 0; i < MAX_DATA_BLOCK * MAX_DATA_BLOCK; ++i) {
- dgd_buf[i] = 255;
- src_buf[i] = 255;
+ dgd_buf[i] = i & 1 ? 255 : 0;
+ src_buf[i] = i & 1 ? 255 : 0;
}
uint8_t *dgd = dgd_buf + wiener_halfwin * MAX_DATA_BLOCK + wiener_halfwin;
uint8_t *src = src_buf;
@@ -389,6 +391,12 @@
::testing::Values(av1_compute_stats_avx2));
#endif // HAVE_AVX2
+#if HAVE_NEON
+
+INSTANTIATE_TEST_SUITE_P(NEON, WienerTest,
+ ::testing::Values(av1_compute_stats_neon));
+#endif // HAVE_NEON
+
} // namespace wiener_lowbd
#if CONFIG_AV1_HIGHBITDEPTH
@@ -531,7 +539,7 @@
class WienerTestHighbd : public ::testing::TestWithParam<WienerTestParam> {
public:
- virtual void SetUp() {
+ void SetUp() override {
src_buf = (uint16_t *)aom_memalign(
32, MAX_DATA_BLOCK * MAX_DATA_BLOCK * sizeof(*src_buf));
ASSERT_NE(src_buf, nullptr);
@@ -540,7 +548,7 @@
ASSERT_NE(dgd_buf, nullptr);
target_func_ = GET_PARAM(0);
}
- virtual void TearDown() {
+ void TearDown() override {
aom_free(src_buf);
aom_free(dgd_buf);
}
@@ -650,9 +658,11 @@
const int src_stride = MAX_DATA_BLOCK;
const int iters = 1;
for (int iter = 0; iter < iters && !HasFatalFailure(); ++iter) {
+ // Fill with alternating extreme values to maximize difference with
+ // the average.
for (int i = 0; i < MAX_DATA_BLOCK * MAX_DATA_BLOCK; ++i) {
- dgd_buf[i] = ((uint16_t)1 << bit_depth) - 1;
- src_buf[i] = ((uint16_t)1 << bit_depth) - 1;
+ dgd_buf[i] = i & 1 ? ((uint16_t)1 << bit_depth) - 1 : 0;
+ src_buf[i] = i & 1 ? ((uint16_t)1 << bit_depth) - 1 : 0;
}
const uint8_t *dgd8 = CONVERT_TO_BYTEPTR(
dgd_buf + wiener_halfwin * MAX_DATA_BLOCK + wiener_halfwin);
@@ -728,6 +738,11 @@
::testing::Values(av1_compute_stats_highbd_avx2));
#endif // HAVE_AVX2
+#if HAVE_NEON
+INSTANTIATE_TEST_SUITE_P(NEON, WienerTestHighbd,
+ ::testing::Values(av1_compute_stats_highbd_neon));
+#endif // HAVE_NEON
+
// A test that reproduces b/274668506: signed integer overflow in
// update_a_sep_sym().
TEST(SearchWienerTest, 10bitSignedIntegerOverflowInUpdateASepSym) {
diff --git a/test/y4m_test.cc b/test/y4m_test.cc
index 515a783..a4ed13f 100644
--- a/test/y4m_test.cc
+++ b/test/y4m_test.cc
@@ -66,7 +66,7 @@
protected:
Y4mVideoSourceTest() : Y4mVideoSource("", 0, 0) {}
- virtual ~Y4mVideoSourceTest() { CloseSource(); }
+ ~Y4mVideoSourceTest() override { CloseSource(); }
virtual void Init(const std::string &file_name, int limit) {
file_name_ = file_name;
@@ -128,7 +128,7 @@
protected:
Y4mVideoWriteTest() : tmpfile_(nullptr) {}
- virtual ~Y4mVideoWriteTest() {
+ ~Y4mVideoWriteTest() override {
delete tmpfile_;
input_file_ = nullptr;
}
@@ -162,7 +162,7 @@
ReplaceInputFile(tmpfile_->file());
}
- virtual void Init(const std::string &file_name, int limit) {
+ void Init(const std::string &file_name, int limit) override {
Y4mVideoSourceTest::Init(file_name, limit);
WriteY4mAndReadBack();
}
diff --git a/test/y4m_video_source.h b/test/y4m_video_source.h
index bf65776..1369e4e 100644
--- a/test/y4m_video_source.h
+++ b/test/y4m_video_source.h
@@ -28,7 +28,7 @@
start_(start), limit_(limit), frame_(0), framerate_numerator_(0),
framerate_denominator_(0), y4m_() {}
- virtual ~Y4mVideoSource() {
+ ~Y4mVideoSource() override {
aom_img_free(img_.get());
CloseSource();
}
@@ -53,33 +53,33 @@
FillFrame();
}
- virtual void Begin() {
+ void Begin() override {
OpenSource();
ReadSourceToStart();
}
- virtual void Next() {
+ void Next() override {
++frame_;
FillFrame();
}
- virtual aom_image_t *img() const {
+ aom_image_t *img() const override {
return (frame_ < limit_) ? img_.get() : nullptr;
}
// Models a stream where Timebase = 1/FPS, so pts == frame.
- virtual aom_codec_pts_t pts() const { return frame_; }
+ aom_codec_pts_t pts() const override { return frame_; }
- virtual unsigned long duration() const { return 1; }
+ unsigned long duration() const override { return 1; }
- virtual aom_rational_t timebase() const {
+ aom_rational_t timebase() const override {
const aom_rational_t t = { framerate_denominator_, framerate_numerator_ };
return t;
}
- virtual unsigned int frame() const { return frame_; }
+ unsigned int frame() const override { return frame_; }
- virtual unsigned int limit() const { return limit_; }
+ unsigned int limit() const override { return limit_; }
virtual void FillFrame() {
ASSERT_NE(input_file_, nullptr);
diff --git a/test/yuv_video_source.h b/test/yuv_video_source.h
index 1b898b5..77d5dfa 100644
--- a/test/yuv_video_source.h
+++ b/test/yuv_video_source.h
@@ -36,12 +36,12 @@
SetSize(width, height, format);
}
- virtual ~YUVVideoSource() {
+ ~YUVVideoSource() override {
aom_img_free(img_);
if (input_file_) fclose(input_file_);
}
- virtual void Begin() {
+ void Begin() override {
if (input_file_) fclose(input_file_);
input_file_ = OpenTestDataFile(file_name_);
ASSERT_NE(input_file_, nullptr)
@@ -53,28 +53,28 @@
FillFrame();
}
- virtual void Next() {
+ void Next() override {
++frame_;
FillFrame();
}
- virtual aom_image_t *img() const {
+ aom_image_t *img() const override {
return (frame_ < limit_) ? img_ : nullptr;
}
// Models a stream where Timebase = 1/FPS, so pts == frame.
- virtual aom_codec_pts_t pts() const { return frame_; }
+ aom_codec_pts_t pts() const override { return frame_; }
- virtual unsigned long duration() const { return 1; }
+ unsigned long duration() const override { return 1; }
- virtual aom_rational_t timebase() const {
+ aom_rational_t timebase() const override {
const aom_rational_t t = { framerate_denominator_, framerate_numerator_ };
return t;
}
- virtual unsigned int frame() const { return frame_; }
+ unsigned int frame() const override { return frame_; }
- virtual unsigned int limit() const { return limit_; }
+ unsigned int limit() const override { return limit_; }
virtual void SetSize(unsigned int width, unsigned int height,
aom_img_fmt format) {
diff --git a/third_party/fastfeat/README.libaom b/third_party/fastfeat/README.libaom
index 8aaee12..556d8b6 100644
--- a/third_party/fastfeat/README.libaom
+++ b/third_party/fastfeat/README.libaom
@@ -41,3 +41,4 @@
Add error checking
Add output argument to hold the scores of the detected features
Add assertion and rewrite comparisons to appease the scan-build static analyzer
+Set output argument *ret_num_corners to -1 to signal memory allocation failure
diff --git a/third_party/fastfeat/fast.c b/third_party/fastfeat/fast.c
index a684a33..c475b4c 100644
--- a/third_party/fastfeat/fast.c
+++ b/third_party/fastfeat/fast.c
@@ -42,7 +42,21 @@
xy* nonmax;
corners = aom_fast9_detect(im, xsize, ysize, stride, b, &num_corners);
+ if(!corners)
+ {
+ // Memory allocation failure
+ *ret_num_corners = -1;
+ return NULL;
+ }
+ // num_corners may be zero.
scores = aom_fast9_score(im, stride, corners, num_corners, b);
+ if(!scores && num_corners > 0)
+ {
+ // Memory allocation failure
+ free(corners);
+ *ret_num_corners = -1;
+ return NULL;
+ }
nonmax = aom_nonmax_suppression(corners, scores, num_corners, ret_scores, ret_num_corners);
free(corners);
diff --git a/third_party/fastfeat/fast.h b/third_party/fastfeat/fast.h
index 7fd199f..228ba85 100644
--- a/third_party/fastfeat/fast.h
+++ b/third_party/fastfeat/fast.h
@@ -37,10 +37,14 @@
int aom_fast9_corner_score(const byte* p, const int pixel[], int bstart);
+// Returns NULL on memory allocation failure.
xy* aom_fast9_detect(const byte* im, int xsize, int ysize, int stride, int b, int* ret_num_corners);
-int* aom_fast9_score(const byte* i, int stride, xy* corners, int num_corners, int b);
+// If num_corners > 0, returns NULL on memory allocation failure.
+int* aom_fast9_score(const byte* i, int stride, const xy* corners, int num_corners, int b);
+// Sets *ret_num_corners to -1 (and returns NULL) on memory allocation failure.
+// Sets *ret_num_corners to 0 if nothing went wrong but no corners were found.
xy* aom_fast9_detect_nonmax(const byte* im, int xsize, int ysize, int stride, int b,
int** ret_scores, int* ret_num_corners);
diff --git a/third_party/fastfeat/fast_9.c b/third_party/fastfeat/fast_9.c
index 345c37f..de55ab5 100644
--- a/third_party/fastfeat/fast_9.c
+++ b/third_party/fastfeat/fast_9.c
@@ -31,9 +31,7 @@
// clang-format off
/*This is mechanically generated code*/
#include <stdlib.h>
-
-typedef struct { int x, y; } xy;
-typedef unsigned char byte;
+#include "fast.h"
int aom_fast9_corner_score(const byte* p, const int pixel[], int bstart)
{
@@ -2988,7 +2986,7 @@
-int* aom_fast9_score(const byte* i, int stride, xy* corners, int num_corners, int b)
+int* aom_fast9_score(const byte* i, int stride, const xy* corners, int num_corners, int b)
{
int* scores = (int*)malloc(sizeof(int)* num_corners);
int n;
@@ -5927,8 +5925,13 @@
if(num_corners == rsize)
{
rsize*=2;
- ret_corners = (xy*)realloc(ret_corners, sizeof(xy)*rsize);
- if(!ret_corners) return NULL;
+ xy* new_ret_corners = (xy*)realloc(ret_corners, sizeof(xy)*rsize);
+ if(!new_ret_corners)
+ {
+ free(ret_corners);
+ return NULL;
+ }
+ ret_corners = new_ret_corners;
}
ret_corners[num_corners].x = x;
ret_corners[num_corners].y = y;
diff --git a/third_party/fastfeat/nonmax.c b/third_party/fastfeat/nonmax.c
index cc0ada7..a6f7da0 100644
--- a/third_party/fastfeat/nonmax.c
+++ b/third_party/fastfeat/nonmax.c
@@ -53,9 +53,10 @@
int point_below = 0;
*ret_scores = 0;
- *ret_num_nonmax = 0;
+ *ret_num_nonmax = -1;
if(!(corners && scores) || num_corners < 1)
{
+ *ret_num_nonmax = 0;
return 0;
}
