Merge tag 'v3.13.2' into main Release v3.13.2 2026-03-17 v3.13.2 This release includes several bug fixes. This release is ABI compatible with the last release. See https://aomedia.googlesource.com/aom/+log/v3.13.1..v3.13.2 for all the commits in this release. - Bug Fixes * x86: enable SSE4.2 in x86_simd_caps https://aomedia-review.googlesource.com/206481 * b:448994065: Working nasm-3.0.0 is not detected with libaom on 3.13.1 * b:480978101: Heap buffer overflow in av1_copy_and_extend_frame during encode triggered by NV12/bordered image with size_align * b:487259772: Out-of-bounds heap read in VP9 encoder via WebCodecs VideoEncoder dimension reconfiguration * b:489473886: Invalid memory read access in aom_sse_avx2 (Mozilla report) Bug: 493253375 Change-Id: Iba4c6ff6a9589da5c828bc25ad3c7fdd2ecf269b
diff --git a/CHANGELOG b/CHANGELOG index fb54ee5..a826d7d 100644 --- a/CHANGELOG +++ b/CHANGELOG
@@ -1,3 +1,15 @@ +yyyy-mm-dd next version + This release is ABI compatible with the last release. + + - Compression Efficiency Improvements + * Some minor quantization matrix formula and delta chroma q tweaks for the + AOM_TUNE_SSIMULACRA2 tuning mode, with up to 0.2% SSIMULACRA 2 score + efficiency gains. + + - Speedups + * Make Adaptive CDEF more decoder friendly by limiting CDEF application to + only regions where high filter strengths are needed. + 2026-03-17 v3.13.2 This release includes several bug fixes. This release is ABI compatible with the last release. See
diff --git a/CMakeLists.txt b/CMakeLists.txt index ac2b7cd..f25119c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt
@@ -151,10 +151,12 @@ "${AOM_ROOT}/aom/aom_codec.h" "${AOM_ROOT}/aom/aom_decoder.h" "${AOM_ROOT}/aom/aom_encoder.h" + "${AOM_ROOT}/aom/aom_ext_ratectrl.h" "${AOM_ROOT}/aom/aom_external_partition.h" "${AOM_ROOT}/aom/aom_frame_buffer.h" "${AOM_ROOT}/aom/aom_image.h" "${AOM_ROOT}/aom/aom_integer.h" + "${AOM_ROOT}/aom/aom_tpl.h" "${AOM_ROOT}/aom/aomcx.h" "${AOM_ROOT}/aom/aomdx.h" "${AOM_ROOT}/aom/internal/aom_codec_internal.h" @@ -799,6 +801,15 @@ list(APPEND AOM_APP_TARGETS lightfield_bitstream_parsing) endif() +if(ENABLE_EXAMPLES AND CONFIG_AV1_ENCODER) + add_executable(low_complexity_mode_encoder + "${AOM_ROOT}/examples/low_complexity_mode_encoder.c" + $<TARGET_OBJECTS:aom_common_app_util> + $<TARGET_OBJECTS:aom_encoder_app_util>) + list(APPEND AOM_EXAMPLE_TARGETS low_complexity_mode_encoder) + list(APPEND AOM_APP_TARGETS low_complexity_mode_encoder) +endif() + foreach(aom_app ${AOM_APP_TARGETS}) target_link_libraries(${aom_app} ${AOM_LIB_LINK_TYPE} aom) endforeach()
diff --git a/aom/aom_codec.h b/aom/aom_codec.h index 84454a2..1ecf5f8 100644 --- a/aom/aom_codec.h +++ b/aom/aom_codec.h
@@ -52,7 +52,7 @@ // configuration is changed based on the parameter. Note that a aom_codec_err_t // is returned, which indicates if the operation was successful or not. // -// Note that for the encoder, the aom_codec_alg_priv_t points to the +// Note that for the encoder, the aom_codec_alg_priv_t points to // the aom_codec_alg_priv structure in av1/av1_cx_iface.c, and for the decoder, // the struct in av1/av1_dx_iface.c. Variables such as AV1_COMP cpi are stored // here and also used in the core algorithm.
diff --git a/aom/aom_decoder.h b/aom/aom_decoder.h index ceef0a6..c202fe2 100644 --- a/aom/aom_decoder.h +++ b/aom/aom_decoder.h
@@ -106,6 +106,10 @@ * call is not thread safe and should be guarded with a lock if being used * in a multithreaded context. * + * On success, aom_codec_destroy() must be used to free resources allocated for + * the decoder context. If aom_codec_dec_init_ver() fails, it is not necessary + * to call aom_codec_destroy() on the decoder context. + * * \param[in] ctx Pointer to this instance's context. * \param[in] iface Pointer to the algorithm interface to use. * \param[in] cfg Configuration to use, if known. May be NULL.
diff --git a/aom/aom_encoder.h b/aom/aom_encoder.h index fa5f643..9d412af 100644 --- a/aom/aom_encoder.h +++ b/aom/aom_encoder.h
@@ -379,6 +379,9 @@ #define AOM_EFLAG_FORCE_KF (1 << 0) /*!\brief Calculate PSNR for this frame, requires g_lag_in_frames to be 0 */ #define AOM_EFLAG_CALCULATE_PSNR (1 << 1) +/*!\brief Freeze internal state, do not update reference buffers, entropy + * tables, rate control state, etc. */ +#define AOM_EFLAG_FREEZE_INTERNAL_STATE (1 << 2) /*!\brief Encoder configuration structure * @@ -887,11 +890,14 @@ */ int tile_heights[MAX_TILE_HEIGHTS]; - /*!\brief Whether encoder should use fixed QP offsets. + /*!\brief Controls how the encoder applies fixed QP offsets * + * If a value of 0 is provided, encoder will adaptively choose QP offsets for + * frames at different levels of the pyramid. * If a value of 1 is provided, encoder will use fixed QP offsets for frames * at different levels of the pyramid. - * If a value of 0 is provided, encoder will NOT use fixed QP offsets. + * If a value of 2 is provided, encoder will use the same QP for all frames + * at different levels of the pyramid (i.e. no QP offsets are applied). * Note: This option is only relevant for --end-usage=q. */ unsigned int use_fixed_qp_offsets; @@ -919,8 +925,9 @@ * is not thread safe and should be guarded with a lock if being used * in a multithreaded context. * - * If aom_codec_enc_init_ver() fails, it is not necessary to call - * aom_codec_destroy() on the encoder context. + * On success, aom_codec_destroy() must be used to free resources allocated for + * the encoder context. If aom_codec_enc_init_ver() fails, it is not necessary + * to call aom_codec_destroy() on the encoder context. * * \param[in] ctx Pointer to this instance's context. * \param[in] iface Pointer to the algorithm interface to use.
diff --git a/aom/aom_ext_ratectrl.h b/aom/aom_ext_ratectrl.h new file mode 100644 index 0000000..88598a3 --- /dev/null +++ b/aom/aom_ext_ratectrl.h
@@ -0,0 +1,663 @@ +/* + * Copyright (c) 2025, Alliance for Open Media. All rights reserved. + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +/*!\file + * \brief Defines structs and callbacks needed for external rate control. + * + */ +#ifndef AOM_AOM_AOM_EXT_RATECTRL_H_ +#define AOM_AOM_AOM_EXT_RATECTRL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "./aom_integer.h" +#include "./aom_tpl.h" + +/*!\brief Current ABI version number + * + * \internal + * If this file is altered in any way that changes the ABI, this value + * must be bumped. Examples include, but are not limited to, changing + * types, removing or reassigning enums, adding/removing/rearranging + * fields to structures. + */ +#define AOM_EXT_RATECTRL_ABI_VERSION (2 + AOM_TPL_ABI_VERSION) + +/*!\brief Corresponds to MAX_STATIC_GF_GROUP_LENGTH defined in ratectrl.h + */ +#define AOM_RC_MAX_STATIC_GF_GROUP_LENGTH 250 + +/*!\brief Max number of ref frames returned by the external RC. + * + * Corresponds to REF_FRAMES defined in enums.h. + */ +#define AOM_RC_MAX_REF_FRAMES 8 + +/*!\brief Max layer depth. + * + * Corresponds to MAX_ARF_LAYERS defined in ratectrl.h. + */ +#define AOM_RC_MAX_ARF_LAYERS 6 + +/*!\brief The type of the external rate control. + * + * This controls what encoder parameters are determined by the external rate + * control. + */ +typedef enum aom_rc_type { + /*! + * The external rate control doesn't determine anything. + * This mode is used as baseline. + */ + AOM_RC_NONE = 0, + /*! + * The external rate control model determines the quantization parameter (QP) + * for each frame. + */ + AOM_RC_QP = 1 << 0, + /*! + * The external rate control model determines the group of picture (GOP) of + * the video sequence. + */ + AOM_RC_GOP = 1 << 1, + /*! + * The external rate control model determines the rate-distortion multiplier + * (rdmult) for the current frame. + */ + AOM_RC_RDMULT = 1 << 2, + /*! + * The external rate control model determines both QP and GOP. + */ + AOM_RC_GOP_QP = AOM_RC_QP | AOM_RC_GOP, + /*! + * The external rate control model determines the QP, GOP and the rdmult. + */ + AOM_RC_GOP_QP_RDMULT = AOM_RC_QP | AOM_RC_GOP | AOM_RC_RDMULT +} aom_rc_type_t; + +/*!\brief The rate control mode for the external rate control model. + */ +typedef enum aom_ext_rc_mode { + AOM_RC_QMODE = 0, + AOM_RC_VBR = 1, + AOM_RC_CQ = 2, +} aom_ext_rc_mode_t; + +/*!\brief Corresponds to FRAME_UPDATE_TYPE defined in ratectrl.h. + */ +typedef enum aom_rc_frame_update_type { + AOM_RC_INVALID_UPDATE_TYPE = -1, + AOM_RC_KF_UPDATE = 0, + AOM_RC_LF_UPDATE = 1, + AOM_RC_GF_UPDATE = 2, + AOM_RC_ARF_UPDATE = 3, + AOM_RC_OVERLAY_UPDATE = 4, + AOM_RC_INTNL_OVERLAY_UPDATE = 5, // Internal Overlay Frame + AOM_RC_INTNL_ARF_UPDATE = 6, // Internal Altref Frame +} aom_rc_frame_update_type_t; + +/*!\brief Name for the ref frames returned by the external RC. + * + * Corresponds to the ref frames defined in enums.h. + */ +typedef enum aom_rc_ref_name { + AOM_RC_INVALID_REF_FRAME = -1, + AOM_RC_INTRA_FRAME, + AOM_RC_LAST_FRAME, + AOM_RC_LAST2_FRAME, + AOM_RC_LAST3_FRAME, + AOM_RC_GOLDEN_FRAME, + AOM_RC_BWDREF_FRAME, + AOM_RC_ALTREF2_FRAME, + AOM_RC_ALTREF_FRAME, +} aom_rc_ref_name_t; + +/*!\brief Abstract rate control model handler + * + * The encoder will receive the model handler from + * aom_rc_funcs_t::create_model(). + */ +typedef void *aom_rc_model_t; + +/*!\brief A reserved value for the q index. + * If the external rate control model returns this value, + * the encoder will use the default q selected by libaom's rate control + * system. + */ +#define AOM_DEFAULT_Q -1 + +/*!\brief A reserved value for the rdmult. + * If the external rate control model returns this value, + * the encoder will use the default rdmult selected by libaom's rate control + * system. + */ +#define AOM_DEFAULT_RDMULT -1 + +/*!\brief Superblock quantization parameters + * Store the superblock quantiztaion parameters + */ +typedef struct aom_sb_parameters { + int q_index; /**< Quantizer step index [0..255]*/ + int rdmult; /**< Superblock level Lagrangian multiplier*/ +} aom_sb_params; + +/*!\brief Encode frame decision made by the external rate control model + * + * The encoder will receive the decision from the external rate control model + * through aom_rc_funcs_t::get_encodeframe_decision(). + */ +typedef struct aom_rc_encodeframe_decision { + int q_index; /**< Required: Quantizer step index [0..255]*/ + int rdmult; /**< Required: Frame level Lagrangian multiplier*/ + // Whether per-superblock delta Q should be used. + // The rate control model should set the value pointed to by this member + // to 1 if per-superblock delta-Q is used for this frame, or 0 otherwise. + // This is a pointer to the flag in cpi->ext_ratectrl. + int *use_delta_q; + /*! + * Optional: Superblock quantization parameters + * It is zero initialized by default. It will be set for key and ARF frames + * but not leaf frames. + */ + aom_sb_params *sb_params_list; +} aom_rc_encodeframe_decision_t; + +/*!\brief Information for the frame to be encoded. + * + * The encoder will send the information to external rate control model through + * aom_rc_funcs_t::get_encodeframe_decision(). + * + */ +typedef struct aom_rc_encodeframe_info { + /*! + * 0: Key frame + * 1: Inter frame + * 2: Alternate reference frame + * 3: Overlay frame + * 4: Golden frame + */ + int frame_type; + int show_index; /**< display index, starts from zero*/ + int coding_index; /**< coding index, starts from zero*/ + /*! + * index of the current frame in this group of picture, starts from zero. + */ + int gop_index; + int ref_frame_coding_indexes[AOM_RC_MAX_REF_FRAMES]; /**< reference frames' + coding indices*/ + /*! + * The validity of the three reference frames. + * 0: Invalid + * 1: Valid + */ + int ref_frame_valid_list[AOM_RC_MAX_REF_FRAMES]; + /*! + * The length of the current GOP. + */ + int gop_size; + /*! + * Whether the current GOP uses an alt ref. + */ + int use_alt_ref; +} aom_rc_encodeframe_info_t; + +/*!\brief Frame coding result + * + * The encoder will send the result to the external rate control model through + * aom_rc_funcs_t::update_encodeframe_result(). + */ +typedef struct aom_rc_encodeframe_result { + int64_t bit_count; /**< number of bits spent on coding the frame*/ + int actual_encoding_qindex; /**< the actual qindex used to encode the frame*/ +} aom_rc_encodeframe_result_t; + +/*!\brief Status returned by rate control callback functions. + */ +typedef enum aom_rc_status { + AOM_RC_OK = 0, + AOM_RC_ERROR = 1, +} aom_rc_status_t; + +/*!\brief First pass frame stats + * This is a mirror of FIRSTPASS_STATS in firstpass.h + */ +typedef struct aom_rc_frame_stats { + /*! + * Frame number in display order, if stats are for a single frame. + * No real meaning for a collection of frames. + */ + double frame; + /*! + * Weight assigned to this frame (or total weight for the collection of + * frames) currently based on intra factor and brightness factor. This is used + * to distribute bits betweeen easier and harder frames. + */ + double weight; + /*! + * Intra prediction error. + */ + double intra_error; + /*! + * Average wavelet energy computed using Discrete Wavelet Transform (DWT). + */ + double frame_avg_wavelet_energy; + /*! + * Best of intra pred error and inter pred error using last frame as ref. + */ + double coded_error; + /*! + * Best of intra pred error and inter pred error using golden frame as ref. + */ + double sr_coded_error; + /*! + * Percentage of blocks with inter pred error < intra pred error. + */ + double pcnt_inter; + /*! + * Percentage of blocks using (inter prediction and) non-zero motion vectors. + */ + double pcnt_motion; + /*! + * Percentage of blocks where golden frame was better than last or intra: + * inter pred error using golden frame < inter pred error using last frame and + * inter pred error using golden frame < intra pred error + */ + double pcnt_second_ref; + /*! + * Percentage of blocks where intra and inter prediction errors were very + * close. Note that this is a 'weighted count', that is, the so blocks may be + * weighted by how close the two errors were. + */ + double pcnt_neutral; + /*! + * Percentage of blocks that have almost no intra error residual + * (i.e. are in effect completely flat and untextured in the intra + * domain). In natural videos this is uncommon, but it is much more + * common in animations, graphics and screen content, so may be used + * as a signal to detect these types of content. + */ + double intra_skip_pct; + /*! + * Image mask rows top and bottom. + */ + double inactive_zone_rows; + /*! + * Image mask columns at left and right edges. + */ + double inactive_zone_cols; + /*! + * Average of row motion vectors. + */ + double MVr; + /*! + * Mean of absolute value of row motion vectors. + */ + double mvr_abs; + /*! + * Mean of column motion vectors. + */ + double MVc; + /*! + * Mean of absolute value of column motion vectors. + */ + double mvc_abs; + /*! + * Variance of row motion vectors. + */ + double MVrv; + /*! + * Variance of column motion vectors. + */ + double MVcv; + /*! + * Value in range [-1,1] indicating fraction of row and column motion vectors + * that point inwards (negative MV value) or outwards (positive MV value). + * For example, value of 1 indicates, all row/column MVs are inwards. + */ + double mv_in_out_count; + /*! + * Count of unique non-zero motion vectors. + */ + double new_mv_count; + /*! + * Duration of the frame / collection of frames. + */ + double duration; + /*! + * 1.0 if stats are for a single frame, OR + * Number of frames in this collection for which the stats are accumulated. + */ + double count; + /*! + * standard deviation for (0, 0) motion prediction error + */ + double raw_error_stdev; + /*! + * Whether the frame contains a flash + */ + int64_t is_flash; + /*! + * Estimated noise variance + */ + double noise_var; + /*! + * Correlation coefficient with the previous frame + */ + double cor_coeff; + /*! + * log of intra_error + */ + double log_intra_error; + /*! + * log of coded_error + */ + double log_coded_error; +} aom_rc_frame_stats_t; + +/*!\brief Collection of first pass frame stats + */ +typedef struct aom_rc_firstpass_stats { + /*! + * Pointer to first pass frame stats. + * The pointed array of aom_rc_frame_stats_t should have length equal to + * number of show frames in the video. + */ + aom_rc_frame_stats_t *frame_stats; + /*! + * Number of show frames in the video. + */ + int num_frames; +} aom_rc_firstpass_stats_t; + +/*!\brief Encode config sent to external rate control model + */ +typedef struct aom_rc_config { + int frame_width; /**< frame width */ + int frame_height; /**< frame height */ + int show_frame_count; /**< number of visible frames in the video */ + int max_gf_interval; /**< max GOP size in number of show frames */ + int min_gf_interval; /**< min GOP size in number of show frames */ + /*! + * Target bitrate in kilobytes per second + */ + int target_bitrate_kbps; + int frame_rate_num; /**< numerator of frame rate */ + int frame_rate_den; /**< denominator of frame rate */ + /*! + * The following fields are only for external rate control models that support + * different rate control modes. + */ + aom_ext_rc_mode_t rc_mode; /**< Q mode or VBR mode */ + int overshoot_percent; /**< for VBR mode only */ + int undershoot_percent; /**< for VBR mode only */ + int min_base_q_index; /**< for VBR mode only */ + int max_base_q_index; /**< for VBR mode only */ + int base_qp; /**< base QP for leaf frames, 0-255 */ + int superblock_size; /**< 64 or 128 */ +} aom_rc_config_t; + +/*!\brief Control what ref frame to use and its index. + */ +typedef struct aom_rc_ref_frame { + /*! + * Ref frame index. + */ + int index[AOM_RC_MAX_REF_FRAMES]; + /*! + * Ref frame name. + */ + aom_rc_ref_name_t name[AOM_RC_MAX_REF_FRAMES]; +} aom_rc_ref_frame_t; + +/*!\brief A single reference frame. + */ +typedef struct aom_rc_single_ref_frame { + int index; /**< Ref frame index. */ + aom_rc_ref_name_t name; /**< Ref frame name. */ +} aom_rc_single_ref_frame_t; + +/*!\brief A single frame in the GOP. + */ +typedef struct aom_rc_gop_frame { + // Basic info + int order_idx; /**< Index in display order within a GOP, starting at 0. */ + int coding_idx; /**< Index in coding order within a GOP, starting at 0. */ + int display_idx; /**< The number of displayed frames preceding this frame in a + GOP. */ + int order_hint; /**< Index in display order since a key frame. */ + int global_order_idx; /**< Index in display order in the whole video chunk. */ + int global_coding_idx; /**< Index in coding order in the whole video chunk. */ + int is_key_frame; /**< Whether this frame is a key frame. */ + aom_rc_frame_update_type_t update_type; /**< The type of frame. */ + + // Reference frame info + /** + * For an overlay or show_existing frame: index of primary reference frame; + * otherwise -1. + */ + int colocated_ref_idx; + /** + * The reference index that this frame should be updated to. -1 when this + * frame will not serve as a reference frame. + */ + int update_ref_idx; + /** + * Additional bits to set in refresh_frame_flags, to allow extra frames to be + * evicted from the DPB. -1 for not set. + */ + int extra_refresh_frame_flags; + /** + * Candidate reference frames which may be used for coding the current frame. + */ + aom_rc_ref_frame_t ref_frame_list; + int layer_depth; /**< Layer depth in the GOP structure. */ + /** + * Primary reference frame (used to update current frame's initial probability + * model). + */ + aom_rc_single_ref_frame_t primary_ref_frame; +} aom_rc_gop_frame_t; + +/*!\brief The decision made by the external rate control model to set the + * group of picture. This struct represents a list of GOPs. + */ +typedef struct aom_rc_gop_decision { + int show_frame_count; /**< Number of visible frames in this GOP. */ + int order_hint_offset; /**< Order hint offset for this GOP. */ + int global_order_idx_offset; /**< Global order index offset for this GOP. */ + int global_coding_idx_offset; /**< Global coding index offset for this GOP. */ + int is_scene_cut; /**< Whether this GOP starts with a scene cut. */ + /** + * The adjustment ratio, based on which the base QP index of this GOP will be + * adjusted from RateControlParam::base_q_index. + */ + double base_q_ratio; + aom_rc_gop_frame_t *gop_frame_list; /**< List of frames in this GOP. */ + int gop_frame_count; +} aom_rc_gop_decision_t; + +/*!\brief The decision made by the external rate control model to set the + * key frame location and the show frame count in the key frame group + */ +typedef struct aom_rc_key_frame_decision { + int key_frame_show_index; /**< This key frame's show index in the video */ + int key_frame_group_size; /**< Show frame count of this key frame group */ +} aom_rc_key_frame_decision_t; + +/*!\brief Create an external rate control model callback prototype + * + * This callback is invoked by the encoder to create an external rate control + * model. + * + * \param[in] priv Callback's private data + * \param[in] ratectrl_config Pointer to aom_rc_config_t + * \param[out] rate_ctrl_model_ptr Pointer to aom_rc_model_t + */ +typedef aom_rc_status_t (*aom_rc_create_model_cb_fn_t)( + void *priv, const aom_rc_config_t *ratectrl_config, + aom_rc_model_t *rate_ctrl_model_ptr); + +/*!\brief Send first pass stats to the external rate control model callback + * prototype + * + * This callback is invoked by the encoder to send first pass stats to the + * external rate control model. + * + * \param[in] rate_ctrl_model rate control model + * \param[in] first_pass_stats first pass stats + */ +typedef aom_rc_status_t (*aom_rc_send_firstpass_stats_cb_fn_t)( + aom_rc_model_t rate_ctrl_model, + const aom_rc_firstpass_stats_t *first_pass_stats); + +/*!\brief Send TPL stats for the current GOP to the external rate control model + * callback prototype + * + * This callback is invoked by the encoder to send TPL stats for the GOP to the + * external rate control model. + * + * \param[in] rate_ctrl_model rate control model + * \param[in] tpl_gop_stats TPL stats for current GOP + */ +typedef aom_rc_status_t (*aom_rc_send_tpl_gop_stats_cb_fn_t)( + aom_rc_model_t rate_ctrl_model, const AomTplGopStats *extrc_tpl_gop_stats); + +/*!\brief Receive encode frame decision callback prototype + * + * This callback is invoked by the encoder to receive encode frame decision from + * the external rate control model. + * + * \param[in] rate_ctrl_model rate control model + * \param[in] frame_gop_index index of the frame in current gop + * \param[out] frame_decision encode decision of the coding frame + */ +typedef aom_rc_status_t (*aom_rc_get_encodeframe_decision_cb_fn_t)( + aom_rc_model_t rate_ctrl_model, const int frame_gop_index, + aom_rc_encodeframe_decision_t *frame_decision); + +/*!\brief Update encode frame result callback prototype + * + * This callback is invoked by the encoder to update encode frame result to the + * external rate control model. + * + * \param[in] rate_ctrl_model rate control model + * \param[out] encode_frame_result encode result of the coding frame + */ +typedef aom_rc_status_t (*aom_rc_update_encodeframe_result_cb_fn_t)( + aom_rc_model_t rate_ctrl_model, + const aom_rc_encodeframe_result_t *encode_frame_result); + +/*!\brief Get the key frame decision from the external rate control model. + * + * This callback is invoked by the encoder to get key frame decision from + * the external rate control model. + * + * \param[in] rate_ctrl_model rate control model + * \param[out] key_frame_decision key frame decision from the model + */ +typedef aom_rc_status_t (*aom_rc_get_key_frame_decision_cb_fn_t)( + aom_rc_model_t rate_ctrl_model, + aom_rc_key_frame_decision_t *key_frame_decision); + +/*!\brief Get the GOP structure from the external rate control model. + * + * This callback is invoked by the encoder to get GOP decisions from + * the external rate control model. + * + * \param[in] rate_ctrl_model rate control model + * \param[out] gop_decision GOP decision from the model + */ +typedef aom_rc_status_t (*aom_rc_get_gop_decision_cb_fn_t)( + aom_rc_model_t rate_ctrl_model, aom_rc_gop_decision_t *gop_decision); + +/*!\brief Get the frame rdmult from the external rate control model. + * + * This callback is invoked by the encoder to get rdmult from + * the external rate control model. + * + * \param[in] rate_ctrl_model rate control model + * \param[in] frame_info information collected from the encoder + * \param[out] rdmult frame rate-distortion multiplier from the model + */ +typedef aom_rc_status_t (*aom_rc_get_frame_rdmult_cb_fn_t)( + aom_rc_model_t rate_ctrl_model, const aom_rc_encodeframe_info_t *frame_info, + int *rdmult); + +/*!\brief Delete the external rate control model callback prototype + * + * This callback is invoked by the encoder to delete the external rate control + * model. + * + * \param[in] rate_ctrl_model rate control model + */ +typedef aom_rc_status_t (*aom_rc_delete_model_cb_fn_t)( + aom_rc_model_t rate_ctrl_model); + +/*!\brief Callback function set for external rate control. + * + * The user can enable external rate control by registering + * a set of callback functions with the codec control flag + * #AOME_SET_EXTERNAL_RATE_CONTROL. + */ +typedef struct aom_rc_funcs { + /*! + * The rate control type of this API. + */ + aom_rc_type_t rc_type; + /*! + * Create an external rate control model. + */ + aom_rc_create_model_cb_fn_t create_model; + /*! + * Send first pass stats to the external rate control model. + */ + aom_rc_send_firstpass_stats_cb_fn_t send_firstpass_stats; + /*! + * Send TPL stats for current GOP to the external rate control model. + */ + aom_rc_send_tpl_gop_stats_cb_fn_t send_tpl_gop_stats; + /*! + * Get encodeframe decision from the external rate control model. + */ + aom_rc_get_encodeframe_decision_cb_fn_t get_encodeframe_decision; + /*! + * Update encodeframe result to the external rate control model. + */ + aom_rc_update_encodeframe_result_cb_fn_t update_encodeframe_result; + /*! + * Get key frame decision from the external rate control model. + */ + aom_rc_get_key_frame_decision_cb_fn_t get_key_frame_decision; + /*! + * Get GOP decisions from the external rate control model. + */ + aom_rc_get_gop_decision_cb_fn_t get_gop_decision; + /*! + * Get rdmult for the frame from the external rate control model. + */ + aom_rc_get_frame_rdmult_cb_fn_t get_frame_rdmult; + /*! + * Delete the external rate control model. + */ + aom_rc_delete_model_cb_fn_t delete_model; + /*! + * Private data for the external rate control model. + */ + void *priv; +} aom_rc_funcs_t; + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_AOM_AOM_EXT_RATECTRL_H_
diff --git a/aom/aom_image.h b/aom/aom_image.h index 248b5b6..3b71086 100644 --- a/aom/aom_image.h +++ b/aom/aom_image.h
@@ -154,17 +154,33 @@ * * While encoding, when metadata is added to an aom_image via * aom_img_add_metadata(), the flag passed along with the metadata will - * determine where the metadata OBU will be placed in the encoded OBU stream. - * Metadata will be emitted into the output stream within the next temporal unit - * if it satisfies the specified insertion flag. + * determine where the metadata OBU will be placed in the encoded OBU stream, + * and whether it's layer-specific. Metadata will be emitted into the output + * stream within the next temporal unit if it satisfies the specified insertion + * flag. * - * During decoding, when the library encounters a metadata OBU, it is always - * flagged as AOM_MIF_ANY_FRAME and emitted with the next output aom_image. + * If the video contains multiple spatial and/or temporal layers, + * a layer-specific metadata OBU only applies to the current frame's layer, as + * determined by the frame's temporal_id and spatial_id. Some metadata types + * cannot be layer-specific, as listed in Section 6.7.1 of the draft AV1 + * specification as of 2025-03-06. + * + * During decoding, when the library encounters a metadata OBU, it is emitted + * with the next output aom_image. Its insert_flag is set to either + * AOM_MIF_ANY_FRAME, or AOM_MIF_ANY_FRAME_LAYER_SPECIFIC if the OBU contains an + * OBU header extension (i.e. the video contains multiple layers AND the + * metadata was added using *_LAYER_SPECIFIC insert flag if using libaom). */ typedef enum aom_metadata_insert_flags { - AOM_MIF_NON_KEY_FRAME = 0, /**< Adds metadata if it's not keyframe */ + AOM_MIF_NON_KEY_FRAME = 0, /**< Adds metadata if it's not a keyframe */ AOM_MIF_KEY_FRAME = 1, /**< Adds metadata only if it's a keyframe */ - AOM_MIF_ANY_FRAME = 2 /**< Adds metadata to any type of frame */ + AOM_MIF_ANY_FRAME = 2, /**< Adds metadata to any type of frame */ + /** Adds layer-specific metadata if it's not a keyframe */ + AOM_MIF_NON_KEY_FRAME_LAYER_SPECIFIC = 16, + /** Adds layer-specific metadata only if it's a keyframe */ + AOM_MIF_KEY_FRAME_LAYER_SPECIFIC = 17, + /** Adds layer-specific metadata to any type of frame */ + AOM_MIF_ANY_FRAME_LAYER_SPECIFIC = 18, } aom_metadata_insert_flags_t; /*!\brief Array of aom_metadata structs for an image. */ @@ -268,26 +284,55 @@ * storage for the image has been allocated elsewhere, and a descriptor is * desired to "wrap" that storage. * - * \param[in] img Pointer to storage for descriptor. If this parameter - * is NULL, the storage for the descriptor will be - * allocated on the heap. - * \param[in] fmt Format for the image - * \param[in] d_w Width of the image. Must not exceed 0x08000000 - * (2^27). - * \param[in] d_h Height of the image. Must not exceed 0x08000000 - * (2^27). - * \param[in] align Alignment, in bytes, of each row in the image - * (stride). Must not exceed 65536. - * \param[in] img_data Storage to use for the image. The storage must - * outlive the returned image descriptor; it can be - * disposed of after calling aom_img_free(). + * \param[in] img Pointer to storage for descriptor. If this + * parameter is NULL, the storage for the descriptor + * will be allocated on the heap. + * \param[in] fmt Format for the image + * \param[in] d_w Width of the image. Must not exceed 0x08000000 + * (2^27). + * \param[in] d_h Height of the image. Must not exceed 0x08000000 + * (2^27). + * \param[in] stride_align Alignment, in bytes, of each row in the image + * (stride). Must not exceed 65536. + * \param[in] img_data Storage to use for the image. The storage must + * outlive the returned image descriptor; it can be + * disposed of after calling aom_img_free(). * * \return Returns a pointer to the initialized image descriptor. If the img * parameter is non-null, the value of the img parameter will be * returned. + * + * \note \a img_data is required to have a minimum allocation size that + * satisfies the requirements of the \a fmt, \a d_w, \a d_h and \a + * stride_align parameters. This size can be calculated as follows (see + * \c img_alloc_helper in the aom_image.c file in the libaom source tree + * for more detail): + * \code + * align = (1 << x_chroma_shift) - 1; + * w = (d_w + align) & ~align; + * align = (1 << y_chroma_shift) - 1; + * h = (d_h + align) & ~align; + * + * s = (fmt & AOM_IMG_FMT_PLANAR) ? w : (uint64_t)bps * w / 8; + * s = (fmt & AOM_IMG_FMT_HIGHBITDEPTH) ? s * 2 : s; + * s = (s + stride_align - 1) & ~((uint64_t)stride_align - 1); + * s = (fmt & AOM_IMG_FMT_HIGHBITDEPTH) ? s / 2 : s; + * alloc_size = (fmt & AOM_IMG_FMT_PLANAR) ? (uint64_t)h * s * bps / 8 + * : (uint64_t)h * s; + * \endcode + * \a x_chroma_shift, \a y_chroma_shift and \a bps can be obtained by calling + * \ref aom_img_wrap with a non-\c NULL \a img_data parameter. The \c + * aom_image_t pointer should \em not be used in other API calls until \em after + * a successful call to \ref aom_img_wrap with a valid image buffer. For + * example: + * \code + * aom_img_wrap(img, fmt, d_w, d_h, stride_align, (unsigned char *)1); + * ... calculate buffer size and allocate buffer as described earlier + * aom_img_wrap(img, fmt, d_w, d_h, stride_align, img_data); + * \endcode */ aom_image_t *aom_img_wrap(aom_image_t *img, aom_img_fmt_t fmt, unsigned int d_w, - unsigned int d_h, unsigned int align, + unsigned int d_h, unsigned int stride_align, unsigned char *img_data); /*!\brief Open a descriptor, allocating storage for the underlying image with a
diff --git a/aom/aom_tpl.h b/aom/aom_tpl.h new file mode 100644 index 0000000..60d879e --- /dev/null +++ b/aom/aom_tpl.h
@@ -0,0 +1,93 @@ +/* + * Copyright (c) 2025, Alliance for Open Media. All rights reserved. + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_AOM_AOM_TPL_H_ +#define AOM_AOM_AOM_TPL_H_ + +#include "./aom_integer.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/*!\brief Current ABI version number + * + * \internal + * If this file is altered in any way that changes the ABI, this value + * must be bumped. Examples include, but are not limited to, changing + * types, removing or reassigning enums, adding/removing/rearranging + * fields to structures + */ +#define AOM_TPL_ABI_VERSION 1 /**<\hideinitializer*/ + +#define AOM_RC_INTER_REFS_PER_FRAME 7 + +// The motion vector in units of full pixel +typedef struct aom_fullpel_mv { + int16_t row; + int16_t col; +} AOM_FULLPEL_MV; + +// The motion vector in units of 1/8-pel +typedef struct aom_mv { + int16_t row; + int16_t col; +} AOM_MV; + +typedef union aom_int_mv { + uint32_t as_int; + AOM_MV as_mv; + AOM_FULLPEL_MV as_fullmv; +} aom_int_mv; /* facilitates faster equality tests and copies */ + +/*!\brief Temporal dependency model stats for each block before propagation */ +typedef struct AomTplBlockStats { + int16_t row; /**< Pixel row of the top left corner */ + int16_t col; /**< Pixel col of the top left corner */ + int64_t srcrf_sse; + int64_t srcrf_dist; + int64_t recrf_sse; + int64_t recrf_dist; + int64_t intra_sse; + int64_t intra_dist; + int64_t cmp_recrf_dist[2]; + int64_t mc_dep_rate; + int64_t mc_dep_dist; + int64_t pred_error[AOM_RC_INTER_REFS_PER_FRAME]; + int32_t intra_cost; + int32_t inter_cost; + int32_t srcrf_rate; + int32_t recrf_rate; + int32_t intra_rate; + int32_t cmp_recrf_rate[2]; + aom_int_mv mv[AOM_RC_INTER_REFS_PER_FRAME]; + int8_t ref_frame_index[2]; +} AomTplBlockStats; + +/*!\brief Temporal dependency model stats for each frame before propagation */ +typedef struct AomTplFrameStats { + int frame_width; /**< Frame width */ + int frame_height; /**< Frame height */ + int num_blocks; /**< Number of blocks. Size of block_stats_list */ + AomTplBlockStats *block_stats_list; /**< List of tpl stats for each block */ +} AomTplFrameStats; + +/*!\brief Temporal dependency model stats for each GOP before propagation */ +typedef struct AomTplGopStats { + int size; /**< GOP size, also the size of frame_stats_list. */ + AomTplFrameStats *frame_stats_list; /**< List of tpl stats for each frame */ +} AomTplGopStats; + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_AOM_AOM_TPL_H_
diff --git a/aom/aomcx.h b/aom/aomcx.h index 3dfaa64..70c2dad 100644 --- a/aom/aomcx.h +++ b/aom/aomcx.h
@@ -18,6 +18,7 @@ */ #include "aom/aom.h" #include "aom/aom_encoder.h" +#include "aom/aom_ext_ratectrl.h" #include "aom/aom_external_partition.h" /*!\file @@ -219,7 +220,7 @@ */ AOME_SET_CPUUSED = 13, - /*!\brief Codec control function to enable automatic set and use alf frames, + /*!\brief Codec control function to enable automatic set and use arf frames, * unsigned int parameter * * - 0 = disable @@ -1595,9 +1596,10 @@ /*!\brief Codec control to set the screen content detection mode, * aom_screen_detection_mode parameter. * - * - 1: AOM_SCREEN_DETECTION_STANDARD = standard (default) + * - 1: AOM_SCREEN_DETECTION_STANDARD = standard (default in good quality and + realtime modes) * - 2: AOM_SCREEN_DETECTION_ANTIALIASING_AWARE = anti-aliased text and - * graphics aware + * graphics aware (default in all intra mode) */ AV1E_SET_SCREEN_CONTENT_DETECTION_MODE = 171, @@ -1617,6 +1619,18 @@ */ AV1E_SET_ENABLE_ADAPTIVE_SHARPNESS = 172, + /*!\brief Codec control function to enable external rate control library. + * + * args: a pointer to aom_rc_funcs_t that contains implementation of callbacks + */ + AV1E_SET_EXTERNAL_RATE_CONTROL = 173, + + /*!\brief Codec control function to get GOP structure from the encoder. + * + * args: a pointer to aom_gop_info_t + */ + AV1E_GET_GOP_INFO, + // Any new encoder control IDs should be added above. // Maximum allowed encoder control ID is 229. // No encoder control ID should be added below. @@ -1718,10 +1732,10 @@ * Changes the encoder to tune for certain types of input material. * * \note - * AOM_TUNE_IQ and AOM_TUNE_SSIMULACRA2 are restricted to all intra mode - * (AOM_USAGE_ALL_INTRA). Setting the tuning option to either AOM_TUNE_IQ or - * AOM_TUNE_SSIMULACRA2 causes the following options to be set (expressed as - * command-line options): + * AOM_TUNE_IQ and AOM_TUNE_SSIMULACRA2 are meant for image encoding. Using + * these tuning modes for videos isn't recommended. + * Setting the tuning option to either AOM_TUNE_IQ or AOM_TUNE_SSIMULACRA2 + * causes the following options to be set (expressed as command-line options): * * --enable-qm=1 * * --qm-min=2 * * --qm-max=10 @@ -1746,16 +1760,30 @@ AOM_TUNE_VMAF_SALIENCY_MAP = 9, /*!\brief Allows detection of the presence of AOM_TUNE_IQ at compile time. */ #define AOM_HAVE_TUNE_IQ 1 - /* Image quality (or intra quality). Increases image quality and consistency, + /* "Image Quality" tuning mode. Increases image quality and consistency, * guided by the SSIMULACRA 2 metric and subjective quality checks. Shares * the rdmult code with AOM_TUNE_SSIM. + * Note: AOM_TUNE_IQ is only meant to be used to encode a still image or a + * layered AVIF image. */ AOM_TUNE_IQ = 10, /*!\brief Allows detection of the presence of AOM_TUNE_SSIMULACRA2 at compile * time. */ #define AOM_HAVE_TUNE_SSIMULACRA2 1 - /* Tune that optimizes for maximum SSIMULACRA 2 scores. Shares the rdmult code - with AOM_TUNE_SSIM. */ + /* A tuning mode that optimizes for maximum SSIMULACRA 2 scores. Shares the + * rdmult code with AOM_TUNE_SSIM. + * Unlike metrics like AOM_TUNE_VMAF_* or AOM_TUNE_BUTTERAUGLI, + * AOM_TUNE_SSIMULACRA2 doesn't use the SSIMULACRA 2 metric for + * rate-distortion optimization decisions. Instead, the tuning mode relies + * purely on hand-crafted heuristics. This means no additional external + * dependencies are required. + * AOM_TUNE_SSIMULACRA2 shares most of the tweaks and optimizations with + * AOM_TUNE_IQ. However, AOM_TUNE_SSIMULACRA2 fine-tunes the encoder in ways + * that have been shown to not come with a corresponding positive impact on + * subjective quality in human evaluations. + * Note: AOM_TUNE_SSIMULACRA2 is only meant to be used to encode a still + * image or a layered AVIF image. + */ AOM_TUNE_SSIMULACRA2 = 11, } aom_tune_metric; @@ -1853,6 +1881,20 @@ AOM_FULL_SUPERFRAME_DROP, /**< Only full superframe can drop. */ } AOM_SVC_FRAME_DROP_MODE; +/*!\brief The GOP structure information determined by the encoder. + * 250 is MAX_STATIC_GF_GROUP_LENGTH defined in av1/firstpass.h. + * This is a subset of GF_GROUP. More fields can be added if needed. + */ +typedef struct aom_gop_info { + int gop_size; /**< The number of frames of this GOP */ + /*! The coding index for each entry in the gop */ + int coding_index[250]; + /*! The display index for each entry in the gop */ + int display_index[250]; + /*! The layer depth for each entry in the gop */ + int layer_depth[250]; +} aom_gop_info_t; + /*!\cond */ /*!\brief Encoder control function parameter type * @@ -2360,6 +2402,12 @@ AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_ADAPTIVE_SHARPNESS, unsigned int) #define AOM_CTRL_AV1E_SET_ENABLE_ADAPTIVE_SHARPNESS +AOM_CTRL_USE_TYPE(AV1E_SET_EXTERNAL_RATE_CONTROL, aom_rc_funcs_t *) +#define AOM_CTRL_AV1E_SET_EXTERNAL_RATE_CONTROL + +AOM_CTRL_USE_TYPE(AV1E_GET_GOP_INFO, aom_gop_info_t *) +#define AOM_CTRL_AV1E_GET_GOP_INFO + /*!\endcond */ /*! @} - end defgroup aom_encoder */ #ifdef __cplusplus
diff --git a/aom/aomdx.h b/aom/aomdx.h index f18cbbf..4c32beb 100644 --- a/aom/aomdx.h +++ b/aom/aomdx.h
@@ -466,6 +466,15 @@ * be used. */ AV1D_GET_MI_INFO, + + /*!\brief Codec control function to set the maximum frame size, unsigned int + * parameter + * + * - 0 = unlimited (default) + * - nonzero = frame size (width * height) must be less than or equal to this + * maximum + */ + AOMD_SET_FRAME_SIZE_LIMIT, }; /*!\cond */ @@ -598,6 +607,9 @@ // The AOM_CTRL_USE_TYPE macro can't be used with AV1D_GET_MI_INFO because // AV1D_GET_MI_INFO takes more than one parameter. #define AOM_CTRL_AV1D_GET_MI_INFO + +AOM_CTRL_USE_TYPE(AOMD_SET_FRAME_SIZE_LIMIT, unsigned int) +#define AOM_CTRL_AOMD_SET_FRAME_SIZE_LIMIT /*!\endcond */ /*! @} - end defgroup aom_decoder */ #ifdef __cplusplus
diff --git a/aom/internal/aom_image_internal.h b/aom/internal/aom_image_internal.h index ef0f166..e11223a 100644 --- a/aom/internal/aom_image_internal.h +++ b/aom/internal/aom_image_internal.h
@@ -29,6 +29,14 @@ aom_metadata_t **metadata_array; /* Array of metadata structs */ }; +/*! \brief Bit in aom_metadata_insert_flags marking metadata as layer-specific. + */ +#define AOM_MIF_LAYER_SPECIFIC 0x10 +/*! \brief Bits in aom_metadata_insert_flags used to signal which frames to + * add the metadata to (keyframes, non keyframes...). + */ +#define AOM_MIF_INSERT_LOCATION_MASK 0x0f + /*!\brief Alloc memory for aom_metadata_array struct. * * Allocate memory for aom_metadata_array struct.
diff --git a/aom/src/aom_image.c b/aom/src/aom_image.c index 497dff3..3522cda 100644 --- a/aom/src/aom_image.c +++ b/aom/src/aom_image.c
@@ -18,6 +18,7 @@ #include "aom/aom_integer.h" #include "aom/internal/aom_image_internal.h" #include "aom_mem/aom_mem.h" +#include "aom/aom_codec.h" static inline unsigned int align_image_dimension(unsigned int d, unsigned int subsampling, @@ -383,6 +384,15 @@ img->metadata = aom_img_metadata_array_alloc(0); if (!img->metadata) return -1; } + // Some metadata types are not allowed to be layer specific, according to + // the Table in Section 6.7.1 of the AV1 specifiction. + // Do not check for OBU_METADATA_TYPE_HDR_CLL or OBU_METADATA_TYPE_HDR_MDCV + // because there are plans to alow them to be layer specific. + if ((insert_flag & AOM_MIF_LAYER_SPECIFIC) && + (type == OBU_METADATA_TYPE_SCALABILITY || + type == OBU_METADATA_TYPE_TIMECODE)) { + return -1; + } aom_metadata_t *metadata = aom_img_metadata_alloc(type, data, sz, insert_flag); if (!metadata) return -1;
diff --git a/aom_dsp/aom_dsp.cmake b/aom_dsp/aom_dsp.cmake index 3047bc7..43ffe68 100644 --- a/aom_dsp/aom_dsp.cmake +++ b/aom_dsp/aom_dsp.cmake
@@ -312,6 +312,7 @@ "${AOM_ROOT}/aom_dsp/arm/sad_neon_dotprod.c" "${AOM_ROOT}/aom_dsp/arm/sadxd_neon_dotprod.c" "${AOM_ROOT}/aom_dsp/arm/sse_neon_dotprod.c" + "${AOM_ROOT}/aom_dsp/arm/subpel_variance_neon_dotprod.c" "${AOM_ROOT}/aom_dsp/arm/sum_squares_neon_dotprod.c" "${AOM_ROOT}/aom_dsp/arm/variance_neon_dotprod.c")
diff --git a/aom_dsp/aom_dsp_rtcd_defs.pl b/aom_dsp/aom_dsp_rtcd_defs.pl index 7a5a615..a3b3564 100755 --- a/aom_dsp/aom_dsp_rtcd_defs.pl +++ b/aom_dsp/aom_dsp_rtcd_defs.pl
@@ -1404,23 +1404,23 @@ specialize qw/aom_variance8x8 sse2 neon neon_dotprod/; specialize qw/aom_variance8x4 sse2 neon neon_dotprod/; specialize qw/aom_variance4x8 sse2 neon neon_dotprod/; - specialize qw/aom_variance4x4 sse2 neon neon_dotprod/; + specialize qw/aom_variance4x4 sse2 neon/; - specialize qw/aom_sub_pixel_variance128x128 avx2 neon ssse3/; - specialize qw/aom_sub_pixel_variance128x64 avx2 neon ssse3/; - specialize qw/aom_sub_pixel_variance64x128 avx2 neon ssse3/; - specialize qw/aom_sub_pixel_variance64x64 avx2 neon ssse3/; - specialize qw/aom_sub_pixel_variance64x32 avx2 neon ssse3/; - specialize qw/aom_sub_pixel_variance32x64 avx2 neon ssse3/; - specialize qw/aom_sub_pixel_variance32x32 avx2 neon ssse3/; - specialize qw/aom_sub_pixel_variance32x16 avx2 neon ssse3/; - specialize qw/aom_sub_pixel_variance16x32 avx2 neon ssse3/; - specialize qw/aom_sub_pixel_variance16x16 avx2 neon ssse3/; - specialize qw/aom_sub_pixel_variance16x8 avx2 neon ssse3/; - specialize qw/aom_sub_pixel_variance8x16 neon ssse3/; - specialize qw/aom_sub_pixel_variance8x8 neon ssse3/; - specialize qw/aom_sub_pixel_variance8x4 neon ssse3/; - specialize qw/aom_sub_pixel_variance4x8 neon ssse3/; + specialize qw/aom_sub_pixel_variance128x128 avx2 neon neon_dotprod ssse3/; + specialize qw/aom_sub_pixel_variance128x64 avx2 neon neon_dotprod ssse3/; + specialize qw/aom_sub_pixel_variance64x128 avx2 neon neon_dotprod ssse3/; + specialize qw/aom_sub_pixel_variance64x64 avx2 neon neon_dotprod ssse3/; + specialize qw/aom_sub_pixel_variance64x32 avx2 neon neon_dotprod ssse3/; + specialize qw/aom_sub_pixel_variance32x64 avx2 neon neon_dotprod ssse3/; + specialize qw/aom_sub_pixel_variance32x32 avx2 neon neon_dotprod ssse3/; + specialize qw/aom_sub_pixel_variance32x16 avx2 neon neon_dotprod ssse3/; + specialize qw/aom_sub_pixel_variance16x32 avx2 neon neon_dotprod ssse3/; + specialize qw/aom_sub_pixel_variance16x16 avx2 neon neon_dotprod ssse3/; + specialize qw/aom_sub_pixel_variance16x8 avx2 neon neon_dotprod ssse3/; + specialize qw/aom_sub_pixel_variance8x16 neon neon_dotprod ssse3/; + specialize qw/aom_sub_pixel_variance8x8 neon neon_dotprod ssse3/; + specialize qw/aom_sub_pixel_variance8x4 neon neon_dotprod ssse3/; + specialize qw/aom_sub_pixel_variance4x8 neon neon_dotprod ssse3/; specialize qw/aom_sub_pixel_variance4x4 neon ssse3/; specialize qw/aom_sub_pixel_avg_variance128x128 avx2 neon ssse3/; @@ -1448,12 +1448,12 @@ specialize qw/aom_variance16x64 neon neon_dotprod sse2 avx2/; specialize qw/aom_variance64x16 neon neon_dotprod sse2 avx2/; - specialize qw/aom_sub_pixel_variance4x16 neon ssse3/; - specialize qw/aom_sub_pixel_variance16x4 neon avx2 ssse3/; - specialize qw/aom_sub_pixel_variance8x32 neon ssse3/; - specialize qw/aom_sub_pixel_variance32x8 neon ssse3/; - specialize qw/aom_sub_pixel_variance16x64 neon avx2 ssse3/; - specialize qw/aom_sub_pixel_variance64x16 neon ssse3/; + specialize qw/aom_sub_pixel_variance4x16 neon neon_dotprod ssse3/; + specialize qw/aom_sub_pixel_variance16x4 neon neon_dotprod avx2 ssse3/; + specialize qw/aom_sub_pixel_variance8x32 neon neon_dotprod ssse3/; + specialize qw/aom_sub_pixel_variance32x8 neon neon_dotprod ssse3/; + specialize qw/aom_sub_pixel_variance16x64 neon neon_dotprod avx2 ssse3/; + specialize qw/aom_sub_pixel_variance64x16 neon neon_dotprod ssse3/; specialize qw/aom_sub_pixel_avg_variance4x16 neon ssse3/; specialize qw/aom_sub_pixel_avg_variance16x4 neon ssse3/; specialize qw/aom_sub_pixel_avg_variance8x32 neon ssse3/;
diff --git a/aom_dsp/arm/aom_convolve8_neon_dotprod.c b/aom_dsp/arm/aom_convolve8_neon_dotprod.c index 7fc9cb1..fd86a39 100644 --- a/aom_dsp/arm/aom_convolve8_neon_dotprod.c +++ b/aom_dsp/arm/aom_convolve8_neon_dotprod.c
@@ -25,25 +25,11 @@ #include "aom_dsp/arm/mem_neon.h" #include "aom_dsp/arm/transpose_neon.h" #include "aom_ports/mem.h" +#include "av1/common/arm/convolve_neon_dotprod.h" // Filter values always sum to 128. #define FILTER_WEIGHT 128 -DECLARE_ALIGNED(16, static const uint8_t, kDotProdPermuteTbl[48]) = { - 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6, - 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10, - 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 -}; - -DECLARE_ALIGNED(16, static const uint8_t, kDotProdMergeBlockTbl[48]) = { - // Shift left and insert new last column in transposed 4x4 block. - 1, 2, 3, 16, 5, 6, 7, 20, 9, 10, 11, 24, 13, 14, 15, 28, - // Shift left and insert two new columns in transposed 4x4 block. - 2, 3, 16, 17, 6, 7, 20, 21, 10, 11, 24, 25, 14, 15, 28, 29, - // Shift left and insert three new columns in transposed 4x4 block. - 3, 16, 17, 18, 7, 20, 21, 22, 11, 24, 25, 26, 15, 28, 29, 30 -}; - static inline int16x4_t convolve8_4_h(const uint8x16_t samples, const int8x8_t filters, const uint8x16x2_t permute_tbl) { @@ -58,12 +44,13 @@ vqtbl1q_s8(samples_128, permute_tbl.val[1]) }; // Accumulate into 128 * FILTER_WEIGHT to account for range transform. - int32x4_t acc = vdupq_n_s32(128 * FILTER_WEIGHT); + // (Divide by 2 since we halved the filter values.) + int32x4_t acc = vdupq_n_s32(128 * FILTER_WEIGHT / 2); int32x4_t sum = vdotq_lane_s32(acc, perm_samples[0], filters, 0); sum = vdotq_lane_s32(sum, perm_samples[1], filters, 1); // Further narrowing and packing is performed by the caller. - return vqmovn_s32(sum); + return vmovn_s32(sum); } static inline uint8x8_t convolve8_8_h(const uint8x16_t samples, @@ -82,7 +69,8 @@ vqtbl1q_s8(samples_128, permute_tbl.val[2]) }; // Accumulate into 128 * FILTER_WEIGHT to account for range transform. - int32x4_t acc = vdupq_n_s32(128 * FILTER_WEIGHT); + // (Divide by 2 since we halved the filter values.) + int32x4_t acc = vdupq_n_s32(128 * FILTER_WEIGHT / 2); // First 4 output values. int32x4_t sum0 = vdotq_lane_s32(acc, perm_samples[0], filters, 0); sum0 = vdotq_lane_s32(sum0, perm_samples[1], filters, 1); @@ -91,14 +79,16 @@ sum1 = vdotq_lane_s32(sum1, perm_samples[2], filters, 1); // Narrow and re-pack. - int16x8_t sum = vcombine_s16(vqmovn_s32(sum0), vqmovn_s32(sum1)); - return vqrshrun_n_s16(sum, FILTER_BITS); + int16x8_t sum = vcombine_s16(vmovn_s32(sum0), vmovn_s32(sum1)); + // We halved the filter values so -1 from right shift. + return vqrshrun_n_s16(sum, FILTER_BITS - 1); } static inline void convolve8_horiz_8tap_neon_dotprod( const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int w, int h) { - const int8x8_t filter = vmovn_s16(vld1q_s16(filter_x)); + // Filter values are even, so halve to reduce intermediate precision reqs. + const int8x8_t filter = vshrn_n_s16(vld1q_s16(filter_x), 1); if (w == 4) { const uint8x16x2_t perm_tbl = vld1q_u8_x2(kDotProdPermuteTbl); @@ -110,8 +100,9 @@ int16x4_t d1 = convolve8_4_h(s1, filter, perm_tbl); int16x4_t d2 = convolve8_4_h(s2, filter, perm_tbl); int16x4_t d3 = convolve8_4_h(s3, filter, perm_tbl); - uint8x8_t d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS); - uint8x8_t d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS); + // We halved the filter values so -1 from right shift. + uint8x8_t d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS - 1); + uint8x8_t d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS - 1); store_u8x4_strided_x2(dst + 0 * dst_stride, dst_stride, d01); store_u8x4_strided_x2(dst + 2 * dst_stride, dst_stride, d23); @@ -284,69 +275,19 @@ } } -static inline void transpose_concat_4x4(int8x8_t a0, int8x8_t a1, int8x8_t a2, - int8x8_t a3, int8x16_t *b) { - // Transpose 8-bit elements and concatenate result rows as follows: - // a0: 00, 01, 02, 03, XX, XX, XX, XX - // a1: 10, 11, 12, 13, XX, XX, XX, XX - // a2: 20, 21, 22, 23, XX, XX, XX, XX - // a3: 30, 31, 32, 33, XX, XX, XX, XX - // - // b: 00, 10, 20, 30, 01, 11, 21, 31, 02, 12, 22, 32, 03, 13, 23, 33 - - int8x16_t a0q = vcombine_s8(a0, vdup_n_s8(0)); - int8x16_t a1q = vcombine_s8(a1, vdup_n_s8(0)); - int8x16_t a2q = vcombine_s8(a2, vdup_n_s8(0)); - int8x16_t a3q = vcombine_s8(a3, vdup_n_s8(0)); - - int8x16_t a01 = vzipq_s8(a0q, a1q).val[0]; - int8x16_t a23 = vzipq_s8(a2q, a3q).val[0]; - - int16x8_t a0123 = - vzipq_s16(vreinterpretq_s16_s8(a01), vreinterpretq_s16_s8(a23)).val[0]; - - *b = vreinterpretq_s8_s16(a0123); -} - -static inline void transpose_concat_8x4(int8x8_t a0, int8x8_t a1, int8x8_t a2, - int8x8_t a3, int8x16_t *b0, - int8x16_t *b1) { - // Transpose 8-bit elements and concatenate result rows as follows: - // a0: 00, 01, 02, 03, 04, 05, 06, 07 - // a1: 10, 11, 12, 13, 14, 15, 16, 17 - // a2: 20, 21, 22, 23, 24, 25, 26, 27 - // a3: 30, 31, 32, 33, 34, 35, 36, 37 - // - // b0: 00, 10, 20, 30, 01, 11, 21, 31, 02, 12, 22, 32, 03, 13, 23, 33 - // b1: 04, 14, 24, 34, 05, 15, 25, 35, 06, 16, 26, 36, 07, 17, 27, 37 - - int8x16_t a0q = vcombine_s8(a0, vdup_n_s8(0)); - int8x16_t a1q = vcombine_s8(a1, vdup_n_s8(0)); - int8x16_t a2q = vcombine_s8(a2, vdup_n_s8(0)); - int8x16_t a3q = vcombine_s8(a3, vdup_n_s8(0)); - - int8x16_t a01 = vzipq_s8(a0q, a1q).val[0]; - int8x16_t a23 = vzipq_s8(a2q, a3q).val[0]; - - int16x8x2_t a0123 = - vzipq_s16(vreinterpretq_s16_s8(a01), vreinterpretq_s16_s8(a23)); - - *b0 = vreinterpretq_s8_s16(a0123.val[0]); - *b1 = vreinterpretq_s8_s16(a0123.val[1]); -} - static inline int16x4_t convolve8_4_v(const int8x16_t samples_lo, const int8x16_t samples_hi, const int8x8_t filters) { // The sample range transform and permutation are performed by the caller. // Accumulate into 128 * FILTER_WEIGHT to account for range transform. - int32x4_t acc = vdupq_n_s32(128 * FILTER_WEIGHT); + // (Divide by 2 since we halved the filter values.) + int32x4_t acc = vdupq_n_s32(128 * FILTER_WEIGHT / 2); int32x4_t sum = vdotq_lane_s32(acc, samples_lo, filters, 0); sum = vdotq_lane_s32(sum, samples_hi, filters, 1); // Further narrowing and packing is performed by the caller. - return vqmovn_s32(sum); + return vmovn_s32(sum); } static inline uint8x8_t convolve8_8_v(const int8x16_t samples0_lo, @@ -357,7 +298,8 @@ // The sample range transform and permutation are performed by the caller. // Accumulate into 128 * FILTER_WEIGHT to account for range transform. - int32x4_t acc = vdupq_n_s32(128 * FILTER_WEIGHT); + // (Divide by 2 since we halved the filter values.) + int32x4_t acc = vdupq_n_s32(128 * FILTER_WEIGHT / 2); // First 4 output values. int32x4_t sum0 = vdotq_lane_s32(acc, samples0_lo, filters, 0); sum0 = vdotq_lane_s32(sum0, samples0_hi, filters, 1); @@ -366,14 +308,16 @@ sum1 = vdotq_lane_s32(sum1, samples1_hi, filters, 1); // Narrow and re-pack. - int16x8_t sum = vcombine_s16(vqmovn_s32(sum0), vqmovn_s32(sum1)); - return vqrshrun_n_s16(sum, FILTER_BITS); + int16x8_t sum = vcombine_s16(vmovn_s32(sum0), vmovn_s32(sum1)); + // We halved the filter values so -1 from right shift. + return vqrshrun_n_s16(sum, FILTER_BITS - 1); } static inline void convolve8_vert_8tap_neon_dotprod( const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_y, int w, int h) { - const int8x8_t filter = vmovn_s16(vld1q_s16(filter_y)); + // Filter values are even, so halve to reduce intermediate precision reqs. + const int8x8_t filter = vshrn_n_s16(vld1q_s16(filter_y), 1); const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(kDotProdMergeBlockTbl); int8x16x2_t samples_LUT; @@ -394,10 +338,10 @@ // This operation combines a conventional transpose and the sample permute // (see horizontal case) required before computing the dot product. int8x16_t s0123, s1234, s2345, s3456; - transpose_concat_4x4(s0, s1, s2, s3, &s0123); - transpose_concat_4x4(s1, s2, s3, s4, &s1234); - transpose_concat_4x4(s2, s3, s4, s5, &s2345); - transpose_concat_4x4(s3, s4, s5, s6, &s3456); + transpose_concat_elems_s8_4x4(s0, s1, s2, s3, &s0123); + transpose_concat_elems_s8_4x4(s1, s2, s3, s4, &s1234); + transpose_concat_elems_s8_4x4(s2, s3, s4, s5, &s2345); + transpose_concat_elems_s8_4x4(s3, s4, s5, s6, &s3456); do { uint8x8_t t7, t8, t9, t10; @@ -409,7 +353,7 @@ int8x8_t s10 = vreinterpret_s8_u8(vsub_u8(t10, vdup_n_u8(128))); int8x16_t s4567, s5678, s6789, s78910; - transpose_concat_4x4(s7, s8, s9, s10, &s78910); + transpose_concat_elems_s8_4x4(s7, s8, s9, s10, &s78910); // Merge new data into block from previous iteration. samples_LUT.val[0] = s3456; @@ -422,8 +366,9 @@ int16x4_t d1 = convolve8_4_v(s1234, s5678, filter); int16x4_t d2 = convolve8_4_v(s2345, s6789, filter); int16x4_t d3 = convolve8_4_v(s3456, s78910, filter); - uint8x8_t d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS); - uint8x8_t d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS); + // We halved the filter values so -1 from right shift. + uint8x8_t d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS - 1); + uint8x8_t d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS - 1); store_u8x4_strided_x2(dst + 0 * dst_stride, dst_stride, d01); store_u8x4_strided_x2(dst + 2 * dst_stride, dst_stride, d23); @@ -462,10 +407,10 @@ // (see horizontal case) required before computing the dot product. int8x16_t s0123_lo, s0123_hi, s1234_lo, s1234_hi, s2345_lo, s2345_hi, s3456_lo, s3456_hi; - transpose_concat_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi); - transpose_concat_8x4(s1, s2, s3, s4, &s1234_lo, &s1234_hi); - transpose_concat_8x4(s2, s3, s4, s5, &s2345_lo, &s2345_hi); - transpose_concat_8x4(s3, s4, s5, s6, &s3456_lo, &s3456_hi); + transpose_concat_elems_s8_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi); + transpose_concat_elems_s8_8x4(s1, s2, s3, s4, &s1234_lo, &s1234_hi); + transpose_concat_elems_s8_8x4(s2, s3, s4, s5, &s2345_lo, &s2345_hi); + transpose_concat_elems_s8_8x4(s3, s4, s5, s6, &s3456_lo, &s3456_hi); do { uint8x8_t t7, t8, t9, t10; @@ -478,7 +423,7 @@ int8x16_t s4567_lo, s4567_hi, s5678_lo, s5678_hi, s6789_lo, s6789_hi, s78910_lo, s78910_hi; - transpose_concat_8x4(s7, s8, s9, s10, &s78910_lo, &s78910_hi); + transpose_concat_elems_s8_8x4(s7, s8, s9, s10, &s78910_lo, &s78910_hi); // Merge new data into block from previous iteration. samples_LUT.val[0] = s3456_lo;
diff --git a/aom_dsp/arm/aom_convolve8_neon_i8mm.c b/aom_dsp/arm/aom_convolve8_neon_i8mm.c index 121e892..33694f8 100644 --- a/aom_dsp/arm/aom_convolve8_neon_i8mm.c +++ b/aom_dsp/arm/aom_convolve8_neon_i8mm.c
@@ -24,88 +24,87 @@ #include "aom_dsp/arm/mem_neon.h" #include "aom_dsp/arm/transpose_neon.h" #include "aom_ports/mem.h" - -DECLARE_ALIGNED(16, static const uint8_t, kMatMulPermuteTbl[32]) = { - // clang-format off - 0, 1, 2, 3, 4, 5, 6, 7, 2, 3, 4, 5, 6, 7, 8, 9, - 4, 5, 6, 7, 8, 9, 10, 11, 6, 7, 8, 9, 10, 11, 12, 13 - // clang-format on -}; - -DECLARE_ALIGNED(16, static const uint8_t, kDotProdPermuteTbl[48]) = { - 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6, - 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10, - 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 -}; - -DECLARE_ALIGNED(16, static const uint8_t, kDotProdMergeBlockTbl[48]) = { - // Shift left and insert new last column in transposed 4x4 block. - 1, 2, 3, 16, 5, 6, 7, 20, 9, 10, 11, 24, 13, 14, 15, 28, - // Shift left and insert two new columns in transposed 4x4 block. - 2, 3, 16, 17, 6, 7, 20, 21, 10, 11, 24, 25, 14, 15, 28, 29, - // Shift left and insert three new columns in transposed 4x4 block. - 3, 16, 17, 18, 7, 20, 21, 22, 11, 24, 25, 26, 15, 28, 29, 30 -}; +#include "av1/common/arm/convolve_neon_dotprod.h" +#include "av1/common/arm/convolve_neon_i8mm.h" static inline int16x4_t convolve8_4_h(const uint8x16_t samples, - const int8x8_t filters, - const uint8x16x2_t permute_tbl) { - // Permute samples ready for dot product. - // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } - // { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } - uint8x16_t permuted_samples[2] = { vqtbl1q_u8(samples, permute_tbl.val[0]), - vqtbl1q_u8(samples, permute_tbl.val[1]) }; + const int8x16_t filters, + const uint8x16_t permute_tbl) { + // Permute samples ready for matrix multiply. + // { 1, 2, 3, 4, 5, 6, 7, 8, 3, 4, 5, 6, 7, 8, 9, 10 } + uint8x16_t perm_samples = vqtbl1q_u8(samples, permute_tbl); - int32x4_t sum = - vusdotq_lane_s32(vdupq_n_s32(0), permuted_samples[0], filters, 0); - sum = vusdotq_lane_s32(sum, permuted_samples[1], filters, 1); + // These instructions multiply a 2x8 matrix (samples) by an 8x2 matrix + // (filter), destructively accumulating into the destination register. + int32x4_t sum = vusmmlaq_s32(vdupq_n_s32(0), perm_samples, filters); - // Further narrowing and packing is performed by the caller. - return vqmovn_s32(sum); + // Tap 0, as well as further narrowing and packing, is applied by the caller. + return vmovn_s32(sum); } static inline uint8x8_t convolve8_8_h(const uint8x16_t samples, - const int8x8_t filters, - const uint8x16x3_t permute_tbl) { - // Permute samples ready for dot product. - // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } - // { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } - // { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 } - uint8x16_t permuted_samples[3] = { vqtbl1q_u8(samples, permute_tbl.val[0]), - vqtbl1q_u8(samples, permute_tbl.val[1]), - vqtbl1q_u8(samples, permute_tbl.val[2]) }; + const int8x16_t filters, + const uint8x8_t f0, + const uint8x16x2_t permute_tbl) { + // Permute samples ready for matrix multiply. + // { 1, 2, 3, 4, 5, 6, 7, 8, 3, 4, 5, 6, 7, 8, 9, 10 } + // { 5, 6, 7, 8, 9, 10, 11, 12, 7, 8, 9, 10, 11, 12, 13, 14 } + uint8x16_t perm_samples[2] = { vqtbl1q_u8(samples, permute_tbl.val[0]), + vqtbl1q_u8(samples, permute_tbl.val[1]) }; - // First 4 output values. - int32x4_t sum0 = - vusdotq_lane_s32(vdupq_n_s32(0), permuted_samples[0], filters, 0); - sum0 = vusdotq_lane_s32(sum0, permuted_samples[1], filters, 1); - // Second 4 output values. - int32x4_t sum1 = - vusdotq_lane_s32(vdupq_n_s32(0), permuted_samples[1], filters, 0); - sum1 = vusdotq_lane_s32(sum1, permuted_samples[2], filters, 1); + // These instructions multiply a 2x8 matrix (samples) by an 8x2 matrix + // (filter), destructively accumulating into the destination register. + int32x4_t sum0123 = vusmmlaq_s32(vdupq_n_s32(0), perm_samples[0], filters); + int32x4_t sum4567 = vusmmlaq_s32(vdupq_n_s32(0), perm_samples[1], filters); // Narrow and re-pack. - int16x8_t sum = vcombine_s16(vqmovn_s32(sum0), vqmovn_s32(sum1)); - return vqrshrun_n_s16(sum, FILTER_BITS); + int16x8_t sum = vcombine_s16(vmovn_s32(sum0123), vmovn_s32(sum4567)); + // Apply tap 0 and accumulate. + sum = vreinterpretq_s16_u16( + vmlsl_u8(vreinterpretq_u16_s16(sum), vget_low_u8(samples), f0)); + + // We halved the filter values so -1 from right shift. + return vqrshrun_n_s16(sum, FILTER_BITS - 1); } static inline void convolve8_horiz_8tap_neon_i8mm( const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int w, int h) { - const int8x8_t filter = vmovn_s16(vld1q_s16(filter_x)); + // Filter values are even, so halve to reduce intermediate precision reqs. + const int8x8_t filter_s8 = vshrn_n_s16(vld1q_s16(filter_x), 1); + // Stagger the filter for use with the matrix multiply instructions. + // { f1, f2, f3, f4, f5, f6, f7, 0, 0, f1, f2, f3, f4, f5, f6, f7 } + const uint8x16_t filter_idx = vld1q_u8(kFilterPermuteTbl); + const int8x16_t filter = + vqtbl1q_s8(vcombine_s8(filter_s8, vdup_n_s8(0)), filter_idx); + + // Since f0 is always negative and samples are unsigned, subtract (unsigned) + // s0 * -f0 to avoid signed overflow. + const uint8x8_t f0 = vdup_n_u8(-filter_x[0] >> 1); if (w == 4) { - const uint8x16x2_t perm_tbl = vld1q_u8_x2(kDotProdPermuteTbl); + const uint8x16_t perm_tbl = vld1q_u8(kMatMul8PermuteTbl); + do { uint8x16_t s0, s1, s2, s3; load_u8_16x4(src, src_stride, &s0, &s1, &s2, &s3); + uint8x8_t s01 = load_u8_4x2(src + 0 * src_stride, src_stride); + uint8x8_t s23 = load_u8_4x2(src + 2 * src_stride, src_stride); - int16x4_t d0 = convolve8_4_h(s0, filter, perm_tbl); - int16x4_t d1 = convolve8_4_h(s1, filter, perm_tbl); - int16x4_t d2 = convolve8_4_h(s2, filter, perm_tbl); - int16x4_t d3 = convolve8_4_h(s3, filter, perm_tbl); - uint8x8_t d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS); - uint8x8_t d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS); + int16x4_t t0 = convolve8_4_h(s0, filter, perm_tbl); + int16x4_t t1 = convolve8_4_h(s1, filter, perm_tbl); + int16x4_t t2 = convolve8_4_h(s2, filter, perm_tbl); + int16x4_t t3 = convolve8_4_h(s3, filter, perm_tbl); + // Apply tap 0 and accumulate. + int16x8_t t01 = vcombine_s16(t0, t1); + int16x8_t t23 = vcombine_s16(t2, t3); + t01 = + vreinterpretq_s16_u16(vmlsl_u8(vreinterpretq_u16_s16(t01), s01, f0)); + t23 = + vreinterpretq_s16_u16(vmlsl_u8(vreinterpretq_u16_s16(t23), s23, f0)); + // We halved the filter values to -1 from right shift. + uint8x8_t d01 = vqrshrun_n_s16(t01, FILTER_BITS - 1); + uint8x8_t d23 = vqrshrun_n_s16(t23, FILTER_BITS - 1); store_u8x4_strided_x2(dst + 0 * dst_stride, dst_stride, d01); store_u8x4_strided_x2(dst + 2 * dst_stride, dst_stride, d23); @@ -115,7 +114,7 @@ h -= 4; } while (h > 0); } else { - const uint8x16x3_t perm_tbl = vld1q_u8_x3(kDotProdPermuteTbl); + const uint8x16x2_t perm_tbl = vld1q_u8_x2(kMatMul8PermuteTbl); do { int width = w; @@ -125,10 +124,10 @@ uint8x16_t s0, s1, s2, s3; load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3); - uint8x8_t d0 = convolve8_8_h(s0, filter, perm_tbl); - uint8x8_t d1 = convolve8_8_h(s1, filter, perm_tbl); - uint8x8_t d2 = convolve8_8_h(s2, filter, perm_tbl); - uint8x8_t d3 = convolve8_8_h(s3, filter, perm_tbl); + uint8x8_t d0 = convolve8_8_h(s0, filter, f0, perm_tbl); + uint8x8_t d1 = convolve8_8_h(s1, filter, f0, perm_tbl); + uint8x8_t d2 = convolve8_8_h(s2, filter, f0, perm_tbl); + uint8x8_t d3 = convolve8_8_h(s3, filter, f0, perm_tbl); store_u8_8x4(d, dst_stride, d0, d1, d2, d3); @@ -189,7 +188,7 @@ vcombine_s8(vext_s8(x_filter, x_filter, 1), x_filter); if (width == 4) { - const uint8x16_t perm_tbl = vld1q_u8(kMatMulPermuteTbl); + const uint8x16_t perm_tbl = vld1q_u8(kMatMul6PermuteTbl); do { uint8x16_t s0, s1, s2, s3; load_u8_16x4(src, src_stride, &s0, &s1, &s2, &s3); @@ -210,7 +209,7 @@ height -= 4; } while (height > 0); } else { - const uint8x16x2_t perm_tbl = vld1q_u8_x2(kMatMulPermuteTbl); + const uint8x16x2_t perm_tbl = vld1q_u8_x2(kMatMul6PermuteTbl); do { int w = width; @@ -266,58 +265,6 @@ } } -static inline void transpose_concat_4x4(uint8x8_t a0, uint8x8_t a1, - uint8x8_t a2, uint8x8_t a3, - uint8x16_t *b) { - // Transpose 8-bit elements and concatenate result rows as follows: - // a0: 00, 01, 02, 03, XX, XX, XX, XX - // a1: 10, 11, 12, 13, XX, XX, XX, XX - // a2: 20, 21, 22, 23, XX, XX, XX, XX - // a3: 30, 31, 32, 33, XX, XX, XX, XX - // - // b: 00, 10, 20, 30, 01, 11, 21, 31, 02, 12, 22, 32, 03, 13, 23, 33 - - uint8x16_t a0q = vcombine_u8(a0, vdup_n_u8(0)); - uint8x16_t a1q = vcombine_u8(a1, vdup_n_u8(0)); - uint8x16_t a2q = vcombine_u8(a2, vdup_n_u8(0)); - uint8x16_t a3q = vcombine_u8(a3, vdup_n_u8(0)); - - uint8x16_t a01 = vzipq_u8(a0q, a1q).val[0]; - uint8x16_t a23 = vzipq_u8(a2q, a3q).val[0]; - - uint16x8_t a0123 = - vzipq_u16(vreinterpretq_u16_u8(a01), vreinterpretq_u16_u8(a23)).val[0]; - - *b = vreinterpretq_u8_u16(a0123); -} - -static inline void transpose_concat_8x4(uint8x8_t a0, uint8x8_t a1, - uint8x8_t a2, uint8x8_t a3, - uint8x16_t *b0, uint8x16_t *b1) { - // Transpose 8-bit elements and concatenate result rows as follows: - // a0: 00, 01, 02, 03, 04, 05, 06, 07 - // a1: 10, 11, 12, 13, 14, 15, 16, 17 - // a2: 20, 21, 22, 23, 24, 25, 26, 27 - // a3: 30, 31, 32, 33, 34, 35, 36, 37 - // - // b0: 00, 10, 20, 30, 01, 11, 21, 31, 02, 12, 22, 32, 03, 13, 23, 33 - // b1: 04, 14, 24, 34, 05, 15, 25, 35, 06, 16, 26, 36, 07, 17, 27, 37 - - uint8x16_t a0q = vcombine_u8(a0, vdup_n_u8(0)); - uint8x16_t a1q = vcombine_u8(a1, vdup_n_u8(0)); - uint8x16_t a2q = vcombine_u8(a2, vdup_n_u8(0)); - uint8x16_t a3q = vcombine_u8(a3, vdup_n_u8(0)); - - uint8x16_t a01 = vzipq_u8(a0q, a1q).val[0]; - uint8x16_t a23 = vzipq_u8(a2q, a3q).val[0]; - - uint16x8x2_t a0123 = - vzipq_u16(vreinterpretq_u16_u8(a01), vreinterpretq_u16_u8(a23)); - - *b0 = vreinterpretq_u8_u16(a0123.val[0]); - *b1 = vreinterpretq_u8_u16(a0123.val[1]); -} - static inline int16x4_t convolve8_4_v(const uint8x16_t samples_lo, const uint8x16_t samples_hi, const int8x8_t filters) { @@ -326,7 +273,7 @@ sum = vusdotq_lane_s32(sum, samples_hi, filters, 1); // Further narrowing and packing is performed by the caller. - return vqmovn_s32(sum); + return vmovn_s32(sum); } static inline uint8x8_t convolve8_8_v(const uint8x16_t samples0_lo, @@ -344,14 +291,16 @@ sum1 = vusdotq_lane_s32(sum1, samples1_hi, filters, 1); // Narrow and re-pack. - int16x8_t sum = vcombine_s16(vqmovn_s32(sum0), vqmovn_s32(sum1)); - return vqrshrun_n_s16(sum, FILTER_BITS); + int16x8_t sum = vcombine_s16(vmovn_s32(sum0), vmovn_s32(sum1)); + // We halved the filter values so -1 from right shift. + return vqrshrun_n_s16(sum, FILTER_BITS - 1); } static inline void convolve8_vert_8tap_neon_i8mm( const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_y, int w, int h) { - const int8x8_t filter = vmovn_s16(vld1q_s16(filter_y)); + // Filter values are even, so halve to reduce intermediate precision reqs. + const int8x8_t filter = vshrn_n_s16(vld1q_s16(filter_y), 1); const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(kDotProdMergeBlockTbl); uint8x16x2_t samples_LUT; @@ -361,19 +310,19 @@ src += 7 * src_stride; // This operation combines a conventional transpose and the sample permute - // (see horizontal case) required before computing the dot product. + // required before computing the dot product. uint8x16_t s0123, s1234, s2345, s3456; - transpose_concat_4x4(s0, s1, s2, s3, &s0123); - transpose_concat_4x4(s1, s2, s3, s4, &s1234); - transpose_concat_4x4(s2, s3, s4, s5, &s2345); - transpose_concat_4x4(s3, s4, s5, s6, &s3456); + transpose_concat_elems_u8_4x4(s0, s1, s2, s3, &s0123); + transpose_concat_elems_u8_4x4(s1, s2, s3, s4, &s1234); + transpose_concat_elems_u8_4x4(s2, s3, s4, s5, &s2345); + transpose_concat_elems_u8_4x4(s3, s4, s5, s6, &s3456); do { uint8x8_t s7, s8, s9, s10; load_u8_8x4(src, src_stride, &s7, &s8, &s9, &s10); uint8x16_t s4567, s5678, s6789, s78910; - transpose_concat_4x4(s7, s8, s9, s10, &s78910); + transpose_concat_elems_u8_4x4(s7, s8, s9, s10, &s78910); // Merge new data into block from previous iteration. samples_LUT.val[0] = s3456; @@ -386,8 +335,9 @@ int16x4_t d1 = convolve8_4_v(s1234, s5678, filter); int16x4_t d2 = convolve8_4_v(s2345, s6789, filter); int16x4_t d3 = convolve8_4_v(s3456, s78910, filter); - uint8x8_t d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS); - uint8x8_t d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS); + // We halved the filter values so -1 from right shift. + uint8x8_t d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS - 1); + uint8x8_t d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS - 1); store_u8x4_strided_x2(dst + 0 * dst_stride, dst_stride, d01); store_u8x4_strided_x2(dst + 2 * dst_stride, dst_stride, d23); @@ -414,13 +364,13 @@ s += 7 * src_stride; // This operation combines a conventional transpose and the sample permute - // (see horizontal case) required before computing the dot product. + // required before computing the dot product. uint8x16_t s0123_lo, s0123_hi, s1234_lo, s1234_hi, s2345_lo, s2345_hi, s3456_lo, s3456_hi; - transpose_concat_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi); - transpose_concat_8x4(s1, s2, s3, s4, &s1234_lo, &s1234_hi); - transpose_concat_8x4(s2, s3, s4, s5, &s2345_lo, &s2345_hi); - transpose_concat_8x4(s3, s4, s5, s6, &s3456_lo, &s3456_hi); + transpose_concat_elems_u8_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi); + transpose_concat_elems_u8_8x4(s1, s2, s3, s4, &s1234_lo, &s1234_hi); + transpose_concat_elems_u8_8x4(s2, s3, s4, s5, &s2345_lo, &s2345_hi); + transpose_concat_elems_u8_8x4(s3, s4, s5, s6, &s3456_lo, &s3456_hi); do { uint8x8_t s7, s8, s9, s10; @@ -428,7 +378,7 @@ uint8x16_t s4567_lo, s4567_hi, s5678_lo, s5678_hi, s6789_lo, s6789_hi, s78910_lo, s78910_hi; - transpose_concat_8x4(s7, s8, s9, s10, &s78910_lo, &s78910_hi); + transpose_concat_elems_u8_8x4(s7, s8, s9, s10, &s78910_lo, &s78910_hi); // Merge new data into block from previous iteration. samples_LUT.val[0] = s3456_lo; @@ -476,6 +426,142 @@ } } +static inline int16x4_t convolve4_4_v(const uint8x16_t samples, + const int8x8_t filters) { + // Sample permutation is performed by the caller. + int32x4_t sum = vusdotq_lane_s32(vdupq_n_s32(0), samples, filters, 0); + + // Further narrowing and packing is performed by the caller. + return vmovn_s32(sum); +} + +static inline uint8x8_t convolve4_8_v(const uint8x16_t samples0, + const uint8x16_t samples1, + const int8x8_t filters) { + // Sample permutation is performed by the caller. + + // First 4 output values. + int32x4_t sum0 = vusdotq_lane_s32(vdupq_n_s32(0), samples0, filters, 0); + // Second 4 output values. + int32x4_t sum1 = vusdotq_lane_s32(vdupq_n_s32(0), samples1, filters, 0); + + // Narrow and re-pack. + int16x8_t sum = vcombine_s16(vmovn_s32(sum0), vmovn_s32(sum1)); + // We halved the filter values so -1 from right shift. + return vqrshrun_n_s16(sum, FILTER_BITS - 1); +} + +static inline void convolve8_vert_4tap_neon_i8mm( + const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, + ptrdiff_t dst_stride, const int16_t *filter_y, int w, int h) { + // Filter values are even, so halve to reduce intermediate precision reqs. + const int16x8_t filter_s16 = + vcombine_s16(vld1_s16(filter_y + 2), vdup_n_s16(0)); + const int8x8_t filter = vshrn_n_s16(filter_s16, 1); + const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(kDotProdMergeBlockTbl); + uint8x16x2_t samples_LUT; + + if (w == 4) { + uint8x8_t s0, s1, s2, s3; + load_u8_8x4(src, src_stride, &s0, &s1, &s2, &s3); + src += 4 * src_stride; + + // This operation combines a conventional transpose and the sample permute + // required before computing the dot product. + uint8x16_t s0123; + transpose_concat_elems_u8_4x4(s0, s1, s2, s3, &s0123); + + do { + uint8x8_t s4, s5, s6, s7; + load_u8_8x4(src, src_stride, &s4, &s5, &s6, &s7); + + uint8x16_t s4567; + transpose_concat_elems_u8_4x4(s4, s5, s6, s7, &s4567); + + // Merge new data into block from previous iteration. + samples_LUT.val[0] = s0123; + samples_LUT.val[1] = s4567; + uint8x16_t s1234 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[0]); + uint8x16_t s2345 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[1]); + uint8x16_t s3456 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[2]); + + int16x4_t d0 = convolve4_4_v(s0123, filter); + int16x4_t d1 = convolve4_4_v(s1234, filter); + int16x4_t d2 = convolve4_4_v(s2345, filter); + int16x4_t d3 = convolve4_4_v(s3456, filter); + // We halved the filter values so -1 from right shift. + uint8x8_t d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS - 1); + uint8x8_t d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS - 1); + + store_u8x4_strided_x2(dst + 0 * dst_stride, dst_stride, d01); + store_u8x4_strided_x2(dst + 2 * dst_stride, dst_stride, d23); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123 = s4567; + + src += 4 * src_stride; + dst += 4 * dst_stride; + h -= 4; + } while (h != 0); + } else { + do { + int height = h; + const uint8_t *s = src; + uint8_t *d = dst; + + uint8x8_t s0, s1, s2, s3; + load_u8_8x4(s, src_stride, &s0, &s1, &s2, &s3); + s += 4 * src_stride; + + // This operation combines a conventional transpose and the sample permute + // required before computing the dot product. + uint8x16_t s0123_lo, s0123_hi; + transpose_concat_elems_u8_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi); + + do { + uint8x8_t s4, s5, s6, s7; + load_u8_8x4(s, src_stride, &s4, &s5, &s6, &s7); + + uint8x16_t s4567_lo, s4567_hi; + transpose_concat_elems_u8_8x4(s4, s5, s6, s7, &s4567_lo, &s4567_hi); + + // Merge new data into block from previous iteration. + samples_LUT.val[0] = s0123_lo; + samples_LUT.val[1] = s4567_lo; + uint8x16_t s1234_lo = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[0]); + uint8x16_t s2345_lo = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[1]); + uint8x16_t s3456_lo = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[2]); + + samples_LUT.val[0] = s0123_hi; + samples_LUT.val[1] = s4567_hi; + uint8x16_t s1234_hi = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[0]); + uint8x16_t s2345_hi = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[1]); + uint8x16_t s3456_hi = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[2]); + + uint8x8_t d0 = convolve4_8_v(s0123_lo, s0123_hi, filter); + uint8x8_t d1 = convolve4_8_v(s1234_lo, s1234_hi, filter); + uint8x8_t d2 = convolve4_8_v(s2345_lo, s2345_hi, filter); + uint8x8_t d3 = convolve4_8_v(s3456_lo, s3456_hi, filter); + + store_u8_8x4(d, dst_stride, d0, d1, d2, d3); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123_lo = s4567_lo; + s0123_hi = s4567_hi; + + s += 4 * src_stride; + d += 4 * dst_stride; + height -= 4; + } while (height != 0); + src += 8; + dst += 8; + w -= 8; + } while (w != 0); + } +} + void aom_convolve8_vert_neon_i8mm(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, @@ -496,8 +582,8 @@ convolve8_vert_2tap_neon(src + 3 * src_stride, src_stride, dst, dst_stride, filter_y, w, h); } else if (filter_taps == 4) { - convolve8_vert_4tap_neon(src + 2 * src_stride, src_stride, dst, dst_stride, - filter_y, w, h); + convolve8_vert_4tap_neon_i8mm(src + 2 * src_stride, src_stride, dst, + dst_stride, filter_y, w, h); } else { convolve8_vert_8tap_neon_i8mm(src, src_stride, dst, dst_stride, filter_y, w, h);
diff --git a/aom_dsp/arm/highbd_convolve8_sve.c b/aom_dsp/arm/highbd_convolve8_sve.c index b5db14b..8af5c5e 100644 --- a/aom_dsp/arm/highbd_convolve8_sve.c +++ b/aom_dsp/arm/highbd_convolve8_sve.c
@@ -20,6 +20,18 @@ #include "aom_dsp/arm/aom_filter.h" #include "aom_dsp/arm/highbd_convolve8_neon.h" #include "aom_dsp/arm/mem_neon.h" +#include "aom_dsp/arm/transpose_neon.h" + +DECLARE_ALIGNED(16, const uint16_t, kHbdDotProdTbl[32]) = { + 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6, + 4, 5, 6, 7, 5, 6, 7, 0, 6, 7, 0, 1, 7, 0, 1, 2, +}; + +// clang-format off +DECLARE_ALIGNED(16, const uint16_t, kDeinterleaveTbl[8]) = { + 0, 2, 4, 6, 1, 3, 5, 7, +}; +// clang-format on static inline uint16x4_t highbd_convolve8_4_h(int16x8_t s[4], int16x8_t filter, uint16x4_t max) { @@ -130,16 +142,6 @@ } } -// clang-format off -DECLARE_ALIGNED(16, static const uint16_t, kDotProdTbl[16]) = { - 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6, -}; - -DECLARE_ALIGNED(16, static const uint16_t, kDeinterleaveTbl[8]) = { - 0, 2, 4, 6, 1, 3, 5, 7, -}; -// clang-format on - static inline uint16x4_t highbd_convolve4_4_h(int16x8_t s, int16x8_t filter, uint16x8x2_t permute_tbl, uint16x4_t max) { @@ -183,7 +185,7 @@ if (width == 4) { const uint16x4_t max = vdup_n_u16((1 << bd) - 1); - uint16x8x2_t permute_tbl = vld1q_u16_x2(kDotProdTbl); + uint16x8x2_t permute_tbl = vld1q_u16_x2(kHbdDotProdTbl); const int16_t *s = (const int16_t *)src; uint16_t *d = dst; @@ -267,97 +269,6 @@ } } -DECLARE_ALIGNED(16, static const uint8_t, kDotProdMergeBlockTbl[48]) = { - // Shift left and insert new last column in transposed 4x4 block. - 2, 3, 4, 5, 6, 7, 16, 17, 10, 11, 12, 13, 14, 15, 24, 25, - // Shift left and insert two new columns in transposed 4x4 block. - 4, 5, 6, 7, 16, 17, 18, 19, 12, 13, 14, 15, 24, 25, 26, 27, - // Shift left and insert three new columns in transposed 4x4 block. - 6, 7, 16, 17, 18, 19, 20, 21, 14, 15, 24, 25, 26, 27, 28, 29 -}; - -static inline void transpose_concat_4x4(int16x4_t s0, int16x4_t s1, - int16x4_t s2, int16x4_t s3, - int16x8_t res[2]) { - // Transpose 16-bit elements and concatenate result rows as follows: - // s0: 00, 01, 02, 03 - // s1: 10, 11, 12, 13 - // s2: 20, 21, 22, 23 - // s3: 30, 31, 32, 33 - // - // res[0]: 00 10 20 30 01 11 21 31 - // res[1]: 02 12 22 32 03 13 23 33 - - int16x8_t s0q = vcombine_s16(s0, vdup_n_s16(0)); - int16x8_t s1q = vcombine_s16(s1, vdup_n_s16(0)); - int16x8_t s2q = vcombine_s16(s2, vdup_n_s16(0)); - int16x8_t s3q = vcombine_s16(s3, vdup_n_s16(0)); - - int32x4_t s01 = vreinterpretq_s32_s16(vzip1q_s16(s0q, s1q)); - int32x4_t s23 = vreinterpretq_s32_s16(vzip1q_s16(s2q, s3q)); - - int32x4x2_t s0123 = vzipq_s32(s01, s23); - - res[0] = vreinterpretq_s16_s32(s0123.val[0]); - res[1] = vreinterpretq_s16_s32(s0123.val[1]); -} - -static inline void transpose_concat_8x4(int16x8_t s0, int16x8_t s1, - int16x8_t s2, int16x8_t s3, - int16x8_t res[4]) { - // Transpose 16-bit elements and concatenate result rows as follows: - // s0: 00, 01, 02, 03, 04, 05, 06, 07 - // s1: 10, 11, 12, 13, 14, 15, 16, 17 - // s2: 20, 21, 22, 23, 24, 25, 26, 27 - // s3: 30, 31, 32, 33, 34, 35, 36, 37 - // - // res_lo[0]: 00 10 20 30 01 11 21 31 - // res_lo[1]: 02 12 22 32 03 13 23 33 - // res_hi[0]: 04 14 24 34 05 15 25 35 - // res_hi[1]: 06 16 26 36 07 17 27 37 - - int16x8x2_t tr01_16 = vzipq_s16(s0, s1); - int16x8x2_t tr23_16 = vzipq_s16(s2, s3); - - int32x4x2_t tr01_32 = vzipq_s32(vreinterpretq_s32_s16(tr01_16.val[0]), - vreinterpretq_s32_s16(tr23_16.val[0])); - int32x4x2_t tr23_32 = vzipq_s32(vreinterpretq_s32_s16(tr01_16.val[1]), - vreinterpretq_s32_s16(tr23_16.val[1])); - - res[0] = vreinterpretq_s16_s32(tr01_32.val[0]); - res[1] = vreinterpretq_s16_s32(tr01_32.val[1]); - res[2] = vreinterpretq_s16_s32(tr23_32.val[0]); - res[3] = vreinterpretq_s16_s32(tr23_32.val[1]); -} - -static inline void aom_tbl2x4_s16(int16x8_t t0[4], int16x8_t t1[4], - uint8x16_t tbl, int16x8_t res[4]) { - int8x16x2_t samples0 = { vreinterpretq_s8_s16(t0[0]), - vreinterpretq_s8_s16(t1[0]) }; - int8x16x2_t samples1 = { vreinterpretq_s8_s16(t0[1]), - vreinterpretq_s8_s16(t1[1]) }; - int8x16x2_t samples2 = { vreinterpretq_s8_s16(t0[2]), - vreinterpretq_s8_s16(t1[2]) }; - int8x16x2_t samples3 = { vreinterpretq_s8_s16(t0[3]), - vreinterpretq_s8_s16(t1[3]) }; - - res[0] = vreinterpretq_s16_s8(vqtbl2q_s8(samples0, tbl)); - res[1] = vreinterpretq_s16_s8(vqtbl2q_s8(samples1, tbl)); - res[2] = vreinterpretq_s16_s8(vqtbl2q_s8(samples2, tbl)); - res[3] = vreinterpretq_s16_s8(vqtbl2q_s8(samples3, tbl)); -} - -static inline void aom_tbl2x2_s16(int16x8_t t0[2], int16x8_t t1[2], - uint8x16_t tbl, int16x8_t res[2]) { - int8x16x2_t samples0 = { vreinterpretq_s8_s16(t0[0]), - vreinterpretq_s8_s16(t1[0]) }; - int8x16x2_t samples1 = { vreinterpretq_s8_s16(t0[1]), - vreinterpretq_s8_s16(t1[1]) }; - - res[0] = vreinterpretq_s16_s8(vqtbl2q_s8(samples0, tbl)); - res[1] = vreinterpretq_s16_s8(vqtbl2q_s8(samples1, tbl)); -} - static inline uint16x4_t highbd_convolve8_4_v(int16x8_t samples_lo[2], int16x8_t samples_hi[2], int16x8_t filter, @@ -377,47 +288,17 @@ return vmin_u16(res, max); } -static inline uint16x8_t highbd_convolve8_8_v(int16x8_t samples_lo[4], - int16x8_t samples_hi[4], - int16x8_t filter, - uint16x8_t max) { - int64x2_t sum[4]; - - sum[0] = aom_svdot_lane_s16(vdupq_n_s64(0), samples_lo[0], filter, 0); - sum[0] = aom_svdot_lane_s16(sum[0], samples_hi[0], filter, 1); - - sum[1] = aom_svdot_lane_s16(vdupq_n_s64(0), samples_lo[1], filter, 0); - sum[1] = aom_svdot_lane_s16(sum[1], samples_hi[1], filter, 1); - - sum[2] = aom_svdot_lane_s16(vdupq_n_s64(0), samples_lo[2], filter, 0); - sum[2] = aom_svdot_lane_s16(sum[2], samples_hi[2], filter, 1); - - sum[3] = aom_svdot_lane_s16(vdupq_n_s64(0), samples_lo[3], filter, 0); - sum[3] = aom_svdot_lane_s16(sum[3], samples_hi[3], filter, 1); - - int32x4_t res0 = vcombine_s32(vmovn_s64(sum[0]), vmovn_s64(sum[1])); - int32x4_t res1 = vcombine_s32(vmovn_s64(sum[2]), vmovn_s64(sum[3])); - - uint16x8_t res = vcombine_u16(vqrshrun_n_s32(res0, FILTER_BITS), - vqrshrun_n_s32(res1, FILTER_BITS)); - - return vminq_u16(res, max); -} - static inline void highbd_convolve8_vert_8tap_sve( const uint16_t *src, ptrdiff_t src_stride, uint16_t *dst, ptrdiff_t dst_stride, const int16_t *filter_y, int width, int height, int bd) { const int16x8_t y_filter = vld1q_s16(filter_y); - uint8x16_t merge_block_tbl[3]; - merge_block_tbl[0] = vld1q_u8(kDotProdMergeBlockTbl); - merge_block_tbl[1] = vld1q_u8(kDotProdMergeBlockTbl + 16); - merge_block_tbl[2] = vld1q_u8(kDotProdMergeBlockTbl + 32); - - if (width == 4) { + do { const uint16x4_t max = vdup_n_u16((1 << bd) - 1); int16_t *s = (int16_t *)src; + uint16_t *d = dst; + int h = height; int16x4_t s0, s1, s2, s3, s4, s5, s6; load_s16_4x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6); @@ -426,10 +307,10 @@ // This operation combines a conventional transpose and the sample permute // required before computing the dot product. int16x8_t s0123[2], s1234[2], s2345[2], s3456[2]; - transpose_concat_4x4(s0, s1, s2, s3, s0123); - transpose_concat_4x4(s1, s2, s3, s4, s1234); - transpose_concat_4x4(s2, s3, s4, s5, s2345); - transpose_concat_4x4(s3, s4, s5, s6, s3456); + transpose_concat_elems_s16_4x4(s0, s1, s2, s3, s0123); + transpose_concat_elems_s16_4x4(s1, s2, s3, s4, s1234); + transpose_concat_elems_s16_4x4(s2, s3, s4, s5, s2345); + transpose_concat_elems_s16_4x4(s3, s4, s5, s6, s3456); do { int16x4_t s7, s8, s9, s10; @@ -438,19 +319,17 @@ int16x8_t s4567[2], s5678[2], s6789[2], s78910[2]; // Transpose and shuffle the 4 lines that were loaded. - transpose_concat_4x4(s7, s8, s9, s10, s78910); - - // Merge new data into block from previous iteration. - aom_tbl2x2_s16(s3456, s78910, merge_block_tbl[0], s4567); - aom_tbl2x2_s16(s3456, s78910, merge_block_tbl[1], s5678); - aom_tbl2x2_s16(s3456, s78910, merge_block_tbl[2], s6789); + transpose_concat_elems_s16_4x4(s4, s5, s6, s7, s4567); + transpose_concat_elems_s16_4x4(s5, s6, s7, s8, s5678); + transpose_concat_elems_s16_4x4(s6, s7, s8, s9, s6789); + transpose_concat_elems_s16_4x4(s7, s8, s9, s10, s78910); uint16x4_t d0 = highbd_convolve8_4_v(s0123, s4567, y_filter, max); uint16x4_t d1 = highbd_convolve8_4_v(s1234, s5678, y_filter, max); uint16x4_t d2 = highbd_convolve8_4_v(s2345, s6789, y_filter, max); uint16x4_t d3 = highbd_convolve8_4_v(s3456, s78910, y_filter, max); - store_u16_4x4(dst, dst_stride, d0, d1, d2, d3); + store_u16_4x4(d, dst_stride, d0, d1, d2, d3); // Prepare block for next iteration - re-using as much as possible. // Shuffle everything up four rows. @@ -463,81 +342,18 @@ s3456[0] = s78910[0]; s3456[1] = s78910[1]; + s4 = s8; + s5 = s9; + s6 = s10; + s += 4 * src_stride; - dst += 4 * dst_stride; - height -= 4; - } while (height != 0); - } else { - const uint16x8_t max = vdupq_n_u16((1 << bd) - 1); - do { - int h = height; - int16_t *s = (int16_t *)src; - uint16_t *d = dst; - - int16x8_t s0, s1, s2, s3, s4, s5, s6; - load_s16_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6); - s += 7 * src_stride; - - // This operation combines a conventional transpose and the sample permute - // required before computing the dot product. - int16x8_t s0123[4], s1234[4], s2345[4], s3456[4]; - transpose_concat_8x4(s0, s1, s2, s3, s0123); - transpose_concat_8x4(s1, s2, s3, s4, s1234); - transpose_concat_8x4(s2, s3, s4, s5, s2345); - transpose_concat_8x4(s3, s4, s5, s6, s3456); - - do { - int16x8_t s7, s8, s9, s10; - load_s16_8x4(s, src_stride, &s7, &s8, &s9, &s10); - - int16x8_t s4567[4], s5678[4], s6789[4], s78910[4]; - - // Transpose and shuffle the 4 lines that were loaded. - transpose_concat_8x4(s7, s8, s9, s10, s78910); - - // Merge new data into block from previous iteration. - aom_tbl2x4_s16(s3456, s78910, merge_block_tbl[0], s4567); - aom_tbl2x4_s16(s3456, s78910, merge_block_tbl[1], s5678); - aom_tbl2x4_s16(s3456, s78910, merge_block_tbl[2], s6789); - - uint16x8_t d0 = highbd_convolve8_8_v(s0123, s4567, y_filter, max); - uint16x8_t d1 = highbd_convolve8_8_v(s1234, s5678, y_filter, max); - uint16x8_t d2 = highbd_convolve8_8_v(s2345, s6789, y_filter, max); - uint16x8_t d3 = highbd_convolve8_8_v(s3456, s78910, y_filter, max); - - store_u16_8x4(d, dst_stride, d0, d1, d2, d3); - - // Prepare block for next iteration - re-using as much as possible. - // Shuffle everything up four rows. - s0123[0] = s4567[0]; - s0123[1] = s4567[1]; - s0123[2] = s4567[2]; - s0123[3] = s4567[3]; - - s1234[0] = s5678[0]; - s1234[1] = s5678[1]; - s1234[2] = s5678[2]; - s1234[3] = s5678[3]; - - s2345[0] = s6789[0]; - s2345[1] = s6789[1]; - s2345[2] = s6789[2]; - s2345[3] = s6789[3]; - - s3456[0] = s78910[0]; - s3456[1] = s78910[1]; - s3456[2] = s78910[2]; - s3456[3] = s78910[3]; - - s += 4 * src_stride; - d += 4 * dst_stride; - h -= 4; - } while (h != 0); - src += 8; - dst += 8; - width -= 8; - } while (width != 0); - } + d += 4 * dst_stride; + h -= 4; + } while (h != 0); + src += 4; + dst += 4; + width -= 4; + } while (width != 0); } void aom_highbd_convolve8_vert_sve(const uint8_t *src8, ptrdiff_t src_stride,
diff --git a/aom_dsp/arm/highbd_convolve8_sve.h b/aom_dsp/arm/highbd_convolve8_sve.h new file mode 100644 index 0000000..1bbdc28 --- /dev/null +++ b/aom_dsp/arm/highbd_convolve8_sve.h
@@ -0,0 +1,20 @@ +/* + * Copyright (c) 2025, Alliance for Open Media. All rights reserved. + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_AOM_DSP_ARM_HIGHBD_CONVOLVE8_SVE_H_ +#define AOM_AOM_DSP_ARM_HIGHBD_CONVOLVE8_SVE_H_ + +#include "aom_ports/mem.h" + +DECLARE_ALIGNED(16, extern const uint16_t, kHbdDotProdTbl[32]); +DECLARE_ALIGNED(16, extern const uint16_t, kDeinterleaveTbl[8]); + +#endif // AOM_AOM_DSP_ARM_HIGHBD_CONVOLVE8_SVE_H_
diff --git a/aom_dsp/arm/highbd_sad_neon.c b/aom_dsp/arm/highbd_sad_neon.c index 115aae8..b8dde45 100644 --- a/aom_dsp/arm/highbd_sad_neon.c +++ b/aom_dsp/arm/highbd_sad_neon.c
@@ -19,15 +19,14 @@ #include "aom_dsp/arm/mem_neon.h" #include "aom_dsp/arm/sum_neon.h" -static inline uint32_t highbd_sad4xh_small_neon(const uint8_t *src_ptr, - int src_stride, - const uint8_t *ref_ptr, - int ref_stride, int h) { +static inline uint32_t highbd_sad4xh_neon(const uint8_t *src_ptr, + int src_stride, + const uint8_t *ref_ptr, + int ref_stride, int h) { const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr); const uint16_t *ref16_ptr = CONVERT_TO_SHORTPTR(ref_ptr); uint32x4_t sum = vdupq_n_u32(0); - int i = h; do { uint16x4_t s = vld1_u16(src16_ptr); uint16x4_t r = vld1_u16(ref16_ptr); @@ -35,248 +34,259 @@ src16_ptr += src_stride; ref16_ptr += ref_stride; - } while (--i != 0); + } while (--h != 0); return horizontal_add_u32x4(sum); } -static inline uint32_t highbd_sad8xh_small_neon(const uint8_t *src_ptr, - int src_stride, - const uint8_t *ref_ptr, - int ref_stride, int h) { +static inline uint32_t highbd_sad8xh_neon(const uint8_t *src_ptr, + int src_stride, + const uint8_t *ref_ptr, + int ref_stride, int h) { const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr); const uint16_t *ref16_ptr = CONVERT_TO_SHORTPTR(ref_ptr); - uint16x8_t sum = vdupq_n_u16(0); - int i = h; - do { - uint16x8_t s = vld1q_u16(src16_ptr); - uint16x8_t r = vld1q_u16(ref16_ptr); - sum = vabaq_u16(sum, s, r); + // 'h_overflow' is the number of 8-wide rows we can process before 16-bit + // accumulators overflow. After hitting this limit accumulate into 32-bit + // elements. 65535 / 4095 ~= 16, so 16 8-wide rows. + const int h_overflow = 16; + // If block height 'h' is smaller than this limit, use 'h' instead. + const int h_limit = h < h_overflow ? h : h_overflow; + assert(h % h_limit == 0); - src16_ptr += src_stride; - ref16_ptr += ref_stride; - } while (--i != 0); - - return horizontal_add_u16x8(sum); -} - -#if !CONFIG_REALTIME_ONLY -static inline uint32_t highbd_sad8xh_large_neon(const uint8_t *src_ptr, - int src_stride, - const uint8_t *ref_ptr, - int ref_stride, int h) { - const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr); - const uint16_t *ref16_ptr = CONVERT_TO_SHORTPTR(ref_ptr); uint32x4_t sum_u32 = vdupq_n_u32(0); - int i = h; do { - uint16x8_t s = vld1q_u16(src16_ptr); - uint16x8_t r = vld1q_u16(ref16_ptr); - uint16x8_t sum_u16 = vabdq_u16(s, r); + uint16x8_t sum_u16 = vdupq_n_u16(0); + + int i = h_limit; + do { + uint16x8_t s0 = vld1q_u16(src16_ptr); + uint16x8_t r0 = vld1q_u16(ref16_ptr); + sum_u16 = vabaq_u16(sum_u16, s0, r0); + + src16_ptr += src_stride; + ref16_ptr += ref_stride; + } while (--i != 0); + sum_u32 = vpadalq_u16(sum_u32, sum_u16); - src16_ptr += src_stride; - ref16_ptr += ref_stride; - } while (--i != 0); + h -= h_limit; + } while (h != 0); return horizontal_add_u32x4(sum_u32); } -#endif // !CONFIG_REALTIME_ONLY -static inline uint32_t highbd_sad16xh_large_neon(const uint8_t *src_ptr, - int src_stride, - const uint8_t *ref_ptr, - int ref_stride, int h) { +static inline uint32_t highbd_sad16xh_neon(const uint8_t *src_ptr, + int src_stride, + const uint8_t *ref_ptr, + int ref_stride, int h) { const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr); const uint16_t *ref16_ptr = CONVERT_TO_SHORTPTR(ref_ptr); - uint32x4_t sum[2] = { vdupq_n_u32(0), vdupq_n_u32(0) }; - int i = h; + // 'h_overflow' is the number of 16-wide rows we can process before 16-bit + // accumulators overflow. After hitting this limit accumulate into 32-bit + // elements. 65535 / 4095 ~= 16, so 16 16-wide rows using two accumulators. + const int h_overflow = 16; + // If block height 'h' is smaller than this limit, use 'h' instead. + const int h_limit = h < h_overflow ? h : h_overflow; + assert(h % h_limit == 0); + + uint32x4_t sum_u32 = vdupq_n_u32(0); + do { - uint16x8_t s0 = vld1q_u16(src16_ptr); - uint16x8_t r0 = vld1q_u16(ref16_ptr); - uint16x8_t diff0 = vabdq_u16(s0, r0); - sum[0] = vpadalq_u16(sum[0], diff0); - - uint16x8_t s1 = vld1q_u16(src16_ptr + 8); - uint16x8_t r1 = vld1q_u16(ref16_ptr + 8); - uint16x8_t diff1 = vabdq_u16(s1, r1); - sum[1] = vpadalq_u16(sum[1], diff1); - - src16_ptr += src_stride; - ref16_ptr += ref_stride; - } while (--i != 0); - - sum[0] = vaddq_u32(sum[0], sum[1]); - return horizontal_add_u32x4(sum[0]); -} - -static inline uint32_t highbd_sadwxh_large_neon(const uint8_t *src_ptr, - int src_stride, - const uint8_t *ref_ptr, - int ref_stride, int w, int h) { - const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr); - const uint16_t *ref16_ptr = CONVERT_TO_SHORTPTR(ref_ptr); - uint32x4_t sum[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0), - vdupq_n_u32(0) }; - - int i = h; - do { - int j = 0; + uint16x8_t sum_u16[2] = { vdupq_n_u16(0), vdupq_n_u16(0) }; + int i = h_limit; do { - uint16x8_t s0 = vld1q_u16(src16_ptr + j); - uint16x8_t r0 = vld1q_u16(ref16_ptr + j); - uint16x8_t diff0 = vabdq_u16(s0, r0); - sum[0] = vpadalq_u16(sum[0], diff0); + uint16x8_t s0 = vld1q_u16(src16_ptr); + uint16x8_t r0 = vld1q_u16(ref16_ptr); + sum_u16[0] = vabaq_u16(sum_u16[0], s0, r0); - uint16x8_t s1 = vld1q_u16(src16_ptr + j + 8); - uint16x8_t r1 = vld1q_u16(ref16_ptr + j + 8); - uint16x8_t diff1 = vabdq_u16(s1, r1); - sum[1] = vpadalq_u16(sum[1], diff1); + uint16x8_t s1 = vld1q_u16(src16_ptr + 8); + uint16x8_t r1 = vld1q_u16(ref16_ptr + 8); + sum_u16[1] = vabaq_u16(sum_u16[1], s1, r1); - uint16x8_t s2 = vld1q_u16(src16_ptr + j + 16); - uint16x8_t r2 = vld1q_u16(ref16_ptr + j + 16); - uint16x8_t diff2 = vabdq_u16(s2, r2); - sum[2] = vpadalq_u16(sum[2], diff2); + src16_ptr += src_stride; + ref16_ptr += ref_stride; + } while (--i != 0); - uint16x8_t s3 = vld1q_u16(src16_ptr + j + 24); - uint16x8_t r3 = vld1q_u16(ref16_ptr + j + 24); - uint16x8_t diff3 = vabdq_u16(s3, r3); - sum[3] = vpadalq_u16(sum[3], diff3); - - j += 32; - } while (j < w); - - src16_ptr += src_stride; - ref16_ptr += ref_stride; - } while (--i != 0); - - sum[0] = vaddq_u32(sum[0], sum[1]); - sum[2] = vaddq_u32(sum[2], sum[3]); - sum[0] = vaddq_u32(sum[0], sum[2]); - - return horizontal_add_u32x4(sum[0]); + sum_u32 = vpadalq_u16(sum_u32, sum_u16[0]); + sum_u32 = vpadalq_u16(sum_u32, sum_u16[1]); + h -= h_limit; + } while (h != 0); + return horizontal_add_u32x4(sum_u32); } -static inline unsigned int highbd_sad128xh_large_neon(const uint8_t *src_ptr, - int src_stride, - const uint8_t *ref_ptr, - int ref_stride, int h) { - return highbd_sadwxh_large_neon(src_ptr, src_stride, ref_ptr, ref_stride, 128, - h); +static inline uint32_t highbd_sadwxh_neon(const uint8_t *src_ptr, + int src_stride, + const uint8_t *ref_ptr, + int ref_stride, int w, int h, + const int h_overflow) { + const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr); + const uint16_t *ref16_ptr = CONVERT_TO_SHORTPTR(ref_ptr); + + const int h_limit = h < h_overflow ? h : h_overflow; + assert(h % h_limit == 0); + + uint32x4_t sum_u32 = vdupq_n_u32(0); + + do { + uint16x8_t sum_u16[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0), + vdupq_n_u16(0) }; + + int i = h_limit; + do { + int j = 0; + do { + uint16x8_t s0 = vld1q_u16(src16_ptr + j); + uint16x8_t r0 = vld1q_u16(ref16_ptr + j); + sum_u16[0] = vabaq_u16(sum_u16[0], s0, r0); + + uint16x8_t s1 = vld1q_u16(src16_ptr + j + 8); + uint16x8_t r1 = vld1q_u16(ref16_ptr + j + 8); + sum_u16[1] = vabaq_u16(sum_u16[1], s1, r1); + + uint16x8_t s2 = vld1q_u16(src16_ptr + j + 16); + uint16x8_t r2 = vld1q_u16(ref16_ptr + j + 16); + sum_u16[2] = vabaq_u16(sum_u16[2], s2, r2); + + uint16x8_t s3 = vld1q_u16(src16_ptr + j + 24); + uint16x8_t r3 = vld1q_u16(ref16_ptr + j + 24); + sum_u16[3] = vabaq_u16(sum_u16[3], s3, r3); + + j += 32; + } while (j < w); + + src16_ptr += src_stride; + ref16_ptr += ref_stride; + } while (--i != 0); + + sum_u32 = vpadalq_u16(sum_u32, sum_u16[0]); + sum_u32 = vpadalq_u16(sum_u32, sum_u16[1]); + sum_u32 = vpadalq_u16(sum_u32, sum_u16[2]); + sum_u32 = vpadalq_u16(sum_u32, sum_u16[3]); + + h -= h_limit; + } while (h != 0); + return horizontal_add_u32x4(sum_u32); } -static inline unsigned int highbd_sad64xh_large_neon(const uint8_t *src_ptr, - int src_stride, - const uint8_t *ref_ptr, - int ref_stride, int h) { - return highbd_sadwxh_large_neon(src_ptr, src_stride, ref_ptr, ref_stride, 64, - h); +static inline uint32_t highbd_sad32xh_neon(const uint8_t *src_ptr, + int src_stride, + const uint8_t *ref_ptr, + int ref_stride, int h) { + // 'h_overflow' is the number of 32-wide rows we can process before 16-bit + // accumulators overflow. After hitting this limit accumulate into 32-bit + // elements. 65535 / 4095 ~= 16, so 16 32-wide rows using four accumulators. + const int h_overflow = 16; + return highbd_sadwxh_neon(src_ptr, src_stride, ref_ptr, ref_stride, 32, h, + h_overflow); } -static inline unsigned int highbd_sad32xh_large_neon(const uint8_t *src_ptr, - int src_stride, - const uint8_t *ref_ptr, - int ref_stride, int h) { - return highbd_sadwxh_large_neon(src_ptr, src_stride, ref_ptr, ref_stride, 32, - h); +static inline uint32_t highbd_sad64xh_neon(const uint8_t *src_ptr, + int src_stride, + const uint8_t *ref_ptr, + int ref_stride, int h) { + // 'h_overflow' is the number of 64-wide rows we can process before 16-bit + // accumulators overflow. After hitting this limit accumulate into 32-bit + // elements. 65535 / 4095 ~= 16, so 8 64-wide rows using four accumulators. + const int h_overflow = 8; + return highbd_sadwxh_neon(src_ptr, src_stride, ref_ptr, ref_stride, 64, h, + h_overflow); } -#define HBD_SAD_WXH_SMALL_NEON(w, h) \ +static inline uint32_t highbd_sad128xh_neon(const uint8_t *src_ptr, + int src_stride, + const uint8_t *ref_ptr, + int ref_stride, int h) { + // 'h_overflow' is the number of 128-wide rows we can process before 16-bit + // accumulators overflow. After hitting this limit accumulate into 32-bit + // elements. 65535 / 4095 ~= 16, so 4 128-wide rows using four accumulators. + const int h_overflow = 4; + return highbd_sadwxh_neon(src_ptr, src_stride, ref_ptr, ref_stride, 128, h, + h_overflow); +} + +#define HBD_SAD_WXH_NEON(w, h) \ unsigned int aom_highbd_sad##w##x##h##_neon( \ const uint8_t *src, int src_stride, const uint8_t *ref, \ int ref_stride) { \ - return highbd_sad##w##xh_small_neon(src, src_stride, ref, ref_stride, \ - (h)); \ + return highbd_sad##w##xh_neon(src, src_stride, ref, ref_stride, (h)); \ } -#define HBD_SAD_WXH_LARGE_NEON(w, h) \ - unsigned int aom_highbd_sad##w##x##h##_neon( \ - const uint8_t *src, int src_stride, const uint8_t *ref, \ - int ref_stride) { \ - return highbd_sad##w##xh_large_neon(src, src_stride, ref, ref_stride, \ - (h)); \ - } +HBD_SAD_WXH_NEON(4, 4) +HBD_SAD_WXH_NEON(4, 8) -HBD_SAD_WXH_SMALL_NEON(4, 4) -HBD_SAD_WXH_SMALL_NEON(4, 8) +HBD_SAD_WXH_NEON(8, 4) +HBD_SAD_WXH_NEON(8, 8) +HBD_SAD_WXH_NEON(8, 16) -HBD_SAD_WXH_SMALL_NEON(8, 4) -HBD_SAD_WXH_SMALL_NEON(8, 8) -HBD_SAD_WXH_SMALL_NEON(8, 16) +HBD_SAD_WXH_NEON(16, 8) +HBD_SAD_WXH_NEON(16, 16) +HBD_SAD_WXH_NEON(16, 32) -HBD_SAD_WXH_LARGE_NEON(16, 8) -HBD_SAD_WXH_LARGE_NEON(16, 16) -HBD_SAD_WXH_LARGE_NEON(16, 32) +HBD_SAD_WXH_NEON(32, 16) +HBD_SAD_WXH_NEON(32, 32) +HBD_SAD_WXH_NEON(32, 64) -HBD_SAD_WXH_LARGE_NEON(32, 16) -HBD_SAD_WXH_LARGE_NEON(32, 32) -HBD_SAD_WXH_LARGE_NEON(32, 64) +HBD_SAD_WXH_NEON(64, 32) +HBD_SAD_WXH_NEON(64, 64) +HBD_SAD_WXH_NEON(64, 128) -HBD_SAD_WXH_LARGE_NEON(64, 32) -HBD_SAD_WXH_LARGE_NEON(64, 64) -HBD_SAD_WXH_LARGE_NEON(64, 128) - -HBD_SAD_WXH_LARGE_NEON(128, 64) -HBD_SAD_WXH_LARGE_NEON(128, 128) +HBD_SAD_WXH_NEON(128, 64) +HBD_SAD_WXH_NEON(128, 128) #if !CONFIG_REALTIME_ONLY -HBD_SAD_WXH_SMALL_NEON(4, 16) +HBD_SAD_WXH_NEON(4, 16) -HBD_SAD_WXH_LARGE_NEON(8, 32) +HBD_SAD_WXH_NEON(8, 32) -HBD_SAD_WXH_LARGE_NEON(16, 4) -HBD_SAD_WXH_LARGE_NEON(16, 64) +HBD_SAD_WXH_NEON(16, 4) +HBD_SAD_WXH_NEON(16, 64) -HBD_SAD_WXH_LARGE_NEON(32, 8) +HBD_SAD_WXH_NEON(32, 8) -HBD_SAD_WXH_LARGE_NEON(64, 16) +HBD_SAD_WXH_NEON(64, 16) #endif // !CONFIG_REALTIME_ONLY -#define HBD_SAD_SKIP_WXH_SMALL_NEON(w, h) \ - unsigned int aom_highbd_sad_skip_##w##x##h##_neon( \ - const uint8_t *src, int src_stride, const uint8_t *ref, \ - int ref_stride) { \ - return 2 * highbd_sad##w##xh_small_neon(src, 2 * src_stride, ref, \ - 2 * ref_stride, (h) / 2); \ +#undef HBD_SAD_WXH_NEON + +#define HBD_SAD_SKIP_WXH_NEON(w, h) \ + unsigned int aom_highbd_sad_skip_##w##x##h##_neon( \ + const uint8_t *src, int src_stride, const uint8_t *ref, \ + int ref_stride) { \ + return 2 * highbd_sad##w##xh_neon(src, 2 * src_stride, ref, \ + 2 * ref_stride, (h) / 2); \ } -#define HBD_SAD_SKIP_WXH_LARGE_NEON(w, h) \ - unsigned int aom_highbd_sad_skip_##w##x##h##_neon( \ - const uint8_t *src, int src_stride, const uint8_t *ref, \ - int ref_stride) { \ - return 2 * highbd_sad##w##xh_large_neon(src, 2 * src_stride, ref, \ - 2 * ref_stride, (h) / 2); \ - } +HBD_SAD_SKIP_WXH_NEON(8, 16) -HBD_SAD_SKIP_WXH_SMALL_NEON(8, 16) +HBD_SAD_SKIP_WXH_NEON(16, 16) +HBD_SAD_SKIP_WXH_NEON(16, 32) -HBD_SAD_SKIP_WXH_LARGE_NEON(16, 16) -HBD_SAD_SKIP_WXH_LARGE_NEON(16, 32) +HBD_SAD_SKIP_WXH_NEON(32, 16) +HBD_SAD_SKIP_WXH_NEON(32, 32) +HBD_SAD_SKIP_WXH_NEON(32, 64) -HBD_SAD_SKIP_WXH_LARGE_NEON(32, 16) -HBD_SAD_SKIP_WXH_LARGE_NEON(32, 32) -HBD_SAD_SKIP_WXH_LARGE_NEON(32, 64) +HBD_SAD_SKIP_WXH_NEON(64, 32) +HBD_SAD_SKIP_WXH_NEON(64, 64) +HBD_SAD_SKIP_WXH_NEON(64, 128) -HBD_SAD_SKIP_WXH_LARGE_NEON(64, 32) -HBD_SAD_SKIP_WXH_LARGE_NEON(64, 64) -HBD_SAD_SKIP_WXH_LARGE_NEON(64, 128) - -HBD_SAD_SKIP_WXH_LARGE_NEON(128, 64) -HBD_SAD_SKIP_WXH_LARGE_NEON(128, 128) +HBD_SAD_SKIP_WXH_NEON(128, 64) +HBD_SAD_SKIP_WXH_NEON(128, 128) #if !CONFIG_REALTIME_ONLY -HBD_SAD_SKIP_WXH_SMALL_NEON(4, 16) +HBD_SAD_SKIP_WXH_NEON(4, 16) -HBD_SAD_SKIP_WXH_SMALL_NEON(8, 32) +HBD_SAD_SKIP_WXH_NEON(8, 32) -HBD_SAD_SKIP_WXH_LARGE_NEON(16, 64) +HBD_SAD_SKIP_WXH_NEON(16, 64) -HBD_SAD_SKIP_WXH_LARGE_NEON(64, 16) +HBD_SAD_SKIP_WXH_NEON(64, 16) #endif // !CONFIG_REALTIME_ONLY +#undef HBD_SAD_SKIP_WXH_NEON + static inline uint32_t highbd_sad8xh_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, @@ -285,24 +295,40 @@ const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr); const uint16_t *ref16_ptr = CONVERT_TO_SHORTPTR(ref_ptr); const uint16_t *pred16_ptr = CONVERT_TO_SHORTPTR(second_pred); - uint32x4_t sum = vdupq_n_u32(0); - int i = h; + // 'h_overflow' is the number of 8-wide rows we can process before 16-bit + // accumulators overflow. After hitting this limit accumulate into 32-bit + // elements. 65535 / 4095 ~= 16, so 16 8-wide rows. + const int h_overflow = 16; + // If block height 'h' is smaller than this limit, use 'h' instead. + const int h_limit = h < h_overflow ? h : h_overflow; + assert(h % h_limit == 0); + + uint32x4_t sum_u32 = vdupq_n_u32(0); + do { - uint16x8_t s = vld1q_u16(src16_ptr); - uint16x8_t r = vld1q_u16(ref16_ptr); - uint16x8_t p = vld1q_u16(pred16_ptr); + uint16x8_t sum_u16 = vdupq_n_u16(0); - uint16x8_t avg = vrhaddq_u16(r, p); - uint16x8_t diff = vabdq_u16(s, avg); - sum = vpadalq_u16(sum, diff); + int i = h_limit; + do { + uint16x8_t s = vld1q_u16(src16_ptr); + uint16x8_t r = vld1q_u16(ref16_ptr); + uint16x8_t p = vld1q_u16(pred16_ptr); - src16_ptr += src_stride; - ref16_ptr += ref_stride; - pred16_ptr += 8; - } while (--i != 0); + uint16x8_t avg = vrhaddq_u16(r, p); + sum_u16 = vabaq_u16(sum_u16, s, avg); - return horizontal_add_u32x4(sum); + src16_ptr += src_stride; + ref16_ptr += ref_stride; + pred16_ptr += 8; + } while (--i != 0); + + sum_u32 = vpadalq_u16(sum_u32, sum_u16); + + h -= h_limit; + } while (h != 0); + + return horizontal_add_u32x4(sum_u32); } static inline uint32_t highbd_sad16xh_avg_neon(const uint8_t *src_ptr, @@ -313,120 +339,157 @@ const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr); const uint16_t *ref16_ptr = CONVERT_TO_SHORTPTR(ref_ptr); const uint16_t *pred16_ptr = CONVERT_TO_SHORTPTR(second_pred); - uint32x4_t sum[2] = { vdupq_n_u32(0), vdupq_n_u32(0) }; - int i = h; + // 'h_overflow' is the number of 16-wide rows we can process before 16-bit + // accumulators overflow. After hitting this limit accumulate into 32-bit + // elements. 65535 / 4095 ~= 16, so 16 16-wide rows using two accumulators. + const int h_overflow = 16; + // If block height 'h' is smaller than this limit, use 'h' instead. + const int h_limit = h < h_overflow ? h : h_overflow; + assert(h % h_limit == 0); + + uint32x4_t sum_u32 = vdupq_n_u32(0); + do { - uint16x8_t s0, s1, r0, r1, p0, p1; - uint16x8_t avg0, avg1, diff0, diff1; + uint16x8_t sum_u16[2] = { vdupq_n_u16(0), vdupq_n_u16(0) }; - s0 = vld1q_u16(src16_ptr); - r0 = vld1q_u16(ref16_ptr); - p0 = vld1q_u16(pred16_ptr); - avg0 = vrhaddq_u16(r0, p0); - diff0 = vabdq_u16(s0, avg0); - sum[0] = vpadalq_u16(sum[0], diff0); + int i = h_limit; + do { + uint16x8_t s0 = vld1q_u16(src16_ptr); + uint16x8_t r0 = vld1q_u16(ref16_ptr); + uint16x8_t p0 = vld1q_u16(pred16_ptr); - s1 = vld1q_u16(src16_ptr + 8); - r1 = vld1q_u16(ref16_ptr + 8); - p1 = vld1q_u16(pred16_ptr + 8); - avg1 = vrhaddq_u16(r1, p1); - diff1 = vabdq_u16(s1, avg1); - sum[1] = vpadalq_u16(sum[1], diff1); + uint16x8_t avg0 = vrhaddq_u16(r0, p0); + sum_u16[0] = vabaq_u16(sum_u16[0], s0, avg0); - src16_ptr += src_stride; - ref16_ptr += ref_stride; - pred16_ptr += 16; - } while (--i != 0); + uint16x8_t s1 = vld1q_u16(src16_ptr + 8); + uint16x8_t r1 = vld1q_u16(ref16_ptr + 8); + uint16x8_t p1 = vld1q_u16(pred16_ptr + 8); - sum[0] = vaddq_u32(sum[0], sum[1]); - return horizontal_add_u32x4(sum[0]); + uint16x8_t avg1 = vrhaddq_u16(r1, p1); + sum_u16[1] = vabaq_u16(sum_u16[1], s1, avg1); + + src16_ptr += src_stride; + ref16_ptr += ref_stride; + pred16_ptr += 16; + } while (--i != 0); + + sum_u32 = vpadalq_u16(sum_u32, sum_u16[0]); + sum_u32 = vpadalq_u16(sum_u32, sum_u16[1]); + + h -= h_limit; + } while (h != 0); + + return horizontal_add_u32x4(sum_u32); } static inline uint32_t highbd_sadwxh_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, - int ref_stride, int w, int h, - const uint8_t *second_pred) { + int ref_stride, + const uint8_t *second_pred, int w, + int h, const int h_overflow) { const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr); const uint16_t *ref16_ptr = CONVERT_TO_SHORTPTR(ref_ptr); const uint16_t *pred16_ptr = CONVERT_TO_SHORTPTR(second_pred); - uint32x4_t sum[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0), - vdupq_n_u32(0) }; - int i = h; + const int h_limit = h < h_overflow ? h : h_overflow; + assert(h % h_limit == 0); + + uint32x4_t sum_u32 = vdupq_n_u32(0); + do { - int j = 0; + uint16x8_t sum_u16[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0), + vdupq_n_u16(0) }; + + int i = h_limit; do { - uint16x8_t s0, s1, s2, s3, r0, r1, r2, r3, p0, p1, p2, p3; - uint16x8_t avg0, avg1, avg2, avg3, diff0, diff1, diff2, diff3; + int j = 0; + do { + uint16x8_t s0 = vld1q_u16(src16_ptr + j); + uint16x8_t r0 = vld1q_u16(ref16_ptr + j); + uint16x8_t p0 = vld1q_u16(pred16_ptr + j); - s0 = vld1q_u16(src16_ptr + j); - r0 = vld1q_u16(ref16_ptr + j); - p0 = vld1q_u16(pred16_ptr + j); - avg0 = vrhaddq_u16(r0, p0); - diff0 = vabdq_u16(s0, avg0); - sum[0] = vpadalq_u16(sum[0], diff0); + uint16x8_t avg0 = vrhaddq_u16(r0, p0); + sum_u16[0] = vabaq_u16(sum_u16[0], s0, avg0); - s1 = vld1q_u16(src16_ptr + j + 8); - r1 = vld1q_u16(ref16_ptr + j + 8); - p1 = vld1q_u16(pred16_ptr + j + 8); - avg1 = vrhaddq_u16(r1, p1); - diff1 = vabdq_u16(s1, avg1); - sum[1] = vpadalq_u16(sum[1], diff1); + uint16x8_t s1 = vld1q_u16(src16_ptr + j + 8); + uint16x8_t r1 = vld1q_u16(ref16_ptr + j + 8); + uint16x8_t p1 = vld1q_u16(pred16_ptr + j + 8); - s2 = vld1q_u16(src16_ptr + j + 16); - r2 = vld1q_u16(ref16_ptr + j + 16); - p2 = vld1q_u16(pred16_ptr + j + 16); - avg2 = vrhaddq_u16(r2, p2); - diff2 = vabdq_u16(s2, avg2); - sum[2] = vpadalq_u16(sum[2], diff2); + uint16x8_t avg1 = vrhaddq_u16(r1, p1); + sum_u16[1] = vabaq_u16(sum_u16[1], s1, avg1); - s3 = vld1q_u16(src16_ptr + j + 24); - r3 = vld1q_u16(ref16_ptr + j + 24); - p3 = vld1q_u16(pred16_ptr + j + 24); - avg3 = vrhaddq_u16(r3, p3); - diff3 = vabdq_u16(s3, avg3); - sum[3] = vpadalq_u16(sum[3], diff3); + uint16x8_t s2 = vld1q_u16(src16_ptr + j + 16); + uint16x8_t r2 = vld1q_u16(ref16_ptr + j + 16); + uint16x8_t p2 = vld1q_u16(pred16_ptr + j + 16); - j += 32; - } while (j < w); + uint16x8_t avg2 = vrhaddq_u16(r2, p2); + sum_u16[2] = vabaq_u16(sum_u16[2], s2, avg2); - src16_ptr += src_stride; - ref16_ptr += ref_stride; - pred16_ptr += w; - } while (--i != 0); + uint16x8_t s3 = vld1q_u16(src16_ptr + j + 24); + uint16x8_t r3 = vld1q_u16(ref16_ptr + j + 24); + uint16x8_t p3 = vld1q_u16(pred16_ptr + j + 24); - sum[0] = vaddq_u32(sum[0], sum[1]); - sum[2] = vaddq_u32(sum[2], sum[3]); - sum[0] = vaddq_u32(sum[0], sum[2]); + uint16x8_t avg3 = vrhaddq_u16(r3, p3); + sum_u16[3] = vabaq_u16(sum_u16[3], s3, avg3); - return horizontal_add_u32x4(sum[0]); + j += 32; + } while (j < w); + + src16_ptr += src_stride; + ref16_ptr += ref_stride; + pred16_ptr += w; + } while (--i != 0); + + sum_u32 = vpadalq_u16(sum_u32, sum_u16[0]); + sum_u32 = vpadalq_u16(sum_u32, sum_u16[1]); + sum_u32 = vpadalq_u16(sum_u32, sum_u16[2]); + sum_u32 = vpadalq_u16(sum_u32, sum_u16[3]); + + h -= h_limit; + } while (h != 0); + + return horizontal_add_u32x4(sum_u32); } -static inline unsigned int highbd_sad128xh_avg_neon( - const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, - int ref_stride, int h, const uint8_t *second_pred) { - return highbd_sadwxh_avg_neon(src_ptr, src_stride, ref_ptr, ref_stride, 128, - h, second_pred); +static inline uint32_t highbd_sad32xh_avg_neon(const uint8_t *src_ptr, + int src_stride, + const uint8_t *ref_ptr, + int ref_stride, int h, + const uint8_t *second_pred) { + // 'h_overflow' is the number of 32-wide rows we can process before 16-bit + // accumulators overflow. After hitting this limit accumulate into 32-bit + // elements. 65535 / 4095 ~= 16, so 16 32-wide rows using four accumulators. + const int h_overflow = 16; + return highbd_sadwxh_avg_neon(src_ptr, src_stride, ref_ptr, ref_stride, + second_pred, 32, h, h_overflow); } -static inline unsigned int highbd_sad64xh_avg_neon(const uint8_t *src_ptr, - int src_stride, - const uint8_t *ref_ptr, - int ref_stride, int h, - const uint8_t *second_pred) { - return highbd_sadwxh_avg_neon(src_ptr, src_stride, ref_ptr, ref_stride, 64, h, - second_pred); +static inline uint32_t highbd_sad64xh_avg_neon(const uint8_t *src_ptr, + int src_stride, + const uint8_t *ref_ptr, + int ref_stride, int h, + const uint8_t *second_pred) { + // 'h_overflow' is the number of 64-wide rows we can process before 16-bit + // accumulators overflow. After hitting this limit accumulate into 32-bit + // elements. 65535 / 4095 ~= 16, so 8 64-wide rows using four accumulators. + const int h_overflow = 8; + return highbd_sadwxh_avg_neon(src_ptr, src_stride, ref_ptr, ref_stride, + second_pred, 64, h, h_overflow); } -static inline unsigned int highbd_sad32xh_avg_neon(const uint8_t *src_ptr, - int src_stride, - const uint8_t *ref_ptr, - int ref_stride, int h, - const uint8_t *second_pred) { - return highbd_sadwxh_avg_neon(src_ptr, src_stride, ref_ptr, ref_stride, 32, h, - second_pred); +static inline uint32_t highbd_sad128xh_avg_neon(const uint8_t *src_ptr, + int src_stride, + const uint8_t *ref_ptr, + int ref_stride, int h, + const uint8_t *second_pred) { + // 'h_overflow' is the number of 128-wide rows we can process before 16-bit + // accumulators overflow. After hitting this limit accumulate into 32-bit + // elements. 65535 / 4095 ~= 16, so 4 128-wide rows using four accumulators. + const int h_overflow = 4; + return highbd_sadwxh_avg_neon(src_ptr, src_stride, ref_ptr, ref_stride, + second_pred, 128, h, h_overflow); } #define HBD_SAD_WXH_AVG_NEON(w, h) \ @@ -464,3 +527,5 @@ HBD_SAD_WXH_AVG_NEON(64, 16) #endif // !CONFIG_REALTIME_ONLY + +#undef HBD_SAD_WXH_AVG_NEON
diff --git a/aom_dsp/arm/highbd_sadxd_neon.c b/aom_dsp/arm/highbd_sadxd_neon.c index 0e43f6d..0b45278 100644 --- a/aom_dsp/arm/highbd_sadxd_neon.c +++ b/aom_dsp/arm/highbd_sadxd_neon.c
@@ -19,11 +19,10 @@ #include "aom_dsp/arm/mem_neon.h" #include "aom_dsp/arm/sum_neon.h" -static inline void highbd_sad4xhx4d_small_neon(const uint8_t *src_ptr, - int src_stride, - const uint8_t *const ref_ptr[4], - int ref_stride, uint32_t res[4], - int h) { +static inline void highbd_sad4xhx4d_neon(const uint8_t *src_ptr, int src_stride, + const uint8_t *const ref_ptr[4], + int ref_stride, uint32_t res[4], + int h) { const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr); const uint16_t *ref16_ptr0 = CONVERT_TO_SHORTPTR(ref_ptr[0]); const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]); @@ -51,317 +50,261 @@ vst1q_u32(res, horizontal_add_4d_u32x4(sum)); } -static inline void highbd_sad8xhx4d_small_neon(const uint8_t *src_ptr, - int src_stride, - const uint8_t *const ref_ptr[4], - int ref_stride, uint32_t res[4], - int h) { +static inline void highbd_sad8xhx4d_neon(const uint8_t *src_ptr, int src_stride, + const uint8_t *const ref_ptr[4], + int ref_stride, uint32_t res[4], + int h) { const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr); const uint16_t *ref16_ptr0 = CONVERT_TO_SHORTPTR(ref_ptr[0]); const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]); const uint16_t *ref16_ptr2 = CONVERT_TO_SHORTPTR(ref_ptr[2]); const uint16_t *ref16_ptr3 = CONVERT_TO_SHORTPTR(ref_ptr[3]); - uint16x8_t sum[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0), - vdupq_n_u16(0) }; - uint32x4_t sum_u32[4]; + // 'h_overflow' is the number of 8-wide rows we can process before 16-bit + // accumulators overflow. After hitting this limit accumulate into 32-bit + // elements. 65535 / 4095 ~= 16, so 16 8-wide rows. + const int h_overflow = 16; + // If block height 'h' is smaller than this limit, use 'h' instead. + const int h_limit = h < h_overflow ? h : h_overflow; + assert(h % h_limit == 0); + uint32x4_t sum_u32[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0), + vdupq_n_u32(0) }; + + int h_tmp = h_limit; int i = 0; do { - uint16x8_t s = vld1q_u16(src16_ptr + i * src_stride); + uint16x8_t sum_u16[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0), + vdupq_n_u16(0) }; - sum[0] = vabaq_u16(sum[0], s, vld1q_u16(ref16_ptr0 + i * ref_stride)); - sum[1] = vabaq_u16(sum[1], s, vld1q_u16(ref16_ptr1 + i * ref_stride)); - sum[2] = vabaq_u16(sum[2], s, vld1q_u16(ref16_ptr2 + i * ref_stride)); - sum[3] = vabaq_u16(sum[3], s, vld1q_u16(ref16_ptr3 + i * ref_stride)); + do { + uint16x8_t s0 = vld1q_u16(src16_ptr + i * src_stride); - } while (++i < h); + sum_u16[0] = + vabaq_u16(sum_u16[0], s0, vld1q_u16(ref16_ptr0 + i * ref_stride)); + sum_u16[1] = + vabaq_u16(sum_u16[1], s0, vld1q_u16(ref16_ptr1 + i * ref_stride)); + sum_u16[2] = + vabaq_u16(sum_u16[2], s0, vld1q_u16(ref16_ptr2 + i * ref_stride)); + sum_u16[3] = + vabaq_u16(sum_u16[3], s0, vld1q_u16(ref16_ptr3 + i * ref_stride)); + } while (++i < h_tmp); - sum_u32[0] = vpaddlq_u16(sum[0]); - sum_u32[1] = vpaddlq_u16(sum[1]); - sum_u32[2] = vpaddlq_u16(sum[2]); - sum_u32[3] = vpaddlq_u16(sum[3]); + sum_u32[0] = vpadalq_u16(sum_u32[0], sum_u16[0]); + sum_u32[1] = vpadalq_u16(sum_u32[1], sum_u16[1]); + sum_u32[2] = vpadalq_u16(sum_u32[2], sum_u16[2]); + sum_u32[3] = vpadalq_u16(sum_u32[3], sum_u16[3]); + + h_tmp += h_limit; + h -= h_limit; + } while (h != 0); + vst1q_u32(res, horizontal_add_4d_u32x4(sum_u32)); } -static inline void sad8_neon(uint16x8_t src, uint16x8_t ref, - uint32x4_t *const sad_sum) { - uint16x8_t abs_diff = vabdq_u16(src, ref); - *sad_sum = vpadalq_u16(*sad_sum, abs_diff); -} - -#if !CONFIG_REALTIME_ONLY -static inline void highbd_sad8xhx4d_large_neon(const uint8_t *src_ptr, - int src_stride, - const uint8_t *const ref_ptr[4], - int ref_stride, uint32_t res[4], - int h) { +static inline void highbd_sadwxhx4d_neon(const uint8_t *src_ptr, int src_stride, + const uint8_t *const ref_ptr[4], + int ref_stride, uint32_t res[4], int w, + int h, const int h_overflow) { const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr); const uint16_t *ref16_ptr0 = CONVERT_TO_SHORTPTR(ref_ptr[0]); const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]); const uint16_t *ref16_ptr2 = CONVERT_TO_SHORTPTR(ref_ptr[2]); const uint16_t *ref16_ptr3 = CONVERT_TO_SHORTPTR(ref_ptr[3]); - uint32x4_t sum[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0), - vdupq_n_u32(0) }; + const int h_limit = h < h_overflow ? h : h_overflow; + assert(h % h_limit == 0); - int i = 0; + uint32x4_t sum_u32[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0), + vdupq_n_u32(0) }; + do { - uint16x8_t s = vld1q_u16(src16_ptr + i * src_stride); - sad8_neon(s, vld1q_u16(ref16_ptr0 + i * ref_stride), &sum[0]); - sad8_neon(s, vld1q_u16(ref16_ptr1 + i * ref_stride), &sum[1]); - sad8_neon(s, vld1q_u16(ref16_ptr2 + i * ref_stride), &sum[2]); - sad8_neon(s, vld1q_u16(ref16_ptr3 + i * ref_stride), &sum[3]); + uint16x8_t sum_u16[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0), + vdupq_n_u16(0) }; - } while (++i < h); - - vst1q_u32(res, horizontal_add_4d_u32x4(sum)); -} -#endif // !CONFIG_REALTIME_ONLY - -static inline void highbd_sad16xhx4d_large_neon(const uint8_t *src_ptr, - int src_stride, - const uint8_t *const ref_ptr[4], - int ref_stride, uint32_t res[4], - int h) { - const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr); - const uint16_t *ref16_ptr0 = CONVERT_TO_SHORTPTR(ref_ptr[0]); - const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]); - const uint16_t *ref16_ptr2 = CONVERT_TO_SHORTPTR(ref_ptr[2]); - const uint16_t *ref16_ptr3 = CONVERT_TO_SHORTPTR(ref_ptr[3]); - - uint32x4_t sum_lo[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0), - vdupq_n_u32(0) }; - uint32x4_t sum_hi[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0), - vdupq_n_u32(0) }; - uint32x4_t sum[4]; - - int i = 0; - do { - uint16x8_t s0 = vld1q_u16(src16_ptr + i * src_stride); - sad8_neon(s0, vld1q_u16(ref16_ptr0 + i * ref_stride), &sum_lo[0]); - sad8_neon(s0, vld1q_u16(ref16_ptr1 + i * ref_stride), &sum_lo[1]); - sad8_neon(s0, vld1q_u16(ref16_ptr2 + i * ref_stride), &sum_lo[2]); - sad8_neon(s0, vld1q_u16(ref16_ptr3 + i * ref_stride), &sum_lo[3]); - - uint16x8_t s1 = vld1q_u16(src16_ptr + i * src_stride + 8); - sad8_neon(s1, vld1q_u16(ref16_ptr0 + i * ref_stride + 8), &sum_hi[0]); - sad8_neon(s1, vld1q_u16(ref16_ptr1 + i * ref_stride + 8), &sum_hi[1]); - sad8_neon(s1, vld1q_u16(ref16_ptr2 + i * ref_stride + 8), &sum_hi[2]); - sad8_neon(s1, vld1q_u16(ref16_ptr3 + i * ref_stride + 8), &sum_hi[3]); - - } while (++i < h); - - sum[0] = vaddq_u32(sum_lo[0], sum_hi[0]); - sum[1] = vaddq_u32(sum_lo[1], sum_hi[1]); - sum[2] = vaddq_u32(sum_lo[2], sum_hi[2]); - sum[3] = vaddq_u32(sum_lo[3], sum_hi[3]); - - vst1q_u32(res, horizontal_add_4d_u32x4(sum)); -} - -static inline void highbd_sadwxhx4d_large_neon(const uint8_t *src_ptr, - int src_stride, - const uint8_t *const ref_ptr[4], - int ref_stride, uint32_t res[4], - int w, int h) { - const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr); - const uint16_t *ref16_ptr0 = CONVERT_TO_SHORTPTR(ref_ptr[0]); - const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]); - const uint16_t *ref16_ptr2 = CONVERT_TO_SHORTPTR(ref_ptr[2]); - const uint16_t *ref16_ptr3 = CONVERT_TO_SHORTPTR(ref_ptr[3]); - - uint32x4_t sum_lo[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0), - vdupq_n_u32(0) }; - uint32x4_t sum_hi[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0), - vdupq_n_u32(0) }; - uint32x4_t sum[4]; - - int i = 0; - do { - int j = 0; + int i = h_limit; do { - uint16x8_t s0 = vld1q_u16(src16_ptr + i * src_stride + j); - sad8_neon(s0, vld1q_u16(ref16_ptr0 + i * ref_stride + j), &sum_lo[0]); - sad8_neon(s0, vld1q_u16(ref16_ptr1 + i * ref_stride + j), &sum_lo[1]); - sad8_neon(s0, vld1q_u16(ref16_ptr2 + i * ref_stride + j), &sum_lo[2]); - sad8_neon(s0, vld1q_u16(ref16_ptr3 + i * ref_stride + j), &sum_lo[3]); + int j = 0; + do { + uint16x8_t s0 = vld1q_u16(src16_ptr + j); - uint16x8_t s1 = vld1q_u16(src16_ptr + i * src_stride + j + 8); - sad8_neon(s1, vld1q_u16(ref16_ptr0 + i * ref_stride + j + 8), &sum_hi[0]); - sad8_neon(s1, vld1q_u16(ref16_ptr1 + i * ref_stride + j + 8), &sum_hi[1]); - sad8_neon(s1, vld1q_u16(ref16_ptr2 + i * ref_stride + j + 8), &sum_hi[2]); - sad8_neon(s1, vld1q_u16(ref16_ptr3 + i * ref_stride + j + 8), &sum_hi[3]); + sum_u16[0] = vabaq_u16(sum_u16[0], s0, vld1q_u16(ref16_ptr0 + j)); + sum_u16[1] = vabaq_u16(sum_u16[1], s0, vld1q_u16(ref16_ptr1 + j)); + sum_u16[2] = vabaq_u16(sum_u16[2], s0, vld1q_u16(ref16_ptr2 + j)); + sum_u16[3] = vabaq_u16(sum_u16[3], s0, vld1q_u16(ref16_ptr3 + j)); - uint16x8_t s2 = vld1q_u16(src16_ptr + i * src_stride + j + 16); - sad8_neon(s2, vld1q_u16(ref16_ptr0 + i * ref_stride + j + 16), - &sum_lo[0]); - sad8_neon(s2, vld1q_u16(ref16_ptr1 + i * ref_stride + j + 16), - &sum_lo[1]); - sad8_neon(s2, vld1q_u16(ref16_ptr2 + i * ref_stride + j + 16), - &sum_lo[2]); - sad8_neon(s2, vld1q_u16(ref16_ptr3 + i * ref_stride + j + 16), - &sum_lo[3]); + uint16x8_t s1 = vld1q_u16(src16_ptr + j + 8); + sum_u16[0] = vabaq_u16(sum_u16[0], s1, vld1q_u16(ref16_ptr0 + j + 8)); + sum_u16[1] = vabaq_u16(sum_u16[1], s1, vld1q_u16(ref16_ptr1 + j + 8)); + sum_u16[2] = vabaq_u16(sum_u16[2], s1, vld1q_u16(ref16_ptr2 + j + 8)); + sum_u16[3] = vabaq_u16(sum_u16[3], s1, vld1q_u16(ref16_ptr3 + j + 8)); - uint16x8_t s3 = vld1q_u16(src16_ptr + i * src_stride + j + 24); - sad8_neon(s3, vld1q_u16(ref16_ptr0 + i * ref_stride + j + 24), - &sum_hi[0]); - sad8_neon(s3, vld1q_u16(ref16_ptr1 + i * ref_stride + j + 24), - &sum_hi[1]); - sad8_neon(s3, vld1q_u16(ref16_ptr2 + i * ref_stride + j + 24), - &sum_hi[2]); - sad8_neon(s3, vld1q_u16(ref16_ptr3 + i * ref_stride + j + 24), - &sum_hi[3]); + j += 16; + } while (j < w); - j += 32; - } while (j < w); + src16_ptr += src_stride; + ref16_ptr0 += ref_stride; + ref16_ptr1 += ref_stride; + ref16_ptr2 += ref_stride; + ref16_ptr3 += ref_stride; + } while (--i != 0); - } while (++i < h); + sum_u32[0] = vpadalq_u16(sum_u32[0], sum_u16[0]); + sum_u32[1] = vpadalq_u16(sum_u32[1], sum_u16[1]); + sum_u32[2] = vpadalq_u16(sum_u32[2], sum_u16[2]); + sum_u32[3] = vpadalq_u16(sum_u32[3], sum_u16[3]); - sum[0] = vaddq_u32(sum_lo[0], sum_hi[0]); - sum[1] = vaddq_u32(sum_lo[1], sum_hi[1]); - sum[2] = vaddq_u32(sum_lo[2], sum_hi[2]); - sum[3] = vaddq_u32(sum_lo[3], sum_hi[3]); + h -= h_limit; + } while (h != 0); - vst1q_u32(res, horizontal_add_4d_u32x4(sum)); + vst1q_u32(res, horizontal_add_4d_u32x4(sum_u32)); } -static inline void highbd_sad128xhx4d_large_neon( - const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_ptr[4], - int ref_stride, uint32_t res[4], int h) { - highbd_sadwxhx4d_large_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, - 128, h); +static inline void highbd_sad16xhx4d_neon(const uint8_t *src_ptr, + int src_stride, + const uint8_t *const ref_ptr[4], + int ref_stride, uint32_t res[4], + int h) { + // 'h_overflow' is the number of 16-wide rows we can process before 16-bit + // accumulators overflow. After hitting this limit accumulate into 32-bit + // elements. 65535 / 4095 ~= 16, so 8 16-wide rows. + const int h_overflow = 8; + highbd_sadwxhx4d_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 16, h, + h_overflow); } -static inline void highbd_sad64xhx4d_large_neon(const uint8_t *src_ptr, - int src_stride, - const uint8_t *const ref_ptr[4], - int ref_stride, uint32_t res[4], - int h) { - highbd_sadwxhx4d_large_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 64, - h); +static inline void highbd_sad32xhx4d_neon(const uint8_t *src_ptr, + int src_stride, + const uint8_t *const ref_ptr[4], + int ref_stride, uint32_t res[4], + int h) { + // 'h_overflow' is the number of 32-wide rows we can process before 16-bit + // accumulators overflow. After hitting this limit accumulate into 32-bit + // elements. 65535 / 4095 ~= 16, so 4 32-wide rows. + const int h_overflow = 4; + highbd_sadwxhx4d_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 32, h, + h_overflow); } -static inline void highbd_sad32xhx4d_large_neon(const uint8_t *src_ptr, - int src_stride, - const uint8_t *const ref_ptr[4], - int ref_stride, uint32_t res[4], - int h) { - highbd_sadwxhx4d_large_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 32, - h); +static inline void highbd_sad64xhx4d_neon(const uint8_t *src_ptr, + int src_stride, + const uint8_t *const ref_ptr[4], + int ref_stride, uint32_t res[4], + int h) { + // 'h_overflow' is the number of 64-wide rows we can process before 16-bit + // accumulators overflow. After hitting this limit accumulate into 32-bit + // elements. 65535 / 4095 ~= 16, so 2 64-wide rows. + const int h_overflow = 2; + highbd_sadwxhx4d_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 64, h, + h_overflow); } -#define HBD_SAD_WXH_4D_SMALL_NEON(w, h) \ +static inline void highbd_sad128xhx4d_neon(const uint8_t *src_ptr, + int src_stride, + const uint8_t *const ref_ptr[4], + int ref_stride, uint32_t res[4], + int h) { + // 'h_overflow' is the number of 128-wide rows we can process before 16-bit + // accumulators overflow. After hitting this limit accumulate into 32-bit + // elements. 65535 / 4095 ~= 16, so 1 128-wide rows. + const int h_overflow = 1; + highbd_sadwxhx4d_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 128, h, + h_overflow); +} + +#define HBD_SAD_WXH_4D_NEON(w, h) \ void aom_highbd_sad##w##x##h##x4d_neon( \ const uint8_t *src, int src_stride, const uint8_t *const ref_array[4], \ int ref_stride, uint32_t sad_array[4]) { \ - highbd_sad##w##xhx4d_small_neon(src, src_stride, ref_array, ref_stride, \ - sad_array, (h)); \ + highbd_sad##w##xhx4d_neon(src, src_stride, ref_array, ref_stride, \ + sad_array, (h)); \ } -#define HBD_SAD_WXH_4D_LARGE_NEON(w, h) \ - void aom_highbd_sad##w##x##h##x4d_neon( \ - const uint8_t *src, int src_stride, const uint8_t *const ref_array[4], \ - int ref_stride, uint32_t sad_array[4]) { \ - highbd_sad##w##xhx4d_large_neon(src, src_stride, ref_array, ref_stride, \ - sad_array, (h)); \ - } +HBD_SAD_WXH_4D_NEON(4, 4) +HBD_SAD_WXH_4D_NEON(4, 8) -HBD_SAD_WXH_4D_SMALL_NEON(4, 4) -HBD_SAD_WXH_4D_SMALL_NEON(4, 8) +HBD_SAD_WXH_4D_NEON(8, 4) +HBD_SAD_WXH_4D_NEON(8, 8) +HBD_SAD_WXH_4D_NEON(8, 16) -HBD_SAD_WXH_4D_SMALL_NEON(8, 4) -HBD_SAD_WXH_4D_SMALL_NEON(8, 8) -HBD_SAD_WXH_4D_SMALL_NEON(8, 16) +HBD_SAD_WXH_4D_NEON(16, 8) +HBD_SAD_WXH_4D_NEON(16, 16) +HBD_SAD_WXH_4D_NEON(16, 32) -HBD_SAD_WXH_4D_LARGE_NEON(16, 8) -HBD_SAD_WXH_4D_LARGE_NEON(16, 16) -HBD_SAD_WXH_4D_LARGE_NEON(16, 32) +HBD_SAD_WXH_4D_NEON(32, 16) +HBD_SAD_WXH_4D_NEON(32, 32) +HBD_SAD_WXH_4D_NEON(32, 64) -HBD_SAD_WXH_4D_LARGE_NEON(32, 16) -HBD_SAD_WXH_4D_LARGE_NEON(32, 32) -HBD_SAD_WXH_4D_LARGE_NEON(32, 64) +HBD_SAD_WXH_4D_NEON(64, 32) +HBD_SAD_WXH_4D_NEON(64, 64) +HBD_SAD_WXH_4D_NEON(64, 128) -HBD_SAD_WXH_4D_LARGE_NEON(64, 32) -HBD_SAD_WXH_4D_LARGE_NEON(64, 64) -HBD_SAD_WXH_4D_LARGE_NEON(64, 128) - -HBD_SAD_WXH_4D_LARGE_NEON(128, 64) -HBD_SAD_WXH_4D_LARGE_NEON(128, 128) +HBD_SAD_WXH_4D_NEON(128, 64) +HBD_SAD_WXH_4D_NEON(128, 128) #if !CONFIG_REALTIME_ONLY -HBD_SAD_WXH_4D_SMALL_NEON(4, 16) +HBD_SAD_WXH_4D_NEON(4, 16) -HBD_SAD_WXH_4D_LARGE_NEON(8, 32) +HBD_SAD_WXH_4D_NEON(8, 32) -HBD_SAD_WXH_4D_LARGE_NEON(16, 4) -HBD_SAD_WXH_4D_LARGE_NEON(16, 64) +HBD_SAD_WXH_4D_NEON(16, 4) +HBD_SAD_WXH_4D_NEON(16, 64) -HBD_SAD_WXH_4D_LARGE_NEON(32, 8) +HBD_SAD_WXH_4D_NEON(32, 8) -HBD_SAD_WXH_4D_LARGE_NEON(64, 16) +HBD_SAD_WXH_4D_NEON(64, 16) #endif // !CONFIG_REALTIME_ONLY -#define HBD_SAD_SKIP_WXH_4D_SMALL_NEON(w, h) \ - void aom_highbd_sad_skip_##w##x##h##x4d_neon( \ - const uint8_t *src, int src_stride, const uint8_t *const ref_array[4], \ - int ref_stride, uint32_t sad_array[4]) { \ - highbd_sad##w##xhx4d_small_neon(src, 2 * src_stride, ref_array, \ - 2 * ref_stride, sad_array, ((h) >> 1)); \ - sad_array[0] <<= 1; \ - sad_array[1] <<= 1; \ - sad_array[2] <<= 1; \ - sad_array[3] <<= 1; \ +#undef HBD_SAD_WXH_4D_NEON + +#define HBD_SAD_SKIP_WXH_4D_NEON(w, h) \ + void aom_highbd_sad_skip_##w##x##h##x4d_neon( \ + const uint8_t *src, int src_stride, const uint8_t *const ref_array[4], \ + int ref_stride, uint32_t sad_array[4]) { \ + highbd_sad##w##xhx4d_neon(src, 2 * src_stride, ref_array, 2 * ref_stride, \ + sad_array, ((h) >> 1)); \ + sad_array[0] <<= 1; \ + sad_array[1] <<= 1; \ + sad_array[2] <<= 1; \ + sad_array[3] <<= 1; \ } -#define HBD_SAD_SKIP_WXH_4D_LARGE_NEON(w, h) \ - void aom_highbd_sad_skip_##w##x##h##x4d_neon( \ - const uint8_t *src, int src_stride, const uint8_t *const ref_array[4], \ - int ref_stride, uint32_t sad_array[4]) { \ - highbd_sad##w##xhx4d_large_neon(src, 2 * src_stride, ref_array, \ - 2 * ref_stride, sad_array, ((h) >> 1)); \ - sad_array[0] <<= 1; \ - sad_array[1] <<= 1; \ - sad_array[2] <<= 1; \ - sad_array[3] <<= 1; \ - } +HBD_SAD_SKIP_WXH_4D_NEON(8, 16) -HBD_SAD_SKIP_WXH_4D_SMALL_NEON(8, 16) +HBD_SAD_SKIP_WXH_4D_NEON(16, 16) +HBD_SAD_SKIP_WXH_4D_NEON(16, 32) -HBD_SAD_SKIP_WXH_4D_LARGE_NEON(16, 16) -HBD_SAD_SKIP_WXH_4D_LARGE_NEON(16, 32) +HBD_SAD_SKIP_WXH_4D_NEON(32, 16) +HBD_SAD_SKIP_WXH_4D_NEON(32, 32) +HBD_SAD_SKIP_WXH_4D_NEON(32, 64) -HBD_SAD_SKIP_WXH_4D_LARGE_NEON(32, 16) -HBD_SAD_SKIP_WXH_4D_LARGE_NEON(32, 32) -HBD_SAD_SKIP_WXH_4D_LARGE_NEON(32, 64) +HBD_SAD_SKIP_WXH_4D_NEON(64, 32) +HBD_SAD_SKIP_WXH_4D_NEON(64, 64) +HBD_SAD_SKIP_WXH_4D_NEON(64, 128) -HBD_SAD_SKIP_WXH_4D_LARGE_NEON(64, 32) -HBD_SAD_SKIP_WXH_4D_LARGE_NEON(64, 64) -HBD_SAD_SKIP_WXH_4D_LARGE_NEON(64, 128) - -HBD_SAD_SKIP_WXH_4D_LARGE_NEON(128, 64) -HBD_SAD_SKIP_WXH_4D_LARGE_NEON(128, 128) +HBD_SAD_SKIP_WXH_4D_NEON(128, 64) +HBD_SAD_SKIP_WXH_4D_NEON(128, 128) #if !CONFIG_REALTIME_ONLY -HBD_SAD_SKIP_WXH_4D_SMALL_NEON(4, 16) +HBD_SAD_SKIP_WXH_4D_NEON(4, 16) -HBD_SAD_SKIP_WXH_4D_SMALL_NEON(8, 32) +HBD_SAD_SKIP_WXH_4D_NEON(8, 32) -HBD_SAD_SKIP_WXH_4D_LARGE_NEON(16, 64) +HBD_SAD_SKIP_WXH_4D_NEON(16, 64) -HBD_SAD_SKIP_WXH_4D_LARGE_NEON(64, 16) +HBD_SAD_SKIP_WXH_4D_NEON(64, 16) #endif // !CONFIG_REALTIME_ONLY -static inline void highbd_sad4xhx3d_small_neon(const uint8_t *src_ptr, - int src_stride, - const uint8_t *const ref_ptr[4], - int ref_stride, uint32_t res[4], - int h) { +#undef HBD_SAD_SKIP_WXH_4D_NEON + +static inline void highbd_sad4xhx3d_neon(const uint8_t *src_ptr, int src_stride, + const uint8_t *const ref_ptr[4], + int ref_stride, uint32_t res[4], + int h) { const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr); const uint16_t *ref16_ptr0 = CONVERT_TO_SHORTPTR(ref_ptr[0]); const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]); const uint16_t *ref16_ptr2 = CONVERT_TO_SHORTPTR(ref_ptr[2]); - uint32x4_t sum[3] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0) }; int i = 0; @@ -382,227 +325,198 @@ res[2] = horizontal_add_u32x4(sum[2]); } -static inline void highbd_sad8xhx3d_small_neon(const uint8_t *src_ptr, - int src_stride, - const uint8_t *const ref_ptr[4], - int ref_stride, uint32_t res[4], - int h) { +static inline void highbd_sad8xhx3d_neon(const uint8_t *src_ptr, int src_stride, + const uint8_t *const ref_ptr[4], + int ref_stride, uint32_t res[4], + int h) { const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr); const uint16_t *ref16_ptr0 = CONVERT_TO_SHORTPTR(ref_ptr[0]); const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]); const uint16_t *ref16_ptr2 = CONVERT_TO_SHORTPTR(ref_ptr[2]); - uint16x8_t sum[3] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0) }; + // 'h_overflow' is the number of 8-wide rows we can process before 16-bit + // accumulators overflow. After hitting this limit accumulate into 32-bit + // elements. 65535 / 4095 ~= 16, so 16 8-wide rows. + const int h_overflow = 16; + // If block height 'h' is smaller than this limit, use 'h' instead. + const int h_limit = h < h_overflow ? h : h_overflow; + assert(h % h_limit == 0); + uint32x4_t sum_u32[3] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0) }; + + int h_tmp = h_limit; int i = 0; do { - uint16x8_t s = vld1q_u16(src16_ptr + i * src_stride); - - sum[0] = vabaq_u16(sum[0], s, vld1q_u16(ref16_ptr0 + i * ref_stride)); - sum[1] = vabaq_u16(sum[1], s, vld1q_u16(ref16_ptr1 + i * ref_stride)); - sum[2] = vabaq_u16(sum[2], s, vld1q_u16(ref16_ptr2 + i * ref_stride)); - - } while (++i < h); - - res[0] = horizontal_add_u32x4(vpaddlq_u16(sum[0])); - res[1] = horizontal_add_u32x4(vpaddlq_u16(sum[1])); - res[2] = horizontal_add_u32x4(vpaddlq_u16(sum[2])); -} - -#if !CONFIG_REALTIME_ONLY -static inline void highbd_sad8xhx3d_large_neon(const uint8_t *src_ptr, - int src_stride, - const uint8_t *const ref_ptr[4], - int ref_stride, uint32_t res[4], - int h) { - const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr); - const uint16_t *ref16_ptr0 = CONVERT_TO_SHORTPTR(ref_ptr[0]); - const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]); - const uint16_t *ref16_ptr2 = CONVERT_TO_SHORTPTR(ref_ptr[2]); - - uint32x4_t sum[3] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0) }; - - int i = 0; - do { - uint16x8_t s = vld1q_u16(src16_ptr + i * src_stride); - uint16x8_t r0 = vld1q_u16(ref16_ptr0 + i * ref_stride); - uint16x8_t r1 = vld1q_u16(ref16_ptr1 + i * ref_stride); - uint16x8_t r2 = vld1q_u16(ref16_ptr2 + i * ref_stride); - - sad8_neon(s, r0, &sum[0]); - sad8_neon(s, r1, &sum[1]); - sad8_neon(s, r2, &sum[2]); - - } while (++i < h); - - res[0] = horizontal_add_u32x4(sum[0]); - res[1] = horizontal_add_u32x4(sum[1]); - res[2] = horizontal_add_u32x4(sum[2]); -} -#endif // !CONFIG_REALTIME_ONLY - -static inline void highbd_sad16xhx3d_large_neon(const uint8_t *src_ptr, - int src_stride, - const uint8_t *const ref_ptr[4], - int ref_stride, uint32_t res[4], - int h) { - const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr); - const uint16_t *ref16_ptr0 = CONVERT_TO_SHORTPTR(ref_ptr[0]); - const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]); - const uint16_t *ref16_ptr2 = CONVERT_TO_SHORTPTR(ref_ptr[2]); - - uint32x4_t sum_lo[3] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0) }; - uint32x4_t sum_hi[3] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0) }; - - int i = 0; - do { - uint16x8_t s0 = vld1q_u16(src16_ptr + i * src_stride); - sad8_neon(s0, vld1q_u16(ref16_ptr0 + i * ref_stride), &sum_lo[0]); - sad8_neon(s0, vld1q_u16(ref16_ptr1 + i * ref_stride), &sum_lo[1]); - sad8_neon(s0, vld1q_u16(ref16_ptr2 + i * ref_stride), &sum_lo[2]); - - uint16x8_t s1 = vld1q_u16(src16_ptr + i * src_stride + 8); - sad8_neon(s1, vld1q_u16(ref16_ptr0 + i * ref_stride + 8), &sum_hi[0]); - sad8_neon(s1, vld1q_u16(ref16_ptr1 + i * ref_stride + 8), &sum_hi[1]); - sad8_neon(s1, vld1q_u16(ref16_ptr2 + i * ref_stride + 8), &sum_hi[2]); - - } while (++i < h); - - res[0] = horizontal_add_u32x4(vaddq_u32(sum_lo[0], sum_hi[0])); - res[1] = horizontal_add_u32x4(vaddq_u32(sum_lo[1], sum_hi[1])); - res[2] = horizontal_add_u32x4(vaddq_u32(sum_lo[2], sum_hi[2])); -} - -static inline void highbd_sadwxhx3d_large_neon(const uint8_t *src_ptr, - int src_stride, - const uint8_t *const ref_ptr[4], - int ref_stride, uint32_t res[4], - int w, int h) { - const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr); - const uint16_t *ref16_ptr0 = CONVERT_TO_SHORTPTR(ref_ptr[0]); - const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]); - const uint16_t *ref16_ptr2 = CONVERT_TO_SHORTPTR(ref_ptr[2]); - - uint32x4_t sum_lo[3] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0) }; - uint32x4_t sum_hi[3] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0) }; - uint32x4_t sum[3]; - - int i = 0; - do { - int j = 0; + uint16x8_t sum_u16[3] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0) }; do { - uint16x8_t s0 = vld1q_u16(src16_ptr + i * src_stride + j); - sad8_neon(s0, vld1q_u16(ref16_ptr0 + i * ref_stride + j), &sum_lo[0]); - sad8_neon(s0, vld1q_u16(ref16_ptr1 + i * ref_stride + j), &sum_lo[1]); - sad8_neon(s0, vld1q_u16(ref16_ptr2 + i * ref_stride + j), &sum_lo[2]); + uint16x8_t s0 = vld1q_u16(src16_ptr + i * src_stride); - uint16x8_t s1 = vld1q_u16(src16_ptr + i * src_stride + j + 8); - sad8_neon(s1, vld1q_u16(ref16_ptr0 + i * ref_stride + j + 8), &sum_hi[0]); - sad8_neon(s1, vld1q_u16(ref16_ptr1 + i * ref_stride + j + 8), &sum_hi[1]); - sad8_neon(s1, vld1q_u16(ref16_ptr2 + i * ref_stride + j + 8), &sum_hi[2]); + sum_u16[0] = + vabaq_u16(sum_u16[0], s0, vld1q_u16(ref16_ptr0 + i * ref_stride)); + sum_u16[1] = + vabaq_u16(sum_u16[1], s0, vld1q_u16(ref16_ptr1 + i * ref_stride)); + sum_u16[2] = + vabaq_u16(sum_u16[2], s0, vld1q_u16(ref16_ptr2 + i * ref_stride)); + } while (++i < h_tmp); - uint16x8_t s2 = vld1q_u16(src16_ptr + i * src_stride + j + 16); - sad8_neon(s2, vld1q_u16(ref16_ptr0 + i * ref_stride + j + 16), - &sum_lo[0]); - sad8_neon(s2, vld1q_u16(ref16_ptr1 + i * ref_stride + j + 16), - &sum_lo[1]); - sad8_neon(s2, vld1q_u16(ref16_ptr2 + i * ref_stride + j + 16), - &sum_lo[2]); + sum_u32[0] = vpadalq_u16(sum_u32[0], sum_u16[0]); + sum_u32[1] = vpadalq_u16(sum_u32[1], sum_u16[1]); + sum_u32[2] = vpadalq_u16(sum_u32[2], sum_u16[2]); - uint16x8_t s3 = vld1q_u16(src16_ptr + i * src_stride + j + 24); - sad8_neon(s3, vld1q_u16(ref16_ptr0 + i * ref_stride + j + 24), - &sum_hi[0]); - sad8_neon(s3, vld1q_u16(ref16_ptr1 + i * ref_stride + j + 24), - &sum_hi[1]); - sad8_neon(s3, vld1q_u16(ref16_ptr2 + i * ref_stride + j + 24), - &sum_hi[2]); + h_tmp += h_limit; + h -= h_limit; + } while (h != 0); - j += 32; - } while (j < w); - - } while (++i < h); - - sum[0] = vaddq_u32(sum_lo[0], sum_hi[0]); - sum[1] = vaddq_u32(sum_lo[1], sum_hi[1]); - sum[2] = vaddq_u32(sum_lo[2], sum_hi[2]); - - res[0] = horizontal_add_u32x4(sum[0]); - res[1] = horizontal_add_u32x4(sum[1]); - res[2] = horizontal_add_u32x4(sum[2]); + res[0] = horizontal_add_u32x4(sum_u32[0]); + res[1] = horizontal_add_u32x4(sum_u32[1]); + res[2] = horizontal_add_u32x4(sum_u32[2]); } -static inline void highbd_sad128xhx3d_large_neon( - const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_ptr[4], - int ref_stride, uint32_t res[4], int h) { - highbd_sadwxhx3d_large_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, - 128, h); +static inline void highbd_sadwxhx3d_neon(const uint8_t *src_ptr, int src_stride, + const uint8_t *const ref_ptr[4], + int ref_stride, uint32_t res[4], int w, + int h, const int h_overflow) { + const uint16_t *src16_ptr = CONVERT_TO_SHORTPTR(src_ptr); + const uint16_t *ref16_ptr0 = CONVERT_TO_SHORTPTR(ref_ptr[0]); + const uint16_t *ref16_ptr1 = CONVERT_TO_SHORTPTR(ref_ptr[1]); + const uint16_t *ref16_ptr2 = CONVERT_TO_SHORTPTR(ref_ptr[2]); + uint32x4_t sum_u32[3] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0) }; + + const int h_limit = h < h_overflow ? h : h_overflow; + assert(h % h_limit == 0); + + do { + uint16x8_t sum_u16[3] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0) }; + + int i = h_limit; + do { + int j = 0; + do { + uint16x8_t s0 = vld1q_u16(src16_ptr + j); + + sum_u16[0] = vabaq_u16(sum_u16[0], s0, vld1q_u16(ref16_ptr0 + j)); + sum_u16[1] = vabaq_u16(sum_u16[1], s0, vld1q_u16(ref16_ptr1 + j)); + sum_u16[2] = vabaq_u16(sum_u16[2], s0, vld1q_u16(ref16_ptr2 + j)); + + uint16x8_t s1 = vld1q_u16(src16_ptr + j + 8); + sum_u16[0] = vabaq_u16(sum_u16[0], s1, vld1q_u16(ref16_ptr0 + j + 8)); + sum_u16[1] = vabaq_u16(sum_u16[1], s1, vld1q_u16(ref16_ptr1 + j + 8)); + sum_u16[2] = vabaq_u16(sum_u16[2], s1, vld1q_u16(ref16_ptr2 + j + 8)); + + j += 16; + } while (j < w); + + src16_ptr += src_stride; + ref16_ptr0 += ref_stride; + ref16_ptr1 += ref_stride; + ref16_ptr2 += ref_stride; + } while (--i != 0); + + sum_u32[0] = vpadalq_u16(sum_u32[0], sum_u16[0]); + sum_u32[1] = vpadalq_u16(sum_u32[1], sum_u16[1]); + sum_u32[2] = vpadalq_u16(sum_u32[2], sum_u16[2]); + + h -= h_limit; + } while (h != 0); + + res[0] = horizontal_add_u32x4(sum_u32[0]); + res[1] = horizontal_add_u32x4(sum_u32[1]); + res[2] = horizontal_add_u32x4(sum_u32[2]); } -static inline void highbd_sad64xhx3d_large_neon(const uint8_t *src_ptr, - int src_stride, - const uint8_t *const ref_ptr[4], - int ref_stride, uint32_t res[4], - int h) { - highbd_sadwxhx3d_large_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 64, - h); +static inline void highbd_sad16xhx3d_neon(const uint8_t *src_ptr, + int src_stride, + const uint8_t *const ref_ptr[4], + int ref_stride, uint32_t res[4], + int h) { + // 'h_overflow' is the number of 16-wide rows we can process before 16-bit + // accumulators overflow. After hitting this limit accumulate into 32-bit + // elements. 65535 / 4095 ~= 16, so 8 16-wide rows. + const int h_overflow = 8; + highbd_sadwxhx3d_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 16, h, + h_overflow); } -static inline void highbd_sad32xhx3d_large_neon(const uint8_t *src_ptr, - int src_stride, - const uint8_t *const ref_ptr[4], - int ref_stride, uint32_t res[4], - int h) { - highbd_sadwxhx3d_large_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 32, - h); +static inline void highbd_sad32xhx3d_neon(const uint8_t *src_ptr, + int src_stride, + const uint8_t *const ref_ptr[4], + int ref_stride, uint32_t res[4], + int h) { + // 'h_overflow' is the number of 32-wide rows we can process before 16-bit + // accumulators overflow. After hitting this limit accumulate into 32-bit + // elements. 65535 / 4095 ~= 16, so 4 32-wide rows. + const int h_overflow = 4; + highbd_sadwxhx3d_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 32, h, + h_overflow); } -#define HBD_SAD_WXH_3D_SMALL_NEON(w, h) \ +static inline void highbd_sad64xhx3d_neon(const uint8_t *src_ptr, + int src_stride, + const uint8_t *const ref_ptr[4], + int ref_stride, uint32_t res[4], + int h) { + // 'h_overflow' is the number of 64-wide rows we can process before 16-bit + // accumulators overflow. After hitting this limit accumulate into 32-bit + // elements. 65535 / 4095 ~= 16, so 2 64-wide rows. + const int h_overflow = 2; + highbd_sadwxhx3d_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 64, h, + h_overflow); +} + +static inline void highbd_sad128xhx3d_neon(const uint8_t *src_ptr, + int src_stride, + const uint8_t *const ref_ptr[4], + int ref_stride, uint32_t res[4], + int h) { + // 'h_overflow' is the number of 128-wide rows we can process before 16-bit + // accumulators overflow. After hitting this limit accumulate into 32-bit + // elements. 65535 / 4095 ~= 16, so 1 128-wide rows. + const int h_overflow = 1; + highbd_sadwxhx3d_neon(src_ptr, src_stride, ref_ptr, ref_stride, res, 128, h, + h_overflow); +} + +#define HBD_SAD_WXH_3D_NEON(w, h) \ void aom_highbd_sad##w##x##h##x3d_neon( \ const uint8_t *src, int src_stride, const uint8_t *const ref_array[4], \ int ref_stride, uint32_t sad_array[4]) { \ - highbd_sad##w##xhx3d_small_neon(src, src_stride, ref_array, ref_stride, \ - sad_array, (h)); \ + highbd_sad##w##xhx3d_neon(src, src_stride, ref_array, ref_stride, \ + sad_array, (h)); \ } -#define HBD_SAD_WXH_3D_LARGE_NEON(w, h) \ - void aom_highbd_sad##w##x##h##x3d_neon( \ - const uint8_t *src, int src_stride, const uint8_t *const ref_array[4], \ - int ref_stride, uint32_t sad_array[4]) { \ - highbd_sad##w##xhx3d_large_neon(src, src_stride, ref_array, ref_stride, \ - sad_array, (h)); \ - } +HBD_SAD_WXH_3D_NEON(4, 4) +HBD_SAD_WXH_3D_NEON(4, 8) -HBD_SAD_WXH_3D_SMALL_NEON(4, 4) -HBD_SAD_WXH_3D_SMALL_NEON(4, 8) +HBD_SAD_WXH_3D_NEON(8, 4) +HBD_SAD_WXH_3D_NEON(8, 8) +HBD_SAD_WXH_3D_NEON(8, 16) -HBD_SAD_WXH_3D_SMALL_NEON(8, 4) -HBD_SAD_WXH_3D_SMALL_NEON(8, 8) -HBD_SAD_WXH_3D_SMALL_NEON(8, 16) +HBD_SAD_WXH_3D_NEON(16, 8) +HBD_SAD_WXH_3D_NEON(16, 16) +HBD_SAD_WXH_3D_NEON(16, 32) -HBD_SAD_WXH_3D_LARGE_NEON(16, 8) -HBD_SAD_WXH_3D_LARGE_NEON(16, 16) -HBD_SAD_WXH_3D_LARGE_NEON(16, 32) +HBD_SAD_WXH_3D_NEON(32, 16) +HBD_SAD_WXH_3D_NEON(32, 32) +HBD_SAD_WXH_3D_NEON(32, 64) -HBD_SAD_WXH_3D_LARGE_NEON(32, 16) -HBD_SAD_WXH_3D_LARGE_NEON(32, 32) -HBD_SAD_WXH_3D_LARGE_NEON(32, 64) +HBD_SAD_WXH_3D_NEON(64, 32) +HBD_SAD_WXH_3D_NEON(64, 64) +HBD_SAD_WXH_3D_NEON(64, 128) -HBD_SAD_WXH_3D_LARGE_NEON(64, 32) -HBD_SAD_WXH_3D_LARGE_NEON(64, 64) -HBD_SAD_WXH_3D_LARGE_NEON(64, 128) - -HBD_SAD_WXH_3D_LARGE_NEON(128, 64) -HBD_SAD_WXH_3D_LARGE_NEON(128, 128) +HBD_SAD_WXH_3D_NEON(128, 64) +HBD_SAD_WXH_3D_NEON(128, 128) #if !CONFIG_REALTIME_ONLY -HBD_SAD_WXH_3D_SMALL_NEON(4, 16) +HBD_SAD_WXH_3D_NEON(4, 16) -HBD_SAD_WXH_3D_LARGE_NEON(8, 32) +HBD_SAD_WXH_3D_NEON(8, 32) -HBD_SAD_WXH_3D_LARGE_NEON(16, 4) -HBD_SAD_WXH_3D_LARGE_NEON(16, 64) +HBD_SAD_WXH_3D_NEON(16, 4) +HBD_SAD_WXH_3D_NEON(16, 64) -HBD_SAD_WXH_3D_LARGE_NEON(32, 8) +HBD_SAD_WXH_3D_NEON(32, 8) -HBD_SAD_WXH_3D_LARGE_NEON(64, 16) +HBD_SAD_WXH_3D_NEON(64, 16) #endif // !CONFIG_REALTIME_ONLY + +#undef HBD_SAD_WXH_3D_NEON
diff --git a/aom_dsp/arm/intrapred_neon.c b/aom_dsp/arm/intrapred_neon.c index f4f7591..be63638 100644 --- a/aom_dsp/arm/intrapred_neon.c +++ b/aom_dsp/arm/intrapred_neon.c
@@ -1170,132 +1170,115 @@ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, }; -static AOM_FORCE_INLINE void dr_prediction_z1_HxW_internal_neon_64( - int H, int W, uint8x8_t *dst, const uint8_t *above, int upsample_above, +static AOM_FORCE_INLINE void dr_prediction_z1_WxH_internal_neon_small( + int W, int H, uint8x8_t *dst, const uint8_t *above, int upsample_above, int dx) { const int frac_bits = 6 - upsample_above; const int max_base_x = ((W + H) - 1) << upsample_above; assert(dx > 0); - // pre-filter above pixels - // store in temp buffers: - // above[x] * 32 + 16 - // above[x+1] - above[x] - // final pixels will be calculated as: - // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 + // above0 * (32 - shift) + above1 * shift + // = (above1 - above0) * shift + above0 * 32 const uint8x8_t a_mbase_x = vdup_n_u8(above[max_base_x]); int x = dx; - for (int r = 0; r < W; r++) { + for (int r = 0; r < H; r++) { int base = x >> frac_bits; int base_max_diff = (max_base_x - base) >> upsample_above; if (base_max_diff <= 0) { - for (int i = r; i < W; ++i) { + for (int i = r; i < H; ++i) { dst[i] = a_mbase_x; // save 4 values } return; } - if (base_max_diff > H) base_max_diff = H; + if (base_max_diff > W) base_max_diff = W; uint8x8x2_t a01_128; uint16x8_t shift; if (upsample_above) { a01_128 = vld2_u8(above + base); - shift = vdupq_n_u16(((x << upsample_above) & 0x3f) >> 1); + shift = vdupq_n_u16(x & 0x1f); // ((x << upsample_above) & 0x3f) >> 1 } else { a01_128.val[0] = vld1_u8(above + base); a01_128.val[1] = vld1_u8(above + base + 1); shift = vdupq_n_u16((x & 0x3f) >> 1); } uint16x8_t diff = vsubl_u8(a01_128.val[1], a01_128.val[0]); - uint16x8_t a32 = vmlal_u8(vdupq_n_u16(16), a01_128.val[0], vdup_n_u8(32)); + uint16x8_t a32 = vshll_n_u8(a01_128.val[0], 5); uint16x8_t res = vmlaq_u16(a32, diff, shift); uint8x8_t mask = vld1_u8(BaseMask[base_max_diff]); - dst[r] = vbsl_u8(mask, vshrn_n_u16(res, 5), a_mbase_x); + dst[r] = vbsl_u8(mask, vrshrn_n_u16(res, 5), a_mbase_x); x += dx; } } -static void dr_prediction_z1_4xN_neon(int N, uint8_t *dst, ptrdiff_t stride, +static void dr_prediction_z1_4xH_neon(int H, uint8_t *dst, ptrdiff_t stride, const uint8_t *above, int upsample_above, int dx) { uint8x8_t dstvec[16]; - dr_prediction_z1_HxW_internal_neon_64(4, N, dstvec, above, upsample_above, - dx); - for (int i = 0; i < N; i++) { + dr_prediction_z1_WxH_internal_neon_small(4, H, dstvec, above, upsample_above, + dx); + for (int i = 0; i < H; i++) { vst1_lane_u32((uint32_t *)(dst + stride * i), vreinterpret_u32_u8(dstvec[i]), 0); } } -static void dr_prediction_z1_8xN_neon(int N, uint8_t *dst, ptrdiff_t stride, +static void dr_prediction_z1_8xH_neon(int H, uint8_t *dst, ptrdiff_t stride, const uint8_t *above, int upsample_above, int dx) { uint8x8_t dstvec[32]; - dr_prediction_z1_HxW_internal_neon_64(8, N, dstvec, above, upsample_above, - dx); - for (int i = 0; i < N; i++) { + dr_prediction_z1_WxH_internal_neon_small(8, H, dstvec, above, upsample_above, + dx); + for (int i = 0; i < H; i++) { vst1_u8(dst + stride * i, dstvec[i]); } } -static AOM_FORCE_INLINE void dr_prediction_z1_HxW_internal_neon( - int H, int W, uint8x16_t *dst, const uint8_t *above, int upsample_above, - int dx) { - const int frac_bits = 6 - upsample_above; - const int max_base_x = ((W + H) - 1) << upsample_above; +static AOM_FORCE_INLINE void dr_prediction_z1_WxH_internal_neon_large( + int W, int H, uint8x16_t *dst, const uint8_t *above, int dx) { + // Here upsample_above is 0 by design of av1_use_intra_edge_upsample defined + // in av1/common/reconintra.h. + const int frac_bits = 6; + const int max_base_x = (W + H) - 1; assert(dx > 0); - // pre-filter above pixels - // store in temp buffers: - // above[x] * 32 + 16 - // above[x+1] - above[x] - // final pixels will be calculated as: - // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 + // above0 * (32 - shift) + above1 * shift + // = (above1 - above0) * shift + above0 * 32 const uint8x16_t a_mbase_x = vdupq_n_u8(above[max_base_x]); int x = dx; - for (int r = 0; r < W; r++) { + for (int r = 0; r < H; r++) { int base = x >> frac_bits; - int base_max_diff = (max_base_x - base) >> upsample_above; + int base_max_diff = max_base_x - base; if (base_max_diff <= 0) { - for (int i = r; i < W; ++i) { + for (int i = r; i < H; ++i) { dst[i] = a_mbase_x; // save 4 values } return; } - if (base_max_diff > H) base_max_diff = H; + if (base_max_diff > W) base_max_diff = W; - uint16x8_t shift; - uint8x16_t a0_128, a1_128; - if (upsample_above) { - uint8x8x2_t v_tmp_a0_128 = vld2_u8(above + base); - a0_128 = vcombine_u8(v_tmp_a0_128.val[0], v_tmp_a0_128.val[1]); - a1_128 = vextq_u8(a0_128, vdupq_n_u8(0), 8); - shift = vdupq_n_u16(x & 0x1f); - } else { - a0_128 = vld1q_u8(above + base); - a1_128 = vld1q_u8(above + base + 1); - shift = vdupq_n_u16((x & 0x3f) >> 1); - } + uint8x16_t a0_128 = vld1q_u8(above + base); + uint8x16_t a1_128 = vld1q_u8(above + base + 1); + uint16x8_t shift = vdupq_n_u16((x & 0x3f) >> 1); + uint16x8_t diff_lo = vsubl_u8(vget_low_u8(a1_128), vget_low_u8(a0_128)); uint16x8_t diff_hi = vsubl_u8(vget_high_u8(a1_128), vget_high_u8(a0_128)); - uint16x8_t a32_lo = - vmlal_u8(vdupq_n_u16(16), vget_low_u8(a0_128), vdup_n_u8(32)); - uint16x8_t a32_hi = - vmlal_u8(vdupq_n_u16(16), vget_high_u8(a0_128), vdup_n_u8(32)); + uint16x8_t a32_lo = vshll_n_u8(vget_low_u8(a0_128), 5); + uint16x8_t a32_hi = vshll_n_u8(vget_high_u8(a0_128), 5); uint16x8_t res_lo = vmlaq_u16(a32_lo, diff_lo, shift); uint16x8_t res_hi = vmlaq_u16(a32_hi, diff_hi, shift); uint8x16_t v_temp = - vcombine_u8(vshrn_n_u16(res_lo, 5), vshrn_n_u16(res_hi, 5)); + vcombine_u8(vrshrn_n_u16(res_lo, 5), vrshrn_n_u16(res_hi, 5)); uint8x16_t mask = vld1q_u8(BaseMask[base_max_diff]); dst[r] = vbslq_u8(mask, v_temp, a_mbase_x); @@ -1304,37 +1287,34 @@ } } -static void dr_prediction_z1_16xN_neon(int N, uint8_t *dst, ptrdiff_t stride, - const uint8_t *above, int upsample_above, - int dx) { +static void dr_prediction_z1_16xH_neon(int H, uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, int dx) { uint8x16_t dstvec[64]; - dr_prediction_z1_HxW_internal_neon(16, N, dstvec, above, upsample_above, dx); - for (int i = 0; i < N; i++) { + dr_prediction_z1_WxH_internal_neon_large(16, H, dstvec, above, dx); + for (int i = 0; i < H; i++) { vst1q_u8(dst + stride * i, dstvec[i]); } } -static AOM_FORCE_INLINE void dr_prediction_z1_32xN_internal_neon( - int N, uint8x16x2_t *dstvec, const uint8_t *above, int dx) { +static AOM_FORCE_INLINE void dr_prediction_z1_32xH_internal_neon( + int H, uint8x16x2_t *dstvec, const uint8_t *above, int dx) { + // Here upsample_above is 0 by design of av1_use_intra_edge_upsample defined + // in av1/common/reconintra.h. const int frac_bits = 6; - const int max_base_x = ((32 + N) - 1); + const int max_base_x = (32 + H) - 1; - // pre-filter above pixels - // store in temp buffers: - // above[x] * 32 + 16 - // above[x+1] - above[x] - // final pixels will be calculated as: - // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 + // above0 * (32 - shift) + above1 * shift + // = (above1 - above0) * shift + above0 * 32 const uint8x16_t a_mbase_x = vdupq_n_u8(above[max_base_x]); int x = dx; - for (int r = 0; r < N; r++) { + for (int r = 0; r < H; r++) { int base = x >> frac_bits; int base_max_diff = (max_base_x - base); if (base_max_diff <= 0) { - for (int i = r; i < N; ++i) { + for (int i = r; i < H; ++i) { dstvec[i].val[0] = a_mbase_x; // save 32 values dstvec[i].val[1] = a_mbase_x; } @@ -1352,17 +1332,17 @@ } else { uint8x16_t a0_128 = vld1q_u8(above + base + j); uint8x16_t a1_128 = vld1q_u8(above + base + j + 1); + uint16x8_t diff_lo = vsubl_u8(vget_low_u8(a1_128), vget_low_u8(a0_128)); uint16x8_t diff_hi = vsubl_u8(vget_high_u8(a1_128), vget_high_u8(a0_128)); - uint16x8_t a32_lo = - vmlal_u8(vdupq_n_u16(16), vget_low_u8(a0_128), vdup_n_u8(32)); - uint16x8_t a32_hi = - vmlal_u8(vdupq_n_u16(16), vget_high_u8(a0_128), vdup_n_u8(32)); + uint16x8_t a32_lo = vshll_n_u8(vget_low_u8(a0_128), 5); + uint16x8_t a32_hi = vshll_n_u8(vget_high_u8(a0_128), 5); uint16x8_t res_lo = vmlaq_u16(a32_lo, diff_lo, shift); uint16x8_t res_hi = vmlaq_u16(a32_hi, diff_hi, shift); - res16[jj] = vcombine_u8(vshrn_n_u16(res_lo, 5), vshrn_n_u16(res_hi, 5)); + res16[jj] = + vcombine_u8(vrshrn_n_u16(res_lo, 5), vrshrn_n_u16(res_hi, 5)); } } @@ -1374,12 +1354,12 @@ } } -static void dr_prediction_z1_32xN_neon(int N, uint8_t *dst, ptrdiff_t stride, +static void dr_prediction_z1_32xH_neon(int H, uint8_t *dst, ptrdiff_t stride, const uint8_t *above, int dx) { uint8x16x2_t dstvec[64]; - dr_prediction_z1_32xN_internal_neon(N, dstvec, above, dx); - for (int i = 0; i < N; i++) { + dr_prediction_z1_32xH_internal_neon(H, dstvec, above, dx); + for (int i = 0; i < H; i++) { vst1q_u8(dst + stride * i, dstvec[i].val[0]); vst1q_u8(dst + stride * i + 16, dstvec[i].val[1]); } @@ -1420,25 +1400,21 @@ #endif } -static void dr_prediction_z1_64xN_neon(int N, uint8_t *dst, ptrdiff_t stride, +static void dr_prediction_z1_64xH_neon(int H, uint8_t *dst, ptrdiff_t stride, const uint8_t *above, int dx) { const int frac_bits = 6; - const int max_base_x = ((64 + N) - 1); + const int max_base_x = (64 + H) - 1; - // pre-filter above pixels - // store in temp buffers: - // above[x] * 32 + 16 - // above[x+1] - above[x] - // final pixels will be calculated as: - // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 + // above0 * (32 - shift) + above1 * shift + // = (above1 - above0) * shift + above0 * 32 const uint8x16_t a_mbase_x = vdupq_n_u8(above[max_base_x]); int x = dx; - for (int r = 0; r < N; r++, dst += stride) { + for (int r = 0; r < H; r++, dst += stride) { int base = x >> frac_bits; if (base >= max_base_x) { - for (int i = r; i < N; ++i) { + for (int i = r; i < H; ++i) { vst1q_u8(dst, a_mbase_x); vst1q_u8(dst + 16, a_mbase_x); vst1q_u8(dst + 32, a_mbase_x); @@ -1449,9 +1425,6 @@ } uint16x8_t shift = vdupq_n_u16((x & 0x3f) >> 1); - uint8x16_t base_inc128 = - vaddq_u8(vdupq_n_u8(base), vcombine_u8(vcreate_u8(0x0706050403020100), - vcreate_u8(0x0F0E0D0C0B0A0908))); for (int j = 0; j < 64; j += 16) { if (base + j >= max_base_x) { @@ -1475,16 +1448,12 @@ uint16x8_t diff_lo = vsubl_u8(vget_low_u8(a1_128), vget_low_u8(a0_128)); uint16x8_t diff_hi = vsubl_u8(vget_high_u8(a1_128), vget_high_u8(a0_128)); - uint16x8_t a32_lo = - vmlal_u8(vdupq_n_u16(16), vget_low_u8(a0_128), vdup_n_u8(32)); - uint16x8_t a32_hi = - vmlal_u8(vdupq_n_u16(16), vget_high_u8(a0_128), vdup_n_u8(32)); + uint16x8_t a32_lo = vshll_n_u8(vget_low_u8(a0_128), 5); + uint16x8_t a32_hi = vshll_n_u8(vget_high_u8(a0_128), 5); uint16x8_t res_lo = vmlaq_u16(a32_lo, diff_lo, shift); uint16x8_t res_hi = vmlaq_u16(a32_hi, diff_hi, shift); vst1q_u8(dst + j, - vcombine_u8(vshrn_n_u16(res_lo, 5), vshrn_n_u16(res_hi, 5))); - - base_inc128 = vaddq_u8(base_inc128, vdupq_n_u8(16)); + vcombine_u8(vrshrn_n_u16(res_lo, 5), vrshrn_n_u16(res_hi, 5))); } } x += dx; @@ -1500,16 +1469,14 @@ switch (bw) { case 4: - dr_prediction_z1_4xN_neon(bh, dst, stride, above, upsample_above, dx); + dr_prediction_z1_4xH_neon(bh, dst, stride, above, upsample_above, dx); break; case 8: - dr_prediction_z1_8xN_neon(bh, dst, stride, above, upsample_above, dx); + dr_prediction_z1_8xH_neon(bh, dst, stride, above, upsample_above, dx); break; - case 16: - dr_prediction_z1_16xN_neon(bh, dst, stride, above, upsample_above, dx); - break; - case 32: dr_prediction_z1_32xN_neon(bh, dst, stride, above, dx); break; - case 64: dr_prediction_z1_64xN_neon(bh, dst, stride, above, dx); break; + case 16: dr_prediction_z1_16xH_neon(bh, dst, stride, above, dx); break; + case 32: dr_prediction_z1_32xH_neon(bh, dst, stride, above, dx); break; + case 64: dr_prediction_z1_64xH_neon(bh, dst, stride, above, dx); break; default: break; } } @@ -1527,44 +1494,40 @@ }; #endif // !AOM_ARCH_AARCH64 -static AOM_FORCE_INLINE void dr_prediction_z2_Nx4_above_neon( +static AOM_FORCE_INLINE void dr_prediction_z2_4xH_above_neon( const uint8_t *above, int upsample_above, int dx, int base_x, int y, uint8x8_t *a0_x, uint8x8_t *a1_x, uint16x4_t *shift0) { - uint16x4_t r6 = vcreate_u16(0x00C0008000400000); - uint16x4_t ydx = vdup_n_u16(y * dx); + const int x = -y * dx; + if (upsample_above) { // Cannot use LD2 here since we only want to load eight bytes, but LD2 can // only load either 16 or 32. uint8x8_t v_tmp = vld1_u8(above + base_x); *a0_x = vuzp_u8(v_tmp, vdup_n_u8(0)).val[0]; *a1_x = vuzp_u8(v_tmp, vdup_n_u8(0)).val[1]; - *shift0 = vand_u16(vsub_u16(r6, ydx), vdup_n_u16(0x1f)); + *shift0 = vdup_n_u16(x & 0x1f); // ((x << upsample_above) & 0x3f) >> 1 } else { *a0_x = load_unaligned_u8_4x1(above + base_x); *a1_x = load_unaligned_u8_4x1(above + base_x + 1); - *shift0 = vand_u16(vhsub_u16(r6, ydx), vdup_n_u16(0x1f)); + *shift0 = vdup_n_u16((x & 0x3f) >> 1); } } -static AOM_FORCE_INLINE void dr_prediction_z2_Nx4_left_neon( +static AOM_FORCE_INLINE void dr_prediction_z2_4xH_left_neon( #if AOM_ARCH_AARCH64 uint8x16x2_t left_vals, #else const uint8_t *left, #endif - int upsample_left, int dy, int r, int min_base_y, int frac_bits_y, - uint16x4_t *a0_y, uint16x4_t *a1_y, uint16x4_t *shift1) { + int upsample_left, int dy, int r, int frac_bits_y, uint16x4_t *a0_y, + uint16x4_t *a1_y, uint16x4_t *shift1) { int16x4_t dy64 = vdup_n_s16(dy); int16x4_t v_1234 = vcreate_s16(0x0004000300020001); int16x4_t v_frac_bits_y = vdup_n_s16(-frac_bits_y); - int16x4_t min_base_y64 = vdup_n_s16(min_base_y); int16x4_t v_r6 = vdup_n_s16(r << 6); int16x4_t y_c64 = vmls_s16(v_r6, v_1234, dy64); int16x4_t base_y_c64 = vshl_s16(y_c64, v_frac_bits_y); - // Values in base_y_c64 range from -2 through 14 inclusive. - base_y_c64 = vmax_s16(base_y_c64, min_base_y64); - #if AOM_ARCH_AARCH64 uint8x8_t left_idx0 = vreinterpret_u8_s16(vadd_s16(base_y_c64, vdup_n_s16(2))); // [0, 16] @@ -1596,19 +1559,17 @@ #endif // AOM_ARCH_AARCH64 if (upsample_left) { - *shift1 = vand_u16(vreinterpret_u16_s16(y_c64), vdup_n_u16(0x1f)); + *shift1 = vand_u16(vreinterpret_u16_s16(y_c64), + vdup_n_u16(0x1f)); // ((y << upsample_left) & 0x3f) >> 1 } else { *shift1 = vand_u16(vshr_n_u16(vreinterpret_u16_s16(y_c64), 1), vdup_n_u16(0x1f)); } } -static AOM_FORCE_INLINE uint8x8_t dr_prediction_z2_Nx8_above_neon( +static AOM_FORCE_INLINE uint8x8_t dr_prediction_z2_8xH_above_neon( const uint8_t *above, int upsample_above, int dx, int base_x, int y) { - uint16x8_t c1234 = vcombine_u16(vcreate_u16(0x0004000300020001), - vcreate_u16(0x0008000700060005)); - uint16x8_t ydx = vdupq_n_u16(y * dx); - uint16x8_t r6 = vshlq_n_u16(vextq_u16(c1234, vdupq_n_u16(0), 2), 6); + const int x = -y * dx; uint16x8_t shift0; uint8x8_t a0_x0; @@ -1617,40 +1578,36 @@ uint8x8x2_t v_tmp = vld2_u8(above + base_x); a0_x0 = v_tmp.val[0]; a1_x0 = v_tmp.val[1]; - shift0 = vandq_u16(vsubq_u16(r6, ydx), vdupq_n_u16(0x1f)); + shift0 = vdupq_n_u16(x & 0x1f); // ((x << upsample_above) & 0x3f) >> 1 } else { a0_x0 = vld1_u8(above + base_x); a1_x0 = vld1_u8(above + base_x + 1); - shift0 = vandq_u16(vhsubq_u16(r6, ydx), vdupq_n_u16(0x1f)); + shift0 = vdupq_n_u16((x & 0x3f) >> 1); } uint16x8_t diff0 = vsubl_u8(a1_x0, a0_x0); // a[x+1] - a[x] - uint16x8_t a32 = - vmlal_u8(vdupq_n_u16(16), a0_x0, vdup_n_u8(32)); // a[x] * 32 + 16 + uint16x8_t a32 = vshll_n_u8(a0_x0, 5); // a[x] * 32 uint16x8_t res = vmlaq_u16(a32, diff0, shift0); - return vshrn_n_u16(res, 5); + + return vrshrn_n_u16(res, 5); } -static AOM_FORCE_INLINE uint8x8_t dr_prediction_z2_Nx8_left_neon( +static AOM_FORCE_INLINE uint8x8_t dr_prediction_z2_8xH_left_neon( #if AOM_ARCH_AARCH64 uint8x16x3_t left_vals, #else const uint8_t *left, #endif - int upsample_left, int dy, int r, int min_base_y, int frac_bits_y) { + int upsample_left, int dy, int r, int frac_bits_y) { int16x8_t v_r6 = vdupq_n_s16(r << 6); int16x8_t dy128 = vdupq_n_s16(dy); int16x8_t v_frac_bits_y = vdupq_n_s16(-frac_bits_y); - int16x8_t min_base_y128 = vdupq_n_s16(min_base_y); uint16x8_t c1234 = vcombine_u16(vcreate_u16(0x0004000300020001), vcreate_u16(0x0008000700060005)); int16x8_t y_c128 = vmlsq_s16(v_r6, vreinterpretq_s16_u16(c1234), dy128); int16x8_t base_y_c128 = vshlq_s16(y_c128, v_frac_bits_y); - // Values in base_y_c128 range from -2 through 31 inclusive. - base_y_c128 = vmaxq_s16(base_y_c128, min_base_y128); - #if AOM_ARCH_AARCH64 uint8x16_t left_idx0 = vreinterpretq_u8_s16(vaddq_s16(base_y_c128, vdupq_n_s16(2))); // [0, 33] @@ -1668,49 +1625,42 @@ uint16x8_t shift1; if (upsample_left) { - shift1 = vandq_u16(vreinterpretq_u16_s16(y_c128), vdupq_n_u16(0x1f)); + shift1 = + vandq_u16(vreinterpretq_u16_s16(y_c128), + vdupq_n_u16(0x1f)); // ((y << upsample_left) & 0x3f) >> 1 } else { shift1 = vshrq_n_u16( vandq_u16(vreinterpretq_u16_s16(y_c128), vdupq_n_u16(0x3f)), 1); } uint16x8_t diff1 = vsubl_u8(a1_x1, a0_x1); - uint16x8_t a32 = vmlal_u8(vdupq_n_u16(16), a0_x1, vdup_n_u8(32)); + uint16x8_t a32 = vshll_n_u8(a0_x1, 5); uint16x8_t res = vmlaq_u16(a32, diff1, shift1); - return vshrn_n_u16(res, 5); + + return vrshrn_n_u16(res, 5); } -static AOM_FORCE_INLINE uint8x16_t dr_prediction_z2_NxW_above_neon( +static AOM_FORCE_INLINE uint8x16_t dr_prediction_z2_WxH_above_neon( const uint8_t *above, int dx, int base_x, int y, int j) { - uint16x8x2_t c0123 = { { vcombine_u16(vcreate_u16(0x0003000200010000), - vcreate_u16(0x0007000600050004)), - vcombine_u16(vcreate_u16(0x000B000A00090008), - vcreate_u16(0x000F000E000D000C)) } }; - uint16x8_t j256 = vdupq_n_u16(j); - uint16x8_t ydx = vdupq_n_u16((uint16_t)(y * dx)); - + const int x = -y * dx; const uint8x16_t a0_x128 = vld1q_u8(above + base_x + j); const uint8x16_t a1_x128 = vld1q_u8(above + base_x + j + 1); - uint16x8_t res6_0 = vshlq_n_u16(vaddq_u16(c0123.val[0], j256), 6); - uint16x8_t res6_1 = vshlq_n_u16(vaddq_u16(c0123.val[1], j256), 6); - uint16x8_t shift0 = - vshrq_n_u16(vandq_u16(vsubq_u16(res6_0, ydx), vdupq_n_u16(0x3f)), 1); - uint16x8_t shift1 = - vshrq_n_u16(vandq_u16(vsubq_u16(res6_1, ydx), vdupq_n_u16(0x3f)), 1); + const uint16x8_t shift = vdupq_n_u16((x & 0x3f) >> 1); + // a[x+1] - a[x] uint16x8_t diff0 = vsubl_u8(vget_low_u8(a1_x128), vget_low_u8(a0_x128)); uint16x8_t diff1 = vsubl_u8(vget_high_u8(a1_x128), vget_high_u8(a0_x128)); - // a[x] * 32 + 16 - uint16x8_t a32_0 = - vmlal_u8(vdupq_n_u16(16), vget_low_u8(a0_x128), vdup_n_u8(32)); - uint16x8_t a32_1 = - vmlal_u8(vdupq_n_u16(16), vget_high_u8(a0_x128), vdup_n_u8(32)); - uint16x8_t res0 = vmlaq_u16(a32_0, diff0, shift0); - uint16x8_t res1 = vmlaq_u16(a32_1, diff1, shift1); - return vcombine_u8(vshrn_n_u16(res0, 5), vshrn_n_u16(res1, 5)); + + // a[x] * 32 + uint16x8_t a32_0 = vshll_n_u8(vget_low_u8(a0_x128), 5); + uint16x8_t a32_1 = vshll_n_u8(vget_high_u8(a0_x128), 5); + uint16x8_t res0 = vmlaq_u16(a32_0, diff0, shift); + uint16x8_t res1 = vmlaq_u16(a32_1, diff1, shift); + + return vcombine_u8(vrshrn_n_u16(res0, 5), vrshrn_n_u16(res1, 5)); } -static AOM_FORCE_INLINE uint8x16_t dr_prediction_z2_NxW_left_neon( +static AOM_FORCE_INLINE uint8x16_t dr_prediction_z2_WxH_left_neon( #if AOM_ARCH_AARCH64 uint8x16x4_t left_vals0, uint8x16x4_t left_vals1, #else @@ -1719,39 +1669,25 @@ int dy, int r, int j) { // here upsample_above and upsample_left are 0 by design of // av1_use_intra_edge_upsample - const int min_base_y = -1; - - int16x8_t min_base_y256 = vdupq_n_s16(min_base_y); - int16x8_t half_min_base_y256 = vdupq_n_s16(min_base_y >> 1); int16x8_t dy256 = vdupq_n_s16(dy); uint16x8_t j256 = vdupq_n_u16(j); - uint16x8x2_t c0123 = { { vcombine_u16(vcreate_u16(0x0003000200010000), - vcreate_u16(0x0007000600050004)), - vcombine_u16(vcreate_u16(0x000B000A00090008), - vcreate_u16(0x000F000E000D000C)) } }; - uint16x8x2_t c1234 = { { vaddq_u16(c0123.val[0], vdupq_n_u16(1)), - vaddq_u16(c0123.val[1], vdupq_n_u16(1)) } }; + uint16x8x2_t c1234 = { { vcombine_u16(vcreate_u16(0x0004000300020001), + vcreate_u16(0x0008000700060005)), + vcombine_u16(vcreate_u16(0x000C000B000A0009), + vcreate_u16(0x0010000F000E000D)) } }; int16x8_t v_r6 = vdupq_n_s16(r << 6); int16x8_t c256_0 = vreinterpretq_s16_u16(vaddq_u16(j256, c1234.val[0])); int16x8_t c256_1 = vreinterpretq_s16_u16(vaddq_u16(j256, c1234.val[1])); - int16x8_t mul16_lo = vreinterpretq_s16_u16( - vminq_u16(vreinterpretq_u16_s16(vmulq_s16(c256_0, dy256)), - vreinterpretq_u16_s16(half_min_base_y256))); - int16x8_t mul16_hi = vreinterpretq_s16_u16( - vminq_u16(vreinterpretq_u16_s16(vmulq_s16(c256_1, dy256)), - vreinterpretq_u16_s16(half_min_base_y256))); - int16x8_t y_c256_lo = vsubq_s16(v_r6, mul16_lo); - int16x8_t y_c256_hi = vsubq_s16(v_r6, mul16_hi); + + int16x8_t y_c256_lo = vmlsq_s16(v_r6, c256_0, dy256); + int16x8_t y_c256_hi = vmlsq_s16(v_r6, c256_1, dy256); int16x8_t base_y_c256_lo = vshrq_n_s16(y_c256_lo, 6); int16x8_t base_y_c256_hi = vshrq_n_s16(y_c256_hi, 6); - base_y_c256_lo = vmaxq_s16(min_base_y256, base_y_c256_lo); - base_y_c256_hi = vmaxq_s16(min_base_y256, base_y_c256_hi); - #if !AOM_ARCH_AARCH64 int16_t min_y = vgetq_lane_s16(base_y_c256_hi, 7); int16_t max_y = vgetq_lane_s16(base_y_c256_lo, 0); @@ -1832,31 +1768,26 @@ uint16x8_t diff_lo = vsubl_u8(a1_y0, a0_y0); uint16x8_t diff_hi = vsubl_u8(a1_y1, a0_y1); // a[x] * 32 + 16 - uint16x8_t a32_lo = vmlal_u8(vdupq_n_u16(16), a0_y0, vdup_n_u8(32)); - uint16x8_t a32_hi = vmlal_u8(vdupq_n_u16(16), a0_y1, vdup_n_u8(32)); + uint16x8_t a32_lo = vshll_n_u8(a0_y0, 5); + uint16x8_t a32_hi = vshll_n_u8(a0_y1, 5); uint16x8_t res0 = vmlaq_u16(a32_lo, diff_lo, shifty_lo); uint16x8_t res1 = vmlaq_u16(a32_hi, diff_hi, shifty_hi); - return vcombine_u8(vshrn_n_u16(res0, 5), vshrn_n_u16(res1, 5)); + return vcombine_u8(vrshrn_n_u16(res0, 5), vrshrn_n_u16(res1, 5)); } -static void dr_prediction_z2_Nx4_neon(int N, uint8_t *dst, ptrdiff_t stride, +static void dr_prediction_z2_4xH_neon(int H, uint8_t *dst, ptrdiff_t stride, const uint8_t *above, const uint8_t *left, int upsample_above, int upsample_left, int dx, int dy) { const int min_base_x = -(1 << upsample_above); - const int min_base_y = -(1 << upsample_left); const int frac_bits_x = 6 - upsample_above; const int frac_bits_y = 6 - upsample_left; assert(dx > 0); - // pre-filter above pixels - // store in temp buffers: - // above[x] * 32 + 16 - // above[x+1] - above[x] - // final pixels will be calculated as: - // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 + // above0 * (32 - shift) + above1 * shift + // = (above1 - above0) * shift + above0 * 32 #if AOM_ARCH_AARCH64 // Use ext rather than loading left + 14 directly to avoid over-read. @@ -1869,7 +1800,7 @@ #define LEFT left #endif // AOM_ARCH_AARCH64 - for (int r = 0; r < N; r++) { + for (int r = 0; r < H; r++) { int y = r + 1; int base_x = (-y * dx) >> frac_bits_x; const int base_min_diff = @@ -1879,22 +1810,21 @@ if (base_min_diff <= 0) { uint8x8_t a0_x_u8, a1_x_u8; uint16x4_t shift0; - dr_prediction_z2_Nx4_above_neon(above, upsample_above, dx, base_x, y, + dr_prediction_z2_4xH_above_neon(above, upsample_above, dx, base_x, y, &a0_x_u8, &a1_x_u8, &shift0); uint8x8_t a0_x = a0_x_u8; uint8x8_t a1_x = a1_x_u8; uint16x8_t diff = vsubl_u8(a1_x, a0_x); // a[x+1] - a[x] - uint16x8_t a32 = - vmlal_u8(vdupq_n_u16(16), a0_x, vdup_n_u8(32)); // a[x] * 32 + 16 + uint16x8_t a32 = vshll_n_u8(a0_x, 5); uint16x8_t res = vmlaq_u16(a32, diff, vcombine_u16(shift0, vdup_n_u16(0))); - uint8x8_t resx = vshrn_n_u16(res, 5); + uint8x8_t resx = vrshrn_n_u16(res, 5); vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(resx), 0); } else if (base_min_diff < 4) { uint8x8_t a0_x_u8, a1_x_u8; uint16x4_t shift0; - dr_prediction_z2_Nx4_above_neon(above, upsample_above, dx, base_x, y, + dr_prediction_z2_4xH_above_neon(above, upsample_above, dx, base_x, y, &a0_x_u8, &a1_x_u8, &shift0); uint16x8_t a0_x = vmovl_u8(a0_x_u8); uint16x8_t a1_x = vmovl_u8(a1_x_u8); @@ -1902,17 +1832,16 @@ uint16x4_t a0_y; uint16x4_t a1_y; uint16x4_t shift1; - dr_prediction_z2_Nx4_left_neon(LEFT, upsample_left, dy, r, min_base_y, - frac_bits_y, &a0_y, &a1_y, &shift1); + dr_prediction_z2_4xH_left_neon(LEFT, upsample_left, dy, r, frac_bits_y, + &a0_y, &a1_y, &shift1); a0_x = vcombine_u16(vget_low_u16(a0_x), a0_y); a1_x = vcombine_u16(vget_low_u16(a1_x), a1_y); uint16x8_t shift = vcombine_u16(shift0, shift1); uint16x8_t diff = vsubq_u16(a1_x, a0_x); // a[x+1] - a[x] - uint16x8_t a32 = - vmlaq_n_u16(vdupq_n_u16(16), a0_x, 32); // a[x] * 32 + 16 + uint16x8_t a32 = vshlq_n_u16(a0_x, 5); uint16x8_t res = vmlaq_u16(a32, diff, shift); - uint8x8_t resx = vshrn_n_u16(res, 5); + uint8x8_t resx = vrshrn_n_u16(res, 5); uint8x8_t resy = vext_u8(resx, vdup_n_u8(0), 4); uint8x8_t mask = vld1_u8(BaseMask[base_min_diff]); @@ -1921,12 +1850,12 @@ } else { uint16x4_t a0_y, a1_y; uint16x4_t shift1; - dr_prediction_z2_Nx4_left_neon(LEFT, upsample_left, dy, r, min_base_y, - frac_bits_y, &a0_y, &a1_y, &shift1); - uint16x4_t diff = vsub_u16(a1_y, a0_y); // a[x+1] - a[x] - uint16x4_t a32 = vmla_n_u16(vdup_n_u16(16), a0_y, 32); // a[x] * 32 + 16 + dr_prediction_z2_4xH_left_neon(LEFT, upsample_left, dy, r, frac_bits_y, + &a0_y, &a1_y, &shift1); + uint16x4_t diff = vsub_u16(a1_y, a0_y); // a[x+1] - a[x] + uint16x4_t a32 = vshl_n_u16(a0_y, 5); uint16x4_t res = vmla_u16(a32, diff, shift1); - uint8x8_t resy = vshrn_n_u16(vcombine_u16(res, vdup_n_u16(0)), 5); + uint8x8_t resy = vrshrn_n_u16(vcombine_u16(res, vdup_n_u16(0)), 5); vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(resy), 0); } @@ -1936,21 +1865,16 @@ #undef LEFT } -static void dr_prediction_z2_Nx8_neon(int N, uint8_t *dst, ptrdiff_t stride, +static void dr_prediction_z2_8xH_neon(int H, uint8_t *dst, ptrdiff_t stride, const uint8_t *above, const uint8_t *left, int upsample_above, int upsample_left, int dx, int dy) { const int min_base_x = -(1 << upsample_above); - const int min_base_y = -(1 << upsample_left); const int frac_bits_x = 6 - upsample_above; const int frac_bits_y = 6 - upsample_left; - // pre-filter above pixels - // store in temp buffers: - // above[x] * 32 + 16 - // above[x+1] - above[x] - // final pixels will be calculated as: - // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 + // above0 * (32 - shift) + above1 * shift + // = (above1 - above0) * shift + above0 * 32 #if AOM_ARCH_AARCH64 // Use ext rather than loading left + 30 directly to avoid over-read. @@ -1965,7 +1889,7 @@ #define LEFT left #endif // AOM_ARCH_AARCH64 - for (int r = 0; r < N; r++) { + for (int r = 0; r < H; r++) { int y = r + 1; int base_x = (-y * dx) >> frac_bits_x; int base_min_diff = @@ -1973,19 +1897,19 @@ if (base_min_diff <= 0) { uint8x8_t resx = - dr_prediction_z2_Nx8_above_neon(above, upsample_above, dx, base_x, y); + dr_prediction_z2_8xH_above_neon(above, upsample_above, dx, base_x, y); vst1_u8(dst, resx); } else if (base_min_diff < 8) { uint8x8_t resx = - dr_prediction_z2_Nx8_above_neon(above, upsample_above, dx, base_x, y); - uint8x8_t resy = dr_prediction_z2_Nx8_left_neon( - LEFT, upsample_left, dy, r, min_base_y, frac_bits_y); + dr_prediction_z2_8xH_above_neon(above, upsample_above, dx, base_x, y); + uint8x8_t resy = dr_prediction_z2_8xH_left_neon(LEFT, upsample_left, dy, + r, frac_bits_y); uint8x8_t mask = vld1_u8(BaseMask[base_min_diff]); uint8x8_t resxy = vbsl_u8(mask, resy, resx); vst1_u8(dst, resxy); } else { - uint8x8_t resy = dr_prediction_z2_Nx8_left_neon( - LEFT, upsample_left, dy, r, min_base_y, frac_bits_y); + uint8x8_t resy = dr_prediction_z2_8xH_left_neon(LEFT, upsample_left, dy, + r, frac_bits_y); vst1_u8(dst, resy); } @@ -1994,7 +1918,7 @@ #undef LEFT } -static void dr_prediction_z2_HxW_neon(int H, int W, uint8_t *dst, +static void dr_prediction_z2_WxH_neon(int W, int H, uint8_t *dst, ptrdiff_t stride, const uint8_t *above, const uint8_t *left, int dx, int dy) { // here upsample_above and upsample_left are 0 by design of @@ -2025,17 +1949,17 @@ if (base_min_diff <= 0) { uint8x16_t resx = - dr_prediction_z2_NxW_above_neon(above, dx, base_x, y, j); + dr_prediction_z2_WxH_above_neon(above, dx, base_x, y, j); vst1q_u8(dst + j, resx); } else if (base_min_diff < 16) { uint8x16_t resx = - dr_prediction_z2_NxW_above_neon(above, dx, base_x, y, j); - uint8x16_t resy = dr_prediction_z2_NxW_left_neon(LEFT, dy, r, j); + dr_prediction_z2_WxH_above_neon(above, dx, base_x, y, j); + uint8x16_t resy = dr_prediction_z2_WxH_left_neon(LEFT, dy, r, j); uint8x16_t mask = vld1q_u8(BaseMask[base_min_diff]); uint8x16_t resxy = vbslq_u8(mask, resy, resx); vst1q_u8(dst + j, resxy); } else { - uint8x16_t resy = dr_prediction_z2_NxW_left_neon(LEFT, dy, r, j); + uint8x16_t resy = dr_prediction_z2_WxH_left_neon(LEFT, dy, r, j); vst1q_u8(dst + j, resy); } } // for j @@ -2054,15 +1978,15 @@ switch (bw) { case 4: - dr_prediction_z2_Nx4_neon(bh, dst, stride, above, left, upsample_above, + dr_prediction_z2_4xH_neon(bh, dst, stride, above, left, upsample_above, upsample_left, dx, dy); break; case 8: - dr_prediction_z2_Nx8_neon(bh, dst, stride, above, left, upsample_above, + dr_prediction_z2_8xH_neon(bh, dst, stride, above, left, upsample_above, upsample_left, dx, dy); break; default: - dr_prediction_z2_HxW_neon(bh, bw, dst, stride, above, left, dx, dy); + dr_prediction_z2_WxH_neon(bw, bh, dst, stride, above, left, dx, dy); break; } } @@ -2134,7 +2058,8 @@ uint8x8_t dstvec[4]; uint8x8x2_t dest; - dr_prediction_z1_HxW_internal_neon_64(4, 4, dstvec, left, upsample_left, dy); + dr_prediction_z1_WxH_internal_neon_small(4, 4, dstvec, left, upsample_left, + dy); z3_transpose_arrays_u8_4x4(dstvec, &dest); store_u8x4_strided_x2(dst + stride * 0, stride, dest.val[0]); store_u8x4_strided_x2(dst + stride * 2, stride, dest.val[1]); @@ -2146,7 +2071,8 @@ uint8x8_t dstvec[8]; uint8x8_t d[8]; - dr_prediction_z1_HxW_internal_neon_64(8, 8, dstvec, left, upsample_left, dy); + dr_prediction_z1_WxH_internal_neon_small(8, 8, dstvec, left, upsample_left, + dy); transpose_arrays_u8_8x8(dstvec, d); store_u8_8x8(dst, stride, d[0], d[1], d[2], d[3], d[4], d[5], d[6], d[7]); } @@ -2157,7 +2083,8 @@ uint8x8_t dstvec[4]; uint8x8x2_t d[2]; - dr_prediction_z1_HxW_internal_neon_64(8, 4, dstvec, left, upsample_left, dy); + dr_prediction_z1_WxH_internal_neon_small(8, 4, dstvec, left, upsample_left, + dy); z3_transpose_arrays_u8_8x4(dstvec, d); store_u8x4_strided_x2(dst + stride * 0, stride, d[0].val[0]); store_u8x4_strided_x2(dst + stride * 2, stride, d[0].val[1]); @@ -2171,7 +2098,8 @@ uint8x8_t dstvec[8]; uint8x8_t d[8]; - dr_prediction_z1_HxW_internal_neon_64(4, 8, dstvec, left, upsample_left, dy); + dr_prediction_z1_WxH_internal_neon_small(4, 8, dstvec, left, upsample_left, + dy); transpose_arrays_u8_8x8(dstvec, d); store_u8_8x4(dst, stride, d[0], d[1], d[2], d[3]); } @@ -2179,10 +2107,12 @@ static void dr_prediction_z3_8x16_neon(uint8_t *dst, ptrdiff_t stride, const uint8_t *left, int upsample_left, int dy) { + (void)upsample_left; + uint8x16_t dstvec[8]; uint8x8_t d[16]; - dr_prediction_z1_HxW_internal_neon(16, 8, dstvec, left, upsample_left, dy); + dr_prediction_z1_WxH_internal_neon_large(16, 8, dstvec, left, dy); transpose_arrays_u8_16x8(dstvec, d); for (int i = 0; i < 16; i++) { vst1_u8(dst + i * stride, d[i]); @@ -2195,7 +2125,8 @@ uint8x8_t dstvec[16]; uint8x16_t d[8]; - dr_prediction_z1_HxW_internal_neon_64(8, 16, dstvec, left, upsample_left, dy); + dr_prediction_z1_WxH_internal_neon_small(8, 16, dstvec, left, upsample_left, + dy); transpose_arrays_u8_8x16(dstvec, d); for (int i = 0; i < 8; i++) { vst1q_u8(dst + i * stride, d[i]); @@ -2206,10 +2137,12 @@ static void dr_prediction_z3_4x16_neon(uint8_t *dst, ptrdiff_t stride, const uint8_t *left, int upsample_left, int dy) { + (void)upsample_left; + uint8x16_t dstvec[4]; uint8x16x2_t d[2]; - dr_prediction_z1_HxW_internal_neon(16, 4, dstvec, left, upsample_left, dy); + dr_prediction_z1_WxH_internal_neon_large(16, 4, dstvec, left, dy); z3_transpose_arrays_u8_16x4(dstvec, d); store_u8x4_strided_x4(dst + stride * 0, stride, d[0].val[0]); store_u8x4_strided_x4(dst + stride * 4, stride, d[0].val[1]); @@ -2223,7 +2156,8 @@ uint8x8_t dstvec[16]; uint8x16_t d[8]; - dr_prediction_z1_HxW_internal_neon_64(4, 16, dstvec, left, upsample_left, dy); + dr_prediction_z1_WxH_internal_neon_small(4, 16, dstvec, left, upsample_left, + dy); transpose_arrays_u8_8x16(dstvec, d); for (int i = 0; i < 4; i++) { vst1q_u8(dst + i * stride, d[i]); @@ -2238,7 +2172,7 @@ uint8x16_t d[32]; uint8x16_t v_zero = vdupq_n_u8(0); - dr_prediction_z1_32xN_internal_neon(8, dstvec, left, dy); + dr_prediction_z1_32xH_internal_neon(8, dstvec, left, dy); for (int i = 8; i < 16; i++) { dstvec[i].val[0] = v_zero; dstvec[i].val[1] = v_zero; @@ -2255,7 +2189,8 @@ uint8x8_t dstvec[32]; uint8x16_t d[16]; - dr_prediction_z1_HxW_internal_neon_64(8, 32, dstvec, left, upsample_left, dy); + dr_prediction_z1_WxH_internal_neon_small(8, 32, dstvec, left, upsample_left, + dy); transpose_arrays_u8_8x16(dstvec, d); transpose_arrays_u8_8x16(dstvec + 16, d + 8); for (int i = 0; i < 8; i++) { @@ -2268,10 +2203,12 @@ static void dr_prediction_z3_16x16_neon(uint8_t *dst, ptrdiff_t stride, const uint8_t *left, int upsample_left, int dy) { + (void)upsample_left; + uint8x16_t dstvec[16]; uint8x16_t d[16]; - dr_prediction_z1_HxW_internal_neon(16, 16, dstvec, left, upsample_left, dy); + dr_prediction_z1_WxH_internal_neon_large(16, 16, dstvec, left, dy); transpose_arrays_u8_16x16(dstvec, d); for (int i = 0; i < 16; i++) { vst1q_u8(dst + i * stride, d[i]); @@ -2285,7 +2222,7 @@ uint8x16x2_t dstvec[32]; uint8x16_t d[64]; - dr_prediction_z1_32xN_internal_neon(32, dstvec, left, dy); + dr_prediction_z1_32xH_internal_neon(32, dstvec, left, dy); transpose_arrays_u8_32x16(dstvec, d); transpose_arrays_u8_32x16(dstvec + 16, d + 32); for (int i = 0; i < 32; i++) { @@ -2300,7 +2237,7 @@ (void)upsample_left; DECLARE_ALIGNED(16, uint8_t, dstT[64 * 64]); - dr_prediction_z1_64xN_neon(64, dstT, 64, left, dy); + dr_prediction_z1_64xH_neon(64, dstT, 64, left, dy); z3_transpose_arrays_u8_16nx16n(dstT, 64, dst, stride, 64, 64); } @@ -2311,7 +2248,7 @@ uint8x16x2_t dstvec[16]; uint8x16_t d[32]; - dr_prediction_z1_32xN_internal_neon(16, dstvec, left, dy); + dr_prediction_z1_32xH_internal_neon(16, dstvec, left, dy); transpose_arrays_u8_32x16(dstvec, d); for (int i = 0; i < 16; i++) { vst1q_u8(dst + 2 * i * stride, d[2 * i + 0]); @@ -2322,9 +2259,11 @@ static void dr_prediction_z3_32x16_neon(uint8_t *dst, ptrdiff_t stride, const uint8_t *left, int upsample_left, int dy) { + (void)upsample_left; + uint8x16_t dstvec[32]; - dr_prediction_z1_HxW_internal_neon(16, 32, dstvec, left, upsample_left, dy); + dr_prediction_z1_WxH_internal_neon_large(16, 32, dstvec, left, dy); for (int i = 0; i < 32; i += 16) { uint8x16_t d[16]; transpose_arrays_u8_16x16(dstvec + i, d); @@ -2340,7 +2279,7 @@ (void)upsample_left; uint8_t dstT[64 * 32]; - dr_prediction_z1_64xN_neon(32, dstT, 64, left, dy); + dr_prediction_z1_64xH_neon(32, dstT, 64, left, dy); z3_transpose_arrays_u8_16nx16n(dstT, 64, dst, stride, 32, 64); } @@ -2350,7 +2289,7 @@ (void)upsample_left; uint8_t dstT[32 * 64]; - dr_prediction_z1_32xN_neon(64, dstT, 32, left, dy); + dr_prediction_z1_32xH_neon(64, dstT, 32, left, dy); z3_transpose_arrays_u8_16nx16n(dstT, 32, dst, stride, 64, 32); } @@ -2361,16 +2300,18 @@ (void)upsample_left; uint8_t dstT[64 * 16]; - dr_prediction_z1_64xN_neon(16, dstT, 64, left, dy); + dr_prediction_z1_64xH_neon(16, dstT, 64, left, dy); z3_transpose_arrays_u8_16nx16n(dstT, 64, dst, stride, 16, 64); } static void dr_prediction_z3_64x16_neon(uint8_t *dst, ptrdiff_t stride, const uint8_t *left, int upsample_left, int dy) { + (void)upsample_left; + uint8x16_t dstvec[64]; - dr_prediction_z1_HxW_internal_neon(16, 64, dstvec, left, upsample_left, dy); + dr_prediction_z1_WxH_internal_neon_large(16, 64, dstvec, left, dy); for (int i = 0; i < 64; i += 16) { uint8x16_t d[16]; transpose_arrays_u8_16x16(dstvec + i, d);
diff --git a/aom_dsp/arm/mem_neon.h b/aom_dsp/arm/mem_neon.h index ad761de..6590a7f 100644 --- a/aom_dsp/arm/mem_neon.h +++ b/aom_dsp/arm/mem_neon.h
@@ -176,7 +176,7 @@ return ret; } -static inline uint8x8_t load_u8_4x2(const uint8_t *p, int stride) { +static inline uint8x8_t load_u8_4x2(const uint8_t *p, ptrdiff_t stride) { uint8x8_t ret = vdup_n_u8(0); ret = vreinterpret_u8_u32( vld1_lane_u32((const uint32_t *)p, vreinterpret_u32_u8(ret), 0)); @@ -186,7 +186,7 @@ return ret; } -static inline uint16x4_t load_u16_2x2(const uint16_t *p, int stride) { +static inline uint16x4_t load_u16_2x2(const uint16_t *p, ptrdiff_t stride) { uint16x4_t ret = vdup_n_u16(0); ret = vreinterpret_u16_u32( vld1_lane_u32((const uint32_t *)p, vreinterpret_u32_u16(ret), 0)); @@ -1194,7 +1194,8 @@ #endif // Load 2 sets of 4 bytes when alignment is not guaranteed. -static inline uint8x8_t load_unaligned_u8(const uint8_t *buf, int stride) { +static inline uint8x8_t load_unaligned_u8(const uint8_t *buf, + ptrdiff_t stride) { uint32_t a; memcpy(&a, buf, 4); buf += stride; @@ -1205,7 +1206,8 @@ } // Load 4 sets of 4 bytes when alignment is not guaranteed. -static inline uint8x16_t load_unaligned_u8q(const uint8_t *buf, int stride) { +static inline uint8x16_t load_unaligned_u8q(const uint8_t *buf, + ptrdiff_t stride) { uint32_t a; uint32x4_t a_u32; if (stride == 4) return vld1q_u8(buf); @@ -1223,7 +1225,8 @@ return vreinterpretq_u8_u32(a_u32); } -static inline uint8x8_t load_unaligned_u8_2x2(const uint8_t *buf, int stride) { +static inline uint8x8_t load_unaligned_u8_2x2(const uint8_t *buf, + ptrdiff_t stride) { uint16_t a; uint16x4_t a_u16; @@ -1263,7 +1266,8 @@ return vreinterpret_u8_u16(a_u32); } -static inline uint8x8_t load_unaligned_u8_4x2(const uint8_t *buf, int stride) { +static inline uint8x8_t load_unaligned_u8_4x2(const uint8_t *buf, + ptrdiff_t stride) { uint32_t a; uint32x2_t a_u32; @@ -1275,14 +1279,14 @@ return vreinterpret_u8_u32(a_u32); } -static inline void load_unaligned_u8_4x4(const uint8_t *buf, int stride, +static inline void load_unaligned_u8_4x4(const uint8_t *buf, ptrdiff_t stride, uint8x8_t *tu0, uint8x8_t *tu1) { *tu0 = load_unaligned_u8_4x2(buf, stride); buf += 2 * stride; *tu1 = load_unaligned_u8_4x2(buf, stride); } -static inline void load_unaligned_u8_3x8(const uint8_t *buf, int stride, +static inline void load_unaligned_u8_3x8(const uint8_t *buf, ptrdiff_t stride, uint8x8_t *tu0, uint8x8_t *tu1, uint8x8_t *tu2) { load_unaligned_u8_4x4(buf, stride, tu0, tu1); @@ -1290,7 +1294,7 @@ *tu2 = load_unaligned_u8_4x2(buf, stride); } -static inline void load_unaligned_u8_4x8(const uint8_t *buf, int stride, +static inline void load_unaligned_u8_4x8(const uint8_t *buf, ptrdiff_t stride, uint8x8_t *tu0, uint8x8_t *tu1, uint8x8_t *tu2, uint8x8_t *tu3) { load_unaligned_u8_4x4(buf, stride, tu0, tu1); @@ -1397,7 +1401,7 @@ } static inline uint16x4_t load_unaligned_u16_2x2(const uint16_t *buf, - int stride) { + ptrdiff_t stride) { uint32_t a; uint32x2_t a_u32; @@ -1418,7 +1422,7 @@ } static inline uint16x8_t load_unaligned_u16_4x2(const uint16_t *buf, - uint32_t stride) { + ptrdiff_t stride) { uint64_t a; uint64x2_t a_u64; @@ -1433,7 +1437,7 @@ } static inline int16x8_t load_unaligned_s16_4x2(const int16_t *buf, - uint32_t stride) { + ptrdiff_t stride) { int64_t a; int64x2_t a_s64; memcpy(&a, buf, 8); @@ -1446,14 +1450,14 @@ return vreinterpretq_s16_s64(a_s64); } -static inline void load_unaligned_u16_4x4(const uint16_t *buf, uint32_t stride, +static inline void load_unaligned_u16_4x4(const uint16_t *buf, ptrdiff_t stride, uint16x8_t *tu0, uint16x8_t *tu1) { *tu0 = load_unaligned_u16_4x2(buf, stride); buf += 2 * stride; *tu1 = load_unaligned_u16_4x2(buf, stride); } -static inline void load_s32_4x4(int32_t *s, int32_t p, int32x4_t *s1, +static inline void load_s32_4x4(int32_t *s, ptrdiff_t p, int32x4_t *s1, int32x4_t *s2, int32x4_t *s3, int32x4_t *s4) { *s1 = vld1q_s32(s); s += p; @@ -1464,7 +1468,7 @@ *s4 = vld1q_s32(s); } -static inline void store_s32_4x4(int32_t *s, int32_t p, int32x4_t s1, +static inline void store_s32_4x4(int32_t *s, ptrdiff_t p, int32x4_t s1, int32x4_t s2, int32x4_t s3, int32x4_t s4) { vst1q_s32(s, s1); s += p; @@ -1475,7 +1479,7 @@ vst1q_s32(s, s4); } -static inline void load_u32_4x4(uint32_t *s, int32_t p, uint32x4_t *s1, +static inline void load_u32_4x4(uint32_t *s, ptrdiff_t p, uint32x4_t *s1, uint32x4_t *s2, uint32x4_t *s3, uint32x4_t *s4) { *s1 = vld1q_u32(s); @@ -1487,7 +1491,7 @@ *s4 = vld1q_u32(s); } -static inline void store_u32_4x4(uint32_t *s, int32_t p, uint32x4_t s1, +static inline void store_u32_4x4(uint32_t *s, ptrdiff_t p, uint32x4_t s1, uint32x4_t s2, uint32x4_t s3, uint32x4_t s4) { vst1q_u32(s, s1); s += p; @@ -1578,14 +1582,14 @@ } // Store two blocks of 16-bits from a single vector. -static inline void store_u8x2_strided_x2(uint8_t *dst, uint32_t dst_stride, +static inline void store_u8x2_strided_x2(uint8_t *dst, ptrdiff_t dst_stride, uint8x8_t src) { store_u8_2x1_lane(dst, src, 0); dst += dst_stride; store_u8_2x1_lane(dst, src, 1); } -static inline void store_u8x2_strided_x4(uint8_t *dst, uint32_t dst_stride, +static inline void store_u8x2_strided_x4(uint8_t *dst, ptrdiff_t dst_stride, uint8x8_t src) { store_u8_2x1_lane(dst, src, 0); dst += dst_stride; @@ -1622,7 +1626,7 @@ } // Store two blocks of 32-bits from a single vector. -static inline void store_u16x2_strided_x2(uint16_t *dst, uint32_t dst_stride, +static inline void store_u16x2_strided_x2(uint16_t *dst, ptrdiff_t dst_stride, uint16x4_t src) { store_u16_2x1_lane(dst, src, 0); dst += dst_stride; @@ -1630,7 +1634,7 @@ } // Store two blocks of 64-bits from a single vector. -static inline void store_u16x4_strided_x2(uint16_t *dst, uint32_t dst_stride, +static inline void store_u16x4_strided_x2(uint16_t *dst, ptrdiff_t dst_stride, uint16x8_t src) { store_u16_4x1_lane(dst, src, 0); dst += dst_stride; @@ -1638,7 +1642,7 @@ } // Store two blocks of 64-bits from a single vector. -static inline void store_s16x4_strided_x2(int16_t *dst, int32_t dst_stride, +static inline void store_s16x4_strided_x2(int16_t *dst, ptrdiff_t dst_stride, int16x8_t src) { store_s16_4x1_lane(dst, src, 0); dst += dst_stride;
diff --git a/aom_dsp/arm/subpel_variance_neon.c b/aom_dsp/arm/subpel_variance_neon.c index 489c8d0..4e74737 100644 --- a/aom_dsp/arm/subpel_variance_neon.c +++ b/aom_dsp/arm/subpel_variance_neon.c
@@ -19,128 +19,7 @@ #include "aom_dsp/variance.h" #include "aom_dsp/arm/mem_neon.h" - -static void var_filter_block2d_bil_w4(const uint8_t *src_ptr, uint8_t *dst_ptr, - int src_stride, int pixel_step, - int dst_height, int filter_offset) { - const uint8x8_t f0 = vdup_n_u8(8 - filter_offset); - const uint8x8_t f1 = vdup_n_u8(filter_offset); - - int i = dst_height; - do { - uint8x8_t s0 = load_unaligned_u8(src_ptr, src_stride); - uint8x8_t s1 = load_unaligned_u8(src_ptr + pixel_step, src_stride); - uint16x8_t blend = vmull_u8(s0, f0); - blend = vmlal_u8(blend, s1, f1); - uint8x8_t blend_u8 = vrshrn_n_u16(blend, 3); - vst1_u8(dst_ptr, blend_u8); - - src_ptr += 2 * src_stride; - dst_ptr += 2 * 4; - i -= 2; - } while (i != 0); -} - -static void var_filter_block2d_bil_w8(const uint8_t *src_ptr, uint8_t *dst_ptr, - int src_stride, int pixel_step, - int dst_height, int filter_offset) { - const uint8x8_t f0 = vdup_n_u8(8 - filter_offset); - const uint8x8_t f1 = vdup_n_u8(filter_offset); - - int i = dst_height; - do { - uint8x8_t s0 = vld1_u8(src_ptr); - uint8x8_t s1 = vld1_u8(src_ptr + pixel_step); - uint16x8_t blend = vmull_u8(s0, f0); - blend = vmlal_u8(blend, s1, f1); - uint8x8_t blend_u8 = vrshrn_n_u16(blend, 3); - vst1_u8(dst_ptr, blend_u8); - - src_ptr += src_stride; - dst_ptr += 8; - } while (--i != 0); -} - -static void var_filter_block2d_bil_large(const uint8_t *src_ptr, - uint8_t *dst_ptr, int src_stride, - int pixel_step, int dst_width, - int dst_height, int filter_offset) { - const uint8x8_t f0 = vdup_n_u8(8 - filter_offset); - const uint8x8_t f1 = vdup_n_u8(filter_offset); - - int i = dst_height; - do { - int j = 0; - do { - uint8x16_t s0 = vld1q_u8(src_ptr + j); - uint8x16_t s1 = vld1q_u8(src_ptr + j + pixel_step); - uint16x8_t blend_l = vmull_u8(vget_low_u8(s0), f0); - blend_l = vmlal_u8(blend_l, vget_low_u8(s1), f1); - uint16x8_t blend_h = vmull_u8(vget_high_u8(s0), f0); - blend_h = vmlal_u8(blend_h, vget_high_u8(s1), f1); - uint8x16_t blend_u8 = - vcombine_u8(vrshrn_n_u16(blend_l, 3), vrshrn_n_u16(blend_h, 3)); - vst1q_u8(dst_ptr + j, blend_u8); - - j += 16; - } while (j < dst_width); - - src_ptr += src_stride; - dst_ptr += dst_width; - } while (--i != 0); -} - -static void var_filter_block2d_bil_w16(const uint8_t *src_ptr, uint8_t *dst_ptr, - int src_stride, int pixel_step, - int dst_height, int filter_offset) { - var_filter_block2d_bil_large(src_ptr, dst_ptr, src_stride, pixel_step, 16, - dst_height, filter_offset); -} - -static void var_filter_block2d_bil_w32(const uint8_t *src_ptr, uint8_t *dst_ptr, - int src_stride, int pixel_step, - int dst_height, int filter_offset) { - var_filter_block2d_bil_large(src_ptr, dst_ptr, src_stride, pixel_step, 32, - dst_height, filter_offset); -} - -static void var_filter_block2d_bil_w64(const uint8_t *src_ptr, uint8_t *dst_ptr, - int src_stride, int pixel_step, - int dst_height, int filter_offset) { - var_filter_block2d_bil_large(src_ptr, dst_ptr, src_stride, pixel_step, 64, - dst_height, filter_offset); -} - -static void var_filter_block2d_bil_w128(const uint8_t *src_ptr, - uint8_t *dst_ptr, int src_stride, - int pixel_step, int dst_height, - int filter_offset) { - var_filter_block2d_bil_large(src_ptr, dst_ptr, src_stride, pixel_step, 128, - dst_height, filter_offset); -} - -static void var_filter_block2d_avg(const uint8_t *src_ptr, uint8_t *dst_ptr, - int src_stride, int pixel_step, - int dst_width, int dst_height) { - // We only specialise on the filter values for large block sizes (>= 16x16.) - assert(dst_width >= 16 && dst_width % 16 == 0); - - int i = dst_height; - do { - int j = 0; - do { - uint8x16_t s0 = vld1q_u8(src_ptr + j); - uint8x16_t s1 = vld1q_u8(src_ptr + j + pixel_step); - uint8x16_t avg = vrhaddq_u8(s0, s1); - vst1q_u8(dst_ptr + j, avg); - - j += 16; - } while (j < dst_width); - - src_ptr += src_stride; - dst_ptr += dst_width; - } while (--i != 0); -} +#include "aom_dsp/arm/subpel_variance_neon.h" #define SUBPEL_VARIANCE_WXH_NEON(w, h, padding) \ unsigned int aom_sub_pixel_variance##w##x##h##_neon( \ @@ -151,61 +30,62 @@ var_filter_block2d_bil_w##w(src, tmp0, src_stride, 1, (h + padding), \ xoffset); \ var_filter_block2d_bil_w##w(tmp0, tmp1, w, w, h, yoffset); \ - return aom_variance##w##x##h(tmp1, w, ref, ref_stride, sse); \ + return aom_variance##w##x##h##_neon(tmp1, w, ref, ref_stride, sse); \ } -#define SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(w, h, padding) \ - unsigned int aom_sub_pixel_variance##w##x##h##_neon( \ - const uint8_t *src, int src_stride, int xoffset, int yoffset, \ - const uint8_t *ref, int ref_stride, unsigned int *sse) { \ - if (xoffset == 0) { \ - if (yoffset == 0) { \ - return aom_variance##w##x##h(src, src_stride, ref, ref_stride, sse); \ - } else if (yoffset == 4) { \ - uint8_t tmp[w * h]; \ - var_filter_block2d_avg(src, tmp, src_stride, src_stride, w, h); \ - return aom_variance##w##x##h(tmp, w, ref, ref_stride, sse); \ - } else { \ - uint8_t tmp[w * h]; \ - var_filter_block2d_bil_w##w(src, tmp, src_stride, src_stride, h, \ - yoffset); \ - return aom_variance##w##x##h(tmp, w, ref, ref_stride, sse); \ - } \ - } else if (xoffset == 4) { \ - uint8_t tmp0[w * (h + padding)]; \ - if (yoffset == 0) { \ - var_filter_block2d_avg(src, tmp0, src_stride, 1, w, h); \ - return aom_variance##w##x##h(tmp0, w, ref, ref_stride, sse); \ - } else if (yoffset == 4) { \ - uint8_t tmp1[w * (h + padding)]; \ - var_filter_block2d_avg(src, tmp0, src_stride, 1, w, (h + padding)); \ - var_filter_block2d_avg(tmp0, tmp1, w, w, w, h); \ - return aom_variance##w##x##h(tmp1, w, ref, ref_stride, sse); \ - } else { \ - uint8_t tmp1[w * (h + padding)]; \ - var_filter_block2d_avg(src, tmp0, src_stride, 1, w, (h + padding)); \ - var_filter_block2d_bil_w##w(tmp0, tmp1, w, w, h, yoffset); \ - return aom_variance##w##x##h(tmp1, w, ref, ref_stride, sse); \ - } \ - } else { \ - uint8_t tmp0[w * (h + padding)]; \ - if (yoffset == 0) { \ - var_filter_block2d_bil_w##w(src, tmp0, src_stride, 1, h, xoffset); \ - return aom_variance##w##x##h(tmp0, w, ref, ref_stride, sse); \ - } else if (yoffset == 4) { \ - uint8_t tmp1[w * h]; \ - var_filter_block2d_bil_w##w(src, tmp0, src_stride, 1, (h + padding), \ - xoffset); \ - var_filter_block2d_avg(tmp0, tmp1, w, w, w, h); \ - return aom_variance##w##x##h(tmp1, w, ref, ref_stride, sse); \ - } else { \ - uint8_t tmp1[w * h]; \ - var_filter_block2d_bil_w##w(src, tmp0, src_stride, 1, (h + padding), \ - xoffset); \ - var_filter_block2d_bil_w##w(tmp0, tmp1, w, w, h, yoffset); \ - return aom_variance##w##x##h(tmp1, w, ref, ref_stride, sse); \ - } \ - } \ +#define SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON(w, h, padding) \ + unsigned int aom_sub_pixel_variance##w##x##h##_neon( \ + const uint8_t *src, int src_stride, int xoffset, int yoffset, \ + const uint8_t *ref, int ref_stride, unsigned int *sse) { \ + if (xoffset == 0) { \ + if (yoffset == 0) { \ + return aom_variance##w##x##h##_neon(src, src_stride, ref, ref_stride, \ + sse); \ + } else if (yoffset == 4) { \ + uint8_t tmp[w * h]; \ + var_filter_block2d_avg(src, tmp, src_stride, src_stride, w, h); \ + return aom_variance##w##x##h##_neon(tmp, w, ref, ref_stride, sse); \ + } else { \ + uint8_t tmp[w * h]; \ + var_filter_block2d_bil_w##w(src, tmp, src_stride, src_stride, h, \ + yoffset); \ + return aom_variance##w##x##h##_neon(tmp, w, ref, ref_stride, sse); \ + } \ + } else if (xoffset == 4) { \ + uint8_t tmp0[w * (h + padding)]; \ + if (yoffset == 0) { \ + var_filter_block2d_avg(src, tmp0, src_stride, 1, w, h); \ + return aom_variance##w##x##h##_neon(tmp0, w, ref, ref_stride, sse); \ + } else if (yoffset == 4) { \ + uint8_t tmp1[w * (h + padding)]; \ + var_filter_block2d_avg(src, tmp0, src_stride, 1, w, (h + padding)); \ + var_filter_block2d_avg(tmp0, tmp1, w, w, w, h); \ + return aom_variance##w##x##h##_neon(tmp1, w, ref, ref_stride, sse); \ + } else { \ + uint8_t tmp1[w * (h + padding)]; \ + var_filter_block2d_avg(src, tmp0, src_stride, 1, w, (h + padding)); \ + var_filter_block2d_bil_w##w(tmp0, tmp1, w, w, h, yoffset); \ + return aom_variance##w##x##h##_neon(tmp1, w, ref, ref_stride, sse); \ + } \ + } else { \ + uint8_t tmp0[w * (h + padding)]; \ + if (yoffset == 0) { \ + var_filter_block2d_bil_w##w(src, tmp0, src_stride, 1, h, xoffset); \ + return aom_variance##w##x##h##_neon(tmp0, w, ref, ref_stride, sse); \ + } else if (yoffset == 4) { \ + uint8_t tmp1[w * h]; \ + var_filter_block2d_bil_w##w(src, tmp0, src_stride, 1, (h + padding), \ + xoffset); \ + var_filter_block2d_avg(tmp0, tmp1, w, w, w, h); \ + return aom_variance##w##x##h##_neon(tmp1, w, ref, ref_stride, sse); \ + } else { \ + uint8_t tmp1[w * h]; \ + var_filter_block2d_bil_w##w(src, tmp0, src_stride, 1, (h + padding), \ + xoffset); \ + var_filter_block2d_bil_w##w(tmp0, tmp1, w, w, h, yoffset); \ + return aom_variance##w##x##h##_neon(tmp1, w, ref, ref_stride, sse); \ + } \ + } \ } SUBPEL_VARIANCE_WXH_NEON(4, 4, 2)
diff --git a/aom_dsp/arm/subpel_variance_neon.h b/aom_dsp/arm/subpel_variance_neon.h new file mode 100644 index 0000000..18baba6 --- /dev/null +++ b/aom_dsp/arm/subpel_variance_neon.h
@@ -0,0 +1,148 @@ +/* + * Copyright (c) 2026, Alliance for Open Media. All rights reserved. + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_AOM_DSP_ARM_SUBPEL_VARIANCE_NEON_H_ +#define AOM_AOM_DSP_ARM_SUBPEL_VARIANCE_NEON_H_ + +#include <arm_neon.h> +#include "aom_dsp/arm/mem_neon.h" +#include "config/aom_dsp_rtcd.h" + +static inline void var_filter_block2d_bil_w4(const uint8_t *src_ptr, + uint8_t *dst_ptr, int src_stride, + int pixel_step, int dst_height, + int filter_offset) { + const uint8x8_t f0 = vdup_n_u8(8 - filter_offset); + const uint8x8_t f1 = vdup_n_u8(filter_offset); + + int i = dst_height; + do { + uint8x8_t s0 = load_unaligned_u8(src_ptr, src_stride); + uint8x8_t s1 = load_unaligned_u8(src_ptr + pixel_step, src_stride); + uint16x8_t blend = vmull_u8(s0, f0); + blend = vmlal_u8(blend, s1, f1); + uint8x8_t blend_u8 = vrshrn_n_u16(blend, 3); + vst1_u8(dst_ptr, blend_u8); + + src_ptr += 2 * src_stride; + dst_ptr += 2 * 4; + i -= 2; + } while (i != 0); +} + +static inline void var_filter_block2d_bil_w8(const uint8_t *src_ptr, + uint8_t *dst_ptr, int src_stride, + int pixel_step, int dst_height, + int filter_offset) { + const uint8x8_t f0 = vdup_n_u8(8 - filter_offset); + const uint8x8_t f1 = vdup_n_u8(filter_offset); + + int i = dst_height; + do { + uint8x8_t s0 = vld1_u8(src_ptr); + uint8x8_t s1 = vld1_u8(src_ptr + pixel_step); + uint16x8_t blend = vmull_u8(s0, f0); + blend = vmlal_u8(blend, s1, f1); + uint8x8_t blend_u8 = vrshrn_n_u16(blend, 3); + vst1_u8(dst_ptr, blend_u8); + + src_ptr += src_stride; + dst_ptr += 8; + } while (--i != 0); +} + +static inline void var_filter_block2d_bil_large(const uint8_t *src_ptr, + uint8_t *dst_ptr, + int src_stride, int pixel_step, + int dst_width, int dst_height, + int filter_offset) { + const uint8x8_t f0 = vdup_n_u8(8 - filter_offset); + const uint8x8_t f1 = vdup_n_u8(filter_offset); + + int i = dst_height; + do { + int j = 0; + do { + uint8x16_t s0 = vld1q_u8(src_ptr + j); + uint8x16_t s1 = vld1q_u8(src_ptr + j + pixel_step); + uint16x8_t blend_l = vmull_u8(vget_low_u8(s0), f0); + blend_l = vmlal_u8(blend_l, vget_low_u8(s1), f1); + uint16x8_t blend_h = vmull_u8(vget_high_u8(s0), f0); + blend_h = vmlal_u8(blend_h, vget_high_u8(s1), f1); + uint8x16_t blend_u8 = + vcombine_u8(vrshrn_n_u16(blend_l, 3), vrshrn_n_u16(blend_h, 3)); + vst1q_u8(dst_ptr + j, blend_u8); + + j += 16; + } while (j < dst_width); + + src_ptr += src_stride; + dst_ptr += dst_width; + } while (--i != 0); +} + +static inline void var_filter_block2d_bil_w16(const uint8_t *src_ptr, + uint8_t *dst_ptr, int src_stride, + int pixel_step, int dst_height, + int filter_offset) { + var_filter_block2d_bil_large(src_ptr, dst_ptr, src_stride, pixel_step, 16, + dst_height, filter_offset); +} + +static inline void var_filter_block2d_bil_w32(const uint8_t *src_ptr, + uint8_t *dst_ptr, int src_stride, + int pixel_step, int dst_height, + int filter_offset) { + var_filter_block2d_bil_large(src_ptr, dst_ptr, src_stride, pixel_step, 32, + dst_height, filter_offset); +} + +static inline void var_filter_block2d_bil_w64(const uint8_t *src_ptr, + uint8_t *dst_ptr, int src_stride, + int pixel_step, int dst_height, + int filter_offset) { + var_filter_block2d_bil_large(src_ptr, dst_ptr, src_stride, pixel_step, 64, + dst_height, filter_offset); +} + +static inline void var_filter_block2d_bil_w128(const uint8_t *src_ptr, + uint8_t *dst_ptr, int src_stride, + int pixel_step, int dst_height, + int filter_offset) { + var_filter_block2d_bil_large(src_ptr, dst_ptr, src_stride, pixel_step, 128, + dst_height, filter_offset); +} + +static inline void var_filter_block2d_avg(const uint8_t *src_ptr, + uint8_t *dst_ptr, int src_stride, + int pixel_step, int dst_width, + int dst_height) { + // We only specialise on the filter values for large block sizes (>= 16x16.) + assert(dst_width >= 16 && dst_width % 16 == 0); + + int i = dst_height; + do { + int j = 0; + do { + uint8x16_t s0 = vld1q_u8(src_ptr + j); + uint8x16_t s1 = vld1q_u8(src_ptr + j + pixel_step); + uint8x16_t avg = vrhaddq_u8(s0, s1); + vst1q_u8(dst_ptr + j, avg); + + j += 16; + } while (j < dst_width); + + src_ptr += src_stride; + dst_ptr += dst_width; + } while (--i != 0); +} + +#endif // AOM_AOM_DSP_ARM_SUBPEL_VARIANCE_NEON_H_
diff --git a/aom_dsp/arm/subpel_variance_neon_dotprod.c b/aom_dsp/arm/subpel_variance_neon_dotprod.c new file mode 100644 index 0000000..6cb12d8 --- /dev/null +++ b/aom_dsp/arm/subpel_variance_neon_dotprod.c
@@ -0,0 +1,245 @@ +/* + * Copyright (c) 2026, Alliance for Open Media. All rights reserved. + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include <arm_neon.h> + +#include "aom_dsp/arm/subpel_variance_neon.h" + +static inline void bil_variance_8xh_neon_dotprod( + const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, + int pixel_step, int h, uint32_t *sse, int *sum, int filter_offset) { + const uint8x8_t f0 = vdup_n_u8(8 - filter_offset); + const uint8x8_t f1 = vdup_n_u8(filter_offset); + + uint32x4_t src_sum = vdupq_n_u32(0); + uint32x4_t ref_sum = vdupq_n_u32(0); + uint32x4_t sse_u32 = vdupq_n_u32(0); + + do { + uint8x8_t s0_lo = vld1_u8(src); + uint8x8_t s1_lo = vld1_u8(src + pixel_step); + uint16x8_t blend_l = vmull_u8(s0_lo, f0); + blend_l = vmlal_u8(blend_l, s1_lo, f1); + uint8x8_t s0_hi = vld1_u8(src + src_stride); + uint8x8_t s1_hi = vld1_u8(src + src_stride + pixel_step); + uint16x8_t blend_h = vmull_u8(s0_hi, f0); + blend_h = vmlal_u8(blend_h, s1_hi, f1); + uint8x16_t s = + vcombine_u8(vrshrn_n_u16(blend_l, 3), vrshrn_n_u16(blend_h, 3)); + uint8x16_t r = load_u8_8x2(ref, ref_stride); + + src_sum = vdotq_u32(src_sum, s, vdupq_n_u8(1)); + ref_sum = vdotq_u32(ref_sum, r, vdupq_n_u8(1)); + + uint8x16_t abs_diff = vabdq_u8(s, r); + sse_u32 = vdotq_u32(sse_u32, abs_diff, abs_diff); + + src += 2 * src_stride; + ref += 2 * ref_stride; + h -= 2; + } while (h != 0); + + int32x4_t sum_diff = + vsubq_s32(vreinterpretq_s32_u32(src_sum), vreinterpretq_s32_u32(ref_sum)); + *sum = vaddvq_s32(sum_diff); + *sse = vaddvq_u32(sse_u32); +} + +static inline void bil_variance_neon_dotprod(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + int pixel_step, int w, int h, + uint32_t *sse, int *sum, + int filter_offset) { + assert(w != 4); + + if (w == 8) { + bil_variance_8xh_neon_dotprod(src, src_stride, ref, ref_stride, pixel_step, + h, sse, sum, filter_offset); + return; + } + + const uint8x8_t f0 = vdup_n_u8(8 - filter_offset); + const uint8x8_t f1 = vdup_n_u8(filter_offset); + + uint32x4_t src_sum = vdupq_n_u32(0); + uint32x4_t ref_sum = vdupq_n_u32(0); + uint32x4_t sse_u32 = vdupq_n_u32(0); + + do { + int i = 0; + do { + uint8x16_t s0 = vld1q_u8(src + i); + uint8x16_t s1 = vld1q_u8(src + i + pixel_step); + uint16x8_t blend_l = vmull_u8(vget_low_u8(s0), f0); + blend_l = vmlal_u8(blend_l, vget_low_u8(s1), f1); + uint16x8_t blend_h = vmull_u8(vget_high_u8(s0), f0); + blend_h = vmlal_u8(blend_h, vget_high_u8(s1), f1); + uint8x16_t s = + vcombine_u8(vrshrn_n_u16(blend_l, 3), vrshrn_n_u16(blend_h, 3)); + uint8x16_t r = vld1q_u8(ref + i); + + src_sum = vdotq_u32(src_sum, s, vdupq_n_u8(1)); + ref_sum = vdotq_u32(ref_sum, r, vdupq_n_u8(1)); + + uint8x16_t abs_diff = vabdq_u8(s, r); + sse_u32 = vdotq_u32(sse_u32, abs_diff, abs_diff); + + i += 16; + } while (i < w); + + src += src_stride; + ref += ref_stride; + } while (--h != 0); + + int32x4_t sum_diff = + vsubq_s32(vreinterpretq_s32_u32(src_sum), vreinterpretq_s32_u32(ref_sum)); + *sum = vaddvq_s32(sum_diff); + *sse = vaddvq_u32(sse_u32); +} + +static inline void avg_variance_neon_dotprod(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + int pixel_step, int w, int h, + uint32_t *sse, int *sum) { + assert(w >= 16 && w % 16 == 0); + uint32x4_t src_sum = vdupq_n_u32(0); + uint32x4_t ref_sum = vdupq_n_u32(0); + uint32x4_t sse_u32 = vdupq_n_u32(0); + + do { + int i = 0; + do { + uint8x16_t s0 = vld1q_u8(src + i); + uint8x16_t s1 = vld1q_u8(src + i + pixel_step); + uint8x16_t s = vrhaddq_u8(s0, s1); + uint8x16_t r = vld1q_u8(ref + i); + + src_sum = vdotq_u32(src_sum, s, vdupq_n_u8(1)); + ref_sum = vdotq_u32(ref_sum, r, vdupq_n_u8(1)); + + uint8x16_t abs_diff = vabdq_u8(s, r); + sse_u32 = vdotq_u32(sse_u32, abs_diff, abs_diff); + + i += 16; + } while (i < w); + + src += src_stride; + ref += ref_stride; + } while (--h != 0); + + int32x4_t sum_diff = + vsubq_s32(vreinterpretq_s32_u32(src_sum), vreinterpretq_s32_u32(ref_sum)); + *sum = vaddvq_s32(sum_diff); + *sse = vaddvq_u32(sse_u32); +} + +#define SUBPEL_VARIANCE_4XH_NEON_DOTPROD(h, padding) \ + unsigned int aom_sub_pixel_variance4x##h##_neon_dotprod( \ + const uint8_t *src, int src_stride, int xoffset, int yoffset, \ + const uint8_t *ref, int ref_stride, uint32_t *sse) { \ + uint8_t tmp0[4 * (h + padding)]; \ + uint8_t tmp1[4 * h]; \ + var_filter_block2d_bil_w4(src, tmp0, src_stride, 1, (h + padding), \ + xoffset); \ + var_filter_block2d_bil_w4(tmp0, tmp1, 4, 4, h, yoffset); \ + return aom_variance4x##h##_neon_dotprod(tmp1, 4, ref, ref_stride, sse); \ + } + +#define SUBPEL_VARIANCE_WXH_NEON_DOTPROD(w, h, shift, padding) \ + unsigned int aom_sub_pixel_variance##w##x##h##_neon_dotprod( \ + const uint8_t *src, int src_stride, int xoffset, int yoffset, \ + const uint8_t *ref, int ref_stride, unsigned int *sse) { \ + uint8_t tmp[w * (h + padding)]; \ + int sum; \ + var_filter_block2d_bil_w##w(src, tmp, src_stride, 1, h + padding, \ + xoffset); \ + bil_variance_neon_dotprod(tmp, w, ref, ref_stride, w, w, h, sse, &sum, \ + yoffset); \ + return *sse - (uint32_t)(((int64_t)sum * sum) >> shift); \ + } + +#define SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON_DOTPROD(w, h, shift, padding) \ + unsigned int aom_sub_pixel_variance##w##x##h##_neon_dotprod( \ + const uint8_t *src, int src_stride, int xoffset, int yoffset, \ + const uint8_t *ref, int ref_stride, unsigned int *sse) { \ + int sum; \ + if (xoffset == 0) { \ + if (yoffset == 0) { \ + return aom_variance##w##x##h##_neon_dotprod(src, src_stride, ref, \ + ref_stride, sse); \ + } else if (yoffset == 4) { \ + avg_variance_neon_dotprod(src, src_stride, ref, ref_stride, \ + src_stride, w, h, sse, &sum); \ + } else { \ + bil_variance_neon_dotprod(src, src_stride, ref, ref_stride, \ + src_stride, w, h, sse, &sum, yoffset); \ + } \ + } else if (xoffset == 4) { \ + uint8_t tmp[w * (h + padding)]; \ + if (yoffset == 0) { \ + avg_variance_neon_dotprod(src, src_stride, ref, ref_stride, 1, w, h, \ + sse, &sum); \ + } else if (yoffset == 4) { \ + var_filter_block2d_avg(src, tmp, src_stride, 1, w, h + padding); \ + avg_variance_neon_dotprod(tmp, w, ref, ref_stride, w, w, h, sse, \ + &sum); \ + } else { \ + var_filter_block2d_avg(src, tmp, src_stride, 1, w, h + padding); \ + bil_variance_neon_dotprod(tmp, w, ref, ref_stride, w, w, h, sse, &sum, \ + yoffset); \ + } \ + } else { \ + uint8_t tmp[w * (h + padding)]; \ + if (yoffset == 0) { \ + bil_variance_neon_dotprod(src, src_stride, ref, ref_stride, 1, w, h, \ + sse, &sum, xoffset); \ + } else if (yoffset == 4) { \ + var_filter_block2d_bil_w##w(src, tmp, src_stride, 1, h + padding, \ + xoffset); \ + avg_variance_neon_dotprod(tmp, w, ref, ref_stride, w, w, h, sse, \ + &sum); \ + } else { \ + var_filter_block2d_bil_w##w(src, tmp, src_stride, 1, h + padding, \ + xoffset); \ + bil_variance_neon_dotprod(tmp, w, ref, ref_stride, w, w, h, sse, &sum, \ + yoffset); \ + } \ + } \ + return *sse - (uint32_t)(((int64_t)sum * sum) >> shift); \ + } + +SUBPEL_VARIANCE_4XH_NEON_DOTPROD(8, 2) +SUBPEL_VARIANCE_WXH_NEON_DOTPROD(8, 4, 5, 1) +SUBPEL_VARIANCE_WXH_NEON_DOTPROD(8, 8, 6, 1) +SUBPEL_VARIANCE_WXH_NEON_DOTPROD(8, 16, 7, 1) +SUBPEL_VARIANCE_WXH_NEON_DOTPROD(16, 8, 7, 1) +SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON_DOTPROD(16, 16, 8, 1) +SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON_DOTPROD(16, 32, 9, 1) +SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON_DOTPROD(32, 16, 9, 1) +SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON_DOTPROD(32, 32, 10, 1) +SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON_DOTPROD(32, 64, 11, 1) +SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON_DOTPROD(64, 32, 11, 1) +SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON_DOTPROD(64, 64, 12, 1) +SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON_DOTPROD(64, 128, 13, 1) +SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON_DOTPROD(128, 64, 13, 1) +SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON_DOTPROD(128, 128, 14, 1) + +#if !CONFIG_REALTIME_ONLY +SUBPEL_VARIANCE_4XH_NEON_DOTPROD(16, 2) +SUBPEL_VARIANCE_WXH_NEON_DOTPROD(8, 32, 8, 1) +SUBPEL_VARIANCE_WXH_NEON_DOTPROD(16, 4, 6, 1) +SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON_DOTPROD(16, 64, 10, 1) +SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON_DOTPROD(32, 8, 8, 1) +SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON_DOTPROD(64, 16, 10, 1) +#endif // !CONFIG_REALTIME_ONLY + +#undef SUBPEL_VARIANCE_WXH_NEON_DOTPROD +#undef SPECIALIZED_SUBPEL_VARIANCE_WXH_NEON_DOTPROD
diff --git a/aom_dsp/arm/transpose_neon.h b/aom_dsp/arm/transpose_neon.h index 88df0d6..2a548bb 100644 --- a/aom_dsp/arm/transpose_neon.h +++ b/aom_dsp/arm/transpose_neon.h
@@ -17,6 +17,155 @@ #include "aom_dsp/aom_dsp_common.h" // For AOM_FORCE_INLINE. #include "config/aom_config.h" +static inline void transpose_concat_elems_u8_4x4(uint8x8_t a0, uint8x8_t a1, + uint8x8_t a2, uint8x8_t a3, + uint8x16_t *b) { + // Transpose 8-bit elements and concatenate result rows as follows: + // a0: 00, 01, 02, 03, XX, XX, XX, XX + // a1: 10, 11, 12, 13, XX, XX, XX, XX + // a2: 20, 21, 22, 23, XX, XX, XX, XX + // a3: 30, 31, 32, 33, XX, XX, XX, XX + // + // b: 00, 10, 20, 30, 01, 11, 21, 31, 02, 12, 22, 32, 03, 13, 23, 33 + + uint8x16_t a0q = vcombine_u8(a0, vdup_n_u8(0)); + uint8x16_t a1q = vcombine_u8(a1, vdup_n_u8(0)); + uint8x16_t a2q = vcombine_u8(a2, vdup_n_u8(0)); + uint8x16_t a3q = vcombine_u8(a3, vdup_n_u8(0)); + + uint8x16_t a02 = vzipq_u8(a0q, a2q).val[0]; + uint8x16_t a13 = vzipq_u8(a1q, a3q).val[0]; + + *b = vzipq_u8(a02, a13).val[0]; +} + +static inline void transpose_concat_elems_u8_8x4(uint8x8_t a0, uint8x8_t a1, + uint8x8_t a2, uint8x8_t a3, + uint8x16_t *b0, + uint8x16_t *b1) { + // Transpose 8-bit elements and concatenate result rows as follows: + // a0: 00, 01, 02, 03, 04, 05, 06, 07 + // a1: 10, 11, 12, 13, 14, 15, 16, 17 + // a2: 20, 21, 22, 23, 24, 25, 26, 27 + // a3: 30, 31, 32, 33, 34, 35, 36, 37 + // + // b0: 00, 10, 20, 30, 01, 11, 21, 31, 02, 12, 22, 32, 03, 13, 23, 33 + // b1: 04, 14, 24, 34, 05, 15, 25, 35, 06, 16, 26, 36, 07, 17, 27, 37 + + uint8x16_t a0q = vcombine_u8(a0, vdup_n_u8(0)); + uint8x16_t a1q = vcombine_u8(a1, vdup_n_u8(0)); + uint8x16_t a2q = vcombine_u8(a2, vdup_n_u8(0)); + uint8x16_t a3q = vcombine_u8(a3, vdup_n_u8(0)); + + uint8x16_t a02 = vzipq_u8(a0q, a2q).val[0]; + uint8x16_t a13 = vzipq_u8(a1q, a3q).val[0]; + + uint8x16x2_t a0123 = vzipq_u8(a02, a13); + + *b0 = a0123.val[0]; + *b1 = a0123.val[1]; +} + +static inline void transpose_concat_elems_s8_4x4(int8x8_t a0, int8x8_t a1, + int8x8_t a2, int8x8_t a3, + int8x16_t *b) { + // Transpose 8-bit elements and concatenate result rows as follows: + // a0: 00, 01, 02, 03, XX, XX, XX, XX + // a1: 10, 11, 12, 13, XX, XX, XX, XX + // a2: 20, 21, 22, 23, XX, XX, XX, XX + // a3: 30, 31, 32, 33, XX, XX, XX, XX + // + // b: 00, 10, 20, 30, 01, 11, 21, 31, 02, 12, 22, 32, 03, 13, 23, 33 + + int8x16_t a0q = vcombine_s8(a0, vdup_n_s8(0)); + int8x16_t a1q = vcombine_s8(a1, vdup_n_s8(0)); + int8x16_t a2q = vcombine_s8(a2, vdup_n_s8(0)); + int8x16_t a3q = vcombine_s8(a3, vdup_n_s8(0)); + + int8x16_t a02 = vzipq_s8(a0q, a2q).val[0]; + int8x16_t a13 = vzipq_s8(a1q, a3q).val[0]; + + *b = vzipq_s8(a02, a13).val[0]; +} + +static inline void transpose_concat_elems_s8_8x4(int8x8_t a0, int8x8_t a1, + int8x8_t a2, int8x8_t a3, + int8x16_t *b0, int8x16_t *b1) { + // Transpose 8-bit elements and concatenate result rows as follows: + // a0: 00, 01, 02, 03, 04, 05, 06, 07 + // a1: 10, 11, 12, 13, 14, 15, 16, 17 + // a2: 20, 21, 22, 23, 24, 25, 26, 27 + // a3: 30, 31, 32, 33, 34, 35, 36, 37 + // + // b0: 00, 10, 20, 30, 01, 11, 21, 31, 02, 12, 22, 32, 03, 13, 23, 33 + // b1: 04, 14, 24, 34, 05, 15, 25, 35, 06, 16, 26, 36, 07, 17, 27, 37 + + int8x16_t a0q = vcombine_s8(a0, vdup_n_s8(0)); + int8x16_t a1q = vcombine_s8(a1, vdup_n_s8(0)); + int8x16_t a2q = vcombine_s8(a2, vdup_n_s8(0)); + int8x16_t a3q = vcombine_s8(a3, vdup_n_s8(0)); + + int8x16_t a02 = vzipq_s8(a0q, a2q).val[0]; + int8x16_t a13 = vzipq_s8(a1q, a3q).val[0]; + + int8x16x2_t a0123 = vzipq_s8(a02, a13); + + *b0 = a0123.val[0]; + *b1 = a0123.val[1]; +} + +static inline void transpose_concat_elems_s16_4x4(int16x4_t s0, int16x4_t s1, + int16x4_t s2, int16x4_t s3, + int16x8_t res[2]) { + // Transpose 16-bit elements and concatenate result rows as follows: + // s0: 00, 01, 02, 03 + // s1: 10, 11, 12, 13 + // s2: 20, 21, 22, 23 + // s3: 30, 31, 32, 33 + // + // res[0]: 00 10 20 30 01 11 21 31 + // res[1]: 02 12 22 32 03 13 23 33 + + int16x8_t s0q = vcombine_s16(s0, vdup_n_s16(0)); + int16x8_t s1q = vcombine_s16(s1, vdup_n_s16(0)); + int16x8_t s2q = vcombine_s16(s2, vdup_n_s16(0)); + int16x8_t s3q = vcombine_s16(s3, vdup_n_s16(0)); + + int16x8_t s02 = vzipq_s16(s0q, s2q).val[0]; + int16x8_t s13 = vzipq_s16(s1q, s3q).val[0]; + + int16x8x2_t s0123 = vzipq_s16(s02, s13); + + res[0] = s0123.val[0]; + res[1] = s0123.val[1]; +} + +static inline void transpose_concat_elems_s16_8x4(int16x8_t s0, int16x8_t s1, + int16x8_t s2, int16x8_t s3, + int16x8_t res[4]) { + // Transpose 16-bit elements and concatenate result rows as follows: + // s0: 00, 01, 02, 03, 04, 05, 06, 07 + // s1: 10, 11, 12, 13, 14, 15, 16, 17 + // s2: 20, 21, 22, 23, 24, 25, 26, 27 + // s3: 30, 31, 32, 33, 34, 35, 36, 37 + // + // res[0]: 00 10 20 30 01 11 21 31 + // res[1]: 02 12 22 32 03 13 23 33 + // res[2]: 04 14 24 34 05 15 25 35 + // res[3]: 06 16 26 36 07 17 27 37 + + int16x8x2_t s02 = vzipq_s16(s0, s2); + int16x8x2_t s13 = vzipq_s16(s1, s3); + + int16x8x2_t s0123_lo = vzipq_s16(s02.val[0], s13.val[0]); + int16x8x2_t s0123_hi = vzipq_s16(s02.val[1], s13.val[1]); + + res[0] = s0123_lo.val[0]; + res[1] = s0123_lo.val[1]; + res[2] = s0123_hi.val[0]; + res[3] = s0123_hi.val[1]; +} + static inline void transpose_elems_u8_8x8( uint8x8_t a0, uint8x8_t a1, uint8x8_t a2, uint8x8_t a3, uint8x8_t a4, uint8x8_t a5, uint8x8_t a6, uint8x8_t a7, uint8x8_t *o0, uint8x8_t *o1,
diff --git a/aom_dsp/arm/variance_neon_dotprod.c b/aom_dsp/arm/variance_neon_dotprod.c index dd03443..3a3965e 100644 --- a/aom_dsp/arm/variance_neon_dotprod.c +++ b/aom_dsp/arm/variance_neon_dotprod.c
@@ -178,7 +178,7 @@ return *sse - (uint32_t)(((int64_t)sum * sum) >> shift); \ } -VARIANCE_WXH_NEON_DOTPROD(4, 4, 4) +// The Armv8.0 Neon implementation is faster than Neon DotProd for 4x4. VARIANCE_WXH_NEON_DOTPROD(4, 8, 5) VARIANCE_WXH_NEON_DOTPROD(8, 4, 5)
diff --git a/aom_dsp/avg.c b/aom_dsp/avg.c index 5cba8a8..6cfbfdd 100644 --- a/aom_dsp/avg.c +++ b/aom_dsp/avg.c
@@ -119,18 +119,20 @@ int16_t buffer2[16]; int16_t *tmp_buf = &buffer[0]; for (idx = 0; idx < 4; ++idx) { - hadamard_col4(src_diff, src_stride, tmp_buf); // src_diff: 9 bit - // dynamic range [-255, 255] + // src_diff: 9 bit (8b), 13 bit (HBD) + // dynamic range [-255, 255] (8b), [-4095, 4095] (HBD) + hadamard_col4(src_diff, src_stride, tmp_buf); tmp_buf += 4; ++src_diff; } tmp_buf = &buffer[0]; for (idx = 0; idx < 4; ++idx) { - hadamard_col4(tmp_buf, 4, buffer2 + 4 * idx); // tmp_buf: 12 bit - // dynamic range [-2040, 2040] - // buffer2: 15 bit - // dynamic range [-16320, 16320] + // tmp_buf: 10 bit (8b), 14 bit (HBD) + // dynamic range [-510, 510] (8b), [-8190, 8190] (HBD) + // buffer2: 11 bit (8b), 15 bit (HBD) + // dynamic range [-1020, 1020] (8b), [-16380, 16380] (HBD) + hadamard_col4(tmp_buf, 4, buffer2 + 4 * idx); ++tmp_buf; }
diff --git a/aom_dsp/grain_table.c b/aom_dsp/grain_table.c index 6c525d9..38ca1e4 100644 --- a/aom_dsp/grain_table.c +++ b/aom_dsp/grain_table.c
@@ -30,6 +30,7 @@ * cCr <ar_coeff_cr_0> .... * E <start-time> ... */ +#include <inttypes.h> #include <string.h> #include <stdio.h> #include "aom_dsp/aom_dsp_common.h" @@ -50,7 +51,6 @@ if (num_read != 5) { aom_internal_error(error_info, AOM_CODEC_ERROR, "Unable to read entry header. Read %d != 5", num_read); - return; } if (pars->update_parameters) { num_read = fscanf(file, "p %d %d %d %d %d %d %d %d %d %d %d %d\n", @@ -63,83 +63,70 @@ aom_internal_error(error_info, AOM_CODEC_ERROR, "Unable to read entry params. Read %d != 12", num_read); - return; } if (!fscanf(file, "\tsY %d ", &pars->num_y_points)) { aom_internal_error(error_info, AOM_CODEC_ERROR, "Unable to read num y points"); - return; } for (int i = 0; i < pars->num_y_points; ++i) { if (2 != fscanf(file, "%d %d", &pars->scaling_points_y[i][0], &pars->scaling_points_y[i][1])) { aom_internal_error(error_info, AOM_CODEC_ERROR, "Unable to read y scaling points"); - return; } } if (!fscanf(file, "\n\tsCb %d", &pars->num_cb_points)) { aom_internal_error(error_info, AOM_CODEC_ERROR, "Unable to read num cb points"); - return; } for (int i = 0; i < pars->num_cb_points; ++i) { if (2 != fscanf(file, "%d %d", &pars->scaling_points_cb[i][0], &pars->scaling_points_cb[i][1])) { aom_internal_error(error_info, AOM_CODEC_ERROR, "Unable to read cb scaling points"); - return; } } if (!fscanf(file, "\n\tsCr %d", &pars->num_cr_points)) { aom_internal_error(error_info, AOM_CODEC_ERROR, "Unable to read num cr points"); - return; } for (int i = 0; i < pars->num_cr_points; ++i) { if (2 != fscanf(file, "%d %d", &pars->scaling_points_cr[i][0], &pars->scaling_points_cr[i][1])) { aom_internal_error(error_info, AOM_CODEC_ERROR, "Unable to read cr scaling points"); - return; } } if (fscanf(file, "\n\tcY")) { aom_internal_error(error_info, AOM_CODEC_ERROR, "Unable to read Y coeffs header (cY)"); - return; } const int n = 2 * pars->ar_coeff_lag * (pars->ar_coeff_lag + 1); for (int i = 0; i < n; ++i) { if (1 != fscanf(file, "%d", &pars->ar_coeffs_y[i])) { aom_internal_error(error_info, AOM_CODEC_ERROR, "Unable to read Y coeffs"); - return; } } if (fscanf(file, "\n\tcCb")) { aom_internal_error(error_info, AOM_CODEC_ERROR, "Unable to read Cb coeffs header (cCb)"); - return; } for (int i = 0; i <= n; ++i) { if (1 != fscanf(file, "%d", &pars->ar_coeffs_cb[i])) { aom_internal_error(error_info, AOM_CODEC_ERROR, "Unable to read Cb coeffs"); - return; } } if (fscanf(file, "\n\tcCr")) { aom_internal_error(error_info, AOM_CODEC_ERROR, "Unable read to Cr coeffs header (cCr)"); - return; } for (int i = 0; i <= n; ++i) { if (1 != fscanf(file, "%d", &pars->ar_coeffs_cr[i])) { aom_internal_error(error_info, AOM_CODEC_ERROR, "Unable to read Cr coeffs"); - return; } } (void)fscanf(file, "\n"); @@ -276,74 +263,84 @@ aom_codec_err_t aom_film_grain_table_read( aom_film_grain_table_t *t, const char *filename, struct aom_internal_error_info *error_info) { - FILE *file = fopen(filename, "rb"); - if (!file) { - aom_internal_error(error_info, AOM_CODEC_ERROR, "Unable to open %s", - filename); - return error_info->error_code; - } + FILE *const file = fopen(filename, "rb"); error_info->error_code = AOM_CODEC_OK; - // Read in one extra character as there should be white space after - // the header. - char magic[9]; - if (!fread(magic, 9, 1, file) || memcmp(magic, kFileMagic, 8)) { - aom_internal_error(error_info, AOM_CODEC_ERROR, - "Unable to read (or invalid) file magic"); - fclose(file); - return error_info->error_code; + if (!file) { + error_info->error_code = AOM_CODEC_ERROR; + return AOM_CODEC_ERROR; } - aom_film_grain_table_entry_t *prev_entry = NULL; - while (!feof(file)) { - aom_film_grain_table_entry_t *entry = aom_malloc(sizeof(*entry)); - if (!entry) { - aom_internal_error(error_info, AOM_CODEC_MEM_ERROR, - "Unable to allocate grain table entry"); - break; + error_info->setjmp = 1; + if (setjmp(error_info->jmp) == 0) { + // Read in one extra character as there should be white space after + // the header. + char magic[9]; + if (!fread(magic, 9, 1, file) || memcmp(magic, kFileMagic, 8)) { + aom_internal_error(error_info, AOM_CODEC_ERROR, + "Unable to read (or invalid) file magic"); } - memset(entry, 0, sizeof(*entry)); - grain_table_entry_read(file, error_info, entry); - entry->next = NULL; - if (prev_entry) prev_entry->next = entry; - if (!t->head) t->head = entry; - t->tail = entry; - prev_entry = entry; + aom_film_grain_table_entry_t *prev_entry = NULL; + for (;;) { + // Check for end of file before attempting to read. + int c = fgetc(file); + if (c == EOF) { + break; + } + ungetc(c, file); - if (error_info->error_code != AOM_CODEC_OK) break; + aom_film_grain_table_entry_t *entry = aom_malloc(sizeof(*entry)); + if (!entry) { + aom_internal_error(error_info, AOM_CODEC_MEM_ERROR, + "Unable to allocate grain table entry"); + } + memset(entry, 0, sizeof(*entry)); + grain_table_entry_read(file, error_info, entry); + entry->next = NULL; + + if (prev_entry) prev_entry->next = entry; + if (!t->head) t->head = entry; + t->tail = entry; + prev_entry = entry; + } } fclose(file); + error_info->setjmp = 0; + return error_info->error_code; } aom_codec_err_t aom_film_grain_table_write( const aom_film_grain_table_t *t, const char *filename, struct aom_internal_error_info *error_info) { + FILE *const file = fopen(filename, "wb"); error_info->error_code = AOM_CODEC_OK; - FILE *file = fopen(filename, "wb"); if (!file) { - aom_internal_error(error_info, AOM_CODEC_ERROR, "Unable to open file %s", - filename); - return error_info->error_code; + error_info->error_code = AOM_CODEC_ERROR; + return AOM_CODEC_ERROR; } - if (!fwrite(kFileMagic, 8, 1, file)) { - aom_internal_error(error_info, AOM_CODEC_ERROR, - "Unable to write file magic"); - fclose(file); - return error_info->error_code; + error_info->setjmp = 1; + if (setjmp(error_info->jmp) == 0) { + if (!fwrite(kFileMagic, 8, 1, file)) { + aom_internal_error(error_info, AOM_CODEC_ERROR, + "Unable to write file magic"); + } + + fprintf(file, "\n"); + aom_film_grain_table_entry_t *entry = t->head; + while (entry) { + grain_table_entry_write(file, entry); + entry = entry->next; + } } - fprintf(file, "\n"); - aom_film_grain_table_entry_t *entry = t->head; - while (entry) { - grain_table_entry_write(file, entry); - entry = entry->next; - } fclose(file); + error_info->setjmp = 0; + return error_info->error_code; }
diff --git a/aom_dsp/prob.h b/aom_dsp/prob.h index bb2c4a9..8bec067 100644 --- a/aom_dsp/prob.h +++ b/aom_dsp/prob.h
@@ -18,7 +18,7 @@ #include "config/aom_config.h" #include "aom_dsp/aom_dsp_common.h" -#include "aom_dsp/entcode.h" + #include "aom_ports/bitops.h" #include "aom_ports/mem.h"
diff --git a/aom_dsp/riscv/mem_rvv.h b/aom_dsp/riscv/mem_rvv.h index e592690..6b44748 100644 --- a/aom_dsp/riscv/mem_rvv.h +++ b/aom_dsp/riscv/mem_rvv.h
@@ -100,6 +100,28 @@ *s6 = __riscv_vle16_v_i16mf2(s, vl); } +static inline void load_s16_4x8(const int16_t *s, int p, vint16mf2_t *const s0, + vint16mf2_t *const s1, vint16mf2_t *const s2, + vint16mf2_t *const s3, vint16mf2_t *const s4, + vint16mf2_t *const s5, vint16mf2_t *const s6, + vint16mf2_t *const s7, size_t vl) { + *s0 = __riscv_vle16_v_i16mf2(s, vl); + s += p; + *s1 = __riscv_vle16_v_i16mf2(s, vl); + s += p; + *s2 = __riscv_vle16_v_i16mf2(s, vl); + s += p; + *s3 = __riscv_vle16_v_i16mf2(s, vl); + s += p; + *s4 = __riscv_vle16_v_i16mf2(s, vl); + s += p; + *s5 = __riscv_vle16_v_i16mf2(s, vl); + s += p; + *s6 = __riscv_vle16_v_i16mf2(s, vl); + s += p; + *s7 = __riscv_vle16_v_i16mf2(s, vl); +} + static inline void load_s16_8x7(const int16_t *s, int p, vint16m1_t *const s0, vint16m1_t *const s1, vint16m1_t *const s2, vint16m1_t *const s3, vint16m1_t *const s4, @@ -150,6 +172,33 @@ *s10 = __riscv_vle16_v_i16mf2(s, vl); } +static inline void load_s16_8x10(const int16_t *s, int p, vint16m1_t *const s0, + vint16m1_t *const s1, vint16m1_t *const s2, + vint16m1_t *const s3, vint16m1_t *const s4, + vint16m1_t *const s5, vint16m1_t *const s6, + vint16m1_t *const s7, vint16m1_t *const s8, + vint16m1_t *const s9, size_t vl) { + *s0 = __riscv_vle16_v_i16m1(s, vl); + s += p; + *s1 = __riscv_vle16_v_i16m1(s, vl); + s += p; + *s2 = __riscv_vle16_v_i16m1(s, vl); + s += p; + *s3 = __riscv_vle16_v_i16m1(s, vl); + s += p; + *s4 = __riscv_vle16_v_i16m1(s, vl); + s += p; + *s5 = __riscv_vle16_v_i16m1(s, vl); + s += p; + *s6 = __riscv_vle16_v_i16m1(s, vl); + s += p; + *s7 = __riscv_vle16_v_i16m1(s, vl); + s += p; + *s8 = __riscv_vle16_v_i16m1(s, vl); + s += p; + *s9 = __riscv_vle16_v_i16m1(s, vl); +} + static inline void load_s16_8x11(const int16_t *s, int p, vint16m1_t *const s0, vint16m1_t *const s1, vint16m1_t *const s2, vint16m1_t *const s3, vint16m1_t *const s4, @@ -295,4 +344,277 @@ __riscv_vse16_v_u16mf2(s, s3, vl); } +static inline void load_u8_8x4(const uint8_t *s, const ptrdiff_t p, + vuint8mf2_t *const s0, vuint8mf2_t *const s1, + vuint8mf2_t *const s2, vuint8mf2_t *const s3, + size_t vl) { + *s0 = __riscv_vle8_v_u8mf2(s, vl); + s += p; + *s1 = __riscv_vle8_v_u8mf2(s, vl); + s += p; + *s2 = __riscv_vle8_v_u8mf2(s, vl); + s += p; + *s3 = __riscv_vle8_v_u8mf2(s, vl); +} + +static inline void load_u8_4x5(const uint8_t *s, const ptrdiff_t p, + vuint8mf4_t *const s0, vuint8mf4_t *const s1, + vuint8mf4_t *const s2, vuint8mf4_t *const s3, + vuint8mf4_t *const s4, size_t vl) { + *s0 = __riscv_vle8_v_u8mf4(s, vl); + s += p; + *s1 = __riscv_vle8_v_u8mf4(s, vl); + s += p; + *s2 = __riscv_vle8_v_u8mf4(s, vl); + s += p; + *s3 = __riscv_vle8_v_u8mf4(s, vl); + s += p; + *s4 = __riscv_vle8_v_u8mf4(s, vl); +} + +static inline void load_u8_4x7(const uint8_t *s, const ptrdiff_t p, + vuint8mf4_t *const s0, vuint8mf4_t *const s1, + vuint8mf4_t *const s2, vuint8mf4_t *const s3, + vuint8mf4_t *const s4, vuint8mf4_t *const s5, + vuint8mf4_t *const s6, size_t vl) { + *s0 = __riscv_vle8_v_u8mf4(s, vl); + s += p; + *s1 = __riscv_vle8_v_u8mf4(s, vl); + s += p; + *s2 = __riscv_vle8_v_u8mf4(s, vl); + s += p; + *s3 = __riscv_vle8_v_u8mf4(s, vl); + s += p; + *s4 = __riscv_vle8_v_u8mf4(s, vl); + s += p; + *s5 = __riscv_vle8_v_u8mf4(s, vl); + s += p; + *s6 = __riscv_vle8_v_u8mf4(s, vl); +} + +static inline void load_u8_8x5(const uint8_t *s, const ptrdiff_t p, + vuint8mf2_t *const s0, vuint8mf2_t *const s1, + vuint8mf2_t *const s2, vuint8mf2_t *const s3, + vuint8mf2_t *const s4, size_t vl) { + *s0 = __riscv_vle8_v_u8mf2(s, vl); + s += p; + *s1 = __riscv_vle8_v_u8mf2(s, vl); + s += p; + *s2 = __riscv_vle8_v_u8mf2(s, vl); + s += p; + *s3 = __riscv_vle8_v_u8mf2(s, vl); + s += p; + *s4 = __riscv_vle8_v_u8mf2(s, vl); +} + +static inline void load_u8_8x7(const uint8_t *s, const ptrdiff_t p, + vuint8mf2_t *const s0, vuint8mf2_t *const s1, + vuint8mf2_t *const s2, vuint8mf2_t *const s3, + vuint8mf2_t *const s4, vuint8mf2_t *const s5, + vuint8mf2_t *const s6, size_t vl) { + *s0 = __riscv_vle8_v_u8mf2(s, vl); + s += p; + *s1 = __riscv_vle8_v_u8mf2(s, vl); + s += p; + *s2 = __riscv_vle8_v_u8mf2(s, vl); + s += p; + *s3 = __riscv_vle8_v_u8mf2(s, vl); + s += p; + *s4 = __riscv_vle8_v_u8mf2(s, vl); + s += p; + *s5 = __riscv_vle8_v_u8mf2(s, vl); + s += p; + *s6 = __riscv_vle8_v_u8mf2(s, vl); +} + +static inline void load_u8_8x8(const uint8_t *s, int p, vuint8mf2_t *const s0, + vuint8mf2_t *const s1, vuint8mf2_t *const s2, + vuint8mf2_t *const s3, vuint8mf2_t *const s4, + vuint8mf2_t *const s5, vuint8mf2_t *const s6, + vuint8mf2_t *const s7, size_t vl) { + *s0 = __riscv_vle8_v_u8mf2(s, vl); + s += p; + *s1 = __riscv_vle8_v_u8mf2(s, vl); + s += p; + *s2 = __riscv_vle8_v_u8mf2(s, vl); + s += p; + *s3 = __riscv_vle8_v_u8mf2(s, vl); + s += p; + *s4 = __riscv_vle8_v_u8mf2(s, vl); + s += p; + *s5 = __riscv_vle8_v_u8mf2(s, vl); + s += p; + *s6 = __riscv_vle8_v_u8mf2(s, vl); + s += p; + *s7 = __riscv_vle8_v_u8mf2(s, vl); +} + +static inline void load_stride_u8_8x8( + const uint8_t *s, int p, vuint8mf2_t *const s0, vuint8mf2_t *const s1, + vuint8mf2_t *const s2, vuint8mf2_t *const s3, vuint8mf2_t *const s4, + vuint8mf2_t *const s5, vuint8mf2_t *const s6, vuint8mf2_t *const s7, + size_t vl) { + *s0 = __riscv_vlse8_v_u8mf2(s + 0, p, vl); + *s1 = __riscv_vlse8_v_u8mf2(s + 1, p, vl); + *s2 = __riscv_vlse8_v_u8mf2(s + 2, p, vl); + *s3 = __riscv_vlse8_v_u8mf2(s + 3, p, vl); + *s4 = __riscv_vlse8_v_u8mf2(s + 4, p, vl); + *s5 = __riscv_vlse8_v_u8mf2(s + 5, p, vl); + *s6 = __riscv_vlse8_v_u8mf2(s + 6, p, vl); + *s7 = __riscv_vlse8_v_u8mf2(s + 7, p, vl); +} + +static inline void store_s16_8x8(int16_t *s, const ptrdiff_t p, + const vint16m1_t s0, const vint16m1_t s1, + const vint16m1_t s2, const vint16m1_t s3, + const vint16m1_t s4, const vint16m1_t s5, + const vint16m1_t s6, const vint16m1_t s7, + size_t vl) { + __riscv_vse16_v_i16m1(s, s0, vl); + s += p; + __riscv_vse16_v_i16m1(s, s1, vl); + s += p; + __riscv_vse16_v_i16m1(s, s2, vl); + s += p; + __riscv_vse16_v_i16m1(s, s3, vl); + s += p; + __riscv_vse16_v_i16m1(s, s4, vl); + s += p; + __riscv_vse16_v_i16m1(s, s5, vl); + s += p; + __riscv_vse16_v_i16m1(s, s6, vl); + s += p; + __riscv_vse16_v_i16m1(s, s7, vl); +} + +static inline void store_u16_8x8(uint16_t *s, const ptrdiff_t p, + const vuint16m1_t s0, const vuint16m1_t s1, + const vuint16m1_t s2, const vuint16m1_t s3, + const vuint16m1_t s4, const vuint16m1_t s5, + const vuint16m1_t s6, const vuint16m1_t s7, + size_t vl) { + __riscv_vse16_v_u16m1(s, s0, vl); + s += p; + __riscv_vse16_v_u16m1(s, s1, vl); + s += p; + __riscv_vse16_v_u16m1(s, s2, vl); + s += p; + __riscv_vse16_v_u16m1(s, s3, vl); + s += p; + __riscv_vse16_v_u16m1(s, s4, vl); + s += p; + __riscv_vse16_v_u16m1(s, s5, vl); + s += p; + __riscv_vse16_v_u16m1(s, s6, vl); + s += p; + __riscv_vse16_v_u16m1(s, s7, vl); +} + +static inline void load_u16_4x4(const uint16_t *s, const ptrdiff_t p, + vuint16mf2_t *const s0, vuint16mf2_t *const s1, + vuint16mf2_t *const s2, vuint16mf2_t *const s3, + size_t vl) { + *s0 = __riscv_vle16_v_u16mf2(s, vl); + s += p; + *s1 = __riscv_vle16_v_u16mf2(s, vl); + s += p; + *s2 = __riscv_vle16_v_u16mf2(s, vl); + s += p; + *s3 = __riscv_vle16_v_u16mf2(s, vl); +} + +static inline void load_u16_8x4(const uint16_t *s, const ptrdiff_t p, + vuint16m1_t *const s0, vuint16m1_t *const s1, + vuint16m1_t *const s2, vuint16m1_t *const s3, + size_t vl) { + *s0 = __riscv_vle16_v_u16m1(s, vl); + s += p; + *s1 = __riscv_vle16_v_u16m1(s, vl); + s += p; + *s2 = __riscv_vle16_v_u16m1(s, vl); + s += p; + *s3 = __riscv_vle16_v_u16m1(s, vl); +} + +static inline void store_u8_8x4(uint8_t *s, const ptrdiff_t p, + const vuint8mf2_t s0, const vuint8mf2_t s1, + const vuint8mf2_t s2, const vuint8mf2_t s3, + size_t vl) { + __riscv_vse8_v_u8mf2(s, s0, vl); + s += p; + __riscv_vse8_v_u8mf2(s, s1, vl); + s += p; + __riscv_vse8_v_u8mf2(s, s2, vl); + s += p; + __riscv_vse8_v_u8mf2(s, s3, vl); +} + +static inline void load_s16_4x3(const int16_t *s, int p, vint16mf2_t *const s0, + vint16mf2_t *const s1, vint16mf2_t *const s2, + size_t vl) { + *s0 = __riscv_vle16_v_i16mf2(s, vl); + s += p; + *s1 = __riscv_vle16_v_i16mf2(s, vl); + s += p; + *s2 = __riscv_vle16_v_i16mf2(s, vl); +} + +static inline void load_s16_8x3(const int16_t *s, int p, vint16m1_t *const s0, + vint16m1_t *const s1, vint16m1_t *const s2, + size_t vl) { + *s0 = __riscv_vle16_v_i16m1(s, vl); + s += p; + *s1 = __riscv_vle16_v_i16m1(s, vl); + s += p; + *s2 = __riscv_vle16_v_i16m1(s, vl); +} + +static inline void load_s32_4x4(const int32_t *s, int p, vint32m1_t *const s0, + vint32m1_t *const s1, vint32m1_t *const s2, + vint32m1_t *const s3, size_t vl) { + *s0 = __riscv_vle32_v_i32m1(s, vl); + s += p; + *s1 = __riscv_vle32_v_i32m1(s, vl); + s += p; + *s2 = __riscv_vle32_v_i32m1(s, vl); + s += p; + *s3 = __riscv_vle32_v_i32m1(s, vl); +} + +static inline void store_s32_4x4(int32_t *s, int p, const vint32m1_t s0, + const vint32m1_t s1, const vint32m1_t s2, + const vint32m1_t s3, size_t vl) { + __riscv_vse32_v_i32m1(s, s0, vl); + s += p; + __riscv_vse32_v_i32m1(s, s1, vl); + s += p; + __riscv_vse32_v_i32m1(s, s2, vl); + s += p; + __riscv_vse32_v_i32m1(s, s3, vl); +} + +static inline void load_u32_4x4(const uint32_t *s, int p, vuint32m1_t *const s0, + vuint32m1_t *const s1, vuint32m1_t *const s2, + vuint32m1_t *const s3, size_t vl) { + *s0 = __riscv_vle32_v_u32m1(s, vl); + s += p; + *s1 = __riscv_vle32_v_u32m1(s, vl); + s += p; + *s2 = __riscv_vle32_v_u32m1(s, vl); + s += p; + *s3 = __riscv_vle32_v_u32m1(s, vl); +} + +static inline void store_u32_4x4(uint32_t *s, int p, const vuint32m1_t s0, + const vuint32m1_t s1, const vuint32m1_t s2, + const vuint32m1_t s3, size_t vl) { + __riscv_vse32_v_u32m1(s, s0, vl); + s += p; + __riscv_vse32_v_u32m1(s, s1, vl); + s += p; + __riscv_vse32_v_u32m1(s, s2, vl); + s += p; + __riscv_vse32_v_u32m1(s, s3, vl); +} + #endif // AOM_AOM_DSP_RISCV_MEM_RVV_H_
diff --git a/aom_dsp/simd/v256_intrinsics_c.h b/aom_dsp/simd/v256_intrinsics_c.h index 20d2709..cbcc1b0 100644 --- a/aom_dsp/simd/v256_intrinsics_c.h +++ b/aom_dsp/simd/v256_intrinsics_c.h
@@ -12,6 +12,7 @@ #ifndef AOM_AOM_DSP_SIMD_V256_INTRINSICS_C_H_ #define AOM_AOM_DSP_SIMD_V256_INTRINSICS_C_H_ +#include <stdbool.h> #include <stdio.h> #include <stdlib.h> #include <string.h> @@ -693,11 +694,13 @@ SIMD_INLINE c_v256 c_v256_wideshuffle_8(c_v256 a, c_v256 b, c_v256 pattern) { c_v256 t; int c; - for (c = 0; c < 32; c++) - t.u8[c] = (pattern.u8[c] < 32 - ? b.u8 - : a.u8)[CONFIG_BIG_ENDIAN ? 31 - (pattern.u8[c] & 31) - : pattern.u8[c] & 31]; + for (c = 0; c < 32; c++) { + const bool from_b_flag = pattern.u8[c] < 32; + const uint8_t *const src = from_b_flag ? b.u8 : a.u8; + const int idx = + CONFIG_BIG_ENDIAN ? 31 - (pattern.u8[c] & 31) : (pattern.u8[c] & 31); + t.u8[c] = src[idx]; + } return t; }
diff --git a/aom_dsp/x86/convolve_avx2.h b/aom_dsp/x86/convolve_avx2.h index 7db8a2d..ffafa13 100644 --- a/aom_dsp/x86/convolve_avx2.h +++ b/aom_dsp/x86/convolve_avx2.h
@@ -16,9 +16,16 @@ #include "aom_ports/mem.h" +#include "aom_dsp/x86/mem_sse2.h" +#include "aom_dsp/x86/synonyms.h" + #include "av1/common/convolve.h" #include "av1/common/filter.h" +#define SECOND_32_BLK (32) +#define THIRD_32_BLK (32 << 1) +#define FOURTH_32_BLK (SECOND_32_BLK + THIRD_32_BLK) + // filters for 16 DECLARE_ALIGNED(32, static const uint8_t, filt_global_avx2[]) = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 0, 1, 1, @@ -47,6 +54,26 @@ }; DECLARE_ALIGNED(32, static const uint8_t, + filt1_global_sse2[16]) = { 0, 1, 1, 2, 2, 3, 3, 4, + 8, 9, 9, 10, 10, 11, 11, 12 }; + +DECLARE_ALIGNED(32, static const uint8_t, + filt2_global_sse2[16]) = { 2, 3, 3, 4, 4, 5, 5, 6, + 10, 11, 11, 12, 12, 13, 13, 14 }; + +DECLARE_ALIGNED(32, static const uint8_t, + filt3_global_sse2[16]) = { 0, 1, 1, 2, 8, 9, 9, 10, + 0, 0, 0, 0, 0, 0, 0, 0 }; + +DECLARE_ALIGNED(32, static const uint8_t, + filt4_global_sse2[16]) = { 2, 3, 3, 4, 10, 11, 11, 12, + 0, 0, 0, 0, 0, 0, 0, 0 }; + +DECLARE_ALIGNED(32, static const uint8_t, + filt5_global_sse2[16]) = { 0, 1, 1, 2, 4, 5, 5, 6, + 0, 0, 0, 0, 0, 0, 0, 0 }; + +DECLARE_ALIGNED(32, static const uint8_t, filt1_global_avx2[32]) = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 }; @@ -66,105 +93,259 @@ 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14 }; -#define CONVOLVE_SR_HORIZONTAL_FILTER_4TAP \ - for (i = 0; i < (im_h - 2); i += 2) { \ - __m256i data = _mm256_castsi128_si256( \ - _mm_loadu_si128((__m128i *)&src_ptr[(i * src_stride) + j])); \ - data = _mm256_inserti128_si256( \ - data, \ - _mm_loadu_si128( \ - (__m128i *)&src_ptr[(i * src_stride) + j + src_stride]), \ - 1); \ - __m256i res = convolve_lowbd_x_4tap(data, coeffs_h + 1, filt); \ - res = \ - _mm256_sra_epi16(_mm256_add_epi16(res, round_const_h), round_shift_h); \ - _mm256_store_si256((__m256i *)&im_block[i * im_stride], res); \ - } \ - __m256i data_1 = _mm256_castsi128_si256( \ - _mm_loadu_si128((__m128i *)&src_ptr[(i * src_stride) + j])); \ - __m256i res = convolve_lowbd_x_4tap(data_1, coeffs_h + 1, filt); \ - res = _mm256_sra_epi16(_mm256_add_epi16(res, round_const_h), round_shift_h); \ +#define CONVOLVE_SR_HOR_FILTER_W4(CONVOLVE_LOWBD) \ + for (i = 0; i < (im_h - 2); i += 2) { \ + __m128i data = \ + load_8bit_8x2_to_1_reg_sse2(&src_ptr[(i * src_stride)], src_stride); \ + __m128i res = CONVOLVE_LOWBD(data, coeffs_h, filt); \ + res = _mm_srai_epi16(_mm_add_epi16(res, round_const_h), 2); \ + _mm_store_si128((__m128i *)&im_block[i * 4], res); \ + } \ + __m128i data_1 = _mm_loadl_epi64((__m128i *)&src_ptr[(i * src_stride)]); \ + __m128i res = CONVOLVE_LOWBD(data_1, coeffs_h, filt); \ + res = _mm_srai_epi16(_mm_add_epi16(res, round_const_h), 2); \ + _mm_storel_epi64((__m128i *)&im_block[i * 4], res); + +#define CONVOLVE_SR_HOR_FILTER_2TAP_W4 \ + CONVOLVE_SR_HOR_FILTER_W4(convolve_lowbd_x_2tap_ssse3) + +#define CONVOLVE_SR_HOR_FILTER_4TAP_W4 \ + CONVOLVE_SR_HOR_FILTER_W4(convolve_lowbd_x_4tap_ssse3) + +static inline void sr_2d_ver_round_and_store_w4(int w, __m256i res, + uint8_t *dst, int dst_stride, + __m256i round_const_v) { + const __m256i res_round = + _mm256_srai_epi32(_mm256_add_epi32(res, round_const_v), 11); + + const __m256i res_16bit = _mm256_packs_epi32(res_round, res_round); + const __m256i res_8b = _mm256_packus_epi16(res_16bit, res_16bit); + + const __m128i r0 = _mm256_castsi256_si128(res_8b); + const __m128i r1 = _mm256_extracti128_si256(res_8b, 1); + + __m128i *const p0 = (__m128i *)dst; + __m128i *const p1 = (__m128i *)(dst + dst_stride); + + if (w == 4) { + xx_storel_32(p0, r0); + xx_storel_32(p1, r1); + } else { + assert(w == 2); + *(uint16_t *)p0 = (uint16_t)_mm_cvtsi128_si32(r0); + *(uint16_t *)p1 = (uint16_t)_mm_cvtsi128_si32(r1); + } +} + +#define CONVOLVE_SR_VER_FILTER_2TAP_W4 \ + __m128i s[2]; \ + s[0] = _mm_loadl_epi64((__m128i *)(im_block + 0 * 4)); \ + \ + for (i = 0; i < h; i += 2) { \ + const int16_t *data = &im_block[i * 4]; \ + s[1] = _mm_loadl_epi64((__m128i *)(data + 1 * 4)); \ + const __m256i src_0 = _mm256_setr_m128i(s[0], s[1]); \ + s[0] = _mm_loadl_epi64((__m128i *)(data + 2 * 4)); \ + const __m256i src_1 = _mm256_setr_m128i(s[1], s[0]); \ + const __m256i ss = _mm256_unpacklo_epi16(src_0, src_1); \ + \ + const __m256i res = _mm256_madd_epi16(ss, coeffs_v[0]); \ + \ + sr_2d_ver_round_and_store_w4(w, res, dst_ptr, dst_stride, round_const_v); \ + dst_ptr += 2 * dst_stride; \ + } + +#define CONVOLVE_SR_VER_FILTER_4TAP_W4 \ + __m128i s[4]; \ + __m256i ss[2]; \ + s[0] = _mm_loadl_epi64((__m128i *)(im_block + 0 * 4)); \ + s[1] = _mm_loadl_epi64((__m128i *)(im_block + 1 * 4)); \ + s[2] = _mm_loadl_epi64((__m128i *)(im_block + 2 * 4)); \ + \ + const __m256i src_0 = _mm256_setr_m128i(s[0], s[1]); \ + const __m256i src_1 = _mm256_setr_m128i(s[1], s[2]); \ + \ + ss[0] = _mm256_unpacklo_epi16(src_0, src_1); \ + \ + for (i = 0; i < h; i += 2) { \ + const int16_t *data = &im_block[i * 4]; \ + s[3] = _mm_loadl_epi64((__m128i *)(data + 3 * 4)); \ + const __m256i src_2 = _mm256_setr_m128i(s[2], s[3]); \ + s[2] = _mm_loadl_epi64((__m128i *)(data + 4 * 4)); \ + const __m256i src_3 = _mm256_setr_m128i(s[3], s[2]); \ + ss[1] = _mm256_unpacklo_epi16(src_2, src_3); \ + \ + const __m256i res = convolve_4tap(ss, coeffs_v); \ + \ + sr_2d_ver_round_and_store_w4(w, res, dst_ptr, dst_stride, round_const_v); \ + dst_ptr += 2 * dst_stride; \ + \ + ss[0] = ss[1]; \ + } + +#define CONVOLVE_SR_VER_FILTER_6TAP_W4 \ + __m128i s[6]; \ + __m256i ss[3]; \ + s[0] = _mm_loadl_epi64((__m128i *)(im_block + 0 * 4)); \ + s[1] = _mm_loadl_epi64((__m128i *)(im_block + 1 * 4)); \ + s[2] = _mm_loadl_epi64((__m128i *)(im_block + 2 * 4)); \ + s[3] = _mm_loadl_epi64((__m128i *)(im_block + 3 * 4)); \ + s[4] = _mm_loadl_epi64((__m128i *)(im_block + 4 * 4)); \ + \ + const __m256i src_0 = _mm256_setr_m128i(s[0], s[1]); \ + const __m256i src_1 = _mm256_setr_m128i(s[1], s[2]); \ + const __m256i src_2 = _mm256_setr_m128i(s[2], s[3]); \ + const __m256i src_3 = _mm256_setr_m128i(s[3], s[4]); \ + \ + ss[0] = _mm256_unpacklo_epi16(src_0, src_1); \ + ss[1] = _mm256_unpacklo_epi16(src_2, src_3); \ + \ + for (i = 0; i < h; i += 2) { \ + const int16_t *data = &im_block[i * 4]; \ + s[5] = _mm_loadl_epi64((__m128i *)(data + 5 * 4)); \ + const __m256i src_4 = _mm256_setr_m128i(s[4], s[5]); \ + s[4] = _mm_loadl_epi64((__m128i *)(data + 6 * 4)); \ + const __m256i src_5 = _mm256_setr_m128i(s[5], s[4]); \ + ss[2] = _mm256_unpacklo_epi16(src_4, src_5); \ + \ + const __m256i res = convolve_6tap(ss, coeffs_v); \ + \ + sr_2d_ver_round_and_store_w4(w, res, dst_ptr, dst_stride, round_const_v); \ + dst_ptr += 2 * dst_stride; \ + \ + ss[0] = ss[1]; \ + ss[1] = ss[2]; \ + } + +#define CONVOLVE_SR_VER_FILTER_8TAP_W4 \ + __m128i s[8]; \ + __m256i ss[4]; \ + s[0] = _mm_loadl_epi64((__m128i *)(im_block + 0 * 4)); \ + s[1] = _mm_loadl_epi64((__m128i *)(im_block + 1 * 4)); \ + s[2] = _mm_loadl_epi64((__m128i *)(im_block + 2 * 4)); \ + s[3] = _mm_loadl_epi64((__m128i *)(im_block + 3 * 4)); \ + s[4] = _mm_loadl_epi64((__m128i *)(im_block + 4 * 4)); \ + s[5] = _mm_loadl_epi64((__m128i *)(im_block + 5 * 4)); \ + s[6] = _mm_loadl_epi64((__m128i *)(im_block + 6 * 4)); \ + \ + const __m256i src_0 = _mm256_setr_m128i(s[0], s[1]); \ + const __m256i src_1 = _mm256_setr_m128i(s[1], s[2]); \ + const __m256i src_2 = _mm256_setr_m128i(s[2], s[3]); \ + const __m256i src_3 = _mm256_setr_m128i(s[3], s[4]); \ + const __m256i src_4 = _mm256_setr_m128i(s[4], s[5]); \ + const __m256i src_5 = _mm256_setr_m128i(s[5], s[6]); \ + \ + ss[0] = _mm256_unpacklo_epi16(src_0, src_1); \ + ss[1] = _mm256_unpacklo_epi16(src_2, src_3); \ + ss[2] = _mm256_unpacklo_epi16(src_4, src_5); \ + \ + for (i = 0; i < h; i += 2) { \ + const int16_t *data = &im_block[i * 4]; \ + s[7] = _mm_loadl_epi64((__m128i *)(data + 7 * 4)); \ + const __m256i src_6 = _mm256_setr_m128i(s[6], s[7]); \ + s[6] = _mm_loadl_epi64((__m128i *)(data + 8 * 4)); \ + const __m256i src_7 = _mm256_setr_m128i(s[7], s[6]); \ + ss[3] = _mm256_unpacklo_epi16(src_6, src_7); \ + \ + const __m256i res = convolve(ss, coeffs_v); \ + \ + sr_2d_ver_round_and_store_w4(w, res, dst_ptr, dst_stride, round_const_v); \ + dst_ptr += 2 * dst_stride; \ + \ + ss[0] = ss[1]; \ + ss[1] = ss[2]; \ + ss[2] = ss[3]; \ + } + +#define CONVOLVE_SR_HORIZONTAL_FILTER(CONVOLVE_LOWBD) \ + for (i = 0; i < (im_h - 2); i += 2) { \ + __m256i data = _mm256_castsi128_si256( \ + _mm_loadu_si128((__m128i *)&src_ptr[(i * src_stride) + j])); \ + data = _mm256_inserti128_si256( \ + data, \ + _mm_loadu_si128( \ + (__m128i *)&src_ptr[(i * src_stride) + j + src_stride]), \ + 1); \ + __m256i res = CONVOLVE_LOWBD(data, coeffs_h, filt); \ + res = _mm256_srai_epi16(_mm256_add_epi16(res, round_const_h), 2); \ + _mm256_store_si256((__m256i *)&im_block[i * im_stride], res); \ + } \ + __m256i data_1 = _mm256_castsi128_si256( \ + _mm_loadu_si128((__m128i *)&src_ptr[(i * src_stride) + j])); \ + __m256i res = CONVOLVE_LOWBD(data_1, coeffs_h, filt); \ + res = _mm256_srai_epi16(_mm256_add_epi16(res, round_const_h), 2); \ _mm256_store_si256((__m256i *)&im_block[i * im_stride], res); +#define CONVOLVE_SR_HORIZONTAL_FILTER_2TAP \ + CONVOLVE_SR_HORIZONTAL_FILTER(convolve_lowbd_x_2tap) + +#define CONVOLVE_SR_HORIZONTAL_FILTER_4TAP \ + CONVOLVE_SR_HORIZONTAL_FILTER(convolve_lowbd_x_4tap) + +#define CONVOLVE_SR_HORIZONTAL_FILTER_6TAP \ + CONVOLVE_SR_HORIZONTAL_FILTER(convolve_lowbd_x_6tap) + +#define CONVOLVE_SR_HORIZONTAL_FILTER_8TAP \ + CONVOLVE_SR_HORIZONTAL_FILTER(convolve_lowbd_x) + +static inline void sr_2d_ver_round_and_store(__m256i res_a, __m256i res_b, + uint8_t *dst, int dst_stride, + __m256i round_const_v) { + const __m256i res_a_round = + _mm256_srai_epi32(_mm256_add_epi32(res_a, round_const_v), 11); + const __m256i res_b_round = + _mm256_srai_epi32(_mm256_add_epi32(res_b, round_const_v), 11); + const __m256i r16 = _mm256_packs_epi32(res_a_round, res_b_round); + const __m256i r8 = _mm256_packus_epi16(r16, r16); + + _mm_storel_epi64((__m128i *)dst, _mm256_castsi256_si128(r8)); + _mm_storel_epi64((__m128i *)(dst + dst_stride), + _mm256_extracti128_si256(r8, 1)); +} + +#define CONVOLVE_SR_VERTICAL_FILTER_2TAP \ + for (i = 0; i < h; i += 2) { \ + __m256i s[2]; \ + const int16_t *data = &im_block[i * im_stride]; \ + const __m256i s1 = _mm256_loadu_si256((__m256i *)(data + 0 * im_stride)); \ + const __m256i s2 = _mm256_loadu_si256((__m256i *)(data + 1 * im_stride)); \ + s[0] = _mm256_unpacklo_epi16(s1, s2); \ + s[1] = _mm256_unpackhi_epi16(s1, s2); \ + \ + __m256i res_a = _mm256_madd_epi16(s[0], coeffs_v[0]); \ + __m256i res_b = _mm256_madd_epi16(s[1], coeffs_v[0]); \ + \ + sr_2d_ver_round_and_store(res_a, res_b, dst_ptr, dst_stride, \ + round_const_v); \ + dst_ptr += 2 * dst_stride; \ + } + #define CONVOLVE_SR_VERTICAL_FILTER_4TAP \ __m256i s[6]; \ __m256i src_0 = _mm256_loadu_si256((__m256i *)(im_block + 0 * im_stride)); \ __m256i src_1 = _mm256_loadu_si256((__m256i *)(im_block + 1 * im_stride)); \ - __m256i src_2 = _mm256_loadu_si256((__m256i *)(im_block + 2 * im_stride)); \ - __m256i src_3 = _mm256_loadu_si256((__m256i *)(im_block + 3 * im_stride)); \ \ s[0] = _mm256_unpacklo_epi16(src_0, src_1); \ - s[1] = _mm256_unpacklo_epi16(src_2, src_3); \ - s[3] = _mm256_unpackhi_epi16(src_0, src_1); \ - s[4] = _mm256_unpackhi_epi16(src_2, src_3); \ + s[2] = _mm256_unpackhi_epi16(src_0, src_1); \ \ for (i = 0; i < h; i += 2) { \ const int16_t *data = &im_block[i * im_stride]; \ - const __m256i s4 = _mm256_loadu_si256((__m256i *)(data + 4 * im_stride)); \ - const __m256i s5 = _mm256_loadu_si256((__m256i *)(data + 5 * im_stride)); \ - s[2] = _mm256_unpacklo_epi16(s4, s5); \ - s[5] = _mm256_unpackhi_epi16(s4, s5); \ + const __m256i s4 = _mm256_loadu_si256((__m256i *)(data + 2 * im_stride)); \ + const __m256i s5 = _mm256_loadu_si256((__m256i *)(data + 3 * im_stride)); \ + s[1] = _mm256_unpacklo_epi16(s4, s5); \ + s[3] = _mm256_unpackhi_epi16(s4, s5); \ \ - __m256i res_a = convolve_4tap(s, coeffs_v + 1); \ - __m256i res_b = convolve_4tap(s + 3, coeffs_v + 1); \ + __m256i res_a = convolve_4tap(s, coeffs_v); \ + __m256i res_b = convolve_4tap(s + 2, coeffs_v); \ \ - res_a = \ - _mm256_sra_epi32(_mm256_add_epi32(res_a, sum_round_v), sum_shift_v); \ - res_b = \ - _mm256_sra_epi32(_mm256_add_epi32(res_b, sum_round_v), sum_shift_v); \ - const __m256i res_a_round = _mm256_sra_epi32( \ - _mm256_add_epi32(res_a, round_const_v), round_shift_v); \ - const __m256i res_b_round = _mm256_sra_epi32( \ - _mm256_add_epi32(res_b, round_const_v), round_shift_v); \ - const __m256i res_16bit = _mm256_packs_epi32(res_a_round, res_b_round); \ - const __m256i res_8b = _mm256_packus_epi16(res_16bit, res_16bit); \ - const __m128i res_0 = _mm256_castsi256_si128(res_8b); \ - const __m128i res_1 = _mm256_extracti128_si256(res_8b, 1); \ - \ - __m128i *const p_0 = (__m128i *)&dst[i * dst_stride + j]; \ - __m128i *const p_1 = (__m128i *)&dst[i * dst_stride + j + dst_stride]; \ - if (w - j > 4) { \ - _mm_storel_epi64(p_0, res_0); \ - _mm_storel_epi64(p_1, res_1); \ - } else if (w == 4) { \ - xx_storel_32(p_0, res_0); \ - xx_storel_32(p_1, res_1); \ - } else { \ - *(uint16_t *)p_0 = (uint16_t)_mm_cvtsi128_si32(res_0); \ - *(uint16_t *)p_1 = (uint16_t)_mm_cvtsi128_si32(res_1); \ - } \ + sr_2d_ver_round_and_store(res_a, res_b, dst_ptr, dst_stride, \ + round_const_v); \ + dst_ptr += 2 * dst_stride; \ \ s[0] = s[1]; \ - s[1] = s[2]; \ - s[3] = s[4]; \ - s[4] = s[5]; \ + s[2] = s[3]; \ } -#define CONVOLVE_SR_HORIZONTAL_FILTER_6TAP \ - for (i = 0; i < (im_h - 2); i += 2) { \ - __m256i data = _mm256_castsi128_si256( \ - _mm_loadu_si128((__m128i *)&src_ptr[(i * src_stride) + j])); \ - data = _mm256_inserti128_si256( \ - data, \ - _mm_loadu_si128( \ - (__m128i *)&src_ptr[(i * src_stride) + j + src_stride]), \ - 1); \ - \ - __m256i res = convolve_lowbd_x_6tap(data, coeffs_h, filt); \ - res = \ - _mm256_sra_epi16(_mm256_add_epi16(res, round_const_h), round_shift_h); \ - _mm256_store_si256((__m256i *)&im_block[i * im_stride], res); \ - } \ - \ - __m256i data_1 = _mm256_castsi128_si256( \ - _mm_loadu_si128((__m128i *)&src_ptr[(i * src_stride) + j])); \ - \ - __m256i res = convolve_lowbd_x_6tap(data_1, coeffs_h, filt); \ - \ - res = _mm256_sra_epi16(_mm256_add_epi16(res, round_const_h), round_shift_h); \ - \ - _mm256_store_si256((__m256i *)&im_block[i * im_stride], res); - #define CONVOLVE_SR_VERTICAL_FILTER_6TAP \ __m256i src_0 = _mm256_loadu_si256((__m256i *)(im_block + 0 * im_stride)); \ __m256i src_1 = _mm256_loadu_si256((__m256i *)(im_block + 1 * im_stride)); \ @@ -190,34 +371,9 @@ __m256i res_a = convolve_6tap(s, coeffs_v); \ __m256i res_b = convolve_6tap(s + 3, coeffs_v); \ \ - res_a = \ - _mm256_sra_epi32(_mm256_add_epi32(res_a, sum_round_v), sum_shift_v); \ - res_b = \ - _mm256_sra_epi32(_mm256_add_epi32(res_b, sum_round_v), sum_shift_v); \ - \ - const __m256i res_a_round = _mm256_sra_epi32( \ - _mm256_add_epi32(res_a, round_const_v), round_shift_v); \ - const __m256i res_b_round = _mm256_sra_epi32( \ - _mm256_add_epi32(res_b, round_const_v), round_shift_v); \ - \ - const __m256i res_16bit = _mm256_packs_epi32(res_a_round, res_b_round); \ - const __m256i res_8b = _mm256_packus_epi16(res_16bit, res_16bit); \ - \ - const __m128i res_0 = _mm256_castsi256_si128(res_8b); \ - const __m128i res_1 = _mm256_extracti128_si256(res_8b, 1); \ - \ - __m128i *const p_0 = (__m128i *)&dst[i * dst_stride + j]; \ - __m128i *const p_1 = (__m128i *)&dst[i * dst_stride + j + dst_stride]; \ - if (w - j > 4) { \ - _mm_storel_epi64(p_0, res_0); \ - _mm_storel_epi64(p_1, res_1); \ - } else if (w == 4) { \ - xx_storel_32(p_0, res_0); \ - xx_storel_32(p_1, res_1); \ - } else { \ - *(uint16_t *)p_0 = (uint16_t)_mm_cvtsi128_si32(res_0); \ - *(uint16_t *)p_1 = (uint16_t)_mm_cvtsi128_si32(res_1); \ - } \ + sr_2d_ver_round_and_store(res_a, res_b, dst_ptr, dst_stride, \ + round_const_v); \ + dst_ptr += 2 * dst_stride; \ \ s[0] = s[1]; \ s[1] = s[2]; \ @@ -226,31 +382,6 @@ s[4] = s[5]; \ } -#define CONVOLVE_SR_HORIZONTAL_FILTER_8TAP \ - for (i = 0; i < (im_h - 2); i += 2) { \ - __m256i data = _mm256_castsi128_si256( \ - _mm_loadu_si128((__m128i *)&src_ptr[(i * src_stride) + j])); \ - data = _mm256_inserti128_si256( \ - data, \ - _mm_loadu_si128( \ - (__m128i *)&src_ptr[(i * src_stride) + j + src_stride]), \ - 1); \ - \ - __m256i res = convolve_lowbd_x(data, coeffs_h, filt); \ - res = \ - _mm256_sra_epi16(_mm256_add_epi16(res, round_const_h), round_shift_h); \ - _mm256_store_si256((__m256i *)&im_block[i * im_stride], res); \ - } \ - \ - __m256i data_1 = _mm256_castsi128_si256( \ - _mm_loadu_si128((__m128i *)&src_ptr[(i * src_stride) + j])); \ - \ - __m256i res = convolve_lowbd_x(data_1, coeffs_h, filt); \ - \ - res = _mm256_sra_epi16(_mm256_add_epi16(res, round_const_h), round_shift_h); \ - \ - _mm256_store_si256((__m256i *)&im_block[i * im_stride], res); - #define CONVOLVE_SR_VERTICAL_FILTER_8TAP \ __m256i src_0 = _mm256_loadu_si256((__m256i *)(im_block + 0 * im_stride)); \ __m256i src_1 = _mm256_loadu_si256((__m256i *)(im_block + 1 * im_stride)); \ @@ -280,34 +411,9 @@ __m256i res_a = convolve(s, coeffs_v); \ __m256i res_b = convolve(s + 4, coeffs_v); \ \ - res_a = \ - _mm256_sra_epi32(_mm256_add_epi32(res_a, sum_round_v), sum_shift_v); \ - res_b = \ - _mm256_sra_epi32(_mm256_add_epi32(res_b, sum_round_v), sum_shift_v); \ - \ - const __m256i res_a_round = _mm256_sra_epi32( \ - _mm256_add_epi32(res_a, round_const_v), round_shift_v); \ - const __m256i res_b_round = _mm256_sra_epi32( \ - _mm256_add_epi32(res_b, round_const_v), round_shift_v); \ - \ - const __m256i res_16bit = _mm256_packs_epi32(res_a_round, res_b_round); \ - const __m256i res_8b = _mm256_packus_epi16(res_16bit, res_16bit); \ - \ - const __m128i res_0 = _mm256_castsi256_si128(res_8b); \ - const __m128i res_1 = _mm256_extracti128_si256(res_8b, 1); \ - \ - __m128i *const p_0 = (__m128i *)&dst[i * dst_stride + j]; \ - __m128i *const p_1 = (__m128i *)&dst[i * dst_stride + j + dst_stride]; \ - if (w - j > 4) { \ - _mm_storel_epi64(p_0, res_0); \ - _mm_storel_epi64(p_1, res_1); \ - } else if (w == 4) { \ - xx_storel_32(p_0, res_0); \ - xx_storel_32(p_1, res_1); \ - } else { \ - *(uint16_t *)p_0 = (uint16_t)_mm_cvtsi128_si32(res_0); \ - *(uint16_t *)p_1 = (uint16_t)_mm_cvtsi128_si32(res_1); \ - } \ + sr_2d_ver_round_and_store(res_a, res_b, dst_ptr, dst_stride, \ + round_const_v); \ + dst_ptr += 2 * dst_stride; \ \ s[0] = s[1]; \ s[1] = s[2]; \ @@ -607,6 +713,174 @@ } \ } while (0) +static inline void prepare_coeffs_2t_ssse3( + const InterpFilterParams *const filter_params, const int32_t subpel_q4, + __m128i *const coeffs /* [4] */) { + const int16_t *const filter = av1_get_interp_filter_subpel_kernel( + filter_params, subpel_q4 & SUBPEL_MASK); + const __m128i coeffs_8 = _mm_loadu_si128((__m128i *)filter); + + // right shift all filter co-efficients by 1 to reduce the bits required. + // This extra right shift will be taken care of at the end while rounding + // the result. + // Since all filter co-efficients are even, this change will not affect the + // end result + assert(_mm_test_all_zeros(_mm_and_si128(coeffs_8, _mm_set1_epi16(1)), + _mm_set1_epi16((short)0xffff))); + + const __m128i coeffs_1 = _mm_srai_epi16(coeffs_8, 1); + + // coeffs 3 4 3 4 3 4 3 4 + coeffs[0] = _mm_shuffle_epi8(coeffs_1, _mm_set1_epi16(0x0806u)); +} + +static inline void prepare_coeffs_4t_ssse3( + const InterpFilterParams *const filter_params, const int32_t subpel_q4, + __m128i *const coeffs /* [4] */) { + const int16_t *const filter = av1_get_interp_filter_subpel_kernel( + filter_params, subpel_q4 & SUBPEL_MASK); + const __m128i coeffs_8 = _mm_loadu_si128((__m128i *)filter); + + // right shift all filter co-efficients by 1 to reduce the bits required. + // This extra right shift will be taken care of at the end while rounding + // the result. + // Since all filter co-efficients are even, this change will not affect the + // end result + assert(_mm_test_all_zeros(_mm_and_si128(coeffs_8, _mm_set1_epi16(1)), + _mm_set1_epi16((short)0xffff))); + + const __m128i coeffs_1 = _mm_srai_epi16(coeffs_8, 1); + + // coeffs 2 3 2 3 2 3 2 3 + coeffs[0] = _mm_shuffle_epi8(coeffs_1, _mm_set1_epi16(0x0604u)); + // coeffs 4 5 4 5 4 5 4 5 + coeffs[1] = _mm_shuffle_epi8(coeffs_1, _mm_set1_epi16(0x0a08u)); +} + +static inline void prepare_coeffs_6t_ssse3( + const InterpFilterParams *const filter_params, const int32_t subpel_q4, + __m128i *const coeffs /* [4] */) { + const int16_t *const filter = av1_get_interp_filter_subpel_kernel( + filter_params, subpel_q4 & SUBPEL_MASK); + const __m128i coeffs_8 = _mm_loadu_si128((__m128i *)filter); + + // right shift all filter co-efficients by 1 to reduce the bits required. + // This extra right shift will be taken care of at the end while rounding + // the result. + // Since all filter co-efficients are even, this change will not affect the + // end result + assert(_mm_test_all_zeros(_mm_and_si128(coeffs_8, _mm_set1_epi16(1)), + _mm_set1_epi16((short)0xffff))); + + const __m128i coeffs_1 = _mm_srai_epi16(coeffs_8, 1); + + // coeffs 2 3 2 3 2 3 2 3 + coeffs[0] = _mm_shuffle_epi8(coeffs_1, _mm_set1_epi16(0x0402u)); + // coeffs 4 5 4 5 4 5 4 5 + coeffs[1] = _mm_shuffle_epi8(coeffs_1, _mm_set1_epi16(0x0806u)); + // coeffs 5 6 5 6 5 6 5 6 + coeffs[2] = _mm_shuffle_epi8(coeffs_1, _mm_set1_epi16(0x0c0au)); +} + +static inline void prepare_coeffs_ssse3( + const InterpFilterParams *const filter_params, const int32_t subpel_q4, + __m128i *const coeffs /* [4] */) { + const int16_t *const filter = av1_get_interp_filter_subpel_kernel( + filter_params, subpel_q4 & SUBPEL_MASK); + const __m128i coeffs_8 = _mm_loadu_si128((__m128i *)filter); + + // right shift all filter co-efficients by 1 to reduce the bits required. + // This extra right shift will be taken care of at the end while rounding + // the result. + // Since all filter co-efficients are even, this change will not affect the + // end result + assert(_mm_test_all_zeros(_mm_and_si128(coeffs_8, _mm_set1_epi16(1)), + _mm_set1_epi16((short)0xffff))); + + const __m128i coeffs_1 = _mm_srai_epi16(coeffs_8, 1); + + // coeffs 0 1 0 1 0 1 0 1 + coeffs[0] = _mm_shuffle_epi8(coeffs_1, _mm_set1_epi16(0x0200u)); + // coeffs 2 3 2 3 2 3 2 3 + coeffs[1] = _mm_shuffle_epi8(coeffs_1, _mm_set1_epi16(0x0604u)); + // coeffs 4 5 4 5 4 5 4 5 + coeffs[2] = _mm_shuffle_epi8(coeffs_1, _mm_set1_epi16(0x0a08u)); + // coeffs 6 7 6 7 6 7 6 7 + coeffs[3] = _mm_shuffle_epi8(coeffs_1, _mm_set1_epi16(0x0e0cu)); +} + +static inline void prepare_coeffs_2t_lowbd( + const InterpFilterParams *const filter_params, const int subpel_q4, + __m256i *const coeffs /* [4] */) { + const int16_t *const filter = av1_get_interp_filter_subpel_kernel( + filter_params, subpel_q4 & SUBPEL_MASK); + const __m128i coeffs_8 = _mm_loadu_si128((__m128i *)filter); + const __m256i filter_coeffs = _mm256_broadcastsi128_si256(coeffs_8); + + // right shift all filter co-efficients by 1 to reduce the bits required. + // This extra right shift will be taken care of at the end while rounding + // the result. + // Since all filter co-efficients are even, this change will not affect the + // end result + assert(_mm_test_all_zeros(_mm_and_si128(coeffs_8, _mm_set1_epi16(1)), + _mm_set1_epi16((int16_t)0xffff))); + + const __m256i coeffs_1 = _mm256_srai_epi16(filter_coeffs, 1); + + // coeffs 3 4 3 4 3 4 3 4 + coeffs[0] = _mm256_shuffle_epi8(coeffs_1, _mm256_set1_epi16(0x0806u)); +} + +static inline void prepare_coeffs_4t_lowbd( + const InterpFilterParams *const filter_params, const int subpel_q4, + __m256i *const coeffs /* [4] */) { + const int16_t *const filter = av1_get_interp_filter_subpel_kernel( + filter_params, subpel_q4 & SUBPEL_MASK); + const __m128i coeffs_8 = _mm_loadu_si128((__m128i *)filter); + const __m256i filter_coeffs = _mm256_broadcastsi128_si256(coeffs_8); + + // right shift all filter co-efficients by 1 to reduce the bits required. + // This extra right shift will be taken care of at the end while rounding + // the result. + // Since all filter co-efficients are even, this change will not affect the + // end result + assert(_mm_test_all_zeros(_mm_and_si128(coeffs_8, _mm_set1_epi16(1)), + _mm_set1_epi16((short)0xffff))); + + const __m256i coeffs_1 = _mm256_srai_epi16(filter_coeffs, 1); + + // coeffs 2 3 2 3 2 3 2 3 + coeffs[0] = _mm256_shuffle_epi8(coeffs_1, _mm256_set1_epi16(0x0604u)); + // coeffs 4 5 4 5 4 5 4 5 + coeffs[1] = _mm256_shuffle_epi8(coeffs_1, _mm256_set1_epi16(0x0a08u)); +} + +static inline void prepare_coeffs_6t_lowbd( + const InterpFilterParams *const filter_params, const int subpel_q4, + __m256i *const coeffs /* [4] */) { + const int16_t *const filter = av1_get_interp_filter_subpel_kernel( + filter_params, subpel_q4 & SUBPEL_MASK); + const __m128i coeffs_8 = _mm_loadu_si128((__m128i *)filter); + const __m256i filter_coeffs = _mm256_broadcastsi128_si256(coeffs_8); + + // right shift all filter co-efficients by 1 to reduce the bits required. + // This extra right shift will be taken care of at the end while rounding + // the result. + // Since all filter co-efficients are even, this change will not affect the + // end result + assert(_mm_test_all_zeros(_mm_and_si128(coeffs_8, _mm_set1_epi16(1)), + _mm_set1_epi16((int16_t)0xffff))); + + const __m256i coeffs_1 = _mm256_srai_epi16(filter_coeffs, 1); + + // coeffs 1 2 1 2 1 2 1 2 + coeffs[0] = _mm256_shuffle_epi8(coeffs_1, _mm256_set1_epi16(0x0402u)); + // coeffs 3 4 3 4 3 4 3 4 + coeffs[1] = _mm256_shuffle_epi8(coeffs_1, _mm256_set1_epi16(0x0806u)); + // coeffs 5 6 5 6 5 6 5 6 + coeffs[2] = _mm256_shuffle_epi8(coeffs_1, _mm256_set1_epi16(0x0c0au)); +} + static inline void prepare_coeffs_lowbd( const InterpFilterParams *const filter_params, const int subpel_q4, __m256i *const coeffs /* [4] */) { @@ -635,30 +909,31 @@ coeffs[3] = _mm256_shuffle_epi8(coeffs_1, _mm256_set1_epi16(0x0e0cu)); } -static inline void prepare_coeffs_6t_lowbd( +static inline void prepare_coeffs_2t( const InterpFilterParams *const filter_params, const int subpel_q4, __m256i *const coeffs /* [4] */) { - const int16_t *const filter = av1_get_interp_filter_subpel_kernel( + const int16_t *filter = av1_get_interp_filter_subpel_kernel( filter_params, subpel_q4 & SUBPEL_MASK); - const __m128i coeffs_8 = _mm_loadu_si128((__m128i *)filter); - const __m256i filter_coeffs = _mm256_broadcastsi128_si256(coeffs_8); - // right shift all filter co-efficients by 1 to reduce the bits required. - // This extra right shift will be taken care of at the end while rounding - // the result. - // Since all filter co-efficients are even, this change will not affect the - // end result - assert(_mm_test_all_zeros(_mm_and_si128(coeffs_8, _mm_set1_epi16(1)), - _mm_set1_epi16((int16_t)0xffff))); + const __m128i coeff_8 = _mm_loadu_si128((__m128i *)(filter + 1)); + const __m256i coeff = _mm256_broadcastsi128_si256(coeff_8); - const __m256i coeffs_1 = _mm256_srai_epi16(filter_coeffs, 1); - - // coeffs 1 2 1 2 1 2 1 2 - coeffs[0] = _mm256_shuffle_epi8(coeffs_1, _mm256_set1_epi16(0x0402u)); // coeffs 3 4 3 4 3 4 3 4 - coeffs[1] = _mm256_shuffle_epi8(coeffs_1, _mm256_set1_epi16(0x0806u)); - // coeffs 5 6 5 6 5 6 5 6 - coeffs[2] = _mm256_shuffle_epi8(coeffs_1, _mm256_set1_epi16(0x0c0au)); + coeffs[0] = _mm256_shuffle_epi32(coeff, 0x55); +} + +static inline void prepare_coeffs_4t( + const InterpFilterParams *const filter_params, const int subpel_q4, + __m256i *const coeffs /* [4] */) { + const int16_t *filter = av1_get_interp_filter_subpel_kernel( + filter_params, subpel_q4 & SUBPEL_MASK); + + const __m128i coeff_8 = _mm_loadu_si128((__m128i *)filter); + const __m256i coeff = _mm256_broadcastsi128_si256(coeff_8); + // coeffs 2 3 2 3 2 3 2 3 + coeffs[0] = _mm256_shuffle_epi32(coeff, 0x55); + // coeffs 4 5 4 5 4 5 4 5 + coeffs[1] = _mm256_shuffle_epi32(coeff, 0xaa); } static inline void prepare_coeffs_6t( @@ -721,6 +996,38 @@ coeffs[5] = _mm256_shuffle_epi32(coeff, 0x55); // coeffs 10 11 10 11.. 10 11 } +static inline __m128i convolve_lowbd_4tap_ssse3(const __m128i ss[2], + const __m128i coeffs[2]) { + const __m128i res_01 = _mm_maddubs_epi16(ss[0], coeffs[0]); + const __m128i res_23 = _mm_maddubs_epi16(ss[1], coeffs[1]); + + return _mm_add_epi16(res_01, res_23); +} + +static inline __m128i convolve_lowbd_6tap_ssse3(const __m128i ss[3], + const __m128i coeffs[3]) { + const __m128i res_01 = _mm_maddubs_epi16(ss[0], coeffs[0]); + const __m128i res_23 = _mm_maddubs_epi16(ss[1], coeffs[1]); + const __m128i res_45 = _mm_maddubs_epi16(ss[2], coeffs[2]); + + const __m128i res = _mm_add_epi16(_mm_add_epi16(res_01, res_45), res_23); + + return res; +} + +static inline __m128i convolve_lowbd_ssse3(const __m128i ss[4], + const __m128i coeffs[4]) { + const __m128i res_01 = _mm_maddubs_epi16(ss[0], coeffs[0]); + const __m128i res_23 = _mm_maddubs_epi16(ss[1], coeffs[1]); + const __m128i res_45 = _mm_maddubs_epi16(ss[2], coeffs[2]); + const __m128i res_67 = _mm_maddubs_epi16(ss[3], coeffs[3]); + + const __m128i res = _mm_add_epi16(_mm_add_epi16(res_01, res_45), + _mm_add_epi16(res_23, res_67)); + + return res; +} + static inline __m256i convolve_lowbd(const __m256i *const s, const __m256i *const coeffs) { const __m256i res_01 = _mm256_maddubs_epi16(s[0], coeffs[0]); @@ -808,6 +1115,26 @@ return res; } +static inline __m128i convolve_lowbd_x_2tap_ssse3(const __m128i data, + const __m128i *const coeffs, + const __m128i *const filt) { + __m128i s; + s = _mm_shuffle_epi8(data, filt[0]); + + return _mm_maddubs_epi16(s, coeffs[0]); +} + +static inline __m128i convolve_lowbd_x_4tap_ssse3(const __m128i data, + const __m128i *const coeffs, + const __m128i *const filt) { + __m128i s[2]; + + s[0] = _mm_shuffle_epi8(data, filt[0]); + s[1] = _mm_shuffle_epi8(data, filt[1]); + + return convolve_lowbd_4tap_ssse3(s, coeffs); +} + static inline __m256i convolve_lowbd_x(const __m256i data, const __m256i *const coeffs, const __m256i *const filt) { @@ -844,6 +1171,15 @@ return convolve_lowbd_4tap(s, coeffs); } +static inline __m256i convolve_lowbd_x_2tap(const __m256i data, + const __m256i *const coeffs, + const __m256i *const filt) { + __m256i s; + s = _mm256_shuffle_epi8(data, filt[0]); + + return _mm256_maddubs_epi16(s, coeffs[0]); +} + static inline void add_store_aligned_256(CONV_BUF_TYPE *const dst, const __m256i *const res, const int do_average) { @@ -919,4 +1255,628 @@ return res_round; } +static inline __m256i round_sr_x_avx2(const __m256i data) { + // we can perform the below steps: + // data = (data + 2) >> 2 + // data = (data + 8) >> 4, + // in the below form as well + // data = (data + 0x22) >> 6 + const __m256i value = _mm256_set1_epi16(34); + const __m256i reg = _mm256_add_epi16(data, value); + return _mm256_srai_epi16(reg, 6); +} + +static inline __m128i convolve_x_4tap_4x2_ssse3(const uint8_t *const src, + const ptrdiff_t src_stride, + __m128i *const coeffs) { + __m128i data[2]; + const __m128i f_l0 = _mm_load_si128((__m128i const *)filt1_global_sse2); + const __m128i f_l1 = _mm_load_si128((__m128i const *)filt2_global_sse2); + const __m128i src_1 = + load_8bit_8x2_to_1_reg_sse2(src, (int)(sizeof(*src) * src_stride)); + + data[0] = _mm_shuffle_epi8(src_1, f_l0); + data[1] = _mm_shuffle_epi8(src_1, f_l1); + return convolve_lowbd_4tap_ssse3(data, coeffs); +} + +static inline __m128i round_sr_x_ssse3(const __m128i data) { + const __m128i val = _mm_set1_epi16(34); + const __m128i reg = _mm_add_epi16(data, val); + return _mm_srai_epi16(reg, 6); +} + +static inline void store_8bit_4x2_sse2(const __m128i reg, uint8_t *const dst, + const ptrdiff_t dst_stride) { + xx_storel_32(dst, reg); + *(uint32_t *)(dst + dst_stride) = + ((uint32_t)_mm_extract_epi16(reg, 3) << 16) | _mm_extract_epi16(reg, 2); +} + +static inline void pack_store_u8_4x2_sse2(const __m128i reg, uint8_t *const dst, + const ptrdiff_t dst_stride) { + const __m128i reg_pack = _mm_packus_epi16(reg, reg); + store_8bit_4x2_sse2(reg_pack, dst, dst_stride); +} + +static inline __m128i convolve_x_4tap_2x2_ssse3(const uint8_t *const src, + const ptrdiff_t src_stride, + __m128i *const coeffs) { + __m128i data[2]; + const __m128i f_0 = _mm_load_si128((__m128i const *)filt3_global_sse2); + const __m128i f_1 = _mm_load_si128((__m128i const *)filt4_global_sse2); + const __m128i reg = + load_8bit_8x2_to_1_reg_sse2(src, (int)(sizeof(*src) * src_stride)); + + data[0] = _mm_shuffle_epi8(reg, f_0); + data[1] = _mm_shuffle_epi8(reg, f_1); + return convolve_lowbd_4tap_ssse3(data, coeffs); +} + +static inline void pack_store_u8_2x2_sse2(const __m128i reg, uint8_t *const dst, + const ptrdiff_t dst_stride) { + const __m128i data = _mm_packus_epi16(reg, reg); + *(int16_t *)dst = (int16_t)_mm_cvtsi128_si32(data); + *(int16_t *)(dst + dst_stride) = (int16_t)_mm_extract_epi16(data, 1); +} + +static inline __m128i convolve_x_2tap_ssse3(const __m128i *data, + const __m128i *coeff) { + return _mm_maddubs_epi16(data[0], coeff[0]); +} + +static inline __m128i load8_x_4x2_sse4(const void *const src, + const ptrdiff_t offset) { + const __m128i s = _mm_cvtsi32_si128(loadu_int32(src)); + return _mm_insert_epi32(s, loadu_int32((uint8_t *)src + offset), 1); +} + +static inline __m128i load_x_u8_4x2_sse4(const uint8_t *const src, + const ptrdiff_t stride) { + return load8_x_4x2_sse4(src, sizeof(*src) * stride); +} + +static inline __m128i convolve_x_2tap_2x2_ssse3(const uint8_t *const src, + const ptrdiff_t stride, + const __m128i *coeffs) { + const __m128i flt = _mm_load_si128((__m128i const *)filt5_global_sse2); + const __m128i reg = load_x_u8_4x2_sse4(src, stride); + const __m128i data = _mm_shuffle_epi8(reg, flt); + return convolve_x_2tap_ssse3(&data, coeffs); +} + +static inline __m128i convolve_x_2tap_4x2_ssse3(const uint8_t *const src, + const ptrdiff_t stride, + const __m128i *coeffs) { + const __m128i flt = _mm_load_si128((__m128i const *)filt1_global_sse2); + const __m128i data = + load_8bit_8x2_to_1_reg_sse2(src, (int)(sizeof(*src) * stride)); + const __m128i res = _mm_shuffle_epi8(data, flt); + return convolve_x_2tap_ssse3(&res, coeffs); +} + +static inline void convolve_x_2tap_8x2_ssse3(const uint8_t *const src, + const ptrdiff_t stride, + const __m128i *coeffs, + __m128i *data) { + __m128i res[2]; + const __m128i reg_00 = _mm_loadu_si128((__m128i *)src); + const __m128i reg_10 = _mm_loadu_si128((__m128i *)(src + stride)); + const __m128i reg_01 = _mm_srli_si128(reg_00, 1); + const __m128i reg_11 = _mm_srli_si128(reg_10, 1); + res[0] = _mm_unpacklo_epi8(reg_00, reg_01); + res[1] = _mm_unpacklo_epi8(reg_10, reg_11); + + data[0] = convolve_x_2tap_ssse3(&res[0], coeffs); + data[1] = convolve_x_2tap_ssse3(&res[1], coeffs); +} + +static inline __m256i loadu_x_8bit_16x2_avx2(const void *const src, + const ptrdiff_t offset) { + const __m128i reg0 = _mm_loadu_si128((__m128i *)src); + const __m128i reg1 = _mm_loadu_si128((__m128i *)((uint8_t *)src + offset)); + return _mm256_setr_m128i(reg0, reg1); +} + +static inline __m256i convolve_x_2tap_avx2(const __m256i *data, + const __m256i *coeffs) { + return _mm256_maddubs_epi16(data[0], coeffs[0]); +} + +static inline void convolve_x_2tap_16x2_avx2(const uint8_t *const src, + const ptrdiff_t stride, + const __m256i *coeffs, + __m256i *data) { + const __m256i reg0 = loadu_x_8bit_16x2_avx2(src, stride); + const __m256i reg1 = loadu_x_8bit_16x2_avx2(src + 1, stride); + const __m256i res0 = _mm256_unpacklo_epi8(reg0, reg1); + const __m256i res1 = _mm256_unpackhi_epi8(reg0, reg1); + data[0] = convolve_x_2tap_avx2(&res0, coeffs); + data[1] = convolve_x_2tap_avx2(&res1, coeffs); +} + +static inline void store_u8_16x2_avx2(const __m256i src, uint8_t *const dst, + const ptrdiff_t stride) { + const __m128i reg0 = _mm256_castsi256_si128(src); + const __m128i reg1 = _mm256_extracti128_si256(src, 1); + _mm_storeu_si128((__m128i *)dst, reg0); + _mm_storeu_si128((__m128i *)((uint8_t *)dst + stride), reg1); +} + +static inline void store_u8_8x2_avx2(const __m256i src, uint8_t *const dst, + const ptrdiff_t stride) { + const __m128i reg0 = _mm256_castsi256_si128(src); + const __m128i reg1 = _mm256_extracti128_si256(src, 1); + _mm_storel_epi64((__m128i *)dst, reg0); + _mm_storel_epi64((__m128i *)(dst + stride), reg1); +} + +static inline void pack_store_16x2_avx2(const __m256i data0, + const __m256i data1, uint8_t *const dst, + const ptrdiff_t stride) { + const __m256i res = _mm256_packus_epi16(data0, data1); + store_u8_16x2_avx2(res, dst, stride); +} + +static inline void pack_store_8x2_avx2(const __m256i data, uint8_t *const dst, + const ptrdiff_t stride) { + const __m256i res = _mm256_packus_epi16(data, data); + store_u8_8x2_avx2(res, dst, stride); +} + +static inline void round_pack_store_16x2_avx2(const __m256i *data, + uint8_t *const dst, + const ptrdiff_t dst_stride) { + __m256i reg[2]; + + reg[0] = round_sr_x_avx2(data[0]); + reg[1] = round_sr_x_avx2(data[1]); + pack_store_16x2_avx2(reg[0], reg[1], dst, dst_stride); +} + +static inline void convolve_x_2tap_32_avx2(const uint8_t *const src, + const __m256i *coeffs, + __m256i *data) { + const __m256i res0 = _mm256_loadu_si256((__m256i *)src); + const __m256i res1 = _mm256_loadu_si256((__m256i *)(src + 1)); + const __m256i reg0 = _mm256_unpacklo_epi8(res0, res1); + const __m256i reg1 = _mm256_unpackhi_epi8(res0, res1); + + data[0] = convolve_x_2tap_avx2(®0, coeffs); + data[1] = convolve_x_2tap_avx2(®1, coeffs); +} + +static inline void pack_store_32_avx2(const __m256i data0, const __m256i data1, + uint8_t *const dst) { + const __m256i reg = _mm256_packus_epi16(data0, data1); + _mm256_storeu_si256((__m256i *)dst, reg); +} + +static inline void round_pack_store_32_avx2(const __m256i *data, + uint8_t *const dst) { + __m256i reg[2]; + + reg[0] = round_sr_x_avx2(data[0]); + reg[1] = round_sr_x_avx2(data[1]); + pack_store_32_avx2(reg[0], reg[1], dst); +} + +static inline void convolve_round_2tap_32_avx2(const uint8_t *const src, + const __m256i *coeffs, + uint8_t *const dst) { + __m256i data[2]; + + convolve_x_2tap_32_avx2(src, coeffs, data); + round_pack_store_32_avx2(data, dst); +} + +static inline void load_avg_store_2tap_32_avx2(const uint8_t *const src, + uint8_t *const dst) { + const __m256i res0 = _mm256_loadu_si256((__m256i *)src); + const __m256i res1 = _mm256_loadu_si256((__m256i *)(src + 1)); + const __m256i data = _mm256_avg_epu8(res0, res1); + _mm256_storeu_si256((__m256i *)dst, data); +} + +static inline __m256i load_convolve_8tap_8x2_avx2(const uint8_t *const src, + const ptrdiff_t stride, + const __m256i *coeffs, + const __m256i *flt) { + const __m256i res = loadu_x_8bit_16x2_avx2(src, stride); + return convolve_lowbd_x(res, coeffs, flt); +} + +static inline void load_convolve_8tap_16x2_avx2(const uint8_t *const src, + const int32_t src_stride, + const __m256i *coeffs, + const __m256i *flt, + __m256i *reg) { + reg[0] = load_convolve_8tap_8x2_avx2(src + 0, src_stride, coeffs, flt); + reg[1] = load_convolve_8tap_8x2_avx2(src + 8, src_stride, coeffs, flt); +} + +static inline void load_convolve_8tap_32_avx2(const uint8_t *const src, + const __m256i *coeffs, + const __m256i *filt, + __m256i *data) { + const __m256i reg_0 = _mm256_loadu_si256((__m256i *)src); + const __m256i reg_8 = _mm256_loadu_si256((__m256i *)(src + 8)); + + data[0] = convolve_lowbd_x(reg_0, coeffs, filt); + data[1] = convolve_lowbd_x(reg_8, coeffs, filt); +} + +static inline void load_convolve_round_8tap_32_avx2(const uint8_t *const src, + const __m256i *coeffs, + const __m256i *filt, + uint8_t *const dst) { + __m256i data[2]; + + load_convolve_8tap_32_avx2(src, coeffs, filt, data); + round_pack_store_32_avx2(data, dst); +} + +static inline void load_convolve_6tap_32_avx2(const uint8_t *const src, + const __m256i *coeffs, + const __m256i *filt, + __m256i *data) { + const __m256i reg0 = _mm256_loadu_si256((__m256i *)src); + const __m256i reg1 = _mm256_loadu_si256((__m256i *)(src + 8)); + + data[0] = convolve_lowbd_x_6tap(reg0, coeffs, filt); + data[1] = convolve_lowbd_x_6tap(reg1, coeffs, filt); +} + +static inline void convolve_sr_store_6tap_32_avx2(const uint8_t *const src, + const __m256i *coeffs, + const __m256i *filt, + uint8_t *const dst) { + __m256i data[2]; + + load_convolve_6tap_32_avx2(src, coeffs, filt, data); + round_pack_store_32_avx2(data, dst); +} + +static inline __m256i load_convolve_6tap_8x2_avx2(const uint8_t *const src, + const ptrdiff_t stride, + const __m256i *coeffs, + const __m256i *filt) { + const __m256i data = loadu_x_8bit_16x2_avx2(src, stride); + return convolve_lowbd_x_6tap(data, coeffs, filt); +} + +static inline void load_convolve_6tap_16x2_avx2(const uint8_t *const src, + const int32_t src_stride, + const __m256i *coeffs, + const __m256i *filt, + __m256i *data) { + data[0] = load_convolve_6tap_8x2_avx2(src + 0, src_stride, coeffs, filt); + data[1] = load_convolve_6tap_8x2_avx2(src + 8, src_stride, coeffs, filt); +} + +static inline __m128i round_sr_y_ssse3(const __m128i data) { + const __m128i value = _mm_set1_epi16(32); + const __m128i reg = _mm_add_epi16(data, value); + return _mm_srai_epi16(reg, FILTER_BITS - 1); +} + +static inline __m256i round_sr_y_avx2(const __m256i data) { + const __m256i value = _mm256_set1_epi16(32); + const __m256i reg = _mm256_add_epi16(data, value); + return _mm256_srai_epi16(reg, FILTER_BITS - 1); +} + +static inline void round_pack_store_y_8x2_avx2(const __m256i res, + uint8_t *const dst, + const ptrdiff_t dst_stride) { + __m256i r; + + r = round_sr_y_avx2(res); + pack_store_8x2_avx2(r, dst, dst_stride); +} + +static inline void round_pack_store_y_16x2_avx2(const __m256i res[2], + uint8_t *const dst, + const ptrdiff_t dst_stride) { + __m256i r[2]; + + r[0] = round_sr_y_avx2(res[0]); + r[1] = round_sr_y_avx2(res[1]); + pack_store_16x2_avx2(r[0], r[1], dst, dst_stride); +} + +static inline void round_pack_store_y_32_avx2(const __m256i res[2], + uint8_t *const dst) { + __m256i r[2]; + + r[0] = round_sr_y_avx2(res[0]); + r[1] = round_sr_y_avx2(res[1]); + pack_store_32_avx2(r[0], r[1], dst); +} + +static inline void round_pack_store_y_32x2_avx2(const __m256i res[4], + uint8_t *const dst, + const ptrdiff_t dst_stride) { + round_pack_store_y_32_avx2(res, dst); + round_pack_store_y_32_avx2(res + 2, dst + dst_stride); +} + +static inline void convolve_y_2tap_2x2_ssse3(const uint8_t *const data, + const ptrdiff_t stride, + const __m128i *coeffs, + __m128i d[2], __m128i *res) { + d[1] = _mm_cvtsi32_si128(loadu_int16(data + 1 * stride)); + const __m128i src_01a = _mm_unpacklo_epi16(d[0], d[1]); + d[0] = _mm_cvtsi32_si128(loadu_int16(data + 2 * stride)); + const __m128i src_12a = _mm_unpacklo_epi16(d[1], d[0]); + + const __m128i s = _mm_unpacklo_epi8(src_01a, src_12a); + + *res = _mm_maddubs_epi16(s, coeffs[0]); +} + +static inline void convolve_y_4tap_2x2_ssse3(const uint8_t *const data, + const ptrdiff_t stride, + const __m128i coeffs[2], + __m128i d[4], __m128i s[2], + __m128i *res) { + d[3] = _mm_cvtsi32_si128(loadu_int16(data + 3 * stride)); + const __m128i src_23a = _mm_unpacklo_epi16(d[2], d[3]); + d[2] = _mm_cvtsi32_si128(loadu_int16(data + 4 * stride)); + const __m128i src_34a = _mm_unpacklo_epi16(d[3], d[2]); + + s[1] = _mm_unpacklo_epi8(src_23a, src_34a); + + *res = convolve_lowbd_4tap_ssse3(s, coeffs); +} + +static inline void convolve_y_6tap_2x2_ssse3(const uint8_t *const data, + const ptrdiff_t stride, + const __m128i coeffs[3], + __m128i d[6], __m128i s[3], + __m128i *res) { + d[5] = _mm_cvtsi32_si128(loadu_int16(data + 5 * stride)); + const __m128i src_45a = _mm_unpacklo_epi16(d[4], d[5]); + d[4] = _mm_cvtsi32_si128(loadu_int16(data + 6 * stride)); + const __m128i src_56a = _mm_unpacklo_epi16(d[5], d[4]); + + s[2] = _mm_unpacklo_epi8(src_45a, src_56a); + + *res = convolve_lowbd_6tap_ssse3(s, coeffs); +} + +static inline void convolve_y_8tap_2x2_ssse3(const uint8_t *const data, + const ptrdiff_t stride, + const __m128i coeffs[4], + __m128i d[8], __m128i s[4], + __m128i *res) { + d[7] = _mm_cvtsi32_si128(loadu_int16(data + 7 * stride)); + const __m128i src_67a = _mm_unpacklo_epi16(d[6], d[7]); + d[6] = _mm_cvtsi32_si128(loadu_int16(data + 8 * stride)); + const __m128i src_78a = _mm_unpacklo_epi16(d[7], d[6]); + + s[3] = _mm_unpacklo_epi8(src_67a, src_78a); + + *res = convolve_lowbd_ssse3(s, coeffs); +} + +static inline void convolve_y_2tap_4x2_ssse3(const uint8_t *const data, + const ptrdiff_t stride, + const __m128i *coeffs, + __m128i d[2], __m128i *res) { + d[1] = _mm_cvtsi32_si128(loadu_int32(data + 1 * stride)); + const __m128i src_01a = _mm_unpacklo_epi32(d[0], d[1]); + d[0] = _mm_cvtsi32_si128(loadu_int32(data + 2 * stride)); + const __m128i src_12a = _mm_unpacklo_epi32(d[1], d[0]); + + const __m128i s = _mm_unpacklo_epi8(src_01a, src_12a); + + *res = _mm_maddubs_epi16(s, coeffs[0]); +} + +static inline void convolve_y_4tap_4x2_ssse3(const uint8_t *const data, + const ptrdiff_t stride, + const __m128i coeffs[2], + __m128i d[4], __m128i s[2], + __m128i *res) { + d[3] = _mm_cvtsi32_si128(loadu_int32(data + 3 * stride)); + const __m128i src_23a = _mm_unpacklo_epi32(d[2], d[3]); + d[2] = _mm_cvtsi32_si128(loadu_int32(data + 4 * stride)); + const __m128i src_34a = _mm_unpacklo_epi32(d[3], d[2]); + + s[1] = _mm_unpacklo_epi8(src_23a, src_34a); + + *res = convolve_lowbd_4tap_ssse3(s, coeffs); +} + +static inline void convolve_y_6tap_4x2_ssse3(const uint8_t *const data, + const ptrdiff_t stride, + const __m128i coeffs[3], + __m128i d[6], __m128i s[3], + __m128i *res) { + d[5] = _mm_cvtsi32_si128(loadu_int32(data + 5 * stride)); + const __m128i src_45a = _mm_unpacklo_epi32(d[4], d[5]); + d[4] = _mm_cvtsi32_si128(loadu_int32(data + 6 * stride)); + const __m128i src_56a = _mm_unpacklo_epi32(d[5], d[4]); + + s[2] = _mm_unpacklo_epi8(src_45a, src_56a); + + *res = convolve_lowbd_6tap_ssse3(s, coeffs); +} + +static inline void convolve_y_8tap_4x2_ssse3(const uint8_t *const data, + const ptrdiff_t stride, + const __m128i coeffs[4], + __m128i d[8], __m128i s[4], + __m128i *res) { + d[7] = _mm_cvtsi32_si128(loadu_int32(data + 7 * stride)); + const __m128i src_67a = _mm_unpacklo_epi32(d[6], d[7]); + d[6] = _mm_cvtsi32_si128(loadu_int32(data + 8 * stride)); + const __m128i src_78a = _mm_unpacklo_epi32(d[7], d[6]); + + s[3] = _mm_unpacklo_epi8(src_67a, src_78a); + + res[0] = convolve_lowbd_ssse3(s, coeffs); +} + +static inline void convolve_y_2tap_8x2_avx2(const uint8_t *const data, + const ptrdiff_t stride, + const __m256i *coeffs, __m128i d[2], + __m256i *res) { + d[1] = _mm_loadu_si128((__m128i *)(data + 1 * stride)); + const __m256i src_01a = _mm256_setr_m128i(d[0], d[1]); + d[0] = _mm_loadu_si128((__m128i *)(data + 2 * stride)); + const __m256i src_12a = _mm256_setr_m128i(d[1], d[0]); + + const __m256i s = _mm256_unpacklo_epi8(src_01a, src_12a); + + *res = _mm256_maddubs_epi16(s, coeffs[0]); +} + +static inline void convolve_y_4tap_8x2_avx2(const uint8_t *const data, + const ptrdiff_t stride, + const __m256i coeffs[2], + __m128i d[4], __m256i s[2], + __m256i *res) { + d[3] = _mm_loadu_si128((__m128i *)(data + 3 * stride)); + const __m256i src_23a = _mm256_setr_m128i(d[2], d[3]); + d[2] = _mm_loadu_si128((__m128i *)(data + 4 * stride)); + const __m256i src_34a = _mm256_setr_m128i(d[3], d[2]); + + s[1] = _mm256_unpacklo_epi8(src_23a, src_34a); + + *res = convolve_lowbd_4tap(s, coeffs); +} + +static inline void convolve_y_6tap_8x2_avx2(const uint8_t *const data, + const ptrdiff_t stride, + const __m256i coeffs[3], + __m128i d[6], __m256i s[3], + __m256i *res) { + d[5] = _mm_loadu_si128((__m128i *)(data + 5 * stride)); + const __m256i src_45a = _mm256_setr_m128i(d[4], d[5]); + d[4] = _mm_loadu_si128((__m128i *)(data + 6 * stride)); + const __m256i src_56a = _mm256_setr_m128i(d[5], d[4]); + + s[2] = _mm256_unpacklo_epi8(src_45a, src_56a); + + *res = convolve_lowbd_6tap(s, coeffs); +} + +static inline void convolve_y_8tap_8x2_avx2(const uint8_t *const data, + const ptrdiff_t stride, + const __m256i coeffs[4], + __m128i d[8], __m256i s[4], + __m256i *res) { + d[7] = _mm_loadu_si128((__m128i *)(data + 7 * stride)); + const __m256i src_67a = _mm256_setr_m128i(d[6], d[7]); + d[6] = _mm_loadu_si128((__m128i *)(data + 8 * stride)); + const __m256i src_78a = _mm256_setr_m128i(d[7], d[6]); + + s[3] = _mm256_unpacklo_epi8(src_67a, src_78a); + + *res = convolve_lowbd(s, coeffs); +} + +static inline void convolve_y_2tap_16x2_avx2(const uint8_t *const data, + const ptrdiff_t stride, + const __m256i *coeffs, + __m128i d[2], __m256i res[2]) { + d[1] = _mm_loadu_si128((__m128i *)(data + 1 * stride)); + const __m256i src_01a = _mm256_setr_m128i(d[0], d[1]); + d[0] = _mm_loadu_si128((__m128i *)(data + 2 * stride)); + const __m256i src_12a = _mm256_setr_m128i(d[1], d[0]); + + const __m256i s0 = _mm256_unpacklo_epi8(src_01a, src_12a); + const __m256i s1 = _mm256_unpackhi_epi8(src_01a, src_12a); + + res[0] = _mm256_maddubs_epi16(s0, coeffs[0]); + res[1] = _mm256_maddubs_epi16(s1, coeffs[0]); +} + +static inline void convolve_y_4tap_16x2_avx2(const uint8_t *const data, + const ptrdiff_t stride, + const __m256i coeffs[2], + __m128i d[4], __m256i s[4], + __m256i res[2]) { + d[3] = _mm_loadu_si128((__m128i *)(data + 3 * stride)); + const __m256i src_23a = _mm256_setr_m128i(d[2], d[3]); + d[2] = _mm_loadu_si128((__m128i *)(data + 4 * stride)); + const __m256i src_34a = _mm256_setr_m128i(d[3], d[2]); + + s[1] = _mm256_unpacklo_epi8(src_23a, src_34a); + s[3] = _mm256_unpackhi_epi8(src_23a, src_34a); + + res[0] = convolve_lowbd_4tap(s, coeffs); + res[1] = convolve_lowbd_4tap(s + 2, coeffs); +} + +static inline void convolve_y_6tap_16x2_avx2(const uint8_t *const data, + const ptrdiff_t stride, + const __m256i coeffs[3], + __m128i d[6], __m256i s[6], + __m256i res[2]) { + d[5] = _mm_loadu_si128((__m128i *)(data + 5 * stride)); + const __m256i src_45a = _mm256_setr_m128i(d[4], d[5]); + d[4] = _mm_loadu_si128((__m128i *)(data + 6 * stride)); + const __m256i src_56a = _mm256_setr_m128i(d[5], d[4]); + + s[2] = _mm256_unpacklo_epi8(src_45a, src_56a); + s[5] = _mm256_unpackhi_epi8(src_45a, src_56a); + + res[0] = convolve_lowbd_6tap(s, coeffs); + res[1] = convolve_lowbd_6tap(s + 3, coeffs); +} + +static inline void convolve_y_8tap_16x2_avx2(const uint8_t *const data, + const ptrdiff_t stride, + const __m256i coeffs[4], + __m128i d[8], __m256i s[8], + __m256i res[2]) { + d[7] = _mm_loadu_si128((__m128i *)(data + 7 * stride)); + const __m256i src_67a = _mm256_setr_m128i(d[6], d[7]); + d[6] = _mm_loadu_si128((__m128i *)(data + 8 * stride)); + const __m256i src_78a = _mm256_setr_m128i(d[7], d[6]); + + s[3] = _mm256_unpacklo_epi8(src_67a, src_78a); + s[7] = _mm256_unpackhi_epi8(src_67a, src_78a); + + res[0] = convolve_lowbd(s, coeffs); + res[1] = convolve_lowbd(s + 4, coeffs); +} + +static inline void convolve_y_2tap_32x2_avx2(const uint8_t *const data, + const ptrdiff_t stride, + const __m256i *coeffs, + __m256i d[2], __m256i res[4]) { + d[1] = _mm256_loadu_si256((__m256i *)(data + 1 * stride)); + const __m256i s00 = _mm256_unpacklo_epi8(d[0], d[1]); + const __m256i s01 = _mm256_unpackhi_epi8(d[0], d[1]); + d[0] = _mm256_loadu_si256((__m256i *)(data + 2 * stride)); + const __m256i s10 = _mm256_unpacklo_epi8(d[1], d[0]); + const __m256i s11 = _mm256_unpackhi_epi8(d[1], d[0]); + + res[0] = _mm256_maddubs_epi16(s00, coeffs[0]); + res[1] = _mm256_maddubs_epi16(s01, coeffs[0]); + res[2] = _mm256_maddubs_epi16(s10, coeffs[0]); + res[3] = _mm256_maddubs_epi16(s11, coeffs[0]); +} + +static inline void convolve_y_4tap_32x2_avx2(const uint8_t *const data, + const ptrdiff_t stride, + const __m256i coeffs[2], + __m256i d[4], __m256i s1[4], + __m256i s2[4], __m256i res[4]) { + d[3] = _mm256_loadu_si256((__m256i *)(data + 3 * stride)); + s1[1] = _mm256_unpacklo_epi8(d[2], d[3]); + s1[3] = _mm256_unpackhi_epi8(d[2], d[3]); + d[2] = _mm256_loadu_si256((__m256i *)(data + 4 * stride)); + s2[1] = _mm256_unpacklo_epi8(d[3], d[2]); + s2[3] = _mm256_unpackhi_epi8(d[3], d[2]); + + res[0] = convolve_lowbd_4tap(s1, coeffs); + res[1] = convolve_lowbd_4tap(s1 + 2, coeffs); + res[2] = convolve_lowbd_4tap(s2, coeffs); + res[3] = convolve_lowbd_4tap(s2 + 2, coeffs); +} #endif // AOM_AOM_DSP_X86_CONVOLVE_AVX2_H_
diff --git a/aom_mem/aom_mem.h b/aom_mem/aom_mem.h index 15cc43d..b332139 100644 --- a/aom_mem/aom_mem.h +++ b/aom_mem/aom_mem.h
@@ -12,6 +12,8 @@ #ifndef AOM_AOM_MEM_AOM_MEM_H_ #define AOM_AOM_MEM_AOM_MEM_H_ +#include <assert.h> + #include "aom/aom_integer.h" #include "config/aom_config.h" @@ -57,6 +59,7 @@ #if CONFIG_DEBUG #define AOM_CHECK_MEM_ERROR(error_info, lval, expr) \ do { \ + assert((error_info)->setjmp); \ lval = (expr); \ if (!lval) \ aom_internal_error(error_info, AOM_CODEC_MEM_ERROR, \ @@ -66,6 +69,7 @@ #else #define AOM_CHECK_MEM_ERROR(error_info, lval, expr) \ do { \ + assert((error_info)->setjmp); \ lval = (expr); \ if (!lval) \ aom_internal_error(error_info, AOM_CODEC_MEM_ERROR, \
diff --git a/aom_scale/aom_scale_rtcd.pl b/aom_scale/aom_scale_rtcd.pl index bcdb898..1f2df8d 100644 --- a/aom_scale/aom_scale_rtcd.pl +++ b/aom_scale/aom_scale_rtcd.pl
@@ -19,8 +19,6 @@ add_proto qw/int aom_yv12_realloc_with_new_border/, "struct yv12_buffer_config *ybf, int new_border, int byte_alignment, bool alloc_pyramid, int num_planes"; -add_proto qw/void aom_yv12_extend_frame_borders/, "struct yv12_buffer_config *ybf, const int num_planes"; - add_proto qw/void aom_yv12_copy_frame/, "const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, const int num_planes"; add_proto qw/void aom_yv12_copy_y/, "const struct yv12_buffer_config *src_ybc, struct yv12_buffer_config *dst_ybc, int use_crop";
diff --git a/aom_scale/generic/yv12extend.c b/aom_scale/generic/yv12extend.c index 019fbfb..8bc01f7 100644 --- a/aom_scale/generic/yv12extend.c +++ b/aom_scale/generic/yv12extend.c
@@ -144,42 +144,6 @@ extend_bottom_border ? bottom : 0, right, v_start, v_end); } -void aom_yv12_extend_frame_borders_c(YV12_BUFFER_CONFIG *ybf, - const int num_planes) { - assert(ybf->border % 2 == 0); - assert(ybf->y_height - ybf->y_crop_height < 16); - assert(ybf->y_width - ybf->y_crop_width < 16); - assert(ybf->y_height - ybf->y_crop_height >= 0); - assert(ybf->y_width - ybf->y_crop_width >= 0); - -#if CONFIG_AV1_HIGHBITDEPTH - if (ybf->flags & YV12_FLAG_HIGHBITDEPTH) { - for (int plane = 0; plane < num_planes; ++plane) { - const int is_uv = plane > 0; - const int plane_border = ybf->border >> is_uv; - extend_plane_high( - ybf->buffers[plane], ybf->strides[is_uv], ybf->crop_widths[is_uv], - ybf->crop_heights[is_uv], plane_border, plane_border, - plane_border + ybf->heights[is_uv] - ybf->crop_heights[is_uv], - plane_border + ybf->widths[is_uv] - ybf->crop_widths[is_uv], 0, - ybf->crop_heights[is_uv]); - } - return; - } -#endif - - for (int plane = 0; plane < num_planes; ++plane) { - const int is_uv = plane > 0; - const int plane_border = ybf->border >> is_uv; - extend_plane(ybf->buffers[plane], ybf->strides[is_uv], - ybf->crop_widths[is_uv], ybf->crop_heights[is_uv], - plane_border, plane_border, - plane_border + ybf->heights[is_uv] - ybf->crop_heights[is_uv], - plane_border + ybf->widths[is_uv] - ybf->crop_widths[is_uv], 0, - ybf->crop_heights[is_uv]); - } -} - static void extend_frame(YV12_BUFFER_CONFIG *const ybf, int ext_size, const int num_planes) { const int ss_x = ybf->subsampling_x; @@ -254,7 +218,7 @@ plane_dst += dst_bc->strides[is_uv]; } } - aom_yv12_extend_frame_borders_c(dst_bc, num_planes); + aom_extend_frame_borders_c(dst_bc, num_planes); return; } #endif @@ -269,7 +233,7 @@ plane_dst += dst_bc->strides[is_uv]; } } - aom_yv12_extend_frame_borders_c(dst_bc, num_planes); + aom_extend_frame_borders_c(dst_bc, num_planes); } void aom_yv12_copy_y_c(const YV12_BUFFER_CONFIG *src_ybc, @@ -482,9 +446,6 @@ // Copy image buffer aom_yv12_copy_frame(ybf, &new_buf, num_planes); - // Extend up to new border - aom_extend_frame_borders(&new_buf, num_planes); - // Now free the old buffer and replace with the new aom_free_frame_buffer(ybf); *ybf = new_buf;
diff --git a/aom_util/aom_thread.c b/aom_util/aom_thread.c index 5a60617..c4b3bf7 100644 --- a/aom_util/aom_thread.c +++ b/aom_util/aom_thread.c
@@ -44,6 +44,7 @@ static THREADFN thread_loop(void *ptr) { AVxWorker *const worker = (AVxWorker *)ptr; +#ifdef HAVE_PTHREAD_SETNAME_NP #ifdef __APPLE__ if (worker->thread_name != NULL) { // Apple's version of pthread_setname_np takes one argument and operates on @@ -66,6 +67,7 @@ pthread_setname_np(pthread_self(), thread_name); } #endif +#endif pthread_mutex_lock(&worker->impl_->mutex_); for (;;) { while (worker->status_ == AVX_WORKER_STATUS_OK) { // wait in idling mode
diff --git a/apps/aomdec.c b/apps/aomdec.c index 9933f6a..e09ebfc 100644 --- a/apps/aomdec.c +++ b/apps/aomdec.c
@@ -9,6 +9,7 @@ * PATENTS file, you can obtain it at www.aomedia.org/license/patent. */ +#include <inttypes.h> #include <stdio.h> #include <stdlib.h> #include <stdarg.h>
diff --git a/apps/aomenc.c b/apps/aomenc.c index 3caae90..d176812 100644 --- a/apps/aomenc.c +++ b/apps/aomenc.c
@@ -14,6 +14,7 @@ #include "config/aom_config.h" #include <assert.h> +#include <inttypes.h> #include <limits.h> #include <math.h> #include <stdarg.h> @@ -1206,10 +1207,12 @@ } config->use_16bit_internal |= config->cfg.g_bit_depth > AOM_BITS_8; +#if CONFIG_REALTIME_ONLY if (global->usage == AOM_USAGE_REALTIME && config->cfg.g_lag_in_frames != 0) { aom_tools_warn("non-zero lag-in-frames option ignored in realtime mode.\n"); config->cfg.g_lag_in_frames = 0; } +#endif if (global->usage == AOM_USAGE_ALL_INTRA) { if (config->cfg.g_lag_in_frames != 0) {
diff --git a/av1/arg_defs.c b/av1/arg_defs.c index 81d00c4..f69ee70 100644 --- a/av1/arg_defs.c +++ b/av1/arg_defs.c
@@ -326,7 +326,7 @@ .enable_tpl_model = ARG_DEF(NULL, "enable-tpl-model", 1, "RDO based on frame temporal dependency " "(0: off, 1: backward source based); " - "required for deltaq mode"), + "required for --deltaq-mode=1"), .enable_keyframe_filtering = ARG_DEF( NULL, "enable-keyframe-filtering", 1, "Apply temporal filtering on key frame " @@ -557,7 +557,7 @@ "Delta qindex mode (0: off, 1: deltaq objective (default), " "2: deltaq placeholder, 3: key frame visual quality, 4: user " "rating based visual quality optimization, 5: HDR video, 6: " - "Variance Boost all intra); requires --enable-tpl-model=1"), + "Variance Boost); --deltaq-mode=1 requires --enable-tpl-model=1"), .deltaq_strength = ARG_DEF(NULL, "deltaq-strength", 1, "Deltaq strength for" " --deltaq-mode=4 and --deltaq-mode=6 (%)"), @@ -641,11 +641,10 @@ .use_fixed_qp_offsets = ARG_DEF(NULL, "use-fixed-qp-offsets", 1, - "Enable fixed QP offsets for frames at different levels of the " - "pyramid. Selected automatically from --cq-level if " - "--fixed-qp-offsets is not provided. If this option is not " - "specified (default), offsets are adaptively chosen by the " - "encoder."), + "Controls how the encoder applies fixed QP offsets for frames at " + "different levels of the pyramid (0: adaptively-chosen offsets " + "from --cq-level if --fixed-qp-offsets is not provided " + "(default), 1: fixed QP offsets, 2: no QP offsets)"), .fixed_qp_offsets = ARG_DEF( NULL, "fixed-qp-offsets", 1, @@ -709,12 +708,15 @@ ARG_DEF(NULL, "sb-qp-sweep", 1, "When set to 1, enable the superblock level qp sweep for a " "given lambda to minimize the rdcost."), - .enable_low_complexity_decode = - ARG_DEF(NULL, "enable-low-complexity-decode", 1, - "Enable low complexity decode (0: false (default), 1: true)"), - .screen_detection_mode = - ARG_DEF(NULL, "screen-detection-mode", 1, - "Screen content detection mode (1: standard (default), " - "2: anti-aliased text and graphics aware)"), + .enable_low_complexity_decode = ARG_DEF( + NULL, "enable-low-complexity-decode", 1, + "Enable low complexity decode (0: false (default), 1: true). As of now, " + "this only supports good-quality encoding (speed 1 to 3) for vertical " + "videos between 608p and 720p."), + .screen_detection_mode = ARG_DEF( + NULL, "screen-detection-mode", 1, + "Screen content detection mode (1: standard (default in good quality and " + "realtime modes), 2: anti-aliased text and graphics aware (default in " + "all intra mode))"), #endif // CONFIG_AV1_ENCODER };
diff --git a/av1/av1.cmake b/av1/av1.cmake index 61533df..c440720 100644 --- a/av1/av1.cmake +++ b/av1/av1.cmake
@@ -94,6 +94,7 @@ if(CONFIG_HIGHWAY) list(APPEND AOM_AV1_COMMON_SOURCES "${AOM_ROOT}/av1/common/selfguided_hwy.h") + list(APPEND AOM_AV1_COMMON_SOURCES "${AOM_ROOT}/av1/common/warp_plane_hwy.h") endif() list(APPEND AOM_AV1_DECODER_SOURCES @@ -126,6 +127,8 @@ "${AOM_ROOT}/av1/encoder/allintra_vis.c" "${AOM_ROOT}/av1/encoder/allintra_vis.h" "${AOM_ROOT}/av1/encoder/enc_enums.h" + "${AOM_ROOT}/av1/encoder/av1_ext_ratectrl.c" + "${AOM_ROOT}/av1/encoder/av1_ext_ratectrl.h" "${AOM_ROOT}/av1/encoder/av1_fwd_txfm1d.c" "${AOM_ROOT}/av1/encoder/av1_fwd_txfm1d.h" "${AOM_ROOT}/av1/encoder/av1_fwd_txfm1d_cfg.h" @@ -321,8 +324,13 @@ "${AOM_ROOT}/av1/common/x86/wiener_convolve_avx2.c") if(CONFIG_HIGHWAY) + list(APPEND AOM_AV1_COMMON_INTRIN_SSE4_1 + "${AOM_ROOT}/av1/common/x86/warp_plane_hwy_sse4.cc") + list(APPEND AOM_AV1_COMMON_INTRIN_AVX2 + "${AOM_ROOT}/av1/common/x86/warp_plane_hwy_avx2.cc") list(APPEND AOM_AV1_COMMON_INTRIN_AVX512 - "${AOM_ROOT}/av1/common/x86/selfguided_hwy_avx512.cc") + "${AOM_ROOT}/av1/common/x86/selfguided_hwy_avx512.cc" + "${AOM_ROOT}/av1/common/x86/warp_plane_hwy_avx512.cc") endif() list(APPEND AOM_AV1_ENCODER_ASM_SSE2 "${AOM_ROOT}/av1/encoder/x86/dct_sse2.asm" @@ -454,11 +462,11 @@ "${AOM_ROOT}/av1/common/arm/av1_convolve_scale_neon_i8mm.c" "${AOM_ROOT}/av1/common/arm/compound_convolve_neon_i8mm.c" "${AOM_ROOT}/av1/common/arm/convolve_neon_i8mm.c" + "${AOM_ROOT}/av1/common/arm/reconintra_neon_i8mm.c" "${AOM_ROOT}/av1/common/arm/resize_neon_i8mm.c" "${AOM_ROOT}/av1/common/arm/warp_plane_neon_i8mm.c") list(APPEND AOM_AV1_COMMON_INTRIN_SVE - "${AOM_ROOT}/av1/common/arm/highbd_warp_plane_sve.c" "${AOM_ROOT}/av1/common/arm/warp_plane_sve.c") list(APPEND AOM_AV1_COMMON_INTRIN_SVE2 @@ -471,7 +479,9 @@ list(APPEND AOM_AV1_COMMON_INTRIN_RVV "${AOM_ROOT}/av1/common/riscv/cdef_block_rvv.c" + "${AOM_ROOT}/av1/common/riscv/compound_convolve_rvv.c" "${AOM_ROOT}/av1/common/riscv/convolve_rvv.c" + "${AOM_ROOT}/av1/common/riscv/highbd_compound_convolve_rvv.c" "${AOM_ROOT}/av1/common/riscv/highbd_convolve_rvv.c") if(CONFIG_THREE_PASS) @@ -542,6 +552,9 @@ "${AOM_ROOT}/av1/common/arm/highbd_warp_plane_neon.c" "${AOM_ROOT}/av1/common/arm/highbd_wiener_convolve_neon.c") + list(APPEND AOM_AV1_COMMON_INTRIN_SVE + "${AOM_ROOT}/av1/common/arm/highbd_warp_plane_sve.c") + list(APPEND AOM_AV1_COMMON_INTRIN_SVE2 "${AOM_ROOT}/av1/common/arm/highbd_compound_convolve_sve2.c" "${AOM_ROOT}/av1/common/arm/highbd_convolve_sve2.c") @@ -591,7 +604,8 @@ "${AOM_ROOT}/av1/common/restoration.h" "${AOM_ROOT}/av1/common/selfguided_hwy.h" "${AOM_ROOT}/av1/common/warped_motion.c" - "${AOM_ROOT}/av1/common/warped_motion.h") + "${AOM_ROOT}/av1/common/warped_motion.h" + "${AOM_ROOT}/av1/common/warp_plane_hwy.h") list(REMOVE_ITEM AOM_AV1_COMMON_INTRIN_SSE2 "${AOM_ROOT}/av1/common/x86/cfl_sse2.c" @@ -601,7 +615,8 @@ list(REMOVE_ITEM AOM_AV1_COMMON_INTRIN_SSE4_1 "${AOM_ROOT}/av1/common/x86/highbd_warp_plane_sse4.c" "${AOM_ROOT}/av1/common/x86/selfguided_sse4.c" - "${AOM_ROOT}/av1/common/x86/warp_plane_sse4.c") + "${AOM_ROOT}/av1/common/x86/warp_plane_sse4.c" + "${AOM_ROOT}/av1/common/x86/warp_plane_hwy_sse4.cc") list( REMOVE_ITEM AOM_AV1_COMMON_INTRIN_SSSE3 @@ -614,10 +629,12 @@ "${AOM_ROOT}/av1/common/x86/highbd_wiener_convolve_avx2.c" "${AOM_ROOT}/av1/common/x86/selfguided_avx2.c" "${AOM_ROOT}/av1/common/x86/warp_plane_avx2.c" + "${AOM_ROOT}/av1/common/x86/warp_plane_hwy_avx2.cc" "${AOM_ROOT}/av1/common/x86/wiener_convolve_avx2.c") list(REMOVE_ITEM AOM_AV1_COMMON_INTRIN_AVX512 - "${AOM_ROOT}/av1/common/x86/selfguided_hwy_avx512.cc") + "${AOM_ROOT}/av1/common/x86/selfguided_hwy_avx512.cc" + "${AOM_ROOT}/av1/common/x86/warp_plane_hwy_avx512.cc") list(REMOVE_ITEM AOM_AV1_COMMON_INTRIN_NEON "${AOM_ROOT}/av1/common/arm/cfl_neon.c" @@ -660,6 +677,7 @@ "${AOM_ROOT}/av1/encoder/arm/temporal_filter_neon_dotprod.c") list(REMOVE_ITEM AOM_AV1_ENCODER_INTRIN_SVE + "${AOM_ROOT}/av1/encoder/arm/highbd_pickrst_sve.c" "${AOM_ROOT}/av1/encoder/arm/pickrst_sve.c") list(REMOVE_ITEM AOM_AV1_ENCODER_SOURCES
diff --git a/av1/av1_cx_iface.c b/av1/av1_cx_iface.c index 07820d6..5abd959 100644 --- a/av1/av1_cx_iface.c +++ b/av1/av1_cx_iface.c
@@ -8,8 +8,11 @@ * Media Patent License 1.0 was not distributed with this source code in the * PATENTS file, you can obtain it at www.aomedia.org/license/patent. */ + +#include <assert.h> #include <limits.h> #include <math.h> +#include <setjmp.h> #include <stdbool.h> #include <stdint.h> #include <stdlib.h> @@ -34,6 +37,7 @@ #include "av1/common/enums.h" #include "av1/common/quant_common.h" #include "av1/common/scale.h" +#include "av1/encoder/av1_ext_ratectrl.h" #include "av1/encoder/bitstream.h" #include "av1/encoder/enc_enums.h" #include "av1/encoder/encoder.h" @@ -48,6 +52,33 @@ #include "common/args_helper.h" +// Creates a setjmp target using `CPI->common.error->jmp` and sets +// `CPI->common.error->setjmp = 1`. Returns `CPI->common.error->error_code` on +// longjmp. This macro expects `aom_codec_alg_priv_t *ctx` to be available. +// This should be accompanied by a call to DISABLE_SETJMP using the same CPI +// before going out of scope. +#define ENABLE_SETJMP(CPI) \ + do { \ + struct aom_internal_error_info *const enable_setjmp_error = \ + (CPI)->common.error; \ + if (setjmp(enable_setjmp_error->jmp)) { \ + enable_setjmp_error->setjmp = 0; \ + ctx->base.err_detail = enable_setjmp_error->has_detail \ + ? enable_setjmp_error->detail \ + : NULL; \ + return enable_setjmp_error->error_code; \ + } \ + enable_setjmp_error->setjmp = 1; \ + } while (0) + +// Sets CPI->common.error->setjmp = 0. +#define DISABLE_SETJMP(CPI) \ + do { \ + struct aom_internal_error_info *const enable_setjmp_error = \ + (CPI)->common.error; \ + enable_setjmp_error->setjmp = 0; \ + } while (0) + struct av1_extracfg { int cpu_used; unsigned int enable_auto_alt_ref; @@ -682,12 +713,6 @@ RANGE_CHECK_BOOL(extra_cfg, lossless); RANGE_CHECK_HI(extra_cfg, aq_mode, AQ_MODE_COUNT - 1); RANGE_CHECK_HI(extra_cfg, deltaq_mode, DELTA_Q_MODE_COUNT - 1); - - if (cfg->g_usage != ALLINTRA && - extra_cfg->deltaq_mode == DELTA_Q_VARIANCE_BOOST) { - ERROR("Variance Boost (deltaq_mode = 6) can only be set in all intra mode"); - } - RANGE_CHECK_HI(extra_cfg, deltalf_mode, 1); RANGE_CHECK_HI(extra_cfg, frame_periodic_boost, 1); #if CONFIG_REALTIME_ONLY @@ -820,7 +845,7 @@ } if (cfg->rc_end_usage == AOM_Q) { - RANGE_CHECK_HI(cfg, use_fixed_qp_offsets, 1); + RANGE_CHECK_HI(cfg, use_fixed_qp_offsets, 2); } else { if (cfg->use_fixed_qp_offsets > 0) { ERROR("--use_fixed_qp_offsets can only be used with --end-usage=q"); @@ -1260,7 +1285,8 @@ q_cfg->deltaq_mode = extra_cfg->deltaq_mode; q_cfg->deltaq_strength = extra_cfg->deltaq_strength; q_cfg->use_fixed_qp_offsets = - cfg->use_fixed_qp_offsets && (rc_cfg->mode == AOM_Q); + (rc_cfg->mode == AOM_Q) ? cfg->use_fixed_qp_offsets : 0; + q_cfg->enable_hdr_deltaq = (q_cfg->deltaq_mode == DELTA_Q_HDR) && (cfg->g_bit_depth == AOM_BITS_10) && @@ -1337,11 +1363,15 @@ oxcf->speed = 7; // Now, low complexity decode mode is only supported for good-quality - // encoding speed 1 to 3. This can be further modified if needed. + // encoding speed 1 to 3 and for vertical videos with a resolution between + // 608p and 720p. This can be further modified if needed. + const int is_low_complexity_decode_mode_supported = + (cfg->g_usage == AOM_USAGE_GOOD_QUALITY) && + (oxcf->speed >= 1 && oxcf->speed <= 3) && + (AOMMIN(cfg->g_w, cfg->g_h) >= 608 && AOMMIN(cfg->g_w, cfg->g_h) <= 1080); oxcf->enable_low_complexity_decode = extra_cfg->enable_low_complexity_decode && - cfg->g_usage == AOM_USAGE_GOOD_QUALITY && oxcf->speed >= 1 && - oxcf->speed <= 3; + is_low_complexity_decode_mode_supported; // Set Color related configuration. color_cfg->color_primaries = extra_cfg->color_primaries; @@ -1351,10 +1381,22 @@ color_cfg->chroma_sample_position = extra_cfg->chroma_sample_position; // Set Group of frames configuration. - // Force lag_in_frames to 0 for REALTIME mode +#if CONFIG_REALTIME_ONLY + // When CONFIG_REALTIME_ONLY=1 and mode=REALTIME, then force lag_in_frames + // = 0. gf_cfg->lag_in_frames = (oxcf->mode == REALTIME) ? 0 : clamp(cfg->g_lag_in_frames, 0, MAX_LAG_BUFFERS); +#else + gf_cfg->lag_in_frames = clamp(cfg->g_lag_in_frames, 0, MAX_LAG_BUFFERS); +#endif + + // Modify lag_in_frames slightly for better coding performance (e.g. better + // temporal filtering result). + if (oxcf->mode == GOOD && gf_cfg->lag_in_frames >= 32 && + gf_cfg->lag_in_frames < 39) + gf_cfg->lag_in_frames = AOMMIN(39, MAX_LAG_BUFFERS); + gf_cfg->enable_auto_arf = extra_cfg->enable_auto_alt_ref; gf_cfg->enable_auto_brf = extra_cfg->enable_auto_bwd_ref; gf_cfg->min_gf_interval = extra_cfg->min_gf_interval; @@ -1575,7 +1617,7 @@ static aom_codec_err_t encoder_set_config(aom_codec_alg_priv_t *ctx, const aom_codec_enc_cfg_t *cfg) { aom_codec_err_t res; - int force_key = 0; + volatile int force_key = 0; if (cfg->g_w != ctx->cfg.g_w || cfg->g_h != ctx->cfg.g_h) { if (cfg->g_lag_in_frames > 1 || cfg->g_pass != AOM_RC_ONE_PASS) @@ -1627,11 +1669,16 @@ bool is_sb_size_changed = false; av1_change_config_seq(ctx->ppi, &ctx->oxcf, &is_sb_size_changed); for (int i = 0; i < ctx->ppi->num_fp_contexts; i++) { - av1_change_config(ctx->ppi->parallel_cpi[i], &ctx->oxcf, - is_sb_size_changed); + AV1_COMP *const cpi = ctx->ppi->parallel_cpi[i]; + ENABLE_SETJMP(cpi); + av1_change_config(cpi, &ctx->oxcf, is_sb_size_changed); + DISABLE_SETJMP(cpi); } if (ctx->ppi->cpi_lap != NULL) { - av1_change_config(ctx->ppi->cpi_lap, &ctx->oxcf, is_sb_size_changed); + AV1_COMP *const cpi = ctx->ppi->cpi_lap; + ENABLE_SETJMP(cpi); + av1_change_config(cpi, &ctx->oxcf, is_sb_size_changed); + DISABLE_SETJMP(cpi); } } @@ -1677,31 +1724,26 @@ } static aom_codec_err_t update_encoder_cfg(aom_codec_alg_priv_t *ctx) { + // Disable denoiser for spatial layers. Bug: 485332522. + // TODO: bug 485332522 - Disable denoiser for spatial layers until + // more testing is done + if (ctx->ppi->cpi->svc.number_spatial_layers > 1) + ctx->extra_cfg.noise_sensitivity = 0; set_encoder_config(&ctx->oxcf, &ctx->cfg, &ctx->extra_cfg); av1_check_fpmt_config(ctx->ppi, &ctx->oxcf); bool is_sb_size_changed = false; av1_change_config_seq(ctx->ppi, &ctx->oxcf, &is_sb_size_changed); for (int i = 0; i < ctx->ppi->num_fp_contexts; i++) { AV1_COMP *const cpi = ctx->ppi->parallel_cpi[i]; - struct aom_internal_error_info *const error = cpi->common.error; - if (setjmp(error->jmp)) { - error->setjmp = 0; - return error->error_code; - } - error->setjmp = 1; + ENABLE_SETJMP(cpi); av1_change_config(cpi, &ctx->oxcf, is_sb_size_changed); - error->setjmp = 0; + DISABLE_SETJMP(cpi); } if (ctx->ppi->cpi_lap != NULL) { AV1_COMP *const cpi_lap = ctx->ppi->cpi_lap; - struct aom_internal_error_info *const error = cpi_lap->common.error; - if (setjmp(error->jmp)) { - error->setjmp = 0; - return error->error_code; - } - error->setjmp = 1; + ENABLE_SETJMP(cpi_lap); av1_change_config(cpi_lap, &ctx->oxcf, is_sb_size_changed); - error->setjmp = 0; + DISABLE_SETJMP(cpi_lap); } return AOM_CODEC_OK; } @@ -1716,6 +1758,20 @@ return res; } +static aom_codec_err_t ctrl_get_gop_info(aom_codec_alg_priv_t *ctx, + va_list args) { + aom_gop_info_t *const gop_info = va_arg(args, aom_gop_info_t *); + if (gop_info == NULL) return AOM_CODEC_INVALID_PARAM; + const GF_GROUP *const gf_group = &ctx->ppi->gf_group; + gop_info->gop_size = gf_group->size; + for (int i = 0; i < gf_group->size; ++i) { + gop_info->coding_index[i] = i; + gop_info->display_index[i] = gf_group->display_idx[i]; + gop_info->layer_depth[i] = gf_group->layer_depth[i]; + } + return AOM_CODEC_OK; +} + static aom_codec_err_t ctrl_set_cpuused(aom_codec_alg_priv_t *ctx, va_list args) { struct av1_extracfg extra_cfg = ctx->extra_cfg; @@ -1838,11 +1894,9 @@ return update_extra_cfg(ctx, &extra_cfg); } -static aom_codec_err_t handle_tuning(aom_codec_alg_priv_t *ctx, - struct av1_extracfg *extra_cfg) { +static aom_codec_err_t handle_tuning(struct av1_extracfg *extra_cfg) { if (extra_cfg->tuning == AOM_TUNE_IQ || extra_cfg->tuning == AOM_TUNE_SSIMULACRA2) { - if (ctx->cfg.g_usage != AOM_USAGE_ALL_INTRA) return AOM_CODEC_INCAPABLE; // Enable QMs as they've been found to be beneficial for images, when used // with alternative QM formulas: // - aom_get_qmlevel_allintra() @@ -1851,8 +1905,8 @@ extra_cfg->enable_qm = 1; extra_cfg->qm_min = QM_FIRST_IQ_SSIMULACRA2; extra_cfg->qm_max = QM_LAST_IQ_SSIMULACRA2; - // We can turn on sharpness, as frames do not have to serve as references to - // others. + // Sharpness has been found to be beneficial for images (better perceptual + // quality). extra_cfg->sharpness = 7; // Using the QM-PSNR metric was found to be beneficial for images (over the // default PSNR metric), as it correlates better with subjective image @@ -1886,7 +1940,7 @@ va_list args) { struct av1_extracfg extra_cfg = ctx->extra_cfg; extra_cfg.tuning = CAST(AOME_SET_TUNING, args); - aom_codec_err_t err = handle_tuning(ctx, &extra_cfg); + aom_codec_err_t err = handle_tuning(&extra_cfg); if (err != AOM_CODEC_OK) return err; return update_extra_cfg(ctx, &extra_cfg); } @@ -2959,14 +3013,20 @@ priv->extra_cfg = default_extra_cfg; // Special handling: - // By default, if omitted: --enable-cdef=1, --qm-min=5, and --qm-max=9 - // Here we set its default values to 0, 4, and 10 respectively when - // --allintra is turned on. - // However, if users set --enable-cdef, --qm-min, or --qm-max, either from - // the command line or aom_codec_control(), the encoder still respects it. + // By default, if omitted: --enable-cdef=1, --screen-detection-mode=1, + // --qm-min=5, and --qm-max=9. + // Here we set its default values to --enable-cdef=0, + // --screen-detection-mode=2, --qm-min=4, and --qm-max=10 when --allintra + // is turned on. + // However, if users set --enable-cdef, --screen-detection-mode, --qm-min, + // or --qm-max, either from the command line or aom_codec_control(), the + // encoder still respects it. if (priv->cfg.g_usage == AOM_USAGE_ALL_INTRA) { // CDEF has been found to blur images, so it's disabled in all-intra mode priv->extra_cfg.enable_cdef = 0; + // Enable "anti-aliased text and graphics aware" screen detection mode. + priv->extra_cfg.screen_detection_mode = + AOM_SCREEN_DETECTION_ANTIALIASING_AWARE; // These QM min/max values have been found to be beneficial for images, // when used with an alternative QM formula (see // aom_get_qmlevel_allintra()). @@ -2989,11 +3049,18 @@ reduce_ratio(&priv->timestamp_ratio); set_encoder_config(&priv->oxcf, &priv->cfg, &priv->extra_cfg); - if (priv->oxcf.rc_cfg.mode != AOM_CBR && - priv->oxcf.pass == AOM_RC_ONE_PASS && priv->oxcf.mode == GOOD) { - // Enable look ahead - enabled for AOM_Q, AOM_CQ, AOM_VBR + if (priv->oxcf.pass == AOM_RC_ONE_PASS) { + // Enable look ahead. +#if CONFIG_REALTIME_ONLY + // When CONFIG_REALTIME_ONLY=1 and mode=REALTIME, then force + // lag_in_frames = 0. + const int lag_in_frames = + (priv->oxcf.mode == REALTIME) ? 0 : (int)priv->cfg.g_lag_in_frames; +#else + const int lag_in_frames = (int)priv->cfg.g_lag_in_frames; +#endif *num_lap_buffers = - AOMMIN((int)priv->cfg.g_lag_in_frames, + AOMMIN(lag_in_frames, AOMMIN(MAX_LAP_BUFFERS, priv->oxcf.kf_cfg.key_freq_max + SCENE_CUT_KEY_TEST_INTERVAL)); if ((int)priv->cfg.g_lag_in_frames - (*num_lap_buffers) >= @@ -3101,6 +3168,7 @@ if (ctx->ppi) { AV1_PRIMARY *ppi = ctx->ppi; + av1_extrc_delete(&ppi->cpi->ext_ratectrl); for (int i = 0; i < MAX_PARALLEL_FRAMES - 1; i++) { if (ppi->parallel_frames_data[i].cx_data) { free(ppi->parallel_frames_data[i].cx_data); @@ -3258,6 +3326,8 @@ if (ppi->use_svc && ppi->cpi->svc.use_flexible_mode == 0 && flags == 0) av1_set_svc_fixed_mode(ppi->cpi); + ppi->b_freeze_internal_state = flags & AOM_EFLAG_FREEZE_INTERNAL_STATE; + // Note(yunqing): While applying encoding flags, always start from enabling // all, and then modifying according to the flags. Previous frame's flags are // overwritten. @@ -3296,6 +3366,16 @@ "Cannot calculate per-frame PSNR when g_lag_in_frames is nonzero"); } + // Don't attempt to freeze internal state when lag is non-zero in order to + // minimize risk of state leaking when e.g. multi-pass and b-frames are + // used. + if ((flags & AOM_EFLAG_FREEZE_INTERNAL_STATE) && + ctx->cfg.g_lag_in_frames != 0) { + aom_internal_error( + &ppi->error, AOM_CODEC_INCAPABLE, + "Cannot freeze internal state when g_lag_in_frames is nonzero"); + } + // Set up internal flags #if CONFIG_INTERNAL_STATS assert(ppi->b_calculate_psnr == 1); @@ -3490,7 +3570,10 @@ #endif // CONFIG_MULTITHREAD // Call for LAP stage - if (cpi_lap != NULL) { + if (cpi_lap != NULL && !is_one_pass_rt_lag_params(cpi)) { + if (cpi_lap->ppi->b_freeze_internal_state) { + av1_save_all_coding_context(cpi_lap); + } AV1_COMP_DATA cpi_lap_data = { 0 }; cpi_lap_data.flush = !img; cpi_lap_data.timestamp_ratio = &ctx->timestamp_ratio; @@ -3499,6 +3582,9 @@ aom_internal_error_copy(&ppi->error, cpi_lap->common.error); } av1_post_encode_updates(cpi_lap, &cpi_lap_data); + if (cpi_lap->ppi->b_freeze_internal_state) { + restore_all_coding_context(cpi_lap); + } } // Recalculate the maximum number of frames that can be encoded in @@ -3517,6 +3603,9 @@ cpi->ref_idx_to_skip = INVALID_IDX; cpi->ref_refresh_index = INVALID_IDX; cpi->refresh_idx_available = false; + if (cpi->ppi->b_freeze_internal_state) { + av1_save_all_coding_context(cpi); + } #if CONFIG_FPMT_TEST simulate_parallel_frame = @@ -3533,6 +3622,15 @@ #endif // CONFIG_FPMT_TEST if (!simulate_parallel_frame) { + // Add a range check here to give an error for frame_parallel_level + // array out-of-bounds access. + if (cpi->gf_frame_index >= MAX_STATIC_GF_GROUP_LENGTH) { + aom_internal_error(&ppi->error, AOM_CODEC_ERROR, + "cpi->gf_frame_index is out of range"); + } + + // May need a better way for checking the frame's frame_parallel_level, + // especially for the first frame of the following gop. if (ppi->gf_group.frame_parallel_level[cpi->gf_frame_index] == 0) { status = av1_get_compressed_data(cpi, &cpi_data); } else if (ppi->gf_group.frame_parallel_level[cpi->gf_frame_index] == @@ -3559,6 +3657,10 @@ ppi->seq_params_locked = 1; av1_post_encode_updates(cpi, &cpi_data); + if (cpi->ppi->b_freeze_internal_state) { + restore_all_coding_context(cpi); + } + #if CONFIG_ENTROPY_STATS if (ppi->cpi->oxcf.pass != 1 && !cpi->common.show_existing_frame) av1_accumulate_frame_counts(&ppi->aggregate_fc, &cpi->counts); @@ -3935,7 +4037,7 @@ AV1_PRIMARY *const ppi = ctx->ppi; AV1_COMP *const cpi = ppi->cpi; aom_svc_params_t *const params = va_arg(args, aom_svc_params_t *); - int64_t target_bandwidth = 0; + volatile int64_t target_bandwidth = 0; // Note svc.use_flexible_mode is set by AV1E_SET_SVC_REF_FRAME_CONFIG. When // it is false (the default) the actual limit is 3 for both spatial and // temporal layers. Given the order of these calls are unpredictable the @@ -3980,7 +4082,8 @@ if (ppi->number_spatial_layers > 1 || ppi->number_temporal_layers > 1) { unsigned int sl, tl; - ctx->ppi->use_svc = 1; + // Disable svc for lag_in_frames > 0. + if (cpi->oxcf.gf_cfg.lag_in_frames > 0) return AOM_CODEC_INVALID_PARAM; const int num_layers = ppi->number_spatial_layers * ppi->number_temporal_layers; for (int layer = 0; layer < num_layers; ++layer) { @@ -3991,6 +4094,7 @@ } } if (!av1_alloc_layer_context(cpi, num_layers)) return AOM_CODEC_MEM_ERROR; + ppi->use_svc = 1; for (sl = 0; sl < ppi->number_spatial_layers; ++sl) { for (tl = 0; tl < ppi->number_temporal_layers; ++tl) { @@ -4019,7 +4123,9 @@ ctx->oxcf.rc_cfg.target_bandwidth = oxcf->rc_cfg.target_bandwidth = target_bandwidth; set_primary_rc_buffer_sizes(oxcf, ppi); + ENABLE_SETJMP(cpi); av1_update_layer_context_change_config(cpi, target_bandwidth); + DISABLE_SETJMP(cpi); check_reset_rc_flag(cpi); } else { // Note av1_init_layer_context() relies on cpi->oxcf. The order of that @@ -4033,7 +4139,9 @@ seq_params->operating_points_cnt_minus_1 = ppi->number_spatial_layers * ppi->number_temporal_layers - 1; + ENABLE_SETJMP(cpi); av1_init_layer_context(cpi); + DISABLE_SETJMP(cpi); // update_encoder_cfg() is somewhat costly and this control may be called // multiple times, so update_encoder_cfg() is only called to ensure frame // and superblock sizes are updated before they're fixed by the first @@ -4164,6 +4272,55 @@ return update_extra_cfg(ctx, &extra_cfg); } +static aom_codec_err_t ctrl_set_external_rate_control(aom_codec_alg_priv_t *ctx, + va_list args) { + aom_rc_funcs_t funcs = *CAST(AV1E_SET_EXTERNAL_RATE_CONTROL, args); + AV1_COMP *cpi = ctx->ppi->cpi; + AOM_EXT_RATECTRL *ext_ratectrl = &cpi->ext_ratectrl; + const AV1EncoderConfig *oxcf = &cpi->oxcf; + if (oxcf->pass == AOM_RC_SECOND_PASS) { + const FRAME_INFO *frame_info = &cpi->frame_info; + aom_rc_config_t ratectrl_config; + aom_codec_err_t codec_status; + memset(&ratectrl_config, 0, sizeof(ratectrl_config)); + + ratectrl_config.frame_width = frame_info->frame_width; + ratectrl_config.frame_height = frame_info->frame_height; + ratectrl_config.show_frame_count = + cpi->ppi->twopass.firstpass_info.stats_count; + ratectrl_config.max_gf_interval = ctx->extra_cfg.max_gf_interval; + ratectrl_config.min_gf_interval = ctx->extra_cfg.min_gf_interval; + ratectrl_config.target_bitrate_kbps = + (int)(oxcf->rc_cfg.target_bandwidth / 1000); + ratectrl_config.frame_rate_num = ctx->cfg.g_timebase.den; + ratectrl_config.frame_rate_den = ctx->cfg.g_timebase.num; + ratectrl_config.overshoot_percent = oxcf->rc_cfg.over_shoot_pct; + ratectrl_config.undershoot_percent = oxcf->rc_cfg.under_shoot_pct; + ratectrl_config.min_base_q_index = oxcf->rc_cfg.best_allowed_q; + ratectrl_config.max_base_q_index = oxcf->rc_cfg.worst_allowed_q; + ratectrl_config.base_qp = ctx->extra_cfg.cq_level; + + const BLOCK_SIZE sb_size = av1_select_sb_size( + oxcf, frame_info->frame_width, frame_info->frame_height, + cpi->ppi->number_spatial_layers); + ratectrl_config.superblock_size = (sb_size == BLOCK_128X128) ? 128 : 64; + + if (ctx->cfg.rc_end_usage == AOM_VBR) { + ratectrl_config.rc_mode = AOM_RC_VBR; + } else if (ctx->cfg.rc_end_usage == AOM_Q) { + ratectrl_config.rc_mode = AOM_RC_QMODE; + } else if (ctx->cfg.rc_end_usage == AOM_CQ) { + ratectrl_config.rc_mode = AOM_RC_CQ; + } + + codec_status = av1_extrc_create(funcs, ratectrl_config, ext_ratectrl); + if (codec_status != AOM_CODEC_OK) { + return codec_status; + } + } + return AOM_CODEC_OK; +} + static aom_codec_err_t encoder_set_option(aom_codec_alg_priv_t *ctx, const char *name, const char *value) { if (ctx == NULL || name == NULL || value == NULL) @@ -4265,7 +4422,7 @@ } else if (arg_match_helper(&arg, &g_av1_codec_arg_defs.tune_metric, argv, err_string)) { extra_cfg.tuning = arg_parse_enum_helper(&arg, err_string); - err = handle_tuning(ctx, &extra_cfg); + err = handle_tuning(&extra_cfg); } #if CONFIG_TUNE_VMAF else if (arg_match_helper(&arg, &g_av1_codec_arg_defs.vmaf_model_path, argv, @@ -4865,6 +5022,7 @@ { AV1E_SET_SCREEN_CONTENT_DETECTION_MODE, ctrl_set_screen_content_detection_mode }, { AV1E_SET_ENABLE_ADAPTIVE_SHARPNESS, ctrl_set_enable_adaptive_sharpness }, + { AV1E_SET_EXTERNAL_RATE_CONTROL, ctrl_set_external_rate_control }, // Getters { AOME_GET_LAST_QUANTIZER, ctrl_get_quantizer }, @@ -4883,6 +5041,7 @@ { AV1E_GET_LUMA_CDEF_STRENGTH, ctrl_get_luma_cdef_strength }, { AV1E_GET_HIGH_MOTION_CONTENT_SCREEN_RTC, ctrl_get_high_motion_content_screen_rtc }, + { AV1E_GET_GOP_INFO, ctrl_get_gop_info }, CTRL_MAP_END, };
diff --git a/av1/av1_dx_iface.c b/av1/av1_dx_iface.c index 3ff284d..50baa48 100644 --- a/av1/av1_dx_iface.c +++ b/av1/av1_dx_iface.c
@@ -62,6 +62,7 @@ unsigned int is_annexb; int operating_point; int output_all_layers; + unsigned int frame_size_limit; AVxWorker *frame_worker; @@ -112,6 +113,8 @@ priv->tile_mode = 0; priv->decode_tile_row = -1; priv->decode_tile_col = -1; + + priv->frame_size_limit = 0; } return AOM_CODEC_OK; @@ -247,13 +250,9 @@ } } - if (aom_get_num_layers_from_operating_point_idc( - operating_point_idc0, &si->number_spatial_layers, - &si->number_temporal_layers) != AOM_CODEC_OK) { - return AOM_CODEC_ERROR; - } - - return AOM_CODEC_OK; + return aom_get_num_layers_from_operating_point_idc( + operating_point_idc0, &si->number_spatial_layers, + &si->number_temporal_layers); } static aom_codec_err_t decoder_peek_si_internal(const uint8_t *data, @@ -495,6 +494,7 @@ frame_worker_data->pbi->dec_tile_col = ctx->decode_tile_col; frame_worker_data->pbi->operating_point = ctx->operating_point; frame_worker_data->pbi->output_all_layers = ctx->output_all_layers; + frame_worker_data->pbi->frame_size_limit = ctx->frame_size_limit; frame_worker_data->pbi->ext_tile_debug = ctx->ext_tile_debug; frame_worker_data->pbi->row_mt = ctx->row_mt; frame_worker_data->pbi->is_fwd_kf_present = 0; @@ -952,8 +952,18 @@ AVxWorker *const worker = ctx->frame_worker; FrameWorkerData *const frame_worker_data = (FrameWorkerData *)worker->data1; image2yuvconfig(&frame->img, &sd); - return av1_set_reference_dec(&frame_worker_data->pbi->common, frame->idx, - frame->use_external_ref, &sd); + + struct aom_internal_error_info *const error = + frame_worker_data->pbi->common.error; + if (setjmp(error->jmp)) { + error->setjmp = 0; + return error->error_code; + } + error->setjmp = 1; + av1_set_reference_dec(&frame_worker_data->pbi->common, frame->idx, + frame->use_external_ref, &sd); + error->setjmp = 0; + return AOM_CODEC_OK; } else { return AOM_CODEC_INVALID_PARAM; } @@ -1578,6 +1588,12 @@ return AOM_CODEC_OK; } +static aom_codec_err_t ctrl_set_frame_size_limit(aom_codec_alg_priv_t *ctx, + va_list args) { + ctx->frame_size_limit = va_arg(args, unsigned int); + return AOM_CODEC_OK; +} + static aom_codec_err_t ctrl_get_accounting(aom_codec_alg_priv_t *ctx, va_list args) { #if !CONFIG_ACCOUNTING @@ -1685,6 +1701,7 @@ { AV1D_SET_ROW_MT, ctrl_set_row_mt }, { AV1D_SET_EXT_REF_PTR, ctrl_set_ext_ref_ptr }, { AV1D_SET_SKIP_FILM_GRAIN, ctrl_set_skip_film_grain }, + { AOMD_SET_FRAME_SIZE_LIMIT, ctrl_set_frame_size_limit }, // Getters { AOMD_GET_FRAME_CORRUPTED, ctrl_get_frame_corrupted },
diff --git a/av1/common/arm/av1_convolve_scale_neon_dotprod.c b/av1/common/arm/av1_convolve_scale_neon_dotprod.c index 1551d3a..340987c 100644 --- a/av1/common/arm/av1_convolve_scale_neon_dotprod.c +++ b/av1/common/arm/av1_convolve_scale_neon_dotprod.c
@@ -28,7 +28,7 @@ #include "av1/common/filter.h" // clang-format off -DECLARE_ALIGNED(16, static const uint8_t, kScale2DotProdPermuteTbl[32]) = { +DECLARE_ALIGNED(16, const uint8_t, kScale2DotProdPermuteTbl[32]) = { 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9, 4, 5, 6, 7, 6, 7, 8, 9, 8, 9, 10, 11, 10, 11, 12, 13 };
diff --git a/av1/common/arm/av1_convolve_scale_neon_dotprod.h b/av1/common/arm/av1_convolve_scale_neon_dotprod.h new file mode 100644 index 0000000..6188124 --- /dev/null +++ b/av1/common/arm/av1_convolve_scale_neon_dotprod.h
@@ -0,0 +1,18 @@ +/* + * Copyright (c) 2025, Alliance for Open Media. All rights reserved. + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef AOM_AV1_COMMON_ARM_AV1_CONVOLVE_SCALE_NEON_DOTPROD_H_ +#define AOM_AV1_COMMON_ARM_AV1_CONVOLVE_SCALE_NEON_DOTPROD_H_ + +#include "aom_ports/mem.h" + +DECLARE_ALIGNED(16, extern const uint8_t, kScale2DotProdPermuteTbl[32]); + +#endif // AOM_AV1_COMMON_ARM_AV1_CONVOLVE_SCALE_NEON_DOTPROD_H_
diff --git a/av1/common/arm/av1_convolve_scale_neon_i8mm.c b/av1/common/arm/av1_convolve_scale_neon_i8mm.c index eb5092b..8ae218bf 100644 --- a/av1/common/arm/av1_convolve_scale_neon_i8mm.c +++ b/av1/common/arm/av1_convolve_scale_neon_i8mm.c
@@ -23,17 +23,11 @@ #include "aom_dsp/arm/transpose_neon.h" #include "aom_ports/mem.h" #include "av1/common/arm/convolve_scale_neon.h" +#include "av1/common/arm/av1_convolve_scale_neon_dotprod.h" #include "av1/common/convolve.h" #include "av1/common/enums.h" #include "av1/common/filter.h" -// clang-format off -DECLARE_ALIGNED(16, static const uint8_t, kScale2DotProdPermuteTbl[32]) = { - 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9, - 4, 5, 6, 7, 6, 7, 8, 9, 8, 9, 10, 11, 10, 11, 12, 13 -}; -// clang-format on - static inline int16x4_t convolve8_4_h(const uint8x8_t s0, const uint8x8_t s1, const uint8x8_t s2, const uint8x8_t s3, const int8x8_t filter,
diff --git a/av1/common/arm/compound_convolve_neon.h b/av1/common/arm/compound_convolve_neon.h index e9837f0..66d0d58 100644 --- a/av1/common/arm/compound_convolve_neon.h +++ b/av1/common/arm/compound_convolve_neon.h
@@ -124,6 +124,46 @@ *d23_u8 = vqrshrun_n_s16(dst_23, FILTER_BITS - ROUND0_BITS); } +static inline void compute_dist_wtd_avg_8x2( + uint16x8_t dd0, uint16x8_t dd1, uint16x8_t d0, uint16x8_t d1, + const uint16_t fwd_offset, const uint16_t bck_offset, + const int16x8_t round_offset, uint8x8_t *d0_u8, uint8x8_t *d1_u8) { + uint32x4_t blend0_lo = vmull_n_u16(vget_low_u16(dd0), fwd_offset); + blend0_lo = vmlal_n_u16(blend0_lo, vget_low_u16(d0), bck_offset); + uint32x4_t blend0_hi = vmull_n_u16(vget_high_u16(dd0), fwd_offset); + blend0_hi = vmlal_n_u16(blend0_hi, vget_high_u16(d0), bck_offset); + + uint32x4_t blend1_lo = vmull_n_u16(vget_low_u16(dd1), fwd_offset); + blend1_lo = vmlal_n_u16(blend1_lo, vget_low_u16(d1), bck_offset); + uint32x4_t blend1_hi = vmull_n_u16(vget_high_u16(dd1), fwd_offset); + blend1_hi = vmlal_n_u16(blend1_hi, vget_high_u16(d1), bck_offset); + + uint16x8_t avg0 = vcombine_u16(vshrn_n_u32(blend0_lo, DIST_PRECISION_BITS), + vshrn_n_u32(blend0_hi, DIST_PRECISION_BITS)); + uint16x8_t avg1 = vcombine_u16(vshrn_n_u32(blend1_lo, DIST_PRECISION_BITS), + vshrn_n_u32(blend1_hi, DIST_PRECISION_BITS)); + + int16x8_t dst0 = vsubq_s16(vreinterpretq_s16_u16(avg0), round_offset); + int16x8_t dst1 = vsubq_s16(vreinterpretq_s16_u16(avg1), round_offset); + + *d0_u8 = vqrshrun_n_s16(dst0, FILTER_BITS - ROUND0_BITS); + *d1_u8 = vqrshrun_n_s16(dst1, FILTER_BITS - ROUND0_BITS); +} + +static inline void compute_basic_avg_8x2(uint16x8_t dd01, uint16x8_t dd23, + uint16x8_t d01, uint16x8_t d23, + const int16x8_t round_offset, + uint8x8_t *d0_u8, uint8x8_t *d1_u8) { + uint16x8_t avg0 = vhaddq_u16(dd01, d01); + uint16x8_t avg1 = vhaddq_u16(dd23, d23); + + int16x8_t dst0 = vsubq_s16(vreinterpretq_s16_u16(avg0), round_offset); + int16x8_t dst1 = vsubq_s16(vreinterpretq_s16_u16(avg1), round_offset); + + *d0_u8 = vqrshrun_n_s16(dst0, FILTER_BITS - ROUND0_BITS); + *d1_u8 = vqrshrun_n_s16(dst1, FILTER_BITS - ROUND0_BITS); +} + static inline void compute_dist_wtd_avg_8x4( uint16x8_t dd0, uint16x8_t dd1, uint16x8_t dd2, uint16x8_t dd3, uint16x8_t d0, uint16x8_t d1, uint16x8_t d2, uint16x8_t d3,
diff --git a/av1/common/arm/compound_convolve_neon_dotprod.c b/av1/common/arm/compound_convolve_neon_dotprod.c index dcf8da1..f4c55de 100644 --- a/av1/common/arm/compound_convolve_neon_dotprod.c +++ b/av1/common/arm/compound_convolve_neon_dotprod.c
@@ -13,16 +13,12 @@ #include <assert.h> #include "aom_dsp/arm/mem_neon.h" +#include "aom_dsp/arm/transpose_neon.h" #include "av1/common/arm/compound_convolve_neon.h" +#include "av1/common/arm/convolve_neon_dotprod.h" #include "config/aom_config.h" #include "config/av1_rtcd.h" -DECLARE_ALIGNED(16, static const uint8_t, dot_prod_permute_tbl[48]) = { - 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6, - 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10, - 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 -}; - static inline int16x4_t convolve4_4_2d_h(uint8x16_t samples, const int8x8_t x_filter, const int32x4_t correction, @@ -97,7 +93,7 @@ int height = im_h; if (w == 4) { - const uint8x16_t permute_tbl = vld1q_u8(dot_prod_permute_tbl); + const uint8x16_t permute_tbl = vld1q_u8(kDotProdPermuteTbl); // 4-tap filters are used for blocks having width <= 4. // Filter values are even, so halve to reduce intermediate precision reqs. const int8x8_t x_filter = @@ -137,7 +133,7 @@ dst_ptr += dst_stride; } while (--height != 0); } else { - const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl); + const uint8x16x3_t permute_tbl = vld1q_u8_x3(kDotProdPermuteTbl); // Filter values are even, so halve to reduce intermediate precision reqs. const int8x8_t x_filter = vshrn_n_s16(x_filter_s16, 1); @@ -349,7 +345,7 @@ int height = h; if (w == 4) { - const uint8x16_t permute_tbl = vld1q_u8(dot_prod_permute_tbl); + const uint8x16_t permute_tbl = vld1q_u8(kDotProdPermuteTbl); // 4-tap filters are used for blocks having width <= 4. // Filter values are even, so halve to reduce intermediate precision reqs. const int8x8_t x_filter = @@ -386,7 +382,7 @@ height -= 4; } while (height != 0); } else { - const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl); + const uint8x16x3_t permute_tbl = vld1q_u8_x3(kDotProdPermuteTbl); // Filter values are even, so halve to reduce intermediate precision reqs. const int8x8_t x_filter = vshrn_n_s16(x_filter_s16, 1); @@ -469,7 +465,7 @@ int height = h; if (w == 4) { - const uint8x16_t permute_tbl = vld1q_u8(dot_prod_permute_tbl); + const uint8x16_t permute_tbl = vld1q_u8(kDotProdPermuteTbl); // 4-tap filters are used for blocks having width <= 4. // Filter values are even, so halve to reduce intermediate precision reqs. const int8x8_t x_filter = @@ -506,7 +502,7 @@ height -= 4; } while (height != 0); } else { - const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl); + const uint8x16x3_t permute_tbl = vld1q_u8_x3(kDotProdPermuteTbl); // Filter values are even, so halve to reduce intermediate precision reqs. const int8x8_t x_filter = vshrn_n_s16(x_filter_s16, 1); @@ -586,7 +582,7 @@ int height = h; if (w == 4) { - const uint8x16_t permute_tbl = vld1q_u8(dot_prod_permute_tbl); + const uint8x16_t permute_tbl = vld1q_u8(kDotProdPermuteTbl); // 4-tap filters are used for blocks having width <= 4. // Filter values are even, so halve to reduce intermediate precision reqs. const int8x8_t x_filter = @@ -614,7 +610,7 @@ height -= 4; } while (height != 0); } else { - const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl); + const uint8x16x3_t permute_tbl = vld1q_u8_x3(kDotProdPermuteTbl); // Filter values are even, so halve to reduce intermediate precision reqs. const int8x8_t x_filter = vshrn_n_s16(x_filter_s16, 1); @@ -668,3 +664,1151 @@ subpel_x_qn, conv_params); } } + +static inline int16x4_t convolve8_4_y(const int8x16_t s0, const int8x16_t s1, + const int8x8_t filters) { + // The sample range transform and permutation are performed by the caller. + // Accumulate into 128 << FILTER_BITS to account for range transform. + // (- 1 since we halved the filters.) + const int32x4_t acc = vdupq_n_s32(128 << (FILTER_BITS - 1)); + int32x4_t sum = vdotq_lane_s32(acc, s0, filters, 0); + sum = vdotq_lane_s32(sum, s1, filters, 1); + + // Further narrowing and packing is performed by the caller. + return vmovn_s32(sum); +} + +static inline uint16x8_t convolve8_8_y(const int8x16_t s0_lo, + const int8x16_t s0_hi, + const int8x16_t s1_lo, + const int8x16_t s1_hi, + const int8x8_t filters, + const int16x8_t round_offset) { + // The sample range transform and permutation are performed by the caller. + // Accumulate into 128 << FILTER_BITS to account for range transform. + // (- 1 since we halved the filters.) + const int32x4_t acc = vdupq_n_s32(128 << (FILTER_BITS - 1)); + + int32x4_t sum0123 = vdotq_lane_s32(acc, s0_lo, filters, 0); + sum0123 = vdotq_lane_s32(sum0123, s1_lo, filters, 1); + + int32x4_t sum4567 = vdotq_lane_s32(acc, s0_hi, filters, 0); + sum4567 = vdotq_lane_s32(sum4567, s1_hi, filters, 1); + + // Narrow and re-pack. + int16x8_t sum = vcombine_s16(vmovn_s32(sum0123), vmovn_s32(sum4567)); + + // We halved the filter values so -1 from right shift. + return vreinterpretq_u16_s16( + vrsraq_n_s16(round_offset, sum, ROUND0_BITS - 1)); +} + +static inline void dist_wtd_convolve_y_8tap_neon_dotprod( + const uint8_t *src_ptr, int src_stride, int w, int h, + const int16_t *y_filter_ptr, ConvolveParams *conv_params) { + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + const int16x8_t round_offset_vec = vdupq_n_s16(round_offset); + + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + + // Filter values are even, so halve to reduce intermediate precision reqs. + const int8x8_t filter = vshrn_n_s16(vld1q_s16(y_filter_ptr), 1); + + const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(kDotProdMergeBlockTbl); + + if (w == 4) { + uint8x8_t t0, t1, t2, t3, t4, t5, t6; + load_u8_8x7(src_ptr, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6); + src_ptr += 7 * src_stride; + + // Transform sample range to [-128, 127] for 8-bit signed dot product. + int8x8_t s0 = vreinterpret_s8_u8(vsub_u8(t0, vdup_n_u8(128))); + int8x8_t s1 = vreinterpret_s8_u8(vsub_u8(t1, vdup_n_u8(128))); + int8x8_t s2 = vreinterpret_s8_u8(vsub_u8(t2, vdup_n_u8(128))); + int8x8_t s3 = vreinterpret_s8_u8(vsub_u8(t3, vdup_n_u8(128))); + int8x8_t s4 = vreinterpret_s8_u8(vsub_u8(t4, vdup_n_u8(128))); + int8x8_t s5 = vreinterpret_s8_u8(vsub_u8(t5, vdup_n_u8(128))); + int8x8_t s6 = vreinterpret_s8_u8(vsub_u8(t6, vdup_n_u8(128))); + + int8x16_t s0123, s1234, s2345, s3456; + transpose_concat_elems_s8_4x4(s0, s1, s2, s3, &s0123); + transpose_concat_elems_s8_4x4(s1, s2, s3, s4, &s1234); + transpose_concat_elems_s8_4x4(s2, s3, s4, s5, &s2345); + transpose_concat_elems_s8_4x4(s3, s4, s5, s6, &s3456); + + do { + uint8x8_t t7, t8, t9, tA; + load_u8_8x4(src_ptr, src_stride, &t7, &t8, &t9, &tA); + + int8x8_t s7 = vreinterpret_s8_u8(vsub_u8(t7, vdup_n_u8(128))); + int8x8_t s8 = vreinterpret_s8_u8(vsub_u8(t8, vdup_n_u8(128))); + int8x8_t s9 = vreinterpret_s8_u8(vsub_u8(t9, vdup_n_u8(128))); + int8x8_t sA = vreinterpret_s8_u8(vsub_u8(tA, vdup_n_u8(128))); + + int8x16_t s4567, s5678, s6789, s789A; + transpose_concat_elems_s8_4x4(s7, s8, s9, sA, &s789A); + + // Merge new data into block from previous iteration. + int8x16x2_t samples_LUT = { { s3456, s789A } }; + s4567 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]); + s5678 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]); + s6789 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]); + + int16x4_t d0 = convolve8_4_y(s0123, s4567, filter); + int16x4_t d1 = convolve8_4_y(s1234, s5678, filter); + int16x4_t d2 = convolve8_4_y(s2345, s6789, filter); + int16x4_t d3 = convolve8_4_y(s3456, s789A, filter); + + // We halved the filter values so -1 from right shift. + int16x8_t d01 = + vrsraq_n_s16(round_offset_vec, vcombine_s16(d0, d1), ROUND0_BITS - 1); + int16x8_t d23 = + vrsraq_n_s16(round_offset_vec, vcombine_s16(d2, d3), ROUND0_BITS - 1); + + store_u16x4_strided_x2(dst_ptr + 0 * dst_stride, dst_stride, + vreinterpretq_u16_s16(d01)); + store_u16x4_strided_x2(dst_ptr + 2 * dst_stride, dst_stride, + vreinterpretq_u16_s16(d23)); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123 = s4567; + s1234 = s5678; + s2345 = s6789; + s3456 = s789A; + + src_ptr += 4 * src_stride; + dst_ptr += 4 * dst_stride; + h -= 4; + } while (h != 0); + } else { + do { + int height = h; + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + + uint8x8_t t0, t1, t2, t3, t4, t5, t6; + load_u8_8x7(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6); + s += 7 * src_stride; + + // Transform sample range to [-128, 127] for 8-bit signed dot product. + int8x8_t s0 = vreinterpret_s8_u8(vsub_u8(t0, vdup_n_u8(128))); + int8x8_t s1 = vreinterpret_s8_u8(vsub_u8(t1, vdup_n_u8(128))); + int8x8_t s2 = vreinterpret_s8_u8(vsub_u8(t2, vdup_n_u8(128))); + int8x8_t s3 = vreinterpret_s8_u8(vsub_u8(t3, vdup_n_u8(128))); + int8x8_t s4 = vreinterpret_s8_u8(vsub_u8(t4, vdup_n_u8(128))); + int8x8_t s5 = vreinterpret_s8_u8(vsub_u8(t5, vdup_n_u8(128))); + int8x8_t s6 = vreinterpret_s8_u8(vsub_u8(t6, vdup_n_u8(128))); + + // This operation combines a conventional transpose and the sample + // permute (see horizontal case) required before computing the dot + // product. + int8x16_t s0123_lo, s0123_hi, s1234_lo, s1234_hi, s2345_lo, s2345_hi, + s3456_lo, s3456_hi; + transpose_concat_elems_s8_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi); + transpose_concat_elems_s8_8x4(s1, s2, s3, s4, &s1234_lo, &s1234_hi); + transpose_concat_elems_s8_8x4(s2, s3, s4, s5, &s2345_lo, &s2345_hi); + transpose_concat_elems_s8_8x4(s3, s4, s5, s6, &s3456_lo, &s3456_hi); + + do { + uint8x8_t t7, t8, t9, tA; + load_u8_8x4(s, src_stride, &t7, &t8, &t9, &tA); + + int8x8_t s7 = vreinterpret_s8_u8(vsub_u8(t7, vdup_n_u8(128))); + int8x8_t s8 = vreinterpret_s8_u8(vsub_u8(t8, vdup_n_u8(128))); + int8x8_t s9 = vreinterpret_s8_u8(vsub_u8(t9, vdup_n_u8(128))); + int8x8_t sA = vreinterpret_s8_u8(vsub_u8(tA, vdup_n_u8(128))); + + int8x16_t s4567_lo, s4567_hi, s5678_lo, s5678_hi, s6789_lo, s6789_hi, + s789A_lo, s789A_hi; + transpose_concat_elems_s8_8x4(s7, s8, s9, sA, &s789A_lo, &s789A_hi); + + // Merge new data into block from previous iteration. + int8x16x2_t samples_LUT_lo = { { s3456_lo, s789A_lo } }; + s4567_lo = vqtbl2q_s8(samples_LUT_lo, merge_block_tbl.val[0]); + s5678_lo = vqtbl2q_s8(samples_LUT_lo, merge_block_tbl.val[1]); + s6789_lo = vqtbl2q_s8(samples_LUT_lo, merge_block_tbl.val[2]); + + int8x16x2_t samples_LUT_hi = { { s3456_hi, s789A_hi } }; + s4567_hi = vqtbl2q_s8(samples_LUT_hi, merge_block_tbl.val[0]); + s5678_hi = vqtbl2q_s8(samples_LUT_hi, merge_block_tbl.val[1]); + s6789_hi = vqtbl2q_s8(samples_LUT_hi, merge_block_tbl.val[2]); + + uint16x8_t d0 = convolve8_8_y(s0123_lo, s0123_hi, s4567_lo, s4567_hi, + filter, round_offset_vec); + uint16x8_t d1 = convolve8_8_y(s1234_lo, s1234_hi, s5678_lo, s5678_hi, + filter, round_offset_vec); + uint16x8_t d2 = convolve8_8_y(s2345_lo, s2345_hi, s6789_lo, s6789_hi, + filter, round_offset_vec); + uint16x8_t d3 = convolve8_8_y(s3456_lo, s3456_hi, s789A_lo, s789A_hi, + filter, round_offset_vec); + + store_u16_8x4(d, dst_stride, d0, d1, d2, d3); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123_lo = s4567_lo; + s0123_hi = s4567_hi; + s1234_lo = s5678_lo; + s1234_hi = s5678_hi; + s2345_lo = s6789_lo; + s2345_hi = s6789_hi; + s3456_lo = s789A_lo; + s3456_hi = s789A_hi; + + s += 4 * src_stride; + d += 4 * dst_stride; + height -= 4; + } while (height != 0); + src_ptr += 8; + dst_ptr += 8; + w -= 8; + } while (w != 0); + } +} + +static inline void dist_wtd_convolve_y_8tap_dist_wtd_avg_neon_dotprod( + const uint8_t *src_ptr, int src_stride, uint8_t *dst8_ptr, + const int dst8_stride, int w, int h, const int16_t *y_filter_ptr, + ConvolveParams *conv_params) { + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + const int16x8_t round_offset_vec = vdupq_n_s16(round_offset); + + const uint16_t fwd_offset = conv_params->fwd_offset; + const uint16_t bck_offset = conv_params->bck_offset; + + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + + // Filter values are even, so halve to reduce intermediate precision reqs. + const int8x8_t filter = vshrn_n_s16(vld1q_s16(y_filter_ptr), 1); + + const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(kDotProdMergeBlockTbl); + + if (w == 4) { + uint8x8_t t0, t1, t2, t3, t4, t5, t6; + load_u8_8x7(src_ptr, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6); + src_ptr += 7 * src_stride; + + // Transform sample range to [-128, 127] for 8-bit signed dot product. + int8x8_t s0 = vreinterpret_s8_u8(vsub_u8(t0, vdup_n_u8(128))); + int8x8_t s1 = vreinterpret_s8_u8(vsub_u8(t1, vdup_n_u8(128))); + int8x8_t s2 = vreinterpret_s8_u8(vsub_u8(t2, vdup_n_u8(128))); + int8x8_t s3 = vreinterpret_s8_u8(vsub_u8(t3, vdup_n_u8(128))); + int8x8_t s4 = vreinterpret_s8_u8(vsub_u8(t4, vdup_n_u8(128))); + int8x8_t s5 = vreinterpret_s8_u8(vsub_u8(t5, vdup_n_u8(128))); + int8x8_t s6 = vreinterpret_s8_u8(vsub_u8(t6, vdup_n_u8(128))); + + int8x16_t s0123, s1234, s2345, s3456; + transpose_concat_elems_s8_4x4(s0, s1, s2, s3, &s0123); + transpose_concat_elems_s8_4x4(s1, s2, s3, s4, &s1234); + transpose_concat_elems_s8_4x4(s2, s3, s4, s5, &s2345); + transpose_concat_elems_s8_4x4(s3, s4, s5, s6, &s3456); + + do { + uint8x8_t t7, t8, t9, tA; + load_u8_8x4(src_ptr, src_stride, &t7, &t8, &t9, &tA); + + int8x8_t s7 = vreinterpret_s8_u8(vsub_u8(t7, vdup_n_u8(128))); + int8x8_t s8 = vreinterpret_s8_u8(vsub_u8(t8, vdup_n_u8(128))); + int8x8_t s9 = vreinterpret_s8_u8(vsub_u8(t9, vdup_n_u8(128))); + int8x8_t sA = vreinterpret_s8_u8(vsub_u8(tA, vdup_n_u8(128))); + + int8x16_t s4567, s5678, s6789, s789A; + transpose_concat_elems_s8_4x4(s7, s8, s9, sA, &s789A); + + // Merge new data into block from previous iteration. + int8x16x2_t samples_LUT = { { s3456, s789A } }; + s4567 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]); + s5678 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]); + s6789 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]); + + int16x4_t d0 = convolve8_4_y(s0123, s4567, filter); + int16x4_t d1 = convolve8_4_y(s1234, s5678, filter); + int16x4_t d2 = convolve8_4_y(s2345, s6789, filter); + int16x4_t d3 = convolve8_4_y(s3456, s789A, filter); + + // We halved the filter values so -1 from right shift. + uint16x8_t d01 = vreinterpretq_u16_s16(vrsraq_n_s16( + round_offset_vec, vcombine_s16(d0, d1), ROUND0_BITS - 1)); + uint16x8_t d23 = vreinterpretq_u16_s16(vrsraq_n_s16( + round_offset_vec, vcombine_s16(d2, d3), ROUND0_BITS - 1)); + + uint16x4_t dd0, dd1, dd2, dd3; + load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3); + + uint8x8_t d0_u8, d1_u8; + compute_dist_wtd_avg_8x2(vcombine_u16(dd0, dd1), vcombine_u16(dd2, dd3), + d01, d23, fwd_offset, bck_offset, + round_offset_vec, &d0_u8, &d1_u8); + + store_u8x4_strided_x2(dst8_ptr + 0 * dst8_stride, dst8_stride, d0_u8); + store_u8x4_strided_x2(dst8_ptr + 2 * dst8_stride, dst8_stride, d1_u8); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123 = s4567; + s1234 = s5678; + s2345 = s6789; + s3456 = s789A; + + src_ptr += 4 * src_stride; + dst_ptr += 4 * dst_stride; + dst8_ptr += 4 * dst8_stride; + h -= 4; + } while (h != 0); + } else { + do { + int height = h; + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + uint8_t *d_u8 = dst8_ptr; + + uint8x8_t t0, t1, t2, t3, t4, t5, t6; + load_u8_8x7(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6); + s += 7 * src_stride; + + // Transform sample range to [-128, 127] for 8-bit signed dot product. + int8x8_t s0 = vreinterpret_s8_u8(vsub_u8(t0, vdup_n_u8(128))); + int8x8_t s1 = vreinterpret_s8_u8(vsub_u8(t1, vdup_n_u8(128))); + int8x8_t s2 = vreinterpret_s8_u8(vsub_u8(t2, vdup_n_u8(128))); + int8x8_t s3 = vreinterpret_s8_u8(vsub_u8(t3, vdup_n_u8(128))); + int8x8_t s4 = vreinterpret_s8_u8(vsub_u8(t4, vdup_n_u8(128))); + int8x8_t s5 = vreinterpret_s8_u8(vsub_u8(t5, vdup_n_u8(128))); + int8x8_t s6 = vreinterpret_s8_u8(vsub_u8(t6, vdup_n_u8(128))); + + // This operation combines a conventional transpose and the sample + // permute (see horizontal case) required before computing the dot + // product. + int8x16_t s0123_lo, s0123_hi, s1234_lo, s1234_hi, s2345_lo, s2345_hi, + s3456_lo, s3456_hi; + transpose_concat_elems_s8_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi); + transpose_concat_elems_s8_8x4(s1, s2, s3, s4, &s1234_lo, &s1234_hi); + transpose_concat_elems_s8_8x4(s2, s3, s4, s5, &s2345_lo, &s2345_hi); + transpose_concat_elems_s8_8x4(s3, s4, s5, s6, &s3456_lo, &s3456_hi); + + do { + uint8x8_t t7, t8, t9, tA; + load_u8_8x4(s, src_stride, &t7, &t8, &t9, &tA); + + int8x8_t s7 = vreinterpret_s8_u8(vsub_u8(t7, vdup_n_u8(128))); + int8x8_t s8 = vreinterpret_s8_u8(vsub_u8(t8, vdup_n_u8(128))); + int8x8_t s9 = vreinterpret_s8_u8(vsub_u8(t9, vdup_n_u8(128))); + int8x8_t sA = vreinterpret_s8_u8(vsub_u8(tA, vdup_n_u8(128))); + + int8x16_t s4567_lo, s4567_hi, s5678_lo, s5678_hi, s6789_lo, s6789_hi, + s789A_lo, s789A_hi; + transpose_concat_elems_s8_8x4(s7, s8, s9, sA, &s789A_lo, &s789A_hi); + + // Merge new data into block from previous iteration. + int8x16x2_t samples_LUT_lo = { { s3456_lo, s789A_lo } }; + s4567_lo = vqtbl2q_s8(samples_LUT_lo, merge_block_tbl.val[0]); + s5678_lo = vqtbl2q_s8(samples_LUT_lo, merge_block_tbl.val[1]); + s6789_lo = vqtbl2q_s8(samples_LUT_lo, merge_block_tbl.val[2]); + + int8x16x2_t samples_LUT_hi = { { s3456_hi, s789A_hi } }; + s4567_hi = vqtbl2q_s8(samples_LUT_hi, merge_block_tbl.val[0]); + s5678_hi = vqtbl2q_s8(samples_LUT_hi, merge_block_tbl.val[1]); + s6789_hi = vqtbl2q_s8(samples_LUT_hi, merge_block_tbl.val[2]); + + uint16x8_t d0 = convolve8_8_y(s0123_lo, s0123_hi, s4567_lo, s4567_hi, + filter, round_offset_vec); + uint16x8_t d1 = convolve8_8_y(s1234_lo, s1234_hi, s5678_lo, s5678_hi, + filter, round_offset_vec); + uint16x8_t d2 = convolve8_8_y(s2345_lo, s2345_hi, s6789_lo, s6789_hi, + filter, round_offset_vec); + uint16x8_t d3 = convolve8_8_y(s3456_lo, s3456_hi, s789A_lo, s789A_hi, + filter, round_offset_vec); + + uint16x8_t dd0, dd1, dd2, dd3; + load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3); + + uint8x8_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_dist_wtd_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset, + bck_offset, round_offset_vec, &d0_u8, &d1_u8, + &d2_u8, &d3_u8); + + store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123_lo = s4567_lo; + s0123_hi = s4567_hi; + s1234_lo = s5678_lo; + s1234_hi = s5678_hi; + s2345_lo = s6789_lo; + s2345_hi = s6789_hi; + s3456_lo = s789A_lo; + s3456_hi = s789A_hi; + + s += 4 * src_stride; + d += 4 * dst_stride; + d_u8 += 4 * dst8_stride; + height -= 4; + } while (height != 0); + src_ptr += 8; + dst_ptr += 8; + dst8_ptr += 8; + w -= 8; + } while (w != 0); + } +} + +static inline void dist_wtd_convolve_y_8tap_avg_neon_dotprod( + const uint8_t *src_ptr, int src_stride, uint8_t *dst8_ptr, + const int dst8_stride, int w, int h, const int16_t *y_filter_ptr, + ConvolveParams *conv_params) { + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + const int16x8_t round_offset_vec = vdupq_n_s16(round_offset); + + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + + // Filter values are even, so halve to reduce intermediate precision reqs. + const int8x8_t filter = vshrn_n_s16(vld1q_s16(y_filter_ptr), 1); + + const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(kDotProdMergeBlockTbl); + + if (w == 4) { + uint8x8_t t0, t1, t2, t3, t4, t5, t6; + load_u8_8x7(src_ptr, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6); + src_ptr += 7 * src_stride; + + // Transform sample range to [-128, 127] for 8-bit signed dot product. + int8x8_t s0 = vreinterpret_s8_u8(vsub_u8(t0, vdup_n_u8(128))); + int8x8_t s1 = vreinterpret_s8_u8(vsub_u8(t1, vdup_n_u8(128))); + int8x8_t s2 = vreinterpret_s8_u8(vsub_u8(t2, vdup_n_u8(128))); + int8x8_t s3 = vreinterpret_s8_u8(vsub_u8(t3, vdup_n_u8(128))); + int8x8_t s4 = vreinterpret_s8_u8(vsub_u8(t4, vdup_n_u8(128))); + int8x8_t s5 = vreinterpret_s8_u8(vsub_u8(t5, vdup_n_u8(128))); + int8x8_t s6 = vreinterpret_s8_u8(vsub_u8(t6, vdup_n_u8(128))); + + int8x16_t s0123, s1234, s2345, s3456; + transpose_concat_elems_s8_4x4(s0, s1, s2, s3, &s0123); + transpose_concat_elems_s8_4x4(s1, s2, s3, s4, &s1234); + transpose_concat_elems_s8_4x4(s2, s3, s4, s5, &s2345); + transpose_concat_elems_s8_4x4(s3, s4, s5, s6, &s3456); + + do { + uint8x8_t t7, t8, t9, tA; + load_u8_8x4(src_ptr, src_stride, &t7, &t8, &t9, &tA); + + int8x8_t s7 = vreinterpret_s8_u8(vsub_u8(t7, vdup_n_u8(128))); + int8x8_t s8 = vreinterpret_s8_u8(vsub_u8(t8, vdup_n_u8(128))); + int8x8_t s9 = vreinterpret_s8_u8(vsub_u8(t9, vdup_n_u8(128))); + int8x8_t sA = vreinterpret_s8_u8(vsub_u8(tA, vdup_n_u8(128))); + + int8x16_t s4567, s5678, s6789, s789A; + transpose_concat_elems_s8_4x4(s7, s8, s9, sA, &s789A); + + // Merge new data into block from previous iteration. + int8x16x2_t samples_LUT = { { s3456, s789A } }; + s4567 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]); + s5678 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]); + s6789 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]); + + int16x4_t d0 = convolve8_4_y(s0123, s4567, filter); + int16x4_t d1 = convolve8_4_y(s1234, s5678, filter); + int16x4_t d2 = convolve8_4_y(s2345, s6789, filter); + int16x4_t d3 = convolve8_4_y(s3456, s789A, filter); + + // We halved the filter values so -1 from right shift. + uint16x8_t d01 = vreinterpretq_u16_s16(vrsraq_n_s16( + round_offset_vec, vcombine_s16(d0, d1), ROUND0_BITS - 1)); + uint16x8_t d23 = vreinterpretq_u16_s16(vrsraq_n_s16( + round_offset_vec, vcombine_s16(d2, d3), ROUND0_BITS - 1)); + + uint16x4_t dd0, dd1, dd2, dd3; + load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3); + + uint8x8_t d0_u8, d1_u8; + compute_basic_avg_8x2(vcombine_u16(dd0, dd1), vcombine_u16(dd2, dd3), d01, + d23, round_offset_vec, &d0_u8, &d1_u8); + + store_u8x4_strided_x2(dst8_ptr + 0 * dst8_stride, dst8_stride, d0_u8); + store_u8x4_strided_x2(dst8_ptr + 2 * dst8_stride, dst8_stride, d1_u8); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123 = s4567; + s1234 = s5678; + s2345 = s6789; + s3456 = s789A; + + src_ptr += 4 * src_stride; + dst_ptr += 4 * dst_stride; + dst8_ptr += 4 * dst8_stride; + h -= 4; + } while (h != 0); + } else { + do { + int height = h; + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + uint8_t *d_u8 = dst8_ptr; + + uint8x8_t t0, t1, t2, t3, t4, t5, t6; + load_u8_8x7(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6); + s += 7 * src_stride; + + // Transform sample range to [-128, 127] for 8-bit signed dot product. + int8x8_t s0 = vreinterpret_s8_u8(vsub_u8(t0, vdup_n_u8(128))); + int8x8_t s1 = vreinterpret_s8_u8(vsub_u8(t1, vdup_n_u8(128))); + int8x8_t s2 = vreinterpret_s8_u8(vsub_u8(t2, vdup_n_u8(128))); + int8x8_t s3 = vreinterpret_s8_u8(vsub_u8(t3, vdup_n_u8(128))); + int8x8_t s4 = vreinterpret_s8_u8(vsub_u8(t4, vdup_n_u8(128))); + int8x8_t s5 = vreinterpret_s8_u8(vsub_u8(t5, vdup_n_u8(128))); + int8x8_t s6 = vreinterpret_s8_u8(vsub_u8(t6, vdup_n_u8(128))); + + // This operation combines a conventional transpose and the sample + // permute (see horizontal case) required before computing the dot + // product. + int8x16_t s0123_lo, s0123_hi, s1234_lo, s1234_hi, s2345_lo, s2345_hi, + s3456_lo, s3456_hi; + transpose_concat_elems_s8_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi); + transpose_concat_elems_s8_8x4(s1, s2, s3, s4, &s1234_lo, &s1234_hi); + transpose_concat_elems_s8_8x4(s2, s3, s4, s5, &s2345_lo, &s2345_hi); + transpose_concat_elems_s8_8x4(s3, s4, s5, s6, &s3456_lo, &s3456_hi); + + do { + uint8x8_t t7, t8, t9, tA; + load_u8_8x4(s, src_stride, &t7, &t8, &t9, &tA); + + int8x8_t s7 = vreinterpret_s8_u8(vsub_u8(t7, vdup_n_u8(128))); + int8x8_t s8 = vreinterpret_s8_u8(vsub_u8(t8, vdup_n_u8(128))); + int8x8_t s9 = vreinterpret_s8_u8(vsub_u8(t9, vdup_n_u8(128))); + int8x8_t sA = vreinterpret_s8_u8(vsub_u8(tA, vdup_n_u8(128))); + + int8x16_t s4567_lo, s4567_hi, s5678_lo, s5678_hi, s6789_lo, s6789_hi, + s789A_lo, s789A_hi; + transpose_concat_elems_s8_8x4(s7, s8, s9, sA, &s789A_lo, &s789A_hi); + + // Merge new data into block from previous iteration. + int8x16x2_t samples_LUT_lo = { { s3456_lo, s789A_lo } }; + s4567_lo = vqtbl2q_s8(samples_LUT_lo, merge_block_tbl.val[0]); + s5678_lo = vqtbl2q_s8(samples_LUT_lo, merge_block_tbl.val[1]); + s6789_lo = vqtbl2q_s8(samples_LUT_lo, merge_block_tbl.val[2]); + + int8x16x2_t samples_LUT_hi = { { s3456_hi, s789A_hi } }; + s4567_hi = vqtbl2q_s8(samples_LUT_hi, merge_block_tbl.val[0]); + s5678_hi = vqtbl2q_s8(samples_LUT_hi, merge_block_tbl.val[1]); + s6789_hi = vqtbl2q_s8(samples_LUT_hi, merge_block_tbl.val[2]); + + uint16x8_t d0 = convolve8_8_y(s0123_lo, s0123_hi, s4567_lo, s4567_hi, + filter, round_offset_vec); + uint16x8_t d1 = convolve8_8_y(s1234_lo, s1234_hi, s5678_lo, s5678_hi, + filter, round_offset_vec); + uint16x8_t d2 = convolve8_8_y(s2345_lo, s2345_hi, s6789_lo, s6789_hi, + filter, round_offset_vec); + uint16x8_t d3 = convolve8_8_y(s3456_lo, s3456_hi, s789A_lo, s789A_hi, + filter, round_offset_vec); + + uint16x8_t dd0, dd1, dd2, dd3; + load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3); + + uint8x8_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_basic_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, + round_offset_vec, &d0_u8, &d1_u8, &d2_u8, &d3_u8); + + store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123_lo = s4567_lo; + s0123_hi = s4567_hi; + s1234_lo = s5678_lo; + s1234_hi = s5678_hi; + s2345_lo = s6789_lo; + s2345_hi = s6789_hi; + s3456_lo = s789A_lo; + s3456_hi = s789A_hi; + + s += 4 * src_stride; + d += 4 * dst_stride; + d_u8 += 4 * dst8_stride; + height -= 4; + } while (height != 0); + src_ptr += 8; + dst_ptr += 8; + dst8_ptr += 8; + w -= 8; + } while (w != 0); + } +} + +static inline int16x4_t convolve4_4_y(const int8x16_t s0, + const int8x8_t filters) { + // The sample range transform and permutation are performed by the caller. + // Accumulate into 128 << FILTER_BITS to account for range transform. + // (- 1 since we halved the filters.) + const int32x4_t acc = vdupq_n_s32(128 << (FILTER_BITS - 1)); + + int32x4_t sum = vdotq_lane_s32(acc, s0, filters, 0); + + // Further narrowing and packing is performed by the caller. + return vmovn_s32(sum); +} + +static inline uint16x8_t convolve4_8_y(const int8x16_t s0, const int8x16_t s1, + const int8x8_t filters, + const int16x8_t round_offset) { + // The sample range transform and permutation are performed by the caller. + // Accumulate into 128 << FILTER_BITS to account for range transform. + // (- 1 since we halved the filters.) + const int32x4_t acc = vdupq_n_s32(128 << (FILTER_BITS - 1)); + + int32x4_t sum0123 = vdotq_lane_s32(acc, s0, filters, 0); + int32x4_t sum4567 = vdotq_lane_s32(acc, s1, filters, 0); + + // Narrow and re-pack. + int16x8_t sum = vcombine_s16(vmovn_s32(sum0123), vmovn_s32(sum4567)); + + // We halved the filter values so -1 from right shift. + return vreinterpretq_u16_s16( + vrsraq_n_s16(round_offset, sum, ROUND0_BITS - 1)); +} + +static inline void dist_wtd_convolve_y_4tap_neon_dotprod( + const uint8_t *src_ptr, int src_stride, int w, int h, + const int16_t *y_filter_ptr, ConvolveParams *conv_params) { + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + const int16x8_t round_offset_vec = vdupq_n_s16(round_offset); + + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + + // Filter values are even, so halve to reduce intermediate precision reqs. + const int16x8_t filter_s16 = + vcombine_s16(vld1_s16(y_filter_ptr + 2), vdup_n_s16(0)); + const int8x8_t filter = vshrn_n_s16(filter_s16, 1); + const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(kDotProdMergeBlockTbl); + int8x16x2_t samples_LUT; + + if (w == 4) { + uint8x8_t t0, t1, t2, t3; + load_u8_8x4(src_ptr, src_stride, &t0, &t1, &t2, &t3); + src_ptr += 4 * src_stride; + + // Transform sample range to [-128, 127] for 8-bit signed dot product. + int8x8_t s0 = vreinterpret_s8_u8(vsub_u8(t0, vdup_n_u8(128))); + int8x8_t s1 = vreinterpret_s8_u8(vsub_u8(t1, vdup_n_u8(128))); + int8x8_t s2 = vreinterpret_s8_u8(vsub_u8(t2, vdup_n_u8(128))); + int8x8_t s3 = vreinterpret_s8_u8(vsub_u8(t3, vdup_n_u8(128))); + + // This operation combines a conventional transpose and the sample permute + // required before computing the dot product. + int8x16_t s0123; + transpose_concat_elems_s8_4x4(s0, s1, s2, s3, &s0123); + + do { + uint8x8_t t4, t5, t6, t7; + load_u8_8x4(src_ptr, src_stride, &t4, &t5, &t6, &t7); + + int8x8_t s4 = vreinterpret_s8_u8(vsub_u8(t4, vdup_n_u8(128))); + int8x8_t s5 = vreinterpret_s8_u8(vsub_u8(t5, vdup_n_u8(128))); + int8x8_t s6 = vreinterpret_s8_u8(vsub_u8(t6, vdup_n_u8(128))); + int8x8_t s7 = vreinterpret_s8_u8(vsub_u8(t7, vdup_n_u8(128))); + + int8x16_t s4567; + transpose_concat_elems_s8_4x4(s4, s5, s6, s7, &s4567); + + // Merge new data into block from previous iteration. + samples_LUT.val[0] = s0123; + samples_LUT.val[1] = s4567; + int8x16_t s1234 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]); + int8x16_t s2345 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]); + int8x16_t s3456 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]); + + int16x4_t d0 = convolve4_4_y(s0123, filter); + int16x4_t d1 = convolve4_4_y(s1234, filter); + int16x4_t d2 = convolve4_4_y(s2345, filter); + int16x4_t d3 = convolve4_4_y(s3456, filter); + + // We halved the filter values so -1 from right shift. + int16x8_t d01 = + vrsraq_n_s16(round_offset_vec, vcombine_s16(d0, d1), ROUND0_BITS - 1); + int16x8_t d23 = + vrsraq_n_s16(round_offset_vec, vcombine_s16(d2, d3), ROUND0_BITS - 1); + + store_u16x4_strided_x2(dst_ptr + 0 * dst_stride, dst_stride, + vreinterpretq_u16_s16(d01)); + store_u16x4_strided_x2(dst_ptr + 2 * dst_stride, dst_stride, + vreinterpretq_u16_s16(d23)); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123 = s4567; + + src_ptr += 4 * src_stride; + dst_ptr += 4 * dst_stride; + h -= 4; + } while (h != 0); + } else { + do { + int height = h; + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + + uint8x8_t t0, t1, t2, t3; + load_u8_8x4(s, src_stride, &t0, &t1, &t2, &t3); + s += 4 * src_stride; + + // Transform sample range to [-128, 127] for 8-bit signed dot product. + int8x8_t s0 = vreinterpret_s8_u8(vsub_u8(t0, vdup_n_u8(128))); + int8x8_t s1 = vreinterpret_s8_u8(vsub_u8(t1, vdup_n_u8(128))); + int8x8_t s2 = vreinterpret_s8_u8(vsub_u8(t2, vdup_n_u8(128))); + int8x8_t s3 = vreinterpret_s8_u8(vsub_u8(t3, vdup_n_u8(128))); + + // This operation combines a conventional transpose and the sample permute + // required before computing the dot product. + int8x16_t s0123_lo, s0123_hi; + transpose_concat_elems_s8_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi); + + do { + uint8x8_t t4, t5, t6, t7; + load_u8_8x4(s, src_stride, &t4, &t5, &t6, &t7); + + int8x8_t s4 = vreinterpret_s8_u8(vsub_u8(t4, vdup_n_u8(128))); + int8x8_t s5 = vreinterpret_s8_u8(vsub_u8(t5, vdup_n_u8(128))); + int8x8_t s6 = vreinterpret_s8_u8(vsub_u8(t6, vdup_n_u8(128))); + int8x8_t s7 = vreinterpret_s8_u8(vsub_u8(t7, vdup_n_u8(128))); + + int8x16_t s4567_lo, s4567_hi; + transpose_concat_elems_s8_8x4(s4, s5, s6, s7, &s4567_lo, &s4567_hi); + + // Merge new data into block from previous iteration. + samples_LUT.val[0] = s0123_lo; + samples_LUT.val[1] = s4567_lo; + int8x16_t s1234_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]); + int8x16_t s2345_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]); + int8x16_t s3456_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]); + + samples_LUT.val[0] = s0123_hi; + samples_LUT.val[1] = s4567_hi; + int8x16_t s1234_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]); + int8x16_t s2345_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]); + int8x16_t s3456_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]); + + uint16x8_t d0 = + convolve4_8_y(s0123_lo, s0123_hi, filter, round_offset_vec); + uint16x8_t d1 = + convolve4_8_y(s1234_lo, s1234_hi, filter, round_offset_vec); + uint16x8_t d2 = + convolve4_8_y(s2345_lo, s2345_hi, filter, round_offset_vec); + uint16x8_t d3 = + convolve4_8_y(s3456_lo, s3456_hi, filter, round_offset_vec); + + store_u16_8x4(d, dst_stride, d0, d1, d2, d3); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123_lo = s4567_lo; + s0123_hi = s4567_hi; + + s += 4 * src_stride; + d += 4 * dst_stride; + height -= 4; + } while (height != 0); + src_ptr += 8; + dst_ptr += 8; + w -= 8; + } while (w != 0); + } +} + +static inline void dist_wtd_convolve_y_4tap_avg_neon_dotprod( + const uint8_t *src_ptr, int src_stride, uint8_t *dst8_ptr, + const int dst8_stride, int w, int h, const int16_t *y_filter_ptr, + ConvolveParams *conv_params) { + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + const int16x8_t round_offset_vec = vdupq_n_s16(round_offset); + + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + + // Filter values are even, so halve to reduce intermediate precision reqs. + const int16x8_t filter_s16 = + vcombine_s16(vld1_s16(y_filter_ptr + 2), vdup_n_s16(0)); + const int8x8_t filter = vshrn_n_s16(filter_s16, 1); + const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(kDotProdMergeBlockTbl); + int8x16x2_t samples_LUT; + + if (w == 4) { + uint8x8_t t0, t1, t2, t3; + load_u8_8x4(src_ptr, src_stride, &t0, &t1, &t2, &t3); + src_ptr += 4 * src_stride; + + // Transform sample range to [-128, 127] for 8-bit signed dot product. + int8x8_t s0 = vreinterpret_s8_u8(vsub_u8(t0, vdup_n_u8(128))); + int8x8_t s1 = vreinterpret_s8_u8(vsub_u8(t1, vdup_n_u8(128))); + int8x8_t s2 = vreinterpret_s8_u8(vsub_u8(t2, vdup_n_u8(128))); + int8x8_t s3 = vreinterpret_s8_u8(vsub_u8(t3, vdup_n_u8(128))); + + // This operation combines a conventional transpose and the sample permute + // required before computing the dot product. + int8x16_t s0123; + transpose_concat_elems_s8_4x4(s0, s1, s2, s3, &s0123); + + do { + uint8x8_t t4, t5, t6, t7; + load_u8_8x4(src_ptr, src_stride, &t4, &t5, &t6, &t7); + + int8x8_t s4 = vreinterpret_s8_u8(vsub_u8(t4, vdup_n_u8(128))); + int8x8_t s5 = vreinterpret_s8_u8(vsub_u8(t5, vdup_n_u8(128))); + int8x8_t s6 = vreinterpret_s8_u8(vsub_u8(t6, vdup_n_u8(128))); + int8x8_t s7 = vreinterpret_s8_u8(vsub_u8(t7, vdup_n_u8(128))); + + int8x16_t s4567; + transpose_concat_elems_s8_4x4(s4, s5, s6, s7, &s4567); + + // Merge new data into block from previous iteration. + samples_LUT.val[0] = s0123; + samples_LUT.val[1] = s4567; + int8x16_t s1234 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]); + int8x16_t s2345 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]); + int8x16_t s3456 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]); + + int16x4_t d0 = convolve4_4_y(s0123, filter); + int16x4_t d1 = convolve4_4_y(s1234, filter); + int16x4_t d2 = convolve4_4_y(s2345, filter); + int16x4_t d3 = convolve4_4_y(s3456, filter); + + // We halved the filter values so -1 from right shift. + uint16x8_t d01 = vreinterpretq_u16_s16(vrsraq_n_s16( + round_offset_vec, vcombine_s16(d0, d1), ROUND0_BITS - 1)); + uint16x8_t d23 = vreinterpretq_u16_s16(vrsraq_n_s16( + round_offset_vec, vcombine_s16(d2, d3), ROUND0_BITS - 1)); + + uint16x4_t dd0, dd1, dd2, dd3; + load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3); + + uint8x8_t d0_u8, d1_u8; + compute_basic_avg_8x2(vcombine_u16(dd0, dd1), vcombine_u16(dd2, dd3), d01, + d23, round_offset_vec, &d0_u8, &d1_u8); + + store_u8x4_strided_x2(dst8_ptr + 0 * dst8_stride, dst8_stride, d0_u8); + store_u8x4_strided_x2(dst8_ptr + 2 * dst8_stride, dst8_stride, d1_u8); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123 = s4567; + + src_ptr += 4 * src_stride; + dst_ptr += 4 * dst_stride; + dst8_ptr += 4 * dst8_stride; + h -= 4; + } while (h != 0); + } else { + do { + int height = h; + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + uint8_t *d_u8 = dst8_ptr; + + uint8x8_t t0, t1, t2, t3; + load_u8_8x4(s, src_stride, &t0, &t1, &t2, &t3); + s += 4 * src_stride; + + // Transform sample range to [-128, 127] for 8-bit signed dot product. + int8x8_t s0 = vreinterpret_s8_u8(vsub_u8(t0, vdup_n_u8(128))); + int8x8_t s1 = vreinterpret_s8_u8(vsub_u8(t1, vdup_n_u8(128))); + int8x8_t s2 = vreinterpret_s8_u8(vsub_u8(t2, vdup_n_u8(128))); + int8x8_t s3 = vreinterpret_s8_u8(vsub_u8(t3, vdup_n_u8(128))); + + // This operation combines a conventional transpose and the sample permute + // required before computing the dot product. + int8x16_t s0123_lo, s0123_hi; + transpose_concat_elems_s8_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi); + + do { + uint8x8_t t4, t5, t6, t7; + load_u8_8x4(s, src_stride, &t4, &t5, &t6, &t7); + + int8x8_t s4 = vreinterpret_s8_u8(vsub_u8(t4, vdup_n_u8(128))); + int8x8_t s5 = vreinterpret_s8_u8(vsub_u8(t5, vdup_n_u8(128))); + int8x8_t s6 = vreinterpret_s8_u8(vsub_u8(t6, vdup_n_u8(128))); + int8x8_t s7 = vreinterpret_s8_u8(vsub_u8(t7, vdup_n_u8(128))); + + int8x16_t s4567_lo, s4567_hi; + transpose_concat_elems_s8_8x4(s4, s5, s6, s7, &s4567_lo, &s4567_hi); + + // Merge new data into block from previous iteration. + samples_LUT.val[0] = s0123_lo; + samples_LUT.val[1] = s4567_lo; + int8x16_t s1234_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]); + int8x16_t s2345_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]); + int8x16_t s3456_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]); + + samples_LUT.val[0] = s0123_hi; + samples_LUT.val[1] = s4567_hi; + int8x16_t s1234_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]); + int8x16_t s2345_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]); + int8x16_t s3456_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]); + + uint16x8_t d0 = + convolve4_8_y(s0123_lo, s0123_hi, filter, round_offset_vec); + uint16x8_t d1 = + convolve4_8_y(s1234_lo, s1234_hi, filter, round_offset_vec); + uint16x8_t d2 = + convolve4_8_y(s2345_lo, s2345_hi, filter, round_offset_vec); + uint16x8_t d3 = + convolve4_8_y(s3456_lo, s3456_hi, filter, round_offset_vec); + + uint16x8_t dd0, dd1, dd2, dd3; + load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3); + + uint8x8_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_basic_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, + round_offset_vec, &d0_u8, &d1_u8, &d2_u8, &d3_u8); + + store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123_lo = s4567_lo; + s0123_hi = s4567_hi; + + s += 4 * src_stride; + d += 4 * dst_stride; + d_u8 += 4 * dst8_stride; + height -= 4; + } while (height != 0); + src_ptr += 8; + dst_ptr += 8; + dst8_ptr += 8; + w -= 8; + } while (w != 0); + } +} + +static inline void dist_wtd_convolve_y_4tap_dist_wtd_avg_neon_dotprod( + const uint8_t *src_ptr, int src_stride, uint8_t *dst8_ptr, + const int dst8_stride, int w, int h, const int16_t *y_filter_ptr, + ConvolveParams *conv_params) { + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + const int16x8_t round_offset_vec = vdupq_n_s16(round_offset); + + const uint16_t fwd_offset = conv_params->fwd_offset; + const uint16_t bck_offset = conv_params->bck_offset; + + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + + // Filter values are even, so halve to reduce intermediate precision reqs. + const int16x8_t filter_s16 = + vcombine_s16(vld1_s16(y_filter_ptr + 2), vdup_n_s16(0)); + const int8x8_t filter = vshrn_n_s16(filter_s16, 1); + const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(kDotProdMergeBlockTbl); + int8x16x2_t samples_LUT; + + if (w == 4) { + uint8x8_t t0, t1, t2, t3; + load_u8_8x4(src_ptr, src_stride, &t0, &t1, &t2, &t3); + src_ptr += 4 * src_stride; + + // Transform sample range to [-128, 127] for 8-bit signed dot product. + int8x8_t s0 = vreinterpret_s8_u8(vsub_u8(t0, vdup_n_u8(128))); + int8x8_t s1 = vreinterpret_s8_u8(vsub_u8(t1, vdup_n_u8(128))); + int8x8_t s2 = vreinterpret_s8_u8(vsub_u8(t2, vdup_n_u8(128))); + int8x8_t s3 = vreinterpret_s8_u8(vsub_u8(t3, vdup_n_u8(128))); + + // This operation combines a conventional transpose and the sample permute + // required before computing the dot product. + int8x16_t s0123; + transpose_concat_elems_s8_4x4(s0, s1, s2, s3, &s0123); + + do { + uint8x8_t t4, t5, t6, t7; + load_u8_8x4(src_ptr, src_stride, &t4, &t5, &t6, &t7); + + int8x8_t s4 = vreinterpret_s8_u8(vsub_u8(t4, vdup_n_u8(128))); + int8x8_t s5 = vreinterpret_s8_u8(vsub_u8(t5, vdup_n_u8(128))); + int8x8_t s6 = vreinterpret_s8_u8(vsub_u8(t6, vdup_n_u8(128))); + int8x8_t s7 = vreinterpret_s8_u8(vsub_u8(t7, vdup_n_u8(128))); + + int8x16_t s4567; + transpose_concat_elems_s8_4x4(s4, s5, s6, s7, &s4567); + + // Merge new data into block from previous iteration. + samples_LUT.val[0] = s0123; + samples_LUT.val[1] = s4567; + int8x16_t s1234 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]); + int8x16_t s2345 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]); + int8x16_t s3456 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]); + + int16x4_t d0 = convolve4_4_y(s0123, filter); + int16x4_t d1 = convolve4_4_y(s1234, filter); + int16x4_t d2 = convolve4_4_y(s2345, filter); + int16x4_t d3 = convolve4_4_y(s3456, filter); + + // We halved the filter values so -1 from right shift. + uint16x8_t d01 = vreinterpretq_u16_s16(vrsraq_n_s16( + round_offset_vec, vcombine_s16(d0, d1), ROUND0_BITS - 1)); + uint16x8_t d23 = vreinterpretq_u16_s16(vrsraq_n_s16( + round_offset_vec, vcombine_s16(d2, d3), ROUND0_BITS - 1)); + + uint16x4_t dd0, dd1, dd2, dd3; + load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3); + + uint8x8_t d0_u8, d1_u8; + compute_dist_wtd_avg_8x2(vcombine_u16(dd0, dd1), vcombine_u16(dd2, dd3), + d01, d23, fwd_offset, bck_offset, + round_offset_vec, &d0_u8, &d1_u8); + + store_u8x4_strided_x2(dst8_ptr + 0 * dst8_stride, dst8_stride, d0_u8); + store_u8x4_strided_x2(dst8_ptr + 2 * dst8_stride, dst8_stride, d1_u8); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123 = s4567; + + src_ptr += 4 * src_stride; + dst_ptr += 4 * dst_stride; + dst8_ptr += 4 * dst8_stride; + h -= 4; + } while (h != 0); + } else { + do { + int height = h; + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + uint8_t *d_u8 = dst8_ptr; + + uint8x8_t t0, t1, t2, t3; + load_u8_8x4(s, src_stride, &t0, &t1, &t2, &t3); + s += 4 * src_stride; + + // Transform sample range to [-128, 127] for 8-bit signed dot product. + int8x8_t s0 = vreinterpret_s8_u8(vsub_u8(t0, vdup_n_u8(128))); + int8x8_t s1 = vreinterpret_s8_u8(vsub_u8(t1, vdup_n_u8(128))); + int8x8_t s2 = vreinterpret_s8_u8(vsub_u8(t2, vdup_n_u8(128))); + int8x8_t s3 = vreinterpret_s8_u8(vsub_u8(t3, vdup_n_u8(128))); + + // This operation combines a conventional transpose and the sample permute + // required before computing the dot product. + int8x16_t s0123_lo, s0123_hi; + transpose_concat_elems_s8_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi); + + do { + uint8x8_t t4, t5, t6, t7; + load_u8_8x4(s, src_stride, &t4, &t5, &t6, &t7); + + int8x8_t s4 = vreinterpret_s8_u8(vsub_u8(t4, vdup_n_u8(128))); + int8x8_t s5 = vreinterpret_s8_u8(vsub_u8(t5, vdup_n_u8(128))); + int8x8_t s6 = vreinterpret_s8_u8(vsub_u8(t6, vdup_n_u8(128))); + int8x8_t s7 = vreinterpret_s8_u8(vsub_u8(t7, vdup_n_u8(128))); + + int8x16_t s4567_lo, s4567_hi; + transpose_concat_elems_s8_8x4(s4, s5, s6, s7, &s4567_lo, &s4567_hi); + + // Merge new data into block from previous iteration. + samples_LUT.val[0] = s0123_lo; + samples_LUT.val[1] = s4567_lo; + int8x16_t s1234_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]); + int8x16_t s2345_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]); + int8x16_t s3456_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]); + + samples_LUT.val[0] = s0123_hi; + samples_LUT.val[1] = s4567_hi; + int8x16_t s1234_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]); + int8x16_t s2345_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]); + int8x16_t s3456_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]); + + uint16x8_t d0 = + convolve4_8_y(s0123_lo, s0123_hi, filter, round_offset_vec); + uint16x8_t d1 = + convolve4_8_y(s1234_lo, s1234_hi, filter, round_offset_vec); + uint16x8_t d2 = + convolve4_8_y(s2345_lo, s2345_hi, filter, round_offset_vec); + uint16x8_t d3 = + convolve4_8_y(s3456_lo, s3456_hi, filter, round_offset_vec); + + uint16x8_t dd0, dd1, dd2, dd3; + load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3); + + uint8x8_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_dist_wtd_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset, + bck_offset, round_offset_vec, &d0_u8, &d1_u8, + &d2_u8, &d3_u8); + + store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123_lo = s4567_lo; + s0123_hi = s4567_hi; + + s += 4 * src_stride; + d += 4 * dst_stride; + d_u8 += 4 * dst8_stride; + height -= 4; + } while (height != 0); + src_ptr += 8; + dst_ptr += 8; + dst8_ptr += 8; + w -= 8; + } while (w != 0); + } +} + +void av1_dist_wtd_convolve_y_neon_dotprod( + const uint8_t *src, int src_stride, uint8_t *dst8, int dst8_stride, int w, + int h, const InterpFilterParams *filter_params_y, const int subpel_y_qn, + ConvolveParams *conv_params) { + assert(w % 4 == 0); + assert(h % 4 == 0); + + const int16_t *y_filter_ptr = av1_get_interp_filter_subpel_kernel( + filter_params_y, subpel_y_qn & SUBPEL_MASK); + + if (get_filter_tap(filter_params_y, subpel_y_qn) <= 4) { + if (conv_params->do_average) { + if (UNLIKELY(conv_params->use_dist_wtd_comp_avg)) { + dist_wtd_convolve_y_4tap_dist_wtd_avg_neon_dotprod( + src - src_stride, src_stride, dst8, dst8_stride, w, h, y_filter_ptr, + conv_params); + } else { + dist_wtd_convolve_y_4tap_avg_neon_dotprod(src - src_stride, src_stride, + dst8, dst8_stride, w, h, + y_filter_ptr, conv_params); + } + } else { + dist_wtd_convolve_y_4tap_neon_dotprod(src - src_stride, src_stride, w, h, + y_filter_ptr, conv_params); + } + } else { // filter tap >= 6 + if (conv_params->do_average) { + if (UNLIKELY(conv_params->use_dist_wtd_comp_avg)) { + dist_wtd_convolve_y_8tap_dist_wtd_avg_neon_dotprod( + src - 3 * src_stride, src_stride, dst8, dst8_stride, w, h, + y_filter_ptr, conv_params); + } else { + dist_wtd_convolve_y_8tap_avg_neon_dotprod( + src - 3 * src_stride, src_stride, dst8, dst8_stride, w, h, + y_filter_ptr, conv_params); + } + } else { + dist_wtd_convolve_y_8tap_neon_dotprod(src - 3 * src_stride, src_stride, w, + h, y_filter_ptr, conv_params); + } + } +}
diff --git a/av1/common/arm/compound_convolve_neon_i8mm.c b/av1/common/arm/compound_convolve_neon_i8mm.c index 0589dfb..aa4438a 100644 --- a/av1/common/arm/compound_convolve_neon_i8mm.c +++ b/av1/common/arm/compound_convolve_neon_i8mm.c
@@ -13,22 +13,12 @@ #include <assert.h> #include "aom_dsp/arm/mem_neon.h" +#include "aom_dsp/arm/transpose_neon.h" #include "av1/common/arm/compound_convolve_neon.h" #include "config/aom_config.h" #include "config/av1_rtcd.h" - -DECLARE_ALIGNED(16, static const uint8_t, kDotProdPermuteTbl[48]) = { - 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6, - 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10, - 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 -}; - -DECLARE_ALIGNED(16, static const uint8_t, kMatMulPermuteTbl[32]) = { - // clang-format off - 0, 1, 2, 3, 4, 5, 6, 7, 2, 3, 4, 5, 6, 7, 8, 9, - 4, 5, 6, 7, 8, 9, 10, 11, 6, 7, 8, 9, 10, 11, 12, 13 - // clang-format on -}; +#include "convolve_neon_dotprod.h" +#include "convolve_neon_i8mm.h" static inline int16x4_t convolve6_4_2d_h(uint8x16_t samples, const int8x16_t x_filter, @@ -90,7 +80,7 @@ int height = im_h; if (w == 4) { - const uint8x16_t permute_tbl = vld1q_u8(kMatMulPermuteTbl); + const uint8x16_t permute_tbl = vld1q_u8(kMatMul6PermuteTbl); do { uint8x16_t s0, s1, s2, s3; load_u8_16x4(src_ptr, src_stride, &s0, &s1, &s2, &s3); @@ -118,7 +108,7 @@ dst_ptr += dst_stride; } while (--height != 0); } else { - const uint8x16x2_t permute_tbl = vld1q_u8_x2(kMatMulPermuteTbl); + const uint8x16x2_t permute_tbl = vld1q_u8_x2(kMatMul6PermuteTbl); do { const uint8_t *s = src_ptr; int16_t *d = dst_ptr; @@ -167,31 +157,29 @@ } static inline int16x8_t convolve8_8_2d_h(uint8x16_t samples, - const int8x8_t x_filter, - const uint8x16x3_t permute_tbl, - const int32x4_t horiz_const) { - uint8x16_t permuted_samples[3]; - int32x4_t sum[2]; + const int8x16_t x_filter, + const uint8x8_t f0, + const uint8x16x2_t permute_tbl, + const uint16x8_t horiz_const) { + // Permute samples ready for matrix multiply. + // { 1, 2, 3, 4, 5, 6, 7, 8, 3, 4, 5, 6, 7, 8, 9, 10 } + // { 5, 6, 7, 8, 9, 10, 11, 12, 7, 8, 9, 10, 11, 12, 13, 14 } + uint8x16_t perm_samples[2] = { vqtbl1q_u8(samples, permute_tbl.val[0]), + vqtbl1q_u8(samples, permute_tbl.val[1]) }; - // Permute samples ready for dot product. - // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } - permuted_samples[0] = vqtbl1q_u8(samples, permute_tbl.val[0]); - // { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } - permuted_samples[1] = vqtbl1q_u8(samples, permute_tbl.val[1]); - // { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 } - permuted_samples[2] = vqtbl1q_u8(samples, permute_tbl.val[2]); + // Calculate partial 7-tap convolution. + int32x4_t sum0123 = vusmmlaq_s32(vdupq_n_s32(0), perm_samples[0], x_filter); + int32x4_t sum4567 = vusmmlaq_s32(vdupq_n_s32(0), perm_samples[1], x_filter); + uint16x8_t sum = vreinterpretq_u16_s16( + vcombine_s16(vmovn_s32(sum0123), vmovn_s32(sum4567))); - // First 4 output values. - sum[0] = vusdotq_lane_s32(horiz_const, permuted_samples[0], x_filter, 0); - sum[0] = vusdotq_lane_s32(sum[0], permuted_samples[1], x_filter, 1); - // Second 4 output values. - sum[1] = vusdotq_lane_s32(horiz_const, permuted_samples[1], x_filter, 0); - sum[1] = vusdotq_lane_s32(sum[1], permuted_samples[2], x_filter, 1); + // Apply tap 0 and accumulate. + sum = vmlsl_u8(sum, vget_low_u8(samples), f0); - // Narrow and re-pack. + sum = vaddq_u16(sum, horiz_const); + // We halved the convolution filter values so -1 from the right shift. - return vcombine_s16(vshrn_n_s32(sum[0], ROUND0_BITS - 1), - vshrn_n_s32(sum[1], ROUND0_BITS - 1)); + return vreinterpretq_s16_u16(vshrq_n_u16(sum, ROUND0_BITS - 1)); } static inline void dist_wtd_convolve_2d_horiz_8tap_neon_i8mm( @@ -201,12 +189,23 @@ // A shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-rounding // shifts - which are generally faster than rounding shifts on modern CPUs. // (The extra -1 is needed because we halved the filter values.) - const int32x4_t horiz_const = vdupq_n_s32((1 << (bd + FILTER_BITS - 2)) + - (1 << ((ROUND0_BITS - 1) - 1))); + const uint16x8_t horiz_const = vdupq_n_u16((1 << (bd + FILTER_BITS - 2)) + + (1 << ((ROUND0_BITS - 1) - 1))); - const uint8x16x3_t permute_tbl = vld1q_u8_x3(kDotProdPermuteTbl); + const uint8x16x2_t permute_tbl = vld1q_u8_x2(kMatMul8PermuteTbl); + // Filter values are even, so halve to reduce intermediate precision reqs. - const int8x8_t x_filter = vshrn_n_s16(vld1q_s16(x_filter_ptr), 1); + const int8x8_t x_filter_s8 = vshrn_n_s16(vld1q_s16(x_filter_ptr), 1); + + // Stagger the filter for use with the matrix multiply instructions. + // { f1, f2, f3, f4, f5, f6, f7, 0, 0, f1, f2, f3, f4, f5, f6, f7 } + const uint8x16_t filter_idx = vld1q_u8(kFilterPermuteTbl); + const int8x16_t x_filter = + vqtbl1q_s8(vcombine_s8(x_filter_s8, vdup_n_s8(0)), filter_idx); + + // Since f0 is always negative and s0 is unsigned, subtract (unsigned) s0 * + // -f0 to avoid signed overflow. + const uint8x8_t f0 = vdup_n_u8(-x_filter_ptr[0] >> 1); const uint8_t *src_ptr = src; int16_t *dst_ptr = im_block; @@ -222,10 +221,14 @@ uint8x16_t s0, s1, s2, s3; load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3); - int16x8_t d0 = convolve8_8_2d_h(s0, x_filter, permute_tbl, horiz_const); - int16x8_t d1 = convolve8_8_2d_h(s1, x_filter, permute_tbl, horiz_const); - int16x8_t d2 = convolve8_8_2d_h(s2, x_filter, permute_tbl, horiz_const); - int16x8_t d3 = convolve8_8_2d_h(s3, x_filter, permute_tbl, horiz_const); + int16x8_t d0 = + convolve8_8_2d_h(s0, x_filter, f0, permute_tbl, horiz_const); + int16x8_t d1 = + convolve8_8_2d_h(s1, x_filter, f0, permute_tbl, horiz_const); + int16x8_t d2 = + convolve8_8_2d_h(s2, x_filter, f0, permute_tbl, horiz_const); + int16x8_t d3 = + convolve8_8_2d_h(s3, x_filter, f0, permute_tbl, horiz_const); store_s16_8x4(d, dst_stride, d0, d1, d2, d3); @@ -246,7 +249,8 @@ do { uint8x16_t s0 = vld1q_u8(s); - int16x8_t d0 = convolve8_8_2d_h(s0, x_filter, permute_tbl, horiz_const); + int16x8_t d0 = + convolve8_8_2d_h(s0, x_filter, f0, permute_tbl, horiz_const); vst1q_s16(d, d0); @@ -366,10 +370,10 @@ return vreinterpretq_u16_s16(res); } -static inline uint16x8_t convolve8_8_x(uint8x16_t samples, - const int8x8_t x_filter, - const uint8x16x3_t permute_tbl, - const int32x4_t round_offset) { +static inline uint16x8_t convolve8_8_x_usdot(uint8x16_t samples, + const int8x8_t x_filter, + const uint8x16x3_t permute_tbl, + const int32x4_t round_offset) { uint8x16_t permuted_samples[3]; int32x4_t sum[2]; @@ -395,6 +399,32 @@ return vreinterpretq_u16_s16(res); } +static inline uint16x8_t convolve8_8_x_usmmla(uint8x16_t samples, + const int8x16_t x_filter, + const uint8x8_t f0, + const uint8x16x2_t permute_tbl, + const uint16x8_t horiz_const) { + // Permute samples ready for matrix multiply. + // { 1, 2, 3, 4, 5, 6, 7, 8, 3, 4, 5, 6, 7, 8, 9, 10 } + // { 5, 6, 7, 8, 9, 10, 11, 12, 7, 8, 9, 10, 11, 12, 13, 14 } + uint8x16_t perm_samples[2] = { vqtbl1q_u8(samples, permute_tbl.val[0]), + vqtbl1q_u8(samples, permute_tbl.val[1]) }; + + // Calculate partial 7-tap convolution. + int32x4_t sum0123 = vusmmlaq_s32(vdupq_n_s32(0), perm_samples[0], x_filter); + int32x4_t sum4567 = vusmmlaq_s32(vdupq_n_s32(0), perm_samples[1], x_filter); + uint16x8_t sum = vreinterpretq_u16_s16( + vcombine_s16(vmovn_s32(sum0123), vmovn_s32(sum4567))); + + // Apply tap 0 and accumulate. + sum = vmlsl_u8(sum, vget_low_u8(samples), f0); + + sum = vaddq_u16(sum, horiz_const); + + // We halved the convolution filter values so -1 from the right shift. + return vshrq_n_u16(sum, ROUND0_BITS - 1); +} + static inline void dist_wtd_convolve_x_dist_wtd_avg_6tap_neon_i8mm( const uint8_t *src, int src_stride, uint16_t *dst, int dst_stride, uint8_t *dst8, int dst8_stride, int w, int h, const int16_t *x_filter_ptr, @@ -421,7 +451,7 @@ vcombine_s8(vext_s8(x_filter_s8, x_filter_s8, 1), x_filter_s8); if (w == 4) { - const uint8x16_t permute_tbl = vld1q_u8(kMatMulPermuteTbl); + const uint8x16_t permute_tbl = vld1q_u8(kMatMul6PermuteTbl); do { uint8x16_t s0, s1, s2, s3; load_u8_16x4(src, src_stride, &s0, &s1, &s2, &s3); @@ -451,7 +481,7 @@ h -= 4; } while (h != 0); } else { - const uint8x16x2_t permute_tbl = vld1q_u8_x2(kMatMulPermuteTbl); + const uint8x16x2_t permute_tbl = vld1q_u8_x2(kMatMul6PermuteTbl); do { const uint8_t *s = src; uint16_t *d = dst; @@ -527,13 +557,13 @@ load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3); uint16x8_t d0 = - convolve8_8_x(s0, x_filter, permute_tbl, round_offset_shim); + convolve8_8_x_usdot(s0, x_filter, permute_tbl, round_offset_shim); uint16x8_t d1 = - convolve8_8_x(s1, x_filter, permute_tbl, round_offset_shim); + convolve8_8_x_usdot(s1, x_filter, permute_tbl, round_offset_shim); uint16x8_t d2 = - convolve8_8_x(s2, x_filter, permute_tbl, round_offset_shim); + convolve8_8_x_usdot(s2, x_filter, permute_tbl, round_offset_shim); uint16x8_t d3 = - convolve8_8_x(s3, x_filter, permute_tbl, round_offset_shim); + convolve8_8_x_usdot(s3, x_filter, permute_tbl, round_offset_shim); uint16x8_t dd0, dd1, dd2, dd3; load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3); @@ -582,7 +612,7 @@ vcombine_s8(vext_s8(x_filter_s8, x_filter_s8, 1), x_filter_s8); if (w == 4) { - const uint8x16_t permute_tbl = vld1q_u8(kMatMulPermuteTbl); + const uint8x16_t permute_tbl = vld1q_u8(kMatMul6PermuteTbl); do { uint8x16_t s0, s1, s2, s3; load_u8_16x4(src, src_stride, &s0, &s1, &s2, &s3); @@ -612,7 +642,7 @@ h -= 4; } while (h != 0); } else { - const uint8x16x2_t permute_tbl = vld1q_u8_x2(kMatMulPermuteTbl); + const uint8x16x2_t permute_tbl = vld1q_u8_x2(kMatMul6PermuteTbl); do { const uint8_t *s = src; uint16_t *d = dst; @@ -668,12 +698,23 @@ // A shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-rounding // shifts - which are generally faster than rounding shifts on modern CPUs. // (The extra -1 is needed because we halved the filter values.) - const int32x4_t round_offset_shim = vdupq_n_s32( + const uint16x8_t round_offset_shim = vdupq_n_u16( (round_offset << (ROUND0_BITS - 1)) + (1 << ((ROUND0_BITS - 1) - 1))); - const uint8x16x3_t permute_tbl = vld1q_u8_x3(kDotProdPermuteTbl); + const uint8x16x2_t permute_tbl = vld1q_u8_x2(kMatMul8PermuteTbl); + // Filter values are even, so halve to reduce intermediate precision reqs. - const int8x8_t x_filter = vshrn_n_s16(vld1q_s16(x_filter_ptr), 1); + const int8x8_t x_filter_s8 = vshrn_n_s16(vld1q_s16(x_filter_ptr), 1); + + // Stagger the filter for use with the matrix multiply instructions. + // { f1, f2, f3, f4, f5, f6, f7, 0, 0, f1, f2, f3, f4, f5, f6, f7 } + const uint8x16_t filter_idx = vld1q_u8(kFilterPermuteTbl); + const int8x16_t x_filter = + vqtbl1q_s8(vcombine_s8(x_filter_s8, vdup_n_s8(0)), filter_idx); + + // Since f0 is always negative and s0 is unsigned, subtract (unsigned) s0 * + // -f0 to avoid signed overflow. + const uint8x8_t f0 = vdup_n_u8(-x_filter_ptr[0] >> 1); do { const uint8_t *s = src; @@ -685,14 +726,14 @@ uint8x16_t s0, s1, s2, s3; load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3); - uint16x8_t d0 = - convolve8_8_x(s0, x_filter, permute_tbl, round_offset_shim); - uint16x8_t d1 = - convolve8_8_x(s1, x_filter, permute_tbl, round_offset_shim); - uint16x8_t d2 = - convolve8_8_x(s2, x_filter, permute_tbl, round_offset_shim); - uint16x8_t d3 = - convolve8_8_x(s3, x_filter, permute_tbl, round_offset_shim); + uint16x8_t d0 = convolve8_8_x_usmmla(s0, x_filter, f0, permute_tbl, + round_offset_shim); + uint16x8_t d1 = convolve8_8_x_usmmla(s1, x_filter, f0, permute_tbl, + round_offset_shim); + uint16x8_t d2 = convolve8_8_x_usmmla(s2, x_filter, f0, permute_tbl, + round_offset_shim); + uint16x8_t d3 = convolve8_8_x_usmmla(s3, x_filter, f0, permute_tbl, + round_offset_shim); uint16x8_t dd0, dd1, dd2, dd3; load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3); @@ -739,7 +780,7 @@ vcombine_s8(vext_s8(x_filter_s8, x_filter_s8, 1), x_filter_s8); if (w == 4) { - const uint8x16_t permute_tbl = vld1q_u8(kMatMulPermuteTbl); + const uint8x16_t permute_tbl = vld1q_u8(kMatMul6PermuteTbl); do { uint8x16_t s0, s1, s2, s3; load_u8_16x4(src, src_stride, &s0, &s1, &s2, &s3); @@ -760,7 +801,7 @@ h -= 4; } while (h != 0); } else { - const uint8x16x2_t permute_tbl = vld1q_u8_x2(kMatMulPermuteTbl); + const uint8x16x2_t permute_tbl = vld1q_u8_x2(kMatMul6PermuteTbl); do { const uint8_t *s = src; uint16_t *d = dst; @@ -805,12 +846,23 @@ // A shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-rounding // shifts - which are generally faster than rounding shifts on modern CPUs. // (The extra -1 is needed because we halved the filter values.) - const int32x4_t round_offset_shim = vdupq_n_s32( + const uint16x8_t round_offset_shim = vdupq_n_u16( (round_offset << (ROUND0_BITS - 1)) + (1 << ((ROUND0_BITS - 1) - 1))); - const uint8x16x3_t permute_tbl = vld1q_u8_x3(kDotProdPermuteTbl); + const uint8x16x2_t permute_tbl = vld1q_u8_x2(kMatMul8PermuteTbl); + // Filter values are even, so halve to reduce intermediate precision reqs. - const int8x8_t x_filter = vshrn_n_s16(vld1q_s16(x_filter_ptr), 1); + const int8x8_t x_filter_s8 = vshrn_n_s16(vld1q_s16(x_filter_ptr), 1); + + // Stagger the filter for use with the matrix multiply instructions. + // { f1, f2, f3, f4, f5, f6, f7, 0, 0, f1, f2, f3, f4, f5, f6, f7 } + const uint8x16_t filter_idx = vld1q_u8(kFilterPermuteTbl); + const int8x16_t x_filter = + vqtbl1q_s8(vcombine_s8(x_filter_s8, vdup_n_s8(0)), filter_idx); + + // Since f0 is always negative and s0 is unsigned, subtract (unsigned) s0 * + // -f0 to avoid signed overflow. + const uint8x8_t f0 = vdup_n_u8(-x_filter_ptr[0] >> 1); do { const uint8_t *s = src; @@ -821,14 +873,14 @@ uint8x16_t s0, s1, s2, s3; load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3); - uint16x8_t d0 = - convolve8_8_x(s0, x_filter, permute_tbl, round_offset_shim); - uint16x8_t d1 = - convolve8_8_x(s1, x_filter, permute_tbl, round_offset_shim); - uint16x8_t d2 = - convolve8_8_x(s2, x_filter, permute_tbl, round_offset_shim); - uint16x8_t d3 = - convolve8_8_x(s3, x_filter, permute_tbl, round_offset_shim); + uint16x8_t d0 = convolve8_8_x_usmmla(s0, x_filter, f0, permute_tbl, + round_offset_shim); + uint16x8_t d1 = convolve8_8_x_usmmla(s1, x_filter, f0, permute_tbl, + round_offset_shim); + uint16x8_t d2 = convolve8_8_x_usmmla(s2, x_filter, f0, permute_tbl, + round_offset_shim); + uint16x8_t d3 = convolve8_8_x_usmmla(s3, x_filter, f0, permute_tbl, + round_offset_shim); store_u16_8x4(d, dst_stride, d0, d1, d2, d3); @@ -892,3 +944,988 @@ x_filter_ptr); } } + +static inline int16x4_t convolve8_4_y(const uint8x16_t s0, const uint8x16_t s1, + const int8x8_t filters) { + int32x4_t sum = vusdotq_lane_s32(vdupq_n_s32(0), s0, filters, 0); + sum = vusdotq_lane_s32(sum, s1, filters, 1); + + // Further narrowing and packing is performed by the caller. + return vmovn_s32(sum); +} + +static inline uint16x8_t convolve8_8_y(const uint8x16_t s0_lo, + const uint8x16_t s0_hi, + const uint8x16_t s1_lo, + const uint8x16_t s1_hi, + const int8x8_t filters, + const int16x8_t round_offset) { + int32x4_t sum0123 = vusdotq_lane_s32(vdupq_n_s32(0), s0_lo, filters, 0); + sum0123 = vusdotq_lane_s32(sum0123, s1_lo, filters, 1); + + int32x4_t sum4567 = vusdotq_lane_s32(vdupq_n_s32(0), s0_hi, filters, 0); + sum4567 = vusdotq_lane_s32(sum4567, s1_hi, filters, 1); + + // Narrow and re-pack. + int16x8_t sum = vcombine_s16(vmovn_s32(sum0123), vmovn_s32(sum4567)); + + // We halved the filter values so -1 from right shift. + return vreinterpretq_u16_s16( + vrsraq_n_s16(round_offset, sum, ROUND0_BITS - 1)); +} + +static inline void dist_wtd_convolve_y_8tap_neon_i8mm( + const uint8_t *src_ptr, int src_stride, int w, int h, + const int16_t *y_filter_ptr, ConvolveParams *conv_params) { + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + const int16x8_t round_offset_vec = vdupq_n_s16(round_offset); + + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + + // Filter values are even, so halve to reduce intermediate precision reqs. + const int8x8_t filter = vshrn_n_s16(vld1q_s16(y_filter_ptr), 1); + + const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(kDotProdMergeBlockTbl); + + if (w == 4) { + uint8x8_t s0, s1, s2, s3, s4, s5, s6; + load_u8_8x7(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6); + src_ptr += 7 * src_stride; + + // This operation combines a conventional transpose and the sample permute + // (see horizontal case) required before computing the dot product. + uint8x16_t s0123, s1234, s2345, s3456; + transpose_concat_elems_u8_4x4(s0, s1, s2, s3, &s0123); + transpose_concat_elems_u8_4x4(s1, s2, s3, s4, &s1234); + transpose_concat_elems_u8_4x4(s2, s3, s4, s5, &s2345); + transpose_concat_elems_u8_4x4(s3, s4, s5, s6, &s3456); + + do { + uint8x8_t s7, s8, s9, sA; + load_u8_8x4(src_ptr, src_stride, &s7, &s8, &s9, &sA); + + uint8x16_t s4567, s5678, s6789, s789A; + transpose_concat_elems_u8_4x4(s7, s8, s9, sA, &s789A); + + // Merge new data into block from previous iteration. + uint8x16x2_t samples_LUT = { { s3456, s789A } }; + s4567 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[0]); + s5678 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[1]); + s6789 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[2]); + + int16x4_t d0 = convolve8_4_y(s0123, s4567, filter); + int16x4_t d1 = convolve8_4_y(s1234, s5678, filter); + int16x4_t d2 = convolve8_4_y(s2345, s6789, filter); + int16x4_t d3 = convolve8_4_y(s3456, s789A, filter); + + // We halved the filter values so -1 from right shift. + int16x8_t d01 = + vrsraq_n_s16(round_offset_vec, vcombine_s16(d0, d1), ROUND0_BITS - 1); + int16x8_t d23 = + vrsraq_n_s16(round_offset_vec, vcombine_s16(d2, d3), ROUND0_BITS - 1); + + store_u16x4_strided_x2(dst_ptr + 0 * dst_stride, dst_stride, + vreinterpretq_u16_s16(d01)); + store_u16x4_strided_x2(dst_ptr + 2 * dst_stride, dst_stride, + vreinterpretq_u16_s16(d23)); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123 = s4567; + s1234 = s5678; + s2345 = s6789; + s3456 = s789A; + + src_ptr += 4 * src_stride; + dst_ptr += 4 * dst_stride; + h -= 4; + } while (h != 0); + } else { + do { + int height = h; + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + + uint8x8_t s0, s1, s2, s3, s4, s5, s6; + load_u8_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6); + s += 7 * src_stride; + + // This operation combines a conventional transpose and the sample + // permute (see horizontal case) required before computing the dot + // product. + uint8x16_t s0123_lo, s0123_hi, s1234_lo, s1234_hi, s2345_lo, s2345_hi, + s3456_lo, s3456_hi; + transpose_concat_elems_u8_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi); + transpose_concat_elems_u8_8x4(s1, s2, s3, s4, &s1234_lo, &s1234_hi); + transpose_concat_elems_u8_8x4(s2, s3, s4, s5, &s2345_lo, &s2345_hi); + transpose_concat_elems_u8_8x4(s3, s4, s5, s6, &s3456_lo, &s3456_hi); + + do { + uint8x8_t s7, s8, s9, sA; + load_u8_8x4(s, src_stride, &s7, &s8, &s9, &sA); + + uint8x16_t s4567_lo, s4567_hi, s5678_lo, s5678_hi, s6789_lo, s6789_hi, + s789A_lo, s789A_hi; + transpose_concat_elems_u8_8x4(s7, s8, s9, sA, &s789A_lo, &s789A_hi); + + // Merge new data into block from previous iteration. + uint8x16x2_t samples_LUT_lo = { { s3456_lo, s789A_lo } }; + s4567_lo = vqtbl2q_u8(samples_LUT_lo, merge_block_tbl.val[0]); + s5678_lo = vqtbl2q_u8(samples_LUT_lo, merge_block_tbl.val[1]); + s6789_lo = vqtbl2q_u8(samples_LUT_lo, merge_block_tbl.val[2]); + + uint8x16x2_t samples_LUT_hi = { { s3456_hi, s789A_hi } }; + s4567_hi = vqtbl2q_u8(samples_LUT_hi, merge_block_tbl.val[0]); + s5678_hi = vqtbl2q_u8(samples_LUT_hi, merge_block_tbl.val[1]); + s6789_hi = vqtbl2q_u8(samples_LUT_hi, merge_block_tbl.val[2]); + + uint16x8_t d0 = convolve8_8_y(s0123_lo, s0123_hi, s4567_lo, s4567_hi, + filter, round_offset_vec); + uint16x8_t d1 = convolve8_8_y(s1234_lo, s1234_hi, s5678_lo, s5678_hi, + filter, round_offset_vec); + uint16x8_t d2 = convolve8_8_y(s2345_lo, s2345_hi, s6789_lo, s6789_hi, + filter, round_offset_vec); + uint16x8_t d3 = convolve8_8_y(s3456_lo, s3456_hi, s789A_lo, s789A_hi, + filter, round_offset_vec); + + store_u16_8x4(d, dst_stride, d0, d1, d2, d3); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123_lo = s4567_lo; + s0123_hi = s4567_hi; + s1234_lo = s5678_lo; + s1234_hi = s5678_hi; + s2345_lo = s6789_lo; + s2345_hi = s6789_hi; + s3456_lo = s789A_lo; + s3456_hi = s789A_hi; + + s += 4 * src_stride; + d += 4 * dst_stride; + height -= 4; + } while (height != 0); + src_ptr += 8; + dst_ptr += 8; + w -= 8; + } while (w != 0); + } +} + +static inline void dist_wtd_convolve_y_8tap_dist_wtd_avg_neon_i8mm( + const uint8_t *src_ptr, int src_stride, uint8_t *dst8_ptr, + const int dst8_stride, int w, int h, const int16_t *y_filter_ptr, + ConvolveParams *conv_params) { + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + const int16x8_t round_offset_vec = vdupq_n_s16(round_offset); + + const uint16_t fwd_offset = conv_params->fwd_offset; + const uint16_t bck_offset = conv_params->bck_offset; + + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + + // Filter values are even, so halve to reduce intermediate precision reqs. + const int8x8_t filter = vshrn_n_s16(vld1q_s16(y_filter_ptr), 1); + + const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(kDotProdMergeBlockTbl); + + if (w == 4) { + uint8x8_t s0, s1, s2, s3, s4, s5, s6; + load_u8_8x7(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6); + src_ptr += 7 * src_stride; + + // This operation combines a conventional transpose and the sample permute + // (see horizontal case) required before computing the dot product. + uint8x16_t s0123, s1234, s2345, s3456; + transpose_concat_elems_u8_4x4(s0, s1, s2, s3, &s0123); + transpose_concat_elems_u8_4x4(s1, s2, s3, s4, &s1234); + transpose_concat_elems_u8_4x4(s2, s3, s4, s5, &s2345); + transpose_concat_elems_u8_4x4(s3, s4, s5, s6, &s3456); + + do { + uint8x8_t s7, s8, s9, sA; + load_u8_8x4(src_ptr, src_stride, &s7, &s8, &s9, &sA); + + uint8x16_t s4567, s5678, s6789, s789A; + transpose_concat_elems_u8_4x4(s7, s8, s9, sA, &s789A); + + // Merge new data into block from previous iteration. + uint8x16x2_t samples_LUT = { { s3456, s789A } }; + s4567 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[0]); + s5678 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[1]); + s6789 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[2]); + + int16x4_t d0 = convolve8_4_y(s0123, s4567, filter); + int16x4_t d1 = convolve8_4_y(s1234, s5678, filter); + int16x4_t d2 = convolve8_4_y(s2345, s6789, filter); + int16x4_t d3 = convolve8_4_y(s3456, s789A, filter); + + // We halved the filter values so -1 from right shift. + uint16x8_t d01 = vreinterpretq_u16_s16(vrsraq_n_s16( + round_offset_vec, vcombine_s16(d0, d1), ROUND0_BITS - 1)); + uint16x8_t d23 = vreinterpretq_u16_s16(vrsraq_n_s16( + round_offset_vec, vcombine_s16(d2, d3), ROUND0_BITS - 1)); + + uint16x4_t dd0, dd1, dd2, dd3; + load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3); + + uint8x8_t d0_u8, d1_u8; + compute_dist_wtd_avg_8x2(vcombine_u16(dd0, dd1), vcombine_u16(dd2, dd3), + d01, d23, fwd_offset, bck_offset, + round_offset_vec, &d0_u8, &d1_u8); + + store_u8x4_strided_x2(dst8_ptr + 0 * dst8_stride, dst8_stride, d0_u8); + store_u8x4_strided_x2(dst8_ptr + 2 * dst8_stride, dst8_stride, d1_u8); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123 = s4567; + s1234 = s5678; + s2345 = s6789; + s3456 = s789A; + + src_ptr += 4 * src_stride; + dst_ptr += 4 * dst_stride; + dst8_ptr += 4 * dst8_stride; + h -= 4; + } while (h != 0); + } else { + do { + int height = h; + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + uint8_t *d_u8 = dst8_ptr; + + uint8x8_t s0, s1, s2, s3, s4, s5, s6; + load_u8_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6); + s += 7 * src_stride; + + // This operation combines a conventional transpose and the sample + // permute (see horizontal case) required before computing the dot + // product. + uint8x16_t s0123_lo, s0123_hi, s1234_lo, s1234_hi, s2345_lo, s2345_hi, + s3456_lo, s3456_hi; + transpose_concat_elems_u8_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi); + transpose_concat_elems_u8_8x4(s1, s2, s3, s4, &s1234_lo, &s1234_hi); + transpose_concat_elems_u8_8x4(s2, s3, s4, s5, &s2345_lo, &s2345_hi); + transpose_concat_elems_u8_8x4(s3, s4, s5, s6, &s3456_lo, &s3456_hi); + + do { + uint8x8_t s7, s8, s9, sA; + load_u8_8x4(s, src_stride, &s7, &s8, &s9, &sA); + + uint8x16_t s4567_lo, s4567_hi, s5678_lo, s5678_hi, s6789_lo, s6789_hi, + s789A_lo, s789A_hi; + transpose_concat_elems_u8_8x4(s7, s8, s9, sA, &s789A_lo, &s789A_hi); + + // Merge new data into block from previous iteration. + uint8x16x2_t samples_LUT_lo = { { s3456_lo, s789A_lo } }; + s4567_lo = vqtbl2q_u8(samples_LUT_lo, merge_block_tbl.val[0]); + s5678_lo = vqtbl2q_u8(samples_LUT_lo, merge_block_tbl.val[1]); + s6789_lo = vqtbl2q_u8(samples_LUT_lo, merge_block_tbl.val[2]); + + uint8x16x2_t samples_LUT_hi = { { s3456_hi, s789A_hi } }; + s4567_hi = vqtbl2q_u8(samples_LUT_hi, merge_block_tbl.val[0]); + s5678_hi = vqtbl2q_u8(samples_LUT_hi, merge_block_tbl.val[1]); + s6789_hi = vqtbl2q_u8(samples_LUT_hi, merge_block_tbl.val[2]); + + uint16x8_t d0 = convolve8_8_y(s0123_lo, s0123_hi, s4567_lo, s4567_hi, + filter, round_offset_vec); + uint16x8_t d1 = convolve8_8_y(s1234_lo, s1234_hi, s5678_lo, s5678_hi, + filter, round_offset_vec); + uint16x8_t d2 = convolve8_8_y(s2345_lo, s2345_hi, s6789_lo, s6789_hi, + filter, round_offset_vec); + uint16x8_t d3 = convolve8_8_y(s3456_lo, s3456_hi, s789A_lo, s789A_hi, + filter, round_offset_vec); + + uint16x8_t dd0, dd1, dd2, dd3; + load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3); + + uint8x8_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_dist_wtd_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset, + bck_offset, round_offset_vec, &d0_u8, &d1_u8, + &d2_u8, &d3_u8); + + store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123_lo = s4567_lo; + s0123_hi = s4567_hi; + s1234_lo = s5678_lo; + s1234_hi = s5678_hi; + s2345_lo = s6789_lo; + s2345_hi = s6789_hi; + s3456_lo = s789A_lo; + s3456_hi = s789A_hi; + + s += 4 * src_stride; + d += 4 * dst_stride; + d_u8 += 4 * dst8_stride; + height -= 4; + } while (height != 0); + src_ptr += 8; + dst_ptr += 8; + dst8_ptr += 8; + w -= 8; + } while (w != 0); + } +} + +static inline void dist_wtd_convolve_y_8tap_avg_neon_i8mm( + const uint8_t *src_ptr, int src_stride, uint8_t *dst8_ptr, + const int dst8_stride, int w, int h, const int16_t *y_filter_ptr, + ConvolveParams *conv_params) { + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + const int16x8_t round_offset_vec = vdupq_n_s16(round_offset); + + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + + // Filter values are even, so halve to reduce intermediate precision reqs. + const int8x8_t filter = vshrn_n_s16(vld1q_s16(y_filter_ptr), 1); + + const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(kDotProdMergeBlockTbl); + + if (w == 4) { + uint8x8_t s0, s1, s2, s3, s4, s5, s6; + load_u8_8x7(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6); + src_ptr += 7 * src_stride; + + // This operation combines a conventional transpose and the sample permute + // (see horizontal case) required before computing the dot product. + uint8x16_t s0123, s1234, s2345, s3456; + transpose_concat_elems_u8_4x4(s0, s1, s2, s3, &s0123); + transpose_concat_elems_u8_4x4(s1, s2, s3, s4, &s1234); + transpose_concat_elems_u8_4x4(s2, s3, s4, s5, &s2345); + transpose_concat_elems_u8_4x4(s3, s4, s5, s6, &s3456); + + do { + uint8x8_t s7, s8, s9, sA; + load_u8_8x4(src_ptr, src_stride, &s7, &s8, &s9, &sA); + + uint8x16_t s4567, s5678, s6789, s789A; + transpose_concat_elems_u8_4x4(s7, s8, s9, sA, &s789A); + + // Merge new data into block from previous iteration. + uint8x16x2_t samples_LUT = { { s3456, s789A } }; + s4567 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[0]); + s5678 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[1]); + s6789 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[2]); + + int16x4_t d0 = convolve8_4_y(s0123, s4567, filter); + int16x4_t d1 = convolve8_4_y(s1234, s5678, filter); + int16x4_t d2 = convolve8_4_y(s2345, s6789, filter); + int16x4_t d3 = convolve8_4_y(s3456, s789A, filter); + + // We halved the filter values so -1 from right shift. + uint16x8_t d01 = vreinterpretq_u16_s16(vrsraq_n_s16( + round_offset_vec, vcombine_s16(d0, d1), ROUND0_BITS - 1)); + uint16x8_t d23 = vreinterpretq_u16_s16(vrsraq_n_s16( + round_offset_vec, vcombine_s16(d2, d3), ROUND0_BITS - 1)); + + uint16x4_t dd0, dd1, dd2, dd3; + load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3); + + uint8x8_t d0_u8, d1_u8; + compute_basic_avg_8x2(vcombine_u16(dd0, dd1), vcombine_u16(dd2, dd3), d01, + d23, round_offset_vec, &d0_u8, &d1_u8); + + store_u8x4_strided_x2(dst8_ptr + 0 * dst8_stride, dst8_stride, d0_u8); + store_u8x4_strided_x2(dst8_ptr + 2 * dst8_stride, dst8_stride, d1_u8); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123 = s4567; + s1234 = s5678; + s2345 = s6789; + s3456 = s789A; + + src_ptr += 4 * src_stride; + dst_ptr += 4 * dst_stride; + dst8_ptr += 4 * dst8_stride; + h -= 4; + } while (h != 0); + } else { + do { + int height = h; + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + uint8_t *d_u8 = dst8_ptr; + + uint8x8_t s0, s1, s2, s3, s4, s5, s6; + load_u8_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6); + s += 7 * src_stride; + + // This operation combines a conventional transpose and the sample + // permute (see horizontal case) required before computing the dot + // product. + uint8x16_t s0123_lo, s0123_hi, s1234_lo, s1234_hi, s2345_lo, s2345_hi, + s3456_lo, s3456_hi; + transpose_concat_elems_u8_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi); + transpose_concat_elems_u8_8x4(s1, s2, s3, s4, &s1234_lo, &s1234_hi); + transpose_concat_elems_u8_8x4(s2, s3, s4, s5, &s2345_lo, &s2345_hi); + transpose_concat_elems_u8_8x4(s3, s4, s5, s6, &s3456_lo, &s3456_hi); + + do { + uint8x8_t s7, s8, s9, sA; + load_u8_8x4(s, src_stride, &s7, &s8, &s9, &sA); + + uint8x16_t s4567_lo, s4567_hi, s5678_lo, s5678_hi, s6789_lo, s6789_hi, + s789A_lo, s789A_hi; + transpose_concat_elems_u8_8x4(s7, s8, s9, sA, &s789A_lo, &s789A_hi); + + // Merge new data into block from previous iteration. + uint8x16x2_t samples_LUT_lo = { { s3456_lo, s789A_lo } }; + s4567_lo = vqtbl2q_u8(samples_LUT_lo, merge_block_tbl.val[0]); + s5678_lo = vqtbl2q_u8(samples_LUT_lo, merge_block_tbl.val[1]); + s6789_lo = vqtbl2q_u8(samples_LUT_lo, merge_block_tbl.val[2]); + + uint8x16x2_t samples_LUT_hi = { { s3456_hi, s789A_hi } }; + s4567_hi = vqtbl2q_u8(samples_LUT_hi, merge_block_tbl.val[0]); + s5678_hi = vqtbl2q_u8(samples_LUT_hi, merge_block_tbl.val[1]); + s6789_hi = vqtbl2q_u8(samples_LUT_hi, merge_block_tbl.val[2]); + + uint16x8_t d0 = convolve8_8_y(s0123_lo, s0123_hi, s4567_lo, s4567_hi, + filter, round_offset_vec); + uint16x8_t d1 = convolve8_8_y(s1234_lo, s1234_hi, s5678_lo, s5678_hi, + filter, round_offset_vec); + uint16x8_t d2 = convolve8_8_y(s2345_lo, s2345_hi, s6789_lo, s6789_hi, + filter, round_offset_vec); + uint16x8_t d3 = convolve8_8_y(s3456_lo, s3456_hi, s789A_lo, s789A_hi, + filter, round_offset_vec); + + uint16x8_t dd0, dd1, dd2, dd3; + load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3); + + uint8x8_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_basic_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, + round_offset_vec, &d0_u8, &d1_u8, &d2_u8, &d3_u8); + + store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123_lo = s4567_lo; + s0123_hi = s4567_hi; + s1234_lo = s5678_lo; + s1234_hi = s5678_hi; + s2345_lo = s6789_lo; + s2345_hi = s6789_hi; + s3456_lo = s789A_lo; + s3456_hi = s789A_hi; + + s += 4 * src_stride; + d += 4 * dst_stride; + d_u8 += 4 * dst8_stride; + height -= 4; + } while (height != 0); + src_ptr += 8; + dst_ptr += 8; + dst8_ptr += 8; + w -= 8; + } while (w != 0); + } +} + +static inline int16x4_t convolve4_4_y(const uint8x16_t s0, + const int8x8_t filters) { + int32x4_t sum = vusdotq_lane_s32(vdupq_n_s32(0), s0, filters, 0); + + // Further narrowing and packing is performed by the caller. + return vmovn_s32(sum); +} + +static inline uint16x8_t convolve4_8_y(const uint8x16_t s0, const uint8x16_t s1, + const int8x8_t filters, + const int16x8_t round_offset) { + int32x4_t sum0123 = vusdotq_lane_s32(vdupq_n_s32(0), s0, filters, 0); + int32x4_t sum4567 = vusdotq_lane_s32(vdupq_n_s32(0), s1, filters, 0); + + // Narrow and re-pack. + int16x8_t sum = vcombine_s16(vmovn_s32(sum0123), vmovn_s32(sum4567)); + + // We halved the filter values so -1 from right shift. + return vreinterpretq_u16_s16( + vrsraq_n_s16(round_offset, sum, ROUND0_BITS - 1)); +} + +static inline void dist_wtd_convolve_y_4tap_neon_i8mm( + const uint8_t *src_ptr, int src_stride, int w, int h, + const int16_t *y_filter_ptr, ConvolveParams *conv_params) { + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + const int16x8_t round_offset_vec = vdupq_n_s16(round_offset); + + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + + // Filter values are even, so halve to reduce intermediate precision reqs. + const int16x8_t filter_s16 = + vcombine_s16(vld1_s16(y_filter_ptr + 2), vdup_n_s16(0)); + const int8x8_t filter = vshrn_n_s16(filter_s16, 1); + const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(kDotProdMergeBlockTbl); + uint8x16x2_t samples_LUT; + + if (w == 4) { + uint8x8_t s0, s1, s2, s3; + load_u8_8x4(src_ptr, src_stride, &s0, &s1, &s2, &s3); + src_ptr += 4 * src_stride; + + // This operation combines a conventional transpose and the sample permute + // required before computing the dot product. + uint8x16_t s0123; + transpose_concat_elems_u8_4x4(s0, s1, s2, s3, &s0123); + + do { + uint8x8_t s4, s5, s6, s7; + load_u8_8x4(src_ptr, src_stride, &s4, &s5, &s6, &s7); + + uint8x16_t s4567; + transpose_concat_elems_u8_4x4(s4, s5, s6, s7, &s4567); + + // Merge new data into block from previous iteration. + samples_LUT.val[0] = s0123; + samples_LUT.val[1] = s4567; + uint8x16_t s1234 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[0]); + uint8x16_t s2345 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[1]); + uint8x16_t s3456 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[2]); + + int16x4_t d0 = convolve4_4_y(s0123, filter); + int16x4_t d1 = convolve4_4_y(s1234, filter); + int16x4_t d2 = convolve4_4_y(s2345, filter); + int16x4_t d3 = convolve4_4_y(s3456, filter); + + // We halved the filter values so -1 from right shift. + int16x8_t d01 = + vrsraq_n_s16(round_offset_vec, vcombine_s16(d0, d1), ROUND0_BITS - 1); + int16x8_t d23 = + vrsraq_n_s16(round_offset_vec, vcombine_s16(d2, d3), ROUND0_BITS - 1); + + store_u16x4_strided_x2(dst_ptr + 0 * dst_stride, dst_stride, + vreinterpretq_u16_s16(d01)); + store_u16x4_strided_x2(dst_ptr + 2 * dst_stride, dst_stride, + vreinterpretq_u16_s16(d23)); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123 = s4567; + + src_ptr += 4 * src_stride; + dst_ptr += 4 * dst_stride; + h -= 4; + } while (h != 0); + } else { + do { + int height = h; + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + + uint8x8_t s0, s1, s2, s3; + load_u8_8x4(s, src_stride, &s0, &s1, &s2, &s3); + s += 4 * src_stride; + + // This operation combines a conventional transpose and the sample permute + // required before computing the dot product. + uint8x16_t s0123_lo, s0123_hi; + transpose_concat_elems_u8_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi); + + do { + uint8x8_t s4, s5, s6, s7; + load_u8_8x4(s, src_stride, &s4, &s5, &s6, &s7); + + uint8x16_t s4567_lo, s4567_hi; + transpose_concat_elems_u8_8x4(s4, s5, s6, s7, &s4567_lo, &s4567_hi); + + // Merge new data into block from previous iteration. + samples_LUT.val[0] = s0123_lo; + samples_LUT.val[1] = s4567_lo; + uint8x16_t s1234_lo = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[0]); + uint8x16_t s2345_lo = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[1]); + uint8x16_t s3456_lo = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[2]); + + samples_LUT.val[0] = s0123_hi; + samples_LUT.val[1] = s4567_hi; + uint8x16_t s1234_hi = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[0]); + uint8x16_t s2345_hi = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[1]); + uint8x16_t s3456_hi = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[2]); + + uint16x8_t d0 = + convolve4_8_y(s0123_lo, s0123_hi, filter, round_offset_vec); + uint16x8_t d1 = + convolve4_8_y(s1234_lo, s1234_hi, filter, round_offset_vec); + uint16x8_t d2 = + convolve4_8_y(s2345_lo, s2345_hi, filter, round_offset_vec); + uint16x8_t d3 = + convolve4_8_y(s3456_lo, s3456_hi, filter, round_offset_vec); + + store_u16_8x4(d, dst_stride, d0, d1, d2, d3); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123_lo = s4567_lo; + s0123_hi = s4567_hi; + + s += 4 * src_stride; + d += 4 * dst_stride; + height -= 4; + } while (height != 0); + src_ptr += 8; + dst_ptr += 8; + w -= 8; + } while (w != 0); + } +} + +static inline void dist_wtd_convolve_y_4tap_avg_neon_i8mm( + const uint8_t *src_ptr, int src_stride, uint8_t *dst8_ptr, + const int dst8_stride, int w, int h, const int16_t *y_filter_ptr, + ConvolveParams *conv_params) { + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + const int16x8_t round_offset_vec = vdupq_n_s16(round_offset); + + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + + // Filter values are even, so halve to reduce intermediate precision reqs. + const int16x8_t filter_s16 = + vcombine_s16(vld1_s16(y_filter_ptr + 2), vdup_n_s16(0)); + const int8x8_t filter = vshrn_n_s16(filter_s16, 1); + const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(kDotProdMergeBlockTbl); + uint8x16x2_t samples_LUT; + + if (w == 4) { + uint8x8_t s0, s1, s2, s3; + load_u8_8x4(src_ptr, src_stride, &s0, &s1, &s2, &s3); + src_ptr += 4 * src_stride; + + // This operation combines a conventional transpose and the sample permute + // required before computing the dot product. + uint8x16_t s0123; + transpose_concat_elems_u8_4x4(s0, s1, s2, s3, &s0123); + + do { + uint8x8_t s4, s5, s6, s7; + load_u8_8x4(src_ptr, src_stride, &s4, &s5, &s6, &s7); + + uint8x16_t s4567; + transpose_concat_elems_u8_4x4(s4, s5, s6, s7, &s4567); + + // Merge new data into block from previous iteration. + samples_LUT.val[0] = s0123; + samples_LUT.val[1] = s4567; + uint8x16_t s1234 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[0]); + uint8x16_t s2345 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[1]); + uint8x16_t s3456 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[2]); + + int16x4_t d0 = convolve4_4_y(s0123, filter); + int16x4_t d1 = convolve4_4_y(s1234, filter); + int16x4_t d2 = convolve4_4_y(s2345, filter); + int16x4_t d3 = convolve4_4_y(s3456, filter); + + // We halved the filter values so -1 from right shift. + uint16x8_t d01 = vreinterpretq_u16_s16(vrsraq_n_s16( + round_offset_vec, vcombine_s16(d0, d1), ROUND0_BITS - 1)); + uint16x8_t d23 = vreinterpretq_u16_s16(vrsraq_n_s16( + round_offset_vec, vcombine_s16(d2, d3), ROUND0_BITS - 1)); + + uint16x4_t dd0, dd1, dd2, dd3; + load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3); + + uint8x8_t d0_u8, d1_u8; + compute_basic_avg_8x2(vcombine_u16(dd0, dd1), vcombine_u16(dd2, dd3), d01, + d23, round_offset_vec, &d0_u8, &d1_u8); + + store_u8x4_strided_x2(dst8_ptr + 0 * dst8_stride, dst8_stride, d0_u8); + store_u8x4_strided_x2(dst8_ptr + 2 * dst8_stride, dst8_stride, d1_u8); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123 = s4567; + + src_ptr += 4 * src_stride; + dst_ptr += 4 * dst_stride; + dst8_ptr += 4 * dst8_stride; + h -= 4; + } while (h != 0); + } else { + do { + int height = h; + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + uint8_t *d_u8 = dst8_ptr; + + uint8x8_t s0, s1, s2, s3; + load_u8_8x4(s, src_stride, &s0, &s1, &s2, &s3); + s += 4 * src_stride; + + // This operation combines a conventional transpose and the sample permute + // required before computing the dot product. + uint8x16_t s0123_lo, s0123_hi; + transpose_concat_elems_u8_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi); + + do { + uint8x8_t s4, s5, s6, s7; + load_u8_8x4(s, src_stride, &s4, &s5, &s6, &s7); + + uint8x16_t s4567_lo, s4567_hi; + transpose_concat_elems_u8_8x4(s4, s5, s6, s7, &s4567_lo, &s4567_hi); + + // Merge new data into block from previous iteration. + samples_LUT.val[0] = s0123_lo; + samples_LUT.val[1] = s4567_lo; + uint8x16_t s1234_lo = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[0]); + uint8x16_t s2345_lo = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[1]); + uint8x16_t s3456_lo = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[2]); + + samples_LUT.val[0] = s0123_hi; + samples_LUT.val[1] = s4567_hi; + uint8x16_t s1234_hi = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[0]); + uint8x16_t s2345_hi = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[1]); + uint8x16_t s3456_hi = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[2]); + + uint16x8_t d0 = + convolve4_8_y(s0123_lo, s0123_hi, filter, round_offset_vec); + uint16x8_t d1 = + convolve4_8_y(s1234_lo, s1234_hi, filter, round_offset_vec); + uint16x8_t d2 = + convolve4_8_y(s2345_lo, s2345_hi, filter, round_offset_vec); + uint16x8_t d3 = + convolve4_8_y(s3456_lo, s3456_hi, filter, round_offset_vec); + + uint16x8_t dd0, dd1, dd2, dd3; + load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3); + + uint8x8_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_basic_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, + round_offset_vec, &d0_u8, &d1_u8, &d2_u8, &d3_u8); + + store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123_lo = s4567_lo; + s0123_hi = s4567_hi; + + s += 4 * src_stride; + d += 4 * dst_stride; + d_u8 += 4 * dst8_stride; + height -= 4; + } while (height != 0); + src_ptr += 8; + dst_ptr += 8; + dst8_ptr += 8; + w -= 8; + } while (w != 0); + } +} + +static inline void dist_wtd_convolve_y_4tap_dist_wtd_avg_neon_i8mm( + const uint8_t *src_ptr, int src_stride, uint8_t *dst8_ptr, + const int dst8_stride, int w, int h, const int16_t *y_filter_ptr, + ConvolveParams *conv_params) { + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + const int16x8_t round_offset_vec = vdupq_n_s16(round_offset); + + const uint16_t fwd_offset = conv_params->fwd_offset; + const uint16_t bck_offset = conv_params->bck_offset; + + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + + // Filter values are even, so halve to reduce intermediate precision reqs. + const int16x8_t filter_s16 = + vcombine_s16(vld1_s16(y_filter_ptr + 2), vdup_n_s16(0)); + const int8x8_t filter = vshrn_n_s16(filter_s16, 1); + const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(kDotProdMergeBlockTbl); + uint8x16x2_t samples_LUT; + + if (w == 4) { + uint8x8_t s0, s1, s2, s3; + load_u8_8x4(src_ptr, src_stride, &s0, &s1, &s2, &s3); + src_ptr += 4 * src_stride; + + // This operation combines a conventional transpose and the sample permute + // required before computing the dot product. + uint8x16_t s0123; + transpose_concat_elems_u8_4x4(s0, s1, s2, s3, &s0123); + + do { + uint8x8_t s4, s5, s6, s7; + load_u8_8x4(src_ptr, src_stride, &s4, &s5, &s6, &s7); + + uint8x16_t s4567; + transpose_concat_elems_u8_4x4(s4, s5, s6, s7, &s4567); + + // Merge new data into block from previous iteration. + samples_LUT.val[0] = s0123; + samples_LUT.val[1] = s4567; + uint8x16_t s1234 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[0]); + uint8x16_t s2345 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[1]); + uint8x16_t s3456 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[2]); + + int16x4_t d0 = convolve4_4_y(s0123, filter); + int16x4_t d1 = convolve4_4_y(s1234, filter); + int16x4_t d2 = convolve4_4_y(s2345, filter); + int16x4_t d3 = convolve4_4_y(s3456, filter); + + // We halved the filter values so -1 from right shift. + uint16x8_t d01 = vreinterpretq_u16_s16(vrsraq_n_s16( + round_offset_vec, vcombine_s16(d0, d1), ROUND0_BITS - 1)); + uint16x8_t d23 = vreinterpretq_u16_s16(vrsraq_n_s16( + round_offset_vec, vcombine_s16(d2, d3), ROUND0_BITS - 1)); + + uint16x4_t dd0, dd1, dd2, dd3; + load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3); + + uint8x8_t d0_u8, d1_u8; + compute_dist_wtd_avg_8x2(vcombine_u16(dd0, dd1), vcombine_u16(dd2, dd3), + d01, d23, fwd_offset, bck_offset, + round_offset_vec, &d0_u8, &d1_u8); + + store_u8x4_strided_x2(dst8_ptr + 0 * dst8_stride, dst8_stride, d0_u8); + store_u8x4_strided_x2(dst8_ptr + 2 * dst8_stride, dst8_stride, d1_u8); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123 = s4567; + + src_ptr += 4 * src_stride; + dst_ptr += 4 * dst_stride; + dst8_ptr += 4 * dst8_stride; + h -= 4; + } while (h != 0); + } else { + do { + int height = h; + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + uint8_t *d_u8 = dst8_ptr; + + uint8x8_t s0, s1, s2, s3; + load_u8_8x4(s, src_stride, &s0, &s1, &s2, &s3); + s += 4 * src_stride; + + // This operation combines a conventional transpose and the sample permute + // required before computing the dot product. + uint8x16_t s0123_lo, s0123_hi; + transpose_concat_elems_u8_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi); + + do { + uint8x8_t s4, s5, s6, s7; + load_u8_8x4(s, src_stride, &s4, &s5, &s6, &s7); + + uint8x16_t s4567_lo, s4567_hi; + transpose_concat_elems_u8_8x4(s4, s5, s6, s7, &s4567_lo, &s4567_hi); + + // Merge new data into block from previous iteration. + samples_LUT.val[0] = s0123_lo; + samples_LUT.val[1] = s4567_lo; + uint8x16_t s1234_lo = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[0]); + uint8x16_t s2345_lo = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[1]); + uint8x16_t s3456_lo = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[2]); + + samples_LUT.val[0] = s0123_hi; + samples_LUT.val[1] = s4567_hi; + uint8x16_t s1234_hi = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[0]); + uint8x16_t s2345_hi = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[1]); + uint8x16_t s3456_hi = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[2]); + + uint16x8_t d0 = + convolve4_8_y(s0123_lo, s0123_hi, filter, round_offset_vec); + uint16x8_t d1 = + convolve4_8_y(s1234_lo, s1234_hi, filter, round_offset_vec); + uint16x8_t d2 = + convolve4_8_y(s2345_lo, s2345_hi, filter, round_offset_vec); + uint16x8_t d3 = + convolve4_8_y(s3456_lo, s3456_hi, filter, round_offset_vec); + + uint16x8_t dd0, dd1, dd2, dd3; + load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3); + + uint8x8_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_dist_wtd_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset, + bck_offset, round_offset_vec, &d0_u8, &d1_u8, + &d2_u8, &d3_u8); + + store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123_lo = s4567_lo; + s0123_hi = s4567_hi; + + s += 4 * src_stride; + d += 4 * dst_stride; + d_u8 += 4 * dst8_stride; + height -= 4; + } while (height != 0); + src_ptr += 8; + dst_ptr += 8; + dst8_ptr += 8; + w -= 8; + } while (w != 0); + } +} + +void av1_dist_wtd_convolve_y_neon_i8mm( + const uint8_t *src, int src_stride, uint8_t *dst8, int dst8_stride, int w, + int h, const InterpFilterParams *filter_params_y, const int subpel_y_qn, + ConvolveParams *conv_params) { + assert(w % 4 == 0); + assert(h % 4 == 0); + + const int16_t *y_filter_ptr = av1_get_interp_filter_subpel_kernel( + filter_params_y, subpel_y_qn & SUBPEL_MASK); + + if (get_filter_tap(filter_params_y, subpel_y_qn) <= 4) { + if (conv_params->do_average) { + if (UNLIKELY(conv_params->use_dist_wtd_comp_avg)) { + dist_wtd_convolve_y_4tap_dist_wtd_avg_neon_i8mm( + src - src_stride, src_stride, dst8, dst8_stride, w, h, y_filter_ptr, + conv_params); + } else { + dist_wtd_convolve_y_4tap_avg_neon_i8mm(src - src_stride, src_stride, + dst8, dst8_stride, w, h, + y_filter_ptr, conv_params); + } + } else { + dist_wtd_convolve_y_4tap_neon_i8mm(src - src_stride, src_stride, w, h, + y_filter_ptr, conv_params); + } + } else { // filter tap >= 6 + if (conv_params->do_average) { + if (UNLIKELY(conv_params->use_dist_wtd_comp_avg)) { + dist_wtd_convolve_y_8tap_dist_wtd_avg_neon_i8mm( + src - 3 * src_stride, src_stride, dst8, dst8_stride, w, h, + y_filter_ptr, conv_params); + } else { + dist_wtd_convolve_y_8tap_avg_neon_i8mm(src - 3 * src_stride, src_stride, + dst8, dst8_stride, w, h, + y_filter_ptr, conv_params); + } + } else { + dist_wtd_convolve_y_8tap_neon_i8mm(src - 3 * src_stride, src_stride, w, h, + y_filter_ptr, conv_params); + } + } +}
diff --git a/av1/common/arm/convolve_neon_dotprod.c b/av1/common/arm/convolve_neon_dotprod.c index 8d0d929..35cdf9e 100644 --- a/av1/common/arm/convolve_neon_dotprod.c +++ b/av1/common/arm/convolve_neon_dotprod.c
@@ -16,18 +16,19 @@ #include "aom_dsp/aom_dsp_common.h" #include "aom_dsp/arm/mem_neon.h" +#include "aom_dsp/arm/transpose_neon.h" #include "aom_ports/mem.h" #include "av1/common/arm/convolve_neon.h" #include "av1/common/convolve.h" #include "av1/common/filter.h" -DECLARE_ALIGNED(16, static const uint8_t, kDotProdPermuteTbl[48]) = { +DECLARE_ALIGNED(16, const uint8_t, kDotProdPermuteTbl[48]) = { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6, 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10, 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 }; -DECLARE_ALIGNED(16, static const uint8_t, kDotProdMergeBlockTbl[48]) = { +DECLARE_ALIGNED(16, const uint8_t, kDotProdMergeBlockTbl[48]) = { // Shift left and insert new last column in transposed 4x4 block. 1, 2, 3, 16, 5, 6, 7, 20, 9, 10, 11, 24, 13, 14, 15, 28, // Shift left and insert two new columns in transposed 4x4 block. @@ -64,7 +65,7 @@ sum = vdotq_laneq_s32(sum, perm_samples[1], filter, 1); sum = vdotq_laneq_s32(sum, perm_samples[2], filter, 2); - return vqrshrn_n_s32(sum, FILTER_BITS); + return vshrn_n_s32(sum, 1); } static inline uint8x8_t convolve12_8_x(uint8x16_t samples[2], @@ -105,9 +106,9 @@ sum4567 = vdotq_laneq_s32(sum4567, perm_samples[3], filter, 2); // Narrow and re-pack. - int16x8_t sum_s16 = vcombine_s16(vqrshrn_n_s32(sum0123, FILTER_BITS), - vqrshrn_n_s32(sum4567, FILTER_BITS)); - return vqmovun_s16(sum_s16); + int16x8_t sum_s16 = + vcombine_s16(vshrn_n_s32(sum0123, 1), vshrn_n_s32(sum4567, 1)); + return vqrshrun_n_s16(sum_s16, FILTER_BITS - 1); } static inline void convolve_x_sr_12tap_neon_dotprod( @@ -134,8 +135,8 @@ int16x4_t d2 = convolve12_4_x(s2, filter, permute_tbl); int16x4_t d3 = convolve12_4_x(s3, filter, permute_tbl); - uint8x8_t d01 = vqmovun_s16(vcombine_s16(d0, d1)); - uint8x8_t d23 = vqmovun_s16(vcombine_s16(d2, d3)); + uint8x8_t d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS - 1); + uint8x8_t d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS - 1); store_u8x4_strided_x2(dst + 0 * dst_stride, dst_stride, d01); store_u8x4_strided_x2(dst + 2 * dst_stride, dst_stride, d23); @@ -387,57 +388,6 @@ } while (h != 0); } -static inline void transpose_concat_4x4(int8x8_t a0, int8x8_t a1, int8x8_t a2, - int8x8_t a3, int8x16_t *b) { - // Transpose 8-bit elements and concatenate result rows as follows: - // a0: 00, 01, 02, 03, XX, XX, XX, XX - // a1: 10, 11, 12, 13, XX, XX, XX, XX - // a2: 20, 21, 22, 23, XX, XX, XX, XX - // a3: 30, 31, 32, 33, XX, XX, XX, XX - // - // b: 00, 10, 20, 30, 01, 11, 21, 31, 02, 12, 22, 32, 03, 13, 23, 33 - - int8x16_t a0q = vcombine_s8(a0, vdup_n_s8(0)); - int8x16_t a1q = vcombine_s8(a1, vdup_n_s8(0)); - int8x16_t a2q = vcombine_s8(a2, vdup_n_s8(0)); - int8x16_t a3q = vcombine_s8(a3, vdup_n_s8(0)); - - int8x16_t a01 = vzipq_s8(a0q, a1q).val[0]; - int8x16_t a23 = vzipq_s8(a2q, a3q).val[0]; - - int16x8_t a0123 = - vzipq_s16(vreinterpretq_s16_s8(a01), vreinterpretq_s16_s8(a23)).val[0]; - - *b = vreinterpretq_s8_s16(a0123); -} - -static inline void transpose_concat_8x4(int8x8_t a0, int8x8_t a1, int8x8_t a2, - int8x8_t a3, int8x16_t *b0, - int8x16_t *b1) { - // Transpose 8-bit elements and concatenate result rows as follows: - // a0: 00, 01, 02, 03, 04, 05, 06, 07 - // a1: 10, 11, 12, 13, 14, 15, 16, 17 - // a2: 20, 21, 22, 23, 24, 25, 26, 27 - // a3: 30, 31, 32, 33, 34, 35, 36, 37 - // - // b0: 00, 10, 20, 30, 01, 11, 21, 31, 02, 12, 22, 32, 03, 13, 23, 33 - // b1: 04, 14, 24, 34, 05, 15, 25, 35, 06, 16, 26, 36, 07, 17, 27, 37 - - int8x16_t a0q = vcombine_s8(a0, vdup_n_s8(0)); - int8x16_t a1q = vcombine_s8(a1, vdup_n_s8(0)); - int8x16_t a2q = vcombine_s8(a2, vdup_n_s8(0)); - int8x16_t a3q = vcombine_s8(a3, vdup_n_s8(0)); - - int8x16_t a01 = vzipq_s8(a0q, a1q).val[0]; - int8x16_t a23 = vzipq_s8(a2q, a3q).val[0]; - - int16x8x2_t a0123 = - vzipq_s16(vreinterpretq_s16_s8(a01), vreinterpretq_s16_s8(a23)); - - *b0 = vreinterpretq_s8_s16(a0123.val[0]); - *b1 = vreinterpretq_s8_s16(a0123.val[1]); -} - static inline int16x4_t convolve12_4_y(const int8x16_t s0, const int8x16_t s1, const int8x16_t s2, const int8x8_t filters_0_7, @@ -450,7 +400,7 @@ sum = vdotq_lane_s32(sum, s2, filters_4_11, 1); // Further narrowing and packing is performed by the caller. - return vqmovn_s32(sum); + return vshrn_n_s32(sum, 1); } static inline uint8x8_t convolve12_8_y( @@ -470,8 +420,9 @@ sum4567 = vdotq_lane_s32(sum4567, s2_hi, filters_4_11, 1); // Narrow and re-pack. - int16x8_t sum = vcombine_s16(vqmovn_s32(sum0123), vqmovn_s32(sum4567)); - return vqrshrun_n_s16(sum, FILTER_BITS); + int16x8_t sum = + vcombine_s16(vshrn_n_s32(sum0123, 1), vshrn_n_s32(sum4567, 1)); + return vqrshrun_n_s16(sum, FILTER_BITS - 1); } static inline void convolve_y_sr_12tap_neon_dotprod( @@ -505,14 +456,14 @@ int8x8_t sA = vreinterpret_s8_u8(vsub_u8(tA, vdup_n_u8(128))); int8x16_t s0123, s1234, s2345, s3456, s4567, s5678, s6789, s789A; - transpose_concat_4x4(s0, s1, s2, s3, &s0123); - transpose_concat_4x4(s1, s2, s3, s4, &s1234); - transpose_concat_4x4(s2, s3, s4, s5, &s2345); - transpose_concat_4x4(s3, s4, s5, s6, &s3456); - transpose_concat_4x4(s4, s5, s6, s7, &s4567); - transpose_concat_4x4(s5, s6, s7, s8, &s5678); - transpose_concat_4x4(s6, s7, s8, s9, &s6789); - transpose_concat_4x4(s7, s8, s9, sA, &s789A); + transpose_concat_elems_s8_4x4(s0, s1, s2, s3, &s0123); + transpose_concat_elems_s8_4x4(s1, s2, s3, s4, &s1234); + transpose_concat_elems_s8_4x4(s2, s3, s4, s5, &s2345); + transpose_concat_elems_s8_4x4(s3, s4, s5, s6, &s3456); + transpose_concat_elems_s8_4x4(s4, s5, s6, s7, &s4567); + transpose_concat_elems_s8_4x4(s5, s6, s7, s8, &s5678); + transpose_concat_elems_s8_4x4(s6, s7, s8, s9, &s6789); + transpose_concat_elems_s8_4x4(s7, s8, s9, sA, &s789A); do { uint8x8_t tB, tC, tD, tE; @@ -524,7 +475,7 @@ int8x8_t sE = vreinterpret_s8_u8(vsub_u8(tE, vdup_n_u8(128))); int8x16_t s89AB, s9ABC, sABCD, sBCDE; - transpose_concat_4x4(sB, sC, sD, sE, &sBCDE); + transpose_concat_elems_s8_4x4(sB, sC, sD, sE, &sBCDE); // Merge new data into block from previous iteration. int8x16x2_t samples_LUT = { { s789A, sBCDE } }; @@ -540,8 +491,8 @@ convolve12_4_y(s2345, s6789, sABCD, filter_0_7, filter_4_11); int16x4_t d3 = convolve12_4_y(s3456, s789A, sBCDE, filter_0_7, filter_4_11); - uint8x8_t d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS); - uint8x8_t d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS); + uint8x8_t d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS - 1); + uint8x8_t d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS - 1); store_u8x4_strided_x2(dst_ptr + 0 * dst_stride, dst_stride, d01); store_u8x4_strided_x2(dst_ptr + 2 * dst_stride, dst_stride, d23); @@ -591,14 +542,14 @@ int8x16_t s0123_lo, s0123_hi, s1234_lo, s1234_hi, s2345_lo, s2345_hi, s3456_lo, s3456_hi, s4567_lo, s4567_hi, s5678_lo, s5678_hi, s6789_lo, s6789_hi, s789A_lo, s789A_hi; - transpose_concat_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi); - transpose_concat_8x4(s1, s2, s3, s4, &s1234_lo, &s1234_hi); - transpose_concat_8x4(s2, s3, s4, s5, &s2345_lo, &s2345_hi); - transpose_concat_8x4(s3, s4, s5, s6, &s3456_lo, &s3456_hi); - transpose_concat_8x4(s4, s5, s6, s7, &s4567_lo, &s4567_hi); - transpose_concat_8x4(s5, s6, s7, s8, &s5678_lo, &s5678_hi); - transpose_concat_8x4(s6, s7, s8, s9, &s6789_lo, &s6789_hi); - transpose_concat_8x4(s7, s8, s9, sA, &s789A_lo, &s789A_hi); + transpose_concat_elems_s8_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi); + transpose_concat_elems_s8_8x4(s1, s2, s3, s4, &s1234_lo, &s1234_hi); + transpose_concat_elems_s8_8x4(s2, s3, s4, s5, &s2345_lo, &s2345_hi); + transpose_concat_elems_s8_8x4(s3, s4, s5, s6, &s3456_lo, &s3456_hi); + transpose_concat_elems_s8_8x4(s4, s5, s6, s7, &s4567_lo, &s4567_hi); + transpose_concat_elems_s8_8x4(s5, s6, s7, s8, &s5678_lo, &s5678_hi); + transpose_concat_elems_s8_8x4(s6, s7, s8, s9, &s6789_lo, &s6789_hi); + transpose_concat_elems_s8_8x4(s7, s8, s9, sA, &s789A_lo, &s789A_hi); do { uint8x8_t tB, tC, tD, tE; @@ -611,7 +562,7 @@ int8x16_t s89AB_lo, s89AB_hi, s9ABC_lo, s9ABC_hi, sABCD_lo, sABCD_hi, sBCDE_lo, sBCDE_hi; - transpose_concat_8x4(sB, sC, sD, sE, &sBCDE_lo, &sBCDE_hi); + transpose_concat_elems_s8_8x4(sB, sC, sD, sE, &sBCDE_lo, &sBCDE_hi); // Merge new data into block from previous iteration. int8x16x2_t samples_LUT_lo = { { s789A_lo, sBCDE_lo } }; @@ -673,12 +624,13 @@ const int8x8_t filters) { // The sample range transform and permutation are performed by the caller. // Accumulate into 128 << FILTER_BITS to account for range transform. - const int32x4_t acc = vdupq_n_s32(128 << FILTER_BITS); + // (- 1 since we halved the filters.) + const int32x4_t acc = vdupq_n_s32(128 << (FILTER_BITS - 1)); int32x4_t sum = vdotq_lane_s32(acc, s0, filters, 0); sum = vdotq_lane_s32(sum, s1, filters, 1); // Further narrowing and packing is performed by the caller. - return vqmovn_s32(sum); + return vmovn_s32(sum); } static inline uint8x8_t convolve8_8_y(const int8x16_t s0_lo, @@ -688,7 +640,8 @@ const int8x8_t filters) { // The sample range transform and permutation are performed by the caller. // Accumulate into 128 << FILTER_BITS to account for range transform. - const int32x4_t acc = vdupq_n_s32(128 << FILTER_BITS); + // (- 1 since we halved the filters.) + const int32x4_t acc = vdupq_n_s32(128 << (FILTER_BITS - 1)); int32x4_t sum0123 = vdotq_lane_s32(acc, s0_lo, filters, 0); sum0123 = vdotq_lane_s32(sum0123, s1_lo, filters, 1); @@ -697,14 +650,16 @@ sum4567 = vdotq_lane_s32(sum4567, s1_hi, filters, 1); // Narrow and re-pack. - int16x8_t sum = vcombine_s16(vqmovn_s32(sum0123), vqmovn_s32(sum4567)); - return vqrshrun_n_s16(sum, FILTER_BITS); + int16x8_t sum = vcombine_s16(vmovn_s32(sum0123), vmovn_s32(sum4567)); + // We halved the filter values so -1 from right shift. + return vqrshrun_n_s16(sum, FILTER_BITS - 1); } static inline void convolve_y_sr_8tap_neon_dotprod( const uint8_t *src_ptr, int src_stride, uint8_t *dst_ptr, int dst_stride, int w, int h, const int16_t *y_filter_ptr) { - const int8x8_t filter = vmovn_s16(vld1q_s16(y_filter_ptr)); + // Filter values are even, so halve to reduce intermediate precision reqs. + const int8x8_t filter = vshrn_n_s16(vld1q_s16(y_filter_ptr), 1); const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(kDotProdMergeBlockTbl); @@ -723,25 +678,25 @@ int8x8_t s6 = vreinterpret_s8_u8(vsub_u8(t6, vdup_n_u8(128))); int8x16_t s0123, s1234, s2345, s3456; - transpose_concat_4x4(s0, s1, s2, s3, &s0123); - transpose_concat_4x4(s1, s2, s3, s4, &s1234); - transpose_concat_4x4(s2, s3, s4, s5, &s2345); - transpose_concat_4x4(s3, s4, s5, s6, &s3456); + transpose_concat_elems_s8_4x4(s0, s1, s2, s3, &s0123); + transpose_concat_elems_s8_4x4(s1, s2, s3, s4, &s1234); + transpose_concat_elems_s8_4x4(s2, s3, s4, s5, &s2345); + transpose_concat_elems_s8_4x4(s3, s4, s5, s6, &s3456); do { - uint8x8_t t7, t8, t9, t10; - load_u8_8x4(src_ptr, src_stride, &t7, &t8, &t9, &t10); + uint8x8_t t7, t8, t9, tA; + load_u8_8x4(src_ptr, src_stride, &t7, &t8, &t9, &tA); int8x8_t s7 = vreinterpret_s8_u8(vsub_u8(t7, vdup_n_u8(128))); int8x8_t s8 = vreinterpret_s8_u8(vsub_u8(t8, vdup_n_u8(128))); int8x8_t s9 = vreinterpret_s8_u8(vsub_u8(t9, vdup_n_u8(128))); - int8x8_t s10 = vreinterpret_s8_u8(vsub_u8(t10, vdup_n_u8(128))); + int8x8_t sA = vreinterpret_s8_u8(vsub_u8(tA, vdup_n_u8(128))); - int8x16_t s4567, s5678, s6789, s78910; - transpose_concat_4x4(s7, s8, s9, s10, &s78910); + int8x16_t s4567, s5678, s6789, s789A; + transpose_concat_elems_s8_4x4(s7, s8, s9, sA, &s789A); // Merge new data into block from previous iteration. - int8x16x2_t samples_LUT = { { s3456, s78910 } }; + int8x16x2_t samples_LUT = { { s3456, s789A } }; s4567 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]); s5678 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]); s6789 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]); @@ -749,9 +704,10 @@ int16x4_t d0 = convolve8_4_y(s0123, s4567, filter); int16x4_t d1 = convolve8_4_y(s1234, s5678, filter); int16x4_t d2 = convolve8_4_y(s2345, s6789, filter); - int16x4_t d3 = convolve8_4_y(s3456, s78910, filter); - uint8x8_t d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS); - uint8x8_t d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS); + int16x4_t d3 = convolve8_4_y(s3456, s789A, filter); + // We halved the filter values so -1 from right shift. + uint8x8_t d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS - 1); + uint8x8_t d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS - 1); store_u8x4_strided_x2(dst_ptr + 0 * dst_stride, dst_stride, d01); store_u8x4_strided_x2(dst_ptr + 2 * dst_stride, dst_stride, d23); @@ -761,7 +717,7 @@ s0123 = s4567; s1234 = s5678; s2345 = s6789; - s3456 = s78910; + s3456 = s789A; src_ptr += 4 * src_stride; dst_ptr += 4 * dst_stride; @@ -791,31 +747,31 @@ // product. int8x16_t s0123_lo, s0123_hi, s1234_lo, s1234_hi, s2345_lo, s2345_hi, s3456_lo, s3456_hi; - transpose_concat_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi); - transpose_concat_8x4(s1, s2, s3, s4, &s1234_lo, &s1234_hi); - transpose_concat_8x4(s2, s3, s4, s5, &s2345_lo, &s2345_hi); - transpose_concat_8x4(s3, s4, s5, s6, &s3456_lo, &s3456_hi); + transpose_concat_elems_s8_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi); + transpose_concat_elems_s8_8x4(s1, s2, s3, s4, &s1234_lo, &s1234_hi); + transpose_concat_elems_s8_8x4(s2, s3, s4, s5, &s2345_lo, &s2345_hi); + transpose_concat_elems_s8_8x4(s3, s4, s5, s6, &s3456_lo, &s3456_hi); do { - uint8x8_t t7, t8, t9, t10; - load_u8_8x4(s, src_stride, &t7, &t8, &t9, &t10); + uint8x8_t t7, t8, t9, tA; + load_u8_8x4(s, src_stride, &t7, &t8, &t9, &tA); int8x8_t s7 = vreinterpret_s8_u8(vsub_u8(t7, vdup_n_u8(128))); int8x8_t s8 = vreinterpret_s8_u8(vsub_u8(t8, vdup_n_u8(128))); int8x8_t s9 = vreinterpret_s8_u8(vsub_u8(t9, vdup_n_u8(128))); - int8x8_t s10 = vreinterpret_s8_u8(vsub_u8(t10, vdup_n_u8(128))); + int8x8_t sA = vreinterpret_s8_u8(vsub_u8(tA, vdup_n_u8(128))); int8x16_t s4567_lo, s4567_hi, s5678_lo, s5678_hi, s6789_lo, s6789_hi, - s78910_lo, s78910_hi; - transpose_concat_8x4(s7, s8, s9, s10, &s78910_lo, &s78910_hi); + s789A_lo, s789A_hi; + transpose_concat_elems_s8_8x4(s7, s8, s9, sA, &s789A_lo, &s789A_hi); // Merge new data into block from previous iteration. - int8x16x2_t samples_LUT_lo = { { s3456_lo, s78910_lo } }; + int8x16x2_t samples_LUT_lo = { { s3456_lo, s789A_lo } }; s4567_lo = vqtbl2q_s8(samples_LUT_lo, merge_block_tbl.val[0]); s5678_lo = vqtbl2q_s8(samples_LUT_lo, merge_block_tbl.val[1]); s6789_lo = vqtbl2q_s8(samples_LUT_lo, merge_block_tbl.val[2]); - int8x16x2_t samples_LUT_hi = { { s3456_hi, s78910_hi } }; + int8x16x2_t samples_LUT_hi = { { s3456_hi, s789A_hi } }; s4567_hi = vqtbl2q_s8(samples_LUT_hi, merge_block_tbl.val[0]); s5678_hi = vqtbl2q_s8(samples_LUT_hi, merge_block_tbl.val[1]); s6789_hi = vqtbl2q_s8(samples_LUT_hi, merge_block_tbl.val[2]); @@ -827,7 +783,7 @@ uint8x8_t d2 = convolve8_8_y(s2345_lo, s2345_hi, s6789_lo, s6789_hi, filter); uint8x8_t d3 = - convolve8_8_y(s3456_lo, s3456_hi, s78910_lo, s78910_hi, filter); + convolve8_8_y(s3456_lo, s3456_hi, s789A_lo, s789A_hi, filter); store_u8_8x4(d, dst_stride, d0, d1, d2, d3); @@ -839,8 +795,170 @@ s1234_hi = s5678_hi; s2345_lo = s6789_lo; s2345_hi = s6789_hi; - s3456_lo = s78910_lo; - s3456_hi = s78910_hi; + s3456_lo = s789A_lo; + s3456_hi = s789A_hi; + + s += 4 * src_stride; + d += 4 * dst_stride; + height -= 4; + } while (height != 0); + src_ptr += 8; + dst_ptr += 8; + w -= 8; + } while (w != 0); + } +} + +static inline int16x4_t convolve4_4_y(const int8x16_t s0, + const int8x8_t filters) { + // The sample range transform and permutation are performed by the caller. + // Accumulate into 128 << FILTER_BITS to account for range transform. + // (- 1 since we halved the filters.) + const int32x4_t acc = vdupq_n_s32(128 << (FILTER_BITS - 1)); + + int32x4_t sum = vdotq_lane_s32(acc, s0, filters, 0); + + // Further narrowing and packing is performed by the caller. + return vmovn_s32(sum); +} + +static inline uint8x8_t convolve4_8_y(const int8x16_t s0, const int8x16_t s1, + const int8x8_t filters) { + // The sample range transform and permutation are performed by the caller. + // Accumulate into 128 << FILTER_BITS to account for range transform. + // (- 1 since we halved the filters.) + const int32x4_t acc = vdupq_n_s32(128 << (FILTER_BITS - 1)); + + int32x4_t sum0123 = vdotq_lane_s32(acc, s0, filters, 0); + int32x4_t sum4567 = vdotq_lane_s32(acc, s1, filters, 0); + + // Narrow and re-pack. + int16x8_t sum = vcombine_s16(vmovn_s32(sum0123), vmovn_s32(sum4567)); + // We halved the filter values so -1 from right shift. + return vqrshrun_n_s16(sum, FILTER_BITS - 1); +} + +static inline void convolve_y_sr_4tap_neon_dotprod( + const uint8_t *src_ptr, int src_stride, uint8_t *dst_ptr, int dst_stride, + int w, int h, const int16_t *y_filter_ptr) { + // Filter values are even, so halve to reduce intermediate precision reqs. + const int16x8_t filter_s16 = + vcombine_s16(vld1_s16(y_filter_ptr + 2), vdup_n_s16(0)); + const int8x8_t filter = vshrn_n_s16(filter_s16, 1); + const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(kDotProdMergeBlockTbl); + int8x16x2_t samples_LUT; + + if (w == 4) { + uint8x8_t t0, t1, t2, t3; + load_u8_8x4(src_ptr, src_stride, &t0, &t1, &t2, &t3); + src_ptr += 4 * src_stride; + + // Transform sample range to [-128, 127] for 8-bit signed dot product. + int8x8_t s0 = vreinterpret_s8_u8(vsub_u8(t0, vdup_n_u8(128))); + int8x8_t s1 = vreinterpret_s8_u8(vsub_u8(t1, vdup_n_u8(128))); + int8x8_t s2 = vreinterpret_s8_u8(vsub_u8(t2, vdup_n_u8(128))); + int8x8_t s3 = vreinterpret_s8_u8(vsub_u8(t3, vdup_n_u8(128))); + + // This operation combines a conventional transpose and the sample permute + // required before computing the dot product. + int8x16_t s0123; + transpose_concat_elems_s8_4x4(s0, s1, s2, s3, &s0123); + + do { + uint8x8_t t4, t5, t6, t7; + load_u8_8x4(src_ptr, src_stride, &t4, &t5, &t6, &t7); + + int8x8_t s4 = vreinterpret_s8_u8(vsub_u8(t4, vdup_n_u8(128))); + int8x8_t s5 = vreinterpret_s8_u8(vsub_u8(t5, vdup_n_u8(128))); + int8x8_t s6 = vreinterpret_s8_u8(vsub_u8(t6, vdup_n_u8(128))); + int8x8_t s7 = vreinterpret_s8_u8(vsub_u8(t7, vdup_n_u8(128))); + + int8x16_t s4567; + transpose_concat_elems_s8_4x4(s4, s5, s6, s7, &s4567); + + // Merge new data into block from previous iteration. + samples_LUT.val[0] = s0123; + samples_LUT.val[1] = s4567; + int8x16_t s1234 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]); + int8x16_t s2345 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]); + int8x16_t s3456 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]); + + int16x4_t d0 = convolve4_4_y(s0123, filter); + int16x4_t d1 = convolve4_4_y(s1234, filter); + int16x4_t d2 = convolve4_4_y(s2345, filter); + int16x4_t d3 = convolve4_4_y(s3456, filter); + // We halved the filter values so -1 from right shift. + uint8x8_t d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS - 1); + uint8x8_t d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS - 1); + + store_u8x4_strided_x2(dst_ptr + 0 * dst_stride, dst_stride, d01); + store_u8x4_strided_x2(dst_ptr + 2 * dst_stride, dst_stride, d23); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123 = s4567; + + src_ptr += 4 * src_stride; + dst_ptr += 4 * dst_stride; + h -= 4; + } while (h != 0); + } else { + do { + int height = h; + const uint8_t *s = src_ptr; + uint8_t *d = dst_ptr; + + uint8x8_t t0, t1, t2, t3; + load_u8_8x4(s, src_stride, &t0, &t1, &t2, &t3); + s += 4 * src_stride; + + // Transform sample range to [-128, 127] for 8-bit signed dot product. + int8x8_t s0 = vreinterpret_s8_u8(vsub_u8(t0, vdup_n_u8(128))); + int8x8_t s1 = vreinterpret_s8_u8(vsub_u8(t1, vdup_n_u8(128))); + int8x8_t s2 = vreinterpret_s8_u8(vsub_u8(t2, vdup_n_u8(128))); + int8x8_t s3 = vreinterpret_s8_u8(vsub_u8(t3, vdup_n_u8(128))); + + // This operation combines a conventional transpose and the sample permute + // required before computing the dot product. + int8x16_t s0123_lo, s0123_hi; + transpose_concat_elems_s8_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi); + + do { + uint8x8_t t4, t5, t6, t7; + load_u8_8x4(s, src_stride, &t4, &t5, &t6, &t7); + + int8x8_t s4 = vreinterpret_s8_u8(vsub_u8(t4, vdup_n_u8(128))); + int8x8_t s5 = vreinterpret_s8_u8(vsub_u8(t5, vdup_n_u8(128))); + int8x8_t s6 = vreinterpret_s8_u8(vsub_u8(t6, vdup_n_u8(128))); + int8x8_t s7 = vreinterpret_s8_u8(vsub_u8(t7, vdup_n_u8(128))); + + int8x16_t s4567_lo, s4567_hi; + transpose_concat_elems_s8_8x4(s4, s5, s6, s7, &s4567_lo, &s4567_hi); + + // Merge new data into block from previous iteration. + samples_LUT.val[0] = s0123_lo; + samples_LUT.val[1] = s4567_lo; + int8x16_t s1234_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]); + int8x16_t s2345_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]); + int8x16_t s3456_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]); + + samples_LUT.val[0] = s0123_hi; + samples_LUT.val[1] = s4567_hi; + int8x16_t s1234_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]); + int8x16_t s2345_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]); + int8x16_t s3456_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]); + + uint8x8_t d0 = convolve4_8_y(s0123_lo, s0123_hi, filter); + uint8x8_t d1 = convolve4_8_y(s1234_lo, s1234_hi, filter); + uint8x8_t d2 = convolve4_8_y(s2345_lo, s2345_hi, filter); + uint8x8_t d3 = convolve4_8_y(s3456_lo, s3456_hi, filter); + + store_u8_8x4(d, dst_stride, d0, d1, d2, d3); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123_lo = s4567_lo; + s0123_hi = s4567_hi; s += 4 * src_stride; d += 4 * dst_stride; @@ -864,27 +982,20 @@ } const int y_filter_taps = get_filter_tap(filter_params_y, subpel_y_qn); - - if (y_filter_taps <= 6) { - av1_convolve_y_sr_neon(src, src_stride, dst, dst_stride, w, h, - filter_params_y, subpel_y_qn); - return; - } - - const int vert_offset = y_filter_taps / 2 - 1; - src -= vert_offset * src_stride; - const int16_t *y_filter_ptr = av1_get_interp_filter_subpel_kernel( filter_params_y, subpel_y_qn & SUBPEL_MASK); - if (y_filter_taps > 8) { - convolve_y_sr_12tap_neon_dotprod(src, src_stride, dst, dst_stride, w, h, - y_filter_ptr); - return; + if (y_filter_taps <= 4) { + convolve_y_sr_4tap_neon_dotprod(src - src_stride, src_stride, dst, + dst_stride, w, h, y_filter_ptr); + } else if (y_filter_taps == 12) { + convolve_y_sr_12tap_neon_dotprod(src - 5 * src_stride, src_stride, dst, + dst_stride, w, h, y_filter_ptr); + } else { + // 6-tap or 8-tap. + convolve_y_sr_8tap_neon_dotprod(src - 3 * src_stride, src_stride, dst, + dst_stride, w, h, y_filter_ptr); } - - convolve_y_sr_8tap_neon_dotprod(src, src_stride, dst, dst_stride, w, h, - y_filter_ptr); } static inline int16x4_t convolve12_4_2d_h(uint8x16_t samples,
diff --git a/av1/common/arm/convolve_neon_dotprod.h b/av1/common/arm/convolve_neon_dotprod.h new file mode 100644 index 0000000..2bdf365 --- /dev/null +++ b/av1/common/arm/convolve_neon_dotprod.h
@@ -0,0 +1,19 @@ +/* + * Copyright (c) 2025, Alliance for Open Media. All rights reserved. + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef AOM_AV1_COMMON_ARM_CONVOLVE_NEON_DOTPROD_H_ +#define AOM_AV1_COMMON_ARM_CONVOLVE_NEON_DOTPROD_H_ + +#include "aom_ports/mem.h" + +DECLARE_ALIGNED(16, extern const uint8_t, kDotProdPermuteTbl[48]); +DECLARE_ALIGNED(16, extern const uint8_t, kDotProdMergeBlockTbl[48]); + +#endif // AOM_AV1_COMMON_ARM_CONVOLVE_NEON_DOTPROD_H_
diff --git a/av1/common/arm/convolve_neon_i8mm.c b/av1/common/arm/convolve_neon_i8mm.c index acd912e..1452841 100644 --- a/av1/common/arm/convolve_neon_i8mm.c +++ b/av1/common/arm/convolve_neon_i8mm.c
@@ -16,19 +16,32 @@ #include "aom_dsp/aom_dsp_common.h" #include "aom_dsp/arm/mem_neon.h" +#include "aom_dsp/arm/transpose_neon.h" #include "aom_ports/mem.h" #include "av1/common/arm/convolve_neon.h" +#include "av1/common/arm/convolve_neon_dotprod.h" #include "av1/common/arm/convolve_neon_i8mm.h" #include "av1/common/convolve.h" #include "av1/common/filter.h" -DECLARE_ALIGNED(16, static const uint8_t, kDotProdMergeBlockTbl[48]) = { - // Shift left and insert new last column in transposed 4x4 block. - 1, 2, 3, 16, 5, 6, 7, 20, 9, 10, 11, 24, 13, 14, 15, 28, - // Shift left and insert two new columns in transposed 4x4 block. - 2, 3, 16, 17, 6, 7, 20, 21, 10, 11, 24, 25, 14, 15, 28, 29, - // Shift left and insert three new columns in transposed 4x4 block. - 3, 16, 17, 18, 7, 20, 21, 22, 11, 24, 25, 26, 15, 28, 29, 30 +DECLARE_ALIGNED(16, const uint8_t, kMatMul6PermuteTbl[32]) = { + // clang-format off + 0, 1, 2, 3, 4, 5, 6, 7, 2, 3, 4, 5, 6, 7, 8, 9, + 4, 5, 6, 7, 8, 9, 10, 11, 6, 7, 8, 9, 10, 11, 12, 13 + // clang-format on +}; + +DECLARE_ALIGNED(16, const uint8_t, kMatMul8PermuteTbl[32]) = { + // clang-format off + 1, 2, 3, 4, 5, 6, 7, 8, 3, 4, 5, 6, 7, 8, 9, 10, + 5, 6, 7, 8, 9, 10, 11, 12, 7, 8, 9, 10, 11, 12, 13, 14 + // clang-format on +}; + +DECLARE_ALIGNED(16, const uint8_t, kFilterPermuteTbl[16]) = { + // clang-format off + 1, 2, 3, 4, 5, 6, 7, 16, 16, 1, 2, 3, 4, 5, 6, 7 + // clang-format on }; static inline int16x4_t convolve12_4_x(uint8x16_t samples[2], @@ -46,7 +59,7 @@ int32x4_t sum = vusmmlaq_s32(horiz_const, perm_samples[0], filter[0]); sum = vusmmlaq_s32(sum, perm_samples[1], filter[1]); - return vqrshrn_n_s32(sum, FILTER_BITS); + return vshrn_n_s32(sum, 1); } static inline uint8x8_t convolve12_8_x(uint8x16_t samples[2], @@ -71,9 +84,9 @@ sum4567 = vusmmlaq_s32(sum4567, perm_samples[3], filter[1]); // Narrow and re-pack. - int16x8_t sum_s16 = vcombine_s16(vqrshrn_n_s32(sum0123, FILTER_BITS), - vqrshrn_n_s32(sum4567, FILTER_BITS)); - return vqmovun_s16(sum_s16); + int16x8_t sum_s16 = + vcombine_s16(vshrn_n_s32(sum0123, 1), vshrn_n_s32(sum4567, 1)); + return vqrshrun_n_s16(sum_s16, FILTER_BITS - 1); } static inline void convolve_x_sr_12tap_neon_i8mm(const uint8_t *src, @@ -104,7 +117,7 @@ const int32x4_t horiz_const = vdupq_n_s32(1 << (ROUND0_BITS - 1)); if (w <= 4) { - const uint8x16_t permute_tbl = vld1q_u8(kMatMulPermuteTbl); + const uint8x16_t permute_tbl = vld1q_u8(kMatMul6PermuteTbl); do { uint8x16_t s0[2], s1[2], s2[2], s3[2]; @@ -116,8 +129,8 @@ int16x4_t d2 = convolve12_4_x(s2, filter, permute_tbl, horiz_const); int16x4_t d3 = convolve12_4_x(s3, filter, permute_tbl, horiz_const); - uint8x8_t d01 = vqmovun_s16(vcombine_s16(d0, d1)); - uint8x8_t d23 = vqmovun_s16(vcombine_s16(d2, d3)); + uint8x8_t d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS - 1); + uint8x8_t d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS - 1); store_u8x4_strided_x2(dst + 0 * dst_stride, dst_stride, d01); store_u8x4_strided_x2(dst + 2 * dst_stride, dst_stride, d23); @@ -127,7 +140,7 @@ h -= 4; } while (h != 0); } else { - const uint8x16x2_t permute_tbl = vld1q_u8_x2(kMatMulPermuteTbl); + const uint8x16x2_t permute_tbl = vld1q_u8_x2(kMatMul6PermuteTbl); do { const uint8_t *s = src; @@ -157,35 +170,49 @@ } } -static inline uint8x8_t convolve8_8_x(uint8x16_t samples, const int8x8_t filter, - const uint8x16x3_t permute_tbl, - const int32x4_t horiz_const) { - // Permute samples ready for dot product. - // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } - // { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } - // { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 } - uint8x16_t perm_samples[3] = { vqtbl1q_u8(samples, permute_tbl.val[0]), - vqtbl1q_u8(samples, permute_tbl.val[1]), - vqtbl1q_u8(samples, permute_tbl.val[2]) }; +static inline uint8x8_t convolve8_8_x(uint8x16_t samples, + const int8x16_t filter, + const uint8x8_t f0, + const uint8x16x2_t permute_tbl, + const int16x8_t horiz_const) { + // Permute samples ready for matrix multiply. + // { 1, 2, 3, 4, 5, 6, 7, 8, 3, 4, 5, 6, 7, 8, 9, 10 } + // { 5, 6, 7, 8, 9, 10, 11, 12, 7, 8, 9, 10, 11, 12, 13, 14 } + uint8x16_t perm_samples[2] = { vqtbl1q_u8(samples, permute_tbl.val[0]), + vqtbl1q_u8(samples, permute_tbl.val[1]) }; - int32x4_t sum0123 = vusdotq_lane_s32(horiz_const, perm_samples[0], filter, 0); - sum0123 = vusdotq_lane_s32(sum0123, perm_samples[1], filter, 1); + // Calculate partial 7-tap convolution. + int32x4_t sum0123 = vusmmlaq_s32(vdupq_n_s32(0), perm_samples[0], filter); + int32x4_t sum4567 = vusmmlaq_s32(vdupq_n_s32(0), perm_samples[1], filter); + int16x8_t sum = vcombine_s16(vmovn_s32(sum0123), vmovn_s32(sum4567)); - int32x4_t sum4567 = vusdotq_lane_s32(horiz_const, perm_samples[1], filter, 0); - sum4567 = vusdotq_lane_s32(sum4567, perm_samples[2], filter, 1); + // Apply tap 0 and accumulate. + sum = vreinterpretq_s16_u16( + vmlsl_u8(vreinterpretq_u16_s16(sum), vget_low_u8(samples), f0)); - int16x8_t sum_s16 = vcombine_s16(vmovn_s32(sum0123), vmovn_s32(sum4567)); + sum = vaddq_s16(sum, horiz_const); + // We halved the convolution filter values so - 1 from the right shift. - return vqrshrun_n_s16(sum_s16, FILTER_BITS - 1); + return vqrshrun_n_s16(sum, FILTER_BITS - 1); } static inline void convolve_x_sr_8tap_neon_i8mm( const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, int width, int height, const int16_t *filter_x, - const int32x4_t horiz_const) { + const int16x8_t horiz_const) { // Filter values are even, so halve to reduce intermediate precision reqs. - const int8x8_t x_filter = vshrn_n_s16(vld1q_s16(filter_x), 1); - const uint8x16x3_t permute_tbl = vld1q_u8_x3(kDotProdPermuteTbl); + const int8x8_t x_filter_s8 = vshrn_n_s16(vld1q_s16(filter_x), 1); + + // Stagger the filter for use with the matrix multiply instructions. + // { f1, f2, f3, f4, f5, f6, f7, 0, 0, f1, f2, f3, f4, f5, f6, f7 } + const uint8x16_t filter_idx = vld1q_u8(kFilterPermuteTbl); + const int8x16_t x_filter = + vqtbl1q_s8(vcombine_s8(x_filter_s8, vdup_n_s8(0)), filter_idx); + + // Since f0 is always negative and s0 is unsigned, subtract (unsigned) s0 * + // -f0 to avoid signed overflow. + const uint8x8_t f0 = vdup_n_u8(-filter_x[0] >> 1); + const uint8x16x2_t permute_tbl = vld1q_u8_x2(kMatMul8PermuteTbl); do { const uint8_t *s = src; @@ -196,10 +223,10 @@ uint8x16_t s0, s1, s2, s3; load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3); - uint8x8_t d0 = convolve8_8_x(s0, x_filter, permute_tbl, horiz_const); - uint8x8_t d1 = convolve8_8_x(s1, x_filter, permute_tbl, horiz_const); - uint8x8_t d2 = convolve8_8_x(s2, x_filter, permute_tbl, horiz_const); - uint8x8_t d3 = convolve8_8_x(s3, x_filter, permute_tbl, horiz_const); + uint8x8_t d0 = convolve8_8_x(s0, x_filter, f0, permute_tbl, horiz_const); + uint8x8_t d1 = convolve8_8_x(s1, x_filter, f0, permute_tbl, horiz_const); + uint8x8_t d2 = convolve8_8_x(s2, x_filter, f0, permute_tbl, horiz_const); + uint8x8_t d3 = convolve8_8_x(s3, x_filter, f0, permute_tbl, horiz_const); store_u8_8x4(d, dst_stride, d0, d1, d2, d3); @@ -261,7 +288,7 @@ vcombine_s8(vext_s8(x_filter_s8, x_filter_s8, 1), x_filter_s8); if (width == 4) { - const uint8x16_t permute_tbl = vld1q_u8(kMatMulPermuteTbl); + const uint8x16_t permute_tbl = vld1q_u8(kMatMul6PermuteTbl); do { uint8x16_t s0, s1, s2, s3; load_u8_16x4(src, src_stride, &s0, &s1, &s2, &s3); @@ -282,7 +309,7 @@ height -= 4; } while (height != 0); } else { - const uint8x16x2_t permute_tbl = vld1q_u8_x2(kMatMulPermuteTbl); + const uint8x16x2_t permute_tbl = vld1q_u8_x2(kMatMul6PermuteTbl); do { const uint8_t *s = src; uint8_t *d = dst; @@ -334,11 +361,12 @@ // right shift by FILTER_BITS instead of two rounding right shifts: first by // ROUND0_BITS, and then subsequently by FILTER_BITS - ROUND0_BITS. // Halve the total because we will halve the filter values. - const int32x4_t horiz_const = vdupq_n_s32((1 << ((ROUND0_BITS - 1)) / 2)); + const int32x4_t horiz_const_s32 = vdupq_n_s32(1 << (ROUND0_BITS - 1) / 2); + const int16x8_t horiz_const_s16 = vdupq_n_s16(1 << (ROUND0_BITS - 1) / 2); if (filter_taps <= 6) { convolve_x_sr_6tap_neon_i8mm(src + 1, src_stride, dst, dst_stride, w, h, - x_filter_ptr, horiz_const); + x_filter_ptr, horiz_const_s32); return; } @@ -349,59 +377,7 @@ } convolve_x_sr_8tap_neon_i8mm(src, src_stride, dst, dst_stride, w, h, - x_filter_ptr, horiz_const); -} - -static inline void transpose_concat_4x4(uint8x8_t a0, uint8x8_t a1, - uint8x8_t a2, uint8x8_t a3, - uint8x16_t *b) { - // Transpose 8-bit elements and concatenate result rows as follows: - // a0: 00, 01, 02, 03, XX, XX, XX, XX - // a1: 10, 11, 12, 13, XX, XX, XX, XX - // a2: 20, 21, 22, 23, XX, XX, XX, XX - // a3: 30, 31, 32, 33, XX, XX, XX, XX - // - // b: 00, 10, 20, 30, 01, 11, 21, 31, 02, 12, 22, 32, 03, 13, 23, 33 - - uint8x16_t a0q = vcombine_u8(a0, vdup_n_u8(0)); - uint8x16_t a1q = vcombine_u8(a1, vdup_n_u8(0)); - uint8x16_t a2q = vcombine_u8(a2, vdup_n_u8(0)); - uint8x16_t a3q = vcombine_u8(a3, vdup_n_u8(0)); - - uint8x16_t a01 = vzipq_u8(a0q, a1q).val[0]; - uint8x16_t a23 = vzipq_u8(a2q, a3q).val[0]; - - uint16x8_t a0123 = - vzipq_u16(vreinterpretq_u16_u8(a01), vreinterpretq_u16_u8(a23)).val[0]; - - *b = vreinterpretq_u8_u16(a0123); -} - -static inline void transpose_concat_8x4(uint8x8_t a0, uint8x8_t a1, - uint8x8_t a2, uint8x8_t a3, - uint8x16_t *b0, uint8x16_t *b1) { - // Transpose 8-bit elements and concatenate result rows as follows: - // a0: 00, 01, 02, 03, 04, 05, 06, 07 - // a1: 10, 11, 12, 13, 14, 15, 16, 17 - // a2: 20, 21, 22, 23, 24, 25, 26, 27 - // a3: 30, 31, 32, 33, 34, 35, 36, 37 - // - // b0: 00, 10, 20, 30, 01, 11, 21, 31, 02, 12, 22, 32, 03, 13, 23, 33 - // b1: 04, 14, 24, 34, 05, 15, 25, 35, 06, 16, 26, 36, 07, 17, 27, 37 - - uint8x16_t a0q = vcombine_u8(a0, vdup_n_u8(0)); - uint8x16_t a1q = vcombine_u8(a1, vdup_n_u8(0)); - uint8x16_t a2q = vcombine_u8(a2, vdup_n_u8(0)); - uint8x16_t a3q = vcombine_u8(a3, vdup_n_u8(0)); - - uint8x16_t a01 = vzipq_u8(a0q, a1q).val[0]; - uint8x16_t a23 = vzipq_u8(a2q, a3q).val[0]; - - uint16x8x2_t a0123 = - vzipq_u16(vreinterpretq_u16_u8(a01), vreinterpretq_u16_u8(a23)); - - *b0 = vreinterpretq_u8_u16(a0123.val[0]); - *b1 = vreinterpretq_u8_u16(a0123.val[1]); + x_filter_ptr, horiz_const_s16); } static inline int16x4_t convolve12_4_y(const uint8x16_t s0, const uint8x16_t s1, @@ -413,7 +389,7 @@ sum = vusdotq_lane_s32(sum, s2, filters_4_11, 1); // Further narrowing and packing is performed by the caller. - return vqmovn_s32(sum); + return vshrn_n_s32(sum, 1); } static inline uint8x8_t convolve12_8_y( @@ -429,8 +405,9 @@ sum4567 = vusdotq_lane_s32(sum4567, s2_hi, filters_4_11, 1); // Narrow and re-pack. - int16x8_t sum = vcombine_s16(vqmovn_s32(sum0123), vqmovn_s32(sum4567)); - return vqrshrun_n_s16(sum, FILTER_BITS); + int16x8_t sum = + vcombine_s16(vshrn_n_s32(sum0123, 1), vshrn_n_s32(sum4567, 1)); + return vqrshrun_n_s16(sum, FILTER_BITS - 1); } static inline void convolve_y_sr_12tap_neon_i8mm(const uint8_t *src_ptr, @@ -455,21 +432,21 @@ // This operation combines a conventional transpose and the sample permute // (see horizontal case) required before computing the dot product. uint8x16_t s0123, s1234, s2345, s3456, s4567, s5678, s6789, s789A; - transpose_concat_4x4(s0, s1, s2, s3, &s0123); - transpose_concat_4x4(s1, s2, s3, s4, &s1234); - transpose_concat_4x4(s2, s3, s4, s5, &s2345); - transpose_concat_4x4(s3, s4, s5, s6, &s3456); - transpose_concat_4x4(s4, s5, s6, s7, &s4567); - transpose_concat_4x4(s5, s6, s7, s8, &s5678); - transpose_concat_4x4(s6, s7, s8, s9, &s6789); - transpose_concat_4x4(s7, s8, s9, sA, &s789A); + transpose_concat_elems_u8_4x4(s0, s1, s2, s3, &s0123); + transpose_concat_elems_u8_4x4(s1, s2, s3, s4, &s1234); + transpose_concat_elems_u8_4x4(s2, s3, s4, s5, &s2345); + transpose_concat_elems_u8_4x4(s3, s4, s5, s6, &s3456); + transpose_concat_elems_u8_4x4(s4, s5, s6, s7, &s4567); + transpose_concat_elems_u8_4x4(s5, s6, s7, s8, &s5678); + transpose_concat_elems_u8_4x4(s6, s7, s8, s9, &s6789); + transpose_concat_elems_u8_4x4(s7, s8, s9, sA, &s789A); do { uint8x8_t sB, sC, sD, sE; load_u8_8x4(src_ptr, src_stride, &sB, &sC, &sD, &sE); uint8x16_t s89AB, s9ABC, sABCD, sBCDE; - transpose_concat_4x4(sB, sC, sD, sE, &sBCDE); + transpose_concat_elems_u8_4x4(sB, sC, sD, sE, &sBCDE); // Merge new data into block from previous iteration. uint8x16x2_t samples_LUT = { { s789A, sBCDE } }; @@ -485,8 +462,8 @@ convolve12_4_y(s2345, s6789, sABCD, filter_0_7, filter_4_11); int16x4_t d3 = convolve12_4_y(s3456, s789A, sBCDE, filter_0_7, filter_4_11); - uint8x8_t d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS); - uint8x8_t d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS); + uint8x8_t d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS - 1); + uint8x8_t d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS - 1); store_u8x4_strided_x2(dst_ptr + 0 * dst_stride, dst_stride, d01); store_u8x4_strided_x2(dst_ptr + 2 * dst_stride, dst_stride, d23); @@ -523,14 +500,14 @@ uint8x16_t s0123_lo, s0123_hi, s1234_lo, s1234_hi, s2345_lo, s2345_hi, s3456_lo, s3456_hi, s4567_lo, s4567_hi, s5678_lo, s5678_hi, s6789_lo, s6789_hi, s789A_lo, s789A_hi; - transpose_concat_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi); - transpose_concat_8x4(s1, s2, s3, s4, &s1234_lo, &s1234_hi); - transpose_concat_8x4(s2, s3, s4, s5, &s2345_lo, &s2345_hi); - transpose_concat_8x4(s3, s4, s5, s6, &s3456_lo, &s3456_hi); - transpose_concat_8x4(s4, s5, s6, s7, &s4567_lo, &s4567_hi); - transpose_concat_8x4(s5, s6, s7, s8, &s5678_lo, &s5678_hi); - transpose_concat_8x4(s6, s7, s8, s9, &s6789_lo, &s6789_hi); - transpose_concat_8x4(s7, s8, s9, sA, &s789A_lo, &s789A_hi); + transpose_concat_elems_u8_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi); + transpose_concat_elems_u8_8x4(s1, s2, s3, s4, &s1234_lo, &s1234_hi); + transpose_concat_elems_u8_8x4(s2, s3, s4, s5, &s2345_lo, &s2345_hi); + transpose_concat_elems_u8_8x4(s3, s4, s5, s6, &s3456_lo, &s3456_hi); + transpose_concat_elems_u8_8x4(s4, s5, s6, s7, &s4567_lo, &s4567_hi); + transpose_concat_elems_u8_8x4(s5, s6, s7, s8, &s5678_lo, &s5678_hi); + transpose_concat_elems_u8_8x4(s6, s7, s8, s9, &s6789_lo, &s6789_hi); + transpose_concat_elems_u8_8x4(s7, s8, s9, sA, &s789A_lo, &s789A_hi); do { uint8x8_t sB, sC, sD, sE; @@ -538,7 +515,7 @@ uint8x16_t s89AB_lo, s89AB_hi, s9ABC_lo, s9ABC_hi, sABCD_lo, sABCD_hi, sBCDE_lo, sBCDE_hi; - transpose_concat_8x4(sB, sC, sD, sE, &sBCDE_lo, &sBCDE_hi); + transpose_concat_elems_u8_8x4(sB, sC, sD, sE, &sBCDE_lo, &sBCDE_hi); // Merge new data into block from previous iteration. uint8x16x2_t samples_LUT_lo = { { s789A_lo, sBCDE_lo } }; @@ -602,7 +579,7 @@ sum = vusdotq_lane_s32(sum, s1, filters, 1); // Further narrowing and packing is performed by the caller. - return vqmovn_s32(sum); + return vmovn_s32(sum); } static inline uint8x8_t convolve8_8_y(const uint8x16_t s0_lo, @@ -617,8 +594,9 @@ sum4567 = vusdotq_lane_s32(sum4567, s1_hi, filters, 1); // Narrow and re-pack. - int16x8_t sum = vcombine_s16(vqmovn_s32(sum0123), vqmovn_s32(sum4567)); - return vqrshrun_n_s16(sum, FILTER_BITS); + int16x8_t sum = vcombine_s16(vmovn_s32(sum0123), vmovn_s32(sum4567)); + // We halved the filter values so -1 from right shift. + return vqrshrun_n_s16(sum, FILTER_BITS - 1); } static inline void convolve_y_sr_8tap_neon_i8mm(const uint8_t *src_ptr, @@ -626,7 +604,8 @@ uint8_t *dst_ptr, int dst_stride, int w, int h, const int16_t *y_filter_ptr) { - const int8x8_t filter = vmovn_s16(vld1q_s16(y_filter_ptr)); + // Filter values are even, so halve to reduce intermediate precision reqs. + const int8x8_t filter = vshrn_n_s16(vld1q_s16(y_filter_ptr), 1); const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(kDotProdMergeBlockTbl); @@ -638,20 +617,20 @@ // This operation combines a conventional transpose and the sample permute // (see horizontal case) required before computing the dot product. uint8x16_t s0123, s1234, s2345, s3456; - transpose_concat_4x4(s0, s1, s2, s3, &s0123); - transpose_concat_4x4(s1, s2, s3, s4, &s1234); - transpose_concat_4x4(s2, s3, s4, s5, &s2345); - transpose_concat_4x4(s3, s4, s5, s6, &s3456); + transpose_concat_elems_u8_4x4(s0, s1, s2, s3, &s0123); + transpose_concat_elems_u8_4x4(s1, s2, s3, s4, &s1234); + transpose_concat_elems_u8_4x4(s2, s3, s4, s5, &s2345); + transpose_concat_elems_u8_4x4(s3, s4, s5, s6, &s3456); do { - uint8x8_t s7, s8, s9, s10; - load_u8_8x4(src_ptr, src_stride, &s7, &s8, &s9, &s10); + uint8x8_t s7, s8, s9, sA; + load_u8_8x4(src_ptr, src_stride, &s7, &s8, &s9, &sA); - uint8x16_t s4567, s5678, s6789, s78910; - transpose_concat_4x4(s7, s8, s9, s10, &s78910); + uint8x16_t s4567, s5678, s6789, s789A; + transpose_concat_elems_u8_4x4(s7, s8, s9, sA, &s789A); // Merge new data into block from previous iteration. - uint8x16x2_t samples_LUT = { { s3456, s78910 } }; + uint8x16x2_t samples_LUT = { { s3456, s789A } }; s4567 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[0]); s5678 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[1]); s6789 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[2]); @@ -659,9 +638,10 @@ int16x4_t d0 = convolve8_4_y(s0123, s4567, filter); int16x4_t d1 = convolve8_4_y(s1234, s5678, filter); int16x4_t d2 = convolve8_4_y(s2345, s6789, filter); - int16x4_t d3 = convolve8_4_y(s3456, s78910, filter); - uint8x8_t d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS); - uint8x8_t d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS); + int16x4_t d3 = convolve8_4_y(s3456, s789A, filter); + // We halved the filter values so -1 from right shift. + uint8x8_t d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS - 1); + uint8x8_t d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS - 1); store_u8x4_strided_x2(dst_ptr + 0 * dst_stride, dst_stride, d01); store_u8x4_strided_x2(dst_ptr + 2 * dst_stride, dst_stride, d23); @@ -671,7 +651,7 @@ s0123 = s4567; s1234 = s5678; s2345 = s6789; - s3456 = s78910; + s3456 = s789A; src_ptr += 4 * src_stride; dst_ptr += 4 * dst_stride; @@ -692,26 +672,26 @@ // product. uint8x16_t s0123_lo, s0123_hi, s1234_lo, s1234_hi, s2345_lo, s2345_hi, s3456_lo, s3456_hi; - transpose_concat_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi); - transpose_concat_8x4(s1, s2, s3, s4, &s1234_lo, &s1234_hi); - transpose_concat_8x4(s2, s3, s4, s5, &s2345_lo, &s2345_hi); - transpose_concat_8x4(s3, s4, s5, s6, &s3456_lo, &s3456_hi); + transpose_concat_elems_u8_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi); + transpose_concat_elems_u8_8x4(s1, s2, s3, s4, &s1234_lo, &s1234_hi); + transpose_concat_elems_u8_8x4(s2, s3, s4, s5, &s2345_lo, &s2345_hi); + transpose_concat_elems_u8_8x4(s3, s4, s5, s6, &s3456_lo, &s3456_hi); do { - uint8x8_t s7, s8, s9, s10; - load_u8_8x4(s, src_stride, &s7, &s8, &s9, &s10); + uint8x8_t s7, s8, s9, sA; + load_u8_8x4(s, src_stride, &s7, &s8, &s9, &sA); uint8x16_t s4567_lo, s4567_hi, s5678_lo, s5678_hi, s6789_lo, s6789_hi, - s78910_lo, s78910_hi; - transpose_concat_8x4(s7, s8, s9, s10, &s78910_lo, &s78910_hi); + s789A_lo, s789A_hi; + transpose_concat_elems_u8_8x4(s7, s8, s9, sA, &s789A_lo, &s789A_hi); // Merge new data into block from previous iteration. - uint8x16x2_t samples_LUT_lo = { { s3456_lo, s78910_lo } }; + uint8x16x2_t samples_LUT_lo = { { s3456_lo, s789A_lo } }; s4567_lo = vqtbl2q_u8(samples_LUT_lo, merge_block_tbl.val[0]); s5678_lo = vqtbl2q_u8(samples_LUT_lo, merge_block_tbl.val[1]); s6789_lo = vqtbl2q_u8(samples_LUT_lo, merge_block_tbl.val[2]); - uint8x16x2_t samples_LUT_hi = { { s3456_hi, s78910_hi } }; + uint8x16x2_t samples_LUT_hi = { { s3456_hi, s789A_hi } }; s4567_hi = vqtbl2q_u8(samples_LUT_hi, merge_block_tbl.val[0]); s5678_hi = vqtbl2q_u8(samples_LUT_hi, merge_block_tbl.val[1]); s6789_hi = vqtbl2q_u8(samples_LUT_hi, merge_block_tbl.val[2]); @@ -723,7 +703,7 @@ uint8x8_t d2 = convolve8_8_y(s2345_lo, s2345_hi, s6789_lo, s6789_hi, filter); uint8x8_t d3 = - convolve8_8_y(s3456_lo, s3456_hi, s78910_lo, s78910_hi, filter); + convolve8_8_y(s3456_lo, s3456_hi, s789A_lo, s789A_hi, filter); store_u8_8x4(d, dst_stride, d0, d1, d2, d3); @@ -735,8 +715,140 @@ s1234_hi = s5678_hi; s2345_lo = s6789_lo; s2345_hi = s6789_hi; - s3456_lo = s78910_lo; - s3456_hi = s78910_hi; + s3456_lo = s789A_lo; + s3456_hi = s789A_hi; + + s += 4 * src_stride; + d += 4 * dst_stride; + height -= 4; + } while (height != 0); + src_ptr += 8; + dst_ptr += 8; + w -= 8; + } while (w != 0); + } +} + +static inline int16x4_t convolve4_4_y(const uint8x16_t s0, + const int8x8_t filters) { + int32x4_t sum = vusdotq_lane_s32(vdupq_n_s32(0), s0, filters, 0); + + // Further narrowing and packing is performed by the caller. + return vmovn_s32(sum); +} + +static inline uint8x8_t convolve4_8_y(const uint8x16_t s0, const uint8x16_t s1, + const int8x8_t filters) { + int32x4_t sum0123 = vusdotq_lane_s32(vdupq_n_s32(0), s0, filters, 0); + int32x4_t sum4567 = vusdotq_lane_s32(vdupq_n_s32(0), s1, filters, 0); + + // Narrow and re-pack. + int16x8_t sum = vcombine_s16(vmovn_s32(sum0123), vmovn_s32(sum4567)); + // We halved the filter values so -1 from right shift. + return vqrshrun_n_s16(sum, FILTER_BITS - 1); +} + +static inline void convolve_y_sr_4tap_neon_i8mm(const uint8_t *src_ptr, + int src_stride, + uint8_t *dst_ptr, + int dst_stride, int w, int h, + const int16_t *y_filter_ptr) { + // Filter values are even, so halve to reduce intermediate precision reqs. + const int16x8_t filter_s16 = + vcombine_s16(vld1_s16(y_filter_ptr + 2), vdup_n_s16(0)); + const int8x8_t filter = vshrn_n_s16(filter_s16, 1); + const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(kDotProdMergeBlockTbl); + uint8x16x2_t samples_LUT; + + if (w == 4) { + uint8x8_t s0, s1, s2, s3; + load_u8_8x4(src_ptr, src_stride, &s0, &s1, &s2, &s3); + src_ptr += 4 * src_stride; + + // This operation combines a conventional transpose and the sample permute + // required before computing the dot product. + uint8x16_t s0123; + transpose_concat_elems_u8_4x4(s0, s1, s2, s3, &s0123); + + do { + uint8x8_t s4, s5, s6, s7; + load_u8_8x4(src_ptr, src_stride, &s4, &s5, &s6, &s7); + + uint8x16_t s4567; + transpose_concat_elems_u8_4x4(s4, s5, s6, s7, &s4567); + + // Merge new data into block from previous iteration. + samples_LUT.val[0] = s0123; + samples_LUT.val[1] = s4567; + uint8x16_t s1234 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[0]); + uint8x16_t s2345 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[1]); + uint8x16_t s3456 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[2]); + + int16x4_t d0 = convolve4_4_y(s0123, filter); + int16x4_t d1 = convolve4_4_y(s1234, filter); + int16x4_t d2 = convolve4_4_y(s2345, filter); + int16x4_t d3 = convolve4_4_y(s3456, filter); + // We halved the filter values so -1 from right shift. + uint8x8_t d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS - 1); + uint8x8_t d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS - 1); + + store_u8x4_strided_x2(dst_ptr + 0 * dst_stride, dst_stride, d01); + store_u8x4_strided_x2(dst_ptr + 2 * dst_stride, dst_stride, d23); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123 = s4567; + + src_ptr += 4 * src_stride; + dst_ptr += 4 * dst_stride; + h -= 4; + } while (h != 0); + } else { + do { + int height = h; + const uint8_t *s = src_ptr; + uint8_t *d = dst_ptr; + + uint8x8_t s0, s1, s2, s3; + load_u8_8x4(s, src_stride, &s0, &s1, &s2, &s3); + s += 4 * src_stride; + + // This operation combines a conventional transpose and the sample permute + // required before computing the dot product. + uint8x16_t s0123_lo, s0123_hi; + transpose_concat_elems_u8_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi); + + do { + uint8x8_t s4, s5, s6, s7; + load_u8_8x4(s, src_stride, &s4, &s5, &s6, &s7); + + uint8x16_t s4567_lo, s4567_hi; + transpose_concat_elems_u8_8x4(s4, s5, s6, s7, &s4567_lo, &s4567_hi); + + // Merge new data into block from previous iteration. + samples_LUT.val[0] = s0123_lo; + samples_LUT.val[1] = s4567_lo; + uint8x16_t s1234_lo = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[0]); + uint8x16_t s2345_lo = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[1]); + uint8x16_t s3456_lo = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[2]); + + samples_LUT.val[0] = s0123_hi; + samples_LUT.val[1] = s4567_hi; + uint8x16_t s1234_hi = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[0]); + uint8x16_t s2345_hi = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[1]); + uint8x16_t s3456_hi = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[2]); + + uint8x8_t d0 = convolve4_8_y(s0123_lo, s0123_hi, filter); + uint8x8_t d1 = convolve4_8_y(s1234_lo, s1234_hi, filter); + uint8x8_t d2 = convolve4_8_y(s2345_lo, s2345_hi, filter); + uint8x8_t d3 = convolve4_8_y(s3456_lo, s3456_hi, filter); + + store_u8_8x4(d, dst_stride, d0, d1, d2, d3); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123_lo = s4567_lo; + s0123_hi = s4567_hi; s += 4 * src_stride; d += 4 * dst_stride; @@ -760,65 +872,70 @@ } const int y_filter_taps = get_filter_tap(filter_params_y, subpel_y_qn); - - if (y_filter_taps <= 6) { - av1_convolve_y_sr_neon(src, src_stride, dst, dst_stride, w, h, - filter_params_y, subpel_y_qn); - return; - } - - const int vert_offset = y_filter_taps / 2 - 1; - src -= vert_offset * src_stride; - const int16_t *y_filter_ptr = av1_get_interp_filter_subpel_kernel( filter_params_y, subpel_y_qn & SUBPEL_MASK); - if (y_filter_taps > 8) { - convolve_y_sr_12tap_neon_i8mm(src, src_stride, dst, dst_stride, w, h, - y_filter_ptr); - return; + if (y_filter_taps <= 4) { + convolve_y_sr_4tap_neon_i8mm(src - src_stride, src_stride, dst, dst_stride, + w, h, y_filter_ptr); + } else if (y_filter_taps == 12) { + convolve_y_sr_12tap_neon_i8mm(src - 5 * src_stride, src_stride, dst, + dst_stride, w, h, y_filter_ptr); + } else { + // 6-tap or 8-tap. + convolve_y_sr_8tap_neon_i8mm(src - 3 * src_stride, src_stride, dst, + dst_stride, w, h, y_filter_ptr); } - convolve_y_sr_8tap_neon_i8mm(src, src_stride, dst, dst_stride, w, h, - y_filter_ptr); } static inline int16x8_t convolve8_8_2d_h(uint8x16_t samples, - const int8x8_t filters, - const uint8x16x3_t permute_tbl, - const int32x4_t horiz_const) { - // Permute samples ready for dot product. - // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } - // { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } - // { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 } - uint8x16_t perm_samples[3] = { vqtbl1q_u8(samples, permute_tbl.val[0]), - vqtbl1q_u8(samples, permute_tbl.val[1]), - vqtbl1q_u8(samples, permute_tbl.val[2]) }; + const int8x16_t x_filter, + const uint8x8_t f0, + const uint8x16x2_t permute_tbl, + const int16x8_t horiz_const) { + // Permute samples ready for matrix multiply. + // { 1, 2, 3, 4, 5, 6, 7, 8, 3, 4, 5, 6, 7, 8, 9, 10 } + // { 5, 6, 7, 8, 9, 10, 11, 12, 7, 8, 9, 10, 11, 12, 13, 14 } + uint8x16_t perm_samples[2] = { vqtbl1q_u8(samples, permute_tbl.val[0]), + vqtbl1q_u8(samples, permute_tbl.val[1]) }; - int32x4_t sum0123 = - vusdotq_lane_s32(horiz_const, perm_samples[0], filters, 0); - sum0123 = vusdotq_lane_s32(sum0123, perm_samples[1], filters, 1); + // Calculate partial 7-tap convolution. + int32x4_t sum0123 = vusmmlaq_s32(vdupq_n_s32(0), perm_samples[0], x_filter); + int32x4_t sum4567 = vusmmlaq_s32(vdupq_n_s32(0), perm_samples[1], x_filter); + int16x8_t sum = vcombine_s16(vmovn_s32(sum0123), vmovn_s32(sum4567)); - int32x4_t sum4567 = - vusdotq_lane_s32(horiz_const, perm_samples[1], filters, 0); - sum4567 = vusdotq_lane_s32(sum4567, perm_samples[2], filters, 1); + // Apply tap 0 and accumulate. + sum = vreinterpretq_s16_u16( + vmlsl_u8(vreinterpretq_u16_s16(sum), vget_low_u8(samples), f0)); - // Narrow and re-pack. + sum = vaddq_s16(sum, horiz_const); + // We halved the convolution filter values so -1 from the right shift. - return vcombine_s16(vshrn_n_s32(sum0123, ROUND0_BITS - 1), - vshrn_n_s32(sum4567, ROUND0_BITS - 1)); + return vshrq_n_s16(sum, ROUND0_BITS - 1); } static inline void convolve_2d_sr_horiz_8tap_neon_i8mm( const uint8_t *src, int src_stride, int16_t *im_block, int im_stride, int w, int im_h, const int16_t *x_filter_ptr) { // Filter values are even, so halve to reduce intermediate precision reqs. - const int8x8_t x_filter = vshrn_n_s16(vld1q_s16(x_filter_ptr), 1); + const int8x8_t x_filter_s8 = vshrn_n_s16(vld1q_s16(x_filter_ptr), 1); + + // Stagger the filter for use with the matrix multiply instructions. + // { f1, f2, f3, f4, f5, f6, f7, 0, 0, f1, f2, f3, f4, f5, f6, f7 } + const uint8x16_t filter_idx = vld1q_u8(kFilterPermuteTbl); + const int8x16_t x_filter = + vqtbl1q_s8(vcombine_s8(x_filter_s8, vdup_n_s8(0)), filter_idx); + + // Since f0 is always negative and s0 is unsigned, subtract (unsigned) s0 * + // -f0 to avoid signed overflow. + const uint8x8_t f0 = vdup_n_u8(-x_filter_ptr[0] >> 1); + const uint8x16x2_t permute_tbl = vld1q_u8_x2(kMatMul8PermuteTbl); const int bd = 8; // This shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-rounding // shifts - which are generally faster than rounding shifts on modern CPUs. // The outermost -1 is needed because we halved the filter values. - const int32x4_t horiz_const = vdupq_n_s32((1 << (bd + FILTER_BITS - 2)) + + const int16x8_t horiz_const = vdupq_n_s16((1 << (bd + FILTER_BITS - 2)) + (1 << ((ROUND0_BITS - 1) - 1))); const uint8_t *src_ptr = src; @@ -826,7 +943,6 @@ int dst_stride = im_stride; int height = im_h; - const uint8x16x3_t permute_tbl = vld1q_u8_x3(kDotProdPermuteTbl); do { const uint8_t *s = src_ptr; int16_t *d = dst_ptr; @@ -836,10 +952,14 @@ uint8x16_t s0, s1, s2, s3; load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3); - int16x8_t d0 = convolve8_8_2d_h(s0, x_filter, permute_tbl, horiz_const); - int16x8_t d1 = convolve8_8_2d_h(s1, x_filter, permute_tbl, horiz_const); - int16x8_t d2 = convolve8_8_2d_h(s2, x_filter, permute_tbl, horiz_const); - int16x8_t d3 = convolve8_8_2d_h(s3, x_filter, permute_tbl, horiz_const); + int16x8_t d0 = + convolve8_8_2d_h(s0, x_filter, f0, permute_tbl, horiz_const); + int16x8_t d1 = + convolve8_8_2d_h(s1, x_filter, f0, permute_tbl, horiz_const); + int16x8_t d2 = + convolve8_8_2d_h(s2, x_filter, f0, permute_tbl, horiz_const); + int16x8_t d3 = + convolve8_8_2d_h(s3, x_filter, f0, permute_tbl, horiz_const); store_s16_8x4(d, dst_stride, d0, d1, d2, d3); @@ -859,7 +979,8 @@ do { uint8x16_t s0 = vld1q_u8(s); - int16x8_t d0 = convolve8_8_2d_h(s0, x_filter, permute_tbl, horiz_const); + int16x8_t d0 = + convolve8_8_2d_h(s0, x_filter, f0, permute_tbl, horiz_const); vst1q_s16(d, d0); s += 8; @@ -1052,7 +1173,7 @@ const int32x4_t horiz_const = vdupq_n_s32((1 << (bd + FILTER_BITS - 2)) + (1 << ((ROUND0_BITS - 1) - 1))); const int16x8_t vert_const = vdupq_n_s16(1 << (bd - 1)); - const uint8x16x2_t permute_tbl = vld1q_u8_x2(kMatMulPermuteTbl); + const uint8x16x2_t permute_tbl = vld1q_u8_x2(kMatMul6PermuteTbl); do { const uint8_t *s = src; @@ -1129,7 +1250,7 @@ const int16x8_t vert_const = vdupq_n_s16(1 << (bd - 1)); if (w == 4) { - const uint8x16_t permute_tbl = vld1q_u8(kMatMulPermuteTbl); + const uint8x16_t permute_tbl = vld1q_u8(kMatMul6PermuteTbl); uint8x16_t h_s0, h_s1, h_s2; load_u8_16x3(src, src_stride, &h_s0, &h_s1, &h_s2); @@ -1172,7 +1293,7 @@ h -= 4; } while (h != 0); } else { - const uint8x16x2_t permute_tbl = vld1q_u8_x2(kMatMulPermuteTbl); + const uint8x16x2_t permute_tbl = vld1q_u8_x2(kMatMul6PermuteTbl); do { int height = h;
diff --git a/av1/common/arm/convolve_neon_i8mm.h b/av1/common/arm/convolve_neon_i8mm.h index 71b7461..38e651f 100644 --- a/av1/common/arm/convolve_neon_i8mm.h +++ b/av1/common/arm/convolve_neon_i8mm.h
@@ -23,18 +23,9 @@ #include "aom_dsp/arm/mem_neon.h" #include "aom_ports/mem.h" -DECLARE_ALIGNED(16, static const uint8_t, kDotProdPermuteTbl[48]) = { - 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6, - 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10, - 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 -}; - -DECLARE_ALIGNED(16, static const uint8_t, kMatMulPermuteTbl[32]) = { - // clang-format off - 0, 1, 2, 3, 4, 5, 6, 7, 2, 3, 4, 5, 6, 7, 8, 9, - 4, 5, 6, 7, 8, 9, 10, 11, 6, 7, 8, 9, 10, 11, 12, 13 - // clang-format on -}; +DECLARE_ALIGNED(16, extern const uint8_t, kMatMul6PermuteTbl[32]); +DECLARE_ALIGNED(16, extern const uint8_t, kMatMul8PermuteTbl[32]); +DECLARE_ALIGNED(16, extern const uint8_t, kFilterPermuteTbl[16]); static inline int16x4_t convolve12_4_2d_h(uint8x16_t samples[2], const int8x16_t filter[2], @@ -110,7 +101,7 @@ vdupq_n_s32((1 << (bd + FILTER_BITS - 1)) + (1 << (ROUND0_BITS - 1))); if (w <= 4) { - const uint8x16_t permute_tbl = vld1q_u8(kMatMulPermuteTbl); + const uint8x16_t permute_tbl = vld1q_u8(kMatMul6PermuteTbl); do { uint8x16_t s0[2], s1[2], s2[2], s3[2]; @@ -141,7 +132,7 @@ } while (--h != 0); } else { - const uint8x16x2_t permute_tbl = vld1q_u8_x2(kMatMulPermuteTbl); + const uint8x16x2_t permute_tbl = vld1q_u8_x2(kMatMul6PermuteTbl); do { const uint8_t *s = src_ptr;
diff --git a/av1/common/arm/convolve_sve2.c b/av1/common/arm/convolve_sve2.c index 536f441..ae00386 100644 --- a/av1/common/arm/convolve_sve2.c +++ b/av1/common/arm/convolve_sve2.c
@@ -23,9 +23,20 @@ #include "aom_dsp/arm/mem_neon.h" #include "aom_dsp/arm/transpose_neon.h" #include "aom_ports/mem.h" -#include "av1/common/arm/highbd_convolve_sve2.h" +#include "av1/common/arm/convolve_sve2.h" #include "av1/common/arm/convolve_neon_i8mm.h" +// clang-format off +DECLARE_ALIGNED(16, const uint16_t, kSVEDotProdMergeBlockTbl[24]) = { + // Shift left and insert new last column in transposed 4x4 block. + 1, 2, 3, 0, 5, 6, 7, 4, + // Shift left and insert two new columns in transposed 4x4 block. + 2, 3, 0, 1, 6, 7, 4, 5, + // Shift left and insert three new columns in transposed 4x4 block. + 3, 0, 1, 2, 7, 4, 5, 6, +}; +// clang-format on + static inline int32x4_t highbd_convolve12_4_2d_v(int16x8_t s0[2], int16x8_t s1[2], int16x8_t s2[2], @@ -52,7 +63,7 @@ const int bd = 8; const int16x8_t sub_const = vdupq_n_s16(1 << (bd - 1)); - uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kDotProdMergeBlockTbl); + uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kSVEDotProdMergeBlockTbl); // Scale indices by size of the true vector length to avoid reading from an // 'undefined' portion of a vector on a system with SVE vectors > 128-bit. uint16x8_t correction0 = @@ -81,21 +92,21 @@ s6789[2], s789A[2]; // This operation combines a conventional transpose and the sample permute // required before computing the dot product. - transpose_concat_4x4(s0, s1, s2, s3, s0123); - transpose_concat_4x4(s1, s2, s3, s4, s1234); - transpose_concat_4x4(s2, s3, s4, s5, s2345); - transpose_concat_4x4(s3, s4, s5, s6, s3456); - transpose_concat_4x4(s4, s5, s6, s7, s4567); - transpose_concat_4x4(s5, s6, s7, s8, s5678); - transpose_concat_4x4(s6, s7, s8, s9, s6789); - transpose_concat_4x4(s7, s8, s9, sA, s789A); + transpose_concat_elems_s16_4x4(s0, s1, s2, s3, s0123); + transpose_concat_elems_s16_4x4(s1, s2, s3, s4, s1234); + transpose_concat_elems_s16_4x4(s2, s3, s4, s5, s2345); + transpose_concat_elems_s16_4x4(s3, s4, s5, s6, s3456); + transpose_concat_elems_s16_4x4(s4, s5, s6, s7, s4567); + transpose_concat_elems_s16_4x4(s5, s6, s7, s8, s5678); + transpose_concat_elems_s16_4x4(s6, s7, s8, s9, s6789); + transpose_concat_elems_s16_4x4(s7, s8, s9, sA, s789A); do { int16x4_t sB, sC, sD, sE; load_s16_4x4(s, src_stride, &sB, &sC, &sD, &sE); int16x8_t s89AB[2], s9ABC[2], sABCD[2], sBCDE[2]; - transpose_concat_4x4(sB, sC, sD, sE, sBCDE); + transpose_concat_elems_s16_4x4(sB, sC, sD, sE, sBCDE); // Merge new data into block from previous iteration. aom_tbl2x2_s16(s789A, sBCDE, merge_block_tbl.val[0], s89AB);
diff --git a/av1/common/arm/convolve_sve2.h b/av1/common/arm/convolve_sve2.h new file mode 100644 index 0000000..bd02153 --- /dev/null +++ b/av1/common/arm/convolve_sve2.h
@@ -0,0 +1,26 @@ +/* + * Copyright (c) 2025, Alliance for Open Media. All rights reserved. + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_AV1_COMMON_ARM_CONVOLVE_SVE2_H_ +#define AOM_AV1_COMMON_ARM_CONVOLVE_SVE2_H_ + +#include "aom_dsp/arm/aom_neon_sve2_bridge.h" +#include "aom_ports/mem.h" + +DECLARE_ALIGNED(16, extern const uint16_t, kSVEDotProdMergeBlockTbl[24]); + +static inline void aom_tbl2x2_s16(int16x8_t t0[2], int16x8_t t1[2], + uint16x8_t tbl, int16x8_t res[2]) { + res[0] = aom_tbl2_s16(t0[0], t1[0], tbl); + res[1] = aom_tbl2_s16(t0[1], t1[1], tbl); +} + +#endif // AOM_AV1_COMMON_ARM_CONVOLVE_SVE2_H_
diff --git a/av1/common/arm/highbd_compound_convolve_sve2.c b/av1/common/arm/highbd_compound_convolve_sve2.c index 668dfbf..e5fbe41 100644 --- a/av1/common/arm/highbd_compound_convolve_sve2.c +++ b/av1/common/arm/highbd_compound_convolve_sve2.c
@@ -18,20 +18,17 @@ #include "aom_dsp/aom_dsp_common.h" #include "aom_dsp/arm/aom_neon_sve_bridge.h" #include "aom_dsp/arm/aom_neon_sve2_bridge.h" +#include "aom_dsp/arm/highbd_convolve8_sve.h" #include "aom_dsp/arm/mem_neon.h" #include "aom_ports/mem.h" #include "av1/common/convolve.h" #include "av1/common/filter.h" #include "av1/common/filter.h" +#include "av1/common/arm/convolve_sve2.h" #include "av1/common/arm/highbd_compound_convolve_neon.h" #include "av1/common/arm/highbd_convolve_neon.h" #include "av1/common/arm/highbd_convolve_sve2.h" -DECLARE_ALIGNED(16, static const uint16_t, kDotProdTbl[32]) = { - 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6, - 4, 5, 6, 7, 5, 6, 7, 0, 6, 7, 0, 1, 7, 0, 1, 2, -}; - static inline uint16x8_t highbd_12_convolve8_8_x(int16x8_t s0[8], int16x8_t filter, int64x2_t offset) { @@ -165,12 +162,6 @@ } while (height != 0); } -// clang-format off -DECLARE_ALIGNED(16, static const uint16_t, kDeinterleaveTbl[8]) = { - 0, 2, 4, 6, 1, 3, 5, 7, -}; -// clang-format on - static inline uint16x4_t highbd_12_convolve4_4_x(int16x8_t s0, int16x8_t filter, int64x2_t offset, uint16x8x2_t permute_tbl) { @@ -212,7 +203,7 @@ const int16x8_t filter = vcombine_s16(x_filter, vdup_n_s16(0)); if (width == 4) { - uint16x8x2_t permute_tbl = vld1q_u16_x2(kDotProdTbl); + uint16x8x2_t permute_tbl = vld1q_u16_x2(kHbdDotProdTbl); const int16_t *s = (const int16_t *)(src); @@ -304,7 +295,7 @@ const int16x8_t filter = vcombine_s16(x_filter, vdup_n_s16(0)); if (width == 4) { - uint16x8x2_t permute_tbl = vld1q_u16_x2(kDotProdTbl); + uint16x8x2_t permute_tbl = vld1q_u16_x2(kHbdDotProdTbl); const int16_t *s = (const int16_t *)(src); @@ -483,7 +474,7 @@ vdupq_n_s64((1 << (12 + FILTER_BITS)) + (1 << (12 + FILTER_BITS - 1))); const int16x8_t y_filter = vld1q_s16(y_filter_ptr); - uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kDotProdMergeBlockTbl); + uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kSVEDotProdMergeBlockTbl); // Scale indices by size of the true vector length to avoid reading from an // 'undefined' portion of a vector on a system with SVE vectors > 128-bit. uint16x8_t correction0 = @@ -506,10 +497,10 @@ // This operation combines a conventional transpose and the sample permute // required before computing the dot product. int16x8_t s0123[2], s1234[2], s2345[2], s3456[2]; - transpose_concat_4x4(s0, s1, s2, s3, s0123); - transpose_concat_4x4(s1, s2, s3, s4, s1234); - transpose_concat_4x4(s2, s3, s4, s5, s2345); - transpose_concat_4x4(s3, s4, s5, s6, s3456); + transpose_concat_elems_s16_4x4(s0, s1, s2, s3, s0123); + transpose_concat_elems_s16_4x4(s1, s2, s3, s4, s1234); + transpose_concat_elems_s16_4x4(s2, s3, s4, s5, s2345); + transpose_concat_elems_s16_4x4(s3, s4, s5, s6, s3456); do { int16x4_t s7, s8, s9, s10; @@ -517,7 +508,7 @@ int16x8_t s4567[2], s5678[2], s6789[2], s789A[2]; // Transpose and shuffle the 4 lines that were loaded. - transpose_concat_4x4(s7, s8, s9, s10, s789A); + transpose_concat_elems_s16_4x4(s7, s8, s9, s10, s789A); // Merge new data into block from previous iteration. aom_tbl2x2_s16(s3456, s789A, merge_block_tbl.val[0], s4567); @@ -559,10 +550,10 @@ // This operation combines a conventional transpose and the sample permute // required before computing the dot product. int16x8_t s0123[4], s1234[4], s2345[4], s3456[4]; - transpose_concat_8x4(s0, s1, s2, s3, s0123); - transpose_concat_8x4(s1, s2, s3, s4, s1234); - transpose_concat_8x4(s2, s3, s4, s5, s2345); - transpose_concat_8x4(s3, s4, s5, s6, s3456); + transpose_concat_elems_s16_8x4(s0, s1, s2, s3, s0123); + transpose_concat_elems_s16_8x4(s1, s2, s3, s4, s1234); + transpose_concat_elems_s16_8x4(s2, s3, s4, s5, s2345); + transpose_concat_elems_s16_8x4(s3, s4, s5, s6, s3456); do { int16x8_t s7, s8, s9, s10; @@ -570,7 +561,7 @@ int16x8_t s4567[4], s5678[4], s6789[4], s789A[4]; // Transpose and shuffle the 4 lines that were loaded. - transpose_concat_8x4(s7, s8, s9, s10, s789A); + transpose_concat_elems_s16_8x4(s7, s8, s9, s10, s789A); // Merge new data into block from previous iteration. aom_tbl2x4_s16(s3456, s789A, merge_block_tbl.val[0], s4567); @@ -659,7 +650,7 @@ vdupq_n_s64((1 << (bd + FILTER_BITS)) + (1 << (bd + FILTER_BITS - 1))); const int16x8_t y_filter = vld1q_s16(y_filter_ptr); - uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kDotProdMergeBlockTbl); + uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kSVEDotProdMergeBlockTbl); // Scale indices by size of the true vector length to avoid reading from an // 'undefined' portion of a vector on a system with SVE vectors > 128-bit. uint16x8_t correction0 = @@ -682,10 +673,10 @@ // This operation combines a conventional transpose and the sample permute // required before computing the dot product. int16x8_t s0123[2], s1234[2], s2345[2], s3456[2]; - transpose_concat_4x4(s0, s1, s2, s3, s0123); - transpose_concat_4x4(s1, s2, s3, s4, s1234); - transpose_concat_4x4(s2, s3, s4, s5, s2345); - transpose_concat_4x4(s3, s4, s5, s6, s3456); + transpose_concat_elems_s16_4x4(s0, s1, s2, s3, s0123); + transpose_concat_elems_s16_4x4(s1, s2, s3, s4, s1234); + transpose_concat_elems_s16_4x4(s2, s3, s4, s5, s2345); + transpose_concat_elems_s16_4x4(s3, s4, s5, s6, s3456); do { int16x4_t s7, s8, s9, s10; @@ -693,7 +684,7 @@ int16x8_t s4567[2], s5678[2], s6789[2], s789A[2]; // Transpose and shuffle the 4 lines that were loaded. - transpose_concat_4x4(s7, s8, s9, s10, s789A); + transpose_concat_elems_s16_4x4(s7, s8, s9, s10, s789A); // Merge new data into block from previous iteration. aom_tbl2x2_s16(s3456, s789A, merge_block_tbl.val[0], s4567); @@ -735,10 +726,10 @@ // This operation combines a conventional transpose and the sample permute // required before computing the dot product. int16x8_t s0123[4], s1234[4], s2345[4], s3456[4]; - transpose_concat_8x4(s0, s1, s2, s3, s0123); - transpose_concat_8x4(s1, s2, s3, s4, s1234); - transpose_concat_8x4(s2, s3, s4, s5, s2345); - transpose_concat_8x4(s3, s4, s5, s6, s3456); + transpose_concat_elems_s16_8x4(s0, s1, s2, s3, s0123); + transpose_concat_elems_s16_8x4(s1, s2, s3, s4, s1234); + transpose_concat_elems_s16_8x4(s2, s3, s4, s5, s2345); + transpose_concat_elems_s16_8x4(s3, s4, s5, s6, s3456); do { int16x8_t s7, s8, s9, s10; @@ -746,7 +737,7 @@ int16x8_t s4567[4], s5678[4], s6789[4], s789A[4]; // Transpose and shuffle the 4 lines that were loaded. - transpose_concat_8x4(s7, s8, s9, s10, s789A); + transpose_concat_elems_s16_8x4(s7, s8, s9, s10, s789A); // Merge new data into block from previous iteration. aom_tbl2x4_s16(s3456, s789A, merge_block_tbl.val[0], s4567); @@ -989,7 +980,7 @@ // a time and then process the last 3 rows separately. if (width == 4) { - uint16x8x2_t permute_tbl = vld1q_u16_x2(kDotProdTbl); + uint16x8x2_t permute_tbl = vld1q_u16_x2(kHbdDotProdTbl); const int16_t *s = (const int16_t *)(src); @@ -1084,7 +1075,7 @@ // a time and then process the last 3 rows separately. if (width == 4) { - uint16x8x2_t permute_tbl = vld1q_u16_x2(kDotProdTbl); + uint16x8x2_t permute_tbl = vld1q_u16_x2(kHbdDotProdTbl); const int16_t *s = (const int16_t *)(src); @@ -1210,7 +1201,7 @@ const int16x8_t y_filter = vld1q_s16(y_filter_ptr); const int64x2_t offset_s64 = vdupq_n_s64(offset); - uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kDotProdMergeBlockTbl); + uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kSVEDotProdMergeBlockTbl); // Scale indices by size of the true vector length to avoid reading from an // 'undefined' portion of a vector on a system with SVE vectors > 128-bit. uint16x8_t correction0 = @@ -1234,10 +1225,10 @@ // This operation combines a conventional transpose and the sample permute // required before computing the dot product. int16x8_t s0123[2], s1234[2], s2345[2], s3456[2]; - transpose_concat_4x4(s0, s1, s2, s3, s0123); - transpose_concat_4x4(s1, s2, s3, s4, s1234); - transpose_concat_4x4(s2, s3, s4, s5, s2345); - transpose_concat_4x4(s3, s4, s5, s6, s3456); + transpose_concat_elems_s16_4x4(s0, s1, s2, s3, s0123); + transpose_concat_elems_s16_4x4(s1, s2, s3, s4, s1234); + transpose_concat_elems_s16_4x4(s2, s3, s4, s5, s2345); + transpose_concat_elems_s16_4x4(s3, s4, s5, s6, s3456); do { int16x4_t s7, s8, s9, s10; @@ -1245,7 +1236,7 @@ int16x8_t s4567[2], s5678[2], s6789[2], s789A[2]; // Transpose and shuffle the 4 lines that were loaded. - transpose_concat_4x4(s7, s8, s9, s10, s789A); + transpose_concat_elems_s16_4x4(s7, s8, s9, s10, s789A); // Merge new data into block from previous iteration. aom_tbl2x2_s16(s3456, s789A, merge_block_tbl.val[0], s4567); @@ -1291,10 +1282,10 @@ // This operation combines a conventional transpose and the sample permute // required before computing the dot product. int16x8_t s0123[4], s1234[4], s2345[4], s3456[4]; - transpose_concat_8x4(s0, s1, s2, s3, s0123); - transpose_concat_8x4(s1, s2, s3, s4, s1234); - transpose_concat_8x4(s2, s3, s4, s5, s2345); - transpose_concat_8x4(s3, s4, s5, s6, s3456); + transpose_concat_elems_s16_8x4(s0, s1, s2, s3, s0123); + transpose_concat_elems_s16_8x4(s1, s2, s3, s4, s1234); + transpose_concat_elems_s16_8x4(s2, s3, s4, s5, s2345); + transpose_concat_elems_s16_8x4(s3, s4, s5, s6, s3456); do { int16x8_t s7, s8, s9, s10; @@ -1302,7 +1293,7 @@ int16x8_t s4567[4], s5678[4], s6789[4], s789A[4]; // Transpose and shuffle the 4 lines that were loaded. - transpose_concat_8x4(s7, s8, s9, s10, s789A); + transpose_concat_elems_s16_8x4(s7, s8, s9, s10, s789A); // Merge new data into block from previous iteration. aom_tbl2x4_s16(s3456, s789A, merge_block_tbl.val[0], s4567);
diff --git a/av1/common/arm/highbd_convolve_sve2.c b/av1/common/arm/highbd_convolve_sve2.c index fcf9d7b..656d8d5 100644 --- a/av1/common/arm/highbd_convolve_sve2.c +++ b/av1/common/arm/highbd_convolve_sve2.c
@@ -18,17 +18,15 @@ #include "aom_dsp/aom_dsp_common.h" #include "aom_dsp/arm/aom_neon_sve_bridge.h" #include "aom_dsp/arm/aom_neon_sve2_bridge.h" +#include "aom_dsp/arm/highbd_convolve8_sve.h" #include "aom_dsp/arm/mem_neon.h" +#include "aom_dsp/arm/transpose_neon.h" #include "aom_ports/mem.h" #include "av1/common/convolve.h" #include "av1/common/filter.h" +#include "av1/common/arm/convolve_sve2.h" #include "av1/common/arm/highbd_convolve_sve2.h" -DECLARE_ALIGNED(16, static const uint16_t, kDotProdTbl[32]) = { - 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6, - 4, 5, 6, 7, 5, 6, 7, 0, 6, 7, 0, 1, 7, 0, 1, 2, -}; - static inline uint16x4_t convolve12_4_x( int16x8_t s0, int16x8_t s1, int16x8_t filter_0_7, int16x8_t filter_4_11, const int64x2_t offset, uint16x8x4_t permute_tbl, uint16x4_t max) { @@ -110,7 +108,7 @@ const int16x8_t y_filter_0_7 = vld1q_s16(y_filter_ptr); const int16x8_t y_filter_4_11 = vld1q_s16(y_filter_ptr + 4); - uint16x8x4_t permute_tbl = vld1q_u16_x4(kDotProdTbl); + uint16x8x4_t permute_tbl = vld1q_u16_x4(kHbdDotProdTbl); // Scale indices by size of the true vector length to avoid reading from an // 'undefined' portion of a vector on a system with SVE vectors > 128-bit. uint16x8_t correction0 = vreinterpretq_u16_u64(vcombine_u64( @@ -252,12 +250,6 @@ } while (height != 0); } -// clang-format off -DECLARE_ALIGNED(16, static const uint16_t, kDeinterleaveTbl[8]) = { - 0, 2, 4, 6, 1, 3, 5, 7, -}; -// clang-format on - static inline uint16x4_t convolve4_4_x(int16x8_t s0, int16x8_t filter, int64x2_t offset, uint16x8x2_t permute_tbl, @@ -304,7 +296,7 @@ if (width == 4) { const uint16x4_t max = vdup_n_u16((1 << bd) - 1); - uint16x8x2_t permute_tbl = vld1q_u16_x2(kDotProdTbl); + uint16x8x2_t permute_tbl = vld1q_u16_x2(kHbdDotProdTbl); const int16_t *s = (const int16_t *)(src); @@ -427,21 +419,6 @@ const int16x8_t y_filter_0_7 = vld1q_s16(y_filter_ptr); const int16x8_t y_filter_4_11 = vld1q_s16(y_filter_ptr + 4); - uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kDotProdMergeBlockTbl); - // Scale indices by size of the true vector length to avoid reading from an - // 'undefined' portion of a vector on a system with SVE vectors > 128-bit. - uint16x8_t correction0 = - vreinterpretq_u16_u64(vdupq_n_u64(svcnth() * 0x0001000000000000ULL)); - merge_block_tbl.val[0] = vaddq_u16(merge_block_tbl.val[0], correction0); - - uint16x8_t correction1 = - vreinterpretq_u16_u64(vdupq_n_u64(svcnth() * 0x0001000100000000ULL)); - merge_block_tbl.val[1] = vaddq_u16(merge_block_tbl.val[1], correction1); - - uint16x8_t correction2 = - vreinterpretq_u16_u64(vdupq_n_u64(svcnth() * 0x0001000100010000ULL)); - merge_block_tbl.val[2] = vaddq_u16(merge_block_tbl.val[2], correction2); - const uint16x4_t max = vdup_n_u16((1 << bd) - 1); do { @@ -456,27 +433,24 @@ int16x8_t s0123[2], s1234[2], s2345[2], s3456[2], s4567[2], s5678[2], s6789[2], s789A[2]; - transpose_concat_4x4(s0, s1, s2, s3, s0123); - transpose_concat_4x4(s1, s2, s3, s4, s1234); - transpose_concat_4x4(s2, s3, s4, s5, s2345); - transpose_concat_4x4(s3, s4, s5, s6, s3456); - transpose_concat_4x4(s4, s5, s6, s7, s4567); - transpose_concat_4x4(s5, s6, s7, s8, s5678); - transpose_concat_4x4(s6, s7, s8, s9, s6789); - transpose_concat_4x4(s7, s8, s9, sA, s789A); + transpose_concat_elems_s16_4x4(s0, s1, s2, s3, s0123); + transpose_concat_elems_s16_4x4(s1, s2, s3, s4, s1234); + transpose_concat_elems_s16_4x4(s2, s3, s4, s5, s2345); + transpose_concat_elems_s16_4x4(s3, s4, s5, s6, s3456); + transpose_concat_elems_s16_4x4(s4, s5, s6, s7, s4567); + transpose_concat_elems_s16_4x4(s5, s6, s7, s8, s5678); + transpose_concat_elems_s16_4x4(s6, s7, s8, s9, s6789); + transpose_concat_elems_s16_4x4(s7, s8, s9, sA, s789A); do { int16x4_t sB, sC, sD, sE; load_s16_4x4(s, src_stride, &sB, &sC, &sD, &sE); int16x8_t s89AB[2], s9ABC[2], sABCD[2], sBCDE[2]; - transpose_concat_4x4(sB, sC, sD, sE, sBCDE); - - // Use the above transpose and reuse data from the previous loop to get - // the rest. - aom_tbl2x2_s16(s789A, sBCDE, merge_block_tbl.val[0], s89AB); - aom_tbl2x2_s16(s789A, sBCDE, merge_block_tbl.val[1], s9ABC); - aom_tbl2x2_s16(s789A, sBCDE, merge_block_tbl.val[2], sABCD); + transpose_concat_elems_s16_4x4(s8, s9, sA, sB, s89AB); + transpose_concat_elems_s16_4x4(s9, sA, sB, sC, s9ABC); + transpose_concat_elems_s16_4x4(sA, sB, sC, sD, sABCD); + transpose_concat_elems_s16_4x4(sB, sC, sD, sE, sBCDE); uint16x4_t d0 = highbd_convolve12_4_y(s0123, s4567, s89AB, y_filter_0_7, y_filter_4_11, max); @@ -508,6 +482,10 @@ s789A[0] = sBCDE[0]; s789A[1] = sBCDE[1]; + s8 = sC; + s9 = sD; + sA = sE; + s += 4 * src_stride; d += 4 * dst_stride; h -= 4; @@ -571,7 +549,7 @@ const int16x8_t y_filter = vld1q_s16(filter_y); - uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kDotProdMergeBlockTbl); + uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kSVEDotProdMergeBlockTbl); // Scale indices by size of the true vector length to avoid reading from an // 'undefined' portion of a vector on a system with SVE vectors > 128-bit. uint16x8_t correction0 = @@ -597,10 +575,10 @@ // This operation combines a conventional transpose and the sample permute // required before computing the dot product. int16x8_t s0123[2], s1234[2], s2345[2], s3456[2]; - transpose_concat_4x4(s0, s1, s2, s3, s0123); - transpose_concat_4x4(s1, s2, s3, s4, s1234); - transpose_concat_4x4(s2, s3, s4, s5, s2345); - transpose_concat_4x4(s3, s4, s5, s6, s3456); + transpose_concat_elems_s16_4x4(s0, s1, s2, s3, s0123); + transpose_concat_elems_s16_4x4(s1, s2, s3, s4, s1234); + transpose_concat_elems_s16_4x4(s2, s3, s4, s5, s2345); + transpose_concat_elems_s16_4x4(s3, s4, s5, s6, s3456); do { int16x4_t s7, s8, s9, s10; @@ -608,7 +586,7 @@ int16x8_t s4567[2], s5678[2], s6789[2], s789A[2]; // Transpose and shuffle the 4 lines that were loaded. - transpose_concat_4x4(s7, s8, s9, s10, s789A); + transpose_concat_elems_s16_4x4(s7, s8, s9, s10, s789A); // Merge new data into block from previous iteration. aom_tbl2x2_s16(s3456, s789A, merge_block_tbl.val[0], s4567); @@ -651,10 +629,10 @@ // This operation combines a conventional transpose and the sample permute // required before computing the dot product. int16x8_t s0123[4], s1234[4], s2345[4], s3456[4]; - transpose_concat_8x4(s0, s1, s2, s3, s0123); - transpose_concat_8x4(s1, s2, s3, s4, s1234); - transpose_concat_8x4(s2, s3, s4, s5, s2345); - transpose_concat_8x4(s3, s4, s5, s6, s3456); + transpose_concat_elems_s16_8x4(s0, s1, s2, s3, s0123); + transpose_concat_elems_s16_8x4(s1, s2, s3, s4, s1234); + transpose_concat_elems_s16_8x4(s2, s3, s4, s5, s2345); + transpose_concat_elems_s16_8x4(s3, s4, s5, s6, s3456); do { int16x8_t s7, s8, s9, s10; @@ -662,7 +640,7 @@ int16x8_t s4567[4], s5678[4], s6789[4], s789A[4]; // Transpose and shuffle the 4 lines that were loaded. - transpose_concat_8x4(s7, s8, s9, s10, s789A); + transpose_concat_elems_s16_8x4(s7, s8, s9, s10, s789A); // Merge new data into block from previous iteration. aom_tbl2x4_s16(s3456, s789A, merge_block_tbl.val[0], s4567); @@ -757,10 +735,10 @@ // This operation combines a conventional transpose and the sample permute // required before computing the dot product. int16x8_t s0123[2], s1234[2], s2345[2], s3456[2]; - transpose_concat_4x4(s0, s1, s2, s3, s0123); - transpose_concat_4x4(s1, s2, s3, s4, s1234); - transpose_concat_4x4(s2, s3, s4, s5, s2345); - transpose_concat_4x4(s3, s4, s5, s6, s3456); + transpose_concat_elems_s16_4x4(s0, s1, s2, s3, s0123); + transpose_concat_elems_s16_4x4(s1, s2, s3, s4, s1234); + transpose_concat_elems_s16_4x4(s2, s3, s4, s5, s2345); + transpose_concat_elems_s16_4x4(s3, s4, s5, s6, s3456); uint16x4_t d0 = highbd_convolve4_4_y(s0123, y_filter, max); uint16x4_t d1 = highbd_convolve4_4_y(s1234, y_filter, max); @@ -797,10 +775,10 @@ // This operation combines a conventional transpose and the sample // permute required before computing the dot product. int16x8_t s0123[4], s1234[4], s2345[4], s3456[4]; - transpose_concat_8x4(s0, s1, s2, s3, s0123); - transpose_concat_8x4(s1, s2, s3, s4, s1234); - transpose_concat_8x4(s2, s3, s4, s5, s2345); - transpose_concat_8x4(s3, s4, s5, s6, s3456); + transpose_concat_elems_s16_8x4(s0, s1, s2, s3, s0123); + transpose_concat_elems_s16_8x4(s1, s2, s3, s4, s1234); + transpose_concat_elems_s16_8x4(s2, s3, s4, s5, s2345); + transpose_concat_elems_s16_8x4(s3, s4, s5, s6, s3456); uint16x8_t d0 = highbd_convolve4_8_y(s0123, y_filter, max); uint16x8_t d1 = highbd_convolve4_8_y(s1234, y_filter, max); @@ -944,7 +922,7 @@ const int16x8_t y_filter_0_7 = vld1q_s16(y_filter_ptr); const int16x8_t y_filter_4_11 = vld1q_s16(y_filter_ptr + 4); - uint16x8x4_t permute_tbl = vld1q_u16_x4(kDotProdTbl); + uint16x8x4_t permute_tbl = vld1q_u16_x4(kHbdDotProdTbl); // Scale indices by size of the true vector length to avoid reading from an // 'undefined' portion of a vector on a system with SVE vectors > 128-bit. uint16x8_t correction0 = vreinterpretq_u16_u64(vcombine_u64( @@ -1131,7 +1109,7 @@ if (width == 4) { const int16_t *s = (const int16_t *)(src); - uint16x8x2_t permute_tbl = vld1q_u16_x2(kDotProdTbl); + uint16x8x2_t permute_tbl = vld1q_u16_x2(kHbdDotProdTbl); do { int16x8_t s0, s1, s2, s3; @@ -1214,7 +1192,7 @@ const int16x8_t y_filter_0_7 = vld1q_s16(y_filter_ptr); const int16x8_t y_filter_4_11 = vld1q_s16(y_filter_ptr + 4); - uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kDotProdMergeBlockTbl); + uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kSVEDotProdMergeBlockTbl); // Scale indices by size of the true vector length to avoid reading from an // 'undefined' portion of a vector on a system with SVE vectors > 128-bit. uint16x8_t correction0 = @@ -1245,21 +1223,21 @@ s6789[2], s789A[2]; // This operation combines a conventional transpose and the sample permute // required before computing the dot product. - transpose_concat_4x4(s0, s1, s2, s3, s0123); - transpose_concat_4x4(s1, s2, s3, s4, s1234); - transpose_concat_4x4(s2, s3, s4, s5, s2345); - transpose_concat_4x4(s3, s4, s5, s6, s3456); - transpose_concat_4x4(s4, s5, s6, s7, s4567); - transpose_concat_4x4(s5, s6, s7, s8, s5678); - transpose_concat_4x4(s6, s7, s8, s9, s6789); - transpose_concat_4x4(s7, s8, s9, sA, s789A); + transpose_concat_elems_s16_4x4(s0, s1, s2, s3, s0123); + transpose_concat_elems_s16_4x4(s1, s2, s3, s4, s1234); + transpose_concat_elems_s16_4x4(s2, s3, s4, s5, s2345); + transpose_concat_elems_s16_4x4(s3, s4, s5, s6, s3456); + transpose_concat_elems_s16_4x4(s4, s5, s6, s7, s4567); + transpose_concat_elems_s16_4x4(s5, s6, s7, s8, s5678); + transpose_concat_elems_s16_4x4(s6, s7, s8, s9, s6789); + transpose_concat_elems_s16_4x4(s7, s8, s9, sA, s789A); do { int16x4_t sB, sC, sD, sE; load_s16_4x4(s, src_stride, &sB, &sC, &sD, &sE); int16x8_t s89AB[2], s9ABC[2], sABCD[2], sBCDE[2]; - transpose_concat_4x4(sB, sC, sD, sE, sBCDE); + transpose_concat_elems_s16_4x4(sB, sC, sD, sE, sBCDE); // Use the above transpose and reuse data from the previous loop to get // the rest. @@ -1357,7 +1335,7 @@ const int32x4_t shift = vdupq_n_s32(-conv_params->round_1); const int16x8_t y_filter = vld1q_s16(filter_y); - uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kDotProdMergeBlockTbl); + uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kSVEDotProdMergeBlockTbl); // Scale indices by size of the true vector length to avoid reading from an // 'undefined' portion of a vector on a system with SVE vectors > 128-bit. uint16x8_t correction0 = @@ -1383,10 +1361,10 @@ // This operation combines a conventional transpose and the sample permute // required before computing the dot product. int16x8_t s0123[2], s1234[2], s2345[2], s3456[2]; - transpose_concat_4x4(s0, s1, s2, s3, s0123); - transpose_concat_4x4(s1, s2, s3, s4, s1234); - transpose_concat_4x4(s2, s3, s4, s5, s2345); - transpose_concat_4x4(s3, s4, s5, s6, s3456); + transpose_concat_elems_s16_4x4(s0, s1, s2, s3, s0123); + transpose_concat_elems_s16_4x4(s1, s2, s3, s4, s1234); + transpose_concat_elems_s16_4x4(s2, s3, s4, s5, s2345); + transpose_concat_elems_s16_4x4(s3, s4, s5, s6, s3456); do { int16x4_t s7, s8, s9, s10; @@ -1394,7 +1372,7 @@ int16x8_t s4567[2], s5678[2], s6789[2], s789A[2]; // Transpose and shuffle the 4 lines that were loaded. - transpose_concat_4x4(s7, s8, s9, s10, s789A); + transpose_concat_elems_s16_4x4(s7, s8, s9, s10, s789A); // Merge new data into block from previous iteration. aom_tbl2x2_s16(s3456, s789A, merge_block_tbl.val[0], s4567); @@ -1442,10 +1420,10 @@ // This operation combines a conventional transpose and the sample permute // required before computing the dot product. int16x8_t s0123[4], s1234[4], s2345[4], s3456[4]; - transpose_concat_8x4(s0, s1, s2, s3, s0123); - transpose_concat_8x4(s1, s2, s3, s4, s1234); - transpose_concat_8x4(s2, s3, s4, s5, s2345); - transpose_concat_8x4(s3, s4, s5, s6, s3456); + transpose_concat_elems_s16_8x4(s0, s1, s2, s3, s0123); + transpose_concat_elems_s16_8x4(s1, s2, s3, s4, s1234); + transpose_concat_elems_s16_8x4(s2, s3, s4, s5, s2345); + transpose_concat_elems_s16_8x4(s3, s4, s5, s6, s3456); do { int16x8_t s7, s8, s9, s10; @@ -1453,7 +1431,7 @@ int16x8_t s4567[4], s5678[4], s6789[4], s789A[4]; // Transpose and shuffle the 4 lines that were loaded. - transpose_concat_8x4(s7, s8, s9, s10, s789A); + transpose_concat_elems_s16_8x4(s7, s8, s9, s10, s789A); // Merge new data into block from previous iteration. aom_tbl2x4_s16(s3456, s789A, merge_block_tbl.val[0], s4567); @@ -1562,10 +1540,10 @@ // This operation combines a conventional transpose and the sample permute // required before computing the dot product. int16x8_t s0123[2], s1234[2], s2345[2], s3456[2]; - transpose_concat_4x4(s0, s1, s2, s3, s0123); - transpose_concat_4x4(s1, s2, s3, s4, s1234); - transpose_concat_4x4(s2, s3, s4, s5, s2345); - transpose_concat_4x4(s3, s4, s5, s6, s3456); + transpose_concat_elems_s16_4x4(s0, s1, s2, s3, s0123); + transpose_concat_elems_s16_4x4(s1, s2, s3, s4, s1234); + transpose_concat_elems_s16_4x4(s2, s3, s4, s5, s2345); + transpose_concat_elems_s16_4x4(s3, s4, s5, s6, s3456); uint16x4_t d0 = highbd_convolve4_4_2d_v(s0123, y_filter, shift, offset, max); @@ -1606,10 +1584,10 @@ // This operation combines a conventional transpose and the sample // permute required before computing the dot product. int16x8_t s0123[4], s1234[4], s2345[4], s3456[4]; - transpose_concat_8x4(s0, s1, s2, s3, s0123); - transpose_concat_8x4(s1, s2, s3, s4, s1234); - transpose_concat_8x4(s2, s3, s4, s5, s2345); - transpose_concat_8x4(s3, s4, s5, s6, s3456); + transpose_concat_elems_s16_8x4(s0, s1, s2, s3, s0123); + transpose_concat_elems_s16_8x4(s1, s2, s3, s4, s1234); + transpose_concat_elems_s16_8x4(s2, s3, s4, s5, s2345); + transpose_concat_elems_s16_8x4(s3, s4, s5, s6, s3456); uint16x8_t d0 = highbd_convolve4_8_2d_v(s0123, y_filter, shift, offset, max);
diff --git a/av1/common/arm/highbd_convolve_sve2.h b/av1/common/arm/highbd_convolve_sve2.h index abbad14..0b59e26 100644 --- a/av1/common/arm/highbd_convolve_sve2.h +++ b/av1/common/arm/highbd_convolve_sve2.h
@@ -16,70 +16,6 @@ #include "aom_dsp/arm/aom_neon_sve2_bridge.h" -// clang-format off -DECLARE_ALIGNED(16, static const uint16_t, kDotProdMergeBlockTbl[24]) = { - // Shift left and insert new last column in transposed 4x4 block. - 1, 2, 3, 0, 5, 6, 7, 4, - // Shift left and insert two new columns in transposed 4x4 block. - 2, 3, 0, 1, 6, 7, 4, 5, - // Shift left and insert three new columns in transposed 4x4 block. - 3, 0, 1, 2, 7, 4, 5, 6, -}; -// clang-format on - -static inline void transpose_concat_4x4(int16x4_t s0, int16x4_t s1, - int16x4_t s2, int16x4_t s3, - int16x8_t res[2]) { - // Transpose 16-bit elements and concatenate result rows as follows: - // s0: 00, 01, 02, 03 - // s1: 10, 11, 12, 13 - // s2: 20, 21, 22, 23 - // s3: 30, 31, 32, 33 - // - // res[0]: 00 10 20 30 01 11 21 31 - // res[1]: 02 12 22 32 03 13 23 33 - - int16x8_t s0q = vcombine_s16(s0, vdup_n_s16(0)); - int16x8_t s1q = vcombine_s16(s1, vdup_n_s16(0)); - int16x8_t s2q = vcombine_s16(s2, vdup_n_s16(0)); - int16x8_t s3q = vcombine_s16(s3, vdup_n_s16(0)); - - int32x4_t s01 = vreinterpretq_s32_s16(vzip1q_s16(s0q, s1q)); - int32x4_t s23 = vreinterpretq_s32_s16(vzip1q_s16(s2q, s3q)); - - int32x4x2_t s0123 = vzipq_s32(s01, s23); - - res[0] = vreinterpretq_s16_s32(s0123.val[0]); - res[1] = vreinterpretq_s16_s32(s0123.val[1]); -} - -static inline void transpose_concat_8x4(int16x8_t s0, int16x8_t s1, - int16x8_t s2, int16x8_t s3, - int16x8_t res[4]) { - // Transpose 16-bit elements and concatenate result rows as follows: - // s0: 00, 01, 02, 03, 04, 05, 06, 07 - // s1: 10, 11, 12, 13, 14, 15, 16, 17 - // s2: 20, 21, 22, 23, 24, 25, 26, 27 - // s3: 30, 31, 32, 33, 34, 35, 36, 37 - // - // res[0]: 00 10 20 30 01 11 21 31 - // res[1]: 02 12 22 32 03 13 23 33 - // res[2]: 04 14 24 34 05 15 25 35 - // res[3]: 06 16 26 36 07 17 27 37 - - int16x8x2_t tr01_16 = vzipq_s16(s0, s1); - int16x8x2_t tr23_16 = vzipq_s16(s2, s3); - int32x4x2_t tr01_32 = vzipq_s32(vreinterpretq_s32_s16(tr01_16.val[0]), - vreinterpretq_s32_s16(tr23_16.val[0])); - int32x4x2_t tr23_32 = vzipq_s32(vreinterpretq_s32_s16(tr01_16.val[1]), - vreinterpretq_s32_s16(tr23_16.val[1])); - - res[0] = vreinterpretq_s16_s32(tr01_32.val[0]); - res[1] = vreinterpretq_s16_s32(tr01_32.val[1]); - res[2] = vreinterpretq_s16_s32(tr23_32.val[0]); - res[3] = vreinterpretq_s16_s32(tr23_32.val[1]); -} - static inline void aom_tbl2x4_s16(int16x8_t t0[4], int16x8_t t1[4], uint16x8_t tbl, int16x8_t res[4]) { res[0] = aom_tbl2_s16(t0[0], t1[0], tbl); @@ -88,10 +24,4 @@ res[3] = aom_tbl2_s16(t0[3], t1[3], tbl); } -static inline void aom_tbl2x2_s16(int16x8_t t0[2], int16x8_t t1[2], - uint16x8_t tbl, int16x8_t res[2]) { - res[0] = aom_tbl2_s16(t0[0], t1[0], tbl); - res[1] = aom_tbl2_s16(t0[1], t1[1], tbl); -} - #endif // AOM_AV1_COMMON_ARM_HIGHBD_CONVOLVE_SVE2_H_
diff --git a/av1/common/arm/reconintra_neon.c b/av1/common/arm/reconintra_neon.c index 81eb224..9ac013f 100644 --- a/av1/common/arm/reconintra_neon.c +++ b/av1/common/arm/reconintra_neon.c
@@ -20,13 +20,11 @@ #include "aom_dsp/arm/sum_neon.h" #define MAX_UPSAMPLE_SZ 16 - -// TODO(aomedia:349436249): enable for armv7 after SIGBUS is fixed. -#if AOM_ARCH_AARCH64 +#define FILTER_INTRA_SCALE_BITS 4 // These kernels are a transposed version of those defined in reconintra.c, // with the absolute value of the negatives taken in the top row. -DECLARE_ALIGNED(16, const uint8_t, +DECLARE_ALIGNED(16, static const uint8_t, av1_filter_intra_taps_neon[FILTER_INTRA_MODES][7][8]) = { // clang-format off { @@ -77,7 +75,22 @@ // clang-format on }; -#define FILTER_INTRA_SCALE_BITS 4 +static inline uint8x8_t filter_intra_predictor( + uint8x8_t s0, uint8x8_t s1, uint8x8_t s2, uint8x8_t s3, uint8x8_t s4, + uint8x8_t s5, uint8x8_t s6, const uint8x8_t f0, const uint8x8_t f1, + const uint8x8_t f2, const uint8x8_t f3, const uint8x8_t f4, + const uint8x8_t f5, const uint8x8_t f6) { + uint16x8_t acc = vmull_u8(s1, f1); + // First row of each filter has all negative values so subtract. + acc = vmlsl_u8(acc, s0, f0); + acc = vmlal_u8(acc, s2, f2); + acc = vmlal_u8(acc, s3, f3); + acc = vmlal_u8(acc, s4, f4); + acc = vmlal_u8(acc, s5, f5); + acc = vmlal_u8(acc, s6, f6); + + return vqrshrun_n_s16(vreinterpretq_s16_u16(acc), FILTER_INTRA_SCALE_BITS); +} void av1_filter_intra_predictor_neon(uint8_t *dst, ptrdiff_t stride, TX_SIZE tx_size, const uint8_t *above, @@ -94,82 +107,118 @@ const uint8x8_t f5 = vld1_u8(av1_filter_intra_taps_neon[mode][5]); const uint8x8_t f6 = vld1_u8(av1_filter_intra_taps_neon[mode][6]); - uint8_t buffer[33][33]; - // Populate the top row in the scratch buffer with data from above. - memcpy(buffer[0], &above[-1], (width + 1) * sizeof(uint8_t)); - // Populate the first column in the scratch buffer with data from the left. - int r = 0; - do { - buffer[r + 1][0] = left[r]; - } while (++r < height); - // Computing 4 cols per iteration (instead of 8) for 8x<h> blocks is faster. if (width <= 8) { - r = 1; - do { - int c = 1; - uint8x8_t s0 = vld1_dup_u8(&buffer[r - 1][c - 1]); - uint8x8_t s5 = vld1_dup_u8(&buffer[r + 0][c - 1]); - uint8x8_t s6 = vld1_dup_u8(&buffer[r + 1][c - 1]); + uint8x8_t s0 = vdup_n_u8(above[-1]); + uint8x8_t s5 = vdup_n_u8(left[0]); + uint8x8_t s6 = vdup_n_u8(left[1]); + int c = 0; + do { + uint8x8_t s1234 = load_unaligned_u8_4x1(above + c); + uint8x8_t s1 = vdup_lane_u8(s1234, 0); + uint8x8_t s2 = vdup_lane_u8(s1234, 1); + uint8x8_t s3 = vdup_lane_u8(s1234, 2); + uint8x8_t s4 = vdup_lane_u8(s1234, 3); + + uint8x8_t res = filter_intra_predictor(s0, s1, s2, s3, s4, s5, s6, f0, f1, + f2, f3, f4, f5, f6); + + store_u8x4_strided_x2(dst + c, stride, res); + + s0 = s4; + s5 = vdup_lane_u8(res, 3); + s6 = vdup_lane_u8(res, 7); + + c += 4; + } while (c < width); + + int r = 2; + do { + s0 = vdup_n_u8(left[r - 1]); + s5 = vdup_n_u8(left[r + 0]); + s6 = vdup_n_u8(left[r + 1]); + + c = 0; do { - uint8x8_t s1234 = load_u8_4x1(&buffer[r - 1][c - 1] + 1); + uint8x8_t s1234 = load_u8_4x1(dst + (r - 1) * stride + c); uint8x8_t s1 = vdup_lane_u8(s1234, 0); uint8x8_t s2 = vdup_lane_u8(s1234, 1); uint8x8_t s3 = vdup_lane_u8(s1234, 2); uint8x8_t s4 = vdup_lane_u8(s1234, 3); - uint16x8_t sum = vmull_u8(s1, f1); - // First row of each filter has all negative values so subtract. - sum = vmlsl_u8(sum, s0, f0); - sum = vmlal_u8(sum, s2, f2); - sum = vmlal_u8(sum, s3, f3); - sum = vmlal_u8(sum, s4, f4); - sum = vmlal_u8(sum, s5, f5); - sum = vmlal_u8(sum, s6, f6); + uint8x8_t res = filter_intra_predictor(s0, s1, s2, s3, s4, s5, s6, f0, + f1, f2, f3, f4, f5, f6); - uint8x8_t res = - vqrshrun_n_s16(vreinterpretq_s16_u16(sum), FILTER_INTRA_SCALE_BITS); - - // Store buffer[r + 0][c] and buffer[r + 1][c]. - store_u8x4_strided_x2(&buffer[r][c], 33, res); - - store_u8x4_strided_x2(dst + (r - 1) * stride + c - 1, stride, res); + store_u8x4_strided_x2(dst + r * stride + c, stride, res); s0 = s4; s5 = vdup_lane_u8(res, 3); s6 = vdup_lane_u8(res, 7); + c += 4; - } while (c < width + 1); + } while (c < width); r += 2; - } while (r < height + 1); + } while (r < height); } else { - r = 1; - do { - int c = 1; - uint8x8_t s0_lo = vld1_dup_u8(&buffer[r - 1][c - 1]); - uint8x8_t s5_lo = vld1_dup_u8(&buffer[r + 0][c - 1]); - uint8x8_t s6_lo = vld1_dup_u8(&buffer[r + 1][c - 1]); + uint8x8_t s0_lo = vdup_n_u8(above[-1]); + uint8x8_t s5_lo = vdup_n_u8(left[0]); + uint8x8_t s6_lo = vdup_n_u8(left[1]); + int c = 0; + do { + uint8x8_t s1234 = vld1_u8(above + c); + uint8x8_t s1_lo = vdup_lane_u8(s1234, 0); + uint8x8_t s2_lo = vdup_lane_u8(s1234, 1); + uint8x8_t s3_lo = vdup_lane_u8(s1234, 2); + uint8x8_t s4_lo = vdup_lane_u8(s1234, 3); + + uint8x8_t res_lo = + filter_intra_predictor(s0_lo, s1_lo, s2_lo, s3_lo, s4_lo, s5_lo, + s6_lo, f0, f1, f2, f3, f4, f5, f6); + + uint8x8_t s0_hi = s4_lo; + uint8x8_t s1_hi = vdup_lane_u8(s1234, 4); + uint8x8_t s2_hi = vdup_lane_u8(s1234, 5); + uint8x8_t s3_hi = vdup_lane_u8(s1234, 6); + uint8x8_t s4_hi = vdup_lane_u8(s1234, 7); + uint8x8_t s5_hi = vdup_lane_u8(res_lo, 3); + uint8x8_t s6_hi = vdup_lane_u8(res_lo, 7); + + uint8x8_t res_hi = + filter_intra_predictor(s0_hi, s1_hi, s2_hi, s3_hi, s4_hi, s5_hi, + s6_hi, f0, f1, f2, f3, f4, f5, f6); + + uint32x2x2_t res = + vzip_u32(vreinterpret_u32_u8(res_lo), vreinterpret_u32_u8(res_hi)); + + vst1_u8(dst + 0 * stride + c, vreinterpret_u8_u32(res.val[0])); + vst1_u8(dst + 1 * stride + c, vreinterpret_u8_u32(res.val[1])); + + s0_lo = s4_hi; + s5_lo = vdup_lane_u8(res_hi, 3); + s6_lo = vdup_lane_u8(res_hi, 7); + c += 8; + } while (c < width); + + int r = 2; + do { + s0_lo = vdup_n_u8(left[r - 1]); + s5_lo = vdup_n_u8(left[r + 0]); + s6_lo = vdup_n_u8(left[r + 1]); + + c = 0; do { - uint8x8_t s1234 = vld1_u8(&buffer[r - 1][c - 1] + 1); + uint8x8_t s1234 = vld1_u8(dst + (r - 1) * stride + c); uint8x8_t s1_lo = vdup_lane_u8(s1234, 0); uint8x8_t s2_lo = vdup_lane_u8(s1234, 1); uint8x8_t s3_lo = vdup_lane_u8(s1234, 2); uint8x8_t s4_lo = vdup_lane_u8(s1234, 3); - uint16x8_t sum_lo = vmull_u8(s1_lo, f1); - // First row of each filter has all negative values so subtract. - sum_lo = vmlsl_u8(sum_lo, s0_lo, f0); - sum_lo = vmlal_u8(sum_lo, s2_lo, f2); - sum_lo = vmlal_u8(sum_lo, s3_lo, f3); - sum_lo = vmlal_u8(sum_lo, s4_lo, f4); - sum_lo = vmlal_u8(sum_lo, s5_lo, f5); - sum_lo = vmlal_u8(sum_lo, s6_lo, f6); - - uint8x8_t res_lo = vqrshrun_n_s16(vreinterpretq_s16_u16(sum_lo), - FILTER_INTRA_SCALE_BITS); + uint8x8_t res_lo = + filter_intra_predictor(s0_lo, s1_lo, s2_lo, s3_lo, s4_lo, s5_lo, + s6_lo, f0, f1, f2, f3, f4, f5, f6); uint8x8_t s0_hi = s4_lo; uint8x8_t s1_hi = vdup_lane_u8(s1234, 4); @@ -179,40 +228,26 @@ uint8x8_t s5_hi = vdup_lane_u8(res_lo, 3); uint8x8_t s6_hi = vdup_lane_u8(res_lo, 7); - uint16x8_t sum_hi = vmull_u8(s1_hi, f1); - // First row of each filter has all negative values so subtract. - sum_hi = vmlsl_u8(sum_hi, s0_hi, f0); - sum_hi = vmlal_u8(sum_hi, s2_hi, f2); - sum_hi = vmlal_u8(sum_hi, s3_hi, f3); - sum_hi = vmlal_u8(sum_hi, s4_hi, f4); - sum_hi = vmlal_u8(sum_hi, s5_hi, f5); - sum_hi = vmlal_u8(sum_hi, s6_hi, f6); - - uint8x8_t res_hi = vqrshrun_n_s16(vreinterpretq_s16_u16(sum_hi), - FILTER_INTRA_SCALE_BITS); + uint8x8_t res_hi = + filter_intra_predictor(s0_hi, s1_hi, s2_hi, s3_hi, s4_hi, s5_hi, + s6_hi, f0, f1, f2, f3, f4, f5, f6); uint32x2x2_t res = vzip_u32(vreinterpret_u32_u8(res_lo), vreinterpret_u32_u8(res_hi)); - vst1_u8(&buffer[r + 0][c], vreinterpret_u8_u32(res.val[0])); - vst1_u8(&buffer[r + 1][c], vreinterpret_u8_u32(res.val[1])); - - vst1_u8(dst + (r - 1) * stride + c - 1, - vreinterpret_u8_u32(res.val[0])); - vst1_u8(dst + (r + 0) * stride + c - 1, - vreinterpret_u8_u32(res.val[1])); + vst1_u8(dst + (r + 0) * stride + c, vreinterpret_u8_u32(res.val[0])); + vst1_u8(dst + (r + 1) * stride + c, vreinterpret_u8_u32(res.val[1])); s0_lo = s4_hi; s5_lo = vdup_lane_u8(res_hi, 3); s6_lo = vdup_lane_u8(res_hi, 7); c += 8; - } while (c < width + 1); + } while (c < width); r += 2; - } while (r < height + 1); + } while (r < height); } } -#endif // AOM_ARCH_AARCH64 void av1_filter_intra_edge_neon(uint8_t *p, int sz, int strength) { if (!strength) return;
diff --git a/av1/common/arm/reconintra_neon_i8mm.c b/av1/common/arm/reconintra_neon_i8mm.c new file mode 100644 index 0000000..21b7170 --- /dev/null +++ b/av1/common/arm/reconintra_neon_i8mm.c
@@ -0,0 +1,182 @@ +/* + * Copyright (c) 2025, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include <arm_neon.h> +#include <assert.h> + +#include "config/aom_config.h" +#include "config/av1_rtcd.h" + +#include "aom_dsp/arm/mem_neon.h" + +#define FILTER_INTRA_SCALE_BITS 4 + +// The input arrays are reordered compared to the C implementation: the first +// four vectors contain the lower elements of the original filter, the next four +// vectors contain the upper elements. This layout allows all 8 multiplications +// to accumulate into a single element using USDOT instructions, instead of +// having two partial sums in adjacent vector elements and needing to combine +// them with an additional pairwise add. +DECLARE_ALIGNED(16, static const int8_t, + av1_filter_intra_taps_neon_i8mm[FILTER_INTRA_MODES][8][8]) = { + { + { -6, 10, 0, 0, -5, 2, 10, 0 }, + { -3, 1, 1, 10, -3, 1, 1, 2 }, + { -4, 6, 0, 0, -3, 2, 6, 0 }, + { -3, 2, 2, 6, -3, 1, 2, 2 }, + { 0, 12, 0, 0, 0, 9, 0, 0 }, + { 0, 7, 0, 0, 10, 5, 0, 0 }, + { 0, 2, 12, 0, 0, 2, 9, 0 }, + { 0, 2, 7, 0, 6, 3, 5, 0 }, + }, + { + { -10, 16, 0, 0, -6, 0, 16, 0 }, + { -4, 0, 0, 16, -2, 0, 0, 0 }, + { -10, 16, 0, 0, -6, 0, 16, 0 }, + { -4, 0, 0, 16, -2, 0, 0, 0 }, + { 0, 10, 0, 0, 0, 6, 0, 0 }, + { 0, 4, 0, 0, 16, 2, 0, 0 }, + { 0, 0, 10, 0, 0, 0, 6, 0 }, + { 0, 0, 4, 0, 16, 0, 2, 0 }, + }, + { + { -8, 8, 0, 0, -8, 0, 8, 0 }, + { -8, 0, 0, 8, -8, 0, 0, 0 }, + { -4, 4, 0, 0, -4, 0, 4, 0 }, + { -4, 0, 0, 4, -4, 0, 0, 0 }, + { 0, 16, 0, 0, 0, 16, 0, 0 }, + { 0, 16, 0, 0, 8, 16, 0, 0 }, + { 0, 0, 16, 0, 0, 0, 16, 0 }, + { 0, 0, 16, 0, 4, 0, 16, 0 }, + }, + { + { -2, 8, 0, 0, -1, 3, 8, 0 }, + { -1, 2, 3, 8, 0, 1, 2, 3 }, + { -1, 4, 0, 0, -1, 3, 4, 0 }, + { -1, 2, 3, 4, -1, 2, 2, 3 }, + { 0, 10, 0, 0, 0, 6, 0, 0 }, + { 0, 4, 0, 0, 8, 2, 0, 0 }, + { 0, 3, 10, 0, 0, 4, 6, 0 }, + { 0, 4, 4, 0, 4, 3, 3, 0 }, + }, + { + { -12, 14, 0, 0, -10, 0, 14, 0 }, + { -9, 0, 0, 14, -8, 0, 0, 0 }, + { -10, 12, 0, 0, -9, 1, 12, 0 }, + { -8, 0, 0, 12, -7, 0, 0, 1 }, + { 0, 14, 0, 0, 0, 12, 0, 0 }, + { 0, 11, 0, 0, 14, 10, 0, 0 }, + { 0, 0, 14, 0, 0, 0, 12, 0 }, + { 0, 1, 11, 0, 12, 1, 9, 0 }, + }, +}; + +static inline uint8x8_t filter_intra_predictor(uint8x16_t p_lo, uint8x16_t p_hi, + const int8x16_t f01, + const int8x16_t f23, + const int8x16_t f45, + const int8x16_t f67) { + int32x4_t acc_0123 = vusdotq_s32(vdupq_n_s32(0), p_lo, f01); + acc_0123 = vusdotq_s32(acc_0123, p_hi, f23); + + int32x4_t acc_4567 = vusdotq_s32(vdupq_n_s32(0), p_lo, f45); + acc_4567 = vusdotq_s32(acc_4567, p_hi, f67); + + const int16x8_t acc = vcombine_s16(vmovn_s32(acc_0123), vmovn_s32(acc_4567)); + + return vqrshrun_n_s16(acc, FILTER_INTRA_SCALE_BITS); +} + +void av1_filter_intra_predictor_neon_i8mm(uint8_t *dst, ptrdiff_t stride, + TX_SIZE tx_size, const uint8_t *above, + const uint8_t *left, int mode) { + const int bw = tx_size_wide[tx_size]; + const int bh = tx_size_high[tx_size]; + + if (bw == 4 || (bw == 8 && bh < 16) || (bw == 16 && bh <= 4) || bw == 32) { + av1_filter_intra_predictor_neon(dst, stride, tx_size, above, left, mode); + return; + } + + assert(bw <= 32 && bh <= 32); + + const int8x16_t f01 = vld1q_s8(av1_filter_intra_taps_neon_i8mm[mode][0]); + const int8x16_t f45 = vld1q_s8(av1_filter_intra_taps_neon_i8mm[mode][2]); + const int8x16_t f23 = vld1q_s8(av1_filter_intra_taps_neon_i8mm[mode][4]); + const int8x16_t f67 = vld1q_s8(av1_filter_intra_taps_neon_i8mm[mode][6]); + + // indexes : 0, 19, 23, -1 + uint8x16_t p_hi_idx = vreinterpretq_u8_u32(vdupq_n_u32(0xFF171300)); + + uint64_t l01 = ((uint64_t)left[0] << 24) | ((uint64_t)left[1] << 56); + uint8x16_t l = vreinterpretq_u8_u64(vdupq_n_u64(l01)); + + int c = 0; + do { + const uint8_t *ptr = above + c - 1; + uint32_t lo = + ptr[0] | (ptr[1] << 8) | (ptr[2] << 16) | ((uint32_t)ptr[3] << 24); + uint8x16_t p_lo = vreinterpretq_u8_u32(vdupq_n_u32(lo)); + + uint8x16x2_t hi; + hi.val[0] = vdupq_n_u8(ptr[4]); + hi.val[1] = l; + uint8x16_t p_hi = vqtbl2q_u8(hi, p_hi_idx); + + const uint8x8_t res = + filter_intra_predictor(p_lo, p_hi, f01, f23, f45, f67); + + store_u8x4_strided_x2(dst + c, stride, res); + + l = vcombine_u8(res, res); + + c += 4; + } while (c < bw); + + dst += 2 * stride; + int r = 2; + while (r < bh) { + const uint8_t *ptr = dst - stride; + uint32_t lo = + left[r - 1] | (ptr[0] << 8) | (ptr[1] << 16) | ((uint32_t)ptr[2] << 24); + uint32_t hi = ptr[3] | (left[r] << 8) | (left[r + 1] << 16); + uint8x16_t p_lo = vreinterpretq_u8_u32(vdupq_n_u32(lo)); + uint8x16_t p_hi = vreinterpretq_u8_u32(vdupq_n_u32(hi)); + + uint8x8_t res = filter_intra_predictor(p_lo, p_hi, f01, f23, f45, f67); + + store_u8x4_strided_x2(dst, stride, res); + + l = vcombine_u8(res, res); + + c = 4; + while (c < bw) { + ptr = dst - stride + c - 1; + lo = ptr[0] | (ptr[1] << 8) | (ptr[2] << 16) | ((uint32_t)ptr[3] << 24); + p_lo = vreinterpretq_u8_u32(vdupq_n_u32(lo)); + + uint8x16x2_t hi_v; + hi_v.val[0] = vdupq_n_u8(ptr[4]); + hi_v.val[1] = l; + p_hi = vqtbl2q_u8(hi_v, p_hi_idx); + + res = filter_intra_predictor(p_lo, p_hi, f01, f23, f45, f67); + + store_u8x4_strided_x2(dst + c, stride, res); + + l = vcombine_u8(res, res); + c += 4; + } + + r += 2; + dst += 2 * stride; + } +}
diff --git a/av1/common/arm/warp_plane_neon.c b/av1/common/arm/warp_plane_neon.c index 497273b..f7b393a 100644 --- a/av1/common/arm/warp_plane_neon.c +++ b/av1/common/arm/warp_plane_neon.c
@@ -143,31 +143,6 @@ return horizontal_filter_8x1_f1_beta0(in, f_s16); } -static AOM_FORCE_INLINE void vertical_filter_4x1_f1(const int16x8_t *src, - int32x4_t *res, int sy) { - int16x4_t s0 = vget_low_s16(src[0]); - int16x4_t s1 = vget_low_s16(src[1]); - int16x4_t s2 = vget_low_s16(src[2]); - int16x4_t s3 = vget_low_s16(src[3]); - int16x4_t s4 = vget_low_s16(src[4]); - int16x4_t s5 = vget_low_s16(src[5]); - int16x4_t s6 = vget_low_s16(src[6]); - int16x4_t s7 = vget_low_s16(src[7]); - - int16x8_t f = vld1q_s16(av1_warped_filter[sy >> WARPEDDIFF_PREC_BITS]); - - int32x4_t m0123 = vmull_lane_s16(s0, vget_low_s16(f), 0); - m0123 = vmlal_lane_s16(m0123, s1, vget_low_s16(f), 1); - m0123 = vmlal_lane_s16(m0123, s2, vget_low_s16(f), 2); - m0123 = vmlal_lane_s16(m0123, s3, vget_low_s16(f), 3); - m0123 = vmlal_lane_s16(m0123, s4, vget_high_s16(f), 0); - m0123 = vmlal_lane_s16(m0123, s5, vget_high_s16(f), 1); - m0123 = vmlal_lane_s16(m0123, s6, vget_high_s16(f), 2); - m0123 = vmlal_lane_s16(m0123, s7, vget_high_s16(f), 3); - - *res = m0123; -} - static AOM_FORCE_INLINE void vertical_filter_4x1_f4(const int16x8_t *src, int32x4_t *res, int sy, int gamma) { @@ -194,43 +169,6 @@ *res = horizontal_add_4d_s32x4(m0123_pairs); } -static AOM_FORCE_INLINE void vertical_filter_8x1_f1(const int16x8_t *src, - int32x4_t *res_low, - int32x4_t *res_high, - int sy) { - int16x8_t s0 = src[0]; - int16x8_t s1 = src[1]; - int16x8_t s2 = src[2]; - int16x8_t s3 = src[3]; - int16x8_t s4 = src[4]; - int16x8_t s5 = src[5]; - int16x8_t s6 = src[6]; - int16x8_t s7 = src[7]; - - int16x8_t f = vld1q_s16(av1_warped_filter[sy >> WARPEDDIFF_PREC_BITS]); - - int32x4_t m0123 = vmull_lane_s16(vget_low_s16(s0), vget_low_s16(f), 0); - m0123 = vmlal_lane_s16(m0123, vget_low_s16(s1), vget_low_s16(f), 1); - m0123 = vmlal_lane_s16(m0123, vget_low_s16(s2), vget_low_s16(f), 2); - m0123 = vmlal_lane_s16(m0123, vget_low_s16(s3), vget_low_s16(f), 3); - m0123 = vmlal_lane_s16(m0123, vget_low_s16(s4), vget_high_s16(f), 0); - m0123 = vmlal_lane_s16(m0123, vget_low_s16(s5), vget_high_s16(f), 1); - m0123 = vmlal_lane_s16(m0123, vget_low_s16(s6), vget_high_s16(f), 2); - m0123 = vmlal_lane_s16(m0123, vget_low_s16(s7), vget_high_s16(f), 3); - - int32x4_t m4567 = vmull_lane_s16(vget_high_s16(s0), vget_low_s16(f), 0); - m4567 = vmlal_lane_s16(m4567, vget_high_s16(s1), vget_low_s16(f), 1); - m4567 = vmlal_lane_s16(m4567, vget_high_s16(s2), vget_low_s16(f), 2); - m4567 = vmlal_lane_s16(m4567, vget_high_s16(s3), vget_low_s16(f), 3); - m4567 = vmlal_lane_s16(m4567, vget_high_s16(s4), vget_high_s16(f), 0); - m4567 = vmlal_lane_s16(m4567, vget_high_s16(s5), vget_high_s16(f), 1); - m4567 = vmlal_lane_s16(m4567, vget_high_s16(s6), vget_high_s16(f), 2); - m4567 = vmlal_lane_s16(m4567, vget_high_s16(s7), vget_high_s16(f), 3); - - *res_low = m0123; - *res_high = m4567; -} - static AOM_FORCE_INLINE void vertical_filter_8x1_f8(const int16x8_t *src, int32x4_t *res_low, int32x4_t *res_high, int sy, @@ -272,13 +210,106 @@ *res_high = horizontal_add_4d_s32x4(m4567_pairs); } +static AOM_FORCE_INLINE void warp_affine_horizontal_neon( + const uint8_t *ref, int width, int height, int stride, int p_width, + int p_height, int16_t alpha, int16_t beta, const int64_t x4, + const int64_t y4, const int i, int16x8_t tmp[]) { + const int height_limit = AOMMIN(8, p_height - i) + 7; + + int32_t ix4 = (int32_t)(x4 >> WARPEDMODEL_PREC_BITS); + int32_t iy4 = (int32_t)(y4 >> WARPEDMODEL_PREC_BITS); + + int32_t sx4 = x4 & ((1 << WARPEDMODEL_PREC_BITS) - 1); + sx4 += alpha * (-4) + beta * (-4) + (1 << (WARPEDDIFF_PREC_BITS - 1)) + + (WARPEDPIXEL_PREC_SHIFTS << WARPEDDIFF_PREC_BITS); + sx4 &= ~((1 << WARP_PARAM_REDUCE_BITS) - 1); + + if (warp_affine_special_case(ref, ix4, iy4, width, height, stride, + height_limit, tmp)) { + return; + } + + static const uint8_t kIotaArr[] = { 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15 }; + const uint8x16_t indx = vld1q_u8(kIotaArr); + + const int out_of_boundary_left = -(ix4 - 6); + const int out_of_boundary_right = (ix4 + 8) - width; + + if (p_width == 4) { + if (beta == 0) { + if (alpha == 0) { + int16x8_t f_s16 = + vld1q_s16(av1_warped_filter[sx4 >> WARPEDDIFF_PREC_BITS]); + APPLY_HORIZONTAL_SHIFT(horizontal_filter_4x1_f1_beta0, f_s16); + } else { + APPLY_HORIZONTAL_SHIFT(horizontal_filter_4x1_f4, sx4, alpha); + } + } else { + if (alpha == 0) { + APPLY_HORIZONTAL_SHIFT(horizontal_filter_4x1_f1, + (sx4 + beta * (k - 3))); + } else { + APPLY_HORIZONTAL_SHIFT(horizontal_filter_4x1_f4, (sx4 + beta * (k - 3)), + alpha); + } + } + } else { + if (beta == 0) { + if (alpha == 0) { + int16x8_t f_s16 = + vld1q_s16(av1_warped_filter[sx4 >> WARPEDDIFF_PREC_BITS]); + APPLY_HORIZONTAL_SHIFT(horizontal_filter_8x1_f1_beta0, f_s16); + } else { + APPLY_HORIZONTAL_SHIFT(horizontal_filter_8x1_f8, sx4, alpha); + } + } else { + if (alpha == 0) { + APPLY_HORIZONTAL_SHIFT(horizontal_filter_8x1_f1, + (sx4 + beta * (k - 3))); + } else { + APPLY_HORIZONTAL_SHIFT(horizontal_filter_8x1_f8, (sx4 + beta * (k - 3)), + alpha); + } + } + } +} + void av1_warp_affine_neon(const int32_t *mat, const uint8_t *ref, int width, int height, int stride, uint8_t *pred, int p_col, int p_row, int p_width, int p_height, int p_stride, int subsampling_x, int subsampling_y, ConvolveParams *conv_params, int16_t alpha, int16_t beta, int16_t gamma, int16_t delta) { - av1_warp_affine_common(mat, ref, width, height, stride, pred, p_col, p_row, - p_width, p_height, p_stride, subsampling_x, - subsampling_y, conv_params, alpha, beta, gamma, delta); + const int w0 = conv_params->fwd_offset; + const int w1 = conv_params->bck_offset; + const int is_compound = conv_params->is_compound; + uint16_t *const dst = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + const int do_average = conv_params->do_average; + const int use_dist_wtd_comp_avg = conv_params->use_dist_wtd_comp_avg; + + assert(IMPLIES(is_compound, dst != NULL)); + assert(IMPLIES(do_average, is_compound)); + + for (int i = 0; i < p_height; i += 8) { + for (int j = 0; j < p_width; j += 8) { + const int32_t src_x = (p_col + j + 4) << subsampling_x; + const int32_t src_y = (p_row + i + 4) << subsampling_y; + const int64_t dst_x = + (int64_t)mat[2] * src_x + (int64_t)mat[3] * src_y + (int64_t)mat[0]; + const int64_t dst_y = + (int64_t)mat[4] * src_x + (int64_t)mat[5] * src_y + (int64_t)mat[1]; + + const int64_t x4 = dst_x >> subsampling_x; + const int64_t y4 = dst_y >> subsampling_y; + + int16x8_t tmp[15]; + warp_affine_horizontal_neon(ref, width, height, stride, p_width, p_height, + alpha, beta, x4, y4, i, tmp); + warp_affine_vertical(pred, p_width, p_height, p_stride, is_compound, dst, + dst_stride, do_average, use_dist_wtd_comp_avg, gamma, + delta, y4, i, j, tmp, w0, w1); + } + } }
diff --git a/av1/common/arm/warp_plane_neon.h b/av1/common/arm/warp_plane_neon.h index 2909df7..0bf5c03 100644 --- a/av1/common/arm/warp_plane_neon.h +++ b/av1/common/arm/warp_plane_neon.h
@@ -24,41 +24,91 @@ #include "av1/common/warped_motion.h" #include "av1/common/scale.h" -static AOM_FORCE_INLINE int16x8_t horizontal_filter_4x1_f4(const uint8x16_t in, - int sx, int alpha); - -static AOM_FORCE_INLINE int16x8_t horizontal_filter_8x1_f8(const uint8x16_t in, - int sx, int alpha); - -static AOM_FORCE_INLINE int16x8_t horizontal_filter_4x1_f1(const uint8x16_t in, - int sx); - -static AOM_FORCE_INLINE int16x8_t horizontal_filter_8x1_f1(const uint8x16_t in, - int sx); - -static AOM_FORCE_INLINE int16x8_t -horizontal_filter_4x1_f1_beta0(const uint8x16_t in, int16x8_t f_s16); - -static AOM_FORCE_INLINE int16x8_t -horizontal_filter_8x1_f1_beta0(const uint8x16_t in, int16x8_t f_s16); - -static AOM_FORCE_INLINE void vertical_filter_4x1_f1(const int16x8_t *src, - int32x4_t *res, int sy); - static AOM_FORCE_INLINE void vertical_filter_4x1_f4(const int16x8_t *src, int32x4_t *res, int sy, int gamma); -static AOM_FORCE_INLINE void vertical_filter_8x1_f1(const int16x8_t *src, - int32x4_t *res_low, - int32x4_t *res_high, - int sy); - static AOM_FORCE_INLINE void vertical_filter_8x1_f8(const int16x8_t *src, int32x4_t *res_low, int32x4_t *res_high, int sy, int gamma); +static AOM_FORCE_INLINE void vertical_filter_4x1_f1(const int16x8_t *src, + int32x4_t *res, int sy) { + int16_t *f_ptr = + (int16_t *)(av1_warped_filter + (sy >> WARPEDDIFF_PREC_BITS)); + int16x8_t f = vld1q_s16(f_ptr); + + int32x4_t m0123; + if (f_ptr[0] != 0) { + m0123 = vmull_lane_s16(vget_low_s16(src[0]), vget_low_s16(f), 0); + } else { + m0123 = vdupq_n_s32(0); + } + if (f_ptr[1] != 0) { + m0123 = vmlal_lane_s16(m0123, vget_low_s16(src[1]), vget_low_s16(f), 1); + } + + m0123 = vmlal_lane_s16(m0123, vget_low_s16(src[2]), vget_low_s16(f), 2); + m0123 = vmlal_lane_s16(m0123, vget_low_s16(src[3]), vget_low_s16(f), 3); + m0123 = vmlal_lane_s16(m0123, vget_low_s16(src[4]), vget_high_s16(f), 0); + m0123 = vmlal_lane_s16(m0123, vget_low_s16(src[5]), vget_high_s16(f), 1); + + if (f_ptr[6] != 0) { + m0123 = vmlal_lane_s16(m0123, vget_low_s16(src[6]), vget_high_s16(f), 2); + } + if (f_ptr[7] != 0) { + m0123 = vmlal_lane_s16(m0123, vget_low_s16(src[7]), vget_high_s16(f), 3); + } + + *res = m0123; +} + +static AOM_FORCE_INLINE void vertical_filter_8x1_f1(const int16x8_t *s, + int32x4_t *res_low, + int32x4_t *res_high, + int sy) { + int16_t *f_ptr = + (int16_t *)(av1_warped_filter + (sy >> WARPEDDIFF_PREC_BITS)); + int16x8_t f = vld1q_s16(f_ptr); + + int32x4_t m0123, m4567; + if (f_ptr[0] != 0) { + m0123 = vmull_lane_s16(vget_low_s16(s[0]), vget_low_s16(f), 0); + m4567 = vmull_lane_s16(vget_high_s16(s[0]), vget_low_s16(f), 0); + } else { + m0123 = vdupq_n_s32(0); + m4567 = vdupq_n_s32(0); + } + if (f_ptr[1] != 0) { + m0123 = vmlal_lane_s16(m0123, vget_low_s16(s[1]), vget_low_s16(f), 1); + m4567 = vmlal_lane_s16(m4567, vget_high_s16(s[1]), vget_low_s16(f), 1); + } + m0123 = vmlal_lane_s16(m0123, vget_low_s16(s[2]), vget_low_s16(f), 2); + m4567 = vmlal_lane_s16(m4567, vget_high_s16(s[2]), vget_low_s16(f), 2); + + m0123 = vmlal_lane_s16(m0123, vget_low_s16(s[3]), vget_low_s16(f), 3); + m4567 = vmlal_lane_s16(m4567, vget_high_s16(s[3]), vget_low_s16(f), 3); + + m0123 = vmlal_lane_s16(m0123, vget_low_s16(s[4]), vget_high_s16(f), 0); + m4567 = vmlal_lane_s16(m4567, vget_high_s16(s[4]), vget_high_s16(f), 0); + + m0123 = vmlal_lane_s16(m0123, vget_low_s16(s[5]), vget_high_s16(f), 1); + m4567 = vmlal_lane_s16(m4567, vget_high_s16(s[5]), vget_high_s16(f), 1); + + if (f_ptr[6] != 0) { + m0123 = vmlal_lane_s16(m0123, vget_low_s16(s[6]), vget_high_s16(f), 2); + m4567 = vmlal_lane_s16(m4567, vget_high_s16(s[6]), vget_high_s16(f), 2); + } + if (f_ptr[7] != 0) { + m0123 = vmlal_lane_s16(m0123, vget_low_s16(s[7]), vget_high_s16(f), 3); + m4567 = vmlal_lane_s16(m4567, vget_high_s16(s[7]), vget_high_s16(f), 3); + } + + *res_low = m0123; + *res_high = m4567; +} + static AOM_FORCE_INLINE void load_filters_4(int16x8_t out[], int offset, int stride) { out[0] = vld1q_s16( @@ -95,21 +145,12 @@ return clamp(iy, 0, height - 1); } -static AOM_FORCE_INLINE void warp_affine_horizontal( - const uint8_t *ref, int width, int height, int stride, int p_width, - int p_height, int16_t alpha, int16_t beta, const int64_t x4, - const int64_t y4, const int i, int16x8_t tmp[]) { +static inline bool warp_affine_special_case(const uint8_t *ref, int32_t ix4, + int32_t iy4, int width, int height, + int stride, const int height_limit, + int16x8_t tmp[]) { const int bd = 8; const int reduce_bits_horiz = ROUND0_BITS; - const int height_limit = AOMMIN(8, p_height - i) + 7; - - int32_t ix4 = (int32_t)(x4 >> WARPEDMODEL_PREC_BITS); - int32_t iy4 = (int32_t)(y4 >> WARPEDMODEL_PREC_BITS); - - int32_t sx4 = x4 & ((1 << WARPEDMODEL_PREC_BITS) - 1); - sx4 += alpha * (-4) + beta * (-4) + (1 << (WARPEDDIFF_PREC_BITS - 1)) + - (WARPEDPIXEL_PREC_SHIFTS << WARPEDDIFF_PREC_BITS); - sx4 &= ~((1 << WARP_PARAM_REDUCE_BITS) - 1); if (ix4 <= -7) { for (int k = 0; k < height_limit; ++k) { @@ -119,7 +160,7 @@ ref[iy * stride] * (1 << (FILTER_BITS - reduce_bits_horiz)); tmp[k] = vdupq_n_s16(dup_val); } - return; + return true; } else if (ix4 >= width + 6) { for (int k = 0; k < height_limit; ++k) { int iy = clamp_iy(iy4 + k - 7, height); @@ -128,15 +169,11 @@ (1 << (FILTER_BITS - reduce_bits_horiz)); tmp[k] = vdupq_n_s16(dup_val); } - return; + return true; } - static const uint8_t kIotaArr[] = { 0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15 }; - const uint8x16_t indx = vld1q_u8(kIotaArr); - - const int out_of_boundary_left = -(ix4 - 6); - const int out_of_boundary_right = (ix4 + 8) - width; + return false; +} #define APPLY_HORIZONTAL_SHIFT(fn, ...) \ do { \ @@ -172,45 +209,6 @@ } \ } while (0) - if (p_width == 4) { - if (beta == 0) { - if (alpha == 0) { - int16x8_t f_s16 = - vld1q_s16(av1_warped_filter[sx4 >> WARPEDDIFF_PREC_BITS]); - APPLY_HORIZONTAL_SHIFT(horizontal_filter_4x1_f1_beta0, f_s16); - } else { - APPLY_HORIZONTAL_SHIFT(horizontal_filter_4x1_f4, sx4, alpha); - } - } else { - if (alpha == 0) { - APPLY_HORIZONTAL_SHIFT(horizontal_filter_4x1_f1, - (sx4 + beta * (k - 3))); - } else { - APPLY_HORIZONTAL_SHIFT(horizontal_filter_4x1_f4, (sx4 + beta * (k - 3)), - alpha); - } - } - } else { - if (beta == 0) { - if (alpha == 0) { - int16x8_t f_s16 = - vld1q_s16(av1_warped_filter[sx4 >> WARPEDDIFF_PREC_BITS]); - APPLY_HORIZONTAL_SHIFT(horizontal_filter_8x1_f1_beta0, f_s16); - } else { - APPLY_HORIZONTAL_SHIFT(horizontal_filter_8x1_f8, sx4, alpha); - } - } else { - if (alpha == 0) { - APPLY_HORIZONTAL_SHIFT(horizontal_filter_8x1_f1, - (sx4 + beta * (k - 3))); - } else { - APPLY_HORIZONTAL_SHIFT(horizontal_filter_8x1_f8, (sx4 + beta * (k - 3)), - alpha); - } - } - } -} - static AOM_FORCE_INLINE void warp_affine_vertical( uint8_t *pred, int p_width, int p_height, int p_stride, int is_compound, uint16_t *dst, int dst_stride, int do_average, int use_dist_wtd_comp_avg, @@ -339,43 +337,4 @@ } } -static AOM_FORCE_INLINE void av1_warp_affine_common( - const int32_t *mat, const uint8_t *ref, int width, int height, int stride, - uint8_t *pred, int p_col, int p_row, int p_width, int p_height, - int p_stride, int subsampling_x, int subsampling_y, - ConvolveParams *conv_params, int16_t alpha, int16_t beta, int16_t gamma, - int16_t delta) { - const int w0 = conv_params->fwd_offset; - const int w1 = conv_params->bck_offset; - const int is_compound = conv_params->is_compound; - uint16_t *const dst = conv_params->dst; - const int dst_stride = conv_params->dst_stride; - const int do_average = conv_params->do_average; - const int use_dist_wtd_comp_avg = conv_params->use_dist_wtd_comp_avg; - - assert(IMPLIES(is_compound, dst != NULL)); - assert(IMPLIES(do_average, is_compound)); - - for (int i = 0; i < p_height; i += 8) { - for (int j = 0; j < p_width; j += 8) { - const int32_t src_x = (p_col + j + 4) << subsampling_x; - const int32_t src_y = (p_row + i + 4) << subsampling_y; - const int64_t dst_x = - (int64_t)mat[2] * src_x + (int64_t)mat[3] * src_y + (int64_t)mat[0]; - const int64_t dst_y = - (int64_t)mat[4] * src_x + (int64_t)mat[5] * src_y + (int64_t)mat[1]; - - const int64_t x4 = dst_x >> subsampling_x; - const int64_t y4 = dst_y >> subsampling_y; - - int16x8_t tmp[15]; - warp_affine_horizontal(ref, width, height, stride, p_width, p_height, - alpha, beta, x4, y4, i, tmp); - warp_affine_vertical(pred, p_width, p_height, p_stride, is_compound, dst, - dst_stride, do_average, use_dist_wtd_comp_avg, gamma, - delta, y4, i, j, tmp, w0, w1); - } - } -} - #endif // AOM_AV1_COMMON_ARM_WARP_PLANE_NEON_H_
diff --git a/av1/common/arm/warp_plane_neon_i8mm.c b/av1/common/arm/warp_plane_neon_i8mm.c index 15ac004..b75f5e7 100644 --- a/av1/common/arm/warp_plane_neon_i8mm.c +++ b/av1/common/arm/warp_plane_neon_i8mm.c
@@ -10,11 +10,11 @@ */ #include "warp_plane_neon.h" +#include "convolve_neon_dotprod.h" +#include "convolve_neon_i8mm.h" -DECLARE_ALIGNED(16, static const uint8_t, usdot_permute_idx[48]) = { - 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6, - 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10, - 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 +DECLARE_ALIGNED(16, const uint8_t, kTblIdx0_3[16]) = { + 0, -1, -1, -1, 1, -1, -1, -1, 2, -1, -1, -1, 3, -1, -1, -1, }; static AOM_FORCE_INLINE int16x8_t horizontal_filter_4x1_f4(const uint8x16_t in, @@ -84,14 +84,56 @@ return vreinterpretq_s16_u16(res); } -static AOM_FORCE_INLINE int16x8_t -horizontal_filter_4x1_f1_beta0(const uint8x16_t in, int16x8_t f_s16) { +static AOM_FORCE_INLINE int16x8_t horizontal_filter_4x1_f1_6tap_beta0( + const uint8x16_t in, const int8x16_t filter, const uint8x16_t perm_tbl) { const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1)); + // Permute samples ready for matrix multiply. + // { 0, 1, 2, 3, 4, 5, 6, 7, 2, 3, 4, 5, 6, 7, 8, 9 } + const uint8x16_t perm_samples = vqtbl1q_u8(in, perm_tbl); + + // These instructions multiply a 2x8 matrix (samples) by an 8x2 matrix + // (filter), destructively accumulating into the destination register. + int32x4_t sum = vusmmlaq_s32(add_const, perm_samples, filter); + + uint16x8_t res = + vcombine_u16(vqrshrun_n_s32(sum, ROUND0_BITS), vdup_n_u16(0)); + + return vreinterpretq_s16_u16(res); +} + +static AOM_FORCE_INLINE int16x8_t horizontal_filter_4x1_f1_8tap_beta0( + const uint8x16_t in, const int8x16_t filter, const int32x4_t f0, + const uint8x16_t perm_tbl, const uint8x16_t tbl_idx0_3) { + const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1)); + + // Permute samples ready for matrix multiply. + // { 1, 2, 3, 4, 5, 6, 7, 8, 3, 4, 5, 6, 7, 8, 9, 10 } + const uint8x16_t perm_samples = vqtbl1q_u8(in, perm_tbl); + // Get samples 0..3 to apply tap 0 after matrix multiply. + const int32x4_t samples_0_3 = + vreinterpretq_s32_u8(vqtbl1q_u8(in, tbl_idx0_3)); + + // Calculate partial 7-tap convolution. + int32x4_t sum = vusmmlaq_s32(add_const, perm_samples, filter); + // Apply tap 0 and accumulate. + sum = vmlaq_s32(sum, samples_0_3, f0); + + uint16x8_t res = + vcombine_u16(vqrshrun_n_s32(sum, ROUND0_BITS), vdup_n_u16(0)); + + return vreinterpretq_s16_u16(res); +} + +static AOM_FORCE_INLINE int16x8_t horizontal_filter_4x1_f1(const uint8x16_t in, + int sx) { + const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1)); + + int16x8_t f_s16 = vld1q_s16(av1_warped_filter[sx >> WARPEDDIFF_PREC_BITS]); int8x16_t f_s8 = vcombine_s8(vmovn_s16(f_s16), vmovn_s16(f_s16)); - uint8x16_t perm0 = vld1q_u8(&usdot_permute_idx[0]); - uint8x16_t perm1 = vld1q_u8(&usdot_permute_idx[16]); + uint8x16_t perm0 = vld1q_u8(&kDotProdPermuteTbl[0]); + uint8x16_t perm1 = vld1q_u8(&kDotProdPermuteTbl[16]); // Permute samples ready for dot product. // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } @@ -107,21 +149,63 @@ return vreinterpretq_s16_u16(res); } -static AOM_FORCE_INLINE int16x8_t horizontal_filter_4x1_f1(const uint8x16_t in, - int sx) { - int16x8_t f_s16 = vld1q_s16(av1_warped_filter[sx >> WARPEDDIFF_PREC_BITS]); - return horizontal_filter_4x1_f1_beta0(in, f_s16); -} - -static AOM_FORCE_INLINE int16x8_t -horizontal_filter_8x1_f1_beta0(const uint8x16_t in, int16x8_t f_s16) { +static AOM_FORCE_INLINE int16x8_t horizontal_filter_8x1_f1_6tap_beta0( + const uint8x16_t in, const int8x16_t filter, const uint8x16x2_t perm_tbl) { const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1)); + // Permute samples ready for matrix multiply. + // { 0, 1, 2, 3, 4, 5, 6, 7, 2, 3, 4, 5, 6, 7, 8, 9 } + // { 4, 5, 6, 7, 8, 9, 10, 11, 6, 7, 8, 9, 10, 11, 12, 13 } + uint8x16_t perm_samples[2] = { vqtbl1q_u8(in, perm_tbl.val[0]), + vqtbl1q_u8(in, perm_tbl.val[1]) }; + + // These instructions multiply a 2x8 matrix (samples) by an 8x2 matrix + // (filter), destructively accumulating into the destination register. + int32x4_t sum0123 = vusmmlaq_s32(add_const, perm_samples[0], filter); + int32x4_t sum4567 = vusmmlaq_s32(add_const, perm_samples[1], filter); + + uint16x8_t res = vcombine_u16(vqrshrun_n_s32(sum0123, ROUND0_BITS), + vqrshrun_n_s32(sum4567, ROUND0_BITS)); + + return vreinterpretq_s16_u16(res); +} + +static AOM_FORCE_INLINE int16x8_t horizontal_filter_8x1_f1_8tap_beta0( + const uint8x16_t in, const int8x16_t filter, const int16x4_t f0, + const uint8x16x2_t perm_tbl) { + const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1)); + + // Permute samples ready for matrix multiply. + // { 1, 2, 3, 4, 5, 6, 7, 8, 3, 4, 5, 6, 7, 8, 9, 10 } + // { 5, 6, 7, 8, 9, 10, 11, 12, 7, 8, 9, 10, 11, 12, 13, 14 } + uint8x16_t perm_samples[2] = { vqtbl1q_u8(in, perm_tbl.val[0]), + vqtbl1q_u8(in, perm_tbl.val[1]) }; + // Get samples 0..7 to apply tap 0 after matrix multiply. + int16x8_t samples_0_7 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(in))); + + // Calculate partial 7-tap convolution. + int32x4_t sum0123 = vusmmlaq_s32(add_const, perm_samples[0], filter); + int32x4_t sum4567 = vusmmlaq_s32(add_const, perm_samples[1], filter); + // Apply tap 0 and accumulate. + sum0123 = vmlal_s16(sum0123, vget_low_s16(samples_0_7), f0); + sum4567 = vmlal_s16(sum4567, vget_high_s16(samples_0_7), f0); + + uint16x8_t res = vcombine_u16(vqrshrun_n_s32(sum0123, ROUND0_BITS), + vqrshrun_n_s32(sum4567, ROUND0_BITS)); + + return vreinterpretq_s16_u16(res); +} + +static AOM_FORCE_INLINE int16x8_t horizontal_filter_8x1_f1(const uint8x16_t in, + int sx) { + const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1)); + + int16x8_t f_s16 = vld1q_s16(av1_warped_filter[sx >> WARPEDDIFF_PREC_BITS]); int8x16_t f_s8 = vcombine_s8(vmovn_s16(f_s16), vmovn_s16(f_s16)); - uint8x16_t perm0 = vld1q_u8(&usdot_permute_idx[0]); - uint8x16_t perm1 = vld1q_u8(&usdot_permute_idx[16]); - uint8x16_t perm2 = vld1q_u8(&usdot_permute_idx[32]); + uint8x16_t perm0 = vld1q_u8(&kDotProdPermuteTbl[0]); + uint8x16_t perm1 = vld1q_u8(&kDotProdPermuteTbl[16]); + uint8x16_t perm2 = vld1q_u8(&kDotProdPermuteTbl[32]); // Permute samples ready for dot product. // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } @@ -142,37 +226,6 @@ return vreinterpretq_s16_u16(res); } -static AOM_FORCE_INLINE int16x8_t horizontal_filter_8x1_f1(const uint8x16_t in, - int sx) { - int16x8_t f_s16 = vld1q_s16(av1_warped_filter[sx >> WARPEDDIFF_PREC_BITS]); - return horizontal_filter_8x1_f1_beta0(in, f_s16); -} - -static AOM_FORCE_INLINE void vertical_filter_4x1_f1(const int16x8_t *src, - int32x4_t *res, int sy) { - int16x4_t s0 = vget_low_s16(src[0]); - int16x4_t s1 = vget_low_s16(src[1]); - int16x4_t s2 = vget_low_s16(src[2]); - int16x4_t s3 = vget_low_s16(src[3]); - int16x4_t s4 = vget_low_s16(src[4]); - int16x4_t s5 = vget_low_s16(src[5]); - int16x4_t s6 = vget_low_s16(src[6]); - int16x4_t s7 = vget_low_s16(src[7]); - - int16x8_t f = vld1q_s16(av1_warped_filter[sy >> WARPEDDIFF_PREC_BITS]); - - int32x4_t m0123 = vmull_lane_s16(s0, vget_low_s16(f), 0); - m0123 = vmlal_lane_s16(m0123, s1, vget_low_s16(f), 1); - m0123 = vmlal_lane_s16(m0123, s2, vget_low_s16(f), 2); - m0123 = vmlal_lane_s16(m0123, s3, vget_low_s16(f), 3); - m0123 = vmlal_lane_s16(m0123, s4, vget_high_s16(f), 0); - m0123 = vmlal_lane_s16(m0123, s5, vget_high_s16(f), 1); - m0123 = vmlal_lane_s16(m0123, s6, vget_high_s16(f), 2); - m0123 = vmlal_lane_s16(m0123, s7, vget_high_s16(f), 3); - - *res = m0123; -} - static AOM_FORCE_INLINE void vertical_filter_4x1_f4(const int16x8_t *src, int32x4_t *res, int sy, int gamma) { @@ -199,43 +252,6 @@ *res = horizontal_add_4d_s32x4(m0123_pairs); } -static AOM_FORCE_INLINE void vertical_filter_8x1_f1(const int16x8_t *src, - int32x4_t *res_low, - int32x4_t *res_high, - int sy) { - int16x8_t s0 = src[0]; - int16x8_t s1 = src[1]; - int16x8_t s2 = src[2]; - int16x8_t s3 = src[3]; - int16x8_t s4 = src[4]; - int16x8_t s5 = src[5]; - int16x8_t s6 = src[6]; - int16x8_t s7 = src[7]; - - int16x8_t f = vld1q_s16(av1_warped_filter[sy >> WARPEDDIFF_PREC_BITS]); - - int32x4_t m0123 = vmull_lane_s16(vget_low_s16(s0), vget_low_s16(f), 0); - m0123 = vmlal_lane_s16(m0123, vget_low_s16(s1), vget_low_s16(f), 1); - m0123 = vmlal_lane_s16(m0123, vget_low_s16(s2), vget_low_s16(f), 2); - m0123 = vmlal_lane_s16(m0123, vget_low_s16(s3), vget_low_s16(f), 3); - m0123 = vmlal_lane_s16(m0123, vget_low_s16(s4), vget_high_s16(f), 0); - m0123 = vmlal_lane_s16(m0123, vget_low_s16(s5), vget_high_s16(f), 1); - m0123 = vmlal_lane_s16(m0123, vget_low_s16(s6), vget_high_s16(f), 2); - m0123 = vmlal_lane_s16(m0123, vget_low_s16(s7), vget_high_s16(f), 3); - - int32x4_t m4567 = vmull_lane_s16(vget_high_s16(s0), vget_low_s16(f), 0); - m4567 = vmlal_lane_s16(m4567, vget_high_s16(s1), vget_low_s16(f), 1); - m4567 = vmlal_lane_s16(m4567, vget_high_s16(s2), vget_low_s16(f), 2); - m4567 = vmlal_lane_s16(m4567, vget_high_s16(s3), vget_low_s16(f), 3); - m4567 = vmlal_lane_s16(m4567, vget_high_s16(s4), vget_high_s16(f), 0); - m4567 = vmlal_lane_s16(m4567, vget_high_s16(s5), vget_high_s16(f), 1); - m4567 = vmlal_lane_s16(m4567, vget_high_s16(s6), vget_high_s16(f), 2); - m4567 = vmlal_lane_s16(m4567, vget_high_s16(s7), vget_high_s16(f), 3); - - *res_low = m0123; - *res_high = m4567; -} - static AOM_FORCE_INLINE void vertical_filter_8x1_f8(const int16x8_t *src, int32x4_t *res_low, int32x4_t *res_high, int sy, @@ -277,6 +293,157 @@ *res_high = horizontal_add_4d_s32x4(m4567_pairs); } +static AOM_FORCE_INLINE void warp_affine_horizontal_neon_i8mm( + const uint8_t *ref, int width, int height, int stride, int p_width, + int p_height, int16_t alpha, int16_t beta, const int64_t x4, + const int64_t y4, const int i, int16x8_t tmp[]) { + const int height_limit = AOMMIN(8, p_height - i) + 7; + + int32_t ix4 = (int32_t)(x4 >> WARPEDMODEL_PREC_BITS); + int32_t iy4 = (int32_t)(y4 >> WARPEDMODEL_PREC_BITS); + + int32_t sx4 = x4 & ((1 << WARPEDMODEL_PREC_BITS) - 1); + sx4 += alpha * (-4) + beta * (-4) + (1 << (WARPEDDIFF_PREC_BITS - 1)) + + (WARPEDPIXEL_PREC_SHIFTS << WARPEDDIFF_PREC_BITS); + sx4 &= ~((1 << WARP_PARAM_REDUCE_BITS) - 1); + + if (warp_affine_special_case(ref, ix4, iy4, width, height, stride, + height_limit, tmp)) { + return; + } + + static const uint8_t kIotaArr[] = { 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15 }; + const uint8x16_t indx = vld1q_u8(kIotaArr); + + const int out_of_boundary_left = -(ix4 - 6); + const int out_of_boundary_right = (ix4 + 8) - width; + + if (p_width == 4) { + if (beta == 0) { + if (alpha == 0) { + int16_t *f_ptr = + (int16_t *)(av1_warped_filter + (sx4 >> WARPEDDIFF_PREC_BITS)); + int16x8_t f_s16 = vld1q_s16(f_ptr); + const int8x8_t x_filter = vmovn_s16(f_s16); + if ((f_ptr[0] | f_ptr[1]) == 0) { + uint8x16_t perm_tbl = vld1q_u8(kMatMul6PermuteTbl); + // Offset the permutation table to match filter layout. + perm_tbl = vaddq_u8(perm_tbl, vdupq_n_u8(2)); + // Stagger filter for use with the matrix multiply instructions. + // { f2, f3, f4, f5, f6, f7, 0, 0, 0, f2, f3, f4, f5, f6, f7, 0 } + const int8x16_t filter = vcombine_s8(vext_s8(x_filter, x_filter, 2), + vext_s8(x_filter, x_filter, 1)); + APPLY_HORIZONTAL_SHIFT(horizontal_filter_4x1_f1_6tap_beta0, filter, + perm_tbl); + } else if ((f_ptr[0] | f_ptr[7]) == 0) { + uint8x16_t perm_tbl = vld1q_u8(kMatMul6PermuteTbl); + // Offset the permutation table to match filter layout. + perm_tbl = vaddq_u8(perm_tbl, vdupq_n_u8(1)); + // Stagger filter for use with the matrix multiply instructions. + // { f1, f2, f3, f4, f5, f6, 0, 0, 0, f1, f2, f3, f4, f5, f6, 0 } + const int8x16_t filter = + vcombine_s8(vext_s8(x_filter, x_filter, 1), x_filter); + APPLY_HORIZONTAL_SHIFT(horizontal_filter_4x1_f1_6tap_beta0, filter, + perm_tbl); + } else if ((f_ptr[6] | f_ptr[7]) == 0) { + const uint8x16_t perm_tbl = vld1q_u8(kMatMul6PermuteTbl); + // Stagger filter for use with the matrix multiply instructions. + // { f0, f1, f2, f3, f4, f5, 0, 0, 0, f0, f1, f2, f3, f4, f5, 0 } + const int8x16_t filter = + vcombine_s8(x_filter, vext_s8(x_filter, x_filter, 7)); + APPLY_HORIZONTAL_SHIFT(horizontal_filter_4x1_f1_6tap_beta0, filter, + perm_tbl); + } else { + const uint8x16_t perm_tbl = vld1q_u8(kMatMul8PermuteTbl); + const uint8x16_t tbl_idx0_3 = vld1q_u8(kTblIdx0_3); + + // Stagger filter for use with the matrix multiply + // instructions. + // { f1, f2, f3, f4, f5, f6, f7, 0, 0, f1, f2, f3, f4, f5, f6, f7 } + const int8x16_t filter = vcombine_s8( + vext_s8(x_filter, vdup_n_s8(0), 1), vset_lane_s8(0, x_filter, 0)); + const int32x4_t f0 = vdupq_n_s32(f_ptr[0]); + + APPLY_HORIZONTAL_SHIFT(horizontal_filter_4x1_f1_8tap_beta0, filter, + f0, perm_tbl, tbl_idx0_3); + } + } else { + APPLY_HORIZONTAL_SHIFT(horizontal_filter_4x1_f4, sx4, alpha); + } + } else { + if (alpha == 0) { + APPLY_HORIZONTAL_SHIFT(horizontal_filter_4x1_f1, + (sx4 + beta * (k - 3))); + } else { + APPLY_HORIZONTAL_SHIFT(horizontal_filter_4x1_f4, (sx4 + beta * (k - 3)), + alpha); + } + } + } else { + if (beta == 0) { + if (alpha == 0) { + int16_t *f_ptr = + (int16_t *)(av1_warped_filter + (sx4 >> WARPEDDIFF_PREC_BITS)); + int16x8_t f_s16 = vld1q_s16(f_ptr); + const int8x8_t x_filter = vmovn_s16(f_s16); + if ((f_ptr[0] | f_ptr[1]) == 0) { + uint8x16x2_t perm_tbl = vld1q_u8_x2(kMatMul6PermuteTbl); + // Offset the permutation table to match filter layout. + perm_tbl.val[0] = vaddq_u8(perm_tbl.val[0], vdupq_n_u8(2)); + perm_tbl.val[1] = vaddq_u8(perm_tbl.val[1], vdupq_n_u8(2)); + // Stagger filter for use with the matrix multiply instructions. + // { f0, f1, f2, f3, f4, f5, 0, 0, 0, f0, f1, f2, f3, f4, f5, 0 } + const int8x16_t filter = vcombine_s8(vext_s8(x_filter, x_filter, 2), + vext_s8(x_filter, x_filter, 1)); + APPLY_HORIZONTAL_SHIFT(horizontal_filter_8x1_f1_6tap_beta0, filter, + perm_tbl); + } else if ((f_ptr[0] | f_ptr[7]) == 0) { + uint8x16x2_t perm_tbl = vld1q_u8_x2(kMatMul6PermuteTbl); + // Offset the permutation table to match filter layout. + perm_tbl.val[0] = vaddq_u8(perm_tbl.val[0], vdupq_n_u8(1)); + perm_tbl.val[1] = vaddq_u8(perm_tbl.val[1], vdupq_n_u8(1)); + // Stagger filter for use with the matrix multiply instructions. + // { f1, f2, f3, f4, f5, f6, 0, 0, 0, f1, f2, f3, f4, f5, f6, 0 } + const int8x16_t filter = + vcombine_s8(vext_s8(x_filter, x_filter, 1), x_filter); + APPLY_HORIZONTAL_SHIFT(horizontal_filter_8x1_f1_6tap_beta0, filter, + perm_tbl); + } else if ((f_ptr[6] | f_ptr[7]) == 0) { + uint8x16x2_t perm_tbl = vld1q_u8_x2(kMatMul6PermuteTbl); + // Stagger filter for use with the matrix multiply instructions. + // { f0, f1, f2, f3, f4, f5, 0, 0, 0, f0, f1, f2, f3, f4, f5, 0 } + const int8x16_t filter = + vcombine_s8(x_filter, vext_s8(x_filter, x_filter, 7)); + APPLY_HORIZONTAL_SHIFT(horizontal_filter_8x1_f1_6tap_beta0, filter, + perm_tbl); + } else { + uint8x16x2_t perm_tbl = vld1q_u8_x2(kMatMul8PermuteTbl); + // Stagger filter for use with the matrix multiply instructions. + // { f1, f2, f3, f4, f5, f6, f7, 0, 0, f1, f2, f3, f4, f5, f6, f7 } + const int8x16_t filter = vcombine_s8( + vext_s8(x_filter, vdup_n_s8(0), 1), vset_lane_s8(0, x_filter, 0)); + + const int16x4_t f0 = vdup_n_s16(f_ptr[0]); + + APPLY_HORIZONTAL_SHIFT(horizontal_filter_8x1_f1_8tap_beta0, filter, + f0, perm_tbl); + } + } else { + APPLY_HORIZONTAL_SHIFT(horizontal_filter_8x1_f8, sx4, alpha); + } + } else { + if (alpha == 0) { + APPLY_HORIZONTAL_SHIFT(horizontal_filter_8x1_f1, + (sx4 + beta * (k - 3))); + } else { + APPLY_HORIZONTAL_SHIFT(horizontal_filter_8x1_f8, (sx4 + beta * (k - 3)), + alpha); + } + } + } +} + void av1_warp_affine_neon_i8mm(const int32_t *mat, const uint8_t *ref, int width, int height, int stride, uint8_t *pred, int p_col, int p_row, int p_width, int p_height, @@ -284,7 +451,35 @@ int subsampling_y, ConvolveParams *conv_params, int16_t alpha, int16_t beta, int16_t gamma, int16_t delta) { - av1_warp_affine_common(mat, ref, width, height, stride, pred, p_col, p_row, - p_width, p_height, p_stride, subsampling_x, - subsampling_y, conv_params, alpha, beta, gamma, delta); + const int w0 = conv_params->fwd_offset; + const int w1 = conv_params->bck_offset; + const int is_compound = conv_params->is_compound; + uint16_t *const dst = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + const int do_average = conv_params->do_average; + const int use_dist_wtd_comp_avg = conv_params->use_dist_wtd_comp_avg; + + assert(IMPLIES(is_compound, dst != NULL)); + assert(IMPLIES(do_average, is_compound)); + + for (int i = 0; i < p_height; i += 8) { + for (int j = 0; j < p_width; j += 8) { + const int32_t src_x = (p_col + j + 4) << subsampling_x; + const int32_t src_y = (p_row + i + 4) << subsampling_y; + const int64_t dst_x = + (int64_t)mat[2] * src_x + (int64_t)mat[3] * src_y + (int64_t)mat[0]; + const int64_t dst_y = + (int64_t)mat[4] * src_x + (int64_t)mat[5] * src_y + (int64_t)mat[1]; + + const int64_t x4 = dst_x >> subsampling_x; + const int64_t y4 = dst_y >> subsampling_y; + + int16x8_t tmp[15]; + warp_affine_horizontal_neon_i8mm(ref, width, height, stride, p_width, + p_height, alpha, beta, x4, y4, i, tmp); + warp_affine_vertical(pred, p_width, p_height, p_stride, is_compound, dst, + dst_stride, do_average, use_dist_wtd_comp_avg, gamma, + delta, y4, i, j, tmp, w0, w1); + } + } }
diff --git a/av1/common/arm/warp_plane_neon_i8mm.h b/av1/common/arm/warp_plane_neon_i8mm.h new file mode 100644 index 0000000..99355dc --- /dev/null +++ b/av1/common/arm/warp_plane_neon_i8mm.h
@@ -0,0 +1,18 @@ +/* + * Copyright (c) 2025, Alliance for Open Media. All rights reserved. + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef AOM_AV1_COMMON_ARM_WARP_PLANE_NEON_I8MM_H_ +#define AOM_AV1_COMMON_ARM_WARP_PLANE_NEON_I8MM_H_ + +#include "aom_ports/mem.h" + +DECLARE_ALIGNED(16, extern const uint8_t, kTblIdx0_3[16]); + +#endif // AOM_AV1_COMMON_ARM_WARP_PLANE_NEON_I8MM_H_
diff --git a/av1/common/arm/warp_plane_sve.c b/av1/common/arm/warp_plane_sve.c index 10aee35..3d21072 100644 --- a/av1/common/arm/warp_plane_sve.c +++ b/av1/common/arm/warp_plane_sve.c
@@ -12,13 +12,10 @@ #include <arm_neon.h> #include "aom_dsp/arm/aom_neon_sve_bridge.h" +#include "convolve_neon_dotprod.h" +#include "convolve_neon_i8mm.h" #include "warp_plane_neon.h" - -DECLARE_ALIGNED(16, static const uint8_t, usdot_permute_idx[48]) = { - 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6, - 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10, - 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 -}; +#include "warp_plane_neon_i8mm.h" static AOM_FORCE_INLINE int16x8_t horizontal_filter_4x1_f4(const uint8x16_t in, int sx, int alpha) { @@ -87,14 +84,56 @@ return vreinterpretq_s16_u16(res); } -static AOM_FORCE_INLINE int16x8_t -horizontal_filter_4x1_f1_beta0(const uint8x16_t in, int16x8_t f_s16) { +static AOM_FORCE_INLINE int16x8_t horizontal_filter_4x1_f1_6tap_beta0( + const uint8x16_t in, const int8x16_t filter, const uint8x16_t perm_tbl) { const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1)); + // Permute samples ready for matrix multiply. + // { 0, 1, 2, 3, 4, 5, 6, 7, 2, 3, 4, 5, 6, 7, 8, 9 } + const uint8x16_t perm_samples = vqtbl1q_u8(in, perm_tbl); + + // These instructions multiply a 2x8 matrix (samples) by an 8x2 matrix + // (filter), destructively accumulating into the destination register. + int32x4_t sum = vusmmlaq_s32(add_const, perm_samples, filter); + + uint16x8_t res = + vcombine_u16(vqrshrun_n_s32(sum, ROUND0_BITS), vdup_n_u16(0)); + + return vreinterpretq_s16_u16(res); +} + +static AOM_FORCE_INLINE int16x8_t horizontal_filter_4x1_f1_8tap_beta0( + const uint8x16_t in, const int8x16_t filter, const int32x4_t f0, + const uint8x16_t perm_tbl, const uint8x16_t tbl_idx0_3) { + const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1)); + + // Permute samples ready for matrix multiply. + // { 1, 2, 3, 4, 5, 6, 7, 8, 3, 4, 5, 6, 7, 8, 9, 10 } + const uint8x16_t perm_samples = vqtbl1q_u8(in, perm_tbl); + // Get samples 0..3 to apply tap 0 after matrix multiply. + const int32x4_t samples_0_3 = + vreinterpretq_s32_u8(vqtbl1q_u8(in, tbl_idx0_3)); + + // Calculate partial 7-tap convolution. + int32x4_t sum = vusmmlaq_s32(add_const, perm_samples, filter); + // Apply tap 0 and accumulate. + sum = vmlaq_s32(sum, samples_0_3, f0); + + uint16x8_t res = + vcombine_u16(vqrshrun_n_s32(sum, ROUND0_BITS), vdup_n_u16(0)); + + return vreinterpretq_s16_u16(res); +} + +static AOM_FORCE_INLINE int16x8_t horizontal_filter_4x1_f1(const uint8x16_t in, + int sx) { + const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1)); + + int16x8_t f_s16 = vld1q_s16(av1_warped_filter[sx >> WARPEDDIFF_PREC_BITS]); int8x16_t f_s8 = vcombine_s8(vmovn_s16(f_s16), vmovn_s16(f_s16)); - uint8x16_t perm0 = vld1q_u8(&usdot_permute_idx[0]); - uint8x16_t perm1 = vld1q_u8(&usdot_permute_idx[16]); + uint8x16_t perm0 = vld1q_u8(&kDotProdPermuteTbl[0]); + uint8x16_t perm1 = vld1q_u8(&kDotProdPermuteTbl[16]); // Permute samples ready for dot product. // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } @@ -110,21 +149,63 @@ return vreinterpretq_s16_u16(res); } -static AOM_FORCE_INLINE int16x8_t horizontal_filter_4x1_f1(const uint8x16_t in, - int sx) { - int16x8_t f_s16 = vld1q_s16(av1_warped_filter[sx >> WARPEDDIFF_PREC_BITS]); - return horizontal_filter_4x1_f1_beta0(in, f_s16); -} - -static AOM_FORCE_INLINE int16x8_t -horizontal_filter_8x1_f1_beta0(const uint8x16_t in, int16x8_t f_s16) { +static AOM_FORCE_INLINE int16x8_t horizontal_filter_8x1_f1_6tap_beta0( + const uint8x16_t in, const int8x16_t filter, const uint8x16x2_t perm_tbl) { const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1)); + // Permute samples ready for matrix multiply. + // { 0, 1, 2, 3, 4, 5, 6, 7, 2, 3, 4, 5, 6, 7, 8, 9 } + // { 4, 5, 6, 7, 8, 9, 10, 11, 6, 7, 8, 9, 10, 11, 12, 13 } + uint8x16_t perm_samples[2] = { vqtbl1q_u8(in, perm_tbl.val[0]), + vqtbl1q_u8(in, perm_tbl.val[1]) }; + + // These instructions multiply a 2x8 matrix (samples) by an 8x2 matrix + // (filter), destructively accumulating into the destination register. + int32x4_t sum0123 = vusmmlaq_s32(add_const, perm_samples[0], filter); + int32x4_t sum4567 = vusmmlaq_s32(add_const, perm_samples[1], filter); + + uint16x8_t res = vcombine_u16(vqrshrun_n_s32(sum0123, ROUND0_BITS), + vqrshrun_n_s32(sum4567, ROUND0_BITS)); + + return vreinterpretq_s16_u16(res); +} + +static AOM_FORCE_INLINE int16x8_t horizontal_filter_8x1_f1_8tap_beta0( + const uint8x16_t in, const int8x16_t filter, const int16x4_t f0, + const uint8x16x2_t perm_tbl) { + const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1)); + + // Permute samples ready for matrix multiply. + // { 1, 2, 3, 4, 5, 6, 7, 8, 3, 4, 5, 6, 7, 8, 9, 10 } + // { 5, 6, 7, 8, 9, 10, 11, 12, 7, 8, 9, 10, 11, 12, 13, 14 } + uint8x16_t perm_samples[2] = { vqtbl1q_u8(in, perm_tbl.val[0]), + vqtbl1q_u8(in, perm_tbl.val[1]) }; + // Get samples 0..7 to apply tap 0 after matrix multiply. + int16x8_t samples_0_7 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(in))); + + // Calculate partial 7-tap convolution. + int32x4_t sum0123 = vusmmlaq_s32(add_const, perm_samples[0], filter); + int32x4_t sum4567 = vusmmlaq_s32(add_const, perm_samples[1], filter); + // Apply tap 0 and accumulate. + sum0123 = vmlal_s16(sum0123, vget_low_s16(samples_0_7), f0); + sum4567 = vmlal_s16(sum4567, vget_high_s16(samples_0_7), f0); + + uint16x8_t res = vcombine_u16(vqrshrun_n_s32(sum0123, ROUND0_BITS), + vqrshrun_n_s32(sum4567, ROUND0_BITS)); + + return vreinterpretq_s16_u16(res); +} + +static AOM_FORCE_INLINE int16x8_t horizontal_filter_8x1_f1(const uint8x16_t in, + int sx) { + const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1)); + + int16x8_t f_s16 = vld1q_s16(av1_warped_filter[sx >> WARPEDDIFF_PREC_BITS]); int8x16_t f_s8 = vcombine_s8(vmovn_s16(f_s16), vmovn_s16(f_s16)); - uint8x16_t perm0 = vld1q_u8(&usdot_permute_idx[0]); - uint8x16_t perm1 = vld1q_u8(&usdot_permute_idx[16]); - uint8x16_t perm2 = vld1q_u8(&usdot_permute_idx[32]); + uint8x16_t perm0 = vld1q_u8(&kDotProdPermuteTbl[0]); + uint8x16_t perm1 = vld1q_u8(&kDotProdPermuteTbl[16]); + uint8x16_t perm2 = vld1q_u8(&kDotProdPermuteTbl[32]); // Permute samples ready for dot product. // { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } @@ -145,37 +226,6 @@ return vreinterpretq_s16_u16(res); } -static AOM_FORCE_INLINE int16x8_t horizontal_filter_8x1_f1(const uint8x16_t in, - int sx) { - int16x8_t f_s16 = vld1q_s16(av1_warped_filter[sx >> WARPEDDIFF_PREC_BITS]); - return horizontal_filter_8x1_f1_beta0(in, f_s16); -} - -static AOM_FORCE_INLINE void vertical_filter_4x1_f1(const int16x8_t *src, - int32x4_t *res, int sy) { - int16x4_t s0 = vget_low_s16(src[0]); - int16x4_t s1 = vget_low_s16(src[1]); - int16x4_t s2 = vget_low_s16(src[2]); - int16x4_t s3 = vget_low_s16(src[3]); - int16x4_t s4 = vget_low_s16(src[4]); - int16x4_t s5 = vget_low_s16(src[5]); - int16x4_t s6 = vget_low_s16(src[6]); - int16x4_t s7 = vget_low_s16(src[7]); - - int16x8_t f = vld1q_s16(av1_warped_filter[sy >> WARPEDDIFF_PREC_BITS]); - - int32x4_t m0123 = vmull_lane_s16(s0, vget_low_s16(f), 0); - m0123 = vmlal_lane_s16(m0123, s1, vget_low_s16(f), 1); - m0123 = vmlal_lane_s16(m0123, s2, vget_low_s16(f), 2); - m0123 = vmlal_lane_s16(m0123, s3, vget_low_s16(f), 3); - m0123 = vmlal_lane_s16(m0123, s4, vget_high_s16(f), 0); - m0123 = vmlal_lane_s16(m0123, s5, vget_high_s16(f), 1); - m0123 = vmlal_lane_s16(m0123, s6, vget_high_s16(f), 2); - m0123 = vmlal_lane_s16(m0123, s7, vget_high_s16(f), 3); - - *res = m0123; -} - static AOM_FORCE_INLINE void vertical_filter_4x1_f4(const int16x8_t *src, int32x4_t *res, int sy, int gamma) { @@ -199,43 +249,6 @@ *res = vcombine_s32(vmovn_s64(m01), vmovn_s64(m23)); } -static AOM_FORCE_INLINE void vertical_filter_8x1_f1(const int16x8_t *src, - int32x4_t *res_low, - int32x4_t *res_high, - int sy) { - int16x8_t s0 = src[0]; - int16x8_t s1 = src[1]; - int16x8_t s2 = src[2]; - int16x8_t s3 = src[3]; - int16x8_t s4 = src[4]; - int16x8_t s5 = src[5]; - int16x8_t s6 = src[6]; - int16x8_t s7 = src[7]; - - int16x8_t f = vld1q_s16(av1_warped_filter[sy >> WARPEDDIFF_PREC_BITS]); - - int32x4_t m0123 = vmull_lane_s16(vget_low_s16(s0), vget_low_s16(f), 0); - m0123 = vmlal_lane_s16(m0123, vget_low_s16(s1), vget_low_s16(f), 1); - m0123 = vmlal_lane_s16(m0123, vget_low_s16(s2), vget_low_s16(f), 2); - m0123 = vmlal_lane_s16(m0123, vget_low_s16(s3), vget_low_s16(f), 3); - m0123 = vmlal_lane_s16(m0123, vget_low_s16(s4), vget_high_s16(f), 0); - m0123 = vmlal_lane_s16(m0123, vget_low_s16(s5), vget_high_s16(f), 1); - m0123 = vmlal_lane_s16(m0123, vget_low_s16(s6), vget_high_s16(f), 2); - m0123 = vmlal_lane_s16(m0123, vget_low_s16(s7), vget_high_s16(f), 3); - - int32x4_t m4567 = vmull_lane_s16(vget_high_s16(s0), vget_low_s16(f), 0); - m4567 = vmlal_lane_s16(m4567, vget_high_s16(s1), vget_low_s16(f), 1); - m4567 = vmlal_lane_s16(m4567, vget_high_s16(s2), vget_low_s16(f), 2); - m4567 = vmlal_lane_s16(m4567, vget_high_s16(s3), vget_low_s16(f), 3); - m4567 = vmlal_lane_s16(m4567, vget_high_s16(s4), vget_high_s16(f), 0); - m4567 = vmlal_lane_s16(m4567, vget_high_s16(s5), vget_high_s16(f), 1); - m4567 = vmlal_lane_s16(m4567, vget_high_s16(s6), vget_high_s16(f), 2); - m4567 = vmlal_lane_s16(m4567, vget_high_s16(s7), vget_high_s16(f), 3); - - *res_low = m0123; - *res_high = m4567; -} - static AOM_FORCE_INLINE void vertical_filter_8x1_f8(const int16x8_t *src, int32x4_t *res_low, int32x4_t *res_high, int sy, @@ -271,13 +284,192 @@ *res_high = vcombine_s32(vmovn_s64(m45), vmovn_s64(m67)); } +static AOM_FORCE_INLINE void warp_affine_horizontal_sve( + const uint8_t *ref, int width, int height, int stride, int p_width, + int p_height, int16_t alpha, int16_t beta, const int64_t x4, + const int64_t y4, const int i, int16x8_t tmp[]) { + const int height_limit = AOMMIN(8, p_height - i) + 7; + + int32_t ix4 = (int32_t)(x4 >> WARPEDMODEL_PREC_BITS); + int32_t iy4 = (int32_t)(y4 >> WARPEDMODEL_PREC_BITS); + + int32_t sx4 = x4 & ((1 << WARPEDMODEL_PREC_BITS) - 1); + sx4 += alpha * (-4) + beta * (-4) + (1 << (WARPEDDIFF_PREC_BITS - 1)) + + (WARPEDPIXEL_PREC_SHIFTS << WARPEDDIFF_PREC_BITS); + sx4 &= ~((1 << WARP_PARAM_REDUCE_BITS) - 1); + + if (warp_affine_special_case(ref, ix4, iy4, width, height, stride, + height_limit, tmp)) { + return; + } + + static const uint8_t kIotaArr[] = { 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15 }; + const uint8x16_t indx = vld1q_u8(kIotaArr); + + const int out_of_boundary_left = -(ix4 - 6); + const int out_of_boundary_right = (ix4 + 8) - width; + + if (p_width == 4) { + if (beta == 0) { + if (alpha == 0) { + int16_t *f_ptr = + (int16_t *)(av1_warped_filter + (sx4 >> WARPEDDIFF_PREC_BITS)); + int16x8_t f_s16 = vld1q_s16(f_ptr); + const int8x8_t x_filter = vmovn_s16(f_s16); + if ((f_ptr[0] | f_ptr[1]) == 0) { + uint8x16_t perm_tbl = vld1q_u8(kMatMul6PermuteTbl); + // Offset the permutation table to match filter layout. + perm_tbl = vaddq_u8(perm_tbl, vdupq_n_u8(2)); + // Stagger filter for use with the matrix multiply instructions. + // { f2, f3, f4, f5, f6, f7, 0, 0, 0, f2, f3, f4, f5, f6, f7, 0 } + const int8x16_t filter = vcombine_s8(vext_s8(x_filter, x_filter, 2), + vext_s8(x_filter, x_filter, 1)); + APPLY_HORIZONTAL_SHIFT(horizontal_filter_4x1_f1_6tap_beta0, filter, + perm_tbl); + } else if ((f_ptr[0] | f_ptr[7]) == 0) { + uint8x16_t perm_tbl = vld1q_u8(kMatMul6PermuteTbl); + // Offset the permutation table to match filter layout. + perm_tbl = vaddq_u8(perm_tbl, vdupq_n_u8(1)); + // Stagger filter for use with the matrix multiply instructions. + // { f1, f2, f3, f4, f5, f6, 0, 0, 0, f1, f2, f3, f4, f5, f6, 0 } + const int8x16_t filter = + vcombine_s8(vext_s8(x_filter, x_filter, 1), x_filter); + APPLY_HORIZONTAL_SHIFT(horizontal_filter_4x1_f1_6tap_beta0, filter, + perm_tbl); + } else if ((f_ptr[6] | f_ptr[7]) == 0) { + const uint8x16_t perm_tbl = vld1q_u8(kMatMul6PermuteTbl); + // Stagger filter for use with the matrix multiply instructions. + // { f0, f1, f2, f3, f4, f5, 0, 0, 0, f0, f1, f2, f3, f4, f5, 0 } + const int8x16_t filter = + vcombine_s8(x_filter, vext_s8(x_filter, x_filter, 7)); + APPLY_HORIZONTAL_SHIFT(horizontal_filter_4x1_f1_6tap_beta0, filter, + perm_tbl); + } else { + const uint8x16_t perm_tbl = vld1q_u8(kMatMul8PermuteTbl); + const uint8x16_t tbl_idx0_3 = vld1q_u8(kTblIdx0_3); + + // Stagger filter for use with the matrix multiply + // instructions. + // { f1, f2, f3, f4, f5, f6, f7, 0, 0, f1, f2, f3, f4, f5, f6, f7 } + const int8x16_t filter = vcombine_s8( + vext_s8(x_filter, vdup_n_s8(0), 1), vset_lane_s8(0, x_filter, 0)); + const int32x4_t f0 = vdupq_n_s32(f_ptr[0]); + + APPLY_HORIZONTAL_SHIFT(horizontal_filter_4x1_f1_8tap_beta0, filter, + f0, perm_tbl, tbl_idx0_3); + } + } else { + APPLY_HORIZONTAL_SHIFT(horizontal_filter_4x1_f4, sx4, alpha); + } + } else { + if (alpha == 0) { + APPLY_HORIZONTAL_SHIFT(horizontal_filter_4x1_f1, + (sx4 + beta * (k - 3))); + } else { + APPLY_HORIZONTAL_SHIFT(horizontal_filter_4x1_f4, (sx4 + beta * (k - 3)), + alpha); + } + } + } else { + if (beta == 0) { + if (alpha == 0) { + int16_t *f_ptr = + (int16_t *)(av1_warped_filter + (sx4 >> WARPEDDIFF_PREC_BITS)); + int16x8_t f_s16 = vld1q_s16(f_ptr); + const int8x8_t x_filter = vmovn_s16(f_s16); + if ((f_ptr[0] | f_ptr[1]) == 0) { + uint8x16x2_t perm_tbl = vld1q_u8_x2(kMatMul6PermuteTbl); + // Offset the permutation table to match filter layout. + perm_tbl.val[0] = vaddq_u8(perm_tbl.val[0], vdupq_n_u8(2)); + perm_tbl.val[1] = vaddq_u8(perm_tbl.val[1], vdupq_n_u8(2)); + // Stagger filter for use with the matrix multiply instructions. + // { f0, f1, f2, f3, f4, f5, 0, 0, 0, f0, f1, f2, f3, f4, f5, 0 } + const int8x16_t filter = vcombine_s8(vext_s8(x_filter, x_filter, 2), + vext_s8(x_filter, x_filter, 1)); + APPLY_HORIZONTAL_SHIFT(horizontal_filter_8x1_f1_6tap_beta0, filter, + perm_tbl); + } else if ((f_ptr[0] | f_ptr[7]) == 0) { + uint8x16x2_t perm_tbl = vld1q_u8_x2(kMatMul6PermuteTbl); + // Offset the permutation table to match filter layout. + perm_tbl.val[0] = vaddq_u8(perm_tbl.val[0], vdupq_n_u8(1)); + perm_tbl.val[1] = vaddq_u8(perm_tbl.val[1], vdupq_n_u8(1)); + // Stagger filter for use with the matrix multiply instructions. + // { f1, f2, f3, f4, f5, f6, 0, 0, 0, f1, f2, f3, f4, f5, f6, 0 } + const int8x16_t filter = + vcombine_s8(vext_s8(x_filter, x_filter, 1), x_filter); + APPLY_HORIZONTAL_SHIFT(horizontal_filter_8x1_f1_6tap_beta0, filter, + perm_tbl); + } else if ((f_ptr[6] | f_ptr[7]) == 0) { + uint8x16x2_t perm_tbl = vld1q_u8_x2(kMatMul6PermuteTbl); + // Stagger filter for use with the matrix multiply instructions. + // { f0, f1, f2, f3, f4, f5, 0, 0, 0, f0, f1, f2, f3, f4, f5, 0 } + const int8x16_t filter = + vcombine_s8(x_filter, vext_s8(x_filter, x_filter, 7)); + APPLY_HORIZONTAL_SHIFT(horizontal_filter_8x1_f1_6tap_beta0, filter, + perm_tbl); + } else { + uint8x16x2_t perm_tbl = vld1q_u8_x2(kMatMul8PermuteTbl); + // Stagger filter for use with the matrix multiply instructions. + // { f1, f2, f3, f4, f5, f6, f7, 0, 0, f1, f2, f3, f4, f5, f6, f7 } + const int8x16_t filter = vcombine_s8( + vext_s8(x_filter, vdup_n_s8(0), 1), vset_lane_s8(0, x_filter, 0)); + + const int16x4_t f0 = vdup_n_s16(f_ptr[0]); + + APPLY_HORIZONTAL_SHIFT(horizontal_filter_8x1_f1_8tap_beta0, filter, + f0, perm_tbl); + } + } else { + APPLY_HORIZONTAL_SHIFT(horizontal_filter_8x1_f8, sx4, alpha); + } + } else { + if (alpha == 0) { + APPLY_HORIZONTAL_SHIFT(horizontal_filter_8x1_f1, + (sx4 + beta * (k - 3))); + } else { + APPLY_HORIZONTAL_SHIFT(horizontal_filter_8x1_f8, (sx4 + beta * (k - 3)), + alpha); + } + } + } +} + void av1_warp_affine_sve(const int32_t *mat, const uint8_t *ref, int width, int height, int stride, uint8_t *pred, int p_col, int p_row, int p_width, int p_height, int p_stride, int subsampling_x, int subsampling_y, ConvolveParams *conv_params, int16_t alpha, int16_t beta, int16_t gamma, int16_t delta) { - av1_warp_affine_common(mat, ref, width, height, stride, pred, p_col, p_row, - p_width, p_height, p_stride, subsampling_x, - subsampling_y, conv_params, alpha, beta, gamma, delta); + const int w0 = conv_params->fwd_offset; + const int w1 = conv_params->bck_offset; + const int is_compound = conv_params->is_compound; + uint16_t *const dst = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + const int do_average = conv_params->do_average; + const int use_dist_wtd_comp_avg = conv_params->use_dist_wtd_comp_avg; + + assert(IMPLIES(is_compound, dst != NULL)); + assert(IMPLIES(do_average, is_compound)); + + for (int i = 0; i < p_height; i += 8) { + for (int j = 0; j < p_width; j += 8) { + const int32_t src_x = (p_col + j + 4) << subsampling_x; + const int32_t src_y = (p_row + i + 4) << subsampling_y; + const int64_t dst_x = + (int64_t)mat[2] * src_x + (int64_t)mat[3] * src_y + (int64_t)mat[0]; + const int64_t dst_y = + (int64_t)mat[4] * src_x + (int64_t)mat[5] * src_y + (int64_t)mat[1]; + + const int64_t x4 = dst_x >> subsampling_x; + const int64_t y4 = dst_y >> subsampling_y; + + int16x8_t tmp[15]; + warp_affine_horizontal_sve(ref, width, height, stride, p_width, p_height, + alpha, beta, x4, y4, i, tmp); + warp_affine_vertical(pred, p_width, p_height, p_stride, is_compound, dst, + dst_stride, do_average, use_dist_wtd_comp_avg, gamma, + delta, y4, i, j, tmp, w0, w1); + } + } }
diff --git a/av1/common/av1_common_int.h b/av1/common/av1_common_int.h index 0961c44..e083c2b 100644 --- a/av1/common/av1_common_int.h +++ b/av1/common/av1_common_int.h
@@ -139,6 +139,7 @@ unsigned int ref_display_order_hint[INTER_REFS_PER_FRAME]; // Frame's level within the hierarchical structure. unsigned int pyramid_level; + int base_qindex; MV_REF *mvs; uint8_t *seg_map; struct segmentation seg; @@ -703,8 +704,9 @@ /*! * Flag indicating whether quantization matrices are being used: - * - If true, qm_level_y, qm_level_u and qm_level_v indicate the level - * indices to be used to access appropriate global quant matrix tables. + * - If true, qmatrix_level_y, qmatrix_level_u and qmatrix_level_v indicate + * the level indices to be used to access appropriate global quant matrix + * tables. * - If false, we implicitly use level index 'NUM_QM_LEVELS - 1'. */ bool using_qmatrix;
diff --git a/av1/common/av1_inv_txfm1d.c b/av1/common/av1_inv_txfm1d.c index 37c2091..49380e8 100644 --- a/av1/common/av1_inv_txfm1d.c +++ b/av1/common/av1_inv_txfm1d.c
@@ -657,12 +657,12 @@ const int8_t *stage_range) { int bit = cos_bit; const int32_t *sinpi = sinpi_arr(bit); - int32_t s0, s1, s2, s3, s4, s5, s6, s7; + int64_t s0, s1, s2, s3, s4, s5, s6, s7; - int32_t x0 = input[0]; - int32_t x1 = input[1]; - int32_t x2 = input[2]; - int32_t x3 = input[3]; + int64_t x0 = input[0]; + int64_t x1 = input[1]; + int64_t x2 = input[2]; + int64_t x3 = input[3]; if (!(x0 | x1 | x2 | x3)) { output[0] = output[1] = output[2] = output[3] = 0; @@ -672,37 +672,37 @@ assert(sinpi[1] + sinpi[2] == sinpi[4]); // stage 1 - s0 = range_check_value(sinpi[1] * x0, stage_range[1] + bit); - s1 = range_check_value(sinpi[2] * x0, stage_range[1] + bit); - s2 = range_check_value(sinpi[3] * x1, stage_range[1] + bit); - s3 = range_check_value(sinpi[4] * x2, stage_range[1] + bit); - s4 = range_check_value(sinpi[1] * x2, stage_range[1] + bit); - s5 = range_check_value(sinpi[2] * x3, stage_range[1] + bit); - s6 = range_check_value(sinpi[4] * x3, stage_range[1] + bit); + s0 = range_check_value64(sinpi[1] * x0, stage_range[1] + bit); + s1 = range_check_value64(sinpi[2] * x0, stage_range[1] + bit); + s2 = range_check_value64(sinpi[3] * x1, stage_range[1] + bit); + s3 = range_check_value64(sinpi[4] * x2, stage_range[1] + bit); + s4 = range_check_value64(sinpi[1] * x2, stage_range[1] + bit); + s5 = range_check_value64(sinpi[2] * x3, stage_range[1] + bit); + s6 = range_check_value64(sinpi[4] * x3, stage_range[1] + bit); // stage 2 // NOTICE: (x0 - x2) here may use one extra bit compared to the // opt_range_row/col specified in av1_gen_inv_stage_range() - s7 = range_check_value((x0 - x2) + x3, stage_range[2]); + s7 = range_check_value64((x0 - x2) + x3, stage_range[2]); // stage 3 - s0 = range_check_value(s0 + s3, stage_range[3] + bit); - s1 = range_check_value(s1 - s4, stage_range[3] + bit); - s3 = range_check_value(s2, stage_range[3] + bit); - s2 = range_check_value(sinpi[3] * s7, stage_range[3] + bit); + s0 = range_check_value64(s0 + s3, stage_range[3] + bit); + s1 = range_check_value64(s1 - s4, stage_range[3] + bit); + s3 = range_check_value64(s2, stage_range[3] + bit); + s2 = range_check_value64(sinpi[3] * s7, stage_range[3] + bit); // stage 4 - s0 = range_check_value(s0 + s5, stage_range[4] + bit); - s1 = range_check_value(s1 - s6, stage_range[4] + bit); + s0 = range_check_value64(s0 + s5, stage_range[4] + bit); + s1 = range_check_value64(s1 - s6, stage_range[4] + bit); // stage 5 - x0 = range_check_value(s0 + s3, stage_range[5] + bit); - x1 = range_check_value(s1 + s3, stage_range[5] + bit); - x2 = range_check_value(s2, stage_range[5] + bit); - x3 = range_check_value(s0 + s1, stage_range[5] + bit); + x0 = range_check_value64(s0 + s3, stage_range[5] + bit); + x1 = range_check_value64(s1 + s3, stage_range[5] + bit); + x2 = range_check_value64(s2, stage_range[5] + bit); + x3 = range_check_value64(s0 + s1, stage_range[5] + bit); // stage 6 - x3 = range_check_value(x3 - s3, stage_range[6] + bit); + x3 = range_check_value64(x3 - s3, stage_range[6] + bit); output[0] = round_shift(x0, bit); output[1] = round_shift(x1, bit);
diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl index 4c90393..a3d0d12 100644 --- a/av1/common/av1_rtcd_defs.pl +++ b/av1/common/av1_rtcd_defs.pl
@@ -126,12 +126,7 @@ # FILTER_INTRA predictor functions add_proto qw/void av1_filter_intra_predictor/, "uint8_t *dst, ptrdiff_t stride, TX_SIZE tx_size, const uint8_t *above, const uint8_t *left, int mode"; -# TODO(aomedia:349436249): enable NEON for armv7 after SIGBUS is fixed. -if (aom_config("AOM_ARCH_ARM") eq "yes" && aom_config("AOM_ARCH_AARCH64") eq "") { - specialize qw/av1_filter_intra_predictor sse4_1/; -} else { - specialize qw/av1_filter_intra_predictor sse4_1 neon/; -} +specialize qw/av1_filter_intra_predictor sse4_1 neon neon_i8mm/; # High bitdepth functions @@ -557,6 +552,9 @@ if ((aom_config("CONFIG_REALTIME_ONLY") ne "yes") || (aom_config("CONFIG_AV1_DECODER") eq "yes")) { add_proto qw/void av1_warp_affine/, "const int32_t *mat, const uint8_t *ref, int width, int height, int stride, uint8_t *pred, int p_col, int p_row, int p_width, int p_height, int p_stride, int subsampling_x, int subsampling_y, ConvolveParams *conv_params, int16_t alpha, int16_t beta, int16_t gamma, int16_t delta"; specialize qw/av1_warp_affine sse4_1 avx2 neon neon_i8mm sve/; + if (aom_config("CONFIG_HIGHWAY") eq "yes") { + specialize qw/av1_warp_affine avx512/; + } } # LOOP_RESTORATION functions @@ -610,15 +608,15 @@ specialize qw/av1_convolve_y_sr sse2 avx2 neon neon_dotprod neon_i8mm rvv/; specialize qw/av1_convolve_y_sr_intrabc neon rvv/; specialize qw/av1_convolve_2d_scale sse4_1 neon neon_dotprod neon_i8mm/; - specialize qw/av1_dist_wtd_convolve_2d ssse3 avx2 neon neon_dotprod neon_i8mm/; - specialize qw/av1_dist_wtd_convolve_2d_copy sse2 avx2 neon/; - specialize qw/av1_dist_wtd_convolve_x sse2 avx2 neon neon_dotprod neon_i8mm/; - specialize qw/av1_dist_wtd_convolve_y sse2 avx2 neon/; + specialize qw/av1_dist_wtd_convolve_2d ssse3 avx2 neon neon_dotprod neon_i8mm rvv/; + specialize qw/av1_dist_wtd_convolve_2d_copy sse2 avx2 neon rvv/; + specialize qw/av1_dist_wtd_convolve_x sse2 avx2 neon neon_dotprod neon_i8mm rvv/; + specialize qw/av1_dist_wtd_convolve_y sse2 avx2 neon neon_dotprod neon_i8mm rvv/; if(aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") { - specialize qw/av1_highbd_dist_wtd_convolve_2d sse4_1 avx2 neon sve2/; - specialize qw/av1_highbd_dist_wtd_convolve_x sse4_1 avx2 neon sve2/; - specialize qw/av1_highbd_dist_wtd_convolve_y sse4_1 avx2 neon sve2/; - specialize qw/av1_highbd_dist_wtd_convolve_2d_copy sse4_1 avx2 neon/; + specialize qw/av1_highbd_dist_wtd_convolve_2d sse4_1 avx2 neon sve2 rvv/; + specialize qw/av1_highbd_dist_wtd_convolve_x sse4_1 avx2 neon sve2 rvv/; + specialize qw/av1_highbd_dist_wtd_convolve_y sse4_1 avx2 neon sve2 rvv/; + specialize qw/av1_highbd_dist_wtd_convolve_2d_copy sse4_1 avx2 neon rvv/; specialize qw/av1_highbd_convolve_2d_sr ssse3 avx2 neon sve2 rvv/; specialize qw/av1_highbd_convolve_2d_sr_intrabc neon rvv/; specialize qw/av1_highbd_convolve_x_sr ssse3 avx2 neon sve2 rvv/;
diff --git a/av1/common/av1_txfm.c b/av1/common/av1_txfm.c index 751851c..854e85b 100644 --- a/av1/common/av1_txfm.c +++ b/av1/common/av1_txfm.c
@@ -9,6 +9,8 @@ * PATENTS file, you can obtain it at www.aomedia.org/license/patent. */ +#include <inttypes.h> + #include "config/aom_dsp_rtcd.h" #include "config/av1_rtcd.h"
diff --git a/av1/common/av1_txfm.h b/av1/common/av1_txfm.h index c3c3123..8603c3d 100644 --- a/av1/common/av1_txfm.h +++ b/av1/common/av1_txfm.h
@@ -13,6 +13,7 @@ #define AOM_AV1_COMMON_AV1_TXFM_H_ #include <assert.h> +#include <inttypes.h> #include <math.h> #include <stdio.h> @@ -94,6 +95,26 @@ return value; } +static inline int64_t range_check_value64(int64_t value, int8_t bit) { +#if CONFIG_COEFFICIENT_RANGE_CHECKING + const int64_t max_value = (1LL << (bit - 1)) - 1; + const int64_t min_value = -(1LL << (bit - 1)); + if (value < min_value || value > max_value) { + fprintf(stderr, "coeff out of bit range, value: %" PRId64 " bit %d\n", + value, bit); +#if !CONFIG_AV1_ENCODER + assert(0); +#endif + } +#endif // CONFIG_COEFFICIENT_RANGE_CHECKING +#if DO_RANGE_CHECK_CLAMP + bit = AOMMIN(bit, 63); + return clamp64(value, -(1LL << (bit - 1)), (1LL << (bit - 1)) - 1); +#endif // DO_RANGE_CHECK_CLAMP + (void)bit; + return value; +} + static inline int32_t round_shift(int64_t value, int bit) { assert(bit >= 1); return (int32_t)((value + (1ll << (bit - 1))) >> bit);
diff --git a/av1/common/blockd.h b/av1/common/blockd.h index b95235f..db91270 100644 --- a/av1/common/blockd.h +++ b/av1/common/blockd.h
@@ -925,6 +925,11 @@ * 'cpi->tile_thr_data[t].td->mb.tmp_pred_bufs'. */ uint8_t *tmp_obmc_bufs[2]; + + /*! + * Temporary buffer used for upsampled prediction. + */ + uint8_t *tmp_upsample_pred; } MACROBLOCKD; /*!\cond */
diff --git a/av1/common/filter.h b/av1/common/filter.h index d1b76bd..fb85ab2 100644 --- a/av1/common/filter.h +++ b/av1/common/filter.h
@@ -301,20 +301,16 @@ if (filter_params->taps == 12) { return 12; } - if (filter[0] | filter[7]) { + if (filter[0] || filter[7]) { return 8; } - if (filter[1] | filter[6]) { + if (filter[1] || filter[6]) { return 6; } -#if CONFIG_SVT_AV1 - if (filter[2] | filter[5]) { + if (filter[2] || filter[5]) { return 4; } return 2; -#else - return 4; -#endif } #ifdef __cplusplus
diff --git a/av1/common/mvref_common.c b/av1/common/mvref_common.c index fdb2c8d..3f86c1d 100644 --- a/av1/common/mvref_common.c +++ b/av1/common/mvref_common.c
@@ -844,6 +844,7 @@ cm->cur_frame->order_hint = cm->current_frame.order_hint; cm->cur_frame->display_order_hint = cm->current_frame.display_order_hint; cm->cur_frame->pyramid_level = cm->current_frame.pyramid_level; + cm->cur_frame->base_qindex = cm->quant_params.base_qindex; cm->cur_frame->filter_level[0] = -1; cm->cur_frame->filter_level[1] = -1; MV_REFERENCE_FRAME ref_frame;
diff --git a/av1/common/quant_common.h b/av1/common/quant_common.h index 965a746..6dc82c0 100644 --- a/av1/common/quant_common.h +++ b/av1/common/quant_common.h
@@ -116,17 +116,17 @@ qm_level = 10; } else if (qindex <= 60) { qm_level = 9; - } else if (qindex <= 100) { + } else if (qindex <= 90) { qm_level = 8; } else if (qindex <= 120) { qm_level = 7; - } else if (qindex <= 140) { + } else if (qindex <= 130) { qm_level = 6; - } else if (qindex <= 160) { + } else if (qindex <= 140) { qm_level = 5; - } else if (qindex <= 200) { + } else if (qindex <= 160) { qm_level = 4; - } else if (qindex <= 220) { + } else if (qindex <= 200) { qm_level = 3; } else { qm_level = 2;
diff --git a/av1/common/riscv/compound_convolve_rvv.c b/av1/common/riscv/compound_convolve_rvv.c new file mode 100644 index 0000000..c1eb587 --- /dev/null +++ b/av1/common/riscv/compound_convolve_rvv.c
@@ -0,0 +1,2964 @@ +/* + * Copyright (c) 2026, Alliance for Open Media. All rights reserved. + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include <assert.h> +#include <riscv_vector.h> + +#include "av1/common/riscv/compound_convolve_rvv.h" +#include "config/aom_config.h" +#include "config/av1_rtcd.h" + +static uint16_t trans_data[64] = { + 0, 8, 16, 24, 32, 40, 48, 56, 1, 9, 17, 25, 33, 41, 49, 57, + 2, 10, 18, 26, 34, 42, 50, 58, 3, 11, 19, 27, 35, 43, 51, 59, + 4, 12, 20, 28, 36, 44, 52, 60, 5, 13, 21, 29, 37, 45, 53, 61, + 6, 14, 22, 30, 38, 46, 54, 62, 7, 15, 23, 31, 39, 47, 55, 63, +}; + +static inline void transpose_elems_inplace_s16_8x8_rvv( + vint16m1_t *d0, vint16m1_t *d1, vint16m1_t *d2, vint16m1_t *d3, + vint16m1_t *d4, vint16m1_t *d5, vint16m1_t *d6, vint16m1_t *d7, + const vuint16m8_t trans_index) { + vint16m8_t matrix_8x8 = __riscv_vundefined_i16m8(); + matrix_8x8 = __riscv_vset_v_i16m1_i16m8(matrix_8x8, 0, *d0); + matrix_8x8 = __riscv_vset_v_i16m1_i16m8(matrix_8x8, 1, *d1); + matrix_8x8 = __riscv_vset_v_i16m1_i16m8(matrix_8x8, 2, *d2); + matrix_8x8 = __riscv_vset_v_i16m1_i16m8(matrix_8x8, 3, *d3); + matrix_8x8 = __riscv_vset_v_i16m1_i16m8(matrix_8x8, 4, *d4); + matrix_8x8 = __riscv_vset_v_i16m1_i16m8(matrix_8x8, 5, *d5); + matrix_8x8 = __riscv_vset_v_i16m1_i16m8(matrix_8x8, 6, *d6); + matrix_8x8 = __riscv_vset_v_i16m1_i16m8(matrix_8x8, 7, *d7); + + // transpose_elems_inplace_s16_8x8 + vint16m8_t trans_matrix = + __riscv_vrgather_vv_i16m8(matrix_8x8, trans_index, 64); + + *d0 = __riscv_vget_v_i16m8_i16m1(trans_matrix, 0); + *d1 = __riscv_vget_v_i16m8_i16m1(trans_matrix, 1); + *d2 = __riscv_vget_v_i16m8_i16m1(trans_matrix, 2); + *d3 = __riscv_vget_v_i16m8_i16m1(trans_matrix, 3); + *d4 = __riscv_vget_v_i16m8_i16m1(trans_matrix, 4); + *d5 = __riscv_vget_v_i16m8_i16m1(trans_matrix, 5); + *d6 = __riscv_vget_v_i16m8_i16m1(trans_matrix, 6); + *d7 = __riscv_vget_v_i16m8_i16m1(trans_matrix, 7); +} + +static inline vint16m1_t convolve4_4_2d_h_rvv( + const vint16m1_t s0, const vint16m1_t s1, const vint16m1_t s2, + const vint16m1_t s3, const int16_t *filter, const int16_t horiz_const, + size_t vl) { + vint16m1_t sum = __riscv_vmul_vx_i16m1(s0, filter[0], vl); + sum = __riscv_vmacc_vx_i16m1(sum, filter[1], s1, vl); + sum = __riscv_vmacc_vx_i16m1(sum, filter[2], s2, vl); + sum = __riscv_vmacc_vx_i16m1(sum, filter[3], s3, vl); + sum = __riscv_vadd_vx_i16m1(sum, horiz_const, vl); + + // We halved the convolution filter values so -1 from the right shift. + return __riscv_vsra_vx_i16m1(sum, ROUND0_BITS - 1, vl); +} + +static inline vint16m1_t convolve8_8_2d_h_rvv( + const vint16m1_t s0, const vint16m1_t s1, const vint16m1_t s2, + const vint16m1_t s3, const vint16m1_t s4, const vint16m1_t s5, + const vint16m1_t s6, const vint16m1_t s7, const int16_t *filter, + const int16_t horiz_const, size_t vl) { + vint16m1_t sum = __riscv_vmul_vx_i16m1(s0, filter[0], vl); + sum = __riscv_vmacc_vx_i16m1(sum, filter[1], s1, vl); + sum = __riscv_vmacc_vx_i16m1(sum, filter[2], s2, vl); + sum = __riscv_vmacc_vx_i16m1(sum, filter[3], s3, vl); + sum = __riscv_vmacc_vx_i16m1(sum, filter[4], s4, vl); + sum = __riscv_vmacc_vx_i16m1(sum, filter[5], s5, vl); + sum = __riscv_vmacc_vx_i16m1(sum, filter[6], s6, vl); + sum = __riscv_vmacc_vx_i16m1(sum, filter[7], s7, vl); + sum = __riscv_vadd_vx_i16m1(sum, horiz_const, vl); + + // We halved the convolution filter values so -1 from the right shift. + return __riscv_vsra_vx_i16m1(sum, ROUND0_BITS - 1, vl); +} + +static inline void dist_wtd_convolve_2d_horiz_rvv( + const uint8_t *src, int src_stride, int16_t *im_block, const int im_stride, + const int16_t *x_filter_ptr, const int im_h, int w) { + const int bd = 8; + // A shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-rounding + // shifts - which are generally faster than rounding shifts on modern CPUs. + // (The extra -1 is needed because we halved the filter values.) + const int16_t horiz_const = + ((1 << (bd + FILTER_BITS - 2)) + (1 << ((ROUND0_BITS - 1) - 1))); + + const uint8_t *src_ptr = src; + int16_t *dst_ptr = im_block; + int dst_stride = im_stride; + int height = im_h; + size_t vl = __riscv_vsetvl_e16m1(w); + + if (w == 4) { + // 4-tap filters are used for blocks having width <= 4. + // Filter values are even, so halve to reduce intermediate precision reqs. + const int16_t *x_filter = x_filter_ptr + 2; + int16_t filter[4]; + for (int i = 0; i < 4; i++) filter[i] = *x_filter++ >> 1; + + src_ptr += 2; + + do { + // load + vuint8mf2_t t0 = __riscv_vle8_v_u8mf2(src_ptr + 0 * src_stride, + 8); // a0 a1 a2 a3 a4 a5 a6 a7 + vuint8mf2_t t1 = __riscv_vle8_v_u8mf2(src_ptr + 1 * src_stride, + 8); // b0 b1 b2 b3 b4 b5 b6 b7 + + __builtin_prefetch(dst_ptr + 0 * dst_stride); + __builtin_prefetch(dst_ptr + 1 * dst_stride); + + // widen to 16-bit + vint16m1_t s00 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t0, 8)); + vint16m1_t s01 = + __riscv_vslide1down_vx_i16m1(s00, 0, 8); // a1 a2 a3 a4 a5 a6 a7 0 + vint16m1_t s02 = + __riscv_vslide1down_vx_i16m1(s01, 0, 8); // a2 a3 a4 a5 a6 a7 0 0 + vint16m1_t s03 = + __riscv_vslide1down_vx_i16m1(s02, 0, 8); // a3 a4 a5 a6 a7 0 0 0 + + vint16m1_t s10 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t1, 8)); + vint16m1_t s11 = + __riscv_vslide1down_vx_i16m1(s10, 0, 8); // b1 b2 b3 b4 b5 b6 b7 0 + vint16m1_t s12 = + __riscv_vslide1down_vx_i16m1(s11, 0, 8); // b2 b3 b4 b5 b6 b7 0 0 + vint16m1_t s13 = + __riscv_vslide1down_vx_i16m1(s12, 0, 8); // b3 b4 b5 b6 b7 0 0 0 + + // perform convolution + vint16m1_t d0 = + convolve4_4_2d_h_rvv(s00, s01, s02, s03, filter, horiz_const, vl); + vint16m1_t d1 = + convolve4_4_2d_h_rvv(s10, s11, s12, s13, filter, horiz_const, vl); + + // store result + __riscv_vse16_v_i16m1(dst_ptr + 0 * dst_stride, d0, vl); + __riscv_vse16_v_i16m1(dst_ptr + 1 * dst_stride, d1, vl); + src_ptr += 2 * src_stride; + dst_ptr += 2 * dst_stride; + height -= 2; + } while (height >= 2); + + if (height > 0) { + vuint8mf2_t t0 = __riscv_vle8_v_u8mf2(src_ptr + 0 * src_stride, + 8); // a0 a1 a2 a3 a4 a5 a6 a7 + + // widen to 16-bit + vint16m1_t s00 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t0, 8)); + vint16m1_t s01 = + __riscv_vslide1down_vx_i16m1(s00, 0, 8); // a1 a2 a3 a4 a5 a6 a7 0 + vint16m1_t s02 = + __riscv_vslide1down_vx_i16m1(s01, 0, 8); // a2 a3 a4 a5 a6 a7 0 0 + vint16m1_t s03 = + __riscv_vslide1down_vx_i16m1(s02, 0, 8); // a3 a4 a5 a6 a7 0 0 0 + + // perform convolution + vint16m1_t d0 = + convolve4_4_2d_h_rvv(s00, s01, s02, s03, filter, horiz_const, vl); + + // store result + __riscv_vse16_v_i16m1(dst_ptr + 0 * dst_stride, d0, vl); + } + } else { + // Filter values are even, so halve to reduce intermediate precision reqs. + int16_t filter[8]; + for (int i = 0; i < 8; i++) filter[i] = *x_filter_ptr++ >> 1; + + // for VLEN = 128 case + if ((vl == 8) && (w > 8) && (height >= 8)) { + vuint16m8_t trans_index = __riscv_vle16_v_u16m8(trans_data, 64); + + while (height >= 8) { + const uint8_t *s = src_ptr; + int16_t *d = dst_ptr; + int width = w; + + // Load + vuint8mf2_t t0, t1, t2, t3, t4, t5, t6, t7; + load_stride_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, + &t7, vl); + + vint16m1_t s0 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t0, vl)); + vint16m1_t s1 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t1, vl)); + vint16m1_t s2 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t2, vl)); + vint16m1_t s3 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t3, vl)); + vint16m1_t s4 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t4, vl)); + vint16m1_t s5 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t5, vl)); + vint16m1_t s6 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t6, vl)); + + __builtin_prefetch(d + 0 * dst_stride); + __builtin_prefetch(d + 1 * dst_stride); + __builtin_prefetch(d + 2 * dst_stride); + __builtin_prefetch(d + 3 * dst_stride); + __builtin_prefetch(d + 4 * dst_stride); + __builtin_prefetch(d + 5 * dst_stride); + __builtin_prefetch(d + 6 * dst_stride); + __builtin_prefetch(d + 7 * dst_stride); + + s += 7; + + do { + load_stride_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, + &t7, vl); + + vint16m1_t s7 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vzext_vf2_u16m1(t0, vl)); + vint16m1_t s8 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vzext_vf2_u16m1(t1, vl)); + vint16m1_t s9 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vzext_vf2_u16m1(t2, vl)); + vint16m1_t s10 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vzext_vf2_u16m1(t3, vl)); + vint16m1_t s11 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vzext_vf2_u16m1(t4, vl)); + vint16m1_t s12 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vzext_vf2_u16m1(t5, vl)); + vint16m1_t s13 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vzext_vf2_u16m1(t6, vl)); + vint16m1_t s14 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vzext_vf2_u16m1(t7, vl)); + + // Perform convolution + vint16m1_t d0 = convolve8_8_2d_h_rvv(s0, s1, s2, s3, s4, s5, s6, s7, + filter, horiz_const, vl); + vint16m1_t d1 = convolve8_8_2d_h_rvv(s1, s2, s3, s4, s5, s6, s7, s8, + filter, horiz_const, vl); + vint16m1_t d2 = convolve8_8_2d_h_rvv(s2, s3, s4, s5, s6, s7, s8, s9, + filter, horiz_const, vl); + vint16m1_t d3 = convolve8_8_2d_h_rvv(s3, s4, s5, s6, s7, s8, s9, s10, + filter, horiz_const, vl); + vint16m1_t d4 = convolve8_8_2d_h_rvv(s4, s5, s6, s7, s8, s9, s10, s11, + filter, horiz_const, vl); + vint16m1_t d5 = convolve8_8_2d_h_rvv(s5, s6, s7, s8, s9, s10, s11, + s12, filter, horiz_const, vl); + vint16m1_t d6 = convolve8_8_2d_h_rvv(s6, s7, s8, s9, s10, s11, s12, + s13, filter, horiz_const, vl); + vint16m1_t d7 = convolve8_8_2d_h_rvv(s7, s8, s9, s10, s11, s12, s13, + s14, filter, horiz_const, vl); + + // transpose + transpose_elems_inplace_s16_8x8_rvv(&d0, &d1, &d2, &d3, &d4, &d5, &d6, + &d7, trans_index); + + // Store result + store_s16_8x8(d, dst_stride, d0, d1, d2, d3, d4, d5, d6, d7, vl); + + // Update sliding window + s0 = __riscv_vmv_v_v_i16m1(s8, vl); + s1 = __riscv_vmv_v_v_i16m1(s9, vl); + s2 = __riscv_vmv_v_v_i16m1(s10, vl); + s3 = __riscv_vmv_v_v_i16m1(s11, vl); + s4 = __riscv_vmv_v_v_i16m1(s12, vl); + s5 = __riscv_vmv_v_v_i16m1(s13, vl); + s6 = __riscv_vmv_v_v_i16m1(s14, vl); + s += 8; + d += 8; + width -= 8; + } while (width != 0); + src_ptr += 8 * src_stride; + dst_ptr += 8 * dst_stride; + height -= 8; + } + } + + while (height > 0) { + const uint8_t *s = src_ptr; + int16_t *d = dst_ptr; + int width = w; + + __builtin_prefetch(d); + + do { + // Load + vuint8mf2_t t0, t1, t2, t3, t4, t5, t6, t7; + load_u8_8x8(s, 1, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7, vl); + + // Convert to 16-bit integers + vint16m1_t s0 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t0, vl)); + vint16m1_t s1 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t1, vl)); + vint16m1_t s2 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t2, vl)); + vint16m1_t s3 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t3, vl)); + vint16m1_t s4 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t4, vl)); + vint16m1_t s5 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t5, vl)); + vint16m1_t s6 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t6, vl)); + vint16m1_t s7 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t7, vl)); + + // Perform convolution + vint16m1_t d0 = convolve8_8_2d_h_rvv(s0, s1, s2, s3, s4, s5, s6, s7, + filter, horiz_const, vl); + + // Store result + __riscv_vse16_v_i16m1(d, d0, vl); + + s += vl; + d += vl; + width -= vl; + } while (width > 0); + src_ptr += src_stride; + dst_ptr += dst_stride; + height--; + } + } +} + +void av1_dist_wtd_convolve_2d_rvv(const uint8_t *src, int src_stride, + uint8_t *dst8, int dst8_stride, int w, int h, + const InterpFilterParams *filter_params_x, + const InterpFilterParams *filter_params_y, + const int subpel_x_qn, const int subpel_y_qn, + ConvolveParams *conv_params) { + assert(w % 4 == 0); + assert(h % 4 == 0); + + DECLARE_ALIGNED(16, int16_t, + im_block[(MAX_SB_SIZE + SUBPEL_TAPS - 1) * MAX_SB_SIZE]); + + const int y_filter_taps = get_filter_tap(filter_params_y, subpel_y_qn); + const int clamped_y_taps = y_filter_taps < 6 ? 6 : y_filter_taps; + + const int im_h = h + clamped_y_taps - 1; + const int im_stride = MAX_SB_SIZE; + const int vert_offset = clamped_y_taps / 2 - 1; + const int horiz_offset = filter_params_x->taps / 2 - 1; + const uint8_t *src_ptr = src - vert_offset * src_stride - horiz_offset; + const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel( + filter_params_x, subpel_x_qn & SUBPEL_MASK); + const int16_t *y_filter_ptr = av1_get_interp_filter_subpel_kernel( + filter_params_y, subpel_y_qn & SUBPEL_MASK); + + dist_wtd_convolve_2d_horiz_rvv(src_ptr, src_stride, im_block, im_stride, + x_filter_ptr, im_h, w); + + if (clamped_y_taps == 6) { + if (conv_params->do_average) { + if (UNLIKELY(conv_params->use_dist_wtd_comp_avg)) { + dist_wtd_convolve_2d_vert_6tap_dist_wtd_avg_rvv( + im_block, im_stride, dst8, dst8_stride, conv_params, y_filter_ptr, + h, w); + } else { + dist_wtd_convolve_2d_vert_6tap_avg_rvv(im_block, im_stride, dst8, + dst8_stride, conv_params, + y_filter_ptr, h, w); + } + } else { + dist_wtd_convolve_2d_vert_6tap_rvv(im_block, im_stride, conv_params, + y_filter_ptr, h, w); + } + } else { + if (conv_params->do_average) { + if (UNLIKELY(conv_params->use_dist_wtd_comp_avg)) { + dist_wtd_convolve_2d_vert_8tap_dist_wtd_avg_rvv( + im_block, im_stride, dst8, dst8_stride, conv_params, y_filter_ptr, + h, w); + } else { + dist_wtd_convolve_2d_vert_8tap_avg_rvv(im_block, im_stride, dst8, + dst8_stride, conv_params, + y_filter_ptr, h, w); + } + } else { + dist_wtd_convolve_2d_vert_8tap_rvv(im_block, im_stride, conv_params, + y_filter_ptr, h, w); + } + } +} + +static inline void dist_wtd_convolve_2d_copy_dist_wtd_avg_rvv( + const uint8_t *src, int src_stride, uint8_t *dst8, int dst8_stride, int w, + int h, ConvolveParams *conv_params) { + assert(w % 4 == 0); + assert(h % 4 == 0); + + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const uint16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + const uint8_t shift_by_bits = 1 << (FILTER_BITS - ROUND0_BITS); + + const uint16_t fwd_offset = conv_params->fwd_offset; + const uint16_t bck_offset = conv_params->bck_offset; + + CONV_BUF_TYPE *dst = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + int height = h; + size_t vl = __riscv_vsetvl_e16m1(w); + + if (w == 4) { + vuint16mf2_t round_offset_u16 = __riscv_vmv_v_x_u16mf2(round_offset, vl); + do { + // load 4 block of uint8_t + vuint8mf4_t s0 = __riscv_vle8_v_u8mf4(src + 0 * src_stride, vl); + vuint8mf4_t s1 = __riscv_vle8_v_u8mf4(src + 1 * src_stride, vl); + vuint8mf4_t s2 = __riscv_vle8_v_u8mf4(src + 2 * src_stride, vl); + vuint8mf4_t s3 = __riscv_vle8_v_u8mf4(src + 3 * src_stride, vl); + + // apply shift and add round offset + vuint16mf2_t d0 = + __riscv_vwmaccu_vx_u16mf2(round_offset_u16, shift_by_bits, s0, vl); + vuint16mf2_t d1 = + __riscv_vwmaccu_vx_u16mf2(round_offset_u16, shift_by_bits, s1, vl); + vuint16mf2_t d2 = + __riscv_vwmaccu_vx_u16mf2(round_offset_u16, shift_by_bits, s2, vl); + vuint16mf2_t d3 = + __riscv_vwmaccu_vx_u16mf2(round_offset_u16, shift_by_bits, s3, vl); + + // load existing dst values + vuint16mf2_t dd0, dd1, dd2, dd3; + load_u16_4x4(dst, dst_stride, &dd0, &dd1, &dd2, &dd3, vl); + + // compute distance-weighted average + vuint8mf4_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_dist_wtd_avg_4x4_rvv(dd0, dd1, dd2, dd3, d0, d1, d2, d3, + fwd_offset, bck_offset, round_offset, &d0_u8, + &d1_u8, &d2_u8, &d3_u8, vl); + + // store results + __riscv_vse8_v_u8mf4(dst8 + 0 * dst8_stride, d0_u8, vl); + __riscv_vse8_v_u8mf4(dst8 + 1 * dst8_stride, d1_u8, vl); + __riscv_vse8_v_u8mf4(dst8 + 2 * dst8_stride, d2_u8, vl); + __riscv_vse8_v_u8mf4(dst8 + 3 * dst8_stride, d3_u8, vl); + + // update pointers + src += 4 * src_stride; + dst += 4 * dst_stride; + dst8 += 4 * dst8_stride; + height -= 4; + } while (height != 0); + } else { + vuint16m1_t round_offset_u16 = __riscv_vmv_v_x_u16m1(round_offset, vl); + do { + const uint8_t *s = src; + CONV_BUF_TYPE *d = dst; + uint8_t *d_u8 = dst8; + int width = w; + + do { + // load 4 block of uint8_t + vuint8mf2_t s0, s1, s2, s3; + load_u8_8x4(s, src_stride, &s0, &s1, &s2, &s3, vl); + + // apply shift and add round offset + vuint16m1_t d0 = + __riscv_vwmaccu_vx_u16m1(round_offset_u16, shift_by_bits, s0, vl); + vuint16m1_t d1 = + __riscv_vwmaccu_vx_u16m1(round_offset_u16, shift_by_bits, s1, vl); + vuint16m1_t d2 = + __riscv_vwmaccu_vx_u16m1(round_offset_u16, shift_by_bits, s2, vl); + vuint16m1_t d3 = + __riscv_vwmaccu_vx_u16m1(round_offset_u16, shift_by_bits, s3, vl); + + // load existing dst values + vuint16m1_t dd0, dd1, dd2, dd3; + load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3, vl); + + // compute distance-weighted average + vuint8mf2_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_dist_wtd_avg_8x4_rvv(dd0, dd1, dd2, dd3, d0, d1, d2, d3, + fwd_offset, bck_offset, round_offset, + &d0_u8, &d1_u8, &d2_u8, &d3_u8, vl); + + // store results + store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8, vl); + + // update pointers + s += vl; + d += vl; + d_u8 += vl; + width -= vl; + } while (width > 0); + src += 4 * src_stride; + dst += 4 * dst_stride; + dst8 += 4 * dst8_stride; + height -= 4; + } while (height != 0); + } +} + +static inline void dist_wtd_convolve_2d_copy_avg_rvv( + const uint8_t *src, int src_stride, uint8_t *dst8, int dst8_stride, int w, + int h, ConvolveParams *conv_params) { + assert(w % 4 == 0); + assert(h % 4 == 0); + + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const uint16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + const uint8_t shift_by_bits = 1 << (FILTER_BITS - ROUND0_BITS); + + CONV_BUF_TYPE *dst = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + int height = h; + size_t vl = __riscv_vsetvl_e16m1(w); + + if (w == 4) { + vuint16mf2_t round_offset_u16 = __riscv_vmv_v_x_u16mf2(round_offset, vl); + do { + // load 4 block of uint8_t + vuint8mf4_t s0 = __riscv_vle8_v_u8mf4(src + 0 * src_stride, vl); + vuint8mf4_t s1 = __riscv_vle8_v_u8mf4(src + 1 * src_stride, vl); + vuint8mf4_t s2 = __riscv_vle8_v_u8mf4(src + 2 * src_stride, vl); + vuint8mf4_t s3 = __riscv_vle8_v_u8mf4(src + 3 * src_stride, vl); + + // apply shift and add round offset + vuint16mf2_t d0 = + __riscv_vwmaccu_vx_u16mf2(round_offset_u16, shift_by_bits, s0, vl); + vuint16mf2_t d1 = + __riscv_vwmaccu_vx_u16mf2(round_offset_u16, shift_by_bits, s1, vl); + vuint16mf2_t d2 = + __riscv_vwmaccu_vx_u16mf2(round_offset_u16, shift_by_bits, s2, vl); + vuint16mf2_t d3 = + __riscv_vwmaccu_vx_u16mf2(round_offset_u16, shift_by_bits, s3, vl); + + // load existing dst values + vuint16mf2_t dd0, dd1, dd2, dd3; + load_u16_4x4(dst, dst_stride, &dd0, &dd1, &dd2, &dd3, vl); + + // compute basic average + vuint8mf4_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_basic_avg_4x4_rvv(dd0, dd1, dd2, dd3, d0, d1, d2, d3, + round_offset, &d0_u8, &d1_u8, &d2_u8, &d3_u8, + vl); + + // store result + __riscv_vse8_v_u8mf4(dst8 + 0 * dst8_stride, d0_u8, vl); + __riscv_vse8_v_u8mf4(dst8 + 1 * dst8_stride, d1_u8, vl); + __riscv_vse8_v_u8mf4(dst8 + 2 * dst8_stride, d2_u8, vl); + __riscv_vse8_v_u8mf4(dst8 + 3 * dst8_stride, d3_u8, vl); + + // update pointers + src += 4 * src_stride; + dst += 4 * dst_stride; + dst8 += 4 * dst8_stride; + height -= 4; + } while (height != 0); + } else { + vuint16m1_t round_offset_u16 = __riscv_vmv_v_x_u16m1(round_offset, vl); + do { + const uint8_t *s = src; + CONV_BUF_TYPE *d = dst; + uint8_t *d_u8 = dst8; + int width = w; + + do { + // load 4 block of uint8_t + vuint8mf2_t s0, s1, s2, s3; + load_u8_8x4(s, src_stride, &s0, &s1, &s2, &s3, vl); + + // apply shift and add round offset + vuint16m1_t d0 = + __riscv_vwmaccu_vx_u16m1(round_offset_u16, shift_by_bits, s0, vl); + vuint16m1_t d1 = + __riscv_vwmaccu_vx_u16m1(round_offset_u16, shift_by_bits, s1, vl); + vuint16m1_t d2 = + __riscv_vwmaccu_vx_u16m1(round_offset_u16, shift_by_bits, s2, vl); + vuint16m1_t d3 = + __riscv_vwmaccu_vx_u16m1(round_offset_u16, shift_by_bits, s3, vl); + + // load existing dst values + vuint16m1_t dd0, dd1, dd2, dd3; + load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3, vl); + + // compute basic average + vuint8mf2_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_basic_avg_8x4_rvv(dd0, dd1, dd2, dd3, d0, d1, d2, d3, + round_offset, &d0_u8, &d1_u8, &d2_u8, &d3_u8, + vl); + + // store results + store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8, vl); + + // update pointers + s += vl; + d += vl; + d_u8 += vl; + width -= vl; + } while (width != 0); + src += 4 * src_stride; + dst += 4 * dst_stride; + dst8 += 4 * dst8_stride; + height -= 4; + } while (height != 0); + } +} + +static inline void dist_wtd_convolve_2d_copy_rvv(const uint8_t *src, + int src_stride, int w, int h, + ConvolveParams *conv_params) { + assert(w % 4 == 0); + assert(h % 4 == 0); + + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const uint16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + const uint8_t shift_by_bits = 1 << (FILTER_BITS - ROUND0_BITS); + + CONV_BUF_TYPE *dst = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + int height = h; + size_t vl = __riscv_vsetvl_e16m1(w); + + if (w == 4) { + vuint16mf2_t round_offset_u16 = __riscv_vmv_v_x_u16mf2(round_offset, vl); + do { + // load 4 block of uint8_t + vuint8mf4_t s0 = __riscv_vle8_v_u8mf4(src + 0 * src_stride, vl); + vuint8mf4_t s1 = __riscv_vle8_v_u8mf4(src + 1 * src_stride, vl); + vuint8mf4_t s2 = __riscv_vle8_v_u8mf4(src + 2 * src_stride, vl); + vuint8mf4_t s3 = __riscv_vle8_v_u8mf4(src + 3 * src_stride, vl); + + // apply shift and add round offset + vuint16mf2_t d0 = + __riscv_vwmaccu_vx_u16mf2(round_offset_u16, shift_by_bits, s0, vl); + vuint16mf2_t d1 = + __riscv_vwmaccu_vx_u16mf2(round_offset_u16, shift_by_bits, s1, vl); + vuint16mf2_t d2 = + __riscv_vwmaccu_vx_u16mf2(round_offset_u16, shift_by_bits, s2, vl); + vuint16mf2_t d3 = + __riscv_vwmaccu_vx_u16mf2(round_offset_u16, shift_by_bits, s3, vl); + + // store result + store_u16_4x4(dst, dst_stride, d0, d1, d2, d3, vl); + + // update pointers + src += 4 * src_stride; + dst += 4 * dst_stride; + height -= 4; + } while (height != 0); + } else { + vuint16m1_t round_offset_u16 = __riscv_vmv_v_x_u16m1(round_offset, vl); + do { + const uint8_t *s = src; + CONV_BUF_TYPE *d = dst; + int width = w; + + do { + // load 4 block of uint8_t + vuint8mf2_t s0, s1, s2, s3; + load_u8_8x4(s, src_stride, &s0, &s1, &s2, &s3, vl); + + // apply shift and add round offset + vuint16m1_t d0 = + __riscv_vwmaccu_vx_u16m1(round_offset_u16, shift_by_bits, s0, vl); + vuint16m1_t d1 = + __riscv_vwmaccu_vx_u16m1(round_offset_u16, shift_by_bits, s1, vl); + vuint16m1_t d2 = + __riscv_vwmaccu_vx_u16m1(round_offset_u16, shift_by_bits, s2, vl); + vuint16m1_t d3 = + __riscv_vwmaccu_vx_u16m1(round_offset_u16, shift_by_bits, s3, vl); + + // store results + store_u16_8x4(d, dst_stride, d0, d1, d2, d3, vl); + + // update pointers + s += vl; + d += vl; + width -= vl; + } while (width != 0); + src += 4 * src_stride; + dst += 4 * dst_stride; + height -= 4; + } while (height != 0); + } +} + +void av1_dist_wtd_convolve_2d_copy_rvv(const uint8_t *src, int src_stride, + uint8_t *dst8, int dst8_stride, int w, + int h, ConvolveParams *conv_params) { + if (conv_params->do_average) { + if (UNLIKELY(conv_params->use_dist_wtd_comp_avg)) { + dist_wtd_convolve_2d_copy_dist_wtd_avg_rvv( + src, src_stride, dst8, dst8_stride, w, h, conv_params); + } else { + dist_wtd_convolve_2d_copy_avg_rvv(src, src_stride, dst8, dst8_stride, w, + h, conv_params); + } + } else { + dist_wtd_convolve_2d_copy_rvv(src, src_stride, w, h, conv_params); + } +} + +static inline vuint16m1_t convolve4_4_x_rvv( + const vint16m1_t s0, const vint16m1_t s1, const vint16m1_t s2, + const vint16m1_t s3, const int16_t *filter, const int16_t round_offset, + size_t vl) { + vint16m1_t sum = __riscv_vmul_vx_i16m1(s0, filter[0], vl); + sum = __riscv_vmacc_vx_i16m1(sum, filter[1], s1, vl); + sum = __riscv_vmacc_vx_i16m1(sum, filter[2], s2, vl); + sum = __riscv_vmacc_vx_i16m1(sum, filter[3], s3, vl); + + // Right shift with rounding: vrsra_n_s16(round_offset, sum, ROUND0_BITS - 1) + // vrsra_n_s16(a, b, n) = a + ((b + (1 << (n-1))) >> n) + // We halved the convolution filter values so -1 from the right shift. + vint16m1_t res = __riscv_vadd_vx_i16m1(sum, 1 << ((ROUND0_BITS - 1) - 1), vl); + res = __riscv_vsra_vx_i16m1(res, ROUND0_BITS - 1, vl); + res = __riscv_vadd_vx_i16m1(res, round_offset, vl); + + // Reinterpret as uint16 + return __riscv_vreinterpret_v_i16m1_u16m1(res); +} + +static inline vuint16m1_t convolve8_8_x_rvv( + const vint16m1_t s0, const vint16m1_t s1, const vint16m1_t s2, + const vint16m1_t s3, const vint16m1_t s4, const vint16m1_t s5, + const vint16m1_t s6, const vint16m1_t s7, const int16_t *filter, + const int16_t round_offset, size_t vl) { + vint16m1_t sum = __riscv_vmul_vx_i16m1(s0, filter[0], vl); + sum = __riscv_vmacc_vx_i16m1(sum, filter[1], s1, vl); + sum = __riscv_vmacc_vx_i16m1(sum, filter[2], s2, vl); + sum = __riscv_vmacc_vx_i16m1(sum, filter[3], s3, vl); + sum = __riscv_vmacc_vx_i16m1(sum, filter[4], s4, vl); + sum = __riscv_vmacc_vx_i16m1(sum, filter[5], s5, vl); + sum = __riscv_vmacc_vx_i16m1(sum, filter[6], s6, vl); + sum = __riscv_vmacc_vx_i16m1(sum, filter[7], s7, vl); + + vint16m1_t res = __riscv_vadd_vx_i16m1(sum, 1 << ((ROUND0_BITS - 1) - 1), vl); + res = __riscv_vsra_vx_i16m1(res, ROUND0_BITS - 1, vl); + res = __riscv_vadd_vx_i16m1(res, round_offset, vl); + + // Reinterpret as uint16 + return __riscv_vreinterpret_v_i16m1_u16m1(res); +} + +static inline void transpose_elems_inplace_u16_8x8_rvv( + vuint16m1_t *d0, vuint16m1_t *d1, vuint16m1_t *d2, vuint16m1_t *d3, + vuint16m1_t *d4, vuint16m1_t *d5, vuint16m1_t *d6, vuint16m1_t *d7, + const vuint16m8_t trans_index) { + vuint16m8_t matrix_8x8 = __riscv_vundefined_u16m8(); + matrix_8x8 = __riscv_vset_v_u16m1_u16m8(matrix_8x8, 0, *d0); + matrix_8x8 = __riscv_vset_v_u16m1_u16m8(matrix_8x8, 1, *d1); + matrix_8x8 = __riscv_vset_v_u16m1_u16m8(matrix_8x8, 2, *d2); + matrix_8x8 = __riscv_vset_v_u16m1_u16m8(matrix_8x8, 3, *d3); + matrix_8x8 = __riscv_vset_v_u16m1_u16m8(matrix_8x8, 4, *d4); + matrix_8x8 = __riscv_vset_v_u16m1_u16m8(matrix_8x8, 5, *d5); + matrix_8x8 = __riscv_vset_v_u16m1_u16m8(matrix_8x8, 6, *d6); + matrix_8x8 = __riscv_vset_v_u16m1_u16m8(matrix_8x8, 7, *d7); + + // transpose_elems_inplace_u16_8x8 + vuint16m8_t trans_matrix = + __riscv_vrgather_vv_u16m8(matrix_8x8, trans_index, 64); + + *d0 = __riscv_vget_v_u16m8_u16m1(trans_matrix, 0); + *d1 = __riscv_vget_v_u16m8_u16m1(trans_matrix, 1); + *d2 = __riscv_vget_v_u16m8_u16m1(trans_matrix, 2); + *d3 = __riscv_vget_v_u16m8_u16m1(trans_matrix, 3); + *d4 = __riscv_vget_v_u16m8_u16m1(trans_matrix, 4); + *d5 = __riscv_vget_v_u16m8_u16m1(trans_matrix, 5); + *d6 = __riscv_vget_v_u16m8_u16m1(trans_matrix, 6); + *d7 = __riscv_vget_v_u16m8_u16m1(trans_matrix, 7); +} + +static inline void dist_wtd_convolve_x_dist_wtd_avg_rvv( + const uint8_t *src, int src_stride, uint8_t *dst8, int dst8_stride, int w, + int h, const InterpFilterParams *filter_params_x, const int subpel_x_qn, + ConvolveParams *conv_params) { + assert(w % 4 == 0); + assert(h % 4 == 0); + + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + const uint16_t fwd_offset = conv_params->fwd_offset; + const uint16_t bck_offset = conv_params->bck_offset; + + // Horizontal filter. + const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel( + filter_params_x, subpel_x_qn & SUBPEL_MASK); + + const int horiz_offset = filter_params_x->taps / 2 - 1; + const uint8_t *src_ptr = src - horiz_offset; + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + uint8_t *dst8_ptr = dst8; + int dst_stride = conv_params->dst_stride; + int height = h; + size_t vl = __riscv_vsetvl_e16m1(w); + + if (w == 4) { + // 4-tap filters are used for blocks having width <= 4. + // Filter values are even, so halve to reduce intermediate precision reqs. + const int16_t *x_filter = x_filter_ptr + 2; + int16_t filter[4]; + for (int i = 0; i < 4; i++) filter[i] = *x_filter++ >> 1; + + src_ptr += 2; + + do { + // Load + vuint8mf2_t t0 = __riscv_vle8_v_u8mf2(src_ptr + 0 * src_stride, + 8); // a0 a1 a2 a3 a4 a5 a6 a7 + vuint8mf2_t t1 = __riscv_vle8_v_u8mf2(src_ptr + 1 * src_stride, + 8); // b0 b1 b2 b3 b4 b5 b6 b7 + + __builtin_prefetch(dst_ptr); + __builtin_prefetch(dst8_ptr); + + // Widen to 16-bit + vint16m1_t s00 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t0, 8)); + vint16m1_t s01 = + __riscv_vslide1down_vx_i16m1(s00, 0, 8); // a1 a2 a3 a4 a5 a6 a7 0 + vint16m1_t s02 = + __riscv_vslide1down_vx_i16m1(s01, 0, 8); // a2 a3 a4 a5 a6 a7 0 0 + vint16m1_t s03 = + __riscv_vslide1down_vx_i16m1(s02, 0, 8); // a3 a4 a5 a6 a7 0 0 0 + + vint16m1_t s10 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t1, 8)); + vint16m1_t s11 = + __riscv_vslide1down_vx_i16m1(s10, 0, 8); // b1 b2 b3 b4 b5 b6 b7 0 + vint16m1_t s12 = + __riscv_vslide1down_vx_i16m1(s11, 0, 8); // b2 b3 b4 b5 b6 b7 0 0 + vint16m1_t s13 = + __riscv_vslide1down_vx_i16m1(s12, 0, 8); // b3 b4 b5 b6 b7 0 0 0 + + // Perform convolution + vuint8mf2_t d01, d11; + vuint16m1_t d0 = + convolve4_4_x_rvv(s00, s01, s02, s03, filter, round_offset, vl); + vuint16m1_t d1 = + convolve4_4_x_rvv(s10, s11, s12, s13, filter, round_offset, vl); + + // weighted average + vuint16m1_t dd0 = __riscv_vle16_v_u16m1(dst_ptr + 0 * dst_stride, vl); + vuint16m1_t dd1 = __riscv_vle16_v_u16m1(dst_ptr + 1 * dst_stride, vl); + compute_dist_wtd_avg_4x1_rvv(dd0, d0, fwd_offset, bck_offset, + round_offset, &d01, vl); + compute_dist_wtd_avg_4x1_rvv(dd1, d1, fwd_offset, bck_offset, + round_offset, &d11, vl); + + // Store result + __riscv_vse8_v_u8mf2(dst8_ptr + 0 * dst8_stride, d01, vl); + __riscv_vse8_v_u8mf2(dst8_ptr + 1 * dst8_stride, d11, vl); + + src_ptr += 2 * src_stride; + dst_ptr += 2 * dst_stride; + dst8_ptr += 2 * dst8_stride; + height -= 2; + } while (height >= 2); + } else { + // Filter values are even, so halve to reduce intermediate precision reqs. + int16_t filter[8]; + for (int i = 0; i < 8; i++) filter[i] = *x_filter_ptr++ >> 1; + + // for VLEN = 128 case + if ((w > 8) && (vl == 8) && (height >= 8)) { + vuint16m8_t trans_index = __riscv_vle16_v_u16m8(trans_data, 64); + + while (height >= 8) { + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + uint8_t *d_u8 = dst8_ptr; + int width = w; + + // Load + vuint8mf2_t t0, t1, t2, t3, t4, t5, t6, t7; + load_stride_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, + &t7, vl); + + vint16m1_t s0 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t0, vl)); + vint16m1_t s1 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t1, vl)); + vint16m1_t s2 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t2, vl)); + vint16m1_t s3 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t3, vl)); + vint16m1_t s4 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t4, vl)); + vint16m1_t s5 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t5, vl)); + vint16m1_t s6 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t6, vl)); + + __builtin_prefetch(d + 0 * dst_stride); + __builtin_prefetch(d + 1 * dst_stride); + __builtin_prefetch(d + 2 * dst_stride); + __builtin_prefetch(d + 3 * dst_stride); + __builtin_prefetch(d + 4 * dst_stride); + __builtin_prefetch(d + 5 * dst_stride); + __builtin_prefetch(d + 6 * dst_stride); + __builtin_prefetch(d + 7 * dst_stride); + + s += 7; + + do { + load_stride_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, + &t7, vl); + + vint16m1_t s7 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vzext_vf2_u16m1(t0, vl)); + vint16m1_t s8 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vzext_vf2_u16m1(t1, vl)); + vint16m1_t s9 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vzext_vf2_u16m1(t2, vl)); + vint16m1_t s10 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vzext_vf2_u16m1(t3, vl)); + vint16m1_t s11 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vzext_vf2_u16m1(t4, vl)); + vint16m1_t s12 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vzext_vf2_u16m1(t5, vl)); + vint16m1_t s13 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vzext_vf2_u16m1(t6, vl)); + vint16m1_t s14 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vzext_vf2_u16m1(t7, vl)); + + // Perform convolution + vuint16m1_t d0 = convolve8_8_x_rvv(s0, s1, s2, s3, s4, s5, s6, s7, + filter, round_offset, vl); + vuint16m1_t d1 = convolve8_8_x_rvv(s1, s2, s3, s4, s5, s6, s7, s8, + filter, round_offset, vl); + vuint16m1_t d2 = convolve8_8_x_rvv(s2, s3, s4, s5, s6, s7, s8, s9, + filter, round_offset, vl); + vuint16m1_t d3 = convolve8_8_x_rvv(s3, s4, s5, s6, s7, s8, s9, s10, + filter, round_offset, vl); + vuint16m1_t d4 = convolve8_8_x_rvv(s4, s5, s6, s7, s8, s9, s10, s11, + filter, round_offset, vl); + vuint16m1_t d5 = convolve8_8_x_rvv(s5, s6, s7, s8, s9, s10, s11, s12, + filter, round_offset, vl); + vuint16m1_t d6 = convolve8_8_x_rvv(s6, s7, s8, s9, s10, s11, s12, s13, + filter, round_offset, vl); + vuint16m1_t d7 = convolve8_8_x_rvv(s7, s8, s9, s10, s11, s12, s13, + s14, filter, round_offset, vl); + + // transpose + transpose_elems_inplace_u16_8x8_rvv(&d0, &d1, &d2, &d3, &d4, &d5, &d6, + &d7, trans_index); + + vuint16m1_t dd0, dd1, dd2, dd3; + load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3, vl); + + vuint8mf2_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_dist_wtd_avg_8x4_rvv(dd0, dd1, dd2, dd3, d0, d1, d2, d3, + fwd_offset, bck_offset, round_offset, + &d0_u8, &d1_u8, &d2_u8, &d3_u8, vl); + store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8, vl); + + vuint16m1_t dd4, dd5, dd6, dd7; + load_u16_8x4(d + 4 * dst_stride, dst_stride, &dd4, &dd5, &dd6, &dd7, + vl); + + vuint8mf2_t d4_u8, d5_u8, d6_u8, d7_u8; + compute_dist_wtd_avg_8x4_rvv(dd4, dd5, dd6, dd7, d4, d5, d6, d7, + fwd_offset, bck_offset, round_offset, + &d4_u8, &d5_u8, &d6_u8, &d7_u8, vl); + store_u8_8x4(d_u8 + 4 * dst8_stride, dst8_stride, d4_u8, d5_u8, d6_u8, + d7_u8, vl); + + // Update sliding window + s0 = __riscv_vmv_v_v_i16m1(s8, vl); + s1 = __riscv_vmv_v_v_i16m1(s9, vl); + s2 = __riscv_vmv_v_v_i16m1(s10, vl); + s3 = __riscv_vmv_v_v_i16m1(s11, vl); + s4 = __riscv_vmv_v_v_i16m1(s12, vl); + s5 = __riscv_vmv_v_v_i16m1(s13, vl); + s6 = __riscv_vmv_v_v_i16m1(s14, vl); + s += 8; + d += 8; + d_u8 += 8; + width -= 8; + } while (width != 0); + src_ptr += 8 * src_stride; + dst_ptr += 8 * dst_stride; + dst8_ptr += 8 * dst8_stride; + height -= 8; + } + } + + while (height > 0) { + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + uint8_t *d_u8 = dst8_ptr; + int width = w; + + do { + // Load + vuint8mf2_t t0, t1, t2, t3, t4, t5, t6, t7; + load_u8_8x8(s, 1, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7, vl); + + // Convert to 16-bit integers + vint16m1_t s0 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t0, vl)); + vint16m1_t s1 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t1, vl)); + vint16m1_t s2 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t2, vl)); + vint16m1_t s3 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t3, vl)); + vint16m1_t s4 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t4, vl)); + vint16m1_t s5 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t5, vl)); + vint16m1_t s6 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t6, vl)); + vint16m1_t s7 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t7, vl)); + + // Perform convolution + vuint16m1_t d0 = convolve8_8_x_rvv(s0, s1, s2, s3, s4, s5, s6, s7, + filter, round_offset, vl); + vuint16m1_t dd0 = __riscv_vle16_v_u16m1(d, vl); + + vuint8mf2_t d0_u8; + compute_dist_wtd_avg_8x1_rvv(dd0, d0, fwd_offset, bck_offset, + round_offset, &d0_u8, vl); + + // Store result + __riscv_vse8_v_u8mf2(d_u8, d0_u8, vl); + + s += vl; + d += vl; + d_u8 += vl; + width -= vl; + } while (width > 0); + src_ptr += src_stride; + dst_ptr += dst_stride; + dst8_ptr += dst8_stride; + height--; + } + } +} + +static inline void dist_wtd_convolve_x_avg_rvv( + const uint8_t *src, int src_stride, uint8_t *dst8, int dst8_stride, int w, + int h, const InterpFilterParams *filter_params_x, const int subpel_x_qn, + ConvolveParams *conv_params) { + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + // Horizontal filter. + const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel( + filter_params_x, subpel_x_qn & SUBPEL_MASK); + + const int horiz_offset = filter_params_x->taps / 2 - 1; + const uint8_t *src_ptr = src - horiz_offset; + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + uint8_t *dst8_ptr = dst8; + int dst_stride = conv_params->dst_stride; + int height = h; + + size_t vl = __riscv_vsetvl_e16m1(w); + if (w == 4) { + // 4-tap filters are used for blocks having width <= 4. + // Filter values are even, so halve to reduce intermediate precision reqs. + const int16_t *x_filter = x_filter_ptr + 2; + int16_t filter[4]; + for (int i = 0; i < 4; i++) filter[i] = *x_filter++ >> 1; + + src_ptr += 2; + + do { + // Load + vuint8mf2_t t0 = __riscv_vle8_v_u8mf2(src_ptr + 0 * src_stride, + 8); // a0 a1 a2 a3 a4 a5 a6 a7 + vuint8mf2_t t1 = __riscv_vle8_v_u8mf2(src_ptr + 1 * src_stride, + 8); // b0 b1 b2 b3 b4 b5 b6 b7 + + __builtin_prefetch(dst_ptr); + __builtin_prefetch(dst8_ptr); + + // Widen to 16-bit + vint16m1_t s00 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t0, 8)); + vint16m1_t s01 = + __riscv_vslide1down_vx_i16m1(s00, 0, 8); // a1 a2 a3 a4 a5 a6 a7 0 + vint16m1_t s02 = + __riscv_vslide1down_vx_i16m1(s01, 0, 8); // a2 a3 a4 a5 a6 a7 0 0 + vint16m1_t s03 = + __riscv_vslide1down_vx_i16m1(s02, 0, 8); // a3 a4 a5 a6 a7 0 0 0 + + vint16m1_t s10 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t1, 8)); + vint16m1_t s11 = + __riscv_vslide1down_vx_i16m1(s10, 0, 8); // b1 b2 b3 b4 b5 b6 b7 0 + vint16m1_t s12 = + __riscv_vslide1down_vx_i16m1(s11, 0, 8); // b2 b3 b4 b5 b6 b7 0 0 + vint16m1_t s13 = + __riscv_vslide1down_vx_i16m1(s12, 0, 8); // b3 b4 b5 b6 b7 0 0 0 + + // Perform convolution + vuint8mf2_t d01, d11; + vuint16m1_t d0 = + convolve4_4_x_rvv(s00, s01, s02, s03, filter, round_offset, vl); + vuint16m1_t d1 = + convolve4_4_x_rvv(s10, s11, s12, s13, filter, round_offset, vl); + + // average + vuint16m1_t dd0 = __riscv_vle16_v_u16m1(dst_ptr + 0 * dst_stride, vl); + vuint16m1_t dd1 = __riscv_vle16_v_u16m1(dst_ptr + 1 * dst_stride, vl); + compute_basic_avg_4x1_rvv(dd0, d0, round_offset, &d01, vl); + compute_basic_avg_4x1_rvv(dd1, d1, round_offset, &d11, vl); + + // Store result + __riscv_vse8_v_u8mf2(dst8_ptr + 0 * dst8_stride, d01, vl); + __riscv_vse8_v_u8mf2(dst8_ptr + 1 * dst8_stride, d11, vl); + + src_ptr += 2 * src_stride; + dst_ptr += 2 * dst_stride; + dst8_ptr += 2 * dst8_stride; + height -= 2; + } while (height >= 2); + } else { + // Filter values are even, so halve to reduce intermediate precision reqs. + int16_t filter[8]; + for (int i = 0; i < 8; i++) filter[i] = *x_filter_ptr++ >> 1; + + // for VLEN = 128 case + if ((w > 8) && (vl == 8) && (height >= 8)) { + vuint16m8_t trans_index = __riscv_vle16_v_u16m8(trans_data, 64); + + while (height >= 8) { + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + uint8_t *d_u8 = dst8_ptr; + int width = w; + + // Load + vuint8mf2_t t0, t1, t2, t3, t4, t5, t6, t7; + load_stride_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, + &t7, vl); + + vint16m1_t s0 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t0, vl)); + vint16m1_t s1 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t1, vl)); + vint16m1_t s2 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t2, vl)); + vint16m1_t s3 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t3, vl)); + vint16m1_t s4 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t4, vl)); + vint16m1_t s5 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t5, vl)); + vint16m1_t s6 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t6, vl)); + + __builtin_prefetch(d + 0 * dst_stride); + __builtin_prefetch(d + 1 * dst_stride); + __builtin_prefetch(d + 2 * dst_stride); + __builtin_prefetch(d + 3 * dst_stride); + __builtin_prefetch(d + 4 * dst_stride); + __builtin_prefetch(d + 5 * dst_stride); + __builtin_prefetch(d + 6 * dst_stride); + __builtin_prefetch(d + 7 * dst_stride); + + s += 7; + + do { + load_stride_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, + &t7, vl); + + vint16m1_t s7 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vzext_vf2_u16m1(t0, vl)); + vint16m1_t s8 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vzext_vf2_u16m1(t1, vl)); + vint16m1_t s9 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vzext_vf2_u16m1(t2, vl)); + vint16m1_t s10 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vzext_vf2_u16m1(t3, vl)); + vint16m1_t s11 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vzext_vf2_u16m1(t4, vl)); + vint16m1_t s12 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vzext_vf2_u16m1(t5, vl)); + vint16m1_t s13 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vzext_vf2_u16m1(t6, vl)); + vint16m1_t s14 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vzext_vf2_u16m1(t7, vl)); + + // Perform convolution + vuint16m1_t d0 = convolve8_8_x_rvv(s0, s1, s2, s3, s4, s5, s6, s7, + filter, round_offset, vl); + vuint16m1_t d1 = convolve8_8_x_rvv(s1, s2, s3, s4, s5, s6, s7, s8, + filter, round_offset, vl); + vuint16m1_t d2 = convolve8_8_x_rvv(s2, s3, s4, s5, s6, s7, s8, s9, + filter, round_offset, vl); + vuint16m1_t d3 = convolve8_8_x_rvv(s3, s4, s5, s6, s7, s8, s9, s10, + filter, round_offset, vl); + vuint16m1_t d4 = convolve8_8_x_rvv(s4, s5, s6, s7, s8, s9, s10, s11, + filter, round_offset, vl); + vuint16m1_t d5 = convolve8_8_x_rvv(s5, s6, s7, s8, s9, s10, s11, s12, + filter, round_offset, vl); + vuint16m1_t d6 = convolve8_8_x_rvv(s6, s7, s8, s9, s10, s11, s12, s13, + filter, round_offset, vl); + vuint16m1_t d7 = convolve8_8_x_rvv(s7, s8, s9, s10, s11, s12, s13, + s14, filter, round_offset, vl); + + // transpose + transpose_elems_inplace_u16_8x8_rvv(&d0, &d1, &d2, &d3, &d4, &d5, &d6, + &d7, trans_index); + + vuint16m1_t dd0, dd1, dd2, dd3; + load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3, vl); + + vuint8mf2_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_basic_avg_8x4_rvv(dd0, dd1, dd2, dd3, d0, d1, d2, d3, + round_offset, &d0_u8, &d1_u8, &d2_u8, + &d3_u8, vl); + store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8, vl); + + vuint16m1_t dd4, dd5, dd6, dd7; + load_u16_8x4(d + 4 * dst_stride, dst_stride, &dd4, &dd5, &dd6, &dd7, + vl); + + vuint8mf2_t d4_u8, d5_u8, d6_u8, d7_u8; + compute_basic_avg_8x4_rvv(dd4, dd5, dd6, dd7, d4, d5, d6, d7, + round_offset, &d4_u8, &d5_u8, &d6_u8, + &d7_u8, vl); + store_u8_8x4(d_u8 + 4 * dst8_stride, dst8_stride, d4_u8, d5_u8, d6_u8, + d7_u8, vl); + + // Update sliding window + s0 = __riscv_vmv_v_v_i16m1(s8, vl); + s1 = __riscv_vmv_v_v_i16m1(s9, vl); + s2 = __riscv_vmv_v_v_i16m1(s10, vl); + s3 = __riscv_vmv_v_v_i16m1(s11, vl); + s4 = __riscv_vmv_v_v_i16m1(s12, vl); + s5 = __riscv_vmv_v_v_i16m1(s13, vl); + s6 = __riscv_vmv_v_v_i16m1(s14, vl); + s += 8; + d += 8; + d_u8 += 8; + width -= 8; + } while (width != 0); + src_ptr += 8 * src_stride; + dst_ptr += 8 * dst_stride; + dst8_ptr += 8 * dst8_stride; + height -= 8; + } + } + + while (height > 0) { + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + uint8_t *d_u8 = dst8_ptr; + int width = w; + + do { + // Load + vuint8mf2_t t0, t1, t2, t3, t4, t5, t6, t7; + load_u8_8x8(s, 1, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7, vl); + + // Convert to 16-bit integers + vint16m1_t s0 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t0, vl)); + vint16m1_t s1 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t1, vl)); + vint16m1_t s2 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t2, vl)); + vint16m1_t s3 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t3, vl)); + vint16m1_t s4 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t4, vl)); + vint16m1_t s5 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t5, vl)); + vint16m1_t s6 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t6, vl)); + vint16m1_t s7 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t7, vl)); + + // Perform convolution + vuint16m1_t d0 = convolve8_8_x_rvv(s0, s1, s2, s3, s4, s5, s6, s7, + filter, round_offset, vl); + vuint16m1_t dd0 = __riscv_vle16_v_u16m1(d, vl); + + vuint8mf2_t d0_u8; + compute_basic_avg_8x1_rvv(dd0, d0, round_offset, &d0_u8, vl); + + // Store result + __riscv_vse8_v_u8mf2(d_u8, d0_u8, vl); + + s += vl; + d += vl; + d_u8 += vl; + width -= vl; + } while (width > 0); + src_ptr += src_stride; + dst_ptr += dst_stride; + dst8_ptr += dst8_stride; + height--; + } + } +} + +static inline void dist_wtd_convolve_x_rvv( + const uint8_t *src, int src_stride, int w, int h, + const InterpFilterParams *filter_params_x, const int subpel_x_qn, + ConvolveParams *conv_params) { + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + // Horizontal filter. + const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel( + filter_params_x, subpel_x_qn & SUBPEL_MASK); + + const int horiz_offset = filter_params_x->taps / 2 - 1; + const uint8_t *src_ptr = src - horiz_offset; + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + int dst_stride = conv_params->dst_stride; + int height = h; + + size_t vl = __riscv_vsetvl_e16m1(w); + if (w == 4) { + // 4-tap filters are used for blocks having width <= 4. + // Filter values are even, so halve to reduce intermediate precision reqs. + const int16_t *x_filter = x_filter_ptr + 2; + int16_t filter[4]; + for (int i = 0; i < 4; i++) filter[i] = *(x_filter + i) >> 1; + + src_ptr += 2; + + do { + // Load + vuint8mf2_t t0 = __riscv_vle8_v_u8mf2(src_ptr + 0 * src_stride, + 8); // a0 a1 a2 a3 a4 a5 a6 a7 + vuint8mf2_t t1 = __riscv_vle8_v_u8mf2(src_ptr + 1 * src_stride, + 8); // b0 b1 b2 b3 b4 b5 b6 b7 + + __builtin_prefetch(dst_ptr); + + // Widen to 16-bit + vint16m1_t s00 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t0, 8)); + vint16m1_t s01 = + __riscv_vslide1down_vx_i16m1(s00, 0, 8); // a1 a2 a3 a4 a5 a6 a7 0 + vint16m1_t s02 = + __riscv_vslide1down_vx_i16m1(s01, 0, 8); // a2 a3 a4 a5 a6 a7 0 0 + vint16m1_t s03 = + __riscv_vslide1down_vx_i16m1(s02, 0, 8); // a3 a4 a5 a6 a7 0 0 0 + + vint16m1_t s10 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t1, 8)); + vint16m1_t s11 = + __riscv_vslide1down_vx_i16m1(s10, 0, 8); // b1 b2 b3 b4 b5 b6 b7 0 + vint16m1_t s12 = + __riscv_vslide1down_vx_i16m1(s11, 0, 8); // b2 b3 b4 b5 b6 b7 0 0 + vint16m1_t s13 = + __riscv_vslide1down_vx_i16m1(s12, 0, 8); // b3 b4 b5 b6 b7 0 0 0 + + // Perform convolution + vuint16m1_t d0 = + convolve4_4_x_rvv(s00, s01, s02, s03, filter, round_offset, vl); + vuint16m1_t d1 = + convolve4_4_x_rvv(s10, s11, s12, s13, filter, round_offset, vl); + + // Store result + __riscv_vse16_v_u16m1(dst_ptr + 0 * dst_stride, d0, vl); + __riscv_vse16_v_u16m1(dst_ptr + 1 * dst_stride, d1, vl); + + src_ptr += 2 * src_stride; + dst_ptr += 2 * dst_stride; + height -= 2; + } while (height >= 2); + } else { + // Filter values are even, so halve to reduce intermediate precision reqs. + int16_t filter[8]; + for (int i = 0; i < 8; i++) filter[i] = *x_filter_ptr++ >> 1; + + // for VLEN = 128 case + if ((w > 8) && (vl == 8) && (height >= 8)) { + vuint16m8_t trans_index = __riscv_vle16_v_u16m8(trans_data, 64); + + while (height >= 8) { + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + int width = w; + + // Load + vuint8mf2_t t0, t1, t2, t3, t4, t5, t6, t7; + load_stride_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, + &t7, vl); + + vint16m1_t s0 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t0, vl)); + vint16m1_t s1 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t1, vl)); + vint16m1_t s2 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t2, vl)); + vint16m1_t s3 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t3, vl)); + vint16m1_t s4 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t4, vl)); + vint16m1_t s5 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t5, vl)); + vint16m1_t s6 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t6, vl)); + + __builtin_prefetch(d + 0 * dst_stride); + __builtin_prefetch(d + 1 * dst_stride); + __builtin_prefetch(d + 2 * dst_stride); + __builtin_prefetch(d + 3 * dst_stride); + __builtin_prefetch(d + 4 * dst_stride); + __builtin_prefetch(d + 5 * dst_stride); + __builtin_prefetch(d + 6 * dst_stride); + __builtin_prefetch(d + 7 * dst_stride); + + s += 7; + + do { + load_stride_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, + &t7, vl); + + vint16m1_t s7 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vzext_vf2_u16m1(t0, vl)); + vint16m1_t s8 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vzext_vf2_u16m1(t1, vl)); + vint16m1_t s9 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vzext_vf2_u16m1(t2, vl)); + vint16m1_t s10 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vzext_vf2_u16m1(t3, vl)); + vint16m1_t s11 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vzext_vf2_u16m1(t4, vl)); + vint16m1_t s12 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vzext_vf2_u16m1(t5, vl)); + vint16m1_t s13 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vzext_vf2_u16m1(t6, vl)); + vint16m1_t s14 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vzext_vf2_u16m1(t7, vl)); + + // Perform convolution + vuint16m1_t d0 = convolve8_8_x_rvv(s0, s1, s2, s3, s4, s5, s6, s7, + filter, round_offset, vl); + vuint16m1_t d1 = convolve8_8_x_rvv(s1, s2, s3, s4, s5, s6, s7, s8, + filter, round_offset, vl); + vuint16m1_t d2 = convolve8_8_x_rvv(s2, s3, s4, s5, s6, s7, s8, s9, + filter, round_offset, vl); + vuint16m1_t d3 = convolve8_8_x_rvv(s3, s4, s5, s6, s7, s8, s9, s10, + filter, round_offset, vl); + vuint16m1_t d4 = convolve8_8_x_rvv(s4, s5, s6, s7, s8, s9, s10, s11, + filter, round_offset, vl); + vuint16m1_t d5 = convolve8_8_x_rvv(s5, s6, s7, s8, s9, s10, s11, s12, + filter, round_offset, vl); + vuint16m1_t d6 = convolve8_8_x_rvv(s6, s7, s8, s9, s10, s11, s12, s13, + filter, round_offset, vl); + vuint16m1_t d7 = convolve8_8_x_rvv(s7, s8, s9, s10, s11, s12, s13, + s14, filter, round_offset, vl); + + // transpose + transpose_elems_inplace_u16_8x8_rvv(&d0, &d1, &d2, &d3, &d4, &d5, &d6, + &d7, trans_index); + + // Store result + store_u16_8x8(d, dst_stride, d0, d1, d2, d3, d4, d5, d6, d7, vl); + + // Update sliding window + s0 = __riscv_vmv_v_v_i16m1(s8, vl); + s1 = __riscv_vmv_v_v_i16m1(s9, vl); + s2 = __riscv_vmv_v_v_i16m1(s10, vl); + s3 = __riscv_vmv_v_v_i16m1(s11, vl); + s4 = __riscv_vmv_v_v_i16m1(s12, vl); + s5 = __riscv_vmv_v_v_i16m1(s13, vl); + s6 = __riscv_vmv_v_v_i16m1(s14, vl); + s += 8; + d += 8; + width -= 8; + } while (width != 0); + src_ptr += 8 * src_stride; + dst_ptr += 8 * dst_stride; + height -= 8; + } + } + + while (height > 0) { + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + int width = w; + + __builtin_prefetch(d); + + do { + // Load + vuint8mf2_t t0, t1, t2, t3, t4, t5, t6, t7; + load_u8_8x8(s, 1, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7, vl); + + // Convert to 16-bit integers + vint16m1_t s0 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t0, vl)); + vint16m1_t s1 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t1, vl)); + vint16m1_t s2 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t2, vl)); + vint16m1_t s3 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t3, vl)); + vint16m1_t s4 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t4, vl)); + vint16m1_t s5 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t5, vl)); + vint16m1_t s6 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t6, vl)); + vint16m1_t s7 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t7, vl)); + + // Perform convolution + vuint16m1_t d0 = convolve8_8_x_rvv(s0, s1, s2, s3, s4, s5, s6, s7, + filter, round_offset, vl); + + // Store result + __riscv_vse16_v_u16m1(d, d0, vl); + + s += vl; + d += vl; + width -= vl; + } while (width > 0); + src_ptr += src_stride; + dst_ptr += dst_stride; + height--; + } + } +} + +void av1_dist_wtd_convolve_x_rvv(const uint8_t *src, int src_stride, + uint8_t *dst8, int dst8_stride, int w, int h, + const InterpFilterParams *filter_params_x, + const int subpel_x_qn, + ConvolveParams *conv_params) { + if (conv_params->do_average) { + if (UNLIKELY(conv_params->use_dist_wtd_comp_avg)) { + dist_wtd_convolve_x_dist_wtd_avg_rvv(src, src_stride, dst8, dst8_stride, + w, h, filter_params_x, subpel_x_qn, + conv_params); + } else { + dist_wtd_convolve_x_avg_rvv(src, src_stride, dst8, dst8_stride, w, h, + filter_params_x, subpel_x_qn, conv_params); + } + } else { + dist_wtd_convolve_x_rvv(src, src_stride, w, h, filter_params_x, subpel_x_qn, + conv_params); + } +} + +static inline vuint16mf2_t convolve6_4_y_rvv( + const vint16mf2_t s0, const vint16mf2_t s1, const vint16mf2_t s2, + const vint16mf2_t s3, const vint16mf2_t s4, const vint16mf2_t s5, + const int16_t *filter, const int16_t round_offset, size_t vl) { + // Filter values at indices 0 and 7 are 0. + vint16mf2_t sum = __riscv_vmul_vx_i16mf2(s0, filter[1], vl); + sum = __riscv_vmacc_vx_i16mf2(sum, filter[2], s1, vl); + sum = __riscv_vmacc_vx_i16mf2(sum, filter[3], s2, vl); + sum = __riscv_vmacc_vx_i16mf2(sum, filter[4], s3, vl); + sum = __riscv_vmacc_vx_i16mf2(sum, filter[5], s4, vl); + sum = __riscv_vmacc_vx_i16mf2(sum, filter[6], s5, vl); + + // Right shift with rounding: vrsra_n_s16(round_offset, sum, ROUND0_BITS - 1) + // vrsra_n_s16(a, b, n) = a + ((b + (1 << (n-1))) >> n) + // We halved the convolution filter values so -1 from the right shift. + vint16mf2_t res = + __riscv_vadd_vx_i16mf2(sum, 1 << ((ROUND0_BITS - 1) - 1), vl); + res = __riscv_vsra_vx_i16mf2(res, ROUND0_BITS - 1, vl); + res = __riscv_vadd_vx_i16mf2(res, round_offset, vl); + + // Reinterpret as uint16 + return __riscv_vreinterpret_v_i16mf2_u16mf2(res); +} + +static inline vuint16m1_t convolve6_8_y_rvv( + const vint16m1_t s0, const vint16m1_t s1, const vint16m1_t s2, + const vint16m1_t s3, const vint16m1_t s4, const vint16m1_t s5, + const int16_t *filter, const int16_t round_offset, size_t vl) { + // Filter values at indices 0 and 7 are 0. + vint16m1_t sum = __riscv_vmul_vx_i16m1(s0, filter[1], vl); + sum = __riscv_vmacc_vx_i16m1(sum, filter[2], s1, vl); + sum = __riscv_vmacc_vx_i16m1(sum, filter[3], s2, vl); + sum = __riscv_vmacc_vx_i16m1(sum, filter[4], s3, vl); + sum = __riscv_vmacc_vx_i16m1(sum, filter[5], s4, vl); + sum = __riscv_vmacc_vx_i16m1(sum, filter[6], s5, vl); + + // Right shift with rounding: vrsra_n_s16(round_offset, sum, ROUND0_BITS - 1) + // vrsra_n_s16(a, b, n) = a + ((b + (1 << (n-1))) >> n) + // We halved the convolution filter values so -1 from the right shift. + vint16m1_t res = __riscv_vadd_vx_i16m1(sum, 1 << ((ROUND0_BITS - 1) - 1), vl); + res = __riscv_vsra_vx_i16m1(res, ROUND0_BITS - 1, vl); + res = __riscv_vadd_vx_i16m1(res, round_offset, vl); + + // Reinterpret as uint16 + return __riscv_vreinterpret_v_i16m1_u16m1(res); +} + +static inline void dist_wtd_convolve_y_6tap_dist_wtd_avg_rvv( + const uint8_t *src_ptr, int src_stride, uint8_t *dst8_ptr, + const int dst8_stride, int w, int h, const int16_t *y_filter, + ConvolveParams *conv_params) { + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + const uint16_t fwd_offset = conv_params->fwd_offset; + const uint16_t bck_offset = conv_params->bck_offset; + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + int width = w; + + if (w == 4 || h == 4) { + size_t vl = __riscv_vsetvl_e16m1(4); + do { + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + uint8_t *d_u8 = dst8_ptr; + int height = h; + + // Load initial 5 rows of data + vuint8mf4_t t0, t1, t2, t3, t4; + load_u8_4x5(s, src_stride, &t0, &t1, &t2, &t3, &t4, vl); + + // Convert to 16-bit + vint16mf2_t s0 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t0, vl)); + vint16mf2_t s1 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t1, vl)); + vint16mf2_t s2 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t2, vl)); + vint16mf2_t s3 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t3, vl)); + vint16mf2_t s4 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t4, vl)); + + s += 5 * src_stride; + + do { + // Load next rows of data + vuint8mf4_t t5 = __riscv_vle8_v_u8mf4(s + 0 * src_stride, vl); + vuint8mf4_t t6 = __riscv_vle8_v_u8mf4(s + 1 * src_stride, vl); + vuint8mf4_t t7 = __riscv_vle8_v_u8mf4(s + 2 * src_stride, vl); + vuint8mf4_t t8 = __riscv_vle8_v_u8mf4(s + 3 * src_stride, vl); + + // Convert to 16-bit + vint16mf2_t s5 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t5, vl)); + vint16mf2_t s6 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t6, vl)); + vint16mf2_t s7 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t7, vl)); + vint16mf2_t s8 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t8, vl)); + + // Perform convolution + vuint16mf2_t d0 = convolve6_4_y_rvv(s0, s1, s2, s3, s4, s5, y_filter, + round_offset, vl); + vuint16mf2_t d1 = convolve6_4_y_rvv(s1, s2, s3, s4, s5, s6, y_filter, + round_offset, vl); + vuint16mf2_t d2 = convolve6_4_y_rvv(s2, s3, s4, s5, s6, s7, y_filter, + round_offset, vl); + vuint16mf2_t d3 = convolve6_4_y_rvv(s3, s4, s5, s6, s7, s8, y_filter, + round_offset, vl); + + vuint16mf2_t dd0, dd1, dd2, dd3; + load_u16_4x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3, vl); + + vuint8mf4_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_dist_wtd_avg_4x4_rvv(dd0, dd1, dd2, dd3, d0, d1, d2, d3, + fwd_offset, bck_offset, round_offset, + &d0_u8, &d1_u8, &d2_u8, &d3_u8, vl); + + // Store result + __riscv_vse8_v_u8mf4(d_u8 + 0 * dst8_stride, d0_u8, vl); + __riscv_vse8_v_u8mf4(d_u8 + 1 * dst8_stride, d1_u8, vl); + __riscv_vse8_v_u8mf4(d_u8 + 2 * dst8_stride, d2_u8, vl); + __riscv_vse8_v_u8mf4(d_u8 + 3 * dst8_stride, d3_u8, vl); + + // Update sliding window + s0 = __riscv_vmv_v_v_i16mf2(s4, vl); + s1 = __riscv_vmv_v_v_i16mf2(s5, vl); + s2 = __riscv_vmv_v_v_i16mf2(s6, vl); + s3 = __riscv_vmv_v_v_i16mf2(s7, vl); + s4 = __riscv_vmv_v_v_i16mf2(s8, vl); + s += 4 * src_stride; + d += 4 * dst_stride; + d_u8 += 4 * dst8_stride; + height -= 4; + } while (height != 0); + src_ptr += 4; + dst_ptr += 4; + dst8_ptr += 4; + width -= 4; + } while (width != 0); + } else { + size_t vl = __riscv_vsetvl_e16m1(w); + do { + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + uint8_t *d_u8 = dst8_ptr; + int height = h; + + // Load initial 5 rows of data + vuint8mf2_t t0, t1, t2, t3, t4; + load_u8_8x5(s, src_stride, &t0, &t1, &t2, &t3, &t4, vl); + + // Convert to 16-bit + vint16m1_t s0 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t0, vl)); + vint16m1_t s1 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t1, vl)); + vint16m1_t s2 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t2, vl)); + vint16m1_t s3 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t3, vl)); + vint16m1_t s4 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t4, vl)); + + s += 5 * src_stride; + + do { + vuint8mf2_t t5, t6, t7; + load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7, vl); + + // Convert to 16-bit + vint16m1_t s5 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t0, vl)); + vint16m1_t s6 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t1, vl)); + vint16m1_t s7 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t2, vl)); + vint16m1_t s8 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t3, vl)); + vint16m1_t s9 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t4, vl)); + vint16m1_t s10 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t5, vl)); + vint16m1_t s11 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t6, vl)); + vint16m1_t s12 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t7, vl)); + + // Perform convolution + vuint16m1_t d0 = convolve6_8_y_rvv(s0, s1, s2, s3, s4, s5, y_filter, + round_offset, vl); + vuint16m1_t d1 = convolve6_8_y_rvv(s1, s2, s3, s4, s5, s6, y_filter, + round_offset, vl); + vuint16m1_t d2 = convolve6_8_y_rvv(s2, s3, s4, s5, s6, s7, y_filter, + round_offset, vl); + vuint16m1_t d3 = convolve6_8_y_rvv(s3, s4, s5, s6, s7, s8, y_filter, + round_offset, vl); + vuint16m1_t d4 = convolve6_8_y_rvv(s4, s5, s6, s7, s8, s9, y_filter, + round_offset, vl); + vuint16m1_t d5 = convolve6_8_y_rvv(s5, s6, s7, s8, s9, s10, y_filter, + round_offset, vl); + vuint16m1_t d6 = convolve6_8_y_rvv(s6, s7, s8, s9, s10, s11, y_filter, + round_offset, vl); + vuint16m1_t d7 = convolve6_8_y_rvv(s7, s8, s9, s10, s11, s12, y_filter, + round_offset, vl); + + vuint16m1_t dd0, dd1, dd2, dd3; + load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3, vl); + + vuint8mf2_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_dist_wtd_avg_8x4_rvv(dd0, dd1, dd2, dd3, d0, d1, d2, d3, + fwd_offset, bck_offset, round_offset, + &d0_u8, &d1_u8, &d2_u8, &d3_u8, vl); + store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8, vl); + d_u8 += 4 * dst8_stride; + + vuint16m1_t dd4, dd5, dd6, dd7; + load_u16_8x4(d + 4 * dst_stride, dst_stride, &dd4, &dd5, &dd6, &dd7, + vl); + + vuint8mf2_t d4_u8, d5_u8, d6_u8, d7_u8; + compute_dist_wtd_avg_8x4_rvv(dd4, dd5, dd6, dd7, d4, d5, d6, d7, + fwd_offset, bck_offset, round_offset, + &d4_u8, &d5_u8, &d6_u8, &d7_u8, vl); + store_u8_8x4(d_u8, dst8_stride, d4_u8, d5_u8, d6_u8, d7_u8, vl); + d_u8 += 4 * dst8_stride; + + // Update sliding window + s0 = __riscv_vmv_v_v_i16m1(s8, vl); + s1 = __riscv_vmv_v_v_i16m1(s9, vl); + s2 = __riscv_vmv_v_v_i16m1(s10, vl); + s3 = __riscv_vmv_v_v_i16m1(s11, vl); + s4 = __riscv_vmv_v_v_i16m1(s12, vl); + s += 8 * src_stride; + d += 8 * dst_stride; + height -= 8; + } while (height != 0); + src_ptr += vl; + dst_ptr += vl; + dst8_ptr += vl; + width -= vl; + } while (width > 0); + } +} + +static inline void dist_wtd_convolve_y_6tap_avg_rvv( + const uint8_t *src_ptr, int src_stride, uint8_t *dst8_ptr, + const int dst8_stride, int w, int h, const int16_t *y_filter, + ConvolveParams *conv_params) { + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + int width = w; + + if (w == 4 || h == 4) { + size_t vl = __riscv_vsetvl_e16m1(4); + do { + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + uint8_t *d_u8 = dst8_ptr; + int height = h; + + // Load initial 5 rows of data + vuint8mf4_t t0, t1, t2, t3, t4; + load_u8_4x5(s, src_stride, &t0, &t1, &t2, &t3, &t4, vl); + + // Convert to 16-bit + vint16mf2_t s0 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t0, vl)); + vint16mf2_t s1 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t1, vl)); + vint16mf2_t s2 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t2, vl)); + vint16mf2_t s3 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t3, vl)); + vint16mf2_t s4 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t4, vl)); + + s += 5 * src_stride; + + do { + // Load next rows of data + vuint8mf4_t t5 = __riscv_vle8_v_u8mf4(s + 0 * src_stride, vl); + vuint8mf4_t t6 = __riscv_vle8_v_u8mf4(s + 1 * src_stride, vl); + vuint8mf4_t t7 = __riscv_vle8_v_u8mf4(s + 2 * src_stride, vl); + vuint8mf4_t t8 = __riscv_vle8_v_u8mf4(s + 3 * src_stride, vl); + + // Convert to 16-bit + vint16mf2_t s5 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t5, vl)); + vint16mf2_t s6 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t6, vl)); + vint16mf2_t s7 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t7, vl)); + vint16mf2_t s8 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t8, vl)); + + // Perform convolution + vuint16mf2_t d0 = convolve6_4_y_rvv(s0, s1, s2, s3, s4, s5, y_filter, + round_offset, vl); + vuint16mf2_t d1 = convolve6_4_y_rvv(s1, s2, s3, s4, s5, s6, y_filter, + round_offset, vl); + vuint16mf2_t d2 = convolve6_4_y_rvv(s2, s3, s4, s5, s6, s7, y_filter, + round_offset, vl); + vuint16mf2_t d3 = convolve6_4_y_rvv(s3, s4, s5, s6, s7, s8, y_filter, + round_offset, vl); + + vuint16mf2_t dd0, dd1, dd2, dd3; + load_u16_4x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3, vl); + + vuint8mf4_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_basic_avg_4x4_rvv(dd0, dd1, dd2, dd3, d0, d1, d2, d3, + round_offset, &d0_u8, &d1_u8, &d2_u8, &d3_u8, + vl); + + // Store result + __riscv_vse8_v_u8mf4(d_u8 + 0 * dst8_stride, d0_u8, vl); + __riscv_vse8_v_u8mf4(d_u8 + 1 * dst8_stride, d1_u8, vl); + __riscv_vse8_v_u8mf4(d_u8 + 2 * dst8_stride, d2_u8, vl); + __riscv_vse8_v_u8mf4(d_u8 + 3 * dst8_stride, d3_u8, vl); + + // Update sliding window + s0 = __riscv_vmv_v_v_i16mf2(s4, vl); + s1 = __riscv_vmv_v_v_i16mf2(s5, vl); + s2 = __riscv_vmv_v_v_i16mf2(s6, vl); + s3 = __riscv_vmv_v_v_i16mf2(s7, vl); + s4 = __riscv_vmv_v_v_i16mf2(s8, vl); + s += 4 * src_stride; + d += 4 * dst_stride; + d_u8 += 4 * dst8_stride; + height -= 4; + } while (height != 0); + src_ptr += 4; + dst_ptr += 4; + dst8_ptr += 4; + width -= 4; + } while (width != 0); + } else { + size_t vl = __riscv_vsetvl_e16m1(w); + do { + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + uint8_t *d_u8 = dst8_ptr; + int height = h; + + // Load initial 5 rows of data + vuint8mf2_t t0, t1, t2, t3, t4; + load_u8_8x5(s, src_stride, &t0, &t1, &t2, &t3, &t4, vl); + + // Convert to 16-bit + vint16m1_t s0 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t0, vl)); + vint16m1_t s1 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t1, vl)); + vint16m1_t s2 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t2, vl)); + vint16m1_t s3 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t3, vl)); + vint16m1_t s4 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t4, vl)); + + s += 5 * src_stride; + + do { + vuint8mf2_t t5, t6, t7; + load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7, vl); + + // Convert to 16-bit + vint16m1_t s5 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t0, vl)); + vint16m1_t s6 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t1, vl)); + vint16m1_t s7 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t2, vl)); + vint16m1_t s8 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t3, vl)); + vint16m1_t s9 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t4, vl)); + vint16m1_t s10 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t5, vl)); + vint16m1_t s11 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t6, vl)); + vint16m1_t s12 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t7, vl)); + + // Perform convolution + vuint16m1_t d0 = convolve6_8_y_rvv(s0, s1, s2, s3, s4, s5, y_filter, + round_offset, vl); + vuint16m1_t d1 = convolve6_8_y_rvv(s1, s2, s3, s4, s5, s6, y_filter, + round_offset, vl); + vuint16m1_t d2 = convolve6_8_y_rvv(s2, s3, s4, s5, s6, s7, y_filter, + round_offset, vl); + vuint16m1_t d3 = convolve6_8_y_rvv(s3, s4, s5, s6, s7, s8, y_filter, + round_offset, vl); + vuint16m1_t d4 = convolve6_8_y_rvv(s4, s5, s6, s7, s8, s9, y_filter, + round_offset, vl); + vuint16m1_t d5 = convolve6_8_y_rvv(s5, s6, s7, s8, s9, s10, y_filter, + round_offset, vl); + vuint16m1_t d6 = convolve6_8_y_rvv(s6, s7, s8, s9, s10, s11, y_filter, + round_offset, vl); + vuint16m1_t d7 = convolve6_8_y_rvv(s7, s8, s9, s10, s11, s12, y_filter, + round_offset, vl); + + vuint16m1_t dd0, dd1, dd2, dd3; + load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3, vl); + + vuint8mf2_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_basic_avg_8x4_rvv(dd0, dd1, dd2, dd3, d0, d1, d2, d3, + round_offset, &d0_u8, &d1_u8, &d2_u8, &d3_u8, + vl); + store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8, vl); + d_u8 += 4 * dst8_stride; + + vuint16m1_t dd4, dd5, dd6, dd7; + load_u16_8x4(d + 4 * dst_stride, dst_stride, &dd4, &dd5, &dd6, &dd7, + vl); + + vuint8mf2_t d4_u8, d5_u8, d6_u8, d7_u8; + compute_basic_avg_8x4_rvv(dd4, dd5, dd6, dd7, d4, d5, d6, d7, + round_offset, &d4_u8, &d5_u8, &d6_u8, &d7_u8, + vl); + store_u8_8x4(d_u8, dst8_stride, d4_u8, d5_u8, d6_u8, d7_u8, vl); + d_u8 += 4 * dst8_stride; + + // Update sliding window + s0 = __riscv_vmv_v_v_i16m1(s8, vl); + s1 = __riscv_vmv_v_v_i16m1(s9, vl); + s2 = __riscv_vmv_v_v_i16m1(s10, vl); + s3 = __riscv_vmv_v_v_i16m1(s11, vl); + s4 = __riscv_vmv_v_v_i16m1(s12, vl); + s += 8 * src_stride; + d += 8 * dst_stride; + height -= 8; + } while (height != 0); + src_ptr += vl; + dst_ptr += vl; + dst8_ptr += vl; + width -= vl; + } while (width > 0); + } +} + +static inline void dist_wtd_convolve_y_6tap_rvv(const uint8_t *src_ptr, + int src_stride, int w, int h, + const int16_t *y_filter, + ConvolveParams *conv_params) { + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + int width = w; + + if (w == 4 || h == 4) { + size_t vl = __riscv_vsetvl_e16m1(4); + do { + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + int height = h; + + // Load initial 5 rows of data + vuint8mf4_t t0, t1, t2, t3, t4; + load_u8_4x5(s, src_stride, &t0, &t1, &t2, &t3, &t4, vl); + + // Convert to 16-bit + vint16mf2_t s0 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t0, vl)); + vint16mf2_t s1 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t1, vl)); + vint16mf2_t s2 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t2, vl)); + vint16mf2_t s3 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t3, vl)); + vint16mf2_t s4 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t4, vl)); + + s += 5 * src_stride; + + do { + // Load next rows of data + vuint8mf4_t t5 = __riscv_vle8_v_u8mf4(s + 0 * src_stride, vl); + vuint8mf4_t t6 = __riscv_vle8_v_u8mf4(s + 1 * src_stride, vl); + vuint8mf4_t t7 = __riscv_vle8_v_u8mf4(s + 2 * src_stride, vl); + vuint8mf4_t t8 = __riscv_vle8_v_u8mf4(s + 3 * src_stride, vl); + + // Convert to 16-bit + vint16mf2_t s5 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t5, vl)); + vint16mf2_t s6 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t6, vl)); + vint16mf2_t s7 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t7, vl)); + vint16mf2_t s8 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t8, vl)); + + // Perform convolution + vuint16mf2_t d0 = convolve6_4_y_rvv(s0, s1, s2, s3, s4, s5, y_filter, + round_offset, vl); + vuint16mf2_t d1 = convolve6_4_y_rvv(s1, s2, s3, s4, s5, s6, y_filter, + round_offset, vl); + vuint16mf2_t d2 = convolve6_4_y_rvv(s2, s3, s4, s5, s6, s7, y_filter, + round_offset, vl); + vuint16mf2_t d3 = convolve6_4_y_rvv(s3, s4, s5, s6, s7, s8, y_filter, + round_offset, vl); + + // Store result + store_u16_4x4(d, dst_stride, d0, d1, d2, d3, vl); + + // Update sliding window + s0 = __riscv_vmv_v_v_i16mf2(s4, vl); + s1 = __riscv_vmv_v_v_i16mf2(s5, vl); + s2 = __riscv_vmv_v_v_i16mf2(s6, vl); + s3 = __riscv_vmv_v_v_i16mf2(s7, vl); + s4 = __riscv_vmv_v_v_i16mf2(s8, vl); + s += 4 * src_stride; + d += 4 * dst_stride; + height -= 4; + } while (height != 0); + src_ptr += 4; + dst_ptr += 4; + width -= 4; + } while (width != 0); + } else { + size_t vl = __riscv_vsetvl_e16m1(w); + do { + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + int height = h; + + // Load initial 5 rows of data + vuint8mf2_t t0, t1, t2, t3, t4; + load_u8_8x5(s, src_stride, &t0, &t1, &t2, &t3, &t4, vl); + + // Convert to 16-bit + vint16m1_t s0 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t0, vl)); + vint16m1_t s1 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t1, vl)); + vint16m1_t s2 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t2, vl)); + vint16m1_t s3 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t3, vl)); + vint16m1_t s4 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t4, vl)); + + s += 5 * src_stride; + + do { + vuint8mf2_t t5, t6, t7; + load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7, vl); + + // Convert to 16-bit + vint16m1_t s5 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t0, vl)); + vint16m1_t s6 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t1, vl)); + vint16m1_t s7 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t2, vl)); + vint16m1_t s8 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t3, vl)); + vint16m1_t s9 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t4, vl)); + vint16m1_t s10 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t5, vl)); + vint16m1_t s11 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t6, vl)); + vint16m1_t s12 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t7, vl)); + + // Perform convolution + vuint16m1_t d0 = convolve6_8_y_rvv(s0, s1, s2, s3, s4, s5, y_filter, + round_offset, vl); + vuint16m1_t d1 = convolve6_8_y_rvv(s1, s2, s3, s4, s5, s6, y_filter, + round_offset, vl); + vuint16m1_t d2 = convolve6_8_y_rvv(s2, s3, s4, s5, s6, s7, y_filter, + round_offset, vl); + vuint16m1_t d3 = convolve6_8_y_rvv(s3, s4, s5, s6, s7, s8, y_filter, + round_offset, vl); + vuint16m1_t d4 = convolve6_8_y_rvv(s4, s5, s6, s7, s8, s9, y_filter, + round_offset, vl); + vuint16m1_t d5 = convolve6_8_y_rvv(s5, s6, s7, s8, s9, s10, y_filter, + round_offset, vl); + vuint16m1_t d6 = convolve6_8_y_rvv(s6, s7, s8, s9, s10, s11, y_filter, + round_offset, vl); + vuint16m1_t d7 = convolve6_8_y_rvv(s7, s8, s9, s10, s11, s12, y_filter, + round_offset, vl); + + // Store result + store_u16_8x8(d, dst_stride, d0, d1, d2, d3, d4, d5, d6, d7, vl); + + // Update sliding window + s0 = __riscv_vmv_v_v_i16m1(s8, vl); + s1 = __riscv_vmv_v_v_i16m1(s9, vl); + s2 = __riscv_vmv_v_v_i16m1(s10, vl); + s3 = __riscv_vmv_v_v_i16m1(s11, vl); + s4 = __riscv_vmv_v_v_i16m1(s12, vl); + s += 8 * src_stride; + d += 8 * dst_stride; + height -= 8; + } while (height != 0); + src_ptr += vl; + dst_ptr += vl; + width -= vl; + } while (width > 0); + } +} + +static inline vuint16mf2_t convolve8_4_y_rvv( + const vint16mf2_t s0, const vint16mf2_t s1, const vint16mf2_t s2, + const vint16mf2_t s3, const vint16mf2_t s4, const vint16mf2_t s5, + const vint16mf2_t s6, const vint16mf2_t s7, const int16_t *filter, + const int16_t round_offset, size_t vl) { + // Filter values at indices 0 and 7 are 0. + vint16mf2_t sum = __riscv_vmul_vx_i16mf2(s0, filter[0], vl); + sum = __riscv_vmacc_vx_i16mf2(sum, filter[1], s1, vl); + sum = __riscv_vmacc_vx_i16mf2(sum, filter[2], s2, vl); + sum = __riscv_vmacc_vx_i16mf2(sum, filter[3], s3, vl); + sum = __riscv_vmacc_vx_i16mf2(sum, filter[4], s4, vl); + sum = __riscv_vmacc_vx_i16mf2(sum, filter[5], s5, vl); + sum = __riscv_vmacc_vx_i16mf2(sum, filter[6], s6, vl); + sum = __riscv_vmacc_vx_i16mf2(sum, filter[7], s7, vl); + + // Right shift with rounding: vrsra_n_s16(round_offset, sum, ROUND0_BITS - 1) + // vrsra_n_s16(a, b, n) = a + ((b + (1 << (n-1))) >> n) + // We halved the convolution filter values so -1 from the right shift. + vint16mf2_t res = + __riscv_vadd_vx_i16mf2(sum, 1 << ((ROUND0_BITS - 1) - 1), vl); + res = __riscv_vsra_vx_i16mf2(res, ROUND0_BITS - 1, vl); + res = __riscv_vadd_vx_i16mf2(res, round_offset, vl); + + // Reinterpret as uint16 + return __riscv_vreinterpret_v_i16mf2_u16mf2(res); +} + +static inline vuint16m1_t convolve8_8_y_rvv( + const vint16m1_t s0, const vint16m1_t s1, const vint16m1_t s2, + const vint16m1_t s3, const vint16m1_t s4, const vint16m1_t s5, + const vint16m1_t s6, const vint16m1_t s7, const int16_t *filter, + const int16_t round_offset, size_t vl) { + // Filter values at indices 0 and 7 are 0. + vint16m1_t sum = __riscv_vmul_vx_i16m1(s0, filter[0], vl); + sum = __riscv_vmacc_vx_i16m1(sum, filter[1], s1, vl); + sum = __riscv_vmacc_vx_i16m1(sum, filter[2], s2, vl); + sum = __riscv_vmacc_vx_i16m1(sum, filter[3], s3, vl); + sum = __riscv_vmacc_vx_i16m1(sum, filter[4], s4, vl); + sum = __riscv_vmacc_vx_i16m1(sum, filter[5], s5, vl); + sum = __riscv_vmacc_vx_i16m1(sum, filter[6], s6, vl); + sum = __riscv_vmacc_vx_i16m1(sum, filter[7], s7, vl); + + // Right shift with rounding: vrsra_n_s16(round_offset, sum, ROUND0_BITS - 1) + // vrsra_n_s16(a, b, n) = a + ((b + (1 << (n-1))) >> n) + // We halved the convolution filter values so -1 from the right shift. + vint16m1_t res = __riscv_vadd_vx_i16m1(sum, 1 << ((ROUND0_BITS - 1) - 1), vl); + res = __riscv_vsra_vx_i16m1(res, ROUND0_BITS - 1, vl); + res = __riscv_vadd_vx_i16m1(res, round_offset, vl); + + // Reinterpret as uint16 + return __riscv_vreinterpret_v_i16m1_u16m1(res); +} + +static inline void dist_wtd_convolve_y_8tap_dist_wtd_avg_rvv( + const uint8_t *src_ptr, int src_stride, uint8_t *dst8_ptr, + const int dst8_stride, int w, int h, const int16_t *y_filter, + ConvolveParams *conv_params) { + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + const uint16_t fwd_offset = conv_params->fwd_offset; + const uint16_t bck_offset = conv_params->bck_offset; + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + int width = w; + + if (w == 4 || h == 4) { + size_t vl = __riscv_vsetvl_e16m1(4); + do { + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + uint8_t *d_u8 = dst8_ptr; + int height = h; + + __builtin_prefetch(s + 0 * src_stride); + __builtin_prefetch(s + 1 * src_stride); + __builtin_prefetch(s + 2 * src_stride); + __builtin_prefetch(s + 3 * src_stride); + + // Load initial 7 rows of data + vuint8mf4_t t0, t1, t2, t3, t4, t5, t6; + load_u8_4x7(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, vl); + + // Convert to 16-bit + vint16mf2_t s0 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t0, vl)); + vint16mf2_t s1 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t1, vl)); + vint16mf2_t s2 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t2, vl)); + vint16mf2_t s3 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t3, vl)); + vint16mf2_t s4 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t4, vl)); + vint16mf2_t s5 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t5, vl)); + vint16mf2_t s6 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t6, vl)); + + s += 7 * src_stride; + + do { + // Load next rows of data + t0 = __riscv_vle8_v_u8mf4(s + 0 * src_stride, vl); + t1 = __riscv_vle8_v_u8mf4(s + 1 * src_stride, vl); + t2 = __riscv_vle8_v_u8mf4(s + 2 * src_stride, vl); + t3 = __riscv_vle8_v_u8mf4(s + 3 * src_stride, vl); + + // Convert to 16-bit + vint16mf2_t s7 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t0, vl)); + vint16mf2_t s8 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t1, vl)); + vint16mf2_t s9 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t2, vl)); + vint16mf2_t s10 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t3, vl)); + + // Perform convolution + vuint16mf2_t d0 = convolve8_4_y_rvv(s0, s1, s2, s3, s4, s5, s6, s7, + y_filter, round_offset, vl); + vuint16mf2_t d1 = convolve8_4_y_rvv(s1, s2, s3, s4, s5, s6, s7, s8, + y_filter, round_offset, vl); + vuint16mf2_t d2 = convolve8_4_y_rvv(s2, s3, s4, s5, s6, s7, s8, s9, + y_filter, round_offset, vl); + vuint16mf2_t d3 = convolve8_4_y_rvv(s3, s4, s5, s6, s7, s8, s9, s10, + y_filter, round_offset, vl); + + __builtin_prefetch(d + 0 * dst_stride); + __builtin_prefetch(d + 1 * dst_stride); + __builtin_prefetch(d + 2 * dst_stride); + __builtin_prefetch(d + 3 * dst_stride); + + __builtin_prefetch(d_u8 + 0 * dst8_stride); + __builtin_prefetch(d_u8 + 1 * dst8_stride); + __builtin_prefetch(d_u8 + 2 * dst8_stride); + __builtin_prefetch(d_u8 + 3 * dst8_stride); + + vuint16mf2_t dd0, dd1, dd2, dd3; + load_u16_4x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3, vl); + + vuint8mf4_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_dist_wtd_avg_4x4_rvv(dd0, dd1, dd2, dd3, d0, d1, d2, d3, + fwd_offset, bck_offset, round_offset, + &d0_u8, &d1_u8, &d2_u8, &d3_u8, vl); + + // Store result + __riscv_vse8_v_u8mf4(d_u8 + 0 * dst8_stride, d0_u8, vl); + __riscv_vse8_v_u8mf4(d_u8 + 1 * dst8_stride, d1_u8, vl); + __riscv_vse8_v_u8mf4(d_u8 + 2 * dst8_stride, d2_u8, vl); + __riscv_vse8_v_u8mf4(d_u8 + 3 * dst8_stride, d3_u8, vl); + + // Update sliding window + s0 = __riscv_vmv_v_v_i16mf2(s4, vl); + s1 = __riscv_vmv_v_v_i16mf2(s5, vl); + s2 = __riscv_vmv_v_v_i16mf2(s6, vl); + s3 = __riscv_vmv_v_v_i16mf2(s7, vl); + s4 = __riscv_vmv_v_v_i16mf2(s8, vl); + s5 = __riscv_vmv_v_v_i16mf2(s9, vl); + s6 = __riscv_vmv_v_v_i16mf2(s10, vl); + s += 4 * src_stride; + d += 4 * dst_stride; + d_u8 += 4 * dst8_stride; + height -= 4; + } while (height != 0); + src_ptr += 4; + dst_ptr += 4; + dst8_ptr += 4; + width -= 4; + } while (width != 0); + } else { + size_t vl = __riscv_vsetvl_e16m1(w); + do { + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + uint8_t *d_u8 = dst8_ptr; + int height = h; + + __builtin_prefetch(s + 0 * src_stride); + __builtin_prefetch(s + 1 * src_stride); + __builtin_prefetch(s + 2 * src_stride); + __builtin_prefetch(s + 3 * src_stride); + __builtin_prefetch(s + 4 * src_stride); + __builtin_prefetch(s + 5 * src_stride); + __builtin_prefetch(s + 6 * src_stride); + + // Load initial 7 rows of data + vuint8mf2_t t0, t1, t2, t3, t4, t5, t6; + load_u8_8x7(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, vl); + + // Convert to 16-bit + vint16m1_t s0 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t0, vl)); + vint16m1_t s1 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t1, vl)); + vint16m1_t s2 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t2, vl)); + vint16m1_t s3 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t3, vl)); + vint16m1_t s4 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t4, vl)); + vint16m1_t s5 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t5, vl)); + vint16m1_t s6 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t6, vl)); + + s += 7 * src_stride; + + do { + vuint8mf2_t t7; + load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7, vl); + + // Convert to 16-bit + vint16m1_t s7 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t0, vl)); + vint16m1_t s8 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t1, vl)); + vint16m1_t s9 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t2, vl)); + vint16m1_t s10 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t3, vl)); + vint16m1_t s11 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t4, vl)); + vint16m1_t s12 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t5, vl)); + vint16m1_t s13 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t6, vl)); + vint16m1_t s14 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t7, vl)); + + __builtin_prefetch(d + 0 * dst_stride); + __builtin_prefetch(d + 1 * dst_stride); + __builtin_prefetch(d + 2 * dst_stride); + __builtin_prefetch(d + 3 * dst_stride); + + // Perform convolution + vuint16m1_t d0 = convolve8_8_y_rvv(s0, s1, s2, s3, s4, s5, s6, s7, + y_filter, round_offset, vl); + vuint16m1_t d1 = convolve8_8_y_rvv(s1, s2, s3, s4, s5, s6, s7, s8, + y_filter, round_offset, vl); + vuint16m1_t d2 = convolve8_8_y_rvv(s2, s3, s4, s5, s6, s7, s8, s9, + y_filter, round_offset, vl); + vuint16m1_t d3 = convolve8_8_y_rvv(s3, s4, s5, s6, s7, s8, s9, s10, + y_filter, round_offset, vl); + vuint16m1_t d4 = convolve8_8_y_rvv(s4, s5, s6, s7, s8, s9, s10, s11, + y_filter, round_offset, vl); + vuint16m1_t d5 = convolve8_8_y_rvv(s5, s6, s7, s8, s9, s10, s11, s12, + y_filter, round_offset, vl); + vuint16m1_t d6 = convolve8_8_y_rvv(s6, s7, s8, s9, s10, s11, s12, s13, + y_filter, round_offset, vl); + vuint16m1_t d7 = convolve8_8_y_rvv(s7, s8, s9, s10, s11, s12, s13, s14, + y_filter, round_offset, vl); + + __builtin_prefetch(d_u8 + 0 * dst8_stride); + __builtin_prefetch(d_u8 + 1 * dst8_stride); + __builtin_prefetch(d_u8 + 2 * dst8_stride); + __builtin_prefetch(d_u8 + 3 * dst8_stride); + + vuint16m1_t dd0, dd1, dd2, dd3; + load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3, vl); + + vuint8mf2_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_dist_wtd_avg_8x4_rvv(dd0, dd1, dd2, dd3, d0, d1, d2, d3, + fwd_offset, bck_offset, round_offset, + &d0_u8, &d1_u8, &d2_u8, &d3_u8, vl); + store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8, vl); + d_u8 += 4 * dst8_stride; + + vuint16m1_t dd4, dd5, dd6, dd7; + load_u16_8x4(d + 4 * dst_stride, dst_stride, &dd4, &dd5, &dd6, &dd7, + vl); + + vuint8mf2_t d4_u8, d5_u8, d6_u8, d7_u8; + compute_dist_wtd_avg_8x4_rvv(dd4, dd5, dd6, dd7, d4, d5, d6, d7, + fwd_offset, bck_offset, round_offset, + &d4_u8, &d5_u8, &d6_u8, &d7_u8, vl); + store_u8_8x4(d_u8, dst8_stride, d4_u8, d5_u8, d6_u8, d7_u8, vl); + d_u8 += 4 * dst8_stride; + + // Update sliding window + s0 = __riscv_vmv_v_v_i16m1(s8, vl); + s1 = __riscv_vmv_v_v_i16m1(s9, vl); + s2 = __riscv_vmv_v_v_i16m1(s10, vl); + s3 = __riscv_vmv_v_v_i16m1(s11, vl); + s4 = __riscv_vmv_v_v_i16m1(s12, vl); + s5 = __riscv_vmv_v_v_i16m1(s13, vl); + s6 = __riscv_vmv_v_v_i16m1(s14, vl); + s += 8 * src_stride; + d += 8 * dst_stride; + height -= 8; + } while (height != 0); + src_ptr += vl; + dst_ptr += vl; + dst8_ptr += vl; + width -= vl; + } while (width > 0); + } +} + +static inline void dist_wtd_convolve_y_8tap_avg_rvv( + const uint8_t *src_ptr, int src_stride, uint8_t *dst8_ptr, + const int dst8_stride, int w, int h, const int16_t *y_filter, + ConvolveParams *conv_params) { + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + int width = w; + + if (w == 4 || h == 4) { + size_t vl = __riscv_vsetvl_e16m1(4); + do { + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + uint8_t *d_u8 = dst8_ptr; + int height = h; + + __builtin_prefetch(s + 0 * src_stride); + __builtin_prefetch(s + 1 * src_stride); + __builtin_prefetch(s + 2 * src_stride); + __builtin_prefetch(s + 3 * src_stride); + + // Load initial 7 rows of data + vuint8mf4_t t0, t1, t2, t3, t4, t5, t6; + load_u8_4x7(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, vl); + + // Convert to 16-bit + vint16mf2_t s0 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t0, vl)); + vint16mf2_t s1 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t1, vl)); + vint16mf2_t s2 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t2, vl)); + vint16mf2_t s3 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t3, vl)); + vint16mf2_t s4 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t4, vl)); + vint16mf2_t s5 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t5, vl)); + vint16mf2_t s6 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t6, vl)); + + s += 7 * src_stride; + + do { + // Load next rows of data + t0 = __riscv_vle8_v_u8mf4(s + 0 * src_stride, vl); + t1 = __riscv_vle8_v_u8mf4(s + 1 * src_stride, vl); + t2 = __riscv_vle8_v_u8mf4(s + 2 * src_stride, vl); + t3 = __riscv_vle8_v_u8mf4(s + 3 * src_stride, vl); + + // Convert to 16-bit + vint16mf2_t s7 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t0, vl)); + vint16mf2_t s8 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t1, vl)); + vint16mf2_t s9 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t2, vl)); + vint16mf2_t s10 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t3, vl)); + + // Perform convolution + vuint16mf2_t d0 = convolve8_4_y_rvv(s0, s1, s2, s3, s4, s5, s6, s7, + y_filter, round_offset, vl); + vuint16mf2_t d1 = convolve8_4_y_rvv(s1, s2, s3, s4, s5, s6, s7, s8, + y_filter, round_offset, vl); + vuint16mf2_t d2 = convolve8_4_y_rvv(s2, s3, s4, s5, s6, s7, s8, s9, + y_filter, round_offset, vl); + vuint16mf2_t d3 = convolve8_4_y_rvv(s3, s4, s5, s6, s7, s8, s9, s10, + y_filter, round_offset, vl); + + __builtin_prefetch(d + 0 * dst_stride); + __builtin_prefetch(d + 1 * dst_stride); + __builtin_prefetch(d + 2 * dst_stride); + __builtin_prefetch(d + 3 * dst_stride); + + __builtin_prefetch(d_u8 + 0 * dst8_stride); + __builtin_prefetch(d_u8 + 1 * dst8_stride); + __builtin_prefetch(d_u8 + 2 * dst8_stride); + __builtin_prefetch(d_u8 + 3 * dst8_stride); + + vuint16mf2_t dd0, dd1, dd2, dd3; + load_u16_4x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3, vl); + + vuint8mf4_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_basic_avg_4x4_rvv(dd0, dd1, dd2, dd3, d0, d1, d2, d3, + round_offset, &d0_u8, &d1_u8, &d2_u8, &d3_u8, + vl); + + // Store result + __riscv_vse8_v_u8mf4(d_u8 + 0 * dst8_stride, d0_u8, vl); + __riscv_vse8_v_u8mf4(d_u8 + 1 * dst8_stride, d1_u8, vl); + __riscv_vse8_v_u8mf4(d_u8 + 2 * dst8_stride, d2_u8, vl); + __riscv_vse8_v_u8mf4(d_u8 + 3 * dst8_stride, d3_u8, vl); + + // Update sliding window + s0 = __riscv_vmv_v_v_i16mf2(s4, vl); + s1 = __riscv_vmv_v_v_i16mf2(s5, vl); + s2 = __riscv_vmv_v_v_i16mf2(s6, vl); + s3 = __riscv_vmv_v_v_i16mf2(s7, vl); + s4 = __riscv_vmv_v_v_i16mf2(s8, vl); + s5 = __riscv_vmv_v_v_i16mf2(s9, vl); + s6 = __riscv_vmv_v_v_i16mf2(s10, vl); + s += 4 * src_stride; + d += 4 * dst_stride; + d_u8 += 4 * dst8_stride; + height -= 4; + } while (height != 0); + src_ptr += 4; + dst_ptr += 4; + dst8_ptr += 4; + width -= 4; + } while (width != 0); + } else { + size_t vl = __riscv_vsetvl_e16m1(w); + do { + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + uint8_t *d_u8 = dst8_ptr; + int height = h; + + __builtin_prefetch(s + 0 * src_stride); + __builtin_prefetch(s + 1 * src_stride); + __builtin_prefetch(s + 2 * src_stride); + __builtin_prefetch(s + 3 * src_stride); + __builtin_prefetch(s + 4 * src_stride); + __builtin_prefetch(s + 5 * src_stride); + __builtin_prefetch(s + 6 * src_stride); + + // Load initial 7 rows of data + vuint8mf2_t t0, t1, t2, t3, t4, t5, t6; + load_u8_8x7(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, vl); + + // Convert to 16-bit + vint16m1_t s0 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t0, vl)); + vint16m1_t s1 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t1, vl)); + vint16m1_t s2 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t2, vl)); + vint16m1_t s3 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t3, vl)); + vint16m1_t s4 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t4, vl)); + vint16m1_t s5 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t5, vl)); + vint16m1_t s6 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t6, vl)); + + s += 7 * src_stride; + + do { + vuint8mf2_t t7; + load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7, vl); + + // Convert to 16-bit + vint16m1_t s7 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t0, vl)); + vint16m1_t s8 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t1, vl)); + vint16m1_t s9 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t2, vl)); + vint16m1_t s10 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t3, vl)); + vint16m1_t s11 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t4, vl)); + vint16m1_t s12 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t5, vl)); + vint16m1_t s13 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t6, vl)); + vint16m1_t s14 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t7, vl)); + + __builtin_prefetch(d + 0 * dst_stride); + __builtin_prefetch(d + 1 * dst_stride); + __builtin_prefetch(d + 2 * dst_stride); + __builtin_prefetch(d + 3 * dst_stride); + + // Perform convolution + vuint16m1_t d0 = convolve8_8_y_rvv(s0, s1, s2, s3, s4, s5, s6, s7, + y_filter, round_offset, vl); + vuint16m1_t d1 = convolve8_8_y_rvv(s1, s2, s3, s4, s5, s6, s7, s8, + y_filter, round_offset, vl); + vuint16m1_t d2 = convolve8_8_y_rvv(s2, s3, s4, s5, s6, s7, s8, s9, + y_filter, round_offset, vl); + vuint16m1_t d3 = convolve8_8_y_rvv(s3, s4, s5, s6, s7, s8, s9, s10, + y_filter, round_offset, vl); + vuint16m1_t d4 = convolve8_8_y_rvv(s4, s5, s6, s7, s8, s9, s10, s11, + y_filter, round_offset, vl); + vuint16m1_t d5 = convolve8_8_y_rvv(s5, s6, s7, s8, s9, s10, s11, s12, + y_filter, round_offset, vl); + vuint16m1_t d6 = convolve8_8_y_rvv(s6, s7, s8, s9, s10, s11, s12, s13, + y_filter, round_offset, vl); + vuint16m1_t d7 = convolve8_8_y_rvv(s7, s8, s9, s10, s11, s12, s13, s14, + y_filter, round_offset, vl); + + __builtin_prefetch(d_u8 + 0 * dst8_stride); + __builtin_prefetch(d_u8 + 1 * dst8_stride); + __builtin_prefetch(d_u8 + 2 * dst8_stride); + __builtin_prefetch(d_u8 + 3 * dst8_stride); + + vuint16m1_t dd0, dd1, dd2, dd3; + load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3, vl); + + vuint8mf2_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_basic_avg_8x4_rvv(dd0, dd1, dd2, dd3, d0, d1, d2, d3, + round_offset, &d0_u8, &d1_u8, &d2_u8, &d3_u8, + vl); + store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8, vl); + d_u8 += 4 * dst8_stride; + + vuint16m1_t dd4, dd5, dd6, dd7; + load_u16_8x4(d + 4 * dst_stride, dst_stride, &dd4, &dd5, &dd6, &dd7, + vl); + + vuint8mf2_t d4_u8, d5_u8, d6_u8, d7_u8; + compute_basic_avg_8x4_rvv(dd4, dd5, dd6, dd7, d4, d5, d6, d7, + round_offset, &d4_u8, &d5_u8, &d6_u8, &d7_u8, + vl); + store_u8_8x4(d_u8, dst8_stride, d4_u8, d5_u8, d6_u8, d7_u8, vl); + d_u8 += 4 * dst8_stride; + + // Update sliding window + s0 = __riscv_vmv_v_v_i16m1(s8, vl); + s1 = __riscv_vmv_v_v_i16m1(s9, vl); + s2 = __riscv_vmv_v_v_i16m1(s10, vl); + s3 = __riscv_vmv_v_v_i16m1(s11, vl); + s4 = __riscv_vmv_v_v_i16m1(s12, vl); + s5 = __riscv_vmv_v_v_i16m1(s13, vl); + s6 = __riscv_vmv_v_v_i16m1(s14, vl); + s += 8 * src_stride; + d += 8 * dst_stride; + height -= 8; + } while (height != 0); + src_ptr += vl; + dst_ptr += vl; + dst8_ptr += vl; + width -= vl; + } while (width > 0); + } +} + +static inline void dist_wtd_convolve_y_8tap_rvv(const uint8_t *src_ptr, + int src_stride, int w, int h, + const int16_t *y_filter, + ConvolveParams *conv_params) { + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + int width = w; + + if (w == 4 || h == 4) { + size_t vl = __riscv_vsetvl_e16m1(4); + do { + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + int height = h; + + __builtin_prefetch(s + 0 * src_stride); + __builtin_prefetch(s + 1 * src_stride); + __builtin_prefetch(s + 2 * src_stride); + __builtin_prefetch(s + 3 * src_stride); + + // Load initial 7 rows of data + vuint8mf4_t t0, t1, t2, t3, t4, t5, t6; + load_u8_4x7(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, vl); + + // Convert to 16-bit + vint16mf2_t s0 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t0, vl)); + vint16mf2_t s1 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t1, vl)); + vint16mf2_t s2 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t2, vl)); + vint16mf2_t s3 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t3, vl)); + vint16mf2_t s4 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t4, vl)); + vint16mf2_t s5 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t5, vl)); + vint16mf2_t s6 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t6, vl)); + + s += 7 * src_stride; + + do { + // Load next rows of data + t0 = __riscv_vle8_v_u8mf4(s + 0 * src_stride, vl); + t1 = __riscv_vle8_v_u8mf4(s + 1 * src_stride, vl); + t2 = __riscv_vle8_v_u8mf4(s + 2 * src_stride, vl); + t3 = __riscv_vle8_v_u8mf4(s + 3 * src_stride, vl); + + // Convert to 16-bit + vint16mf2_t s7 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t0, vl)); + vint16mf2_t s8 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t1, vl)); + vint16mf2_t s9 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t2, vl)); + vint16mf2_t s10 = __riscv_vreinterpret_v_u16mf2_i16mf2( + __riscv_vzext_vf2_u16mf2(t3, vl)); + + // Perform convolution + vuint16mf2_t d0 = convolve8_4_y_rvv(s0, s1, s2, s3, s4, s5, s6, s7, + y_filter, round_offset, vl); + vuint16mf2_t d1 = convolve8_4_y_rvv(s1, s2, s3, s4, s5, s6, s7, s8, + y_filter, round_offset, vl); + vuint16mf2_t d2 = convolve8_4_y_rvv(s2, s3, s4, s5, s6, s7, s8, s9, + y_filter, round_offset, vl); + vuint16mf2_t d3 = convolve8_4_y_rvv(s3, s4, s5, s6, s7, s8, s9, s10, + y_filter, round_offset, vl); + + // Store result + store_u16_4x4(d, dst_stride, d0, d1, d2, d3, vl); + + // Update sliding window + s0 = __riscv_vmv_v_v_i16mf2(s4, vl); + s1 = __riscv_vmv_v_v_i16mf2(s5, vl); + s2 = __riscv_vmv_v_v_i16mf2(s6, vl); + s3 = __riscv_vmv_v_v_i16mf2(s7, vl); + s4 = __riscv_vmv_v_v_i16mf2(s8, vl); + s5 = __riscv_vmv_v_v_i16mf2(s9, vl); + s6 = __riscv_vmv_v_v_i16mf2(s10, vl); + s += 4 * src_stride; + d += 4 * dst_stride; + height -= 4; + } while (height != 0); + src_ptr += 4; + dst_ptr += 4; + width -= 4; + } while (width != 0); + } else { + size_t vl = __riscv_vsetvl_e16m1(w); + do { + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + int height = h; + + __builtin_prefetch(s + 0 * src_stride); + __builtin_prefetch(s + 1 * src_stride); + __builtin_prefetch(s + 2 * src_stride); + __builtin_prefetch(s + 3 * src_stride); + __builtin_prefetch(s + 4 * src_stride); + __builtin_prefetch(s + 5 * src_stride); + __builtin_prefetch(s + 6 * src_stride); + + // Load initial 7 rows of data + vuint8mf2_t t0, t1, t2, t3, t4, t5, t6; + load_u8_8x7(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, vl); + + // Convert to 16-bit + vint16m1_t s0 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t0, vl)); + vint16m1_t s1 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t1, vl)); + vint16m1_t s2 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t2, vl)); + vint16m1_t s3 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t3, vl)); + vint16m1_t s4 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t4, vl)); + vint16m1_t s5 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t5, vl)); + vint16m1_t s6 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t6, vl)); + + s += 7 * src_stride; + + do { + vuint8mf2_t t7; + load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7, vl); + + // Convert to 16-bit + vint16m1_t s7 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t0, vl)); + vint16m1_t s8 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t1, vl)); + vint16m1_t s9 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t2, vl)); + vint16m1_t s10 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t3, vl)); + vint16m1_t s11 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t4, vl)); + vint16m1_t s12 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t5, vl)); + vint16m1_t s13 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t6, vl)); + vint16m1_t s14 = + __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(t7, vl)); + + __builtin_prefetch(d + 0 * dst_stride); + __builtin_prefetch(d + 1 * dst_stride); + __builtin_prefetch(d + 2 * dst_stride); + __builtin_prefetch(d + 3 * dst_stride); + + // Perform convolution + vuint16m1_t d0 = convolve8_8_y_rvv(s0, s1, s2, s3, s4, s5, s6, s7, + y_filter, round_offset, vl); + vuint16m1_t d1 = convolve8_8_y_rvv(s1, s2, s3, s4, s5, s6, s7, s8, + y_filter, round_offset, vl); + vuint16m1_t d2 = convolve8_8_y_rvv(s2, s3, s4, s5, s6, s7, s8, s9, + y_filter, round_offset, vl); + vuint16m1_t d3 = convolve8_8_y_rvv(s3, s4, s5, s6, s7, s8, s9, s10, + y_filter, round_offset, vl); + vuint16m1_t d4 = convolve8_8_y_rvv(s4, s5, s6, s7, s8, s9, s10, s11, + y_filter, round_offset, vl); + vuint16m1_t d5 = convolve8_8_y_rvv(s5, s6, s7, s8, s9, s10, s11, s12, + y_filter, round_offset, vl); + vuint16m1_t d6 = convolve8_8_y_rvv(s6, s7, s8, s9, s10, s11, s12, s13, + y_filter, round_offset, vl); + vuint16m1_t d7 = convolve8_8_y_rvv(s7, s8, s9, s10, s11, s12, s13, s14, + y_filter, round_offset, vl); + + // Store result + store_u16_8x8(d, dst_stride, d0, d1, d2, d3, d4, d5, d6, d7, vl); + + // Update sliding window + s0 = __riscv_vmv_v_v_i16m1(s8, vl); + s1 = __riscv_vmv_v_v_i16m1(s9, vl); + s2 = __riscv_vmv_v_v_i16m1(s10, vl); + s3 = __riscv_vmv_v_v_i16m1(s11, vl); + s4 = __riscv_vmv_v_v_i16m1(s12, vl); + s5 = __riscv_vmv_v_v_i16m1(s13, vl); + s6 = __riscv_vmv_v_v_i16m1(s14, vl); + s += 8 * src_stride; + d += 8 * dst_stride; + height -= 8; + } while (height != 0); + src_ptr += vl; + dst_ptr += vl; + width -= vl; + } while (width > 0); + } +} +void av1_dist_wtd_convolve_y_rvv(const uint8_t *src, int src_stride, + uint8_t *dst8, int dst8_stride, int w, int h, + const InterpFilterParams *filter_params_y, + const int subpel_y_qn, + ConvolveParams *conv_params) { + assert(w % 4 == 0); + assert(h % 4 == 0); + + // Vertical filter. + const int16_t *y_filter_ptr = av1_get_interp_filter_subpel_kernel( + filter_params_y, subpel_y_qn & SUBPEL_MASK); + // Filter values are even, so downshift by 1 to reduce intermediate + // precision requirements. + int16_t y_filter[8]; + for (int i = 0; i < 8; i++) y_filter[i] = *y_filter_ptr++ >> 1; + + const int vert_offset = filter_params_y->taps / 2 - 1; + const uint8_t *src_ptr = src - (vert_offset * src_stride); + + if (get_filter_tap(filter_params_y, subpel_y_qn) <= 6) { + if (conv_params->do_average) { + if (UNLIKELY(conv_params->use_dist_wtd_comp_avg)) { + dist_wtd_convolve_y_6tap_dist_wtd_avg_rvv(src_ptr + src_stride, + src_stride, dst8, dst8_stride, + w, h, y_filter, conv_params); + } else { + dist_wtd_convolve_y_6tap_avg_rvv(src_ptr + src_stride, src_stride, dst8, + dst8_stride, w, h, y_filter, + conv_params); + } + } else { + dist_wtd_convolve_y_6tap_rvv(src_ptr + src_stride, src_stride, w, h, + y_filter, conv_params); + } + } else { + if (conv_params->do_average) { + if (UNLIKELY(conv_params->use_dist_wtd_comp_avg)) { + dist_wtd_convolve_y_8tap_dist_wtd_avg_rvv(src_ptr, src_stride, dst8, + dst8_stride, w, h, y_filter, + conv_params); + } else { + dist_wtd_convolve_y_8tap_avg_rvv(src_ptr, src_stride, dst8, dst8_stride, + w, h, y_filter, conv_params); + } + } else { + dist_wtd_convolve_y_8tap_rvv(src_ptr, src_stride, w, h, y_filter, + conv_params); + } + } +}
diff --git a/av1/common/riscv/compound_convolve_rvv.h b/av1/common/riscv/compound_convolve_rvv.h new file mode 100644 index 0000000..61e71fc --- /dev/null +++ b/av1/common/riscv/compound_convolve_rvv.h
@@ -0,0 +1,1140 @@ +/* + * Copyright (c) 2026, Alliance for Open Media. All rights reserved. + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_AV1_COMMON_RISCV_COMPOUND_CONVOLVE_RVV_H_ +#define AOM_AV1_COMMON_RISCV_COMPOUND_CONVOLVE_RVV_H_ + +#include <riscv_vector.h> + +#include "aom_dsp/riscv/mem_rvv.h" +#include "av1/common/convolve.h" +#include "av1/common/enums.h" +#include "av1/common/filter.h" + +static inline void compute_dist_wtd_avg_4x1_rvv(const vuint16m1_t dd0, + const vuint16m1_t d0, + const uint16_t fwd_offset, + const uint16_t bck_offset, + const int16_t round_offset, + vuint8mf2_t *d0_u8, size_t vl) { + // blend0 = dd0 * fwd_offset + d0 * bck_offset (32-bit accumulation) + vuint32m2_t vblend0 = __riscv_vwmulu_vx_u32m2(dd0, fwd_offset, vl); + vblend0 = __riscv_vwmaccu_vx_u32m2(vblend0, bck_offset, d0, vl); + + // avg0 = vblend0 >> DIST_PRECISION_BITS (narrow to 16-bit) + vuint16m1_t vavg0 = __riscv_vnsrl_wx_u16m1(vblend0, DIST_PRECISION_BITS, vl); + + // dst0 = (int16_t)vavg0 - round_offset + vint16m1_t vdst0 = __riscv_vsub_vx_i16m1( + __riscv_vreinterpret_v_u16m1_i16m1(vavg0), round_offset, vl); + + // Pack to 8 lanes for rounding and narrowing + vint16m1_t vec_zero = __riscv_vmv_s_x_i16m1(0, vl); + vdst0 = + __riscv_vslideup_vx_i16m1(vdst0, vec_zero, 4, 8); // upper 4 lanes = 0 + vuint16m1_t d0_clip = + __riscv_vreinterpret_v_i16m1_u16m1(__riscv_vmax_vx_i16m1(vdst0, 0, vl)); +#if __riscv_v_intrinsic == 11000 + *d0_u8 = __riscv_vnclipu_wx_u8mf2(d0_clip, FILTER_BITS - ROUND0_BITS, vl); +#elif __riscv_v_intrinsic >= 12000 + *d0_u8 = __riscv_vnclipu_wx_u8mf2(d0_clip, FILTER_BITS - ROUND0_BITS, + __RISCV_VXRM_RNU, vl); +#endif +} + +static inline void compute_basic_avg_4x1_rvv(const vuint16m1_t dd0, + const vuint16m1_t d0, + const int16_t round_offset, + vuint8mf2_t *d0_u8, size_t vl) { + // avg0 = (dd0 + d0) >> 1 (vector halving add, no rounding) +#if __riscv_v_intrinsic == 11000 + vuint16m1_t vavg0 = + __riscv_vnsrl_wx_u16m1(__riscv_vwaddu_vv_u32m2(dd0, d0, vl), 1, vl); +#elif __riscv_v_intrinsic >= 12000 + vuint16m1_t vavg0 = __riscv_vaaddu_vv_u16m1(dd0, d0, __RISCV_VXRM_RDN, vl); +#endif + + // dst0 = (int16_t)vavg0 - round_offset + vint16m1_t vdst0 = __riscv_vsub_vx_i16m1( + __riscv_vreinterpret_v_u16m1_i16m1(vavg0), round_offset, vl); + + // Pack to 8 lanes for rounding and narrowing + vint16m1_t vec_zero = __riscv_vmv_s_x_i16m1(0, vl); + vdst0 = + __riscv_vslideup_vx_i16m1(vdst0, vec_zero, 4, 8); // upper 4 lanes = 0 + vuint16m1_t d0_clip = + __riscv_vreinterpret_v_i16m1_u16m1(__riscv_vmax_vx_i16m1(vdst0, 0, vl)); +#if __riscv_v_intrinsic == 11000 + *d0_u8 = __riscv_vnclipu_wx_u8mf2(d0_clip, FILTER_BITS - ROUND0_BITS, vl); +#elif __riscv_v_intrinsic >= 12000 + *d0_u8 = __riscv_vnclipu_wx_u8mf2(d0_clip, FILTER_BITS - ROUND0_BITS, + __RISCV_VXRM_RNU, vl); +#endif +} + +static inline void compute_dist_wtd_avg_8x1_rvv(const vuint16m1_t dd0, + const vuint16m1_t d0, + const uint16_t fwd_offset, + const uint16_t bck_offset, + const int16_t round_offset, + vuint8mf2_t *d0_u8, size_t vl) { + // blend0 = dd0 * fwd_offset + d0 * bck_offset (32-bit accumulation) + vuint32m2_t vblend0 = __riscv_vwmulu_vx_u32m2(dd0, fwd_offset, vl); + vblend0 = __riscv_vwmaccu_vx_u32m2(vblend0, bck_offset, d0, vl); + + // avg0 = vblend0 >> DIST_PRECISION_BITS (narrow to 16-bit) + vuint16m1_t vavg0 = __riscv_vnsrl_wx_u16m1(vblend0, DIST_PRECISION_BITS, vl); + + // dst0 = (int16_t)vavg0 - round_offset + vint16m1_t vdst0 = __riscv_vsub_vx_i16m1( + __riscv_vreinterpret_v_u16m1_i16m1(vavg0), round_offset, vl); + + // Rounding shift right and narrow to u8 + vuint16m1_t d0_clip = + __riscv_vreinterpret_v_i16m1_u16m1(__riscv_vmax_vx_i16m1(vdst0, 0, vl)); +#if __riscv_v_intrinsic == 11000 + *d0_u8 = __riscv_vnclipu_wx_u8mf2(d0_clip, FILTER_BITS - ROUND0_BITS, vl); +#elif __riscv_v_intrinsic >= 12000 + *d0_u8 = __riscv_vnclipu_wx_u8mf2(d0_clip, FILTER_BITS - ROUND0_BITS, + __RISCV_VXRM_RNU, vl); +#endif +} + +static inline void compute_basic_avg_8x1_rvv(const vuint16m1_t dd0, + const vuint16m1_t d0, + const int16_t round_offset, + vuint8mf2_t *d0_u8, size_t vl) { + // avg0 = (dd0 + d0) >> 1 (vector halving add, no rounding) +#if __riscv_v_intrinsic == 11000 + vuint16m1_t vavg0 = + __riscv_vnsrl_wx_u16m1(__riscv_vwaddu_vv_u32m2(dd0, d0, vl), 1, vl); +#elif __riscv_v_intrinsic >= 12000 + vuint16m1_t vavg0 = __riscv_vaaddu_vv_u16m1(dd0, d0, __RISCV_VXRM_RDN, vl); +#endif + + // dst0 = (int16_t)vavg0 - round_offset + vint16m1_t vdst0 = __riscv_vsub_vx_i16m1( + __riscv_vreinterpret_v_u16m1_i16m1(vavg0), round_offset, vl); + + // Rounding shift right and narrow to u8 + vuint16m1_t d0_clip = + __riscv_vreinterpret_v_i16m1_u16m1(__riscv_vmax_vx_i16m1(vdst0, 0, vl)); +#if __riscv_v_intrinsic == 11000 + *d0_u8 = __riscv_vnclipu_wx_u8mf2(d0_clip, FILTER_BITS - ROUND0_BITS, vl); +#elif __riscv_v_intrinsic >= 12000 + *d0_u8 = __riscv_vnclipu_wx_u8mf2(d0_clip, FILTER_BITS - ROUND0_BITS, + __RISCV_VXRM_RNU, vl); +#endif +} + +static inline void compute_dist_wtd_avg_4x4_rvv( + const vuint16mf2_t dd0, const vuint16mf2_t dd1, const vuint16mf2_t dd2, + const vuint16mf2_t dd3, const vuint16mf2_t d0, const vuint16mf2_t d1, + const vuint16mf2_t d2, const vuint16mf2_t d3, const uint16_t fwd_offset, + const uint16_t bck_offset, const int16_t round_offset, vuint8mf4_t *d0_u8, + vuint8mf4_t *d1_u8, vuint8mf4_t *d2_u8, vuint8mf4_t *d3_u8, size_t vl) { + // blend = dd* * fwd_offset + d* * bck_offset (32-bit accumulation) + vuint32m1_t vblend0 = __riscv_vwmulu_vx_u32m1(dd0, fwd_offset, vl); + vblend0 = __riscv_vwmaccu_vx_u32m1(vblend0, bck_offset, d0, vl); + + vuint32m1_t vblend1 = __riscv_vwmulu_vx_u32m1(dd1, fwd_offset, vl); + vblend1 = __riscv_vwmaccu_vx_u32m1(vblend1, bck_offset, d1, vl); + + vuint32m1_t vblend2 = __riscv_vwmulu_vx_u32m1(dd2, fwd_offset, vl); + vblend2 = __riscv_vwmaccu_vx_u32m1(vblend2, bck_offset, d2, vl); + + vuint32m1_t vblend3 = __riscv_vwmulu_vx_u32m1(dd3, fwd_offset, vl); + vblend3 = __riscv_vwmaccu_vx_u32m1(vblend3, bck_offset, d3, vl); + + // avg = blend >> DIST_PRECISION_BITS (narrow to 16-bit) + vuint16mf2_t vavg0 = + __riscv_vnsrl_wx_u16mf2(vblend0, DIST_PRECISION_BITS, vl); + vuint16mf2_t vavg1 = + __riscv_vnsrl_wx_u16mf2(vblend1, DIST_PRECISION_BITS, vl); + vuint16mf2_t vavg2 = + __riscv_vnsrl_wx_u16mf2(vblend2, DIST_PRECISION_BITS, vl); + vuint16mf2_t vavg3 = + __riscv_vnsrl_wx_u16mf2(vblend3, DIST_PRECISION_BITS, vl); + + // dst = vavg - round_offset + vint16mf2_t vdst0 = __riscv_vsub_vx_i16mf2( + __riscv_vreinterpret_v_u16mf2_i16mf2(vavg0), round_offset, vl); + vint16mf2_t vdst1 = __riscv_vsub_vx_i16mf2( + __riscv_vreinterpret_v_u16mf2_i16mf2(vavg1), round_offset, vl); + vint16mf2_t vdst2 = __riscv_vsub_vx_i16mf2( + __riscv_vreinterpret_v_u16mf2_i16mf2(vavg2), round_offset, vl); + vint16mf2_t vdst3 = __riscv_vsub_vx_i16mf2( + __riscv_vreinterpret_v_u16mf2_i16mf2(vavg3), round_offset, vl); + + // Saturating Rounded Shift right and narrow to u8 + const int shift = FILTER_BITS - ROUND0_BITS; + vuint16mf2_t d0_clip = __riscv_vreinterpret_v_i16mf2_u16mf2( + __riscv_vmax_vx_i16mf2(vdst0, 0, vl)); + vuint16mf2_t d1_clip = __riscv_vreinterpret_v_i16mf2_u16mf2( + __riscv_vmax_vx_i16mf2(vdst1, 0, vl)); + vuint16mf2_t d2_clip = __riscv_vreinterpret_v_i16mf2_u16mf2( + __riscv_vmax_vx_i16mf2(vdst2, 0, vl)); + vuint16mf2_t d3_clip = __riscv_vreinterpret_v_i16mf2_u16mf2( + __riscv_vmax_vx_i16mf2(vdst3, 0, vl)); +#if __riscv_v_intrinsic == 11000 + *d0_u8 = __riscv_vnclipu_wx_u8mf4(d0_clip, shift, vl); + *d1_u8 = __riscv_vnclipu_wx_u8mf4(d1_clip, shift, vl); + *d2_u8 = __riscv_vnclipu_wx_u8mf4(d2_clip, shift, vl); + *d3_u8 = __riscv_vnclipu_wx_u8mf4(d3_clip, shift, vl); +#elif __riscv_v_intrinsic >= 12000 + *d0_u8 = __riscv_vnclipu_wx_u8mf4(d0_clip, shift, __RISCV_VXRM_RNU, vl); + *d1_u8 = __riscv_vnclipu_wx_u8mf4(d1_clip, shift, __RISCV_VXRM_RNU, vl); + *d2_u8 = __riscv_vnclipu_wx_u8mf4(d2_clip, shift, __RISCV_VXRM_RNU, vl); + *d3_u8 = __riscv_vnclipu_wx_u8mf4(d3_clip, shift, __RISCV_VXRM_RNU, vl); +#endif +} + +static inline void compute_basic_avg_4x4_rvv( + const vuint16mf2_t dd0, const vuint16mf2_t dd1, const vuint16mf2_t dd2, + const vuint16mf2_t dd3, const vuint16mf2_t d0, const vuint16mf2_t d1, + const vuint16mf2_t d2, const vuint16mf2_t d3, const int16_t round_offset, + vuint8mf4_t *d0_u8, vuint8mf4_t *d1_u8, vuint8mf4_t *d2_u8, + vuint8mf4_t *d3_u8, size_t vl) { + // avg = (dd + d) >> 1 (vector halving add) +#if __riscv_v_intrinsic == 11000 + vuint16mf2_t vavg0 = + __riscv_vnsrl_wx_u16mf2(__riscv_vwaddu_vv_u32m1(dd0, d0, vl), 1, vl); + vuint16mf2_t vavg1 = + __riscv_vnsrl_wx_u16mf2(__riscv_vwaddu_vv_u32m1(dd1, d1, vl), 1, vl); + vuint16mf2_t vavg2 = + __riscv_vnsrl_wx_u16mf2(__riscv_vwaddu_vv_u32m1(dd2, d2, vl), 1, vl); + vuint16mf2_t vavg3 = + __riscv_vnsrl_wx_u16mf2(__riscv_vwaddu_vv_u32m1(dd3, d3, vl), 1, vl); +#elif __riscv_v_intrinsic >= 12000 + vuint16mf2_t vavg0 = __riscv_vaaddu_vv_u16mf2(dd0, d0, __RISCV_VXRM_RDN, vl); + vuint16mf2_t vavg1 = __riscv_vaaddu_vv_u16mf2(dd1, d1, __RISCV_VXRM_RDN, vl); + vuint16mf2_t vavg2 = __riscv_vaaddu_vv_u16mf2(dd2, d2, __RISCV_VXRM_RDN, vl); + vuint16mf2_t vavg3 = __riscv_vaaddu_vv_u16mf2(dd3, d3, __RISCV_VXRM_RDN, vl); +#endif + + // dst_01 = (int16x8_t) vavg01 - round_offset + vint16mf2_t vdst0 = __riscv_vsub_vx_i16mf2( + __riscv_vreinterpret_v_u16mf2_i16mf2(vavg0), round_offset, 8); + vint16mf2_t vdst1 = __riscv_vsub_vx_i16mf2( + __riscv_vreinterpret_v_u16mf2_i16mf2(vavg1), round_offset, 8); + vint16mf2_t vdst2 = __riscv_vsub_vx_i16mf2( + __riscv_vreinterpret_v_u16mf2_i16mf2(vavg2), round_offset, 8); + vint16mf2_t vdst3 = __riscv_vsub_vx_i16mf2( + __riscv_vreinterpret_v_u16mf2_i16mf2(vavg3), round_offset, 8); + + // Saturating Rounded Shift right and narrow to u8 + const int shift = FILTER_BITS - ROUND0_BITS; + vuint16mf2_t d0_clip = __riscv_vreinterpret_v_i16mf2_u16mf2( + __riscv_vmax_vx_i16mf2(vdst0, 0, vl)); + vuint16mf2_t d1_clip = __riscv_vreinterpret_v_i16mf2_u16mf2( + __riscv_vmax_vx_i16mf2(vdst1, 0, vl)); + vuint16mf2_t d2_clip = __riscv_vreinterpret_v_i16mf2_u16mf2( + __riscv_vmax_vx_i16mf2(vdst2, 0, vl)); + vuint16mf2_t d3_clip = __riscv_vreinterpret_v_i16mf2_u16mf2( + __riscv_vmax_vx_i16mf2(vdst3, 0, vl)); +#if __riscv_v_intrinsic == 11000 + *d0_u8 = __riscv_vnclipu_wx_u8mf4(d0_clip, shift, vl); + *d1_u8 = __riscv_vnclipu_wx_u8mf4(d1_clip, shift, vl); + *d2_u8 = __riscv_vnclipu_wx_u8mf4(d2_clip, shift, vl); + *d3_u8 = __riscv_vnclipu_wx_u8mf4(d3_clip, shift, vl); +#elif __riscv_v_intrinsic >= 12000 + *d0_u8 = __riscv_vnclipu_wx_u8mf4(d0_clip, shift, __RISCV_VXRM_RNU, vl); + *d1_u8 = __riscv_vnclipu_wx_u8mf4(d1_clip, shift, __RISCV_VXRM_RNU, vl); + *d2_u8 = __riscv_vnclipu_wx_u8mf4(d2_clip, shift, __RISCV_VXRM_RNU, vl); + *d3_u8 = __riscv_vnclipu_wx_u8mf4(d3_clip, shift, __RISCV_VXRM_RNU, vl); +#endif +} + +static inline void compute_dist_wtd_avg_8x4_rvv( + const vuint16m1_t dd0, const vuint16m1_t dd1, const vuint16m1_t dd2, + const vuint16m1_t dd3, const vuint16m1_t d0, const vuint16m1_t d1, + const vuint16m1_t d2, const vuint16m1_t d3, const uint16_t fwd_offset, + const uint16_t bck_offset, const int16_t round_offset, vuint8mf2_t *d0_u8, + vuint8mf2_t *d1_u8, vuint8mf2_t *d2_u8, vuint8mf2_t *d3_u8, size_t vl) { + // blend = dd* * fwd_offset + d* * bck_offset (32-bit accumulation) + vuint32m2_t vblend0 = __riscv_vwmulu_vx_u32m2(dd0, fwd_offset, vl); + vblend0 = __riscv_vwmaccu_vx_u32m2(vblend0, bck_offset, d0, vl); + + vuint32m2_t vblend1 = __riscv_vwmulu_vx_u32m2(dd1, fwd_offset, vl); + vblend1 = __riscv_vwmaccu_vx_u32m2(vblend1, bck_offset, d1, vl); + + vuint32m2_t vblend2 = __riscv_vwmulu_vx_u32m2(dd2, fwd_offset, vl); + vblend2 = __riscv_vwmaccu_vx_u32m2(vblend2, bck_offset, d2, vl); + + vuint32m2_t vblend3 = __riscv_vwmulu_vx_u32m2(dd3, fwd_offset, vl); + vblend3 = __riscv_vwmaccu_vx_u32m2(vblend3, bck_offset, d3, vl); + + // avg = blend >> DIST_PRECISION_BITS (narrow to 16-bit) + vuint16m1_t vavg0 = __riscv_vnsrl_wx_u16m1(vblend0, DIST_PRECISION_BITS, vl); + vuint16m1_t vavg1 = __riscv_vnsrl_wx_u16m1(vblend1, DIST_PRECISION_BITS, vl); + vuint16m1_t vavg2 = __riscv_vnsrl_wx_u16m1(vblend2, DIST_PRECISION_BITS, vl); + vuint16m1_t vavg3 = __riscv_vnsrl_wx_u16m1(vblend3, DIST_PRECISION_BITS, vl); + + // dst = vavg - round_offset + vint16m1_t vdst0 = __riscv_vsub_vx_i16m1( + __riscv_vreinterpret_v_u16m1_i16m1(vavg0), round_offset, vl); + vint16m1_t vdst1 = __riscv_vsub_vx_i16m1( + __riscv_vreinterpret_v_u16m1_i16m1(vavg1), round_offset, vl); + vint16m1_t vdst2 = __riscv_vsub_vx_i16m1( + __riscv_vreinterpret_v_u16m1_i16m1(vavg2), round_offset, vl); + vint16m1_t vdst3 = __riscv_vsub_vx_i16m1( + __riscv_vreinterpret_v_u16m1_i16m1(vavg3), round_offset, vl); + + // Saturating Rounded Shift right and narrow to u8 + const int shift = FILTER_BITS - ROUND0_BITS; + vuint16m1_t d0_clip = + __riscv_vreinterpret_v_i16m1_u16m1(__riscv_vmax_vx_i16m1(vdst0, 0, vl)); + vuint16m1_t d1_clip = + __riscv_vreinterpret_v_i16m1_u16m1(__riscv_vmax_vx_i16m1(vdst1, 0, vl)); + vuint16m1_t d2_clip = + __riscv_vreinterpret_v_i16m1_u16m1(__riscv_vmax_vx_i16m1(vdst2, 0, vl)); + vuint16m1_t d3_clip = + __riscv_vreinterpret_v_i16m1_u16m1(__riscv_vmax_vx_i16m1(vdst3, 0, vl)); +#if __riscv_v_intrinsic == 11000 + *d0_u8 = __riscv_vnclipu_wx_u8mf2(d0_clip, shift, vl); + *d1_u8 = __riscv_vnclipu_wx_u8mf2(d1_clip, shift, vl); + *d2_u8 = __riscv_vnclipu_wx_u8mf2(d2_clip, shift, vl); + *d3_u8 = __riscv_vnclipu_wx_u8mf2(d3_clip, shift, vl); +#elif __riscv_v_intrinsic >= 12000 + *d0_u8 = __riscv_vnclipu_wx_u8mf2(d0_clip, shift, __RISCV_VXRM_RNU, vl); + *d1_u8 = __riscv_vnclipu_wx_u8mf2(d1_clip, shift, __RISCV_VXRM_RNU, vl); + *d2_u8 = __riscv_vnclipu_wx_u8mf2(d2_clip, shift, __RISCV_VXRM_RNU, vl); + *d3_u8 = __riscv_vnclipu_wx_u8mf2(d3_clip, shift, __RISCV_VXRM_RNU, vl); +#endif +} + +static inline void compute_basic_avg_8x4_rvv( + const vuint16m1_t dd0, const vuint16m1_t dd1, const vuint16m1_t dd2, + const vuint16m1_t dd3, const vuint16m1_t d0, const vuint16m1_t d1, + const vuint16m1_t d2, const vuint16m1_t d3, const int16_t round_offset, + vuint8mf2_t *d0_u8, vuint8mf2_t *d1_u8, vuint8mf2_t *d2_u8, + vuint8mf2_t *d3_u8, size_t vl) { + // avg = (dd + d) >> 1 (vector halving add) +#if __riscv_v_intrinsic == 11000 + vuint16m1_t vavg0 = + __riscv_vnsrl_wx_u16m1(__riscv_vwaddu_vv_u32m2(dd0, d0, vl), 1, vl); + vuint16m1_t vavg1 = + __riscv_vnsrl_wx_u16m1(__riscv_vwaddu_vv_u32m2(dd1, d1, vl), 1, vl); + vuint16m1_t vavg2 = + __riscv_vnsrl_wx_u16m1(__riscv_vwaddu_vv_u32m2(dd2, d2, vl), 1, vl); + vuint16m1_t vavg3 = + __riscv_vnsrl_wx_u16m1(__riscv_vwaddu_vv_u32m2(dd3, d3, vl), 1, vl); +#elif __riscv_v_intrinsic >= 12000 + vuint16m1_t vavg0 = __riscv_vaaddu_vv_u16m1(dd0, d0, __RISCV_VXRM_RDN, vl); + vuint16m1_t vavg1 = __riscv_vaaddu_vv_u16m1(dd1, d1, __RISCV_VXRM_RDN, vl); + vuint16m1_t vavg2 = __riscv_vaaddu_vv_u16m1(dd2, d2, __RISCV_VXRM_RDN, vl); + vuint16m1_t vavg3 = __riscv_vaaddu_vv_u16m1(dd3, d3, __RISCV_VXRM_RDN, vl); +#endif + + // vdst = (int16x8_t) vavg - round_offset + vint16m1_t vdst0 = __riscv_vsub_vx_i16m1( + __riscv_vreinterpret_v_u16m1_i16m1(vavg0), round_offset, vl); + vint16m1_t vdst1 = __riscv_vsub_vx_i16m1( + __riscv_vreinterpret_v_u16m1_i16m1(vavg1), round_offset, vl); + vint16m1_t vdst2 = __riscv_vsub_vx_i16m1( + __riscv_vreinterpret_v_u16m1_i16m1(vavg2), round_offset, vl); + vint16m1_t vdst3 = __riscv_vsub_vx_i16m1( + __riscv_vreinterpret_v_u16m1_i16m1(vavg3), round_offset, vl); + + // Saturating Rounded Shift right and narrow to u8 + const int shift = FILTER_BITS - ROUND0_BITS; + vuint16m1_t d0_clip = + __riscv_vreinterpret_v_i16m1_u16m1(__riscv_vmax_vx_i16m1(vdst0, 0, vl)); + vuint16m1_t d1_clip = + __riscv_vreinterpret_v_i16m1_u16m1(__riscv_vmax_vx_i16m1(vdst1, 0, vl)); + vuint16m1_t d2_clip = + __riscv_vreinterpret_v_i16m1_u16m1(__riscv_vmax_vx_i16m1(vdst2, 0, vl)); + vuint16m1_t d3_clip = + __riscv_vreinterpret_v_i16m1_u16m1(__riscv_vmax_vx_i16m1(vdst3, 0, vl)); +#if __riscv_v_intrinsic == 11000 + *d0_u8 = __riscv_vnclipu_wx_u8mf2(d0_clip, shift, vl); + *d1_u8 = __riscv_vnclipu_wx_u8mf2(d1_clip, shift, vl); + *d2_u8 = __riscv_vnclipu_wx_u8mf2(d2_clip, shift, vl); + *d3_u8 = __riscv_vnclipu_wx_u8mf2(d3_clip, shift, vl); +#elif __riscv_v_intrinsic >= 12000 + *d0_u8 = __riscv_vnclipu_wx_u8mf2(d0_clip, shift, __RISCV_VXRM_RNU, vl); + *d1_u8 = __riscv_vnclipu_wx_u8mf2(d1_clip, shift, __RISCV_VXRM_RNU, vl); + *d2_u8 = __riscv_vnclipu_wx_u8mf2(d2_clip, shift, __RISCV_VXRM_RNU, vl); + *d3_u8 = __riscv_vnclipu_wx_u8mf2(d3_clip, shift, __RISCV_VXRM_RNU, vl); +#endif +} + +static inline vuint16mf2_t convolve6_4_2d_v_rvv( + const vint16mf2_t s0, const vint16mf2_t s1, const vint16mf2_t s2, + const vint16mf2_t s3, const vint16mf2_t s4, const vint16mf2_t s5, + const int16_t *filter, const int32_t offset_const) { + // Filter values at indices 0 and 7 are 0. + vint32m1_t sum = __riscv_vwmul_vx_i32m1(s0, filter[1], 4); + sum = __riscv_vwmacc_vx_i32m1(sum, filter[2], s1, 4); + sum = __riscv_vwmacc_vx_i32m1(sum, filter[3], s2, 4); + sum = __riscv_vwmacc_vx_i32m1(sum, filter[4], s3, 4); + sum = __riscv_vwmacc_vx_i32m1(sum, filter[5], s4, 4); + sum = __riscv_vwmacc_vx_i32m1(sum, filter[6], s5, 4); + sum = __riscv_vadd_vx_i32m1(sum, offset_const, 4); + + // Round and shift + vuint32m1_t d0 = + __riscv_vreinterpret_v_i32m1_u32m1(__riscv_vmax_vx_i32m1(sum, 0, 4)); + +#if __riscv_v_intrinsic == 11000 + return __riscv_vnclipu_wx_u16mf2(d0, COMPOUND_ROUND1_BITS, 4); +#elif __riscv_v_intrinsic >= 12000 + return __riscv_vnclipu_wx_u16mf2(d0, COMPOUND_ROUND1_BITS, __RISCV_VXRM_RNU, + 4); +#endif +} + +static inline vuint16m1_t convolve6_8_2d_v_rvv( + const vint16m1_t s0, const vint16m1_t s1, const vint16m1_t s2, + const vint16m1_t s3, const vint16m1_t s4, const vint16m1_t s5, + const int16_t *filter, const int32_t offset_const, size_t vl) { + // Filter values at indices 0 and 7 are 0. + vint32m2_t sum = __riscv_vwmul_vx_i32m2(s0, filter[1], vl); + sum = __riscv_vwmacc_vx_i32m2(sum, filter[2], s1, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, filter[3], s2, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, filter[4], s3, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, filter[5], s4, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, filter[6], s5, vl); + sum = __riscv_vadd_vx_i32m2(sum, offset_const, vl); + + // Round and shift + vuint32m2_t d0 = + __riscv_vreinterpret_v_i32m2_u32m2(__riscv_vmax_vx_i32m2(sum, 0, vl)); + +#if __riscv_v_intrinsic == 11000 + return __riscv_vnclipu_wx_u16m1(d0, COMPOUND_ROUND1_BITS, vl); +#elif __riscv_v_intrinsic >= 12000 + return __riscv_vnclipu_wx_u16m1(d0, COMPOUND_ROUND1_BITS, __RISCV_VXRM_RNU, + vl); +#endif +} + +static inline void dist_wtd_convolve_2d_vert_6tap_dist_wtd_avg_rvv( + int16_t *src_ptr, const int src_stride, uint8_t *dst8_ptr, int dst8_stride, + ConvolveParams *conv_params, const int16_t *y_filter, int h, int w) { + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const int32_t offset_const = 1 << offset_bits; + const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + + const uint16_t fwd_offset = conv_params->fwd_offset; + const uint16_t bck_offset = conv_params->bck_offset; + + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + size_t vl = __riscv_vsetvl_e16m1(w); + + if (w == 4) { + // load initial 5 rows of data + vint16mf2_t s0, s1, s2, s3, s4; + load_s16_4x5(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4, vl); + src_ptr += 5 * src_stride; + + do { + // load next 4 rows of data + vint16mf2_t s5, s6, s7, s8; + load_s16_4x4(src_ptr, src_stride, &s5, &s6, &s7, &s8, vl); + + // perform 6-tap convolution + vuint16mf2_t d0, d1, d2, d3; + d0 = convolve6_4_2d_v_rvv(s0, s1, s2, s3, s4, s5, y_filter, offset_const); + d1 = convolve6_4_2d_v_rvv(s1, s2, s3, s4, s5, s6, y_filter, offset_const); + d2 = convolve6_4_2d_v_rvv(s2, s3, s4, s5, s6, s7, y_filter, offset_const); + d3 = convolve6_4_2d_v_rvv(s3, s4, s5, s6, s7, s8, y_filter, offset_const); + + // weighted average + vuint16mf2_t dd0, dd1, dd2, dd3; + load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3, vl); + vuint8mf4_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_dist_wtd_avg_4x4_rvv(dd0, dd1, dd2, dd3, d0, d1, d2, d3, + fwd_offset, bck_offset, round_offset, &d0_u8, + &d1_u8, &d2_u8, &d3_u8, vl); + + // store results + __riscv_vse8_v_u8mf4(dst8_ptr + 0 * dst8_stride, d0_u8, vl); + __riscv_vse8_v_u8mf4(dst8_ptr + 1 * dst8_stride, d1_u8, vl); + __riscv_vse8_v_u8mf4(dst8_ptr + 2 * dst8_stride, d2_u8, vl); + __riscv_vse8_v_u8mf4(dst8_ptr + 3 * dst8_stride, d3_u8, vl); + + // update sliding window + s0 = __riscv_vmv_v_v_i16mf2(s4, vl); + s1 = __riscv_vmv_v_v_i16mf2(s5, vl); + s2 = __riscv_vmv_v_v_i16mf2(s6, vl); + s3 = __riscv_vmv_v_v_i16mf2(s7, vl); + s4 = __riscv_vmv_v_v_i16mf2(s8, vl); + src_ptr += 4 * src_stride; + dst_ptr += 4 * dst_stride; + dst8_ptr += 4 * dst8_stride; + h -= 4; + } while (h != 0); + } else { + do { + int16_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + uint8_t *d_u8 = dst8_ptr; + int height = h; + + // Load initial 5 rows of data + vint16m1_t s0, s1, s2, s3, s4; + load_s16_8x5(s, src_stride, &s0, &s1, &s2, &s3, &s4, vl); + s += 5 * src_stride; + + do { + // load next 4 rows of data + vint16m1_t s5, s6, s7, s8; + load_s16_8x4(s, src_stride, &s5, &s6, &s7, &s8, vl); + + // perform 6-tap convolution + vuint16m1_t d0, d1, d2, d3; + d0 = convolve6_8_2d_v_rvv(s0, s1, s2, s3, s4, s5, y_filter, + offset_const, vl); + d1 = convolve6_8_2d_v_rvv(s1, s2, s3, s4, s5, s6, y_filter, + offset_const, vl); + d2 = convolve6_8_2d_v_rvv(s2, s3, s4, s5, s6, s7, y_filter, + offset_const, vl); + d3 = convolve6_8_2d_v_rvv(s3, s4, s5, s6, s7, s8, y_filter, + offset_const, vl); + + // weighted average + vuint16m1_t dd0, dd1, dd2, dd3; + load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3, vl); + + vuint8mf2_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_dist_wtd_avg_8x4_rvv(dd0, dd1, dd2, dd3, d0, d1, d2, d3, + fwd_offset, bck_offset, round_offset, + &d0_u8, &d1_u8, &d2_u8, &d3_u8, vl); + + // store results + store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8, vl); + d_u8 += 4 * dst8_stride; + + // update sliding window + s0 = __riscv_vmv_v_v_i16m1(s4, vl); + s1 = __riscv_vmv_v_v_i16m1(s5, vl); + s2 = __riscv_vmv_v_v_i16m1(s6, vl); + s3 = __riscv_vmv_v_v_i16m1(s7, vl); + s4 = __riscv_vmv_v_v_i16m1(s8, vl); + s += 4 * src_stride; + d += 4 * dst_stride; + height -= 4; + } while (height != 0); + src_ptr += vl; + dst_ptr += vl; + dst8_ptr += vl; + w -= vl; + } while (w > 0); + } +} + +static inline void dist_wtd_convolve_2d_vert_6tap_avg_rvv( + int16_t *src_ptr, const int src_stride, uint8_t *dst8_ptr, int dst8_stride, + ConvolveParams *conv_params, const int16_t *y_filter, int h, int w) { + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const int32_t offset_const = 1 << offset_bits; + const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + size_t vl = __riscv_vsetvl_e16m1(w); + + if (w == 4) { + // load initial 5 rows of data + vint16mf2_t s0, s1, s2, s3, s4; + load_s16_4x5(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4, vl); + src_ptr += 5 * src_stride; + + do { + // load next 4 rows of data + vint16mf2_t s5, s6, s7, s8; + load_s16_4x4(src_ptr, src_stride, &s5, &s6, &s7, &s8, vl); + + // perform 6-tap convolution + vuint16mf2_t d0, d1, d2, d3; + d0 = convolve6_4_2d_v_rvv(s0, s1, s2, s3, s4, s5, y_filter, offset_const); + d1 = convolve6_4_2d_v_rvv(s1, s2, s3, s4, s5, s6, y_filter, offset_const); + d2 = convolve6_4_2d_v_rvv(s2, s3, s4, s5, s6, s7, y_filter, offset_const); + d3 = convolve6_4_2d_v_rvv(s3, s4, s5, s6, s7, s8, y_filter, offset_const); + + // average + vuint16mf2_t dd0, dd1, dd2, dd3; + load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3, vl); + + vuint8mf4_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_basic_avg_4x4_rvv(dd0, dd1, dd2, dd3, d0, d1, d2, d3, + round_offset, &d0_u8, &d1_u8, &d2_u8, &d3_u8, + vl); + + // Store result + __riscv_vse8_v_u8mf4(dst8_ptr + 0 * dst8_stride, d0_u8, vl); + __riscv_vse8_v_u8mf4(dst8_ptr + 1 * dst8_stride, d1_u8, vl); + __riscv_vse8_v_u8mf4(dst8_ptr + 2 * dst8_stride, d2_u8, vl); + __riscv_vse8_v_u8mf4(dst8_ptr + 3 * dst8_stride, d3_u8, vl); + dst8_ptr += 4 * dst8_stride; + + // Update sliding window + s0 = __riscv_vmv_v_v_i16mf2(s4, vl); + s1 = __riscv_vmv_v_v_i16mf2(s5, vl); + s2 = __riscv_vmv_v_v_i16mf2(s6, vl); + s3 = __riscv_vmv_v_v_i16mf2(s7, vl); + s4 = __riscv_vmv_v_v_i16mf2(s8, vl); + src_ptr += 4 * src_stride; + dst_ptr += 4 * dst_stride; + h -= 4; + } while (h != 0); + } else { + do { + int16_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + uint8_t *d_u8 = dst8_ptr; + int height = h; + + // Load initial 5 rows of data + vint16m1_t s0, s1, s2, s3, s4; + load_s16_8x5(s, src_stride, &s0, &s1, &s2, &s3, &s4, vl); + s += 5 * src_stride; + + do { + // load next 4 rows of data + vint16m1_t s5, s6, s7, s8; + load_s16_8x4(s, src_stride, &s5, &s6, &s7, &s8, vl); + + // perform 6-tap convolution + vuint16m1_t d0, d1, d2, d3; + d0 = convolve6_8_2d_v_rvv(s0, s1, s2, s3, s4, s5, y_filter, + offset_const, vl); + d1 = convolve6_8_2d_v_rvv(s1, s2, s3, s4, s5, s6, y_filter, + offset_const, vl); + d2 = convolve6_8_2d_v_rvv(s2, s3, s4, s5, s6, s7, y_filter, + offset_const, vl); + d3 = convolve6_8_2d_v_rvv(s3, s4, s5, s6, s7, s8, y_filter, + offset_const, vl); + + // average + vuint16m1_t dd0, dd1, dd2, dd3; + load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3, vl); + + vuint8mf2_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_basic_avg_8x4_rvv(dd0, dd1, dd2, dd3, d0, d1, d2, d3, + round_offset, &d0_u8, &d1_u8, &d2_u8, &d3_u8, + vl); + + // store results + store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8, vl); + d_u8 += 4 * dst8_stride; + + // update sliding window + s0 = __riscv_vmv_v_v_i16m1(s4, vl); + s1 = __riscv_vmv_v_v_i16m1(s5, vl); + s2 = __riscv_vmv_v_v_i16m1(s6, vl); + s3 = __riscv_vmv_v_v_i16m1(s7, vl); + s4 = __riscv_vmv_v_v_i16m1(s8, vl); + s += 4 * src_stride; + d += 4 * dst_stride; + height -= 4; + } while (height != 0); + src_ptr += vl; + dst_ptr += vl; + dst8_ptr += vl; + w -= vl; + } while (w > 0); + } +} + +static inline void dist_wtd_convolve_2d_vert_6tap_rvv( + int16_t *src_ptr, const int src_stride, ConvolveParams *conv_params, + const int16_t *y_filter, int h, int w) { + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const int32_t offset_const = 1 << offset_bits; + + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + size_t vl = __riscv_vsetvl_e16m1(w); + + if (w == 4) { + // load initial 5 rows of data + vint16mf2_t s0, s1, s2, s3, s4; + load_s16_4x5(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4, vl); + src_ptr += 5 * src_stride; + + do { + // load next 4 rows of data + vint16mf2_t s5, s6, s7, s8; + load_s16_4x4(src_ptr, src_stride, &s5, &s6, &s7, &s8, vl); + + // perform 6-tap convolution + vuint16mf2_t d0, d1, d2, d3; + d0 = convolve6_4_2d_v_rvv(s0, s1, s2, s3, s4, s5, y_filter, offset_const); + d1 = convolve6_4_2d_v_rvv(s1, s2, s3, s4, s5, s6, y_filter, offset_const); + d2 = convolve6_4_2d_v_rvv(s2, s3, s4, s5, s6, s7, y_filter, offset_const); + d3 = convolve6_4_2d_v_rvv(s3, s4, s5, s6, s7, s8, y_filter, offset_const); + + // Store result + store_u16_4x4(dst_ptr, dst_stride, d0, d1, d2, d3, vl); + + // Update sliding window + s0 = __riscv_vmv_v_v_i16mf2(s4, vl); + s1 = __riscv_vmv_v_v_i16mf2(s5, vl); + s2 = __riscv_vmv_v_v_i16mf2(s6, vl); + s3 = __riscv_vmv_v_v_i16mf2(s7, vl); + s4 = __riscv_vmv_v_v_i16mf2(s8, vl); + src_ptr += 4 * src_stride; + dst_ptr += 4 * dst_stride; + h -= 4; + } while (h != 0); + } else { + do { + int16_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + int height = h; + + // Load initial 5 rows of data + vint16m1_t s0, s1, s2, s3, s4; + load_s16_8x5(s, src_stride, &s0, &s1, &s2, &s3, &s4, vl); + s += 5 * src_stride; + + do { + // load next 4 rows of data + vint16m1_t s5, s6, s7, s8; + load_s16_8x4(s, src_stride, &s5, &s6, &s7, &s8, vl); + + // perform 6-tap convolution + vuint16m1_t d0, d1, d2, d3; + d0 = convolve6_8_2d_v_rvv(s0, s1, s2, s3, s4, s5, y_filter, + offset_const, vl); + d1 = convolve6_8_2d_v_rvv(s1, s2, s3, s4, s5, s6, y_filter, + offset_const, vl); + d2 = convolve6_8_2d_v_rvv(s2, s3, s4, s5, s6, s7, y_filter, + offset_const, vl); + d3 = convolve6_8_2d_v_rvv(s3, s4, s5, s6, s7, s8, y_filter, + offset_const, vl); + + // store results + store_u16_8x4(d, dst_stride, d0, d1, d2, d3, vl); + + // update sliding window + s0 = __riscv_vmv_v_v_i16m1(s4, vl); + s1 = __riscv_vmv_v_v_i16m1(s5, vl); + s2 = __riscv_vmv_v_v_i16m1(s6, vl); + s3 = __riscv_vmv_v_v_i16m1(s7, vl); + s4 = __riscv_vmv_v_v_i16m1(s8, vl); + s += 4 * src_stride; + d += 4 * dst_stride; + height -= 4; + } while (height != 0); + src_ptr += vl; + dst_ptr += vl; + w -= vl; + } while (w > 0); + } +} + +static inline vuint16mf2_t convolve8_4_2d_v_rvv( + const vint16mf2_t s0, const vint16mf2_t s1, const vint16mf2_t s2, + const vint16mf2_t s3, const vint16mf2_t s4, const vint16mf2_t s5, + const vint16mf2_t s6, const vint16mf2_t s7, const int16_t *filter, + const int32_t offset_const) { + vint32m1_t sum = __riscv_vwmul_vx_i32m1(s0, filter[0], 4); + sum = __riscv_vwmacc_vx_i32m1(sum, filter[1], s1, 4); + sum = __riscv_vwmacc_vx_i32m1(sum, filter[2], s2, 4); + sum = __riscv_vwmacc_vx_i32m1(sum, filter[3], s3, 4); + sum = __riscv_vwmacc_vx_i32m1(sum, filter[4], s4, 4); + sum = __riscv_vwmacc_vx_i32m1(sum, filter[5], s5, 4); + sum = __riscv_vwmacc_vx_i32m1(sum, filter[6], s6, 4); + sum = __riscv_vwmacc_vx_i32m1(sum, filter[7], s7, 4); + sum = __riscv_vadd_vx_i32m1(sum, offset_const, 4); + + // Round and shift + vuint32m1_t d0 = + __riscv_vreinterpret_v_i32m1_u32m1(__riscv_vmax_vx_i32m1(sum, 0, 4)); + +#if __riscv_v_intrinsic == 11000 + return __riscv_vnclipu_wx_u16mf2(d0, COMPOUND_ROUND1_BITS, 4); +#elif __riscv_v_intrinsic >= 12000 + return __riscv_vnclipu_wx_u16mf2(d0, COMPOUND_ROUND1_BITS, __RISCV_VXRM_RNU, + 4); +#endif +} + +static inline vuint16m1_t convolve8_8_2d_v_rvv( + const vint16m1_t s0, const vint16m1_t s1, const vint16m1_t s2, + const vint16m1_t s3, const vint16m1_t s4, const vint16m1_t s5, + const vint16m1_t s6, const vint16m1_t s7, const int16_t *filter, + const int32_t offset_const, size_t vl) { + vint32m2_t sum = __riscv_vwmul_vx_i32m2(s0, filter[0], vl); + sum = __riscv_vwmacc_vx_i32m2(sum, filter[1], s1, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, filter[2], s2, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, filter[3], s3, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, filter[4], s4, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, filter[5], s5, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, filter[6], s6, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, filter[7], s7, vl); + sum = __riscv_vadd_vx_i32m2(sum, offset_const, vl); + + // Round and shift + vuint32m2_t d0 = + __riscv_vreinterpret_v_i32m2_u32m2(__riscv_vmax_vx_i32m2(sum, 0, vl)); + +#if __riscv_v_intrinsic == 11000 + return __riscv_vnclipu_wx_u16m1(d0, COMPOUND_ROUND1_BITS, vl); +#elif __riscv_v_intrinsic >= 12000 + return __riscv_vnclipu_wx_u16m1(d0, COMPOUND_ROUND1_BITS, __RISCV_VXRM_RNU, + vl); +#endif +} + +static inline void dist_wtd_convolve_2d_vert_8tap_dist_wtd_avg_rvv( + int16_t *src_ptr, const int src_stride, uint8_t *dst8_ptr, int dst8_stride, + ConvolveParams *conv_params, const int16_t *y_filter, int h, int w) { + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const int32_t offset_const = 1 << offset_bits; + const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + const uint16_t fwd_offset = conv_params->fwd_offset; + const uint16_t bck_offset = conv_params->bck_offset; + + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + size_t vl = __riscv_vsetvl_e16m1(w); + + if (w == 4) { + // load initial 7 rows of data + vint16mf2_t s0, s1, s2, s3, s4, s5, s6; + load_s16_4x7(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, vl); + src_ptr += 7 * src_stride; + + do { + // load next 4 rows of data + vint16mf2_t s7, s8, s9, s10; + load_s16_4x4(src_ptr, src_stride, &s7, &s8, &s9, &s10, vl); + + // Perform convolution + vuint16mf2_t d0, d1, d2, d3; + d0 = convolve8_4_2d_v_rvv(s0, s1, s2, s3, s4, s5, s6, s7, y_filter, + offset_const); + d1 = convolve8_4_2d_v_rvv(s1, s2, s3, s4, s5, s6, s7, s8, y_filter, + offset_const); + d2 = convolve8_4_2d_v_rvv(s2, s3, s4, s5, s6, s7, s8, s9, y_filter, + offset_const); + d3 = convolve8_4_2d_v_rvv(s3, s4, s5, s6, s7, s8, s9, s10, y_filter, + offset_const); + + // weighted average + vuint16mf2_t dd0, dd1, dd2, dd3; + load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3, vl); + vuint8mf4_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_dist_wtd_avg_4x4_rvv(dd0, dd1, dd2, dd3, d0, d1, d2, d3, + fwd_offset, bck_offset, round_offset, &d0_u8, + &d1_u8, &d2_u8, &d3_u8, vl); + + // store results + __riscv_vse8_v_u8mf4(dst8_ptr + 0 * dst8_stride, d0_u8, vl); + __riscv_vse8_v_u8mf4(dst8_ptr + 1 * dst8_stride, d1_u8, vl); + __riscv_vse8_v_u8mf4(dst8_ptr + 2 * dst8_stride, d2_u8, vl); + __riscv_vse8_v_u8mf4(dst8_ptr + 3 * dst8_stride, d3_u8, vl); + + // update sliding window + s0 = __riscv_vmv_v_v_i16mf2(s4, vl); + s1 = __riscv_vmv_v_v_i16mf2(s5, vl); + s2 = __riscv_vmv_v_v_i16mf2(s6, vl); + s3 = __riscv_vmv_v_v_i16mf2(s7, vl); + s4 = __riscv_vmv_v_v_i16mf2(s8, vl); + s5 = __riscv_vmv_v_v_i16mf2(s9, vl); + s6 = __riscv_vmv_v_v_i16mf2(s10, vl); + src_ptr += 4 * src_stride; + dst_ptr += 4 * dst_stride; + dst8_ptr += 4 * dst8_stride; + h -= 4; + } while (h != 0); + } else { + do { + int16_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + uint8_t *d_u8 = dst8_ptr; + int height = h; + + // load initial 7 rows of data + vint16m1_t s0, s1, s2, s3, s4, s5, s6; + load_s16_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, vl); + s += 7 * src_stride; + + do { + // load next 4 rows of data + vint16m1_t s7, s8, s9, s10; + load_s16_8x4(s, src_stride, &s7, &s8, &s9, &s10, vl); + + // perform convolution + vuint16m1_t d0 = convolve8_8_2d_v_rvv(s0, s1, s2, s3, s4, s5, s6, s7, + y_filter, offset_const, vl); + vuint16m1_t d1 = convolve8_8_2d_v_rvv(s1, s2, s3, s4, s5, s6, s7, s8, + y_filter, offset_const, vl); + vuint16m1_t d2 = convolve8_8_2d_v_rvv(s2, s3, s4, s5, s6, s7, s8, s9, + y_filter, offset_const, vl); + vuint16m1_t d3 = convolve8_8_2d_v_rvv(s3, s4, s5, s6, s7, s8, s9, s10, + y_filter, offset_const, vl); + + // weighted average + vuint16m1_t dd0, dd1, dd2, dd3; + load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3, vl); + + vuint8mf2_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_dist_wtd_avg_8x4_rvv(dd0, dd1, dd2, dd3, d0, d1, d2, d3, + fwd_offset, bck_offset, round_offset, + &d0_u8, &d1_u8, &d2_u8, &d3_u8, vl); + + // store results + store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8, vl); + d_u8 += 4 * dst8_stride; + + // update sliding window + s0 = __riscv_vmv_v_v_i16m1(s4, vl); + s1 = __riscv_vmv_v_v_i16m1(s5, vl); + s2 = __riscv_vmv_v_v_i16m1(s6, vl); + s3 = __riscv_vmv_v_v_i16m1(s7, vl); + s4 = __riscv_vmv_v_v_i16m1(s8, vl); + s5 = __riscv_vmv_v_v_i16m1(s9, vl); + s6 = __riscv_vmv_v_v_i16m1(s10, vl); + s += 4 * src_stride; + d += 4 * dst_stride; + height -= 4; + } while (height != 0); + src_ptr += vl; + dst_ptr += vl; + dst8_ptr += vl; + w -= vl; + } while (w > 0); + } +} + +static inline void dist_wtd_convolve_2d_vert_8tap_avg_rvv( + int16_t *src_ptr, const int src_stride, uint8_t *dst8_ptr, int dst8_stride, + ConvolveParams *conv_params, const int16_t *y_filter, int h, int w) { + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const int32_t offset_const = 1 << offset_bits; + const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + size_t vl = __riscv_vsetvl_e16m1(w); + + if (w == 4) { + // load initial 7 rows of data + vint16mf2_t s0, s1, s2, s3, s4, s5, s6; + load_s16_4x7(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, vl); + src_ptr += 7 * src_stride; + + do { + // load next 4 rows of data + vint16mf2_t s7, s8, s9, s10; + load_s16_4x4(src_ptr, src_stride, &s7, &s8, &s9, &s10, vl); + + // Perform convolution + vuint16mf2_t d0, d1, d2, d3; + d0 = convolve8_4_2d_v_rvv(s0, s1, s2, s3, s4, s5, s6, s7, y_filter, + offset_const); + d1 = convolve8_4_2d_v_rvv(s1, s2, s3, s4, s5, s6, s7, s8, y_filter, + offset_const); + d2 = convolve8_4_2d_v_rvv(s2, s3, s4, s5, s6, s7, s8, s9, y_filter, + offset_const); + d3 = convolve8_4_2d_v_rvv(s3, s4, s5, s6, s7, s8, s9, s10, y_filter, + offset_const); + + // average + vuint16mf2_t dd0, dd1, dd2, dd3; + load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3, vl); + + vuint8mf4_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_basic_avg_4x4_rvv(dd0, dd1, dd2, dd3, d0, d1, d2, d3, + round_offset, &d0_u8, &d1_u8, &d2_u8, &d3_u8, + vl); + + // Store result + __riscv_vse8_v_u8mf4(dst8_ptr + 0 * dst8_stride, d0_u8, vl); + __riscv_vse8_v_u8mf4(dst8_ptr + 1 * dst8_stride, d1_u8, vl); + __riscv_vse8_v_u8mf4(dst8_ptr + 2 * dst8_stride, d2_u8, vl); + __riscv_vse8_v_u8mf4(dst8_ptr + 3 * dst8_stride, d3_u8, vl); + dst8_ptr += 4 * dst8_stride; + + // update sliding window + s0 = __riscv_vmv_v_v_i16mf2(s4, vl); + s1 = __riscv_vmv_v_v_i16mf2(s5, vl); + s2 = __riscv_vmv_v_v_i16mf2(s6, vl); + s3 = __riscv_vmv_v_v_i16mf2(s7, vl); + s4 = __riscv_vmv_v_v_i16mf2(s8, vl); + s5 = __riscv_vmv_v_v_i16mf2(s9, vl); + s6 = __riscv_vmv_v_v_i16mf2(s10, vl); + src_ptr += 4 * src_stride; + dst_ptr += 4 * dst_stride; + h -= 4; + } while (h != 0); + } else { + do { + int16_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + uint8_t *d_u8 = dst8_ptr; + int height = h; + + // load initial 7 rows of data + vint16m1_t s0, s1, s2, s3, s4, s5, s6; + load_s16_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, vl); + s += 7 * src_stride; + + do { + // load next 4 rows of data + vint16m1_t s7, s8, s9, s10; + load_s16_8x4(s, src_stride, &s7, &s8, &s9, &s10, vl); + + // perform convolution + vuint16m1_t d0 = convolve8_8_2d_v_rvv(s0, s1, s2, s3, s4, s5, s6, s7, + y_filter, offset_const, vl); + vuint16m1_t d1 = convolve8_8_2d_v_rvv(s1, s2, s3, s4, s5, s6, s7, s8, + y_filter, offset_const, vl); + vuint16m1_t d2 = convolve8_8_2d_v_rvv(s2, s3, s4, s5, s6, s7, s8, s9, + y_filter, offset_const, vl); + vuint16m1_t d3 = convolve8_8_2d_v_rvv(s3, s4, s5, s6, s7, s8, s9, s10, + y_filter, offset_const, vl); + + // average + vuint16m1_t dd0, dd1, dd2, dd3; + load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3, vl); + + vuint8mf2_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_basic_avg_8x4_rvv(dd0, dd1, dd2, dd3, d0, d1, d2, d3, + round_offset, &d0_u8, &d1_u8, &d2_u8, &d3_u8, + vl); + + // store results + store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8, vl); + d_u8 += 4 * dst8_stride; + + // update sliding window + s0 = __riscv_vmv_v_v_i16m1(s4, vl); + s1 = __riscv_vmv_v_v_i16m1(s5, vl); + s2 = __riscv_vmv_v_v_i16m1(s6, vl); + s3 = __riscv_vmv_v_v_i16m1(s7, vl); + s4 = __riscv_vmv_v_v_i16m1(s8, vl); + s5 = __riscv_vmv_v_v_i16m1(s9, vl); + s6 = __riscv_vmv_v_v_i16m1(s10, vl); + s += 4 * src_stride; + d += 4 * dst_stride; + height -= 4; + } while (height != 0); + src_ptr += vl; + dst_ptr += vl; + dst8_ptr += vl; + w -= vl; + } while (w > 0); + } +} + +static inline void dist_wtd_convolve_2d_vert_8tap_rvv( + int16_t *src_ptr, const int src_stride, ConvolveParams *conv_params, + const int16_t *y_filter, int h, int w) { + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const int32_t offset_const = 1 << offset_bits; + + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + size_t vl = __riscv_vsetvl_e16m1(w); + + if (w == 4) { + // load initial 7 rows of data + vint16mf2_t s0, s1, s2, s3, s4, s5, s6; + load_s16_4x7(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, vl); + src_ptr += 7 * src_stride; + + do { + // load next 4 rows of data + vint16mf2_t s7, s8, s9, s10; + load_s16_4x4(src_ptr, src_stride, &s7, &s8, &s9, &s10, vl); + + // perform convolution + vuint16mf2_t d0, d1, d2, d3; + d0 = convolve8_4_2d_v_rvv(s0, s1, s2, s3, s4, s5, s6, s7, y_filter, + offset_const); + d1 = convolve8_4_2d_v_rvv(s1, s2, s3, s4, s5, s6, s7, s8, y_filter, + offset_const); + d2 = convolve8_4_2d_v_rvv(s2, s3, s4, s5, s6, s7, s8, s9, y_filter, + offset_const); + d3 = convolve8_4_2d_v_rvv(s3, s4, s5, s6, s7, s8, s9, s10, y_filter, + offset_const); + + // store result + store_u16_4x4(dst_ptr, dst_stride, d0, d1, d2, d3, vl); + + // update sliding window + s0 = __riscv_vmv_v_v_i16mf2(s4, vl); + s1 = __riscv_vmv_v_v_i16mf2(s5, vl); + s2 = __riscv_vmv_v_v_i16mf2(s6, vl); + s3 = __riscv_vmv_v_v_i16mf2(s7, vl); + s4 = __riscv_vmv_v_v_i16mf2(s8, vl); + s5 = __riscv_vmv_v_v_i16mf2(s9, vl); + s6 = __riscv_vmv_v_v_i16mf2(s10, vl); + src_ptr += 4 * src_stride; + dst_ptr += 4 * dst_stride; + h -= 4; + } while (h != 0); + } else { + do { + int16_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + int height = h; + + // load initial 7 rows of data + vint16m1_t s0, s1, s2, s3, s4, s5, s6; + load_s16_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, vl); + s += 7 * src_stride; + + do { + // load next 4 rows of data + vint16m1_t s7, s8, s9, s10; + load_s16_8x4(s, src_stride, &s7, &s8, &s9, &s10, vl); + + // perform convolution + vuint16m1_t d0 = convolve8_8_2d_v_rvv(s0, s1, s2, s3, s4, s5, s6, s7, + y_filter, offset_const, vl); + vuint16m1_t d1 = convolve8_8_2d_v_rvv(s1, s2, s3, s4, s5, s6, s7, s8, + y_filter, offset_const, vl); + vuint16m1_t d2 = convolve8_8_2d_v_rvv(s2, s3, s4, s5, s6, s7, s8, s9, + y_filter, offset_const, vl); + vuint16m1_t d3 = convolve8_8_2d_v_rvv(s3, s4, s5, s6, s7, s8, s9, s10, + y_filter, offset_const, vl); + + // store results + store_u16_8x4(d, dst_stride, d0, d1, d2, d3, vl); + + // update sliding window + s0 = __riscv_vmv_v_v_i16m1(s4, vl); + s1 = __riscv_vmv_v_v_i16m1(s5, vl); + s2 = __riscv_vmv_v_v_i16m1(s6, vl); + s3 = __riscv_vmv_v_v_i16m1(s7, vl); + s4 = __riscv_vmv_v_v_i16m1(s8, vl); + s5 = __riscv_vmv_v_v_i16m1(s9, vl); + s6 = __riscv_vmv_v_v_i16m1(s10, vl); + s += 4 * src_stride; + d += 4 * dst_stride; + height -= 4; + } while (height != 0); + src_ptr += vl; + dst_ptr += vl; + w -= vl; + } while (w > 0); + } +} + +#endif // AOM_AV1_COMMON_RISCV_COMPOUND_CONVOLVE_RVV_H_
diff --git a/av1/common/riscv/highbd_compound_convolve_rvv.c b/av1/common/riscv/highbd_compound_convolve_rvv.c new file mode 100644 index 0000000..9f2f869 --- /dev/null +++ b/av1/common/riscv/highbd_compound_convolve_rvv.c
@@ -0,0 +1,1386 @@ +/* + * Copyright (c) 2026, Alliance for Open Media. All rights reserved. + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include <riscv_vector.h> +#include <assert.h> + +#include "config/aom_config.h" +#include "config/av1_rtcd.h" + +#include "av1/common/convolve.h" +#include "av1/common/filter.h" +#include "av1/common/riscv/highbd_compound_convolve_rvv.h" + +static inline vuint16mf2_t highbd_convolve6_4_rvv( + const vint16mf2_t s0, const vint16mf2_t s1, const vint16mf2_t s2, + const vint16mf2_t s3, const vint16mf2_t s4, const vint16mf2_t s5, + const int16_t *filter, const vint32m1_t offset_vec, + const int32_t round_bits, const size_t vl) { + // Values at indices 0 and 7 of filter are zero. + vint32m1_t sum = __riscv_vwmacc_vx_i32m1(offset_vec, filter[1], s0, vl); + sum = __riscv_vwmacc_vx_i32m1(sum, filter[2], s1, vl); + sum = __riscv_vwmacc_vx_i32m1(sum, filter[3], s2, vl); + sum = __riscv_vwmacc_vx_i32m1(sum, filter[4], s3, vl); + sum = __riscv_vwmacc_vx_i32m1(sum, filter[5], s4, vl); + sum = __riscv_vwmacc_vx_i32m1(sum, filter[6], s5, vl); + + vuint32m1_t sum_u32 = + __riscv_vreinterpret_v_i32m1_u32m1(__riscv_vmax_vx_i32m1(sum, 0, vl)); +#if __riscv_v_intrinsic >= 12000 + vuint16mf2_t res = + __riscv_vnclipu_wx_u16mf2(sum_u32, round_bits, __RISCV_VXRM_RDN, vl); +#else + vuint16mf2_t res = __riscv_vnsrl_wx_u16mf2(sum_u32, round_bits, vl); +#endif + return res; +} + +static inline vuint16m1_t highbd_convolve6_8_rvv( + const vint16m1_t s0, const vint16m1_t s1, const vint16m1_t s2, + const vint16m1_t s3, const vint16m1_t s4, const vint16m1_t s5, + const int16_t *filter, const vint32m2_t offset_vec, + const int32_t round_bits, const size_t vl) { + // Values at indices 0 and 7 of filter are zero. + vint32m2_t sum = __riscv_vwmacc_vx_i32m2(offset_vec, filter[1], s0, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, filter[2], s1, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, filter[3], s2, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, filter[4], s3, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, filter[5], s4, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, filter[6], s5, vl); + + vuint32m2_t sum_u32 = + __riscv_vreinterpret_v_i32m2_u32m2(__riscv_vmax_vx_i32m2(sum, 0, vl)); +#if __riscv_v_intrinsic >= 12000 + vuint16m1_t res = + __riscv_vnclipu_wx_u16m1(sum_u32, round_bits, __RISCV_VXRM_RDN, vl); +#else + vuint16m1_t res = __riscv_vnsrl_wx_u16m1(sum_u32, round_bits, vl); +#endif + return res; +} + +static inline void highbd_dist_wtd_convolve_x_6tap_rvv( + const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride, + int w, int h, const int16_t *x_filter_ptr, const int offset, + const int round_bits) { + int height = h; + const size_t vl = __riscv_vsetvl_e16m1(w); + vint32m2_t offset_vec = __riscv_vmv_v_x_i32m2(offset, vl); + + do { + int width = w; + const int16_t *s = (const int16_t *)src_ptr; + uint16_t *d = dst_ptr; + + do { + vint16m1_t s00, s01, s02, s03, s04, s05; + vint16m1_t s10, s11, s12, s13, s14, s15; + vint16m1_t s20, s21, s22, s23, s24, s25; + vint16m1_t s30, s31, s32, s33, s34, s35; + + // Load 6 consecutive 8-element vectors from 4 rows + load_s16_8x6(s + 0 * src_stride, 1, &s00, &s01, &s02, &s03, &s04, &s05, + vl); + load_s16_8x6(s + 1 * src_stride, 1, &s10, &s11, &s12, &s13, &s14, &s15, + vl); + load_s16_8x6(s + 2 * src_stride, 1, &s20, &s21, &s22, &s23, &s24, &s25, + vl); + load_s16_8x6(s + 3 * src_stride, 1, &s30, &s31, &s32, &s33, &s34, &s35, + vl); + + // Convolve each row + vuint16m1_t d0 = + highbd_convolve6_8_rvv(s00, s01, s02, s03, s04, s05, x_filter_ptr, + offset_vec, round_bits, vl); + vuint16m1_t d1 = + highbd_convolve6_8_rvv(s10, s11, s12, s13, s14, s15, x_filter_ptr, + offset_vec, round_bits, vl); + vuint16m1_t d2 = + highbd_convolve6_8_rvv(s20, s21, s22, s23, s24, s25, x_filter_ptr, + offset_vec, round_bits, vl); + vuint16m1_t d3 = + highbd_convolve6_8_rvv(s30, s31, s32, s33, s34, s35, x_filter_ptr, + offset_vec, round_bits, vl); + + // Store results with stride + store_u16_8x4(d, dst_stride, d0, d1, d2, d3, vl); + + s += vl; + d += vl; + width -= vl; + } while (width > 0); + src_ptr += 4 * src_stride; + dst_ptr += 4 * dst_stride; + height -= 4; + } while (height != 0); +} + +static inline vuint16mf2_t highbd_convolve8_4_rvv( + const vint16mf2_t s0, const vint16mf2_t s1, const vint16mf2_t s2, + const vint16mf2_t s3, const vint16mf2_t s4, const vint16mf2_t s5, + const vint16mf2_t s6, const vint16mf2_t s7, const int16_t *filter, + const vint32m1_t offset_vec, const int32_t round_bits, const size_t vl) { + vint32m1_t sum = __riscv_vwmacc_vx_i32m1(offset_vec, filter[0], s0, vl); + sum = __riscv_vwmacc_vx_i32m1(sum, filter[1], s1, vl); + sum = __riscv_vwmacc_vx_i32m1(sum, filter[2], s2, vl); + sum = __riscv_vwmacc_vx_i32m1(sum, filter[3], s3, vl); + sum = __riscv_vwmacc_vx_i32m1(sum, filter[4], s4, vl); + sum = __riscv_vwmacc_vx_i32m1(sum, filter[5], s5, vl); + sum = __riscv_vwmacc_vx_i32m1(sum, filter[6], s6, vl); + sum = __riscv_vwmacc_vx_i32m1(sum, filter[7], s7, vl); + + vuint32m1_t sum_u32 = + __riscv_vreinterpret_v_i32m1_u32m1(__riscv_vmax_vx_i32m1(sum, 0, vl)); +#if __riscv_v_intrinsic >= 12000 + vuint16mf2_t res = + __riscv_vnclipu_wx_u16mf2(sum_u32, round_bits, __RISCV_VXRM_RDN, vl); +#else + vuint16mf2_t res = __riscv_vnsrl_wx_u16mf2(sum_u32, round_bits, vl); +#endif + return res; +} + +static inline vuint16m1_t highbd_convolve8_8_rvv( + const vint16m1_t s0, const vint16m1_t s1, const vint16m1_t s2, + const vint16m1_t s3, const vint16m1_t s4, const vint16m1_t s5, + const vint16m1_t s6, const vint16m1_t s7, const int16_t *filter, + const vint32m2_t offset_vec, const int32_t round_bits, const size_t vl) { + vint32m2_t sum = __riscv_vwmacc_vx_i32m2(offset_vec, filter[0], s0, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, filter[1], s1, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, filter[2], s2, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, filter[3], s3, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, filter[4], s4, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, filter[5], s5, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, filter[6], s6, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, filter[7], s7, vl); + + vuint32m2_t sum_u32 = + __riscv_vreinterpret_v_i32m2_u32m2(__riscv_vmax_vx_i32m2(sum, 0, vl)); +#if __riscv_v_intrinsic >= 12000 + vuint16m1_t res = + __riscv_vnclipu_wx_u16m1(sum_u32, round_bits, __RISCV_VXRM_RDN, vl); +#else + vuint16m1_t res = __riscv_vnsrl_wx_u16m1(sum_u32, round_bits, vl); +#endif + return res; +} + +static inline vuint16mf2_t highbd_convolve4_4_x_rvv( + const vint16mf2_t s0, const vint16mf2_t s1, const vint16mf2_t s2, + const vint16mf2_t s3, const int16_t *filter, const vint32m1_t offset_vec, + const int round_bits, const size_t vl) { + vint32m1_t sum = __riscv_vwmacc_vx_i32m1(offset_vec, filter[0], s0, vl); + sum = __riscv_vwmacc_vx_i32m1(sum, filter[1], s1, vl); + sum = __riscv_vwmacc_vx_i32m1(sum, filter[2], s2, vl); + sum = __riscv_vwmacc_vx_i32m1(sum, filter[3], s3, vl); + + vuint32m1_t sum_u32 = + __riscv_vreinterpret_v_i32m1_u32m1(__riscv_vmax_vx_i32m1(sum, 0, vl)); +#if __riscv_v_intrinsic >= 12000 + vuint16mf2_t res = + __riscv_vnclipu_wx_u16mf2(sum_u32, round_bits, __RISCV_VXRM_RDN, vl); +#else + vuint16mf2_t res = __riscv_vnsrl_wx_u16mf2(sum_u32, round_bits, vl); +#endif + return res; +} + +static inline void highbd_dist_wtd_convolve_x_rvv( + const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride, + int w, int h, const int16_t *x_filter_ptr, const int offset, + const int round_bits) { + size_t vl = __riscv_vsetvl_e16m1(w); + if (w == 4) { + // 4-tap filters are used for blocks having width == 4. + const int16_t *filter = x_filter_ptr + 2; // Skip first 2 taps + const int16_t *s = (const int16_t *)(src_ptr + 2); + uint16_t *d = dst_ptr; + vint32m1_t offset_vec = __riscv_vmv_v_x_i32m1(offset, vl); + + do { + // Load 4 taps for 4 rows + vint16mf2_t s00, s01, s02, s03; + vint16mf2_t s10, s11, s12, s13; + vint16mf2_t s20, s21, s22, s23; + vint16mf2_t s30, s31, s32, s33; + + load_s16_4x4(s + 0 * src_stride, 1, &s00, &s01, &s02, &s03, vl); + load_s16_4x4(s + 1 * src_stride, 1, &s10, &s11, &s12, &s13, vl); + load_s16_4x4(s + 2 * src_stride, 1, &s20, &s21, &s22, &s23, vl); + load_s16_4x4(s + 3 * src_stride, 1, &s30, &s31, &s32, &s33, vl); + + // Convolve each row + vuint16mf2_t d0 = highbd_convolve4_4_x_rvv(s00, s01, s02, s03, filter, + offset_vec, round_bits, vl); + vuint16mf2_t d1 = highbd_convolve4_4_x_rvv(s10, s11, s12, s13, filter, + offset_vec, round_bits, vl); + vuint16mf2_t d2 = highbd_convolve4_4_x_rvv(s20, s21, s22, s23, filter, + offset_vec, round_bits, vl); + vuint16mf2_t d3 = highbd_convolve4_4_x_rvv(s30, s31, s32, s33, filter, + offset_vec, round_bits, vl); + + // Store results + store_u16_4x4(d, dst_stride, d0, d1, d2, d3, vl); + + s += 4 * src_stride; + d += 4 * dst_stride; + h -= 4; + } while (h != 0); + } else { + // 8-tap filter path + const int16_t *filter = x_filter_ptr; + int height = h; + vint32m2_t offset_vec = __riscv_vmv_v_x_i32m2(offset, vl); + + do { + int width = w; + const int16_t *s = (const int16_t *)src_ptr; + uint16_t *d = dst_ptr; + + do { + vint16m1_t s00, s01, s02, s03, s04, s05, s06, s07; + vint16m1_t s10, s11, s12, s13, s14, s15, s16, s17; + vint16m1_t s20, s21, s22, s23, s24, s25, s26, s27; + vint16m1_t s30, s31, s32, s33, s34, s35, s36, s37; + + // Load elements for each of 4 rows + load_s16_8x8(s + 0 * src_stride, 1, &s00, &s01, &s02, &s03, &s04, &s05, + &s06, &s07, vl); + load_s16_8x8(s + 1 * src_stride, 1, &s10, &s11, &s12, &s13, &s14, &s15, + &s16, &s17, vl); + load_s16_8x8(s + 2 * src_stride, 1, &s20, &s21, &s22, &s23, &s24, &s25, + &s26, &s27, vl); + load_s16_8x8(s + 3 * src_stride, 1, &s30, &s31, &s32, &s33, &s34, &s35, + &s36, &s37, vl); + + // Convolve each row + vuint16m1_t d0 = + highbd_convolve8_8_rvv(s00, s01, s02, s03, s04, s05, s06, s07, + filter, offset_vec, round_bits, vl); + vuint16m1_t d1 = + highbd_convolve8_8_rvv(s10, s11, s12, s13, s14, s15, s16, s17, + filter, offset_vec, round_bits, vl); + vuint16m1_t d2 = + highbd_convolve8_8_rvv(s20, s21, s22, s23, s24, s25, s26, s27, + filter, offset_vec, round_bits, vl); + vuint16m1_t d3 = + highbd_convolve8_8_rvv(s30, s31, s32, s33, s34, s35, s36, s37, + filter, offset_vec, round_bits, vl); + + // Store results + store_u16_8x4(d, dst_stride, d0, d1, d2, d3, vl); + + s += vl; + d += vl; + width -= vl; + } while (width > 0); + src_ptr += 4 * src_stride; + dst_ptr += 4 * dst_stride; + height -= 4; + } while (height != 0); + } +} + +void av1_highbd_dist_wtd_convolve_x_rvv( + const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, + int h, const InterpFilterParams *filter_params_x, const int subpel_x_qn, + ConvolveParams *conv_params, int bd) { + DECLARE_ALIGNED(16, uint16_t, + im_block[(MAX_SB_SIZE + MAX_FILTER_TAP) * MAX_SB_SIZE]); + CONV_BUF_TYPE *dst16 = conv_params->dst; + const int x_filter_taps = get_filter_tap(filter_params_x, subpel_x_qn); + int dst16_stride = conv_params->dst_stride; + const int im_stride = MAX_SB_SIZE; + const int horiz_offset = filter_params_x->taps / 2 - 1; + assert(FILTER_BITS == COMPOUND_ROUND1_BITS); + const int offset_convolve = (1 << (conv_params->round_0 - 1)) + + (1 << (bd + FILTER_BITS)) + + (1 << (bd + FILTER_BITS - 1)); + + const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel( + filter_params_x, subpel_x_qn & SUBPEL_MASK); + + src -= horiz_offset; + int round_bits, shift_bits, offset_bits; + + // horizontal filter + if (bd == 12) { + round_bits = ROUND0_BITS + 2; + shift_bits = ROUND_SHIFT - 2; + offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS - 2; + } else { + round_bits = ROUND0_BITS; + shift_bits = ROUND_SHIFT; + offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + } + + if (conv_params->do_average) { + if (x_filter_taps <= 6 && w != 4) { + highbd_dist_wtd_convolve_x_6tap_rvv(src + 1, src_stride, im_block, + im_stride, w, h, x_filter_ptr, + offset_convolve, round_bits); + } else { + highbd_dist_wtd_convolve_x_rvv(src, src_stride, im_block, im_stride, w, h, + x_filter_ptr, offset_convolve, round_bits); + } + if (conv_params->use_dist_wtd_comp_avg) { + highbd_dist_wtd_comp_avg_rvv(im_block, im_stride, dst, dst_stride, w, h, + conv_params, bd, shift_bits, offset_bits); + } else { + highbd_comp_avg_rvv(im_block, im_stride, dst, dst_stride, w, h, + conv_params, bd, shift_bits, offset_bits); + } + } else { + if (x_filter_taps <= 6 && w != 4) { + highbd_dist_wtd_convolve_x_6tap_rvv(src + 1, src_stride, dst16, + dst16_stride, w, h, x_filter_ptr, + offset_convolve, round_bits); + } else { + highbd_dist_wtd_convolve_x_rvv(src, src_stride, dst16, dst16_stride, w, h, + x_filter_ptr, offset_convolve, round_bits); + } + } +} + +static inline void highbd_dist_wtd_convolve_y_6tap_rvv( + const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride, + int w, int h, const int16_t *y_filter_ptr, const int offset, + const int round_bits) { + const int16_t *filter = y_filter_ptr; + size_t vl = __riscv_vsetvl_e16m1(w); + + if (w == 4) { + const int16_t *s = (const int16_t *)src_ptr; + uint16_t *d = dst_ptr; + vint32m1_t offset_vec = __riscv_vmv_v_x_i32m1(offset, vl); + + // Load initial 5 rows for 6-tap filter + vint16mf2_t s0, s1, s2, s3, s4; + load_s16_4x5(s, src_stride, &s0, &s1, &s2, &s3, &s4, vl); + s += 5 * src_stride; + + do { + // Load next 4 rows + vint16mf2_t s5, s6, s7, s8; + load_s16_4x4(s, src_stride, &s5, &s6, &s7, &s8, vl); + + // Convolve 4 output rows + vuint16mf2_t d0 = highbd_convolve6_4_rvv(s0, s1, s2, s3, s4, s5, filter, + offset_vec, round_bits, vl); + vuint16mf2_t d1 = highbd_convolve6_4_rvv(s1, s2, s3, s4, s5, s6, filter, + offset_vec, round_bits, vl); + vuint16mf2_t d2 = highbd_convolve6_4_rvv(s2, s3, s4, s5, s6, s7, filter, + offset_vec, round_bits, vl); + vuint16mf2_t d3 = highbd_convolve6_4_rvv(s3, s4, s5, s6, s7, s8, filter, + offset_vec, round_bits, vl); + + store_u16_4x4(d, dst_stride, d0, d1, d2, d3, vl); + + // Shift window: s0-s4 become s4-s8 for next iteration + s0 = s4; + s1 = s5; + s2 = s6; + s3 = s7; + s4 = s8; + s += 4 * src_stride; + d += 4 * dst_stride; + h -= 4; + } while (h != 0); + } else { + vint32m2_t offset_vec = __riscv_vmv_v_x_i32m2(offset, vl); + do { + int height = h; + const int16_t *s = (const int16_t *)src_ptr; + uint16_t *d = dst_ptr; + + // Load initial 5 rows for 6-tap filter + vint16m1_t s0, s1, s2, s3, s4; + load_s16_8x5(s, src_stride, &s0, &s1, &s2, &s3, &s4, vl); + s += 5 * src_stride; + + do { + // Load next 4 rows + vint16m1_t s5, s6, s7, s8; + load_s16_8x4(s, src_stride, &s5, &s6, &s7, &s8, vl); + + // Convolve 4 output rows + vuint16m1_t d0 = highbd_convolve6_8_rvv(s0, s1, s2, s3, s4, s5, filter, + offset_vec, round_bits, vl); + vuint16m1_t d1 = highbd_convolve6_8_rvv(s1, s2, s3, s4, s5, s6, filter, + offset_vec, round_bits, vl); + vuint16m1_t d2 = highbd_convolve6_8_rvv(s2, s3, s4, s5, s6, s7, filter, + offset_vec, round_bits, vl); + vuint16m1_t d3 = highbd_convolve6_8_rvv(s3, s4, s5, s6, s7, s8, filter, + offset_vec, round_bits, vl); + + store_u16_8x4(d, dst_stride, d0, d1, d2, d3, vl); + + // Shift window: s0-s4 become s4-s8 for next iteration + s0 = s4; + s1 = s5; + s2 = s6; + s3 = s7; + s4 = s8; + s += 4 * src_stride; + d += 4 * dst_stride; + height -= 4; + } while (height != 0); + src_ptr += vl; + dst_ptr += vl; + w -= vl; + } while (w > 0); + } +} + +static inline vuint16mf2_t highbd_convolve4_4_rvv( + const vint16mf2_t s0, const vint16mf2_t s1, const vint16mf2_t s2, + const vint16mf2_t s3, const int16_t *filter, const int32_t offset, + const int32_t round_bits, size_t vl) { + vint32m1_t sum = __riscv_vwmul_vx_i32m1(s0, filter[0], vl); + sum = __riscv_vwmacc_vx_i32m1(sum, filter[1], s1, vl); + sum = __riscv_vwmacc_vx_i32m1(sum, filter[2], s2, vl); + sum = __riscv_vwmacc_vx_i32m1(sum, filter[3], s3, vl); + sum = __riscv_vadd_vx_i32m1(sum, offset, vl); + + vuint32m1_t sum_u32 = + __riscv_vreinterpret_v_i32m1_u32m1(__riscv_vmax_vx_i32m1(sum, 0, vl)); +#if __riscv_v_intrinsic >= 12000 + vuint16mf2_t res = + __riscv_vnclipu_wx_u16mf2(sum_u32, round_bits, __RISCV_VXRM_RDN, vl); +#else + vuint16mf2_t res = __riscv_vnsrl_wx_u16mf2(sum_u32, round_bits, vl); +#endif + return res; +} + +static inline vuint16m1_t highbd_convolve4_8_rvv( + const vint16m1_t s0, const vint16m1_t s1, const vint16m1_t s2, + const vint16m1_t s3, const int16_t *filter, const int32_t offset, + const int32_t round_bits, size_t vl) { + vint32m2_t sum = __riscv_vwmul_vx_i32m2(s0, filter[0], vl); + sum = __riscv_vwmacc_vx_i32m2(sum, filter[1], s1, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, filter[2], s2, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, filter[3], s3, vl); + sum = __riscv_vadd_vx_i32m2(sum, offset, vl); + + vuint32m2_t sum_u32 = + __riscv_vreinterpret_v_i32m2_u32m2(__riscv_vmax_vx_i32m2(sum, 0, vl)); +#if __riscv_v_intrinsic >= 12000 + vuint16m1_t res = + __riscv_vnclipu_wx_u16m1(sum_u32, round_bits, __RISCV_VXRM_RDN, vl); +#else + vuint16m1_t res = __riscv_vnsrl_wx_u16m1(sum_u32, round_bits, vl); +#endif + return res; +} + +static inline void highbd_dist_wtd_convolve_y_4tap_rvv( + const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride, + int w, int h, const int16_t *y_filter_ptr, const int offset, + const int round_bits) { + const int16_t *filter = y_filter_ptr + 2; + size_t vl = __riscv_vsetvl_e16m1(w); + + if (w == 4) { + const int16_t *s = (const int16_t *)src_ptr; + uint16_t *d = dst_ptr; + + // Load initial 3 rows + vint16mf2_t s0, s1, s2; + load_s16_4x3(s, src_stride, &s0, &s1, &s2, vl); + s += 3 * src_stride; + + do { + // Load next 4 rows + vint16mf2_t s3, s4, s5, s6; + load_s16_4x4(s, src_stride, &s3, &s4, &s5, &s6, vl); + + // Convolve 4 output rows + vuint16mf2_t d0 = highbd_convolve4_4_rvv(s0, s1, s2, s3, filter, offset, + round_bits, vl); + vuint16mf2_t d1 = highbd_convolve4_4_rvv(s1, s2, s3, s4, filter, offset, + round_bits, vl); + vuint16mf2_t d2 = highbd_convolve4_4_rvv(s2, s3, s4, s5, filter, offset, + round_bits, vl); + vuint16mf2_t d3 = highbd_convolve4_4_rvv(s3, s4, s5, s6, filter, offset, + round_bits, vl); + + store_u16_4x4(d, dst_stride, d0, d1, d2, d3, vl); + + // Shift window + s0 = s4; + s1 = s5; + s2 = s6; + + s += 4 * src_stride; + d += 4 * dst_stride; + h -= 4; + } while (h != 0); + } else { + do { + int height = h; + const int16_t *s = (const int16_t *)src_ptr; + uint16_t *d = dst_ptr; + + // Load initial 3 rows + vint16m1_t s0, s1, s2; + load_s16_8x3(s, src_stride, &s0, &s1, &s2, vl); + s += 3 * src_stride; + + do { + // Load next 4 rows + vint16m1_t s3, s4, s5, s6; + load_s16_8x4(s, src_stride, &s3, &s4, &s5, &s6, vl); + + // Convolve 4 output rows + vuint16m1_t d0 = highbd_convolve4_8_rvv(s0, s1, s2, s3, filter, offset, + round_bits, vl); + vuint16m1_t d1 = highbd_convolve4_8_rvv(s1, s2, s3, s4, filter, offset, + round_bits, vl); + vuint16m1_t d2 = highbd_convolve4_8_rvv(s2, s3, s4, s5, filter, offset, + round_bits, vl); + vuint16m1_t d3 = highbd_convolve4_8_rvv(s3, s4, s5, s6, filter, offset, + round_bits, vl); + + store_u16_8x4(d, dst_stride, d0, d1, d2, d3, vl); + + // Shift window + s0 = s4; + s1 = s5; + s2 = s6; + + s += 4 * src_stride; + d += 4 * dst_stride; + height -= 4; + } while (height != 0); + src_ptr += vl; + dst_ptr += vl; + w -= vl; + } while (w > 0); + } +} + +static inline void highbd_dist_wtd_convolve_y_8tap_rvv( + const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride, + int w, int h, const int16_t *y_filter_ptr, const int offset, + const int32_t round_bits) { + const int16_t *filter = y_filter_ptr; + size_t vl = __riscv_vsetvl_e16m1(w); + + if (w == 4) { + const int16_t *s = (const int16_t *)src_ptr; + uint16_t *d = dst_ptr; + vint32m1_t offset_vec = __riscv_vmv_v_x_i32m1(offset, vl); + + // Load initial 7 rows + vint16mf2_t s0, s1, s2, s3, s4, s5, s6; + load_s16_4x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, vl); + s += 7 * src_stride; + + do { + // Load next 4 rows + vint16mf2_t s7, s8, s9, s10; + load_s16_4x4(s, src_stride, &s7, &s8, &s9, &s10, vl); + + // Convolve 4 output rows + vuint16mf2_t d0 = highbd_convolve8_4_rvv( + s0, s1, s2, s3, s4, s5, s6, s7, filter, offset_vec, round_bits, vl); + vuint16mf2_t d1 = highbd_convolve8_4_rvv( + s1, s2, s3, s4, s5, s6, s7, s8, filter, offset_vec, round_bits, vl); + vuint16mf2_t d2 = highbd_convolve8_4_rvv( + s2, s3, s4, s5, s6, s7, s8, s9, filter, offset_vec, round_bits, vl); + vuint16mf2_t d3 = highbd_convolve8_4_rvv( + s3, s4, s5, s6, s7, s8, s9, s10, filter, offset_vec, round_bits, vl); + + store_u16_4x4(d, dst_stride, d0, d1, d2, d3, vl); + + // Shift window + s0 = s4; + s1 = s5; + s2 = s6; + s3 = s7; + s4 = s8; + s5 = s9; + s6 = s10; + + s += 4 * src_stride; + d += 4 * dst_stride; + h -= 4; + } while (h != 0); + } else { + vint32m2_t offset_vec = __riscv_vmv_v_x_i32m2(offset, vl); + do { + int height = h; + const int16_t *s = (const int16_t *)src_ptr; + uint16_t *d = dst_ptr; + + // Load initial 7 rows + vint16m1_t s0, s1, s2, s3, s4, s5, s6; + load_s16_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, vl); + s += 7 * src_stride; + + do { + // Load next 4 rows + vint16m1_t s7, s8, s9, s10; + load_s16_8x4(s, src_stride, &s7, &s8, &s9, &s10, vl); + + // Convolve 4 output rows + vuint16m1_t d0 = highbd_convolve8_8_rvv( + s0, s1, s2, s3, s4, s5, s6, s7, filter, offset_vec, round_bits, vl); + vuint16m1_t d1 = highbd_convolve8_8_rvv( + s1, s2, s3, s4, s5, s6, s7, s8, filter, offset_vec, round_bits, vl); + vuint16m1_t d2 = highbd_convolve8_8_rvv( + s2, s3, s4, s5, s6, s7, s8, s9, filter, offset_vec, round_bits, vl); + vuint16m1_t d3 = + highbd_convolve8_8_rvv(s3, s4, s5, s6, s7, s8, s9, s10, filter, + offset_vec, round_bits, vl); + + store_u16_8x4(d, dst_stride, d0, d1, d2, d3, vl); + + // Shift window + s0 = s4; + s1 = s5; + s2 = s6; + s3 = s7; + s4 = s8; + s5 = s9; + s6 = s10; + + s += 4 * src_stride; + d += 4 * dst_stride; + height -= 4; + } while (height != 0); + src_ptr += vl; + dst_ptr += vl; + w -= vl; + } while (w > 0); + } +} + +void av1_highbd_dist_wtd_convolve_y_rvv( + const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, + int h, const InterpFilterParams *filter_params_y, const int subpel_y_qn, + ConvolveParams *conv_params, int bd) { + DECLARE_ALIGNED(16, uint16_t, + im_block[(MAX_SB_SIZE + MAX_FILTER_TAP) * MAX_SB_SIZE]); + CONV_BUF_TYPE *dst16 = conv_params->dst; + const int y_filter_taps = get_filter_tap(filter_params_y, subpel_y_qn); + int dst16_stride = conv_params->dst_stride; + const int im_stride = MAX_SB_SIZE; + const int vert_offset = filter_params_y->taps / 2 - 1; + assert(FILTER_BITS == COMPOUND_ROUND1_BITS); + const int round_offset_conv = (1 << (conv_params->round_0 - 1)) + + (1 << (bd + FILTER_BITS)) + + (1 << (bd + FILTER_BITS - 1)); + + const int16_t *y_filter_ptr = av1_get_interp_filter_subpel_kernel( + filter_params_y, subpel_y_qn & SUBPEL_MASK); + + src -= vert_offset * src_stride; + int round_bits, shift_bits, offset_bits; + + if (bd == 12) { + round_bits = ROUND0_BITS + 2; + shift_bits = ROUND_SHIFT - 2; + offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS - 2; + } else { + round_bits = ROUND0_BITS; + shift_bits = ROUND_SHIFT; + offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + } + + if (conv_params->do_average) { + if (y_filter_taps <= 4) { + highbd_dist_wtd_convolve_y_4tap_rvv( + src + 2 * src_stride, src_stride, im_block, im_stride, w, h, + y_filter_ptr, round_offset_conv, round_bits); + } else if (y_filter_taps == 6) { + highbd_dist_wtd_convolve_y_6tap_rvv( + src + src_stride, src_stride, im_block, im_stride, w, h, y_filter_ptr, + round_offset_conv, round_bits); + } else { + highbd_dist_wtd_convolve_y_8tap_rvv(src, src_stride, im_block, im_stride, + w, h, y_filter_ptr, round_offset_conv, + round_bits); + } + if (conv_params->use_dist_wtd_comp_avg) { + highbd_dist_wtd_comp_avg_rvv(im_block, im_stride, dst, dst_stride, w, h, + conv_params, bd, shift_bits, offset_bits); + } else { + highbd_comp_avg_rvv(im_block, im_stride, dst, dst_stride, w, h, + conv_params, bd, shift_bits, offset_bits); + } + } else { + if (y_filter_taps <= 4) { + highbd_dist_wtd_convolve_y_4tap_rvv( + src + 2 * src_stride, src_stride, dst16, dst16_stride, w, h, + y_filter_ptr, round_offset_conv, round_bits); + } else if (y_filter_taps == 6) { + highbd_dist_wtd_convolve_y_6tap_rvv(src + src_stride, src_stride, dst16, + dst16_stride, w, h, y_filter_ptr, + round_offset_conv, round_bits); + } else { + highbd_dist_wtd_convolve_y_8tap_rvv(src, src_stride, dst16, dst16_stride, + w, h, y_filter_ptr, round_offset_conv, + round_bits); + } + } +} + +static inline void highbd_2d_copy_rvv(const uint16_t *src_ptr, int src_stride, + uint16_t *dst_ptr, int dst_stride, int w, + int h, int round_bits, const int offset) { + assert(h % 4 == 0); + + const uint16_t round_val = (1 << round_bits); + int height = h; + + if (w <= 4) { + const size_t vl = __riscv_vsetvl_e16mf2(w); + vuint16mf2_t vec_offset = __riscv_vmv_v_x_u16mf2(offset, vl); + + do { + vuint16mf2_t s0, s1, s2, s3; + load_u16_4x4(src_ptr, src_stride, &s0, &s1, &s2, &s3, vl); + + vuint16mf2_t d0 = __riscv_vmacc_vx_u16mf2(vec_offset, round_val, s0, vl); + vuint16mf2_t d1 = __riscv_vmacc_vx_u16mf2(vec_offset, round_val, s1, vl); + vuint16mf2_t d2 = __riscv_vmacc_vx_u16mf2(vec_offset, round_val, s2, vl); + vuint16mf2_t d3 = __riscv_vmacc_vx_u16mf2(vec_offset, round_val, s3, vl); + + store_u16_4x4(dst_ptr, dst_stride, d0, d1, d2, d3, vl); + + src_ptr += 4 * src_stride; + dst_ptr += 4 * dst_stride; + height -= 4; + } while (height > 0); + } else { + size_t vl_max = __riscv_vsetvlmax_e16m1(); + vuint16m1_t vec_offset = __riscv_vmv_v_x_u16m1(offset, vl_max); + + do { + const uint16_t *s = src_ptr; + uint16_t *d = dst_ptr; + int width = w; + + do { + size_t vl = __riscv_vsetvl_e16m1(width); + vuint16m1_t s0, s1, s2, s3; + load_u16_8x4(s, src_stride, &s0, &s1, &s2, &s3, vl); + + // apply shift and add round offset + vuint16m1_t d0 = __riscv_vmacc_vx_u16m1(vec_offset, round_val, s0, vl); + vuint16m1_t d1 = __riscv_vmacc_vx_u16m1(vec_offset, round_val, s1, vl); + vuint16m1_t d2 = __riscv_vmacc_vx_u16m1(vec_offset, round_val, s2, vl); + vuint16m1_t d3 = __riscv_vmacc_vx_u16m1(vec_offset, round_val, s3, vl); + + // store results + store_u16_8x4(d, dst_stride, d0, d1, d2, d3, vl); + + s += vl; + d += vl; + width -= vl; + } while (width > 0); + src_ptr += 4 * src_stride; + dst_ptr += 4 * dst_stride; + height -= 4; + } while (height > 0); + } +} + +void av1_highbd_dist_wtd_convolve_2d_copy_rvv(const uint16_t *src, + int src_stride, uint16_t *dst, + int dst_stride, int w, int h, + ConvolveParams *conv_params, + int bd) { + DECLARE_ALIGNED(16, uint16_t, + im_block[(MAX_SB_SIZE + MAX_FILTER_TAP) * MAX_SB_SIZE]); + + const int im_stride = MAX_SB_SIZE; + CONV_BUF_TYPE *dst16 = conv_params->dst; + int dst16_stride = conv_params->dst_stride; + const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0; + const int round_offset = (1 << (offset_bits - conv_params->round_1)) + + (1 << (offset_bits - conv_params->round_1 - 1)); + const int round_bits = + 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1; + assert(round_bits >= 0); + + if (conv_params->do_average) { + highbd_2d_copy_rvv(src, src_stride, im_block, im_stride, w, h, round_bits, + round_offset); + } else { + highbd_2d_copy_rvv(src, src_stride, dst16, dst16_stride, w, h, round_bits, + round_offset); + } + + if (conv_params->do_average) { + int shift_bits; + if (bd == 12) { + shift_bits = ROUND_SHIFT - 2; + } else { + shift_bits = ROUND_SHIFT; + } + if (conv_params->use_dist_wtd_comp_avg) { + highbd_dist_wtd_comp_avg_rvv(im_block, im_stride, dst, dst_stride, w, h, + conv_params, bd, shift_bits, offset_bits); + } else { + highbd_comp_avg_rvv(im_block, im_stride, dst, dst_stride, w, h, + conv_params, bd, shift_bits, offset_bits); + } + } +} + +static inline vuint16mf2_t highbd_convolve6_4_2d_v_rvv( + const vint16mf2_t s0, const vint16mf2_t s1, const vint16mf2_t s2, + const vint16mf2_t s3, const vint16mf2_t s4, const vint16mf2_t s5, + const int16_t *y_filter, const vint32m1_t offset, size_t vl) { + // Values at indices 0 and 7 of y_filter are zero. + // Accumulate widening multiplications into the offset vector. + vint32m1_t sum = __riscv_vwmacc_vx_i32m1(offset, y_filter[1], s0, vl); + sum = __riscv_vwmacc_vx_i32m1(sum, y_filter[2], s1, vl); + sum = __riscv_vwmacc_vx_i32m1(sum, y_filter[3], s2, vl); + sum = __riscv_vwmacc_vx_i32m1(sum, y_filter[4], s3, vl); + sum = __riscv_vwmacc_vx_i32m1(sum, y_filter[5], s4, vl); + sum = __riscv_vwmacc_vx_i32m1(sum, y_filter[6], s5, vl); + + vuint32m1_t sum_u32 = + __riscv_vreinterpret_v_i32m1_u32m1(__riscv_vmax_vx_i32m1(sum, 0, vl)); +#if __riscv_v_intrinsic >= 12000 + return __riscv_vnclipu_wx_u16mf2(sum_u32, COMPOUND_ROUND1_BITS, + __RISCV_VXRM_RNU, vl); +#else + return __riscv_vnclipu_wx_u16mf2(sum_u32, COMPOUND_ROUND1_BITS, vl); +#endif +} + +static inline vuint16m1_t highbd_convolve6_8_2d_v_rvv( + const vint16m1_t s0, const vint16m1_t s1, const vint16m1_t s2, + const vint16m1_t s3, const vint16m1_t s4, const vint16m1_t s5, + const int16_t *y_filter, const vint32m2_t offset, size_t vl) { + // Values at indices 0 and 7 of y_filter are zero. + // Perform widening multiply-accumulate starting with the offset vector. + vint32m2_t sum = __riscv_vwmacc_vx_i32m2(offset, y_filter[1], s0, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, y_filter[2], s1, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, y_filter[3], s2, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, y_filter[4], s3, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, y_filter[5], s4, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, y_filter[6], s5, vl); + + vuint32m2_t sum_u32 = + __riscv_vreinterpret_v_i32m2_u32m2(__riscv_vmax_vx_i32m2(sum, 0, vl)); +#if __riscv_v_intrinsic >= 12000 + return __riscv_vnclipu_wx_u16m1(sum_u32, COMPOUND_ROUND1_BITS, + __RISCV_VXRM_RNU, vl); +#else + return __riscv_vnclipu_wx_u16m1(sum_u32, COMPOUND_ROUND1_BITS, vl); +#endif +} + +static inline void highbd_dist_wtd_convolve_2d_vert_6tap_rvv( + const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride, + int w, int h, const int16_t *y_filter_ptr, int offset) { + size_t vl = __riscv_vsetvl_e16m1(w); + if (w == 4) { + // Broadcast offset to a 32-bit vector + vint32m1_t offset_vec = __riscv_vmv_v_x_i32m1(offset, vl); + const int16_t *s = (const int16_t *)src_ptr; + uint16_t *d = dst_ptr; + + // Load initial 5 rows + vint16mf2_t s0, s1, s2, s3, s4; + load_s16_4x5(s, src_stride, &s0, &s1, &s2, &s3, &s4, vl); + s += 5 * src_stride; + + do { + vint16mf2_t s5, s6, s7, s8; + load_s16_4x4(s, src_stride, &s5, &s6, &s7, &s8, vl); + + // Perform 6-tap convolution for 4 rows + vuint16mf2_t d0 = highbd_convolve6_4_2d_v_rvv( + s0, s1, s2, s3, s4, s5, y_filter_ptr, offset_vec, vl); + vuint16mf2_t d1 = highbd_convolve6_4_2d_v_rvv( + s1, s2, s3, s4, s5, s6, y_filter_ptr, offset_vec, vl); + vuint16mf2_t d2 = highbd_convolve6_4_2d_v_rvv( + s2, s3, s4, s5, s6, s7, y_filter_ptr, offset_vec, vl); + vuint16mf2_t d3 = highbd_convolve6_4_2d_v_rvv( + s3, s4, s5, s6, s7, s8, y_filter_ptr, offset_vec, vl); + + store_u16_4x4(d, dst_stride, d0, d1, d2, d3, vl); + + // Update sliding window state + s0 = s4; + s1 = s5; + s2 = s6; + s3 = s7; + s4 = s8; + s += 4 * src_stride; + d += 4 * dst_stride; + h -= 4; + } while (h != 0); + } else { + // w > 4 case + do { + vint32m2_t offset_vec = __riscv_vmv_v_x_i32m2(offset, vl); + const int16_t *s = (const int16_t *)src_ptr; + uint16_t *d = dst_ptr; + int height = h; + + vint16m1_t s0, s1, s2, s3, s4; + load_s16_8x5(s, src_stride, &s0, &s1, &s2, &s3, &s4, vl); + s += 5 * src_stride; + + do { + vint16m1_t s5, s6, s7, s8; + load_s16_8x4(s, src_stride, &s5, &s6, &s7, &s8, vl); + + vuint16m1_t d0 = highbd_convolve6_8_2d_v_rvv( + s0, s1, s2, s3, s4, s5, y_filter_ptr, offset_vec, vl); + vuint16m1_t d1 = highbd_convolve6_8_2d_v_rvv( + s1, s2, s3, s4, s5, s6, y_filter_ptr, offset_vec, vl); + vuint16m1_t d2 = highbd_convolve6_8_2d_v_rvv( + s2, s3, s4, s5, s6, s7, y_filter_ptr, offset_vec, vl); + vuint16m1_t d3 = highbd_convolve6_8_2d_v_rvv( + s3, s4, s5, s6, s7, s8, y_filter_ptr, offset_vec, vl); + + store_u16_8x4(d, dst_stride, d0, d1, d2, d3, vl); + + s0 = s4; + s1 = s5; + s2 = s6; + s3 = s7; + s4 = s8; + s += 4 * src_stride; + d += 4 * dst_stride; + height -= 4; + } while (height != 0); + + src_ptr += vl; + dst_ptr += vl; + w -= (int)vl; + } while (w > 0); + } +} + +static inline vuint16mf2_t highbd_convolve8_4_2d_v_rvv( + const vint16mf2_t s0, const vint16mf2_t s1, const vint16mf2_t s2, + const vint16mf2_t s3, const vint16mf2_t s4, const vint16mf2_t s5, + const vint16mf2_t s6, const vint16mf2_t s7, const int16_t *y_filter, + const vint32m1_t offset, size_t vl) { + // Perform widening multiply-accumulate for all 8 taps. + // Accumulate directly into the offset vector. + vint32m1_t sum = __riscv_vwmacc_vx_i32m1(offset, y_filter[0], s0, vl); + sum = __riscv_vwmacc_vx_i32m1(sum, y_filter[1], s1, vl); + sum = __riscv_vwmacc_vx_i32m1(sum, y_filter[2], s2, vl); + sum = __riscv_vwmacc_vx_i32m1(sum, y_filter[3], s3, vl); + sum = __riscv_vwmacc_vx_i32m1(sum, y_filter[4], s4, vl); + sum = __riscv_vwmacc_vx_i32m1(sum, y_filter[5], s5, vl); + sum = __riscv_vwmacc_vx_i32m1(sum, y_filter[6], s6, vl); + sum = __riscv_vwmacc_vx_i32m1(sum, y_filter[7], s7, vl); + + // NEON vqrshrun_n_s32 -> RVV vnclipu_wx (with default RNU rounding) + vuint32m1_t sum_u32 = + __riscv_vreinterpret_v_i32m1_u32m1(__riscv_vmax_vx_i32m1(sum, 0, vl)); +#if __riscv_v_intrinsic >= 12000 + return __riscv_vnclipu_wx_u16mf2(sum_u32, COMPOUND_ROUND1_BITS, + __RISCV_VXRM_RNU, vl); +#else + return __riscv_vnclipu_wx_u16mf2(sum_u32, COMPOUND_ROUND1_BITS, vl); +#endif +} + +static inline vuint16m1_t highbd_convolve8_8_2d_v_rvv( + const vint16m1_t s0, const vint16m1_t s1, const vint16m1_t s2, + const vint16m1_t s3, const vint16m1_t s4, const vint16m1_t s5, + const vint16m1_t s6, const vint16m1_t s7, const int16_t *y_filter, + const vint32m2_t offset, size_t vl) { + // Perform widening multiply-accumulate for all 8 taps. + vint32m2_t sum = __riscv_vwmacc_vx_i32m2(offset, y_filter[0], s0, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, y_filter[1], s1, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, y_filter[2], s2, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, y_filter[3], s3, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, y_filter[4], s4, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, y_filter[5], s5, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, y_filter[6], s6, vl); + sum = __riscv_vwmacc_vx_i32m2(sum, y_filter[7], s7, vl); + + // NEON vqrshrun_n_s32 -> RVV vnclipu_wx (with default RNU rounding) + vuint32m2_t sum_u32 = + __riscv_vreinterpret_v_i32m2_u32m2(__riscv_vmax_vx_i32m2(sum, 0, vl)); +#if __riscv_v_intrinsic >= 12000 + return __riscv_vnclipu_wx_u16m1(sum_u32, COMPOUND_ROUND1_BITS, + __RISCV_VXRM_RNU, vl); +#else + return __riscv_vnclipu_wx_u16m1(sum_u32, COMPOUND_ROUND1_BITS, vl); +#endif +} + +static inline void highbd_dist_wtd_convolve_2d_vert_8tap_rvv( + const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride, + int w, int h, const int16_t *y_filter_ptr, int offset) { + size_t vl = __riscv_vsetvl_e16m1(w); + if (w == 4) { + vint32m1_t offset_vec = __riscv_vmv_v_x_i32m1(offset, vl); + const int16_t *s = (const int16_t *)src_ptr; + uint16_t *d = dst_ptr; + + // Load initial 7 rows for 8-tap filter + vint16mf2_t s0, s1, s2, s3, s4, s5, s6; + load_s16_4x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, vl); + s += 7 * src_stride; + + do { + vint16mf2_t s7, s8, s9, s10; + load_s16_4x4(s, src_stride, &s7, &s8, &s9, &s10, vl); + + // Perform 8-tap convolution for 4 rows + vuint16mf2_t d0 = highbd_convolve8_4_2d_v_rvv( + s0, s1, s2, s3, s4, s5, s6, s7, y_filter_ptr, offset_vec, vl); + vuint16mf2_t d1 = highbd_convolve8_4_2d_v_rvv( + s1, s2, s3, s4, s5, s6, s7, s8, y_filter_ptr, offset_vec, vl); + vuint16mf2_t d2 = highbd_convolve8_4_2d_v_rvv( + s2, s3, s4, s5, s6, s7, s8, s9, y_filter_ptr, offset_vec, vl); + vuint16mf2_t d3 = highbd_convolve8_4_2d_v_rvv( + s3, s4, s5, s6, s7, s8, s9, s10, y_filter_ptr, offset_vec, vl); + + store_u16_4x4(d, dst_stride, d0, d1, d2, d3, vl); + + // Update sliding window state + s0 = s4; + s1 = s5; + s2 = s6; + s3 = s7; + s4 = s8; + s5 = s9; + s6 = s10; + s += 4 * src_stride; + d += 4 * dst_stride; + h -= 4; + } while (h != 0); + } else { + // w > 4 case + vint32m2_t offset_vec = __riscv_vmv_v_x_i32m2(offset, vl); + do { + const int16_t *s = (const int16_t *)src_ptr; + uint16_t *d = dst_ptr; + int height = h; + + vint16m1_t s0, s1, s2, s3, s4, s5, s6; + load_s16_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, vl); + s += 7 * src_stride; + + do { + vint16m1_t s7, s8, s9, s10; + load_s16_8x4(s, src_stride, &s7, &s8, &s9, &s10, vl); + + vuint16m1_t d0 = highbd_convolve8_8_2d_v_rvv( + s0, s1, s2, s3, s4, s5, s6, s7, y_filter_ptr, offset_vec, vl); + vuint16m1_t d1 = highbd_convolve8_8_2d_v_rvv( + s1, s2, s3, s4, s5, s6, s7, s8, y_filter_ptr, offset_vec, vl); + vuint16m1_t d2 = highbd_convolve8_8_2d_v_rvv( + s2, s3, s4, s5, s6, s7, s8, s9, y_filter_ptr, offset_vec, vl); + vuint16m1_t d3 = highbd_convolve8_8_2d_v_rvv( + s3, s4, s5, s6, s7, s8, s9, s10, y_filter_ptr, offset_vec, vl); + + store_u16_8x4(d, dst_stride, d0, d1, d2, d3, vl); + + s0 = s4; + s1 = s5; + s2 = s6; + s3 = s7; + s4 = s8; + s5 = s9; + s6 = s10; + s += 4 * src_stride; + d += 4 * dst_stride; + height -= 4; + } while (height != 0); + + src_ptr += vl; + dst_ptr += vl; + w -= (int)vl; + } while (w > 0); + } +} + +static inline void highbd_dist_wtd_convolve_2d_horiz_6tap_rvv( + const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride, + int w, int h, const int16_t *x_filter_ptr, const int offset, + const int32_t round_bits) { + const int16_t *filter = x_filter_ptr; + + size_t vl = __riscv_vsetvl_e16m1(w); + vint32m2_t offset_vec = __riscv_vmv_v_x_i32m2(offset, vl); + do { + const int16_t *s = (const int16_t *)src_ptr; + uint16_t *d = dst_ptr; + int height = h; + + // Process 4 rows at a time to reduce loop overhead + while (height >= 4) { + vint16m1_t s00, s01, s02, s03, s04, s05; + vint16m1_t s10, s11, s12, s13, s14, s15; + vint16m1_t s20, s21, s22, s23, s24, s25; + vint16m1_t s30, s31, s32, s33, s34, s35; + + // Load 6 vectors per row with stride 1 (horizontal overlap) + load_s16_8x6(s + 0 * src_stride, 1, &s00, &s01, &s02, &s03, &s04, &s05, + vl); + load_s16_8x6(s + 1 * src_stride, 1, &s10, &s11, &s12, &s13, &s14, &s15, + vl); + load_s16_8x6(s + 2 * src_stride, 1, &s20, &s21, &s22, &s23, &s24, &s25, + vl); + load_s16_8x6(s + 3 * src_stride, 1, &s30, &s31, &s32, &s33, &s34, &s35, + vl); + + vuint16m1_t d0 = highbd_convolve6_8_rvv( + s00, s01, s02, s03, s04, s05, filter, offset_vec, round_bits, vl); + vuint16m1_t d1 = highbd_convolve6_8_rvv( + s10, s11, s12, s13, s14, s15, filter, offset_vec, round_bits, vl); + vuint16m1_t d2 = highbd_convolve6_8_rvv( + s20, s21, s22, s23, s24, s25, filter, offset_vec, round_bits, vl); + vuint16m1_t d3 = highbd_convolve6_8_rvv( + s30, s31, s32, s33, s34, s35, filter, offset_vec, round_bits, vl); + + store_u16_8x4(d, dst_stride, d0, d1, d2, d3, vl); + + s += 4 * src_stride; + d += 4 * dst_stride; + height -= 4; + } + + // Handle remaining rows + while (height > 0) { + vint16m1_t s00, s01, s02, s03, s04, s05; + load_s16_8x6(s, 1, &s00, &s01, &s02, &s03, &s04, &s05, vl); + vuint16m1_t d0 = highbd_convolve6_8_rvv( + s00, s01, s02, s03, s04, s05, filter, offset_vec, round_bits, vl); + __riscv_vse16_v_u16m1(d, d0, vl); + + s += src_stride; + d += dst_stride; + height--; + } + + src_ptr += vl; + dst_ptr += vl; + w -= (int)vl; + } while (w > 0); +} + +static inline void highbd_dist_wtd_convolve_2d_horiz_rvv( + const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride, + int w, int h, const int16_t *x_filter_ptr, const int offset, + const int32_t round_bits) { + assert(h >= 5); + + if (w == 4) { + size_t vl = __riscv_vsetvl_e16mf2(4); + vint32m1_t offset_vec = __riscv_vmv_v_x_i32m1(offset, vl); + const int16_t *filter = x_filter_ptr + 2; // Use middle 4 taps + const int16_t *s = (const int16_t *)(src_ptr + 1); + uint16_t *d = dst_ptr; + + while (h >= 4) { + vint16mf2_t s00, s01, s02, s03; + vint16mf2_t s10, s11, s12, s13; + vint16mf2_t s20, s21, s22, s23; + vint16mf2_t s30, s31, s32, s33; + + load_s16_4x4(s + 0 * src_stride, 1, &s00, &s01, &s02, &s03, vl); + load_s16_4x4(s + 1 * src_stride, 1, &s10, &s11, &s12, &s13, vl); + load_s16_4x4(s + 2 * src_stride, 1, &s20, &s21, &s22, &s23, vl); + load_s16_4x4(s + 3 * src_stride, 1, &s30, &s31, &s32, &s33, vl); + + vuint16mf2_t d0 = highbd_convolve4_4_x_rvv(s00, s01, s02, s03, filter, + offset_vec, round_bits, vl); + vuint16mf2_t d1 = highbd_convolve4_4_x_rvv(s10, s11, s12, s13, filter, + offset_vec, round_bits, vl); + vuint16mf2_t d2 = highbd_convolve4_4_x_rvv(s20, s21, s22, s23, filter, + offset_vec, round_bits, vl); + vuint16mf2_t d3 = highbd_convolve4_4_x_rvv(s30, s31, s32, s33, filter, + offset_vec, round_bits, vl); + + store_u16_4x4(d, dst_stride, d0, d1, d2, d3, vl); + + s += 4 * src_stride; + d += 4 * dst_stride; + h -= 4; + } + + while (h > 0) { + vint16mf2_t s00, s01, s02, s03; + load_s16_4x4(s, 1, &s00, &s01, &s02, &s03, vl); + vuint16mf2_t d0 = highbd_convolve4_4_x_rvv(s00, s01, s02, s03, filter, + offset_vec, round_bits, vl); + __riscv_vse16_v_u16mf2(d, d0, vl); + s += src_stride; + d += dst_stride; + h--; + } + } else { + // w > 4 case using 8-tap filter + const int16_t *filter = x_filter_ptr; + size_t vl = __riscv_vsetvl_e16m1(w); + vint32m2_t offset_vec = __riscv_vmv_v_x_i32m2(offset, vl); + do { + const int16_t *s_base = (const int16_t *)src_ptr; + uint16_t *d_base = dst_ptr; + int height = h; + + while (height >= 4) { + vint16m1_t s00, s01, s02, s03, s04, s05, s06, s07; + vint16m1_t s10, s11, s12, s13, s14, s15, s16, s17; + vint16m1_t s20, s21, s22, s23, s24, s25, s26, s27; + vint16m1_t s30, s31, s32, s33, s34, s35, s36, s37; + + load_s16_8x8(s_base + 0 * src_stride, 1, &s00, &s01, &s02, &s03, &s04, + &s05, &s06, &s07, vl); + load_s16_8x8(s_base + 1 * src_stride, 1, &s10, &s11, &s12, &s13, &s14, + &s15, &s16, &s17, vl); + load_s16_8x8(s_base + 2 * src_stride, 1, &s20, &s21, &s22, &s23, &s24, + &s25, &s26, &s27, vl); + load_s16_8x8(s_base + 3 * src_stride, 1, &s30, &s31, &s32, &s33, &s34, + &s35, &s36, &s37, vl); + + vuint16m1_t d0 = + highbd_convolve8_8_rvv(s00, s01, s02, s03, s04, s05, s06, s07, + filter, offset_vec, round_bits, vl); + vuint16m1_t d1 = + highbd_convolve8_8_rvv(s10, s11, s12, s13, s14, s15, s16, s17, + filter, offset_vec, round_bits, vl); + vuint16m1_t d2 = + highbd_convolve8_8_rvv(s20, s21, s22, s23, s24, s25, s26, s27, + filter, offset_vec, round_bits, vl); + vuint16m1_t d3 = + highbd_convolve8_8_rvv(s30, s31, s32, s33, s34, s35, s36, s37, + filter, offset_vec, round_bits, vl); + + store_u16_8x4(d_base, dst_stride, d0, d1, d2, d3, vl); + + s_base += 4 * src_stride; + d_base += 4 * dst_stride; + height -= 4; + } + + while (height > 0) { + vint16m1_t s00, s01, s02, s03, s04, s05, s06, s07; + load_s16_8x8(s_base, 1, &s00, &s01, &s02, &s03, &s04, &s05, &s06, &s07, + vl); + vuint16m1_t d0 = + highbd_convolve8_8_rvv(s00, s01, s02, s03, s04, s05, s06, s07, + filter, offset_vec, round_bits, vl); + __riscv_vse16_v_u16m1(d_base, d0, vl); + s_base += src_stride; + d_base += dst_stride; + height--; + } + + src_ptr += vl; + dst_ptr += vl; + w -= (int)vl; + } while (w > 0); + } +} + +void av1_highbd_dist_wtd_convolve_2d_rvv( + const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, + int h, const InterpFilterParams *filter_params_x, + const InterpFilterParams *filter_params_y, const int subpel_x_qn, + const int subpel_y_qn, ConvolveParams *conv_params, int bd) { + DECLARE_ALIGNED(16, uint16_t, + im_block[(MAX_SB_SIZE + MAX_FILTER_TAP) * MAX_SB_SIZE]); + DECLARE_ALIGNED(16, uint16_t, + im_block2[(MAX_SB_SIZE + MAX_FILTER_TAP) * MAX_SB_SIZE]); + + CONV_BUF_TYPE *dst16 = conv_params->dst; + int dst16_stride = conv_params->dst_stride; + const int x_filter_taps = get_filter_tap(filter_params_x, subpel_x_qn); + const int clamped_x_taps = x_filter_taps < 6 ? 6 : x_filter_taps; + const int y_filter_taps = get_filter_tap(filter_params_y, subpel_y_qn); + const int clamped_y_taps = y_filter_taps < 6 ? 6 : y_filter_taps; + + const int im_h = h + clamped_y_taps - 1; + const int im_stride = MAX_SB_SIZE; + const int vert_offset = clamped_y_taps / 2 - 1; + const int horiz_offset = clamped_x_taps / 2 - 1; + + const int round_offset_conv_x = + (1 << (bd + FILTER_BITS - 1)) + (1 << (conv_params->round_0 - 1)); + const int y_offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0; + const int round_offset_conv_y = (1 << y_offset_bits); + + const uint16_t *src_ptr = src - vert_offset * src_stride - horiz_offset; + + const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel( + filter_params_x, subpel_x_qn & SUBPEL_MASK); + const int16_t *y_filter_ptr = av1_get_interp_filter_subpel_kernel( + filter_params_y, subpel_y_qn & SUBPEL_MASK); + + // 1. Horizontal Filter Pass + int round_bits; + if (bd == 12) { + round_bits = ROUND0_BITS + 2; + } else { + round_bits = ROUND0_BITS; + } + if (x_filter_taps <= 6 && w != 4) { + highbd_dist_wtd_convolve_2d_horiz_6tap_rvv(src_ptr, src_stride, im_block, + im_stride, w, im_h, x_filter_ptr, + round_offset_conv_x, round_bits); + } else { + highbd_dist_wtd_convolve_2d_horiz_rvv(src_ptr, src_stride, im_block, + im_stride, w, im_h, x_filter_ptr, + round_offset_conv_x, round_bits); + } + + // 2. Vertical Filter Pass + if (y_filter_taps <= 6) { + if (conv_params->do_average) { + highbd_dist_wtd_convolve_2d_vert_6tap_rvv(im_block, im_stride, im_block2, + im_stride, w, h, y_filter_ptr, + round_offset_conv_y); + } else { + highbd_dist_wtd_convolve_2d_vert_6tap_rvv( + im_block, im_stride, dst16, dst16_stride, w, h, y_filter_ptr, + round_offset_conv_y); + } + } else { + if (conv_params->do_average) { + highbd_dist_wtd_convolve_2d_vert_8tap_rvv(im_block, im_stride, im_block2, + im_stride, w, h, y_filter_ptr, + round_offset_conv_y); + } else { + highbd_dist_wtd_convolve_2d_vert_8tap_rvv( + im_block, im_stride, dst16, dst16_stride, w, h, y_filter_ptr, + round_offset_conv_y); + } + } + + // 3. Compound Averaging + if (conv_params->do_average) { + int shift_bits, offset_bits; + if (bd == 12) { + shift_bits = ROUND_SHIFT - 2; + offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS - 2; + } else { + shift_bits = ROUND_SHIFT; + offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + } + if (conv_params->use_dist_wtd_comp_avg) { + highbd_dist_wtd_comp_avg_rvv(im_block2, im_stride, dst, dst_stride, w, h, + conv_params, bd, shift_bits, offset_bits); + } else { + highbd_comp_avg_rvv(im_block2, im_stride, dst, dst_stride, w, h, + conv_params, bd, shift_bits, offset_bits); + } + } +}
diff --git a/av1/common/riscv/highbd_compound_convolve_rvv.h b/av1/common/riscv/highbd_compound_convolve_rvv.h new file mode 100644 index 0000000..67ae66f --- /dev/null +++ b/av1/common/riscv/highbd_compound_convolve_rvv.h
@@ -0,0 +1,190 @@ +/* + * Copyright (c) 2026, Alliance for Open Media. All rights reserved. + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_AV1_COMMON_RISCV_HIGHBD_COMPOUND_CONVOLVE_RVV_H_ +#define AOM_AV1_COMMON_RISCV_HIGHBD_COMPOUND_CONVOLVE_RVV_H_ + +#include <riscv_vector.h> +#include <assert.h> + +#include "config/aom_config.h" +#include "config/av1_rtcd.h" +#include "aom_dsp/riscv/mem_rvv.h" +#include "aom_ports/mem.h" + +#define ROUND_SHIFT 2 * FILTER_BITS - ROUND0_BITS - COMPOUND_ROUND1_BITS + +static inline void highbd_comp_avg_rvv(const uint16_t *src_ptr, int src_stride, + uint16_t *dst_ptr, int dst_stride, int w, + int h, ConvolveParams *conv_params, + const int bd, const int shift_bits, + const int offset_bits) { + const uint16_t offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + const uint16_t max_val = (1 << bd) - 1; + const int32_t round_shift_val = 1 << (shift_bits - 1); + + CONV_BUF_TYPE *ref_ptr = conv_params->dst; + const int ref_stride = conv_params->dst_stride; + const size_t vl = __riscv_vsetvl_e16m1(w); + + if (w == 4) { + do { + vuint16m1_t src = __riscv_vle16_v_u16m1(src_ptr, vl); + vuint16m1_t ref = __riscv_vle16_v_u16m1(ref_ptr, vl); + +#if __riscv_v_intrinsic >= 12000 + vuint16m1_t avg = __riscv_vaaddu_vv_u16m1(src, ref, __RISCV_VXRM_RDN, vl); +#else + vuint16m1_t avg = + __riscv_vnsrl_wx_u16m1(__riscv_vwaddu_vv_u32m2(src, ref, vl), 1, vl); +#endif + vint32m2_t diff = __riscv_vreinterpret_v_u32m2_i32m2( + __riscv_vwsubu_vx_u32m2(avg, offset, vl)); + diff = __riscv_vadd_vx_i32m2(diff, round_shift_val, vl); + vint16m1_t d0 = __riscv_vnsra_wx_i16m1(diff, shift_bits, vl); + vuint16m1_t d0_u16 = + __riscv_vreinterpret_v_i16m1_u16m1(__riscv_vmax_vx_i16m1(d0, 0, vl)); + d0_u16 = __riscv_vminu_vx_u16m1(d0_u16, max_val, vl); + + __riscv_vse16_v_u16m1(dst_ptr, d0_u16, vl); + + src_ptr += src_stride; + ref_ptr += ref_stride; + dst_ptr += dst_stride; + } while (--h != 0); + } else { + do { + int width = w; + const uint16_t *src_line = src_ptr; + const uint16_t *ref_line = (const uint16_t *)ref_ptr; + uint16_t *dst_line = dst_ptr; + + while (width > 0) { + vuint16m1_t s = __riscv_vle16_v_u16m1(src_line, vl); + vuint16m1_t r = __riscv_vle16_v_u16m1(ref_line, vl); + +#if __riscv_v_intrinsic >= 12000 + vuint16m1_t avg = __riscv_vaaddu_vv_u16m1(s, r, __RISCV_VXRM_RDN, vl); +#else + vuint16m1_t avg = + __riscv_vnsrl_wx_u16m1(__riscv_vwaddu_vv_u32m2(s, r, vl), 1, vl); +#endif + vint32m2_t diff = __riscv_vreinterpret_v_u32m2_i32m2( + __riscv_vwsubu_vx_u32m2(avg, offset, vl)); + diff = __riscv_vadd_vx_i32m2(diff, round_shift_val, vl); + vint16m1_t d0 = __riscv_vnsra_wx_i16m1(diff, shift_bits, vl); + vuint16m1_t d0_u16 = __riscv_vreinterpret_v_i16m1_u16m1( + __riscv_vmax_vx_i16m1(d0, 0, vl)); + d0_u16 = __riscv_vminu_vx_u16m1(d0_u16, max_val, vl); + + __riscv_vse16_v_u16m1(dst_line, d0_u16, vl); + + src_line += vl; + ref_line += vl; + dst_line += vl; + width -= vl; + } + + src_ptr += src_stride; + ref_ptr += ref_stride; + dst_ptr += dst_stride; + } while (--h != 0); + } +} + +static inline void highbd_dist_wtd_comp_avg_rvv( + const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride, + int w, int h, ConvolveParams *conv_params, const int bd, + const int shift_bits, const int offset_bits) { + const int offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + const uint16_t max_val = (1 << bd) - 1; + const uint16_t fwd = conv_params->fwd_offset; + const uint16_t bck = conv_params->bck_offset; + + CONV_BUF_TYPE *ref_ptr = conv_params->dst; + const int ref_stride = conv_params->dst_stride; + + const size_t vl = __riscv_vsetvl_e16m1(w); + vuint32m2_t offset_vec = __riscv_vmv_v_x_u32m2((uint32_t)offset, vl); + if (w == 4) { + do { + vuint16m1_t src = __riscv_vle16_v_u16m1(src_ptr, vl); + vuint16m1_t ref = __riscv_vle16_v_u16m1(ref_ptr, vl); + + vuint32m2_t wtd = __riscv_vwmulu_vx_u32m2(ref, fwd, vl); + wtd = __riscv_vwmaccu_vx_u32m2(wtd, bck, src, vl); + wtd = __riscv_vsrl_vx_u32m2(wtd, DIST_PRECISION_BITS, vl); + + vint32m2_t diff = __riscv_vreinterpret_v_u32m2_i32m2( + __riscv_vsub_vv_u32m2(wtd, offset_vec, vl)); + + vuint32m2_t d0 = __riscv_vreinterpret_v_i32m2_u32m2( + __riscv_vmax_vx_i32m2(diff, 0, vl)); +#if __riscv_v_intrinsic >= 12000 + vuint16m1_t d0_u16 = + __riscv_vnclipu_wx_u16m1(d0, shift_bits, __RISCV_VXRM_RNU, vl); +#else + vuint16m1_t d0_u16 = __riscv_vnclipu_wx_u16m1(d0, shift_bits, vl); +#endif + d0_u16 = __riscv_vminu_vx_u16m1(d0_u16, max_val, vl); + + __riscv_vse16_v_u16m1(dst_ptr, d0_u16, vl); + + src_ptr += src_stride; + dst_ptr += dst_stride; + ref_ptr += ref_stride; + } while (--h != 0); + } else { + do { + int width = w; + const uint16_t *src_line = src_ptr; + const uint16_t *ref_line = (const uint16_t *)ref_ptr; + uint16_t *dst_line = dst_ptr; + + while (width > 0) { + vuint16m1_t s = __riscv_vle16_v_u16m1(src_line, vl); + vuint16m1_t r = __riscv_vle16_v_u16m1(ref_line, vl); + + vuint32m2_t wtd = __riscv_vwmulu_vx_u32m2(r, fwd, vl); + wtd = __riscv_vwmaccu_vx_u32m2(wtd, bck, s, vl); + wtd = __riscv_vsrl_vx_u32m2(wtd, DIST_PRECISION_BITS, vl); + + vint32m2_t diff = __riscv_vreinterpret_v_u32m2_i32m2( + __riscv_vsub_vv_u32m2(wtd, offset_vec, vl)); + + vuint32m2_t d0 = __riscv_vreinterpret_v_i32m2_u32m2( + __riscv_vmax_vx_i32m2(diff, 0, vl)); +#if __riscv_v_intrinsic >= 12000 + vuint16m1_t d0_u16 = + __riscv_vnclipu_wx_u16m1(d0, shift_bits, __RISCV_VXRM_RNU, vl); +#else + vuint16m1_t d0_u16 = __riscv_vnclipu_wx_u16m1(d0, shift_bits, vl); +#endif + d0_u16 = __riscv_vminu_vx_u16m1(d0_u16, max_val, vl); + + __riscv_vse16_v_u16m1(dst_line, d0_u16, vl); + + src_line += vl; + ref_line += vl; + dst_line += vl; + width -= vl; + } + + src_ptr += src_stride; + dst_ptr += dst_stride; + ref_ptr += ref_stride; + } while (--h != 0); + } +} + +#endif // AOM_AV1_COMMON_RISCV_HIGHBD_COMPOUND_CONVOLVE_RVV_H_
diff --git a/av1/common/seg_common.h b/av1/common/seg_common.h index 3ba5052..3b88893 100644 --- a/av1/common/seg_common.h +++ b/av1/common/seg_common.h
@@ -12,6 +12,8 @@ #ifndef AOM_AV1_COMMON_SEG_COMMON_H_ #define AOM_AV1_COMMON_SEG_COMMON_H_ +#include <string.h> + #include "aom_dsp/prob.h" #ifdef __cplusplus
diff --git a/av1/common/warp_plane_hwy.h b/av1/common/warp_plane_hwy.h new file mode 100644 index 0000000..61d5fc6 --- /dev/null +++ b/av1/common/warp_plane_hwy.h
@@ -0,0 +1,1487 @@ +/* + * Copyright (c) 2025, Alliance for Open Media. All rights reserved. + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_AV1_COMMON_WARP_PLANE_HWY_H_ +#define AOM_AV1_COMMON_WARP_PLANE_HWY_H_ + +#include "av1/common/warped_motion.h" +#include "config/av1_rtcd.h" +#include "third_party/highway/hwy/highway.h" + +HWY_BEFORE_NAMESPACE(); + +namespace { +namespace HWY_NAMESPACE { + +namespace hn = hwy::HWY_NAMESPACE; + +constexpr hn::ScalableTag<uint8_t> uint8xN_tag; +constexpr hn::ScalableTag<uint16_t> uint16xN_tag; + +constexpr hn::ScalableTag<int16_t> int16xN_tag; +constexpr hn::ScalableTag<int32_t> int32xN_tag; +constexpr hn::ScalableTag<int64_t> int64xN_tag; + +constexpr hn::CappedTag<uint8_t, 32> uint8x32_tag; +constexpr hn::CappedTag<int16_t, 16> int16x16_tag; + +constexpr hn::FixedTag<uint8_t, 4> uint8x4_tag; +constexpr hn::FixedTag<uint8_t, 8> uint8x8_tag; +constexpr hn::FixedTag<uint8_t, 16> uint8x16_tag; +constexpr hn::FixedTag<uint16_t, 4> uint16x4_tag; +constexpr hn::FixedTag<uint16_t, 8> uint16x8_tag; + +constexpr hn::FixedTag<int8_t, 8> int8x8_tag; +constexpr hn::FixedTag<int8_t, 16> int8x16_tag; +constexpr hn::FixedTag<int16_t, 8> int16x8_tag; +constexpr hn::FixedTag<int32_t, 4> int32x4_tag; +constexpr hn::FixedTag<int64_t, 2> int64x2_tag; + +constexpr hn::ScalableTag<int8_t> coeff_tag; + +using IVec16 = hn::Vec<decltype(int16xN_tag)>; +using IVec32 = hn::Vec<decltype(int32xN_tag)>; +using IVec8x16 = hn::Vec<decltype(int8x16_tag)>; + +template <typename D> +HWY_ATTR inline void FilterPixelsHorizontal(D tag, const hn::VFromD<D> src, + int16_t *HWY_RESTRICT horz_out, + int8_t *HWY_RESTRICT coeff, + const IVec16 round_const, + const int shift, int row) { + constexpr hn::Repartition<int8_t, D> int8_tag; + constexpr hn::Repartition<int16_t, D> result_tag; + constexpr hn::Repartition<uint16_t, D> unsigned_result_tag; + // N.B. coeffs are stored to support the maximum vector width, which may not + // be the vector width being filtered on now. + const auto coeff0 = hn::Load(int8_tag, coeff + hn::MaxLanes(coeff_tag) * 0); + const auto coeff1 = hn::Load(int8_tag, coeff + hn::MaxLanes(coeff_tag) * 1); + const auto coeff2 = hn::Load(int8_tag, coeff + hn::MaxLanes(coeff_tag) * 2); + const auto coeff3 = hn::Load(int8_tag, coeff + hn::MaxLanes(coeff_tag) * 3); + + const auto shuffle0 = hn::Dup128VecFromValues( + uint8xN_tag, 0, 2, 2, 4, 4, 6, 6, 8, 1, 3, 3, 5, 5, 7, 7, 9 // + ); + const auto shuffle1 = hn::Dup128VecFromValues( + uint8xN_tag, 4, 6, 6, 8, 8, 10, 10, 12, 5, 7, 7, 9, 9, 11, 11, 13 // + ); + const auto shuffle2 = hn::Dup128VecFromValues( + uint8xN_tag, 1, 3, 3, 5, 5, 7, 7, 9, 2, 4, 4, 6, 6, 8, 8, 10 // + ); + const auto shuffle3 = hn::Dup128VecFromValues( + uint8xN_tag, 5, 7, 7, 9, 9, 11, 11, 13, 6, 8, 8, 10, 10, 12, 12, 14 // + ); + + const auto src_0 = + hn::TableLookupBytes(src, hn::ResizeBitCast(tag, shuffle0)); + const auto src_1 = + hn::TableLookupBytes(src, hn::ResizeBitCast(tag, shuffle1)); + const auto src_2 = + hn::TableLookupBytes(src, hn::ResizeBitCast(tag, shuffle2)); + const auto src_3 = + hn::TableLookupBytes(src, hn::ResizeBitCast(tag, shuffle3)); + + const auto res_02 = hn::SatWidenMulPairwiseAdd(result_tag, src_0, coeff0); + const auto res_46 = hn::SatWidenMulPairwiseAdd(result_tag, src_1, coeff1); + const auto res_13 = hn::SatWidenMulPairwiseAdd(result_tag, src_2, coeff2); + const auto res_57 = hn::SatWidenMulPairwiseAdd(result_tag, src_3, coeff3); + + const auto res_even = hn::Add(res_02, res_46); + const auto res_odd = hn::Add(res_13, res_57); + + const auto res = hn::Add(hn::Add(res_even, res_odd), + hn::ResizeBitCast(result_tag, round_const)); + + hn::Store(hn::BitCast(result_tag, + hn::ShiftRightSame( + hn::BitCast(unsigned_result_tag, res), shift)), + result_tag, horz_out + row * hn::MaxLanes(int16x8_tag)); +} + +HWY_ATTR HWY_INLINE IVec8x16 LoadAV1Filter8Bit(unsigned int offset) { + return hn::LoadN(int8x16_tag, av1_filter_8bit[offset >> WARPEDDIFF_PREC_BITS], + 8); +} + +template <typename D> +HWY_ATTR HWY_INLINE hn::VFromD<D> LoadAV1Filter8BitLower(D int8_tag, + unsigned int offset) { + return hn::LoadN(int8_tag, av1_filter_8bit[offset >> WARPEDDIFF_PREC_BITS], + 8); +} + +template <int Block, typename D> +HWY_ATTR HWY_INLINE hn::VFromD<D> LoadAV1Filter8BitUpper(D int8_tag, + unsigned int offset, + hn::VFromD<D> src) { + (void)int8_tag; + return hn::InsertBlock<Block>( + src, hn::LoadN(int8x16_tag, + av1_filter_8bit[offset >> WARPEDDIFF_PREC_BITS], 8)); +} + +template <typename D> +HWY_ATTR inline void PrepareHorizontalFilterCoefficients( + D int16_tag, int alpha, int beta, int sx, int8_t *HWY_RESTRICT coeff) { + constexpr auto int8_tag = hn::Repartition<int8_t, D>(); + constexpr auto int32_tag = hn::Repartition<int32_t, D>(); + constexpr auto int64_tag = hn::Repartition<int64_t, D>(); + + auto tmp_0 = LoadAV1Filter8BitLower(int8_tag, sx + 0 * alpha); + auto tmp_1 = LoadAV1Filter8BitLower(int8_tag, sx + 1 * alpha); + auto tmp_2 = LoadAV1Filter8BitLower(int8_tag, sx + 2 * alpha); + auto tmp_3 = LoadAV1Filter8BitLower(int8_tag, sx + 3 * alpha); + auto tmp_4 = LoadAV1Filter8BitLower(int8_tag, sx + 4 * alpha); + auto tmp_5 = LoadAV1Filter8BitLower(int8_tag, sx + 5 * alpha); + auto tmp_6 = LoadAV1Filter8BitLower(int8_tag, sx + 6 * alpha); + auto tmp_7 = LoadAV1Filter8BitLower(int8_tag, sx + 7 * alpha); + + if constexpr (int16_tag.MaxBlocks() >= 2) { + tmp_0 = LoadAV1Filter8BitUpper<1>(int8_tag, sx + beta + 0 * alpha, tmp_0); + tmp_1 = LoadAV1Filter8BitUpper<1>(int8_tag, sx + beta + 1 * alpha, tmp_1); + tmp_2 = LoadAV1Filter8BitUpper<1>(int8_tag, sx + beta + 2 * alpha, tmp_2); + tmp_3 = LoadAV1Filter8BitUpper<1>(int8_tag, sx + beta + 3 * alpha, tmp_3); + tmp_4 = LoadAV1Filter8BitUpper<1>(int8_tag, sx + beta + 4 * alpha, tmp_4); + tmp_5 = LoadAV1Filter8BitUpper<1>(int8_tag, sx + beta + 5 * alpha, tmp_5); + tmp_6 = LoadAV1Filter8BitUpper<1>(int8_tag, sx + beta + 6 * alpha, tmp_6); + tmp_7 = LoadAV1Filter8BitUpper<1>(int8_tag, sx + beta + 7 * alpha, tmp_7); + } + + if constexpr (int16_tag.MaxBlocks() >= 3) { + tmp_0 = + LoadAV1Filter8BitUpper<2>(int8_tag, sx + beta * 2 + 0 * alpha, tmp_0); + tmp_1 = + LoadAV1Filter8BitUpper<2>(int8_tag, sx + beta * 2 + 1 * alpha, tmp_1); + tmp_2 = + LoadAV1Filter8BitUpper<2>(int8_tag, sx + beta * 2 + 2 * alpha, tmp_2); + tmp_3 = + LoadAV1Filter8BitUpper<2>(int8_tag, sx + beta * 2 + 3 * alpha, tmp_3); + tmp_4 = + LoadAV1Filter8BitUpper<2>(int8_tag, sx + beta * 2 + 4 * alpha, tmp_4); + tmp_5 = + LoadAV1Filter8BitUpper<2>(int8_tag, sx + beta * 2 + 5 * alpha, tmp_5); + tmp_6 = + LoadAV1Filter8BitUpper<2>(int8_tag, sx + beta * 2 + 6 * alpha, tmp_6); + tmp_7 = + LoadAV1Filter8BitUpper<2>(int8_tag, sx + beta * 2 + 7 * alpha, tmp_7); + + tmp_0 = + LoadAV1Filter8BitUpper<3>(int8_tag, sx + beta * 3 + 0 * alpha, tmp_0); + tmp_1 = + LoadAV1Filter8BitUpper<3>(int8_tag, sx + beta * 3 + 1 * alpha, tmp_1); + tmp_2 = + LoadAV1Filter8BitUpper<3>(int8_tag, sx + beta * 3 + 2 * alpha, tmp_2); + tmp_3 = + LoadAV1Filter8BitUpper<3>(int8_tag, sx + beta * 3 + 3 * alpha, tmp_3); + tmp_4 = + LoadAV1Filter8BitUpper<3>(int8_tag, sx + beta * 3 + 4 * alpha, tmp_4); + tmp_5 = + LoadAV1Filter8BitUpper<3>(int8_tag, sx + beta * 3 + 5 * alpha, tmp_5); + tmp_6 = + LoadAV1Filter8BitUpper<3>(int8_tag, sx + beta * 3 + 6 * alpha, tmp_6); + tmp_7 = + LoadAV1Filter8BitUpper<3>(int8_tag, sx + beta * 3 + 7 * alpha, tmp_7); + } + + const auto tmp_0_16 = hn::BitCast(int16_tag, tmp_0); + const auto tmp_1_16 = hn::BitCast(int16_tag, tmp_1); + const auto tmp_2_16 = hn::BitCast(int16_tag, tmp_2); + const auto tmp_3_16 = hn::BitCast(int16_tag, tmp_3); + const auto tmp_4_16 = hn::BitCast(int16_tag, tmp_4); + const auto tmp_5_16 = hn::BitCast(int16_tag, tmp_5); + const auto tmp_6_16 = hn::BitCast(int16_tag, tmp_6); + const auto tmp_7_16 = hn::BitCast(int16_tag, tmp_7); + + const auto tmp_12 = hn::ZipLower(int32_tag, tmp_0_16, tmp_2_16); + const auto tmp_13 = hn::ZipLower(int32_tag, tmp_1_16, tmp_3_16); + const auto tmp_14 = hn::ZipLower(int32_tag, tmp_4_16, tmp_6_16); + const auto tmp_15 = hn::ZipLower(int32_tag, tmp_5_16, tmp_7_16); + + const auto res_0 = hn::ZipLower(int64_tag, tmp_12, tmp_14); + const auto res_1 = hn::ZipUpper(int64_tag, tmp_12, tmp_14); + const auto res_2 = hn::ZipLower(int64_tag, tmp_13, tmp_15); + const auto res_3 = hn::ZipUpper(int64_tag, tmp_13, tmp_15); + + hn::Store(hn::BitCast(int8_tag, hn::InterleaveLower(int64_tag, res_0, res_2)), + int8_tag, coeff + hn::MaxLanes(coeff_tag) * 0); + hn::Store(hn::BitCast(int8_tag, hn::InterleaveUpper(int64_tag, res_0, res_2)), + int8_tag, coeff + hn::MaxLanes(coeff_tag) * 1); + hn::Store(hn::BitCast(int8_tag, hn::InterleaveLower(int64_tag, res_1, res_3)), + int8_tag, coeff + hn::MaxLanes(coeff_tag) * 2); + hn::Store(hn::BitCast(int8_tag, hn::InterleaveUpper(int64_tag, res_1, res_3)), + int8_tag, coeff + hn::MaxLanes(coeff_tag) * 3); +} + +template <typename D> +HWY_ATTR inline void PrepareHorizontalFilterCoefficientsBeta0( + D int16_tag, int alpha, int beta, int sx, int8_t *HWY_RESTRICT coeff) { + (void)int16_tag; + (void)beta; + const auto tmp_0 = + hn::BitCast(int16x8_tag, LoadAV1Filter8Bit(sx + 0 * alpha)); + const auto tmp_1 = + hn::BitCast(int16x8_tag, LoadAV1Filter8Bit(sx + 1 * alpha)); + const auto tmp_2 = + hn::BitCast(int16x8_tag, LoadAV1Filter8Bit(sx + 2 * alpha)); + const auto tmp_3 = + hn::BitCast(int16x8_tag, LoadAV1Filter8Bit(sx + 3 * alpha)); + const auto tmp_4 = + hn::BitCast(int16x8_tag, LoadAV1Filter8Bit(sx + 4 * alpha)); + const auto tmp_5 = + hn::BitCast(int16x8_tag, LoadAV1Filter8Bit(sx + 5 * alpha)); + const auto tmp_6 = + hn::BitCast(int16x8_tag, LoadAV1Filter8Bit(sx + 6 * alpha)); + const auto tmp_7 = + hn::BitCast(int16x8_tag, LoadAV1Filter8Bit(sx + 7 * alpha)); + + const auto tmp_02 = hn::ZipLower(int32x4_tag, tmp_0, tmp_2); + const auto tmp_13 = hn::ZipLower(int32x4_tag, tmp_1, tmp_3); + const auto tmp_46 = hn::ZipLower(int32x4_tag, tmp_4, tmp_6); + const auto tmp_57 = hn::ZipLower(int32x4_tag, tmp_5, tmp_7); + + constexpr auto int8_tag = hn::Repartition<int8_t, D>(); + constexpr auto int32_tag = hn::Repartition<int32_t, D>(); + constexpr auto int64_tag = hn::Repartition<int64_t, D>(); + + const auto broadcast_12 = + hn::BroadcastBlock<0>(hn::ResizeBitCast(int32_tag, tmp_02)); + const auto broadcast_13 = + hn::BroadcastBlock<0>(hn::ResizeBitCast(int32_tag, tmp_13)); + const auto broadcast_14 = + hn::BroadcastBlock<0>(hn::ResizeBitCast(int32_tag, tmp_46)); + const auto broadcast_15 = + hn::BroadcastBlock<0>(hn::ResizeBitCast(int32_tag, tmp_57)); + + const auto res_0 = hn::ZipLower(int64_tag, broadcast_12, broadcast_14); + const auto res_1 = hn::ZipUpper(int64_tag, broadcast_12, broadcast_14); + const auto res_2 = hn::ZipLower(int64_tag, broadcast_13, broadcast_15); + const auto res_3 = hn::ZipUpper(int64_tag, broadcast_13, broadcast_15); + + hn::Store(hn::BitCast(int8_tag, hn::InterleaveLower(int64_tag, res_0, res_2)), + int8_tag, coeff + hn::MaxLanes(coeff_tag) * 0); + hn::Store(hn::BitCast(int8_tag, hn::InterleaveUpper(int64_tag, res_0, res_2)), + int8_tag, coeff + hn::MaxLanes(coeff_tag) * 1); + hn::Store(hn::BitCast(int8_tag, hn::InterleaveLower(int64_tag, res_1, res_3)), + int8_tag, coeff + hn::MaxLanes(coeff_tag) * 2); + hn::Store(hn::BitCast(int8_tag, hn::InterleaveUpper(int64_tag, res_1, res_3)), + int8_tag, coeff + hn::MaxLanes(coeff_tag) * 3); +} + +template <typename D> +HWY_ATTR inline void PrepareHorizontalFilterCoefficientsAlpha0( + D int16_tag, int alpha, int beta, int sx, int8_t *HWY_RESTRICT coeff) { + (void)alpha; + constexpr auto int8_tag = hn::Repartition<int8_t, D>(); + auto tmp_0 = LoadAV1Filter8BitLower(int8_tag, sx); + if constexpr (int16_tag.MaxBlocks() >= 2) { + tmp_0 = LoadAV1Filter8BitUpper<1>(int8_tag, sx + beta, tmp_0); + } + if constexpr (int16_tag.MaxBlocks() >= 3) { + tmp_0 = LoadAV1Filter8BitUpper<2>(int8_tag, sx + beta * 2, tmp_0); + tmp_0 = LoadAV1Filter8BitUpper<3>(int8_tag, sx + beta * 3, tmp_0); + } + const auto res_0 = hn::BitCast(int16_tag, tmp_0); + + hn::Store(hn::BitCast(int8_tag, hn::Broadcast<0>(res_0)), int8_tag, + coeff + hn::MaxLanes(coeff_tag) * 0); + hn::Store(hn::BitCast(int8_tag, hn::Broadcast<1>(res_0)), int8_tag, + coeff + hn::MaxLanes(coeff_tag) * 1); + hn::Store(hn::BitCast(int8_tag, hn::Broadcast<2>(res_0)), int8_tag, + coeff + hn::MaxLanes(coeff_tag) * 2); + hn::Store(hn::BitCast(int8_tag, hn::Broadcast<3>(res_0)), int8_tag, + coeff + hn::MaxLanes(coeff_tag) * 3); +} + +template <typename D> +HWY_ATTR inline void HorizontalFilter(D tag, const hn::VFromD<D> src, + int16_t *HWY_RESTRICT horz_out, int sx, + int alpha, int beta, int row, + const IVec16 round_const, + const int reduce_bits_horiz) { + HWY_ALIGN int8_t coeff[4 * hn::MaxLanes(coeff_tag)]; + PrepareHorizontalFilterCoefficients(hn::Repartition<int16_t, D>(), alpha, + beta, sx, coeff); + FilterPixelsHorizontal(tag, src, horz_out, coeff, round_const, + reduce_bits_horiz, row); +} + +template <typename D> +HWY_ATTR inline void PrepareLastHorizontalFilterCoefficients( + D int16_tag, int alpha, int beta, int sx, int8_t *HWY_RESTRICT coeff) { + (void)int16_tag; + (void)beta; + const auto tmp_0 = + hn::BitCast(int16x8_tag, LoadAV1Filter8Bit(sx + 0 * alpha)); + const auto tmp_1 = + hn::BitCast(int16x8_tag, LoadAV1Filter8Bit(sx + 1 * alpha)); + const auto tmp_2 = + hn::BitCast(int16x8_tag, LoadAV1Filter8Bit(sx + 2 * alpha)); + const auto tmp_3 = + hn::BitCast(int16x8_tag, LoadAV1Filter8Bit(sx + 3 * alpha)); + const auto tmp_4 = + hn::BitCast(int16x8_tag, LoadAV1Filter8Bit(sx + 4 * alpha)); + const auto tmp_5 = + hn::BitCast(int16x8_tag, LoadAV1Filter8Bit(sx + 5 * alpha)); + const auto tmp_6 = + hn::BitCast(int16x8_tag, LoadAV1Filter8Bit(sx + 6 * alpha)); + const auto tmp_7 = + hn::BitCast(int16x8_tag, LoadAV1Filter8Bit(sx + 7 * alpha)); + + const auto tmp_8 = hn::ZipLower(int32x4_tag, tmp_0, tmp_2); + const auto tmp_9 = hn::ZipLower(int32x4_tag, tmp_1, tmp_3); + const auto tmp_10 = hn::ZipLower(int32x4_tag, tmp_4, tmp_6); + const auto tmp_11 = hn::ZipLower(int32x4_tag, tmp_5, tmp_7); + + const auto tmp_12 = hn::ZipLower(int64x2_tag, tmp_8, tmp_10); + const auto tmp_13 = hn::ZipUpper(int64x2_tag, tmp_8, tmp_10); + const auto tmp_14 = hn::ZipLower(int64x2_tag, tmp_9, tmp_11); + const auto tmp_15 = hn::ZipUpper(int64x2_tag, tmp_9, tmp_11); + + const auto tmp_16 = hn::InterleaveLower(int64x2_tag, tmp_12, tmp_14); + const auto tmp_17 = hn::InterleaveUpper(int64x2_tag, tmp_12, tmp_14); + const auto tmp_18 = hn::InterleaveLower(int64x2_tag, tmp_13, tmp_15); + const auto tmp_19 = hn::InterleaveUpper(int64x2_tag, tmp_13, tmp_15); + + constexpr auto int8_tag = hn::Repartition<int8_t, D>(); + + const auto tmp_20 = hn::ResizeBitCast(int8_tag, tmp_16); + const auto tmp_21 = hn::ResizeBitCast(int8_tag, tmp_17); + const auto tmp_22 = hn::ResizeBitCast(int8_tag, tmp_18); + const auto tmp_23 = hn::ResizeBitCast(int8_tag, tmp_19); + + hn::Store(hn::BroadcastBlock<0>(tmp_20), int8_tag, + coeff + hn::MaxLanes(coeff_tag) * 0); + hn::Store(hn::BroadcastBlock<0>(tmp_21), int8_tag, + coeff + hn::MaxLanes(coeff_tag) * 1); + hn::Store(hn::BroadcastBlock<0>(tmp_22), int8_tag, + coeff + hn::MaxLanes(coeff_tag) * 2); + hn::Store(hn::BroadcastBlock<0>(tmp_23), int8_tag, + coeff + hn::MaxLanes(coeff_tag) * 3); +} + +template <typename D> +HWY_ATTR HWY_INLINE hn::VFromD<D> LoadRowsClamped( + D tag, const uint8_t *HWY_RESTRICT ref, const int stride, const int iy, + const int height) { + constexpr hn::BlockDFromD<D> block_tag; + const int iy0 = clamp(iy + 0, 0, height - 1); + auto src = hn::ResizeBitCast(tag, hn::LoadU(block_tag, ref + iy0 * stride)); + if constexpr (tag.MaxBlocks() >= 2) { + const int iy1 = clamp(iy + 1, 0, height - 1); + const auto src_1 = hn::LoadU(block_tag, ref + iy1 * stride); + src = hn::InsertBlock<1>(src, src_1); + } + if constexpr (tag.MaxBlocks() >= 3) { + const int iy2 = clamp(iy + 2, 0, height - 1); + const auto src_2 = hn::LoadU(block_tag, ref + iy2 * stride); + const int iy3 = clamp(iy + 3, 0, height - 1); + const auto src_3 = hn::LoadU(block_tag, ref + iy3 * stride); + src = hn::InsertBlock<2>(src, src_2); + src = hn::InsertBlock<3>(src, src_3); + } + return src; +} + +template <void (*PrepareCoeffs)(int alpha, int beta, int sx, + int8_t *HWY_RESTRICT coeffs), + typename D> +HWY_ATTR int WarpHorizontalFilterLoop( + D tag, const uint8_t *HWY_RESTRICT ref, int16_t *HWY_RESTRICT horz_out, + int stride, int32_t ix4, int32_t iy4, int32_t sx4, int alpha, int beta, + int p_height, int height, int i, const IVec16 round_const, + const int reduce_bits_horiz, int k, int8_t *HWY_RESTRICT coeff) { + constexpr int kNumRows = tag.MaxBlocks(); + for (; k < AOMMIN(8, p_height - i) - kNumRows; k += kNumRows) { + const auto src = + LoadRowsClamped(tag, ref + ix4 - 7, stride, iy4 + k, height); + if constexpr (PrepareCoeffs != nullptr) { + int sx = sx4 + beta * (k + 4); + PrepareCoeffs(alpha, beta, sx, coeff); + } + FilterPixelsHorizontal(tag, src, horz_out, coeff, round_const, + reduce_bits_horiz, k + 7); + } + return k; +} + +enum class HorizontalFilterCoeffs { + kAlpha0, + kBeta0, + kDefault, +}; + +template <bool IsLast, HorizontalFilterCoeffs Filter, typename D> +HWY_ATTR void WarpHorizontalPrepareCoeffs(int alpha, int beta, int sx, + int8_t *HWY_RESTRICT coeffs) { + D int16_tag; + switch (Filter) { + case HorizontalFilterCoeffs::kAlpha0: + PrepareHorizontalFilterCoefficientsAlpha0(int16_tag, alpha, beta, sx, + coeffs); + return; + case HorizontalFilterCoeffs::kBeta0: + PrepareHorizontalFilterCoefficientsBeta0(int16_tag, alpha, beta, sx, + coeffs); + return; + case HorizontalFilterCoeffs::kDefault: + default: + if (IsLast) { + PrepareLastHorizontalFilterCoefficients(int16_tag, alpha, beta, sx, + coeffs); + } else { + PrepareHorizontalFilterCoefficients(int16_tag, alpha, beta, sx, coeffs); + } + return; + } +} + +template <bool InnerCoeffUpdate, HorizontalFilterCoeffs Filter> +HWY_ATTR inline void WarpHorizontalFilterTemplate( + const uint8_t *HWY_RESTRICT ref, int16_t *HWY_RESTRICT horz_out, int stride, + int32_t ix4, int32_t iy4, int32_t sx4, int alpha, int beta, int p_height, + int height, int i, const IVec16 round_const, const int reduce_bits_horiz) { + int k = -7, iy; + HWY_ALIGN int8_t coeff[4 * hn::MaxLanes(coeff_tag)]; + if constexpr (!InnerCoeffUpdate) { + WarpHorizontalPrepareCoeffs<false, Filter, decltype(int16xN_tag)>( + alpha, beta, sx4, coeff); + } + if constexpr (uint8xN_tag.MaxBlocks() >= 3) { + k = WarpHorizontalFilterLoop<( + InnerCoeffUpdate + ? WarpHorizontalPrepareCoeffs<false, Filter, decltype(int16xN_tag)> + : nullptr)>(uint8xN_tag, ref, horz_out, stride, ix4, iy4, sx4, + alpha, beta, p_height, height, i, round_const, + reduce_bits_horiz, k, coeff); + } + if constexpr (uint8xN_tag.MaxBlocks() >= 2) { + k = WarpHorizontalFilterLoop<( + InnerCoeffUpdate + ? WarpHorizontalPrepareCoeffs<false, Filter, decltype(int16x16_tag)> + : nullptr)>(uint8x32_tag, ref, horz_out, stride, ix4, iy4, sx4, + alpha, beta, p_height, height, i, round_const, + reduce_bits_horiz, k, coeff); + } + if constexpr (uint8xN_tag.MaxBlocks() == 1) { + k = WarpHorizontalFilterLoop<( + InnerCoeffUpdate + ? WarpHorizontalPrepareCoeffs<true, Filter, decltype(int16x8_tag)> + : nullptr)>(uint8x16_tag, ref, horz_out, stride, ix4, iy4, sx4, + alpha, beta, p_height, height, i, round_const, + reduce_bits_horiz, k, coeff); + } + iy = iy4 + k; + iy = clamp(iy, 0, height - 1); + const auto src = hn::LoadU(uint8x16_tag, ref + iy * stride + ix4 - 7); + if constexpr (InnerCoeffUpdate) { + int sx = sx4 + beta * (k + 4); + WarpHorizontalPrepareCoeffs<true, Filter, decltype(int16x8_tag)>( + alpha, beta, sx, coeff); + } + FilterPixelsHorizontal(uint8x16_tag, src, horz_out, coeff, round_const, + reduce_bits_horiz, k + 7); +} + +HWY_ATTR inline void UnpackWeightsAndSetRoundConst( + ConvolveParams *HWY_RESTRICT conv_params, const int round_bits, + const int offset_bits, IVec16 &HWY_RESTRICT res_sub_const, + IVec16 &HWY_RESTRICT round_bits_const, IVec16 &HWY_RESTRICT wt) { + res_sub_const = + hn::Set(int16xN_tag, -(1 << (offset_bits - conv_params->round_1)) - + (1 << (offset_bits - conv_params->round_1 - 1))); + round_bits_const = hn::Set(int16xN_tag, ((1 << round_bits) >> 1)); + + const auto w0 = static_cast<int16_t>(conv_params->fwd_offset); + const auto w1 = static_cast<int16_t>(conv_params->bck_offset); + const auto wt0 = hn::Set(int16xN_tag, w0); + const auto wt1 = hn::Set(int16xN_tag, w1); + wt = hn::InterleaveLower(wt0, wt1); +} + +HWY_ATTR HWY_INLINE IVec16 LoadAV1WarpedFilter(size_t offset) { + return hn::LoadN(int16xN_tag, + av1_warped_filter[offset >> WARPEDDIFF_PREC_BITS], 8); +} + +HWY_ATTR HWY_INLINE IVec16 LoadAV1WarpedFilterLower(size_t offset) { + return hn::ResizeBitCast( + int16xN_tag, + hn::Load(int16x8_tag, av1_warped_filter[offset >> WARPEDDIFF_PREC_BITS])); +} + +template <int Block> +HWY_ATTR HWY_INLINE IVec16 LoadAV1WarpedFilterUpper(size_t offset, IVec16 src) { + return hn::InsertBlock<Block>( + src, + hn::Load(int16x8_tag, av1_warped_filter[offset >> WARPEDDIFF_PREC_BITS])); +} + +HWY_ATTR inline void PrepareVerticalFilterCoeffs(int gamma, int delta, int sy, + int16_t *HWY_RESTRICT coeffs) { + auto filt_00 = LoadAV1WarpedFilterLower(sy + 0 * gamma); + auto filt_01 = LoadAV1WarpedFilterLower(sy + 2 * gamma); + auto filt_02 = LoadAV1WarpedFilterLower(sy + 4 * gamma); + auto filt_03 = LoadAV1WarpedFilterLower(sy + 6 * gamma); + + if constexpr (int16xN_tag.MaxBlocks() >= 2) { + filt_00 = LoadAV1WarpedFilterUpper<1>(sy + delta + 0 * gamma, filt_00); + filt_01 = LoadAV1WarpedFilterUpper<1>(sy + delta + 2 * gamma, filt_01); + filt_02 = LoadAV1WarpedFilterUpper<1>(sy + delta + 4 * gamma, filt_02); + filt_03 = LoadAV1WarpedFilterUpper<1>(sy + delta + 6 * gamma, filt_03); + } + + if constexpr (int16xN_tag.MaxBlocks() >= 3) { + filt_00 = LoadAV1WarpedFilterUpper<2>(sy + 2 * delta + 0 * gamma, filt_00); + filt_01 = LoadAV1WarpedFilterUpper<2>(sy + 2 * delta + 2 * gamma, filt_01); + filt_02 = LoadAV1WarpedFilterUpper<2>(sy + 2 * delta + 4 * gamma, filt_02); + filt_03 = LoadAV1WarpedFilterUpper<2>(sy + 2 * delta + 6 * gamma, filt_03); + + filt_00 = LoadAV1WarpedFilterUpper<3>(sy + 3 * delta + 0 * gamma, filt_00); + filt_01 = LoadAV1WarpedFilterUpper<3>(sy + 3 * delta + 2 * gamma, filt_01); + filt_02 = LoadAV1WarpedFilterUpper<3>(sy + 3 * delta + 4 * gamma, filt_02); + filt_03 = LoadAV1WarpedFilterUpper<3>(sy + 3 * delta + 6 * gamma, filt_03); + } + + auto filt_0 = hn::BitCast(int32xN_tag, filt_00); + auto filt_1 = hn::BitCast(int32xN_tag, filt_01); + auto filt_2 = hn::BitCast(int32xN_tag, filt_02); + auto filt_3 = hn::BitCast(int32xN_tag, filt_03); + + auto res_0 = hn::ZipLower(int64xN_tag, filt_0, filt_1); + auto res_1 = hn::ZipLower(int64xN_tag, filt_2, filt_3); + auto res_2 = hn::ZipUpper(int64xN_tag, filt_0, filt_1); + auto res_3 = hn::ZipUpper(int64xN_tag, filt_2, filt_3); + + hn::Store( + hn::BitCast(int16xN_tag, hn::InterleaveLower(int64xN_tag, res_0, res_1)), + int16xN_tag, coeffs + 0 * hn::MaxLanes(int16xN_tag)); + hn::Store( + hn::BitCast(int16xN_tag, hn::InterleaveUpper(int64xN_tag, res_0, res_1)), + int16xN_tag, coeffs + 1 * hn::MaxLanes(int16xN_tag)); + hn::Store( + hn::BitCast(int16xN_tag, hn::InterleaveLower(int64xN_tag, res_2, res_3)), + int16xN_tag, coeffs + 2 * hn::MaxLanes(int16xN_tag)); + hn::Store( + hn::BitCast(int16xN_tag, hn::InterleaveUpper(int64xN_tag, res_2, res_3)), + int16xN_tag, coeffs + 3 * hn::MaxLanes(int16xN_tag)); + + filt_00 = LoadAV1WarpedFilterLower(sy + 1 * gamma); + filt_01 = LoadAV1WarpedFilterLower(sy + 3 * gamma); + filt_02 = LoadAV1WarpedFilterLower(sy + 5 * gamma); + filt_03 = LoadAV1WarpedFilterLower(sy + 7 * gamma); + + if constexpr (int16xN_tag.MaxBlocks() >= 2) { + filt_00 = LoadAV1WarpedFilterUpper<1>(sy + delta + 1 * gamma, filt_00); + filt_01 = LoadAV1WarpedFilterUpper<1>(sy + delta + 3 * gamma, filt_01); + filt_02 = LoadAV1WarpedFilterUpper<1>(sy + delta + 5 * gamma, filt_02); + filt_03 = LoadAV1WarpedFilterUpper<1>(sy + delta + 7 * gamma, filt_03); + } + + if constexpr (int16xN_tag.MaxBlocks() >= 3) { + filt_00 = LoadAV1WarpedFilterUpper<2>(sy + 2 * delta + 1 * gamma, filt_00); + filt_01 = LoadAV1WarpedFilterUpper<2>(sy + 2 * delta + 3 * gamma, filt_01); + filt_02 = LoadAV1WarpedFilterUpper<2>(sy + 2 * delta + 5 * gamma, filt_02); + filt_03 = LoadAV1WarpedFilterUpper<2>(sy + 2 * delta + 7 * gamma, filt_03); + + filt_00 = LoadAV1WarpedFilterUpper<3>(sy + 3 * delta + 1 * gamma, filt_00); + filt_01 = LoadAV1WarpedFilterUpper<3>(sy + 3 * delta + 3 * gamma, filt_01); + filt_02 = LoadAV1WarpedFilterUpper<3>(sy + 3 * delta + 5 * gamma, filt_02); + filt_03 = LoadAV1WarpedFilterUpper<3>(sy + 3 * delta + 7 * gamma, filt_03); + } + + filt_0 = hn::BitCast(int32xN_tag, filt_00); + filt_1 = hn::BitCast(int32xN_tag, filt_01); + filt_2 = hn::BitCast(int32xN_tag, filt_02); + filt_3 = hn::BitCast(int32xN_tag, filt_03); + + res_0 = hn::ZipLower(int64xN_tag, filt_0, filt_1); + res_1 = hn::ZipLower(int64xN_tag, filt_2, filt_3); + res_2 = hn::ZipUpper(int64xN_tag, filt_0, filt_1); + res_3 = hn::ZipUpper(int64xN_tag, filt_2, filt_3); + + hn::Store( + hn::BitCast(int16xN_tag, hn::InterleaveLower(int64xN_tag, res_0, res_1)), + int16xN_tag, coeffs + 4 * hn::MaxLanes(int16xN_tag)); + hn::Store( + hn::BitCast(int16xN_tag, hn::InterleaveUpper(int64xN_tag, res_0, res_1)), + int16xN_tag, coeffs + 5 * hn::MaxLanes(int16xN_tag)); + hn::Store( + hn::BitCast(int16xN_tag, hn::InterleaveLower(int64xN_tag, res_2, res_3)), + int16xN_tag, coeffs + 6 * hn::MaxLanes(int16xN_tag)); + hn::Store( + hn::BitCast(int16xN_tag, hn::InterleaveUpper(int64xN_tag, res_2, res_3)), + int16xN_tag, coeffs + 7 * hn::MaxLanes(int16xN_tag)); +} + +HWY_ATTR inline void PrepareVerticalFilterCoeffsDelta0( + int gamma, int delta, int sy, int16_t *HWY_RESTRICT coeffs) { + (void)delta; + auto filt_00 = LoadAV1WarpedFilter(sy + 0 * gamma); + auto filt_01 = LoadAV1WarpedFilter(sy + 2 * gamma); + auto filt_02 = LoadAV1WarpedFilter(sy + 4 * gamma); + auto filt_03 = LoadAV1WarpedFilter(sy + 6 * gamma); + + auto filt_10 = hn::BitCast(int32xN_tag, hn::BroadcastBlock<0>(filt_00)); + auto filt_11 = hn::BitCast(int32xN_tag, hn::BroadcastBlock<0>(filt_01)); + auto filt_12 = hn::BitCast(int32xN_tag, hn::BroadcastBlock<0>(filt_02)); + auto filt_13 = hn::BitCast(int32xN_tag, hn::BroadcastBlock<0>(filt_03)); + + auto res_0 = hn::ZipLower(int64xN_tag, filt_10, filt_11); + auto res_1 = hn::ZipLower(int64xN_tag, filt_12, filt_13); + auto res_2 = hn::ZipUpper(int64xN_tag, filt_10, filt_11); + auto res_3 = hn::ZipUpper(int64xN_tag, filt_12, filt_13); + + hn::Store( + hn::BitCast(int16xN_tag, hn::InterleaveLower(int64xN_tag, res_0, res_1)), + int16xN_tag, coeffs + 0 * hn::MaxLanes(int16xN_tag)); + hn::Store( + hn::BitCast(int16xN_tag, hn::InterleaveUpper(int64xN_tag, res_0, res_1)), + int16xN_tag, coeffs + 1 * hn::MaxLanes(int16xN_tag)); + hn::Store( + hn::BitCast(int16xN_tag, hn::InterleaveLower(int64xN_tag, res_2, res_3)), + int16xN_tag, coeffs + 2 * hn::MaxLanes(int16xN_tag)); + hn::Store( + hn::BitCast(int16xN_tag, hn::InterleaveUpper(int64xN_tag, res_2, res_3)), + int16xN_tag, coeffs + 3 * hn::MaxLanes(int16xN_tag)); + + filt_00 = LoadAV1WarpedFilter(sy + 1 * gamma); + filt_01 = LoadAV1WarpedFilter(sy + 3 * gamma); + filt_02 = LoadAV1WarpedFilter(sy + 5 * gamma); + filt_03 = LoadAV1WarpedFilter(sy + 7 * gamma); + + filt_10 = hn::BitCast(int32xN_tag, hn::BroadcastBlock<0>(filt_00)); + filt_11 = hn::BitCast(int32xN_tag, hn::BroadcastBlock<0>(filt_01)); + filt_12 = hn::BitCast(int32xN_tag, hn::BroadcastBlock<0>(filt_02)); + filt_13 = hn::BitCast(int32xN_tag, hn::BroadcastBlock<0>(filt_03)); + + res_0 = hn::ZipLower(int64xN_tag, filt_10, filt_11); + res_1 = hn::ZipLower(int64xN_tag, filt_12, filt_13); + res_2 = hn::ZipUpper(int64xN_tag, filt_10, filt_11); + res_3 = hn::ZipUpper(int64xN_tag, filt_12, filt_13); + + hn::Store( + hn::BitCast(int16xN_tag, hn::InterleaveLower(int64xN_tag, res_0, res_1)), + int16xN_tag, coeffs + 4 * hn::MaxLanes(int16xN_tag)); + hn::Store( + hn::BitCast(int16xN_tag, hn::InterleaveUpper(int64xN_tag, res_0, res_1)), + int16xN_tag, coeffs + 5 * hn::MaxLanes(int16xN_tag)); + hn::Store( + hn::BitCast(int16xN_tag, hn::InterleaveLower(int64xN_tag, res_2, res_3)), + int16xN_tag, coeffs + 6 * hn::MaxLanes(int16xN_tag)); + hn::Store( + hn::BitCast(int16xN_tag, hn::InterleaveUpper(int64xN_tag, res_2, res_3)), + int16xN_tag, coeffs + 7 * hn::MaxLanes(int16xN_tag)); +} + +HWY_ATTR inline void PrepareVerticalFilterCoeffsGamma0( + int gamma, int delta, int sy, int16_t *HWY_RESTRICT coeffs) { + (void)gamma; + auto filt_0 = LoadAV1WarpedFilterLower(sy); + if constexpr (int16xN_tag.MaxBlocks() >= 2) { + filt_0 = LoadAV1WarpedFilterUpper<1>(sy + delta, filt_0); + } + if constexpr (int16xN_tag.MaxBlocks() >= 3) { + filt_0 = LoadAV1WarpedFilterUpper<2>(sy + 2 * delta, filt_0); + filt_0 = LoadAV1WarpedFilterUpper<3>(sy + 3 * delta, filt_0); + } + auto res_0 = hn::BitCast(int32xN_tag, filt_0); + + auto broadcast_0 = hn::BitCast(int16xN_tag, hn::Broadcast<0>(res_0)); + auto broadcast_1 = hn::BitCast(int16xN_tag, hn::Broadcast<1>(res_0)); + auto broadcast_2 = hn::BitCast(int16xN_tag, hn::Broadcast<2>(res_0)); + auto broadcast_3 = hn::BitCast(int16xN_tag, hn::Broadcast<3>(res_0)); + + hn::Store(broadcast_0, int16xN_tag, coeffs + 0 * hn::MaxLanes(int16xN_tag)); + hn::Store(broadcast_1, int16xN_tag, coeffs + 1 * hn::MaxLanes(int16xN_tag)); + hn::Store(broadcast_2, int16xN_tag, coeffs + 2 * hn::MaxLanes(int16xN_tag)); + hn::Store(broadcast_3, int16xN_tag, coeffs + 3 * hn::MaxLanes(int16xN_tag)); + hn::Store(broadcast_0, int16xN_tag, coeffs + 4 * hn::MaxLanes(int16xN_tag)); + hn::Store(broadcast_1, int16xN_tag, coeffs + 5 * hn::MaxLanes(int16xN_tag)); + hn::Store(broadcast_2, int16xN_tag, coeffs + 6 * hn::MaxLanes(int16xN_tag)); + hn::Store(broadcast_3, int16xN_tag, coeffs + 7 * hn::MaxLanes(int16xN_tag)); +} + +HWY_ATTR inline void FilterPixelsVertical( + int16_t *HWY_RESTRICT horz_out, int16_t *HWY_RESTRICT src_lo, + int16_t *HWY_RESTRICT src_hi, int16_t *HWY_RESTRICT coeffs, + IVec32 &HWY_RESTRICT res_lo, IVec32 &HWY_RESTRICT res_hi, int row) { + if constexpr (int16xN_tag.MaxBlocks() >= 3) { + const auto horz_out_4 = + hn::Load(int16xN_tag, horz_out + (row + 4) * hn::MaxLanes(int16x8_tag)); + const auto horz_out_5 = hn::LoadU( + int16xN_tag, horz_out + (row + 5) * hn::MaxLanes(int16x8_tag)); + const auto horz_out_6 = hn::LoadU( + int16xN_tag, horz_out + (row + 6) * hn::MaxLanes(int16x8_tag)); + const auto horz_out_7 = hn::LoadU( + int16xN_tag, horz_out + (row + 7) * hn::MaxLanes(int16x8_tag)); + const auto src_lo_2 = + hn::InterleaveLower(int16xN_tag, horz_out_4, horz_out_5); + const auto src_hi_2 = + hn::InterleaveUpper(int16xN_tag, horz_out_4, horz_out_5); + const auto src_lo_3 = + hn::InterleaveLower(int16xN_tag, horz_out_6, horz_out_7); + const auto src_hi_3 = + hn::InterleaveUpper(int16xN_tag, horz_out_6, horz_out_7); + hn::Store(src_lo_2, int16xN_tag, src_lo + 2 * hn::MaxLanes(int16xN_tag)); + hn::Store(src_hi_2, int16xN_tag, src_hi + 2 * hn::MaxLanes(int16xN_tag)); + hn::Store(src_lo_3, int16xN_tag, src_lo + 3 * hn::MaxLanes(int16xN_tag)); + hn::Store(src_hi_3, int16xN_tag, src_hi + 3 * hn::MaxLanes(int16xN_tag)); + } else if constexpr (int16xN_tag.MaxBlocks() == 2) { + const auto horz_out_6 = + hn::Load(int16xN_tag, horz_out + (row + 6) * hn::MaxLanes(int16x8_tag)); + const auto horz_out_8 = + hn::Load(int16xN_tag, horz_out + (row + 8) * hn::MaxLanes(int16x8_tag)); + const auto horz_out_7 = + hn::ConcatLowerUpper(int16xN_tag, horz_out_8, horz_out_6); + const auto src_lo_3 = + hn::InterleaveLower(int16xN_tag, horz_out_6, horz_out_7); + const auto src_hi_3 = + hn::InterleaveUpper(int16xN_tag, horz_out_6, horz_out_7); + hn::Store(src_lo_3, int16xN_tag, src_lo + 3 * hn::MaxLanes(int16xN_tag)); + hn::Store(src_hi_3, int16xN_tag, src_hi + 3 * hn::MaxLanes(int16xN_tag)); + } else if constexpr (int16xN_tag.MaxBlocks() == 1) { + const auto horz_out_6 = + hn::Load(int16x8_tag, horz_out + (row + 6) * hn::MaxLanes(int16x8_tag)); + const auto horz_out_7 = + hn::Load(int16x8_tag, horz_out + (row + 7) * hn::MaxLanes(int16x8_tag)); + const auto src_lo_3 = + hn::InterleaveLower(int16x8_tag, horz_out_6, horz_out_7); + const auto src_hi_3 = + hn::InterleaveUpper(int16x8_tag, horz_out_6, horz_out_7); + hn::Store(src_lo_3, int16x8_tag, src_lo + 3 * hn::MaxLanes(int16x8_tag)); + hn::Store(src_hi_3, int16x8_tag, src_hi + 3 * hn::MaxLanes(int16x8_tag)); + } + + const auto coeff_0 = + hn::Load(int16xN_tag, coeffs + 0 * hn::MaxLanes(int16xN_tag)); + const auto coeff_1 = + hn::Load(int16xN_tag, coeffs + 1 * hn::MaxLanes(int16xN_tag)); + const auto coeff_2 = + hn::Load(int16xN_tag, coeffs + 2 * hn::MaxLanes(int16xN_tag)); + const auto coeff_3 = + hn::Load(int16xN_tag, coeffs + 3 * hn::MaxLanes(int16xN_tag)); + const auto coeff_4 = + hn::Load(int16xN_tag, coeffs + 4 * hn::MaxLanes(int16xN_tag)); + const auto coeff_5 = + hn::Load(int16xN_tag, coeffs + 5 * hn::MaxLanes(int16xN_tag)); + const auto coeff_6 = + hn::Load(int16xN_tag, coeffs + 6 * hn::MaxLanes(int16xN_tag)); + const auto coeff_7 = + hn::Load(int16xN_tag, coeffs + 7 * hn::MaxLanes(int16xN_tag)); + + const auto src_lo_0 = + hn::Load(int16xN_tag, src_lo + 0 * hn::MaxLanes(int16xN_tag)); + const auto src_lo_1 = + hn::Load(int16xN_tag, src_lo + 1 * hn::MaxLanes(int16xN_tag)); + const auto src_lo_2 = + hn::Load(int16xN_tag, src_lo + 2 * hn::MaxLanes(int16xN_tag)); + const auto src_lo_3 = + hn::Load(int16xN_tag, src_lo + 3 * hn::MaxLanes(int16xN_tag)); + const auto src_hi_0 = + hn::Load(int16xN_tag, src_hi + 0 * hn::MaxLanes(int16xN_tag)); + const auto src_hi_1 = + hn::Load(int16xN_tag, src_hi + 1 * hn::MaxLanes(int16xN_tag)); + const auto src_hi_2 = + hn::Load(int16xN_tag, src_hi + 2 * hn::MaxLanes(int16xN_tag)); + const auto src_hi_3 = + hn::Load(int16xN_tag, src_hi + 3 * hn::MaxLanes(int16xN_tag)); + + auto even_sum0 = hn::Zero(int32xN_tag); + auto even_sum1 = hn::Zero(int32xN_tag); + even_sum0 = hn::ReorderWidenMulAccumulate(int32xN_tag, src_lo_0, coeff_0, + even_sum0, even_sum1); + even_sum0 = hn::ReorderWidenMulAccumulate(int32xN_tag, src_lo_1, coeff_1, + even_sum0, even_sum1); + even_sum0 = hn::ReorderWidenMulAccumulate(int32xN_tag, src_lo_2, coeff_2, + even_sum0, even_sum1); + even_sum0 = hn::ReorderWidenMulAccumulate(int32xN_tag, src_lo_3, coeff_3, + even_sum0, even_sum1); + auto res_even = hn::RearrangeToOddPlusEven(even_sum0, even_sum1); + + auto odd_sum0 = hn::Zero(int32xN_tag); + auto odd_sum1 = hn::Zero(int32xN_tag); + odd_sum0 = hn::ReorderWidenMulAccumulate(int32xN_tag, src_hi_0, coeff_4, + odd_sum0, odd_sum1); + odd_sum0 = hn::ReorderWidenMulAccumulate(int32xN_tag, src_hi_1, coeff_5, + odd_sum0, odd_sum1); + odd_sum0 = hn::ReorderWidenMulAccumulate(int32xN_tag, src_hi_2, coeff_6, + odd_sum0, odd_sum1); + odd_sum0 = hn::ReorderWidenMulAccumulate(int32xN_tag, src_hi_3, coeff_7, + odd_sum0, odd_sum1); + auto res_odd = hn::RearrangeToOddPlusEven(odd_sum0, odd_sum1); + + // Rearrange pixels back into the order 0 ... 7 + res_lo = hn::InterleaveLower(int32xN_tag, res_even, res_odd); + res_hi = hn::InterleaveUpper(int32xN_tag, res_even, res_odd); +} + +template <typename DS, typename DR, typename A, typename B, typename C> +HWY_ATTR HWY_INLINE void StoreRows(DS store_tag, DR row_tag, hn::VFromD<DR> vec, + A stride, B y, C x, + hn::TFromD<DS> *HWY_RESTRICT out) { + hn::TFromD<DS> *HWY_RESTRICT pointers[row_tag.MaxBlocks()]; + for (int i = 0; i < static_cast<int>(row_tag.MaxBlocks()); ++i) { + pointers[i] = &out[(y + i) * stride + x]; + } + hn::Store(hn::ResizeBitCast(store_tag, hn::ExtractBlock<0>(vec)), store_tag, + pointers[0]); + if constexpr (row_tag.MaxBlocks() >= 2) { + hn::Store(hn::ResizeBitCast(store_tag, hn::ExtractBlock<1>(vec)), store_tag, + pointers[1]); + } + if constexpr (row_tag.MaxBlocks() >= 3) { + hn::Store(hn::ResizeBitCast(store_tag, hn::ExtractBlock<2>(vec)), store_tag, + pointers[2]); + hn::Store(hn::ResizeBitCast(store_tag, hn::ExtractBlock<3>(vec)), store_tag, + pointers[3]); + } +} + +HWY_ATTR HWY_INLINE void StoreVerticalFilterOutput( + IVec32 res_lo, IVec32 res_hi, const IVec32 res_add_const, const IVec16 wt, + const IVec16 res_sub_const, const IVec16 round_bits_const, + uint8_t *HWY_RESTRICT pred, ConvolveParams *HWY_RESTRICT conv_params, int i, + int j, int k, const int reduce_bits_vert, int p_stride, int p_width, + const int round_bits) { + constexpr int kNumRows = uint16xN_tag.MaxBlocks(); + if (conv_params->is_compound) { + uint16_t *HWY_RESTRICT pointers[kNumRows]; + for (int row = 0; row < kNumRows; ++row) { + pointers[row] = + &conv_params->dst[(i + k + row) * conv_params->dst_stride + j]; + } + + res_lo = + hn::ShiftRightSame(hn::Add(res_lo, res_add_const), reduce_bits_vert); + + const auto temp_lo_16 = hn::ReorderDemote2To(uint16xN_tag, res_lo, res_lo); + if (conv_params->do_average) { + auto p_16 = + hn::ResizeBitCast(uint16xN_tag, hn::Load(uint16x4_tag, pointers[0])); + if constexpr (kNumRows >= 2) { + p_16 = hn::InsertBlock<1>( + p_16, hn::ResizeBitCast(uint16x8_tag, + hn::Load(uint16x4_tag, pointers[1]))); + } + if constexpr (kNumRows >= 3) { + p_16 = hn::InsertBlock<2>( + p_16, hn::ResizeBitCast(uint16x8_tag, + hn::Load(uint16x4_tag, pointers[2]))); + p_16 = hn::InsertBlock<3>( + p_16, hn::ResizeBitCast(uint16x8_tag, + hn::Load(uint16x4_tag, pointers[3]))); + } + auto res_lo_16 = hn::Undefined(int16xN_tag); + if (conv_params->use_dist_wtd_comp_avg) { + const auto p_16_lo = + hn::BitCast(int16xN_tag, hn::InterleaveLower(p_16, temp_lo_16)); + const auto wt_res_lo = + hn::WidenMulPairwiseAdd(int32xN_tag, p_16_lo, wt); + const auto shifted_32 = hn::ShiftRight<DIST_PRECISION_BITS>(wt_res_lo); + res_lo_16 = hn::BitCast( + int16xN_tag, + hn::ReorderDemote2To(uint16xN_tag, shifted_32, shifted_32)); + } else { + res_lo_16 = hn::ShiftRight<1>( + hn::BitCast(int16xN_tag, hn::Add(p_16, temp_lo_16))); + } + res_lo_16 = hn::Add(res_lo_16, res_sub_const); + res_lo_16 = + hn::ShiftRightSame(hn::Add(res_lo_16, round_bits_const), round_bits); + const auto res_8_lo = + hn::ReorderDemote2To(uint8xN_tag, res_lo_16, res_lo_16); + StoreRows(uint8x4_tag, uint8xN_tag, res_8_lo, p_stride, i + k, j, pred); + } else { + hn::Store( + hn::ResizeBitCast(uint16x4_tag, hn::ExtractBlock<0>(temp_lo_16)), + uint16x4_tag, pointers[0]); + if constexpr (kNumRows >= 2) { + hn::Store( + hn::ResizeBitCast(uint16x4_tag, hn::ExtractBlock<1>(temp_lo_16)), + uint16x4_tag, pointers[1]); + } + if constexpr (kNumRows >= 3) { + hn::Store( + hn::ResizeBitCast(uint16x4_tag, hn::ExtractBlock<2>(temp_lo_16)), + uint16x4_tag, pointers[2]); + hn::Store( + hn::ResizeBitCast(uint16x4_tag, hn::ExtractBlock<3>(temp_lo_16)), + uint16x4_tag, pointers[3]); + } + } + if (p_width > 4) { + uint16_t *HWY_RESTRICT pointers4[kNumRows]; + for (int row = 0; row < kNumRows; ++row) { + pointers4[row] = + &conv_params->dst[(i + k + row) * conv_params->dst_stride + j + 4]; + } + res_hi = + hn::ShiftRightSame(hn::Add(res_hi, res_add_const), reduce_bits_vert); + const auto temp_hi_16 = + hn::ReorderDemote2To(uint16xN_tag, res_hi, res_hi); + if (conv_params->do_average) { + auto p4_16 = hn::ResizeBitCast(uint16xN_tag, + hn::Load(uint16x4_tag, pointers4[0])); + if constexpr (kNumRows >= 2) { + p4_16 = hn::InsertBlock<1>( + p4_16, hn::ResizeBitCast(uint16x8_tag, + hn::Load(uint16x4_tag, pointers4[1]))); + } + if constexpr (kNumRows >= 3) { + p4_16 = hn::InsertBlock<2>( + p4_16, hn::ResizeBitCast(uint16x8_tag, + hn::Load(uint16x4_tag, pointers4[2]))); + p4_16 = hn::InsertBlock<3>( + p4_16, hn::ResizeBitCast(uint16x8_tag, + hn::Load(uint16x4_tag, pointers4[3]))); + } + + auto res_hi_16 = hn::Undefined(int16xN_tag); + if (conv_params->use_dist_wtd_comp_avg) { + const auto p_16_hi = + hn::BitCast(int16xN_tag, hn::InterleaveLower(p4_16, temp_hi_16)); + const auto wt_res_hi = + hn::WidenMulPairwiseAdd(int32xN_tag, p_16_hi, wt); + const auto shifted_32 = + hn::ShiftRight<DIST_PRECISION_BITS>(wt_res_hi); + res_hi_16 = hn::BitCast( + int16xN_tag, + hn::ReorderDemote2To(uint16xN_tag, shifted_32, shifted_32)); + } else { + res_hi_16 = hn::ShiftRight<1>( + hn::BitCast(int16xN_tag, hn::Add(p4_16, temp_hi_16))); + } + res_hi_16 = hn::Add(res_hi_16, res_sub_const); + res_hi_16 = hn::ShiftRightSame(hn::Add(res_hi_16, round_bits_const), + round_bits); + const auto res_8_hi = + hn::ReorderDemote2To(uint8xN_tag, res_hi_16, res_hi_16); + StoreRows(uint8x4_tag, uint8xN_tag, res_8_hi, p_stride, i + k, j + 4, + pred); + } else { + hn::Store(hn::ResizeBitCast(uint16x4_tag, temp_hi_16), uint16x4_tag, + pointers4[0]); + if constexpr (kNumRows >= 2) { + hn::Store( + hn::ResizeBitCast(uint16x4_tag, hn::ExtractBlock<1>(temp_hi_16)), + uint16x4_tag, pointers4[1]); + } + if constexpr (kNumRows >= 3) { + hn::Store( + hn::ResizeBitCast(uint16x4_tag, hn::ExtractBlock<2>(temp_hi_16)), + uint16x4_tag, pointers4[2]); + hn::Store( + hn::ResizeBitCast(uint16x4_tag, hn::ExtractBlock<3>(temp_hi_16)), + uint16x4_tag, pointers4[3]); + } + } + } + } else { + const auto res_lo_round = + hn::ShiftRightSame(hn::Add(res_lo, res_add_const), reduce_bits_vert); + const auto res_hi_round = + hn::ShiftRightSame(hn::Add(res_hi, res_add_const), reduce_bits_vert); + + const auto res_16bit = + hn::ReorderDemote2To(int16xN_tag, res_lo_round, res_hi_round); + const auto res_8bit = + hn::ReorderDemote2To(uint8xN_tag, res_16bit, res_16bit); + // Store, blending with 'pred' if needed + if (p_width == 4) { + StoreRows(uint8x4_tag, uint8xN_tag, res_8bit, p_stride, i + k, j, pred); + } else { + StoreRows(uint8x8_tag, uint8xN_tag, res_8bit, p_stride, i + k, j, pred); + } + } +} + +template <bool InnerCoeffUpdate, + void (*PrepareCoeffs)(int gamma, int delta, int sy, + int16_t *HWY_RESTRICT coeffs)> +HWY_ATTR inline void WarpVerticalFilterTemplate( + uint8_t *HWY_RESTRICT pred, int16_t *HWY_RESTRICT horz_out, + ConvolveParams *HWY_RESTRICT conv_params, int16_t gamma, int16_t delta, + int p_height, int p_stride, int p_width, int i, int j, int sy4, + const int reduce_bits_vert, const IVec32 res_add_const, + const int round_bits, const IVec16 res_sub_const, + const IVec16 round_bits_const, const IVec16 wt) { + HWY_ALIGN int16_t src_lo[4 * hn::MaxLanes(int16xN_tag)]; + HWY_ALIGN int16_t src_hi[4 * hn::MaxLanes(int16xN_tag)]; + if constexpr (int16xN_tag.MaxBlocks() >= 3) { + const auto horz_out_0 = + hn::Load(int16xN_tag, horz_out + 0 * hn::MaxLanes(int16x8_tag)); + const auto horz_out_1 = + hn::LoadU(int16xN_tag, horz_out + 1 * hn::MaxLanes(int16x8_tag)); + const auto horz_out_2 = + hn::LoadU(int16xN_tag, horz_out + 2 * hn::MaxLanes(int16x8_tag)); + const auto horz_out_3 = + hn::LoadU(int16xN_tag, horz_out + 3 * hn::MaxLanes(int16x8_tag)); + hn::Store(hn::InterleaveLower(int16xN_tag, horz_out_0, horz_out_1), + int16xN_tag, src_lo + 0 * hn::MaxLanes(int16xN_tag)); + hn::Store(hn::InterleaveUpper(int16xN_tag, horz_out_0, horz_out_1), + int16xN_tag, src_hi + 0 * hn::MaxLanes(int16xN_tag)); + hn::Store(hn::InterleaveLower(int16xN_tag, horz_out_2, horz_out_3), + int16xN_tag, src_lo + 1 * hn::MaxLanes(int16xN_tag)); + hn::Store(hn::InterleaveUpper(int16xN_tag, horz_out_2, horz_out_3), + int16xN_tag, src_hi + 1 * hn::MaxLanes(int16xN_tag)); + } else if constexpr (int16xN_tag.MaxBlocks() == 2) { + const auto horz_out_0 = + hn::Load(int16xN_tag, horz_out + 0 * hn::MaxLanes(int16xN_tag)); + const auto horz_out_2 = + hn::Load(int16xN_tag, horz_out + 1 * hn::MaxLanes(int16xN_tag)); + const auto horz_out_4 = + hn::Load(int16xN_tag, horz_out + 2 * hn::MaxLanes(int16xN_tag)); + const auto horz_out_6 = + hn::Load(int16xN_tag, horz_out + 3 * hn::MaxLanes(int16xN_tag)); + const auto horz_out_1 = + hn::ConcatLowerUpper(int16xN_tag, horz_out_2, horz_out_0); + const auto horz_out_3 = + hn::ConcatLowerUpper(int16xN_tag, horz_out_4, horz_out_2); + const auto horz_out_5 = + hn::ConcatLowerUpper(int16xN_tag, horz_out_6, horz_out_4); + hn::Store(hn::InterleaveLower(int16xN_tag, horz_out_0, horz_out_1), + int16xN_tag, src_lo + 0 * hn::MaxLanes(int16xN_tag)); + hn::Store(hn::InterleaveUpper(int16xN_tag, horz_out_0, horz_out_1), + int16xN_tag, src_hi + 0 * hn::MaxLanes(int16xN_tag)); + hn::Store(hn::InterleaveLower(int16xN_tag, horz_out_2, horz_out_3), + int16xN_tag, src_lo + 1 * hn::MaxLanes(int16xN_tag)); + hn::Store(hn::InterleaveUpper(int16xN_tag, horz_out_2, horz_out_3), + int16xN_tag, src_hi + 1 * hn::MaxLanes(int16xN_tag)); + hn::Store(hn::InterleaveLower(int16xN_tag, horz_out_4, horz_out_5), + int16xN_tag, src_lo + 2 * hn::MaxLanes(int16xN_tag)); + hn::Store(hn::InterleaveUpper(int16xN_tag, horz_out_4, horz_out_5), + int16xN_tag, src_hi + 2 * hn::MaxLanes(int16xN_tag)); + } else { + const auto horz_out_0 = + hn::Load(int16xN_tag, horz_out + 0 * hn::MaxLanes(int16xN_tag)); + const auto horz_out_1 = + hn::Load(int16xN_tag, horz_out + 1 * hn::MaxLanes(int16xN_tag)); + const auto horz_out_2 = + hn::Load(int16xN_tag, horz_out + 2 * hn::MaxLanes(int16xN_tag)); + const auto horz_out_3 = + hn::Load(int16xN_tag, horz_out + 3 * hn::MaxLanes(int16xN_tag)); + const auto horz_out_4 = + hn::Load(int16xN_tag, horz_out + 4 * hn::MaxLanes(int16xN_tag)); + const auto horz_out_5 = + hn::Load(int16xN_tag, horz_out + 5 * hn::MaxLanes(int16xN_tag)); + hn::Store(hn::InterleaveLower(int16xN_tag, horz_out_0, horz_out_1), + int16xN_tag, src_lo + 0 * hn::MaxLanes(int16xN_tag)); + hn::Store(hn::InterleaveUpper(int16xN_tag, horz_out_0, horz_out_1), + int16xN_tag, src_hi + 0 * hn::MaxLanes(int16xN_tag)); + hn::Store(hn::InterleaveLower(int16xN_tag, horz_out_2, horz_out_3), + int16xN_tag, src_lo + 1 * hn::MaxLanes(int16xN_tag)); + hn::Store(hn::InterleaveUpper(int16xN_tag, horz_out_2, horz_out_3), + int16xN_tag, src_hi + 1 * hn::MaxLanes(int16xN_tag)); + hn::Store(hn::InterleaveLower(int16xN_tag, horz_out_4, horz_out_5), + int16xN_tag, src_lo + 2 * hn::MaxLanes(int16xN_tag)); + hn::Store(hn::InterleaveUpper(int16xN_tag, horz_out_4, horz_out_5), + int16xN_tag, src_hi + 2 * hn::MaxLanes(int16xN_tag)); + } + + HWY_ALIGN int16_t coeffs[8 * hn::MaxLanes(int16xN_tag)]; + if constexpr (!InnerCoeffUpdate) { + PrepareCoeffs(gamma, delta, sy4, coeffs); + } + + for (int k = -4; k < AOMMIN(4, p_height - i - 4); + k += static_cast<int>(int16xN_tag.MaxBlocks())) { + if constexpr (InnerCoeffUpdate) { + int sy = sy4 + delta * (k + 4); + PrepareCoeffs(gamma, delta, sy, coeffs); + } + + IVec32 res_lo, res_hi; + FilterPixelsVertical(horz_out, src_lo, src_hi, coeffs, res_lo, res_hi, + k + 4); + StoreVerticalFilterOutput(res_lo, res_hi, res_add_const, wt, res_sub_const, + round_bits_const, pred, conv_params, i, j, k + 4, + reduce_bits_vert, p_stride, p_width, round_bits); + + if constexpr (int16xN_tag.MaxBlocks() >= 3) { + hn::Store(hn::Load(int16xN_tag, src_lo + 2 * hn::MaxLanes(int16xN_tag)), + int16xN_tag, src_lo + 0 * hn::MaxLanes(int16xN_tag)); + hn::Store(hn::Load(int16xN_tag, src_lo + 3 * hn::MaxLanes(int16xN_tag)), + int16xN_tag, src_lo + 1 * hn::MaxLanes(int16xN_tag)); + hn::Store(hn::Load(int16xN_tag, src_hi + 2 * hn::MaxLanes(int16xN_tag)), + int16xN_tag, src_hi + 0 * hn::MaxLanes(int16xN_tag)); + hn::Store(hn::Load(int16xN_tag, src_hi + 3 * hn::MaxLanes(int16xN_tag)), + int16xN_tag, src_hi + 1 * hn::MaxLanes(int16xN_tag)); + } else if constexpr (int16xN_tag.MaxBlocks() == 2) { + hn::Store(hn::Load(int16xN_tag, src_lo + 1 * hn::MaxLanes(int16xN_tag)), + int16xN_tag, src_lo + 0 * hn::MaxLanes(int16xN_tag)); + hn::Store(hn::Load(int16xN_tag, src_lo + 2 * hn::MaxLanes(int16xN_tag)), + int16xN_tag, src_lo + 1 * hn::MaxLanes(int16xN_tag)); + hn::Store(hn::Load(int16xN_tag, src_lo + 3 * hn::MaxLanes(int16xN_tag)), + int16xN_tag, src_lo + 2 * hn::MaxLanes(int16xN_tag)); + hn::Store(hn::Load(int16xN_tag, src_hi + 1 * hn::MaxLanes(int16xN_tag)), + int16xN_tag, src_hi + 0 * hn::MaxLanes(int16xN_tag)); + hn::Store(hn::Load(int16xN_tag, src_hi + 2 * hn::MaxLanes(int16xN_tag)), + int16xN_tag, src_hi + 1 * hn::MaxLanes(int16xN_tag)); + hn::Store(hn::Load(int16xN_tag, src_hi + 3 * hn::MaxLanes(int16xN_tag)), + int16xN_tag, src_hi + 2 * hn::MaxLanes(int16xN_tag)); + } else if constexpr (int16xN_tag.MaxBlocks() == 1) { + const auto src_lo_0 = + hn::Load(int16xN_tag, src_lo + 0 * hn::MaxLanes(int16xN_tag)); + const auto src_lo_1 = + hn::Load(int16xN_tag, src_lo + 1 * hn::MaxLanes(int16xN_tag)); + const auto src_lo_2 = + hn::Load(int16xN_tag, src_lo + 2 * hn::MaxLanes(int16xN_tag)); + const auto src_lo_3 = + hn::Load(int16xN_tag, src_lo + 3 * hn::MaxLanes(int16xN_tag)); + const auto src_lo_0_new = hn::InterleaveEven( + hn::ShiftRightLanes<1>(int16xN_tag, src_lo_0), src_lo_1); + const auto src_lo_1_new = hn::InterleaveEven( + hn::ShiftRightLanes<1>(int16xN_tag, src_lo_1), src_lo_2); + const auto src_lo_2_new = hn::InterleaveEven( + hn::ShiftRightLanes<1>(int16xN_tag, src_lo_2), src_lo_3); + hn::Store(src_lo_0_new, int16xN_tag, + src_lo + 0 * hn::MaxLanes(int16xN_tag)); + hn::Store(src_lo_1_new, int16xN_tag, + src_lo + 1 * hn::MaxLanes(int16xN_tag)); + hn::Store(src_lo_2_new, int16xN_tag, + src_lo + 2 * hn::MaxLanes(int16xN_tag)); + const auto src_hi_0 = + hn::Load(int16xN_tag, src_hi + 0 * hn::MaxLanes(int16xN_tag)); + const auto src_hi_1 = + hn::Load(int16xN_tag, src_hi + 1 * hn::MaxLanes(int16xN_tag)); + const auto src_hi_2 = + hn::Load(int16xN_tag, src_hi + 2 * hn::MaxLanes(int16xN_tag)); + const auto src_hi_3 = + hn::Load(int16xN_tag, src_hi + 3 * hn::MaxLanes(int16xN_tag)); + const auto src_hi_0_new = hn::InterleaveEven( + hn::ShiftRightLanes<1>(int16xN_tag, src_hi_0), src_hi_1); + const auto src_hi_1_new = hn::InterleaveEven( + hn::ShiftRightLanes<1>(int16xN_tag, src_hi_1), src_hi_2); + const auto src_hi_2_new = hn::InterleaveEven( + hn::ShiftRightLanes<1>(int16xN_tag, src_hi_2), src_hi_3); + hn::Store(src_hi_0_new, int16xN_tag, + src_hi + 0 * hn::MaxLanes(int16xN_tag)); + hn::Store(src_hi_1_new, int16xN_tag, + src_hi + 1 * hn::MaxLanes(int16xN_tag)); + hn::Store(src_hi_2_new, int16xN_tag, + src_hi + 2 * hn::MaxLanes(int16xN_tag)); + } + } +} + +HWY_ATTR inline void PrepareWarpVerticalFilter( + uint8_t *HWY_RESTRICT pred, int16_t *HWY_RESTRICT horz_out, + ConvolveParams *HWY_RESTRICT conv_params, int16_t gamma, int16_t delta, + int p_height, int p_stride, int p_width, int i, int j, int sy4, + const int reduce_bits_vert, const IVec32 res_add_const, + const int round_bits, const IVec16 res_sub_const, + const IVec16 round_bits_const, const IVec16 wt) { + if (gamma == 0 && delta == 0) + WarpVerticalFilterTemplate<false, PrepareVerticalFilterCoeffsGamma0>( + pred, horz_out, conv_params, gamma, delta, p_height, p_stride, p_width, + i, j, sy4, reduce_bits_vert, res_add_const, round_bits, res_sub_const, + round_bits_const, wt); + else if (gamma == 0 && delta != 0) + WarpVerticalFilterTemplate<true, PrepareVerticalFilterCoeffsGamma0>( + pred, horz_out, conv_params, gamma, delta, p_height, p_stride, p_width, + i, j, sy4, reduce_bits_vert, res_add_const, round_bits, res_sub_const, + round_bits_const, wt); + else if (gamma != 0 && delta == 0) + WarpVerticalFilterTemplate<false, PrepareVerticalFilterCoeffsDelta0>( + pred, horz_out, conv_params, gamma, delta, p_height, p_stride, p_width, + i, j, sy4, reduce_bits_vert, res_add_const, round_bits, res_sub_const, + round_bits_const, wt); + else + WarpVerticalFilterTemplate<true, PrepareVerticalFilterCoeffs>( + pred, horz_out, conv_params, gamma, delta, p_height, p_stride, p_width, + i, j, sy4, reduce_bits_vert, res_add_const, round_bits, res_sub_const, + round_bits_const, wt); +} + +HWY_ATTR inline void PrepareWarpHorizontalFilter( + const uint8_t *HWY_RESTRICT ref, int16_t *HWY_RESTRICT horz_out, int stride, + int32_t ix4, int32_t iy4, int32_t sx4, int alpha, int beta, int p_height, + int height, int i, const IVec16 round_const, const int reduce_bits_horiz) { + if (alpha == 0 && beta == 0) + WarpHorizontalFilterTemplate<false, HorizontalFilterCoeffs::kAlpha0>( + ref, horz_out, stride, ix4, iy4, sx4, alpha, beta, p_height, height, i, + round_const, reduce_bits_horiz); + else if (alpha == 0 && beta != 0) + WarpHorizontalFilterTemplate<true, HorizontalFilterCoeffs::kAlpha0>( + ref, horz_out, stride, ix4, iy4, sx4, alpha, beta, p_height, height, i, + round_const, reduce_bits_horiz); + else if (alpha != 0 && beta == 0) + WarpHorizontalFilterTemplate<false, HorizontalFilterCoeffs::kBeta0>( + ref, horz_out, stride, ix4, iy4, sx4, alpha, beta, p_height, height, i, + round_const, reduce_bits_horiz); + else + WarpHorizontalFilterTemplate<true, HorizontalFilterCoeffs::kDefault>( + ref, horz_out, stride, ix4, iy4, sx4, alpha, beta, p_height, height, i, + round_const, reduce_bits_horiz); +} + +template <typename D> +HWY_ATTR HWY_INLINE int WarpHorizontalFilterOutOfBoundsSetLoop( + D tag, const uint8_t *HWY_RESTRICT ref, int height, int stride, + int p_height, int i, int iy4, int16_t const4, int16_t const5, int offset, + int k, int16_t *HWY_RESTRICT horz_out) { + constexpr int kNumRows = tag.MaxBlocks(); + for (; k < AOMMIN(8, p_height - i) - kNumRows; k += kNumRows) { + int iy = clamp(iy4 + k + 0, 0, height - 1); + auto src = hn::ResizeBitCast( + tag, hn::Set(int16x8_tag, const4 + ref[iy * stride + offset] * const5)); + if constexpr (kNumRows >= 2) { + iy = clamp(iy4 + k + 1, 0, height - 1); + src = hn::InsertBlock<1>( + src, + hn::Set(int16x8_tag, const4 + ref[iy * stride + offset] * const5)); + } + if constexpr (kNumRows >= 3) { + iy = clamp(iy4 + k + 2, 0, height - 1); + src = hn::InsertBlock<2>( + src, + hn::Set(int16x8_tag, const4 + ref[iy * stride + offset] * const5)); + iy = clamp(iy4 + k + 3, 0, height - 1); + src = hn::InsertBlock<3>( + src, + hn::Set(int16x8_tag, const4 + ref[iy * stride + offset] * const5)); + } + hn::Store(src, tag, horz_out + (k + 7) * hn::MaxLanes(int16x8_tag)); + } + return k; +} + +HWY_ATTR void WarpHorizontalFilterOutOfBoundsSet( + const uint8_t *HWY_RESTRICT ref, int height, int stride, int p_height, + int i, int iy4, int16_t const4, int16_t const5, int offset, + int16_t *HWY_RESTRICT horz_out) { + int k = -7, iy; + if constexpr (int16xN_tag.MaxBlocks() >= 3) { + k = WarpHorizontalFilterOutOfBoundsSetLoop(int16xN_tag, ref, height, stride, + p_height, i, iy4, const4, const5, + offset, k, horz_out); + } + if constexpr (int16xN_tag.MaxBlocks() >= 2) { + k = WarpHorizontalFilterOutOfBoundsSetLoop(int16x16_tag, ref, height, + stride, p_height, i, iy4, const4, + const5, offset, k, horz_out); + } + if constexpr (int16xN_tag.MaxBlocks() == 1) { + k = WarpHorizontalFilterOutOfBoundsSetLoop(int16x8_tag, ref, height, stride, + p_height, i, iy4, const4, const5, + offset, k, horz_out); + } + iy = iy4 + k; + iy = clamp(iy4 + k, 0, height - 1); + hn::Store(hn::Set(int16x8_tag, const4 + ref[iy * stride + offset] * const5), + int16x8_tag, horz_out + (k + 7) * hn::MaxLanes(int16x8_tag)); +} + +template <typename D> +HWY_ATTR int WarpHorizontalFilterOutOfBoundsPadLoop( + D tag, const uint8_t *HWY_RESTRICT ref, int stride, int32_t ix4, + int32_t iy4, int32_t sx4, int alpha, int beta, int p_height, int height, + int i, const IVec16 round_const, const int reduce_bits_horiz, + int out_of_boundary_left, int out_of_boundary_right, int k, + int16_t *HWY_RESTRICT horz_out) { + constexpr int kNumRows = tag.MaxBlocks(); + for (; k < (AOMMIN(8, p_height - i) - kNumRows); k += kNumRows) { + auto src = LoadRowsClamped(tag, ref + ix4 - 7, stride, iy4 + k, height); + if (out_of_boundary_left >= 0) { + const auto shuffle_reg_left = + hn::LoadDup128(tag, warp_pad_left[out_of_boundary_left]); + src = hn::TableLookupBytes(src, shuffle_reg_left); + } + if (out_of_boundary_right >= 0) { + const auto shuffle_reg_right = + hn::LoadDup128(tag, warp_pad_right[out_of_boundary_right]); + src = hn::TableLookupBytes(src, shuffle_reg_right); + } + int sx = sx4 + beta * (k + 4); + HorizontalFilter(tag, src, horz_out, sx, alpha, beta, k + 7, round_const, + reduce_bits_horiz); + } + return k; +} + +HWY_ATTR void WarpHorizontalFilterOutOfBoundsPad( + const uint8_t *HWY_RESTRICT ref, int stride, int32_t ix4, int32_t iy4, + int32_t sx4, int alpha, int beta, int p_height, int width, int height, + int i, const IVec16 round_const, const int reduce_bits_horiz, + int16_t *HWY_RESTRICT horz_out) { + const int out_of_boundary_left = -(ix4 - 6); + const int out_of_boundary_right = (ix4 + 8) - width; + int k = -7, iy, sx; + if constexpr (uint8xN_tag.MaxBlocks() >= 3) { + k = WarpHorizontalFilterOutOfBoundsPadLoop( + uint8xN_tag, ref, stride, ix4, iy4, sx4, alpha, beta, p_height, height, + i, round_const, reduce_bits_horiz, out_of_boundary_left, + out_of_boundary_right, k, horz_out); + } + if constexpr (uint8xN_tag.MaxBlocks() >= 2) { + k = WarpHorizontalFilterOutOfBoundsPadLoop( + uint8x32_tag, ref, stride, ix4, iy4, sx4, alpha, beta, p_height, height, + i, round_const, reduce_bits_horiz, out_of_boundary_left, + out_of_boundary_right, k, horz_out); + } + if constexpr (uint8xN_tag.MaxBlocks() == 1) { + k = WarpHorizontalFilterOutOfBoundsPadLoop( + uint8xN_tag, ref, stride, ix4, iy4, sx4, alpha, beta, p_height, height, + i, round_const, reduce_bits_horiz, out_of_boundary_left, + out_of_boundary_right, k, horz_out); + } + iy = iy4 + k; + iy = clamp(iy, 0, height - 1); + auto src = hn::LoadU(uint8x16_tag, ref + iy * stride + ix4 - 7); + if (out_of_boundary_left >= 0) { + const auto shuffle_reg_left = + hn::LoadU(uint8x16_tag, warp_pad_left[out_of_boundary_left]); + src = hn::TableLookupBytes(src, shuffle_reg_left); + } + if (out_of_boundary_right >= 0) { + const auto shuffle_reg_right = + hn::LoadU(uint8x16_tag, warp_pad_right[out_of_boundary_right]); + src = hn::TableLookupBytes(src, shuffle_reg_right); + } + sx = sx4 + beta * (k + 4); + HWY_ALIGN int8_t coeff[4 * hn::MaxLanes(coeff_tag)]; + PrepareLastHorizontalFilterCoefficients(int16xN_tag, alpha, beta, sx, coeff); + FilterPixelsHorizontal(uint8x16_tag, src, horz_out, coeff, round_const, + reduce_bits_horiz, k + 7); +} + +HWY_ATTR void WarpAffine(const int32_t *HWY_RESTRICT mat, + const uint8_t *HWY_RESTRICT ref, int width, int height, + int stride, uint8_t *HWY_RESTRICT pred, int p_col, + int p_row, int p_width, int p_height, int p_stride, + int subsampling_x, int subsampling_y, + ConvolveParams *HWY_RESTRICT conv_params, + int16_t alpha, int16_t beta, int16_t gamma, + int16_t delta) { + int i, j; + const int bd = 8; + const int reduce_bits_horiz = conv_params->round_0; + const int reduce_bits_vert = conv_params->is_compound + ? conv_params->round_1 + : 2 * FILTER_BITS - reduce_bits_horiz; + const int offset_bits_horiz = bd + FILTER_BITS - 1; + assert(IMPLIES(conv_params->is_compound, conv_params->dst != NULL)); + + const int offset_bits_vert = bd + 2 * FILTER_BITS - reduce_bits_horiz; + const auto reduce_bits_vert_const = + hn::Set(int32xN_tag, ((1 << reduce_bits_vert) >> 1)); + const auto res_add_const = hn::Set(int32xN_tag, 1 << offset_bits_vert); + const int round_bits = + 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1; + const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0; + assert(IMPLIES(conv_params->do_average, conv_params->is_compound)); + + const auto round_const = hn::Set( + int16xN_tag, (1 << offset_bits_horiz) + ((1 << reduce_bits_horiz) >> 1)); + + IVec16 res_sub_const, round_bits_const, wt; + UnpackWeightsAndSetRoundConst(conv_params, round_bits, offset_bits, + res_sub_const, round_bits_const, wt); + + IVec32 res_add_const_1; + if (conv_params->is_compound == 1) { + res_add_const_1 = hn::Add(reduce_bits_vert_const, res_add_const); + } else { + res_add_const_1 = hn::Set(int32xN_tag, -(1 << (bd + reduce_bits_vert - 1)) + + ((1 << reduce_bits_vert) >> 1)); + } + const int32_t const1 = alpha * (-4) + beta * (-4) + + (1 << (WARPEDDIFF_PREC_BITS - 1)) + + (WARPEDPIXEL_PREC_SHIFTS << WARPEDDIFF_PREC_BITS); + const int32_t const2 = gamma * (-4) + delta * (-4) + + (1 << (WARPEDDIFF_PREC_BITS - 1)) + + (WARPEDPIXEL_PREC_SHIFTS << WARPEDDIFF_PREC_BITS); + const int32_t const3 = ((1 << WARP_PARAM_REDUCE_BITS) - 1); + const int16_t const4 = (1 << (bd + FILTER_BITS - reduce_bits_horiz - 1)); + const int16_t const5 = (1 << (FILTER_BITS - reduce_bits_horiz)); + + for (i = 0; i < p_height; i += 8) { + for (j = 0; j < p_width; j += 8) { + HWY_ALIGN int16_t horz_out[8 * 16 + hn::MaxLanes(int16xN_tag)]; + const int32_t src_x = (p_col + j + 4) << subsampling_x; + const int32_t src_y = (p_row + i + 4) << subsampling_y; + const int64_t dst_x = + (int64_t)mat[2] * src_x + (int64_t)mat[3] * src_y + (int64_t)mat[0]; + const int64_t dst_y = + (int64_t)mat[4] * src_x + (int64_t)mat[5] * src_y + (int64_t)mat[1]; + const int64_t x4 = dst_x >> subsampling_x; + const int64_t y4 = dst_y >> subsampling_y; + + int32_t ix4 = (int32_t)(x4 >> WARPEDMODEL_PREC_BITS); + int32_t sx4 = x4 & ((1 << WARPEDMODEL_PREC_BITS) - 1); + int32_t iy4 = (int32_t)(y4 >> WARPEDMODEL_PREC_BITS); + int32_t sy4 = y4 & ((1 << WARPEDMODEL_PREC_BITS) - 1); + + // Add in all the constant terms, including rounding and offset + sx4 += const1; + sy4 += const2; + + sx4 &= ~const3; + sy4 &= ~const3; + + // Horizontal filter + // If the block is aligned such that, after clamping, every sample + // would be taken from the leftmost/rightmost column, then we can + // skip the expensive horizontal filter. + + if (ix4 <= -7) { + WarpHorizontalFilterOutOfBoundsSet(ref, height, stride, p_height, i, + iy4, const4, const5, 0, horz_out); + } else if (ix4 >= width + 6) { + WarpHorizontalFilterOutOfBoundsSet(ref, height, stride, p_height, i, + iy4, const4, const5, width - 1, + horz_out); + } else if (((ix4 - 7) < 0) || ((ix4 + 9) > width)) { + WarpHorizontalFilterOutOfBoundsPad( + ref, stride, ix4, iy4, sx4, alpha, beta, p_height, width, height, i, + round_const, reduce_bits_horiz, horz_out); + } else { + PrepareWarpHorizontalFilter(ref, horz_out, stride, ix4, iy4, sx4, alpha, + beta, p_height, height, i, round_const, + reduce_bits_horiz); + } + + // Vertical filter + PrepareWarpVerticalFilter(pred, horz_out, conv_params, gamma, delta, + p_height, p_stride, p_width, i, j, sy4, + reduce_bits_vert, res_add_const_1, round_bits, + res_sub_const, round_bits_const, wt); + } + } +} + +} // namespace HWY_NAMESPACE +} // namespace + +#define MAKE_WARP_AFFINE(suffix) \ + extern "C" void av1_warp_affine_##suffix( \ + const int32_t *HWY_RESTRICT mat, const uint8_t *HWY_RESTRICT ref, \ + int width, int height, int stride, uint8_t *HWY_RESTRICT pred, \ + int p_col, int p_row, int p_width, int p_height, int p_stride, \ + int subsampling_x, int subsampling_y, \ + ConvolveParams *HWY_RESTRICT conv_params, int16_t alpha, int16_t beta, \ + int16_t gamma, int16_t delta); \ + HWY_ATTR void av1_warp_affine_##suffix( \ + const int32_t *HWY_RESTRICT mat, const uint8_t *HWY_RESTRICT ref, \ + int width, int height, int stride, uint8_t *HWY_RESTRICT pred, \ + int p_col, int p_row, int p_width, int p_height, int p_stride, \ + int subsampling_x, int subsampling_y, \ + ConvolveParams *HWY_RESTRICT conv_params, int16_t alpha, int16_t beta, \ + int16_t gamma, int16_t delta) { \ + HWY_NAMESPACE::WarpAffine(mat, ref, width, height, stride, pred, p_col, \ + p_row, p_width, p_height, p_stride, \ + subsampling_x, subsampling_y, conv_params, \ + alpha, beta, gamma, delta); \ + } + +HWY_AFTER_NAMESPACE(); + +#endif // AOM_AV1_COMMON_WARP_PLANE_HWY_H_
diff --git a/av1/common/x86/av1_inv_txfm_avx2.c b/av1/common/x86/av1_inv_txfm_avx2.c index edee095..78ea98a 100644 --- a/av1/common/x86/av1_inv_txfm_avx2.c +++ b/av1/common/x86/av1_inv_txfm_avx2.c
@@ -9,6 +9,8 @@ * PATENTS file, you can obtain it at www.aomedia.org/license/patent. */ +#include <assert.h> + #include "config/aom_config.h" #include "config/av1_rtcd.h" @@ -2226,16 +2228,8 @@ lowbd_inv_txfm2d_add_8x8_avx2(input, output, stride, tx_type, tx_size, eob); break; - case TX_16X16: - case TX_32X32: - case TX_64X64: - case TX_16X32: - case TX_32X16: - case TX_32X64: - case TX_64X32: - case TX_16X64: - case TX_64X16: default: + assert(tx_size < TX_SIZES_ALL); lowbd_inv_txfm2d_add_universe_avx2(input, output, stride, tx_type, tx_size, eob); break;
diff --git a/av1/common/x86/convolve_2d_avx2.c b/av1/common/x86/convolve_2d_avx2.c index 521326c..be53424 100644 --- a/av1/common/x86/convolve_2d_avx2.c +++ b/av1/common/x86/convolve_2d_avx2.c
@@ -13,21 +13,81 @@ #include "config/av1_rtcd.h" -#if CONFIG_SVT_AV1 -#include "third_party/SVT-AV1/convolve_2d_avx2.h" -#endif - #include "aom_dsp/x86/convolve_avx2.h" #include "aom_dsp/aom_filter.h" #include "aom_dsp/x86/synonyms.h" #include "av1/common/convolve.h" -static void convolve_2d_sr_general_avx2( - const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, - int h, const InterpFilterParams *filter_params_x, - const InterpFilterParams *filter_params_y, const int subpel_x_qn, - const int subpel_y_qn, ConvolveParams *conv_params) { +static void convolve_2d_sr_w4_avx2( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int32_t w, int32_t h, const InterpFilterParams *filter_params_x, + const InterpFilterParams *filter_params_y, const int32_t subpel_x_qn, + const int32_t subpel_y_qn, ConvolveParams *conv_params) { + int i; + DECLARE_ALIGNED(32, int16_t, im_block[(MAX_SB_SIZE + MAX_FILTER_TAP) * 4]); + uint8_t *dst_ptr = dst; + assert(conv_params->round_0 == 3); + assert(conv_params->round_1 == 11); + + const __m128i round_const_h = _mm_set1_epi16(1 << (conv_params->round_0 - 2)); + const __m256i round_const_v = + _mm256_set1_epi32(1 << (conv_params->round_1 - 1)); + + __m128i filt[2], coeffs_h[2] = { 0 }; + __m256i coeffs_v[4] = { 0 }; + + const int horiz_tap = get_filter_tap(filter_params_x, subpel_x_qn); + const int vert_tap = get_filter_tap(filter_params_y, subpel_y_qn); + + assert(horiz_tap == 2 || horiz_tap == 4); + assert(vert_tap == 2 || vert_tap == 4 || vert_tap == 6 || vert_tap == 8); + + if (horiz_tap == 2) + prepare_coeffs_2t_ssse3(filter_params_x, subpel_x_qn, coeffs_h); + else + prepare_coeffs_4t_ssse3(filter_params_x, subpel_x_qn, coeffs_h); + + if (vert_tap == 2) + prepare_coeffs_2t(filter_params_y, subpel_y_qn, coeffs_v); + else if (vert_tap == 4) + prepare_coeffs_4t(filter_params_y, subpel_y_qn, coeffs_v); + else if (vert_tap == 6) + prepare_coeffs_6t(filter_params_y, subpel_y_qn, coeffs_v); + else + prepare_coeffs(filter_params_y, subpel_y_qn, coeffs_v); + + int im_h = h + vert_tap - 1; + const int fo_vert = vert_tap / 2 - 1; + const int fo_horiz = horiz_tap / 2 - 1; + const uint8_t *const src_ptr = src - fo_vert * src_stride - fo_horiz; + + filt[0] = _mm_load_si128((__m128i const *)filt1_global_sse2); + filt[1] = _mm_load_si128((__m128i const *)filt2_global_sse2); + + if (horiz_tap == 2) { + CONVOLVE_SR_HOR_FILTER_2TAP_W4 + } else { + CONVOLVE_SR_HOR_FILTER_4TAP_W4 + } + + if (vert_tap == 2) { + CONVOLVE_SR_VER_FILTER_2TAP_W4 + } else if (vert_tap == 4) { + CONVOLVE_SR_VER_FILTER_4TAP_W4 + } else if (vert_tap == 6) { + CONVOLVE_SR_VER_FILTER_6TAP_W4 + } else { + CONVOLVE_SR_VER_FILTER_8TAP_W4 + } +} + +static void convolve_2d_sr_avx2(const uint8_t *src, int src_stride, + uint8_t *dst, int dst_stride, int w, int h, + const InterpFilterParams *filter_params_x, + const InterpFilterParams *filter_params_y, + const int subpel_x_qn, const int subpel_y_qn, + ConvolveParams *conv_params) { if (filter_params_x->taps > 8) { const int bd = 8; int im_stride = 8, i; @@ -69,43 +129,40 @@ CONVOLVE_SR_VERTICAL_FILTER_12TAP } } else { - const int bd = 8; int im_stride = 8, i; DECLARE_ALIGNED(32, int16_t, im_block[(MAX_SB_SIZE + MAX_FILTER_TAP) * 8]); - const int bits = - FILTER_BITS * 2 - conv_params->round_0 - conv_params->round_1; - const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0; - assert(conv_params->round_0 > 0); + assert(conv_params->round_0 == 3); + assert(conv_params->round_1 == 11); const __m256i round_const_h = - _mm256_set1_epi16(((1 << (conv_params->round_0 - 1)) >> 1) + - (1 << (bd + FILTER_BITS - 2))); - const __m128i round_shift_h = _mm_cvtsi32_si128(conv_params->round_0 - 1); + _mm256_set1_epi16(1 << (conv_params->round_0 - 2)); + const __m256i round_const_v = + _mm256_set1_epi32(1 << (conv_params->round_1 - 1)); - const __m256i sum_round_v = _mm256_set1_epi32( - (1 << offset_bits) + ((1 << conv_params->round_1) >> 1)); - const __m128i sum_shift_v = _mm_cvtsi32_si128(conv_params->round_1); - - const __m256i round_const_v = _mm256_set1_epi32( - ((1 << bits) >> 1) - (1 << (offset_bits - conv_params->round_1)) - - ((1 << (offset_bits - conv_params->round_1)) >> 1)); - const __m128i round_shift_v = _mm_cvtsi32_si128(bits); - - __m256i filt[4], coeffs_h[4], coeffs_v[4]; - - prepare_coeffs_lowbd(filter_params_x, subpel_x_qn, coeffs_h); - prepare_coeffs(filter_params_y, subpel_y_qn, coeffs_v); + __m256i filt[4], coeffs_h[4] = { 0 }, coeffs_v[4] = { 0 }; int horiz_tap = get_filter_tap(filter_params_x, subpel_x_qn); int vert_tap = get_filter_tap(filter_params_y, subpel_y_qn); - if (horiz_tap == 6) + assert(horiz_tap == 2 || horiz_tap == 4 || horiz_tap == 6 || + horiz_tap == 8); + assert(vert_tap == 2 || vert_tap == 4 || vert_tap == 6 || vert_tap == 8); + + if (horiz_tap == 2) + prepare_coeffs_2t_lowbd(filter_params_x, subpel_x_qn, coeffs_h); + else if (horiz_tap == 4) + prepare_coeffs_4t_lowbd(filter_params_x, subpel_x_qn, coeffs_h); + else if (horiz_tap == 6) prepare_coeffs_6t_lowbd(filter_params_x, subpel_x_qn, coeffs_h); else prepare_coeffs_lowbd(filter_params_x, subpel_x_qn, coeffs_h); - if (vert_tap == 6) + if (vert_tap == 2) + prepare_coeffs_2t(filter_params_y, subpel_y_qn, coeffs_v); + else if (vert_tap == 4) + prepare_coeffs_4t(filter_params_y, subpel_y_qn, coeffs_v); + else if (vert_tap == 6) prepare_coeffs_6t(filter_params_y, subpel_y_qn, coeffs_v); else prepare_coeffs(filter_params_y, subpel_y_qn, coeffs_v); @@ -121,7 +178,9 @@ filt[3] = _mm256_load_si256((__m256i const *)filt4_global_avx2); for (int j = 0; j < w; j += 8) { - if (horiz_tap == 4) { + if (horiz_tap == 2) { + CONVOLVE_SR_HORIZONTAL_FILTER_2TAP + } else if (horiz_tap == 4) { CONVOLVE_SR_HORIZONTAL_FILTER_4TAP } else if (horiz_tap == 6) { CONVOLVE_SR_HORIZONTAL_FILTER_6TAP @@ -129,7 +188,10 @@ CONVOLVE_SR_HORIZONTAL_FILTER_8TAP } - if (vert_tap == 4) { + uint8_t *dst_ptr = dst + j; + if (vert_tap == 2) { + CONVOLVE_SR_VERTICAL_FILTER_2TAP + } else if (vert_tap == 4) { CONVOLVE_SR_VERTICAL_FILTER_4TAP } else if (vert_tap == 6) { CONVOLVE_SR_VERTICAL_FILTER_6TAP @@ -145,23 +207,16 @@ int32_t w, int32_t h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int32_t subpel_x_qn, const int32_t subpel_y_qn, ConvolveParams *conv_params) { -#if CONFIG_SVT_AV1 const int32_t tap_x = get_filter_tap(filter_params_x, subpel_x_qn); const int32_t tap_y = get_filter_tap(filter_params_y, subpel_y_qn); - const bool use_general = (tap_x == 12 || tap_y == 12); - if (use_general) { - convolve_2d_sr_general_avx2(src, src_stride, dst, dst_stride, w, h, - filter_params_x, filter_params_y, subpel_x_qn, - subpel_y_qn, conv_params); + const bool use_12tap = (tap_x == 12 || tap_y == 12); + if (w <= 4 && !use_12tap) { + convolve_2d_sr_w4_avx2(src, src_stride, dst, dst_stride, w, h, + filter_params_x, filter_params_y, subpel_x_qn, + subpel_y_qn, conv_params); } else { - av1_convolve_2d_sr_specialized_avx2(src, src_stride, dst, dst_stride, w, h, - filter_params_x, filter_params_y, - subpel_x_qn, subpel_y_qn, conv_params); + convolve_2d_sr_avx2(src, src_stride, dst, dst_stride, w, h, filter_params_x, + filter_params_y, subpel_x_qn, subpel_y_qn, conv_params); } -#else - convolve_2d_sr_general_avx2(src, src_stride, dst, dst_stride, w, h, - filter_params_x, filter_params_y, subpel_x_qn, - subpel_y_qn, conv_params); -#endif }
diff --git a/av1/common/x86/convolve_avx2.c b/av1/common/x86/convolve_avx2.c index d250d88..c2c3c7c 100644 --- a/av1/common/x86/convolve_avx2.c +++ b/av1/common/x86/convolve_avx2.c
@@ -13,243 +13,512 @@ #include "config/av1_rtcd.h" -#if CONFIG_SVT_AV1 -#include "third_party/SVT-AV1/convolve_avx2.h" -#endif - #include "aom_dsp/aom_dsp_common.h" #include "aom_dsp/x86/convolve_avx2.h" #include "aom_dsp/x86/convolve_common_intrin.h" #include "aom_dsp/x86/synonyms.h" -static inline void av1_convolve_y_sr_general_avx2( - const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, - int h, const InterpFilterParams *filter_params_y, const int subpel_y_qn) { - // right shift is F-1 because we are already dividing - // filter co-efficients by 2 - const int right_shift_bits = (FILTER_BITS - 1); - __m128i right_shift = _mm_cvtsi32_si128(right_shift_bits); - __m256i right_shift_const = _mm256_set1_epi16((1 << right_shift_bits) >> 1); - - __m256i coeffs[6], s[12]; - __m128i d[10]; +void av1_convolve_y_sr_avx2(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, int32_t w, + int32_t h, + const InterpFilterParams *filter_params_y, + const int32_t subpel_y_qn) { + __m128i coeffs_128[4]; + __m256i coeffs[6]; + int x = 0, y = h; int i, vert_tap = get_filter_tap(filter_params_y, subpel_y_qn); + assert(vert_tap == 2 || vert_tap == 4 || vert_tap == 6 || vert_tap == 8 || + vert_tap == 12); + assert(!(w % 2)); + assert(!(h % 2)); - if (vert_tap == 6) - prepare_coeffs_6t_lowbd(filter_params_y, subpel_y_qn, coeffs); - else if (vert_tap == 12) { - prepare_coeffs_12taps(filter_params_y, subpel_y_qn, coeffs); - } else { - prepare_coeffs_lowbd(filter_params_y, subpel_y_qn, coeffs); - } + const int fo_vert = vert_tap / 2 - 1; + const uint8_t *const src_ptr = src - fo_vert * src_stride; + const uint8_t *data = src_ptr; + uint8_t *dst_ptr = dst; - // vert_filt as 4 tap - if (vert_tap == 4) { - const int fo_vert = 1; - const uint8_t *const src_ptr = src - fo_vert * src_stride; - for (int j = 0; j < w; j += 16) { - const uint8_t *data = &src_ptr[j]; - d[0] = _mm_loadu_si128((__m128i *)(data + 0 * src_stride)); - d[1] = _mm_loadu_si128((__m128i *)(data + 1 * src_stride)); - d[2] = _mm_loadu_si128((__m128i *)(data + 2 * src_stride)); - d[3] = _mm_loadu_si128((__m128i *)(data + 3 * src_stride)); - d[4] = _mm_loadu_si128((__m128i *)(data + 4 * src_stride)); + if (vert_tap == 2) { + if (subpel_y_qn != 8) { + if (w <= 4) { + prepare_coeffs_2t_ssse3(filter_params_y, subpel_y_qn, coeffs_128); + __m128i d[2], res; + if (w == 2) { + d[0] = _mm_cvtsi32_si128(loadu_int16(data)); - // Load lines a and b. Line a to lower 128, line b to upper 128 - const __m256i src_01a = _mm256_permute2x128_si256( - _mm256_castsi128_si256(d[0]), _mm256_castsi128_si256(d[1]), 0x20); + do { + convolve_y_2tap_2x2_ssse3(data, src_stride, coeffs_128, d, &res); + res = round_sr_y_ssse3(res); + pack_store_u8_2x2_sse2(res, dst_ptr, dst_stride); - const __m256i src_12a = _mm256_permute2x128_si256( - _mm256_castsi128_si256(d[1]), _mm256_castsi128_si256(d[2]), 0x20); - - const __m256i src_23a = _mm256_permute2x128_si256( - _mm256_castsi128_si256(d[2]), _mm256_castsi128_si256(d[3]), 0x20); - - const __m256i src_34a = _mm256_permute2x128_si256( - _mm256_castsi128_si256(d[3]), _mm256_castsi128_si256(d[4]), 0x20); - - s[0] = _mm256_unpacklo_epi8(src_01a, src_12a); - s[1] = _mm256_unpacklo_epi8(src_23a, src_34a); - - s[3] = _mm256_unpackhi_epi8(src_01a, src_12a); - s[4] = _mm256_unpackhi_epi8(src_23a, src_34a); - - for (i = 0; i < h; i += 2) { - data = &src_ptr[i * src_stride + j]; - d[5] = _mm_loadu_si128((__m128i *)(data + 5 * src_stride)); - const __m256i src_45a = _mm256_permute2x128_si256( - _mm256_castsi128_si256(d[4]), _mm256_castsi128_si256(d[5]), 0x20); - - d[4] = _mm_loadu_si128((__m128i *)(data + 6 * src_stride)); - const __m256i src_56a = _mm256_permute2x128_si256( - _mm256_castsi128_si256(d[5]), _mm256_castsi128_si256(d[4]), 0x20); - - s[2] = _mm256_unpacklo_epi8(src_45a, src_56a); - s[5] = _mm256_unpackhi_epi8(src_45a, src_56a); - - const __m256i res_lo = convolve_lowbd_4tap(s, coeffs + 1); - /* rounding code */ - // shift by F - 1 - const __m256i res_16b_lo = _mm256_sra_epi16( - _mm256_add_epi16(res_lo, right_shift_const), right_shift); - // 8 bit conversion and saturation to uint8 - __m256i res_8b_lo = _mm256_packus_epi16(res_16b_lo, res_16b_lo); - - if (w - j > 8) { - const __m256i res_hi = convolve_lowbd_4tap(s + 3, coeffs + 1); - - /* rounding code */ - // shift by F - 1 - const __m256i res_16b_hi = _mm256_sra_epi16( - _mm256_add_epi16(res_hi, right_shift_const), right_shift); - // 8 bit conversion and saturation to uint8 - __m256i res_8b_hi = _mm256_packus_epi16(res_16b_hi, res_16b_hi); - - __m256i res_a = _mm256_unpacklo_epi64(res_8b_lo, res_8b_hi); - - const __m128i res_0 = _mm256_castsi256_si128(res_a); - const __m128i res_1 = _mm256_extracti128_si256(res_a, 1); - - _mm_storeu_si128((__m128i *)&dst[i * dst_stride + j], res_0); - _mm_storeu_si128((__m128i *)&dst[i * dst_stride + j + dst_stride], - res_1); + dst_ptr += 2 * dst_stride; + data += 2 * src_stride; + y -= 2; + } while (y > 0); } else { - const __m128i res_0 = _mm256_castsi256_si128(res_8b_lo); - const __m128i res_1 = _mm256_extracti128_si256(res_8b_lo, 1); - if (w - j > 4) { - _mm_storel_epi64((__m128i *)&dst[i * dst_stride + j], res_0); - _mm_storel_epi64((__m128i *)&dst[i * dst_stride + j + dst_stride], - res_1); - } else if (w - j > 2) { - xx_storel_32(&dst[i * dst_stride + j], res_0); - xx_storel_32(&dst[i * dst_stride + j + dst_stride], res_1); - } else { - __m128i *const p_0 = (__m128i *)&dst[i * dst_stride + j]; - __m128i *const p_1 = - (__m128i *)&dst[i * dst_stride + j + dst_stride]; - *(uint16_t *)p_0 = (uint16_t)_mm_cvtsi128_si32(res_0); - *(uint16_t *)p_1 = (uint16_t)_mm_cvtsi128_si32(res_1); - } - } - s[0] = s[1]; - s[1] = s[2]; + assert(w == 4); + d[0] = _mm_cvtsi32_si128(loadu_int32(data)); - s[3] = s[4]; - s[4] = s[5]; + do { + convolve_y_2tap_4x2_ssse3(data, src_stride, coeffs_128, d, &res); + res = round_sr_y_ssse3(res); + pack_store_u8_4x2_sse2(res, dst_ptr, dst_stride); + + dst_ptr += 2 * dst_stride; + data += 2 * src_stride; + y -= 2; + } while (y > 0); + } + } else { + prepare_coeffs_2t_lowbd(filter_params_y, subpel_y_qn, coeffs); + + if (w == 8) { + __m128i d[2]; + d[0] = _mm_loadl_epi64((__m128i *)data); + + do { + __m256i res; + convolve_y_2tap_8x2_avx2(data, src_stride, coeffs, d, &res); + round_pack_store_y_8x2_avx2(res, dst_ptr, dst_stride); + + dst_ptr += 2 * dst_stride; + data += 2 * src_stride; + y -= 2; + + } while (y > 0); + + } else if (w == 16) { + __m128i d[2]; + d[0] = _mm_loadu_si128((__m128i *)data); + + do { + __m256i res[2]; + convolve_y_2tap_16x2_avx2(data, src_stride, coeffs, d, res); + round_pack_store_y_16x2_avx2(res, dst_ptr, dst_stride); + + dst_ptr += 2 * dst_stride; + data += 2 * src_stride; + y -= 2; + } while (y > 0); + + } else { + assert(!(w % 32)); + + __m256i d[2]; + do { + data = src_ptr + x; + dst_ptr = dst + x; + y = h; + + d[0] = _mm256_loadu_si256((__m256i *)data); + + do { + __m256i res[4]; + convolve_y_2tap_32x2_avx2(data, src_stride, coeffs, d, res); + round_pack_store_y_32x2_avx2(res, dst_ptr, dst_stride); + + dst_ptr += 2 * dst_stride; + data += 2 * src_stride; + y -= 2; + } while (y > 0); + + x += 32; + } while (x < w); + } + } + } else { + if (w <= 16) { + __m128i s[2], res; + + if (w == 2) { + s[0] = _mm_cvtsi32_si128(*(int16_t *)data); + + do { + s[1] = _mm_cvtsi32_si128(*(int16_t *)(data + src_stride)); + res = _mm_avg_epu8(s[0], s[1]); + *(int16_t *)dst_ptr = (int16_t)_mm_cvtsi128_si32(res); + s[0] = _mm_cvtsi32_si128(*(int16_t *)(data + 2 * src_stride)); + res = _mm_avg_epu8(s[1], s[0]); + *(int16_t *)(dst_ptr + dst_stride) = + (int16_t)_mm_cvtsi128_si32(res); + + data += 2 * src_stride; + dst_ptr += 2 * dst_stride; + y -= 2; + } while (y > 0); + } else if (w == 4) { + s[0] = _mm_cvtsi32_si128(loadu_int32(data)); + + do { + s[1] = _mm_cvtsi32_si128(loadu_int32(data + src_stride)); + res = _mm_avg_epu8(s[0], s[1]); + xx_storel_32(dst_ptr, res); + s[0] = _mm_cvtsi32_si128(loadu_int32(data + 2 * src_stride)); + res = _mm_avg_epu8(s[1], s[0]); + xx_storel_32(dst_ptr + dst_stride, res); + + data += 2 * src_stride; + dst_ptr += 2 * dst_stride; + y -= 2; + } while (y > 0); + } else if (w == 8) { + s[0] = _mm_loadl_epi64((__m128i *)data); + + do { + s[1] = _mm_loadl_epi64((__m128i *)(data + src_stride)); + res = _mm_avg_epu8(s[0], s[1]); + _mm_storel_epi64((__m128i *)dst_ptr, res); + s[0] = _mm_loadl_epi64((__m128i *)(data + 2 * src_stride)); + res = _mm_avg_epu8(s[1], s[0]); + _mm_storel_epi64((__m128i *)(dst_ptr + dst_stride), res); + + data += 2 * src_stride; + dst_ptr += 2 * dst_stride; + y -= 2; + } while (y > 0); + } else { + assert(w == 16); + + s[0] = _mm_loadu_si128((__m128i *)data); + + do { + s[1] = _mm_loadu_si128((__m128i *)(data + src_stride)); + res = _mm_avg_epu8(s[0], s[1]); + _mm_storeu_si128((__m128i *)dst_ptr, res); + s[0] = _mm_loadu_si128((__m128i *)(data + 2 * src_stride)); + res = _mm_avg_epu8(s[1], s[0]); + _mm_storeu_si128((__m128i *)(dst_ptr + dst_stride), res); + + data += 2 * src_stride; + dst_ptr += 2 * dst_stride; + y -= 2; + } while (y > 0); + } + } else { + assert(!(w % 32)); + + __m256i s[2], res; + do { + data = src_ptr + x; + dst_ptr = dst + x; + y = h; + + s[0] = _mm256_loadu_si256((__m256i *)data); + + do { + s[1] = _mm256_loadu_si256((__m256i *)(data + src_stride)); + res = _mm256_avg_epu8(s[0], s[1]); + _mm256_storeu_si256((__m256i *)dst_ptr, res); + s[0] = _mm256_loadu_si256((__m256i *)(data + 2 * src_stride)); + res = _mm256_avg_epu8(s[1], s[0]); + _mm256_storeu_si256((__m256i *)(dst_ptr + dst_stride), res); + + data += 2 * src_stride; + dst_ptr += 2 * dst_stride; + y -= 2; + } while (y > 0); + + x += 32; + } while (x < w); + } + } + } else if (vert_tap == 4) { + if (w <= 4) { + prepare_coeffs_4t_ssse3(filter_params_y, subpel_y_qn, coeffs_128); + __m128i d[4], s[2]; + + if (w == 2) { + d[0] = _mm_cvtsi32_si128(loadu_int16(data + 0 * src_stride)); + d[1] = _mm_cvtsi32_si128(loadu_int16(data + 1 * src_stride)); + d[2] = _mm_cvtsi32_si128(loadu_int16(data + 2 * src_stride)); + + const __m128i src_01a = _mm_unpacklo_epi16(d[0], d[1]); + const __m128i src_12a = _mm_unpacklo_epi16(d[1], d[2]); + + s[0] = _mm_unpacklo_epi8(src_01a, src_12a); + do { + __m128i res; + convolve_y_4tap_2x2_ssse3(data, src_stride, coeffs_128, d, s, &res); + res = round_sr_y_ssse3(res); + pack_store_u8_2x2_sse2(res, dst_ptr, dst_stride); + + dst_ptr += 2 * dst_stride; + data += 2 * src_stride; + y -= 2; + + s[0] = s[1]; + } while (y > 0); + + } else { + assert(w == 4); + + d[0] = _mm_cvtsi32_si128(loadu_int32(data + 0 * src_stride)); + d[1] = _mm_cvtsi32_si128(loadu_int32(data + 1 * src_stride)); + d[2] = _mm_cvtsi32_si128(loadu_int32(data + 2 * src_stride)); + + const __m128i src_01a = _mm_unpacklo_epi32(d[0], d[1]); + const __m128i src_12a = _mm_unpacklo_epi32(d[1], d[2]); + + s[0] = _mm_unpacklo_epi8(src_01a, src_12a); + do { + __m128i res; + convolve_y_4tap_4x2_ssse3(data, src_stride, coeffs_128, d, s, &res); + res = round_sr_y_ssse3(res); + pack_store_u8_4x2_sse2(res, dst_ptr, dst_stride); + + dst_ptr += 2 * dst_stride; + data += 2 * src_stride; + y -= 2; + + s[0] = s[1]; + } while (y > 0); + } + } else { + prepare_coeffs_4t_lowbd(filter_params_y, subpel_y_qn, coeffs); + + if (w == 8) { + __m128i d[4]; + __m256i s[2]; + + d[0] = _mm_loadl_epi64((__m128i *)(data + 0 * src_stride)); + d[1] = _mm_loadl_epi64((__m128i *)(data + 1 * src_stride)); + d[2] = _mm_loadl_epi64((__m128i *)(data + 2 * src_stride)); + + const __m256i src_01a = _mm256_setr_m128i(d[0], d[1]); + const __m256i src_12a = _mm256_setr_m128i(d[1], d[2]); + + s[0] = _mm256_unpacklo_epi8(src_01a, src_12a); + do { + __m256i res; + convolve_y_4tap_8x2_avx2(data, src_stride, coeffs, d, s, &res); + round_pack_store_y_8x2_avx2(res, dst_ptr, dst_stride); + + dst_ptr += 2 * dst_stride; + data += 2 * src_stride; + y -= 2; + + s[0] = s[1]; + } while (y > 0); + } else if (w == 16) { + __m128i d[4]; + __m256i s[4]; + + d[0] = _mm_loadu_si128((__m128i *)(data + 0 * src_stride)); + d[1] = _mm_loadu_si128((__m128i *)(data + 1 * src_stride)); + d[2] = _mm_loadu_si128((__m128i *)(data + 2 * src_stride)); + + const __m256i src_01a = _mm256_setr_m128i(d[0], d[1]); + const __m256i src_12a = _mm256_setr_m128i(d[1], d[2]); + + s[0] = _mm256_unpacklo_epi8(src_01a, src_12a); + s[2] = _mm256_unpackhi_epi8(src_01a, src_12a); + + do { + __m256i res[2]; + convolve_y_4tap_16x2_avx2(data, src_stride, coeffs, d, s, res); + round_pack_store_y_16x2_avx2(res, dst_ptr, dst_stride); + + dst_ptr += 2 * dst_stride; + data += 2 * src_stride; + y -= 2; + + s[0] = s[1]; + s[2] = s[3]; + } while (y > 0); + } else { + assert(!(w % 32)); + + __m256i d[4], s1[4], s2[4]; + do { + data = src_ptr + x; + dst_ptr = dst + x; + y = h; + + d[0] = _mm256_loadu_si256((__m256i *)(data + 0 * src_stride)); + d[1] = _mm256_loadu_si256((__m256i *)(data + 1 * src_stride)); + d[2] = _mm256_loadu_si256((__m256i *)(data + 2 * src_stride)); + + s1[0] = _mm256_unpacklo_epi8(d[0], d[1]); + s1[2] = _mm256_unpackhi_epi8(d[0], d[1]); + + s2[0] = _mm256_unpacklo_epi8(d[1], d[2]); + s2[2] = _mm256_unpackhi_epi8(d[1], d[2]); + + do { + __m256i res[4]; + convolve_y_4tap_32x2_avx2(data, src_stride, coeffs, d, s1, s2, res); + round_pack_store_y_32x2_avx2(res, dst_ptr, dst_stride); + + dst_ptr += 2 * dst_stride; + data += 2 * src_stride; + y -= 2; + + s1[0] = s1[1]; + s1[2] = s1[3]; + + s2[0] = s2[1]; + s2[2] = s2[3]; + } while (y > 0); + + x += 32; + } while (x < w); } } } else if (vert_tap == 6) { - const int fo_vert = vert_tap / 2 - 1; - const uint8_t *const src_ptr = src - fo_vert * src_stride; + if (w <= 4) { + prepare_coeffs_6t_ssse3(filter_params_y, subpel_y_qn, coeffs_128); - for (int j = 0; j < w; j += 16) { - const uint8_t *data = &src_ptr[j]; - __m256i src6; + __m128i d[6], s[3]; + if (w == 2) { + d[0] = _mm_cvtsi32_si128(loadu_int16(data + 0 * src_stride)); + d[1] = _mm_cvtsi32_si128(loadu_int16(data + 1 * src_stride)); + d[2] = _mm_cvtsi32_si128(loadu_int16(data + 2 * src_stride)); + d[3] = _mm_cvtsi32_si128(loadu_int16(data + 3 * src_stride)); + d[4] = _mm_cvtsi32_si128(loadu_int16(data + 4 * src_stride)); - d[0] = _mm_loadu_si128((__m128i *)(data + 0 * src_stride)); - d[1] = _mm_loadu_si128((__m128i *)(data + 1 * src_stride)); - d[2] = _mm_loadu_si128((__m128i *)(data + 2 * src_stride)); - d[3] = _mm_loadu_si128((__m128i *)(data + 3 * src_stride)); - // Load lines a and b. Line a to lower 128, line b to upper 128 - const __m256i src_01a = _mm256_permute2x128_si256( - _mm256_castsi128_si256(d[0]), _mm256_castsi128_si256(d[1]), 0x20); + const __m128i src_01a = _mm_unpacklo_epi16(d[0], d[1]); + const __m128i src_12a = _mm_unpacklo_epi16(d[1], d[2]); + const __m128i src_23a = _mm_unpacklo_epi16(d[2], d[3]); + const __m128i src_34a = _mm_unpacklo_epi16(d[3], d[4]); - const __m256i src_12a = _mm256_permute2x128_si256( - _mm256_castsi128_si256(d[1]), _mm256_castsi128_si256(d[2]), 0x20); + s[0] = _mm_unpacklo_epi8(src_01a, src_12a); + s[1] = _mm_unpacklo_epi8(src_23a, src_34a); - const __m256i src_23a = _mm256_permute2x128_si256( - _mm256_castsi128_si256(d[2]), _mm256_castsi128_si256(d[3]), 0x20); + do { + __m128i res; + convolve_y_6tap_2x2_ssse3(data, src_stride, coeffs_128, d, s, &res); + res = round_sr_y_ssse3(res); + pack_store_u8_2x2_sse2(res, dst_ptr, dst_stride); - src6 = _mm256_castsi128_si256( - _mm_loadu_si128((__m128i *)(data + 4 * src_stride))); - const __m256i src_34a = - _mm256_permute2x128_si256(_mm256_castsi128_si256(d[3]), src6, 0x20); + dst_ptr += 2 * dst_stride; + data += 2 * src_stride; + y -= 2; - s[0] = _mm256_unpacklo_epi8(src_01a, src_12a); - s[1] = _mm256_unpacklo_epi8(src_23a, src_34a); + s[0] = s[1]; + s[1] = s[2]; + } while (y > 0); - s[3] = _mm256_unpackhi_epi8(src_01a, src_12a); - s[4] = _mm256_unpackhi_epi8(src_23a, src_34a); + } else { + assert(w == 4); + d[0] = _mm_cvtsi32_si128(loadu_int32(data + 0 * src_stride)); + d[1] = _mm_cvtsi32_si128(loadu_int32(data + 1 * src_stride)); + d[2] = _mm_cvtsi32_si128(loadu_int32(data + 2 * src_stride)); + d[3] = _mm_cvtsi32_si128(loadu_int32(data + 3 * src_stride)); + d[4] = _mm_cvtsi32_si128(loadu_int32(data + 4 * src_stride)); - for (i = 0; i < h; i += 2) { - data = &src_ptr[i * src_stride + j]; - const __m256i src_45a = _mm256_permute2x128_si256( - src6, - _mm256_castsi128_si256( - _mm_loadu_si128((__m128i *)(data + 5 * src_stride))), - 0x20); + const __m128i src_01a = _mm_unpacklo_epi32(d[0], d[1]); + const __m128i src_12a = _mm_unpacklo_epi32(d[1], d[2]); + const __m128i src_23a = _mm_unpacklo_epi32(d[2], d[3]); + const __m128i src_34a = _mm_unpacklo_epi32(d[3], d[4]); - src6 = _mm256_castsi128_si256( - _mm_loadu_si128((__m128i *)(data + 6 * src_stride))); - const __m256i src_56a = _mm256_permute2x128_si256( - _mm256_castsi128_si256( - _mm_loadu_si128((__m128i *)(data + 5 * src_stride))), - src6, 0x20); + s[0] = _mm_unpacklo_epi8(src_01a, src_12a); + s[1] = _mm_unpacklo_epi8(src_23a, src_34a); - s[2] = _mm256_unpacklo_epi8(src_45a, src_56a); - s[5] = _mm256_unpackhi_epi8(src_45a, src_56a); + do { + __m128i res; + convolve_y_6tap_4x2_ssse3(data, src_stride, coeffs_128, d, s, &res); + res = round_sr_y_ssse3(res); + pack_store_u8_4x2_sse2(res, dst_ptr, dst_stride); - const __m256i res_lo = convolve_lowbd_6tap(s, coeffs); + dst_ptr += 2 * dst_stride; + data += 2 * src_stride; + y -= 2; - /* rounding code */ - // shift by F - 1 - const __m256i res_16b_lo = _mm256_sra_epi16( - _mm256_add_epi16(res_lo, right_shift_const), right_shift); - // 8 bit conversion and saturation to uint8 - __m256i res_8b_lo = _mm256_packus_epi16(res_16b_lo, res_16b_lo); + s[0] = s[1]; + s[1] = s[2]; + } while (y > 0); + } + } else { + prepare_coeffs_6t_lowbd(filter_params_y, subpel_y_qn, coeffs); - if (w - j > 8) { - const __m256i res_hi = convolve_lowbd_6tap(s + 3, coeffs); + if (w == 8) { + __m128i d[6]; + __m256i s[3]; - /* rounding code */ - // shift by F - 1 - const __m256i res_16b_hi = _mm256_sra_epi16( - _mm256_add_epi16(res_hi, right_shift_const), right_shift); - // 8 bit conversion and saturation to uint8 - __m256i res_8b_hi = _mm256_packus_epi16(res_16b_hi, res_16b_hi); + d[0] = _mm_loadl_epi64((__m128i *)(data + 0 * src_stride)); + d[1] = _mm_loadl_epi64((__m128i *)(data + 1 * src_stride)); + d[2] = _mm_loadl_epi64((__m128i *)(data + 2 * src_stride)); + d[3] = _mm_loadl_epi64((__m128i *)(data + 3 * src_stride)); + d[4] = _mm_loadl_epi64((__m128i *)(data + 4 * src_stride)); - __m256i res_a = _mm256_unpacklo_epi64(res_8b_lo, res_8b_hi); + const __m256i src_01a = _mm256_setr_m128i(d[0], d[1]); + const __m256i src_12a = _mm256_setr_m128i(d[1], d[2]); + const __m256i src_23a = _mm256_setr_m128i(d[2], d[3]); + const __m256i src_34a = _mm256_setr_m128i(d[3], d[4]); - const __m128i res_0 = _mm256_castsi256_si128(res_a); - const __m128i res_1 = _mm256_extracti128_si256(res_a, 1); + s[0] = _mm256_unpacklo_epi8(src_01a, src_12a); + s[1] = _mm256_unpacklo_epi8(src_23a, src_34a); - _mm_storeu_si128((__m128i *)&dst[i * dst_stride + j], res_0); - _mm_storeu_si128((__m128i *)&dst[i * dst_stride + j + dst_stride], - res_1); - } else { - const __m128i res_0 = _mm256_castsi256_si128(res_8b_lo); - const __m128i res_1 = _mm256_extracti128_si256(res_8b_lo, 1); - if (w - j > 4) { - _mm_storel_epi64((__m128i *)&dst[i * dst_stride + j], res_0); - _mm_storel_epi64((__m128i *)&dst[i * dst_stride + j + dst_stride], - res_1); - } else if (w - j > 2) { - xx_storel_32(&dst[i * dst_stride + j], res_0); - xx_storel_32(&dst[i * dst_stride + j + dst_stride], res_1); - } else { - __m128i *const p_0 = (__m128i *)&dst[i * dst_stride + j]; - __m128i *const p_1 = - (__m128i *)&dst[i * dst_stride + j + dst_stride]; - *(uint16_t *)p_0 = (uint16_t)_mm_cvtsi128_si32(res_0); - *(uint16_t *)p_1 = (uint16_t)_mm_cvtsi128_si32(res_1); - } - } - s[0] = s[1]; - s[1] = s[2]; - s[3] = s[4]; - s[4] = s[5]; + do { + __m256i res; + convolve_y_6tap_8x2_avx2(data, src_stride, coeffs, d, s, &res); + round_pack_store_y_8x2_avx2(res, dst_ptr, dst_stride); + + dst_ptr += 2 * dst_stride; + data += 2 * src_stride; + y -= 2; + + s[0] = s[1]; + s[1] = s[2]; + } while (y > 0); + + } else { + assert(!(w % 16)); + + __m128i d[6]; + __m256i s[6]; + do { + data = src_ptr + x; + dst_ptr = dst + x; + y = h; + + d[0] = _mm_loadu_si128((__m128i *)(data + 0 * src_stride)); + d[1] = _mm_loadu_si128((__m128i *)(data + 1 * src_stride)); + d[2] = _mm_loadu_si128((__m128i *)(data + 2 * src_stride)); + d[3] = _mm_loadu_si128((__m128i *)(data + 3 * src_stride)); + d[4] = _mm_loadu_si128((__m128i *)(data + 4 * src_stride)); + + const __m256i src_01a = _mm256_setr_m128i(d[0], d[1]); + const __m256i src_12a = _mm256_setr_m128i(d[1], d[2]); + const __m256i src_23a = _mm256_setr_m128i(d[2], d[3]); + const __m256i src_34a = _mm256_setr_m128i(d[3], d[4]); + + s[0] = _mm256_unpacklo_epi8(src_01a, src_12a); + s[1] = _mm256_unpacklo_epi8(src_23a, src_34a); + + s[3] = _mm256_unpackhi_epi8(src_01a, src_12a); + s[4] = _mm256_unpackhi_epi8(src_23a, src_34a); + + do { + __m256i res[2]; + convolve_y_6tap_16x2_avx2(data, src_stride, coeffs, d, s, res); + round_pack_store_y_16x2_avx2(res, dst_ptr, dst_stride); + + dst_ptr += 2 * dst_stride; + data += 2 * src_stride; + y -= 2; + + s[0] = s[1]; + s[1] = s[2]; + + s[3] = s[4]; + s[4] = s[5]; + } while (y > 0); + + x += 16; + } while (x < w); } } } else if (vert_tap == 12) { // vert_tap == 12 - const int fo_vert = filter_params_y->taps / 2 - 1; - const uint8_t *const src_ptr = src - fo_vert * src_stride; + __m128i d[12]; + __m256i s[12]; + prepare_coeffs_12taps(filter_params_y, subpel_y_qn, coeffs); const __m256i v_zero = _mm256_setzero_si256(); - right_shift = _mm_cvtsi32_si128(FILTER_BITS); - right_shift_const = _mm256_set1_epi32((1 << FILTER_BITS) >> 1); + __m128i right_shift = _mm_cvtsi32_si128(FILTER_BITS); + __m256i right_shift_const = _mm256_set1_epi32((1 << FILTER_BITS) >> 1); for (int j = 0; j < w; j += 8) { - const uint8_t *data = &src_ptr[j]; + data = &src_ptr[j]; __m256i src10; d[0] = _mm_loadl_epi64((__m128i *)(data + 0 * src_stride)); @@ -393,193 +662,242 @@ } } } else { - const int fo_vert = filter_params_y->taps / 2 - 1; - const uint8_t *const src_ptr = src - fo_vert * src_stride; + assert(vert_tap == 8); - for (int j = 0; j < w; j += 16) { - const uint8_t *data = &src_ptr[j]; - __m256i src6; + if (w <= 4) { + prepare_coeffs_ssse3(filter_params_y, subpel_y_qn, coeffs_128); - d[0] = _mm_loadu_si128((__m128i *)(data + 0 * src_stride)); - d[1] = _mm_loadu_si128((__m128i *)(data + 1 * src_stride)); - d[2] = _mm_loadu_si128((__m128i *)(data + 2 * src_stride)); - d[3] = _mm_loadu_si128((__m128i *)(data + 3 * src_stride)); - d[4] = _mm_loadu_si128((__m128i *)(data + 4 * src_stride)); - d[5] = _mm_loadu_si128((__m128i *)(data + 5 * src_stride)); - // Load lines a and b. Line a to lower 128, line b to upper 128 - const __m256i src_01a = _mm256_permute2x128_si256( - _mm256_castsi128_si256(d[0]), _mm256_castsi128_si256(d[1]), 0x20); + __m128i d[8], s[4], res; + if (w == 2) { + d[0] = _mm_cvtsi32_si128(loadu_int16(data + 0 * src_stride)); + d[1] = _mm_cvtsi32_si128(loadu_int16(data + 1 * src_stride)); + d[2] = _mm_cvtsi32_si128(loadu_int16(data + 2 * src_stride)); + d[3] = _mm_cvtsi32_si128(loadu_int16(data + 3 * src_stride)); + d[4] = _mm_cvtsi32_si128(loadu_int16(data + 4 * src_stride)); + d[5] = _mm_cvtsi32_si128(loadu_int16(data + 5 * src_stride)); + d[6] = _mm_cvtsi32_si128(loadu_int16(data + 6 * src_stride)); - const __m256i src_12a = _mm256_permute2x128_si256( - _mm256_castsi128_si256(d[1]), _mm256_castsi128_si256(d[2]), 0x20); + const __m128i src_01a = _mm_unpacklo_epi16(d[0], d[1]); + const __m128i src_12a = _mm_unpacklo_epi16(d[1], d[2]); + const __m128i src_23a = _mm_unpacklo_epi16(d[2], d[3]); + const __m128i src_34a = _mm_unpacklo_epi16(d[3], d[4]); + const __m128i src_45a = _mm_unpacklo_epi16(d[4], d[5]); + const __m128i src_56a = _mm_unpacklo_epi16(d[5], d[6]); - const __m256i src_23a = _mm256_permute2x128_si256( - _mm256_castsi128_si256(d[2]), _mm256_castsi128_si256(d[3]), 0x20); + s[0] = _mm_unpacklo_epi8(src_01a, src_12a); + s[1] = _mm_unpacklo_epi8(src_23a, src_34a); + s[2] = _mm_unpacklo_epi8(src_45a, src_56a); - const __m256i src_34a = _mm256_permute2x128_si256( - _mm256_castsi128_si256(d[3]), _mm256_castsi128_si256(d[4]), 0x20); + do { + convolve_y_8tap_2x2_ssse3(data, src_stride, coeffs_128, d, s, &res); + res = round_sr_y_ssse3(res); + pack_store_u8_2x2_sse2(res, dst_ptr, dst_stride); - const __m256i src_45a = _mm256_permute2x128_si256( - _mm256_castsi128_si256(d[4]), _mm256_castsi128_si256(d[5]), 0x20); + dst_ptr += 2 * dst_stride; + data += 2 * src_stride; + y -= 2; - src6 = _mm256_castsi128_si256( - _mm_loadu_si128((__m128i *)(data + 6 * src_stride))); - const __m256i src_56a = - _mm256_permute2x128_si256(_mm256_castsi128_si256(d[5]), src6, 0x20); + s[0] = s[1]; + s[1] = s[2]; + s[2] = s[3]; + } while (y > 0); - s[0] = _mm256_unpacklo_epi8(src_01a, src_12a); - s[1] = _mm256_unpacklo_epi8(src_23a, src_34a); - s[2] = _mm256_unpacklo_epi8(src_45a, src_56a); + } else { + assert(w == 4); - s[4] = _mm256_unpackhi_epi8(src_01a, src_12a); - s[5] = _mm256_unpackhi_epi8(src_23a, src_34a); - s[6] = _mm256_unpackhi_epi8(src_45a, src_56a); + d[0] = _mm_cvtsi32_si128(loadu_int32(data + 0 * src_stride)); + d[1] = _mm_cvtsi32_si128(loadu_int32(data + 1 * src_stride)); + d[2] = _mm_cvtsi32_si128(loadu_int32(data + 2 * src_stride)); + d[3] = _mm_cvtsi32_si128(loadu_int32(data + 3 * src_stride)); + d[4] = _mm_cvtsi32_si128(loadu_int32(data + 4 * src_stride)); + d[5] = _mm_cvtsi32_si128(loadu_int32(data + 5 * src_stride)); + d[6] = _mm_cvtsi32_si128(loadu_int32(data + 6 * src_stride)); - for (i = 0; i < h; i += 2) { - data = &src_ptr[i * src_stride + j]; - const __m256i src_67a = _mm256_permute2x128_si256( - src6, - _mm256_castsi128_si256( - _mm_loadu_si128((__m128i *)(data + 7 * src_stride))), - 0x20); + const __m128i src_01a = _mm_unpacklo_epi32(d[0], d[1]); + const __m128i src_12a = _mm_unpacklo_epi32(d[1], d[2]); + const __m128i src_23a = _mm_unpacklo_epi32(d[2], d[3]); + const __m128i src_34a = _mm_unpacklo_epi32(d[3], d[4]); + const __m128i src_45a = _mm_unpacklo_epi32(d[4], d[5]); + const __m128i src_56a = _mm_unpacklo_epi32(d[5], d[6]); - src6 = _mm256_castsi128_si256( - _mm_loadu_si128((__m128i *)(data + 8 * src_stride))); - const __m256i src_78a = _mm256_permute2x128_si256( - _mm256_castsi128_si256( - _mm_loadu_si128((__m128i *)(data + 7 * src_stride))), - src6, 0x20); + s[0] = _mm_unpacklo_epi8(src_01a, src_12a); + s[1] = _mm_unpacklo_epi8(src_23a, src_34a); + s[2] = _mm_unpacklo_epi8(src_45a, src_56a); - s[3] = _mm256_unpacklo_epi8(src_67a, src_78a); - s[7] = _mm256_unpackhi_epi8(src_67a, src_78a); + do { + convolve_y_8tap_4x2_ssse3(data, src_stride, coeffs_128, d, s, &res); + res = round_sr_y_ssse3(res); + pack_store_u8_4x2_sse2(res, dst_ptr, dst_stride); - const __m256i res_lo = convolve_lowbd(s, coeffs); + dst_ptr += 2 * dst_stride; + data += 2 * src_stride; + y -= 2; - /* rounding code */ - // shift by F - 1 - const __m256i res_16b_lo = _mm256_sra_epi16( - _mm256_add_epi16(res_lo, right_shift_const), right_shift); - // 8 bit conversion and saturation to uint8 - __m256i res_8b_lo = _mm256_packus_epi16(res_16b_lo, res_16b_lo); + s[0] = s[1]; + s[1] = s[2]; + s[2] = s[3]; + } while (y > 0); + } + } else { + prepare_coeffs_lowbd(filter_params_y, subpel_y_qn, coeffs); - if (w - j > 8) { - const __m256i res_hi = convolve_lowbd(s + 4, coeffs); + if (w == 8) { + __m128i d[8]; + __m256i s[4]; - /* rounding code */ - // shift by F - 1 - const __m256i res_16b_hi = _mm256_sra_epi16( - _mm256_add_epi16(res_hi, right_shift_const), right_shift); - // 8 bit conversion and saturation to uint8 - __m256i res_8b_hi = _mm256_packus_epi16(res_16b_hi, res_16b_hi); + d[0] = _mm_loadl_epi64((__m128i *)(data + 0 * src_stride)); + d[1] = _mm_loadl_epi64((__m128i *)(data + 1 * src_stride)); + d[2] = _mm_loadl_epi64((__m128i *)(data + 2 * src_stride)); + d[3] = _mm_loadl_epi64((__m128i *)(data + 3 * src_stride)); + d[4] = _mm_loadl_epi64((__m128i *)(data + 4 * src_stride)); + d[5] = _mm_loadl_epi64((__m128i *)(data + 5 * src_stride)); + d[6] = _mm_loadl_epi64((__m128i *)(data + 6 * src_stride)); - __m256i res_a = _mm256_unpacklo_epi64(res_8b_lo, res_8b_hi); + const __m256i src_01a = _mm256_setr_m128i(d[0], d[1]); + const __m256i src_12a = _mm256_setr_m128i(d[1], d[2]); + const __m256i src_23a = _mm256_setr_m128i(d[2], d[3]); + const __m256i src_34a = _mm256_setr_m128i(d[3], d[4]); + const __m256i src_45a = _mm256_setr_m128i(d[4], d[5]); + const __m256i src_56a = _mm256_setr_m128i(d[5], d[6]); - const __m128i res_0 = _mm256_castsi256_si128(res_a); - const __m128i res_1 = _mm256_extracti128_si256(res_a, 1); + s[0] = _mm256_unpacklo_epi8(src_01a, src_12a); + s[1] = _mm256_unpacklo_epi8(src_23a, src_34a); + s[2] = _mm256_unpacklo_epi8(src_45a, src_56a); - _mm_storeu_si128((__m128i *)&dst[i * dst_stride + j], res_0); - _mm_storeu_si128((__m128i *)&dst[i * dst_stride + j + dst_stride], - res_1); - } else { - const __m128i res_0 = _mm256_castsi256_si128(res_8b_lo); - const __m128i res_1 = _mm256_extracti128_si256(res_8b_lo, 1); - if (w - j > 4) { - _mm_storel_epi64((__m128i *)&dst[i * dst_stride + j], res_0); - _mm_storel_epi64((__m128i *)&dst[i * dst_stride + j + dst_stride], - res_1); - } else if (w - j > 2) { - xx_storel_32(&dst[i * dst_stride + j], res_0); - xx_storel_32(&dst[i * dst_stride + j + dst_stride], res_1); - } else { - __m128i *const p_0 = (__m128i *)&dst[i * dst_stride + j]; - __m128i *const p_1 = - (__m128i *)&dst[i * dst_stride + j + dst_stride]; - *(uint16_t *)p_0 = (uint16_t)_mm_cvtsi128_si32(res_0); - *(uint16_t *)p_1 = (uint16_t)_mm_cvtsi128_si32(res_1); - } - } - s[0] = s[1]; - s[1] = s[2]; - s[2] = s[3]; + do { + __m256i res; + convolve_y_8tap_8x2_avx2(data, src_stride, coeffs, d, s, &res); + round_pack_store_y_8x2_avx2(res, dst_ptr, dst_stride); - s[4] = s[5]; - s[5] = s[6]; - s[6] = s[7]; + dst_ptr += 2 * dst_stride; + data += 2 * src_stride; + y -= 2; + + s[0] = s[1]; + s[1] = s[2]; + s[2] = s[3]; + } while (y > 0); + + } else { + assert(!(w % 16)); + + __m128i d[8]; + __m256i s[8]; + do { + data = src_ptr + x; + dst_ptr = dst + x; + y = h; + + d[0] = _mm_loadu_si128((__m128i *)(data + 0 * src_stride)); + d[1] = _mm_loadu_si128((__m128i *)(data + 1 * src_stride)); + d[2] = _mm_loadu_si128((__m128i *)(data + 2 * src_stride)); + d[3] = _mm_loadu_si128((__m128i *)(data + 3 * src_stride)); + d[4] = _mm_loadu_si128((__m128i *)(data + 4 * src_stride)); + d[5] = _mm_loadu_si128((__m128i *)(data + 5 * src_stride)); + d[6] = _mm_loadu_si128((__m128i *)(data + 6 * src_stride)); + + const __m256i src_01a = _mm256_setr_m128i(d[0], d[1]); + const __m256i src_12a = _mm256_setr_m128i(d[1], d[2]); + const __m256i src_23a = _mm256_setr_m128i(d[2], d[3]); + const __m256i src_34a = _mm256_setr_m128i(d[3], d[4]); + const __m256i src_45a = _mm256_setr_m128i(d[4], d[5]); + const __m256i src_56a = _mm256_setr_m128i(d[5], d[6]); + + s[0] = _mm256_unpacklo_epi8(src_01a, src_12a); + s[1] = _mm256_unpacklo_epi8(src_23a, src_34a); + s[2] = _mm256_unpacklo_epi8(src_45a, src_56a); + + s[4] = _mm256_unpackhi_epi8(src_01a, src_12a); + s[5] = _mm256_unpackhi_epi8(src_23a, src_34a); + s[6] = _mm256_unpackhi_epi8(src_45a, src_56a); + + do { + __m256i res[2]; + convolve_y_8tap_16x2_avx2(data, src_stride, coeffs, d, s, res); + round_pack_store_y_16x2_avx2(res, dst_ptr, dst_stride); + + dst_ptr += 2 * dst_stride; + data += 2 * src_stride; + y -= 2; + + s[0] = s[1]; + s[1] = s[2]; + s[2] = s[3]; + + s[4] = s[5]; + s[5] = s[6]; + s[6] = s[7]; + } while (y > 0); + + x += 16; + } while (x < w); } } } } -void av1_convolve_y_sr_avx2(const uint8_t *src, int32_t src_stride, +void av1_convolve_x_sr_avx2(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, int32_t w, int32_t h, - const InterpFilterParams *filter_params_y, - const int32_t subpel_y_qn) { -#if CONFIG_SVT_AV1 - const int vert_tap = get_filter_tap(filter_params_y, subpel_y_qn); - - if (vert_tap == 12) { - av1_convolve_y_sr_general_avx2(src, src_stride, dst, dst_stride, w, h, - filter_params_y, subpel_y_qn); - } else { - av1_convolve_y_sr_specialized_avx2(src, src_stride, dst, dst_stride, w, h, - filter_params_y, subpel_y_qn); - } -#else - av1_convolve_y_sr_general_avx2(src, src_stride, dst, dst_stride, w, h, - filter_params_y, subpel_y_qn); -#endif -} - -static inline void av1_convolve_x_sr_general_avx2( - const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, - int h, const InterpFilterParams *filter_params_x, const int subpel_x_qn, - ConvolveParams *conv_params) { + const InterpFilterParams *filter_params_x, + const int32_t subpel_x_qn, + ConvolveParams *conv_params) { const int bits = FILTER_BITS - conv_params->round_0; - const __m128i round_shift = _mm_cvtsi32_si128(bits); - __m256i round_0_const = - _mm256_set1_epi16((1 << (conv_params->round_0 - 1)) >> 1); - __m128i round_0_shift = _mm_cvtsi32_si128(conv_params->round_0 - 1); - __m256i round_const = _mm256_set1_epi16((1 << bits) >> 1); - int i, horiz_tap = get_filter_tap(filter_params_x, subpel_x_qn); + int i, j, horiz_tap = get_filter_tap(filter_params_x, subpel_x_qn); assert(bits >= 0); assert((FILTER_BITS - conv_params->round_1) >= 0 || ((conv_params->round_0 + conv_params->round_1) == 2 * FILTER_BITS)); assert(conv_params->round_0 > 0); - __m256i coeffs[6], filt[4]; - filt[0] = _mm256_load_si256((__m256i const *)(filt_global_avx2)); - filt[1] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32)); + assert(horiz_tap == 2 || horiz_tap == 4 || horiz_tap == 6 || horiz_tap == 8 || + horiz_tap == 12); + assert((!(w % 2)) || (w <= 128)); + assert((h % 2) == 0); - if (horiz_tap == 6) - prepare_coeffs_6t_lowbd(filter_params_x, subpel_x_qn, coeffs); - else if (horiz_tap == 12) { - prepare_coeffs_12taps(filter_params_x, subpel_x_qn, coeffs); - } else { - prepare_coeffs_lowbd(filter_params_x, subpel_x_qn, coeffs); - } + __m256i coeffs[6] = { 0 }, filt[4] = { 0 }; + __m128i coeffs_128[4] = { 0 }; + i = 0; // horz_filt as 4 tap if (horiz_tap == 4) { - const int fo_horiz = 1; - const uint8_t *const src_ptr = src - fo_horiz; - if (w <= 8) { - for (i = 0; i < h; i += 2) { - const __m256i data = _mm256_permute2x128_si256( - _mm256_castsi128_si256( - _mm_loadu_si128((__m128i *)(&src_ptr[i * src_stride]))), - _mm256_castsi128_si256(_mm_loadu_si128( - (__m128i *)(&src_ptr[i * src_stride + src_stride]))), - 0x20); + // since fo_horiz = 1 + const uint8_t *src_ptr = src - 1; + if (w == 2) { + prepare_coeffs_4t_ssse3(filter_params_x, subpel_x_qn, coeffs_128); + do { + const __m128i res = + convolve_x_4tap_2x2_ssse3(src_ptr, src_stride, coeffs_128); + const __m128i reg = round_sr_x_ssse3(res); + pack_store_u8_2x2_sse2(reg, dst, dst_stride); + src_ptr += 2 * src_stride; + dst += 2 * dst_stride; + h -= 2; + } while (h); + } else if (w == 4) { + prepare_coeffs_4t_ssse3(filter_params_x, subpel_x_qn, coeffs_128); + do { + const __m128i reg = + convolve_x_4tap_4x2_ssse3(src_ptr, src_stride, coeffs_128); + const __m128i res = round_sr_x_ssse3(reg); + pack_store_u8_4x2_sse2(res, dst, dst_stride); + src_ptr += 2 * src_stride; + dst += 2 * dst_stride; + h -= 2; + } while (h); + } else if (w == 8) { + prepare_coeffs_lowbd(filter_params_x, subpel_x_qn, coeffs); + filt[0] = _mm256_load_si256((__m256i const *)(filt_global_avx2)); + filt[1] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32)); + do { + const __m256i data = _mm256_setr_m128i( + _mm_loadu_si128((__m128i *)(&src_ptr[i * src_stride])), + _mm_loadu_si128( + (__m128i *)(&src_ptr[i * src_stride + src_stride]))); __m256i res_16b = convolve_lowbd_x_4tap(data, coeffs + 1, filt); - res_16b = _mm256_sra_epi16(_mm256_add_epi16(res_16b, round_0_const), - round_0_shift); - - res_16b = _mm256_sra_epi16(_mm256_add_epi16(res_16b, round_const), - round_shift); + res_16b = round_sr_x_avx2(res_16b); /* rounding code */ // 8 bit conversion and saturation to uint8 @@ -588,22 +906,18 @@ const __m128i res_0 = _mm256_castsi256_si128(res_8b); const __m128i res_1 = _mm256_extracti128_si256(res_8b, 1); - if (w > 4) { - _mm_storel_epi64((__m128i *)&dst[i * dst_stride], res_0); - _mm_storel_epi64((__m128i *)&dst[i * dst_stride + dst_stride], res_1); - } else if (w > 2) { - xx_storel_32(&dst[i * dst_stride], res_0); - xx_storel_32(&dst[i * dst_stride + dst_stride], res_1); - } else { - __m128i *const p_0 = (__m128i *)&dst[i * dst_stride]; - __m128i *const p_1 = (__m128i *)&dst[i * dst_stride + dst_stride]; - *(uint16_t *)p_0 = (uint16_t)_mm_cvtsi128_si32(res_0); - *(uint16_t *)p_1 = (uint16_t)_mm_cvtsi128_si32(res_1); - } - } + _mm_storel_epi64((__m128i *)&dst[i * dst_stride], res_0); + _mm_storel_epi64((__m128i *)&dst[i * dst_stride + dst_stride], res_1); + i += 2; + } while (i < h); } else { - for (i = 0; i < h; ++i) { - for (int j = 0; j < w; j += 16) { + assert(!(w % 16)); + prepare_coeffs_lowbd(filter_params_x, subpel_x_qn, coeffs); + filt[0] = _mm256_load_si256((__m256i const *)(filt_global_avx2)); + filt[1] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32)); + do { + j = 0; + do { // 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15 16 17 // 18 19 20 21 22 23 const __m256i data = _mm256_inserti128_si256( @@ -613,11 +927,7 @@ __m256i res_16b = convolve_lowbd_x_4tap(data, coeffs + 1, filt); - res_16b = _mm256_sra_epi16(_mm256_add_epi16(res_16b, round_0_const), - round_0_shift); - - res_16b = _mm256_sra_epi16(_mm256_add_epi16(res_16b, round_const), - round_shift); + res_16b = round_sr_x_avx2(res_16b); /* rounding code */ // 8 bit conversion and saturation to uint8 @@ -628,31 +938,28 @@ res_8b = _mm256_permute4x64_epi64(res_8b, 216); __m128i res = _mm256_castsi256_si128(res_8b); _mm_storeu_si128((__m128i *)&dst[i * dst_stride + j], res); - } - } + j += 16; + } while (j < w); + i++; + } while (i < h); } } else if (horiz_tap == 6) { - const int fo_horiz = horiz_tap / 2 - 1; - const uint8_t *const src_ptr = src - fo_horiz; + // since (horiz_tap/2 - 1 == 2) + const uint8_t *src_ptr = src - 2; + prepare_coeffs_6t_lowbd(filter_params_x, subpel_x_qn, coeffs); + filt[0] = _mm256_load_si256((__m256i const *)(filt_global_avx2)); + filt[1] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32)); filt[2] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32 * 2)); - filt[3] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32 * 3)); - - if (w <= 8) { - for (i = 0; i < h; i += 2) { - const __m256i data = _mm256_permute2x128_si256( - _mm256_castsi128_si256( - _mm_loadu_si128((__m128i *)(&src_ptr[i * src_stride]))), - _mm256_castsi128_si256(_mm_loadu_si128( - (__m128i *)(&src_ptr[i * src_stride + src_stride]))), - 0x20); + if (w == 8) { + do { + const __m256i data = _mm256_setr_m128i( + _mm_loadu_si128((__m128i *)(&src_ptr[i * src_stride])), + _mm_loadu_si128( + (__m128i *)(&src_ptr[i * src_stride + src_stride]))); __m256i res_16b = convolve_lowbd_x_6tap(data, coeffs, filt); - res_16b = _mm256_sra_epi16(_mm256_add_epi16(res_16b, round_0_const), - round_0_shift); - - res_16b = _mm256_sra_epi16(_mm256_add_epi16(res_16b, round_const), - round_shift); + res_16b = round_sr_x_avx2(res_16b); /* rounding code */ // 8 bit conversion and saturation to uint8 @@ -660,60 +967,132 @@ const __m128i res_0 = _mm256_castsi256_si128(res_8b); const __m128i res_1 = _mm256_extracti128_si256(res_8b, 1); - if (w > 4) { - _mm_storel_epi64((__m128i *)&dst[i * dst_stride], res_0); - _mm_storel_epi64((__m128i *)&dst[i * dst_stride + dst_stride], res_1); - } else if (w > 2) { - xx_storel_32(&dst[i * dst_stride], res_0); - xx_storel_32(&dst[i * dst_stride + dst_stride], res_1); - } else { - __m128i *const p_0 = (__m128i *)&dst[i * dst_stride]; - __m128i *const p_1 = (__m128i *)&dst[i * dst_stride + dst_stride]; - *(uint16_t *)p_0 = _mm_cvtsi128_si32(res_0); - *(uint16_t *)p_1 = _mm_cvtsi128_si32(res_1); - } - } + _mm_storel_epi64((__m128i *)&dst[i * dst_stride], res_0); + _mm_storel_epi64((__m128i *)&dst[i * dst_stride + dst_stride], res_1); + i += 2; + } while (i < h); + } else if (w == 16) { + do { + __m256i data[2] = { 0 }; + + load_convolve_6tap_16x2_avx2(src_ptr, src_stride, coeffs, filt, data); + round_pack_store_16x2_avx2(data, dst, dst_stride); + src_ptr += 2 * src_stride; + dst += 2 * dst_stride; + h -= 2; + } while (h); + } else if (w == 32) { + do { + convolve_sr_store_6tap_32_avx2(src_ptr, coeffs, filt, dst); + src_ptr += src_stride; + dst += dst_stride; + } while ((--h) > 0); + } else if (w == 64) { + do { + convolve_sr_store_6tap_32_avx2(src_ptr, coeffs, filt, dst); + convolve_sr_store_6tap_32_avx2(src_ptr + 32, coeffs, filt, dst + 32); + src_ptr += src_stride; + dst += dst_stride; + } while ((--h) > 0); } else { - for (i = 0; i < h; ++i) { - for (int j = 0; j < w; j += 16) { - // 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15 16 17 - // 18 19 20 21 22 23 - const __m256i data = _mm256_inserti128_si256( - _mm256_loadu_si256((__m256i *)&src_ptr[(i * src_stride) + j]), - _mm_loadu_si128((__m128i *)&src_ptr[(i * src_stride) + (j + 8)]), - 1); + assert(w == 128); - __m256i res_16b = convolve_lowbd_x_6tap(data, coeffs, filt); + do { + convolve_sr_store_6tap_32_avx2(src_ptr, coeffs, filt, dst); + convolve_sr_store_6tap_32_avx2(src_ptr + SECOND_32_BLK, coeffs, filt, + dst + SECOND_32_BLK); + convolve_sr_store_6tap_32_avx2(src_ptr + THIRD_32_BLK, coeffs, filt, + dst + THIRD_32_BLK); + convolve_sr_store_6tap_32_avx2(src_ptr + FOURTH_32_BLK, coeffs, filt, + dst + FOURTH_32_BLK); + src_ptr += src_stride; + dst += dst_stride; + } while ((--h) > 0); + } + } else if (horiz_tap == 8) { + // since (horiz_tap / 2 - 1) == 3 + const uint8_t *src_ptr = src - 3; + prepare_coeffs_lowbd(filter_params_x, subpel_x_qn, coeffs); + filt[0] = _mm256_load_si256((__m256i const *)(filt_global_avx2)); + filt[1] = + _mm256_load_si256((__m256i const *)(filt_global_avx2 + SECOND_32_BLK)); + filt[2] = + _mm256_load_si256((__m256i const *)(filt_global_avx2 + THIRD_32_BLK)); + filt[3] = + _mm256_load_si256((__m256i const *)(filt_global_avx2 + FOURTH_32_BLK)); - res_16b = _mm256_sra_epi16(_mm256_add_epi16(res_16b, round_0_const), - round_0_shift); + if (w == 8) { + do { + const __m256i data = _mm256_setr_m128i( + _mm_loadu_si128((__m128i *)(&src_ptr[i * src_stride])), + _mm_loadu_si128( + (__m128i *)(&src_ptr[i * src_stride + src_stride]))); - res_16b = _mm256_sra_epi16(_mm256_add_epi16(res_16b, round_const), - round_shift); + __m256i res_16b = convolve_lowbd_x(data, coeffs, filt); - /* rounding code */ - // 8 bit conversion and saturation to uint8 - __m256i res_8b = _mm256_packus_epi16(res_16b, res_16b); + res_16b = round_sr_x_avx2(res_16b); - // Store values into the destination buffer - // 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 - res_8b = _mm256_permute4x64_epi64(res_8b, 216); - __m128i res = _mm256_castsi256_si128(res_8b); - _mm_storeu_si128((__m128i *)&dst[i * dst_stride + j], res); - } - } + /* rounding code */ + // 8 bit conversion and saturation to uint8 + __m256i res_8b = _mm256_packus_epi16(res_16b, res_16b); + + const __m128i res_0 = _mm256_castsi256_si128(res_8b); + const __m128i res_1 = _mm256_extracti128_si256(res_8b, 1); + _mm_storel_epi64((__m128i *)&dst[i * dst_stride], res_0); + _mm_storel_epi64((__m128i *)&dst[i * dst_stride + dst_stride], res_1); + i += 2; + } while (i < h); + } else if (w == 16) { + do { + __m256i data[2] = { 0 }; + + load_convolve_8tap_16x2_avx2(src_ptr, src_stride, coeffs, filt, data); + round_pack_store_16x2_avx2(data, dst, dst_stride); + src_ptr += 2 * src_stride; + dst += 2 * dst_stride; + h -= 2; + } while (h); + } else if (w == 32) { + do { + load_convolve_round_8tap_32_avx2(src_ptr, coeffs, filt, dst); + src_ptr += src_stride; + dst += dst_stride; + } while ((--h) > 0); + } else if (w == 64) { + do { + load_convolve_round_8tap_32_avx2(src_ptr, coeffs, filt, dst); + load_convolve_round_8tap_32_avx2(src_ptr + 32, coeffs, filt, dst + 32); + src_ptr += src_stride; + dst += dst_stride; + } while ((--h) > 0); + } else { + assert(w == 128); + do { + load_convolve_round_8tap_32_avx2(src_ptr, coeffs, filt, dst); + load_convolve_round_8tap_32_avx2(src_ptr + SECOND_32_BLK, coeffs, filt, + dst + SECOND_32_BLK); + load_convolve_round_8tap_32_avx2(src_ptr + THIRD_32_BLK, coeffs, filt, + dst + THIRD_32_BLK); + load_convolve_round_8tap_32_avx2(src_ptr + FOURTH_32_BLK, coeffs, filt, + dst + FOURTH_32_BLK); + src_ptr += src_stride; + dst += dst_stride; + } while ((--h) > 0); } } else if (horiz_tap == 12) { // horiz_tap == 12 const int fo_horiz = filter_params_x->taps / 2 - 1; + prepare_coeffs_12taps(filter_params_x, subpel_x_qn, coeffs); + const __m128i round_shift = _mm_cvtsi32_si128(bits); const uint8_t *const src_ptr = src - fo_horiz; const __m256i v_zero = _mm256_setzero_si256(); - round_0_const = _mm256_set1_epi32((1 << (conv_params->round_0)) >> 1); - round_const = _mm256_set1_epi32((1 << bits) >> 1); - round_0_shift = _mm_cvtsi32_si128(conv_params->round_0); - __m256i s[6]; + __m256i round_0_const = + _mm256_set1_epi32((1 << (conv_params->round_0)) >> 1); + __m256i round_const = _mm256_set1_epi32((1 << bits) >> 1); + __m128i round_0_shift = _mm_cvtsi32_si128(conv_params->round_0); + __m256i s[6] = { 0 }; if (w <= 4) { - for (i = 0; i < h; i += 2) { + do { const __m256i data = _mm256_permute2x128_si256( _mm256_castsi128_si256( _mm_loadu_si128((__m128i *)(&src_ptr[i * src_stride]))), @@ -780,10 +1159,13 @@ *(uint16_t *)&dst[i * dst_stride + dst_stride] = (uint16_t)_mm_cvtsi128_si32(res_1); } - } + i += 2; + } while (i < h); } else { - for (i = 0; i < h; i++) { - for (int j = 0; j < w; j += 8) { + assert(!(w % 8)); + do { + j = 0; + do { const __m256i data = _mm256_permute2x128_si256( _mm256_castsi128_si256( _mm_loadu_si128((__m128i *)(&src_ptr[i * src_stride + j]))), @@ -826,103 +1208,189 @@ const __m128i res_1 = _mm256_extracti128_si256(res_8b_lo, 1); *(int *)&dst[i * dst_stride + j] = _mm_cvtsi128_si32(res_0); *(int *)&dst[i * dst_stride + j + 4] = _mm_cvtsi128_si32(res_1); - } - } + + j += 8; + } while (j < w); + i++; + } while (i < h); } } else { - const int fo_horiz = filter_params_x->taps / 2 - 1; - const uint8_t *const src_ptr = src - fo_horiz; - filt[2] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32 * 2)); - filt[3] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32 * 3)); + assert(horiz_tap == 2); + // since (filter_params_x->taps / 2 - 1) == 0 + const uint8_t *src_ptr = src; + if (subpel_x_qn != 8) { + if (w <= 8) { + prepare_coeffs_2t_ssse3(filter_params_x, subpel_x_qn, coeffs_128); - if (w <= 8) { - for (i = 0; i < h; i += 2) { - const __m256i data = _mm256_permute2x128_si256( - _mm256_castsi128_si256( - _mm_loadu_si128((__m128i *)(&src_ptr[i * src_stride]))), - _mm256_castsi128_si256(_mm_loadu_si128( - (__m128i *)(&src_ptr[i * src_stride + src_stride]))), - 0x20); - - __m256i res_16b = convolve_lowbd_x(data, coeffs, filt); - - res_16b = _mm256_sra_epi16(_mm256_add_epi16(res_16b, round_0_const), - round_0_shift); - - res_16b = _mm256_sra_epi16(_mm256_add_epi16(res_16b, round_const), - round_shift); - - /* rounding code */ - // 8 bit conversion and saturation to uint8 - __m256i res_8b = _mm256_packus_epi16(res_16b, res_16b); - - const __m128i res_0 = _mm256_castsi256_si128(res_8b); - const __m128i res_1 = _mm256_extracti128_si256(res_8b, 1); - if (w > 4) { - _mm_storel_epi64((__m128i *)&dst[i * dst_stride], res_0); - _mm_storel_epi64((__m128i *)&dst[i * dst_stride + dst_stride], res_1); - } else if (w > 2) { - xx_storel_32(&dst[i * dst_stride], res_0); - xx_storel_32(&dst[i * dst_stride + dst_stride], res_1); + if (w == 2) { + do { + const __m128i data = + convolve_x_2tap_2x2_ssse3(src_ptr, src_stride, coeffs_128); + const __m128i reg = round_sr_x_ssse3(data); + pack_store_u8_2x2_sse2(reg, dst, dst_stride); + src_ptr += 2 * src_stride; + dst += 2 * dst_stride; + h -= 2; + } while (h); + } else if (w == 4) { + do { + const __m128i data = + convolve_x_2tap_4x2_ssse3(src_ptr, src_stride, coeffs_128); + const __m128i reg = round_sr_x_ssse3(data); + pack_store_u8_4x2_sse2(reg, dst, dst_stride); + src_ptr += 2 * src_stride; + dst += 2 * dst_stride; + h -= 2; + } while (h); } else { - __m128i *const p_0 = (__m128i *)&dst[i * dst_stride]; - __m128i *const p_1 = (__m128i *)&dst[i * dst_stride + dst_stride]; - *(uint16_t *)p_0 = (uint16_t)_mm_cvtsi128_si32(res_0); - *(uint16_t *)p_1 = (uint16_t)_mm_cvtsi128_si32(res_1); + assert(w == 8); + + do { + __m128i data[2] = { 0 }; + + convolve_x_2tap_8x2_ssse3(src_ptr, src_stride, coeffs_128, data); + data[0] = round_sr_x_ssse3(data[0]); + data[1] = round_sr_x_ssse3(data[1]); + const __m128i reg = _mm_packus_epi16(data[0], data[1]); + _mm_storel_epi64((__m128i *)dst, reg); + _mm_storeh_epi64((__m128i *)(dst + dst_stride), reg); + + src_ptr += 2 * src_stride; + dst += 2 * dst_stride; + h -= 2; + } while (h); + } + } else { + prepare_coeffs_2t_lowbd(filter_params_x, subpel_x_qn, coeffs); + + if (w == 16) { + do { + __m256i data[2] = { 0 }; + + convolve_x_2tap_16x2_avx2(src_ptr, src_stride, coeffs, data); + round_pack_store_16x2_avx2(data, dst, dst_stride); + src_ptr += 2 * src_stride; + dst += 2 * dst_stride; + h -= 2; + } while (h); + } else if (w == 32) { + do { + convolve_round_2tap_32_avx2(src_ptr, coeffs, dst); + src_ptr += src_stride; + dst += dst_stride; + } while ((--h) > 0); + } else if (w == 64) { + do { + convolve_round_2tap_32_avx2(src_ptr, coeffs, dst); + convolve_round_2tap_32_avx2(src_ptr + SECOND_32_BLK, coeffs, + dst + SECOND_32_BLK); + src_ptr += src_stride; + dst += dst_stride; + } while ((--h) > 0); + } else { + assert(w == 128); + + do { + convolve_round_2tap_32_avx2(src_ptr, coeffs, dst); + convolve_round_2tap_32_avx2(src_ptr + (SECOND_32_BLK), coeffs, + dst + (SECOND_32_BLK)); + convolve_round_2tap_32_avx2(src_ptr + (THIRD_32_BLK), coeffs, + dst + (THIRD_32_BLK)); + convolve_round_2tap_32_avx2(src_ptr + (FOURTH_32_BLK), coeffs, + dst + (FOURTH_32_BLK)); + src_ptr += src_stride; + dst += dst_stride; + } while ((--h) > 0); } } } else { - for (i = 0; i < h; ++i) { - for (int j = 0; j < w; j += 16) { - // 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15 16 17 - // 18 19 20 21 22 23 - const __m256i data = _mm256_inserti128_si256( - _mm256_loadu_si256((__m256i *)&src_ptr[(i * src_stride) + j]), - _mm_loadu_si128((__m128i *)&src_ptr[(i * src_stride) + (j + 8)]), - 1); + if (w == 2) { + do { + __m128i data = load_x_u8_4x2_sse4(src_ptr, src_stride); + const __m128i reg1 = _mm_srli_si128(data, 1); + const __m128i reg2 = _mm_avg_epu8(data, reg1); + *(uint16_t *)dst = (uint16_t)_mm_cvtsi128_si32(reg2); + *(uint16_t *)(dst + dst_stride) = _mm_extract_epi16(reg2, 2); - __m256i res_16b = convolve_lowbd_x(data, coeffs, filt); + src_ptr += 2 * src_stride; + dst += 2 * dst_stride; + h -= 2; + } while (h); + } else if (w == 4) { + do { + __m128i data = load_8bit_8x2_to_1_reg_sse2( + src_ptr, (int)(sizeof(*src_ptr) * src_stride)); + const __m128i reg1 = _mm_srli_si128(data, 1); + const __m128i reg2 = _mm_avg_epu8(data, reg1); + xx_storel_32(dst, reg2); + *(int32_t *)(dst + dst_stride) = _mm_extract_epi32(reg2, 2); - res_16b = _mm256_sra_epi16(_mm256_add_epi16(res_16b, round_0_const), - round_0_shift); + src_ptr += 2 * src_stride; + dst += 2 * dst_stride; + h -= 2; + } while (h); + } else if (w == 8) { + do { + const __m128i data00 = _mm_loadu_si128((__m128i *)src_ptr); + const __m128i data10 = + _mm_loadu_si128((__m128i *)(src_ptr + src_stride)); + const __m128i data01 = _mm_srli_si128(data00, 1); + const __m128i data11 = _mm_srli_si128(data10, 1); + const __m128i reg0 = _mm_avg_epu8(data00, data01); + const __m128i reg1 = _mm_avg_epu8(data10, data11); + _mm_storel_epi64((__m128i *)dst, reg0); + _mm_storel_epi64((__m128i *)(dst + dst_stride), reg1); - res_16b = _mm256_sra_epi16(_mm256_add_epi16(res_16b, round_const), - round_shift); + src_ptr += 2 * src_stride; + dst += 2 * dst_stride; + h -= 2; + } while (h); + } else if (w == 16) { + do { + const __m128i data00 = _mm_loadu_si128((__m128i *)src_ptr); + const __m128i data01 = _mm_loadu_si128((__m128i *)(src_ptr + 1)); + const __m128i data10 = + _mm_loadu_si128((__m128i *)(src_ptr + src_stride)); + const __m128i data11 = + _mm_loadu_si128((__m128i *)(src_ptr + src_stride + 1)); + const __m128i reg0 = _mm_avg_epu8(data00, data01); + const __m128i reg1 = _mm_avg_epu8(data10, data11); + _mm_storeu_si128((__m128i *)dst, reg0); + _mm_storeu_si128((__m128i *)(dst + dst_stride), reg1); - /* rounding code */ - // 8 bit conversion and saturation to uint8 - __m256i res_8b = _mm256_packus_epi16(res_16b, res_16b); + src_ptr += 2 * src_stride; + dst += 2 * dst_stride; + h -= 2; + } while (h); + } else if (w == 32) { + do { + load_avg_store_2tap_32_avx2(src_ptr, dst); + src_ptr += src_stride; + dst += dst_stride; + } while ((--h) > 0); + } else if (w == 64) { + do { + load_avg_store_2tap_32_avx2(src_ptr, dst); + load_avg_store_2tap_32_avx2(src_ptr + (SECOND_32_BLK), + dst + (SECOND_32_BLK)); + src_ptr += src_stride; + dst += dst_stride; + } while ((--h) > 0); + } else { + assert(w == 128); - // Store values into the destination buffer - // 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 - res_8b = _mm256_permute4x64_epi64(res_8b, 216); - __m128i res = _mm256_castsi256_si128(res_8b); - _mm_storeu_si128((__m128i *)&dst[i * dst_stride + j], res); - } + do { + load_avg_store_2tap_32_avx2(src_ptr, dst); + load_avg_store_2tap_32_avx2(src_ptr + (SECOND_32_BLK), + dst + (SECOND_32_BLK)); + load_avg_store_2tap_32_avx2(src_ptr + (THIRD_32_BLK), + dst + (THIRD_32_BLK)); + load_avg_store_2tap_32_avx2(src_ptr + (FOURTH_32_BLK), + dst + (FOURTH_32_BLK)); + src_ptr += src_stride; + dst += dst_stride; + } while ((--h) > 0); } } } } - -void av1_convolve_x_sr_avx2(const uint8_t *src, int32_t src_stride, - uint8_t *dst, int32_t dst_stride, int32_t w, - int32_t h, - const InterpFilterParams *filter_params_x, - const int32_t subpel_x_qn, - ConvolveParams *conv_params) { -#if CONFIG_SVT_AV1 - const int horz_tap = get_filter_tap(filter_params_x, subpel_x_qn); - - if (horz_tap == 12) { - av1_convolve_x_sr_general_avx2(src, src_stride, dst, dst_stride, w, h, - filter_params_x, subpel_x_qn, conv_params); - } else { - av1_convolve_x_sr_specialized_avx2(src, src_stride, dst, dst_stride, w, h, - filter_params_x, subpel_x_qn, - conv_params); - } -#else - av1_convolve_x_sr_general_avx2(src, src_stride, dst, dst_stride, w, h, - filter_params_x, subpel_x_qn, conv_params); -#endif -}
diff --git a/av1/common/x86/warp_plane_avx2.c b/av1/common/x86/warp_plane_avx2.c index a780939..4f6f910 100644 --- a/av1/common/x86/warp_plane_avx2.c +++ b/av1/common/x86/warp_plane_avx2.c
@@ -14,6 +14,8 @@ #include "av1/common/warped_motion.h" #include "aom_dsp/x86/synonyms.h" +#if !CONFIG_HIGHWAY + DECLARE_ALIGNED(32, static const uint8_t, shuffle_alpha0_mask01_avx2[32]) = { 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 @@ -1208,3 +1210,5 @@ } } } + +#endif // !CONFIG_HIGHWAY
diff --git a/av1/common/x86/warp_plane_hwy_avx2.cc b/av1/common/x86/warp_plane_hwy_avx2.cc new file mode 100644 index 0000000..a721383 --- /dev/null +++ b/av1/common/x86/warp_plane_hwy_avx2.cc
@@ -0,0 +1,17 @@ +/* + * Copyright (c) 2025, Alliance for Open Media. All rights reserved. + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#define HWY_BASELINE_TARGETS HWY_AVX2 +#define HWY_BROKEN_32BIT 0 + +#include "av1/common/warp_plane_hwy.h" + +MAKE_WARP_AFFINE(avx2)
diff --git a/av1/common/x86/warp_plane_hwy_avx512.cc b/av1/common/x86/warp_plane_hwy_avx512.cc new file mode 100644 index 0000000..a0e2a87 --- /dev/null +++ b/av1/common/x86/warp_plane_hwy_avx512.cc
@@ -0,0 +1,17 @@ +/* + * Copyright (c) 2025, Alliance for Open Media. All rights reserved. + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#define HWY_BASELINE_TARGETS HWY_AVX3_DL +#define HWY_BROKEN_32BIT 0 + +#include "av1/common/warp_plane_hwy.h" + +MAKE_WARP_AFFINE(avx512)
diff --git a/av1/common/x86/warp_plane_hwy_sse4.cc b/av1/common/x86/warp_plane_hwy_sse4.cc new file mode 100644 index 0000000..f7c74fb7 --- /dev/null +++ b/av1/common/x86/warp_plane_hwy_sse4.cc
@@ -0,0 +1,17 @@ +/* + * Copyright (c) 2025, Alliance for Open Media. All rights reserved. + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#define HWY_BASELINE_TARGETS HWY_SSE4 +#define HWY_BROKEN_32BIT 0 + +#include "av1/common/warp_plane_hwy.h" + +MAKE_WARP_AFFINE(sse4_1)
diff --git a/av1/common/x86/warp_plane_sse4.c b/av1/common/x86/warp_plane_sse4.c index eb02683..11de2b0 100644 --- a/av1/common/x86/warp_plane_sse4.c +++ b/av1/common/x86/warp_plane_sse4.c
@@ -137,6 +137,8 @@ }; /* clang-format on */ +#if !CONFIG_HIGHWAY + // Shuffle masks: we want to convert a sequence of bytes 0, 1, 2, ..., 15 // in an SSE register into two sequences: // 0, 2, 2, 4, ..., 12, 12, 14, <don't care> @@ -906,3 +908,5 @@ } } } + +#endif // !CONFIG_HIGHWAY
diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c index ca9f0a4..b6ee3ac 100644 --- a/av1/decoder/decodeframe.c +++ b/av1/decoder/decodeframe.c
@@ -373,6 +373,9 @@ int mi_col, aom_reader *r, PARTITION_TYPE partition, BLOCK_SIZE bsize) { +#if CONFIG_COLLECT_COMPONENT_TIMING + start_timing(pbi, decode_mbmi_block_time); +#endif AV1_COMMON *const cm = &pbi->common; const SequenceHeader *const seq_params = cm->seq_params; const int bw = mi_size_wide[bsize]; @@ -396,6 +399,9 @@ aom_internal_error(xd->error_info, AOM_CODEC_CORRUPT_FRAME, "Invalid block size."); } +#if CONFIG_COLLECT_COMPONENT_TIMING + end_timing(pbi, decode_mbmi_block_time); +#endif } typedef struct PadBlock { @@ -902,6 +908,9 @@ static inline void decode_token_recon_block(AV1Decoder *const pbi, ThreadData *const td, aom_reader *r, BLOCK_SIZE bsize) { +#if CONFIG_COLLECT_COMPONENT_TIMING + start_timing(pbi, decode_token_recon_block_time); +#endif AV1_COMMON *const cm = &pbi->common; DecoderCodingBlock *const dcb = &td->dcb; MACROBLOCKD *const xd = &dcb->xd; @@ -909,6 +918,9 @@ MB_MODE_INFO *mbmi = xd->mi[0]; if (!is_inter_block(mbmi)) { +#if CONFIG_COLLECT_COMPONENT_TIMING + start_timing(pbi, decode_token_recon_block_intra_time); +#endif int row, col; assert(bsize == get_plane_block_size(bsize, xd->plane[0].subsampling_x, xd->plane[0].subsampling_y)); @@ -948,8 +960,20 @@ } } } +#if CONFIG_COLLECT_COMPONENT_TIMING + end_timing(pbi, decode_token_recon_block_intra_time); +#endif } else { +#if CONFIG_COLLECT_COMPONENT_TIMING + start_timing(pbi, decode_token_recon_block_inter_time); +#endif +#if CONFIG_COLLECT_COMPONENT_TIMING + start_timing(pbi, predict_inter_block_time); +#endif td->predict_inter_block_visit(cm, dcb, bsize); +#if CONFIG_COLLECT_COMPONENT_TIMING + end_timing(pbi, predict_inter_block_time); +#endif // Reconstruction if (!mbmi->skip_txfm) { int eobtotal = 0; @@ -994,9 +1018,15 @@ blk_row += bh_var_tx) { for (blk_col = col >> ss_x; blk_col < unit_width; blk_col += bw_var_tx) { +#if CONFIG_COLLECT_COMPONENT_TIMING + start_timing(pbi, decode_reconstruct_tx_inter_time); +#endif decode_reconstruct_tx(cm, td, r, mbmi, plane, plane_bsize, blk_row, blk_col, block, max_tx_size, &eobtotal); +#if CONFIG_COLLECT_COMPONENT_TIMING + end_timing(pbi, decode_reconstruct_tx_inter_time); +#endif block += step; } } @@ -1005,9 +1035,15 @@ } } td->cfl_store_inter_block_visit(cm, xd); +#if CONFIG_COLLECT_COMPONENT_TIMING + end_timing(pbi, decode_token_recon_block_inter_time); +#endif } av1_visit_palette(pbi, xd, r, set_color_index_map_offset); +#if CONFIG_COLLECT_COMPONENT_TIMING + end_timing(pbi, decode_token_recon_block_time); +#endif } static inline void set_inter_tx_size(MB_MODE_INFO *mbmi, int stride_log2, @@ -1128,6 +1164,9 @@ int mi_col, aom_reader *r, PARTITION_TYPE partition, BLOCK_SIZE bsize) { +#if CONFIG_COLLECT_COMPONENT_TIMING + start_timing(pbi, parse_decode_block_time); +#endif DecoderCodingBlock *const dcb = &td->dcb; MACROBLOCKD *const xd = &dcb->xd; decode_mbmi_block(pbi, dcb, mi_row, mi_col, r, partition, bsize); @@ -1180,6 +1219,9 @@ if (mbmi->skip_txfm) av1_reset_entropy_context(xd, bsize, num_planes); decode_token_recon_block(pbi, td, r, bsize); +#if CONFIG_COLLECT_COMPONENT_TIMING + end_timing(pbi, parse_decode_block_time); +#endif } static inline void set_offsets_for_pred_and_recon(AV1Decoder *const pbi, @@ -1907,14 +1949,19 @@ } } -static inline void resize_context_buffers(AV1_COMMON *cm, int width, - int height) { +static inline void resize_context_buffers(AV1_COMMON *cm, int width, int height, + unsigned int frame_size_limit) { #if CONFIG_SIZE_LIMIT if (width > DECODE_WIDTH_LIMIT || height > DECODE_HEIGHT_LIMIT) aom_internal_error(cm->error, AOM_CODEC_CORRUPT_FRAME, "Dimensions of %dx%d beyond allowed size of %dx%d.", width, height, DECODE_WIDTH_LIMIT, DECODE_HEIGHT_LIMIT); #endif + if (frame_size_limit && (uint64_t)width * height > frame_size_limit) { + aom_internal_error(cm->error, AOM_CODEC_CORRUPT_FRAME, + "Dimensions of %dx%d beyond allowed size of %u.", width, + height, frame_size_limit); + } if (cm->width != width || cm->height != height) { const int new_mi_rows = CEIL_POWER_OF_TWO(height, MI_SIZE_LOG2); const int new_mi_cols = CEIL_POWER_OF_TWO(width, MI_SIZE_LOG2); @@ -1977,6 +2024,7 @@ static inline void setup_frame_size(AV1_COMMON *cm, int frame_size_override_flag, + unsigned int frame_size_limit, struct aom_read_bit_buffer *rb) { const SequenceHeader *const seq_params = cm->seq_params; int width, height; @@ -1996,7 +2044,7 @@ } setup_superres(cm, rb, &width, &height); - resize_context_buffers(cm, width, height); + resize_context_buffers(cm, width, height, frame_size_limit); setup_render_size(cm, rb); setup_buffer_pool(cm); } @@ -2015,6 +2063,7 @@ } static inline void setup_frame_size_with_refs(AV1_COMMON *cm, + unsigned int frame_size_limit, struct aom_read_bit_buffer *rb) { int width, height; int found = 0; @@ -2037,7 +2086,7 @@ cm->render_width = buf->render_width; cm->render_height = buf->render_height; setup_superres(cm, rb, &width, &height); - resize_context_buffers(cm, width, height); + resize_context_buffers(cm, width, height, frame_size_limit); found = 1; break; } @@ -2051,7 +2100,7 @@ read_frame_size(rb, num_bits_width, num_bits_height, &width, &height); setup_superres(cm, rb, &width, &height); - resize_context_buffers(cm, width, height); + resize_context_buffers(cm, width, height, frame_size_limit); setup_render_size(cm, rb); } @@ -2718,6 +2767,9 @@ static inline void decode_tile(AV1Decoder *pbi, ThreadData *const td, int tile_row, int tile_col) { +#if CONFIG_COLLECT_COMPONENT_TIMING + start_timing(pbi, decode_tile_time); +#endif TileInfo tile_info; AV1_COMMON *const cm = &pbi->common; @@ -2741,9 +2793,25 @@ mi_col += cm->seq_params->mib_size) { set_cb_buffer(pbi, dcb, &td->cb_buffer_base, num_planes, 0, 0); +#if CONFIG_INSPECTION + const int start_bits = aom_reader_tell(td->bit_reader); +#endif // Bit-stream parsing and decoding of the superblock decode_partition(pbi, td, mi_row, mi_col, td->bit_reader, cm->seq_params->sb_size, 0x3); +#if CONFIG_INSPECTION + const int end_bits = aom_reader_tell(td->bit_reader); +#endif + +#if CONFIG_INSPECTION + if (pbi->sb_bits) { + const int mib_size = cm->seq_params->mib_size; + const int sb_cols = (cm->mi_params.mi_cols + mib_size - 1) / mib_size; + const int sb_row = mi_row / mib_size; + const int sb_col = mi_col / mib_size; + pbi->sb_bits[sb_row * sb_cols + sb_col] = end_bits - start_bits; + } +#endif if (aom_reader_has_overflowed(td->bit_reader)) { aom_merge_corrupted_flag(&dcb->corrupted, 1); @@ -2755,12 +2823,27 @@ int corrupted = (check_trailing_bits_after_symbol_coder(td->bit_reader)) ? 1 : 0; aom_merge_corrupted_flag(&dcb->corrupted, corrupted); +#if CONFIG_COLLECT_COMPONENT_TIMING + end_timing(pbi, decode_tile_time); +#endif } static const uint8_t *decode_tiles(AV1Decoder *pbi, const uint8_t *data, const uint8_t *data_end, int start_tile, int end_tile) { AV1_COMMON *const cm = &pbi->common; +#if CONFIG_INSPECTION + const int mib_size = cm->seq_params->mib_size; + const int sb_cols = (cm->mi_params.mi_cols + mib_size - 1) / mib_size; + const int sb_rows = (cm->mi_params.mi_rows + mib_size - 1) / mib_size; + const int required_size = sb_cols * sb_rows; + if (pbi->sb_bits_alloc_size < required_size) { + aom_free(pbi->sb_bits); + CHECK_MEM_ERROR(cm, pbi->sb_bits, + (int *)aom_malloc(sizeof(*pbi->sb_bits) * required_size)); + pbi->sb_bits_alloc_size = required_size; + } +#endif ThreadData *const td = &pbi->td; CommonTileParams *const tiles = &cm->tiles; const int tile_cols = tiles->cols; @@ -4845,7 +4928,7 @@ } if (current_frame->frame_type == KEY_FRAME) { - setup_frame_size(cm, frame_size_override_flag, rb); + setup_frame_size(cm, frame_size_override_flag, pbi->frame_size_limit, rb); if (features->allow_screen_content_tools && !av1_superres_scaled(cm)) features->allow_intrabc = aom_rb_read_bit(rb); @@ -4857,7 +4940,7 @@ if (current_frame->frame_type == INTRA_ONLY_FRAME) { cm->cur_frame->film_grain_params_present = seq_params->film_grain_params_present; - setup_frame_size(cm, frame_size_override_flag, rb); + setup_frame_size(cm, frame_size_override_flag, pbi->frame_size_limit, rb); if (features->allow_screen_content_tools && !av1_superres_scaled(cm)) features->allow_intrabc = aom_rb_read_bit(rb); @@ -4934,9 +5017,10 @@ } if (!features->error_resilient_mode && frame_size_override_flag) { - setup_frame_size_with_refs(cm, rb); + setup_frame_size_with_refs(cm, pbi->frame_size_limit, rb); } else { - setup_frame_size(cm, frame_size_override_flag, rb); + setup_frame_size(cm, frame_size_override_flag, pbi->frame_size_limit, + rb); } if (features->cur_frame_force_integer_mv) { @@ -5276,6 +5360,9 @@ const uint8_t *data_end, const uint8_t **p_data_end, int start_tile, int end_tile, int initialize_flag) { +#if CONFIG_COLLECT_COMPONENT_TIMING + start_timing(pbi, av1_decode_tg_tiles_and_wrapup_time); +#endif AV1_COMMON *const cm = &pbi->common; CommonTileParams *const tiles = &cm->tiles; MACROBLOCKD *const xd = &pbi->dcb.xd; @@ -5285,6 +5372,9 @@ if (initialize_flag) setup_frame_info(pbi); const int num_planes = av1_num_planes(cm); +#if CONFIG_COLLECT_COMPONENT_TIMING + start_timing(pbi, decode_tiles_time); +#endif if (pbi->max_threads > 1 && !(tiles->large_scale && !pbi->ext_tile_debug) && pbi->row_mt) *p_data_end = @@ -5294,6 +5384,9 @@ *p_data_end = decode_tiles_mt(pbi, data, data_end, start_tile, end_tile); else *p_data_end = decode_tiles(pbi, data, data_end, start_tile, end_tile); +#if CONFIG_COLLECT_COMPONENT_TIMING + end_timing(pbi, decode_tiles_time); +#endif // If the bit stream is monochrome, set the U and V buffers to a constant. if (num_planes < 3) { @@ -5309,11 +5402,17 @@ av1_alloc_cdef_sync(cm, &pbi->cdef_sync, pbi->num_workers); if (!cm->features.allow_intrabc && !tiles->single_tile_decoding) { +#if CONFIG_COLLECT_COMPONENT_TIMING + start_timing(pbi, av1_loop_filter_frame_time); +#endif if (cm->lf.filter_level[0] || cm->lf.filter_level[1]) { av1_loop_filter_frame_mt(&cm->cur_frame->buf, cm, &pbi->dcb.xd, 0, num_planes, 0, pbi->tile_workers, pbi->num_workers, &pbi->lf_row_sync, 0); } +#if CONFIG_COLLECT_COMPONENT_TIMING + end_timing(pbi, av1_loop_filter_frame_time); +#endif const int do_cdef = !pbi->skip_loop_filter && !cm->features.coded_lossless && @@ -5328,6 +5427,10 @@ // Frame border extension is not required in the decoder // as it happens in extend_mc_border(). int do_extend_border_mt = 0; + +#if CONFIG_COLLECT_COMPONENT_TIMING + start_timing(pbi, cdef_and_lr_time); +#endif if (!optimized_loop_restoration) { if (do_loop_restoration) av1_loop_restoration_save_boundary_lines(&pbi->common.cur_frame->buf, @@ -5377,6 +5480,9 @@ } } } +#if CONFIG_COLLECT_COMPONENT_TIMING + end_timing(pbi, cdef_and_lr_time); +#endif } if (!pbi->dcb.corrupted) { @@ -5404,4 +5510,8 @@ if (cm->show_frame && !cm->seq_params->order_hint_info.enable_order_hint) { ++cm->current_frame.frame_number; } + +#if CONFIG_COLLECT_COMPONENT_TIMING + end_timing(pbi, av1_decode_tg_tiles_and_wrapup_time); +#endif }
diff --git a/av1/decoder/decoder.c b/av1/decoder/decoder.c index 0ae0b86..81a88b4 100644 --- a/av1/decoder/decoder.c +++ b/av1/decoder/decoder.c
@@ -148,6 +148,11 @@ aom_get_worker_interface()->init(&pbi->lf_worker); pbi->lf_worker.thread_name = "aom lf worker"; +#if CONFIG_INSPECTION + pbi->sb_bits = NULL; + pbi->sb_bits_alloc_size = 0; +#endif + return pbi; } @@ -226,6 +231,9 @@ #if CONFIG_ACCOUNTING aom_accounting_clear(&pbi->accounting); #endif +#if CONFIG_INSPECTION + aom_free(pbi->sb_bits); +#endif av1_free_mc_tmp_buf(&pbi->td); aom_img_metadata_array_free(pbi->metadata); av1_remove_common(&pbi->common); @@ -260,16 +268,17 @@ const YV12_BUFFER_CONFIG *const cfg = get_ref_frame(cm, idx); if (cfg == NULL) { - aom_internal_error(&pbi->error, AOM_CODEC_ERROR, "No reference frame"); - return AOM_CODEC_ERROR; + aom_set_error(&pbi->error, AOM_CODEC_ERROR, "No reference frame"); + return pbi->error.error_code; } - if (!equal_dimensions(cfg, sd)) - aom_internal_error(&pbi->error, AOM_CODEC_ERROR, - "Incorrect buffer dimensions"); - else - aom_yv12_copy_frame(cfg, sd, num_planes); + if (!equal_dimensions(cfg, sd)) { + aom_set_error(&pbi->error, AOM_CODEC_ERROR, "Incorrect buffer dimensions"); + return pbi->error.error_code; + } - return pbi->error.error_code; + aom_yv12_copy_frame(cfg, sd, num_planes); + + return AOM_CODEC_OK; } static int equal_dimensions_and_border(const YV12_BUFFER_CONFIG *a, @@ -286,43 +295,40 @@ int use_external_ref, YV12_BUFFER_CONFIG *sd) { const int num_planes = av1_num_planes(cm); - YV12_BUFFER_CONFIG *ref_buf = NULL; - + // Ensure that aom_internal_error() calls longjmp(). + assert(cm->error->setjmp); // Get the destination reference buffer. - ref_buf = get_ref_frame(cm, idx); + YV12_BUFFER_CONFIG *ref_buf = get_ref_frame(cm, idx); if (ref_buf == NULL) { aom_internal_error(cm->error, AOM_CODEC_ERROR, "No reference frame"); - return AOM_CODEC_ERROR; } if (!use_external_ref) { if (!equal_dimensions(ref_buf, sd)) { aom_internal_error(cm->error, AOM_CODEC_ERROR, "Incorrect buffer dimensions"); - } else { - // Overwrite the reference frame buffer. - aom_yv12_copy_frame(sd, ref_buf, num_planes); } + // Overwrite the reference frame buffer. + aom_yv12_copy_frame(sd, ref_buf, num_planes); } else { if (!equal_dimensions_and_border(ref_buf, sd)) { aom_internal_error(cm->error, AOM_CODEC_ERROR, "Incorrect buffer dimensions"); - } else { - // Overwrite the reference frame buffer pointers. - // Once we no longer need the external reference buffer, these pointers - // are restored. - ref_buf->store_buf_adr[0] = ref_buf->y_buffer; - ref_buf->store_buf_adr[1] = ref_buf->u_buffer; - ref_buf->store_buf_adr[2] = ref_buf->v_buffer; - ref_buf->y_buffer = sd->y_buffer; - ref_buf->u_buffer = sd->u_buffer; - ref_buf->v_buffer = sd->v_buffer; - ref_buf->use_external_reference_buffers = 1; } + // Overwrite the reference frame buffer pointers. + // Once we no longer need the external reference buffer, these pointers + // are restored. + ref_buf->store_buf_adr[0] = ref_buf->y_buffer; + ref_buf->store_buf_adr[1] = ref_buf->u_buffer; + ref_buf->store_buf_adr[2] = ref_buf->v_buffer; + ref_buf->y_buffer = sd->y_buffer; + ref_buf->u_buffer = sd->u_buffer; + ref_buf->v_buffer = sd->v_buffer; + ref_buf->use_external_reference_buffers = 1; } - return cm->error->error_code; + return AOM_CODEC_OK; } aom_codec_err_t av1_copy_new_frame_dec(AV1_COMMON *cm, @@ -330,13 +336,14 @@ YV12_BUFFER_CONFIG *sd) { const int num_planes = av1_num_planes(cm); - if (!equal_dimensions_and_border(new_frame, sd)) - aom_internal_error(cm->error, AOM_CODEC_ERROR, - "Incorrect buffer dimensions"); - else - aom_yv12_copy_frame(new_frame, sd, num_planes); + if (!equal_dimensions_and_border(new_frame, sd)) { + aom_set_error(cm->error, AOM_CODEC_ERROR, "Incorrect buffer dimensions"); + return cm->error->error_code; + } - return cm->error->error_code; + aom_yv12_copy_frame(new_frame, sd, num_planes); + + return AOM_CODEC_OK; } static void release_current_frame(AV1Decoder *pbi) {
diff --git a/av1/decoder/decoder.h b/av1/decoder/decoder.h index d6610cd..02ad429 100644 --- a/av1/decoder/decoder.h +++ b/av1/decoder/decoder.h
@@ -229,6 +229,53 @@ int alloc_tile_cols; } AV1DecTileMT; +#if CONFIG_COLLECT_COMPONENT_TIMING +#include "aom_ports/aom_timer.h" +// Adjust the following to add new components. +enum { + decode_mbmi_block_time, + decode_token_recon_block_intra_time, + predict_inter_block_time, + decode_reconstruct_tx_inter_time, + decode_token_recon_block_inter_time, + decode_token_recon_block_time, + parse_decode_block_time, + decode_tile_time, + decode_tiles_time, + av1_loop_filter_frame_time, + cdef_and_lr_time, + av1_decode_tg_tiles_and_wrapup_time, + aom_decode_frame_from_obus_time, + kTimingComponents, +} UENUM1BYTE(TIMING_COMPONENT); + +static inline char const *get_component_name(int index) { + switch (index) { + case decode_mbmi_block_time: return "decode_mbmi_block_time"; + case decode_token_recon_block_intra_time: + return "decode_token_recon_block_intra_time"; + case predict_inter_block_time: return "predict_inter_block_time"; + case decode_reconstruct_tx_inter_time: + return "decode_reconstruct_tx_inter_time"; + case decode_token_recon_block_inter_time: + return "decode_token_recon_block_inter_time"; + case decode_token_recon_block_time: return "decode_token_recon_block_time"; + case parse_decode_block_time: return "parse_decode_block_time"; + case decode_tile_time: return "decode_tile_time"; + case decode_tiles_time: return "decode_tiles_time"; + case av1_loop_filter_frame_time: return "av1_loop_filter_frame_time"; + case cdef_and_lr_time: return "cdef_and_lr_time"; + case av1_decode_tg_tiles_and_wrapup_time: + return "av1_decode_tg_tiles_and_wrapup_time"; + case aom_decode_frame_from_obus_time: + return "aom_decode_frame_from_obus_time"; + + default: assert(0); + } + return "error"; +} +#endif + typedef struct AV1Decoder { DecoderCodingBlock dcb; @@ -287,9 +334,13 @@ #if CONFIG_INSPECTION aom_inspect_cb inspect_cb; void *inspect_ctx; + int *sb_bits; + int sb_bits_alloc_size; #endif int operating_point; int current_operating_point; + // If nonzero, the maximum frame size (width * height). 0 means unlimited. + unsigned int frame_size_limit; int seen_frame_header; // The expected start_tile (tg_start syntax element) of the next tile group. int next_start_tile; @@ -369,6 +420,18 @@ * Number of spatial layers: may be > 1 for SVC (scalable vector coding). */ unsigned int number_spatial_layers; + +#if CONFIG_COLLECT_COMPONENT_TIMING + /*! + * component_time[] are initialized to zero while decoder starts. + */ + uint64_t component_time[kTimingComponents]; + struct aom_usec_timer component_timer[kTimingComponents]; + /*! + * frame_component_time[] are initialized to zero at beginning of each frame. + */ + uint64_t frame_component_time[kTimingComponents]; +#endif } AV1Decoder; // Returns 0 on success. Sets pbi->common.error.error_code to a nonzero error @@ -445,6 +508,28 @@ /*!\endcond */ +#if CONFIG_COLLECT_COMPONENT_TIMING +static inline void start_timing(AV1Decoder *pbi, int component) { + aom_usec_timer_start(&pbi->component_timer[component]); +} +static inline void end_timing(AV1Decoder *pbi, int component) { + aom_usec_timer_mark(&pbi->component_timer[component]); + pbi->frame_component_time[component] += + aom_usec_timer_elapsed(&pbi->component_timer[component]); +} + +static inline char const *get_frame_type_enum(int type) { + switch (type) { + case 0: return "KEY_FRAME"; + case 1: return "INTER_FRAME"; + case 2: return "INTRA_ONLY_FRAME"; + case 3: return "S_FRAME"; + default: assert(0); + } + return "error"; +} +#endif + #ifdef __cplusplus } // extern "C" #endif
diff --git a/av1/decoder/decodetxb.c b/av1/decoder/decodetxb.c index 17d67cb..e37f239 100644 --- a/av1/decoder/decodetxb.c +++ b/av1/decoder/decodetxb.c
@@ -30,7 +30,6 @@ if (length > 20) { aom_internal_error(xd->error_info, AOM_CODEC_CORRUPT_FRAME, "Invalid length in read_golomb"); - break; } } @@ -341,6 +340,8 @@ TXB_CTX txb_ctx; get_txb_ctx(plane_bsize, tx_size, plane, pd->above_entropy_context + col, pd->left_entropy_context + row, &txb_ctx); + + assert(dcb->xd.error_info->setjmp); const uint8_t cul_level = read_coeffs_txb(cm, dcb, r, row, col, plane, &txb_ctx, tx_size); av1_set_entropy_contexts(xd, pd, plane, plane_bsize, tx_size, cul_level, col,
diff --git a/av1/decoder/inspection.c b/av1/decoder/inspection.c index f84ed17..0eb6f38 100644 --- a/av1/decoder/inspection.c +++ b/av1/decoder/inspection.c
@@ -156,6 +156,16 @@ } // delta_q mi->current_qindex = mbmi->current_qindex; + + if (pbi->sb_bits) { + const int mib_size = cm->seq_params->mib_size; + const int sb_cols = (cm->mi_params.mi_cols + mib_size - 1) / mib_size; + const int sb_row = j / mib_size; + const int sb_col = i / mib_size; + mi->sb_bits = (int16_t)pbi->sb_bits[sb_row * sb_cols + sb_col]; + } else { + mi->sb_bits = 0; + } } } return 1;
diff --git a/av1/decoder/inspection.h b/av1/decoder/inspection.h index 7d1c14d..132147b 100644 --- a/av1/decoder/inspection.h +++ b/av1/decoder/inspection.h
@@ -55,6 +55,7 @@ int16_t intrabc; int16_t palette; int16_t uv_palette; + int16_t sb_bits; }; typedef struct insp_frame_data insp_frame_data;
diff --git a/av1/decoder/obu.c b/av1/decoder/obu.c index ea4b1b6..1075885 100644 --- a/av1/decoder/obu.c +++ b/av1/decoder/obu.c
@@ -66,27 +66,18 @@ return 0; } -static int byte_alignment(AV1_COMMON *const cm, - struct aom_read_bit_buffer *const rb) { +static void byte_alignment(struct aom_internal_error_info *error_info, + struct aom_read_bit_buffer *const rb) { while (rb->bit_offset & 7) { if (aom_rb_read_bit(rb)) { - cm->error->error_code = AOM_CODEC_CORRUPT_FRAME; - return -1; + aom_internal_error(error_info, AOM_CODEC_CORRUPT_FRAME, + "byte_alignment() is not all 0 bits"); } } - return 0; } static uint32_t read_temporal_delimiter_obu(void) { return 0; } -// Returns a boolean that indicates success. -static int read_bitstream_level(AV1_LEVEL *seq_level_idx, - struct aom_read_bit_buffer *rb) { - *seq_level_idx = aom_rb_read_literal(rb, LEVEL_BITS); - if (!is_valid_seq_level_idx(*seq_level_idx)) return 0; - return 1; -} - // Returns whether two sequence headers are consistent with each other. // Note that the 'op_params' field is not compared per Section 7.5 in the spec: // Within a particular coded video sequence, the contents of @@ -100,7 +91,8 @@ // On success, sets pbi->sequence_header_ready to 1 and returns the number of // bytes read from 'rb'. -// On failure, sets pbi->common.error.error_code and returns 0. +// On failure, calls aom_internal_error() and does not return, or sets +// pbi->error.error_code and returns 0. static uint32_t read_sequence_header_obu(AV1Decoder *pbi, struct aom_read_bit_buffer *rb) { AV1_COMMON *const cm = &pbi->common; @@ -136,10 +128,7 @@ seq_params->operating_points_cnt_minus_1 = 0; seq_params->operating_point_idc[0] = 0; seq_params->has_nonzero_operating_point_idc = false; - if (!read_bitstream_level(&seq_params->seq_level_idx[0], rb)) { - pbi->error.error_code = AOM_CODEC_UNSUP_BITSTREAM; - return 0; - } + seq_params->seq_level_idx[0] = aom_rb_read_literal(rb, LEVEL_BITS); seq_params->tier[0] = 0; seq_params->op_params[0].decoder_model_param_present_flag = 0; seq_params->op_params[0].display_model_param_present_flag = 0; @@ -163,10 +152,7 @@ aom_rb_read_literal(rb, OP_POINTS_IDC_BITS); if (seq_params->operating_point_idc[i] != 0) seq_params->has_nonzero_operating_point_idc = true; - if (!read_bitstream_level(&seq_params->seq_level_idx[i], rb)) { - pbi->error.error_code = AOM_CODEC_UNSUP_BITSTREAM; - return 0; - } + seq_params->seq_level_idx[i] = aom_rb_read_literal(rb, LEVEL_BITS); // This is the seq_level_idx[i] > 7 check in the spec. seq_level_idx 7 // is equivalent to level 3.3. if (seq_params->seq_level_idx[i] >= SEQ_LEVEL_4_0) @@ -196,7 +182,7 @@ aom_internal_error(&pbi->error, AOM_CODEC_UNSUP_BITSTREAM, "AV1 does not support this combination of " "profile, level, and tier."); - // Buffer size in bits/s is bitrate in bits/s * 1 s + // Buffer size in bits is bitrate in bits/s * 1 s seq_params->op_params[i].buffer_size = seq_params->op_params[i].bitrate; } if (seq_params->timing_info_present && @@ -236,10 +222,11 @@ operating_point = 0; pbi->current_operating_point = seq_params->operating_point_idc[operating_point]; - if (aom_get_num_layers_from_operating_point_idc( - pbi->current_operating_point, &pbi->number_spatial_layers, - &pbi->number_temporal_layers) != AOM_CODEC_OK) { - pbi->error.error_code = AOM_CODEC_ERROR; + aom_codec_err_t status = aom_get_num_layers_from_operating_point_idc( + pbi->current_operating_point, &pbi->number_spatial_layers, + &pbi->number_temporal_layers); + if (status != AOM_CODEC_OK) { + pbi->error.error_code = status; return 0; } @@ -293,11 +280,11 @@ } // On success, returns the tile group header size. On failure, calls -// aom_internal_error() and returns -1. -static int32_t read_tile_group_header(AV1Decoder *pbi, - struct aom_read_bit_buffer *rb, - int *start_tile, int *end_tile, - int tile_start_implicit) { +// aom_internal_error() and does not return. +static uint32_t read_tile_group_header(AV1Decoder *pbi, + struct aom_read_bit_buffer *rb, + int *start_tile, int *end_tile, + int tile_start_implicit) { AV1_COMMON *const cm = &pbi->common; CommonTileParams *const tiles = &cm->tiles; uint32_t saved_bit_offset = rb->bit_offset; @@ -310,7 +297,6 @@ aom_internal_error( &pbi->error, AOM_CODEC_UNSUP_BITSTREAM, "For OBU_FRAME type obu tile_start_and_end_present_flag must be 0"); - return -1; } } if (tiles->large_scale || num_tiles == 1 || @@ -326,47 +312,43 @@ aom_internal_error(&pbi->error, AOM_CODEC_CORRUPT_FRAME, "tg_start (%d) must be equal to %d", *start_tile, pbi->next_start_tile); - return -1; } if (*start_tile > *end_tile) { aom_internal_error( &pbi->error, AOM_CODEC_CORRUPT_FRAME, "tg_end (%d) must be greater than or equal to tg_start (%d)", *end_tile, *start_tile); - return -1; } if (*end_tile >= num_tiles) { aom_internal_error(&pbi->error, AOM_CODEC_CORRUPT_FRAME, "tg_end (%d) must be less than NumTiles (%d)", *end_tile, num_tiles); - return -1; } pbi->next_start_tile = (*end_tile == num_tiles - 1) ? 0 : *end_tile + 1; return ((rb->bit_offset - saved_bit_offset + 7) >> 3); } -// On success, returns the tile group OBU size. On failure, sets -// pbi->common.error.error_code and returns 0. +// On success, returns the tile group OBU size. On failure, calls +// aom_internal_error() and does not return. static uint32_t read_one_tile_group_obu( AV1Decoder *pbi, struct aom_read_bit_buffer *rb, int is_first_tg, const uint8_t *data, const uint8_t *data_end, const uint8_t **p_data_end, int *is_last_tg, int tile_start_implicit) { AV1_COMMON *const cm = &pbi->common; int start_tile, end_tile; - int32_t header_size, tg_payload_size; assert((rb->bit_offset & 7) == 0); assert(rb->bit_buffer + aom_rb_bytes_read(rb) == data); - header_size = read_tile_group_header(pbi, rb, &start_tile, &end_tile, - tile_start_implicit); - if (header_size == -1 || byte_alignment(cm, rb)) return 0; + uint32_t header_size = read_tile_group_header(pbi, rb, &start_tile, &end_tile, + tile_start_implicit); + byte_alignment(&pbi->error, rb); data += header_size; av1_decode_tg_tiles_and_wrapup(pbi, data, data_end, p_data_end, start_tile, end_tile, is_first_tg); - tg_payload_size = (uint32_t)(*p_data_end - data); + uint32_t tg_payload_size = (uint32_t)(*p_data_end - data); *is_last_tg = end_tile == cm->tiles.rows * cm->tiles.cols - 1; return header_size + tg_payload_size; @@ -482,7 +464,7 @@ // Only called while large_scale_tile = 1. // // On success, returns the tile list OBU size. On failure, sets -// pbi->common.error.error_code and returns 0. +// pbi->error.error_code and returns 0. static uint32_t read_and_decode_one_tile_list(AV1Decoder *pbi, struct aom_read_bit_buffer *rb, const uint8_t *data, @@ -628,7 +610,7 @@ // On failure, calls aom_internal_error() and does not return. static void read_metadata_itut_t35(AV1Decoder *const pbi, const uint8_t *data, - size_t sz) { + size_t sz, bool has_obu_extension_header) { if (sz == 0) { aom_internal_error(&pbi->error, AOM_CODEC_CORRUPT_FRAME, "itu_t_t35_country_code is missing"); @@ -657,7 +639,9 @@ data[end_index]); } alloc_read_metadata(pbi, OBU_METADATA_TYPE_ITUT_T35, data, end_index, - AOM_MIF_ANY_FRAME); + has_obu_extension_header + ? AOM_MIF_ANY_FRAME_LAYER_SPECIFIC + : AOM_MIF_ANY_FRAME); } // On success, returns the number of bytes read from 'data'. On failure, calls @@ -782,9 +766,10 @@ // Checks the metadata for correct syntax but ignores the parsed metadata. // // On success, returns the number of bytes read from 'data'. On failure, sets -// pbi->common.error.error_code and returns 0, or calls aom_internal_error() -// and does not return. -static size_t read_metadata(AV1Decoder *pbi, const uint8_t *data, size_t sz) { +// pbi->error.error_code and returns 0, or calls aom_internal_error() and does +// not return. +static size_t read_metadata(AV1Decoder *pbi, const uint8_t *data, size_t sz, + bool has_obu_extension_header) { size_t type_length; uint64_t type_value; if (aom_uleb_decode(data, sz, &type_value, &type_length) < 0) { @@ -803,7 +788,8 @@ } if (metadata_type == OBU_METADATA_TYPE_ITUT_T35) { // read_metadata_itut_t35() checks trailing bits. - read_metadata_itut_t35(pbi, data + type_length, sz - type_length); + read_metadata_itut_t35(pbi, data + type_length, sz - type_length, + has_obu_extension_header); return sz; } else if (metadata_type == OBU_METADATA_TYPE_HDR_CLL) { size_t bytes_read = @@ -841,10 +827,10 @@ return type_length + (rb.bit_offset >> 3); } -// On success, returns 'sz'. On failure, sets pbi->common.error.error_code and -// returns 0. -static size_t read_padding(AV1_COMMON *const cm, const uint8_t *data, - size_t sz) { +// On success, returns 'sz'. On failure, sets error_info->error_code and returns +// 0. +static size_t read_padding(struct aom_internal_error_info *error_info, + const uint8_t *data, size_t sz) { // The spec allows a padding OBU to be header-only (i.e., obu_size = 0). So // check trailing bits only if sz > 0. if (sz > 0) { @@ -852,7 +838,7 @@ // trailing byte should be 0x80. See https://crbug.com/aomedia/2393. const uint8_t last_nonzero_byte = get_last_nonzero_byte(data, sz); if (last_nonzero_byte != 0x80) { - cm->error->error_code = AOM_CODEC_CORRUPT_FRAME; + error_info->error_code = AOM_CODEC_CORRUPT_FRAME; return 0; } } @@ -865,6 +851,9 @@ int aom_decode_frame_from_obus(struct AV1Decoder *pbi, const uint8_t *data, const uint8_t *data_end, const uint8_t **p_data_end) { +#if CONFIG_COLLECT_COMPONENT_TIMING + start_timing(pbi, aom_decode_frame_from_obus_time); +#endif AV1_COMMON *const cm = &pbi->common; int frame_decoding_finished = 0; int is_first_tg_obu_received = 1; @@ -880,6 +869,9 @@ const uint8_t *frame_header = data; uint32_t frame_header_size = 0; ObuHeader obu_header; + + assert(pbi->error.setjmp); + memset(&obu_header, 0, sizeof(obu_header)); pbi->seen_frame_header = 0; pbi->next_start_tile = 0; @@ -1033,8 +1025,7 @@ if (obu_header.type != OBU_FRAME) break; obu_payload_offset = frame_header_size; // Byte align the reader before reading the tile group. - // byte_alignment() has set pbi->error.error_code if it returns -1. - if (byte_alignment(cm, &rb)) return -1; + byte_alignment(&pbi->error, &rb); AOM_FALLTHROUGH_INTENDED; // fall through to read tile group. case OBU_TILE_GROUP: if (!pbi->seen_frame_header) { @@ -1049,7 +1040,6 @@ pbi, &rb, is_first_tg_obu_received, data + obu_payload_offset, data + payload_size, p_data_end, &frame_decoding_finished, obu_header.type == OBU_FRAME); - if (pbi->error.error_code != AOM_CODEC_OK) return -1; is_first_tg_obu_received = 0; if (frame_decoding_finished) { pbi->seen_frame_header = 0; @@ -1057,10 +1047,12 @@ } pbi->num_tile_groups++; break; - case OBU_METADATA: - decoded_payload_size = read_metadata(pbi, data, payload_size); + case OBU_METADATA: { + decoded_payload_size = + read_metadata(pbi, data, payload_size, obu_header.has_extension); if (pbi->error.error_code != AOM_CODEC_OK) return -1; break; + } case OBU_TILE_LIST: if (CONFIG_NORMAL_TILE_MODE) { pbi->error.error_code = AOM_CODEC_UNSUP_BITSTREAM; @@ -1082,7 +1074,7 @@ if (pbi->error.error_code != AOM_CODEC_OK) return -1; break; case OBU_PADDING: - decoded_payload_size = read_padding(cm, data, payload_size); + decoded_payload_size = read_padding(&pbi->error, data, payload_size); if (pbi->error.error_code != AOM_CODEC_OK) return -1; break; default: @@ -1115,5 +1107,29 @@ } if (pbi->error.error_code != AOM_CODEC_OK) return -1; + +#if CONFIG_COLLECT_COMPONENT_TIMING + end_timing(pbi, aom_decode_frame_from_obus_time); + + // Print out timing information. + int i; + fprintf(stderr, + "\n Frame number: %d, Frame type: %s, Show Frame: %d, Show existing " + "frame: %d\n", + cm->current_frame.frame_number, + get_frame_type_enum(cm->current_frame.frame_type), cm->show_frame, + cm->show_existing_frame); + // Exclude show_existing_frame since it doesn't take much time. + if (!cm->show_existing_frame) { + for (i = 0; i < kTimingComponents; i++) { + pbi->component_time[i] += pbi->frame_component_time[i]; + fprintf(stderr, " %s: %" PRId64 " us (total: %" PRId64 " us)\n", + get_component_name(i), pbi->frame_component_time[i], + pbi->component_time[i]); + pbi->frame_component_time[i] = 0; + } + } +#endif + return frame_decoding_finished; }
diff --git a/av1/encoder/aq_cyclicrefresh.c b/av1/encoder/aq_cyclicrefresh.c index d0135fd..d3aabdb 100644 --- a/av1/encoder/aq_cyclicrefresh.c +++ b/av1/encoder/aq_cyclicrefresh.c
@@ -439,6 +439,8 @@ // should we enable cyclic refresh on this frame. cr->apply_cyclic_refresh = 1; if (frame_is_intra_only(cm) || is_lossless_requested(&cpi->oxcf.rc_cfg) || + (is_one_pass_rt_lag_params(cpi) && + (cpi->refresh_frame.alt_ref_frame || cpi->rc.is_src_frame_alt_ref)) || cpi->roi.enabled || cpi->rc.high_motion_content_screen_rtc || scene_change_detected || svc->temporal_layer_id > 0 || svc->prev_number_spatial_layers != svc->number_spatial_layers ||
diff --git a/av1/encoder/arm/av1_highbd_quantize_neon.c b/av1/encoder/arm/av1_highbd_quantize_neon.c index c1016db..fd4749d 100644 --- a/av1/encoder/arm/av1_highbd_quantize_neon.c +++ b/av1/encoder/arm/av1_highbd_quantize_neon.c
@@ -60,15 +60,15 @@ static inline int16x8_t get_max_lane_eob(const int16_t *iscan, int16x8_t v_eobmax, uint16x8_t v_mask) { - const int16x8_t v_iscan = vld1q_s16(&iscan[0]); - const int16x8_t v_iscan_plus1 = vaddq_s16(v_iscan, vdupq_n_s16(1)); - const int16x8_t v_nz_iscan = vbslq_s16(v_mask, v_iscan_plus1, vdupq_n_s16(0)); + const int16x8_t v_iscan = vld1q_s16(iscan); + const int16x8_t v_nz_iscan = vbslq_s16(v_mask, v_iscan, vdupq_n_s16(-1)); return vmaxq_s16(v_eobmax, v_nz_iscan); } static inline uint16_t get_max_eob(int16x8_t v_eobmax) { #if AOM_ARCH_AARCH64 - return (uint16_t)vmaxvq_s16(v_eobmax); + int16_t max_val = vmaxvq_s16(v_eobmax); + return (uint16_t)max_val + 1; #else const int16x4_t v_eobmax_3210 = vmax_s16(vget_low_s16(v_eobmax), vget_high_s16(v_eobmax)); @@ -80,7 +80,7 @@ vshr_n_s64(vreinterpret_s64_s16(v_eobmax_tmp), 16); const int16x4_t v_eobmax_final = vmax_s16(v_eobmax_tmp, vreinterpret_s16_s64(v_eobmax_xxx3)); - return (uint16_t)vget_lane_s16(v_eobmax_final, 0); + return (uint16_t)vget_lane_s16(v_eobmax_final, 0) + 1; #endif }
diff --git a/av1/encoder/arm/highbd_pickrst_neon.c b/av1/encoder/arm/highbd_pickrst_neon.c index bb85a4b..499eb58 100644 --- a/av1/encoder/arm/highbd_pickrst_neon.c +++ b/av1/encoder/arm/highbd_pickrst_neon.c
@@ -463,64 +463,34 @@ { const int16_t *d_t = d; - if (height % 2) { - int32x4_t deltas[(WIENER_WIN + 1) * 2] = { vdupq_n_s32(0) }; - int32x4_t deltas_tr[(WIENER_WIN + 1) * 2] = { vdupq_n_s32(0) }; - int16x8_t ds[WIENER_WIN * 2]; + int32x4_t deltas[WIENER_WIN_CHROMA] = { vdupq_n_s32(0) }; + int16x8_t ds[WIENER_WIN_CHROMA + 1]; - load_s16_8x4(d_t, d_stride, &ds[0], &ds[2], &ds[4], &ds[6]); - load_s16_8x4(d_t + width, d_stride, &ds[1], &ds[3], &ds[5], &ds[7]); - d_t += 4 * d_stride; + ds[0] = load_unaligned_s16_4x2(d_t + 0 * d_stride, width); + ds[1] = load_unaligned_s16_4x2(d_t + 1 * d_stride, width); + ds[2] = load_unaligned_s16_4x2(d_t + 2 * d_stride, width); + ds[3] = load_unaligned_s16_4x2(d_t + 3 * d_stride, width); - step3_win5_oneline_neon(&d_t, d_stride, width, height, ds, deltas); - transpose_arrays_s32_8x8(deltas, deltas_tr); + step3_win5_neon(d_t + 4 * d_stride, d_stride, width, height, ds, deltas); - update_5_stats_neon(H + 0 * wiener_win * wiener_win2 + 0 * wiener_win, - deltas_tr[0], vgetq_lane_s32(deltas_tr[4], 0), - H + 1 * wiener_win * wiener_win2 + 1 * wiener_win); + transpose_elems_inplace_s32_4x4(&deltas[0], &deltas[1], &deltas[2], + &deltas[3]); - update_5_stats_neon(H + 1 * wiener_win * wiener_win2 + 1 * wiener_win, - deltas_tr[1], vgetq_lane_s32(deltas_tr[5], 0), - H + 2 * wiener_win * wiener_win2 + 2 * wiener_win); + update_5_stats_neon(H + 0 * wiener_win * wiener_win2 + 0 * wiener_win, + deltas[0], vgetq_lane_s32(deltas[4], 0), + H + 1 * wiener_win * wiener_win2 + 1 * wiener_win); - update_5_stats_neon(H + 2 * wiener_win * wiener_win2 + 2 * wiener_win, - deltas_tr[2], vgetq_lane_s32(deltas_tr[6], 0), - H + 3 * wiener_win * wiener_win2 + 3 * wiener_win); + update_5_stats_neon(H + 1 * wiener_win * wiener_win2 + 1 * wiener_win, + deltas[1], vgetq_lane_s32(deltas[4], 1), + H + 2 * wiener_win * wiener_win2 + 2 * wiener_win); - update_5_stats_neon(H + 3 * wiener_win * wiener_win2 + 3 * wiener_win, - deltas_tr[3], vgetq_lane_s32(deltas_tr[7], 0), - H + 4 * wiener_win * wiener_win2 + 4 * wiener_win); + update_5_stats_neon(H + 2 * wiener_win * wiener_win2 + 2 * wiener_win, + deltas[2], vgetq_lane_s32(deltas[4], 2), + H + 3 * wiener_win * wiener_win2 + 3 * wiener_win); - } else { - int32x4_t deltas[WIENER_WIN_CHROMA * 2] = { vdupq_n_s32(0) }; - int16x8_t ds[WIENER_WIN_CHROMA * 2]; - - ds[0] = load_unaligned_s16_4x2(d_t + 0 * d_stride, width); - ds[1] = load_unaligned_s16_4x2(d_t + 1 * d_stride, width); - ds[2] = load_unaligned_s16_4x2(d_t + 2 * d_stride, width); - ds[3] = load_unaligned_s16_4x2(d_t + 3 * d_stride, width); - - step3_win5_neon(d_t + 4 * d_stride, d_stride, width, height, ds, deltas); - - transpose_elems_inplace_s32_4x4(&deltas[0], &deltas[1], &deltas[2], - &deltas[3]); - - update_5_stats_neon(H + 0 * wiener_win * wiener_win2 + 0 * wiener_win, - deltas[0], vgetq_lane_s32(deltas[4], 0), - H + 1 * wiener_win * wiener_win2 + 1 * wiener_win); - - update_5_stats_neon(H + 1 * wiener_win * wiener_win2 + 1 * wiener_win, - deltas[1], vgetq_lane_s32(deltas[4], 1), - H + 2 * wiener_win * wiener_win2 + 2 * wiener_win); - - update_5_stats_neon(H + 2 * wiener_win * wiener_win2 + 2 * wiener_win, - deltas[2], vgetq_lane_s32(deltas[4], 2), - H + 3 * wiener_win * wiener_win2 + 3 * wiener_win); - - update_5_stats_neon(H + 3 * wiener_win * wiener_win2 + 3 * wiener_win, - deltas[3], vgetq_lane_s32(deltas[4], 3), - H + 4 * wiener_win * wiener_win2 + 4 * wiener_win); - } + update_5_stats_neon(H + 3 * wiener_win * wiener_win2 + 3 * wiener_win, + deltas[3], vgetq_lane_s32(deltas[4], 3), + H + 4 * wiener_win * wiener_win2 + 4 * wiener_win); } // Step 4: Derive the top and left edge of each square. No square in top and
diff --git a/av1/encoder/arm/pickrst_neon.c b/av1/encoder/arm/pickrst_neon.c index f83e693..70ee3b6 100644 --- a/av1/encoder/arm/pickrst_neon.c +++ b/av1/encoder/arm/pickrst_neon.c
@@ -1027,64 +1027,34 @@ { const int16_t *d_t = d; - if (height % 2) { - int32x4_t deltas[(WIENER_WIN + 1) * 2] = { vdupq_n_s32(0) }; - int32x4_t deltas_tr[(WIENER_WIN + 1) * 2] = { vdupq_n_s32(0) }; - int16x8_t ds[WIENER_WIN * 2]; + int32x4_t deltas[WIENER_WIN_CHROMA] = { vdupq_n_s32(0) }; + int16x8_t ds[WIENER_WIN_CHROMA + 1]; - load_s16_8x4(d_t, d_stride, &ds[0], &ds[2], &ds[4], &ds[6]); - load_s16_8x4(d_t + width, d_stride, &ds[1], &ds[3], &ds[5], &ds[7]); - d_t += 4 * d_stride; + ds[0] = load_unaligned_s16_4x2(d_t + 0 * d_stride, width); + ds[1] = load_unaligned_s16_4x2(d_t + 1 * d_stride, width); + ds[2] = load_unaligned_s16_4x2(d_t + 2 * d_stride, width); + ds[3] = load_unaligned_s16_4x2(d_t + 3 * d_stride, width); - step3_win5_oneline_neon(&d_t, d_stride, width, height, ds, deltas); - transpose_arrays_s32_8x8(deltas, deltas_tr); + step3_win5_neon(d_t + 4 * d_stride, d_stride, width, height, ds, deltas); - update_5_stats_neon(H + 0 * wiener_win * wiener_win2 + 0 * wiener_win, - deltas_tr[0], vgetq_lane_s32(deltas_tr[4], 0), - H + 1 * wiener_win * wiener_win2 + 1 * wiener_win); + transpose_elems_inplace_s32_4x4(&deltas[0], &deltas[1], &deltas[2], + &deltas[3]); - update_5_stats_neon(H + 1 * wiener_win * wiener_win2 + 1 * wiener_win, - deltas_tr[1], vgetq_lane_s32(deltas_tr[5], 0), - H + 2 * wiener_win * wiener_win2 + 2 * wiener_win); + update_5_stats_neon(H + 0 * wiener_win * wiener_win2 + 0 * wiener_win, + deltas[0], vgetq_lane_s32(deltas[4], 0), + H + 1 * wiener_win * wiener_win2 + 1 * wiener_win); - update_5_stats_neon(H + 2 * wiener_win * wiener_win2 + 2 * wiener_win, - deltas_tr[2], vgetq_lane_s32(deltas_tr[6], 0), - H + 3 * wiener_win * wiener_win2 + 3 * wiener_win); + update_5_stats_neon(H + 1 * wiener_win * wiener_win2 + 1 * wiener_win, + deltas[1], vgetq_lane_s32(deltas[4], 1), + H + 2 * wiener_win * wiener_win2 + 2 * wiener_win); - update_5_stats_neon(H + 3 * wiener_win * wiener_win2 + 3 * wiener_win, - deltas_tr[3], vgetq_lane_s32(deltas_tr[7], 0), - H + 4 * wiener_win * wiener_win2 + 4 * wiener_win); + update_5_stats_neon(H + 2 * wiener_win * wiener_win2 + 2 * wiener_win, + deltas[2], vgetq_lane_s32(deltas[4], 2), + H + 3 * wiener_win * wiener_win2 + 3 * wiener_win); - } else { - int32x4_t deltas[WIENER_WIN_CHROMA * 2] = { vdupq_n_s32(0) }; - int16x8_t ds[WIENER_WIN_CHROMA * 2]; - - ds[0] = load_unaligned_s16_4x2(d_t + 0 * d_stride, width); - ds[1] = load_unaligned_s16_4x2(d_t + 1 * d_stride, width); - ds[2] = load_unaligned_s16_4x2(d_t + 2 * d_stride, width); - ds[3] = load_unaligned_s16_4x2(d_t + 3 * d_stride, width); - - step3_win5_neon(d_t + 4 * d_stride, d_stride, width, height, ds, deltas); - - transpose_elems_inplace_s32_4x4(&deltas[0], &deltas[1], &deltas[2], - &deltas[3]); - - update_5_stats_neon(H + 0 * wiener_win * wiener_win2 + 0 * wiener_win, - deltas[0], vgetq_lane_s32(deltas[4], 0), - H + 1 * wiener_win * wiener_win2 + 1 * wiener_win); - - update_5_stats_neon(H + 1 * wiener_win * wiener_win2 + 1 * wiener_win, - deltas[1], vgetq_lane_s32(deltas[4], 1), - H + 2 * wiener_win * wiener_win2 + 2 * wiener_win); - - update_5_stats_neon(H + 2 * wiener_win * wiener_win2 + 2 * wiener_win, - deltas[2], vgetq_lane_s32(deltas[4], 2), - H + 3 * wiener_win * wiener_win2 + 3 * wiener_win); - - update_5_stats_neon(H + 3 * wiener_win * wiener_win2 + 3 * wiener_win, - deltas[3], vgetq_lane_s32(deltas[4], 3), - H + 4 * wiener_win * wiener_win2 + 4 * wiener_win); - } + update_5_stats_neon(H + 3 * wiener_win * wiener_win2 + 3 * wiener_win, + deltas[3], vgetq_lane_s32(deltas[4], 3), + H + 4 * wiener_win * wiener_win2 + 4 * wiener_win); } // Step 4: Derive the top and left edge of each square. No square in top and
diff --git a/av1/encoder/arm/pickrst_neon.h b/av1/encoder/arm/pickrst_neon.h index 356c42d..2eeece8 100644 --- a/av1/encoder/arm/pickrst_neon.h +++ b/av1/encoder/arm/pickrst_neon.h
@@ -422,9 +422,8 @@ dst[4] = src[4] + delta4; } -static inline void compute_delta_step3_two_lines(int32x4_t *sum, - const int16x8_t src, - const int16x8_t dgd) { +static inline void compute_delta_step3_win5(int32x4_t *sum, const int16x8_t src, + const int16x8_t dgd) { *sum = vmlsl_s16(*sum, vget_low_s16(src), vget_low_s16(dgd)); *sum = vmlal_s16(*sum, vget_high_s16(src), vget_high_s16(dgd)); } @@ -437,16 +436,16 @@ ds[4] = load_unaligned_s16_4x2(d + 0 * d_stride, width); ds[5] = load_unaligned_s16_4x2(d + 1 * d_stride, width); - compute_delta_step3_two_lines(&deltas[0], ds[0], ds[0]); - compute_delta_step3_two_lines(&deltas[1], ds[0], ds[1]); - compute_delta_step3_two_lines(&deltas[2], ds[0], ds[2]); - compute_delta_step3_two_lines(&deltas[3], ds[0], ds[3]); - compute_delta_step3_two_lines(&deltas[4], ds[0], ds[4]); - compute_delta_step3_two_lines(&deltas[0], ds[1], ds[1]); - compute_delta_step3_two_lines(&deltas[1], ds[1], ds[2]); - compute_delta_step3_two_lines(&deltas[2], ds[1], ds[3]); - compute_delta_step3_two_lines(&deltas[3], ds[1], ds[4]); - compute_delta_step3_two_lines(&deltas[4], ds[1], ds[5]); + compute_delta_step3_win5(&deltas[0], ds[0], ds[0]); + compute_delta_step3_win5(&deltas[1], ds[0], ds[1]); + compute_delta_step3_win5(&deltas[2], ds[0], ds[2]); + compute_delta_step3_win5(&deltas[3], ds[0], ds[3]); + compute_delta_step3_win5(&deltas[4], ds[0], ds[4]); + compute_delta_step3_win5(&deltas[0], ds[1], ds[1]); + compute_delta_step3_win5(&deltas[1], ds[1], ds[2]); + compute_delta_step3_win5(&deltas[2], ds[1], ds[3]); + compute_delta_step3_win5(&deltas[3], ds[1], ds[4]); + compute_delta_step3_win5(&deltas[4], ds[1], ds[5]); ds[0] = ds[2]; ds[1] = ds[3]; @@ -455,36 +454,17 @@ d += 2 * d_stride; y -= 2; - } while (y); -} + } while (y > 1); -static inline void step3_win5_oneline_neon(const int16_t **const d, - const int32_t d_stride, - const int32_t width, - const int32_t height, int16x8_t *ds, - int32x4_t *deltas) { - int32_t y = height; - do { - ds[8] = vld1q_s16(*d); - ds[9] = vld1q_s16(*d + width); + if (y) { + ds[4] = load_unaligned_s16_4x2(d + 0 * d_stride, width); - compute_delta_step3(&deltas[0], &deltas[4], ds[0], ds[1], ds[0], ds[1]); - compute_delta_step3(&deltas[1], &deltas[5], ds[0], ds[1], ds[2], ds[3]); - compute_delta_step3(&deltas[2], &deltas[6], ds[0], ds[1], ds[4], ds[5]); - compute_delta_step3(&deltas[3], &deltas[7], ds[0], ds[1], ds[6], ds[7]); - compute_delta_step3(&deltas[8], &deltas[12], ds[0], ds[1], ds[8], ds[9]); - - ds[0] = ds[2]; - ds[1] = ds[3]; - ds[2] = ds[4]; - ds[3] = ds[5]; - ds[4] = ds[6]; - ds[5] = ds[7]; - ds[6] = ds[8]; - ds[7] = ds[9]; - - *d += d_stride; - } while (--y); + compute_delta_step3_win5(&deltas[0], ds[0], ds[0]); + compute_delta_step3_win5(&deltas[1], ds[0], ds[1]); + compute_delta_step3_win5(&deltas[2], ds[0], ds[2]); + compute_delta_step3_win5(&deltas[3], ds[0], ds[3]); + compute_delta_step3_win5(&deltas[4], ds[0], ds[4]); + } } static inline void derive_triangle_win5_neon(const int16x8_t *d_is,
diff --git a/av1/encoder/arm/pickrst_sve.h b/av1/encoder/arm/pickrst_sve.h index 94c0375..ad03680 100644 --- a/av1/encoder/arm/pickrst_sve.h +++ b/av1/encoder/arm/pickrst_sve.h
@@ -21,12 +21,12 @@ // Swap each half of the dgd vectors so that we can accumulate the result of // the dot-products directly in the destination matrix. static inline int16x8x2_t transpose_dgd(int16x8_t dgd0, int16x8_t dgd1) { - int16x8_t dgd_trn0 = vreinterpretq_s16_s64( + int16x8x2_t dgd_trn; + dgd_trn.val[0] = vreinterpretq_s16_s64( vzip1q_s64(vreinterpretq_s64_s16(dgd0), vreinterpretq_s64_s16(dgd1))); - int16x8_t dgd_trn1 = vreinterpretq_s16_s64( + dgd_trn.val[1] = vreinterpretq_s16_s64( vzip2q_s64(vreinterpretq_s64_s16(dgd0), vreinterpretq_s64_s16(dgd1))); - - return (struct int16x8x2_t){ dgd_trn0, dgd_trn1 }; + return dgd_trn; } static inline void compute_M_one_row_win5(int16x8_t src, int16x8_t dgd[5], @@ -527,64 +527,34 @@ { const int16_t *d_t = d; - if (height % 2) { - int32x4_t deltas[(WIENER_WIN + 1) * 2] = { vdupq_n_s32(0) }; - int32x4_t deltas_tr[(WIENER_WIN + 1) * 2] = { vdupq_n_s32(0) }; - int16x8_t ds[WIENER_WIN * 2]; + int32x4_t deltas[WIENER_WIN_CHROMA] = { vdupq_n_s32(0) }; + int16x8_t ds[WIENER_WIN_CHROMA + 1]; - load_s16_8x4(d_t, d_stride, &ds[0], &ds[2], &ds[4], &ds[6]); - load_s16_8x4(d_t + width, d_stride, &ds[1], &ds[3], &ds[5], &ds[7]); - d_t += 4 * d_stride; + ds[0] = load_unaligned_s16_4x2(d_t + 0 * d_stride, width); + ds[1] = load_unaligned_s16_4x2(d_t + 1 * d_stride, width); + ds[2] = load_unaligned_s16_4x2(d_t + 2 * d_stride, width); + ds[3] = load_unaligned_s16_4x2(d_t + 3 * d_stride, width); - step3_win5_oneline_neon(&d_t, d_stride, width, height, ds, deltas); - transpose_arrays_s32_8x8(deltas, deltas_tr); + step3_win5_neon(d_t + 4 * d_stride, d_stride, width, height, ds, deltas); - update_5_stats_neon(H + 0 * wiener_win * wiener_win2 + 0 * wiener_win, - deltas_tr[0], vgetq_lane_s32(deltas_tr[4], 0), - H + 1 * wiener_win * wiener_win2 + 1 * wiener_win); + transpose_elems_inplace_s32_4x4(&deltas[0], &deltas[1], &deltas[2], + &deltas[3]); - update_5_stats_neon(H + 1 * wiener_win * wiener_win2 + 1 * wiener_win, - deltas_tr[1], vgetq_lane_s32(deltas_tr[5], 0), - H + 2 * wiener_win * wiener_win2 + 2 * wiener_win); + update_5_stats_neon(H + 0 * wiener_win * wiener_win2 + 0 * wiener_win, + deltas[0], vgetq_lane_s32(deltas[4], 0), + H + 1 * wiener_win * wiener_win2 + 1 * wiener_win); - update_5_stats_neon(H + 2 * wiener_win * wiener_win2 + 2 * wiener_win, - deltas_tr[2], vgetq_lane_s32(deltas_tr[6], 0), - H + 3 * wiener_win * wiener_win2 + 3 * wiener_win); + update_5_stats_neon(H + 1 * wiener_win * wiener_win2 + 1 * wiener_win, + deltas[1], vgetq_lane_s32(deltas[4], 1), + H + 2 * wiener_win * wiener_win2 + 2 * wiener_win); - update_5_stats_neon(H + 3 * wiener_win * wiener_win2 + 3 * wiener_win, - deltas_tr[3], vgetq_lane_s32(deltas_tr[7], 0), - H + 4 * wiener_win * wiener_win2 + 4 * wiener_win); + update_5_stats_neon(H + 2 * wiener_win * wiener_win2 + 2 * wiener_win, + deltas[2], vgetq_lane_s32(deltas[4], 2), + H + 3 * wiener_win * wiener_win2 + 3 * wiener_win); - } else { - int32x4_t deltas[WIENER_WIN_CHROMA * 2] = { vdupq_n_s32(0) }; - int16x8_t ds[WIENER_WIN_CHROMA * 2]; - - ds[0] = load_unaligned_s16_4x2(d_t + 0 * d_stride, width); - ds[1] = load_unaligned_s16_4x2(d_t + 1 * d_stride, width); - ds[2] = load_unaligned_s16_4x2(d_t + 2 * d_stride, width); - ds[3] = load_unaligned_s16_4x2(d_t + 3 * d_stride, width); - - step3_win5_neon(d_t + 4 * d_stride, d_stride, width, height, ds, deltas); - - transpose_elems_inplace_s32_4x4(&deltas[0], &deltas[1], &deltas[2], - &deltas[3]); - - update_5_stats_neon(H + 0 * wiener_win * wiener_win2 + 0 * wiener_win, - deltas[0], vgetq_lane_s32(deltas[4], 0), - H + 1 * wiener_win * wiener_win2 + 1 * wiener_win); - - update_5_stats_neon(H + 1 * wiener_win * wiener_win2 + 1 * wiener_win, - deltas[1], vgetq_lane_s32(deltas[4], 1), - H + 2 * wiener_win * wiener_win2 + 2 * wiener_win); - - update_5_stats_neon(H + 2 * wiener_win * wiener_win2 + 2 * wiener_win, - deltas[2], vgetq_lane_s32(deltas[4], 2), - H + 3 * wiener_win * wiener_win2 + 3 * wiener_win); - - update_5_stats_neon(H + 3 * wiener_win * wiener_win2 + 3 * wiener_win, - deltas[3], vgetq_lane_s32(deltas[4], 3), - H + 4 * wiener_win * wiener_win2 + 4 * wiener_win); - } + update_5_stats_neon(H + 3 * wiener_win * wiener_win2 + 3 * wiener_win, + deltas[3], vgetq_lane_s32(deltas[4], 3), + H + 4 * wiener_win * wiener_win2 + 4 * wiener_win); } // Step 4: Derive the top and left edge of each square. No square in top and
diff --git a/av1/encoder/arm/quantize_neon.c b/av1/encoder/arm/quantize_neon.c index cbeafc7..e9c0ca7 100644 --- a/av1/encoder/arm/quantize_neon.c +++ b/av1/encoder/arm/quantize_neon.c
@@ -29,7 +29,8 @@ static inline uint16_t get_max_eob(int16x8_t v_eobmax) { #if AOM_ARCH_AARCH64 - return (uint16_t)vmaxvq_s16(v_eobmax); + int16_t max_val = vmaxvq_s16(v_eobmax); + return (uint16_t)max_val + 1; #else const int16x4_t v_eobmax_3210 = vmax_s16(vget_low_s16(v_eobmax), vget_high_s16(v_eobmax)); @@ -41,16 +42,15 @@ vshr_n_s64(vreinterpret_s64_s16(v_eobmax_tmp), 16); const int16x4_t v_eobmax_final = vmax_s16(v_eobmax_tmp, vreinterpret_s16_s64(v_eobmax_xxx3)); - return (uint16_t)vget_lane_s16(v_eobmax_final, 0); + return (uint16_t)vget_lane_s16(v_eobmax_final, 0) + 1; #endif } static inline int16x8_t get_max_lane_eob(const int16_t *iscan, int16x8_t v_eobmax, uint16x8_t v_mask) { - const int16x8_t v_iscan = vld1q_s16(&iscan[0]); - const int16x8_t v_iscan_plus1 = vaddq_s16(v_iscan, vdupq_n_s16(1)); - const int16x8_t v_nz_iscan = vbslq_s16(v_mask, v_iscan_plus1, vdupq_n_s16(0)); + const int16x8_t v_iscan = vld1q_s16(iscan); + const int16x8_t v_nz_iscan = vbslq_s16(v_mask, v_iscan, vdupq_n_s16(-1)); return vmaxq_s16(v_eobmax, v_nz_iscan); } @@ -351,6 +351,33 @@ iscan, 2); } +static inline uint16x8_t quantize_b_logscale0_8( + int16x8_t coeff, int16x8_t abs, uint16x8_t cond, int16x8_t round, + int16x8_t dequant, int16x8_t quant, int16x8_t quant_shift, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr) { + const int16x8_t zero = vdupq_n_s16(0); + + int16x8_t coeff_sign = vreinterpretq_s16_u16(vcltq_s16(coeff, zero)); + + int16x8_t tmp = vqaddq_s16(abs, round); + tmp = vsraq_n_s16(tmp, vqdmulhq_s16(tmp, quant), 1); + tmp = vqdmulhq_s16(tmp, quant_shift); + + int16x8_t qcoeff = vsubq_s16(veorq_s16(tmp, coeff_sign), coeff_sign); + qcoeff = vbslq_s16(cond, qcoeff, zero); + store_s16q_to_tran_low(qcoeff_ptr, qcoeff); + + int16x8_t dqcoeff = vmulq_s16(tmp, dequant); + dqcoeff = vsubq_s16(veorq_s16(dqcoeff, coeff_sign), coeff_sign); + dqcoeff = vbslq_s16(cond, dqcoeff, zero); + store_s16q_to_tran_low(dqcoeff_ptr, dqcoeff); + + uint16x8_t tmp_mask = vcgtq_s16(tmp, zero); + uint16x8_t nz_mask = vandq_u16(tmp_mask, cond); + + return nz_mask; +} + void aom_quantize_b_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, @@ -361,91 +388,61 @@ (void)quant_shift_ptr; (void)scan; - const int zbins[2] = { zbin_ptr[0], zbin_ptr[1] }; + int16x8_t v_eobmax_76543210 = vdupq_n_s16(-1); - memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); - memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + int16x8_t v_zbins = vdupq_n_s16(zbin_ptr[1]); + int16x8_t v_round = vdupq_n_s16(round_ptr[1]); + int16x8_t v_dequant = vdupq_n_s16(dequant_ptr[1]); + int16x8_t v_quant = vdupq_n_s16(quant_ptr[1]); + // Shift by 1 in order to save one shift in the kernel function. + int16x8_t v_quant_shift = vdupq_n_s16(quant_shift_ptr[1] >> 1); - const int16x8_t zero = vdupq_n_s16(0); - int16x8_t v_eobmax_76543210 = vreinterpretq_s16_u16(vceqq_s16(zero, zero)); - - int16x8_t vzbins = vdupq_n_s16(zbins[1]), vround = vdupq_n_s16(round_ptr[1]); - int16x8_t vdequant = vdupq_n_s16(dequant_ptr[1]); - int16x8_t vquant = vdupq_n_s16(quant_ptr[1]); - int16x8_t vquant_shift = vdupq_n_s16(quant_shift_ptr[1]); - - int16x8_t v_coeff = load_tran_low_to_s16q(&coeff_ptr[0]); - int16x8_t v_coeff_sign = vshrq_n_s16(v_coeff, 15); + int16x8_t v_zbins0 = vsetq_lane_s16(zbin_ptr[0], v_zbins, 0); + int16x8_t v_coeff = load_tran_low_to_s16q(coeff_ptr); int16x8_t v_abs = vabsq_s16(v_coeff); + uint16x8_t v_cond = vcgeq_s16(v_abs, v_zbins0); - vzbins = vsetq_lane_s16(zbins[0], vzbins, 0); - - uint16x8_t vcond = vcgeq_s16(v_abs, vzbins); - uint64_t nz_check = vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(vcond)), 0); + uint64_t nz_check = vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(v_cond)), 0); if (nz_check) { - vround = vsetq_lane_s16(round_ptr[0], vround, 0); - vquant = vsetq_lane_s16(quant_ptr[0], vquant, 0); - vdequant = vsetq_lane_s16(dequant_ptr[0], vdequant, 0); - vquant_shift = vsetq_lane_s16(quant_shift_ptr[0], vquant_shift, 0); + int16x8_t v_round0 = vsetq_lane_s16(round_ptr[0], v_round, 0); + int16x8_t v_quant0 = vsetq_lane_s16(quant_ptr[0], v_quant, 0); + int16x8_t v_dequant0 = vsetq_lane_s16(dequant_ptr[0], v_dequant, 0); + // Shift by 1 in order to save one shift in the kernel function. + int16x8_t v_quant_shift0 = + vsetq_lane_s16(quant_shift_ptr[0] >> 1, v_quant_shift, 0); - int16x8_t vtmp = vqaddq_s16(v_abs, vround); - int16x8_t vtmp2 = vsraq_n_s16(vtmp, vqdmulhq_s16(vtmp, vquant), 1); - vtmp2 = vshrq_n_s16(vqdmulhq_s16(vtmp2, vquant_shift), 1); + const uint16x8_t v_nz_mask = quantize_b_logscale0_8( + v_coeff, v_abs, v_cond, v_round0, v_dequant0, v_quant0, v_quant_shift0, + qcoeff_ptr, dqcoeff_ptr); - int16x8_t vdest = vsubq_s16(veorq_s16(vtmp2, v_coeff_sign), v_coeff_sign); - int16x8_t coeff_nz_mask = - vbslq_s16(vcond, vdest, load_tran_low_to_s16q(&qcoeff_ptr[0])); - store_s16q_to_tran_low(&qcoeff_ptr[0], coeff_nz_mask); - int16x8_t v_deq_abs = vmulq_s16(vtmp2, vdequant); - - vdest = vsubq_s16(veorq_s16(v_deq_abs, v_coeff_sign), v_coeff_sign); - coeff_nz_mask = - vbslq_s16(vcond, vdest, load_tran_low_to_s16q(&dqcoeff_ptr[0])); - store_s16q_to_tran_low(&dqcoeff_ptr[0], coeff_nz_mask); - - vround = vsetq_lane_s16(round_ptr[1], vround, 0); - vquant = vsetq_lane_s16(quant_ptr[1], vquant, 0); - vdequant = vsetq_lane_s16(dequant_ptr[1], vdequant, 0); - vquant_shift = vsetq_lane_s16(quant_shift_ptr[1], vquant_shift, 0); - - uint16x8_t vtmp_mask = vcgtq_s16(vtmp2, zero); - const uint16x8_t v_nz_mask = vandq_u16(vtmp_mask, vcond); - int16x8_t v_iscan = vld1q_s16(&iscan[0]); - vcond = vandq_u16(v_nz_mask, vcgtq_s16(v_iscan, v_eobmax_76543210)); - v_eobmax_76543210 = vbslq_s16(vcond, v_iscan, v_eobmax_76543210); + int16x8_t v_iscan = vld1q_s16(iscan); + int16x8_t v_eobmax = vmaxq_s16(v_iscan, v_eobmax_76543210); + v_eobmax_76543210 = vbslq_s16(v_nz_mask, v_eobmax, v_eobmax_76543210); + } else { + store_s16q_to_tran_low(qcoeff_ptr, vdupq_n_s16(0)); + store_s16q_to_tran_low(dqcoeff_ptr, vdupq_n_s16(0)); } - vzbins = vsetq_lane_s16(zbins[1], vzbins, 0); for (int i = 8; i < n_coeffs; i += 8) { - v_coeff = load_tran_low_to_s16q(&coeff_ptr[i]); - v_coeff_sign = vshrq_n_s16(v_coeff, 15); + v_coeff = load_tran_low_to_s16q(coeff_ptr + i); v_abs = vabsq_s16(v_coeff); - vcond = vcgeq_s16(v_abs, vzbins); + v_cond = vcgeq_s16(v_abs, v_zbins); - nz_check = vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(vcond)), 0); + nz_check = vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(v_cond)), 0); if (nz_check) { - int16x8_t vtmp = vqaddq_s16(v_abs, vround); - int16x8_t vtmp2 = vsraq_n_s16(vtmp, vqdmulhq_s16(vtmp, vquant), 1); + const uint16x8_t v_nz_mask = quantize_b_logscale0_8( + v_coeff, v_abs, v_cond, v_round, v_dequant, v_quant, v_quant_shift, + qcoeff_ptr + i, dqcoeff_ptr + i); - vtmp2 = vshrq_n_s16(vqdmulhq_s16(vtmp2, vquant_shift), 1); - int16x8_t vdest = vsubq_s16(veorq_s16(vtmp2, v_coeff_sign), v_coeff_sign); - int16x8_t coeff_nz_mask = - vbslq_s16(vcond, vdest, load_tran_low_to_s16q(&qcoeff_ptr[i])); - store_s16q_to_tran_low(&qcoeff_ptr[i], coeff_nz_mask); - int16x8_t v_deq_abs = vmulq_s16(vtmp2, vdequant); - vdest = vsubq_s16(veorq_s16(v_deq_abs, v_coeff_sign), v_coeff_sign); - coeff_nz_mask = - vbslq_s16(vcond, vdest, load_tran_low_to_s16q(&dqcoeff_ptr[i])); - store_s16q_to_tran_low(&dqcoeff_ptr[i], coeff_nz_mask); - - uint16x8_t vtmp_mask = vcgtq_s16(vtmp2, zero); - const uint16x8_t v_nz_mask = vandq_u16(vtmp_mask, vcond); - int16x8_t v_iscan = vld1q_s16(&iscan[i]); - vcond = vandq_u16(v_nz_mask, vcgtq_s16(v_iscan, v_eobmax_76543210)); - v_eobmax_76543210 = vbslq_s16(vcond, v_iscan, v_eobmax_76543210); + int16x8_t v_iscan = vld1q_s16(iscan + i); + int16x8_t v_eobmax = vmaxq_s16(v_iscan, v_eobmax_76543210); + v_eobmax_76543210 = vbslq_s16(v_nz_mask, v_eobmax, v_eobmax_76543210); + } else { + store_s16q_to_tran_low(qcoeff_ptr + i, vdupq_n_s16(0)); + store_s16q_to_tran_low(dqcoeff_ptr + i, vdupq_n_s16(0)); } } - *eob_ptr = get_max_eob(v_eobmax_76543210) + 1; + *eob_ptr = get_max_eob(v_eobmax_76543210); } #define QM_MULL_SHIFT(x0, x1) \ @@ -580,7 +577,7 @@ v_eobmax_76543210 = vbslq_s16(vcond, v_iscan, v_eobmax_76543210); } } - *eob_ptr = get_max_eob(v_eobmax_76543210) + 1; + *eob_ptr = get_max_eob(v_eobmax_76543210); } static void aom_quantize_b_helper_32x32_neon( @@ -718,7 +715,7 @@ v_eobmax_76543210 = vbslq_s16(vcond, v_iscan, v_eobmax_76543210); } } - *eob_ptr = get_max_eob(v_eobmax_76543210) + 1; + *eob_ptr = get_max_eob(v_eobmax_76543210); } static void aom_quantize_b_helper_64x64_neon( @@ -867,7 +864,7 @@ v_eobmax_76543210 = vbslq_s16(vcond, v_iscan, v_eobmax_76543210); } } - *eob_ptr = get_max_eob(v_eobmax_76543210) + 1; + *eob_ptr = get_max_eob(v_eobmax_76543210); } void aom_quantize_b_helper_neon( @@ -899,6 +896,35 @@ } } +static inline uint16x8_t quantize_b_logscale1_8( + int16x8_t coeff, int16x8_t abs, uint16x8_t cond, int16x8_t round, + int16x8_t dequant, int16x8_t quant, int16x8_t quant_shift, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr) { + const int16x8_t zero = vdupq_n_s16(0); + + int16x8_t coeff_sign = vreinterpretq_s16_u16(vcltq_s16(coeff, zero)); + + int16x8_t tmp = vqaddq_s16(abs, round); + tmp = vsraq_n_s16(tmp, vqdmulhq_s16(tmp, quant), 1); + tmp = vqdmulhq_s16(tmp, quant_shift); + + int16x8_t qcoeff = vsubq_s16(veorq_s16(tmp, coeff_sign), coeff_sign); + qcoeff = vbslq_s16(cond, qcoeff, zero); + store_s16q_to_tran_low(qcoeff_ptr, qcoeff); + + // Shift by log_scale = 1. + int16x8_t dqcoeff = vreinterpretq_s16_u16(vhaddq_u16( + vreinterpretq_u16_s16(vmulq_s16(tmp, dequant)), vdupq_n_u16(0))); + dqcoeff = vsubq_s16(veorq_s16(dqcoeff, coeff_sign), coeff_sign); + dqcoeff = vbslq_s16(cond, dqcoeff, zero); + store_s16q_to_tran_low(dqcoeff_ptr, dqcoeff); + + uint16x8_t tmp_mask = vcgtq_s16(tmp, zero); + const uint16x8_t nz_mask = vandq_u16(tmp_mask, cond); + + return nz_mask; +} + void aom_quantize_b_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, @@ -907,10 +933,100 @@ tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan) { - aom_quantize_b_helper_neon(coeff_ptr, n_coeffs, zbin_ptr, round_ptr, - quant_ptr, quant_shift_ptr, qcoeff_ptr, - dqcoeff_ptr, dequant_ptr, eob_ptr, scan, iscan, - NULL, NULL, 1); + (void)scan; + + const int log_scale = 1; + const int zbins[2] = { ROUND_POWER_OF_TWO(zbin_ptr[0], log_scale), + ROUND_POWER_OF_TWO(zbin_ptr[1], log_scale) }; + const int rounds[2] = { ROUND_POWER_OF_TWO(round_ptr[0], log_scale), + ROUND_POWER_OF_TWO(round_ptr[1], log_scale) }; + + int16x8_t v_eobmax_76543210 = vdupq_n_s16(-1); + + int16x8_t v_zbins = vdupq_n_s16(zbins[1]); + int16x8_t v_round = vdupq_n_s16(rounds[1]); + int16x8_t v_dequant = vdupq_n_s16(dequant_ptr[1]); + int16x8_t v_quant = vdupq_n_s16(quant_ptr[1]); + int16x8_t v_quant_shift = vdupq_n_s16(quant_shift_ptr[1]); + + int16x8_t v_zbins0 = vsetq_lane_s16(zbins[0], v_zbins, 0); + int16x8_t v_coeff = load_tran_low_to_s16q(coeff_ptr); + int16x8_t v_abs = vabsq_s16(v_coeff); + uint16x8_t v_cond = vcgeq_s16(v_abs, v_zbins0); + + uint64_t nz_check = vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(v_cond)), 0); + if (nz_check) { + int16x8_t v_round0 = vsetq_lane_s16(rounds[0], v_round, 0); + int16x8_t v_quant0 = vsetq_lane_s16(quant_ptr[0], v_quant, 0); + int16x8_t v_dequant0 = vsetq_lane_s16(dequant_ptr[0], v_dequant, 0); + int16x8_t v_quant_shift0 = + vsetq_lane_s16(quant_shift_ptr[0], v_quant_shift, 0); + + const uint16x8_t v_nz_mask = quantize_b_logscale1_8( + v_coeff, v_abs, v_cond, v_round0, v_dequant0, v_quant0, v_quant_shift0, + qcoeff_ptr, dqcoeff_ptr); + + int16x8_t v_iscan = vld1q_s16(iscan); + int16x8_t v_eobmax = vmaxq_s16(v_iscan, v_eobmax_76543210); + v_eobmax_76543210 = vbslq_s16(v_nz_mask, v_eobmax, v_eobmax_76543210); + } else { + store_s16q_to_tran_low(qcoeff_ptr, vdupq_n_s16(0)); + store_s16q_to_tran_low(dqcoeff_ptr, vdupq_n_s16(0)); + } + + for (int i = 8; i < n_coeffs; i += 8) { + v_coeff = load_tran_low_to_s16q(coeff_ptr + i); + v_abs = vabsq_s16(v_coeff); + v_cond = vcgeq_s16(v_abs, v_zbins); + + nz_check = vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(v_cond)), 0); + if (nz_check) { + const uint16x8_t v_nz_mask = quantize_b_logscale1_8( + v_coeff, v_abs, v_cond, v_round, v_dequant, v_quant, v_quant_shift, + qcoeff_ptr + i, dqcoeff_ptr + i); + + int16x8_t v_iscan = vld1q_s16(iscan + i); + int16x8_t v_eobmax = vmaxq_s16(v_iscan, v_eobmax_76543210); + v_eobmax_76543210 = vbslq_s16(v_nz_mask, v_eobmax, v_eobmax_76543210); + } else { + store_s16q_to_tran_low(qcoeff_ptr + i, vdupq_n_s16(0)); + store_s16q_to_tran_low(dqcoeff_ptr + i, vdupq_n_s16(0)); + } + } + *eob_ptr = get_max_eob(v_eobmax_76543210); +} + +static inline uint16x8_t quantize_b_logscale2_8( + int16x8_t coeff, int16x8_t abs, uint16x8_t cond, int16x8_t round, + int16x8_t dequant, int16x8_t quant, int16x8_t quant_shift, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr) { + const int16x8_t zero = vdupq_n_s16(0); + const int16x8_t one = vdupq_n_s16(1); + + int16x8_t coeff_sign = vreinterpretq_s16_u16(vcltq_s16(coeff, zero)); + + int16x8_t tmp = vqaddq_s16(abs, round); + tmp = vsraq_n_s16(tmp, vqdmulhq_s16(tmp, quant), 1); + int16x8_t ones = vandq_s16(vshrq_n_s16(vmulq_s16(tmp, quant_shift), 14), one); + tmp = vqdmulhq_s16(tmp, quant_shift); + tmp = vaddq_s16(vshlq_s16(tmp, one), ones); + + int16x8_t qcoeff = vsubq_s16(veorq_s16(tmp, coeff_sign), coeff_sign); + qcoeff = vbslq_s16(cond, qcoeff, zero); + store_s16q_to_tran_low(qcoeff_ptr, qcoeff); + + // Shift right by log_scale = 2. + int16x8_t dqcoeff = vreinterpretq_s16_u16( + vshrq_n_u16(vreinterpretq_u16_s16(vmulq_s16(tmp, dequant)), 2)); + dqcoeff = vorrq_s16(vshlq_n_s16(vqdmulhq_s16(tmp, dequant), 13), dqcoeff); + dqcoeff = vsubq_s16(veorq_s16(dqcoeff, coeff_sign), coeff_sign); + dqcoeff = vbslq_s16(cond, dqcoeff, zero); + store_s16q_to_tran_low(dqcoeff_ptr, dqcoeff); + + uint16x8_t tmp_mask = vcgtq_s16(tmp, zero); + const uint16x8_t nz_mask = vandq_u16(tmp_mask, cond); + + return nz_mask; } void aom_quantize_b_64x64_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, @@ -921,8 +1037,65 @@ tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan) { - aom_quantize_b_helper_neon(coeff_ptr, n_coeffs, zbin_ptr, round_ptr, - quant_ptr, quant_shift_ptr, qcoeff_ptr, - dqcoeff_ptr, dequant_ptr, eob_ptr, scan, iscan, - NULL, NULL, 2); + (void)scan; + + const int log_scale = 2; + const int zbins[2] = { ROUND_POWER_OF_TWO(zbin_ptr[0], log_scale), + ROUND_POWER_OF_TWO(zbin_ptr[1], log_scale) }; + const int rounds[2] = { ROUND_POWER_OF_TWO(round_ptr[0], log_scale), + ROUND_POWER_OF_TWO(round_ptr[1], log_scale) }; + + int16x8_t v_eobmax_76543210 = vdupq_n_s16(-1); + + int16x8_t v_zbins = vdupq_n_s16(zbins[1]); + int16x8_t v_round = vdupq_n_s16(rounds[1]); + int16x8_t v_dequant = vdupq_n_s16(dequant_ptr[1]); + int16x8_t v_quant = vdupq_n_s16(quant_ptr[1]); + int16x8_t v_quant_shift = vdupq_n_s16(quant_shift_ptr[1]); + + int16x8_t v_zbins0 = vsetq_lane_s16(zbins[0], v_zbins, 0); + int16x8_t v_coeff = load_tran_low_to_s16q(coeff_ptr); + int16x8_t v_abs = vabsq_s16(v_coeff); + uint16x8_t v_cond = vcgeq_s16(v_abs, v_zbins0); + + uint64_t nz_check = vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(v_cond)), 0); + if (nz_check) { + int16x8_t v_round0 = vsetq_lane_s16(rounds[0], v_round, 0); + int16x8_t v_quant0 = vsetq_lane_s16(quant_ptr[0], v_quant, 0); + int16x8_t v_dequant0 = vsetq_lane_s16(dequant_ptr[0], v_dequant, 0); + int16x8_t v_quant_shift0 = + vsetq_lane_s16(quant_shift_ptr[0], v_quant_shift, 0); + + const uint16x8_t v_nz_mask = quantize_b_logscale2_8( + v_coeff, v_abs, v_cond, v_round0, v_dequant0, v_quant0, v_quant_shift0, + qcoeff_ptr, dqcoeff_ptr); + + int16x8_t v_iscan = vld1q_s16(iscan); + int16x8_t v_eobmax = vmaxq_s16(v_iscan, v_eobmax_76543210); + v_eobmax_76543210 = vbslq_s16(v_nz_mask, v_eobmax, v_eobmax_76543210); + } else { + store_s16q_to_tran_low(qcoeff_ptr, vdupq_n_s16(0)); + store_s16q_to_tran_low(dqcoeff_ptr, vdupq_n_s16(0)); + } + + for (int i = 8; i < n_coeffs; i += 8) { + v_coeff = load_tran_low_to_s16q(coeff_ptr + i); + v_abs = vabsq_s16(v_coeff); + v_cond = vcgeq_s16(v_abs, v_zbins); + + nz_check = vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(v_cond)), 0); + if (nz_check) { + const uint16x8_t v_nz_mask = quantize_b_logscale2_8( + v_coeff, v_abs, v_cond, v_round, v_dequant, v_quant, v_quant_shift, + qcoeff_ptr + i, dqcoeff_ptr + i); + + int16x8_t v_iscan = vld1q_s16(iscan + i); + int16x8_t v_eobmax = vmaxq_s16(v_iscan, v_eobmax_76543210); + v_eobmax_76543210 = vbslq_s16(v_nz_mask, v_eobmax, v_eobmax_76543210); + } else { + store_s16q_to_tran_low(qcoeff_ptr + i, vdupq_n_s16(0)); + store_s16q_to_tran_low(dqcoeff_ptr + i, vdupq_n_s16(0)); + } + } + *eob_ptr = get_max_eob(v_eobmax_76543210); }
diff --git a/av1/encoder/arm/temporal_filter_neon.c b/av1/encoder/arm/temporal_filter_neon.c index 132045f..3f62af6 100644 --- a/av1/encoder/arm/temporal_filter_neon.c +++ b/av1/encoder/arm/temporal_filter_neon.c
@@ -192,6 +192,8 @@ } } +// TODO: bug aomedia:493082083 - Modify this function to support TF_BLOCK_SIZE +// of 64x64. void av1_apply_temporal_filter_neon( const YV12_BUFFER_CONFIG *frame_to_filter, const MACROBLOCKD *mbd, const BLOCK_SIZE block_size, const int mb_row, const int mb_col, @@ -199,6 +201,13 @@ const int *subblock_mses, const int q_factor, const int filter_strength, int tf_wgt_calc_lvl, const uint8_t *pred, uint32_t *accum, uint16_t *count) { + if (block_size == BLOCK_64X64) { + av1_apply_temporal_filter_c(frame_to_filter, mbd, block_size, mb_row, + mb_col, num_planes, noise_levels, subblock_mvs, + subblock_mses, q_factor, filter_strength, + tf_wgt_calc_lvl, pred, accum, count); + return; + } const int is_high_bitdepth = frame_to_filter->flags & YV12_FLAG_HIGHBITDEPTH; assert(block_size == BLOCK_32X32 && "Only support 32x32 block with Neon!"); assert(TF_WINDOW_LENGTH == 5 && "Only support window length 5 with Neon!");
diff --git a/av1/encoder/arm/temporal_filter_neon_dotprod.c b/av1/encoder/arm/temporal_filter_neon_dotprod.c index c8fd699..fc6a125 100644 --- a/av1/encoder/arm/temporal_filter_neon_dotprod.c +++ b/av1/encoder/arm/temporal_filter_neon_dotprod.c
@@ -214,6 +214,8 @@ } } +// TODO: bug aomedia:493082083 - Modify this function to support TF_BLOCK_SIZE +// of 64x64. void av1_apply_temporal_filter_neon_dotprod( const YV12_BUFFER_CONFIG *frame_to_filter, const MACROBLOCKD *mbd, const BLOCK_SIZE block_size, const int mb_row, const int mb_col, @@ -221,6 +223,13 @@ const int *subblock_mses, const int q_factor, const int filter_strength, int tf_wgt_calc_lvl, const uint8_t *pred, uint32_t *accum, uint16_t *count) { + if (block_size == BLOCK_64X64) { + av1_apply_temporal_filter_c(frame_to_filter, mbd, block_size, mb_row, + mb_col, num_planes, noise_levels, subblock_mvs, + subblock_mses, q_factor, filter_strength, + tf_wgt_calc_lvl, pred, accum, count); + return; + } const int is_high_bitdepth = frame_to_filter->flags & YV12_FLAG_HIGHBITDEPTH; assert(block_size == BLOCK_32X32 && "Only support 32x32 block with Neon!"); assert(TF_WINDOW_LENGTH == 5 && "Only support window length 5 with Neon!");
diff --git a/av1/encoder/av1_ext_ratectrl.c b/av1/encoder/av1_ext_ratectrl.c new file mode 100644 index 0000000..90ef4b2 --- /dev/null +++ b/av1/encoder/av1_ext_ratectrl.c
@@ -0,0 +1,196 @@ +/* + * Copyright (c) 2025, Alliance for Open Media. All rights reserved. + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "aom/aom_ext_ratectrl.h" +#include "av1/encoder/av1_ext_ratectrl.h" + +aom_codec_err_t av1_extrc_init(AOM_EXT_RATECTRL *ext_ratectrl) { + if (ext_ratectrl == NULL) { + return AOM_CODEC_INVALID_PARAM; + } + av1_zero(*ext_ratectrl); + return AOM_CODEC_OK; +} + +aom_codec_err_t av1_extrc_create(aom_rc_funcs_t funcs, + aom_rc_config_t ratectrl_config, + AOM_EXT_RATECTRL *ext_ratectrl) { + aom_rc_status_t rc_status; + aom_rc_firstpass_stats_t *rc_firstpass_stats; + if (ext_ratectrl == NULL) { + return AOM_CODEC_INVALID_PARAM; + } + av1_extrc_delete(ext_ratectrl); + ext_ratectrl->funcs = funcs; + ext_ratectrl->ratectrl_config = ratectrl_config; + rc_status = ext_ratectrl->funcs.create_model(ext_ratectrl->funcs.priv, + &ext_ratectrl->ratectrl_config, + &ext_ratectrl->model); + if (rc_status == AOM_RC_ERROR) { + return AOM_CODEC_ERROR; + } + rc_firstpass_stats = &ext_ratectrl->rc_firstpass_stats; + rc_firstpass_stats->num_frames = ratectrl_config.show_frame_count; + rc_firstpass_stats->frame_stats = + aom_malloc(sizeof(*rc_firstpass_stats->frame_stats) * + rc_firstpass_stats->num_frames); + if (rc_firstpass_stats->frame_stats == NULL) { + return AOM_CODEC_MEM_ERROR; + } + + ext_ratectrl->ready = 1; + return AOM_CODEC_OK; +} + +static void gen_rc_firstpass_stats(const FIRSTPASS_STATS *stats, + aom_rc_frame_stats_t *rc_frame_stats) { + rc_frame_stats->frame = stats->frame; + rc_frame_stats->weight = stats->weight; + rc_frame_stats->intra_error = stats->intra_error; + rc_frame_stats->coded_error = stats->coded_error; + rc_frame_stats->sr_coded_error = stats->sr_coded_error; + rc_frame_stats->frame_avg_wavelet_energy = stats->frame_avg_wavelet_energy; + rc_frame_stats->pcnt_inter = stats->pcnt_inter; + rc_frame_stats->pcnt_motion = stats->pcnt_motion; + rc_frame_stats->pcnt_second_ref = stats->pcnt_second_ref; + rc_frame_stats->pcnt_neutral = stats->pcnt_neutral; + rc_frame_stats->intra_skip_pct = stats->intra_skip_pct; + rc_frame_stats->inactive_zone_rows = stats->inactive_zone_rows; + rc_frame_stats->inactive_zone_cols = stats->inactive_zone_cols; + rc_frame_stats->MVr = stats->MVr; + rc_frame_stats->mvr_abs = stats->mvr_abs; + rc_frame_stats->MVc = stats->MVc; + rc_frame_stats->mvc_abs = stats->mvc_abs; + rc_frame_stats->MVrv = stats->MVrv; + rc_frame_stats->MVcv = stats->MVcv; + rc_frame_stats->mv_in_out_count = stats->mv_in_out_count; + rc_frame_stats->duration = stats->duration; + rc_frame_stats->count = stats->count; + rc_frame_stats->new_mv_count = stats->new_mv_count; + rc_frame_stats->raw_error_stdev = stats->raw_error_stdev; + rc_frame_stats->is_flash = stats->is_flash; + rc_frame_stats->noise_var = stats->noise_var; + rc_frame_stats->cor_coeff = stats->cor_coeff; + rc_frame_stats->log_intra_error = stats->log_intra_error; + rc_frame_stats->log_coded_error = stats->log_coded_error; +} + +aom_codec_err_t av1_extrc_send_firstpass_stats( + AOM_EXT_RATECTRL *ext_ratectrl, const FIRSTPASS_INFO *first_pass_info) { + assert(first_pass_info != NULL); + assert(ext_ratectrl != NULL); + if (ext_ratectrl->ready) { + aom_rc_status_t rc_status; + aom_rc_firstpass_stats_t *rc_firstpass_stats = + &ext_ratectrl->rc_firstpass_stats; + assert(rc_firstpass_stats->num_frames == first_pass_info->stats_buf_size); + for (int i = 0; i < rc_firstpass_stats->num_frames; ++i) { + gen_rc_firstpass_stats(&first_pass_info->stats_buf[i], + &rc_firstpass_stats->frame_stats[i]); + } + rc_status = ext_ratectrl->funcs.send_firstpass_stats(ext_ratectrl->model, + rc_firstpass_stats); + if (rc_status == AOM_RC_ERROR) { + return AOM_CODEC_ERROR; + } + } + return AOM_CODEC_OK; +} + +aom_codec_err_t av1_extrc_send_tpl_stats( + AOM_EXT_RATECTRL *ext_ratectrl, const AomTplGopStats *extrc_tpl_gop_stats) { + assert(ext_ratectrl != NULL); + assert(extrc_tpl_gop_stats != NULL); + if (ext_ratectrl->ready && ext_ratectrl->funcs.send_tpl_gop_stats != NULL) { + aom_rc_status_t rc_status = ext_ratectrl->funcs.send_tpl_gop_stats( + ext_ratectrl->model, extrc_tpl_gop_stats); + if (rc_status == AOM_RC_ERROR) { + return AOM_CODEC_ERROR; + } + } + return AOM_CODEC_OK; +} + +aom_codec_err_t av1_extrc_get_gop_decision( + AOM_EXT_RATECTRL *ext_ratectrl, aom_rc_gop_decision_t *gop_decision) { + aom_rc_status_t rc_status; + assert(ext_ratectrl != NULL); + assert(gop_decision != NULL); + if ((ext_ratectrl->funcs.rc_type & AOM_RC_GOP) == 0) { + return AOM_CODEC_INVALID_PARAM; + } + rc_status = + ext_ratectrl->funcs.get_gop_decision(ext_ratectrl->model, gop_decision); + if (rc_status == AOM_RC_ERROR) { + return AOM_CODEC_ERROR; + } + return AOM_CODEC_OK; +} + +aom_codec_err_t av1_extrc_get_encodeframe_decision( + AOM_EXT_RATECTRL *ext_ratectrl, int gop_index, + aom_rc_encodeframe_decision_t *encode_frame_decision) { + assert(ext_ratectrl != NULL); + assert(ext_ratectrl->ready && (ext_ratectrl->funcs.rc_type & AOM_RC_QP) != 0); + assert(encode_frame_decision != NULL); + aom_rc_status_t rc_status = ext_ratectrl->funcs.get_encodeframe_decision( + ext_ratectrl->model, gop_index, encode_frame_decision); + if (rc_status == AOM_RC_ERROR) { + return AOM_CODEC_ERROR; + } + return AOM_CODEC_OK; +} + +aom_codec_err_t av1_extrc_update_encodeframe_result( + AOM_EXT_RATECTRL *ext_ratectrl, int64_t bit_count, + int actual_encoding_qindex) { + assert(ext_ratectrl != NULL); + if (ext_ratectrl->ready) { + aom_rc_status_t rc_status; + aom_rc_encodeframe_result_t encode_frame_result; + encode_frame_result.bit_count = bit_count; + encode_frame_result.actual_encoding_qindex = actual_encoding_qindex; + rc_status = ext_ratectrl->funcs.update_encodeframe_result( + ext_ratectrl->model, &encode_frame_result); + if (rc_status == AOM_RC_ERROR) { + return AOM_CODEC_ERROR; + } + } + return AOM_CODEC_OK; +} + +aom_codec_err_t av1_extrc_get_key_frame_decision( + AOM_EXT_RATECTRL *ext_ratectrl, + aom_rc_key_frame_decision_t *key_frame_decision) { + assert(ext_ratectrl != NULL); + if (!ext_ratectrl->ready || (ext_ratectrl->funcs.rc_type & AOM_RC_GOP) == 0) { + return AOM_CODEC_INVALID_PARAM; + } + aom_rc_status_t rc_status = ext_ratectrl->funcs.get_key_frame_decision( + ext_ratectrl->model, key_frame_decision); + return rc_status == AOM_RC_OK ? AOM_CODEC_OK : AOM_CODEC_ERROR; +} + +aom_codec_err_t av1_extrc_delete(AOM_EXT_RATECTRL *ext_ratectrl) { + if (ext_ratectrl == NULL) { + return AOM_CODEC_INVALID_PARAM; + } + if (ext_ratectrl->ready) { + aom_rc_status_t rc_status = + ext_ratectrl->funcs.delete_model(ext_ratectrl->model); + if (rc_status == AOM_RC_ERROR) { + return AOM_CODEC_ERROR; + } + aom_free(ext_ratectrl->rc_firstpass_stats.frame_stats); + aom_free(ext_ratectrl->sb_params_list); + } + return av1_extrc_init(ext_ratectrl); +}
diff --git a/av1/encoder/av1_ext_ratectrl.h b/av1/encoder/av1_ext_ratectrl.h new file mode 100644 index 0000000..bf7c4a9 --- /dev/null +++ b/av1/encoder/av1_ext_ratectrl.h
@@ -0,0 +1,70 @@ +/* + * Copyright (c) 2025, Alliance for Open Media. All rights reserved. + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_AV1_ENCODER_AV1_EXT_RATECTRL_H_ +#define AOM_AV1_ENCODER_AV1_EXT_RATECTRL_H_ + +#include "aom/aom_codec.h" +#include "aom/aom_ext_ratectrl.h" +#include "aom/aom_tpl.h" +#include "av1/encoder/firstpass.h" + +typedef struct AOM_EXT_RATECTRL { + int ready; + int ext_rdmult; + // Whether per-superblock delta Q should be used. This is set on frame basis. + // Key, golden and regular ARF frames use delta Q. All other frames do not. + // This flag is added so the sb_params_list array doesn't have to be freed + // and allocated repeatedly. + int use_delta_q; + aom_rc_model_t model; + aom_rc_funcs_t funcs; + aom_rc_config_t ratectrl_config; + aom_rc_firstpass_stats_t rc_firstpass_stats; + aom_sb_params *sb_params_list; +} AOM_EXT_RATECTRL; + +aom_codec_err_t av1_extrc_init(AOM_EXT_RATECTRL *ext_ratectrl); + +aom_codec_err_t av1_extrc_create(aom_rc_funcs_t funcs, + aom_rc_config_t ratectrl_config, + AOM_EXT_RATECTRL *ext_ratectrl); + +aom_codec_err_t av1_extrc_delete(AOM_EXT_RATECTRL *ext_ratectrl); + +aom_codec_err_t av1_extrc_send_firstpass_stats( + AOM_EXT_RATECTRL *ext_ratectrl, const FIRSTPASS_INFO *first_pass_info); + +aom_codec_err_t av1_extrc_send_tpl_stats( + AOM_EXT_RATECTRL *ext_ratectrl, const AomTplGopStats *extrc_tpl_gop_stats); + +aom_codec_err_t av1_extrc_get_encodeframe_decision( + AOM_EXT_RATECTRL *ext_ratectrl, int gop_index, + aom_rc_encodeframe_decision_t *encode_frame_decision); + +aom_codec_err_t av1_extrc_update_encodeframe_result( + AOM_EXT_RATECTRL *ext_ratectrl, int64_t bit_count, + int actual_encoding_qindex); + +aom_codec_err_t av1_extrc_get_key_frame_decision( + AOM_EXT_RATECTRL *ext_ratectrl, + aom_rc_key_frame_decision_t *key_frame_decision); + +aom_codec_err_t av1_extrc_get_gop_decision(AOM_EXT_RATECTRL *ext_ratectrl, + aom_rc_gop_decision_t *gop_decision); + +aom_codec_err_t av1_extrc_get_frame_rdmult( + AOM_EXT_RATECTRL *ext_ratectrl, int show_index, int coding_index, + int gop_index, FRAME_UPDATE_TYPE update_type, int gop_size, int use_alt_ref, + RefCntBuffer *ref_frame_bufs[AOM_RC_MAX_REF_FRAMES], int ref_frame_flags, + int *rdmult); + +#endif // AOM_AV1_ENCODER_AV1_EXT_RATECTRL_H_
diff --git a/av1/encoder/av1_quantize.c b/av1/encoder/av1_quantize.c index 1673d53..e7367c8 100644 --- a/av1/encoder/av1_quantize.c +++ b/av1/encoder/av1_quantize.c
@@ -885,8 +885,7 @@ // Disable deltaq in lossless mode. if (enable_chroma_deltaq && q) { - if (is_allintra && - (tuning == AOM_TUNE_IQ || tuning == AOM_TUNE_SSIMULACRA2)) { + if (tuning == AOM_TUNE_IQ || tuning == AOM_TUNE_SSIMULACRA2) { int chroma_dc_delta_q = 0; int chroma_ac_delta_q = 0; @@ -902,7 +901,13 @@ // The ramp-down of chroma increase was determined by generating the // convex hull of SSIMULACRA 2 scores (for all boosts from 0-16), and // finding a linear equation that fits the convex hull. - chroma_dc_delta_q = -clamp((quant_params->base_qindex / 2) - 14, 0, 16); + // For the SSIMULACRA 2 tune only: a convex hull of SSIMULACRA 2 scores + // (mean, 10th percentile) was generated by decreasing chroma delta_q + // by 3 different values (16, 20 and 24). 20 was selected as it had the + // best 10th percentile performance. + int offset = (tuning == AOM_TUNE_SSIMULACRA2) ? 20 : 16; + chroma_dc_delta_q = + -clamp((quant_params->base_qindex / 2) - 14, 0, offset); chroma_ac_delta_q = chroma_dc_delta_q; } else if (cm->seq_params->subsampling_x == 1 && cm->seq_params->subsampling_y == 0) { @@ -980,30 +985,28 @@ int (*get_luma_qmlevel)(int, int, int); int (*get_chroma_qmlevel)(int, int, int); - if (is_allintra) { - if (tuning == AOM_TUNE_IQ || tuning == AOM_TUNE_SSIMULACRA2) { - if (tuning == AOM_TUNE_SSIMULACRA2) { - // Use luma QM formula specifically tailored for tune SSIMULACRA2 - get_luma_qmlevel = aom_get_qmlevel_luma_ssimulacra2; - } else { - get_luma_qmlevel = aom_get_qmlevel_allintra; - } - - if (cm->seq_params->subsampling_x == 0 && - cm->seq_params->subsampling_y == 0) { - // 4:4:4 subsampling mode has 4x the number of chroma coefficients - // compared to 4:2:0 (2x on each dimension). This means the encoder - // should use lower chroma QM levels that more closely match the scaling - // of an equivalent 4:2:0 chroma QM. - get_chroma_qmlevel = aom_get_qmlevel_444_chroma; - } else { - // For all other chroma subsampling modes, use the all intra QM formula - get_chroma_qmlevel = aom_get_qmlevel_allintra; - } + if (tuning == AOM_TUNE_IQ || tuning == AOM_TUNE_SSIMULACRA2) { + if (tuning == AOM_TUNE_SSIMULACRA2) { + // Use luma QM formula specifically tailored for tune SSIMULACRA2 + get_luma_qmlevel = aom_get_qmlevel_luma_ssimulacra2; } else { get_luma_qmlevel = aom_get_qmlevel_allintra; + } + + if (cm->seq_params->subsampling_x == 0 && + cm->seq_params->subsampling_y == 0) { + // 4:4:4 subsampling mode has 4x the number of chroma coefficients + // compared to 4:2:0 (2x on each dimension). This means the encoder + // should use lower chroma QM levels that more closely match the scaling + // of an equivalent 4:2:0 chroma QM. + get_chroma_qmlevel = aom_get_qmlevel_444_chroma; + } else { + // For all other chroma subsampling modes, use the all intra QM formula get_chroma_qmlevel = aom_get_qmlevel_allintra; } + } else if (is_allintra) { + get_luma_qmlevel = aom_get_qmlevel_allintra; + get_chroma_qmlevel = aom_get_qmlevel_allintra; } else { get_luma_qmlevel = aom_get_qmlevel; get_chroma_qmlevel = aom_get_qmlevel;
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c index a6a92bf..178a84a 100644 --- a/av1/encoder/bitstream.c +++ b/av1/encoder/bitstream.c
@@ -4222,23 +4222,24 @@ for (size_t i = 0; i < arr->sz; i++) { aom_metadata_t *current_metadata = arr->metadata_array[i]; if (current_metadata && current_metadata->payload) { + const int metadata_insert_location = + current_metadata->insert_flag & AOM_MIF_INSERT_LOCATION_MASK; if ((cm->current_frame.frame_type == KEY_FRAME && - current_metadata->insert_flag == AOM_MIF_KEY_FRAME) || + metadata_insert_location == AOM_MIF_KEY_FRAME) || (cm->current_frame.frame_type != KEY_FRAME && - current_metadata->insert_flag == AOM_MIF_NON_KEY_FRAME) || - current_metadata->insert_flag == AOM_MIF_ANY_FRAME) { + metadata_insert_location == AOM_MIF_NON_KEY_FRAME) || + metadata_insert_location == AOM_MIF_ANY_FRAME) { // OBU header is either one or two bytes. if (dst_size < 2) { aom_internal_error(cm->error, AOM_CODEC_ERROR, "av1_write_metadata_array: output buffer full"); } - // According to the AV1 spec draft version (as of git commit 5e04f) - // Section 6.7.1, some metadata types can be layer specific, but we - // currently only support non-layer specific metadata. + const bool is_layer_specific_obu = + (current_metadata->insert_flag & AOM_MIF_LAYER_SPECIFIC) != 0; obu_header_size = av1_write_obu_header( &cpi->ppi->level_params, &cpi->frame_header_count, OBU_METADATA, cm->seq_params->has_nonzero_operating_point_idc, - /*is_layer_specific_obu=*/false, 0, dst); + is_layer_specific_obu, 0, dst); assert(obu_header_size <= 2); obu_payload_size = av1_write_metadata_obu(current_metadata, dst + obu_header_size,
diff --git a/av1/encoder/block.h b/av1/encoder/block.h index ecbc1aa..6734f34 100644 --- a/av1/encoder/block.h +++ b/av1/encoder/block.h
@@ -441,6 +441,11 @@ */ int use_default_intra_tx_type; + /*! Whether to limit the intra transform search type to the ones in the table + * av1_derived_intra_tx_used_flag[INTRA_MODES]. + */ + int use_derived_intra_tx_type_set; + /*! Probability threshold used for conditionally forcing tx type*/ int default_inter_tx_type_prob_thresh; @@ -952,6 +957,11 @@ * prediction. */ uint8_t *tmp_pred_bufs[2]; + + /*! + * Buffer used for upsampled prediction. + */ + uint8_t *upsample_pred; /**@}*/ /***************************************************************************** @@ -1352,9 +1362,9 @@ //! Flag to indicate to test the superblock MV for the coding block in the // nonrd_pickmode. int sb_me_block; - //! Flag to indicate superblock selected column scroll. + //! Counter for superblock selected column scroll. int sb_col_scroll; - //! Flag to indicate superblock selected row scroll. + //! Counter for superblock selected row scroll. int sb_row_scroll; //! Motion vector from superblock MV derived from int_pro_motion() in // the variance_partitioning.
diff --git a/av1/encoder/compound_type.c b/av1/encoder/compound_type.c index d5dcf74..f8364f1 100644 --- a/av1/encoder/compound_type.c +++ b/av1/encoder/compound_type.c
@@ -1307,8 +1307,6 @@ const bool skip_mv_refinement_for_avg_distwtd = enable_fast_compound_mode_search == 3 || (enable_fast_compound_mode_search == 2 && (this_mode != NEW_NEWMV)); - const bool skip_mv_refinement_for_diffwtd = - (!enable_fast_compound_mode_search && cur_type == COMPOUND_DIFFWTD); // Case COMPOUND_AVERAGE and COMPOUND_DISTWTD if (cur_type < COMPOUND_WEDGE) { @@ -1376,12 +1374,26 @@ int eval_txfm = prune_mode_by_skip_rd(cpi, x, xd, bsize, ref_skip_rd, rs2 + *rate_mv); + int64_t est_rd = INT64_MAX; + RD_STATS est_rd_stats; if (eval_txfm) { - RD_STATS est_rd_stats; - estimate_yrd_for_sb(cpi, bsize, x, INT64_MAX, &est_rd_stats); - + est_rd = estimate_yrd_for_sb(cpi, bsize, x, INT64_MAX, &est_rd_stats); + } + if (est_rd != INT64_MAX) { best_rd_cur = RDCOST(x->rdmult, rs2 + tmp_rate_mv + est_rd_stats.rate, est_rd_stats.dist); + int rate_sum; + uint8_t tmp_skip_txfm_sb; + int64_t dist_sum, tmp_skip_sse_sb; + model_rd_sb_fn[MODELRD_TYPE_MASKED_COMPOUND]( + cpi, bsize, x, xd, 0, 0, &rate_sum, &dist_sum, &tmp_skip_txfm_sb, + &tmp_skip_sse_sb, NULL, NULL, NULL); + comp_model_rd_cur = + RDCOST(x->rdmult, rs2 + tmp_rate_mv + rate_sum, dist_sum); + // Backup rate and distortion for future reuse + backup_stats(cur_type, comp_rate, comp_dist, comp_model_rate, + comp_model_dist, rate_sum, dist_sum, &est_rd_stats, + comp_rs2, rs2); } } @@ -1524,7 +1536,8 @@ mbmi->mv[1] = tmp_mv[1]; tmp_rate_mv = best_rate_mv; rs2 = best_rs2; - } else if (skip_mv_refinement_for_diffwtd) { + } else if (!enable_fast_compound_mode_search && + cur_type == COMPOUND_DIFFWTD) { int_mv tmp_mv[2]; int best_mask_index = 0; rs2 += get_interinter_compound_mask_rate(&x->mode_costs, mbmi);
diff --git a/av1/encoder/context_tree.c b/av1/encoder/context_tree.c index 4273fae..74b82e5 100644 --- a/av1/encoder/context_tree.c +++ b/av1/encoder/context_tree.c
@@ -25,8 +25,6 @@ dst_ctx->best_mode_index = src_ctx->best_mode_index; #endif // CONFIG_INTERNAL_STATS - memcpy(dst_ctx->blk_skip, src_ctx->blk_skip, - sizeof(uint8_t) * src_ctx->num_4x4_blk); av1_copy_array(dst_ctx->tx_type_map, src_ctx->tx_type_map, src_ctx->num_4x4_blk); @@ -84,8 +82,6 @@ const int num_pix = block_size_wide[bsize] * block_size_high[bsize]; const int num_blk = num_pix / 16; - AOM_CHECK_MEM_ERROR(&error, ctx->blk_skip, - aom_calloc(num_blk, sizeof(*ctx->blk_skip))); AOM_CHECK_MEM_ERROR(&error, ctx->tx_type_map, aom_calloc(num_blk, sizeof(*ctx->tx_type_map))); ctx->num_4x4_blk = num_blk; @@ -119,7 +115,6 @@ } void av1_reset_pmc(PICK_MODE_CONTEXT *ctx) { - av1_zero_array(ctx->blk_skip, ctx->num_4x4_blk); av1_zero_array(ctx->tx_type_map, ctx->num_4x4_blk); av1_invalid_rd_stats(&ctx->rd_stats); } @@ -127,8 +122,6 @@ void av1_free_pmc(PICK_MODE_CONTEXT *ctx, int num_planes) { if (ctx == NULL) return; - aom_free(ctx->blk_skip); - ctx->blk_skip = NULL; aom_free(ctx->tx_type_map); for (int i = 0; i < num_planes; ++i) { ctx->coeff[i] = NULL;
diff --git a/av1/encoder/context_tree.h b/av1/encoder/context_tree.h index 6b6d851..1578793 100644 --- a/av1/encoder/context_tree.h +++ b/av1/encoder/context_tree.h
@@ -38,7 +38,6 @@ MB_MODE_INFO mic; MB_MODE_INFO_EXT_FRAME mbmi_ext_best; uint8_t *color_index_map[2]; - uint8_t *blk_skip; tran_low_t *coeff[MAX_MB_PLANE]; tran_low_t *qcoeff[MAX_MB_PLANE];
diff --git a/av1/encoder/cost.c b/av1/encoder/cost.c index e4d15e2..4bf8fce 100644 --- a/av1/encoder/cost.c +++ b/av1/encoder/cost.c
@@ -10,6 +10,7 @@ */ #include <assert.h> +#include "aom_dsp/entcode.h" #include "av1/encoder/cost.h" #include "av1/common/entropy.h"
diff --git a/av1/encoder/encode_strategy.c b/av1/encoder/encode_strategy.c index b18b21f..d8232f9 100644 --- a/av1/encoder/encode_strategy.c +++ b/av1/encoder/encode_strategy.c
@@ -611,6 +611,11 @@ } } +static int get_new_fb_map_idx_rc(int new_fb_map_idx) { + if (new_fb_map_idx == INVALID_IDX) return 0; + return 1 << new_fb_map_idx; +} + int av1_get_refresh_frame_flags( const AV1_COMP *const cpi, const EncodeFrameParams *const frame_params, FRAME_UPDATE_TYPE frame_update_type, int gf_index, int cur_disp_order, @@ -637,12 +642,16 @@ #if !CONFIG_REALTIME_ONLY if (cpi->use_ducky_encode && cpi->ducky_encode_info.frame_info.gop_mode == DUCKY_ENCODE_GOP_MODE_RCL) { - int new_fb_map_idx = cpi->ppi->gf_group.update_ref_idx[gf_index]; - if (new_fb_map_idx == INVALID_IDX) return 0; - return 1 << new_fb_map_idx; + return get_new_fb_map_idx_rc(gf_group->update_ref_idx[gf_index]); } #endif // !CONFIG_REALTIME_ONLY + if (cpi->ext_ratectrl.ready && + (cpi->ext_ratectrl.funcs.rc_type & AOM_RC_GOP) != 0 && + cpi->ext_ratectrl.funcs.get_gop_decision != NULL) { + return get_new_fb_map_idx_rc(gf_group->update_ref_idx[gf_index]); + } + int refresh_mask = 0; if (ext_refresh_frame_flags->update_pending) { if (rtc_ref->set_ref_frame_config || @@ -787,7 +796,8 @@ if (tf_buf != NULL) { frame_input->source = tf_buf; show_existing_alt_ref = av1_check_show_filtered_frame( - tf_buf, &frame_diff, q_index, cm->seq_params->bit_depth); + tf_buf, &frame_diff, q_index, cm->seq_params->bit_depth, + cpi->oxcf.algo_cfg.enable_overlay); if (show_existing_alt_ref) { cpi->common.showable_frame |= 1; } else { @@ -822,8 +832,9 @@ // TODO(angiebird): Reuse tf_info->tf_buf here. av1_temporal_filter(cpi, arf_src_index, cpi->gf_frame_index, &frame_diff, tf_buf_second_arf); - show_existing_alt_ref = av1_check_show_filtered_frame( - tf_buf_second_arf, &frame_diff, q_index, cm->seq_params->bit_depth); + show_existing_alt_ref = + av1_check_show_filtered_frame(tf_buf_second_arf, &frame_diff, q_index, + cm->seq_params->bit_depth, 1); if (show_existing_alt_ref) { aom_extend_frame_borders(tf_buf_second_arf, av1_num_planes(cm)); frame_input->source = tf_buf_second_arf; @@ -869,11 +880,18 @@ if (frame_params->frame_type != KEY_FRAME) { // In rare case, it's possible to have non ARF/GF update_type here. // We should set allow_tpl to zero in the situation - allow_tpl = - allow_tpl && (update_type == ARF_UPDATE || update_type == GF_UPDATE || - (cpi->use_ducky_encode && - cpi->ducky_encode_info.frame_info.gop_mode == - DUCKY_ENCODE_GOP_MODE_RCL)); + bool frame_type_allow_tpl = + update_type == ARF_UPDATE || update_type == GF_UPDATE; + // When external rate control is used, sometimes the sequence ends with + // a GOP with only 1 frame which is a leaf frame. TPL stats needs to be + // calculated for it as well. + if (av1_use_tpl_for_extrc(&cpi->ext_ratectrl)) { + frame_type_allow_tpl |= update_type == LF_UPDATE; + } + allow_tpl = allow_tpl && (frame_type_allow_tpl || + (cpi->use_ducky_encode && + cpi->ducky_encode_info.frame_info.gop_mode == + DUCKY_ENCODE_GOP_MODE_RCL)); } if (allow_tpl) { @@ -995,6 +1013,21 @@ // Initialize reference frame mappings. for (int i = 0; i < REF_FRAMES; ++i) remapped_ref_idx[i] = INVALID_IDX; + if (cpi->ppi->gf_group.use_ext_ref_frame_map[gf_index]) { + for (int rf = LAST_FRAME; rf < REF_FRAMES; ++rf) { + if (cpi->ppi->gf_group.ref_frame_list[gf_index][rf] != INVALID_IDX) { + remapped_ref_idx[rf - LAST_FRAME] = + (int)cpi->ppi->gf_group.ref_frame_list[gf_index][rf]; + } + } + for (int i = 0; i < REF_FRAMES; ++i) { + if (remapped_ref_idx[i] == INVALID_IDX) { + remapped_ref_idx[i] = 0; + } + } + return; + } + #if !CONFIG_REALTIME_ONLY if (cpi->use_ducky_encode && cpi->ducky_encode_info.frame_info.gop_mode == DUCKY_ENCODE_GOP_MODE_RCL) { @@ -1123,7 +1156,7 @@ // Do not map GOLDEN and ALTREF based on their pyramid level if all reference // frames have the same level. - if (n_min_level_refs <= n_bufs) { + if (n_min_level_refs < n_bufs) { // Map the GOLDEN_FRAME. if (golden_idx > -1) add_ref_to_slot(&buffer_map[golden_idx], remapped_ref_idx, GOLDEN_FRAME); @@ -1142,8 +1175,8 @@ // Continue if the current ref slot is already full. if (remapped_ref_idx[frame - LAST_FRAME] != INVALID_IDX) continue; // Find the next unmapped reference buffer - // in decreasing ouptut order relative to current picture. - int next_buf_max = 0; + // in decreasing output order relative to current picture. + int next_buf_max = -1; int next_disp_order = INT_MIN; for (buf_map_idx = n_bufs - 1; buf_map_idx >= 0; buf_map_idx--) { if (!buffer_map[buf_map_idx].used && @@ -1164,8 +1197,8 @@ // Continue if the current ref slot is already full. if (remapped_ref_idx[frame - LAST_FRAME] != INVALID_IDX) continue; // Find the next unmapped reference buffer - // in increasing ouptut order relative to current picture. - int next_buf_max = 0; + // in increasing output order relative to current picture. + int next_buf_max = -1; int next_disp_order = INT_MAX; for (buf_map_idx = n_bufs - 1; buf_map_idx >= 0; buf_map_idx--) { if (!buffer_map[buf_map_idx].used && @@ -1302,8 +1335,7 @@ return -1; } - // TODO(sarahparker) finish bit allocation for one pass pyramid - if (has_no_stats_stage(cpi)) { + if (has_no_stats_stage(cpi) && !is_one_pass_rt_lag_params(cpi)) { gf_cfg->gf_max_pyr_height = AOMMIN(gf_cfg->gf_max_pyr_height, USE_ALTREF_FOR_ONE_PASS); gf_cfg->gf_min_pyr_height = @@ -1494,6 +1526,21 @@ *frame_flags); if (use_rtc_reference_structure_one_layer(cpi)) av1_set_rtc_reference_structure_one_layer(cpi, cpi->gf_frame_index == 0); + } else if (is_one_pass_rt_lag_params(cpi) && oxcf->rc_cfg.mode == AOM_CBR) { + // For realtime mode with lookahead in CBR: the current frame buffer_level + // is used to update the frame target_bandwidth, so we need to call + // av1_calc_i/pframe_target_size_one_pass_cbr() here for every frame. + int target; + const FRAME_UPDATE_TYPE cur_update_type = + gf_group->update_type[cpi->gf_frame_index]; + if (cur_update_type == KF_UPDATE) { + target = av1_calc_iframe_target_size_one_pass_cbr(cpi); + } else { + target = av1_calc_pframe_target_size_one_pass_cbr(cpi, cur_update_type); + } + gf_group->bit_allocation[cpi->gf_frame_index] = target; + av1_rc_set_frame_target(cpi, target, cm->width, cm->height); + cpi->rc.base_frame_target = target; } #endif #if CONFIG_COLLECT_COMPONENT_TIMING
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c index 423f2f9..5846e03 100644 --- a/av1/encoder/encodeframe.c +++ b/av1/encoder/encodeframe.c
@@ -175,6 +175,8 @@ void av1_init_rtc_counters(MACROBLOCK *const x) { av1_init_cyclic_refresh_counters(x); x->cnt_zeromv = 0; + x->sb_col_scroll = 0; + x->sb_row_scroll = 0; } void av1_accumulate_rtc_counters(AV1_COMP *cpi, const MACROBLOCK *const x) { @@ -306,15 +308,25 @@ const int delta_q_res = delta_q_info->delta_q_res; int current_qindex = cm->quant_params.base_qindex; + const int sb_row = mi_row >> cm->seq_params->mib_size_log2; + const int sb_col = mi_col >> cm->seq_params->mib_size_log2; + const int sb_cols = + CEIL_POWER_OF_TWO(cm->mi_params.mi_cols, cm->seq_params->mib_size_log2); + const int sb_index = sb_row * sb_cols + sb_col; if (cpi->use_ducky_encode && cpi->ducky_encode_info.frame_info.qp_mode == DUCKY_ENCODE_FRAME_MODE_QINDEX) { - const int sb_row = mi_row >> cm->seq_params->mib_size_log2; - const int sb_col = mi_col >> cm->seq_params->mib_size_log2; - const int sb_cols = - CEIL_POWER_OF_TWO(cm->mi_params.mi_cols, cm->seq_params->mib_size_log2); - const int sb_index = sb_row * sb_cols + sb_col; current_qindex = cpi->ducky_encode_info.frame_info.superblock_encode_qindex[sb_index]; + } else if (cpi->ext_ratectrl.ready && + (cpi->ext_ratectrl.funcs.rc_type & AOM_RC_QP) != 0 && + cpi->ext_ratectrl.funcs.get_encodeframe_decision != NULL && + cpi->ext_ratectrl.sb_params_list != NULL) { + if (cpi->ext_ratectrl.use_delta_q) { + const int q_index = cpi->ext_ratectrl.sb_params_list[sb_index].q_index; + if (q_index != AOM_DEFAULT_Q) { + current_qindex = q_index; + } + } } else if (cpi->oxcf.q_cfg.deltaq_mode == DELTA_Q_PERCEPTUAL) { if (DELTA_Q_PERCEPTUAL_MODULATION == 1) { const int block_wavelet_energy_level = @@ -1281,8 +1293,6 @@ x->sb_me_block = 0; x->sb_me_partition = 0; x->sb_me_mv.as_int = 0; - x->sb_col_scroll = 0; - x->sb_row_scroll = 0; x->sb_force_fixed_part = 1; x->color_palette_thresh = 64; x->force_color_check_block_level = 0; @@ -1688,6 +1698,71 @@ : INTERP_SKIP_LUMA_SKIP_CHROMA; } +/*!\cond */ +typedef struct { + // Scoring function for usefulness of references (the lower score, the more + // useful) + int score; + // Index in the reference buffer + int index; +} RefScoreData; +/*!\endcond */ + +// Comparison function to sort reference frames in ascending score order. +static int compare_score_data_asc(const void *a, const void *b) { + const RefScoreData *ra = (const RefScoreData *)a; + const RefScoreData *rb = (const RefScoreData *)b; + + const int score_diff = ra->score - rb->score; + if (score_diff != 0) return score_diff; + + return ra->index - rb->index; +} + +// Determines whether a given reference frame is "good" based on temporal +// distance and base_qindex. The "good" reference frames are not allowed to be +// pruned by the speed feature "prune_single_ref" frame at block level. +static inline void setup_keep_single_ref_frame_mask(AV1_COMP *cpi) { + const int prune_single_ref = cpi->sf.inter_sf.prune_single_ref; + const AV1_COMMON *const cm = &cpi->common; + + if (prune_single_ref != 1 || frame_is_intra_only(cm)) { + cpi->keep_single_ref_frame_mask = + (prune_single_ref == 0) ? ((1 << REF_FRAMES) - 1) : 0; + return; + } + RefScoreData ref_score_data[INTER_REFS_PER_FRAME]; + for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) { + ref_score_data[i].score = INT_MAX; + ref_score_data[i].index = i; + } + + // Calculate score for each reference frame based on relative distance to + // the current frame and its base_qindex. A lower score means that the + // reference is potentially more useful. + for (MV_REFERENCE_FRAME ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; + ++ref_frame) { + if (cpi->ref_frame_flags & av1_ref_frame_flag_list[ref_frame]) { + const RefFrameDistanceInfo *const ref_frame_dist_info = + &cpi->ref_frame_dist_info; + const RefCntBuffer *const buf = get_ref_frame_buf(cm, ref_frame); + ref_score_data[ref_frame - LAST_FRAME].score = + abs(ref_frame_dist_info->ref_relative_dist[ref_frame - LAST_FRAME]) + + buf->base_qindex; + } + } + + qsort(ref_score_data, INTER_REFS_PER_FRAME, sizeof(ref_score_data[0]), + compare_score_data_asc); + + cpi->keep_single_ref_frame_mask = 0; + const int num_frames_to_keep = 3; + for (int i = 0; i < num_frames_to_keep; ++i) { + const int idx = ref_score_data[i].index; + cpi->keep_single_ref_frame_mask |= 1 << idx; + } +} + static inline void setup_prune_ref_frame_mask(AV1_COMP *cpi) { if ((!cpi->oxcf.ref_frm_cfg.enable_onesided_comp || cpi->sf.inter_sf.disable_onesided_comp) && @@ -2255,6 +2330,9 @@ cpi->prune_ref_frame_mask = 0; // Figure out which ref frames can be skipped at frame level. setup_prune_ref_frame_mask(cpi); + // Disable certain reference frame pruning based on temporal distance and + // quality of that reference frame. + setup_keep_single_ref_frame_mask(cpi); x->txfm_search_info.txb_split_count = 0; #if CONFIG_SPEED_STATS
diff --git a/av1/encoder/encodeframe_utils.c b/av1/encoder/encodeframe_utils.c index 79da14f..0dc7ef7 100644 --- a/av1/encoder/encodeframe_utils.c +++ b/av1/encoder/encodeframe_utils.c
@@ -9,6 +9,9 @@ * PATENTS file, you can obtain it at www.aomedia.org/license/patent. */ +#include "aom/aom_codec.h" +#include "aom/internal/aom_codec_internal.h" + #include "av1/common/common_data.h" #include "av1/common/quant_common.h" #include "av1/common/reconintra.h" @@ -182,25 +185,33 @@ struct macroblock_plane *const p = x->plane; struct macroblockd_plane *const pd = xd->plane; const MB_MODE_INFO *const mi = &ctx->mic; + if (ctx == NULL) { + aom_internal_error(cm->error, AOM_CODEC_ERROR, + "ctx is NULL in av1_update_state: %d %d %d %d %d %d \n", + cm->current_frame.frame_type, cm->width, cm->height, + mi_col, mi_row, bsize); + } + if (mi == NULL) { + aom_internal_error(cm->error, AOM_CODEC_ERROR, + "mi is NULL in av1_update_state: %d %d %d %d %d %d \n", + cm->current_frame.frame_type, cm->width, cm->height, + mi_col, mi_row, bsize); + } MB_MODE_INFO *const mi_addr = xd->mi[0]; const struct segmentation *const seg = &cm->seg; assert(bsize < BLOCK_SIZES_ALL); - const int bw = mi_size_wide[mi->bsize]; - const int bh = mi_size_high[mi->bsize]; + assert(mi->bsize == bsize); + const int bw = mi_size_wide[bsize]; + const int bh = mi_size_high[bsize]; const int mis = mi_params->mi_stride; const int mi_width = mi_size_wide[bsize]; const int mi_height = mi_size_high[bsize]; TxfmSearchInfo *txfm_info = &x->txfm_search_info; - assert(mi->bsize == bsize); - *mi_addr = *mi; copy_mbmi_ext_frame_to_mbmi_ext(&x->mbmi_ext, &ctx->mbmi_ext_best, av1_ref_frame_type(ctx->mic.ref_frame)); - memcpy(txfm_info->blk_skip, ctx->blk_skip, - sizeof(txfm_info->blk_skip[0]) * ctx->num_4x4_blk); - txfm_info->skip_txfm = ctx->rd_stats.skip_txfm; xd->tx_type_map = ctx->tx_type_map;
diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c index 447cae4..bf5c697 100644 --- a/av1/encoder/encodemb.c +++ b/av1/encoder/encodemb.c
@@ -248,18 +248,6 @@ } } -// Settings for optimization type. NOTE: To set optimization type for all intra -// frames, both `KEY_BLOCK_OPT_TYPE` and `INTRA_BLOCK_OPT_TYPE` should be set. -// TODO(yjshen): These settings are hard-coded and look okay for now. They -// should be made configurable later. -// Blocks of key frames ONLY. -static const OPT_TYPE KEY_BLOCK_OPT_TYPE = TRELLIS_DROPOUT_OPT; -// Blocks of intra frames (key frames EXCLUSIVE). -static const OPT_TYPE INTRA_BLOCK_OPT_TYPE = TRELLIS_DROPOUT_OPT; -// Blocks of inter frames. (NOTE: Dropout optimization is DISABLED by default -// if trellis optimization is on for inter frames.) -static const OPT_TYPE INTER_BLOCK_OPT_TYPE = TRELLIS_DROPOUT_OPT; - enum { QUANT_FUNC_LOWBD = 0, QUANT_FUNC_HIGHBD = 1, @@ -404,16 +392,13 @@ ENTROPY_CONTEXT *a, *l; int dummy_rate_cost = 0; - const int bw = mi_size_wide[plane_bsize]; dst = &pd->dst.buf[(blk_row * pd->dst.stride + blk_col) << MI_SIZE_LOG2]; a = &args->ta[blk_col]; l = &args->tl[blk_row]; TX_TYPE tx_type = DCT_DCT; - const int blk_skip_idx = blk_row * bw + blk_col; - if (!is_blk_skip(x->txfm_search_info.blk_skip, plane, blk_skip_idx) && - !mbmi->skip_mode) { + if (!mbmi->skip_mode) { tx_type = av1_get_tx_type(xd, pd->plane_type, blk_row, blk_col, tx_size, cm->features.reduced_tx_set_used); TxfmParam txfm_param; @@ -432,23 +417,12 @@ &quant_param); av1_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize, &txfm_param, &quant_param); - - // Whether trellis or dropout optimization is required for inter frames. - const bool do_trellis = INTER_BLOCK_OPT_TYPE == TRELLIS_OPT || - INTER_BLOCK_OPT_TYPE == TRELLIS_DROPOUT_OPT; - const bool do_dropout = INTER_BLOCK_OPT_TYPE == DROPOUT_OPT || - INTER_BLOCK_OPT_TYPE == TRELLIS_DROPOUT_OPT; - - if (quant_param.use_optimize_b && do_trellis) { + if (use_trellis) { TXB_CTX txb_ctx; get_txb_ctx(plane_bsize, tx_size, plane, a, l, &txb_ctx); av1_optimize_b(args->cpi, x, plane, block, tx_size, tx_type, &txb_ctx, &dummy_rate_cost); } - if (!quant_param.use_optimize_b && do_dropout) { - av1_dropout_qcoeff(x, plane, block, tx_size, tx_type, - cm->quant_params.base_qindex); - } } else { p->eobs[block] = 0; p->txb_entropy_ctx[block] = 0; @@ -733,7 +707,6 @@ const AV1_COMMON *const cm = &cpi->common; MACROBLOCK *const x = args->x; MACROBLOCKD *const xd = &x->e_mbd; - MB_MODE_INFO *mbmi = xd->mi[0]; struct macroblock_plane *const p = &x->plane[plane]; struct macroblockd_plane *const pd = &xd->plane[plane]; tran_low_t *dqcoeff = p->dqcoeff + BLOCK_OFFSET(block); @@ -746,9 +719,7 @@ av1_predict_intra_block_facade(cm, xd, plane, blk_col, blk_row, tx_size); TX_TYPE tx_type = DCT_DCT; - const int bw = mi_size_wide[plane_bsize]; - if (plane == 0 && is_blk_skip(x->txfm_search_info.blk_skip, plane, - blk_row * bw + blk_col)) { + if (xd->mi[0]->skip_txfm) { *eob = 0; p->txb_entropy_ctx[block] = 0; } else { @@ -777,32 +748,12 @@ av1_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize, &txfm_param, &quant_param); - - // Whether trellis or dropout optimization is required for key frames and - // intra frames. - const bool do_trellis = (frame_is_intra_only(cm) && - (KEY_BLOCK_OPT_TYPE == TRELLIS_OPT || - KEY_BLOCK_OPT_TYPE == TRELLIS_DROPOUT_OPT)) || - (!frame_is_intra_only(cm) && - (INTRA_BLOCK_OPT_TYPE == TRELLIS_OPT || - INTRA_BLOCK_OPT_TYPE == TRELLIS_DROPOUT_OPT)); - const bool do_dropout = (frame_is_intra_only(cm) && - (KEY_BLOCK_OPT_TYPE == DROPOUT_OPT || - KEY_BLOCK_OPT_TYPE == TRELLIS_DROPOUT_OPT)) || - (!frame_is_intra_only(cm) && - (INTRA_BLOCK_OPT_TYPE == DROPOUT_OPT || - INTRA_BLOCK_OPT_TYPE == TRELLIS_DROPOUT_OPT)); - - if (quant_param.use_optimize_b && do_trellis) { + if (use_trellis) { TXB_CTX txb_ctx; get_txb_ctx(plane_bsize, tx_size, plane, a, l, &txb_ctx); av1_optimize_b(args->cpi, x, plane, block, tx_size, tx_type, &txb_ctx, &dummy_rate_cost); } - if (do_dropout) { - av1_dropout_qcoeff(x, plane, block, tx_size, tx_type, - cm->quant_params.base_qindex); - } } if (*eob) { @@ -827,10 +778,6 @@ update_txk_array(xd, blk_row, blk_col, tx_size, DCT_DCT); } - // For intra mode, skipped blocks are so rare that transmitting - // skip_txfm = 1 is very expensive. - mbmi->skip_txfm = 0; - #if !CONFIG_REALTIME_ONLY if (plane == AOM_PLANE_Y && xd->cfl.store_y) { cfl_store_tx(xd, blk_row, blk_col, tx_size, plane_bsize);
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c index df01bae..85895ad 100644 --- a/av1/encoder/encoder.c +++ b/av1/encoder/encoder.c
@@ -553,7 +553,7 @@ aom_internal_error( &ppi->error, AOM_CODEC_UNSUP_BITSTREAM, "AV1 does not support this combination of profile, level, and tier."); - // Buffer size in bits/s is bitrate in bits/s * 1 s + // Buffer size in bits is bitrate in bits/s * 1 s seq_params->op_params[i].buffer_size = seq_params->op_params[i].bitrate; } } @@ -879,6 +879,7 @@ const FrameDimensionCfg *const frm_dim_cfg = &cpi->oxcf.frm_dim_cfg; const RateControlCfg *const rc_cfg = &oxcf->rc_cfg; FeatureFlags *const features = &cm->features; + const int is_highbitdepth = seq_params->use_highbitdepth; // in case of LAP, lag in frames is set according to number of lap buffers // calculated at init time. This stores and restores LAP's lag in frames to @@ -976,6 +977,14 @@ } } + if (x->upsample_pred == NULL) { + CHECK_MEM_ERROR( + cm, x->upsample_pred, + aom_memalign(16, (1 + is_highbitdepth) * ((MAX_SB_SIZE + 16) + 16) * + MAX_SB_SIZE * sizeof(*x->upsample_pred))); + x->e_mbd.tmp_upsample_pred = x->upsample_pred; + } + av1_reset_segment_features(cm); av1_set_high_precision_mv(cpi, 1, 0); @@ -1750,6 +1759,8 @@ for (int frame = 0; frame < MAX_LAG_BUFFERS; ++frame) { aom_free(tpl_data->tpl_stats_pool[frame]); aom_free_frame_buffer(&tpl_data->tpl_rec_pool[frame]); + aom_free_frame_buffer(&tpl_data->prev_gop_arf_src); + tpl_data->prev_gop_arf_disp_order = -1; tpl_data->tpl_stats_pool[frame] = NULL; } @@ -3146,20 +3157,24 @@ // This is for rtc temporal filtering case. if (is_psnr_calc_enabled(cpi) && cpi->sf.rt_sf.use_rtc_tf) { - const SequenceHeader *seq_params = cm->seq_params; - if (cpi->orig_source.buffer_alloc_sz == 0 || - cpi->rc.prev_coded_width != cpi->oxcf.frm_dim_cfg.width || - cpi->rc.prev_coded_height != cpi->oxcf.frm_dim_cfg.height) { - // Allocate a source buffer to store the true source for psnr calculation. - if (aom_alloc_frame_buffer( - &cpi->orig_source, cpi->oxcf.frm_dim_cfg.width, - cpi->oxcf.frm_dim_cfg.height, seq_params->subsampling_x, - seq_params->subsampling_y, seq_params->use_highbitdepth, - cpi->oxcf.border_in_pixels, cm->features.byte_alignment, false, - 0)) + cpi->orig_source.y_crop_width != cpi->source->y_crop_width || + cpi->orig_source.y_crop_height != cpi->source->y_crop_height || + cpi->orig_source.subsampling_x != cpi->source->subsampling_x || + cpi->orig_source.subsampling_y != cpi->source->subsampling_y || + cpi->orig_source.flags != cpi->source->flags) { + // Allocate a source buffer to store the original source for psnr + // calculation. + const int use_highbitdepth = + (cpi->source->flags & YV12_FLAG_HIGHBITDEPTH) != 0; + if (aom_alloc_frame_buffer(&cpi->orig_source, cpi->source->y_crop_width, + cpi->source->y_crop_height, + cpi->source->subsampling_x, + cpi->source->subsampling_y, use_highbitdepth, + cpi->oxcf.border_in_pixels, + cm->features.byte_alignment, false, 0)) aom_internal_error(cm->error, AOM_CODEC_MEM_ERROR, - "Failed to allocate scaled buffer"); + "Failed to allocate cpi->orig_source buffer"); } aom_yv12_copy_y(cpi->source, &cpi->orig_source, 1); @@ -3175,6 +3190,11 @@ // frame. if (!frame_is_intra_only(cm)) av1_pick_and_set_high_precision_mv(cpi, q); + if (svc->temporal_layer_id == 0) { + cpi->rc.num_col_blscroll_last_tl0 = 0; + cpi->rc.num_row_blscroll_last_tl0 = 0; + } + // transform / motion compensation build reconstruction frame av1_encode_frame(cpi); @@ -3422,6 +3442,38 @@ } } + if (cpi->ext_ratectrl.ready && + (cpi->ext_ratectrl.funcs.rc_type & AOM_RC_QP) != 0 && + cpi->ext_ratectrl.funcs.get_encodeframe_decision != NULL) { + aom_codec_err_t codec_status; + aom_rc_encodeframe_decision_t encode_frame_decision; + const int sb_rows = CEIL_POWER_OF_TWO(cm->mi_params.mi_rows, + cm->seq_params->mib_size_log2); + const int sb_cols = CEIL_POWER_OF_TWO(cm->mi_params.mi_cols, + cm->seq_params->mib_size_log2); + // This assumes frame sizes don't change when used with external RC. + // cpi->ext_ratectrl is zero'ed at init. + if (cpi->ext_ratectrl.sb_params_list == NULL) { + CHECK_MEM_ERROR( + cm, cpi->ext_ratectrl.sb_params_list, + (aom_sb_params *)aom_calloc( + sb_rows * sb_cols, sizeof(*cpi->ext_ratectrl.sb_params_list))); + } + encode_frame_decision.sb_params_list = cpi->ext_ratectrl.sb_params_list; + encode_frame_decision.use_delta_q = &cpi->ext_ratectrl.use_delta_q; + codec_status = av1_extrc_get_encodeframe_decision( + &cpi->ext_ratectrl, cpi->gf_frame_index, &encode_frame_decision); + if (codec_status != AOM_CODEC_OK) { + aom_internal_error(cm->error, codec_status, + "av1_extrc_get_encodeframe_decision() failed"); + } + // If the external model recommends a reserved value, we use the default + // q. + if (encode_frame_decision.q_index != AOM_DEFAULT_Q) { + q = encode_frame_decision.q_index; + } + } + av1_set_quantizer(cm, q_cfg->qm_minlevel, q_cfg->qm_maxlevel, q, q_cfg->enable_chroma_deltaq, q_cfg->enable_hdr_deltaq, oxcf->mode == ALLINTRA, oxcf->tune_cfg.tuning); @@ -3563,6 +3615,13 @@ // Ducky encode currently does not support recode loop. loop = 0; } + + // Do not recode if external rate control is used. + if (cpi->ext_ratectrl.ready && + (cpi->ext_ratectrl.funcs.rc_type & AOM_RC_QP) != 0 && + cpi->ext_ratectrl.funcs.get_encodeframe_decision != NULL) { + loop = 0; + } #if CONFIG_BITRATE_ACCURACY || CONFIG_RD_COMMAND loop = 0; // turn off recode loop when CONFIG_BITRATE_ACCURACY is on #endif // CONFIG_BITRATE_ACCURACY || CONFIG_RD_COMMAND @@ -3981,6 +4040,7 @@ section->frame_avg_wavelet_energy -= frame->frame_avg_wavelet_energy; section->coded_error -= frame->coded_error; section->sr_coded_error -= frame->sr_coded_error; + section->lt_coded_error -= frame->lt_coded_error; section->pcnt_inter -= frame->pcnt_inter; section->pcnt_motion -= frame->pcnt_motion; section->pcnt_second_ref -= frame->pcnt_second_ref; @@ -4512,9 +4572,50 @@ cpi->ppi->gf_group.layer_depth[cpi->gf_frame_index], current_frame->display_order_hint, cpi->ppi->gf_group.max_layer_depth); + const GF_GROUP *gf_group = &cpi->ppi->gf_group; + // Check if this is the last frame in the gop. If so, make a copy of the + // source for TPL. + if (cpi->oxcf.algo_cfg.enable_tpl_model && + gf_group->update_type[cpi->gf_frame_index] != OVERLAY_UPDATE && + gf_group->update_type[cpi->gf_frame_index] != INTNL_OVERLAY_UPDATE) { + int is_last = 1; + for (int i = 0; i < gf_group->size; ++i) { + if (gf_group->display_idx[i] > + (int64_t)current_frame->display_order_hint) { + is_last = 0; + break; + } + } + if (is_last) { + cpi->ppi->tpl_data.prev_gop_arf_disp_order = -1; + const AV1EncoderConfig *const oxcf = &cpi->oxcf; + int ret = aom_realloc_frame_buffer( + &cpi->ppi->tpl_data.prev_gop_arf_src, oxcf->frm_dim_cfg.width, + oxcf->frm_dim_cfg.height, cm->seq_params->subsampling_x, + cm->seq_params->subsampling_y, cm->seq_params->use_highbitdepth, + cpi->oxcf.border_in_pixels, cm->features.byte_alignment, NULL, NULL, + NULL, cpi->alloc_pyramid, 0); + if (ret) + aom_internal_error(cm->error, AOM_CODEC_MEM_ERROR, + "Failed to allocate tpl prev_gop_arf_src buf."); + + // Currently it is not supported if source/refernece is resized. + if (cpi->source->y_width == cpi->ppi->tpl_data.prev_gop_arf_src.y_width && + cpi->source->y_height == + cpi->ppi->tpl_data.prev_gop_arf_src.y_height) { + // Copy the content from source to this buffer for next gop. + aom_yv12_copy_frame(cpi->source, &cpi->ppi->tpl_data.prev_gop_arf_src, + av1_num_planes(cm)); + + cpi->ppi->tpl_data.prev_gop_arf_disp_order = + current_frame->display_order_hint; + } + } + } + if (is_stat_generation_stage(cpi)) { #if !CONFIG_REALTIME_ONLY - if (cpi->oxcf.q_cfg.use_fixed_qp_offsets) + if (cpi->oxcf.q_cfg.use_fixed_qp_offsets != 0) av1_noop_first_pass_frame(cpi, frame_input->ts_duration); else av1_first_pass(cpi, frame_input->ts_duration); @@ -4529,6 +4630,16 @@ return AOM_CODEC_ERROR; } + if (cpi->ext_ratectrl.ready && + cpi->ext_ratectrl.funcs.update_encodeframe_result != NULL) { + aom_codec_err_t codec_status = av1_extrc_update_encodeframe_result( + &cpi->ext_ratectrl, (*frame_size) << 3, cm->quant_params.base_qindex); + if (codec_status != AOM_CODEC_OK) { + aom_internal_error(cm->error, codec_status, + "av1_extrc_update_encodeframe_result() failed"); + } + } + return AOM_CODEC_OK; } @@ -4577,6 +4688,33 @@ const int subsampling_y = sd->subsampling_y; const int use_highbitdepth = (sd->flags & YV12_FLAG_HIGHBITDEPTH) != 0; + // Note: Regarding profile setting, the following checks are added to help + // choose a proper profile for the input video. The criterion is that all + // bitstreams must be designated as the lowest profile that match its content. + // E.G. A bitstream that contains 4:4:4 video must be designated as High + // Profile in the seq header, and likewise a bitstream that contains 4:2:2 + // bitstream must be designated as Professional Profile in the sequence + // header. + if ((seq_params->profile == PROFILE_0) && !seq_params->monochrome && + (subsampling_x != 1 || subsampling_y != 1)) { + aom_set_error(cm->error, AOM_CODEC_INVALID_PARAM, + "Non-4:2:0 color format requires profile 1 or 2"); + return -1; + } + if ((seq_params->profile == PROFILE_1) && + !(subsampling_x == 0 && subsampling_y == 0)) { + aom_set_error(cm->error, AOM_CODEC_INVALID_PARAM, + "Profile 1 requires 4:4:4 color format"); + return -1; + } + if ((seq_params->profile == PROFILE_2) && + (seq_params->bit_depth <= AOM_BITS_10) && + !(subsampling_x == 1 && subsampling_y == 0)) { + aom_set_error(cm->error, AOM_CODEC_INVALID_PARAM, + "Profile 2 bit-depth <= 10 requires 4:2:2 color format"); + return -1; + } + #if CONFIG_TUNE_VMAF if (!is_stat_generation_stage(cpi) && cpi->oxcf.tune_cfg.tuning == AOM_TUNE_VMAF_WITH_PREPROCESSING) { @@ -4639,33 +4777,6 @@ cpi->ppi->total_time_receive_data += aom_usec_timer_elapsed(&timer); #endif - // Note: Regarding profile setting, the following checks are added to help - // choose a proper profile for the input video. The criterion is that all - // bitstreams must be designated as the lowest profile that match its content. - // E.G. A bitstream that contains 4:4:4 video must be designated as High - // Profile in the seq header, and likewise a bitstream that contains 4:2:2 - // bitstream must be designated as Professional Profile in the sequence - // header. - if ((seq_params->profile == PROFILE_0) && !seq_params->monochrome && - (subsampling_x != 1 || subsampling_y != 1)) { - aom_set_error(cm->error, AOM_CODEC_INVALID_PARAM, - "Non-4:2:0 color format requires profile 1 or 2"); - res = -1; - } - if ((seq_params->profile == PROFILE_1) && - !(subsampling_x == 0 && subsampling_y == 0)) { - aom_set_error(cm->error, AOM_CODEC_INVALID_PARAM, - "Profile 1 requires 4:4:4 color format"); - res = -1; - } - if ((seq_params->profile == PROFILE_2) && - (seq_params->bit_depth <= AOM_BITS_10) && - !(subsampling_x == 1 && subsampling_y == 0)) { - aom_set_error(cm->error, AOM_CODEC_INVALID_PARAM, - "Profile 2 bit-depth <= 10 requires 4:2:2 color format"); - res = -1; - } - return res; } @@ -4955,16 +5066,10 @@ static inline void update_gf_group_index(AV1_COMP *cpi) { // Increment the gf group index ready for the next frame. - if (is_one_pass_rt_params(cpi) && - cpi->svc.spatial_layer_id == cpi->svc.number_spatial_layers - 1) { - ++cpi->gf_frame_index; - // Reset gf_frame_index in case it reaches MAX_STATIC_GF_GROUP_LENGTH - // for real time encoding. - if (cpi->gf_frame_index == MAX_STATIC_GF_GROUP_LENGTH) - cpi->gf_frame_index = 0; - } else { - ++cpi->gf_frame_index; - } + ++cpi->gf_frame_index; + // Reset gf_frame_index in case it reaches MAX_STATIC_GF_GROUP_LENGTH. + if (cpi->gf_frame_index == MAX_STATIC_GF_GROUP_LENGTH) + cpi->gf_frame_index = 0; } static void update_fb_of_context_type(const AV1_COMP *const cpi, @@ -5074,6 +5179,25 @@ AV1_PRIMARY *const ppi = cpi->ppi; AV1_COMMON *const cm = &cpi->common; + if (ppi->b_freeze_internal_state) { + // Should not update encoder state, just necessary work to get the + // expected output and then return early. + + // Note *size = 0 indicates a dropped frame for which psnr is not calculated + if (ppi->b_calculate_psnr && cpi_data->frame_size > 0) { + if (cm->show_existing_frame || + (!is_stat_generation_stage(cpi) && cm->show_frame)) { + generate_psnr_packet(cpi); + } + } + + if (cpi_data->pop_lookahead == 1) { + av1_lookahead_pop(cpi->ppi->lookahead, cpi_data->flush, + cpi->compressor_stage); + } + return; + } + update_gm_stats(cpi); #if !CONFIG_REALTIME_ONLY
diff --git a/av1/encoder/encoder.h b/av1/encoder/encoder.h index ab19b1c..77df0a9 100644 --- a/av1/encoder/encoder.h +++ b/av1/encoder/encoder.h
@@ -34,6 +34,7 @@ #include "av1/common/timing.h" #include "av1/encoder/aq_cyclicrefresh.h" +#include "av1/encoder/av1_ext_ratectrl.h" #include "av1/encoder/av1_quantize.h" #include "av1/encoder/block.h" #include "av1/encoder/context_tree.h" @@ -166,9 +167,8 @@ DELTA_Q_PERCEPTUAL = 2, // Modulation to improve video perceptual quality DELTA_Q_PERCEPTUAL_AI = 3, // Perceptual quality opt for all intra mode DELTA_Q_USER_RATING_BASED = 4, // User rating based delta q mode - DELTA_Q_HDR = 5, // QP adjustment based on HDR block pixel average - DELTA_Q_VARIANCE_BOOST = - 6, // Variance Boost style modulation for all intra mode + DELTA_Q_HDR = 5, // QP adjustment based on HDR block pixel average + DELTA_Q_VARIANCE_BOOST = 6, // Variance Boost style modulation DELTA_Q_MODE_COUNT // This should always be the last member of the enum } UENUM1BYTE(DELTAQ_MODE); @@ -790,9 +790,10 @@ } InputCfg; typedef struct { - // If true, encoder will use fixed QP offsets, that are either: - // - Given by the user, and stored in 'fixed_qp_offsets' array, OR - // - Picked automatically from cq_level. + // Controls how the encoder applies fixed QP offsets. + // If the value is 0, QP offsets are chosen adaptively. + // If the value is 1, fixed QP offsets are picked automatically from cq_level. + // If the value is 2, no QP offsets will be applied. int use_fixed_qp_offsets; // Indicates the minimum flatness of the quantization matrix. int qm_minlevel; @@ -827,9 +828,10 @@ * For values 1-7, eob and skip block optimization are * avoided and rdmult is adjusted in favor of block sharpness. * - * In all-intra mode: it also sets the `loop_filter_sharpness` syntax element - * in the bitstream. Larger values increasingly reduce how much the filtering - * can change the sample values on block edges to favor perceived sharpness. + * In all-intra mode or tune IQ or SSIMULACRA2: it also sets the + * `loop_filter_sharpness` syntax element in the bitstream. Larger values + * increasingly reduce how much the filtering can change the sample values on + * block edges to favor perceived sharpness. */ int sharpness; @@ -1488,6 +1490,7 @@ CONV_BUF_TYPE *tmp_conv_dst; uint64_t abs_sum_level; uint8_t *tmp_pred_bufs[2]; + uint8_t *upsample_pred; uint8_t *wiener_tmp_pred_buf; int intrabc_used; int deltaq_used; @@ -2379,12 +2382,21 @@ double max_scale; } WeberStats; +/*! + * \brief This structure stores different types of frame indices. + */ +typedef struct { + int show_frame_count; +} FRAME_INDEX_SET; + typedef struct { struct loopfilter lf; CdefInfo cdef_info; YV12_BUFFER_CONFIG copy_buffer; RATE_CONTROL rc; MV_STATS mv_stats; + unsigned int frame_number; + FRAME_INDEX_SET frame_index_set; } CODING_CONTEXT; typedef struct { @@ -2400,13 +2412,6 @@ int subsampling_y; } FRAME_INFO; -/*! - * \brief This structure stores different types of frame indices. - */ -typedef struct { - int show_frame_count; -} FRAME_INDEX_SET; - /*!\endcond */ /*! @@ -2888,6 +2893,13 @@ * when --deltaq-mode=3. */ AV1EncRowMultiThreadSync intra_row_mt_sync; + + /*! + * If set to 1, the encoder should not update any internal state after + * completing the encode. E.g. no updates to reference buffers, CDF + * tables or RC state. + */ + int b_freeze_internal_state; } AV1_PRIMARY; /*! @@ -3328,6 +3340,13 @@ int prune_ref_frame_mask; /*! + * Mark the reference frames which are important (based on the temporal + * distance and quality) to prevent pruning the reference frame at block + * level. + */ + int keep_single_ref_frame_mask; + + /*! * Loop Restoration context. */ AV1LrStruct lr_ctxt; @@ -3697,6 +3716,16 @@ * ROI map. */ aom_roi_map_t roi; + + /*! + * External rate control. + */ + AOM_EXT_RATECTRL ext_ratectrl; + + /*! + * Store TPL stats before propagation + */ + AomTplGopStats extrc_tpl_gop_stats; } AV1_COMP; /*! @@ -3767,9 +3796,9 @@ void av1_initialize_enc(unsigned int usage, enum aom_rc_mode end_usage); -struct AV1_COMP *av1_create_compressor(AV1_PRIMARY *ppi, +struct AV1_COMP *av1_create_compressor(struct AV1_PRIMARY *ppi, const AV1EncoderConfig *oxcf, - BufferPool *const pool, + struct BufferPool *const pool, COMPRESSOR_STAGE stage, int lap_lag_in_frames); @@ -4107,7 +4136,8 @@ static inline int is_stat_consumption_stage(const AV1_COMP *const cpi) { return (is_stat_consumption_stage_twopass(cpi) || (cpi->oxcf.pass == AOM_RC_ONE_PASS && - (cpi->compressor_stage == ENCODE_STAGE) && cpi->ppi->lap_enabled)); + (cpi->compressor_stage == ENCODE_STAGE) && cpi->ppi->lap_enabled && + cpi->oxcf.mode != REALTIME)); } // Decide whether 'dv_costs' need to be allocated/stored during the encoding. @@ -4128,11 +4158,17 @@ static inline int has_no_stats_stage(const AV1_COMP *const cpi) { assert( IMPLIES(!cpi->ppi->lap_enabled, cpi->compressor_stage == ENCODE_STAGE)); - return (cpi->oxcf.pass == AOM_RC_ONE_PASS && !cpi->ppi->lap_enabled); + return (cpi->oxcf.pass == AOM_RC_ONE_PASS && + (!cpi->ppi->lap_enabled || cpi->oxcf.mode == REALTIME)); } /*!\cond */ +static inline int is_one_pass_rt_lag_params(const AV1_COMP *cpi) { + return cpi->oxcf.pass == AOM_RC_ONE_PASS && + cpi->oxcf.gf_cfg.lag_in_frames > 0 && cpi->oxcf.mode == REALTIME; +} + static inline int is_one_pass_rt_params(const AV1_COMP *cpi) { return has_no_stats_stage(cpi) && cpi->oxcf.gf_cfg.lag_in_frames == 0 && (cpi->oxcf.mode == REALTIME || cpi->svc.number_spatial_layers > 1);
diff --git a/av1/encoder/encoder_alloc.h b/av1/encoder/encoder_alloc.h index 13c3241..97b5273 100644 --- a/av1/encoder/encoder_alloc.h +++ b/av1/encoder/encoder_alloc.h
@@ -333,6 +333,7 @@ aom_free(cpi->td.mb.palette_buffer); release_compound_type_rd_buffers(&cpi->td.mb.comp_rd_buffer); aom_free(cpi->td.mb.tmp_conv_dst); + aom_free(cpi->td.mb.upsample_pred); for (int j = 0; j < 2; ++j) { aom_free(cpi->td.mb.tmp_pred_bufs[j]); } @@ -375,6 +376,10 @@ aom_free(cpi->mb_delta_q); cpi->mb_delta_q = NULL; + +#if !CONFIG_REALTIME_ONLY + av1_free_tpl_gop_stats(&cpi->extrc_tpl_gop_stats); +#endif } static inline void allocate_gradient_info_for_hog(AV1_COMP *cpi) { @@ -471,6 +476,7 @@ aom_free(td->tctx); aom_free(td->palette_buffer); aom_free(td->tmp_conv_dst); + aom_free(td->upsample_pred); release_compound_type_rd_buffers(&td->comp_rd_buffer); for (int j = 0; j < 2; ++j) { aom_free(td->tmp_pred_bufs[j]);
diff --git a/av1/encoder/encoder_utils.c b/av1/encoder/encoder_utils.c index 89ba731..2bc36d3 100644 --- a/av1/encoder/encoder_utils.c +++ b/av1/encoder/encoder_utils.c
@@ -9,6 +9,7 @@ * PATENTS file, you can obtain it at www.aomedia.org/license/patent. */ +#include <assert.h> #include <string.h> #include "aom/aomcx.h" @@ -681,55 +682,65 @@ } #endif - // Decide q and q bounds. - *q = av1_rc_pick_q_and_bounds(cpi, cm->width, cm->height, cpi->gf_frame_index, - bottom_index, top_index); + if (cpi->oxcf.q_cfg.use_fixed_qp_offsets == 2 && + cpi->oxcf.rc_cfg.mode == AOM_Q) { + // Disable scaling, and use the same q for all frames of the pyramid + *q = cpi->oxcf.rc_cfg.cq_level; + *top_index = *bottom_index = *q; + cpi->ppi->p_rc.arf_q = *q; + } else { + // Decide q and q bounds. + *q = av1_rc_pick_q_and_bounds(cpi, cm->width, cm->height, + cpi->gf_frame_index, bottom_index, top_index); - if (cpi->oxcf.rc_cfg.mode == AOM_CBR && cpi->rc.force_max_q) { - *q = cpi->rc.worst_quality; - cpi->rc.force_max_q = 0; - } + if (cpi->oxcf.rc_cfg.mode == AOM_CBR && cpi->rc.force_max_q) { + *q = cpi->rc.worst_quality; + cpi->rc.force_max_q = 0; + } #if !CONFIG_REALTIME_ONLY - if (cpi->oxcf.rc_cfg.mode == AOM_Q && - cpi->ppi->tpl_data.tpl_frame[cpi->gf_frame_index].is_valid && - !is_lossless_requested(&cpi->oxcf.rc_cfg)) { - const RateControlCfg *const rc_cfg = &cpi->oxcf.rc_cfg; - const int tpl_q = av1_tpl_get_q_index( - &cpi->ppi->tpl_data, cpi->gf_frame_index, cpi->rc.active_worst_quality, - cm->seq_params->bit_depth); - *q = clamp(tpl_q, rc_cfg->best_allowed_q, rc_cfg->worst_allowed_q); - *top_index = *bottom_index = *q; - if (gf_group->update_type[cpi->gf_frame_index] == ARF_UPDATE) - cpi->ppi->p_rc.arf_q = *q; - } - - if (cpi->oxcf.q_cfg.use_fixed_qp_offsets && cpi->oxcf.rc_cfg.mode == AOM_Q) { - if (is_frame_tpl_eligible(gf_group, cpi->gf_frame_index)) { - const double qratio_grad = - cpi->ppi->p_rc.baseline_gf_interval > 20 ? 0.2 : 0.3; - const double qstep_ratio = - 0.2 + - (1.0 - (double)cpi->rc.active_worst_quality / MAXQ) * qratio_grad; - *q = av1_get_q_index_from_qstep_ratio( - cpi->rc.active_worst_quality, qstep_ratio, cm->seq_params->bit_depth); + if (cpi->oxcf.rc_cfg.mode == AOM_Q && + cpi->ppi->tpl_data.tpl_frame[cpi->gf_frame_index].is_valid && + !is_lossless_requested(&cpi->oxcf.rc_cfg)) { + const RateControlCfg *const rc_cfg = &cpi->oxcf.rc_cfg; + const int tpl_q = av1_tpl_get_q_index( + &cpi->ppi->tpl_data, cpi->gf_frame_index, + cpi->rc.active_worst_quality, cm->seq_params->bit_depth); + *q = clamp(tpl_q, rc_cfg->best_allowed_q, rc_cfg->worst_allowed_q); *top_index = *bottom_index = *q; - if (gf_group->update_type[cpi->gf_frame_index] == ARF_UPDATE || - gf_group->update_type[cpi->gf_frame_index] == KF_UPDATE || - gf_group->update_type[cpi->gf_frame_index] == GF_UPDATE) + if (gf_group->update_type[cpi->gf_frame_index] == ARF_UPDATE) cpi->ppi->p_rc.arf_q = *q; - } else if (gf_group->layer_depth[cpi->gf_frame_index] < - gf_group->max_layer_depth) { - int this_height = gf_group->layer_depth[cpi->gf_frame_index]; - int arf_q = cpi->ppi->p_rc.arf_q; - while (this_height > 1) { - arf_q = (arf_q + cpi->oxcf.rc_cfg.cq_level + 1) / 2; - --this_height; - } - *top_index = *bottom_index = *q = arf_q; } - } + + if (cpi->oxcf.q_cfg.use_fixed_qp_offsets == 1 && + cpi->oxcf.rc_cfg.mode == AOM_Q) { + if (is_frame_tpl_eligible(gf_group, cpi->gf_frame_index)) { + const double qratio_grad = + cpi->ppi->p_rc.baseline_gf_interval > 20 ? 0.2 : 0.3; + const double qstep_ratio = + 0.2 + + (1.0 - (double)cpi->rc.active_worst_quality / MAXQ) * qratio_grad; + *q = av1_get_q_index_from_qstep_ratio(cpi->rc.active_worst_quality, + qstep_ratio, + cm->seq_params->bit_depth); + *top_index = *bottom_index = *q; + if (gf_group->update_type[cpi->gf_frame_index] == ARF_UPDATE || + gf_group->update_type[cpi->gf_frame_index] == KF_UPDATE || + gf_group->update_type[cpi->gf_frame_index] == GF_UPDATE) + cpi->ppi->p_rc.arf_q = *q; + } else if (gf_group->layer_depth[cpi->gf_frame_index] < + gf_group->max_layer_depth) { + int this_height = gf_group->layer_depth[cpi->gf_frame_index]; + int arf_q = cpi->ppi->p_rc.arf_q; + while (this_height > 1) { + arf_q = (arf_q + cpi->oxcf.rc_cfg.cq_level + 1) / 2; + --this_height; + } + *top_index = *bottom_index = *q = arf_q; + } + } #endif + } // Configure experimental use of segmentation for enhanced coding of // static regions if indicated. @@ -738,6 +749,9 @@ if (is_stat_consumption_stage_twopass(cpi) && cpi->sf.hl_sf.static_segmentation) configure_static_seg_features(cpi); + + if (cpi->oxcf.rc_cfg.over_shoot_pct == 0) *top_index = MAXQ; + if (cpi->oxcf.rc_cfg.under_shoot_pct == 0) *bottom_index = MINQ; } #if !CONFIG_REALTIME_ONLY @@ -1314,6 +1328,8 @@ if (!cm->seq_params->reduced_still_picture_hdr && encode_show_existing_frame(cm)) { + assert(cpi->existing_fb_idx_to_show >= 0 && + cpi->existing_fb_idx_to_show < REF_FRAMES); RefCntBuffer *const frame_to_show = cm->ref_frame_map[cpi->existing_fb_idx_to_show]; @@ -1544,6 +1560,8 @@ cc->cdef_info = cm->cdef_info; cc->rc = cpi->rc; cc->mv_stats = cpi->ppi->mv_stats; + cc->frame_number = cpi->common.current_frame.frame_number; + cc->frame_index_set = cpi->frame_index_set; } void av1_save_all_coding_context(AV1_COMP *cpi) {
diff --git a/av1/encoder/encoder_utils.h b/av1/encoder/encoder_utils.h index ca33eec..8691e48 100644 --- a/av1/encoder/encoder_utils.h +++ b/av1/encoder/encoder_utils.h
@@ -889,6 +889,8 @@ restore_cdef_coding_context(&cm->cdef_info, &cc->cdef_info); cpi->rc = cc->rc; cpi->ppi->mv_stats = cc->mv_stats; + cpi->common.current_frame.frame_number = cc->frame_number; + cpi->frame_index_set = cc->frame_index_set; } static inline int equal_dimensions_and_border(const YV12_BUFFER_CONFIG *a,
diff --git a/av1/encoder/ethread.c b/av1/encoder/ethread.c index fb6a13a..4f5fe5d 100644 --- a/av1/encoder/ethread.c +++ b/av1/encoder/ethread.c
@@ -965,6 +965,7 @@ assert(p_mt_info->workers != NULL); assert(p_mt_info->tile_thr_data != NULL); + const int is_highbitdepth = ppi->seq_params.use_highbitdepth; int num_workers = p_mt_info->num_workers; int num_enc_workers = av1_get_num_mod_workers_for_alloc(p_mt_info, MOD_ENC); assert(num_enc_workers <= num_workers); @@ -994,6 +995,11 @@ "Failed to allocate PICK_MODE_CONTEXT"); } + AOM_CHECK_MEM_ERROR( + &ppi->error, td->upsample_pred, + aom_memalign(16, (1 + is_highbitdepth) * ((MAX_SB_SIZE + 16) + 16) * + MAX_SB_SIZE * sizeof(*td->upsample_pred))); + if (!is_first_pass && i < num_enc_workers) { // Set up sms_tree. if (av1_setup_sms_tree(ppi->cpi, td)) { @@ -1635,6 +1641,7 @@ thread_data->td->mb.palette_buffer = thread_data->td->palette_buffer; thread_data->td->mb.comp_rd_buffer = thread_data->td->comp_rd_buffer; thread_data->td->mb.tmp_conv_dst = thread_data->td->tmp_conv_dst; + thread_data->td->mb.upsample_pred = thread_data->td->upsample_pred; for (int j = 0; j < 2; ++j) { thread_data->td->mb.tmp_pred_bufs[j] = thread_data->td->tmp_pred_bufs[j]; @@ -1646,6 +1653,8 @@ thread_data->td->src_var_info_of_4x4_sub_blocks; thread_data->td->mb.e_mbd.tmp_conv_dst = thread_data->td->mb.tmp_conv_dst; + thread_data->td->mb.e_mbd.tmp_upsample_pred = + thread_data->td->mb.upsample_pred; for (int j = 0; j < 2; ++j) { thread_data->td->mb.e_mbd.tmp_obmc_bufs[j] = thread_data->td->mb.tmp_pred_bufs[j]; @@ -2296,6 +2305,9 @@ } thread_data->td->mb.tmp_conv_dst = thread_data->td->tmp_conv_dst; thread_data->td->mb.e_mbd.tmp_conv_dst = thread_data->td->mb.tmp_conv_dst; + thread_data->td->mb.upsample_pred = thread_data->td->upsample_pred; + thread_data->td->mb.e_mbd.tmp_upsample_pred = + thread_data->td->mb.upsample_pred; } } } @@ -2472,6 +2484,9 @@ aom_internal_error(cpi->common.error, AOM_CODEC_MEM_ERROR, "Error allocating temporal filter data"); } + thread_data->td->mb.upsample_pred = thread_data->td->upsample_pred; + thread_data->td->mb.e_mbd.tmp_upsample_pred = + thread_data->td->mb.upsample_pred; } } }
diff --git a/av1/encoder/firstpass.c b/av1/encoder/firstpass.c index 4436f00..e8c15cb 100644 --- a/av1/encoder/firstpass.c +++ b/av1/encoder/firstpass.c
@@ -97,6 +97,7 @@ section->log_intra_error = 0.0; section->log_coded_error = 0.0; section->sr_coded_error = 0.0; + section->lt_coded_error = 0.0; section->pcnt_inter = 0.0; section->pcnt_motion = 0.0; section->pcnt_second_ref = 0.0; @@ -129,6 +130,7 @@ section->frame_avg_wavelet_energy += frame->frame_avg_wavelet_energy; section->coded_error += frame->coded_error; section->sr_coded_error += frame->sr_coded_error; + section->lt_coded_error += frame->lt_coded_error; section->pcnt_inter += frame->pcnt_inter; section->pcnt_motion += frame->pcnt_motion; section->pcnt_second_ref += frame->pcnt_second_ref; @@ -254,7 +256,8 @@ // for first pass test. static int get_search_range(int width, int height) { int sr = 0; - const int dim = AOMMIN(width, height); + int dim = AOMMIN(width, height); + dim = AOMMAX(dim, MI_SIZE); while ((dim << sr) < MAX_FULL_PEL_VAL) ++sr; return sr; @@ -520,6 +523,7 @@ xd->lossless[xd->mi[0]->segment_id] = (qindex == 0); xd->mi[0]->mode = DC_PRED; xd->mi[0]->tx_size = TX_4X4; + xd->mi[0]->skip_txfm = 0; if (cpi->sf.fp_sf.disable_recon) first_pass_predict_intra_block_for_luma_plane(seq_params, x, bsize); @@ -679,10 +683,10 @@ // golden_frame: the frame buffer of the golden frame. // unit_row: row index in the unit of first pass block size. // unit_col: column index in the unit of first pass block size. -// recon_yoffset: the y offset of the reconstructed frame buffer, +// recon_yoffset: the y offset of the reconstructed frame buffer, // indicating the starting point of the current block. -// recont_uvoffset: the u/v offset of the reconstructed frame buffer, -// indicating the starting point of the current block. +// recon_uvoffset: the u/v offset of the reconstructed frame buffer, +// indicating the starting point of the current block. // src_yoffset: the y offset of the source frame buffer. // fp_block_size: first pass block size. // this_intra_error: the intra prediction error of this block. @@ -701,7 +705,8 @@ // this_inter_error static int firstpass_inter_prediction( AV1_COMP *cpi, ThreadData *td, const YV12_BUFFER_CONFIG *const last_frame, - const YV12_BUFFER_CONFIG *const golden_frame, const int unit_row, + const YV12_BUFFER_CONFIG *const golden_frame, + const YV12_BUFFER_CONFIG *const last2_frame, const int unit_row, const int unit_col, const int recon_yoffset, const int recon_uvoffset, const int src_yoffset, const BLOCK_SIZE fp_block_size, const int this_intra_error, const int raw_motion_err_counts, @@ -769,6 +774,19 @@ } } + int last2_motion_error = motion_error; + if ((current_frame->frame_number > 2) && last2_frame != NULL) { + FULLPEL_MV tmp_mv = kZeroFullMv; + // Assume 0,0 motion with no mv overhead. + av1_setup_pre_planes(xd, 0, last2_frame, 0, 0, NULL, 1); + xd->plane[0].pre[0].buf += recon_yoffset; + last2_motion_error = + get_prediction_error_bitdepth(is_high_bitdepth, bitdepth, bsize, + &x->plane[0].src, &xd->plane[0].pre[0]); + first_pass_motion_search(cpi, x, &kZeroMv, &tmp_mv, &last2_motion_error); + stats->lt_coded_error += AOMMIN(last2_motion_error, this_intra_error); + } + // Motion search in 2nd reference frame. int gf_motion_error = motion_error; if ((current_frame->frame_number > 1) && golden_frame != NULL) { @@ -858,6 +876,7 @@ double f_h) { fps->coded_error /= num_mbs_16x16; fps->sr_coded_error /= num_mbs_16x16; + fps->lt_coded_error /= num_mbs_16x16; fps->intra_error /= num_mbs_16x16; fps->frame_avg_wavelet_energy /= num_mbs_16x16; fps->log_coded_error = log1p(fps->coded_error); @@ -913,6 +932,7 @@ fps.frame = frame_number; fps.coded_error = (double)(stats->coded_error >> 8) + min_err; fps.sr_coded_error = (double)(stats->sr_coded_error >> 8) + min_err; + fps.lt_coded_error = (double)(stats->lt_coded_error >> 8) + min_err; fps.intra_error = (double)(stats->intra_error >> 8) + min_err; fps.frame_avg_wavelet_energy = (double)stats->frame_avg_wavelet_energy; fps.count = 1.0; @@ -1032,6 +1052,7 @@ stats.new_mv_count += mb_stat.new_mv_count; stats.second_ref_count += mb_stat.second_ref_count; stats.sr_coded_error += mb_stat.sr_coded_error; + stats.lt_coded_error += mb_stat.lt_coded_error; stats.sum_in_vectors += mb_stat.sum_in_vectors; stats.sum_mvc += mb_stat.sum_mvc; stats.sum_mvc_abs += mb_stat.sum_mvc_abs; @@ -1125,8 +1146,8 @@ MACROBLOCKD *const xd = &x->e_mbd; TileInfo *tile = &tile_data->tile_info; const int qindex = find_fp_qindex(seq_params->bit_depth); - const int fp_block_size_width = block_size_high[fp_block_size]; - const int fp_block_size_height = block_size_wide[fp_block_size]; + const int fp_block_size_width = block_size_wide[fp_block_size]; + const int fp_block_size_height = block_size_high[fp_block_size]; const int unit_width = mi_size_wide[fp_block_size]; const int unit_width_log2 = mi_size_wide_log2[fp_block_size]; const int unit_height_log2 = mi_size_high_log2[fp_block_size]; @@ -1149,6 +1170,10 @@ if (!golden_frame) { golden_frame = get_ref_frame_yv12_buf(cm, GOLDEN_FRAME); } + const YV12_BUFFER_CONFIG *last2_frame = + av1_get_scaled_ref_frame(cpi, LAST2_FRAME); + if (!last2_frame) last2_frame = get_ref_frame_yv12_buf(cm, LAST2_FRAME); + YV12_BUFFER_CONFIG *const this_frame = &cm->cur_frame->buf; PICK_MODE_CONTEXT *ctx = td->firstpass_ctx; @@ -1169,6 +1194,8 @@ const int src_y_stride = cpi->source->y_stride; const int recon_y_stride = this_frame->y_stride; const int recon_uv_stride = this_frame->uv_stride; + const int uv_mb_width = + fp_block_size_width >> (this_frame->y_width > this_frame->uv_width); const int uv_mb_height = fp_block_size_height >> (this_frame->y_height > this_frame->uv_height); @@ -1182,7 +1209,7 @@ int src_yoffset = (unit_row * src_y_stride * fp_block_size_height) + (unit_col_start * fp_block_size_width); int recon_uvoffset = (unit_row * recon_uv_stride * uv_mb_height) + - (unit_col_start * uv_mb_height); + (unit_col_start * uv_mb_width); // Set up limit values for motion vectors to prevent them extending // outside the UMV borders. @@ -1222,10 +1249,10 @@ if (!frame_is_intra_only(cm)) { const int this_inter_error = firstpass_inter_prediction( - cpi, td, last_frame, golden_frame, unit_row, unit_col, recon_yoffset, - recon_uvoffset, src_yoffset, fp_block_size, this_intra_error, - raw_motion_err_counts, raw_motion_err_list, best_ref_mv, &best_ref_mv, - &last_mv, mb_stats); + cpi, td, last_frame, golden_frame, last2_frame, unit_row, unit_col, + recon_yoffset, recon_uvoffset, src_yoffset, fp_block_size, + this_intra_error, raw_motion_err_counts, raw_motion_err_list, + best_ref_mv, &best_ref_mv, &last_mv, mb_stats); if (unit_col_in_tile == 0) { *first_top_mv = last_mv; } @@ -1234,18 +1261,19 @@ } else { mb_stats->sr_coded_error += this_intra_error; mb_stats->coded_error += this_intra_error; + mb_stats->lt_coded_error += this_intra_error; } // Adjust to the next column of MBs. x->plane[0].src.buf += fp_block_size_width; if (num_planes > 1) { - x->plane[1].src.buf += uv_mb_height; - x->plane[2].src.buf += uv_mb_height; + x->plane[1].src.buf += uv_mb_width; + x->plane[2].src.buf += uv_mb_width; } recon_yoffset += fp_block_size_width; src_yoffset += fp_block_size_width; - recon_uvoffset += uv_mb_height; + recon_uvoffset += uv_mb_width; mb_stats++; enc_row_mt->sync_write_ptr(row_mt_sync, unit_row_in_tile, unit_col_in_tile, @@ -1288,7 +1316,8 @@ const int qindex = find_fp_qindex(seq_params->bit_depth); const int ref_frame_flags_backup = cpi->ref_frame_flags; cpi->ref_frame_flags = av1_ref_frame_flag_list[LAST_FRAME] | - av1_ref_frame_flag_list[GOLDEN_FRAME]; + av1_ref_frame_flag_list[GOLDEN_FRAME] | + av1_ref_frame_flag_list[LAST2_FRAME]; // Detect if the key frame is screen content type. if (frame_is_intra_only(cm)) { @@ -1342,6 +1371,8 @@ const YV12_BUFFER_CONFIG *last_frame = NULL; const YV12_BUFFER_CONFIG *golden_frame = NULL; + const YV12_BUFFER_CONFIG *last2_frame = NULL; + if (!frame_is_intra_only(cm)) { av1_scale_references(cpi, EIGHTTAP_REGULAR, 0, 0); last_frame = av1_is_scaled(get_ref_scale_factors_const(cm, LAST_FRAME)) @@ -1350,6 +1381,9 @@ golden_frame = av1_is_scaled(get_ref_scale_factors_const(cm, GOLDEN_FRAME)) ? av1_get_scaled_ref_frame(cpi, GOLDEN_FRAME) : get_ref_frame_yv12_buf(cm, GOLDEN_FRAME); + last2_frame = av1_is_scaled(get_ref_scale_factors_const(cm, LAST2_FRAME)) + ? av1_get_scaled_ref_frame(cpi, LAST2_FRAME) + : get_ref_frame_yv12_buf(cm, LAST2_FRAME); } YV12_BUFFER_CONFIG *const this_frame = &cm->cur_frame->buf; @@ -1439,6 +1473,12 @@ current_frame->frame_number, ts_duration, fp_block_size); + if (this_frame_stats->pcnt_inter < 0.2 && last2_frame != NULL) { + assign_frame_buffer_p( + &cm->ref_frame_map[get_ref_frame_map_idx(cm, LAST2_FRAME)], + cm->ref_frame_map[get_ref_frame_map_idx(cm, GOLDEN_FRAME)]); + } + // Copy the previous Last Frame back into gf buffer if the prediction is good // enough... but also don't allow it to lag too far. if ((twopass->sr_update_lag > 3) || @@ -1471,6 +1511,13 @@ cm->ref_frame_map[get_ref_frame_map_idx(cm, LAST_FRAME)]); } + if (current_frame->frame_number == 0 && + get_ref_frame_map_idx(cm, LAST2_FRAME) != INVALID_IDX) { + assign_frame_buffer_p( + &cm->ref_frame_map[get_ref_frame_map_idx(cm, LAST2_FRAME)], + cm->ref_frame_map[get_ref_frame_map_idx(cm, LAST_FRAME)]); + } + print_reconstruction_frame(last_frame, current_frame->frame_number, /*do_print=*/0);
diff --git a/av1/encoder/firstpass.h b/av1/encoder/firstpass.h index a1b7e7d..639eda7 100644 --- a/av1/encoder/firstpass.h +++ b/av1/encoder/firstpass.h
@@ -69,6 +69,10 @@ */ double sr_coded_error; /*! + * Best of intra pred error and inter pred error using long term frame as ref. + */ + double lt_coded_error; + /*! * Percentage of blocks with inter pred error < intra pred error. */ double pcnt_inter; @@ -393,6 +397,9 @@ int skip_frame_as_ref[MAX_STATIC_GF_GROUP_LENGTH]; // Indicates whether a switch frame is due. bool is_sframe_due; + // Indicates whether the ref frame map is overridden by the external rate + // control. + int use_ext_ref_frame_map[MAX_STATIC_GF_GROUP_LENGTH]; /*!\endcond */ } GF_GROUP; /*!\cond */ @@ -482,6 +489,9 @@ int64_t coded_error; // Best of intra pred error and inter pred error using golden frame as ref. int64_t sr_coded_error; + // Best of coded error using long term reference. + int64_t lt_coded_error; + // Count of motion vector. int mv_count; // Count of blocks that pick inter prediction (inter pred error is smaller
diff --git a/av1/encoder/gop_structure.c b/av1/encoder/gop_structure.c index 15ec0eb..01c3e76 100644 --- a/av1/encoder/gop_structure.c +++ b/av1/encoder/gop_structure.c
@@ -17,10 +17,12 @@ #include "aom/aom_codec.h" #include "aom/aom_encoder.h" +#include "aom/aom_ext_ratectrl.h" #include "av1/common/av1_common_int.h" #include "av1/encoder/encoder.h" +#include "av1/encoder/av1_ext_ratectrl.h" #include "av1/encoder/firstpass.h" #include "av1/encoder/gop_structure.h" #include "av1/encoder/pass2_strategy.h" @@ -65,7 +67,7 @@ GF_GROUP *const gf_group, int *cur_frame_idx, int *frame_ind, int *parallel_frame_count, int max_parallel_frames, int do_frame_parallel_encode, int *first_frame_index, int *cur_disp_index, - int layer_depth, int start, int end) { + int layer_depth, int start, int end, const bool scale_max_boost) { gf_group->update_type[*frame_ind] = LF_UPDATE; gf_group->arf_src_offset[*frame_ind] = 0; gf_group->cur_frame_idx[*frame_ind] = *cur_frame_idx; @@ -76,7 +78,7 @@ gf_group->display_idx[*frame_ind] = (*cur_disp_index); gf_group->arf_boost[*frame_ind] = av1_calc_arf_boost(twopass, twopass_frame, p_rc, frame_info, start, - end - start, 0, NULL, NULL, 0); + end - start, 0, NULL, NULL, 0, scale_max_boost); ++(*cur_disp_index); // Set the level of parallelism for the LF_UPDATE frame. @@ -118,7 +120,7 @@ int *parallel_frame_count, int max_parallel_frames, int do_frame_parallel_encode, int *first_frame_index, int depth_thr, int *cur_disp_idx, int layer_depth, int arf_src_offset, int offset, - int f_frames, int b_frames) { + int f_frames, int b_frames, const bool scale_max_boost) { gf_group->update_type[*frame_ind] = INTNL_ARF_UPDATE; gf_group->arf_src_offset[*frame_ind] = arf_src_offset; gf_group->cur_frame_idx[*frame_ind] = *cur_frame_idx; @@ -129,7 +131,7 @@ (*cur_disp_idx) + gf_group->arf_src_offset[*frame_ind]; gf_group->arf_boost[*frame_ind] = av1_calc_arf_boost(twopass, twopass_frame, p_rc, frame_info, offset, - f_frames, b_frames, NULL, NULL, 0); + f_frames, b_frames, NULL, NULL, 0, scale_max_boost); if (do_frame_parallel_encode) { if (depth_thr != INT_MAX) { @@ -177,7 +179,8 @@ RATE_CONTROL *rc, FRAME_INFO *frame_info, int start, int end, int *cur_frame_idx, int *frame_ind, int *parallel_frame_count, int max_parallel_frames, int do_frame_parallel_encode, - int *first_frame_index, int depth_thr, int *cur_disp_idx, int layer_depth) { + int *first_frame_index, int depth_thr, int *cur_disp_idx, int layer_depth, + const bool scale_max_boost) { const int num_frames_to_process = end - start; // Either we are at the last level of the pyramid, or we don't have enough @@ -186,11 +189,11 @@ num_frames_to_process < 3) { // Leaf nodes. while (start < end) { - set_params_for_leaf_frames(twopass, twopass_frame, p_rc, frame_info, - gf_group, cur_frame_idx, frame_ind, - parallel_frame_count, max_parallel_frames, - do_frame_parallel_encode, first_frame_index, - cur_disp_idx, layer_depth, start, end); + set_params_for_leaf_frames( + twopass, twopass_frame, p_rc, frame_info, gf_group, cur_frame_idx, + frame_ind, parallel_frame_count, max_parallel_frames, + do_frame_parallel_encode, first_frame_index, cur_disp_idx, + layer_depth, start, end, scale_max_boost); ++start; } } else { @@ -202,7 +205,7 @@ twopass, twopass_frame, p_rc, frame_info, gf_group, cur_frame_idx, frame_ind, parallel_frame_count, max_parallel_frames, do_frame_parallel_encode, first_frame_index, INT_MAX, cur_disp_idx, - layer_depth, arf_src_offset, m, end - m, m - start); + layer_depth, arf_src_offset, m, end - m, m - start, scale_max_boost); // If encode reordering is enabled, configure the multi-layers accordingly // and return. For e.g., the encode order for gf-interval 16 after @@ -224,7 +227,7 @@ frame_ind, parallel_frame_count, max_parallel_frames, do_frame_parallel_encode, first_frame_index, depth_thr, cur_disp_idx, layer_depth + 1, arf_src_offsets[i], offset[i], - f_frames[i], b_frames[i]); + f_frames[i], b_frames[i], scale_max_boost); } // Initialize the start and end indices to configure LF_UPDATE frames. @@ -240,7 +243,8 @@ twopass, twopass_frame, gf_group, p_rc, rc, frame_info, start_idx[i], end_idx[i], cur_frame_idx, frame_ind, parallel_frame_count, max_parallel_frames, do_frame_parallel_encode, - first_frame_index, depth_thr, cur_disp_idx, layer_depth + 2); + first_frame_index, depth_thr, cur_disp_idx, layer_depth + 2, + scale_max_boost); if (layer_depth_for_intnl_overlay[i] != INVALID_IDX) set_params_for_intnl_overlay_frames( gf_group, cur_frame_idx, frame_ind, first_frame_index, @@ -254,7 +258,7 @@ twopass, twopass_frame, gf_group, p_rc, rc, frame_info, start, m, cur_frame_idx, frame_ind, parallel_frame_count, max_parallel_frames, do_frame_parallel_encode, first_frame_index, depth_thr, cur_disp_idx, - layer_depth + 1); + layer_depth + 1, scale_max_boost); // Overlay for internal ARF. set_params_for_intnl_overlay_frames(gf_group, cur_frame_idx, frame_ind, @@ -266,7 +270,7 @@ twopass, twopass_frame, gf_group, p_rc, rc, frame_info, m + 1, end, cur_frame_idx, frame_ind, parallel_frame_count, max_parallel_frames, do_frame_parallel_encode, first_frame_index, depth_thr, cur_disp_idx, - layer_depth + 1); + layer_depth + 1, scale_max_boost); } } @@ -379,7 +383,7 @@ int *cur_frame_idx, int *frame_ind, int *count_arf_frames, int *doh_gf_index_map, int *parallel_frame_count, int *first_frame_index, int *cur_disp_index, int gf_interval, int layer_depth, - int max_parallel_frames) { + int max_parallel_frames, const bool scale_max_boost) { assert(layer_depth == 2); assert(gf_group->max_layer_depth_allowed >= 4); int layer, node_start, node_end = 0; @@ -416,7 +420,7 @@ set_params_for_leaf_frames( twopass, twopass_frame, p_rc, frame_info, gf_group, cur_frame_idx, frame_ind, parallel_frame_count, max_parallel_frames, 1, - first_frame_index, cur_disp_index, layer, 0, 0); + first_frame_index, cur_disp_index, layer, 0, 0, scale_max_boost); } else { // In order to obtain the layer depths of INTNL_OVERLAY_UPDATE frames, get // the gf index of corresponding INTNL_ARF_UPDATE frames. @@ -436,7 +440,8 @@ RATE_CONTROL *rc, FRAME_INFO *frame_info, int start, int end, int *cur_frame_idx, int *frame_ind, int *parallel_frame_count, int max_parallel_frames, int do_frame_parallel_encode, - int *first_frame_index, int *cur_disp_idx, int layer_depth) { + int *first_frame_index, int *cur_disp_idx, int layer_depth, + const bool scale_max_boost) { const int num_frames_to_process = end - start; // Either we are at the last level of the pyramid, or we don't have enough @@ -452,7 +457,7 @@ gf_group->layer_depth[*frame_ind] = MAX_ARF_LAYERS; gf_group->arf_boost[*frame_ind] = av1_calc_arf_boost(twopass, twopass_frame, p_rc, frame_info, start, - end - start, 0, NULL, NULL, 0); + end - start, 0, NULL, NULL, 0, scale_max_boost); gf_group->frame_type[*frame_ind] = INTER_FRAME; gf_group->refbuf_state[*frame_ind] = REFBUF_UPDATE; gf_group->max_layer_depth = @@ -498,7 +503,7 @@ // Get the boost factor for intermediate ARF frames. gf_group->arf_boost[*frame_ind] = av1_calc_arf_boost(twopass, twopass_frame, p_rc, frame_info, m, end - m, - m - start, NULL, NULL, 0); + m - start, NULL, NULL, 0, scale_max_boost); ++(*frame_ind); // Frames displayed before this internal ARF. @@ -506,7 +511,7 @@ frame_info, start, m, cur_frame_idx, frame_ind, parallel_frame_count, max_parallel_frames, do_frame_parallel_encode, first_frame_index, - cur_disp_idx, layer_depth + 1); + cur_disp_idx, layer_depth + 1, scale_max_boost); // Overlay for internal ARF. gf_group->update_type[*frame_ind] = INTNL_OVERLAY_UPDATE; @@ -528,7 +533,7 @@ frame_info, m + 1, end, cur_frame_idx, frame_ind, parallel_frame_count, max_parallel_frames, do_frame_parallel_encode, first_frame_index, - cur_disp_idx, layer_depth + 1); + cur_disp_idx, layer_depth + 1, scale_max_boost); } } @@ -665,6 +670,8 @@ gf_group->max_layer_depth_allowed >= 4); int first_frame_index = cur_frame_index; + const bool scale_max_boost = (cpi->oxcf.mode != REALTIME); + if (do_frame_parallel_encode) { // construct_multi_layer_gf_structure() takes the input parameter // 'gf_interval' as p_rc->baseline_gf_interval - 1 . Below code computes the @@ -708,7 +715,7 @@ arf_frame_stats, &cur_frame_index, &frame_index, &count_arf_frames, doh_gf_index_map, ¶llel_frame_count, &first_frame_index, &cur_disp_index, actual_gf_length, use_altref + 1, - cpi->ppi->num_fp_contexts); + cpi->ppi->num_fp_contexts, scale_max_boost); // Set gf_group->skip_frame_refresh. for (int i = 0; i < actual_gf_length; i++) { @@ -736,7 +743,7 @@ cur_frame_index, gf_interval, &cur_frame_index, &frame_index, ¶llel_frame_count, cpi->ppi->num_fp_contexts, do_frame_parallel_encode, &first_frame_index, depth_thr, - &cur_disp_index, use_altref + 1); + &cur_disp_index, use_altref + 1, scale_max_boost); } is_multi_layer_configured = 1; } @@ -748,7 +755,7 @@ &cur_frame_index, &frame_index, ¶llel_frame_count, cpi->ppi->num_fp_contexts, do_frame_parallel_encode, &first_frame_index, - &cur_disp_index, use_altref + 1); + &cur_disp_index, use_altref + 1, scale_max_boost); if (use_altref) { gf_group->update_type[frame_index] = OVERLAY_UPDATE; @@ -839,7 +846,54 @@ gf_group->max_layer_depth = AOMMIN(log_gop_length, MAX_ARF_LAYERS); } -void av1_gop_setup_structure(AV1_COMP *cpi) { +static void construct_gop_structure_from_rc( + GF_GROUP *gf_group, aom_rc_gop_decision_t *rc_gop_decision) { + gf_group->size = rc_gop_decision->gop_frame_count; + for (int frame_index = 0; frame_index < gf_group->size; ++frame_index) { + aom_rc_gop_frame_t *gop_frame_rc = + &rc_gop_decision->gop_frame_list[frame_index]; + gf_group->update_type[frame_index] = gop_frame_rc->update_type; + gf_group->layer_depth[frame_index] = gop_frame_rc->layer_depth; + gf_group->update_ref_idx[frame_index] = gop_frame_rc->update_ref_idx; + // `display_idx` means differently in libaom and RC. + // - in libaom: it is display order index in the GOP, equivalent to + // `order_idx` in RC + // - in RC: it is the number of display frames precedeing this frame, which + // is equivalent to `cur_frame_idx` in libaom. + gf_group->display_idx[frame_index] = gop_frame_rc->order_idx; + gf_group->cur_frame_idx[frame_index] = gop_frame_rc->display_idx; + switch (gf_group->update_type[frame_index]) { + case LF_UPDATE: + case INTNL_OVERLAY_UPDATE: + gf_group->arf_src_offset[frame_index] = 0; + break; + case ARF_UPDATE: + case INTNL_ARF_UPDATE: + gf_group->arf_src_offset[frame_index] = + gop_frame_rc->order_idx - gop_frame_rc->display_idx; + break; + default: gf_group->arf_src_offset[frame_index] = 0; + } + gf_group->frame_type[frame_index] = + gop_frame_rc->is_key_frame ? KEY_FRAME : INTER_FRAME; + gf_group->refbuf_state[frame_index] = + gop_frame_rc->is_key_frame ? REFBUF_RESET : REFBUF_UPDATE; + // Always override the ref frame map from external RC. + gf_group->use_ext_ref_frame_map[frame_index] = 1; + for (int i = 0; i < REF_FRAMES; ++i) { + gf_group->ref_frame_list[frame_index][i] = INVALID_IDX; + } + for (int i = 0; i < AOM_RC_MAX_REF_FRAMES; ++i) { + int ref_name = gop_frame_rc->ref_frame_list.name[i]; + int buf_idx = gop_frame_rc->ref_frame_list.index[i]; + if (ref_name >= LAST_FRAME && ref_name <= ALTREF_FRAME) { + gf_group->ref_frame_list[frame_index][ref_name] = (int8_t)buf_idx; + } + } + } +} + +void av1_gop_setup_structure(AV1_COMP *cpi, const int is_final_pass) { RATE_CONTROL *const rc = &cpi->rc; PRIMARY_RATE_CONTROL *const p_rc = &cpi->ppi->p_rc; GF_GROUP *const gf_group = &cpi->ppi->gf_group; @@ -848,26 +902,45 @@ const int key_frame = rc->frames_since_key == 0; FRAME_UPDATE_TYPE first_frame_update_type = ARF_UPDATE; - if (key_frame) { - first_frame_update_type = KF_UPDATE; - if (cpi->oxcf.kf_max_pyr_height != -1) { - gf_group->max_layer_depth_allowed = AOMMIN( - cpi->oxcf.kf_max_pyr_height, gf_group->max_layer_depth_allowed); + // define_gf_group() is called twice in av1_set_second_pass_params() with + // `is_final_pass` being 0 and 1 separately. But only one GOP can be advanced + // with the external RC. That is only done when `is_final_pass` is true. + if (cpi->ext_ratectrl.ready && + (cpi->ext_ratectrl.funcs.rc_type & AOM_RC_GOP) != 0 && + cpi->ext_ratectrl.funcs.get_gop_decision != NULL && is_final_pass) { + aom_rc_gop_decision_t gop_decision; + aom_codec_err_t codec_status = + av1_extrc_get_gop_decision(&cpi->ext_ratectrl, &gop_decision); + if (codec_status != AOM_CODEC_OK) { + aom_internal_error(cpi->common.error, codec_status, + "av1_extrc_get_gop_decision() failed"); } - } else if (!cpi->ppi->gf_state.arf_gf_boost_lst) { - first_frame_update_type = GF_UPDATE; + construct_gop_structure_from_rc(gf_group, &gop_decision); + if (gop_decision.gop_frame_list[0].is_key_frame) { + rc->frames_since_key = 0; + } + } else { + if (key_frame) { + first_frame_update_type = KF_UPDATE; + if (cpi->oxcf.kf_max_pyr_height != -1) { + gf_group->max_layer_depth_allowed = AOMMIN( + cpi->oxcf.kf_max_pyr_height, gf_group->max_layer_depth_allowed); + } + } else if (!cpi->ppi->gf_state.arf_gf_boost_lst) { + first_frame_update_type = GF_UPDATE; + } + + if (cpi->oxcf.algo_cfg.sharpness == 3) + gf_group->max_layer_depth_allowed = + AOMMIN(gf_group->max_layer_depth_allowed, 2); + + gf_group->size = construct_multi_layer_gf_structure( + cpi, twopass, gf_group, rc, frame_info, p_rc->baseline_gf_interval, + first_frame_update_type); + + if (gf_group->max_layer_depth_allowed == 0) + set_ld_layer_depth(gf_group, p_rc->baseline_gf_interval); } - - if (cpi->oxcf.algo_cfg.sharpness == 3) - gf_group->max_layer_depth_allowed = - AOMMIN(gf_group->max_layer_depth_allowed, 2); - - gf_group->size = construct_multi_layer_gf_structure( - cpi, twopass, gf_group, rc, frame_info, p_rc->baseline_gf_interval, - first_frame_update_type); - - if (gf_group->max_layer_depth_allowed == 0) - set_ld_layer_depth(gf_group, p_rc->baseline_gf_interval); } int av1_gop_check_forward_keyframe(const GF_GROUP *gf_group,
diff --git a/av1/encoder/gop_structure.h b/av1/encoder/gop_structure.h index 92380ab..14ef5bf 100644 --- a/av1/encoder/gop_structure.h +++ b/av1/encoder/gop_structure.h
@@ -36,11 +36,13 @@ * the group. It does this primarily by updateing entries in * cpi->twopass.gf_group.update_type[]. * - * \param[in] cpi Top - level encoder instance structure + * \param[in] cpi Top - level encoder instance structure + * \param[in] is_final_pass Whether it is the second call to + * define_gf_group(). * * \remark No return value but this function updates group data structures. */ -void av1_gop_setup_structure(struct AV1_COMP *cpi); +void av1_gop_setup_structure(struct AV1_COMP *cpi, const int is_final_pass); /*!\brief Check whether a frame in the GOP is a forward key frame *
diff --git a/av1/encoder/interp_search.h b/av1/encoder/interp_search.h index 28b036d..428d1e4 100644 --- a/av1/encoder/interp_search.h +++ b/av1/encoder/interp_search.h
@@ -128,12 +128,12 @@ /*! * Stack to store full pixel search start mv of NEWMV mode. */ - FULLPEL_MV start_mv_stack[(MAX_REF_MV_SEARCH - 1) * 2]; + FULLPEL_MV start_mv_stack[MAX_REF_MV_SEARCH * 2]; /*! * Stack to store ref_mv_idx of NEWMV mode. */ - uint8_t ref_mv_idx_stack[(MAX_REF_MV_SEARCH - 1) * 2]; + uint8_t ref_mv_idx_stack[MAX_REF_MV_SEARCH * 2]; /*! * Count of mvs in start mv stack.
diff --git a/av1/encoder/intra_mode_search.c b/av1/encoder/intra_mode_search.c index e8904c7..6f61a7c 100644 --- a/av1/encoder/intra_mode_search.c +++ b/av1/encoder/intra_mode_search.c
@@ -298,8 +298,6 @@ best_tx_size = mbmi->tx_size; filter_intra_mode_info = mbmi->filter_intra_mode_info; av1_copy_array(best_tx_type_map, xd->tx_type_map, ctx->num_4x4_blk); - memcpy(ctx->blk_skip, x->txfm_search_info.blk_skip, - sizeof(x->txfm_search_info.blk_skip[0]) * ctx->num_4x4_blk); *rate = this_rate; *rate_tokenonly = tokenonly_rd_stats.rate; *distortion = tokenonly_rd_stats.dist; @@ -626,7 +624,7 @@ } else { av1_init_rd_stats(rd_stats); av1_txfm_rd_in_plane(x, cpi, rd_stats, INT64_MAX, 0, plane, plane_bsize, - tx_size, FTXS_NONE, 0); + tx_size, FTXS_NONE); av1_rd_cost_update(x->rdmult, rd_stats); cfl_cost = rd_stats->rdcost; } @@ -1048,7 +1046,6 @@ x->palette_buffer->best_palette_color_map; uint8_t *const color_map = xd->plane[0].color_index_map; MB_MODE_INFO best_mbmi_palette = *mbmi; - uint8_t best_blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE]; uint8_t best_tx_type_map[MAX_MIB_SIZE * MAX_MIB_SIZE]; const ModeCosts *mode_costs = &x->mode_costs; const int *const intra_mode_cost = @@ -1064,18 +1061,15 @@ RD_STATS rd_stats_y; av1_invalid_rd_stats(&rd_stats_y); - av1_rd_pick_palette_intra_sby(cpi, x, bsize, intra_mode_cost[DC_PRED], - &best_mbmi_palette, best_palette_color_map, - &best_rd_palette, &rd_stats_y.rate, NULL, - &rd_stats_y.dist, &rd_stats_y.skip_txfm, NULL, - ctx, best_blk_skip, best_tx_type_map); + av1_rd_pick_palette_intra_sby( + cpi, x, bsize, intra_mode_cost[DC_PRED], &best_mbmi_palette, + best_palette_color_map, &best_rd_palette, &rd_stats_y.rate, NULL, + &rd_stats_y.dist, &rd_stats_y.skip_txfm, NULL, ctx, best_tx_type_map); if (rd_stats_y.rate == INT_MAX || pmi->palette_size[0] == 0) { this_rd_cost->rdcost = INT64_MAX; return skippable; } - memcpy(x->txfm_search_info.blk_skip, best_blk_skip, - sizeof(best_blk_skip[0]) * bsize_to_num_blk(bsize)); av1_copy_array(xd->tx_type_map, best_tx_type_map, ctx->num_4x4_blk); memcpy(color_map, best_palette_color_map, rows * cols * sizeof(best_palette_color_map[0])); @@ -1137,7 +1131,6 @@ x->palette_buffer->best_palette_color_map; uint8_t *const color_map = xd->plane[0].color_index_map; MB_MODE_INFO best_mbmi_palette = *mbmi; - uint8_t best_blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE]; uint8_t best_tx_type_map[MAX_MIB_SIZE * MAX_MIB_SIZE]; const ModeCosts *mode_costs = &x->mode_costs; const int *const intra_mode_cost = @@ -1153,18 +1146,15 @@ RD_STATS rd_stats_y; av1_invalid_rd_stats(&rd_stats_y); - av1_rd_pick_palette_intra_sby(cpi, x, bsize, intra_mode_cost[DC_PRED], - &best_mbmi_palette, best_palette_color_map, - &best_rd_palette, &rd_stats_y.rate, NULL, - &rd_stats_y.dist, &rd_stats_y.skip_txfm, NULL, - ctx, best_blk_skip, best_tx_type_map); + av1_rd_pick_palette_intra_sby( + cpi, x, bsize, intra_mode_cost[DC_PRED], &best_mbmi_palette, + best_palette_color_map, &best_rd_palette, &rd_stats_y.rate, NULL, + &rd_stats_y.dist, &rd_stats_y.skip_txfm, NULL, ctx, best_tx_type_map); if (rd_stats_y.rate == INT_MAX || pmi->palette_size[0] == 0) { this_rd_cost->rdcost = INT64_MAX; return; } - memcpy(x->txfm_search_info.blk_skip, best_blk_skip, - sizeof(best_blk_skip[0]) * bsize_to_num_blk(bsize)); av1_copy_array(xd->tx_type_map, best_tx_type_map, ctx->num_4x4_blk); memcpy(color_map, best_palette_color_map, rows * cols * sizeof(best_palette_color_map[0])); @@ -1231,8 +1221,6 @@ *rate_tokenonly = this_rate_tokenonly; *distortion = rd_stats.dist; *skippable = rd_stats.skip_txfm; - av1_copy_array(ctx->blk_skip, x->txfm_search_info.blk_skip, - ctx->num_4x4_blk); av1_copy_array(ctx->tx_type_map, xd->tx_type_map, ctx->num_4x4_blk); return 1; } @@ -1262,9 +1250,6 @@ int filter_intra_selected_flag = 0; TX_SIZE best_tx_size = mbmi->tx_size; FILTER_INTRA_MODE best_fi_mode = FILTER_DC_PRED; - uint8_t best_blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE]; - memcpy(best_blk_skip, x->txfm_search_info.blk_skip, - sizeof(best_blk_skip[0]) * ctx->num_4x4_blk); uint8_t best_tx_type_map[MAX_MIB_SIZE * MAX_MIB_SIZE]; av1_copy_array(best_tx_type_map, xd->tx_type_map, ctx->num_4x4_blk); mbmi->filter_intra_mode_info.use_filter_intra = 1; @@ -1285,8 +1270,6 @@ if (this_rd_tmp < best_rd_so_far) { best_tx_size = mbmi->tx_size; av1_copy_array(best_tx_type_map, xd->tx_type_map, ctx->num_4x4_blk); - memcpy(best_blk_skip, x->txfm_search_info.blk_skip, - sizeof(best_blk_skip[0]) * ctx->num_4x4_blk); best_fi_mode = fi_mode; *rd_stats_y = rd_stats_y_fi; filter_intra_selected_flag = 1; @@ -1296,8 +1279,6 @@ mbmi->tx_size = best_tx_size; av1_copy_array(xd->tx_type_map, best_tx_type_map, ctx->num_4x4_blk); - memcpy(x->txfm_search_info.blk_skip, best_blk_skip, - sizeof(x->txfm_search_info.blk_skip[0]) * ctx->num_4x4_blk); if (filter_intra_selected_flag) { mbmi->filter_intra_mode_info.use_filter_intra = 1; @@ -1325,15 +1306,7 @@ const int mode_cost = mode_costs->mbmode_cost[size_group_lookup[bsize]][mode] + ref_frame_cost; const int skip_ctx = av1_get_skip_txfm_context(xd); - - int known_rate = mode_cost; - const int intra_cost_penalty = av1_get_intra_cost_penalty( - cm->quant_params.base_qindex, cm->quant_params.y_dc_delta_q, - cm->seq_params->bit_depth); - - if (mode != DC_PRED && mode != PAETH_PRED) known_rate += intra_cost_penalty; - known_rate += AOMMIN(mode_costs->skip_txfm_cost[skip_ctx][0], - mode_costs->skip_txfm_cost[skip_ctx][1]); + int known_rate = mode_cost + mode_costs->skip_mode_cost[skip_ctx][0]; const int64_t known_rd = RDCOST(x->rdmult, known_rate, 0); if (known_rd > best_rd) { intra_search_state->skip_intra_modes = 1; @@ -1424,28 +1397,25 @@ const int try_palette = cpi->oxcf.tool_cfg.enable_palette && av1_allow_palette(cm->features.allow_screen_content_tools, mbmi->bsize); - assert(intra_search_state->rate_uv_intra == INT_MAX); - if (intra_search_state->rate_uv_intra == INT_MAX) { - // If no good uv-predictor had been found, search for it. - const TX_SIZE uv_tx = av1_get_tx_size(AOM_PLANE_U, xd); - av1_rd_pick_intra_sbuv_mode(cpi, x, &intra_search_state->rate_uv_intra, - &intra_search_state->rate_uv_tokenonly, - &intra_search_state->dist_uvs, - &intra_search_state->skip_uvs, bsize, uv_tx); - intra_search_state->mode_uv = mbmi->uv_mode; - if (try_palette) intra_search_state->pmi_uv = *pmi; - intra_search_state->uv_angle_delta = mbmi->angle_delta[PLANE_TYPE_UV]; + // If no good uv-predictor had been found, search for it. + const TX_SIZE uv_tx = av1_get_tx_size(AOM_PLANE_U, xd); + av1_rd_pick_intra_sbuv_mode(cpi, x, &intra_search_state->rate_uv_intra, + &intra_search_state->rate_uv_tokenonly, + &intra_search_state->dist_uvs, + &intra_search_state->skip_uvs, bsize, uv_tx); + intra_search_state->mode_uv = mbmi->uv_mode; + if (try_palette) intra_search_state->pmi_uv = *pmi; + intra_search_state->uv_angle_delta = mbmi->angle_delta[PLANE_TYPE_UV]; - const int uv_rate = intra_search_state->rate_uv_tokenonly; - const int64_t uv_dist = intra_search_state->dist_uvs; - const int64_t uv_rd = RDCOST(x->rdmult, uv_rate, uv_dist); - if (uv_rd > best_rd) { - // If there is no good intra uv-mode available, we can skip all intra - // modes. - intra_search_state->skip_intra_modes = 1; - return 0; - } + const int uv_rate = intra_search_state->rate_uv_tokenonly; + const int64_t uv_dist = intra_search_state->dist_uvs; + const int64_t uv_rd = RDCOST(x->rdmult, uv_rate, uv_dist); + if (uv_rd > best_rd) { + // If there is no good intra uv-mode available, we can skip all intra + // modes. + intra_search_state->skip_intra_modes = 1; + return 0; } // If we are here, then the encoder has found at least one good intra uv @@ -1686,18 +1656,16 @@ *rate_tokenonly = this_rate_tokenonly; *distortion = this_distortion; *skippable = s; - memcpy(ctx->blk_skip, x->txfm_search_info.blk_skip, - sizeof(x->txfm_search_info.blk_skip[0]) * ctx->num_4x4_blk); av1_copy_array(ctx->tx_type_map, xd->tx_type_map, ctx->num_4x4_blk); } } // Searches palette if (try_palette) { - av1_rd_pick_palette_intra_sby( - cpi, x, bsize, bmode_costs[DC_PRED], &best_mbmi, best_palette_color_map, - &best_rd, rate, rate_tokenonly, distortion, skippable, &beat_best_rd, - ctx, ctx->blk_skip, ctx->tx_type_map); + av1_rd_pick_palette_intra_sby(cpi, x, bsize, bmode_costs[DC_PRED], + &best_mbmi, best_palette_color_map, &best_rd, + rate, rate_tokenonly, distortion, skippable, + &beat_best_rd, ctx, ctx->tx_type_map); } // Searches filter_intra
diff --git a/av1/encoder/mcomp.c b/av1/encoder/mcomp.c index 06145ce..b89b12f 100644 --- a/av1/encoder/mcomp.c +++ b/av1/encoder/mcomp.c
@@ -2572,7 +2572,7 @@ unsigned int besterr; #if CONFIG_AV1_HIGHBITDEPTH if (is_cur_buf_hbd(xd)) { - DECLARE_ALIGNED(16, uint16_t, pred16[MAX_SB_SQUARE]); + uint16_t *pred16 = (uint16_t *)(xd->tmp_upsample_pred); uint8_t *pred8 = CONVERT_TO_BYTEPTR(pred16); if (second_pred != NULL) { if (mask) { @@ -2593,7 +2593,7 @@ } besterr = vfp->vf(pred8, w, src, src_stride, sse); } else { - DECLARE_ALIGNED(16, uint8_t, pred[MAX_SB_SQUARE]); + uint8_t *pred = xd->tmp_upsample_pred; if (second_pred != NULL) { if (mask) { aom_comp_mask_upsampled_pred( @@ -2614,7 +2614,7 @@ besterr = vfp->vf(pred, w, src, src_stride, sse); } #else - DECLARE_ALIGNED(16, uint8_t, pred[MAX_SB_SQUARE]); + uint8_t *pred = xd->tmp_upsample_pred; if (second_pred != NULL) { if (mask) { aom_comp_mask_upsampled_pred(xd, cm, mi_row, mi_col, this_mv, pred, @@ -2895,7 +2895,7 @@ best_mv->col + diag_step.col }; int has_better_mv = 0; - if (var_params->subpel_search_type != USE_2_TAPS_ORIG) { + if (var_params->subpel_search_type > USE_2_TAPS) { check_better(xd, cm, &row_bias_mv, best_mv, mv_limits, var_params, mv_cost_params, besterr, sse1, distortion, &has_better_mv); check_better(xd, cm, &col_bias_mv, best_mv, mv_limits, var_params, @@ -3326,7 +3326,7 @@ *distortion = start_mv_stats->distortion; *sse1 = start_mv_stats->sse; } else { - if (subpel_search_type != USE_2_TAPS_ORIG) { + if (subpel_search_type > USE_2_TAPS) { besterr = upsampled_setup_center_error(xd, cm, bestmv, var_params, mv_cost_params, sse1, distortion); } else { @@ -3346,7 +3346,7 @@ } MV diag_step; - if (subpel_search_type != USE_2_TAPS_ORIG) { + if (subpel_search_type > USE_2_TAPS) { diag_step = first_level_check(xd, cm, iter_center_mv, bestmv, hstep, mv_limits, var_params, mv_cost_params, &besterr, sse1, distortion); @@ -3682,21 +3682,23 @@ const int mi_col = xd->mi_col; unsigned int besterr; - DECLARE_ALIGNED(16, uint8_t, pred[2 * MAX_SB_SQUARE]); #if CONFIG_AV1_HIGHBITDEPTH if (is_cur_buf_hbd(xd)) { - uint8_t *pred8 = CONVERT_TO_BYTEPTR(pred); + uint16_t *pred16 = (uint16_t *)(xd->tmp_upsample_pred); + uint8_t *pred8 = CONVERT_TO_BYTEPTR(pred16); aom_highbd_upsampled_pred(xd, cm, mi_row, mi_col, this_mv, pred8, w, h, subpel_x_q3, subpel_y_q3, ref, ref_stride, xd->bd, subpel_search_type); besterr = vfp->ovf(pred8, w, wsrc, mask, sse); } else { + uint8_t *pred = xd->tmp_upsample_pred; aom_upsampled_pred(xd, cm, mi_row, mi_col, this_mv, pred, w, h, subpel_x_q3, subpel_y_q3, ref, ref_stride, subpel_search_type); besterr = vfp->ovf(pred, w, wsrc, mask, sse); } #else + uint8_t *pred = xd->tmp_upsample_pred; aom_upsampled_pred(xd, cm, mi_row, mi_col, this_mv, pred, w, h, subpel_x_q3, subpel_y_q3, ref, ref_stride, subpel_search_type);
diff --git a/av1/encoder/motion_search_facade.c b/av1/encoder/motion_search_facade.c index 5169989..0bd7d9d 100644 --- a/av1/encoder/motion_search_facade.c +++ b/av1/encoder/motion_search_facade.c
@@ -169,6 +169,8 @@ else start_mv = get_fullmv_from_mv(&ref_mv); + const FULLPEL_MV fullpel_ref_mv = start_mv; + // cand stores start_mv and all possible MVs in a SB. cand_mv_t cand[MAX_TPL_BLK_IN_SB * MAX_TPL_BLK_IN_SB + 1]; av1_zero(cand); @@ -183,54 +185,74 @@ const int cand_cnt = AOMMIN(2, cnt); // TODO(any): Test the speed feature for OBMC_CAUSAL mode. - if (cpi->sf.mv_sf.skip_fullpel_search_using_startmv && + if (cpi->sf.mv_sf.skip_fullpel_search_using_startmv_refmv && mbmi->motion_mode == SIMPLE_TRANSLATION) { - const int stack_size = args->start_mv_cnt; for (int cand_idx = 0; cand_idx < cand_cnt; cand_idx++) { int_mv *fmv_cand = &cand[cand_idx].fmv; int skip_cand_mv = 0; // Check difference between mvs in the stack and candidate mv. - for (int stack_idx = 0; stack_idx < stack_size; stack_idx++) { - const uint8_t this_ref_mv_idx = args->ref_mv_idx_stack[stack_idx]; - const FULLPEL_MV *fmv_stack = &args->start_mv_stack[stack_idx]; + for (int stack_idx = 0; stack_idx < args->start_mv_cnt; stack_idx++) { + uint8_t this_ref_mv_idx = args->ref_mv_idx_stack[stack_idx]; const int this_newmv_valid = args->single_newmv_valid[this_ref_mv_idx][ref]; - const int row_diff = abs(fmv_stack->row - fmv_cand->as_fullmv.row); - const int col_diff = abs(fmv_stack->col - fmv_cand->as_fullmv.col); - if (!this_newmv_valid) continue; + if (!this_newmv_valid && this_ref_mv_idx != mbmi->ref_mv_idx) continue; - if (cpi->sf.mv_sf.skip_fullpel_search_using_startmv >= 2) { + const FULLPEL_MV *fmv_stack = &args->start_mv_stack[stack_idx]; + const int start_mv_row_diff = + abs(fmv_stack->row - fmv_cand->as_fullmv.row); + const int start_mv_col_diff = + abs(fmv_stack->col - fmv_cand->as_fullmv.col); + + if (mbmi->mode == NEAR_NEWMV || mbmi->mode == NEW_NEARMV) { + assert(has_second_ref(mbmi)); + this_ref_mv_idx += 1; + } + const MV this_ref_mv = + av1_get_ref_mv_from_stack(ref_idx, mbmi->ref_frame, this_ref_mv_idx, + &x->mbmi_ext) + .as_mv; + + assert(IMPLIES(args->ref_mv_idx_stack[stack_idx] == mbmi->ref_mv_idx, + this_ref_mv.row == ref_mv.row)); + assert(IMPLIES(args->ref_mv_idx_stack[stack_idx] == mbmi->ref_mv_idx, + this_ref_mv.col == ref_mv.col)); + + const FULLPEL_MV this_fullpel_ref_mv = get_fullmv_from_mv(&this_ref_mv); + const int ref_mv_row_diff = + abs(this_fullpel_ref_mv.row - fullpel_ref_mv.row); + const int ref_mv_col_diff = + abs(this_fullpel_ref_mv.col - fullpel_ref_mv.col); + + if (cpi->sf.mv_sf.skip_fullpel_search_using_startmv_refmv >= 2) { // Prunes the current start_mv candidate, if the absolute mv // difference of both row and column are <= 1. - if (row_diff <= 1 && col_diff <= 1) { + if (start_mv_row_diff <= 1 && start_mv_col_diff <= 1 && + ref_mv_row_diff <= 1 && ref_mv_col_diff <= 1) { skip_cand_mv = 1; break; } - } else if (cpi->sf.mv_sf.skip_fullpel_search_using_startmv >= 1) { + } else if (cpi->sf.mv_sf.skip_fullpel_search_using_startmv_refmv >= 1) { // Prunes the current start_mv candidate, if the sum of the absolute // mv difference of row and column is <= 1. - if (row_diff + col_diff <= 1) { + if ((start_mv_row_diff + start_mv_col_diff <= 1) && + (ref_mv_row_diff + ref_mv_col_diff <= 1)) { skip_cand_mv = 1; break; } } } if (skip_cand_mv) { - // Ensure atleast one full-pel motion search is not pruned. - assert(mbmi->ref_mv_idx != 0); // Mark the candidate mv as invalid so that motion search gets skipped. cand[cand_idx].fmv.as_int = INVALID_MV; } else { // Store start_mv candidate and corresponding ref_mv_idx of full-pel - // search in the mv stack (except last ref_mv_idx). - if (mbmi->ref_mv_idx != MAX_REF_MV_SEARCH - 1) { - assert(args->start_mv_cnt < (MAX_REF_MV_SEARCH - 1) * 2); - args->start_mv_stack[args->start_mv_cnt] = fmv_cand->as_fullmv; - args->ref_mv_idx_stack[args->start_mv_cnt] = mbmi->ref_mv_idx; - args->start_mv_cnt++; - } + // search in the mv stack. + assert(args->start_mv_cnt < MAX_REF_MV_SEARCH * 2); + args->start_mv_stack[args->start_mv_cnt] = fmv_cand->as_fullmv; + args->ref_mv_idx_stack[args->start_mv_cnt] = mbmi->ref_mv_idx; + args->start_mv_cnt++; } } }
diff --git a/av1/encoder/motion_search_facade.h b/av1/encoder/motion_search_facade.h index a1cbb77..5e7f417 100644 --- a/av1/encoder/motion_search_facade.h +++ b/av1/encoder/motion_search_facade.h
@@ -123,10 +123,17 @@ const int sf_blk_search_method = mv_sf->use_bsize_dependent_search_method; const int min_dim = AOMMIN(block_size_wide[bsize], block_size_high[bsize]); const int qband = x->qindex >> (QINDEX_BITS - 2); - const bool use_faster_search_method = - (sf_blk_search_method == 1 && min_dim >= 32) || - (sf_blk_search_method >= 2 && min_dim >= 16 && - x->content_state_sb.source_sad_nonrd <= kMedSad && qband < 3); + const int min_dim_th[4] = { 128, 64, 32, 16 }; + bool use_faster_search_method = false; + + if (sf_blk_search_method >= 1 && sf_blk_search_method <= 3) { + use_faster_search_method = + (min_dim >= min_dim_th[sf_blk_search_method - 1]); + } else if (sf_blk_search_method == 4) { + use_faster_search_method = + (min_dim >= min_dim_th[sf_blk_search_method - 1] && + x->content_state_sb.source_sad_nonrd <= kMedSad && qband < 3); + } if (use_faster_search_method) { search_method = av1_get_faster_search_method(search_method);
diff --git a/av1/encoder/nonrd_opt.c b/av1/encoder/nonrd_opt.c index 1acf7fb..a196e0c 100644 --- a/av1/encoder/nonrd_opt.c +++ b/av1/encoder/nonrd_opt.c
@@ -708,7 +708,6 @@ PRED_BUFFER *tmp_buffers, PRED_BUFFER **this_mode_pred, RD_STATS *best_rdc, BEST_PICKMODE *best_pickmode, - PICK_MODE_CONTEXT *ctx, unsigned int *best_sad_norm) { AV1_COMMON *const cm = &cpi->common; MACROBLOCKD *const xd = &x->e_mbd; @@ -924,14 +923,8 @@ mi->uv_mode = this_mode; mi->mv[0].as_int = INVALID_MV; mi->mv[1].as_int = INVALID_MV; - if (!this_rdc.skip_txfm) - memset(ctx->blk_skip, 0, - sizeof(x->txfm_search_info.blk_skip[0]) * ctx->num_4x4_blk); } } - if (best_pickmode->best_ref_frame == INTRA_FRAME) - memset(ctx->blk_skip, 0, - sizeof(x->txfm_search_info.blk_skip[0]) * ctx->num_4x4_blk); mi->tx_size = best_pickmode->best_tx_size; *best_sad_norm = args.best_sad >>
diff --git a/av1/encoder/nonrd_opt.h b/av1/encoder/nonrd_opt.h index 0bdf73e..9f81c0a 100644 --- a/av1/encoder/nonrd_opt.h +++ b/av1/encoder/nonrd_opt.h
@@ -21,7 +21,9 @@ #define RTC_MODES (AOMMAX(RTC_INTER_MODES, RTC_INTRA_MODES)) #define CALC_BIASED_RDCOST(rdcost) (7 * (rdcost) >> 3) #define NUM_COMP_INTER_MODES_RT (6) +#define NUM_COMP_INTER_MODES_RT_FULL (10) #define NUM_INTER_MODES 12 +#define NUM_INTER_MODES_FULL 28 #define CAP_TX_SIZE_FOR_BSIZE_GT32(tx_mode_search_type, bsize) \ (((tx_mode_search_type) != ONLY_4X4 && (bsize) > BLOCK_32X32) ? true : false) #define TX_SIZE_FOR_BSIZE_GT32 (TX_16X16) @@ -144,6 +146,23 @@ { ALTREF_FRAME, GLOBALMV }, { ALTREF_FRAME, NEWMV }, }; +static const REF_MODE ref_mode_set_full[NUM_INTER_MODES_FULL] = { + { LAST_FRAME, NEARESTMV }, { LAST_FRAME, NEARMV }, + { LAST_FRAME, GLOBALMV }, { LAST_FRAME, NEWMV }, + { GOLDEN_FRAME, NEARESTMV }, { GOLDEN_FRAME, NEARMV }, + { GOLDEN_FRAME, GLOBALMV }, { GOLDEN_FRAME, NEWMV }, + { ALTREF_FRAME, NEARESTMV }, { ALTREF_FRAME, NEARMV }, + { ALTREF_FRAME, GLOBALMV }, { ALTREF_FRAME, NEWMV }, + { LAST2_FRAME, NEARESTMV }, { LAST2_FRAME, NEARMV }, + { LAST2_FRAME, GLOBALMV }, { LAST2_FRAME, NEWMV }, + { LAST3_FRAME, NEARESTMV }, { LAST3_FRAME, NEARMV }, + { LAST3_FRAME, GLOBALMV }, { LAST3_FRAME, NEWMV }, + { BWDREF_FRAME, NEARESTMV }, { BWDREF_FRAME, NEARMV }, + { BWDREF_FRAME, GLOBALMV }, { BWDREF_FRAME, NEWMV }, + { ALTREF2_FRAME, NEARESTMV }, { ALTREF2_FRAME, NEARMV }, + { ALTREF2_FRAME, GLOBALMV }, { ALTREF2_FRAME, NEWMV }, +}; + static const COMP_REF_MODE comp_ref_mode_set[NUM_COMP_INTER_MODES_RT] = { { { LAST_FRAME, GOLDEN_FRAME }, GLOBAL_GLOBALMV }, { { LAST_FRAME, GOLDEN_FRAME }, NEAREST_NEARESTMV }, @@ -153,6 +172,20 @@ { { LAST_FRAME, ALTREF_FRAME }, NEAREST_NEARESTMV }, }; +static const COMP_REF_MODE + comp_ref_mode_set_full[NUM_COMP_INTER_MODES_RT_FULL] = { + { { LAST_FRAME, GOLDEN_FRAME }, GLOBAL_GLOBALMV }, + { { LAST_FRAME, GOLDEN_FRAME }, NEAREST_NEARESTMV }, + { { LAST_FRAME, LAST2_FRAME }, GLOBAL_GLOBALMV }, + { { LAST_FRAME, LAST2_FRAME }, NEAREST_NEARESTMV }, + { { LAST_FRAME, ALTREF_FRAME }, GLOBAL_GLOBALMV }, + { { LAST_FRAME, ALTREF_FRAME }, NEAREST_NEARESTMV }, + { { LAST_FRAME, BWDREF_FRAME }, GLOBAL_GLOBALMV }, + { { LAST_FRAME, BWDREF_FRAME }, NEAREST_NEARESTMV }, + { { LAST_FRAME, ALTREF2_FRAME }, GLOBAL_GLOBALMV }, + { { LAST_FRAME, ALTREF2_FRAME }, NEAREST_NEARESTMV }, + }; + static const int_interpfilters filters_ref_set[9] = { [0].as_filters = { EIGHTTAP_REGULAR, EIGHTTAP_REGULAR }, [1].as_filters = { EIGHTTAP_SMOOTH, EIGHTTAP_SMOOTH }, @@ -466,7 +499,8 @@ &frame_mv[NEARESTMV][ref_frame], &frame_mv[NEARMV][ref_frame], 0); frame_mv[GLOBALMV][ref_frame] = mbmi_ext->global_mvs[ref_frame]; // Early exit for non-LAST frame if force_skip_low_temp_var is set. - if (!ref_is_scaled && bsize >= BLOCK_8X8 && !skip_pred_mv && + if (!is_one_pass_rt_lag_params(cpi) && !ref_is_scaled && + bsize >= BLOCK_8X8 && !skip_pred_mv && !(force_skip_low_temp_var && ref_frame != LAST_FRAME)) { av1_mv_pred(cpi, x, yv12_mb[ref_frame][0].buf, yv12->y_stride, ref_frame, bsize); @@ -528,7 +562,7 @@ cpi->oxcf.tune_cfg.content == AOM_CONTENT_SCREEN && cpi->oxcf.speed >= 11 && cpi->rc.high_source_sad && ((cpi->sf.rt_sf.prune_palette_search_nonrd > 2) || - (cpi->sf.rt_sf.rc_compute_spatial_var_sc && + (cpi->sf.rt_sf.rc_compute_spatial_var_sc_kf && cpi->rc.frame_spatial_variance < 1200 && cpi->rc.perc_spatial_flat_blocks < 5 && cpi->rc.percent_blocks_with_motion > 98 && source_variance < 4000))); @@ -585,7 +619,6 @@ PRED_BUFFER *tmp_buffers, PRED_BUFFER **this_mode_pred, RD_STATS *best_rdc, BEST_PICKMODE *best_pickmode, - PICK_MODE_CONTEXT *ctx, unsigned int *best_sad_norm); #endif // AOM_AV1_ENCODER_NONRD_OPT_H_
diff --git a/av1/encoder/nonrd_pickmode.c b/av1/encoder/nonrd_pickmode.c index d7d2636..f201006 100644 --- a/av1/encoder/nonrd_pickmode.c +++ b/av1/encoder/nonrd_pickmode.c
@@ -77,8 +77,9 @@ // Copy best inter mode parameters to best_pickmode static inline void update_search_state_nonrd( InterModeSearchStateNonrd *search_state, MB_MODE_INFO *const mi, - TxfmSearchInfo *txfm_info, RD_STATS *nonskip_rdc, PICK_MODE_CONTEXT *ctx, + TxfmSearchInfo *txfm_info, RD_STATS *nonskip_rdc, PREDICTION_MODE this_best_mode, const int64_t sse_y) { + (void)txfm_info; BEST_PICKMODE *const best_pickmode = &search_state->best_pickmode; best_pickmode->best_sse = sse_y; @@ -93,10 +94,6 @@ best_pickmode->best_mode_skip_txfm = search_state->this_rdc.skip_txfm; best_pickmode->best_mode_initial_skip_flag = (nonskip_rdc->rate == INT_MAX && search_state->this_rdc.skip_txfm); - if (!best_pickmode->best_mode_skip_txfm) { - memcpy(ctx->blk_skip, txfm_info->blk_skip, - sizeof(txfm_info->blk_skip[0]) * ctx->num_4x4_blk); - } } static inline int subpel_select(AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bsize, @@ -314,7 +311,8 @@ int_mv *this_ref_frm_newmv = &frame_mv[NEWMV][ref_frame]; unsigned int y_sad_zero; if (ref_frame > LAST_FRAME && cpi->oxcf.rc_cfg.mode == AOM_CBR && - (cpi->ref_frame_flags & AOM_LAST_FLAG) && gf_temporal_ref) { + (cpi->ref_frame_flags & AOM_LAST_FLAG) && gf_temporal_ref && + !is_one_pass_rt_lag_params(cpi)) { int tmp_sad; int dis; @@ -374,7 +372,8 @@ const MACROBLOCKD *xd, const ModeCosts *mode_costs, int segment_id, BLOCK_SIZE bsize, - unsigned int *ref_costs_single) { + unsigned int *ref_costs_single, + int is_one_pass_rt_lag) { int seg_ref_active = segfeature_active(&cm->seg, segment_id, SEG_LVL_REF_FRAME); if (seg_ref_active) { @@ -398,6 +397,18 @@ ref_costs_single[GOLDEN_FRAME] += mode_costs->single_ref_cost[0][1][0]; ref_costs_single[ALTREF_FRAME] += mode_costs->single_ref_cost[0][0][1]; ref_costs_single[ALTREF_FRAME] += mode_costs->single_ref_cost[0][2][0]; + if (is_one_pass_rt_lag) { + ref_costs_single[ALTREF2_FRAME] = base_cost; + ref_costs_single[BWDREF_FRAME] = base_cost; + ref_costs_single[LAST2_FRAME] = base_cost; + ref_costs_single[LAST3_FRAME] = base_cost; + ref_costs_single[LAST2_FRAME] += mode_costs->single_ref_cost[0][0][0]; + ref_costs_single[LAST3_FRAME] += mode_costs->single_ref_cost[0][0][0]; + ref_costs_single[BWDREF_FRAME] += mode_costs->single_ref_cost[0][0][1]; + ref_costs_single[BWDREF_FRAME] += mode_costs->single_ref_cost[0][1][0]; + ref_costs_single[ALTREF2_FRAME] += mode_costs->single_ref_cost[0][0][1]; + ref_costs_single[ALTREF2_FRAME] += mode_costs->single_ref_cost[0][2][0]; + } } } @@ -1665,15 +1676,6 @@ if (this_rdc.rdcost < best_rdc.rdcost) { best_rdc = this_rdc; best_mode = this_mode; - if (!this_rdc.skip_txfm) { - if (flat_blocks_screen && args.skippable && best_rdc.dist < 20000) { - memcpy(ctx->blk_skip, x->txfm_search_info.blk_skip, - sizeof(x->txfm_search_info.blk_skip[0]) * ctx->num_4x4_blk); - } else { - memset(ctx->blk_skip, 0, - sizeof(x->txfm_search_info.blk_skip[0]) * ctx->num_4x4_blk); - } - } } if (this_mode == DC_PRED) { if (flat_blocks_screen && args.skippable && this_rdc.dist > 0) @@ -1699,7 +1701,6 @@ try_palette &= prune; } if (try_palette) { - const TxfmSearchInfo *txfm_info = &x->txfm_search_info; const unsigned int intra_ref_frame_cost = 0; x->color_palette_thresh = (best_sad_norm < 500) ? 32 : 64; @@ -1714,10 +1715,6 @@ best_rdc.rate = this_rdc.rate; best_rdc.dist = this_rdc.dist; best_rdc.rdcost = this_rdc.rdcost; - if (!this_rdc.skip_txfm) { - memcpy(ctx->blk_skip, txfm_info->blk_skip, - sizeof(txfm_info->blk_skip[0]) * ctx->num_4x4_blk); - } if (xd->tx_type_map[0] != DCT_DCT) av1_copy_array(ctx->tx_type_map, xd->tx_type_map, ctx->num_4x4_blk); } else { @@ -1735,8 +1732,6 @@ // do it here again in case the above logic changes. if (is_lossless_requested(&cpi->oxcf.rc_cfg)) { x->txfm_search_info.skip_txfm = 0; - memset(ctx->blk_skip, 0, - sizeof(x->txfm_search_info.blk_skip[0]) * ctx->num_4x4_blk); } #if CONFIG_INTERNAL_STATS @@ -2232,14 +2227,12 @@ const int *use_ref_frame_mask, int comp_index, bool comp_use_zero_zeromv_only, MV_REFERENCE_FRAME *last_comp_ref_frame, BLOCK_SIZE bsize) { - const MV_REFERENCE_FRAME *rf = comp_ref_mode_set[comp_index].ref_frame; + const MV_REFERENCE_FRAME *rf = + is_one_pass_rt_lag_params(cpi) + ? comp_ref_mode_set_full[comp_index].ref_frame + : comp_ref_mode_set[comp_index].ref_frame; int skip_gf = 0; int skip_alt = 0; - *this_mode = comp_ref_mode_set[comp_index].pred_mode; - *ref_frame = rf[0]; - *ref_frame2 = rf[1]; - assert(*ref_frame == LAST_FRAME); - assert(*this_mode == GLOBAL_GLOBALMV || *this_mode == NEAREST_NEARESTMV); if (x->source_variance < 50 && bsize > BLOCK_16X16) { if (x->color_sensitivity_sb_g[COLOR_SENS_IDX(AOM_PLANE_U)] == 1 || x->color_sensitivity_sb_g[COLOR_SENS_IDX(AOM_PLANE_V)] == 1) @@ -2371,7 +2364,8 @@ // Estimate cost for single reference frames estimate_single_ref_frame_costs(cm, xd, mode_costs, segment_id, bsize, - search_state->ref_costs_single); + search_state->ref_costs_single, + is_one_pass_rt_lag_params(cpi)); // Reset flag to indicate modes evaluated av1_zero(search_state->mode_checked); @@ -2407,9 +2401,17 @@ &search_state->use_scaled_ref_frame[LAST_FRAME]); } // Update mask to use all reference frame - get_ref_frame_use_mask(cpi, x, mi, mi_row, mi_col, bsize, gf_temporal_ref, - search_state->use_ref_frame_mask, - force_skip_low_temp_var); + if (!is_one_pass_rt_lag_params(cpi)) { + get_ref_frame_use_mask(cpi, x, mi, mi_row, mi_col, bsize, gf_temporal_ref, + search_state->use_ref_frame_mask, + force_skip_low_temp_var); + } else { + for (MV_REFERENCE_FRAME ref_frame_iter = LAST_FRAME; + ref_frame_iter <= ALTREF_FRAME; ++ref_frame_iter) { + search_state->use_ref_frame_mask[ref_frame_iter] = + cpi->ref_frame_flags & (1 << (ref_frame_iter - 1)); + } + } skip_pred_mv = x->force_zeromv_skip_for_blk || (x->nonrd_prune_ref_frame_search > 2 && @@ -2450,6 +2452,15 @@ // for allowed compound modes, setup ref mv stack and reference frame. if (idx >= num_inter_modes) { const int comp_index = idx - num_inter_modes; + const COMP_REF_MODE *const comp_modes = is_one_pass_rt_lag_params(cpi) + ? comp_ref_mode_set_full + : comp_ref_mode_set; + const MV_REFERENCE_FRAME *rf = comp_modes[comp_index].ref_frame; + *this_mode = comp_modes[comp_index].pred_mode; + *ref_frame = rf[0]; + *ref_frame2 = rf[1]; + assert(*ref_frame == LAST_FRAME); + assert(*this_mode == GLOBAL_GLOBALMV || *this_mode == NEAREST_NEARESTMV); if (!setup_compound_params_from_comp_idx( cpi, x, search_state->yv12_mb, this_mode, ref_frame, ref_frame2, search_state->frame_mv, search_state->use_ref_frame_mask, @@ -2459,9 +2470,15 @@ } *is_single_pred = 0; } else { - *this_mode = ref_mode_set[idx].pred_mode; - *ref_frame = ref_mode_set[idx].ref_frame; *ref_frame2 = NONE_FRAME; + const REF_MODE *const single_modes = + is_one_pass_rt_lag_params(cpi) ? ref_mode_set_full : ref_mode_set; + *this_mode = single_modes[idx].pred_mode; + *ref_frame = single_modes[idx].ref_frame; + } + + if (is_one_pass_rt_lag_params(cpi) && cpi->rc.is_src_frame_alt_ref) { + if (*this_mode != GLOBALMV || *ref_frame != ALTREF_FRAME) return true; } if (cpi->sf.rt_sf.skip_newmv_mode_sad_screen && cpi->rc.high_source_sad && @@ -2540,6 +2557,7 @@ // Skip compound mode based on variance of previously evaluated single // reference modes. if (rt_sf->prune_compoundmode_with_singlemode_var && !*is_single_pred && + !is_one_pass_rt_lag_params(cpi) && prune_compoundmode_with_singlemode_var( *this_mode, *ref_frame, *ref_frame2, search_state->frame_mv, search_state->mode_checked, search_state->vars, @@ -2596,6 +2614,8 @@ return true; } + if (is_one_pass_rt_lag_params(cpi)) return false; + // Skip mode based on block size, reference frame mode and other block // properties. if (skip_mode_by_bsize_and_ref_frame( @@ -2682,6 +2702,8 @@ RD_STATS nonskip_rdc; av1_invalid_rd_stats(&nonskip_rdc); + (void)ctx; + if (x->sb_me_block && this_mode == NEWMV && ref_frame == LAST_FRAME) { // Set the NEWMV_LAST to the sb MV. search_state->frame_mv[NEWMV][LAST_FRAME].as_int = x->sb_me_mv.as_int; @@ -3034,7 +3056,7 @@ if (search_state->this_rdc.rdcost < search_state->best_rdc.rdcost) { search_state->best_rdc = search_state->this_rdc; *best_early_term = this_early_term; - update_search_state_nonrd(search_state, mi, txfm_info, &nonskip_rdc, ctx, + update_search_state_nonrd(search_state, mi, txfm_info, &nonskip_rdc, this_best_mode, sse_y); // This is needed for the compound modes. @@ -3074,7 +3096,6 @@ struct macroblockd_plane *const pd = &xd->plane[0]; const int bw = block_size_wide[bsize]; const int bh = block_size_high[bsize]; - TxfmSearchInfo *txfm_info = &x->txfm_search_info; BEST_PICKMODE *const best_pickmode = &search_state->best_pickmode; // TODO(marpan): Only allow for 8 bit-depth for now, re-enable for 10/12 bit @@ -3129,10 +3150,6 @@ best_pickmode->tx_type = IDTX; search_state->best_rdc.rdcost = idx_rdcost; best_pickmode->best_mode_skip_txfm = idtx_rdc.skip_txfm; - if (!idtx_rdc.skip_txfm) { - memcpy(ctx->blk_skip, txfm_info->blk_skip, - sizeof(txfm_info->blk_skip[0]) * ctx->num_4x4_blk); - } xd->tx_type_map[0] = best_pickmode->tx_type; memset(ctx->tx_type_map, best_pickmode->tx_type, ctx->num_4x4_blk); memset(xd->tx_type_map, best_pickmode->tx_type, ctx->num_4x4_blk); @@ -3176,10 +3193,7 @@ if (x->color_sensitivity[COLOR_SENS_IDX(AOM_PLANE_U)] || x->color_sensitivity[COLOR_SENS_IDX(AOM_PLANE_V)]) search_state->this_rdc.skip_txfm = 0; - if (!search_state->this_rdc.skip_txfm) { - memcpy(ctx->blk_skip, txfm_info->blk_skip, - sizeof(txfm_info->blk_skip[0]) * ctx->num_4x4_blk); - } + if (xd->tx_type_map[0] != DCT_DCT) av1_copy_array(ctx->tx_type_map, xd->tx_type_map, ctx->num_4x4_blk); } @@ -3268,7 +3282,8 @@ int best_early_term = 0; int force_skip_low_temp_var = 0; unsigned int sse_zeromv_norm = UINT_MAX; - const int num_inter_modes = NUM_INTER_MODES; + const int num_inter_modes = + is_one_pass_rt_lag_params(cpi) ? NUM_INTER_MODES_FULL : NUM_INTER_MODES; const REAL_TIME_SPEED_FEATURES *const rt_sf = &cpi->sf.rt_sf; bool check_globalmv = rt_sf->check_globalmv_on_single_ref; PRED_BUFFER tmp_buffer[4]; @@ -3299,7 +3314,9 @@ int_mv svc_mv = { .as_int = 0 }; int force_mv_inter_layer = 0; bool comp_use_zero_zeromv_only = 0; - int tot_num_comp_modes = NUM_COMP_INTER_MODES_RT; + int tot_num_comp_modes = is_one_pass_rt_lag_params(cpi) + ? NUM_COMP_INTER_MODES_RT_FULL + : NUM_COMP_INTER_MODES_RT; #if CONFIG_AV1_TEMPORAL_DENOISING const int denoise_recheck_zeromv = 1; AV1_PICKMODE_CTX_DEN ctx_den; @@ -3396,9 +3413,14 @@ tx_mode_to_biggest_tx_size[txfm_params->tx_mode_search_type]), TX_16X16); - fill_single_inter_mode_costs(search_state.single_inter_mode_costs, - num_inter_modes, ref_mode_set, mode_costs, - mbmi_ext->mode_context); + if (!is_one_pass_rt_lag_params(cpi)) + fill_single_inter_mode_costs(search_state.single_inter_mode_costs, + num_inter_modes, ref_mode_set, mode_costs, + mbmi_ext->mode_context); + else + fill_single_inter_mode_costs(search_state.single_inter_mode_costs, + num_inter_modes, ref_mode_set_full, mode_costs, + mbmi_ext->mode_context); MV_REFERENCE_FRAME last_comp_ref_frame = NONE_FRAME; @@ -3584,7 +3606,7 @@ search_state.ref_costs_single[INTRA_FRAME], reuse_inter_pred, &orig_dst, tmp_buffer, &this_mode_pred, &search_state.best_rdc, - best_pickmode, ctx, &best_intra_sad_norm); + best_pickmode, &best_intra_sad_norm); int skip_idtx_palette = (x->color_sensitivity[COLOR_SENS_IDX(AOM_PLANE_U)] || x->color_sensitivity[COLOR_SENS_IDX(AOM_PLANE_V)]) && @@ -3625,7 +3647,6 @@ // For lossless: always force the skip flags off. if (is_lossless_requested(&cpi->oxcf.rc_cfg)) { txfm_info->skip_txfm = 0; - memset(ctx->blk_skip, 0, sizeof(ctx->blk_skip[0]) * ctx->num_4x4_blk); } else { txfm_info->skip_txfm = best_pickmode->best_mode_skip_txfm; }
diff --git a/av1/encoder/palette.c b/av1/encoder/palette.c index 79a2fa5..14c8970 100644 --- a/av1/encoder/palette.c +++ b/av1/encoder/palette.c
@@ -232,9 +232,9 @@ int n, uint16_t *color_cache, int n_cache, bool do_header_rd_based_gating, MB_MODE_INFO *best_mbmi, uint8_t *best_palette_color_map, int64_t *best_rd, int *rate, int *rate_tokenonly, int64_t *distortion, uint8_t *skippable, - int *beat_best_rd, PICK_MODE_CONTEXT *ctx, uint8_t *blk_skip, - uint8_t *tx_type_map, int *beat_best_palette_rd, - bool *do_header_rd_based_breakout, int discount_color_cost) { + int *beat_best_rd, PICK_MODE_CONTEXT *ctx, uint8_t *tx_type_map, + int *beat_best_palette_rd, bool *do_header_rd_based_breakout, + int discount_color_cost) { if (do_header_rd_based_breakout != NULL) *do_header_rd_based_breakout = false; optimize_palette_colors(color_cache, n_cache, n, 1, centroids, cpi->common.seq_params->bit_depth); @@ -314,8 +314,6 @@ memcpy(best_palette_color_map, color_map, block_width * block_height * sizeof(color_map[0])); *best_mbmi = *mbmi; - memcpy(blk_skip, x->txfm_search_info.blk_skip, - sizeof(x->txfm_search_info.blk_skip[0]) * ctx->num_4x4_blk); av1_copy_array(tx_type_map, xd->tx_type_map, ctx->num_4x4_blk); if (rate) *rate = this_rate; if (rate_tokenonly) *rate_tokenonly = tokenonly_rd_stats.rate; @@ -342,7 +340,7 @@ int n_cache, MB_MODE_INFO *best_mbmi, uint8_t *best_palette_color_map, int64_t *best_rd, int *rate, int *rate_tokenonly, int64_t *distortion, uint8_t *skippable, int *beat_best_rd, PICK_MODE_CONTEXT *ctx, - uint8_t *best_blk_skip, uint8_t *tx_type_map, int discount_color_cost) { + uint8_t *tx_type_map, int discount_color_cost) { int16_t centroids[PALETTE_MAX_SIZE]; int n = start_n; int top_color_winner = end_n; @@ -357,9 +355,9 @@ palette_rd_y(cpi, x, mbmi, bsize, dc_mode_cost, data, centroids, n, color_cache, n_cache, do_header_rd_based_gating, best_mbmi, best_palette_color_map, best_rd, rate, rate_tokenonly, - distortion, skippable, beat_best_rd, ctx, best_blk_skip, - tx_type_map, &beat_best_palette_rd, - &do_header_rd_based_breakout, discount_color_cost); + distortion, skippable, beat_best_rd, ctx, tx_type_map, + &beat_best_palette_rd, &do_header_rd_based_breakout, + discount_color_cost); *last_n_searched = n; if (do_header_rd_based_breakout) { // Terminate palette_size search by setting last_n_searched to end_n. @@ -389,8 +387,8 @@ int n_cache, MB_MODE_INFO *best_mbmi, uint8_t *best_palette_color_map, int64_t *best_rd, int *rate, int *rate_tokenonly, int64_t *distortion, uint8_t *skippable, int *beat_best_rd, PICK_MODE_CONTEXT *ctx, - uint8_t *best_blk_skip, uint8_t *tx_type_map, uint8_t *color_map, - int data_points, int discount_color_cost) { + uint8_t *tx_type_map, uint8_t *color_map, int data_points, + int discount_color_cost) { int16_t centroids[PALETTE_MAX_SIZE]; const int max_itr = 50; int n = start_n; @@ -410,9 +408,9 @@ palette_rd_y(cpi, x, mbmi, bsize, dc_mode_cost, data, centroids, n, color_cache, n_cache, do_header_rd_based_gating, best_mbmi, best_palette_color_map, best_rd, rate, rate_tokenonly, - distortion, skippable, beat_best_rd, ctx, best_blk_skip, - tx_type_map, &beat_best_palette_rd, - &do_header_rd_based_breakout, discount_color_cost); + distortion, skippable, beat_best_rd, ctx, tx_type_map, + &beat_best_palette_rd, &do_header_rd_based_breakout, + discount_color_cost); *last_n_searched = n; if (do_header_rd_based_breakout) { // Terminate palette_size search by setting last_n_searched to end_n. @@ -543,8 +541,7 @@ const AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bsize, int dc_mode_cost, MB_MODE_INFO *best_mbmi, uint8_t *best_palette_color_map, int64_t *best_rd, int *rate, int *rate_tokenonly, int64_t *distortion, uint8_t *skippable, - int *beat_best_rd, PICK_MODE_CONTEXT *ctx, uint8_t *best_blk_skip, - uint8_t *tx_type_map) { + int *beat_best_rd, PICK_MODE_CONTEXT *ctx, uint8_t *tx_type_map) { MACROBLOCKD *const xd = &x->e_mbd; MB_MODE_INFO *const mbmi = xd->mi[0]; assert(!is_inter_block(mbmi)); @@ -662,7 +659,7 @@ cpi, x, mbmi, bsize, dc_mode_cost, data, top_colors, min_n, max_n + 1, step_size, do_header_rd_based_gating, &unused, color_cache, n_cache, best_mbmi, best_palette_color_map, best_rd, rate, rate_tokenonly, - distortion, skippable, beat_best_rd, ctx, best_blk_skip, tx_type_map, + distortion, skippable, beat_best_rd, ctx, tx_type_map, discount_color_cost); // Evaluate neighbors for the winner color (if winner is found) in the // above coarse search for dominant colors @@ -676,8 +673,8 @@ stage2_max_n + 1, stage2_step_size, /*do_header_rd_based_gating=*/false, &unused, color_cache, n_cache, best_mbmi, best_palette_color_map, best_rd, rate, rate_tokenonly, - distortion, skippable, beat_best_rd, ctx, best_blk_skip, - tx_type_map, discount_color_cost); + distortion, skippable, beat_best_rd, ctx, tx_type_map, + discount_color_cost); } // K-means clustering. // Perform k-means coarse palette search to find the winner candidate @@ -686,8 +683,7 @@ min_n, max_n + 1, step_size, do_header_rd_based_gating, &unused, color_cache, n_cache, best_mbmi, best_palette_color_map, best_rd, rate, rate_tokenonly, distortion, skippable, beat_best_rd, ctx, - best_blk_skip, tx_type_map, color_map, rows * cols, - discount_color_cost); + tx_type_map, color_map, rows * cols, discount_color_cost); // Evaluate neighbors for the winner color (if winner is found) in the // above coarse search for k-means if (k_means_winner <= max_n) { @@ -700,8 +696,8 @@ start_n_stage2, end_n_stage2 + 1, step_size_stage2, /*do_header_rd_based_gating=*/false, &unused, color_cache, n_cache, best_mbmi, best_palette_color_map, best_rd, rate, rate_tokenonly, - distortion, skippable, beat_best_rd, ctx, best_blk_skip, - tx_type_map, color_map, rows * cols, discount_color_cost); + distortion, skippable, beat_best_rd, ctx, tx_type_map, color_map, + rows * cols, discount_color_cost); } } else { const int max_n = AOMMIN(colors, PALETTE_MAX_SIZE), @@ -712,7 +708,7 @@ cpi, x, mbmi, bsize, dc_mode_cost, data, top_colors, min_n, max_n + 1, 1, do_header_rd_based_gating, &last_n_searched, color_cache, n_cache, best_mbmi, best_palette_color_map, best_rd, rate, rate_tokenonly, - distortion, skippable, beat_best_rd, ctx, best_blk_skip, tx_type_map, + distortion, skippable, beat_best_rd, ctx, tx_type_map, discount_color_cost); if (last_n_searched < max_n) { // Search in descending order until we get to the previous best @@ -721,7 +717,7 @@ last_n_searched, -1, /*do_header_rd_based_gating=*/false, &unused, color_cache, n_cache, best_mbmi, best_palette_color_map, best_rd, rate, rate_tokenonly, distortion, skippable, beat_best_rd, ctx, - best_blk_skip, tx_type_map, discount_color_cost); + tx_type_map, discount_color_cost); } // K-means clustering. if (colors == PALETTE_MIN_SIZE) { @@ -733,8 +729,7 @@ color_cache, n_cache, /*do_header_rd_based_gating=*/false, best_mbmi, best_palette_color_map, best_rd, rate, rate_tokenonly, distortion, skippable, beat_best_rd, ctx, - best_blk_skip, tx_type_map, NULL, NULL, - discount_color_cost); + tx_type_map, NULL, NULL, discount_color_cost); } else { // Perform k-means palette search in ascending order last_n_searched = min_n; @@ -743,8 +738,7 @@ min_n, max_n + 1, 1, do_header_rd_based_gating, &last_n_searched, color_cache, n_cache, best_mbmi, best_palette_color_map, best_rd, rate, rate_tokenonly, distortion, skippable, beat_best_rd, ctx, - best_blk_skip, tx_type_map, color_map, rows * cols, - discount_color_cost); + tx_type_map, color_map, rows * cols, discount_color_cost); if (last_n_searched < max_n) { // Search in descending order until we get to the previous best perform_k_means_palette_search( @@ -752,8 +746,8 @@ max_n, last_n_searched, -1, /*do_header_rd_based_gating=*/false, &unused, color_cache, n_cache, best_mbmi, best_palette_color_map, best_rd, rate, rate_tokenonly, distortion, skippable, - beat_best_rd, ctx, best_blk_skip, tx_type_map, color_map, - rows * cols, discount_color_cost); + beat_best_rd, ctx, tx_type_map, color_map, rows * cols, + discount_color_cost); } } }
diff --git a/av1/encoder/palette.h b/av1/encoder/palette.h index ce10949..ac1112f 100644 --- a/av1/encoder/palette.h +++ b/av1/encoder/palette.h
@@ -174,7 +174,7 @@ int dc_mode_cost, MB_MODE_INFO *best_mbmi, uint8_t *best_palette_color_map, int64_t *best_rd, int *rate, int *rate_tokenonly, int64_t *distortion, uint8_t *skippable, int *beat_best_rd, struct PICK_MODE_CONTEXT *ctx, - uint8_t *best_blk_skip, uint8_t *tx_type_map); + uint8_t *tx_type_map); /*!\brief Search for the best palette in the chroma plane. *
diff --git a/av1/encoder/partition_model_weights.h b/av1/encoder/partition_model_weights.h index 28d960d..576b981 100644 --- a/av1/encoder/partition_model_weights.h +++ b/av1/encoder/partition_model_weights.h
@@ -1317,6 +1317,842 @@ #define FEATURE_SIZE 18 #define LABEL_SIZE 4 +#define NEW_LABEL_SIZE 3 + +static const float av1_partition4_search_thresh[6][3][5] = { + // Aggressiveness = 0 + { + // lowres + { 1.0f, 0.7451170578367811f, 0.7001978100387174f, 0.7410514819747771f, + 1.0f }, + // midres + { 1.0f, 0.7494756052316782f, 0.6892602739627772f, 0.8573332311578094f, + 1.0f }, + // hdres + { 1.0f, 0.7494756052316782f, 0.6892602739627772f, 0.8573332311578094f, + 1.0f }, + }, + // Aggressiveness = 1 + { + // lowres + { 1.0f, 0.8885108052540076f, 0.6485475489414385f, 0.8310364661853259f, + 1.0f }, + // midres + { 1.0f, 0.6669407157796841f, 0.7813399048525052f, 0.8876257905832826f, + 1.0f }, + // hdres + { 1.0f, 0.6669407157796841f, 0.7813399048525052f, 0.8876257905832826f, + 1.0f }, + }, + // Aggressiveness = 2 + { + // lowres + { 1.0f, 0.8885108052540076f, 0.6485475489414385f, 0.8310364661853259f, + 1.0f }, + // midres + { 1.0f, 0.6669407157796841f, 0.7813399048525052f, 0.8876257905832826f, + 1.0f }, + // hdres + { 1.0f, 0.6669407157796841f, 0.7813399048525052f, 0.8876257905832826f, + 1.0f }, + }, + // Aggressiveness = 3 + { + // lowres + { 1.0f, 1.0f, 1.0f, 1.0f, 1.0f }, + // midres + { 1.0f, 1.0f, 1.0f, 1.0f, 1.0f }, + // hdres + { 1.0f, 1.0f, 1.0f, 1.0f, 1.0f }, + }, + // Aggressiveness = 4 + { + // lowres + { 1.0f, 1.0f, 1.0f, 1.0f, 1.0f }, + // midres + { 1.0f, 1.0f, 1.0f, 1.0f, 1.0f }, + // hdres + { 1.0f, 1.0f, 1.0f, 1.0f, 1.0f }, + }, + // Aggressiveness = 5 + { + // lowres + { 1.0f, 1.0f, 1.0f, 1.0f, 1.0f }, + // midres + { 1.0f, 1.0f, 1.0f, 1.0f, 1.0f }, + // hdres + { 1.0f, 1.0f, 1.0f, 1.0f, 1.0f }, + }, +}; + +static const float av1_partition4_not_search_thresh[6][3][5] = { + // Aggressiveness = 0 + { + // lowres + { 0.0f, 0.001052842338862181f, 0.03450079872362423f, + 0.002759167859733889f, 0.0f }, + // midres + { 0.0f, 0.07053592384393781f, 0.03838345436035215f, 0.005675936031290841f, + 0.0f }, + // hdres + { 0.0f, 0.07053592384393781f, 0.03838345436035215f, 0.005675936031290841f, + 0.0f }, + }, + // Aggressiveness = 1 + { + // lowres + { 0.0f, 0.003841755695390817f, 0.04822550751756426f, + 0.008487399657253631f, 0.0f }, + // midres + { 0.0f, 0.0011047168521408242f, 0.04736033094945354f, + 0.01638628311248205f, 0.0f }, + // hdres + { 0.0f, 0.0011047168521408242f, 0.04736033094945354f, + 0.01638628311248205f, 0.0f }, + }, + // Aggressiveness = 2 + { + // lowres + { 0.0f, 0.003841755695390817f, 0.04822550751756426f, + 0.008487399657253631f, 0.0f }, + // midres + { 0.0f, 0.0011047168521408242f, 0.04736033094945354f, + 0.01638628311248205f, 0.0f }, + // hdres + { 0.0f, 0.0011047168521408242f, 0.04736033094945354f, + 0.01638628311248205f, 0.0f }, + + }, + // Aggressiveness = 3 + { + // lowres + { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f }, + // midres + { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f }, + // hdres + { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f }, + }, + // Aggressiveness = 4 + { + // lowres + { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f }, + // midres + { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f }, + // hdres + { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f }, + }, + // Aggressiveness = 5 + { + // lowres + { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f }, + // midres + { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f }, + // hdres + { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f }, + }, +}; + +static const float av1_partition4_nn_mean_64[FEATURE_SIZE] = { + 1.3752231712399923f, 7.8818296620238755f, 0.70302558915341073f, + 0.89354391968945868f, 0.69912435960273867f, 0.894260991860082f, + 0.3926311487269134f, 0.396667092758365f, 0.39248475739497596f, + 0.60199542223376412f, 0.79385711361043609f, 0.71296985320460649f, + 0.70907736145028422f, 0.79726457629807679f, 0.78323554106033988f, + 0.70020724711817484f, 0.703606514382454f, 0.812416459726476f, +}; + +static const float av1_partition4_nn_std_64[FEATURE_SIZE] = { + 1.5210330851172169f, 3.2502345451890569f, 0.31526449400836393f, + 0.28189232462592312f, 0.31774840567547408f, 0.28162114702432761f, + 0.32736466480428811f, 0.32670076454358127f, 0.327642806340881f, + 0.39886024415245425f, 0.58334698402992669f, 0.43936371898261278f, + 0.43195994078504846f, 0.6085133217391806f, 0.58153035818647314f, + 0.43649370363303375f, 0.43352660981855345f, 0.6044169876719907f +}; + +static const float av1_partition4_nn_mean_32[FEATURE_SIZE] = { + 1.2246859239905685f, 8.2993701373465054f, 0.766659262212363f, + 0.94652387284747963f, 0.7600040783826757f, 0.95272747538676006f, + 0.65159866132436739f, 0.65257588048947879f, 0.66423008846285458f, + 0.76469913589630689f, 0.82278748860287532f, 0.74684876377199483f, + 0.747579068836797f, 0.81934887236245424f, 0.83835878037641287f, + 0.75032880096189358f, 0.75488102829056769f, 0.84759980602684326f, +}; + +static const float av1_partition4_nn_std_32[FEATURE_SIZE] = { + 1.6075309663410215f, 2.5730348757929398f, 0.26321073080207291f, + 0.16943329319298692f, 0.2651485345364597f, 0.16189549806595446f, + 0.37708176589960463f, 0.37678428010229964f, 0.376236054947869f, + 0.35251481923629219f, 0.54812998882840647f, 0.40597153804110392f, + 0.4062410958984316f, 0.55966403917740415f, 0.551261263412241f, + 0.39617447478383311f, 0.3975763070782759f, 0.56159171003404f +}; + +static const float av1_partition4_nn_mean_16[FEATURE_SIZE] = { + 1.0045835791325202f, 8.4894105104733484f, 0.662368560626388f, + 0.907764748376923f, 0.6627159222349438f, 0.936766105471581f, + 0.45489736309556683f, 0.46132801974042803f, 0.46931301089726096f, + 0.85059414467649674f, 0.779699826074789f, 0.70609877051124081f, + 0.711834073256916f, 0.79712849358962179f, 0.8351636078095156f, + 0.75367264126747469f, 0.75873775601873783f, 0.8466653756621495f +}; + +static const float av1_partition4_nn_std_16[FEATURE_SIZE] = { + 1.6427502751165233f, 2.3099057132670047f, 0.24533039885709371f, + 0.19978306019327963f, 0.24478191346772621f, 0.17204070228378546f, + 0.33179173716308064f, 0.33283691607603066f, 0.33758170469103288f, + 0.3009885171419176f, 0.550933084988767f, 0.41342659059456571f, + 0.41384314531287553f, 0.56715063304567692f, 0.54874118341614453f, + 0.39768773547851133f, 0.397536461220648f, 0.55655009687449353f +}; + +static const float *const av1_partition4_nn_mean[5] = { + NULL, + av1_partition4_nn_mean_64, + av1_partition4_nn_mean_32, + av1_partition4_nn_mean_16, + NULL, +}; + +static const float *const av1_partition4_nn_std[5] = { + NULL, + av1_partition4_nn_std_64, + av1_partition4_nn_std_32, + av1_partition4_nn_std_16, + NULL, +}; + +static const float + av1_hd_4_partition_nn_weights_64_layer0[FEATURE_SIZE * 24] = { + 0.187540665268898f, 0.008867358788847923f, 1.3245491981506348f, + 0.18199755251407623f, -1.320246934890747f, -0.11215319484472275f, + -0.2073437124490738f, -1.6804399490356445f, 1.0172151327133179f, + -0.04451597109436989f, 0.3141838610172272f, -0.10051298141479492f, + 0.19940334558486938f, 0.3332000970840454f, -0.19904619455337524f, + 0.575351357460022f, -0.3605368435382843f, -0.2180677056312561f, + -0.08732779324054718f, -0.5763136148452759f, -0.4628996253013611f, + -0.19735047221183777f, -0.2669188976287842f, -0.18305927515029907f, + 1.9265873432159424f, 0.44848161935806274f, 0.48354148864746094f, + 0.883162796497345f, -0.2056751400232315f, 0.26105034351348877f, + 0.22955726087093353f, -0.07927738130092621f, 0.22645841538906097f, + -0.07682249695062637f, 0.46167850494384766f, 0.03855176270008087f, + 0.27271342277526855f, -0.032083675265312195f, 1.2821667194366455f, + 0.2666876018047333f, -0.09309065341949463f, -0.15320968627929688f, + -1.6731971502304077f, -1.3742161989212036f, 0.9385388493537903f, + -0.008384025655686855f, 0.24508674442768097f, 0.1386878341436386f, + 0.02423120103776455f, 0.19301238656044006f, 0.13091489672660828f, + -0.6223750710487366f, -0.051762085407972336f, 0.3243873119354248f, + -0.17509253323078156f, -0.30384597182273865f, 2.6790037155151367f, + 0.20672129094600677f, -0.2797843813896179f, 0.4765399992465973f, + -0.45524242520332336f, 0.39230310916900635f, -0.23308643698692322f, + -0.3281370997428894f, -0.33606868982315063f, -0.24563899636268616f, + -0.11075028777122498f, 0.26341673731803894f, -0.1780794858932495f, + -0.3037680685520172f, 0.6263296604156494f, -0.7237133383750916f, + -0.28761041164398193f, 0.09403169900178909f, 0.5554772019386292f, + 0.08419027924537659f, -0.20748449862003326f, 0.8960804343223572f, + 1.731931447982788f, 0.9879814386367798f, -0.7795932292938232f, + -0.08355121314525604f, -1.2876038551330566f, 0.2683749794960022f, + 0.046085603535175323f, 0.6003941297531128f, 0.39504218101501465f, + 0.42627424001693726f, 0.2481756955385208f, 0.2539105713367462f, + 0.3717970848083496f, -1.372495174407959f, 1.779883623123169f, + 0.0023461435921490192f, -0.4034990966320038f, -0.005759629420936108f, + -0.3803994655609131f, -0.3320770561695099f, 2.070441961288452f, + 0.3284848630428314f, 0.35943546891212463f, -0.7431299090385437f, + 0.327751100063324f, 0.42656365036964417f, 0.0988604873418808f, + 0.17223680019378662f, -0.14063750207424164f, 0.01995408907532692f, + -1.0309178829193115f, -0.18265926837921143f, -0.9445133805274963f, + 0.1541268527507782f, 0.7334791421890259f, 1.5515707731246948f, + -0.14698545634746552f, 0.49188539385795593f, -0.6401307582855225f, + -0.24233347177505493f, -0.24929864704608917f, 0.7331327795982361f, + -0.24525558948516846f, -0.6303561925888062f, 0.2914137542247772f, + -0.1501816362142563f, 0.33382299542427063f, 0.20683197677135468f, + 0.6331838965415955f, 0.21398495137691498f, -0.7407512068748474f, + -0.38856998085975647f, 1.4056344032287598f, 0.2766624987125397f, + -0.47957277297973633f, 0.4720071256160736f, -2.237626791000366f, + 0.17220094799995422f, 0.1279810518026352f, -0.170374795794487f, + 0.15627238154411316f, 0.06910572201013565f, 0.1091245785355568f, + 0.13457776606082916f, 0.0723930299282074f, 0.18878978490829468f, + -0.9244682788848877f, 0.49643731117248535f, 0.00938953272998333f, + 0.817326009273529f, -0.872970700263977f, 1.2162615060806274f, + 1.6745234727859497f, 0.7649728655815125f, 0.6812664270401001f, + 0.7555683851242065f, -0.3641526997089386f, -0.5201636552810669f, + 0.33287081122398376f, -0.015199027955532074f, 0.11200721561908722f, + 0.1262284219264984f, -0.30170801281929016f, 0.29431891441345215f, + -0.8517263531684875f, -0.011521201580762863f, -0.2496422827243805f, + 0.4014142155647278f, -1.337280035018921f, -0.4126336872577667f, + 1.4324754476547241f, 0.5764614343643188f, 0.9383725523948669f, + 0.15321384370326996f, 0.16989222168922424f, 0.3319786489009857f, + -0.07834333181381226f, 0.13870780169963837f, 0.06260085105895996f, + 0.1040545254945755f, -0.04136618599295616f, -0.1875564157962799f, + 0.12765266001224518f, 0.119786836206913f, 2.6260948181152344f, + -0.718943178653717f, 2.216468572616577f, 0.2265452891588211f, + 0.3913697898387909f, 1.3077619075775146f, 0.8483156561851501f, + 0.17309871315956116f, -0.032115768641233444f, 0.06523145735263824f, + -0.039812054485082626f, 0.20178377628326416f, 0.09728340059518814f, + 0.06543758511543274f, -0.018573174253106117f, 0.22792796790599823f, + -0.25049784779548645f, -0.26964524388313293f, 0.3470650613307953f, + 0.07785002142190933f, 0.9330359697341919f, 0.13356055319309235f, + 0.5518191456794739f, -0.03740643337368965f, -1.0142114162445068f, + -0.3127419948577881f, 0.6786038279533386f, -0.746333658695221f, + 0.7910341024398804f, -0.5631489753723145f, 0.13812024891376495f, + 0.5231190919876099f, -0.4686489403247833f, 0.09117632359266281f, + 0.7155937552452087f, -0.7463059425354004f, -0.4379642903804779f, + 0.48007652163505554f, 0.0414154939353466f, 0.08547665923833847f, + -0.08740316331386566f, -0.3762202262878418f, 1.650416374206543f, + 0.6305139660835266f, 0.2807236909866333f, 0.3252447545528412f, + -1.2813769578933716f, 0.1806870996952057f, -0.07484257966279984f, + -0.4510236084461212f, -0.32734254002571106f, -0.2525199055671692f, + -0.6823204755783081f, 0.4419010579586029f, -0.13107456266880035f, + 1.5115501880645752f, -1.0415934324264526f, 0.6362841725349426f, + 1.1276127099990845f, 1.6090517044067383f, 1.4753515720367432f, + -0.5725038647651672f, 0.4504983127117157f, 0.0542747899889946f, + 0.23436619341373444f, -0.09478037059307098f, 0.17619547247886658f, + 0.5189757943153381f, 0.3518279492855072f, -0.20417530834674835f, + -0.3984152376651764f, 1.4910639524459839f, 0.896291196346283f, + -0.07476390153169632f, -0.38988080620765686f, -0.36142808198928833f, + -0.5949490070343018f, -0.34817272424697876f, 2.390124559402466f, + 0.036864519119262695f, -0.29277393221855164f, 0.2582416832447052f, + 0.12344545125961304f, -0.19278477132320404f, -0.3593710958957672f, + 0.052842043340206146f, -0.4351940155029297f, 0.39930951595306396f, + -0.05878692492842674f, -0.13607178628444672f, -0.3738308250904083f, + 0.07306762039661407f, 0.14370614290237427f, 0.7443045377731323f, + -0.6571366786956787f, 1.4732838869094849f, 0.07397766411304474f, + -0.026601698249578476f, 0.16516001522541046f, -0.09079989790916443f, + 0.08782806247472763f, -0.02378779835999012f, 1.2652226686477661f, + 1.1744853258132935f, 1.0768793821334839f, 1.2959463596343994f, + 0.18875858187675476f, -0.16353435814380646f, 0.12929506599903107f, + -0.028879646211862564f, 1.7920417785644531f, 0.2096109390258789f, + 0.9538826942443848f, -1.3157790899276733f, 1.025888204574585f, + 0.14866796135902405f, 0.11838590353727341f, -0.1063050851225853f, + 0.10628897696733475f, -0.07982262223958969f, -0.26907843351364136f, + 0.010558247566223145f, 0.13393597304821014f, 0.22190389037132263f, + -0.19494540989398956f, 0.3906410038471222f, -0.9874037504196167f, + 0.651418149471283f, -0.8602163791656494f, 0.32449445128440857f, + 1.2791974544525146f, 1.0395246744155884f, 1.5486201047897339f, + -0.7350376844406128f, -0.6816073060035706f, 0.016160577535629272f, + -0.18916243314743042f, -0.26918724179267883f, 0.4815976321697235f, + 0.2128375917673111f, 0.32033786177635193f, 0.5065921545028687f, + -0.9307171106338501f, 0.007387777790427208f, -0.45773541927337646f, + -0.19515591859817505f, -0.28307247161865234f, 1.3757314682006836f, + 1.5909974575042725f, 1.4594842195510864f, 1.3418850898742676f, + 1.449257493019104f, 0.1290784776210785f, -0.02872903272509575f, + 0.0876379907131195f, 0.0945722684264183f, -0.11620496213436127f, + 0.03221723809838295f, 0.01832464151084423f, -0.0350685641169548f, + 0.2947345972061157f, -0.6470182538032532f, 0.7246863842010498f, + -0.04240961745381355f, -0.8078872561454773f, 0.2905123829841614f, + 0.9782440066337585f, -0.7729941010475159f, 0.9055317044258118f, + 0.037821024656295776f, 0.123793825507164f, 0.17636357247829437f, + 0.022636424750089645f, 0.31781089305877686f, 0.4737413227558136f, + -0.6823905110359192f, 1.179976463317871f, -0.6974776983261108f, + -0.7578544616699219f, 0.4514416754245758f, 0.8697171211242676f, + -0.45408639311790466f, 0.06114231050014496f, 0.4886225461959839f, + -0.902211606502533f, 0.2965964078903198f, 0.0068778120912611485f, + 0.7137743830680847f, 0.3116300702095032f, -0.2853015661239624f, + 0.9911389350891113f, -0.21889007091522217f, 0.2786492705345154f, + -0.1595863401889801f, -0.2824990749359131f, 0.6888416409492493f, + -0.8445804715156555f, -0.07846039533615112f, -0.9218422174453735f, + -0.36275193095207214f, -0.24915170669555664f, -0.022376133129000664f, + 0.9710543751716614f, 1.1314961910247803f, 0.8179596662521362f, + -0.12418682873249054f, 0.1507333666086197f, -0.22915267944335938f, + 0.0621408149600029f, -0.18360091745853424f, 0.045976653695106506f, + -0.039411406964063644f, 0.09902478009462357f, 0.02785186842083931f, + -0.11535168439149857f, -0.021012280136346817f, -0.8498482704162598f, + -0.08284465223550797f, 1.7017556428909302f, 0.09805655479431152f, + -0.819100558757782f, 1.2941229343414307f, -1.4690512418746948f, + -0.14031103253364563f, -0.11753756552934647f, -0.09136147797107697f, + -0.22433844208717346f, 0.553429126739502f, 0.03767363727092743f, + 0.013898416422307491f, 0.25408342480659485f, 0.047054700553417206f, + 0.38260170817375183f, -0.10799504816532135f, 0.7206213474273682f, + 0.40167760848999023f, 0.33209502696990967f, -0.05047299340367317f, + -1.4984959363937378f, -0.7021075487136841f, 0.33913472294807434f, + 0.1524118036031723f, -0.15081769227981567f, -0.27317649126052856f, + -0.15303443372249603f, -0.6619091033935547f, -0.1907443255186081f, + 0.0709405466914177f, 0.2743484377861023f, -0.12694808840751648f + }; + +static const float av1_hd_4_partition_nn_bias_64_layer0[24] = { + -0.017236731946468353f, 1.2984086275100708f, -0.5433139204978943f, + -0.1250726878643036f, 0.1385560780763626f, -0.715609073638916f, + 0.23101328313350677f, -0.5218021273612976f, 0.4044009745121002f, + 1.2596054077148438f, -1.4503605365753174f, -0.09927619248628616f, + -0.692095160484314f, 0.339007705450058f, -0.9172008633613586f, + -0.6032222509384155f, 1.5522794723510742f, -0.2724784016609192f, + 1.1838854551315308f, 0.1453675925731659f, -0.8429049253463745f, + 1.4830207824707031f, 0.4452385902404785f, -0.19120316207408905f +}; + +static const float + av1_hd_4_partition_nn_weights_64_layer1[24 * NEW_LABEL_SIZE] = { + -0.13316302001476288f, 0.3058302104473114f, 0.1070905551314354f, + 0.14361895620822906f, -0.4736453890800476f, -0.33969804644584656f, + 0.030571846291422844f, 0.33224594593048096f, -0.06829124689102173f, + 0.21595115959644318f, 0.8105601668357849f, -0.13124717772006989f, + -0.2749757170677185f, -0.7534167170524597f, 0.006968092638999224f, + 0.04543568938970566f, -0.22467342019081116f, -0.47277456521987915f, + 0.7455835938453674f, -0.17081353068351746f, -0.33045464754104614f, + 0.2652832865715027f, -0.12469493597745895f, 0.010430725291371346f, + -0.6428546905517578f, -0.06037088856101036f, -0.6064530611038208f, + 0.06202490255236626f, 0.0295811016112566f, 0.2661149799823761f, + 0.09020768105983734f, 0.2919042408466339f, -0.5376702547073364f, + -0.6015531420707703f, -1.8029735088348389f, 0.07257948815822601f, + 0.11942317336797714f, -0.28495341539382935f, -0.5178318619728088f, + 0.416162371635437f, -0.34815943241119385f, -0.2532649636268616f, + -0.8336657285690308f, -0.28184619545936584f, -0.07333157956600189f, + -0.0849723070859909f, -0.012794683687388897f, -0.300246000289917f, + 0.30970948934555054f, 0.06398778408765793f, 0.3188731074333191f, + -0.32415783405303955f, -0.23729059100151062f, -1.2901034355163574f, + -0.3617444634437561f, -0.3765851557254791f, -0.6880390644073486f, + -0.08437152206897736f, -3.1945059299468994f, -0.41093024611473083f, + -0.5344216227531433f, 0.11081822216510773f, -0.4143705666065216f, + -0.28478118777275085f, 0.37445685267448425f, 0.3433232307434082f, + -0.5481890439987183f, 0.31730446219444275f, -0.04274527728557587f, + -0.17332714796066284f, -0.6555780172348022f, -0.1539755016565323f + }; + +static const float av1_hd_4_partition_nn_bias_64_layer1[NEW_LABEL_SIZE] = { + 0.9356993436813354f, -0.9818968772888184f, -0.5307729244232178f +}; + +static const float + av1_hd_4_partition_nn_weights_32_layer0[FEATURE_SIZE * 32] = { + -1.0885568857192993f, 0.7276804447174072f, 0.3899334967136383f, + 1.786031723022461f, 0.047051746398210526f, 0.7338393330574036f, + -1.0688824653625488f, 0.7300519347190857f, 0.5556117296218872f, + 0.15455445647239685f, 0.23771341145038605f, 0.17510148882865906f, + 0.07676580548286438f, 0.33183372020721436f, 0.029779063537716866f, + -0.038887616246938705f, 0.12371493875980377f, -0.1936332881450653f, + 0.4791683554649353f, 0.01103244535624981f, 0.22002530097961426f, + 0.1311550885438919f, -0.25968194007873535f, 0.6977970004081726f, + 0.4742209017276764f, -1.7808669805526733f, 0.5283331871032715f, + 0.12193027883768082f, 0.24376845359802246f, -0.05244125798344612f, + -0.012799731455743313f, -0.11021115630865097f, 0.2635785937309265f, + 0.5267615914344788f, -0.6972594261169434f, 0.5042043924331665f, + 0.5771448612213135f, 0.16394701600074768f, 0.6534922122955322f, + 0.44265198707580566f, 0.001729517593048513f, -0.24405747652053833f, + -0.46474769711494446f, -1.2396752834320068f, 0.4316665828227997f, + -0.40889325737953186f, 0.21149589121341705f, 0.059366464614868164f, + -0.23183700442314148f, 0.26841700077056885f, -0.18424294888973236f, + 0.018943576142191887f, 0.1582796424627304f, -0.04466909542679787f, + 0.1756608933210373f, -0.14650224149227142f, 0.6032110452651978f, + 0.23037950694561005f, 0.05099782347679138f, -0.11585348099470139f, + -0.6360398530960083f, -0.504222571849823f, 1.2582234144210815f, + 0.639336109161377f, -0.08372239023447037f, 0.23684656620025635f, + -0.8035249710083008f, 0.6960340738296509f, -0.1946241408586502f, + 0.26888877153396606f, 0.0608806349337101f, -0.0793396532535553f, + -0.21731014549732208f, -0.44480085372924805f, 0.9669836759567261f, + -0.03727146238088608f, -1.048531413078308f, 0.09407489746809006f, + 0.7357499599456787f, -0.1052381843328476f, 0.7807036638259888f, + -1.2890628576278687f, -0.27905717492103577f, -0.2336570918560028f, + 0.24939477443695068f, -0.1286485493183136f, -0.017393384128808975f, + 0.2383008748292923f, -0.2378801703453064f, 0.046414244920015335f, + 0.04329893738031387f, 0.09692811220884323f, -0.23152312636375427f, + 0.26998770236968994f, 0.11675097048282623f, 0.048871126025915146f, + -0.5766774415969849f, 0.5075773000717163f, -0.16599853336811066f, + 0.07355321198701859f, 1.2679802179336548f, -0.470909059047699f, + -0.7353945970535278f, 0.2871803045272827f, 0.04421471804380417f, + 0.028569340705871582f, 0.07299203425645828f, 0.17615129053592682f, + -0.6759434938430786f, 0.3759593665599823f, -0.03345653414726257f, + 0.15250401198863983f, 1.6251002550125122f, -0.32556575536727905f, + -0.4355400800704956f, 0.18039146065711975f, -0.6343501210212708f, + -0.26397016644477844f, 0.1334858238697052f, 0.17045122385025024f, + 0.06101013347506523f, 0.15618449449539185f, -0.05479817092418671f, + 0.19636914134025574f, -0.171217679977417f, -0.2721777856349945f, + -0.03099956177175045f, -0.09452107548713684f, -0.2100360095500946f, + -0.08023128658533096f, 0.5380405783653259f, 0.04074520245194435f, + -0.45956486463546753f, -0.26170605421066284f, -0.036938007920980453f, + 0.3081861436367035f, 0.7264460921287537f, 0.4579879939556122f, + 0.5227369070053101f, 0.7139924764633179f, -1.3371893167495728f, + 0.8577211499214172f, -0.05075681209564209f, 0.2690950036048889f, + -0.3681940734386444f, 0.08500909060239792f, 0.8850646018981934f, + 0.4467816948890686f, 0.28028661012649536f, 0.27217888832092285f, + 0.7745168209075928f, 0.7731392979621887f, 1.2513463497161865f, + -0.35894691944122314f, 0.060745496302843094f, -0.04395193234086037f, + 0.11425885558128357f, -0.0328013151884079f, -0.1833721548318863f, + -0.07130905985832214f, 0.05300029367208481f, -0.10985758155584335f, + 0.006968754343688488f, 0.9198938012123108f, -0.17839254438877106f, + 0.26150068640708923f, 0.3576587736606598f, -0.23587310314178467f, + -0.0849660113453865f, 0.6160711050033569f, -0.18021133542060852f, + -0.30802857875823975f, 0.43131136894226074f, 0.1159980297088623f, + -0.25173503160476685f, 0.1161213293671608f, -0.1237974539399147f, + -0.7531319260597229f, -0.00878852792084217f, 0.2545885741710663f, + 0.007603700738400221f, -0.654176652431488f, 0.44965240359306335f, + 0.7931277751922607f, 0.18094922602176666f, -0.17669828236103058f, + 0.41031819581985474f, 0.12209723144769669f, -0.2814178168773651f, + -0.5220250487327576f, 0.270843505859375f, -1.7025750875473022f, + 0.019062979146838188f, 0.03137283772230148f, -0.17925892770290375f, + -0.42655882239341736f, -0.6994401216506958f, -0.07981792837381363f, + -0.1174769252538681f, -0.17482306063175201f, 0.07917492091655731f, + 0.5423614382743835f, -0.19157607853412628f, -0.05202263593673706f, + 0.04898262768983841f, 0.9668978452682495f, -0.27084100246429443f, + 0.1934838443994522f, 0.20127049088478088f, 0.07396796345710754f, + -0.06063545122742653f, -0.04891745001077652f, -0.5657607316970825f, + 0.5004168152809143f, -0.8361988663673401f, -0.1273525059223175f, + 0.8776155710220337f, -0.10246502608060837f, 0.46023881435394287f, + 0.07886356115341187f, 0.9126405119895935f, 0.17006607353687286f, + -0.513926088809967f, 0.1184217631816864f, 0.4032308757305145f, + -0.6753421425819397f, -0.043491754680871964f, 0.028111552819609642f, + -0.03926699236035347f, -0.1501464545726776f, -0.0014241111930459738f, + -0.004508028272539377f, 0.1367940455675125f, -0.1648765653371811f, + -0.3467996418476105f, 0.8369541168212891f, 0.8667364120483398f, + 0.6326888799667358f, 0.0986432433128357f, 0.20588631927967072f, + 0.03708481788635254f, 0.7360354065895081f, -0.43546396493911743f, + 0.2251778244972229f, 0.5427256226539612f, -0.08095406740903854f, + 0.9812734723091125f, 0.6085113286972046f, 0.853574812412262f, + 0.6173090934753418f, 0.6338220238685608f, 0.5749073624610901f, + 0.16422225534915924f, -0.024263106286525726f, -0.26709797978401184f, + -0.27645328640937805f, -0.18558774888515472f, 0.19723981618881226f, + 0.3469878137111664f, -0.853264331817627f, 0.1911182701587677f, + -0.15484920144081116f, 0.1289137303829193f, 0.16144156455993652f, + 0.28407949209213257f, 0.06683658063411713f, 0.10071670264005661f, + -0.13541074097156525f, 0.8519083857536316f, -1.3628060817718506f, + 0.044920407235622406f, -0.9304912090301514f, -0.0543082021176815f, + -0.5795891880989075f, 0.7444899678230286f, 0.34974098205566406f, + -0.5224471688270569f, 0.7189369797706604f, 0.002385274972766638f, + 0.28222599625587463f, -0.48682257533073425f, -0.24234719574451447f, + -0.04255838692188263f, 0.12397146970033646f, 0.618983805179596f, + 0.20875586569309235f, 0.1820639818906784f, 0.6538670659065247f, + -0.42951807379722595f, -0.9420881271362305f, 0.7871586680412292f, + 0.3056884706020355f, -0.22634023427963257f, -0.4248776435852051f, + 0.7732703685760498f, -0.559977650642395f, 0.08991166949272156f, + -0.5344432592391968f, 0.4384763538837433f, 0.0689520537853241f, + 0.054295241832733154f, 0.4064214527606964f, 0.33730319142341614f, + -0.5907483696937561f, -0.25099942088127136f, 0.4158354103565216f, + -0.49657127261161804f, -0.05017169192433357f, -0.19485296308994293f, + 0.6315739750862122f, 0.026322035118937492f, -0.022450678050518036f, + 0.7323209643363953f, 0.04370767995715141f, -0.24496810138225555f, + 0.11060530692338943f, -0.17143982648849487f, 0.9523182511329651f, + -0.6612598896026611f, -0.37088659405708313f, -0.06733931601047516f, + 0.10264944285154343f, -0.5029839277267456f, 0.05873759835958481f, + -0.9268806576728821f, 0.08673790097236633f, -0.05127667635679245f, + -0.12422613799571991f, 1.112686276435852f, 0.8936125636100769f, + -0.13199946284294128f, -0.7763554453849792f, -0.43546125292778015f, + 0.2081739604473114f, -0.02353234775364399f, 0.12321862578392029f, + 0.21226866543293f, -0.13369852304458618f, -0.07170510292053223f, + 0.022296279668807983f, -0.056023065000772476f, 0.07766319811344147f, + 0.538470983505249f, -0.09681082516908646f, 0.04744873568415642f, + 0.15997207164764404f, 0.1942814141511917f, -0.606421709060669f, + -1.6964457035064697f, 1.8364115953445435f, -1.2303802967071533f, + 0.3570907413959503f, 0.010215475223958492f, -0.08084885030984879f, + 0.07036393880844116f, 0.08225959539413452f, 0.1582154929637909f, + 0.051107257604599f, 0.007791138254106045f, -0.07944256067276001f, + 0.1391010731458664f, -0.13120266795158386f, -0.22520364820957184f, + 0.5478659868240356f, 0.2474595457315445f, 0.5424359440803528f, + 0.2528058588504791f, -0.3962598741054535f, 0.3255935311317444f, + 0.03695685788989067f, -0.07569821178913116f, 0.11970824003219604f, + -0.09008245170116425f, 0.23039402067661285f, 0.22283776104450226f, + -1.016753911972046f, -0.3937026262283325f, 0.5536431670188904f, + -0.0240758266299963f, 0.4818081855773926f, 0.05481531843543053f, + 0.2614268660545349f, -0.0978444516658783f, 0.5413931608200073f, + -0.12983573973178864f, 0.11426673084497452f, -0.8978381156921387f, + 2.279249429702759f, 0.2615402340888977f, 0.21105653047561646f, + 0.5502492785453796f, 0.21621979773044586f, 0.084201879799366f, + 0.3490462899208069f, -0.026039229705929756f, -0.10916426032781601f, + -0.07801277190446854f, 0.6263523101806641f, 0.6516022086143494f, + 0.4567950665950775f, -0.1453220695257187f, 0.2806280255317688f, + 0.1846635937690735f, -0.39070093631744385f, 0.2953556478023529f, + -0.6223417520523071f, 0.7452172040939331f, -0.43376561999320984f, + 0.08578093349933624f, 0.047524344176054f, -0.1837213784456253f, + -0.1024935394525528f, 0.11444173008203506f, -0.6773277521133423f, + 0.27492958307266235f, -0.09823667258024216f, -0.22346174716949463f, + 0.0019273801008239388f, -2.3131091594696045f, 0.21303993463516235f, + 0.24907292425632477f, -0.2713790833950043f, -0.010520088486373425f, + 0.008633764460682869f, 0.09717638790607452f, -0.06675714254379272f, + -0.15131525695323944f, -0.06854774802923203f, 0.18723946809768677f, + 0.18391983211040497f, -0.03460600599646568f, 0.04807600751519203f, + 0.3075238764286041f, -0.5352725386619568f, -0.49505266547203064f, + 0.13530148565769196f, 0.6102568507194519f, 0.07696415483951569f, + 0.7194110155105591f, 0.06245575100183487f, -0.5269941091537476f, + 0.6319996118545532f, 0.19854962825775146f, 0.16299931704998016f, + -0.36398473381996155f, -0.29766127467155457f, -0.8556507229804993f, + 0.19275438785552979f, 0.10368930548429489f, -0.18639041483402252f, + -0.13361744582653046f, 0.20305071771144867f, -0.2384209781885147f, + -0.05955479294061661f, -0.1649128794670105f, 0.406151682138443f, + 0.6018882393836975f, 0.7667891383171082f, -1.7823314666748047f, + -0.6050052642822266f, -0.33709582686424255f, -0.07998203486204147f, + -0.48412758111953735f, -0.05572658032178879f, 0.1572490930557251f, + 0.2134942263364792f, 0.14582860469818115f, 0.18256354331970215f, + -0.377498060464859f, -0.9428679347038269f, 0.7819222211837769f, + 0.40199485421180725f, 0.37610843777656555f, 0.09845725446939468f, + -0.2473520040512085f, 1.216661810874939f, 0.30756333470344543f, + -0.6023632287979126f, -0.43360501527786255f, -0.13598665595054626f, + -0.2213943898677826f, -0.05226043984293938f, -0.4922758936882019f, + -0.0975913479924202f, -0.21766597032546997f, -0.3971554636955261f, + 0.7108594179153442f, -0.7833206057548523f, -0.1663222461938858f, + -0.10129307955503464f, 0.694123387336731f, -0.18871963024139404f, + -0.20957666635513306f, -0.6052191853523254f, -0.4168408513069153f, + 0.5734631419181824f, -0.16406677663326263f, -0.025442425161600113f, + 0.2279360443353653f, -0.39298707246780396f, 0.6339749097824097f, + 0.811789870262146f, 0.5733523964881897f, 0.7220039367675781f, + -0.2420847862958908f, -0.036595702171325684f, 0.16434960067272186f, + 0.1654539257287979f, -0.05197849124670029f, 1.6182829141616821f, + 0.10106345266103745f, -0.4488222897052765f, 1.9005721807479858f, + 1.1414402723312378f, -0.0009835668606683612f, -0.053397852927446365f, + 0.053350090980529785f, -0.01116836816072464f, -0.20531748235225677f, + 0.031078532338142395f, -0.12687280774116516f, 0.04337241128087044f, + -0.19423863291740417f, -0.1944684386253357f, -1.0170292854309082f, + 0.253818154335022f, 0.2104361206293106f, 0.303714781999588f, + 0.24978181719779968f, -0.13854654133319855f, 0.43723681569099426f, + -0.35904571413993835f, 0.3720517158508301f, 0.37751492857933044f, + -0.22321447730064392f, 0.2643660604953766f, -1.2104402780532837f, + -0.1291334182024002f, -0.799103856086731f, -0.6578419804573059f, + 0.1304577738046646f, -0.310467928647995f, -0.5715723633766174f, + -0.2751500606536865f, -1.4782752990722656f, 0.056794919073581696f, + -0.42365702986717224f, 0.29207560420036316f, -0.7994689345359802f, + 0.6121898293495178f, -0.13781405985355377f, -0.17137859761714935f, + -0.06067662686109543f, -0.2110602855682373f, -0.14195752143859863f, + -0.033988095819950104f, -0.290365606546402f, -0.11846823245286942f, + -0.5680235624313354f, -0.11251579970121384f, 0.11803086847066879f, + 0.17645932734012604f, 0.173707515001297f, -0.08128058165311813f, + 1.9146397113800049f, -2.025925636291504f, -0.2421492487192154f, + 0.1311802864074707f, 0.007723645307123661f, 0.03510530665516853f, + 0.01055131945759058f, -0.001576860318891704f, 0.12198811024427414f, + 0.051576755940914154f, 0.29626473784446716f, -0.21319039165973663f + }; + +static const float av1_hd_4_partition_nn_bias_32_layer0[32] = { + 0.2225671112537384f, -1.488280177116394f, -1.25272536277771f, + 0.30195170640945435f, -1.5750389099121094f, -0.15505032241344452f, + 0.4310544729232788f, -0.7423095703125f, 0.3700641989707947f, + -0.6942943930625916f, -1.8595184087753296f, 0.9343048930168152f, + -0.2134159654378891f, -1.0248521566390991f, -0.3916787803173065f, + -0.13198702037334442f, -0.004904405679553747f, -0.02703840285539627f, + 0.6067211031913757f, -0.783298909664154f, -0.43120503425598145f, + -0.5023682117462158f, -0.6118841171264648f, -1.6906251907348633f, + -0.22733944654464722f, -0.1254553198814392f, -1.4340238571166992f, + -0.22106066346168518f, -1.1736090183258057f, -0.19184525310993195f, + 1.6934525966644287f, 0.06706780195236206f +}; + +static const float + av1_hd_4_partition_nn_weights_32_layer1[32 * NEW_LABEL_SIZE] = { + 0.16060389578342438f, -0.21775491535663605f, 0.19317272305488586f, + 0.16914533078670502f, -0.45330461859703064f, 0.2787623107433319f, + 0.31299149990081787f, 0.05981941148638725f, 0.26470449566841125f, + 0.09401649236679077f, 0.009454675950109959f, 0.07412216812372208f, + 0.1894051879644394f, -0.005571697372943163f, -0.29581448435783386f, + 0.1277410089969635f, -0.2395242303609848f, -0.07200955599546432f, + -0.06016874685883522f, -0.052163951098918915f, -0.17527195811271667f, + -0.35842975974082947f, -0.2974209785461426f, -0.28416672348976135f, + -0.13073766231536865f, 0.06440825760364532f, 0.11089038848876953f, + 0.04264971986413002f, -0.047677211463451385f, -0.15897323191165924f, + 0.20627695322036743f, 0.14130337536334991f, -0.34490400552749634f, + -0.5835226774215698f, -0.2576460540294647f, 0.45405450463294983f, + -0.06914465129375458f, 0.47381001710891724f, 0.2275693118572235f, + -0.18174265325069427f, -0.32693126797676086f, -0.18394632637500763f, + 0.32419729232788086f, -0.312508225440979f, -0.03289115056395531f, + -0.0015484002651646733f, -0.411852091550827f, -0.025346053764224052f, + -0.5360790491104126f, 0.0021535682026296854f, -0.1873963326215744f, + -0.6050364375114441f, -0.32903650403022766f, 0.054302141070365906f, + -0.2966037094593048f, 0.09255900233983994f, -0.3126637041568756f, + 0.3107044994831085f, -0.41937246918678284f, 0.42857909202575684f, + 0.413314551115036f, 0.08698277920484543f, -0.2702505588531494f, + -0.3499393165111542f, 0.1393149048089981f, -0.07625073194503784f, + 0.22380144894123077f, 0.07417313754558563f, -0.030424892902374268f, + 0.010089819319546223f, -0.18347591161727905f, 0.2737013101577759f, + -0.002240510890260339f, 0.08045932650566101f, -0.3990366756916046f, + 0.28912147879600525f, -0.25353744626045227f, -0.398074209690094f, + -0.04493967071175575f, -0.26016902923583984f, -0.0123167484998703f, + -0.37009286880493164f, -0.5511913895606995f, 0.4690968990325928f, + 0.04034169763326645f, -0.04233121499419212f, -0.006188324186950922f, + 0.662132740020752f, -0.19957122206687927f, -0.2447078973054886f, + -0.37380027770996094f, -0.019658207893371582f, -0.8110345005989075f, + 0.04405290633440018f, -0.6720242500305176f, 0.28009140491485596f + }; + +static const float av1_hd_4_partition_nn_bias_32_layer1[NEW_LABEL_SIZE] = { + 0.2621873617172241f, -0.3843003511428833f, 0.061671461910009384f +}; + +static const float + av1_hd_4_partition_nn_weights_16_layer0[FEATURE_SIZE * 24] = { + 0.016253290697932243f, -1.297595500946045f, -0.27500706911087036f, + -0.015488669276237488f, 0.13751500844955444f, 0.35479483008384705f, + 0.14572829008102417f, 0.9681192636489868f, 0.1723741590976715f, + 0.6096186637878418f, -0.27040380239486694f, 0.10235186666250229f, + -1.328527569770813f, -0.21975912153720856f, -0.646583616733551f, + -0.055015575140714645f, -0.2533200681209564f, -0.3775864541530609f, + 0.14515726268291473f, 0.7094276547431946f, 0.1637670248746872f, + -0.44492098689079285f, 0.45630398392677307f, -0.09169628471136093f, + -1.2921737432479858f, -0.7724092602729797f, -0.3599521517753601f, + -0.5058088302612305f, 0.05034554377198219f, -0.13767042756080627f, + -0.063873291015625f, -0.22768817842006683f, 0.4867677688598633f, + 0.30664312839508057f, 0.197816863656044f, 0.7504525184631348f, + -1.1621628999710083f, 1.2124617099761963f, 0.25230467319488525f, + -0.05503295734524727f, -1.4942923784255981f, -0.32912740111351013f, + 0.3235040307044983f, -0.2510537803173065f, 0.771889328956604f, + -0.3342115581035614f, 0.5448428988456726f, 0.33309075236320496f, + 0.323270320892334f, 0.7348728775978088f, 0.08185584098100662f, + 0.2510865032672882f, 0.3047282099723816f, -0.1991347074508667f, + -0.12788036465644836f, 0.2775464951992035f, 0.8931537866592407f, + 0.056438326835632324f, -0.318469375371933f, 0.09217683225870132f, + 0.5232695937156677f, -1.103989601135254f, 1.3018217086791992f, + -0.009919252246618271f, -0.2791503667831421f, 0.061792079359292984f, + 0.07982557266950607f, -0.0922408252954483f, 0.6757814884185791f, + -0.7555142641067505f, -0.5889946222305298f, 0.6173131465911865f, + -0.8960505723953247f, -0.5027951002120972f, 0.13630244135856628f, + 0.6446100473403931f, -0.4130464196205139f, -0.4017779231071472f, + 0.20977409183979034f, -0.4052879810333252f, 1.3482177257537842f, + 0.14319704473018646f, 0.10465224087238312f, 0.7072924971580505f, + -0.10870729386806488f, -0.519813597202301f, -0.044540468603372574f, + 0.033170267939567566f, 0.09132528305053711f, -0.06453194469213486f, + -0.21186839044094086f, -0.05110237002372742f, -0.20925769209861755f, + -0.21288996934890747f, 1.6771467924118042f, 0.5502645373344421f, + -1.7048338651657104f, 0.16985715925693512f, -1.4219897985458374f, + -0.15837456285953522f, -0.2376108467578888f, -0.0772041529417038f, + -0.10152464359998703f, -0.12048980593681335f, -0.04174394533038139f, + 0.03342839702963829f, -0.11870015412569046f, 0.045642685145139694f, + -0.17824901640415192f, -0.39495736360549927f, 0.023219818249344826f, + 0.06599095463752747f, -1.3451234102249146f, -0.36058682203292847f, + -0.6323454976081848f, 0.6011573672294617f, 0.5714403986930847f, + -0.12171386182308197f, -0.2662772238254547f, 0.11267754435539246f, + 0.05486325919628143f, -0.005583520047366619f, -0.4404391944408417f, + 0.9215461611747742f, -0.7242451906204224f, 0.35334518551826477f, + 0.10570390522480011f, -0.25813040137290955f, -0.14261184632778168f, + 0.23588822782039642f, 0.8678246140480042f, 0.18859107792377472f, + -1.3121229410171509f, -0.3789387047290802f, -0.12254016101360321f, + 0.32620975375175476f, -0.020516367629170418f, -0.02394155040383339f, + -0.04741834104061127f, 0.6299850940704346f, 0.4153035879135132f, + 0.24157185852527618f, 0.1356470286846161f, 0.23200926184654236f, + 0.1951109915971756f, 0.18018396198749542f, 1.8084784746170044f, + 0.2517470717430115f, -0.73732590675354f, -0.3096546232700348f, + -1.2763310670852661f, -1.914092779159546f, 1.0422425270080566f, + -0.10959721356630325f, 0.004097330383956432f, 0.029040994122624397f, + 0.08122090995311737f, 0.12759143114089966f, -0.10597894340753555f, + -0.08978335559368134f, 0.06638552993535995f, -0.1336660236120224f, + -0.006535878870636225f, -0.41592198610305786f, 0.5584124326705933f, + 0.2367665022611618f, 0.196196049451828f, -0.07954944670200348f, + 0.5095464587211609f, -0.09126847237348557f, -0.21107706427574158f, + 0.029520299285650253f, -1.2078922986984253f, 0.3940129578113556f, + 0.7266209721565247f, -1.5306856632232666f, -0.1339416354894638f, + 0.1284932792186737f, -0.21777178347110748f, 0.07416583597660065f, + -0.09128864854574203f, 0.054931897670030594f, 2.331453323364258f, + 0.3878869414329529f, 3.0721747875213623f, -0.20134149491786957f, + 0.4023342430591583f, 1.0996533632278442f, 0.804786741733551f, + 0.05523429065942764f, 0.3615642488002777f, 0.28358790278434753f, + 1.0130475759506226f, 0.6530840992927551f, 0.5036265850067139f, + 0.07072196900844574f, 1.1965914964675903f, 0.15188759565353394f, + -0.4931938946247101f, -0.5595028400421143f, 0.4211946129798889f, + 0.4326711595058441f, 1.68171226978302f, -0.5289874076843262f, + 0.5924357771873474f, -1.0259782075881958f, 0.8006095886230469f, + -0.011948885396122932f, -0.06665080040693283f, 0.017298908904194832f, + 0.19788117706775665f, 0.030706975609064102f, 0.4442530870437622f, + -0.045532263815402985f, 0.9565532207489014f, -0.13111355900764465f, + -0.17530879378318787f, -1.7634141445159912f, 0.4697858691215515f, + 0.1104392409324646f, -0.2023022472858429f, -0.22376781702041626f, + 0.8732469081878662f, 0.8716534376144409f, 0.5823152661323547f, + -0.18083855509757996f, -1.308184266090393f, -0.654875636100769f, + 0.02435542456805706f, -0.0694618821144104f, -0.8239381909370422f, + -0.3262384235858917f, -0.4603644609451294f, -0.34901463985443115f, + -0.45511960983276367f, 0.09622194617986679f, -0.9455815553665161f, + -0.1763199418783188f, -0.9315241575241089f, -0.15244387090206146f, + 1.9309401512145996f, 1.658923625946045f, 0.6284119486808777f, + 1.4863715171813965f, -0.10375337302684784f, -0.10615994036197662f, + 0.06735323369503021f, -0.21195504069328308f, 0.09150348603725433f, + 0.03171848878264427f, 0.08611147850751877f, -0.186605304479599f, + 0.1885748654603958f, -0.03468483313918114f, 0.45047804713249207f, + 0.28302377462387085f, -0.06926856935024261f, -0.3081843852996826f, + 1.321082353591919f, 1.173419713973999f, -0.6945704221725464f, + -0.0016985656693577766f, 0.491913765668869f, 0.44896093010902405f, + 0.22076819837093353f, -0.8053346276283264f, 0.469636470079422f, + 0.3976723849773407f, 0.3846956193447113f, 0.3820555508136749f, + -0.5353395342826843f, -1.3723238706588745f, -0.6204023957252502f, + 0.013284329324960709f, 1.4644020795822144f, 1.1471366882324219f, + 0.5871153473854065f, 1.382630705833435f, -0.19867826998233795f, + -0.36002323031425476f, -0.5623172521591187f, -0.6787152886390686f, + -0.46966180205345154f, -0.06944172829389572f, -0.37814465165138245f, + -0.2501605749130249f, 0.23046647012233734f, -0.7194182872772217f, + 0.03376816213130951f, -0.5843089818954468f, -0.9881840944290161f, + 0.3359870910644531f, 0.4855765104293823f, -0.014989004470407963f, + -0.506977915763855f, -0.6273276805877686f, 0.6636883020401001f, + 0.3604613244533539f, -0.5910298824310303f, 0.7819033861160278f, + -0.279474139213562f, 0.42845603823661804f, 0.04737255349755287f, + 0.4609934985637665f, 0.295590877532959f, 0.1710357666015625f, + -0.5633265376091003f, -0.3976214528083801f, -0.3271988034248352f, + 1.0001078844070435f, -0.8906325101852417f, 1.7183932065963745f, + 0.657141923904419f, -0.45635029673576355f, 1.7355040311813354f, + -1.08721923828125f, -0.055488500744104385f, 0.060083694756031036f, + 0.04308626428246498f, 0.16541598737239838f, -0.004631219431757927f, + -0.05671098455786705f, -0.04844706505537033f, -0.04004818573594093f, + 0.5423532724380493f, -0.22633561491966248f, -0.27722683548927307f, + -0.14303728938102722f, 0.09665144979953766f, 0.10049419850111008f, + 0.0016488885739818215f, -0.9644765853881836f, -0.37397530674934387f, + -0.16206716001033783f, 0.46272340416908264f, 0.5197669267654419f, + 0.22682012617588043f, -0.14623111486434937f, 0.07362930476665497f, + -0.38951820135116577f, -0.31893840432167053f, -0.06790059059858322f, + -0.9926403760910034f, -0.02509547397494316f, -0.2454613894224167f, + 1.5675048828125f, -0.9967404007911682f, -0.4881826937198639f, + 2.075866937637329f, 0.9609254002571106f, 0.9962529540061951f, + 0.28117066621780396f, -0.049589112401008606f, -0.22676287591457367f, + 0.02461501583456993f, -0.015650441870093346f, 0.062187422066926956f, + 0.03559637814760208f, -0.022159814834594727f, 0.12390958517789841f, + -0.3556448221206665f, -0.509732186794281f, 0.23648309707641602f, + -0.02601865492761135f, -1.9125454425811768f, 0.2455364614725113f, + -0.2373759150505066f, 0.5479041337966919f, -0.09244018793106079f, + 0.4884181618690491f, -0.24114874005317688f, -0.14978568255901337f, + -0.17928700149059296f, -0.16638562083244324f, 0.20396184921264648f, + 0.35705462098121643f, 0.027246948331594467f, -0.03450827673077583f, + -1.0188283920288086f, -0.10113411396741867f, -0.9864289164543152f, + -0.10024290531873703f, -0.1372683346271515f, 1.2718243598937988f, + 1.9245606660842896f, 1.5185524225234985f, 0.8519737720489502f, + 1.2267814874649048f, -0.04248328134417534f, 0.03472750261425972f, + -0.010050246492028236f, 0.011352861300110817f, 0.0025208336301147938f, + -0.08251981437206268f, 0.013549050316214561f, 0.03250717744231224f, + -0.08791140466928482f, -0.6399267315864563f, 0.6154654026031494f, + 0.17678213119506836f, -0.25917088985443115f, 0.4593951106071472f, + 0.9585320353507996f, 0.6765605211257935f, 0.18305635452270508f, + -0.31955450773239136f, 0.18701691925525665f, -1.0847679376602173f, + -1.4102541208267212f, 0.5045204162597656f, 0.15169933438301086f, + 0.24514462053775787f, 0.3000636696815491f, 0.032767392694950104f, + 0.28788086771965027f, 0.17492298781871796f, 1.1169567108154297f, + 0.1887919008731842f, 1.3159228563308716f, 0.5523889064788818f, + -0.3944258689880371f, -1.4844435453414917f, 1.5090211629867554f, + 0.3398161232471466f, -0.006244915537536144f, 0.2332850992679596f, + -0.04527553915977478f, -0.004237487446516752f, -0.3812176585197449f, + 0.3342042565345764f, 0.09058798104524612f, 0.06223088502883911f + }; + +static const float av1_hd_4_partition_nn_bias_16_layer0[24] = { + 0.17428483068943024f, -0.18228434026241302f, -0.22544708847999573f, + 0.9856225252151489f, 0.7987083196640015f, -0.2589545249938965f, + -0.34056714177131653f, 0.518294632434845f, -0.8025332093238831f, + -0.04026109725236893f, -2.161693811416626f, 0.9118471145629883f, + 0.928778350353241f, 1.0625823736190796f, 0.11546365916728973f, + 0.006691100541502237f, -1.3805687427520752f, -0.11042501777410507f, + -0.2545872926712036f, 1.2440357208251953f, -1.6608234643936157f, + 0.9239633083343506f, -0.7040714621543884f, 1.1720298528671265f +}; + +static const float + av1_hd_4_partition_nn_weights_16_layer1[24 * NEW_LABEL_SIZE] = { + -0.0328470803797245f, 0.23792479932308197f, 0.25561416149139404f, + -0.020317574962973595f, 0.11895804107189178f, 0.11310578882694244f, + -0.11336013674736023f, 0.16002647578716278f, 0.18786849081516266f, + 0.10171400010585785f, 2.3350460529327393f, -0.2828865051269531f, + -0.1283670961856842f, 0.3708970248699188f, -0.21812289953231812f, + -0.0782235786318779f, -0.2884467840194702f, -0.36653709411621094f, + 0.28875768184661865f, 0.3709397315979004f, -0.1924125701189041f, + 0.8257001042366028f, -0.3084794580936432f, 0.10378926247358322f, + 0.10832912474870682f, 0.23994174599647522f, -0.09522973001003265f, + -0.20610085129737854f, -0.2905101180076599f, 0.18631568551063538f, + -0.3959210515022278f, 0.056451812386512756f, -1.2908092737197876f, + 0.3916683793067932f, -8.122172355651855f, -0.12142644822597504f, + -0.4428127706050873f, -0.24993056058883667f, 0.2432871013879776f, + -0.4911264181137085f, 0.23390810191631317f, -0.08834259957075119f, + -0.0922209843993187f, -0.791261613368988f, -0.2527330815792084f, + -0.8133859038352966f, 0.14507368206977844f, -0.45458677411079407f, + -0.5710628628730774f, -0.2644200026988983f, 0.2170594483613968f, + 0.40066513419151306f, -0.18364030122756958f, -0.8001738786697388f, + 0.14335019886493683f, -0.2577047646045685f, 0.11850440502166748f, + -0.14166106283664703f, -2.869997024536133f, 0.11104889959096909f, + -0.5155230760574341f, -0.8204565048217773f, -0.2842898666858673f, + -0.4841044247150421f, -0.23635634779930115f, 0.3735388517379761f, + 0.19305679202079773f, -0.5582130551338196f, 0.5497944951057434f, + -0.4491751790046692f, -0.2788706123828888f, 0.5158282518386841f + }; + +static const float av1_hd_4_partition_nn_bias_16_layer1[NEW_LABEL_SIZE] = { + 0.8354771137237549f, -0.7445926666259766f, -0.5160333514213562f +}; static const float av1_4_partition_nn_weights_16_layer0[FEATURE_SIZE * 24] = { -2.032866f, 0.056691f, 0.495960f, 0.778785f, 0.548153f, -0.806942f, @@ -1426,21 +2262,39 @@ 0.401786f, }; -static const NN_CONFIG av1_4_partition_nnconfig_16 = { - FEATURE_SIZE, // num_inputs - LABEL_SIZE, // num_outputs - 1, // num_hidden_layers +static const NN_CONFIG av1_4_partition_nnconfig_16[2] = { { - 24, // num_hidden_nodes + FEATURE_SIZE, // num_inputs + LABEL_SIZE, // num_outputs + 1, // num_hidden_layers + { + 24, // num_hidden_nodes + }, + { + av1_4_partition_nn_weights_16_layer0, + av1_4_partition_nn_weights_16_layer1, + }, + { + av1_4_partition_nn_bias_16_layer0, + av1_4_partition_nn_bias_16_layer1, + }, }, { - av1_4_partition_nn_weights_16_layer0, - av1_4_partition_nn_weights_16_layer1, - }, - { - av1_4_partition_nn_bias_16_layer0, - av1_4_partition_nn_bias_16_layer1, - }, + FEATURE_SIZE, // num_inputs + NEW_LABEL_SIZE, // num_outputs + 1, // num_hidden_layers + { + 24, // num_hidden_nodes + }, + { + av1_hd_4_partition_nn_weights_16_layer0, + av1_hd_4_partition_nn_weights_16_layer1, + }, + { + av1_hd_4_partition_nn_bias_16_layer0, + av1_hd_4_partition_nn_bias_16_layer1, + }, + } }; static const float av1_4_partition_nn_weights_32_layer0[FEATURE_SIZE * 32] = { @@ -1583,21 +2437,39 @@ -0.261961f, }; -static const NN_CONFIG av1_4_partition_nnconfig_32 = { - FEATURE_SIZE, // num_inputs - LABEL_SIZE, // num_outputs - 1, // num_hidden_layers +static const NN_CONFIG av1_4_partition_nnconfig_32[2] = { { - 32, // num_hidden_nodes + FEATURE_SIZE, // num_inputs + LABEL_SIZE, // num_outputs + 1, // num_hidden_layers + { + 32, // num_hidden_nodes + }, + { + av1_4_partition_nn_weights_32_layer0, + av1_4_partition_nn_weights_32_layer1, + }, + { + av1_4_partition_nn_bias_32_layer0, + av1_4_partition_nn_bias_32_layer1, + }, }, { - av1_4_partition_nn_weights_32_layer0, - av1_4_partition_nn_weights_32_layer1, - }, - { - av1_4_partition_nn_bias_32_layer0, - av1_4_partition_nn_bias_32_layer1, - }, + FEATURE_SIZE, // num_inputs + NEW_LABEL_SIZE, // num_outputs + 1, // num_hidden_layers + { + 32, // num_hidden_nodes + }, + { + av1_hd_4_partition_nn_weights_32_layer0, + av1_hd_4_partition_nn_weights_32_layer1, + }, + { + av1_hd_4_partition_nn_bias_32_layer0, + av1_hd_4_partition_nn_bias_32_layer1, + }, + } }; static const float av1_4_partition_nn_weights_64_layer0[FEATURE_SIZE * 24] = { @@ -1708,84 +2580,105 @@ 0.040013f, }; -static const NN_CONFIG av1_4_partition_nnconfig_64 = { - FEATURE_SIZE, // num_inputs - LABEL_SIZE, // num_outputs - 1, // num_hidden_layers +static const NN_CONFIG av1_4_partition_nnconfig_64[2] = { { - 24, // num_hidden_nodes + FEATURE_SIZE, // num_inputs + LABEL_SIZE, // num_outputs + 1, // num_hidden_layers + { + 24, // num_hidden_nodes + }, + { + av1_4_partition_nn_weights_64_layer0, + av1_4_partition_nn_weights_64_layer1, + }, + { + av1_4_partition_nn_bias_64_layer0, + av1_4_partition_nn_bias_64_layer1, + }, }, { - av1_4_partition_nn_weights_64_layer0, - av1_4_partition_nn_weights_64_layer1, - }, - { - av1_4_partition_nn_bias_64_layer0, - av1_4_partition_nn_bias_64_layer1, - }, + FEATURE_SIZE, // num_inputs + NEW_LABEL_SIZE, // num_outputs + 1, // num_hidden_layers + { + 24, // num_hidden_nodes + }, + { + av1_hd_4_partition_nn_weights_64_layer0, + av1_hd_4_partition_nn_weights_64_layer1, + }, + { + av1_hd_4_partition_nn_bias_64_layer0, + av1_hd_4_partition_nn_bias_64_layer1, + }, + } }; #undef FEATURE_SIZE #undef LABEL_SIZE +#undef NEW_LABEL_SIZE #define FEATURE_SIZE 4 // Mean and std static const float av1_hd_partition_breakout_nn_mean_128[FEATURE_SIZE] = { - 10.730792598027795f, - 376.6180794798389f, - 699.9787168227473f, - 2069.426780349039f, + 1.7569518370024242f, + 5.5324198861333107f, + 5.5260479508123526f, + 7.1913091289363038f, }; static const float av1_hd_partition_breakout_nn_std_128[FEATURE_SIZE] = { - 29.023640354060735f, 659.3967891826734f, 940.2882285903872f, - 2566.2070264789245f + 1.3286579037156365f, 1.4302357105525316f, 1.947275803679912f, + 1.4856880036033451f }; static const float av1_hd_partition_breakout_nn_mean_64[FEATURE_SIZE] = { - 22.220171631578605f, - 360.49969866754554f, - 642.6605866566223f, - 1022.2158389850164f, + 2.5083185600034166f, + 5.5459875545856132f, + 5.4419578738009537f, + 6.3575563591302293f, }; static const float av1_hd_partition_breakout_nn_std_64[FEATURE_SIZE] = { - 69.0770174031301f, 758.5544028348198f, 891.8245485291498f, 1901.3873859786233f + 1.5327874427371651f, 1.4413133143150851f, 1.852577618179309f, + 1.7441221691208793f }; static const float av1_hd_partition_breakout_nn_mean_32[FEATURE_SIZE] = { - 56.23900643356807f, - 411.708483646884f, - 552.7719521312714f, - 671.8117635469166f, + 3.2925380941946631f, + 5.6592503897046313f, + 5.2656875189877344f, + 5.88623228906336f, }; static const float av1_hd_partition_breakout_nn_std_32[FEATURE_SIZE] = { - 198.97960359590587f, 959.3829390828911f, 830.9980036115545f, - 1498.4982976375336f + 1.6894083970930263f, 1.4685304613224999f, 1.8598497961630693f, + 1.7648307635114531f }; static const float av1_hd_partition_breakout_nn_mean_16[FEATURE_SIZE] = { - 136.9843430416749f, - 532.9528184025893f, - 504.7980827006724f, - 445.7892729738405f, + 4.334899946924148f, + 5.8802354638427667f, + 5.2098555971571239f, + 5.5808807097219715f, }; static const float av1_hd_partition_breakout_nn_std_16[FEATURE_SIZE] = { - 478.52767648003527f, 1308.8412547684493f, 795.1318845991449f, - 1176.5665198192673f + 1.7269713123782717f, 1.5227226730026808f, 1.9255508380289716f, + 1.6959046024771494f }; static const float av1_hd_partition_breakout_nn_mean_8[FEATURE_SIZE] = { - 275.0553416698255f, - 566.7785552508784f, - 406.48781378761095f, - 247.1067473273997f, + 5.17688933331104f, + 6.0068390263641165f, + 4.9884361306237377f, + 5.0973575838899237f, }; static const float av1_hd_partition_breakout_nn_std_8[FEATURE_SIZE] = { - 935.5925922140688f, 1435.299322400821f, 678.9806893808023f, 791.3637162723987f + 1.693000978888298f, 1.4741850818874491f, 1.8398258859567682f, + 1.6434668808089992f }; static const float *const av1_hd_partition_breakout_nn_mean[5] = { @@ -1802,418 +2695,397 @@ static const float av1_hd_partition_breakout_nn_weights_128_layer0[FEATURE_SIZE * 32] = { - 0.4625445008277893f, 0.046699944883584976f, - 0.01686757430434227f, -0.20717839896678925f, - 3.7204904556274414f, -0.09939002245664597f, - -0.03257349878549576f, -0.02249263785779476f, - 4.27296257019043f, -0.2201482355594635f, - -0.02630791626870632f, -0.8290131092071533f, - 4.106664657592773f, -0.03872422128915787f, - -0.0008374569588340819f, 0.008610394783318043f, - 2.8612468242645264f, 0.029475420713424683f, - 0.01878979615867138f, -0.07678258419036865f, - 2.642153024673462f, -0.7023047208786011f, - -0.16276267170906067f, -1.7577937841415405f, - 3.9672751426696777f, -0.019311338663101196f, - -0.00475256098434329f, 7.173077028710395e-05f, - -0.06531551480293274f, -0.07583651691675186f, - -0.02518392540514469f, 0.005303085781633854f, - 4.6276326179504395f, -0.4294697642326355f, - -0.06581428647041321f, -0.4343421459197998f, - -3.5704429149627686f, -0.3216317892074585f, - 0.2882029414176941f, 0.6410080194473267f, - -3.4585084915161133f, 0.24044956266880035f, - 0.2667864263057709f, 0.35314950346946716f, - -0.9072761535644531f, 0.4799148142337799f, - -1.9291492700576782f, -0.14138458669185638f, - -4.956376075744629f, -0.4101237654685974f, - -0.00801194179803133f, 0.016341974958777428f, - 10.001052856445312f, -0.10566182434558868f, - -0.008026782423257828f, -0.04508136212825775f, - -0.11174708604812622f, -0.1031208261847496f, - 0.004450094420462847f, -0.0714421346783638f, - -3.7611849308013916f, -0.42005446553230286f, - -0.39350515604019165f, 0.47153064608573914f, - -0.36057817935943604f, -2.4768900871276855f, - -0.0051008667796850204f, 0.588866114616394f, - 3.6189825534820557f, -0.018487676978111267f, - 0.033942464739084244f, 0.3527982831001282f, - 2.350560426712036f, -1.840644359588623f, - 0.1856888085603714f, -1.4500601291656494f, - -2.9771881103515625f, 0.15055160224437714f, - -0.7389538884162903f, -0.02373156137764454f, - 3.244478464126587f, 0.09777998179197311f, - 0.03143433853983879f, -0.3365783095359802f, - -3.579131841659546f, -0.3401176333427429f, - 0.3683990240097046f, 0.6233412027359009f, - -5.737433433532715f, -1.0643895864486694f, - -0.0862712413072586f, -0.8468044400215149f, - 4.366209983825684f, -0.03896212950348854f, - -0.005277127958834171f, -0.054806072264909744f, - 4.20292854309082f, -0.33671531081199646f, - -0.04025593772530556f, -0.4954434335231781f, - 1.4843676090240479f, -0.05833450332283974f, - -1.2679489850997925f, 0.526577353477478f, - 4.186404705047607f, -0.019188573583960533f, - 0.03789565712213516f, 0.35230082273483276f, - 3.3375051021575928f, -0.01609140820801258f, - 0.027185490354895592f, 0.25255802273750305f, - 4.186381816864014f, -0.10051476955413818f, - -0.03126750886440277f, -0.02391117438673973f, - 3.763073205947876f, -0.04791776463389397f, - -2.3893446268630214e-05f, 0.010260583832859993f, - -2.701782464981079f, -0.3117542862892151f, - -0.08752211928367615f, 0.4682832360267639f, - 2.9983580112457275f, -0.47988685965538025f, - 0.2418350726366043f, 0.451855331659317f + -0.24421754479408264f, 0.5826895833015442f, -0.468494713306427f, + 0.05104721337556839f, -0.3030756115913391f, -0.47995227575302124f, + 0.30220189690589905f, 0.6468079686164856f, -0.6581971049308777f, + -0.08561690151691437f, -0.25164133310317993f, -0.7251309752464294f, + 0.28622835874557495f, -0.3805813491344452f, -1.2038640975952148f, + -0.03289572894573212f, -0.37501949071884155f, -0.6697373390197754f, + 0.10718680173158646f, 0.39980190992355347f, -0.6406639218330383f, + 0.3059859871864319f, -0.04687154293060303f, -0.0283386018127203f, + 0.2230420708656311f, -0.024931879714131355f, 0.029338961467146873f, + -0.39070868492126465f, 0.12055488675832748f, 0.44709470868110657f, + -0.2240329086780548f, 0.3899381160736084f, 0.7343757748603821f, + 0.3804762363433838f, -0.6964248418807983f, 0.031393926590681076f, + 1.1329405307769775f, 0.2145451456308365f, 0.15080654621124268f, + -0.5583563446998596f, 0.7352061867713928f, 0.03883780539035797f, + -0.003559485776349902f, -0.6891534328460693f, -0.2841552793979645f, + -0.033437542617321014f, -0.05212170630693436f, 0.46065035462379456f, + 0.49060872197151184f, -1.0210745334625244f, -0.20526815950870514f, + 0.3765658140182495f, -0.050877176225185394f, -0.4010245203971863f, + 0.2002274990081787f, -0.45646876096725464f, 0.028925752267241478f, + 0.44919347763061523f, -0.02976205199956894f, -0.14882135391235352f, + -0.23057465255260468f, -0.1957021802663803f, -0.02549680881202221f, + -0.8748094439506531f, -0.48586851358413696f, -0.4607398211956024f, + -0.19540899991989136f, 0.857029139995575f, -0.37086406350135803f, + -0.15118056535720825f, 0.18724219501018524f, 0.9177047610282898f, + -0.04173488914966583f, -0.3510979413986206f, 0.17673315107822418f, + -0.4069298505783081f, 0.020244762301445007f, -0.6126715540885925f, + 0.6862805485725403f, 0.1272497922182083f, -0.4310610890388489f, + -0.5232776403427124f, -0.26814699172973633f, 0.8925990462303162f, + -0.12301289290189743f, -0.5783265829086304f, 0.6845996379852295f, + 0.22650295495986938f, -0.11178003996610641f, -0.2689964771270752f, + -1.2071754932403564f, 0.8008237481117249f, -0.07050541788339615f, + -0.572569727897644f, 0.27917924523353577f, -0.6794210076332092f, + 0.37107107043266296f, 0.04119817912578583f, 0.08820059895515442f, + -0.29868319630622864f, -0.8464934229850769f, -0.6945635080337524f, + -0.6215496063232422f, 0.08221497386693954f, -0.6280669569969177f, + -0.44894012808799744f, 0.5003432631492615f, 0.4358958899974823f, + -0.03372211381793022f, 0.523205041885376f, -0.03965454921126366f, + -0.13546960055828094f, -0.3377343714237213f, -0.3924517333507538f, + -0.1991700977087021f, 0.6752629280090332f, -1.0135334730148315f, + 0.5485906600952148f, -0.020843572914600372f, -0.1461324244737625f, + -0.7540525794029236f, 0.21933406591415405f, -0.5565986633300781f, + 0.1918906718492508f, -0.40834516286849976f, 0.9896414279937744f, + 0.11563636362552643f, -0.017263229936361313f }; static const float av1_hd_partition_breakout_nn_bias_128_layer0[32] = { - -0.09511213004589081f, 0.9938786625862122f, 1.4839892387390137f, - 1.0876423120498657f, 0.8019428253173828f, 0.2738432288169861f, - 1.071292519569397f, -0.09138987958431244f, 1.2080857753753662f, - -0.6339017748832703f, -0.4984428584575653f, -1.2238473892211914f, - -1.3475229740142822f, 2.4443564414978027f, -0.16558663547039032f, - 0.033367302268743515f, -0.7447078227996826f, 1.0401278734207153f, - -0.4657363295555115f, -1.0536733865737915f, 0.8565444946289062f, - -0.5986987948417664f, -0.789206862449646f, 1.1764090061187744f, - 1.3952962160110474f, -0.17746582627296448f, 1.187064290046692f, - 0.9370515942573547f, 1.1067490577697754f, 1.003738522529602f, - -2.1517810821533203f, 0.8461123704910278f + 0.2676561176776886f, -0.29510897397994995f, -1.1071470975875854f, + 0.583503007888794f, -0.472633957862854f, -0.44106626510620117f, + 0.6721175909042358f, -0.42975232005119324f, -0.5145384073257446f, + 0.15512774884700775f, 1.0278737545013428f, -0.6500952839851379f, + 1.0237534046173096f, 0.5816664099693298f, 0.6633380055427551f, + -1.3322787284851074f, -0.4271796643733978f, 0.06402774155139923f, + 0.5194029808044434f, 0.03623528033494949f, -0.34109270572662354f, + -0.15831837058067322f, -0.07824128866195679f, 0.8782597184181213f, + -0.014673121273517609f, -0.20494380593299866f, -0.3248092532157898f, + 0.6707151532173157f, -0.2768913507461548f, -0.31312206387519836f, + -0.6461104154586792f, 0.16326004266738892f }; static const float av1_hd_partition_breakout_nn_weights_128_layer1[32] = { - -0.00859279278665781f, -2.9574432373046875f, -4.146053314208984f, - -3.348489761352539f, -0.05143212527036667f, -2.0400538444519043f, - -4.622243404388428f, -0.05732434615492821f, -5.16105318069458f, - 1.1596481800079346f, 0.8911784291267395f, 8.114648818969727f, - 19.255064010620117f, 11.491721153259277f, -0.2828827202320099f, - 1.5688998699188232f, 0.7430379986763f, -1.0839532613754272f, - -1.3526417016983032f, 5.244595527648926f, -4.02890682220459f, - 0.8462737798690796f, 5.2530364990234375f, -4.688608646392822f, - -4.285271644592285f, 0.3642231523990631f, -2.4636125564575195f, - -0.8168565034866333f, -6.247287273406982f, -1.9640626907348633f, - -2.2665669918060303f, -0.7841953039169312f + -0.31587469577789307f, 0.2444663941860199f, 1.1686184406280518f, + -1.136441707611084f, 0.462795227766037f, 0.16067592799663544f, + -0.08983717858791351f, -0.1526145339012146f, 1.2807552814483643f, + -1.8411308526992798f, -2.5061304569244385f, -0.33663204312324524f, + -1.3827414512634277f, -0.2779307961463928f, -0.47907790541648865f, + 1.9458348751068115f, 0.3252658247947693f, 0.6043151617050171f, + -0.2127833366394043f, 0.3176756501197815f, 0.9920678734779358f, + 0.49299460649490356f, 0.8988507390022278f, -0.7701797485351562f, + 0.29247382283210754f, 0.7251673340797424f, 0.460658997297287f, + -0.3683358430862427f, 0.6006630659103394f, 0.7680628299713135f, + 0.8999216556549072f, -0.529272735118866f }; static const float av1_hd_partition_breakout_nn_bias_128_layer1[1] = { - 0.1112925186753273f + -0.22755585610866547f }; static const float av1_hd_partition_breakout_nn_weights_64_layer0[FEATURE_SIZE * 32] = { - -2.9600589275360107f, 0.042996086180210114f, -0.5422487258911133f, - -0.16088658571243286f, 2.665553331375122f, 0.17293912172317505f, - -0.011884346604347229f, -1.3571131229400635f, -4.151463508605957f, - -0.09992470592260361f, -0.03328946977853775f, -0.19198642671108246f, - -4.485296726226807f, 0.0004126991261728108f, 0.00528523325920105f, - 0.0014443921390920877f, 3.302469253540039f, -0.054684557020664215f, - 0.0016338867135345936f, -2.7690939903259277f, 2.5339231491088867f, - 0.5221245884895325f, -0.17638295888900757f, -0.31949788331985474f, - 2.323598623275757f, 0.2059188336133957f, -0.01752222701907158f, - -1.1489571332931519f, 2.984546422958374f, -0.19020183384418488f, - 0.010538210161030293f, -1.4752967357635498f, 1.8535363674163818f, - 0.8166176080703735f, -0.5628044009208679f, -1.9997116327285767f, - -2.098487377166748f, 0.3857003152370453f, -1.727634310722351f, - -0.15402595698833466f, 3.4408957958221436f, -0.2683435082435608f, - 0.023594189435243607f, 0.31177008152008057f, 3.039790391921997f, - 0.4344753623008728f, 0.00961180217564106f, -0.45155903697013855f, - -2.201700210571289f, -2.646042585372925f, 0.01605065166950226f, - 0.13390818238258362f, -3.6872377395629883f, 0.7149689793586731f, - -0.04385269060730934f, -0.10101474821567535f, 2.27207088470459f, - 0.34321531653404236f, 0.04720620810985565f, -0.7443615198135376f, - -5.059787750244141f, 0.02228701300919056f, -0.0019030789844691753f, - 0.0005637781578116119f, 3.129986524581909f, 0.09729813039302826f, - -0.024177810177206993f, -1.8108325004577637f, 3.3083548545837402f, - -0.7547611594200134f, -0.44618573784828186f, -0.21931181848049164f, - -2.2004897594451904f, -2.6458916664123535f, 0.09130378812551498f, - 0.2665710747241974f, -0.5643004775047302f, -0.33926868438720703f, - -0.6195669174194336f, 0.7023015022277832f, -5.160871982574463f, - -0.0034803124144673347f, -0.011646432802081108f, 0.012036978267133236f, - -3.1459012031555176f, -1.4449714422225952f, -0.0737914890050888f, - 0.6170333623886108f, 2.4652867317199707f, -0.05448657274246216f, - -0.16370250284671783f, -0.6195785999298096f, -4.647854328155518f, - -0.00691827479749918f, 0.038473278284072876f, 0.032320134341716766f, - -4.32373571395874f, 0.004058001562952995f, -0.004842042922973633f, - -0.011608919128775597f, 2.67899489402771f, -0.21639853715896606f, - -0.004101857542991638f, -2.9161124229431152f, 4.8421549797058105f, - 0.04978768154978752f, -0.04711421951651573f, -0.06902165710926056f, - -0.16757740080356598f, 0.021471548825502396f, -0.09624961763620377f, - -0.05160382017493248f, -4.769743919372559f, 0.1121208518743515f, - -0.024247704073786736f, -0.20753318071365356f, 0.47400805354118347f, - 0.46464693546295166f, -2.335646390914917f, -0.00663331151008606f, - -1.8362538814544678f, -1.3952279090881348f, -0.0224999338388443f, - 0.9369595050811768f, -0.32786062359809875f, -1.2076143026351929f, - -0.4175379276275635f, 0.920979917049408f + 0.4476310610771179f, -0.8712111711502075f, 0.1288183629512787f, + 0.792915940284729f, -0.37857282161712646f, -0.5999379754066467f, + -0.09074056148529053f, 0.7428027987480164f, -0.806922197341919f, + -0.06513206660747528f, 0.0015326066641137004f, 0.03463340178132057f, + -0.45006656646728516f, 0.8719941973686218f, -0.3037683367729187f, + -0.4873082637786865f, 0.3746185898780823f, -0.35810524225234985f, + 0.16044893860816956f, -0.5600897073745728f, 0.7878753542900085f, + -0.057914119213819504f, -0.05587376281619072f, -0.32516810297966003f, + -0.054352328181266785f, 0.7050285339355469f, 0.4860455393791199f, + -0.26461395621299744f, 0.4708733856678009f, 0.6615039110183716f, + -0.6717810034751892f, 0.25766628980636597f, -0.20877176523208618f, + 0.4233146607875824f, -0.8261363506317139f, 0.6218927502632141f, + -0.7742594480514526f, -0.09597370028495789f, -0.016565624624490738f, + 0.3243916630744934f, 0.730843722820282f, 0.3434459865093231f, + -0.6115328073501587f, 0.05172838270664215f, -0.8085055947303772f, + -0.23082391917705536f, 0.25912967324256897f, 0.48390132188796997f, + -0.24071218073368073f, -0.21271266043186188f, 0.15473750233650208f, + 1.1897283792495728f, -0.605708122253418f, 0.8830835223197937f, + -0.8744847774505615f, -0.10007809847593307f, 0.14527341723442078f, + -0.3217686712741852f, -0.6968852281570435f, -0.12154795229434967f, + -0.8508146405220032f, -0.23319056630134583f, 0.27581697702407837f, + 0.49674510955810547f, 0.2984752655029297f, -0.24128034710884094f, + -0.6872956156730652f, 0.6757507920265198f, -0.9962611198425293f, + -0.24671325087547302f, 0.04458463191986084f, 0.5092823505401611f, + -0.755521833896637f, 0.1556125432252884f, -0.19932909309864044f, + 0.8233707547187805f, -0.33803892135620117f, 0.1641087830066681f, + -0.1640886515378952f, 0.2807005047798157f, 0.11740092933177948f, + 0.9441748261451721f, -0.19026336073875427f, 0.1905575692653656f, + -0.7156860828399658f, -0.30425164103507996f, -0.6542183756828308f, + -0.048906467854976654f, 0.6393331289291382f, -0.5186219811439514f, + -0.15262047946453094f, 0.729802668094635f, 0.4545252323150635f, + -0.5133960843086243f, 0.26521825790405273f, -0.4991665482521057f, + -0.4927463233470917f, -0.07089114189147949f, 0.33401384949684143f, + 0.3023838400840759f, 0.7638855576515198f, -0.4981137812137604f, + -0.5541877150535583f, -0.11165370792150497f, 0.3648005425930023f, + 0.5878415107727051f, 0.11315485090017319f, -1.0285885334014893f, + -0.45433279871940613f, -0.7583989500999451f, -0.14650657773017883f, + 0.24613532423973083f, -0.2773364782333374f, -0.9591728448867798f, + 0.5280625224113464f, 0.5242233276367188f, -0.39398059248924255f, + -0.8652032613754272f, 0.5243239998817444f, 0.5806688666343689f, + 0.42181527614593506f, -0.09341983497142792f, 0.12693725526332855f, + -0.39904987812042236f, 0.15719516575336456f, 0.16289645433425903f, + 0.20234838128089905f, -0.25414201617240906f }; static const float av1_hd_partition_breakout_nn_bias_64_layer0[32] = { - -0.9125985503196716f, 0.9354565739631653f, -0.9121711850166321f, - -1.1538583040237427f, 0.08968774974346161f, 1.1123112440109253f, - 1.4973655939102173f, 0.608467698097229f, 0.18589675426483154f, - -1.441516399383545f, 0.9556304812431335f, 0.7926673889160156f, - -1.4284138679504395f, -0.7179064750671387f, -0.047664791345596313f, - -1.2288035154342651f, 0.6012410521507263f, 0.18902179598808289f, - -0.89959716796875f, 0.23802867531776428f, -1.300124168395996f, - -0.08421888202428818f, 0.28658464550971985f, -0.27196547389030457f, - -1.0962705612182617f, -0.350778728723526f, 0.8084302544593811f, - -0.11797107756137848f, -0.9147244691848755f, -0.9423892498016357f, - -0.46693262457847595f, -1.179872989654541f + -0.34709370136260986f, -0.009843609295785427f, -1.6221565008163452f, + -0.668915867805481f, 0.9886232018470764f, 0.6522868275642395f, + -0.37716832756996155f, -0.07447462528944016f, 0.3766818642616272f, + -1.1512787342071533f, 0.40794867277145386f, -0.14982998371124268f, + -0.19331441819667816f, -0.9201817512512207f, 0.48975181579589844f, + -0.137226402759552f, -0.5480761528015137f, -0.14310558140277863f, + -0.035678427666425705f, -0.724799633026123f, 0.6203687787055969f, + -0.26913538575172424f, -0.45061108469963074f, 1.1059123277664185f, + -0.9563385844230652f, 0.40938442945480347f, 0.4411890506744385f, + 0.1871682107448578f, 0.004259683191776276f, -0.050544340163469315f, + 0.789713442325592f, 0.26867425441741943f }; static const float av1_hd_partition_breakout_nn_weights_64_layer1[32] = { - 1.888397216796875f, -1.6998655796051025f, 2.5516481399536133f, - 31.333364486694336f, -7.420342922210693f, -1.175126552581787f, - -2.0703518390655518f, -1.0818297863006592f, -1.0498319864273071f, - 12.22414779663086f, -0.6940112113952637f, -0.3731234669685364f, - 11.562948226928711f, 0.6934157013893127f, 1.0499767065048218f, - 15.071033477783203f, -2.1736562252044678f, 0.47170618176460266f, - 1.8899933099746704f, 0.2540336847305298f, 27.493698120117188f, - 0.8192774057388306f, -0.011624461971223354f, 1.5983129739761353f, - 34.282840728759766f, -15.804471969604492f, 1.1542948484420776f, - -0.09561372548341751f, 4.60494327545166f, 1.2926712036132812f, - 0.4446795880794525f, -0.7026389241218567f + 0.7450170516967773f, 0.1674197018146515f, 8.603666305541992f, + -1.4572426080703735f, -0.6806990504264832f, -0.36511117219924927f, + -0.21973447501659393f, -0.33904269337654114f, 0.494196355342865f, + 2.428866386413574f, -0.33053985238075256f, 0.5259426236152649f, + 0.7364671230316162f, 1.8298554420471191f, -0.20149026811122894f, + 0.4657254219055176f, 0.9128966331481934f, 0.41023728251457214f, + 0.4559532403945923f, -0.49007904529571533f, -0.6186080574989319f, + 0.6019283533096313f, 0.41913628578186035f, -0.9652310609817505f, + -1.266809105873108f, -0.6605119109153748f, -1.0554698705673218f, + 0.28501376509666443f, 0.40640202164649963f, 0.4747793972492218f, + -0.3028396964073181f, -0.020081687718629837f }; static const float av1_hd_partition_breakout_nn_bias_64_layer1[1] = { - -0.15468640625476837f + -0.35164690017700195f }; static const float av1_hd_partition_breakout_nn_weights_32_layer0[FEATURE_SIZE * 32] = { - -0.18351112306118011f, -0.15651831030845642f, 0.0013713851803913713f, - 0.01054045557975769f, -3.780832529067993f, -0.15531419217586517f, - 0.03428387641906738f, -0.027534646913409233f, -4.079058647155762f, - 0.05176529288291931f, -0.0065496210008859634f, -0.08574667572975159f, - -1.9617663621902466f, 0.3167760670185089f, -0.33277764916419983f, - 0.7234531044960022f, 2.2539572715759277f, -0.02695995196700096f, - 0.005501582287251949f, -3.854844093322754f, -2.5093464851379395f, - -1.8817286491394043f, -0.13509777188301086f, 0.8389045000076294f, - -0.8123356103897095f, -2.210233449935913f, 0.04169511795043945f, - 1.3030999898910522f, 1.0332109928131104f, 0.15076060593128204f, - -0.02417255938053131f, -3.10256290435791f, -3.5063133239746094f, - 0.1668155938386917f, -0.11349064111709595f, -0.08673816174268723f, - 0.2023429572582245f, 0.2143218219280243f, -2.2571842670440674f, - 0.1890597939491272f, -2.0899853706359863f, -0.5377482771873474f, - -0.24231456220149994f, 0.7975266575813293f, -1.104709267616272f, - 0.42163029313087463f, -1.588227391242981f, -0.5404914021492004f, - 0.39026254415512085f, 0.3967314660549164f, -2.0958690643310547f, - -0.1487407088279724f, -1.2271692752838135f, -2.4342153072357178f, - 0.0073483348824083805f, 0.4332350790500641f, 1.8399561643600464f, - 0.20915934443473816f, 0.014152542687952518f, -2.279658079147339f, - 2.1412830352783203f, -1.2423213720321655f, 0.10377844423055649f, - 0.0026721248868852854f, -1.515641212463379f, 0.011470284312963486f, - 0.433901309967041f, 0.03397373855113983f, 1.9424909353256226f, - 0.007673368323594332f, -0.019963057711720467f, -3.2635231018066406f, - -3.814363956451416f, -0.008298925124108791f, -0.0006046147318556905f, - -0.03078455477952957f, -2.5249812602996826f, -1.5319974422454834f, - 0.03523765504360199f, 0.47840240597724915f, -2.459254503250122f, - -0.060729049146175385f, 0.3428601920604706f, 1.0676825046539307f, - -3.196352481842041f, 0.00556157436221838f, -0.014416366815567017f, - 0.00013636458606924862f, 0.5342910289764404f, 0.08967946469783783f, - -1.079426884651184f, 0.0459836907684803f, 1.169906735420227f, - 0.23663409054279327f, 0.15936784446239471f, 0.12904685735702515f, - -1.941832423210144f, -2.3517515659332275f, -0.4566086530685425f, - 1.001920223236084f, -2.479144334793091f, -1.4872301816940308f, - 0.44298961758613586f, 0.0910615548491478f, 0.34078770875930786f, - -1.0173975229263306f, 0.2069491446018219f, 0.8345462679862976f, - 1.5491070747375488f, 0.20149222016334534f, 0.2451339066028595f, - -1.176571011543274f, -2.3429925441741943f, 0.2398013025522232f, - -0.9587415456771851f, -0.05250968039035797f, -3.1348395347595215f, - -0.55987948179245f, -0.040084246546030045f, 0.29037174582481384f, - 1.663692593574524f, 0.6011162400245667f, 0.014114673249423504f, - -0.9607003331184387f, -1.0632480382919312f, -0.9293855428695679f, - -0.7134267091751099f, 0.9321397542953491f + -0.8952129483222961f, -0.23612700402736664f, -0.022868705913424492f, + 0.504967451095581f, 0.3275434970855713f, 1.1059952974319458f, + -0.34120088815689087f, -0.07344819605350494f, 0.15775921940803528f, + -0.04879460483789444f, 0.1439337134361267f, 1.2064639329910278f, + 0.006475068163126707f, -0.4494408369064331f, -0.10751494765281677f, + 1.1100929975509644f, 0.12578976154327393f, 0.4349801242351532f, + 0.5385650396347046f, -0.4146132171154022f, -0.6666572690010071f, + 0.1782642900943756f, -0.43637481331825256f, -0.522347629070282f, + -0.7048850655555725f, -0.21371017396450043f, 0.1338515281677246f, + 0.9378389716148376f, -0.9088003635406494f, 0.3172213137149811f, + -0.6936320066452026f, 0.6413770318031311f, -0.6204543113708496f, + -0.20075663924217224f, 0.011177961714565754f, 0.3956983685493469f, + -0.23885823786258698f, 0.7718412280082703f, 0.013668450526893139f, + -0.11212749779224396f, -0.3808520436286926f, -0.019013067707419395f, + -0.5678153038024902f, 1.038125991821289f, -0.0636964738368988f, + -0.9305693507194519f, 0.5443329811096191f, 0.5650880932807922f, + -0.2772569954395294f, -0.7757272720336914f, 0.02383701130747795f, + 0.5687315464019775f, -0.4549035429954529f, -0.16180022060871124f, + 0.033420149236917496f, 0.5288522839546204f, -0.3871588110923767f, + 0.712287962436676f, 0.008839298039674759f, -0.4081243574619293f, + -0.44496551156044006f, -0.49737873673439026f, 0.5079053044319153f, + 0.4881437122821808f, 0.6699126362800598f, 0.28043776750564575f, + -0.5355777740478516f, 0.2881544232368469f, 0.33460190892219543f, + -0.26208120584487915f, -0.44822946190834045f, -0.733630895614624f, + -0.35417506098747253f, 0.6161319613456726f, -0.0993569940328598f, + -0.7897841334342957f, -0.551094651222229f, 0.4497978985309601f, + -0.31630903482437134f, 0.46718651056289673f, -0.19858941435813904f, + 1.0053157806396484f, -0.3194795548915863f, 0.31316739320755005f, + -0.035107459872961044f, -0.7919036746025085f, 0.04070507362484932f, + -0.325952410697937f, -0.8687034845352173f, -0.30160313844680786f, + 0.16643138229846954f, 1.1927493810653687f, -0.5958532094955444f, + -0.38102859258651733f, -0.06056114658713341f, 1.063319206237793f, + -0.028595328330993652f, 0.7656055688858032f, 0.5335075855255127f, + -0.6592297554016113f, -0.3134363293647766f, -0.8477844595909119f, + 0.14632871747016907f, 0.5351374745368958f, -0.007659543305635452f, + -0.8012065291404724f, 0.47477757930755615f, 0.4271889626979828f, + -0.6191681623458862f, -0.14256015419960022f, 0.038999997079372406f, + 0.4897593557834625f, -0.713534414768219f, 0.07782091945409775f, + -0.28364527225494385f, -0.17344607412815094f, -0.24871549010276794f, + 0.6229367852210999f, 0.35328975319862366f, -0.48995962738990784f, + 0.9903460741043091f, -0.1367904245853424f, -0.14807336032390594f, + -0.28647252917289734f, 0.6976669430732727f, -0.5536572933197021f, + 0.3724263310432434f, -0.7798140645027161f }; static const float av1_hd_partition_breakout_nn_bias_32_layer0[32] = { - -0.11753689497709274f, -0.7140882015228271f, -0.9722395539283752f, - -0.6984077095985413f, -0.726676881313324f, -0.36886507272720337f, - -0.17676463723182678f, -0.10097962617874146f, -0.29938116669654846f, - -1.727449893951416f, 0.5804333090782166f, -1.2764358520507812f, - -1.108852505683899f, -0.8272318840026855f, 0.6785213947296143f, - 0.11738266795873642f, -2.057903528213501f, -0.37823784351348877f, - -0.9595664143562317f, -1.0248923301696777f, -0.23907040059566498f, - -0.8479264974594116f, -0.13894951343536377f, -1.7720012664794922f, - -1.6527719497680664f, -0.20360442996025085f, -1.7053756713867188f, - 0.8720151782035828f, -1.0570132732391357f, -0.7993026971817017f, - 0.46637022495269775f, -0.8374565243721008f + 0.5906000733375549f, 0.8543934226036072f, 0.43217557668685913f, + -0.643294632434845f, 0.30037060379981995f, -0.7660125494003296f, + 0.1731051206588745f, -0.7480033040046692f, -0.6353603005409241f, + 0.4871520698070526f, 0.7321553230285645f, 0.2810375988483429f, + -0.43400588631629944f, -0.17055644094944f, -0.8143026232719421f, + 0.13951803743839264f, -1.2939090728759766f, -0.5045616626739502f, + -1.295732021331787f, -0.9267586469650269f, -0.13748344779014587f, + -0.8858642578125f, 0.24190591275691986f, 0.34041425585746765f, + 0.05495613068342209f, 0.25397151708602905f, 0.24138814210891724f, + 0.12242670357227325f, -0.5556198358535767f, -0.46377164125442505f, + 0.6218398213386536f, 0.7764495015144348f }; static const float av1_hd_partition_breakout_nn_weights_32_layer1[32] = { - -0.329619824886322f, 4.491082191467285f, 11.859853744506836f, - -0.2909659743309021f, -20.884370803833008f, 1.2395148277282715f, - 0.7091589570045471f, -2.458725690841675f, 1.714850664138794f, - 2.3455934524536133f, 0.7239453792572021f, 11.537474632263184f, - 3.062530755996704f, 2.7946059703826904f, -1.4265711307525635f, - 0.44554367661476135f, -1.534838080406189f, -4.844552993774414f, - 18.15884780883789f, 5.826704978942871f, 0.5580787062644958f, - 11.919114112854004f, 0.24757426977157593f, 0.30521970987319946f, - -1.6404640674591064f, 0.848618745803833f, -0.8315955996513367f, - -0.8414434194564819f, 6.57717227935791f, 2.829108476638794f, - -0.25260433554649353f, -0.4767799377441406f + 0.6254786252975464f, -0.724627673625946f, 0.5428725481033325f, + 0.8609197735786438f, -0.33933258056640625f, 0.6795636415481567f, + 0.5508725643157959f, 1.4960711002349854f, 0.8083735108375549f, + -0.3456595838069916f, 0.5514245629310608f, 0.28833192586898804f, + 0.7953433394432068f, -0.009960951283574104f, -1.7862937450408936f, + 0.1605624407529831f, 1.6885508298873901f, -0.49641919136047363f, + -4.162553310394287f, -2.510878086090088f, 0.3701621890068054f, + -0.5248422622680664f, 0.5657362937927246f, 0.48716917634010315f, + -0.23934587836265564f, 0.45801791548728943f, 0.3940700590610504f, + 0.07297442853450775f, 0.287155419588089f, -0.0904238149523735f, + -0.608720064163208f, -0.7513124942779541f }; static const float av1_hd_partition_breakout_nn_bias_32_layer1[1] = { - 0.04493425786495209f + 0.1117519736289978f }; static const float av1_hd_partition_breakout_nn_weights_16_layer0[FEATURE_SIZE * 32] = { - -0.03770807012915611f, 0.1042923852801323f, 0.3589613735675812f, - 0.8915500044822693f, -0.025765806436538696f, -0.8080071806907654f, - 0.11935761570930481f, 1.081723690032959f, -1.0217446088790894f, - -0.05894668400287628f, -0.0417763888835907f, 1.9492236375808716f, - -1.6240522861480713f, -1.0665417909622192f, -0.4230113625526428f, - 2.6151089668273926f, -1.0250322818756104f, -1.013023853302002f, - -0.3417854607105255f, 1.2021280527114868f, -2.932366132736206f, - -1.5600024461746216f, 0.013506319373846054f, 1.827286958694458f, - 0.5162537097930908f, 0.37939924001693726f, -0.12543700635433197f, - -3.9469857215881348f, -0.5308187007904053f, 0.055751170963048935f, - -2.1356542110443115f, 0.9524611830711365f, -1.2815498113632202f, - -0.2583577334880829f, 0.5526014566421509f, -0.991321861743927f, - 0.4657832682132721f, -0.32342609763145447f, 0.012717160396277905f, - -4.607941150665283f, -0.7786141633987427f, -0.7440438270568848f, - -0.10956363379955292f, 1.9957225322723389f, -3.557837724685669f, - -0.06580297648906708f, -0.019815972074866295f, 0.48607689142227173f, - 0.9143388271331787f, 0.7489029765129089f, 0.23694343864917755f, - -2.3731346130371094f, -2.455751895904541f, 0.02998301573097706f, - -1.352641224861145f, 1.4107884168624878f, -2.7167775630950928f, - -1.3195676803588867f, -0.0024603719357401133f, 1.568859338760376f, - -1.368638277053833f, -0.6318024396896362f, -0.8460041284561157f, - 0.7431659698486328f, -0.17625141143798828f, -0.25151070952415466f, - 0.19983969628810883f, 2.0860755443573f, -3.904813289642334f, - -0.38988929986953735f, -0.002895161509513855f, 0.714531660079956f, - -3.0156004428863525f, -0.6497556567192078f, 0.32678696513175964f, - 0.5911594033241272f, -1.1386737823486328f, -0.6372389793395996f, - -0.029834266752004623f, 2.0517866611480713f, -1.0608230829238892f, - -0.955141544342041f, -0.10072463750839233f, 1.5374855995178223f, - -1.2410826683044434f, -2.169691562652588f, 0.003649030579254031f, - 2.0521841049194336f, -0.9012426733970642f, 0.6359571218490601f, - 0.10054442286491394f, 0.9355012774467468f, -2.433013439178467f, - -1.3059970140457153f, -0.062273263931274414f, 1.2142263650894165f, - 0.07180827856063843f, -0.02799983322620392f, 0.012213055975735188f, - -0.15036505460739136f, -1.5302343368530273f, -0.9248730540275574f, - -1.2439353466033936f, 1.6203926801681519f, -0.26789119839668274f, - 0.29344648122787476f, -0.12080803513526917f, 2.095102548599243f, - -3.6576807498931885f, 0.32156985998153687f, -0.00601181061938405f, - 0.3735950291156769f, -1.2174468040466309f, -0.21403838694095612f, - -2.081902503967285f, 1.0142937898635864f, -1.2942562103271484f, - -2.4739151000976562f, 0.11531462520360947f, 1.7982516288757324f, - -2.203162670135498f, 0.1353112757205963f, -1.742370843887329f, - 0.44314906001091003f, -1.1483821868896484f, -0.6968573331832886f, - -0.059433888643980026f, 0.9399102330207825f + 0.3427221179008484f, 0.06073892489075661f, -0.06882427632808685f, + 1.1336252689361572f, 0.097809799015522f, -0.6385823488235474f, + 0.43692731857299805f, -0.6029260754585266f, -0.13368098437786102f, + -0.6389026045799255f, 0.2193300873041153f, 0.6348177790641785f, + 0.22468054294586182f, 0.13958360254764557f, -0.05150606855750084f, + -0.28101637959480286f, -0.4977607727050781f, -0.3215360641479492f, + -0.16947278380393982f, 1.0890949964523315f, -1.02256441116333f, + 0.34789103269577026f, 0.34546756744384766f, -0.08185210078954697f, + 0.24066677689552307f, -1.2535117864608765f, 0.04398220032453537f, + 0.8025804162025452f, -0.2541569769382477f, 0.3501926362514496f, + -0.3649175763130188f, -0.2356642484664917f, 0.8068639039993286f, + -0.23699556291103363f, -0.2195892035961151f, -0.1547853797674179f, + -0.15287409722805023f, 0.4898677468299866f, -0.3860497772693634f, + 0.32813069224357605f, -0.7604209184646606f, -0.6611303091049194f, + 0.006376428063958883f, 1.0097582340240479f, 0.18538667261600494f, + 0.5587815642356873f, 0.30065974593162537f, -0.8898136615753174f, + 0.63029944896698f, 0.839385986328125f, -0.2562950849533081f, + -0.5478677749633789f, -0.7537129521369934f, 0.019417811185121536f, + 0.23349952697753906f, 0.07770441472530365f, 0.0019170257728546858f, + 0.003552661743015051f, 0.012630019336938858f, 0.03414732217788696f, + -0.04959012567996979f, -0.21843530237674713f, -0.5894723534584045f, + 0.21037738025188446f, -0.4222605526447296f, -0.5836210250854492f, + 0.011957126669585705f, 0.0031051053665578365f, -0.5127609968185425f, + -0.7208832502365112f, 0.05614267289638519f, 1.1000267267227173f, + -0.7031041383743286f, -0.20904500782489777f, 0.2827422320842743f, + 0.5169914960861206f, -0.03972400724887848f, -0.24878446757793427f, + -0.6129308938980103f, 0.3691819906234741f, -0.5902302861213684f, + -0.44936394691467285f, -0.20858828723430634f, -0.1926502138376236f, + 0.015391144901514053f, -0.05831986665725708f, 0.0012774848146364093f, + 0.9318875670433044f, -0.07232701778411865f, -0.07282842695713043f, + -0.3941495716571808f, 0.6986988186836243f, -0.6321771144866943f, + -0.1466921716928482f, -0.18214966356754303f, -0.2929930090904236f, + -0.048453543335199356f, 0.5302963852882385f, 0.3090183138847351f, + -0.7714007496833801f, -0.03153527155518532f, -0.28210026025772095f, + -0.0035391938872635365f, 1.1437654495239258f, 0.3694542348384857f, + -0.8283377885818481f, -0.06223354861140251f, 0.7067912817001343f, + -0.5538484454154968f, -0.3097091019153595f, 0.24007144570350647f, + 0.5142718553543091f, 0.5205507278442383f, -0.7828086018562317f, + 0.02196805737912655f, -0.5540323853492737f, -0.5541141629219055f, + -0.5407395958900452f, 0.83812016248703f, 0.5495181679725647f, + -0.36925947666168213f, -0.5186708569526672f, 0.035871513187885284f, + 0.9290533661842346f, 0.7525129914283752f, 0.16506646573543549f, + 0.015168293379247189f, -0.22260211408138275f }; static const float av1_hd_partition_breakout_nn_bias_16_layer0[32] = { - -2.500817060470581f, -1.1564264297485352f, 0.2977291941642761f, - -0.255332350730896f, -0.44481080770492554f, -0.6241381168365479f, - -0.4780181348323822f, -0.6444894075393677f, -1.2649517059326172f, - -1.3098241090774536f, 0.10575855523347855f, -0.6572186946868896f, - -0.13408327102661133f, -2.2591185569763184f, -0.6009128093719482f, - -0.9144248962402344f, 0.5553908944129944f, -0.40132245421409607f, - 0.3925309181213379f, 0.9891119003295898f, 0.7784745097160339f, - -0.47138360142707825f, -1.2775310277938843f, -0.3515026867389679f, - -0.4055931568145752f, -0.4989534318447113f, 0.7892708778381348f, - -0.5805770754814148f, -1.247983694076538f, -0.46460986137390137f, - -1.3504952192306519f, -0.29709282517433167f + 0.3051907420158386f, -0.7940329313278198f, 0.37281763553619385f, + -0.47349175810813904f, 0.9813013076782227f, -0.6225157976150513f, + -0.3041200339794159f, 0.18893377482891083f, -0.04121162369847298f, + 0.0799027606844902f, -0.06257568299770355f, 0.004709158092737198f, + 0.4165017306804657f, 0.40799248218536377f, -0.1605808585882187f, + 0.4690406322479248f, 0.2635003328323364f, 0.37693503499031067f, + 0.509822428226471f, 0.6932690739631653f, 0.3800438344478607f, + -0.9304105639457703f, 0.688030481338501f, -1.478393316268921f, + -0.08685281872749329f, -0.7200548648834229f, -0.9290581345558167f, + 0.47607457637786865f, -0.2131124883890152f, 0.4063669741153717f, + 0.6758500337600708f, 0.4055664539337158f }; static const float av1_hd_partition_breakout_nn_weights_16_layer1[32] = { - 0.37168681621551514f, 1.6819430589675903f, -0.24184460937976837f, - -0.5973770022392273f, -0.9343476295471191f, 9.948596000671387f, - -2.4088406562805176f, 0.7863813638687134f, -0.4910288453102112f, - -8.380901336669922f, -0.41362130641937256f, 5.82503604888916f, - -0.4502200186252594f, -1.826108694076538f, 4.183648109436035f, - 2.714763879776001f, -0.26116943359375f, 2.803555727005005f, - 0.8531390428543091f, 1.0828324556350708f, 0.8189658522605896f, - -2.4576377868652344f, -0.3627878427505493f, 2.138772487640381f, - -0.016609186306595802f, 0.7119042873382568f, 0.5726228356361389f, - 1.508818507194519f, 7.78598690032959f, 2.531076669692993f, - 6.370460510253906f, -0.04699355363845825f + 0.4050608277320862f, -0.41333451867103577f, 0.17236362397670746f, + 0.024612678214907646f, 0.9329894781112671f, -0.838448166847229f, + -1.4745935201644897f, 0.09989786148071289f, -0.10163675248622894f, + 0.17781522870063782f, 2.2620022296905518f, -0.3684936761856079f, + -0.6526914834976196f, 0.2942826449871063f, -0.35902100801467896f, + 0.10697637498378754f, 0.20386190712451935f, 1.2234491109848022f, + 0.30403849482536316f, 0.35678544640541077f, 0.29488465189933777f, + 1.736344814300537f, 0.5220925807952881f, -1.7380571365356445f, + -0.2642151713371277f, 0.7513654828071594f, -1.9644155502319336f, + 0.1906810700893402f, -0.35255730152130127f, 0.5716033577919006f, + 0.653854250907898f, -0.26972928643226624f }; static const float av1_hd_partition_breakout_nn_bias_16_layer1[1] = { - 0.42323046922683716f + 0.35929590463638306f }; static const float av1_hd_partition_breakout_nn_weights_8_layer0[FEATURE_SIZE * 32] = { - -0.9726589918136597f, -0.5411639213562012f, 0.7087171673774719f, - 1.7313759326934814f, -2.402184009552002f, -0.047024089843034744f, - -0.027457909658551216f, 1.6469149589538574f, 0.20576165616512299f, - -0.7170919179916382f, -0.10603219270706177f, 2.669275999069214f, - -0.2676953077316284f, -0.16688808798789978f, -0.03148927167057991f, - 1.4055346250534058f, -0.7267357110977173f, -1.3847031593322754f, - 0.21190257370471954f, -1.012730598449707f, -1.2875311374664307f, - -0.9335927963256836f, 0.9185367226600647f, -0.262455552816391f, - -1.8908239603042603f, -0.7330380082130432f, 0.580338716506958f, - 1.0240660905838013f, -0.741085946559906f, -0.4254375696182251f, - 1.0939886569976807f, 0.8454716205596924f, 0.028208844363689423f, - -1.108168363571167f, -0.5589184165000916f, 2.6788110733032227f, - -2.6662967205047607f, -0.7947397232055664f, 0.0377255417406559f, - 2.2486352920532227f, -0.6704809069633484f, 0.7823348641395569f, - 0.18206557631492615f, 1.8088916540145874f, -0.2850566804409027f, - -0.2228451520204544f, -0.0077590541914105415f, 0.267416387796402f, - -0.11488392949104309f, -0.3818961977958679f, 0.15274959802627563f, - 1.7191359996795654f, -0.1928352564573288f, 0.1540122777223587f, - -1.7266273498535156f, 0.16492529213428497f, -0.26917898654937744f, - 0.0453917570412159f, 0.006803665775805712f, -3.431412696838379f, - -0.8123732209205627f, 0.20750433206558228f, 0.043349288403987885f, - 1.5046347379684448f, -0.43123146891593933f, 0.496737003326416f, - 0.13186758756637573f, 1.348464012145996f, -1.0381258726119995f, - -0.14341726899147034f, -0.12307563424110413f, -0.47205984592437744f, - -0.26413795351982117f, -0.4825311005115509f, 0.01020186860114336f, - 1.8038893938064575f, 0.2135389894247055f, -0.5266213417053223f, - -0.34630000591278076f, 2.1957969665527344f, -2.6997740268707275f, - 0.22156886756420135f, 0.01723811775445938f, 0.9266343116760254f, - -0.29436543583869934f, -0.21920204162597656f, 0.050585292279720306f, - 1.8021506071090698f, 0.28632524609565735f, -1.0584778785705566f, - -0.004406697116792202f, 1.6804509162902832f, 0.1515306681394577f, - -0.5138083696365356f, -1.493039846420288f, 1.1335597038269043f, - -2.868360757827759f, -0.7500830888748169f, -0.018952427431941032f, - 1.7541661262512207f, -1.2600187063217163f, -0.19970472157001495f, - -0.027286890894174576f, -2.7354300022125244f, -1.349392056465149f, - -2.1341166496276855f, 0.1937991976737976f, -0.12994812428951263f, - -0.8666549921035767f, 0.3629917502403259f, -0.2922564446926117f, - 1.5711613893508911f, -1.5161727666854858f, -0.07508653402328491f, - -0.015387722291052341f, 1.1435129642486572f, 0.08286841958761215f, - -0.4614250361919403f, -0.26704689860343933f, 2.349350929260254f, - -0.16499075293540955f, 0.025024881586432457f, 0.001498897559940815f, - -0.04137025028467178f, -0.8304779529571533f, -0.20447227358818054f, - 0.009074501693248749f, 0.8690717816352844f + -1.1174798011779785f, -0.02540789172053337f, -0.024517325684428215f, + 0.9283695220947266f, 0.514511227607727f, 0.008708865381777287f, + -0.7743955254554749f, 0.03391582891345024f, -0.4907812476158142f, + 0.06405021250247955f, 0.009607836604118347f, -0.06413879245519638f, + -0.3100389838218689f, 0.027276739478111267f, 0.02754838392138481f, + 0.2741911709308624f, -0.33674532175064087f, -0.12841999530792236f, + -0.07583238184452057f, 1.2658021450042725f, -0.07618967443704605f, + 0.018406564369797707f, 0.36800676584243774f, 1.1263890266418457f, + 0.29222336411476135f, -0.1937945932149887f, -0.781943678855896f, + 0.08040471374988556f, 0.06932314485311508f, 0.27592363953590393f, + 0.6615497469902039f, -0.3753996193408966f, 0.04914139211177826f, + -0.18584918975830078f, 0.3851499855518341f, -0.14877784252166748f, + 0.8386304974555969f, -0.9132646322250366f, 0.09750116616487503f, + -0.38142505288124084f, -0.27932387590408325f, -0.3158399760723114f, + 0.3219115734100342f, -0.11211645603179932f, -0.1601802557706833f, + 0.199840247631073f, -0.1438094526529312f, 1.4131516218185425f, + -0.6507886648178101f, -0.35280969738960266f, -0.07980803400278091f, + 0.9108555912971497f, -0.49617353081703186f, -0.2692374289035797f, + 0.019175099208950996f, 0.5736552476882935f, -0.48202967643737793f, + -0.26134830713272095f, 0.018342891708016396f, 0.6154197454452515f, + -0.18230408430099487f, -0.8209198713302612f, 0.6161503195762634f, + 0.4757719039916992f, 0.5258967876434326f, -0.9027984142303467f, + -0.47966063022613525f, 0.6204502582550049f, -1.342071294784546f, + 0.12434060871601105f, 0.14531831443309784f, -0.15101315081119537f, + -0.35774746537208557f, 0.08184701204299927f, -0.04071677848696709f, + 1.2944822311401367f, -0.06492389738559723f, 0.34913739562034607f, + -0.08534345775842667f, 1.3283460140228271f, -0.7555822134017944f, + 0.3319704234600067f, -0.17644117772579193f, 0.5248327255249023f, + -0.46480831503868103f, -0.23956067860126495f, 0.0013285222230479121f, + 0.5432708859443665f, 0.30651840567588806f, 0.37756049633026123f, + -0.03282826766371727f, 0.06274658441543579f, -0.2130001038312912f, + 0.2907213270664215f, 0.09125600755214691f, -0.133547842502594f, + -0.05953352898359299f, 0.032438505440950394f, 0.05902266874909401f, + -0.043947484344244f, -1.100358486175537f, 0.2626442313194275f, + 0.056724611669778824f, -0.19415289163589478f, 0.010464449413120747f, + -0.385423481464386f, -0.29392293095588684f, 0.5806636214256287f, + -0.28265658020973206f, 1.0120196342468262f, 0.17220932245254517f, + -0.9805957078933716f, -0.368004709482193f, -0.42056915163993835f, + -0.1005094051361084f, -0.740735650062561f, 0.8810465335845947f, + -0.4035398066043854f, 0.06286423653364182f, -0.26255688071250916f, + -0.24484986066818237f, -0.1917300969362259f, 0.23190940916538239f, + -0.5081801414489746f, -0.909449577331543f, 0.6952412128448486f, + -0.41929101943969727f, -0.28803330659866333f }; static const float av1_hd_partition_breakout_nn_bias_8_layer0[32] = { - 0.7538588047027588f, -0.0891113132238388f, 0.6207568049430847f, - 0.1754276007413864f, -0.9016708731651306f, -0.5710541009902954f, - 0.3960225582122803f, -0.5215008854866028f, 0.1935194879770279f, - 0.1463070660829544f, -0.5508337020874023f, -0.25621265172958374f, - -0.21759319305419922f, -0.6702320575714111f, -0.510668158531189f, - -0.11305645108222961f, 0.9119726419448853f, 0.932460367679596f, - 0.5708784461021423f, 0.9256153702735901f, 0.4665280282497406f, - 0.2577301561832428f, 0.4606616199016571f, 0.4183802306652069f, - -0.11034739017486572f, -1.2057222127914429f, -1.5402069091796875f, - 0.36954033374786377f, -0.07251504063606262f, 0.4745430052280426f, - -0.4834096133708954f, -0.08527179062366486f + 0.9311190843582153f, -0.04600553959608078f, 0.4736453592777252f, + -0.21461233496665955f, -0.6837071776390076f, 0.41968896985054016f, + 0.5667760372161865f, -0.295064777135849f, 0.6580685973167419f, + -0.12489219009876251f, 0.6441367864608765f, 0.017892301082611084f, + 0.3467213213443756f, 0.24851299822330475f, 0.27420252561569214f, + 0.6503739356994629f, -0.9539271593093872f, -0.7639755606651306f, + 0.3438030183315277f, -0.65008944272995f, 0.1381210833787918f, + 0.17897674441337585f, 0.6611759066581726f, -0.6359773278236389f, + 0.753626823425293f, -0.8921093940734863f, 0.7966955304145813f, + -0.20982062816619873f, 0.2511371076107025f, -0.23097757995128632f, + 1.1271780729293823f, 0.26889708638191223f }; static const float av1_hd_partition_breakout_nn_weights_8_layer1[32] = { - 1.1129406690597534f, -1.5291675329208374f, -0.8876321911811829f, - 0.30224356055259705f, -0.40715256333351135f, -0.5691804885864258f, - 0.8208304643630981f, -0.6657942533493042f, -1.1190624237060547f, - 3.5144896507263184f, -0.4905086159706116f, -0.04449653998017311f, - 0.8738682270050049f, 0.38138872385025024f, -2.4828171730041504f, - -0.31892910599708557f, 0.4230459928512573f, 0.6047843098640442f, - 0.5003713369369507f, 1.1065287590026855f, 0.97183758020401f, - 0.2900541126728058f, 0.5317407846450806f, 0.6475094556808472f, - 4.615662574768066f, -7.686334133148193f, -0.9662672877311707f, - 0.6237889528274536f, -0.5734347701072693f, -0.4399879276752472f, - -0.016914604231715202f, -0.6294510960578918f + 2.6572957038879395f, 0.17708347737789154f, 0.09978588670492172f, + -0.15976227819919586f, 2.394906997680664f, 0.2525644302368164f, + 0.4016534388065338f, -0.18184596300125122f, 0.20857582986354828f, + -0.6041266918182373f, 0.1736220121383667f, 0.5656698346138f, + 1.886559247970581f, 1.0891855955123901f, 0.3225744664669037f, + 0.5683131217956543f, -1.198813796043396f, -1.226926326751709f, + 0.42729490995407104f, -1.0998389720916748f, -1.6383575201034546f, + 0.47224658727645874f, 0.3989439904689789f, 0.003893804969266057f, + 0.2592338025569916f, -0.9896919131278992f, 0.23605197668075562f, + -1.2522234916687012f, 0.44355013966560364f, -0.37252721190452576f, + 0.4963497519493103f, -0.596142053604126f }; static const float av1_hd_partition_breakout_nn_bias_8_layer1[1] = { - 1.0166552066802979f + 0.8022159934043884f }; static const float @@ -3059,19 +3931,19 @@ { // lowres { - 2.312415f, // p = 0.9099 - 1.848067f, // p = 0.8639 - 2.708050f, // p = 0.9375 - 1.567311f, // p = 0.8274 - 3.167687f, // p = 0.9596 + 1.587045f, // p = 0.830200 + 1.982054f, // p = 0.878900 + 2.150334f, // p = 0.895700 + 2.093809f, // p = 0.890300 + 2.428837f, // p = 0.919000 }, // midres { - 2.312415f, // p = 0.9099 - 1.848067f, // p = 0.8639 - 2.708050f, // p = 0.9375 - 1.567311f, // p = 0.8274 - 3.167687f, // p = 0.9596 + 2.044592f, // p = 0.885400 + 1.961544f, // p = 0.876700 + 2.940236f, // p = 0.949800 + 2.301486f, // p = 0.909000 + 2.265745f, // p = 0.906000 }, // hdres { @@ -3086,19 +3958,19 @@ { // lowres { - 2.312415f, // p = 0.9099 - 1.848067f, // p = 0.8639 - 2.708050f, // p = 0.9375 - 1.567311f, // p = 0.8274 - 3.167687f, // p = 0.9596 + 1.390049f, // p = 0.800600 + 1.784893f, // p = 0.856300 + 3.199090f, // p = 0.960800 + 2.119666f, // p = 0.892800 + 2.026000f, // p = 0.883500 }, // midres { - 2.312415f, // p = 0.9099 - 1.848067f, // p = 0.8639 - 2.708050f, // p = 0.9375 - 1.567311f, // p = 0.8274 - 3.167687f, // p = 0.9596 + 1.587045f, // p = 0.830200 + 1.982054f, // p = 0.878900 + 2.150334f, // p = 0.895700 + 2.093809f, // p = 0.890300 + 2.428837f, // p = 0.919000 }, // hdres { @@ -3210,19 +4082,19 @@ { // lowres { - -2.197225f, // p = 0.1 - -5.289093f, // p = 0.005021 - -2.522730f, // p = 0.07428 - -7.174085f, // p = 0.0007656 - -6.316499f, // p = 0.001803 + -2.758650f, // p = 0.059600 + -4.385780f, // p = 0.012300 + -4.230259f, // p = 0.014340 + -6.455747f, // p = 0.001569 + -8.541803f, // p = 0.000195 }, // midres { - -2.197225f, // p = 0.1 - -5.289093f, // p = 0.005021 - -2.522730f, // p = 0.07428 - -7.174085f, // p = 0.0007656 - -6.316499f, // p = 0.001803 + -4.881569f, // p = 0.007528 + -4.656635f, // p = 0.009409 + -3.037505f, // p = 0.045760 + -6.679364f, // p = 0.001255 + -5.944256f, // p = 0.002614 }, // hdres { @@ -3237,19 +4109,19 @@ { // lowres { - -2.197225f, // p = 0.1 - -5.289093f, // p = 0.005021 - -2.522730f, // p = 0.07428 - -7.174085f, // p = 0.0007656 - -6.316499f, // p = 0.001803 + -4.381672f, // p = 0.012350 + -4.034317f, // p = 0.017390 + -3.986288f, // p = 0.018230 + -7.325495f, // p = 0.000658 + -7.908147f, // p = 0.000368 }, // midres { - -2.197225f, // p = 0.1 - -5.289093f, // p = 0.005021 - -2.522730f, // p = 0.07428 - -7.174085f, // p = 0.0007656 - -6.316499f, // p = 0.001803 + -2.758650f, // p = 0.059600 + -4.385780f, // p = 0.012300 + -4.230259f, // p = 0.014340 + -6.455747f, // p = 0.001569 + -8.541803f, // p = 0.000195 }, // hdres {
diff --git a/av1/encoder/partition_search.c b/av1/encoder/partition_search.c index 81ac00b..c3b667d 100644 --- a/av1/encoder/partition_search.c +++ b/av1/encoder/partition_search.c
@@ -10,6 +10,7 @@ */ #include <float.h> +#include <inttypes.h> #include "config/aom_config.h" @@ -41,7 +42,9 @@ #include "av1/encoder/tune_vmaf.h" #endif -#define COLLECT_MOTION_SEARCH_FEATURE_SB 0 +#ifndef COLLECT_MOTION_SEARCH_FEATURE_SB +#define COLLECT_MOTION_SEARCH_FEATURE_SB CONFIG_PARTITION_SEARCH_ORDER +#endif #if CONFIG_PARTITION_SEARCH_ORDER void av1_reset_part_sf(PARTITION_SPEED_FEATURES *part_sf) { @@ -96,6 +99,7 @@ #endif // CONFIG_PARTITION_SEARCH_ORDER #if !CONFIG_REALTIME_ONLY +#if COLLECT_MOTION_SEARCH_FEATURE_SB // If input |features| is NULL, write tpl stats to file for each super block. // Otherwise, store tpl stats to |features|. // The tpl stats is computed in the unit of tpl_bsize_1d (16x16). @@ -204,6 +208,7 @@ } } } +#endif // COLLECT_MOTION_SEARCH_FEATURE_SB #endif // !CONFIG_REALTIME_ONLY static void update_txfm_count(MACROBLOCK *x, MACROBLOCKD *xd, @@ -412,7 +417,6 @@ const int mi_col = xd->mi_col; if (!is_inter) { xd->cfl.store_y = store_cfl_required(cm, xd); - mbmi->skip_txfm = 1; for (int plane = 0; plane < num_planes; ++plane) { av1_encode_intra_block_plane(cpi, x, bsize, plane, dry_run, cpi->optimize_seg_arr[mbmi->segment_id]); @@ -2107,6 +2111,7 @@ (subsampling_x + subsampling_y))); } + if (!is_inter_block(xd->mi[0])) xd->mi[0]->skip_txfm = 0; encode_superblock(cpi, tile_data, td, tp, dry_run, bsize, rate); if (!dry_run) { update_cb_offsets(x, bsize, subsampling_x, subsampling_y); @@ -4560,6 +4565,7 @@ av1_restore_context(x, x_ctx, mi_row, mi_col, bsize, av1_num_planes(cm)); } +#if COLLECT_MOTION_SEARCH_FEATURE_SB // The max number of nodes in the partition tree. // The number of leaf nodes is (128x128) / (4x4) = 1024. // The number of All possible parent nodes is 1 + 2 + ... + 512 = 1023. @@ -4623,6 +4629,7 @@ fclose(pfile); } +#endif // COLLECT_MOTION_SEARCH_FEATURE_SB #if CONFIG_PARTITION_SEARCH_ORDER static void verify_write_partition_tree(const AV1_COMP *const cpi, @@ -4911,18 +4918,77 @@ node->partitioning = partitioning; bsize = node->block_size; } - if (partitioning == PARTITION_SPLIT) { - const BLOCK_SIZE subsize = get_partition_subsize(bsize, PARTITION_SPLIT); - for (int i = 0; i < 4; ++i) { - if (node != NULL) { // Suppress warning - node->split[i] = av1_alloc_pc_tree_node(subsize); - if (!node->split[i]) - aom_internal_error(error_info, AOM_CODEC_MEM_ERROR, - "Failed to allocate PC_TREE"); - node->split[i]->index = i; - tree_node_queue[last_idx] = node->split[i]; - ++last_idx; - } + if (partitioning != PARTITION_NONE) { + const BLOCK_SIZE subsize = get_partition_subsize(bsize, partitioning); + // Smaller block size for AB partition + const BLOCK_SIZE subsize2 = get_partition_subsize(bsize, PARTITION_SPLIT); + + switch (partitioning) { + case PARTITION_SPLIT: + case PARTITION_HORZ_4: + case PARTITION_VERT_4: + for (int i = 0; i < 4; ++i) { + if (node != NULL) { // Suppress warning + node->split[i] = av1_alloc_pc_tree_node(subsize); + if (!node->split[i]) + aom_internal_error(error_info, AOM_CODEC_MEM_ERROR, + "Failed to allocate PC_TREE"); + node->split[i]->index = i; + tree_node_queue[last_idx] = node->split[i]; + ++last_idx; + } + } + break; + case PARTITION_HORZ: + case PARTITION_VERT: + for (int i = 0; i < 2; ++i) { + if (node != NULL) { // Suppress warning + node->split[i] = av1_alloc_pc_tree_node(subsize); + if (!node->split[i]) + aom_internal_error(error_info, AOM_CODEC_MEM_ERROR, + "Failed to allocate PC_TREE"); + node->split[i]->index = i; + tree_node_queue[last_idx] = node->split[i]; + ++last_idx; + } + } + break; + case PARTITION_HORZ_A: + case PARTITION_VERT_A: + if (node != NULL) { // Suppress warning + node->split[0] = av1_alloc_pc_tree_node(subsize2); + node->split[1] = av1_alloc_pc_tree_node(subsize2); + node->split[2] = av1_alloc_pc_tree_node(subsize); + for (int i = 0; i < 3; ++i) { + if (!node->split[i]) + aom_internal_error(error_info, AOM_CODEC_MEM_ERROR, + "Failed to allocate PC_TREE"); + node->split[i]->index = i; + tree_node_queue[last_idx] = node->split[i]; + ++last_idx; + } + } + break; + case PARTITION_HORZ_B: + case PARTITION_VERT_B: + if (node != NULL) { // Suppress warning + node->split[0] = av1_alloc_pc_tree_node(subsize); + node->split[1] = av1_alloc_pc_tree_node(subsize2); + node->split[2] = av1_alloc_pc_tree_node(subsize2); + for (int i = 0; i < 3; ++i) { + if (!node->split[i]) + aom_internal_error(error_info, AOM_CODEC_MEM_ERROR, + "Failed to allocate PC_TREE"); + node->split[i]->index = i; + tree_node_queue[last_idx] = node->split[i]; + ++last_idx; + } + } + break; + case PARTITION_NONE: + default: + // Should not hit this case. + assert(0); } } --num_nodes; @@ -5554,16 +5620,6 @@ part_search_state.partition_rect_allowed[VERT] &= !blk_params.has_cols; } -#ifndef NDEBUG - // Nothing should rely on the default value of this array (which is just - // leftover from encoding the previous block. Setting it to fixed pattern - // when debugging. - // bit 0, 1, 2 are blk_skip of each plane - // bit 4, 5, 6 are initialization checking of each plane - memset(x->txfm_search_info.blk_skip, 0x77, - sizeof(x->txfm_search_info.blk_skip)); -#endif // NDEBUG - assert(mi_size_wide[bsize] == mi_size_high[bsize]); // Set buffers and offsets. @@ -5594,12 +5650,13 @@ // external ML model. // TODO(chengchen): reduce motion search. This function is similar to // av1_get_max_min_partition_features(). - if (COLLECT_MOTION_SEARCH_FEATURE_SB && !frame_is_intra_only(cm) && - bsize == cm->seq_params->sb_size) { +#if COLLECT_MOTION_SEARCH_FEATURE_SB + if (!frame_is_intra_only(cm) && bsize == cm->seq_params->sb_size) { av1_collect_motion_search_features_sb(cpi, td, tile_data, mi_row, mi_col, bsize, /*features=*/NULL); collect_tpl_stats_sb(cpi, bsize, mi_row, mi_col, /*features=*/NULL); } +#endif // !COLLECT_MOTION_SEARCH_FEATURE_SB // Update rd cost of the bound using the current multiplier. av1_rd_cost_update(x->rdmult, &best_rdc); @@ -5861,12 +5918,11 @@ const int emit_output = multi_pass_mode != SB_DRY_PASS; const RUN_TYPE run_type = emit_output ? OUTPUT_ENABLED : DRY_RUN_NORMAL; +#if COLLECT_MOTION_SEARCH_FEATURE_SB // Write partition tree to file. Not used by default. - if (COLLECT_MOTION_SEARCH_FEATURE_SB) { - write_partition_tree(cpi, pc_tree, bsize, mi_row, mi_col); - ++cpi->sb_counter; - } - + write_partition_tree(cpi, pc_tree, bsize, mi_row, mi_col); + ++cpi->sb_counter; +#endif // COLLECT_MOTION_SEARCH_FEATURE_SB set_cb_offsets(x->cb_offset, 0, 0); encode_sb(cpi, td, tile_data, tp, mi_row, mi_col, run_type, bsize, pc_tree, NULL);
diff --git a/av1/encoder/partition_strategy.c b/av1/encoder/partition_strategy.c index 526ccb1e..4b44ba4 100644 --- a/av1/encoder/partition_strategy.c +++ b/av1/encoder/partition_strategy.c
@@ -210,7 +210,6 @@ bit_depth, &output)) { aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR, "Error allocating CNN data"); - return; } } else { uint8_t *image[1] = { x->plane[AOM_PLANE_Y].src.buf - stride - 1 }; @@ -219,7 +218,6 @@ cnn_config, &thread_data, &output)) { aom_internal_error(xd->error_info, AOM_CODEC_MEM_ERROR, "Error allocating CNN data"); - return; } } @@ -1323,12 +1321,14 @@ #define FEATURES 18 #define LABELS 4 +#define NEW_LABELS 3 // Use a ML model to predict if horz4 and vert4 should be considered. void av1_ml_prune_4_partition(AV1_COMP *const cpi, MACROBLOCK *const x, int part_ctx, int64_t best_rd, PartitionSearchState *part_state, int *part4_allowed, unsigned int pb_source_variance) { + const AV1_COMMON *const cm = &cpi->common; const PartitionBlkParams blk_params = part_state->part_blk_params; const int mi_row = blk_params.mi_row; const int mi_col = blk_params.mi_col; @@ -1345,15 +1345,34 @@ if (best_rd >= 1000000000) return; int64_t *horz_rd = rect_part_rd[HORZ4]; int64_t *vert_rd = rect_part_rd[VERT4]; + + const int is_720p_or_larger = AOMMIN(cm->width, cm->height) >= 720; + const int is_480p_or_larger = AOMMIN(cm->width, cm->height) >= 480; + // res_idx is 0 for res < 480p, 1 for 480p, 2 for 720p+ + const int res_idx = is_480p_or_larger + is_720p_or_larger; + + const int bsize_idx = convert_bsize_to_idx(bsize); + if (bsize_idx < 0) return; + const float *ml_mean = av1_partition4_nn_mean[bsize_idx]; + const float *ml_std = av1_partition4_nn_std[bsize_idx]; + + int ml_model_index = (cpi->sf.part_sf.ml_4_partition_search_level_index < 3); + const NN_CONFIG *nn_config = NULL; // 4-way partitions are only allowed for these three square block sizes. switch (bsize) { - case BLOCK_16X16: nn_config = &av1_4_partition_nnconfig_16; break; - case BLOCK_32X32: nn_config = &av1_4_partition_nnconfig_32; break; - case BLOCK_64X64: nn_config = &av1_4_partition_nnconfig_64; break; + case BLOCK_16X16: + nn_config = &av1_4_partition_nnconfig_16[ml_model_index]; + break; + case BLOCK_32X32: + nn_config = &av1_4_partition_nnconfig_32[ml_model_index]; + break; + case BLOCK_64X64: + nn_config = &av1_4_partition_nnconfig_64[ml_model_index]; + break; default: assert(0 && "Unexpected bsize."); } - if (!nn_config) return; + if (!nn_config || !ml_mean || !ml_std) return; // Generate features. float features[FEATURES]; @@ -1437,6 +1456,12 @@ } assert(feature_index == FEATURES); + if (ml_model_index) { + for (int idx = 0; idx < FEATURES; idx++) { + features[idx] = (features[idx] - ml_mean[idx]) / ml_std[idx]; + } + } + // Write features to file if (!frame_is_intra_only(&cpi->common)) { write_features_to_file(cpi->oxcf.partition_info_path, @@ -1444,34 +1469,61 @@ FEATURES, 7, bsize, mi_row, mi_col); } - // Calculate scores using the NN model. - float score[LABELS] = { 0.0f }; - av1_nn_predict(features, nn_config, 1, score); - int int_score[LABELS]; - int max_score = -1000; - for (int i = 0; i < LABELS; ++i) { - int_score[i] = (int)(100 * score[i]); - max_score = AOMMAX(int_score[i], max_score); - } + if (ml_model_index == 0) { + // Calculate scores using the NN model. + float score[LABELS] = { 0.0f }; + av1_nn_predict(features, nn_config, 1, score); + int int_score[LABELS]; + int max_score = -1000; + for (int i = 0; i < LABELS; ++i) { + int_score[i] = (int)(100 * score[i]); + max_score = AOMMAX(int_score[i], max_score); + } - // Make decisions based on the model scores. - int thresh = max_score; - switch (bsize) { - case BLOCK_16X16: thresh -= 500; break; - case BLOCK_32X32: thresh -= 500; break; - case BLOCK_64X64: thresh -= 200; break; - default: break; - } - av1_zero_array(part4_allowed, NUM_PART4_TYPES); - for (int i = 0; i < LABELS; ++i) { - if (int_score[i] >= thresh) { - if ((i >> 0) & 1) part4_allowed[HORZ4] = 1; - if ((i >> 1) & 1) part4_allowed[VERT4] = 1; + // Make decisions based on the model scores. + int thresh = max_score; + switch (bsize) { + case BLOCK_16X16: thresh -= 500; break; + case BLOCK_32X32: thresh -= 500; break; + case BLOCK_64X64: thresh -= 200; break; + default: break; + } + av1_zero_array(part4_allowed, NUM_PART4_TYPES); + for (int i = 0; i < LABELS; ++i) { + if (int_score[i] >= thresh) { + if ((i >> 0) & 1) part4_allowed[HORZ4] = 1; + if ((i >> 1) & 1) part4_allowed[VERT4] = 1; + } + } + } else { + // Calculate scores using the NN model. + float score[NEW_LABELS] = { 0.0f }; + float probs[NEW_LABELS] = { 0.0f }; + av1_nn_predict(features, nn_config, 1, score); + + av1_nn_softmax(score, probs, NEW_LABELS); + + // Make decisions based on the model scores. + const float search_thresh = av1_partition4_search_thresh + [cpi->sf.part_sf.ml_4_partition_search_level_index][res_idx][bsize_idx]; + const float not_search_thresh = av1_partition4_not_search_thresh + [cpi->sf.part_sf.ml_4_partition_search_level_index][res_idx][bsize_idx]; + + for (int i = 1; i < NEW_LABELS; ++i) { + if (probs[i] >= search_thresh) { + if (i == 1) part4_allowed[HORZ4] = 1; + if (i == 2) part4_allowed[VERT4] = 1; + } + if (probs[i] < not_search_thresh) { + if (i == 1) part4_allowed[HORZ4] = 0; + if (i == 2) part4_allowed[VERT4] = 0; + } } } } #undef FEATURES #undef LABELS +#undef NEW_LABELS #define FEATURES 4 void av1_ml_predict_breakout(AV1_COMP *const cpi, const MACROBLOCK *const x, @@ -1536,16 +1588,29 @@ float rate_f = (float)AOMMIN(rd_stats->rate, INT_MAX); rate_f = ((float)x->rdmult / 128.0f / 512.0f / (float)(1 << num_pels_log2)) * rate_f; - features[feature_index++] = rate_f; + features[feature_index++] = + cpi->sf.part_sf.ml_partition_search_breakout_model_index + ? log1pf((float)rate_f) + : rate_f; const float dist_f = (float)(AOMMIN(rd_stats->dist, INT_MAX) >> num_pels_log2); - features[feature_index++] = dist_f; + features[feature_index++] = + cpi->sf.part_sf.ml_partition_search_breakout_model_index + ? log1pf((float)dist_f) + : dist_f; - features[feature_index++] = (float)pb_source_variance; + features[feature_index++] = + cpi->sf.part_sf.ml_partition_search_breakout_model_index + ? log1pf((float)pb_source_variance) + : (float)pb_source_variance; const int dc_q = (int)x->plane[0].dequant_QTX[0] >> (bit_depth - 8); - features[feature_index++] = (float)(dc_q * dc_q) / 256.0f; + features[feature_index++] = + cpi->sf.part_sf.ml_partition_search_breakout_model_index + ? log1pf((float)(dc_q * dc_q) / 256.0f) + : (float)(dc_q * dc_q) / 256.0f; + assert(feature_index == FEATURES); if (cpi->sf.part_sf.ml_partition_search_breakout_model_index) { @@ -1719,6 +1784,7 @@ av1_is_whole_blk_in_frame(blk_params, mi_params); if (try_intra_cnn_based_part_prune) { + assert(x->e_mbd.error_info->setjmp); intra_mode_cnn_partition(&cpi->common, x, x->part_search_info.quad_tree_idx, cpi->sf.part_sf.intra_cnn_based_part_prune_level, part_state); @@ -1927,6 +1993,7 @@ // Pruning: pruning out some ab partitions using a DNN taking rd costs of // sub-blocks from previous basic partition types. if (cpi->sf.part_sf.ml_prune_partition && ext_partition_allowed && + part_cfg->enable_ab_partitions && part_state->partition_rect_allowed[HORZ] && part_state->partition_rect_allowed[VERT]) { // TODO(huisu@google.com): x->source_variance may not be the current
diff --git a/av1/encoder/pass2_strategy.c b/av1/encoder/pass2_strategy.c index 99dcfb8..0c448cf 100644 --- a/av1/encoder/pass2_strategy.c +++ b/av1/encoder/pass2_strategy.c
@@ -164,12 +164,28 @@ return (int)max_bits; } +static const double q_pow_term[(QINDEX_RANGE >> 5) + 1] = { 0.65, 0.70, 0.75, + 0.80, 0.85, 0.90, + 0.95, 0.95, 0.95 }; +#define ERR_DIVISOR 96.0 +static double calc_correction_factor(double err_per_mb, int q) { + const double error_term = err_per_mb / ERR_DIVISOR; + const int index = q >> 5; + // Adjustment to power term based on qindex + const double power_term = + q_pow_term[index] + + (((q_pow_term[index + 1] - q_pow_term[index]) * (q % 32)) / 32.0); + assert(error_term >= 0.0); + return fclamp(pow(error_term, power_term), 0.05, 5.0); +} + // Based on history adjust expectations of bits per macroblock. static void twopass_update_bpm_factor(AV1_COMP *cpi, int rate_err_tol) { TWO_PASS *const twopass = &cpi->ppi->twopass; const PRIMARY_RATE_CONTROL *const p_rc = &cpi->ppi->p_rc; // Based on recent history adjust expectations of bits per macroblock. + double damp_fac = AOMMAX(5.0, rate_err_tol / 10.0); double rate_err_factor = 1.0; const double adj_limit = AOMMAX(0.2, (double)(100 - rate_err_tol) / 200.0); const double min_fac = 1.0 - adj_limit; @@ -204,7 +220,9 @@ #endif // CONFIG_THREE_PASS int err_estimate = p_rc->rate_error_estimate; + int64_t bits_left = twopass->bits_left; int64_t total_actual_bits = p_rc->total_actual_bits; + int64_t bits_off_target = p_rc->vbr_bits_off_target; double rolling_arf_group_actual_bits = (double)twopass->rolling_arf_group_actual_bits; double rolling_arf_group_target_bits = @@ -219,6 +237,10 @@ : 0; total_actual_bits = simulate_parallel_frame ? p_rc->temp_total_actual_bits : p_rc->total_actual_bits; + bits_off_target = simulate_parallel_frame ? p_rc->temp_vbr_bits_off_target + : p_rc->vbr_bits_off_target; + bits_left = + simulate_parallel_frame ? p_rc->temp_bits_left : twopass->bits_left; rolling_arf_group_target_bits = (double)(simulate_parallel_frame ? p_rc->temp_rolling_arf_group_target_bits @@ -231,22 +253,22 @@ : p_rc->rate_error_estimate; #endif - if ((p_rc->bits_off_target && total_actual_bits > 0) && - (rolling_arf_group_target_bits >= 1.0)) { - if (rolling_arf_group_actual_bits > rolling_arf_group_target_bits) { - double error_fraction = - (rolling_arf_group_actual_bits - rolling_arf_group_target_bits) / - rolling_arf_group_target_bits; - error_fraction = (error_fraction > 1.0) ? 1.0 : error_fraction; - rate_err_factor = 1.0 + error_fraction; + if (p_rc->bits_off_target && total_actual_bits > 0) { + if (cpi->ppi->lap_enabled) { + rate_err_factor = rolling_arf_group_actual_bits / + DOUBLE_DIVIDE_CHECK(rolling_arf_group_target_bits); } else { - double error_fraction = - (rolling_arf_group_target_bits - rolling_arf_group_actual_bits) / - rolling_arf_group_target_bits; - rate_err_factor = 1.0 - error_fraction; + rate_err_factor = 1.0 - ((double)(bits_off_target) / + AOMMAX(total_actual_bits, bits_left)); } - rate_err_factor = fclamp(rate_err_factor, min_fac, max_fac); + // Adjustment is damped if this is 1 pass with look ahead processing + // (as there are only ever a few frames of data) and for all but the first + // GOP in normal two pass. + if ((twopass->bpm_factor != 1.0) || cpi->ppi->lap_enabled) { + rate_err_factor = 1.0 + ((rate_err_factor - 1.0) / damp_fac); + } + rate_err_factor = AOMMAX(min_fac, AOMMIN(max_fac, rate_err_factor)); } // Is the rate control trending in the right direction. Only make @@ -254,44 +276,36 @@ if ((rate_err_factor < 1.0 && err_estimate >= 0) || (rate_err_factor > 1.0 && err_estimate <= 0)) { twopass->bpm_factor *= rate_err_factor; - twopass->bpm_factor = fclamp(twopass->bpm_factor, min_fac, max_fac); + if (rate_err_tol >= 100) { + twopass->bpm_factor = + AOMMAX(min_fac, AOMMIN(max_fac, twopass->bpm_factor)); + } else { + twopass->bpm_factor = AOMMAX(0.1, AOMMIN(10.0, twopass->bpm_factor)); + } } } -static const double q_div_term[(QINDEX_RANGE >> 4) + 1] = { - 18.0, 30.0, 38.0, 44.0, 47.0, 50.0, 52.0, 54.0, 56.0, - 58.0, 60.0, 62.0, 64.0, 66.0, 68.0, 70.0, 72.0 -}; - -#define EPMB_SCALER 1250000 -static double calc_correction_factor(double err_per_mb, int q) { - double power_term = 0.90; - const int index = q >> 4; - const double divisor = - q_div_term[index] + - (((q_div_term[index + 1] - q_div_term[index]) * (q % 16)) / 16.0); - double error_term = EPMB_SCALER * pow(err_per_mb, power_term); - return error_term / divisor; +static int qbpm_enumerator(int rate_err_tol) { + return 1200000 + ((300000 * AOMMIN(75, AOMMAX(rate_err_tol - 25, 0))) / 75); } // Similar to find_qindex_by_rate() function in ratectrl.c, but includes // calculation of a correction_factor. -static int find_qindex_by_rate_with_correction(uint64_t desired_bits_per_mb, - aom_bit_depth_t bit_depth, - double error_per_mb, - double group_weight_factor, - int best_qindex, - int worst_qindex) { +static int find_qindex_by_rate_with_correction( + uint64_t desired_bits_per_mb, aom_bit_depth_t bit_depth, + double error_per_mb, double group_weight_factor, int rate_err_tol, + int best_qindex, int worst_qindex) { assert(best_qindex <= worst_qindex); int low = best_qindex; int high = worst_qindex; while (low < high) { const int mid = (low + high) >> 1; - const double q_factor = calc_correction_factor(error_per_mb, mid); + const double mid_factor = calc_correction_factor(error_per_mb, mid); const double q = av1_convert_qindex_to_q(mid, bit_depth); + const int enumerator = qbpm_enumerator(rate_err_tol); const uint64_t mid_bits_per_mb = - (uint64_t)((q_factor * group_weight_factor) / q); + (uint64_t)((enumerator * mid_factor * group_weight_factor) / q); if (mid_bits_per_mb > desired_bits_per_mb) { low = mid + 1; @@ -343,10 +357,6 @@ const uint64_t target_norm_bits_per_mb = ((uint64_t)av_target_bandwidth << BPER_MB_NORMBITS) / active_mbs; int rate_err_tol = AOMMIN(rc_cfg->under_shoot_pct, rc_cfg->over_shoot_pct); - const double size_factor = - (active_mbs < 500) ? 0.925 : ((active_mbs > 3000) ? 1.05 : 1.0); - const double speed_factor = - AOMMIN(1.02, (0.975 + (0.005 * cpi->oxcf.speed))); // Update bpm correction factor based on previous GOP rate error. twopass_update_bpm_factor(cpi, rate_err_tol); @@ -355,8 +365,7 @@ // content at the given rate. int q = find_qindex_by_rate_with_correction( target_norm_bits_per_mb, cpi->common.seq_params->bit_depth, - av_err_per_mb, - cpi->ppi->twopass.bpm_factor * speed_factor * size_factor, + av_err_per_mb, cpi->ppi->twopass.bpm_factor, rate_err_tol, rc->best_quality, rc->worst_quality); // Restriction on active max q for constrained quality mode. @@ -574,10 +583,15 @@ } } +// scale_max_boost = (cpi->oxcf.mode != REALTIME) in most cases as it was only +// tuned in non-rtc cases. The only exception is when we derive +// gfu_boost_average, we pass scale_max_boost = false for better coding +// efficiency. static double calc_frame_boost(const PRIMARY_RATE_CONTROL *p_rc, const FRAME_INFO *frame_info, const FIRSTPASS_STATS *this_frame, - double this_frame_mv_in_out, double max_boost) { + double this_frame_mv_in_out, double max_boost, + const bool scale_max_boost) { double frame_boost; const double lq = av1_convert_qindex_to_q(p_rc->avg_frame_qindex[INTER_FRAME], frame_info->bit_depth); @@ -593,11 +607,14 @@ // Increase boost for frames where new data coming into frame (e.g. zoom out). // Slightly reduce boost if there is a net balance of motion out of the frame // (zoom in). The range for this_frame_mv_in_out is -1.0 to +1.0. - if (this_frame_mv_in_out > 0.0) + if (this_frame_mv_in_out > 0.0) { frame_boost += frame_boost * (this_frame_mv_in_out * 2.0); + if (scale_max_boost) max_boost += max_boost * (this_frame_mv_in_out * 2.0); + } // In the extreme case the boost is halved. - else + else { frame_boost += frame_boost * (this_frame_mv_in_out / 2.0); + } return AOMMIN(frame_boost, max_boost * boost_q_correction); } @@ -664,7 +681,7 @@ const PRIMARY_RATE_CONTROL *p_rc, FRAME_INFO *frame_info, int offset, int f_frames, int b_frames, int *num_fpstats_used, int *num_fpstats_required, - int project_gfu_boost) { + int project_gfu_boost, const bool scale_max_boost) { int i; GF_GROUP_STATS gf_stats; init_gf_stats(&gf_stats); @@ -697,10 +714,10 @@ : gf_stats.decay_accumulator; } - boost_score += - gf_stats.decay_accumulator * - calc_frame_boost(p_rc, frame_info, this_frame, - gf_stats.this_frame_mv_in_out, GF_MAX_BOOST); + boost_score += gf_stats.decay_accumulator * + calc_frame_boost(p_rc, frame_info, this_frame, + gf_stats.this_frame_mv_in_out, GF_MAX_BOOST, + scale_max_boost); if (num_fpstats_used) (*num_fpstats_used)++; } @@ -733,10 +750,10 @@ : gf_stats.decay_accumulator; } - boost_score += - gf_stats.decay_accumulator * - calc_frame_boost(p_rc, frame_info, this_frame, - gf_stats.this_frame_mv_in_out, GF_MAX_BOOST); + boost_score += gf_stats.decay_accumulator * + calc_frame_boost(p_rc, frame_info, this_frame, + gf_stats.this_frame_mv_in_out, GF_MAX_BOOST, + scale_max_boost); if (num_fpstats_used) (*num_fpstats_used)++; } arf_boost += (int)boost_score; @@ -1032,9 +1049,10 @@ // If almost totally static, we will not use the the max GF length later, // so we can continue for more frames. - if (((frame_index - cur_start) >= active_max_gf_interval + 1) && - !is_almost_static(gf_stats->zero_motion_accumulator, - twopass->kf_zeromotion_pct, cpi->ppi->lap_enabled)) { + if ((frame_index - cur_start) >= active_max_gf_interval + 1 && + ((cpi->oxcf.mode != REALTIME) || + !is_almost_static(gf_stats->zero_motion_accumulator, + twopass->kf_zeromotion_pct, cpi->ppi->lap_enabled))) { return 1; } return 0; @@ -2170,10 +2188,12 @@ * case of one pass encoding where no lookahead stats are avialable. * * \param[in] cpi Top-level encoder structure + * \param[in] is_final_pass Whether it is the second call to + * define_gf_group(). * * \remark Nothing is returned. Instead, cpi->ppi->gf_group is changed. */ -static void define_gf_group_pass0(AV1_COMP *cpi) { +static void define_gf_group_pass0(AV1_COMP *cpi, const int is_final_pass) { RATE_CONTROL *const rc = &cpi->rc; PRIMARY_RATE_CONTROL *const p_rc = &cpi->ppi->p_rc; GF_GROUP *const gf_group = &cpi->ppi->gf_group; @@ -2181,13 +2201,9 @@ const GFConfig *const gf_cfg = &oxcf->gf_cfg; int target; - if (oxcf->q_cfg.aq_mode == CYCLIC_REFRESH_AQ) { - av1_cyclic_refresh_set_golden_update(cpi); - } else { - p_rc->baseline_gf_interval = p_rc->gf_intervals[p_rc->cur_gf_index]; - rc->intervals_till_gf_calculate_due--; - p_rc->cur_gf_index++; - } + p_rc->baseline_gf_interval = p_rc->gf_intervals[p_rc->cur_gf_index]; + rc->intervals_till_gf_calculate_due--; + p_rc->cur_gf_index++; // correct frames_to_key when lookahead queue is flushing correct_frames_to_key(cpi); @@ -2199,17 +2215,28 @@ p_rc->constrained_gf_group = (p_rc->baseline_gf_interval >= rc->frames_to_key) ? 1 : 0; - gf_group->max_layer_depth_allowed = oxcf->gf_cfg.gf_max_pyr_height; + // Default for pyr_height if inputs not set. + if (oxcf->gf_cfg.gf_max_pyr_height == 0 || + oxcf->gf_cfg.gf_min_pyr_height == 0) { + gf_group->max_layer_depth_allowed = 1; + } else { + gf_group->max_layer_depth_allowed = oxcf->gf_cfg.gf_max_pyr_height; + } // Rare case when the look-ahead is less than the target GOP length, can't // generate ARF frame. + // Also disable ARF frame if the motion content (source_sad) is high in + // lookahead buffer. + const uint64_t thresh_sad = 8 * 64 * 64; if (p_rc->baseline_gf_interval > gf_cfg->lag_in_frames || !is_altref_enabled(gf_cfg->lag_in_frames, gf_cfg->enable_auto_arf) || - p_rc->baseline_gf_interval < rc->min_gf_interval) + p_rc->baseline_gf_interval < rc->min_gf_interval || + rc->frame_source_sad_lag[0] > thresh_sad) { gf_group->max_layer_depth_allowed = 0; + } // Set up the structure of this Group-Of-Pictures (same as GF_GROUP) - av1_gop_setup_structure(cpi); + av1_gop_setup_structure(cpi, is_final_pass); // Allocate bits to each of the frames in the GF group. // TODO(sarahparker) Extend this to work with pyramid structure. @@ -2392,7 +2419,58 @@ const AV1EncoderConfig *const oxcf = &cpi->oxcf; const RateControlCfg *const rc_cfg = &oxcf->rc_cfg; + if (cpi->oxcf.mode != REALTIME) { + TWO_PASS_FRAME stats_in_backup = cpi->twopass_frame; + int gfu_boost_sum = 0; + int gfu_count = 0; + int accumulate_i = 0; + if (rc->frames_since_key == 0) { + for (int k = 0; k < MAX_NUM_GF_INTERVALS; k++) { + if (p_rc->gf_intervals[k] == 0) { + break; + } + + int new_i = p_rc->gf_intervals[k]; + int ext_len_new = new_i - (k == 0 ? is_intra_only : 0); + if (use_alt_ref) { + if (accumulate_i >= rc->frames_to_key) { + break; + } + const int forward_frames = + (rc->frames_to_key - accumulate_i - new_i >= ext_len_new) + ? ext_len_new + : AOMMAX(0, rc->frames_to_key - accumulate_i - new_i); + if (k) { + cpi->twopass_frame.stats_in += new_i; + if (cpi->twopass_frame.stats_in >= + twopass->stats_buf_ctx->stats_in_end) { + cpi->twopass_frame.stats_in = + twopass->stats_buf_ctx->stats_in_end; + } + } + reset_fpf_position(&cpi->twopass_frame, cpi->twopass_frame.stats_in); + // Calculate the boost for alt ref. Note that we pass the + // scale_max_boost=false to derive gfu_boost_average, which can help + // the coding efficiency for some clips with global motion. + int gfu_boost_tmp = av1_calc_arf_boost( + twopass, &cpi->twopass_frame, p_rc, frame_info, alt_offset, + forward_frames, ext_len_new, &p_rc->num_stats_used_for_gfu_boost, + &p_rc->num_stats_required_for_gfu_boost, cpi->ppi->lap_enabled, + /*scale_max_boost=*/false); + gfu_boost_sum += gfu_boost_tmp; + } + gfu_count++; + accumulate_i += new_i; + } + assert(gfu_count > 0); + p_rc->gfu_boost_average = gfu_boost_sum / gfu_count; + } + cpi->twopass_frame = stats_in_backup; + } + int ext_len = i - is_intra_only; + const bool scale_max_boost = (cpi->oxcf.mode != REALTIME); + if (use_alt_ref) { const int forward_frames = (rc->frames_to_key - i >= ext_len) ? ext_len @@ -2402,15 +2480,17 @@ p_rc->gfu_boost = av1_calc_arf_boost( twopass, &cpi->twopass_frame, p_rc, frame_info, alt_offset, forward_frames, ext_len, &p_rc->num_stats_used_for_gfu_boost, - &p_rc->num_stats_required_for_gfu_boost, cpi->ppi->lap_enabled); + &p_rc->num_stats_required_for_gfu_boost, cpi->ppi->lap_enabled, + scale_max_boost); } else { reset_fpf_position(&cpi->twopass_frame, start_pos); - p_rc->gfu_boost = AOMMIN( - MAX_GF_BOOST, - av1_calc_arf_boost( - twopass, &cpi->twopass_frame, p_rc, frame_info, alt_offset, ext_len, - 0, &p_rc->num_stats_used_for_gfu_boost, - &p_rc->num_stats_required_for_gfu_boost, cpi->ppi->lap_enabled)); + p_rc->gfu_boost = + AOMMIN(MAX_GF_BOOST, + av1_calc_arf_boost(twopass, &cpi->twopass_frame, p_rc, + frame_info, alt_offset, ext_len, 0, + &p_rc->num_stats_used_for_gfu_boost, + &p_rc->num_stats_required_for_gfu_boost, + cpi->ppi->lap_enabled, scale_max_boost)); } #define LAST_ALR_BOOST_FACTOR 0.2f @@ -2533,7 +2613,7 @@ } if (has_no_stats_stage(cpi)) { - define_gf_group_pass0(cpi); + define_gf_group_pass0(cpi, is_final_pass); return; } @@ -2578,10 +2658,11 @@ int use_alt_ref; if (can_disable_arf) { use_alt_ref = - !is_almost_static(gf_stats.zero_motion_accumulator, - twopass->kf_zeromotion_pct, cpi->ppi->lap_enabled) && p_rc->use_arf_in_this_kf_group && (i < gf_cfg->lag_in_frames) && - (i >= MIN_GF_INTERVAL); + (i >= MIN_GF_INTERVAL) && + ((cpi->oxcf.mode != REALTIME) || + !is_almost_static(gf_stats.zero_motion_accumulator, + twopass->kf_zeromotion_pct, cpi->ppi->lap_enabled)); } else { use_alt_ref = p_rc->use_arf_in_this_kf_group && (i < gf_cfg->lag_in_frames) && (i > 2); @@ -2634,7 +2715,7 @@ update_gop_length(rc, p_rc, i, is_final_pass); // Set up the structure of this Group-Of-Pictures (same as GF_GROUP) - av1_gop_setup_structure(cpi); + av1_gop_setup_structure(cpi, is_final_pass); set_gop_bits_boost(cpi, i, is_intra_only, is_final_pass, use_alt_ref, alt_offset, start_pos, &gf_stats); @@ -2710,7 +2791,7 @@ update_gop_length(rc, p_rc, i, is_final_pass); // Set up the structure of this Group-Of-Pictures (same as GF_GROUP) - av1_gop_setup_structure(cpi); + av1_gop_setup_structure(cpi, is_final_pass); set_gop_bits_boost(cpi, i, is_intra_only, is_final_pass, use_alt_ref, 0, start_pos, &gf_stats); @@ -2976,6 +3057,7 @@ : cpi->common.mi_params.MBs; const int future_stats_count = av1_firstpass_info_future_count(firstpass_info, 0); + while (frames_to_key < future_stats_count && frames_to_key < num_frames_to_detect_scenecut) { // Provided that we are not at the end of the file... @@ -2990,7 +3072,21 @@ oxcf->rc_cfg.mode, cpi->ppi->p_rc.enable_scenecut_detection, num_mbs); if (scenecut_detected) { - break; + int test_next_gop = 0; + + for (int idx = 0; idx < 32; ++idx) { + const FIRSTPASS_STATS *next_stats = + av1_firstpass_info_peek(firstpass_info, frames_to_key + idx); + + if (next_stats == NULL) continue; + + if (cpi->common.current_frame.frame_number + frames_to_key + idx > + 2 && + next_stats->lt_coded_error * 2.5 < next_stats->coded_error) + test_next_gop = 1; + } + + if (!test_next_gop) break; } } @@ -3260,15 +3356,30 @@ kf_raw_err = this_frame->intra_error; kf_mod_err = calculate_modified_err(frame_info, twopass, oxcf, this_frame); - // We assume the current frame is a key frame and we are looking for the next - // key frame. Therefore search_start_idx = 1 - frames_to_key = define_kf_interval(cpi, firstpass_info, kf_cfg->key_freq_max, - /*search_start_idx=*/1); - - if (frames_to_key != -1) { - rc->frames_to_key = AOMMIN(kf_cfg->key_freq_max, frames_to_key); + if (cpi->ext_ratectrl.ready && + (cpi->ext_ratectrl.funcs.rc_type & AOM_RC_GOP) != 0 && + cpi->ext_ratectrl.funcs.get_key_frame_decision != NULL) { + aom_rc_key_frame_decision_t key_frame_decision; + aom_codec_err_t codec_status = av1_extrc_get_key_frame_decision( + &cpi->ext_ratectrl, &key_frame_decision); + if (codec_status == AOM_CODEC_OK) { + rc->frames_to_key = key_frame_decision.key_frame_group_size; + } else { + aom_internal_error(cpi->common.error, codec_status, + "av1_extrc_get_key_frame_decision() failed"); + } } else { - rc->frames_to_key = kf_cfg->key_freq_max; + // We assume the current frame is a key frame and we are looking for the + // next key frame. Therefore search_start_idx = 1 + frames_to_key = + define_kf_interval(cpi, firstpass_info, kf_cfg->key_freq_max, + /*search_start_idx=*/1); + + if (frames_to_key != -1) { + rc->frames_to_key = AOMMIN(kf_cfg->key_freq_max, frames_to_key); + } else { + rc->frames_to_key = kf_cfg->key_freq_max; + } } if (cpi->ppi->lap_enabled) correct_frames_to_key(cpi); @@ -3713,6 +3824,105 @@ first_stats->cor_coeff = 1.0; } +static void get_one_pass_rt_lag_params(AV1_COMP *cpi, unsigned int frame_flags, + EncodeFrameParams *const frame_params) { + RATE_CONTROL *const rc = &cpi->rc; + PRIMARY_RATE_CONTROL *const p_rc = &cpi->ppi->p_rc; + GF_GROUP *const gf_group = &cpi->ppi->gf_group; + const AV1EncoderConfig *const oxcf = &cpi->oxcf; + // Check forced key frames. + const int frames_to_next_forced_key = detect_app_forced_key(cpi); + if (frames_to_next_forced_key == 0) { + rc->frames_to_key = 0; + frame_flags &= FRAMEFLAGS_KEY; + } else if (frames_to_next_forced_key > 0 && + frames_to_next_forced_key < rc->frames_to_key) { + rc->frames_to_key = frames_to_next_forced_key; + } + frame_params->frame_type = gf_group->frame_type[cpi->gf_frame_index]; + if (cpi->gf_frame_index < gf_group->size && !(frame_flags & FRAMEFLAGS_KEY)) { + av1_setup_target_rate(cpi); + } + // Keyframe processing. + if (rc->frames_to_key <= 0) { + const KeyFrameCfg *const kf_cfg = &oxcf->kf_cfg; + assert(rc->frames_to_key == 0); + // Define next KF group. + frame_params->frame_type = KEY_FRAME; + rc->frames_since_key = 0; + // Use arfs if possible. + p_rc->use_arf_in_this_kf_group = is_altref_enabled( + oxcf->gf_cfg.lag_in_frames, oxcf->gf_cfg.enable_auto_arf); + // Reset the GF group data structures. + av1_zero(*gf_group); + cpi->gf_frame_index = 0; + // KF is always a GF so clear frames till next gf counter. + rc->frames_till_gf_update_due = 0; + int num_frames_to_app_forced_key = detect_app_forced_key(cpi); + p_rc->this_key_frame_forced = + cpi->common.current_frame.frame_number != 0 && rc->frames_to_key == 0; + if (num_frames_to_app_forced_key != -1) + rc->frames_to_key = num_frames_to_app_forced_key; + else + rc->frames_to_key = AOMMAX(1, kf_cfg->key_freq_max); + correct_frames_to_key(cpi); + p_rc->kf_boost = DEFAULT_KF_BOOST; + gf_group->update_type[0] = KF_UPDATE; + } + // Define a new GF/ARF group. (Should always enter here for key frames). + if (cpi->gf_frame_index == gf_group->size) { + int max_gop_length = + (oxcf->gf_cfg.lag_in_frames >= 32) + ? AOMMIN(MAX_GF_INTERVAL, oxcf->gf_cfg.lag_in_frames - + oxcf->algo_cfg.arnr_max_frames / 2) + : MAX_GF_LENGTH_LAP; + // Limit the max gop length for the last gop in 1 pass setting. + max_gop_length = AOMMIN(max_gop_length, rc->frames_to_key); + // Go through source frames in lookahead buffer and compute source metrics: + // scene change, frame average source sad, etc. + int num_frames = 1; + int scene_change_gop_frame_index = 0; + rc->frame_source_sad_lag[0] = 0; + rc->avg_source_sad = 0; + for (int i = 1; i < max_gop_length; i++) { + EncodeFrameInput frame_input; + memset(&frame_input, 0, sizeof(frame_input)); + struct lookahead_entry *e = + av1_lookahead_peek(cpi->ppi->lookahead, i, cpi->compressor_stage); + struct lookahead_entry *e_prev = + av1_lookahead_peek(cpi->ppi->lookahead, i - 1, cpi->compressor_stage); + if (e != NULL && e_prev != NULL) { + frame_input.source = &e->img; + frame_input.last_source = &e_prev->img; + rc->high_source_sad_lag[i] = -1; + rc->frame_source_sad_lag[i] = 0; + av1_rc_scene_detection_onepass_rt(cpi, &frame_input); + rc->high_source_sad_lag[i] = rc->high_source_sad; + rc->frame_source_sad_lag[i] = rc->frame_source_sad; + if (rc->high_source_sad_lag[i] == 1 && i > 1) { + // Scene change, so exit and constrain the gop to this frame. + scene_change_gop_frame_index = i; + break; + } + num_frames++; + rc->frame_source_sad_lag[0] += rc->frame_source_sad_lag[i]; + } + } + if (scene_change_gop_frame_index > 0) + max_gop_length = AOMMIN(max_gop_length, scene_change_gop_frame_index); + rc->frame_source_sad_lag[0] = rc->frame_source_sad_lag[0] / num_frames; + calculate_gf_length(cpi, max_gop_length, MAX_NUM_GF_INTERVALS); + define_gf_group(cpi, frame_params, 0); + rc->frames_till_gf_update_due = p_rc->baseline_gf_interval; + frame_params->frame_type = gf_group->frame_type[cpi->gf_frame_index]; + av1_setup_target_rate(cpi); + // Reset the source_sad parameters for the encoding. + rc->high_source_sad = 0; + rc->frame_source_sad = UINT64_MAX; + rc->avg_source_sad = 0; + } +} + void av1_get_second_pass_params(AV1_COMP *cpi, EncodeFrameParams *const frame_params, unsigned int frame_flags) { @@ -3722,6 +3932,19 @@ GF_GROUP *const gf_group = &cpi->ppi->gf_group; const AV1EncoderConfig *const oxcf = &cpi->oxcf; + if (is_one_pass_rt_lag_params(cpi)) { + get_one_pass_rt_lag_params(cpi, frame_flags, frame_params); + return; + } + + if (cpi->ext_ratectrl.ready && + (cpi->ext_ratectrl.funcs.rc_type & AOM_RC_GOP) != 0 && + cpi->ext_ratectrl.funcs.get_gop_decision != NULL) { + frame_params->show_frame = + !(gf_group->update_type[cpi->gf_frame_index] == ARF_UPDATE || + gf_group->update_type[cpi->gf_frame_index] == INTNL_ARF_UPDATE); + } + if (cpi->use_ducky_encode && cpi->ducky_encode_info.frame_info.gop_mode == DUCKY_ENCODE_GOP_MODE_RCL) { frame_params->frame_type = gf_group->frame_type[cpi->gf_frame_index]; @@ -3735,6 +3958,16 @@ return; } + if (cpi->common.current_frame.frame_number == 0 && + cpi->ext_ratectrl.funcs.send_firstpass_stats != NULL) { + const aom_codec_err_t codec_status = av1_extrc_send_firstpass_stats( + &cpi->ext_ratectrl, &cpi->ppi->twopass.firstpass_info); + if (codec_status != AOM_CODEC_OK) { + aom_internal_error(cpi->common.error, codec_status, + "av1_extrc_send_firstpass_stats() failed"); + } + } + const FIRSTPASS_STATS *const start_pos = cpi->twopass_frame.stats_in; int update_total_stats = 0; @@ -3805,6 +4038,8 @@ frame_params->frame_type = KEY_FRAME; find_next_key_frame(cpi, &this_frame); this_frame = this_frame_copy; + // Mark prev gop arf source as unusable + cpi->ppi->tpl_data.prev_gop_arf_disp_order = -1; } if (rc->frames_to_fwd_kf <= 0) @@ -3924,7 +4159,6 @@ if (rc->intervals_till_gf_calculate_due == 0 || 1) { calculate_gf_length(cpi, max_gop_length, MAX_NUM_GF_INTERVALS); } - if (max_gop_length > 16 && oxcf->algo_cfg.enable_tpl_model && oxcf->gf_cfg.lag_in_frames >= 32 && cpi->sf.tpl_sf.gop_length_decision_method != 3) { @@ -4274,7 +4508,7 @@ int maxq_adj_limit; minq_adj_limit = (rc_cfg->mode == AOM_CQ ? MINQ_ADJ_LIMIT_CQ : MINQ_ADJ_LIMIT); - maxq_adj_limit = (rc->worst_quality - rc->active_worst_quality); + maxq_adj_limit = rc->worst_quality - rc->active_worst_quality; // Undershoot if ((rc_cfg->under_shoot_pct < 100) && @@ -4286,9 +4520,8 @@ if ((pct_error >= rc_cfg->under_shoot_pct) && (p_rc->rate_error_estimate > 0)) { twopass->extend_minq += 1; - twopass->extend_maxq -= 1; } - + twopass->extend_maxq -= 1; // Overshoot } else if ((rc_cfg->over_shoot_pct < 100) && (p_rc->rolling_actual_bits > p_rc->rolling_target_bits)) { @@ -4300,8 +4533,18 @@ if ((pct_error >= rc_cfg->over_shoot_pct) && (p_rc->rate_error_estimate < 0)) { twopass->extend_maxq += 1; - twopass->extend_minq -= 1; } + twopass->extend_minq -= 1; + } else { + // Adjustment for extreme local overshoot. + // Only applies when normal adjustment above is not used (e.g. + // when threshold is set to 100). + if (rc->projected_frame_size > (2 * rc->base_frame_target) && + rc->projected_frame_size > (2 * rc->avg_frame_bandwidth)) + ++twopass->extend_maxq; + // Unwind extreme overshoot adjustment. + else if (p_rc->rolling_target_bits > p_rc->rolling_actual_bits) + --twopass->extend_maxq; } twopass->extend_minq = clamp(twopass->extend_minq, -minq_adj_limit, minq_adj_limit);
diff --git a/av1/encoder/pass2_strategy.h b/av1/encoder/pass2_strategy.h index 1ca39ec..be14d8e 100644 --- a/av1/encoder/pass2_strategy.h +++ b/av1/encoder/pass2_strategy.h
@@ -109,7 +109,7 @@ const PRIMARY_RATE_CONTROL *p_rc, FRAME_INFO *frame_info, int offset, int f_frames, int b_frames, int *num_fpstats_used, int *num_fpstats_required, - int project_gfu_boost); + int project_gfu_boost, const bool scale_max_boost); void av1_gop_bit_allocation(const AV1_COMP *cpi, RATE_CONTROL *const rc, GF_GROUP *gf_group, int is_key_frame, int use_arf,
diff --git a/av1/encoder/pickcdef.c b/av1/encoder/pickcdef.c index 27aea1d..d498c06 100644 --- a/av1/encoder/pickcdef.c +++ b/av1/encoder/pickcdef.c
@@ -730,7 +730,7 @@ } void av1_pick_cdef_from_qp(AV1_COMMON *const cm, int skip_cdef, - int is_screen_content) { + int is_screen_content, bool avoid_uv_cdef) { const int bd = cm->seq_params->bit_depth; const int q = av1_ac_quant_QTX(cm->quant_params.base_qindex, 0, bd) >> (bd - 8); @@ -794,7 +794,8 @@ cdef_info->cdef_strengths[0] = predicted_y_f1 * CDEF_SEC_STRENGTHS + predicted_y_f2; cdef_info->cdef_uv_strengths[0] = - predicted_uv_f1 * CDEF_SEC_STRENGTHS + predicted_uv_f2; + avoid_uv_cdef ? 0 + : predicted_uv_f1 * CDEF_SEC_STRENGTHS + predicted_uv_f2; // mbmi->cdef_strength is already set in the encoding stage. We don't need to // set it again here. @@ -822,15 +823,16 @@ void av1_cdef_search(AV1_COMP *cpi) { AV1_COMMON *cm = &cpi->common; CDEF_CONTROL cdef_control = cpi->oxcf.tool_cfg.cdef_control; + const bool apply_adaptive_cdef = + cdef_control == CDEF_ADAPTIVE && + (cpi->oxcf.rc_cfg.mode == AOM_Q || cpi->oxcf.rc_cfg.mode == AOM_CQ); assert(cdef_control != CDEF_NONE); // For CDEF_ADAPTIVE, turning off CDEF around qindex 32 was best for still // pictures if ((cdef_control == CDEF_REFERENCE && cpi->ppi->rtc_ref.non_reference_frame) || - (cdef_control == CDEF_ADAPTIVE && cpi->oxcf.mode == ALLINTRA && - (cpi->oxcf.rc_cfg.mode == AOM_Q || cpi->oxcf.rc_cfg.mode == AOM_CQ) && - cpi->oxcf.rc_cfg.cq_level <= 32)) { + (apply_adaptive_cdef && cpi->oxcf.rc_cfg.cq_level <= 32)) { CdefInfo *const cdef_info = &cm->cdef_info; cdef_info->nb_cdef_strengths = 1; cdef_info->cdef_bits = 0; @@ -842,7 +844,8 @@ // Indicate if external RC is used for testing const int rtc_ext_rc = cpi->rc.rtc_external_ratectrl; if (rtc_ext_rc) { - av1_pick_cdef_from_qp(cm, 0, 0); + av1_pick_cdef_from_qp(cm, /*skip_cdef=*/0, /*is_screen_content=*/0, + /*avoid_uv_cdef=*/false); return; } CDEF_PICK_METHOD pick_method = cpi->sf.lpf_sf.cdef_pick_method; @@ -852,8 +855,14 @@ AOMMAX(cpi->sf.rt_sf.screen_content_cdef_filter_qindex_thresh, cpi->rc.best_quality + 5) && cpi->oxcf.tune_cfg.content == AOM_CONTENT_SCREEN; - av1_pick_cdef_from_qp(cm, cpi->sf.rt_sf.skip_cdef_sb, - use_screen_content_model); + + // For adaptive CDEF, do not apply CDEF to chroma channels. + // This is done to reduce decode time, as CDEF is a relatively-expensive + // filter to compute. + const bool avoid_uv_cdef = apply_adaptive_cdef; + + av1_pick_cdef_from_qp(cm, cpi->sf.rt_sf.skip_cdef_sb != 0, + use_screen_content_model, avoid_uv_cdef); return; } const CommonModeInfoParams *const mi_params = &cm->mi_params; @@ -896,7 +905,25 @@ const int max_signaling_bits = joint_strengths == 1 ? 0 : get_msb(joint_strengths - 1) + 1; int rdmult = cpi->td.mb.rdmult; - for (int i = 0; i <= 3; i++) { + + // For adaptive CDEF, reduce primary and secondary CDEF strengths for + // qindexes up to 220. + const bool should_reduce_cdef_strengths = + apply_adaptive_cdef && cpi->oxcf.rc_cfg.cq_level <= 220; + // For adaptive CDEF with strength reduction, zero out CDEF strengths with + // low values (luma and/or chroma). This is done to reduce decode time, as + // CDEF is a relatively-expensive filter to compute. + const bool should_zero_cdef_strengths = + should_reduce_cdef_strengths && cpi->sf.lpf_sf.zero_low_cdef_strengths; + // If running adaptive CDEF with strength zeroing, let search derive at least + // two CDEF strengths (i.e. at least 1 CDEF signaling bit), unless search was + // explicitly set to search for 1 strength only (i.e. 0 CDEF signaling bits). + // Doing so will help find opportunities to zero out low strengths to reduce + // overall decode time. + const int min_signaling_bits = + (should_zero_cdef_strengths && max_signaling_bits > 0) ? 1 : 0; + + for (int i = min_signaling_bits; i <= 3; i++) { if (i > max_signaling_bits) break; int best_lev0[CDEF_MAX_STRENGTHS] = { 0 }; int best_lev1[CDEF_MAX_STRENGTHS] = { 0 }; @@ -946,18 +973,19 @@ if (fast) { for (int j = 0; j < cdef_info->nb_cdef_strengths; j++) { const int luma_strength = cdef_info->cdef_strengths[j]; - const int chroma_strength = cdef_info->cdef_uv_strengths[j]; int pri_strength, sec_strength; STORE_CDEF_FILTER_STRENGTH(cdef_info->cdef_strengths[j], pick_method, luma_strength); - STORE_CDEF_FILTER_STRENGTH(cdef_info->cdef_uv_strengths[j], pick_method, - chroma_strength); + if (num_planes > 1) { + const int chroma_strength = cdef_info->cdef_uv_strengths[j]; + STORE_CDEF_FILTER_STRENGTH(cdef_info->cdef_uv_strengths[j], pick_method, + chroma_strength); + } } } - // For CDEF_ADAPTIVE, set primary and secondary CDEF at reduced strength for - // qindexes 33 through 220. + // Perform CDEF strength reduction. // Note 1: for odd strengths, the 0.5 discarded by ">> 1" is a significant // part of the strength when the strength is small, and because there are // few strength levels, odd strengths are reduced significantly more than a @@ -970,27 +998,54 @@ // resulting SSIMULACRA 2 scores were either exactly the same (at cpu-used 6), // or within noise level (at cpu-used 3). Given that there were no discernible // improvements, this special mapping was left out for reduced strength. - if (cdef_control == CDEF_ADAPTIVE && cpi->oxcf.mode == ALLINTRA && - (cpi->oxcf.rc_cfg.mode == AOM_Q || cpi->oxcf.rc_cfg.mode == AOM_CQ) && - cpi->oxcf.rc_cfg.cq_level <= 220) { + if (should_reduce_cdef_strengths) { for (int j = 0; j < cdef_info->nb_cdef_strengths; j++) { const int luma_strength = cdef_info->cdef_strengths[j]; - const int chroma_strength = cdef_info->cdef_uv_strengths[j]; - const int new_pri_luma_strength = (luma_strength / CDEF_SEC_STRENGTHS) >> 1; const int new_sec_luma_strength = (luma_strength % CDEF_SEC_STRENGTHS) >> 1; - const int new_pri_chroma_strength = - (chroma_strength / CDEF_SEC_STRENGTHS) >> 1; - const int new_sec_chroma_strength = - (chroma_strength % CDEF_SEC_STRENGTHS) >> 1; cdef_info->cdef_strengths[j] = new_pri_luma_strength * CDEF_SEC_STRENGTHS + new_sec_luma_strength; - cdef_info->cdef_uv_strengths[j] = - new_pri_chroma_strength * CDEF_SEC_STRENGTHS + - new_sec_chroma_strength; + + int new_pri_chroma_strength; + int new_sec_chroma_strength; + + if (num_planes > 1) { + const int chroma_strength = cdef_info->cdef_uv_strengths[j]; + new_pri_chroma_strength = (chroma_strength / CDEF_SEC_STRENGTHS) >> 1; + new_sec_chroma_strength = (chroma_strength % CDEF_SEC_STRENGTHS) >> 1; + + cdef_info->cdef_uv_strengths[j] = + new_pri_chroma_strength * CDEF_SEC_STRENGTHS + + new_sec_chroma_strength; + } + + // Zero out entries with low CDEF luma (and optional chroma) strengths. + // The low-strength thresholds were empirically derived from subjective + // testing and SSIMULACRA 2 scores. These strike a balance between + // perceptual quality gains and a reasonable single-threaded decode time + // increase (~10%) over --enable-cdef 0. There's an overall 0.18 point + // loss in SSIMULACRA 2 scores over no CDEF strength zeroing at speed 6, + // QP 30 on the CLIC 2020 dataset. + if (should_zero_cdef_strengths) { + const bool is_low_luma_strength = + new_pri_luma_strength <= 4 && new_sec_luma_strength <= 1; + + if (is_low_luma_strength) { + cdef_info->cdef_strengths[j] = 0; + } + if (num_planes > 1) { + const bool is_low_chroma_strength = + new_pri_chroma_strength <= 4 && new_sec_chroma_strength <= 1; + + if (is_low_luma_strength || is_low_chroma_strength) { + // Disable CDEF on chroma if we've disabled it on luma + cdef_info->cdef_uv_strengths[j] = 0; + } + } + } } }
diff --git a/av1/encoder/pickcdef.h b/av1/encoder/pickcdef.h index 6056ad9..70fc0e2 100644 --- a/av1/encoder/pickcdef.h +++ b/av1/encoder/pickcdef.h
@@ -251,10 +251,11 @@ * \param[in,out] cm Pointer to top level common structure * \param[in] skip_cdef Flag to skip CDEF filtering * \param[in] is_screen_content Flag indicating screen content + * \param[in] avoid_uv_cdef Flag to avoid assigning UV CDEF strengths * */ void av1_pick_cdef_from_qp(AV1_COMMON *const cm, int skip_cdef, - int is_screen_content); + int is_screen_content, bool avoid_uv_cdef); #ifdef __cplusplus } // extern "C"
diff --git a/av1/encoder/picklpf.c b/av1/encoder/picklpf.c index bc776d7..d7333aa 100644 --- a/av1/encoder/picklpf.c +++ b/av1/encoder/picklpf.c
@@ -217,10 +217,17 @@ int disable_filter_rt_screen = 0; (void)sd; - // Enable loop filter sharpness only for allintra encoding mode, - // as frames do not have to serve as references to others - lf->sharpness_level = - cpi->oxcf.mode == ALLINTRA ? cpi->oxcf.algo_cfg.sharpness : 0; + // Enable loop filter sharpness only for all-intra encoding mode, + // or tune IQ or SSIMULACRA2. This is because: + // - All-intra: frames do not have to serve as references to others + // - Tune IQ/SSIMULACRA2: enabling loop filter sharpness has been found to + // be beneficial for sharpness perception + if (cpi->oxcf.mode == ALLINTRA || cpi->oxcf.tune_cfg.tuning == AOM_TUNE_IQ || + cpi->oxcf.tune_cfg.tuning == AOM_TUNE_SSIMULACRA2) { + lf->sharpness_level = cpi->oxcf.algo_cfg.sharpness; + } else { + lf->sharpness_level = 0; + } if (cpi->oxcf.algo_cfg.enable_adaptive_sharpness) { // Loop filter sharpness levels are highly nonlinear. Visually, lf sharpness @@ -228,7 +235,7 @@ // written to pick levels 0, 1 and 7 to keep it simple. int max_lf_sharpness; - if (cm->quant_params.base_qindex <= 120) { + if (cm->quant_params.base_qindex <= 112) { max_lf_sharpness = 7; } else if (cm->quant_params.base_qindex <= 160) { max_lf_sharpness = 1; @@ -409,9 +416,7 @@ unsigned int pyramid_level = cm->current_frame.pyramid_level; if (pyramid_level > 1) { int filter_threshold; - if (pyramid_level >= 5) - filter_threshold = 32; - else if (pyramid_level >= 4) + if (pyramid_level >= 4) filter_threshold = 16; else filter_threshold = 8;
diff --git a/av1/encoder/pickrst.c b/av1/encoder/pickrst.c index 3913d58..cd79ed9 100644 --- a/av1/encoder/pickrst.c +++ b/av1/encoder/pickrst.c
@@ -848,25 +848,27 @@ flt0, flt1, flt_stride, exqd, &err, error_info); get_best_error(&besterr, err, exqd, bestxqd, &bestep, ep); } - // evaluate left and right ep of winner in seed ep - int bestep_ref = bestep; - for (ep = bestep_ref - 1; ep < bestep_ref + 2; ep += 2) { - if (ep < SGRPROJ_EP_GRP1_START_IDX || ep > SGRPROJ_EP_GRP1_END_IDX) - continue; - int64_t err; - compute_sgrproj_err(dat8, width, height, dat_stride, src8, src_stride, - use_highbitdepth, bit_depth, pu_width, pu_height, ep, - flt0, flt1, flt_stride, exqd, &err, error_info); - get_best_error(&besterr, err, exqd, bestxqd, &bestep, ep); - } - // evaluate last two group - for (idx = 0; idx < SGRPROJ_EP_GRP2_3_SEARCH_COUNT; idx++) { - ep = sgproj_ep_grp2_3[idx][bestep]; - int64_t err; - compute_sgrproj_err(dat8, width, height, dat_stride, src8, src_stride, - use_highbitdepth, bit_depth, pu_width, pu_height, ep, - flt0, flt1, flt_stride, exqd, &err, error_info); - get_best_error(&besterr, err, exqd, bestxqd, &bestep, ep); + if (enable_sgr_ep_pruning < 2) { + // evaluate left and right ep of winner in seed ep + int bestep_ref = bestep; + for (ep = bestep_ref - 1; ep < bestep_ref + 2; ep += 2) { + if (ep < SGRPROJ_EP_GRP1_START_IDX || ep > SGRPROJ_EP_GRP1_END_IDX) + continue; + int64_t err; + compute_sgrproj_err(dat8, width, height, dat_stride, src8, src_stride, + use_highbitdepth, bit_depth, pu_width, pu_height, + ep, flt0, flt1, flt_stride, exqd, &err, error_info); + get_best_error(&besterr, err, exqd, bestxqd, &bestep, ep); + } + // evaluate last two group + for (idx = 0; idx < SGRPROJ_EP_GRP2_3_SEARCH_COUNT; idx++) { + ep = sgproj_ep_grp2_3[idx][bestep]; + int64_t err; + compute_sgrproj_err(dat8, width, height, dat_stride, src8, src_stride, + use_highbitdepth, bit_depth, pu_width, pu_height, + ep, flt0, flt1, flt_stride, exqd, &err, error_info); + get_best_error(&besterr, err, exqd, bestxqd, &bestep, ep); + } } }
diff --git a/av1/encoder/ratectrl.c b/av1/encoder/ratectrl.c index 1b55fe2..491e7a5 100644 --- a/av1/encoder/ratectrl.c +++ b/av1/encoder/ratectrl.c
@@ -41,6 +41,9 @@ #define USE_UNRESTRICTED_Q_IN_CQ_MODE 0 +#define RES_NUM 2 +#define MODE_NUM 2 + // Max rate target for 1080P and below encodes under normal circumstances // (1920 * 1080 / (16 * 16)) * MAX_MB_RATE bits per MB #define MAX_MB_RATE 250 @@ -68,35 +71,53 @@ } \ } while (0) +#define ASSIGN_MINQ_TABLE_2(bit_depth, name, res_idx, mode_idx) \ + do { \ + switch (bit_depth) { \ + case AOM_BITS_8: name = name##_8[mode_idx][res_idx]; break; \ + case AOM_BITS_10: name = name##_10[mode_idx][res_idx]; break; \ + case AOM_BITS_12: name = name##_12[mode_idx][res_idx]; break; \ + default: \ + assert(0 && \ + "bit_depth should be AOM_BITS_8, AOM_BITS_10" \ + " or AOM_BITS_12"); \ + name = NULL; \ + } \ + } while (0) + // Tables relating active max Q to active min Q -static int kf_low_motion_minq_8[QINDEX_RANGE]; -static int kf_high_motion_minq_8[QINDEX_RANGE]; -static int arfgf_low_motion_minq_8[QINDEX_RANGE]; -static int arfgf_high_motion_minq_8[QINDEX_RANGE]; -static int inter_minq_8[QINDEX_RANGE]; +static int kf_low_motion_minq_8[MODE_NUM][RES_NUM][QINDEX_RANGE]; +static int kf_high_motion_minq_8[MODE_NUM][RES_NUM][QINDEX_RANGE]; +static int arfgf_low_motion_minq_8[MODE_NUM][RES_NUM][QINDEX_RANGE]; +static int arfgf_high_motion_minq_8[MODE_NUM][RES_NUM][QINDEX_RANGE]; +static int inter_minq_8[MODE_NUM][RES_NUM][QINDEX_RANGE]; static int rtc_minq_8[QINDEX_RANGE]; -static int kf_low_motion_minq_10[QINDEX_RANGE]; -static int kf_high_motion_minq_10[QINDEX_RANGE]; -static int arfgf_low_motion_minq_10[QINDEX_RANGE]; -static int arfgf_high_motion_minq_10[QINDEX_RANGE]; -static int inter_minq_10[QINDEX_RANGE]; +static int kf_low_motion_minq_10[MODE_NUM][RES_NUM][QINDEX_RANGE]; +static int kf_high_motion_minq_10[MODE_NUM][RES_NUM][QINDEX_RANGE]; +static int arfgf_low_motion_minq_10[MODE_NUM][RES_NUM][QINDEX_RANGE]; +static int arfgf_high_motion_minq_10[MODE_NUM][RES_NUM][QINDEX_RANGE]; +static int inter_minq_10[MODE_NUM][RES_NUM][QINDEX_RANGE]; static int rtc_minq_10[QINDEX_RANGE]; -static int kf_low_motion_minq_12[QINDEX_RANGE]; -static int kf_high_motion_minq_12[QINDEX_RANGE]; -static int arfgf_low_motion_minq_12[QINDEX_RANGE]; -static int arfgf_high_motion_minq_12[QINDEX_RANGE]; -static int inter_minq_12[QINDEX_RANGE]; +static int kf_low_motion_minq_12[MODE_NUM][RES_NUM][QINDEX_RANGE]; +static int kf_high_motion_minq_12[MODE_NUM][RES_NUM][QINDEX_RANGE]; +static int arfgf_low_motion_minq_12[MODE_NUM][RES_NUM][QINDEX_RANGE]; +static int arfgf_high_motion_minq_12[MODE_NUM][RES_NUM][QINDEX_RANGE]; +static int inter_minq_12[MODE_NUM][RES_NUM][QINDEX_RANGE]; static int rtc_minq_12[QINDEX_RANGE]; -static int gf_high = 2400; -static int gf_low = 300; -#ifdef STRICT_RC -static int kf_high = 3200; -#else -static int kf_high = 5000; -#endif -static int kf_low = 400; +static int gf_high_1 = 2875; +static int gf_low_1 = 562; +static int gf_high_2 = 4994; +static int gf_low_2 = 100; + +static int kf_high = 8000; +static int kf_low = 553; + +static int gf_high_rtc = 2400; +static int gf_low_rtc = 300; +static int kf_high_rtc = 5000; +static int kf_low_rtc = 400; // How many times less pixels there are to encode given the current scaling. // Temporary replacement for rcf_mult and rate_thresh_mult. @@ -120,18 +141,41 @@ return av1_find_qindex(minqtarget, bit_depth, 0, QINDEX_RANGE - 1); } -static void init_minq_luts(int *kf_low_m, int *kf_high_m, int *arfgf_low, - int *arfgf_high, int *inter, int *rtc, +static double x1[MODE_NUM][RES_NUM][5] = { + { + { 0.1771, 0.379, 0.3279, 0.6634, 1.385 }, + { 0.1917, 0.3760, 0.34570, 0.6916, 1.14820 }, + }, + { + { 0.15, 0.45, 0.30, 0.55, 0.90 }, + { 0.15, 0.45, 0.30, 0.55, 0.90 }, + }, +}; + +static void init_minq_luts(int kf_low_m[MODE_NUM][RES_NUM][QINDEX_RANGE], + int kf_high_m[MODE_NUM][RES_NUM][QINDEX_RANGE], + int arfgf_low[MODE_NUM][RES_NUM][QINDEX_RANGE], + int arfgf_high[MODE_NUM][RES_NUM][QINDEX_RANGE], + int inter[MODE_NUM][RES_NUM][QINDEX_RANGE], int *rtc, aom_bit_depth_t bit_depth) { int i; - for (i = 0; i < QINDEX_RANGE; i++) { - const double maxq = av1_convert_qindex_to_q(i, bit_depth); - kf_low_m[i] = get_minq_index(maxq, 0.000001, -0.0004, 0.150, bit_depth); - kf_high_m[i] = get_minq_index(maxq, 0.0000021, -0.00125, 0.45, bit_depth); - arfgf_low[i] = get_minq_index(maxq, 0.0000015, -0.0009, 0.30, bit_depth); - arfgf_high[i] = get_minq_index(maxq, 0.0000021, -0.00125, 0.55, bit_depth); - inter[i] = get_minq_index(maxq, 0.00000271, -0.00113, 0.90, bit_depth); - rtc[i] = get_minq_index(maxq, 0.00000271, -0.00113, 0.70, bit_depth); + for (int mode = 0; mode < MODE_NUM; mode++) { + for (int res = 0; res < RES_NUM; res++) { + for (i = 0; i < QINDEX_RANGE; i++) { + const double maxq = av1_convert_qindex_to_q(i, bit_depth); + kf_low_m[mode][res][i] = get_minq_index(maxq, 0.000001, -0.0004, + x1[mode][res][0], bit_depth); + kf_high_m[mode][res][i] = get_minq_index(maxq, 0.0000021, -0.00125, + x1[mode][res][1], bit_depth); + arfgf_low[mode][res][i] = get_minq_index(maxq, 0.0000015, -0.0009, + x1[mode][res][2], bit_depth); + arfgf_high[mode][res][i] = get_minq_index(maxq, 0.0000021, -0.00125, + x1[mode][res][3], bit_depth); + inter[mode][res][i] = get_minq_index(maxq, 0.00000271, -0.00113, + x1[mode][res][4], bit_depth); + rtc[i] = get_minq_index(maxq, 0.00000271, -0.00113, 0.70, bit_depth); + } + } } } @@ -499,6 +543,7 @@ rc->force_max_q = 0; rc->postencode_drop = 0; rc->frames_since_scene_change = 0; + rc->last_frame_low_source_sad = 0; } static bool check_buffer_below_thresh(AV1_COMP *cpi, int64_t buffer_level, @@ -1122,34 +1167,58 @@ } } +static int gfboost_thresh[3] = { 4000, 4000, 3000 }; + static int get_kf_active_quality(const PRIMARY_RATE_CONTROL *const p_rc, int q, - aom_bit_depth_t bit_depth) { + aom_bit_depth_t bit_depth, const int res_idx, + const bool rtc_mode) { int *kf_low_motion_minq; int *kf_high_motion_minq; - ASSIGN_MINQ_TABLE(bit_depth, kf_low_motion_minq); - ASSIGN_MINQ_TABLE(bit_depth, kf_high_motion_minq); - return get_active_quality(q, p_rc->kf_boost, kf_low, kf_high, + ASSIGN_MINQ_TABLE_2(bit_depth, kf_low_motion_minq, res_idx > 1, rtc_mode); + ASSIGN_MINQ_TABLE_2(bit_depth, kf_high_motion_minq, res_idx > 1, rtc_mode); + + int kf_low_local = rtc_mode ? kf_low_rtc : kf_low; + int kf_high_local = rtc_mode ? kf_high_rtc : kf_high; + + return get_active_quality(q, p_rc->kf_boost, kf_low_local, kf_high_local, kf_low_motion_minq, kf_high_motion_minq); } -static int get_gf_active_quality_no_rc(int gfu_boost, int q, - aom_bit_depth_t bit_depth) { +static int get_gf_active_quality_no_rc(const PRIMARY_RATE_CONTROL *const p_rc, + int q, aom_bit_depth_t bit_depth, + const int res_idx, const bool rtc_mode) { int *arfgf_low_motion_minq; int *arfgf_high_motion_minq; - ASSIGN_MINQ_TABLE(bit_depth, arfgf_low_motion_minq); - ASSIGN_MINQ_TABLE(bit_depth, arfgf_high_motion_minq); - return get_active_quality(q, gfu_boost, gf_low, gf_high, + ASSIGN_MINQ_TABLE_2(bit_depth, arfgf_low_motion_minq, res_idx > 1, rtc_mode); + ASSIGN_MINQ_TABLE_2(bit_depth, arfgf_high_motion_minq, res_idx > 1, rtc_mode); + int gf_low_local, gf_high_local; + + if (!rtc_mode) { + gf_low_local = (p_rc->gfu_boost_average < gfboost_thresh[res_idx]) + ? gf_low_1 + : gf_low_2; + gf_high_local = (p_rc->gfu_boost_average < gfboost_thresh[res_idx]) + ? gf_high_1 + : gf_high_2; + } else { + gf_low_local = gf_low_rtc; + gf_high_local = gf_high_rtc; + } + + return get_active_quality(q, p_rc->gfu_boost, gf_low_local, gf_high_local, arfgf_low_motion_minq, arfgf_high_motion_minq); } static int get_gf_active_quality(const PRIMARY_RATE_CONTROL *const p_rc, int q, - aom_bit_depth_t bit_depth) { - return get_gf_active_quality_no_rc(p_rc->gfu_boost, q, bit_depth); + aom_bit_depth_t bit_depth, const int res_idx, + const bool rtc_mode) { + return get_gf_active_quality_no_rc(p_rc, q, bit_depth, res_idx, rtc_mode); } -static int get_gf_high_motion_quality(int q, aom_bit_depth_t bit_depth) { +static int get_gf_high_motion_quality(int q, aom_bit_depth_t bit_depth, + const int res_idx, const bool rtc_mode) { int *arfgf_high_motion_minq; - ASSIGN_MINQ_TABLE(bit_depth, arfgf_high_motion_minq); + ASSIGN_MINQ_TABLE_2(bit_depth, arfgf_high_motion_minq, res_idx > 1, rtc_mode); return arfgf_high_motion_minq[q]; } @@ -1209,7 +1278,26 @@ int adjustment = 0; int active_worst_quality; int ambient_qp; - if (frame_is_intra_only(cm)) return rc->worst_quality; + if (frame_is_intra_only(cm)) { + // Allow for active_worst_quality to go lower than rc->worst_quality (max) + // under certain conditions: that the frame spatial variance is below + // threshold and the buffer is full/stable. Also check the encoded vs target + // size for the last keyframe. + if (cpi->sf.rt_sf.rc_compute_spatial_var_sc_kf && + svc->number_spatial_layers == 1 && rc->frame_spatial_variance < 1000 && + p_rc->buffer_level > p_rc->optimal_buffer_level && + p_rc->optimal_buffer_level > (rc->avg_frame_bandwidth << 3) && + rc->last_encoded_size_keyframe < (rc->last_target_size_keyframe << 3)) { + if (p_rc->buffer_level > ((9 * p_rc->optimal_buffer_level) >> 3)) + ambient_qp = p_rc->avg_frame_qindex[INTER_FRAME]; + else + ambient_qp = + (rc->worst_quality + p_rc->avg_frame_qindex[INTER_FRAME]) >> 1; + return AOMMIN(rc->worst_quality, AOMMAX(ambient_qp, rc->best_quality)); + } else { + return rc->worst_quality; + } + } // For ambient_qp we use minimum of avg_frame_qindex[KEY_FRAME/INTER_FRAME] // for the first few frames following key frame. These are both initialized // to worst_quality and updated with (3/4, 1/4) average in postencode_update. @@ -1290,6 +1378,12 @@ int active_best_quality = rc->best_quality; ASSIGN_MINQ_TABLE(bit_depth, rtc_minq); + const int is_608p_or_larger = AOMMIN(cm->width, cm->height) >= 608; + const int is_480p_or_larger = AOMMIN(cm->width, cm->height) >= 480; + // res_idx is 0 for res < 480p, 1 for 480p, 2 for 608p+ + const int res_idx = is_480p_or_larger + is_608p_or_larger; + const bool rtc_mode = (cpi->oxcf.mode == REALTIME); + if (frame_is_intra_only(cm)) { // Handle the special case for key frames forced when we have reached // the maximum key frame interval. Here force the Q to a range @@ -1304,8 +1398,9 @@ // not first frame of one pass and kf_boost is set double q_adj_factor = 1.0; double q_val; - active_best_quality = get_kf_active_quality( - p_rc, p_rc->avg_frame_qindex[KEY_FRAME], bit_depth); + active_best_quality = + get_kf_active_quality(p_rc, p_rc->avg_frame_qindex[KEY_FRAME], + bit_depth, res_idx, rtc_mode); // Allow somewhat lower kf minq with small image formats. if ((width * height) <= (352 * 288)) { q_adj_factor -= 0.25; @@ -1327,7 +1422,8 @@ p_rc->avg_frame_qindex[INTER_FRAME] < active_worst_quality) { q = p_rc->avg_frame_qindex[INTER_FRAME]; } - active_best_quality = get_gf_active_quality(p_rc, q, bit_depth); + active_best_quality = + get_gf_active_quality(p_rc, q, bit_depth, res_idx, rtc_mode); } else { // Use the lower of active_worst_quality and recent/average Q. FRAME_TYPE frame_type = @@ -1499,6 +1595,12 @@ const RefreshFrameInfo *const refresh_frame = &cpi->refresh_frame; const enum aom_rc_mode rc_mode = oxcf->rc_cfg.mode; + const int is_608p_or_larger = AOMMIN(cm->width, cm->height) >= 608; + const int is_480p_or_larger = AOMMIN(cm->width, cm->height) >= 480; + // res_idx is 0 for res < 480p, 1 for 480p, 2 for 608p+ + const int res_idx = is_480p_or_larger + is_608p_or_larger; + const bool rtc_mode = (cpi->oxcf.mode == REALTIME); + assert(has_no_stats_stage(cpi)); assert(rc_mode == AOM_VBR || (!USE_UNRESTRICTED_Q_IN_CQ_MODE && rc_mode == AOM_CQ) || @@ -1513,7 +1615,7 @@ int active_worst_quality = calc_active_worst_quality_no_stats_vbr(cpi); int q; int *inter_minq; - ASSIGN_MINQ_TABLE(bit_depth, inter_minq); + ASSIGN_MINQ_TABLE_2(bit_depth, inter_minq, res_idx > 1, rtc_mode); if (frame_is_intra_only(cm)) { if (rc_mode == AOM_Q) { @@ -1539,8 +1641,9 @@ } else { // not first frame of one pass and kf_boost is set double q_adj_factor = 1.0; - active_best_quality = get_kf_active_quality( - p_rc, p_rc->avg_frame_qindex[KEY_FRAME], bit_depth); + active_best_quality = + get_kf_active_quality(p_rc, p_rc->avg_frame_qindex[KEY_FRAME], + bit_depth, res_idx, rtc_mode); // Allow somewhat lower kf minq with small image formats. if ((width * height) <= (352 * 288)) { @@ -1567,7 +1670,8 @@ // For constrained quality don't allow Q less than the cq level if (rc_mode == AOM_CQ) { if (q < cq_level) q = cq_level; - active_best_quality = get_gf_active_quality(p_rc, q, bit_depth); + active_best_quality = + get_gf_active_quality(p_rc, q, bit_depth, res_idx, rtc_mode); // Constrained quality use slightly lower active best. active_best_quality = active_best_quality * 15 / 16; } else if (rc_mode == AOM_Q) { @@ -1579,7 +1683,8 @@ : av1_compute_qdelta(rc, q_val, q_val * 0.50, bit_depth); active_best_quality = AOMMAX(qindex + delta_qindex, rc->best_quality); } else { - active_best_quality = get_gf_active_quality(p_rc, q, bit_depth); + active_best_quality = + get_gf_active_quality(p_rc, q, bit_depth, res_idx, rtc_mode); } } else { if (rc_mode == AOM_Q) { @@ -1717,6 +1822,12 @@ int active_worst_quality = *active_worst; const int bit_depth = cm->seq_params->bit_depth; + const int is_608p_or_larger = AOMMIN(cm->width, cm->height) >= 608; + const int is_480p_or_larger = AOMMIN(cm->width, cm->height) >= 480; + // res_idx is 0 for res < 480p, 1 for 480p, 2 for 608p+ + const int res_idx = is_480p_or_larger + is_608p_or_larger; + const bool rtc_mode = (cpi->oxcf.mode == REALTIME); + if (rc->frames_to_key <= 1 && oxcf->rc_cfg.mode == AOM_Q) { // If the next frame is also a key frame or the current frame is the // only frame in the sequence in AOM_Q mode, just use the cq_level @@ -1762,8 +1873,8 @@ double q_val; // Baseline value derived from active_worst_quality and kf boost. - active_best_quality = - get_kf_active_quality(p_rc, active_worst_quality, bit_depth); + active_best_quality = get_kf_active_quality(p_rc, active_worst_quality, + bit_depth, res_idx, rtc_mode); if (cpi->is_screen_content_type) { active_best_quality /= 2; } @@ -1843,8 +1954,8 @@ } #else (void)is_intrl_arf_boost; - active_best_quality -= cpi->ppi->twopass.extend_minq / 8; - active_worst_quality += cpi->ppi->twopass.extend_maxq / 4; + active_best_quality -= cpi->ppi->twopass.extend_minq / 4; + active_worst_quality += cpi->ppi->twopass.extend_maxq; #endif } @@ -1953,14 +2064,20 @@ const RefreshFrameInfo *const refresh_frame = &cpi->refresh_frame; const GF_GROUP *gf_group = &cpi->ppi->gf_group; const enum aom_rc_mode rc_mode = oxcf->rc_cfg.mode; - int *inter_minq; - ASSIGN_MINQ_TABLE(bit_depth, inter_minq); int active_best_quality = 0; - const int is_intrl_arf_boost = - gf_group->update_type[gf_index] == INTNL_ARF_UPDATE; - int is_leaf_frame = - !(gf_group->update_type[gf_index] == ARF_UPDATE || - gf_group->update_type[gf_index] == GF_UPDATE || is_intrl_arf_boost); + FRAME_UPDATE_TYPE update_type = gf_group->update_type[gf_index]; + const int is_intrl_arf_boost = update_type == INTNL_ARF_UPDATE; + int is_leaf_frame = !(update_type == ARF_UPDATE || update_type == GF_UPDATE || + is_intrl_arf_boost); + + const int is_608p_or_larger = AOMMIN(cm->width, cm->height) >= 608; + const int is_480p_or_larger = AOMMIN(cm->width, cm->height) >= 480; + // res_idx is 0 for res < 480p, 1 for 480p, 2 for 608p+ + const int res_idx = is_480p_or_larger + is_608p_or_larger; + const bool rtc_mode = (cpi->oxcf.mode == REALTIME); + + int *inter_minq; + ASSIGN_MINQ_TABLE_2(bit_depth, inter_minq, res_idx > 1, rtc_mode); // TODO(jingning): Consider to rework this hack that covers issues incurred // in lightfield setting. @@ -1968,7 +2085,8 @@ is_leaf_frame = !(refresh_frame->golden_frame || refresh_frame->alt_ref_frame || is_intrl_arf_boost); } - const int is_overlay_frame = rc->is_src_frame_alt_ref; + const int is_overlay_frame = + update_type == OVERLAY_UPDATE || update_type == INTNL_OVERLAY_UPDATE; if (is_leaf_frame || is_overlay_frame) { if (rc_mode == AOM_Q) return cq_level; @@ -1992,10 +2110,12 @@ q = p_rc->avg_frame_qindex[INTER_FRAME]; } if (rc_mode == AOM_CQ && q < cq_level) q = cq_level; - active_best_quality = get_gf_active_quality(p_rc, q, bit_depth); + active_best_quality = + get_gf_active_quality(p_rc, q, bit_depth, res_idx, rtc_mode); // Constrained quality use slightly lower active best. if (rc_mode == AOM_CQ) active_best_quality = active_best_quality * 15 / 16; - const int min_boost = get_gf_high_motion_quality(q, bit_depth); + const int min_boost = + get_gf_high_motion_quality(q, bit_depth, res_idx, rtc_mode); const int boost = min_boost - active_best_quality; active_best_quality = min_boost - (int)(boost * p_rc->arf_boost_factor); if (!is_intrl_arf_boost) return active_best_quality; @@ -2461,6 +2581,10 @@ } if (cpi->refresh_frame.golden_frame) rc->frame_num_last_gf_refresh = current_frame->frame_number; + + if (rc->frame_source_sad < 10000) + rc->last_frame_low_source_sad = rc->frame_number_encoded; + rc->prev_coded_width = cm->width; rc->prev_coded_height = cm->height; rc->frame_number_encoded++; @@ -2567,38 +2691,30 @@ static void set_gf_interval_range(const AV1_COMP *const cpi, RATE_CONTROL *const rc) { const AV1EncoderConfig *const oxcf = &cpi->oxcf; + // Set Maximum gf/arf interval + rc->max_gf_interval = oxcf->gf_cfg.max_gf_interval; + rc->min_gf_interval = oxcf->gf_cfg.min_gf_interval; + if (rc->min_gf_interval == 0) + rc->min_gf_interval = av1_rc_get_default_min_gf_interval( + oxcf->frm_dim_cfg.width, oxcf->frm_dim_cfg.height, cpi->framerate); + if (rc->max_gf_interval == 0) + rc->max_gf_interval = + get_default_max_gf_interval(cpi->framerate, rc->min_gf_interval); + /* + * Extended max interval for genuinely static scenes like slide shows. + * The no.of.stats available in the case of LAP is limited, + * hence setting to max_gf_interval. + */ + if (cpi->ppi->lap_enabled) + rc->static_scene_max_gf_interval = rc->max_gf_interval + 1; + else + rc->static_scene_max_gf_interval = MAX_STATIC_GF_GROUP_LENGTH; - // Special case code for 1 pass fixed Q mode tests - if ((has_no_stats_stage(cpi)) && (oxcf->rc_cfg.mode == AOM_Q)) { - rc->max_gf_interval = oxcf->gf_cfg.max_gf_interval; - rc->min_gf_interval = oxcf->gf_cfg.min_gf_interval; - rc->static_scene_max_gf_interval = rc->min_gf_interval + 1; - } else { - // Set Maximum gf/arf interval - rc->max_gf_interval = oxcf->gf_cfg.max_gf_interval; - rc->min_gf_interval = oxcf->gf_cfg.min_gf_interval; - if (rc->min_gf_interval == 0) - rc->min_gf_interval = av1_rc_get_default_min_gf_interval( - oxcf->frm_dim_cfg.width, oxcf->frm_dim_cfg.height, cpi->framerate); - if (rc->max_gf_interval == 0) - rc->max_gf_interval = - get_default_max_gf_interval(cpi->framerate, rc->min_gf_interval); - /* - * Extended max interval for genuinely static scenes like slide shows. - * The no.of.stats available in the case of LAP is limited, - * hence setting to max_gf_interval. - */ - if (cpi->ppi->lap_enabled) - rc->static_scene_max_gf_interval = rc->max_gf_interval + 1; - else - rc->static_scene_max_gf_interval = MAX_STATIC_GF_GROUP_LENGTH; + if (rc->max_gf_interval > rc->static_scene_max_gf_interval) + rc->max_gf_interval = rc->static_scene_max_gf_interval; - if (rc->max_gf_interval > rc->static_scene_max_gf_interval) - rc->max_gf_interval = rc->static_scene_max_gf_interval; - - // Clamp min to max - rc->min_gf_interval = AOMMIN(rc->min_gf_interval, rc->max_gf_interval); - } + // Clamp min to max + rc->min_gf_interval = AOMMIN(rc->min_gf_interval, rc->max_gf_interval); } void av1_rc_update_framerate(AV1_COMP *cpi, int width, int height) { @@ -2723,7 +2839,7 @@ int av1_calc_pframe_target_size_one_pass_vbr( const AV1_COMP *const cpi, FRAME_UPDATE_TYPE frame_update_type) { - static const int af_ratio = 10; + const int af_ratio = is_one_pass_rt_lag_params(cpi) ? 6 : 10; const RATE_CONTROL *const rc = &cpi->rc; const PRIMARY_RATE_CONTROL *const p_rc = &cpi->ppi->p_rc; int64_t target; @@ -2756,7 +2872,14 @@ const RATE_CONTROL *rc = &cpi->rc; const PRIMARY_RATE_CONTROL *p_rc = &cpi->ppi->p_rc; const RateControlCfg *rc_cfg = &oxcf->rc_cfg; - const int64_t diff = p_rc->optimal_buffer_level - p_rc->buffer_level; + const RefreshFrameInfo *const refresh_frame = &cpi->refresh_frame; + int64_t diff = p_rc->optimal_buffer_level - p_rc->buffer_level; + // For refresh alt or golden: keep diff negative to force setting + // higher target bandwidth on these frames. + // Only realtime mode with lookahead. + if (is_one_pass_rt_lag_params(cpi) && + (refresh_frame->alt_ref_frame || refresh_frame->golden_frame)) + diff = AOMMIN(diff, -p_rc->optimal_buffer_level); const int64_t one_pct_bits = 1 + p_rc->optimal_buffer_level / 100; int min_frame_target = AOMMAX(rc->avg_frame_bandwidth >> 4, FRAME_OVERHEAD_BITS); @@ -3143,21 +3266,8 @@ return best_sad; } -/*!\brief Check for scene detection, for 1 pass real-time mode. - * - * Compute average source sad (temporal sad: between current source and - * previous source) over a subset of superblocks. Use this is detect big changes - * in content and set the \c cpi->rc.high_source_sad flag. - * - * \ingroup rate_control - * \param[in] cpi Top level encoder structure - * \param[in] frame_input Current and last input source frames - * - * \remark Nothing is returned. Instead the flag \c cpi->rc.high_source_sad - * is set if scene change is detected, and \c cpi->rc.avg_source_sad is updated. - */ -static void rc_scene_detection_onepass_rt(AV1_COMP *cpi, - const EncodeFrameInput *frame_input) { +void av1_rc_scene_detection_onepass_rt(AV1_COMP *cpi, + const EncodeFrameInput *frame_input) { AV1_COMMON *const cm = &cpi->common; RATE_CONTROL *const rc = &cpi->rc; YV12_BUFFER_CONFIG const *const unscaled_src = frame_input->source; @@ -3176,8 +3286,14 @@ width = cpi->oxcf.frm_dim_cfg.width; height = cpi->oxcf.frm_dim_cfg.height; } + // Set src_sad_blk_64x64 to NULL also for number_spatial layers > 1, as + // it is never allocated for number_spatial_layers > 1 (see the condition + // under which we allocate cpi->src_sad_blk_64x64 later in this function). + // This is guard against the case where the number_spatial_layers + // is changed dynamically without re-alloc of encoder. if (width != cm->render_width || height != cm->render_height || - unscaled_src == NULL || unscaled_last_src == NULL) { + cpi->svc.number_spatial_layers > 1 || unscaled_src == NULL || + unscaled_last_src == NULL) { aom_free(cpi->src_sad_blk_64x64); cpi->src_sad_blk_64x64 = NULL; } @@ -3243,7 +3359,7 @@ rc->prev_frame_is_dropped || cpi->svc.number_temporal_layers > 1; // Store blkwise SAD for later use. Disable for spatial layers for now. if (width == cm->render_width && height == cm->render_height && - cpi->svc.number_spatial_layers == 1) { + cpi->svc.number_spatial_layers == 1 && !is_one_pass_rt_lag_params(cpi)) { if (cpi->src_sad_blk_64x64 == NULL) { CHECK_MEM_ERROR(cm, cpi->src_sad_blk_64x64, (uint64_t *)aom_calloc(sb_cols * sb_rows, @@ -3301,11 +3417,13 @@ // between current and previous frame value(s). Use minimum threshold // for cases where there is small change from content that is completely // static. + const int thresh_zero_sad_samples = + avg_sad > 8 * min_thresh ? 3 * (num_samples >> 2) : num_samples >> 1; if (!light_change && avg_sad > AOMMAX(min_thresh, (unsigned int)(rc->avg_source_sad * thresh)) && rc->frames_since_key > 1 + cpi->svc.number_spatial_layers && - num_zero_temp_sad < 3 * (num_samples >> 2)) + num_zero_temp_sad < thresh_zero_sad_samples) rc->high_source_sad = 1; else rc->high_source_sad = 0; @@ -3321,14 +3439,23 @@ } // Update the high_motion_content_screen_rtc flag on TL0. Avoid the update // if too many consecutive frame drops occurred. - const uint64_t thresh_high_motion = 9 * 64 * 64; + // The threshold_high_motion is kept to a large value, to account for + // mis-detection for scroll for scaled input (where scroll motion can be + // subpel and not detected below). The threshold may be improved when better + // scroll detection (for subpel) is added. + const int scale = + (unscaled_src->y_width * unscaled_src->y_height > 1920 * 1080) ? 30 : 10; + const uint64_t thresh_high_motion = scale * 64 * 64; if (cpi->svc.temporal_layer_id == 0 && rc->drop_count_consec < 3) { cpi->rc.high_motion_content_screen_rtc = 0; - if (cpi->oxcf.speed >= 11 && + if (cpi->oxcf.speed >= 11 && !is_one_pass_rt_lag_params(cpi) && cpi->oxcf.tune_cfg.content == AOM_CONTENT_SCREEN && + rc->num_col_blscroll_last_tl0 < 5 && + rc->num_row_blscroll_last_tl0 < 5 && rc->percent_blocks_with_motion > 40 && rc->prev_avg_source_sad > thresh_high_motion && rc->avg_source_sad > thresh_high_motion && + rc->frame_number_encoded - rc->last_frame_low_source_sad > 12 && rc->avg_frame_low_motion < 60 && unscaled_src->y_width >= 1280 && unscaled_src->y_height >= 720) { cpi->rc.high_motion_content_screen_rtc = 1; @@ -3800,15 +3927,15 @@ svc->spatial_layer_id == 0) { if (rc->prev_coded_width == cm->width && rc->prev_coded_height == cm->height) { - rc_scene_detection_onepass_rt(cpi, frame_input); + av1_rc_scene_detection_onepass_rt(cpi, frame_input); } else { aom_free(cpi->src_sad_blk_64x64); cpi->src_sad_blk_64x64 = NULL; } } - if (((*frame_type == KEY_FRAME && cpi->sf.rt_sf.rc_adjust_keyframe) || - (cpi->sf.rt_sf.rc_compute_spatial_var_sc && rc->high_source_sad)) && - svc->spatial_layer_id == 0 && cm->seq_params->bit_depth == 8 && + if (cpi->sf.rt_sf.rc_compute_spatial_var_sc_kf && + (*frame_type == KEY_FRAME || rc->high_source_sad) && + svc->spatial_layer_id == 0 && !cm->seq_params->use_highbitdepth && cpi->oxcf.rc_cfg.max_intra_bitrate_pct > 0) rc_spatial_act_onepass_rt(cpi, frame_input->source->y_buffer, frame_input->source->y_stride); @@ -3832,10 +3959,6 @@ resize_reset_rc(cpi, resize_pending_params->width, resize_pending_params->height, cm->width, cm->height); } - if (svc->temporal_layer_id == 0) { - rc->num_col_blscroll_last_tl0 = 0; - rc->num_row_blscroll_last_tl0 = 0; - } // Set the GF interval and update flag. if (!rc->rtc_external_ratectrl) set_gf_interval_update_onepass_rt(cpi, *frame_type); @@ -3904,7 +4027,7 @@ // For easy scene changes used lower QP, otherwise set max-q. // If rt_sf->compute_spatial_var_sc is enabled relax the max-q // condition based on frame spatial variance. - if (cpi->sf.rt_sf.rc_compute_spatial_var_sc) { + if (cpi->sf.rt_sf.rc_compute_spatial_var_sc_kf) { if (cpi->rc.frame_spatial_variance < 100) { *q = (cpi->rc.worst_quality + *q) >> 1; } else if (cpi->rc.frame_spatial_variance < 400 ||
diff --git a/av1/encoder/ratectrl.h b/av1/encoder/ratectrl.h index 88c048f..deea12d 100644 --- a/av1/encoder/ratectrl.h +++ b/av1/encoder/ratectrl.h
@@ -190,10 +190,21 @@ int sframe_due; int high_source_sad; + /*! + * The high_source_sad flag, which indicates scene change, for each frame [i] + * in the lookahead buffer, used for the encoding mode: is_one_pass_rt_lag. + */ + int high_source_sad_lag[MAX_GF_INTERVAL]; int high_motion_content_screen_rtc; uint64_t avg_source_sad; uint64_t prev_avg_source_sad; uint64_t frame_source_sad; + /*! + * The frame source sad for each frame [i] in the lookahead buffer, + * used for the encoding mode: is_one_pass_rt_lag. + */ + uint64_t frame_source_sad_lag[MAX_GF_INTERVAL]; + unsigned int last_frame_low_source_sad; uint64_t frame_spatial_variance; int static_since_last_scene_change; int last_encoded_size_keyframe; @@ -318,6 +329,11 @@ int gfu_boost; /*! + * Average boost factor of ARFs and GFs within a kf interval + */ + int gfu_boost_average; + + /*! * Stores the determined gf group lengths for a set of gf groups */ int gf_intervals[MAX_NUM_GF_INTERVALS]; @@ -830,6 +846,22 @@ */ int av1_postencode_drop_cbr(struct AV1_COMP *cpi, size_t *size); +/*!\brief Check for scene detection, for 1 pass real-time mode. + * + * Compute average source sad (temporal sad: between current source and + * previous source) over a subset of superblocks. Use this is detect big changes + * in content and set the \c cpi->rc.high_source_sad flag. + * + * \ingroup rate_control + * \param[in] cpi Top level encoder structure + * \param[in] frame_input Current and last input source frames + * + * \remark Nothing is returned. Instead the flag \c cpi->rc.high_source_sad + * is set if scene change is detected, and \c cpi->rc.avg_source_sad is updated. + */ +void av1_rc_scene_detection_onepass_rt( + struct AV1_COMP *cpi, const struct EncodeFrameInput *frame_input); + #ifdef __cplusplus } // extern "C" #endif
diff --git a/av1/encoder/rd.c b/av1/encoder/rd.c index 43e6a78..fcca312 100644 --- a/av1/encoder/rd.c +++ b/av1/encoder/rd.c
@@ -439,7 +439,7 @@ const aom_tune_metric tuning) { int64_t rdmult = av1_compute_rd_mult_based_on_qindex(bit_depth, update_type, qindex, tuning); - if (is_stat_consumption_stage && !use_fixed_qp_offsets && + if (is_stat_consumption_stage && (use_fixed_qp_offsets == 0) && (frame_type != KEY_FRAME)) { // Layer depth adjustment rdmult = (rdmult * rd_layer_depth_factor[layer_depth]) >> 7;
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c index 869a23c..3e49001 100644 --- a/av1/encoder/rdopt.c +++ b/av1/encoder/rdopt.c
@@ -784,7 +784,25 @@ } static void adjust_rdcost(const AV1_COMP *cpi, const MACROBLOCK *x, - RD_STATS *rd_cost) { + RD_STATS *rd_cost, bool is_inter_pred) { + if ((cpi->oxcf.tune_cfg.tuning == AOM_TUNE_IQ || + cpi->oxcf.tune_cfg.tuning == AOM_TUNE_SSIMULACRA2) && + is_inter_pred) { + // Tune IQ and SSIMULACRA2 are often used to encode layered AVIFs, where + // keyframes can be encoded at a lower quality (i.e. higher QP) than + // inter-coded frames. + // In this case, libaom tends to underestimate the true RD cost of inter + // prediction candidates, causing encoded file size to increase without a + // corresponding increase in quality. + // To compensate for this effect, make inter block candidates appear more + // expensive to the encoder to slightly bias toward intra prediction. + // Doing this increases overall compression efficiency, while still allowing + // the encoder to pick inter prediction when it's beneficial. + rd_cost->dist += rd_cost->dist >> 3; + rd_cost->rdcost += rd_cost->rdcost >> 3; + return; + } + if (cpi->oxcf.algo_cfg.sharpness != 3) return; if (frame_is_kf_gf_arf(cpi)) return; @@ -807,7 +825,14 @@ } static void adjust_cost(const AV1_COMP *cpi, const MACROBLOCK *x, - int64_t *rd_cost) { + int64_t *rd_cost, bool is_inter_pred) { + if ((cpi->oxcf.tune_cfg.tuning == AOM_TUNE_IQ || + cpi->oxcf.tune_cfg.tuning == AOM_TUNE_SSIMULACRA2) && + is_inter_pred) { + *rd_cost += *rd_cost >> 3; + return; + } + if (cpi->oxcf.algo_cfg.sharpness != 3) return; if (frame_is_kf_gf_arf(cpi)) return; @@ -1401,29 +1426,34 @@ *mode_index_end = SIMPLE_TRANSLATION; } -// Increase rd cost of warp mode for low complexity decoding. -static inline void increase_warp_mode_rd(const MB_MODE_INFO *const best_mbmi, - const MB_MODE_INFO *const this_mbmi, - int64_t *const best_scaled_rd, - int64_t *const this_scaled_rd, - int rd_bias_scale_pct) { - // Check rd bias percentage is non-zero. - if (!rd_bias_scale_pct) return; +// Increase rd cost of warp and obmc motion modes for low complexity decoding. +static inline void increase_motion_mode_rd(const MB_MODE_INFO *const best_mbmi, + const MB_MODE_INFO *const this_mbmi, + int64_t *const best_scaled_rd, + int64_t *const this_scaled_rd, + int rd_warp_bias_scale_pct, + float rd_obmc_bias_scale_pct) { if (*best_scaled_rd == INT64_MAX || *this_scaled_rd == INT64_MAX) return; - // Experiments have been performed with increasing the RD cost of warp mode at - // the below locations of inter mode evaluation. + // Experiments have been performed with increasing the RD cost of warp and + // obmc motion modes at the below locations of inter mode evaluation. // (1). Inter mode evaluation loop in av1_rd_pick_inter_mode(). // (2). Motion mode evaluation during handle_inter_mode() call. // (3). Motion mode evaluation for winner motion modes. // (4). Tx search for best inter candidates. // Based on the speed quality trade-off results of this speed feature, the rd // bias logic is enabled only at (2), (3) and (4). - const double rd_bias_scale = rd_bias_scale_pct / 100.0; + const double rd_warp_bias_scale = rd_warp_bias_scale_pct / 100.0; + const double rd_obmc_bias_scale = rd_obmc_bias_scale_pct / 100.0; if (best_mbmi->motion_mode == WARPED_CAUSAL) - *best_scaled_rd += (int64_t)(rd_bias_scale * *best_scaled_rd); + *best_scaled_rd += (int64_t)(rd_warp_bias_scale * *best_scaled_rd); + else if (best_mbmi->motion_mode == OBMC_CAUSAL) + *best_scaled_rd += (int64_t)(rd_obmc_bias_scale * *best_scaled_rd); + if (this_mbmi->motion_mode == WARPED_CAUSAL) - *this_scaled_rd += (int64_t)(rd_bias_scale * *this_scaled_rd); + *this_scaled_rd += (int64_t)(rd_warp_bias_scale * *this_scaled_rd); + else if (this_mbmi->motion_mode == OBMC_CAUSAL) + *this_scaled_rd += (int64_t)(rd_obmc_bias_scale * *this_scaled_rd); } /*!\brief AV1 motion mode search @@ -1512,7 +1542,6 @@ const int rate2_nocoeff = rd_stats->rate; int best_xskip_txfm = 0; RD_STATS best_rd_stats, best_rd_stats_y, best_rd_stats_uv; - uint8_t best_blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE]; uint8_t best_tx_type_map[MAX_MIB_SIZE * MAX_MIB_SIZE]; const int rate_mv0 = *rate_mv; const int interintra_allowed = cm->seq_params->enable_interintra_compound && @@ -1847,9 +1876,13 @@ } } - adjust_cost(cpi, x, &this_yrd); - adjust_rdcost(cpi, x, rd_stats); - adjust_rdcost(cpi, x, rd_stats_y); + if (this_yrd < INT64_MAX) { + adjust_cost(cpi, x, &this_yrd, /*is_inter_pred=*/true); + } + adjust_rdcost(cpi, x, rd_stats, /*is_inter_pred=*/true); + if (rd_stats_y->rdcost < INT64_MAX) { + adjust_rdcost(cpi, x, rd_stats_y, /*is_inter_pred=*/true); + } const int64_t tmp_rd = RDCOST(x->rdmult, rd_stats->rate, rd_stats->dist); if (mode_index == 0) { @@ -1858,8 +1891,10 @@ int64_t best_scaled_rd = best_rd; int64_t this_scaled_rd = tmp_rd; if (mode_index != 0) - increase_warp_mode_rd(&best_mbmi, mbmi, &best_scaled_rd, &this_scaled_rd, - cpi->sf.inter_sf.bias_warp_mode_rd_scale_pct); + increase_motion_mode_rd(&best_mbmi, mbmi, &best_scaled_rd, + &this_scaled_rd, + cpi->sf.inter_sf.bias_warp_mode_rd_scale_pct, + cpi->sf.inter_sf.bias_obmc_mode_rd_scale_pct); if (mode_index == 0 || this_scaled_rd < best_scaled_rd) { // Update best_rd data if this is the best motion mode so far @@ -1870,8 +1905,6 @@ best_rate_mv = tmp_rate_mv; *yrd = this_yrd; if (num_planes > 1) best_rd_stats_uv = *rd_stats_uv; - memcpy(best_blk_skip, txfm_info->blk_skip, - sizeof(txfm_info->blk_skip[0]) * xd->height * xd->width); av1_copy_array(best_tx_type_map, xd->tx_type_map, xd->height * xd->width); best_xskip_txfm = mbmi->skip_txfm; } @@ -1888,8 +1921,6 @@ *rd_stats = best_rd_stats; *rd_stats_y = best_rd_stats_y; if (num_planes > 1) *rd_stats_uv = best_rd_stats_uv; - memcpy(txfm_info->blk_skip, best_blk_skip, - sizeof(txfm_info->blk_skip[0]) * xd->height * xd->width); av1_copy_array(xd->tx_type_map, best_tx_type_map, xd->height * xd->width); txfm_info->skip_txfm = best_xskip_txfm; @@ -2554,28 +2585,17 @@ static inline int prune_modes_based_on_tpl_stats( PruneInfoFromTpl *inter_cost_info_from_tpl, const int *refs, int ref_mv_idx, const PREDICTION_MODE this_mode, int prune_mode_level) { + const int is_ref_last2 = refs[0] == LAST2_FRAME || refs[1] == LAST2_FRAME; + if (prune_mode_level == 1 && !is_ref_last2) return 0; + const int have_newmv = have_newmv_in_inter_mode(this_mode); - if ((prune_mode_level < 2) && have_newmv) return 0; + if ((prune_mode_level == 2) && have_newmv) return 0; const int64_t best_inter_cost = inter_cost_info_from_tpl->best_inter_cost; if (best_inter_cost == INT64_MAX) return 0; - const int prune_level = prune_mode_level - 1; int64_t cur_inter_cost; - const int is_globalmv = - (this_mode == GLOBALMV) || (this_mode == GLOBAL_GLOBALMV); - const int prune_index = is_globalmv ? MAX_REF_MV_SEARCH : ref_mv_idx; - - // Thresholds used for pruning: - // Lower value indicates aggressive pruning and higher value indicates - // conservative pruning which is set based on ref_mv_idx and speed feature. - // 'prune_index' 0, 1, 2 corresponds to ref_mv indices 0, 1 and 2. prune_index - // 3 corresponds to GLOBALMV/GLOBAL_GLOBALMV - static const int tpl_inter_mode_prune_mul_factor[3][MAX_REF_MV_SEARCH + 1] = { - { 6, 6, 6, 4 }, { 6, 4, 4, 4 }, { 5, 4, 4, 4 } - }; - const int is_comp_pred = (refs[1] > INTRA_FRAME); if (!is_comp_pred) { cur_inter_cost = inter_cost_info_from_tpl->ref_inter_cost[refs[0] - 1]; @@ -2589,6 +2609,22 @@ cur_inter_cost = AOMMAX(inter_cost_ref0, inter_cost_ref1); } + if (is_ref_last2) return (cur_inter_cost > best_inter_cost); + + const int is_globalmv = + (this_mode == GLOBALMV) || (this_mode == GLOBAL_GLOBALMV); + const int prune_index = is_globalmv ? MAX_REF_MV_SEARCH : ref_mv_idx; + const int prune_level = prune_mode_level - 2; + + // Thresholds used for pruning: + // Lower value indicates aggressive pruning and higher value indicates + // conservative pruning which is set based on ref_mv_idx and speed feature. + // 'prune_index' 0, 1, 2 corresponds to ref_mv indices 0, 1 and 2. + // prune_index 3 corresponds to GLOBALMV/GLOBAL_GLOBALMV + static const int tpl_inter_mode_prune_mul_factor[3][MAX_REF_MV_SEARCH + 1] = { + { 6, 6, 6, 4 }, { 6, 4, 4, 4 }, { 5, 4, 4, 4 } + }; + // Prune the mode if cur_inter_cost is greater than threshold times // best_inter_cost if (cur_inter_cost > @@ -3061,7 +3097,6 @@ const int8_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame); RD_STATS best_rd_stats, best_rd_stats_y, best_rd_stats_uv; int64_t best_rd = INT64_MAX; - uint8_t best_blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE]; uint8_t best_tx_type_map[MAX_MIB_SIZE * MAX_MIB_SIZE]; int64_t best_yrd = INT64_MAX; MB_MODE_INFO best_mbmi = *mbmi; @@ -3326,8 +3361,6 @@ best_rd = tmp_rd; best_mbmi = *mbmi; best_xskip_txfm = txfm_info->skip_txfm; - memcpy(best_blk_skip, txfm_info->blk_skip, - sizeof(best_blk_skip[0]) * xd->height * xd->width); av1_copy_array(best_tx_type_map, xd->tx_type_map, xd->height * xd->width); motion_mode_cand->rate_mv = rate_mv; @@ -3353,8 +3386,6 @@ txfm_info->skip_txfm = best_xskip_txfm; assert(IMPLIES(mbmi->comp_group_idx == 1, mbmi->interinter_comp.type != COMPOUND_AVERAGE)); - memcpy(txfm_info->blk_skip, best_blk_skip, - sizeof(best_blk_skip[0]) * xd->height * xd->width); av1_copy_array(xd->tx_type_map, best_tx_type_map, xd->height * xd->width); rd_stats->rdcost = RDCOST(x->rdmult, rd_stats->rate, rd_stats->dist); @@ -3388,7 +3419,6 @@ MACROBLOCKD *const xd = &x->e_mbd; const TileInfo *tile = &xd->tile; MB_MODE_INFO *mbmi = xd->mi[0]; - TxfmSearchInfo *txfm_info = &x->txfm_search_info; const int mi_row = xd->mi_row; const int mi_col = xd->mi_col; @@ -3439,7 +3469,6 @@ MB_MODE_INFO best_mbmi = *mbmi; RD_STATS best_rdstats = *rd_stats; - uint8_t best_blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE] = { 0 }; uint8_t best_tx_type_map[MAX_MIB_SIZE * MAX_MIB_SIZE]; av1_copy_array(best_tx_type_map, xd->tx_type_map, ctx->num_4x4_blk); @@ -3565,15 +3594,11 @@ best_rd = rd_stats_yuv.rdcost; best_mbmi = *mbmi; best_rdstats = rd_stats_yuv; - memcpy(best_blk_skip, txfm_info->blk_skip, - sizeof(txfm_info->blk_skip[0]) * xd->height * xd->width); av1_copy_array(best_tx_type_map, xd->tx_type_map, xd->height * xd->width); } } *mbmi = best_mbmi; *rd_stats = best_rdstats; - memcpy(txfm_info->blk_skip, best_blk_skip, - sizeof(txfm_info->blk_skip[0]) * xd->height * xd->width); av1_copy_array(xd->tx_type_map, best_tx_type_map, ctx->num_4x4_blk); #if CONFIG_RD_DEBUG mbmi->rd_stats = *rd_stats; @@ -3593,7 +3618,6 @@ MACROBLOCKD *const xd = &x->e_mbd; MB_MODE_INFO *const mbmi = xd->mi[0]; const int num_planes = av1_num_planes(cm); - TxfmSearchInfo *txfm_info = &x->txfm_search_info; int rate_y = 0, rate_uv = 0, rate_y_tokenonly = 0, rate_uv_tokenonly = 0; uint8_t y_skip_txfm = 0, uv_skip_txfm = 0; int64_t dist_y = 0, dist_uv = 0; @@ -3618,8 +3642,6 @@ // Set up the tx variables for reproducing the y predictions in case we // need it for chroma-from-luma. if (xd->is_chroma_ref && store_cfl_required_rdo(cm, x)) { - memcpy(txfm_info->blk_skip, ctx->blk_skip, - sizeof(txfm_info->blk_skip[0]) * ctx->num_4x4_blk); av1_copy_array(xd->tx_type_map, ctx->tx_type_map, ctx->num_4x4_blk); } const TX_SIZE max_uv_tx_size = av1_get_tx_size(AOM_PLANE_U, xd); @@ -3643,13 +3665,11 @@ best_rd = rd_cost->rdcost; if (rd_pick_intrabc_mode_sb(cpi, x, ctx, rd_cost, bsize, best_rd) < best_rd) { ctx->rd_stats.skip_txfm = mbmi->skip_txfm; - memcpy(ctx->blk_skip, txfm_info->blk_skip, - sizeof(txfm_info->blk_skip[0]) * ctx->num_4x4_blk); assert(rd_cost->rate != INT_MAX); } if (rd_cost->rate == INT_MAX) return; - ctx->mic = *xd->mi[0]; + ctx->mic = *mbmi; av1_copy_mbmi_ext_to_mbmi_ext_frame(&ctx->mbmi_ext_best, &x->mbmi_ext, av1_ref_frame_type(xd->mi[0]->ref_frame)); av1_copy_array(ctx->tx_type_map, xd->tx_type_map, ctx->num_4x4_blk); @@ -3830,7 +3850,6 @@ MACROBLOCKD *const xd = &x->e_mbd; MB_MODE_INFO *const mbmi = xd->mi[0]; TxfmSearchParams *txfm_params = &x->txfm_search_params; - TxfmSearchInfo *txfm_info = &x->txfm_search_info; int64_t best_rd; const int num_planes = av1_num_planes(cm); @@ -3910,8 +3929,6 @@ INT64_MAX); memset(mbmi->inter_tx_size, mbmi->tx_size, sizeof(mbmi->inter_tx_size)); - for (int i = 0; i < xd->height * xd->width; ++i) - set_blk_skip(txfm_info->blk_skip, 0, i, rd_stats_y.skip_txfm); } } else { av1_pick_uniform_tx_size_type_yrd(cpi, x, &rd_stats_y, bsize, @@ -3951,7 +3968,6 @@ if (best_rd > this_rd) { *best_mbmode = *mbmi; *best_mode_index = winner_mode_index; - av1_copy_array(ctx->blk_skip, txfm_info->blk_skip, ctx->num_4x4_blk); av1_copy_array(ctx->tx_type_map, xd->tx_type_map, ctx->num_4x4_blk); rd_cost->rate = this_rate; rd_cost->dist = rd_stats_y.dist + rd_stats_uv.dist; @@ -4149,11 +4165,10 @@ 0) { // Prune inter modes when relative dist of ALTREF2 and ALTREF is close // to the relative dist of LAST_FRAME. - if (inter_sf->alt_ref_search_fp == 1 && - (abs(cpi->ref_frame_dist_info - .ref_relative_dist[ref_frame - LAST_FRAME]) > - 1.5 * abs(cpi->ref_frame_dist_info - .ref_relative_dist[LAST_FRAME - LAST_FRAME]))) { + if (abs(cpi->ref_frame_dist_info + .ref_relative_dist[ref_frame - LAST_FRAME] - + cpi->ref_frame_dist_info + .ref_relative_dist[LAST_FRAME - LAST_FRAME]) > 4) { continue; } if (x->pred_mv_sad[ref_frame] > sad_thresh) @@ -4194,8 +4209,8 @@ // Prune reference frames which are not the closest to the current // frame and with large pred_mv_sad. if (inter_sf->prune_single_ref) { - assert(inter_sf->prune_single_ref > 0 && inter_sf->prune_single_ref < 3); - const double prune_threshes[2] = { 1.20, 1.05 }; + assert(inter_sf->prune_single_ref > 0 && inter_sf->prune_single_ref < 4); + const double prune_threshes[3] = { 1.20, 1.20, 1.05 }; for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { const RefFrameDistanceInfo *const ref_frame_dist_info = @@ -4203,8 +4218,10 @@ const int is_closest_ref = (ref_frame == ref_frame_dist_info->nearest_past_ref) || (ref_frame == ref_frame_dist_info->nearest_future_ref); + const int ref_idx = ref_frame - LAST_FRAME; - if (!is_closest_ref) { + if (!(cpi->keep_single_ref_frame_mask & (1 << ref_idx) || + is_closest_ref)) { const int dir = (ref_frame_dist_info->ref_relative_dist[ref_frame - LAST_FRAME] < 0) ? 0 @@ -5117,7 +5134,6 @@ const int skip_ctx = av1_get_skip_txfm_context(xd); const int skip_txfm = mbmi->skip_txfm && !is_mode_intra(av1_mode_defs[new_best_mode].mode); - const TxfmSearchInfo *txfm_info = &x->txfm_search_info; search_state->best_rd = new_best_rd_stats->rdcost; search_state->best_mode_index = new_best_mode; @@ -5137,8 +5153,6 @@ search_state->best_rate_uv = new_best_rd_stats_uv->rate; } search_state->best_y_rdcost = *new_best_rd_stats_y; - memcpy(ctx->blk_skip, txfm_info->blk_skip, - sizeof(txfm_info->blk_skip[0]) * ctx->num_4x4_blk); av1_copy_array(ctx->tx_type_map, xd->tx_type_map, ctx->num_4x4_blk); } @@ -5241,9 +5255,10 @@ int64_t best_scaled_rd = search_state->best_rd; int64_t this_scaled_rd = rd_stats.rdcost; if (search_state->best_mode_index != THR_INVALID) - increase_warp_mode_rd(&search_state->best_mbmode, mbmi, &best_scaled_rd, - &this_scaled_rd, - cpi->sf.inter_sf.bias_warp_mode_rd_scale_pct); + increase_motion_mode_rd(&search_state->best_mbmode, mbmi, + &best_scaled_rd, &this_scaled_rd, + cpi->sf.inter_sf.bias_warp_mode_rd_scale_pct, + cpi->sf.inter_sf.bias_obmc_mode_rd_scale_pct); if (this_scaled_rd < best_scaled_rd) { *yrd = this_yrd; @@ -5583,9 +5598,10 @@ int64_t best_scaled_rd = search_state->best_rd; int64_t this_scaled_rd = rd_stats.rdcost; - increase_warp_mode_rd(&search_state->best_mbmode, mbmi, &best_scaled_rd, - &this_scaled_rd, - cpi->sf.inter_sf.bias_warp_mode_rd_scale_pct); + increase_motion_mode_rd(&search_state->best_mbmode, mbmi, &best_scaled_rd, + &this_scaled_rd, + cpi->sf.inter_sf.bias_warp_mode_rd_scale_pct, + cpi->sf.inter_sf.bias_obmc_mode_rd_scale_pct); if (this_scaled_rd < best_rd_in_this_partition) { best_rd_in_this_partition = rd_stats.rdcost; *yrd = this_yrd; @@ -5725,7 +5741,6 @@ int best_mode_cost_y = -1; MB_MODE_INFO best_mbmi = *xd->mi[0]; THR_MODES best_mode_enum = THR_INVALID; - uint8_t best_blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE]; uint8_t best_tx_type_map[MAX_MIB_SIZE * MAX_MIB_SIZE]; const int num_4x4 = bsize_to_num_blk(bsize); @@ -5742,6 +5757,8 @@ if (bh > 4 || bw > 4) return; } + mbmi->skip_txfm = 0; + for (int mode_idx = 0; mode_idx < LUMA_MODE_COUNT; ++mode_idx) { if (sf->intra_sf.skip_intra_in_interframe && search_state->intra_search_state.skip_intra_modes) @@ -5799,7 +5816,7 @@ &best_model_rd, top_intra_model_rd); if (intra_rd_y < INT64_MAX) { - adjust_cost(cpi, x, &intra_rd_y); + adjust_cost(cpi, x, &intra_rd_y, /*is_inter_pred=*/false); } if (is_luma_result_valid && intra_rd_y < yrd_threshold) { @@ -5810,8 +5827,6 @@ best_rd_y = intra_rd_y; best_mbmi = *mbmi; best_mode_enum = mode_enum; - memcpy(best_blk_skip, x->txfm_search_info.blk_skip, - sizeof(best_blk_skip[0]) * num_4x4); av1_copy_array(best_tx_type_map, xd->tx_type_map, num_4x4); } } @@ -5825,8 +5840,6 @@ // Restores the best luma mode *mbmi = best_mbmi; - memcpy(x->txfm_search_info.blk_skip, best_blk_skip, - sizeof(best_blk_skip[0]) * num_4x4); av1_copy_array(xd->tx_type_map, best_tx_type_map, num_4x4); // Performs chroma search @@ -5887,7 +5900,7 @@ intra_rd_stats.rdcost = this_rd; - adjust_rdcost(cpi, x, &intra_rd_stats); + adjust_rdcost(cpi, x, &intra_rd_stats, /*is_inter_pred=*/false); // Collect mode stats for multiwinner mode processing const int txfm_search_done = 1; @@ -6273,11 +6286,8 @@ ref_frame > INTRA_FRAME && second_ref_frame == NONE_FRAME; const int comp_pred = second_ref_frame > INTRA_FRAME; - init_mbmi(mbmi, this_mode, ref_frames, cm); - txfm_info->skip_txfm = 0; sf_args.num_single_modes_processed += is_single_pred; - set_ref_ptrs(cm, xd, ref_frame, second_ref_frame); #if CONFIG_COLLECT_COMPONENT_TIMING start_timing(cpi, skip_inter_mode_time); #endif @@ -6289,6 +6299,9 @@ #endif if (is_skip_inter_mode) continue; + init_mbmi(mbmi, this_mode, ref_frames, cm); + set_ref_ptrs(cm, xd, ref_frame, second_ref_frame); + // Select prediction reference frames. for (i = 0; i < num_planes; i++) { xd->plane[i].pre[0] = yv12_mb[ref_frame][i]; @@ -6359,8 +6372,8 @@ ref_frame_rd[ref_frame] = this_rd; } - adjust_cost(cpi, x, &this_rd); - adjust_rdcost(cpi, x, &rd_stats); + adjust_cost(cpi, x, &this_rd, /*is_inter_pred=*/true); + adjust_rdcost(cpi, x, &rd_stats, /*is_inter_pred=*/true); // Did this mode help, i.e., is it the new best mode if (this_rd < search_state.best_rd) { @@ -6479,8 +6492,6 @@ search_state.best_mbmode = *mbmi; search_state.best_skip2 = 0; search_state.best_mode_skippable = this_skippable; - memcpy(ctx->blk_skip, txfm_info->blk_skip, - sizeof(txfm_info->blk_skip[0]) * ctx->num_4x4_blk); av1_copy_array(ctx->tx_type_map, xd->tx_type_map, ctx->num_4x4_blk); } }
diff --git a/av1/encoder/rdopt_utils.h b/av1/encoder/rdopt_utils.h index 5214800..5063d34 100644 --- a/av1/encoder/rdopt_utils.h +++ b/av1/encoder/rdopt_utils.h
@@ -558,6 +558,7 @@ case DEFAULT_EVAL: txfm_params->default_inter_tx_type_prob_thresh = INT_MAX; txfm_params->use_default_intra_tx_type = 0; + txfm_params->use_derived_intra_tx_type_set = 0; txfm_params->skip_txfm_level = winner_mode_params->skip_txfm_level[DEFAULT_EVAL]; txfm_params->predict_dc_level = @@ -576,8 +577,10 @@ break; case MODE_EVAL: txfm_params->use_default_intra_tx_type = - (cpi->sf.tx_sf.tx_type_search.fast_intra_tx_type_search || + (cpi->sf.tx_sf.tx_type_search.fast_intra_tx_type_search == 2 || cpi->oxcf.txfm_cfg.use_intra_default_tx_only); + txfm_params->use_derived_intra_tx_type_set = + cpi->sf.tx_sf.tx_type_search.fast_intra_tx_type_search == 1; txfm_params->default_inter_tx_type_prob_thresh = cpi->sf.tx_sf.tx_type_search.fast_inter_tx_type_prob_thresh; txfm_params->skip_txfm_level = @@ -607,6 +610,7 @@ case WINNER_MODE_EVAL: txfm_params->default_inter_tx_type_prob_thresh = INT_MAX; txfm_params->use_default_intra_tx_type = 0; + txfm_params->use_derived_intra_tx_type_set = 0; txfm_params->skip_txfm_level = winner_mode_params->skip_txfm_level[WINNER_MODE_EVAL]; txfm_params->predict_dc_level =
diff --git a/av1/encoder/reconinter_enc.c b/av1/encoder/reconinter_enc.c index fd751cd..0185438 100644 --- a/av1/encoder/reconinter_enc.c +++ b/av1/encoder/reconinter_enc.c
@@ -523,14 +523,9 @@ int ref_stride, const uint8_t *mask, int mask_stride, int invert_mask, int subpel_search) { - if (subpel_x_q3 | subpel_y_q3) { - aom_upsampled_pred(xd, cm, mi_row, mi_col, mv, comp_pred, width, height, - subpel_x_q3, subpel_y_q3, ref, ref_stride, - subpel_search); - ref = comp_pred; - ref_stride = width; - } - aom_comp_mask_pred(comp_pred, pred, width, height, ref, ref_stride, mask, + aom_upsampled_pred(xd, cm, mi_row, mi_col, mv, comp_pred, width, height, + subpel_x_q3, subpel_y_q3, ref, ref_stride, subpel_search); + aom_comp_mask_pred(comp_pred, pred, width, height, comp_pred, width, mask, mask_stride, invert_mask); }
diff --git a/av1/encoder/speed_features.c b/av1/encoder/speed_features.c index a981f08..7939f7c 100644 --- a/av1/encoder/speed_features.c +++ b/av1/encoder/speed_features.c
@@ -223,6 +223,7 @@ } if (speed >= 1) { + sf->part_sf.ml_4_partition_search_level_index = 1; if (is_720p_or_larger) { sf->part_sf.use_square_partition_only_threshold = BLOCK_128X128; } else if (is_480p_or_larger) { @@ -232,12 +233,11 @@ } if (is_720p_or_larger) { - sf->part_sf.ml_partition_search_breakout_thresh[0] = 0.9999999f; - sf->part_sf.ml_partition_search_breakout_thresh[1] = 0.9999999f; - sf->part_sf.ml_partition_search_breakout_thresh[2] = 0.9618367258814811f; - sf->part_sf.ml_partition_search_breakout_thresh[3] = 0.9990705139233304f; - sf->part_sf.ml_partition_search_breakout_thresh[4] = 0.9648891196441841f; - + sf->part_sf.ml_partition_search_breakout_thresh[0] = 0.5f; + sf->part_sf.ml_partition_search_breakout_thresh[1] = 0.5042595622791082f; + sf->part_sf.ml_partition_search_breakout_thresh[2] = 0.5f; + sf->part_sf.ml_partition_search_breakout_thresh[3] = 0.8378425823517456f; + sf->part_sf.ml_partition_search_breakout_thresh[4] = 0.8047585616503903f; sf->part_sf.ml_partition_search_breakout_model_index = 1; } else { sf->part_sf.ml_partition_search_breakout_thresh[0] = -1.0f; @@ -250,20 +250,19 @@ } if (speed >= 2) { + sf->part_sf.ml_4_partition_search_level_index = 2; if (is_720p_or_larger) { sf->part_sf.use_square_partition_only_threshold = BLOCK_64X64; - } else if (is_480p_or_larger) { - sf->part_sf.use_square_partition_only_threshold = BLOCK_32X32; } else { sf->part_sf.use_square_partition_only_threshold = BLOCK_32X32; } if (is_720p_or_larger) { - sf->part_sf.ml_partition_search_breakout_thresh[0] = 0.9583713938680828f; - sf->part_sf.ml_partition_search_breakout_thresh[1] = 0.9999999f; - sf->part_sf.ml_partition_search_breakout_thresh[2] = 0.9634239069901543f; - sf->part_sf.ml_partition_search_breakout_thresh[3] = 0.9000000000000001f; - sf->part_sf.ml_partition_search_breakout_thresh[4] = 0.9196596355880025f; + sf->part_sf.ml_partition_search_breakout_thresh[0] = 0.5f; + sf->part_sf.ml_partition_search_breakout_thresh[1] = 0.5042595622791082f; + sf->part_sf.ml_partition_search_breakout_thresh[2] = 0.5f; + sf->part_sf.ml_partition_search_breakout_thresh[3] = 0.8378425823517456f; + sf->part_sf.ml_partition_search_breakout_thresh[4] = 0.8047585616503903f; sf->part_sf.ml_partition_search_breakout_model_index = 1; } @@ -285,6 +284,7 @@ if (speed >= 3) { sf->part_sf.ml_early_term_after_part_split_level = 0; + sf->part_sf.ml_4_partition_search_level_index = 3; if (is_720p_or_larger) { for (int i = 0; i < PARTITION_BLOCK_SIZES; ++i) { @@ -501,7 +501,7 @@ sf->tpl_sf.search_method = FAST_BIGDIA; sf->tx_sf.tx_type_search.winner_mode_tx_type_pruning = 2; - sf->tx_sf.tx_type_search.fast_intra_tx_type_search = 1; + sf->tx_sf.tx_type_search.fast_intra_tx_type_search = 2; sf->tx_sf.tx_type_search.prune_2d_txfm_mode = TX_TYPE_PRUNE_3; sf->tx_sf.tx_type_search.prune_tx_type_est_rd = 1; @@ -560,7 +560,7 @@ // target image quality is very low. sf->part_sf.default_max_partition_size = BLOCK_32X32; - sf->mv_sf.use_bsize_dependent_search_method = 1; + sf->mv_sf.use_bsize_dependent_search_method = 3; sf->mv_sf.intrabc_search_level = 1; sf->tx_sf.tx_type_search.winner_mode_tx_type_pruning = 3; @@ -636,9 +636,10 @@ if (speed < 1 || speed > 3) return; const AV1_COMMON *const cm = &cpi->common; - const bool is_720p_or_larger = AOMMIN(cm->width, cm->height) >= 720; - const bool is_between_608p_and_720p = AOMMIN(cm->width, cm->height) >= 608 && - AOMMIN(cm->width, cm->height) <= 720; + const bool is_between_608p_and_1080p = AOMMIN(cm->width, cm->height) >= 608 && + AOMMIN(cm->width, cm->height) <= 1080; + const bool is_between_720p_and_1080p = AOMMIN(cm->width, cm->height) >= 720 && + AOMMIN(cm->width, cm->height) <= 1080; const bool is_vertical_video = cm->width < cm->height; const FRAME_UPDATE_TYPE update_type = @@ -647,7 +648,7 @@ const int is_key_frame = frame_is_intra_only(cm); // Speed features for vertical videos - if (is_vertical_video && is_between_608p_and_720p) { + if (is_vertical_video && is_between_608p_and_1080p) { const int leaf_and_overlay_frames = (update_type == LF_UPDATE || update_type == OVERLAY_UPDATE || update_type == INTNL_OVERLAY_UPDATE); @@ -659,6 +660,7 @@ sf->lpf_sf.switchable_lr_with_bias_level = 1; sf->inter_sf.bias_warp_mode_rd_scale_pct = 4; + sf->inter_sf.bias_obmc_mode_rd_scale_pct = 1.5f; sf->part_sf.split_partition_penalty_level = is_key_frame ? 0 : 2; @@ -668,20 +670,13 @@ } // Speed features for regular videos - if (!is_vertical_video && is_720p_or_larger) { + if (!is_vertical_video && is_between_720p_and_1080p) { sf->gm_sf.gm_erroradv_tr_level = 1; sf->hl_sf.ref_frame_mvs_lvl = 1; - sf->lpf_sf.dual_sgr_penalty_level = boosted ? 1 : 2; - sf->lpf_sf.switchable_lr_with_bias_level = 1; - sf->lpf_sf.skip_loop_filter_using_filt_error = - (update_type != OVERLAY_UPDATE && update_type != INTNL_OVERLAY_UPDATE && - cpi->common.current_frame.pyramid_level > 1) - ? 1 - : 0; - sf->inter_sf.bias_warp_mode_rd_scale_pct = 4; + sf->inter_sf.bias_obmc_mode_rd_scale_pct = 1.5f; sf->part_sf.split_partition_penalty_level = is_key_frame ? 0 : 2; @@ -772,7 +767,8 @@ } if (speed >= 1) { - if (is_480p_or_lesser) sf->inter_sf.skip_newmv_in_drl = 1; + sf->part_sf.ml_4_partition_search_level_index = 1; + sf->inter_sf.skip_newmv_in_drl = 1; if (is_720p_or_larger) { sf->part_sf.use_square_partition_only_threshold = BLOCK_128X128; @@ -783,11 +779,11 @@ } if (is_720p_or_larger) { - sf->part_sf.ml_partition_search_breakout_thresh[0] = 0.9999999f; - sf->part_sf.ml_partition_search_breakout_thresh[1] = 0.9999999f; - sf->part_sf.ml_partition_search_breakout_thresh[2] = 0.9618367258814811f; - sf->part_sf.ml_partition_search_breakout_thresh[3] = 0.9990705139233304f; - sf->part_sf.ml_partition_search_breakout_thresh[4] = 0.9648891196441841f; + sf->part_sf.ml_partition_search_breakout_thresh[0] = 0.5f; + sf->part_sf.ml_partition_search_breakout_thresh[1] = 0.5042595622791082f; + sf->part_sf.ml_partition_search_breakout_thresh[2] = 0.5f; + sf->part_sf.ml_partition_search_breakout_thresh[3] = 0.8378425823517456f; + sf->part_sf.ml_partition_search_breakout_thresh[4] = 0.8047585616503903f; sf->part_sf.ml_partition_search_breakout_model_index = 1; } else { sf->part_sf.ml_partition_search_breakout_thresh[0] = -1.0f; @@ -802,20 +798,19 @@ } if (speed >= 2) { + sf->part_sf.ml_4_partition_search_level_index = 2; if (is_720p_or_larger) { sf->part_sf.use_square_partition_only_threshold = BLOCK_64X64; - } else if (is_480p_or_larger) { - sf->part_sf.use_square_partition_only_threshold = BLOCK_32X32; } else { sf->part_sf.use_square_partition_only_threshold = BLOCK_32X32; } if (is_720p_or_larger) { - sf->part_sf.ml_partition_search_breakout_thresh[0] = 0.9583713938680828f; - sf->part_sf.ml_partition_search_breakout_thresh[1] = 0.9999999f; - sf->part_sf.ml_partition_search_breakout_thresh[2] = 0.9634239069901543f; - sf->part_sf.ml_partition_search_breakout_thresh[3] = 0.9000000000000001f; - sf->part_sf.ml_partition_search_breakout_thresh[4] = 0.9196596355880025f; + sf->part_sf.ml_partition_search_breakout_thresh[0] = 0.5f; + sf->part_sf.ml_partition_search_breakout_thresh[1] = 0.5042595622791082f; + sf->part_sf.ml_partition_search_breakout_thresh[2] = 0.5f; + sf->part_sf.ml_partition_search_breakout_thresh[3] = 0.8378425823517456f; + sf->part_sf.ml_partition_search_breakout_thresh[4] = 0.8047585616503903f; sf->part_sf.ml_partition_search_breakout_model_index = 1; } @@ -892,6 +887,8 @@ sf->part_sf.ml_partition_search_breakout_model_index = 0; } + sf->part_sf.ml_4_partition_search_level_index = 3; + if (is_720p_or_larger) { sf->part_sf.partition_search_breakout_dist_thr = (1 << 25); sf->part_sf.partition_search_breakout_rate_thr = 200; @@ -940,7 +937,7 @@ if (is_480p_or_larger) { sf->tx_sf.tx_type_search.prune_tx_type_using_stats = 2; } else { - sf->mv_sf.skip_fullpel_search_using_startmv = boosted ? 0 : 1; + sf->mv_sf.skip_fullpel_search_using_startmv_refmv = boosted ? 0 : 1; } sf->inter_sf.disable_interinter_wedge_var_thresh = UINT_MAX; @@ -974,7 +971,7 @@ sf->inter_sf.skip_newmv_in_drl = 4; sf->inter_sf.prune_comp_ref_frames = 1; - sf->mv_sf.skip_fullpel_search_using_startmv = boosted ? 0 : 1; + sf->mv_sf.skip_fullpel_search_using_startmv_refmv = boosted ? 0 : 1; if (!is_720p_or_larger) { sf->inter_sf.mv_cost_upd_level = INTERNAL_COST_UPD_SBROW_SET; @@ -993,9 +990,11 @@ sf->inter_sf.prune_nearmv_using_neighbors = PRUNE_NEARMV_LEVEL2; } - if (is_720p_or_larger) + if (is_720p_or_larger) { sf->part_sf.ext_part_eval_based_on_cur_best = (allow_screen_content_tools || frame_is_intra_only(cm)) ? 0 : 1; + sf->part_sf.auto_max_partition_based_on_simple_motion = NOT_IN_USE; + } if (is_480p_or_larger) { sf->tpl_sf.reduce_num_frames = 1; @@ -1008,11 +1007,9 @@ sf->inter_sf.prune_comp_ref_frames = 2; sf->inter_sf.prune_nearest_near_mv_using_refmv_weight = (boosted || allow_screen_content_tools) ? 0 : 1; - sf->mv_sf.skip_fullpel_search_using_startmv = boosted ? 0 : 2; + sf->mv_sf.skip_fullpel_search_using_startmv_refmv = boosted ? 0 : 2; - if (is_720p_or_larger) { - sf->part_sf.auto_max_partition_based_on_simple_motion = NOT_IN_USE; - } else if (is_480p_or_larger) { + if (is_480p_or_larger && !is_720p_or_larger) { sf->part_sf.auto_max_partition_based_on_simple_motion = DIRECT_PRED; } @@ -1047,6 +1044,12 @@ sf->inter_sf.prune_ref_mv_idx_search = 1; } + if (is_720p_or_larger) { + sf->mv_sf.use_bsize_dependent_search_method = 1; + } else { + sf->mv_sf.use_bsize_dependent_search_method = 2; + } + if (!is_720p_or_larger) { sf->tx_sf.tx_type_search.fast_inter_tx_type_prob_thresh = is_boosted_arf2_bwd_type ? 450 : 150; @@ -1059,6 +1062,11 @@ if (cpi->oxcf.enable_low_complexity_decode) set_good_speed_features_lc_dec_framesize_dependent(cpi, sf, speed); + + if (cpi->oxcf.tune_cfg.tuning == AOM_TUNE_IQ || + cpi->oxcf.tune_cfg.tuning == AOM_TUNE_SSIMULACRA2) { + sf->intra_sf.skip_intra_in_interframe = 0; + } } static void set_good_speed_features_framesize_independent( @@ -1083,7 +1091,6 @@ sf->gm_sf.prune_ref_frame_for_gm_search = boosted ? 0 : 1; sf->gm_sf.disable_gm_search_based_on_stats = 1; - sf->part_sf.less_rectangular_check_level = 1; sf->part_sf.ml_prune_partition = 1; sf->part_sf.prune_ext_partition_types_search_level = 1; sf->part_sf.prune_part4_search = 2; @@ -1106,8 +1113,10 @@ sf->inter_sf.reduce_inter_modes = boosted ? 1 : 2; sf->inter_sf.selective_ref_frame = 1; sf->inter_sf.use_dist_wtd_comp_flag = DIST_WTD_COMP_SKIP_MV_SEARCH; + sf->inter_sf.enable_fast_compound_mode_search = 1; sf->interp_sf.use_fast_interpolation_filter_search = 1; + sf->interp_sf.disable_dual_filter = 1; sf->intra_sf.intra_pruning_with_hog = 1; @@ -1132,6 +1141,8 @@ sf->rd_sf.perform_coeff_opt = 1; sf->hl_sf.superres_auto_search_type = SUPERRES_AUTO_DUAL; + sf->lpf_sf.reduce_wiener_window_size = 1; + if (speed >= 1) { sf->hl_sf.adjust_num_frames_for_arf_filtering = allow_screen_content_tools ? 0 : 1; @@ -1164,6 +1175,8 @@ sf->inter_sf.reuse_inter_intra_mode = 1; sf->inter_sf.selective_ref_frame = 2; sf->inter_sf.skip_arf_compound = 1; + sf->inter_sf.prune_comp_using_best_single_mode_ref = 2; + sf->inter_sf.use_dist_wtd_comp_flag = DIST_WTD_COMP_DISABLED; sf->interp_sf.use_interp_filter = 1; @@ -1214,21 +1227,16 @@ sf->inter_sf.fast_interintra_wedge_search = 1; sf->inter_sf.prune_comp_search_by_single_result = boosted ? 4 : 1; sf->inter_sf.prune_ext_comp_using_neighbors = 1; - sf->inter_sf.prune_comp_using_best_single_mode_ref = 2; sf->inter_sf.prune_comp_type_by_comp_avg = 2; sf->inter_sf.selective_ref_frame = 3; - sf->inter_sf.use_dist_wtd_comp_flag = DIST_WTD_COMP_DISABLED; - sf->inter_sf.enable_fast_compound_mode_search = 1; sf->inter_sf.reuse_mask_search_results = 1; set_txfm_rd_gate_level(sf->inter_sf.txfm_rd_gate_level, boosted ? 0 : 1); sf->inter_sf.inter_mode_txfm_breakout = boosted ? 0 : 1; sf->inter_sf.alt_ref_search_fp = 1; + sf->inter_sf.prune_inter_modes_based_on_tpl = 1; sf->interp_sf.adaptive_interp_filter_search = 1; - sf->interp_sf.disable_dual_filter = 1; - sf->intra_sf.disable_smooth_intra = - !frame_is_intra_only(&cpi->common) || (cpi->rc.frames_to_key > 1); sf->intra_sf.intra_pruning_with_hog = 2; sf->intra_sf.skip_intra_in_interframe = is_inter_frame ? 2 : 1; sf->intra_sf.skip_filter_intra_in_inter_frames = 1; @@ -1242,7 +1250,6 @@ sf->lpf_sf.prune_wiener_based_on_src_var = 1; sf->lpf_sf.prune_sgr_based_on_wiener = 1; sf->lpf_sf.disable_loop_restoration_chroma = boosted ? 0 : 1; - sf->lpf_sf.reduce_wiener_window_size = boosted ? 0 : 1; // TODO(any): Re-evaluate this feature set to 1 in speed 2. sf->tpl_sf.allow_compound_pred = 0; @@ -1256,7 +1263,6 @@ sf->gm_sf.prune_zero_mv_with_sse = 1; sf->gm_sf.num_refinement_steps = 0; - sf->part_sf.less_rectangular_check_level = 2; sf->part_sf.simple_motion_search_prune_agg = allow_screen_content_tools ? SIMPLE_AGG_LVL0 @@ -1264,7 +1270,6 @@ sf->part_sf.prune_ext_part_using_split_info = 1; sf->part_sf.simple_motion_search_rect_split = 1; - sf->mv_sf.full_pixel_search_level = 1; sf->mv_sf.subpel_search_method = SUBPEL_TREE_PRUNED; sf->mv_sf.search_method = DIAMOND; sf->mv_sf.disable_second_mv = 2; @@ -1278,13 +1283,14 @@ // TODO(any): Experiment with the early exit mechanism for speeds 0, 1 and 2 // and clean-up the speed feature sf->inter_sf.perform_best_rd_based_gating_for_chroma = 1; - sf->inter_sf.prune_inter_modes_based_on_tpl = boosted ? 0 : 1; + sf->inter_sf.prune_inter_modes_based_on_tpl = boosted ? 1 : 2; sf->inter_sf.prune_comp_search_by_single_result = boosted ? 4 : 2; sf->inter_sf.selective_ref_frame = 5; sf->inter_sf.reuse_compound_type_decision = 1; set_txfm_rd_gate_level(sf->inter_sf.txfm_rd_gate_level, boosted ? 0 : (is_boosted_arf2_bwd_type ? 1 : 2)); sf->inter_sf.inter_mode_txfm_breakout = boosted ? 0 : 2; + sf->inter_sf.prune_single_ref = 1; sf->interp_sf.adaptive_interp_filter_search = 2; @@ -1301,7 +1307,6 @@ sf->tpl_sf.prune_intra_modes = 1; sf->tpl_sf.reduce_first_step_size = 6; sf->tpl_sf.subpel_force_stop = QUARTER_PEL; - sf->tpl_sf.gop_length_decision_method = 1; sf->tx_sf.adaptive_txb_search_level = boosted ? 2 : 3; sf->tx_sf.tx_type_search.use_skip_flag_prediction = 2; @@ -1327,6 +1332,8 @@ } if (speed >= 4) { + sf->hl_sf.weight_calc_level_in_tf = 1; + sf->mv_sf.subpel_search_method = SUBPEL_TREE_PRUNED_MORE; sf->gm_sf.prune_zero_mv_with_sse = 2; @@ -1346,7 +1353,7 @@ sf->inter_sf.txfm_rd_gate_level[TX_SEARCH_MOTION_MODE] = boosted ? 0 : 5; sf->inter_sf.txfm_rd_gate_level[TX_SEARCH_COMP_TYPE_MODE] = boosted ? 0 : 3; - sf->inter_sf.prune_inter_modes_based_on_tpl = boosted ? 0 : 2; + sf->inter_sf.prune_inter_modes_based_on_tpl = boosted ? 1 : 3; sf->inter_sf.prune_ext_comp_using_neighbors = 2; sf->inter_sf.prune_obmc_prob_thresh = INT_MAX; sf->inter_sf.disable_interinter_wedge_var_thresh = UINT_MAX; @@ -1385,7 +1392,6 @@ } if (speed >= 5) { - sf->hl_sf.weight_calc_level_in_tf = 1; sf->hl_sf.adjust_num_frames_for_arf_filtering = allow_screen_content_tools ? 0 : 2; @@ -1401,7 +1407,7 @@ sf->mv_sf.warp_search_method = WARP_SEARCH_DIAMOND; sf->inter_sf.prune_inter_modes_if_skippable = 1; - sf->inter_sf.prune_single_ref = is_boosted_arf2_bwd_type ? 0 : 1; + sf->inter_sf.prune_single_ref = is_boosted_arf2_bwd_type ? 0 : 2; sf->inter_sf.txfm_rd_gate_level[TX_SEARCH_DEFAULT] = boosted ? 0 : 4; sf->inter_sf.txfm_rd_gate_level[TX_SEARCH_COMP_TYPE_MODE] = boosted ? 0 : 5; sf->inter_sf.enable_fast_compound_mode_search = 2; @@ -1409,6 +1415,7 @@ sf->interp_sf.skip_interp_filter_search = boosted ? 0 : 1; sf->intra_sf.chroma_intra_pruning_with_hog = 3; + sf->intra_sf.disable_smooth_intra = 1; // TODO(any): Extend multi-winner mode processing support for inter frames sf->winner_mode_sf.multi_winner_mode_type = @@ -1416,7 +1423,7 @@ : MULTI_WINNER_MODE_OFF; // Disable Self-guided Loop restoration filter. - sf->lpf_sf.disable_sgr_filter = true; + sf->lpf_sf.enable_sgr_ep_pruning = 2; sf->lpf_sf.disable_wiener_coeff_refine_search = true; sf->tpl_sf.prune_starting_mv = 3; @@ -1436,9 +1443,9 @@ sf->gm_sf.downsample_level = 2; - sf->inter_sf.prune_inter_modes_based_on_tpl = boosted ? 0 : 3; + sf->inter_sf.prune_inter_modes_based_on_tpl = boosted ? 1 : 4; sf->inter_sf.selective_ref_frame = 6; - sf->inter_sf.prune_single_ref = is_boosted_arf2_bwd_type ? 0 : 2; + sf->inter_sf.prune_single_ref = is_boosted_arf2_bwd_type ? 0 : 3; sf->inter_sf.prune_ext_comp_using_neighbors = 3; sf->intra_sf.chroma_intra_pruning_with_hog = 4; @@ -1455,7 +1462,6 @@ sf->part_sf.prune_part4_search = 3; sf->mv_sf.simple_motion_subpel_force_stop = FULL_PEL; - sf->mv_sf.use_bsize_dependent_search_method = 1; sf->tpl_sf.gop_length_decision_method = 3; @@ -1474,6 +1480,32 @@ sf->tx_sf.adaptive_txb_search_level = 0; sf->tx_sf.tx_type_search.use_skip_flag_prediction = 0; } + + // Set speed features for the IQ and SSIMULACRA2 tuning modes + // Layered image encoding has different requirements than regular video + // coding. + // Mainly, most of these speed features undo an implicit assumption that + // keyframes are encoded at a better quality than inter-coded frames. + // This means the encoder needs to be more thorough at considering and + // performing RDO on intra block candidates vs. inter block candidates for + // the best compression efficiency. + // Finally, enabling certain coding tools are beneficial for layered image + // encoding in general. + if (cpi->oxcf.tune_cfg.tuning == AOM_TUNE_IQ || + cpi->oxcf.tune_cfg.tuning == AOM_TUNE_SSIMULACRA2) { + sf->intra_sf.skip_intra_in_interframe = 0; + sf->inter_sf.inter_mode_rd_model_estimation = 0; + sf->mv_sf.use_intrabc = 1; + + // Don't prune intra candidates too aggressively, as it can cause more + // expensive inter candidates to be chosen instead + if (sf->intra_sf.intra_pruning_with_hog > 3) { + sf->intra_sf.intra_pruning_with_hog = 3; + } + if (sf->intra_sf.chroma_intra_pruning_with_hog > 3) { + sf->intra_sf.chroma_intra_pruning_with_hog = 3; + } + } } static void set_rt_speed_feature_framesize_dependent(const AV1_COMP *const cpi, @@ -1703,7 +1735,7 @@ if (cpi->oxcf.tune_cfg.content == AOM_CONTENT_SCREEN) { // TODO(marpan): Check settings for speed 7 and 8. if (speed >= 7) { - sf->rt_sf.reduce_mv_pel_precision_highmotion = 1; + sf->rt_sf.reduce_mv_pel_precision_highmotion = 0; sf->mv_sf.use_bsize_dependent_search_method = 0; sf->rt_sf.skip_cdef_sb = 1; sf->rt_sf.increase_color_thresh_palette = 1; @@ -1722,7 +1754,6 @@ sf->rt_sf.nonrd_prune_ref_frame_search = 3; sf->rt_sf.var_part_split_threshold_shift = 10; sf->mv_sf.subpel_search_method = SUBPEL_TREE_PRUNED_MORE; - sf->rt_sf.reduce_mv_pel_precision_highmotion = 3; sf->rt_sf.reduce_mv_pel_precision_lowcomplex = 1; sf->lpf_sf.cdef_pick_method = CDEF_PICK_FROM_Q; sf->rt_sf.nonrd_check_partition_merge_mode = 0; @@ -1740,7 +1771,7 @@ sf->rt_sf.prune_palette_search_nonrd = 1; sf->rt_sf.prune_intra_mode_using_best_sad_so_far = true; sf->rt_sf.rc_faster_convergence_static = 1; - sf->rt_sf.rc_compute_spatial_var_sc = 1; + sf->rt_sf.rc_compute_spatial_var_sc_kf = 1; } if (speed >= 11) { sf->rt_sf.skip_lf_screen = 2; @@ -1826,10 +1857,42 @@ // temporal filtering which may change the input source during encoding: // this causes an issue on resized frames when psnr is calculated, // so disable it here for frames that are resized (encoding width/height - // different from configured width/height). - if (is_psnr_calc_enabled(cpi) && (cpi->oxcf.frm_dim_cfg.width != cm->width || - cpi->oxcf.frm_dim_cfg.height != cm->height)) + // different from configured width/height). Also disable for spatial layers. + // Bug: 491358681 + if ((is_psnr_calc_enabled(cpi) && + (cpi->oxcf.frm_dim_cfg.width != cm->width || + cpi->oxcf.frm_dim_cfg.height != cm->height)) || + cpi->svc.number_spatial_layers > 1) sf->rt_sf.use_rtc_tf = 0; + + // This speed feature is causing artifacts with active_maps enabled, so + // disable for now. + if (cpi->active_map.enabled) + sf->rt_sf.set_zeromv_skip_based_on_source_sad = 0; + + if (is_one_pass_rt_lag_params(cpi)) { + const RefreshFrameInfo *const refresh_frame = &cpi->refresh_frame; + if (refresh_frame->alt_ref_frame) { + sf->rt_sf.source_metrics_sb_nonrd = 0; + sf->rt_sf.var_part_based_on_qidx = 0; + } + sf->rt_sf.use_nonrd_altref_frame = 1; + // For non-zero lag: disable the 3 speed features below for now, + // until further testing. + sf->rt_sf.use_rtc_tf = 0; + sf->rt_sf.nonrd_check_partition_merge_mode = 0; + sf->rt_sf.nonrd_check_partition_split = 0; + // These (nonrd) speed features that force zeromv-LAST early in partition + // are disabled since for src_frame_alt_ref frame the zeromv-ALTREF_FRAME + // mode is forced in the nonrd_pickmode. + if (cpi->rc.is_src_frame_alt_ref) { + sf->rt_sf.increase_source_sad_thresh = 0; + sf->rt_sf.part_early_exit_zeromv = 0; + } + // This feature is for CBR mode, turning if off means the gop interval + // will not be changed after encoding. + sf->rt_sf.gf_refresh_based_on_qp = 0; + } } static void set_rt_speed_features_framesize_independent(AV1_COMP *cpi, @@ -1940,7 +2003,7 @@ sf->tx_sf.inter_tx_size_search_init_depth_sqr = 1; sf->tx_sf.tx_type_search.prune_2d_txfm_mode = TX_TYPE_PRUNE_3; sf->tx_sf.refine_fast_tx_search_results = 0; - sf->tx_sf.tx_type_search.fast_intra_tx_type_search = 1; + sf->tx_sf.tx_type_search.fast_intra_tx_type_search = 2; sf->tx_sf.tx_type_search.use_skip_flag_prediction = 2; sf->tx_sf.tx_type_search.winner_mode_tx_type_pruning = 4; @@ -1968,7 +2031,7 @@ sf->rt_sf.mode_search_skip_flags |= FLAG_SKIP_INTRA_DIRMISMATCH; sf->rt_sf.num_inter_modes_for_tx_search = 5; sf->rt_sf.prune_inter_modes_using_temp_var = 1; - sf->rt_sf.use_real_time_ref_set = 1; + sf->rt_sf.use_real_time_ref_set = is_one_pass_rt_lag_params(cpi) ? 0 : 1; sf->rt_sf.use_simple_rd_model = 1; sf->rt_sf.prune_inter_modes_with_golden_ref = boosted ? 0 : 1; // TODO(any): This sf could be removed. @@ -1997,6 +2060,13 @@ sf->rt_sf.use_fast_fixed_part = 0; sf->rt_sf.increase_source_sad_thresh = 0; + if (is_one_pass_rt_lag_params(cpi) && speed <= 6) { + sf->hl_sf.frame_parameter_update = 1; + sf->inter_sf.use_dist_wtd_comp_flag = 0; + sf->inter_sf.disable_masked_comp = 1; + sf->inter_sf.disable_onesided_comp = 1; + } + if (speed >= 6) { sf->mv_sf.use_fullpel_costlist = 1; @@ -2096,7 +2166,7 @@ sf->rt_sf.var_part_based_on_qidx = 4; sf->rt_sf.partition_direct_merging = 1; sf->rt_sf.prune_compoundmode_with_singlemode_var = false; - sf->mv_sf.use_bsize_dependent_search_method = 2; + sf->mv_sf.use_bsize_dependent_search_method = 4; sf->rt_sf.prune_hv_pred_modes_using_src_sad = true; } if (speed >= 9) { @@ -2124,6 +2194,11 @@ cpi->oxcf.tune_cfg.content == AOM_CONTENT_SCREEN) { sf->winner_mode_sf.dc_blk_pred_level = 3; } + + if (cpi->oxcf.tune_cfg.tuning == AOM_TUNE_IQ || + cpi->oxcf.tune_cfg.tuning == AOM_TUNE_SSIMULACRA2) { + sf->intra_sf.skip_intra_in_interframe = 0; + } } static inline void init_hl_sf(HIGH_LEVEL_SPEED_FEATURES *hl_sf) { @@ -2152,7 +2227,7 @@ } static inline void init_tpl_sf(TPL_SPEED_FEATURES *tpl_sf) { - tpl_sf->gop_length_decision_method = 0; + tpl_sf->gop_length_decision_method = 1; tpl_sf->prune_intra_modes = 0; tpl_sf->prune_starting_mv = 0; tpl_sf->reduce_first_step_size = 0; @@ -2200,6 +2275,7 @@ -1; // -1 means not enabled. } part_sf->ml_partition_search_breakout_model_index = 0; + part_sf->ml_4_partition_search_level_index = 0; part_sf->simple_motion_search_prune_agg = SIMPLE_AGG_LVL0; part_sf->simple_motion_search_split = 0; part_sf->simple_motion_search_prune_rect = 0; @@ -2243,7 +2319,7 @@ mv_sf->use_downsampled_sad = 0; mv_sf->disable_extensive_joint_motion_search = 0; mv_sf->disable_second_mv = 0; - mv_sf->skip_fullpel_search_using_startmv = 0; + mv_sf->skip_fullpel_search_using_startmv_refmv = 0; mv_sf->warp_search_method = WARP_SEARCH_SQUARE; mv_sf->warp_search_iters = 8; mv_sf->use_intrabc = 1; @@ -2301,6 +2377,7 @@ inter_sf->limit_txfm_eval_per_mode = 0; inter_sf->skip_arf_compound = 0; inter_sf->bias_warp_mode_rd_scale_pct = 0; + inter_sf->bias_obmc_mode_rd_scale_pct = 0.0f; set_txfm_rd_gate_level(inter_sf->txfm_rd_gate_level, 0); } @@ -2420,6 +2497,7 @@ lpf_sf->lpf_pick = LPF_PICK_FROM_FULL_IMAGE; lpf_sf->use_coarse_filter_level_search = 0; lpf_sf->cdef_pick_method = CDEF_FULL_SEARCH; + lpf_sf->zero_low_cdef_strengths = 0; // Set decoder side speed feature to use less dual sgr modes lpf_sf->dual_sgr_penalty_level = 0; // Enable Wiener and Self-guided Loop restoration filters by default. @@ -2454,7 +2532,7 @@ rt_sf->overshoot_detection_cbr = NO_DETECTION; rt_sf->check_scene_detection = 0; rt_sf->rc_adjust_keyframe = 0; - rt_sf->rc_compute_spatial_var_sc = 0; + rt_sf->rc_compute_spatial_var_sc_kf = 0; rt_sf->prefer_large_partition_blocks = 0; rt_sf->use_temporal_noise_estimate = 0; rt_sf->fullpel_search_step_param = 0; @@ -2742,31 +2820,15 @@ if (speed < 1 || speed > 3) return; const AV1_COMMON *const cm = &cpi->common; - const bool is_between_608p_and_720p = AOMMIN(cm->width, cm->height) >= 608 && - AOMMIN(cm->width, cm->height) <= 720; - const bool is_720p_or_larger = AOMMIN(cm->width, cm->height) >= 720; + const bool is_between_608p_and_1080p = AOMMIN(cm->width, cm->height) >= 608 && + AOMMIN(cm->width, cm->height) <= 1080; const bool is_vertical_video = cm->width < cm->height; - const FRAME_UPDATE_TYPE update_type = - get_frame_update_type(&cpi->ppi->gf_group, cpi->gf_frame_index); - const bool leaf_and_overlay_frames = - (update_type == LF_UPDATE || update_type == OVERLAY_UPDATE || - update_type == INTNL_OVERLAY_UPDATE); // Speed features for vertical videos - if (is_vertical_video && is_between_608p_and_720p) { + if (is_vertical_video && is_between_608p_and_1080p) { sf->lpf_sf.min_lr_unit_size = RESTORATION_UNITSIZE_MAX >> 1; sf->lpf_sf.max_lr_unit_size = RESTORATION_UNITSIZE_MAX >> 1; } - - // Speed features for regular videos - if (!is_vertical_video && is_720p_or_larger) { - if (speed <= 2 && leaf_and_overlay_frames) { - // For 720p and above, only enable this feature for leaf and overlay - // frames to avoid quality degradation on ARF frames. - sf->lpf_sf.min_lr_unit_size = RESTORATION_UNITSIZE_MAX >> 1; - sf->lpf_sf.max_lr_unit_size = RESTORATION_UNITSIZE_MAX >> 1; - } - } } // Override some speed features based on qindex @@ -2783,6 +2845,13 @@ const int is_arf2_bwd_type = cpi->ppi->gf_group.update_type[cpi->gf_frame_index] == INTNL_ARF_UPDATE; + if (cpi->oxcf.mode == ALLINTRA || cpi->oxcf.tune_cfg.tuning == AOM_TUNE_IQ || + cpi->oxcf.tune_cfg.tuning == AOM_TUNE_SSIMULACRA2) { + if (cm->quant_params.base_qindex <= 140) { + sf->lpf_sf.zero_low_cdef_strengths = 1; + } + } + if (cpi->oxcf.mode == REALTIME) { if (speed >= 6) { const int qindex_thresh = boosted ? 190 : (is_720p_or_larger ? 120 : 150); @@ -2865,9 +2934,9 @@ } } - if (speed >= 4) { + if (speed >= 3) { // Disable rectangular partitions for lower quantizers - const int aggr = AOMMIN(1, speed - 4); + const int aggr = (speed <= 4) ? 0 : 1; const int qindex_thresh[2] = { 65, 80 }; int disable_rect_part; disable_rect_part = !boosted; @@ -2894,8 +2963,13 @@ sf->mv_sf.search_method = NSTEP_8PT; } } + sf->part_sf.less_rectangular_check_level = 1; } + if (speed == 3) + sf->part_sf.less_rectangular_check_level = + (cm->quant_params.base_qindex >= 170) ? 1 : 2; + if (speed >= 4) { // Disable LR search at low and high quantizers and enable only for // mid-quantizer range. @@ -2904,9 +2978,11 @@ const int qindex_high[2] = { 180, 160 }; if (cm->quant_params.base_qindex <= qindex_low[is_720p_or_larger] || cm->quant_params.base_qindex > qindex_high[is_720p_or_larger]) { - sf->lpf_sf.disable_loop_restoration_luma = 1; + sf->lpf_sf.disable_sgr_filter = true; + sf->lpf_sf.disable_wiener_coeff_refine_search = true; } } + sf->part_sf.less_rectangular_check_level = 2; } if (speed == 1) {
diff --git a/av1/encoder/speed_features.h b/av1/encoder/speed_features.h index 01f55cd..3d81c3b 100644 --- a/av1/encoder/speed_features.h +++ b/av1/encoder/speed_features.h
@@ -256,6 +256,11 @@ typedef struct { TX_TYPE_PRUNE_MODE prune_2d_txfm_mode; + + // Limit the intra transform search type. + // 1 : Limit the intra transform search type to the ones in the table + // av1_derived_intra_tx_used_flag[INTRA_MODES]. + // 2 : Limit the intra transform search type to the default transform. int fast_intra_tx_type_search; // INT_MAX: Disable fast search. @@ -673,6 +678,9 @@ // ML based partition search breakout model index int ml_partition_search_breakout_model_index; + // ML based partition search breakout model index + int ml_4_partition_search_level_index; + // Aggressiveness levels for pruning split and rectangular partitions based on // simple_motion_search. SIMPLE_AGG_LVL0 to SIMPLE_AGG_LVL5 correspond to // simple motion search based pruning. QIDX_BASED_AGG_LVL1 corresponds to @@ -858,7 +866,9 @@ SEARCH_METHODS search_method; // Enable the use of faster, less accurate mv search method - // 0: disable, 1: if bsize >= BLOCK_32X32, 2: based on bsize, SAD and qp + // 0: Disable + // 1 - 3: Disable for larger bsize + // 4: Based on bsize, SAD and qp // TODO(chiyotsai@google.com): Take the clip's resolution and mv activity into // account. int use_bsize_dependent_search_method; @@ -953,11 +963,14 @@ // 2: disable second MV int disable_second_mv; - // Skips full pixel search based on start mv of prior ref_mv_idx. + // Skips full pixel search based on closeness of start mv and ref mv + // of previous search. // 0: Disabled - // 1: Skips the full pixel search upto 4 neighbor full-pel MV positions. - // 2: Skips the full pixel search upto 8 neighbor full-pel MV positions. - int skip_fullpel_search_using_startmv; + // 1: Skips the full pixel search upto 4 neighbor full-pel start MV and ref MV + // positions. + // 2: Skips the full pixel search upto 8 neighbor full-pel start MV and ref MV + // positions. + int skip_fullpel_search_using_startmv_refmv; // Method to use for refining WARPED_CAUSAL motion vectors // TODO(rachelbarker): Can this be unified with OBMC in some way? @@ -1010,7 +1023,7 @@ int alt_ref_search_fp; // Prune reference frames for single prediction modes based on temporal - // distance and pred MV SAD. Feasible values are 0, 1, 2. The feature is + // distance and pred MV SAD. Feasible values are 0, 1, 2, 3. The feature is // disabled for 0. An increasing value indicates more aggressive pruning // threshold. int prune_single_ref; @@ -1146,7 +1159,9 @@ // Prune inter modes based on tpl stats // 0 : no pruning - // 1 - 3 indicate increasing aggressiveness in order. + // 1 : Allow pruning of LAST2 frame + // 2 - 4: Allow pruning of all reference frames with increased aggressiveness + // of pruning in order int prune_inter_modes_based_on_tpl; // Skip NEARMV and NEAR_NEARMV modes using ref frames of above and left @@ -1219,6 +1234,11 @@ // encoder decisions are biased against local warp, favoring low complexity // modes. int bias_warp_mode_rd_scale_pct; + + // Percentage of scaling used to increase the rd cost of obmc motion mode so + // that encoder decisions are biased against local obmc, favoring low + // complexity modes. + float bias_obmc_mode_rd_scale_pct; } INTER_MODE_SPEED_FEATURES; typedef struct INTERP_FILTER_SPEED_FEATURES { @@ -1560,6 +1580,12 @@ // Control how the CDEF strength is determined. CDEF_PICK_METHOD cdef_pick_method; + // Decoder side speed feature for adaptive CDEF with strength reduction. + // Zero out values with low CDEF strengths (luma and/or chroma). This is + // done as CDEF is a relatively-expensive filter to compute during decode. + // This speed feature is only enabled in all intra mode. + bool zero_low_cdef_strengths; + // Decoder side speed feature to add penalty for use of dual-sgr filters. // Takes values 0 - 10, 0 indicating no penalty and each additional level // adding a penalty of 1% @@ -1574,7 +1600,14 @@ // search_switchable() int switchable_lr_with_bias_level; - // prune sgr ep using binary search like mechanism + // Enable fast search in self guided restoration + // 0 : Search over all 16 SGR projection parameters listed + // in av1_sgr_params[SGRPROJ_PARAMS]. + // 1 : Approximate search using binary search like mechanism, + // a total of 8 SGR projection parameters are searched. + // 2 : Search only 'ep' values in + // sgproj_ep_grp1_seed[SGRPROJ_EP_GRP1_SEARCH_COUNT], + // a total of 4 SGR projection parameters are searched. int enable_sgr_ep_pruning; // Disable loop restoration for Chroma plane @@ -1724,8 +1757,8 @@ // For keyframes in rtc: adjust the rc_bits_per_mb, to reduce overshoot. int rc_adjust_keyframe; - // On scene change: compute spatial variance. - int rc_compute_spatial_var_sc; + // On scene change or keyframe: compute spatial variance. + int rc_compute_spatial_var_sc_kf; // For nonrd mode: Prefer larger partition blks in variance based partitioning // 0: disabled, 1-3: increasing aggressiveness
diff --git a/av1/encoder/temporal_filter.c b/av1/encoder/temporal_filter.c index 1fab6bf..daab015 100644 --- a/av1/encoder/temporal_filter.c +++ b/av1/encoder/temporal_filter.c
@@ -48,8 +48,90 @@ // Forward Declaration. static void tf_determine_block_partition(const MV block_mv, const int block_mse, + const MV *midblock_mvs, + const int *midblock_mses, MV *subblock_mvs, int *subblock_mses); +// Do motion search for 1 subblock. The block size can be 32x32 or 16x16. +static void subblock_motion_search( + AV1_COMP *cpi, MACROBLOCK *mb, const YV12_BUFFER_CONFIG *frame_to_filter, + const YV12_BUFFER_CONFIG *ref_frame, BLOCK_SIZE tf_block_size, int mb_row, + int mb_col, BLOCK_SIZE subblock_size, int idx, int subblock_ofst_i, + int subblock_ofst_j, FULLPEL_MOTION_SEARCH_PARAMS *full_ms_params, + SUBPEL_MOTION_SEARCH_PARAMS *ms_params, const MV *ref_mv, + FULLPEL_MV start_mv, const search_site_config *search_site_cfg, + SEARCH_METHODS search_method, MV_COST_TYPE mv_cost_type, int step_param, + SUBPEL_SEARCH_TYPE subpel_ms_type, MV *subblock_mvs, int *subblock_mses, + FULLPEL_MV_STATS *best_mv_stats, int q) { + MACROBLOCKD *const mbd = &mb->e_mbd; + + const int mb_height = block_size_high[tf_block_size]; + const int mb_width = block_size_wide[tf_block_size]; + const int mi_h = mi_size_high_log2[tf_block_size]; + const int mi_w = mi_size_wide_log2[tf_block_size]; + + const int y_stride = frame_to_filter->y_stride; + assert(y_stride == ref_frame->y_stride); + const int y_offset = mb_row * mb_height * y_stride + mb_col * mb_width; + + const int subblock_height = block_size_high[subblock_size]; + const int subblock_width = block_size_wide[subblock_size]; + const int subblock_pels = subblock_height * subblock_width; + + int_mv best_mv; // Searched motion vector. + unsigned int sse, error; + int distortion; + int cost_list[5]; + + av1_set_mv_row_limits(&cpi->common.mi_params, &mb->mv_limits, + (mb_row << mi_h) + (subblock_ofst_i >> MI_SIZE_LOG2), + (subblock_height >> MI_SIZE_LOG2), + cpi->oxcf.border_in_pixels); + av1_set_mv_col_limits(&cpi->common.mi_params, &mb->mv_limits, + (mb_col << mi_w) + (subblock_ofst_j >> MI_SIZE_LOG2), + (subblock_width >> MI_SIZE_LOG2), + cpi->oxcf.border_in_pixels); + const int boffset = subblock_ofst_i * y_stride + subblock_ofst_j; + mb->plane[0].src.buf = frame_to_filter->y_buffer + y_offset + boffset; + mbd->plane[0].pre[0].buf = ref_frame->y_buffer + y_offset + boffset; + av1_make_default_fullpel_ms_params(full_ms_params, cpi, mb, subblock_size, + ref_mv, start_mv, search_site_cfg, + search_method, + /*fine_search_interval=*/0); + full_ms_params->run_mesh_search = 1; + full_ms_params->mv_cost_params.mv_cost_type = mv_cost_type; + + // May need to re-tune prune_mesh_search after increase TF + // block to 64x64. + if (cpi->sf.mv_sf.prune_mesh_search == PRUNE_MESH_SEARCH_LVL_1) { + // Enable prune_mesh_search based on q for PRUNE_MESH_SEARCH_LVL_1. + full_ms_params->prune_mesh_search = (q <= 20) ? 0 : 1; + full_ms_params->mesh_search_mv_diff_threshold = 2; + } + + av1_full_pixel_search(start_mv, full_ms_params, step_param, + cond_cost_list(cpi, cost_list), &best_mv.as_fullmv, + best_mv_stats, NULL); + + av1_make_default_subpel_ms_params(ms_params, cpi, mb, subblock_size, ref_mv, + cost_list); + ms_params->forced_stop = EIGHTH_PEL; + ms_params->var_params.subpel_search_type = subpel_ms_type; + // Since we are merely refining the result from full pixel + // search, we don't need regularization for subpel search + ms_params->mv_cost_params.mv_cost_type = MV_COST_NONE; + best_mv_stats->err_cost = 0; + + MV subpel_start_mv = get_mv_from_fullmv(&best_mv.as_fullmv); + assert(av1_is_subpelmv_in_range(&ms_params->mv_limits, subpel_start_mv)); + error = cpi->mv_search_params.find_fractional_mv_step( + &mb->e_mbd, &cpi->common, ms_params, subpel_start_mv, best_mv_stats, + &best_mv.as_mv, &distortion, &sse, NULL); + + subblock_mses[idx] = DIVIDE_AND_ROUND(error, subblock_pels); + subblock_mvs[idx] = best_mv.as_mv; +} + // This function returns the minimum and maximum log variances for 4x4 sub // blocks in the current block. static inline void get_log_var_4x4sub_blk( @@ -87,6 +169,13 @@ static int get_q(const AV1_COMP *cpi) { const GF_GROUP *gf_group = &cpi->ppi->gf_group; const FRAME_TYPE frame_type = gf_group->frame_type[cpi->gf_frame_index]; + + if (cpi->oxcf.rc_cfg.mode == AOM_Q) { + int cq_level = cpi->oxcf.rc_cfg.cq_level; + return (int)av1_convert_qindex_to_q(cq_level, + cpi->common.seq_params->bit_depth); + } + const int q = (int)av1_convert_qindex_to_q(cpi->ppi->p_rc.avg_frame_qindex[frame_type], cpi->common.seq_params->bit_depth); @@ -229,6 +318,10 @@ FULLPEL_MV_STATS best_mv_stats; int block_mse = INT_MAX; MV block_mv = kZeroMv; + // 32x32 block motion search results. + int midblock_mses[4] = { INT_MAX, INT_MAX, INT_MAX, INT_MAX }; + MV midblock_mvs[4] = { kZeroMv, kZeroMv, kZeroMv, kZeroMv }; + const int q = get_q(cpi); av1_make_default_fullpel_ms_params(&full_ms_params, cpi, mb, block_size, @@ -282,54 +375,49 @@ *is_dc_diff_large = 50 * error < sse; if (src_var <= 2 * (int64_t)distortion) *is_low_cntras = 1; + // On 4 mid-blocks in the 64x64 tf block. if (allow_me_for_sub_blks) { - // On 4 sub-blocks. - const BLOCK_SIZE subblock_size = av1_ss_size_lookup[block_size][1][1]; - const int subblock_height = block_size_high[subblock_size]; - const int subblock_width = block_size_wide[subblock_size]; - const int subblock_pels = subblock_height * subblock_width; - start_mv = get_fullmv_from_mv(ref_mv); + // midblock_size is 32x32, which corresponds to each 32x32 block in the + // 64x64 block. + const BLOCK_SIZE midblock_size = av1_ss_size_lookup[block_size][1][1]; + const int midblock_height = block_size_high[midblock_size]; + const int midblock_width = block_size_wide[midblock_size]; + int midblock_idx = 0; + for (int i = 0; i < mb_height; i += midblock_height) { + for (int j = 0; j < mb_width; j += midblock_width) { + start_mv = get_fullmv_from_mv(ref_mv); - int subblock_idx = 0; - for (int i = 0; i < mb_height; i += subblock_height) { - for (int j = 0; j < mb_width; j += subblock_width) { - const int offset = i * y_stride + j; - mb->plane[0].src.buf = frame_to_filter->y_buffer + y_offset + offset; - mbd->plane[0].pre[0].buf = ref_frame->y_buffer + y_offset + offset; - av1_make_default_fullpel_ms_params( - &full_ms_params, cpi, mb, subblock_size, &baseline_mv, start_mv, - search_site_cfg, search_method, - /*fine_search_interval=*/0); - full_ms_params.run_mesh_search = 1; - full_ms_params.mv_cost_params.mv_cost_type = mv_cost_type; + subblock_motion_search( + cpi, mb, frame_to_filter, ref_frame, block_size, mb_row, mb_col, + midblock_size, midblock_idx, i, j, &full_ms_params, &ms_params, + &baseline_mv, start_mv, search_site_cfg, search_method, + mv_cost_type, step_param, subpel_search_type, midblock_mvs, + midblock_mses, &best_mv_stats, q); - if (cpi->sf.mv_sf.prune_mesh_search == PRUNE_MESH_SEARCH_LVL_1) { - // Enable prune_mesh_search based on q for PRUNE_MESH_SEARCH_LVL_1. - full_ms_params.prune_mesh_search = (q <= 20) ? 0 : 1; - full_ms_params.mesh_search_mv_diff_threshold = 2; + // On 4 sub-blocks in 1 mid-block. + { + const BLOCK_SIZE subblock_size = + av1_ss_size_lookup[midblock_size][1][1]; + const int subblock_height = block_size_high[subblock_size]; + const int subblock_width = block_size_wide[subblock_size]; + + start_mv = get_fullmv_from_mv(&midblock_mvs[midblock_idx]); + + int bidx = midblock_idx * 4; + for (int bi = 0; bi < midblock_height; bi += subblock_height) { + for (int bj = 0; bj < midblock_width; bj += subblock_width) { + subblock_motion_search( + cpi, mb, frame_to_filter, ref_frame, block_size, mb_row, + mb_col, subblock_size, bidx, (i + bi), (j + bj), + &full_ms_params, &ms_params, &baseline_mv, start_mv, + search_site_cfg, search_method, mv_cost_type, step_param, + subpel_search_type, subblock_mvs, subblock_mses, + &best_mv_stats, q); + ++bidx; + } + } } - av1_full_pixel_search(start_mv, &full_ms_params, step_param, - cond_cost_list(cpi, cost_list), - &best_mv.as_fullmv, &best_mv_stats, NULL); - - av1_make_default_subpel_ms_params(&ms_params, cpi, mb, subblock_size, - &baseline_mv, cost_list); - ms_params.forced_stop = EIGHTH_PEL; - ms_params.var_params.subpel_search_type = subpel_search_type; - // Since we are merely refining the result from full pixel search, we - // don't need regularization for subpel search - ms_params.mv_cost_params.mv_cost_type = MV_COST_NONE; - best_mv_stats.err_cost = 0; - - subpel_start_mv = get_mv_from_fullmv(&best_mv.as_fullmv); - assert( - av1_is_subpelmv_in_range(&ms_params.mv_limits, subpel_start_mv)); - error = cpi->mv_search_params.find_fractional_mv_step( - &mb->e_mbd, &cpi->common, &ms_params, subpel_start_mv, - &best_mv_stats, &best_mv.as_mv, &distortion, &sse, NULL); - subblock_mses[subblock_idx] = DIVIDE_AND_ROUND(error, subblock_pels); - subblock_mvs[subblock_idx] = best_mv.as_mv; - ++subblock_idx; + ++midblock_idx; } } } @@ -341,11 +429,11 @@ // Make partition decision. if (allow_me_for_sub_blks) { - tf_determine_block_partition(block_mv, block_mse, subblock_mvs, - subblock_mses); + tf_determine_block_partition(block_mv, block_mse, midblock_mvs, + midblock_mses, subblock_mvs, subblock_mses); } else { - // Copy 32X32 block mv and mse values to sub blocks - for (int i = 0; i < 4; ++i) { + // Copy 64X64 block mv and mse values to sub blocks + for (int i = 0; i < NUM_16X16; ++i) { subblock_mvs[i] = block_mv; subblock_mses[i] = block_mse; } @@ -365,6 +453,8 @@ // block_mv: Motion vector for the entire block (ONLY as reference). // block_mse: Motion search error (MSE) for the entire block (ONLY as // reference). +// midblock_mvs: Pointer to the motion vectors for 4 mid-blocks. +// midblock_mses: Pointer to the search errors (MSE) for 4 mid-blocks. // subblock_mvs: Pointer to the motion vectors for 4 sub-blocks (will be // modified based on the partition decision). // subblock_mses: Pointer to the search errors (MSE) for 4 sub-blocks (will @@ -373,23 +463,53 @@ // Nothing will be returned. Results are saved in `subblock_mvs` and // `subblock_mses`. static void tf_determine_block_partition(const MV block_mv, const int block_mse, + const MV *midblock_mvs, + const int *midblock_mses, MV *subblock_mvs, int *subblock_mses) { int min_subblock_mse = INT_MAX; int max_subblock_mse = INT_MIN; int64_t sum_subblock_mse = 0; - for (int i = 0; i < 4; ++i) { + int i; + + // Go through 4 32x32 blocks. + for (int idx = 0; idx < 4; ++idx) { + min_subblock_mse = INT_MAX; + max_subblock_mse = INT_MIN; + sum_subblock_mse = 0; + + const int sub_idx = idx * 4; + for (i = sub_idx; i < sub_idx + 4; ++i) { + sum_subblock_mse += subblock_mses[i]; + min_subblock_mse = AOMMIN(min_subblock_mse, subblock_mses[i]); + max_subblock_mse = AOMMAX(max_subblock_mse, subblock_mses[i]); + } + + if (((midblock_mses[idx] * 15 <= sum_subblock_mse * 4) && + max_subblock_mse - min_subblock_mse < 48) || + ((midblock_mses[idx] * 14 <= sum_subblock_mse * 4) && + max_subblock_mse - min_subblock_mse < 24)) { // No split. + for (i = sub_idx; i < sub_idx + 4; ++i) { + subblock_mvs[i] = midblock_mvs[idx]; + subblock_mses[i] = midblock_mses[idx]; + } + } + } + + min_subblock_mse = INT_MAX; + max_subblock_mse = INT_MIN; + sum_subblock_mse = 0; + for (i = 0; i < NUM_16X16; ++i) { sum_subblock_mse += subblock_mses[i]; min_subblock_mse = AOMMIN(min_subblock_mse, subblock_mses[i]); max_subblock_mse = AOMMAX(max_subblock_mse, subblock_mses[i]); } - // TODO(any): The following magic numbers may be tuned to improve the - // performance OR find a way to get rid of these magic numbers. - if (((block_mse * 15 < sum_subblock_mse * 4) && - max_subblock_mse - min_subblock_mse < 48) || - ((block_mse * 14 < sum_subblock_mse * 4) && - max_subblock_mse - min_subblock_mse < 24)) { // No split. - for (int i = 0; i < 4; ++i) { + if (((block_mse * 15 <= sum_subblock_mse) && + (max_subblock_mse - min_subblock_mse) * 16 < sum_subblock_mse * 3) || + ((block_mse * 14 <= sum_subblock_mse) && + (max_subblock_mse - min_subblock_mse) * 8 < + sum_subblock_mse)) { // No split. + for (i = 0; i < NUM_16X16; ++i) { subblock_mvs[i] = block_mv; subblock_mses[i] = block_mse; } @@ -459,35 +579,45 @@ const int plane_w = mb_width >> subsampling_x; // Plane width. const int plane_y = mb_y >> subsampling_y; // Y-coord (Top-left). const int plane_x = mb_x >> subsampling_x; // X-coord (Top-left). - const int h = plane_h >> 1; // Sub-block height. - const int w = plane_w >> 1; // Sub-block width. - const int is_y_plane = (plane == 0); // Is Y-plane? + + const int h32 = plane_h >> 1; // 32x32 sub-block height. + const int w32 = plane_w >> 1; // 32x32 sub-block width. + const int h16 = plane_h >> 2; // 16x16 sub-block height. + const int w16 = plane_w >> 2; // 16x16 sub-block width. + + const int is_y_plane = (plane == 0); // Is Y-plane? const struct buf_2d ref_buf = { NULL, ref_frame->buffers[plane], ref_frame->widths[is_y_plane ? 0 : 1], ref_frame->heights[is_y_plane ? 0 : 1], ref_frame->strides[is_y_plane ? 0 : 1] }; - // Handle each subblock. - int subblock_idx = 0; - for (int i = 0; i < plane_h; i += h) { - for (int j = 0; j < plane_w; j += w) { - // Choose proper motion vector. - const MV mv = subblock_mvs[subblock_idx++]; - assert(mv.row >= INT16_MIN && mv.row <= INT16_MAX && - mv.col >= INT16_MIN && mv.col <= INT16_MAX); + const int sub_y[4] = { 0, 0, h32, h32 }; + const int sub_x[4] = { 0, w32, 0, w32 }; + // Handle each 16x16 subblock. + for (int idx = 0; idx < 4; ++idx) { + int subblock_idx = idx * 4; + for (int i = 0; i < h32; i += h16) { + for (int j = 0; j < w32; j += w16) { + // Choose proper motion vector. + const MV mv = subblock_mvs[subblock_idx++]; + assert(mv.row >= INT16_MIN && mv.row <= INT16_MAX && + mv.col >= INT16_MIN && mv.col <= INT16_MAX); - const int y = plane_y + i; - const int x = plane_x + j; + const int y = plane_y + sub_y[idx] + i; + const int x = plane_x + sub_x[idx] + j; - // Build predictior for each sub-block on current plane. - InterPredParams inter_pred_params; - av1_init_inter_params(&inter_pred_params, w, h, y, x, subsampling_x, - subsampling_y, bit_depth, is_high_bitdepth, - is_intrabc, scale, &ref_buf, interp_filters); - inter_pred_params.conv_params = get_conv_params(0, plane, bit_depth); - av1_enc_build_one_inter_predictor(&pred[plane_offset + i * plane_w + j], - plane_w, &mv, &inter_pred_params); + // Build predictior for each sub-block on current plane. + InterPredParams inter_pred_params; + av1_init_inter_params(&inter_pred_params, w16, h16, y, x, + subsampling_x, subsampling_y, bit_depth, + is_high_bitdepth, is_intrabc, scale, &ref_buf, + interp_filters); + inter_pred_params.conv_params = get_conv_params(0, plane, bit_depth); + av1_enc_build_one_inter_predictor( + &pred[plane_offset + (sub_y[idx] + i) * plane_w + sub_x[idx] + j], + plane_w, &mv, &inter_pred_params); + } } } plane_offset += plane_h * plane_w; @@ -704,8 +834,10 @@ const double n_decay = 0.5 + log(2 * noise_levels[plane] + 5.0); decay_factor[plane] = 1 / (n_decay * q_decay * s_decay); } - double d_factor[4] = { 0 }; - for (int subblock_idx = 0; subblock_idx < 4; subblock_idx++) { + + // Figure out each 16x16 block's d_factor beforehand. + double d_factor[NUM_16X16] = { 0 }; + for (int subblock_idx = 0; subblock_idx < NUM_16X16; subblock_idx++) { // Larger motion vector -> smaller filtering weight. const MV mv = subblock_mvs[subblock_idx]; const double distance = sqrt(pow(mv.row, 2) + pow(mv.col, 2)); @@ -789,7 +921,13 @@ // Combine window error and block error, and normalize it. const double window_error = sum_square_diff * inv_num_ref_pixels; - const int subblock_idx = (i >= h / 2) * 2 + (j >= w / 2); + + // 16x16 block index + const int y32 = i / (h / 2); + const int x32 = j / (w / 2); + const int y16 = (i % (h / 2)) / (h / 4); + const int x16 = (j % (w / 2)) / (w / 4); + const int subblock_idx = (y32 * 2 + x32) * 4 + (y16 * 2 + x16); const double block_error = (double)subblock_mses[subblock_idx]; const double combined_error = weight_factor * window_error + block_error * inv_factor; @@ -923,6 +1061,17 @@ const GF_GROUP *gf_group = &cpi->ppi->gf_group; const FRAME_TYPE frame_type = gf_group->frame_type[cpi->gf_frame_index]; + // Determine whether the video is with `YUV 4:2:2` format, since the avx2/sse2 + // function only supports square block size. We will use C function instead + // for videos with `YUV 4:2:2` format. + int is_yuv422_format = 0; + for (int plane = 1; plane < num_planes; ++plane) { + if (mbd->plane[plane].subsampling_x != mbd->plane[plane].subsampling_y) { + is_yuv422_format = 1; + break; + } + } + // Do filtering. FRAME_DIFF *diff = &td->tf_data.diff; av1_set_mv_row_limits(&cpi->common.mi_params, &mb->mv_limits, @@ -960,8 +1109,16 @@ if (frames[frame] == NULL) continue; // Motion search. - MV subblock_mvs[4] = { kZeroMv, kZeroMv, kZeroMv, kZeroMv }; - int subblock_mses[4] = { INT_MAX, INT_MAX, INT_MAX, INT_MAX }; + // block size is 64x64. 16 16x16 in 1 64x64. + // Store motion search results in 16x16 units. + MV subblock_mvs[NUM_16X16] = { kZeroMv, kZeroMv, kZeroMv, kZeroMv, + kZeroMv, kZeroMv, kZeroMv, kZeroMv, + kZeroMv, kZeroMv, kZeroMv, kZeroMv, + kZeroMv, kZeroMv, kZeroMv, kZeroMv }; + int subblock_mses[NUM_16X16] = { INT_MAX, INT_MAX, INT_MAX, INT_MAX, + INT_MAX, INT_MAX, INT_MAX, INT_MAX, + INT_MAX, INT_MAX, INT_MAX, INT_MAX, + INT_MAX, INT_MAX, INT_MAX, INT_MAX }; int is_dc_diff_large = 0; int is_low_cntras = 0; @@ -999,7 +1156,8 @@ // only supports 32x32 block size and 5x5 filtering window. if (is_frame_high_bitdepth(frame_to_filter)) { // for high bit-depth #if CONFIG_AV1_HIGHBITDEPTH - if (TF_BLOCK_SIZE == BLOCK_32X32 && TF_WINDOW_LENGTH == 5) { + if (!is_yuv422_format && TF_BLOCK_SIZE == BLOCK_32X32 && + TF_WINDOW_LENGTH == 5) { av1_highbd_apply_temporal_filter( frame_to_filter, mbd, block_size, mb_row, mb_col, num_planes, noise_levels, subblock_mvs, subblock_mses, q_factor, @@ -1015,7 +1173,8 @@ #endif // CONFIG_AV1_HIGHBITDEPTH } else { // for 8-bit - if (TF_BLOCK_SIZE == BLOCK_32X32 && TF_WINDOW_LENGTH == 5) { + if (!is_yuv422_format && TF_BLOCK_SIZE == BLOCK_64X64 && + TF_WINDOW_LENGTH == 5) { av1_apply_temporal_filter( frame_to_filter, mbd, block_size, mb_row, mb_col, num_planes, noise_levels, subblock_mvs, subblock_mses, q_factor, @@ -1206,10 +1365,10 @@ num_before = AOMMIN(is_forward_keyframe ? num_frames / 2 : 0, max_before); num_after = AOMMIN(num_frames - 1, max_after); } else { - int gfu_boost = av1_calc_arf_boost(&cpi->ppi->twopass, &cpi->twopass_frame, - &cpi->ppi->p_rc, &cpi->frame_info, - filter_frame_lookahead_idx, max_before, - max_after, NULL, NULL, 0); + int gfu_boost = av1_calc_arf_boost( + &cpi->ppi->twopass, &cpi->twopass_frame, &cpi->ppi->p_rc, + &cpi->frame_info, filter_frame_lookahead_idx, max_before, max_after, + NULL, NULL, 0, cpi->oxcf.mode != REALTIME); num_frames = AOMMIN(num_frames, gfu_boost / 150); num_frames += !(num_frames & 1); // Make the number odd. @@ -1430,7 +1589,10 @@ int av1_check_show_filtered_frame(const YV12_BUFFER_CONFIG *frame, const FRAME_DIFF *frame_diff, int q_index, - aom_bit_depth_t bit_depth) { + aom_bit_depth_t bit_depth, + int enable_overlay) { + if (!enable_overlay) return 1; + const int frame_height = frame->y_crop_height; const int frame_width = frame->y_crop_width; const int block_height = block_size_high[TF_BLOCK_SIZE];
diff --git a/av1/encoder/temporal_filter.h b/av1/encoder/temporal_filter.h index 9585942..5d380a2 100644 --- a/av1/encoder/temporal_filter.h +++ b/av1/encoder/temporal_filter.h
@@ -26,15 +26,18 @@ // TODO(wtc): These two variables are only used in avx2, sse2, neon // implementations, where the block size is still hard coded to TF_BLOCK_SIZE. // This should be fixed to align with the c implementation. -#define BH 32 -#define BW 32 +#define BH 64 +#define BW 64 // Block size used in temporal filtering. -#define TF_BLOCK_SIZE BLOCK_32X32 +#define TF_BLOCK_SIZE BLOCK_64X64 // Window size for temporal filtering. #define TF_WINDOW_LENGTH 5 +// Number of 16x16 blocks within one 64x64 TF block. +#define NUM_16X16 16 + // A constant number, sqrt(pi / 2), used for noise estimation. static const double SQRT_PI_BY_2 = 1.25331413732; @@ -338,17 +341,19 @@ * to make decision. * * \ingroup src_frame_proc - * \param[in] frame filtered frame's buffer - * \param[in] frame_diff structure of sse and sum of the - * filtered frame. - * \param[in] q_index q_index used for this frame - * \param[in] bit_depth bit depth + * \param[in] frame filtered frame's buffer + * \param[in] frame_diff structure of sse and sum of the + * filtered frame. + * \param[in] q_index q_index used for this frame + * \param[in] bit_depth bit depth + * \param[in] enable_overlay arf overlay is enabled or disabled * \return return 1 if this frame can be shown directly, otherwise * return 0 */ int av1_check_show_filtered_frame(const YV12_BUFFER_CONFIG *frame, const FRAME_DIFF *frame_diff, int q_index, - aom_bit_depth_t bit_depth); + aom_bit_depth_t bit_depth, + int enable_overlay); /*!\cond */ // Allocates memory for members of TemporalFilterData.
diff --git a/av1/encoder/tpl_model.c b/av1/encoder/tpl_model.c index a8c2f83..546d100 100644 --- a/av1/encoder/tpl_model.c +++ b/av1/encoder/tpl_model.c
@@ -200,6 +200,16 @@ aom_internal_error(&ppi->error, AOM_CODEC_MEM_ERROR, "Failed to allocate frame buffer"); } + + if (aom_alloc_frame_buffer( + &tpl_data->prev_gop_arf_src, width, height, seq_params->subsampling_x, + seq_params->subsampling_y, seq_params->use_highbitdepth, + tpl_data->border_in_pixels, byte_alignment, false, + alloc_y_plane_only)) + aom_internal_error(&ppi->error, AOM_CODEC_MEM_ERROR, + "Failed to allocate prev gop arf buffer"); + + tpl_data->prev_gop_arf_disp_order = -1; } static inline int32_t tpl_get_satd_cost(BitDepthInfo bd_info, int16_t *src_diff, @@ -1014,9 +1024,58 @@ tpl_stats->srcrf_dist = recon_error << TPL_DEP_COST_SCALE_LOG2; tpl_stats->srcrf_sse = pred_error << TPL_DEP_COST_SCALE_LOG2; + const YV12_BUFFER_CONFIG *ref_frame_ptr[2]; + + if (best_mode == NEW_NEWMV) { + ref_frame_ptr[0] = tpl_data->ref_frame[comp_ref_frames[best_cmp_rf_idx][0]]; + ref_frame_ptr[1] = + tpl_data->src_ref_frame[comp_ref_frames[best_cmp_rf_idx][1]]; + get_rate_distortion(&rate_cost, &recon_error, &pred_error, src_diff, coeff, + qcoeff, dqcoeff, cm, x, ref_frame_ptr, rec_buffer_pool, + rec_stride_pool, tx_size, best_mode, mi_row, mi_col, + use_y_only_rate_distortion, 0 /*do_recon*/, NULL); + tpl_stats->cmp_recrf_dist[0] = recon_error << TPL_DEP_COST_SCALE_LOG2; + tpl_stats->cmp_recrf_rate[0] = rate_cost; + + rate_cost = 0; + ref_frame_ptr[0] = + tpl_data->src_ref_frame[comp_ref_frames[best_cmp_rf_idx][0]]; + ref_frame_ptr[1] = tpl_data->ref_frame[comp_ref_frames[best_cmp_rf_idx][1]]; + get_rate_distortion(&rate_cost, &recon_error, &pred_error, src_diff, coeff, + qcoeff, dqcoeff, cm, x, ref_frame_ptr, rec_buffer_pool, + rec_stride_pool, tx_size, best_mode, mi_row, mi_col, + use_y_only_rate_distortion, 0 /*do_recon*/, NULL); + tpl_stats->cmp_recrf_dist[1] = recon_error << TPL_DEP_COST_SCALE_LOG2; + tpl_stats->cmp_recrf_rate[1] = rate_cost; + } + // Final encode rate_cost = 0; - const YV12_BUFFER_CONFIG *ref_frame_ptr[2]; + // Add uv bound if needed + if (best_mode == D203_PRED && xd->left_available && + mi_row + tx_size_high_unit[tx_size] < xd->tile.mi_row_end) { + const int num_planes = use_y_only_rate_distortion ? 1 : av1_num_planes(cm); + for (int plane = 1; plane < num_planes; ++plane) { + struct macroblockd_plane *pd = &xd->plane[plane]; + int dst_mb_offset_uv = + ((mi_row * MI_SIZE) >> pd->subsampling_y) * rec_stride_pool[plane] + + ((mi_col * MI_SIZE) >> pd->subsampling_x); + uint8_t *dst_uv_buffer = rec_buffer_pool[plane] + dst_mb_offset_uv; + int dst_uv_buffer_stride = rec_stride_pool[plane]; + int bh_uv = (bh >> pd->subsampling_y); + + if (is_cur_buf_hbd(xd)) { + uint16_t *dst_uv = CONVERT_TO_SHORTPTR(dst_uv_buffer); + for (int i = 0; i < bh_uv; ++i) + dst_uv[(bh_uv + i) * dst_uv_buffer_stride - 1] = + dst_uv[(bh_uv - 1) * dst_uv_buffer_stride - 1]; + } else { + for (int i = 0; i < bh_uv; ++i) + dst_uv_buffer[(bh_uv + i) * dst_uv_buffer_stride - 1] = + dst_uv_buffer[(bh_uv - 1) * dst_uv_buffer_stride - 1]; + } + } + } ref_frame_ptr[0] = best_mode == NEW_NEWMV @@ -1046,17 +1105,11 @@ tpl_stats->recrf_dist = AOMMAX(tpl_stats->srcrf_dist, tpl_stats->recrf_dist); tpl_stats->recrf_rate = AOMMAX(tpl_stats->srcrf_rate, tpl_stats->recrf_rate); - if (best_mode == NEW_NEWMV) { - ref_frame_ptr[0] = tpl_data->ref_frame[comp_ref_frames[best_cmp_rf_idx][0]]; - ref_frame_ptr[1] = - tpl_data->src_ref_frame[comp_ref_frames[best_cmp_rf_idx][1]]; - get_rate_distortion(&rate_cost, &recon_error, &pred_error, src_diff, coeff, - qcoeff, dqcoeff, cm, x, ref_frame_ptr, rec_buffer_pool, - rec_stride_pool, tx_size, best_mode, mi_row, mi_col, - use_y_only_rate_distortion, 1 /*do_recon*/, NULL); - tpl_stats->cmp_recrf_dist[0] = recon_error << TPL_DEP_COST_SCALE_LOG2; - tpl_stats->cmp_recrf_rate[0] = rate_cost; - + if (best_mode == NEWMV) { + tpl_stats->mv[best_rf_idx] = best_mv[0]; + tpl_stats->ref_frame_index[0] = best_rf_idx; + tpl_stats->ref_frame_index[1] = NONE_FRAME; + } else if (best_mode == NEW_NEWMV) { tpl_stats->cmp_recrf_dist[0] = AOMMAX(tpl_stats->srcrf_dist, tpl_stats->cmp_recrf_dist[0]); tpl_stats->cmp_recrf_rate[0] = @@ -1067,17 +1120,6 @@ tpl_stats->cmp_recrf_rate[0] = AOMMIN(tpl_stats->recrf_rate, tpl_stats->cmp_recrf_rate[0]); - rate_cost = 0; - ref_frame_ptr[0] = - tpl_data->src_ref_frame[comp_ref_frames[best_cmp_rf_idx][0]]; - ref_frame_ptr[1] = tpl_data->ref_frame[comp_ref_frames[best_cmp_rf_idx][1]]; - get_rate_distortion(&rate_cost, &recon_error, &pred_error, src_diff, coeff, - qcoeff, dqcoeff, cm, x, ref_frame_ptr, rec_buffer_pool, - rec_stride_pool, tx_size, best_mode, mi_row, mi_col, - use_y_only_rate_distortion, 1 /*do_recon*/, NULL); - tpl_stats->cmp_recrf_dist[1] = recon_error << TPL_DEP_COST_SCALE_LOG2; - tpl_stats->cmp_recrf_rate[1] = rate_cost; - tpl_stats->cmp_recrf_dist[1] = AOMMAX(tpl_stats->srcrf_dist, tpl_stats->cmp_recrf_dist[1]); tpl_stats->cmp_recrf_rate[1] = @@ -1087,13 +1129,7 @@ AOMMIN(tpl_stats->recrf_dist, tpl_stats->cmp_recrf_dist[1]); tpl_stats->cmp_recrf_rate[1] = AOMMIN(tpl_stats->recrf_rate, tpl_stats->cmp_recrf_rate[1]); - } - if (best_mode == NEWMV) { - tpl_stats->mv[best_rf_idx] = best_mv[0]; - tpl_stats->ref_frame_index[0] = best_rf_idx; - tpl_stats->ref_frame_index[1] = NONE_FRAME; - } else if (best_mode == NEW_NEWMV) { tpl_stats->ref_frame_index[0] = comp_ref_frames[best_cmp_rf_idx][0]; tpl_stats->ref_frame_index[1] = comp_ref_frames[best_cmp_rf_idx][1]; tpl_stats->mv[tpl_stats->ref_frame_index[0]] = best_mv[0]; @@ -1375,12 +1411,6 @@ const int base_qindex = cpi->use_ducky_encode ? gf_group->q_val[frame_idx] : pframe_qindex; - // The TPL model is only meant to be run in inter mode, so ensure that we are - // not running in all intra mode, which implies we are not tuning for image - // quality (IQ) or SSIMULACRA2. - assert(cpi->oxcf.tune_cfg.tuning != AOM_TUNE_IQ && - cpi->oxcf.tune_cfg.tuning != AOM_TUNE_SSIMULACRA2 && - cpi->oxcf.mode != ALLINTRA); // Get rd multiplier set up. rdmult = av1_compute_rd_mult( base_qindex, cm->seq_params->bit_depth, @@ -1427,6 +1457,53 @@ gf_group->layer_depth[frame_idx] >= layer_depth_th; } +static void tpl_store_before_propagation(AV1_COMP *cpi, + AomTplBlockStats *tpl_block_stats, + TplDepStats *src_stats, int mi_row, + int mi_col) { + TplParams *const tpl_data = &cpi->ppi->tpl_data; + GF_GROUP *gf_group = &cpi->ppi->gf_group; + + tpl_block_stats->row = mi_row * MI_SIZE; + tpl_block_stats->col = mi_col * MI_SIZE; + tpl_block_stats->srcrf_sse = src_stats->srcrf_sse; + // These need to be scaled down for external RC as libaom scales them up + // first. See b/274644689. + tpl_block_stats->srcrf_dist = + src_stats->srcrf_dist >> TPL_DEP_COST_SCALE_LOG2; + tpl_block_stats->recrf_sse = src_stats->recrf_sse >> TPL_DEP_COST_SCALE_LOG2; + tpl_block_stats->recrf_dist = + src_stats->recrf_dist >> TPL_DEP_COST_SCALE_LOG2; + tpl_block_stats->intra_sse = src_stats->intra_sse >> TPL_DEP_COST_SCALE_LOG2; + tpl_block_stats->intra_dist = + src_stats->intra_dist >> TPL_DEP_COST_SCALE_LOG2; + tpl_block_stats->cmp_recrf_dist[0] = src_stats->cmp_recrf_dist[0]; + tpl_block_stats->cmp_recrf_dist[1] = src_stats->cmp_recrf_dist[1]; + tpl_block_stats->mc_dep_rate = src_stats->mc_dep_rate; + tpl_block_stats->mc_dep_dist = src_stats->mc_dep_dist; + tpl_block_stats->inter_cost = src_stats->inter_cost; + tpl_block_stats->intra_cost = src_stats->intra_cost; + tpl_block_stats->srcrf_rate = src_stats->srcrf_rate; + tpl_block_stats->recrf_rate = src_stats->recrf_rate; + tpl_block_stats->intra_rate = src_stats->intra_rate; + tpl_block_stats->cmp_recrf_rate[0] = src_stats->cmp_recrf_rate[0]; + tpl_block_stats->cmp_recrf_rate[1] = src_stats->cmp_recrf_rate[1]; + tpl_block_stats->ref_frame_index[0] = + gf_group->ref_frame_list[tpl_data->frame_idx] + [LAST_FRAME + src_stats->ref_frame_index[0]]; + tpl_block_stats->ref_frame_index[1] = + gf_group->ref_frame_list[tpl_data->frame_idx] + [LAST_FRAME + src_stats->ref_frame_index[1]]; + for (int ref = 0; ref < AOM_RC_INTER_REFS_PER_FRAME; ++ref) { + tpl_block_stats->mv[ref].as_mv.col = src_stats->mv[ref].as_mv.col; + tpl_block_stats->mv[ref].as_mv.row = src_stats->mv[ref].as_mv.row; + tpl_block_stats->mv[ref].as_fullmv.col = src_stats->mv[ref].as_fullmv.col; + tpl_block_stats->mv[ref].as_fullmv.row = src_stats->mv[ref].as_fullmv.row; + tpl_block_stats->mv[ref].as_int = src_stats->mv[ref].as_int; + tpl_block_stats->pred_error[ref] = src_stats->pred_error[ref]; + } +} + // This function stores the motion estimation dependencies of all the blocks in // a row void av1_mc_flow_dispenser_row(AV1_COMP *cpi, TplTxfmStats *tpl_txfm_stats, @@ -1440,7 +1517,6 @@ TplParams *const tpl_data = &cpi->ppi->tpl_data; TplDepFrame *tpl_frame = &tpl_data->tpl_frame[tpl_data->frame_idx]; MACROBLOCKD *xd = &x->e_mbd; - const int tplb_cols_in_tile = ROUND_POWER_OF_TWO(mi_params->mi_cols, mi_size_wide_log2[bsize]); const int tplb_row = ROUND_POWER_OF_TWO(mi_row, mi_size_high_log2[bsize]); @@ -1476,6 +1552,17 @@ // Motion flow dependency dispenser. tpl_model_store(tpl_frame->tpl_stats_ptr, mi_row, mi_col, tpl_frame->stride, &tpl_stats, tpl_data->tpl_stats_block_mis_log2); + + if (av1_use_tpl_for_extrc(&cpi->ext_ratectrl)) { + AomTplFrameStats *tpl_frame_stats_before_propagation = + &cpi->extrc_tpl_gop_stats.frame_stats_list[tpl_data->frame_idx]; + AomTplBlockStats *block_stats = + &tpl_frame_stats_before_propagation + ->block_stats_list[mi_row * tpl_frame->mi_cols + mi_col]; + tpl_store_before_propagation(cpi, block_stats, &tpl_stats, mi_row, + mi_col); + } + (*tpl_row_mt->sync_write_ptr)(&tpl_data->tpl_mt_sync, tplb_row, tplb_col_in_tile, tplb_cols_in_tile); } @@ -1521,7 +1608,7 @@ } } -static inline void init_gop_frames_for_tpl( +static inline int init_gop_frames_for_tpl( AV1_COMP *cpi, const EncodeFrameParams *const init_frame_params, GF_GROUP *gf_group, int *tpl_group_frames, int *pframe_qindex) { AV1_COMMON *cm = &cpi->common; @@ -1544,7 +1631,12 @@ tpl_data->tpl_frame[-i - 1].rec_picture = NULL; tpl_data->tpl_frame[-i - 1].frame_display_index = 0; } else { - tpl_data->tpl_frame[-i - 1].gf_picture = &cm->ref_frame_map[i]->buf; + if (cm->ref_frame_map[i]->display_order_hint == + tpl_data->prev_gop_arf_disp_order) { + tpl_data->tpl_frame[-i - 1].gf_picture = &tpl_data->prev_gop_arf_src; + } else { + tpl_data->tpl_frame[-i - 1].gf_picture = &cm->ref_frame_map[i]->buf; + } tpl_data->tpl_frame[-i - 1].rec_picture = &cm->ref_frame_map[i]->buf; tpl_data->tpl_frame[-i - 1].frame_display_index = cm->ref_frame_map[i]->display_order_hint; @@ -1722,6 +1814,8 @@ ++extend_frame_count; ++frame_display_index; } + + return extend_frame_count; } void av1_init_tpl_stats(TplParams *const tpl_data) { @@ -1856,6 +1950,78 @@ return exp((mc_dep_cost_base - intra_cost_base) / cbcmp_base); } +void av1_free_tpl_gop_stats(AomTplGopStats *extrc_tpl_gop_stats) { + if (extrc_tpl_gop_stats == NULL || + extrc_tpl_gop_stats->frame_stats_list == NULL) { + return; + } + for (int frame_index = 0; frame_index < extrc_tpl_gop_stats->size; + ++frame_index) { + AomTplFrameStats *this_frame_stats = + &extrc_tpl_gop_stats->frame_stats_list[frame_index]; + aom_free(this_frame_stats->block_stats_list); + this_frame_stats->block_stats_list = NULL; + } + aom_free(extrc_tpl_gop_stats->frame_stats_list); + extrc_tpl_gop_stats->frame_stats_list = NULL; + extrc_tpl_gop_stats->size = 0; +} + +static void init_tpl_stats_before_propagation( + struct aom_internal_error_info *error_info, + AomTplGopStats *extrc_tpl_gop_stats, TplParams *tpl_stats, + int tpl_gop_frames, int frame_width, int frame_height) { + av1_free_tpl_gop_stats(extrc_tpl_gop_stats); + AOM_CHECK_MEM_ERROR( + error_info, extrc_tpl_gop_stats->frame_stats_list, + aom_calloc(tpl_gop_frames, + sizeof(*extrc_tpl_gop_stats->frame_stats_list))); + extrc_tpl_gop_stats->size = tpl_gop_frames; + for (int frame_index = 0; frame_index < tpl_gop_frames; ++frame_index) { + const int mi_rows = tpl_stats->tpl_frame[frame_index].mi_rows; + const int mi_cols = tpl_stats->tpl_frame[frame_index].mi_cols; + AomTplFrameStats *this_frame_stats = + &extrc_tpl_gop_stats->frame_stats_list[frame_index]; + AOM_CHECK_MEM_ERROR( + error_info, this_frame_stats->block_stats_list, + aom_calloc(mi_rows * mi_cols, + sizeof(*this_frame_stats->block_stats_list))); + this_frame_stats->num_blocks = mi_rows * mi_cols; + this_frame_stats->frame_width = frame_width; + this_frame_stats->frame_height = frame_height; + } +} + +static void trim_tpl_stats(struct aom_internal_error_info *error_info, + AomTplGopStats *extrc_tpl_gop_stats, + int extra_frames) { + int i; + AomTplFrameStats *new_frame_stats; + const int new_size = extrc_tpl_gop_stats->size - extra_frames; + if (extrc_tpl_gop_stats->size <= extra_frames) + aom_internal_error( + error_info, AOM_CODEC_ERROR, + "The number of frames in AomTplGopStats is fewer than expected."); + AOM_CHECK_MEM_ERROR(error_info, new_frame_stats, + aom_calloc(new_size, sizeof(*new_frame_stats))); + for (i = 0; i < new_size; i++) { + AomTplFrameStats *frame_stats = &extrc_tpl_gop_stats->frame_stats_list[i]; + const int num_blocks = frame_stats->num_blocks; + new_frame_stats[i].num_blocks = frame_stats->num_blocks; + new_frame_stats[i].frame_width = frame_stats->frame_width; + new_frame_stats[i].frame_height = frame_stats->frame_height; + new_frame_stats[i].num_blocks = num_blocks; + AOM_CHECK_MEM_ERROR( + error_info, new_frame_stats[i].block_stats_list, + aom_calloc(num_blocks, sizeof(*new_frame_stats[i].block_stats_list))); + memcpy(new_frame_stats[i].block_stats_list, frame_stats->block_stats_list, + num_blocks * sizeof(*new_frame_stats[i].block_stats_list)); + } + av1_free_tpl_gop_stats(extrc_tpl_gop_stats); + extrc_tpl_gop_stats->size = new_size; + extrc_tpl_gop_stats->frame_stats_list = new_frame_stats; +} + int av1_tpl_setup_stats(AV1_COMP *cpi, int gop_eval, const EncodeFrameParams *const frame_params) { #if CONFIG_COLLECT_COMPONENT_TIMING @@ -1890,13 +2056,19 @@ int pframe_qindex; int tpl_gf_group_frames; - init_gop_frames_for_tpl(cpi, frame_params, gf_group, &tpl_gf_group_frames, - &pframe_qindex); + int extended_frame_count = init_gop_frames_for_tpl( + cpi, frame_params, gf_group, &tpl_gf_group_frames, &pframe_qindex); cpi->ppi->p_rc.base_layer_qp = pframe_qindex; av1_init_tpl_stats(tpl_data); + if (av1_use_tpl_for_extrc(&cpi->ext_ratectrl)) { + init_tpl_stats_before_propagation( + cpi->common.error, &cpi->extrc_tpl_gop_stats, tpl_data, + tpl_gf_group_frames, cpi->common.width, cpi->common.height); + } + TplBuffers *tpl_tmp_buffers = &cpi->td.tpl_tmp_buffers; if (!tpl_alloc_temp_buffers(tpl_tmp_buffers, tpl_data->tpl_bsize_1d)) { aom_internal_error(cpi->common.error, AOM_CODEC_MEM_ERROR, @@ -1962,6 +2134,19 @@ num_planes); } + if (av1_use_tpl_for_extrc(&cpi->ext_ratectrl)) { + // TPL stats has extra frames from next GOP. Trim those extra frames for + // external RC. + trim_tpl_stats(cpi->common.error, &cpi->extrc_tpl_gop_stats, + extended_frame_count); + const aom_codec_err_t codec_status = + av1_extrc_send_tpl_stats(&cpi->ext_ratectrl, &cpi->extrc_tpl_gop_stats); + if (codec_status != AOM_CODEC_OK) { + aom_internal_error(cpi->common.error, codec_status, + "av1_extrc_send_tpl_stats() failed"); + } + } + for (int frame_idx = tpl_gf_group_frames - 1; frame_idx >= cpi->gf_frame_index; --frame_idx) { if (skip_tpl_for_frame(gf_group, frame_idx, gop_eval, approx_gop_eval, @@ -2230,7 +2415,7 @@ if (qstep <= target_qstep) break; } } else { - for (qindex = leaf_qindex; qindex <= MAXQ; ++qindex) { + for (qindex = leaf_qindex; qindex < MAXQ; ++qindex) { const double qstep = av1_dc_quant_QTX(qindex, 0, bit_depth); if (qstep >= target_qstep) break; }
diff --git a/av1/encoder/tpl_model.h b/av1/encoder/tpl_model.h index 3690671..ba6fd9c 100644 --- a/av1/encoder/tpl_model.h +++ b/av1/encoder/tpl_model.h
@@ -29,11 +29,13 @@ #include "config/aom_config.h" +#include "aom/aom_tpl.h" #include "aom_scale/yv12config.h" #include "aom_util/aom_pthread.h" #include "av1/common/mv.h" #include "av1/common/scale.h" +#include "av1/encoder/av1_ext_ratectrl.h" #include "av1/encoder/block.h" #include "av1/encoder/lookahead.h" #include "av1/encoder/ratectrl.h" @@ -236,6 +238,16 @@ const YV12_BUFFER_CONFIG *ref_frame[INTER_REFS_PER_FRAME]; /*! + * The buffer for the past gop's last frame's src. + */ + YV12_BUFFER_CONFIG prev_gop_arf_src; + + /*! + * Display order of the past gop's last frame. + */ + int64_t prev_gop_arf_disp_order; + + /*! * Parameters related to synchronization for top-right dependency in row based * multi-threading of tpl */ @@ -404,6 +416,10 @@ void av1_read_rd_command(const char *filepath, RD_COMMAND *rd_command); #endif // CONFIG_RD_COMMAND +static inline bool av1_use_tpl_for_extrc(AOM_EXT_RATECTRL const *ext_rc) { + return ext_rc->ready && ext_rc->funcs.send_tpl_gop_stats != NULL; +} + /*!\brief Allocate buffers used by tpl model * * \param[in] Top-level encode/decode structure @@ -686,6 +702,12 @@ double av1_tpl_compute_frame_mv_entropy(const TplDepFrame *tpl_frame, uint8_t right_shift); +/*!\brief Free the memory allocated for cpi->extrc_tpl_gop_stats. + * + * \param[in] extrc_tpl_gop_stats TPL stats for the GOP used for external RC. + */ +void av1_free_tpl_gop_stats(AomTplGopStats *extrc_tpl_gop_stats); + #if CONFIG_RATECTRL_LOG typedef struct { int coding_frame_count;
diff --git a/av1/encoder/tx_search.c b/av1/encoder/tx_search.c index 334bce1..1829a25 100644 --- a/av1/encoder/tx_search.c +++ b/av1/encoder/tx_search.c
@@ -9,6 +9,8 @@ * PATENTS file, you can obtain it at www.aomedia.org/license/patent. */ +#include <inttypes.h> + #include "av1/common/cfl.h" #include "av1/common/reconintra.h" #include "av1/encoder/block.h" @@ -111,8 +113,6 @@ MACROBLOCKD *const xd = &x->e_mbd; MB_MODE_INFO *const mbmi = xd->mi[0]; mbmi->tx_size = mb_rd_info->tx_size; - memcpy(x->txfm_search_info.blk_skip, mb_rd_info->blk_skip, - sizeof(mb_rd_info->blk_skip[0]) * n4); av1_copy(mbmi->inter_tx_size, mb_rd_info->inter_tx_size); av1_copy_array(xd->tx_type_map, mb_rd_info->tx_type_map, n4); *rd_stats = mb_rd_info->rd_stats; @@ -247,8 +247,6 @@ memset(xd->tx_type_map, DCT_DCT, sizeof(xd->tx_type_map[0]) * n4); memset(mbmi->inter_tx_size, tx_size, sizeof(mbmi->inter_tx_size)); mbmi->tx_size = tx_size; - for (int i = 0; i < n4; ++i) - set_blk_skip(x->txfm_search_info.blk_skip, 0, i, 1); rd_stats->skip_txfm = 1; if (is_cur_buf_hbd(xd)) dist = ROUND_POWER_OF_TWO(dist, (xd->bd - 8) * 2); rd_stats->dist = rd_stats->sse = (dist << 4); @@ -297,8 +295,6 @@ const MB_MODE_INFO *const mbmi = xd->mi[0]; mb_rd_info->hash_value = hash; mb_rd_info->tx_size = mbmi->tx_size; - memcpy(mb_rd_info->blk_skip, x->txfm_search_info.blk_skip, - sizeof(mb_rd_info->blk_skip[0]) * n4); av1_copy(mb_rd_info->inter_tx_size, mbmi->inter_tx_size); av1_copy_array(mb_rd_info->tx_type_map, xd->tx_type_map, n4); mb_rd_info->rd_stats = *rd_stats; @@ -1819,6 +1815,9 @@ } else if (fast_tx_search) { allowed_tx_mask = 0x0c01; // V_DCT, H_DCT, DCT_DCT allowed_tx_mask &= ext_tx_used_flag; + } else if (!is_inter && txfm_params->use_derived_intra_tx_type_set) { + allowed_tx_mask = av1_derived_intra_tx_used_flag[intra_dir]; + allowed_tx_mask &= ext_tx_used_flag; } else { assert(plane == 0); allowed_tx_mask = ext_tx_used_flag; @@ -2021,8 +2020,8 @@ int block, int blk_row, int blk_col, BLOCK_SIZE plane_bsize, TX_SIZE tx_size, const TXB_CTX *const txb_ctx, - FAST_TX_SEARCH_MODE ftxs_mode, int skip_trellis, - int64_t ref_best_rd, RD_STATS *best_rd_stats) { + FAST_TX_SEARCH_MODE ftxs_mode, int64_t ref_best_rd, + RD_STATS *best_rd_stats) { const AV1_COMMON *cm = &cpi->common; MACROBLOCKD *xd = &x->e_mbd; MB_MODE_INFO *mbmi = xd->mi[0]; @@ -2038,8 +2037,8 @@ plane ? 0 : blk_row * xd->tx_type_map_stride + blk_col; av1_invalid_rd_stats(best_rd_stats); - skip_trellis |= !is_trellis_used(cpi->optimize_seg_arr[xd->mi[0]->segment_id], - DRY_RUN_NORMAL); + int skip_trellis = !is_trellis_used( + cpi->optimize_seg_arr[xd->mi[0]->segment_id], DRY_RUN_NORMAL); uint8_t best_txb_ctx = 0; // txk_allowed = TX_TYPES: >1 tx types are allowed @@ -2338,9 +2337,8 @@ int64_t ref_rdcost) { assert(is_inter_block(x->e_mbd.mi[0])); RD_STATS this_rd_stats; - const int skip_trellis = 0; search_tx_type(cpi, x, 0, block, blk_row, blk_col, plane_bsize, tx_size, - txb_ctx, ftxs_mode, skip_trellis, ref_rdcost, &this_rd_stats); + txb_ctx, ftxs_mode, ref_rdcost, &this_rd_stats); av1_merge_rd_stats(rd_stats, &this_rd_stats); } @@ -2354,7 +2352,6 @@ MACROBLOCKD *const xd = &x->e_mbd; MB_MODE_INFO *const mbmi = xd->mi[0]; struct macroblock_plane *const p = &x->plane[0]; - const int bw = mi_size_wide[plane_bsize]; const ENTROPY_CONTEXT *const pta = ta + blk_col; const ENTROPY_CONTEXT *const ptl = tl + blk_row; const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size); @@ -2384,8 +2381,6 @@ update_txk_array(xd, blk_row, blk_col, tx_size, DCT_DCT); } rd_stats->skip_txfm = pick_skip_txfm; - set_blk_skip(x->txfm_search_info.blk_skip, 0, blk_row * bw + blk_col, - pick_skip_txfm); if (tx_size > TX_4X4 && depth < MAX_VARTX_DEPTH) rd_stats->rate += x->mode_costs.txfm_partition_cost[txfm_partition_ctx][0]; @@ -2641,9 +2636,6 @@ } mbmi->tx_size = tx_size; update_txk_array(xd, blk_row, blk_col, tx_size, no_split.tx_type); - const int bw = mi_size_wide[plane_bsize]; - set_blk_skip(x->txfm_search_info.blk_skip, 0, blk_row * bw + blk_col, - rd_stats->skip_txfm); } else { *rd_stats = split_rd_stats; if (split_rd_stats.rdcost == INT64_MAX) *is_cost_valid = 0; @@ -2740,10 +2732,9 @@ const int64_t skip_txfm_rd = is_inter_block(mbmi) ? RDCOST(x->rdmult, skip_txfm_rate, 0) : INT64_MAX; const int64_t no_skip_txfm_rd = RDCOST(x->rdmult, no_skip_txfm_rate, 0); - const int skip_trellis = 0; av1_txfm_rd_in_plane(x, cpi, rd_stats, ref_best_rd, AOMMIN(no_skip_txfm_rd, skip_txfm_rd), AOM_PLANE_Y, bs, - mbmi->tx_size, FTXS_NONE, skip_trellis); + mbmi->tx_size, FTXS_NONE); } static inline void choose_smallest_tx_size(const AV1_COMP *const cpi, @@ -2754,9 +2745,8 @@ mbmi->tx_size = TX_4X4; // TODO(any) : Pass this_rd based on skip/non-skip cost - const int skip_trellis = 0; av1_txfm_rd_in_plane(x, cpi, rd_stats, ref_best_rd, 0, 0, bs, mbmi->tx_size, - FTXS_NONE, skip_trellis); + FTXS_NONE); } #if !CONFIG_REALTIME_ONLY @@ -2840,15 +2830,12 @@ * \param[in] tx_size The given transform size * \param[in] ftxs_mode Transform search mode specifying desired speed and quality tradeoff - * \param[in] skip_trellis Binary flag indicating if trellis optimization - should be skipped * \return An int64_t value that is the best RD cost found. */ static int64_t uniform_txfm_yrd(const AV1_COMP *const cpi, MACROBLOCK *x, RD_STATS *rd_stats, int64_t ref_best_rd, BLOCK_SIZE bs, TX_SIZE tx_size, - FAST_TX_SEARCH_MODE ftxs_mode, - int skip_trellis) { + FAST_TX_SEARCH_MODE ftxs_mode) { assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed_bsize(bs))); MACROBLOCKD *const xd = &x->e_mbd; MB_MODE_INFO *const mbmi = xd->mi[0]; @@ -2875,7 +2862,7 @@ mbmi->tx_size = tx_size; av1_txfm_rd_in_plane(x, cpi, rd_stats, ref_best_rd, AOMMIN(no_this_rd, skip_txfm_rd), AOM_PLANE_Y, bs, - tx_size, ftxs_mode, skip_trellis); + tx_size, ftxs_mode); if (rd_stats->rate == INT_MAX) return INT64_MAX; int64_t rd; @@ -2940,15 +2927,12 @@ init_depth = MAX_TX_DEPTH; } - const int skip_trellis = 0; uint8_t best_txk_type_map[MAX_MIB_SIZE * MAX_MIB_SIZE]; - uint8_t best_blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE]; TX_SIZE best_tx_size = max_rect_tx_size; int64_t best_rd = INT64_MAX; const int num_blks = bsize_to_num_blk(bs); x->rd_model = FULL_TXFM_RD; int64_t rd[MAX_TX_DEPTH + 1] = { INT64_MAX, INT64_MAX, INT64_MAX }; - TxfmSearchInfo *txfm_info = &x->txfm_search_info; for (int tx_size = start_tx, depth = init_depth; depth <= MAX_TX_DEPTH; depth++, tx_size = sub_tx_size_map[tx_size]) { if ((!cpi->oxcf.txfm_cfg.enable_tx64 && @@ -2977,9 +2961,8 @@ ? AOMMIN(ref_best_rd, best_rd) : ref_best_rd; rd[depth] = uniform_txfm_yrd(cpi, x, &this_rd_stats, rd_thresh, bs, tx_size, - FTXS_NONE, skip_trellis); + FTXS_NONE); if (rd[depth] < best_rd) { - av1_copy_array(best_blk_skip, txfm_info->blk_skip, num_blks); av1_copy_array(best_txk_type_map, xd->tx_type_map, num_blks); best_tx_size = tx_size; best_rd = rd[depth]; @@ -2997,7 +2980,6 @@ if (rd_stats->rate != INT_MAX) { mbmi->tx_size = best_tx_size; av1_copy_array(xd->tx_type_map, best_txk_type_map, num_blks); - av1_copy_array(txfm_info->blk_skip, best_blk_skip, num_blks); } #if !CONFIG_REALTIME_ONLY @@ -3049,8 +3031,8 @@ TXB_CTX txb_ctx; get_txb_ctx(plane_bsize, tx_size, plane, a, l, &txb_ctx); search_tx_type(cpi, x, plane, block, blk_row, blk_col, plane_bsize, tx_size, - &txb_ctx, args->ftxs_mode, args->skip_trellis, - args->best_rd - args->current_rd, &this_rd_stats); + &txb_ctx, args->ftxs_mode, args->best_rd - args->current_rd, + &this_rd_stats); #if !CONFIG_REALTIME_ONLY if (plane == AOM_PLANE_Y && xd->cfl.store_y) { @@ -3064,16 +3046,6 @@ #endif // CONFIG_RD_DEBUG av1_set_txb_context(x, plane, block, tx_size, a, l); - const int blk_idx = - blk_row * (block_size_wide[plane_bsize] >> MI_SIZE_LOG2) + blk_col; - - TxfmSearchInfo *txfm_info = &x->txfm_search_info; - if (plane == 0) - set_blk_skip(txfm_info->blk_skip, plane, blk_idx, - x->plane[plane].eobs[block] == 0); - else - set_blk_skip(txfm_info->blk_skip, plane, blk_idx, 0); - int64_t rd; if (is_inter) { const int64_t no_skip_txfm_rd = @@ -3256,21 +3228,17 @@ rd_stats->zero_rate = zero_blk_rate; tx_type_rd(cpi, x, tx_size, blk_row, blk_col, block, plane_bsize, &txb_ctx, rd_stats, ftxs_mode, ref_best_rd); - const int mi_width = mi_size_wide[plane_bsize]; - TxfmSearchInfo *txfm_info = &x->txfm_search_info; if (RDCOST(x->rdmult, rd_stats->rate, rd_stats->dist) >= RDCOST(x->rdmult, zero_blk_rate, rd_stats->sse) || rd_stats->skip_txfm == 1) { rd_stats->rate = zero_blk_rate; rd_stats->dist = rd_stats->sse; rd_stats->skip_txfm = 1; - set_blk_skip(txfm_info->blk_skip, 0, blk_row * mi_width + blk_col, 1); x->plane[0].eobs[block] = 0; x->plane[0].txb_entropy_ctx[block] = 0; update_txk_array(xd, blk_row, blk_col, tx_size, DCT_DCT); } else { rd_stats->skip_txfm = 0; - set_blk_skip(txfm_info->blk_skip, 0, blk_row * mi_width + blk_col, 0); } if (tx_size > TX_4X4 && depth < MAX_VARTX_DEPTH) rd_stats->rate += x->mode_costs.txfm_partition_cost[ctx][0]; @@ -3672,7 +3640,6 @@ av1_subtract_plane(x, plane_bsize, plane); } - const int skip_trellis = 0; const TX_SIZE uv_tx_size = av1_get_tx_size(AOM_PLANE_U, xd); int is_cost_valid = 1; for (int plane = 1; plane < MAX_MB_PLANE; ++plane) { @@ -3687,7 +3654,7 @@ chroma_ref_best_rd != INT64_MAX) chroma_ref_best_rd = ref_best_rd - AOMMIN(this_rd, skip_txfm_rd); av1_txfm_rd_in_plane(x, cpi, &this_rd_stats, chroma_ref_best_rd, 0, plane, - plane_bsize, uv_tx_size, FTXS_NONE, skip_trellis); + plane_bsize, uv_tx_size, FTXS_NONE); if (this_rd_stats.rate == INT_MAX) { is_cost_valid = 0; break; @@ -3712,8 +3679,7 @@ void av1_txfm_rd_in_plane(MACROBLOCK *x, const AV1_COMP *cpi, RD_STATS *rd_stats, int64_t ref_best_rd, int64_t current_rd, int plane, BLOCK_SIZE plane_bsize, - TX_SIZE tx_size, FAST_TX_SEARCH_MODE ftxs_mode, - int skip_trellis) { + TX_SIZE tx_size, FAST_TX_SEARCH_MODE ftxs_mode) { assert(IMPLIES(plane == 0, x->e_mbd.mi[0]->tx_size == tx_size)); if (!cpi->oxcf.txfm_cfg.enable_tx64 && @@ -3736,7 +3702,7 @@ args.best_rd = ref_best_rd; args.current_rd = current_rd; args.ftxs_mode = ftxs_mode; - args.skip_trellis = skip_trellis; + args.skip_trellis = 0; av1_init_rd_stats(&args.rd_stats); av1_get_entropy_contexts(plane_bsize, pd, args.t_above, args.t_left); @@ -3792,8 +3758,6 @@ } else { av1_pick_uniform_tx_size_type_yrd(cpi, x, rd_stats_y, bsize, rd_thresh); memset(mbmi->inter_tx_size, mbmi->tx_size, sizeof(mbmi->inter_tx_size)); - for (int i = 0; i < xd->height * xd->width; ++i) - set_blk_skip(x->txfm_search_info.blk_skip, 0, i, rd_stats_y->skip_txfm); } if (rd_stats_y->rate == INT_MAX) return 0;
diff --git a/av1/encoder/tx_search.h b/av1/encoder/tx_search.h index 9181304..a16d549 100644 --- a/av1/encoder/tx_search.h +++ b/av1/encoder/tx_search.h
@@ -152,16 +152,13 @@ * \param[in] tx_size The given transform size * \param[in] ftxs_mode Transform search mode specifying desired speed and quality tradeoff - * \param[in] skip_trellis Binary flag indicating if trellis optimization - should be skipped * * \remark Nothing is returned. The RD results will be saved in rd_stats. */ void av1_txfm_rd_in_plane(MACROBLOCK *x, const AV1_COMP *cpi, RD_STATS *rd_stats, int64_t ref_best_rd, int64_t current_rd, int plane, BLOCK_SIZE plane_bsize, - TX_SIZE tx_size, FAST_TX_SEARCH_MODE ftxs_mode, - int skip_trellis); + TX_SIZE tx_size, FAST_TX_SEARCH_MODE ftxs_mode); /*!\brief Recursive transform size and type search. *
diff --git a/av1/encoder/txb_rdopt.c b/av1/encoder/txb_rdopt.c index 6f0fdff..cf76ffd 100644 --- a/av1/encoder/txb_rdopt.c +++ b/av1/encoder/txb_rdopt.c
@@ -80,7 +80,6 @@ const tran_low_t *tcoeff, tran_low_t *qcoeff, tran_low_t *dqcoeff, uint8_t *levels, int sharpness, const qm_val_t *iqmatrix, const qm_val_t *qmatrix) { - const int dqv = get_dqv(dequant, scan[si], iqmatrix); (void)eob; // this simple version assumes the coeff's scan_idx is not DC (scan_idx != 0) // and not the last (scan_idx != eob - 1) @@ -104,6 +103,7 @@ return; } + const int dqv = get_dqv(dequant, scan[si], iqmatrix); const int64_t dist = get_coeff_dist(abs_tqc, abs_dqc, shift, qmatrix, ci); const int64_t rd = RDCOST(rdmult, rate, dist); @@ -135,7 +135,6 @@ const LV_MAP_COEFF_COST *txb_costs, const tran_low_t *tcoeff, tran_low_t *qcoeff, tran_low_t *dqcoeff, uint8_t *levels, int sharpness, const qm_val_t *iqmatrix, const qm_val_t *qmatrix) { - const int dqv = get_dqv(dequant, scan[si], iqmatrix); assert(si != *eob - 1); const int ci = scan[si]; const tran_low_t qc = qcoeff[ci]; @@ -144,6 +143,7 @@ if (qc == 0) { *accu_rate += txb_costs->base_cost[coeff_ctx][0]; } else { + const int dqv = get_dqv(dequant, scan[si], iqmatrix); int lower_level = 0; const tran_low_t abs_qc = abs(qc); const tran_low_t tqc = tcoeff[ci]; @@ -299,6 +299,36 @@ return 0; } +static AOM_FORCE_INLINE void update_coeff_eob_facade( + int *accu_rate, int64_t *accu_dist, int *eob, int *nz_num, int *nz_ci, + int *si, TX_SIZE tx_size, TX_CLASS tx_class, int bhl, int width, + int dc_sign_ctx, int64_t rdmult, int shift, const int16_t *dequant, + const int16_t *scan, const LV_MAP_EOB_COST *txb_eob_costs, + const LV_MAP_COEFF_COST *txb_costs, const tran_low_t *tcoeff, + tran_low_t *qcoeff, tran_low_t *dqcoeff, uint8_t *levels, int sharpness, + const qm_val_t *iqmatrix, const qm_val_t *qmatrix, int max_nz_num) { + for (; *si >= 0 && *nz_num <= max_nz_num; --*si) { + update_coeff_eob(accu_rate, accu_dist, eob, nz_num, nz_ci, *si, tx_size, + tx_class, bhl, width, dc_sign_ctx, rdmult, shift, dequant, + scan, txb_eob_costs, txb_costs, tcoeff, qcoeff, dqcoeff, + levels, sharpness, iqmatrix, qmatrix); + } +} + +static AOM_FORCE_INLINE void update_coeff_simple_facade( + int *accu_rate, int *si, int eob, TX_SIZE tx_size, TX_CLASS tx_class, + int bhl, int64_t rdmult, int shift, const int16_t *dequant, + const int16_t *scan, const LV_MAP_COEFF_COST *txb_costs, + const tran_low_t *tcoeff, tran_low_t *qcoeff, tran_low_t *dqcoeff, + uint8_t *levels, int sharpness, const qm_val_t *iqmatrix, + const qm_val_t *qmatrix) { + for (; *si >= 1; --*si) { + update_coeff_simple(accu_rate, *si, eob, tx_size, tx_class, bhl, rdmult, + shift, dequant, scan, txb_costs, tcoeff, qcoeff, + dqcoeff, levels, sharpness, iqmatrix, qmatrix); + } +} + int av1_optimize_txb(const struct AV1_COMP *cpi, MACROBLOCK *x, int plane, int block, TX_SIZE tx_size, TX_TYPE tx_type, const TXB_CTX *const txb_ctx, int *rate_cost, @@ -400,13 +430,11 @@ #define UPDATE_COEFF_EOB_CASE(tx_class_literal) \ case tx_class_literal: \ - for (; si >= 0 && nz_num <= max_nz_num; --si) { \ - update_coeff_eob(&accu_rate, &accu_dist, &eob, &nz_num, nz_ci, si, \ - tx_size, tx_class_literal, bhl, width, \ - txb_ctx->dc_sign_ctx, rdmult, shift, dequant, scan, \ - txb_eob_costs, txb_costs, tcoeff, qcoeff, dqcoeff, \ - levels, sharpness, iqmatrix, qmatrix); \ - } \ + update_coeff_eob_facade( \ + &accu_rate, &accu_dist, &eob, &nz_num, nz_ci, &si, tx_size, \ + tx_class_literal, bhl, width, txb_ctx->dc_sign_ctx, rdmult, shift, \ + dequant, scan, txb_eob_costs, txb_costs, tcoeff, qcoeff, dqcoeff, \ + levels, sharpness, iqmatrix, qmatrix, max_nz_num); \ break switch (tx_class) { UPDATE_COEFF_EOB_CASE(TX_CLASS_2D); @@ -421,14 +449,12 @@ non_skip_cost, qcoeff, dqcoeff); } -#define UPDATE_COEFF_SIMPLE_CASE(tx_class_literal) \ - case tx_class_literal: \ - for (; si >= 1; --si) { \ - update_coeff_simple(&accu_rate, si, eob, tx_size, tx_class_literal, bhl, \ - rdmult, shift, dequant, scan, txb_costs, tcoeff, \ - qcoeff, dqcoeff, levels, sharpness, iqmatrix, \ - qmatrix); \ - } \ +#define UPDATE_COEFF_SIMPLE_CASE(tx_class_literal) \ + case tx_class_literal: \ + update_coeff_simple_facade(&accu_rate, &si, eob, tx_size, \ + tx_class_literal, bhl, rdmult, shift, dequant, \ + scan, txb_costs, tcoeff, qcoeff, dqcoeff, \ + levels, sharpness, iqmatrix, qmatrix); \ break switch (tx_class) { UPDATE_COEFF_SIMPLE_CASE(TX_CLASS_2D);
diff --git a/av1/encoder/var_based_part.c b/av1/encoder/var_based_part.c index 3027726..d09b0ca 100644 --- a/av1/encoder/var_based_part.c +++ b/av1/encoder/var_based_part.c
@@ -1027,6 +1027,10 @@ const YV12_BUFFER_CONFIG *yv12_alt = get_ref_frame_yv12_buf(cm, ALTREF_FRAME); const struct scale_factors *const sf = get_ref_scale_factors_const(cm, LAST_FRAME); + const struct scale_factors *const sf_g = + get_ref_scale_factors_const(cm, GOLDEN_FRAME); + const struct scale_factors *const sf_alt = + get_ref_scale_factors_const(cm, ALTREF_FRAME); struct buf_2d dst; unsigned int uv_sad_g = 0; unsigned int uv_sad_alt = 0; @@ -1063,7 +1067,7 @@ uint8_t *src = (plane == 1) ? yv12_g->u_buffer : yv12_g->v_buffer; setup_pred_plane(&dst, xd->mi[0]->bsize, src, yv12_g->uv_crop_width, yv12_g->uv_crop_height, yv12_g->uv_stride, xd->mi_row, - xd->mi_col, sf, xd->plane[plane].subsampling_x, + xd->mi_col, sf_g, xd->plane[plane].subsampling_x, xd->plane[plane].subsampling_y); uv_sad_g = cpi->ppi->fn_ptr[bs].sdf(p->src.buf, p->src.stride, dst.buf, dst.stride); @@ -1074,7 +1078,7 @@ uint8_t *src = (plane == 1) ? yv12_alt->u_buffer : yv12_alt->v_buffer; setup_pred_plane(&dst, xd->mi[0]->bsize, src, yv12_alt->uv_crop_width, yv12_alt->uv_crop_height, yv12_alt->uv_stride, - xd->mi_row, xd->mi_col, sf, + xd->mi_row, xd->mi_col, sf_alt, xd->plane[plane].subsampling_x, xd->plane[plane].subsampling_y); uv_sad_alt = cpi->ppi->fn_ptr[bs].sdf(p->src.buf, p->src.stride, @@ -1368,9 +1372,9 @@ x->sb_me_mv.as_int = mi->mv[0].as_int; if (cpi->svc.temporal_layer_id == 0) { if (abs(mi->mv[0].as_mv.col) > 16 && abs(mi->mv[0].as_mv.row) == 0) - x->sb_col_scroll = 1; + x->sb_col_scroll++; else if (abs(mi->mv[0].as_mv.row) > 16 && abs(mi->mv[0].as_mv.col) == 0) - x->sb_row_scroll = 1; + x->sb_row_scroll++; } } else { x->sb_me_partition = 0;
diff --git a/av1/encoder/x86/reconinter_enc_sse2.c b/av1/encoder/x86/reconinter_enc_sse2.c index a18e172..8ab566a 100644 --- a/av1/encoder/x86/reconinter_enc_sse2.c +++ b/av1/encoder/x86/reconinter_enc_sse2.c
@@ -126,8 +126,7 @@ aom_convolve8_vert(ref, ref_stride, comp_pred, width, NULL, -1, kernel, 16, width, height); } else { - DECLARE_ALIGNED(16, uint8_t, - temp[((MAX_SB_SIZE * 2 + 16) + 16) * MAX_SB_SIZE]); + uint8_t *temp = comp_pred; const int16_t *const kernel_x = av1_get_interp_filter_subpel_kernel(filter, subpel_x_q3 << 1); const int16_t *const kernel_y = @@ -230,8 +229,7 @@ aom_highbd_convolve8_vert(ref8, ref_stride, comp_pred8, width, NULL, -1, kernel, 16, width, height, bd); } else { - DECLARE_ALIGNED(16, uint16_t, - temp[((MAX_SB_SIZE + 16) + 16) * MAX_SB_SIZE]); + uint16_t *temp = CONVERT_TO_SHORTPTR(comp_pred8); const int16_t *const kernel_x = av1_get_interp_filter_subpel_kernel(filter, subpel_x_q3 << 1); const int16_t *const kernel_y =
diff --git a/av1/encoder/x86/temporal_filter_avx2.c b/av1/encoder/x86/temporal_filter_avx2.c index f3d85e1..45d563b 100644 --- a/av1/encoder/x86/temporal_filter_avx2.c +++ b/av1/encoder/x86/temporal_filter_avx2.c
@@ -229,62 +229,35 @@ return (count < 16) ? -1.0 : (double)accum / (6 * count) * SQRT_PI_BY_2; } -static AOM_FORCE_INLINE void get_squared_error_16x16_avx2( +static AOM_FORCE_INLINE void get_squared_error_avx2( const uint8_t *frame1, const unsigned int stride, const uint8_t *frame2, const unsigned int stride2, const int block_width, const int block_height, uint16_t *frame_sse, const unsigned int sse_stride) { - (void)block_width; const uint8_t *src1 = frame1; const uint8_t *src2 = frame2; uint16_t *dst = frame_sse; for (int i = 0; i < block_height; i++) { - __m128i vf1_128, vf2_128; - __m256i vf1, vf2, vdiff1, vsqdiff1; + for (int j = 0; j < block_width; j += 32) { + __m256i vsrc1, vsrc2, vmin, vmax, vdiff, vdiff1, vdiff2, vres1, vres2; - vf1_128 = _mm_loadu_si128((__m128i *)(src1)); - vf2_128 = _mm_loadu_si128((__m128i *)(src2)); - vf1 = _mm256_cvtepu8_epi16(vf1_128); - vf2 = _mm256_cvtepu8_epi16(vf2_128); - vdiff1 = _mm256_sub_epi16(vf1, vf2); - vsqdiff1 = _mm256_mullo_epi16(vdiff1, vdiff1); + vsrc1 = _mm256_loadu_si256((__m256i *)(src1 + j)); + vsrc2 = _mm256_loadu_si256((__m256i *)(src2 + j)); + vmax = _mm256_max_epu8(vsrc1, vsrc2); + vmin = _mm256_min_epu8(vsrc1, vsrc2); + vdiff = _mm256_subs_epu8(vmax, vmin); - _mm256_storeu_si256((__m256i *)(dst), vsqdiff1); + __m128i vtmp1 = _mm256_castsi256_si128(vdiff); + __m128i vtmp2 = _mm256_extracti128_si256(vdiff, 1); + vdiff1 = _mm256_cvtepu8_epi16(vtmp1); + vdiff2 = _mm256_cvtepu8_epi16(vtmp2); + + vres1 = _mm256_mullo_epi16(vdiff1, vdiff1); + vres2 = _mm256_mullo_epi16(vdiff2, vdiff2); + _mm256_storeu_si256((__m256i *)(dst + j), vres1); + _mm256_storeu_si256((__m256i *)(dst + 16 + j), vres2); + } // Set zero to uninitialized memory to avoid uninitialized loads later - *(int *)(dst + 16) = _mm_cvtsi128_si32(_mm_setzero_si128()); - - src1 += stride, src2 += stride2; - dst += sse_stride; - } -} - -static AOM_FORCE_INLINE void get_squared_error_32x32_avx2( - const uint8_t *frame1, const unsigned int stride, const uint8_t *frame2, - const unsigned int stride2, const int block_width, const int block_height, - uint16_t *frame_sse, const unsigned int sse_stride) { - (void)block_width; - const uint8_t *src1 = frame1; - const uint8_t *src2 = frame2; - uint16_t *dst = frame_sse; - for (int i = 0; i < block_height; i++) { - __m256i vsrc1, vsrc2, vmin, vmax, vdiff, vdiff1, vdiff2, vres1, vres2; - - vsrc1 = _mm256_loadu_si256((__m256i *)src1); - vsrc2 = _mm256_loadu_si256((__m256i *)src2); - vmax = _mm256_max_epu8(vsrc1, vsrc2); - vmin = _mm256_min_epu8(vsrc1, vsrc2); - vdiff = _mm256_subs_epu8(vmax, vmin); - - __m128i vtmp1 = _mm256_castsi256_si128(vdiff); - __m128i vtmp2 = _mm256_extracti128_si256(vdiff, 1); - vdiff1 = _mm256_cvtepu8_epi16(vtmp1); - vdiff2 = _mm256_cvtepu8_epi16(vtmp2); - - vres1 = _mm256_mullo_epi16(vdiff1, vdiff1); - vres2 = _mm256_mullo_epi16(vdiff2, vdiff2); - _mm256_storeu_si256((__m256i *)(dst), vres1); - _mm256_storeu_si256((__m256i *)(dst + 16), vres2); - // Set zero to uninitialized memory to avoid uninitialized loads later - *(int *)(dst + 32) = _mm_cvtsi128_si32(_mm_setzero_si128()); + *(int *)(dst + block_width) = _mm_cvtsi128_si32(_mm_setzero_si128()); src1 += stride; src2 += stride2; @@ -351,18 +324,13 @@ const double inv_num_ref_pixels, const double decay_factor, const double inv_factor, const double weight_factor, double *d_factor, int tf_wgt_calc_lvl) { - assert(((block_width == 16) || (block_width == 32)) && - ((block_height == 16) || (block_height == 32))); + assert(((block_width == 64) || (block_width == 32)) && + ((block_height == 64) || (block_height == 32))); uint32_t acc_5x5_sse[BH][BW]; - if (block_width == 32) { - get_squared_error_32x32_avx2(frame1, stride, frame2, stride2, block_width, - block_height, frame_sse, SSE_STRIDE); - } else { - get_squared_error_16x16_avx2(frame1, stride, frame2, stride2, block_width, - block_height, frame_sse, SSE_STRIDE); - } + get_squared_error_avx2(frame1, stride, frame2, stride2, block_width, + block_height, frame_sse, SSE_STRIDE); __m256i vsrc[5]; @@ -409,21 +377,28 @@ } } - double subblock_mses_scaled[4]; - double d_factor_decayed[4]; - for (int idx = 0; idx < 4; idx++) { + double subblock_mses_scaled[NUM_16X16]; + double d_factor_decayed[NUM_16X16]; + for (int idx = 0; idx < NUM_16X16; idx++) { subblock_mses_scaled[idx] = subblock_mses[idx] * inv_factor; d_factor_decayed[idx] = d_factor[idx] * decay_factor; } if (tf_wgt_calc_lvl == 0) { for (int i = 0, k = 0; i < block_height; i++) { - const int y_blk_raster_offset = (i >= block_height / 2) * 2; + const int y32_blk_raster_offset = (i >= (block_height >> 1)) << 1; + const int y16_blk_raster_offset = + ((i % (block_height >> 1)) >= (block_height >> 2)) << 1; for (int j = 0; j < block_width; j++, k++) { const int pixel_value = frame2[i * stride2 + j]; uint32_t diff_sse = acc_5x5_sse[i][j] + luma_sse_sum[i * BW + j]; const double window_error = diff_sse * inv_num_ref_pixels; - const int subblock_idx = y_blk_raster_offset + (j >= block_width / 2); + const int x32_blk_raster_offset = (j >= (block_width >> 1)); + const int x16_blk_raster_offset = + ((j % (block_width >> 1)) >= (block_width >> 2)); + const int subblock_idx = + ((y32_blk_raster_offset + x32_blk_raster_offset) << 2) + + (y16_blk_raster_offset + x16_blk_raster_offset); const double combined_error = weight_factor * window_error + subblock_mses_scaled[subblock_idx]; @@ -436,8 +411,8 @@ } } } else { - __m256d subblock_mses_reg[4]; - __m256d d_factor_mul_n_decay_qr_invs[4]; + __m256d subblock_mses_reg[NUM_16X16]; + __m256d d_factor_mul_n_decay_qr_invs[NUM_16X16]; const __m256 zero = _mm256_set1_ps(0.0f); const __m256 point_five = _mm256_set1_ps(0.5f); const __m256 seven = _mm256_set1_ps(7.0f); @@ -445,17 +420,15 @@ const __m256d weight_factor_256bit = _mm256_set1_pd(weight_factor); const __m256 tf_weight_scale = _mm256_set1_ps((float)TF_WEIGHT_SCALE); // Maintain registers to hold mse and d_factor at subblock level. - subblock_mses_reg[0] = _mm256_set1_pd(subblock_mses_scaled[0]); - subblock_mses_reg[1] = _mm256_set1_pd(subblock_mses_scaled[1]); - subblock_mses_reg[2] = _mm256_set1_pd(subblock_mses_scaled[2]); - subblock_mses_reg[3] = _mm256_set1_pd(subblock_mses_scaled[3]); - d_factor_mul_n_decay_qr_invs[0] = _mm256_set1_pd(d_factor_decayed[0]); - d_factor_mul_n_decay_qr_invs[1] = _mm256_set1_pd(d_factor_decayed[1]); - d_factor_mul_n_decay_qr_invs[2] = _mm256_set1_pd(d_factor_decayed[2]); - d_factor_mul_n_decay_qr_invs[3] = _mm256_set1_pd(d_factor_decayed[3]); + for (int i = 0; i < NUM_16X16; i++) { + subblock_mses_reg[i] = _mm256_set1_pd(subblock_mses_scaled[i]); + d_factor_mul_n_decay_qr_invs[i] = _mm256_set1_pd(d_factor_decayed[i]); + } for (int i = 0; i < block_height; i++) { - const int y_blk_raster_offset = (i >= block_height / 2) * 2; + const int y32_blk_raster_offset = (i >= (block_height >> 1)) << 1; + const int y16_blk_raster_offset = + ((i % (block_height >> 1)) >= (block_height >> 2)) << 1; uint32_t *luma_sse_sum_temp = luma_sse_sum + i * BW; for (int j = 0; j < block_width; j += 8) { const __m256i acc_sse = @@ -477,9 +450,13 @@ const __m256d window_error_2 = _mm256_mul_pd(diff_sse_pd_2, inv_num_ref_pixel_256bit); - // const int subblock_idx = y_blk_raster_offset + (j >= block_width / - // 2); - const int subblock_idx = y_blk_raster_offset + (j >= block_width / 2); + // const int subblock_idx = (y32 * 2 + x32) * 4 + (y16 * 2 + x16); + const int x32_blk_raster_offset = (j >= (block_width >> 1)); + const int x16_blk_raster_offset = + ((j % (block_width >> 1)) >= (block_width >> 2)); + const int subblock_idx = + ((y32_blk_raster_offset + x32_blk_raster_offset) << 2) + + (y16_blk_raster_offset + x16_blk_raster_offset); const __m256d blk_error = subblock_mses_reg[subblock_idx]; // const double combined_error = @@ -555,7 +532,7 @@ int tf_wgt_calc_lvl, const uint8_t *pred, uint32_t *accum, uint16_t *count) { const int is_high_bitdepth = frame_to_filter->flags & YV12_FLAG_HIGHBITDEPTH; - assert(block_size == BLOCK_32X32 && "Only support 32x32 block with avx2!"); + assert(block_size == BLOCK_64X64 && "Only support 64x64 block with avx2!"); assert(TF_WINDOW_LENGTH == 5 && "Only support window length 5 with avx2!"); assert(!is_high_bitdepth && "Only support low bit-depth with avx2!"); assert(num_planes >= 1 && num_planes <= MAX_MB_PLANE); @@ -584,16 +561,20 @@ // Smaller strength -> smaller filtering weight. double s_decay = pow((double)filter_strength / TF_STRENGTH_THRESHOLD, 2); s_decay = CLIP(s_decay, 1e-5, 1); - double d_factor[4] = { 0 }; - uint16_t frame_sse[SSE_STRIDE * BH] = { 0 }; - uint32_t luma_sse_sum[BW * BH] = { 0 }; + double d_factor[NUM_16X16] = { 0 }; + uint16_t *frame_sse = + (uint16_t *)aom_memalign(32, sizeof(frame_sse[0]) * SSE_STRIDE * BH); + uint32_t *luma_sse_sum = + (uint32_t *)aom_memalign(32, sizeof(luma_sse_sum[0]) * BW * BH); + memset(frame_sse, 0, sizeof(frame_sse[0]) * SSE_STRIDE * BH); + memset(luma_sse_sum, 0, sizeof(luma_sse_sum[0]) * BW * BH); - for (int subblock_idx = 0; subblock_idx < 4; subblock_idx++) { + double distance_threshold = min_frame_size * TF_SEARCH_DISTANCE_THRESHOLD; + distance_threshold = AOMMAX(distance_threshold, 1); + for (int subblock_idx = 0; subblock_idx < NUM_16X16; subblock_idx++) { // Larger motion vector -> smaller filtering weight. const MV mv = subblock_mvs[subblock_idx]; const double distance = sqrt(pow(mv.row, 2) + pow(mv.col, 2)); - double distance_threshold = min_frame_size * TF_SEARCH_DISTANCE_THRESHOLD; - distance_threshold = AOMMAX(distance_threshold, 1); d_factor[subblock_idx] = distance / distance_threshold; d_factor[subblock_idx] = AOMMAX(d_factor[subblock_idx], 1); } @@ -624,13 +605,35 @@ // will be more accurate. The luma sse sum is reused in both chroma // planes. if (plane == AOM_PLANE_U) { - for (unsigned int i = 0, k = 0; i < plane_h; i++) { - for (unsigned int j = 0; j < plane_w; j++, k++) { - for (int ii = 0; ii < (1 << ss_y_shift); ++ii) { - for (int jj = 0; jj < (1 << ss_x_shift); ++jj) { - const int yy = (i << ss_y_shift) + ii; // Y-coord on Y-plane. - const int xx = (j << ss_x_shift) + jj; // X-coord on Y-plane. - luma_sse_sum[i * BW + j] += frame_sse[yy * SSE_STRIDE + xx]; + if (ss_x_shift == 1 && ss_y_shift == 1) { + const __m256i zero_reg = _mm256_setzero_si256(); + for (unsigned int i = 0; i < plane_h; i++) { + const uint16_t *src_0 = &frame_sse[2 * i * SSE_STRIDE]; + const uint16_t *src_1 = &frame_sse[(2 * i + 1) * SSE_STRIDE]; + for (unsigned int j = 0; j < plane_w; j += 8) { + const __m256i reg0 = _mm256_loadu_si256((__m256i *)(src_0 + j * 2)); + const __m256i reg1 = _mm256_loadu_si256((__m256i *)(src_1 + j * 2)); + + const __m256i reg0_lo = _mm256_unpacklo_epi16(reg0, zero_reg); + const __m256i reg0_hi = _mm256_unpackhi_epi16(reg0, zero_reg); + const __m256i reg1_lo = _mm256_unpacklo_epi16(reg1, zero_reg); + const __m256i reg1_hi = _mm256_unpackhi_epi16(reg1, zero_reg); + + const __m256i reg_0 = _mm256_add_epi32(reg0_lo, reg1_lo); + const __m256i reg_1 = _mm256_add_epi32(reg0_hi, reg1_hi); + const __m256i res = _mm256_hadd_epi32(reg_0, reg_1); + _mm256_storeu_si256((__m256i *)&luma_sse_sum[i * BW + j], res); + } + } + } else { + for (unsigned int i = 0, k = 0; i < plane_h; i++) { + for (unsigned int j = 0; j < plane_w; j++, k++) { + for (int ii = 0; ii < (1 << ss_y_shift); ++ii) { + for (int jj = 0; jj < (1 << ss_x_shift); ++jj) { + const int yy = (i << ss_y_shift) + ii; // Y-coord on Y-plane. + const int xx = (j << ss_x_shift) + jj; // X-coord on Y-plane. + luma_sse_sum[i * BW + j] += frame_sse[yy * SSE_STRIDE + xx]; + } } } } @@ -644,4 +647,6 @@ weight_factor, d_factor, tf_wgt_calc_lvl); plane_offset += plane_h * plane_w; } + aom_free(frame_sse); + aom_free(luma_sse_sum); }
diff --git a/av1/encoder/x86/temporal_filter_sse2.c b/av1/encoder/x86/temporal_filter_sse2.c index 8a8c947..7fcea7e 100644 --- a/av1/encoder/x86/temporal_filter_sse2.c +++ b/av1/encoder/x86/temporal_filter_sse2.c
@@ -110,8 +110,8 @@ const double inv_num_ref_pixels, const double decay_factor, const double inv_factor, const double weight_factor, double *d_factor, int tf_wgt_calc_lvl) { - assert(((block_width == 16) || (block_width == 32)) && - ((block_height == 16) || (block_height == 32))); + assert(((block_width == 64) || (block_width == 32)) && + ((block_height == 64) || (block_height == 32))); uint32_t acc_5x5_sse[BH][BW]; @@ -170,21 +170,28 @@ } } - double subblock_mses_scaled[4]; - double d_factor_decayed[4]; - for (int idx = 0; idx < 4; idx++) { + double subblock_mses_scaled[NUM_16X16]; + double d_factor_decayed[NUM_16X16]; + for (int idx = 0; idx < NUM_16X16; idx++) { subblock_mses_scaled[idx] = subblock_mses[idx] * inv_factor; d_factor_decayed[idx] = d_factor[idx] * decay_factor; } if (tf_wgt_calc_lvl == 0) { for (int i = 0, k = 0; i < block_height; i++) { - const int y_blk_raster_offset = (i >= block_height / 2) * 2; + const int y32_blk_raster_offset = (i >= (block_height >> 1)) << 1; + const int y16_blk_raster_offset = + ((i % (block_height >> 1)) >= (block_height >> 2)) << 1; for (int j = 0; j < block_width; j++, k++) { const int pixel_value = frame2[i * stride2 + j]; uint32_t diff_sse = acc_5x5_sse[i][j] + luma_sse_sum[i * BW + j]; const double window_error = diff_sse * inv_num_ref_pixels; - const int subblock_idx = y_blk_raster_offset + (j >= block_width / 2); + const int x32_blk_raster_offset = (j >= (block_width >> 1)); + const int x16_blk_raster_offset = + ((j % (block_width >> 1)) >= (block_width >> 2)); + const int subblock_idx = + ((y32_blk_raster_offset + x32_blk_raster_offset) << 2) + + (y16_blk_raster_offset + x16_blk_raster_offset); const double combined_error = weight_factor * window_error + subblock_mses_scaled[subblock_idx]; @@ -198,13 +205,20 @@ } } else { for (int i = 0, k = 0; i < block_height; i++) { - const int y_blk_raster_offset = (i >= block_height / 2) * 2; + const int y32_blk_raster_offset = (i >= (block_height >> 1)) << 1; + const int y16_blk_raster_offset = + ((i % (block_height >> 1)) >= (block_height >> 2)) << 1; for (int j = 0; j < block_width; j++, k++) { const int pixel_value = frame2[i * stride2 + j]; uint32_t diff_sse = acc_5x5_sse[i][j] + luma_sse_sum[i * BW + j]; const double window_error = diff_sse * inv_num_ref_pixels; - const int subblock_idx = y_blk_raster_offset + (j >= block_width / 2); + const int x32_blk_raster_offset = (j >= (block_width >> 1)); + const int x16_blk_raster_offset = + ((j % (block_width >> 1)) >= (block_width >> 2)); + const int subblock_idx = + ((y32_blk_raster_offset + x32_blk_raster_offset) << 2) + + (y16_blk_raster_offset + x16_blk_raster_offset); const double combined_error = weight_factor * window_error + subblock_mses_scaled[subblock_idx]; @@ -228,7 +242,7 @@ int tf_wgt_calc_lvl, const uint8_t *pred, uint32_t *accum, uint16_t *count) { const int is_high_bitdepth = frame_to_filter->flags & YV12_FLAG_HIGHBITDEPTH; - assert(block_size == BLOCK_32X32 && "Only support 32x32 block with sse2!"); + assert(block_size == BLOCK_64X64 && "Only support 64x64 block with sse2!"); assert(TF_WINDOW_LENGTH == 5 && "Only support window length 5 with sse2!"); assert(!is_high_bitdepth && "Only support low bit-depth with sse2!"); assert(num_planes >= 1 && num_planes <= MAX_MB_PLANE); @@ -257,16 +271,20 @@ // Smaller strength -> smaller filtering weight. double s_decay = pow((double)filter_strength / TF_STRENGTH_THRESHOLD, 2); s_decay = CLIP(s_decay, 1e-5, 1); - double d_factor[4] = { 0 }; - uint16_t frame_sse[SSE_STRIDE * BH] = { 0 }; - uint32_t luma_sse_sum[BW * BH] = { 0 }; + double d_factor[NUM_16X16] = { 0 }; + uint16_t *frame_sse = + (uint16_t *)aom_memalign(32, sizeof(frame_sse[0]) * SSE_STRIDE * BH); + uint32_t *luma_sse_sum = + (uint32_t *)aom_memalign(32, sizeof(luma_sse_sum[0]) * BW * BH); + memset(frame_sse, 0, sizeof(frame_sse[0]) * SSE_STRIDE * BH); + memset(luma_sse_sum, 0, sizeof(luma_sse_sum[0]) * BW * BH); - for (int subblock_idx = 0; subblock_idx < 4; subblock_idx++) { + double distance_threshold = min_frame_size * TF_SEARCH_DISTANCE_THRESHOLD; + distance_threshold = AOMMAX(distance_threshold, 1); + for (int subblock_idx = 0; subblock_idx < NUM_16X16; subblock_idx++) { // Larger motion vector -> smaller filtering weight. const MV mv = subblock_mvs[subblock_idx]; const double distance = sqrt(pow(mv.row, 2) + pow(mv.col, 2)); - double distance_threshold = min_frame_size * TF_SEARCH_DISTANCE_THRESHOLD; - distance_threshold = AOMMAX(distance_threshold, 1); d_factor[subblock_idx] = distance / distance_threshold; d_factor[subblock_idx] = AOMMAX(d_factor[subblock_idx], 1); } @@ -317,4 +335,6 @@ weight_factor, d_factor, tf_wgt_calc_lvl); plane_offset += plane_h * plane_w; } + aom_free(frame_sse); + aom_free(luma_sse_sum); }
diff --git a/av1/ratectrl_rtc.cc b/av1/ratectrl_rtc.cc index 7ee58e9..9d176e5 100644 --- a/av1/ratectrl_rtc.cc +++ b/av1/ratectrl_rtc.cc
@@ -16,6 +16,7 @@ #include "aom/aom_encoder.h" #include "aom/aomcx.h" +#include "aom/internal/aom_codec_internal.h" #include "aom_dsp/aom_dsp_common.h" #include "aom_mem/aom_mem.h" #include "av1/common/common.h" @@ -72,6 +73,29 @@ namespace aom { +// Creates a setjmp target using `CPI->common.error->jmp` and sets +// `CPI->common.error->setjmp = 1`. Returns false on longjmp. This should be +// accompanied by a call to DISABLE_SETJMP using the same CPI before going out +// of scope. +#define ENABLE_SETJMP(CPI) \ + do { \ + struct aom_internal_error_info *const enable_setjmp_error = \ + (CPI)->common.error; \ + if (setjmp(enable_setjmp_error->jmp)) { \ + enable_setjmp_error->setjmp = 0; \ + return false; \ + } \ + enable_setjmp_error->setjmp = 1; \ + } while (0) + +// Sets CPI->common.error->setjmp = 0. +#define DISABLE_SETJMP(CPI) \ + do { \ + struct aom_internal_error_info *const enable_setjmp_error = \ + (CPI)->common.error; \ + enable_setjmp_error->setjmp = 0; \ + } while (0) + std::unique_ptr<AV1RateControlRTC> AV1RateControlRTC::Create( const AV1RateControlRtcConfig &cfg) { std::unique_ptr<AV1RateControlRTC> rc_api(new (std::nothrow) @@ -84,6 +108,7 @@ static_cast<AV1_PRIMARY *>(aom_memalign(32, sizeof(AV1_PRIMARY))); if (!rc_api->cpi_->ppi) return nullptr; av1_zero(*rc_api->cpi_->ppi); + rc_api->cpi_->common.error = &rc_api->cpi_->ppi->error; rc_api->cpi_->common.seq_params = &rc_api->cpi_->ppi->seq_params; av1_zero(*rc_api->cpi_->common.seq_params); if (!rc_api->InitRateControl(cfg)) return nullptr; @@ -224,7 +249,7 @@ av1_new_framerate(cpi_, cpi_->framerate); if (cpi_->svc.number_temporal_layers > 1 || cpi_->svc.number_spatial_layers > 1) { - int64_t target_bandwidth_svc = 0; + volatile int64_t target_bandwidth_svc = 0; for (int sl = 0; sl < cpi_->svc.number_spatial_layers; ++sl) { for (int tl = 0; tl < cpi_->svc.number_temporal_layers; ++tl) { const int layer = @@ -246,10 +271,14 @@ } } - if (cm->current_frame.frame_number == 0) av1_init_layer_context(cpi_); + ENABLE_SETJMP(cpi_); + if (cm->current_frame.frame_number == 0) { + av1_init_layer_context(cpi_); + } // This is needed to initialize external RC flag in layer context structure. cpi_->rc.rtc_external_ratectrl = 1; av1_update_layer_context_change_config(cpi_, target_bandwidth_svc); + DISABLE_SETJMP(cpi_); } check_reset_rc_flag(cpi_); return true; @@ -362,7 +391,8 @@ } AV1CdefInfo AV1RateControlRTC::GetCdefInfo() const { - av1_pick_cdef_from_qp(&cpi_->common, 0, 0); + av1_pick_cdef_from_qp(&cpi_->common, /*skip_cdef=*/0, /*is_screen_content=*/0, + /*avoid_uv_cdef=*/false); AV1CdefInfo cdef_level; cdef_level.cdef_strength_y = cpi_->common.cdef_info.cdef_strengths[0]; cdef_level.cdef_strength_uv = cpi_->common.cdef_info.cdef_uv_strengths[0];
diff --git a/build/cmake/aom_config_defaults.cmake b/build/cmake/aom_config_defaults.cmake index a8efb4a..1ca6310 100644 --- a/build/cmake/aom_config_defaults.cmake +++ b/build/cmake/aom_config_defaults.cmake
@@ -60,6 +60,8 @@ set_aom_detect_var(HAVE_FEXCEPT 0 "Internal flag, GNU fenv.h present for target.") set_aom_detect_var(HAVE_PTHREAD_H 0 "Internal flag, target pthread support.") +set_aom_detect_var(HAVE_PTHREAD_SETNAME_NP 0 + "Internal flag, target has pthread_setname_np API support.") set_aom_detect_var(HAVE_UNISTD_H 0 "Internal flag, unistd.h present for target.") set_aom_detect_var(HAVE_WXWIDGETS 0 "WxWidgets present.")
diff --git a/build/cmake/aom_configure.cmake b/build/cmake/aom_configure.cmake index 07e354a..b34748d 100644 --- a/build/cmake/aom_configure.cmake +++ b/build/cmake/aom_configure.cmake
@@ -14,6 +14,7 @@ set(AOM_BUILD_CMAKE_AOM_CONFIGURE_CMAKE_ 1) include(FindThreads) +include(CheckSymbolExists) include("${AOM_ROOT}/build/cmake/aom_config_defaults.cmake") include("${AOM_ROOT}/build/cmake/aom_experiment_deps.cmake") @@ -258,6 +259,15 @@ set(HAVE_PTHREAD_H ${CMAKE_USE_PTHREADS_INIT}) aom_check_source_compiles("unistd_check" "#include <unistd.h>" HAVE_UNISTD_H) +if(HAVE_PTHREAD_H) + # Check for pthread setname_np API + aom_push_var(CMAKE_REQUIRED_DEFINITIONS "-D_GNU_SOURCE") + check_symbol_exists(pthread_setname_np pthread.h _HAS_PTHREAD_SETNAME_NP) + aom_pop_var(CMAKE_REQUIRED_DEFINITIONS) + + set(HAVE_PTHREAD_SETNAME_NP ${_HAS_PTHREAD_SETNAME_NP}) +endif() + if(NOT WIN32) aom_push_var(CMAKE_REQUIRED_LIBRARIES "m") aom_check_c_compiles("fenv_check" "#define _GNU_SOURCE @@ -432,6 +442,20 @@ unset(AOM_LIB_LINK_TYPE) endif() +# Sanitize boolean variables to ensure they are 0 or 1. +foreach(aom_config_var ${AOM_CONFIG_VARS}) + if( + NOT aom_config_var MATCHES + "AOM_RTCD_FLAGS|CONFIG_MAX_DECODE_PROFILE|DECODE_HEIGHT_LIMIT|DECODE_WIDTH_LIMIT" + ) + if(${aom_config_var}) + set(${aom_config_var} 1) + else() + set(${aom_config_var} 0) + endif() + endif() +endforeach() + # Generate aom_config templates. set(aom_config_asm_template "${AOM_CONFIG_DIR}/config/aom_config.asm.cmake") set(aom_config_h_template "${AOM_CONFIG_DIR}/config/aom_config.h.cmake")
diff --git a/build/cmake/aom_install.cmake b/build/cmake/aom_install.cmake index 4291782..1ac795e 100644 --- a/build/cmake/aom_install.cmake +++ b/build/cmake/aom_install.cmake
@@ -20,7 +20,9 @@ if(CONFIG_AV1_ENCODER) list(APPEND AOM_INSTALL_INCS "${AOM_ROOT}/aom/aomcx.h" "${AOM_ROOT}/aom/aom_encoder.h" - "${AOM_ROOT}/aom/aom_external_partition.h") + "${AOM_ROOT}/aom/aom_ext_ratectrl.h" + "${AOM_ROOT}/aom/aom_external_partition.h" + "${AOM_ROOT}/aom/aom_tpl.h") endif() # Generate aom.pc and setup dependencies to ensure it is created when necessary.
diff --git a/build/cmake/rtcd.pl b/build/cmake/rtcd.pl index ba19761..10f6597 100755 --- a/build/cmake/rtcd.pl +++ b/build/cmake/rtcd.pl
@@ -221,11 +221,9 @@ # sub common_top() { my $include_guard = uc($opts{sym})."_H_"; - my @time = localtime; - my $year = $time[5] + 1900; print <<EOF; /* - * Copyright (c) ${year}, Alliance for Open Media. All rights reserved. + * Copyright (c) 2017, Alliance for Open Media. All rights reserved. * * This source code is subject to the terms of the BSD 2 Clause License and * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
diff --git a/docs.cmake b/docs.cmake index 154cb0d..1b76161 100644 --- a/docs.cmake +++ b/docs.cmake
@@ -199,6 +199,16 @@ "Lightfield bitstream parsing example.") endif() +if(CONFIG_AV1_DECODER AND CONFIG_AV1_ENCODER) + set(AOM_DOXYGEN_EXAMPLE_SOURCES + ${AOM_DOXYGEN_EXAMPLE_SOURCES} + "${AOM_ROOT}/examples/low_complexity_mode_encoder.c") + + set(AOM_DOXYGEN_EXAMPLE_DESCRIPTIONS + ${AOM_DOXYGEN_EXAMPLE_DESCRIPTIONS} + "Low complexity decode mode encoder example.") +endif() + # Iterates over list named by $list_name and appends each item to $AOM_DOXYFILE # as values assigned to $var_name with no line breaks between list items. # Appends a new line after the entire config variable is expanded.
diff --git a/examples/inspect.c b/examples/inspect.c index 9042eb8..43fb049 100644 --- a/examples/inspect.c +++ b/examples/inspect.c
@@ -65,7 +65,8 @@ INTRABC_LAYER = 1 << 17, PALETTE_LAYER = 1 << 18, UV_PALETTE_LAYER = 1 << 19, - ALL_LAYERS = (1 << 20) - 1 + BITS_LAYER = 1 << 20, + ALL_LAYERS = (1 << 21) - 1 } LayerType; static LayerType layers = 0; @@ -115,6 +116,8 @@ ARG_DEF("plt", "palette", 0, "Dump Palette Size"); static const arg_def_t dump_uv_palette_arg = ARG_DEF("uvp", "uv_palette", 0, "Dump UV Palette Size"); +static const arg_def_t dump_bits_arg = + ARG_DEF("sb", "sb_bits", 0, "Dump Bits of each superblock"); static const arg_def_t usage_arg = ARG_DEF("h", "help", 0, "Help"); static const arg_def_t skip_non_transform_arg = ARG_DEF( "snt", "skip_non_transform", 1, "Skip is counted as a non transform."); @@ -149,6 +152,7 @@ &dump_intrabc_arg, &dump_palette_arg, &dump_uv_palette_arg, + &dump_bits_arg, &usage_arg, &skip_non_transform_arg, &combined_arg, @@ -275,6 +279,7 @@ { "compound_type", offsetof(insp_mi_data, compound_type) }, { "referenceFrame", offsetof(insp_mi_data, ref_frame) }, { "skip", offsetof(insp_mi_data, skip) }, + { "sb_bits", offsetof(insp_mi_data, sb_bits) }, }; int parm_count = sizeof(parm_offsets) / sizeof(parm_offsets[0]); @@ -704,6 +709,10 @@ buf += put_block_info(buf, palette_map, "uv_palette", offsetof(insp_mi_data, uv_palette), 0); } + if (layers & BITS_LAYER) { + buf += put_block_info(buf, NULL, "sb_bits", offsetof(insp_mi_data, sb_bits), + 0); + } if (combined_parm_count > 0) buf += put_combined(buf); if (layers & REFERENCE_FRAME_LAYER) { buf += put_block_info(buf, refs_map, "referenceFrame", @@ -928,6 +937,8 @@ layers |= PALETTE_LAYER; else if (arg_match(&arg, &dump_uv_palette_arg, argi)) layers |= UV_PALETTE_LAYER; + else if (arg_match(&arg, &dump_bits_arg, argi)) + layers |= BITS_LAYER; else if (arg_match(&arg, &dump_all_arg, argi)) layers |= ALL_LAYERS; else if (arg_match(&arg, &compress_arg, argi))
diff --git a/examples/lightfield_encoder.c b/examples/lightfield_encoder.c index 63d72d4..dc8e890 100644 --- a/examples/lightfield_encoder.c +++ b/examples/lightfield_encoder.c
@@ -434,7 +434,6 @@ // lf_blocksize X lf_blocksize images will all use the reference image // in the middle of the block of images. int lf_blocksize; - aom_codec_ctx_t codec; aom_codec_enc_cfg_t cfg; aom_image_t raw; aom_image_t raw_shift; @@ -485,7 +484,8 @@ // Configuration res = aom_codec_enc_config_default(encoder, &cfg, 0); - if (res) die_codec(&codec, "Failed to get default codec config."); + if (res) + die("Failed to get default codec config: %s", aom_codec_err_to_string(res)); cfg.g_w = w; cfg.g_h = h;
diff --git a/examples/lossless_encoder.c b/examples/lossless_encoder.c index 194c717..c3d3b93 100644 --- a/examples/lossless_encoder.c +++ b/examples/lossless_encoder.c
@@ -96,7 +96,8 @@ aom_codec_ctx_t codec; res = aom_codec_enc_config_default(encoder, &cfg, 0); - if (res) die_codec(&codec, "Failed to get default codec config."); + if (res) + die("Failed to get default codec config: %s", aom_codec_err_to_string(res)); cfg.g_w = info.frame_width; cfg.g_h = info.frame_height;
diff --git a/examples/low_complexity_mode_encoder.c b/examples/low_complexity_mode_encoder.c new file mode 100644 index 0000000..31d8f06 --- /dev/null +++ b/examples/low_complexity_mode_encoder.c
@@ -0,0 +1,177 @@ +/* + * Copyright (c) 2026, Alliance for Open Media. All rights reserved. + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +// Low Complexity (LC) Mode Encoder +// ================================ +// +// This is an example of a low complexity mode encoder. It takes an input +// file in Y4M format, passes it through the encoder with LC mode on, and +// writes the compressed frames to disk in IVF format. + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "aom/aom_encoder.h" +#include "aom/aomcx.h" +#include "common/tools_common.h" +#include "common/video_writer.h" + +static const char *exec_name; + +void usage_exit(void) { + fprintf(stderr, + "Usage: %s <codec> <width> <height> <infile> <outfile> " + "<keyframe-interval> <frames to encode>\n", + exec_name); + exit(EXIT_FAILURE); +} + +static int encode_frame(aom_codec_ctx_t *codec, aom_image_t *img, + int frame_index, int flags, AvxVideoWriter *writer) { + int got_pkts = 0; + aom_codec_iter_t iter = NULL; + const aom_codec_cx_pkt_t *pkt = NULL; + const aom_codec_err_t res = + aom_codec_encode(codec, img, frame_index, 1, flags); + if (res != AOM_CODEC_OK) die_codec(codec, "Failed to encode frame"); + + while ((pkt = aom_codec_get_cx_data(codec, &iter)) != NULL) { + got_pkts = 1; + + if (pkt->kind == AOM_CODEC_CX_FRAME_PKT) { + const int keyframe = (pkt->data.frame.flags & AOM_FRAME_IS_KEY) != 0; + if (!aom_video_writer_write_frame(writer, pkt->data.frame.buf, + pkt->data.frame.sz, + pkt->data.frame.pts)) { + die_codec(codec, "Failed to write compressed frame"); + } + printf(keyframe ? "K" : "."); + fflush(stdout); + } + } + + return got_pkts; +} + +int main(int argc, char **argv) { + FILE *infile = NULL; + aom_codec_ctx_t codec; + aom_codec_enc_cfg_t cfg; + int frame_count = 0; + aom_image_t raw; + aom_codec_err_t res; + AvxVideoInfo info; + AvxVideoWriter *writer = NULL; + const int fps = 30; + const int bitrate = 400; + int keyframe_interval = 0; + int max_frames = 0; + int frames_encoded = 0; + const char *codec_arg = NULL; + const char *width_arg = NULL; + const char *height_arg = NULL; + const char *infile_arg = NULL; + const char *outfile_arg = NULL; + const char *keyframe_interval_arg = NULL; + const int usage = 0; // AOM_USAGE_GOOD_QUALITY + const int speed = 2; + + exec_name = argv[0]; + + // Clear explicitly, as simply assigning "{ 0 }" generates + // "missing-field-initializers" warning in some compilers. + memset(&info, 0, sizeof(info)); + + if (argc != 8) die("Invalid number of arguments"); + + codec_arg = argv[1]; + width_arg = argv[2]; + height_arg = argv[3]; + infile_arg = argv[4]; + outfile_arg = argv[5]; + keyframe_interval_arg = argv[6]; + max_frames = (int)strtol(argv[7], NULL, 0); + + aom_codec_iface_t *encoder = get_aom_encoder_by_short_name(codec_arg); + if (!encoder) die("Unsupported codec."); + + info.codec_fourcc = get_fourcc_by_aom_encoder(encoder); + info.frame_width = (int)strtol(width_arg, NULL, 0); + info.frame_height = (int)strtol(height_arg, NULL, 0); + info.time_base.numerator = 1; + info.time_base.denominator = fps; + + if (info.frame_width <= 0 || info.frame_height <= 0 || + (info.frame_width % 2) != 0 || (info.frame_height % 2) != 0) { + die("Invalid frame size: %dx%d", info.frame_width, info.frame_height); + } + + if (!aom_img_alloc(&raw, AOM_IMG_FMT_I420, info.frame_width, + info.frame_height, 1)) { + die("Failed to allocate image."); + } + + keyframe_interval = (int)strtol(keyframe_interval_arg, NULL, 0); + if (keyframe_interval < 0) die("Invalid keyframe interval value."); + + printf("Using %s\n", aom_codec_iface_name(encoder)); + + res = aom_codec_enc_config_default(encoder, &cfg, usage); + if (res) + die("Failed to get default codec config: %s", aom_codec_err_to_string(res)); + + cfg.g_w = info.frame_width; + cfg.g_h = info.frame_height; + cfg.g_timebase.num = info.time_base.numerator; + cfg.g_timebase.den = info.time_base.denominator; + cfg.rc_target_bitrate = bitrate; + cfg.g_lag_in_frames = 35; + + writer = aom_video_writer_open(outfile_arg, kContainerIVF, &info); + if (!writer) die("Failed to open %s for writing.", outfile_arg); + + if (!(infile = fopen(infile_arg, "rb"))) + die("Failed to open %s for reading.", infile_arg); + + if (aom_codec_enc_init(&codec, encoder, &cfg, 0)) + die("Failed to initialize encoder"); + + if (aom_codec_control(&codec, AOME_SET_CPUUSED, speed)) + die_codec(&codec, "Failed to set cpu-used"); + + if (aom_codec_control(&codec, AV1E_SET_ENABLE_LOW_COMPLEXITY_DECODE, 1)) + die_codec(&codec, "Failed to set low-complexity mode"); + + // Encode frames. + while (aom_img_read(&raw, infile)) { + int flags = 0; + if (keyframe_interval > 0 && frame_count % keyframe_interval == 0) + flags |= AOM_EFLAG_FORCE_KF; + encode_frame(&codec, &raw, frame_count++, flags, writer); + frames_encoded++; + if (max_frames > 0 && frames_encoded >= max_frames) break; + } + + // Flush encoder. + while (encode_frame(&codec, NULL, -1, 0, writer)) continue; + + printf("\n"); + fclose(infile); + printf("Processed %d frames.\n", frame_count); + + aom_img_free(&raw); + if (aom_codec_destroy(&codec)) die_codec(&codec, "Failed to destroy codec."); + + aom_video_writer_close(writer); + + return EXIT_SUCCESS; +}
diff --git a/examples/multilayer_metadata.cc b/examples/multilayer_metadata.cc index 9427029..7c7149a 100644 --- a/examples/multilayer_metadata.cc +++ b/examples/multilayer_metadata.cc
@@ -12,12 +12,12 @@ #include "examples/multilayer_metadata.h" #include <assert.h> -#include <inttypes.h> #include <limits.h> #include <stdio.h> #include <stdlib.h> #include <string.h> +#include <cinttypes> #include <cmath> #include <fstream> #include <iostream> @@ -430,6 +430,49 @@ return true; } +bool parse_multilayer_layer_local_metadata( + std::ifstream &file, int min_indent, int *line_idx, + std::vector<FrameLocalMetadata> &frames) { + bool has_list_prefix; + int indent = -1; + std::string field_name; + ParsedValue value; + bool syntax_error; + while (parse_line(file, min_indent, /*is_list=*/true, &indent, + &has_list_prefix, line_idx, &field_name, &value, + &syntax_error)) { + if (has_list_prefix) { + frames.push_back({}); + } + if (frames.empty()) { + fprintf(stderr, "Error: Missing list prefix '-' at line %d\n", *line_idx); + return false; + } + + if (field_name == "frame_idx") { + RETURN_IF_FALSE( + value.IntegerValueInRange(0, std::numeric_limits<long>::max(), + *line_idx, &frames.back().frame_idx)); + } else if (field_name == "alpha") { + RETURN_IF_FALSE(parse_multilayer_layer_alpha( + file, + /*min_indent=*/indent + 1, line_idx, &frames.back().alpha)); + + } else if (field_name == "depth") { + RETURN_IF_FALSE(parse_multilayer_layer_depth( + file, + /*min_indent=*/indent + 1, line_idx, &frames.back().depth)); + + } else { + fprintf(stderr, "Error: Unknown field %s at line %d\n", + field_name.c_str(), *line_idx); + return false; + } + } + if (syntax_error) return false; + return true; +} + bool validate_layer(const LayerMetadata &layer, bool layer_has_alpha, bool layer_has_depth) { if (layer_has_alpha != (layer.layer_type == MULTILAYER_LAYER_TYPE_ALPHA && @@ -475,7 +518,8 @@ // Validate the previous layer. if (!layers.empty()) { - validate_layer(layers.back(), layer_has_alpha, layer_has_depth); + RETURN_IF_FALSE( + validate_layer(layers.back(), layer_has_alpha, layer_has_depth)); } if (layers.size() == 1 && layers.back().layer_color_description.second) { fprintf(stderr, @@ -520,24 +564,25 @@ layer->layer_color_description = value_present(color_properties); } else if ((field_name == "alpha")) { layer_has_alpha = true; - RETURN_IF_FALSE(parse_multilayer_layer_alpha( - file, - /*min_indent=*/indent + 1, line_idx, &layer->global_alpha_info)); + RETURN_IF_FALSE(parse_multilayer_layer_alpha(file, + /*min_indent=*/indent + 1, + line_idx, &layer->alpha)); } else if (field_name == "depth") { layer_has_depth = true; - RETURN_IF_FALSE(parse_multilayer_layer_depth( - file, - /*min_indent=*/indent + 1, line_idx, &layer->global_depth_info)); - if ((layer->global_depth_info.d_min.second || - layer->global_depth_info.d_max.second) && - layer->global_depth_info.disparity_ref_view_id == - (layers.size() - 1)) { + RETURN_IF_FALSE(parse_multilayer_layer_depth(file, + /*min_indent=*/indent + 1, + line_idx, &layer->depth)); + if ((layer->depth.d_min.second || layer->depth.d_max.second) && + layer->depth.disparity_ref_view_id == (layers.size() - 1)) { fprintf(stderr, "disparity_ref_view_id must be different from the layer's id " "for layer %d (zero-based index)\n", static_cast<int>(layers.size()) - 1); return false; } + } else if (field_name == "local_metadata") { + RETURN_IF_FALSE(parse_multilayer_layer_local_metadata( + file, /*min_indent=*/indent + 1, line_idx, layer->local_metadata)); } else { fprintf(stderr, "Error: Unknown field %s at line %d\n", field_name.c_str(), *line_idx); @@ -545,7 +590,8 @@ } } if (syntax_error) return false; - validate_layer(layers.back(), layer_has_alpha, layer_has_depth); + RETURN_IF_FALSE( + validate_layer(layers.back(), layer_has_alpha, layer_has_depth)); return true; } @@ -786,6 +832,33 @@ return true; } +void print_alpha_information(const AlphaInformation &alpha) { + printf(" alpha_simple_flag: %d\n", alpha.alpha_simple_flag); + if (!alpha.alpha_simple_flag) { + printf(" alpha_bit_depth: %d\n", alpha.alpha_bit_depth); + printf(" alpha_clip_idc: %d\n", alpha.alpha_clip_idc); + printf(" alpha_incr_flag: %d\n", alpha.alpha_incr_flag); + printf(" alpha_transparent_value: %hu\n", alpha.alpha_transparent_value); + printf(" alpha_opaque_value: %hu\n", alpha.alpha_opaque_value); + printf(" alpha_color_description: %s\n", + format_color_properties(alpha.alpha_color_description).c_str()); + } +} + +void print_depth_information(const DepthInformation &depth) { + printf(" z_near: %s\n", + format_depth_representation_element(depth.z_near).c_str()); + printf(" z_far: %s\n", + format_depth_representation_element(depth.z_far).c_str()); + printf(" d_min: %s\n", + format_depth_representation_element(depth.d_min).c_str()); + printf(" d_max: %s\n", + format_depth_representation_element(depth.d_max).c_str()); + printf(" depth_representation_type: %d\n", + depth.depth_representation_type); + printf(" disparity_ref_view_id: %d\n", depth.disparity_ref_view_id); +} + } // namespace double depth_representation_element_to_double( @@ -881,45 +954,21 @@ printf(" layer_color_description: %s\n", format_color_properties(layer.layer_color_description).c_str()); if (layer.layer_type == MULTILAYER_LAYER_TYPE_ALPHA) { - printf(" alpha:\n"); - printf(" alpha_use_idc: %d\n", layer.global_alpha_info.alpha_use_idc); - printf(" alpha_simple_flag: %d\n", - layer.global_alpha_info.alpha_simple_flag); - if (!layer.global_alpha_info.alpha_simple_flag) { - printf(" alpha_bit_depth: %d\n", - layer.global_alpha_info.alpha_bit_depth); - printf(" alpha_clip_idc: %d\n", - layer.global_alpha_info.alpha_clip_idc); - printf(" alpha_incr_flag: %d\n", - layer.global_alpha_info.alpha_incr_flag); - printf(" alpha_transparent_value: %hu\n", - layer.global_alpha_info.alpha_transparent_value); - printf(" alpha_opaque_value: %hu\n", - layer.global_alpha_info.alpha_opaque_value); - printf(" alpha_color_description: %s\n", - format_color_properties( - layer.global_alpha_info.alpha_color_description) - .c_str()); + if (layer.layer_metadata_scope >= SCOPE_GLOBAL) { + printf(" global alpha:\n"); + print_alpha_information(layer.alpha); + } + for (const FrameLocalMetadata &local_metadata : layer.local_metadata) { + printf(" local alpha for frame %ld:\n", local_metadata.frame_idx); + print_alpha_information(local_metadata.alpha); } } else if (layer.layer_type == MULTILAYER_LAYER_TYPE_DEPTH) { - printf(" depth:\n"); - printf(" z_near: %s\n", - format_depth_representation_element(layer.global_depth_info.z_near) - .c_str()); - printf(" z_far: %s\n", - format_depth_representation_element(layer.global_depth_info.z_far) - .c_str()); - printf(" d_min: %s\n", - format_depth_representation_element(layer.global_depth_info.d_min) - .c_str()); - printf(" d_max: %s\n", - format_depth_representation_element(layer.global_depth_info.d_max) - .c_str()); - printf(" depth_representation_type: %d\n", - layer.global_depth_info.depth_representation_type); - printf(" disparity_ref_view_id: %d\n", - layer.global_depth_info.disparity_ref_view_id); - printf("\n"); + printf(" global depth:\n"); + print_depth_information(layer.depth); + for (const FrameLocalMetadata &local_metadata : layer.local_metadata) { + printf(" local depth for frame %ld:\n", local_metadata.frame_idx); + print_depth_information(local_metadata.depth); + } } } printf("\n");
diff --git a/examples/multilayer_metadata.h b/examples/multilayer_metadata.h index 392f1c5..1ac1c3c 100644 --- a/examples/multilayer_metadata.h +++ b/examples/multilayer_metadata.h
@@ -61,7 +61,7 @@ std::pair<DepthRepresentationElement, bool> z_far; std::pair<DepthRepresentationElement, bool> d_min; std::pair<DepthRepresentationElement, bool> d_max; - uint8_t depth_representation_type; // [0, 15] + uint8_t depth_representation_type; // [0, 2]. Values 3 to 15 are reserved. // Only relevant if d_min or d_max are present. uint8_t disparity_ref_view_id; // [0, 3] }; @@ -110,6 +110,14 @@ // 4 to 7 are reserved. }; +struct FrameLocalMetadata { + long frame_idx; + // Relevant for MULTILAYER_LAYER_TYPE_ALPHA with scope != SCOPE_GLOBAL. + AlphaInformation alpha; + // Relevant for MULTILAYER_LAYER_TYPE_DEPTH with scope != SCOPE_GLOBAL. + DepthInformation depth; +}; + struct LayerMetadata { LayerType layer_type; // [0, 31] bool luma_plane_only_flag; @@ -121,9 +129,12 @@ std::pair<ColorProperties, bool> layer_color_description; // Relevant for MULTILAYER_LAYER_TYPE_ALPHA with scope >= SCOPE_GLOBAL. - AlphaInformation global_alpha_info; + AlphaInformation alpha; // Relevant for MULTILAYER_LAYER_TYPE_DEPTH with scope >= SCOPE_GLOBAL. - DepthInformation global_depth_info; + DepthInformation depth; + + // Relevant when scope != SCOPE_GLOBAL. + std::vector<FrameLocalMetadata> local_metadata; }; struct MultilayerMetadata {
diff --git a/examples/noise_model.c b/examples/noise_model.c index 1816e33..e23185e 100644 --- a/examples/noise_model.c +++ b/examples/noise_model.c
@@ -38,6 +38,7 @@ * --auto-alt-ref=2 --bit-depth=8 --film-grain-table=film_grain.tbl \ * -o denoised_with_grain_params.ivf denoised.854_480.yuv */ +#include <inttypes.h> #include <math.h> #include <stdio.h> #include <stdlib.h>
diff --git a/examples/scalable_encoder.c b/examples/scalable_encoder.c index 91d35a2..963b4c3 100644 --- a/examples/scalable_encoder.c +++ b/examples/scalable_encoder.c
@@ -186,7 +186,8 @@ aom_codec_ctx_t codec; res = aom_codec_enc_config_default(encoder, &cfg, 0); - if (res) die_codec(&codec, "Failed to get default codec config."); + if (res) + die("Failed to get default codec config: %s", aom_codec_err_to_string(res)); cfg.g_w = info.frame_width; cfg.g_h = info.frame_height;
diff --git a/examples/set_maps.c b/examples/set_maps.c index 4f41b47..0d03983 100644 --- a/examples/set_maps.c +++ b/examples/set_maps.c
@@ -167,7 +167,8 @@ printf("Using %s\n", aom_codec_iface_name(encoder)); res = aom_codec_enc_config_default(encoder, &cfg, usage); - if (res) die_codec(&codec, "Failed to get default codec config."); + if (res) + die("Failed to get default codec config: %s", aom_codec_err_to_string(res)); cfg.g_w = info.frame_width; cfg.g_h = info.frame_height;
diff --git a/examples/simple_encoder.c b/examples/simple_encoder.c index b08f97b..f5ddb3b 100644 --- a/examples/simple_encoder.c +++ b/examples/simple_encoder.c
@@ -212,7 +212,8 @@ printf("Using %s\n", aom_codec_iface_name(encoder)); res = aom_codec_enc_config_default(encoder, &cfg, usage); - if (res) die_codec(&codec, "Failed to get default codec config."); + if (res) + die("Failed to get default codec config: %s", aom_codec_err_to_string(res)); cfg.g_w = info.frame_width; cfg.g_h = info.frame_height;
diff --git a/examples/svc_encoder_rtc.cc b/examples/svc_encoder_rtc.cc index 10c2102..c7956bc 100644 --- a/examples/svc_encoder_rtc.cc +++ b/examples/svc_encoder_rtc.cc
@@ -13,13 +13,13 @@ // encoding scheme for RTC video applications. #include <assert.h> -#include <inttypes.h> #include <limits.h> #include <math.h> #include <stdio.h> #include <stdlib.h> #include <string.h> +#include <cinttypes> #include <memory> #include "config/aom_config.h" @@ -1463,8 +1463,55 @@ } } +static void write_alpha_information( + struct aom_write_bit_buffer *buffer, + const libaom_examples::AlphaInformation &alpha_info) { + write_literal(buffer, alpha_info.alpha_use_idc, 2); + write_literal(buffer, alpha_info.alpha_simple_flag, 1); + if (!alpha_info.alpha_simple_flag) { + write_literal(buffer, alpha_info.alpha_bit_depth, 3, /*offset=*/8); + write_literal(buffer, alpha_info.alpha_clip_idc, 2); + write_literal(buffer, alpha_info.alpha_incr_flag, 1); + write_literal(buffer, alpha_info.alpha_transparent_value, + alpha_info.alpha_bit_depth + 1); + write_literal(buffer, alpha_info.alpha_opaque_value, + alpha_info.alpha_bit_depth + 1); + if (buffer->bit_offset % 8 != 0) { + // ai_byte_alignment_bits + write_literal(buffer, 0, 8 - (buffer->bit_offset % 8)); + } + assert(buffer->bit_offset % 8 == 0); + + write_literal(buffer, 0, 6); // ai_reserved_6bits + write_color_properties(buffer, alpha_info.alpha_color_description); + } else { + write_literal(buffer, 0, 5); // ai_reserved_5bits + } +} + +static void write_depth_information( + struct aom_write_bit_buffer *buffer, + const libaom_examples::DepthInformation &depth_info) { + write_literal(buffer, depth_info.z_near.second, 1); + write_literal(buffer, depth_info.z_far.second, 1); + write_literal(buffer, depth_info.d_min.second, 1); + write_literal(buffer, depth_info.d_max.second, 1); + write_literal(buffer, depth_info.depth_representation_type, 4); + if (depth_info.d_min.second || depth_info.d_max.second) { + write_literal(buffer, depth_info.disparity_ref_view_id, 2); + } + write_depth_representation_element(buffer, depth_info.z_near); + write_depth_representation_element(buffer, depth_info.z_far); + write_depth_representation_element(buffer, depth_info.d_min); + write_depth_representation_element(buffer, depth_info.d_max); + if (buffer->bit_offset % 8 != 0) { + write_literal(buffer, 0, 8 - (buffer->bit_offset % 8)); + } +} + static void add_multilayer_metadata( - aom_image_t *frame, const libaom_examples::MultilayerMetadata &multilayer) { + aom_image_t *frame, const libaom_examples::MultilayerMetadata &multilayer, + int frame_idx, int spatial_id) { // Large enough buffer for the multilayer metadata. // Each layer's metadata is less than 100 bytes and there are at most 4 // layers. @@ -1506,51 +1553,12 @@ if (layer.layer_type == libaom_examples::MULTILAYER_LAYER_TYPE_ALPHA && layer.layer_metadata_scope >= libaom_examples::SCOPE_GLOBAL) { - const libaom_examples::AlphaInformation &alpha_info = - layer.global_alpha_info; - write_literal(&buffer, alpha_info.alpha_use_idc, 2); - write_literal(&buffer, alpha_info.alpha_simple_flag, 1); - if (!alpha_info.alpha_simple_flag) { - write_literal(&buffer, alpha_info.alpha_bit_depth, 3, /*offset=*/8); - write_literal(&buffer, alpha_info.alpha_clip_idc, 2); - write_literal(&buffer, alpha_info.alpha_incr_flag, 1); - write_literal(&buffer, alpha_info.alpha_transparent_value, - alpha_info.alpha_bit_depth + 1); - write_literal(&buffer, alpha_info.alpha_opaque_value, - alpha_info.alpha_bit_depth + 1); - if (buffer.bit_offset % 8 != 0) { - // ai_byte_alignment_bits - write_literal(&buffer, 0, 8 - (buffer.bit_offset % 8)); - } - assert(buffer.bit_offset % 8 == 0); - - write_literal(&buffer, 0, 6); // ai_reserved_6bits - write_color_properties(&buffer, alpha_info.alpha_color_description); - } else { - write_literal(&buffer, 0, 5); // ai_reserved_5bits - } - + write_alpha_information(&buffer, layer.alpha); assert(buffer.bit_offset % 8 == 0); } else if (layer.layer_type == libaom_examples::MULTILAYER_LAYER_TYPE_DEPTH && layer.layer_metadata_scope >= libaom_examples::SCOPE_GLOBAL) { - const libaom_examples::DepthInformation &depth_info = - layer.global_depth_info; - write_literal(&buffer, depth_info.z_near.second, 1); - write_literal(&buffer, depth_info.z_far.second, 1); - write_literal(&buffer, depth_info.d_min.second, 1); - write_literal(&buffer, depth_info.d_max.second, 1); - write_literal(&buffer, depth_info.depth_representation_type, 4); - if (depth_info.d_min.second || depth_info.d_max.second) { - write_literal(&buffer, depth_info.disparity_ref_view_id, 2); - } - write_depth_representation_element(&buffer, depth_info.z_near); - write_depth_representation_element(&buffer, depth_info.z_far); - write_depth_representation_element(&buffer, depth_info.d_min); - write_depth_representation_element(&buffer, depth_info.d_max); - if (buffer.bit_offset % 8 != 0) { - write_literal(&buffer, 0, 8 - (buffer.bit_offset % 8)); - } + write_depth_information(&buffer, layer.depth); assert(buffer.bit_offset % 8 == 0); } @@ -1572,6 +1580,36 @@ AOM_MIF_KEY_FRAME)) { die("Error: Failed to add metadata\n"); } + + if ((int)multilayer.layers.size() > spatial_id) { + const libaom_examples::LayerMetadata &layer = multilayer.layers[spatial_id]; + for (const libaom_examples::FrameLocalMetadata &local_metadata : + layer.local_metadata) { + if (local_metadata.frame_idx == frame_idx) { + if (layer.layer_type == libaom_examples::MULTILAYER_LAYER_TYPE_ALPHA) { + buffer = { data.data(), 0 }; + write_alpha_information(&buffer, local_metadata.alpha); + if (aom_img_add_metadata(frame, + 34 /*METADATA_TYPE_ALPHA_INFORMATION*/, + buffer.bit_buffer, buffer.bit_offset / 8, + AOM_MIF_ANY_FRAME_LAYER_SPECIFIC)) { + die("Error: Failed to add metadata\n"); + } + } else if (layer.layer_type == + libaom_examples::MULTILAYER_LAYER_TYPE_DEPTH) { + buffer = { data.data(), 0 }; + write_depth_information(&buffer, local_metadata.depth); + if (aom_img_add_metadata(frame, + 35 /*METADATA_TYPE_DEPTH_INFORMATION*/, + buffer.bit_buffer, buffer.bit_offset / 8, + AOM_MIF_ANY_FRAME_LAYER_SPECIFIC)) { + die("Error: Failed to add metadata\n"); + } + } + break; + } + } + } } #if CONFIG_AV1_DECODER @@ -1637,7 +1675,7 @@ } #endif // CONFIG_AV1_DECODER -struct psnr_stats { +struct PsnrStats { // The second element of these arrays is reserved for high bitdepth. uint64_t psnr_sse_total[2]; uint64_t psnr_samples_total[2]; @@ -1645,19 +1683,22 @@ int psnr_count[2]; }; -static void show_psnr(struct psnr_stats *psnr_stream, double peak) { - double ovpsnr; +static void show_psnr(struct PsnrStats *psnr_stream, double peak, + int num_layers) { + for (int sl = 0; sl < num_layers; ++sl) { + if (!psnr_stream[sl].psnr_count[0]) continue; - if (!psnr_stream->psnr_count[0]) return; + fprintf(stderr, "\nPSNR (Layer %d, Overall/Avg/Y/U/V)", sl); + const double ovpsnr = + sse_to_psnr((double)psnr_stream[sl].psnr_samples_total[0], peak, + (double)psnr_stream[sl].psnr_sse_total[0]); + fprintf(stderr, " %.3f", ovpsnr); - fprintf(stderr, "\nPSNR (Overall/Avg/Y/U/V)"); - ovpsnr = sse_to_psnr((double)psnr_stream->psnr_samples_total[0], peak, - (double)psnr_stream->psnr_sse_total[0]); - fprintf(stderr, " %.3f", ovpsnr); - - for (int i = 0; i < 4; i++) { - fprintf(stderr, " %.3f", - psnr_stream->psnr_totals[0][i] / psnr_stream->psnr_count[0]); + for (int i = 0; i < 4; i++) { + fprintf( + stderr, " %.3f", + psnr_stream[sl].psnr_totals[0][i] / psnr_stream[sl].psnr_count[0]); + } } fprintf(stderr, "\n"); } @@ -2095,7 +2136,7 @@ } frame_avail = 1; - struct psnr_stats psnr_stream; + struct PsnrStats psnr_stream[MAX_NUM_SPATIAL_LAYERS]; memset(&psnr_stream, 0, sizeof(psnr_stream)); while (frame_avail || got_data) { struct aom_usec_timer timer; @@ -2133,7 +2174,7 @@ &ref_frame_comp_pred); } if (app_input.multilayer_metadata_file != NULL) { - add_multilayer_metadata(&raw, multilayer_metadata); + add_multilayer_metadata(&raw, multilayer_metadata, frame_cnt, slx); } // Set the speed per layer. if (test_speed_per_layer) { @@ -2376,12 +2417,17 @@ break; case AOM_CODEC_PSNR_PKT: if (app_input.show_psnr) { - psnr_stream.psnr_sse_total[0] += pkt->data.psnr.sse[0]; - psnr_stream.psnr_samples_total[0] += pkt->data.psnr.samples[0]; - for (int plane = 0; plane < 4; plane++) { - psnr_stream.psnr_totals[0][plane] += pkt->data.psnr.psnr[plane]; + const int sl = layer_id.spatial_layer_id; + const int show_psnr_hbd = + (cfg.g_input_bit_depth > 8 || cfg.g_bit_depth > AOM_BITS_8); + const int hbd = show_psnr_hbd; + psnr_stream[sl].psnr_sse_total[hbd] += pkt->data.psnr.sse[0]; + psnr_stream[sl].psnr_samples_total[hbd] += + pkt->data.psnr.samples[0]; + for (i = 0; i < 4; i++) { + psnr_stream[sl].psnr_totals[hbd][i] += pkt->data.psnr.psnr[i]; } - psnr_stream.psnr_count[0]++; + psnr_stream[sl].psnr_count[hbd]++; } break; default: break; @@ -2432,7 +2478,10 @@ 1000000 * (double)frame_cnt / (double)cx_time); if (app_input.show_psnr) { - show_psnr(&psnr_stream, 255.0); + const int show_psnr_hbd = + (cfg.g_input_bit_depth > 8 || cfg.g_bit_depth > AOM_BITS_8); + show_psnr(psnr_stream, (double)((1 << (show_psnr_hbd ? 12 : 8)) - 1), + ss_number_layers); } if (aom_codec_destroy(&codec)) die_codec(&codec, "Failed to destroy encoder");
diff --git a/examples/twopass_encoder.c b/examples/twopass_encoder.c index 4d7c653..285bef2 100644 --- a/examples/twopass_encoder.c +++ b/examples/twopass_encoder.c
@@ -187,7 +187,6 @@ int main(int argc, char **argv) { FILE *infile = NULL; int w, h; - aom_codec_ctx_t codec; aom_codec_enc_cfg_t cfg; aom_image_t raw; aom_codec_err_t res; @@ -225,7 +224,8 @@ // Configuration res = aom_codec_enc_config_default(encoder, &cfg, 0); - if (res) die_codec(&codec, "Failed to get default codec config."); + if (res) + die("Failed to get default codec config: %s", aom_codec_err_to_string(res)); cfg.g_w = w; cfg.g_h = h;
diff --git a/test/allintra_end_to_end_test.cc b/test/allintra_end_to_end_test.cc index ba0f0a9..a33aaa2 100644 --- a/test/allintra_end_to_end_test.cc +++ b/test/allintra_end_to_end_test.cc
@@ -24,13 +24,13 @@ const unsigned int kFrames = 20; const int kBitrate = 500; -typedef struct { +struct TestVideoParam { const char *filename; unsigned int input_bit_depth; aom_img_fmt fmt; aom_bit_depth_t bit_depth; unsigned int profile; -} TestVideoParam; +}; std::ostream &operator<<(std::ostream &os, const TestVideoParam &test_arg) { return os << "TestVideoParam { filename:" << test_arg.filename
diff --git a/test/altref_test.cc b/test/altref_test.cc index 354a5a8..ee1a347 100644 --- a/test/altref_test.cc +++ b/test/altref_test.cc
@@ -15,14 +15,14 @@ #include "test/i420_video_source.h" #include "test/util.h" namespace { -typedef struct { +struct AltRefTestParams { const unsigned int min_kf_dist; const unsigned int max_kf_dist; const unsigned int min_gf_interval; const unsigned int max_gf_interval; const unsigned int lag_in_frames; libaom_test::TestMode encoding_mode; -} AltRefTestParams; +}; static const AltRefTestParams TestParams[] = { { 0, 10, 4, 8, 10, ::libaom_test::kOnePassGood }, @@ -113,11 +113,11 @@ ::testing::ValuesIn(TestParams), ::testing::Values(AOM_Q, AOM_VBR, AOM_CBR, AOM_CQ)); -typedef struct { +struct gfIntervalParam { const ::libaom_test::TestMode encoding_mode; const unsigned int min_gf_interval; const unsigned int max_gf_interval; -} gfIntervalParam; +}; const gfIntervalParam gfTestParams[] = { // single pass
diff --git a/test/arf_freq_test.cc b/test/arf_freq_test.cc index 1236ce2..d01eaed 100644 --- a/test/arf_freq_test.cc +++ b/test/arf_freq_test.cc
@@ -28,7 +28,7 @@ #define ARF_NOT_SEEN 1000001 #define ARF_SEEN_ONCE 1000000 -typedef struct { +struct TestVideoParam { const char *filename; unsigned int width; unsigned int height; @@ -38,12 +38,12 @@ aom_img_fmt fmt; aom_bit_depth_t bit_depth; unsigned int profile; -} TestVideoParam; +}; -typedef struct { +struct TestEncodeParam { libaom_test::TestMode mode; int cpu_used; -} TestEncodeParam; +}; const TestVideoParam kTestVectors[] = { // artificially increase framerate to trigger default check
diff --git a/test/av1_convolve_scale_test.cc b/test/av1_convolve_scale_test.cc index a8344fe..8be4b87 100644 --- a/test/av1_convolve_scale_test.cc +++ b/test/av1_convolve_scale_test.cc
@@ -175,7 +175,7 @@ } } -typedef tuple<int, int> BlockDimension; +using BlockDimension = tuple<int, int>; struct BaseParams { BaseParams(BlockDimension dimensions) : dims(dimensions) {} @@ -321,19 +321,19 @@ ConvolveParams convolve_params_; }; -typedef tuple<int, int> BlockDimension; +using BlockDimension = tuple<int, int>; -typedef void (*LowbdConvolveFunc)(const uint8_t *src, int src_stride, - uint8_t *dst, int dst_stride, int w, int h, - const InterpFilterParams *filter_params_x, - const InterpFilterParams *filter_params_y, - const int subpel_x_qn, const int x_step_qn, - const int subpel_y_qn, const int y_step_qn, - ConvolveParams *conv_params); +using LowbdConvolveFunc = void (*)(const uint8_t *src, int src_stride, + uint8_t *dst, int dst_stride, int w, int h, + const InterpFilterParams *filter_params_x, + const InterpFilterParams *filter_params_y, + const int subpel_x_qn, const int x_step_qn, + const int subpel_y_qn, const int y_step_qn, + ConvolveParams *conv_params); // Test parameter list: // <tst_fun, dims, avg> -typedef tuple<LowbdConvolveFunc, BlockDimension> LowBDParams; +using LowBDParams = tuple<LowbdConvolveFunc, BlockDimension>; class LowBDConvolveScaleTest : public ConvolveScaleTestBase<uint8_t>, @@ -417,17 +417,17 @@ #endif // HAVE_SSE4_1 #if CONFIG_AV1_HIGHBITDEPTH -typedef void (*HighbdConvolveFunc)(const uint16_t *src, int src_stride, - uint16_t *dst, int dst_stride, int w, int h, - const InterpFilterParams *filter_params_x, - const InterpFilterParams *filter_params_y, - const int subpel_x_qn, const int x_step_qn, - const int subpel_y_qn, const int y_step_qn, - ConvolveParams *conv_params, int bd); +using HighbdConvolveFunc = void (*)(const uint16_t *src, int src_stride, + uint16_t *dst, int dst_stride, int w, int h, + const InterpFilterParams *filter_params_x, + const InterpFilterParams *filter_params_y, + const int subpel_x_qn, const int x_step_qn, + const int subpel_y_qn, const int y_step_qn, + ConvolveParams *conv_params, int bd); // Test parameter list: // <tst_fun, dims, avg, bd> -typedef tuple<HighbdConvolveFunc, BlockDimension, int> HighBDParams; +using HighBDParams = tuple<HighbdConvolveFunc, BlockDimension, int>; class HighBDConvolveScaleTest : public ConvolveScaleTestBase<uint16_t>,
diff --git a/test/av1_convolve_test.cc b/test/av1_convolve_test.cc index b14c6b7..9dee488 100644 --- a/test/av1_convolve_test.cc +++ b/test/av1_convolve_test.cc
@@ -318,11 +318,11 @@ //////////////////////////////////////////////////////// // Single reference convolve-x functions (low bit-depth) //////////////////////////////////////////////////////// -typedef void (*convolve_x_func)(const uint8_t *src, int src_stride, - uint8_t *dst, int dst_stride, int w, int h, - const InterpFilterParams *filter_params_x, - const int subpel_x_qn, - ConvolveParams *conv_params); +using convolve_x_func = void (*)(const uint8_t *src, int src_stride, + uint8_t *dst, int dst_stride, int w, int h, + const InterpFilterParams *filter_params_x, + const int subpel_x_qn, + ConvolveParams *conv_params); class AV1ConvolveXTest : public AV1ConvolveTest<convolve_x_func> { public: @@ -401,7 +401,7 @@ } aom_usec_timer_mark(&timer); const double time2 = static_cast<double>(aom_usec_timer_elapsed(&timer)); - printf("%d %3dx%-3d:%7.2f/%7.2fns (%3.2f)\n", filter, width, height, time1, + printf("%d %3dx%-3d:%7.2f/%7.2fus (%3.2f)\n", filter, width, height, time1, time2, time1 / time2); } }; @@ -509,7 +509,7 @@ aom_usec_timer_mark(&timer); const double time2 = static_cast<double>(aom_usec_timer_elapsed(&timer)); - printf("%d %3dx%-3d:%7.2f/%7.2fns (%3.2f)\n", filter, width, height, time1, + printf("%d %3dx%-3d:%7.2f/%7.2fus (%3.2f)\n", filter, width, height, time1, time2, time1 / time2); } }; @@ -535,10 +535,10 @@ ///////////////////////////////////////////////////////// // Single reference convolve-x functions (high bit-depth) ///////////////////////////////////////////////////////// -typedef void (*highbd_convolve_x_func)( - const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, - int h, const InterpFilterParams *filter_params_x, const int subpel_x_qn, - ConvolveParams *conv_params, int bd); +using highbd_convolve_x_func = + void (*)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, + int w, int h, const InterpFilterParams *filter_params_x, + const int subpel_x_qn, ConvolveParams *conv_params, int bd); class AV1ConvolveXHighbdTest : public AV1ConvolveTest<highbd_convolve_x_func> { public: @@ -618,7 +618,7 @@ } aom_usec_timer_mark(&timer); const double time2 = static_cast<double>(aom_usec_timer_elapsed(&timer)); - printf("%d %3dx%-3d:%7.2f/%7.2fns (%3.2f)\n", filter, width, height, time1, + printf("%d %3dx%-3d:%7.2f/%7.2fus (%3.2f)\n", filter, width, height, time1, time2, time1 / time2); } }; @@ -725,7 +725,7 @@ aom_usec_timer_mark(&timer); const double time2 = static_cast<double>(aom_usec_timer_elapsed(&timer)); - printf("%d %3dx%-3d:%7.2f/%7.2fns (%3.2f)\n", filter, width, height, time1, + printf("%d %3dx%-3d:%7.2f/%7.2fus (%3.2f)\n", filter, width, height, time1, time2, time1 / time2); } }; @@ -754,10 +754,10 @@ //////////////////////////////////////////////////////// // Single reference convolve-y functions (low bit-depth) //////////////////////////////////////////////////////// -typedef void (*convolve_y_func)(const uint8_t *src, int src_stride, - uint8_t *dst, int dst_stride, int w, int h, - const InterpFilterParams *filter_params_y, - const int subpel_y_qn); +using convolve_y_func = void (*)(const uint8_t *src, int src_stride, + uint8_t *dst, int dst_stride, int w, int h, + const InterpFilterParams *filter_params_y, + const int subpel_y_qn); class AV1ConvolveYTest : public AV1ConvolveTest<convolve_y_func> { public: @@ -826,7 +826,7 @@ } aom_usec_timer_mark(&timer); const double time2 = static_cast<double>(aom_usec_timer_elapsed(&timer)); - printf("%d %3dx%-3d:%7.2f/%7.2fns (%3.2f)\n", filter, width, height, time1, + printf("%d %3dx%-3d:%7.2f/%7.2fus (%3.2f)\n", filter, width, height, time1, time2, time1 / time2); } }; @@ -925,7 +925,7 @@ aom_usec_timer_mark(&timer); const double time2 = static_cast<double>(aom_usec_timer_elapsed(&timer)); - printf("%d %3dx%-3d:%7.2f/%7.2fns (%3.2f)\n", filter, width, height, time1, + printf("%d %3dx%-3d:%7.2f/%7.2fus (%3.2f)\n", filter, width, height, time1, time2, time1 / time2); } }; @@ -951,10 +951,10 @@ ///////////////////////////////////////////////////////// // Single reference convolve-y functions (high bit-depth) ///////////////////////////////////////////////////////// -typedef void (*highbd_convolve_y_func)( - const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, - int h, const InterpFilterParams *filter_params_y, const int subpel_y_qn, - int bd); +using highbd_convolve_y_func = + void (*)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, + int w, int h, const InterpFilterParams *filter_params_y, + const int subpel_y_qn, int bd); class AV1ConvolveYHighbdTest : public AV1ConvolveTest<highbd_convolve_y_func> { public: @@ -1023,7 +1023,7 @@ } aom_usec_timer_mark(&timer); const double time2 = static_cast<double>(aom_usec_timer_elapsed(&timer)); - printf("%d %3dx%-3d:%7.2f/%7.2fns (%3.2f)\n", filter, width, height, time1, + printf("%d %3dx%-3d:%7.2f/%7.2fus (%3.2f)\n", filter, width, height, time1, time2, time1 / time2); } }; @@ -1122,7 +1122,7 @@ aom_usec_timer_mark(&timer); const double time2 = static_cast<double>(aom_usec_timer_elapsed(&timer)); - printf("%d %3dx%-3d:%7.2f/%7.2fns (%3.2f)\n", filter, width, height, time1, + printf("%d %3dx%-3d:%7.2f/%7.2fus (%3.2f)\n", filter, width, height, time1, time2, time1 / time2); } }; @@ -1151,9 +1151,9 @@ ////////////////////////////////////////////////////////////// // Single reference convolve-copy functions (low bit-depth) ////////////////////////////////////////////////////////////// -typedef void (*convolve_copy_func)(const uint8_t *src, ptrdiff_t src_stride, - uint8_t *dst, ptrdiff_t dst_stride, int w, - int h); +using convolve_copy_func = void (*)(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, int w, + int h); class AV1ConvolveCopyTest : public AV1ConvolveTest<convolve_copy_func> { public: @@ -1195,9 +1195,9 @@ /////////////////////////////////////////////////////////////// // Single reference convolve-copy functions (high bit-depth) /////////////////////////////////////////////////////////////// -typedef void (*highbd_convolve_copy_func)(const uint16_t *src, - ptrdiff_t src_stride, uint16_t *dst, - ptrdiff_t dst_stride, int w, int h); +using highbd_convolve_copy_func = void (*)(const uint16_t *src, + ptrdiff_t src_stride, uint16_t *dst, + ptrdiff_t dst_stride, int w, int h); class AV1ConvolveCopyHighbdTest : public AV1ConvolveTest<highbd_convolve_copy_func> { @@ -1241,12 +1241,12 @@ ///////////////////////////////////////////////////////// // Single reference convolve-2D functions (low bit-depth) ///////////////////////////////////////////////////////// -typedef void (*convolve_2d_func)(const uint8_t *src, int src_stride, - uint8_t *dst, int dst_stride, int w, int h, - const InterpFilterParams *filter_params_x, - const InterpFilterParams *filter_params_y, - const int subpel_x_qn, const int subpel_y_qn, - ConvolveParams *conv_params); +using convolve_2d_func = void (*)(const uint8_t *src, int src_stride, + uint8_t *dst, int dst_stride, int w, int h, + const InterpFilterParams *filter_params_x, + const InterpFilterParams *filter_params_y, + const int subpel_x_qn, const int subpel_y_qn, + ConvolveParams *conv_params); class AV1Convolve2DTest : public AV1ConvolveTest<convolve_2d_func> { public: @@ -1338,7 +1338,7 @@ } aom_usec_timer_mark(&timer); const double time2 = static_cast<double>(aom_usec_timer_elapsed(&timer)); - printf("%d - %d %3dx%-3d:%7.2f/%7.2fns (%3.2f)\n", h_f, v_f, width, height, + printf("%d - %d %3dx%-3d:%7.2f/%7.2fus (%3.2f)\n", h_f, v_f, width, height, time1, time2, time1 / time2); } }; @@ -1456,7 +1456,7 @@ aom_usec_timer_mark(&timer); const double time2 = static_cast<double>(aom_usec_timer_elapsed(&timer)); - printf("%d - %d %3dx%-3d:%7.2f/%7.2fns (%3.2f)\n", h_f, v_f, width, height, + printf("%d - %d %3dx%-3d:%7.2f/%7.2fus (%3.2f)\n", h_f, v_f, width, height, time1, time2, time1 / time2); } }; @@ -1483,11 +1483,11 @@ // Single reference convolve-2d functions (high bit-depth) ////////////////////////////////////////////////////////// -typedef void (*highbd_convolve_2d_func)( - const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, - int h, const InterpFilterParams *filter_params_x, - const InterpFilterParams *filter_params_y, const int subpel_x_qn, - const int subpel_y_qn, ConvolveParams *conv_params, int bd); +using highbd_convolve_2d_func = + void (*)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, + int w, int h, const InterpFilterParams *filter_params_x, + const InterpFilterParams *filter_params_y, const int subpel_x_qn, + const int subpel_y_qn, ConvolveParams *conv_params, int bd); class AV1Convolve2DHighbdTest : public AV1ConvolveTest<highbd_convolve_2d_func> { @@ -1581,7 +1581,7 @@ } aom_usec_timer_mark(&timer); const double time2 = static_cast<double>(aom_usec_timer_elapsed(&timer)); - printf("%d - %d %3dx%-3d:%7.2f/%7.2fns (%3.2f)\n", h_f, v_f, width, height, + printf("%d - %d %3dx%-3d:%7.2f/%7.2fus (%3.2f)\n", h_f, v_f, width, height, time1, time2, time1 / time2); } }; @@ -1696,7 +1696,7 @@ aom_usec_timer_mark(&timer); const double time2 = static_cast<double>(aom_usec_timer_elapsed(&timer)); - printf("%d - %d %3dx%-3d:%7.2f/%7.2fns (%3.2f)\n", h_f, v_f, width, height, + printf("%d - %d %3dx%-3d:%7.2f/%7.2fus (%3.2f)\n", h_f, v_f, width, height, time1, time2, time1 / time2); } }; @@ -1951,6 +1951,11 @@ BuildLowbdLumaParams(av1_dist_wtd_convolve_x_neon_i8mm)); #endif +#if HAVE_RVV +INSTANTIATE_TEST_SUITE_P(RVV, AV1ConvolveXCompoundTest, + BuildLowbdLumaParams(av1_dist_wtd_convolve_x_rvv)); +#endif + #if CONFIG_AV1_HIGHBITDEPTH ///////////////////////////////////////////////// // Compound convolve-x functions (high bit-depth) @@ -2046,6 +2051,12 @@ BuildHighbdLumaParams(av1_highbd_dist_wtd_convolve_x_neon)); #endif +#if HAVE_RVV +INSTANTIATE_TEST_SUITE_P( + RVV, AV1ConvolveXHighbdCompoundTest, + BuildHighbdLumaParams(av1_highbd_dist_wtd_convolve_x_rvv)); +#endif + #if HAVE_SVE2 INSTANTIATE_TEST_SUITE_P( SVE2, AV1ConvolveXHighbdCompoundTest, @@ -2092,6 +2103,23 @@ BuildLowbdLumaParams(av1_dist_wtd_convolve_y_neon)); #endif +#if HAVE_NEON_DOTPROD +INSTANTIATE_TEST_SUITE_P( + NEON_DOTPROD, AV1ConvolveYCompoundTest, + BuildLowbdLumaParams(av1_dist_wtd_convolve_y_neon_dotprod)); +#endif + +#if HAVE_NEON_I8MM +INSTANTIATE_TEST_SUITE_P( + NEON_I8MM, AV1ConvolveYCompoundTest, + BuildLowbdLumaParams(av1_dist_wtd_convolve_y_neon_i8mm)); +#endif + +#if HAVE_RVV +INSTANTIATE_TEST_SUITE_P(RVV, AV1ConvolveYCompoundTest, + BuildLowbdLumaParams(av1_dist_wtd_convolve_y_rvv)); +#endif + #if CONFIG_AV1_HIGHBITDEPTH ///////////////////////////////////////////////// // Compound convolve-y functions (high bit-depth) @@ -2132,6 +2160,12 @@ BuildHighbdLumaParams(av1_highbd_dist_wtd_convolve_y_neon)); #endif +#if HAVE_RVV +INSTANTIATE_TEST_SUITE_P( + RVV, AV1ConvolveYHighbdCompoundTest, + BuildHighbdLumaParams(av1_highbd_dist_wtd_convolve_y_rvv)); +#endif + #if HAVE_SVE2 INSTANTIATE_TEST_SUITE_P( SVE2, AV1ConvolveYHighbdCompoundTest, @@ -2143,9 +2177,9 @@ ////////////////////////////////////////////////////// // Compound convolve-2d-copy functions (low bit-depth) ////////////////////////////////////////////////////// -typedef void (*compound_conv_2d_copy_func)(const uint8_t *src, int src_stride, - uint8_t *dst, int dst_stride, int w, - int h, ConvolveParams *conv_params); +using compound_conv_2d_copy_func = void (*)(const uint8_t *src, int src_stride, + uint8_t *dst, int dst_stride, int w, + int h, ConvolveParams *conv_params); class AV1Convolve2DCopyCompoundTest : public AV1ConvolveTest<compound_conv_2d_copy_func> { @@ -2218,7 +2252,7 @@ } aom_usec_timer_mark(&timer); const double time2 = static_cast<double>(aom_usec_timer_elapsed(&timer)); - printf("Dist Weighted: %d %3dx%-3d:%7.2f/%7.2fns (%3.2f)\n", + printf("Dist Weighted: %d %3dx%-3d:%7.2f/%7.2fus (%3.2f)\n", compound.UseDistWtdCompAvg(), width, height, time1, time2, time1 / time2); } @@ -2262,15 +2296,19 @@ BuildLowbdLumaParams(av1_dist_wtd_convolve_2d_copy_neon)); #endif +#if HAVE_RVV +INSTANTIATE_TEST_SUITE_P( + RVV, AV1Convolve2DCopyCompoundTest, + BuildLowbdLumaParams(av1_dist_wtd_convolve_2d_copy_rvv)); +#endif + #if CONFIG_AV1_HIGHBITDEPTH /////////////////////////////////////////////////////// // Compound convolve-2d-copy functions (high bit-depth) /////////////////////////////////////////////////////// -typedef void (*highbd_compound_conv_2d_copy_func)(const uint16_t *src, - int src_stride, uint16_t *dst, - int dst_stride, int w, int h, - ConvolveParams *conv_params, - int bd); +using highbd_compound_conv_2d_copy_func = + void (*)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, + int w, int h, ConvolveParams *conv_params, int bd); class AV1Convolve2DCopyHighbdCompoundTest : public AV1ConvolveTest<highbd_compound_conv_2d_copy_func> { @@ -2348,6 +2386,12 @@ BuildHighbdLumaParams(av1_highbd_dist_wtd_convolve_2d_copy_neon)); #endif +#if HAVE_RVV +INSTANTIATE_TEST_SUITE_P( + RVV, AV1Convolve2DCopyHighbdCompoundTest, + BuildHighbdLumaParams(av1_highbd_dist_wtd_convolve_2d_copy_rvv)); +#endif + #endif // CONFIG_AV1_HIGHBITDEPTH ///////////////////////////////////////////////// @@ -2443,6 +2487,11 @@ BuildLowbdLumaParams(av1_dist_wtd_convolve_2d_neon)); #endif +#if HAVE_RVV +INSTANTIATE_TEST_SUITE_P(RVV, AV1Convolve2DCompoundTest, + BuildLowbdLumaParams(av1_dist_wtd_convolve_2d_rvv)); +#endif + #if HAVE_NEON_DOTPROD INSTANTIATE_TEST_SUITE_P( NEON_DOTPROD, AV1Convolve2DCompoundTest, @@ -2560,6 +2609,12 @@ BuildHighbdLumaParams(av1_highbd_dist_wtd_convolve_2d_sve2)); #endif +#if HAVE_RVV +INSTANTIATE_TEST_SUITE_P( + RVV, AV1Convolve2DHighbdCompoundTest, + BuildHighbdLumaParams(av1_highbd_dist_wtd_convolve_2d_rvv)); +#endif + #endif // CONFIG_AV1_HIGHBITDEPTH } // namespace
diff --git a/test/av1_external_partition_test.cc b/test/av1_external_partition_test.cc index 65fa001..1aecb34 100644 --- a/test/av1_external_partition_test.cc +++ b/test/av1_external_partition_test.cc
@@ -31,11 +31,11 @@ constexpr int kFrameNum = 8; constexpr int kVersion = 1; -typedef struct TestData { +struct TestData { int version = kVersion; -} TestData; +}; -typedef struct ToyModel { +struct ToyModel { TestData *data; aom_ext_part_config_t config; aom_ext_part_funcs_t funcs; @@ -44,7 +44,7 @@ int frame_width; int frame_height; BLOCK_SIZE block_size; -} ToyModel; +}; // Note: // if CONFIG_PARTITION_SEARCH_ORDER = 0, we test APIs designed for the baseline
diff --git a/test/av1_fwd_txfm2d_test.cc b/test/av1_fwd_txfm2d_test.cc index d694a32..483cf81 100644 --- a/test/av1_fwd_txfm2d_test.cc +++ b/test/av1_fwd_txfm2d_test.cc
@@ -35,7 +35,7 @@ namespace { // tx_type_, tx_size_, max_error_, max_avg_error_ -typedef std::tuple<TX_TYPE, TX_SIZE, double, double> AV1FwdTxfm2dParam; +using AV1FwdTxfm2dParam = std::tuple<TX_TYPE, TX_SIZE, double, double>; class AV1FwdTxfm2d : public ::testing::TestWithParam<AV1FwdTxfm2dParam> { public: @@ -238,8 +238,8 @@ } } -typedef void (*lowbd_fwd_txfm_func)(const int16_t *src_diff, tran_low_t *coeff, - int diff_stride, TxfmParam *txfm_param); +using lowbd_fwd_txfm_func = void (*)(const int16_t *src_diff, tran_low_t *coeff, + int diff_stride, TxfmParam *txfm_param); void AV1FwdTxfm2dMatchTest(TX_SIZE tx_size, lowbd_fwd_txfm_func target_func) { const int bd = 8; @@ -356,7 +356,7 @@ } } -typedef std::tuple<TX_SIZE, lowbd_fwd_txfm_func> LbdFwdTxfm2dParam; +using LbdFwdTxfm2dParam = std::tuple<TX_SIZE, lowbd_fwd_txfm_func>; class AV1FwdTxfm2dTest : public ::testing::TestWithParam<LbdFwdTxfm2dParam> {}; GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(AV1FwdTxfm2dTest); @@ -521,8 +521,9 @@ #endif // HAVE_NEON -typedef void (*Highbd_fwd_txfm_func)(const int16_t *src_diff, tran_low_t *coeff, - int diff_stride, TxfmParam *txfm_param); +using Highbd_fwd_txfm_func = void (*)(const int16_t *src_diff, + tran_low_t *coeff, int diff_stride, + TxfmParam *txfm_param); void AV1HighbdFwdTxfm2dMatchTest(TX_SIZE tx_size, Highbd_fwd_txfm_func target_func) { @@ -648,7 +649,7 @@ } } -typedef std::tuple<TX_SIZE, Highbd_fwd_txfm_func> HighbdFwdTxfm2dParam; +using HighbdFwdTxfm2dParam = std::tuple<TX_SIZE, Highbd_fwd_txfm_func>; class AV1HighbdFwdTxfm2dTest : public ::testing::TestWithParam<HighbdFwdTxfm2dParam> {};
diff --git a/test/av1_highbd_iht_test.cc b/test/av1_highbd_iht_test.cc index 24cf9b0..5da77fc 100644 --- a/test/av1_highbd_iht_test.cc +++ b/test/av1_highbd_iht_test.cc
@@ -29,11 +29,11 @@ using libaom_test::ACMRandom; using std::tuple; -typedef void (*HbdHtFunc)(const int16_t *input, int32_t *output, int stride, - TX_TYPE tx_type, int bd); - -typedef void (*IHbdHtFunc)(const int32_t *coeff, uint16_t *output, int stride, +using HbdHtFunc = void (*)(const int16_t *input, int32_t *output, int stride, TX_TYPE tx_type, int bd); + +using IHbdHtFunc = void (*)(const int32_t *coeff, uint16_t *output, int stride, + TX_TYPE tx_type, int bd); static const char *tx_type_name[] = { "DCT_DCT", "ADST_DCT", @@ -59,7 +59,7 @@ // num_coeffs, // tx_type, // bit_depth> -typedef tuple<HbdHtFunc, IHbdHtFunc, IHbdHtFunc, int, TX_TYPE, int> IHbdHtParam; +using IHbdHtParam = tuple<HbdHtFunc, IHbdHtFunc, IHbdHtFunc, int, TX_TYPE, int>; class AV1HighbdInvHTNxN : public ::testing::TestWithParam<IHbdHtParam> { public: @@ -193,10 +193,10 @@ ::testing::ValuesIn(kArrayIhtParam)); #endif // HAVE_SSE4_1 -typedef void (*HighbdInvTxfm2dFunc)(const int32_t *input, uint8_t *output, - int stride, const TxfmParam *txfm_param); +using HighbdInvTxfm2dFunc = void (*)(const int32_t *input, uint8_t *output, + int stride, const TxfmParam *txfm_param); -typedef std::tuple<const HighbdInvTxfm2dFunc> AV1HighbdInvTxfm2dParam; +using AV1HighbdInvTxfm2dParam = std::tuple<const HighbdInvTxfm2dFunc>; class AV1HighbdInvTxfm2d : public ::testing::TestWithParam<AV1HighbdInvTxfm2dParam> { public:
diff --git a/test/av1_horz_only_frame_superres_test.cc b/test/av1_horz_only_frame_superres_test.cc index 20b1a97..d32ebd5 100644 --- a/test/av1_horz_only_frame_superres_test.cc +++ b/test/av1_horz_only_frame_superres_test.cc
@@ -257,14 +257,14 @@ TestImage<Pixel> *image_; }; -typedef void (*LowBDConvolveHorizRsFunc)(const uint8_t *src, int src_stride, - uint8_t *dst, int dst_stride, int w, - int h, const int16_t *x_filters, - const int x0_qn, const int x_step_qn); +using LowBDConvolveHorizRsFunc = void (*)(const uint8_t *src, int src_stride, + uint8_t *dst, int dst_stride, int w, + int h, const int16_t *x_filters, + const int x0_qn, const int x_step_qn); // Test parameter list: // <tst_fun_> -typedef tuple<LowBDConvolveHorizRsFunc> LowBDParams; +using LowBDParams = tuple<LowBDConvolveHorizRsFunc>; class LowBDConvolveHorizRSTest : public ConvolveHorizRSTestBase<uint8_t>, @@ -322,15 +322,15 @@ #endif #if CONFIG_AV1_HIGHBITDEPTH -typedef void (*HighBDConvolveHorizRsFunc)(const uint16_t *src, int src_stride, - uint16_t *dst, int dst_stride, int w, - int h, const int16_t *x_filters, - const int x0_qn, const int x_step_qn, - int bd); +using HighBDConvolveHorizRsFunc = void (*)(const uint16_t *src, int src_stride, + uint16_t *dst, int dst_stride, int w, + int h, const int16_t *x_filters, + const int x0_qn, const int x_step_qn, + int bd); // Test parameter list: // <tst_fun_, bd_> -typedef tuple<HighBDConvolveHorizRsFunc, int> HighBDParams; +using HighBDParams = tuple<HighBDConvolveHorizRsFunc, int>; class HighBDConvolveHorizRSTest : public ConvolveHorizRSTestBase<uint16_t>,
diff --git a/test/av1_inv_txfm1d_test.cc b/test/av1_inv_txfm1d_test.cc index 156fb40..30b9a14 100644 --- a/test/av1_inv_txfm1d_test.cc +++ b/test/av1_inv_txfm1d_test.cc
@@ -16,8 +16,6 @@ #include "av1/common/av1_inv_txfm1d.h" #include "av1/encoder/av1_fwd_txfm1d.h" -typedef TX_SIZE TxSize; - using libaom_test::ACMRandom; using libaom_test::input_base; @@ -75,7 +73,7 @@ ASSERT_EQ(NELEMENTS(inv_txfm_func_ls), TX_SIZES); for (int i = 0; i < count_test_block; ++i) { // choose a random transform to test - const TxSize tx_size = static_cast<TxSize>(rnd.Rand8() % TX_SIZES); + const TX_SIZE tx_size = static_cast<TX_SIZE>(rnd.Rand8() % TX_SIZES); const int txfm_size = txfm_size_ls[tx_size]; const TxfmFunc inv_txfm_func = inv_txfm_func_ls[tx_size][0]; @@ -154,4 +152,15 @@ } } +TEST(av1_inv_txfm1d, iadst4_overflow_bug471847850) { + // Test case to trigger integer overflow in av1_iadst4 + int32_t input[4] = { 300000, 0, 300000, 300000 }; + int32_t output[4]; + + av1_iadst4(input, output, 12, range_bit); + + // Verify that the transform completes and the output is greater than zero. + EXPECT_GT(output[0], 0); +} + } // namespace
diff --git a/test/av1_inv_txfm2d_test.cc b/test/av1_inv_txfm2d_test.cc index 1f62daf..c26fe70 100644 --- a/test/av1_inv_txfm2d_test.cc +++ b/test/av1_inv_txfm2d_test.cc
@@ -38,14 +38,11 @@ using std::vector; -typedef TX_TYPE TxType; -typedef TX_SIZE TxSize; - namespace { // AV1InvTxfm2dParam argument list: // tx_type_, tx_size_, max_error_, max_avg_error_ -typedef std::tuple<TxType, TxSize, int, double> AV1InvTxfm2dParam; +using AV1InvTxfm2dParam = std::tuple<TX_TYPE, TX_SIZE, int, double>; class AV1InvTxfm2d : public ::testing::TestWithParam<AV1InvTxfm2dParam> { public: @@ -91,7 +88,7 @@ } double ref_coeffs[64 * 64] = { 0 }; ASSERT_LE(txfm2d_size, NELEMENTS(ref_coeffs)); - ASSERT_EQ(tx_type_, static_cast<TxType>(DCT_DCT)); + ASSERT_EQ(tx_type_, static_cast<TX_TYPE>(DCT_DCT)); libaom_test::reference_hybrid_2d(ref_input, ref_coeffs, tx_type_, tx_size_); DECLARE_ALIGNED(16, int32_t, ref_coeffs_int[64 * 64]) = { 0 }; @@ -146,8 +143,8 @@ int max_error_; double max_avg_error_; - TxType tx_type_; - TxSize tx_size_; + TX_TYPE tx_type_; + TX_SIZE tx_size_; }; static const int max_error_ls[TX_SIZES_ALL] = { @@ -200,8 +197,8 @@ const int max_error = max_error_ls[s]; const double avg_error = avg_error_ls[s]; for (int t = 0; t < TX_TYPES; ++t) { - const TxType tx_type = static_cast<TxType>(t); - const TxSize tx_size = static_cast<TxSize>(s); + const TX_TYPE tx_type = static_cast<TX_TYPE>(t); + const TX_SIZE tx_size = static_cast<TX_SIZE>(s); if (libaom_test::IsTxSizeTypeValid(tx_size, tx_type)) { param_list.push_back( AV1InvTxfm2dParam(tx_type, tx_size, max_error, avg_error)); @@ -223,18 +220,18 @@ int8_t high_range = libaom_test::high_range_arr[bd_idx]; for (int tx_size = 0; tx_size < TX_SIZES_ALL; ++tx_size) { for (int tx_type = 0; tx_type < TX_TYPES; ++tx_type) { - if (libaom_test::IsTxSizeTypeValid(static_cast<TxSize>(tx_size), - static_cast<TxType>(tx_type)) == + if (libaom_test::IsTxSizeTypeValid(static_cast<TX_SIZE>(tx_size), + static_cast<TX_TYPE>(tx_type)) == false) { continue; } TXFM_2D_FLIP_CFG cfg; - av1_get_inv_txfm_cfg(static_cast<TxType>(tx_type), - static_cast<TxSize>(tx_size), &cfg); + av1_get_inv_txfm_cfg(static_cast<TX_TYPE>(tx_type), + static_cast<TX_SIZE>(tx_size), &cfg); int8_t stage_range_col[MAX_TXFM_STAGE_NUM]; int8_t stage_range_row[MAX_TXFM_STAGE_NUM]; av1_gen_inv_stage_range(stage_range_col, stage_range_row, &cfg, - static_cast<TxSize>(tx_size), bd); + static_cast<TX_SIZE>(tx_size), bd); libaom_test::txfm_stage_range_check(stage_range_col, cfg.stage_num_col, cfg.cos_bit_col, low_range, high_range); @@ -246,11 +243,11 @@ } } -typedef std::tuple<const LbdInvTxfm2dFunc> AV1LbdInvTxfm2dParam; +using AV1LbdInvTxfm2dParam = std::tuple<const LbdInvTxfm2dFunc>; class AV1LbdInvTxfm2d : public ::testing::TestWithParam<AV1LbdInvTxfm2dParam> { public: void SetUp() override { target_func_ = GET_PARAM(0); } - void RunAV1InvTxfm2dTest(TxType tx_type, TxSize tx_size, int run_times, + void RunAV1InvTxfm2dTest(TX_TYPE tx_type, TX_SIZE tx_size, int run_times, int gt_int16 = 0); private: @@ -258,7 +255,7 @@ }; GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(AV1LbdInvTxfm2d); -void AV1LbdInvTxfm2d::RunAV1InvTxfm2dTest(TxType tx_type, TxSize tx_size, +void AV1LbdInvTxfm2d::RunAV1InvTxfm2dTest(TX_TYPE tx_type, TX_SIZE tx_size, int run_times, int gt_int16) { FwdTxfm2dFunc fwd_func_ = libaom_test::fwd_txfm_func_ls[tx_size]; InvTxfm2dFunc ref_func_ = libaom_test::inv_txfm_func_ls[tx_size]; @@ -340,21 +337,23 @@ TEST_P(AV1LbdInvTxfm2d, match) { for (int j = 0; j < (int)(TX_SIZES_ALL); ++j) { for (int i = 0; i < (int)TX_TYPES; ++i) { - if (libaom_test::IsTxSizeTypeValid(static_cast<TxSize>(j), - static_cast<TxType>(i))) { - RunAV1InvTxfm2dTest(static_cast<TxType>(i), static_cast<TxSize>(j), 1); + if (libaom_test::IsTxSizeTypeValid(static_cast<TX_SIZE>(j), + static_cast<TX_TYPE>(i))) { + RunAV1InvTxfm2dTest(static_cast<TX_TYPE>(i), static_cast<TX_SIZE>(j), + 1); } } } } TEST_P(AV1LbdInvTxfm2d, gt_int16) { - static const TxType types[] = { DCT_DCT, ADST_DCT, FLIPADST_DCT, IDTX, - V_DCT, H_DCT, H_ADST, H_FLIPADST }; + static const TX_TYPE types[] = { + DCT_DCT, ADST_DCT, FLIPADST_DCT, IDTX, V_DCT, H_DCT, H_ADST, H_FLIPADST + }; for (int j = 0; j < (int)(TX_SIZES_ALL); ++j) { - const TxSize sz = static_cast<TxSize>(j); + const TX_SIZE sz = static_cast<TX_SIZE>(j); for (uint8_t i = 0; i < sizeof(types) / sizeof(types[0]); ++i) { - const TxType tp = types[i]; + const TX_TYPE tp = types[i]; if (libaom_test::IsTxSizeTypeValid(sz, tp)) { RunAV1InvTxfm2dTest(tp, sz, 1, 1); } @@ -365,9 +364,9 @@ TEST_P(AV1LbdInvTxfm2d, DISABLED_Speed) { for (int j = 1; j < (int)(TX_SIZES_ALL); ++j) { for (int i = 0; i < (int)TX_TYPES; ++i) { - if (libaom_test::IsTxSizeTypeValid(static_cast<TxSize>(j), - static_cast<TxType>(i))) { - RunAV1InvTxfm2dTest(static_cast<TxType>(i), static_cast<TxSize>(j), + if (libaom_test::IsTxSizeTypeValid(static_cast<TX_SIZE>(j), + static_cast<TX_TYPE>(i))) { + RunAV1InvTxfm2dTest(static_cast<TX_TYPE>(i), static_cast<TX_SIZE>(j), 10000000); } } @@ -377,7 +376,7 @@ #if HAVE_SSSE3 extern "C" void av1_lowbd_inv_txfm2d_add_ssse3(const int32_t *input, uint8_t *output, int stride, - TxType tx_type, TxSize tx_size, + TX_TYPE tx_type, TX_SIZE tx_size, int eob); INSTANTIATE_TEST_SUITE_P(SSSE3, AV1LbdInvTxfm2d, ::testing::Values(av1_lowbd_inv_txfm2d_add_ssse3)); @@ -386,7 +385,7 @@ #if HAVE_AVX2 extern "C" void av1_lowbd_inv_txfm2d_add_avx2(const int32_t *input, uint8_t *output, int stride, - TxType tx_type, TxSize tx_size, + TX_TYPE tx_type, TX_SIZE tx_size, int eob); INSTANTIATE_TEST_SUITE_P(AVX2, AV1LbdInvTxfm2d,
diff --git a/test/av1_k_means_test.cc b/test/av1_k_means_test.cc index db738472..58dcb01 100644 --- a/test/av1_k_means_test.cc +++ b/test/av1_k_means_test.cc
@@ -28,20 +28,20 @@ #include "test/util.h" namespace AV1Kmeans { -typedef void (*av1_calc_indices_dim1_func)(const int16_t *data, - const int16_t *centroids, - uint8_t *indices, - int64_t *total_dist, int n, int k); -typedef void (*av1_calc_indices_dim2_func)(const int16_t *data, - const int16_t *centroids, - uint8_t *indices, - int64_t *total_dist, int n, int k); +using av1_calc_indices_dim1_func = void (*)(const int16_t *data, + const int16_t *centroids, + uint8_t *indices, + int64_t *total_dist, int n, int k); +using av1_calc_indices_dim2_func = void (*)(const int16_t *data, + const int16_t *centroids, + uint8_t *indices, + int64_t *total_dist, int n, int k); -typedef std::tuple<av1_calc_indices_dim1_func, BLOCK_SIZE> - av1_calc_indices_dim1Param; +using av1_calc_indices_dim1Param = + std::tuple<av1_calc_indices_dim1_func, BLOCK_SIZE>; -typedef std::tuple<av1_calc_indices_dim2_func, BLOCK_SIZE> - av1_calc_indices_dim2Param; +using av1_calc_indices_dim2Param = + std::tuple<av1_calc_indices_dim2_func, BLOCK_SIZE>; class AV1KmeansTest1 : public ::testing::TestWithParam<av1_calc_indices_dim1Param> {
diff --git a/test/av1_key_value_api_test.cc b/test/av1_key_value_api_test.cc index 03cdeee..d3e9dc1 100644 --- a/test/av1_key_value_api_test.cc +++ b/test/av1_key_value_api_test.cc
@@ -21,7 +21,7 @@ #include "gtest/gtest.h" namespace { -typedef std::tuple<const char *, const char *> KeyValParam; +using KeyValParam = std::tuple<const char *, const char *>; class BaseKeyValAPI : public testing::Test { public:
diff --git a/test/av1_nn_predict_test.cc b/test/av1_nn_predict_test.cc index 38981be..b05b83c 100644 --- a/test/av1_nn_predict_test.cc +++ b/test/av1_nn_predict_test.cc
@@ -24,11 +24,11 @@ #include "test/acm_random.h" namespace { -typedef void (*NnPredict_Func)(const float *const input_nodes, - const NN_CONFIG *const nn_config, - int reduce_prec, float *const output); +using NnPredict_Func = void (*)(const float *const input_nodes, + const NN_CONFIG *const nn_config, + int reduce_prec, float *const output); -typedef std::tuple<const NnPredict_Func> NnPredictTestParam; +using NnPredictTestParam = std::tuple<const NnPredict_Func>; const float epsilon = 1e-3f; // Error threshold for functional equivalence
diff --git a/test/av1_quantize_test.cc b/test/av1_quantize_test.cc index 101186b..4f0c178 100644 --- a/test/av1_quantize_test.cc +++ b/test/av1_quantize_test.cc
@@ -22,7 +22,7 @@ namespace { -typedef void (*QuantizeFpFunc)( +using QuantizeFpFunc = void (*)( const tran_low_t *coeff_ptr, intptr_t count, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
diff --git a/test/av1_round_shift_array_test.cc b/test/av1_round_shift_array_test.cc index e9731b1..39e99f8 100644 --- a/test/av1_round_shift_array_test.cc +++ b/test/av1_round_shift_array_test.cc
@@ -25,7 +25,7 @@ namespace AV1CompRoundShift { -typedef void (*comp_round_shift_array_func)(int32_t *arr, int size, int bit); +using comp_round_shift_array_func = void (*)(int32_t *arr, int size, int bit); #if HAVE_SSE4_1 || HAVE_NEON const int kValidBitCheck[] = { @@ -33,8 +33,8 @@ }; #endif // HAVE_SSE4_1 || HAVE_NEON -typedef std::tuple<comp_round_shift_array_func, BLOCK_SIZE, int> - CompRoundShiftParam; +using CompRoundShiftParam = + std::tuple<comp_round_shift_array_func, BLOCK_SIZE, int>; class AV1CompRoundShiftTest : public ::testing::TestWithParam<CompRoundShiftParam> {
diff --git a/test/av1_temporal_denoiser_test.cc b/test/av1_temporal_denoiser_test.cc index 35a682a..0a80381 100644 --- a/test/av1_temporal_denoiser_test.cc +++ b/test/av1_temporal_denoiser_test.cc
@@ -32,12 +32,12 @@ const int kNumPixels = 128 * 128; -typedef int (*Av1DenoiserFilterFunc)(const uint8_t *sig, int sig_stride, - const uint8_t *mc_avg, int mc_avg_stride, - uint8_t *avg, int avg_stride, - int increase_denoising, BLOCK_SIZE bs, - int motion_magnitude); -typedef std::tuple<Av1DenoiserFilterFunc, BLOCK_SIZE> AV1DenoiserTestParam; +using Av1DenoiserFilterFunc = int (*)(const uint8_t *sig, int sig_stride, + const uint8_t *mc_avg, int mc_avg_stride, + uint8_t *avg, int avg_stride, + int increase_denoising, BLOCK_SIZE bs, + int motion_magnitude); +using AV1DenoiserTestParam = std::tuple<Av1DenoiserFilterFunc, BLOCK_SIZE>; class AV1DenoiserTest : public ::testing::Test,
diff --git a/test/av1_txfm_test.h b/test/av1_txfm_test.h index a7f4a2b..e50dace 100644 --- a/test/av1_txfm_test.h +++ b/test/av1_txfm_test.h
@@ -76,12 +76,12 @@ template <typename Type> void fliplrud(Type *dest, int width, int height, int stride); -typedef void (*TxfmFunc)(const int32_t *in, int32_t *out, const int8_t cos_bit, - const int8_t *range_bit); +using TxfmFunc = void (*)(const int32_t *in, int32_t *out, const int8_t cos_bit, + const int8_t *range_bit); -typedef void (*InvTxfm2dFunc)(const int32_t *, uint16_t *, int, TX_TYPE, int); -typedef void (*LbdInvTxfm2dFunc)(const int32_t *, uint8_t *, int, TX_TYPE, - TX_SIZE, int); +using InvTxfm2dFunc = void (*)(const int32_t *, uint16_t *, int, TX_TYPE, int); +using LbdInvTxfm2dFunc = void (*)(const int32_t *, uint8_t *, int, TX_TYPE, + TX_SIZE, int); static const int bd = 10; static const int input_base = (1 << bd);
diff --git a/test/av1_wedge_utils_test.cc b/test/av1_wedge_utils_test.cc index af11494..9719d2b 100644 --- a/test/av1_wedge_utils_test.cc +++ b/test/av1_wedge_utils_test.cc
@@ -156,9 +156,9 @@ // av1_wedge_sse_from_residuals - optimizations ////////////////////////////////////////////////////////////////////////////// -typedef uint64_t (*FSSE)(const int16_t *r1, const int16_t *d, const uint8_t *m, - int N); -typedef libaom_test::FuncParam<FSSE> TestFuncsFSSE; +using FSSE = uint64_t (*)(const int16_t *r1, const int16_t *d, const uint8_t *m, + int N); +using TestFuncsFSSE = libaom_test::FuncParam<FSSE>; class WedgeUtilsSSEOptTest : public FunctionEquivalenceTest<FSSE> { protected: @@ -222,9 +222,9 @@ // av1_wedge_sign_from_residuals ////////////////////////////////////////////////////////////////////////////// -typedef int8_t (*FSign)(const int16_t *ds, const uint8_t *m, int N, - int64_t limit); -typedef libaom_test::FuncParam<FSign> TestFuncsFSign; +using FSign = int8_t (*)(const int16_t *ds, const uint8_t *m, int N, + int64_t limit); +using TestFuncsFSign = libaom_test::FuncParam<FSign>; class WedgeUtilsSignOptTest : public FunctionEquivalenceTest<FSign> { protected: @@ -324,8 +324,8 @@ // av1_wedge_compute_delta_squares ////////////////////////////////////////////////////////////////////////////// -typedef void (*FDS)(int16_t *d, const int16_t *a, const int16_t *b, int N); -typedef libaom_test::FuncParam<FDS> TestFuncsFDS; +using FDS = void (*)(int16_t *d, const int16_t *a, const int16_t *b, int N); +using TestFuncsFDS = libaom_test::FuncParam<FDS>; class WedgeUtilsDeltaSquaresOptTest : public FunctionEquivalenceTest<FDS> { protected:
diff --git a/test/avg_test.cc b/test/avg_test.cc index d1698fc..d2b4dbc 100644 --- a/test/avg_test.cc +++ b/test/avg_test.cc
@@ -117,11 +117,11 @@ ACMRandom rnd_; }; -typedef unsigned int (*AverageFunction)(const uint8_t *s, int pitch); +using AverageFunction = unsigned int (*)(const uint8_t *s, int pitch); // Arguments: width, height, bit_depth, buffer start offset, block size, avg // function. -typedef std::tuple<int, int, int, int, int, AverageFunction> AvgFunc; +using AvgFunc = std::tuple<int, int, int, int, int, AverageFunction>; template <typename Pixel> class AverageTest : public AverageTestBase<Pixel>, @@ -216,13 +216,13 @@ int64_t opt_elapsed_time_ = 0; }; -typedef void (*AverageFunction_8x8_quad)(const uint8_t *s, int pitch, int x_idx, - int y_idx, int *avg); +using AverageFunction_8x8_quad = void (*)(const uint8_t *s, int pitch, + int x_idx, int y_idx, int *avg); // Arguments: width, height, bit_depth, buffer start offset, block size, avg // function. -typedef std::tuple<int, int, int, int, int, AverageFunction_8x8_quad> - AvgFunc_8x8_quad; +using AvgFunc_8x8_quad = + std::tuple<int, int, int, int, int, AverageFunction_8x8_quad>; template <typename Pixel> class AverageTest_8x8_quad @@ -350,12 +350,12 @@ } #endif // CONFIG_AV1_HIGHBITDEPTH -typedef void (*IntProRowFunc)(int16_t *hbuf, uint8_t const *ref, - const int ref_stride, const int width, - const int height, int norm_factor); +using IntProRowFunc = void (*)(int16_t *hbuf, uint8_t const *ref, + const int ref_stride, const int width, + const int height, int norm_factor); // Params: width, height, asm function, c function. -typedef std::tuple<int, int, IntProRowFunc, IntProRowFunc> IntProRowParam; +using IntProRowParam = std::tuple<int, int, IntProRowFunc, IntProRowFunc>; class IntProRowTest : public AverageTestBase<uint8_t>, public ::testing::WithParamInterface<IntProRowParam> { @@ -452,12 +452,12 @@ }; GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(IntProRowTest); -typedef void (*IntProColFunc)(int16_t *vbuf, uint8_t const *ref, - const int ref_stride, const int width, - const int height, int norm_factor); +using IntProColFunc = void (*)(int16_t *vbuf, uint8_t const *ref, + const int ref_stride, const int width, + const int height, int norm_factor); // Params: width, height, asm function, c function. -typedef std::tuple<int, int, IntProColFunc, IntProColFunc> IntProColParam; +using IntProColParam = std::tuple<int, int, IntProColFunc, IntProColFunc>; class IntProColTest : public AverageTestBase<uint8_t>, public ::testing::WithParamInterface<IntProColParam> { @@ -633,10 +633,10 @@ static const int num_random_cmp = 50; }; -typedef int (*VectorVarFunc)(const int16_t *ref, const int16_t *src, - const int bwl); +using VectorVarFunc = int (*)(const int16_t *ref, const int16_t *src, + const int bwl); -typedef std::tuple<int, VectorVarFunc, VectorVarFunc> VecVarFunc; +using VecVarFunc = std::tuple<int, VectorVarFunc, VectorVarFunc>; class VectorVarTest : public VectorVarTestBase, public ::testing::WithParamInterface<VecVarFunc> { @@ -858,8 +858,8 @@ #endif // HAVE_NEON #endif // CONFIG_AV1_HIGHBITDEPTH -typedef int (*SatdFunc)(const tran_low_t *coeffs, int length); -typedef int (*SatdLpFunc)(const int16_t *coeffs, int length); +using SatdFunc = int (*)(const tran_low_t *coeffs, int length); +using SatdLpFunc = int (*)(const int16_t *coeffs, int length); template <typename SatdFuncType> struct SatdTestParam {
diff --git a/test/blend_a64_mask_1d_test.cc b/test/blend_a64_mask_1d_test.cc index feee2d4..d20baa5 100644 --- a/test/blend_a64_mask_1d_test.cc +++ b/test/blend_a64_mask_1d_test.cc
@@ -114,10 +114,10 @@ // 8 bit version ////////////////////////////////////////////////////////////////////////////// -typedef void (*F8B)(uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, - uint32_t src0_stride, const uint8_t *src1, - uint32_t src1_stride, const uint8_t *mask, int w, int h); -typedef libaom_test::FuncParam<F8B> TestFuncs; +using F8B = void (*)(uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, + uint32_t src0_stride, const uint8_t *src1, + uint32_t src1_stride, const uint8_t *mask, int w, int h); +using TestFuncs = libaom_test::FuncParam<F8B>; class BlendA64Mask1DTest8B : public BlendA64Mask1DTest<F8B, uint8_t> { protected: @@ -219,11 +219,11 @@ // High bit-depth version ////////////////////////////////////////////////////////////////////////////// #if CONFIG_AV1_HIGHBITDEPTH -typedef void (*FHBD)(uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, - uint32_t src0_stride, const uint8_t *src1, - uint32_t src1_stride, const uint8_t *mask, int w, int h, - int bd); -typedef libaom_test::FuncParam<FHBD> TestFuncsHBD; +using FHBD = void (*)(uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, + uint32_t src0_stride, const uint8_t *src1, + uint32_t src1_stride, const uint8_t *mask, int w, int h, + int bd); +using TestFuncsHBD = libaom_test::FuncParam<FHBD>; class BlendA64Mask1DTestHBD : public BlendA64Mask1DTest<FHBD, uint16_t> { protected:
diff --git a/test/blend_a64_mask_test.cc b/test/blend_a64_mask_test.cc index 43d0162..c9dad97 100644 --- a/test/blend_a64_mask_test.cc +++ b/test/blend_a64_mask_test.cc
@@ -157,11 +157,11 @@ // 8 bit version ////////////////////////////////////////////////////////////////////////////// -typedef void (*F8B)(uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, - uint32_t src0_stride, const uint8_t *src1, - uint32_t src1_stride, const uint8_t *mask, - uint32_t mask_stride, int w, int h, int subx, int suby); -typedef libaom_test::FuncParam<F8B> TestFuncs; +using F8B = void (*)(uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, + uint32_t src0_stride, const uint8_t *src1, + uint32_t src1_stride, const uint8_t *mask, + uint32_t mask_stride, int w, int h, int subx, int suby); +using TestFuncs = libaom_test::FuncParam<F8B>; class BlendA64MaskTest8B : public BlendA64MaskTest<F8B, uint8_t, uint8_t> { protected: @@ -266,12 +266,13 @@ // 8 bit _d16 version ////////////////////////////////////////////////////////////////////////////// -typedef void (*F8B_D16)(uint8_t *dst, uint32_t dst_stride, const uint16_t *src0, - uint32_t src0_stride, const uint16_t *src1, - uint32_t src1_stride, const uint8_t *mask, - uint32_t mask_stride, int w, int h, int subx, int suby, - ConvolveParams *conv_params); -typedef libaom_test::FuncParam<F8B_D16> TestFuncs_d16; +using F8B_D16 = void (*)(uint8_t *dst, uint32_t dst_stride, + const uint16_t *src0, uint32_t src0_stride, + const uint16_t *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int w, + int h, int subx, int suby, + ConvolveParams *conv_params); +using TestFuncs_d16 = libaom_test::FuncParam<F8B_D16>; class BlendA64MaskTest8B_d16 : public BlendA64MaskTest<F8B_D16, uint16_t, uint8_t> { @@ -387,12 +388,12 @@ // High bit-depth version ////////////////////////////////////////////////////////////////////////////// #if CONFIG_AV1_HIGHBITDEPTH -typedef void (*FHBD)(uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, - uint32_t src0_stride, const uint8_t *src1, - uint32_t src1_stride, const uint8_t *mask, - uint32_t mask_stride, int w, int h, int subx, int suby, - int bd); -typedef libaom_test::FuncParam<FHBD> TestFuncsHBD; +using FHBD = void (*)(uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, + uint32_t src0_stride, const uint8_t *src1, + uint32_t src1_stride, const uint8_t *mask, + uint32_t mask_stride, int w, int h, int subx, int suby, + int bd); +using TestFuncsHBD = libaom_test::FuncParam<FHBD>; class BlendA64MaskTestHBD : public BlendA64MaskTest<FHBD, uint16_t, uint16_t> { protected: @@ -490,13 +491,13 @@ // HBD _d16 version ////////////////////////////////////////////////////////////////////////////// -typedef void (*FHBD_D16)(uint8_t *dst, uint32_t dst_stride, - const CONV_BUF_TYPE *src0, uint32_t src0_stride, - const CONV_BUF_TYPE *src1, uint32_t src1_stride, - const uint8_t *mask, uint32_t mask_stride, int w, - int h, int subx, int suby, ConvolveParams *conv_params, - const int bd); -typedef libaom_test::FuncParam<FHBD_D16> TestFuncsHBD_d16; +using FHBD_D16 = void (*)(uint8_t *dst, uint32_t dst_stride, + const CONV_BUF_TYPE *src0, uint32_t src0_stride, + const CONV_BUF_TYPE *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int w, + int h, int subx, int suby, + ConvolveParams *conv_params, const int bd); +using TestFuncsHBD_d16 = libaom_test::FuncParam<FHBD_D16>; class BlendA64MaskTestHBD_d16 : public BlendA64MaskTest<FHBD_D16, uint16_t, uint16_t> {
diff --git a/test/cdef_test.cc b/test/cdef_test.cc index 958e6e5..6f985f0 100644 --- a/test/cdef_test.cc +++ b/test/cdef_test.cc
@@ -32,9 +32,9 @@ using CdefFilterBlockFunctions = std::array<cdef_filter_block_func, 4>; -typedef std::tuple<CdefFilterBlockFunctions, CdefFilterBlockFunctions, - BLOCK_SIZE, int, int> - cdef_dir_param_t; +using cdef_dir_param_t = + std::tuple<CdefFilterBlockFunctions, CdefFilterBlockFunctions, BLOCK_SIZE, + int, int>; class CDEFBlockTest : public ::testing::TestWithParam<cdef_dir_param_t> { public: @@ -56,13 +56,13 @@ }; GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(CDEFBlockTest); -typedef CDEFBlockTest CDEFBlockHighbdTest; +using CDEFBlockHighbdTest = CDEFBlockTest; GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(CDEFBlockHighbdTest); -typedef CDEFBlockTest CDEFSpeedTest; +using CDEFSpeedTest = CDEFBlockTest; GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(CDEFSpeedTest); -typedef CDEFBlockTest CDEFSpeedHighbdTest; +using CDEFSpeedHighbdTest = CDEFBlockTest; GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(CDEFSpeedHighbdTest); int64_t test_cdef(BLOCK_SIZE bsize, int iterations, @@ -206,10 +206,10 @@ << "SIMD time: " << elapsed_time << " us" << std::endl; } -typedef int (*find_dir_t)(const uint16_t *img, int stride, int32_t *var, - int coeff_shift); +using find_dir_t = int (*)(const uint16_t *img, int stride, int32_t *var, + int coeff_shift); -typedef std::tuple<find_dir_t, find_dir_t> find_dir_param_t; +using find_dir_param_t = std::tuple<find_dir_t, find_dir_t>; class CDEFFindDirTest : public ::testing::TestWithParam<find_dir_param_t> { public: @@ -225,7 +225,7 @@ }; GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(CDEFFindDirTest); -typedef CDEFFindDirTest CDEFFindDirSpeedTest; +using CDEFFindDirSpeedTest = CDEFFindDirTest; GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(CDEFFindDirSpeedTest); void test_finddir(int (*finddir)(const uint16_t *img, int stride, int32_t *var, @@ -293,11 +293,11 @@ << "SIMD time: " << elapsed_time << " us" << std::endl; } -typedef void (*find_dir_dual_t)(const uint16_t *img1, const uint16_t *img2, - int stride, int32_t *var1, int32_t *var2, - int coeff_shift, int *out1, int *out2); +using find_dir_dual_t = void (*)(const uint16_t *img1, const uint16_t *img2, + int stride, int32_t *var1, int32_t *var2, + int coeff_shift, int *out1, int *out2); -typedef std::tuple<find_dir_dual_t, find_dir_dual_t> find_dir_dual_param_t; +using find_dir_dual_param_t = std::tuple<find_dir_dual_t, find_dir_dual_t>; class CDEFFindDirDualTest : public ::testing::TestWithParam<find_dir_dual_param_t> { @@ -314,7 +314,7 @@ }; GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(CDEFFindDirDualTest); -typedef CDEFFindDirDualTest CDEFFindDirDualSpeedTest; +using CDEFFindDirDualSpeedTest = CDEFFindDirDualTest; GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(CDEFFindDirDualSpeedTest); void test_finddir_dual(
diff --git a/test/cfl_test.cc b/test/cfl_test.cc index 3f93305..6002fd8 100644 --- a/test/cfl_test.cc +++ b/test/cfl_test.cc
@@ -172,8 +172,8 @@ } }; -typedef cfl_subtract_average_fn (*sub_avg_fn)(TX_SIZE tx_size); -typedef std::tuple<TX_SIZE, sub_avg_fn> sub_avg_param; +using sub_avg_fn = cfl_subtract_average_fn (*)(TX_SIZE tx_size); +using sub_avg_param = std::tuple<TX_SIZE, sub_avg_fn>; class CFLSubAvgTest : public ::testing::TestWithParam<sub_avg_param>, public CFLTestWithData<uint16_t> { public: @@ -280,10 +280,10 @@ } }; -typedef cfl_subsample_lbd_fn (*get_subsample_lbd_fn)(TX_SIZE tx_size); -typedef std::tuple<TX_SIZE, get_subsample_lbd_fn, get_subsample_lbd_fn, - get_subsample_lbd_fn> - subsample_lbd_param; +using get_subsample_lbd_fn = cfl_subsample_lbd_fn (*)(TX_SIZE tx_size); +using subsample_lbd_param = + std::tuple<TX_SIZE, get_subsample_lbd_fn, get_subsample_lbd_fn, + get_subsample_lbd_fn>; class CFLSubsampleLBDTest : public CFLSubsampleTest<subsample_lbd_param, cfl_subsample_lbd_fn, uint8_t> { @@ -324,10 +324,10 @@ } #if CONFIG_AV1_HIGHBITDEPTH -typedef cfl_subsample_hbd_fn (*get_subsample_hbd_fn)(TX_SIZE tx_size); -typedef std::tuple<TX_SIZE, get_subsample_hbd_fn, get_subsample_hbd_fn, - get_subsample_hbd_fn> - subsample_hbd_param; +using get_subsample_hbd_fn = cfl_subsample_hbd_fn (*)(TX_SIZE tx_size); +using subsample_hbd_param = + std::tuple<TX_SIZE, get_subsample_hbd_fn, get_subsample_hbd_fn, + get_subsample_hbd_fn>; class CFLSubsampleHBDTest : public CFLSubsampleTest<subsample_hbd_param, cfl_subsample_hbd_fn, uint16_t> { @@ -368,8 +368,8 @@ } #endif // CONFIG_AV1_HIGHBITDEPTH -typedef cfl_predict_lbd_fn (*get_predict_fn)(TX_SIZE tx_size); -typedef std::tuple<TX_SIZE, get_predict_fn> predict_param; +using get_predict_fn = cfl_predict_lbd_fn (*)(TX_SIZE tx_size); +using predict_param = std::tuple<TX_SIZE, get_predict_fn>; class CFLPredictTest : public ::testing::TestWithParam<predict_param>, public CFLTestWithAlignedData<uint8_t> { public: @@ -417,8 +417,8 @@ } #if CONFIG_AV1_HIGHBITDEPTH -typedef cfl_predict_hbd_fn (*get_predict_fn_hbd)(TX_SIZE tx_size); -typedef std::tuple<TX_SIZE, get_predict_fn_hbd> predict_param_hbd; +using get_predict_fn_hbd = cfl_predict_hbd_fn (*)(TX_SIZE tx_size); +using predict_param_hbd = std::tuple<TX_SIZE, get_predict_fn_hbd>; class CFLPredictHBDTest : public ::testing::TestWithParam<predict_param_hbd>, public CFLTestWithAlignedData<uint16_t> { public:
diff --git a/test/cnn_test.cc b/test/cnn_test.cc index 3012451..fdf71d0 100644 --- a/test/cnn_test.cc +++ b/test/cnn_test.cc
@@ -2510,13 +2510,13 @@ namespace { -typedef void (*CNNConvolveNoMaxpoolPaddingValidFunc)( - const float **input, int in_width, int in_height, int in_stride, - const CNN_LAYER_CONFIG *layer_config, float **output, int out_stride, - int start_idx, int cstep, int channel_step); +using CNNConvolveNoMaxpoolPaddingValidFunc = + void (*)(const float **input, int in_width, int in_height, int in_stride, + const CNN_LAYER_CONFIG *layer_config, float **output, + int out_stride, int start_idx, int cstep, int channel_step); -typedef libaom_test::FuncParam<CNNConvolveNoMaxpoolPaddingValidFunc> - CNNConvolveTestFuncs; +using CNNConvolveTestFuncs = + libaom_test::FuncParam<CNNConvolveNoMaxpoolPaddingValidFunc>; class CNNConvolveTest : public ::testing::TestWithParam<CNNConvolveTestFuncs> { protected:
diff --git a/test/comp_mask_pred_test.cc b/test/comp_mask_pred_test.cc index 953e481..f57160c 100644 --- a/test/comp_mask_pred_test.cc +++ b/test/comp_mask_pred_test.cc
@@ -30,14 +30,14 @@ #include "test/util.h" namespace { -typedef void (*comp_mask_pred_func)(uint8_t *comp_pred, const uint8_t *pred, - int width, int height, const uint8_t *ref, - int ref_stride, const uint8_t *mask, - int mask_stride, int invert_mask); +using comp_mask_pred_func = void (*)(uint8_t *comp_pred, const uint8_t *pred, + int width, int height, const uint8_t *ref, + int ref_stride, const uint8_t *mask, + int mask_stride, int invert_mask); -typedef void (*comp_avg_pred_func)(uint8_t *comp_pred, const uint8_t *pred, - int width, int height, const uint8_t *ref, - int ref_stride); +using comp_avg_pred_func = void (*)(uint8_t *comp_pred, const uint8_t *pred, + int width, int height, const uint8_t *ref, + int ref_stride); #if HAVE_SSSE3 || HAVE_SSE2 || HAVE_AVX2 || HAVE_NEON const BLOCK_SIZE kCompMaskPredParams[] = { @@ -81,9 +81,11 @@ void AV1CompMaskPredBase::SetUp() { rnd_.Reset(libaom_test::ACMRandom::DeterministicSeed()); av1_init_wedge_masks(); - comp_pred1_ = (uint8_t *)aom_memalign(16, MAX_SB_SQUARE); + comp_pred1_ = + (uint8_t *)aom_memalign(16, ((MAX_SB_SIZE + 16) + 16) * MAX_SB_SIZE); ASSERT_NE(comp_pred1_, nullptr); - comp_pred2_ = (uint8_t *)aom_memalign(16, MAX_SB_SQUARE); + comp_pred2_ = + (uint8_t *)aom_memalign(16, ((MAX_SB_SIZE + 16) + 16) * MAX_SB_SIZE); ASSERT_NE(comp_pred2_, nullptr); pred_ = (uint8_t *)aom_memalign(16, MAX_SB_SQUARE); ASSERT_NE(pred_, nullptr); @@ -111,7 +113,7 @@ aom_free(ref_buffer_); } -typedef std::tuple<comp_mask_pred_func, BLOCK_SIZE> CompMaskPredParam; +using CompMaskPredParam = std::tuple<comp_mask_pred_func, BLOCK_SIZE>; class AV1CompMaskPredTest : public AV1CompMaskPredBase, @@ -207,14 +209,15 @@ }; #endif -typedef void (*upsampled_pred_func)(MACROBLOCKD *xd, const AV1_COMMON *const cm, - int mi_row, int mi_col, const MV *const mv, - uint8_t *comp_pred, int width, int height, - int subpel_x_q3, int subpel_y_q3, - const uint8_t *ref, int ref_stride, - int subpel_search); +using upsampled_pred_func = void (*)(MACROBLOCKD *xd, + const AV1_COMMON *const cm, int mi_row, + int mi_col, const MV *const mv, + uint8_t *comp_pred, int width, int height, + int subpel_x_q3, int subpel_y_q3, + const uint8_t *ref, int ref_stride, + int subpel_search); -typedef std::tuple<upsampled_pred_func, BLOCK_SIZE> UpsampledPredParam; +using UpsampledPredParam = std::tuple<upsampled_pred_func, BLOCK_SIZE>; class AV1UpsampledPredTest : public AV1CompMaskPredBase, @@ -299,7 +302,7 @@ ::testing::ValuesIn(kValidBlockSize))); #endif -typedef std::tuple<comp_avg_pred_func, BLOCK_SIZE> CompAvgPredParam; +using CompAvgPredParam = std::tuple<comp_avg_pred_func, BLOCK_SIZE>; class AV1CompAvgPredTest : public ::testing::TestWithParam<CompAvgPredParam> { public: @@ -454,11 +457,11 @@ rnd_.Reset(libaom_test::ACMRandom::DeterministicSeed()); av1_init_wedge_masks(); - comp_pred1_ = - (uint16_t *)aom_memalign(16, MAX_SB_SQUARE * sizeof(*comp_pred1_)); + comp_pred1_ = (uint16_t *)aom_memalign( + 16, ((MAX_SB_SIZE + 16) + 16) * MAX_SB_SIZE * sizeof(*comp_pred1_)); ASSERT_NE(comp_pred1_, nullptr); - comp_pred2_ = - (uint16_t *)aom_memalign(16, MAX_SB_SQUARE * sizeof(*comp_pred2_)); + comp_pred2_ = (uint16_t *)aom_memalign( + 16, ((MAX_SB_SIZE + 16) + 16) * MAX_SB_SIZE * sizeof(*comp_pred2_)); ASSERT_NE(comp_pred2_, nullptr); pred_ = (uint16_t *)aom_memalign(16, MAX_SB_SQUARE * sizeof(*pred_)); ASSERT_NE(pred_, nullptr); @@ -481,14 +484,14 @@ aom_free(ref_buffer_); } -typedef void (*highbd_comp_mask_pred_func)(uint8_t *comp_pred8, - const uint8_t *pred8, int width, - int height, const uint8_t *ref8, - int ref_stride, const uint8_t *mask, - int mask_stride, int invert_mask); +using highbd_comp_mask_pred_func = void (*)(uint8_t *comp_pred8, + const uint8_t *pred8, int width, + int height, const uint8_t *ref8, + int ref_stride, const uint8_t *mask, + int mask_stride, int invert_mask); -typedef std::tuple<highbd_comp_mask_pred_func, BLOCK_SIZE, int> - HighbdCompMaskPredParam; +using HighbdCompMaskPredParam = + std::tuple<highbd_comp_mask_pred_func, BLOCK_SIZE, int>; class AV1HighbdCompMaskPredTest : public AV1HighbdCompMaskPredTestBase, @@ -607,14 +610,14 @@ ::testing::Range(8, 13, 2))); #endif -typedef void (*highbd_upsampled_pred_func)( - MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col, - const MV *const mv, uint8_t *comp_pred8, int width, int height, - int subpel_x_q3, int subpel_y_q3, const uint8_t *ref8, int ref_stride, - int bd, int subpel_search); +using highbd_upsampled_pred_func = + void (*)(MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, + int mi_col, const MV *const mv, uint8_t *comp_pred8, int width, + int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref8, + int ref_stride, int bd, int subpel_search); -typedef std::tuple<highbd_upsampled_pred_func, BLOCK_SIZE, int> - HighbdUpsampledPredParam; +using HighbdUpsampledPredParam = + std::tuple<highbd_upsampled_pred_func, BLOCK_SIZE, int>; class AV1HighbdUpsampledPredTest : public AV1HighbdCompMaskPredTestBase, @@ -728,13 +731,13 @@ ::testing::Range(8, 13, 2))); #endif -typedef void (*highbd_comp_avg_pred_func)(uint8_t *comp_pred, - const uint8_t *pred, int width, - int height, const uint8_t *ref, - int ref_stride); +using highbd_comp_avg_pred_func = void (*)(uint8_t *comp_pred, + const uint8_t *pred, int width, + int height, const uint8_t *ref, + int ref_stride); -typedef std::tuple<highbd_comp_avg_pred_func, BLOCK_SIZE, int> - HighbdCompAvgPredParam; +using HighbdCompAvgPredParam = + std::tuple<highbd_comp_avg_pred_func, BLOCK_SIZE, int>; class AV1HighbdCompAvgPredTest : public ::testing::TestWithParam<HighbdCompAvgPredParam> {
diff --git a/test/convolve_test.cc b/test/convolve_test.cc index 09e5f64..578d9a2 100644 --- a/test/convolve_test.cc +++ b/test/convolve_test.cc
@@ -40,11 +40,11 @@ static const int kNumFilterBanks = SWITCHABLE_FILTERS; static const int kNumFilters = 16; -typedef void (*ConvolveFunc)(const uint8_t *src, ptrdiff_t src_stride, - uint8_t *dst, ptrdiff_t dst_stride, - const int16_t *filter_x, int filter_x_stride, - const int16_t *filter_y, int filter_y_stride, - int w, int h); +using ConvolveFunc = void (*)(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int filter_x_stride, + const int16_t *filter_y, int filter_y_stride, + int w, int h); struct ConvolveFunctions { ConvolveFunctions(ConvolveFunc h8, ConvolveFunc v8, int bd) @@ -55,7 +55,7 @@ int use_highbd_; // 0 if high bitdepth not used, else the actual bit depth. }; -typedef std::tuple<int, int, const ConvolveFunctions *> ConvolveParam; +using ConvolveParam = std::tuple<int, int, const ConvolveFunctions *>; #define ALL_SIZES_64(convolve_fn) \ make_tuple(4, 4, &convolve_fn), make_tuple(8, 4, &convolve_fn), \ @@ -940,13 +940,13 @@ #endif #endif // HAVE_SVE -typedef void (*ConvolveScale2DFunc)(const uint8_t *src, ptrdiff_t src_stride, - uint8_t *dst, ptrdiff_t dst_stride, - const InterpKernel *filter, int x0_q4, - int x_step_q4, int y0_q4, int y_step_q4, - int w, int h); +using ConvolveScale2DFunc = void (*)(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const InterpKernel *filter, int x0_q4, + int x_step_q4, int y0_q4, int y_step_q4, + int w, int h); -typedef std::tuple<int, int, ConvolveScale2DFunc> ConvolveScale2DParam; +using ConvolveScale2DParam = std::tuple<int, int, ConvolveScale2DFunc>; class ConvolveScale2DTest : public ::testing::TestWithParam<ConvolveScale2DParam> {
diff --git a/test/corner_match_test.cc b/test/corner_match_test.cc index a805329..e1353b8 100644 --- a/test/corner_match_test.cc +++ b/test/corner_match_test.cc
@@ -27,19 +27,19 @@ using libaom_test::ACMRandom; -typedef bool (*ComputeMeanStddevFunc)(const unsigned char *frame, int stride, - int x, int y, double *mean, - double *one_over_stddev); -typedef double (*ComputeCorrFunc)(const unsigned char *frame1, int stride1, - int x1, int y1, double mean1, - double one_over_stddev1, - const unsigned char *frame2, int stride2, - int x2, int y2, double mean2, - double one_over_stddev2); +using ComputeMeanStddevFunc = bool (*)(const unsigned char *frame, int stride, + int x, int y, double *mean, + double *one_over_stddev); +using ComputeCorrFunc = double (*)(const unsigned char *frame1, int stride1, + int x1, int y1, double mean1, + double one_over_stddev1, + const unsigned char *frame2, int stride2, + int x2, int y2, double mean2, + double one_over_stddev2); using std::make_tuple; using std::tuple; -typedef tuple<int, ComputeMeanStddevFunc, ComputeCorrFunc> CornerMatchParam; +using CornerMatchParam = tuple<int, ComputeMeanStddevFunc, ComputeCorrFunc>; class AV1CornerMatchTest : public ::testing::TestWithParam<CornerMatchParam> { public:
diff --git a/test/datarate_test.cc b/test/datarate_test.cc index 3d86128..c777d4a 100644 --- a/test/datarate_test.cc +++ b/test/datarate_test.cc
@@ -52,7 +52,7 @@ void FillFrame() override { // Read frame from input_file and scale up. ASSERT_NE(input_file_, nullptr); - fread(img_input_->img_data, raw_size_, 1, input_file_); + ASSERT_EQ(fread(img_input_->img_data, raw_size_, 1, input_file_), 1); libyuv::I420Scale( img_input_->planes[AOM_PLANE_Y], img_input_->stride[AOM_PLANE_Y], img_input_->planes[AOM_PLANE_U], img_input_->stride[AOM_PLANE_U], @@ -103,37 +103,15 @@ ::libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288, 30, 1, 0, 140); const int bitrate_array[2] = { 400, 800 }; - cfg_.rc_target_bitrate = bitrate_array[GET_PARAM(4)]; - ResetModel(); - ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); - ASSERT_GE(effective_datarate_, cfg_.rc_target_bitrate * 0.7) - << " The datarate for the file is lower than target by too much!"; - // FIXME(jingning): Lower this test threshold after vbr mode can render - // sufficiently accurate bit rate. - ASSERT_LE(effective_datarate_, cfg_.rc_target_bitrate * 1.45) - << " The datarate for the file is greater than target by too much!"; + RunBasicRateTargetingTest(&video, bitrate_array[GET_PARAM(4)], 0.7, 1.45); } virtual void BasicRateTargetingCBRTest() { - cfg_.rc_buf_initial_sz = 500; - cfg_.rc_buf_optimal_sz = 500; - cfg_.rc_buf_sz = 1000; - cfg_.rc_dropframe_thresh = 1; - cfg_.rc_min_quantizer = 0; - cfg_.rc_max_quantizer = 63; - cfg_.rc_end_usage = AOM_CBR; - cfg_.g_lag_in_frames = 0; - + SetUpCBR(); ::libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288, 30, 1, 0, 140); const int bitrate_array[2] = { 150, 550 }; - cfg_.rc_target_bitrate = bitrate_array[GET_PARAM(4)]; - ResetModel(); - ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); - ASSERT_GE(effective_datarate_, cfg_.rc_target_bitrate * 0.85) - << " The datarate for the file is lower than target by too much!"; - ASSERT_LE(effective_datarate_, cfg_.rc_target_bitrate * 1.19) - << " The datarate for the file is greater than target by too much!"; + RunBasicRateTargetingTest(&video, bitrate_array[GET_PARAM(4)], 0.85, 1.19); } #if CONFIG_LIBYUV @@ -143,10 +121,8 @@ // with the flag avif_mode_. This test upsamples a QVGA clip to the target // resolution, using libyuv for the scaling. virtual void BasicRateTargetingCBRAssertAvifModeTest() { - cfg_.rc_min_quantizer = 0; - cfg_.rc_max_quantizer = 63; - cfg_.rc_end_usage = AOM_CBR; - cfg_.g_lag_in_frames = 0; + SetUpCBR(); + cfg_.rc_dropframe_thresh = 0; ResizingVideoSource video(2456, 2054, 320, 240, "pixel_capture_w320h240.yuv", 100); const int bitrate_array[2] = { 1000, 2000 }; @@ -164,14 +140,10 @@ #endif // CONFIG_LIBYUV virtual void BasicRateTargetingCBRSpikeTest() { - cfg_.rc_buf_initial_sz = 500; - cfg_.rc_buf_optimal_sz = 500; - cfg_.rc_buf_sz = 1000; + SetUpCBR(); cfg_.rc_dropframe_thresh = 0; cfg_.rc_min_quantizer = 2; cfg_.rc_max_quantizer = 56; - cfg_.rc_end_usage = AOM_CBR; - cfg_.g_lag_in_frames = 0; cfg_.kf_max_dist = 3000; cfg_.kf_min_dist = 3000; @@ -192,14 +164,10 @@ } virtual void BasicRateTargetingCBRDynamicBitrateTest() { - cfg_.rc_buf_initial_sz = 500; - cfg_.rc_buf_optimal_sz = 500; - cfg_.rc_buf_sz = 1000; + SetUpCBR(); cfg_.rc_dropframe_thresh = 0; cfg_.rc_min_quantizer = 2; cfg_.rc_max_quantizer = 56; - cfg_.rc_end_usage = AOM_CBR; - cfg_.g_lag_in_frames = 0; cfg_.kf_max_dist = 3000; cfg_.kf_min_dist = 3000; @@ -226,112 +194,56 @@ virtual void BasicRateTargetingMultiThreadCBRTest() { ::libaom_test::I420VideoSource video("niklas_640_480_30.yuv", 640, 480, 30, 1, 0, 400); - cfg_.rc_buf_initial_sz = 500; - cfg_.rc_buf_optimal_sz = 500; - cfg_.rc_buf_sz = 1000; - cfg_.rc_dropframe_thresh = 1; - cfg_.rc_min_quantizer = 0; - cfg_.rc_max_quantizer = 63; - cfg_.rc_end_usage = AOM_CBR; - cfg_.g_lag_in_frames = 0; + SetUpCBR(); cfg_.g_threads = 4; const int bitrate_array[2] = { 250, 650 }; - cfg_.rc_target_bitrate = bitrate_array[GET_PARAM(4)]; ResetModel(); tile_columns_ = 2; - ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); - ASSERT_GE(static_cast<double>(cfg_.rc_target_bitrate), - effective_datarate_ * 0.85) - << " The datarate for the file exceeds the target by too much!"; - ASSERT_LE(static_cast<double>(cfg_.rc_target_bitrate), - effective_datarate_ * 1.15) - << " The datarate for the file missed the target!" - << cfg_.rc_target_bitrate << " " << effective_datarate_; + RunBasicRateTargetingTestReversed(&video, bitrate_array[GET_PARAM(4)], 0.85, + 1.15); } virtual void ErrorResilienceOnSceneCuts() { if (GET_PARAM(4) > 0) return; - cfg_.rc_buf_initial_sz = 500; - cfg_.rc_buf_optimal_sz = 500; - cfg_.rc_buf_sz = 1000; + SetUpCBR(); cfg_.rc_dropframe_thresh = 0; cfg_.g_error_resilient = 1; - cfg_.rc_min_quantizer = 0; - cfg_.rc_max_quantizer = 63; - cfg_.rc_end_usage = AOM_CBR; - cfg_.g_lag_in_frames = 0; ::libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288, 30, 1, 0, 300); - cfg_.rc_target_bitrate = 500; - ResetModel(); - ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); - ASSERT_GE(effective_datarate_, cfg_.rc_target_bitrate * 0.85) - << " The datarate for the file is lower than target by too much!"; - ASSERT_LE(effective_datarate_, cfg_.rc_target_bitrate * 1.15) - << " The datarate for the file is greater than target by too much!"; + RunBasicRateTargetingTest(&video, 500, 0.85, 1.15); } virtual void BasicRateTargetingCBRPeriodicKeyFrameTest() { - cfg_.rc_buf_initial_sz = 500; - cfg_.rc_buf_optimal_sz = 500; - cfg_.rc_buf_sz = 1000; - cfg_.rc_dropframe_thresh = 1; - cfg_.rc_min_quantizer = 0; - cfg_.rc_max_quantizer = 63; - cfg_.rc_end_usage = AOM_CBR; - cfg_.g_lag_in_frames = 0; + SetUpCBR(); // Periodic keyframe cfg_.kf_max_dist = 50; ::libaom_test::I420VideoSource video("pixel_capture_w320h240.yuv", 320, 240, 30, 1, 0, 310); const int bitrate_array[2] = { 150, 550 }; - cfg_.rc_target_bitrate = bitrate_array[GET_PARAM(4)]; - ResetModel(); - ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); - ASSERT_GE(effective_datarate_, cfg_.rc_target_bitrate * 0.85) - << " The datarate for the file is lower than target by too much!"; - ASSERT_LE(effective_datarate_, cfg_.rc_target_bitrate * 1.15) - << " The datarate for the file is greater than target by too much!"; + RunBasicRateTargetingTest(&video, bitrate_array[GET_PARAM(4)], 0.85, 1.15); } virtual void CBRPeriodicKeyFrameOnSceneCuts() { if (GET_PARAM(4) > 0) return; - cfg_.rc_buf_initial_sz = 500; - cfg_.rc_buf_optimal_sz = 500; - cfg_.rc_buf_sz = 1000; + SetUpCBR(); cfg_.rc_dropframe_thresh = 0; - cfg_.rc_min_quantizer = 0; - cfg_.rc_max_quantizer = 63; - cfg_.rc_end_usage = AOM_CBR; - cfg_.g_lag_in_frames = 0; // Periodic keyframe cfg_.kf_max_dist = 30; cfg_.kf_min_dist = 30; ::libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288, 30, 1, 0, 300); - cfg_.rc_target_bitrate = 500; - ResetModel(); - ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); - ASSERT_GE(effective_datarate_, cfg_.rc_target_bitrate * 0.85) - << " The datarate for the file is lower than target by too much!"; - ASSERT_LE(effective_datarate_, cfg_.rc_target_bitrate * 1.3) - << " The datarate for the file is greater than target by too much!"; + RunBasicRateTargetingTest(&video, 500, 0.85, 1.3); } virtual void BasicRateTargetingAQModeOnOffCBRTest() { if (GET_PARAM(4) > 0) return; - cfg_.rc_buf_initial_sz = 500; - cfg_.rc_buf_optimal_sz = 500; - cfg_.rc_buf_sz = 1000; + SetUpCBR(); cfg_.rc_dropframe_thresh = 0; cfg_.rc_min_quantizer = 2; - cfg_.rc_max_quantizer = 63; - cfg_.rc_end_usage = AOM_CBR; - cfg_.g_lag_in_frames = 0; cfg_.g_error_resilient = 0; cfg_.g_pass = AOM_RC_ONE_PASS; cfg_.g_usage = AOM_USAGE_REALTIME; @@ -339,13 +251,7 @@ ::libaom_test::I420VideoSource video("pixel_capture_w320h240.yuv", 320, 240, 30, 1, 0, 310); - cfg_.rc_target_bitrate = 60; - ResetModel(); - ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); - ASSERT_GE(effective_datarate_, cfg_.rc_target_bitrate * 0.85) - << " The datarate for the file is lower than target by too much!"; - ASSERT_LE(effective_datarate_, cfg_.rc_target_bitrate * 1.15) - << " The datarate for the file is greater than target by too much!"; + RunBasicRateTargetingTest(&video, 60, 0.85, 1.15); } virtual void BasicRateTargeting444CBRScreenTest() { @@ -354,26 +260,13 @@ cfg_.g_profile = 1; cfg_.g_timebase = video.timebase(); - cfg_.rc_buf_initial_sz = 500; - cfg_.rc_buf_optimal_sz = 500; - cfg_.rc_buf_sz = 1000; - cfg_.rc_dropframe_thresh = 1; - cfg_.rc_min_quantizer = 0; - cfg_.rc_max_quantizer = 63; - cfg_.rc_end_usage = AOM_CBR; + SetUpCBR(); const int bitrate_array[2] = { 250, 650 }; - cfg_.rc_target_bitrate = bitrate_array[GET_PARAM(4)]; ResetModel(); screen_mode_ = true; - ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); - ASSERT_GE(static_cast<double>(cfg_.rc_target_bitrate), - effective_datarate_ * 0.85) - << " The datarate for the file exceeds the target by too much!"; - ASSERT_LE(static_cast<double>(cfg_.rc_target_bitrate), - effective_datarate_ * 1.15) - << " The datarate for the file missed the target!" - << cfg_.rc_target_bitrate << " " << effective_datarate_; + RunBasicRateTargetingTestReversed(&video, bitrate_array[GET_PARAM(4)], 0.85, + 1.15); } virtual void BasicRateTargetingSuperresCBR() { @@ -383,29 +276,15 @@ cfg_.g_profile = 0; cfg_.g_timebase = video.timebase(); - cfg_.rc_buf_initial_sz = 500; - cfg_.rc_buf_optimal_sz = 500; - cfg_.rc_buf_sz = 1000; - cfg_.rc_dropframe_thresh = 1; - cfg_.rc_min_quantizer = 0; - cfg_.rc_max_quantizer = 63; - cfg_.rc_end_usage = AOM_CBR; + SetUpCBR(); cfg_.rc_superres_mode = AOM_SUPERRES_FIXED; cfg_.rc_superres_denominator = 16; cfg_.rc_superres_kf_denominator = 16; const int bitrate_array[2] = { 250, 650 }; - cfg_.rc_target_bitrate = bitrate_array[GET_PARAM(4)]; - ResetModel(); - ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); - ASSERT_GE(static_cast<double>(cfg_.rc_target_bitrate), - effective_datarate_ * 0.85) - << " The datarate for the file exceeds the target by too much!"; - ASSERT_LE(static_cast<double>(cfg_.rc_target_bitrate), - effective_datarate_ * 1.15) - << " The datarate for the file missed the target!" - << cfg_.rc_target_bitrate << " " << effective_datarate_; + RunBasicRateTargetingTestReversed(&video, bitrate_array[GET_PARAM(4)], 0.85, + 1.15); } virtual void BasicRateTargetingSuperresCBRMultiThreads() { @@ -415,13 +294,7 @@ cfg_.g_profile = 0; cfg_.g_timebase = video.timebase(); - cfg_.rc_buf_initial_sz = 500; - cfg_.rc_buf_optimal_sz = 500; - cfg_.rc_buf_sz = 1000; - cfg_.rc_dropframe_thresh = 1; - cfg_.rc_min_quantizer = 0; - cfg_.rc_max_quantizer = 63; - cfg_.rc_end_usage = AOM_CBR; + SetUpCBR(); cfg_.g_threads = 2; cfg_.rc_superres_mode = AOM_SUPERRES_FIXED; @@ -429,17 +302,81 @@ cfg_.rc_superres_kf_denominator = 16; const int bitrate_array[2] = { 250, 650 }; - cfg_.rc_target_bitrate = bitrate_array[GET_PARAM(4)]; ResetModel(); tile_columns_ = 1; + RunBasicRateTargetingTestReversed(&video, bitrate_array[GET_PARAM(4)], 0.85, + 1.15); + } + + virtual void BasicRateTargetingQvgaCBRKf() { + ::libaom_test::I420VideoSource video("desktop1.320_180.yuv", 320, 180, 30, + 1, 0, 500); + + cfg_.g_profile = 0; + cfg_.g_timebase = video.timebase(); + + SetUpCBR(); + cfg_.kf_max_dist = 10; + cfg_.kf_min_dist = 10; + const int bitrate_array[2] = { 250, 650 }; + ResetModel(); + RunBasicRateTargetingTestReversed(&video, bitrate_array[GET_PARAM(4)], 0.5, + 1.5); + } + + virtual void BasicRateTargetingQvga2CBRKf() { + ::libaom_test::I420VideoSource video("desktopqvga2.320_240.yuv", 320, 240, + 30, 1, 0, 500); + + cfg_.g_profile = 0; + cfg_.g_timebase = video.timebase(); + + SetUpCBR(); + cfg_.kf_max_dist = 10; + cfg_.kf_min_dist = 10; + const int bitrate_array[2] = { 250, 650 }; + ResetModel(); + RunBasicRateTargetingTestReversed(&video, bitrate_array[GET_PARAM(4)], 0.5, + 1.5); + } + + virtual void BasicRateTargetingCifCBRKf() { + ::libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, + 288, 30, 1, 0, 140); + + cfg_.g_profile = 0; + cfg_.g_timebase = video.timebase(); + + SetUpCBR(); + cfg_.kf_max_dist = 10; + cfg_.kf_min_dist = 10; + const int bitrate_array[2] = { 250, 650 }; + ResetModel(); + RunBasicRateTargetingTestReversed(&video, bitrate_array[GET_PARAM(4)], 0.5, + 1.5); + } + + virtual void BasicRateTargetingVBRLagRealtime() { + ::libaom_test::I420VideoSource video("niklas_640_480_30.yuv", 320, 240, 30, + 1, 0, 200); + cfg_.rc_min_quantizer = 0; + cfg_.rc_max_quantizer = 63; + cfg_.g_error_resilient = 0; + cfg_.rc_end_usage = AOM_VBR; + cfg_.g_lag_in_frames = 48; + cfg_.g_pass = AOM_RC_ONE_PASS; + cfg_.g_usage = AOM_USAGE_REALTIME; + cfg_.g_profile = 0; + cfg_.g_timebase = video.timebase(); + cfg_.g_threads = 1; + + ResetModel(); + lag_realtime_mode_ = 1; ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); - ASSERT_GE(static_cast<double>(cfg_.rc_target_bitrate), - effective_datarate_ * 0.85) - << " The datarate for the file exceeds the target by too much!"; - ASSERT_LE(static_cast<double>(cfg_.rc_target_bitrate), - effective_datarate_ * 1.15) - << " The datarate for the file missed the target!" - << cfg_.rc_target_bitrate << " " << effective_datarate_; + ASSERT_GE(effective_datarate_, cfg_.rc_target_bitrate * 0.85) + << " The datarate for the file is lower than target by too much!"; + ASSERT_LE(effective_datarate_, cfg_.rc_target_bitrate * 2.0) + << " The datarate for the file is greater than target by too much!"; } }; @@ -463,17 +400,12 @@ } virtual void ChangingDropFrameThreshTest() { - cfg_.rc_buf_initial_sz = 500; - cfg_.rc_buf_optimal_sz = 500; - cfg_.rc_buf_sz = 1000; + SetUpCBR(); cfg_.rc_undershoot_pct = 20; cfg_.rc_undershoot_pct = 20; cfg_.rc_dropframe_thresh = 10; - cfg_.rc_min_quantizer = 0; cfg_.rc_max_quantizer = 50; - cfg_.rc_end_usage = AOM_CBR; cfg_.rc_target_bitrate = 200; - cfg_.g_lag_in_frames = 0; cfg_.g_error_resilient = 1; // TODO(marpan): Investigate datarate target failures with a smaller // keyframe interval (128). @@ -594,17 +526,12 @@ } virtual void ChangingSpeedTest() { - cfg_.rc_buf_initial_sz = 500; - cfg_.rc_buf_optimal_sz = 500; - cfg_.rc_buf_sz = 1000; + SetUpCBR(); cfg_.rc_undershoot_pct = 20; cfg_.rc_undershoot_pct = 20; cfg_.rc_dropframe_thresh = 10; - cfg_.rc_min_quantizer = 0; cfg_.rc_max_quantizer = 50; - cfg_.rc_end_usage = AOM_CBR; cfg_.rc_target_bitrate = 200; - cfg_.g_lag_in_frames = 0; cfg_.g_error_resilient = 1; // TODO(marpan): Investigate datarate target failures with a smaller // keyframe interval (128). @@ -691,6 +618,25 @@ BasicRateTargetingSuperresCBRMultiThreads(); } +// Check basic rate targeting for QVGA CBR with short keyframe spacing. +TEST_P(DatarateTestRealtime, BasicRateTargetingQvgaCBRKf) { + BasicRateTargetingQvgaCBRKf(); +} + +// Check basic rate targeting for QVGA CBR with short keyframe spacing. +TEST_P(DatarateTestRealtime, BasicRateTargetingQvga2CBRKf) { + BasicRateTargetingQvga2CBRKf(); +} + +// Check basic rate targeting for CIF CBR with short keyframe spacing. +TEST_P(DatarateTestRealtime, BasicRateTargetingCifCBRKf) { + BasicRateTargetingCifCBRKf(); +} + +TEST_P(DatarateTestRealtime, BasicRateTargetingVBRLagRealtime) { + BasicRateTargetingVBRLagRealtime(); +} + // Check that (1) the first dropped frame gets earlier and earlier // as the drop frame threshold is increased, and (2) that the total number of // frame drops does not decrease as we increase frame drop threshold. @@ -743,16 +689,11 @@ }; TEST_P(DatarateTestSetFrameQpRealtime, SetFrameQpOnePass) { - cfg_.rc_buf_initial_sz = 500; - cfg_.rc_buf_optimal_sz = 500; - cfg_.rc_buf_sz = 1000; + SetUpCBR(); cfg_.rc_undershoot_pct = 20; cfg_.rc_undershoot_pct = 20; - cfg_.rc_min_quantizer = 0; cfg_.rc_max_quantizer = 50; - cfg_.rc_end_usage = AOM_CBR; cfg_.rc_target_bitrate = 200; - cfg_.g_lag_in_frames = 0; cfg_.g_error_resilient = 1; cfg_.kf_max_dist = 9999; cfg_.rc_dropframe_thresh = 0; @@ -829,12 +770,12 @@ AV1_INSTANTIATE_TEST_SUITE(DatarateTestRealtime, ::testing::Values(::libaom_test::kRealTime), - ::testing::Range(7, 12), ::testing::Values(0, 3), + ::testing::Range(6, 12), ::testing::Values(0, 3), ::testing::Values(0, 1)); AV1_INSTANTIATE_TEST_SUITE(DatarateTestFrameDropRealtime, ::testing::Values(::libaom_test::kRealTime), - ::testing::Range(7, 12), ::testing::Values(0, 3)); + ::testing::Range(6, 12), ::testing::Values(0, 3)); AV1_INSTANTIATE_TEST_SUITE(DatarateTestSpeedChangeRealtime, ::testing::Values(::libaom_test::kRealTime),
diff --git a/test/datarate_test.h b/test/datarate_test.h index 9064d1d..9b91cb4 100644 --- a/test/datarate_test.h +++ b/test/datarate_test.h
@@ -30,6 +30,17 @@ protected: ~DatarateTest() override = default; + virtual void SetUpCBR() { + cfg_.rc_buf_initial_sz = 500; + cfg_.rc_buf_optimal_sz = 500; + cfg_.rc_buf_sz = 1000; + cfg_.rc_dropframe_thresh = 1; + cfg_.rc_min_quantizer = 0; + cfg_.rc_max_quantizer = 63; + cfg_.rc_end_usage = AOM_CBR; + cfg_.g_lag_in_frames = 0; + } + virtual void ResetModel() { last_pts_ = 0; bits_in_buffer_model_ = cfg_.rc_target_bitrate * cfg_.rc_buf_initial_sz; @@ -58,6 +69,7 @@ effective_datarate_dynamic_[i] = 0.0; } avif_mode_ = 0; + lag_realtime_mode_ = 0; } void PreEncodeFrameHook(::libaom_test::VideoSource *video, @@ -106,6 +118,12 @@ encoder->Control(AOME_SET_CQ_LEVEL, 0); encoder->Control(AV1E_SET_AQ_MODE, (aq_mode_ > 0) ? 1 : 0); } + if (lag_realtime_mode_) { + encoder->Control(AV1E_SET_MIN_GF_INTERVAL, 16); + encoder->Control(AV1E_SET_MAX_GF_INTERVAL, 16); + encoder->Control(AV1E_SET_GF_MAX_PYRAMID_HEIGHT, 1); + encoder->Control(AV1E_SET_GF_MIN_PYRAMID_HEIGHT, 1); + } } if (speed_change_test_) { @@ -211,6 +229,33 @@ } } + void RunBasicRateTargetingTest(::libaom_test::VideoSource *video, + const int bitrate, double low_rate_err_limit, + double high_rate_err_limit) { + cfg_.rc_target_bitrate = bitrate; + ResetModel(); + ASSERT_NO_FATAL_FAILURE(RunLoop(video)); + ASSERT_GE(effective_datarate_, cfg_.rc_target_bitrate * low_rate_err_limit) + << " The datarate for the file is lower than target by too much!"; + ASSERT_LE(effective_datarate_, cfg_.rc_target_bitrate * high_rate_err_limit) + << " The datarate for the file is greater than target by too much!"; + } + + void RunBasicRateTargetingTestReversed(::libaom_test::VideoSource *video, + const int bitrate, + double low_rate_err_limit, + double high_rate_err_limit) { + cfg_.rc_target_bitrate = bitrate; + ASSERT_NO_FATAL_FAILURE(RunLoop(video)); + ASSERT_GE(static_cast<double>(cfg_.rc_target_bitrate), + effective_datarate_ * low_rate_err_limit) + << " The datarate for the file exceeds the target by too much!"; + ASSERT_LE(static_cast<double>(cfg_.rc_target_bitrate), + effective_datarate_ * high_rate_err_limit) + << " The datarate for the file missed the target!" + << cfg_.rc_target_bitrate << " " << effective_datarate_; + } + aom_codec_pts_t last_pts_; double timebase_; int frame_number_; // Counter for number of non-dropped/encoded frames. @@ -244,6 +289,7 @@ int64_t bits_total_dynamic_[3]; int frame_number_dynamic_[3]; int avif_mode_; + int lag_realtime_mode_; }; } // namespace
diff --git a/test/decode_frame_size_limit_test.cc b/test/decode_frame_size_limit_test.cc new file mode 100644 index 0000000..11d4921 --- /dev/null +++ b/test/decode_frame_size_limit_test.cc
@@ -0,0 +1,107 @@ +/* + * Copyright (c) 2026, Alliance for Open Media. All rights reserved. + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include <ostream> + +#include "gtest/gtest.h" +#include "test/codec_factory.h" +#include "test/decode_test_driver.h" +#include "test/ivf_video_source.h" +#include "test/util.h" +#include "test/video_source.h" + +namespace { + +struct DecodeParam { + const char *filename; + unsigned int width; + unsigned int height; +}; + +std::ostream &operator<<(std::ostream &os, const DecodeParam &dp) { + return os << "file: " << dp.filename; +} + +class DecodeFrameSizeLimitTest + : public ::libaom_test::DecoderTest, + public ::libaom_test::CodecTestWithParam<DecodeParam> { + protected: + DecodeFrameSizeLimitTest() + : DecoderTest(GET_PARAM(0)), width_(GET_PARAM(1).width), + height_(GET_PARAM(1).height) {} + + ~DecodeFrameSizeLimitTest() override = default; + + void PreDecodeFrameHook(const libaom_test::CompressedVideoSource &video, + libaom_test::Decoder *decoder) override { + if (video.frame_number() == 0) + decoder->Control(AOMD_SET_FRAME_SIZE_LIMIT, frame_size_limit_); + } + + void DecompressedFrameHook(const aom_image_t &img, + const unsigned int /*frame_number*/) override { + EXPECT_EQ(img.d_w, width_); + EXPECT_EQ(img.d_h, height_); + } + + bool HandleDecodeResult(const aom_codec_err_t res_dec, + const libaom_test::CompressedVideoSource & /*video*/, + libaom_test::Decoder * /*decoder*/) override { + bool expect_failure = + frame_size_limit_ && width_ * height_ > frame_size_limit_; + if (expect_failure) { + EXPECT_EQ(res_dec, AOM_CODEC_CORRUPT_FRAME); + } else { + EXPECT_EQ(res_dec, AOM_CODEC_OK); + } + + return !HasFailure(); + } + + void RunTest() { + const DecodeParam input = GET_PARAM(1); + aom_codec_dec_cfg_t cfg = { 1, 0, 0, !FORCE_HIGHBITDEPTH_DECODING }; + libaom_test::IVFVideoSource decode_video(input.filename); + decode_video.Init(); + + ASSERT_NO_FATAL_FAILURE(RunLoop(&decode_video, cfg)); + } + + protected: + unsigned int frame_size_limit_ = 0; + + private: + unsigned int width_; + unsigned int height_; +}; + +TEST_P(DecodeFrameSizeLimitTest, Unlimited) { RunTest(); } + +TEST_P(DecodeFrameSizeLimitTest, LimitedBig) { + frame_size_limit_ = 226 * 210; + RunTest(); +} + +TEST_P(DecodeFrameSizeLimitTest, LimitedSmall) { + frame_size_limit_ = 226 * 210 - 1; + RunTest(); +} + +const DecodeParam kAV1DecodeFrameSizeLimitTests[] = { + // { filename, width, height } + { "av1-1-b8-01-size-16x16.ivf", 16, 16 }, + { "av1-1-b8-01-size-226x210.ivf", 226, 210 }, +}; + +AV1_INSTANTIATE_TEST_SUITE(DecodeFrameSizeLimitTest, + ::testing::ValuesIn(kAV1DecodeFrameSizeLimitTests)); + +} // namespace
diff --git a/test/decode_perf_test.cc b/test/decode_perf_test.cc index 14b0f9e..c751678 100644 --- a/test/decode_perf_test.cc +++ b/test/decode_perf_test.cc
@@ -37,7 +37,7 @@ /* DecodePerfTest takes a tuple of filename + number of threads to decode with */ -typedef std::tuple<const char *, unsigned> DecodePerfParam; +using DecodePerfParam = std::tuple<const char *, unsigned int>; // TODO(jimbankoski): Add actual test vectors here when available. // const DecodePerfParam kAV1DecodePerfVectors[] = {};
diff --git a/test/dr_prediction_test.cc b/test/dr_prediction_test.cc index de90ec7..4e4a06b 100644 --- a/test/dr_prediction_test.cc +++ b/test/dr_prediction_test.cc
@@ -50,19 +50,19 @@ using libaom_test::ACMRandom; -typedef void (*DrPred_Hbd)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, - const uint16_t *above, const uint16_t *left, - int upsample_above, int upsample_left, int dx, - int dy, int bd); +using DrPred_Hbd = void (*)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, + const uint16_t *above, const uint16_t *left, + int upsample_above, int upsample_left, int dx, + int dy, int bd); -typedef void (*DrPred)(uint8_t *dst, ptrdiff_t stride, int bw, int bh, - const uint8_t *above, const uint8_t *left, - int upsample_above, int upsample_left, int dx, int dy, - int bd); +using DrPred = void (*)(uint8_t *dst, ptrdiff_t stride, int bw, int bh, + const uint8_t *above, const uint8_t *left, + int upsample_above, int upsample_left, int dx, int dy, + int bd); -typedef void (*Z1_Lbd)(uint8_t *dst, ptrdiff_t stride, int bw, int bh, - const uint8_t *above, const uint8_t *left, - int upsample_above, int dx, int dy); +using Z1_Lbd = void (*)(uint8_t *dst, ptrdiff_t stride, int bw, int bh, + const uint8_t *above, const uint8_t *left, + int upsample_above, int dx, int dy); template <Z1_Lbd fn> void z1_wrapper(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, @@ -72,9 +72,9 @@ fn(dst, stride, bw, bh, above, left, upsample_above, dx, dy); } -typedef void (*Z2_Lbd)(uint8_t *dst, ptrdiff_t stride, int bw, int bh, - const uint8_t *above, const uint8_t *left, - int upsample_above, int upsample_left, int dx, int dy); +using Z2_Lbd = void (*)(uint8_t *dst, ptrdiff_t stride, int bw, int bh, + const uint8_t *above, const uint8_t *left, + int upsample_above, int upsample_left, int dx, int dy); template <Z2_Lbd fn> void z2_wrapper(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, @@ -84,9 +84,9 @@ fn(dst, stride, bw, bh, above, left, upsample_above, upsample_left, dx, dy); } -typedef void (*Z3_Lbd)(uint8_t *dst, ptrdiff_t stride, int bw, int bh, - const uint8_t *above, const uint8_t *left, - int upsample_left, int dx, int dy); +using Z3_Lbd = void (*)(uint8_t *dst, ptrdiff_t stride, int bw, int bh, + const uint8_t *above, const uint8_t *left, + int upsample_left, int dx, int dy); template <Z3_Lbd fn> void z3_wrapper(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, @@ -96,9 +96,9 @@ fn(dst, stride, bw, bh, above, left, upsample_left, dx, dy); } -typedef void (*Z1_Hbd)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, - const uint16_t *above, const uint16_t *left, - int upsample_above, int dx, int dy, int bd); +using Z1_Hbd = void (*)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, + const uint16_t *above, const uint16_t *left, + int upsample_above, int dx, int dy, int bd); template <Z1_Hbd fn> void z1_wrapper_hbd(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, @@ -109,10 +109,10 @@ fn(dst, stride, bw, bh, above, left, upsample_above, dx, dy, bd); } -typedef void (*Z2_Hbd)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, - const uint16_t *above, const uint16_t *left, - int upsample_above, int upsample_left, int dx, int dy, - int bd); +using Z2_Hbd = void (*)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, + const uint16_t *above, const uint16_t *left, + int upsample_above, int upsample_left, int dx, int dy, + int bd); template <Z2_Hbd fn> void z2_wrapper_hbd(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, @@ -123,9 +123,9 @@ bd); } -typedef void (*Z3_Hbd)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, - const uint16_t *above, const uint16_t *left, - int upsample_left, int dx, int dy, int bd); +using Z3_Hbd = void (*)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, + const uint16_t *above, const uint16_t *left, + int upsample_left, int dx, int dy, int bd); template <Z3_Hbd fn> void z3_wrapper_hbd(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left,
diff --git a/test/encode_api_test.cc b/test/encode_api_test.cc index bccbf30..af86bb8 100644 --- a/test/encode_api_test.cc +++ b/test/encode_api_test.cc
@@ -25,6 +25,11 @@ #include "aom/aom_encoder.h" #include "aom/aom_image.h" +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/util.h" +#include "test/video_source.h" + namespace { #if CONFIG_REALTIME_ONLY @@ -185,21 +190,6 @@ EXPECT_EQ(AOM_CODEC_OK, aom_codec_destroy(&enc)); } -TEST(EncodeAPI, TuneIqNotAllIntra) { - aom_codec_iface_t *iface = aom_codec_av1_cx(); - aom_codec_enc_cfg_t cfg; - ASSERT_EQ(aom_codec_enc_config_default(iface, &cfg, AOM_USAGE_REALTIME), - AOM_CODEC_OK); - - aom_codec_ctx_t enc; - ASSERT_EQ(aom_codec_enc_init(&enc, iface, &cfg, 0), AOM_CODEC_OK); - - ASSERT_EQ(aom_codec_control(&enc, AOME_SET_TUNING, AOM_TUNE_SSIMULACRA2), - AOM_CODEC_INCAPABLE); - - ASSERT_EQ(aom_codec_destroy(&enc), AOM_CODEC_OK); -} - void EncodeSetSFrameOnFirstFrame(aom_img_fmt fmt, aom_codec_flags_t flag) { constexpr int kWidth = 2; constexpr int kHeight = 128; @@ -1367,6 +1357,521 @@ } #endif // !CONFIG_REALTIME_ONLY +TEST(EncodeAPI, FreezeInternalState) { + aom_codec_iface_t *iface = aom_codec_av1_cx(); + aom_codec_enc_cfg_t cfg; + ASSERT_EQ(aom_codec_enc_config_default(iface, &cfg, kUsage), AOM_CODEC_OK); + cfg.g_w = 176; + cfg.g_h = 144; + cfg.rc_target_bitrate = 200; + cfg.g_lag_in_frames = 0; // Needed for single frame updates + + aom_codec_ctx_t enc; + ASSERT_EQ(aom_codec_enc_init(&enc, iface, &cfg, AOM_CODEC_USE_PSNR), + AOM_CODEC_OK); + + aom_image_t *image = CreateGrayImage(AOM_IMG_FMT_I420, cfg.g_w, cfg.g_h); + ASSERT_NE(image, nullptr); + + // Encode Frame A (Keyframe) + ASSERT_EQ(aom_codec_encode(&enc, image, /*pts=*/0, /*duration=*/1, + /*flags=*/AOM_EFLAG_FORCE_KF), + AOM_CODEC_OK); + + aom_codec_iter_t iter = nullptr; + while (aom_codec_get_cx_data(&enc, &iter) != nullptr) { + // Drain packets + } + + std::vector<uint8_t> bitstream1; + bool psnr1 = false; + + // Encode Frame B with freeze flag + ASSERT_EQ(aom_codec_encode(&enc, image, /*pts=*/1, /*duration=*/1, + /*flags=*/AOM_EFLAG_FREEZE_INTERNAL_STATE | + AOM_EFLAG_CALCULATE_PSNR), + AOM_CODEC_OK); + iter = nullptr; + const aom_codec_cx_pkt_t *pkt; + while ((pkt = aom_codec_get_cx_data(&enc, &iter)) != nullptr) { + if (pkt->kind == AOM_CODEC_CX_FRAME_PKT) { + bitstream1.assign((uint8_t *)pkt->data.frame.buf, + (uint8_t *)pkt->data.frame.buf + pkt->data.frame.sz); + } else if (pkt->kind == AOM_CODEC_PSNR_PKT) { + psnr1 = true; + } + } + EXPECT_TRUE(psnr1); + EXPECT_FALSE(bitstream1.empty()); + + std::vector<uint8_t> bitstream2; + bool psnr2 = false; + + // Encode Frame B again without freeze flag + ASSERT_EQ(aom_codec_encode(&enc, image, /*pts=*/1, /*duration=*/1, + /*flags=*/AOM_EFLAG_CALCULATE_PSNR), + AOM_CODEC_OK); + iter = nullptr; + while ((pkt = aom_codec_get_cx_data(&enc, &iter)) != nullptr) { + if (pkt->kind == AOM_CODEC_CX_FRAME_PKT) { + bitstream2.assign((uint8_t *)pkt->data.frame.buf, + (uint8_t *)pkt->data.frame.buf + pkt->data.frame.sz); + } else if (pkt->kind == AOM_CODEC_PSNR_PKT) { + psnr2 = true; + } + } + EXPECT_TRUE(psnr2); + EXPECT_FALSE(bitstream2.empty()); + + // Bitstreams should be identical + EXPECT_EQ(bitstream1, bitstream2); + + // Encode one more frame for good measure. + ASSERT_EQ(aom_codec_encode(&enc, image, /*pts=*/2, /*duration=*/1, + /*flags=*/0), + AOM_CODEC_OK); + while (aom_codec_get_cx_data(&enc, &iter) != nullptr) { + // Drain packets + } + + aom_img_free(image); + ASSERT_EQ(aom_codec_destroy(&enc), AOM_CODEC_OK); +} + +TEST(EncodeAPI, FreezeInternalStateSVC) { + aom_codec_iface_t *iface = aom_codec_av1_cx(); + aom_codec_enc_cfg_t cfg; + ASSERT_EQ(aom_codec_enc_config_default(iface, &cfg, AOM_USAGE_REALTIME), + AOM_CODEC_OK); + cfg.g_w = 176; + cfg.g_h = 144; + cfg.rc_target_bitrate = 300; + cfg.g_lag_in_frames = 0; + cfg.rc_end_usage = AOM_CBR; + + aom_codec_ctx_t enc; + ASSERT_EQ(aom_codec_enc_init(&enc, iface, &cfg, AOM_CODEC_USE_PSNR), + AOM_CODEC_OK); + + ASSERT_EQ(aom_codec_control(&enc, AOME_SET_CPUUSED, 7), AOM_CODEC_OK); + + aom_svc_params_t svc_params = {}; + svc_params.number_spatial_layers = 2; + svc_params.number_temporal_layers = 1; + svc_params.max_quantizers[0] = 56; + svc_params.min_quantizers[0] = 10; + svc_params.max_quantizers[1] = 56; + svc_params.min_quantizers[1] = 10; + svc_params.scaling_factor_num[0] = 1; + svc_params.scaling_factor_den[0] = 2; + svc_params.scaling_factor_num[1] = 1; + svc_params.scaling_factor_den[1] = 1; + svc_params.layer_target_bitrate[0] = cfg.rc_target_bitrate * 2 / 3; + svc_params.layer_target_bitrate[1] = cfg.rc_target_bitrate; + svc_params.framerate_factor[0] = 1; + ASSERT_EQ(aom_codec_control(&enc, AV1E_SET_SVC_PARAMS, &svc_params), + AOM_CODEC_OK); + + aom_image_t *image = CreateGrayImage(AOM_IMG_FMT_I420, cfg.g_w, cfg.g_h); + ASSERT_NE(image, nullptr); + + aom_svc_layer_id_t layer_id = {}; + aom_svc_ref_frame_config_t ref_frame_config = {}; + + // Encode SL0 - Keyframe + layer_id.spatial_layer_id = 0; + ASSERT_EQ(aom_codec_control(&enc, AV1E_SET_SVC_LAYER_ID, &layer_id), + AOM_CODEC_OK); + ASSERT_EQ(aom_codec_encode(&enc, image, /*pts=*/0, /*duration=*/1, + /*flags=*/AOM_EFLAG_FORCE_KF), + AOM_CODEC_OK); + + aom_codec_iter_t iter = nullptr; + while (aom_codec_get_cx_data(&enc, &iter) != nullptr) { + } // Drain + + // Encode SL1 - Delta Frame with Freeze + layer_id.spatial_layer_id = 1; + ASSERT_EQ(aom_codec_control(&enc, AV1E_SET_SVC_LAYER_ID, &layer_id), + AOM_CODEC_OK); + ref_frame_config.refresh[0] = 1; + ref_frame_config.reference[0] = 1; + ref_frame_config.ref_idx[0] = 0; + ASSERT_EQ( + aom_codec_control(&enc, AV1E_SET_SVC_REF_FRAME_CONFIG, &ref_frame_config), + AOM_CODEC_OK); + + std::vector<uint8_t> bitstream1; + bool psnr1 = false; + ASSERT_EQ(aom_codec_encode(&enc, image, /*pts=*/1, /*duration=*/1, + /*flags=*/AOM_EFLAG_FREEZE_INTERNAL_STATE | + AOM_EFLAG_CALCULATE_PSNR), + AOM_CODEC_OK); + iter = nullptr; + const aom_codec_cx_pkt_t *pkt; + while ((pkt = aom_codec_get_cx_data(&enc, &iter)) != nullptr) { + if (pkt->kind == AOM_CODEC_CX_FRAME_PKT) { + bitstream1.assign((uint8_t *)pkt->data.frame.buf, + (uint8_t *)pkt->data.frame.buf + pkt->data.frame.sz); + } else if (pkt->kind == AOM_CODEC_PSNR_PKT) { + psnr1 = true; + } + } + EXPECT_TRUE(psnr1); + EXPECT_FALSE(bitstream1.empty()); + + // Encode SL1 - Delta Frame again without Freeze + std::vector<uint8_t> bitstream2; + bool psnr2 = false; + ASSERT_EQ(aom_codec_encode(&enc, image, /*pts=*/1, /*duration=*/1, + /*flags=*/AOM_EFLAG_CALCULATE_PSNR), + AOM_CODEC_OK); + iter = nullptr; + while ((pkt = aom_codec_get_cx_data(&enc, &iter)) != nullptr) { + if (pkt->kind == AOM_CODEC_CX_FRAME_PKT) { + bitstream2.assign((uint8_t *)pkt->data.frame.buf, + (uint8_t *)pkt->data.frame.buf + pkt->data.frame.sz); + } else if (pkt->kind == AOM_CODEC_PSNR_PKT) { + psnr2 = true; + } + } + EXPECT_TRUE(psnr2); + EXPECT_FALSE(bitstream2.empty()); + + EXPECT_EQ(bitstream1, bitstream2); + + // Encode one more frame for good measure. + layer_id.spatial_layer_id = 0; + ASSERT_EQ(aom_codec_control(&enc, AV1E_SET_SVC_LAYER_ID, &layer_id), + AOM_CODEC_OK); + ref_frame_config.refresh[0] = 1; + ref_frame_config.reference[0] = 1; + ref_frame_config.ref_idx[0] = 0; + ASSERT_EQ(aom_codec_encode(&enc, image, /*pts=*/2, /*duration=*/1, + /*flags=*/0), + AOM_CODEC_OK); + while (aom_codec_get_cx_data(&enc, &iter) != nullptr) { + // Drain packets + } + + aom_img_free(image); + ASSERT_EQ(aom_codec_destroy(&enc), AOM_CODEC_OK); +} + +TEST(EncodeAPI, FreezeInternalStateTemporalLayers) { + aom_codec_iface_t *iface = aom_codec_av1_cx(); + aom_codec_enc_cfg_t cfg; + ASSERT_EQ(aom_codec_enc_config_default(iface, &cfg, AOM_USAGE_REALTIME), + AOM_CODEC_OK); + cfg.g_w = 176; + cfg.g_h = 144; + cfg.rc_target_bitrate = 300; + cfg.g_lag_in_frames = 0; + cfg.rc_end_usage = AOM_CBR; + + aom_codec_ctx_t enc; + ASSERT_EQ(aom_codec_enc_init(&enc, iface, &cfg, AOM_CODEC_USE_PSNR), + AOM_CODEC_OK); + + ASSERT_EQ(aom_codec_control(&enc, AOME_SET_CPUUSED, 7), AOM_CODEC_OK); + + aom_svc_params_t svc_params = {}; + svc_params.number_spatial_layers = 1; + svc_params.number_temporal_layers = 2; + svc_params.max_quantizers[0] = 56; + svc_params.min_quantizers[0] = 10; + svc_params.max_quantizers[1] = 56; + svc_params.min_quantizers[1] = 10; + svc_params.scaling_factor_num[0] = 1; + svc_params.scaling_factor_den[0] = 1; + svc_params.layer_target_bitrate[0] = cfg.rc_target_bitrate * 2 / 3; + svc_params.layer_target_bitrate[1] = cfg.rc_target_bitrate; + svc_params.framerate_factor[0] = 1; + svc_params.framerate_factor[1] = 2; + ASSERT_EQ(aom_codec_control(&enc, AV1E_SET_SVC_PARAMS, &svc_params), + AOM_CODEC_OK); + + aom_image_t *image = CreateGrayImage(AOM_IMG_FMT_I420, cfg.g_w, cfg.g_h); + ASSERT_NE(image, nullptr); + + aom_svc_layer_id_t layer_id = {}; + aom_svc_ref_frame_config_t ref_frame_config = {}; + + // Encode a Keyframe. + layer_id.spatial_layer_id = 0; + layer_id.temporal_layer_id = 0; + ASSERT_EQ(aom_codec_control(&enc, AV1E_SET_SVC_LAYER_ID, &layer_id), + AOM_CODEC_OK); + ASSERT_EQ(aom_codec_encode(&enc, image, /*pts=*/0, /*duration=*/1, + /*flags=*/AOM_EFLAG_FORCE_KF), + AOM_CODEC_OK); + + aom_codec_iter_t iter = nullptr; + while (aom_codec_get_cx_data(&enc, &iter) != nullptr) { + } // Drain + + // Encode a TL1 - Delta Frame + layer_id.temporal_layer_id = 1; + ASSERT_EQ(aom_codec_control(&enc, AV1E_SET_SVC_LAYER_ID, &layer_id), + AOM_CODEC_OK); + ref_frame_config.refresh[0] = 0; + ref_frame_config.reference[0] = 1; + ref_frame_config.ref_idx[0] = 0; + ASSERT_EQ( + aom_codec_control(&enc, AV1E_SET_SVC_REF_FRAME_CONFIG, &ref_frame_config), + AOM_CODEC_OK); + + ASSERT_EQ(aom_codec_encode(&enc, image, /*pts=*/1, /*duration=*/1, + /*flags=*/0), + AOM_CODEC_OK); + + while (aom_codec_get_cx_data(&enc, &iter) != nullptr) { + } // Drain + + // Encode a TL0 - Delta Frame, with frozen state. + layer_id.temporal_layer_id = 0; + ASSERT_EQ(aom_codec_control(&enc, AV1E_SET_SVC_LAYER_ID, &layer_id), + AOM_CODEC_OK); + ref_frame_config.refresh[0] = 1; + ref_frame_config.reference[0] = 1; + ref_frame_config.ref_idx[0] = 0; + + std::vector<uint8_t> bitstream1; + bool psnr1 = false; + ASSERT_EQ(aom_codec_encode(&enc, image, /*pts=*/2, /*duration=*/1, + /*flags=*/AOM_EFLAG_FREEZE_INTERNAL_STATE | + AOM_EFLAG_CALCULATE_PSNR), + AOM_CODEC_OK); + iter = nullptr; + const aom_codec_cx_pkt_t *pkt; + while ((pkt = aom_codec_get_cx_data(&enc, &iter)) != nullptr) { + if (pkt->kind == AOM_CODEC_CX_FRAME_PKT) { + bitstream1.assign((uint8_t *)pkt->data.frame.buf, + (uint8_t *)pkt->data.frame.buf + pkt->data.frame.sz); + } else if (pkt->kind == AOM_CODEC_PSNR_PKT) { + psnr1 = true; + } + } + EXPECT_TRUE(psnr1); + EXPECT_FALSE(bitstream1.empty()); + + // Encode TL1 - Delta Frame again without frozen state. + std::vector<uint8_t> bitstream2; + bool psnr2 = false; + ASSERT_EQ(aom_codec_encode(&enc, image, /*pts=*/2, /*duration=*/1, + /*flags=*/AOM_EFLAG_CALCULATE_PSNR), + AOM_CODEC_OK); + iter = nullptr; + while ((pkt = aom_codec_get_cx_data(&enc, &iter)) != nullptr) { + if (pkt->kind == AOM_CODEC_CX_FRAME_PKT) { + bitstream2.assign((uint8_t *)pkt->data.frame.buf, + (uint8_t *)pkt->data.frame.buf + pkt->data.frame.sz); + } else if (pkt->kind == AOM_CODEC_PSNR_PKT) { + psnr2 = true; + } + } + EXPECT_TRUE(psnr2); + EXPECT_FALSE(bitstream2.empty()); + + EXPECT_EQ(bitstream1, bitstream2); + + // Encode one more frame for good measure. + layer_id.temporal_layer_id = 1; + ASSERT_EQ(aom_codec_control(&enc, AV1E_SET_SVC_LAYER_ID, &layer_id), + AOM_CODEC_OK); + ref_frame_config.refresh[0] = 0; + ref_frame_config.reference[0] = 1; + ref_frame_config.ref_idx[0] = 0; + ASSERT_EQ(aom_codec_encode(&enc, image, /*pts=*/3, /*duration=*/1, + /*flags=*/0), + AOM_CODEC_OK); + while (aom_codec_get_cx_data(&enc, &iter) != nullptr) { + // Drain packets + } + + aom_img_free(image); + ASSERT_EQ(aom_codec_destroy(&enc), AOM_CODEC_OK); +} + +TEST(EncodeAPI, FreezeInternalStateNotAllowedWithNonZeroLag) { + aom_codec_iface_t *iface = aom_codec_av1_cx(); + aom_codec_enc_cfg_t cfg; + ASSERT_EQ(aom_codec_enc_config_default(iface, &cfg, kUsage), AOM_CODEC_OK); + cfg.g_w = 176; + cfg.g_h = 144; + cfg.rc_target_bitrate = 200; + cfg.g_lag_in_frames = + 1; // Not supported with AOM_EFLAG_FREEZE_INTERNAL_STATE. + + aom_codec_ctx_t enc; + ASSERT_EQ(aom_codec_enc_init(&enc, iface, &cfg, 0), AOM_CODEC_OK); + + aom_image_t *image = CreateGrayImage(AOM_IMG_FMT_I420, cfg.g_w, cfg.g_h); + ASSERT_NE(image, nullptr); + + ASSERT_EQ(aom_codec_encode( + &enc, image, /*pts=*/0, /*duration=*/1, + /*flags=*/AOM_EFLAG_FREEZE_INTERNAL_STATE | AOM_EFLAG_FORCE_KF), + AOM_CODEC_INCAPABLE); + + aom_img_free(image); + ASSERT_EQ(aom_codec_destroy(&enc), AOM_CODEC_OK); +} + +#if !CONFIG_REALTIME_ONLY +// This test is based on the issue:449376308. The segfault in +// av1_update_layer_context_change_config() is triggered +// by doing svc with nonzero lag_in_frames and good_quality usage. +// Note good_quality mode is needed because for realtime mode lag_in_frames +// is forced to 0 in set_encoder_config() . +TEST(EncodeAPI, Issue449376308) { + aom_codec_iface_t *iface = aom_codec_av1_cx(); + aom_codec_enc_cfg_t cfg; + ASSERT_EQ(aom_codec_enc_config_default(iface, &cfg, AOM_USAGE_GOOD_QUALITY), + AOM_CODEC_OK); + cfg.g_w = 320; + cfg.g_h = 240; + cfg.g_timebase.num = 1; + cfg.g_timebase.den = 30; + cfg.rc_target_bitrate = 1000; + cfg.g_lag_in_frames = 25; + aom_codec_ctx_t enc; + ASSERT_EQ(aom_codec_enc_init(&enc, iface, &cfg, 0), AOM_CODEC_OK); + // AV1E_SET_SVC_PARAMS + aom_svc_params_t svc_params = {}; + svc_params.number_spatial_layers = 3; + svc_params.number_temporal_layers = 2; + for (int i = 0; i < AOM_MAX_LAYERS; i++) { + svc_params.max_quantizers[i] = 30; + svc_params.min_quantizers[i] = 0; + svc_params.layer_target_bitrate[i] = 1000; + } + for (int i = 0; i < AOM_MAX_SS_LAYERS; i++) { + svc_params.scaling_factor_num[i] = 1; + svc_params.scaling_factor_den[i] = 1; + } + for (int i = 0; i < AOM_MAX_TS_LAYERS; i++) { + svc_params.framerate_factor[i] = 1; + } + // set_svc_params should fail since lag_in_frames > 0. + EXPECT_NE(aom_codec_control(&enc, AV1E_SET_SVC_PARAMS, &svc_params), + AOM_CODEC_OK); + ASSERT_EQ(aom_codec_destroy(&enc), AOM_CODEC_OK); +} +#endif + +// This test is based on the issue:449341177. The issue occurs in the +// codec_destroy after invalid params (quantizer out of range) are passed +// to the set_svc_params control. The issue can occur for realtime mode +// with lag_in_frames = 0. +TEST(EncodeAPI, Issue449341177) { + aom_codec_iface_t *iface = aom_codec_av1_cx(); + aom_codec_enc_cfg_t cfg; + ASSERT_EQ(aom_codec_enc_config_default(iface, &cfg, AOM_USAGE_REALTIME), + AOM_CODEC_OK); + cfg.g_w = 320; + cfg.g_h = 240; + cfg.rc_target_bitrate = 500; + cfg.g_timebase.num = 1; + cfg.g_timebase.den = 30; + cfg.rc_end_usage = AOM_CBR; + cfg.g_lag_in_frames = 0; + aom_codec_ctx_t enc; + ASSERT_EQ(aom_codec_enc_init(&enc, iface, &cfg, 0), AOM_CODEC_OK); + // AV1E_SET_SVC_PARAMS + aom_svc_params_t svc_params = {}; + svc_params.number_spatial_layers = 3; + svc_params.number_temporal_layers = 2; + for (int i = 0; i < AOM_MAX_LAYERS; i++) { + // Set quantizer out of range. + svc_params.max_quantizers[i] = 80; + svc_params.min_quantizers[i] = 0; + svc_params.layer_target_bitrate[i] = 1000; + } + for (int i = 0; i < AOM_MAX_SS_LAYERS; i++) { + svc_params.scaling_factor_num[i] = 1; + svc_params.scaling_factor_den[i] = 1; + } + for (int i = 0; i < AOM_MAX_TS_LAYERS; i++) { + svc_params.framerate_factor[i] = 1; + } + // set_svc_params should fail because svc_params.max_quantizer[i] is set out + // of range. + EXPECT_NE(aom_codec_control(&enc, AV1E_SET_SVC_PARAMS, &svc_params), + AOM_CODEC_OK); + ASSERT_EQ(aom_codec_destroy(&enc), AOM_CODEC_OK); +} + +// This test is based on the issue:471095598. For profile 2 444 12 bit, +// assert is triggered on first frame in intra_mode_search.c, the function +// av1_count_colors_highbd, for good quality mode. +// Disabled until the assert issue is fixed. +TEST(EncodeAPI, DISABLED_Issue471095598) { + aom_codec_iface_t *encoder_iface = aom_codec_av1_cx(); + aom_codec_enc_cfg_t cfg; + ASSERT_EQ( + aom_codec_enc_config_default(encoder_iface, &cfg, AOM_USAGE_GOOD_QUALITY), + AOM_CODEC_OK); + cfg.g_profile = 2; + cfg.g_bit_depth = AOM_BITS_12; + cfg.g_lag_in_frames = 0; + aom_codec_ctx_t encoder; + ASSERT_EQ(aom_codec_enc_init(&encoder, encoder_iface, &cfg, + AOM_CODEC_USE_HIGHBITDEPTH), + AOM_CODEC_OK); + aom_image_t *img = CreateGrayImage(AOM_IMG_FMT_I44416, cfg.g_w, cfg.g_h); + ASSERT_EQ(aom_codec_encode(&encoder, img, 0, 1, 0), AOM_CODEC_OK); + aom_img_free(img); + ASSERT_EQ(aom_codec_destroy(&encoder), AOM_CODEC_OK); +} + +#if !CONFIG_REALTIME_ONLY +class GetGopInfoTest : public ::libaom_test::EncoderTest, + public ::testing::Test { + protected: + GetGopInfoTest() : EncoderTest(&::libaom_test::kAV1) {} + ~GetGopInfoTest() override = default; + + void SetUp() override { + InitializeConfig(::libaom_test::kTwoPassGood); + cfg_.g_w = 176; + cfg_.g_h = 144; + cfg_.rc_target_bitrate = 200; + cfg_.g_lag_in_frames = 25; + cfg_.g_limit = kFrameLimit; + } + + void PreEncodeFrameHook(::libaom_test::VideoSource *video, + ::libaom_test::Encoder *encoder) override { + if (video->frame() == 0) { + encoder->Control(AOME_SET_CPUUSED, 3); + } + } + + void PostEncodeFrameHook(::libaom_test::Encoder *encoder) override { + if (cfg_.g_pass == AOM_RC_FIRST_PASS) return; + encoder->Control(AV1E_GET_GOP_INFO, &gop_info_); + } + + void FramePktHook(const aom_codec_cx_pkt_t * /*pkt*/) override { + // This is verified here (not in PostEncodeFrameHook) because + // PostEncodeFrameHook is also called when encoder is reading frames into + // lookahead buffer, when GOP structure hasn't been determined. + ASSERT_GT(gop_info_.gop_size, 0); + } + + aom_gop_info_t gop_info_; + static constexpr int kFrameLimit = 10; +}; + +TEST_F(GetGopInfoTest, GetGopInfo) { + ::libaom_test::RandomVideoSource video; + video.SetSize(cfg_.g_w, cfg_.g_h); + video.set_limit(kFrameLimit); + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); +} +#endif // !CONFIG_REALTIME_ONLY + TEST(EncodeAPI, SizeAlignOverflow) { aom_codec_iface_t *iface = aom_codec_av1_cx(); aom_codec_enc_cfg_t cfg;
diff --git a/test/encode_test_driver.cc b/test/encode_test_driver.cc index 6ad10bd..6d42f4b 100644 --- a/test/encode_test_driver.cc +++ b/test/encode_test_driver.cc
@@ -196,7 +196,7 @@ ASSERT_NO_FATAL_FAILURE(video->Begin()); encoder->InitEncoder(video); - if (mode_ == kRealTime) { + if (mode_ == kRealTime && cfg_.g_lag_in_frames == 0) { encoder->Control(AOME_SET_ENABLEAUTOALTREF, 0); }
diff --git a/test/encode_test_driver.h b/test/encode_test_driver.h index a589ae2..a05d645 100644 --- a/test/encode_test_driver.h +++ b/test/encode_test_driver.h
@@ -159,7 +159,16 @@ const aom_codec_err_t res = aom_codec_control(&encoder_, ctrl_id, arg); ASSERT_EQ(AOM_CODEC_OK, res) << EncoderError(); } -#endif + void Control(int ctrl_id, aom_rc_funcs_t *arg) { + const aom_codec_err_t res = aom_codec_control(&encoder_, ctrl_id, arg); + ASSERT_EQ(AOM_CODEC_OK, res) << EncoderError(); + } + + void Control(int ctrl_id, aom_gop_info_t *arg) { + const aom_codec_err_t res = aom_codec_control(&encoder_, ctrl_id, arg); + ASSERT_EQ(AOM_CODEC_OK, res) << EncoderError(); + } +#endif // CONFIG_AV1_ENCODER void SetOption(const char *name, const char *value) { const aom_codec_err_t res = aom_codec_set_option(&encoder_, name, value);
diff --git a/test/encodetxb_test.cc b/test/encodetxb_test.cc index 55047e7..1881191 100644 --- a/test/encodetxb_test.cc +++ b/test/encodetxb_test.cc
@@ -32,11 +32,11 @@ namespace { using libaom_test::ACMRandom; -typedef void (*GetNzMapContextsFunc)(const uint8_t *const levels, - const int16_t *const scan, - const uint16_t eob, const TX_SIZE tx_size, - const TX_CLASS tx_class, - int8_t *const coeff_contexts); +using GetNzMapContextsFunc = void (*)(const uint8_t *const levels, + const int16_t *const scan, + const uint16_t eob, const TX_SIZE tx_size, + const TX_CLASS tx_class, + int8_t *const coeff_contexts); class EncodeTxbTest : public ::testing::TestWithParam<GetNzMapContextsFunc> { public: @@ -202,11 +202,11 @@ ::testing::Values(av1_get_nz_map_contexts_neon)); #endif -typedef void (*av1_txb_init_levels_func)(const tran_low_t *const coeff, - const int width, const int height, - uint8_t *const levels); +using av1_txb_init_levels_func = void (*)(const tran_low_t *const coeff, + const int width, const int height, + uint8_t *const levels); -typedef std::tuple<av1_txb_init_levels_func, int> TxbInitLevelParam; +using TxbInitLevelParam = std::tuple<av1_txb_init_levels_func, int>; class EncodeTxbInitLevelTest : public ::testing::TestWithParam<TxbInitLevelParam> {
diff --git a/test/end_to_end_psnr_test.cc b/test/end_to_end_psnr_test.cc index e0e744b..b006509 100644 --- a/test/end_to_end_psnr_test.cc +++ b/test/end_to_end_psnr_test.cc
@@ -38,13 +38,13 @@ { 34.9, 44.3, 38.5, 40.8 } }; -typedef struct { +struct TestVideoParam { const char *filename; unsigned int input_bit_depth; aom_img_fmt fmt; aom_bit_depth_t bit_depth; unsigned int profile; -} TestVideoParam; +}; std::ostream &operator<<(std::ostream &os, const TestVideoParam &test_arg) { return os << "TestVideoParam { filename:" << test_arg.filename @@ -81,6 +81,12 @@ { 1, 41.8 }, { 2, 41.4 }, { 3, 41.1 }, + } }, + { "SDR_Dance_ldsn_1080p.y4m", + { + { 1, 32.6 }, + { 2, 32.5 }, + { 3, 32.3 }, } } }; // Encoding modes tested @@ -93,6 +99,7 @@ const TestVideoParam kTestVectorsLcMode[] = { { "SDR_Animal_oqo7.y4m", 8, AOM_IMG_FMT_I420, AOM_BITS_8, 0 }, { "SDR_Health_rtd0_720p.y4m", 8, AOM_IMG_FMT_I420, AOM_BITS_8, 0 }, + { "SDR_Dance_ldsn_1080p.y4m", 8, AOM_IMG_FMT_I420, AOM_BITS_8, 0 }, }; // Speed settings tested @@ -138,8 +145,12 @@ if (video->frame() == 0) { if (encoding_mode_ == ::libaom_test::kLowComplexityDecode) { ASSERT_LE(cpu_used_, 3); + if (video->img()->d_h >= 1920) { + encoder->Control(AV1E_SET_TILE_ROWS, 2); + } else { + encoder->Control(AV1E_SET_TILE_ROWS, 1); + } encoder->Control(AV1E_SET_TILE_COLUMNS, 1); - encoder->Control(AV1E_SET_TILE_ROWS, 1); encoder->Control(AV1E_SET_ENABLE_CDEF, 0); encoder->Control(AV1E_SET_ENABLE_LOW_COMPLEXITY_DECODE, 1); } else {
diff --git a/test/end_to_end_qmpsnr_test.cc b/test/end_to_end_qmpsnr_test.cc index c5911ff..64aee79 100644 --- a/test/end_to_end_qmpsnr_test.cc +++ b/test/end_to_end_qmpsnr_test.cc
@@ -28,13 +28,13 @@ const double kSsimThreshold[] = { 83.4, 83.4, 83.4, 83.3, 83.3, 83.0, 82.3, 81.1, 81.1 }; -typedef struct { +struct TestVideoParam { const char *filename; unsigned int input_bit_depth; aom_img_fmt fmt; aom_bit_depth_t bit_depth; unsigned int profile; -} TestVideoParam; +}; std::ostream &operator<<(std::ostream &os, const TestVideoParam &test_arg) { return os << "TestVideoParam { filename:" << test_arg.filename
diff --git a/test/end_to_end_ssim_test.cc b/test/end_to_end_ssim_test.cc index 2b3fb87..730788b 100644 --- a/test/end_to_end_ssim_test.cc +++ b/test/end_to_end_ssim_test.cc
@@ -26,13 +26,13 @@ const double kSsimThreshold[] = { 83.4, 83.4, 83.4, 83.3, 83.3, 83.0, 82.3, 81.1, 81.1 }; -typedef struct { +struct TestVideoParam { const char *filename; unsigned int input_bit_depth; aom_img_fmt fmt; aom_bit_depth_t bit_depth; unsigned int profile; -} TestVideoParam; +}; std::ostream &operator<<(std::ostream &os, const TestVideoParam &test_arg) { return os << "TestVideoParam { filename:" << test_arg.filename
diff --git a/test/examples.sh b/test/examples.sh index 6869d02..ecc9a2b 100755 --- a/test/examples.sh +++ b/test/examples.sh
@@ -19,7 +19,7 @@ exclude_list="best_encode examples run_encodes tools_common" if [ "$(realtime_only_build)" = "yes" ]; then - exclude_list="${exclude_list} twopass_encoder simple_decoder lightfield_test" + exclude_list="${exclude_list} twopass_encoder simple_decoder lightfield_test low_complexity_test" fi # Filter out the scripts in $exclude_list.
diff --git a/test/ext_ratectrl_test.cc b/test/ext_ratectrl_test.cc new file mode 100644 index 0000000..57944d9 --- /dev/null +++ b/test/ext_ratectrl_test.cc
@@ -0,0 +1,408 @@ +/* + * Copyright (c) 2025, Alliance for Open Media. All rights reserved. + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/util.h" +#include "test/y4m_video_source.h" +#include "aom/aom_ext_ratectrl.h" +#include "gtest/gtest.h" + +namespace { + +const int kFrameNum = 5; + +// A mock rate control model. +struct MockRateCtrlModel {}; + +// A mock private data for rate control callbacks. +struct MockRC {}; +struct MockRC g_priv; + +// A flag to indicate if create_model() is called. +bool is_create_model_called = false; + +// A flag to indicate if delete_model() is called. +bool is_delete_model_called = false; + +// A flag to indicate if send_firstpass_stats() is called. +bool is_send_firstpass_stats_called = false; + +// A flag to indicate if send_tpl_gop_stats() is called. +bool is_send_extrc_tpl_gop_stats_called = false; + +// A flag to indicate if get_gop_decision() is called. +bool is_get_gop_decision_called = false; + +// A flag to indicate if update_encodeframe_result() is called. +bool is_update_encodeframe_result_called = false; + +// A flag to indicate if get_key_frame_decision() is called. +bool is_get_key_frame_decision_called = false; + +// Variables to store the parameters passed to update_encodeframe_result(). +int64_t bit_count = 0; +int actual_encoding_qindex = 0; + +// Construct a single ALTREF GOP. +const int kGopFrameCount = kFrameNum + 1; +aom_rc_gop_frame_t gop_frame_list[kGopFrameCount]; + +aom_rc_status_t mock_get_key_frame_decision( + aom_rc_model_t /*ratectrl_model*/, + aom_rc_key_frame_decision_t *key_frame_decision) { + key_frame_decision->key_frame_group_size = 1; + is_get_key_frame_decision_called = true; + return AOM_RC_OK; +} + +aom_rc_status_t mock_get_gop_decision(aom_rc_model_t /*ratectrl_model*/, + aom_rc_gop_decision_t *gop_decision) { + gop_decision->gop_frame_count = kGopFrameCount; + gop_decision->gop_frame_list = gop_frame_list; + static const aom_rc_ref_name_t ref_names[] = { + AOM_RC_LAST_FRAME, AOM_RC_LAST2_FRAME, AOM_RC_LAST3_FRAME, + AOM_RC_GOLDEN_FRAME, AOM_RC_BWDREF_FRAME, AOM_RC_ALTREF2_FRAME, + AOM_RC_ALTREF_FRAME + }; + for (int i = 0; i < kGopFrameCount; ++i) { + auto current_gop_frame = &gop_decision->gop_frame_list[i]; + current_gop_frame->coding_idx = i; + // Set all references to buffer 0 (KF) by default. + for (int j = 0; j < 7; ++j) { + current_gop_frame->ref_frame_list.name[j] = ref_names[j]; + current_gop_frame->ref_frame_list.index[j] = 0; + } + current_gop_frame->update_ref_idx = -1; + current_gop_frame->primary_ref_frame.name = AOM_RC_INVALID_REF_FRAME; + current_gop_frame->primary_ref_frame.index = -1; + + if (i == 0) { + // Key frame + current_gop_frame->is_key_frame = true; + current_gop_frame->update_type = AOM_RC_KF_UPDATE; + current_gop_frame->layer_depth = 0; + current_gop_frame->display_idx = 0; + current_gop_frame->update_ref_idx = 0; + current_gop_frame->order_idx = 0; + } else { + current_gop_frame->is_key_frame = false; + if (i == 1) { + // ALTREF + current_gop_frame->update_type = AOM_RC_ARF_UPDATE; + current_gop_frame->layer_depth = 1; + current_gop_frame->display_idx = 1; + current_gop_frame->update_ref_idx = 1; + current_gop_frame->order_idx = 4; + } else if (i == 5) { + // Overlay frame + current_gop_frame->is_key_frame = false; + current_gop_frame->update_type = AOM_RC_OVERLAY_UPDATE; + current_gop_frame->layer_depth = AOM_RC_MAX_ARF_LAYERS - 1; + current_gop_frame->display_idx = 4; + current_gop_frame->order_idx = 4; + } else { + // Leaf frames + current_gop_frame->is_key_frame = false; + current_gop_frame->update_type = AOM_RC_LF_UPDATE; + current_gop_frame->layer_depth = AOM_RC_MAX_ARF_LAYERS - 1; + current_gop_frame->display_idx = i - 1; // Key in the front + current_gop_frame->order_idx = i - 1; + current_gop_frame->update_ref_idx = 2; + } + // For leaf and overlay frames, set ALTREF to buffer 1. + if (i >= 2) { + current_gop_frame->ref_frame_list.index[AOM_RC_ALTREF_FRAME] = 1; + } + // For leaf frames after the first leaf, use buffer 2 as LAST. + if (i > 2 && i < 5) { + current_gop_frame->ref_frame_list.index[AOM_RC_LAST_FRAME] = 2; + } + + current_gop_frame->primary_ref_frame.name = AOM_RC_LAST_FRAME; + current_gop_frame->primary_ref_frame.index = (i > 2 && i < 5) ? 2 : 0; + } + } + gop_decision->global_order_idx_offset = 0; + is_get_gop_decision_called = true; + return AOM_RC_OK; +} + +aom_rc_status_t mock_create_model(void *priv, + const aom_rc_config_t *ratectrl_config, + aom_rc_model_t *ratectrl_model) { + (void)priv; + (void)ratectrl_config; + EXPECT_NE(ratectrl_model, nullptr); + *ratectrl_model = (aom_rc_model_t)(new MockRateCtrlModel()); + is_create_model_called = true; + return AOM_RC_OK; +} + +aom_rc_status_t mock_delete_model(aom_rc_model_t ratectrl_model) { + EXPECT_NE(ratectrl_model, nullptr); + delete (MockRateCtrlModel *)ratectrl_model; + is_delete_model_called = true; + return AOM_RC_OK; +} + +aom_rc_status_t mock_send_firstpass_stats( + aom_rc_model_t ratectrl_model, + const aom_rc_firstpass_stats_t *firstpass_stats) { + EXPECT_NE(ratectrl_model, nullptr); + EXPECT_NE(firstpass_stats, nullptr); + EXPECT_EQ(firstpass_stats->num_frames, kFrameNum); + EXPECT_NE(firstpass_stats->frame_stats, nullptr); + is_send_firstpass_stats_called = true; + return AOM_RC_OK; +} + +aom_rc_status_t mock_send_extrc_tpl_gop_stats( + aom_rc_model_t ratectrl_model, const AomTplGopStats *extrc_tpl_gop_stats) { + EXPECT_NE(ratectrl_model, nullptr); + EXPECT_NE(extrc_tpl_gop_stats, nullptr); + EXPECT_GT(extrc_tpl_gop_stats->size, 0); + EXPECT_NE(extrc_tpl_gop_stats->frame_stats_list, nullptr); + is_send_extrc_tpl_gop_stats_called = true; + return AOM_RC_OK; +} + +aom_rc_status_t mock_update_encodeframe_result( + aom_rc_model_t /* ratectrl_model */, + const aom_rc_encodeframe_result_t *encode_frame_result) { + EXPECT_NE(encode_frame_result, nullptr); + bit_count = encode_frame_result->bit_count; + actual_encoding_qindex = encode_frame_result->actual_encoding_qindex; + is_update_encodeframe_result_called = true; + return AOM_RC_OK; +} + +class ExtRateCtrlTest : public ::libaom_test::EncoderTest, + public ::libaom_test::CodecTestWith2Params<int, int> { + protected: + ExtRateCtrlTest() : EncoderTest(GET_PARAM(0)), cpu_used_(GET_PARAM(2)) { + aom_rc_funcs_t *rc_funcs = &rc_funcs_; + rc_funcs->priv = &g_priv; + rc_funcs->rc_type = AOM_RC_QP; + rc_funcs->create_model = mock_create_model; + rc_funcs->delete_model = mock_delete_model; + rc_funcs->send_firstpass_stats = mock_send_firstpass_stats; + rc_funcs->send_tpl_gop_stats = mock_send_extrc_tpl_gop_stats; + rc_funcs->get_gop_decision = nullptr; + rc_funcs->get_key_frame_decision = nullptr; + rc_funcs->get_encodeframe_decision = nullptr; + rc_funcs->update_encodeframe_result = nullptr; + } + ~ExtRateCtrlTest() override = default; + + void SetUp() override { + InitializeConfig(static_cast<libaom_test::TestMode>(GET_PARAM(1))); + cfg_.g_threads = 1; + cfg_.g_limit = kFrameNum; + is_create_model_called = false; + is_delete_model_called = false; + is_send_firstpass_stats_called = false; + is_send_extrc_tpl_gop_stats_called = false; + is_get_gop_decision_called = false; + is_update_encodeframe_result_called = false; + is_get_key_frame_decision_called = false; + } + + void PreEncodeFrameHook(::libaom_test::VideoSource *video, + ::libaom_test::Encoder *encoder) override { + if (video->frame() == 0) { + encoder->Control(AOME_SET_CPUUSED, cpu_used_); + encoder->Control(AV1E_SET_EXTERNAL_RATE_CONTROL, &rc_funcs_); + } + current_encoder_ = encoder; + } + + ::libaom_test::Encoder *current_encoder_; + aom_rc_funcs_t rc_funcs_; + int cpu_used_; +}; + +TEST_P(ExtRateCtrlTest, TestExternalRateCtrl) { + ::libaom_test::Y4mVideoSource video("screendata.y4m", 0, kFrameNum); + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + EXPECT_TRUE(is_create_model_called); + EXPECT_TRUE(is_send_firstpass_stats_called); + EXPECT_TRUE(is_send_extrc_tpl_gop_stats_called); + EXPECT_TRUE(is_delete_model_called); +} + +AV1_INSTANTIATE_TEST_SUITE(ExtRateCtrlTest, + ::testing::Values(::libaom_test::kTwoPassGood), + ::testing::Values(3)); + +// Corresponds to QP 43 for 8-bit video. +const int kFrameQindex = 172; + +aom_rc_status_t mock_get_encodeframe_decision_const_q( + aom_rc_model_t /*ratectrl_model*/, int /*frame_gop_index*/, + aom_rc_encodeframe_decision_t *frame_decision) { + // Set constant QP. + frame_decision->q_index = kFrameQindex; + frame_decision->sb_params_list = NULL; // Initialize to NULL + return AOM_RC_OK; +} + +class ExtRateCtrlQpTest : public ExtRateCtrlTest { + protected: + ExtRateCtrlQpTest() { + rc_funcs_.get_encodeframe_decision = mock_get_encodeframe_decision_const_q; + } + ~ExtRateCtrlQpTest() override = default; + + void SetUp() override { + InitializeConfig(static_cast<libaom_test::TestMode>(GET_PARAM(1))); + cfg_.g_threads = 1; + cfg_.g_limit = kFrameNum; + is_create_model_called = false; + is_delete_model_called = false; + } + + void FramePktHook(const aom_codec_cx_pkt_t *pkt) override { + if (pkt->kind != AOM_CODEC_CX_FRAME_PKT) return; + int q_index = -1; + current_encoder_->Control(AOME_GET_LAST_QUANTIZER, &q_index); + EXPECT_EQ(q_index, kFrameQindex); + } +}; + +TEST_P(ExtRateCtrlQpTest, TestExternalRateCtrlConstQp) { + ::libaom_test::Y4mVideoSource video("screendata.y4m", 0, kFrameNum); + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + EXPECT_TRUE(is_create_model_called); + EXPECT_TRUE(is_delete_model_called); +} + +AV1_INSTANTIATE_TEST_SUITE(ExtRateCtrlQpTest, + ::testing::Values(::libaom_test::kTwoPassGood), + ::testing::Values(3)); + +class ExtRateCtrlUpdateEncodeFrameResultTest : public ExtRateCtrlTest { + protected: + ExtRateCtrlUpdateEncodeFrameResultTest() { + rc_funcs_.rc_type = AOM_RC_QP; + rc_funcs_.get_encodeframe_decision = mock_get_encodeframe_decision_const_q; + rc_funcs_.update_encodeframe_result = mock_update_encodeframe_result; + } + ~ExtRateCtrlUpdateEncodeFrameResultTest() override = default; + + void SetUp() override { + ExtRateCtrlTest::SetUp(); + is_update_encodeframe_result_called = false; + bit_count = 0; + actual_encoding_qindex = 0; + } + + void FramePktHook(const aom_codec_cx_pkt_t *pkt) override { + if (pkt->kind != AOM_CODEC_CX_FRAME_PKT) return; + int q_index = -1; + current_encoder_->Control(AOME_GET_LAST_QUANTIZER, &q_index); + EXPECT_EQ(q_index, kFrameQindex); + EXPECT_EQ(actual_encoding_qindex, kFrameQindex); + // The second packet includes both invisible and visible frames. To verify + // frame size, there's no move_offset (obu_header_size + obu_payload_size) + // for vis_frame_size. + const int pkt_size_padding = pkt_count_ == 1 ? 0 : 2; + // Due to av1 packing invisible and visible frames together, we can only + // verify the size of the visible frame in the packet using vis_frame_size. + EXPECT_EQ((bit_count >> 3) + pkt_size_padding, + pkt->data.frame.vis_frame_size); + pkt_count_++; + } + + private: + int pkt_count_ = 0; +}; + +TEST_P(ExtRateCtrlUpdateEncodeFrameResultTest, + TestExternalRateCtrlUpdateEncodeFrameResult) { + ::libaom_test::Y4mVideoSource video("screendata.y4m", 0, kFrameNum); + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + EXPECT_TRUE(is_create_model_called); + EXPECT_TRUE(is_delete_model_called); + EXPECT_TRUE(is_update_encodeframe_result_called); +} + +AV1_INSTANTIATE_TEST_SUITE(ExtRateCtrlUpdateEncodeFrameResultTest, + ::testing::Values(::libaom_test::kTwoPassGood), + ::testing::Values(3)); + +class ExtRateCtrlGopTest : public ExtRateCtrlTest { + protected: + ExtRateCtrlGopTest() { + rc_funcs_.rc_type = AOM_RC_GOP; + rc_funcs_.get_gop_decision = mock_get_gop_decision; + } + + void PostEncodeFrameHook(::libaom_test::Encoder *encoder) override { + if (cfg_.g_pass == AOM_RC_FIRST_PASS) return; + encoder->Control(AV1E_GET_GOP_INFO, &gop_info_); + } + + void FramePktHook(const aom_codec_cx_pkt_t * /*pkt*/) override { + // This is verified here (not in PostEncodeFrameHook) because + // PostEncodeFrameHook is also called when encoder is reading frames into + // lookahead buffer, when GOP structure hasn't been determined. + ASSERT_EQ(gop_info_.gop_size, kGopFrameCount); + } + + ~ExtRateCtrlGopTest() override = default; + aom_gop_info_t gop_info_; +}; + +TEST_P(ExtRateCtrlGopTest, TestExternalRateCtrlGop) { + ::libaom_test::Y4mVideoSource video("screendata.y4m", 0, kFrameNum); + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + EXPECT_TRUE(is_create_model_called); + EXPECT_TRUE(is_get_gop_decision_called); + EXPECT_TRUE(is_delete_model_called); +} + +AV1_INSTANTIATE_TEST_SUITE(ExtRateCtrlGopTest, + ::testing::Values(::libaom_test::kTwoPassGood), + ::testing::Values(3)); + +class ExtRateCtrlKeyFrameTest : public ExtRateCtrlTest { + protected: + ExtRateCtrlKeyFrameTest() { + rc_funcs_.rc_type = AOM_RC_GOP; + rc_funcs_.get_key_frame_decision = mock_get_key_frame_decision; + } + + ~ExtRateCtrlKeyFrameTest() override = default; + + void SetUp() override { + ExtRateCtrlTest::SetUp(); + is_get_key_frame_decision_called = false; + } + + void FramePktHook(const aom_codec_cx_pkt_t *pkt) override { + if (pkt->kind != AOM_CODEC_CX_FRAME_PKT) return; + EXPECT_TRUE(pkt->data.frame.flags & AOM_FRAME_IS_KEY); + } +}; + +TEST_P(ExtRateCtrlKeyFrameTest, TestExternalRateCtrlKeyFrame) { + ::libaom_test::Y4mVideoSource video("screendata.y4m", 0, kFrameNum); + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + EXPECT_TRUE(is_create_model_called); + EXPECT_TRUE(is_get_key_frame_decision_called); + EXPECT_TRUE(is_delete_model_called); +} + +AV1_INSTANTIATE_TEST_SUITE(ExtRateCtrlKeyFrameTest, + ::testing::Values(::libaom_test::kTwoPassGood), + ::testing::Values(3)); +} // namespace
diff --git a/test/fft_test.cc b/test/fft_test.cc index 1d3d59a..a17249d 100644 --- a/test/fft_test.cc +++ b/test/fft_test.cc
@@ -25,7 +25,7 @@ namespace { -typedef void (*tform_fun_t)(const float *input, float *temp, float *output); +using tform_fun_t = void (*)(const float *input, float *temp, float *output); // Simple 1D FFT implementation template <typename InputType>
diff --git a/test/filterintra_test.cc b/test/filterintra_test.cc index d425287..0375f09 100644 --- a/test/filterintra_test.cc +++ b/test/filterintra_test.cc
@@ -25,15 +25,15 @@ using libaom_test::ACMRandom; using std::tuple; -typedef void (*Predictor)(uint8_t *dst, ptrdiff_t stride, TX_SIZE tx_size, - const uint8_t *above, const uint8_t *left, int mode); +using Predictor = void (*)(uint8_t *dst, ptrdiff_t stride, TX_SIZE tx_size, + const uint8_t *above, const uint8_t *left, int mode); // Note: // Test parameter list: // Reference predictor, optimized predictor, prediction mode, tx size // -typedef tuple<Predictor, Predictor, int> PredFuncMode; -typedef tuple<PredFuncMode, TX_SIZE> PredParams; +using PredFuncMode = tuple<Predictor, Predictor, int>; +using PredParams = tuple<PredFuncMode, TX_SIZE>; const int MaxTxSize = 32; @@ -171,8 +171,6 @@ #endif // HAVE_SSE4_1 #if HAVE_NEON -// TODO(aomedia:349436249): enable for armv7 after SIGBUS is fixed. -#if AOM_ARCH_AARCH64 const PredFuncMode kPredFuncMdArrayNEON[] = { make_tuple(&av1_filter_intra_predictor_c, &av1_filter_intra_predictor_neon, FILTER_DC_PRED), @@ -194,9 +192,31 @@ NEON, AV1FilterIntraPredTest, ::testing::Combine(::testing::ValuesIn(kPredFuncMdArrayNEON), ::testing::ValuesIn(kTxSizeNEON))); -#else // !AOM_ARCH_AARCH64 -GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(AV1FilterIntraPredTest); -#endif // AOM_ARCH_AARCH64 #endif // HAVE_NEON +#if HAVE_NEON_I8MM +const PredFuncMode kPredFuncMdArrayNEON_I8MM[] = { + make_tuple(&av1_filter_intra_predictor_c, + &av1_filter_intra_predictor_neon_i8mm, FILTER_DC_PRED), + make_tuple(&av1_filter_intra_predictor_c, + &av1_filter_intra_predictor_neon_i8mm, FILTER_V_PRED), + make_tuple(&av1_filter_intra_predictor_c, + &av1_filter_intra_predictor_neon_i8mm, FILTER_H_PRED), + make_tuple(&av1_filter_intra_predictor_c, + &av1_filter_intra_predictor_neon_i8mm, FILTER_D157_PRED), + make_tuple(&av1_filter_intra_predictor_c, + &av1_filter_intra_predictor_neon_i8mm, FILTER_PAETH_PRED), +}; + +const TX_SIZE kTxSizeNEON_I8MM[] = { TX_4X4, TX_8X8, TX_16X16, TX_32X32, + TX_4X8, TX_8X4, TX_8X16, TX_16X8, + TX_16X32, TX_32X16, TX_4X16, TX_16X4, + TX_8X32, TX_32X8 }; + +INSTANTIATE_TEST_SUITE_P( + NEON_I8MM, AV1FilterIntraPredTest, + ::testing::Combine(::testing::ValuesIn(kPredFuncMdArrayNEON_I8MM), + ::testing::ValuesIn(kTxSizeNEON_I8MM))); +#endif // HAVE_NEON_I8MM + } // namespace
diff --git a/test/frame_resize_test.cc b/test/frame_resize_test.cc index a9acb24..f9557e0 100644 --- a/test/frame_resize_test.cc +++ b/test/frame_resize_test.cc
@@ -30,7 +30,7 @@ const int kIters = 1000; -typedef tuple<int, int> FrameDimension; +using FrameDimension = tuple<int, int>; // Check that two 8-bit output buffers are identical. void AssertOutputBufferEq(const uint8_t *p1, const uint8_t *p2, int width, @@ -50,12 +50,12 @@ } } -typedef bool (*LowBDResizeFunc)(uint8_t *intbuf, uint8_t *output, - int out_stride, int height, int height2, - int stride, int start_wd); +using LowBDResizeFunc = bool (*)(uint8_t *intbuf, uint8_t *output, + int out_stride, int height, int height2, + int stride, int start_wd); // Test parameter list: // <tst_fun, dims> -typedef tuple<LowBDResizeFunc, FrameDimension> ResizeTestParams; +using ResizeTestParams = tuple<LowBDResizeFunc, FrameDimension>; class AV1ResizeYTest : public ::testing::TestWithParam<ResizeTestParams> { public: @@ -166,11 +166,11 @@ ::testing::ValuesIn(kFrameDim))); #endif -typedef void (*LowBDResize_x_Func)(const uint8_t *const input, int in_stride, - uint8_t *intbuf, int height, - int filtered_length, int width2); +using LowBDResize_x_Func = void (*)(const uint8_t *const input, int in_stride, + uint8_t *intbuf, int height, + int filtered_length, int width2); -typedef tuple<LowBDResize_x_Func, FrameDimension> Resize_x_TestParams; +using Resize_x_TestParams = tuple<LowBDResize_x_Func, FrameDimension>; class AV1ResizeXTest : public ::testing::TestWithParam<Resize_x_TestParams> { public:
diff --git a/test/frame_size_tests.cc b/test/frame_size_tests.cc index 65269aa..4ed8354 100644 --- a/test/frame_size_tests.cc +++ b/test/frame_size_tests.cc
@@ -327,10 +327,10 @@ ::testing::Combine(::testing::Values(AOM_USAGE_ALL_INTRA), ::testing::Values(AOM_Q), ::testing::Range(6, 10))); -typedef struct { +struct FrameSizeParam { unsigned int width; unsigned int height; -} FrameSizeParam; +}; const FrameSizeParam FrameSizeTestParams[] = { { 96, 96 }, { 176, 144 } };
diff --git a/test/fwht4x4_test.cc b/test/fwht4x4_test.cc index 8e9600a..c68879e 100644 --- a/test/fwht4x4_test.cc +++ b/test/fwht4x4_test.cc
@@ -31,13 +31,13 @@ using libaom_test::ACMRandom; namespace { -typedef void (*FdctFunc)(const int16_t *in, tran_low_t *out, int stride); -typedef void (*IdctFunc)(const tran_low_t *in, uint8_t *out, int stride); +using FdctFunc = void (*)(const int16_t *in, tran_low_t *out, int stride); +using IdctFunc = void (*)(const tran_low_t *in, uint8_t *out, int stride); using libaom_test::FhtFunc; -typedef std::tuple<FdctFunc, IdctFunc, TX_TYPE, aom_bit_depth_t, int, FdctFunc> - Dct4x4Param; +using Dct4x4Param = + std::tuple<FdctFunc, IdctFunc, TX_TYPE, aom_bit_depth_t, int, FdctFunc>; void fwht4x4_ref(const int16_t *in, tran_low_t *out, int stride, TxfmParam * /*txfm_param*/) {
diff --git a/test/hash_test.cc b/test/hash_test.cc index f824184..adc072e 100644 --- a/test/hash_test.cc +++ b/test/hash_test.cc
@@ -24,10 +24,10 @@ namespace { -typedef uint32_t (*get_crc32c_value_func)(void *calculator, const uint8_t *p, - size_t length); +using get_crc32c_value_func = uint32_t (*)(void *calculator, const uint8_t *p, + size_t length); -typedef std::tuple<get_crc32c_value_func, int> HashParam; +using HashParam = std::tuple<get_crc32c_value_func, int>; class AV1Crc32cHashTest : public ::testing::TestWithParam<HashParam> { public:
diff --git a/test/hbd_metrics_test.cc b/test/hbd_metrics_test.cc index af050a0..6dfe4cc 100644 --- a/test/hbd_metrics_test.cc +++ b/test/hbd_metrics_test.cc
@@ -29,11 +29,11 @@ namespace { -typedef double (*LBDMetricFunc)(const YV12_BUFFER_CONFIG *source, - const YV12_BUFFER_CONFIG *dest); -typedef double (*HBDMetricFunc)(const YV12_BUFFER_CONFIG *source, - const YV12_BUFFER_CONFIG *dest, uint32_t in_bd, - uint32_t bd); +using LBDMetricFunc = double (*)(const YV12_BUFFER_CONFIG *source, + const YV12_BUFFER_CONFIG *dest); +using HBDMetricFunc = double (*)(const YV12_BUFFER_CONFIG *source, + const YV12_BUFFER_CONFIG *dest, uint32_t in_bd, + uint32_t bd); double compute_hbd_psnr(const YV12_BUFFER_CONFIG *source, const YV12_BUFFER_CONFIG *dest, uint32_t in_bd, @@ -173,8 +173,8 @@ HBDMetricFunc hbd_metric_; }; -typedef std::tuple<LBDMetricFunc, HBDMetricFunc, int, int, double> - MetricTestTParam; +using MetricTestTParam = + std::tuple<LBDMetricFunc, HBDMetricFunc, int, int, double>; class HBDMetricsTest : public HBDMetricsTestBase, public ::testing::TestWithParam<MetricTestTParam> { public:
diff --git a/test/hiprec_convolve_test_util.h b/test/hiprec_convolve_test_util.h index 52a43e9..d33baa5 100644 --- a/test/hiprec_convolve_test_util.h +++ b/test/hiprec_convolve_test_util.h
@@ -29,14 +29,14 @@ namespace AV1HiprecConvolve { -typedef void (*hiprec_convolve_func)(const uint8_t *src, ptrdiff_t src_stride, - uint8_t *dst, ptrdiff_t dst_stride, - const int16_t *filter_x, int x_step_q4, - const int16_t *filter_y, int y_step_q4, - int w, int h, - const WienerConvolveParams *conv_params); +using hiprec_convolve_func = void (*)(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, + int w, int h, + const WienerConvolveParams *conv_params); -typedef std::tuple<int, int, int, hiprec_convolve_func> HiprecConvolveParam; +using HiprecConvolveParam = std::tuple<int, int, int, hiprec_convolve_func>; ::testing::internal::ParamGenerator<HiprecConvolveParam> BuildParams( hiprec_convolve_func filter); @@ -58,14 +58,14 @@ #if CONFIG_AV1_HIGHBITDEPTH namespace AV1HighbdHiprecConvolve { -typedef void (*highbd_hiprec_convolve_func)( - const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, - ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, - const int16_t *filter_y, int y_step_q4, int w, int h, - const WienerConvolveParams *conv_params, int bps); +using highbd_hiprec_convolve_func = + void (*)(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, + ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, int w, int h, + const WienerConvolveParams *conv_params, int bps); -typedef std::tuple<int, int, int, int, highbd_hiprec_convolve_func> - HighbdHiprecConvolveParam; +using HighbdHiprecConvolveParam = + std::tuple<int, int, int, int, highbd_hiprec_convolve_func>; ::testing::internal::ParamGenerator<HighbdHiprecConvolveParam> BuildParams( highbd_hiprec_convolve_func filter);
diff --git a/test/horver_correlation_test.cc b/test/horver_correlation_test.cc index bab3912..c9212de 100644 --- a/test/horver_correlation_test.cc +++ b/test/horver_correlation_test.cc
@@ -26,10 +26,10 @@ using libaom_test::ACMRandom; namespace { -typedef void (*HorverFunc)(const int16_t *diff, int stride, int w, int h, - float *hcorr, float *vcorr); +using HorverFunc = void (*)(const int16_t *diff, int stride, int w, int h, + float *hcorr, float *vcorr); -typedef std::tuple<const HorverFunc> HorverTestParam; +using HorverTestParam = std::tuple<const HorverFunc>; class HorverTest : public ::testing::TestWithParam<HorverTestParam> { public:
diff --git a/test/horz_superres_test.cc b/test/horz_superres_test.cc index fee0cca..6d4ca8a 100644 --- a/test/horz_superres_test.cc +++ b/test/horz_superres_test.cc
@@ -32,7 +32,7 @@ const int kBitrate = 40; -typedef struct { +struct TestVideoParam { const char *filename; aom_img_fmt fmt; aom_bit_depth_t bit_depth; @@ -41,7 +41,7 @@ unsigned int screen_content; double psnr_threshold; // used by modes other than AOM_SUPERRES_AUTO double psnr_threshold2; // used by AOM_SUPERRES_AUTO -} TestVideoParam; +}; std::ostream &operator<<(std::ostream &os, const TestVideoParam &test_arg) { return os << "TestVideoParam { filename:" << test_arg.filename @@ -69,7 +69,7 @@ AOM_SUPERRES_AUTO }; // Superres denominators and superres kf denominators to be tested -typedef tuple<int, int> SuperresDenominatorPair; +using SuperresDenominatorPair = tuple<int, int>; const SuperresDenominatorPair kSuperresDenominators[] = { make_tuple(16, 9), make_tuple(13, 11), make_tuple(9, 9), make_tuple(13, 13), make_tuple(11, 16), make_tuple(8, 16), @@ -77,7 +77,7 @@ }; // Superres q thresholds and superres kf q thresholds to be tested -typedef tuple<int, int> SuperresQThresholdPair; +using SuperresQThresholdPair = tuple<int, int>; const SuperresQThresholdPair kSuperresQThresholds[] = { make_tuple(63, 63), make_tuple(63, 41), make_tuple(17, 63), make_tuple(41, 11), make_tuple(1, 37), make_tuple(11, 11), @@ -88,9 +88,8 @@ // Test parameter list: // <[needed for EncoderTest], test_video_param_, superres_mode_> -typedef tuple<const libaom_test::CodecFactory *, TestVideoParam, - aom_superres_mode> - HorzSuperresTestParam; +using HorzSuperresTestParam = + tuple<const libaom_test::CodecFactory *, TestVideoParam, aom_superres_mode>; class HorzSuperresEndToEndTest : public ::testing::TestWithParam<HorzSuperresTestParam>, @@ -187,9 +186,9 @@ // Test parameter list: // <[needed for EncoderTest], test_video_param_, tuple(superres_denom_, // superres_kf_denom_)> -typedef tuple<const libaom_test::CodecFactory *, TestVideoParam, - SuperresDenominatorPair> - HorzSuperresFixedTestParam; +using HorzSuperresFixedTestParam = + tuple<const libaom_test::CodecFactory *, TestVideoParam, + SuperresDenominatorPair>; class HorzSuperresFixedEndToEndTest : public ::testing::TestWithParam<HorzSuperresFixedTestParam>, @@ -297,9 +296,9 @@ // Test parameter list: // <[needed for EncoderTest], test_video_param_, // tuple(superres_qthresh_,superres_kf_qthresh_)> -typedef tuple<const libaom_test::CodecFactory *, TestVideoParam, - SuperresQThresholdPair> - HorzSuperresQThreshTestParam; +using HorzSuperresQThreshTestParam = + tuple<const libaom_test::CodecFactory *, TestVideoParam, + SuperresQThresholdPair>; class HorzSuperresQThreshEndToEndTest : public ::testing::TestWithParam<HorzSuperresQThreshTestParam>,
diff --git a/test/intra_edge_test.cc b/test/intra_edge_test.cc index d5cc3d5..9b992d2 100644 --- a/test/intra_edge_test.cc +++ b/test/intra_edge_test.cc
@@ -62,8 +62,8 @@ int size_; }; -typedef void (*UP8B)(uint8_t *p, int size); -typedef libaom_test::FuncParam<UP8B> TestFuncs; +using UP8B = void (*)(uint8_t *p, int size); +using TestFuncs = libaom_test::FuncParam<UP8B>; class UpsampleTest8B : public UpsampleTest<UP8B, uint8_t> { protected: @@ -154,8 +154,8 @@ int strength_; }; -typedef void (*FE8B)(uint8_t *p, int size, int strength); -typedef libaom_test::FuncParam<FE8B> FilterEdgeTestFuncs; +using FE8B = void (*)(uint8_t *p, int size, int strength); +using FilterEdgeTestFuncs = libaom_test::FuncParam<FE8B>; class FilterEdgeTest8B : public FilterEdgeTest<FE8B, uint8_t> { protected: @@ -212,8 +212,8 @@ #if CONFIG_AV1_HIGHBITDEPTH -typedef void (*UPHB)(uint16_t *p, int size, int bd); -typedef libaom_test::FuncParam<UPHB> TestFuncsHBD; +using UPHB = void (*)(uint16_t *p, int size, int bd); +using TestFuncsHBD = libaom_test::FuncParam<UPHB>; class UpsampleTestHB : public UpsampleTest<UPHB, uint16_t> { protected: @@ -281,8 +281,8 @@ av1_highbd_upsample_intra_edge_neon))); #endif // HAVE_NEON -typedef void (*FEHB)(uint16_t *p, int size, int strength); -typedef libaom_test::FuncParam<FEHB> FilterEdgeTestFuncsHBD; +using FEHB = void (*)(uint16_t *p, int size, int strength); +using FilterEdgeTestFuncsHBD = libaom_test::FuncParam<FEHB>; class FilterEdgeTestHB : public FilterEdgeTest<FEHB, uint16_t> { protected:
diff --git a/test/intrapred_test.cc b/test/intrapred_test.cc index bf9ab75..f3a246b 100644 --- a/test/intrapred_test.cc +++ b/test/intrapred_test.cc
@@ -30,11 +30,11 @@ const int count_test_block = 100000; -typedef void (*HighbdIntraPred)(uint16_t *dst, ptrdiff_t stride, - const uint16_t *above, const uint16_t *left, - int bps); -typedef void (*IntraPred)(uint8_t *dst, ptrdiff_t stride, const uint8_t *above, - const uint8_t *left); +using HighbdIntraPred = void (*)(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, const uint16_t *left, + int bps); +using IntraPred = void (*)(uint8_t *dst, ptrdiff_t stride, const uint8_t *above, + const uint8_t *left); } // namespace
diff --git a/test/invalid_file_test.cc b/test/invalid_file_test.cc index 9a052b4..4cbf29e 100644 --- a/test/invalid_file_test.cc +++ b/test/invalid_file_test.cc
@@ -13,6 +13,7 @@ #include <ostream> #include <string> +#include "config/aom_config.h" #include "gtest/gtest.h" #include "test/codec_factory.h" #include "test/ivf_video_source.h" @@ -131,7 +132,7 @@ { 1, "invalid-google-142530197-1.ivf", nullptr }, { 4, "invalid-oss-fuzz-9463.ivf", "invalid-oss-fuzz-9463.ivf.res.2" }, { 1, "invalid-oss-fuzz-9720.ivf", nullptr }, - { 1, "invalid-oss-fuzz-10389.ivf", "invalid-oss-fuzz-10389.ivf.res.4" }, + { 1, "invalid-oss-fuzz-10389.ivf", "invalid-oss-fuzz-10389.ivf.res.5" }, #if !defined(CHROMIUM) && !CONFIG_SIZE_LIMIT || \ (CONFIG_SIZE_LIMIT && DECODE_WIDTH_LIMIT >= 5120 && \ DECODE_HEIGHT_LIMIT >= 180) @@ -151,11 +152,7 @@ { 1, "invalid-oss-fuzz-10227.ivf", nullptr }, { 4, "invalid-oss-fuzz-10555.ivf", nullptr }, { 1, "invalid-oss-fuzz-10705.ivf", nullptr }, -#if CONFIG_CWG_C013 { 1, "invalid-oss-fuzz-10723.ivf", "invalid-oss-fuzz-10723.ivf.res.3" }, -#else - { 1, "invalid-oss-fuzz-10723.ivf", "invalid-oss-fuzz-10723.ivf.res.2" }, -#endif { 1, "invalid-oss-fuzz-10779.ivf", nullptr }, { 1, "invalid-oss-fuzz-11477.ivf", nullptr }, { 1, "invalid-oss-fuzz-11479.ivf", "invalid-oss-fuzz-11479.ivf.res.2" },
diff --git a/test/kf_test.cc b/test/kf_test.cc index 3d76873..5d7cfdb 100644 --- a/test/kf_test.cc +++ b/test/kf_test.cc
@@ -107,10 +107,10 @@ } } -typedef struct { +struct kfIntervalParam { const unsigned int min_kf_dist; const unsigned int max_kf_dist; -} kfIntervalParam; +}; const kfIntervalParam kfTestParams[] = { { 1, 1 }, { 0, 10 }, { 10, 10 }, { 0, 30 }, { 30, 30 }
diff --git a/test/level_test.cc b/test/level_test.cc index dd8981c..70f4b21 100644 --- a/test/level_test.cc +++ b/test/level_test.cc
@@ -135,12 +135,12 @@ // To save run time, we only test speed 4. if (cpu_used_ == 4) { libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288, - 30, 1, 0, 30); + 30, 1, 0, 40); target_level_ = kLevelKeepStats; cfg_.rc_target_bitrate = 1000; - cfg_.g_limit = 30; + cfg_.g_limit = 40; ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); - ASSERT_LE(level_[0], 0); + ASSERT_EQ(level_[0], 0); } } @@ -148,12 +148,12 @@ // To save run time, we only test speed 4. if (cpu_used_ == 4) { libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288, - 30, 1, 0, 30); + 30, 1, 0, 40); target_level_ = kLevelKeepStats; cfg_.rc_target_bitrate = 4000; - cfg_.g_limit = 30; + cfg_.g_limit = 40; ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); - ASSERT_LE(level_[0], 4); + ASSERT_EQ(level_[0], 4); } } @@ -166,7 +166,7 @@ target_level_ = target_level; cfg_.rc_target_bitrate = 4000; ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); - ASSERT_LE(level_[0], target_level); + ASSERT_EQ(level_[0], target_level); } }
diff --git a/test/loopfilter_control_test.cc b/test/loopfilter_control_test.cc index 30f04b2..f498559 100644 --- a/test/loopfilter_control_test.cc +++ b/test/loopfilter_control_test.cc
@@ -46,13 +46,13 @@ { 2, { { 0, 31.0 }, { 3, 31.0 } } }, { 3, { { 0, 31.0 }, { 3, 31.0 } } } } } }; -typedef struct { +struct TestVideoParam { const char *filename; unsigned int input_bit_depth; aom_img_fmt fmt; aom_bit_depth_t bit_depth; unsigned int profile; -} TestVideoParam; +}; std::ostream &operator<<(std::ostream &os, const TestVideoParam &test_arg) { return os << "TestVideoParam { filename:" << test_arg.filename
diff --git a/test/low_complexity_test.sh b/test/low_complexity_test.sh new file mode 100755 index 0000000..2521b2b --- /dev/null +++ b/test/low_complexity_test.sh
@@ -0,0 +1,89 @@ +#!/bin/sh +## Copyright (c) 2026, Alliance for Open Media. All rights reserved. +## +## This source code is subject to the terms of the BSD 2 Clause License and +## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +## was not distributed with this source code in the LICENSE file, you can +## obtain it at www.aomedia.org/license/software. If the Alliance for Open +## Media Patent License 1.0 was not distributed with this source code in the +## PATENTS file, you can obtain it at www.aomedia.org/license/patent. +## +## This file tests the libaom low-complexity decode mode using simple_encoder +## and simple_decoder example. The decoder outputs the instruction counts that +## can be used for decoder speed monitoring. +## +. $(dirname $0)/tools_common.sh + +# Environment check: $infile is required. +low_complexity_test_verify_environment() { + if [ ! "$(av1_encode_available)" = "yes" ] || \ + [ ! "$(av1_decode_available)" = "yes" ]; then + echo "Encoder and decoder needs to be available." + return 1 + fi + + local infile="${LIBAOM_TEST_DATA_PATH}/SDR_Sports_6mug_608p_30fps_90f.yuv" + if [ ! -e "${infile}" ]; then + echo "Libaom test data must exist in LIBAOM_TEST_DATA_PATH." + return 1 + fi + + command -v perf >/dev/null 2>&1 + if [ $? -ne 0 ]; then + echo "Perf does not exist. Please install it." + return 1 + fi +} + +# Runs libaom low-complexity decode mode test +low_complexity_test() { + local img_width=608 + local img_height=1080 + + # Encode in low-complexity decode mode + local encoder="${LIBAOM_BIN_PATH}/low_complexity_mode_encoder${AOM_TEST_EXE_SUFFIX}" + local input_file="${LIBAOM_TEST_DATA_PATH}/SDR_Sports_6mug_608p_30fps_90f.yuv" + local ivf_file="${AOM_TEST_OUTPUT_DIR}/low_complexity_mode_encoder.bin" + + if [ ! -x "${encoder}" ]; then + elog "${encoder} does not exist or is not executable." + return 1 + fi + + echo "The LC mode encoder is started." + local enc_cmd="$(echo "${AOM_TEST_PREFIX}" "${encoder}" av1 "${img_width}" \ + "${img_height}" "${input_file}" "${ivf_file}" 150 90 "\ + "${devnull})" + echo "$enc_cmd" + eval "$enc_cmd" || return 1 + + [ -e "${ivf_file}" ] || return 1 + echo "The LC mode encoder is completed." + + # Use simple_decoder to decode the generated bitstream and collect the + # instruction counts. + local decoder="${LIBAOM_BIN_PATH}/simple_decoder${AOM_TEST_EXE_SUFFIX}" + local perfstat_file="${AOM_TEST_OUTPUT_DIR}/perfstat.txt" + local perf_prefix="perf stat --no-big-num -e instructions:u -o ${perfstat_file}" + local output_file="${AOM_TEST_OUTPUT_DIR}/low_complexity_mode_encoder.out" + + if [ ! -x "${decoder}" ]; then + elog "${decoder} does not exist or is not executable." + return 1 + fi + + local dec_cmd="$(echo "${perf_prefix}" "${AOM_TEST_PREFIX}" "${decoder}" \ + "${ivf_file}" "${output_file}" ${devnull})" + echo "$dec_cmd" + eval "$dec_cmd" || return 1 + + # Get perf user instruction count from perf output + local instruction_count="$(awk '/instructions:u/ {print $1}' \ + "${perfstat_file}")" + echo ${instruction_count} +} + +low_complexity_decode_mode_test="low_complexity_test" + +run_tests low_complexity_test_verify_environment \ + "${low_complexity_decode_mode_test}"
diff --git a/test/lpf_test.cc b/test/lpf_test.cc index edb107b..da9d162 100644 --- a/test/lpf_test.cc +++ b/test/lpf_test.cc
@@ -45,16 +45,16 @@ const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, \ const uint8_t *thresh1 -typedef void (*loop_op_t)(uint8_t *s, LOOP_PARAM); -typedef void (*dual_loop_op_t)(uint8_t *s, DUAL_LOOP_PARAM); -typedef void (*hbdloop_op_t)(uint16_t *s, LOOP_PARAM, int bd); -typedef void (*hbddual_loop_op_t)(uint16_t *s, DUAL_LOOP_PARAM, int bd); +using loop_op_t = void (*)(uint8_t *s, LOOP_PARAM); +using dual_loop_op_t = void (*)(uint8_t *s, DUAL_LOOP_PARAM); +using hbdloop_op_t = void (*)(uint16_t *s, LOOP_PARAM, int bd); +using hbddual_loop_op_t = void (*)(uint16_t *s, DUAL_LOOP_PARAM, int bd); -typedef std::tuple<hbdloop_op_t, hbdloop_op_t, int> hbdloop_param_t; -typedef std::tuple<hbddual_loop_op_t, hbddual_loop_op_t, int> - hbddual_loop_param_t; -typedef std::tuple<loop_op_t, loop_op_t, int> loop_param_t; -typedef std::tuple<dual_loop_op_t, dual_loop_op_t, int> dual_loop_param_t; +using hbdloop_param_t = std::tuple<hbdloop_op_t, hbdloop_op_t, int>; +using hbddual_loop_param_t = + std::tuple<hbddual_loop_op_t, hbddual_loop_op_t, int>; +using loop_param_t = std::tuple<loop_op_t, loop_op_t, int>; +using dual_loop_param_t = std::tuple<dual_loop_op_t, dual_loop_op_t, int>; template <typename Pixel_t, int PIXEL_WIDTH_t> void InitInput(Pixel_t *s, Pixel_t *ref_s, ACMRandom *rnd, const uint8_t limit, @@ -161,15 +161,15 @@ } #if CONFIG_AV1_HIGHBITDEPTH -typedef LoopTestParam<hbdloop_op_t, hbdloop_param_t> Loop8Test6Param_hbd; +using Loop8Test6Param_hbd = LoopTestParam<hbdloop_op_t, hbdloop_param_t>; GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(Loop8Test6Param_hbd); -typedef LoopTestParam<hbddual_loop_op_t, hbddual_loop_param_t> - Loop8Test9Param_hbd; +using Loop8Test9Param_hbd = + LoopTestParam<hbddual_loop_op_t, hbddual_loop_param_t>; GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(Loop8Test9Param_hbd); #endif -typedef LoopTestParam<loop_op_t, loop_param_t> Loop8Test6Param_lbd; +using Loop8Test6Param_lbd = LoopTestParam<loop_op_t, loop_param_t>; GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(Loop8Test6Param_lbd); -typedef LoopTestParam<dual_loop_op_t, dual_loop_param_t> Loop8Test9Param_lbd; +using Loop8Test9Param_lbd = LoopTestParam<dual_loop_op_t, dual_loop_param_t>; GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(Loop8Test9Param_lbd); #define OPCHECK(a, b) \
diff --git a/test/masked_sad_test.cc b/test/masked_sad_test.cc index 1b9e953..4de17ae 100644 --- a/test/masked_sad_test.cc +++ b/test/masked_sad_test.cc
@@ -28,20 +28,20 @@ namespace { const int number_of_iterations = 200; -typedef unsigned int (*MaskedSADFunc)(const uint8_t *src, int src_stride, - const uint8_t *ref, int ref_stride, - const uint8_t *second_pred, - const uint8_t *msk, int msk_stride, - int invert_mask); -typedef std::tuple<MaskedSADFunc, MaskedSADFunc> MaskedSADParam; +using MaskedSADFunc = unsigned int (*)(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + const uint8_t *second_pred, + const uint8_t *msk, int msk_stride, + int invert_mask); +using MaskedSADParam = std::tuple<MaskedSADFunc, MaskedSADFunc>; -typedef void (*MaskedSADx4Func)(const uint8_t *src, int src_stride, - const uint8_t *ref[], int ref_stride, - const uint8_t *second_pred, const uint8_t *msk, - int msk_stride, int invert_mask, - unsigned sads[]); +using MaskedSADx4Func = void (*)(const uint8_t *src, int src_stride, + const uint8_t *ref[], int ref_stride, + const uint8_t *second_pred, const uint8_t *msk, + int msk_stride, int invert_mask, + unsigned sads[]); -typedef std::tuple<MaskedSADx4Func, MaskedSADx4Func> MaskedSADx4Param; +using MaskedSADx4Param = std::tuple<MaskedSADx4Func, MaskedSADx4Func>; class MaskedSADTestBase : public ::testing::Test { public: @@ -195,13 +195,13 @@ TEST_P(MaskedSADTest, DISABLED_Speed) { runMaskedSADTest(2000000); } #if CONFIG_AV1_HIGHBITDEPTH -typedef unsigned int (*HighbdMaskedSADFunc)(const uint8_t *src, int src_stride, - const uint8_t *ref, int ref_stride, - const uint8_t *second_pred, - const uint8_t *msk, int msk_stride, - int invert_mask); -typedef std::tuple<HighbdMaskedSADFunc, HighbdMaskedSADFunc> - HighbdMaskedSADParam; +using HighbdMaskedSADFunc = unsigned int (*)(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + const uint8_t *second_pred, + const uint8_t *msk, int msk_stride, + int invert_mask); +using HighbdMaskedSADParam = + std::tuple<HighbdMaskedSADFunc, HighbdMaskedSADFunc>; class HighbdMaskedSADTest : public ::testing::TestWithParam<HighbdMaskedSADParam> {
diff --git a/test/masked_variance_test.cc b/test/masked_variance_test.cc index 4d9ebe9..e527efa 100644 --- a/test/masked_variance_test.cc +++ b/test/masked_variance_test.cc
@@ -32,13 +32,13 @@ namespace { const int number_of_iterations = 200; -typedef unsigned int (*MaskedSubPixelVarianceFunc)( +using MaskedSubPixelVarianceFunc = unsigned int (*)( const uint8_t *src, int src_stride, int xoffset, int yoffset, const uint8_t *ref, int ref_stride, const uint8_t *second_pred, const uint8_t *msk, int msk_stride, int invert_mask, unsigned int *sse); -typedef std::tuple<MaskedSubPixelVarianceFunc, MaskedSubPixelVarianceFunc> - MaskedSubPixelVarianceParam; +using MaskedSubPixelVarianceParam = + std::tuple<MaskedSubPixelVarianceFunc, MaskedSubPixelVarianceFunc>; class MaskedSubPixelVarianceTest : public ::testing::TestWithParam<MaskedSubPixelVarianceParam> { @@ -170,9 +170,9 @@ } #if CONFIG_AV1_HIGHBITDEPTH -typedef std::tuple<MaskedSubPixelVarianceFunc, MaskedSubPixelVarianceFunc, - aom_bit_depth_t> - HighbdMaskedSubPixelVarianceParam; +using HighbdMaskedSubPixelVarianceParam = + std::tuple<MaskedSubPixelVarianceFunc, MaskedSubPixelVarianceFunc, + aom_bit_depth_t>; class HighbdMaskedSubPixelVarianceTest : public ::testing::TestWithParam<HighbdMaskedSubPixelVarianceParam> {
diff --git a/test/metadata_test.cc b/test/metadata_test.cc index da0fb13..c34f4da 100644 --- a/test/metadata_test.cc +++ b/test/metadata_test.cc
@@ -35,6 +35,12 @@ 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17 }; +const size_t kMetadataPayloadSizeT35Two = 10; +// 0xB5 stands for the itut t35 metadata country code for the Unites States +const uint8_t kMetadataPayloadT35Two[kMetadataPayloadSizeT35] = { + 0xB5, 0x01, 0x02, 0x42, 0xff, 0xff, 0x00, 0x07, 0x08, 0x09 +}; + const size_t kMetadataPayloadSizeMdcv = 24; // Arbitrary content. const uint8_t kMetadataPayloadMdcv[kMetadataPayloadSizeMdcv] = { @@ -96,11 +102,18 @@ ASSERT_EQ(aom_img_add_metadata(current_frame, OBU_METADATA_TYPE_ITUT_T35, nullptr, 0, AOM_MIF_ANY_FRAME), -1); + ASSERT_EQ(aom_img_add_metadata(current_frame, OBU_METADATA_TYPE_ITUT_T35, kMetadataPayloadT35, kMetadataPayloadSizeT35, AOM_MIF_ANY_FRAME), 0); + ASSERT_EQ( + aom_img_add_metadata(current_frame, OBU_METADATA_TYPE_ITUT_T35, + kMetadataPayloadT35Two, kMetadataPayloadSizeT35Two, + AOM_MIF_ANY_FRAME_LAYER_SPECIFIC), + 0); + ASSERT_EQ(aom_img_add_metadata(current_frame, OBU_METADATA_TYPE_HDR_MDCV, kMetadataPayloadMdcv, kMetadataPayloadSizeMdcv, AOM_MIF_KEY_FRAME), @@ -143,32 +156,42 @@ ASSERT_NE(img.metadata, nullptr); - ASSERT_EQ(img.metadata->sz, is_key_frame ? 3 : 1); + ASSERT_EQ(img.metadata->sz, is_key_frame ? 4 : 2); - ASSERT_EQ(OBU_METADATA_TYPE_ITUT_T35, - img.metadata->metadata_array[0]->type); - ASSERT_EQ(kMetadataPayloadSizeT35, img.metadata->metadata_array[0]->sz); + aom_metadata_t *metadata = img.metadata->metadata_array[0]; + ASSERT_EQ(metadata->type, OBU_METADATA_TYPE_ITUT_T35); + ASSERT_EQ(metadata->insert_flag, AOM_MIF_ANY_FRAME); + ASSERT_EQ(metadata->sz, kMetadataPayloadSizeT35); EXPECT_EQ( - memcmp(kMetadataPayloadT35, img.metadata->metadata_array[0]->payload, - kMetadataPayloadSizeT35), + memcmp(kMetadataPayloadT35, metadata->payload, kMetadataPayloadSizeT35), 0); - if (is_key_frame) { - ASSERT_EQ(OBU_METADATA_TYPE_HDR_MDCV, - img.metadata->metadata_array[1]->type); - ASSERT_EQ(kMetadataPayloadSizeMdcv, img.metadata->metadata_array[1]->sz); - EXPECT_EQ( - memcmp(kMetadataPayloadMdcv, img.metadata->metadata_array[1]->payload, - kMetadataPayloadSizeMdcv), - 0); + metadata = img.metadata->metadata_array[1]; + ASSERT_EQ(metadata->type, OBU_METADATA_TYPE_ITUT_T35); + // AOM_MIF_ANY_FRAME and not AOM_MIF_ANY_FRAME_LAYER_SPECIFIC because the + // stream does not contain layers. + ASSERT_EQ(metadata->insert_flag, AOM_MIF_ANY_FRAME); + ASSERT_EQ(metadata->sz, kMetadataPayloadSizeT35Two); + EXPECT_EQ(memcmp(kMetadataPayloadT35Two, metadata->payload, + kMetadataPayloadSizeT35Two), + 0); - ASSERT_EQ(OBU_METADATA_TYPE_HDR_CLL, - img.metadata->metadata_array[2]->type); - ASSERT_EQ(kMetadataPayloadSizeCll, img.metadata->metadata_array[2]->sz); - EXPECT_EQ( - memcmp(kMetadataPayloadCll, img.metadata->metadata_array[2]->payload, - kMetadataPayloadSizeCll), - 0); + if (is_key_frame) { + metadata = img.metadata->metadata_array[2]; + ASSERT_EQ(metadata->type, OBU_METADATA_TYPE_HDR_MDCV); + ASSERT_EQ(metadata->insert_flag, AOM_MIF_ANY_FRAME); + ASSERT_EQ(metadata->sz, kMetadataPayloadSizeMdcv); + EXPECT_EQ(memcmp(kMetadataPayloadMdcv, metadata->payload, + kMetadataPayloadSizeMdcv), + 0); + + metadata = img.metadata->metadata_array[3]; + ASSERT_EQ(metadata->type, OBU_METADATA_TYPE_HDR_CLL); + ASSERT_EQ(metadata->insert_flag, AOM_MIF_ANY_FRAME); + ASSERT_EQ(metadata->sz, kMetadataPayloadSizeCll); + EXPECT_EQ(memcmp(kMetadataPayloadCll, metadata->payload, + kMetadataPayloadSizeCll), + 0); } } @@ -243,6 +266,12 @@ AOM_MIF_ANY_FRAME), 0); + ASSERT_EQ( + aom_img_add_metadata(current_frame, OBU_METADATA_TYPE_ITUT_T35, + kMetadataPayloadT35Two, kMetadataPayloadSizeT35Two, + AOM_MIF_ANY_FRAME_LAYER_SPECIFIC), + 0); + ASSERT_EQ(aom_img_add_metadata(current_frame, OBU_METADATA_TYPE_HDR_MDCV, kMetadataPayloadMdcv, kMetadataPayloadSizeMdcv, AOM_MIF_KEY_FRAME), @@ -288,32 +317,40 @@ ASSERT_NE(img.metadata, nullptr); - ASSERT_EQ(img.metadata->sz, is_key_frame ? 3 : 1); + ASSERT_EQ(img.metadata->sz, is_key_frame ? 4 : 2); - ASSERT_EQ(OBU_METADATA_TYPE_ITUT_T35, - img.metadata->metadata_array[0]->type); - ASSERT_EQ(kMetadataPayloadSizeT35, img.metadata->metadata_array[0]->sz); + aom_metadata_t *metadata = img.metadata->metadata_array[0]; + ASSERT_EQ(metadata->type, OBU_METADATA_TYPE_ITUT_T35); + ASSERT_EQ(metadata->insert_flag, AOM_MIF_ANY_FRAME); + ASSERT_EQ(metadata->sz, kMetadataPayloadSizeT35); EXPECT_EQ( - memcmp(kMetadataPayloadT35, img.metadata->metadata_array[0]->payload, - kMetadataPayloadSizeT35), + memcmp(kMetadataPayloadT35, metadata->payload, kMetadataPayloadSizeT35), 0); - if (is_key_frame) { - ASSERT_EQ(OBU_METADATA_TYPE_HDR_MDCV, - img.metadata->metadata_array[1]->type); - ASSERT_EQ(kMetadataPayloadSizeMdcv, img.metadata->metadata_array[1]->sz); - EXPECT_EQ( - memcmp(kMetadataPayloadMdcv, img.metadata->metadata_array[1]->payload, - kMetadataPayloadSizeMdcv), - 0); + metadata = img.metadata->metadata_array[1]; + ASSERT_EQ(metadata->type, OBU_METADATA_TYPE_ITUT_T35); + ASSERT_EQ(metadata->insert_flag, AOM_MIF_ANY_FRAME_LAYER_SPECIFIC); + ASSERT_EQ(metadata->sz, kMetadataPayloadSizeT35Two); + EXPECT_EQ(memcmp(kMetadataPayloadT35Two, metadata->payload, + kMetadataPayloadSizeT35Two), + 0); - ASSERT_EQ(OBU_METADATA_TYPE_HDR_CLL, - img.metadata->metadata_array[2]->type); - ASSERT_EQ(kMetadataPayloadSizeCll, img.metadata->metadata_array[2]->sz); - EXPECT_EQ( - memcmp(kMetadataPayloadCll, img.metadata->metadata_array[2]->payload, - kMetadataPayloadSizeCll), - 0); + if (is_key_frame) { + metadata = img.metadata->metadata_array[2]; + ASSERT_EQ(metadata->type, OBU_METADATA_TYPE_HDR_MDCV); + ASSERT_EQ(metadata->insert_flag, AOM_MIF_ANY_FRAME); + ASSERT_EQ(metadata->sz, kMetadataPayloadSizeMdcv); + EXPECT_EQ(memcmp(kMetadataPayloadMdcv, metadata->payload, + kMetadataPayloadSizeMdcv), + 0); + + metadata = img.metadata->metadata_array[3]; + ASSERT_EQ(metadata->type, OBU_METADATA_TYPE_HDR_CLL); + ASSERT_EQ(metadata->insert_flag, AOM_MIF_ANY_FRAME); + ASSERT_EQ(metadata->sz, kMetadataPayloadSizeCll); + EXPECT_EQ(memcmp(kMetadataPayloadCll, metadata->payload, + kMetadataPayloadSizeCll), + 0); } } @@ -402,6 +439,33 @@ -1); } +TEST(MetadataTest, AddLayerSpecificMetadataToImage) { + aom_image_t image; + image.metadata = nullptr; + + ASSERT_EQ( + aom_img_add_metadata( + &image, OBU_METADATA_TYPE_ITUT_T35, kMetadataPayloadT35, + kMetadataPayloadSizeT35, + (aom_metadata_insert_flags_t)(AOM_MIF_ANY_FRAME_LAYER_SPECIFIC)), + 0); + aom_img_metadata_array_free(image.metadata); +} + +TEST(MetadataTest, AddLayerSpecificMetadataToImageNotAllowed) { + aom_image_t image; + image.metadata = nullptr; + + // OBU_METADATA_TYPE_SCALABILITY cannot be layer specific. + ASSERT_EQ( + aom_img_add_metadata( + &image, OBU_METADATA_TYPE_SCALABILITY, kMetadataPayloadT35, + kMetadataPayloadSizeT35, + (aom_metadata_insert_flags_t)(AOM_MIF_ANY_FRAME_LAYER_SPECIFIC)), + -1); + aom_img_metadata_array_free(image.metadata); +} + TEST(MetadataTest, RemoveMetadataFromImage) { aom_image_t image; image.metadata = nullptr; @@ -454,9 +518,13 @@ kMetadataPayloadT35, kMetadataPayloadSizeT35, AOM_MIF_ANY_FRAME), 0); + ASSERT_EQ(aom_img_add_metadata(&image, OBU_METADATA_TYPE_ITUT_T35, + kMetadataPayloadT35, kMetadataPayloadSizeT35, + AOM_MIF_ANY_FRAME_LAYER_SPECIFIC), + 0); EXPECT_EQ(aom_img_get_metadata(nullptr, 0), nullptr); - EXPECT_EQ(aom_img_get_metadata(&image, 1u), nullptr); + EXPECT_EQ(aom_img_get_metadata(&image, 2u), nullptr); EXPECT_EQ(aom_img_get_metadata(&image, 10u), nullptr); const aom_metadata_t *metadata = aom_img_get_metadata(&image, 0); @@ -465,6 +533,15 @@ EXPECT_EQ( memcmp(kMetadataPayloadT35, metadata->payload, kMetadataPayloadSizeT35), 0); + EXPECT_EQ(metadata->insert_flag, AOM_MIF_ANY_FRAME); + + metadata = aom_img_get_metadata(&image, 1); + ASSERT_NE(metadata, nullptr); + ASSERT_EQ(metadata->sz, kMetadataPayloadSizeT35); + EXPECT_EQ( + memcmp(kMetadataPayloadT35, metadata->payload, kMetadataPayloadSizeT35), + 0); + EXPECT_EQ(metadata->insert_flag, AOM_MIF_ANY_FRAME_LAYER_SPECIFIC); aom_img_metadata_array_free(image.metadata); }
diff --git a/test/minmax_test.cc b/test/minmax_test.cc index b87ee97..bd1a965 100644 --- a/test/minmax_test.cc +++ b/test/minmax_test.cc
@@ -26,8 +26,8 @@ using ::libaom_test::ACMRandom; -typedef void (*MinMaxFunc)(const uint8_t *a, int a_stride, const uint8_t *b, - int b_stride, int *min, int *max); +using MinMaxFunc = void (*)(const uint8_t *a, int a_stride, const uint8_t *b, + int b_stride, int *min, int *max); class MinMaxTest : public ::testing::TestWithParam<MinMaxFunc> { public:
diff --git a/test/monochrome_test.cc b/test/monochrome_test.cc index 78afd4f..da2b3d9 100644 --- a/test/monochrome_test.cc +++ b/test/monochrome_test.cc
@@ -34,7 +34,7 @@ // kPsnrFluctuation represents the maximum allowed psnr fluctuation w.r.t first // frame. The indices correspond to one/two-pass, allintra and realtime // encoding modes. -const double kPsnrFluctuation[3] = { 2.5, 0.3, 17.0 }; +const double kPsnrFluctuation[3] = { 8.0, 0.3, 17.0 }; class MonochromeTest : public ::libaom_test::CodecTestWith3Params<libaom_test::TestMode, int,
diff --git a/test/multilayer_metadata_test.cc b/test/multilayer_metadata_test.cc index 33ce45f..faaf75f 100644 --- a/test/multilayer_metadata_test.cc +++ b/test/multilayer_metadata_test.cc
@@ -22,27 +22,27 @@ TEST(MultilayerMetadataTest, ParseAlpha) { const std::string metadata = R"( -use_case: 1 # global alpha -layers: - - layer_type: 5 # alpha - luma_plane_only_flag: 1 - layer_metadata_scope: 2 # global - alpha: - alpha_use_idc: 1 # premultiplied - alpha_bit_depth: 8 - alpha_transparent_value: 0 - alpha_opaque_value: 4 + use_case: 1 # global alpha + layers: + - layer_type: 5 # alpha + luma_plane_only_flag: 1 + layer_metadata_scope: 2 # global + alpha: + alpha_use_idc: 1 # premultiplied + alpha_bit_depth: 8 + alpha_transparent_value: 0 + alpha_opaque_value: 4 - - layer_type: 1 # texture - luma_plane_only_flag: 0 - layer_metadata_scope: 2 # global - layer_color_description: - color_range: 1 - color_primaries: 1 - transfer_characteristics: 13 - matrix_coefficients: 6 + - layer_type: 1 # texture + luma_plane_only_flag: 0 + layer_metadata_scope: 2 # global + layer_color_description: + color_range: 1 + color_primaries: 1 + transfer_characteristics: 13 + matrix_coefficients: 6 - )"; + )"; libaom_test::TempOutFile tmp_file(/*text_mode=*/true); fprintf(tmp_file.file(), "%s", metadata.c_str()); fflush(tmp_file.file()); @@ -55,10 +55,10 @@ EXPECT_EQ(multilayer.layers[0].layer_type, 5); EXPECT_EQ(multilayer.layers[0].luma_plane_only_flag, 1); EXPECT_EQ(multilayer.layers[0].layer_metadata_scope, 2); - EXPECT_EQ(multilayer.layers[0].global_alpha_info.alpha_use_idc, 1); - EXPECT_EQ(multilayer.layers[0].global_alpha_info.alpha_bit_depth, 8); - EXPECT_EQ(multilayer.layers[0].global_alpha_info.alpha_transparent_value, 0); - EXPECT_EQ(multilayer.layers[0].global_alpha_info.alpha_opaque_value, 4); + EXPECT_EQ(multilayer.layers[0].alpha.alpha_use_idc, 1); + EXPECT_EQ(multilayer.layers[0].alpha.alpha_bit_depth, 8); + EXPECT_EQ(multilayer.layers[0].alpha.alpha_transparent_value, 0); + EXPECT_EQ(multilayer.layers[0].alpha.alpha_opaque_value, 4); EXPECT_EQ(multilayer.layers[1].layer_type, 1); EXPECT_EQ(multilayer.layers[1].luma_plane_only_flag, 0); EXPECT_EQ(multilayer.layers[1].layer_metadata_scope, 2); @@ -76,26 +76,26 @@ TEST(MultilayerMetadataTest, ParseDepth) { const std::string metadata = R"( -use_case: 2 # global depth -layers: - - layer_type: 6 # depth - luma_plane_only_flag: 1 - layer_metadata_scope: 2 # global - depth: - z_near: 1.456 - z_far: 9.786 - depth_representation_type: 2 + use_case: 2 # global depth + layers: + - layer_type: 6 # depth + luma_plane_only_flag: 1 + layer_metadata_scope: 2 # global + depth: + z_near: 1.456 + z_far: 9.786 + depth_representation_type: 2 - - layer_type: 1 # texture - luma_plane_only_flag: 0 - layer_metadata_scope: 2 # global - layer_color_description: - color_range: 1 - color_primaries: 1 - transfer_characteristics: 13 - matrix_coefficients: 6 + - layer_type: 1 # texture + luma_plane_only_flag: 0 + layer_metadata_scope: 2 # global + layer_color_description: + color_range: 1 + color_primaries: 1 + transfer_characteristics: 13 + matrix_coefficients: 6 - )"; + )"; libaom_test::TempOutFile tmp_file(/*text_mode=*/true); fprintf(tmp_file.file(), "%s", metadata.c_str()); fflush(tmp_file.file()); @@ -107,16 +107,15 @@ EXPECT_EQ(multilayer.layers[0].layer_type, 6); EXPECT_EQ(multilayer.layers[0].luma_plane_only_flag, 1); EXPECT_EQ(multilayer.layers[0].layer_metadata_scope, 2); - EXPECT_TRUE(multilayer.layers[0].global_depth_info.z_near.second); + EXPECT_TRUE(multilayer.layers[0].depth.z_near.second); EXPECT_NEAR(depth_representation_element_to_double( - multilayer.layers[0].global_depth_info.z_near.first), + multilayer.layers[0].depth.z_near.first), 1.456, 0.00001); - EXPECT_TRUE(multilayer.layers[0].global_depth_info.z_far.second); + EXPECT_TRUE(multilayer.layers[0].depth.z_far.second); EXPECT_NEAR(depth_representation_element_to_double( - multilayer.layers[0].global_depth_info.z_far.first), + multilayer.layers[0].depth.z_far.first), 9.786, 0.00001); - EXPECT_EQ(multilayer.layers[0].global_depth_info.depth_representation_type, - 2); + EXPECT_EQ(multilayer.layers[0].depth.depth_representation_type, 2); EXPECT_EQ(multilayer.layers[1].layer_type, 1); EXPECT_EQ(multilayer.layers[1].luma_plane_only_flag, 0); EXPECT_EQ(multilayer.layers[1].layer_metadata_scope, 2); @@ -132,22 +131,115 @@ 6); } -TEST(MultilayerMetadataTest, ParseInvalid) { +TEST(MultilayerMetadataTest, ParseLocalDepth) { const std::string metadata = R"( - -use_case: 1 # global alpha +use_case: 4 # depth layers: - - layer_type: 5 # alpha + - layer_type: 6 # depth luma_plane_only_flag: 1 - layer_metadata_scope: 2 # global + layer_metadata_scope: 3 # mixed + depth: + z_near: 1.456 + z_far: 9.786 + depth_representation_type: 2 + local_metadata: + - frame_idx: 4 + depth: + z_near: 2.78933 + z_far: 20.663 + depth_representation_type: 0 + - frame_idx: 100 + depth: + z_near: 0 + z_far: 24 + depth_representation_type: 0 - layer_type: 1 # texture luma_plane_only_flag: 0 - layer_metadata_scope: 2 # global + layer_metadata_scope: 3 # mixed + layer_color_description: + color_range: 1 + color_primaries: 1 + transfer_characteristics: 13 + matrix_coefficients: 6 + )"; + libaom_test::TempOutFile tmp_file(/*text_mode=*/true); + fprintf(tmp_file.file(), "%s", metadata.c_str()); + fflush(tmp_file.file()); + + MultilayerMetadata multilayer; + EXPECT_TRUE(parse_multilayer_file(tmp_file.file_name().c_str(), &multilayer)); + EXPECT_EQ(multilayer.use_case, 4); + ASSERT_EQ(multilayer.layers.size(), 2); + EXPECT_EQ(multilayer.layers[0].layer_type, 6); + EXPECT_EQ(multilayer.layers[0].luma_plane_only_flag, 1); + EXPECT_EQ(multilayer.layers[0].layer_metadata_scope, 3); + EXPECT_TRUE(multilayer.layers[0].depth.z_near.second); + EXPECT_NEAR(depth_representation_element_to_double( + multilayer.layers[0].depth.z_near.first), + 1.456, 0.00001); + EXPECT_TRUE(multilayer.layers[0].depth.z_far.second); + EXPECT_NEAR(depth_representation_element_to_double( + multilayer.layers[0].depth.z_far.first), + 9.786, 0.00001); + EXPECT_EQ(multilayer.layers[0].depth.depth_representation_type, 2); + ASSERT_EQ(multilayer.layers[0].local_metadata.size(), 2); + EXPECT_EQ(multilayer.layers[0].local_metadata[0].frame_idx, 4); + EXPECT_TRUE(multilayer.layers[0].local_metadata[0].depth.z_near.second); + EXPECT_NEAR(depth_representation_element_to_double( + multilayer.layers[0].local_metadata[0].depth.z_near.first), + 2.78933, 0.00001); + EXPECT_TRUE(multilayer.layers[0].local_metadata[0].depth.z_far.second); + EXPECT_NEAR(depth_representation_element_to_double( + multilayer.layers[0].local_metadata[0].depth.z_far.first), + 20.663, 0.00001); + EXPECT_EQ( + multilayer.layers[0].local_metadata[0].depth.depth_representation_type, + 0); + EXPECT_EQ(multilayer.layers[0].local_metadata[1].frame_idx, 100); + EXPECT_TRUE(multilayer.layers[0].local_metadata[1].depth.z_near.second); + EXPECT_NEAR(depth_representation_element_to_double( + multilayer.layers[0].local_metadata[1].depth.z_near.first), + 0, 0.00001); + EXPECT_TRUE(multilayer.layers[0].local_metadata[1].depth.z_far.second); + EXPECT_NEAR(depth_representation_element_to_double( + multilayer.layers[0].local_metadata[1].depth.z_far.first), + 24, 0.00001); + EXPECT_EQ( + multilayer.layers[0].local_metadata[1].depth.depth_representation_type, + 0); + EXPECT_EQ(multilayer.layers[1].layer_type, 1); + EXPECT_EQ(multilayer.layers[1].luma_plane_only_flag, 0); + EXPECT_EQ(multilayer.layers[1].layer_metadata_scope, 3); + EXPECT_TRUE(multilayer.layers[1].layer_color_description.second); + EXPECT_EQ(multilayer.layers[1].layer_color_description.first.color_range, 1); + EXPECT_EQ(multilayer.layers[1].layer_color_description.first.color_primaries, + 1); + EXPECT_EQ(multilayer.layers[1] + .layer_color_description.first.transfer_characteristics, + 13); + EXPECT_EQ( + multilayer.layers[1].layer_color_description.first.matrix_coefficients, + 6); + EXPECT_EQ(multilayer.layers[1].local_metadata.size(), 0); +} + +TEST(MultilayerMetadataTest, ParseInvalid) { + const std::string metadata = R"( + +use_case: 3 # alpha +layers: + - layer_type: 5 # alpha + luma_plane_only_flag: 1 + layer_metadata_scope: 3 # mixed + + - layer_type: 1 # texture + luma_plane_only_flag: 0 + layer_metadata_scope: 3 # mixed - layer_type: 6 # depth => bad layer type luma_plane_only_flag: 1 - layer_metadata_scope: 2 # global + layer_metadata_scope: 3 # mixed )"; libaom_test::TempOutFile tmp_file(/*text_mode=*/true); fprintf(tmp_file.file(), "%s", metadata.c_str()); @@ -162,16 +254,16 @@ TEST(MultilayerMetadataTest, ParseBadIndent) { const std::string metadata = R"( -use_case: 1 # global alpha -layers: - - layer_type: 5 # alpha - luma_plane_only_flag: 1 - layer_metadata_scope: 2 # global + use_case: 1 # global alpha + layers: + - layer_type: 5 # alpha + luma_plane_only_flag: 1 + layer_metadata_scope: 2 # global - - layer_type: 1 # texture - luma_plane_only_flag: 0 - layer_metadata_scope: 2 # global - )"; + - layer_type: 1 # texture + luma_plane_only_flag: 0 + layer_metadata_scope: 2 # global + )"; libaom_test::TempOutFile tmp_file(/*text_mode=*/true); fprintf(tmp_file.file(), "%s", metadata.c_str()); fflush(tmp_file.file()); @@ -185,17 +277,17 @@ TEST(MultilayerMetadataTest, ParseUnknownField) { const std::string metadata = R"( -use_case: 1 # global alpha -layers: - - layer_type: 5 # alpha - luma_plane_only_flag: 1 - layer_metadata_scope: 2 # global - foobar: 42 + use_case: 1 # global alpha + layers: + - layer_type: 5 # alpha + luma_plane_only_flag: 1 + layer_metadata_scope: 2 # global + foobar: 42 - - layer_type: 1 # texture - luma_plane_only_flag: 0 - layer_metadata_scope: 2 # global - )"; + - layer_type: 1 # texture + luma_plane_only_flag: 0 + layer_metadata_scope: 2 # global + )"; libaom_test::TempOutFile tmp_file(/*text_mode=*/true); fprintf(tmp_file.file(), "%s", metadata.c_str()); fflush(tmp_file.file());
diff --git a/test/noise_model_test.cc b/test/noise_model_test.cc index 9f05a44..23f300c 100644 --- a/test/noise_model_test.cc +++ b/test/noise_model_test.cc
@@ -359,7 +359,7 @@ // (uint8_t and uint16_t) and bit depths (8, 10, 12). template <typename T, int bit_depth, bool use_highbd> struct BitDepthParams { - typedef T data_type_t; + using data_type_t = T; static const int kBitDepth = bit_depth; static const bool kUseHighBD = use_highbd; }; @@ -368,7 +368,7 @@ class FlatBlockEstimatorTest : public ::testing::Test, public T { public: void SetUp() override { random_.Reset(171); } - typedef std::vector<typename T::data_type_t> VecType; + using VecType = std::vector<typename T::data_type_t>; VecType data_; libaom_test::ACMRandom random_; }; @@ -527,11 +527,9 @@ REGISTER_TYPED_TEST_SUITE_P(FlatBlockEstimatorTest, ExtractBlock, FindFlatBlocks); -typedef ::testing::Types<BitDepthParams<uint8_t, 8, false>, // lowbd - BitDepthParams<uint16_t, 8, true>, // lowbd in 16-bit - BitDepthParams<uint16_t, 10, true>, // highbd data - BitDepthParams<uint16_t, 12, true> > - AllBitDepthParams; +using AllBitDepthParams = ::testing::Types< + BitDepthParams<uint8_t, 8, false>, BitDepthParams<uint16_t, 8, true>, + BitDepthParams<uint16_t, 10, true>, BitDepthParams<uint16_t, 12, true>>; // Note the empty final argument can be removed if C++20 is made the minimum // requirement. INSTANTIATE_TYPED_TEST_SUITE_P(FlatBlockInstatiation, FlatBlockEstimatorTest,
diff --git a/test/obmc_sad_test.cc b/test/obmc_sad_test.cc index dd6484f..17f3251 100644 --- a/test/obmc_sad_test.cc +++ b/test/obmc_sad_test.cc
@@ -28,9 +28,9 @@ static const int kIterations = 1000; static const int kMaskMax = 64; -typedef unsigned int (*ObmcSadF)(const uint8_t *pre, int pre_stride, - const int32_t *wsrc, const int32_t *mask); -typedef libaom_test::FuncParam<ObmcSadF> TestFuncs; +using ObmcSadF = unsigned int (*)(const uint8_t *pre, int pre_stride, + const int32_t *wsrc, const int32_t *mask); +using TestFuncs = libaom_test::FuncParam<ObmcSadF>; //////////////////////////////////////////////////////////////////////////////// // 8 bit
diff --git a/test/obmc_variance_test.cc b/test/obmc_variance_test.cc index 39b816e..16482f7 100644 --- a/test/obmc_variance_test.cc +++ b/test/obmc_variance_test.cc
@@ -30,10 +30,10 @@ static const int kIterations = 1000; static const int kMaskMax = 64; -typedef unsigned int (*ObmcVarF)(const uint8_t *pre, int pre_stride, - const int32_t *wsrc, const int32_t *mask, - unsigned int *sse); -typedef libaom_test::FuncParam<ObmcVarF> TestFuncs; +using ObmcVarF = unsigned int (*)(const uint8_t *pre, int pre_stride, + const int32_t *wsrc, const int32_t *mask, + unsigned int *sse); +using TestFuncs = libaom_test::FuncParam<ObmcVarF>; //////////////////////////////////////////////////////////////////////////////// // 8 bit
diff --git a/test/pickrst_test.cc b/test/pickrst_test.cc index fd25b0c..98b686d 100644 --- a/test/pickrst_test.cc +++ b/test/pickrst_test.cc
@@ -29,7 +29,7 @@ namespace pickrst_test_lowbd { static const int kIterations = 100; -typedef int64_t (*lowbd_pixel_proj_error_func)( +using lowbd_pixel_proj_error_func = int64_t (*)( const uint8_t *src8, int width, int height, int src_stride, const uint8_t *dat8, int dat_stride, int32_t *flt0, int flt0_stride, int32_t *flt1, int flt1_stride, int xq[2], const sgr_params_type *params); @@ -38,7 +38,7 @@ // 8 bit //////////////////////////////////////////////////////////////////////////////// -typedef std::tuple<const lowbd_pixel_proj_error_func> PixelProjErrorTestParam; +using PixelProjErrorTestParam = std::tuple<const lowbd_pixel_proj_error_func>; class PixelProjErrorTest : public ::testing::TestWithParam<PixelProjErrorTestParam> { @@ -201,7 +201,7 @@ namespace pickrst_test_highbd { static const int kIterations = 100; -typedef int64_t (*highbd_pixel_proj_error_func)( +using highbd_pixel_proj_error_func = int64_t (*)( const uint8_t *src8, int width, int height, int src_stride, const uint8_t *dat8, int dat_stride, int32_t *flt0, int flt0_stride, int32_t *flt1, int flt1_stride, int xq[2], const sgr_params_type *params); @@ -210,7 +210,7 @@ // High bit-depth //////////////////////////////////////////////////////////////////////////////// -typedef std::tuple<const highbd_pixel_proj_error_func> PixelProjErrorTestParam; +using PixelProjErrorTestParam = std::tuple<const highbd_pixel_proj_error_func>; class PixelProjHighbdErrorTest : public ::testing::TestWithParam<PixelProjErrorTestParam> { @@ -379,15 +379,15 @@ namespace get_proj_subspace_test_lowbd { static const int kIterations = 100; -typedef void (*set_get_proj_subspace)(const uint8_t *src8, int width, - int height, int src_stride, - const uint8_t *dat8, int dat_stride, - int32_t *flt0, int flt0_stride, - int32_t *flt1, int flt1_stride, - int64_t H[2][2], int64_t C[2], - const sgr_params_type *params); +using set_get_proj_subspace = void (*)(const uint8_t *src8, int width, + int height, int src_stride, + const uint8_t *dat8, int dat_stride, + int32_t *flt0, int flt0_stride, + int32_t *flt1, int flt1_stride, + int64_t H[2][2], int64_t C[2], + const sgr_params_type *params); -typedef std::tuple<const set_get_proj_subspace> GetProjSubspaceTestParam; +using GetProjSubspaceTestParam = std::tuple<const set_get_proj_subspace>; class GetProjSubspaceTest : public ::testing::TestWithParam<GetProjSubspaceTestParam> { @@ -564,15 +564,15 @@ namespace get_proj_subspace_test_hbd { static const int kIterations = 100; -typedef void (*set_get_proj_subspace_hbd)(const uint8_t *src8, int width, - int height, int src_stride, - const uint8_t *dat8, int dat_stride, - int32_t *flt0, int flt0_stride, - int32_t *flt1, int flt1_stride, - int64_t H[2][2], int64_t C[2], - const sgr_params_type *params); +using set_get_proj_subspace_hbd = void (*)(const uint8_t *src8, int width, + int height, int src_stride, + const uint8_t *dat8, int dat_stride, + int32_t *flt0, int flt0_stride, + int32_t *flt1, int flt1_stride, + int64_t H[2][2], int64_t C[2], + const sgr_params_type *params); -typedef std::tuple<const set_get_proj_subspace_hbd> GetProjSubspaceHBDTestParam; +using GetProjSubspaceHBDTestParam = std::tuple<const set_get_proj_subspace_hbd>; class GetProjSubspaceTestHBD : public ::testing::TestWithParam<GetProjSubspaceHBDTestParam> {
diff --git a/test/quant_test.cc b/test/quant_test.cc index 120eae7..505d19d 100644 --- a/test/quant_test.cc +++ b/test/quant_test.cc
@@ -91,10 +91,10 @@ ::testing::Range(5, 9)); #if !CONFIG_REALTIME_ONLY -typedef struct { +struct QuantParam { const unsigned int min_q; const unsigned int max_q; -} QuantParam; +}; const QuantParam QuantTestParams[] = { { 0, 10 }, { 0, 60 }, { 20, 35 }, { 35, 50 }, { 50, 63 }
diff --git a/test/quantize_func_test.cc b/test/quantize_func_test.cc index 782f93c..64d9ce4 100644 --- a/test/quantize_func_test.cc +++ b/test/quantize_func_test.cc
@@ -43,9 +43,9 @@ const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, \ const int16_t *iscan -typedef void (*LPQuantizeFunc)(LP_QUANTIZE_PARAM_LIST); -typedef void (*QuantizeFunc)(QUAN_PARAM_LIST); -typedef void (*QuantizeFuncHbd)(QUAN_PARAM_LIST, int log_scale); +using LPQuantizeFunc = void (*)(LP_QUANTIZE_PARAM_LIST); +using QuantizeFunc = void (*)(QUAN_PARAM_LIST); +using QuantizeFuncHbd = void (*)(QUAN_PARAM_LIST, int log_scale); #undef LP_QUANTIZE_PARAM_LIST @@ -83,10 +83,10 @@ using QuantizeParam = tuple<FuncType, FuncType, TX_SIZE, QuantType, aom_bit_depth_t>; -typedef struct { +struct QuanTable { QUANTS quant; Dequants dequant; -} QuanTable; +}; const int kTestNum = 1000;
diff --git a/test/rt_end_to_end_test.cc b/test/rt_end_to_end_test.cc index dced915..9459aae 100644 --- a/test/rt_end_to_end_test.cc +++ b/test/rt_end_to_end_test.cc
@@ -33,40 +33,40 @@ std::unordered_map<int, std::unordered_map<int, double>>> kPsnrThreshold = { { "park_joy_90p_8_420.y4m", { { 5, { { 0, 34.0 }, { 3, 35.0 } } }, - { 6, { { 0, 34.0 }, { 3, 35.0 } } }, + { 6, { { 0, 33.8 }, { 3, 34.8 } } }, { 7, { { 0, 33.0 }, { 3, 34.0 } } }, { 8, { { 0, 33.0 }, { 3, 34.0 } } }, { 9, { { 0, 33.0 }, { 3, 34.0 } } }, { 10, { { 0, 33.0 }, { 3, 34.0 } } } } }, { "paris_352_288_30.y4m", { { 5, { { 0, 35.0 }, { 3, 35.0 } } }, - { 6, { { 0, 35.0 }, { 3, 35.0 } } }, + { 6, { { 0, 34.8 }, { 3, 34.8 } } }, { 7, { { 0, 34.0 }, { 3, 34.0 } } }, { 8, { { 0, 34.0 }, { 3, 35.0 } } }, { 9, { { 0, 34.0 }, { 3, 34.0 } } }, { 10, { { 0, 34.0 }, { 3, 34.0 } } } } }, { "niklas_1280_720_30.y4m", { { 5, { { 0, 32.0 }, { 3, 32.0 } } }, - { 6, { { 0, 32.0 }, { 3, 32.0 } } }, + { 6, { { 0, 31.8 }, { 3, 31.8 } } }, { 7, { { 0, 31.0 }, { 3, 31.0 } } }, { 8, { { 0, 31.0 }, { 3, 31.0 } } }, { 9, { { 0, 31.0 }, { 3, 31.0 } } }, { 10, { { 0, 31.0 }, { 3, 31.0 } } } } }, { "hantro_collage_w352h288_nv12.yuv", { { 5, { { 0, 32.0 }, { 3, 32.0 } } }, - { 6, { { 0, 32.0 }, { 3, 32.0 } } }, - { 7, { { 0, 32.0 }, { 3, 32.0 } } }, - { 8, { { 0, 32.0 }, { 3, 32.0 } } }, + { 6, { { 0, 31.8 }, { 3, 31.8 } } }, + { 7, { { 0, 31.8 }, { 3, 31.8 } } }, + { 8, { { 0, 31.8 }, { 3, 31.8 } } }, { 9, { { 0, 31.0 }, { 3, 31.0 } } }, { 10, { { 0, 31.0 }, { 3, 31.0 } } } } } }; -typedef struct { +struct TestVideoParam { const char *filename; unsigned int input_bit_depth; aom_img_fmt fmt; aom_bit_depth_t bit_depth; unsigned int profile; -} TestVideoParam; +}; std::ostream &operator<<(std::ostream &os, const TestVideoParam &test_arg) { return os << "TestVideoParam { filename:" << test_arg.filename
diff --git a/test/sad_test.cc b/test/sad_test.cc index af73b68..416f0ee 100644 --- a/test/sad_test.cc +++ b/test/sad_test.cc
@@ -26,41 +26,41 @@ #include "aom_mem/aom_mem.h" #include "aom_ports/mem.h" -typedef unsigned int (*SadMxNFunc)(const uint8_t *src_ptr, int src_stride, - const uint8_t *ref_ptr, int ref_stride); -typedef std::tuple<int, int, SadMxNFunc, int> SadMxNParam; +using SadMxNFunc = unsigned int (*)(const uint8_t *src_ptr, int src_stride, + const uint8_t *ref_ptr, int ref_stride); +using SadMxNParam = std::tuple<int, int, SadMxNFunc, int>; -typedef unsigned int (*SadSkipMxNFunc)(const uint8_t *src_ptr, int src_stride, - const uint8_t *ref_ptr, int ref_stride); -typedef std::tuple<int, int, SadSkipMxNFunc, int> SadSkipMxNParam; +using SadSkipMxNFunc = unsigned int (*)(const uint8_t *src_ptr, int src_stride, + const uint8_t *ref_ptr, int ref_stride); +using SadSkipMxNParam = std::tuple<int, int, SadSkipMxNFunc, int>; -typedef uint32_t (*SadMxNAvgFunc)(const uint8_t *src_ptr, int src_stride, - const uint8_t *ref_ptr, int ref_stride, - const uint8_t *second_pred); -typedef std::tuple<int, int, SadMxNAvgFunc, int> SadMxNAvgParam; +using SadMxNAvgFunc = uint32_t (*)(const uint8_t *src_ptr, int src_stride, + const uint8_t *ref_ptr, int ref_stride, + const uint8_t *second_pred); +using SadMxNAvgParam = std::tuple<int, int, SadMxNAvgFunc, int>; -typedef unsigned int (*DistWtdSadMxhFunc)(const uint8_t *src_ptr, - int src_stride, - const uint8_t *ref_ptr, - int ref_stride, int width, - int height); -typedef std::tuple<int, int, DistWtdSadMxhFunc, int> DistWtdSadMxhParam; +using DistWtdSadMxhFunc = unsigned int (*)(const uint8_t *src_ptr, + int src_stride, + const uint8_t *ref_ptr, + int ref_stride, int width, + int height); +using DistWtdSadMxhParam = std::tuple<int, int, DistWtdSadMxhFunc, int>; -typedef void (*SadMxNx4Func)(const uint8_t *src_ptr, int src_stride, - const uint8_t *const ref_ptr[], int ref_stride, - uint32_t *sad_array); -typedef std::tuple<int, int, SadMxNx4Func, int> SadMxNx4Param; +using SadMxNx4Func = void (*)(const uint8_t *src_ptr, int src_stride, + const uint8_t *const ref_ptr[], int ref_stride, + uint32_t *sad_array); +using SadMxNx4Param = std::tuple<int, int, SadMxNx4Func, int>; -typedef void (*SadSkipMxNx4Func)(const uint8_t *src_ptr, int src_stride, +using SadSkipMxNx4Func = void (*)(const uint8_t *src_ptr, int src_stride, + const uint8_t *const ref_ptr[], + int ref_stride, uint32_t *sad_array); +using SadSkipMxNx4Param = std::tuple<int, int, SadSkipMxNx4Func, int>; + +using SadMxNx4AvgFunc = void (*)(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_ptr[], int ref_stride, + const uint8_t *second_pred, uint32_t *sad_array); -typedef std::tuple<int, int, SadSkipMxNx4Func, int> SadSkipMxNx4Param; - -typedef void (*SadMxNx4AvgFunc)(const uint8_t *src_ptr, int src_stride, - const uint8_t *const ref_ptr[], int ref_stride, - const uint8_t *second_pred, - uint32_t *sad_array); -typedef std::tuple<int, int, SadMxNx4AvgFunc, int> SadMxNx4AvgParam; +using SadMxNx4AvgParam = std::tuple<int, int, SadMxNx4AvgFunc, int>; using libaom_test::ACMRandom;
diff --git a/test/selfguided_filter_test.cc b/test/selfguided_filter_test.cc index 7540036..4a2d814 100644 --- a/test/selfguided_filter_test.cc +++ b/test/selfguided_filter_test.cc
@@ -31,13 +31,13 @@ using std::make_tuple; using std::tuple; -typedef int (*SgrFunc)(const uint8_t *dat8, int width, int height, int stride, - int eps, const int *xqd, uint8_t *dst8, int dst_stride, - int32_t *tmpbuf, int bit_depth, int highbd); +using SgrFunc = int (*)(const uint8_t *dat8, int width, int height, int stride, + int eps, const int *xqd, uint8_t *dst8, int dst_stride, + int32_t *tmpbuf, int bit_depth, int highbd); // Test parameter list: // <tst_fun_> -typedef tuple<SgrFunc> FilterTestParam; +using FilterTestParam = tuple<SgrFunc>; class AV1SelfguidedFilterTest : public ::testing::TestWithParam<FilterTestParam> { @@ -232,7 +232,7 @@ #if CONFIG_AV1_HIGHBITDEPTH // Test parameter list: // <tst_fun_, bit_depth> -typedef tuple<SgrFunc, int> HighbdFilterTestParam; +using HighbdFilterTestParam = tuple<SgrFunc, int>; class AV1HighbdSelfguidedFilterTest : public ::testing::TestWithParam<HighbdFilterTestParam> {
diff --git a/test/simd_cmp_impl.inc b/test/simd_cmp_impl.inc index 0a9a195..a2373d7 100644 --- a/test/simd_cmp_impl.inc +++ b/test/simd_cmp_impl.inc
@@ -463,13 +463,13 @@ return c_v256_ssd_s16_sum(::c_v256_ssd_s16(c_v256_ssd_s16_init(), a, b)); } -typedef void (*fptr)(); +using fptr = void (*)(); -typedef struct { +struct mapping { const char *name; fptr ref; fptr simd; -} mapping; +}; #define MAP(name) \ { #name, reinterpret_cast < fptr>(c_##name), reinterpret_cast < fptr>(name) }
diff --git a/test/simd_impl.h b/test/simd_impl.h index 20737a0..cb5b2d6 100644 --- a/test/simd_impl.h +++ b/test/simd_impl.h
@@ -35,9 +35,9 @@ }; // Create one typedef for each function signature -#define TYPEDEF_SIMD(name) \ - typedef TestIntrinsic<std::tuple<uint32_t, uint32_t, const char *> > \ - ARCH_POSTFIX(name) +#define TYPEDEF_SIMD(name) \ + using ARCH_POSTFIX(name) = \ + TestIntrinsic<std::tuple<uint32_t, uint32_t, const char *> > TYPEDEF_SIMD(V64_U8); TYPEDEF_SIMD(V64_U16); @@ -87,17 +87,17 @@ TYPEDEF_SIMD(V64_V256); // Google Test allows up to 50 tests per case, so split the largest -typedef ARCH_POSTFIX(V64_V64) ARCH_POSTFIX(V64_V64_Part2); -typedef ARCH_POSTFIX(V64_V64V64) ARCH_POSTFIX(V64_V64V64_Part2); -typedef ARCH_POSTFIX(V128_V128) ARCH_POSTFIX(V128_V128_Part2); -typedef ARCH_POSTFIX(V128_V128) ARCH_POSTFIX(V128_V128_Part3); -typedef ARCH_POSTFIX(V128_V128) ARCH_POSTFIX(V128_V128_Part4); -typedef ARCH_POSTFIX(V128_V128V128) ARCH_POSTFIX(V128_V128V128_Part2); -typedef ARCH_POSTFIX(V256_V256) ARCH_POSTFIX(V256_V256_Part2); -typedef ARCH_POSTFIX(V256_V256) ARCH_POSTFIX(V256_V256_Part3); -typedef ARCH_POSTFIX(V256_V256) ARCH_POSTFIX(V256_V256_Part4); -typedef ARCH_POSTFIX(V256_V256) ARCH_POSTFIX(V256_V256_Part5); -typedef ARCH_POSTFIX(V256_V256V256) ARCH_POSTFIX(V256_V256V256_Part2); +using ARCH_POSTFIX(V64_V64_Part2) = ARCH_POSTFIX(V64_V64); +using ARCH_POSTFIX(V64_V64V64_Part2) = ARCH_POSTFIX(V64_V64V64); +using ARCH_POSTFIX(V128_V128_Part2) = ARCH_POSTFIX(V128_V128); +using ARCH_POSTFIX(V128_V128_Part3) = ARCH_POSTFIX(V128_V128); +using ARCH_POSTFIX(V128_V128_Part4) = ARCH_POSTFIX(V128_V128); +using ARCH_POSTFIX(V128_V128V128_Part2) = ARCH_POSTFIX(V128_V128V128); +using ARCH_POSTFIX(V256_V256_Part2) = ARCH_POSTFIX(V256_V256); +using ARCH_POSTFIX(V256_V256_Part3) = ARCH_POSTFIX(V256_V256); +using ARCH_POSTFIX(V256_V256_Part4) = ARCH_POSTFIX(V256_V256); +using ARCH_POSTFIX(V256_V256_Part5) = ARCH_POSTFIX(V256_V256); +using ARCH_POSTFIX(V256_V256V256_Part2) = ARCH_POSTFIX(V256_V256V256); // These functions are machine tuned located elsewhere template <typename c_ret, typename c_arg>
diff --git a/test/sse_sum_test.cc b/test/sse_sum_test.cc index 54fbeae..8b33cf1 100644 --- a/test/sse_sum_test.cc +++ b/test/sse_sum_test.cc
@@ -35,9 +35,9 @@ namespace { const int kNumIterations = 10000; -typedef uint64_t (*SSI16Func)(const int16_t *src, int src_stride, int width, - int height, int *sum); -typedef libaom_test::FuncParam<SSI16Func> TestFuncs; +using SSI16Func = uint64_t (*)(const int16_t *src, int src_stride, int width, + int height, int *sum); +using TestFuncs = libaom_test::FuncParam<SSI16Func>; class SumSSETest : public ::testing::TestWithParam<TestFuncs> { public:
diff --git a/test/subtract_test.cc b/test/subtract_test.cc index ca2b2a9..c478210 100644 --- a/test/subtract_test.cc +++ b/test/subtract_test.cc
@@ -24,10 +24,10 @@ #include "aom_mem/aom_mem.h" #include "aom_ports/mem.h" -typedef void (*SubtractFunc)(int rows, int cols, int16_t *diff_ptr, - ptrdiff_t diff_stride, const uint8_t *src_ptr, - ptrdiff_t src_stride, const uint8_t *pred_ptr, - ptrdiff_t pred_stride); +using SubtractFunc = void (*)(int rows, int cols, int16_t *diff_ptr, + ptrdiff_t diff_stride, const uint8_t *src_ptr, + ptrdiff_t src_stride, const uint8_t *pred_ptr, + ptrdiff_t pred_stride); namespace {
diff --git a/test/sum_squares_test.cc b/test/sum_squares_test.cc index 26d0361..09b3c05 100644 --- a/test/sum_squares_test.cc +++ b/test/sum_squares_test.cc
@@ -38,9 +38,9 @@ static const int16_t kInt13Max = (1 << 12) - 1; -typedef uint64_t (*SSI16Func)(const int16_t *src, int stride, int width, - int height); -typedef libaom_test::FuncParam<SSI16Func> TestFuncs; +using SSI16Func = uint64_t (*)(const int16_t *src, int stride, int width, + int height); +using TestFuncs = libaom_test::FuncParam<SSI16Func>; class SumSquaresTest : public ::testing::TestWithParam<TestFuncs> { public: @@ -191,8 +191,8 @@ // 1D version ////////////////////////////////////////////////////////////////////////////// -typedef uint64_t (*F1D)(const int16_t *src, uint32_t n); -typedef libaom_test::FuncParam<F1D> TestFuncs1D; +using F1D = uint64_t (*)(const int16_t *src, uint32_t n); +using TestFuncs1D = libaom_test::FuncParam<F1D>; class SumSquares1DTest : public FunctionEquivalenceTest<F1D> { protected: @@ -261,11 +261,11 @@ #endif // HAVE_SVE -typedef int64_t (*SSEFunc)(const uint8_t *a, int a_stride, const uint8_t *b, - int b_stride, int width, int height); -typedef libaom_test::FuncParam<SSEFunc> TestSSEFuncs; +using SSEFunc = int64_t (*)(const uint8_t *a, int a_stride, const uint8_t *b, + int b_stride, int width, int height); +using TestSSEFuncs = libaom_test::FuncParam<SSEFunc>; -typedef std::tuple<TestSSEFuncs, int> SSETestParam; +using SSETestParam = std::tuple<TestSSEFuncs, int>; class SSETest : public ::testing::TestWithParam<SSETestParam> { public: @@ -471,11 +471,11 @@ // get_blk sum squares test functions ////////////////////////////////////////////////////////////////////////////// -typedef void (*sse_sum_func)(const int16_t *data, int stride, int bw, int bh, - int *x_sum, int64_t *x2_sum); -typedef libaom_test::FuncParam<sse_sum_func> TestSSE_SumFuncs; +using sse_sum_func = void (*)(const int16_t *data, int stride, int bw, int bh, + int *x_sum, int64_t *x2_sum); +using TestSSE_SumFuncs = libaom_test::FuncParam<sse_sum_func>; -typedef std::tuple<TestSSE_SumFuncs, TX_SIZE> SSE_SumTestParam; +using SSE_SumTestParam = std::tuple<TestSSE_SumFuncs, TX_SIZE>; class SSE_Sum_Test : public ::testing::TestWithParam<SSE_SumTestParam> { public: @@ -631,8 +631,8 @@ // 2D Variance test functions ////////////////////////////////////////////////////////////////////////////// -typedef uint64_t (*Var2DFunc)(uint8_t *src, int stride, int width, int height); -typedef libaom_test::FuncParam<Var2DFunc> TestFuncVar2D; +using Var2DFunc = uint64_t (*)(uint8_t *src, int stride, int width, int height); +using TestFuncVar2D = libaom_test::FuncParam<Var2DFunc>; const uint16_t test_block_size[2] = { 128, 256 };
diff --git a/test/svc_datarate_test.cc b/test/svc_datarate_test.cc index 762a36b..0df6782 100644 --- a/test/svc_datarate_test.cc +++ b/test/svc_datarate_test.cc
@@ -1225,6 +1225,46 @@ #endif } + virtual void BasicRateTargetingSVC3TL1SLQvgaLowFramerateTest() { + SetUpCbr(); + cfg_.g_error_resilient = 0; + cfg_.g_threads = 2; + cfg_.kf_max_dist = 30; + cfg_.kf_min_dist = 30; + cfg_.rc_dropframe_thresh = 0; + cfg_.rc_min_quantizer = 2; + cfg_.rc_max_quantizer = 50; + + ::libaom_test::I420VideoSource video("desktop1.320_180.yuv", 320, 180, 10, + 1, 0, 800); + const int bitrate_array[2] = { 50, 200 }; + cfg_.rc_target_bitrate = bitrate_array[GET_PARAM(4)]; + ResetModel(); + tile_columns_ = 1; + SetTargetBitratesFor1SL3TL(); + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + } + + virtual void BasicRateTargetingSVC3TL1SLQvgaHighBitrateLowFramerateTest() { + SetUpCbr(); + cfg_.g_error_resilient = 0; + cfg_.g_threads = 2; + cfg_.kf_max_dist = 30; + cfg_.kf_min_dist = 30; + cfg_.rc_dropframe_thresh = 0; + cfg_.rc_min_quantizer = 2; + cfg_.rc_max_quantizer = 50; + + ::libaom_test::I420VideoSource video("desktop1.320_180.yuv", 320, 180, 10, + 1, 0, 800); + const int bitrate_array[2] = { 500, 1000 }; + cfg_.rc_target_bitrate = bitrate_array[GET_PARAM(4)]; + ResetModel(); + tile_columns_ = 1; + SetTargetBitratesFor1SL3TL(); + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + } + virtual void SetFrameQpSVC3TL1SLTest() { SetUpCbr(); cfg_.g_error_resilient = 1; @@ -2347,6 +2387,19 @@ BasicRateTargetingSVC3TL1SLTest(); } +// Check basic rate targeting for CBR, for 3 temporal layers, 1 spatial, +// QVGA, low framerate. +TEST_P(DatarateTestSVC, BasicRateTargetingSVC3TL1SLQvgaLowFrameate) { + BasicRateTargetingSVC3TL1SLQvgaLowFramerateTest(); +} + +// Check basic rate targeting for CBR, for 3 temporal layers, 1 spatial, +// QVGA, high bitrate and low framerate. +TEST_P(DatarateTestSVC, + BasicRateTargetingSVC3TL1SLQvgaHighBitrateLowFramerateTest) { + BasicRateTargetingSVC3TL1SLQvgaHighBitrateLowFramerateTest(); +} + TEST_P(DatarateTestSVC, SetFrameQpSVC3TL1SL) { SetFrameQpSVC3TL1SLTest(); } TEST_P(DatarateTestSVC, SetFrameQpSVC3TL3SL) { SetFrameQpSVC3TL3SLTest(); } @@ -2723,10 +2776,18 @@ EXPECT_EQ(aom_codec_destroy(&enc), AOM_CODEC_OK); } +// Speed 6 takes too long on valgrind, so do only 1 bitrate and one aq_mode. +#ifdef AOM_VALGRIND_BUILD AV1_INSTANTIATE_TEST_SUITE(DatarateTestSVC, ::testing::Values(::libaom_test::kRealTime), - ::testing::Range(7, 12), ::testing::Values(0, 3), + ::testing::Range(6, 12), ::testing::Values(3), + ::testing::Values(1)); +#else // AOM_VALGRIND_BUILD +AV1_INSTANTIATE_TEST_SUITE(DatarateTestSVC, + ::testing::Values(::libaom_test::kRealTime), + ::testing::Range(6, 12), ::testing::Values(0, 3), ::testing::Values(0, 1)); +#endif } // namespace } // namespace datarate_test
diff --git a/test/temporal_filter_test.cc b/test/temporal_filter_test.cc index cda06c5..c9b4224 100644 --- a/test/temporal_filter_test.cc +++ b/test/temporal_filter_test.cc
@@ -37,22 +37,22 @@ #if !CONFIG_REALTIME_ONLY namespace { -typedef enum { +enum ColorFormat { I400, // Monochrome I420, // 4:2:0 I422, // 4:2:2 I444, // 4:4:4 -} ColorFormat; +}; static const char *color_fmt_str[] = { "I400", "I420", "I422", "I444" }; -typedef void (*TemporalFilterFunc)( +using TemporalFilterFunc = void (*)( const YV12_BUFFER_CONFIG *frame_to_filter, const MACROBLOCKD *mbd, const BLOCK_SIZE block_size, const int mb_row, const int mb_col, const int num_planes, const double *noise_level, const MV *subblock_mvs, const int *subblock_mses, const int q_factor, const int filter_strength, int tf_wgt_calc_lvl, const uint8_t *pred, uint32_t *accum, uint16_t *count); -typedef libaom_test::FuncParam<TemporalFilterFunc> TemporalFilterFuncParam; +using TemporalFilterFuncParam = libaom_test::FuncParam<TemporalFilterFunc>; -typedef std::tuple<TemporalFilterFuncParam, int> TemporalFilterWithParam; +using TemporalFilterWithParam = std::tuple<TemporalFilterFuncParam, int>; class TemporalFilterTest : public ::testing::TestWithParam<TemporalFilterWithParam> { @@ -131,9 +131,10 @@ ColorFormat color_fmt) { aom_usec_timer ref_timer, test_timer; const BLOCK_SIZE block_size = TF_BLOCK_SIZE; - static_assert(block_size == BLOCK_32X32, ""); - const int width = 32; - const int height = 32; + static_assert(block_size == BLOCK_64X64, ""); + const int width = 64; + const int height = 64; + const int pels = width * height; int num_planes = MAX_MB_PLANE; int subsampling_x = 0; int subsampling_y = 0; @@ -165,18 +166,23 @@ } double sigma[MAX_MB_PLANE] = { 2.1002103677063437, 2.1002103677063437, 2.1002103677063437 }; - DECLARE_ALIGNED(16, unsigned int, accumulator_ref[1024 * 3]); - DECLARE_ALIGNED(16, uint16_t, count_ref[1024 * 3]); - memset(accumulator_ref, 0, 1024 * 3 * sizeof(accumulator_ref[0])); - memset(count_ref, 0, 1024 * 3 * sizeof(count_ref[0])); - DECLARE_ALIGNED(16, unsigned int, accumulator_mod[1024 * 3]); - DECLARE_ALIGNED(16, uint16_t, count_mod[1024 * 3]); - memset(accumulator_mod, 0, 1024 * 3 * sizeof(accumulator_mod[0])); - memset(count_mod, 0, 1024 * 3 * sizeof(count_mod[0])); + DECLARE_ALIGNED(16, unsigned int, accumulator_ref[pels * 3]); + DECLARE_ALIGNED(16, uint16_t, count_ref[pels * 3]); + memset(accumulator_ref, 0, pels * 3 * sizeof(accumulator_ref[0])); + memset(count_ref, 0, pels * 3 * sizeof(count_ref[0])); + DECLARE_ALIGNED(16, unsigned int, accumulator_mod[pels * 3]); + DECLARE_ALIGNED(16, uint16_t, count_mod[pels * 3]); + memset(accumulator_mod, 0, pels * 3 * sizeof(accumulator_mod[0])); + memset(count_mod, 0, pels * 3 * sizeof(count_mod[0])); - static_assert(width == 32 && height == 32, ""); - const MV subblock_mvs[4] = { { 0, 0 }, { 5, 5 }, { 7, 8 }, { 2, 10 } }; - const int subblock_mses[4] = { 15, 16, 17, 18 }; + static_assert(width == 64 && height == 64, ""); + const MV subblock_mvs[NUM_16X16] = { + { 0, 0 }, { 5, 5 }, { 7, 8 }, { 2, 10 }, { 0, 0 }, { 5, 5 }, + { 7, 8 }, { 2, 10 }, { 0, 0 }, { 5, 5 }, { 7, 8 }, { 2, 10 }, + { 0, 0 }, { 5, 5 }, { 7, 8 }, { 2, 10 } + }; + const int subblock_mses[NUM_16X16] = { 15, 16, 17, 18, 15, 16, 17, 18, + 15, 16, 17, 18, 15, 16, 17, 18 }; const int q_factor = 12; const int filter_strength = 5; const int mb_row = 0; @@ -190,10 +196,10 @@ frame_to_filter->heights[PLANE_TYPE_UV] = height >> subsampling_y; frame_to_filter->strides[PLANE_TYPE_Y] = stride; frame_to_filter->strides[PLANE_TYPE_UV] = stride >> subsampling_x; - DECLARE_ALIGNED(16, uint8_t, src[1024 * 3]); + DECLARE_ALIGNED(16, uint8_t, src[pels * 3]); frame_to_filter->buffer_alloc = src; frame_to_filter->flags = 0; // Only support low bit-depth test. - memcpy(src, src1_, 1024 * 3 * sizeof(uint8_t)); + memcpy(src, src1_, pels * 3 * sizeof(uint8_t)); std::unique_ptr<MACROBLOCKD> mbd(new (std::nothrow) MACROBLOCKD); ASSERT_NE(mbd, nullptr); @@ -300,23 +306,26 @@ Values(0, 1))); #endif // HAVE_SSE2 -#if HAVE_NEON -TemporalFilterFuncParam temporal_filter_test_neon[] = { TemporalFilterFuncParam( - &av1_apply_temporal_filter_c, &av1_apply_temporal_filter_neon) }; -INSTANTIATE_TEST_SUITE_P(NEON, TemporalFilterTest, - Combine(ValuesIn(temporal_filter_test_neon), - Values(0, 1))); -#endif // HAVE_NEON +// av1_apply_temporal_filter_c works on 64x64 TF block now, the SIMD function +// needs to be updated. +// #if HAVE_NEON +// TemporalFilterFuncParam temporal_filter_test_neon[] = { +// TemporalFilterFuncParam( +// &av1_apply_temporal_filter_c, &av1_apply_temporal_filter_neon) }; +// INSTANTIATE_TEST_SUITE_P(NEON, TemporalFilterTest, +// Combine(ValuesIn(temporal_filter_test_neon), +// Values(0, 1))); +// #endif // HAVE_NEON -#if HAVE_NEON_DOTPROD -TemporalFilterFuncParam temporal_filter_test_neon_dotprod[] = { - TemporalFilterFuncParam(&av1_apply_temporal_filter_c, - &av1_apply_temporal_filter_neon_dotprod) -}; -INSTANTIATE_TEST_SUITE_P(NEON_DOTPROD, TemporalFilterTest, - Combine(ValuesIn(temporal_filter_test_neon_dotprod), - Values(0, 1))); -#endif // HAVE_NEON_DOTPROD +// #if HAVE_NEON_DOTPROD +// TemporalFilterFuncParam temporal_filter_test_neon_dotprod[] = { +// TemporalFilterFuncParam(&av1_apply_temporal_filter_c, +// &av1_apply_temporal_filter_neon_dotprod) +// }; +// INSTANTIATE_TEST_SUITE_P(NEON_DOTPROD, TemporalFilterTest, +// Combine(ValuesIn(temporal_filter_test_neon_dotprod), +// Values(0, 1))); +// #endif // HAVE_NEON_DOTPROD #if HAVE_AVX2 || HAVE_NEON // Width and height for which av1_estimate_noise_from_single_plane() will be @@ -325,11 +334,11 @@ const int kHeights[] = { 2160, 1080, 720, 600, 480, 240, 237 }; #endif // HAVE_AVX2 || HAVE_NEON -typedef double (*EstimateNoiseFunc)(const uint8_t *src, int height, int width, - int stride, int edge_thresh); +using EstimateNoiseFunc = double (*)(const uint8_t *src, int height, int width, + int stride, int edge_thresh); -typedef std::tuple<EstimateNoiseFunc, EstimateNoiseFunc, int, int> - EstimateNoiseWithParam; +using EstimateNoiseWithParam = + std::tuple<EstimateNoiseFunc, EstimateNoiseFunc, int, int>; class EstimateNoiseTest : public ::testing::TestWithParam<EstimateNoiseWithParam> { @@ -423,16 +432,16 @@ #if CONFIG_AV1_HIGHBITDEPTH -typedef void (*HBDTemporalFilterFunc)( +using HBDTemporalFilterFunc = void (*)( const YV12_BUFFER_CONFIG *frame_to_filter, const MACROBLOCKD *mbd, const BLOCK_SIZE block_size, const int mb_row, const int mb_col, const int num_planes, const double *noise_level, const MV *subblock_mvs, const int *subblock_mses, const int q_factor, const int filter_strength, int tf_wgt_calc_lvl, const uint8_t *pred, uint32_t *accum, uint16_t *count); -typedef libaom_test::FuncParam<HBDTemporalFilterFunc> - HBDTemporalFilterFuncParam; +using HBDTemporalFilterFuncParam = + libaom_test::FuncParam<HBDTemporalFilterFunc>; -typedef std::tuple<HBDTemporalFilterFuncParam, int> HBDTemporalFilterWithParam; +using HBDTemporalFilterWithParam = std::tuple<HBDTemporalFilterFuncParam, int>; class HBDTemporalFilterTest : public ::testing::TestWithParam<HBDTemporalFilterWithParam> { @@ -514,9 +523,9 @@ ColorFormat color_fmt) { aom_usec_timer ref_timer, test_timer; const BLOCK_SIZE block_size = TF_BLOCK_SIZE; - static_assert(block_size == BLOCK_32X32, ""); - const int width = 32; - const int height = 32; + static_assert(block_size == BLOCK_64X64, ""); + const int width = 64; + const int height = 64; int num_planes = MAX_MB_PLANE; int subsampling_x = 0; int subsampling_y = 0; @@ -557,7 +566,7 @@ memset(accumulator_mod, 0, 1024 * 3 * sizeof(accumulator_mod[0])); memset(count_mod, 0, 1024 * 3 * sizeof(count_mod[0])); - static_assert(width == 32 && height == 32, ""); + static_assert(width == 64 && height == 64, ""); const MV subblock_mvs[4] = { { 0, 0 }, { 5, 5 }, { 7, 8 }, { 2, 10 } }; const int subblock_mses[4] = { 15, 16, 17, 18 }; const int q_factor = 12; @@ -667,34 +676,39 @@ RunTest(1, 100000, 10, I422); RunTest(1, 100000, 10, I444); } -#if HAVE_SSE2 -HBDTemporalFilterFuncParam HBDtemporal_filter_test_sse2[] = { - HBDTemporalFilterFuncParam(&av1_highbd_apply_temporal_filter_c, - &av1_highbd_apply_temporal_filter_sse2) -}; -INSTANTIATE_TEST_SUITE_P(SSE2, HBDTemporalFilterTest, - Combine(ValuesIn(HBDtemporal_filter_test_sse2), - Values(0, 1))); -#endif // HAVE_SSE2 -#if HAVE_AVX2 -HBDTemporalFilterFuncParam HBDtemporal_filter_test_avx2[] = { - HBDTemporalFilterFuncParam(&av1_highbd_apply_temporal_filter_c, - &av1_highbd_apply_temporal_filter_avx2) -}; -INSTANTIATE_TEST_SUITE_P(AVX2, HBDTemporalFilterTest, - Combine(ValuesIn(HBDtemporal_filter_test_avx2), - Values(0, 1))); -#endif // HAVE_AVX2 -#if HAVE_NEON -HBDTemporalFilterFuncParam HBDtemporal_filter_test_neon[] = { - HBDTemporalFilterFuncParam(&av1_highbd_apply_temporal_filter_c, - &av1_highbd_apply_temporal_filter_neon) -}; -INSTANTIATE_TEST_SUITE_P(NEON, HBDTemporalFilterTest, - Combine(ValuesIn(HBDtemporal_filter_test_neon), - Values(0, 1))); -#endif // HAVE_NEON +// av1_apply_temporal_filter_c works on 64x64 TF block now, the SIMD function +// needs to be updated. +// #if HAVE_SSE2 +// HBDTemporalFilterFuncParam HBDtemporal_filter_test_sse2[] = { +// HBDTemporalFilterFuncParam(&av1_highbd_apply_temporal_filter_c, +// &av1_highbd_apply_temporal_filter_sse2) +//}; +// INSTANTIATE_TEST_SUITE_P(SSE2, HBDTemporalFilterTest, +// Combine(ValuesIn(HBDtemporal_filter_test_sse2), +// Values(0, 1))); +// #endif // HAVE_SSE2 +// #if HAVE_AVX2 +// HBDTemporalFilterFuncParam HBDtemporal_filter_test_avx2[] = { +// HBDTemporalFilterFuncParam(&av1_highbd_apply_temporal_filter_c, +// &av1_highbd_apply_temporal_filter_avx2) +//}; +// INSTANTIATE_TEST_SUITE_P(AVX2, HBDTemporalFilterTest, +// Combine(ValuesIn(HBDtemporal_filter_test_avx2), +// Values(0, 1))); +// #endif // HAVE_AVX2 + +// av1_apply_temporal_filter_c works on 64x64 TF block now, the SIMD function +// needs to be updated. +// #if HAVE_NEON +// HBDTemporalFilterFuncParam HBDtemporal_filter_test_neon[] = { +// HBDTemporalFilterFuncParam(&av1_highbd_apply_temporal_filter_c, +// &av1_highbd_apply_temporal_filter_neon) +//}; +// INSTANTIATE_TEST_SUITE_P(NEON, HBDTemporalFilterTest, +// Combine(ValuesIn(HBDtemporal_filter_test_neon), +// Values(0, 1))); +// #endif // HAVE_NEON using HBDEstimateNoiseFunc = double (*)(const uint16_t *src, int height, int width, int stride, int bit_depth,
diff --git a/test/test-data.sha1 b/test/test-data.sha1 index 9a9453d..e302ee2 100644 --- a/test/test-data.sha1 +++ b/test/test-data.sha1
@@ -16,7 +16,7 @@ 91a5bedeb4832c1c2900736cc0f644bb63971bbc *invalid-oss-fuzz-10227.ivf b055f06b9a95aaa5697fa26497b592a47843a7c8 *invalid-oss-fuzz-10227.ivf.res b2d0a29a65879436bf483d04865faca7d11cc2ee *invalid-oss-fuzz-10389.ivf -f4ce175af1d871ed1603c8936f6b78e968f93c85 *invalid-oss-fuzz-10389.ivf.res.4 +ddd9f38cd81a88e53ef71879adbaa40bac02e634 *invalid-oss-fuzz-10389.ivf.res.5 11df8e9a068669c678097d460b63609d3da73828 *invalid-oss-fuzz-10555.ivf b055f06b9a95aaa5697fa26497b592a47843a7c8 *invalid-oss-fuzz-10555.ivf.res cf5945085fe85456a1f74bf4cc7998b88b3f4b62 *invalid-oss-fuzz-10705.ivf @@ -577,3 +577,5 @@ 79ae6f87ddde75a8bfbc47251a6aefcc7c3f996b *av1.rawfile 5605058cf45cc959c34c8407fad9b9ff2ec7ca28 *SDR_Animal_oqo7.y4m 234e498cc29b8f4bb4f0fbaeda867fbefbddcaa8 *SDR_Health_rtd0_720p.y4m +514c8334a883f1b156433c3e56f42a0330231ace *SDR_Dance_ldsn_1080p.y4m +e4c7ad520f69dc09227eb999540ff518964d30f6 *SDR_Sports_6mug_608p_30fps_90f.yuv
diff --git a/test/test.cmake b/test/test.cmake index 273a41e..ddef74d 100644 --- a/test/test.cmake +++ b/test/test.cmake
@@ -51,6 +51,7 @@ add_to_libaom_test_srcs(AOM_UNIT_TEST_COMMON_SOURCES) list(APPEND AOM_UNIT_TEST_DECODER_SOURCES "${AOM_ROOT}/test/decode_api_test.cc" + "${AOM_ROOT}/test/decode_frame_size_limit_test.cc" "${AOM_ROOT}/test/decode_scalability_test.cc" "${AOM_ROOT}/test/external_frame_buffer_test.cc" "${AOM_ROOT}/test/invalid_file_test.cc" @@ -77,6 +78,7 @@ "${AOM_ROOT}/test/encode_test_driver.cc" "${AOM_ROOT}/test/encode_test_driver.h" "${AOM_ROOT}/test/end_to_end_psnr_test.cc" + "${AOM_ROOT}/test/ext_ratectrl_test.cc" "${AOM_ROOT}/test/forced_max_frame_width_height_test.cc" "${AOM_ROOT}/test/force_key_frame_test.cc" "${AOM_ROOT}/test/gf_pyr_height_test.cc" @@ -129,6 +131,7 @@ "${AOM_ROOT}/test/deltaq_mode_test.cc" "${AOM_ROOT}/test/dropframe_encode_test.cc" "${AOM_ROOT}/test/end_to_end_psnr_test.cc" + "${AOM_ROOT}/test/ext_ratectrl_test.cc" "${AOM_ROOT}/test/force_key_frame_test.cc" "${AOM_ROOT}/test/gf_pyr_height_test.cc" "${AOM_ROOT}/test/horz_superres_test.cc" @@ -235,6 +238,7 @@ "${AOM_ROOT}/test/subtract_test.cc" "${AOM_ROOT}/test/sum_squares_test.cc" "${AOM_ROOT}/test/sse_sum_test.cc" + "${AOM_ROOT}/test/use_fixed_qp_offsets_test.cc" "${AOM_ROOT}/test/variance_test.cc" "${AOM_ROOT}/test/warp_filter_test.cc" "${AOM_ROOT}/test/warp_filter_test_util.cc"
diff --git a/test/test_data_util.cmake b/test/test_data_util.cmake index 04dc2aa..10a67fe 100644 --- a/test/test_data_util.cmake +++ b/test/test_data_util.cmake
@@ -44,7 +44,9 @@ "crowd_run_360p_10_150f.y4m" "wikipedia_420_360p_60f.y4m" "SDR_Animal_oqo7.y4m" - "SDR_Health_rtd0_720p.y4m") + "SDR_Health_rtd0_720p.y4m" + "SDR_Dance_ldsn_1080p.y4m" + "SDR_Sports_6mug_608p_30fps_90f.yuv") if(ENABLE_DECODE_PERF_TESTS AND CONFIG_AV1_ENCODER) list(APPEND AOM_TEST_DATA_FILE_NAMES "niklas_1280_720_30.yuv") @@ -551,7 +553,7 @@ "invalid-oss-fuzz-10227.ivf" "invalid-oss-fuzz-10227.ivf.res" "invalid-oss-fuzz-10389.ivf" - "invalid-oss-fuzz-10389.ivf.res.4" + "invalid-oss-fuzz-10389.ivf.res.5" "invalid-oss-fuzz-10555.ivf" "invalid-oss-fuzz-10555.ivf.res" "invalid-oss-fuzz-10705.ivf"
diff --git a/test/test_intra_pred_speed.cc b/test/test_intra_pred_speed.cc index 65468d9..80e1719 100644 --- a/test/test_intra_pred_speed.cc +++ b/test/test_intra_pred_speed.cc
@@ -35,8 +35,8 @@ // 0: Generate MD5 array as required #define APPLY_UNIT_TESTS 1 -typedef void (*AvxPredFunc)(uint8_t *dst, ptrdiff_t y_stride, - const uint8_t *above, const uint8_t *left); +using AvxPredFunc = void (*)(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, const uint8_t *left); const int kBPS = 64; const int kTotalPixels = kBPS * kBPS; @@ -87,7 +87,7 @@ // ----------------------------------------------------------------------------- // Low Bittdepth -typedef IntraPredTestMem<uint8_t> Av1IntraPredTestMem; +using Av1IntraPredTestMem = IntraPredTestMem<uint8_t>; static const char *const kTxSizeStrings[TX_SIZES_ALL] = { "4X4", "8X8", "16X16", "32X32", "64X64", "4X8", "8X4", @@ -1007,11 +1007,11 @@ // High Bitdepth namespace { -typedef void (*AvxHighbdPredFunc)(uint16_t *dst, ptrdiff_t y_stride, - const uint16_t *above, const uint16_t *left, - int bd); +using AvxHighbdPredFunc = void (*)(uint16_t *dst, ptrdiff_t y_stride, + const uint16_t *above, const uint16_t *left, + int bd); -typedef IntraPredTestMem<uint16_t> Av1HighbdIntraPredTestMem; +using Av1HighbdIntraPredTestMem = IntraPredTestMem<uint16_t>; void TestHighbdIntraPred(TX_SIZE tx_size, AvxHighbdPredFunc const *pred_funcs, const char *const signatures[]) {
diff --git a/test/test_vector_test.cc b/test/test_vector_test.cc index b521e93..6c132b6 100644 --- a/test/test_vector_test.cc +++ b/test/test_vector_test.cc
@@ -34,7 +34,7 @@ const int kFileName = 1; const int kRowMT = 2; -typedef std::tuple<int, const char *, int> DecodeParam; +using DecodeParam = std::tuple<int, const char *, int>; class TestVectorTest : public ::libaom_test::DecoderTest, public ::libaom_test::CodecTestWithParam<DecodeParam> {
diff --git a/test/tile_config_test.cc b/test/tile_config_test.cc index ed3216f..ff7a460 100644 --- a/test/tile_config_test.cc +++ b/test/tile_config_test.cc
@@ -19,14 +19,14 @@ #include "test/util.h" namespace { -typedef struct { +struct uniformTileConfigParam { // Superblock size const unsigned int sb_size; // log2(number of tile rows) const unsigned int tile_rows; // log2(number of tile columns) const unsigned int tile_cols; -} uniformTileConfigParam; +}; const libaom_test::TestMode kTestModeParams[] = #if CONFIG_REALTIME_ONLY @@ -42,7 +42,7 @@ { 64, 2, 2 }, { 64, 3, 3 }, { 64, 4, 4 } }; -typedef struct { +struct nonUniformTileConfigParam { // Superblock size const unsigned int sb_size; // number of tile widths @@ -53,7 +53,7 @@ const unsigned int tile_height_count; // list of tile heights int tile_heights[AOM_MAX_TILE_ROWS]; -} nonUniformTileConfigParam; +}; const nonUniformTileConfigParam nonUniformTileConfigParams[] = { { 64, 1, { 3 }, 1, { 3 } }, { 64, 2, { 1, 2 }, 2, { 1, 2 } }, @@ -271,14 +271,14 @@ ::testing::ValuesIn(nonUniformTileConfigParams), ::testing::Values(AOM_Q, AOM_VBR, AOM_CBR, AOM_CQ)); -typedef struct { +struct TileGroupConfigParams { // Number of tile groups to set. const int num_tg; // Number of tile rows to set const int num_tile_rows; // Number of tile columns to set const int num_tile_cols; -} TileGroupConfigParams; +}; static const TileGroupConfigParams tileGroupTestParams[] = { { 5, 4, 4 }, { 3, 3, 3 }, { 5, 3, 3 }, { 7, 5, 5 }, { 7, 3, 3 }, { 7, 4, 4 }
diff --git a/test/use_fixed_qp_offsets_test.cc b/test/use_fixed_qp_offsets_test.cc new file mode 100644 index 0000000..1131f8a --- /dev/null +++ b/test/use_fixed_qp_offsets_test.cc
@@ -0,0 +1,84 @@ +/* + * Copyright (c) 2026, Alliance for Open Media. All rights reserved. + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#include "config/aom_config.h" + +#include "gtest/gtest.h" +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/i420_video_source.h" +#include "test/util.h" + +namespace { + +const ::libaom_test::TestMode kTestMode[] = +#if CONFIG_REALTIME_ONLY + { ::libaom_test::kRealTime }; +#else + { ::libaom_test::kRealTime, ::libaom_test::kOnePassGood }; +#endif + +const int kUseFixedQPOffsetsMode[] = { 1, 2 }; + +class UseFixedQPOffsetsTest + : public ::libaom_test::CodecTestWith2Params<libaom_test::TestMode, int>, + public ::libaom_test::EncoderTest { + protected: + UseFixedQPOffsetsTest() : EncoderTest(GET_PARAM(0)) {} + ~UseFixedQPOffsetsTest() override = default; + + void SetUp() override { + InitializeConfig(GET_PARAM(1)); + cfg_.kf_max_dist = 9999; + cfg_.rc_end_usage = AOM_Q; + cfg_.use_fixed_qp_offsets = GET_PARAM(2); + } + + void PreEncodeFrameHook(::libaom_test::VideoSource *video, + ::libaom_test::Encoder *encoder) override { + if (video->frame() == 0) { + encoder->Control(AOME_SET_CPUUSED, 6); + encoder->Control(AOME_SET_CQ_LEVEL, frame_qp_); + } + } + + void PostEncodeFrameHook(::libaom_test::Encoder *encoder) override { + int qp = 0; + encoder->Control(AOME_GET_LAST_QUANTIZER_64, &qp); + + // If a call to encoder->EncodeFrame() results in a last QP of 0, + // interpret as the frame being read into the lookahead buffer. + if (qp == 0) return; + + if (cfg_.use_fixed_qp_offsets == 2) { + // Setting use_fixed_qp_offsets = 2 means every frame should use the same + // QP + ASSERT_EQ(qp, frame_qp_); + } else { + ASSERT_LE(qp, frame_qp_); + } + } + + void DoTest() { + ::libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, + 288, 30, 1, 0, 33); + frame_qp_ = 35; + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + } + + int frame_qp_; +}; + +TEST_P(UseFixedQPOffsetsTest, TestQPOffsets) { DoTest(); } + +AV1_INSTANTIATE_TEST_SUITE(UseFixedQPOffsetsTest, + ::testing::ValuesIn(kTestMode), + ::testing::ValuesIn(kUseFixedQPOffsetsMode)); +} // namespace
diff --git a/test/variance_test.cc b/test/variance_test.cc index 1999cc9..ff5caeb 100644 --- a/test/variance_test.cc +++ b/test/variance_test.cc
@@ -30,39 +30,40 @@ namespace { -typedef uint64_t (*MseWxH16bitFunc)(uint8_t *dst, int dstride, uint16_t *src, - int sstride, int w, int h); -typedef uint64_t (*Mse16xH16bitFunc)(uint8_t *dst, int dstride, uint16_t *src, - int w, int h); -typedef unsigned int (*VarianceMxNFunc)(const uint8_t *a, int a_stride, - const uint8_t *b, int b_stride, - unsigned int *sse); -typedef void (*GetSseSum8x8QuadFunc)(const uint8_t *a, int a_stride, - const uint8_t *b, int b_stride, - uint32_t *sse8x8, int *sum8x8, - unsigned int *tot_sse, int *tot_sum, - uint32_t *var8x8); -typedef void (*GetSseSum16x16DualFunc)(const uint8_t *a, int a_stride, - const uint8_t *b, int b_stride, - uint32_t *sse16x16, - unsigned int *tot_sse, int *tot_sum, - uint32_t *var16x16); -typedef unsigned int (*SubpixVarMxNFunc)(const uint8_t *a, int a_stride, - int xoffset, int yoffset, +using MseWxH16bitFunc = uint64_t (*)(uint8_t *dst, int dstride, uint16_t *src, + int sstride, int w, int h); +using Mse16xH16bitFunc = uint64_t (*)(uint8_t *dst, int dstride, uint16_t *src, + int w, int h); +using VarianceMxNFunc = unsigned int (*)(const uint8_t *a, int a_stride, const uint8_t *b, int b_stride, unsigned int *sse); -typedef unsigned int (*SubpixAvgVarMxNFunc)(const uint8_t *a, int a_stride, - int xoffset, int yoffset, - const uint8_t *b, int b_stride, - uint32_t *sse, - const uint8_t *second_pred); -typedef unsigned int (*SumOfSquaresFunction)(const int16_t *src); +using GetSseSum8x8QuadFunc = void (*)(const uint8_t *a, int a_stride, + const uint8_t *b, int b_stride, + uint32_t *sse8x8, int *sum8x8, + unsigned int *tot_sse, int *tot_sum, + uint32_t *var8x8); +using GetSseSum16x16DualFunc = void (*)(const uint8_t *a, int a_stride, + const uint8_t *b, int b_stride, + uint32_t *sse16x16, + unsigned int *tot_sse, int *tot_sum, + uint32_t *var16x16); +using SubpixVarMxNFunc = unsigned int (*)(const uint8_t *a, int a_stride, + int xoffset, int yoffset, + const uint8_t *b, int b_stride, + unsigned int *sse); +using SubpixAvgVarMxNFunc = unsigned int (*)(const uint8_t *a, int a_stride, + int xoffset, int yoffset, + const uint8_t *b, int b_stride, + uint32_t *sse, + const uint8_t *second_pred); +using SumOfSquaresFunction = unsigned int (*)(const int16_t *src); #if !CONFIG_REALTIME_ONLY -typedef uint32_t (*ObmcSubpelVarFunc)(const uint8_t *pre, int pre_stride, - int xoffset, int yoffset, - const int32_t *wsrc, const int32_t *mask, - unsigned int *sse); +using ObmcSubpelVarFunc = uint32_t (*)(const uint8_t *pre, int pre_stride, + int xoffset, int yoffset, + const int32_t *wsrc, const int32_t *mask, + unsigned int *sse); + #endif using libaom_test::ACMRandom; @@ -1437,7 +1438,7 @@ static const int kMaskMax = 64; -typedef TestParams<ObmcSubpelVarFunc> ObmcSubpelVarianceParams; +using ObmcSubpelVarianceParams = TestParams<ObmcSubpelVarFunc>; template <typename FunctionType> class ObmcVarianceTest @@ -1592,19 +1593,19 @@ #endif // !CONFIG_REALTIME_ONLY -typedef MseWxHTestClass<MseWxH16bitFunc> MseWxHTest; -typedef Mse16xHTestClass<Mse16xH16bitFunc> Mse16xHTest; -typedef MainTestClass<VarianceMxNFunc> AvxMseTest; -typedef MainTestClass<VarianceMxNFunc> AvxVarianceTest; -typedef MainTestClass<GetSseSum8x8QuadFunc> GetSseSum8x8QuadTest; -typedef MainTestClass<GetSseSum16x16DualFunc> GetSseSum16x16DualTest; -typedef SubpelVarianceTest<SubpixVarMxNFunc> AvxSubpelVarianceTest; -typedef SubpelVarianceTest<SubpixAvgVarMxNFunc> AvxSubpelAvgVarianceTest; +using MseWxHTest = MseWxHTestClass<MseWxH16bitFunc>; +using Mse16xHTest = Mse16xHTestClass<Mse16xH16bitFunc>; +using AvxMseTest = MainTestClass<VarianceMxNFunc>; +using AvxVarianceTest = MainTestClass<VarianceMxNFunc>; +using GetSseSum8x8QuadTest = MainTestClass<GetSseSum8x8QuadFunc>; +using GetSseSum16x16DualTest = MainTestClass<GetSseSum16x16DualFunc>; +using AvxSubpelVarianceTest = SubpelVarianceTest<SubpixVarMxNFunc>; +using AvxSubpelAvgVarianceTest = SubpelVarianceTest<SubpixAvgVarMxNFunc>; #if !CONFIG_REALTIME_ONLY -typedef ObmcVarianceTest<ObmcSubpelVarFunc> AvxObmcSubpelVarianceTest; +using AvxObmcSubpelVarianceTest = ObmcVarianceTest<ObmcSubpelVarFunc>; #endif -typedef TestParams<MseWxH16bitFunc> MseWxHParams; -typedef TestParams<Mse16xH16bitFunc> Mse16xHParams; +using MseWxHParams = TestParams<MseWxH16bitFunc>; +using Mse16xHParams = TestParams<Mse16xH16bitFunc>; TEST_P(MseWxHTest, RefMse) { RefMatchTestMse(); } TEST_P(MseWxHTest, DISABLED_SpeedMse) { SpeedTest(); } @@ -1659,14 +1660,14 @@ ::testing::Values(aom_get_mb_ss_c)); #endif // !CONFIG_REALTIME_ONLY -typedef TestParams<VarianceMxNFunc> MseParams; +using MseParams = TestParams<VarianceMxNFunc>; INSTANTIATE_TEST_SUITE_P(C, AvxMseTest, ::testing::Values(MseParams(4, 4, &aom_mse16x16_c), MseParams(4, 3, &aom_mse16x8_c), MseParams(3, 4, &aom_mse8x16_c), MseParams(3, 3, &aom_mse8x8_c))); -typedef TestParams<VarianceMxNFunc> VarianceParams; +using VarianceParams = TestParams<VarianceMxNFunc>; const VarianceParams kArrayVariance_c[] = { VarianceParams(7, 7, &aom_variance128x128_c), VarianceParams(7, 6, &aom_variance128x64_c), @@ -1696,7 +1697,7 @@ INSTANTIATE_TEST_SUITE_P(C, AvxVarianceTest, ::testing::ValuesIn(kArrayVariance_c)); -typedef TestParams<GetSseSum8x8QuadFunc> GetSseSumParams; +using GetSseSumParams = TestParams<GetSseSum8x8QuadFunc>; const GetSseSumParams kArrayGetSseSum8x8Quad_c[] = { GetSseSumParams(7, 7, &aom_get_var_sse_sum_8x8_quad_c, 0), GetSseSumParams(6, 6, &aom_get_var_sse_sum_8x8_quad_c, 0), @@ -1706,7 +1707,7 @@ INSTANTIATE_TEST_SUITE_P(C, GetSseSum8x8QuadTest, ::testing::ValuesIn(kArrayGetSseSum8x8Quad_c)); -typedef TestParams<GetSseSum16x16DualFunc> GetSseSumParamsDual; +using GetSseSumParamsDual = TestParams<GetSseSum16x16DualFunc>; const GetSseSumParamsDual kArrayGetSseSum16x16Dual_c[] = { GetSseSumParamsDual(7, 7, &aom_get_var_sse_sum_16x16_dual_c, 0), GetSseSumParamsDual(6, 6, &aom_get_var_sse_sum_16x16_dual_c, 0), @@ -1717,7 +1718,7 @@ INSTANTIATE_TEST_SUITE_P(C, GetSseSum16x16DualTest, ::testing::ValuesIn(kArrayGetSseSum16x16Dual_c)); -typedef TestParams<SubpixVarMxNFunc> SubpelVarianceParams; +using SubpelVarianceParams = TestParams<SubpixVarMxNFunc>; const SubpelVarianceParams kArraySubpelVariance_c[] = { SubpelVarianceParams(7, 7, &aom_sub_pixel_variance128x128_c, 0), SubpelVarianceParams(7, 6, &aom_sub_pixel_variance128x64_c, 0), @@ -1747,7 +1748,7 @@ INSTANTIATE_TEST_SUITE_P(C, AvxSubpelVarianceTest, ::testing::ValuesIn(kArraySubpelVariance_c)); -typedef TestParams<SubpixAvgVarMxNFunc> SubpelAvgVarianceParams; +using SubpelAvgVarianceParams = TestParams<SubpixAvgVarMxNFunc>; const SubpelAvgVarianceParams kArraySubpelAvgVariance_c[] = { SubpelAvgVarianceParams(7, 7, &aom_sub_pixel_avg_variance128x128_c, 0), SubpelAvgVarianceParams(7, 6, &aom_sub_pixel_avg_variance128x64_c, 0), @@ -1808,9 +1809,8 @@ #endif #if CONFIG_AV1_HIGHBITDEPTH -typedef uint64_t (*MseHBDWxH16bitFunc)(uint16_t *dst, int dstride, - uint16_t *src, int sstride, int w, - int h); +using MseHBDWxH16bitFunc = uint64_t (*)(uint16_t *, int, uint16_t *, int, int, + int); template <typename FunctionType> class MseHBDWxHTestClass @@ -1909,15 +1909,15 @@ } } -typedef TestParams<MseHBDWxH16bitFunc> MseHBDWxHParams; -typedef MseHBDWxHTestClass<MseHBDWxH16bitFunc> MseHBDWxHTest; -typedef MainTestClass<VarianceMxNFunc> AvxHBDMseTest; +using MseHBDWxHParams = TestParams<MseHBDWxH16bitFunc>; +using MseHBDWxHTest = MseHBDWxHTestClass<MseHBDWxH16bitFunc>; +using AvxHBDMseTest = MainTestClass<VarianceMxNFunc>; GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(AvxHBDMseTest); -typedef MainTestClass<VarianceMxNFunc> AvxHBDVarianceTest; -typedef SubpelVarianceTest<SubpixVarMxNFunc> AvxHBDSubpelVarianceTest; -typedef SubpelVarianceTest<SubpixAvgVarMxNFunc> AvxHBDSubpelAvgVarianceTest; +using AvxHBDVarianceTest = MainTestClass<VarianceMxNFunc>; +using AvxHBDSubpelVarianceTest = SubpelVarianceTest<SubpixVarMxNFunc>; +using AvxHBDSubpelAvgVarianceTest = SubpelVarianceTest<SubpixAvgVarMxNFunc>; #if !CONFIG_REALTIME_ONLY -typedef ObmcVarianceTest<ObmcSubpelVarFunc> AvxHBDObmcSubpelVarianceTest; +using AvxHBDObmcSubpelVarianceTest = ObmcVarianceTest<ObmcSubpelVarFunc>; #endif GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(AvxHBDObmcSubpelVarianceTest); @@ -3661,7 +3661,6 @@ VarianceParams(3, 3, &aom_variance8x8_neon_dotprod), VarianceParams(3, 2, &aom_variance8x4_neon_dotprod), VarianceParams(2, 3, &aom_variance4x8_neon_dotprod), - VarianceParams(2, 2, &aom_variance4x4_neon_dotprod), #if !CONFIG_REALTIME_ONLY VarianceParams(2, 4, &aom_variance4x16_neon_dotprod), VarianceParams(4, 2, &aom_variance16x4_neon_dotprod), @@ -3675,6 +3674,35 @@ INSTANTIATE_TEST_SUITE_P(NEON_DOTPROD, AvxVarianceTest, ::testing::ValuesIn(kArrayVariance_neon_dotprod)); +const SubpelVarianceParams kArraySubpelVariance_neon_dotprod[] = { + SubpelVarianceParams(7, 7, &aom_sub_pixel_variance128x128_neon_dotprod, 0), + SubpelVarianceParams(7, 6, &aom_sub_pixel_variance128x64_neon_dotprod, 0), + SubpelVarianceParams(6, 7, &aom_sub_pixel_variance64x128_neon_dotprod, 0), + SubpelVarianceParams(6, 6, &aom_sub_pixel_variance64x64_neon_dotprod, 0), + SubpelVarianceParams(6, 5, &aom_sub_pixel_variance64x32_neon_dotprod, 0), + SubpelVarianceParams(5, 6, &aom_sub_pixel_variance32x64_neon_dotprod, 0), + SubpelVarianceParams(5, 5, &aom_sub_pixel_variance32x32_neon_dotprod, 0), + SubpelVarianceParams(5, 4, &aom_sub_pixel_variance32x16_neon_dotprod, 0), + SubpelVarianceParams(4, 5, &aom_sub_pixel_variance16x32_neon_dotprod, 0), + SubpelVarianceParams(4, 4, &aom_sub_pixel_variance16x16_neon_dotprod, 0), + SubpelVarianceParams(4, 3, &aom_sub_pixel_variance16x8_neon_dotprod, 0), + SubpelVarianceParams(3, 4, &aom_sub_pixel_variance8x16_neon_dotprod, 0), + SubpelVarianceParams(3, 3, &aom_sub_pixel_variance8x8_neon_dotprod, 0), + SubpelVarianceParams(3, 2, &aom_sub_pixel_variance8x4_neon_dotprod, 0), + SubpelVarianceParams(2, 3, &aom_sub_pixel_variance4x8_neon_dotprod, 0), +#if !CONFIG_REALTIME_ONLY + SubpelVarianceParams(6, 4, &aom_sub_pixel_variance64x16_neon_dotprod, 0), + SubpelVarianceParams(4, 6, &aom_sub_pixel_variance16x64_neon_dotprod, 0), + SubpelVarianceParams(5, 3, &aom_sub_pixel_variance32x8_neon_dotprod, 0), + SubpelVarianceParams(3, 5, &aom_sub_pixel_variance8x32_neon_dotprod, 0), + SubpelVarianceParams(4, 2, &aom_sub_pixel_variance16x4_neon_dotprod, 0), + SubpelVarianceParams(2, 4, &aom_sub_pixel_variance4x16_neon_dotprod, 0), +#endif +}; +INSTANTIATE_TEST_SUITE_P( + NEON_DOTPROD, AvxSubpelVarianceTest, + ::testing::ValuesIn(kArraySubpelVariance_neon_dotprod)); + const GetSseSumParams kArrayGetSseSum8x8Quad_neon_dotprod[] = { GetSseSumParams(7, 7, &aom_get_var_sse_sum_8x8_quad_neon_dotprod, 0), GetSseSumParams(6, 6, &aom_get_var_sse_sum_8x8_quad_neon_dotprod, 0),
diff --git a/test/warp_filter_test.cc b/test/warp_filter_test.cc index 3a35075..530f2aa 100644 --- a/test/warp_filter_test.cc +++ b/test/warp_filter_test.cc
@@ -66,6 +66,12 @@ #endif // CONFIG_AV1_HIGHBITDEPTH #endif // HAVE_AVX2 +#if CONFIG_HIGHWAY && HAVE_AVX512 +INSTANTIATE_TEST_SUITE_P( + AVX512, AV1WarpFilterTest, + libaom_test::AV1WarpFilter::BuildParams(av1_warp_affine_avx512)); +#endif // CONFIG_HIGHWAY && HAVE_AVX512 + #if HAVE_NEON INSTANTIATE_TEST_SUITE_P( NEON, AV1WarpFilterTest,
diff --git a/test/warp_filter_test_util.cc b/test/warp_filter_test_util.cc index 9dadbaf..19683f0 100644 --- a/test/warp_filter_test_util.cc +++ b/test/warp_filter_test_util.cc
@@ -258,9 +258,13 @@ conv_params.fwd_offset = quant_dist_lookup_table[jj][ii]; conv_params.bck_offset = quant_dist_lookup_table[jj][1 - ii]; } - av1_warp_affine_c(mat, input, w, h, stride, output.get(), 32, 32, - out_w, out_h, out_w, sub_x, sub_y, &conv_params, - alpha, beta, gamma, delta); + // Around 6% of the time, use a random column/row to test + // out-of-bounds filtering cases. + const int col = rnd_.Rand8() < 16 ? rnd_.Rand8() : 32; + const int row = rnd_.Rand8() < 16 ? rnd_.Rand8() : 32; + av1_warp_affine_c(mat, input, w, h, stride, output.get(), col, + row, out_w, out_h, out_w, sub_x, sub_y, + &conv_params, alpha, beta, gamma, delta); if (use_no_round) { conv_params = get_conv_params_no_round( do_average, 0, dstb.get(), out_w, 1, bd); @@ -272,9 +276,9 @@ conv_params.fwd_offset = quant_dist_lookup_table[jj][ii]; conv_params.bck_offset = quant_dist_lookup_table[jj][1 - ii]; } - test_impl(mat, input, w, h, stride, output2.get(), 32, 32, out_w, - out_h, out_w, sub_x, sub_y, &conv_params, alpha, beta, - gamma, delta); + test_impl(mat, input, w, h, stride, output2.get(), col, row, + out_w, out_h, out_w, sub_x, sub_y, &conv_params, alpha, + beta, gamma, delta); if (use_no_round) { for (int j = 0; j < out_w * out_h; ++j) ASSERT_EQ(dsta[j], dstb[j]) @@ -458,8 +462,12 @@ conv_params.bck_offset = quant_dist_lookup_table[jj][1 - ii]; } + // Around 6% of the time, use a random column/row to test + // out-of-bounds filtering cases. + const int col = rnd_.Rand8() < 16 ? rnd_.Rand8() : 32; + const int row = rnd_.Rand8() < 16 ? rnd_.Rand8() : 32; av1_highbd_warp_affine_c(mat, input, w, h, stride, output.get(), - 32, 32, out_w, out_h, out_w, sub_x, + col, row, out_w, out_h, out_w, sub_x, sub_y, bd, &conv_params, alpha, beta, gamma, delta); if (use_no_round) { @@ -475,9 +483,9 @@ conv_params.fwd_offset = quant_dist_lookup_table[jj][ii]; conv_params.bck_offset = quant_dist_lookup_table[jj][1 - ii]; } - test_impl(mat, input, w, h, stride, output2.get(), 32, 32, out_w, - out_h, out_w, sub_x, sub_y, bd, &conv_params, alpha, - beta, gamma, delta); + test_impl(mat, input, w, h, stride, output2.get(), col, row, + out_w, out_h, out_w, sub_x, sub_y, bd, &conv_params, + alpha, beta, gamma, delta); if (use_no_round) { for (int j = 0; j < out_w * out_h; ++j)
diff --git a/test/warp_filter_test_util.h b/test/warp_filter_test_util.h index 7e401d8..25fa110 100644 --- a/test/warp_filter_test_util.h +++ b/test/warp_filter_test_util.h
@@ -34,16 +34,16 @@ namespace AV1WarpFilter { -typedef void (*warp_affine_func)(const int32_t *mat, const uint8_t *ref, - int width, int height, int stride, - uint8_t *pred, int p_col, int p_row, - int p_width, int p_height, int p_stride, - int subsampling_x, int subsampling_y, - ConvolveParams *conv_params, int16_t alpha, - int16_t beta, int16_t gamma, int16_t delta); +using warp_affine_func = void (*)(const int32_t *mat, const uint8_t *ref, + int width, int height, int stride, + uint8_t *pred, int p_col, int p_row, + int p_width, int p_height, int p_stride, + int subsampling_x, int subsampling_y, + ConvolveParams *conv_params, int16_t alpha, + int16_t beta, int16_t gamma, int16_t delta); -typedef std::tuple<int, int, int, warp_affine_func> WarpTestParam; -typedef std::tuple<WarpTestParam, int, int, int, int> WarpTestParams; +using WarpTestParam = std::tuple<int, int, int, warp_affine_func>; +using WarpTestParams = std::tuple<WarpTestParam, int, int, int, int>; ::testing::internal::ParamGenerator<WarpTestParams> BuildParams( warp_affine_func filter); @@ -64,19 +64,17 @@ #if CONFIG_AV1_HIGHBITDEPTH namespace AV1HighbdWarpFilter { -typedef void (*highbd_warp_affine_func)(const int32_t *mat, const uint16_t *ref, - int width, int height, int stride, - uint16_t *pred, int p_col, int p_row, - int p_width, int p_height, int p_stride, - int subsampling_x, int subsampling_y, - int bd, ConvolveParams *conv_params, - int16_t alpha, int16_t beta, - int16_t gamma, int16_t delta); +using highbd_warp_affine_func = + void (*)(const int32_t *mat, const uint16_t *ref, int width, int height, + int stride, uint16_t *pred, int p_col, int p_row, int p_width, + int p_height, int p_stride, int subsampling_x, int subsampling_y, + int bd, ConvolveParams *conv_params, int16_t alpha, int16_t beta, + int16_t gamma, int16_t delta); -typedef std::tuple<int, int, int, int, highbd_warp_affine_func> - HighbdWarpTestParam; -typedef std::tuple<HighbdWarpTestParam, int, int, int, int> - HighbdWarpTestParams; +using HighbdWarpTestParam = + std::tuple<int, int, int, int, highbd_warp_affine_func>; +using HighbdWarpTestParams = + std::tuple<HighbdWarpTestParam, int, int, int, int>; ::testing::internal::ParamGenerator<HighbdWarpTestParams> BuildParams( highbd_warp_affine_func filter);
diff --git a/test/wiener_test.cc b/test/wiener_test.cc index fe6e40e..8d5950d 100644 --- a/test/wiener_test.cc +++ b/test/wiener_test.cc
@@ -9,6 +9,7 @@ * PATENTS file, you can obtain it at www.aomedia.org/license/patent. */ +#include <cinttypes> #include <tuple> #include <utility> #include <vector> @@ -175,18 +176,18 @@ } static const int kIterations = 100; -typedef void (*compute_stats_Func)(int wiener_win, const uint8_t *dgd, - const uint8_t *src, int16_t *dgd_avg, - int16_t *src_avg, int h_start, int h_end, - int v_start, int v_end, int dgd_stride, - int src_stride, int64_t *M, int64_t *H, - int use_downsampled_wiener_stats); +using compute_stats_Func = void (*)(int wiener_win, const uint8_t *dgd, + const uint8_t *src, int16_t *dgd_avg, + int16_t *src_avg, int h_start, int h_end, + int v_start, int v_end, int dgd_stride, + int src_stride, int64_t *M, int64_t *H, + int use_downsampled_wiener_stats); //////////////////////////////////////////////////////////////////////////////// // 8 bit //////////////////////////////////////////////////////////////////////////////// -typedef std::tuple<const compute_stats_Func> WienerTestParam; +using WienerTestParam = std::tuple<const compute_stats_Func>; class WienerTest : public ::testing::TestWithParam<WienerTestParam> { public: @@ -537,14 +538,14 @@ } static const int kIterations = 100; -typedef void (*compute_stats_Func)(int wiener_win, const uint8_t *dgd, - const uint8_t *src, int16_t *d, int16_t *s, - int h_start, int h_end, int v_start, - int v_end, int dgd_stride, int src_stride, - int64_t *M, int64_t *H, - aom_bit_depth_t bit_depth); +using compute_stats_Func = void (*)(int wiener_win, const uint8_t *dgd, + const uint8_t *src, int16_t *d, int16_t *s, + int h_start, int h_end, int v_start, + int v_end, int dgd_stride, int src_stride, + int64_t *M, int64_t *H, + aom_bit_depth_t bit_depth); -typedef std::tuple<const compute_stats_Func> WienerTestParam; +using WienerTestParam = std::tuple<const compute_stats_Func>; class WienerTestHighbd : public ::testing::TestWithParam<WienerTestParam> { public:
diff --git a/third_party/SVT-AV1/README.libaom b/third_party/SVT-AV1/README.libaom index 1dadd49..069f7c1 100644 --- a/third_party/SVT-AV1/README.libaom +++ b/third_party/SVT-AV1/README.libaom
@@ -1,8 +1,12 @@ +Name: SVT-AV1 URL: https://gitlab.com/AOMediaCodec/SVT-AV1 - -Version: 8ff99c90359330d2e807757c9425560bbc452ff3 -License: BSD-3-clause clear +Version: N/A +Update Mechanism: Manual +Revision: 8ff99c90359330d2e807757c9425560bbc452ff3 +License: BSD-3-Clause-Clear License File: LICENSE.md +Shipped in Chromium: no +Security Critical: yes Description: Port the x86 intrinsics used for single reference convolve reconstructions.
diff --git a/third_party/fastfeat/LICENSE b/third_party/fastfeat/LICENSE index f347008..ee48fe6 100644 --- a/third_party/fastfeat/LICENSE +++ b/third_party/fastfeat/LICENSE
@@ -13,8 +13,8 @@ notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - *Neither the name of the University of Cambridge nor the names of - its contributors may be used to endorse or promote products derived + *Neither the name of the University of Cambridge nor the names of + its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
diff --git a/third_party/fastfeat/README.libaom b/third_party/fastfeat/README.libaom index 75e164f..2aab235 100644 --- a/third_party/fastfeat/README.libaom +++ b/third_party/fastfeat/README.libaom
@@ -1,7 +1,12 @@ +Name: FAST feature detectors in C Version 2.0 URL: https://github.com/edrosten/fast-C-src -Version: 391d5e939eb1545d24c10533d7de424db8d9c191 -License: BSD +Version: N/A +Update Mechanism: Manual +Revision: 391d5e939eb1545d24c10533d7de424db8d9c191 +License: BSD-3-Clause License File: LICENSE +Shipped in Chromium: no +Security Critical: yes Description: Library to compute FAST features with non-maximum suppression.
diff --git a/third_party/googletest/README.libaom b/third_party/googletest/README.libaom index 3b21c8e..a7d7e7b 100644 --- a/third_party/googletest/README.libaom +++ b/third_party/googletest/README.libaom
@@ -1,7 +1,13 @@ +Name: Google Test: Google's C++ Testing Framework +Short Name: googletest URL: https://github.com/google/googletest -Version: release-1.12.1 -License: BSD -License File: LICENSE +Version: 1.12.1 +Update Mechanism: Manual +Revision: 58d77fa8070e8cec2dc1ed015d66b454c8d78850 +License: BSD-3-Clause +License File: src/LICENSE +Shipped in Chromium: no +Security Critical: no Description: Google's framework for writing C++ tests on a variety of platforms
diff --git a/third_party/highway/README.libaom b/third_party/highway/README.libaom index 28fdd58..7fc5db9 100644 --- a/third_party/highway/README.libaom +++ b/third_party/highway/README.libaom
@@ -1,8 +1,13 @@ +Name: Highway: C++ library for SIMD +Short Name: highway URL: https://github.com/google/highway - -Version: e92c12750d18c372867809b882dd3ec6874ecc73 -License: BSD-3-clause clear +Version: N/A +Update Mechanism: Manual +Revision: e92c12750d18c372867809b882dd3ec6874ecc73 +License: BSD-3-Clause License File: LICENSE-BSD3 +Shipped in Chromium: no +Security Critical: yes Description: Highway is a C++ library that provides portable SIMD/vector intrinsics.
diff --git a/third_party/libwebm/README.libaom b/third_party/libwebm/README.libaom index 6e43487..850b9ce 100644 --- a/third_party/libwebm/README.libaom +++ b/third_party/libwebm/README.libaom
@@ -1,7 +1,12 @@ +Name: libwebm URL: https://chromium.googlesource.com/webm/libwebm -Version: 3b630045052e1e4d563207ab9e3be8d137c26067 -License: BSD +Version: N/A +Update Mechanism: Manual +Revision: 3b630045052e1e4d563207ab9e3be8d137c26067 +License: BSD-3-Clause License File: LICENSE.TXT +Shipped in Chromium: no +Security Critical: yes Description: libwebm is used to handle WebM container I/O.
diff --git a/third_party/libyuv/README.libaom b/third_party/libyuv/README.libaom index fcdf921..cdf1652 100644 --- a/third_party/libyuv/README.libaom +++ b/third_party/libyuv/README.libaom
@@ -1,8 +1,12 @@ Name: libyuv URL: https://chromium.googlesource.com/libyuv/libyuv/ -Version: dfaf7534e0e536f7e5ef8ddd7326797bd09b8622 -License: BSD +Version: N/A +Update Mechanism: Manual +Revision: dfaf7534e0e536f7e5ef8ddd7326797bd09b8622 +License: BSD-3-Clause License File: LICENSE +Shipped in Chromium: no +Security Critical: yes Description: libyuv is an open source project that includes YUV conversion and scaling
diff --git a/third_party/vector/README.libaom b/third_party/vector/README.libaom index 4b6e4dd..f8e06b0 100644 --- a/third_party/vector/README.libaom +++ b/third_party/vector/README.libaom
@@ -1,8 +1,12 @@ Name: vector URL: https://github.com/goldsborough/vector -Version: commit-id: 40efe82 +Version: N/A +Update Mechanism: Manual +Revision: 40efe82 License: MIT License File: LICENSE +Shipped in Chromium: yes +Security Critical: yes Description: A feature-complete, generic and customizable resizable
diff --git a/third_party/x86inc/README.libaom b/third_party/x86inc/README.libaom index 6b92358..d6f8901 100644 --- a/third_party/x86inc/README.libaom +++ b/third_party/x86inc/README.libaom
@@ -1,7 +1,12 @@ +Name: x86inc.asm URL: https://git.videolan.org/git/x264.git -Version: 3e5aed95cc470f37e2db3e6506a8deb89b527720 +Version: N/A +Update Mechanism: Manual +Revision: 3e5aed95cc470f37e2db3e6506a8deb89b527720 License: ISC License File: LICENSE +Shipped in Chromium: yes +Security Critical: yes Description: x264/libav's framework for x86 assembly. Contains a variety of macros and