| /* |
| * Copyright (c) 2016, Alliance for Open Media. All rights reserved |
| * |
| * This source code is subject to the terms of the BSD 2 Clause License and |
| * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License |
| * was not distributed with this source code in the LICENSE file, you can |
| * obtain it at www.aomedia.org/license/software. If the Alliance for Open |
| * Media Patent License 1.0 was not distributed with this source code in the |
| * PATENTS file, you can obtain it at www.aomedia.org/license/patent. |
| */ |
| |
| #ifndef AOM_AV1_ENCODER_TEMPORAL_FILTER_H_ |
| #define AOM_AV1_ENCODER_TEMPORAL_FILTER_H_ |
| |
| #include <stdbool.h> |
| |
| #ifdef __cplusplus |
| extern "C" { |
| #endif |
| /*!\cond */ |
| struct AV1_COMP; |
| struct AV1EncoderConfig; |
| struct ThreadData; |
| // TODO(wtc): These two variables are only used in avx2, sse2, neon |
| // implementations, where the block size is still hard coded to TF_BLOCK_SIZE. |
| // This should be fixed to align with the c implementation. |
| #define BH 32 |
| #define BW 32 |
| |
| // Block size used in temporal filtering. |
| #define TF_BLOCK_SIZE BLOCK_32X32 |
| |
| // Window size for temporal filtering. |
| #define TF_WINDOW_LENGTH 5 |
| |
| // A constant number, sqrt(pi / 2), used for noise estimation. |
| static const double SQRT_PI_BY_2 = 1.25331413732; |
| |
| // Hyper-parameters used to compute filtering weight. These hyper-parameters can |
| // be tuned for a better performance. |
| // 0. A scale factor used in temporal filtering to raise the filter weight from |
| // `double` with range [0, 1] to `int` with range [0, 1000]. |
| #define TF_WEIGHT_SCALE 1000 |
| // 1. Weight factor used to balance the weighted-average between window error |
| // and block error. The weight is for window error while the weight for block |
| // error is always set as 1. |
| #define TF_WINDOW_BLOCK_BALANCE_WEIGHT 5 |
| // 2. Threshold for using q to adjust the filtering weight. Concretely, when |
| // using a small q (high bitrate), we would like to reduce the filtering |
| // strength such that more detailed information can be preserved. Hence, when |
| // q is smaller than this threshold, we will adjust the filtering weight |
| // based on the q-value. |
| #define TF_Q_DECAY_THRESHOLD 20 |
| // 3. Normalization factor used to normalize the motion search error. Since the |
| // motion search error can be large and uncontrollable, we will simply |
| // normalize it before using it to compute the filtering weight. |
| #define TF_SEARCH_ERROR_NORM_WEIGHT 20 |
| // 4. Threshold for using `arnr_strength` to adjust the filtering strength. |
| // Concretely, users can use `arnr_strength` arguments to control the |
| // strength of temporal filtering. When `arnr_strength` is small enough ( |
| // i.e., smaller than this threshold), we will adjust the filtering weight |
| // based on the strength value. |
| #define TF_STRENGTH_THRESHOLD 4 |
| // 5. Threshold for using motion search distance to adjust the filtering weight. |
| // Concretely, larger motion search vector leads to a higher probability of |
| // unreliable search. Hence, we would like to reduce the filtering strength |
| // when the distance is large enough. Considering that the distance actually |
| // relies on the frame size, this threshold is also a resolution-based |
| // threshold. Taking 720p videos as an instance, if this field equals to 0.1, |
| // then the actual threshold will be 720 * 0.1 = 72. Similarly, the threshold |
| // for 360p videos will be 360 * 0.1 = 36. |
| #define TF_SEARCH_DISTANCE_THRESHOLD 0.1 |
| // 6. Threshold to identify if the q is in a relative high range. |
| // Above this cutoff q, a stronger filtering is applied. |
| // For a high q, the quantization throws away more information, and thus a |
| // stronger filtering is less likely to distort the encoded quality, while a |
| // stronger filtering could reduce bit rates. |
| // Ror a low q, more details are expected to be retained. Filtering is thus |
| // more conservative. |
| #define TF_QINDEX_CUTOFF 128 |
| |
| #define NOISE_ESTIMATION_EDGE_THRESHOLD 50 |
| |
| // Sum and SSE source vs filtered frame difference returned by |
| // temporal filter. |
| typedef struct { |
| int64_t sum; |
| int64_t sse; |
| } FRAME_DIFF; |
| |
| /*!\endcond */ |
| |
| /*! |
| * \brief Parameters related to temporal filtering. |
| */ |
| typedef struct { |
| /*! |
| * Frame buffers used for temporal filtering. |
| */ |
| YV12_BUFFER_CONFIG *frames[MAX_LAG_BUFFERS]; |
| /*! |
| * Number of frames in the frame buffer. |
| */ |
| int num_frames; |
| |
| /*! |
| * Output filtered frame |
| */ |
| YV12_BUFFER_CONFIG *output_frame; |
| |
| /*! |
| * Index of the frame to be filtered. |
| */ |
| int filter_frame_idx; |
| /*! |
| * Whether to accumulate diff for show existing condition check. |
| */ |
| int compute_frame_diff; |
| /*! |
| * Frame scaling factor. |
| */ |
| struct scale_factors sf; |
| /*! |
| * Estimated noise levels for each plane in the frame. |
| */ |
| double noise_levels[MAX_MB_PLANE]; |
| /*! |
| * Number of pixels in the temporal filtering block across all planes. |
| */ |
| int num_pels; |
| /*! |
| * Number of temporal filtering block rows. |
| */ |
| int mb_rows; |
| /*! |
| * Number of temporal filtering block columns. |
| */ |
| int mb_cols; |
| /*! |
| * Whether the frame is high-bitdepth or not. |
| */ |
| int is_highbitdepth; |
| /*! |
| * Quantization factor used in temporal filtering. |
| */ |
| int q_factor; |
| } TemporalFilterCtx; |
| |
| /*! |
| * buffer count in TEMPORAL_FILTER_INFO |
| * Currently we only apply filtering on KEY and ARF after |
| * define_gf_group(). Hence, the count is two. |
| */ |
| #define TF_INFO_BUF_COUNT 2 |
| |
| /*! |
| * \brief Temporal filter info for a gop |
| */ |
| typedef struct TEMPORAL_FILTER_INFO { |
| /*! |
| * A flag indicate whether temporal filter shoud be applied. |
| * This flag will stored the result of |
| * av1_is_temporal_filter_on() |
| */ |
| int is_temporal_filter_on; |
| /*! |
| * buffers used for temporal filtering in a GOP |
| * index 0 for key frame and index 1 for ARF |
| */ |
| YV12_BUFFER_CONFIG tf_buf[TF_INFO_BUF_COUNT]; |
| |
| /*! |
| * buffers used for temporal filtering for |
| * INTNL_ARF_UPDATE |
| * Check av1_gop_is_second_arf() for the |
| * definition of second_arf in detail |
| */ |
| YV12_BUFFER_CONFIG tf_buf_second_arf; |
| /*! |
| * whether to show the buffer directly or not. |
| */ |
| FRAME_DIFF frame_diff[TF_INFO_BUF_COUNT]; |
| /*! |
| * the corresponding gf_index for the buffer. |
| */ |
| int tf_buf_gf_index[TF_INFO_BUF_COUNT]; |
| /*! |
| * the display_index offset between next show frame and the frames in the GOP |
| */ |
| int tf_buf_display_index_offset[TF_INFO_BUF_COUNT]; |
| /*! |
| * whether the buf is valid or not. |
| */ |
| int tf_buf_valid[TF_INFO_BUF_COUNT]; |
| } TEMPORAL_FILTER_INFO; |
| |
| /*!\brief Check whether we should apply temporal filter at all. |
| * \param[in] oxcf AV1 encoder config |
| * |
| * \return 1: temporal filter is on 0: temporal is off |
| */ |
| int av1_is_temporal_filter_on(const struct AV1EncoderConfig *oxcf); |
| |
| /*!\brief Allocate buffers for TEMPORAL_FILTER_INFO |
| * \param[in,out] tf_info Temporal filter info for a gop |
| * \param[in,out] cpi Top level encoder instance structure |
| * |
| * \return True on success, false on memory allocation failure. |
| */ |
| bool av1_tf_info_alloc(TEMPORAL_FILTER_INFO *tf_info, |
| const struct AV1_COMP *cpi); |
| |
| /*!\brief Free buffers for TEMPORAL_FILTER_INFO |
| * \param[in,out] tf_info Temporal filter info for a gop |
| */ |
| void av1_tf_info_free(TEMPORAL_FILTER_INFO *tf_info); |
| |
| /*!\brief Reset validity of tf_buf in TEMPORAL_FILTER_INFO |
| * \param[in,out] tf_info Temporal filter info for a gop |
| */ |
| void av1_tf_info_reset(TEMPORAL_FILTER_INFO *tf_info); |
| |
| /*!\brief Apply temporal filter for key frame and ARF in a gop |
| * \param[in,out] tf_info Temporal filter info for a gop |
| * \param[in,out] cpi Top level encoder instance structure |
| * \param[in] gf_group GF/ARF group data structure |
| */ |
| void av1_tf_info_filtering(TEMPORAL_FILTER_INFO *tf_info, struct AV1_COMP *cpi, |
| const GF_GROUP *gf_group); |
| |
| /*!\brief Get a filtered buffer from TEMPORAL_FILTER_INFO |
| * \param[in,out] tf_info Temporal filter info for a gop |
| * \param[in] gf_index gf_index for the target buffer |
| * \param[out] show_tf_buf whether the target buffer can be shown |
| * directly |
| */ |
| YV12_BUFFER_CONFIG *av1_tf_info_get_filtered_buf(TEMPORAL_FILTER_INFO *tf_info, |
| int gf_index, |
| FRAME_DIFF *frame_diff); |
| |
| /*!\cond */ |
| |
| // Data related to temporal filtering. |
| typedef struct { |
| // Source vs filtered frame error. |
| FRAME_DIFF diff; |
| // Pointer to temporary block info used to store state in temporal filtering |
| // process. |
| MB_MODE_INFO *tmp_mbmi; |
| // Pointer to accumulator buffer used in temporal filtering process. |
| uint32_t *accum; |
| // Pointer to count buffer used in temporal filtering process. |
| uint16_t *count; |
| // Pointer to predictor used in temporal filtering process. |
| uint8_t *pred; |
| } TemporalFilterData; |
| |
| // Data related to temporal filter multi-thread synchronization. |
| typedef struct { |
| #if CONFIG_MULTITHREAD |
| // Mutex lock used for dispatching jobs. |
| pthread_mutex_t *mutex_; |
| #endif // CONFIG_MULTITHREAD |
| // Next temporal filter block row to be filtered. |
| int next_tf_row; |
| // Initialized to false, set to true by the worker thread that encounters an |
| // error in order to abort the processing of other worker threads. |
| bool tf_mt_exit; |
| } AV1TemporalFilterSync; |
| |
| // Estimates noise level from a given frame using a single plane (Y, U, or V). |
| // This is an adaptation of the mehtod in the following paper: |
| // Shen-Chuan Tai, Shih-Ming Yang, "A fast method for image noise |
| // estimation using Laplacian operator and adaptive edge detection", |
| // Proc. 3rd International Symposium on Communications, Control and |
| // Signal Processing, 2008, St Julians, Malta. |
| // Inputs: |
| // frame: Pointer to the frame to estimate noise level from. |
| // noise_level: Pointer to store the estimated noise. |
| // plane_from: Index of the starting plane used for noise estimation. |
| // Commonly, 0 for Y-plane, 1 for U-plane, and 2 for V-plane. |
| // plane_to: Index of the end plane used for noise estimation. |
| // bit_depth: Actual bit-depth instead of the encoding bit-depth of the frame. |
| // edge_thresh: Edge threshold. |
| void av1_estimate_noise_level(const YV12_BUFFER_CONFIG *frame, |
| double *noise_level, int plane_from, int plane_to, |
| int bit_depth, int edge_thresh); |
| /*!\endcond */ |
| |
| /*!\brief Does temporal filter for a given macroblock row. |
| * |
| * \ingroup src_frame_proc |
| * \param[in] cpi Top level encoder instance structure |
| * \param[in] td Pointer to thread data |
| * \param[in] mb_row Macroblock row to be filtered |
| filtering |
| * |
| * \remark Nothing will be returned, but the contents of td->diff will be |
| modified. |
| */ |
| void av1_tf_do_filtering_row(struct AV1_COMP *cpi, struct ThreadData *td, |
| int mb_row); |
| |
| /*!\brief Performs temporal filtering if needed on a source frame. |
| * For example to create a filtered alternate reference frame (ARF) |
| * |
| * In this function, the lookahead index is different from the 0-based |
| * real index. For example, if we want to filter the first frame in the |
| * pre-fetched buffer `cpi->lookahead`, the lookahead index will be -1 instead |
| * of 0. More concretely, 0 indicates the first LOOKAHEAD frame, which is the |
| * second frame in the pre-fetched buffer. Another example: if we want to filter |
| * the 17-th frame, which is an ARF, the lookahead index is 15 instead of 16. |
| * Futhermore, negative number is used for key frame in one-pass mode, where key |
| * frame is filtered with the frames before it instead of after it. For example, |
| * -15 means to filter the 17-th frame, which is a key frame in one-pass mode. |
| * |
| * \ingroup src_frame_proc |
| * \param[in] cpi Top level encoder instance |
| * structure |
| * \param[in] filter_frame_lookahead_idx The index of the |
| * to-filter frame in the lookahead |
| * buffer cpi->lookahead. |
| * \param[in] gf_frame_index Index of GOP |
| * \param[in,out] frame_diff structure of sse and sum of the |
| * filtered frame. |
| * \param[out] output_frame Ouput filtered frame. |
| */ |
| void av1_temporal_filter(struct AV1_COMP *cpi, |
| const int filter_frame_lookahead_idx, |
| int gf_frame_index, FRAME_DIFF *frame_diff, |
| YV12_BUFFER_CONFIG *output_frame); |
| |
| /*!\brief Check whether a filtered frame can be show directly |
| * |
| * This function will use the filtered frame's sse and current q index |
| * to make decision. |
| * |
| * \ingroup src_frame_proc |
| * \param[in] frame filtered frame's buffer |
| * \param[in] frame_diff structure of sse and sum of the |
| * filtered frame. |
| * \param[in] q_index q_index used for this frame |
| * \param[in] bit_depth bit depth |
| * \return return 1 if this frame can be shown directly, otherwise |
| * return 0 |
| */ |
| int av1_check_show_filtered_frame(const YV12_BUFFER_CONFIG *frame, |
| const FRAME_DIFF *frame_diff, int q_index, |
| aom_bit_depth_t bit_depth); |
| |
| /*!\cond */ |
| // Helper function to get `q` used for encoding. |
| int av1_get_q(const struct AV1_COMP *cpi); |
| |
| // Allocates memory for members of TemporalFilterData. |
| // Inputs: |
| // tf_data: Pointer to the structure containing temporal filter related data. |
| // num_pels: Number of pixels in the block across all planes. |
| // is_high_bitdepth: Whether the frame is high-bitdepth or not. |
| // Returns: |
| // True if allocation is successful and false otherwise. |
| static AOM_INLINE bool tf_alloc_and_reset_data(TemporalFilterData *tf_data, |
| int num_pels, |
| int is_high_bitdepth) { |
| tf_data->tmp_mbmi = (MB_MODE_INFO *)aom_calloc(1, sizeof(*tf_data->tmp_mbmi)); |
| tf_data->accum = |
| (uint32_t *)aom_memalign(16, num_pels * sizeof(*tf_data->accum)); |
| tf_data->count = |
| (uint16_t *)aom_memalign(16, num_pels * sizeof(*tf_data->count)); |
| if (is_high_bitdepth) |
| tf_data->pred = CONVERT_TO_BYTEPTR( |
| aom_memalign(32, num_pels * 2 * sizeof(*tf_data->pred))); |
| else |
| tf_data->pred = |
| (uint8_t *)aom_memalign(32, num_pels * sizeof(*tf_data->pred)); |
| // In case of an allocation failure, other successfully allocated buffers will |
| // be freed by the tf_dealloc_data() call in encoder_destroy(). |
| if (!(tf_data->tmp_mbmi && tf_data->accum && tf_data->count && tf_data->pred)) |
| return false; |
| memset(&tf_data->diff, 0, sizeof(tf_data->diff)); |
| return true; |
| } |
| |
| // Setup macroblockd params for temporal filtering process. |
| // Inputs: |
| // mbd: Pointer to the block for filtering. |
| // tf_data: Pointer to the structure containing temporal filter related data. |
| // scale: Scaling factor. |
| // Returns: |
| // Nothing will be returned. Contents of mbd will be modified. |
| static AOM_INLINE void tf_setup_macroblockd(MACROBLOCKD *mbd, |
| TemporalFilterData *tf_data, |
| const struct scale_factors *scale) { |
| mbd->block_ref_scale_factors[0] = scale; |
| mbd->block_ref_scale_factors[1] = scale; |
| mbd->mi = &tf_data->tmp_mbmi; |
| mbd->mi[0]->motion_mode = SIMPLE_TRANSLATION; |
| } |
| |
| // Deallocates the memory allocated for members of TemporalFilterData. |
| // Inputs: |
| // tf_data: Pointer to the structure containing temporal filter related data. |
| // is_high_bitdepth: Whether the frame is high-bitdepth or not. |
| // Returns: |
| // Nothing will be returned. |
| static AOM_INLINE void tf_dealloc_data(TemporalFilterData *tf_data, |
| int is_high_bitdepth) { |
| if (is_high_bitdepth) |
| tf_data->pred = (uint8_t *)CONVERT_TO_SHORTPTR(tf_data->pred); |
| aom_free(tf_data->tmp_mbmi); |
| tf_data->tmp_mbmi = NULL; |
| aom_free(tf_data->accum); |
| tf_data->accum = NULL; |
| aom_free(tf_data->count); |
| tf_data->count = NULL; |
| aom_free(tf_data->pred); |
| tf_data->pred = NULL; |
| } |
| |
| // Saves the state prior to temporal filter process. |
| // Inputs: |
| // mbd: Pointer to the block for filtering. |
| // input_mbmi: Backup block info to save input state. |
| // input_buffer: Backup buffer pointer to save input state. |
| // num_planes: Number of planes. |
| // Returns: |
| // Nothing will be returned. Contents of input_mbmi and input_buffer will be |
| // modified. |
| static INLINE void tf_save_state(MACROBLOCKD *mbd, MB_MODE_INFO ***input_mbmi, |
| uint8_t **input_buffer, int num_planes) { |
| for (int i = 0; i < num_planes; i++) { |
| input_buffer[i] = mbd->plane[i].pre[0].buf; |
| } |
| *input_mbmi = mbd->mi; |
| } |
| |
| // Restores the initial state after temporal filter process. |
| // Inputs: |
| // mbd: Pointer to the block for filtering. |
| // input_mbmi: Backup block info from where input state is restored. |
| // input_buffer: Backup buffer pointer from where input state is restored. |
| // num_planes: Number of planes. |
| // Returns: |
| // Nothing will be returned. Contents of mbd will be modified. |
| static INLINE void tf_restore_state(MACROBLOCKD *mbd, MB_MODE_INFO **input_mbmi, |
| uint8_t **input_buffer, int num_planes) { |
| for (int i = 0; i < num_planes; i++) { |
| mbd->plane[i].pre[0].buf = input_buffer[i]; |
| } |
| mbd->mi = input_mbmi; |
| } |
| |
| /*!\endcond */ |
| #ifdef __cplusplus |
| } // extern "C" |
| #endif |
| |
| #endif // AOM_AV1_ENCODER_TEMPORAL_FILTER_H_ |