blob: 4e62312bd58beba67eb75c0a7672bcf4d6c74de6 [file] [log] [blame]
Yaowu Xuc27fc142016-08-22 16:08:15 -07001/*
James Zernb7c05bd2024-06-11 19:15:10 -07002 * Copyright (c) 2016, Alliance for Open Media. All rights reserved.
Yaowu Xuc27fc142016-08-22 16:08:15 -07003 *
Yaowu Xu2ab7ff02016-09-02 12:04:54 -07004 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Yaowu Xuc27fc142016-08-22 16:08:15 -070010 */
11
chiyotsaic2f38412020-06-10 16:07:21 -070012/*! \file
13 * Declares various structs used to encode the current partition block.
14 */
James Zerne1cbb132018-08-22 14:10:36 -070015#ifndef AOM_AV1_ENCODER_BLOCK_H_
16#define AOM_AV1_ENCODER_BLOCK_H_
Yaowu Xuc27fc142016-08-22 16:08:15 -070017
chiyotsai681e4942020-09-30 11:08:17 -070018#include "av1/common/blockd.h"
Yaowu Xuc27fc142016-08-22 16:08:15 -070019#include "av1/common/entropymv.h"
20#include "av1/common/entropy.h"
chiyotsai85d715a2020-05-02 15:10:33 -070021#include "av1/common/enums.h"
Yaowu Xuc27fc142016-08-22 16:08:15 -070022#include "av1/common/mvref_common.h"
chiyotsai36035d12019-06-19 17:39:48 -070023
Ravi Chaudharyb61cdea2019-07-05 15:01:20 +053024#include "av1/encoder/enc_enums.h"
chiyotsaifbebe9d2022-05-23 13:31:41 -070025#include "av1/encoder/mcomp_structs.h"
chiyotsai36035d12019-06-19 17:39:48 -070026#if !CONFIG_REALTIME_ONLY
27#include "av1/encoder/partition_cnn_weights.h"
28#endif
29
Vincent Rabaud0d7608b2021-06-22 16:07:23 +020030#include "av1/encoder/hash_motion.h"
Yaowu Xuc27fc142016-08-22 16:08:15 -070031
32#ifdef __cplusplus
33extern "C" {
34#endif
35
Yunqing Wangb17bfa42020-08-14 14:50:33 -070036//! Minimum linear dimension of a tpl block
37#define MIN_TPL_BSIZE_1D 16
38//! Maximum number of tpl block in a super block
39#define MAX_TPL_BLK_IN_SB (MAX_SB_SIZE / MIN_TPL_BSIZE_1D)
chiyotsaic2f38412020-06-10 16:07:21 -070040//! Number of txfm hash records kept for the partition block.
41#define RD_RECORD_BUFFER_LEN 8
chiyotsaiff73c532020-04-21 12:50:12 -070042
S Hamsalekhab7a5b232021-08-06 13:11:33 +053043/*! Maximum value taken by transform type probabilities */
44#define MAX_TX_TYPE_PROB 1024
Anupam Pandey45caf952022-12-15 11:16:15 +053045
46//! Compute color sensitivity index for given plane
47#define COLOR_SENS_IDX(plane) ((plane)-1)
Anupam Pandeyc5ee9a02023-01-06 12:15:48 +053048
49//! Enable timer statistics of mode search in non-rd
50#define COLLECT_NONRD_PICK_MODE_STAT 0
51
52/*!\cond */
53#if COLLECT_NONRD_PICK_MODE_STAT
54#include "aom_ports/aom_timer.h"
55
56typedef struct _mode_search_stat_nonrd {
57 int32_t num_blocks[BLOCK_SIZES];
58 int64_t total_block_times[BLOCK_SIZES];
59 int32_t num_searches[BLOCK_SIZES][MB_MODE_COUNT];
60 int32_t num_nonskipped_searches[BLOCK_SIZES][MB_MODE_COUNT];
61 int64_t search_times[BLOCK_SIZES][MB_MODE_COUNT];
62 int64_t nonskipped_search_times[BLOCK_SIZES][MB_MODE_COUNT];
63 int64_t ms_time[BLOCK_SIZES][MB_MODE_COUNT];
64 int64_t ifs_time[BLOCK_SIZES][MB_MODE_COUNT];
65 int64_t model_rd_time[BLOCK_SIZES][MB_MODE_COUNT];
66 int64_t txfm_time[BLOCK_SIZES][MB_MODE_COUNT];
67 struct aom_usec_timer timer1;
68 struct aom_usec_timer timer2;
69 struct aom_usec_timer bsize_timer;
70} mode_search_stat_nonrd;
71#endif // COLLECT_NONRD_PICK_MODE_STAT
72/*!\endcond */
73
chiyotsaic2f38412020-06-10 16:07:21 -070074/*! \brief Superblock level encoder info
75 *
76 * SuperblockEnc stores superblock level information used by the encoder for
77 * more efficient encoding. Currently this is mostly used to store TPL data
78 * for the current superblock.
79 */
chiyotsaiff73c532020-04-21 12:50:12 -070080typedef struct {
chiyotsaic2f38412020-06-10 16:07:21 -070081 //! Maximum partition size for the sb.
chiyotsai68eefbe2020-05-01 15:07:58 -070082 BLOCK_SIZE min_partition_size;
chiyotsaic2f38412020-06-10 16:07:21 -070083 //! Minimum partition size for the sb.
chiyotsai68eefbe2020-05-01 15:07:58 -070084 BLOCK_SIZE max_partition_size;
85
chiyotsaic2f38412020-06-10 16:07:21 -070086 /*****************************************************************************
87 * \name TPL Info
88 *
Yunqing Wangb17bfa42020-08-14 14:50:33 -070089 * Information gathered from tpl_model at tpl block precision for the
chiyotsaic2f38412020-06-10 16:07:21 -070090 * superblock to speed up the encoding process..
91 ****************************************************************************/
92 /**@{*/
93 //! Number of TPL blocks in this superblock.
chiyotsaiff73c532020-04-21 12:50:12 -070094 int tpl_data_count;
chiyotsaic2f38412020-06-10 16:07:21 -070095 //! TPL's estimate of inter cost for each tpl block.
Yunqing Wangb17bfa42020-08-14 14:50:33 -070096 int64_t tpl_inter_cost[MAX_TPL_BLK_IN_SB * MAX_TPL_BLK_IN_SB];
chiyotsaic2f38412020-06-10 16:07:21 -070097 //! TPL's estimate of tpl cost for each tpl block.
Yunqing Wangb17bfa42020-08-14 14:50:33 -070098 int64_t tpl_intra_cost[MAX_TPL_BLK_IN_SB * MAX_TPL_BLK_IN_SB];
chiyotsaic2f38412020-06-10 16:07:21 -070099 //! Motion vectors found by TPL model for each tpl block.
Yunqing Wangb17bfa42020-08-14 14:50:33 -0700100 int_mv tpl_mv[MAX_TPL_BLK_IN_SB * MAX_TPL_BLK_IN_SB][INTER_REFS_PER_FRAME];
chiyotsaic2f38412020-06-10 16:07:21 -0700101 //! TPL's stride for the arrays in this struct.
chiyotsaiff73c532020-04-21 12:50:12 -0700102 int tpl_stride;
chiyotsaic2f38412020-06-10 16:07:21 -0700103 /**@}*/
chiyotsaiff73c532020-04-21 12:50:12 -0700104} SuperBlockEnc;
105
chiyotsaic2f38412020-06-10 16:07:21 -0700106/*! \brief Stores the best performing modes.
107 */
Cherma Rajan A835f7a62019-09-25 11:04:39 +0530108typedef struct {
chiyotsaic2f38412020-06-10 16:07:21 -0700109 //! The mbmi used to reconstruct the winner mode.
Cherma Rajan A835f7a62019-09-25 11:04:39 +0530110 MB_MODE_INFO mbmi;
chiyotsaic2f38412020-06-10 16:07:21 -0700111 //! Rdstats of the winner mode.
Cherma Rajan Ad71ace62019-11-25 12:58:37 +0530112 RD_STATS rd_cost;
chiyotsaic2f38412020-06-10 16:07:21 -0700113 //! Rdcost of the winner mode
Cherma Rajan A835f7a62019-09-25 11:04:39 +0530114 int64_t rd;
chiyotsaic2f38412020-06-10 16:07:21 -0700115 //! Luma rate of the winner mode.
Cherma Rajan Ad71ace62019-11-25 12:58:37 +0530116 int rate_y;
chiyotsaic2f38412020-06-10 16:07:21 -0700117 //! Chroma rate of the winner mode.
Cherma Rajan Ad71ace62019-11-25 12:58:37 +0530118 int rate_uv;
chiyotsaic2f38412020-06-10 16:07:21 -0700119 //! The color map needed to reconstruct palette mode.
120 uint8_t color_index_map[MAX_SB_SQUARE];
121 //! The current winner mode.
Cherma Rajan Ad71ace62019-11-25 12:58:37 +0530122 THR_MODES mode_index;
Cherma Rajan A835f7a62019-09-25 11:04:39 +0530123} WinnerModeStats;
Yue Chenbd934232019-08-05 14:23:39 -0700124
chiyotsaic2f38412020-06-10 16:07:21 -0700125/*! \brief Each source plane of the current macroblock
126 *
127 * This struct also stores the txfm buffers and quantizer settings.
128 */
Yaowu Xuc27fc142016-08-22 16:08:15 -0700129typedef struct macroblock_plane {
chiyotsaic2f38412020-06-10 16:07:21 -0700130 //! Stores source - pred so the txfm can be computed later
Chethan Kumar R Ea494e2b2021-03-18 17:19:44 +0530131 int16_t *src_diff;
chiyotsaic2f38412020-06-10 16:07:21 -0700132 //! Dequantized coefficients
Urvang Joshi9543ad72020-04-17 16:59:46 -0700133 tran_low_t *dqcoeff;
chiyotsaic2f38412020-06-10 16:07:21 -0700134 //! Quantized coefficients
Yaowu Xuc27fc142016-08-22 16:08:15 -0700135 tran_low_t *qcoeff;
chiyotsaic2f38412020-06-10 16:07:21 -0700136 //! Transformed coefficients
Yaowu Xuc27fc142016-08-22 16:08:15 -0700137 tran_low_t *coeff;
chiyotsaic2f38412020-06-10 16:07:21 -0700138 //! Location of the end of qcoeff (end of block).
Yaowu Xuc27fc142016-08-22 16:08:15 -0700139 uint16_t *eobs;
chiyotsaic2f38412020-06-10 16:07:21 -0700140 //! Contexts used to code the transform coefficients.
Angie Chiang74e23072017-03-24 14:54:23 -0700141 uint8_t *txb_entropy_ctx;
chiyotsaic2f38412020-06-10 16:07:21 -0700142 //! A buffer containing the source frame.
Yaowu Xuc27fc142016-08-22 16:08:15 -0700143 struct buf_2d src;
144
chiyotsaic2f38412020-06-10 16:07:21 -0700145 /*! \name Quantizer Settings
146 *
147 * \attention These are used/accessed only in the quantization process.
148 * RDO does not and *must not* depend on any of these values.
149 * All values below share the coefficient scale/shift used in TX.
150 */
151 /**@{*/
152 //! Quantization step size used by AV1_XFORM_QUANT_FP.
Monty Montgomery125c0fc2017-10-26 00:44:35 -0400153 const int16_t *quant_fp_QTX;
chiyotsaic2f38412020-06-10 16:07:21 -0700154 //! Offset used for rounding in the quantizer process by AV1_XFORM_QUANT_FP.
Monty Montgomery125c0fc2017-10-26 00:44:35 -0400155 const int16_t *round_fp_QTX;
chiyotsaic2f38412020-06-10 16:07:21 -0700156 //! Quantization step size used by AV1_XFORM_QUANT_B.
Monty Montgomery125c0fc2017-10-26 00:44:35 -0400157 const int16_t *quant_QTX;
chiyotsaic2f38412020-06-10 16:07:21 -0700158 //! Offset used for rounding in the quantizer process by AV1_XFORM_QUANT_B.
Monty Montgomery125c0fc2017-10-26 00:44:35 -0400159 const int16_t *round_QTX;
chiyotsaic2f38412020-06-10 16:07:21 -0700160 //! Scale factor to shift coefficients toward zero. Only used by QUANT_B.
161 const int16_t *quant_shift_QTX;
162 //! Size of the quantization bin around 0. Only Used by QUANT_B
163 const int16_t *zbin_QTX;
164 //! Dequantizer
Monty Montgomery125c0fc2017-10-26 00:44:35 -0400165 const int16_t *dequant_QTX;
chiyotsaic2f38412020-06-10 16:07:21 -0700166 /**@}*/
Yaowu Xuc27fc142016-08-22 16:08:15 -0700167} MACROBLOCK_PLANE;
168
chiyotsaic2f38412020-06-10 16:07:21 -0700169/*! \brief Costs for encoding the coefficients within a level.
170 *
171 * Covers everything including txb_skip, eob, dc_sign,
172 */
Jingning Handfd72322017-08-09 14:04:12 -0700173typedef struct {
chiyotsaic2f38412020-06-10 16:07:21 -0700174 //! Cost to skip txfm for the current txfm block.
Jingning Handfd72322017-08-09 14:04:12 -0700175 int txb_skip_cost[TXB_SKIP_CONTEXTS][2];
chiyotsaic2f38412020-06-10 16:07:21 -0700176 /*! \brief Cost for encoding the base_eob of a level.
177 *
178 * Decoder uses base_eob to derive the base_level as base_eob := base_eob+1.
179 */
Dake He3fe369c2017-11-16 17:56:44 -0800180 int base_eob_cost[SIG_COEF_CONTEXTS_EOB][3];
chiyotsaic2f38412020-06-10 16:07:21 -0700181 /*! \brief Cost for encoding the base level of a coefficient.
182 *
183 * Decoder derives coeff_base as coeff_base := base_eob + 1.
184 */
Wenyao Liuf7e53752019-01-22 17:34:44 +0800185 int base_cost[SIG_COEF_CONTEXTS][8];
chiyotsaic2f38412020-06-10 16:07:21 -0700186 /*! \brief Cost for encoding the last non-zero coefficient.
187 *
188 * Eob is derived from eob_extra at the decoder as eob := eob_extra + 1
189 */
Angie Chiang7ab884e2017-10-18 15:57:12 -0700190 int eob_extra_cost[EOB_COEF_CONTEXTS][2];
chiyotsaic2f38412020-06-10 16:07:21 -0700191 //! Cost for encoding the dc_sign
Jingning Handfd72322017-08-09 14:04:12 -0700192 int dc_sign_cost[DC_SIGN_CONTEXTS][2];
chiyotsaic2f38412020-06-10 16:07:21 -0700193 //! Cost for encoding an increment to the coefficient
Wenyao Liuf7e53752019-01-22 17:34:44 +0800194 int lps_cost[LEVEL_CONTEXTS][COEFF_BASE_RANGE + 1 + COEFF_BASE_RANGE + 1];
Jingning Handfd72322017-08-09 14:04:12 -0700195} LV_MAP_COEFF_COST;
Jingning Hanf5a4d3b2017-08-27 23:01:19 -0700196
chiyotsaic2f38412020-06-10 16:07:21 -0700197/*! \brief Costs for encoding the eob.
198 */
Johannb0ef6ff2018-02-08 14:32:21 -0800199typedef struct {
chiyotsaic2f38412020-06-10 16:07:21 -0700200 //! eob_cost.
Johannb0ef6ff2018-02-08 14:32:21 -0800201 int eob_cost[2][11];
202} LV_MAP_EOB_COST;
Dake He0db7d0e2017-12-21 15:23:20 -0800203
chiyotsaic2f38412020-06-10 16:07:21 -0700204/*! \brief Stores the transforms coefficients for the whole superblock.
205 */
Jingning Hanf5a4d3b2017-08-27 23:01:19 -0700206typedef struct {
chiyotsaic2f38412020-06-10 16:07:21 -0700207 //! The transformed coefficients.
Jayasanker J2350ca32020-09-10 23:49:00 +0530208 tran_low_t *tcoeff[MAX_MB_PLANE];
chiyotsaic2f38412020-06-10 16:07:21 -0700209 //! Where the transformed coefficients end.
Jayasanker J2350ca32020-09-10 23:49:00 +0530210 uint16_t *eobs[MAX_MB_PLANE];
chiyotsaic2f38412020-06-10 16:07:21 -0700211 /*! \brief Transform block entropy contexts.
212 *
213 * Each element is used as a bit field.
214 * - Bits 0~3: txb_skip_ctx
215 * - Bits 4~5: dc_sign_ctx.
216 */
Jayasanker J2350ca32020-09-10 23:49:00 +0530217 uint8_t *entropy_ctx[MAX_MB_PLANE];
Jingning Hanf5a4d3b2017-08-27 23:01:19 -0700218} CB_COEFF_BUFFER;
Jingning Handfd72322017-08-09 14:04:12 -0700219
chiyotsaic2f38412020-06-10 16:07:21 -0700220/*! \brief Extended mode info derived from mbmi.
221 */
Yaowu Xuc27fc142016-08-22 16:08:15 -0700222typedef struct {
Angie Chiangc484abe2017-03-20 15:43:11 -0700223 // TODO(angiebird): Reduce the buffer size according to sb_type
chiyotsaic2f38412020-06-10 16:07:21 -0700224 //! The reference mv list for the current block.
Ravi Chaudharyfa73e202019-08-19 12:41:26 +0530225 CANDIDATE_MV ref_mv_stack[MODE_CTX_REF_FRAMES][USABLE_REF_MV_STACK_SIZE];
chiyotsaic2f38412020-06-10 16:07:21 -0700226 //! The weights used to compute the ref mvs.
Ravi Chaudharyfa73e202019-08-19 12:41:26 +0530227 uint16_t weight[MODE_CTX_REF_FRAMES][USABLE_REF_MV_STACK_SIZE];
chiyotsaic2f38412020-06-10 16:07:21 -0700228 //! Number of ref mvs in the drl.
Satish Kumar Suman69e93292018-11-28 16:05:33 +0530229 uint8_t ref_mv_count[MODE_CTX_REF_FRAMES];
chiyotsaic2f38412020-06-10 16:07:21 -0700230 //! Global mvs
231 int_mv global_mvs[REF_FRAMES];
232 //! Context used to encode the current mode.
233 int16_t mode_context[MODE_CTX_REF_FRAMES];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700234} MB_MODE_INFO_EXT;
235
chiyotsaic2f38412020-06-10 16:07:21 -0700236/*! \brief Stores best extended mode information at frame level.
237 *
238 * The frame level in here is used in bitstream preparation stage. The
239 * information in \ref MB_MODE_INFO_EXT are copied to this struct to save
240 * memory.
241 */
Remya0cce44c2019-08-16 11:57:24 +0530242typedef struct {
chiyotsaic2f38412020-06-10 16:07:21 -0700243 //! \copydoc MB_MODE_INFO_EXT::ref_mv_stack
Ravi Chaudharyfa73e202019-08-19 12:41:26 +0530244 CANDIDATE_MV ref_mv_stack[USABLE_REF_MV_STACK_SIZE];
chiyotsaic2f38412020-06-10 16:07:21 -0700245 //! \copydoc MB_MODE_INFO_EXT::weight
Ravi Chaudharyfa73e202019-08-19 12:41:26 +0530246 uint16_t weight[USABLE_REF_MV_STACK_SIZE];
chiyotsaic2f38412020-06-10 16:07:21 -0700247 //! \copydoc MB_MODE_INFO_EXT::ref_mv_count
Remya0cce44c2019-08-16 11:57:24 +0530248 uint8_t ref_mv_count;
chiyotsaic2f38412020-06-10 16:07:21 -0700249 // TODO(Ravi/Remya): Reduce the buffer size of global_mvs
250 //! \copydoc MB_MODE_INFO_EXT::global_mvs
251 int_mv global_mvs[REF_FRAMES];
252 //! \copydoc MB_MODE_INFO_EXT::mode_context
253 int16_t mode_context;
254 //! Offset of current coding block's coeff buffer relative to the sb.
Jayasanker J2350ca32020-09-10 23:49:00 +0530255 uint16_t cb_offset[PLANE_TYPES];
Remya0cce44c2019-08-16 11:57:24 +0530256} MB_MODE_INFO_EXT_FRAME;
257
venkat sanampudi4dad14a2021-10-21 20:26:18 +0530258/*! \brief Inter-mode txfm results for a partition block.
chiyotsaic2f38412020-06-10 16:07:21 -0700259 */
Alex Converse0fa0f422017-04-24 12:51:14 -0700260typedef struct {
chiyotsaic2f38412020-06-10 16:07:21 -0700261 //! Txfm size used if the current mode is intra mode.
Hui Su1ddf2312017-08-19 15:21:34 -0700262 TX_SIZE tx_size;
chiyotsaic2f38412020-06-10 16:07:21 -0700263 //! Txfm sizes used if the current mode is inter mode.
Hui Su7167d952018-02-01 16:33:12 -0800264 TX_SIZE inter_tx_size[INTER_TX_SIZE_BUF_LEN];
chiyotsaic2f38412020-06-10 16:07:21 -0700265 //! Map showing which txfm block skips the txfm process.
Hui Suf4b79c72018-03-22 13:14:36 -0700266 uint8_t blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE];
Sarah Parkerf492c1f2021-03-02 21:01:56 -0800267 //! Map showing the txfm types for each block.
Hui Su52b7ddc2019-10-10 16:27:16 -0700268 uint8_t tx_type_map[MAX_MIB_SIZE * MAX_MIB_SIZE];
chiyotsaic2f38412020-06-10 16:07:21 -0700269 //! Rd_stats for the whole partition block.
Hui Su1ddf2312017-08-19 15:21:34 -0700270 RD_STATS rd_stats;
chiyotsaic2f38412020-06-10 16:07:21 -0700271 //! Hash value of the current record.
Hui Su1ddf2312017-08-19 15:21:34 -0700272 uint32_t hash_value;
Hui Su6cb17c12018-03-09 12:56:20 -0800273} MB_RD_INFO;
Hui Su1ddf2312017-08-19 15:21:34 -0700274
venkat sanampudi4dad14a2021-10-21 20:26:18 +0530275/*! \brief Hash records of the inter-mode transform results
276 *
277 * Hash records of the inter-mode transform results for a whole partition block
278 * based on the residue. Since this operates on the partition block level, this
279 * can give us a whole txfm partition tree.
chiyotsaic2f38412020-06-10 16:07:21 -0700280 */
Hui Su1ddf2312017-08-19 15:21:34 -0700281typedef struct {
venkat sanampudi4dad14a2021-10-21 20:26:18 +0530282 /*! Circular buffer that stores the inter-mode txfm results of a partition
283 * block.
284 */
285 MB_RD_INFO mb_rd_info[RD_RECORD_BUFFER_LEN];
Jingning Han0ecc39c2021-09-22 16:07:06 -0700286 //! Index to insert the newest rd record.
Hui Su1ddf2312017-08-19 15:21:34 -0700287 int index_start;
chiyotsaic2f38412020-06-10 16:07:21 -0700288 //! Number of info stored in this record.
Hui Su1ddf2312017-08-19 15:21:34 -0700289 int num;
chiyotsaic2f38412020-06-10 16:07:21 -0700290 //! Hash function
291 CRC32C crc_calculator;
Hui Su6cb17c12018-03-09 12:56:20 -0800292} MB_RD_RECORD;
Hui Su1ddf2312017-08-19 15:21:34 -0700293
chiyotsaic2f38412020-06-10 16:07:21 -0700294//! Number of compound rd stats
Ranjit Kumar Tulabandua1ebb572018-12-24 12:13:54 +0530295#define MAX_COMP_RD_STATS 64
chiyotsaic2f38412020-06-10 16:07:21 -0700296/*! \brief Rdcost stats in compound mode.
297 */
Ranjit Kumar Tulabandua1ebb572018-12-24 12:13:54 +0530298typedef struct {
chiyotsaic2f38412020-06-10 16:07:21 -0700299 //! Rate of the compound modes.
Ranjit Kumar Tulabandua1ebb572018-12-24 12:13:54 +0530300 int32_t rate[COMPOUND_TYPES];
chiyotsaic2f38412020-06-10 16:07:21 -0700301 //! Distortion of the compound modes.
Ranjit Kumar Tulabandua1ebb572018-12-24 12:13:54 +0530302 int64_t dist[COMPOUND_TYPES];
chiyotsaic2f38412020-06-10 16:07:21 -0700303 //! Estimated rate of the compound modes.
Venkat457e32e2019-12-19 17:44:05 +0530304 int32_t model_rate[COMPOUND_TYPES];
chiyotsaic2f38412020-06-10 16:07:21 -0700305 //! Estimated distortion of the compound modes.
Venkat457e32e2019-12-19 17:44:05 +0530306 int64_t model_dist[COMPOUND_TYPES];
chiyotsaic2f38412020-06-10 16:07:21 -0700307 //! Rate need to send the mask type.
venkat sanampudic88148e2020-01-03 12:57:28 +0530308 int comp_rs2[COMPOUND_TYPES];
chiyotsaic2f38412020-06-10 16:07:21 -0700309 //! Motion vector for each predictor.
Ranjit Kumar Tulabandua1ebb572018-12-24 12:13:54 +0530310 int_mv mv[2];
chiyotsaic2f38412020-06-10 16:07:21 -0700311 //! Ref frame for each predictor.
Hui Sud06ff662019-01-23 16:53:05 -0800312 MV_REFERENCE_FRAME ref_frames[2];
chiyotsaic2f38412020-06-10 16:07:21 -0700313 //! Current prediction mode.
Ranjit Kumar Tulabandua1ebb572018-12-24 12:13:54 +0530314 PREDICTION_MODE mode;
chiyotsaic2f38412020-06-10 16:07:21 -0700315 //! Current interpolation filter.
Ravi Chaudhary1e4f94b2019-06-20 16:19:49 +0530316 int_interpfilters filter;
chiyotsaic2f38412020-06-10 16:07:21 -0700317 //! Refmv index in the drl.
Ranjit Kumar Tulabandua1ebb572018-12-24 12:13:54 +0530318 int ref_mv_idx;
chiyotsaic2f38412020-06-10 16:07:21 -0700319 //! Whether the predictors are GLOBALMV.
Ranjit Kumar Tulabandua1ebb572018-12-24 12:13:54 +0530320 int is_global[2];
chiyotsaic2f38412020-06-10 16:07:21 -0700321 //! Current parameters for interinter mode.
venkat sanampudic88148e2020-01-03 12:57:28 +0530322 INTERINTER_COMPOUND_DATA interinter_comp;
Ranjit Kumar Tulabandua1ebb572018-12-24 12:13:54 +0530323} COMP_RD_STATS;
324
chiyotsaic2f38412020-06-10 16:07:21 -0700325/*! \brief Contains buffers used to speed up rdopt for obmc.
326 *
327 * See the comments for calc_target_weighted_pred for details.
328 */
Hui Su38711e72019-06-11 10:49:47 -0700329typedef struct {
chiyotsaic2f38412020-06-10 16:07:21 -0700330 /*! \brief A new source weighted with the above and left predictors.
331 *
332 * Used to efficiently construct multiple obmc predictors during rdopt.
333 */
chiyotsaid2b12212020-04-28 20:57:19 -0700334 int32_t *wsrc;
chiyotsaic2f38412020-06-10 16:07:21 -0700335 /*! \brief A new mask constructed from the original horz/vert mask.
336 *
337 * \copydetails wsrc
338 */
chiyotsaid2b12212020-04-28 20:57:19 -0700339 int32_t *mask;
chiyotsaic2f38412020-06-10 16:07:21 -0700340 /*! \brief Prediction from the up predictor.
341 *
342 * Used to build the obmc predictor.
343 */
chiyotsaid2b12212020-04-28 20:57:19 -0700344 uint8_t *above_pred;
chiyotsaic2f38412020-06-10 16:07:21 -0700345 /*! \brief Prediction from the up predictor.
346 *
347 * \copydetails above_pred
348 */
chiyotsaid2b12212020-04-28 20:57:19 -0700349 uint8_t *left_pred;
350} OBMCBuffer;
351
chiyotsaic2f38412020-06-10 16:07:21 -0700352/*! \brief Contains color maps used in palette mode.
353 */
354typedef struct {
355 //! The best color map found.
356 uint8_t best_palette_color_map[MAX_PALETTE_SQUARE];
357 //! A temporary buffer used for k-means clustering.
Vincent Rabaud8693e6a2022-12-04 23:35:06 +0100358 int16_t kmeans_data_buf[2 * MAX_PALETTE_SQUARE];
chiyotsaic2f38412020-06-10 16:07:21 -0700359} PALETTE_BUFFER;
360
361/*! \brief Contains buffers used by av1_compound_type_rd()
362 *
363 * For sizes and alignment of these arrays, refer to
364 * alloc_compound_type_rd_buffers() function.
365 */
366typedef struct {
367 //! First prediction.
368 uint8_t *pred0;
369 //! Second prediction.
370 uint8_t *pred1;
371 //! Source - first prediction.
372 int16_t *residual1;
373 //! Second prediction - first prediction.
374 int16_t *diff10;
375 //! Backup of the best segmentation mask.
376 uint8_t *tmp_best_mask_buf;
377} CompoundTypeRdBuffers;
378
379/*! \brief Holds some parameters related to partitioning schemes in AV1.
380 */
chiyotsai68eefbe2020-05-01 15:07:58 -0700381// TODO(chiyotsai@google.com): Consolidate this with SIMPLE_MOTION_DATA_TREE
382typedef struct {
383#if !CONFIG_REALTIME_ONLY
384 // The following 4 parameters are used for cnn-based partitioning on intra
385 // frame.
chiyotsaic2f38412020-06-10 16:07:21 -0700386 /*! \brief Current index on the partition block quad tree.
387 *
388 * Used to index into the cnn buffer for partition decision.
389 */
chiyotsai68eefbe2020-05-01 15:07:58 -0700390 int quad_tree_idx;
chiyotsaic2f38412020-06-10 16:07:21 -0700391 //! Whether the CNN buffer contains valid output.
chiyotsai68eefbe2020-05-01 15:07:58 -0700392 int cnn_output_valid;
chiyotsaic2f38412020-06-10 16:07:21 -0700393 //! A buffer used by our segmentation CNN for intra-frame partitioning.
chiyotsai68eefbe2020-05-01 15:07:58 -0700394 float cnn_buffer[CNN_OUT_BUF_SIZE];
chiyotsaic2f38412020-06-10 16:07:21 -0700395 //! log of the quantization parameter of the ancestor BLOCK_64X64.
chiyotsai68eefbe2020-05-01 15:07:58 -0700396 float log_q;
397#endif
398
chiyotsaic2f38412020-06-10 16:07:21 -0700399 /*! \brief Variance of the subblocks in the superblock.
400 *
401 * This is used by rt mode for variance based partitioning.
402 * The indices corresponds to the following block sizes:
403 * - 0 - 128x128
404 * - 1-2 - 128x64
405 * - 3-4 - 64x128
406 * - 5-8 - 64x64
407 * - 9-16 - 64x32
408 * - 17-24 - 32x64
409 * - 25-40 - 32x32
410 * - 41-104 - 16x16
411 */
chiyotsai68eefbe2020-05-01 15:07:58 -0700412 uint8_t variance_low[105];
413} PartitionSearchInfo;
414
Ranjit Kumar Tulabandua00e5962022-06-07 19:58:10 +0530415/*!\cond */
416enum {
417 /**
418 * Do not prune transform depths.
419 */
420 TX_PRUNE_NONE = 0,
421 /**
422 * Prune largest transform (depth 0) based on NN model.
423 */
424 TX_PRUNE_LARGEST = 1,
425 /**
426 * Prune split transforms (depth>=1) based on NN model.
427 */
428 TX_PRUNE_SPLIT = 2,
429} UENUM1BYTE(TX_PRUNE_TYPE);
430/*!\endcond */
431
chiyotsaic2f38412020-06-10 16:07:21 -0700432/*! \brief Defines the parameters used to perform txfm search.
433 *
434 * For the most part, this determines how various speed features are used.
435 */
chiyotsaia36d9002020-04-29 16:48:21 -0700436typedef struct {
chiyotsaic2f38412020-06-10 16:07:21 -0700437 /*! \brief Whether to limit the intra txfm search type to the default txfm.
438 *
439 * This could either be a result of either sequence parameter or speed
440 * features.
441 */
chiyotsaia36d9002020-04-29 16:48:21 -0700442 int use_default_intra_tx_type;
S Hamsalekhaf48f5302021-08-10 16:51:42 +0530443
S Hamsalekhab7a5b232021-08-06 13:11:33 +0530444 /*! Probability threshold used for conditionally forcing tx type*/
445 int default_inter_tx_type_prob_thresh;
chiyotsaia36d9002020-04-29 16:48:21 -0700446
chiyotsaic2f38412020-06-10 16:07:21 -0700447 //! Whether to prune 2d transforms based on 1d transform results.
chiyotsaia36d9002020-04-29 16:48:21 -0700448 int prune_2d_txfm_mode;
449
chiyotsaic2f38412020-06-10 16:07:21 -0700450 /*! \brief Variable from \ref WinnerModeParams based on current eval mode.
451 *
452 * See the documentation for \ref WinnerModeParams for more detail.
453 */
Akshata Jadhavbd0eb432021-01-13 14:33:47 +0530454 unsigned int coeff_opt_thresholds[2];
455 /*! \copydoc coeff_opt_thresholds */
chiyotsaia36d9002020-04-29 16:48:21 -0700456 unsigned int tx_domain_dist_threshold;
Akshata Jadhavbd0eb432021-01-13 14:33:47 +0530457 /*! \copydoc coeff_opt_thresholds */
chiyotsaia36d9002020-04-29 16:48:21 -0700458 TX_SIZE_SEARCH_METHOD tx_size_search_method;
Akshata Jadhavbd0eb432021-01-13 14:33:47 +0530459 /*! \copydoc coeff_opt_thresholds */
chiyotsaia36d9002020-04-29 16:48:21 -0700460 unsigned int use_transform_domain_distortion;
Akshata Jadhavbd0eb432021-01-13 14:33:47 +0530461 /*! \copydoc coeff_opt_thresholds */
chiyotsaia36d9002020-04-29 16:48:21 -0700462 unsigned int skip_txfm_level;
463
chiyotsaic2f38412020-06-10 16:07:21 -0700464 /*! \brief How to search for the optimal tx_size
465 *
466 * If ONLY_4X4, use TX_4X4; if TX_MODE_LARGEST, use the largest tx_size for
467 * the current partition block; if TX_MODE_SELECT, search through the whole
468 * tree.
469 *
470 * \attention
471 * Although this looks suspicious similar to a bitstream element, this
472 * tx_mode_search_type is only used internally by the encoder, and is *not*
473 * written to the bitstream. It determines what kind of tx_mode would be
474 * searched. For example, we might set it to TX_MODE_LARGEST to find a good
475 * candidate, then code it as TX_MODE_SELECT.
476 */
chiyotsaia36d9002020-04-29 16:48:21 -0700477 TX_MODE tx_mode_search_type;
Ravi Chaudhary6eaea622020-08-28 10:28:24 +0530478
479 /*!
Mudassir Galagnath788fcc12022-07-26 17:20:52 +0530480 * Determines whether a block can be predicted as transform skip or DC only
481 * based on residual mean and variance.
482 * Type 0 : No skip block or DC only block prediction
483 * Type 1 : Prediction of skip block based on residual mean and variance
484 * Type 2 : Prediction of skip block or DC only block based on residual mean
485 * and variance
Ravi Chaudhary6eaea622020-08-28 10:28:24 +0530486 */
487 unsigned int predict_dc_level;
Luca Versari6807a7f2021-12-20 21:41:20 +0100488
489 /*!
490 * Whether or not we should use the quantization matrix as weights for PSNR
491 * during RD search.
492 */
493 int use_qm_dist_metric;
venkat sanampudi7b82db52022-04-26 22:44:41 +0530494
495 /*!
496 * Keep track of previous mode evaluation stage type. This will be used to
497 * reset mb rd hash record when mode evaluation type changes.
498 */
499 int mode_eval_type;
Ranjit Kumar Tulabandua00e5962022-06-07 19:58:10 +0530500
501#if !CONFIG_REALTIME_ONLY
502 //! Indicates the transform depths for which RD evaluation is skipped.
503 TX_PRUNE_TYPE nn_prune_depths_for_intra_tx;
504
505 /*! \brief Indicates if NN model should be invoked to prune transform depths.
506 *
507 * Used to signal whether NN model should be evaluated to prune the R-D
508 * evaluation of specific transform depths.
509 */
510 bool enable_nn_prune_intra_tx_depths;
511#endif
chiyotsaia36d9002020-04-29 16:48:21 -0700512} TxfmSearchParams;
513
chiyotsaic2f38412020-06-10 16:07:21 -0700514/*!\cond */
chiyotsai4c1e5c62020-04-30 17:54:14 -0700515#define MAX_NUM_8X8_TXBS ((MAX_MIB_SIZE >> 1) * (MAX_MIB_SIZE >> 1))
516#define MAX_NUM_16X16_TXBS ((MAX_MIB_SIZE >> 2) * (MAX_MIB_SIZE >> 2))
517#define MAX_NUM_32X32_TXBS ((MAX_MIB_SIZE >> 3) * (MAX_MIB_SIZE >> 3))
518#define MAX_NUM_64X64_TXBS ((MAX_MIB_SIZE >> 4) * (MAX_MIB_SIZE >> 4))
chiyotsaic2f38412020-06-10 16:07:21 -0700519/*!\endcond */
chiyotsai4c1e5c62020-04-30 17:54:14 -0700520
chiyotsaic2f38412020-06-10 16:07:21 -0700521/*! \brief Stores various encoding/search decisions related to txfm search.
522 *
523 * This struct contains a cache of previous txfm results, and some buffers for
524 * the current txfm decision.
525 */
chiyotsai4c1e5c62020-04-30 17:54:14 -0700526typedef struct {
chiyotsaic2f38412020-06-10 16:07:21 -0700527 //! Whether to skip transform and quantization on a partition block level.
Kyle Siefring932f4022022-08-24 14:50:26 -0400528 uint8_t skip_txfm;
chiyotsai4c1e5c62020-04-30 17:54:14 -0700529
chiyotsaic2f38412020-06-10 16:07:21 -0700530 /*! \brief Whether to skip transform and quantization on a txfm block level.
531 *
532 * Skips transform and quantization on a transform block level inside the
533 * current partition block. Each element of this array is used as a bit-field.
534 * So for example, the we are skipping on the luma plane, then the last bit
535 * would be set to 1.
536 */
chiyotsai4c1e5c62020-04-30 17:54:14 -0700537 uint8_t blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE];
538
chiyotsaic2f38412020-06-10 16:07:21 -0700539 /*! \brief Transform types inside the partition block
540 *
541 * Keeps a record of what kind of transform to use for each of the transform
542 * block inside the partition block.
543 * \attention The buffer here is *never* directly used. Instead, this just
544 * allocates the memory for MACROBLOCKD::tx_type_map during rdopt on the
545 * partition block. So if we need to save memory, we could move the allocation
546 * to pick_sb_mode instead.
547 */
chiyotsai4c1e5c62020-04-30 17:54:14 -0700548 uint8_t tx_type_map_[MAX_MIB_SIZE * MAX_MIB_SIZE];
549
venkat sanampudi4dad14a2021-10-21 20:26:18 +0530550 //! Txfm hash records of inter-modes.
venkat sanampudie61793d2021-10-13 14:26:05 +0530551 MB_RD_RECORD *mb_rd_record;
chiyotsai4c1e5c62020-04-30 17:54:14 -0700552
chiyotsaic2f38412020-06-10 16:07:21 -0700553 /*! \brief Number of txb splits.
554 *
555 * Keep track of how many times we've used split tx partition for transform
556 * blocks. Somewhat misleadingly, this parameter doesn't actually keep track
557 * of the count of the current block. Instead, it's a cumulative count across
558 * of the whole frame. The main usage is that if txb_split_count is zero, then
559 * we can signal TX_MODE_LARGEST at frame level.
560 */
chiyotsai4c1e5c62020-04-30 17:54:14 -0700561 // TODO(chiyotsai@google.com): Move this to a more appropriate location such
562 // as ThreadData.
563 unsigned int txb_split_count;
564#if CONFIG_SPEED_STATS
chiyotsaic2f38412020-06-10 16:07:21 -0700565 //! For debugging. Used to check how many txfm searches we are doing.
chiyotsai4c1e5c62020-04-30 17:54:14 -0700566 unsigned int tx_search_count;
567#endif // CONFIG_SPEED_STATS
568} TxfmSearchInfo;
chiyotsaic2f38412020-06-10 16:07:21 -0700569#undef MAX_NUM_8X8_TXBS
570#undef MAX_NUM_16X16_TXBS
571#undef MAX_NUM_32X32_TXBS
572#undef MAX_NUM_64X64_TXBS
chiyotsai4c1e5c62020-04-30 17:54:14 -0700573
chiyotsaic2f38412020-06-10 16:07:21 -0700574/*! \brief Holds the entropy costs for various modes sent to the bitstream.
575 *
576 * \attention This does not include the costs for mv and transformed
577 * coefficients.
578 */
chiyotsai9a06d182020-05-01 17:12:12 -0700579typedef struct {
chiyotsaic2f38412020-06-10 16:07:21 -0700580 /*****************************************************************************
581 * \name Partition Costs
582 ****************************************************************************/
583 /**@{*/
584 //! Cost for coding the partition.
chiyotsai9a06d182020-05-01 17:12:12 -0700585 int partition_cost[PARTITION_CONTEXTS][EXT_PARTITION_TYPES];
chiyotsaic2f38412020-06-10 16:07:21 -0700586 /**@}*/
chiyotsai9a06d182020-05-01 17:12:12 -0700587
chiyotsaic2f38412020-06-10 16:07:21 -0700588 /*****************************************************************************
589 * \name Intra Costs: General
590 ****************************************************************************/
591 /**@{*/
592 //! Luma mode cost for inter frame.
593 int mbmode_cost[BLOCK_SIZE_GROUPS][INTRA_MODES];
594 //! Luma mode cost for intra frame.
595 int y_mode_costs[INTRA_MODES][INTRA_MODES][INTRA_MODES];
596 //! Chroma mode cost
chiyotsai9a06d182020-05-01 17:12:12 -0700597 int intra_uv_mode_cost[CFL_ALLOWED_TYPES][INTRA_MODES][UV_INTRA_MODES];
chiyotsaic2f38412020-06-10 16:07:21 -0700598 //! filter_intra_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700599 int filter_intra_cost[BLOCK_SIZES_ALL][2];
chiyotsaic2f38412020-06-10 16:07:21 -0700600 //! filter_intra_mode_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700601 int filter_intra_mode_cost[FILTER_INTRA_MODES];
chiyotsaic2f38412020-06-10 16:07:21 -0700602 //! angle_delta_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700603 int angle_delta_cost[DIRECTIONAL_MODES][2 * MAX_ANGLE_DELTA + 1];
604
chiyotsaic2f38412020-06-10 16:07:21 -0700605 //! Rate rate associated with each alpha codeword
606 int cfl_cost[CFL_JOINT_SIGNS][CFL_PRED_PLANES][CFL_ALPHABET_SIZE];
607 /**@}*/
608
609 /*****************************************************************************
610 * \name Intra Costs: Screen Contents
611 ****************************************************************************/
612 /**@{*/
613 //! intrabc_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700614 int intrabc_cost[2];
615
chiyotsaic2f38412020-06-10 16:07:21 -0700616 //! palette_y_size_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700617 int palette_y_size_cost[PALATTE_BSIZE_CTXS][PALETTE_SIZES];
chiyotsaic2f38412020-06-10 16:07:21 -0700618 //! palette_uv_size_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700619 int palette_uv_size_cost[PALATTE_BSIZE_CTXS][PALETTE_SIZES];
chiyotsaic2f38412020-06-10 16:07:21 -0700620 //! palette_y_color_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700621 int palette_y_color_cost[PALETTE_SIZES][PALETTE_COLOR_INDEX_CONTEXTS]
622 [PALETTE_COLORS];
chiyotsaic2f38412020-06-10 16:07:21 -0700623 //! palette_uv_color_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700624 int palette_uv_color_cost[PALETTE_SIZES][PALETTE_COLOR_INDEX_CONTEXTS]
625 [PALETTE_COLORS];
chiyotsaic2f38412020-06-10 16:07:21 -0700626 //! palette_y_mode_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700627 int palette_y_mode_cost[PALATTE_BSIZE_CTXS][PALETTE_Y_MODE_CONTEXTS][2];
chiyotsaic2f38412020-06-10 16:07:21 -0700628 //! palette_uv_mode_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700629 int palette_uv_mode_cost[PALETTE_UV_MODE_CONTEXTS][2];
chiyotsaic2f38412020-06-10 16:07:21 -0700630 /**@}*/
chiyotsai9a06d182020-05-01 17:12:12 -0700631
chiyotsaic2f38412020-06-10 16:07:21 -0700632 /*****************************************************************************
633 * \name Inter Costs: MV Modes
634 ****************************************************************************/
635 /**@{*/
636 //! skip_mode_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700637 int skip_mode_cost[SKIP_MODE_CONTEXTS][2];
chiyotsaic2f38412020-06-10 16:07:21 -0700638 //! newmv_mode_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700639 int newmv_mode_cost[NEWMV_MODE_CONTEXTS][2];
chiyotsaic2f38412020-06-10 16:07:21 -0700640 //! zeromv_mode_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700641 int zeromv_mode_cost[GLOBALMV_MODE_CONTEXTS][2];
chiyotsaic2f38412020-06-10 16:07:21 -0700642 //! refmv_mode_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700643 int refmv_mode_cost[REFMV_MODE_CONTEXTS][2];
chiyotsaic2f38412020-06-10 16:07:21 -0700644 //! drl_mode_cost0
chiyotsai9a06d182020-05-01 17:12:12 -0700645 int drl_mode_cost0[DRL_MODE_CONTEXTS][2];
chiyotsaic2f38412020-06-10 16:07:21 -0700646 /**@}*/
chiyotsai9a06d182020-05-01 17:12:12 -0700647
chiyotsaic2f38412020-06-10 16:07:21 -0700648 /*****************************************************************************
649 * \name Inter Costs: Ref Frame Types
650 ****************************************************************************/
651 /**@{*/
652 //! single_ref_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700653 int single_ref_cost[REF_CONTEXTS][SINGLE_REFS - 1][2];
chiyotsaic2f38412020-06-10 16:07:21 -0700654 //! comp_inter_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700655 int comp_inter_cost[COMP_INTER_CONTEXTS][2];
chiyotsaic2f38412020-06-10 16:07:21 -0700656 //! comp_ref_type_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700657 int comp_ref_type_cost[COMP_REF_TYPE_CONTEXTS]
658 [CDF_SIZE(COMP_REFERENCE_TYPES)];
chiyotsaic2f38412020-06-10 16:07:21 -0700659 //! uni_comp_ref_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700660 int uni_comp_ref_cost[UNI_COMP_REF_CONTEXTS][UNIDIR_COMP_REFS - 1]
661 [CDF_SIZE(2)];
chiyotsaic2f38412020-06-10 16:07:21 -0700662 /*! \brief Cost for signaling ref_frame[0] in bidir-comp mode
663 *
664 * Includes LAST_FRAME, LAST2_FRAME, LAST3_FRAME, and GOLDEN_FRAME.
665 */
chiyotsai9a06d182020-05-01 17:12:12 -0700666 int comp_ref_cost[REF_CONTEXTS][FWD_REFS - 1][2];
chiyotsaic2f38412020-06-10 16:07:21 -0700667 /*! \brief Cost for signaling ref_frame[1] in bidir-comp mode
668 *
669 * Includes ALTREF_FRAME, ALTREF2_FRAME, and BWDREF_FRAME.
670 */
chiyotsai9a06d182020-05-01 17:12:12 -0700671 int comp_bwdref_cost[REF_CONTEXTS][BWD_REFS - 1][2];
chiyotsaic2f38412020-06-10 16:07:21 -0700672 /**@}*/
chiyotsai9a06d182020-05-01 17:12:12 -0700673
chiyotsaic2f38412020-06-10 16:07:21 -0700674 /*****************************************************************************
675 * \name Inter Costs: Compound Types
676 ****************************************************************************/
677 /**@{*/
678 //! intra_inter_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700679 int intra_inter_cost[INTRA_INTER_CONTEXTS][2];
chiyotsaic2f38412020-06-10 16:07:21 -0700680 //! inter_compound_mode_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700681 int inter_compound_mode_cost[INTER_MODE_CONTEXTS][INTER_COMPOUND_MODES];
chiyotsaic2f38412020-06-10 16:07:21 -0700682 //! compound_type_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700683 int compound_type_cost[BLOCK_SIZES_ALL][MASKED_COMPOUND_TYPES];
chiyotsaic2f38412020-06-10 16:07:21 -0700684 //! wedge_idx_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700685 int wedge_idx_cost[BLOCK_SIZES_ALL][16];
chiyotsaic2f38412020-06-10 16:07:21 -0700686 //! interintra_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700687 int interintra_cost[BLOCK_SIZE_GROUPS][2];
chiyotsaic2f38412020-06-10 16:07:21 -0700688 //! wedge_interintra_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700689 int wedge_interintra_cost[BLOCK_SIZES_ALL][2];
chiyotsaic2f38412020-06-10 16:07:21 -0700690 //! interintra_mode_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700691 int interintra_mode_cost[BLOCK_SIZE_GROUPS][INTERINTRA_MODES];
chiyotsaic2f38412020-06-10 16:07:21 -0700692 /**@}*/
chiyotsai9a06d182020-05-01 17:12:12 -0700693
chiyotsaic2f38412020-06-10 16:07:21 -0700694 /*****************************************************************************
695 * \name Inter Costs: Compound Masks
696 ****************************************************************************/
697 /**@{*/
698 //! comp_idx_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700699 int comp_idx_cost[COMP_INDEX_CONTEXTS][2];
chiyotsaic2f38412020-06-10 16:07:21 -0700700 //! comp_group_idx_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700701 int comp_group_idx_cost[COMP_GROUP_IDX_CONTEXTS][2];
chiyotsaic2f38412020-06-10 16:07:21 -0700702 /**@}*/
chiyotsai9a06d182020-05-01 17:12:12 -0700703
chiyotsaic2f38412020-06-10 16:07:21 -0700704 /*****************************************************************************
705 * \name Inter Costs: Motion Modes/Filters
706 ****************************************************************************/
707 /**@{*/
708 //! motion_mode_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700709 int motion_mode_cost[BLOCK_SIZES_ALL][MOTION_MODES];
chiyotsaic2f38412020-06-10 16:07:21 -0700710 //! motion_mode_cost1
chiyotsai9a06d182020-05-01 17:12:12 -0700711 int motion_mode_cost1[BLOCK_SIZES_ALL][2];
chiyotsaic2f38412020-06-10 16:07:21 -0700712 //! switchable_interp_costs
chiyotsai9a06d182020-05-01 17:12:12 -0700713 int switchable_interp_costs[SWITCHABLE_FILTER_CONTEXTS][SWITCHABLE_FILTERS];
chiyotsaic2f38412020-06-10 16:07:21 -0700714 /**@}*/
chiyotsai9a06d182020-05-01 17:12:12 -0700715
chiyotsaic2f38412020-06-10 16:07:21 -0700716 /*****************************************************************************
717 * \name Txfm Mode Costs
718 ****************************************************************************/
719 /**@{*/
720 //! skip_txfm_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700721 int skip_txfm_cost[SKIP_CONTEXTS][2];
chiyotsaic2f38412020-06-10 16:07:21 -0700722 //! tx_size_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700723 int tx_size_cost[TX_SIZES - 1][TX_SIZE_CONTEXTS][TX_SIZES];
chiyotsaic2f38412020-06-10 16:07:21 -0700724 //! txfm_partition_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700725 int txfm_partition_cost[TXFM_PARTITION_CONTEXTS][2];
chiyotsaic2f38412020-06-10 16:07:21 -0700726 //! inter_tx_type_costs
chiyotsai9a06d182020-05-01 17:12:12 -0700727 int inter_tx_type_costs[EXT_TX_SETS_INTER][EXT_TX_SIZES][TX_TYPES];
chiyotsaic2f38412020-06-10 16:07:21 -0700728 //! intra_tx_type_costs
chiyotsai9a06d182020-05-01 17:12:12 -0700729 int intra_tx_type_costs[EXT_TX_SETS_INTRA][EXT_TX_SIZES][INTRA_MODES]
730 [TX_TYPES];
chiyotsaic2f38412020-06-10 16:07:21 -0700731 /**@}*/
chiyotsai9a06d182020-05-01 17:12:12 -0700732
chiyotsaic2f38412020-06-10 16:07:21 -0700733 /*****************************************************************************
734 * \name Restoration Mode Costs
735 ****************************************************************************/
736 /**@{*/
737 //! switchable_restore_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700738 int switchable_restore_cost[RESTORE_SWITCHABLE_TYPES];
chiyotsaic2f38412020-06-10 16:07:21 -0700739 //! wiener_restore_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700740 int wiener_restore_cost[2];
chiyotsaic2f38412020-06-10 16:07:21 -0700741 //! sgrproj_restore_cost
chiyotsai9a06d182020-05-01 17:12:12 -0700742 int sgrproj_restore_cost[2];
chiyotsaic2f38412020-06-10 16:07:21 -0700743 /**@}*/
Jingning Han564fe2c2022-03-08 23:35:01 -0800744
745 /*****************************************************************************
746 * \name Segmentation Mode Costs
747 ****************************************************************************/
748 /**@{*/
749 //! tmp_pred_cost
750 int tmp_pred_cost[SEG_TEMPORAL_PRED_CTXS][2];
751 //! spatial_pred_cost
752 int spatial_pred_cost[SPATIAL_PREDICTION_PROBS][MAX_SEGMENTS];
753 /**@}*/
chiyotsai9a06d182020-05-01 17:12:12 -0700754} ModeCosts;
755
chiyotsaic2f38412020-06-10 16:07:21 -0700756/*! \brief Holds mv costs for encoding and motion search.
757 */
chiyotsai9a06d182020-05-01 17:12:12 -0700758typedef struct {
chiyotsaic2f38412020-06-10 16:07:21 -0700759 /*****************************************************************************
chiyotsaic2f38412020-06-10 16:07:21 -0700760 * \name Encoding Costs
761 * Here are the entropy costs needed to encode a given mv.
762 * \ref nmv_cost_alloc and \ref nmv_cost_hp_alloc are two arrays that holds
763 * the memory for holding the mv cost. But since the motion vectors can be
764 * negative, we shift them to the middle and store the resulting pointer in
765 * \ref nmv_cost and \ref nmv_cost_hp for easier referencing. Finally, \ref
766 * mv_cost_stack points to the \ref nmv_cost with the mv precision we are
767 * currently working with. In essence, only \ref mv_cost_stack is needed for
768 * motion search, the other can be considered private.
769 ****************************************************************************/
770 /**@{*/
771 //! Costs for coding the zero components.
chiyotsai9a06d182020-05-01 17:12:12 -0700772 int nmv_joint_cost[MV_JOINTS];
773
chiyotsaic2f38412020-06-10 16:07:21 -0700774 //! Allocates memory for 1/4-pel motion vector costs.
chiyotsai9a06d182020-05-01 17:12:12 -0700775 int nmv_cost_alloc[2][MV_VALS];
chiyotsaic2f38412020-06-10 16:07:21 -0700776 //! Allocates memory for 1/8-pel motion vector costs.
chiyotsai9a06d182020-05-01 17:12:12 -0700777 int nmv_cost_hp_alloc[2][MV_VALS];
chiyotsaic2f38412020-06-10 16:07:21 -0700778 //! Points to the middle of \ref nmv_cost_alloc
chiyotsai9a06d182020-05-01 17:12:12 -0700779 int *nmv_cost[2];
chiyotsaic2f38412020-06-10 16:07:21 -0700780 //! Points to the middle of \ref nmv_cost_hp_alloc
chiyotsai9a06d182020-05-01 17:12:12 -0700781 int *nmv_cost_hp[2];
chiyotsaic2f38412020-06-10 16:07:21 -0700782 //! Points to the nmv_cost_hp in use.
chiyotsai9a06d182020-05-01 17:12:12 -0700783 int **mv_cost_stack;
chiyotsaic2f38412020-06-10 16:07:21 -0700784 /**@}*/
chiyotsai9a06d182020-05-01 17:12:12 -0700785} MvCosts;
786
chiyotsai41fd15c2021-03-15 14:12:02 -0700787/*! \brief Holds mv costs for intrabc.
788 */
789typedef struct {
790 /*! Costs for coding the joint mv. */
791 int joint_mv[MV_JOINTS];
792
793 /*! \brief Cost of transmitting the actual motion vector.
chiyotsaiad4d3ea2021-03-17 13:53:01 -0700794 * dv_costs_alloc[0][i] is the cost of motion vector with horizontal
795 * component (mv_row) equal to i - MV_MAX. dv_costs_alloc[1][i] is the cost of
chiyotsai41fd15c2021-03-15 14:12:02 -0700796 * motion vector with vertical component (mv_col) equal to i - MV_MAX.
797 */
798 int dv_costs_alloc[2][MV_VALS];
799
800 /*! Points to the middle of \ref dv_costs_alloc. */
801 int *dv_costs[2];
802} IntraBCMVCosts;
803
chiyotsaic2f38412020-06-10 16:07:21 -0700804/*! \brief Holds the costs needed to encode the coefficients
805 */
chiyotsai9a06d182020-05-01 17:12:12 -0700806typedef struct {
chiyotsaic2f38412020-06-10 16:07:21 -0700807 //! Costs for coding the coefficients.
chiyotsai9a06d182020-05-01 17:12:12 -0700808 LV_MAP_COEFF_COST coeff_costs[TX_SIZES][PLANE_TYPES];
chiyotsaic2f38412020-06-10 16:07:21 -0700809 //! Costs for coding the eobs.
chiyotsai9a06d182020-05-01 17:12:12 -0700810 LV_MAP_EOB_COST eob_costs[7][2];
811} CoeffCosts;
812
chiyotsaic2f38412020-06-10 16:07:21 -0700813/*!\cond */
814// 4: NEAREST, NEW, NEAR, GLOBAL
815#define SINGLE_REF_MODES ((REF_FRAMES - 1) * 4)
816/*!\endcond */
Ravi Chaudhary5d970f42018-09-25 11:25:32 +0530817struct inter_modes_info;
chiyotsaic2f38412020-06-10 16:07:21 -0700818
Hui Su69af3f52020-10-06 16:28:57 -0700819/*! \brief Holds the motion samples for warp motion model estimation
820 */
821typedef struct {
822 //! Number of samples.
823 int num;
824 //! Sample locations in current frame.
825 int pts[16];
826 //! Sample location in the reference frame.
827 int pts_inref[16];
828} WARP_SAMPLE_INFO;
829
Marco Paniconi988b34a2020-11-09 12:41:13 -0800830/*!\cond */
831typedef enum {
Marco Paniconifa924c52022-01-11 14:57:41 -0800832 kZeroSad = 0,
Neeraj Gadgil710cc542022-08-22 12:05:25 +0530833 kVeryLowSad = 1,
834 kLowSad = 2,
835 kMedSad = 3,
836 kHighSad = 4
Marco Paniconi988b34a2020-11-09 12:41:13 -0800837} SOURCE_SAD;
838
839typedef struct {
Neeraj Gadgil6ad85e82022-08-18 14:38:01 +0530840 //! SAD levels in non-rd path
Neeraj Gadgil710cc542022-08-22 12:05:25 +0530841 SOURCE_SAD source_sad_nonrd;
Neeraj Gadgilb538d632022-04-13 15:18:02 +0530842 //! SAD levels in rd-path for var-based part qindex thresholds
843 SOURCE_SAD source_sad_rd;
Marco Paniconi988b34a2020-11-09 12:41:13 -0800844 int lighting_change;
845 int low_sumdiff;
846} CONTENT_STATE_SB;
Jayasanker J759b3202021-03-24 19:12:32 +0530847
848// Structure to hold pixel level gradient info.
849typedef struct {
850 uint16_t abs_dx_abs_dy_sum;
851 int8_t hist_bin_idx;
852 bool is_dx_zero;
853} PixelLevelGradientInfo;
854
Mudassir Galagnathad268362021-12-27 10:57:44 +0530855// Structure to hold the variance and log(1 + variance) for 4x4 sub-blocks.
856typedef struct {
857 double log_var;
858 int var;
859} Block4x4VarInfo;
860
chiyotsaid2c9d9d2022-04-28 14:23:33 -0700861#ifndef NDEBUG
862typedef struct SetOffsetsLoc {
863 int mi_row;
864 int mi_col;
865 BLOCK_SIZE bsize;
866} SetOffsetsLoc;
867#endif // NDEBUG
868
Marco Paniconi988b34a2020-11-09 12:41:13 -0800869/*!\endcond */
870
chiyotsaic2f38412020-06-10 16:07:21 -0700871/*! \brief Encoder's parameters related to the current coding block.
872 *
873 * This struct contains most of the information the encoder needs to encode the
874 * current coding block. This includes the src and pred buffer, a copy of the
875 * decoder's view of the current block, the txfm coefficients. This struct also
876 * contains various buffers and data used to speed up the encoding process.
877 */
878typedef struct macroblock {
879 /*****************************************************************************
880 * \name Source, Buffers and Decoder
881 ****************************************************************************/
882 /**@{*/
883 /*! \brief Each of the encoding plane.
884 *
885 * An array holding the src buffer for each of plane of the current block. It
886 * also contains the txfm and quantized txfm coefficients.
887 */
Yaowu Xuc27fc142016-08-22 16:08:15 -0700888 struct macroblock_plane plane[MAX_MB_PLANE];
889
chiyotsaic2f38412020-06-10 16:07:21 -0700890 /*! \brief Decoder's view of current coding block.
891 *
892 * Contains the encoder's copy of what the decoder sees in the current block.
893 * Most importantly, this struct contains pointers to mbmi that is used in
894 * final bitstream packing.
895 */
Yaowu Xuc27fc142016-08-22 16:08:15 -0700896 MACROBLOCKD e_mbd;
chiyotsai85d715a2020-05-02 15:10:33 -0700897
chiyotsaic2f38412020-06-10 16:07:21 -0700898 /*! \brief Derived coding information.
899 *
900 * Contains extra information not transmitted in the bitstream but are
901 * derived. For example, this contains the stack of ref_mvs.
902 */
chiyotsai0b90c412020-09-29 14:48:16 -0700903 MB_MODE_INFO_EXT mbmi_ext;
chiyotsai85d715a2020-05-02 15:10:33 -0700904
chiyotsaic2f38412020-06-10 16:07:21 -0700905 /*! \brief Finalized mbmi_ext for the whole frame.
906 *
907 * Contains the finalized info in mbmi_ext that gets used at the frame level
908 * for bitstream packing.
909 */
Remya0cce44c2019-08-16 11:57:24 +0530910 MB_MODE_INFO_EXT_FRAME *mbmi_ext_frame;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700911
chiyotsaic2f38412020-06-10 16:07:21 -0700912 //! Entropy context for the current row.
chiyotsai85d715a2020-05-02 15:10:33 -0700913 FRAME_CONTEXT *row_ctx;
chiyotsaic2f38412020-06-10 16:07:21 -0700914 /*! \brief Entropy context for the current tile.
915 *
916 * This context will be used to update color_map_cdf pointer which would be
917 * used during pack bitstream. For single thread and tile-multithreading case
918 * this pointer will be same as xd->tile_ctx, but for the case of row-mt:
919 * xd->tile_ctx will point to a temporary context while tile_pb_ctx will point
920 * to the accurate tile context.
921 */
chiyotsai85d715a2020-05-02 15:10:33 -0700922 FRAME_CONTEXT *tile_pb_ctx;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700923
chiyotsaic2f38412020-06-10 16:07:21 -0700924 /*! \brief Buffer of transformed coefficients
925 *
926 * Points to cb_coef_buff in the AV1_COMP struct, which contains the finalized
927 * coefficients. This is here to conveniently copy the best coefficients to
928 * frame level for bitstream packing. Since CB_COEFF_BUFFER is allocated on a
929 * superblock level, we need to combine it with cb_offset to get the proper
930 * position for the current coding block.
931 */
chiyotsai85d715a2020-05-02 15:10:33 -0700932 CB_COEFF_BUFFER *cb_coef_buff;
chiyotsaic2f38412020-06-10 16:07:21 -0700933 //! Offset of current coding block's coeff buffer relative to the sb.
Jayasanker J2350ca32020-09-10 23:49:00 +0530934 uint16_t cb_offset[PLANE_TYPES];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700935
chiyotsaic2f38412020-06-10 16:07:21 -0700936 //! Modified source and masks used for fast OBMC search.
chiyotsaid2b12212020-04-28 20:57:19 -0700937 OBMCBuffer obmc_buffer;
chiyotsaic2f38412020-06-10 16:07:21 -0700938 //! Buffer to store the best palette map.
Yaowu Xuc27fc142016-08-22 16:08:15 -0700939 PALETTE_BUFFER *palette_buffer;
chiyotsaic2f38412020-06-10 16:07:21 -0700940 //! Buffer used for compound_type_rd().
Hui Su38711e72019-06-11 10:49:47 -0700941 CompoundTypeRdBuffers comp_rd_buffer;
chiyotsaic2f38412020-06-10 16:07:21 -0700942 //! Buffer to store convolution during averaging process in compound mode.
Urvang Joshi0a4cfad2018-09-07 11:10:39 -0700943 CONV_BUF_TYPE *tmp_conv_dst;
chiyotsai2a897eb2020-04-28 19:22:13 -0700944
chiyotsaic2f38412020-06-10 16:07:21 -0700945 /*! \brief Temporary buffer to hold prediction.
946 *
947 * Points to a buffer that is used to hold temporary prediction results. This
948 * is used in two ways:
Sarah Parkerf492c1f2021-03-02 21:01:56 -0800949 * - This is a temporary buffer used to ping-pong the prediction in
chiyotsaic2f38412020-06-10 16:07:21 -0700950 * handle_inter_mode.
951 * - xd->tmp_obmc_bufs also points to this buffer, and is used in ombc
952 * prediction.
953 */
chiyotsai2a897eb2020-04-28 19:22:13 -0700954 uint8_t *tmp_pred_bufs[2];
chiyotsaic2f38412020-06-10 16:07:21 -0700955 /**@}*/
Urvang Joshi0a4cfad2018-09-07 11:10:39 -0700956
chiyotsaic2f38412020-06-10 16:07:21 -0700957 /*****************************************************************************
958 * \name Rdopt Costs
959 ****************************************************************************/
960 /**@{*/
961 /*! \brief Quantization index for the current partition block.
962 *
963 * This is used to as the index to find quantization parameter for luma and
964 * chroma transformed coefficients.
965 */
chiyotsai9a06d182020-05-01 17:12:12 -0700966 int qindex;
967
chiyotsaic2f38412020-06-10 16:07:21 -0700968 /*! \brief Difference between frame-level qindex and current qindex.
969 *
970 * This is used to track whether a non-zero delta for qindex is used at least
971 * once in the current frame.
972 */
chiyotsai9a06d182020-05-01 17:12:12 -0700973 int delta_qindex;
974
Lin Zheng9b1d9d92022-11-07 16:59:15 +0000975 /*! \brief Difference between frame-level qindex and qindex used to
linzhen49073062022-10-31 00:42:59 +0000976 * compute rdmult (lambda).
977 *
978 * rdmult_delta_qindex is assigned the same as delta_qindex before qp sweep.
979 * During qp sweep, delta_qindex is changed and used to calculate the actual
980 * quant params, while rdmult_delta_qindex remains the same, and is used to
981 * calculate the rdmult in "set_deltaq_rdmult".
982 */
983 int rdmult_delta_qindex;
984
Lin Zheng9b1d9d92022-11-07 16:59:15 +0000985 /*! \brief Current qindex (before being adjusted by delta_q_res) used to
986 * derive rdmult_delta_qindex.
linzhen49073062022-10-31 00:42:59 +0000987 */
988 int rdmult_cur_qindex;
989
chiyotsaic2f38412020-06-10 16:07:21 -0700990 /*! \brief Rate-distortion multiplier.
991 *
992 * The rd multiplier used to determine the rate-distortion trade-off. This is
993 * roughly proportional to the inverse of q-index for a given frame, but this
994 * can be manipulated for better rate-control. For example, in tune_ssim
995 * mode, this is scaled by a factor related to the variance of the current
996 * block.
997 */
chiyotsai9a06d182020-05-01 17:12:12 -0700998 int rdmult;
999
James Zernf8caf012022-03-03 16:11:01 -08001000 //! Intra only, per sb rd adjustment.
Paul Wilkinsc19e09e2022-02-08 14:46:39 +00001001 int intra_sb_rdmult_modifier;
1002
Jingning Han3b96bb32021-11-18 15:05:42 -08001003 //! Superblock level distortion propagation factor.
1004 double rb;
1005
chiyotsaic2f38412020-06-10 16:07:21 -07001006 //! Energy in the current source coding block. Used to calculate \ref rdmult
chiyotsai85d715a2020-05-02 15:10:33 -07001007 int mb_energy;
chiyotsaic2f38412020-06-10 16:07:21 -07001008 //! Energy in the current source superblock. Used to calculate \ref rdmult
chiyotsai85d715a2020-05-02 15:10:33 -07001009 int sb_energy_level;
1010
chiyotsaic2f38412020-06-10 16:07:21 -07001011 //! The rate needed to signal a mode to the bitstream.
chiyotsai9a06d182020-05-01 17:12:12 -07001012 ModeCosts mode_costs;
1013
chiyotsaic2f38412020-06-10 16:07:21 -07001014 //! The rate needed to encode a new motion vector to the bitstream and some
1015 //! multipliers for motion search.
Fyodor Kyslov648c6502021-02-02 18:41:10 -08001016 MvCosts *mv_costs;
chiyotsai9a06d182020-05-01 17:12:12 -07001017
chiyotsai41fd15c2021-03-15 14:12:02 -07001018 /*! The rate needed to encode a new motion vector to the bitstream in intrabc
1019 * mode.
1020 */
1021 IntraBCMVCosts *dv_costs;
1022
chiyotsaic2f38412020-06-10 16:07:21 -07001023 //! The rate needed to signal the txfm coefficients to the bitstream.
chiyotsai9a06d182020-05-01 17:12:12 -07001024 CoeffCosts coeff_costs;
chiyotsaic2f38412020-06-10 16:07:21 -07001025 /**@}*/
chiyotsai9a06d182020-05-01 17:12:12 -07001026
Fyodor Kyslov648c6502021-02-02 18:41:10 -08001027 /*****************************************************************************
1028 * \name Rate to Distortion Multipliers
1029 ****************************************************************************/
1030 /**@{*/
1031 //! A multiplier that converts mv cost to l2 error.
1032 int errorperbit;
1033 //! A multiplier that converts mv cost to l1 error.
1034 int sadperbit;
1035 /**@}*/
1036
chiyotsaic2f38412020-06-10 16:07:21 -07001037 /******************************************************************************
1038 * \name Segmentation
1039 *****************************************************************************/
1040 /**@{*/
1041 /*! \brief Skip mode for the segment
1042 *
1043 * A syntax element of the segmentation mode. In skip_block mode, all mvs are
1044 * set 0 and all txfms are skipped.
1045 */
chiyotsai85d715a2020-05-02 15:10:33 -07001046 int seg_skip_block;
Cherma Rajan Ad27a1472021-02-19 11:35:40 +05301047
1048 /*! \brief Number of segment 1 blocks
1049 * Actual number of (4x4) blocks that were applied delta-q,
1050 * for segment 1.
1051 */
1052 int actual_num_seg1_blocks;
1053
1054 /*!\brief Number of segment 2 blocks
1055 * Actual number of (4x4) blocks that were applied delta-q,
1056 * for segment 2.
1057 */
1058 int actual_num_seg2_blocks;
1059
1060 /*!\brief Number of zero motion vectors
1061 */
1062 int cnt_zeromv;
Marco Paniconi7ae16352022-03-08 00:19:24 -08001063
Ranjit Kumar Tulabanducd4a3072022-09-02 15:53:57 +05301064 /*!\brief Flag to force zeromv-skip at superblock level, for nonrd path.
1065 *
1066 * 0/1 imply zeromv-skip is disabled/enabled. 2 implies that the blocks
1067 * in the superblock may be marked as zeromv-skip at block level.
Marco Paniconi7ae16352022-03-08 00:19:24 -08001068 */
Ranjit Kumar Tulabanducd4a3072022-09-02 15:53:57 +05301069 int force_zeromv_skip_for_sb;
1070
1071 /*!\brief Flag to force zeromv-skip at block level, for nonrd path.
1072 */
1073 int force_zeromv_skip_for_blk;
Nithya V S8bb0c792022-03-14 15:48:24 +05301074
1075 /*! \brief Previous segment id for which qmatrices were updated.
1076 * This is used to bypass setting of qmatrices if no change in qindex.
1077 */
1078 int prev_segment_id;
chiyotsaic2f38412020-06-10 16:07:21 -07001079 /**@}*/
chiyotsai85d715a2020-05-02 15:10:33 -07001080
chiyotsaic2f38412020-06-10 16:07:21 -07001081 /*****************************************************************************
1082 * \name Superblock
1083 ****************************************************************************/
1084 /**@{*/
1085 //! Information on a whole superblock level.
chiyotsai85d715a2020-05-02 15:10:33 -07001086 // TODO(chiyotsai@google.com): Refactor this out of macroblock
1087 SuperBlockEnc sb_enc;
1088
chiyotsaic2f38412020-06-10 16:07:21 -07001089 /*! \brief Characteristics of the current superblock.
1090 *
1091 * Characteristics like whether the block has high sad, low sad, etc. This is
1092 * only used by av1 realtime mode.
1093 */
Marco Paniconi988b34a2020-11-09 12:41:13 -08001094 CONTENT_STATE_SB content_state_sb;
chiyotsaic2f38412020-06-10 16:07:21 -07001095 /**@}*/
chiyotsai85d715a2020-05-02 15:10:33 -07001096
chiyotsaic2f38412020-06-10 16:07:21 -07001097 /*****************************************************************************
chiyotsai3afb03e2020-09-08 13:30:12 -07001098 * \name Reference Frame Search
chiyotsaic2f38412020-06-10 16:07:21 -07001099 ****************************************************************************/
1100 /**@{*/
1101 /*! \brief Sum absolute distortion of the predicted mv for each ref frame.
1102 *
1103 * This is used to measure how viable a reference frame is.
1104 */
chiyotsai85d715a2020-05-02 15:10:33 -07001105 int pred_mv_sad[REF_FRAMES];
Deepa K G9f7fe722022-04-28 12:33:31 +05301106 /*! \brief The minimum of \ref pred_mv_sad.
1107 *
1108 * Index 0 stores the minimum \ref pred_mv_sad across past reference frames.
1109 * Index 1 stores the minimum \ref pred_mv_sad across future reference frames.
1110 */
1111 int best_pred_mv_sad[2];
Marco Paniconiaaf2c562021-06-07 14:22:11 -07001112 //! The sad of the 1st mv ref (nearest).
Marco Paniconi737820e2021-05-24 11:03:38 -07001113 int pred_mv0_sad[REF_FRAMES];
Marco Paniconiaaf2c562021-06-07 14:22:11 -07001114 //! The sad of the 2nd mv ref (near).
Marco Paniconi737820e2021-05-24 11:03:38 -07001115 int pred_mv1_sad[REF_FRAMES];
chiyotsai85d715a2020-05-02 15:10:33 -07001116
chiyotsaic2f38412020-06-10 16:07:21 -07001117 /*! \brief Disables certain ref frame pruning based on tpl.
1118 *
1119 * Determines whether a given ref frame is "good" based on data from the TPL
1120 * model. If so, this stops selective_ref frame from pruning the given ref
1121 * frame at block level.
1122 */
chiyotsai85d715a2020-05-02 15:10:33 -07001123 uint8_t tpl_keep_ref_frame[REF_FRAMES];
1124
Hui Su69af3f52020-10-06 16:28:57 -07001125 /*! \brief Warp motion samples buffer.
1126 *
1127 * Store the motion samples used for warp motion.
1128 */
1129 WARP_SAMPLE_INFO warp_sample_info[REF_FRAMES];
1130
chiyotsaic2f38412020-06-10 16:07:21 -07001131 /*! \brief Reference frames picked by the square subblocks in a superblock.
1132 *
1133 * Keeps track of ref frames that are selected by square partition blocks
1134 * within a superblock, in MI resolution. They can be used to prune ref frames
1135 * for rectangular blocks.
1136 */
chiyotsai85d715a2020-05-02 15:10:33 -07001137 int picked_ref_frames_mask[MAX_MIB_SIZE * MAX_MIB_SIZE];
1138
chiyotsaic2f38412020-06-10 16:07:21 -07001139 /*! \brief Prune ref frames in real-time mode.
1140 *
1141 * Determines whether to prune reference frames in real-time mode. For the
1142 * most part, this is the same as nonrd_prune_ref_frame_search in
1143 * cpi->sf.rt_sf.nonrd_prune_ref_frame_search, but this can be selectively
1144 * turned off if the only frame available is GOLDEN_FRAME.
1145 */
chiyotsai85d715a2020-05-02 15:10:33 -07001146 int nonrd_prune_ref_frame_search;
chiyotsaic2f38412020-06-10 16:07:21 -07001147 /**@}*/
chiyotsai85d715a2020-05-02 15:10:33 -07001148
chiyotsaic2f38412020-06-10 16:07:21 -07001149 /*****************************************************************************
1150 * \name Partition Search
1151 ****************************************************************************/
1152 /**@{*/
1153 //! Stores some partition-search related buffers.
chiyotsai68eefbe2020-05-01 15:07:58 -07001154 PartitionSearchInfo part_search_info;
1155
chiyotsaic2f38412020-06-10 16:07:21 -07001156 /*! \brief Whether to disable some features to force a mode in current block.
1157 *
1158 * In some cases, our speed features can be overly aggressive and remove all
1159 * modes search in the superblock. When this happens, we set
1160 * must_find_valid_partition to 1 to reduce the number of speed features, and
1161 * recode the superblock again.
1162 */
chiyotsai85d715a2020-05-02 15:10:33 -07001163 int must_find_valid_partition;
chiyotsaic2f38412020-06-10 16:07:21 -07001164 /**@}*/
chiyotsai85d715a2020-05-02 15:10:33 -07001165
chiyotsaic2f38412020-06-10 16:07:21 -07001166 /*****************************************************************************
1167 * \name Prediction Mode Search
1168 ****************************************************************************/
1169 /**@{*/
1170 /*! \brief Inter skip mode.
1171 *
1172 * Skip mode tries to use the closest forward and backward references for
1173 * inter prediction. Skip here means to skip transmitting the reference
1174 * frames, not to be confused with skip_txfm.
1175 */
chiyotsai85d715a2020-05-02 15:10:33 -07001176 int skip_mode;
1177
chiyotsaic2f38412020-06-10 16:07:21 -07001178 /*! \brief Factors used for rd-thresholding.
1179 *
1180 * Determines a rd threshold to determine whether to continue searching the
1181 * current mode. If the current best rd is already <= threshold, then we skip
1182 * the current mode.
1183 */
chiyotsai85d715a2020-05-02 15:10:33 -07001184 int thresh_freq_fact[BLOCK_SIZES_ALL][MAX_MODES];
1185
chiyotsaic2f38412020-06-10 16:07:21 -07001186 /*! \brief Tracks the winner modes in the current coding block.
1187 *
1188 * Winner mode is a two-pass strategy to find the best prediction mode. In the
1189 * first pass, we search the prediction modes with a limited set of txfm
1190 * options, and keep the top modes. These modes are called the winner modes.
1191 * In the second pass, we retry the winner modes with more thorough txfm
1192 * options.
1193 */
Chethan Kumar R Ea494e2b2021-03-18 17:19:44 +05301194 WinnerModeStats *winner_mode_stats;
chiyotsaic2f38412020-06-10 16:07:21 -07001195 //! Tracks how many winner modes there are.
chiyotsai85d715a2020-05-02 15:10:33 -07001196 int winner_mode_count;
1197
chiyotsaic2f38412020-06-10 16:07:21 -07001198 /*! \brief The model used for rd-estimation to avoid txfm
1199 *
1200 * These are for inter_mode_rd_model_estimation, which is another two pass
1201 * approach. In this speed feature, we collect data in the first couple frames
1202 * to build an rd model to estimate the rdcost of a prediction model based on
1203 * the residue error. Once enough data is collected, this speed feature uses
1204 * the estimated rdcost to find the most performant prediction mode. Then we
1205 * follow up with a second pass find the best transform for the mode.
1206 * Determines if one would go with reduced complexity transform block
1207 * search model to select prediction modes, or full complexity model
1208 * to select transform kernel.
1209 */
chiyotsai85d715a2020-05-02 15:10:33 -07001210 TXFM_RD_MODEL rd_model;
1211
chiyotsaic2f38412020-06-10 16:07:21 -07001212 /*! \brief Stores the inter mode information needed to build an rd model.
1213 *
1214 * These are for inter_mode_rd_model_estimation, which is another two pass
1215 * approach. In this speed feature, we collect data in the first couple frames
1216 * to build an rd model to estimate the rdcost of a prediction model based on
1217 * the residue error. Once enough data is collected, this speed feature uses
1218 * the estimated rdcost to find the most performant prediction mode. Then we
1219 * follow up with a second pass find the best transform for the mode.
1220 */
chiyotsai85d715a2020-05-02 15:10:33 -07001221 // TODO(any): try to consolidate this speed feature with winner mode
1222 // processing.
1223 struct inter_modes_info *inter_modes_info;
1224
chiyotsaic2f38412020-06-10 16:07:21 -07001225 //! How to blend the compound predictions.
chiyotsai85d715a2020-05-02 15:10:33 -07001226 uint8_t compound_idx;
1227
chiyotsaic2f38412020-06-10 16:07:21 -07001228 //! A caches of results of compound type search so they can be reused later.
chiyotsai85d715a2020-05-02 15:10:33 -07001229 COMP_RD_STATS comp_rd_stats[MAX_COMP_RD_STATS];
chiyotsaic2f38412020-06-10 16:07:21 -07001230 //! The idx for the latest compound mode in the cache \ref comp_rd_stats.
chiyotsai85d715a2020-05-02 15:10:33 -07001231 int comp_rd_stats_idx;
1232
chiyotsaic2f38412020-06-10 16:07:21 -07001233 /*! \brief Whether to recompute the luma prediction.
1234 *
1235 * In interpolation search, we can usually skip recalculating the luma
1236 * prediction because it is already calculated by a previous predictor. This
1237 * flag signifies that some modes might have been skipped, so we need to
1238 * rebuild the prediction.
1239 */
chiyotsai85d715a2020-05-02 15:10:33 -07001240 int recalc_luma_mc_data;
1241
chiyotsaic2f38412020-06-10 16:07:21 -07001242 /*! \brief Data structure to speed up intrabc search.
1243 *
1244 * Contains the hash table, hash function, and buffer used for intrabc.
1245 */
1246 IntraBCHashInfo intrabc_hash_info;
chiyotsai3afb03e2020-09-08 13:30:12 -07001247
Aniket Wanareb69e9902021-03-09 14:01:33 +05301248 /*! \brief Whether to reuse the mode stored in mb_mode_cache. */
1249 int use_mb_mode_cache;
1250 /*! \brief The mode to reuse during \ref av1_rd_pick_intra_mode_sb and
1251 * \ref av1_rd_pick_inter_mode. */
1252 const MB_MODE_INFO *mb_mode_cache;
Jayasanker J759b3202021-03-24 19:12:32 +05301253 /*! \brief Pointer to the buffer which caches gradient information.
1254 *
1255 * Pointer to the array of structures to store gradient information of each
1256 * pixel in a superblock. The buffer constitutes of MAX_SB_SQUARE pixel level
1257 * structures for each of the plane types (PLANE_TYPE_Y and PLANE_TYPE_UV).
1258 */
1259 PixelLevelGradientInfo *pixel_gradient_info;
1260 /*! \brief Flags indicating the availability of cached gradient info. */
1261 bool is_sb_gradient_cached[PLANE_TYPES];
Cherma Rajan A0c4568d2022-09-30 17:17:19 +05301262
1263 /*! \brief Flag to reuse predicted samples of inter block. */
1264 bool reuse_inter_pred;
chiyotsaic2f38412020-06-10 16:07:21 -07001265 /**@}*/
1266
1267 /*****************************************************************************
1268 * \name MV Search
1269 ****************************************************************************/
1270 /**@{*/
1271 /*! \brief Context used to determine the initial step size in motion search.
1272 *
1273 * This context is defined as the \f$l_\inf\f$ norm of the best ref_mvs for
1274 * each frame.
1275 */
1276 unsigned int max_mv_context[REF_FRAMES];
1277
1278 /*! \brief Limit for the range of motion vectors.
1279 *
1280 * These define limits to motion vector components to prevent them from
1281 * extending outside the UMV borders
1282 */
1283 FullMvLimits mv_limits;
chiyotsaifbebe9d2022-05-23 13:31:41 -07001284
1285 /*! \brief Buffer for storing the search site config.
1286 *
1287 * When resize mode or super resolution mode is on, the stride of the
1288 * reference frame does not always match what's specified in \ref
1289 * MotionVectorSearchParams::search_site_cfg. When his happens, we update the
1290 * search_sine_config buffer here and use it for motion search.
1291 */
1292 search_site_config search_site_cfg_buf[NUM_DISTINCT_SEARCH_METHODS];
chiyotsaic2f38412020-06-10 16:07:21 -07001293 /**@}*/
1294
1295 /*****************************************************************************
1296 * \name Txfm Search
1297 ****************************************************************************/
1298 /**@{*/
1299 /*! \brief Parameters that control how motion search is done.
1300 *
1301 * Stores various txfm search related parameters such as txfm_type, txfm_size,
1302 * trellis eob search, etc.
1303 */
chiyotsaia36d9002020-04-29 16:48:21 -07001304 TxfmSearchParams txfm_search_params;
Nithya V Sd0276ac2019-10-24 11:31:06 +05301305
chiyotsaic2f38412020-06-10 16:07:21 -07001306 /*! \brief Results of the txfm searches that have been done.
1307 *
1308 * Caches old txfm search results and keeps the current txfm decisions to
1309 * facilitate rdopt.
1310 */
chiyotsai4c1e5c62020-04-30 17:54:14 -07001311 TxfmSearchInfo txfm_search_info;
1312
chiyotsaic2f38412020-06-10 16:07:21 -07001313 /*! \brief Whether there is a strong color activity.
1314 *
1315 * Used in REALTIME coding mode to enhance the visual quality at the boundary
1316 * of moving color objects.
1317 */
Anupam Pandey45caf952022-12-15 11:16:15 +05301318 uint8_t color_sensitivity_sb[MAX_MB_PLANE - 1];
Marco Paniconic6a503f2022-08-15 23:16:21 -07001319 //! Color sensitivity flag for the superblock for golden reference.
Anupam Pandey45caf952022-12-15 11:16:15 +05301320 uint8_t color_sensitivity_sb_g[MAX_MB_PLANE - 1];
Marco Paniconi555b3aa2023-03-03 09:40:28 -08001321 //! Color sensitivity flag for the superblock for altref reference.
1322 uint8_t color_sensitivity_sb_alt[MAX_MB_PLANE - 1];
Marco Paniconiaaf2c562021-06-07 14:22:11 -07001323 //! Color sensitivity flag for the coding block.
Anupam Pandey45caf952022-12-15 11:16:15 +05301324 uint8_t color_sensitivity[MAX_MB_PLANE - 1];
Marco Paniconif289cc92023-06-27 21:31:04 -07001325 //! Coding block distortion value for uv/color, minimum over the inter modes.
1326 int64_t min_dist_inter_uv;
Wan-Teh Chang48b78522023-10-20 15:54:39 -07001327
Marco Paniconi06f3b3b2024-05-09 22:26:27 -07001328 //! Threshold on the number of colors for testing palette mode.
1329 int color_palette_thresh;
1330
Wan-Teh Chang48b78522023-10-20 15:54:39 -07001331 //! The buffer used by search_tx_type() to swap dqcoeff in macroblockd_plane
1332 // so we can keep dqcoeff of the best tx_type.
1333 tran_low_t *dqcoeff_buf;
chiyotsaic2f38412020-06-10 16:07:21 -07001334 /**@}*/
Jingning Han185d23b2020-03-04 09:12:05 -08001335
chiyotsaic2f38412020-06-10 16:07:21 -07001336 /*****************************************************************************
1337 * \name Misc
1338 ****************************************************************************/
1339 /**@{*/
1340 //! Variance of the source frame.
chiyotsai85d715a2020-05-02 15:10:33 -07001341 unsigned int source_variance;
Marco Paniconibdef9a92023-07-06 15:20:46 -07001342 //! Flag to indicate coding block is zero sad.
1343 int block_is_zero_sad;
Marco Paniconi5a4ea092023-08-09 14:36:15 -07001344 //! Flag to indicate superblock ME in variance partition is determined to be
1345 // good/reliable, and so the superblock MV will be tested in the
1346 // nonrd_pickmode. This is only used for LAST_FRAME.
1347 int sb_me_partition;
1348 //! Flag to indicate to test the superblock MV for the coding block in the
1349 // nonrd_pickmode.
1350 int sb_me_block;
1351 //! Motion vector from superblock MV derived from int_pro_motion() in
1352 // the variance_partitioning.
1353 int_mv sb_me_mv;
Marco Paniconi0414f4e2024-02-22 22:49:24 +00001354 //! Flag to indicate if a fixed partition should be used, only if the
Jerome Jiang16bfbae2024-03-05 08:25:57 -05001355 // speed feature rt_sf->use_fast_fixed_part is enabled.
Marco Paniconi0414f4e2024-02-22 22:49:24 +00001356 int sb_force_fixed_part;
chiyotsaic2f38412020-06-10 16:07:21 -07001357 //! SSE of the current predictor.
chiyotsai85d715a2020-05-02 15:10:33 -07001358 unsigned int pred_sse[REF_FRAMES];
Fyodor Kyslov28af7872020-09-25 18:41:45 -07001359 //! Prediction for ML based partition.
Fyodor Kyslov2c528682021-01-25 11:03:31 -08001360#if CONFIG_RT_ML_PARTITIONING
Fyodor Kyslov28af7872020-09-25 18:41:45 -07001361 DECLARE_ALIGNED(16, uint8_t, est_pred[128 * 128]);
Fyodor Kyslov2c528682021-01-25 11:03:31 -08001362#endif
chiyotsaic2f38412020-06-10 16:07:21 -07001363 /**@}*/
Vishnu Teja Manyam6ee38272021-08-12 10:33:44 +05301364
1365 /*! \brief NONE partition evaluated for merge.
1366 *
1367 * In variance based partitioning scheme, NONE & SPLIT partitions are
1368 * evaluated to check the SPLIT can be merged as NONE. This flag signifies the
1369 * partition is evaluated in the scheme.
1370 */
1371 int try_merge_partition;
Mudassir Galagnathad268362021-12-27 10:57:44 +05301372
1373 /*! \brief Pointer to buffer which caches sub-block variances in a superblock.
1374 *
1375 * Pointer to the array of structures to store source variance information of
1376 * each 4x4 sub-block in a superblock. Block4x4VarInfo structure is used to
Paul Wilkins2c8db512022-03-02 14:35:08 +00001377 * store source variance and log of source variance of each 4x4 sub-block.
Mudassir Galagnathad268362021-12-27 10:57:44 +05301378 */
1379 Block4x4VarInfo *src_var_info_of_4x4_sub_blocks;
chiyotsaid2c9d9d2022-04-28 14:23:33 -07001380#ifndef NDEBUG
1381 /*! \brief A hash to make sure av1_set_offsets is called */
1382 SetOffsetsLoc last_set_offsets_loc;
1383#endif // NDEBUG
Anupam Pandeyc5ee9a02023-01-06 12:15:48 +05301384
1385#if COLLECT_NONRD_PICK_MODE_STAT
1386 mode_search_stat_nonrd ms_stat_nonrd;
1387#endif // COLLECT_NONRD_PICK_MODE_STAT
Lin Zheng4c3c2c12023-03-02 01:49:58 +00001388
1389 /*!\brief Number of pixels in current thread that choose palette mode in the
1390 * fast encoding stage for screen content tool detemination.
1391 */
1392 int palette_pixels;
Satheesh Kumar7b231692023-10-10 17:25:30 +05301393
1394 /*!\brief Pointer to the structure which stores the statistics used by
1395 * sb-level multi-pass encoding.
1396 */
1397 struct SB_FIRST_PASS_STATS *sb_stats_cache;
1398
1399 /*!\brief Pointer to the structure which stores the statistics used by
1400 * first-pass when superblock is searched twice consecutively.
1401 */
1402 struct SB_FIRST_PASS_STATS *sb_fp_stats;
Satheesh Kumar7a4901e2023-10-27 11:49:42 +05301403
1404#if CONFIG_PARTITION_SEARCH_ORDER
1405 /*!\brief Pointer to RD_STATS structure to be used in
1406 * av1_rd_partition_search().
1407 */
1408 RD_STATS *rdcost;
1409#endif // CONFIG_PARTITION_SEARCH_ORDER
chiyotsaic2f38412020-06-10 16:07:21 -07001410} MACROBLOCK;
1411#undef SINGLE_REF_MODES
Yaowu Xuc27fc142016-08-22 16:08:15 -07001412
chiyotsaic2f38412020-06-10 16:07:21 -07001413/*!\cond */
Vincent Rabaud5a44b622021-06-29 12:07:14 +02001414// Zeroes out 'n_stats' elements in the array x->winner_mode_stats.
1415// It only zeroes out what is necessary in 'color_index_map' (just the block
1416// size, not the whole array).
Wan-Teh Chang12c64e82024-08-08 16:02:19 -07001417static inline void zero_winner_mode_stats(BLOCK_SIZE bsize, int n_stats,
Vincent Rabaud5a44b622021-06-29 12:07:14 +02001418 WinnerModeStats *stats) {
Mudassir Galagnathb9bdc592022-03-30 22:05:27 +05301419 // When winner mode stats are not required, the memory allocation is avoided
1420 // for x->winner_mode_stats. The stats pointer will be NULL in such cases.
1421 if (stats == NULL) return;
1422
Vincent Rabaud5a44b622021-06-29 12:07:14 +02001423 const int block_height = block_size_high[bsize];
1424 const int block_width = block_size_wide[bsize];
1425 for (int i = 0; i < n_stats; ++i) {
1426 WinnerModeStats *const stat = &stats[i];
1427 memset(&stat->mbmi, 0, sizeof(stat->mbmi));
1428 memset(&stat->rd_cost, 0, sizeof(stat->rd_cost));
1429 memset(&stat->rd, 0, sizeof(stat->rd));
1430 memset(&stat->rate_y, 0, sizeof(stat->rate_y));
1431 memset(&stat->rate_uv, 0, sizeof(stat->rate_uv));
1432 // Do not reset the whole array as it is CPU intensive.
1433 memset(&stat->color_index_map, 0,
1434 block_width * block_height * sizeof(stat->color_index_map[0]));
1435 memset(&stat->mode_index, 0, sizeof(stat->mode_index));
1436 }
1437}
1438
Wan-Teh Chang12c64e82024-08-08 16:02:19 -07001439static inline int is_rect_tx_allowed_bsize(BLOCK_SIZE bsize) {
Frederic Barbier0f191da2018-01-03 17:29:26 +01001440 static const char LUT[BLOCK_SIZES_ALL] = {
1441 0, // BLOCK_4X4
1442 1, // BLOCK_4X8
1443 1, // BLOCK_8X4
1444 0, // BLOCK_8X8
1445 1, // BLOCK_8X16
1446 1, // BLOCK_16X8
1447 0, // BLOCK_16X16
1448 1, // BLOCK_16X32
1449 1, // BLOCK_32X16
1450 0, // BLOCK_32X32
1451 1, // BLOCK_32X64
1452 1, // BLOCK_64X32
1453 0, // BLOCK_64X64
Frederic Barbier0f191da2018-01-03 17:29:26 +01001454 0, // BLOCK_64X128
1455 0, // BLOCK_128X64
1456 0, // BLOCK_128X128
Frederic Barbier0f191da2018-01-03 17:29:26 +01001457 1, // BLOCK_4X16
1458 1, // BLOCK_16X4
1459 1, // BLOCK_8X32
1460 1, // BLOCK_32X8
1461 1, // BLOCK_16X64
1462 1, // BLOCK_64X16
Frederic Barbier0f191da2018-01-03 17:29:26 +01001463 };
1464
1465 return LUT[bsize];
1466}
1467
Wan-Teh Chang12c64e82024-08-08 16:02:19 -07001468static inline int is_rect_tx_allowed(const MACROBLOCKD *xd,
Frederic Barbier0f191da2018-01-03 17:29:26 +01001469 const MB_MODE_INFO *mbmi) {
chiyotsai0f5cd052020-08-27 14:37:44 -07001470 return is_rect_tx_allowed_bsize(mbmi->bsize) &&
Frederic Barbier0f191da2018-01-03 17:29:26 +01001471 !xd->lossless[mbmi->segment_id];
1472}
1473
Wan-Teh Chang12c64e82024-08-08 16:02:19 -07001474static inline int tx_size_to_depth(TX_SIZE tx_size, BLOCK_SIZE bsize) {
Urvang Joshidd0376f2018-05-02 16:37:25 -07001475 TX_SIZE ctx_size = max_txsize_rect_lookup[bsize];
Frederic Barbier0f191da2018-01-03 17:29:26 +01001476 int depth = 0;
1477 while (tx_size != ctx_size) {
1478 depth++;
Frederic Barbier4b56b102018-03-30 16:09:34 +02001479 ctx_size = sub_tx_size_map[ctx_size];
Frederic Barbier0f191da2018-01-03 17:29:26 +01001480 assert(depth <= MAX_TX_DEPTH);
1481 }
1482 return depth;
1483}
1484
Wan-Teh Chang12c64e82024-08-08 16:02:19 -07001485static inline void set_blk_skip(uint8_t txb_skip[], int plane, int blk_idx,
Grant Hsu39248c32018-09-18 10:38:44 +08001486 int skip) {
1487 if (skip)
chiyotsai4c1e5c62020-04-30 17:54:14 -07001488 txb_skip[blk_idx] |= 1UL << plane;
Grant Hsu39248c32018-09-18 10:38:44 +08001489 else
chiyotsai4c1e5c62020-04-30 17:54:14 -07001490 txb_skip[blk_idx] &= ~(1UL << plane);
Grant Hsu39248c32018-09-18 10:38:44 +08001491#ifndef NDEBUG
1492 // Set chroma planes to uninitialized states when luma is set to check if
1493 // it will be set later
1494 if (plane == 0) {
chiyotsai4c1e5c62020-04-30 17:54:14 -07001495 txb_skip[blk_idx] |= 1UL << (1 + 4);
1496 txb_skip[blk_idx] |= 1UL << (2 + 4);
Grant Hsu39248c32018-09-18 10:38:44 +08001497 }
1498
1499 // Clear the initialization checking bit
chiyotsai4c1e5c62020-04-30 17:54:14 -07001500 txb_skip[blk_idx] &= ~(1UL << (plane + 4));
Grant Hsu39248c32018-09-18 10:38:44 +08001501#endif
1502}
1503
Wan-Teh Chang12c64e82024-08-08 16:02:19 -07001504static inline int is_blk_skip(uint8_t *txb_skip, int plane, int blk_idx) {
Grant Hsu39248c32018-09-18 10:38:44 +08001505#ifndef NDEBUG
1506 // Check if this is initialized
chiyotsai4c1e5c62020-04-30 17:54:14 -07001507 assert(!(txb_skip[blk_idx] & (1UL << (plane + 4))));
Grant Hsu39248c32018-09-18 10:38:44 +08001508
1509 // The magic number is 0x77, this is to test if there is garbage data
chiyotsai4c1e5c62020-04-30 17:54:14 -07001510 assert((txb_skip[blk_idx] & 0x88) == 0);
Grant Hsu39248c32018-09-18 10:38:44 +08001511#endif
chiyotsai4c1e5c62020-04-30 17:54:14 -07001512 return (txb_skip[blk_idx] >> plane) & 1;
Grant Hsu39248c32018-09-18 10:38:44 +08001513}
1514
chiyotsaic2f38412020-06-10 16:07:21 -07001515/*!\endcond */
1516
Yaowu Xuc27fc142016-08-22 16:08:15 -07001517#ifdef __cplusplus
1518} // extern "C"
1519#endif
1520
James Zerne1cbb132018-08-22 14:10:36 -07001521#endif // AOM_AV1_ENCODER_BLOCK_H_