blob: 87456f5722c0b51e13344a04e6586b6088884c55 [file] [log] [blame]
Yaowu Xuc27fc142016-08-22 16:08:15 -07001/*
Yaowu Xu2ab7ff02016-09-02 12:04:54 -07002 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Yaowu Xuc27fc142016-08-22 16:08:15 -07003 *
Yaowu Xu2ab7ff02016-09-02 12:04:54 -07004 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Yaowu Xuc27fc142016-08-22 16:08:15 -070010 */
11
James Zerne1cbb132018-08-22 14:10:36 -070012#ifndef AOM_AV1_ENCODER_BLOCK_H_
13#define AOM_AV1_ENCODER_BLOCK_H_
Yaowu Xuc27fc142016-08-22 16:08:15 -070014
15#include "av1/common/entropymv.h"
16#include "av1/common/entropy.h"
Yaowu Xuc27fc142016-08-22 16:08:15 -070017#include "av1/common/mvref_common.h"
chiyotsai36035d12019-06-19 17:39:48 -070018
Ravi Chaudharyb61cdea2019-07-05 15:01:20 +053019#include "av1/encoder/enc_enums.h"
chiyotsai36035d12019-06-19 17:39:48 -070020#if !CONFIG_REALTIME_ONLY
21#include "av1/encoder/partition_cnn_weights.h"
22#endif
23
Hui Su1ddf2312017-08-19 15:21:34 -070024#include "av1/encoder/hash.h"
Yaowu Xuc27fc142016-08-22 16:08:15 -070025
26#ifdef __cplusplus
27extern "C" {
28#endif
29
Yunqing Wang61b2eac2019-09-24 16:45:16 -070030#define MC_FLOW_BSIZE_1D 16
31#define MC_FLOW_NUM_PELS (MC_FLOW_BSIZE_1D * MC_FLOW_BSIZE_1D)
32#define MAX_MC_FLOW_BLK_IN_SB (MAX_SB_SIZE / MC_FLOW_BSIZE_1D)
Cherma Rajan Ad71ace62019-11-25 12:58:37 +053033#define MAX_WINNER_MODE_COUNT_INTRA 3
34#define MAX_WINNER_MODE_COUNT_INTER 1
chiyotsaiff73c532020-04-21 12:50:12 -070035
36// SuperblockEnc stores superblock level information used by the encoder for
37// more efficient encoding.
38typedef struct {
39 // Below are information gathered from tpl_model used to speed up the encoding
40 // process.
41 int tpl_data_count;
42 int64_t tpl_inter_cost[MAX_MC_FLOW_BLK_IN_SB * MAX_MC_FLOW_BLK_IN_SB];
43 int64_t tpl_intra_cost[MAX_MC_FLOW_BLK_IN_SB * MAX_MC_FLOW_BLK_IN_SB];
44 int_mv tpl_mv[MAX_MC_FLOW_BLK_IN_SB * MAX_MC_FLOW_BLK_IN_SB]
45 [INTER_REFS_PER_FRAME];
46 int tpl_stride;
47} SuperBlockEnc;
48
Cherma Rajan A835f7a62019-09-25 11:04:39 +053049typedef struct {
50 MB_MODE_INFO mbmi;
Cherma Rajan Ad71ace62019-11-25 12:58:37 +053051 RD_STATS rd_cost;
Cherma Rajan A835f7a62019-09-25 11:04:39 +053052 int64_t rd;
Cherma Rajan Ad71ace62019-11-25 12:58:37 +053053 int rate_y;
54 int rate_uv;
Cherma Rajan A835f7a62019-09-25 11:04:39 +053055 uint8_t color_index_map[64 * 64];
Cherma Rajan Ad71ace62019-11-25 12:58:37 +053056 THR_MODES mode_index;
Cherma Rajan A835f7a62019-09-25 11:04:39 +053057} WinnerModeStats;
Yue Chenbd934232019-08-05 14:23:39 -070058
Yaowu Xuc27fc142016-08-22 16:08:15 -070059typedef struct {
60 unsigned int sse;
61 int sum;
62 unsigned int var;
Urvang Joshi454280d2016-10-14 16:51:44 -070063} DIFF;
Yaowu Xuc27fc142016-08-22 16:08:15 -070064
Debargha Mukherjee8a191ef2020-03-24 00:47:21 -070065enum {
66 NO_TRELLIS_OPT, // No trellis optimization
67 FULL_TRELLIS_OPT, // Trellis optimization in all stages
68 FINAL_PASS_TRELLIS_OPT, // Trellis optimization in only the final encode pass
69 NO_ESTIMATE_YRD_TRELLIS_OPT // Disable trellis in estimate_yrd_for_sb
70} UENUM1BYTE(TRELLIS_OPT_TYPE);
71
Yaowu Xuc27fc142016-08-22 16:08:15 -070072typedef struct macroblock_plane {
Ravi Chaudharyd4a5a502019-06-07 12:13:32 +053073 DECLARE_ALIGNED(32, int16_t, src_diff[MAX_SB_SQUARE]);
Urvang Joshi9543ad72020-04-17 16:59:46 -070074 tran_low_t *dqcoeff;
Yaowu Xuc27fc142016-08-22 16:08:15 -070075 tran_low_t *qcoeff;
76 tran_low_t *coeff;
77 uint16_t *eobs;
Angie Chiang74e23072017-03-24 14:54:23 -070078 uint8_t *txb_entropy_ctx;
Yaowu Xuc27fc142016-08-22 16:08:15 -070079 struct buf_2d src;
80
81 // Quantizer setings
Monty Montgomery125c0fc2017-10-26 00:44:35 -040082 // These are used/accessed only in the quantization process
83 // RDO does not / must not depend on any of these values
84 // All values below share the coefficient scale/shift used in TX
85 const int16_t *quant_fp_QTX;
86 const int16_t *round_fp_QTX;
87 const int16_t *quant_QTX;
88 const int16_t *quant_shift_QTX;
89 const int16_t *zbin_QTX;
90 const int16_t *round_QTX;
91 const int16_t *dequant_QTX;
Yaowu Xuc27fc142016-08-22 16:08:15 -070092} MACROBLOCK_PLANE;
93
Jingning Handfd72322017-08-09 14:04:12 -070094typedef struct {
95 int txb_skip_cost[TXB_SKIP_CONTEXTS][2];
Dake He3fe369c2017-11-16 17:56:44 -080096 int base_eob_cost[SIG_COEF_CONTEXTS_EOB][3];
Wenyao Liuf7e53752019-01-22 17:34:44 +080097 int base_cost[SIG_COEF_CONTEXTS][8];
Angie Chiang7ab884e2017-10-18 15:57:12 -070098 int eob_extra_cost[EOB_COEF_CONTEXTS][2];
Jingning Handfd72322017-08-09 14:04:12 -070099 int dc_sign_cost[DC_SIGN_CONTEXTS][2];
Wenyao Liuf7e53752019-01-22 17:34:44 +0800100 int lps_cost[LEVEL_CONTEXTS][COEFF_BASE_RANGE + 1 + COEFF_BASE_RANGE + 1];
Jingning Handfd72322017-08-09 14:04:12 -0700101} LV_MAP_COEFF_COST;
Jingning Hanf5a4d3b2017-08-27 23:01:19 -0700102
Johannb0ef6ff2018-02-08 14:32:21 -0800103typedef struct {
104 int eob_cost[2][11];
105} LV_MAP_EOB_COST;
Dake He0db7d0e2017-12-21 15:23:20 -0800106
Jingning Hanf5a4d3b2017-08-27 23:01:19 -0700107typedef struct {
108 tran_low_t tcoeff[MAX_MB_PLANE][MAX_SB_SQUARE];
109 uint16_t eobs[MAX_MB_PLANE][MAX_SB_SQUARE / (TX_SIZE_W_MIN * TX_SIZE_H_MIN)];
Hui Suc328ea22019-10-01 15:01:27 -0700110 // Transform block entropy contexts.
111 // Bits 0~3: txb_skip_ctx; bits 4~5: dc_sign_ctx.
112 uint8_t entropy_ctx[MAX_MB_PLANE]
113 [MAX_SB_SQUARE / (TX_SIZE_W_MIN * TX_SIZE_H_MIN)];
Jingning Hanf5a4d3b2017-08-27 23:01:19 -0700114} CB_COEFF_BUFFER;
Jingning Handfd72322017-08-09 14:04:12 -0700115
Yaowu Xuc27fc142016-08-22 16:08:15 -0700116typedef struct {
Angie Chiangc484abe2017-03-20 15:43:11 -0700117 // TODO(angiebird): Reduce the buffer size according to sb_type
Ravi Chaudharyfa73e202019-08-19 12:41:26 +0530118 CANDIDATE_MV ref_mv_stack[MODE_CTX_REF_FRAMES][USABLE_REF_MV_STACK_SIZE];
119 uint16_t weight[MODE_CTX_REF_FRAMES][USABLE_REF_MV_STACK_SIZE];
Angie Chiang4c9b6022018-04-10 16:16:45 -0700120 int_mv global_mvs[REF_FRAMES];
Satish Kumar Suman69e93292018-11-28 16:05:33 +0530121 int16_t mode_context[MODE_CTX_REF_FRAMES];
122 uint8_t ref_mv_count[MODE_CTX_REF_FRAMES];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700123} MB_MODE_INFO_EXT;
124
Visheshd1317912020-04-07 14:39:44 +0530125// Structure to store best mode information at frame level. This
Remya0cce44c2019-08-16 11:57:24 +0530126// frame level information will be used during bitstream preparation stage.
127typedef struct {
Ravi Chaudharyfa73e202019-08-19 12:41:26 +0530128 CANDIDATE_MV ref_mv_stack[USABLE_REF_MV_STACK_SIZE];
129 uint16_t weight[USABLE_REF_MV_STACK_SIZE];
Remya0cce44c2019-08-16 11:57:24 +0530130 // TODO(Ravi/Remya): Reduce the buffer size of global_mvs
131 int_mv global_mvs[REF_FRAMES];
132 int cb_offset;
133 int16_t mode_context;
134 uint8_t ref_mv_count;
135} MB_MODE_INFO_EXT_FRAME;
136
Alex Converse0fa0f422017-04-24 12:51:14 -0700137typedef struct {
Hui Su473cf892017-11-08 18:14:31 -0800138 uint8_t best_palette_color_map[MAX_PALETTE_SQUARE];
Hui Su5891f982017-12-18 16:18:23 -0800139 int kmeans_data_buf[2 * MAX_PALETTE_SQUARE];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700140} PALETTE_BUFFER;
141
Hui Su1ddf2312017-08-19 15:21:34 -0700142typedef struct {
Hui Su1ddf2312017-08-19 15:21:34 -0700143 TX_SIZE tx_size;
Hui Su7167d952018-02-01 16:33:12 -0800144 TX_SIZE inter_tx_size[INTER_TX_SIZE_BUF_LEN];
Hui Suf4b79c72018-03-22 13:14:36 -0700145 uint8_t blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE];
Hui Su52b7ddc2019-10-10 16:27:16 -0700146 uint8_t tx_type_map[MAX_MIB_SIZE * MAX_MIB_SIZE];
Hui Su1ddf2312017-08-19 15:21:34 -0700147 RD_STATS rd_stats;
148 uint32_t hash_value;
Hui Su6cb17c12018-03-09 12:56:20 -0800149} MB_RD_INFO;
Hui Su1ddf2312017-08-19 15:21:34 -0700150
151#define RD_RECORD_BUFFER_LEN 8
152typedef struct {
Hui Su6cb17c12018-03-09 12:56:20 -0800153 MB_RD_INFO tx_rd_info[RD_RECORD_BUFFER_LEN]; // Circular buffer.
Hui Su1ddf2312017-08-19 15:21:34 -0700154 int index_start;
155 int num;
Peng Bin8a204cd2018-04-08 13:07:35 +0800156 CRC32C crc_calculator; // Hash function.
Hui Su6cb17c12018-03-09 12:56:20 -0800157} MB_RD_RECORD;
Hui Su1ddf2312017-08-19 15:21:34 -0700158
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700159typedef struct {
160 int64_t dist;
Jingning Han73bc2aa2018-02-02 14:31:39 -0800161 int64_t sse;
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700162 int rate;
Hui Su8c2b9132017-12-09 10:40:15 -0800163 uint16_t eob;
Jingning Han73bc2aa2018-02-02 14:31:39 -0800164 TX_TYPE tx_type;
Jingning Han45027c62017-12-11 11:47:15 -0800165 uint16_t entropy_context;
Jingning Hand7e99112017-12-13 09:47:45 -0800166 uint8_t txb_entropy_ctx;
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700167 uint8_t valid;
Hui Su950b9122018-02-03 10:21:40 -0800168 uint8_t fast; // This is not being used now.
Sachin Kumar Garg4ec28cb2019-06-06 19:33:58 +0530169 uint8_t perform_block_coeff_opt;
Hui Su6cb17c12018-03-09 12:56:20 -0800170} TXB_RD_INFO;
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700171
172#define TX_SIZE_RD_RECORD_BUFFER_LEN 256
173typedef struct {
174 uint32_t hash_vals[TX_SIZE_RD_RECORD_BUFFER_LEN];
Hui Su6cb17c12018-03-09 12:56:20 -0800175 TXB_RD_INFO tx_rd_info[TX_SIZE_RD_RECORD_BUFFER_LEN];
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700176 int index_start;
177 int num;
Hui Su6cb17c12018-03-09 12:56:20 -0800178} TXB_RD_RECORD;
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700179
180typedef struct tx_size_rd_info_node {
Hui Su6cb17c12018-03-09 12:56:20 -0800181 TXB_RD_INFO *rd_info_array; // Points to array of size TX_TYPES.
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700182 struct tx_size_rd_info_node *children[4];
Hui Su6cb17c12018-03-09 12:56:20 -0800183} TXB_RD_INFO_NODE;
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700184
Peng Bin525e2da2018-09-06 11:32:58 +0800185// Simple translation rd state for prune_comp_search_by_single_result
186typedef struct {
187 RD_STATS rd_stats;
188 RD_STATS rd_stats_y;
189 RD_STATS rd_stats_uv;
190 uint8_t blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE];
Hui Su52b7ddc2019-10-10 16:27:16 -0700191 uint8_t tx_type_map[MAX_MIB_SIZE * MAX_MIB_SIZE];
chiyotsai8c004e12020-04-17 15:52:08 -0700192 uint8_t skip_txfm;
193 uint8_t disable_skip_txfm;
Peng Bin525e2da2018-09-06 11:32:58 +0800194 uint8_t early_skipped;
195} SimpleRDState;
196
197// 4: NEAREST, NEW, NEAR, GLOBAL
198#define SINGLE_REF_MODES ((REF_FRAMES - 1) * 4)
199
Ranjit Kumar Tulabandua1ebb572018-12-24 12:13:54 +0530200#define MAX_COMP_RD_STATS 64
201typedef struct {
202 int32_t rate[COMPOUND_TYPES];
203 int64_t dist[COMPOUND_TYPES];
Venkat457e32e2019-12-19 17:44:05 +0530204 int32_t model_rate[COMPOUND_TYPES];
205 int64_t model_dist[COMPOUND_TYPES];
venkat sanampudic88148e2020-01-03 12:57:28 +0530206 int comp_rs2[COMPOUND_TYPES];
Ranjit Kumar Tulabandua1ebb572018-12-24 12:13:54 +0530207 int_mv mv[2];
Hui Sud06ff662019-01-23 16:53:05 -0800208 MV_REFERENCE_FRAME ref_frames[2];
Ranjit Kumar Tulabandua1ebb572018-12-24 12:13:54 +0530209 PREDICTION_MODE mode;
Ravi Chaudhary1e4f94b2019-06-20 16:19:49 +0530210 int_interpfilters filter;
Ranjit Kumar Tulabandua1ebb572018-12-24 12:13:54 +0530211 int ref_mv_idx;
212 int is_global[2];
venkat sanampudic88148e2020-01-03 12:57:28 +0530213 INTERINTER_COMPOUND_DATA interinter_comp;
Ranjit Kumar Tulabandua1ebb572018-12-24 12:13:54 +0530214} COMP_RD_STATS;
215
Satish Kumar Sumanb8983c42020-01-23 11:06:19 +0530216// Struct for buffers used by av1_compound_type_rd() function.
Hui Su38711e72019-06-11 10:49:47 -0700217// For sizes and alignment of these arrays, refer to
218// alloc_compound_type_rd_buffers() function.
219typedef struct {
220 uint8_t *pred0;
221 uint8_t *pred1;
222 int16_t *residual1; // src - pred1
223 int16_t *diff10; // pred1 - pred0
224 uint8_t *tmp_best_mask_buf; // backup of the best segmentation mask
225} CompoundTypeRdBuffers;
226
chiyotsaid2b12212020-04-28 20:57:19 -0700227// Struct for buffers used to speed up rdopt for obmc.
228// See the comments for calc_target_weighted_pred for details.
229typedef struct {
230 // A new source weighted with the above and left predictors for efficient
231 // rdopt in obmc mode.
232 int32_t *wsrc;
233 // A new mask constructed from the original left and horizontal masks for
234 // fast obmc rdopt.
235 int32_t *mask;
236 // Holds a prediction using the above/left predictor. This is used to build
237 // the obmc predictor.
238 uint8_t *above_pred;
239 uint8_t *left_pred;
240} OBMCBuffer;
241
chiyotsaic95e3642020-04-10 13:17:06 -0700242typedef struct {
243 // A multiplier that converts mv cost to l2 error.
244 int errorperbit;
245 // A multiplier that converts mv cost to l1 error.
246 int sadperbit;
247
248 int nmv_joint_cost[MV_JOINTS];
249
250 // Below are the entropy costs needed to encode a given mv.
251 // nmv_costs_(hp_)alloc are two arrays that holds the memory
252 // for holding the mv cost. But since the motion vectors can be negative, we
253 // shift them to the middle and store the resulting pointer in nmvcost(_hp)
254 // for easier referencing. Finally, nmv_cost_stack points to the nmvcost array
255 // with the mv precision we are currently working with. In essence, only
256 // mv_cost_stack is needed for motion search, the other can be considered
257 // private.
258 int nmv_cost_alloc[2][MV_VALS];
259 int nmv_cost_hp_alloc[2][MV_VALS];
260 int *nmv_cost[2];
261 int *nmv_cost_hp[2];
262 int **mv_cost_stack;
263} MvCostInfo;
chiyotsai9365bff2020-01-31 14:40:05 -0800264
Ravi Chaudhary5d970f42018-09-25 11:25:32 +0530265struct inter_modes_info;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700266typedef struct macroblock MACROBLOCK;
267struct macroblock {
268 struct macroblock_plane plane[MAX_MB_PLANE];
269
Jingning Han66965a22018-01-25 09:53:41 -0800270 // Determine if one would go with reduced complexity transform block
271 // search model to select prediction modes, or full complexity model
272 // to select transform kernel.
Jingning Handd8600f2018-01-23 09:06:32 -0800273 int rd_model;
274
elliottkdfc59242019-04-25 16:38:51 -0700275 // prune_comp_search_by_single_result (3:MAX_REF_MV_SEARCH)
Peng Bin525e2da2018-09-06 11:32:58 +0800276 SimpleRDState simple_rd_state[SINGLE_REF_MODES][3];
277
Hui Su6cb17c12018-03-09 12:56:20 -0800278 // Inter macroblock RD search info.
279 MB_RD_RECORD mb_rd_record;
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700280
Hui Su6cb17c12018-03-09 12:56:20 -0800281 // Inter transform block RD search info. for square TX sizes.
282 TXB_RD_RECORD txb_rd_record_8X8[(MAX_MIB_SIZE >> 1) * (MAX_MIB_SIZE >> 1)];
283 TXB_RD_RECORD txb_rd_record_16X16[(MAX_MIB_SIZE >> 2) * (MAX_MIB_SIZE >> 2)];
284 TXB_RD_RECORD txb_rd_record_32X32[(MAX_MIB_SIZE >> 3) * (MAX_MIB_SIZE >> 3)];
285 TXB_RD_RECORD txb_rd_record_64X64[(MAX_MIB_SIZE >> 4) * (MAX_MIB_SIZE >> 4)];
286
287 // Intra transform block RD search info. for square TX sizes.
288 TXB_RD_RECORD txb_rd_record_intra;
Hui Suec5a1ab2018-03-08 15:09:12 -0800289
Yaowu Xuc27fc142016-08-22 16:08:15 -0700290 MACROBLOCKD e_mbd;
291 MB_MODE_INFO_EXT *mbmi_ext;
Remya0cce44c2019-08-16 11:57:24 +0530292 MB_MODE_INFO_EXT_FRAME *mbmi_ext_frame;
Cherma Rajan A835f7a62019-09-25 11:04:39 +0530293 // Array of mode stats for winner mode processing
Cherma Rajan Ad71ace62019-11-25 12:58:37 +0530294 WinnerModeStats winner_mode_stats[AOMMAX(MAX_WINNER_MODE_COUNT_INTRA,
295 MAX_WINNER_MODE_COUNT_INTER)];
Cherma Rajan A835f7a62019-09-25 11:04:39 +0530296 int winner_mode_count;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700297 int skip_block;
David Barkerd7d78c82016-10-24 10:55:35 +0100298 int qindex;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700299
Urvang Joshi69a986c2020-04-27 16:13:11 -0700300 // The difference between the frame-level base qindex and the qindex used for
301 // the current superblock. This is used to track whether a non-zero delta for
302 // qindex is used at least once in the current frame.
303 int delta_qindex;
304
Yaowu Xuc27fc142016-08-22 16:08:15 -0700305 int rdmult;
306 int mb_energy;
Yue Chende730472018-06-20 10:13:15 -0700307 int sb_energy_level;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700308
Jingning Han9777afc2016-10-20 15:17:43 -0700309 unsigned int txb_split_count;
Debargha Mukherjee0857e662019-01-04 16:22:09 -0800310#if CONFIG_SPEED_STATS
311 unsigned int tx_search_count;
312#endif // CONFIG_SPEED_STATS
Jingning Han9777afc2016-10-20 15:17:43 -0700313
Yaowu Xuc27fc142016-08-22 16:08:15 -0700314 // These are set to their default values at the beginning, and then adjusted
315 // further in the encoding process.
316 BLOCK_SIZE min_partition_size;
317 BLOCK_SIZE max_partition_size;
318
Zoe Liu27deb382018-03-27 15:13:56 -0700319 unsigned int max_mv_context[REF_FRAMES];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700320 unsigned int source_variance;
Yue Chen4f719c22019-02-15 15:13:27 -0800321 unsigned int simple_motion_pred_sse;
Zoe Liu27deb382018-03-27 15:13:56 -0700322 unsigned int pred_sse[REF_FRAMES];
323 int pred_mv_sad[REF_FRAMES];
Lokeshwar Reddy Bce8b7592019-07-09 18:19:55 +0530324 int best_pred_mv_sad;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700325
chiyotsaid2b12212020-04-28 20:57:19 -0700326 // Buffers used to hold/create predictions during rdopt
327 OBMCBuffer obmc_buffer;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700328 PALETTE_BUFFER *palette_buffer;
Hui Su38711e72019-06-11 10:49:47 -0700329 CompoundTypeRdBuffers comp_rd_buffer;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700330
chiyotsaid2b12212020-04-28 20:57:19 -0700331 // A buffer used for convolution during the averaging prediction in compound
332 // mode.
Urvang Joshi0a4cfad2018-09-07 11:10:39 -0700333 CONV_BUF_TYPE *tmp_conv_dst;
chiyotsai2a897eb2020-04-28 19:22:13 -0700334
335 // Points to a buffer that is used to hold temporary prediction results. This
336 // is used in two ways:
337 // 1. This is a temporary buffer used to pingpong the prediction in
338 // handle_inter_mode.
339 // 2. xd->tmp_obmc_bufs also points to this buffer, and is used in ombc
chiyotsaid2b12212020-04-28 20:57:19 -0700340 // prediction.
chiyotsai2a897eb2020-04-28 19:22:13 -0700341 uint8_t *tmp_pred_bufs[2];
Urvang Joshi0a4cfad2018-09-07 11:10:39 -0700342
Ravi Chaudharye5215172018-12-21 18:47:25 +0530343 FRAME_CONTEXT *row_ctx;
Ravi Chaudhary982ac042018-11-02 14:30:29 +0530344 // This context will be used to update color_map_cdf pointer which would be
345 // used during pack bitstream. For single thread and tile-multithreading case
346 // this ponter will be same as xd->tile_ctx, but for the case of row-mt:
347 // xd->tile_ctx will point to a temporary context while tile_pb_ctx will point
348 // to the accurate tile context.
349 FRAME_CONTEXT *tile_pb_ctx;
Ravi Chaudhary84a280a2018-09-24 16:09:48 +0530350
Ravi Chaudhary5d970f42018-09-25 11:25:32 +0530351 struct inter_modes_info *inter_modes_info;
Ravi Chaudhary5d970f42018-09-25 11:25:32 +0530352
chiyotsai82f36c92020-04-09 16:18:02 -0700353 // Contains the hash table, hash function, and buffer used for intrabc
354 IntraBCHashInfo intrabc_hash_info;
Ravi Chaudhary783d6a32018-08-28 18:21:02 +0530355
Yaowu Xuc27fc142016-08-22 16:08:15 -0700356 // These define limits to motion vector components to prevent them
357 // from extending outside the UMV borders
chiyotsai2ad959b2020-02-12 14:29:32 -0800358 FullMvLimits mv_limits;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700359
chiyotsaic95e3642020-04-10 13:17:06 -0700360 // Stores the entropy cost needed to encode a motion vector.
361 MvCostInfo mv_cost_info;
362
Hui Suf4b79c72018-03-22 13:14:36 -0700363 uint8_t blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE];
Hui Su52b7ddc2019-10-10 16:27:16 -0700364 uint8_t tx_type_map[MAX_MIB_SIZE * MAX_MIB_SIZE];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700365
chiyotsai8c004e12020-04-17 15:52:08 -0700366 // Forces the coding block to skip transform and quantization.
367 int skip_txfm;
368 int skip_txfm_cost[SKIP_CONTEXTS][2];
Zoe Liu1eed2df2017-10-16 17:13:15 -0700369
chiyotsai8c004e12020-04-17 15:52:08 -0700370 // Skip mode tries to use the closest forward and backward references for
371 // inter prediction. Skip here means to skip transmitting the reference
372 // frames, not to be confused with skip_txfm.
Zoe Liuf40a9572017-10-13 12:37:19 -0700373 int skip_mode; // 0: off; 1: on
chiyotsai8c004e12020-04-17 15:52:08 -0700374 int skip_mode_cost[SKIP_MODE_CONTEXTS][2];
Zoe Liuf40a9572017-10-13 12:37:19 -0700375
Jingning Handfd72322017-08-09 14:04:12 -0700376 LV_MAP_COEFF_COST coeff_costs[TX_SIZES][PLANE_TYPES];
Dake He0db7d0e2017-12-21 15:23:20 -0800377 LV_MAP_EOB_COST eob_costs[7][2];
Jingning Hanf5a4d3b2017-08-27 23:01:19 -0700378 uint16_t cb_offset;
Jingning Handfd72322017-08-09 14:04:12 -0700379
Yue Chenb23d00a2017-07-28 17:01:21 -0700380 // mode costs
Yue Chen170678a2017-10-17 13:43:10 -0700381 int intra_inter_cost[INTRA_INTER_CONTEXTS][2];
382
Yue Chenb23d00a2017-07-28 17:01:21 -0700383 int mbmode_cost[BLOCK_SIZE_GROUPS][INTRA_MODES];
384 int newmv_mode_cost[NEWMV_MODE_CONTEXTS][2];
Sarah Parker2b9ec2e2017-10-30 17:34:08 -0700385 int zeromv_mode_cost[GLOBALMV_MODE_CONTEXTS][2];
Yue Chenb23d00a2017-07-28 17:01:21 -0700386 int refmv_mode_cost[REFMV_MODE_CONTEXTS][2];
387 int drl_mode_cost0[DRL_MODE_CONTEXTS][2];
388
Hui Su9d0c03d2017-12-27 16:05:23 -0800389 int comp_inter_cost[COMP_INTER_CONTEXTS][2];
Hui Su3d30b4b2017-12-27 16:47:59 -0800390 int single_ref_cost[REF_CONTEXTS][SINGLE_REFS - 1][2];
Hui Su6b3d1e32018-01-05 11:25:40 -0800391 int comp_ref_type_cost[COMP_REF_TYPE_CONTEXTS]
392 [CDF_SIZE(COMP_REFERENCE_TYPES)];
Hui Sua7e3bfe2018-01-05 12:14:48 -0800393 int uni_comp_ref_cost[UNI_COMP_REF_CONTEXTS][UNIDIR_COMP_REFS - 1]
394 [CDF_SIZE(2)];
Hui Su0bdf5f52018-01-05 14:54:32 -0800395 // Cost for signaling ref_frame[0] (LAST_FRAME, LAST2_FRAME, LAST3_FRAME or
396 // GOLDEN_FRAME) in bidir-comp mode.
Zoe Liu3b353472018-02-12 13:58:22 -0800397 int comp_ref_cost[REF_CONTEXTS][FWD_REFS - 1][2];
Hui Su0bdf5f52018-01-05 14:54:32 -0800398 // Cost for signaling ref_frame[1] (ALTREF_FRAME, ALTREF2_FRAME, or
399 // BWDREF_FRAME) in bidir-comp mode.
Zoe Liu3b353472018-02-12 13:58:22 -0800400 int comp_bwdref_cost[REF_CONTEXTS][BWD_REFS - 1][2];
Yue Chenb23d00a2017-07-28 17:01:21 -0700401 int inter_compound_mode_cost[INTER_MODE_CONTEXTS][INTER_COMPOUND_MODES];
Debargha Mukherjee54eabb52019-02-01 16:54:33 -0800402 int compound_type_cost[BLOCK_SIZES_ALL][MASKED_COMPOUND_TYPES];
Yue Chen73335fa2017-12-20 23:33:41 -0800403 int wedge_idx_cost[BLOCK_SIZES_ALL][16];
Yue Cheneaf128a2017-10-16 17:01:36 -0700404 int interintra_cost[BLOCK_SIZE_GROUPS][2];
405 int wedge_interintra_cost[BLOCK_SIZES_ALL][2];
Yue Chenb23d00a2017-07-28 17:01:21 -0700406 int interintra_mode_cost[BLOCK_SIZE_GROUPS][INTERINTRA_MODES];
Yue Chenb23d00a2017-07-28 17:01:21 -0700407 int motion_mode_cost[BLOCK_SIZES_ALL][MOTION_MODES];
Yue Chenb23d00a2017-07-28 17:01:21 -0700408 int motion_mode_cost1[BLOCK_SIZES_ALL][2];
David Michael Barrcb3a8ef2018-01-06 15:48:49 +0900409 int intra_uv_mode_cost[CFL_ALLOWED_TYPES][INTRA_MODES][UV_INTRA_MODES];
Yue Chenb23d00a2017-07-28 17:01:21 -0700410 int y_mode_costs[INTRA_MODES][INTRA_MODES][INTRA_MODES];
Yue Chen45dfb792018-03-01 13:02:40 -0800411 int filter_intra_cost[BLOCK_SIZES_ALL][2];
Yue Chen994dba22017-12-19 15:27:26 -0800412 int filter_intra_mode_cost[FILTER_INTRA_MODES];
Yue Chenb23d00a2017-07-28 17:01:21 -0700413 int switchable_interp_costs[SWITCHABLE_FILTER_CONTEXTS][SWITCHABLE_FILTERS];
Sebastien Alaiwana6a486c2017-11-07 17:04:27 +0100414 int partition_cost[PARTITION_CONTEXTS][EXT_PARTITION_TYPES];
Hui Suc1f411b2017-12-19 15:58:28 -0800415 int palette_y_size_cost[PALATTE_BSIZE_CTXS][PALETTE_SIZES];
416 int palette_uv_size_cost[PALATTE_BSIZE_CTXS][PALETTE_SIZES];
Yue Chenb23d00a2017-07-28 17:01:21 -0700417 int palette_y_color_cost[PALETTE_SIZES][PALETTE_COLOR_INDEX_CONTEXTS]
418 [PALETTE_COLORS];
419 int palette_uv_color_cost[PALETTE_SIZES][PALETTE_COLOR_INDEX_CONTEXTS]
420 [PALETTE_COLORS];
Hui Suc1f411b2017-12-19 15:58:28 -0800421 int palette_y_mode_cost[PALATTE_BSIZE_CTXS][PALETTE_Y_MODE_CONTEXTS][2];
Yue Chendab2ca92017-10-16 17:48:48 -0700422 int palette_uv_mode_cost[PALETTE_UV_MODE_CONTEXTS][2];
David Michael Barr38e560c2017-08-16 21:46:37 +0900423 // The rate associated with each alpha codeword
424 int cfl_cost[CFL_JOINT_SIGNS][CFL_PRED_PLANES][CFL_ALPHABET_SIZE];
Yue Chenb23d00a2017-07-28 17:01:21 -0700425 int tx_size_cost[TX_SIZES - 1][TX_SIZE_CONTEXTS][TX_SIZES];
Yue Chen171c17d2017-10-16 18:08:22 -0700426 int txfm_partition_cost[TXFM_PARTITION_CONTEXTS][2];
Yue Chenb23d00a2017-07-28 17:01:21 -0700427 int inter_tx_type_costs[EXT_TX_SETS_INTER][EXT_TX_SIZES][TX_TYPES];
428 int intra_tx_type_costs[EXT_TX_SETS_INTRA][EXT_TX_SIZES][INTRA_MODES]
429 [TX_TYPES];
Joe Young3ca43bf2017-10-06 15:12:46 -0700430 int angle_delta_cost[DIRECTIONAL_MODES][2 * MAX_ANGLE_DELTA + 1];
Yue Chenb23d00a2017-07-28 17:01:21 -0700431 int switchable_restore_cost[RESTORE_SWITCHABLE_TYPES];
Debargha Mukherjeebc732ef2017-10-12 12:40:25 -0700432 int wiener_restore_cost[2];
433 int sgrproj_restore_cost[2];
Hui Su6c8584f2017-09-14 15:37:02 -0700434 int intrabc_cost[2];
Yue Chenb23d00a2017-07-28 17:01:21 -0700435
Yaowu Xuc27fc142016-08-22 16:08:15 -0700436 // Used to store sub partition's choices.
Zoe Liu27deb382018-03-27 15:13:56 -0700437 MV pred_mv[REF_FRAMES];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700438
Hui Su835bbd52019-03-05 23:18:15 -0800439 // Ref frames that are selected by square partition blocks within a super-
440 // block, in MI resolution. They can be used to prune ref frames for
441 // rectangular blocks.
442 int picked_ref_frames_mask[32 * 32];
443
Yaowu Xuc27fc142016-08-22 16:08:15 -0700444 // use default transform and skip transform type search for intra modes
445 int use_default_intra_tx_type;
446 // use default transform and skip transform type search for inter modes
447 int use_default_inter_tx_type;
Cheng Chen46970612017-10-24 14:53:36 -0700448 int comp_idx_cost[COMP_INDEX_CONTEXTS][2];
Cheng Chen2ef24ea2017-11-29 12:22:24 -0800449 int comp_group_idx_cost[COMP_GROUP_IDX_CONTEXTS][2];
Hui Su845a7412018-03-08 12:21:00 -0800450 int must_find_valid_partition;
Ranjit Kumar Tulabandu8105bf42018-07-27 14:16:55 +0530451 int recalc_luma_mc_data; // Flag to indicate recalculation of MC data during
452 // interpolation filter search
Cherma Rajan A12478f32019-10-31 10:03:34 +0530453 int prune_mode;
Cherma Rajan A628efce2019-09-18 18:55:12 +0530454 uint32_t tx_domain_dist_threshold;
Cherma Rajan Aba4e1562019-08-29 14:45:22 +0530455 int use_transform_domain_distortion;
elliottk1dbb8c12018-11-06 22:28:59 -0800456 // The likelihood of an edge existing in the block (using partial Canny edge
457 // detection). For reference, 556 is the value returned for a solid
458 // vertical black/white edge.
459 uint16_t edge_strength;
elliottk17dd3ea2019-02-01 11:59:57 -0800460 // The strongest edge strength seen along the x/y axis.
461 uint16_t edge_strength_x;
462 uint16_t edge_strength_y;
Remya05be56a2019-04-15 19:09:08 +0530463 uint8_t compound_idx;
Ranjit Kumar Tulabandua1ebb572018-12-24 12:13:54 +0530464
465 // [Saved stat index]
466 COMP_RD_STATS comp_rd_stats[MAX_COMP_RD_STATS];
467 int comp_rd_stats_idx;
Hui Suf1b11062019-04-17 14:18:26 -0700468
469 CB_COEFF_BUFFER *cb_coef_buff;
chiyotsai36035d12019-06-19 17:39:48 -0700470
Ranjit Kumar Tulabandu824f9db2019-07-05 15:16:37 +0530471 // Threshold used to decide the applicability of R-D optimization of
472 // quantized coeffs
473 uint32_t coeff_opt_dist_threshold;
474
chiyotsai36035d12019-06-19 17:39:48 -0700475#if !CONFIG_REALTIME_ONLY
476 int quad_tree_idx;
chiyotsai52ac0042019-07-01 09:48:03 -0700477 int cnn_output_valid;
chiyotsai36035d12019-06-19 17:39:48 -0700478 float cnn_buffer[CNN_OUT_BUF_SIZE];
479 float log_q;
480#endif
Ravi Chaudharyb61cdea2019-07-05 15:01:20 +0530481 int thresh_freq_fact[BLOCK_SIZES_ALL][MAX_MODES];
Jerome Jianga996e772020-02-25 11:40:13 -0800482 // 0 - 128x128
483 // 1-2 - 128x64
484 // 3-4 - 64x128
485 // 5-8 - 64x64
486 // 9-16 - 64x32
487 // 17-24 - 32x64
488 // 25-40 - 32x32
489 // 41-104 - 16x16
Fyodor Kyslov59745d92019-07-30 18:16:54 -0700490 uint8_t variance_low[105];
Marco Paniconi1bfbd9f2020-02-25 20:05:08 -0800491 uint8_t content_state_sb;
Fyodor Kyslovabef54f2019-08-14 14:43:12 -0700492 // Strong color activity detection. Used in REALTIME coding mode to enhance
493 // the visual quality at the boundary of moving color objects.
494 uint8_t color_sensitivity[2];
Fyodor Kyslov54ebdad2020-02-27 12:47:02 -0800495 int nonrd_prune_ref_frame_search;
Cherma Rajan A3ed030f2019-08-16 18:56:00 +0530496
497 // Used to control the tx size search evaluation for mode processing
498 // (normal/winner mode)
499 int tx_size_search_method;
chiyotsaif271f9f2019-11-25 16:44:25 -0800500 // This tx_mode_search_type is used internally by the encoder, and is not
501 // written to the bitstream. It determines what kind of tx_mode should be
502 // searched. For example, we might set it to TX_MODE_LARGEST to find a good
503 // candidate, then use TX_MODE_SELECT on it
504 TX_MODE tx_mode_search_type;
Yunqing Wang61b2eac2019-09-24 16:45:16 -0700505
Nithya V Sd0276ac2019-10-24 11:31:06 +0530506 // Used to control aggressiveness of skip flag prediction for mode processing
507 // (normal/winner mode)
508 unsigned int predict_skip_level;
509
Hui Su922db602020-03-18 13:39:29 -0700510 uint8_t search_ref_frame[REF_FRAMES];
Jingning Han185d23b2020-03-04 09:12:05 -0800511
chiyotsaiff73c532020-04-21 12:50:12 -0700512 // The information on a whole superblock level.
513 // TODO(chiyotsai@google.com): Refactor this out of macroblock
514 SuperBlockEnc sb_enc;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700515};
516
Yunqing Wang8317cac2019-09-25 10:52:51 -0700517// Only consider full SB, MC_FLOW_BSIZE_1D = 16.
518static INLINE int tpl_blocks_in_sb(BLOCK_SIZE bsize) {
519 switch (bsize) {
520 case BLOCK_64X64: return 16;
521 case BLOCK_128X128: return 64;
522 default: assert(0);
523 }
524 return -1;
525}
526
Frederic Barbier0f191da2018-01-03 17:29:26 +0100527static INLINE int is_rect_tx_allowed_bsize(BLOCK_SIZE bsize) {
528 static const char LUT[BLOCK_SIZES_ALL] = {
529 0, // BLOCK_4X4
530 1, // BLOCK_4X8
531 1, // BLOCK_8X4
532 0, // BLOCK_8X8
533 1, // BLOCK_8X16
534 1, // BLOCK_16X8
535 0, // BLOCK_16X16
536 1, // BLOCK_16X32
537 1, // BLOCK_32X16
538 0, // BLOCK_32X32
539 1, // BLOCK_32X64
540 1, // BLOCK_64X32
541 0, // BLOCK_64X64
Frederic Barbier0f191da2018-01-03 17:29:26 +0100542 0, // BLOCK_64X128
543 0, // BLOCK_128X64
544 0, // BLOCK_128X128
Frederic Barbier0f191da2018-01-03 17:29:26 +0100545 1, // BLOCK_4X16
546 1, // BLOCK_16X4
547 1, // BLOCK_8X32
548 1, // BLOCK_32X8
549 1, // BLOCK_16X64
550 1, // BLOCK_64X16
Frederic Barbier0f191da2018-01-03 17:29:26 +0100551 };
552
553 return LUT[bsize];
554}
555
556static INLINE int is_rect_tx_allowed(const MACROBLOCKD *xd,
557 const MB_MODE_INFO *mbmi) {
558 return is_rect_tx_allowed_bsize(mbmi->sb_type) &&
559 !xd->lossless[mbmi->segment_id];
560}
561
Frederic Barbier4b56b102018-03-30 16:09:34 +0200562static INLINE int tx_size_to_depth(TX_SIZE tx_size, BLOCK_SIZE bsize) {
Urvang Joshidd0376f2018-05-02 16:37:25 -0700563 TX_SIZE ctx_size = max_txsize_rect_lookup[bsize];
Frederic Barbier0f191da2018-01-03 17:29:26 +0100564 int depth = 0;
565 while (tx_size != ctx_size) {
566 depth++;
Frederic Barbier4b56b102018-03-30 16:09:34 +0200567 ctx_size = sub_tx_size_map[ctx_size];
Frederic Barbier0f191da2018-01-03 17:29:26 +0100568 assert(depth <= MAX_TX_DEPTH);
569 }
570 return depth;
571}
572
Grant Hsu39248c32018-09-18 10:38:44 +0800573static INLINE void set_blk_skip(MACROBLOCK *x, int plane, int blk_idx,
574 int skip) {
575 if (skip)
576 x->blk_skip[blk_idx] |= 1UL << plane;
577 else
578 x->blk_skip[blk_idx] &= ~(1UL << plane);
579#ifndef NDEBUG
580 // Set chroma planes to uninitialized states when luma is set to check if
581 // it will be set later
582 if (plane == 0) {
583 x->blk_skip[blk_idx] |= 1UL << (1 + 4);
584 x->blk_skip[blk_idx] |= 1UL << (2 + 4);
585 }
586
587 // Clear the initialization checking bit
588 x->blk_skip[blk_idx] &= ~(1UL << (plane + 4));
589#endif
590}
591
592static INLINE int is_blk_skip(MACROBLOCK *x, int plane, int blk_idx) {
593#ifndef NDEBUG
594 // Check if this is initialized
595 assert(!(x->blk_skip[blk_idx] & (1UL << (plane + 4))));
596
597 // The magic number is 0x77, this is to test if there is garbage data
598 assert((x->blk_skip[blk_idx] & 0x88) == 0);
599#endif
600 return (x->blk_skip[blk_idx] >> plane) & 1;
601}
602
Yaowu Xuc27fc142016-08-22 16:08:15 -0700603#ifdef __cplusplus
604} // extern "C"
605#endif
606
James Zerne1cbb132018-08-22 14:10:36 -0700607#endif // AOM_AV1_ENCODER_BLOCK_H_