blob: 7cd8590ca7bef4bceb97655735d702e9bf2f9546 [file] [log] [blame]
Yaowu Xuc27fc142016-08-22 16:08:15 -07001/*
Yaowu Xu2ab7ff02016-09-02 12:04:54 -07002 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Yaowu Xuc27fc142016-08-22 16:08:15 -07003 *
Yaowu Xu2ab7ff02016-09-02 12:04:54 -07004 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Yaowu Xuc27fc142016-08-22 16:08:15 -070010 */
11
James Zerne1cbb132018-08-22 14:10:36 -070012#ifndef AOM_AV1_ENCODER_BLOCK_H_
13#define AOM_AV1_ENCODER_BLOCK_H_
Yaowu Xuc27fc142016-08-22 16:08:15 -070014
15#include "av1/common/entropymv.h"
16#include "av1/common/entropy.h"
Yaowu Xuc27fc142016-08-22 16:08:15 -070017#include "av1/common/mvref_common.h"
Hui Su1ddf2312017-08-19 15:21:34 -070018#include "av1/encoder/hash.h"
Yushin Cho55104332017-08-14 16:15:43 -070019#if CONFIG_DIST_8X8
20#include "aom/aomcx.h"
21#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -070022
23#ifdef __cplusplus
24extern "C" {
25#endif
26
27typedef struct {
28 unsigned int sse;
29 int sum;
30 unsigned int var;
Urvang Joshi454280d2016-10-14 16:51:44 -070031} DIFF;
Yaowu Xuc27fc142016-08-22 16:08:15 -070032
33typedef struct macroblock_plane {
34 DECLARE_ALIGNED(16, int16_t, src_diff[MAX_SB_SQUARE]);
35 tran_low_t *qcoeff;
36 tran_low_t *coeff;
37 uint16_t *eobs;
Angie Chiang74e23072017-03-24 14:54:23 -070038 uint8_t *txb_entropy_ctx;
Yaowu Xuc27fc142016-08-22 16:08:15 -070039 struct buf_2d src;
40
41 // Quantizer setings
Monty Montgomery125c0fc2017-10-26 00:44:35 -040042 // These are used/accessed only in the quantization process
43 // RDO does not / must not depend on any of these values
44 // All values below share the coefficient scale/shift used in TX
45 const int16_t *quant_fp_QTX;
46 const int16_t *round_fp_QTX;
47 const int16_t *quant_QTX;
48 const int16_t *quant_shift_QTX;
49 const int16_t *zbin_QTX;
50 const int16_t *round_QTX;
51 const int16_t *dequant_QTX;
Yaowu Xuc27fc142016-08-22 16:08:15 -070052} MACROBLOCK_PLANE;
53
Jingning Handfd72322017-08-09 14:04:12 -070054typedef struct {
55 int txb_skip_cost[TXB_SKIP_CONTEXTS][2];
Dake He3fe369c2017-11-16 17:56:44 -080056 int base_eob_cost[SIG_COEF_CONTEXTS_EOB][3];
Ola Hugosson13892102017-11-06 08:01:44 +010057 int base_cost[SIG_COEF_CONTEXTS][4];
Angie Chiang7ab884e2017-10-18 15:57:12 -070058 int eob_extra_cost[EOB_COEF_CONTEXTS][2];
Jingning Handfd72322017-08-09 14:04:12 -070059 int dc_sign_cost[DC_SIGN_CONTEXTS][2];
Angie Chiang26d3e452017-09-29 17:40:02 -070060 int lps_cost[LEVEL_CONTEXTS][COEFF_BASE_RANGE + 1];
Jingning Handfd72322017-08-09 14:04:12 -070061} LV_MAP_COEFF_COST;
Jingning Hanf5a4d3b2017-08-27 23:01:19 -070062
Johannb0ef6ff2018-02-08 14:32:21 -080063typedef struct {
64 int eob_cost[2][11];
65} LV_MAP_EOB_COST;
Dake He0db7d0e2017-12-21 15:23:20 -080066
Jingning Hanf5a4d3b2017-08-27 23:01:19 -070067typedef struct {
68 tran_low_t tcoeff[MAX_MB_PLANE][MAX_SB_SQUARE];
69 uint16_t eobs[MAX_MB_PLANE][MAX_SB_SQUARE / (TX_SIZE_W_MIN * TX_SIZE_H_MIN)];
70 uint8_t txb_skip_ctx[MAX_MB_PLANE]
71 [MAX_SB_SQUARE / (TX_SIZE_W_MIN * TX_SIZE_H_MIN)];
72 int dc_sign_ctx[MAX_MB_PLANE]
73 [MAX_SB_SQUARE / (TX_SIZE_W_MIN * TX_SIZE_H_MIN)];
74} CB_COEFF_BUFFER;
Jingning Handfd72322017-08-09 14:04:12 -070075
Yaowu Xuc27fc142016-08-22 16:08:15 -070076typedef struct {
Angie Chiangc484abe2017-03-20 15:43:11 -070077 // TODO(angiebird): Reduce the buffer size according to sb_type
Jingning Hanf5a4d3b2017-08-27 23:01:19 -070078 tran_low_t *tcoeff[MAX_MB_PLANE];
79 uint16_t *eobs[MAX_MB_PLANE];
80 uint8_t *txb_skip_ctx[MAX_MB_PLANE];
81 int *dc_sign_ctx[MAX_MB_PLANE];
Yaowu Xuc27fc142016-08-22 16:08:15 -070082 CANDIDATE_MV ref_mv_stack[MODE_CTX_REF_FRAMES][MAX_REF_MV_STACK_SIZE];
Angie Chiang4c9b6022018-04-10 16:16:45 -070083 int_mv global_mvs[REF_FRAMES];
Yaowu Xuc27fc142016-08-22 16:08:15 -070084 int16_t compound_mode_context[MODE_CTX_REF_FRAMES];
Satish Kumar Suman69e93292018-11-28 16:05:33 +053085 int16_t mode_context[MODE_CTX_REF_FRAMES];
86 uint8_t ref_mv_count[MODE_CTX_REF_FRAMES];
Yaowu Xuc27fc142016-08-22 16:08:15 -070087} MB_MODE_INFO_EXT;
88
Alex Converse0fa0f422017-04-24 12:51:14 -070089typedef struct {
90 int col_min;
91 int col_max;
92 int row_min;
93 int row_max;
94} MvLimits;
95
Yaowu Xuc27fc142016-08-22 16:08:15 -070096typedef struct {
Hui Su473cf892017-11-08 18:14:31 -080097 uint8_t best_palette_color_map[MAX_PALETTE_SQUARE];
Hui Su5891f982017-12-18 16:18:23 -080098 int kmeans_data_buf[2 * MAX_PALETTE_SQUARE];
Yaowu Xuc27fc142016-08-22 16:08:15 -070099} PALETTE_BUFFER;
100
Hui Su1ddf2312017-08-19 15:21:34 -0700101typedef struct {
Hui Su1ddf2312017-08-19 15:21:34 -0700102 TX_SIZE tx_size;
Hui Su7167d952018-02-01 16:33:12 -0800103 TX_SIZE inter_tx_size[INTER_TX_SIZE_BUF_LEN];
Hui Suf4b79c72018-03-22 13:14:36 -0700104 uint8_t blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE];
Hui Sude1a5db2018-02-22 15:44:49 -0800105 TX_TYPE txk_type[TXK_TYPE_BUF_LEN];
Hui Su1ddf2312017-08-19 15:21:34 -0700106 RD_STATS rd_stats;
107 uint32_t hash_value;
Hui Su6cb17c12018-03-09 12:56:20 -0800108} MB_RD_INFO;
Hui Su1ddf2312017-08-19 15:21:34 -0700109
110#define RD_RECORD_BUFFER_LEN 8
111typedef struct {
Hui Su6cb17c12018-03-09 12:56:20 -0800112 MB_RD_INFO tx_rd_info[RD_RECORD_BUFFER_LEN]; // Circular buffer.
Hui Su1ddf2312017-08-19 15:21:34 -0700113 int index_start;
114 int num;
Peng Bin8a204cd2018-04-08 13:07:35 +0800115 CRC32C crc_calculator; // Hash function.
Hui Su6cb17c12018-03-09 12:56:20 -0800116} MB_RD_RECORD;
Hui Su1ddf2312017-08-19 15:21:34 -0700117
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700118typedef struct {
119 int64_t dist;
Jingning Han73bc2aa2018-02-02 14:31:39 -0800120 int64_t sse;
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700121 int rate;
Hui Su8c2b9132017-12-09 10:40:15 -0800122 uint16_t eob;
Jingning Han73bc2aa2018-02-02 14:31:39 -0800123 TX_TYPE tx_type;
Jingning Han45027c62017-12-11 11:47:15 -0800124 uint16_t entropy_context;
Jingning Hand7e99112017-12-13 09:47:45 -0800125 uint8_t txb_entropy_ctx;
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700126 uint8_t valid;
Hui Su950b9122018-02-03 10:21:40 -0800127 uint8_t fast; // This is not being used now.
Hui Su6cb17c12018-03-09 12:56:20 -0800128} TXB_RD_INFO;
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700129
130#define TX_SIZE_RD_RECORD_BUFFER_LEN 256
131typedef struct {
132 uint32_t hash_vals[TX_SIZE_RD_RECORD_BUFFER_LEN];
Hui Su6cb17c12018-03-09 12:56:20 -0800133 TXB_RD_INFO tx_rd_info[TX_SIZE_RD_RECORD_BUFFER_LEN];
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700134 int index_start;
135 int num;
Hui Su6cb17c12018-03-09 12:56:20 -0800136} TXB_RD_RECORD;
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700137
138typedef struct tx_size_rd_info_node {
Hui Su6cb17c12018-03-09 12:56:20 -0800139 TXB_RD_INFO *rd_info_array; // Points to array of size TX_TYPES.
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700140 struct tx_size_rd_info_node *children[4];
Hui Su6cb17c12018-03-09 12:56:20 -0800141} TXB_RD_INFO_NODE;
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700142
Peng Bin525e2da2018-09-06 11:32:58 +0800143// Simple translation rd state for prune_comp_search_by_single_result
144typedef struct {
145 RD_STATS rd_stats;
146 RD_STATS rd_stats_y;
147 RD_STATS rd_stats_uv;
148 uint8_t blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE];
149 uint8_t skip;
150 uint8_t disable_skip;
151 uint8_t early_skipped;
152} SimpleRDState;
153
154// 4: NEAREST, NEW, NEAR, GLOBAL
155#define SINGLE_REF_MODES ((REF_FRAMES - 1) * 4)
156
Hui Suaaf5a612018-04-12 15:33:42 -0700157// Region size for mode decision sampling in the first pass of partition
158// search(two_pass_partition_search speed feature), in units of mi size(4).
159// Used by the mode_pruning_based_on_two_pass_partition_search speed feature.
160#define FIRST_PARTITION_PASS_SAMPLE_REGION 8
161#define FIRST_PARTITION_PASS_SAMPLE_REGION_LOG2 3
162#define FIRST_PARTITION_PASS_STATS_TABLES \
163 (MAX_MIB_SIZE >> FIRST_PARTITION_PASS_SAMPLE_REGION_LOG2) * \
164 (MAX_MIB_SIZE >> FIRST_PARTITION_PASS_SAMPLE_REGION_LOG2)
165#define FIRST_PARTITION_PASS_STATS_STRIDE \
166 (MAX_MIB_SIZE_LOG2 - FIRST_PARTITION_PASS_SAMPLE_REGION_LOG2)
167
168static INLINE int av1_first_partition_pass_stats_index(int mi_row, int mi_col) {
169 const int row =
170 (mi_row & MAX_MIB_MASK) >> FIRST_PARTITION_PASS_SAMPLE_REGION_LOG2;
171 const int col =
172 (mi_col & MAX_MIB_MASK) >> FIRST_PARTITION_PASS_SAMPLE_REGION_LOG2;
173 return (row << FIRST_PARTITION_PASS_STATS_STRIDE) + col;
174}
175
176typedef struct {
177 uint8_t ref0_counts[REF_FRAMES]; // Counters for ref_frame[0].
178 uint8_t ref1_counts[REF_FRAMES]; // Counters for ref_frame[1].
179 int sample_counts; // Number of samples collected.
180} FIRST_PARTITION_PASS_STATS;
181
Peng Bin706a3b22018-05-30 19:42:05 +0800182#define MAX_INTERP_FILTER_STATS 64
183typedef struct {
184 InterpFilters filters;
Peng Bind482d172018-06-14 15:55:51 +0800185 int_mv mv[2];
Peng Bin706a3b22018-05-30 19:42:05 +0800186 int8_t ref_frames[2];
Grant Hsu97202d02018-06-28 12:51:49 +0800187 COMPOUND_TYPE comp_type;
Peng Bin706a3b22018-05-30 19:42:05 +0800188} INTERPOLATION_FILTER_STATS;
189
Ranjit Kumar Tulabandua1ebb572018-12-24 12:13:54 +0530190#define MAX_COMP_RD_STATS 64
191typedef struct {
192 int32_t rate[COMPOUND_TYPES];
193 int64_t dist[COMPOUND_TYPES];
194 int_mv mv[2];
195 int8_t ref_frames[2];
196 PREDICTION_MODE mode;
197 InterpFilters filter;
198 int ref_mv_idx;
199 int is_global[2];
200} COMP_RD_STATS;
201
Ravi Chaudhary5d970f42018-09-25 11:25:32 +0530202#if CONFIG_COLLECT_INTER_MODE_RD_STATS
203struct inter_modes_info;
204#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -0700205typedef struct macroblock MACROBLOCK;
206struct macroblock {
207 struct macroblock_plane plane[MAX_MB_PLANE];
208
Jingning Han66965a22018-01-25 09:53:41 -0800209 // Determine if one would go with reduced complexity transform block
210 // search model to select prediction modes, or full complexity model
211 // to select transform kernel.
Jingning Handd8600f2018-01-23 09:06:32 -0800212 int rd_model;
213
Jingning Han4489c262018-01-26 16:43:59 -0800214 // Indicate if the encoder is running in the first pass partition search.
215 // In that case, apply certain speed features therein to reduce the overhead
216 // cost in the first pass search.
Jingning Hanc58ccc32018-01-26 07:46:38 -0800217 int cb_partition_scan;
218
Hui Suaaf5a612018-04-12 15:33:42 -0700219 FIRST_PARTITION_PASS_STATS
220 first_partition_pass_stats[FIRST_PARTITION_PASS_STATS_TABLES];
Hui Subb628a72018-04-05 19:48:23 -0700221
Peng Bin706a3b22018-05-30 19:42:05 +0800222 // [comp_idx][saved stat_idx]
223 INTERPOLATION_FILTER_STATS interp_filter_stats[2][MAX_INTERP_FILTER_STATS];
224 int interp_filter_stats_idx[2];
225
Peng Bin525e2da2018-09-06 11:32:58 +0800226 // prune_comp_search_by_single_result (3:MAX_REF_MV_SERCH)
227 SimpleRDState simple_rd_state[SINGLE_REF_MODES][3];
228
Jingning Han4489c262018-01-26 16:43:59 -0800229 // Activate constrained coding block partition search range.
230 int use_cb_search_range;
231
Hui Su6cb17c12018-03-09 12:56:20 -0800232 // Inter macroblock RD search info.
233 MB_RD_RECORD mb_rd_record;
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700234
Hui Su6cb17c12018-03-09 12:56:20 -0800235 // Inter transform block RD search info. for square TX sizes.
236 TXB_RD_RECORD txb_rd_record_8X8[(MAX_MIB_SIZE >> 1) * (MAX_MIB_SIZE >> 1)];
237 TXB_RD_RECORD txb_rd_record_16X16[(MAX_MIB_SIZE >> 2) * (MAX_MIB_SIZE >> 2)];
238 TXB_RD_RECORD txb_rd_record_32X32[(MAX_MIB_SIZE >> 3) * (MAX_MIB_SIZE >> 3)];
239 TXB_RD_RECORD txb_rd_record_64X64[(MAX_MIB_SIZE >> 4) * (MAX_MIB_SIZE >> 4)];
240
241 // Intra transform block RD search info. for square TX sizes.
242 TXB_RD_RECORD txb_rd_record_intra;
Hui Suec5a1ab2018-03-08 15:09:12 -0800243
Yaowu Xuc27fc142016-08-22 16:08:15 -0700244 MACROBLOCKD e_mbd;
245 MB_MODE_INFO_EXT *mbmi_ext;
246 int skip_block;
David Barkerd7d78c82016-10-24 10:55:35 +0100247 int qindex;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700248
249 // The equivalent error at the current rdmult of one whole bit (not one
250 // bitcost unit).
251 int errorperbit;
252 // The equivalend SAD error of one (whole) bit at the current quantizer
253 // for large blocks.
254 int sadperbit16;
255 // The equivalend SAD error of one (whole) bit at the current quantizer
256 // for sub-8x8 blocks.
257 int sadperbit4;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700258 int rdmult;
Yue Chen7cae98f2018-08-24 10:43:16 -0700259 int cb_rdmult;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700260 int mb_energy;
Yue Chende730472018-06-20 10:13:15 -0700261 int sb_energy_level;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700262 int *m_search_count_ptr;
263 int *ex_search_count_ptr;
264
Jingning Han9777afc2016-10-20 15:17:43 -0700265 unsigned int txb_split_count;
Debargha Mukherjee0857e662019-01-04 16:22:09 -0800266#if CONFIG_SPEED_STATS
267 unsigned int tx_search_count;
268#endif // CONFIG_SPEED_STATS
Jingning Han9777afc2016-10-20 15:17:43 -0700269
Yaowu Xuc27fc142016-08-22 16:08:15 -0700270 // These are set to their default values at the beginning, and then adjusted
271 // further in the encoding process.
272 BLOCK_SIZE min_partition_size;
273 BLOCK_SIZE max_partition_size;
274
Zoe Liu27deb382018-03-27 15:13:56 -0700275 unsigned int max_mv_context[REF_FRAMES];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700276 unsigned int source_variance;
Zoe Liu27deb382018-03-27 15:13:56 -0700277 unsigned int pred_sse[REF_FRAMES];
278 int pred_mv_sad[REF_FRAMES];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700279
Jingning Hanf050fc12018-03-09 14:53:33 -0800280 int nmv_vec_cost[MV_JOINTS];
281 int *nmvcost[2];
282 int *nmvcost_hp[2];
283 int **mv_cost_stack;
Alex Conversea127a792017-05-23 15:27:21 -0700284
Yue Chene9638cc2016-10-10 12:37:54 -0700285 int32_t *wsrc_buf;
286 int32_t *mask_buf;
Jingning Hand064cf02017-06-01 10:00:39 -0700287 uint8_t *above_pred_buf;
288 uint8_t *left_pred_buf;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700289
290 PALETTE_BUFFER *palette_buffer;
291
Urvang Joshi0a4cfad2018-09-07 11:10:39 -0700292 CONV_BUF_TYPE *tmp_conv_dst;
293 uint8_t *tmp_obmc_bufs[2];
294
Ravi Chaudharye5215172018-12-21 18:47:25 +0530295 FRAME_CONTEXT *row_ctx;
Ravi Chaudhary982ac042018-11-02 14:30:29 +0530296 // This context will be used to update color_map_cdf pointer which would be
297 // used during pack bitstream. For single thread and tile-multithreading case
298 // this ponter will be same as xd->tile_ctx, but for the case of row-mt:
299 // xd->tile_ctx will point to a temporary context while tile_pb_ctx will point
300 // to the accurate tile context.
301 FRAME_CONTEXT *tile_pb_ctx;
Ravi Chaudhary84a280a2018-09-24 16:09:48 +0530302
Ravi Chaudhary5d970f42018-09-25 11:25:32 +0530303#if CONFIG_COLLECT_INTER_MODE_RD_STATS
304 struct inter_modes_info *inter_modes_info;
305#endif
306
Ravi Chaudhary783d6a32018-08-28 18:21:02 +0530307 // buffer for hash value calculation of a block
308 // used only in av1_get_block_hash_value()
309 // [first hash/second hash]
310 // [two buffers used ping-pong]
311 uint32_t *hash_value_buffer[2][2];
312
313 CRC_CALCULATOR crc_calculator1;
314 CRC_CALCULATOR crc_calculator2;
315 int g_crc_initialized;
316
Yaowu Xuc27fc142016-08-22 16:08:15 -0700317 // These define limits to motion vector components to prevent them
318 // from extending outside the UMV borders
Alex Converse0fa0f422017-04-24 12:51:14 -0700319 MvLimits mv_limits;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700320
Hui Suf4b79c72018-03-22 13:14:36 -0700321 uint8_t blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700322
323 int skip;
Jingning Han8efdbc82017-02-19 14:40:03 -0800324 int skip_chroma_rd;
Zoe Liu1eed2df2017-10-16 17:13:15 -0700325 int skip_cost[SKIP_CONTEXTS][2];
326
Zoe Liuf40a9572017-10-13 12:37:19 -0700327 int skip_mode; // 0: off; 1: on
328 int skip_mode_cost[SKIP_CONTEXTS][2];
329
Zoe Liu104d62e2017-12-07 12:44:45 -0800330 int compound_idx;
Zoe Liuf40a9572017-10-13 12:37:19 -0700331
Jingning Handfd72322017-08-09 14:04:12 -0700332 LV_MAP_COEFF_COST coeff_costs[TX_SIZES][PLANE_TYPES];
Dake He0db7d0e2017-12-21 15:23:20 -0800333 LV_MAP_EOB_COST eob_costs[7][2];
Jingning Hanf5a4d3b2017-08-27 23:01:19 -0700334 uint16_t cb_offset;
Jingning Handfd72322017-08-09 14:04:12 -0700335
Yue Chenb23d00a2017-07-28 17:01:21 -0700336 // mode costs
Yue Chen170678a2017-10-17 13:43:10 -0700337 int intra_inter_cost[INTRA_INTER_CONTEXTS][2];
338
Yue Chenb23d00a2017-07-28 17:01:21 -0700339 int mbmode_cost[BLOCK_SIZE_GROUPS][INTRA_MODES];
340 int newmv_mode_cost[NEWMV_MODE_CONTEXTS][2];
Sarah Parker2b9ec2e2017-10-30 17:34:08 -0700341 int zeromv_mode_cost[GLOBALMV_MODE_CONTEXTS][2];
Yue Chenb23d00a2017-07-28 17:01:21 -0700342 int refmv_mode_cost[REFMV_MODE_CONTEXTS][2];
343 int drl_mode_cost0[DRL_MODE_CONTEXTS][2];
344
Hui Su9d0c03d2017-12-27 16:05:23 -0800345 int comp_inter_cost[COMP_INTER_CONTEXTS][2];
Hui Su3d30b4b2017-12-27 16:47:59 -0800346 int single_ref_cost[REF_CONTEXTS][SINGLE_REFS - 1][2];
Hui Su6b3d1e32018-01-05 11:25:40 -0800347 int comp_ref_type_cost[COMP_REF_TYPE_CONTEXTS]
348 [CDF_SIZE(COMP_REFERENCE_TYPES)];
Hui Sua7e3bfe2018-01-05 12:14:48 -0800349 int uni_comp_ref_cost[UNI_COMP_REF_CONTEXTS][UNIDIR_COMP_REFS - 1]
350 [CDF_SIZE(2)];
Hui Su0bdf5f52018-01-05 14:54:32 -0800351 // Cost for signaling ref_frame[0] (LAST_FRAME, LAST2_FRAME, LAST3_FRAME or
352 // GOLDEN_FRAME) in bidir-comp mode.
Zoe Liu3b353472018-02-12 13:58:22 -0800353 int comp_ref_cost[REF_CONTEXTS][FWD_REFS - 1][2];
Hui Su0bdf5f52018-01-05 14:54:32 -0800354 // Cost for signaling ref_frame[1] (ALTREF_FRAME, ALTREF2_FRAME, or
355 // BWDREF_FRAME) in bidir-comp mode.
Zoe Liu3b353472018-02-12 13:58:22 -0800356 int comp_bwdref_cost[REF_CONTEXTS][BWD_REFS - 1][2];
Yue Chenb23d00a2017-07-28 17:01:21 -0700357 int inter_compound_mode_cost[INTER_MODE_CONTEXTS][INTER_COMPOUND_MODES];
Cheng Chen2ef24ea2017-11-29 12:22:24 -0800358 int compound_type_cost[BLOCK_SIZES_ALL][COMPOUND_TYPES - 1];
Yue Chen73335fa2017-12-20 23:33:41 -0800359 int wedge_idx_cost[BLOCK_SIZES_ALL][16];
Yue Cheneaf128a2017-10-16 17:01:36 -0700360 int interintra_cost[BLOCK_SIZE_GROUPS][2];
361 int wedge_interintra_cost[BLOCK_SIZES_ALL][2];
Yue Chenb23d00a2017-07-28 17:01:21 -0700362 int interintra_mode_cost[BLOCK_SIZE_GROUPS][INTERINTRA_MODES];
Yue Chenb23d00a2017-07-28 17:01:21 -0700363 int motion_mode_cost[BLOCK_SIZES_ALL][MOTION_MODES];
Yue Chenb23d00a2017-07-28 17:01:21 -0700364 int motion_mode_cost1[BLOCK_SIZES_ALL][2];
David Michael Barrcb3a8ef2018-01-06 15:48:49 +0900365 int intra_uv_mode_cost[CFL_ALLOWED_TYPES][INTRA_MODES][UV_INTRA_MODES];
Yue Chenb23d00a2017-07-28 17:01:21 -0700366 int y_mode_costs[INTRA_MODES][INTRA_MODES][INTRA_MODES];
Yue Chen45dfb792018-03-01 13:02:40 -0800367 int filter_intra_cost[BLOCK_SIZES_ALL][2];
Yue Chen994dba22017-12-19 15:27:26 -0800368 int filter_intra_mode_cost[FILTER_INTRA_MODES];
Yue Chenb23d00a2017-07-28 17:01:21 -0700369 int switchable_interp_costs[SWITCHABLE_FILTER_CONTEXTS][SWITCHABLE_FILTERS];
Sebastien Alaiwana6a486c2017-11-07 17:04:27 +0100370 int partition_cost[PARTITION_CONTEXTS][EXT_PARTITION_TYPES];
Hui Suc1f411b2017-12-19 15:58:28 -0800371 int palette_y_size_cost[PALATTE_BSIZE_CTXS][PALETTE_SIZES];
372 int palette_uv_size_cost[PALATTE_BSIZE_CTXS][PALETTE_SIZES];
Yue Chenb23d00a2017-07-28 17:01:21 -0700373 int palette_y_color_cost[PALETTE_SIZES][PALETTE_COLOR_INDEX_CONTEXTS]
374 [PALETTE_COLORS];
375 int palette_uv_color_cost[PALETTE_SIZES][PALETTE_COLOR_INDEX_CONTEXTS]
376 [PALETTE_COLORS];
Hui Suc1f411b2017-12-19 15:58:28 -0800377 int palette_y_mode_cost[PALATTE_BSIZE_CTXS][PALETTE_Y_MODE_CONTEXTS][2];
Yue Chendab2ca92017-10-16 17:48:48 -0700378 int palette_uv_mode_cost[PALETTE_UV_MODE_CONTEXTS][2];
David Michael Barr38e560c2017-08-16 21:46:37 +0900379 // The rate associated with each alpha codeword
380 int cfl_cost[CFL_JOINT_SIGNS][CFL_PRED_PLANES][CFL_ALPHABET_SIZE];
Yue Chenb23d00a2017-07-28 17:01:21 -0700381 int tx_size_cost[TX_SIZES - 1][TX_SIZE_CONTEXTS][TX_SIZES];
Yue Chen171c17d2017-10-16 18:08:22 -0700382 int txfm_partition_cost[TXFM_PARTITION_CONTEXTS][2];
Yue Chenb23d00a2017-07-28 17:01:21 -0700383 int inter_tx_type_costs[EXT_TX_SETS_INTER][EXT_TX_SIZES][TX_TYPES];
384 int intra_tx_type_costs[EXT_TX_SETS_INTRA][EXT_TX_SIZES][INTRA_MODES]
385 [TX_TYPES];
Joe Young3ca43bf2017-10-06 15:12:46 -0700386 int angle_delta_cost[DIRECTIONAL_MODES][2 * MAX_ANGLE_DELTA + 1];
Yue Chenb23d00a2017-07-28 17:01:21 -0700387 int switchable_restore_cost[RESTORE_SWITCHABLE_TYPES];
Debargha Mukherjeebc732ef2017-10-12 12:40:25 -0700388 int wiener_restore_cost[2];
389 int sgrproj_restore_cost[2];
Hui Su6c8584f2017-09-14 15:37:02 -0700390 int intrabc_cost[2];
Yue Chenb23d00a2017-07-28 17:01:21 -0700391
Yaowu Xuc27fc142016-08-22 16:08:15 -0700392 // Used to store sub partition's choices.
Zoe Liu27deb382018-03-27 15:13:56 -0700393 MV pred_mv[REF_FRAMES];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700394
395 // Store the best motion vector during motion search
396 int_mv best_mv;
397 // Store the second best motion vector during full-pixel motion search
398 int_mv second_best_mv;
399
Venkate846e2b2018-09-21 07:31:14 +0530400 // Store the fractional best motion vector during sub/Qpel-pixel motion search
401 int_mv fractional_best_mv[3];
402
Yaowu Xuc27fc142016-08-22 16:08:15 -0700403 // use default transform and skip transform type search for intra modes
404 int use_default_intra_tx_type;
405 // use default transform and skip transform type search for inter modes
406 int use_default_inter_tx_type;
Yushin Chob7b60c52017-07-14 16:18:52 -0700407#if CONFIG_DIST_8X8
Yushin Cho55104332017-08-14 16:15:43 -0700408 int using_dist_8x8;
409 aom_tune_metric tune_metric;
Yushin Chob7b60c52017-07-14 16:18:52 -0700410#endif // CONFIG_DIST_8X8
Cheng Chen46970612017-10-24 14:53:36 -0700411 int comp_idx_cost[COMP_INDEX_CONTEXTS][2];
Cheng Chen2ef24ea2017-11-29 12:22:24 -0800412 int comp_group_idx_cost[COMP_GROUP_IDX_CONTEXTS][2];
Hui Su6bbd9322018-01-26 11:04:13 -0800413 // Bit flags for pruning tx type search, tx split, etc.
414 int tx_search_prune[EXT_TX_SET_TYPES];
Hui Su845a7412018-03-08 12:21:00 -0800415 int must_find_valid_partition;
Hui Su7b67ff32018-03-07 11:52:04 -0800416 int tx_split_prune_flag; // Flag to skip tx split RD search.
Ranjit Kumar Tulabandu8105bf42018-07-27 14:16:55 +0530417 int recalc_luma_mc_data; // Flag to indicate recalculation of MC data during
418 // interpolation filter search
elliottk1dbb8c12018-11-06 22:28:59 -0800419 // The likelihood of an edge existing in the block (using partial Canny edge
420 // detection). For reference, 556 is the value returned for a solid
421 // vertical black/white edge.
422 uint16_t edge_strength;
Ranjit Kumar Tulabandua1ebb572018-12-24 12:13:54 +0530423
424 // [Saved stat index]
425 COMP_RD_STATS comp_rd_stats[MAX_COMP_RD_STATS];
426 int comp_rd_stats_idx;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700427};
428
Frederic Barbier0f191da2018-01-03 17:29:26 +0100429static INLINE int is_rect_tx_allowed_bsize(BLOCK_SIZE bsize) {
430 static const char LUT[BLOCK_SIZES_ALL] = {
431 0, // BLOCK_4X4
432 1, // BLOCK_4X8
433 1, // BLOCK_8X4
434 0, // BLOCK_8X8
435 1, // BLOCK_8X16
436 1, // BLOCK_16X8
437 0, // BLOCK_16X16
438 1, // BLOCK_16X32
439 1, // BLOCK_32X16
440 0, // BLOCK_32X32
441 1, // BLOCK_32X64
442 1, // BLOCK_64X32
443 0, // BLOCK_64X64
Frederic Barbier0f191da2018-01-03 17:29:26 +0100444 0, // BLOCK_64X128
445 0, // BLOCK_128X64
446 0, // BLOCK_128X128
Frederic Barbier0f191da2018-01-03 17:29:26 +0100447 1, // BLOCK_4X16
448 1, // BLOCK_16X4
449 1, // BLOCK_8X32
450 1, // BLOCK_32X8
451 1, // BLOCK_16X64
452 1, // BLOCK_64X16
Frederic Barbier0f191da2018-01-03 17:29:26 +0100453 };
454
455 return LUT[bsize];
456}
457
458static INLINE int is_rect_tx_allowed(const MACROBLOCKD *xd,
459 const MB_MODE_INFO *mbmi) {
460 return is_rect_tx_allowed_bsize(mbmi->sb_type) &&
461 !xd->lossless[mbmi->segment_id];
462}
463
Frederic Barbier4b56b102018-03-30 16:09:34 +0200464static INLINE int tx_size_to_depth(TX_SIZE tx_size, BLOCK_SIZE bsize) {
Urvang Joshidd0376f2018-05-02 16:37:25 -0700465 TX_SIZE ctx_size = max_txsize_rect_lookup[bsize];
Frederic Barbier0f191da2018-01-03 17:29:26 +0100466 int depth = 0;
467 while (tx_size != ctx_size) {
468 depth++;
Frederic Barbier4b56b102018-03-30 16:09:34 +0200469 ctx_size = sub_tx_size_map[ctx_size];
Frederic Barbier0f191da2018-01-03 17:29:26 +0100470 assert(depth <= MAX_TX_DEPTH);
471 }
472 return depth;
473}
474
Grant Hsu39248c32018-09-18 10:38:44 +0800475static INLINE void set_blk_skip(MACROBLOCK *x, int plane, int blk_idx,
476 int skip) {
477 if (skip)
478 x->blk_skip[blk_idx] |= 1UL << plane;
479 else
480 x->blk_skip[blk_idx] &= ~(1UL << plane);
481#ifndef NDEBUG
482 // Set chroma planes to uninitialized states when luma is set to check if
483 // it will be set later
484 if (plane == 0) {
485 x->blk_skip[blk_idx] |= 1UL << (1 + 4);
486 x->blk_skip[blk_idx] |= 1UL << (2 + 4);
487 }
488
489 // Clear the initialization checking bit
490 x->blk_skip[blk_idx] &= ~(1UL << (plane + 4));
491#endif
492}
493
494static INLINE int is_blk_skip(MACROBLOCK *x, int plane, int blk_idx) {
495#ifndef NDEBUG
496 // Check if this is initialized
497 assert(!(x->blk_skip[blk_idx] & (1UL << (plane + 4))));
498
499 // The magic number is 0x77, this is to test if there is garbage data
500 assert((x->blk_skip[blk_idx] & 0x88) == 0);
501#endif
502 return (x->blk_skip[blk_idx] >> plane) & 1;
503}
504
Yaowu Xuc27fc142016-08-22 16:08:15 -0700505#ifdef __cplusplus
506} // extern "C"
507#endif
508
James Zerne1cbb132018-08-22 14:10:36 -0700509#endif // AOM_AV1_ENCODER_BLOCK_H_