blob: 28cc1b4659f61d65bc54f6cb473b7ea5671e13ac [file] [log] [blame]
Yaowu Xuc27fc142016-08-22 16:08:15 -07001/*
Yaowu Xu2ab7ff02016-09-02 12:04:54 -07002 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Yaowu Xuc27fc142016-08-22 16:08:15 -07003 *
Yaowu Xu2ab7ff02016-09-02 12:04:54 -07004 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Yaowu Xuc27fc142016-08-22 16:08:15 -070010 */
11
Yaowu Xuf883b422016-08-30 14:01:10 -070012#ifndef AV1_ENCODER_BLOCK_H_
13#define AV1_ENCODER_BLOCK_H_
Yaowu Xuc27fc142016-08-22 16:08:15 -070014
15#include "av1/common/entropymv.h"
16#include "av1/common/entropy.h"
Yaowu Xuc27fc142016-08-22 16:08:15 -070017#include "av1/common/mvref_common.h"
Hui Su1ddf2312017-08-19 15:21:34 -070018#include "av1/encoder/hash.h"
Yushin Cho55104332017-08-14 16:15:43 -070019#if CONFIG_DIST_8X8
20#include "aom/aomcx.h"
21#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -070022
23#ifdef __cplusplus
24extern "C" {
25#endif
26
27typedef struct {
28 unsigned int sse;
29 int sum;
30 unsigned int var;
Urvang Joshi454280d2016-10-14 16:51:44 -070031} DIFF;
Yaowu Xuc27fc142016-08-22 16:08:15 -070032
33typedef struct macroblock_plane {
34 DECLARE_ALIGNED(16, int16_t, src_diff[MAX_SB_SQUARE]);
35 tran_low_t *qcoeff;
36 tran_low_t *coeff;
37 uint16_t *eobs;
Angie Chiang74e23072017-03-24 14:54:23 -070038 uint8_t *txb_entropy_ctx;
Yaowu Xuc27fc142016-08-22 16:08:15 -070039 struct buf_2d src;
40
41 // Quantizer setings
Monty Montgomery125c0fc2017-10-26 00:44:35 -040042 // These are used/accessed only in the quantization process
43 // RDO does not / must not depend on any of these values
44 // All values below share the coefficient scale/shift used in TX
45 const int16_t *quant_fp_QTX;
46 const int16_t *round_fp_QTX;
47 const int16_t *quant_QTX;
48 const int16_t *quant_shift_QTX;
49 const int16_t *zbin_QTX;
50 const int16_t *round_QTX;
51 const int16_t *dequant_QTX;
Yaowu Xuc27fc142016-08-22 16:08:15 -070052} MACROBLOCK_PLANE;
53
Jingning Handfd72322017-08-09 14:04:12 -070054typedef struct {
55 int txb_skip_cost[TXB_SKIP_CONTEXTS][2];
Dake He3fe369c2017-11-16 17:56:44 -080056 int base_eob_cost[SIG_COEF_CONTEXTS_EOB][3];
Ola Hugosson13892102017-11-06 08:01:44 +010057 int base_cost[SIG_COEF_CONTEXTS][4];
Angie Chiang7ab884e2017-10-18 15:57:12 -070058 int eob_extra_cost[EOB_COEF_CONTEXTS][2];
Jingning Handfd72322017-08-09 14:04:12 -070059 int dc_sign_cost[DC_SIGN_CONTEXTS][2];
Angie Chiang26d3e452017-09-29 17:40:02 -070060 int lps_cost[LEVEL_CONTEXTS][COEFF_BASE_RANGE + 1];
Jingning Handfd72322017-08-09 14:04:12 -070061} LV_MAP_COEFF_COST;
Jingning Hanf5a4d3b2017-08-27 23:01:19 -070062
Johannb0ef6ff2018-02-08 14:32:21 -080063typedef struct {
64 int eob_cost[2][11];
65} LV_MAP_EOB_COST;
Dake He0db7d0e2017-12-21 15:23:20 -080066
Jingning Hanf5a4d3b2017-08-27 23:01:19 -070067typedef struct {
68 tran_low_t tcoeff[MAX_MB_PLANE][MAX_SB_SQUARE];
69 uint16_t eobs[MAX_MB_PLANE][MAX_SB_SQUARE / (TX_SIZE_W_MIN * TX_SIZE_H_MIN)];
70 uint8_t txb_skip_ctx[MAX_MB_PLANE]
71 [MAX_SB_SQUARE / (TX_SIZE_W_MIN * TX_SIZE_H_MIN)];
72 int dc_sign_ctx[MAX_MB_PLANE]
73 [MAX_SB_SQUARE / (TX_SIZE_W_MIN * TX_SIZE_H_MIN)];
74} CB_COEFF_BUFFER;
Jingning Handfd72322017-08-09 14:04:12 -070075
Yaowu Xuc27fc142016-08-22 16:08:15 -070076typedef struct {
77 int_mv ref_mvs[MODE_CTX_REF_FRAMES][MAX_MV_REF_CANDIDATES];
78 int16_t mode_context[MODE_CTX_REF_FRAMES];
Angie Chiangc484abe2017-03-20 15:43:11 -070079 // TODO(angiebird): Reduce the buffer size according to sb_type
Jingning Hanf5a4d3b2017-08-27 23:01:19 -070080 tran_low_t *tcoeff[MAX_MB_PLANE];
81 uint16_t *eobs[MAX_MB_PLANE];
82 uint8_t *txb_skip_ctx[MAX_MB_PLANE];
83 int *dc_sign_ctx[MAX_MB_PLANE];
Yaowu Xuc27fc142016-08-22 16:08:15 -070084 uint8_t ref_mv_count[MODE_CTX_REF_FRAMES];
85 CANDIDATE_MV ref_mv_stack[MODE_CTX_REF_FRAMES][MAX_REF_MV_STACK_SIZE];
Angie Chiang4c9b6022018-04-10 16:16:45 -070086 int_mv global_mvs[REF_FRAMES];
Yaowu Xuc27fc142016-08-22 16:08:15 -070087 int16_t compound_mode_context[MODE_CTX_REF_FRAMES];
Yaowu Xuc27fc142016-08-22 16:08:15 -070088} MB_MODE_INFO_EXT;
89
Alex Converse0fa0f422017-04-24 12:51:14 -070090typedef struct {
91 int col_min;
92 int col_max;
93 int row_min;
94 int row_max;
95} MvLimits;
96
Yaowu Xuc27fc142016-08-22 16:08:15 -070097typedef struct {
Hui Su473cf892017-11-08 18:14:31 -080098 uint8_t best_palette_color_map[MAX_PALETTE_SQUARE];
Hui Su5891f982017-12-18 16:18:23 -080099 int kmeans_data_buf[2 * MAX_PALETTE_SQUARE];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700100} PALETTE_BUFFER;
101
Hui Su1ddf2312017-08-19 15:21:34 -0700102typedef struct {
Hui Su1ddf2312017-08-19 15:21:34 -0700103 TX_SIZE tx_size;
Hui Su7167d952018-02-01 16:33:12 -0800104 TX_SIZE inter_tx_size[INTER_TX_SIZE_BUF_LEN];
Hui Suf4b79c72018-03-22 13:14:36 -0700105 uint8_t blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE];
Hui Sude1a5db2018-02-22 15:44:49 -0800106 TX_TYPE txk_type[TXK_TYPE_BUF_LEN];
Hui Su1ddf2312017-08-19 15:21:34 -0700107 RD_STATS rd_stats;
108 uint32_t hash_value;
Hui Su6cb17c12018-03-09 12:56:20 -0800109} MB_RD_INFO;
Hui Su1ddf2312017-08-19 15:21:34 -0700110
111#define RD_RECORD_BUFFER_LEN 8
112typedef struct {
Hui Su6cb17c12018-03-09 12:56:20 -0800113 MB_RD_INFO tx_rd_info[RD_RECORD_BUFFER_LEN]; // Circular buffer.
Hui Su1ddf2312017-08-19 15:21:34 -0700114 int index_start;
115 int num;
Peng Bin8a204cd2018-04-08 13:07:35 +0800116 CRC32C crc_calculator; // Hash function.
Hui Su6cb17c12018-03-09 12:56:20 -0800117} MB_RD_RECORD;
Hui Su1ddf2312017-08-19 15:21:34 -0700118
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700119typedef struct {
120 int64_t dist;
Jingning Han73bc2aa2018-02-02 14:31:39 -0800121 int64_t sse;
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700122 int rate;
Hui Su8c2b9132017-12-09 10:40:15 -0800123 uint16_t eob;
Jingning Han73bc2aa2018-02-02 14:31:39 -0800124 TX_TYPE tx_type;
Jingning Han45027c62017-12-11 11:47:15 -0800125 uint16_t entropy_context;
Jingning Hand7e99112017-12-13 09:47:45 -0800126 uint8_t txb_entropy_ctx;
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700127 uint8_t valid;
Hui Su950b9122018-02-03 10:21:40 -0800128 uint8_t fast; // This is not being used now.
Hui Su6cb17c12018-03-09 12:56:20 -0800129} TXB_RD_INFO;
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700130
131#define TX_SIZE_RD_RECORD_BUFFER_LEN 256
132typedef struct {
133 uint32_t hash_vals[TX_SIZE_RD_RECORD_BUFFER_LEN];
Hui Su6cb17c12018-03-09 12:56:20 -0800134 TXB_RD_INFO tx_rd_info[TX_SIZE_RD_RECORD_BUFFER_LEN];
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700135 int index_start;
136 int num;
Hui Su6cb17c12018-03-09 12:56:20 -0800137} TXB_RD_RECORD;
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700138
139typedef struct tx_size_rd_info_node {
Hui Su6cb17c12018-03-09 12:56:20 -0800140 TXB_RD_INFO *rd_info_array; // Points to array of size TX_TYPES.
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700141 struct tx_size_rd_info_node *children[4];
Hui Su6cb17c12018-03-09 12:56:20 -0800142} TXB_RD_INFO_NODE;
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700143
Hui Suaaf5a612018-04-12 15:33:42 -0700144// Region size for mode decision sampling in the first pass of partition
145// search(two_pass_partition_search speed feature), in units of mi size(4).
146// Used by the mode_pruning_based_on_two_pass_partition_search speed feature.
147#define FIRST_PARTITION_PASS_SAMPLE_REGION 8
148#define FIRST_PARTITION_PASS_SAMPLE_REGION_LOG2 3
149#define FIRST_PARTITION_PASS_STATS_TABLES \
150 (MAX_MIB_SIZE >> FIRST_PARTITION_PASS_SAMPLE_REGION_LOG2) * \
151 (MAX_MIB_SIZE >> FIRST_PARTITION_PASS_SAMPLE_REGION_LOG2)
152#define FIRST_PARTITION_PASS_STATS_STRIDE \
153 (MAX_MIB_SIZE_LOG2 - FIRST_PARTITION_PASS_SAMPLE_REGION_LOG2)
154
155static INLINE int av1_first_partition_pass_stats_index(int mi_row, int mi_col) {
156 const int row =
157 (mi_row & MAX_MIB_MASK) >> FIRST_PARTITION_PASS_SAMPLE_REGION_LOG2;
158 const int col =
159 (mi_col & MAX_MIB_MASK) >> FIRST_PARTITION_PASS_SAMPLE_REGION_LOG2;
160 return (row << FIRST_PARTITION_PASS_STATS_STRIDE) + col;
161}
162
163typedef struct {
164 uint8_t ref0_counts[REF_FRAMES]; // Counters for ref_frame[0].
165 uint8_t ref1_counts[REF_FRAMES]; // Counters for ref_frame[1].
166 int sample_counts; // Number of samples collected.
167} FIRST_PARTITION_PASS_STATS;
168
Peng Bin706a3b22018-05-30 19:42:05 +0800169#define MAX_INTERP_FILTER_STATS 64
170typedef struct {
171 InterpFilters filters;
Peng Bind482d172018-06-14 15:55:51 +0800172 int_mv mv[2];
Peng Bin706a3b22018-05-30 19:42:05 +0800173 int8_t ref_frames[2];
174} INTERPOLATION_FILTER_STATS;
175
Yaowu Xuc27fc142016-08-22 16:08:15 -0700176typedef struct macroblock MACROBLOCK;
177struct macroblock {
178 struct macroblock_plane plane[MAX_MB_PLANE];
179
Jingning Han66965a22018-01-25 09:53:41 -0800180 // Determine if one would go with reduced complexity transform block
181 // search model to select prediction modes, or full complexity model
182 // to select transform kernel.
Jingning Handd8600f2018-01-23 09:06:32 -0800183 int rd_model;
184
Jingning Han4489c262018-01-26 16:43:59 -0800185 // Indicate if the encoder is running in the first pass partition search.
186 // In that case, apply certain speed features therein to reduce the overhead
187 // cost in the first pass search.
Jingning Hanc58ccc32018-01-26 07:46:38 -0800188 int cb_partition_scan;
189
Hui Suaaf5a612018-04-12 15:33:42 -0700190 FIRST_PARTITION_PASS_STATS
191 first_partition_pass_stats[FIRST_PARTITION_PASS_STATS_TABLES];
Hui Subb628a72018-04-05 19:48:23 -0700192
Peng Bin706a3b22018-05-30 19:42:05 +0800193 // [comp_idx][saved stat_idx]
194 INTERPOLATION_FILTER_STATS interp_filter_stats[2][MAX_INTERP_FILTER_STATS];
195 int interp_filter_stats_idx[2];
196
Jingning Han4489c262018-01-26 16:43:59 -0800197 // Activate constrained coding block partition search range.
198 int use_cb_search_range;
199
Hui Su6cb17c12018-03-09 12:56:20 -0800200 // Inter macroblock RD search info.
201 MB_RD_RECORD mb_rd_record;
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700202
Hui Su6cb17c12018-03-09 12:56:20 -0800203 // Inter transform block RD search info. for square TX sizes.
204 TXB_RD_RECORD txb_rd_record_8X8[(MAX_MIB_SIZE >> 1) * (MAX_MIB_SIZE >> 1)];
205 TXB_RD_RECORD txb_rd_record_16X16[(MAX_MIB_SIZE >> 2) * (MAX_MIB_SIZE >> 2)];
206 TXB_RD_RECORD txb_rd_record_32X32[(MAX_MIB_SIZE >> 3) * (MAX_MIB_SIZE >> 3)];
207 TXB_RD_RECORD txb_rd_record_64X64[(MAX_MIB_SIZE >> 4) * (MAX_MIB_SIZE >> 4)];
208
209 // Intra transform block RD search info. for square TX sizes.
210 TXB_RD_RECORD txb_rd_record_intra;
Hui Suec5a1ab2018-03-08 15:09:12 -0800211
Yaowu Xuc27fc142016-08-22 16:08:15 -0700212 MACROBLOCKD e_mbd;
213 MB_MODE_INFO_EXT *mbmi_ext;
214 int skip_block;
David Barkerd7d78c82016-10-24 10:55:35 +0100215 int qindex;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700216
217 // The equivalent error at the current rdmult of one whole bit (not one
218 // bitcost unit).
219 int errorperbit;
220 // The equivalend SAD error of one (whole) bit at the current quantizer
221 // for large blocks.
222 int sadperbit16;
223 // The equivalend SAD error of one (whole) bit at the current quantizer
224 // for sub-8x8 blocks.
225 int sadperbit4;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700226 int rdmult;
227 int mb_energy;
228 int *m_search_count_ptr;
229 int *ex_search_count_ptr;
230
Jingning Han9777afc2016-10-20 15:17:43 -0700231 unsigned int txb_split_count;
Jingning Han9777afc2016-10-20 15:17:43 -0700232
Yaowu Xuc27fc142016-08-22 16:08:15 -0700233 // These are set to their default values at the beginning, and then adjusted
234 // further in the encoding process.
235 BLOCK_SIZE min_partition_size;
236 BLOCK_SIZE max_partition_size;
237
Zoe Liu27deb382018-03-27 15:13:56 -0700238 int mv_best_ref_index[REF_FRAMES];
239 unsigned int max_mv_context[REF_FRAMES];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700240 unsigned int source_variance;
Zoe Liu27deb382018-03-27 15:13:56 -0700241 unsigned int pred_sse[REF_FRAMES];
242 int pred_mv_sad[REF_FRAMES];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700243
Yaowu Xuc27fc142016-08-22 16:08:15 -0700244 int *nmvjointcost;
Jingning Hanf050fc12018-03-09 14:53:33 -0800245 int nmv_vec_cost[MV_JOINTS];
246 int *nmvcost[2];
247 int *nmvcost_hp[2];
248 int **mv_cost_stack;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700249 int **mvcost;
Alex Conversea127a792017-05-23 15:27:21 -0700250
Yue Chene9638cc2016-10-10 12:37:54 -0700251 int32_t *wsrc_buf;
252 int32_t *mask_buf;
Jingning Hand064cf02017-06-01 10:00:39 -0700253 uint8_t *above_pred_buf;
254 uint8_t *left_pred_buf;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700255
256 PALETTE_BUFFER *palette_buffer;
257
258 // These define limits to motion vector components to prevent them
259 // from extending outside the UMV borders
Alex Converse0fa0f422017-04-24 12:51:14 -0700260 MvLimits mv_limits;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700261
Hui Suf4b79c72018-03-22 13:14:36 -0700262 uint8_t blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE];
263 uint8_t blk_skip_drl[MAX_MIB_SIZE * MAX_MIB_SIZE];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700264
265 int skip;
Jingning Han8efdbc82017-02-19 14:40:03 -0800266 int skip_chroma_rd;
Zoe Liu1eed2df2017-10-16 17:13:15 -0700267 int skip_cost[SKIP_CONTEXTS][2];
268
Zoe Liuf40a9572017-10-13 12:37:19 -0700269 int skip_mode; // 0: off; 1: on
270 int skip_mode_cost[SKIP_CONTEXTS][2];
271
Zoe Liu104d62e2017-12-07 12:44:45 -0800272 int compound_idx;
Zoe Liuf40a9572017-10-13 12:37:19 -0700273
Jingning Handfd72322017-08-09 14:04:12 -0700274 LV_MAP_COEFF_COST coeff_costs[TX_SIZES][PLANE_TYPES];
Dake He0db7d0e2017-12-21 15:23:20 -0800275 LV_MAP_EOB_COST eob_costs[7][2];
Jingning Hanf5a4d3b2017-08-27 23:01:19 -0700276 uint16_t cb_offset;
Jingning Handfd72322017-08-09 14:04:12 -0700277
Yue Chenb23d00a2017-07-28 17:01:21 -0700278 // mode costs
Yue Chen170678a2017-10-17 13:43:10 -0700279 int intra_inter_cost[INTRA_INTER_CONTEXTS][2];
280
Yue Chenb23d00a2017-07-28 17:01:21 -0700281 int mbmode_cost[BLOCK_SIZE_GROUPS][INTRA_MODES];
282 int newmv_mode_cost[NEWMV_MODE_CONTEXTS][2];
Sarah Parker2b9ec2e2017-10-30 17:34:08 -0700283 int zeromv_mode_cost[GLOBALMV_MODE_CONTEXTS][2];
Yue Chenb23d00a2017-07-28 17:01:21 -0700284 int refmv_mode_cost[REFMV_MODE_CONTEXTS][2];
285 int drl_mode_cost0[DRL_MODE_CONTEXTS][2];
286
Hui Su9d0c03d2017-12-27 16:05:23 -0800287 int comp_inter_cost[COMP_INTER_CONTEXTS][2];
Hui Su3d30b4b2017-12-27 16:47:59 -0800288 int single_ref_cost[REF_CONTEXTS][SINGLE_REFS - 1][2];
Hui Su6b3d1e32018-01-05 11:25:40 -0800289 int comp_ref_type_cost[COMP_REF_TYPE_CONTEXTS]
290 [CDF_SIZE(COMP_REFERENCE_TYPES)];
Hui Sua7e3bfe2018-01-05 12:14:48 -0800291 int uni_comp_ref_cost[UNI_COMP_REF_CONTEXTS][UNIDIR_COMP_REFS - 1]
292 [CDF_SIZE(2)];
Hui Su0bdf5f52018-01-05 14:54:32 -0800293 // Cost for signaling ref_frame[0] (LAST_FRAME, LAST2_FRAME, LAST3_FRAME or
294 // GOLDEN_FRAME) in bidir-comp mode.
Zoe Liu3b353472018-02-12 13:58:22 -0800295 int comp_ref_cost[REF_CONTEXTS][FWD_REFS - 1][2];
Hui Su0bdf5f52018-01-05 14:54:32 -0800296 // Cost for signaling ref_frame[1] (ALTREF_FRAME, ALTREF2_FRAME, or
297 // BWDREF_FRAME) in bidir-comp mode.
Zoe Liu3b353472018-02-12 13:58:22 -0800298 int comp_bwdref_cost[REF_CONTEXTS][BWD_REFS - 1][2];
Yue Chenb23d00a2017-07-28 17:01:21 -0700299 int inter_compound_mode_cost[INTER_MODE_CONTEXTS][INTER_COMPOUND_MODES];
Cheng Chen2ef24ea2017-11-29 12:22:24 -0800300 int compound_type_cost[BLOCK_SIZES_ALL][COMPOUND_TYPES - 1];
Yue Chen73335fa2017-12-20 23:33:41 -0800301 int wedge_idx_cost[BLOCK_SIZES_ALL][16];
Yue Cheneaf128a2017-10-16 17:01:36 -0700302 int interintra_cost[BLOCK_SIZE_GROUPS][2];
303 int wedge_interintra_cost[BLOCK_SIZES_ALL][2];
Yue Chenb23d00a2017-07-28 17:01:21 -0700304 int interintra_mode_cost[BLOCK_SIZE_GROUPS][INTERINTRA_MODES];
Yue Chenb23d00a2017-07-28 17:01:21 -0700305 int motion_mode_cost[BLOCK_SIZES_ALL][MOTION_MODES];
Yue Chenb23d00a2017-07-28 17:01:21 -0700306 int motion_mode_cost1[BLOCK_SIZES_ALL][2];
David Michael Barrcb3a8ef2018-01-06 15:48:49 +0900307 int intra_uv_mode_cost[CFL_ALLOWED_TYPES][INTRA_MODES][UV_INTRA_MODES];
Yue Chenb23d00a2017-07-28 17:01:21 -0700308 int y_mode_costs[INTRA_MODES][INTRA_MODES][INTRA_MODES];
Yue Chen45dfb792018-03-01 13:02:40 -0800309 int filter_intra_cost[BLOCK_SIZES_ALL][2];
Yue Chen994dba22017-12-19 15:27:26 -0800310 int filter_intra_mode_cost[FILTER_INTRA_MODES];
Yue Chenb23d00a2017-07-28 17:01:21 -0700311 int switchable_interp_costs[SWITCHABLE_FILTER_CONTEXTS][SWITCHABLE_FILTERS];
Sebastien Alaiwana6a486c2017-11-07 17:04:27 +0100312 int partition_cost[PARTITION_CONTEXTS][EXT_PARTITION_TYPES];
Hui Suc1f411b2017-12-19 15:58:28 -0800313 int palette_y_size_cost[PALATTE_BSIZE_CTXS][PALETTE_SIZES];
314 int palette_uv_size_cost[PALATTE_BSIZE_CTXS][PALETTE_SIZES];
Yue Chenb23d00a2017-07-28 17:01:21 -0700315 int palette_y_color_cost[PALETTE_SIZES][PALETTE_COLOR_INDEX_CONTEXTS]
316 [PALETTE_COLORS];
317 int palette_uv_color_cost[PALETTE_SIZES][PALETTE_COLOR_INDEX_CONTEXTS]
318 [PALETTE_COLORS];
Hui Suc1f411b2017-12-19 15:58:28 -0800319 int palette_y_mode_cost[PALATTE_BSIZE_CTXS][PALETTE_Y_MODE_CONTEXTS][2];
Yue Chendab2ca92017-10-16 17:48:48 -0700320 int palette_uv_mode_cost[PALETTE_UV_MODE_CONTEXTS][2];
David Michael Barr38e560c2017-08-16 21:46:37 +0900321 // The rate associated with each alpha codeword
322 int cfl_cost[CFL_JOINT_SIGNS][CFL_PRED_PLANES][CFL_ALPHABET_SIZE];
Yue Chenb23d00a2017-07-28 17:01:21 -0700323 int tx_size_cost[TX_SIZES - 1][TX_SIZE_CONTEXTS][TX_SIZES];
Yue Chen171c17d2017-10-16 18:08:22 -0700324 int txfm_partition_cost[TXFM_PARTITION_CONTEXTS][2];
Yue Chenb23d00a2017-07-28 17:01:21 -0700325 int inter_tx_type_costs[EXT_TX_SETS_INTER][EXT_TX_SIZES][TX_TYPES];
326 int intra_tx_type_costs[EXT_TX_SETS_INTRA][EXT_TX_SIZES][INTRA_MODES]
327 [TX_TYPES];
Joe Young3ca43bf2017-10-06 15:12:46 -0700328 int angle_delta_cost[DIRECTIONAL_MODES][2 * MAX_ANGLE_DELTA + 1];
Yue Chenb23d00a2017-07-28 17:01:21 -0700329 int switchable_restore_cost[RESTORE_SWITCHABLE_TYPES];
Debargha Mukherjeebc732ef2017-10-12 12:40:25 -0700330 int wiener_restore_cost[2];
331 int sgrproj_restore_cost[2];
Hui Su6c8584f2017-09-14 15:37:02 -0700332 int intrabc_cost[2];
Yue Chenb23d00a2017-07-28 17:01:21 -0700333
Yaowu Xuc27fc142016-08-22 16:08:15 -0700334 // Used to store sub partition's choices.
Zoe Liu27deb382018-03-27 15:13:56 -0700335 MV pred_mv[REF_FRAMES];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700336
337 // Store the best motion vector during motion search
338 int_mv best_mv;
339 // Store the second best motion vector during full-pixel motion search
340 int_mv second_best_mv;
341
Yaowu Xuc27fc142016-08-22 16:08:15 -0700342 // use default transform and skip transform type search for intra modes
343 int use_default_intra_tx_type;
344 // use default transform and skip transform type search for inter modes
345 int use_default_inter_tx_type;
Yushin Chob7b60c52017-07-14 16:18:52 -0700346#if CONFIG_DIST_8X8
Yushin Cho55104332017-08-14 16:15:43 -0700347 int using_dist_8x8;
348 aom_tune_metric tune_metric;
Yushin Cho49fc74d2018-05-11 13:04:18 -0700349 DECLARE_ALIGNED(16, int16_t, pred_luma[MAX_SB_SQUARE]);
Yushin Chob7b60c52017-07-14 16:18:52 -0700350#endif // CONFIG_DIST_8X8
Cheng Chen46970612017-10-24 14:53:36 -0700351 int comp_idx_cost[COMP_INDEX_CONTEXTS][2];
Cheng Chen2ef24ea2017-11-29 12:22:24 -0800352 int comp_group_idx_cost[COMP_GROUP_IDX_CONTEXTS][2];
Hui Su6bbd9322018-01-26 11:04:13 -0800353 // Bit flags for pruning tx type search, tx split, etc.
354 int tx_search_prune[EXT_TX_SET_TYPES];
Hui Su845a7412018-03-08 12:21:00 -0800355 int must_find_valid_partition;
Hui Su7b67ff32018-03-07 11:52:04 -0800356 int tx_split_prune_flag; // Flag to skip tx split RD search.
Yaowu Xuc27fc142016-08-22 16:08:15 -0700357};
358
Frederic Barbier0f191da2018-01-03 17:29:26 +0100359static INLINE int is_rect_tx_allowed_bsize(BLOCK_SIZE bsize) {
360 static const char LUT[BLOCK_SIZES_ALL] = {
361 0, // BLOCK_4X4
362 1, // BLOCK_4X8
363 1, // BLOCK_8X4
364 0, // BLOCK_8X8
365 1, // BLOCK_8X16
366 1, // BLOCK_16X8
367 0, // BLOCK_16X16
368 1, // BLOCK_16X32
369 1, // BLOCK_32X16
370 0, // BLOCK_32X32
371 1, // BLOCK_32X64
372 1, // BLOCK_64X32
373 0, // BLOCK_64X64
Frederic Barbier0f191da2018-01-03 17:29:26 +0100374 0, // BLOCK_64X128
375 0, // BLOCK_128X64
376 0, // BLOCK_128X128
Frederic Barbier0f191da2018-01-03 17:29:26 +0100377 1, // BLOCK_4X16
378 1, // BLOCK_16X4
379 1, // BLOCK_8X32
380 1, // BLOCK_32X8
381 1, // BLOCK_16X64
382 1, // BLOCK_64X16
Frederic Barbier0f191da2018-01-03 17:29:26 +0100383 };
384
385 return LUT[bsize];
386}
387
388static INLINE int is_rect_tx_allowed(const MACROBLOCKD *xd,
389 const MB_MODE_INFO *mbmi) {
390 return is_rect_tx_allowed_bsize(mbmi->sb_type) &&
391 !xd->lossless[mbmi->segment_id];
392}
393
Frederic Barbier4b56b102018-03-30 16:09:34 +0200394static INLINE int tx_size_to_depth(TX_SIZE tx_size, BLOCK_SIZE bsize) {
Urvang Joshidd0376f2018-05-02 16:37:25 -0700395 TX_SIZE ctx_size = max_txsize_rect_lookup[bsize];
Frederic Barbier0f191da2018-01-03 17:29:26 +0100396 int depth = 0;
397 while (tx_size != ctx_size) {
398 depth++;
Frederic Barbier4b56b102018-03-30 16:09:34 +0200399 ctx_size = sub_tx_size_map[ctx_size];
Frederic Barbier0f191da2018-01-03 17:29:26 +0100400 assert(depth <= MAX_TX_DEPTH);
401 }
402 return depth;
403}
404
Yaowu Xuc27fc142016-08-22 16:08:15 -0700405#ifdef __cplusplus
406} // extern "C"
407#endif
408
Yaowu Xuf883b422016-08-30 14:01:10 -0700409#endif // AV1_ENCODER_BLOCK_H_