blob: d5c95deb847ffcd41c5403dceee7ab08e26f177a [file] [log] [blame]
Yaowu Xuc27fc142016-08-22 16:08:15 -07001/*
Yaowu Xu2ab7ff02016-09-02 12:04:54 -07002 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Yaowu Xuc27fc142016-08-22 16:08:15 -07003 *
Yaowu Xu2ab7ff02016-09-02 12:04:54 -07004 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Yaowu Xuc27fc142016-08-22 16:08:15 -070010 */
11
Yaowu Xuf883b422016-08-30 14:01:10 -070012#ifndef AV1_ENCODER_BLOCK_H_
13#define AV1_ENCODER_BLOCK_H_
Yaowu Xuc27fc142016-08-22 16:08:15 -070014
15#include "av1/common/entropymv.h"
16#include "av1/common/entropy.h"
Yaowu Xuc27fc142016-08-22 16:08:15 -070017#include "av1/common/mvref_common.h"
Hui Su1ddf2312017-08-19 15:21:34 -070018#include "av1/encoder/hash.h"
Yushin Cho55104332017-08-14 16:15:43 -070019#if CONFIG_DIST_8X8
20#include "aom/aomcx.h"
21#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -070022
23#ifdef __cplusplus
24extern "C" {
25#endif
26
27typedef struct {
28 unsigned int sse;
29 int sum;
30 unsigned int var;
Urvang Joshi454280d2016-10-14 16:51:44 -070031} DIFF;
Yaowu Xuc27fc142016-08-22 16:08:15 -070032
33typedef struct macroblock_plane {
34 DECLARE_ALIGNED(16, int16_t, src_diff[MAX_SB_SQUARE]);
35 tran_low_t *qcoeff;
36 tran_low_t *coeff;
37 uint16_t *eobs;
Angie Chiang74e23072017-03-24 14:54:23 -070038 uint8_t *txb_entropy_ctx;
Yaowu Xuc27fc142016-08-22 16:08:15 -070039 struct buf_2d src;
40
41 // Quantizer setings
Monty Montgomery125c0fc2017-10-26 00:44:35 -040042 // These are used/accessed only in the quantization process
43 // RDO does not / must not depend on any of these values
44 // All values below share the coefficient scale/shift used in TX
45 const int16_t *quant_fp_QTX;
46 const int16_t *round_fp_QTX;
47 const int16_t *quant_QTX;
48 const int16_t *quant_shift_QTX;
49 const int16_t *zbin_QTX;
50 const int16_t *round_QTX;
51 const int16_t *dequant_QTX;
Yaowu Xuc27fc142016-08-22 16:08:15 -070052} MACROBLOCK_PLANE;
53
hui suc0cf71d2017-07-20 16:38:50 -070054typedef int av1_coeff_cost[PLANE_TYPES][REF_TYPES][COEF_BANDS][COEFF_CONTEXTS]
55 [TAIL_TOKENS];
Yaowu Xuc27fc142016-08-22 16:08:15 -070056
Jingning Handfd72322017-08-09 14:04:12 -070057typedef struct {
58 int txb_skip_cost[TXB_SKIP_CONTEXTS][2];
Dake He3fe369c2017-11-16 17:56:44 -080059 int base_eob_cost[SIG_COEF_CONTEXTS_EOB][3];
Ola Hugosson13892102017-11-06 08:01:44 +010060 int base_cost[SIG_COEF_CONTEXTS][4];
Angie Chiang7ab884e2017-10-18 15:57:12 -070061 int eob_extra_cost[EOB_COEF_CONTEXTS][2];
Jingning Handfd72322017-08-09 14:04:12 -070062 int dc_sign_cost[DC_SIGN_CONTEXTS][2];
Angie Chiang26d3e452017-09-29 17:40:02 -070063 int lps_cost[LEVEL_CONTEXTS][COEFF_BASE_RANGE + 1];
Jingning Handfd72322017-08-09 14:04:12 -070064} LV_MAP_COEFF_COST;
Jingning Hanf5a4d3b2017-08-27 23:01:19 -070065
Johannb0ef6ff2018-02-08 14:32:21 -080066typedef struct {
67 int eob_cost[2][11];
68} LV_MAP_EOB_COST;
Dake He0db7d0e2017-12-21 15:23:20 -080069
Jingning Hanf5a4d3b2017-08-27 23:01:19 -070070typedef struct {
71 tran_low_t tcoeff[MAX_MB_PLANE][MAX_SB_SQUARE];
72 uint16_t eobs[MAX_MB_PLANE][MAX_SB_SQUARE / (TX_SIZE_W_MIN * TX_SIZE_H_MIN)];
73 uint8_t txb_skip_ctx[MAX_MB_PLANE]
74 [MAX_SB_SQUARE / (TX_SIZE_W_MIN * TX_SIZE_H_MIN)];
75 int dc_sign_ctx[MAX_MB_PLANE]
76 [MAX_SB_SQUARE / (TX_SIZE_W_MIN * TX_SIZE_H_MIN)];
77} CB_COEFF_BUFFER;
Jingning Handfd72322017-08-09 14:04:12 -070078
Yaowu Xuc27fc142016-08-22 16:08:15 -070079typedef struct {
80 int_mv ref_mvs[MODE_CTX_REF_FRAMES][MAX_MV_REF_CANDIDATES];
81 int16_t mode_context[MODE_CTX_REF_FRAMES];
Angie Chiangc484abe2017-03-20 15:43:11 -070082 // TODO(angiebird): Reduce the buffer size according to sb_type
Jingning Hanf5a4d3b2017-08-27 23:01:19 -070083 tran_low_t *tcoeff[MAX_MB_PLANE];
84 uint16_t *eobs[MAX_MB_PLANE];
85 uint8_t *txb_skip_ctx[MAX_MB_PLANE];
86 int *dc_sign_ctx[MAX_MB_PLANE];
Yaowu Xuc27fc142016-08-22 16:08:15 -070087 uint8_t ref_mv_count[MODE_CTX_REF_FRAMES];
88 CANDIDATE_MV ref_mv_stack[MODE_CTX_REF_FRAMES][MAX_REF_MV_STACK_SIZE];
Yaowu Xuc27fc142016-08-22 16:08:15 -070089 int16_t compound_mode_context[MODE_CTX_REF_FRAMES];
Yaowu Xuc27fc142016-08-22 16:08:15 -070090} MB_MODE_INFO_EXT;
91
Alex Converse0fa0f422017-04-24 12:51:14 -070092typedef struct {
93 int col_min;
94 int col_max;
95 int row_min;
96 int row_max;
97} MvLimits;
98
Yaowu Xuc27fc142016-08-22 16:08:15 -070099typedef struct {
Hui Su473cf892017-11-08 18:14:31 -0800100 uint8_t best_palette_color_map[MAX_PALETTE_SQUARE];
Hui Su5891f982017-12-18 16:18:23 -0800101 int kmeans_data_buf[2 * MAX_PALETTE_SQUARE];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700102} PALETTE_BUFFER;
103
Hui Su1ddf2312017-08-19 15:21:34 -0700104typedef struct {
Hui Su1ddf2312017-08-19 15:21:34 -0700105 TX_SIZE tx_size;
Hui Su7167d952018-02-01 16:33:12 -0800106 TX_SIZE inter_tx_size[INTER_TX_SIZE_BUF_LEN];
Hui Su1ddf2312017-08-19 15:21:34 -0700107 uint8_t blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE * 8];
Hui Sude1a5db2018-02-22 15:44:49 -0800108 TX_TYPE txk_type[TXK_TYPE_BUF_LEN];
Hui Su1ddf2312017-08-19 15:21:34 -0700109 RD_STATS rd_stats;
110 uint32_t hash_value;
111} TX_RD_INFO;
112
113#define RD_RECORD_BUFFER_LEN 8
114typedef struct {
115 TX_RD_INFO tx_rd_info[RD_RECORD_BUFFER_LEN]; // Circular buffer.
116 int index_start;
117 int num;
118 CRC_CALCULATOR crc_calculator; // Hash function.
119} TX_RD_RECORD;
120
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700121typedef struct {
122 int64_t dist;
Jingning Han73bc2aa2018-02-02 14:31:39 -0800123 int64_t sse;
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700124 int rate;
Hui Su8c2b9132017-12-09 10:40:15 -0800125 uint16_t eob;
Jingning Han73bc2aa2018-02-02 14:31:39 -0800126 TX_TYPE tx_type;
Jingning Han45027c62017-12-11 11:47:15 -0800127 uint16_t entropy_context;
Jingning Hand7e99112017-12-13 09:47:45 -0800128 uint8_t txb_entropy_ctx;
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700129 uint8_t valid;
Hui Su950b9122018-02-03 10:21:40 -0800130 uint8_t fast; // This is not being used now.
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700131} TX_SIZE_RD_INFO;
132
133#define TX_SIZE_RD_RECORD_BUFFER_LEN 256
134typedef struct {
135 uint32_t hash_vals[TX_SIZE_RD_RECORD_BUFFER_LEN];
Jingning Han73bc2aa2018-02-02 14:31:39 -0800136 TX_SIZE_RD_INFO tx_rd_info[TX_SIZE_RD_RECORD_BUFFER_LEN];
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700137 int index_start;
138 int num;
139} TX_SIZE_RD_RECORD;
140
141typedef struct tx_size_rd_info_node {
142 TX_SIZE_RD_INFO *rd_info_array; // Points to array of size TX_TYPES.
143 struct tx_size_rd_info_node *children[4];
144} TX_SIZE_RD_INFO_NODE;
145
Yaowu Xuc27fc142016-08-22 16:08:15 -0700146typedef struct macroblock MACROBLOCK;
147struct macroblock {
148 struct macroblock_plane plane[MAX_MB_PLANE];
149
Hui Su1ddf2312017-08-19 15:21:34 -0700150 // Save the transform RD search info.
151 TX_RD_RECORD tx_rd_record;
152
Jingning Han66965a22018-01-25 09:53:41 -0800153 // Determine if one would go with reduced complexity transform block
154 // search model to select prediction modes, or full complexity model
155 // to select transform kernel.
Jingning Handd8600f2018-01-23 09:06:32 -0800156 int rd_model;
157
Jingning Han4489c262018-01-26 16:43:59 -0800158 // Indicate if the encoder is running in the first pass partition search.
159 // In that case, apply certain speed features therein to reduce the overhead
160 // cost in the first pass search.
Jingning Hanc58ccc32018-01-26 07:46:38 -0800161 int cb_partition_scan;
162
Jingning Han4489c262018-01-26 16:43:59 -0800163 // Activate constrained coding block partition search range.
164 int use_cb_search_range;
165
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700166 // Also save RD info on the TX size search level for square TX sizes.
167 TX_SIZE_RD_RECORD
168 tx_size_rd_record_8X8[(MAX_MIB_SIZE >> 1) * (MAX_MIB_SIZE >> 1)];
169 TX_SIZE_RD_RECORD
170 tx_size_rd_record_16X16[(MAX_MIB_SIZE >> 2) * (MAX_MIB_SIZE >> 2)];
171 TX_SIZE_RD_RECORD
172 tx_size_rd_record_32X32[(MAX_MIB_SIZE >> 3) * (MAX_MIB_SIZE >> 3)];
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700173 TX_SIZE_RD_RECORD
174 tx_size_rd_record_64X64[(MAX_MIB_SIZE >> 4) * (MAX_MIB_SIZE >> 4)];
Alexander Bokovc5ddf062017-10-17 16:41:46 -0700175
Yaowu Xuc27fc142016-08-22 16:08:15 -0700176 MACROBLOCKD e_mbd;
177 MB_MODE_INFO_EXT *mbmi_ext;
178 int skip_block;
David Barkerd7d78c82016-10-24 10:55:35 +0100179 int qindex;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700180
181 // The equivalent error at the current rdmult of one whole bit (not one
182 // bitcost unit).
183 int errorperbit;
184 // The equivalend SAD error of one (whole) bit at the current quantizer
185 // for large blocks.
186 int sadperbit16;
187 // The equivalend SAD error of one (whole) bit at the current quantizer
188 // for sub-8x8 blocks.
189 int sadperbit4;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700190 int rdmult;
191 int mb_energy;
192 int *m_search_count_ptr;
193 int *ex_search_count_ptr;
194
Jingning Han9777afc2016-10-20 15:17:43 -0700195 unsigned int txb_split_count;
Jingning Han9777afc2016-10-20 15:17:43 -0700196
Yaowu Xuc27fc142016-08-22 16:08:15 -0700197 // These are set to their default values at the beginning, and then adjusted
198 // further in the encoding process.
199 BLOCK_SIZE min_partition_size;
200 BLOCK_SIZE max_partition_size;
201
202 int mv_best_ref_index[TOTAL_REFS_PER_FRAME];
203 unsigned int max_mv_context[TOTAL_REFS_PER_FRAME];
204 unsigned int source_variance;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700205 unsigned int pred_sse[TOTAL_REFS_PER_FRAME];
206 int pred_mv_sad[TOTAL_REFS_PER_FRAME];
207
Yaowu Xuc27fc142016-08-22 16:08:15 -0700208 int *nmvjointcost;
209 int nmv_vec_cost[NMV_CONTEXTS][MV_JOINTS];
210 int *nmvcost[NMV_CONTEXTS][2];
211 int *nmvcost_hp[NMV_CONTEXTS][2];
212 int **mv_cost_stack[NMV_CONTEXTS];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700213 int **mvcost;
Alex Conversea127a792017-05-23 15:27:21 -0700214
Yue Chene9638cc2016-10-10 12:37:54 -0700215 int32_t *wsrc_buf;
216 int32_t *mask_buf;
Jingning Hand064cf02017-06-01 10:00:39 -0700217 uint8_t *above_pred_buf;
218 uint8_t *left_pred_buf;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700219
220 PALETTE_BUFFER *palette_buffer;
221
222 // These define limits to motion vector components to prevent them
223 // from extending outside the UMV borders
Alex Converse0fa0f422017-04-24 12:51:14 -0700224 MvLimits mv_limits;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700225
Jingning Hanca3f0392018-02-15 09:54:49 -0800226 uint8_t blk_skip[MAX_MB_PLANE][MAX_MIB_SIZE * MAX_MIB_SIZE];
227 uint8_t blk_skip_drl[MAX_MB_PLANE][MAX_MIB_SIZE * MAX_MIB_SIZE];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700228
229 int skip;
Jingning Han8efdbc82017-02-19 14:40:03 -0800230 int skip_chroma_rd;
Zoe Liu1eed2df2017-10-16 17:13:15 -0700231 int skip_cost[SKIP_CONTEXTS][2];
232
Zoe Liuf40a9572017-10-13 12:37:19 -0700233 int skip_mode; // 0: off; 1: on
234 int skip_mode_cost[SKIP_CONTEXTS][2];
235
236 int64_t skip_mode_rdcost; // -1: Not set
237 int skip_mode_rate;
238 int64_t skip_mode_sse;
239 int64_t skip_mode_dist;
240 MV_REFERENCE_FRAME skip_mode_ref_frame[2];
241 int_mv skip_mode_mv[2];
Zoe Liu104d62e2017-12-07 12:44:45 -0800242#if CONFIG_JNT_COMP
243 int compound_idx;
244#endif // CONFIG_JNT_COMP
Zoe Liuf40a9572017-10-13 12:37:19 -0700245 int skip_mode_index_candidate;
246 int skip_mode_index;
Zoe Liuf40a9572017-10-13 12:37:19 -0700247
Jingning Handfd72322017-08-09 14:04:12 -0700248 LV_MAP_COEFF_COST coeff_costs[TX_SIZES][PLANE_TYPES];
Dake He0db7d0e2017-12-21 15:23:20 -0800249 LV_MAP_EOB_COST eob_costs[7][2];
Jingning Hanf5a4d3b2017-08-27 23:01:19 -0700250 uint16_t cb_offset;
Jingning Handfd72322017-08-09 14:04:12 -0700251
hui suc0cf71d2017-07-20 16:38:50 -0700252 av1_coeff_cost token_head_costs[TX_SIZES];
253 av1_coeff_cost token_tail_costs[TX_SIZES];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700254
Yue Chenb23d00a2017-07-28 17:01:21 -0700255 // mode costs
Yue Chen170678a2017-10-17 13:43:10 -0700256 int intra_inter_cost[INTRA_INTER_CONTEXTS][2];
257
Yue Chenb23d00a2017-07-28 17:01:21 -0700258 int mbmode_cost[BLOCK_SIZE_GROUPS][INTRA_MODES];
259 int newmv_mode_cost[NEWMV_MODE_CONTEXTS][2];
Sarah Parker2b9ec2e2017-10-30 17:34:08 -0700260 int zeromv_mode_cost[GLOBALMV_MODE_CONTEXTS][2];
Yue Chenb23d00a2017-07-28 17:01:21 -0700261 int refmv_mode_cost[REFMV_MODE_CONTEXTS][2];
262 int drl_mode_cost0[DRL_MODE_CONTEXTS][2];
263
Hui Su9d0c03d2017-12-27 16:05:23 -0800264 int comp_inter_cost[COMP_INTER_CONTEXTS][2];
Hui Su3d30b4b2017-12-27 16:47:59 -0800265 int single_ref_cost[REF_CONTEXTS][SINGLE_REFS - 1][2];
Hui Su6b3d1e32018-01-05 11:25:40 -0800266 int comp_ref_type_cost[COMP_REF_TYPE_CONTEXTS]
267 [CDF_SIZE(COMP_REFERENCE_TYPES)];
Hui Sua7e3bfe2018-01-05 12:14:48 -0800268 int uni_comp_ref_cost[UNI_COMP_REF_CONTEXTS][UNIDIR_COMP_REFS - 1]
269 [CDF_SIZE(2)];
Hui Su0bdf5f52018-01-05 14:54:32 -0800270 // Cost for signaling ref_frame[0] (LAST_FRAME, LAST2_FRAME, LAST3_FRAME or
271 // GOLDEN_FRAME) in bidir-comp mode.
Zoe Liu3b353472018-02-12 13:58:22 -0800272 int comp_ref_cost[REF_CONTEXTS][FWD_REFS - 1][2];
Hui Su0bdf5f52018-01-05 14:54:32 -0800273 // Cost for signaling ref_frame[1] (ALTREF_FRAME, ALTREF2_FRAME, or
274 // BWDREF_FRAME) in bidir-comp mode.
Zoe Liu3b353472018-02-12 13:58:22 -0800275 int comp_bwdref_cost[REF_CONTEXTS][BWD_REFS - 1][2];
Yue Chenb23d00a2017-07-28 17:01:21 -0700276 int inter_compound_mode_cost[INTER_MODE_CONTEXTS][INTER_COMPOUND_MODES];
Cheng Chen2ef24ea2017-11-29 12:22:24 -0800277#if CONFIG_JNT_COMP
278 int compound_type_cost[BLOCK_SIZES_ALL][COMPOUND_TYPES - 1];
279#else
Yue Chena4245512017-08-31 11:58:08 -0700280 int compound_type_cost[BLOCK_SIZES_ALL][COMPOUND_TYPES];
Cheng Chen2ef24ea2017-11-29 12:22:24 -0800281#endif // CONFIG_JNT_COMP
Yue Chen73335fa2017-12-20 23:33:41 -0800282#if WEDGE_IDX_ENTROPY_CODING
283 int wedge_idx_cost[BLOCK_SIZES_ALL][16];
284#endif
Yue Cheneaf128a2017-10-16 17:01:36 -0700285 int interintra_cost[BLOCK_SIZE_GROUPS][2];
286 int wedge_interintra_cost[BLOCK_SIZES_ALL][2];
Yue Chenb23d00a2017-07-28 17:01:21 -0700287 int interintra_mode_cost[BLOCK_SIZE_GROUPS][INTERINTRA_MODES];
Yue Chenb23d00a2017-07-28 17:01:21 -0700288 int motion_mode_cost[BLOCK_SIZES_ALL][MOTION_MODES];
Yue Chenb23d00a2017-07-28 17:01:21 -0700289 int motion_mode_cost1[BLOCK_SIZES_ALL][2];
David Michael Barrcb3a8ef2018-01-06 15:48:49 +0900290#if CONFIG_CFL
291 int intra_uv_mode_cost[CFL_ALLOWED_TYPES][INTRA_MODES][UV_INTRA_MODES];
292#else
Yue Chenb23d00a2017-07-28 17:01:21 -0700293 int intra_uv_mode_cost[INTRA_MODES][UV_INTRA_MODES];
David Michael Barrcb3a8ef2018-01-06 15:48:49 +0900294#endif
Yue Chenb23d00a2017-07-28 17:01:21 -0700295 int y_mode_costs[INTRA_MODES][INTRA_MODES][INTRA_MODES];
Yue Chen4eba69b2017-11-09 22:37:35 -0800296 int filter_intra_cost[TX_SIZES_ALL][2];
Yue Chen994dba22017-12-19 15:27:26 -0800297 int filter_intra_mode_cost[FILTER_INTRA_MODES];
Yue Chenb23d00a2017-07-28 17:01:21 -0700298 int switchable_interp_costs[SWITCHABLE_FILTER_CONTEXTS][SWITCHABLE_FILTERS];
Sebastien Alaiwana6a486c2017-11-07 17:04:27 +0100299 int partition_cost[PARTITION_CONTEXTS][EXT_PARTITION_TYPES];
Hui Suc1f411b2017-12-19 15:58:28 -0800300 int palette_y_size_cost[PALATTE_BSIZE_CTXS][PALETTE_SIZES];
301 int palette_uv_size_cost[PALATTE_BSIZE_CTXS][PALETTE_SIZES];
Yue Chenb23d00a2017-07-28 17:01:21 -0700302 int palette_y_color_cost[PALETTE_SIZES][PALETTE_COLOR_INDEX_CONTEXTS]
303 [PALETTE_COLORS];
304 int palette_uv_color_cost[PALETTE_SIZES][PALETTE_COLOR_INDEX_CONTEXTS]
305 [PALETTE_COLORS];
Hui Suc1f411b2017-12-19 15:58:28 -0800306 int palette_y_mode_cost[PALATTE_BSIZE_CTXS][PALETTE_Y_MODE_CONTEXTS][2];
Yue Chendab2ca92017-10-16 17:48:48 -0700307 int palette_uv_mode_cost[PALETTE_UV_MODE_CONTEXTS][2];
David Michael Barr38e560c2017-08-16 21:46:37 +0900308#if CONFIG_CFL
309 // The rate associated with each alpha codeword
310 int cfl_cost[CFL_JOINT_SIGNS][CFL_PRED_PLANES][CFL_ALPHABET_SIZE];
311#endif // CONFIG_CFL
Yue Chenb23d00a2017-07-28 17:01:21 -0700312 int tx_size_cost[TX_SIZES - 1][TX_SIZE_CONTEXTS][TX_SIZES];
Yue Chen171c17d2017-10-16 18:08:22 -0700313 int txfm_partition_cost[TXFM_PARTITION_CONTEXTS][2];
Yue Chenb23d00a2017-07-28 17:01:21 -0700314 int inter_tx_type_costs[EXT_TX_SETS_INTER][EXT_TX_SIZES][TX_TYPES];
315 int intra_tx_type_costs[EXT_TX_SETS_INTRA][EXT_TX_SIZES][INTRA_MODES]
316 [TX_TYPES];
Hui Su79c763b2018-01-18 13:18:46 -0800317#if CONFIG_EXT_INTRA_MOD
Joe Young3ca43bf2017-10-06 15:12:46 -0700318 int angle_delta_cost[DIRECTIONAL_MODES][2 * MAX_ANGLE_DELTA + 1];
Hui Su79c763b2018-01-18 13:18:46 -0800319#endif // CONFIG_EXT_INTRA_MOD
Yue Chenb23d00a2017-07-28 17:01:21 -0700320 int switchable_restore_cost[RESTORE_SWITCHABLE_TYPES];
Debargha Mukherjeebc732ef2017-10-12 12:40:25 -0700321 int wiener_restore_cost[2];
322 int sgrproj_restore_cost[2];
Hui Su6c8584f2017-09-14 15:37:02 -0700323 int intrabc_cost[2];
Yue Chenb23d00a2017-07-28 17:01:21 -0700324
Yaowu Xuc27fc142016-08-22 16:08:15 -0700325 int optimize;
326
Yaowu Xuc27fc142016-08-22 16:08:15 -0700327 // Used to store sub partition's choices.
328 MV pred_mv[TOTAL_REFS_PER_FRAME];
329
330 // Store the best motion vector during motion search
331 int_mv best_mv;
332 // Store the second best motion vector during full-pixel motion search
333 int_mv second_best_mv;
334
Yaowu Xuc27fc142016-08-22 16:08:15 -0700335 // use default transform and skip transform type search for intra modes
336 int use_default_intra_tx_type;
337 // use default transform and skip transform type search for inter modes
338 int use_default_inter_tx_type;
Yushin Chob7b60c52017-07-14 16:18:52 -0700339#if CONFIG_DIST_8X8
Yushin Cho55104332017-08-14 16:15:43 -0700340 int using_dist_8x8;
341 aom_tune_metric tune_metric;
Yushin Chob7b60c52017-07-14 16:18:52 -0700342#endif // CONFIG_DIST_8X8
Cheng Chen46970612017-10-24 14:53:36 -0700343#if CONFIG_JNT_COMP
344 int comp_idx_cost[COMP_INDEX_CONTEXTS][2];
Cheng Chen2ef24ea2017-11-29 12:22:24 -0800345 int comp_group_idx_cost[COMP_GROUP_IDX_CONTEXTS][2];
Cheng Chen46970612017-10-24 14:53:36 -0700346#endif // CONFIG_JNT_COMP
Hui Su6bbd9322018-01-26 11:04:13 -0800347 // Bit flags for pruning tx type search, tx split, etc.
348 int tx_search_prune[EXT_TX_SET_TYPES];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700349};
350
Frederic Barbier0f191da2018-01-03 17:29:26 +0100351static INLINE int is_rect_tx_allowed_bsize(BLOCK_SIZE bsize) {
352 static const char LUT[BLOCK_SIZES_ALL] = {
353 0, // BLOCK_4X4
354 1, // BLOCK_4X8
355 1, // BLOCK_8X4
356 0, // BLOCK_8X8
357 1, // BLOCK_8X16
358 1, // BLOCK_16X8
359 0, // BLOCK_16X16
360 1, // BLOCK_16X32
361 1, // BLOCK_32X16
362 0, // BLOCK_32X32
363 1, // BLOCK_32X64
364 1, // BLOCK_64X32
365 0, // BLOCK_64X64
Frederic Barbier0f191da2018-01-03 17:29:26 +0100366 0, // BLOCK_64X128
367 0, // BLOCK_128X64
368 0, // BLOCK_128X128
Frederic Barbier0f191da2018-01-03 17:29:26 +0100369 1, // BLOCK_4X16
370 1, // BLOCK_16X4
371 1, // BLOCK_8X32
372 1, // BLOCK_32X8
373 1, // BLOCK_16X64
374 1, // BLOCK_64X16
Frederic Barbier0f191da2018-01-03 17:29:26 +0100375 1, // BLOCK_32X128
376 1, // BLOCK_128X32
Frederic Barbier0f191da2018-01-03 17:29:26 +0100377 };
378
379 return LUT[bsize];
380}
381
382static INLINE int is_rect_tx_allowed(const MACROBLOCKD *xd,
383 const MB_MODE_INFO *mbmi) {
384 return is_rect_tx_allowed_bsize(mbmi->sb_type) &&
385 !xd->lossless[mbmi->segment_id];
386}
387
388static INLINE int tx_size_to_depth(TX_SIZE tx_size, BLOCK_SIZE bsize,
389 int is_inter) {
Yaowu Xu25ff26a2018-02-26 11:20:10 -0800390 TX_SIZE ctx_size = get_max_rect_tx_size(bsize);
Frederic Barbier0f191da2018-01-03 17:29:26 +0100391 int depth = 0;
392 while (tx_size != ctx_size) {
393 depth++;
394 ctx_size = sub_tx_size_map[is_inter][ctx_size];
395 assert(depth <= MAX_TX_DEPTH);
396 }
397 return depth;
398}
399
Yaowu Xuc27fc142016-08-22 16:08:15 -0700400#ifdef __cplusplus
401} // extern "C"
402#endif
403
Yaowu Xuf883b422016-08-30 14:01:10 -0700404#endif // AV1_ENCODER_BLOCK_H_