blob: 5b34c7b35b382b9d07bcc47080e7b6a6020ea584 [file] [log] [blame]
Yaowu Xuc27fc142016-08-22 16:08:15 -07001/*
Yaowu Xu2ab7ff02016-09-02 12:04:54 -07002 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Yaowu Xuc27fc142016-08-22 16:08:15 -07003 *
Yaowu Xu2ab7ff02016-09-02 12:04:54 -07004 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Yaowu Xuc27fc142016-08-22 16:08:15 -070010 */
11
Yaowu Xuf883b422016-08-30 14:01:10 -070012#ifndef AV1_COMMON_BLOCKD_H_
13#define AV1_COMMON_BLOCKD_H_
Yaowu Xuc27fc142016-08-22 16:08:15 -070014
Yaowu Xuf883b422016-08-30 14:01:10 -070015#include "./aom_config.h"
Yaowu Xuc27fc142016-08-22 16:08:15 -070016
Yaowu Xuf883b422016-08-30 14:01:10 -070017#include "aom_dsp/aom_dsp_common.h"
Yaowu Xuc27fc142016-08-22 16:08:15 -070018#include "aom_ports/mem.h"
19#include "aom_scale/yv12config.h"
20
21#include "av1/common/common_data.h"
22#include "av1/common/quant_common.h"
23#include "av1/common/entropy.h"
24#include "av1/common/entropymode.h"
25#include "av1/common/mv.h"
26#include "av1/common/scale.h"
27#include "av1/common/seg_common.h"
28#include "av1/common/tile_common.h"
Yushin Cho77bba8d2016-11-04 16:36:56 -070029#if CONFIG_PVQ
30#include "av1/common/pvq.h"
31#include "av1/common/pvq_state.h"
32#include "av1/decoder/decint.h"
33#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -070034#ifdef __cplusplus
35extern "C" {
36#endif
37
Jingning Han69d21012017-05-14 16:51:27 -070038#if (CONFIG_CHROMA_SUB8X8 || CONFIG_CHROMA_2X2)
39#define SUB8X8_COMP_REF 0
40#else
41#define SUB8X8_COMP_REF 1
42#endif
Jingning Hanc41a5492017-02-24 11:18:52 -080043
Yaowu Xuc27fc142016-08-22 16:08:15 -070044#define MAX_MB_PLANE 3
45
David Barkerac37fa32016-12-02 12:30:21 +000046#if CONFIG_EXT_INTER
Sarah Parkerb9f757c2017-01-06 17:12:24 -080047
Debargha Mukherjee1edf9a32017-01-07 18:54:20 -080048#if CONFIG_COMPOUND_SEGMENT
Debargha Mukherjee1edf9a32017-01-07 18:54:20 -080049// Set COMPOUND_SEGMENT_TYPE to one of the three
50// 0: Uniform
51// 1: Difference weighted
52#define COMPOUND_SEGMENT_TYPE 1
Debargha Mukherjee1edf9a32017-01-07 18:54:20 -080053#define MAX_SEG_MASK_BITS 1
Yaowu Xuf35f5272017-05-10 08:00:02 -070054
Sarah Parkerb9f757c2017-01-06 17:12:24 -080055// SEG_MASK_TYPES should not surpass 1 << MAX_SEG_MASK_BITS
56typedef enum {
Yaowu Xuf35f5272017-05-10 08:00:02 -070057#if COMPOUND_SEGMENT_TYPE == 0
Sarah Parkerb9f757c2017-01-06 17:12:24 -080058 UNIFORM_45 = 0,
59 UNIFORM_45_INV,
Debargha Mukherjee1edf9a32017-01-07 18:54:20 -080060#elif COMPOUND_SEGMENT_TYPE == 1
Sarah Parker7bb84f32017-05-09 15:17:46 -070061 DIFFWTD_38 = 0,
62 DIFFWTD_38_INV,
Yaowu Xuf35f5272017-05-10 08:00:02 -070063#endif // COMPOUND_SEGMENT_TYPE
Debargha Mukherjee1edf9a32017-01-07 18:54:20 -080064 SEG_MASK_TYPES,
65} SEG_MASK_TYPE;
66
Sarah Parkerb9f757c2017-01-06 17:12:24 -080067#endif // CONFIG_COMPOUND_SEGMENT
Debargha Mukherjee1edf9a32017-01-07 18:54:20 -080068#endif // CONFIG_EXT_INTER
David Barkerac37fa32016-12-02 12:30:21 +000069
Yaowu Xuc27fc142016-08-22 16:08:15 -070070typedef enum {
71 KEY_FRAME = 0,
72 INTER_FRAME = 1,
Soo-Chul Han65c00ae2017-09-07 13:12:35 -040073#if CONFIG_OBU
74 INTRA_ONLY_FRAME = 2, // replaces intra-only
75 S_FRAME = 3,
76#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -070077 FRAME_TYPES,
78} FRAME_TYPE;
79
Debargha Mukherjee0f248c42017-09-07 12:40:18 -070080static INLINE int is_comp_ref_allowed(BLOCK_SIZE bsize) {
81 (void)bsize;
82#if SUB8X8_COMP_REF
83 return 1;
84#else
85 return AOMMIN(block_size_wide[bsize], block_size_high[bsize]) >= 8;
86#endif // SUB8X8_COMP_REF
87}
88
Yaowu Xuc27fc142016-08-22 16:08:15 -070089static INLINE int is_inter_mode(PREDICTION_MODE mode) {
90#if CONFIG_EXT_INTER
91 return mode >= NEARESTMV && mode <= NEW_NEWMV;
92#else
93 return mode >= NEARESTMV && mode <= NEWMV;
94#endif // CONFIG_EXT_INTER
95}
96
Yushin Cho77bba8d2016-11-04 16:36:56 -070097#if CONFIG_PVQ
98typedef struct PVQ_INFO {
99 int theta[PVQ_MAX_PARTITIONS];
Yushin Cho77bba8d2016-11-04 16:36:56 -0700100 int qg[PVQ_MAX_PARTITIONS];
101 int k[PVQ_MAX_PARTITIONS];
Yushin Cho48f84db2016-11-07 21:20:17 -0800102 od_coeff y[OD_TXSIZE_MAX * OD_TXSIZE_MAX];
Yushin Cho77bba8d2016-11-04 16:36:56 -0700103 int nb_bands;
104 int off[PVQ_MAX_PARTITIONS];
105 int size[PVQ_MAX_PARTITIONS];
106 int skip_rest;
107 int skip_dir;
ltrudeaue1c09292017-01-20 15:42:13 -0500108 int bs; // log of the block size minus two,
109 // i.e. equivalent to aom's TX_SIZE
110 // Block skip info, indicating whether DC/AC, is coded.
111 PVQ_SKIP_TYPE ac_dc_coded; // bit0: DC coded, bit1 : AC coded (1 means coded)
Yushin Cho77bba8d2016-11-04 16:36:56 -0700112 tran_low_t dq_dc_residue;
113} PVQ_INFO;
114
115typedef struct PVQ_QUEUE {
116 PVQ_INFO *buf; // buffer for pvq info, stored in encoding order
117 int curr_pos; // curr position to write PVQ_INFO
118 int buf_len; // allocated buffer length
119 int last_pos; // last written position of PVQ_INFO in a tile
120} PVQ_QUEUE;
121#endif
122
Wei-Ting Lin482551b2017-08-03 12:29:24 -0700123#if CONFIG_NCOBMC_ADAPT_WEIGHT
124typedef struct superblock_mi_boundaries {
125 int mi_row_begin;
126 int mi_col_begin;
127 int mi_row_end;
128 int mi_col_end;
129} SB_MI_BD;
130
Wei-Ting Linb586de82017-08-03 15:17:38 -0700131typedef struct { int16_t KERNEL[4][MAX_SB_SIZE][MAX_SB_SIZE]; } NCOBMC_KERNELS;
Wei-Ting Lin482551b2017-08-03 12:29:24 -0700132#endif
133
David Barkerac37fa32016-12-02 12:30:21 +0000134typedef struct {
135 uint8_t *plane[MAX_MB_PLANE];
136 int stride[MAX_MB_PLANE];
137} BUFFER_SET;
138
Yaowu Xuc27fc142016-08-22 16:08:15 -0700139#if CONFIG_EXT_INTER
140static INLINE int is_inter_singleref_mode(PREDICTION_MODE mode) {
Zoe Liu7f24e1b2017-03-17 17:42:05 -0700141 return mode >= NEARESTMV && mode <= NEWMV;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700142}
Zoe Liu85b66462017-04-20 14:28:19 -0700143static INLINE int is_inter_compound_mode(PREDICTION_MODE mode) {
144 return mode >= NEAREST_NEARESTMV && mode <= NEW_NEWMV;
145}
Zoe Liu239f06b2017-04-20 13:10:55 -0700146#if CONFIG_COMPOUND_SINGLEREF
147static INLINE int is_inter_singleref_comp_mode(PREDICTION_MODE mode) {
148 return mode >= SR_NEAREST_NEARMV && mode <= SR_NEW_NEWMV;
149}
Zoe Liu85b66462017-04-20 14:28:19 -0700150static INLINE int is_inter_anyref_comp_mode(PREDICTION_MODE mode) {
151 return is_inter_compound_mode(mode) || is_inter_singleref_comp_mode(mode);
Yaowu Xuc27fc142016-08-22 16:08:15 -0700152}
Zoe Liu85b66462017-04-20 14:28:19 -0700153#endif // CONFIG_COMPOUND_SINGLEREF
Yaowu Xuc27fc142016-08-22 16:08:15 -0700154
155static INLINE PREDICTION_MODE compound_ref0_mode(PREDICTION_MODE mode) {
Urvang Joshi5a9ea002017-05-22 15:25:18 -0700156 static PREDICTION_MODE lut[] = {
Urvang Joshi102245d2016-11-28 13:05:36 -0800157 MB_MODE_COUNT, // DC_PRED
158 MB_MODE_COUNT, // V_PRED
159 MB_MODE_COUNT, // H_PRED
160 MB_MODE_COUNT, // D45_PRED
161 MB_MODE_COUNT, // D135_PRED
162 MB_MODE_COUNT, // D117_PRED
163 MB_MODE_COUNT, // D153_PRED
164 MB_MODE_COUNT, // D207_PRED
165 MB_MODE_COUNT, // D63_PRED
Urvang Joshi102245d2016-11-28 13:05:36 -0800166 MB_MODE_COUNT, // SMOOTH_PRED
Urvang Joshi5a9ea002017-05-22 15:25:18 -0700167#if CONFIG_SMOOTH_HV
168 MB_MODE_COUNT, // SMOOTH_V_PRED
169 MB_MODE_COUNT, // SMOOTH_H_PRED
170#endif // CONFIG_SMOOTH_HV
Urvang Joshi102245d2016-11-28 13:05:36 -0800171 MB_MODE_COUNT, // TM_PRED
172 MB_MODE_COUNT, // NEARESTMV
173 MB_MODE_COUNT, // NEARMV
174 MB_MODE_COUNT, // ZEROMV
175 MB_MODE_COUNT, // NEWMV
Zoe Liu239f06b2017-04-20 13:10:55 -0700176#if CONFIG_COMPOUND_SINGLEREF
177 NEARESTMV, // SR_NEAREST_NEARMV
Zoe Liu85b66462017-04-20 14:28:19 -0700178 // NEARESTMV, // SR_NEAREST_NEWMV
Zoe Liu239f06b2017-04-20 13:10:55 -0700179 NEARMV, // SR_NEAR_NEWMV
180 ZEROMV, // SR_ZERO_NEWMV
181 NEWMV, // SR_NEW_NEWMV
182#endif // CONFIG_COMPOUND_SINGLEREF
183 NEARESTMV, // NEAREST_NEARESTMV
Zoe Liu239f06b2017-04-20 13:10:55 -0700184 NEARMV, // NEAR_NEARMV
185 NEARESTMV, // NEAREST_NEWMV
186 NEWMV, // NEW_NEARESTMV
187 NEARMV, // NEAR_NEWMV
188 NEWMV, // NEW_NEARMV
189 ZEROMV, // ZERO_ZEROMV
190 NEWMV, // NEW_NEWMV
Yaowu Xuc27fc142016-08-22 16:08:15 -0700191 };
Urvang Joshi5a9ea002017-05-22 15:25:18 -0700192 assert(NELEMENTS(lut) == MB_MODE_COUNT);
Zoe Liu85b66462017-04-20 14:28:19 -0700193#if CONFIG_COMPOUND_SINGLEREF
194 assert(is_inter_anyref_comp_mode(mode));
195#else // !CONFIG_COMPOUND_SINGLEREF
Yaowu Xuc27fc142016-08-22 16:08:15 -0700196 assert(is_inter_compound_mode(mode));
Zoe Liu85b66462017-04-20 14:28:19 -0700197#endif // CONFIG_COMPOUND_SINGLEREF
Yaowu Xuc27fc142016-08-22 16:08:15 -0700198 return lut[mode];
199}
200
201static INLINE PREDICTION_MODE compound_ref1_mode(PREDICTION_MODE mode) {
Urvang Joshi5a9ea002017-05-22 15:25:18 -0700202 static PREDICTION_MODE lut[] = {
Urvang Joshi102245d2016-11-28 13:05:36 -0800203 MB_MODE_COUNT, // DC_PRED
204 MB_MODE_COUNT, // V_PRED
205 MB_MODE_COUNT, // H_PRED
206 MB_MODE_COUNT, // D45_PRED
207 MB_MODE_COUNT, // D135_PRED
208 MB_MODE_COUNT, // D117_PRED
209 MB_MODE_COUNT, // D153_PRED
210 MB_MODE_COUNT, // D207_PRED
211 MB_MODE_COUNT, // D63_PRED
Urvang Joshi102245d2016-11-28 13:05:36 -0800212 MB_MODE_COUNT, // SMOOTH_PRED
Urvang Joshi5a9ea002017-05-22 15:25:18 -0700213#if CONFIG_SMOOTH_HV
214 MB_MODE_COUNT, // SMOOTH_V_PRED
215 MB_MODE_COUNT, // SMOOTH_H_PRED
216#endif // CONFIG_SMOOTH_HV
Urvang Joshi102245d2016-11-28 13:05:36 -0800217 MB_MODE_COUNT, // TM_PRED
218 MB_MODE_COUNT, // NEARESTMV
219 MB_MODE_COUNT, // NEARMV
220 MB_MODE_COUNT, // ZEROMV
221 MB_MODE_COUNT, // NEWMV
Zoe Liu239f06b2017-04-20 13:10:55 -0700222#if CONFIG_COMPOUND_SINGLEREF
Zoe Liu85b66462017-04-20 14:28:19 -0700223 NEARMV, // SR_NEAREST_NEARMV
224 // NEWMV, // SR_NEAREST_NEWMV
Zoe Liu239f06b2017-04-20 13:10:55 -0700225 NEWMV, // SR_NEAR_NEWMV
226 NEWMV, // SR_ZERO_NEWMV
227 NEWMV, // SR_NEW_NEWMV
228#endif // CONFIG_COMPOUND_SINGLEREF
229 NEARESTMV, // NEAREST_NEARESTMV
Zoe Liu239f06b2017-04-20 13:10:55 -0700230 NEARMV, // NEAR_NEARMV
231 NEWMV, // NEAREST_NEWMV
232 NEARESTMV, // NEW_NEARESTMV
233 NEWMV, // NEAR_NEWMV
234 NEARMV, // NEW_NEARMV
235 ZEROMV, // ZERO_ZEROMV
236 NEWMV, // NEW_NEWMV
Yaowu Xuc27fc142016-08-22 16:08:15 -0700237 };
Urvang Joshi5a9ea002017-05-22 15:25:18 -0700238 assert(NELEMENTS(lut) == MB_MODE_COUNT);
Zoe Liu85b66462017-04-20 14:28:19 -0700239#if CONFIG_COMPOUND_SINGLEREF
240 assert(is_inter_anyref_comp_mode(mode));
241#else // !CONFIG_COMPOUND_SINGLEREF
Yaowu Xuc27fc142016-08-22 16:08:15 -0700242 assert(is_inter_compound_mode(mode));
Zoe Liu85b66462017-04-20 14:28:19 -0700243#endif // CONFIG_COMPOUND_SINGLEREF
Yaowu Xuc27fc142016-08-22 16:08:15 -0700244 return lut[mode];
245}
246
David Barker3dfba992017-04-03 16:10:09 +0100247static INLINE int have_nearmv_in_inter_mode(PREDICTION_MODE mode) {
Debargha Mukherjeebb6e1342017-04-17 16:05:04 -0700248 return (mode == NEARMV || mode == NEAR_NEARMV || mode == NEAR_NEWMV ||
Zoe Liu85b66462017-04-20 14:28:19 -0700249#if CONFIG_COMPOUND_SINGLEREF
250 mode == SR_NEAREST_NEARMV || mode == SR_NEAR_NEWMV ||
251#endif // CONFIG_COMPOUND_SINGLEREF
Debargha Mukherjeebb6e1342017-04-17 16:05:04 -0700252 mode == NEW_NEARMV);
David Barker3dfba992017-04-03 16:10:09 +0100253}
254
Yaowu Xuc27fc142016-08-22 16:08:15 -0700255static INLINE int have_newmv_in_inter_mode(PREDICTION_MODE mode) {
Zoe Liu7f24e1b2017-03-17 17:42:05 -0700256 return (mode == NEWMV || mode == NEW_NEWMV || mode == NEAREST_NEWMV ||
Zoe Liu85b66462017-04-20 14:28:19 -0700257#if CONFIG_COMPOUND_SINGLEREF
258 /* mode == SR_NEAREST_NEWMV || */ mode == SR_NEAR_NEWMV ||
259 mode == SR_ZERO_NEWMV || mode == SR_NEW_NEWMV ||
260#endif // CONFIG_COMPOUND_SINGLEREF
Zoe Liu7f24e1b2017-03-17 17:42:05 -0700261 mode == NEW_NEARESTMV || mode == NEAR_NEWMV || mode == NEW_NEARMV);
Yaowu Xuc27fc142016-08-22 16:08:15 -0700262}
Sarah Parker6fdc8532016-11-16 17:47:13 -0800263
Sarah Parker2e604882017-01-17 17:31:25 -0800264static INLINE int use_masked_motion_search(COMPOUND_TYPE type) {
Debargha Mukherjeec5f735f2017-04-26 03:25:28 +0000265#if CONFIG_WEDGE
Sarah Parker2e604882017-01-17 17:31:25 -0800266 return (type == COMPOUND_WEDGE);
Debargha Mukherjeec5f735f2017-04-26 03:25:28 +0000267#else
268 (void)type;
269 return 0;
270#endif
Sarah Parker2e604882017-01-17 17:31:25 -0800271}
272
Sarah Parker6fdc8532016-11-16 17:47:13 -0800273static INLINE int is_masked_compound_type(COMPOUND_TYPE type) {
Debargha Mukherjeec5f735f2017-04-26 03:25:28 +0000274#if CONFIG_COMPOUND_SEGMENT && CONFIG_WEDGE
Sarah Parker569edda2016-12-14 14:57:38 -0800275 return (type == COMPOUND_WEDGE || type == COMPOUND_SEG);
Debargha Mukherjeec5f735f2017-04-26 03:25:28 +0000276#elif !CONFIG_COMPOUND_SEGMENT && CONFIG_WEDGE
Sarah Parker6fdc8532016-11-16 17:47:13 -0800277 return (type == COMPOUND_WEDGE);
Debargha Mukherjeec5f735f2017-04-26 03:25:28 +0000278#elif CONFIG_COMPOUND_SEGMENT && !CONFIG_WEDGE
279 return (type == COMPOUND_SEG);
Sarah Parker569edda2016-12-14 14:57:38 -0800280#endif // CONFIG_COMPOUND_SEGMENT
Debargha Mukherjeec5f735f2017-04-26 03:25:28 +0000281 (void)type;
282 return 0;
Sarah Parker6fdc8532016-11-16 17:47:13 -0800283}
Zoe Liu85b66462017-04-20 14:28:19 -0700284
285#else // !CONFIG_EXT_INTER
Yaowu Xuc27fc142016-08-22 16:08:15 -0700286
David Barker3dfba992017-04-03 16:10:09 +0100287static INLINE int have_nearmv_in_inter_mode(PREDICTION_MODE mode) {
288 return (mode == NEARMV);
289}
290
Yaowu Xuc27fc142016-08-22 16:08:15 -0700291static INLINE int have_newmv_in_inter_mode(PREDICTION_MODE mode) {
292 return (mode == NEWMV);
293}
294#endif // CONFIG_EXT_INTER
295
296/* For keyframes, intra block modes are predicted by the (already decoded)
297 modes for the Y blocks to the left and above us; for interframes, there
298 is a single probability table. */
299
300typedef struct {
301 PREDICTION_MODE as_mode;
302 int_mv as_mv[2]; // first, second inter predictor motion vectors
Yaowu Xuf5bbbfa2016-09-26 09:13:38 -0700303 int_mv pred_mv[2];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700304#if CONFIG_EXT_INTER
305 int_mv ref_mv[2];
306#endif // CONFIG_EXT_INTER
307} b_mode_info;
308
309typedef int8_t MV_REFERENCE_FRAME;
310
311typedef struct {
312 // Number of base colors for Y (0) and UV (1)
313 uint8_t palette_size[2];
hui sufa4ff852017-05-15 12:20:50 -0700314 // Value of base colors for Y, U, and V
Yaowu Xuc27fc142016-08-22 16:08:15 -0700315 uint16_t palette_colors[3 * PALETTE_MAX_SIZE];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700316} PALETTE_MODE_INFO;
317
hui su5db97432016-10-14 16:10:14 -0700318#if CONFIG_FILTER_INTRA
Yue Chen8d8638a2017-02-21 13:28:16 +0800319#define USE_3TAP_INTRA_FILTER 1 // 0: 4-tap; 1: 3-tap
Yaowu Xuc27fc142016-08-22 16:08:15 -0700320typedef struct {
321 // 1: an ext intra mode is used; 0: otherwise.
hui su5db97432016-10-14 16:10:14 -0700322 uint8_t use_filter_intra_mode[PLANE_TYPES];
323 FILTER_INTRA_MODE filter_intra_mode[PLANE_TYPES];
324} FILTER_INTRA_MODE_INFO;
325#endif // CONFIG_FILTER_INTRA
Yaowu Xuc27fc142016-08-22 16:08:15 -0700326
Angie Chiang9a44f5f2016-11-06 12:19:06 -0800327#if CONFIG_VAR_TX
Angie Chiang7c2b7f22016-11-07 16:00:00 -0800328#if CONFIG_RD_DEBUG
Angie Chiang9a44f5f2016-11-06 12:19:06 -0800329#define TXB_COEFF_COST_MAP_SIZE (2 * MAX_MIB_SIZE)
Angie Chiang7c2b7f22016-11-07 16:00:00 -0800330#endif
331#endif
Angie Chiang9a44f5f2016-11-06 12:19:06 -0800332
Angie Chiang9a44f5f2016-11-06 12:19:06 -0800333typedef struct RD_STATS {
334 int rate;
335 int64_t dist;
Angie Chiang7bbd3b12017-04-26 11:06:09 -0700336 // Please be careful of using rdcost, it's not guaranteed to be set all the
337 // time.
338 // TODO(angiebird): Create a set of functions to manipulate the RD_STATS. In
339 // these functions, make sure rdcost is always up-to-date according to
340 // rate/dist.
Angie Chiang2a2a7dd2017-04-25 16:08:47 -0700341 int64_t rdcost;
Angie Chiang9a44f5f2016-11-06 12:19:06 -0800342 int64_t sse;
Angie Chiang7bbd3b12017-04-26 11:06:09 -0700343 int skip; // sse should equal to dist when skip == 1
Jingning Han3bce7542017-07-25 10:53:57 -0700344 int64_t ref_rdcost;
345 int zero_rate;
Jingning Han1a7f0a82017-07-27 09:48:05 -0700346 uint8_t invalid_rate;
Yushin Chob7b60c52017-07-14 16:18:52 -0700347#if CONFIG_DIST_8X8 && CONFIG_CB4X4
Yushin Cho63927c42017-05-23 15:41:05 -0700348 int64_t dist_y;
349#endif
Angie Chiang9a44f5f2016-11-06 12:19:06 -0800350#if CONFIG_RD_DEBUG
351 int txb_coeff_cost[MAX_MB_PLANE];
Angie Chiang3963d632016-11-10 18:41:40 -0800352#if CONFIG_VAR_TX
Angie Chiang9a44f5f2016-11-06 12:19:06 -0800353 int txb_coeff_cost_map[MAX_MB_PLANE][TXB_COEFF_COST_MAP_SIZE]
354 [TXB_COEFF_COST_MAP_SIZE];
Angie Chiang3963d632016-11-10 18:41:40 -0800355#endif // CONFIG_VAR_TX
356#endif // CONFIG_RD_DEBUG
Angie Chiang9a44f5f2016-11-06 12:19:06 -0800357} RD_STATS;
Angie Chiang9a44f5f2016-11-06 12:19:06 -0800358
Sarah Parker6fdc8532016-11-16 17:47:13 -0800359#if CONFIG_EXT_INTER
Sarah Parker2d0e9b72017-05-04 01:34:16 +0000360// This struct is used to group function args that are commonly
361// sent together in functions related to interinter compound modes
Sarah Parker6fdc8532016-11-16 17:47:13 -0800362typedef struct {
Sarah Parker2d0e9b72017-05-04 01:34:16 +0000363#if CONFIG_WEDGE
Sarah Parker6fdc8532016-11-16 17:47:13 -0800364 int wedge_index;
365 int wedge_sign;
Sarah Parker2d0e9b72017-05-04 01:34:16 +0000366#endif // CONFIG_WEDGE
Sarah Parker569edda2016-12-14 14:57:38 -0800367#if CONFIG_COMPOUND_SEGMENT
Sarah Parkerb9f757c2017-01-06 17:12:24 -0800368 SEG_MASK_TYPE mask_type;
Sarah Parker2d0e9b72017-05-04 01:34:16 +0000369 uint8_t *seg_mask;
Sarah Parker569edda2016-12-14 14:57:38 -0800370#endif // CONFIG_COMPOUND_SEGMENT
Sarah Parker2d0e9b72017-05-04 01:34:16 +0000371 COMPOUND_TYPE interinter_compound_type;
Sarah Parker6fdc8532016-11-16 17:47:13 -0800372} INTERINTER_COMPOUND_DATA;
373#endif // CONFIG_EXT_INTER
374
Yaowu Xuc27fc142016-08-22 16:08:15 -0700375// This structure now relates to 8x8 block regions.
Luc Trudeauf5334002017-04-25 12:21:26 -0400376typedef struct MB_MODE_INFO {
Yaowu Xuc27fc142016-08-22 16:08:15 -0700377 // Common for both INTER and INTRA blocks
378 BLOCK_SIZE sb_type;
379 PREDICTION_MODE mode;
380 TX_SIZE tx_size;
381#if CONFIG_VAR_TX
382 // TODO(jingning): This effectively assigned a separate entry for each
383 // 8x8 block. Apparently it takes much more space than needed.
384 TX_SIZE inter_tx_size[MAX_MIB_SIZE][MAX_MIB_SIZE];
Jingning Hane67b38a2016-11-04 10:30:00 -0700385 TX_SIZE min_tx_size;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700386#endif
387 int8_t skip;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700388 int8_t segment_id;
389#if CONFIG_SUPERTX
390 // Minimum of all segment IDs under the current supertx block.
391 int8_t segment_id_supertx;
392#endif // CONFIG_SUPERTX
393 int8_t seg_id_predicted; // valid only when temporal_update is enabled
394
Sarah Parkerc5ccd4c2017-08-03 11:27:50 -0700395#if CONFIG_MRC_TX
396 int valid_mrc_mask;
397#endif // CONFIG_MRC_TX
398
Yaowu Xuc27fc142016-08-22 16:08:15 -0700399 // Only for INTRA blocks
Luc Trudeaud6d9eee2017-07-12 12:36:50 -0400400 UV_PREDICTION_MODE uv_mode;
Urvang Joshic6300aa2017-06-01 14:46:23 -0700401
Yaowu Xuc27fc142016-08-22 16:08:15 -0700402 PALETTE_MODE_INFO palette_mode_info;
Alex Converse28744302017-04-13 14:46:22 -0700403#if CONFIG_INTRABC
404 uint8_t use_intrabc;
405#endif // CONFIG_INTRABC
Yaowu Xuc27fc142016-08-22 16:08:15 -0700406
407// Only for INTER blocks
408#if CONFIG_DUAL_FILTER
James Zern7b9407a2016-05-18 23:48:05 -0700409 InterpFilter interp_filter[4];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700410#else
James Zern7b9407a2016-05-18 23:48:05 -0700411 InterpFilter interp_filter;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700412#endif
413 MV_REFERENCE_FRAME ref_frame[2];
414 TX_TYPE tx_type;
Angie Chiangcd9b03f2017-04-16 13:37:13 -0700415#if CONFIG_TXK_SEL
Angie Chiang808d8592017-04-06 18:36:55 -0700416 TX_TYPE txk_type[MAX_SB_SQUARE / (TX_SIZE_W_MIN * TX_SIZE_H_MIN)];
417#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -0700418
hui su5db97432016-10-14 16:10:14 -0700419#if CONFIG_FILTER_INTRA
420 FILTER_INTRA_MODE_INFO filter_intra_mode_info;
421#endif // CONFIG_FILTER_INTRA
Yaowu Xuc27fc142016-08-22 16:08:15 -0700422#if CONFIG_EXT_INTRA
hui sueda3d762016-12-06 16:58:23 -0800423 // The actual prediction angle is the base angle + (angle_delta * step).
Yaowu Xuc27fc142016-08-22 16:08:15 -0700424 int8_t angle_delta[2];
hui sueda3d762016-12-06 16:58:23 -0800425#if CONFIG_INTRA_INTERP
Yaowu Xuc27fc142016-08-22 16:08:15 -0700426 // To-Do (huisu): this may be replaced by interp_filter
427 INTRA_FILTER intra_filter;
hui sueda3d762016-12-06 16:58:23 -0800428#endif // CONFIG_INTRA_INTERP
Yaowu Xuc27fc142016-08-22 16:08:15 -0700429#endif // CONFIG_EXT_INTRA
430
431#if CONFIG_EXT_INTER
Yue Chen4d26acb2017-05-01 12:28:34 -0700432#if CONFIG_INTERINTRA
Sarah Parker2d0e9b72017-05-04 01:34:16 +0000433 // interintra members
Yaowu Xuc27fc142016-08-22 16:08:15 -0700434 INTERINTRA_MODE interintra_mode;
Yue Chen4d26acb2017-05-01 12:28:34 -0700435#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -0700436 // TODO(debargha): Consolidate these flags
437 int use_wedge_interintra;
438 int interintra_wedge_index;
439 int interintra_wedge_sign;
Sarah Parker2d0e9b72017-05-04 01:34:16 +0000440 // interinter members
441 COMPOUND_TYPE interinter_compound_type;
442#if CONFIG_WEDGE
443 int wedge_index;
444 int wedge_sign;
445#endif // CONFIG_WEDGE
446#if CONFIG_COMPOUND_SEGMENT
447 SEG_MASK_TYPE mask_type;
448#endif // CONFIG_COMPOUND_SEGMENT
Yaowu Xuc27fc142016-08-22 16:08:15 -0700449#endif // CONFIG_EXT_INTER
Yue Chencb60b182016-10-13 15:18:22 -0700450 MOTION_MODE motion_mode;
Yue Chen5329a2b2017-02-28 17:33:00 +0800451#if CONFIG_MOTION_VAR
452 int overlappable_neighbors[2];
Wei-Ting Lin85a8f702017-06-22 13:55:15 -0700453#if CONFIG_NCOBMC_ADAPT_WEIGHT
454 // Applying different weighting kernels in ncobmc
455 // In current implementation, interpolation modes only defined for squared
456 // blocks. A rectangular block is divided into two squared blocks and each
457 // squared block has an interpolation mode.
458 NCOBMC_MODE ncobmc_mode[2];
Wei-Ting Lin482551b2017-08-03 12:29:24 -0700459#endif // CONFIG_NCOBMC_ADAPT_WEIGHT
Yue Chen5329a2b2017-02-28 17:33:00 +0800460#endif // CONFIG_MOTION_VAR
Yaowu Xuc27fc142016-08-22 16:08:15 -0700461 int_mv mv[2];
462 int_mv pred_mv[2];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700463 uint8_t ref_mv_idx;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700464#if CONFIG_EXT_PARTITION_TYPES
465 PARTITION_TYPE partition;
466#endif
467#if CONFIG_NEW_QUANT
468 int dq_off_index;
469 int send_dq_bit;
470#endif // CONFIG_NEW_QUANT
471 /* deringing gain *per-superblock* */
Jean-Marc Valin5f5c1322017-03-21 16:20:21 -0400472 int8_t cdef_strength;
Arild Fuldseth07441162016-08-15 15:07:52 +0200473#if CONFIG_DELTA_Q
474 int current_q_index;
Fangwen Fu231fe422017-04-24 17:52:29 -0700475#if CONFIG_EXT_DELTA_Q
476 int current_delta_lf_from_base;
477#endif
Arild Fuldseth07441162016-08-15 15:07:52 +0200478#endif
Angie Chiangd4022822016-11-02 18:30:25 -0700479#if CONFIG_RD_DEBUG
Angie Chiang9a44f5f2016-11-06 12:19:06 -0800480 RD_STATS rd_stats;
Angie Chiangd4022822016-11-02 18:30:25 -0700481 int mi_row;
482 int mi_col;
483#endif
Yue Chen69f18e12016-09-08 14:48:15 -0700484#if CONFIG_WARPED_MOTION
485 int num_proj_ref[2];
486 WarpedMotionParams wm_params[2];
487#endif // CONFIG_WARPED_MOTION
Ryan Lei9b02b0e2017-01-30 15:52:20 -0800488
Luc Trudeauf5334002017-04-25 12:21:26 -0400489#if CONFIG_CFL
490 // Index of the alpha Cb and alpha Cr combination
Luc Trudeaua9bd85f2017-05-11 14:37:56 -0400491 int cfl_alpha_idx;
David Michael Barrf6eaa152017-07-19 19:42:28 +0900492 // Joint sign of alpha Cb and alpha Cr
493 int cfl_alpha_signs;
Luc Trudeauf5334002017-04-25 12:21:26 -0400494#endif
495
Ryan Lei9b02b0e2017-01-30 15:52:20 -0800496 BOUNDARY_TYPE boundary_info;
Cheng Chenf572cd32017-08-25 18:34:51 -0700497#if CONFIG_LPF_SB
498 uint8_t filt_lvl;
499#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -0700500} MB_MODE_INFO;
501
502typedef struct MODE_INFO {
503 MB_MODE_INFO mbmi;
504 b_mode_info bmi[4];
505} MODE_INFO;
506
Alex Converse28744302017-04-13 14:46:22 -0700507#if CONFIG_INTRABC
508static INLINE int is_intrabc_block(const MB_MODE_INFO *mbmi) {
509 return mbmi->use_intrabc;
510}
511#endif
512
Yaowu Xuc27fc142016-08-22 16:08:15 -0700513static INLINE PREDICTION_MODE get_y_mode(const MODE_INFO *mi, int block) {
Jingning Hand7d20472016-12-14 11:13:48 -0800514#if CONFIG_CB4X4
515 (void)block;
516 return mi->mbmi.mode;
517#else
Yaowu Xuc27fc142016-08-22 16:08:15 -0700518 return mi->mbmi.sb_type < BLOCK_8X8 ? mi->bmi[block].as_mode : mi->mbmi.mode;
Jingning Hand7d20472016-12-14 11:13:48 -0800519#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -0700520}
521
Luc Trudeaud6d9eee2017-07-12 12:36:50 -0400522#if CONFIG_CFL
523static INLINE PREDICTION_MODE get_uv_mode(UV_PREDICTION_MODE mode) {
524 static const PREDICTION_MODE uv2y[UV_INTRA_MODES] = {
Urvang Joshi93b543a2017-06-01 17:32:41 -0700525 DC_PRED, // UV_DC_PRED
526 V_PRED, // UV_V_PRED
527 H_PRED, // UV_H_PRED
528 D45_PRED, // UV_D45_PRED
529 D135_PRED, // UV_D135_PRED
530 D117_PRED, // UV_D117_PRED
531 D153_PRED, // UV_D153_PRED
532 D207_PRED, // UV_D207_PRED
533 D63_PRED, // UV_D63_PRED
Luc Trudeaud6d9eee2017-07-12 12:36:50 -0400534 SMOOTH_PRED, // UV_SMOOTH_PRED
535#if CONFIG_SMOOTH_HV
536 SMOOTH_V_PRED, // UV_SMOOTH_V_PRED
537 SMOOTH_H_PRED, // UV_SMOOTH_H_PRED
538#endif // CONFIG_SMOOTH_HV
Luc Trudeaud6d9eee2017-07-12 12:36:50 -0400539 TM_PRED, // UV_TM_PRED
Luc Trudeau6e1cd782017-06-21 13:52:36 -0400540 DC_PRED, // CFL_PRED
Luc Trudeaud6d9eee2017-07-12 12:36:50 -0400541 };
542 return uv2y[mode];
543}
544#else
545static INLINE PREDICTION_MODE get_uv_mode(PREDICTION_MODE mode) { return mode; }
546#endif // CONFIG_CFL
547
Yaowu Xuc27fc142016-08-22 16:08:15 -0700548static INLINE int is_inter_block(const MB_MODE_INFO *mbmi) {
Alex Converse28744302017-04-13 14:46:22 -0700549#if CONFIG_INTRABC
550 if (is_intrabc_block(mbmi)) return 1;
551#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -0700552 return mbmi->ref_frame[0] > INTRA_FRAME;
553}
554
555static INLINE int has_second_ref(const MB_MODE_INFO *mbmi) {
556 return mbmi->ref_frame[1] > INTRA_FRAME;
557}
558
Zoe Liuc082bbc2017-05-17 13:31:37 -0700559#if CONFIG_EXT_COMP_REFS
560static INLINE int has_uni_comp_refs(const MB_MODE_INFO *mbmi) {
561 return has_second_ref(mbmi) && (!((mbmi->ref_frame[0] >= BWDREF_FRAME) ^
562 (mbmi->ref_frame[1] >= BWDREF_FRAME)));
563}
564
565static INLINE MV_REFERENCE_FRAME comp_ref0(int ref_idx) {
566 static const MV_REFERENCE_FRAME lut[] = {
567 LAST_FRAME, // LAST_LAST2_FRAMES,
Zoe Liufcf5fa22017-06-26 16:00:38 -0700568 LAST_FRAME, // LAST_LAST3_FRAMES,
Zoe Liuc082bbc2017-05-17 13:31:37 -0700569 LAST_FRAME, // LAST_GOLDEN_FRAMES,
570 BWDREF_FRAME, // BWDREF_ALTREF_FRAMES,
571 };
572 assert(NELEMENTS(lut) == UNIDIR_COMP_REFS);
573 return lut[ref_idx];
574}
575
576static INLINE MV_REFERENCE_FRAME comp_ref1(int ref_idx) {
577 static const MV_REFERENCE_FRAME lut[] = {
578 LAST2_FRAME, // LAST_LAST2_FRAMES,
Zoe Liufcf5fa22017-06-26 16:00:38 -0700579 LAST3_FRAME, // LAST_LAST3_FRAMES,
Zoe Liuc082bbc2017-05-17 13:31:37 -0700580 GOLDEN_FRAME, // LAST_GOLDEN_FRAMES,
581 ALTREF_FRAME, // BWDREF_ALTREF_FRAMES,
582 };
583 assert(NELEMENTS(lut) == UNIDIR_COMP_REFS);
584 return lut[ref_idx];
585}
586#endif // CONFIG_EXT_COMP_REFS
587
Yaowu Xuf883b422016-08-30 14:01:10 -0700588PREDICTION_MODE av1_left_block_mode(const MODE_INFO *cur_mi,
589 const MODE_INFO *left_mi, int b);
Yaowu Xuc27fc142016-08-22 16:08:15 -0700590
Yaowu Xuf883b422016-08-30 14:01:10 -0700591PREDICTION_MODE av1_above_block_mode(const MODE_INFO *cur_mi,
592 const MODE_INFO *above_mi, int b);
Yaowu Xuc27fc142016-08-22 16:08:15 -0700593
Sarah Parker19234cc2017-03-10 16:43:25 -0800594#if CONFIG_GLOBAL_MOTION
595static INLINE int is_global_mv_block(const MODE_INFO *mi, int block,
596 TransformationType type) {
597 PREDICTION_MODE mode = get_y_mode(mi, block);
598#if GLOBAL_SUB8X8_USED
599 const int block_size_allowed = 1;
600#else
601 const BLOCK_SIZE bsize = mi->mbmi.sb_type;
Debargha Mukherjeec17a4432017-08-28 22:22:45 -0700602 const int block_size_allowed =
603 AOMMIN(block_size_wide[bsize], block_size_high[bsize]) >= 8;
Sarah Parker19234cc2017-03-10 16:43:25 -0800604#endif // GLOBAL_SUB8X8_USED
605#if CONFIG_EXT_INTER
606 return (mode == ZEROMV || mode == ZERO_ZEROMV) && type > TRANSLATION &&
607 block_size_allowed;
608#else
609 return mode == ZEROMV && type > TRANSLATION && block_size_allowed;
610#endif // CONFIG_EXT_INTER
611}
612#endif // CONFIG_GLOBAL_MOTION
613
Yaowu Xuc27fc142016-08-22 16:08:15 -0700614enum mv_precision { MV_PRECISION_Q3, MV_PRECISION_Q4 };
615
616struct buf_2d {
617 uint8_t *buf;
618 uint8_t *buf0;
619 int width;
620 int height;
621 int stride;
622};
623
624typedef struct macroblockd_plane {
625 tran_low_t *dqcoeff;
626 PLANE_TYPE plane_type;
627 int subsampling_x;
628 int subsampling_y;
629 struct buf_2d dst;
630 struct buf_2d pre[2];
631 ENTROPY_CONTEXT *above_context;
632 ENTROPY_CONTEXT *left_context;
633 int16_t seg_dequant[MAX_SEGMENTS][2];
634#if CONFIG_NEW_QUANT
clang-format67948d32016-09-07 22:40:40 -0700635 dequant_val_type_nuq seg_dequant_nuq[MAX_SEGMENTS][QUANT_PROFILES]
636 [COEF_BANDS];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700637#endif
638 uint8_t *color_index_map;
639
640 // number of 4x4s in current block
641 uint16_t n4_w, n4_h;
642 // log2 of n4_w, n4_h
643 uint8_t n4_wl, n4_hl;
Jingning Hanc47fe6c2016-10-21 16:40:47 -0700644 // block size in pixels
645 uint8_t width, height;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700646
647#if CONFIG_AOM_QM
Sarah Parker172f3c12017-09-08 14:23:36 -0700648 qm_val_t *seg_iqmatrix[MAX_SEGMENTS][2][TX_SIZES_ALL];
649 qm_val_t *seg_qmatrix[MAX_SEGMENTS][2][TX_SIZES_ALL];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700650#endif
651 // encoder
652 const int16_t *dequant;
653#if CONFIG_NEW_QUANT
654 const dequant_val_type_nuq *dequant_val_nuq[QUANT_PROFILES];
655#endif // CONFIG_NEW_QUANT
Yushin Cho77bba8d2016-11-04 16:36:56 -0700656
Yushin Chob7b60c52017-07-14 16:18:52 -0700657#if CONFIG_PVQ || CONFIG_DIST_8X8
Yushin Cho77bba8d2016-11-04 16:36:56 -0700658 DECLARE_ALIGNED(16, int16_t, pred[MAX_SB_SQUARE]);
Yushin Chob7b60c52017-07-14 16:18:52 -0700659#endif
660#if CONFIG_PVQ
Yushin Cho77bba8d2016-11-04 16:36:56 -0700661 // PVQ: forward transformed predicted image, a reference for PVQ.
662 tran_low_t *pvq_ref_coeff;
663#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -0700664} MACROBLOCKD_PLANE;
665
Jingning Han3468df12016-12-05 17:53:16 -0800666#define BLOCK_OFFSET(x, i) \
667 ((x) + (i) * (1 << (tx_size_wide_log2[0] + tx_size_high_log2[0])))
Yaowu Xuc27fc142016-08-22 16:08:15 -0700668
669typedef struct RefBuffer {
Yaowu Xuc27fc142016-08-22 16:08:15 -0700670 int idx;
671 YV12_BUFFER_CONFIG *buf;
672 struct scale_factors sf;
Zoe Liu7b1ec7a2017-05-24 22:28:24 -0700673#if CONFIG_VAR_REFS
Zoe Liu7b1ec7a2017-05-24 22:28:24 -0700674 int is_valid;
675#endif // CONFIG_VAR_REFS
Yaowu Xuc27fc142016-08-22 16:08:15 -0700676} RefBuffer;
677
Yi Luo2ab63cb2017-05-11 16:44:22 -0700678#if CONFIG_ADAPT_SCAN
679typedef int16_t EobThresholdMD[TX_TYPES][EOB_THRESHOLD_NUM];
680#endif
Luc Trudeauf8164152017-04-11 16:20:51 -0400681
Rupert Swarbrick6c545212017-09-01 17:17:25 +0100682#if CONFIG_LOOP_RESTORATION
683typedef struct {
684 DECLARE_ALIGNED(16, InterpKernel, vfilter);
685 DECLARE_ALIGNED(16, InterpKernel, hfilter);
686} WienerInfo;
687
688typedef struct {
689 int ep;
690 int xqd[2];
691} SgrprojInfo;
692#endif // CONFIG_LOOP_RESTORATION
693
David Michael Barr5b2021e2017-08-17 18:12:39 +0900694#if CONFIG_CFL
Luc Trudeau03b7e7d2017-09-19 13:20:52 -0400695#if CONFIG_CHROMA_SUB8X8 && CONFIG_DEBUG
696#define CFL_SUB8X8_VAL_MI_SIZE (4)
697#define CFL_SUB8X8_VAL_MI_SQUARE \
698 (CFL_SUB8X8_VAL_MI_SIZE * CFL_SUB8X8_VAL_MI_SIZE)
699#endif // CONFIG_CHROMA_SUB8X8 && CONFIG_DEBUG
David Michael Barr5b2021e2017-08-17 18:12:39 +0900700typedef struct cfl_ctx {
701 // Pixel buffer containing the luma pixels used as prediction for chroma
702 // TODO(ltrudeau) Convert to uint16 for HBD support
703 uint8_t y_pix[MAX_SB_SQUARE];
704
705 // Pixel buffer containing the downsampled luma pixels used as prediction for
706 // chroma
707 // TODO(ltrudeau) Convert to uint16 for HBD support
708 uint8_t y_down_pix[MAX_SB_SQUARE];
709
710 // Height and width of the luma prediction block currently in the pixel buffer
711 int y_height, y_width;
712
713 // Height and width of the chroma prediction block currently associated with
714 // this context
715 int uv_height, uv_width;
716
717 // Transform level averages of the luma reconstructed values over the entire
718 // prediction unit
719 // Fixed point y_averages is Q12.3:
720 // * Worst case division is 1/1024
721 // * Max error will be 1/16th.
722 // Note: 3 is chosen so that y_averages fits in 15 bits when 12 bit input is
723 // used
Luc Trudeauf1cf7172017-09-19 13:32:05 -0400724 int y_averages_q3[MAX_NUM_TXB_SQUARE];
David Michael Barr5b2021e2017-08-17 18:12:39 +0900725 int y_averages_stride;
726
727 int are_parameters_computed;
728
729 // Chroma subsampling
730 int subsampling_x, subsampling_y;
731
732 // Block level DC_PRED for each chromatic plane
733 int dc_pred[CFL_PRED_PLANES];
734
735 int mi_row, mi_col;
736
737 // Whether the reconstructed luma pixels need to be stored
738 int store_y;
739
740#if CONFIG_CB4X4
741 int is_chroma_reference;
742#if CONFIG_CHROMA_SUB8X8 && CONFIG_DEBUG
743 // The prediction used for sub8x8 blocks originates from multiple luma blocks,
744 // this array is used to validate that cfl_store() is called only once for
745 // each luma block
Luc Trudeau03b7e7d2017-09-19 13:20:52 -0400746 uint8_t sub8x8_val[CFL_SUB8X8_VAL_MI_SQUARE];
David Michael Barr5b2021e2017-08-17 18:12:39 +0900747#endif // CONFIG_CHROMA_SUB8X8 && CONFIG_DEBUG
748#endif // CONFIG_CB4X4
749} CFL_CTX;
750#endif // CONFIG_CFL
751
Yaowu Xuc27fc142016-08-22 16:08:15 -0700752typedef struct macroblockd {
753 struct macroblockd_plane plane[MAX_MB_PLANE];
754 uint8_t bmode_blocks_wl;
755 uint8_t bmode_blocks_hl;
756
757 FRAME_COUNTS *counts;
758 TileInfo tile;
759
760 int mi_stride;
761
762 MODE_INFO **mi;
763 MODE_INFO *left_mi;
764 MODE_INFO *above_mi;
765 MB_MODE_INFO *left_mbmi;
766 MB_MODE_INFO *above_mbmi;
767
768 int up_available;
769 int left_available;
Jingning Han3da18d62017-05-02 12:43:58 -0700770#if CONFIG_CHROMA_SUB8X8
771 int chroma_up_available;
772 int chroma_left_available;
773#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -0700774
Yaowu Xuf883b422016-08-30 14:01:10 -0700775 const aom_prob (*partition_probs)[PARTITION_TYPES - 1];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700776
Rupert Swarbrick57486c52017-08-14 10:53:49 +0100777 /* Distance of MB away from frame edges in subpixels (1/8th pixel) */
Yaowu Xuc27fc142016-08-22 16:08:15 -0700778 int mb_to_left_edge;
779 int mb_to_right_edge;
780 int mb_to_top_edge;
781 int mb_to_bottom_edge;
782
783 FRAME_CONTEXT *fc;
784
785 /* pointers to reference frames */
Urvang Joshi52648442016-10-13 17:27:51 -0700786 const RefBuffer *block_refs[2];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700787
788 /* pointer to current frame */
789 const YV12_BUFFER_CONFIG *cur_buf;
790
Alex Conversee816b312017-05-01 09:51:24 -0700791#if CONFIG_INTRABC
792 /* Scale of the current frame with respect to itself */
793 struct scale_factors sf_identity;
794#endif
795
Yaowu Xuc27fc142016-08-22 16:08:15 -0700796 ENTROPY_CONTEXT *above_context[MAX_MB_PLANE];
797 ENTROPY_CONTEXT left_context[MAX_MB_PLANE][2 * MAX_MIB_SIZE];
798
799 PARTITION_CONTEXT *above_seg_context;
800 PARTITION_CONTEXT left_seg_context[MAX_MIB_SIZE];
801
802#if CONFIG_VAR_TX
803 TXFM_CONTEXT *above_txfm_context;
804 TXFM_CONTEXT *left_txfm_context;
Jingning Han331662e2017-05-30 17:03:32 -0700805 TXFM_CONTEXT left_txfm_context_buffer[2 * MAX_MIB_SIZE];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700806
807 TX_SIZE max_tx_size;
808#if CONFIG_SUPERTX
809 TX_SIZE supertx_size;
810#endif
811#endif
812
Rupert Swarbrick6c545212017-09-01 17:17:25 +0100813#if CONFIG_LOOP_RESTORATION
814 WienerInfo wiener_info[MAX_MB_PLANE];
815 SgrprojInfo sgrproj_info[MAX_MB_PLANE];
816#endif // CONFIG_LOOP_RESTORATION
817
Jingning Hanff6ee6a2016-12-07 09:55:21 -0800818 // block dimension in the unit of mode_info.
Yaowu Xuc27fc142016-08-22 16:08:15 -0700819 uint8_t n8_w, n8_h;
820
Yaowu Xuc27fc142016-08-22 16:08:15 -0700821 uint8_t ref_mv_count[MODE_CTX_REF_FRAMES];
822 CANDIDATE_MV ref_mv_stack[MODE_CTX_REF_FRAMES][MAX_REF_MV_STACK_SIZE];
823 uint8_t is_sec_rect;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700824
Yushin Cho77bba8d2016-11-04 16:36:56 -0700825#if CONFIG_PVQ
826 daala_dec_ctx daala_dec;
827#endif
Thomas Daviesf77d4ad2017-01-10 18:55:42 +0000828 FRAME_CONTEXT *tile_ctx;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700829 /* Bit depth: 8, 10, 12 */
830 int bd;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700831
Debargha Mukherjee3c42c092016-09-29 09:17:36 -0700832 int qindex[MAX_SEGMENTS];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700833 int lossless[MAX_SEGMENTS];
834 int corrupted;
RogerZhou3b635242017-09-19 10:06:46 -0700835#if CONFIG_AMVR
836 int cur_frame_mv_precision_level;
837// same with that in AV1_COMMON
838#endif
Yaowu Xuf883b422016-08-30 14:01:10 -0700839 struct aom_internal_error_info *error_info;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700840#if CONFIG_GLOBAL_MOTION
David Barkercf3d0b02016-11-10 10:14:49 +0000841 WarpedMotionParams *global_motion;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700842#endif // CONFIG_GLOBAL_MOTION
Arild Fuldseth07441162016-08-15 15:07:52 +0200843#if CONFIG_DELTA_Q
844 int prev_qindex;
845 int delta_qindex;
846 int current_qindex;
Fangwen Fu231fe422017-04-24 17:52:29 -0700847#if CONFIG_EXT_DELTA_Q
848 // Since actual frame level loop filtering level value is not available
849 // at the beginning of the tile (only available during actual filtering)
850 // at encoder side.we record the delta_lf (against the frame level loop
851 // filtering level) and code the delta between previous superblock's delta
852 // lf and current delta lf. It is equivalent to the delta between previous
853 // superblock's actual lf and current lf.
854 int prev_delta_lf_from_base;
855 int current_delta_lf_from_base;
856#endif
Arild Fuldseth07441162016-08-15 15:07:52 +0200857#endif
Yi Luof8e87b42017-04-14 17:20:27 -0700858#if CONFIG_ADAPT_SCAN
859 const EobThresholdMD *eob_threshold_md;
860#endif
Luc Trudeauf8164152017-04-11 16:20:51 -0400861
Sarah Parker2d0e9b72017-05-04 01:34:16 +0000862#if CONFIG_EXT_INTER && CONFIG_COMPOUND_SEGMENT
863 DECLARE_ALIGNED(16, uint8_t, seg_mask[2 * MAX_SB_SQUARE]);
864#endif // CONFIG_EXT_INTER && CONFIG_COMPOUND_SEGMENT
865
Sarah Parker5c6744b2017-08-25 17:27:45 -0700866#if CONFIG_MRC_TX
867 uint8_t *mrc_mask;
868#endif // CONFIG_MRC_TX
869
Luc Trudeauf8164152017-04-11 16:20:51 -0400870#if CONFIG_CFL
871 CFL_CTX *cfl;
872#endif
Wei-Ting Lin482551b2017-08-03 12:29:24 -0700873
874#if CONFIG_NCOBMC_ADAPT_WEIGHT
875 uint8_t *ncobmc_pred_buf[MAX_MB_PLANE];
876 int ncobmc_pred_buf_stride[MAX_MB_PLANE];
877 SB_MI_BD sb_mi_bd;
878#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -0700879} MACROBLOCKD;
880
Yi Luo51281092017-06-26 16:36:15 -0700881static INLINE int get_bitdepth_data_path_index(const MACROBLOCKD *xd) {
882 return xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH ? 1 : 0;
883}
884
Yaowu Xuc27fc142016-08-22 16:08:15 -0700885static INLINE BLOCK_SIZE get_subsize(BLOCK_SIZE bsize,
886 PARTITION_TYPE partition) {
887 if (partition == PARTITION_INVALID)
Urvang Joshicb586f32016-09-20 11:36:33 -0700888 return BLOCK_INVALID;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700889 else
890 return subsize_lookup[partition][bsize];
891}
892
893static const TX_TYPE intra_mode_to_tx_type_context[INTRA_MODES] = {
894 DCT_DCT, // DC
895 ADST_DCT, // V
896 DCT_ADST, // H
897 DCT_DCT, // D45
898 ADST_ADST, // D135
899 ADST_DCT, // D117
900 DCT_ADST, // D153
901 DCT_ADST, // D207
902 ADST_DCT, // D63
Urvang Joshi6be4a542016-11-03 15:24:05 -0700903 ADST_ADST, // SMOOTH
Urvang Joshie6ca8e82017-03-15 14:57:41 -0700904#if CONFIG_SMOOTH_HV
905 ADST_DCT, // SMOOTH_V
906 DCT_ADST, // SMOOTH_H
907#endif // CONFIG_SMOOTH_HV
Yaowu Xuc27fc142016-08-22 16:08:15 -0700908 ADST_ADST, // TM
909};
910
911#if CONFIG_SUPERTX
912static INLINE int supertx_enabled(const MB_MODE_INFO *mbmi) {
Jingning Han93531242016-12-20 11:54:36 -0800913 TX_SIZE max_tx_size = txsize_sqr_map[mbmi->tx_size];
914 return tx_size_wide[max_tx_size] >
915 AOMMIN(block_size_wide[mbmi->sb_type], block_size_high[mbmi->sb_type]);
Yaowu Xuc27fc142016-08-22 16:08:15 -0700916}
917#endif // CONFIG_SUPERTX
918
Jingning Hanb83e64b2017-03-01 14:52:04 -0800919#define USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4 1
Debargha Mukherjee5a488a62016-11-22 22:24:10 -0800920
Sarah Parker076437f2017-03-14 17:39:53 -0700921#if CONFIG_RECT_TX
Sarah Parker076437f2017-03-14 17:39:53 -0700922static INLINE int is_rect_tx(TX_SIZE tx_size) { return tx_size >= TX_SIZES; }
923#endif // CONFIG_RECT_TX
924
Sarah Parker2e08d962017-08-01 19:51:20 -0700925#if CONFIG_MRC_TX
926#define USE_MRC_INTRA 0
927#define USE_MRC_INTER 1
Sarah Parker99e7daa2017-08-29 10:30:13 -0700928#define SIGNAL_MRC_MASK_INTRA (USE_MRC_INTRA && 0)
929#define SIGNAL_MRC_MASK_INTER (USE_MRC_INTER && 1)
930#define SIGNAL_ANY_MRC_MASK (SIGNAL_MRC_MASK_INTRA || SIGNAL_MRC_MASK_INTER)
Sarah Parker2e08d962017-08-01 19:51:20 -0700931#endif // CONFIG_MRC_TX
932
Jingning Hanb83e64b2017-03-01 14:52:04 -0800933#if CONFIG_EXT_TX
Yaowu Xuc27fc142016-08-22 16:08:15 -0700934#define ALLOW_INTRA_EXT_TX 1
Yaowu Xuc27fc142016-08-22 16:08:15 -0700935
Hui Suddbcde22017-09-18 17:22:02 -0700936// Number of transform types in each set type
937static const int av1_num_ext_tx_set[EXT_TX_SET_TYPES] = {
938 1, 2,
Sarah Parker53f93db2017-07-11 17:20:04 -0700939#if CONFIG_MRC_TX
Hui Suddbcde22017-09-18 17:22:02 -0700940 2, 3,
941#endif // CONFIG_MRC_TX
942 5, 7, 12, 16,
943};
944
Hui Su4a9be2a2017-09-19 14:41:49 -0700945static const int av1_ext_tx_set_idx_to_type[2][AOMMAX(EXT_TX_SETS_INTRA,
946 EXT_TX_SETS_INTER)] = {
947 {
948 // Intra
949 EXT_TX_SET_DCTONLY, EXT_TX_SET_DTT4_IDTX_1DDCT, EXT_TX_SET_DTT4_IDTX,
Hui Suddbcde22017-09-18 17:22:02 -0700950#if CONFIG_MRC_TX
Hui Su4a9be2a2017-09-19 14:41:49 -0700951 EXT_TX_SET_MRC_DCT,
Hui Suddbcde22017-09-18 17:22:02 -0700952#endif // CONFIG_MRC_TX
Hui Su4a9be2a2017-09-19 14:41:49 -0700953 },
954 {
955 // Inter
956 EXT_TX_SET_DCTONLY, EXT_TX_SET_ALL16, EXT_TX_SET_DTT9_IDTX_1DDCT,
957 EXT_TX_SET_DCT_IDTX,
Hui Suddbcde22017-09-18 17:22:02 -0700958#if CONFIG_MRC_TX
Hui Su4a9be2a2017-09-19 14:41:49 -0700959 EXT_TX_SET_MRC_DCT_IDTX,
Sarah Parker53f93db2017-07-11 17:20:04 -0700960#endif // CONFIG_MRC_TX
Hui Su4a9be2a2017-09-19 14:41:49 -0700961 }
Hui Suddbcde22017-09-18 17:22:02 -0700962};
963
964static const int use_intra_ext_tx_for_txsize[EXT_TX_SETS_INTRA][EXT_TX_SIZES] =
965 {
966#if CONFIG_CHROMA_2X2
967 { 1, 1, 1, 1, 1 }, // unused
968 { 0, 1, 1, 0, 0 },
969 { 0, 0, 0, 1, 0 },
970#if CONFIG_MRC_TX
971 { 0, 0, 0, 0, 1 },
972#endif // CONFIG_MRC_TX
973#else // CONFIG_CHROMA_2X2
974 { 1, 1, 1, 1 }, // unused
975 { 1, 1, 0, 0 },
976 { 0, 0, 1, 0 },
977#if CONFIG_MRC_TX
978 { 0, 0, 0, 1 },
979#endif // CONFIG_MRC_TX
980#endif // CONFIG_CHROMA_2X2
981 };
982
983static const int use_inter_ext_tx_for_txsize[EXT_TX_SETS_INTER][EXT_TX_SIZES] =
984 {
985#if CONFIG_CHROMA_2X2
986 { 1, 1, 1, 1, 1 }, // unused
987 { 0, 1, 1, 0, 0 }, { 0, 0, 0, 1, 0 }, { 0, 0, 0, 0, 1 },
988#if CONFIG_MRC_TX
989 { 0, 0, 0, 0, 1 },
990#endif // CONFIG_MRC_TX
991#else // CONFIG_CHROMA_2X2
992 { 1, 1, 1, 1 }, // unused
993 { 1, 1, 0, 0 }, { 0, 0, 1, 0 }, { 0, 0, 0, 1 },
994#if CONFIG_MRC_TX
995 { 0, 0, 0, 1 },
996#endif // CONFIG_MRC_TX
997#endif // CONFIG_CHROMA_2X2
998 };
999
1000// 1D Transforms used in inter set, this needs to be changed if
1001// ext_tx_used_inter is changed
1002static const int ext_tx_used_inter_1D[EXT_TX_SETS_INTER][TX_TYPES_1D] = {
1003 { 1, 0, 0, 0 }, { 1, 1, 1, 1 }, { 1, 1, 1, 1 }, { 1, 0, 0, 1 },
1004#if CONFIG_MRC_TX
1005 { 1, 0, 0, 1 },
1006#endif // CONFIG_MRC_TX
1007};
1008
1009#if CONFIG_MRC_TX
1010static const int av1_ext_tx_used[EXT_TX_SET_TYPES][TX_TYPES] = {
1011 {
1012 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1013 },
1014 {
1015 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0,
1016 },
1017 {
1018 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1,
1019 },
1020 {
1021 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1,
1022 },
1023 {
1024 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0,
1025 },
1026 {
1027 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0,
1028 },
1029 {
1030 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,
1031 },
1032 {
1033 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0,
1034 },
1035};
1036#else // CONFIG_MRC_TX
1037static const int av1_ext_tx_used[EXT_TX_SET_TYPES][TX_TYPES] = {
1038 {
1039 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1040 },
1041 {
1042 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0,
1043 },
1044 {
1045 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0,
1046 },
1047 {
1048 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0,
1049 },
1050 {
1051 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0,
1052 },
1053 {
1054 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1055 },
Debargha Mukherjee08542b92017-02-21 01:08:14 -08001056};
Sarah Parker53f93db2017-07-11 17:20:04 -07001057#endif // CONFIG_MRC_TX
Debargha Mukherjee08542b92017-02-21 01:08:14 -08001058
1059static INLINE TxSetType get_ext_tx_set_type(TX_SIZE tx_size, BLOCK_SIZE bs,
Sarah Parker5effe3f2017-02-23 12:49:10 -08001060 int is_inter, int use_reduced_set) {
Yue Chen56e226e2017-05-02 16:21:40 -07001061 const TX_SIZE tx_size_sqr_up = txsize_sqr_up_map[tx_size];
1062 const TX_SIZE tx_size_sqr = txsize_sqr_map[tx_size];
Debargha Mukherjee094c9432017-02-22 10:31:25 -08001063#if CONFIG_CB4X4 && USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4
Jingning Han1a00cff2016-12-28 14:53:14 -08001064 (void)bs;
Yue Chen56e226e2017-05-02 16:21:40 -07001065 if (tx_size_sqr > TX_32X32) return EXT_TX_SET_DCTONLY;
Jingning Han1a00cff2016-12-28 14:53:14 -08001066#else
Yue Chen56e226e2017-05-02 16:21:40 -07001067 if (tx_size_sqr > TX_32X32 || bs < BLOCK_8X8) return EXT_TX_SET_DCTONLY;
Jingning Han1a00cff2016-12-28 14:53:14 -08001068#endif
Sarah Parker5effe3f2017-02-23 12:49:10 -08001069 if (use_reduced_set)
1070 return is_inter ? EXT_TX_SET_DCT_IDTX : EXT_TX_SET_DTT4_IDTX;
Sarah Parker53f93db2017-07-11 17:20:04 -07001071#if CONFIG_MRC_TX
Sarah Parker2e08d962017-08-01 19:51:20 -07001072 if (tx_size == TX_32X32) {
1073 if (is_inter && USE_MRC_INTER)
1074 return EXT_TX_SET_MRC_DCT_IDTX;
1075 else if (!is_inter && USE_MRC_INTRA)
1076 return EXT_TX_SET_MRC_DCT;
1077 }
Sarah Parker53f93db2017-07-11 17:20:04 -07001078#endif // CONFIG_MRC_TX
Yue Chen56e226e2017-05-02 16:21:40 -07001079 if (tx_size_sqr_up == TX_32X32)
Debargha Mukherjee08542b92017-02-21 01:08:14 -08001080 return is_inter ? EXT_TX_SET_DCT_IDTX : EXT_TX_SET_DCTONLY;
1081 if (is_inter)
Yue Chen56e226e2017-05-02 16:21:40 -07001082 return (tx_size_sqr == TX_16X16 ? EXT_TX_SET_DTT9_IDTX_1DDCT
1083 : EXT_TX_SET_ALL16);
Debargha Mukherjee08542b92017-02-21 01:08:14 -08001084 else
Yue Chen56e226e2017-05-02 16:21:40 -07001085 return (tx_size_sqr == TX_16X16 ? EXT_TX_SET_DTT4_IDTX
1086 : EXT_TX_SET_DTT4_IDTX_1DDCT);
Debargha Mukherjee08542b92017-02-21 01:08:14 -08001087}
1088
Hui Su4a9be2a2017-09-19 14:41:49 -07001089// Maps tx set types to the indices.
1090static const int ext_tx_set_index[2][EXT_TX_SET_TYPES] = {
1091 {
1092 // Intra
1093 0, -1,
1094#if CONFIG_MRC_TX
1095 3, -1,
1096#endif // CONFIG_MRC_TX
1097 2, 1, -1, -1,
1098 },
1099 {
1100 // Inter
1101 0, 3,
1102#if CONFIG_MRC_TX
1103 -1, 4,
1104#endif // CONFIG_MRC_TX
1105 -1, -1, 2, 1,
1106 },
1107};
1108
Debargha Mukherjee08542b92017-02-21 01:08:14 -08001109static INLINE int get_ext_tx_set(TX_SIZE tx_size, BLOCK_SIZE bs, int is_inter,
Sarah Parker5effe3f2017-02-23 12:49:10 -08001110 int use_reduced_set) {
Debargha Mukherjee08542b92017-02-21 01:08:14 -08001111 const TxSetType set_type =
Sarah Parker5effe3f2017-02-23 12:49:10 -08001112 get_ext_tx_set_type(tx_size, bs, is_inter, use_reduced_set);
Hui Su4a9be2a2017-09-19 14:41:49 -07001113 return ext_tx_set_index[is_inter][set_type];
Yaowu Xuc27fc142016-08-22 16:08:15 -07001114}
1115
Sarah Parkere68a3e42017-02-16 14:03:24 -08001116static INLINE int get_ext_tx_types(TX_SIZE tx_size, BLOCK_SIZE bs, int is_inter,
Sarah Parker5effe3f2017-02-23 12:49:10 -08001117 int use_reduced_set) {
1118 const int set_type =
1119 get_ext_tx_set_type(tx_size, bs, is_inter, use_reduced_set);
Hui Suddbcde22017-09-18 17:22:02 -07001120 return av1_num_ext_tx_set[set_type];
Yaowu Xuc27fc142016-08-22 16:08:15 -07001121}
1122
1123#if CONFIG_RECT_TX
1124static INLINE int is_rect_tx_allowed_bsize(BLOCK_SIZE bsize) {
Rupert Swarbrick93c39e92017-07-12 11:11:02 +01001125 static const char LUT[BLOCK_SIZES_ALL] = {
Timothy B. Terriberry81ec2612017-04-26 16:53:47 -07001126#if CONFIG_CHROMA_2X2 || CONFIG_CHROMA_SUB8X8
Jingning Hanf4e097b2017-01-20 09:23:58 -08001127 0, // BLOCK_2X2
1128 0, // BLOCK_2X4
1129 0, // BLOCK_4X2
1130#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -07001131 0, // BLOCK_4X4
1132 1, // BLOCK_4X8
1133 1, // BLOCK_8X4
1134 0, // BLOCK_8X8
1135 1, // BLOCK_8X16
1136 1, // BLOCK_16X8
1137 0, // BLOCK_16X16
1138 1, // BLOCK_16X32
1139 1, // BLOCK_32X16
1140 0, // BLOCK_32X32
1141 0, // BLOCK_32X64
1142 0, // BLOCK_64X32
1143 0, // BLOCK_64X64
1144#if CONFIG_EXT_PARTITION
1145 0, // BLOCK_64X128
1146 0, // BLOCK_128X64
1147 0, // BLOCK_128X128
1148#endif // CONFIG_EXT_PARTITION
Rupert Swarbrick93c39e92017-07-12 11:11:02 +01001149 0, // BLOCK_4X16
1150 0, // BLOCK_16X4
1151 0, // BLOCK_8X32
1152 0, // BLOCK_32X8
Rupert Swarbrick72678572017-08-02 12:05:26 +01001153 0, // BLOCK_16X64
1154 0, // BLOCK_64X16
Yaowu Xuc27fc142016-08-22 16:08:15 -07001155 };
1156
1157 return LUT[bsize];
1158}
1159
Yue Chen49587a72016-09-28 17:09:47 -07001160static INLINE int is_rect_tx_allowed(const MACROBLOCKD *xd,
1161 const MB_MODE_INFO *mbmi) {
Urvang Joshifeb925f2016-12-05 10:37:29 -08001162 return is_rect_tx_allowed_bsize(mbmi->sb_type) &&
Yue Chen49587a72016-09-28 17:09:47 -07001163 !xd->lossless[mbmi->segment_id];
Yaowu Xuc27fc142016-08-22 16:08:15 -07001164}
Yue Chend6bdd462017-07-19 16:05:43 -07001165#endif // CONFIG_RECT_TX
1166#endif // CONFIG_EXT_TX
Yue Chen56e226e2017-05-02 16:21:40 -07001167
Yue Chend6bdd462017-07-19 16:05:43 -07001168#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
Yue Chen56e226e2017-05-02 16:21:40 -07001169static INLINE int is_quarter_tx_allowed_bsize(BLOCK_SIZE bsize) {
Rupert Swarbrick93c39e92017-07-12 11:11:02 +01001170 static const char LUT_QTTX[BLOCK_SIZES_ALL] = {
Timothy B. Terriberry81ec2612017-04-26 16:53:47 -07001171#if CONFIG_CHROMA_2X2 || CONFIG_CHROMA_SUB8X8
Yue Chen56e226e2017-05-02 16:21:40 -07001172 0, // BLOCK_2X2
1173 0, // BLOCK_2X4
1174 0, // BLOCK_4X2
1175#endif
1176 0, // BLOCK_4X4
1177 0, // BLOCK_4X8
1178 0, // BLOCK_8X4
1179 0, // BLOCK_8X8
1180 1, // BLOCK_8X16
1181 1, // BLOCK_16X8
1182 0, // BLOCK_16X16
1183 0, // BLOCK_16X32
1184 0, // BLOCK_32X16
1185 0, // BLOCK_32X32
1186 0, // BLOCK_32X64
1187 0, // BLOCK_64X32
1188 0, // BLOCK_64X64
1189#if CONFIG_EXT_PARTITION
1190 0, // BLOCK_64X128
1191 0, // BLOCK_128X64
1192 0, // BLOCK_128X128
1193#endif // CONFIG_EXT_PARTITION
Rupert Swarbrick93c39e92017-07-12 11:11:02 +01001194 0, // BLOCK_4X16
1195 0, // BLOCK_16X4
1196 0, // BLOCK_8X32
1197 0, // BLOCK_32X8
Rupert Swarbrick72678572017-08-02 12:05:26 +01001198 0, // BLOCK_16X64
1199 0, // BLOCK_64X16
Yue Chen56e226e2017-05-02 16:21:40 -07001200 };
1201
1202 return LUT_QTTX[bsize];
1203}
1204
1205static INLINE int is_quarter_tx_allowed(const MACROBLOCKD *xd,
1206 const MB_MODE_INFO *mbmi,
1207 int is_inter) {
1208 return is_quarter_tx_allowed_bsize(mbmi->sb_type) && is_inter &&
1209 !xd->lossless[mbmi->segment_id];
1210}
Yue Chend6bdd462017-07-19 16:05:43 -07001211#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -07001212
1213static INLINE TX_SIZE tx_size_from_tx_mode(BLOCK_SIZE bsize, TX_MODE tx_mode,
1214 int is_inter) {
1215 const TX_SIZE largest_tx_size = tx_mode_to_biggest_tx_size[tx_mode];
Debargha Mukherjee428bbb22017-03-17 07:30:24 -07001216#if (CONFIG_VAR_TX || CONFIG_EXT_TX) && CONFIG_RECT_TX
Urvang Joshifeb925f2016-12-05 10:37:29 -08001217 const TX_SIZE max_rect_tx_size = max_txsize_rect_lookup[bsize];
Jingning Han70e5f3f2016-11-09 17:03:07 -08001218#else
Yaowu Xuc27fc142016-08-22 16:08:15 -07001219 const TX_SIZE max_tx_size = max_txsize_lookup[bsize];
Debargha Mukherjee428bbb22017-03-17 07:30:24 -07001220#endif // (CONFIG_VAR_TX || CONFIG_EXT_TX) && CONFIG_RECT_TX
Urvang Joshifeb925f2016-12-05 10:37:29 -08001221 (void)is_inter;
Debargha Mukherjee428bbb22017-03-17 07:30:24 -07001222#if CONFIG_VAR_TX && CONFIG_RECT_TX
Urvang Joshifeb925f2016-12-05 10:37:29 -08001223#if CONFIG_CB4X4
1224 if (bsize == BLOCK_4X4)
1225 return AOMMIN(max_txsize_lookup[bsize], largest_tx_size);
1226#else
1227 if (bsize < BLOCK_8X8)
1228 return AOMMIN(max_txsize_lookup[bsize], largest_tx_size);
Jingning Han70e5f3f2016-11-09 17:03:07 -08001229#endif
Urvang Joshifeb925f2016-12-05 10:37:29 -08001230 if (txsize_sqr_map[max_rect_tx_size] <= largest_tx_size)
1231 return max_rect_tx_size;
1232 else
1233 return largest_tx_size;
1234#elif CONFIG_EXT_TX && CONFIG_RECT_TX
1235 if (txsize_sqr_up_map[max_rect_tx_size] <= largest_tx_size) {
1236 return max_rect_tx_size;
Yaowu Xuc27fc142016-08-22 16:08:15 -07001237 } else {
Urvang Joshifeb925f2016-12-05 10:37:29 -08001238 return largest_tx_size;
Yaowu Xuc27fc142016-08-22 16:08:15 -07001239 }
1240#else
Yaowu Xuf883b422016-08-30 14:01:10 -07001241 return AOMMIN(max_tx_size, largest_tx_size);
Debargha Mukherjee428bbb22017-03-17 07:30:24 -07001242#endif // CONFIG_VAR_TX && CONFIG_RECT_TX
Yaowu Xuc27fc142016-08-22 16:08:15 -07001243}
1244
hui su5db97432016-10-14 16:10:14 -07001245#if CONFIG_EXT_INTRA
hui su0a6731f2017-04-26 15:23:47 -07001246#define MAX_ANGLE_DELTA 3
1247#define ANGLE_STEP 3
hui su5db97432016-10-14 16:10:14 -07001248extern const int16_t dr_intra_derivative[90];
hui su02c79742017-05-16 17:19:04 -07001249static const uint8_t mode_to_angle_map[] = {
Urvang Joshi93b543a2017-06-01 17:32:41 -07001250 0, 90, 180, 45, 135, 111, 157, 203, 67, 0, 0,
hui su02c79742017-05-16 17:19:04 -07001251#if CONFIG_SMOOTH_HV
1252 0, 0,
1253#endif // CONFIG_SMOOTH_HV
hui su5db97432016-10-14 16:10:14 -07001254};
hui sueda3d762016-12-06 16:58:23 -08001255#if CONFIG_INTRA_INTERP
hui su5db97432016-10-14 16:10:14 -07001256// Returns whether filter selection is needed for a given
1257// intra prediction angle.
Yaowu Xuf883b422016-08-30 14:01:10 -07001258int av1_is_intra_filter_switchable(int angle);
hui sueda3d762016-12-06 16:58:23 -08001259#endif // CONFIG_INTRA_INTERP
Yaowu Xuc27fc142016-08-22 16:08:15 -07001260#endif // CONFIG_EXT_INTRA
1261
Monty Montgomerycb55dad2017-07-11 16:59:52 -04001262#if CONFIG_DCT_ONLY
1263#define FIXED_TX_TYPE 1
1264#else
Yaowu Xuc27fc142016-08-22 16:08:15 -07001265#define FIXED_TX_TYPE 0
Monty Montgomerycb55dad2017-07-11 16:59:52 -04001266#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -07001267
Angie Chiang752ccce2017-04-09 13:41:13 -07001268// Converts block_index for given transform size to index of the block in raster
1269// order.
1270static INLINE int av1_block_index_to_raster_order(TX_SIZE tx_size,
1271 int block_idx) {
1272 // For transform size 4x8, the possible block_idx values are 0 & 2, because
1273 // block_idx values are incremented in steps of size 'tx_width_unit x
1274 // tx_height_unit'. But, for this transform size, block_idx = 2 corresponds to
1275 // block number 1 in raster order, inside an 8x8 MI block.
1276 // For any other transform size, the two indices are equivalent.
1277 return (tx_size == TX_4X8 && block_idx == 2) ? 1 : block_idx;
1278}
1279
1280// Inverse of above function.
1281// Note: only implemented for transform sizes 4x4, 4x8 and 8x4 right now.
1282static INLINE int av1_raster_order_to_block_index(TX_SIZE tx_size,
1283 int raster_order) {
1284 assert(tx_size == TX_4X4 || tx_size == TX_4X8 || tx_size == TX_8X4);
1285 // We ensure that block indices are 0 & 2 if tx size is 4x8 or 8x4.
1286 return (tx_size == TX_4X4) ? raster_order : (raster_order > 0) ? 2 : 0;
1287}
1288
Yaowu Xuc27fc142016-08-22 16:08:15 -07001289static INLINE TX_TYPE get_default_tx_type(PLANE_TYPE plane_type,
1290 const MACROBLOCKD *xd, int block_idx,
1291 TX_SIZE tx_size) {
1292 const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
1293
Monty Montgomerycb55dad2017-07-11 16:59:52 -04001294 if (CONFIG_DCT_ONLY || is_inter_block(mbmi) || plane_type != PLANE_TYPE_Y ||
Yaowu Xuc27fc142016-08-22 16:08:15 -07001295 xd->lossless[mbmi->segment_id] || tx_size >= TX_32X32)
1296 return DCT_DCT;
1297
1298 return intra_mode_to_tx_type_context[plane_type == PLANE_TYPE_Y
1299 ? get_y_mode(xd->mi[0], block_idx)
Luc Trudeaud6d9eee2017-07-12 12:36:50 -04001300 : get_uv_mode(mbmi->uv_mode)];
Yaowu Xuc27fc142016-08-22 16:08:15 -07001301}
1302
hui su45b64752017-07-12 16:54:35 -07001303static INLINE TX_TYPE av1_get_tx_type(PLANE_TYPE plane_type,
Jingning Han19b5c8f2017-07-06 15:10:12 -07001304 const MACROBLOCKD *xd, int blk_row,
1305 int blk_col, int block, TX_SIZE tx_size) {
Yaowu Xuc27fc142016-08-22 16:08:15 -07001306 const MODE_INFO *const mi = xd->mi[0];
1307 const MB_MODE_INFO *const mbmi = &mi->mbmi;
Jingning Han19b5c8f2017-07-06 15:10:12 -07001308 (void)blk_row;
1309 (void)blk_col;
Alex Conversedaa15e42017-05-02 14:27:16 -07001310#if CONFIG_INTRABC && (!CONFIG_EXT_TX || CONFIG_TXK_SEL)
1311 // TODO(aconverse@google.com): Handle INTRABC + EXT_TX + TXK_SEL
Alex Converse28744302017-04-13 14:46:22 -07001312 if (is_intrabc_block(mbmi)) return DCT_DCT;
Alex Conversedaa15e42017-05-02 14:27:16 -07001313#endif // CONFIG_INTRABC && (!CONFIG_EXT_TX || CONFIG_TXK_SEL)
hui su45b64752017-07-12 16:54:35 -07001314
1315#if CONFIG_TXK_SEL
1316 TX_TYPE tx_type;
Jingning Han19b5c8f2017-07-06 15:10:12 -07001317 if (xd->lossless[mbmi->segment_id] || txsize_sqr_map[tx_size] >= TX_32X32) {
hui su45b64752017-07-12 16:54:35 -07001318 tx_type = DCT_DCT;
1319 } else {
Jingning Han19b5c8f2017-07-06 15:10:12 -07001320 if (plane_type == PLANE_TYPE_Y)
1321 tx_type = mbmi->txk_type[(blk_row << 4) + blk_col];
1322 else if (is_inter_block(mbmi))
1323 tx_type = mbmi->txk_type[(blk_row << 5) + (blk_col << 1)];
1324 else
1325 tx_type = intra_mode_to_tx_type_context[mbmi->uv_mode];
hui su45b64752017-07-12 16:54:35 -07001326 }
1327 assert(tx_type >= DCT_DCT && tx_type < TX_TYPES);
1328 return tx_type;
1329#endif // CONFIG_TXK_SEL
1330
James Zern7cdaa602017-04-20 20:12:46 -07001331#if FIXED_TX_TYPE
Angie Chiangc2ebfbf2017-04-15 13:56:58 -07001332 const int block_raster_idx = av1_block_index_to_raster_order(tx_size, block);
James Zern7cdaa602017-04-20 20:12:46 -07001333 return get_default_tx_type(plane_type, xd, block_raster_idx, tx_size);
hui su45b64752017-07-12 16:54:35 -07001334#endif // FIXED_TX_TYPE
1335
1336#if CONFIG_EXT_TX
Sarah Parker53f93db2017-07-11 17:20:04 -07001337#if CONFIG_MRC_TX
1338 if (mbmi->tx_type == MRC_DCT) {
Sarah Parker2e08d962017-08-01 19:51:20 -07001339 assert(((is_inter_block(mbmi) && USE_MRC_INTER) ||
1340 (!is_inter_block(mbmi) && USE_MRC_INTRA)) &&
1341 "INVALID BLOCK TYPE FOR MRC_DCT");
Sarah Parker53f93db2017-07-11 17:20:04 -07001342 if (plane_type == PLANE_TYPE_Y) {
1343 assert(tx_size == TX_32X32);
1344 return mbmi->tx_type;
1345 }
1346 return DCT_DCT;
1347 }
1348#endif // CONFIG_MRC_TX
Yaowu Xuc27fc142016-08-22 16:08:15 -07001349 if (xd->lossless[mbmi->segment_id] || txsize_sqr_map[tx_size] > TX_32X32 ||
1350 (txsize_sqr_map[tx_size] >= TX_32X32 && !is_inter_block(mbmi)))
Yaowu Xuc27fc142016-08-22 16:08:15 -07001351 return DCT_DCT;
Jingning Han8260d8b2016-12-28 13:48:43 -08001352 if (mbmi->sb_type >= BLOCK_8X8 || CONFIG_CB4X4) {
Yaowu Xuc27fc142016-08-22 16:08:15 -07001353 if (plane_type == PLANE_TYPE_Y) {
1354#if !ALLOW_INTRA_EXT_TX
1355 if (is_inter_block(mbmi))
1356#endif // ALLOW_INTRA_EXT_TX
1357 return mbmi->tx_type;
1358 }
Jingning Han1a00cff2016-12-28 14:53:14 -08001359
1360 if (is_inter_block(mbmi)) {
1361// UV Inter only
Timothy B. Terriberryfe67ed62017-04-26 16:53:47 -07001362#if CONFIG_CHROMA_2X2
Jingning Han1a00cff2016-12-28 14:53:14 -08001363 if (tx_size < TX_4X4) return DCT_DCT;
1364#endif
Debargha Mukherjee5a488a62016-11-22 22:24:10 -08001365 return (mbmi->tx_type == IDTX && txsize_sqr_map[tx_size] >= TX_32X32)
Yaowu Xuc27fc142016-08-22 16:08:15 -07001366 ? DCT_DCT
1367 : mbmi->tx_type;
Jingning Han1a00cff2016-12-28 14:53:14 -08001368 }
Yaowu Xuc27fc142016-08-22 16:08:15 -07001369 }
1370
Jingning Han8260d8b2016-12-28 13:48:43 -08001371#if CONFIG_CB4X4
Thomas Daedeb7e72f32017-04-26 13:40:31 -07001372 (void)block;
Timothy B. Terriberryfe67ed62017-04-26 16:53:47 -07001373#if CONFIG_CHROMA_2X2
Jingning Han1a00cff2016-12-28 14:53:14 -08001374 if (tx_size < TX_4X4)
1375 return DCT_DCT;
1376 else
hui su45b64752017-07-12 16:54:35 -07001377#endif // CONFIG_CHROMA_2X2
Luc Trudeaud6d9eee2017-07-12 12:36:50 -04001378 return intra_mode_to_tx_type_context[get_uv_mode(mbmi->uv_mode)];
Timothy B. Terriberryfe67ed62017-04-26 16:53:47 -07001379#else // CONFIG_CB4X4
Yaowu Xuc27fc142016-08-22 16:08:15 -07001380 // Sub8x8-Inter/Intra OR UV-Intra
hui su45b64752017-07-12 16:54:35 -07001381 if (is_inter_block(mbmi)) { // Sub8x8-Inter
Yaowu Xuc27fc142016-08-22 16:08:15 -07001382 return DCT_DCT;
hui su45b64752017-07-12 16:54:35 -07001383 } else { // Sub8x8 Intra OR UV-Intra
1384 const int block_raster_idx =
1385 av1_block_index_to_raster_order(tx_size, block);
Yaowu Xuc27fc142016-08-22 16:08:15 -07001386 return intra_mode_to_tx_type_context[plane_type == PLANE_TYPE_Y
Angie Chiang752ccce2017-04-09 13:41:13 -07001387 ? get_y_mode(mi, block_raster_idx)
Luc Trudeaud6d9eee2017-07-12 12:36:50 -04001388 : get_uv_mode(mbmi->uv_mode)];
hui su45b64752017-07-12 16:54:35 -07001389 }
Thomas Daedeb7e72f32017-04-26 13:40:31 -07001390#endif // CONFIG_CB4X4
Yaowu Xuc27fc142016-08-22 16:08:15 -07001391#else // CONFIG_EXT_TX
Angie Chiang752ccce2017-04-09 13:41:13 -07001392 (void)block;
Sarah Parker53f93db2017-07-11 17:20:04 -07001393#if CONFIG_MRC_TX
1394 if (mbmi->tx_type == MRC_DCT) {
1395 if (plane_type == PLANE_TYPE_Y && !xd->lossless[mbmi->segment_id]) {
1396 assert(tx_size == TX_32X32);
1397 return mbmi->tx_type;
1398 }
1399 return DCT_DCT;
1400 }
1401#endif // CONFIG_MRC_TX
Yaowu Xuc27fc142016-08-22 16:08:15 -07001402 if (plane_type != PLANE_TYPE_Y || xd->lossless[mbmi->segment_id] ||
1403 txsize_sqr_map[tx_size] >= TX_32X32)
1404 return DCT_DCT;
1405 return mbmi->tx_type;
1406#endif // CONFIG_EXT_TX
1407}
1408
Yaowu Xuf883b422016-08-30 14:01:10 -07001409void av1_setup_block_planes(MACROBLOCKD *xd, int ss_x, int ss_y);
Yaowu Xuc27fc142016-08-22 16:08:15 -07001410
Yaowu Xu4ff59b52017-04-24 12:41:56 -07001411static INLINE int tx_size_to_depth(TX_SIZE tx_size) {
Timothy B. Terriberryfe0fb1d2017-05-18 12:15:16 -07001412 return (int)(tx_size - TX_SIZE_LUMA_MIN);
Jingning Han4e1737a2016-10-25 16:05:02 -07001413}
1414
Yaowu Xu4ff59b52017-04-24 12:41:56 -07001415static INLINE TX_SIZE depth_to_tx_size(int depth) {
Timothy B. Terriberryfe0fb1d2017-05-18 12:15:16 -07001416 return (TX_SIZE)(depth + TX_SIZE_LUMA_MIN);
Jingning Han4e1737a2016-10-25 16:05:02 -07001417}
1418
hui su0c6244b2017-07-12 17:11:43 -07001419static INLINE TX_SIZE av1_get_uv_tx_size(const MB_MODE_INFO *mbmi,
1420 const struct macroblockd_plane *pd) {
Timothy B. Terriberryfe67ed62017-04-26 16:53:47 -07001421#if CONFIG_CHROMA_2X2
Jingning Han93531242016-12-20 11:54:36 -08001422 assert(mbmi->tx_size > TX_2X2);
hui su0c6244b2017-07-12 17:11:43 -07001423#endif // CONFIG_CHROMA_2X2
Jingning Han93531242016-12-20 11:54:36 -08001424
Yaowu Xuc27fc142016-08-22 16:08:15 -07001425#if CONFIG_SUPERTX
1426 if (supertx_enabled(mbmi))
1427 return uvsupertx_size_lookup[txsize_sqr_map[mbmi->tx_size]]
1428 [pd->subsampling_x][pd->subsampling_y];
1429#endif // CONFIG_SUPERTX
Jingning Hancabd9892016-12-01 12:28:42 -08001430
hui su0c6244b2017-07-12 17:11:43 -07001431 const TX_SIZE uv_txsize =
1432 uv_txsize_lookup[mbmi->sb_type][mbmi->tx_size][pd->subsampling_x]
1433 [pd->subsampling_y];
Debargha Mukherjee2f123402016-08-30 17:43:38 -07001434 assert(uv_txsize != TX_INVALID);
1435 return uv_txsize;
Yaowu Xuc27fc142016-08-22 16:08:15 -07001436}
1437
hui su0c6244b2017-07-12 17:11:43 -07001438static INLINE TX_SIZE av1_get_tx_size(int plane, const MACROBLOCKD *xd) {
Angie Chiang80b82262017-02-24 11:39:47 -08001439 const MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
hui su0c6244b2017-07-12 17:11:43 -07001440 if (plane == 0) return mbmi->tx_size;
Angie Chiang80b82262017-02-24 11:39:47 -08001441 const MACROBLOCKD_PLANE *pd = &xd->plane[plane];
hui su0c6244b2017-07-12 17:11:43 -07001442 return av1_get_uv_tx_size(mbmi, pd);
Angie Chiang80b82262017-02-24 11:39:47 -08001443}
1444
Yaowu Xuc27fc142016-08-22 16:08:15 -07001445static INLINE BLOCK_SIZE
1446get_plane_block_size(BLOCK_SIZE bsize, const struct macroblockd_plane *pd) {
1447 return ss_size_lookup[bsize][pd->subsampling_x][pd->subsampling_y];
1448}
1449
Timothy B. Terriberrya2d5cde2017-05-10 18:33:50 -07001450void av1_reset_skip_context(MACROBLOCKD *xd, int mi_row, int mi_col,
1451 BLOCK_SIZE bsize);
Yaowu Xuc27fc142016-08-22 16:08:15 -07001452
1453typedef void (*foreach_transformed_block_visitor)(int plane, int block,
1454 int blk_row, int blk_col,
1455 BLOCK_SIZE plane_bsize,
1456 TX_SIZE tx_size, void *arg);
1457
Yaowu Xuf883b422016-08-30 14:01:10 -07001458void av1_foreach_transformed_block_in_plane(
Yaowu Xuc27fc142016-08-22 16:08:15 -07001459 const MACROBLOCKD *const xd, BLOCK_SIZE bsize, int plane,
1460 foreach_transformed_block_visitor visit, void *arg);
1461
Angie Chiang0397eda2017-03-15 16:57:14 -07001462#if CONFIG_LV_MAP
1463void av1_foreach_transformed_block(const MACROBLOCKD *const xd,
Jingning Han94652b82017-04-04 09:45:02 -07001464 BLOCK_SIZE bsize, int mi_row, int mi_col,
Angie Chiang0397eda2017-03-15 16:57:14 -07001465 foreach_transformed_block_visitor visit,
1466 void *arg);
1467#endif
1468
iole moccagattaf25a4cf2016-11-11 23:57:57 -08001469#if CONFIG_COEF_INTERLEAVE
1470static INLINE int get_max_4x4_size(int num_4x4, int mb_to_edge,
1471 int subsampling) {
1472 return num_4x4 + (mb_to_edge >= 0 ? 0 : mb_to_edge >> (5 + subsampling));
1473}
1474
1475void av1_foreach_transformed_block_interleave(
1476 const MACROBLOCKD *const xd, BLOCK_SIZE bsize,
1477 foreach_transformed_block_visitor visit, void *arg);
1478#endif
1479
Yaowu Xuf883b422016-08-30 14:01:10 -07001480void av1_set_contexts(const MACROBLOCKD *xd, struct macroblockd_plane *pd,
Jingning Haneee43152016-12-05 09:58:45 -08001481 int plane, TX_SIZE tx_size, int has_eob, int aoff,
1482 int loff);
Yaowu Xuc27fc142016-08-22 16:08:15 -07001483
1484#if CONFIG_EXT_INTER
1485static INLINE int is_interintra_allowed_bsize(const BLOCK_SIZE bsize) {
Debargha Mukherjee37f6fe62017-02-10 21:44:13 -08001486#if CONFIG_INTERINTRA
Yaowu Xuc27fc142016-08-22 16:08:15 -07001487 // TODO(debargha): Should this be bsize < BLOCK_LARGEST?
1488 return (bsize >= BLOCK_8X8) && (bsize < BLOCK_64X64);
Debargha Mukherjee37f6fe62017-02-10 21:44:13 -08001489#else
1490 (void)bsize;
1491 return 0;
1492#endif // CONFIG_INTERINTRA
Yaowu Xuc27fc142016-08-22 16:08:15 -07001493}
1494
1495static INLINE int is_interintra_allowed_mode(const PREDICTION_MODE mode) {
Debargha Mukherjee37f6fe62017-02-10 21:44:13 -08001496#if CONFIG_INTERINTRA
Yaowu Xuc27fc142016-08-22 16:08:15 -07001497 return (mode >= NEARESTMV) && (mode <= NEWMV);
Debargha Mukherjee37f6fe62017-02-10 21:44:13 -08001498#else
1499 (void)mode;
1500 return 0;
1501#endif // CONFIG_INTERINTRA
Yaowu Xuc27fc142016-08-22 16:08:15 -07001502}
1503
1504static INLINE int is_interintra_allowed_ref(const MV_REFERENCE_FRAME rf[2]) {
Debargha Mukherjee37f6fe62017-02-10 21:44:13 -08001505#if CONFIG_INTERINTRA
Yaowu Xuc27fc142016-08-22 16:08:15 -07001506 return (rf[0] > INTRA_FRAME) && (rf[1] <= INTRA_FRAME);
Debargha Mukherjee37f6fe62017-02-10 21:44:13 -08001507#else
1508 (void)rf;
1509 return 0;
1510#endif // CONFIG_INTERINTRA
Yaowu Xuc27fc142016-08-22 16:08:15 -07001511}
1512
1513static INLINE int is_interintra_allowed(const MB_MODE_INFO *mbmi) {
1514 return is_interintra_allowed_bsize(mbmi->sb_type) &&
1515 is_interintra_allowed_mode(mbmi->mode) &&
1516 is_interintra_allowed_ref(mbmi->ref_frame);
1517}
1518
Yaowu Xu4ff59b52017-04-24 12:41:56 -07001519static INLINE int is_interintra_allowed_bsize_group(int group) {
Yaowu Xuc27fc142016-08-22 16:08:15 -07001520 int i;
Rupert Swarbrick93c39e92017-07-12 11:11:02 +01001521 for (i = 0; i < BLOCK_SIZES_ALL; i++) {
Urvang Joshicb586f32016-09-20 11:36:33 -07001522 if (size_group_lookup[i] == group &&
1523 is_interintra_allowed_bsize((BLOCK_SIZE)i)) {
Yaowu Xuc27fc142016-08-22 16:08:15 -07001524 return 1;
Urvang Joshicb586f32016-09-20 11:36:33 -07001525 }
Yaowu Xuc27fc142016-08-22 16:08:15 -07001526 }
1527 return 0;
1528}
1529
1530static INLINE int is_interintra_pred(const MB_MODE_INFO *mbmi) {
1531 return (mbmi->ref_frame[1] == INTRA_FRAME) && is_interintra_allowed(mbmi);
1532}
1533#endif // CONFIG_EXT_INTER
1534
Sarah Parker106b3cb2017-04-21 12:13:37 -07001535#if CONFIG_VAR_TX
1536static INLINE int get_vartx_max_txsize(const MB_MODE_INFO *const mbmi,
1537 BLOCK_SIZE bsize) {
1538#if CONFIG_CB4X4
1539 (void)mbmi;
1540 return max_txsize_rect_lookup[bsize];
1541#endif // CONFIG_C4X4
1542 return mbmi->sb_type < BLOCK_8X8 ? max_txsize_rect_lookup[mbmi->sb_type]
1543 : max_txsize_rect_lookup[bsize];
1544}
1545#endif // CONFIG_VAR_TX
1546
Yue Chencb60b182016-10-13 15:18:22 -07001547#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
1548static INLINE int is_motion_variation_allowed_bsize(BLOCK_SIZE bsize) {
Debargha Mukherjeec17a4432017-08-28 22:22:45 -07001549 return AOMMIN(block_size_wide[bsize], block_size_high[bsize]) >= 8;
Yue Chencb60b182016-10-13 15:18:22 -07001550}
1551
Yue Chen8636da62017-04-03 01:23:44 -07001552static INLINE int is_motion_variation_allowed_compound(
1553 const MB_MODE_INFO *mbmi) {
Zoe Liu85b66462017-04-20 14:28:19 -07001554#if CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF
1555 if (!has_second_ref(mbmi) && !is_inter_singleref_comp_mode(mbmi->mode))
1556#else
Yue Chen8636da62017-04-03 01:23:44 -07001557 if (!has_second_ref(mbmi))
Zoe Liu85b66462017-04-20 14:28:19 -07001558#endif // CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF
Yue Chen8636da62017-04-03 01:23:44 -07001559 return 1;
1560 else
1561 return 0;
1562}
1563
Yue Chen5329a2b2017-02-28 17:33:00 +08001564#if CONFIG_MOTION_VAR
Yue Chen1bd42be2017-03-15 18:07:04 -07001565// input: log2 of length, 0(4), 1(8), ...
1566static const int max_neighbor_obmc[6] = { 0, 1, 2, 3, 4, 4 };
1567
Yue Chen5329a2b2017-02-28 17:33:00 +08001568static INLINE int check_num_overlappable_neighbors(const MB_MODE_INFO *mbmi) {
Yue Chen1bd42be2017-03-15 18:07:04 -07001569 return !(mbmi->overlappable_neighbors[0] == 0 &&
1570 mbmi->overlappable_neighbors[1] == 0);
Yue Chen5329a2b2017-02-28 17:33:00 +08001571}
Wei-Ting Lin20885282017-08-28 17:18:18 -07001572#if CONFIG_NCOBMC_ADAPT_WEIGHT
1573static INLINE NCOBMC_MODE ncobmc_mode_allowed_bsize(BLOCK_SIZE bsize) {
1574 if (bsize < BLOCK_8X8 || bsize >= BLOCK_64X64)
1575 return NO_OVERLAP;
1576 else
1577 return MAX_NCOBMC_MODES;
1578}
1579#endif // CONFIG_NCOBMC_ADAPT_WEIGHT
1580#endif // CONFIG_MOTION_VAR
Yue Chen5329a2b2017-02-28 17:33:00 +08001581
Sarah Parker19234cc2017-03-10 16:43:25 -08001582static INLINE MOTION_MODE motion_mode_allowed(
Sarah Parker0eea89f2017-07-11 11:56:36 -07001583#if CONFIG_GLOBAL_MOTION
Sarah Parker19234cc2017-03-10 16:43:25 -08001584 int block, const WarpedMotionParams *gm_params,
Sarah Parker0eea89f2017-07-11 11:56:36 -07001585#endif // CONFIG_GLOBAL_MOTION
Yue Chen52c51732017-07-11 15:08:30 -07001586#if CONFIG_WARPED_MOTION
1587 const MACROBLOCKD *xd,
1588#endif
Sarah Parker19234cc2017-03-10 16:43:25 -08001589 const MODE_INFO *mi) {
1590 const MB_MODE_INFO *mbmi = &mi->mbmi;
RogerZhou3b635242017-09-19 10:06:46 -07001591#if CONFIG_AMVR
1592 if (xd->cur_frame_mv_precision_level == 0) {
1593#endif
Sarah Parker0eea89f2017-07-11 11:56:36 -07001594#if CONFIG_GLOBAL_MOTION
RogerZhou3b635242017-09-19 10:06:46 -07001595 const TransformationType gm_type = gm_params[mbmi->ref_frame[0]].wmtype;
1596 if (is_global_mv_block(mi, block, gm_type)) return SIMPLE_TRANSLATION;
Sarah Parker0eea89f2017-07-11 11:56:36 -07001597#endif // CONFIG_GLOBAL_MOTION
RogerZhou3b635242017-09-19 10:06:46 -07001598#if CONFIG_AMVR
1599 }
1600#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -07001601#if CONFIG_EXT_INTER
Yue Chen69f18e12016-09-08 14:48:15 -07001602 if (is_motion_variation_allowed_bsize(mbmi->sb_type) &&
Yue Chen8636da62017-04-03 01:23:44 -07001603 is_inter_mode(mbmi->mode) && mbmi->ref_frame[1] != INTRA_FRAME &&
1604 is_motion_variation_allowed_compound(mbmi)) {
Yaowu Xuc27fc142016-08-22 16:08:15 -07001605#else
Yue Chen69f18e12016-09-08 14:48:15 -07001606 if (is_motion_variation_allowed_bsize(mbmi->sb_type) &&
Yue Chen8636da62017-04-03 01:23:44 -07001607 is_inter_mode(mbmi->mode) && is_motion_variation_allowed_compound(mbmi)) {
Yaowu Xuc27fc142016-08-22 16:08:15 -07001608#endif // CONFIG_EXT_INTER
Yue Chen5329a2b2017-02-28 17:33:00 +08001609#if CONFIG_MOTION_VAR
1610 if (!check_num_overlappable_neighbors(mbmi)) return SIMPLE_TRANSLATION;
1611#endif
Yue Chen69f18e12016-09-08 14:48:15 -07001612#if CONFIG_WARPED_MOTION
Yue Chen52c51732017-07-11 15:08:30 -07001613 if (!has_second_ref(mbmi) && mbmi->num_proj_ref[0] >= 1 &&
Wei-Ting Lin20885282017-08-28 17:18:18 -07001614 !av1_is_scaled(&(xd->block_refs[0]->sf))) {
RogerZhou3b635242017-09-19 10:06:46 -07001615#if CONFIG_AMVR
1616 if (xd->cur_frame_mv_precision_level) {
1617 return OBMC_CAUSAL;
1618 }
1619#endif
Yue Chen69f18e12016-09-08 14:48:15 -07001620 return WARPED_CAUSAL;
Hui Su12231bd2017-09-07 18:01:15 -07001621 }
RogerZhou3b635242017-09-19 10:06:46 -07001622
Yue Chen69f18e12016-09-08 14:48:15 -07001623#endif // CONFIG_WARPED_MOTION
1624#if CONFIG_MOTION_VAR
Wei-Ting Lin20885282017-08-28 17:18:18 -07001625#if CONFIG_NCOBMC_ADAPT_WEIGHT
Hui Su12231bd2017-09-07 18:01:15 -07001626 if (ncobmc_mode_allowed_bsize(mbmi->sb_type) < NO_OVERLAP)
1627 return NCOBMC_ADAPT_WEIGHT;
1628 else
Wei-Ting Lin20885282017-08-28 17:18:18 -07001629#endif
Hui Su12231bd2017-09-07 18:01:15 -07001630 return OBMC_CAUSAL;
Yue Chen69f18e12016-09-08 14:48:15 -07001631#else
1632 return SIMPLE_TRANSLATION;
1633#endif // CONFIG_MOTION_VAR
1634 } else {
1635 return SIMPLE_TRANSLATION;
1636 }
Yaowu Xuc27fc142016-08-22 16:08:15 -07001637}
1638
Sarah Parker19234cc2017-03-10 16:43:25 -08001639static INLINE void assert_motion_mode_valid(MOTION_MODE mode,
Sarah Parker0eea89f2017-07-11 11:56:36 -07001640#if CONFIG_GLOBAL_MOTION
Sarah Parker19234cc2017-03-10 16:43:25 -08001641 int block,
1642 const WarpedMotionParams *gm_params,
Sarah Parker0eea89f2017-07-11 11:56:36 -07001643#endif // CONFIG_GLOBAL_MOTION
Yue Chen52c51732017-07-11 15:08:30 -07001644#if CONFIG_WARPED_MOTION
1645 const MACROBLOCKD *xd,
1646#endif
Sarah Parker19234cc2017-03-10 16:43:25 -08001647 const MODE_INFO *mi) {
1648 const MOTION_MODE last_motion_mode_allowed = motion_mode_allowed(
Sarah Parker0eea89f2017-07-11 11:56:36 -07001649#if CONFIG_GLOBAL_MOTION
Sarah Parker19234cc2017-03-10 16:43:25 -08001650 block, gm_params,
Sarah Parker0eea89f2017-07-11 11:56:36 -07001651#endif // CONFIG_GLOBAL_MOTION
Yue Chen52c51732017-07-11 15:08:30 -07001652#if CONFIG_WARPED_MOTION
1653 xd,
1654#endif
Sarah Parker19234cc2017-03-10 16:43:25 -08001655 mi);
Wei-Ting Lin20885282017-08-28 17:18:18 -07001656
Sarah Parker19234cc2017-03-10 16:43:25 -08001657 // Check that the input mode is not illegal
1658 if (last_motion_mode_allowed < mode)
1659 assert(0 && "Illegal motion mode selected");
1660}
1661
Yue Chencb60b182016-10-13 15:18:22 -07001662#if CONFIG_MOTION_VAR
Yaowu Xuc27fc142016-08-22 16:08:15 -07001663static INLINE int is_neighbor_overlappable(const MB_MODE_INFO *mbmi) {
1664 return (is_inter_block(mbmi));
1665}
Yue Chencb60b182016-10-13 15:18:22 -07001666#endif // CONFIG_MOTION_VAR
1667#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
Yaowu Xuc27fc142016-08-22 16:08:15 -07001668
Urvang Joshi56ba91b2017-01-10 13:22:09 -08001669// Returns sub-sampled dimensions of the given block.
1670// The output values for 'rows_within_bounds' and 'cols_within_bounds' will
RogerZhou3b635242017-09-19 10:06:46 -07001671// differ from 'height' and 'width' when part of the block is outside the
1672// right
Urvang Joshi56ba91b2017-01-10 13:22:09 -08001673// and/or bottom image boundary.
1674static INLINE void av1_get_block_dimensions(BLOCK_SIZE bsize, int plane,
1675 const MACROBLOCKD *xd, int *width,
1676 int *height,
1677 int *rows_within_bounds,
1678 int *cols_within_bounds) {
1679 const int block_height = block_size_high[bsize];
1680 const int block_width = block_size_wide[bsize];
1681 const int block_rows = (xd->mb_to_bottom_edge >= 0)
1682 ? block_height
1683 : (xd->mb_to_bottom_edge >> 3) + block_height;
1684 const int block_cols = (xd->mb_to_right_edge >= 0)
1685 ? block_width
1686 : (xd->mb_to_right_edge >> 3) + block_width;
1687 const struct macroblockd_plane *const pd = &xd->plane[plane];
1688 assert(IMPLIES(plane == PLANE_TYPE_Y, pd->subsampling_x == 0));
1689 assert(IMPLIES(plane == PLANE_TYPE_Y, pd->subsampling_y == 0));
1690 assert(block_width >= block_cols);
1691 assert(block_height >= block_rows);
1692 if (width) *width = block_width >> pd->subsampling_x;
1693 if (height) *height = block_height >> pd->subsampling_y;
1694 if (rows_within_bounds) *rows_within_bounds = block_rows >> pd->subsampling_y;
1695 if (cols_within_bounds) *cols_within_bounds = block_cols >> pd->subsampling_x;
1696}
1697
Sarah Parkerefd8af22017-08-25 16:36:06 -07001698/* clang-format off */
1699typedef aom_cdf_prob (*MapCdf)[PALETTE_COLOR_INDEX_CONTEXTS]
1700 [CDF_SIZE(PALETTE_COLORS)];
1701typedef const int (*ColorCost)[PALETTE_SIZES][PALETTE_COLOR_INDEX_CONTEXTS]
1702 [PALETTE_COLORS];
1703/* clang-format on */
1704
1705typedef struct {
1706 int rows;
1707 int cols;
1708 int n_colors;
1709 int plane_width;
1710 int plane_height;
1711 uint8_t *color_map;
1712 MapCdf map_cdf;
1713 ColorCost color_cost;
1714} Av1ColorMapParam;
1715
Yue Chen19e7aa82016-11-30 14:05:39 -08001716#if CONFIG_GLOBAL_MOTION
1717static INLINE int is_nontrans_global_motion(const MACROBLOCKD *xd) {
1718 const MODE_INFO *mi = xd->mi[0];
1719 const MB_MODE_INFO *const mbmi = &mi->mbmi;
1720 int ref;
1721#if CONFIG_CB4X4
1722 const int unify_bsize = 1;
1723#else
1724 const int unify_bsize = 0;
1725#endif
1726
1727 // First check if all modes are ZEROMV
1728 if (mbmi->sb_type >= BLOCK_8X8 || unify_bsize) {
1729#if CONFIG_EXT_INTER
1730 if (mbmi->mode != ZEROMV && mbmi->mode != ZERO_ZEROMV) return 0;
1731#else
1732 if (mbmi->mode != ZEROMV) return 0;
1733#endif // CONFIG_EXT_INTER
1734 } else {
1735#if CONFIG_EXT_INTER
David Barkera0c16382017-07-07 17:01:30 +01001736 if ((mi->bmi[0].as_mode != ZEROMV && mi->bmi[0].as_mode != ZERO_ZEROMV) ||
1737 (mi->bmi[1].as_mode != ZEROMV && mi->bmi[1].as_mode != ZERO_ZEROMV) ||
1738 (mi->bmi[2].as_mode != ZEROMV && mi->bmi[2].as_mode != ZERO_ZEROMV) ||
1739 (mi->bmi[3].as_mode != ZEROMV && mi->bmi[3].as_mode != ZERO_ZEROMV))
Yue Chen19e7aa82016-11-30 14:05:39 -08001740 return 0;
1741#else
1742 if (mi->bmi[0].as_mode != ZEROMV || mi->bmi[1].as_mode != ZEROMV ||
1743 mi->bmi[2].as_mode != ZEROMV || mi->bmi[3].as_mode != ZEROMV)
1744 return 0;
1745#endif // CONFIG_EXT_INTER
1746 }
Jingning Han3ca0e672017-04-14 19:48:05 -07001747
1748#if !GLOBAL_SUB8X8_USED
1749 if (mbmi->sb_type < BLOCK_8X8) return 0;
1750#endif
1751
Yue Chen19e7aa82016-11-30 14:05:39 -08001752 // Now check if all global motion is non translational
1753 for (ref = 0; ref < 1 + has_second_ref(mbmi); ++ref) {
1754 if (xd->global_motion[mbmi->ref_frame[ref]].wmtype <= TRANSLATION) return 0;
1755 }
1756 return 1;
1757}
1758#endif // CONFIG_GLOBAL_MOTION
1759
Yaowu Xu4ff59b52017-04-24 12:41:56 -07001760static INLINE PLANE_TYPE get_plane_type(int plane) {
Luc Trudeau005feb62017-02-22 13:34:01 -05001761 return (plane == 0) ? PLANE_TYPE_Y : PLANE_TYPE_UV;
1762}
1763
Angie Chiang155bf9a2017-08-06 19:52:57 -07001764static INLINE void transpose_uint8(uint8_t *dst, int dst_stride,
1765 const uint8_t *src, int src_stride, int w,
1766 int h) {
1767 int r, c;
1768 for (r = 0; r < h; ++r)
1769 for (c = 0; c < w; ++c) dst[c * dst_stride + r] = src[r * src_stride + c];
1770}
1771
1772static INLINE void transpose_uint16(uint16_t *dst, int dst_stride,
1773 const uint16_t *src, int src_stride, int w,
1774 int h) {
1775 int r, c;
1776 for (r = 0; r < h; ++r)
1777 for (c = 0; c < w; ++c) dst[c * dst_stride + r] = src[r * src_stride + c];
1778}
1779
1780static INLINE void transpose_int16(int16_t *dst, int dst_stride,
1781 const int16_t *src, int src_stride, int w,
1782 int h) {
1783 int r, c;
1784 for (r = 0; r < h; ++r)
1785 for (c = 0; c < w; ++c) dst[c * dst_stride + r] = src[r * src_stride + c];
1786}
1787
1788static INLINE void transpose_int32(int32_t *dst, int dst_stride,
1789 const int32_t *src, int src_stride, int w,
1790 int h) {
1791 int r, c;
1792 for (r = 0; r < h; ++r)
1793 for (c = 0; c < w; ++c) dst[c * dst_stride + r] = src[r * src_stride + c];
1794}
1795
Yaowu Xuc27fc142016-08-22 16:08:15 -07001796#ifdef __cplusplus
1797} // extern "C"
1798#endif
1799
Yaowu Xuf883b422016-08-30 14:01:10 -07001800#endif // AV1_COMMON_BLOCKD_H_