blob: ebb20bb6572ca360cc1723c4d961911f6e2cc2c0 [file] [log] [blame]
Yaowu Xuc27fc142016-08-22 16:08:15 -07001/*
Yaowu Xu2ab7ff02016-09-02 12:04:54 -07002 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Yaowu Xuc27fc142016-08-22 16:08:15 -07003 *
Yaowu Xu2ab7ff02016-09-02 12:04:54 -07004 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Yaowu Xuc27fc142016-08-22 16:08:15 -070010 */
11
Yaowu Xuf883b422016-08-30 14:01:10 -070012#ifndef AV1_COMMON_BLOCKD_H_
13#define AV1_COMMON_BLOCKD_H_
Yaowu Xuc27fc142016-08-22 16:08:15 -070014
Yaowu Xuf883b422016-08-30 14:01:10 -070015#include "./aom_config.h"
Yaowu Xuc27fc142016-08-22 16:08:15 -070016
Yaowu Xuf883b422016-08-30 14:01:10 -070017#include "aom_dsp/aom_dsp_common.h"
Yaowu Xuc27fc142016-08-22 16:08:15 -070018#include "aom_ports/mem.h"
19#include "aom_scale/yv12config.h"
20
21#include "av1/common/common_data.h"
22#include "av1/common/quant_common.h"
23#include "av1/common/entropy.h"
24#include "av1/common/entropymode.h"
25#include "av1/common/mv.h"
26#include "av1/common/scale.h"
27#include "av1/common/seg_common.h"
28#include "av1/common/tile_common.h"
Yushin Cho77bba8d2016-11-04 16:36:56 -070029#if CONFIG_PVQ
30#include "av1/common/pvq.h"
31#include "av1/common/pvq_state.h"
32#include "av1/decoder/decint.h"
33#endif
Luc Trudeaubaeb3752017-04-24 11:19:25 -040034#if CONFIG_CFL
35#include "av1/common/cfl.h"
36#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -070037#ifdef __cplusplus
38extern "C" {
39#endif
40
Jingning Han69d21012017-05-14 16:51:27 -070041#if (CONFIG_CHROMA_SUB8X8 || CONFIG_CHROMA_2X2)
42#define SUB8X8_COMP_REF 0
43#else
44#define SUB8X8_COMP_REF 1
45#endif
Jingning Hanc41a5492017-02-24 11:18:52 -080046
Yaowu Xuc27fc142016-08-22 16:08:15 -070047#define MAX_MB_PLANE 3
48
David Barkerac37fa32016-12-02 12:30:21 +000049#if CONFIG_EXT_INTER
Sarah Parkerb9f757c2017-01-06 17:12:24 -080050
Debargha Mukherjee1edf9a32017-01-07 18:54:20 -080051#if CONFIG_COMPOUND_SEGMENT
Debargha Mukherjee1edf9a32017-01-07 18:54:20 -080052// Set COMPOUND_SEGMENT_TYPE to one of the three
53// 0: Uniform
54// 1: Difference weighted
55#define COMPOUND_SEGMENT_TYPE 1
Debargha Mukherjee1edf9a32017-01-07 18:54:20 -080056#define MAX_SEG_MASK_BITS 1
Yaowu Xuf35f5272017-05-10 08:00:02 -070057
Sarah Parkerb9f757c2017-01-06 17:12:24 -080058// SEG_MASK_TYPES should not surpass 1 << MAX_SEG_MASK_BITS
59typedef enum {
Yaowu Xuf35f5272017-05-10 08:00:02 -070060#if COMPOUND_SEGMENT_TYPE == 0
Sarah Parkerb9f757c2017-01-06 17:12:24 -080061 UNIFORM_45 = 0,
62 UNIFORM_45_INV,
Debargha Mukherjee1edf9a32017-01-07 18:54:20 -080063#elif COMPOUND_SEGMENT_TYPE == 1
Sarah Parker7bb84f32017-05-09 15:17:46 -070064 DIFFWTD_38 = 0,
65 DIFFWTD_38_INV,
Yaowu Xuf35f5272017-05-10 08:00:02 -070066#endif // COMPOUND_SEGMENT_TYPE
Debargha Mukherjee1edf9a32017-01-07 18:54:20 -080067 SEG_MASK_TYPES,
68} SEG_MASK_TYPE;
69
Sarah Parkerb9f757c2017-01-06 17:12:24 -080070#endif // CONFIG_COMPOUND_SEGMENT
Debargha Mukherjee1edf9a32017-01-07 18:54:20 -080071#endif // CONFIG_EXT_INTER
David Barkerac37fa32016-12-02 12:30:21 +000072
Yaowu Xuc27fc142016-08-22 16:08:15 -070073typedef enum {
74 KEY_FRAME = 0,
75 INTER_FRAME = 1,
76 FRAME_TYPES,
77} FRAME_TYPE;
78
Yaowu Xuc27fc142016-08-22 16:08:15 -070079static INLINE int is_inter_mode(PREDICTION_MODE mode) {
80#if CONFIG_EXT_INTER
81 return mode >= NEARESTMV && mode <= NEW_NEWMV;
82#else
83 return mode >= NEARESTMV && mode <= NEWMV;
84#endif // CONFIG_EXT_INTER
85}
86
Yushin Cho77bba8d2016-11-04 16:36:56 -070087#if CONFIG_PVQ
88typedef struct PVQ_INFO {
89 int theta[PVQ_MAX_PARTITIONS];
Yushin Cho77bba8d2016-11-04 16:36:56 -070090 int qg[PVQ_MAX_PARTITIONS];
91 int k[PVQ_MAX_PARTITIONS];
Yushin Cho48f84db2016-11-07 21:20:17 -080092 od_coeff y[OD_TXSIZE_MAX * OD_TXSIZE_MAX];
Yushin Cho77bba8d2016-11-04 16:36:56 -070093 int nb_bands;
94 int off[PVQ_MAX_PARTITIONS];
95 int size[PVQ_MAX_PARTITIONS];
96 int skip_rest;
97 int skip_dir;
ltrudeaue1c09292017-01-20 15:42:13 -050098 int bs; // log of the block size minus two,
99 // i.e. equivalent to aom's TX_SIZE
100 // Block skip info, indicating whether DC/AC, is coded.
101 PVQ_SKIP_TYPE ac_dc_coded; // bit0: DC coded, bit1 : AC coded (1 means coded)
Yushin Cho77bba8d2016-11-04 16:36:56 -0700102 tran_low_t dq_dc_residue;
103} PVQ_INFO;
104
105typedef struct PVQ_QUEUE {
106 PVQ_INFO *buf; // buffer for pvq info, stored in encoding order
107 int curr_pos; // curr position to write PVQ_INFO
108 int buf_len; // allocated buffer length
109 int last_pos; // last written position of PVQ_INFO in a tile
110} PVQ_QUEUE;
111#endif
112
Wei-Ting Lin482551b2017-08-03 12:29:24 -0700113#if CONFIG_NCOBMC_ADAPT_WEIGHT
114typedef struct superblock_mi_boundaries {
115 int mi_row_begin;
116 int mi_col_begin;
117 int mi_row_end;
118 int mi_col_end;
119} SB_MI_BD;
120
Wei-Ting Linb586de82017-08-03 15:17:38 -0700121typedef struct { int16_t KERNEL[4][MAX_SB_SIZE][MAX_SB_SIZE]; } NCOBMC_KERNELS;
Wei-Ting Lin482551b2017-08-03 12:29:24 -0700122#endif
123
David Barkerac37fa32016-12-02 12:30:21 +0000124typedef struct {
125 uint8_t *plane[MAX_MB_PLANE];
126 int stride[MAX_MB_PLANE];
127} BUFFER_SET;
128
Yaowu Xuc27fc142016-08-22 16:08:15 -0700129#if CONFIG_EXT_INTER
130static INLINE int is_inter_singleref_mode(PREDICTION_MODE mode) {
Zoe Liu7f24e1b2017-03-17 17:42:05 -0700131 return mode >= NEARESTMV && mode <= NEWMV;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700132}
Zoe Liu85b66462017-04-20 14:28:19 -0700133static INLINE int is_inter_compound_mode(PREDICTION_MODE mode) {
134 return mode >= NEAREST_NEARESTMV && mode <= NEW_NEWMV;
135}
Zoe Liu239f06b2017-04-20 13:10:55 -0700136#if CONFIG_COMPOUND_SINGLEREF
137static INLINE int is_inter_singleref_comp_mode(PREDICTION_MODE mode) {
138 return mode >= SR_NEAREST_NEARMV && mode <= SR_NEW_NEWMV;
139}
Zoe Liu85b66462017-04-20 14:28:19 -0700140static INLINE int is_inter_anyref_comp_mode(PREDICTION_MODE mode) {
141 return is_inter_compound_mode(mode) || is_inter_singleref_comp_mode(mode);
Yaowu Xuc27fc142016-08-22 16:08:15 -0700142}
Zoe Liu85b66462017-04-20 14:28:19 -0700143#endif // CONFIG_COMPOUND_SINGLEREF
Yaowu Xuc27fc142016-08-22 16:08:15 -0700144
145static INLINE PREDICTION_MODE compound_ref0_mode(PREDICTION_MODE mode) {
Urvang Joshi5a9ea002017-05-22 15:25:18 -0700146 static PREDICTION_MODE lut[] = {
Urvang Joshi102245d2016-11-28 13:05:36 -0800147 MB_MODE_COUNT, // DC_PRED
148 MB_MODE_COUNT, // V_PRED
149 MB_MODE_COUNT, // H_PRED
150 MB_MODE_COUNT, // D45_PRED
151 MB_MODE_COUNT, // D135_PRED
152 MB_MODE_COUNT, // D117_PRED
153 MB_MODE_COUNT, // D153_PRED
154 MB_MODE_COUNT, // D207_PRED
155 MB_MODE_COUNT, // D63_PRED
Urvang Joshi102245d2016-11-28 13:05:36 -0800156 MB_MODE_COUNT, // SMOOTH_PRED
Urvang Joshi5a9ea002017-05-22 15:25:18 -0700157#if CONFIG_SMOOTH_HV
158 MB_MODE_COUNT, // SMOOTH_V_PRED
159 MB_MODE_COUNT, // SMOOTH_H_PRED
160#endif // CONFIG_SMOOTH_HV
Urvang Joshi102245d2016-11-28 13:05:36 -0800161 MB_MODE_COUNT, // TM_PRED
162 MB_MODE_COUNT, // NEARESTMV
163 MB_MODE_COUNT, // NEARMV
164 MB_MODE_COUNT, // ZEROMV
165 MB_MODE_COUNT, // NEWMV
Zoe Liu239f06b2017-04-20 13:10:55 -0700166#if CONFIG_COMPOUND_SINGLEREF
167 NEARESTMV, // SR_NEAREST_NEARMV
Zoe Liu85b66462017-04-20 14:28:19 -0700168 // NEARESTMV, // SR_NEAREST_NEWMV
Zoe Liu239f06b2017-04-20 13:10:55 -0700169 NEARMV, // SR_NEAR_NEWMV
170 ZEROMV, // SR_ZERO_NEWMV
171 NEWMV, // SR_NEW_NEWMV
172#endif // CONFIG_COMPOUND_SINGLEREF
173 NEARESTMV, // NEAREST_NEARESTMV
Zoe Liu239f06b2017-04-20 13:10:55 -0700174 NEARMV, // NEAR_NEARMV
175 NEARESTMV, // NEAREST_NEWMV
176 NEWMV, // NEW_NEARESTMV
177 NEARMV, // NEAR_NEWMV
178 NEWMV, // NEW_NEARMV
179 ZEROMV, // ZERO_ZEROMV
180 NEWMV, // NEW_NEWMV
Yaowu Xuc27fc142016-08-22 16:08:15 -0700181 };
Urvang Joshi5a9ea002017-05-22 15:25:18 -0700182 assert(NELEMENTS(lut) == MB_MODE_COUNT);
Zoe Liu85b66462017-04-20 14:28:19 -0700183#if CONFIG_COMPOUND_SINGLEREF
184 assert(is_inter_anyref_comp_mode(mode));
185#else // !CONFIG_COMPOUND_SINGLEREF
Yaowu Xuc27fc142016-08-22 16:08:15 -0700186 assert(is_inter_compound_mode(mode));
Zoe Liu85b66462017-04-20 14:28:19 -0700187#endif // CONFIG_COMPOUND_SINGLEREF
Yaowu Xuc27fc142016-08-22 16:08:15 -0700188 return lut[mode];
189}
190
191static INLINE PREDICTION_MODE compound_ref1_mode(PREDICTION_MODE mode) {
Urvang Joshi5a9ea002017-05-22 15:25:18 -0700192 static PREDICTION_MODE lut[] = {
Urvang Joshi102245d2016-11-28 13:05:36 -0800193 MB_MODE_COUNT, // DC_PRED
194 MB_MODE_COUNT, // V_PRED
195 MB_MODE_COUNT, // H_PRED
196 MB_MODE_COUNT, // D45_PRED
197 MB_MODE_COUNT, // D135_PRED
198 MB_MODE_COUNT, // D117_PRED
199 MB_MODE_COUNT, // D153_PRED
200 MB_MODE_COUNT, // D207_PRED
201 MB_MODE_COUNT, // D63_PRED
Urvang Joshi102245d2016-11-28 13:05:36 -0800202 MB_MODE_COUNT, // SMOOTH_PRED
Urvang Joshi5a9ea002017-05-22 15:25:18 -0700203#if CONFIG_SMOOTH_HV
204 MB_MODE_COUNT, // SMOOTH_V_PRED
205 MB_MODE_COUNT, // SMOOTH_H_PRED
206#endif // CONFIG_SMOOTH_HV
Urvang Joshi102245d2016-11-28 13:05:36 -0800207 MB_MODE_COUNT, // TM_PRED
208 MB_MODE_COUNT, // NEARESTMV
209 MB_MODE_COUNT, // NEARMV
210 MB_MODE_COUNT, // ZEROMV
211 MB_MODE_COUNT, // NEWMV
Zoe Liu239f06b2017-04-20 13:10:55 -0700212#if CONFIG_COMPOUND_SINGLEREF
Zoe Liu85b66462017-04-20 14:28:19 -0700213 NEARMV, // SR_NEAREST_NEARMV
214 // NEWMV, // SR_NEAREST_NEWMV
Zoe Liu239f06b2017-04-20 13:10:55 -0700215 NEWMV, // SR_NEAR_NEWMV
216 NEWMV, // SR_ZERO_NEWMV
217 NEWMV, // SR_NEW_NEWMV
218#endif // CONFIG_COMPOUND_SINGLEREF
219 NEARESTMV, // NEAREST_NEARESTMV
Zoe Liu239f06b2017-04-20 13:10:55 -0700220 NEARMV, // NEAR_NEARMV
221 NEWMV, // NEAREST_NEWMV
222 NEARESTMV, // NEW_NEARESTMV
223 NEWMV, // NEAR_NEWMV
224 NEARMV, // NEW_NEARMV
225 ZEROMV, // ZERO_ZEROMV
226 NEWMV, // NEW_NEWMV
Yaowu Xuc27fc142016-08-22 16:08:15 -0700227 };
Urvang Joshi5a9ea002017-05-22 15:25:18 -0700228 assert(NELEMENTS(lut) == MB_MODE_COUNT);
Zoe Liu85b66462017-04-20 14:28:19 -0700229#if CONFIG_COMPOUND_SINGLEREF
230 assert(is_inter_anyref_comp_mode(mode));
231#else // !CONFIG_COMPOUND_SINGLEREF
Yaowu Xuc27fc142016-08-22 16:08:15 -0700232 assert(is_inter_compound_mode(mode));
Zoe Liu85b66462017-04-20 14:28:19 -0700233#endif // CONFIG_COMPOUND_SINGLEREF
Yaowu Xuc27fc142016-08-22 16:08:15 -0700234 return lut[mode];
235}
236
David Barker3dfba992017-04-03 16:10:09 +0100237static INLINE int have_nearmv_in_inter_mode(PREDICTION_MODE mode) {
Debargha Mukherjeebb6e1342017-04-17 16:05:04 -0700238 return (mode == NEARMV || mode == NEAR_NEARMV || mode == NEAR_NEWMV ||
Zoe Liu85b66462017-04-20 14:28:19 -0700239#if CONFIG_COMPOUND_SINGLEREF
240 mode == SR_NEAREST_NEARMV || mode == SR_NEAR_NEWMV ||
241#endif // CONFIG_COMPOUND_SINGLEREF
Debargha Mukherjeebb6e1342017-04-17 16:05:04 -0700242 mode == NEW_NEARMV);
David Barker3dfba992017-04-03 16:10:09 +0100243}
244
Yaowu Xuc27fc142016-08-22 16:08:15 -0700245static INLINE int have_newmv_in_inter_mode(PREDICTION_MODE mode) {
Zoe Liu7f24e1b2017-03-17 17:42:05 -0700246 return (mode == NEWMV || mode == NEW_NEWMV || mode == NEAREST_NEWMV ||
Zoe Liu85b66462017-04-20 14:28:19 -0700247#if CONFIG_COMPOUND_SINGLEREF
248 /* mode == SR_NEAREST_NEWMV || */ mode == SR_NEAR_NEWMV ||
249 mode == SR_ZERO_NEWMV || mode == SR_NEW_NEWMV ||
250#endif // CONFIG_COMPOUND_SINGLEREF
Zoe Liu7f24e1b2017-03-17 17:42:05 -0700251 mode == NEW_NEARESTMV || mode == NEAR_NEWMV || mode == NEW_NEARMV);
Yaowu Xuc27fc142016-08-22 16:08:15 -0700252}
Sarah Parker6fdc8532016-11-16 17:47:13 -0800253
Sarah Parker2e604882017-01-17 17:31:25 -0800254static INLINE int use_masked_motion_search(COMPOUND_TYPE type) {
Debargha Mukherjeec5f735f2017-04-26 03:25:28 +0000255#if CONFIG_WEDGE
Sarah Parker2e604882017-01-17 17:31:25 -0800256 return (type == COMPOUND_WEDGE);
Debargha Mukherjeec5f735f2017-04-26 03:25:28 +0000257#else
258 (void)type;
259 return 0;
260#endif
Sarah Parker2e604882017-01-17 17:31:25 -0800261}
262
Sarah Parker6fdc8532016-11-16 17:47:13 -0800263static INLINE int is_masked_compound_type(COMPOUND_TYPE type) {
Debargha Mukherjeec5f735f2017-04-26 03:25:28 +0000264#if CONFIG_COMPOUND_SEGMENT && CONFIG_WEDGE
Sarah Parker569edda2016-12-14 14:57:38 -0800265 return (type == COMPOUND_WEDGE || type == COMPOUND_SEG);
Debargha Mukherjeec5f735f2017-04-26 03:25:28 +0000266#elif !CONFIG_COMPOUND_SEGMENT && CONFIG_WEDGE
Sarah Parker6fdc8532016-11-16 17:47:13 -0800267 return (type == COMPOUND_WEDGE);
Debargha Mukherjeec5f735f2017-04-26 03:25:28 +0000268#elif CONFIG_COMPOUND_SEGMENT && !CONFIG_WEDGE
269 return (type == COMPOUND_SEG);
Sarah Parker569edda2016-12-14 14:57:38 -0800270#endif // CONFIG_COMPOUND_SEGMENT
Debargha Mukherjeec5f735f2017-04-26 03:25:28 +0000271 (void)type;
272 return 0;
Sarah Parker6fdc8532016-11-16 17:47:13 -0800273}
Zoe Liu85b66462017-04-20 14:28:19 -0700274
275#else // !CONFIG_EXT_INTER
Yaowu Xuc27fc142016-08-22 16:08:15 -0700276
David Barker3dfba992017-04-03 16:10:09 +0100277static INLINE int have_nearmv_in_inter_mode(PREDICTION_MODE mode) {
278 return (mode == NEARMV);
279}
280
Yaowu Xuc27fc142016-08-22 16:08:15 -0700281static INLINE int have_newmv_in_inter_mode(PREDICTION_MODE mode) {
282 return (mode == NEWMV);
283}
284#endif // CONFIG_EXT_INTER
285
286/* For keyframes, intra block modes are predicted by the (already decoded)
287 modes for the Y blocks to the left and above us; for interframes, there
288 is a single probability table. */
289
290typedef struct {
291 PREDICTION_MODE as_mode;
292 int_mv as_mv[2]; // first, second inter predictor motion vectors
Yaowu Xuf5bbbfa2016-09-26 09:13:38 -0700293 int_mv pred_mv[2];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700294#if CONFIG_EXT_INTER
295 int_mv ref_mv[2];
296#endif // CONFIG_EXT_INTER
297} b_mode_info;
298
299typedef int8_t MV_REFERENCE_FRAME;
300
301typedef struct {
302 // Number of base colors for Y (0) and UV (1)
303 uint8_t palette_size[2];
hui sufa4ff852017-05-15 12:20:50 -0700304 // Value of base colors for Y, U, and V
Yaowu Xuc27fc142016-08-22 16:08:15 -0700305 uint16_t palette_colors[3 * PALETTE_MAX_SIZE];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700306} PALETTE_MODE_INFO;
307
hui su5db97432016-10-14 16:10:14 -0700308#if CONFIG_FILTER_INTRA
Yue Chen8d8638a2017-02-21 13:28:16 +0800309#define USE_3TAP_INTRA_FILTER 1 // 0: 4-tap; 1: 3-tap
Yaowu Xuc27fc142016-08-22 16:08:15 -0700310typedef struct {
311 // 1: an ext intra mode is used; 0: otherwise.
hui su5db97432016-10-14 16:10:14 -0700312 uint8_t use_filter_intra_mode[PLANE_TYPES];
313 FILTER_INTRA_MODE filter_intra_mode[PLANE_TYPES];
314} FILTER_INTRA_MODE_INFO;
315#endif // CONFIG_FILTER_INTRA
Yaowu Xuc27fc142016-08-22 16:08:15 -0700316
Angie Chiang9a44f5f2016-11-06 12:19:06 -0800317#if CONFIG_VAR_TX
Angie Chiang7c2b7f22016-11-07 16:00:00 -0800318#if CONFIG_RD_DEBUG
Angie Chiang9a44f5f2016-11-06 12:19:06 -0800319#define TXB_COEFF_COST_MAP_SIZE (2 * MAX_MIB_SIZE)
Angie Chiang7c2b7f22016-11-07 16:00:00 -0800320#endif
321#endif
Angie Chiang9a44f5f2016-11-06 12:19:06 -0800322
Angie Chiang9a44f5f2016-11-06 12:19:06 -0800323typedef struct RD_STATS {
324 int rate;
325 int64_t dist;
Angie Chiang7bbd3b12017-04-26 11:06:09 -0700326 // Please be careful of using rdcost, it's not guaranteed to be set all the
327 // time.
328 // TODO(angiebird): Create a set of functions to manipulate the RD_STATS. In
329 // these functions, make sure rdcost is always up-to-date according to
330 // rate/dist.
Angie Chiang2a2a7dd2017-04-25 16:08:47 -0700331 int64_t rdcost;
Angie Chiang9a44f5f2016-11-06 12:19:06 -0800332 int64_t sse;
Angie Chiang7bbd3b12017-04-26 11:06:09 -0700333 int skip; // sse should equal to dist when skip == 1
Jingning Han3bce7542017-07-25 10:53:57 -0700334 int64_t ref_rdcost;
335 int zero_rate;
Jingning Han1a7f0a82017-07-27 09:48:05 -0700336 uint8_t invalid_rate;
Yushin Chob7b60c52017-07-14 16:18:52 -0700337#if CONFIG_DIST_8X8 && CONFIG_CB4X4
Yushin Cho63927c42017-05-23 15:41:05 -0700338 int64_t dist_y;
339#endif
Angie Chiang9a44f5f2016-11-06 12:19:06 -0800340#if CONFIG_RD_DEBUG
341 int txb_coeff_cost[MAX_MB_PLANE];
Angie Chiang3963d632016-11-10 18:41:40 -0800342#if CONFIG_VAR_TX
Angie Chiang9a44f5f2016-11-06 12:19:06 -0800343 int txb_coeff_cost_map[MAX_MB_PLANE][TXB_COEFF_COST_MAP_SIZE]
344 [TXB_COEFF_COST_MAP_SIZE];
Angie Chiang3963d632016-11-10 18:41:40 -0800345#endif // CONFIG_VAR_TX
346#endif // CONFIG_RD_DEBUG
Angie Chiang9a44f5f2016-11-06 12:19:06 -0800347} RD_STATS;
Angie Chiang9a44f5f2016-11-06 12:19:06 -0800348
Sarah Parker6fdc8532016-11-16 17:47:13 -0800349#if CONFIG_EXT_INTER
Sarah Parker2d0e9b72017-05-04 01:34:16 +0000350// This struct is used to group function args that are commonly
351// sent together in functions related to interinter compound modes
Sarah Parker6fdc8532016-11-16 17:47:13 -0800352typedef struct {
Sarah Parker2d0e9b72017-05-04 01:34:16 +0000353#if CONFIG_WEDGE
Sarah Parker6fdc8532016-11-16 17:47:13 -0800354 int wedge_index;
355 int wedge_sign;
Sarah Parker2d0e9b72017-05-04 01:34:16 +0000356#endif // CONFIG_WEDGE
Sarah Parker569edda2016-12-14 14:57:38 -0800357#if CONFIG_COMPOUND_SEGMENT
Sarah Parkerb9f757c2017-01-06 17:12:24 -0800358 SEG_MASK_TYPE mask_type;
Sarah Parker2d0e9b72017-05-04 01:34:16 +0000359 uint8_t *seg_mask;
Sarah Parker569edda2016-12-14 14:57:38 -0800360#endif // CONFIG_COMPOUND_SEGMENT
Sarah Parker2d0e9b72017-05-04 01:34:16 +0000361 COMPOUND_TYPE interinter_compound_type;
Sarah Parker6fdc8532016-11-16 17:47:13 -0800362} INTERINTER_COMPOUND_DATA;
363#endif // CONFIG_EXT_INTER
364
Yaowu Xuc27fc142016-08-22 16:08:15 -0700365// This structure now relates to 8x8 block regions.
Luc Trudeauf5334002017-04-25 12:21:26 -0400366typedef struct MB_MODE_INFO {
Yaowu Xuc27fc142016-08-22 16:08:15 -0700367 // Common for both INTER and INTRA blocks
368 BLOCK_SIZE sb_type;
369 PREDICTION_MODE mode;
370 TX_SIZE tx_size;
371#if CONFIG_VAR_TX
372 // TODO(jingning): This effectively assigned a separate entry for each
373 // 8x8 block. Apparently it takes much more space than needed.
374 TX_SIZE inter_tx_size[MAX_MIB_SIZE][MAX_MIB_SIZE];
Jingning Hane67b38a2016-11-04 10:30:00 -0700375 TX_SIZE min_tx_size;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700376#endif
377 int8_t skip;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700378 int8_t segment_id;
379#if CONFIG_SUPERTX
380 // Minimum of all segment IDs under the current supertx block.
381 int8_t segment_id_supertx;
382#endif // CONFIG_SUPERTX
383 int8_t seg_id_predicted; // valid only when temporal_update is enabled
384
Sarah Parkerc5ccd4c2017-08-03 11:27:50 -0700385#if CONFIG_MRC_TX
386 int valid_mrc_mask;
387#endif // CONFIG_MRC_TX
388
Yaowu Xuc27fc142016-08-22 16:08:15 -0700389 // Only for INTRA blocks
Luc Trudeaud6d9eee2017-07-12 12:36:50 -0400390 UV_PREDICTION_MODE uv_mode;
Urvang Joshic6300aa2017-06-01 14:46:23 -0700391
Yaowu Xuc27fc142016-08-22 16:08:15 -0700392 PALETTE_MODE_INFO palette_mode_info;
Alex Converse28744302017-04-13 14:46:22 -0700393#if CONFIG_INTRABC
394 uint8_t use_intrabc;
395#endif // CONFIG_INTRABC
Yaowu Xuc27fc142016-08-22 16:08:15 -0700396
397// Only for INTER blocks
398#if CONFIG_DUAL_FILTER
James Zern7b9407a2016-05-18 23:48:05 -0700399 InterpFilter interp_filter[4];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700400#else
James Zern7b9407a2016-05-18 23:48:05 -0700401 InterpFilter interp_filter;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700402#endif
403 MV_REFERENCE_FRAME ref_frame[2];
404 TX_TYPE tx_type;
Angie Chiangcd9b03f2017-04-16 13:37:13 -0700405#if CONFIG_TXK_SEL
Angie Chiang808d8592017-04-06 18:36:55 -0700406 TX_TYPE txk_type[MAX_SB_SQUARE / (TX_SIZE_W_MIN * TX_SIZE_H_MIN)];
407#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -0700408
hui su5db97432016-10-14 16:10:14 -0700409#if CONFIG_FILTER_INTRA
410 FILTER_INTRA_MODE_INFO filter_intra_mode_info;
411#endif // CONFIG_FILTER_INTRA
Yaowu Xuc27fc142016-08-22 16:08:15 -0700412#if CONFIG_EXT_INTRA
hui sueda3d762016-12-06 16:58:23 -0800413 // The actual prediction angle is the base angle + (angle_delta * step).
Yaowu Xuc27fc142016-08-22 16:08:15 -0700414 int8_t angle_delta[2];
hui sueda3d762016-12-06 16:58:23 -0800415#if CONFIG_INTRA_INTERP
Yaowu Xuc27fc142016-08-22 16:08:15 -0700416 // To-Do (huisu): this may be replaced by interp_filter
417 INTRA_FILTER intra_filter;
hui sueda3d762016-12-06 16:58:23 -0800418#endif // CONFIG_INTRA_INTERP
Yaowu Xuc27fc142016-08-22 16:08:15 -0700419#endif // CONFIG_EXT_INTRA
420
421#if CONFIG_EXT_INTER
Yue Chen4d26acb2017-05-01 12:28:34 -0700422#if CONFIG_INTERINTRA
Sarah Parker2d0e9b72017-05-04 01:34:16 +0000423 // interintra members
Yaowu Xuc27fc142016-08-22 16:08:15 -0700424 INTERINTRA_MODE interintra_mode;
Yue Chen4d26acb2017-05-01 12:28:34 -0700425#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -0700426 // TODO(debargha): Consolidate these flags
427 int use_wedge_interintra;
428 int interintra_wedge_index;
429 int interintra_wedge_sign;
Sarah Parker2d0e9b72017-05-04 01:34:16 +0000430 // interinter members
431 COMPOUND_TYPE interinter_compound_type;
432#if CONFIG_WEDGE
433 int wedge_index;
434 int wedge_sign;
435#endif // CONFIG_WEDGE
436#if CONFIG_COMPOUND_SEGMENT
437 SEG_MASK_TYPE mask_type;
438#endif // CONFIG_COMPOUND_SEGMENT
Yaowu Xuc27fc142016-08-22 16:08:15 -0700439#endif // CONFIG_EXT_INTER
Yue Chencb60b182016-10-13 15:18:22 -0700440 MOTION_MODE motion_mode;
Yue Chen5329a2b2017-02-28 17:33:00 +0800441#if CONFIG_MOTION_VAR
442 int overlappable_neighbors[2];
Wei-Ting Lin85a8f702017-06-22 13:55:15 -0700443#if CONFIG_NCOBMC_ADAPT_WEIGHT
444 // Applying different weighting kernels in ncobmc
445 // In current implementation, interpolation modes only defined for squared
446 // blocks. A rectangular block is divided into two squared blocks and each
447 // squared block has an interpolation mode.
448 NCOBMC_MODE ncobmc_mode[2];
Wei-Ting Lin482551b2017-08-03 12:29:24 -0700449#endif // CONFIG_NCOBMC_ADAPT_WEIGHT
Yue Chen5329a2b2017-02-28 17:33:00 +0800450#endif // CONFIG_MOTION_VAR
Yaowu Xuc27fc142016-08-22 16:08:15 -0700451 int_mv mv[2];
452 int_mv pred_mv[2];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700453 uint8_t ref_mv_idx;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700454#if CONFIG_EXT_PARTITION_TYPES
455 PARTITION_TYPE partition;
456#endif
457#if CONFIG_NEW_QUANT
458 int dq_off_index;
459 int send_dq_bit;
460#endif // CONFIG_NEW_QUANT
461 /* deringing gain *per-superblock* */
Jean-Marc Valin5f5c1322017-03-21 16:20:21 -0400462 int8_t cdef_strength;
Arild Fuldseth07441162016-08-15 15:07:52 +0200463#if CONFIG_DELTA_Q
464 int current_q_index;
Fangwen Fu231fe422017-04-24 17:52:29 -0700465#if CONFIG_EXT_DELTA_Q
466 int current_delta_lf_from_base;
467#endif
Arild Fuldseth07441162016-08-15 15:07:52 +0200468#endif
Angie Chiangd4022822016-11-02 18:30:25 -0700469#if CONFIG_RD_DEBUG
Angie Chiang9a44f5f2016-11-06 12:19:06 -0800470 RD_STATS rd_stats;
Angie Chiangd4022822016-11-02 18:30:25 -0700471 int mi_row;
472 int mi_col;
473#endif
Yue Chen69f18e12016-09-08 14:48:15 -0700474#if CONFIG_WARPED_MOTION
475 int num_proj_ref[2];
476 WarpedMotionParams wm_params[2];
477#endif // CONFIG_WARPED_MOTION
Ryan Lei9b02b0e2017-01-30 15:52:20 -0800478
Luc Trudeauf5334002017-04-25 12:21:26 -0400479#if CONFIG_CFL
480 // Index of the alpha Cb and alpha Cr combination
Luc Trudeaua9bd85f2017-05-11 14:37:56 -0400481 int cfl_alpha_idx;
David Michael Barrf6eaa152017-07-19 19:42:28 +0900482 // Joint sign of alpha Cb and alpha Cr
483 int cfl_alpha_signs;
Luc Trudeauf5334002017-04-25 12:21:26 -0400484#endif
485
Ryan Lei9b02b0e2017-01-30 15:52:20 -0800486 BOUNDARY_TYPE boundary_info;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700487} MB_MODE_INFO;
488
489typedef struct MODE_INFO {
490 MB_MODE_INFO mbmi;
491 b_mode_info bmi[4];
492} MODE_INFO;
493
Alex Converse28744302017-04-13 14:46:22 -0700494#if CONFIG_INTRABC
495static INLINE int is_intrabc_block(const MB_MODE_INFO *mbmi) {
496 return mbmi->use_intrabc;
497}
498#endif
499
Yaowu Xuc27fc142016-08-22 16:08:15 -0700500static INLINE PREDICTION_MODE get_y_mode(const MODE_INFO *mi, int block) {
Jingning Hand7d20472016-12-14 11:13:48 -0800501#if CONFIG_CB4X4
502 (void)block;
503 return mi->mbmi.mode;
504#else
Yaowu Xuc27fc142016-08-22 16:08:15 -0700505 return mi->mbmi.sb_type < BLOCK_8X8 ? mi->bmi[block].as_mode : mi->mbmi.mode;
Jingning Hand7d20472016-12-14 11:13:48 -0800506#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -0700507}
508
Luc Trudeaud6d9eee2017-07-12 12:36:50 -0400509#if CONFIG_CFL
510static INLINE PREDICTION_MODE get_uv_mode(UV_PREDICTION_MODE mode) {
511 static const PREDICTION_MODE uv2y[UV_INTRA_MODES] = {
Urvang Joshi93b543a2017-06-01 17:32:41 -0700512 DC_PRED, // UV_DC_PRED
513 V_PRED, // UV_V_PRED
514 H_PRED, // UV_H_PRED
515 D45_PRED, // UV_D45_PRED
516 D135_PRED, // UV_D135_PRED
517 D117_PRED, // UV_D117_PRED
518 D153_PRED, // UV_D153_PRED
519 D207_PRED, // UV_D207_PRED
520 D63_PRED, // UV_D63_PRED
Luc Trudeaud6d9eee2017-07-12 12:36:50 -0400521 SMOOTH_PRED, // UV_SMOOTH_PRED
522#if CONFIG_SMOOTH_HV
523 SMOOTH_V_PRED, // UV_SMOOTH_V_PRED
524 SMOOTH_H_PRED, // UV_SMOOTH_H_PRED
525#endif // CONFIG_SMOOTH_HV
Luc Trudeaud6d9eee2017-07-12 12:36:50 -0400526 TM_PRED, // UV_TM_PRED
Luc Trudeau6e1cd782017-06-21 13:52:36 -0400527 DC_PRED, // CFL_PRED
Luc Trudeaud6d9eee2017-07-12 12:36:50 -0400528 };
529 return uv2y[mode];
530}
531#else
532static INLINE PREDICTION_MODE get_uv_mode(PREDICTION_MODE mode) { return mode; }
533#endif // CONFIG_CFL
534
Yaowu Xuc27fc142016-08-22 16:08:15 -0700535static INLINE int is_inter_block(const MB_MODE_INFO *mbmi) {
Alex Converse28744302017-04-13 14:46:22 -0700536#if CONFIG_INTRABC
537 if (is_intrabc_block(mbmi)) return 1;
538#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -0700539 return mbmi->ref_frame[0] > INTRA_FRAME;
540}
541
542static INLINE int has_second_ref(const MB_MODE_INFO *mbmi) {
543 return mbmi->ref_frame[1] > INTRA_FRAME;
544}
545
Zoe Liuc082bbc2017-05-17 13:31:37 -0700546#if CONFIG_EXT_COMP_REFS
547static INLINE int has_uni_comp_refs(const MB_MODE_INFO *mbmi) {
548 return has_second_ref(mbmi) && (!((mbmi->ref_frame[0] >= BWDREF_FRAME) ^
549 (mbmi->ref_frame[1] >= BWDREF_FRAME)));
550}
551
552static INLINE MV_REFERENCE_FRAME comp_ref0(int ref_idx) {
553 static const MV_REFERENCE_FRAME lut[] = {
554 LAST_FRAME, // LAST_LAST2_FRAMES,
Zoe Liufcf5fa22017-06-26 16:00:38 -0700555 LAST_FRAME, // LAST_LAST3_FRAMES,
Zoe Liuc082bbc2017-05-17 13:31:37 -0700556 LAST_FRAME, // LAST_GOLDEN_FRAMES,
557 BWDREF_FRAME, // BWDREF_ALTREF_FRAMES,
558 };
559 assert(NELEMENTS(lut) == UNIDIR_COMP_REFS);
560 return lut[ref_idx];
561}
562
563static INLINE MV_REFERENCE_FRAME comp_ref1(int ref_idx) {
564 static const MV_REFERENCE_FRAME lut[] = {
565 LAST2_FRAME, // LAST_LAST2_FRAMES,
Zoe Liufcf5fa22017-06-26 16:00:38 -0700566 LAST3_FRAME, // LAST_LAST3_FRAMES,
Zoe Liuc082bbc2017-05-17 13:31:37 -0700567 GOLDEN_FRAME, // LAST_GOLDEN_FRAMES,
568 ALTREF_FRAME, // BWDREF_ALTREF_FRAMES,
569 };
570 assert(NELEMENTS(lut) == UNIDIR_COMP_REFS);
571 return lut[ref_idx];
572}
573#endif // CONFIG_EXT_COMP_REFS
574
Yaowu Xuf883b422016-08-30 14:01:10 -0700575PREDICTION_MODE av1_left_block_mode(const MODE_INFO *cur_mi,
576 const MODE_INFO *left_mi, int b);
Yaowu Xuc27fc142016-08-22 16:08:15 -0700577
Yaowu Xuf883b422016-08-30 14:01:10 -0700578PREDICTION_MODE av1_above_block_mode(const MODE_INFO *cur_mi,
579 const MODE_INFO *above_mi, int b);
Yaowu Xuc27fc142016-08-22 16:08:15 -0700580
Sarah Parker19234cc2017-03-10 16:43:25 -0800581#if CONFIG_GLOBAL_MOTION
582static INLINE int is_global_mv_block(const MODE_INFO *mi, int block,
583 TransformationType type) {
584 PREDICTION_MODE mode = get_y_mode(mi, block);
585#if GLOBAL_SUB8X8_USED
586 const int block_size_allowed = 1;
587#else
588 const BLOCK_SIZE bsize = mi->mbmi.sb_type;
Debargha Mukherjeec17a4432017-08-28 22:22:45 -0700589 const int block_size_allowed =
590 AOMMIN(block_size_wide[bsize], block_size_high[bsize]) >= 8;
Sarah Parker19234cc2017-03-10 16:43:25 -0800591#endif // GLOBAL_SUB8X8_USED
592#if CONFIG_EXT_INTER
593 return (mode == ZEROMV || mode == ZERO_ZEROMV) && type > TRANSLATION &&
594 block_size_allowed;
595#else
596 return mode == ZEROMV && type > TRANSLATION && block_size_allowed;
597#endif // CONFIG_EXT_INTER
598}
599#endif // CONFIG_GLOBAL_MOTION
600
Yaowu Xuc27fc142016-08-22 16:08:15 -0700601enum mv_precision { MV_PRECISION_Q3, MV_PRECISION_Q4 };
602
603struct buf_2d {
604 uint8_t *buf;
605 uint8_t *buf0;
606 int width;
607 int height;
608 int stride;
609};
610
611typedef struct macroblockd_plane {
612 tran_low_t *dqcoeff;
613 PLANE_TYPE plane_type;
614 int subsampling_x;
615 int subsampling_y;
616 struct buf_2d dst;
617 struct buf_2d pre[2];
618 ENTROPY_CONTEXT *above_context;
619 ENTROPY_CONTEXT *left_context;
620 int16_t seg_dequant[MAX_SEGMENTS][2];
621#if CONFIG_NEW_QUANT
clang-format67948d32016-09-07 22:40:40 -0700622 dequant_val_type_nuq seg_dequant_nuq[MAX_SEGMENTS][QUANT_PROFILES]
623 [COEF_BANDS];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700624#endif
625 uint8_t *color_index_map;
626
627 // number of 4x4s in current block
628 uint16_t n4_w, n4_h;
629 // log2 of n4_w, n4_h
630 uint8_t n4_wl, n4_hl;
Jingning Hanc47fe6c2016-10-21 16:40:47 -0700631 // block size in pixels
632 uint8_t width, height;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700633
634#if CONFIG_AOM_QM
Thomas Davies6675adf2017-05-04 17:39:21 +0100635 const qm_val_t *seg_iqmatrix[MAX_SEGMENTS][2][TX_SIZES_ALL];
636 const qm_val_t *seg_qmatrix[MAX_SEGMENTS][2][TX_SIZES_ALL];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700637#endif
638 // encoder
639 const int16_t *dequant;
640#if CONFIG_NEW_QUANT
641 const dequant_val_type_nuq *dequant_val_nuq[QUANT_PROFILES];
642#endif // CONFIG_NEW_QUANT
Yushin Cho77bba8d2016-11-04 16:36:56 -0700643
Yushin Chob7b60c52017-07-14 16:18:52 -0700644#if CONFIG_PVQ || CONFIG_DIST_8X8
Yushin Cho77bba8d2016-11-04 16:36:56 -0700645 DECLARE_ALIGNED(16, int16_t, pred[MAX_SB_SQUARE]);
Yushin Chob7b60c52017-07-14 16:18:52 -0700646#endif
647#if CONFIG_PVQ
Yushin Cho77bba8d2016-11-04 16:36:56 -0700648 // PVQ: forward transformed predicted image, a reference for PVQ.
649 tran_low_t *pvq_ref_coeff;
650#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -0700651} MACROBLOCKD_PLANE;
652
Jingning Han3468df12016-12-05 17:53:16 -0800653#define BLOCK_OFFSET(x, i) \
654 ((x) + (i) * (1 << (tx_size_wide_log2[0] + tx_size_high_log2[0])))
Yaowu Xuc27fc142016-08-22 16:08:15 -0700655
656typedef struct RefBuffer {
Yaowu Xuc27fc142016-08-22 16:08:15 -0700657 int idx;
658 YV12_BUFFER_CONFIG *buf;
659 struct scale_factors sf;
Zoe Liu7b1ec7a2017-05-24 22:28:24 -0700660#if CONFIG_VAR_REFS
Zoe Liu7b1ec7a2017-05-24 22:28:24 -0700661 int is_valid;
662#endif // CONFIG_VAR_REFS
Yaowu Xuc27fc142016-08-22 16:08:15 -0700663} RefBuffer;
664
Yi Luo2ab63cb2017-05-11 16:44:22 -0700665#if CONFIG_ADAPT_SCAN
666typedef int16_t EobThresholdMD[TX_TYPES][EOB_THRESHOLD_NUM];
667#endif
Luc Trudeauf8164152017-04-11 16:20:51 -0400668
Rupert Swarbrick6c545212017-09-01 17:17:25 +0100669#if CONFIG_LOOP_RESTORATION
670typedef struct {
671 DECLARE_ALIGNED(16, InterpKernel, vfilter);
672 DECLARE_ALIGNED(16, InterpKernel, hfilter);
673} WienerInfo;
674
675typedef struct {
676 int ep;
677 int xqd[2];
678} SgrprojInfo;
679#endif // CONFIG_LOOP_RESTORATION
680
Yaowu Xuc27fc142016-08-22 16:08:15 -0700681typedef struct macroblockd {
682 struct macroblockd_plane plane[MAX_MB_PLANE];
683 uint8_t bmode_blocks_wl;
684 uint8_t bmode_blocks_hl;
685
686 FRAME_COUNTS *counts;
687 TileInfo tile;
688
689 int mi_stride;
690
691 MODE_INFO **mi;
692 MODE_INFO *left_mi;
693 MODE_INFO *above_mi;
694 MB_MODE_INFO *left_mbmi;
695 MB_MODE_INFO *above_mbmi;
696
697 int up_available;
698 int left_available;
Jingning Han3da18d62017-05-02 12:43:58 -0700699#if CONFIG_CHROMA_SUB8X8
700 int chroma_up_available;
701 int chroma_left_available;
702#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -0700703
Yaowu Xuf883b422016-08-30 14:01:10 -0700704 const aom_prob (*partition_probs)[PARTITION_TYPES - 1];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700705
Rupert Swarbrick57486c52017-08-14 10:53:49 +0100706 /* Distance of MB away from frame edges in subpixels (1/8th pixel) */
Yaowu Xuc27fc142016-08-22 16:08:15 -0700707 int mb_to_left_edge;
708 int mb_to_right_edge;
709 int mb_to_top_edge;
710 int mb_to_bottom_edge;
711
712 FRAME_CONTEXT *fc;
713
714 /* pointers to reference frames */
Urvang Joshi52648442016-10-13 17:27:51 -0700715 const RefBuffer *block_refs[2];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700716
717 /* pointer to current frame */
718 const YV12_BUFFER_CONFIG *cur_buf;
719
Alex Conversee816b312017-05-01 09:51:24 -0700720#if CONFIG_INTRABC
721 /* Scale of the current frame with respect to itself */
722 struct scale_factors sf_identity;
723#endif
724
Yaowu Xuc27fc142016-08-22 16:08:15 -0700725 ENTROPY_CONTEXT *above_context[MAX_MB_PLANE];
726 ENTROPY_CONTEXT left_context[MAX_MB_PLANE][2 * MAX_MIB_SIZE];
727
728 PARTITION_CONTEXT *above_seg_context;
729 PARTITION_CONTEXT left_seg_context[MAX_MIB_SIZE];
730
731#if CONFIG_VAR_TX
732 TXFM_CONTEXT *above_txfm_context;
733 TXFM_CONTEXT *left_txfm_context;
Jingning Han331662e2017-05-30 17:03:32 -0700734 TXFM_CONTEXT left_txfm_context_buffer[2 * MAX_MIB_SIZE];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700735
736 TX_SIZE max_tx_size;
737#if CONFIG_SUPERTX
738 TX_SIZE supertx_size;
739#endif
740#endif
741
Rupert Swarbrick6c545212017-09-01 17:17:25 +0100742#if CONFIG_LOOP_RESTORATION
743 WienerInfo wiener_info[MAX_MB_PLANE];
744 SgrprojInfo sgrproj_info[MAX_MB_PLANE];
745#endif // CONFIG_LOOP_RESTORATION
746
Jingning Hanff6ee6a2016-12-07 09:55:21 -0800747 // block dimension in the unit of mode_info.
Yaowu Xuc27fc142016-08-22 16:08:15 -0700748 uint8_t n8_w, n8_h;
749
Yaowu Xuc27fc142016-08-22 16:08:15 -0700750 uint8_t ref_mv_count[MODE_CTX_REF_FRAMES];
751 CANDIDATE_MV ref_mv_stack[MODE_CTX_REF_FRAMES][MAX_REF_MV_STACK_SIZE];
752 uint8_t is_sec_rect;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700753
Yushin Cho77bba8d2016-11-04 16:36:56 -0700754#if CONFIG_PVQ
755 daala_dec_ctx daala_dec;
756#endif
Thomas Daviesf77d4ad2017-01-10 18:55:42 +0000757 FRAME_CONTEXT *tile_ctx;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700758 /* Bit depth: 8, 10, 12 */
759 int bd;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700760
Debargha Mukherjee3c42c092016-09-29 09:17:36 -0700761 int qindex[MAX_SEGMENTS];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700762 int lossless[MAX_SEGMENTS];
763 int corrupted;
764
Yaowu Xuf883b422016-08-30 14:01:10 -0700765 struct aom_internal_error_info *error_info;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700766#if CONFIG_GLOBAL_MOTION
David Barkercf3d0b02016-11-10 10:14:49 +0000767 WarpedMotionParams *global_motion;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700768#endif // CONFIG_GLOBAL_MOTION
Arild Fuldseth07441162016-08-15 15:07:52 +0200769#if CONFIG_DELTA_Q
770 int prev_qindex;
771 int delta_qindex;
772 int current_qindex;
Fangwen Fu231fe422017-04-24 17:52:29 -0700773#if CONFIG_EXT_DELTA_Q
774 // Since actual frame level loop filtering level value is not available
775 // at the beginning of the tile (only available during actual filtering)
776 // at encoder side.we record the delta_lf (against the frame level loop
777 // filtering level) and code the delta between previous superblock's delta
778 // lf and current delta lf. It is equivalent to the delta between previous
779 // superblock's actual lf and current lf.
780 int prev_delta_lf_from_base;
781 int current_delta_lf_from_base;
782#endif
Arild Fuldseth07441162016-08-15 15:07:52 +0200783#endif
Yi Luof8e87b42017-04-14 17:20:27 -0700784#if CONFIG_ADAPT_SCAN
785 const EobThresholdMD *eob_threshold_md;
786#endif
Luc Trudeauf8164152017-04-11 16:20:51 -0400787
Sarah Parker2d0e9b72017-05-04 01:34:16 +0000788#if CONFIG_EXT_INTER && CONFIG_COMPOUND_SEGMENT
789 DECLARE_ALIGNED(16, uint8_t, seg_mask[2 * MAX_SB_SQUARE]);
790#endif // CONFIG_EXT_INTER && CONFIG_COMPOUND_SEGMENT
791
Sarah Parker5c6744b2017-08-25 17:27:45 -0700792#if CONFIG_MRC_TX
793 uint8_t *mrc_mask;
794#endif // CONFIG_MRC_TX
795
Luc Trudeauf8164152017-04-11 16:20:51 -0400796#if CONFIG_CFL
797 CFL_CTX *cfl;
798#endif
Wei-Ting Lin482551b2017-08-03 12:29:24 -0700799
800#if CONFIG_NCOBMC_ADAPT_WEIGHT
801 uint8_t *ncobmc_pred_buf[MAX_MB_PLANE];
802 int ncobmc_pred_buf_stride[MAX_MB_PLANE];
803 SB_MI_BD sb_mi_bd;
804#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -0700805} MACROBLOCKD;
806
Yi Luo51281092017-06-26 16:36:15 -0700807static INLINE int get_bitdepth_data_path_index(const MACROBLOCKD *xd) {
808 return xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH ? 1 : 0;
809}
810
Yaowu Xuc27fc142016-08-22 16:08:15 -0700811static INLINE BLOCK_SIZE get_subsize(BLOCK_SIZE bsize,
812 PARTITION_TYPE partition) {
813 if (partition == PARTITION_INVALID)
Urvang Joshicb586f32016-09-20 11:36:33 -0700814 return BLOCK_INVALID;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700815 else
816 return subsize_lookup[partition][bsize];
817}
818
819static const TX_TYPE intra_mode_to_tx_type_context[INTRA_MODES] = {
820 DCT_DCT, // DC
821 ADST_DCT, // V
822 DCT_ADST, // H
823 DCT_DCT, // D45
824 ADST_ADST, // D135
825 ADST_DCT, // D117
826 DCT_ADST, // D153
827 DCT_ADST, // D207
828 ADST_DCT, // D63
Urvang Joshi6be4a542016-11-03 15:24:05 -0700829 ADST_ADST, // SMOOTH
Urvang Joshie6ca8e82017-03-15 14:57:41 -0700830#if CONFIG_SMOOTH_HV
831 ADST_DCT, // SMOOTH_V
832 DCT_ADST, // SMOOTH_H
833#endif // CONFIG_SMOOTH_HV
Yaowu Xuc27fc142016-08-22 16:08:15 -0700834 ADST_ADST, // TM
835};
836
837#if CONFIG_SUPERTX
838static INLINE int supertx_enabled(const MB_MODE_INFO *mbmi) {
Jingning Han93531242016-12-20 11:54:36 -0800839 TX_SIZE max_tx_size = txsize_sqr_map[mbmi->tx_size];
840 return tx_size_wide[max_tx_size] >
841 AOMMIN(block_size_wide[mbmi->sb_type], block_size_high[mbmi->sb_type]);
Yaowu Xuc27fc142016-08-22 16:08:15 -0700842}
843#endif // CONFIG_SUPERTX
844
Jingning Hanb83e64b2017-03-01 14:52:04 -0800845#define USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4 1
Debargha Mukherjee5a488a62016-11-22 22:24:10 -0800846
Sarah Parker076437f2017-03-14 17:39:53 -0700847#if CONFIG_RECT_TX
Sarah Parker076437f2017-03-14 17:39:53 -0700848static INLINE int is_rect_tx(TX_SIZE tx_size) { return tx_size >= TX_SIZES; }
849#endif // CONFIG_RECT_TX
850
Sarah Parker2e08d962017-08-01 19:51:20 -0700851#if CONFIG_MRC_TX
852#define USE_MRC_INTRA 0
853#define USE_MRC_INTER 1
854#endif // CONFIG_MRC_TX
855
Jingning Hanb83e64b2017-03-01 14:52:04 -0800856#if CONFIG_EXT_TX
Yaowu Xuc27fc142016-08-22 16:08:15 -0700857#define ALLOW_INTRA_EXT_TX 1
Yaowu Xuc27fc142016-08-22 16:08:15 -0700858
Debargha Mukherjee08542b92017-02-21 01:08:14 -0800859typedef enum {
860 // DCT only
861 EXT_TX_SET_DCTONLY = 0,
862 // DCT + Identity only
Sarah Parker53f93db2017-07-11 17:20:04 -0700863 EXT_TX_SET_DCT_IDTX,
864#if CONFIG_MRC_TX
865 // DCT + MRC_DCT
866 EXT_TX_SET_MRC_DCT,
867 // DCT + MRC_DCT + IDTX
868 EXT_TX_SET_MRC_DCT_IDTX,
869#endif // CONFIG_MRC_TX
Debargha Mukherjee08542b92017-02-21 01:08:14 -0800870 // Discrete Trig transforms w/o flip (4) + Identity (1)
Sarah Parker53f93db2017-07-11 17:20:04 -0700871 EXT_TX_SET_DTT4_IDTX,
Debargha Mukherjee08542b92017-02-21 01:08:14 -0800872 // Discrete Trig transforms w/o flip (4) + Identity (1) + 1D Hor/vert DCT (2)
Sarah Parker53f93db2017-07-11 17:20:04 -0700873 EXT_TX_SET_DTT4_IDTX_1DDCT,
Debargha Mukherjee08542b92017-02-21 01:08:14 -0800874 // Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver DCT (2)
Sarah Parker53f93db2017-07-11 17:20:04 -0700875 EXT_TX_SET_DTT9_IDTX_1DDCT,
Debargha Mukherjee08542b92017-02-21 01:08:14 -0800876 // Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver (6)
Sarah Parker53f93db2017-07-11 17:20:04 -0700877 EXT_TX_SET_ALL16,
Debargha Mukherjee08542b92017-02-21 01:08:14 -0800878 EXT_TX_SET_TYPES
879} TxSetType;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700880
Sarah Parker53f93db2017-07-11 17:20:04 -0700881#if CONFIG_MRC_TX
882// Number of transform types in each set type
883static const int num_ext_tx_set[EXT_TX_SET_TYPES] = {
884 1, 2, 2, 3, 5, 7, 12, 16
885};
886
887// Maps intra set index to the set type
888static const int ext_tx_set_type_intra[EXT_TX_SETS_INTRA] = {
889 EXT_TX_SET_DCTONLY, EXT_TX_SET_DTT4_IDTX_1DDCT, EXT_TX_SET_DTT4_IDTX,
890 EXT_TX_SET_MRC_DCT
891};
892
893// Maps inter set index to the set type
894static const int ext_tx_set_type_inter[EXT_TX_SETS_INTER] = {
895 EXT_TX_SET_DCTONLY, EXT_TX_SET_ALL16, EXT_TX_SET_DTT9_IDTX_1DDCT,
896 EXT_TX_SET_DCT_IDTX, EXT_TX_SET_MRC_DCT_IDTX
897};
898
899// Maps set types above to the indices used for intra
900static const int ext_tx_set_index_intra[EXT_TX_SET_TYPES] = { 0, -1, 3, -1,
901 2, 1, -1, -1 };
902
903// Maps set types above to the indices used for inter
904static const int ext_tx_set_index_inter[EXT_TX_SET_TYPES] = { 0, 3, -1, 4,
905 -1, -1, 2, 1 };
906#else // CONFIG_MRC_TX
Debargha Mukherjee08542b92017-02-21 01:08:14 -0800907// Number of transform types in each set type
908static const int num_ext_tx_set[EXT_TX_SET_TYPES] = { 1, 2, 5, 7, 12, 16 };
909
910// Maps intra set index to the set type
911static const int ext_tx_set_type_intra[EXT_TX_SETS_INTRA] = {
912 EXT_TX_SET_DCTONLY, EXT_TX_SET_DTT4_IDTX_1DDCT, EXT_TX_SET_DTT4_IDTX
913};
914
915// Maps inter set index to the set type
916static const int ext_tx_set_type_inter[EXT_TX_SETS_INTER] = {
917 EXT_TX_SET_DCTONLY, EXT_TX_SET_ALL16, EXT_TX_SET_DTT9_IDTX_1DDCT,
918 EXT_TX_SET_DCT_IDTX
919};
920
921// Maps set types above to the indices used for intra
922static const int ext_tx_set_index_intra[EXT_TX_SET_TYPES] = { 0, -1, 2,
923 1, -1, -1 };
924
925// Maps set types above to the indices used for inter
926static const int ext_tx_set_index_inter[EXT_TX_SET_TYPES] = {
927 0, 3, -1, -1, 2, 1
928};
Sarah Parker53f93db2017-07-11 17:20:04 -0700929#endif // CONFIG_MRC_TX
Debargha Mukherjee08542b92017-02-21 01:08:14 -0800930
931static INLINE TxSetType get_ext_tx_set_type(TX_SIZE tx_size, BLOCK_SIZE bs,
Sarah Parker5effe3f2017-02-23 12:49:10 -0800932 int is_inter, int use_reduced_set) {
Yue Chen56e226e2017-05-02 16:21:40 -0700933 const TX_SIZE tx_size_sqr_up = txsize_sqr_up_map[tx_size];
934 const TX_SIZE tx_size_sqr = txsize_sqr_map[tx_size];
Debargha Mukherjee094c9432017-02-22 10:31:25 -0800935#if CONFIG_CB4X4 && USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4
Jingning Han1a00cff2016-12-28 14:53:14 -0800936 (void)bs;
Yue Chen56e226e2017-05-02 16:21:40 -0700937 if (tx_size_sqr > TX_32X32) return EXT_TX_SET_DCTONLY;
Jingning Han1a00cff2016-12-28 14:53:14 -0800938#else
Yue Chen56e226e2017-05-02 16:21:40 -0700939 if (tx_size_sqr > TX_32X32 || bs < BLOCK_8X8) return EXT_TX_SET_DCTONLY;
Jingning Han1a00cff2016-12-28 14:53:14 -0800940#endif
Sarah Parker5effe3f2017-02-23 12:49:10 -0800941 if (use_reduced_set)
942 return is_inter ? EXT_TX_SET_DCT_IDTX : EXT_TX_SET_DTT4_IDTX;
Sarah Parker53f93db2017-07-11 17:20:04 -0700943#if CONFIG_MRC_TX
Sarah Parker2e08d962017-08-01 19:51:20 -0700944 if (tx_size == TX_32X32) {
945 if (is_inter && USE_MRC_INTER)
946 return EXT_TX_SET_MRC_DCT_IDTX;
947 else if (!is_inter && USE_MRC_INTRA)
948 return EXT_TX_SET_MRC_DCT;
949 }
Sarah Parker53f93db2017-07-11 17:20:04 -0700950#endif // CONFIG_MRC_TX
Yue Chen56e226e2017-05-02 16:21:40 -0700951 if (tx_size_sqr_up == TX_32X32)
Debargha Mukherjee08542b92017-02-21 01:08:14 -0800952 return is_inter ? EXT_TX_SET_DCT_IDTX : EXT_TX_SET_DCTONLY;
953 if (is_inter)
Yue Chen56e226e2017-05-02 16:21:40 -0700954 return (tx_size_sqr == TX_16X16 ? EXT_TX_SET_DTT9_IDTX_1DDCT
955 : EXT_TX_SET_ALL16);
Debargha Mukherjee08542b92017-02-21 01:08:14 -0800956 else
Yue Chen56e226e2017-05-02 16:21:40 -0700957 return (tx_size_sqr == TX_16X16 ? EXT_TX_SET_DTT4_IDTX
958 : EXT_TX_SET_DTT4_IDTX_1DDCT);
Debargha Mukherjee08542b92017-02-21 01:08:14 -0800959}
960
961static INLINE int get_ext_tx_set(TX_SIZE tx_size, BLOCK_SIZE bs, int is_inter,
Sarah Parker5effe3f2017-02-23 12:49:10 -0800962 int use_reduced_set) {
Debargha Mukherjee08542b92017-02-21 01:08:14 -0800963 const TxSetType set_type =
Sarah Parker5effe3f2017-02-23 12:49:10 -0800964 get_ext_tx_set_type(tx_size, bs, is_inter, use_reduced_set);
Debargha Mukherjee08542b92017-02-21 01:08:14 -0800965 return is_inter ? ext_tx_set_index_inter[set_type]
966 : ext_tx_set_index_intra[set_type];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700967}
968
Sarah Parker53f93db2017-07-11 17:20:04 -0700969#if CONFIG_MRC_TX
970static const int use_intra_ext_tx_for_txsize[EXT_TX_SETS_INTRA][EXT_TX_SIZES] =
971 {
972#if CONFIG_CHROMA_2X2
973 { 1, 1, 1, 1, 1 }, // unused
974 { 0, 1, 1, 0, 0 },
975 { 0, 0, 0, 1, 0 },
976 { 0, 0, 0, 0, 1 },
977#else
978 { 1, 1, 1, 1 }, // unused
979 { 1, 1, 0, 0 },
980 { 0, 0, 1, 0 },
981 { 0, 0, 0, 1 },
982#endif // CONFIG_CHROMA_2X2
983 };
984
985static const int use_inter_ext_tx_for_txsize[EXT_TX_SETS_INTER][EXT_TX_SIZES] =
986 {
987#if CONFIG_CHROMA_2X2
988 { 1, 1, 1, 1, 1 }, // unused
989 { 0, 1, 1, 0, 0 }, { 0, 0, 0, 1, 0 },
990 { 0, 0, 0, 0, 1 }, { 0, 0, 0, 0, 1 },
991#else
992 { 1, 1, 1, 1 }, // unused
993 { 1, 1, 0, 0 }, { 0, 0, 1, 0 }, { 0, 0, 0, 1 }, { 0, 0, 0, 1 },
994#endif // CONFIG_CHROMA_2X2
995 };
996
997// Transform types used in each intra set
998static const int ext_tx_used_intra[EXT_TX_SETS_INTRA][TX_TYPES] = {
999 { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
1000 { 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0 },
1001 { 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 },
1002 { 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1 },
1003};
1004
1005// Numbers of transform types used in each intra set
1006static const int ext_tx_cnt_intra[EXT_TX_SETS_INTRA] = { 1, 7, 5, 2 };
1007
1008// Transform types used in each inter set
1009static const int ext_tx_used_inter[EXT_TX_SETS_INTER][TX_TYPES] = {
1010 { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
1011 { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0 },
1012 { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0 },
1013 { 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 },
1014 { 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1 },
1015};
1016
1017// Numbers of transform types used in each inter set
1018static const int ext_tx_cnt_inter[EXT_TX_SETS_INTER] = { 1, 16, 12, 2, 3 };
1019
1020// 1D Transforms used in inter set, this needs to be changed if
1021// ext_tx_used_inter is changed
1022static const int ext_tx_used_inter_1D[EXT_TX_SETS_INTER][TX_TYPES_1D] = {
1023 { 1, 0, 0, 0 }, { 1, 1, 1, 1 }, { 1, 1, 1, 1 }, { 1, 0, 0, 1 }, { 1, 0, 0, 1 }
1024};
1025#else // CONFIG_MRC_TX
clang-format55ce9e02017-02-15 22:27:12 -08001026static const int use_intra_ext_tx_for_txsize[EXT_TX_SETS_INTRA][EXT_TX_SIZES] =
1027 {
Timothy B. Terriberryfe67ed62017-04-26 16:53:47 -07001028#if CONFIG_CHROMA_2X2
clang-format55ce9e02017-02-15 22:27:12 -08001029 { 1, 1, 1, 1, 1 }, // unused
1030 { 0, 1, 1, 0, 0 },
1031 { 0, 0, 0, 1, 0 },
Jingning Han1a00cff2016-12-28 14:53:14 -08001032#else
clang-format55ce9e02017-02-15 22:27:12 -08001033 { 1, 1, 1, 1 }, // unused
1034 { 1, 1, 0, 0 },
1035 { 0, 0, 1, 0 },
Timothy B. Terriberryfe67ed62017-04-26 16:53:47 -07001036#endif // CONFIG_CHROMA_2X2
clang-format55ce9e02017-02-15 22:27:12 -08001037 };
Yaowu Xuc27fc142016-08-22 16:08:15 -07001038
clang-format55ce9e02017-02-15 22:27:12 -08001039static const int use_inter_ext_tx_for_txsize[EXT_TX_SETS_INTER][EXT_TX_SIZES] =
1040 {
Timothy B. Terriberryfe67ed62017-04-26 16:53:47 -07001041#if CONFIG_CHROMA_2X2
clang-format55ce9e02017-02-15 22:27:12 -08001042 { 1, 1, 1, 1, 1 }, // unused
1043 { 0, 1, 1, 0, 0 },
1044 { 0, 0, 0, 1, 0 },
1045 { 0, 0, 0, 0, 1 },
Jingning Han1a00cff2016-12-28 14:53:14 -08001046#else
clang-format55ce9e02017-02-15 22:27:12 -08001047 { 1, 1, 1, 1 }, // unused
1048 { 1, 1, 0, 0 },
1049 { 0, 0, 1, 0 },
1050 { 0, 0, 0, 1 },
Timothy B. Terriberryfe67ed62017-04-26 16:53:47 -07001051#endif // CONFIG_CHROMA_2X2
clang-format55ce9e02017-02-15 22:27:12 -08001052 };
Yaowu Xuc27fc142016-08-22 16:08:15 -07001053
1054// Transform types used in each intra set
1055static const int ext_tx_used_intra[EXT_TX_SETS_INTRA][TX_TYPES] = {
1056 { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
1057 { 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0 },
1058 { 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0 },
1059};
1060
Thomas Daviesb1bedf52017-03-17 14:03:28 +00001061// Numbers of transform types used in each intra set
1062static const int ext_tx_cnt_intra[EXT_TX_SETS_INTRA] = { 1, 7, 5 };
1063
Yaowu Xuc27fc142016-08-22 16:08:15 -07001064// Transform types used in each inter set
1065static const int ext_tx_used_inter[EXT_TX_SETS_INTER][TX_TYPES] = {
1066 { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
1067 { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 },
1068 { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0 },
1069 { 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0 },
1070};
1071
Thomas Daviesb1bedf52017-03-17 14:03:28 +00001072// Numbers of transform types used in each inter set
1073static const int ext_tx_cnt_inter[EXT_TX_SETS_INTER] = { 1, 16, 12, 2 };
1074
Yaowu Xuc27fc142016-08-22 16:08:15 -07001075// 1D Transforms used in inter set, this needs to be changed if
1076// ext_tx_used_inter is changed
1077static const int ext_tx_used_inter_1D[EXT_TX_SETS_INTER][TX_TYPES_1D] = {
1078 { 1, 0, 0, 0 }, { 1, 1, 1, 1 }, { 1, 1, 1, 1 }, { 1, 0, 0, 1 },
1079};
Sarah Parker53f93db2017-07-11 17:20:04 -07001080#endif // CONFIG_MRC_TX
Yaowu Xuc27fc142016-08-22 16:08:15 -07001081
Sarah Parkere68a3e42017-02-16 14:03:24 -08001082static INLINE int get_ext_tx_types(TX_SIZE tx_size, BLOCK_SIZE bs, int is_inter,
Sarah Parker5effe3f2017-02-23 12:49:10 -08001083 int use_reduced_set) {
1084 const int set_type =
1085 get_ext_tx_set_type(tx_size, bs, is_inter, use_reduced_set);
Debargha Mukherjee08542b92017-02-21 01:08:14 -08001086 return num_ext_tx_set[set_type];
Yaowu Xuc27fc142016-08-22 16:08:15 -07001087}
1088
1089#if CONFIG_RECT_TX
1090static INLINE int is_rect_tx_allowed_bsize(BLOCK_SIZE bsize) {
Rupert Swarbrick93c39e92017-07-12 11:11:02 +01001091 static const char LUT[BLOCK_SIZES_ALL] = {
Timothy B. Terriberry81ec2612017-04-26 16:53:47 -07001092#if CONFIG_CHROMA_2X2 || CONFIG_CHROMA_SUB8X8
Jingning Hanf4e097b2017-01-20 09:23:58 -08001093 0, // BLOCK_2X2
1094 0, // BLOCK_2X4
1095 0, // BLOCK_4X2
1096#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -07001097 0, // BLOCK_4X4
1098 1, // BLOCK_4X8
1099 1, // BLOCK_8X4
1100 0, // BLOCK_8X8
1101 1, // BLOCK_8X16
1102 1, // BLOCK_16X8
1103 0, // BLOCK_16X16
1104 1, // BLOCK_16X32
1105 1, // BLOCK_32X16
1106 0, // BLOCK_32X32
1107 0, // BLOCK_32X64
1108 0, // BLOCK_64X32
1109 0, // BLOCK_64X64
1110#if CONFIG_EXT_PARTITION
1111 0, // BLOCK_64X128
1112 0, // BLOCK_128X64
1113 0, // BLOCK_128X128
1114#endif // CONFIG_EXT_PARTITION
Rupert Swarbrick93c39e92017-07-12 11:11:02 +01001115 0, // BLOCK_4X16
1116 0, // BLOCK_16X4
1117 0, // BLOCK_8X32
1118 0, // BLOCK_32X8
Rupert Swarbrick72678572017-08-02 12:05:26 +01001119 0, // BLOCK_16X64
1120 0, // BLOCK_64X16
Yaowu Xuc27fc142016-08-22 16:08:15 -07001121 };
1122
1123 return LUT[bsize];
1124}
1125
Yue Chen49587a72016-09-28 17:09:47 -07001126static INLINE int is_rect_tx_allowed(const MACROBLOCKD *xd,
1127 const MB_MODE_INFO *mbmi) {
Urvang Joshifeb925f2016-12-05 10:37:29 -08001128 return is_rect_tx_allowed_bsize(mbmi->sb_type) &&
Yue Chen49587a72016-09-28 17:09:47 -07001129 !xd->lossless[mbmi->segment_id];
Yaowu Xuc27fc142016-08-22 16:08:15 -07001130}
Yue Chend6bdd462017-07-19 16:05:43 -07001131#endif // CONFIG_RECT_TX
1132#endif // CONFIG_EXT_TX
Yue Chen56e226e2017-05-02 16:21:40 -07001133
Yue Chend6bdd462017-07-19 16:05:43 -07001134#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
Yue Chen56e226e2017-05-02 16:21:40 -07001135static INLINE int is_quarter_tx_allowed_bsize(BLOCK_SIZE bsize) {
Rupert Swarbrick93c39e92017-07-12 11:11:02 +01001136 static const char LUT_QTTX[BLOCK_SIZES_ALL] = {
Timothy B. Terriberry81ec2612017-04-26 16:53:47 -07001137#if CONFIG_CHROMA_2X2 || CONFIG_CHROMA_SUB8X8
Yue Chen56e226e2017-05-02 16:21:40 -07001138 0, // BLOCK_2X2
1139 0, // BLOCK_2X4
1140 0, // BLOCK_4X2
1141#endif
1142 0, // BLOCK_4X4
1143 0, // BLOCK_4X8
1144 0, // BLOCK_8X4
1145 0, // BLOCK_8X8
1146 1, // BLOCK_8X16
1147 1, // BLOCK_16X8
1148 0, // BLOCK_16X16
1149 0, // BLOCK_16X32
1150 0, // BLOCK_32X16
1151 0, // BLOCK_32X32
1152 0, // BLOCK_32X64
1153 0, // BLOCK_64X32
1154 0, // BLOCK_64X64
1155#if CONFIG_EXT_PARTITION
1156 0, // BLOCK_64X128
1157 0, // BLOCK_128X64
1158 0, // BLOCK_128X128
1159#endif // CONFIG_EXT_PARTITION
Rupert Swarbrick93c39e92017-07-12 11:11:02 +01001160 0, // BLOCK_4X16
1161 0, // BLOCK_16X4
1162 0, // BLOCK_8X32
1163 0, // BLOCK_32X8
Rupert Swarbrick72678572017-08-02 12:05:26 +01001164 0, // BLOCK_16X64
1165 0, // BLOCK_64X16
Yue Chen56e226e2017-05-02 16:21:40 -07001166 };
1167
1168 return LUT_QTTX[bsize];
1169}
1170
1171static INLINE int is_quarter_tx_allowed(const MACROBLOCKD *xd,
1172 const MB_MODE_INFO *mbmi,
1173 int is_inter) {
1174 return is_quarter_tx_allowed_bsize(mbmi->sb_type) && is_inter &&
1175 !xd->lossless[mbmi->segment_id];
1176}
Yue Chend6bdd462017-07-19 16:05:43 -07001177#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -07001178
1179static INLINE TX_SIZE tx_size_from_tx_mode(BLOCK_SIZE bsize, TX_MODE tx_mode,
1180 int is_inter) {
1181 const TX_SIZE largest_tx_size = tx_mode_to_biggest_tx_size[tx_mode];
Debargha Mukherjee428bbb22017-03-17 07:30:24 -07001182#if (CONFIG_VAR_TX || CONFIG_EXT_TX) && CONFIG_RECT_TX
Urvang Joshifeb925f2016-12-05 10:37:29 -08001183 const TX_SIZE max_rect_tx_size = max_txsize_rect_lookup[bsize];
Jingning Han70e5f3f2016-11-09 17:03:07 -08001184#else
Yaowu Xuc27fc142016-08-22 16:08:15 -07001185 const TX_SIZE max_tx_size = max_txsize_lookup[bsize];
Debargha Mukherjee428bbb22017-03-17 07:30:24 -07001186#endif // (CONFIG_VAR_TX || CONFIG_EXT_TX) && CONFIG_RECT_TX
Urvang Joshifeb925f2016-12-05 10:37:29 -08001187 (void)is_inter;
Debargha Mukherjee428bbb22017-03-17 07:30:24 -07001188#if CONFIG_VAR_TX && CONFIG_RECT_TX
Urvang Joshifeb925f2016-12-05 10:37:29 -08001189#if CONFIG_CB4X4
1190 if (bsize == BLOCK_4X4)
1191 return AOMMIN(max_txsize_lookup[bsize], largest_tx_size);
1192#else
1193 if (bsize < BLOCK_8X8)
1194 return AOMMIN(max_txsize_lookup[bsize], largest_tx_size);
Jingning Han70e5f3f2016-11-09 17:03:07 -08001195#endif
Urvang Joshifeb925f2016-12-05 10:37:29 -08001196 if (txsize_sqr_map[max_rect_tx_size] <= largest_tx_size)
1197 return max_rect_tx_size;
1198 else
1199 return largest_tx_size;
1200#elif CONFIG_EXT_TX && CONFIG_RECT_TX
1201 if (txsize_sqr_up_map[max_rect_tx_size] <= largest_tx_size) {
1202 return max_rect_tx_size;
Yaowu Xuc27fc142016-08-22 16:08:15 -07001203 } else {
Urvang Joshifeb925f2016-12-05 10:37:29 -08001204 return largest_tx_size;
Yaowu Xuc27fc142016-08-22 16:08:15 -07001205 }
1206#else
Yaowu Xuf883b422016-08-30 14:01:10 -07001207 return AOMMIN(max_tx_size, largest_tx_size);
Debargha Mukherjee428bbb22017-03-17 07:30:24 -07001208#endif // CONFIG_VAR_TX && CONFIG_RECT_TX
Yaowu Xuc27fc142016-08-22 16:08:15 -07001209}
1210
hui su5db97432016-10-14 16:10:14 -07001211#if CONFIG_EXT_INTRA
hui su0a6731f2017-04-26 15:23:47 -07001212#define MAX_ANGLE_DELTA 3
1213#define ANGLE_STEP 3
hui su5db97432016-10-14 16:10:14 -07001214extern const int16_t dr_intra_derivative[90];
hui su02c79742017-05-16 17:19:04 -07001215static const uint8_t mode_to_angle_map[] = {
Urvang Joshi93b543a2017-06-01 17:32:41 -07001216 0, 90, 180, 45, 135, 111, 157, 203, 67, 0, 0,
hui su02c79742017-05-16 17:19:04 -07001217#if CONFIG_SMOOTH_HV
1218 0, 0,
1219#endif // CONFIG_SMOOTH_HV
hui su5db97432016-10-14 16:10:14 -07001220};
hui sueda3d762016-12-06 16:58:23 -08001221#if CONFIG_INTRA_INTERP
hui su5db97432016-10-14 16:10:14 -07001222// Returns whether filter selection is needed for a given
1223// intra prediction angle.
Yaowu Xuf883b422016-08-30 14:01:10 -07001224int av1_is_intra_filter_switchable(int angle);
hui sueda3d762016-12-06 16:58:23 -08001225#endif // CONFIG_INTRA_INTERP
Yaowu Xuc27fc142016-08-22 16:08:15 -07001226#endif // CONFIG_EXT_INTRA
1227
Monty Montgomerycb55dad2017-07-11 16:59:52 -04001228#if CONFIG_DCT_ONLY
1229#define FIXED_TX_TYPE 1
1230#else
Yaowu Xuc27fc142016-08-22 16:08:15 -07001231#define FIXED_TX_TYPE 0
Monty Montgomerycb55dad2017-07-11 16:59:52 -04001232#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -07001233
Angie Chiang752ccce2017-04-09 13:41:13 -07001234// Converts block_index for given transform size to index of the block in raster
1235// order.
1236static INLINE int av1_block_index_to_raster_order(TX_SIZE tx_size,
1237 int block_idx) {
1238 // For transform size 4x8, the possible block_idx values are 0 & 2, because
1239 // block_idx values are incremented in steps of size 'tx_width_unit x
1240 // tx_height_unit'. But, for this transform size, block_idx = 2 corresponds to
1241 // block number 1 in raster order, inside an 8x8 MI block.
1242 // For any other transform size, the two indices are equivalent.
1243 return (tx_size == TX_4X8 && block_idx == 2) ? 1 : block_idx;
1244}
1245
1246// Inverse of above function.
1247// Note: only implemented for transform sizes 4x4, 4x8 and 8x4 right now.
1248static INLINE int av1_raster_order_to_block_index(TX_SIZE tx_size,
1249 int raster_order) {
1250 assert(tx_size == TX_4X4 || tx_size == TX_4X8 || tx_size == TX_8X4);
1251 // We ensure that block indices are 0 & 2 if tx size is 4x8 or 8x4.
1252 return (tx_size == TX_4X4) ? raster_order : (raster_order > 0) ? 2 : 0;
1253}
1254
Yaowu Xuc27fc142016-08-22 16:08:15 -07001255static INLINE TX_TYPE get_default_tx_type(PLANE_TYPE plane_type,
1256 const MACROBLOCKD *xd, int block_idx,
1257 TX_SIZE tx_size) {
1258 const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
1259
Monty Montgomerycb55dad2017-07-11 16:59:52 -04001260 if (CONFIG_DCT_ONLY || is_inter_block(mbmi) || plane_type != PLANE_TYPE_Y ||
Yaowu Xuc27fc142016-08-22 16:08:15 -07001261 xd->lossless[mbmi->segment_id] || tx_size >= TX_32X32)
1262 return DCT_DCT;
1263
1264 return intra_mode_to_tx_type_context[plane_type == PLANE_TYPE_Y
1265 ? get_y_mode(xd->mi[0], block_idx)
Luc Trudeaud6d9eee2017-07-12 12:36:50 -04001266 : get_uv_mode(mbmi->uv_mode)];
Yaowu Xuc27fc142016-08-22 16:08:15 -07001267}
1268
hui su45b64752017-07-12 16:54:35 -07001269static INLINE TX_TYPE av1_get_tx_type(PLANE_TYPE plane_type,
Jingning Han19b5c8f2017-07-06 15:10:12 -07001270 const MACROBLOCKD *xd, int blk_row,
1271 int blk_col, int block, TX_SIZE tx_size) {
Yaowu Xuc27fc142016-08-22 16:08:15 -07001272 const MODE_INFO *const mi = xd->mi[0];
1273 const MB_MODE_INFO *const mbmi = &mi->mbmi;
Jingning Han19b5c8f2017-07-06 15:10:12 -07001274 (void)blk_row;
1275 (void)blk_col;
Alex Conversedaa15e42017-05-02 14:27:16 -07001276#if CONFIG_INTRABC && (!CONFIG_EXT_TX || CONFIG_TXK_SEL)
1277 // TODO(aconverse@google.com): Handle INTRABC + EXT_TX + TXK_SEL
Alex Converse28744302017-04-13 14:46:22 -07001278 if (is_intrabc_block(mbmi)) return DCT_DCT;
Alex Conversedaa15e42017-05-02 14:27:16 -07001279#endif // CONFIG_INTRABC && (!CONFIG_EXT_TX || CONFIG_TXK_SEL)
hui su45b64752017-07-12 16:54:35 -07001280
1281#if CONFIG_TXK_SEL
1282 TX_TYPE tx_type;
Jingning Han19b5c8f2017-07-06 15:10:12 -07001283 if (xd->lossless[mbmi->segment_id] || txsize_sqr_map[tx_size] >= TX_32X32) {
hui su45b64752017-07-12 16:54:35 -07001284 tx_type = DCT_DCT;
1285 } else {
Jingning Han19b5c8f2017-07-06 15:10:12 -07001286 if (plane_type == PLANE_TYPE_Y)
1287 tx_type = mbmi->txk_type[(blk_row << 4) + blk_col];
1288 else if (is_inter_block(mbmi))
1289 tx_type = mbmi->txk_type[(blk_row << 5) + (blk_col << 1)];
1290 else
1291 tx_type = intra_mode_to_tx_type_context[mbmi->uv_mode];
hui su45b64752017-07-12 16:54:35 -07001292 }
1293 assert(tx_type >= DCT_DCT && tx_type < TX_TYPES);
1294 return tx_type;
1295#endif // CONFIG_TXK_SEL
1296
James Zern7cdaa602017-04-20 20:12:46 -07001297#if FIXED_TX_TYPE
Angie Chiangc2ebfbf2017-04-15 13:56:58 -07001298 const int block_raster_idx = av1_block_index_to_raster_order(tx_size, block);
James Zern7cdaa602017-04-20 20:12:46 -07001299 return get_default_tx_type(plane_type, xd, block_raster_idx, tx_size);
hui su45b64752017-07-12 16:54:35 -07001300#endif // FIXED_TX_TYPE
1301
1302#if CONFIG_EXT_TX
Sarah Parker53f93db2017-07-11 17:20:04 -07001303#if CONFIG_MRC_TX
1304 if (mbmi->tx_type == MRC_DCT) {
Sarah Parker2e08d962017-08-01 19:51:20 -07001305 assert(((is_inter_block(mbmi) && USE_MRC_INTER) ||
1306 (!is_inter_block(mbmi) && USE_MRC_INTRA)) &&
1307 "INVALID BLOCK TYPE FOR MRC_DCT");
Sarah Parker53f93db2017-07-11 17:20:04 -07001308 if (plane_type == PLANE_TYPE_Y) {
1309 assert(tx_size == TX_32X32);
1310 return mbmi->tx_type;
1311 }
1312 return DCT_DCT;
1313 }
1314#endif // CONFIG_MRC_TX
Yaowu Xuc27fc142016-08-22 16:08:15 -07001315 if (xd->lossless[mbmi->segment_id] || txsize_sqr_map[tx_size] > TX_32X32 ||
1316 (txsize_sqr_map[tx_size] >= TX_32X32 && !is_inter_block(mbmi)))
Yaowu Xuc27fc142016-08-22 16:08:15 -07001317 return DCT_DCT;
Jingning Han8260d8b2016-12-28 13:48:43 -08001318 if (mbmi->sb_type >= BLOCK_8X8 || CONFIG_CB4X4) {
Yaowu Xuc27fc142016-08-22 16:08:15 -07001319 if (plane_type == PLANE_TYPE_Y) {
1320#if !ALLOW_INTRA_EXT_TX
1321 if (is_inter_block(mbmi))
1322#endif // ALLOW_INTRA_EXT_TX
1323 return mbmi->tx_type;
1324 }
Jingning Han1a00cff2016-12-28 14:53:14 -08001325
1326 if (is_inter_block(mbmi)) {
1327// UV Inter only
Timothy B. Terriberryfe67ed62017-04-26 16:53:47 -07001328#if CONFIG_CHROMA_2X2
Jingning Han1a00cff2016-12-28 14:53:14 -08001329 if (tx_size < TX_4X4) return DCT_DCT;
1330#endif
Debargha Mukherjee5a488a62016-11-22 22:24:10 -08001331 return (mbmi->tx_type == IDTX && txsize_sqr_map[tx_size] >= TX_32X32)
Yaowu Xuc27fc142016-08-22 16:08:15 -07001332 ? DCT_DCT
1333 : mbmi->tx_type;
Jingning Han1a00cff2016-12-28 14:53:14 -08001334 }
Yaowu Xuc27fc142016-08-22 16:08:15 -07001335 }
1336
Jingning Han8260d8b2016-12-28 13:48:43 -08001337#if CONFIG_CB4X4
Thomas Daedeb7e72f32017-04-26 13:40:31 -07001338 (void)block;
Timothy B. Terriberryfe67ed62017-04-26 16:53:47 -07001339#if CONFIG_CHROMA_2X2
Jingning Han1a00cff2016-12-28 14:53:14 -08001340 if (tx_size < TX_4X4)
1341 return DCT_DCT;
1342 else
hui su45b64752017-07-12 16:54:35 -07001343#endif // CONFIG_CHROMA_2X2
Luc Trudeaud6d9eee2017-07-12 12:36:50 -04001344 return intra_mode_to_tx_type_context[get_uv_mode(mbmi->uv_mode)];
Timothy B. Terriberryfe67ed62017-04-26 16:53:47 -07001345#else // CONFIG_CB4X4
Yaowu Xuc27fc142016-08-22 16:08:15 -07001346 // Sub8x8-Inter/Intra OR UV-Intra
hui su45b64752017-07-12 16:54:35 -07001347 if (is_inter_block(mbmi)) { // Sub8x8-Inter
Yaowu Xuc27fc142016-08-22 16:08:15 -07001348 return DCT_DCT;
hui su45b64752017-07-12 16:54:35 -07001349 } else { // Sub8x8 Intra OR UV-Intra
1350 const int block_raster_idx =
1351 av1_block_index_to_raster_order(tx_size, block);
Yaowu Xuc27fc142016-08-22 16:08:15 -07001352 return intra_mode_to_tx_type_context[plane_type == PLANE_TYPE_Y
Angie Chiang752ccce2017-04-09 13:41:13 -07001353 ? get_y_mode(mi, block_raster_idx)
Luc Trudeaud6d9eee2017-07-12 12:36:50 -04001354 : get_uv_mode(mbmi->uv_mode)];
hui su45b64752017-07-12 16:54:35 -07001355 }
Thomas Daedeb7e72f32017-04-26 13:40:31 -07001356#endif // CONFIG_CB4X4
Yaowu Xuc27fc142016-08-22 16:08:15 -07001357#else // CONFIG_EXT_TX
Angie Chiang752ccce2017-04-09 13:41:13 -07001358 (void)block;
Sarah Parker53f93db2017-07-11 17:20:04 -07001359#if CONFIG_MRC_TX
1360 if (mbmi->tx_type == MRC_DCT) {
1361 if (plane_type == PLANE_TYPE_Y && !xd->lossless[mbmi->segment_id]) {
1362 assert(tx_size == TX_32X32);
1363 return mbmi->tx_type;
1364 }
1365 return DCT_DCT;
1366 }
1367#endif // CONFIG_MRC_TX
Yaowu Xuc27fc142016-08-22 16:08:15 -07001368 if (plane_type != PLANE_TYPE_Y || xd->lossless[mbmi->segment_id] ||
1369 txsize_sqr_map[tx_size] >= TX_32X32)
1370 return DCT_DCT;
1371 return mbmi->tx_type;
1372#endif // CONFIG_EXT_TX
1373}
1374
Yaowu Xuf883b422016-08-30 14:01:10 -07001375void av1_setup_block_planes(MACROBLOCKD *xd, int ss_x, int ss_y);
Yaowu Xuc27fc142016-08-22 16:08:15 -07001376
Yaowu Xu4ff59b52017-04-24 12:41:56 -07001377static INLINE int tx_size_to_depth(TX_SIZE tx_size) {
Timothy B. Terriberryfe0fb1d2017-05-18 12:15:16 -07001378 return (int)(tx_size - TX_SIZE_LUMA_MIN);
Jingning Han4e1737a2016-10-25 16:05:02 -07001379}
1380
Yaowu Xu4ff59b52017-04-24 12:41:56 -07001381static INLINE TX_SIZE depth_to_tx_size(int depth) {
Timothy B. Terriberryfe0fb1d2017-05-18 12:15:16 -07001382 return (TX_SIZE)(depth + TX_SIZE_LUMA_MIN);
Jingning Han4e1737a2016-10-25 16:05:02 -07001383}
1384
hui su0c6244b2017-07-12 17:11:43 -07001385static INLINE TX_SIZE av1_get_uv_tx_size(const MB_MODE_INFO *mbmi,
1386 const struct macroblockd_plane *pd) {
Timothy B. Terriberryfe67ed62017-04-26 16:53:47 -07001387#if CONFIG_CHROMA_2X2
Jingning Han93531242016-12-20 11:54:36 -08001388 assert(mbmi->tx_size > TX_2X2);
hui su0c6244b2017-07-12 17:11:43 -07001389#endif // CONFIG_CHROMA_2X2
Jingning Han93531242016-12-20 11:54:36 -08001390
Yaowu Xuc27fc142016-08-22 16:08:15 -07001391#if CONFIG_SUPERTX
1392 if (supertx_enabled(mbmi))
1393 return uvsupertx_size_lookup[txsize_sqr_map[mbmi->tx_size]]
1394 [pd->subsampling_x][pd->subsampling_y];
1395#endif // CONFIG_SUPERTX
Jingning Hancabd9892016-12-01 12:28:42 -08001396
hui su0c6244b2017-07-12 17:11:43 -07001397 const TX_SIZE uv_txsize =
1398 uv_txsize_lookup[mbmi->sb_type][mbmi->tx_size][pd->subsampling_x]
1399 [pd->subsampling_y];
Debargha Mukherjee2f123402016-08-30 17:43:38 -07001400 assert(uv_txsize != TX_INVALID);
1401 return uv_txsize;
Yaowu Xuc27fc142016-08-22 16:08:15 -07001402}
1403
hui su0c6244b2017-07-12 17:11:43 -07001404static INLINE TX_SIZE av1_get_tx_size(int plane, const MACROBLOCKD *xd) {
Angie Chiang80b82262017-02-24 11:39:47 -08001405 const MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
hui su0c6244b2017-07-12 17:11:43 -07001406 if (plane == 0) return mbmi->tx_size;
Angie Chiang80b82262017-02-24 11:39:47 -08001407 const MACROBLOCKD_PLANE *pd = &xd->plane[plane];
hui su0c6244b2017-07-12 17:11:43 -07001408 return av1_get_uv_tx_size(mbmi, pd);
Angie Chiang80b82262017-02-24 11:39:47 -08001409}
1410
Yaowu Xuc27fc142016-08-22 16:08:15 -07001411static INLINE BLOCK_SIZE
1412get_plane_block_size(BLOCK_SIZE bsize, const struct macroblockd_plane *pd) {
1413 return ss_size_lookup[bsize][pd->subsampling_x][pd->subsampling_y];
1414}
1415
Timothy B. Terriberrya2d5cde2017-05-10 18:33:50 -07001416void av1_reset_skip_context(MACROBLOCKD *xd, int mi_row, int mi_col,
1417 BLOCK_SIZE bsize);
Yaowu Xuc27fc142016-08-22 16:08:15 -07001418
1419typedef void (*foreach_transformed_block_visitor)(int plane, int block,
1420 int blk_row, int blk_col,
1421 BLOCK_SIZE plane_bsize,
1422 TX_SIZE tx_size, void *arg);
1423
Yaowu Xuf883b422016-08-30 14:01:10 -07001424void av1_foreach_transformed_block_in_plane(
Yaowu Xuc27fc142016-08-22 16:08:15 -07001425 const MACROBLOCKD *const xd, BLOCK_SIZE bsize, int plane,
1426 foreach_transformed_block_visitor visit, void *arg);
1427
Angie Chiang0397eda2017-03-15 16:57:14 -07001428#if CONFIG_LV_MAP
1429void av1_foreach_transformed_block(const MACROBLOCKD *const xd,
Jingning Han94652b82017-04-04 09:45:02 -07001430 BLOCK_SIZE bsize, int mi_row, int mi_col,
Angie Chiang0397eda2017-03-15 16:57:14 -07001431 foreach_transformed_block_visitor visit,
1432 void *arg);
1433#endif
1434
iole moccagattaf25a4cf2016-11-11 23:57:57 -08001435#if CONFIG_COEF_INTERLEAVE
1436static INLINE int get_max_4x4_size(int num_4x4, int mb_to_edge,
1437 int subsampling) {
1438 return num_4x4 + (mb_to_edge >= 0 ? 0 : mb_to_edge >> (5 + subsampling));
1439}
1440
1441void av1_foreach_transformed_block_interleave(
1442 const MACROBLOCKD *const xd, BLOCK_SIZE bsize,
1443 foreach_transformed_block_visitor visit, void *arg);
1444#endif
1445
Yaowu Xuf883b422016-08-30 14:01:10 -07001446void av1_set_contexts(const MACROBLOCKD *xd, struct macroblockd_plane *pd,
Jingning Haneee43152016-12-05 09:58:45 -08001447 int plane, TX_SIZE tx_size, int has_eob, int aoff,
1448 int loff);
Yaowu Xuc27fc142016-08-22 16:08:15 -07001449
1450#if CONFIG_EXT_INTER
1451static INLINE int is_interintra_allowed_bsize(const BLOCK_SIZE bsize) {
Debargha Mukherjee37f6fe62017-02-10 21:44:13 -08001452#if CONFIG_INTERINTRA
Yaowu Xuc27fc142016-08-22 16:08:15 -07001453 // TODO(debargha): Should this be bsize < BLOCK_LARGEST?
1454 return (bsize >= BLOCK_8X8) && (bsize < BLOCK_64X64);
Debargha Mukherjee37f6fe62017-02-10 21:44:13 -08001455#else
1456 (void)bsize;
1457 return 0;
1458#endif // CONFIG_INTERINTRA
Yaowu Xuc27fc142016-08-22 16:08:15 -07001459}
1460
1461static INLINE int is_interintra_allowed_mode(const PREDICTION_MODE mode) {
Debargha Mukherjee37f6fe62017-02-10 21:44:13 -08001462#if CONFIG_INTERINTRA
Yaowu Xuc27fc142016-08-22 16:08:15 -07001463 return (mode >= NEARESTMV) && (mode <= NEWMV);
Debargha Mukherjee37f6fe62017-02-10 21:44:13 -08001464#else
1465 (void)mode;
1466 return 0;
1467#endif // CONFIG_INTERINTRA
Yaowu Xuc27fc142016-08-22 16:08:15 -07001468}
1469
1470static INLINE int is_interintra_allowed_ref(const MV_REFERENCE_FRAME rf[2]) {
Debargha Mukherjee37f6fe62017-02-10 21:44:13 -08001471#if CONFIG_INTERINTRA
Yaowu Xuc27fc142016-08-22 16:08:15 -07001472 return (rf[0] > INTRA_FRAME) && (rf[1] <= INTRA_FRAME);
Debargha Mukherjee37f6fe62017-02-10 21:44:13 -08001473#else
1474 (void)rf;
1475 return 0;
1476#endif // CONFIG_INTERINTRA
Yaowu Xuc27fc142016-08-22 16:08:15 -07001477}
1478
1479static INLINE int is_interintra_allowed(const MB_MODE_INFO *mbmi) {
1480 return is_interintra_allowed_bsize(mbmi->sb_type) &&
1481 is_interintra_allowed_mode(mbmi->mode) &&
1482 is_interintra_allowed_ref(mbmi->ref_frame);
1483}
1484
Yaowu Xu4ff59b52017-04-24 12:41:56 -07001485static INLINE int is_interintra_allowed_bsize_group(int group) {
Yaowu Xuc27fc142016-08-22 16:08:15 -07001486 int i;
Rupert Swarbrick93c39e92017-07-12 11:11:02 +01001487 for (i = 0; i < BLOCK_SIZES_ALL; i++) {
Urvang Joshicb586f32016-09-20 11:36:33 -07001488 if (size_group_lookup[i] == group &&
1489 is_interintra_allowed_bsize((BLOCK_SIZE)i)) {
Yaowu Xuc27fc142016-08-22 16:08:15 -07001490 return 1;
Urvang Joshicb586f32016-09-20 11:36:33 -07001491 }
Yaowu Xuc27fc142016-08-22 16:08:15 -07001492 }
1493 return 0;
1494}
1495
1496static INLINE int is_interintra_pred(const MB_MODE_INFO *mbmi) {
1497 return (mbmi->ref_frame[1] == INTRA_FRAME) && is_interintra_allowed(mbmi);
1498}
1499#endif // CONFIG_EXT_INTER
1500
Sarah Parker106b3cb2017-04-21 12:13:37 -07001501#if CONFIG_VAR_TX
1502static INLINE int get_vartx_max_txsize(const MB_MODE_INFO *const mbmi,
1503 BLOCK_SIZE bsize) {
1504#if CONFIG_CB4X4
1505 (void)mbmi;
1506 return max_txsize_rect_lookup[bsize];
1507#endif // CONFIG_C4X4
1508 return mbmi->sb_type < BLOCK_8X8 ? max_txsize_rect_lookup[mbmi->sb_type]
1509 : max_txsize_rect_lookup[bsize];
1510}
1511#endif // CONFIG_VAR_TX
1512
Yue Chencb60b182016-10-13 15:18:22 -07001513#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
1514static INLINE int is_motion_variation_allowed_bsize(BLOCK_SIZE bsize) {
Debargha Mukherjeec17a4432017-08-28 22:22:45 -07001515 return AOMMIN(block_size_wide[bsize], block_size_high[bsize]) >= 8;
Yue Chencb60b182016-10-13 15:18:22 -07001516}
1517
Yue Chen8636da62017-04-03 01:23:44 -07001518static INLINE int is_motion_variation_allowed_compound(
1519 const MB_MODE_INFO *mbmi) {
Zoe Liu85b66462017-04-20 14:28:19 -07001520#if CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF
1521 if (!has_second_ref(mbmi) && !is_inter_singleref_comp_mode(mbmi->mode))
1522#else
Yue Chen8636da62017-04-03 01:23:44 -07001523 if (!has_second_ref(mbmi))
Zoe Liu85b66462017-04-20 14:28:19 -07001524#endif // CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF
Yue Chen8636da62017-04-03 01:23:44 -07001525 return 1;
1526 else
1527 return 0;
1528}
1529
Yue Chen5329a2b2017-02-28 17:33:00 +08001530#if CONFIG_MOTION_VAR
Yue Chen1bd42be2017-03-15 18:07:04 -07001531// input: log2 of length, 0(4), 1(8), ...
1532static const int max_neighbor_obmc[6] = { 0, 1, 2, 3, 4, 4 };
1533
Yue Chen5329a2b2017-02-28 17:33:00 +08001534static INLINE int check_num_overlappable_neighbors(const MB_MODE_INFO *mbmi) {
Yue Chen1bd42be2017-03-15 18:07:04 -07001535 return !(mbmi->overlappable_neighbors[0] == 0 &&
1536 mbmi->overlappable_neighbors[1] == 0);
Yue Chen5329a2b2017-02-28 17:33:00 +08001537}
Wei-Ting Lin20885282017-08-28 17:18:18 -07001538#if CONFIG_NCOBMC_ADAPT_WEIGHT
1539static INLINE NCOBMC_MODE ncobmc_mode_allowed_bsize(BLOCK_SIZE bsize) {
1540 if (bsize < BLOCK_8X8 || bsize >= BLOCK_64X64)
1541 return NO_OVERLAP;
1542 else
1543 return MAX_NCOBMC_MODES;
1544}
1545#endif // CONFIG_NCOBMC_ADAPT_WEIGHT
1546#endif // CONFIG_MOTION_VAR
Yue Chen5329a2b2017-02-28 17:33:00 +08001547
Sarah Parker19234cc2017-03-10 16:43:25 -08001548static INLINE MOTION_MODE motion_mode_allowed(
Sarah Parker0eea89f2017-07-11 11:56:36 -07001549#if CONFIG_GLOBAL_MOTION
Sarah Parker19234cc2017-03-10 16:43:25 -08001550 int block, const WarpedMotionParams *gm_params,
Sarah Parker0eea89f2017-07-11 11:56:36 -07001551#endif // CONFIG_GLOBAL_MOTION
Yue Chen52c51732017-07-11 15:08:30 -07001552#if CONFIG_WARPED_MOTION
1553 const MACROBLOCKD *xd,
1554#endif
Sarah Parker19234cc2017-03-10 16:43:25 -08001555 const MODE_INFO *mi) {
1556 const MB_MODE_INFO *mbmi = &mi->mbmi;
Sarah Parker0eea89f2017-07-11 11:56:36 -07001557#if CONFIG_GLOBAL_MOTION
Sarah Parker19234cc2017-03-10 16:43:25 -08001558 const TransformationType gm_type = gm_params[mbmi->ref_frame[0]].wmtype;
1559 if (is_global_mv_block(mi, block, gm_type)) return SIMPLE_TRANSLATION;
Sarah Parker0eea89f2017-07-11 11:56:36 -07001560#endif // CONFIG_GLOBAL_MOTION
Yaowu Xuc27fc142016-08-22 16:08:15 -07001561#if CONFIG_EXT_INTER
Yue Chen69f18e12016-09-08 14:48:15 -07001562 if (is_motion_variation_allowed_bsize(mbmi->sb_type) &&
Yue Chen8636da62017-04-03 01:23:44 -07001563 is_inter_mode(mbmi->mode) && mbmi->ref_frame[1] != INTRA_FRAME &&
1564 is_motion_variation_allowed_compound(mbmi)) {
Yaowu Xuc27fc142016-08-22 16:08:15 -07001565#else
Yue Chen69f18e12016-09-08 14:48:15 -07001566 if (is_motion_variation_allowed_bsize(mbmi->sb_type) &&
Yue Chen8636da62017-04-03 01:23:44 -07001567 is_inter_mode(mbmi->mode) && is_motion_variation_allowed_compound(mbmi)) {
Yaowu Xuc27fc142016-08-22 16:08:15 -07001568#endif // CONFIG_EXT_INTER
Yue Chen5329a2b2017-02-28 17:33:00 +08001569#if CONFIG_MOTION_VAR
1570 if (!check_num_overlappable_neighbors(mbmi)) return SIMPLE_TRANSLATION;
1571#endif
Yue Chen69f18e12016-09-08 14:48:15 -07001572#if CONFIG_WARPED_MOTION
Yue Chen52c51732017-07-11 15:08:30 -07001573 if (!has_second_ref(mbmi) && mbmi->num_proj_ref[0] >= 1 &&
Wei-Ting Lin20885282017-08-28 17:18:18 -07001574 !av1_is_scaled(&(xd->block_refs[0]->sf))) {
Yue Chen69f18e12016-09-08 14:48:15 -07001575 return WARPED_CAUSAL;
Wei-Ting Lin20885282017-08-28 17:18:18 -07001576 } else {
Yue Chen69f18e12016-09-08 14:48:15 -07001577#endif // CONFIG_WARPED_MOTION
1578#if CONFIG_MOTION_VAR
Wei-Ting Lin20885282017-08-28 17:18:18 -07001579#if CONFIG_NCOBMC_ADAPT_WEIGHT
1580 if (ncobmc_mode_allowed_bsize(mbmi->sb_type) < NO_OVERLAP)
1581 return NCOBMC_ADAPT_WEIGHT;
1582 else
1583#endif
1584 return OBMC_CAUSAL;
1585#if CONFIG_WARPED_MOTION
1586 }
1587#endif // CONFIG_WARPED_MOTION
Yue Chen69f18e12016-09-08 14:48:15 -07001588#else
1589 return SIMPLE_TRANSLATION;
1590#endif // CONFIG_MOTION_VAR
1591 } else {
1592 return SIMPLE_TRANSLATION;
1593 }
Yaowu Xuc27fc142016-08-22 16:08:15 -07001594}
1595
Sarah Parker19234cc2017-03-10 16:43:25 -08001596static INLINE void assert_motion_mode_valid(MOTION_MODE mode,
Sarah Parker0eea89f2017-07-11 11:56:36 -07001597#if CONFIG_GLOBAL_MOTION
Sarah Parker19234cc2017-03-10 16:43:25 -08001598 int block,
1599 const WarpedMotionParams *gm_params,
Sarah Parker0eea89f2017-07-11 11:56:36 -07001600#endif // CONFIG_GLOBAL_MOTION
Yue Chen52c51732017-07-11 15:08:30 -07001601#if CONFIG_WARPED_MOTION
1602 const MACROBLOCKD *xd,
1603#endif
Sarah Parker19234cc2017-03-10 16:43:25 -08001604 const MODE_INFO *mi) {
1605 const MOTION_MODE last_motion_mode_allowed = motion_mode_allowed(
Sarah Parker0eea89f2017-07-11 11:56:36 -07001606#if CONFIG_GLOBAL_MOTION
Sarah Parker19234cc2017-03-10 16:43:25 -08001607 block, gm_params,
Sarah Parker0eea89f2017-07-11 11:56:36 -07001608#endif // CONFIG_GLOBAL_MOTION
Yue Chen52c51732017-07-11 15:08:30 -07001609#if CONFIG_WARPED_MOTION
1610 xd,
1611#endif
Sarah Parker19234cc2017-03-10 16:43:25 -08001612 mi);
Wei-Ting Lin20885282017-08-28 17:18:18 -07001613
Sarah Parker19234cc2017-03-10 16:43:25 -08001614 // Check that the input mode is not illegal
1615 if (last_motion_mode_allowed < mode)
1616 assert(0 && "Illegal motion mode selected");
1617}
1618
Yue Chencb60b182016-10-13 15:18:22 -07001619#if CONFIG_MOTION_VAR
Yaowu Xuc27fc142016-08-22 16:08:15 -07001620static INLINE int is_neighbor_overlappable(const MB_MODE_INFO *mbmi) {
1621 return (is_inter_block(mbmi));
1622}
Yue Chencb60b182016-10-13 15:18:22 -07001623#endif // CONFIG_MOTION_VAR
1624#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
Yaowu Xuc27fc142016-08-22 16:08:15 -07001625
Urvang Joshi56ba91b2017-01-10 13:22:09 -08001626// Returns sub-sampled dimensions of the given block.
1627// The output values for 'rows_within_bounds' and 'cols_within_bounds' will
1628// differ from 'height' and 'width' when part of the block is outside the right
1629// and/or bottom image boundary.
1630static INLINE void av1_get_block_dimensions(BLOCK_SIZE bsize, int plane,
1631 const MACROBLOCKD *xd, int *width,
1632 int *height,
1633 int *rows_within_bounds,
1634 int *cols_within_bounds) {
1635 const int block_height = block_size_high[bsize];
1636 const int block_width = block_size_wide[bsize];
1637 const int block_rows = (xd->mb_to_bottom_edge >= 0)
1638 ? block_height
1639 : (xd->mb_to_bottom_edge >> 3) + block_height;
1640 const int block_cols = (xd->mb_to_right_edge >= 0)
1641 ? block_width
1642 : (xd->mb_to_right_edge >> 3) + block_width;
1643 const struct macroblockd_plane *const pd = &xd->plane[plane];
1644 assert(IMPLIES(plane == PLANE_TYPE_Y, pd->subsampling_x == 0));
1645 assert(IMPLIES(plane == PLANE_TYPE_Y, pd->subsampling_y == 0));
1646 assert(block_width >= block_cols);
1647 assert(block_height >= block_rows);
1648 if (width) *width = block_width >> pd->subsampling_x;
1649 if (height) *height = block_height >> pd->subsampling_y;
1650 if (rows_within_bounds) *rows_within_bounds = block_rows >> pd->subsampling_y;
1651 if (cols_within_bounds) *cols_within_bounds = block_cols >> pd->subsampling_x;
1652}
1653
Sarah Parkerefd8af22017-08-25 16:36:06 -07001654/* clang-format off */
1655typedef aom_cdf_prob (*MapCdf)[PALETTE_COLOR_INDEX_CONTEXTS]
1656 [CDF_SIZE(PALETTE_COLORS)];
1657typedef const int (*ColorCost)[PALETTE_SIZES][PALETTE_COLOR_INDEX_CONTEXTS]
1658 [PALETTE_COLORS];
1659/* clang-format on */
1660
1661typedef struct {
1662 int rows;
1663 int cols;
1664 int n_colors;
1665 int plane_width;
1666 int plane_height;
1667 uint8_t *color_map;
1668 MapCdf map_cdf;
1669 ColorCost color_cost;
1670} Av1ColorMapParam;
1671
Yue Chen19e7aa82016-11-30 14:05:39 -08001672#if CONFIG_GLOBAL_MOTION
1673static INLINE int is_nontrans_global_motion(const MACROBLOCKD *xd) {
1674 const MODE_INFO *mi = xd->mi[0];
1675 const MB_MODE_INFO *const mbmi = &mi->mbmi;
1676 int ref;
1677#if CONFIG_CB4X4
1678 const int unify_bsize = 1;
1679#else
1680 const int unify_bsize = 0;
1681#endif
1682
1683 // First check if all modes are ZEROMV
1684 if (mbmi->sb_type >= BLOCK_8X8 || unify_bsize) {
1685#if CONFIG_EXT_INTER
1686 if (mbmi->mode != ZEROMV && mbmi->mode != ZERO_ZEROMV) return 0;
1687#else
1688 if (mbmi->mode != ZEROMV) return 0;
1689#endif // CONFIG_EXT_INTER
1690 } else {
1691#if CONFIG_EXT_INTER
David Barkera0c16382017-07-07 17:01:30 +01001692 if ((mi->bmi[0].as_mode != ZEROMV && mi->bmi[0].as_mode != ZERO_ZEROMV) ||
1693 (mi->bmi[1].as_mode != ZEROMV && mi->bmi[1].as_mode != ZERO_ZEROMV) ||
1694 (mi->bmi[2].as_mode != ZEROMV && mi->bmi[2].as_mode != ZERO_ZEROMV) ||
1695 (mi->bmi[3].as_mode != ZEROMV && mi->bmi[3].as_mode != ZERO_ZEROMV))
Yue Chen19e7aa82016-11-30 14:05:39 -08001696 return 0;
1697#else
1698 if (mi->bmi[0].as_mode != ZEROMV || mi->bmi[1].as_mode != ZEROMV ||
1699 mi->bmi[2].as_mode != ZEROMV || mi->bmi[3].as_mode != ZEROMV)
1700 return 0;
1701#endif // CONFIG_EXT_INTER
1702 }
Jingning Han3ca0e672017-04-14 19:48:05 -07001703
1704#if !GLOBAL_SUB8X8_USED
1705 if (mbmi->sb_type < BLOCK_8X8) return 0;
1706#endif
1707
Yue Chen19e7aa82016-11-30 14:05:39 -08001708 // Now check if all global motion is non translational
1709 for (ref = 0; ref < 1 + has_second_ref(mbmi); ++ref) {
1710 if (xd->global_motion[mbmi->ref_frame[ref]].wmtype <= TRANSLATION) return 0;
1711 }
1712 return 1;
1713}
1714#endif // CONFIG_GLOBAL_MOTION
1715
Yaowu Xu4ff59b52017-04-24 12:41:56 -07001716static INLINE PLANE_TYPE get_plane_type(int plane) {
Luc Trudeau005feb62017-02-22 13:34:01 -05001717 return (plane == 0) ? PLANE_TYPE_Y : PLANE_TYPE_UV;
1718}
1719
Angie Chiang155bf9a2017-08-06 19:52:57 -07001720static INLINE void transpose_uint8(uint8_t *dst, int dst_stride,
1721 const uint8_t *src, int src_stride, int w,
1722 int h) {
1723 int r, c;
1724 for (r = 0; r < h; ++r)
1725 for (c = 0; c < w; ++c) dst[c * dst_stride + r] = src[r * src_stride + c];
1726}
1727
1728static INLINE void transpose_uint16(uint16_t *dst, int dst_stride,
1729 const uint16_t *src, int src_stride, int w,
1730 int h) {
1731 int r, c;
1732 for (r = 0; r < h; ++r)
1733 for (c = 0; c < w; ++c) dst[c * dst_stride + r] = src[r * src_stride + c];
1734}
1735
1736static INLINE void transpose_int16(int16_t *dst, int dst_stride,
1737 const int16_t *src, int src_stride, int w,
1738 int h) {
1739 int r, c;
1740 for (r = 0; r < h; ++r)
1741 for (c = 0; c < w; ++c) dst[c * dst_stride + r] = src[r * src_stride + c];
1742}
1743
1744static INLINE void transpose_int32(int32_t *dst, int dst_stride,
1745 const int32_t *src, int src_stride, int w,
1746 int h) {
1747 int r, c;
1748 for (r = 0; r < h; ++r)
1749 for (c = 0; c < w; ++c) dst[c * dst_stride + r] = src[r * src_stride + c];
1750}
1751
Yaowu Xuc27fc142016-08-22 16:08:15 -07001752#ifdef __cplusplus
1753} // extern "C"
1754#endif
1755
Yaowu Xuf883b422016-08-30 14:01:10 -07001756#endif // AV1_COMMON_BLOCKD_H_