blob: 7927a1d5ef4882f99e6b63faa7c0a5f4b6c6f6d6 [file] [log] [blame]
Yaowu Xuc27fc142016-08-22 16:08:15 -07001/*
Yaowu Xu2ab7ff02016-09-02 12:04:54 -07002 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Yaowu Xuc27fc142016-08-22 16:08:15 -07003 *
Yaowu Xu2ab7ff02016-09-02 12:04:54 -07004 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Yaowu Xuc27fc142016-08-22 16:08:15 -070010 */
11
Yaowu Xuf883b422016-08-30 14:01:10 -070012#ifndef AV1_COMMON_BLOCKD_H_
13#define AV1_COMMON_BLOCKD_H_
Yaowu Xuc27fc142016-08-22 16:08:15 -070014
Yaowu Xuf883b422016-08-30 14:01:10 -070015#include "./aom_config.h"
Yaowu Xuc27fc142016-08-22 16:08:15 -070016
Yaowu Xuf883b422016-08-30 14:01:10 -070017#include "aom_dsp/aom_dsp_common.h"
Yaowu Xuc27fc142016-08-22 16:08:15 -070018#include "aom_ports/mem.h"
19#include "aom_scale/yv12config.h"
20
21#include "av1/common/common_data.h"
22#include "av1/common/quant_common.h"
23#include "av1/common/entropy.h"
24#include "av1/common/entropymode.h"
25#include "av1/common/mv.h"
26#include "av1/common/scale.h"
27#include "av1/common/seg_common.h"
28#include "av1/common/tile_common.h"
Yushin Chod0b77ac2017-10-20 17:33:16 -070029
Yaowu Xuc27fc142016-08-22 16:08:15 -070030#ifdef __cplusplus
31extern "C" {
32#endif
33
Debargha Mukherjee6cf2b462018-01-12 15:53:50 -080034#define USE_B_QUANT_NO_TRELLIS 1
Debargha Mukherjee6cf2b462018-01-12 15:53:50 -080035
Yaowu Xuc27fc142016-08-22 16:08:15 -070036#define MAX_MB_PLANE 3
37
Debargha Mukherjee1edf9a32017-01-07 18:54:20 -080038// Set COMPOUND_SEGMENT_TYPE to one of the three
39// 0: Uniform
40// 1: Difference weighted
41#define COMPOUND_SEGMENT_TYPE 1
Debargha Mukherjee1edf9a32017-01-07 18:54:20 -080042#define MAX_SEG_MASK_BITS 1
Yaowu Xuf35f5272017-05-10 08:00:02 -070043
Sarah Parkerb9f757c2017-01-06 17:12:24 -080044// SEG_MASK_TYPES should not surpass 1 << MAX_SEG_MASK_BITS
45typedef enum {
Yaowu Xuf35f5272017-05-10 08:00:02 -070046#if COMPOUND_SEGMENT_TYPE == 0
Sarah Parkerb9f757c2017-01-06 17:12:24 -080047 UNIFORM_45 = 0,
48 UNIFORM_45_INV,
Debargha Mukherjee1edf9a32017-01-07 18:54:20 -080049#elif COMPOUND_SEGMENT_TYPE == 1
Sarah Parker7bb84f32017-05-09 15:17:46 -070050 DIFFWTD_38 = 0,
51 DIFFWTD_38_INV,
Yaowu Xuf35f5272017-05-10 08:00:02 -070052#endif // COMPOUND_SEGMENT_TYPE
Debargha Mukherjee1edf9a32017-01-07 18:54:20 -080053 SEG_MASK_TYPES,
54} SEG_MASK_TYPE;
55
Yaowu Xuc27fc142016-08-22 16:08:15 -070056typedef enum {
57 KEY_FRAME = 0,
58 INTER_FRAME = 1,
Soo-Chul Han65c00ae2017-09-07 13:12:35 -040059 INTRA_ONLY_FRAME = 2, // replaces intra-only
60 S_FRAME = 3,
Yaowu Xuc27fc142016-08-22 16:08:15 -070061 FRAME_TYPES,
62} FRAME_TYPE;
63
Debargha Mukherjee0f248c42017-09-07 12:40:18 -070064static INLINE int is_comp_ref_allowed(BLOCK_SIZE bsize) {
Debargha Mukherjee0f248c42017-09-07 12:40:18 -070065 return AOMMIN(block_size_wide[bsize], block_size_high[bsize]) >= 8;
Debargha Mukherjee0f248c42017-09-07 12:40:18 -070066}
67
Yaowu Xuc27fc142016-08-22 16:08:15 -070068static INLINE int is_inter_mode(PREDICTION_MODE mode) {
Yaowu Xuc27fc142016-08-22 16:08:15 -070069 return mode >= NEARESTMV && mode <= NEW_NEWMV;
Yaowu Xuc27fc142016-08-22 16:08:15 -070070}
71
David Barkerac37fa32016-12-02 12:30:21 +000072typedef struct {
73 uint8_t *plane[MAX_MB_PLANE];
74 int stride[MAX_MB_PLANE];
75} BUFFER_SET;
76
Yaowu Xuc27fc142016-08-22 16:08:15 -070077static INLINE int is_inter_singleref_mode(PREDICTION_MODE mode) {
Zoe Liu7f24e1b2017-03-17 17:42:05 -070078 return mode >= NEARESTMV && mode <= NEWMV;
Yaowu Xuc27fc142016-08-22 16:08:15 -070079}
Zoe Liu85b66462017-04-20 14:28:19 -070080static INLINE int is_inter_compound_mode(PREDICTION_MODE mode) {
81 return mode >= NEAREST_NEARESTMV && mode <= NEW_NEWMV;
82}
Yaowu Xuc27fc142016-08-22 16:08:15 -070083
84static INLINE PREDICTION_MODE compound_ref0_mode(PREDICTION_MODE mode) {
Urvang Joshi5a9ea002017-05-22 15:25:18 -070085 static PREDICTION_MODE lut[] = {
Urvang Joshi102245d2016-11-28 13:05:36 -080086 MB_MODE_COUNT, // DC_PRED
87 MB_MODE_COUNT, // V_PRED
88 MB_MODE_COUNT, // H_PRED
89 MB_MODE_COUNT, // D45_PRED
90 MB_MODE_COUNT, // D135_PRED
Hui Su439bcb52018-02-21 10:52:16 -080091 MB_MODE_COUNT, // D113_PRED
92 MB_MODE_COUNT, // D157_PRED
93 MB_MODE_COUNT, // D203_PRED
94 MB_MODE_COUNT, // D67_PRED
Urvang Joshi102245d2016-11-28 13:05:36 -080095 MB_MODE_COUNT, // SMOOTH_PRED
Urvang Joshi5a9ea002017-05-22 15:25:18 -070096 MB_MODE_COUNT, // SMOOTH_V_PRED
97 MB_MODE_COUNT, // SMOOTH_H_PRED
Urvang Joshi96d1c0a2017-10-10 13:15:32 -070098 MB_MODE_COUNT, // PAETH_PRED
Urvang Joshi102245d2016-11-28 13:05:36 -080099 MB_MODE_COUNT, // NEARESTMV
100 MB_MODE_COUNT, // NEARMV
Sarah Parker2b9ec2e2017-10-30 17:34:08 -0700101 MB_MODE_COUNT, // GLOBALMV
Urvang Joshi102245d2016-11-28 13:05:36 -0800102 MB_MODE_COUNT, // NEWMV
Sebastien Alaiwan34d55662017-11-15 09:36:03 +0100103 NEARESTMV, // NEAREST_NEARESTMV
104 NEARMV, // NEAR_NEARMV
105 NEARESTMV, // NEAREST_NEWMV
106 NEWMV, // NEW_NEARESTMV
107 NEARMV, // NEAR_NEWMV
108 NEWMV, // NEW_NEARMV
109 GLOBALMV, // GLOBAL_GLOBALMV
110 NEWMV, // NEW_NEWMV
Yaowu Xuc27fc142016-08-22 16:08:15 -0700111 };
Urvang Joshi5a9ea002017-05-22 15:25:18 -0700112 assert(NELEMENTS(lut) == MB_MODE_COUNT);
Yaowu Xuc27fc142016-08-22 16:08:15 -0700113 assert(is_inter_compound_mode(mode));
114 return lut[mode];
115}
116
117static INLINE PREDICTION_MODE compound_ref1_mode(PREDICTION_MODE mode) {
Urvang Joshi5a9ea002017-05-22 15:25:18 -0700118 static PREDICTION_MODE lut[] = {
Urvang Joshi102245d2016-11-28 13:05:36 -0800119 MB_MODE_COUNT, // DC_PRED
120 MB_MODE_COUNT, // V_PRED
121 MB_MODE_COUNT, // H_PRED
122 MB_MODE_COUNT, // D45_PRED
123 MB_MODE_COUNT, // D135_PRED
Hui Su439bcb52018-02-21 10:52:16 -0800124 MB_MODE_COUNT, // D113_PRED
125 MB_MODE_COUNT, // D157_PRED
126 MB_MODE_COUNT, // D203_PRED
127 MB_MODE_COUNT, // D67_PRED
Urvang Joshi102245d2016-11-28 13:05:36 -0800128 MB_MODE_COUNT, // SMOOTH_PRED
Urvang Joshi5a9ea002017-05-22 15:25:18 -0700129 MB_MODE_COUNT, // SMOOTH_V_PRED
130 MB_MODE_COUNT, // SMOOTH_H_PRED
Urvang Joshi96d1c0a2017-10-10 13:15:32 -0700131 MB_MODE_COUNT, // PAETH_PRED
Urvang Joshi102245d2016-11-28 13:05:36 -0800132 MB_MODE_COUNT, // NEARESTMV
133 MB_MODE_COUNT, // NEARMV
Sarah Parker2b9ec2e2017-10-30 17:34:08 -0700134 MB_MODE_COUNT, // GLOBALMV
Urvang Joshi102245d2016-11-28 13:05:36 -0800135 MB_MODE_COUNT, // NEWMV
Sebastien Alaiwan34d55662017-11-15 09:36:03 +0100136 NEARESTMV, // NEAREST_NEARESTMV
137 NEARMV, // NEAR_NEARMV
138 NEWMV, // NEAREST_NEWMV
139 NEARESTMV, // NEW_NEARESTMV
140 NEWMV, // NEAR_NEWMV
141 NEARMV, // NEW_NEARMV
142 GLOBALMV, // GLOBAL_GLOBALMV
143 NEWMV, // NEW_NEWMV
Yaowu Xuc27fc142016-08-22 16:08:15 -0700144 };
Urvang Joshi5a9ea002017-05-22 15:25:18 -0700145 assert(NELEMENTS(lut) == MB_MODE_COUNT);
Yaowu Xuc27fc142016-08-22 16:08:15 -0700146 assert(is_inter_compound_mode(mode));
147 return lut[mode];
148}
149
David Barker3dfba992017-04-03 16:10:09 +0100150static INLINE int have_nearmv_in_inter_mode(PREDICTION_MODE mode) {
Debargha Mukherjeebb6e1342017-04-17 16:05:04 -0700151 return (mode == NEARMV || mode == NEAR_NEARMV || mode == NEAR_NEWMV ||
152 mode == NEW_NEARMV);
David Barker3dfba992017-04-03 16:10:09 +0100153}
154
Yaowu Xuc27fc142016-08-22 16:08:15 -0700155static INLINE int have_newmv_in_inter_mode(PREDICTION_MODE mode) {
Zoe Liu7f24e1b2017-03-17 17:42:05 -0700156 return (mode == NEWMV || mode == NEW_NEWMV || mode == NEAREST_NEWMV ||
157 mode == NEW_NEARESTMV || mode == NEAR_NEWMV || mode == NEW_NEARMV);
Yaowu Xuc27fc142016-08-22 16:08:15 -0700158}
Sarah Parker6fdc8532016-11-16 17:47:13 -0800159
Sarah Parker2e604882017-01-17 17:31:25 -0800160static INLINE int use_masked_motion_search(COMPOUND_TYPE type) {
161 return (type == COMPOUND_WEDGE);
162}
163
Sarah Parker6fdc8532016-11-16 17:47:13 -0800164static INLINE int is_masked_compound_type(COMPOUND_TYPE type) {
Sarah Parker569edda2016-12-14 14:57:38 -0800165 return (type == COMPOUND_WEDGE || type == COMPOUND_SEG);
Sarah Parker6fdc8532016-11-16 17:47:13 -0800166}
Zoe Liu85b66462017-04-20 14:28:19 -0700167
Yaowu Xuc27fc142016-08-22 16:08:15 -0700168/* For keyframes, intra block modes are predicted by the (already decoded)
169 modes for the Y blocks to the left and above us; for interframes, there
170 is a single probability table. */
171
Yaowu Xuc27fc142016-08-22 16:08:15 -0700172typedef int8_t MV_REFERENCE_FRAME;
173
174typedef struct {
175 // Number of base colors for Y (0) and UV (1)
176 uint8_t palette_size[2];
hui sufa4ff852017-05-15 12:20:50 -0700177 // Value of base colors for Y, U, and V
Yaowu Xuc27fc142016-08-22 16:08:15 -0700178 uint16_t palette_colors[3 * PALETTE_MAX_SIZE];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700179} PALETTE_MODE_INFO;
180
hui su5db97432016-10-14 16:10:14 -0700181#if CONFIG_FILTER_INTRA
Yaowu Xuc27fc142016-08-22 16:08:15 -0700182typedef struct {
Yue Chenb0571872017-12-18 18:12:59 -0800183 uint8_t use_filter_intra;
184 FILTER_INTRA_MODE filter_intra_mode;
hui su5db97432016-10-14 16:10:14 -0700185} FILTER_INTRA_MODE_INFO;
Yue Chen57b8ff62017-10-10 23:37:31 -0700186
187static const PREDICTION_MODE fimode_to_intradir[FILTER_INTRA_MODES] = {
Hui Su69ae7b82018-02-20 20:10:51 -0800188 DC_PRED, V_PRED, H_PRED, D157_PRED, DC_PRED
Yue Chen57b8ff62017-10-10 23:37:31 -0700189};
hui su5db97432016-10-14 16:10:14 -0700190#endif // CONFIG_FILTER_INTRA
Yaowu Xuc27fc142016-08-22 16:08:15 -0700191
Angie Chiang7c2b7f22016-11-07 16:00:00 -0800192#if CONFIG_RD_DEBUG
Jingning Hanf32f6782018-02-15 09:29:15 -0800193#define TXB_COEFF_COST_MAP_SIZE (MAX_MIB_SIZE)
Angie Chiang7c2b7f22016-11-07 16:00:00 -0800194#endif
Angie Chiang9a44f5f2016-11-06 12:19:06 -0800195
Angie Chiang9a44f5f2016-11-06 12:19:06 -0800196typedef struct RD_STATS {
197 int rate;
198 int64_t dist;
Angie Chiang7bbd3b12017-04-26 11:06:09 -0700199 // Please be careful of using rdcost, it's not guaranteed to be set all the
200 // time.
201 // TODO(angiebird): Create a set of functions to manipulate the RD_STATS. In
202 // these functions, make sure rdcost is always up-to-date according to
203 // rate/dist.
Angie Chiang2a2a7dd2017-04-25 16:08:47 -0700204 int64_t rdcost;
Angie Chiang9a44f5f2016-11-06 12:19:06 -0800205 int64_t sse;
Angie Chiang7bbd3b12017-04-26 11:06:09 -0700206 int skip; // sse should equal to dist when skip == 1
Jingning Han3bce7542017-07-25 10:53:57 -0700207 int64_t ref_rdcost;
208 int zero_rate;
Jingning Han1a7f0a82017-07-27 09:48:05 -0700209 uint8_t invalid_rate;
Angie Chiang9a44f5f2016-11-06 12:19:06 -0800210#if CONFIG_RD_DEBUG
211 int txb_coeff_cost[MAX_MB_PLANE];
212 int txb_coeff_cost_map[MAX_MB_PLANE][TXB_COEFF_COST_MAP_SIZE]
213 [TXB_COEFF_COST_MAP_SIZE];
Angie Chiang3963d632016-11-10 18:41:40 -0800214#endif // CONFIG_RD_DEBUG
Angie Chiang9a44f5f2016-11-06 12:19:06 -0800215} RD_STATS;
Angie Chiang9a44f5f2016-11-06 12:19:06 -0800216
Sarah Parker2d0e9b72017-05-04 01:34:16 +0000217// This struct is used to group function args that are commonly
218// sent together in functions related to interinter compound modes
Sarah Parker6fdc8532016-11-16 17:47:13 -0800219typedef struct {
Sarah Parker6fdc8532016-11-16 17:47:13 -0800220 int wedge_index;
221 int wedge_sign;
Sarah Parkerb9f757c2017-01-06 17:12:24 -0800222 SEG_MASK_TYPE mask_type;
Sarah Parker2d0e9b72017-05-04 01:34:16 +0000223 uint8_t *seg_mask;
Sarah Parker2d0e9b72017-05-04 01:34:16 +0000224 COMPOUND_TYPE interinter_compound_type;
Sarah Parker6fdc8532016-11-16 17:47:13 -0800225} INTERINTER_COMPOUND_DATA;
Sarah Parker6fdc8532016-11-16 17:47:13 -0800226
Hui Su7167d952018-02-01 16:33:12 -0800227#define INTER_TX_SIZE_BUF_LEN 16
Hui Sude1a5db2018-02-22 15:44:49 -0800228#if CONFIG_TXK_SEL
229#define TXK_TYPE_BUF_LEN 64
230#endif
Hui Su7167d952018-02-01 16:33:12 -0800231// This structure now relates to 4x4 block regions.
Luc Trudeauf5334002017-04-25 12:21:26 -0400232typedef struct MB_MODE_INFO {
Yaowu Xuc27fc142016-08-22 16:08:15 -0700233 // Common for both INTER and INTRA blocks
234 BLOCK_SIZE sb_type;
235 PREDICTION_MODE mode;
236 TX_SIZE tx_size;
Jingning Hane67b38a2016-11-04 10:30:00 -0700237 TX_SIZE min_tx_size;
Hui Su7167d952018-02-01 16:33:12 -0800238 uint8_t inter_tx_size[INTER_TX_SIZE_BUF_LEN];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700239 int8_t skip;
Zoe Liuf704a1c2017-10-02 16:55:59 -0700240#if CONFIG_EXT_SKIP
241 int8_t skip_mode;
242#endif // CONFIG_EXT_SKIP
Yaowu Xuc27fc142016-08-22 16:08:15 -0700243 int8_t segment_id;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700244 int8_t seg_id_predicted; // valid only when temporal_update is enabled
245
246 // Only for INTRA blocks
Luc Trudeaud6d9eee2017-07-12 12:36:50 -0400247 UV_PREDICTION_MODE uv_mode;
Urvang Joshic6300aa2017-06-01 14:46:23 -0700248
Yaowu Xuc27fc142016-08-22 16:08:15 -0700249 PALETTE_MODE_INFO palette_mode_info;
Alex Converse28744302017-04-13 14:46:22 -0700250 uint8_t use_intrabc;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700251
Rupert Swarbrick27e90292017-09-28 17:46:50 +0100252 // Only for INTER blocks
253 InterpFilters interp_filters;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700254 MV_REFERENCE_FRAME ref_frame[2];
255 TX_TYPE tx_type;
Angie Chiangcd9b03f2017-04-16 13:37:13 -0700256#if CONFIG_TXK_SEL
Hui Sude1a5db2018-02-22 15:44:49 -0800257 TX_TYPE txk_type[TXK_TYPE_BUF_LEN];
Angie Chiang808d8592017-04-06 18:36:55 -0700258#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -0700259
hui su5db97432016-10-14 16:10:14 -0700260#if CONFIG_FILTER_INTRA
261 FILTER_INTRA_MODE_INFO filter_intra_mode_info;
262#endif // CONFIG_FILTER_INTRA
hui sueda3d762016-12-06 16:58:23 -0800263 // The actual prediction angle is the base angle + (angle_delta * step).
Luc Trudeau7bb3a4f2018-02-12 09:59:21 -0500264 int8_t angle_delta[PLANE_TYPES];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700265
Sarah Parker2d0e9b72017-05-04 01:34:16 +0000266 // interintra members
Yaowu Xuc27fc142016-08-22 16:08:15 -0700267 INTERINTRA_MODE interintra_mode;
268 // TODO(debargha): Consolidate these flags
269 int use_wedge_interintra;
270 int interintra_wedge_index;
271 int interintra_wedge_sign;
Sarah Parker2d0e9b72017-05-04 01:34:16 +0000272 // interinter members
273 COMPOUND_TYPE interinter_compound_type;
Sarah Parker2d0e9b72017-05-04 01:34:16 +0000274 int wedge_index;
275 int wedge_sign;
Sarah Parker2d0e9b72017-05-04 01:34:16 +0000276 SEG_MASK_TYPE mask_type;
Yue Chencb60b182016-10-13 15:18:22 -0700277 MOTION_MODE motion_mode;
Yue Chen5329a2b2017-02-28 17:33:00 +0800278 int overlappable_neighbors[2];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700279 int_mv mv[2];
280 int_mv pred_mv[2];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700281 uint8_t ref_mv_idx;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700282#if CONFIG_EXT_PARTITION_TYPES
283 PARTITION_TYPE partition;
284#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -0700285 /* deringing gain *per-superblock* */
Jean-Marc Valin5f5c1322017-03-21 16:20:21 -0400286 int8_t cdef_strength;
Arild Fuldseth07441162016-08-15 15:07:52 +0200287 int current_q_index;
Fangwen Fu231fe422017-04-24 17:52:29 -0700288#if CONFIG_EXT_DELTA_Q
289 int current_delta_lf_from_base;
Cheng Chena97394f2017-09-27 15:05:14 -0700290 int curr_delta_lf[FRAME_LF_COUNT];
Fangwen Fu231fe422017-04-24 17:52:29 -0700291#endif
Angie Chiangd4022822016-11-02 18:30:25 -0700292#if CONFIG_RD_DEBUG
Angie Chiang9a44f5f2016-11-06 12:19:06 -0800293 RD_STATS rd_stats;
Angie Chiangd4022822016-11-02 18:30:25 -0700294 int mi_row;
295 int mi_col;
296#endif
Yue Chen69f18e12016-09-08 14:48:15 -0700297 int num_proj_ref[2];
298 WarpedMotionParams wm_params[2];
Ryan Lei9b02b0e2017-01-30 15:52:20 -0800299
Luc Trudeauf5334002017-04-25 12:21:26 -0400300#if CONFIG_CFL
301 // Index of the alpha Cb and alpha Cr combination
Luc Trudeaua9bd85f2017-05-11 14:37:56 -0400302 int cfl_alpha_idx;
David Michael Barrf6eaa152017-07-19 19:42:28 +0900303 // Joint sign of alpha Cb and alpha Cr
304 int cfl_alpha_signs;
Luc Trudeauf5334002017-04-25 12:21:26 -0400305#endif
306
Cheng Chenca6958c2017-10-10 14:00:50 -0700307#if CONFIG_JNT_COMP
308 int compound_idx;
Cheng Chen33a13d92017-11-28 16:49:59 -0800309 int comp_group_idx;
Cheng Chenca6958c2017-10-10 14:00:50 -0700310#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -0700311} MB_MODE_INFO;
312
Johann6b41d4d2018-02-08 14:32:53 -0800313typedef struct MODE_INFO {
314 MB_MODE_INFO mbmi;
315} MODE_INFO;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700316
Hui Su27df8342017-11-07 15:16:05 -0800317#define NO_FILTER_FOR_IBC 1 // Disable in-loop filters for frame with intrabc
318
Alex Converse28744302017-04-13 14:46:22 -0700319static INLINE int is_intrabc_block(const MB_MODE_INFO *mbmi) {
320 return mbmi->use_intrabc;
321}
Alex Converse28744302017-04-13 14:46:22 -0700322
Luc Trudeaud6d9eee2017-07-12 12:36:50 -0400323static INLINE PREDICTION_MODE get_uv_mode(UV_PREDICTION_MODE mode) {
Luc Trudeau5d5078e2017-12-13 16:43:16 -0500324 assert(mode < UV_INTRA_MODES);
Luc Trudeau2f052ee2018-01-16 14:43:06 -0500325 static const PREDICTION_MODE uv2y[] = {
Urvang Joshib7301cd2017-11-09 15:08:56 -0800326 DC_PRED, // UV_DC_PRED
327 V_PRED, // UV_V_PRED
328 H_PRED, // UV_H_PRED
329 D45_PRED, // UV_D45_PRED
330 D135_PRED, // UV_D135_PRED
Hui Su69ae7b82018-02-20 20:10:51 -0800331 D113_PRED, // UV_D113_PRED
332 D157_PRED, // UV_D157_PRED
333 D203_PRED, // UV_D203_PRED
334 D67_PRED, // UV_D67_PRED
Urvang Joshib7301cd2017-11-09 15:08:56 -0800335 SMOOTH_PRED, // UV_SMOOTH_PRED
Luc Trudeaud6d9eee2017-07-12 12:36:50 -0400336 SMOOTH_V_PRED, // UV_SMOOTH_V_PRED
337 SMOOTH_H_PRED, // UV_SMOOTH_H_PRED
Urvang Joshi96d1c0a2017-10-10 13:15:32 -0700338 PAETH_PRED, // UV_PAETH_PRED
David Barker01faff92018-02-09 13:57:45 +0000339#if CONFIG_CFL
Luc Trudeau2f052ee2018-01-16 14:43:06 -0500340 DC_PRED, // UV_CFL_PRED
David Barker01faff92018-02-09 13:57:45 +0000341#endif // CONFIG_CFL
Luc Trudeau2f052ee2018-01-16 14:43:06 -0500342 INTRA_INVALID, // UV_INTRA_MODES
343 INTRA_INVALID, // UV_MODE_INVALID
Luc Trudeaud6d9eee2017-07-12 12:36:50 -0400344 };
345 return uv2y[mode];
346}
Luc Trudeaud6d9eee2017-07-12 12:36:50 -0400347
Yaowu Xuc27fc142016-08-22 16:08:15 -0700348static INLINE int is_inter_block(const MB_MODE_INFO *mbmi) {
Hui Su293f2812018-02-26 14:41:18 -0800349 return is_intrabc_block(mbmi) || mbmi->ref_frame[0] > INTRA_FRAME;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700350}
351
352static INLINE int has_second_ref(const MB_MODE_INFO *mbmi) {
353 return mbmi->ref_frame[1] > INTRA_FRAME;
354}
355
Zoe Liuc082bbc2017-05-17 13:31:37 -0700356#if CONFIG_EXT_COMP_REFS
357static INLINE int has_uni_comp_refs(const MB_MODE_INFO *mbmi) {
358 return has_second_ref(mbmi) && (!((mbmi->ref_frame[0] >= BWDREF_FRAME) ^
359 (mbmi->ref_frame[1] >= BWDREF_FRAME)));
360}
Zoe Liu057f97b2018-01-24 13:45:25 -0800361#endif // CONFIG_EXT_COMP_REFS
Zoe Liuc082bbc2017-05-17 13:31:37 -0700362
363static INLINE MV_REFERENCE_FRAME comp_ref0(int ref_idx) {
364 static const MV_REFERENCE_FRAME lut[] = {
Zoe Liu057f97b2018-01-24 13:45:25 -0800365 LAST_FRAME, // LAST_LAST2_FRAMES,
366 LAST_FRAME, // LAST_LAST3_FRAMES,
367 LAST_FRAME, // LAST_GOLDEN_FRAMES,
368 BWDREF_FRAME, // BWDREF_ALTREF_FRAMES,
369 LAST2_FRAME, // LAST2_LAST3_FRAMES
370 LAST2_FRAME, // LAST2_GOLDEN_FRAMES,
371 LAST3_FRAME, // LAST3_GOLDEN_FRAMES,
372 BWDREF_FRAME, // BWDREF_ALTREF2_FRAMES,
373 ALTREF2_FRAME, // ALTREF2_ALTREF_FRAMES,
Zoe Liuc082bbc2017-05-17 13:31:37 -0700374 };
Zoe Liu057f97b2018-01-24 13:45:25 -0800375 assert(NELEMENTS(lut) == TOTAL_UNIDIR_COMP_REFS);
Zoe Liuc082bbc2017-05-17 13:31:37 -0700376 return lut[ref_idx];
377}
378
379static INLINE MV_REFERENCE_FRAME comp_ref1(int ref_idx) {
380 static const MV_REFERENCE_FRAME lut[] = {
Zoe Liu057f97b2018-01-24 13:45:25 -0800381 LAST2_FRAME, // LAST_LAST2_FRAMES,
382 LAST3_FRAME, // LAST_LAST3_FRAMES,
383 GOLDEN_FRAME, // LAST_GOLDEN_FRAMES,
384 ALTREF_FRAME, // BWDREF_ALTREF_FRAMES,
385 LAST3_FRAME, // LAST2_LAST3_FRAMES
386 GOLDEN_FRAME, // LAST2_GOLDEN_FRAMES,
387 GOLDEN_FRAME, // LAST3_GOLDEN_FRAMES,
388 ALTREF2_FRAME, // BWDREF_ALTREF2_FRAMES,
389 ALTREF_FRAME, // ALTREF2_ALTREF_FRAMES,
Zoe Liuc082bbc2017-05-17 13:31:37 -0700390 };
Zoe Liu057f97b2018-01-24 13:45:25 -0800391 assert(NELEMENTS(lut) == TOTAL_UNIDIR_COMP_REFS);
Zoe Liuc082bbc2017-05-17 13:31:37 -0700392 return lut[ref_idx];
393}
Zoe Liuc082bbc2017-05-17 13:31:37 -0700394
Jingning Han9010e202017-12-14 14:48:09 -0800395PREDICTION_MODE av1_left_block_mode(const MODE_INFO *left_mi);
Yaowu Xuc27fc142016-08-22 16:08:15 -0700396
Jingning Han9010e202017-12-14 14:48:09 -0800397PREDICTION_MODE av1_above_block_mode(const MODE_INFO *above_mi);
Yaowu Xuc27fc142016-08-22 16:08:15 -0700398
Luc Trudeau2eb9b842017-12-13 11:19:16 -0500399static INLINE int is_global_mv_block(const MODE_INFO *mi,
Sarah Parker19234cc2017-03-10 16:43:25 -0800400 TransformationType type) {
Luc Trudeau2eb9b842017-12-13 11:19:16 -0500401 const MB_MODE_INFO *const mbmi = &mi->mbmi;
402 const PREDICTION_MODE mode = mbmi->mode;
Sarah Parker19234cc2017-03-10 16:43:25 -0800403#if GLOBAL_SUB8X8_USED
404 const int block_size_allowed = 1;
405#else
Luc Trudeau2eb9b842017-12-13 11:19:16 -0500406 const BLOCK_SIZE bsize = mbmi->sb_type;
Debargha Mukherjeec17a4432017-08-28 22:22:45 -0700407 const int block_size_allowed =
408 AOMMIN(block_size_wide[bsize], block_size_high[bsize]) >= 8;
Sarah Parker19234cc2017-03-10 16:43:25 -0800409#endif // GLOBAL_SUB8X8_USED
Sarah Parker2b9ec2e2017-10-30 17:34:08 -0700410 return (mode == GLOBALMV || mode == GLOBAL_GLOBALMV) && type > TRANSLATION &&
Sarah Parker19234cc2017-03-10 16:43:25 -0800411 block_size_allowed;
Sarah Parker19234cc2017-03-10 16:43:25 -0800412}
Sarah Parker19234cc2017-03-10 16:43:25 -0800413
Angie Chiang5b5f4df2017-12-06 10:41:12 -0800414#if CONFIG_MISMATCH_DEBUG
415static INLINE void mi_to_pixel_loc(int *pixel_c, int *pixel_r, int mi_col,
416 int mi_row, int tx_blk_col, int tx_blk_row,
417 int subsampling_x, int subsampling_y) {
418 *pixel_c = ((mi_col >> subsampling_x) << MI_SIZE_LOG2) +
419 (tx_blk_col << tx_size_wide_log2[0]);
420 *pixel_r = ((mi_row >> subsampling_y) << MI_SIZE_LOG2) +
421 (tx_blk_row << tx_size_high_log2[0]);
422}
423#endif
424
Yaowu Xuc27fc142016-08-22 16:08:15 -0700425enum mv_precision { MV_PRECISION_Q3, MV_PRECISION_Q4 };
426
427struct buf_2d {
428 uint8_t *buf;
429 uint8_t *buf0;
430 int width;
431 int height;
432 int stride;
433};
434
435typedef struct macroblockd_plane {
436 tran_low_t *dqcoeff;
437 PLANE_TYPE plane_type;
438 int subsampling_x;
439 int subsampling_y;
440 struct buf_2d dst;
441 struct buf_2d pre[2];
442 ENTROPY_CONTEXT *above_context;
443 ENTROPY_CONTEXT *left_context;
Monty Montgomery125c0fc2017-10-26 00:44:35 -0400444
445 // The dequantizers below are true dequntizers used only in the
446 // dequantization process. They have the same coefficient
447 // shift/scale as TX.
448 int16_t seg_dequant_QTX[MAX_SEGMENTS][2];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700449 uint8_t *color_index_map;
450
Jingning Hanc47fe6c2016-10-21 16:40:47 -0700451 // block size in pixels
452 uint8_t width, height;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700453
Thomas Daviesdd3cf832017-10-20 15:49:57 +0100454 qm_val_t *seg_iqmatrix[MAX_SEGMENTS][TX_SIZES_ALL];
455 qm_val_t *seg_qmatrix[MAX_SEGMENTS][TX_SIZES_ALL];
Monty Montgomery125c0fc2017-10-26 00:44:35 -0400456
457 // the 'dequantizers' below are not literal dequantizer values.
458 // They're used by encoder RDO to generate ad-hoc lambda values.
459 // They use a hardwired Q3 coeff shift and do not necessarily match
460 // the TX scale in use.
461 const int16_t *dequant_Q3;
Yushin Cho77bba8d2016-11-04 16:36:56 -0700462
Yushin Chod0b77ac2017-10-20 17:33:16 -0700463#if CONFIG_DIST_8X8
Yushin Cho77bba8d2016-11-04 16:36:56 -0700464 DECLARE_ALIGNED(16, int16_t, pred[MAX_SB_SQUARE]);
Yushin Chob7b60c52017-07-14 16:18:52 -0700465#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -0700466} MACROBLOCKD_PLANE;
467
Jingning Han3468df12016-12-05 17:53:16 -0800468#define BLOCK_OFFSET(x, i) \
469 ((x) + (i) * (1 << (tx_size_wide_log2[0] + tx_size_high_log2[0])))
Yaowu Xuc27fc142016-08-22 16:08:15 -0700470
471typedef struct RefBuffer {
Zoe Liub57fee02018-02-15 17:41:41 -0800472 int idx; // frame buf idx
473#if CONFIG_FRAME_REFS_SIGNALING
474 int map_idx; // frame map idx
475#endif // CONFIG_FRAME_REFS_SIGNALING
Yaowu Xuc27fc142016-08-22 16:08:15 -0700476 YV12_BUFFER_CONFIG *buf;
477 struct scale_factors sf;
478} RefBuffer;
479
Rupert Swarbrick6c545212017-09-01 17:17:25 +0100480#if CONFIG_LOOP_RESTORATION
481typedef struct {
482 DECLARE_ALIGNED(16, InterpKernel, vfilter);
483 DECLARE_ALIGNED(16, InterpKernel, hfilter);
484} WienerInfo;
485
486typedef struct {
487 int ep;
488 int xqd[2];
489} SgrprojInfo;
490#endif // CONFIG_LOOP_RESTORATION
491
David Michael Barr5b2021e2017-08-17 18:12:39 +0900492#if CONFIG_CFL
Hui Su9fa96232017-10-23 15:46:04 -0700493#if CONFIG_DEBUG
Luc Trudeau03b7e7d2017-09-19 13:20:52 -0400494#define CFL_SUB8X8_VAL_MI_SIZE (4)
495#define CFL_SUB8X8_VAL_MI_SQUARE \
496 (CFL_SUB8X8_VAL_MI_SIZE * CFL_SUB8X8_VAL_MI_SIZE)
Hui Su9fa96232017-10-23 15:46:04 -0700497#endif // CONFIG_DEBUG
Luc Trudeaue425f472017-12-08 14:48:46 -0500498#define CFL_MAX_BLOCK_SIZE (BLOCK_32X32)
Luc Trudeau467205a2017-12-12 23:23:47 -0500499#define CFL_BUF_LINE (32)
Luc Trudeau365f73b2017-12-23 00:07:29 -0500500#define CFL_BUF_LINE_I128 (CFL_BUF_LINE >> 3)
501#define CFL_BUF_LINE_I256 (CFL_BUF_LINE >> 4)
Luc Trudeau467205a2017-12-12 23:23:47 -0500502#define CFL_BUF_SQUARE (CFL_BUF_LINE * CFL_BUF_LINE)
David Michael Barr5b2021e2017-08-17 18:12:39 +0900503typedef struct cfl_ctx {
Luc Trudeau4e26d662017-09-11 13:08:40 -0400504 // The CfL prediction buffer is used in two steps:
505 // 1. Stores Q3 reconstructed luma pixels
506 // (only Q2 is required, but Q3 is used to avoid shifts)
507 // 2. Stores Q3 AC contributions (step1 - tx block avg)
Luc Trudeau467205a2017-12-12 23:23:47 -0500508 int16_t pred_buf_q3[CFL_BUF_SQUARE];
509
510 // Cache the DC_PRED when performing RDO, so it does not have to be recomputed
511 // for every scaling parameter
512 int dc_pred_is_cached[CFL_PRED_PLANES];
513 // The DC_PRED cache is disable when decoding
514 int use_dc_pred_cache;
515 // Only cache the first row of the DC_PRED
516 int16_t dc_pred_cache[CFL_PRED_PLANES][CFL_BUF_LINE];
David Michael Barr5b2021e2017-08-17 18:12:39 +0900517
Luc Trudeau4e26d662017-09-11 13:08:40 -0400518 // Height and width currently used in the CfL prediction buffer.
519 int buf_height, buf_width;
David Michael Barr5b2021e2017-08-17 18:12:39 +0900520
David Michael Barr5b2021e2017-08-17 18:12:39 +0900521 int are_parameters_computed;
522
523 // Chroma subsampling
524 int subsampling_x, subsampling_y;
525
David Michael Barr5b2021e2017-08-17 18:12:39 +0900526 int mi_row, mi_col;
527
528 // Whether the reconstructed luma pixels need to be stored
529 int store_y;
530
David Michael Barr1f8d0952017-10-11 17:46:39 +0900531#if CONFIG_DEBUG
532 int rate;
533#endif // CONFIG_DEBUG
534
David Michael Barr5b2021e2017-08-17 18:12:39 +0900535 int is_chroma_reference;
David Michael Barr5b2021e2017-08-17 18:12:39 +0900536} CFL_CTX;
537#endif // CONFIG_CFL
538
Cheng Chenf78632e2017-10-20 15:30:51 -0700539#if CONFIG_JNT_COMP
540typedef struct jnt_comp_params {
Cheng Chen8263f802017-11-14 15:50:00 -0800541 int use_jnt_comp_avg;
Cheng Chenf78632e2017-10-20 15:30:51 -0700542 int fwd_offset;
543 int bck_offset;
544} JNT_COMP_PARAMS;
545#endif // CONFIG_JNT_COMP
546
Yaowu Xuc27fc142016-08-22 16:08:15 -0700547typedef struct macroblockd {
548 struct macroblockd_plane plane[MAX_MB_PLANE];
549 uint8_t bmode_blocks_wl;
550 uint8_t bmode_blocks_hl;
551
552 FRAME_COUNTS *counts;
553 TileInfo tile;
554
555 int mi_stride;
556
557 MODE_INFO **mi;
558 MODE_INFO *left_mi;
559 MODE_INFO *above_mi;
560 MB_MODE_INFO *left_mbmi;
561 MB_MODE_INFO *above_mbmi;
David Barkerd3afdb92018-02-14 15:45:58 +0000562 MB_MODE_INFO *chroma_left_mbmi;
563 MB_MODE_INFO *chroma_above_mbmi;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700564
565 int up_available;
566 int left_available;
Jingning Han3da18d62017-05-02 12:43:58 -0700567 int chroma_up_available;
568 int chroma_left_available;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700569
Rupert Swarbrick57486c52017-08-14 10:53:49 +0100570 /* Distance of MB away from frame edges in subpixels (1/8th pixel) */
Yaowu Xuc27fc142016-08-22 16:08:15 -0700571 int mb_to_left_edge;
572 int mb_to_right_edge;
573 int mb_to_top_edge;
574 int mb_to_bottom_edge;
575
576 FRAME_CONTEXT *fc;
577
578 /* pointers to reference frames */
Urvang Joshi52648442016-10-13 17:27:51 -0700579 const RefBuffer *block_refs[2];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700580
581 /* pointer to current frame */
582 const YV12_BUFFER_CONFIG *cur_buf;
583
584 ENTROPY_CONTEXT *above_context[MAX_MB_PLANE];
585 ENTROPY_CONTEXT left_context[MAX_MB_PLANE][2 * MAX_MIB_SIZE];
586
587 PARTITION_CONTEXT *above_seg_context;
588 PARTITION_CONTEXT left_seg_context[MAX_MIB_SIZE];
589
Yaowu Xuc27fc142016-08-22 16:08:15 -0700590 TXFM_CONTEXT *above_txfm_context;
591 TXFM_CONTEXT *left_txfm_context;
Jingning Han331662e2017-05-30 17:03:32 -0700592 TXFM_CONTEXT left_txfm_context_buffer[2 * MAX_MIB_SIZE];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700593
Rupert Swarbrick6c545212017-09-01 17:17:25 +0100594#if CONFIG_LOOP_RESTORATION
595 WienerInfo wiener_info[MAX_MB_PLANE];
596 SgrprojInfo sgrproj_info[MAX_MB_PLANE];
597#endif // CONFIG_LOOP_RESTORATION
598
Jingning Hanff6ee6a2016-12-07 09:55:21 -0800599 // block dimension in the unit of mode_info.
Yaowu Xuc27fc142016-08-22 16:08:15 -0700600 uint8_t n8_w, n8_h;
601
Yaowu Xuc27fc142016-08-22 16:08:15 -0700602 uint8_t ref_mv_count[MODE_CTX_REF_FRAMES];
603 CANDIDATE_MV ref_mv_stack[MODE_CTX_REF_FRAMES][MAX_REF_MV_STACK_SIZE];
604 uint8_t is_sec_rect;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700605
Zoe Liufa8bad12018-01-23 14:32:31 -0800606 // Counts of each reference frame in the above and left neighboring blocks.
607 // NOTE: Take into account both single and comp references.
608 uint8_t neighbors_ref_counts[TOTAL_REFS_PER_FRAME];
609
Thomas Daviesf77d4ad2017-01-10 18:55:42 +0000610 FRAME_CONTEXT *tile_ctx;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700611 /* Bit depth: 8, 10, 12 */
612 int bd;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700613
Debargha Mukherjee3c42c092016-09-29 09:17:36 -0700614 int qindex[MAX_SEGMENTS];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700615 int lossless[MAX_SEGMENTS];
616 int corrupted;
RogerZhou3b635242017-09-19 10:06:46 -0700617#if CONFIG_AMVR
RogerZhou10a03802017-10-26 11:49:48 -0700618 int cur_frame_force_integer_mv;
RogerZhou3b635242017-09-19 10:06:46 -0700619// same with that in AV1_COMMON
620#endif
Yaowu Xuf883b422016-08-30 14:01:10 -0700621 struct aom_internal_error_info *error_info;
Luc Trudeauf3bf8b12017-12-08 14:38:41 -0500622 const WarpedMotionParams *global_motion;
Arild Fuldseth07441162016-08-15 15:07:52 +0200623 int prev_qindex;
624 int delta_qindex;
625 int current_qindex;
Fangwen Fu231fe422017-04-24 17:52:29 -0700626#if CONFIG_EXT_DELTA_Q
627 // Since actual frame level loop filtering level value is not available
628 // at the beginning of the tile (only available during actual filtering)
629 // at encoder side.we record the delta_lf (against the frame level loop
630 // filtering level) and code the delta between previous superblock's delta
631 // lf and current delta lf. It is equivalent to the delta between previous
632 // superblock's actual lf and current lf.
633 int prev_delta_lf_from_base;
634 int current_delta_lf_from_base;
Cheng Chena97394f2017-09-27 15:05:14 -0700635 // For this experiment, we have four frame filter levels for different plane
636 // and direction. So, to support the per superblock update, we need to add
637 // a few more params as below.
638 // 0: delta loop filter level for y plane vertical
639 // 1: delta loop filter level for y plane horizontal
640 // 2: delta loop filter level for u plane
641 // 3: delta loop filter level for v plane
642 // To make it consistent with the reference to each filter level in segment,
643 // we need to -1, since
644 // SEG_LVL_ALT_LF_Y_V = 1;
645 // SEG_LVL_ALT_LF_Y_H = 2;
646 // SEG_LVL_ALT_LF_U = 3;
647 // SEG_LVL_ALT_LF_V = 4;
648 int prev_delta_lf[FRAME_LF_COUNT];
649 int curr_delta_lf[FRAME_LF_COUNT];
Fangwen Fu231fe422017-04-24 17:52:29 -0700650#endif
Luc Trudeauf8164152017-04-11 16:20:51 -0400651
Sarah Parker2d0e9b72017-05-04 01:34:16 +0000652 DECLARE_ALIGNED(16, uint8_t, seg_mask[2 * MAX_SB_SQUARE]);
Sarah Parker2d0e9b72017-05-04 01:34:16 +0000653
Luc Trudeauf8164152017-04-11 16:20:51 -0400654#if CONFIG_CFL
Luc Trudeau1e84af52017-11-25 15:00:28 -0500655 CFL_CTX cfl;
Luc Trudeauf8164152017-04-11 16:20:51 -0400656#endif
Cheng Chenf78632e2017-10-20 15:30:51 -0700657
658#if CONFIG_JNT_COMP
659 JNT_COMP_PARAMS jcp_param;
660#endif
Yaowu Xuc27fc142016-08-22 16:08:15 -0700661} MACROBLOCKD;
662
Yi Luo51281092017-06-26 16:36:15 -0700663static INLINE int get_bitdepth_data_path_index(const MACROBLOCKD *xd) {
664 return xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH ? 1 : 0;
665}
666
Yaowu Xuc27fc142016-08-22 16:08:15 -0700667static INLINE BLOCK_SIZE get_subsize(BLOCK_SIZE bsize,
668 PARTITION_TYPE partition) {
669 if (partition == PARTITION_INVALID)
Urvang Joshicb586f32016-09-20 11:36:33 -0700670 return BLOCK_INVALID;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700671 else
672 return subsize_lookup[partition][bsize];
673}
674
Luc Trudeau9cea9932017-12-13 21:14:13 -0500675static TX_TYPE intra_mode_to_tx_type_context(const MB_MODE_INFO *mbmi,
676 PLANE_TYPE plane_type) {
677 static const TX_TYPE _intra_mode_to_tx_type_context[INTRA_MODES] = {
678 DCT_DCT, // DC
679 ADST_DCT, // V
680 DCT_ADST, // H
681 DCT_DCT, // D45
682 ADST_ADST, // D135
683 ADST_DCT, // D117
684 DCT_ADST, // D153
685 DCT_ADST, // D207
686 ADST_DCT, // D63
687 ADST_ADST, // SMOOTH
688 ADST_DCT, // SMOOTH_V
689 DCT_ADST, // SMOOTH_H
690 ADST_ADST, // PAETH
691 };
692 return plane_type == PLANE_TYPE_Y
693 ? _intra_mode_to_tx_type_context[mbmi->mode]
694 : _intra_mode_to_tx_type_context[get_uv_mode(mbmi->uv_mode)];
695}
Yaowu Xuc27fc142016-08-22 16:08:15 -0700696
Jingning Hanb83e64b2017-03-01 14:52:04 -0800697#define USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4 1
Debargha Mukherjee5a488a62016-11-22 22:24:10 -0800698
Sarah Parker076437f2017-03-14 17:39:53 -0700699static INLINE int is_rect_tx(TX_SIZE tx_size) { return tx_size >= TX_SIZES; }
Sarah Parker076437f2017-03-14 17:39:53 -0700700
Rupert Swarbrickfcff0b22017-10-05 09:26:04 +0100701static INLINE int block_signals_txsize(BLOCK_SIZE bsize) {
Rupert Swarbrickfcff0b22017-10-05 09:26:04 +0100702 return bsize > BLOCK_4X4;
Rupert Swarbrickfcff0b22017-10-05 09:26:04 +0100703}
704
Hui Suddbcde22017-09-18 17:22:02 -0700705// Number of transform types in each set type
706static const int av1_num_ext_tx_set[EXT_TX_SET_TYPES] = {
Sarah Parkercec7ba12017-11-03 16:46:09 -0700707 1, 2, 5, 7, 7, 10, 12, 16, 16,
Hui Suddbcde22017-09-18 17:22:02 -0700708};
709
Hui Suddbcde22017-09-18 17:22:02 -0700710static const int av1_ext_tx_used[EXT_TX_SET_TYPES][TX_TYPES] = {
Johann6b41d4d2018-02-08 14:32:53 -0800711 { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
712 { 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0 },
713 { 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0 },
714 { 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0 },
715 { 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0 },
716 { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0 },
717 { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0 },
718 { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 },
719 { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 },
Debargha Mukherjee08542b92017-02-21 01:08:14 -0800720};
721
722static INLINE TxSetType get_ext_tx_set_type(TX_SIZE tx_size, BLOCK_SIZE bs,
Sarah Parker5effe3f2017-02-23 12:49:10 -0800723 int is_inter, int use_reduced_set) {
Yue Chen56e226e2017-05-02 16:21:40 -0700724 const TX_SIZE tx_size_sqr_up = txsize_sqr_up_map[tx_size];
725 const TX_SIZE tx_size_sqr = txsize_sqr_map[tx_size];
Debargha Mukherjee6ea917e2017-10-19 09:31:29 -0700726#if USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4
Jingning Han1a00cff2016-12-28 14:53:14 -0800727 (void)bs;
Debargha Mukherjee570423c2017-10-01 00:35:20 -0700728 if (tx_size_sqr_up > TX_32X32) return EXT_TX_SET_DCTONLY;
Jingning Han1a00cff2016-12-28 14:53:14 -0800729#else
Debargha Mukherjee570423c2017-10-01 00:35:20 -0700730 if (tx_size_sqr_up > TX_32X32 || bs < BLOCK_8X8) return EXT_TX_SET_DCTONLY;
Debargha Mukherjee6ea917e2017-10-19 09:31:29 -0700731#endif // USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4
Yue Chen56e226e2017-05-02 16:21:40 -0700732 if (tx_size_sqr_up == TX_32X32)
Debargha Mukherjee08542b92017-02-21 01:08:14 -0800733 return is_inter ? EXT_TX_SET_DCT_IDTX : EXT_TX_SET_DCTONLY;
Sarah Parkerc7fa12c2018-01-29 14:48:46 -0800734 if (use_reduced_set)
735 return is_inter ? EXT_TX_SET_DCT_IDTX : EXT_TX_SET_DTT4_IDTX;
Hui Su3e509e32018-01-29 11:42:46 -0800736 if (is_inter) {
737 return (tx_size_sqr == TX_16X16 ? EXT_TX_SET_DTT9_IDTX_1DDCT
Sarah Parkercec7ba12017-11-03 16:46:09 -0700738 : EXT_TX_SET_ALL16);
Hui Su3e509e32018-01-29 11:42:46 -0800739 } else {
740 return (tx_size_sqr == TX_16X16 ? EXT_TX_SET_DTT4_IDTX
Yue Chen56e226e2017-05-02 16:21:40 -0700741 : EXT_TX_SET_DTT4_IDTX_1DDCT);
Hui Su3e509e32018-01-29 11:42:46 -0800742 }
Debargha Mukherjee08542b92017-02-21 01:08:14 -0800743}
744
Hui Su4a9be2a2017-09-19 14:41:49 -0700745// Maps tx set types to the indices.
746static const int ext_tx_set_index[2][EXT_TX_SET_TYPES] = {
Johann6b41d4d2018-02-08 14:32:53 -0800747 { // Intra
748 0, -1, 2, -1, 1, -1, -1, -1, -16 },
749 { // Inter
750 0, 3, -1, -1, -1, -1, 2, -1, 1 },
Hui Su4a9be2a2017-09-19 14:41:49 -0700751};
752
Debargha Mukherjee08542b92017-02-21 01:08:14 -0800753static INLINE int get_ext_tx_set(TX_SIZE tx_size, BLOCK_SIZE bs, int is_inter,
Sarah Parker5effe3f2017-02-23 12:49:10 -0800754 int use_reduced_set) {
Debargha Mukherjee08542b92017-02-21 01:08:14 -0800755 const TxSetType set_type =
Sarah Parker5effe3f2017-02-23 12:49:10 -0800756 get_ext_tx_set_type(tx_size, bs, is_inter, use_reduced_set);
Hui Su4a9be2a2017-09-19 14:41:49 -0700757 return ext_tx_set_index[is_inter][set_type];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700758}
759
Sarah Parkere68a3e42017-02-16 14:03:24 -0800760static INLINE int get_ext_tx_types(TX_SIZE tx_size, BLOCK_SIZE bs, int is_inter,
Sarah Parker5effe3f2017-02-23 12:49:10 -0800761 int use_reduced_set) {
762 const int set_type =
763 get_ext_tx_set_type(tx_size, bs, is_inter, use_reduced_set);
Hui Suddbcde22017-09-18 17:22:02 -0700764 return av1_num_ext_tx_set[set_type];
Yaowu Xuc27fc142016-08-22 16:08:15 -0700765}
766
Debargha Mukherjee3ebb0d02017-12-14 05:05:18 -0800767#define TXSIZEMAX(t1, t2) (tx_size_2d[(t1)] >= tx_size_2d[(t2)] ? (t1) : (t2))
768#define TXSIZEMIN(t1, t2) (tx_size_2d[(t1)] <= tx_size_2d[(t2)] ? (t1) : (t2))
769
Yaowu Xu25ff26a2018-02-26 11:20:10 -0800770static INLINE TX_SIZE get_max_rect_tx_size(BLOCK_SIZE bsize) {
Jingning Hand26fde22018-02-21 09:24:37 -0800771 return max_txsize_rect_lookup[bsize];
Yue Chen0797a202017-10-27 17:24:56 -0700772}
773
Yaowu Xu25ff26a2018-02-26 11:20:10 -0800774static INLINE TX_SIZE tx_size_from_tx_mode(BLOCK_SIZE bsize, TX_MODE tx_mode) {
Yaowu Xuc27fc142016-08-22 16:08:15 -0700775 const TX_SIZE largest_tx_size = tx_mode_to_biggest_tx_size[tx_mode];
Yaowu Xu25ff26a2018-02-26 11:20:10 -0800776 const TX_SIZE max_rect_tx_size = get_max_rect_tx_size(bsize);
Urvang Joshifeb925f2016-12-05 10:37:29 -0800777 if (bsize == BLOCK_4X4)
778 return AOMMIN(max_txsize_lookup[bsize], largest_tx_size);
Urvang Joshifeb925f2016-12-05 10:37:29 -0800779 if (txsize_sqr_map[max_rect_tx_size] <= largest_tx_size)
780 return max_rect_tx_size;
781 else
782 return largest_tx_size;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700783}
784
hui su5db97432016-10-14 16:10:14 -0700785extern const int16_t dr_intra_derivative[90];
hui su02c79742017-05-16 17:19:04 -0700786static const uint8_t mode_to_angle_map[] = {
Urvang Joshib7301cd2017-11-09 15:08:56 -0800787 0, 90, 180, 45, 135, 113, 157, 203, 67, 0, 0, 0, 0,
hui su5db97432016-10-14 16:10:14 -0700788};
Yaowu Xuc27fc142016-08-22 16:08:15 -0700789
Yue Chen1a5ab9f2017-12-14 14:53:51 -0800790#if CONFIG_FILTER_INTRA
Yue Chen1a5ab9f2017-12-14 14:53:51 -0800791static INLINE int av1_filter_intra_allowed_txsize(TX_SIZE tx) {
Yue Chen1a5ab9f2017-12-14 14:53:51 -0800792 if (tx == TX_INVALID) return 0;
793
Yue Chen1a5ab9f2017-12-14 14:53:51 -0800794 return tx_size_wide[tx] <= 32 && tx_size_high[tx] <= 32;
Yue Chen1a5ab9f2017-12-14 14:53:51 -0800795}
796
797static INLINE TX_SIZE av1_max_tx_size_for_filter_intra(BLOCK_SIZE bsize,
798 TX_MODE tx_mode) {
Yaowu Xu25ff26a2018-02-26 11:20:10 -0800799 const TX_SIZE max_tx_size = tx_size_from_tx_mode(bsize, tx_mode);
Yue Chen1a5ab9f2017-12-14 14:53:51 -0800800
801 if (tx_mode != TX_MODE_SELECT) return max_tx_size;
802
803 int depth = 0;
804 TX_SIZE tx_size = max_tx_size;
805 while (depth < MAX_TX_DEPTH && tx_size != TX_4X4) {
806 if (av1_filter_intra_allowed_txsize(tx_size)) return tx_size;
807 depth++;
808 tx_size = sub_tx_size_map[0][tx_size];
809 }
810 return TX_INVALID;
811}
812#endif
813
Angie Chiang752ccce2017-04-09 13:41:13 -0700814// Converts block_index for given transform size to index of the block in raster
815// order.
816static INLINE int av1_block_index_to_raster_order(TX_SIZE tx_size,
817 int block_idx) {
818 // For transform size 4x8, the possible block_idx values are 0 & 2, because
819 // block_idx values are incremented in steps of size 'tx_width_unit x
820 // tx_height_unit'. But, for this transform size, block_idx = 2 corresponds to
821 // block number 1 in raster order, inside an 8x8 MI block.
822 // For any other transform size, the two indices are equivalent.
823 return (tx_size == TX_4X8 && block_idx == 2) ? 1 : block_idx;
824}
825
826// Inverse of above function.
827// Note: only implemented for transform sizes 4x4, 4x8 and 8x4 right now.
828static INLINE int av1_raster_order_to_block_index(TX_SIZE tx_size,
829 int raster_order) {
830 assert(tx_size == TX_4X4 || tx_size == TX_4X8 || tx_size == TX_8X4);
831 // We ensure that block indices are 0 & 2 if tx size is 4x8 or 8x4.
832 return (tx_size == TX_4X4) ? raster_order : (raster_order > 0) ? 2 : 0;
833}
834
Yaowu Xuc27fc142016-08-22 16:08:15 -0700835static INLINE TX_TYPE get_default_tx_type(PLANE_TYPE plane_type,
Luc Trudeau2eb9b842017-12-13 11:19:16 -0500836 const MACROBLOCKD *xd,
Yaowu Xuc27fc142016-08-22 16:08:15 -0700837 TX_SIZE tx_size) {
838 const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
839
Hui Su7448fc22018-01-11 16:47:45 -0800840 if (is_inter_block(mbmi) || plane_type != PLANE_TYPE_Y ||
Yaowu Xuc27fc142016-08-22 16:08:15 -0700841 xd->lossless[mbmi->segment_id] || tx_size >= TX_32X32)
842 return DCT_DCT;
843
Luc Trudeau9cea9932017-12-13 21:14:13 -0500844 return intra_mode_to_tx_type_context(mbmi, plane_type);
Yaowu Xuc27fc142016-08-22 16:08:15 -0700845}
846
Sarah Parker90024e42017-10-06 16:50:47 -0700847static INLINE BLOCK_SIZE
848get_plane_block_size(BLOCK_SIZE bsize, const struct macroblockd_plane *pd) {
849 return ss_size_lookup[bsize][pd->subsampling_x][pd->subsampling_y];
850}
851
Hui Su7167d952018-02-01 16:33:12 -0800852static INLINE int av1_get_txb_size_index(BLOCK_SIZE bsize, int blk_row,
853 int blk_col) {
Jingning Hand26fde22018-02-21 09:24:37 -0800854 TX_SIZE txs = max_txsize_rect_lookup[bsize];
Hui Su7167d952018-02-01 16:33:12 -0800855 for (int level = 0; level < MAX_VARTX_DEPTH - 1; ++level)
856 txs = sub_tx_size_map[1][txs];
857 const int tx_w = tx_size_wide_unit[txs];
858 const int tx_h = tx_size_high_unit[txs];
859 const int bw_uint = mi_size_wide[bsize];
860 const int stride = bw_uint / tx_w;
861 const int index = (blk_row / tx_h) * stride + (blk_col / tx_w);
862 assert(index < INTER_TX_SIZE_BUF_LEN);
863 return index;
864}
865
Hui Sude1a5db2018-02-22 15:44:49 -0800866#if CONFIG_TXK_SEL
867static INLINE int av1_get_txk_type_index(BLOCK_SIZE bsize, int blk_row,
868 int blk_col) {
869 TX_SIZE txs = max_txsize_rect_lookup[bsize];
870 for (int level = 0; level < MAX_VARTX_DEPTH; ++level)
871 txs = sub_tx_size_map[1][txs];
872 const int tx_w = tx_size_wide_unit[txs];
873 const int tx_h = tx_size_high_unit[txs];
874 const int bw_uint = mi_size_wide[bsize];
875 const int stride = bw_uint / tx_w;
876 const int index = (blk_row / tx_h) * stride + (blk_col / tx_w);
877 assert(index < TXK_TYPE_BUF_LEN);
878 return index;
879}
880#endif // CONFIG_TXK_SEL
881
hui su45b64752017-07-12 16:54:35 -0700882static INLINE TX_TYPE av1_get_tx_type(PLANE_TYPE plane_type,
Jingning Han19b5c8f2017-07-06 15:10:12 -0700883 const MACROBLOCKD *xd, int blk_row,
Sarah Parker7c71cc02018-01-29 12:27:58 -0800884 int blk_col, TX_SIZE tx_size,
885 int reduced_tx_set) {
Yaowu Xuc27fc142016-08-22 16:08:15 -0700886 const MODE_INFO *const mi = xd->mi[0];
887 const MB_MODE_INFO *const mbmi = &mi->mbmi;
Sarah Parker90024e42017-10-06 16:50:47 -0700888 const struct macroblockd_plane *const pd = &xd->plane[plane_type];
889 const BLOCK_SIZE plane_bsize = get_plane_block_size(mbmi->sb_type, pd);
Sarah Parker7c71cc02018-01-29 12:27:58 -0800890 const TxSetType tx_set_type = get_ext_tx_set_type(
891 tx_size, plane_bsize, is_inter_block(mbmi), reduced_tx_set);
Sarah Parker90024e42017-10-06 16:50:47 -0700892
hui su45b64752017-07-12 16:54:35 -0700893#if CONFIG_TXK_SEL
894 TX_TYPE tx_type;
Thomas Daedeaf73d532018-01-30 14:21:32 -0800895 if (xd->lossless[mbmi->segment_id] || txsize_sqr_up_map[tx_size] > TX_32X32) {
hui su45b64752017-07-12 16:54:35 -0700896 tx_type = DCT_DCT;
897 } else {
Angie Chiangbce07f12017-12-01 16:34:31 -0800898 if (plane_type == PLANE_TYPE_Y) {
Hui Sude1a5db2018-02-22 15:44:49 -0800899 const int txk_type_idx =
900 av1_get_txk_type_index(mbmi->sb_type, blk_row, blk_col);
901 tx_type = mbmi->txk_type[txk_type_idx];
Angie Chiangbce07f12017-12-01 16:34:31 -0800902 } else if (is_inter_block(mbmi)) {
903 // scale back to y plane's coordinate
904 blk_row <<= pd->subsampling_y;
905 blk_col <<= pd->subsampling_x;
Hui Sude1a5db2018-02-22 15:44:49 -0800906 const int txk_type_idx =
907 av1_get_txk_type_index(mbmi->sb_type, blk_row, blk_col);
908 tx_type = mbmi->txk_type[txk_type_idx];
Angie Chiangbce07f12017-12-01 16:34:31 -0800909 } else {
910 // In intra mode, uv planes don't share the same prediction mode as y
911 // plane, so the tx_type should not be shared
Luc Trudeau9cea9932017-12-13 21:14:13 -0500912 tx_type = intra_mode_to_tx_type_context(mbmi, PLANE_TYPE_UV);
Angie Chiangbce07f12017-12-01 16:34:31 -0800913 }
hui su45b64752017-07-12 16:54:35 -0700914 }
Angie Chiang840c66e2017-12-11 16:43:38 -0800915 assert(tx_type < TX_TYPES);
Angie Chiang2ac18682017-12-01 18:02:59 -0800916 if (!av1_ext_tx_used[tx_set_type][tx_type]) return DCT_DCT;
hui su45b64752017-07-12 16:54:35 -0700917 return tx_type;
Hui Su02e84152018-01-29 10:23:26 -0800918#else
919 (void)blk_row;
920 (void)blk_col;
hui su45b64752017-07-12 16:54:35 -0700921#endif // CONFIG_TXK_SEL
922
Jingning Handa6df482018-01-07 07:39:31 -0800923 if (is_inter_block(mbmi) && !av1_ext_tx_used[tx_set_type][mbmi->tx_type])
924 return DCT_DCT;
925
Yaowu Xuc27fc142016-08-22 16:08:15 -0700926 if (xd->lossless[mbmi->segment_id] || txsize_sqr_map[tx_size] > TX_32X32 ||
927 (txsize_sqr_map[tx_size] >= TX_32X32 && !is_inter_block(mbmi)))
Yaowu Xuc27fc142016-08-22 16:08:15 -0700928 return DCT_DCT;
Debargha Mukherjee6ea917e2017-10-19 09:31:29 -0700929 if (plane_type == PLANE_TYPE_Y) {
Urvang Joshi978152a2018-01-09 12:59:25 -0800930 return mbmi->tx_type;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700931 }
932
Debargha Mukherjee6ea917e2017-10-19 09:31:29 -0700933 if (is_inter_block(mbmi)) {
934 // UV Inter only
935 return (mbmi->tx_type == IDTX && txsize_sqr_map[tx_size] >= TX_32X32)
936 ? DCT_DCT
937 : mbmi->tx_type;
938 }
939
Sarah Parker90024e42017-10-06 16:50:47 -0700940 // UV Intra only
Luc Trudeau9cea9932017-12-13 21:14:13 -0500941 const TX_TYPE intra_type = intra_mode_to_tx_type_context(mbmi, PLANE_TYPE_UV);
Sarah Parker90024e42017-10-06 16:50:47 -0700942 if (!av1_ext_tx_used[tx_set_type][intra_type]) return DCT_DCT;
943 return intra_type;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700944}
945
Imdad Sardharwallaaf8e2642018-01-19 11:46:34 +0000946void av1_setup_block_planes(MACROBLOCKD *xd, int ss_x, int ss_y,
947 const int num_planes);
Yaowu Xuc27fc142016-08-22 16:08:15 -0700948
Debargha Mukherjee0fa057f2017-12-06 17:06:29 -0800949static INLINE int bsize_to_max_depth(BLOCK_SIZE bsize, int is_inter) {
Yaowu Xu25ff26a2018-02-26 11:20:10 -0800950 TX_SIZE tx_size = get_max_rect_tx_size(bsize);
Debargha Mukherjee0fa057f2017-12-06 17:06:29 -0800951 int depth = 0;
952 while (depth < MAX_TX_DEPTH && tx_size != TX_4X4) {
953 depth++;
Debargha Mukherjeee4e18fc2017-12-06 23:43:24 -0800954 tx_size = sub_tx_size_map[is_inter][tx_size];
Debargha Mukherjee0fa057f2017-12-06 17:06:29 -0800955 }
956 return depth;
Jingning Han4e1737a2016-10-25 16:05:02 -0700957}
958
Debargha Mukherjeee4e18fc2017-12-06 23:43:24 -0800959static INLINE int bsize_to_tx_size_cat(BLOCK_SIZE bsize, int is_inter) {
Yaowu Xu25ff26a2018-02-26 11:20:10 -0800960 TX_SIZE tx_size = get_max_rect_tx_size(bsize);
Debargha Mukherjeee4e18fc2017-12-06 23:43:24 -0800961 assert(tx_size != TX_4X4);
962 int depth = 0;
963 while (tx_size != TX_4X4) {
964 depth++;
965 tx_size = sub_tx_size_map[is_inter][tx_size];
966 assert(depth < 10);
967 }
968 assert(depth <= MAX_TX_CATS);
969 return depth - 1;
970}
971
Debargha Mukherjee0fa057f2017-12-06 17:06:29 -0800972static INLINE TX_SIZE depth_to_tx_size(int depth, BLOCK_SIZE bsize,
973 int is_inter) {
Yaowu Xu25ff26a2018-02-26 11:20:10 -0800974 TX_SIZE max_tx_size = get_max_rect_tx_size(bsize);
Debargha Mukherjee0fa057f2017-12-06 17:06:29 -0800975 TX_SIZE tx_size = max_tx_size;
Debargha Mukherjeee4e18fc2017-12-06 23:43:24 -0800976 for (int d = 0; d < depth; ++d) tx_size = sub_tx_size_map[is_inter][tx_size];
Debargha Mukherjee0fa057f2017-12-06 17:06:29 -0800977 return tx_size;
Jingning Han4e1737a2016-10-25 16:05:02 -0700978}
979
Yaowu Xu25ff26a2018-02-26 11:20:10 -0800980static INLINE TX_SIZE av1_get_max_uv_txsize(BLOCK_SIZE bsize, int ss_x,
981 int ss_y) {
Debargha Mukherjee80592c72017-12-16 08:23:34 -0800982 const BLOCK_SIZE plane_bsize = ss_size_lookup[bsize][ss_x][ss_y];
983 assert(plane_bsize < BLOCK_SIZES_ALL);
Jingning Hand26fde22018-02-21 09:24:37 -0800984 TX_SIZE uv_tx = max_txsize_rect_lookup[plane_bsize];
Debargha Mukherjee80592c72017-12-16 08:23:34 -0800985 switch (uv_tx) {
986 case TX_64X64:
987 case TX_64X32:
988 case TX_32X64: return TX_32X32;
989 case TX_64X16: return TX_32X16;
990 case TX_16X64: return TX_16X32;
991 default: break;
Luc Trudeaud1941f32017-11-22 14:17:21 -0500992 }
Debargha Mukherjee80592c72017-12-16 08:23:34 -0800993 return uv_tx;
994}
995
996static INLINE TX_SIZE av1_get_uv_tx_size(const MB_MODE_INFO *mbmi, int ss_x,
997 int ss_y) {
Yaowu Xu25ff26a2018-02-26 11:20:10 -0800998 return av1_get_max_uv_txsize(mbmi->sb_type, ss_x, ss_y);
Yaowu Xuc27fc142016-08-22 16:08:15 -0700999}
1000
hui su0c6244b2017-07-12 17:11:43 -07001001static INLINE TX_SIZE av1_get_tx_size(int plane, const MACROBLOCKD *xd) {
Angie Chiang80b82262017-02-24 11:39:47 -08001002 const MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
Debargha Mukherjee80592c72017-12-16 08:23:34 -08001003 if (xd->lossless[mbmi->segment_id]) return TX_4X4;
hui su0c6244b2017-07-12 17:11:43 -07001004 if (plane == 0) return mbmi->tx_size;
Angie Chiang80b82262017-02-24 11:39:47 -08001005 const MACROBLOCKD_PLANE *pd = &xd->plane[plane];
Debargha Mukherjee80592c72017-12-16 08:23:34 -08001006 return av1_get_uv_tx_size(mbmi, pd->subsampling_x, pd->subsampling_y);
Angie Chiang80b82262017-02-24 11:39:47 -08001007}
1008
Timothy B. Terriberrya2d5cde2017-05-10 18:33:50 -07001009void av1_reset_skip_context(MACROBLOCKD *xd, int mi_row, int mi_col,
Imdad Sardharwallaaf8e2642018-01-19 11:46:34 +00001010 BLOCK_SIZE bsize, const int num_planes);
Yaowu Xuc27fc142016-08-22 16:08:15 -07001011
Rupert Swarbrick76405202017-11-07 16:35:55 +00001012#if CONFIG_LOOP_RESTORATION
Imdad Sardharwallaaf8e2642018-01-19 11:46:34 +00001013void av1_reset_loop_restoration(MACROBLOCKD *xd, const int num_planes);
Rupert Swarbrick76405202017-11-07 16:35:55 +00001014#endif // CONFIG_LOOP_RESTORATION
1015
Yaowu Xuc27fc142016-08-22 16:08:15 -07001016typedef void (*foreach_transformed_block_visitor)(int plane, int block,
1017 int blk_row, int blk_col,
1018 BLOCK_SIZE plane_bsize,
1019 TX_SIZE tx_size, void *arg);
1020
Yaowu Xuf883b422016-08-30 14:01:10 -07001021void av1_foreach_transformed_block_in_plane(
Yaowu Xuc27fc142016-08-22 16:08:15 -07001022 const MACROBLOCKD *const xd, BLOCK_SIZE bsize, int plane,
1023 foreach_transformed_block_visitor visit, void *arg);
1024
Angie Chiang0397eda2017-03-15 16:57:14 -07001025void av1_foreach_transformed_block(const MACROBLOCKD *const xd,
Jingning Han94652b82017-04-04 09:45:02 -07001026 BLOCK_SIZE bsize, int mi_row, int mi_col,
Angie Chiang0397eda2017-03-15 16:57:14 -07001027 foreach_transformed_block_visitor visit,
Imdad Sardharwallaaf8e2642018-01-19 11:46:34 +00001028 void *arg, const int num_planes);
Angie Chiang0397eda2017-03-15 16:57:14 -07001029
Yaowu Xuf883b422016-08-30 14:01:10 -07001030void av1_set_contexts(const MACROBLOCKD *xd, struct macroblockd_plane *pd,
Jingning Haneee43152016-12-05 09:58:45 -08001031 int plane, TX_SIZE tx_size, int has_eob, int aoff,
1032 int loff);
Yaowu Xuc27fc142016-08-22 16:08:15 -07001033
Yue Chenb8aa3992017-12-13 15:27:52 -08001034#define MAX_INTERINTRA_SB_SQUARE 32 * 32
Yaowu Xuc27fc142016-08-22 16:08:15 -07001035static INLINE int is_interintra_allowed_bsize(const BLOCK_SIZE bsize) {
Yue Chen9ddd4092017-10-30 16:13:39 -07001036 return (bsize >= BLOCK_8X8) && (bsize <= BLOCK_32X32);
Yaowu Xuc27fc142016-08-22 16:08:15 -07001037}
1038
1039static INLINE int is_interintra_allowed_mode(const PREDICTION_MODE mode) {
1040 return (mode >= NEARESTMV) && (mode <= NEWMV);
1041}
1042
1043static INLINE int is_interintra_allowed_ref(const MV_REFERENCE_FRAME rf[2]) {
1044 return (rf[0] > INTRA_FRAME) && (rf[1] <= INTRA_FRAME);
1045}
1046
1047static INLINE int is_interintra_allowed(const MB_MODE_INFO *mbmi) {
1048 return is_interintra_allowed_bsize(mbmi->sb_type) &&
1049 is_interintra_allowed_mode(mbmi->mode) &&
1050 is_interintra_allowed_ref(mbmi->ref_frame);
1051}
1052
Yaowu Xu4ff59b52017-04-24 12:41:56 -07001053static INLINE int is_interintra_allowed_bsize_group(int group) {
Yaowu Xuc27fc142016-08-22 16:08:15 -07001054 int i;
Rupert Swarbrick93c39e92017-07-12 11:11:02 +01001055 for (i = 0; i < BLOCK_SIZES_ALL; i++) {
Urvang Joshicb586f32016-09-20 11:36:33 -07001056 if (size_group_lookup[i] == group &&
1057 is_interintra_allowed_bsize((BLOCK_SIZE)i)) {
Yaowu Xuc27fc142016-08-22 16:08:15 -07001058 return 1;
Urvang Joshicb586f32016-09-20 11:36:33 -07001059 }
Yaowu Xuc27fc142016-08-22 16:08:15 -07001060 }
1061 return 0;
1062}
1063
1064static INLINE int is_interintra_pred(const MB_MODE_INFO *mbmi) {
Yue Chenb8aa3992017-12-13 15:27:52 -08001065 return mbmi->ref_frame[0] > INTRA_FRAME &&
1066 mbmi->ref_frame[1] == INTRA_FRAME && is_interintra_allowed(mbmi);
Yaowu Xuc27fc142016-08-22 16:08:15 -07001067}
Yaowu Xuc27fc142016-08-22 16:08:15 -07001068
Debargha Mukherjee891a8772017-11-22 10:09:37 -08001069static INLINE int get_vartx_max_txsize(const MACROBLOCKD *xd, BLOCK_SIZE bsize,
1070 int subsampled) {
Yaowu Xu25ff26a2018-02-26 11:20:10 -08001071 TX_SIZE max_txsize = xd->lossless[xd->mi[0]->mbmi.segment_id]
1072 ? TX_4X4
1073 : get_max_rect_tx_size(bsize);
Rupert Swarbrick4e7b7d62017-09-28 17:30:44 +01001074
Yaowu Xud3d41592018-02-14 13:26:52 -08001075#if CONFIG_EXT_PARTITION
Rupert Swarbrick4e7b7d62017-09-28 17:30:44 +01001076 // The decoder is designed so that it can process 64x64 luma pixels at a
1077 // time. If this is a chroma plane with subsampling and bsize corresponds to
1078 // a subsampled BLOCK_128X128 then the lookup above will give TX_64X64. That
1079 // mustn't be used for the subsampled plane (because it would be bigger than
1080 // a 64x64 luma block) so we round down to TX_32X32.
Debargha Mukherjee1fa24462017-10-19 14:33:39 -07001081 if (subsampled && txsize_sqr_up_map[max_txsize] == TX_64X64) {
Debargha Mukherjee1a8664e2017-12-20 15:54:49 -08001082 if (max_txsize == TX_16X64)
1083 max_txsize = TX_16X32;
1084 else if (max_txsize == TX_64X16)
1085 max_txsize = TX_32X16;
1086 else
1087 max_txsize = TX_32X32;
Debargha Mukherjee1fa24462017-10-19 14:33:39 -07001088 }
Rupert Swarbrick4e7b7d62017-09-28 17:30:44 +01001089#else
1090 (void)subsampled;
1091#endif
1092
1093 return max_txsize;
Sarah Parker106b3cb2017-04-21 12:13:37 -07001094}
Sarah Parker106b3cb2017-04-21 12:13:37 -07001095
Yue Chencb60b182016-10-13 15:18:22 -07001096static INLINE int is_motion_variation_allowed_bsize(BLOCK_SIZE bsize) {
Debargha Mukherjeec17a4432017-08-28 22:22:45 -07001097 return AOMMIN(block_size_wide[bsize], block_size_high[bsize]) >= 8;
Yue Chencb60b182016-10-13 15:18:22 -07001098}
1099
Yue Chen8636da62017-04-03 01:23:44 -07001100static INLINE int is_motion_variation_allowed_compound(
1101 const MB_MODE_INFO *mbmi) {
1102 if (!has_second_ref(mbmi))
1103 return 1;
1104 else
1105 return 0;
1106}
1107
Yue Chen1bd42be2017-03-15 18:07:04 -07001108// input: log2 of length, 0(4), 1(8), ...
1109static const int max_neighbor_obmc[6] = { 0, 1, 2, 3, 4, 4 };
1110
Yue Chen5329a2b2017-02-28 17:33:00 +08001111static INLINE int check_num_overlappable_neighbors(const MB_MODE_INFO *mbmi) {
Yue Chen1bd42be2017-03-15 18:07:04 -07001112 return !(mbmi->overlappable_neighbors[0] == 0 &&
1113 mbmi->overlappable_neighbors[1] == 0);
Yue Chen5329a2b2017-02-28 17:33:00 +08001114}
Yue Chen5329a2b2017-02-28 17:33:00 +08001115
Sebastien Alaiwan48795802017-10-30 12:07:13 +01001116static INLINE MOTION_MODE
Luc Trudeau2eb9b842017-12-13 11:19:16 -05001117motion_mode_allowed(const WarpedMotionParams *gm_params, const MACROBLOCKD *xd,
1118 const MODE_INFO *mi) {
Sarah Parker19234cc2017-03-10 16:43:25 -08001119 const MB_MODE_INFO *mbmi = &mi->mbmi;
RogerZhou3b635242017-09-19 10:06:46 -07001120#if CONFIG_AMVR
RogerZhou10a03802017-10-26 11:49:48 -07001121 if (xd->cur_frame_force_integer_mv == 0) {
RogerZhou3b635242017-09-19 10:06:46 -07001122#endif
RogerZhou3b635242017-09-19 10:06:46 -07001123 const TransformationType gm_type = gm_params[mbmi->ref_frame[0]].wmtype;
Luc Trudeau2eb9b842017-12-13 11:19:16 -05001124 if (is_global_mv_block(mi, gm_type)) return SIMPLE_TRANSLATION;
RogerZhou3b635242017-09-19 10:06:46 -07001125#if CONFIG_AMVR
1126 }
1127#endif
Yue Chen69f18e12016-09-08 14:48:15 -07001128 if (is_motion_variation_allowed_bsize(mbmi->sb_type) &&
Yue Chen8636da62017-04-03 01:23:44 -07001129 is_inter_mode(mbmi->mode) && mbmi->ref_frame[1] != INTRA_FRAME &&
1130 is_motion_variation_allowed_compound(mbmi)) {
Yue Chen5329a2b2017-02-28 17:33:00 +08001131 if (!check_num_overlappable_neighbors(mbmi)) return SIMPLE_TRANSLATION;
Zoe Liu70539b12017-12-02 19:03:36 -08001132 assert(!has_second_ref(mbmi));
1133 if (mbmi->num_proj_ref[0] >= 1 &&
Wei-Ting Lin20885282017-08-28 17:18:18 -07001134 !av1_is_scaled(&(xd->block_refs[0]->sf))) {
RogerZhou3b635242017-09-19 10:06:46 -07001135#if CONFIG_AMVR
RogerZhou10a03802017-10-26 11:49:48 -07001136 if (xd->cur_frame_force_integer_mv) {
RogerZhou3b635242017-09-19 10:06:46 -07001137 return OBMC_CAUSAL;
1138 }
1139#endif
Yue Chen69f18e12016-09-08 14:48:15 -07001140 return WARPED_CAUSAL;
Hui Su12231bd2017-09-07 18:01:15 -07001141 }
Yue Chen80daf0c2017-11-02 17:14:18 -07001142 return OBMC_CAUSAL;
Yue Chen69f18e12016-09-08 14:48:15 -07001143 } else {
1144 return SIMPLE_TRANSLATION;
1145 }
Yaowu Xuc27fc142016-08-22 16:08:15 -07001146}
1147
Luc Trudeau2eb9b842017-12-13 11:19:16 -05001148static INLINE void assert_motion_mode_valid(MOTION_MODE mode,
Sarah Parker19234cc2017-03-10 16:43:25 -08001149 const WarpedMotionParams *gm_params,
Yue Chen52c51732017-07-11 15:08:30 -07001150 const MACROBLOCKD *xd,
Sarah Parker19234cc2017-03-10 16:43:25 -08001151 const MODE_INFO *mi) {
Sebastien Alaiwan48795802017-10-30 12:07:13 +01001152 const MOTION_MODE last_motion_mode_allowed =
Luc Trudeau2eb9b842017-12-13 11:19:16 -05001153 motion_mode_allowed(gm_params, xd, mi);
Wei-Ting Lin20885282017-08-28 17:18:18 -07001154
Sarah Parker19234cc2017-03-10 16:43:25 -08001155 // Check that the input mode is not illegal
1156 if (last_motion_mode_allowed < mode)
1157 assert(0 && "Illegal motion mode selected");
1158}
1159
Yaowu Xuc27fc142016-08-22 16:08:15 -07001160static INLINE int is_neighbor_overlappable(const MB_MODE_INFO *mbmi) {
1161 return (is_inter_block(mbmi));
1162}
Yaowu Xuc27fc142016-08-22 16:08:15 -07001163
Hui Sue87fb232017-10-05 15:00:15 -07001164static INLINE int av1_allow_palette(int allow_screen_content_tools,
1165 BLOCK_SIZE sb_type) {
Hui Su8b618f62017-12-20 12:03:35 -08001166 return allow_screen_content_tools && block_size_wide[sb_type] <= 64 &&
Hui Su0f9dafb2018-02-09 17:20:29 -08001167 block_size_high[sb_type] <= 64 && sb_type >= BLOCK_8X8;
Hui Sue87fb232017-10-05 15:00:15 -07001168}
1169
Urvang Joshi56ba91b2017-01-10 13:22:09 -08001170// Returns sub-sampled dimensions of the given block.
1171// The output values for 'rows_within_bounds' and 'cols_within_bounds' will
RogerZhou3b635242017-09-19 10:06:46 -07001172// differ from 'height' and 'width' when part of the block is outside the
1173// right
Urvang Joshi56ba91b2017-01-10 13:22:09 -08001174// and/or bottom image boundary.
1175static INLINE void av1_get_block_dimensions(BLOCK_SIZE bsize, int plane,
1176 const MACROBLOCKD *xd, int *width,
1177 int *height,
1178 int *rows_within_bounds,
1179 int *cols_within_bounds) {
1180 const int block_height = block_size_high[bsize];
1181 const int block_width = block_size_wide[bsize];
1182 const int block_rows = (xd->mb_to_bottom_edge >= 0)
1183 ? block_height
1184 : (xd->mb_to_bottom_edge >> 3) + block_height;
1185 const int block_cols = (xd->mb_to_right_edge >= 0)
1186 ? block_width
1187 : (xd->mb_to_right_edge >> 3) + block_width;
1188 const struct macroblockd_plane *const pd = &xd->plane[plane];
1189 assert(IMPLIES(plane == PLANE_TYPE_Y, pd->subsampling_x == 0));
1190 assert(IMPLIES(plane == PLANE_TYPE_Y, pd->subsampling_y == 0));
1191 assert(block_width >= block_cols);
1192 assert(block_height >= block_rows);
Hui Su8b618f62017-12-20 12:03:35 -08001193 const int plane_block_width = block_width >> pd->subsampling_x;
1194 const int plane_block_height = block_height >> pd->subsampling_y;
1195 // Special handling for chroma sub8x8.
1196 const int is_chroma_sub8_x = plane > 0 && plane_block_width < 4;
1197 const int is_chroma_sub8_y = plane > 0 && plane_block_height < 4;
1198 if (width) *width = plane_block_width + 2 * is_chroma_sub8_x;
1199 if (height) *height = plane_block_height + 2 * is_chroma_sub8_y;
1200 if (rows_within_bounds) {
1201 *rows_within_bounds =
1202 (block_rows >> pd->subsampling_y) + 2 * is_chroma_sub8_y;
1203 }
1204 if (cols_within_bounds) {
1205 *cols_within_bounds =
1206 (block_cols >> pd->subsampling_x) + 2 * is_chroma_sub8_x;
1207 }
Urvang Joshi56ba91b2017-01-10 13:22:09 -08001208}
1209
Sarah Parkerefd8af22017-08-25 16:36:06 -07001210/* clang-format off */
1211typedef aom_cdf_prob (*MapCdf)[PALETTE_COLOR_INDEX_CONTEXTS]
1212 [CDF_SIZE(PALETTE_COLORS)];
1213typedef const int (*ColorCost)[PALETTE_SIZES][PALETTE_COLOR_INDEX_CONTEXTS]
1214 [PALETTE_COLORS];
1215/* clang-format on */
1216
1217typedef struct {
1218 int rows;
1219 int cols;
1220 int n_colors;
1221 int plane_width;
1222 int plane_height;
1223 uint8_t *color_map;
1224 MapCdf map_cdf;
1225 ColorCost color_cost;
1226} Av1ColorMapParam;
1227
Yue Chen19e7aa82016-11-30 14:05:39 -08001228static INLINE int is_nontrans_global_motion(const MACROBLOCKD *xd) {
1229 const MODE_INFO *mi = xd->mi[0];
1230 const MB_MODE_INFO *const mbmi = &mi->mbmi;
1231 int ref;
Yue Chen19e7aa82016-11-30 14:05:39 -08001232
Sarah Parker2b9ec2e2017-10-30 17:34:08 -07001233 // First check if all modes are GLOBALMV
1234 if (mbmi->mode != GLOBALMV && mbmi->mode != GLOBAL_GLOBALMV) return 0;
Jingning Han3ca0e672017-04-14 19:48:05 -07001235
1236#if !GLOBAL_SUB8X8_USED
Jingning Han19d05212017-12-22 12:09:49 -08001237 if (AOMMIN(mi_size_wide[mbmi->sb_type], mi_size_high[mbmi->sb_type]) < 8)
1238 return 0;
Jingning Han3ca0e672017-04-14 19:48:05 -07001239#endif
1240
Yue Chen19e7aa82016-11-30 14:05:39 -08001241 // Now check if all global motion is non translational
1242 for (ref = 0; ref < 1 + has_second_ref(mbmi); ++ref) {
1243 if (xd->global_motion[mbmi->ref_frame[ref]].wmtype <= TRANSLATION) return 0;
1244 }
1245 return 1;
1246}
Yue Chen19e7aa82016-11-30 14:05:39 -08001247
Yaowu Xu4ff59b52017-04-24 12:41:56 -07001248static INLINE PLANE_TYPE get_plane_type(int plane) {
Luc Trudeau005feb62017-02-22 13:34:01 -05001249 return (plane == 0) ? PLANE_TYPE_Y : PLANE_TYPE_UV;
1250}
1251
Angie Chiang155bf9a2017-08-06 19:52:57 -07001252static INLINE void transpose_uint8(uint8_t *dst, int dst_stride,
1253 const uint8_t *src, int src_stride, int w,
1254 int h) {
1255 int r, c;
1256 for (r = 0; r < h; ++r)
1257 for (c = 0; c < w; ++c) dst[c * dst_stride + r] = src[r * src_stride + c];
1258}
1259
1260static INLINE void transpose_uint16(uint16_t *dst, int dst_stride,
1261 const uint16_t *src, int src_stride, int w,
1262 int h) {
1263 int r, c;
1264 for (r = 0; r < h; ++r)
1265 for (c = 0; c < w; ++c) dst[c * dst_stride + r] = src[r * src_stride + c];
1266}
1267
1268static INLINE void transpose_int16(int16_t *dst, int dst_stride,
1269 const int16_t *src, int src_stride, int w,
1270 int h) {
1271 int r, c;
1272 for (r = 0; r < h; ++r)
1273 for (c = 0; c < w; ++c) dst[c * dst_stride + r] = src[r * src_stride + c];
1274}
1275
1276static INLINE void transpose_int32(int32_t *dst, int dst_stride,
1277 const int32_t *src, int src_stride, int w,
1278 int h) {
1279 int r, c;
1280 for (r = 0; r < h; ++r)
1281 for (c = 0; c < w; ++c) dst[c * dst_stride + r] = src[r * src_stride + c];
1282}
1283
Urvang Joshi80893152017-10-27 11:51:14 -07001284static INLINE int av1_get_max_eob(TX_SIZE tx_size) {
Urvang Joshi030cea92017-12-05 15:27:09 -08001285 if (tx_size == TX_64X64 || tx_size == TX_64X32 || tx_size == TX_32X64) {
1286 return 1024;
1287 }
1288 if (tx_size == TX_16X64 || tx_size == TX_64X16) {
1289 return 512;
1290 }
Urvang Joshi030cea92017-12-05 15:27:09 -08001291 return tx_size_2d[tx_size];
Urvang Joshi80893152017-10-27 11:51:14 -07001292}
1293
Debargha Mukherjee695abd72017-12-16 10:15:45 -08001294static INLINE TX_SIZE av1_get_adjusted_tx_size(TX_SIZE tx_size) {
Debargha Mukherjee695abd72017-12-16 10:15:45 -08001295 if (tx_size == TX_64X64 || tx_size == TX_64X32 || tx_size == TX_32X64) {
1296 return TX_32X32;
1297 }
1298 if (tx_size == TX_16X64) {
1299 return TX_16X32;
1300 }
1301 if (tx_size == TX_64X16) {
1302 return TX_32X16;
1303 }
Debargha Mukherjee695abd72017-12-16 10:15:45 -08001304 return tx_size;
1305}
1306
Yaowu Xuc27fc142016-08-22 16:08:15 -07001307#ifdef __cplusplus
1308} // extern "C"
1309#endif
1310
Yaowu Xuf883b422016-08-30 14:01:10 -07001311#endif // AV1_COMMON_BLOCKD_H_