blob: 31b86abe643390d379c9d23b6950579fe302765e [file] [log] [blame]
Vishesha45092c2021-01-25 00:28:11 +05301/*
2 * Copyright (c) 2021, Alliance for Open Media. All rights reserved
3 *
4 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10 */
11
12#include "av1/encoder/txb_rdopt.h"
Vishesh14be3cd2021-01-27 15:21:15 +053013#include "av1/encoder/txb_rdopt_utils.h"
Vishesha45092c2021-01-25 00:28:11 +053014
15#include "av1/common/idct.h"
Vishesha45092c2021-01-25 00:28:11 +053016
17static INLINE void update_coeff_general(
18 int *accu_rate, int64_t *accu_dist, int si, int eob, TX_SIZE tx_size,
19 TX_CLASS tx_class, int bwl, int height, int64_t rdmult, int shift,
20 int dc_sign_ctx, const int16_t *dequant, const int16_t *scan,
21 const LV_MAP_COEFF_COST *txb_costs, const tran_low_t *tcoeff,
22 tran_low_t *qcoeff, tran_low_t *dqcoeff, uint8_t *levels,
23 const qm_val_t *iqmatrix) {
24 const int dqv = get_dqv(dequant, scan[si], iqmatrix);
25 const int ci = scan[si];
26 const tran_low_t qc = qcoeff[ci];
27 const int is_last = si == (eob - 1);
28 const int coeff_ctx = get_lower_levels_ctx_general(
29 is_last, si, bwl, height, levels, ci, tx_size, tx_class);
30 if (qc == 0) {
31 *accu_rate += txb_costs->base_cost[coeff_ctx][0];
32 } else {
33 const int sign = (qc < 0) ? 1 : 0;
34 const tran_low_t abs_qc = abs(qc);
35 const tran_low_t tqc = tcoeff[ci];
36 const tran_low_t dqc = dqcoeff[ci];
37 const int64_t dist = get_coeff_dist(tqc, dqc, shift);
38 const int64_t dist0 = get_coeff_dist(tqc, 0, shift);
39 const int rate =
40 get_coeff_cost_general(is_last, ci, abs_qc, sign, coeff_ctx,
41 dc_sign_ctx, txb_costs, bwl, tx_class, levels);
42 const int64_t rd = RDCOST(rdmult, rate, dist);
43
44 tran_low_t qc_low, dqc_low;
45 tran_low_t abs_qc_low;
46 int64_t dist_low, rd_low;
47 int rate_low;
48 if (abs_qc == 1) {
49 abs_qc_low = qc_low = dqc_low = 0;
50 dist_low = dist0;
51 rate_low = txb_costs->base_cost[coeff_ctx][0];
52 } else {
53 get_qc_dqc_low(abs_qc, sign, dqv, shift, &qc_low, &dqc_low);
54 abs_qc_low = abs_qc - 1;
55 dist_low = get_coeff_dist(tqc, dqc_low, shift);
56 rate_low =
57 get_coeff_cost_general(is_last, ci, abs_qc_low, sign, coeff_ctx,
58 dc_sign_ctx, txb_costs, bwl, tx_class, levels);
59 }
60
61 rd_low = RDCOST(rdmult, rate_low, dist_low);
62 if (rd_low < rd) {
63 qcoeff[ci] = qc_low;
64 dqcoeff[ci] = dqc_low;
65 levels[get_padded_idx(ci, bwl)] = AOMMIN(abs_qc_low, INT8_MAX);
66 *accu_rate += rate_low;
67 *accu_dist += dist_low - dist0;
68 } else {
69 *accu_rate += rate;
70 *accu_dist += dist - dist0;
71 }
72 }
73}
74
75static AOM_FORCE_INLINE void update_coeff_simple(
76 int *accu_rate, int si, int eob, TX_SIZE tx_size, TX_CLASS tx_class,
77 int bwl, int64_t rdmult, int shift, const int16_t *dequant,
78 const int16_t *scan, const LV_MAP_COEFF_COST *txb_costs,
79 const tran_low_t *tcoeff, tran_low_t *qcoeff, tran_low_t *dqcoeff,
80 uint8_t *levels, const qm_val_t *iqmatrix) {
81 const int dqv = get_dqv(dequant, scan[si], iqmatrix);
82 (void)eob;
83 // this simple version assumes the coeff's scan_idx is not DC (scan_idx != 0)
84 // and not the last (scan_idx != eob - 1)
85 assert(si != eob - 1);
86 assert(si > 0);
87 const int ci = scan[si];
88 const tran_low_t qc = qcoeff[ci];
89 const int coeff_ctx =
90 get_lower_levels_ctx(levels, ci, bwl, tx_size, tx_class);
91 if (qc == 0) {
92 *accu_rate += txb_costs->base_cost[coeff_ctx][0];
93 } else {
94 const tran_low_t abs_qc = abs(qc);
95 const tran_low_t abs_tqc = abs(tcoeff[ci]);
96 const tran_low_t abs_dqc = abs(dqcoeff[ci]);
97 int rate_low = 0;
98 const int rate = get_two_coeff_cost_simple(
99 ci, abs_qc, coeff_ctx, txb_costs, bwl, tx_class, levels, &rate_low);
100 if (abs_dqc < abs_tqc) {
101 *accu_rate += rate;
102 return;
103 }
104
105 const int64_t dist = get_coeff_dist(abs_tqc, abs_dqc, shift);
106 const int64_t rd = RDCOST(rdmult, rate, dist);
107
108 const tran_low_t abs_qc_low = abs_qc - 1;
109 const tran_low_t abs_dqc_low = (abs_qc_low * dqv) >> shift;
110 const int64_t dist_low = get_coeff_dist(abs_tqc, abs_dqc_low, shift);
111 const int64_t rd_low = RDCOST(rdmult, rate_low, dist_low);
112
113 if (rd_low < rd) {
114 const int sign = (qc < 0) ? 1 : 0;
115 qcoeff[ci] = (-sign ^ abs_qc_low) + sign;
116 dqcoeff[ci] = (-sign ^ abs_dqc_low) + sign;
117 levels[get_padded_idx(ci, bwl)] = AOMMIN(abs_qc_low, INT8_MAX);
118 *accu_rate += rate_low;
119 } else {
120 *accu_rate += rate;
121 }
122 }
123}
124
125static AOM_FORCE_INLINE void update_coeff_eob(
126 int *accu_rate, int64_t *accu_dist, int *eob, int *nz_num, int *nz_ci,
127 int si, TX_SIZE tx_size, TX_CLASS tx_class, int bwl, int height,
128 int dc_sign_ctx, int64_t rdmult, int shift, const int16_t *dequant,
129 const int16_t *scan, const LV_MAP_EOB_COST *txb_eob_costs,
130 const LV_MAP_COEFF_COST *txb_costs, const tran_low_t *tcoeff,
131 tran_low_t *qcoeff, tran_low_t *dqcoeff, uint8_t *levels, int sharpness,
132 const qm_val_t *iqmatrix) {
133 const int dqv = get_dqv(dequant, scan[si], iqmatrix);
134 assert(si != *eob - 1);
135 const int ci = scan[si];
136 const tran_low_t qc = qcoeff[ci];
137 const int coeff_ctx =
138 get_lower_levels_ctx(levels, ci, bwl, tx_size, tx_class);
139 if (qc == 0) {
140 *accu_rate += txb_costs->base_cost[coeff_ctx][0];
141 } else {
142 int lower_level = 0;
143 const tran_low_t abs_qc = abs(qc);
144 const tran_low_t tqc = tcoeff[ci];
145 const tran_low_t dqc = dqcoeff[ci];
146 const int sign = (qc < 0) ? 1 : 0;
147 const int64_t dist0 = get_coeff_dist(tqc, 0, shift);
148 int64_t dist = get_coeff_dist(tqc, dqc, shift) - dist0;
149 int rate =
150 get_coeff_cost_general(0, ci, abs_qc, sign, coeff_ctx, dc_sign_ctx,
151 txb_costs, bwl, tx_class, levels);
152 int64_t rd = RDCOST(rdmult, *accu_rate + rate, *accu_dist + dist);
153
154 tran_low_t qc_low, dqc_low;
155 tran_low_t abs_qc_low;
156 int64_t dist_low, rd_low;
157 int rate_low;
158 if (abs_qc == 1) {
159 abs_qc_low = 0;
160 dqc_low = qc_low = 0;
161 dist_low = 0;
162 rate_low = txb_costs->base_cost[coeff_ctx][0];
163 rd_low = RDCOST(rdmult, *accu_rate + rate_low, *accu_dist);
164 } else {
165 get_qc_dqc_low(abs_qc, sign, dqv, shift, &qc_low, &dqc_low);
166 abs_qc_low = abs_qc - 1;
167 dist_low = get_coeff_dist(tqc, dqc_low, shift) - dist0;
168 rate_low =
169 get_coeff_cost_general(0, ci, abs_qc_low, sign, coeff_ctx,
170 dc_sign_ctx, txb_costs, bwl, tx_class, levels);
171 rd_low = RDCOST(rdmult, *accu_rate + rate_low, *accu_dist + dist_low);
172 }
173
174 int lower_level_new_eob = 0;
175 const int new_eob = si + 1;
176 const int coeff_ctx_new_eob = get_lower_levels_ctx_eob(bwl, height, si);
177 const int new_eob_cost =
178 get_eob_cost(new_eob, txb_eob_costs, txb_costs, tx_class);
179 int rate_coeff_eob =
180 new_eob_cost + get_coeff_cost_eob(ci, abs_qc, sign, coeff_ctx_new_eob,
181 dc_sign_ctx, txb_costs, bwl,
182 tx_class);
183 int64_t dist_new_eob = dist;
184 int64_t rd_new_eob = RDCOST(rdmult, rate_coeff_eob, dist_new_eob);
185
186 if (abs_qc_low > 0) {
187 const int rate_coeff_eob_low =
188 new_eob_cost + get_coeff_cost_eob(ci, abs_qc_low, sign,
189 coeff_ctx_new_eob, dc_sign_ctx,
190 txb_costs, bwl, tx_class);
191 const int64_t dist_new_eob_low = dist_low;
192 const int64_t rd_new_eob_low =
193 RDCOST(rdmult, rate_coeff_eob_low, dist_new_eob_low);
194 if (rd_new_eob_low < rd_new_eob) {
195 lower_level_new_eob = 1;
196 rd_new_eob = rd_new_eob_low;
197 rate_coeff_eob = rate_coeff_eob_low;
198 dist_new_eob = dist_new_eob_low;
199 }
200 }
201
202 if (rd_low < rd) {
203 lower_level = 1;
204 rd = rd_low;
205 rate = rate_low;
206 dist = dist_low;
207 }
208
209 if (sharpness == 0 && rd_new_eob < rd) {
210 for (int ni = 0; ni < *nz_num; ++ni) {
211 int last_ci = nz_ci[ni];
212 levels[get_padded_idx(last_ci, bwl)] = 0;
213 qcoeff[last_ci] = 0;
214 dqcoeff[last_ci] = 0;
215 }
216 *eob = new_eob;
217 *nz_num = 0;
218 *accu_rate = rate_coeff_eob;
219 *accu_dist = dist_new_eob;
220 lower_level = lower_level_new_eob;
221 } else {
222 *accu_rate += rate;
223 *accu_dist += dist;
224 }
225
226 if (lower_level) {
227 qcoeff[ci] = qc_low;
228 dqcoeff[ci] = dqc_low;
229 levels[get_padded_idx(ci, bwl)] = AOMMIN(abs_qc_low, INT8_MAX);
230 }
231 if (qcoeff[ci]) {
232 nz_ci[*nz_num] = ci;
233 ++*nz_num;
234 }
235 }
236}
237
238static INLINE void update_skip(int *accu_rate, int64_t accu_dist, int *eob,
239 int nz_num, int *nz_ci, int64_t rdmult,
240 int skip_cost, int non_skip_cost,
241 tran_low_t *qcoeff, tran_low_t *dqcoeff,
242 int sharpness) {
243 const int64_t rd = RDCOST(rdmult, *accu_rate + non_skip_cost, accu_dist);
244 const int64_t rd_new_eob = RDCOST(rdmult, skip_cost, 0);
245 if (sharpness == 0 && rd_new_eob < rd) {
246 for (int i = 0; i < nz_num; ++i) {
247 const int ci = nz_ci[i];
248 qcoeff[ci] = 0;
249 dqcoeff[ci] = 0;
250 // no need to set up levels because this is the last step
251 // levels[get_padded_idx(ci, bwl)] = 0;
252 }
253 *accu_rate = 0;
254 *eob = 0;
255 }
256}
257
258// TODO(angiebird): use this function whenever it's possible
259static int get_tx_type_cost(const MACROBLOCK *x, const MACROBLOCKD *xd,
260 int plane, TX_SIZE tx_size, TX_TYPE tx_type,
261 int reduced_tx_set_used) {
262 if (plane > 0) return 0;
263
264 const TX_SIZE square_tx_size = txsize_sqr_map[tx_size];
265
266 const MB_MODE_INFO *mbmi = xd->mi[0];
267 const int is_inter = is_inter_block(mbmi);
268 if (get_ext_tx_types(tx_size, is_inter, reduced_tx_set_used) > 1 &&
269 !xd->lossless[xd->mi[0]->segment_id]) {
270 const int ext_tx_set =
271 get_ext_tx_set(tx_size, is_inter, reduced_tx_set_used);
272 if (is_inter) {
273 if (ext_tx_set > 0)
274 return x->mode_costs
275 .inter_tx_type_costs[ext_tx_set][square_tx_size][tx_type];
276 } else {
277 if (ext_tx_set > 0) {
278 PREDICTION_MODE intra_dir;
279 if (mbmi->filter_intra_mode_info.use_filter_intra)
280 intra_dir = fimode_to_intradir[mbmi->filter_intra_mode_info
281 .filter_intra_mode];
282 else
283 intra_dir = mbmi->mode;
284 return x->mode_costs.intra_tx_type_costs[ext_tx_set][square_tx_size]
285 [intra_dir][tx_type];
286 }
287 }
288 }
289 return 0;
290}
291
292int av1_optimize_txb(const struct AV1_COMP *cpi, MACROBLOCK *x, int plane,
293 int block, TX_SIZE tx_size, TX_TYPE tx_type,
294 const TXB_CTX *const txb_ctx, int *rate_cost,
295 int sharpness) {
296 MACROBLOCKD *xd = &x->e_mbd;
297 const struct macroblock_plane *p = &x->plane[plane];
298 const SCAN_ORDER *scan_order = get_scan(tx_size, tx_type);
299 const int16_t *scan = scan_order->scan;
300 const int shift = av1_get_tx_scale(tx_size);
301 int eob = p->eobs[block];
302 const int16_t *dequant = p->dequant_QTX;
303 const qm_val_t *iqmatrix =
304 av1_get_iqmatrix(&cpi->common.quant_params, xd, plane, tx_size, tx_type);
305 const int block_offset = BLOCK_OFFSET(block);
306 tran_low_t *qcoeff = p->qcoeff + block_offset;
307 tran_low_t *dqcoeff = p->dqcoeff + block_offset;
308 const tran_low_t *tcoeff = p->coeff + block_offset;
309 const CoeffCosts *coeff_costs = &x->coeff_costs;
310
311 // This function is not called if eob = 0.
312 assert(eob > 0);
313
314 const AV1_COMMON *cm = &cpi->common;
315 const PLANE_TYPE plane_type = get_plane_type(plane);
316 const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size);
317 const TX_CLASS tx_class = tx_type_to_class[tx_type];
318 const MB_MODE_INFO *mbmi = xd->mi[0];
319 const int bwl = get_txb_bwl(tx_size);
320 const int width = get_txb_wide(tx_size);
321 const int height = get_txb_high(tx_size);
322 assert(width == (1 << bwl));
323 const int is_inter = is_inter_block(mbmi);
324 const LV_MAP_COEFF_COST *txb_costs =
325 &coeff_costs->coeff_costs[txs_ctx][plane_type];
326 const int eob_multi_size = txsize_log2_minus4[tx_size];
327 const LV_MAP_EOB_COST *txb_eob_costs =
328 &coeff_costs->eob_costs[eob_multi_size][plane_type];
329
330 const int rshift =
331 (sharpness +
332 (cpi->oxcf.q_cfg.aq_mode == VARIANCE_AQ && mbmi->segment_id < 4
333 ? 7 - mbmi->segment_id
334 : 2) +
335 (cpi->oxcf.q_cfg.aq_mode != VARIANCE_AQ &&
336 cpi->oxcf.q_cfg.deltaq_mode == DELTA_Q_PERCEPTUAL &&
337 cm->delta_q_info.delta_q_present_flag && x->sb_energy_level < 0
338 ? (3 - x->sb_energy_level)
339 : 0));
340 const int64_t rdmult =
341 (((int64_t)x->rdmult *
342 (plane_rd_mult[is_inter][plane_type] << (2 * (xd->bd - 8)))) +
343 2) >>
344 rshift;
345
346 uint8_t levels_buf[TX_PAD_2D];
347 uint8_t *const levels = set_levels(levels_buf, width);
348
349 if (eob > 1) av1_txb_init_levels(qcoeff, width, height, levels);
350
351 // TODO(angirbird): check iqmatrix
352
353 const int non_skip_cost = txb_costs->txb_skip_cost[txb_ctx->txb_skip_ctx][0];
354 const int skip_cost = txb_costs->txb_skip_cost[txb_ctx->txb_skip_ctx][1];
355 const int eob_cost = get_eob_cost(eob, txb_eob_costs, txb_costs, tx_class);
356 int accu_rate = eob_cost;
357 int64_t accu_dist = 0;
358 int si = eob - 1;
359 const int ci = scan[si];
360 const tran_low_t qc = qcoeff[ci];
361 const tran_low_t abs_qc = abs(qc);
362 const int sign = qc < 0;
363 const int max_nz_num = 2;
364 int nz_num = 1;
365 int nz_ci[3] = { ci, 0, 0 };
366 if (abs_qc >= 2) {
367 update_coeff_general(&accu_rate, &accu_dist, si, eob, tx_size, tx_class,
368 bwl, height, rdmult, shift, txb_ctx->dc_sign_ctx,
369 dequant, scan, txb_costs, tcoeff, qcoeff, dqcoeff,
370 levels, iqmatrix);
371 --si;
372 } else {
373 assert(abs_qc == 1);
374 const int coeff_ctx = get_lower_levels_ctx_eob(bwl, height, si);
375 accu_rate +=
376 get_coeff_cost_eob(ci, abs_qc, sign, coeff_ctx, txb_ctx->dc_sign_ctx,
377 txb_costs, bwl, tx_class);
378 const tran_low_t tqc = tcoeff[ci];
379 const tran_low_t dqc = dqcoeff[ci];
380 const int64_t dist = get_coeff_dist(tqc, dqc, shift);
381 const int64_t dist0 = get_coeff_dist(tqc, 0, shift);
382 accu_dist += dist - dist0;
383 --si;
384 }
385
386#define UPDATE_COEFF_EOB_CASE(tx_class_literal) \
387 case tx_class_literal: \
388 for (; si >= 0 && nz_num <= max_nz_num; --si) { \
389 update_coeff_eob(&accu_rate, &accu_dist, &eob, &nz_num, nz_ci, si, \
390 tx_size, tx_class_literal, bwl, height, \
391 txb_ctx->dc_sign_ctx, rdmult, shift, dequant, scan, \
392 txb_eob_costs, txb_costs, tcoeff, qcoeff, dqcoeff, \
393 levels, sharpness, iqmatrix); \
394 } \
395 break;
396 switch (tx_class) {
397 UPDATE_COEFF_EOB_CASE(TX_CLASS_2D);
398 UPDATE_COEFF_EOB_CASE(TX_CLASS_HORIZ);
399 UPDATE_COEFF_EOB_CASE(TX_CLASS_VERT);
400#undef UPDATE_COEFF_EOB_CASE
401 default: assert(false);
402 }
403
404 if (si == -1 && nz_num <= max_nz_num) {
405 update_skip(&accu_rate, accu_dist, &eob, nz_num, nz_ci, rdmult, skip_cost,
406 non_skip_cost, qcoeff, dqcoeff, sharpness);
407 }
408
409#define UPDATE_COEFF_SIMPLE_CASE(tx_class_literal) \
410 case tx_class_literal: \
411 for (; si >= 1; --si) { \
412 update_coeff_simple(&accu_rate, si, eob, tx_size, tx_class_literal, bwl, \
413 rdmult, shift, dequant, scan, txb_costs, tcoeff, \
414 qcoeff, dqcoeff, levels, iqmatrix); \
415 } \
416 break;
417 switch (tx_class) {
418 UPDATE_COEFF_SIMPLE_CASE(TX_CLASS_2D);
419 UPDATE_COEFF_SIMPLE_CASE(TX_CLASS_HORIZ);
420 UPDATE_COEFF_SIMPLE_CASE(TX_CLASS_VERT);
421#undef UPDATE_COEFF_SIMPLE_CASE
422 default: assert(false);
423 }
424
425 // DC position
426 if (si == 0) {
427 // no need to update accu_dist because it's not used after this point
428 int64_t dummy_dist = 0;
429 update_coeff_general(&accu_rate, &dummy_dist, si, eob, tx_size, tx_class,
430 bwl, height, rdmult, shift, txb_ctx->dc_sign_ctx,
431 dequant, scan, txb_costs, tcoeff, qcoeff, dqcoeff,
432 levels, iqmatrix);
433 }
434
435 const int tx_type_cost = get_tx_type_cost(x, xd, plane, tx_size, tx_type,
436 cm->features.reduced_tx_set_used);
437 if (eob == 0)
438 accu_rate += skip_cost;
439 else
440 accu_rate += non_skip_cost + tx_type_cost;
441
442 p->eobs[block] = eob;
443 p->txb_entropy_ctx[block] =
444 av1_get_txb_entropy_context(qcoeff, scan_order, p->eobs[block]);
445
446 *rate_cost = accu_rate;
447 return eob;
448}
449
Vishesha45092c2021-01-25 00:28:11 +0530450static AOM_FORCE_INLINE int warehouse_efficients_txb(
451 const MACROBLOCK *x, const int plane, const int block,
452 const TX_SIZE tx_size, const TXB_CTX *const txb_ctx,
453 const struct macroblock_plane *p, const int eob,
454 const PLANE_TYPE plane_type, const LV_MAP_COEFF_COST *const coeff_costs,
455 const MACROBLOCKD *const xd, const TX_TYPE tx_type, const TX_CLASS tx_class,
456 int reduced_tx_set_used) {
457 const tran_low_t *const qcoeff = p->qcoeff + BLOCK_OFFSET(block);
458 const int txb_skip_ctx = txb_ctx->txb_skip_ctx;
459 const int bwl = get_txb_bwl(tx_size);
460 const int width = get_txb_wide(tx_size);
461 const int height = get_txb_high(tx_size);
462 const SCAN_ORDER *const scan_order = get_scan(tx_size, tx_type);
463 const int16_t *const scan = scan_order->scan;
464 uint8_t levels_buf[TX_PAD_2D];
465 uint8_t *const levels = set_levels(levels_buf, width);
466 DECLARE_ALIGNED(16, int8_t, coeff_contexts[MAX_TX_SQUARE]);
467 const int eob_multi_size = txsize_log2_minus4[tx_size];
468 const LV_MAP_EOB_COST *const eob_costs =
469 &x->coeff_costs.eob_costs[eob_multi_size][plane_type];
470 int cost = coeff_costs->txb_skip_cost[txb_skip_ctx][0];
471
472 av1_txb_init_levels(qcoeff, width, height, levels);
473
474 cost += get_tx_type_cost(x, xd, plane, tx_size, tx_type, reduced_tx_set_used);
475
476 cost += get_eob_cost(eob, eob_costs, coeff_costs, tx_class);
477
478 av1_get_nz_map_contexts(levels, scan, eob, tx_size, tx_class, coeff_contexts);
479
480 const int(*lps_cost)[COEFF_BASE_RANGE + 1 + COEFF_BASE_RANGE + 1] =
481 coeff_costs->lps_cost;
482 int c = eob - 1;
483 {
484 const int pos = scan[c];
485 const tran_low_t v = qcoeff[pos];
486 const int sign = AOMSIGN(v);
487 const int level = (v ^ sign) - sign;
488 const int coeff_ctx = coeff_contexts[pos];
489 cost += coeff_costs->base_eob_cost[coeff_ctx][AOMMIN(level, 3) - 1];
490
491 if (v) {
492 // sign bit cost
493 if (level > NUM_BASE_LEVELS) {
494 const int ctx = get_br_ctx_eob(pos, bwl, tx_class);
495 cost += get_br_cost(level, lps_cost[ctx]);
496 }
497 if (c) {
498 cost += av1_cost_literal(1);
499 } else {
500 const int sign01 = (sign ^ sign) - sign;
501 const int dc_sign_ctx = txb_ctx->dc_sign_ctx;
502 cost += coeff_costs->dc_sign_cost[dc_sign_ctx][sign01];
503 return cost;
504 }
505 }
506 }
507 const int(*base_cost)[8] = coeff_costs->base_cost;
508 for (c = eob - 2; c >= 1; --c) {
509 const int pos = scan[c];
510 const int coeff_ctx = coeff_contexts[pos];
511 const tran_low_t v = qcoeff[pos];
512 const int level = abs(v);
513 cost += base_cost[coeff_ctx][AOMMIN(level, 3)];
514 if (v) {
515 // sign bit cost
516 cost += av1_cost_literal(1);
517 if (level > NUM_BASE_LEVELS) {
518 const int ctx = get_br_ctx(levels, pos, bwl, tx_class);
519 cost += get_br_cost(level, lps_cost[ctx]);
520 }
521 }
522 }
523 // c == 0 after previous loop
524 {
525 const int pos = scan[c];
526 const tran_low_t v = qcoeff[pos];
527 const int coeff_ctx = coeff_contexts[pos];
528 const int sign = AOMSIGN(v);
529 const int level = (v ^ sign) - sign;
530 cost += base_cost[coeff_ctx][AOMMIN(level, 3)];
531
532 if (v) {
533 // sign bit cost
534 const int sign01 = (sign ^ sign) - sign;
535 const int dc_sign_ctx = txb_ctx->dc_sign_ctx;
536 cost += coeff_costs->dc_sign_cost[dc_sign_ctx][sign01];
537 if (level > NUM_BASE_LEVELS) {
538 const int ctx = get_br_ctx(levels, pos, bwl, tx_class);
539 cost += get_br_cost(level, lps_cost[ctx]);
540 }
541 }
542 }
543 return cost;
544}
545
Vishesha45092c2021-01-25 00:28:11 +0530546int av1_cost_coeffs_txb_estimate(const MACROBLOCK *x, const int plane,
547 const int block, const TX_SIZE tx_size,
548 const TX_TYPE tx_type) {
549 assert(plane == 0);
550
551 int cost = 0;
552 const struct macroblock_plane *p = &x->plane[plane];
553 const SCAN_ORDER *scan_order = get_scan(tx_size, tx_type);
554 const int16_t *scan = scan_order->scan;
555 tran_low_t *qcoeff = p->qcoeff + BLOCK_OFFSET(block);
556
557 int eob = p->eobs[block];
558
559 // coeffs
560 int c = eob - 1;
561 // eob
562 {
563 const int pos = scan[c];
564 const tran_low_t v = abs(qcoeff[pos]) - 1;
565 cost += (v << (AV1_PROB_COST_SHIFT + 2));
566 }
567 // other coeffs
568 for (c = eob - 2; c >= 0; c--) {
569 const int pos = scan[c];
570 const tran_low_t v = abs(qcoeff[pos]);
571 const int idx = AOMMIN(v, 14);
572
573 cost += costLUT[idx];
574 }
575
576 // const_term does not contain DC, and log(e) does not contain eob, so both
577 // (eob-1)
578 cost += (const_term + loge_par) * (eob - 1);
579
580 return cost;
581}
582
583static AOM_FORCE_INLINE int warehouse_efficients_txb_laplacian(
584 const MACROBLOCK *x, const int plane, const int block,
585 const TX_SIZE tx_size, const TXB_CTX *const txb_ctx, const int eob,
586 const PLANE_TYPE plane_type, const LV_MAP_COEFF_COST *const coeff_costs,
587 const MACROBLOCKD *const xd, const TX_TYPE tx_type, const TX_CLASS tx_class,
588 int reduced_tx_set_used) {
589 const int txb_skip_ctx = txb_ctx->txb_skip_ctx;
590
591 const int eob_multi_size = txsize_log2_minus4[tx_size];
592 const LV_MAP_EOB_COST *const eob_costs =
593 &x->coeff_costs.eob_costs[eob_multi_size][plane_type];
594 int cost = coeff_costs->txb_skip_cost[txb_skip_ctx][0];
595
596 cost += get_tx_type_cost(x, xd, plane, tx_size, tx_type, reduced_tx_set_used);
597
598 cost += get_eob_cost(eob, eob_costs, coeff_costs, tx_class);
599
600 cost += av1_cost_coeffs_txb_estimate(x, plane, block, tx_size, tx_type);
601 return cost;
602}
603
604int av1_cost_coeffs_txb(const MACROBLOCK *x, const int plane, const int block,
605 const TX_SIZE tx_size, const TX_TYPE tx_type,
606 const TXB_CTX *const txb_ctx, int reduced_tx_set_used) {
607 const struct macroblock_plane *p = &x->plane[plane];
608 const int eob = p->eobs[block];
609 const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size);
610 const PLANE_TYPE plane_type = get_plane_type(plane);
611 const LV_MAP_COEFF_COST *const coeff_costs =
612 &x->coeff_costs.coeff_costs[txs_ctx][plane_type];
613 if (eob == 0) {
614 return coeff_costs->txb_skip_cost[txb_ctx->txb_skip_ctx][1];
615 }
616
617 const MACROBLOCKD *const xd = &x->e_mbd;
618 const TX_CLASS tx_class = tx_type_to_class[tx_type];
619
620 return warehouse_efficients_txb(x, plane, block, tx_size, txb_ctx, p, eob,
621 plane_type, coeff_costs, xd, tx_type,
622 tx_class, reduced_tx_set_used);
623}
624
625int av1_cost_coeffs_txb_laplacian(const MACROBLOCK *x, const int plane,
626 const int block, const TX_SIZE tx_size,
627 const TX_TYPE tx_type,
628 const TXB_CTX *const txb_ctx,
629 const int reduced_tx_set_used,
630 const int adjust_eob) {
631 const struct macroblock_plane *p = &x->plane[plane];
632 int eob = p->eobs[block];
633
634 if (adjust_eob) {
635 const SCAN_ORDER *scan_order = get_scan(tx_size, tx_type);
636 const int16_t *scan = scan_order->scan;
637 tran_low_t *tcoeff = p->coeff + BLOCK_OFFSET(block);
638 tran_low_t *qcoeff = p->qcoeff + BLOCK_OFFSET(block);
639 tran_low_t *dqcoeff = p->dqcoeff + BLOCK_OFFSET(block);
640 update_coeff_eob_fast(&eob, av1_get_tx_scale(tx_size), p->dequant_QTX, scan,
641 tcoeff, qcoeff, dqcoeff);
642 p->eobs[block] = eob;
643 }
644
645 const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size);
646 const PLANE_TYPE plane_type = get_plane_type(plane);
647 const LV_MAP_COEFF_COST *const coeff_costs =
648 &x->coeff_costs.coeff_costs[txs_ctx][plane_type];
649 if (eob == 0) {
650 return coeff_costs->txb_skip_cost[txb_ctx->txb_skip_ctx][1];
651 }
652
653 const MACROBLOCKD *const xd = &x->e_mbd;
654 const TX_CLASS tx_class = tx_type_to_class[tx_type];
655
656 return warehouse_efficients_txb_laplacian(
657 x, plane, block, tx_size, txb_ctx, eob, plane_type, coeff_costs, xd,
658 tx_type, tx_class, reduced_tx_set_used);
659}