Vishesh | a45092c | 2021-01-25 00:28:11 +0530 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2021, Alliance for Open Media. All rights reserved |
| 3 | * |
| 4 | * This source code is subject to the terms of the BSD 2 Clause License and |
| 5 | * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License |
| 6 | * was not distributed with this source code in the LICENSE file, you can |
| 7 | * obtain it at www.aomedia.org/license/software. If the Alliance for Open |
| 8 | * Media Patent License 1.0 was not distributed with this source code in the |
| 9 | * PATENTS file, you can obtain it at www.aomedia.org/license/patent. |
| 10 | */ |
| 11 | |
| 12 | #include "av1/encoder/txb_rdopt.h" |
Vishesh | 14be3cd | 2021-01-27 15:21:15 +0530 | [diff] [blame] | 13 | #include "av1/encoder/txb_rdopt_utils.h" |
Vishesh | a45092c | 2021-01-25 00:28:11 +0530 | [diff] [blame] | 14 | |
| 15 | #include "av1/common/idct.h" |
Vishesh | a45092c | 2021-01-25 00:28:11 +0530 | [diff] [blame] | 16 | |
| 17 | static INLINE void update_coeff_general( |
| 18 | int *accu_rate, int64_t *accu_dist, int si, int eob, TX_SIZE tx_size, |
| 19 | TX_CLASS tx_class, int bwl, int height, int64_t rdmult, int shift, |
| 20 | int dc_sign_ctx, const int16_t *dequant, const int16_t *scan, |
| 21 | const LV_MAP_COEFF_COST *txb_costs, const tran_low_t *tcoeff, |
| 22 | tran_low_t *qcoeff, tran_low_t *dqcoeff, uint8_t *levels, |
| 23 | const qm_val_t *iqmatrix) { |
| 24 | const int dqv = get_dqv(dequant, scan[si], iqmatrix); |
| 25 | const int ci = scan[si]; |
| 26 | const tran_low_t qc = qcoeff[ci]; |
| 27 | const int is_last = si == (eob - 1); |
| 28 | const int coeff_ctx = get_lower_levels_ctx_general( |
| 29 | is_last, si, bwl, height, levels, ci, tx_size, tx_class); |
| 30 | if (qc == 0) { |
| 31 | *accu_rate += txb_costs->base_cost[coeff_ctx][0]; |
| 32 | } else { |
| 33 | const int sign = (qc < 0) ? 1 : 0; |
| 34 | const tran_low_t abs_qc = abs(qc); |
| 35 | const tran_low_t tqc = tcoeff[ci]; |
| 36 | const tran_low_t dqc = dqcoeff[ci]; |
| 37 | const int64_t dist = get_coeff_dist(tqc, dqc, shift); |
| 38 | const int64_t dist0 = get_coeff_dist(tqc, 0, shift); |
| 39 | const int rate = |
| 40 | get_coeff_cost_general(is_last, ci, abs_qc, sign, coeff_ctx, |
| 41 | dc_sign_ctx, txb_costs, bwl, tx_class, levels); |
| 42 | const int64_t rd = RDCOST(rdmult, rate, dist); |
| 43 | |
| 44 | tran_low_t qc_low, dqc_low; |
| 45 | tran_low_t abs_qc_low; |
| 46 | int64_t dist_low, rd_low; |
| 47 | int rate_low; |
| 48 | if (abs_qc == 1) { |
| 49 | abs_qc_low = qc_low = dqc_low = 0; |
| 50 | dist_low = dist0; |
| 51 | rate_low = txb_costs->base_cost[coeff_ctx][0]; |
| 52 | } else { |
| 53 | get_qc_dqc_low(abs_qc, sign, dqv, shift, &qc_low, &dqc_low); |
| 54 | abs_qc_low = abs_qc - 1; |
| 55 | dist_low = get_coeff_dist(tqc, dqc_low, shift); |
| 56 | rate_low = |
| 57 | get_coeff_cost_general(is_last, ci, abs_qc_low, sign, coeff_ctx, |
| 58 | dc_sign_ctx, txb_costs, bwl, tx_class, levels); |
| 59 | } |
| 60 | |
| 61 | rd_low = RDCOST(rdmult, rate_low, dist_low); |
| 62 | if (rd_low < rd) { |
| 63 | qcoeff[ci] = qc_low; |
| 64 | dqcoeff[ci] = dqc_low; |
| 65 | levels[get_padded_idx(ci, bwl)] = AOMMIN(abs_qc_low, INT8_MAX); |
| 66 | *accu_rate += rate_low; |
| 67 | *accu_dist += dist_low - dist0; |
| 68 | } else { |
| 69 | *accu_rate += rate; |
| 70 | *accu_dist += dist - dist0; |
| 71 | } |
| 72 | } |
| 73 | } |
| 74 | |
| 75 | static AOM_FORCE_INLINE void update_coeff_simple( |
| 76 | int *accu_rate, int si, int eob, TX_SIZE tx_size, TX_CLASS tx_class, |
| 77 | int bwl, int64_t rdmult, int shift, const int16_t *dequant, |
| 78 | const int16_t *scan, const LV_MAP_COEFF_COST *txb_costs, |
| 79 | const tran_low_t *tcoeff, tran_low_t *qcoeff, tran_low_t *dqcoeff, |
| 80 | uint8_t *levels, const qm_val_t *iqmatrix) { |
| 81 | const int dqv = get_dqv(dequant, scan[si], iqmatrix); |
| 82 | (void)eob; |
| 83 | // this simple version assumes the coeff's scan_idx is not DC (scan_idx != 0) |
| 84 | // and not the last (scan_idx != eob - 1) |
| 85 | assert(si != eob - 1); |
| 86 | assert(si > 0); |
| 87 | const int ci = scan[si]; |
| 88 | const tran_low_t qc = qcoeff[ci]; |
| 89 | const int coeff_ctx = |
| 90 | get_lower_levels_ctx(levels, ci, bwl, tx_size, tx_class); |
| 91 | if (qc == 0) { |
| 92 | *accu_rate += txb_costs->base_cost[coeff_ctx][0]; |
| 93 | } else { |
| 94 | const tran_low_t abs_qc = abs(qc); |
| 95 | const tran_low_t abs_tqc = abs(tcoeff[ci]); |
| 96 | const tran_low_t abs_dqc = abs(dqcoeff[ci]); |
| 97 | int rate_low = 0; |
| 98 | const int rate = get_two_coeff_cost_simple( |
| 99 | ci, abs_qc, coeff_ctx, txb_costs, bwl, tx_class, levels, &rate_low); |
| 100 | if (abs_dqc < abs_tqc) { |
| 101 | *accu_rate += rate; |
| 102 | return; |
| 103 | } |
| 104 | |
| 105 | const int64_t dist = get_coeff_dist(abs_tqc, abs_dqc, shift); |
| 106 | const int64_t rd = RDCOST(rdmult, rate, dist); |
| 107 | |
| 108 | const tran_low_t abs_qc_low = abs_qc - 1; |
| 109 | const tran_low_t abs_dqc_low = (abs_qc_low * dqv) >> shift; |
| 110 | const int64_t dist_low = get_coeff_dist(abs_tqc, abs_dqc_low, shift); |
| 111 | const int64_t rd_low = RDCOST(rdmult, rate_low, dist_low); |
| 112 | |
| 113 | if (rd_low < rd) { |
| 114 | const int sign = (qc < 0) ? 1 : 0; |
| 115 | qcoeff[ci] = (-sign ^ abs_qc_low) + sign; |
| 116 | dqcoeff[ci] = (-sign ^ abs_dqc_low) + sign; |
| 117 | levels[get_padded_idx(ci, bwl)] = AOMMIN(abs_qc_low, INT8_MAX); |
| 118 | *accu_rate += rate_low; |
| 119 | } else { |
| 120 | *accu_rate += rate; |
| 121 | } |
| 122 | } |
| 123 | } |
| 124 | |
| 125 | static AOM_FORCE_INLINE void update_coeff_eob( |
| 126 | int *accu_rate, int64_t *accu_dist, int *eob, int *nz_num, int *nz_ci, |
| 127 | int si, TX_SIZE tx_size, TX_CLASS tx_class, int bwl, int height, |
| 128 | int dc_sign_ctx, int64_t rdmult, int shift, const int16_t *dequant, |
| 129 | const int16_t *scan, const LV_MAP_EOB_COST *txb_eob_costs, |
| 130 | const LV_MAP_COEFF_COST *txb_costs, const tran_low_t *tcoeff, |
| 131 | tran_low_t *qcoeff, tran_low_t *dqcoeff, uint8_t *levels, int sharpness, |
| 132 | const qm_val_t *iqmatrix) { |
| 133 | const int dqv = get_dqv(dequant, scan[si], iqmatrix); |
| 134 | assert(si != *eob - 1); |
| 135 | const int ci = scan[si]; |
| 136 | const tran_low_t qc = qcoeff[ci]; |
| 137 | const int coeff_ctx = |
| 138 | get_lower_levels_ctx(levels, ci, bwl, tx_size, tx_class); |
| 139 | if (qc == 0) { |
| 140 | *accu_rate += txb_costs->base_cost[coeff_ctx][0]; |
| 141 | } else { |
| 142 | int lower_level = 0; |
| 143 | const tran_low_t abs_qc = abs(qc); |
| 144 | const tran_low_t tqc = tcoeff[ci]; |
| 145 | const tran_low_t dqc = dqcoeff[ci]; |
| 146 | const int sign = (qc < 0) ? 1 : 0; |
| 147 | const int64_t dist0 = get_coeff_dist(tqc, 0, shift); |
| 148 | int64_t dist = get_coeff_dist(tqc, dqc, shift) - dist0; |
| 149 | int rate = |
| 150 | get_coeff_cost_general(0, ci, abs_qc, sign, coeff_ctx, dc_sign_ctx, |
| 151 | txb_costs, bwl, tx_class, levels); |
| 152 | int64_t rd = RDCOST(rdmult, *accu_rate + rate, *accu_dist + dist); |
| 153 | |
| 154 | tran_low_t qc_low, dqc_low; |
| 155 | tran_low_t abs_qc_low; |
| 156 | int64_t dist_low, rd_low; |
| 157 | int rate_low; |
| 158 | if (abs_qc == 1) { |
| 159 | abs_qc_low = 0; |
| 160 | dqc_low = qc_low = 0; |
| 161 | dist_low = 0; |
| 162 | rate_low = txb_costs->base_cost[coeff_ctx][0]; |
| 163 | rd_low = RDCOST(rdmult, *accu_rate + rate_low, *accu_dist); |
| 164 | } else { |
| 165 | get_qc_dqc_low(abs_qc, sign, dqv, shift, &qc_low, &dqc_low); |
| 166 | abs_qc_low = abs_qc - 1; |
| 167 | dist_low = get_coeff_dist(tqc, dqc_low, shift) - dist0; |
| 168 | rate_low = |
| 169 | get_coeff_cost_general(0, ci, abs_qc_low, sign, coeff_ctx, |
| 170 | dc_sign_ctx, txb_costs, bwl, tx_class, levels); |
| 171 | rd_low = RDCOST(rdmult, *accu_rate + rate_low, *accu_dist + dist_low); |
| 172 | } |
| 173 | |
| 174 | int lower_level_new_eob = 0; |
| 175 | const int new_eob = si + 1; |
| 176 | const int coeff_ctx_new_eob = get_lower_levels_ctx_eob(bwl, height, si); |
| 177 | const int new_eob_cost = |
| 178 | get_eob_cost(new_eob, txb_eob_costs, txb_costs, tx_class); |
| 179 | int rate_coeff_eob = |
| 180 | new_eob_cost + get_coeff_cost_eob(ci, abs_qc, sign, coeff_ctx_new_eob, |
| 181 | dc_sign_ctx, txb_costs, bwl, |
| 182 | tx_class); |
| 183 | int64_t dist_new_eob = dist; |
| 184 | int64_t rd_new_eob = RDCOST(rdmult, rate_coeff_eob, dist_new_eob); |
| 185 | |
| 186 | if (abs_qc_low > 0) { |
| 187 | const int rate_coeff_eob_low = |
| 188 | new_eob_cost + get_coeff_cost_eob(ci, abs_qc_low, sign, |
| 189 | coeff_ctx_new_eob, dc_sign_ctx, |
| 190 | txb_costs, bwl, tx_class); |
| 191 | const int64_t dist_new_eob_low = dist_low; |
| 192 | const int64_t rd_new_eob_low = |
| 193 | RDCOST(rdmult, rate_coeff_eob_low, dist_new_eob_low); |
| 194 | if (rd_new_eob_low < rd_new_eob) { |
| 195 | lower_level_new_eob = 1; |
| 196 | rd_new_eob = rd_new_eob_low; |
| 197 | rate_coeff_eob = rate_coeff_eob_low; |
| 198 | dist_new_eob = dist_new_eob_low; |
| 199 | } |
| 200 | } |
| 201 | |
| 202 | if (rd_low < rd) { |
| 203 | lower_level = 1; |
| 204 | rd = rd_low; |
| 205 | rate = rate_low; |
| 206 | dist = dist_low; |
| 207 | } |
| 208 | |
| 209 | if (sharpness == 0 && rd_new_eob < rd) { |
| 210 | for (int ni = 0; ni < *nz_num; ++ni) { |
| 211 | int last_ci = nz_ci[ni]; |
| 212 | levels[get_padded_idx(last_ci, bwl)] = 0; |
| 213 | qcoeff[last_ci] = 0; |
| 214 | dqcoeff[last_ci] = 0; |
| 215 | } |
| 216 | *eob = new_eob; |
| 217 | *nz_num = 0; |
| 218 | *accu_rate = rate_coeff_eob; |
| 219 | *accu_dist = dist_new_eob; |
| 220 | lower_level = lower_level_new_eob; |
| 221 | } else { |
| 222 | *accu_rate += rate; |
| 223 | *accu_dist += dist; |
| 224 | } |
| 225 | |
| 226 | if (lower_level) { |
| 227 | qcoeff[ci] = qc_low; |
| 228 | dqcoeff[ci] = dqc_low; |
| 229 | levels[get_padded_idx(ci, bwl)] = AOMMIN(abs_qc_low, INT8_MAX); |
| 230 | } |
| 231 | if (qcoeff[ci]) { |
| 232 | nz_ci[*nz_num] = ci; |
| 233 | ++*nz_num; |
| 234 | } |
| 235 | } |
| 236 | } |
| 237 | |
| 238 | static INLINE void update_skip(int *accu_rate, int64_t accu_dist, int *eob, |
| 239 | int nz_num, int *nz_ci, int64_t rdmult, |
| 240 | int skip_cost, int non_skip_cost, |
| 241 | tran_low_t *qcoeff, tran_low_t *dqcoeff, |
| 242 | int sharpness) { |
| 243 | const int64_t rd = RDCOST(rdmult, *accu_rate + non_skip_cost, accu_dist); |
| 244 | const int64_t rd_new_eob = RDCOST(rdmult, skip_cost, 0); |
| 245 | if (sharpness == 0 && rd_new_eob < rd) { |
| 246 | for (int i = 0; i < nz_num; ++i) { |
| 247 | const int ci = nz_ci[i]; |
| 248 | qcoeff[ci] = 0; |
| 249 | dqcoeff[ci] = 0; |
| 250 | // no need to set up levels because this is the last step |
| 251 | // levels[get_padded_idx(ci, bwl)] = 0; |
| 252 | } |
| 253 | *accu_rate = 0; |
| 254 | *eob = 0; |
| 255 | } |
| 256 | } |
| 257 | |
| 258 | // TODO(angiebird): use this function whenever it's possible |
| 259 | static int get_tx_type_cost(const MACROBLOCK *x, const MACROBLOCKD *xd, |
| 260 | int plane, TX_SIZE tx_size, TX_TYPE tx_type, |
| 261 | int reduced_tx_set_used) { |
| 262 | if (plane > 0) return 0; |
| 263 | |
| 264 | const TX_SIZE square_tx_size = txsize_sqr_map[tx_size]; |
| 265 | |
| 266 | const MB_MODE_INFO *mbmi = xd->mi[0]; |
| 267 | const int is_inter = is_inter_block(mbmi); |
| 268 | if (get_ext_tx_types(tx_size, is_inter, reduced_tx_set_used) > 1 && |
| 269 | !xd->lossless[xd->mi[0]->segment_id]) { |
| 270 | const int ext_tx_set = |
| 271 | get_ext_tx_set(tx_size, is_inter, reduced_tx_set_used); |
| 272 | if (is_inter) { |
| 273 | if (ext_tx_set > 0) |
| 274 | return x->mode_costs |
| 275 | .inter_tx_type_costs[ext_tx_set][square_tx_size][tx_type]; |
| 276 | } else { |
| 277 | if (ext_tx_set > 0) { |
| 278 | PREDICTION_MODE intra_dir; |
| 279 | if (mbmi->filter_intra_mode_info.use_filter_intra) |
| 280 | intra_dir = fimode_to_intradir[mbmi->filter_intra_mode_info |
| 281 | .filter_intra_mode]; |
| 282 | else |
| 283 | intra_dir = mbmi->mode; |
| 284 | return x->mode_costs.intra_tx_type_costs[ext_tx_set][square_tx_size] |
| 285 | [intra_dir][tx_type]; |
| 286 | } |
| 287 | } |
| 288 | } |
| 289 | return 0; |
| 290 | } |
| 291 | |
| 292 | int av1_optimize_txb(const struct AV1_COMP *cpi, MACROBLOCK *x, int plane, |
| 293 | int block, TX_SIZE tx_size, TX_TYPE tx_type, |
| 294 | const TXB_CTX *const txb_ctx, int *rate_cost, |
| 295 | int sharpness) { |
| 296 | MACROBLOCKD *xd = &x->e_mbd; |
| 297 | const struct macroblock_plane *p = &x->plane[plane]; |
| 298 | const SCAN_ORDER *scan_order = get_scan(tx_size, tx_type); |
| 299 | const int16_t *scan = scan_order->scan; |
| 300 | const int shift = av1_get_tx_scale(tx_size); |
| 301 | int eob = p->eobs[block]; |
| 302 | const int16_t *dequant = p->dequant_QTX; |
| 303 | const qm_val_t *iqmatrix = |
| 304 | av1_get_iqmatrix(&cpi->common.quant_params, xd, plane, tx_size, tx_type); |
| 305 | const int block_offset = BLOCK_OFFSET(block); |
| 306 | tran_low_t *qcoeff = p->qcoeff + block_offset; |
| 307 | tran_low_t *dqcoeff = p->dqcoeff + block_offset; |
| 308 | const tran_low_t *tcoeff = p->coeff + block_offset; |
| 309 | const CoeffCosts *coeff_costs = &x->coeff_costs; |
| 310 | |
| 311 | // This function is not called if eob = 0. |
| 312 | assert(eob > 0); |
| 313 | |
| 314 | const AV1_COMMON *cm = &cpi->common; |
| 315 | const PLANE_TYPE plane_type = get_plane_type(plane); |
| 316 | const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size); |
| 317 | const TX_CLASS tx_class = tx_type_to_class[tx_type]; |
| 318 | const MB_MODE_INFO *mbmi = xd->mi[0]; |
| 319 | const int bwl = get_txb_bwl(tx_size); |
| 320 | const int width = get_txb_wide(tx_size); |
| 321 | const int height = get_txb_high(tx_size); |
| 322 | assert(width == (1 << bwl)); |
| 323 | const int is_inter = is_inter_block(mbmi); |
| 324 | const LV_MAP_COEFF_COST *txb_costs = |
| 325 | &coeff_costs->coeff_costs[txs_ctx][plane_type]; |
| 326 | const int eob_multi_size = txsize_log2_minus4[tx_size]; |
| 327 | const LV_MAP_EOB_COST *txb_eob_costs = |
| 328 | &coeff_costs->eob_costs[eob_multi_size][plane_type]; |
| 329 | |
| 330 | const int rshift = |
| 331 | (sharpness + |
| 332 | (cpi->oxcf.q_cfg.aq_mode == VARIANCE_AQ && mbmi->segment_id < 4 |
| 333 | ? 7 - mbmi->segment_id |
| 334 | : 2) + |
| 335 | (cpi->oxcf.q_cfg.aq_mode != VARIANCE_AQ && |
| 336 | cpi->oxcf.q_cfg.deltaq_mode == DELTA_Q_PERCEPTUAL && |
| 337 | cm->delta_q_info.delta_q_present_flag && x->sb_energy_level < 0 |
| 338 | ? (3 - x->sb_energy_level) |
| 339 | : 0)); |
| 340 | const int64_t rdmult = |
| 341 | (((int64_t)x->rdmult * |
| 342 | (plane_rd_mult[is_inter][plane_type] << (2 * (xd->bd - 8)))) + |
| 343 | 2) >> |
| 344 | rshift; |
| 345 | |
| 346 | uint8_t levels_buf[TX_PAD_2D]; |
| 347 | uint8_t *const levels = set_levels(levels_buf, width); |
| 348 | |
| 349 | if (eob > 1) av1_txb_init_levels(qcoeff, width, height, levels); |
| 350 | |
| 351 | // TODO(angirbird): check iqmatrix |
| 352 | |
| 353 | const int non_skip_cost = txb_costs->txb_skip_cost[txb_ctx->txb_skip_ctx][0]; |
| 354 | const int skip_cost = txb_costs->txb_skip_cost[txb_ctx->txb_skip_ctx][1]; |
| 355 | const int eob_cost = get_eob_cost(eob, txb_eob_costs, txb_costs, tx_class); |
| 356 | int accu_rate = eob_cost; |
| 357 | int64_t accu_dist = 0; |
| 358 | int si = eob - 1; |
| 359 | const int ci = scan[si]; |
| 360 | const tran_low_t qc = qcoeff[ci]; |
| 361 | const tran_low_t abs_qc = abs(qc); |
| 362 | const int sign = qc < 0; |
| 363 | const int max_nz_num = 2; |
| 364 | int nz_num = 1; |
| 365 | int nz_ci[3] = { ci, 0, 0 }; |
| 366 | if (abs_qc >= 2) { |
| 367 | update_coeff_general(&accu_rate, &accu_dist, si, eob, tx_size, tx_class, |
| 368 | bwl, height, rdmult, shift, txb_ctx->dc_sign_ctx, |
| 369 | dequant, scan, txb_costs, tcoeff, qcoeff, dqcoeff, |
| 370 | levels, iqmatrix); |
| 371 | --si; |
| 372 | } else { |
| 373 | assert(abs_qc == 1); |
| 374 | const int coeff_ctx = get_lower_levels_ctx_eob(bwl, height, si); |
| 375 | accu_rate += |
| 376 | get_coeff_cost_eob(ci, abs_qc, sign, coeff_ctx, txb_ctx->dc_sign_ctx, |
| 377 | txb_costs, bwl, tx_class); |
| 378 | const tran_low_t tqc = tcoeff[ci]; |
| 379 | const tran_low_t dqc = dqcoeff[ci]; |
| 380 | const int64_t dist = get_coeff_dist(tqc, dqc, shift); |
| 381 | const int64_t dist0 = get_coeff_dist(tqc, 0, shift); |
| 382 | accu_dist += dist - dist0; |
| 383 | --si; |
| 384 | } |
| 385 | |
| 386 | #define UPDATE_COEFF_EOB_CASE(tx_class_literal) \ |
| 387 | case tx_class_literal: \ |
| 388 | for (; si >= 0 && nz_num <= max_nz_num; --si) { \ |
| 389 | update_coeff_eob(&accu_rate, &accu_dist, &eob, &nz_num, nz_ci, si, \ |
| 390 | tx_size, tx_class_literal, bwl, height, \ |
| 391 | txb_ctx->dc_sign_ctx, rdmult, shift, dequant, scan, \ |
| 392 | txb_eob_costs, txb_costs, tcoeff, qcoeff, dqcoeff, \ |
| 393 | levels, sharpness, iqmatrix); \ |
| 394 | } \ |
| 395 | break; |
| 396 | switch (tx_class) { |
| 397 | UPDATE_COEFF_EOB_CASE(TX_CLASS_2D); |
| 398 | UPDATE_COEFF_EOB_CASE(TX_CLASS_HORIZ); |
| 399 | UPDATE_COEFF_EOB_CASE(TX_CLASS_VERT); |
| 400 | #undef UPDATE_COEFF_EOB_CASE |
| 401 | default: assert(false); |
| 402 | } |
| 403 | |
| 404 | if (si == -1 && nz_num <= max_nz_num) { |
| 405 | update_skip(&accu_rate, accu_dist, &eob, nz_num, nz_ci, rdmult, skip_cost, |
| 406 | non_skip_cost, qcoeff, dqcoeff, sharpness); |
| 407 | } |
| 408 | |
| 409 | #define UPDATE_COEFF_SIMPLE_CASE(tx_class_literal) \ |
| 410 | case tx_class_literal: \ |
| 411 | for (; si >= 1; --si) { \ |
| 412 | update_coeff_simple(&accu_rate, si, eob, tx_size, tx_class_literal, bwl, \ |
| 413 | rdmult, shift, dequant, scan, txb_costs, tcoeff, \ |
| 414 | qcoeff, dqcoeff, levels, iqmatrix); \ |
| 415 | } \ |
| 416 | break; |
| 417 | switch (tx_class) { |
| 418 | UPDATE_COEFF_SIMPLE_CASE(TX_CLASS_2D); |
| 419 | UPDATE_COEFF_SIMPLE_CASE(TX_CLASS_HORIZ); |
| 420 | UPDATE_COEFF_SIMPLE_CASE(TX_CLASS_VERT); |
| 421 | #undef UPDATE_COEFF_SIMPLE_CASE |
| 422 | default: assert(false); |
| 423 | } |
| 424 | |
| 425 | // DC position |
| 426 | if (si == 0) { |
| 427 | // no need to update accu_dist because it's not used after this point |
| 428 | int64_t dummy_dist = 0; |
| 429 | update_coeff_general(&accu_rate, &dummy_dist, si, eob, tx_size, tx_class, |
| 430 | bwl, height, rdmult, shift, txb_ctx->dc_sign_ctx, |
| 431 | dequant, scan, txb_costs, tcoeff, qcoeff, dqcoeff, |
| 432 | levels, iqmatrix); |
| 433 | } |
| 434 | |
| 435 | const int tx_type_cost = get_tx_type_cost(x, xd, plane, tx_size, tx_type, |
| 436 | cm->features.reduced_tx_set_used); |
| 437 | if (eob == 0) |
| 438 | accu_rate += skip_cost; |
| 439 | else |
| 440 | accu_rate += non_skip_cost + tx_type_cost; |
| 441 | |
| 442 | p->eobs[block] = eob; |
| 443 | p->txb_entropy_ctx[block] = |
| 444 | av1_get_txb_entropy_context(qcoeff, scan_order, p->eobs[block]); |
| 445 | |
| 446 | *rate_cost = accu_rate; |
| 447 | return eob; |
| 448 | } |
| 449 | |
Vishesh | a45092c | 2021-01-25 00:28:11 +0530 | [diff] [blame] | 450 | static AOM_FORCE_INLINE int warehouse_efficients_txb( |
| 451 | const MACROBLOCK *x, const int plane, const int block, |
| 452 | const TX_SIZE tx_size, const TXB_CTX *const txb_ctx, |
| 453 | const struct macroblock_plane *p, const int eob, |
| 454 | const PLANE_TYPE plane_type, const LV_MAP_COEFF_COST *const coeff_costs, |
| 455 | const MACROBLOCKD *const xd, const TX_TYPE tx_type, const TX_CLASS tx_class, |
| 456 | int reduced_tx_set_used) { |
| 457 | const tran_low_t *const qcoeff = p->qcoeff + BLOCK_OFFSET(block); |
| 458 | const int txb_skip_ctx = txb_ctx->txb_skip_ctx; |
| 459 | const int bwl = get_txb_bwl(tx_size); |
| 460 | const int width = get_txb_wide(tx_size); |
| 461 | const int height = get_txb_high(tx_size); |
| 462 | const SCAN_ORDER *const scan_order = get_scan(tx_size, tx_type); |
| 463 | const int16_t *const scan = scan_order->scan; |
| 464 | uint8_t levels_buf[TX_PAD_2D]; |
| 465 | uint8_t *const levels = set_levels(levels_buf, width); |
| 466 | DECLARE_ALIGNED(16, int8_t, coeff_contexts[MAX_TX_SQUARE]); |
| 467 | const int eob_multi_size = txsize_log2_minus4[tx_size]; |
| 468 | const LV_MAP_EOB_COST *const eob_costs = |
| 469 | &x->coeff_costs.eob_costs[eob_multi_size][plane_type]; |
| 470 | int cost = coeff_costs->txb_skip_cost[txb_skip_ctx][0]; |
| 471 | |
| 472 | av1_txb_init_levels(qcoeff, width, height, levels); |
| 473 | |
| 474 | cost += get_tx_type_cost(x, xd, plane, tx_size, tx_type, reduced_tx_set_used); |
| 475 | |
| 476 | cost += get_eob_cost(eob, eob_costs, coeff_costs, tx_class); |
| 477 | |
| 478 | av1_get_nz_map_contexts(levels, scan, eob, tx_size, tx_class, coeff_contexts); |
| 479 | |
| 480 | const int(*lps_cost)[COEFF_BASE_RANGE + 1 + COEFF_BASE_RANGE + 1] = |
| 481 | coeff_costs->lps_cost; |
| 482 | int c = eob - 1; |
| 483 | { |
| 484 | const int pos = scan[c]; |
| 485 | const tran_low_t v = qcoeff[pos]; |
| 486 | const int sign = AOMSIGN(v); |
| 487 | const int level = (v ^ sign) - sign; |
| 488 | const int coeff_ctx = coeff_contexts[pos]; |
| 489 | cost += coeff_costs->base_eob_cost[coeff_ctx][AOMMIN(level, 3) - 1]; |
| 490 | |
| 491 | if (v) { |
| 492 | // sign bit cost |
| 493 | if (level > NUM_BASE_LEVELS) { |
| 494 | const int ctx = get_br_ctx_eob(pos, bwl, tx_class); |
| 495 | cost += get_br_cost(level, lps_cost[ctx]); |
| 496 | } |
| 497 | if (c) { |
| 498 | cost += av1_cost_literal(1); |
| 499 | } else { |
| 500 | const int sign01 = (sign ^ sign) - sign; |
| 501 | const int dc_sign_ctx = txb_ctx->dc_sign_ctx; |
| 502 | cost += coeff_costs->dc_sign_cost[dc_sign_ctx][sign01]; |
| 503 | return cost; |
| 504 | } |
| 505 | } |
| 506 | } |
| 507 | const int(*base_cost)[8] = coeff_costs->base_cost; |
| 508 | for (c = eob - 2; c >= 1; --c) { |
| 509 | const int pos = scan[c]; |
| 510 | const int coeff_ctx = coeff_contexts[pos]; |
| 511 | const tran_low_t v = qcoeff[pos]; |
| 512 | const int level = abs(v); |
| 513 | cost += base_cost[coeff_ctx][AOMMIN(level, 3)]; |
| 514 | if (v) { |
| 515 | // sign bit cost |
| 516 | cost += av1_cost_literal(1); |
| 517 | if (level > NUM_BASE_LEVELS) { |
| 518 | const int ctx = get_br_ctx(levels, pos, bwl, tx_class); |
| 519 | cost += get_br_cost(level, lps_cost[ctx]); |
| 520 | } |
| 521 | } |
| 522 | } |
| 523 | // c == 0 after previous loop |
| 524 | { |
| 525 | const int pos = scan[c]; |
| 526 | const tran_low_t v = qcoeff[pos]; |
| 527 | const int coeff_ctx = coeff_contexts[pos]; |
| 528 | const int sign = AOMSIGN(v); |
| 529 | const int level = (v ^ sign) - sign; |
| 530 | cost += base_cost[coeff_ctx][AOMMIN(level, 3)]; |
| 531 | |
| 532 | if (v) { |
| 533 | // sign bit cost |
| 534 | const int sign01 = (sign ^ sign) - sign; |
| 535 | const int dc_sign_ctx = txb_ctx->dc_sign_ctx; |
| 536 | cost += coeff_costs->dc_sign_cost[dc_sign_ctx][sign01]; |
| 537 | if (level > NUM_BASE_LEVELS) { |
| 538 | const int ctx = get_br_ctx(levels, pos, bwl, tx_class); |
| 539 | cost += get_br_cost(level, lps_cost[ctx]); |
| 540 | } |
| 541 | } |
| 542 | } |
| 543 | return cost; |
| 544 | } |
| 545 | |
Vishesh | a45092c | 2021-01-25 00:28:11 +0530 | [diff] [blame] | 546 | int av1_cost_coeffs_txb_estimate(const MACROBLOCK *x, const int plane, |
| 547 | const int block, const TX_SIZE tx_size, |
| 548 | const TX_TYPE tx_type) { |
| 549 | assert(plane == 0); |
| 550 | |
| 551 | int cost = 0; |
| 552 | const struct macroblock_plane *p = &x->plane[plane]; |
| 553 | const SCAN_ORDER *scan_order = get_scan(tx_size, tx_type); |
| 554 | const int16_t *scan = scan_order->scan; |
| 555 | tran_low_t *qcoeff = p->qcoeff + BLOCK_OFFSET(block); |
| 556 | |
| 557 | int eob = p->eobs[block]; |
| 558 | |
| 559 | // coeffs |
| 560 | int c = eob - 1; |
| 561 | // eob |
| 562 | { |
| 563 | const int pos = scan[c]; |
| 564 | const tran_low_t v = abs(qcoeff[pos]) - 1; |
| 565 | cost += (v << (AV1_PROB_COST_SHIFT + 2)); |
| 566 | } |
| 567 | // other coeffs |
| 568 | for (c = eob - 2; c >= 0; c--) { |
| 569 | const int pos = scan[c]; |
| 570 | const tran_low_t v = abs(qcoeff[pos]); |
| 571 | const int idx = AOMMIN(v, 14); |
| 572 | |
| 573 | cost += costLUT[idx]; |
| 574 | } |
| 575 | |
| 576 | // const_term does not contain DC, and log(e) does not contain eob, so both |
| 577 | // (eob-1) |
| 578 | cost += (const_term + loge_par) * (eob - 1); |
| 579 | |
| 580 | return cost; |
| 581 | } |
| 582 | |
| 583 | static AOM_FORCE_INLINE int warehouse_efficients_txb_laplacian( |
| 584 | const MACROBLOCK *x, const int plane, const int block, |
| 585 | const TX_SIZE tx_size, const TXB_CTX *const txb_ctx, const int eob, |
| 586 | const PLANE_TYPE plane_type, const LV_MAP_COEFF_COST *const coeff_costs, |
| 587 | const MACROBLOCKD *const xd, const TX_TYPE tx_type, const TX_CLASS tx_class, |
| 588 | int reduced_tx_set_used) { |
| 589 | const int txb_skip_ctx = txb_ctx->txb_skip_ctx; |
| 590 | |
| 591 | const int eob_multi_size = txsize_log2_minus4[tx_size]; |
| 592 | const LV_MAP_EOB_COST *const eob_costs = |
| 593 | &x->coeff_costs.eob_costs[eob_multi_size][plane_type]; |
| 594 | int cost = coeff_costs->txb_skip_cost[txb_skip_ctx][0]; |
| 595 | |
| 596 | cost += get_tx_type_cost(x, xd, plane, tx_size, tx_type, reduced_tx_set_used); |
| 597 | |
| 598 | cost += get_eob_cost(eob, eob_costs, coeff_costs, tx_class); |
| 599 | |
| 600 | cost += av1_cost_coeffs_txb_estimate(x, plane, block, tx_size, tx_type); |
| 601 | return cost; |
| 602 | } |
| 603 | |
| 604 | int av1_cost_coeffs_txb(const MACROBLOCK *x, const int plane, const int block, |
| 605 | const TX_SIZE tx_size, const TX_TYPE tx_type, |
| 606 | const TXB_CTX *const txb_ctx, int reduced_tx_set_used) { |
| 607 | const struct macroblock_plane *p = &x->plane[plane]; |
| 608 | const int eob = p->eobs[block]; |
| 609 | const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size); |
| 610 | const PLANE_TYPE plane_type = get_plane_type(plane); |
| 611 | const LV_MAP_COEFF_COST *const coeff_costs = |
| 612 | &x->coeff_costs.coeff_costs[txs_ctx][plane_type]; |
| 613 | if (eob == 0) { |
| 614 | return coeff_costs->txb_skip_cost[txb_ctx->txb_skip_ctx][1]; |
| 615 | } |
| 616 | |
| 617 | const MACROBLOCKD *const xd = &x->e_mbd; |
| 618 | const TX_CLASS tx_class = tx_type_to_class[tx_type]; |
| 619 | |
| 620 | return warehouse_efficients_txb(x, plane, block, tx_size, txb_ctx, p, eob, |
| 621 | plane_type, coeff_costs, xd, tx_type, |
| 622 | tx_class, reduced_tx_set_used); |
| 623 | } |
| 624 | |
| 625 | int av1_cost_coeffs_txb_laplacian(const MACROBLOCK *x, const int plane, |
| 626 | const int block, const TX_SIZE tx_size, |
| 627 | const TX_TYPE tx_type, |
| 628 | const TXB_CTX *const txb_ctx, |
| 629 | const int reduced_tx_set_used, |
| 630 | const int adjust_eob) { |
| 631 | const struct macroblock_plane *p = &x->plane[plane]; |
| 632 | int eob = p->eobs[block]; |
| 633 | |
| 634 | if (adjust_eob) { |
| 635 | const SCAN_ORDER *scan_order = get_scan(tx_size, tx_type); |
| 636 | const int16_t *scan = scan_order->scan; |
| 637 | tran_low_t *tcoeff = p->coeff + BLOCK_OFFSET(block); |
| 638 | tran_low_t *qcoeff = p->qcoeff + BLOCK_OFFSET(block); |
| 639 | tran_low_t *dqcoeff = p->dqcoeff + BLOCK_OFFSET(block); |
| 640 | update_coeff_eob_fast(&eob, av1_get_tx_scale(tx_size), p->dequant_QTX, scan, |
| 641 | tcoeff, qcoeff, dqcoeff); |
| 642 | p->eobs[block] = eob; |
| 643 | } |
| 644 | |
| 645 | const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size); |
| 646 | const PLANE_TYPE plane_type = get_plane_type(plane); |
| 647 | const LV_MAP_COEFF_COST *const coeff_costs = |
| 648 | &x->coeff_costs.coeff_costs[txs_ctx][plane_type]; |
| 649 | if (eob == 0) { |
| 650 | return coeff_costs->txb_skip_cost[txb_ctx->txb_skip_ctx][1]; |
| 651 | } |
| 652 | |
| 653 | const MACROBLOCKD *const xd = &x->e_mbd; |
| 654 | const TX_CLASS tx_class = tx_type_to_class[tx_type]; |
| 655 | |
| 656 | return warehouse_efficients_txb_laplacian( |
| 657 | x, plane, block, tx_size, txb_ctx, eob, plane_type, coeff_costs, xd, |
| 658 | tx_type, tx_class, reduced_tx_set_used); |
| 659 | } |