Refactor: Replace rate dist sse skip by RD_STATS in VAR_TX
This is to facilitate implementation of rd_debug tool; it doesn't change
coding behavior.
Change-Id: I0eb82b31473883ba6652ed11dca09b9ec4530183
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index b844594..04c9fec 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -6170,11 +6170,12 @@
ENTROPY_CONTEXT ctxl[2 * MAX_MIB_SIZE];
const struct macroblockd_plane *const pd = &xd->plane[plane];
int coeff_ctx = 1;
+ RD_STATS this_rd_stats;
- this_rate = 0;
- this_dist = 0;
- pnsse = 0;
- pnskip = 1;
+ this_rd_stats.rate = 0;
+ this_rd_stats.dist = 0;
+ this_rd_stats.sse = 0;
+ this_rd_stats.skip = 1;
tx_size = max_txsize_lookup[bsize];
tx_size =
@@ -6184,8 +6185,13 @@
av1_subtract_plane(x, bsize, plane);
av1_tx_block_rd_b(cpi, x, tx_size, 0, 0, plane, 0,
- get_plane_block_size(bsize, pd), coeff_ctx, &this_rate,
- &this_dist, &pnsse, &pnskip);
+ get_plane_block_size(bsize, pd), coeff_ctx,
+ &this_rd_stats);
+
+ this_rate = this_rd_stats.rate;
+ this_dist = this_rd_stats.dist;
+ pnsse = this_rd_stats.sse;
+ pnskip = this_rd_stats.skip;
#else
tx_size = max_txsize_lookup[bsize];
tx_size =
@@ -6213,7 +6219,9 @@
ENTROPY_CONTEXT ctxl[2 * MAX_MIB_SIZE];
const struct macroblockd_plane *const pd = &xd->plane[0];
int coeff_ctx = 1;
+ RD_STATS this_rd_stats;
#endif // CONFIG_VAR_TX
+
#if CONFIG_EXT_TX
if (!ext_tx_used_inter[ext_tx_set][tx_type]) continue;
#else
@@ -6222,15 +6230,20 @@
mbmi->tx_type = tx_type;
#if CONFIG_VAR_TX
- this_rate = 0;
- this_dist = 0;
- pnsse = 0;
- pnskip = 1;
+ this_rd_stats.rate = 0;
+ this_rd_stats.dist = 0;
+ this_rd_stats.sse = 0;
+ this_rd_stats.skip = 1;
av1_get_entropy_contexts(bsize, tx_size, pd, ctxa, ctxl);
coeff_ctx = combine_entropy_contexts(ctxa[0], ctxl[0]);
- av1_tx_block_rd_b(cpi, x, tx_size, 0, 0, 0, 0, bsize, coeff_ctx, &this_rate,
- &this_dist, &pnsse, &pnskip);
+ av1_tx_block_rd_b(cpi, x, tx_size, 0, 0, 0, 0, bsize, coeff_ctx,
+ &this_rd_stats);
+
+ this_rate = this_rd_stats.rate;
+ this_dist = this_rd_stats.dist;
+ pnsse = this_rd_stats.sse;
+ pnskip = this_rd_stats.skip;
#else
av1_txfm_rd_in_plane_supertx(x, cpi, &this_rate, &this_dist, &pnskip,
&pnsse, INT64_MAX, 0, bsize, tx_size, 0);
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 4922d61..62edfa0 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -2906,8 +2906,7 @@
#if CONFIG_VAR_TX
void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
int blk_row, int blk_col, int plane, int block,
- int plane_bsize, int coeff_ctx, int *rate, int64_t *dist,
- int64_t *bsse, int *skip) {
+ int plane_bsize, int coeff_ctx, RD_STATS *rd_stats) {
const AV1_COMMON *const cm = &cpi->common;
MACROBLOCKD *xd = &x->e_mbd;
const struct macroblock_plane *const p = &x->plane[plane];
@@ -2994,7 +2993,7 @@
if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH)
tmp = ROUND_POWER_OF_TWO(tmp, (xd->bd - 8) * 2);
#endif // CONFIG_AOM_HIGHBITDEPTH
- *bsse += tmp * 16;
+ rd_stats->sse += tmp * 16;
if (p->eobs[block] > 0) {
INV_TXFM_PARAM inv_txfm_param;
@@ -3035,10 +3034,10 @@
tmp = this_dist;
}
}
- *dist += tmp * 16;
- *rate += av1_cost_coeffs(cm, x, plane, block, coeff_ctx, tx_size,
- scan_order->scan, scan_order->neighbors, 0);
- *skip &= (p->eobs[block] == 0);
+ rd_stats->dist += tmp * 16;
+ rd_stats->rate += av1_cost_coeffs(cm, x, plane, block, coeff_ctx, tx_size,
+ scan_order->scan, scan_order->neighbors, 0);
+ rd_stats->skip &= (p->eobs[block] == 0);
}
static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
@@ -3046,8 +3045,8 @@
int depth, BLOCK_SIZE plane_bsize,
ENTROPY_CONTEXT *ta, ENTROPY_CONTEXT *tl,
TXFM_CONTEXT *tx_above, TXFM_CONTEXT *tx_left,
- int *rate, int64_t *dist, int64_t *bsse, int *skip,
- int64_t ref_best_rd, int *is_cost_valid) {
+ RD_STATS *rd_stats, int64_t ref_best_rd,
+ int *is_cost_valid) {
MACROBLOCKD *const xd = &x->e_mbd;
MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
struct macroblock_plane *const p = &x->plane[plane];
@@ -3086,10 +3085,10 @@
coeff_ctx = get_entropy_context(tx_size, pta, ptl);
- *rate = 0;
- *dist = 0;
- *bsse = 0;
- *skip = 1;
+ rd_stats->rate = 0;
+ rd_stats->dist = 0;
+ rd_stats->sse = 0;
+ rd_stats->skip = 1;
if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
@@ -3099,25 +3098,26 @@
if (cpi->common.tx_mode == TX_MODE_SELECT || tx_size == TX_4X4) {
inter_tx_size[0][0] = tx_size;
av1_tx_block_rd_b(cpi, x, tx_size, blk_row, blk_col, plane, block,
- plane_bsize, coeff_ctx, rate, dist, bsse, skip);
+ plane_bsize, coeff_ctx, rd_stats);
- if ((RDCOST(x->rdmult, x->rddiv, *rate, *dist) >=
- RDCOST(x->rdmult, x->rddiv, zero_blk_rate, *bsse) ||
- *skip == 1) &&
+ if ((RDCOST(x->rdmult, x->rddiv, rd_stats->rate, rd_stats->dist) >=
+ RDCOST(x->rdmult, x->rddiv, zero_blk_rate, rd_stats->sse) ||
+ rd_stats->skip == 1) &&
!xd->lossless[mbmi->segment_id]) {
- *rate = zero_blk_rate;
- *dist = *bsse;
- *skip = 1;
+ rd_stats->rate = zero_blk_rate;
+ rd_stats->dist = rd_stats->sse;
+ rd_stats->skip = 1;
x->blk_skip[plane][blk_row * bw + blk_col] = 1;
p->eobs[block] = 0;
} else {
x->blk_skip[plane][blk_row * bw + blk_col] = 0;
- *skip = 0;
+ rd_stats->skip = 0;
}
if (tx_size > TX_4X4 && depth < MAX_VARTX_DEPTH)
- *rate += av1_cost_bit(cpi->common.fc->txfm_partition_prob[ctx], 0);
- this_rd = RDCOST(x->rdmult, x->rddiv, *rate, *dist);
+ rd_stats->rate +=
+ av1_cost_bit(cpi->common.fc->txfm_partition_prob[ctx], 0);
+ this_rd = RDCOST(x->rdmult, x->rddiv, rd_stats->rate, rd_stats->dist);
tmp_eob = p->eobs[block];
}
@@ -3125,10 +3125,7 @@
const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
const int bsl = tx_size_wide_unit[sub_txs];
int sub_step = tx_size_wide_unit[sub_txs] * tx_size_high_unit[sub_txs];
- int this_rate;
- int64_t this_dist;
- int64_t this_bsse;
- int this_skip;
+ RD_STATS this_rd_stats;
int this_cost_valid = 1;
int64_t tmp_rd = 0;
@@ -3141,12 +3138,12 @@
int offsetc = (i & 0x01) * bsl;
select_tx_block(cpi, x, blk_row + offsetr, blk_col + offsetc, plane,
block + i * sub_step, sub_txs, depth + 1, plane_bsize, ta,
- tl, tx_above, tx_left, &this_rate, &this_dist, &this_bsse,
- &this_skip, ref_best_rd - tmp_rd, &this_cost_valid);
- sum_rate += this_rate;
- sum_dist += this_dist;
- sum_bsse += this_bsse;
- all_skip &= this_skip;
+ tl, tx_above, tx_left, &this_rd_stats,
+ ref_best_rd - tmp_rd, &this_cost_valid);
+ sum_rate += this_rd_stats.rate;
+ sum_dist += this_rd_stats.dist;
+ sum_bsse += this_rd_stats.sse;
+ all_skip &= this_rd_stats.skip;
tmp_rd = RDCOST(x->rdmult, x->rddiv, sum_rate, sum_dist);
if (this_rd < tmp_rd) break;
}
@@ -3165,29 +3162,29 @@
inter_tx_size[idy][idx] = tx_size;
mbmi->tx_size = tx_size;
if (this_rd == INT64_MAX) *is_cost_valid = 0;
- x->blk_skip[plane][blk_row * bw + blk_col] = *skip;
+ x->blk_skip[plane][blk_row * bw + blk_col] = rd_stats->skip;
} else {
- *rate = sum_rate;
- *dist = sum_dist;
- *bsse = sum_bsse;
- *skip = all_skip;
+ rd_stats->rate = sum_rate;
+ rd_stats->dist = sum_dist;
+ rd_stats->sse = sum_bsse;
+ rd_stats->skip = all_skip;
if (sum_rd == INT64_MAX) *is_cost_valid = 0;
}
}
-static void inter_block_yrd(const AV1_COMP *cpi, MACROBLOCK *x, int *rate,
- int64_t *distortion, int *skippable, int64_t *sse,
- BLOCK_SIZE bsize, int64_t ref_best_rd) {
+static void inter_block_yrd(const AV1_COMP *cpi, MACROBLOCK *x,
+ RD_STATS *rd_stats, BLOCK_SIZE bsize,
+ int64_t ref_best_rd) {
MACROBLOCKD *const xd = &x->e_mbd;
int is_cost_valid = 1;
int64_t this_rd = 0;
if (ref_best_rd < 0) is_cost_valid = 0;
- *rate = 0;
- *distortion = 0;
- *sse = 0;
- *skippable = 1;
+ rd_stats->rate = 0;
+ rd_stats->dist = 0;
+ rd_stats->sse = 0;
+ rd_stats->skip = 1;
if (is_cost_valid) {
const struct macroblockd_plane *const pd = &xd->plane[0];
@@ -3205,8 +3202,11 @@
TXFM_CONTEXT tx_above[MAX_MIB_SIZE];
TXFM_CONTEXT tx_left[MAX_MIB_SIZE];
- int pnrate = 0, pnskip = 1;
- int64_t pndist = 0, pnsse = 0;
+ RD_STATS pn_rd_stats;
+ pn_rd_stats.rate = 0;
+ pn_rd_stats.skip = 1;
+ pn_rd_stats.dist = 0;
+ pn_rd_stats.sse = 0;
av1_get_entropy_contexts(bsize, TX_4X4, pd, ctxa, ctxl);
memcpy(tx_above, xd->above_txfm_context,
@@ -3218,35 +3218,35 @@
for (idx = 0; idx < mi_width; idx += bw) {
select_tx_block(cpi, x, idy, idx, 0, block, max_tx_size,
mi_height != mi_width, plane_bsize, ctxa, ctxl,
- tx_above, tx_left, &pnrate, &pndist, &pnsse, &pnskip,
- ref_best_rd - this_rd, &is_cost_valid);
- *rate += pnrate;
- *distortion += pndist;
- *sse += pnsse;
- *skippable &= pnskip;
- this_rd += AOMMIN(RDCOST(x->rdmult, x->rddiv, pnrate, pndist),
- RDCOST(x->rdmult, x->rddiv, 0, pnsse));
+ tx_above, tx_left, &pn_rd_stats, ref_best_rd - this_rd,
+ &is_cost_valid);
+ rd_stats->rate += pn_rd_stats.rate;
+ rd_stats->dist += pn_rd_stats.dist;
+ rd_stats->sse += pn_rd_stats.sse;
+ rd_stats->skip &= pn_rd_stats.skip;
+ this_rd += AOMMIN(
+ RDCOST(x->rdmult, x->rddiv, pn_rd_stats.rate, pn_rd_stats.dist),
+ RDCOST(x->rdmult, x->rddiv, 0, pn_rd_stats.sse));
block += step;
}
}
}
- this_rd = AOMMIN(RDCOST(x->rdmult, x->rddiv, *rate, *distortion),
- RDCOST(x->rdmult, x->rddiv, 0, *sse));
+ this_rd = AOMMIN(RDCOST(x->rdmult, x->rddiv, rd_stats->rate, rd_stats->dist),
+ RDCOST(x->rdmult, x->rddiv, 0, rd_stats->sse));
if (this_rd > ref_best_rd) is_cost_valid = 0;
if (!is_cost_valid) {
// reset cost value
- *rate = INT_MAX;
- *distortion = INT64_MAX;
- *sse = INT64_MAX;
- *skippable = 0;
+ rd_stats->rate = INT_MAX;
+ rd_stats->dist = INT64_MAX;
+ rd_stats->sse = INT64_MAX;
+ rd_stats->skip = 0;
}
}
static int64_t select_tx_size_fix_type(const AV1_COMP *cpi, MACROBLOCK *x,
- int *rate, int64_t *dist, int *skippable,
- int64_t *sse, BLOCK_SIZE bsize,
+ RD_STATS *rd_stats, BLOCK_SIZE bsize,
int64_t ref_best_rd, TX_TYPE tx_type) {
const AV1_COMMON *const cm = &cpi->common;
MACROBLOCKD *const xd = &x->e_mbd;
@@ -3258,55 +3258,52 @@
int64_t rd;
mbmi->tx_type = tx_type;
- inter_block_yrd(cpi, x, rate, dist, skippable, sse, bsize, ref_best_rd);
+ inter_block_yrd(cpi, x, rd_stats, bsize, ref_best_rd);
#if CONFIG_EXT_TX && CONFIG_RECT_TX
if (is_rect_tx_allowed(xd, mbmi)) {
- int rate_rect_tx, skippable_rect_tx = 0;
- int64_t dist_rect_tx, sse_rect_tx, rd_rect_tx;
+ RD_STATS rect_rd_stats;
+ int64_t rd_rect_tx;
int tx_size_cat = inter_tx_size_cat_lookup[bsize];
TX_SIZE tx_size = max_txsize_rect_lookup[bsize];
TX_SIZE var_tx_size = mbmi->tx_size;
- txfm_rd_in_plane(x, cpi, &rate_rect_tx, &dist_rect_tx, &skippable_rect_tx,
- &sse_rect_tx, ref_best_rd, 0, bsize, tx_size,
- cpi->sf.use_fast_coef_costing);
+ txfm_rd_in_plane(x, cpi, &rect_rd_stats.rate, &rect_rd_stats.dist,
+ &rect_rd_stats.skip, &rect_rd_stats.sse, ref_best_rd, 0,
+ bsize, tx_size, cpi->sf.use_fast_coef_costing);
- if (*rate != INT_MAX) {
- *rate += av1_cost_bit(cm->fc->rect_tx_prob[tx_size_cat], 0);
- if (*skippable) {
- rd = RDCOST(x->rdmult, x->rddiv, s1, *sse);
+ if (rd_stats->rate != INT_MAX) {
+ rd_stats->rate += av1_cost_bit(cm->fc->rect_tx_prob[tx_size_cat], 0);
+ if (rd_stats->skip) {
+ rd = RDCOST(x->rdmult, x->rddiv, s1, rd_stats->sse);
} else {
- rd = RDCOST(x->rdmult, x->rddiv, *rate + s0, *dist);
+ rd = RDCOST(x->rdmult, x->rddiv, rd_stats->rate + s0, rd_stats->dist);
if (is_inter && !xd->lossless[xd->mi[0]->mbmi.segment_id] &&
- !(*skippable))
- rd = AOMMIN(rd, RDCOST(x->rdmult, x->rddiv, s1, *sse));
+ !rd_stats->skip)
+ rd = AOMMIN(rd, RDCOST(x->rdmult, x->rddiv, s1, rd_stats->sse));
}
} else {
rd = INT64_MAX;
}
- if (rate_rect_tx != INT_MAX) {
- rate_rect_tx += av1_cost_bit(cm->fc->rect_tx_prob[tx_size_cat], 1);
- if (skippable_rect_tx) {
- rd_rect_tx = RDCOST(x->rdmult, x->rddiv, s1, sse_rect_tx);
+ if (rect_rd_stats.rate != INT_MAX) {
+ rect_rd_stats.rate += av1_cost_bit(cm->fc->rect_tx_prob[tx_size_cat], 1);
+ if (rect_rd_stats.skip) {
+ rd_rect_tx = RDCOST(x->rdmult, x->rddiv, s1, rect_rd_stats.sse);
} else {
- rd_rect_tx =
- RDCOST(x->rdmult, x->rddiv, rate_rect_tx + s0, dist_rect_tx);
+ rd_rect_tx = RDCOST(x->rdmult, x->rddiv, rect_rd_stats.rate + s0,
+ rect_rd_stats.dist);
if (is_inter && !xd->lossless[xd->mi[0]->mbmi.segment_id] &&
- !(skippable_rect_tx))
- rd_rect_tx =
- AOMMIN(rd_rect_tx, RDCOST(x->rdmult, x->rddiv, s1, sse_rect_tx));
+ !(rect_rd_stats.skip))
+ rd_rect_tx = AOMMIN(
+ rd_rect_tx, RDCOST(x->rdmult, x->rddiv, s1, rect_rd_stats.sse));
}
} else {
rd_rect_tx = INT64_MAX;
}
if (rd_rect_tx < rd) {
- *rate = rate_rect_tx;
- *dist = dist_rect_tx;
- *sse = sse_rect_tx;
- *skippable = skippable_rect_tx;
- if (!xd->lossless[mbmi->segment_id]) x->blk_skip[0][0] = *skippable;
+ *rd_stats = rect_rd_stats;
+ if (!xd->lossless[mbmi->segment_id]) x->blk_skip[0][0] = rd_stats->skip;
mbmi->tx_size = tx_size;
mbmi->inter_tx_size[0][0] = mbmi->tx_size;
} else {
@@ -3315,7 +3312,7 @@
}
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
- if (*rate == INT_MAX) return INT64_MAX;
+ if (rd_stats->rate == INT_MAX) return INT64_MAX;
#if CONFIG_EXT_TX
if (get_ext_tx_types(mbmi->tx_size, bsize, is_inter) > 1 &&
@@ -3323,41 +3320,41 @@
int ext_tx_set = get_ext_tx_set(mbmi->tx_size, bsize, is_inter);
if (is_inter) {
if (ext_tx_set > 0)
- *rate +=
+ rd_stats->rate +=
cpi->inter_tx_type_costs[ext_tx_set][txsize_sqr_map[mbmi->tx_size]]
[mbmi->tx_type];
} else {
if (ext_tx_set > 0 && ALLOW_INTRA_EXT_TX)
- *rate += cpi->intra_tx_type_costs[ext_tx_set][mbmi->tx_size][mbmi->mode]
- [mbmi->tx_type];
+ rd_stats->rate += cpi->intra_tx_type_costs[ext_tx_set][mbmi->tx_size]
+ [mbmi->mode][mbmi->tx_type];
}
}
#else // CONFIG_EXT_TX
if (mbmi->tx_size < TX_32X32 && !xd->lossless[xd->mi[0]->mbmi.segment_id]) {
if (is_inter)
- *rate += cpi->inter_tx_type_costs[mbmi->tx_size][mbmi->tx_type];
+ rd_stats->rate += cpi->inter_tx_type_costs[mbmi->tx_size][mbmi->tx_type];
else
- *rate +=
+ rd_stats->rate +=
cpi->intra_tx_type_costs[mbmi->tx_size]
[intra_mode_to_tx_type_context[mbmi->mode]]
[mbmi->tx_type];
}
#endif // CONFIG_EXT_TX
- if (*skippable)
- rd = RDCOST(x->rdmult, x->rddiv, s1, *sse);
+ if (rd_stats->skip)
+ rd = RDCOST(x->rdmult, x->rddiv, s1, rd_stats->sse);
else
- rd = RDCOST(x->rdmult, x->rddiv, *rate + s0, *dist);
+ rd = RDCOST(x->rdmult, x->rddiv, rd_stats->rate + s0, rd_stats->dist);
- if (is_inter && !xd->lossless[xd->mi[0]->mbmi.segment_id] && !(*skippable))
- rd = AOMMIN(rd, RDCOST(x->rdmult, x->rddiv, s1, *sse));
+ if (is_inter && !xd->lossless[xd->mi[0]->mbmi.segment_id] &&
+ !(rd_stats->skip))
+ rd = AOMMIN(rd, RDCOST(x->rdmult, x->rddiv, s1, rd_stats->sse));
return rd;
}
-static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x, int *rate,
- int64_t *distortion, int *skippable,
- int64_t *sse, BLOCK_SIZE bsize,
+static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x,
+ RD_STATS *rd_stats, BLOCK_SIZE bsize,
int64_t ref_best_rd) {
const TX_SIZE max_tx_size = max_txsize_lookup[bsize];
MACROBLOCKD *const xd = &x->e_mbd;
@@ -3383,16 +3380,17 @@
prune = prune_tx_types(cpi, bsize, x, xd, 0);
#endif
- *distortion = INT64_MAX;
- *rate = INT_MAX;
- *skippable = 0;
- *sse = INT64_MAX;
+ rd_stats->dist = INT64_MAX;
+ rd_stats->rate = INT_MAX;
+ rd_stats->skip = 0;
+ rd_stats->sse = INT64_MAX;
for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) {
- int this_rate = 0;
- int this_skip = 1;
- int64_t this_dist = 0;
- int64_t this_sse = 0;
+ RD_STATS this_rd_stats;
+ this_rd_stats.rate = 0;
+ this_rd_stats.skip = 1;
+ this_rd_stats.dist = 0;
+ this_rd_stats.sse = 0;
#if CONFIG_EXT_TX
if (is_inter) {
if (!ext_tx_used_inter[ext_tx_set][tx_type]) continue;
@@ -3415,15 +3413,12 @@
tx_type != get_default_tx_type(0, xd, 0, max_tx_size))
continue;
- rd = select_tx_size_fix_type(cpi, x, &this_rate, &this_dist, &this_skip,
- &this_sse, bsize, ref_best_rd, tx_type);
+ rd = select_tx_size_fix_type(cpi, x, &this_rd_stats, bsize, ref_best_rd,
+ tx_type);
if (rd < best_rd) {
best_rd = rd;
- *distortion = this_dist;
- *rate = this_rate;
- *skippable = this_skip;
- *sse = this_sse;
+ *rd_stats = this_rd_stats;
best_tx_type = mbmi->tx_type;
best_tx = mbmi->tx_size;
memcpy(best_blk_skip, x->blk_skip[0], sizeof(best_blk_skip[0]) * n4);
@@ -3444,8 +3439,7 @@
static void tx_block_rd(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
int blk_col, int plane, int block, TX_SIZE tx_size,
BLOCK_SIZE plane_bsize, ENTROPY_CONTEXT *above_ctx,
- ENTROPY_CONTEXT *left_ctx, int *rate, int64_t *dist,
- int64_t *bsse, int *skip) {
+ ENTROPY_CONTEXT *left_ctx, RD_STATS *rd_stats) {
MACROBLOCKD *const xd = &x->e_mbd;
MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
struct macroblock_plane *const p = &x->plane[plane];
@@ -3473,7 +3467,7 @@
ENTROPY_CONTEXT *tl = left_ctx + blk_row;
coeff_ctx = get_entropy_context(tx_size, ta, tl);
av1_tx_block_rd_b(cpi, x, tx_size, blk_row, blk_col, plane, block,
- plane_bsize, coeff_ctx, rate, dist, bsse, skip);
+ plane_bsize, coeff_ctx, rd_stats);
for (i = 0; i < tx_size_wide_unit[tx_size]; ++i)
ta[i] = !(p->eobs[block] == 0);
@@ -3492,16 +3486,16 @@
int offsetc = (i & 0x01) * bsl;
tx_block_rd(cpi, x, blk_row + offsetr, blk_col + offsetc, plane,
block + i * step, sub_txs, plane_bsize, above_ctx, left_ctx,
- rate, dist, bsse, skip);
+ rd_stats);
}
}
}
// Return value 0: early termination triggered, no valid rd cost available;
// 1: rd cost values are valid.
-static int inter_block_uvrd(const AV1_COMP *cpi, MACROBLOCK *x, int *rate,
- int64_t *distortion, int *skippable, int64_t *sse,
- BLOCK_SIZE bsize, int64_t ref_best_rd) {
+static int inter_block_uvrd(const AV1_COMP *cpi, MACROBLOCK *x,
+ RD_STATS *rd_stats, BLOCK_SIZE bsize,
+ int64_t ref_best_rd) {
MACROBLOCKD *const xd = &x->e_mbd;
MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
int plane;
@@ -3510,14 +3504,15 @@
if (ref_best_rd < 0) is_cost_valid = 0;
- *rate = 0;
- *distortion = 0;
- *sse = 0;
- *skippable = 1;
+ rd_stats->rate = 0;
+ rd_stats->dist = 0;
+ rd_stats->sse = 0;
+ rd_stats->skip = 1;
#if CONFIG_EXT_TX && CONFIG_RECT_TX
if (is_rect_tx(mbmi->tx_size)) {
- return super_block_uvrd(cpi, x, rate, distortion, skippable, sse, bsize,
+ return super_block_uvrd(cpi, x, &rd_stats->rate, &rd_stats->dist,
+ &rd_stats->skip, &rd_stats->sse, bsize,
ref_best_rd);
}
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
@@ -3538,33 +3533,37 @@
int idx, idy;
int block = 0;
const int step = bh * bw;
- int pnrate = 0, pnskip = 1;
- int64_t pndist = 0, pnsse = 0;
ENTROPY_CONTEXT ta[2 * MAX_MIB_SIZE];
ENTROPY_CONTEXT tl[2 * MAX_MIB_SIZE];
+ RD_STATS pn_rd_stats;
+ pn_rd_stats.rate = 0;
+ pn_rd_stats.skip = 1;
+ pn_rd_stats.dist = 0;
+ pn_rd_stats.sse = 0;
av1_get_entropy_contexts(bsize, TX_4X4, pd, ta, tl);
for (idy = 0; idy < mi_height; idy += bh) {
for (idx = 0; idx < mi_width; idx += bw) {
tx_block_rd(cpi, x, idy, idx, plane, block, max_tx_size, plane_bsize,
- ta, tl, &pnrate, &pndist, &pnsse, &pnskip);
+ ta, tl, &pn_rd_stats);
block += step;
}
}
- if (pnrate == INT_MAX) {
+ if (pn_rd_stats.rate == INT_MAX) {
is_cost_valid = 0;
break;
}
- *rate += pnrate;
- *distortion += pndist;
- *sse += pnsse;
- *skippable &= pnskip;
+ rd_stats->rate += pn_rd_stats.rate;
+ rd_stats->dist += pn_rd_stats.dist;
+ rd_stats->sse += pn_rd_stats.sse;
+ rd_stats->skip &= pn_rd_stats.skip;
- this_rd = AOMMIN(RDCOST(x->rdmult, x->rddiv, *rate, *distortion),
- RDCOST(x->rdmult, x->rddiv, 0, *sse));
+ this_rd =
+ AOMMIN(RDCOST(x->rdmult, x->rddiv, rd_stats->rate, rd_stats->dist),
+ RDCOST(x->rdmult, x->rddiv, 0, rd_stats->sse));
if (this_rd > ref_best_rd) {
is_cost_valid = 0;
@@ -3574,10 +3573,10 @@
if (!is_cost_valid) {
// reset cost value
- *rate = INT_MAX;
- *distortion = INT64_MAX;
- *sse = INT64_MAX;
- *skippable = 0;
+ rd_stats->rate = INT_MAX;
+ rd_stats->dist = INT64_MAX;
+ rd_stats->sse = INT64_MAX;
+ rd_stats->skip = 0;
}
return is_cost_valid;
@@ -7493,13 +7492,21 @@
int skippable_y, skippable_uv;
int64_t sseuv = INT64_MAX;
int64_t rdcosty = INT64_MAX;
+ int is_cost_valid_uv = 0;
+#if CONFIG_VAR_TX
+ RD_STATS rd_stats_uv;
+#endif
// Y cost and distortion
av1_subtract_plane(x, bsize, 0);
#if CONFIG_VAR_TX
if (cm->tx_mode == TX_MODE_SELECT && !xd->lossless[mbmi->segment_id]) {
- select_tx_type_yrd(cpi, x, rate_y, &distortion_y, &skippable_y, psse,
- bsize, ref_best_rd);
+ RD_STATS rd_stats_y;
+ select_tx_type_yrd(cpi, x, &rd_stats_y, bsize, ref_best_rd);
+ *rate_y = rd_stats_y.rate;
+ distortion_y = rd_stats_y.dist;
+ skippable_y = rd_stats_y.skip;
+ *psse = rd_stats_y.sse;
} else {
int idx, idy;
super_block_yrd(cpi, x, rate_y, &distortion_y, &skippable_y, psse,
@@ -7537,13 +7544,18 @@
rdcosty = AOMMIN(rdcosty, RDCOST(x->rdmult, x->rddiv, 0, *psse));
#if CONFIG_VAR_TX
- if (!inter_block_uvrd(cpi, x, rate_uv, &distortion_uv, &skippable_uv,
- &sseuv, bsize, ref_best_rd - rdcosty))
+ is_cost_valid_uv =
+ inter_block_uvrd(cpi, x, &rd_stats_uv, bsize, ref_best_rd - rdcosty);
+ *rate_uv = rd_stats_uv.rate;
+ distortion_uv = rd_stats_uv.dist;
+ skippable_uv = rd_stats_uv.skip;
+ sseuv = rd_stats_uv.sse;
#else
- if (!super_block_uvrd(cpi, x, rate_uv, &distortion_uv, &skippable_uv,
- &sseuv, bsize, ref_best_rd - rdcosty))
+ is_cost_valid_uv =
+ super_block_uvrd(cpi, x, rate_uv, &distortion_uv, &skippable_uv,
+ &sseuv, bsize, ref_best_rd - rdcosty);
#endif // CONFIG_VAR_TX
- {
+ if (!is_cost_valid_uv) {
*rate2 = INT_MAX;
*distortion = INT64_MAX;
#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
@@ -9201,6 +9213,9 @@
}
if (is_inter_mode(mbmi->mode)) {
+#if CONFIG_VAR_TX
+ RD_STATS rd_stats_uv;
+#endif
av1_build_inter_predictors_sb(xd, mi_row, mi_col, bsize);
#if CONFIG_MOTION_VAR
if (mbmi->motion_mode == OBMC_CAUSAL)
@@ -9210,8 +9225,12 @@
av1_subtract_plane(x, bsize, 0);
#if CONFIG_VAR_TX
if (cm->tx_mode == TX_MODE_SELECT || xd->lossless[mbmi->segment_id]) {
- select_tx_type_yrd(cpi, x, &rate_y, &dist_y, &skip_y, &sse_y, bsize,
- INT64_MAX);
+ RD_STATS rd_stats_y;
+ select_tx_type_yrd(cpi, x, &rd_stats_y, bsize, INT64_MAX);
+ rate_y = rd_stats_y.rate;
+ dist_y = rd_stats_y.dist;
+ sse_y = rd_stats_y.sse;
+ skip_y = rd_stats_y.skip;
} else {
int idx, idy;
super_block_yrd(cpi, x, &rate_y, &dist_y, &skip_y, &sse_y, bsize,
@@ -9223,8 +9242,11 @@
sizeof(uint8_t) * xd->n8_h * xd->n8_w * 4);
}
- inter_block_uvrd(cpi, x, &rate_uv, &dist_uv, &skip_uv, &sse_uv, bsize,
- INT64_MAX);
+ inter_block_uvrd(cpi, x, &rd_stats_uv, bsize, INT64_MAX);
+ rate_uv = rd_stats_uv.rate;
+ dist_uv = rd_stats_uv.dist;
+ skip_uv = rd_stats_uv.skip;
+ sse_uv = rd_stats_uv.sse;
#else
super_block_yrd(cpi, x, &rate_y, &dist_y, &skip_y, &sse_y, bsize,
INT64_MAX);
@@ -10357,16 +10379,24 @@
if (tmp_best_rdu > 0) {
// If even the 'Y' rd value of split is higher than best so far
// then dont bother looking at UV
+ int is_cost_valid_uv;
+#if CONFIG_VAR_TX
+ RD_STATS rd_stats_uv;
+#endif
av1_build_inter_predictors_sbuv(&x->e_mbd, mi_row, mi_col, BLOCK_8X8);
#if CONFIG_VAR_TX
- if (!inter_block_uvrd(cpi, x, &rate_uv, &distortion_uv, &uv_skippable,
- &uv_sse, BLOCK_8X8, tmp_best_rdu))
- continue;
+ is_cost_valid_uv =
+ inter_block_uvrd(cpi, x, &rd_stats_uv, BLOCK_8X8, tmp_best_rdu);
+ rate_uv = rd_stats_uv.rate;
+ distortion_uv = rd_stats_uv.dist;
+ uv_skippable = rd_stats_uv.skip;
+ uv_sse = rd_stats_uv.sse;
#else
- if (!super_block_uvrd(cpi, x, &rate_uv, &distortion_uv, &uv_skippable,
- &uv_sse, BLOCK_8X8, tmp_best_rdu))
- continue;
+ is_cost_valid_uv =
+ super_block_uvrd(cpi, x, &rate_uv, &distortion_uv, &uv_skippable,
+ &uv_sse, BLOCK_8X8, tmp_best_rdu);
#endif
+ if (!is_cost_valid_uv) continue;
rate2 += rate_uv;
distortion2 += distortion_uv;
skippable = skippable && uv_skippable;
diff --git a/av1/encoder/rdopt.h b/av1/encoder/rdopt.h
index 5d9fc12..8c65770 100644
--- a/av1/encoder/rdopt.h
+++ b/av1/encoder/rdopt.h
@@ -26,6 +26,16 @@
struct macroblock;
struct RD_COST;
+#if CONFIG_VAR_TX
+// TODO(angiebird): Merge RD_COST and RD_STATS
+typedef struct RD_STATS {
+ int rate;
+ int64_t dist;
+ int64_t sse;
+ int skip;
+} RD_STATS;
+#endif
+
int av1_cost_coeffs(const AV1_COMMON *const cm, MACROBLOCK *x, int plane,
int block, int coeff_ctx, TX_SIZE tx_size,
const int16_t *scan, const int16_t *nb,
@@ -77,8 +87,7 @@
#if CONFIG_VAR_TX
void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
int blk_row, int blk_col, int plane, int block,
- int plane_bsize, int coeff_ctx, int *rate, int64_t *dist,
- int64_t *bsse, int *skip);
+ int plane_bsize, int coeff_ctx, RD_STATS *rd_stats);
#endif
void av1_txfm_rd_in_plane_supertx(MACROBLOCK *x, const AV1_COMP *cpi, int *rate,
