Merge "Add min_tx_size variable to recursive transform block partition system" into nextgenv2
diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index 439a6b4..31836fb 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -238,6 +238,7 @@
// TODO(jingning): This effectively assigned a separate entry for each
// 8x8 block. Apparently it takes much more space than needed.
TX_SIZE inter_tx_size[MAX_MIB_SIZE][MAX_MIB_SIZE];
+ TX_SIZE min_tx_size;
#endif
int8_t skip;
int8_t segment_id;
diff --git a/av1/common/onyxc_int.h b/av1/common/onyxc_int.h
index 1291b65..20270cb 100644
--- a/av1/common/onyxc_int.h
+++ b/av1/common/onyxc_int.h
@@ -741,6 +741,11 @@
}
#if CONFIG_VAR_TX
+static INLINE TX_SIZE get_min_tx_size(const TX_SIZE tx_size) {
+ if (tx_size >= TX_SIZES_ALL) assert(0);
+ return txsize_sqr_map[tx_size];
+}
+
static INLINE void set_txfm_ctx(TXFM_CONTEXT *txfm_ctx, uint8_t txs, int len) {
int i;
for (i = 0; i < len; ++i) txfm_ctx[i] = txs;
diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c
index 088167e..a318c2f 100644
--- a/av1/decoder/decodemv.c
+++ b/av1/decoder/decodemv.c
@@ -315,6 +315,7 @@
for (idx = 0; idx < tx_size_wide_unit[tx_size] / 2; ++idx)
inter_tx_size[idy][idx] = tx_size;
mbmi->tx_size = tx_size;
+ mbmi->min_tx_size = AOMMIN(mbmi->min_tx_size, get_min_tx_size(tx_size));
if (counts) ++counts->txfm_partition[ctx][0];
txfm_partition_update(xd->above_txfm_context + tx_col,
xd->left_txfm_context + tx_row, tx_size);
@@ -333,6 +334,7 @@
if (tx_size == TX_8X8) {
inter_tx_size[0][0] = TX_4X4;
mbmi->tx_size = TX_4X4;
+ mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
txfm_partition_update(xd->above_txfm_context + tx_col,
xd->left_txfm_context + tx_row, TX_4X4);
return;
@@ -352,6 +354,7 @@
for (idx = 0; idx < tx_size_wide_unit[tx_size] / 2; ++idx)
inter_tx_size[idy][idx] = tx_size;
mbmi->tx_size = tx_size;
+ mbmi->min_tx_size = AOMMIN(mbmi->min_tx_size, get_min_tx_size(tx_size));
if (counts) ++counts->txfm_partition[ctx][0];
txfm_partition_update(xd->above_txfm_context + tx_col,
xd->left_txfm_context + tx_row, tx_size);
@@ -672,7 +675,11 @@
#endif
aom_reader *r) {
const int inter_block = is_inter_block(mbmi);
+#if CONFIG_VAR_TX
+ const TX_SIZE tx_size = inter_block ? mbmi->min_tx_size : mbmi->tx_size;
+#else
const TX_SIZE tx_size = mbmi->tx_size;
+#endif
if (!FIXED_TX_TYPE) {
#if CONFIG_EXT_TX
if (get_ext_tx_types(tx_size, mbmi->sb_type, inter_block) > 1 &&
@@ -1856,6 +1863,7 @@
set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, xd);
} else {
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
+ mbmi->min_tx_size = TX_SIZES_ALL;
for (idy = 0; idy < height; idy += bh)
for (idx = 0; idx < width; idx += bw)
read_tx_size_vartx(cm, xd, mbmi, xd->counts, max_tx_size,
@@ -1877,7 +1885,7 @@
for (idx = 0; idx < width; ++idx)
mbmi->inter_tx_size[idy >> 1][idx >> 1] = mbmi->tx_size;
}
-
+ mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, xd);
}
#else
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 4b47dd4..3b66fa7 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -1132,7 +1132,11 @@
#endif
aom_writer *w) {
const int is_inter = is_inter_block(mbmi);
+#if CONFIG_VAR_TX
+ const TX_SIZE tx_size = is_inter ? mbmi->min_tx_size : mbmi->tx_size;
+#else
const TX_SIZE tx_size = mbmi->tx_size;
+#endif
if (!FIXED_TX_TYPE) {
#if CONFIG_EXT_TX
const BLOCK_SIZE bsize = mbmi->sb_type;
@@ -1187,13 +1191,6 @@
&ext_tx_encodings[mbmi->tx_type]);
#endif
}
- } else {
- if (!mbmi->skip) {
-#if CONFIG_SUPERTX
- if (!supertx_enabled)
-#endif // CONFIG_SUPERTX
- assert(mbmi->tx_type == DCT_DCT);
- }
}
#endif // CONFIG_EXT_TX
}
@@ -1927,6 +1924,7 @@
(*tok)++;
}
}
+
#if CONFIG_RD_DEBUG
for (plane = 0; plane < MAX_MB_PLANE; ++plane) {
if (m->mbmi.rd_stats.txb_coeff_cost[plane] != txb_coeff_cost[plane]) {
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index f4ca136..baf61e9 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -1295,6 +1295,9 @@
for (i = 0; i < 1; ++i)
memcpy(x->blk_skip[i], ctx->blk_skip[i],
sizeof(uint8_t) * ctx->num_4x4_blk);
+
+ if (!is_inter_block(mbmi) || mbmi->skip)
+ mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
#endif // CONFIG_VAR_TX
#if CONFIG_VAR_TX
@@ -1497,6 +1500,7 @@
for (i = 0; i < 1; ++i)
memcpy(ctx->blk_skip[i], x->blk_skip[i],
sizeof(uint8_t) * ctx->num_4x4_blk);
+ ctx->mic.mbmi.min_tx_size = get_min_tx_size(supertx_size);
#endif // CONFIG_VAR_TX
ctx->mic.mbmi.tx_size = supertx_size;
ctx->skip = x->skip;
@@ -5317,6 +5321,9 @@
}
}
#endif // CONFIG_PALETTE
+#if CONFIG_VAR_TX
+ mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
+#endif
av1_tokenize_sb(cpi, td, t, dry_run, AOMMAX(bsize, BLOCK_8X8), rate);
} else {
int ref;
@@ -5344,6 +5351,7 @@
av1_encode_sb((AV1_COMMON *)cm, x, AOMMAX(bsize, BLOCK_8X8));
#if CONFIG_VAR_TX
+ if (mbmi->skip) mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
#if CONFIG_EXT_TX && CONFIG_RECT_TX
if (is_rect_tx(mbmi->tx_size))
av1_tokenize_sb(cpi, td, t, dry_run, AOMMAX(bsize, BLOCK_8X8), rate);
@@ -5357,29 +5365,35 @@
}
if (!dry_run) {
+ const int is_inter = is_inter_block(mbmi);
+#if CONFIG_VAR_TX
+ TX_SIZE tx_size =
+ is_inter && !mbmi->skip ? mbmi->min_tx_size : mbmi->tx_size;
+#else
+ TX_SIZE tx_size = mbmi->tx_size;
+#endif
if (cm->tx_mode == TX_MODE_SELECT && mbmi->sb_type >= BLOCK_8X8 &&
!(is_inter_block(mbmi) && (mbmi->skip || seg_skip))) {
- const int is_inter = is_inter_block(mbmi);
const int tx_size_ctx = get_tx_size_context(xd);
const int tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize]
: intra_tx_size_cat_lookup[bsize];
- const TX_SIZE coded_tx_size = txsize_sqr_up_map[mbmi->tx_size];
+ const TX_SIZE coded_tx_size = txsize_sqr_up_map[tx_size];
const int depth = tx_size_to_depth(coded_tx_size);
#if CONFIG_EXT_TX && CONFIG_RECT_TX
- assert(IMPLIES(is_rect_tx(mbmi->tx_size), is_rect_tx_allowed(xd, mbmi)));
+ assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed(xd, mbmi)));
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
#if CONFIG_VAR_TX
#if CONFIG_EXT_TX && CONFIG_RECT_TX
if (is_rect_tx_allowed(xd, mbmi)) {
- td->counts->rect_tx[tx_size_cat][is_rect_tx(mbmi->tx_size)]++;
+ td->counts->rect_tx[tx_size_cat][is_rect_tx(tx_size)]++;
}
- if (!is_rect_tx_allowed(xd, mbmi) || !is_rect_tx(mbmi->tx_size)) {
+ if (!is_rect_tx_allowed(xd, mbmi) || !is_rect_tx(tx_size)) {
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
if (is_inter) {
tx_partition_count_update(cm, x, bsize, mi_row, mi_col, td->counts);
} else {
++td->counts->tx_size[tx_size_cat][tx_size_ctx][depth];
- if (mbmi->tx_size != max_txsize_lookup[bsize]) ++x->txb_split_count;
+ if (tx_size != max_txsize_lookup[bsize]) ++x->txb_split_count;
}
#if CONFIG_EXT_TX && CONFIG_RECT_TX
}
@@ -5390,57 +5404,57 @@
#endif
} else {
int i, j;
- TX_SIZE tx_size;
+ TX_SIZE intra_tx_size;
// The new intra coding scheme requires no change of transform size
if (is_inter_block(&mi->mbmi)) {
if (xd->lossless[mbmi->segment_id]) {
- tx_size = TX_4X4;
+ intra_tx_size = TX_4X4;
} else {
- tx_size = tx_size_from_tx_mode(bsize, cm->tx_mode, 1);
+ intra_tx_size = tx_size_from_tx_mode(bsize, cm->tx_mode, 1);
}
#if CONFIG_EXT_TX && CONFIG_RECT_TX
++td->counts->tx_size_implied[max_txsize_lookup[bsize]]
- [txsize_sqr_up_map[mbmi->tx_size]];
+ [txsize_sqr_up_map[tx_size]];
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
} else {
- tx_size = (bsize >= BLOCK_8X8) ? mbmi->tx_size : TX_4X4;
+ intra_tx_size = (bsize >= BLOCK_8X8) ? tx_size : TX_4X4;
}
for (j = 0; j < mi_height; j++)
for (i = 0; i < mi_width; i++)
if (mi_col + i < cm->mi_cols && mi_row + j < cm->mi_rows)
- mi_8x8[mis * j + i]->mbmi.tx_size = tx_size;
+ mi_8x8[mis * j + i]->mbmi.tx_size = intra_tx_size;
#if CONFIG_VAR_TX
- if (mbmi->tx_size != max_txsize_lookup[bsize]) ++x->txb_split_count;
+ mbmi->min_tx_size = get_min_tx_size(intra_tx_size);
+ if (intra_tx_size != max_txsize_lookup[bsize]) ++x->txb_split_count;
#endif
}
- ++td->counts->tx_size_totals[txsize_sqr_map[mbmi->tx_size]];
+ ++td->counts->tx_size_totals[txsize_sqr_map[tx_size]];
++td->counts
->tx_size_totals[txsize_sqr_map[get_uv_tx_size(mbmi, &xd->plane[1])]];
#if CONFIG_EXT_TX
- if (get_ext_tx_types(mbmi->tx_size, bsize, is_inter_block(mbmi)) > 1 &&
+ if (get_ext_tx_types(tx_size, bsize, is_inter_block(mbmi)) > 1 &&
cm->base_qindex > 0 && !mbmi->skip &&
!segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
- int eset = get_ext_tx_set(mbmi->tx_size, bsize, is_inter_block(mbmi));
+ int eset = get_ext_tx_set(tx_size, bsize, is_inter_block(mbmi));
if (eset > 0) {
if (is_inter_block(mbmi)) {
- ++td->counts->inter_ext_tx[eset][txsize_sqr_map[mbmi->tx_size]]
+ ++td->counts->inter_ext_tx[eset][txsize_sqr_map[tx_size]]
[mbmi->tx_type];
} else {
- ++td->counts->intra_ext_tx[eset][mbmi->tx_size][mbmi->mode]
- [mbmi->tx_type];
+ ++td->counts->intra_ext_tx[eset][tx_size][mbmi->mode][mbmi->tx_type];
}
}
}
#else
- if (mbmi->tx_size < TX_32X32 && cm->base_qindex > 0 && !mbmi->skip &&
+ if (tx_size < TX_32X32 && cm->base_qindex > 0 && !mbmi->skip &&
!segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
if (is_inter_block(mbmi)) {
- ++td->counts->inter_ext_tx[mbmi->tx_size][mbmi->tx_type];
+ ++td->counts->inter_ext_tx[tx_size][mbmi->tx_type];
} else {
- ++td->counts->intra_ext_tx[mbmi->tx_size]
+ ++td->counts->intra_ext_tx[tx_size]
[intra_mode_to_tx_type_context[mbmi->mode]]
[mbmi->tx_type];
}
@@ -5461,7 +5475,7 @@
if (dry_run) tx_partition_set_contexts(cm, xd, bsize, mi_row, mi_col);
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
} else {
- TX_SIZE tx_size;
+ TX_SIZE tx_size = mbmi->tx_size;
// The new intra coding scheme requires no change of transform size
if (is_inter_block(mbmi))
#if CONFIG_EXT_TX && CONFIG_RECT_TX
@@ -5477,7 +5491,7 @@
max_txsize_lookup[bsize]);
#endif
else
- tx_size = (bsize >= BLOCK_8X8) ? mbmi->tx_size : TX_4X4;
+ tx_size = (bsize >= BLOCK_8X8) ? tx_size : TX_4X4;
mbmi->tx_size = tx_size;
set_txfm_ctxs(tx_size, xd->n8_w, xd->n8_h, xd);
}
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 8334842..7daa992 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -1581,7 +1581,9 @@
*sse = INT64_MAX;
mbmi->tx_size = tx_size_from_tx_mode(bs, cm->tx_mode, is_inter);
-
+#if CONFIG_VAR_TX
+ mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
+#endif
#if CONFIG_EXT_TX
ext_tx_set = get_ext_tx_set(mbmi->tx_size, bs, is_inter);
#endif // CONFIG_EXT_TX
@@ -1725,6 +1727,9 @@
mbmi->tx_size = TX_4X4;
mbmi->tx_type = DCT_DCT;
+#if CONFIG_VAR_TX
+ mbmi->min_tx_size = get_min_tx_size(TX_4X4);
+#endif
txfm_rd_in_plane(x, cpi, rate, distortion, skip, sse, ref_best_rd, 0, bs,
mbmi->tx_size, cpi->sf.use_fast_coef_costing);
@@ -1783,6 +1788,10 @@
mbmi->tx_size = best_tx;
mbmi->tx_type = best_tx_type;
+#if CONFIG_VAR_TX
+ mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
+#endif
+
#if !CONFIG_EXT_TX
if (mbmi->tx_size >= TX_32X32) assert(mbmi->tx_type == DCT_DCT);
#endif
@@ -3434,8 +3443,12 @@
int s0 = av1_cost_bit(skip_prob, 0);
int s1 = av1_cost_bit(skip_prob, 1);
int64_t rd;
+ int row, col;
+ const int max_blocks_high = max_block_high(xd, bsize, 0);
+ const int max_blocks_wide = max_block_wide(xd, bsize, 0);
mbmi->tx_type = tx_type;
+ mbmi->min_tx_size = TX_SIZES_ALL;
inter_block_yrd(cpi, x, rd_stats, bsize, ref_best_rd);
#if CONFIG_EXT_TX && CONFIG_RECT_TX
if (is_rect_tx_allowed(xd, mbmi)) {
@@ -3492,31 +3505,32 @@
if (rd_stats->rate == INT_MAX) return INT64_MAX;
+ for (row = 0; row < max_blocks_high / 2; ++row)
+ for (col = 0; col < max_blocks_wide / 2; ++col)
+ mbmi->min_tx_size = AOMMIN(
+ mbmi->min_tx_size, get_min_tx_size(mbmi->inter_tx_size[row][col]));
+
#if CONFIG_EXT_TX
- if (get_ext_tx_types(mbmi->tx_size, bsize, is_inter) > 1 &&
+ if (get_ext_tx_types(mbmi->min_tx_size, bsize, is_inter) > 1 &&
!xd->lossless[xd->mi[0]->mbmi.segment_id]) {
- int ext_tx_set = get_ext_tx_set(mbmi->tx_size, bsize, is_inter);
+ int ext_tx_set = get_ext_tx_set(mbmi->min_tx_size, bsize, is_inter);
if (is_inter) {
if (ext_tx_set > 0)
rd_stats->rate +=
- cpi->inter_tx_type_costs[ext_tx_set][txsize_sqr_map[mbmi->tx_size]]
+ cpi->inter_tx_type_costs[ext_tx_set]
+ [txsize_sqr_map[mbmi->min_tx_size]]
[mbmi->tx_type];
} else {
if (ext_tx_set > 0 && ALLOW_INTRA_EXT_TX)
- rd_stats->rate += cpi->intra_tx_type_costs[ext_tx_set][mbmi->tx_size]
- [mbmi->mode][mbmi->tx_type];
+ rd_stats->rate +=
+ cpi->intra_tx_type_costs[ext_tx_set][mbmi->min_tx_size][mbmi->mode]
+ [mbmi->tx_type];
}
}
#else // CONFIG_EXT_TX
- if (mbmi->tx_size < TX_32X32 && !xd->lossless[xd->mi[0]->mbmi.segment_id]) {
- if (is_inter)
- rd_stats->rate += cpi->inter_tx_type_costs[mbmi->tx_size][mbmi->tx_type];
- else
- rd_stats->rate +=
- cpi->intra_tx_type_costs[mbmi->tx_size]
- [intra_mode_to_tx_type_context[mbmi->mode]]
- [mbmi->tx_type];
- }
+ if (mbmi->min_tx_size < TX_32X32 && !xd->lossless[xd->mi[0]->mbmi.segment_id])
+ rd_stats->rate +=
+ cpi->inter_tx_type_costs[mbmi->min_tx_size][mbmi->tx_type];
#endif // CONFIG_EXT_TX
if (rd_stats->skip)
@@ -3543,6 +3557,7 @@
const int is_inter = is_inter_block(mbmi);
TX_SIZE best_tx_size[MAX_MIB_SIZE][MAX_MIB_SIZE];
TX_SIZE best_tx = max_txsize_lookup[bsize];
+ TX_SIZE best_min_tx_size = TX_SIZES_ALL;
uint8_t best_blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE * 4];
const int n4 = 1 << (num_pels_log2_lookup[bsize] - 4);
int idx, idy;
@@ -3576,7 +3591,6 @@
if (!ext_tx_used_intra[ext_tx_set][tx_type]) continue;
}
#else // CONFIG_EXT_TX
- if (max_tx_size >= TX_32X32 && tx_type != DCT_DCT) continue;
if (is_inter && cpi->sf.tx_type_search.prune_mode > NO_PRUNE &&
!do_tx_type_search(tx_type, prune))
continue;
@@ -3585,6 +3599,9 @@
tx_type != get_default_tx_type(0, xd, 0, max_tx_size))
continue;
+ if (xd->lossless[mbmi->segment_id])
+ if (tx_type != DCT_DCT) continue;
+
rd = select_tx_size_fix_type(cpi, x, &this_rd_stats, bsize, ref_best_rd,
tx_type);
@@ -3593,6 +3610,7 @@
*rd_stats = this_rd_stats;
best_tx_type = mbmi->tx_type;
best_tx = mbmi->tx_size;
+ best_min_tx_size = mbmi->min_tx_size;
memcpy(best_blk_skip, x->blk_skip[0], sizeof(best_blk_skip[0]) * n4);
for (idy = 0; idy < xd->n8_h; ++idy)
for (idx = 0; idx < xd->n8_w; ++idx)
@@ -3605,6 +3623,7 @@
for (idx = 0; idx < xd->n8_w; ++idx)
mbmi->inter_tx_size[idy][idx] = best_tx_size[idy][idx];
mbmi->tx_size = best_tx;
+ mbmi->min_tx_size = best_min_tx_size;
#if CONFIG_RD_DEBUG
// record plane y's transform block coefficient cost
mbmi->rd_stats = *rd_stats;
@@ -9593,6 +9612,8 @@
for (i = 0; i < MAX_MB_PLANE; ++i)
memcpy(ctx->blk_skip[i], x->blk_skip[i],
sizeof(uint8_t) * ctx->num_4x4_blk);
+
+ best_mbmode.min_tx_size = mbmi->min_tx_size;
#endif
rd_cost->rate += (rate_y + rate_uv - best_rate_y - best_rate_uv);
rd_cost->dist = dist_y + dist_uv;
@@ -10409,6 +10430,7 @@
#if CONFIG_VAR_TX
mbmi->inter_tx_size[0][0] = mbmi->tx_size;
+ mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
#endif
if (ref_frame == INTRA_FRAME) {
