Make RECT_TX(>=8x8) work with VAR_TX
Bitstream syntax:
For a rectangular inter block, 'rect_tx' flag is sent to indicate if
the biggest rect tx is used. If no, continue to decode regular
recursive tx partition.
Change-Id: I127e35cc619b65acb5e9a0717f399cdcdb73fbf0
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 3bd5adc..7c33aea 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -1121,9 +1121,25 @@
const int width = num_4x4_blocks_wide_lookup[bsize];
const int height = num_4x4_blocks_high_lookup[bsize];
int idx, idy;
- for (idy = 0; idy < height; idy += bs)
- for (idx = 0; idx < width; idx += bs)
- write_tx_size_vartx(cm, xd, mbmi, max_tx_size, idy, idx, w);
+
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+ if (is_rect_tx_allowed(mbmi)) {
+ int tx_size_cat = inter_tx_size_cat_lookup[bsize];
+
+ aom_write(w, is_rect_tx(mbmi->tx_size),
+ cm->fc->rect_tx_prob[tx_size_cat]);
+ }
+
+ if (is_rect_tx(mbmi->tx_size)) {
+ set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, xd);
+ } else {
+#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
+ for (idy = 0; idy < height; idy += bs)
+ for (idx = 0; idx < width; idx += bs)
+ write_tx_size_vartx(cm, xd, mbmi, max_tx_size, idy, idx, w);
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+ }
+#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
} else {
set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, xd);
write_selected_tx_size(cm, xd, w);
@@ -1650,7 +1666,7 @@
TX_SIZE tx_size =
plane ? get_uv_tx_size(mbmi, &xd->plane[plane]) : mbmi->tx_size;
- if (is_inter_block(mbmi) && tx_size < TX_SIZES) {
+ if (is_inter_block(mbmi) && !is_rect_tx(tx_size)) {
#else
if (is_inter_block(mbmi)) {
#endif
@@ -3271,6 +3287,13 @@
#if CONFIG_VAR_TX
update_txfm_partition_probs(cm, header_bc, counts);
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+ if (cm->tx_mode == TX_MODE_SELECT) {
+ for (i = 1; i < TX_SIZES - 1; ++i)
+ av1_cond_prob_diff_update(header_bc, &fc->rect_tx_prob[i],
+ counts->rect_tx[i]);
+ }
+#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
#endif
update_skip_probs(cm, header_bc, counts);
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index f172fbb..200b02c 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -5031,7 +5031,7 @@
av1_encode_sb(x, AOMMAX(bsize, BLOCK_8X8));
#if CONFIG_VAR_TX
#if CONFIG_EXT_TX && CONFIG_RECT_TX
- if (mbmi->tx_size >= TX_SIZES)
+ if (is_rect_tx(mbmi->tx_size))
av1_tokenize_sb(cpi, td, t, !output_enabled, AOMMAX(bsize, BLOCK_8X8));
else
#endif
@@ -5054,8 +5054,17 @@
assert(IMPLIES(is_rect_tx(mbmi->tx_size), is_rect_tx_allowed(mbmi)));
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
#if CONFIG_VAR_TX
- if (is_inter)
- tx_partition_count_update(cm, xd, bsize, mi_row, mi_col, td->counts);
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+ if (is_rect_tx_allowed(mbmi)) {
+ td->counts->rect_tx[tx_size_cat][is_rect_tx(mbmi->tx_size)]++;
+ }
+ if (!is_rect_tx_allowed(mbmi) || !is_rect_tx(mbmi->tx_size)) {
+#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
+ if (is_inter)
+ tx_partition_count_update(cm, xd, bsize, mi_row, mi_col, td->counts);
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+ }
+#endif
#endif
++td->counts->tx_size[tx_size_cat][tx_size_ctx][coded_tx_size];
} else {
@@ -5118,6 +5127,11 @@
is_inter_block(mbmi) && !(mbmi->skip || seg_skip)) {
if (!output_enabled)
tx_partition_set_contexts(cm, xd, bsize, mi_row, mi_col);
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+ if (is_rect_tx(mbmi->tx_size)) {
+ set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, xd);
+ }
+#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
} else {
TX_SIZE tx_size;
// The new intra coding scheme requires no change of transform size
diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c
index 5a21ebd..718bddb 100644
--- a/av1/encoder/encodemb.c
+++ b/av1/encoder/encodemb.c
@@ -791,7 +791,7 @@
#endif
#if CONFIG_VAR_TX
- // Assert not magic number (uninitialised).
+ // Assert not magic number (uninitialized).
assert(x->blk_skip[plane][(blk_row << bwl) + blk_col] != 234);
if (x->blk_skip[plane][(blk_row << bwl) + blk_col] == 0) {
@@ -976,9 +976,6 @@
int idx, idy;
int block = 0;
int step = num_4x4_blocks_txsize_lookup[max_tx_size];
-#if CONFIG_EXT_TX && CONFIG_RECT_TX
- const TX_SIZE tx_size = plane ? get_uv_tx_size(mbmi, pd) : mbmi->tx_size;
-#endif
av1_get_entropy_contexts(bsize, TX_4X4, pd, ctx.ta[plane], ctx.tl[plane]);
#else
const struct macroblockd_plane *const pd = &xd->plane[plane];
@@ -991,7 +988,7 @@
#if CONFIG_VAR_TX
#if CONFIG_EXT_TX && CONFIG_RECT_TX
- if (tx_size >= TX_SIZES) {
+ if (is_rect_tx(mbmi->tx_size)) {
av1_foreach_transformed_block_in_plane(xd, bsize, plane, encode_block,
&arg);
} else {
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 0178721..ec80ee0 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -2814,6 +2814,61 @@
return best_rd;
}
+// Return value 0: early termination triggered, no valid rd cost available;
+// 1: rd cost values are valid.
+static int super_block_uvrd(const AV1_COMP *cpi, MACROBLOCK *x, int *rate,
+ int64_t *distortion, int *skippable, int64_t *sse,
+ BLOCK_SIZE bsize, int64_t ref_best_rd) {
+ MACROBLOCKD *const xd = &x->e_mbd;
+ MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
+ const TX_SIZE uv_tx_size = get_uv_tx_size(mbmi, &xd->plane[1]);
+ int plane;
+ int pnrate = 0, pnskip = 1;
+ int64_t pndist = 0, pnsse = 0;
+ int is_cost_valid = 1;
+
+ if (ref_best_rd < 0) is_cost_valid = 0;
+
+ if (is_inter_block(mbmi) && is_cost_valid) {
+ int plane;
+ for (plane = 1; plane < MAX_MB_PLANE; ++plane)
+ av1_subtract_plane(x, bsize, plane);
+ }
+
+ *rate = 0;
+ *distortion = 0;
+ *sse = 0;
+ *skippable = 1;
+
+ for (plane = 1; plane < MAX_MB_PLANE; ++plane) {
+ txfm_rd_in_plane(x, cpi, &pnrate, &pndist, &pnskip, &pnsse, ref_best_rd,
+ plane, bsize, uv_tx_size, cpi->sf.use_fast_coef_costing);
+ if (pnrate == INT_MAX) {
+ is_cost_valid = 0;
+ break;
+ }
+ *rate += pnrate;
+ *distortion += pndist;
+ *sse += pnsse;
+ *skippable &= pnskip;
+ if (RDCOST(x->rdmult, x->rddiv, *rate, *distortion) > ref_best_rd &&
+ RDCOST(x->rdmult, x->rddiv, 0, *sse) > ref_best_rd) {
+ is_cost_valid = 0;
+ break;
+ }
+ }
+
+ if (!is_cost_valid) {
+ // reset cost value
+ *rate = INT_MAX;
+ *distortion = INT64_MAX;
+ *sse = INT64_MAX;
+ *skippable = 0;
+ }
+
+ return is_cost_valid;
+}
+
#if CONFIG_VAR_TX
void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
int blk_row, int blk_col, int plane, int block,
@@ -3196,6 +3251,61 @@
mbmi->tx_type = tx_type;
inter_block_yrd(cpi, x, rate, dist, skippable, sse, bsize, ref_best_rd);
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+ if (is_rect_tx_allowed(mbmi)) {
+ int rate_rect_tx, skippable_rect_tx = 0;
+ int64_t dist_rect_tx, sse_rect_tx, rd, rd_rect_tx;
+ int tx_size_cat = inter_tx_size_cat_lookup[bsize];
+ TX_SIZE tx_size = max_txsize_rect_lookup[bsize];
+ TX_SIZE var_tx_size = mbmi->tx_size;
+
+ txfm_rd_in_plane(x, cpi, &rate_rect_tx, &dist_rect_tx, &skippable_rect_tx,
+ &sse_rect_tx, ref_best_rd, 0, bsize, tx_size,
+ cpi->sf.use_fast_coef_costing);
+
+ if (*rate != INT_MAX) {
+ *rate += av1_cost_bit(cm->fc->rect_tx_prob[tx_size_cat], 0);
+ if (*skippable) {
+ rd = RDCOST(x->rdmult, x->rddiv, s1, *sse);
+ } else {
+ rd = RDCOST(x->rdmult, x->rddiv, *rate + s0, *dist);
+ if (is_inter && !xd->lossless[xd->mi[0]->mbmi.segment_id] &&
+ !(*skippable))
+ rd = AOMMIN(rd, RDCOST(x->rdmult, x->rddiv, s1, *sse));
+ }
+ } else {
+ rd = INT64_MAX;
+ }
+
+ if (rate_rect_tx != INT_MAX) {
+ rate_rect_tx += av1_cost_bit(cm->fc->rect_tx_prob[tx_size_cat], 1);
+ if (skippable_rect_tx) {
+ rd_rect_tx = RDCOST(x->rdmult, x->rddiv, s1, sse_rect_tx);
+ } else {
+ rd_rect_tx =
+ RDCOST(x->rdmult, x->rddiv, rate_rect_tx + s0, dist_rect_tx);
+ if (is_inter && !xd->lossless[xd->mi[0]->mbmi.segment_id] &&
+ !(skippable_rect_tx))
+ rd_rect_tx =
+ AOMMIN(rd_rect_tx, RDCOST(x->rdmult, x->rddiv, s1, sse_rect_tx));
+ }
+ } else {
+ rd_rect_tx = INT64_MAX;
+ }
+
+ if (rd_rect_tx < rd) {
+ *rate = rate_rect_tx;
+ *dist = dist_rect_tx;
+ *sse = sse_rect_tx;
+ *skippable = skippable_rect_tx;
+ if (!xd->lossless[mbmi->segment_id]) x->blk_skip[0][0] = *skippable;
+ mbmi->tx_size = tx_size;
+ mbmi->inter_tx_size[0][0] = mbmi->tx_size;
+ } else {
+ mbmi->tx_size = var_tx_size;
+ }
+ }
+#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
if (*rate == INT_MAX) return INT64_MAX;
@@ -3409,17 +3519,24 @@
if (ref_best_rd < 0) is_cost_valid = 0;
+ *rate = 0;
+ *distortion = 0;
+ *sse = 0;
+ *skippable = 1;
+
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+ if (is_rect_tx(mbmi->tx_size)) {
+ return super_block_uvrd(cpi, x, rate, distortion, skippable, sse, bsize,
+ ref_best_rd);
+ }
+#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
+
if (is_inter_block(mbmi) && is_cost_valid) {
int plane;
for (plane = 1; plane < MAX_MB_PLANE; ++plane)
av1_subtract_plane(x, bsize, plane);
}
- *rate = 0;
- *distortion = 0;
- *sse = 0;
- *skippable = 1;
-
for (plane = 1; plane < MAX_MB_PLANE; ++plane) {
const struct macroblockd_plane *const pd = &xd->plane[plane];
const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd);
@@ -3477,61 +3594,6 @@
}
#endif // CONFIG_VAR_TX
-// Return value 0: early termination triggered, no valid rd cost available;
-// 1: rd cost values are valid.
-static int super_block_uvrd(const AV1_COMP *cpi, MACROBLOCK *x, int *rate,
- int64_t *distortion, int *skippable, int64_t *sse,
- BLOCK_SIZE bsize, int64_t ref_best_rd) {
- MACROBLOCKD *const xd = &x->e_mbd;
- MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
- const TX_SIZE uv_tx_size = get_uv_tx_size(mbmi, &xd->plane[1]);
- int plane;
- int pnrate = 0, pnskip = 1;
- int64_t pndist = 0, pnsse = 0;
- int is_cost_valid = 1;
-
- if (ref_best_rd < 0) is_cost_valid = 0;
-
- if (is_inter_block(mbmi) && is_cost_valid) {
- int plane;
- for (plane = 1; plane < MAX_MB_PLANE; ++plane)
- av1_subtract_plane(x, bsize, plane);
- }
-
- *rate = 0;
- *distortion = 0;
- *sse = 0;
- *skippable = 1;
-
- for (plane = 1; plane < MAX_MB_PLANE; ++plane) {
- txfm_rd_in_plane(x, cpi, &pnrate, &pndist, &pnskip, &pnsse, ref_best_rd,
- plane, bsize, uv_tx_size, cpi->sf.use_fast_coef_costing);
- if (pnrate == INT_MAX) {
- is_cost_valid = 0;
- break;
- }
- *rate += pnrate;
- *distortion += pndist;
- *sse += pnsse;
- *skippable &= pnskip;
- if (RDCOST(x->rdmult, x->rddiv, *rate, *distortion) > ref_best_rd &&
- RDCOST(x->rdmult, x->rddiv, 0, *sse) > ref_best_rd) {
- is_cost_valid = 0;
- break;
- }
- }
-
- if (!is_cost_valid) {
- // reset cost value
- *rate = INT_MAX;
- *distortion = INT64_MAX;
- *sse = INT64_MAX;
- *skippable = 0;
- }
-
- return is_cost_valid;
-}
-
static void rd_pick_palette_intra_sbuv(
AV1_COMP *cpi, MACROBLOCK *x, int dc_mode_cost,
PALETTE_MODE_INFO *palette_mode_info, uint8_t *best_palette_color_map,
