rect_tx_ext: work with var_tx
Change-Id: Ie2c34490dc50cb242bcd701308e6b55243883b15
diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index 4ba0def..8ea6462 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -1085,8 +1085,10 @@
return is_rect_tx_allowed_bsize(mbmi->sb_type) &&
!xd->lossless[mbmi->segment_id];
}
+#endif // CONFIG_RECT_TX
+#endif // CONFIG_EXT_TX
-#if CONFIG_RECT_TX_EXT
+#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
static INLINE int is_quarter_tx_allowed_bsize(BLOCK_SIZE bsize) {
static const char LUT_QTTX[BLOCK_SIZES_ALL] = {
#if CONFIG_CHROMA_2X2 || CONFIG_CHROMA_SUB8X8
@@ -1127,9 +1129,7 @@
return is_quarter_tx_allowed_bsize(mbmi->sb_type) && is_inter &&
!xd->lossless[mbmi->segment_id];
}
-#endif // CONFIG_RECT_TX_EXT
-#endif // CONFIG_RECT_TX
-#endif // CONFIG_EXT_TX
+#endif
static INLINE TX_SIZE tx_size_from_tx_mode(BLOCK_SIZE bsize, TX_MODE tx_mode,
int is_inter) {
diff --git a/av1/common/common_data.h b/av1/common/common_data.h
index 3aa437d..f49c733 100644
--- a/av1/common/common_data.h
+++ b/av1/common/common_data.h
@@ -655,13 +655,20 @@
TX_32X32, TX_32X32, TX_32X32,
#endif // CONFIG_EXT_PARTITION
#endif // CONFIG_TX64X64
+#if CONFIG_RECT_TX_EXT
+ // 4x16, 16x4, 8x32
+ TX_4X16, TX_16X4, TX_8X32,
+ // 32x8
+ TX_32X8
+#else
// 4x16, 16x4, 8x32
TX_4X8, TX_8X4, TX_8X16,
// 32x8
TX_16X8
+#endif
};
-#if CONFIG_EXT_TX && CONFIG_RECT_TX_EXT
+#if CONFIG_RECT_TX_EXT
static const TX_SIZE quarter_txsize_lookup[BLOCK_SIZES_ALL] = {
#if CONFIG_CHROMA_2X2 || CONFIG_CHROMA_SUB8X8
// 2X2, 2X4, 4X2,
@@ -686,7 +693,7 @@
// 32x8
TX_32X8
};
-#endif // CONFIG_EXT_TX && CONFIG_RECT_TX_EXT
+#endif
#else
#define max_txsize_rect_lookup max_txsize_lookup
#endif // CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
diff --git a/av1/common/entropy.h b/av1/common/entropy.h
index 51803db..190b792 100644
--- a/av1/common/entropy.h
+++ b/av1/common/entropy.h
@@ -315,7 +315,7 @@
*(const uint64_t *)(l + 16) | *(const uint64_t *)(l + 24));
break;
#endif // CONFIG_TX64X64
-#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
case TX_4X16:
above_ec = !!*(const uint16_t *)a;
left_ec = !!*(const uint64_t *)l;
@@ -332,7 +332,7 @@
above_ec = !!(*(const uint64_t *)a | *(const uint64_t *)(a + 8));
left_ec = !!*(const uint32_t *)l;
break;
-#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#endif
default: assert(0 && "Invalid transform size."); break;
}
return combine_entropy_contexts(above_ec, left_ec);
@@ -385,7 +385,7 @@
left_ec = !!(*(const uint64_t *)l | *(const uint64_t *)(l + 8));
break;
#endif // CONFIG_TX64X64
-#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
case TX_4X16:
above_ec = a[0] != 0;
left_ec = !!*(const uint32_t *)l;
@@ -402,7 +402,7 @@
above_ec = !!*(const uint64_t *)a;
left_ec = !!*(const uint16_t *)l;
break;
-#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#endif
default: assert(0 && "Invalid transform size."); break;
}
return combine_entropy_contexts(above_ec, left_ec);
diff --git a/av1/common/entropymode.c b/av1/common/entropymode.c
index c4d75c5..9faa03e 100644
--- a/av1/common/entropymode.c
+++ b/av1/common/entropymode.c
@@ -1737,9 +1737,9 @@
#endif // CONFIG_TX64X64
};
-#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
static const aom_prob default_quarter_tx_size_prob = 192;
-#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#endif
#if CONFIG_LOOP_RESTORATION
const aom_tree_index
@@ -4952,9 +4952,9 @@
av1_copy(fc->comp_inter_mode_prob, default_comp_inter_mode_p);
#endif // CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF
av1_copy(fc->tx_size_probs, default_tx_size_prob);
-#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
fc->quarter_tx_size_prob = default_quarter_tx_size_prob;
-#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#endif
#if CONFIG_VAR_TX
av1_copy(fc->txfm_partition_prob, default_txfm_partition_probs);
#if CONFIG_NEW_MULTISYMBOL
@@ -5231,10 +5231,10 @@
aom_tree_merge_probs(av1_tx_size_tree[i], pre_fc->tx_size_probs[i][j],
counts->tx_size[i][j], fc->tx_size_probs[i][j]);
}
-#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
fc->quarter_tx_size_prob = av1_mode_mv_merge_probs(
pre_fc->quarter_tx_size_prob, counts->quarter_tx_size);
-#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#endif
}
#if CONFIG_VAR_TX
diff --git a/av1/common/entropymode.h b/av1/common/entropymode.h
index 83beb6a..3253247 100644
--- a/av1/common/entropymode.h
+++ b/av1/common/entropymode.h
@@ -277,9 +277,9 @@
aom_prob comp_inter_mode_prob[COMP_INTER_MODE_CONTEXTS];
#endif // CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF
aom_prob tx_size_probs[MAX_TX_DEPTH][TX_SIZE_CONTEXTS][MAX_TX_DEPTH];
-#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
aom_prob quarter_tx_size_prob;
-#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#endif
#if CONFIG_VAR_TX
aom_prob txfm_partition_prob[TXFM_PARTITION_CONTEXTS];
#if CONFIG_NEW_MULTISYMBOL
@@ -467,9 +467,9 @@
// belong into this structure.
unsigned int tx_size_totals[TX_SIZES];
unsigned int tx_size[MAX_TX_DEPTH][TX_SIZE_CONTEXTS][MAX_TX_DEPTH + 1];
-#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
unsigned int quarter_tx_size[2];
-#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#endif
#if CONFIG_VAR_TX
unsigned int txfm_partition[TXFM_PARTITION_CONTEXTS][2];
#endif
diff --git a/av1/common/idct.c b/av1/common/idct.c
index 09f01d3..7915b70 100644
--- a/av1/common/idct.c
+++ b/av1/common/idct.c
@@ -1623,7 +1623,7 @@
}
// These will be used by the masked-tx experiment in the future.
-#if CONFIG_RECT_TX && CONFIG_EXT_TX && CONFIG_RECT_TX_EXT
+#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
static void inv_txfm_add_4x16(const tran_low_t *input, uint8_t *dest,
int stride, const TxfmParam *txfm_param) {
#if CONFIG_LGT
@@ -1659,7 +1659,7 @@
av1_iht32x8_256_add(input, dest, stride, txfm_param);
#endif
}
-#endif // CONFIG_RECT_TX && CONFIG_EXT_TX && CONFIG_RECT_TX_EXT
+#endif
static void inv_txfm_add_8x16(const tran_low_t *input, uint8_t *dest,
int stride, const TxfmParam *txfm_param) {
@@ -2146,7 +2146,7 @@
#if CONFIG_CHROMA_2X2
case TX_2X2: inv_txfm_add_2x2(input, dest, stride, txfm_param); break;
#endif
-#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
case TX_32X8: inv_txfm_add_32x8(input, dest, stride, txfm_param); break;
case TX_8X32: inv_txfm_add_8x32(input, dest, stride, txfm_param); break;
case TX_16X4: inv_txfm_add_16x4(input, dest, stride, txfm_param); break;
diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c
index 21e9add..80ef8e8 100644
--- a/av1/decoder/decodeframe.c
+++ b/av1/decoder/decodeframe.c
@@ -723,17 +723,33 @@
pd->dst.stride, max_scan_line, eob);
*eob_total += eob;
} else {
+#if CONFIG_RECT_TX_EXT
+ int is_qttx = plane_tx_size == quarter_txsize_lookup[plane_bsize];
+ const TX_SIZE sub_txs = is_qttx ? plane_tx_size : sub_tx_size_map[tx_size];
+ if (is_qttx) assert(blk_row == 0 && blk_col == 0 && block == 0);
+#else
const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
+ assert(sub_txs < tx_size);
+#endif
const int bsl = tx_size_wide_unit[sub_txs];
int sub_step = tx_size_wide_unit[sub_txs] * tx_size_high_unit[sub_txs];
- assert(sub_txs < tx_size);
int i;
assert(bsl > 0);
for (i = 0; i < 4; ++i) {
+#if CONFIG_RECT_TX_EXT
+ int is_wide_tx = tx_size_wide_unit[sub_txs] > tx_size_high_unit[sub_txs];
+ const int offsetr =
+ is_qttx ? (is_wide_tx ? i * tx_size_high_unit[sub_txs] : 0)
+ : blk_row + ((i >> 1) * bsl);
+ const int offsetc =
+ is_qttx ? (is_wide_tx ? 0 : i * tx_size_wide_unit[sub_txs])
+ : blk_col + (i & 0x01) * bsl;
+#else
const int offsetr = blk_row + (i >> 1) * bsl;
const int offsetc = blk_col + (i & 0x01) * bsl;
+#endif
if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue;
@@ -4961,10 +4977,10 @@
}
#endif
-#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
if (cm->tx_mode == TX_MODE_SELECT)
av1_diff_update_prob(&r, &fc->quarter_tx_size_prob, ACCT_STR);
-#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#endif
#if CONFIG_LV_MAP
av1_read_txb_probs(fc, cm->tx_mode, &r);
diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c
index 0149688..7c85442 100644
--- a/av1/decoder/decodemv.c
+++ b/av1/decoder/decodemv.c
@@ -543,16 +543,22 @@
#if CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
if (coded_tx_size > max_txsize_lookup[bsize]) {
assert(coded_tx_size == max_txsize_lookup[bsize] + 1);
-#if CONFIG_EXT_TX && CONFIG_RECT_TX_EXT
+#if CONFIG_RECT_TX_EXT
if (is_quarter_tx_allowed(xd, &xd->mi[0]->mbmi, is_inter)) {
- int quarter_tx = aom_read(r, cm->fc->quarter_tx_size_prob, ACCT_STR);
- FRAME_COUNTS *counts = xd->counts;
+ int quarter_tx;
- if (counts) ++counts->quarter_tx_size[quarter_tx];
+ if (quarter_txsize_lookup[bsize] != max_txsize_lookup[bsize]) {
+ quarter_tx = aom_read(r, cm->fc->quarter_tx_size_prob, ACCT_STR);
+ FRAME_COUNTS *counts = xd->counts;
+
+ if (counts) ++counts->quarter_tx_size[quarter_tx];
+ } else {
+ quarter_tx = 1;
+ }
return quarter_tx ? quarter_txsize_lookup[bsize]
: max_txsize_rect_lookup[bsize];
}
-#endif // CONFIG_EXT_TX && CONFIG_RECT_TX_EXT
+#endif // CONFIG_RECT_TX_EXT
return max_txsize_rect_lookup[bsize];
}
@@ -2860,6 +2866,26 @@
for (idx = 0; idx < width; idx += bw)
read_tx_size_vartx(cm, xd, mbmi, xd->counts, max_tx_size,
height != width, idy, idx, r);
+#if CONFIG_RECT_TX_EXT
+ if (is_quarter_tx_allowed(xd, mbmi, inter_block) &&
+ mbmi->tx_size == max_tx_size) {
+ int quarter_tx;
+
+ if (quarter_txsize_lookup[bsize] != max_tx_size) {
+ quarter_tx = aom_read(r, cm->fc->quarter_tx_size_prob, ACCT_STR);
+ if (xd->counts) ++xd->counts->quarter_tx_size[quarter_tx];
+ } else {
+ quarter_tx = 1;
+ }
+ if (quarter_tx) {
+ mbmi->tx_size = quarter_txsize_lookup[bsize];
+ for (idy = 0; idy < tx_size_high_unit[max_tx_size] / 2; ++idy)
+ for (idx = 0; idx < tx_size_wide_unit[max_tx_size] / 2; ++idx)
+ mbmi->inter_tx_size[idy][idx] = mbmi->tx_size;
+ mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
+ }
+ }
+#endif
} else {
mbmi->tx_size = read_tx_size(cm, xd, inter_block, !mbmi->skip, r);
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 2331307..9241572 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -355,7 +355,12 @@
return;
}
+#if CONFIG_RECT_TX_EXT
+ if (tx_size == mbmi->inter_tx_size[tx_row][tx_col] ||
+ mbmi->tx_size == quarter_txsize_lookup[mbmi->sb_type]) {
+#else
if (tx_size == mbmi->inter_tx_size[tx_row][tx_col]) {
+#endif
#if CONFIG_NEW_MULTISYMBOL
aom_write_symbol(w, 0, ec_ctx->txfm_partition_cdf[ctx], 2);
#else
@@ -364,6 +369,7 @@
txfm_partition_update(xd->above_txfm_context + blk_col,
xd->left_txfm_context + blk_row, tx_size, tx_size);
+ // TODO(yuec): set correct txfm partition update for qttx
} else {
const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
const int bsl = tx_size_wide_unit[sub_txs];
@@ -427,11 +433,11 @@
aom_write_symbol(w, depth, ec_ctx->tx_size_cdf[tx_size_cat][tx_size_ctx],
tx_size_cat + 2);
-#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
if (is_quarter_tx_allowed(xd, mbmi, is_inter) && tx_size != coded_tx_size)
aom_write(w, tx_size == quarter_txsize_lookup[bsize],
cm->fc->quarter_tx_size_prob);
-#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#endif
}
}
@@ -963,15 +969,30 @@
token_stats->cost += tmp_token_stats.cost;
#endif
} else {
+#if CONFIG_RECT_TX_EXT
+ int is_qttx = plane_tx_size == quarter_txsize_lookup[plane_bsize];
+ const TX_SIZE sub_txs = is_qttx ? plane_tx_size : sub_tx_size_map[tx_size];
+#else
const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
+#endif
const int bsl = tx_size_wide_unit[sub_txs];
int i;
assert(bsl > 0);
for (i = 0; i < 4; ++i) {
+#if CONFIG_RECT_TX_EXT
+ int is_wide_tx = tx_size_wide_unit[sub_txs] > tx_size_high_unit[sub_txs];
+ const int offsetr =
+ is_qttx ? (is_wide_tx ? i * tx_size_high_unit[sub_txs] : 0)
+ : blk_row + (i >> 1) * bsl;
+ const int offsetc =
+ is_qttx ? (is_wide_tx ? 0 : i * tx_size_wide_unit[sub_txs])
+ : blk_col + (i & 0x01) * bsl;
+#else
const int offsetr = blk_row + (i >> 1) * bsl;
const int offsetc = blk_col + (i & 0x01) * bsl;
+#endif
const int step = tx_size_wide_unit[sub_txs] * tx_size_high_unit[sub_txs];
if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue;
@@ -1748,6 +1769,15 @@
for (idx = 0; idx < width; idx += bw)
write_tx_size_vartx(cm, xd, mbmi, max_tx_size, height != width, idy,
idx, w);
+#if CONFIG_RECT_TX_EXT
+ if (is_quarter_tx_allowed(xd, mbmi, is_inter_block(mbmi)) &&
+ quarter_txsize_lookup[bsize] != max_tx_size &&
+ (mbmi->tx_size == quarter_txsize_lookup[bsize] ||
+ mbmi->tx_size == max_tx_size)) {
+ aom_write(w, mbmi->tx_size != max_tx_size,
+ cm->fc->quarter_tx_size_prob);
+ }
+#endif
} else {
set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, skip, xd);
write_selected_tx_size(cm, xd, w);
@@ -4551,11 +4581,11 @@
#if CONFIG_LOOP_RESTORATION
encode_restoration(cm, header_bc);
#endif // CONFIG_LOOP_RESTORATION
-#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
if (cm->tx_mode == TX_MODE_SELECT)
av1_cond_prob_diff_update(header_bc, &cm->fc->quarter_tx_size_prob,
cm->counts.quarter_tx_size, probwt);
-#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#endif
#if CONFIG_LV_MAP
av1_write_txb_probs(cpi, header_bc);
#endif // CONFIG_LV_MAP
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index ab1218a..d13eb42 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -5474,10 +5474,15 @@
#endif // CONFIG_EXT_INTER
#if CONFIG_VAR_TX
+#if CONFIG_RECT_TX_EXT
+ if (cm->tx_mode == TX_MODE_SELECT && cpi->td.mb.txb_split_count == 0 &&
+ counts->quarter_tx_size[1] == 0)
+#else
if (cm->tx_mode == TX_MODE_SELECT && cpi->td.mb.txb_split_count == 0)
+#endif
cm->tx_mode = ALLOW_32X32 + CONFIG_TX64X64;
#else
-#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#if CONFIG_RECT_TX_EXT && CONFIG_EXT_TX
if (cm->tx_mode == TX_MODE_SELECT && counts->quarter_tx_size[1] == 0) {
#else
if (cm->tx_mode == TX_MODE_SELECT) {
@@ -5766,9 +5771,17 @@
if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
+#if CONFIG_RECT_TX_EXT
+ if (tx_size == plane_tx_size ||
+ mbmi->tx_size == quarter_txsize_lookup[mbmi->sb_type]) {
+#else
if (tx_size == plane_tx_size) {
+#endif
++counts->txfm_partition[ctx][0];
- mbmi->tx_size = tx_size;
+#if CONFIG_RECT_TX_EXT
+ if (tx_size == plane_tx_size)
+#endif
+ mbmi->tx_size = tx_size;
txfm_partition_update(xd->above_txfm_context + blk_col,
xd->left_txfm_context + blk_row, tx_size, tx_size);
} else {
@@ -6082,13 +6095,16 @@
++td->counts->tx_size[tx_size_cat][tx_size_ctx][depth];
#endif
-#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+
+#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
if (is_quarter_tx_allowed(xd, mbmi, is_inter) &&
- mbmi->tx_size != txsize_sqr_up_map[mbmi->tx_size]) {
- ++td->counts->quarter_tx_size[mbmi->tx_size ==
- quarter_txsize_lookup[mbmi->sb_type]];
+ quarter_txsize_lookup[bsize] != max_txsize_rect_lookup[bsize] &&
+ (mbmi->tx_size == quarter_txsize_lookup[bsize] ||
+ mbmi->tx_size == max_txsize_rect_lookup[bsize])) {
+ ++td->counts
+ ->quarter_tx_size[mbmi->tx_size == quarter_txsize_lookup[bsize]];
}
-#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#endif
#if CONFIG_EXT_TX && CONFIG_RECT_TX
assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed(xd, mbmi)));
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c
index 5b91cec..e7f4d31 100644
--- a/av1/encoder/encodemb.c
+++ b/av1/encoder/encodemb.c
@@ -797,16 +797,32 @@
encode_block(plane, block, blk_row, blk_col, plane_bsize, tx_size, arg);
} else {
assert(tx_size < TX_SIZES_ALL);
+#if CONFIG_RECT_TX_EXT
+ int is_qttx = plane_tx_size == quarter_txsize_lookup[plane_bsize];
+ const TX_SIZE sub_txs = is_qttx ? plane_tx_size : sub_tx_size_map[tx_size];
+ if (is_qttx) assert(blk_row == 0 && blk_col == 0 && block == 0);
+#else
const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
assert(sub_txs < tx_size);
+#endif
// This is the square transform block partition entry point.
int bsl = tx_size_wide_unit[sub_txs];
int i;
assert(bsl > 0);
for (i = 0; i < 4; ++i) {
+#if CONFIG_RECT_TX_EXT
+ int is_wide_tx = tx_size_wide_unit[sub_txs] > tx_size_high_unit[sub_txs];
+ const int offsetr =
+ is_qttx ? (is_wide_tx ? i * tx_size_high_unit[sub_txs] : 0)
+ : blk_row + ((i >> 1) * bsl);
+ const int offsetc =
+ is_qttx ? (is_wide_tx ? 0 : i * tx_size_wide_unit[sub_txs])
+ : blk_col + ((i & 0x01) * bsl);
+#else
const int offsetr = blk_row + ((i >> 1) * bsl);
const int offsetc = blk_col + ((i & 0x01) * bsl);
+#endif
int step = tx_size_wide_unit[sub_txs] * tx_size_high_unit[sub_txs];
if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue;
diff --git a/av1/encoder/hybrid_fwd_txfm.c b/av1/encoder/hybrid_fwd_txfm.c
index 81b895c..85f4b7d 100644
--- a/av1/encoder/hybrid_fwd_txfm.c
+++ b/av1/encoder/hybrid_fwd_txfm.c
@@ -143,7 +143,7 @@
}
#endif // CONFIG_TX64X64
-#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
static void fwd_txfm_16x4(const int16_t *src_diff, tran_low_t *coeff,
int diff_stride, TxfmParam *txfm_param) {
#if CONFIG_LGT
@@ -179,7 +179,7 @@
av1_fht8x32(src_diff, coeff, diff_stride, txfm_param);
#endif
}
-#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#endif
#if CONFIG_CHROMA_2X2
static void highbd_fwd_txfm_2x2(const int16_t *src_diff, tran_low_t *coeff,
@@ -483,7 +483,7 @@
#if CONFIG_CHROMA_2X2
case TX_2X2: fwd_txfm_2x2(src_diff, coeff, diff_stride, txfm_param); break;
#endif
-#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
case TX_4X16:
fwd_txfm_4x16(src_diff, coeff, diff_stride, txfm_param);
break;
@@ -496,7 +496,7 @@
case TX_32X8:
fwd_txfm_32x8(src_diff, coeff, diff_stride, txfm_param);
break;
-#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#endif
default: assert(0); break;
}
}
diff --git a/av1/encoder/rd.c b/av1/encoder/rd.c
index 9601184..da3b6e2 100644
--- a/av1/encoder/rd.c
+++ b/av1/encoder/rd.c
@@ -669,7 +669,7 @@
for (i = 0; i < num_4x4_h; i += 8)
t_left[i] = !!*(const uint64_t *)&left[i];
break;
-#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
case TX_4X16:
for (i = 0; i < num_4x4_w; i += 2)
t_above[i] = !!*(const uint16_t *)&above[i];
@@ -696,7 +696,7 @@
for (i = 0; i < num_4x4_h; i += 4)
t_left[i] = !!*(const uint32_t *)&left[i];
break;
-#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#endif
default: assert(0 && "Invalid transform size."); break;
}
@@ -770,7 +770,7 @@
for (i = 0; i < num_4x4_h; i += 4)
t_left[i] = !!*(const uint32_t *)&left[i];
break;
-#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
case TX_4X16:
memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w);
for (i = 0; i < num_4x4_h; i += 4)
@@ -793,7 +793,7 @@
for (i = 0; i < num_4x4_h; i += 2)
t_left[i] = !!*(const uint16_t *)&left[i];
break;
-#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#endif
default: assert(0 && "Invalid transform size."); break;
}
}
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 17d882c..dc6837e 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -1556,9 +1556,9 @@
BLOCK_SIZE plane_bsize, int blk_row, int blk_col,
BLOCK_SIZE tx_bsize, int *width, int *height,
int *visible_width, int *visible_height) {
-#if !(CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT)
+#if !(CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX))
assert(tx_bsize <= plane_bsize);
-#endif // !(CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT)
+#endif
int txb_height = block_size_high[tx_bsize];
int txb_width = block_size_wide[tx_bsize];
const int block_height = block_size_high[plane_bsize];
@@ -1606,7 +1606,7 @@
x->qindex);
#endif // CONFIG_DIST_8X8
-#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
if ((txb_rows == visible_rows && txb_cols == visible_cols) &&
tx_bsize < BLOCK_SIZES) {
#else
@@ -2191,11 +2191,11 @@
const int depth = tx_size_to_depth(coded_tx_size);
const int tx_size_ctx = get_tx_size_context(xd);
int r_tx_size = cpi->tx_size_cost[tx_size_cat][tx_size_ctx][depth];
-#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
if (is_quarter_tx_allowed(xd, mbmi, is_inter) && tx_size != coded_tx_size)
r_tx_size += av1_cost_bit(cm->fc->quarter_tx_size_prob,
tx_size == quarter_txsize_lookup[bsize]);
-#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#endif
return r_tx_size;
} else {
return 0;
@@ -4483,6 +4483,24 @@
TX_TYPE best_tx_type = TX_TYPES;
int txk_idx = (blk_row << 4) + blk_col;
#endif
+#if CONFIG_RECT_TX_EXT
+ TX_SIZE quarter_txsize = quarter_txsize_lookup[mbmi->sb_type];
+ int check_qttx = is_quarter_tx_allowed(xd, mbmi, is_inter_block(mbmi)) &&
+ tx_size == max_txsize_rect_lookup[mbmi->sb_type] &&
+ quarter_txsize != tx_size;
+ int is_qttx_picked = 0;
+ int eobs_qttx[2] = { 0, 0 };
+ int skip_qttx[2] = { 0, 0 };
+ int block_offset_qttx = check_qttx
+ ? tx_size_wide_unit[quarter_txsize] *
+ tx_size_high_unit[quarter_txsize]
+ : 0;
+ int blk_row_offset, blk_col_offset;
+ int is_wide_qttx =
+ tx_size_wide_unit[quarter_txsize] > tx_size_high_unit[quarter_txsize];
+ blk_row_offset = is_wide_qttx ? tx_size_high_unit[quarter_txsize] : 0;
+ blk_col_offset = is_wide_qttx ? 0 : tx_size_wide_unit[quarter_txsize];
+#endif
av1_init_rd_stats(&sum_rd_stats);
@@ -4504,7 +4522,7 @@
zero_blk_rate =
av1_cost_bit(xd->fc->txb_skip[txs_ctx][txb_ctx.txb_skip_ctx], 1);
#else
- const int tx_size_ctx = txsize_sqr_map[tx_size];
+ int tx_size_ctx = txsize_sqr_map[tx_size];
int coeff_ctx = get_entropy_context(tx_size, pta, ptl);
zero_blk_rate = x->token_costs[tx_size_ctx][pd->plane_type][1][0][0]
[coeff_ctx][EOB_TOKEN];
@@ -4552,6 +4570,12 @@
if (tx_size > TX_4X4 && depth < MAX_VARTX_DEPTH)
rd_stats->rate +=
av1_cost_bit(cpi->common.fc->txfm_partition_prob[ctx], 0);
+#if CONFIG_RECT_TX_EXT
+ if (check_qttx) {
+ assert(blk_row == 0 && blk_col == 0);
+ rd_stats->rate += av1_cost_bit(cpi->common.fc->quarter_tx_size_prob, 0);
+ }
+#endif
this_rd = RDCOST(x->rdmult, rd_stats->rate, rd_stats->dist);
#if CONFIG_LV_MAP
tmp_eob = p->txb_entropy_ctx[block];
@@ -4562,6 +4586,102 @@
#if CONFIG_TXK_SEL
best_tx_type = mbmi->txk_type[txk_idx];
#endif
+
+#if CONFIG_RECT_TX_EXT
+ if (check_qttx) {
+ assert(blk_row == 0 && blk_col == 0 && block == 0 && plane == 0);
+
+ RD_STATS rd_stats_tmp, rd_stats_qttx;
+ int64_t rd_qttx;
+
+ av1_init_rd_stats(&rd_stats_qttx);
+ av1_init_rd_stats(&rd_stats_tmp);
+
+ av1_tx_block_rd_b(cpi, x, quarter_txsize, 0, 0, plane, 0, plane_bsize,
+ pta, ptl, &rd_stats_qttx);
+
+ tx_size_ctx = txsize_sqr_map[quarter_txsize];
+ coeff_ctx = get_entropy_context(quarter_txsize, pta, ptl);
+ zero_blk_rate = x->token_costs[tx_size_ctx][pd->plane_type][1][0][0]
+ [coeff_ctx][EOB_TOKEN];
+ if ((RDCOST(x->rdmult, rd_stats_qttx.rate, rd_stats_qttx.dist) >=
+ RDCOST(x->rdmult, zero_blk_rate, rd_stats_qttx.sse) ||
+ rd_stats_qttx.skip == 1) &&
+ !xd->lossless[mbmi->segment_id]) {
+#if CONFIG_RD_DEBUG
+ av1_update_txb_coeff_cost(&rd_stats_qttx, plane, quarter_txsize, 0, 0,
+ zero_blk_rate - rd_stats_qttx.rate);
+#endif // CONFIG_RD_DEBUG
+ rd_stats_qttx.rate = zero_blk_rate;
+ rd_stats_qttx.dist = rd_stats_qttx.sse;
+ rd_stats_qttx.skip = 1;
+ x->blk_skip[plane][blk_row * bw + blk_col] = 1;
+ skip_qttx[0] = 1;
+ p->eobs[block] = 0;
+ } else {
+ x->blk_skip[plane][blk_row * bw + blk_col] = 0;
+ skip_qttx[0] = 0;
+ rd_stats->skip = 0;
+ }
+
+ // Second tx block
+ av1_tx_block_rd_b(cpi, x, quarter_txsize, blk_row_offset, blk_col_offset,
+ plane, block_offset_qttx, plane_bsize, pta, ptl,
+ &rd_stats_tmp);
+
+ av1_set_txb_context(x, plane, 0, quarter_txsize, pta, ptl);
+ coeff_ctx = get_entropy_context(quarter_txsize, pta + blk_col_offset,
+ ptl + blk_row_offset);
+ zero_blk_rate = x->token_costs[tx_size_ctx][pd->plane_type][1][0][0]
+ [coeff_ctx][EOB_TOKEN];
+ if ((RDCOST(x->rdmult, rd_stats_tmp.rate, rd_stats_tmp.dist) >=
+ RDCOST(x->rdmult, zero_blk_rate, rd_stats_tmp.sse) ||
+ rd_stats_tmp.skip == 1) &&
+ !xd->lossless[mbmi->segment_id]) {
+#if CONFIG_RD_DEBUG
+ av1_update_txb_coeff_cost(&rd_stats_tmp, plane, quarter_txsize, 0, 0,
+ zero_blk_rate - rd_stats_tmp.rate);
+#endif // CONFIG_RD_DEBUG
+ rd_stats_tmp.rate = zero_blk_rate;
+ rd_stats_tmp.dist = rd_stats_tmp.sse;
+ rd_stats_tmp.skip = 1;
+ x->blk_skip[plane][blk_row_offset * bw + blk_col_offset] = 1;
+ skip_qttx[1] = 1;
+ p->eobs[block_offset_qttx] = 0;
+ } else {
+ x->blk_skip[plane][blk_row_offset * bw + blk_col_offset] = 0;
+ skip_qttx[1] = 0;
+ rd_stats_tmp.skip = 0;
+ }
+
+ av1_merge_rd_stats(&rd_stats_qttx, &rd_stats_tmp);
+
+ if (tx_size > TX_4X4 && depth < MAX_VARTX_DEPTH) {
+ rd_stats_qttx.rate +=
+ av1_cost_bit(cpi->common.fc->txfm_partition_prob[ctx], 0);
+ }
+ rd_stats_qttx.rate +=
+ av1_cost_bit(cpi->common.fc->quarter_tx_size_prob, 1);
+ rd_qttx = RDCOST(x->rdmult, rd_stats_qttx.rate, rd_stats_qttx.dist);
+#if CONFIG_LV_MAP
+ eobs_qttx[0] = p->txb_entropy_ctx[0];
+ eobs_qttx[1] = p->txb_entropy_ctx[block_offset_qttx];
+#else
+ eobs_qttx[0] = p->eobs[0];
+ eobs_qttx[1] = p->eobs[block_offset_qttx];
+#endif
+ if (rd_qttx < this_rd) {
+ is_qttx_picked = 1;
+ this_rd = rd_qttx;
+ rd_stats->rate = rd_stats_qttx.rate;
+ rd_stats->dist = rd_stats_qttx.dist;
+ rd_stats->sse = rd_stats_qttx.sse;
+ rd_stats->skip = rd_stats_qttx.skip;
+ rd_stats->rdcost = rd_stats_qttx.rdcost;
+ }
+ av1_get_entropy_contexts(plane_bsize, 0, pd, ta, tl);
+ }
+#endif
}
#if CONFIG_MRC_TX
@@ -4704,27 +4824,61 @@
if (this_rd < sum_rd) {
int idx, idy;
+#if CONFIG_RECT_TX_EXT
+ TX_SIZE tx_size_selected = is_qttx_picked ? quarter_txsize : tx_size;
+#else
+ TX_SIZE tx_size_selected = tx_size;
+#endif
+#if CONFIG_RECT_TX_EXT
+ if (is_qttx_picked) {
+ assert(blk_row == 0 && blk_col == 0 && plane == 0);
#if CONFIG_LV_MAP
- p->txb_entropy_ctx[block] = tmp_eob;
+ p->txb_entropy_ctx[0] = eobs_qttx[0];
+ p->txb_entropy_ctx[block_offset_qttx] = eobs_qttx[1];
+#else
+ p->eobs[0] = eobs_qttx[0];
+ p->eobs[block_offset_qttx] = eobs_qttx[1];
+#endif
+ } else {
+#endif
+#if CONFIG_LV_MAP
+ p->txb_entropy_ctx[block] = tmp_eob;
#else
p->eobs[block] = tmp_eob;
#endif
+#if CONFIG_RECT_TX_EXT
+ }
+#endif
- av1_set_txb_context(x, plane, block, tx_size, pta, ptl);
+ av1_set_txb_context(x, plane, block, tx_size_selected, pta, ptl);
+#if CONFIG_RECT_TX_EXT
+ if (is_qttx_picked)
+ av1_set_txb_context(x, plane, block_offset_qttx, tx_size_selected,
+ pta + blk_col_offset, ptl + blk_row_offset);
+#endif
txfm_partition_update(tx_above + blk_col, tx_left + blk_row, tx_size,
tx_size);
- inter_tx_size[0][0] = tx_size;
+ inter_tx_size[0][0] = tx_size_selected;
for (idy = 0; idy < tx_size_high_unit[tx_size] / 2; ++idy)
for (idx = 0; idx < tx_size_wide_unit[tx_size] / 2; ++idx)
- inter_tx_size[idy][idx] = tx_size;
- mbmi->tx_size = tx_size;
+ inter_tx_size[idy][idx] = tx_size_selected;
+ mbmi->tx_size = tx_size_selected;
#if CONFIG_TXK_SEL
mbmi->txk_type[txk_idx] = best_tx_type;
#endif
if (this_rd == INT64_MAX) *is_cost_valid = 0;
- x->blk_skip[plane][blk_row * bw + blk_col] = rd_stats->skip;
+#if CONFIG_RECT_TX_EXT
+ if (is_qttx_picked) {
+ x->blk_skip[plane][0] = skip_qttx[0];
+ x->blk_skip[plane][blk_row_offset * bw + blk_col_offset] = skip_qttx[1];
+ } else {
+#endif
+ x->blk_skip[plane][blk_row * bw + blk_col] = rd_stats->skip;
+#if CONFIG_RECT_TX_EXT
+ }
+#endif
} else {
*rd_stats = sum_rd_stats;
if (sum_rd == INT64_MAX) *is_cost_valid = 0;
diff --git a/av1/encoder/tokenize.c b/av1/encoder/tokenize.c
index 719799a..b9db891 100644
--- a/av1/encoder/tokenize.c
+++ b/av1/encoder/tokenize.c
@@ -597,16 +597,31 @@
cost_coeffs_b(plane, block, blk_row, blk_col, plane_bsize, tx_size, arg);
#endif
} else {
+#if CONFIG_RECT_TX_EXT
+ int is_qttx = plane_tx_size == quarter_txsize_lookup[plane_bsize];
+ const TX_SIZE sub_txs = is_qttx ? plane_tx_size : sub_tx_size_map[tx_size];
+#else
// Half the block size in transform block unit.
const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
+#endif
const int bsl = tx_size_wide_unit[sub_txs];
int i;
assert(bsl > 0);
for (i = 0; i < 4; ++i) {
+#if CONFIG_RECT_TX_EXT
+ int is_wide_tx = tx_size_wide_unit[sub_txs] > tx_size_high_unit[sub_txs];
+ const int offsetr =
+ is_qttx ? (is_wide_tx ? i * tx_size_high_unit[sub_txs] : 0)
+ : blk_row + ((i >> 1) * bsl);
+ const int offsetc =
+ is_qttx ? (is_wide_tx ? 0 : i * tx_size_wide_unit[sub_txs])
+ : blk_col + ((i & 0x01) * bsl);
+#else
const int offsetr = blk_row + ((i >> 1) * bsl);
const int offsetc = blk_col + ((i & 0x01) * bsl);
+#endif
int step = tx_size_wide_unit[sub_txs] * tx_size_high_unit[sub_txs];
diff --git a/configure b/configure
index c8e1f3e..0e8d933 100755
--- a/configure
+++ b/configure
@@ -555,6 +555,7 @@
enabled ext_comp_refs && enable_feature var_refs
enabled ext_comp_refs && disable_feature one_sided_compound
enabled altref2 && enable_feature ext_refs
+ enabled rect_tx_ext && enable_feature rect_tx
if ! enabled delta_q && enabled ext_delta_q; then
log_echo "ext_delta_q requires delta_q, so disabling ext_delta_q"
diff --git a/tools/aom_entropy_optimizer.c b/tools/aom_entropy_optimizer.c
index 65b7d29..b892cc1 100644
--- a/tools/aom_entropy_optimizer.c
+++ b/tools/aom_entropy_optimizer.c
@@ -678,7 +678,7 @@
/* Transform size */
// TODO(yuec): av1_tx_size_tree has variable sizes, so needs special handling
-#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
cts_each_dim[0] = 2;
optimize_entropy_table(&fc.quarter_tx_size[0], probsfile, 1, cts_each_dim,
NULL, 1,
