Miscellaneous fixes for var-tx
Lots of small bug fixes, mainly around the transform size coding:
* The loop filter was accidentally using the non-subsampled
block size for the V plane, due to comparing a plane index
(0, 1, or 2) against PLANE_TYPE_UV (== 1)
* We allowed an initial update of the transform partition probabilities
even on frames where we know they will never be used
(because tx_mode != TX_MODE_SELECT).
Further, these probabilities would not be reverted at the end
of the frame, leading to the probability delta persisting across frames.
Change this to behave more like the non-var-tx transform size coding,
where probability deltas are only coded for frames with
tx_mode == TX_MODE_SELECT, and the deltas only apply for one frame.
* Fix decoder for the case where the video as a whole isn't lossless,
and we have tx_mode == TX_MODE_SELECT, but the current segment
*is* lossless.
Note that the encoder already does the right thing in this case.
* Don't allow the transform splitting to recurse "below" 4x4.
This is really just a refactor, but means we can increase the
maximum depth when subdividing rectangular transforms if we
want to, whereas the previous code would have needed special cases
for 4x8 and 8x4 transforms.
* Finally, when we hit the maximum splitting depth, don't update
the counts as if we had coded a 'no split' symbol.
Change-Id: Iaebdacc9de81d2e93d3c49241e719bbc02e32682
diff --git a/av1/common/av1_loopfilter.c b/av1/common/av1_loopfilter.c
index 8d1b76c..b2501c4 100644
--- a/av1/common/av1_loopfilter.c
+++ b/av1/common/av1_loopfilter.c
@@ -2739,7 +2739,7 @@
const uint32_t scale_horz,
const uint32_t scale_vert) {
const MB_MODE_INFO *mbmi = &mi->mbmi;
- TX_SIZE tx_size = (plane == PLANE_TYPE_Y)
+ TX_SIZE tx_size = (plane == AOM_PLANE_Y)
? mbmi->tx_size
: av1_get_uv_tx_size(mbmi, plane_ptr);
assert(tx_size < TX_SIZES_ALL);
@@ -2774,9 +2774,9 @@
assert(mb_tx_size < TX_SIZES_ALL);
- tx_size = (plane == PLANE_TYPE_UV)
- ? uv_txsize_lookup[bsize][mb_tx_size][0][0]
- : mb_tx_size;
+ tx_size = (plane == AOM_PLANE_Y)
+ ? mb_tx_size
+ : uv_txsize_lookup[bsize][mb_tx_size][0][0];
assert(tx_size < TX_SIZES_ALL);
}
#else
diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c
index fd6c9f3..10e1512 100644
--- a/av1/decoder/decodeframe.c
+++ b/av1/decoder/decodeframe.c
@@ -4926,8 +4926,9 @@
#if !CONFIG_NEW_MULTISYMBOL
#if CONFIG_VAR_TX
- for (i = 0; i < TXFM_PARTITION_CONTEXTS; ++i)
- av1_diff_update_prob(&r, &fc->txfm_partition_prob[i], ACCT_STR);
+ if (cm->tx_mode == TX_MODE_SELECT)
+ for (i = 0; i < TXFM_PARTITION_CONTEXTS; ++i)
+ av1_diff_update_prob(&r, &fc->txfm_partition_prob[i], ACCT_STR);
#endif // CONFIG_VAR_TX
for (i = 0; i < SKIP_CONTEXTS; ++i)
av1_diff_update_prob(&r, &fc->skip_probs[i], ACCT_STR);
diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c
index 25a0ab9..846c4bc 100644
--- a/av1/decoder/decodemv.c
+++ b/av1/decoder/decodemv.c
@@ -428,6 +428,7 @@
[MAX_MIB_SIZE] =
(TX_SIZE(*)[MAX_MIB_SIZE]) & mbmi->inter_tx_size[tx_row][tx_col];
if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
+ assert(tx_size > TX_4X4);
if (depth == MAX_VARTX_DEPTH) {
int idx, idy;
@@ -437,7 +438,6 @@
inter_tx_size[idy][idx] = tx_size;
mbmi->tx_size = tx_size;
mbmi->min_tx_size = AOMMIN(mbmi->min_tx_size, get_min_tx_size(tx_size));
- if (counts) ++counts->txfm_partition[ctx][0];
txfm_partition_update(xd->above_txfm_context + blk_col,
xd->left_txfm_context + blk_row, tx_size, tx_size);
return;
@@ -456,7 +456,7 @@
if (counts) ++counts->txfm_partition[ctx][1];
- if (tx_size == TX_8X8) {
+ if (sub_txs == TX_4X4) {
int idx, idy;
inter_tx_size[0][0] = sub_txs;
for (idy = 0; idy < tx_size_high_unit[tx_size] / 2; ++idy)
@@ -2861,7 +2861,7 @@
#else
bsize >= BLOCK_8X8 &&
#endif
- !mbmi->skip && inter_block) {
+ !mbmi->skip && inter_block && !xd->lossless[mbmi->segment_id]) {
const TX_SIZE max_tx_size = max_txsize_rect_lookup[bsize];
const int bh = tx_size_high_unit[max_tx_size];
const int bw = tx_size_wide_unit[max_tx_size];
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 9561159..968b903 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -376,7 +376,7 @@
aom_write(w, 1, cm->fc->txfm_partition_prob[ctx]);
#endif
- if (tx_size == TX_8X8) {
+ if (sub_txs == TX_4X4) {
txfm_partition_update(xd->above_txfm_context + blk_col,
xd->left_txfm_context + blk_row, sub_txs, tx_size);
return;
@@ -401,7 +401,7 @@
counts->txfm_partition[k], probwt);
}
#endif // CONFIG_NEW_MULTISYMBOL
-#endif
+#endif // CONFIG_VAR_TX
static void write_selected_tx_size(const AV1_COMMON *cm, const MACROBLOCKD *xd,
aom_writer *w) {
@@ -4701,7 +4701,8 @@
#endif // CONFIG_LV_MAP
#if CONFIG_VAR_TX && !CONFIG_NEW_MULTISYMBOL
- update_txfm_partition_probs(cm, header_bc, counts, probwt);
+ if (cm->tx_mode == TX_MODE_SELECT)
+ update_txfm_partition_probs(cm, header_bc, counts, probwt);
#endif
#if !CONFIG_NEW_MULTISYMBOL
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 062b3b3..db20aa3 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -5806,6 +5806,18 @@
const TX_SIZE plane_tx_size = mbmi->inter_tx_size[tx_row][tx_col];
if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
+ assert(tx_size > TX_4X4);
+
+ if (depth == MAX_VARTX_DEPTH) {
+// Don't add to counts in this case
+#if CONFIG_RECT_TX_EXT
+ if (tx_size == plane_tx_size)
+#endif
+ mbmi->tx_size = tx_size;
+ txfm_partition_update(xd->above_txfm_context + blk_col,
+ xd->left_txfm_context + blk_row, tx_size, tx_size);
+ return;
+ }
#if CONFIG_RECT_TX_EXT
if (tx_size == plane_tx_size ||
@@ -5829,7 +5841,7 @@
++counts->txfm_partition[ctx][1];
++x->txb_split_count;
- if (tx_size == TX_8X8) {
+ if (sub_txs == TX_4X4) {
mbmi->inter_tx_size[tx_row][tx_col] = TX_4X4;
mbmi->tx_size = TX_4X4;
txfm_partition_update(xd->above_txfm_context + blk_col,
