Fix unit test failure for RECT_TX + VAR_TX
Disable rect_tx because we only support 4x4 Walsh-Hadamard transform
in lossless mode.
Fixes failure in ./test_libaom --gtest_filter=*Large*ScreencastQ0/1
Configuration: --enable-experimental --enable-var-tx --enable-rect-tx
--enable-ref-mv --enable-ext_intra --enable-ext_tx --enable-debug
--disable-optimizations
Change-Id: Ib6b3494c7dcf7182f1cab9b138388d054851a23d
diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index 94eb089..266f8fe 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -516,8 +516,10 @@
return LUT[bsize];
}
-static INLINE int is_rect_tx_allowed(const MB_MODE_INFO *mbmi) {
- return is_inter_block(mbmi) && is_rect_tx_allowed_bsize(mbmi->sb_type);
+static INLINE int is_rect_tx_allowed(const MACROBLOCKD *xd,
+ const MB_MODE_INFO *mbmi) {
+ return is_inter_block(mbmi) && is_rect_tx_allowed_bsize(mbmi->sb_type) &&
+ !xd->lossless[mbmi->segment_id];
}
static INLINE int is_rect_tx(TX_SIZE tx_size) { return tx_size >= TX_SIZES; }
diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c
index 6492f62..7be3e75 100644
--- a/av1/decoder/decodemv.c
+++ b/av1/decoder/decodemv.c
@@ -1633,7 +1633,8 @@
int idx, idy;
int tx_size_cat = inter_tx_size_cat_lookup[bsize];
#if CONFIG_EXT_TX && CONFIG_RECT_TX
- int is_rect_tx_allowed = inter_block && is_rect_tx_allowed_bsize(bsize);
+ int is_rect_tx_allowed = inter_block && is_rect_tx_allowed_bsize(bsize) &&
+ !xd->lossless[mbmi->segment_id];
int use_rect_tx = 0;
if (is_rect_tx_allowed) {
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 7636151..f9ae154 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -406,7 +406,7 @@
const TX_SIZE coded_tx_size = txsize_sqr_up_map[tx_size];
#if CONFIG_EXT_TX && CONFIG_RECT_TX
- assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed(mbmi)));
+ assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed(xd, mbmi)));
assert(
IMPLIES(is_rect_tx(tx_size), tx_size == max_txsize_rect_lookup[bsize]));
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
@@ -1132,7 +1132,7 @@
int idx, idy;
#if CONFIG_EXT_TX && CONFIG_RECT_TX
- if (is_rect_tx_allowed(mbmi)) {
+ if (is_rect_tx_allowed(xd, mbmi)) {
int tx_size_cat = inter_tx_size_cat_lookup[bsize];
aom_write(w, is_rect_tx(mbmi->tx_size),
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index cfc4718..cc73d4c 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -5141,14 +5141,14 @@
: intra_tx_size_cat_lookup[bsize];
const TX_SIZE coded_tx_size = txsize_sqr_up_map[mbmi->tx_size];
#if CONFIG_EXT_TX && CONFIG_RECT_TX
- assert(IMPLIES(is_rect_tx(mbmi->tx_size), is_rect_tx_allowed(mbmi)));
+ assert(IMPLIES(is_rect_tx(mbmi->tx_size), is_rect_tx_allowed(xd, mbmi)));
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
#if CONFIG_VAR_TX
#if CONFIG_EXT_TX && CONFIG_RECT_TX
- if (is_rect_tx_allowed(mbmi)) {
+ if (is_rect_tx_allowed(xd, mbmi)) {
td->counts->rect_tx[tx_size_cat][is_rect_tx(mbmi->tx_size)]++;
}
- if (!is_rect_tx_allowed(mbmi) || !is_rect_tx(mbmi->tx_size)) {
+ if (!is_rect_tx_allowed(xd, mbmi) || !is_rect_tx(mbmi->tx_size)) {
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
if (is_inter)
tx_partition_count_update(cm, xd, bsize, mi_row, mi_col, td->counts);
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 9721abe..8b0a7bd 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -1385,14 +1385,14 @@
const int is_inter = is_inter_block(mbmi);
#if CONFIG_EXT_TX
#if CONFIG_RECT_TX
- int evaulate_rect_tx = 0;
+ int evaluate_rect_tx = 0;
#endif // CONFIG_RECT_TX
int ext_tx_set;
#endif // CONFIG_EXT_TX
if (tx_select) {
#if CONFIG_EXT_TX && CONFIG_RECT_TX
- evaulate_rect_tx = is_rect_tx_allowed(mbmi);
+ evaluate_rect_tx = is_rect_tx_allowed(xd, mbmi);
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
start_tx = max_tx_size;
end_tx = 0;
@@ -1400,8 +1400,8 @@
const TX_SIZE chosen_tx_size =
tx_size_from_tx_mode(bs, cm->tx_mode, is_inter);
#if CONFIG_EXT_TX && CONFIG_RECT_TX
- evaulate_rect_tx = is_rect_tx(chosen_tx_size);
- assert(IMPLIES(evaulate_rect_tx, is_rect_tx_allowed(mbmi)));
+ evaluate_rect_tx = is_rect_tx(chosen_tx_size);
+ assert(IMPLIES(evaluate_rect_tx, is_rect_tx_allowed(xd, mbmi)));
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
start_tx = chosen_tx_size;
end_tx = chosen_tx_size;
@@ -1415,7 +1415,7 @@
mbmi->tx_type = tx_type;
#if CONFIG_EXT_TX && CONFIG_RECT_TX
- if (evaulate_rect_tx) {
+ if (evaluate_rect_tx) {
const TX_SIZE rect_tx_size = max_txsize_rect_lookup[bs];
const int ext_tx_set = get_ext_tx_set(rect_tx_size, bs, 1);
if (ext_tx_used_inter[ext_tx_set][tx_type]) {
@@ -3214,7 +3214,7 @@
mbmi->tx_type = tx_type;
inter_block_yrd(cpi, x, rate, dist, skippable, sse, bsize, ref_best_rd);
#if CONFIG_EXT_TX && CONFIG_RECT_TX
- if (is_rect_tx_allowed(mbmi)) {
+ if (is_rect_tx_allowed(xd, mbmi)) {
int rate_rect_tx, skippable_rect_tx = 0;
int64_t dist_rect_tx, sse_rect_tx, rd, rd_rect_tx;
int tx_size_cat = inter_tx_size_cat_lookup[bsize];
