Avoid using MRC_DCT when the mask produced is invalid
If the mask is invalid, do not allow the encoder to select MRC_DCT.
Currently the mask is invalid if it is all 1 or all 0, but these
criteria will likely expand in a future patch.
Change-Id: I77230ea8357bfdb2bf1e6338903d44bbf1db22d1
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 3fccb3c..1b86b61 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -2159,6 +2159,13 @@
}
#endif // DISABLE_TRELLISQ_SEARCH
+#if CONFIG_MRC_TX
+ if (mbmi->tx_type == MRC_DCT && !mbmi->valid_mrc_mask) {
+ args->exit_early = 1;
+ return;
+ }
+#endif // CONFIG_MRC_TX
+
if (!is_inter_block(mbmi)) {
struct macroblock_plane *const p = &x->plane[plane];
av1_inverse_transform_block_facade(xd, plane, block, blk_row, blk_col,
