commit e3aabb3577440afff21913970bafd85bcb613708
author Lester Lu <kslu@google.com> Wed Aug 11 15:51:46 2021 -0700
committer Lester Lu <kslu@google.com> Thu Aug 12 14:21:58 2021 -0700
tree 458933054690b4e5966939eac7bccc7c07fc776c
parent de1b1e120b9c3809a9fdccdac251720aef3dda44

Make CONFIG_DST_32X32 work with MDTX

Fix a bug when CONFIG_MODE_DEP_INTER_TX and CONFIG_DST_32X32 are
both on.

Change-Id: I78d21f862df1e34553fad62fa3b72c255ff4f86d

av1/common/av1_inv_txfm2d.c

diff --git a/av1/common/av1_inv_txfm2d.c b/av1/common/av1_inv_txfm2d.c
index 108ed95..da40f9d 100644
--- a/av1/common/av1_inv_txfm2d.c
+++ b/av1/common/av1_inv_txfm2d.c
@@ -238,6 +238,8 @@
   if (cfg->txfm_type_row == TXFM_TYPE_ADST4) {
     memcpy(cfg->stage_range_row, iadst4_range, sizeof(iadst4_range));
   }
+  assert(cfg->txfm_type_col < TXFM_TYPE_INVALID);
+  assert(cfg->txfm_type_row < TXFM_TYPE_INVALID);
   cfg->mode = mode;
 #if CONFIG_MODE_DEP_INTRA_TX && CONFIG_MODE_DEP_NONSEP_INTRA_TX
   if (use_nstx(tx_type, tx_size, mode)) {

av1/common/av1_txfm.c

diff --git a/av1/common/av1_txfm.c b/av1/common/av1_txfm.c
index 59d21f7..b171095 100644
--- a/av1/common/av1_txfm.c
+++ b/av1/common/av1_txfm.c
@@ -396,9 +396,14 @@
   { TXFM_TYPE_DCT8, TXFM_TYPE_ADST8, TXFM_TYPE_ADST8, TXFM_TYPE_IDENTITY8,
     TXFM_TYPE_MDTX8, TXFM_TYPE_MDTX8 },
   { TXFM_TYPE_DCT16, TXFM_TYPE_ADST16, TXFM_TYPE_ADST16, TXFM_TYPE_IDENTITY16,
-    TXFM_TYPE_MDTX16, TXFM_TYPE_ADST16 },
+    TXFM_TYPE_MDTX16, TXFM_TYPE_MDTX16 },
+#if CONFIG_DST_32X32
+  { TXFM_TYPE_DCT32, TXFM_TYPE_ADST32, TXFM_TYPE_ADST32, TXFM_TYPE_IDENTITY32,
+    TXFM_TYPE_ADST32, TXFM_TYPE_ADST32 },
+#else
   { TXFM_TYPE_DCT32, TXFM_TYPE_INVALID, TXFM_TYPE_INVALID, TXFM_TYPE_IDENTITY32,
     TXFM_TYPE_INVALID, TXFM_TYPE_INVALID },
+#endif
   { TXFM_TYPE_DCT64, TXFM_TYPE_INVALID, TXFM_TYPE_INVALID, TXFM_TYPE_INVALID,
     TXFM_TYPE_INVALID, TXFM_TYPE_INVALID }
 };

av1/encoder/av1_fwd_txfm2d.c

diff --git a/av1/encoder/av1_fwd_txfm2d.c b/av1/encoder/av1_fwd_txfm2d.c
index 3a1cc7c..a58fafb 100644
--- a/av1/encoder/av1_fwd_txfm2d.c
+++ b/av1/encoder/av1_fwd_txfm2d.c
@@ -855,6 +855,8 @@
   cfg->cos_bit_row = av1_fwd_cos_bit_row[txw_idx][txh_idx];
   cfg->txfm_type_col = av1_txfm_type_ls[txh_idx][tx_type_1d_col];
   cfg->txfm_type_row = av1_txfm_type_ls[txw_idx][tx_type_1d_row];
+  assert(cfg->txfm_type_col < TXFM_TYPE_INVALID);
+  assert(cfg->txfm_type_row < TXFM_TYPE_INVALID);
   cfg->mode = mode;
 #if CONFIG_MODE_DEP_INTRA_TX && CONFIG_MODE_DEP_NONSEP_INTRA_TX
   if (use_nstx(tx_type, tx_size, mode)) {