AOM_QM: Fix bug with applying QMs to 1D and identity transforms.

Only apply quantisation matrices when the transform is 2D.

Change-Id: Iae9ac910c11199f7944a80d10d334db3b711059d

av1/common/enums.h

diff --git a/av1/common/enums.h b/av1/common/enums.h
index d105f94..0437e08 100644
--- a/av1/common/enums.h
+++ b/av1/common/enums.h
@@ -237,6 +237,12 @@
   TX_TYPES,
 } TX_TYPE;
 
+#if CONFIG_EXT_TX
+#define IS_2D_TRANSFORM(tx_type) (tx_type < IDTX)
+#else
+#define IS_2D_TRANSFORM(tx_type) 1
+#endif
+
 typedef enum {
   TILE_LEFT_BOUNDARY = 1,
   TILE_RIGHT_BOUNDARY = 2,

av1/decoder/detokenize.c

diff --git a/av1/decoder/detokenize.c b/av1/decoder/detokenize.c
index 2e3309c..b48e8b1 100644
--- a/av1/decoder/detokenize.c
+++ b/av1/decoder/detokenize.c
@@ -120,6 +120,8 @@
   const int ref = is_inter_block(&xd->mi[0]->mbmi);
 #if CONFIG_AOM_QM
   const qm_val_t *iqmatrix = iqm[!ref][tx_size];
+#else
+  (void)tx_type;
 #endif  // CONFIG_AOM_QM
   int band, c = 0;
   const int tx_size_ctx = txsize_sqr_map[tx_size];
@@ -142,7 +144,6 @@
 #if CONFIG_NEW_QUANT
   const tran_low_t *dqv_val = &dq_val[0][0];
 #endif  // CONFIG_NEW_QUANT
-  (void)tx_type;
 
   if (counts) {
 #if !CONFIG_EC_ADAPT
@@ -226,8 +227,10 @@
     v = dq_shift ? ROUND_POWER_OF_TWO(v, dq_shift) : v;
 #else
 #if CONFIG_AOM_QM
-    dqv = ((iqmatrix[scan[c]] * (int)dqv) + (1 << (AOM_QM_BITS - 1))) >>
-          AOM_QM_BITS;
+    // Apply quant matrix only for 2D transforms
+    if (IS_2D_TRANSFORM(tx_type))
+      dqv = ((iqmatrix[scan[c]] * (int)dqv) + (1 << (AOM_QM_BITS - 1))) >>
+            AOM_QM_BITS;
 #endif
     v = (val * dqv) >> dq_shift;
 #endif

av1/encoder/encodemb.c

diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c
index b4624a4..db63049 100644
--- a/av1/encoder/encodemb.c
+++ b/av1/encoder/encodemb.c
@@ -164,7 +164,11 @@
   const int shift = av1_get_tx_scale(tx_size);
 #if CONFIG_AOM_QM
   int seg_id = xd->mi[0]->mbmi.segment_id;
-  const qm_val_t *iqmatrix = pd->seg_iqmatrix[seg_id][!ref][tx_size];
+  // Use a flat matrix (i.e. no weighting) for 1D and Identity transforms
+  const qm_val_t *iqmatrix =
+      IS_2D_TRANSFORM(tx_type)
+          ? pd->seg_iqmatrix[seg_id][!ref][tx_size]
+          : cm->giqmatrix[NUM_QM_LEVELS - 1][0][0][tx_size];
 #endif
 #if CONFIG_NEW_QUANT
   int dq = get_dq_profile_from_ctx(mb->qindex, ctx, ref, plane_type);
@@ -452,7 +456,6 @@
   mb->plane[plane].eobs[block] = final_eob;
   return final_eob;
 }
-
 #endif  // !CONFIG_LV_MAP
 
 int av1_optimize_b(const AV1_COMMON *cm, MACROBLOCK *mb, int plane, int block,
@@ -556,8 +559,14 @@
   const int diff_stride = block_size_wide[plane_bsize];
 #if CONFIG_AOM_QM
   int seg_id = mbmi->segment_id;
-  const qm_val_t *qmatrix = pd->seg_qmatrix[seg_id][!is_inter][tx_size];
-  const qm_val_t *iqmatrix = pd->seg_iqmatrix[seg_id][!is_inter][tx_size];
+  // Use a flat matrix (i.e. no weighting) for 1D and Identity transforms
+  const qm_val_t *qmatrix =
+      IS_2D_TRANSFORM(tx_type) ? pd->seg_qmatrix[seg_id][!is_inter][tx_size]
+                               : cm->gqmatrix[NUM_QM_LEVELS - 1][0][0][tx_size];
+  const qm_val_t *iqmatrix =
+      IS_2D_TRANSFORM(tx_type)
+          ? pd->seg_iqmatrix[seg_id][!is_inter][tx_size]
+          : cm->giqmatrix[NUM_QM_LEVELS - 1][0][0][tx_size];
 #endif
 
   FWD_TXFM_PARAM fwd_txfm_param;