commit be44e512ea4e6e204b4540ae28d08d227d6744f4
author Thomas Davies <thdavies@cisco.com> Wed Jun 07 11:41:07 2017 +0100
committer Thomas Davies <thdavies@cisco.com> Tue Jun 13 10:32:04 2017 +0100
tree 37686809c19df4c260527586e908420de3a3abf6
parent 85b66463a2ceb895c7842350f7a992082c64b523

AOM_QM: Fix bug with applying QMs to 1D and identity transforms.

Only apply quantisation matrices when the transform is 2D.

Change-Id: Iae9ac910c11199f7944a80d10d334db3b711059d

av1/encoder/encodemb.c

diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c
index b4624a4..db63049 100644
--- a/av1/encoder/encodemb.c
+++ b/av1/encoder/encodemb.c
@@ -164,7 +164,11 @@
   const int shift = av1_get_tx_scale(tx_size);
 #if CONFIG_AOM_QM
   int seg_id = xd->mi[0]->mbmi.segment_id;
-  const qm_val_t *iqmatrix = pd->seg_iqmatrix[seg_id][!ref][tx_size];
+  // Use a flat matrix (i.e. no weighting) for 1D and Identity transforms
+  const qm_val_t *iqmatrix =
+      IS_2D_TRANSFORM(tx_type)
+          ? pd->seg_iqmatrix[seg_id][!ref][tx_size]
+          : cm->giqmatrix[NUM_QM_LEVELS - 1][0][0][tx_size];
 #endif
 #if CONFIG_NEW_QUANT
   int dq = get_dq_profile_from_ctx(mb->qindex, ctx, ref, plane_type);
@@ -452,7 +456,6 @@
   mb->plane[plane].eobs[block] = final_eob;
   return final_eob;
 }
-
 #endif  // !CONFIG_LV_MAP
 
 int av1_optimize_b(const AV1_COMMON *cm, MACROBLOCK *mb, int plane, int block,
@@ -556,8 +559,14 @@
   const int diff_stride = block_size_wide[plane_bsize];
 #if CONFIG_AOM_QM
   int seg_id = mbmi->segment_id;
-  const qm_val_t *qmatrix = pd->seg_qmatrix[seg_id][!is_inter][tx_size];
-  const qm_val_t *iqmatrix = pd->seg_iqmatrix[seg_id][!is_inter][tx_size];
+  // Use a flat matrix (i.e. no weighting) for 1D and Identity transforms
+  const qm_val_t *qmatrix =
+      IS_2D_TRANSFORM(tx_type) ? pd->seg_qmatrix[seg_id][!is_inter][tx_size]
+                               : cm->gqmatrix[NUM_QM_LEVELS - 1][0][0][tx_size];
+  const qm_val_t *iqmatrix =
+      IS_2D_TRANSFORM(tx_type)
+          ? pd->seg_iqmatrix[seg_id][!is_inter][tx_size]
+          : cm->giqmatrix[NUM_QM_LEVELS - 1][0][0][tx_size];
 #endif
 
   FWD_TXFM_PARAM fwd_txfm_param;