Integrate lv_map with aom_qm
BUG=aomedia:717
Change-Id: Ib06a12039cb72665c1ee534cc2246ac3d23f878d
diff --git a/av1/decoder/decodetxb.c b/av1/decoder/decodetxb.c
index 2317f95..f9d2f72 100644
--- a/av1/decoder/decodetxb.c
+++ b/av1/decoder/decodetxb.c
@@ -49,6 +49,21 @@
return eob;
}
+#if !CONFIG_NEW_QUANT
+static INLINE int get_dqv(const int16_t *dequant, int coeff_idx,
+ const qm_val_t *iqmatrix) {
+ int dqv = dequant[!!coeff_idx];
+#if CONFIG_AOM_QM
+ if (iqmatrix != NULL)
+ dqv =
+ ((iqmatrix[coeff_idx] * dqv) + (1 << (AOM_QM_BITS - 1))) >> AOM_QM_BITS;
+#else
+ (void)iqmatrix;
+#endif
+ return dqv;
+}
+#endif
+
uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *const xd,
aom_reader *const r, const int blk_row,
const int blk_col, const int plane,
@@ -115,6 +130,16 @@
#endif
const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col,
tx_size, cm->reduced_tx_set_used);
+#if CONFIG_AOM_QM
+ const TX_SIZE qm_tx_size = av1_get_adjusted_tx_size(tx_size);
+ const qm_val_t *iqmatrix =
+ IS_2D_TRANSFORM(tx_type)
+ ? pd->seg_iqmatrix[mbmi->segment_id][qm_tx_size]
+ : cm->giqmatrix[NUM_QM_LEVELS - 1][0][qm_tx_size];
+#else
+ const qm_val_t *iqmatrix = NULL;
+#endif
+ (void)iqmatrix;
const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, mbmi);
const int16_t *const scan = scan_order->scan;
int dummy;
@@ -229,7 +254,7 @@
v = av1_dequant_abscoeff_nuq(level, dequant[!!c], dqv_val, nq_shift);
#endif // CONFIG_AOM_QM
#else
- v = level * dequant[!!c];
+ v = level * get_dqv(dequant, scan[c], iqmatrix);
v = v >> shift;
#endif // CONFIG_NEW_QUANT
tcoeffs[pos] = v;
@@ -288,7 +313,7 @@
t = av1_dequant_abscoeff_nuq(*level, dequant[!!pos], dqv_val, nq_shift);
#endif // CONFIG_AOM_QM
#else
- t = *level * dequant[!!pos];
+ t = *level * get_dqv(dequant, pos, iqmatrix);
t = t >> shift;
#endif // CONFIG_NEW_QUANT
if (signs[pos]) t = -t;
@@ -309,7 +334,7 @@
t = av1_dequant_abscoeff_nuq(t, dequant[!!pos], dqv_val, nq_shift);
#endif // CONFIG_AOM_QM
#else
- t = t * dequant[!!pos];
+ t = t * get_dqv(dequant, pos, iqmatrix);
t = t >> shift;
#endif // CONFIG_NEW_QUANT
if (signs[pos]) t = -t;
diff --git a/av1/encoder/encodetxb.c b/av1/encoder/encodetxb.c
index 89ebcb0..1ee5089 100644
--- a/av1/encoder/encodetxb.c
+++ b/av1/encoder/encodetxb.c
@@ -137,25 +137,30 @@
return qc > 0 ? qc - 1 : qc + 1;
}
-static INLINE tran_low_t qcoeff_to_dqcoeff(tran_low_t qc,
+static INLINE tran_low_t qcoeff_to_dqcoeff(tran_low_t qc, int coeff_idx,
#if CONFIG_NEW_QUANT
#if CONFIG_AOM_QM
- int dq_idx, int is_ac_coeff,
+ int dq_idx,
#else
const tran_low_t *nq_dq,
#endif // CONFIG_AOM_QM
#endif // CONFIG_NEW_QUANT
- int dqv, int shift) {
+ int dqv, int shift,
+ const qm_val_t *iqmatrix) {
int sgn = qc < 0 ? -1 : 1;
#if CONFIG_NEW_QUANT
#if CONFIG_AOM_QM
+ int is_ac_coeff = coeff_idx != 0;
int dqcoeff = av1_dequant_coeff_nuq(abs(qc), dqv, dq_idx, is_ac_coeff, shift);
#else
+ (void)coeff_idx;
int dqcoeff = av1_dequant_coeff_nuq(abs(qc), dqv, nq_dq, shift);
#endif // CONFIG_AOM_QM
return sgn * dqcoeff;
#endif // CONFIG_NEW_QUANT
-
+ if (iqmatrix != NULL)
+ dqv =
+ ((iqmatrix[coeff_idx] * dqv) + (1 << (AOM_QM_BITS - 1))) >> AOM_QM_BITS;
return sgn * ((abs(qc) * dqv) >> shift);
}
@@ -299,27 +304,29 @@
stats->rate = qc_cost;
return;
} else {
- const tran_low_t dqc = qcoeff_to_dqcoeff(qc,
+ const tran_low_t dqc =
+ qcoeff_to_dqcoeff(qc, coeff_idx,
#if CONFIG_NEW_QUANT
#if CONFIG_AOM_QM
- dq_idx, coeff_idx != 0,
+ dq_idx,
#else
- nq_dequant_val,
+ nq_dequant_val,
#endif // CONFIG_AOM_QM
#endif // CONFIG_NEW_QUANT
- dqv, txb_info->shift);
+ dqv, txb_info->shift, txb_info->iqmatrix);
const int64_t dqc_dist = get_coeff_dist(tqc, dqc, txb_info->shift);
// distortion difference when coefficient is quantized to 0
- const tran_low_t dqc0 = qcoeff_to_dqcoeff(0,
+ const tran_low_t dqc0 =
+ qcoeff_to_dqcoeff(0, coeff_idx,
#if CONFIG_NEW_QUANT
#if CONFIG_AOM_QM
- dq_idx, coeff_idx != 0,
+ dq_idx,
#else
- nq_dequant_val,
+ nq_dequant_val,
#endif // CONFIG_AOM_QM
#endif // CONFIG_NEW_QUANT
- dqv, txb_info->shift);
+ dqv, txb_info->shift, txb_info->iqmatrix);
stats->dist0 = get_coeff_dist(tqc, dqc0, txb_info->shift);
stats->dist = dqc_dist - stats->dist0;
@@ -336,15 +343,16 @@
if (stats->low_qc == 0) {
stats->dist_low = 0;
} else {
- stats->low_dqc = qcoeff_to_dqcoeff(stats->low_qc,
+ stats->low_dqc =
+ qcoeff_to_dqcoeff(stats->low_qc, coeff_idx,
#if CONFIG_NEW_QUANT
#if CONFIG_AOM_QM
- dq_idx, coeff_idx != 0,
+ dq_idx,
#else
- nq_dequant_val,
+ nq_dequant_val,
#endif // CONFIG_AOM_QM
#endif // CONFIG_NEW_QUANT
- dqv, txb_info->shift);
+ dqv, txb_info->shift, txb_info->iqmatrix);
const int64_t low_dqc_dist =
get_coeff_dist(tqc, stats->low_dqc, txb_info->shift);
stats->dist_low = low_dqc_dist - stats->dist0;
@@ -392,15 +400,16 @@
const tran_low_t *nq_dequant_val = txb_info->nq_dequant_vals[coeff_idx != 0];
#endif // CONFIG_AOM_QM
#endif // CONFIG_NEW_QUANT
- txb_info->dqcoeff[coeff_idx] = qcoeff_to_dqcoeff(qc,
+ txb_info->dqcoeff[coeff_idx] =
+ qcoeff_to_dqcoeff(qc, coeff_idx,
#if CONFIG_NEW_QUANT
#if CONFIG_AOM_QM
- dq_idx, coeff_idx != 0,
+ dq_idx,
#else
- nq_dequant_val,
+ nq_dequant_val,
#endif // CONFIG_AOM_QM
#endif // CONFIG_NEW_QUANT
- dqv, txb_info->shift);
+ dqv, txb_info->shift, txb_info->iqmatrix);
}
static INLINE void av1_txb_init_levels(const tran_low_t *const coeff,
@@ -2065,7 +2074,15 @@
2;
uint8_t levels_buf[TX_PAD_2D];
uint8_t *const levels = set_levels(levels_buf, width);
-
+#if CONFIG_AOM_QM
+ const TX_SIZE qm_tx_size = av1_get_adjusted_tx_size(tx_size);
+ const qm_val_t *iqmatrix =
+ IS_2D_TRANSFORM(tx_type)
+ ? pd->seg_iqmatrix[mbmi->segment_id][qm_tx_size]
+ : cm->giqmatrix[NUM_QM_LEVELS - 1][0][qm_tx_size];
+#else
+ const qm_val_t *iqmatrix = NULL;
+#endif
assert(width == (1 << bwl));
TxbInfo txb_info = {
qcoeff,
@@ -2092,7 +2109,8 @@
scan_order,
txb_ctx,
rdmult,
- &cm->coeff_ctx_table
+ &cm->coeff_ctx_table,
+ iqmatrix,
};
// Hash based trellis (hbt) speed feature: avoid expensive optimize_txb calls
diff --git a/av1/encoder/encodetxb.h b/av1/encoder/encodetxb.h
index 714b16c..ec4d04f 100644
--- a/av1/encoder/encodetxb.h
+++ b/av1/encoder/encodetxb.h
@@ -49,6 +49,7 @@
TXB_CTX *txb_ctx;
int64_t rdmult;
const LV_MAP_CTX_TABLE *coeff_ctx_table;
+ const qm_val_t *iqmatrix;
} TxbInfo;
typedef struct TxbCache {
