av1_encoder : sharpness affect
This patch changes the effect of the sharpness parameter to the encoder.
It no longer affects the thresholds in the loop filter. Now it acts this
way:
sharpness
0 => acts as before.
1 => stops performing eob and skip block optimization
1-7 => changes rdmult in trellis to favor lower distortion
The effect is images / video include more detail at the same bitrate
but sometimes also more artifacts.
+ when using variance based aq : lower segment ids
also favor lower distortion
Change-Id: Ibfc95bfea3ac1312df01c954f98036bb98b58ff9
diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c
index 5a5b3be..69c8eda 100644
--- a/av1/encoder/encodemb.c
+++ b/av1/encoder/encodemb.c
@@ -115,7 +115,7 @@
(void)fast_mode;
return av1_optimize_txb_new(cpi, mb, plane, blk_row, blk_col, block, tx_size,
- &txb_ctx, rate_cost);
+ &txb_ctx, rate_cost, cpi->oxcf.sharpness);
}
typedef enum QUANT_FUNC {
diff --git a/av1/encoder/encodetxb.c b/av1/encoder/encodetxb.c
index d35f7cc..1aadeb1 100644
--- a/av1/encoder/encodetxb.c
+++ b/av1/encoder/encodetxb.c
@@ -1423,7 +1423,7 @@
int dc_sign_ctx, int64_t rdmult, int shift, const int16_t *dequant,
const int16_t *scan, const LV_MAP_EOB_COST *txb_eob_costs,
const LV_MAP_COEFF_COST *txb_costs, const tran_low_t *tcoeff,
- tran_low_t *qcoeff, tran_low_t *dqcoeff, uint8_t *levels) {
+ tran_low_t *qcoeff, tran_low_t *dqcoeff, uint8_t *levels, int sharpness) {
const int dqv = dequant[si != 0];
assert(si != *eob - 1);
const int ci = scan[si];
@@ -1498,7 +1498,7 @@
dist = dist_low;
}
- if (rd_new_eob < rd) {
+ if (sharpness == 0 && rd_new_eob < rd) {
for (int ni = 0; ni < *nz_num; ++ni) {
int last_ci = nz_ci[ni];
// levels[get_padded_idx(last_ci, bwl)] = 0;
@@ -1534,10 +1534,11 @@
static INLINE void update_skip(int *accu_rate, int64_t accu_dist, int *eob,
int nz_num, int *nz_ci, int64_t rdmult,
int skip_cost, int non_skip_cost,
- tran_low_t *qcoeff, tran_low_t *dqcoeff) {
+ tran_low_t *qcoeff, tran_low_t *dqcoeff,
+ int sharpness) {
const int64_t rd = RDCOST(rdmult, *accu_rate + non_skip_cost, accu_dist);
const int64_t rd_new_eob = RDCOST(rdmult, skip_cost, 0);
- if (rd_new_eob < rd) {
+ if (sharpness == 0 && rd_new_eob < rd) {
for (int i = 0; i < nz_num; ++i) {
const int ci = nz_ci[i];
qcoeff[ci] = 0;
@@ -1552,7 +1553,7 @@
int av1_optimize_txb_new(const struct AV1_COMP *cpi, MACROBLOCK *x, int plane,
int blk_row, int blk_col, int block, TX_SIZE tx_size,
- TXB_CTX *txb_ctx, int *rate_cost) {
+ TXB_CTX *txb_ctx, int *rate_cost, int sharpness) {
const AV1_COMMON *cm = &cpi->common;
MACROBLOCKD *xd = &x->e_mbd;
const PLANE_TYPE plane_type = get_plane_type(plane);
@@ -1583,10 +1584,13 @@
const int64_t rdmult =
((x->rdmult * plane_rd_mult[is_inter][plane_type] << (2 * (xd->bd - 8))) +
2) >>
- 2;
+ (sharpness + (cpi->oxcf.aq_mode == VARIANCE_AQ && mbmi->segment_id < 4
+ ? 7 - mbmi->segment_id
+ : 2));
uint8_t levels_buf[TX_PAD_2D];
uint8_t *const levels = set_levels(levels_buf, width);
+
av1_txb_init_levels(qcoeff, width, height, levels);
// TODO(angirbird): check iqmatrix
@@ -1633,7 +1637,7 @@
tx_size, tx_class_literal, bwl, height, \
txb_ctx->dc_sign_ctx, rdmult, shift, dequant, scan, \
txb_eob_costs, txb_costs, tcoeff, qcoeff, dqcoeff, \
- levels); \
+ levels, sharpness); \
} \
break;
switch (tx_class) {
@@ -1646,7 +1650,7 @@
if (si == -1 && nz_num <= max_nz_num) {
update_skip(&accu_rate, accu_dist, &eob, nz_num, nz_ci, rdmult, skip_cost,
- non_skip_cost, qcoeff, dqcoeff);
+ non_skip_cost, qcoeff, dqcoeff, sharpness);
}
#define UPDATE_COEFF_SIMPLE_CASE(tx_class_literal) \
diff --git a/av1/encoder/encodetxb.h b/av1/encoder/encodetxb.h
index 8eba04d..62a6017 100644
--- a/av1/encoder/encodetxb.h
+++ b/av1/encoder/encodetxb.h
@@ -79,7 +79,7 @@
void hbt_destroy();
int av1_optimize_txb_new(const AV1_COMP *cpi, MACROBLOCK *x, int plane,
int blk_row, int blk_col, int block, TX_SIZE tx_size,
- TXB_CTX *txb_ctx, int *rate_cost);
+ TXB_CTX *txb_ctx, int *rate_cost, int sharpness);
#ifdef __cplusplus
}
#endif
diff --git a/av1/encoder/picklpf.c b/av1/encoder/picklpf.c
index 66f42fa..5f802a7 100644
--- a/av1/encoder/picklpf.c
+++ b/av1/encoder/picklpf.c
@@ -194,7 +194,7 @@
struct loopfilter *const lf = &cm->lf;
(void)sd;
- lf->sharpness_level = cm->frame_type == KEY_FRAME ? 0 : cpi->oxcf.sharpness;
+ lf->sharpness_level = 0;
if (method == LPF_PICK_MINIMAL_LPF) {
lf->filter_level[0] = 0;
