Add rd weighting for filtered modes. Previous rd weighting did not account for filtered modes so tended to slightly prefer filtered modes. Change-Id: Ia25e65099fb8832778192d9bfdcd4603eb3a8c2c
diff --git a/av1/encoder/intra_mode_search.c b/av1/encoder/intra_mode_search.c index 4e86b77..47f04f9 100644 --- a/av1/encoder/intra_mode_search.c +++ b/av1/encoder/intra_mode_search.c
@@ -83,6 +83,84 @@ 0x2a01, // SMOOTH_H_PRED: 0010 1010 0000 0001 0x3201 // PAETH_PRED: 0011 0010 0000 0001 }; + +DECLARE_ALIGNED(16, static const uint8_t, all_zeros[MAX_SB_SIZE]) = { 0 }; +DECLARE_ALIGNED(16, static const uint16_t, + highbd_all_zeros[MAX_SB_SIZE]) = { 0 }; +// Returns a factor to be applied to the RD value based on how well the +// reconstructed block variance matches the source variance. +static double intra_rd_variance_factor(const AV1_COMP *cpi, MACROBLOCK *x, + BLOCK_SIZE bs) { + MACROBLOCKD *xd = &x->e_mbd; + double variance_rd_factor = 1.0; + double src_var = 0.0; + double rec_var = 0.0; + double var_diff = 0.0; + double threshold = 1.0 - (0.25 * cpi->oxcf.speed); + unsigned int sse; + int i, j; + int right_overflow = + (xd->mb_to_right_edge < 0) ? ((-xd->mb_to_right_edge) >> 3) : 0; + int bottom_overflow = + (xd->mb_to_bottom_edge < 0) ? ((-xd->mb_to_bottom_edge) >> 3) : 0; + + const int bw = MI_SIZE * mi_size_wide[bs] - right_overflow; + const int bh = MI_SIZE * mi_size_high[bs] - bottom_overflow; + const int blocks = (bw * bh) / 16; + + for (i = 0; i < bh; i += 4) { + for (j = 0; j < bw; j += 4) { + if (is_cur_buf_hbd(xd)) { + src_var += + log(1.0 + cpi->ppi->fn_ptr[BLOCK_4X4].vf( + x->plane[0].src.buf + i * x->plane[0].src.stride + j, + x->plane[0].src.stride, + CONVERT_TO_BYTEPTR(highbd_all_zeros), 0, &sse) / + 16.0); + rec_var += log( + 1.0 + cpi->ppi->fn_ptr[BLOCK_4X4].vf( + xd->plane[0].dst.buf + i * xd->plane[0].dst.stride + j, + xd->plane[0].dst.stride, + CONVERT_TO_BYTEPTR(highbd_all_zeros), 0, &sse) / + 16.0); + } else { + src_var += + log(1.0 + cpi->ppi->fn_ptr[BLOCK_4X4].vf( + x->plane[0].src.buf + i * x->plane[0].src.stride + j, + x->plane[0].src.stride, all_zeros, 0, &sse) / + 16.0); + rec_var += log( + 1.0 + cpi->ppi->fn_ptr[BLOCK_4X4].vf( + xd->plane[0].dst.buf + i * xd->plane[0].dst.stride + j, + xd->plane[0].dst.stride, all_zeros, 0, &sse) / + 16.0); + } + } + } + src_var /= (double)blocks; + rec_var /= (double)blocks; + + // Dont allow 0 to prevent / 0 below. + src_var += 0.000001; + rec_var += 0.000001; + + if (src_var >= rec_var) { + var_diff = (src_var - rec_var); + if ((var_diff > 0.5) && (rec_var < threshold)) { + variance_rd_factor = 1.0 + ((var_diff * 2) / src_var); + } + } else { + var_diff = (rec_var - src_var); + if ((var_diff > 0.5) && (src_var < threshold)) { + variance_rd_factor = 1.0 + (var_diff / (2 * src_var)); + } + } + + // Limit adjustment; + variance_rd_factor = AOMMIN(3.0, variance_rd_factor); + + return variance_rd_factor; +} /*!\endcond */ /*!\brief Search for the best filter_intra mode when coding intra frame. @@ -148,6 +226,11 @@ intra_mode_info_cost_y(cpi, x, mbmi, bsize, mode_cost); this_rd = RDCOST(x->rdmult, this_rate, tokenonly_rd_stats.dist); + // Visual quality adjustment based on recon vs source variance. + if ((cpi->oxcf.mode == ALLINTRA) && (this_rd != INT64_MAX)) { + this_rd = (int64_t)(this_rd * intra_rd_variance_factor(cpi, x, bsize)); + } + // Collect mode stats for multiwinner mode processing const int txfm_search_done = 1; store_winner_mode_stats( @@ -1086,84 +1169,6 @@ return 1; } -DECLARE_ALIGNED(16, static const uint8_t, all_zeros[MAX_SB_SIZE]) = { 0 }; -DECLARE_ALIGNED(16, static const uint16_t, - highbd_all_zeros[MAX_SB_SIZE]) = { 0 }; -// Returns a factor to be applied to the RD value based on how well the -// reconstructed block variance matches the source variance. -static double intra_rd_variance_factor(const AV1_COMP *cpi, MACROBLOCK *x, - BLOCK_SIZE bs) { - MACROBLOCKD *xd = &x->e_mbd; - double variance_rd_factor = 1.0; - double src_var = 0.0; - double rec_var = 0.0; - double var_diff = 0.0; - double threshold = 1.0 - (0.25 * cpi->oxcf.speed); - unsigned int sse; - int i, j; - int right_overflow = - (xd->mb_to_right_edge < 0) ? ((-xd->mb_to_right_edge) >> 3) : 0; - int bottom_overflow = - (xd->mb_to_bottom_edge < 0) ? ((-xd->mb_to_bottom_edge) >> 3) : 0; - - const int bw = MI_SIZE * mi_size_wide[bs] - right_overflow; - const int bh = MI_SIZE * mi_size_high[bs] - bottom_overflow; - const int blocks = (bw * bh) / 16; - - for (i = 0; i < bh; i += 4) { - for (j = 0; j < bw; j += 4) { - if (is_cur_buf_hbd(xd)) { - src_var += - log(1.0 + cpi->ppi->fn_ptr[BLOCK_4X4].vf( - x->plane[0].src.buf + i * x->plane[0].src.stride + j, - x->plane[0].src.stride, - CONVERT_TO_BYTEPTR(highbd_all_zeros), 0, &sse) / - 16.0); - rec_var += log( - 1.0 + cpi->ppi->fn_ptr[BLOCK_4X4].vf( - xd->plane[0].dst.buf + i * xd->plane[0].dst.stride + j, - xd->plane[0].dst.stride, - CONVERT_TO_BYTEPTR(highbd_all_zeros), 0, &sse) / - 16.0); - } else { - src_var += - log(1.0 + cpi->ppi->fn_ptr[BLOCK_4X4].vf( - x->plane[0].src.buf + i * x->plane[0].src.stride + j, - x->plane[0].src.stride, all_zeros, 0, &sse) / - 16.0); - rec_var += log( - 1.0 + cpi->ppi->fn_ptr[BLOCK_4X4].vf( - xd->plane[0].dst.buf + i * xd->plane[0].dst.stride + j, - xd->plane[0].dst.stride, all_zeros, 0, &sse) / - 16.0); - } - } - } - src_var /= (double)blocks; - rec_var /= (double)blocks; - - // Dont allow 0 to prevent / 0 below. - src_var += 0.000001; - rec_var += 0.000001; - - if (src_var >= rec_var) { - var_diff = (src_var - rec_var); - if ((var_diff > 0.5) && (rec_var < threshold)) { - variance_rd_factor = 1.0 + ((var_diff * 2) / src_var); - } - } else { - var_diff = (rec_var - src_var); - if ((var_diff > 0.5) && (src_var < threshold)) { - variance_rd_factor = 1.0 + (var_diff / (2 * src_var)); - } - } - - // Limit adjustment; - variance_rd_factor = AOMMIN(3.0, variance_rd_factor); - - return variance_rd_factor; -} - // Finds the best non-intrabc mode on an intra frame. int64_t av1_rd_pick_intra_sby_mode(const AV1_COMP *const cpi, MACROBLOCK *x, int *rate, int *rate_tokenonly,