Fix integer cost list calculation
Integer cost_list is not consistent on whether mv_cost is included. In
some scenarios, mv_cost can even be added twice. Furthermore, the wrong
mv_cost is used for the center mv in calc_int_cost_list.
This commit fixes both of these inconsistencies.
Change-Id: I678a4fe3708acd369dcca3723d038c4756918eb8
diff --git a/av1/encoder/mcomp.c b/av1/encoder/mcomp.c
index 2450f947..a72c07e 100644
--- a/av1/encoder/mcomp.c
+++ b/av1/encoder/mcomp.c
@@ -348,14 +348,16 @@
// fullpixel motion search. The resulting list can be used to speed up subpel
// motion search later.
static INLINE void calc_int_cost_list(const MACROBLOCK *x,
- const MV *const ref_mv, int sadpb,
+ const MV *const ref_mv,
const aom_variance_fn_ptr_t *fn_ptr,
const FULLPEL_MV *best_mv,
int *cost_list) {
- static const MV neighbors[4] = { { 0, -1 }, { 1, 0 }, { 0, 1 }, { -1, 0 } };
+ static const FULLPEL_MV neighbors[4] = {
+ { 0, -1 }, { 1, 0 }, { 0, 1 }, { -1, 0 }
+ };
const struct buf_2d *const what = &x->plane[0].src;
const struct buf_2d *const in_what = &x->e_mbd.plane[0].pre[0];
- const FULLPEL_MV full_ref_mv = get_fullmv_from_mv(ref_mv);
+ const MV sub_best_mv = get_mv_from_fullmv(best_mv);
const int br = best_mv->row;
const int bc = best_mv->col;
const MV_COST_TYPE mv_cost_type = x->mv_cost_type;
@@ -364,7 +366,9 @@
cost_list[0] =
fn_ptr->vf(what->buf, what->stride, get_buf_from_mv(in_what, best_mv),
in_what->stride, &sse) +
- mvsad_err_cost(x, best_mv, &full_ref_mv, sadpb);
+ mv_err_cost(&sub_best_mv, ref_mv, x->nmv_vec_cost,
+ CONVERT_TO_CONST_MVCOST(x->mv_cost_stack), x->errorperbit,
+ mv_cost_type);
if (check_bounds(&x->mv_limits, br, bc, 1)) {
for (int i = 0; i < 4; i++) {
const FULLPEL_MV neighbor_mv = { br + neighbors[i].row,
@@ -402,7 +406,9 @@
const aom_variance_fn_ptr_t *fn_ptr,
const FULLPEL_MV *best_mv, int *cost_list,
const int bestsad) {
- static const MV neighbors[4] = { { 0, -1 }, { 1, 0 }, { 0, 1 }, { -1, 0 } };
+ static const FULLPEL_MV neighbors[4] = {
+ { 0, -1 }, { 1, 0 }, { 0, 1 }, { -1, 0 }
+ };
const struct buf_2d *const what = &x->plane[0].src;
const struct buf_2d *const in_what = &x->e_mbd.plane[0].pre[0];
const int br = best_mv->row;
@@ -411,6 +417,8 @@
if (cost_list[0] == INT_MAX) {
cost_list[0] = bestsad;
+ cost_list[0] += mvsad_err_cost(x, best_mv, &full_ref_mv, sadpb);
+
if (check_bounds(&x->mv_limits, br, bc, 1)) {
for (int i = 0; i < 4; i++) {
const FULLPEL_MV this_mv = { br + neighbors[i].row,
@@ -418,21 +426,26 @@
cost_list[i + 1] =
fn_ptr->sdf(what->buf, what->stride,
get_buf_from_mv(in_what, &this_mv), in_what->stride);
+ cost_list[i + 1] += mvsad_err_cost(x, &this_mv, &full_ref_mv, sadpb);
}
} else {
for (int i = 0; i < 4; i++) {
const FULLPEL_MV this_mv = { br + neighbors[i].row,
bc + neighbors[i].col };
- if (!av1_is_fullmv_in_range(&x->mv_limits, this_mv))
+ if (!av1_is_fullmv_in_range(&x->mv_limits, this_mv)) {
cost_list[i + 1] = INT_MAX;
- else
+ } else {
cost_list[i + 1] =
fn_ptr->sdf(what->buf, what->stride,
get_buf_from_mv(in_what, &this_mv), in_what->stride);
+ cost_list[i + 1] += mvsad_err_cost(x, &this_mv, &full_ref_mv, sadpb);
+ }
}
}
} else {
if (x->mv_cost_type != MV_COST_NONE) {
+ cost_list[0] += mvsad_err_cost(x, best_mv, &full_ref_mv, sadpb);
+
for (int i = 0; i < 4; i++) {
const FULLPEL_MV this_mv = { br + neighbors[i].row,
bc + neighbors[i].col };
@@ -446,9 +459,11 @@
#define CHECK_BETTER \
if (thissad < bestsad) { \
+ int tmp_thissad = thissad; \
if (use_mvcost) \
thissad += mvsad_err_cost(x, &this_mv, &full_ref_mv, sad_per_bit); \
if (thissad < bestsad) { \
+ raw_bestsad = tmp_thissad; \
bestsad = thissad; \
best_site = i; \
} \
@@ -472,7 +487,7 @@
const struct buf_2d *const in_what = &xd->plane[0].pre[0];
const int last_is_4 = num_candidates[0] == 4;
int br, bc;
- int bestsad = INT_MAX;
+ int bestsad = INT_MAX, raw_bestsad = INT_MAX;
int thissad;
int k = -1;
const FULLPEL_MV full_ref_mv = get_fullmv_from_mv(ref_mv);
@@ -489,9 +504,10 @@
}
// Work out the start point for the search
- bestsad = vfp->sdf(what->buf, what->stride,
- get_buf_from_mv(in_what, start_mv), in_what->stride) +
- mvsad_err_cost(x, start_mv, &full_ref_mv, sad_per_bit);
+ raw_bestsad = vfp->sdf(what->buf, what->stride,
+ get_buf_from_mv(in_what, start_mv), in_what->stride);
+ bestsad =
+ raw_bestsad + mvsad_err_cost(x, start_mv, &full_ref_mv, sad_per_bit);
// Search all possible scales up to the search param around the center point
// pick the scale of the point that is best as the starting scale of
@@ -616,7 +632,7 @@
// Note: If we enter the if below, then cost_list must be non-NULL.
if (s == 0) {
- cost_list[0] = bestsad;
+ cost_list[0] = raw_bestsad;
if (!do_init_search || s != best_init_s) {
if (check_bounds(&x->mv_limits, br, bc, 1 << s)) {
for (i = 0; i < num_candidates[s]; i++) {
@@ -653,7 +669,7 @@
next_chkpts_indices[2] = (k == num_candidates[s] - 1) ? 0 : k + 1;
cost_list[1] = cost_list[2] = cost_list[3] = cost_list[4] = INT_MAX;
cost_list[((k + 2) % 4) + 1] = cost_list[0];
- cost_list[0] = bestsad;
+ cost_list[0] = raw_bestsad;
if (check_bounds(&x->mv_limits, br, bc, 1 << s)) {
for (i = 0; i < PATTERN_CANDIDATES_REF; i++) {
@@ -704,7 +720,7 @@
calc_int_sad_list(x, ref_mv, sad_per_bit, vfp, &best_int_mv, cost_list,
bestsad);
} else {
- calc_int_cost_list(x, ref_mv, sad_per_bit, vfp, &best_int_mv, cost_list);
+ calc_int_cost_list(x, ref_mv, vfp, &best_int_mv, cost_list);
}
}
x->best_mv.as_mv.row = br;
@@ -1153,8 +1169,7 @@
// Return cost list.
if (cost_list) {
- calc_int_cost_list(x, ref_mv, sadpb, fn_ptr, &x->best_mv.as_fullmv,
- cost_list);
+ calc_int_cost_list(x, ref_mv, fn_ptr, &x->best_mv.as_fullmv, cost_list);
}
return bestsme;
}
@@ -1212,7 +1227,7 @@
// Return cost list.
if (cost_list) {
- calc_int_cost_list(x, ref_mv, sadpb, fn_ptr, best_mv, cost_list);
+ calc_int_cost_list(x, ref_mv, fn_ptr, best_mv, cost_list);
}
return bestsme;
}
