Fix a potential segfault when NSTEP is used on scaled reference This commit makes sure that search_sites_cfg is always update once per frame to avoid segfault due to using incorrect y_stride on scaled refrences. Change-Id: If11a7ea41be7880c3e179fe416ab13f641e8f7d7
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c index be39189..c6b3c88 100644 --- a/av1/encoder/encoder.c +++ b/av1/encoder/encoder.c
@@ -3556,18 +3556,30 @@ } static void init_motion_estimation(AV1_COMP *cpi) { - int y_stride = cpi->scaled_source.y_stride; - int y_stride_src = (cpi->oxcf.resize_mode || cpi->oxcf.superres_mode) - ? y_stride - : cpi->lookahead->buf->img.y_stride; + const int y_stride = cpi->scaled_source.y_stride; + const int y_stride_src = (cpi->oxcf.resize_mode || cpi->oxcf.superres_mode) + ? y_stride + : cpi->lookahead->buf->img.y_stride; + // Update if ss_cfg is uninitialized or the current frame has a new stride + const int should_update = !cpi->ss_cfg[SS_CFG_SRC].stride || + !cpi->ss_cfg[SS_CFG_LOOKAHEAD].stride || + (y_stride != cpi->ss_cfg[SS_CFG_SRC].stride); - if (cpi->sf.mv.search_method == NSTEP) { - av1_init3smotion_compensation(&cpi->ss_cfg[SS_CFG_SRC], y_stride); - av1_init3smotion_compensation(&cpi->ss_cfg[SS_CFG_LOOKAHEAD], y_stride_src); - } else if (cpi->sf.mv.search_method == DIAMOND) { + if (!should_update) { + return; + } + + if (cpi->sf.mv.search_method == DIAMOND) { av1_init_dsmotion_compensation(&cpi->ss_cfg[SS_CFG_SRC], y_stride); av1_init_dsmotion_compensation(&cpi->ss_cfg[SS_CFG_LOOKAHEAD], y_stride_src); + } else { + // Update the offsets in search_sites as y_stride can change due to scaled + // references. This update allows NSTEP to be used on scaled references as + // long as sf.mv.search_method is not DIAMOND. Currently in the codebae, + // sf.mv.search_method is never set to DIAMOND. + av1_init3smotion_compensation(&cpi->ss_cfg[SS_CFG_SRC], y_stride); + av1_init3smotion_compensation(&cpi->ss_cfg[SS_CFG_LOOKAHEAD], y_stride_src); } }
diff --git a/av1/encoder/mcomp.c b/av1/encoder/mcomp.c index a3f7d25..1a64d1a 100644 --- a/av1/encoder/mcomp.c +++ b/av1/encoder/mcomp.c
@@ -123,6 +123,7 @@ cfg->ss[0].mv.col = cfg->ss[0].mv.row = 0; cfg->ss[0].offset = 0; + cfg->stride = stride; for (len = MAX_FIRST_STEP; len > 0; len /= 2) { // Generate offsets for 4 search sites per step. @@ -144,6 +145,7 @@ cfg->ss[0].mv.col = cfg->ss[0].mv.row = 0; cfg->ss[0].offset = 0; + cfg->stride = stride; for (len = MAX_FIRST_STEP; len > 0; len /= 2) { // Generate offsets for 8 search sites per step.
diff --git a/av1/encoder/mcomp.h b/av1/encoder/mcomp.h index 71547da..de4d931 100644 --- a/av1/encoder/mcomp.h +++ b/av1/encoder/mcomp.h
@@ -47,6 +47,7 @@ search_site ss[8 * MAX_MVSEARCH_STEPS + 1]; int ss_count; int searches_per_step; + int stride; } search_site_config; typedef struct {