commit 836b69b9a0a96cc7bb1e9eaec441150b0d64bba6
author chiyotsai <chiyotsai@google.com> Tue Apr 09 13:41:24 2019 -0700
committer Chi Yo Tsai <chiyotsai@google.com> Wed Apr 10 20:44:18 2019 +0000
tree 47595ead0af241b7367037dcf93886552d088b1e
parent 6fdf93469276de7cce28a2f98c684066596c532d

Fix a potential segfault when NSTEP is used on scaled reference

This commit makes sure that search_sites_cfg is always update once per
frame to avoid segfault due to using incorrect y_stride on scaled
refrences.

Change-Id: If11a7ea41be7880c3e179fe416ab13f641e8f7d7

av1/encoder/encoder.c

diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index be39189..c6b3c88 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -3556,18 +3556,30 @@
 }
 
 static void init_motion_estimation(AV1_COMP *cpi) {
-  int y_stride = cpi->scaled_source.y_stride;
-  int y_stride_src = (cpi->oxcf.resize_mode || cpi->oxcf.superres_mode)
-                         ? y_stride
-                         : cpi->lookahead->buf->img.y_stride;
+  const int y_stride = cpi->scaled_source.y_stride;
+  const int y_stride_src = (cpi->oxcf.resize_mode || cpi->oxcf.superres_mode)
+                               ? y_stride
+                               : cpi->lookahead->buf->img.y_stride;
+  // Update if ss_cfg is uninitialized or the current frame has a new stride
+  const int should_update = !cpi->ss_cfg[SS_CFG_SRC].stride ||
+                            !cpi->ss_cfg[SS_CFG_LOOKAHEAD].stride ||
+                            (y_stride != cpi->ss_cfg[SS_CFG_SRC].stride);
 
-  if (cpi->sf.mv.search_method == NSTEP) {
-    av1_init3smotion_compensation(&cpi->ss_cfg[SS_CFG_SRC], y_stride);
-    av1_init3smotion_compensation(&cpi->ss_cfg[SS_CFG_LOOKAHEAD], y_stride_src);
-  } else if (cpi->sf.mv.search_method == DIAMOND) {
+  if (!should_update) {
+    return;
+  }
+
+  if (cpi->sf.mv.search_method == DIAMOND) {
     av1_init_dsmotion_compensation(&cpi->ss_cfg[SS_CFG_SRC], y_stride);
     av1_init_dsmotion_compensation(&cpi->ss_cfg[SS_CFG_LOOKAHEAD],
                                    y_stride_src);
+  } else {
+    // Update the offsets in search_sites as y_stride can change due to scaled
+    // references. This update allows NSTEP to be used on scaled references as
+    // long as sf.mv.search_method is not DIAMOND. Currently in the codebae,
+    // sf.mv.search_method is never set to DIAMOND.
+    av1_init3smotion_compensation(&cpi->ss_cfg[SS_CFG_SRC], y_stride);
+    av1_init3smotion_compensation(&cpi->ss_cfg[SS_CFG_LOOKAHEAD], y_stride_src);
   }
 }
 

av1/encoder/mcomp.c

diff --git a/av1/encoder/mcomp.c b/av1/encoder/mcomp.c
index a3f7d25..1a64d1a 100644
--- a/av1/encoder/mcomp.c
+++ b/av1/encoder/mcomp.c
@@ -123,6 +123,7 @@
 
   cfg->ss[0].mv.col = cfg->ss[0].mv.row = 0;
   cfg->ss[0].offset = 0;
+  cfg->stride = stride;
 
   for (len = MAX_FIRST_STEP; len > 0; len /= 2) {
     // Generate offsets for 4 search sites per step.
@@ -144,6 +145,7 @@
 
   cfg->ss[0].mv.col = cfg->ss[0].mv.row = 0;
   cfg->ss[0].offset = 0;
+  cfg->stride = stride;
 
   for (len = MAX_FIRST_STEP; len > 0; len /= 2) {
     // Generate offsets for 8 search sites per step.

av1/encoder/mcomp.h

diff --git a/av1/encoder/mcomp.h b/av1/encoder/mcomp.h
index 71547da..de4d931 100644
--- a/av1/encoder/mcomp.h
+++ b/av1/encoder/mcomp.h
@@ -47,6 +47,7 @@
   search_site ss[8 * MAX_MVSEARCH_STEPS + 1];
   int ss_count;
   int searches_per_step;
+  int stride;
 } search_site_config;
 
 typedef struct {