commit 081783dc671240054acdee61d8ff8c9ff064e8de
author Sarah Parker <sarahparker@google.com> Tue Oct 18 16:27:04 2016 -0700
committer Sarah Parker <sarahparker@google.com> Tue Oct 18 16:43:54 2016 -0700
tree 38d4dd2c2d7650eb7a4337fe42ccae844e46f1e9
parent 40f1d487adde872b080a6bea52b917fc916cf7ac

Add clamping to parameter search

This fixes mismatches due to overflowing low precision parameters.

Change-Id: If34e39ca7ab0adc9688d46b0e8ed62cbb6fdaff0

av1/encoder/encodeframe.c

diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 773e368..3f93d53 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -4447,7 +4447,9 @@
     best_param = curr_param;
     for (i = 0; i < n_refinements; i++) {
       // look to the left
-      *param = curr_param - step;
+      *param =
+          (int16_t)clamp(curr_param - step, p < 2 ? GM_TRANS_MIN : GM_ALPHA_MIN,
+                         p < 2 ? GM_TRANS_MAX : GM_ALPHA_MAX);
       step_error =
           av1_warp_erroradv(wm,
 #if CONFIG_AOM_HIGHBITDEPTH
@@ -4464,7 +4466,9 @@
       }
 
       // look to the right
-      *param = curr_param + step;
+      *param =
+          (int16_t)clamp(curr_param + step, p < 2 ? GM_TRANS_MIN : GM_ALPHA_MIN,
+                         p < 2 ? GM_TRANS_MAX : GM_ALPHA_MAX);
       step_error =
           av1_warp_erroradv(wm,
 #if CONFIG_AOM_HIGHBITDEPTH