commit 9057858f246df9bc6215dca8eaa41c20f84ef5e6
author Rachel Barker <rachelbarker@google.com> Wed May 31 21:41:08 2023 +0000
committer Rachel Barker <rachelbarker@google.com> Wed Jun 21 16:21:03 2023 +0000
tree 0465dec63716bc775fad808fa594ed8e850236a8
parent 9da597672e0072ab3957fbfae2581beb2b2adb43

Fix interaction of WARP_DELTA and EXT_WARP_FILTER

Change WARP_DELTA logic so that it handles the edges of the model
range allowed by av1_reduce_warp_model().

No change of output when CONFIG_EXT_WARP_FILTER == 0; fixes some rare
mismatches when CONFIG_EXT_WARP_FILTER == 1

av1/encoder/bitstream.c

diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index a07a261..d861e7a 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -448,6 +448,9 @@
   int coded_value = (value / WARP_DELTA_STEP) + WARP_DELTA_CODED_MAX;
   assert(0 <= coded_value && coded_value < WARP_DELTA_NUM_SYMBOLS);
 
+  // Check that the decoder will receive the correct value
+  assert((coded_value - WARP_DELTA_CODED_MAX) * WARP_DELTA_STEP == value);
+
   aom_write_symbol(w, coded_value,
                    xd->tile_ctx->warp_delta_param_cdf[index_type],
                    WARP_DELTA_NUM_SYMBOLS);

av1/encoder/mcomp.c

diff --git a/av1/encoder/mcomp.c b/av1/encoder/mcomp.c
index 601e6b5..4687d81 100644
--- a/av1/encoder/mcomp.c
+++ b/av1/encoder/mcomp.c
@@ -5559,16 +5559,36 @@
     }
 
     for (int param_index = 2; param_index < 4; param_index++) {
+      // Calculate the same range as av1_reduce_warp_model(), as we should
+      // reject any model which lies outside the clamping range.
+      int offset = (param_index == 2 || param_index == 5)
+                       ? (1 << WARPEDMODEL_PREC_BITS)
+                       : 0;
+#if CONFIG_EXT_WARP_FILTER
+      const int max_value =
+          (1 << (WARPEDMODEL_PREC_BITS - 1)) - (1 << WARP_PARAM_REDUCE_BITS);
+      const int min_value = -max_value;
+#else
+      const int min_value = -(1 << (WARPEDMODEL_PREC_BITS - 1));
+      const int max_value =
+          (1 << (WARPEDMODEL_PREC_BITS - 1)) - (1 << WARP_PARAM_REDUCE_BITS);
+#endif  // CONFIG_EXT_WARP_FILTER
+
       // Try increasing the parameter
       *params = best_wm_params;
       params->wmmat[param_index] += step_size;
+      assert(params->wmmat[param_index] - offset >= min_value);
       delta = params->wmmat[param_index] - base_params.wmmat[param_index];
       if (abs(delta) > WARP_DELTA_MAX) {
+        // Reject values we can't encode
+        inc_rd = UINT64_MAX;
+      } else if (params->wmmat[param_index] - offset > max_value) {
+        // Reject models which are outside the range allowed by
+        // av1_reduce_warp_model()
         inc_rd = UINT64_MAX;
       } else {
         params->wmmat[4] = -params->wmmat[3];
         params->wmmat[5] = params->wmmat[2];
-        av1_reduce_warp_model(params);
         valid = av1_get_shear_params(params);
         params->invalid = !valid;
         if (valid) {
@@ -5589,13 +5609,18 @@
       // Try decreasing the parameter
       *params = best_wm_params;
       params->wmmat[param_index] -= step_size;
+      assert(params->wmmat[param_index] - offset <= max_value);
       delta = params->wmmat[param_index] - base_params.wmmat[param_index];
       if (abs(delta) > WARP_DELTA_MAX) {
+        // Reject values we can't encode
+        dec_rd = UINT64_MAX;
+      } else if (params->wmmat[param_index] - offset < min_value) {
+        // Reject models which are outside the range allowed by
+        // av1_reduce_warp_model()
         dec_rd = UINT64_MAX;
       } else {
         params->wmmat[4] = -params->wmmat[3];
         params->wmmat[5] = params->wmmat[2];
-        av1_reduce_warp_model(params);
         valid = av1_get_shear_params(params);
         params->invalid = !valid;
         if (valid) {