commit 687c56424b75de044f29bc55d760fafd1da9bdb4
author Paul Wilkins <paulwilkins@google.com> Mon Oct 04 15:39:33 2021 +0100
committer Paul Wilkins <paulwilkins@google.com> Mon Oct 11 14:16:51 2021 +0000
tree 19e9ea37f2169ae2e5180b69dbe8b0eafc657ca0
parent aa6113ebbc5880d3f57ab57220ccc9f3923d9e13

Change to partition and mode search weighting.

The use of an integer divisor was truncating values.
Most significantly small variance values were all being
truncated to 0.

Adjustment of thresholds along with the higher precision
slightly reduces artifacts on one problem clip.

Change-Id: I7948b670144909fb6445102a0dc536b1b3741f2c

av1/encoder/aq_variance.c

diff --git a/av1/encoder/aq_variance.c b/av1/encoder/aq_variance.c
index 05dd33a..3273ef8 100644
--- a/av1/encoder/aq_variance.c
+++ b/av1/encoder/aq_variance.c
@@ -124,13 +124,13 @@
                           x->plane[0].src.buf + i * x->plane[0].src.stride + j,
                           x->plane[0].src.stride,
                           CONVERT_TO_BYTEPTR(av1_highbd_all_zeros), 0, &sse) /
-                          16);
+                          16.0);
       } else {
         var +=
             log(1.0 + cpi->ppi->fn_ptr[BLOCK_4X4].vf(
                           x->plane[0].src.buf + i * x->plane[0].src.stride + j,
                           x->plane[0].src.stride, av1_all_zeros, 0, &sse) /
-                          16);
+                          16.0);
       }
     }
   }

av1/encoder/intra_mode_search.c

diff --git a/av1/encoder/intra_mode_search.c b/av1/encoder/intra_mode_search.c
index 26cea2d..4e86b77 100644
--- a/av1/encoder/intra_mode_search.c
+++ b/av1/encoder/intra_mode_search.c
@@ -1118,24 +1118,24 @@
                           x->plane[0].src.buf + i * x->plane[0].src.stride + j,
                           x->plane[0].src.stride,
                           CONVERT_TO_BYTEPTR(highbd_all_zeros), 0, &sse) /
-                          16);
+                          16.0);
         rec_var += log(
             1.0 + cpi->ppi->fn_ptr[BLOCK_4X4].vf(
                       xd->plane[0].dst.buf + i * xd->plane[0].dst.stride + j,
                       xd->plane[0].dst.stride,
                       CONVERT_TO_BYTEPTR(highbd_all_zeros), 0, &sse) /
-                      16);
+                      16.0);
       } else {
         src_var +=
             log(1.0 + cpi->ppi->fn_ptr[BLOCK_4X4].vf(
                           x->plane[0].src.buf + i * x->plane[0].src.stride + j,
                           x->plane[0].src.stride, all_zeros, 0, &sse) /
-                          16);
+                          16.0);
         rec_var += log(
             1.0 + cpi->ppi->fn_ptr[BLOCK_4X4].vf(
                       xd->plane[0].dst.buf + i * xd->plane[0].dst.stride + j,
                       xd->plane[0].dst.stride, all_zeros, 0, &sse) /
-                      16);
+                      16.0);
       }
     }
   }

av1/encoder/partition_search.c

diff --git a/av1/encoder/partition_search.c b/av1/encoder/partition_search.c
index bd8a099..e4d5671 100644
--- a/av1/encoder/partition_search.c
+++ b/av1/encoder/partition_search.c
@@ -4505,7 +4505,7 @@
   const int bh = MI_SIZE * mi_size_high[bs] - bottom_overflow;
 
   // Initialize min to a large value and max to 0 at
-  *var_min = 10.0;
+  *var_min = 99.0;
   *var_max = 0.0;
 
   for (i = 0; i < bh; i += 4) {
@@ -4516,13 +4516,13 @@
                           x->plane[0].src.buf + i * x->plane[0].src.stride + j,
                           x->plane[0].src.stride,
                           CONVERT_TO_BYTEPTR(highbd_all_zeros), 0, &sse) /
-                          16);
+                          16.0);
       } else {
         var =
             log(1.0 + cpi->ppi->fn_ptr[BLOCK_4X4].vf(
                           x->plane[0].src.buf + i * x->plane[0].src.stride + j,
                           x->plane[0].src.stride, all_zeros, 0, &sse) /
-                          16);
+                          16.0);
       }
       *var_min = AOMMIN(*var_min, var);
       *var_max = AOMMAX(*var_max, var);
@@ -4716,7 +4716,7 @@
     double var_min, var_max;
     log_sub_block_var(cpi, x, bsize, &var_min, &var_max);
 
-    if ((var_min < 0.5) && ((var_max - var_min) > 3.0)) {
+    if ((var_min < 0.272) && ((var_max - var_min) > 3.0)) {
       part_search_state.partition_none_allowed = 0;
       part_search_state.terminate_partition_search = 0;
       part_search_state.do_square_split = 1;