commit ca52fb5f369c825420b7f6483da602753bd6183c
author Sarah Parker <sarahparker@google.com> Tue Nov 28 14:25:43 2017 -0800
committer Sarah Parker <sarahparker@google.com> Wed Nov 29 21:02:33 2017 +0000
tree fcd315cbc94e8971d51d2d8733c9f5b09720386c
parent abdf65564851796795dbdbd0e0d21c1f7d65c9b1

Make new_quant bin sizes uniform

This gives a -0.05% change in performance over the regular
quantizer. The new_quant bins will remain uniform as we
experiment with different parameters.

Change-Id: Ic92c74fb1183cf1c2460bbcf1e223452f46d8210

av1/common/quant_common.c

diff --git a/av1/common/quant_common.c b/av1/common/quant_common.c
index a67d649..998ae36 100644
--- a/av1/common/quant_common.c
+++ b/av1/common/quant_common.c
@@ -42,28 +42,28 @@
       { { 64, 128, 128 }, 0 },  // band 5
   },
   {
-      { { 64, 128, 128 }, 4 },   // dc, band 0
-      { { 64, 128, 128 }, 6 },   // band 1
-      { { 64, 128, 128 }, 8 },   // band 2
-      { { 64, 128, 128 }, 10 },  // band 3
-      { { 72, 128, 128 }, 12 },  // band 4
-      { { 80, 128, 128 }, 14 }   // band 5
+      { { 64, 128, 128 }, 0 },  // dc, band 0
+      { { 64, 128, 128 }, 0 },  // band 1
+      { { 64, 128, 128 }, 0 },  // band 2
+      { { 64, 128, 128 }, 0 },  // band 3
+      { { 64, 128, 128 }, 0 },  // band 4
+      { { 64, 128, 128 }, 0 }   // band 5
   },
   {
-      { { 64, 128, 128 }, 6 },   // dc, band 0
-      { { 64, 128, 128 }, 8 },   // band 1
-      { { 64, 128, 128 }, 10 },  // band 2
-      { { 64, 128, 128 }, 12 },  // band 3
-      { { 72, 128, 128 }, 14 },  // band 4
-      { { 80, 128, 128 }, 16 }   // band 5
+      { { 64, 128, 128 }, 0 },  // dc, band 0
+      { { 64, 128, 128 }, 0 },  // band 1
+      { { 64, 128, 128 }, 0 },  // band 2
+      { { 64, 128, 128 }, 0 },  // band 3
+      { { 64, 128, 128 }, 0 },  // band 4
+      { { 64, 128, 128 }, 0 }   // band 5
   },
   {
-      { { 64, 128, 128 }, 8 },   // dc, band 0
-      { { 64, 128, 128 }, 10 },  // band 1
-      { { 64, 128, 128 }, 12 },  // band 2
-      { { 72, 128, 128 }, 14 },  // band 3
-      { { 76, 128, 128 }, 16 },  // band 4
-      { { 80, 128, 128 }, 18 }   // band 5
+      { { 64, 128, 128 }, 0 },  // dc, band 0
+      { { 64, 128, 128 }, 0 },  // band 1
+      { { 64, 128, 128 }, 0 },  // band 2
+      { { 64, 128, 128 }, 0 },  // band 3
+      { { 64, 128, 128 }, 0 },  // band 4
+      { { 64, 128, 128 }, 0 }   // band 5
   }
 };
 

av1/encoder/av1_quantize.c

diff --git a/av1/encoder/av1_quantize.c b/av1/encoder/av1_quantize.c
index de43f4e..11621c4 100644
--- a/av1/encoder/av1_quantize.c
+++ b/av1/encoder/av1_quantize.c
@@ -101,7 +101,7 @@
   int i, q;
   int tmp = clamp(abs_coeff, INT16_MIN, INT16_MAX);
   for (i = 0; i < NUQ_KNOTS; i++) {
-    if (tmp < cuml_bins_ptr[i]) {
+    if (tmp <= cuml_bins_ptr[i]) {
       q = i;
       break;
     }
@@ -131,7 +131,7 @@
   int i, q;
   int tmp = clamp(abs_coeff, INT16_MIN, INT16_MAX);
   for (i = 0; i < NUQ_KNOTS; i++) {
-    if (tmp < ROUND_POWER_OF_TWO(cuml_bins_ptr[i], logsizeby16)) {
+    if (tmp <= ROUND_POWER_OF_TWO(cuml_bins_ptr[i], logsizeby16)) {
       q = i;
       break;
     }