commit 60f2a229afa9f35e7918dd76068748983e515f3a
author Monty Montgomery <cmontgomery@mozilla.com> Wed Nov 01 19:48:38 2017 -0400
committer Christopher Montgomery <cmontgomery@mozilla.com> Mon Nov 20 14:40:24 2017 +0000
tree 877d80b278b8d45c6a5e6a1a28acfc26764c9c51
parent 605d63f3adf5a098ca16b3645e0f95d1dd3c129d

Add Daala TX fixed-coeff-depth capability to quantization

This patch completes the work to add fixed-depth TX domain support to
the quantization and dequantization code.  At present, it is active but
configured to behave identically to current AV1 master as RDO and TX
have not yet been updated to also support this functionality.

subset-1:
monty-rest-of-stack-noshift-s1@2017-11-13T14:37:42.541Z ->
 monty-rest-of-stack-quant-s1@2017-11-13T14:38:43.774Z

  PSNR | PSNR Cb | PSNR Cr | PSNR HVS |   SSIM | MS SSIM | CIEDE 2000
0.0000 |  0.0000 |  0.0000 |   0.0000 | 0.0000 |  0.0000 |     0.0000

objective-1-fast --limit=4:
monty-rest-of-stack-noshift-o1f4@2017-11-13T14:37:16.992Z ->
 monty-rest-of-stack-quant-o1f4@2017-11-13T14:38:28.828Z

  PSNR | PSNR Cb | PSNR Cr | PSNR HVS |   SSIM | MS SSIM | CIEDE 2000
0.0000 |  0.0000 |  0.0000 |   0.0000 | 0.0000 |  0.0000 |     0.0000

Change-Id: I3773a1fc128136c9fea227f4b547576a8aa6efa3

av1/encoder/ratectrl.c

diff --git a/av1/encoder/ratectrl.c b/av1/encoder/ratectrl.c
index 4bde2fa..0e5b4a4 100644
--- a/av1/encoder/ratectrl.c
+++ b/av1/encoder/ratectrl.c
@@ -157,15 +157,15 @@
 // Convert the index to a real Q value (scaled down to match old Q values)
 #if CONFIG_HIGHBITDEPTH
   switch (bit_depth) {
-    case AOM_BITS_8: return av1_ac_quant(qindex, 0, bit_depth) / 4.0;
-    case AOM_BITS_10: return av1_ac_quant(qindex, 0, bit_depth) / 16.0;
-    case AOM_BITS_12: return av1_ac_quant(qindex, 0, bit_depth) / 64.0;
+    case AOM_BITS_8: return av1_ac_quant_Q3(qindex, 0, bit_depth) / 4.0;
+    case AOM_BITS_10: return av1_ac_quant_Q3(qindex, 0, bit_depth) / 16.0;
+    case AOM_BITS_12: return av1_ac_quant_Q3(qindex, 0, bit_depth) / 64.0;
     default:
       assert(0 && "bit_depth should be AOM_BITS_8, AOM_BITS_10 or AOM_BITS_12");
       return -1.0;
   }
 #else
-  return av1_ac_quant(qindex, 0, bit_depth) / 4.0;
+  return av1_ac_quant_Q3(qindex, 0, bit_depth) / 4.0;
 #endif
 }