commit 837262bf4f75067fff8a1e7b00b2afbe7cb9d607
author Thomas Daede <tdaede@mozilla.com> Mon Nov 06 20:07:01 2017 -0800
committer Thomas Daede <tdaede@mozilla.com> Fri Jan 19 06:10:59 2018 +0000
tree 51775e6a1c6ba43bb4c25ff4272d4c441834feb9
parent 79c763bfea8b8749fc5de1df921e24e63e286c4a

Allow CDF precision to be configured with CDF_PROB_BITS.

This does not change the precision used for od_ec_enc_bool_q15
(used by aom_write and aom_write_bit), which means these two
paths will use different CDF precision for now.

The test that verifies that q15's code identically to CDFs
is disabled.

Change-Id: I4cd40a60d472eb58f37ac4fda1056b5dfe8b39a5

aom_dsp/entenc.c

diff --git a/aom_dsp/entenc.c b/aom_dsp/entenc.c
index ca07244..02bee67 100644
--- a/aom_dsp/entenc.c
+++ b/aom_dsp/entenc.c
@@ -16,6 +16,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include "aom_dsp/entenc.h"
+#include "aom_dsp/prob.h"
 
 /*A range encoder.
   See entdec.c and the references for implementation details \cite{Mar79,MNW98}.
@@ -143,9 +144,11 @@
 }
 
 /*Encodes a symbol given its frequency in Q15.
-  fl: 32768 minus the cumulative frequency of all symbols that come before the
+  fl: CDF_PROB_TOP minus the cumulative frequency of all symbols that come
+  before the
        one to be encoded.
-  fh: 32768 minus the cumulative frequency of all symbols up to and including
+  fh: CDF_PROB_TOP minus the cumulative frequency of all symbols up to and
+  including
        the one to be encoded.*/
 static void od_ec_encode_q15(od_ec_enc *enc, unsigned fl, unsigned fh, int s,
                              int nsyms) {
@@ -158,21 +161,25 @@
   OD_ASSERT(32768U <= r);
   OD_ASSERT(fh <= fl);
   OD_ASSERT(fl <= 32768U);
+  OD_ASSERT(7 - EC_PROB_SHIFT - CDF_SHIFT >= 0);
   const int N = nsyms - 1;
-  if (fl < 32768U) {
-    u = ((r >> 8) * (uint32_t)(fl >> EC_PROB_SHIFT) >> (7 - EC_PROB_SHIFT)) +
+  if (fl < CDF_PROB_TOP) {
+    u = ((r >> 8) * (uint32_t)(fl >> EC_PROB_SHIFT) >>
+         (7 - EC_PROB_SHIFT - CDF_SHIFT)) +
         EC_MIN_PROB * (N - (s - 1));
-    v = ((r >> 8) * (uint32_t)(fh >> EC_PROB_SHIFT) >> (7 - EC_PROB_SHIFT)) +
+    v = ((r >> 8) * (uint32_t)(fh >> EC_PROB_SHIFT) >>
+         (7 - EC_PROB_SHIFT - CDF_SHIFT)) +
         EC_MIN_PROB * (N - (s + 0));
     l += r - u;
     r = u - v;
   } else {
-    r -= ((r >> 8) * (uint32_t)(fh >> EC_PROB_SHIFT) >> (7 - EC_PROB_SHIFT)) +
+    r -= ((r >> 8) * (uint32_t)(fh >> EC_PROB_SHIFT) >>
+          (7 - EC_PROB_SHIFT - CDF_SHIFT)) +
          EC_MIN_PROB * (N - (s + 0));
   }
   od_ec_enc_normalize(enc, l, r);
 #if OD_MEASURE_EC_OVERHEAD
-  enc->entropy -= OD_LOG2((double)(OD_ICDF(fh) - OD_ICDF(fl)) / 32768.);
+  enc->entropy -= OD_LOG2((double)(OD_ICDF(fh) - OD_ICDF(fl)) / CDF_PROB_TOP.);
   enc->nb_symbols++;
 #endif
 }
@@ -195,8 +202,7 @@
   r = val ? v : r - v;
   od_ec_enc_normalize(enc, l, r);
 #if OD_MEASURE_EC_OVERHEAD
-  enc->entropy -=
-      OD_LOG2((double)(val ? 32768 - OD_ICDF(f) : OD_ICDF(f)) / 32768.);
+  enc->entropy -= OD_LOG2((double)(val ? f : (32768 - f)) / 32768.);
   enc->nb_symbols++;
 #endif
 }
@@ -214,7 +220,7 @@
   (void)nsyms;
   OD_ASSERT(s >= 0);
   OD_ASSERT(s < nsyms);
-  OD_ASSERT(icdf[nsyms - 1] == OD_ICDF(32768U));
+  OD_ASSERT(icdf[nsyms - 1] == OD_ICDF(CDF_PROB_TOP));
   od_ec_encode_q15(enc, s > 0 ? icdf[s - 1] : OD_ICDF(0), icdf[s], s, nsyms);
 }