Allow CDF precision to be configured with CDF_PROB_BITS. This does not change the precision used for od_ec_enc_bool_q15 (used by aom_write and aom_write_bit), which means these two paths will use different CDF precision for now. The test that verifies that q15's code identically to CDFs is disabled. Change-Id: I4cd40a60d472eb58f37ac4fda1056b5dfe8b39a5
diff --git a/aom_dsp/entcode.h b/aom_dsp/entcode.h index e779b6d..78eac4f 100644 --- a/aom_dsp/entcode.h +++ b/aom_dsp/entcode.h
@@ -14,6 +14,7 @@ #include <limits.h> #include <stddef.h> #include "av1/common/odintrin.h" +#include "aom_dsp/prob.h" #define EC_PROB_SHIFT 6 #define EC_MIN_PROB 4 // must be <= (1<<EC_PROB_SHIFT)/16 @@ -31,11 +32,7 @@ 3 => 1/8th bits.*/ #define OD_BITRES (3) -/*The value stored in an iCDF is 32768 minus the actual Q15 cumulative - probability (an "inverse" CDF). - This function converts from one representation to the other (and is its own - inverse).*/ -#define OD_ICDF(x) (32768U - (x)) +#define OD_ICDF AOM_ICDF /*See entcode.c for further documentation.*/
diff --git a/aom_dsp/entdec.c b/aom_dsp/entdec.c index 569cef0..0e94804 100644 --- a/aom_dsp/entdec.c +++ b/aom_dsp/entdec.c
@@ -14,6 +14,7 @@ #endif #include "aom_dsp/entdec.h" +#include "aom_dsp/prob.h" /*A range decoder. This is an entropy decoder based upon \cite{Mar79}, which is itself a @@ -171,8 +172,8 @@ /*Decodes a symbol given an inverse cumulative distribution function (CDF) table in Q15. - icdf: 32768 minus the CDF, such that symbol s falls in the range - [s > 0 ? (32768 - icdf[s - 1]) : 0, 32768 - icdf[s]). + icdf: CDF_PROB_TOP minus the CDF, such that symbol s falls in the range + [s > 0 ? (CDF_PROB_TOP - icdf[s - 1]) : 0, CDF_PROB_TOP - icdf[s]). The values must be monotonically non-increasing, and icdf[nsyms - 1] must be 0. nsyms: The number of symbols in the alphabet. @@ -191,15 +192,16 @@ const int N = nsyms - 1; OD_ASSERT(dif >> (OD_EC_WINDOW_SIZE - 16) < r); - OD_ASSERT(icdf[nsyms - 1] == OD_ICDF(32768U)); + OD_ASSERT(icdf[nsyms - 1] == OD_ICDF(CDF_PROB_TOP)); OD_ASSERT(32768U <= r); + OD_ASSERT(7 - EC_PROB_SHIFT - CDF_SHIFT >= 0); c = (unsigned)(dif >> (OD_EC_WINDOW_SIZE - 16)); v = r; ret = -1; do { u = v; v = ((r >> 8) * (uint32_t)(icdf[++ret] >> EC_PROB_SHIFT) >> - (7 - EC_PROB_SHIFT)); + (7 - EC_PROB_SHIFT - CDF_SHIFT)); v += EC_MIN_PROB * (N - ret); } while (c < v); OD_ASSERT(v < u);
diff --git a/aom_dsp/entenc.c b/aom_dsp/entenc.c index ca07244..02bee67 100644 --- a/aom_dsp/entenc.c +++ b/aom_dsp/entenc.c
@@ -16,6 +16,7 @@ #include <stdlib.h> #include <string.h> #include "aom_dsp/entenc.h" +#include "aom_dsp/prob.h" /*A range encoder. See entdec.c and the references for implementation details \cite{Mar79,MNW98}. @@ -143,9 +144,11 @@ } /*Encodes a symbol given its frequency in Q15. - fl: 32768 minus the cumulative frequency of all symbols that come before the + fl: CDF_PROB_TOP minus the cumulative frequency of all symbols that come + before the one to be encoded. - fh: 32768 minus the cumulative frequency of all symbols up to and including + fh: CDF_PROB_TOP minus the cumulative frequency of all symbols up to and + including the one to be encoded.*/ static void od_ec_encode_q15(od_ec_enc *enc, unsigned fl, unsigned fh, int s, int nsyms) { @@ -158,21 +161,25 @@ OD_ASSERT(32768U <= r); OD_ASSERT(fh <= fl); OD_ASSERT(fl <= 32768U); + OD_ASSERT(7 - EC_PROB_SHIFT - CDF_SHIFT >= 0); const int N = nsyms - 1; - if (fl < 32768U) { - u = ((r >> 8) * (uint32_t)(fl >> EC_PROB_SHIFT) >> (7 - EC_PROB_SHIFT)) + + if (fl < CDF_PROB_TOP) { + u = ((r >> 8) * (uint32_t)(fl >> EC_PROB_SHIFT) >> + (7 - EC_PROB_SHIFT - CDF_SHIFT)) + EC_MIN_PROB * (N - (s - 1)); - v = ((r >> 8) * (uint32_t)(fh >> EC_PROB_SHIFT) >> (7 - EC_PROB_SHIFT)) + + v = ((r >> 8) * (uint32_t)(fh >> EC_PROB_SHIFT) >> + (7 - EC_PROB_SHIFT - CDF_SHIFT)) + EC_MIN_PROB * (N - (s + 0)); l += r - u; r = u - v; } else { - r -= ((r >> 8) * (uint32_t)(fh >> EC_PROB_SHIFT) >> (7 - EC_PROB_SHIFT)) + + r -= ((r >> 8) * (uint32_t)(fh >> EC_PROB_SHIFT) >> + (7 - EC_PROB_SHIFT - CDF_SHIFT)) + EC_MIN_PROB * (N - (s + 0)); } od_ec_enc_normalize(enc, l, r); #if OD_MEASURE_EC_OVERHEAD - enc->entropy -= OD_LOG2((double)(OD_ICDF(fh) - OD_ICDF(fl)) / 32768.); + enc->entropy -= OD_LOG2((double)(OD_ICDF(fh) - OD_ICDF(fl)) / CDF_PROB_TOP.); enc->nb_symbols++; #endif } @@ -195,8 +202,7 @@ r = val ? v : r - v; od_ec_enc_normalize(enc, l, r); #if OD_MEASURE_EC_OVERHEAD - enc->entropy -= - OD_LOG2((double)(val ? 32768 - OD_ICDF(f) : OD_ICDF(f)) / 32768.); + enc->entropy -= OD_LOG2((double)(val ? f : (32768 - f)) / 32768.); enc->nb_symbols++; #endif } @@ -214,7 +220,7 @@ (void)nsyms; OD_ASSERT(s >= 0); OD_ASSERT(s < nsyms); - OD_ASSERT(icdf[nsyms - 1] == OD_ICDF(32768U)); + OD_ASSERT(icdf[nsyms - 1] == OD_ICDF(CDF_PROB_TOP)); od_ec_encode_q15(enc, s > 0 ? icdf[s - 1] : OD_ICDF(0), icdf[s], s, nsyms); }
diff --git a/aom_dsp/prob.h b/aom_dsp/prob.h index 561dda3..c717c9a 100644 --- a/aom_dsp/prob.h +++ b/aom_dsp/prob.h
@@ -36,8 +36,15 @@ #define CDF_PROB_BITS 15 #define CDF_PROB_TOP (1 << CDF_PROB_BITS) +#define CDF_INIT_TOP 32768 +#define CDF_SHIFT (15 - CDF_PROB_BITS) +/*The value stored in an iCDF is CDF_PROB_TOP minus the actual cumulative + probability (an "inverse" CDF). + This function converts from one representation to the other (and is its own + inverse).*/ +#define AOM_ICDF(x) (CDF_PROB_TOP - (x)) -#define AOM_ICDF OD_ICDF +#if CDF_SHIFT == 0 #define AOM_CDF2(a0) AOM_ICDF(a0), AOM_ICDF(CDF_PROB_TOP), 0 #define AOM_CDF3(a0, a1) AOM_ICDF(a0), AOM_ICDF(a1), AOM_ICDF(CDF_PROB_TOP), 0 @@ -99,6 +106,521 @@ AOM_ICDF(a11), AOM_ICDF(a12), AOM_ICDF(a13), AOM_ICDF(a14), \ AOM_ICDF(CDF_PROB_TOP), 0 +#else +#define AOM_CDF2(a0) \ + AOM_ICDF((((a0)-1) * ((CDF_INIT_TOP >> CDF_SHIFT) - 2) + \ + ((CDF_INIT_TOP - 2) >> 1)) / \ + ((CDF_INIT_TOP - 2)) + \ + 1) \ + , AOM_ICDF(CDF_PROB_TOP), 0 +#define AOM_CDF3(a0, a1) \ + AOM_ICDF((((a0)-1) * ((CDF_INIT_TOP >> CDF_SHIFT) - 3) + \ + ((CDF_INIT_TOP - 3) >> 1)) / \ + ((CDF_INIT_TOP - 3)) + \ + 1) \ + , AOM_ICDF((((a1)-2) * ((CDF_INIT_TOP >> CDF_SHIFT) - 3) + \ + ((CDF_INIT_TOP - 3) >> 1)) / \ + ((CDF_INIT_TOP - 3)) + \ + 2), \ + AOM_ICDF(CDF_PROB_TOP), 0 +#define AOM_CDF4(a0, a1, a2) \ + AOM_ICDF((((a0)-1) * ((CDF_INIT_TOP >> CDF_SHIFT) - 4) + \ + ((CDF_INIT_TOP - 4) >> 1)) / \ + ((CDF_INIT_TOP - 4)) + \ + 1) \ + , AOM_ICDF((((a1)-2) * ((CDF_INIT_TOP >> CDF_SHIFT) - 4) + \ + ((CDF_INIT_TOP - 4) >> 1)) / \ + ((CDF_INIT_TOP - 4)) + \ + 2), \ + AOM_ICDF((((a2)-3) * ((CDF_INIT_TOP >> CDF_SHIFT) - 4) + \ + ((CDF_INIT_TOP - 4) >> 1)) / \ + ((CDF_INIT_TOP - 4)) + \ + 3), \ + AOM_ICDF(CDF_PROB_TOP), 0 +#define AOM_CDF5(a0, a1, a2, a3) \ + AOM_ICDF((((a0)-1) * ((CDF_INIT_TOP >> CDF_SHIFT) - 5) + \ + ((CDF_INIT_TOP - 5) >> 1)) / \ + ((CDF_INIT_TOP - 5)) + \ + 1) \ + , AOM_ICDF((((a1)-2) * ((CDF_INIT_TOP >> CDF_SHIFT) - 5) + \ + ((CDF_INIT_TOP - 5) >> 1)) / \ + ((CDF_INIT_TOP - 5)) + \ + 2), \ + AOM_ICDF((((a2)-3) * ((CDF_INIT_TOP >> CDF_SHIFT) - 5) + \ + ((CDF_INIT_TOP - 5) >> 1)) / \ + ((CDF_INIT_TOP - 5)) + \ + 3), \ + AOM_ICDF((((a3)-4) * ((CDF_INIT_TOP >> CDF_SHIFT) - 5) + \ + ((CDF_INIT_TOP - 5) >> 1)) / \ + ((CDF_INIT_TOP - 5)) + \ + 4), \ + AOM_ICDF(CDF_PROB_TOP), 0 +#define AOM_CDF6(a0, a1, a2, a3, a4) \ + AOM_ICDF((((a0)-1) * ((CDF_INIT_TOP >> CDF_SHIFT) - 6) + \ + ((CDF_INIT_TOP - 6) >> 1)) / \ + ((CDF_INIT_TOP - 6)) + \ + 1) \ + , AOM_ICDF((((a1)-2) * ((CDF_INIT_TOP >> CDF_SHIFT) - 6) + \ + ((CDF_INIT_TOP - 6) >> 1)) / \ + ((CDF_INIT_TOP - 6)) + \ + 2), \ + AOM_ICDF((((a2)-3) * ((CDF_INIT_TOP >> CDF_SHIFT) - 6) + \ + ((CDF_INIT_TOP - 6) >> 1)) / \ + ((CDF_INIT_TOP - 6)) + \ + 3), \ + AOM_ICDF((((a3)-4) * ((CDF_INIT_TOP >> CDF_SHIFT) - 6) + \ + ((CDF_INIT_TOP - 6) >> 1)) / \ + ((CDF_INIT_TOP - 6)) + \ + 4), \ + AOM_ICDF((((a4)-5) * ((CDF_INIT_TOP >> CDF_SHIFT) - 6) + \ + ((CDF_INIT_TOP - 6) >> 1)) / \ + ((CDF_INIT_TOP - 6)) + \ + 5), \ + AOM_ICDF(CDF_PROB_TOP), 0 +#define AOM_CDF7(a0, a1, a2, a3, a4, a5) \ + AOM_ICDF((((a0)-1) * ((CDF_INIT_TOP >> CDF_SHIFT) - 7) + \ + ((CDF_INIT_TOP - 7) >> 1)) / \ + ((CDF_INIT_TOP - 7)) + \ + 1) \ + , AOM_ICDF((((a1)-2) * ((CDF_INIT_TOP >> CDF_SHIFT) - 7) + \ + ((CDF_INIT_TOP - 7) >> 1)) / \ + ((CDF_INIT_TOP - 7)) + \ + 2), \ + AOM_ICDF((((a2)-3) * ((CDF_INIT_TOP >> CDF_SHIFT) - 7) + \ + ((CDF_INIT_TOP - 7) >> 1)) / \ + ((CDF_INIT_TOP - 7)) + \ + 3), \ + AOM_ICDF((((a3)-4) * ((CDF_INIT_TOP >> CDF_SHIFT) - 7) + \ + ((CDF_INIT_TOP - 7) >> 1)) / \ + ((CDF_INIT_TOP - 7)) + \ + 4), \ + AOM_ICDF((((a4)-5) * ((CDF_INIT_TOP >> CDF_SHIFT) - 7) + \ + ((CDF_INIT_TOP - 7) >> 1)) / \ + ((CDF_INIT_TOP - 7)) + \ + 5), \ + AOM_ICDF((((a5)-6) * ((CDF_INIT_TOP >> CDF_SHIFT) - 7) + \ + ((CDF_INIT_TOP - 7) >> 1)) / \ + ((CDF_INIT_TOP - 7)) + \ + 6), \ + AOM_ICDF(CDF_PROB_TOP), 0 +#define AOM_CDF8(a0, a1, a2, a3, a4, a5, a6) \ + AOM_ICDF((((a0)-1) * ((CDF_INIT_TOP >> CDF_SHIFT) - 8) + \ + ((CDF_INIT_TOP - 8) >> 1)) / \ + ((CDF_INIT_TOP - 8)) + \ + 1) \ + , AOM_ICDF((((a1)-2) * ((CDF_INIT_TOP >> CDF_SHIFT) - 8) + \ + ((CDF_INIT_TOP - 8) >> 1)) / \ + ((CDF_INIT_TOP - 8)) + \ + 2), \ + AOM_ICDF((((a2)-3) * ((CDF_INIT_TOP >> CDF_SHIFT) - 8) + \ + ((CDF_INIT_TOP - 8) >> 1)) / \ + ((CDF_INIT_TOP - 8)) + \ + 3), \ + AOM_ICDF((((a3)-4) * ((CDF_INIT_TOP >> CDF_SHIFT) - 8) + \ + ((CDF_INIT_TOP - 8) >> 1)) / \ + ((CDF_INIT_TOP - 8)) + \ + 4), \ + AOM_ICDF((((a4)-5) * ((CDF_INIT_TOP >> CDF_SHIFT) - 8) + \ + ((CDF_INIT_TOP - 8) >> 1)) / \ + ((CDF_INIT_TOP - 8)) + \ + 5), \ + AOM_ICDF((((a5)-6) * ((CDF_INIT_TOP >> CDF_SHIFT) - 8) + \ + ((CDF_INIT_TOP - 8) >> 1)) / \ + ((CDF_INIT_TOP - 8)) + \ + 6), \ + AOM_ICDF((((a6)-7) * ((CDF_INIT_TOP >> CDF_SHIFT) - 8) + \ + ((CDF_INIT_TOP - 8) >> 1)) / \ + ((CDF_INIT_TOP - 8)) + \ + 7), \ + AOM_ICDF(CDF_PROB_TOP), 0 +#define AOM_CDF9(a0, a1, a2, a3, a4, a5, a6, a7) \ + AOM_ICDF((((a0)-1) * ((CDF_INIT_TOP >> CDF_SHIFT) - 9) + \ + ((CDF_INIT_TOP - 9) >> 1)) / \ + ((CDF_INIT_TOP - 9)) + \ + 1) \ + , AOM_ICDF((((a1)-2) * ((CDF_INIT_TOP >> CDF_SHIFT) - 9) + \ + ((CDF_INIT_TOP - 9) >> 1)) / \ + ((CDF_INIT_TOP - 9)) + \ + 2), \ + AOM_ICDF((((a2)-3) * ((CDF_INIT_TOP >> CDF_SHIFT) - 9) + \ + ((CDF_INIT_TOP - 9) >> 1)) / \ + ((CDF_INIT_TOP - 9)) + \ + 3), \ + AOM_ICDF((((a3)-4) * ((CDF_INIT_TOP >> CDF_SHIFT) - 9) + \ + ((CDF_INIT_TOP - 9) >> 1)) / \ + ((CDF_INIT_TOP - 9)) + \ + 4), \ + AOM_ICDF((((a4)-5) * ((CDF_INIT_TOP >> CDF_SHIFT) - 9) + \ + ((CDF_INIT_TOP - 9) >> 1)) / \ + ((CDF_INIT_TOP - 9)) + \ + 5), \ + AOM_ICDF((((a5)-6) * ((CDF_INIT_TOP >> CDF_SHIFT) - 9) + \ + ((CDF_INIT_TOP - 9) >> 1)) / \ + ((CDF_INIT_TOP - 9)) + \ + 6), \ + AOM_ICDF((((a6)-7) * ((CDF_INIT_TOP >> CDF_SHIFT) - 9) + \ + ((CDF_INIT_TOP - 9) >> 1)) / \ + ((CDF_INIT_TOP - 9)) + \ + 7), \ + AOM_ICDF((((a7)-8) * ((CDF_INIT_TOP >> CDF_SHIFT) - 9) + \ + ((CDF_INIT_TOP - 9) >> 1)) / \ + ((CDF_INIT_TOP - 9)) + \ + 8), \ + AOM_ICDF(CDF_PROB_TOP), 0 +#define AOM_CDF10(a0, a1, a2, a3, a4, a5, a6, a7, a8) \ + AOM_ICDF((((a0)-1) * ((CDF_INIT_TOP >> CDF_SHIFT) - 10) + \ + ((CDF_INIT_TOP - 10) >> 1)) / \ + ((CDF_INIT_TOP - 10)) + \ + 1) \ + , AOM_ICDF((((a1)-2) * ((CDF_INIT_TOP >> CDF_SHIFT) - 10) + \ + ((CDF_INIT_TOP - 10) >> 1)) / \ + ((CDF_INIT_TOP - 10)) + \ + 2), \ + AOM_ICDF((((a2)-3) * ((CDF_INIT_TOP >> CDF_SHIFT) - 10) + \ + ((CDF_INIT_TOP - 10) >> 1)) / \ + ((CDF_INIT_TOP - 10)) + \ + 3), \ + AOM_ICDF((((a3)-4) * ((CDF_INIT_TOP >> CDF_SHIFT) - 10) + \ + ((CDF_INIT_TOP - 10) >> 1)) / \ + ((CDF_INIT_TOP - 10)) + \ + 4), \ + AOM_ICDF((((a4)-5) * ((CDF_INIT_TOP >> CDF_SHIFT) - 10) + \ + ((CDF_INIT_TOP - 10) >> 1)) / \ + ((CDF_INIT_TOP - 10)) + \ + 5), \ + AOM_ICDF((((a5)-6) * ((CDF_INIT_TOP >> CDF_SHIFT) - 10) + \ + ((CDF_INIT_TOP - 10) >> 1)) / \ + ((CDF_INIT_TOP - 10)) + \ + 6), \ + AOM_ICDF((((a6)-7) * ((CDF_INIT_TOP >> CDF_SHIFT) - 10) + \ + ((CDF_INIT_TOP - 10) >> 1)) / \ + ((CDF_INIT_TOP - 10)) + \ + 7), \ + AOM_ICDF((((a7)-8) * ((CDF_INIT_TOP >> CDF_SHIFT) - 10) + \ + ((CDF_INIT_TOP - 10) >> 1)) / \ + ((CDF_INIT_TOP - 10)) + \ + 8), \ + AOM_ICDF((((a8)-9) * ((CDF_INIT_TOP >> CDF_SHIFT) - 10) + \ + ((CDF_INIT_TOP - 10) >> 1)) / \ + ((CDF_INIT_TOP - 10)) + \ + 9), \ + AOM_ICDF(CDF_PROB_TOP), 0 +#define AOM_CDF11(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) \ + AOM_ICDF((((a0)-1) * ((CDF_INIT_TOP >> CDF_SHIFT) - 11) + \ + ((CDF_INIT_TOP - 11) >> 1)) / \ + ((CDF_INIT_TOP - 11)) + \ + 1) \ + , AOM_ICDF((((a1)-2) * ((CDF_INIT_TOP >> CDF_SHIFT) - 11) + \ + ((CDF_INIT_TOP - 11) >> 1)) / \ + ((CDF_INIT_TOP - 11)) + \ + 2), \ + AOM_ICDF((((a2)-3) * ((CDF_INIT_TOP >> CDF_SHIFT) - 11) + \ + ((CDF_INIT_TOP - 11) >> 1)) / \ + ((CDF_INIT_TOP - 11)) + \ + 3), \ + AOM_ICDF((((a3)-4) * ((CDF_INIT_TOP >> CDF_SHIFT) - 11) + \ + ((CDF_INIT_TOP - 11) >> 1)) / \ + ((CDF_INIT_TOP - 11)) + \ + 4), \ + AOM_ICDF((((a4)-5) * ((CDF_INIT_TOP >> CDF_SHIFT) - 11) + \ + ((CDF_INIT_TOP - 11) >> 1)) / \ + ((CDF_INIT_TOP - 11)) + \ + 5), \ + AOM_ICDF((((a5)-6) * ((CDF_INIT_TOP >> CDF_SHIFT) - 11) + \ + ((CDF_INIT_TOP - 11) >> 1)) / \ + ((CDF_INIT_TOP - 11)) + \ + 6), \ + AOM_ICDF((((a6)-7) * ((CDF_INIT_TOP >> CDF_SHIFT) - 11) + \ + ((CDF_INIT_TOP - 11) >> 1)) / \ + ((CDF_INIT_TOP - 11)) + \ + 7), \ + AOM_ICDF((((a7)-8) * ((CDF_INIT_TOP >> CDF_SHIFT) - 11) + \ + ((CDF_INIT_TOP - 11) >> 1)) / \ + ((CDF_INIT_TOP - 11)) + \ + 8), \ + AOM_ICDF((((a8)-9) * ((CDF_INIT_TOP >> CDF_SHIFT) - 11) + \ + ((CDF_INIT_TOP - 11) >> 1)) / \ + ((CDF_INIT_TOP - 11)) + \ + 9), \ + AOM_ICDF((((a9)-10) * ((CDF_INIT_TOP >> CDF_SHIFT) - 11) + \ + ((CDF_INIT_TOP - 11) >> 1)) / \ + ((CDF_INIT_TOP - 11)) + \ + 10), \ + AOM_ICDF(CDF_PROB_TOP), 0 +#define AOM_CDF12(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10) \ + AOM_ICDF((((a0)-1) * ((CDF_INIT_TOP >> CDF_SHIFT) - 12) + \ + ((CDF_INIT_TOP - 12) >> 1)) / \ + ((CDF_INIT_TOP - 12)) + \ + 1) \ + , AOM_ICDF((((a1)-2) * ((CDF_INIT_TOP >> CDF_SHIFT) - 12) + \ + ((CDF_INIT_TOP - 12) >> 1)) / \ + ((CDF_INIT_TOP - 12)) + \ + 2), \ + AOM_ICDF((((a2)-3) * ((CDF_INIT_TOP >> CDF_SHIFT) - 12) + \ + ((CDF_INIT_TOP - 12) >> 1)) / \ + ((CDF_INIT_TOP - 12)) + \ + 3), \ + AOM_ICDF((((a3)-4) * ((CDF_INIT_TOP >> CDF_SHIFT) - 12) + \ + ((CDF_INIT_TOP - 12) >> 1)) / \ + ((CDF_INIT_TOP - 12)) + \ + 4), \ + AOM_ICDF((((a4)-5) * ((CDF_INIT_TOP >> CDF_SHIFT) - 12) + \ + ((CDF_INIT_TOP - 12) >> 1)) / \ + ((CDF_INIT_TOP - 12)) + \ + 5), \ + AOM_ICDF((((a5)-6) * ((CDF_INIT_TOP >> CDF_SHIFT) - 12) + \ + ((CDF_INIT_TOP - 12) >> 1)) / \ + ((CDF_INIT_TOP - 12)) + \ + 6), \ + AOM_ICDF((((a6)-7) * ((CDF_INIT_TOP >> CDF_SHIFT) - 12) + \ + ((CDF_INIT_TOP - 12) >> 1)) / \ + ((CDF_INIT_TOP - 12)) + \ + 7), \ + AOM_ICDF((((a7)-8) * ((CDF_INIT_TOP >> CDF_SHIFT) - 12) + \ + ((CDF_INIT_TOP - 12) >> 1)) / \ + ((CDF_INIT_TOP - 12)) + \ + 8), \ + AOM_ICDF((((a8)-9) * ((CDF_INIT_TOP >> CDF_SHIFT) - 12) + \ + ((CDF_INIT_TOP - 12) >> 1)) / \ + ((CDF_INIT_TOP - 12)) + \ + 9), \ + AOM_ICDF((((a9)-10) * ((CDF_INIT_TOP >> CDF_SHIFT) - 12) + \ + ((CDF_INIT_TOP - 12) >> 1)) / \ + ((CDF_INIT_TOP - 12)) + \ + 10), \ + AOM_ICDF((((a10)-11) * ((CDF_INIT_TOP >> CDF_SHIFT) - 12) + \ + ((CDF_INIT_TOP - 12) >> 1)) / \ + ((CDF_INIT_TOP - 12)) + \ + 11), \ + AOM_ICDF(CDF_PROB_TOP), 0 +#define AOM_CDF13(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11) \ + AOM_ICDF((((a0)-1) * ((CDF_INIT_TOP >> CDF_SHIFT) - 13) + \ + ((CDF_INIT_TOP - 13) >> 1)) / \ + ((CDF_INIT_TOP - 13)) + \ + 1) \ + , AOM_ICDF((((a1)-2) * ((CDF_INIT_TOP >> CDF_SHIFT) - 13) + \ + ((CDF_INIT_TOP - 13) >> 1)) / \ + ((CDF_INIT_TOP - 13)) + \ + 2), \ + AOM_ICDF((((a2)-3) * ((CDF_INIT_TOP >> CDF_SHIFT) - 13) + \ + ((CDF_INIT_TOP - 13) >> 1)) / \ + ((CDF_INIT_TOP - 13)) + \ + 3), \ + AOM_ICDF((((a3)-4) * ((CDF_INIT_TOP >> CDF_SHIFT) - 13) + \ + ((CDF_INIT_TOP - 13) >> 1)) / \ + ((CDF_INIT_TOP - 13)) + \ + 4), \ + AOM_ICDF((((a4)-5) * ((CDF_INIT_TOP >> CDF_SHIFT) - 13) + \ + ((CDF_INIT_TOP - 13) >> 1)) / \ + ((CDF_INIT_TOP - 13)) + \ + 5), \ + AOM_ICDF((((a5)-6) * ((CDF_INIT_TOP >> CDF_SHIFT) - 13) + \ + ((CDF_INIT_TOP - 13) >> 1)) / \ + ((CDF_INIT_TOP - 13)) + \ + 6), \ + AOM_ICDF((((a6)-7) * ((CDF_INIT_TOP >> CDF_SHIFT) - 13) + \ + ((CDF_INIT_TOP - 13) >> 1)) / \ + ((CDF_INIT_TOP - 13)) + \ + 7), \ + AOM_ICDF((((a7)-8) * ((CDF_INIT_TOP >> CDF_SHIFT) - 13) + \ + ((CDF_INIT_TOP - 13) >> 1)) / \ + ((CDF_INIT_TOP - 13)) + \ + 8), \ + AOM_ICDF((((a8)-9) * ((CDF_INIT_TOP >> CDF_SHIFT) - 13) + \ + ((CDF_INIT_TOP - 13) >> 1)) / \ + ((CDF_INIT_TOP - 13)) + \ + 9), \ + AOM_ICDF((((a9)-10) * ((CDF_INIT_TOP >> CDF_SHIFT) - 13) + \ + ((CDF_INIT_TOP - 13) >> 1)) / \ + ((CDF_INIT_TOP - 13)) + \ + 10), \ + AOM_ICDF((((a10)-11) * ((CDF_INIT_TOP >> CDF_SHIFT) - 13) + \ + ((CDF_INIT_TOP - 13) >> 1)) / \ + ((CDF_INIT_TOP - 13)) + \ + 11), \ + AOM_ICDF((((a11)-12) * ((CDF_INIT_TOP >> CDF_SHIFT) - 13) + \ + ((CDF_INIT_TOP - 13) >> 1)) / \ + ((CDF_INIT_TOP - 13)) + \ + 12), \ + AOM_ICDF(CDF_PROB_TOP), 0 +#define AOM_CDF14(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12) \ + AOM_ICDF((((a0)-1) * ((CDF_INIT_TOP >> CDF_SHIFT) - 14) + \ + ((CDF_INIT_TOP - 14) >> 1)) / \ + ((CDF_INIT_TOP - 14)) + \ + 1) \ + , AOM_ICDF((((a1)-2) * ((CDF_INIT_TOP >> CDF_SHIFT) - 14) + \ + ((CDF_INIT_TOP - 14) >> 1)) / \ + ((CDF_INIT_TOP - 14)) + \ + 2), \ + AOM_ICDF((((a2)-3) * ((CDF_INIT_TOP >> CDF_SHIFT) - 14) + \ + ((CDF_INIT_TOP - 14) >> 1)) / \ + ((CDF_INIT_TOP - 14)) + \ + 3), \ + AOM_ICDF((((a3)-4) * ((CDF_INIT_TOP >> CDF_SHIFT) - 14) + \ + ((CDF_INIT_TOP - 14) >> 1)) / \ + ((CDF_INIT_TOP - 14)) + \ + 4), \ + AOM_ICDF((((a4)-5) * ((CDF_INIT_TOP >> CDF_SHIFT) - 14) + \ + ((CDF_INIT_TOP - 14) >> 1)) / \ + ((CDF_INIT_TOP - 14)) + \ + 5), \ + AOM_ICDF((((a5)-6) * ((CDF_INIT_TOP >> CDF_SHIFT) - 14) + \ + ((CDF_INIT_TOP - 14) >> 1)) / \ + ((CDF_INIT_TOP - 14)) + \ + 6), \ + AOM_ICDF((((a6)-7) * ((CDF_INIT_TOP >> CDF_SHIFT) - 14) + \ + ((CDF_INIT_TOP - 14) >> 1)) / \ + ((CDF_INIT_TOP - 14)) + \ + 7), \ + AOM_ICDF((((a7)-8) * ((CDF_INIT_TOP >> CDF_SHIFT) - 14) + \ + ((CDF_INIT_TOP - 14) >> 1)) / \ + ((CDF_INIT_TOP - 14)) + \ + 8), \ + AOM_ICDF((((a8)-9) * ((CDF_INIT_TOP >> CDF_SHIFT) - 14) + \ + ((CDF_INIT_TOP - 14) >> 1)) / \ + ((CDF_INIT_TOP - 14)) + \ + 9), \ + AOM_ICDF((((a9)-10) * ((CDF_INIT_TOP >> CDF_SHIFT) - 14) + \ + ((CDF_INIT_TOP - 14) >> 1)) / \ + ((CDF_INIT_TOP - 14)) + \ + 10), \ + AOM_ICDF((((a10)-11) * ((CDF_INIT_TOP >> CDF_SHIFT) - 14) + \ + ((CDF_INIT_TOP - 14) >> 1)) / \ + ((CDF_INIT_TOP - 14)) + \ + 11), \ + AOM_ICDF((((a11)-12) * ((CDF_INIT_TOP >> CDF_SHIFT) - 14) + \ + ((CDF_INIT_TOP - 14) >> 1)) / \ + ((CDF_INIT_TOP - 14)) + \ + 12), \ + AOM_ICDF((((a12)-13) * ((CDF_INIT_TOP >> CDF_SHIFT) - 14) + \ + ((CDF_INIT_TOP - 14) >> 1)) / \ + ((CDF_INIT_TOP - 14)) + \ + 13), \ + AOM_ICDF(CDF_PROB_TOP), 0 +#define AOM_CDF15(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13) \ + AOM_ICDF((((a0)-1) * ((CDF_INIT_TOP >> CDF_SHIFT) - 15) + \ + ((CDF_INIT_TOP - 15) >> 1)) / \ + ((CDF_INIT_TOP - 15)) + \ + 1) \ + , AOM_ICDF((((a1)-2) * ((CDF_INIT_TOP >> CDF_SHIFT) - 15) + \ + ((CDF_INIT_TOP - 15) >> 1)) / \ + ((CDF_INIT_TOP - 15)) + \ + 2), \ + AOM_ICDF((((a2)-3) * ((CDF_INIT_TOP >> CDF_SHIFT) - 15) + \ + ((CDF_INIT_TOP - 15) >> 1)) / \ + ((CDF_INIT_TOP - 15)) + \ + 3), \ + AOM_ICDF((((a3)-4) * ((CDF_INIT_TOP >> CDF_SHIFT) - 15) + \ + ((CDF_INIT_TOP - 15) >> 1)) / \ + ((CDF_INIT_TOP - 15)) + \ + 4), \ + AOM_ICDF((((a4)-5) * ((CDF_INIT_TOP >> CDF_SHIFT) - 15) + \ + ((CDF_INIT_TOP - 15) >> 1)) / \ + ((CDF_INIT_TOP - 15)) + \ + 5), \ + AOM_ICDF((((a5)-6) * ((CDF_INIT_TOP >> CDF_SHIFT) - 15) + \ + ((CDF_INIT_TOP - 15) >> 1)) / \ + ((CDF_INIT_TOP - 15)) + \ + 6), \ + AOM_ICDF((((a6)-7) * ((CDF_INIT_TOP >> CDF_SHIFT) - 15) + \ + ((CDF_INIT_TOP - 15) >> 1)) / \ + ((CDF_INIT_TOP - 15)) + \ + 7), \ + AOM_ICDF((((a7)-8) * ((CDF_INIT_TOP >> CDF_SHIFT) - 15) + \ + ((CDF_INIT_TOP - 15) >> 1)) / \ + ((CDF_INIT_TOP - 15)) + \ + 8), \ + AOM_ICDF((((a8)-9) * ((CDF_INIT_TOP >> CDF_SHIFT) - 15) + \ + ((CDF_INIT_TOP - 15) >> 1)) / \ + ((CDF_INIT_TOP - 15)) + \ + 9), \ + AOM_ICDF((((a9)-10) * ((CDF_INIT_TOP >> CDF_SHIFT) - 15) + \ + ((CDF_INIT_TOP - 15) >> 1)) / \ + ((CDF_INIT_TOP - 15)) + \ + 10), \ + AOM_ICDF((((a10)-11) * ((CDF_INIT_TOP >> CDF_SHIFT) - 15) + \ + ((CDF_INIT_TOP - 15) >> 1)) / \ + ((CDF_INIT_TOP - 15)) + \ + 11), \ + AOM_ICDF((((a11)-12) * ((CDF_INIT_TOP >> CDF_SHIFT) - 15) + \ + ((CDF_INIT_TOP - 15) >> 1)) / \ + ((CDF_INIT_TOP - 15)) + \ + 12), \ + AOM_ICDF((((a12)-13) * ((CDF_INIT_TOP >> CDF_SHIFT) - 15) + \ + ((CDF_INIT_TOP - 15) >> 1)) / \ + ((CDF_INIT_TOP - 15)) + \ + 13), \ + AOM_ICDF((((a13)-14) * ((CDF_INIT_TOP >> CDF_SHIFT) - 15) + \ + ((CDF_INIT_TOP - 15) >> 1)) / \ + ((CDF_INIT_TOP - 15)) + \ + 14), \ + AOM_ICDF(CDF_PROB_TOP), 0 +#define AOM_CDF16(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, \ + a14) \ + AOM_ICDF((((a0)-1) * ((CDF_INIT_TOP >> CDF_SHIFT) - 16) + \ + ((CDF_INIT_TOP - 16) >> 1)) / \ + ((CDF_INIT_TOP - 16)) + \ + 1) \ + , AOM_ICDF((((a1)-2) * ((CDF_INIT_TOP >> CDF_SHIFT) - 16) + \ + ((CDF_INIT_TOP - 16) >> 1)) / \ + ((CDF_INIT_TOP - 16)) + \ + 2), \ + AOM_ICDF((((a2)-3) * ((CDF_INIT_TOP >> CDF_SHIFT) - 16) + \ + ((CDF_INIT_TOP - 16) >> 1)) / \ + ((CDF_INIT_TOP - 16)) + \ + 3), \ + AOM_ICDF((((a3)-4) * ((CDF_INIT_TOP >> CDF_SHIFT) - 16) + \ + ((CDF_INIT_TOP - 16) >> 1)) / \ + ((CDF_INIT_TOP - 16)) + \ + 4), \ + AOM_ICDF((((a4)-5) * ((CDF_INIT_TOP >> CDF_SHIFT) - 16) + \ + ((CDF_INIT_TOP - 16) >> 1)) / \ + ((CDF_INIT_TOP - 16)) + \ + 5), \ + AOM_ICDF((((a5)-6) * ((CDF_INIT_TOP >> CDF_SHIFT) - 16) + \ + ((CDF_INIT_TOP - 16) >> 1)) / \ + ((CDF_INIT_TOP - 16)) + \ + 6), \ + AOM_ICDF((((a6)-7) * ((CDF_INIT_TOP >> CDF_SHIFT) - 16) + \ + ((CDF_INIT_TOP - 16) >> 1)) / \ + ((CDF_INIT_TOP - 16)) + \ + 7), \ + AOM_ICDF((((a7)-8) * ((CDF_INIT_TOP >> CDF_SHIFT) - 16) + \ + ((CDF_INIT_TOP - 16) >> 1)) / \ + ((CDF_INIT_TOP - 16)) + \ + 8), \ + AOM_ICDF((((a8)-9) * ((CDF_INIT_TOP >> CDF_SHIFT) - 16) + \ + ((CDF_INIT_TOP - 16) >> 1)) / \ + ((CDF_INIT_TOP - 16)) + \ + 9), \ + AOM_ICDF((((a9)-10) * ((CDF_INIT_TOP >> CDF_SHIFT) - 16) + \ + ((CDF_INIT_TOP - 16) >> 1)) / \ + ((CDF_INIT_TOP - 16)) + \ + 10), \ + AOM_ICDF((((a10)-11) * ((CDF_INIT_TOP >> CDF_SHIFT) - 16) + \ + ((CDF_INIT_TOP - 16) >> 1)) / \ + ((CDF_INIT_TOP - 16)) + \ + 11), \ + AOM_ICDF((((a11)-12) * ((CDF_INIT_TOP >> CDF_SHIFT) - 16) + \ + ((CDF_INIT_TOP - 16) >> 1)) / \ + ((CDF_INIT_TOP - 16)) + \ + 12), \ + AOM_ICDF((((a12)-13) * ((CDF_INIT_TOP >> CDF_SHIFT) - 16) + \ + ((CDF_INIT_TOP - 16) >> 1)) / \ + ((CDF_INIT_TOP - 16)) + \ + 13), \ + AOM_ICDF((((a13)-14) * ((CDF_INIT_TOP >> CDF_SHIFT) - 16) + \ + ((CDF_INIT_TOP - 16) >> 1)) / \ + ((CDF_INIT_TOP - 16)) + \ + 14), \ + AOM_ICDF((((a14)-15) * ((CDF_INIT_TOP >> CDF_SHIFT) - 16) + \ + ((CDF_INIT_TOP - 16) >> 1)) / \ + ((CDF_INIT_TOP - 16)) + \ + 15), \ + AOM_ICDF(CDF_PROB_TOP), 0 + +#endif + #define MAX_PROB 255 #define BR_NODE 1
diff --git a/av1/common/entropy.c b/av1/common/entropy.c index 3e371b1..727ca09 100644 --- a/av1/common/entropy.c +++ b/av1/common/entropy.c
@@ -530,12 +530,16 @@ probNZ = CDF_PROB_TOP - phead[ZERO_TOKEN + is_dc] - (is_dc ? phead[0] : 0); prob1 = phead[is_dc + ONE_TOKEN_EOB] + phead[is_dc + ONE_TOKEN_NEOB]; prob_idx = - AOMMIN(COEFF_PROB_MODELS - 1, AOMMAX(0, ((256 * prob1) / probNZ) - 1)); + AOMMIN(COEFF_PROB_MODELS - 1, AOMMAX(0, (256 * prob1 / probNZ) - 1)); sum = 0; for (i = 0; i < TAIL_TOKENS; ++i) { sum += av1_pareto8_tail_probs[prob_idx][i]; - cdf_tail[i] = AOM_ICDF(sum); + cdf_tail[i] = AOM_ICDF( + ((sum - (i + 1)) * ((CDF_INIT_TOP >> CDF_SHIFT) - TAIL_TOKENS) + + ((CDF_INIT_TOP - TAIL_TOKENS) >> 1)) / + ((CDF_INIT_TOP - TAIL_TOKENS)) + + (i + 1)); } }
diff --git a/test/ec_test.cc b/test/ec_test.cc index 780ef8c..e6a5ea6 100644 --- a/test/ec_test.cc +++ b/test/ec_test.cc
@@ -55,13 +55,14 @@ for (j = 0; j < sz; j++) { data[j] = rand() / ((RAND_MAX >> 1) + 1); - fts[j] = 15; - fz[j] = (rand() % 32766) >> (15 - fts[j]); + fts[j] = CDF_PROB_BITS; + fz[j] = (rand() % (CDF_PROB_TOP - 2)) >> (CDF_PROB_BITS - fts[j]); fz[j] = OD_MAXI(fz[j], 1); enc_method[j] = 3 + (rand() & 1); switch (enc_method[j]) { case 3: { - od_ec_encode_bool_q15(&enc, data[j], OD_ICDF(fz[j] << (15 - fts[j]))); + od_ec_encode_bool_q15(&enc, data[j], + OD_ICDF(fz[j] << (CDF_PROB_BITS - fts[j]))); break; } case 4: { @@ -69,7 +70,6 @@ cdf[0] = OD_ICDF(fz[j]); cdf[1] = OD_ICDF(1U << fts[j]); od_ec_encode_cdf_q15(&enc, data[j], cdf, 2); - enc_method[j]++; break; } } @@ -90,10 +90,15 @@ << " (Random seed: " << seed << ").\n"; for (j = 0; j < sz; j++) { int dec_method; - dec_method = 3 + (rand() & 1); + if (CDF_SHIFT == 0) { + dec_method = 3 + (rand() & 1); + } else { + dec_method = enc_method[j]; + } switch (dec_method) { case 3: { - sym = od_ec_decode_bool_q15(&dec, OD_ICDF(fz[j] << (15 - fts[j]))); + sym = od_ec_decode_bool_q15( + &dec, OD_ICDF(fz[j] << (CDF_PROB_BITS - fts[j]))); break; } case 4: { @@ -101,7 +106,6 @@ cdf[0] = OD_ICDF(fz[j]); cdf[1] = OD_ICDF(1U << fts[j]); sym = od_ec_decode_cdf_q15(&dec, cdf, 2); - dec_method++; break; } } @@ -126,28 +130,30 @@ free(fz); } od_ec_enc_reset(&enc); - od_ec_encode_bool_q15(&enc, 0, OD_ICDF(16384)); - od_ec_encode_bool_q15(&enc, 0, OD_ICDF(16384)); - od_ec_encode_bool_q15(&enc, 0, OD_ICDF(16384)); - od_ec_encode_bool_q15(&enc, 0, OD_ICDF(16384)); - od_ec_encode_bool_q15(&enc, 0, OD_ICDF(24576)); - od_ec_enc_patch_initial_bits(&enc, 3, 2); - EXPECT_FALSE(enc.error) << "od_ec_enc_patch_initial_bits() failed.\n"; - od_ec_enc_patch_initial_bits(&enc, 0, 5); - EXPECT_TRUE(enc.error) - << "od_ec_enc_patch_initial_bits() didn't fail when it should have.\n"; - od_ec_enc_reset(&enc); - od_ec_encode_bool_q15(&enc, 0, OD_ICDF(16384)); - od_ec_encode_bool_q15(&enc, 0, OD_ICDF(16384)); - od_ec_encode_bool_q15(&enc, 1, OD_ICDF(32256)); - od_ec_encode_bool_q15(&enc, 0, OD_ICDF(24576)); - od_ec_enc_patch_initial_bits(&enc, 0, 2); - EXPECT_FALSE(enc.error) << "od_ec_enc_patch_initial_bits() failed.\n"; - ptr = od_ec_enc_done(&enc, &ptr_sz); - EXPECT_EQ(ptr_sz, 2u); - EXPECT_EQ(ptr[0], 63) - << "Got " << ptr[0] - << " when expecting 63 for od_ec_enc_patch_initial_bits().\n"; + if (CDF_SHIFT == 0) { + od_ec_encode_bool_q15(&enc, 0, OD_ICDF(16384)); + od_ec_encode_bool_q15(&enc, 0, OD_ICDF(16384)); + od_ec_encode_bool_q15(&enc, 0, OD_ICDF(16384)); + od_ec_encode_bool_q15(&enc, 0, OD_ICDF(16384)); + od_ec_encode_bool_q15(&enc, 0, OD_ICDF(24576)); + od_ec_enc_patch_initial_bits(&enc, 3, 2); + EXPECT_FALSE(enc.error) << "od_ec_enc_patch_initial_bits() failed.\n"; + od_ec_enc_patch_initial_bits(&enc, 0, 5); + EXPECT_TRUE(enc.error) + << "od_ec_enc_patch_initial_bits() didn't fail when it should have.\n"; + od_ec_enc_reset(&enc); + od_ec_encode_bool_q15(&enc, 0, OD_ICDF(16384)); + od_ec_encode_bool_q15(&enc, 0, OD_ICDF(16384)); + od_ec_encode_bool_q15(&enc, 1, OD_ICDF(32256)); + od_ec_encode_bool_q15(&enc, 0, OD_ICDF(24576)); + od_ec_enc_patch_initial_bits(&enc, 0, 2); + EXPECT_FALSE(enc.error) << "od_ec_enc_patch_initial_bits() failed.\n"; + ptr = od_ec_enc_done(&enc, &ptr_sz); + EXPECT_EQ(ptr_sz, 2u); + EXPECT_EQ(ptr[0], 63) + << "Got " << ptr[0] + << " when expecting 63 for od_ec_enc_patch_initial_bits().\n"; + } od_ec_enc_clear(&enc); EXPECT_EQ(ret, 0); }