commit 62a94a649dda9c2608e1cae15283f13f77d2f13a
author Alex Converse <aconverse@google.com> Fri Sep 09 10:51:48 2016 -0700
committer Alex Converse <aconverse@google.com> Fri Oct 14 14:05:50 2016 -0700
tree b18455f164078f48c9a4cd2e601d921f6e3d4743
parent fbabcad67c1b277608b6bee26816d9b4746d7a88

Switch rANS to 15 bit precision, and adjust L_BASE.

This causes rANS to operate at the same precision as the Daala EC.

aom/master stats: rans10uabs8lbase12 → rans15uabs8lbase15

objective-1-fast
PSNR YCbCr:      0.01%      0.01%      0.01%
   PSNRHVS:      0.01%
      SSIM:      0.01%
    MSSSIM:      0.01%
 CIEDE2000:      0.01%

subset1
PSNR YCbCr:     -0.01%     -0.00%     -0.00%
   PSNRHVS:     -0.01%
      SSIM:     -0.01%
    MSSSIM:     -0.01%
 CIEDE2000:     -0.01%

(cherry picked from aom/master commit ddbc2e2a68bfc997dc61fca5bcaac3a75245e965)

Change-Id: I6ef0a4f6198784b3712a61af9f105d560a22eaea

test/ans_test.cc

diff --git a/test/ans_test.cc b/test/ans_test.cc
index ba8e3c7..74fdaf6 100644
--- a/test/ans_test.cc
+++ b/test/ans_test.cc
@@ -74,7 +74,8 @@
   return ans_read_end(&d);
 }
 
-const aom_cdf_prob spareto65[] = { 260, 188, 138, 102, 133, 122, 64, 15, 1, 1 };
+const aom_cdf_prob spareto65[] = { 8320, 6018, 4402, 3254, 4259,
+                                   3919, 2057, 492,  45,   2 };
 
 const int kRansSymbols =
     static_cast<int>(sizeof(spareto65) / sizeof(spareto65[0]));
@@ -94,7 +95,8 @@
   std::vector<int> ret;
   libaom_test::ACMRandom gen(18543637);
   for (int i = 0; i < iters; ++i) {
-    int sym = p_to_sym[gen.Rand8() * 4];
+    int sym =
+        p_to_sym[((gen.Rand8() << 8) + gen.Rand8()) & (RANS_PRECISION - 1)];
     ret.push_back(sym);
   }
   return ret;