commit 62a94a649dda9c2608e1cae15283f13f77d2f13a
author Alex Converse <aconverse@google.com> Fri Sep 09 10:51:48 2016 -0700
committer Alex Converse <aconverse@google.com> Fri Oct 14 14:05:50 2016 -0700
tree b18455f164078f48c9a4cd2e601d921f6e3d4743
parent fbabcad67c1b277608b6bee26816d9b4746d7a88

Switch rANS to 15 bit precision, and adjust L_BASE.

This causes rANS to operate at the same precision as the Daala EC.

aom/master stats: rans10uabs8lbase12 → rans15uabs8lbase15

objective-1-fast
PSNR YCbCr:      0.01%      0.01%      0.01%
   PSNRHVS:      0.01%
      SSIM:      0.01%
    MSSSIM:      0.01%
 CIEDE2000:      0.01%

subset1
PSNR YCbCr:     -0.01%     -0.00%     -0.00%
   PSNRHVS:     -0.01%
      SSIM:     -0.01%
    MSSSIM:     -0.01%
 CIEDE2000:     -0.01%

(cherry picked from aom/master commit ddbc2e2a68bfc997dc61fca5bcaac3a75245e965)

Change-Id: I6ef0a4f6198784b3712a61af9f105d560a22eaea

aom_dsp/ans.h

diff --git a/aom_dsp/ans.h b/aom_dsp/ans.h
index 5927e58..08735bb 100644
--- a/aom_dsp/ans.h
+++ b/aom_dsp/ans.h
@@ -26,10 +26,12 @@
 typedef uint8_t AnsP8;
 #define ANS_P8_PRECISION 256u
 #define ANS_P8_SHIFT 8
-#define RANS_PRECISION 1024u
-#define RANS_PROB_BITS 10
+#define RANS_PROB_BITS 15
+#define RANS_PRECISION (1u << RANS_PROB_BITS)
 
-#define L_BASE (RANS_PRECISION * 4)  // L_BASE % precision must be 0
+// L_BASE % PRECISION must be 0. Increasing L_BASE beyond 2**15 will cause uabs
+// to overflow.
+#define L_BASE (RANS_PRECISION)
 #define IO_BASE 256
 // Range I = { L_BASE, L_BASE + 1, ..., L_BASE * IO_BASE - 1 }
 

aom_dsp/ansreader.h

diff --git a/aom_dsp/ansreader.h b/aom_dsp/ansreader.h
index 1f66531..c46778b 100644
--- a/aom_dsp/ansreader.h
+++ b/aom_dsp/ansreader.h
@@ -111,8 +111,12 @@
     if (offset < 3) return 1;
     ans->buf_offset = offset - 3;
     ans->state = mem_get_le24(buf + offset - 3) & 0x3FFFFF;
+  } else if ((buf[offset - 1] & 0xE0) == 0xE0) {
+    if (offset < 4) return 1;
+    ans->buf_offset = offset - 4;
+    ans->state = mem_get_le32(buf + offset - 4) & 0x1FFFFFFF;
   } else {
-    // x == 3 implies this byte is a superframe marker
+    // 110xxxxx implies this byte is a superframe marker
     return 1;
   }
   ans->state += L_BASE;

aom_dsp/answriter.h

diff --git a/aom_dsp/answriter.h b/aom_dsp/answriter.h
index 0ac1bda..298b255 100644
--- a/aom_dsp/answriter.h
+++ b/aom_dsp/answriter.h
@@ -54,6 +54,9 @@
   } else if (state < (1 << 22)) {
     mem_put_le24(ans->buf + ans->buf_offset, (0x02 << 22) + state);
     return ans->buf_offset + 3;
+  } else if (state < (1 << 29)) {
+    mem_put_le32(ans->buf + ans->buf_offset, (0x07 << 29) + state);
+    return ans->buf_offset + 4;
   } else {
     assert(0 && "State is too large to be serialized");
     return ans->buf_offset;