commit 736ddef553664a7421dea82adfc50392977b2275
author Thomas Davies <thdavies@cisco.com> Thu Nov 09 09:46:08 2017 +0000
committer Thomas Davies <thdavies@cisco.com> Fri Nov 17 11:27:40 2017 +0000
tree d5f545a6ad5f4abfde5df8edcc406c82803d7a32
parent 53c08960ea268cb5392772b65b2cbc186a3c2bb9

LV_MAP_MULTI: use reduced probability in EC.

Do not round the lv_map probabilities but instead
reduce the precision of the entropy coder to 9 bits
for all CDFs, and modify operation so that the EC
interval is never 0.

Change-Id: I1b4e51019613106ae3b68519b20637bc0d7b51be

av1/encoder/cost.c

diff --git a/av1/encoder/cost.c b/av1/encoder/cost.c
index e33df53..240a16c 100644
--- a/av1/encoder/cost.c
+++ b/av1/encoder/cost.c
@@ -71,7 +71,7 @@
   int i;
   aom_cdf_prob prev_cdf = 0;
   for (i = 0;; ++i) {
-    const aom_cdf_prob p15 = AOM_ICDF(cdf[i]) - prev_cdf;
+    aom_cdf_prob p15 = AOM_ICDF(cdf[i]) - prev_cdf;
     prev_cdf = AOM_ICDF(cdf[i]);
 
     if (inv_map)

av1/encoder/encodetxb.c

diff --git a/av1/encoder/encodetxb.c b/av1/encoder/encodetxb.c
index 21d4a1d..08c7135 100644
--- a/av1/encoder/encodetxb.c
+++ b/av1/encoder/encodetxb.c
@@ -377,8 +377,8 @@
     coeff_ctx =
         get_nz_map_ctx(levels, c, scan, bwl, height, tx_type, c == eob - 1);
     tran_low_t v = tcoeff[scan[c]];
-    aom_write_cdf4(w, AOMMIN(abs(v), 3),
-                   ec_ctx->coeff_base_cdf[txs_ctx][plane_type][coeff_ctx], 4);
+    aom_write_symbol(w, AOMMIN(abs(v), 3),
+                     ec_ctx->coeff_base_cdf[txs_ctx][plane_type][coeff_ctx], 4);
 #else
     coeff_ctx = get_nz_map_ctx(levels, c, scan, bwl, height, tx_type);
     tran_low_t v = tcoeff[scan[c]];
@@ -473,8 +473,8 @@
 #if CONFIG_LV_MAP_MULTI
       for (idx = 0; idx < COEFF_BASE_RANGE; idx += BR_CDF_SIZE - 1) {
         int k = AOMMIN(base_range - idx, BR_CDF_SIZE - 1);
-        aom_write_cdf4(w, k, ec_ctx->coeff_br_cdf[txs_ctx][plane_type][ctx],
-                       BR_CDF_SIZE);
+        aom_write_symbol(w, k, ec_ctx->coeff_br_cdf[txs_ctx][plane_type][ctx],
+                         BR_CDF_SIZE);
         if (k < BR_CDF_SIZE - 1) break;
       }
       if (base_range < COEFF_BASE_RANGE) continue;