commit 70a80a8143302e809dce81e7d88f8ac283ccace7
author Angie Chiang <angiebird@google.com> Mon Jul 24 13:48:02 2017 -0700
committer Angie Chiang <angiebird@google.com> Mon Jul 24 22:45:42 2017 +0000
tree 2731580cd30919d0281febdd2b6c3331664999d0
parent d39f6b330065f416e552b1d557fa1d2316c1af42

Let adapt_scan's prob precision be adjustable

Change-Id: Ie1f5db74d3b22a46abd09e0207770935622074a5

av1/common/scan.c

diff --git a/av1/common/scan.c b/av1/common/scan.c
index a0881a9..2076c8b 100644
--- a/av1/common/scan.c
+++ b/av1/common/scan.c
@@ -6605,7 +6605,7 @@
 }
 
 static void update_scan_prob(AV1_COMMON *cm, TX_SIZE tx_size, TX_TYPE tx_type,
-                             int rate_16) {
+                             int rate) {
   FRAME_CONTEXT *pre_fc = cm->pre_fc;
   uint32_t *prev_non_zero_prob = get_non_zero_prob(pre_fc, tx_size, tx_type);
   uint32_t *non_zero_prob = get_non_zero_prob(cm->fc, tx_size, tx_type);
@@ -6615,13 +6615,18 @@
   int i;
   for (i = 0; i < tx2d_size; i++) {
     int64_t curr_prob =
-        block_num == 0 ? 0 : (non_zero_count[i] << 16) / block_num;
+        block_num == 0
+            ? 0
+            : (non_zero_count[i] << ADAPT_SCAN_PROB_PRECISION) / block_num;
     int64_t prev_prob = prev_non_zero_prob[i];
     int64_t pred_prob =
-        (curr_prob * rate_16 + prev_prob * ((1 << 16) - rate_16)) >> 16;
+        (curr_prob * rate +
+         prev_prob * ((1 << ADAPT_SCAN_PROB_PRECISION) - rate)) >>
+        ADAPT_SCAN_PROB_PRECISION;
     // TODO(angiebird): reduce the bit usage of probabilities and remove
     // clamp_64()
-    non_zero_prob[i] = clamp_64(pred_prob, 0, UINT16_MAX);
+    non_zero_prob[i] =
+        clamp_64(pred_prob, 0, (1 << ADAPT_SCAN_PROB_PRECISION) - 1);
   }
 }
 
@@ -6840,7 +6845,7 @@
 #endif  // CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
     TX_TYPE tx_type;
     for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) {
-      update_scan_prob(cm, tx_size, tx_type, ADAPT_SCAN_UPDATE_RATE_16);
+      update_scan_prob(cm, tx_size, tx_type, ADAPT_SCAN_UPDATE_RATE);
       update_scan_order_facade(cm, tx_size, tx_type);
       update_eob_threshold(cm, tx_size, tx_type);
     }