commit fe2a959e42c411baf4517541a0bf833e42228bfc
author Angie Chiang <angiebird@google.com> Thu Jan 12 14:54:48 2017 -0800
committer Angie Chiang <angiebird@google.com> Sat Jan 14 02:04:30 2017 +0000
tree 063960691fdb05ff830a5d56cb1fc203a8ce3afd
parent 8f315b19fe2897278b005fe20f6bbad5e7c4b69d

Use default scan order as a tie breaker

Change-Id: I85f059b6e2c48bcdf2edd3b7bf896fdccbaaa703

av1/common/scan.c

diff --git a/av1/common/scan.c b/av1/common/scan.c
index 0376b82..bf161ab 100644
--- a/av1/common/scan.c
+++ b/av1/common/scan.c
@@ -6458,15 +6458,17 @@
   return *(const uint32_t *)b > *(const uint32_t *)a ? 1 : -1;
 }
 
-void av1_augment_prob(uint32_t *prob, int size, int tx1d_size) {
+void av1_augment_prob(TX_SIZE tx_size, TX_TYPE tx_type, uint32_t *prob) {
+  // TODO(angiebird): check if we need is_inter here
+  const SCAN_ORDER *sc = get_default_scan(tx_size, tx_type, 0);
+  const int tx1d_size = tx_size_wide[tx_size];
   int r, c;
-  for (r = 0; r < size; r++) {
-    for (c = 0; c < size; c++) {
-      const int coeff_idx = r * tx1d_size + c;
-      const int idx = r * size + c;
+  for (r = 0; r < tx1d_size; r++) {
+    for (c = 0; c < tx1d_size; c++) {
+      const int idx = r * tx1d_size + c;
       const uint32_t mask_16 = ((1 << 16) - 1);
-      const uint32_t tie_breaker = ~(((r + c) << COEFF_IDX_BITS) | coeff_idx);
-      // prob[idx]: 16 bits  r+c: 6 bits  coeff_idx: 10 bits
+      const uint32_t tie_breaker = ~((uint32_t)sc->iscan[idx]);
+      // prob[idx]: 16 bits  dummy: 6 bits  scan_idx: 10 bits
       prob[idx] = (prob[idx] << 16) | (mask_16 & tie_breaker);
     }
   }
@@ -6519,18 +6521,20 @@
   neighbors[tx2d_size * MAX_NEIGHBORS + 1] = scan[0];
 }
 
-void av1_update_sort_order(TX_SIZE tx_size, const uint32_t *non_zero_prob,
-                           int16_t *sort_order) {
+void av1_update_sort_order(TX_SIZE tx_size, TX_TYPE tx_type,
+                           const uint32_t *non_zero_prob, int16_t *sort_order) {
+  const SCAN_ORDER *sc = get_default_scan(tx_size, tx_type, 0);
   uint32_t temp[COEFF_IDX_SIZE];
-  const int tx1d_size = tx_size_wide[tx_size];
   const int tx2d_size = tx_size_2d[tx_size];
   int sort_idx;
   assert(tx2d_size <= COEFF_IDX_SIZE);
   memcpy(temp, non_zero_prob, tx2d_size * sizeof(*non_zero_prob));
-  av1_augment_prob(temp, tx1d_size, tx1d_size);
+  av1_augment_prob(tx_size, tx_type, temp);
   qsort(temp, tx2d_size, sizeof(*temp), cmp_prob);
   for (sort_idx = 0; sort_idx < tx2d_size; ++sort_idx) {
-    const int coeff_idx = (temp[sort_idx] & COEFF_IDX_MASK) ^ COEFF_IDX_MASK;
+    const int default_scan_idx =
+        (temp[sort_idx] & COEFF_IDX_MASK) ^ COEFF_IDX_MASK;
+    const int coeff_idx = sc->scan[default_scan_idx];
     sort_order[sort_idx] = coeff_idx;
   }
 }
@@ -6562,7 +6566,7 @@
   int16_t *iscan = get_adapt_iscan(cm->fc, tx_size, tx_type);
   int16_t *nb = get_adapt_nb(cm->fc, tx_size, tx_type);
   assert(tx_size_2d[tx_size] <= COEFF_IDX_SIZE);
-  av1_update_sort_order(tx_size, non_zero_prob, sort_order);
+  av1_update_sort_order(tx_size, tx_type, non_zero_prob, sort_order);
   av1_update_scan_order(tx_size, sort_order, scan, iscan);
   av1_update_neighbors(tx_size, scan, iscan, nb);
 }