Use default scan order as a tie breaker
Change-Id: I85f059b6e2c48bcdf2edd3b7bf896fdccbaaa703
diff --git a/av1/common/scan.c b/av1/common/scan.c
index 0376b82..bf161ab 100644
--- a/av1/common/scan.c
+++ b/av1/common/scan.c
@@ -6458,15 +6458,17 @@
return *(const uint32_t *)b > *(const uint32_t *)a ? 1 : -1;
}
-void av1_augment_prob(uint32_t *prob, int size, int tx1d_size) {
+void av1_augment_prob(TX_SIZE tx_size, TX_TYPE tx_type, uint32_t *prob) {
+ // TODO(angiebird): check if we need is_inter here
+ const SCAN_ORDER *sc = get_default_scan(tx_size, tx_type, 0);
+ const int tx1d_size = tx_size_wide[tx_size];
int r, c;
- for (r = 0; r < size; r++) {
- for (c = 0; c < size; c++) {
- const int coeff_idx = r * tx1d_size + c;
- const int idx = r * size + c;
+ for (r = 0; r < tx1d_size; r++) {
+ for (c = 0; c < tx1d_size; c++) {
+ const int idx = r * tx1d_size + c;
const uint32_t mask_16 = ((1 << 16) - 1);
- const uint32_t tie_breaker = ~(((r + c) << COEFF_IDX_BITS) | coeff_idx);
- // prob[idx]: 16 bits r+c: 6 bits coeff_idx: 10 bits
+ const uint32_t tie_breaker = ~((uint32_t)sc->iscan[idx]);
+ // prob[idx]: 16 bits dummy: 6 bits scan_idx: 10 bits
prob[idx] = (prob[idx] << 16) | (mask_16 & tie_breaker);
}
}
@@ -6519,18 +6521,20 @@
neighbors[tx2d_size * MAX_NEIGHBORS + 1] = scan[0];
}
-void av1_update_sort_order(TX_SIZE tx_size, const uint32_t *non_zero_prob,
- int16_t *sort_order) {
+void av1_update_sort_order(TX_SIZE tx_size, TX_TYPE tx_type,
+ const uint32_t *non_zero_prob, int16_t *sort_order) {
+ const SCAN_ORDER *sc = get_default_scan(tx_size, tx_type, 0);
uint32_t temp[COEFF_IDX_SIZE];
- const int tx1d_size = tx_size_wide[tx_size];
const int tx2d_size = tx_size_2d[tx_size];
int sort_idx;
assert(tx2d_size <= COEFF_IDX_SIZE);
memcpy(temp, non_zero_prob, tx2d_size * sizeof(*non_zero_prob));
- av1_augment_prob(temp, tx1d_size, tx1d_size);
+ av1_augment_prob(tx_size, tx_type, temp);
qsort(temp, tx2d_size, sizeof(*temp), cmp_prob);
for (sort_idx = 0; sort_idx < tx2d_size; ++sort_idx) {
- const int coeff_idx = (temp[sort_idx] & COEFF_IDX_MASK) ^ COEFF_IDX_MASK;
+ const int default_scan_idx =
+ (temp[sort_idx] & COEFF_IDX_MASK) ^ COEFF_IDX_MASK;
+ const int coeff_idx = sc->scan[default_scan_idx];
sort_order[sort_idx] = coeff_idx;
}
}
@@ -6562,7 +6566,7 @@
int16_t *iscan = get_adapt_iscan(cm->fc, tx_size, tx_type);
int16_t *nb = get_adapt_nb(cm->fc, tx_size, tx_type);
assert(tx_size_2d[tx_size] <= COEFF_IDX_SIZE);
- av1_update_sort_order(tx_size, non_zero_prob, sort_order);
+ av1_update_sort_order(tx_size, tx_type, non_zero_prob, sort_order);
av1_update_scan_order(tx_size, sort_order, scan, iscan);
av1_update_neighbors(tx_size, scan, iscan, nb);
}
diff --git a/av1/common/scan.h b/av1/common/scan.h
index 01f37b6..71868d0 100644
--- a/av1/common/scan.h
+++ b/av1/common/scan.h
@@ -39,11 +39,11 @@
// embed r + c and coeff_idx info with nonzero probabilities. When sorting the
// nonzero probabilities, if there is a tie, the coefficient with smaller r + c
// will be scanned first
-void av1_augment_prob(uint32_t *prob, int size, int tx1d_size);
+void av1_augment_prob(TX_SIZE tx_size, TX_TYPE tx_type, uint32_t *prob);
// apply quick sort on nonzero probabilities to obtain a sort order
-void av1_update_sort_order(TX_SIZE tx_size, const uint32_t *non_zero_prob,
- int16_t *sort_order);
+void av1_update_sort_order(TX_SIZE tx_size, TX_TYPE tx_type,
+ const uint32_t *non_zero_prob, int16_t *sort_order);
// apply topological sort on the nonzero probabilities sorting order to
// guarantee each to-be-scanned coefficient's upper and left coefficient will be
@@ -67,13 +67,9 @@
1;
}
-static INLINE const SCAN_ORDER *get_scan(const AV1_COMMON *cm, TX_SIZE tx_size,
- TX_TYPE tx_type, int is_inter) {
-#if CONFIG_ADAPT_SCAN
- (void)is_inter;
- return &cm->fc->sc[tx_size][tx_type];
-#else // CONFIG_ADAPT_SCAN
- (void)cm;
+static INLINE const SCAN_ORDER *get_default_scan(TX_SIZE tx_size,
+ TX_TYPE tx_type,
+ int is_inter) {
#if CONFIG_EXT_TX || CONFIG_VAR_TX
return is_inter ? &av1_inter_scan_orders[tx_size][tx_type]
: &av1_intra_scan_orders[tx_size][tx_type];
@@ -81,6 +77,16 @@
(void)is_inter;
return &av1_intra_scan_orders[tx_size][tx_type];
#endif // CONFIG_EXT_TX
+}
+
+static INLINE const SCAN_ORDER *get_scan(const AV1_COMMON *cm, TX_SIZE tx_size,
+ TX_TYPE tx_type, int is_inter) {
+#if CONFIG_ADAPT_SCAN
+ (void)is_inter;
+ return &cm->fc->sc[tx_size][tx_type];
+#else // CONFIG_ADAPT_SCAN
+ (void)cm;
+ return get_default_scan(tx_size, tx_type, is_inter);
#endif // CONFIG_ADAPT_SCAN
}
