Add try_level_down() This function computes the overall (i.e. self and neighbors') cost difference caused by downgrading a coefficient by one level at a specific location Change-Id: I1b7b6acfe06ed06b9a2ff48b5bb11527646d1aa8
diff --git a/av1/encoder/encodetxb.c b/av1/encoder/encodetxb.c index bac23e9..3748798 100644 --- a/av1/encoder/encodetxb.c +++ b/av1/encoder/encodetxb.c
@@ -589,10 +589,161 @@ return qc > 0 ? qc - 1 : qc + 1; } -// TODO(angiebird): add static to this function once it's called -int try_self_level_down(tran_low_t *low_coeff, int coeff_idx, - TxbCache *txb_cache, TxbProbs *txb_probs, - TxbInfo *txb_info) { +static INLINE int get_mag_from_mag_arr(const int *mag_arr) { + int mag; + if (mag_arr[1] > 0) { + mag = mag_arr[0]; + } else if (mag_arr[0] > 0) { + mag = mag_arr[0] - 1; + } else { + // no neighbor + assert(mag_arr[0] == 0 && mag_arr[1] == 0); + mag = 0; + } + return mag; +} + +static int neighbor_level_down_update(int *new_count, int *new_mag, int count, + const int *mag, int coeff_idx, + tran_low_t abs_nb_coeff, int nb_coeff_idx, + int level, const TxbInfo *txb_info) { + *new_count = count; + *new_mag = get_mag_from_mag_arr(mag); + + int update = 0; + // check if br_count changes + if (abs_nb_coeff == level) { + update = 1; + *new_count -= 1; + assert(*new_count >= 0); + } + const int row = coeff_idx >> txb_info->bwl; + const int col = coeff_idx - (row << txb_info->bwl); + const int nb_row = nb_coeff_idx >> txb_info->bwl; + const int nb_col = nb_coeff_idx - (nb_row << txb_info->bwl); + + // check if mag changes + if (nb_row >= row && nb_col >= col) { + if (abs_nb_coeff == mag[0]) { + assert(mag[1] > 0); + if (mag[1] == 1) { + // the nb is the only qc with max mag + *new_mag -= 1; + assert(*new_mag >= 0); + update = 1; + } + } + } + return update; +} + +static int try_neighbor_level_down_br(int coeff_idx, int nb_coeff_idx, + const TxbCache *txb_cache, + const TxbProbs *txb_probs, + const TxbInfo *txb_info) { + const tran_low_t qc = txb_info->qcoeff[coeff_idx]; + const tran_low_t abs_qc = abs(qc); + const int level = NUM_BASE_LEVELS + 1; + if (abs_qc < level) return 0; + + const tran_low_t nb_coeff = txb_info->qcoeff[nb_coeff_idx]; + const tran_low_t abs_nb_coeff = abs(nb_coeff); + const int count = txb_cache->br_count_arr[coeff_idx]; + const int *mag = txb_cache->br_mag_arr[coeff_idx]; + int new_count; + int new_mag; + const int update = + neighbor_level_down_update(&new_count, &new_mag, count, mag, coeff_idx, + abs_nb_coeff, nb_coeff_idx, level, txb_info); + if (update) { + const int row = coeff_idx >> txb_info->bwl; + const int col = coeff_idx - (row << txb_info->bwl); + const int ctx = txb_cache->br_ctx_arr[coeff_idx][0]; + const int org_cost = get_br_cost(abs_qc, ctx, txb_probs->coeff_lps); + + const int new_ctx = get_br_ctx_from_count_mag(row, col, new_count, new_mag); + const int new_cost = get_br_cost(abs_qc, new_ctx, txb_probs->coeff_lps); + const int cost_diff = -org_cost + new_cost; + return cost_diff; + } else { + return 0; + } +} + +static int try_neighbor_level_down_base(int coeff_idx, int nb_coeff_idx, + const TxbCache *txb_cache, + const TxbProbs *txb_probs, + const TxbInfo *txb_info) { + const tran_low_t qc = txb_info->qcoeff[coeff_idx]; + const tran_low_t abs_qc = abs(qc); + + int cost_diff = 0; + for (int base_idx = 0; base_idx < NUM_BASE_LEVELS; ++base_idx) { + const int level = base_idx + 1; + if (abs_qc < level) continue; + + const tran_low_t nb_coeff = txb_info->qcoeff[nb_coeff_idx]; + const tran_low_t abs_nb_coeff = abs(nb_coeff); + + const int count = txb_cache->base_count_arr[base_idx][coeff_idx]; + const int *mag = txb_cache->base_mag_arr[coeff_idx]; + int new_count; + int new_mag; + const int update = + neighbor_level_down_update(&new_count, &new_mag, count, mag, coeff_idx, + abs_nb_coeff, nb_coeff_idx, level, txb_info); + if (update) { + const int row = coeff_idx >> txb_info->bwl; + const int col = coeff_idx - (row << txb_info->bwl); + const int ctx = txb_cache->base_ctx_arr[base_idx][coeff_idx][0]; + const int org_cost = + get_base_cost(abs_qc, ctx, txb_probs->coeff_base, base_idx); + + const int new_ctx = + get_base_ctx_from_count_mag(row, col, new_count, new_mag, level); + const int new_cost = + get_base_cost(abs_qc, new_ctx, txb_probs->coeff_base, base_idx); + cost_diff += -org_cost + new_cost; + } + } + return cost_diff; +} + +static int try_neighbor_level_down_nz(int coeff_idx, int nb_coeff_idx, + const TxbCache *txb_cache, + const TxbProbs *txb_probs, + TxbInfo *txb_info) { + // assume eob doesn't change + const tran_low_t qc = txb_info->qcoeff[coeff_idx]; + const tran_low_t abs_qc = abs(qc); + const tran_low_t nb_coeff = txb_info->qcoeff[nb_coeff_idx]; + const tran_low_t abs_nb_coeff = abs(nb_coeff); + if (abs_nb_coeff != 1) return 0; + const int16_t *iscan = txb_info->scan_order->iscan; + const int scan_idx = iscan[coeff_idx]; + if (scan_idx == txb_info->seg_eob) return 0; + const int nb_scan_idx = iscan[nb_coeff_idx]; + if (nb_scan_idx < scan_idx) { + const int count = txb_cache->nz_count_arr[coeff_idx]; + assert(count > 0); + txb_info->qcoeff[nb_coeff_idx] = get_lower_coeff(nb_coeff); + const int new_ctx = get_nz_map_ctx_from_count( + count - 1, txb_info->qcoeff, coeff_idx, txb_info->bwl, iscan); + txb_info->qcoeff[nb_coeff_idx] = nb_coeff; + const int ctx = txb_cache->nz_ctx_arr[coeff_idx][0]; + const int is_nz = abs_qc > 0; + const int org_cost = av1_cost_bit(txb_probs->nz_map[ctx], is_nz); + const int new_cost = av1_cost_bit(txb_probs->nz_map[new_ctx], is_nz); + const int cost_diff = new_cost - org_cost; + return cost_diff; + } else { + return 0; + } +} + +static int try_self_level_down(tran_low_t *low_coeff, int coeff_idx, + const TxbCache *txb_cache, + const TxbProbs *txb_probs, TxbInfo *txb_info) { const tran_low_t qc = txb_info->qcoeff[coeff_idx]; if (qc == 0) { *low_coeff = 0; @@ -603,9 +754,9 @@ int cost_diff; if (*low_coeff == 0) { const int scan_idx = txb_info->scan_order->iscan[coeff_idx]; - aom_prob level_prob = + const aom_prob level_prob = get_level_prob(abs_qc, coeff_idx, txb_cache, txb_probs); - aom_prob low_level_prob = + const aom_prob low_level_prob = get_level_prob(abs(*low_coeff), coeff_idx, txb_cache, txb_probs); if (scan_idx < txb_info->seg_eob) { // When level-0, we code the binary of abs_qc > level @@ -649,6 +800,102 @@ return cost_diff; } +#define COST_MAP_SIZE 5 +#define COST_MAP_OFFSET 2 + +static INLINE int check_nz_neighbor(tran_low_t qc) { return abs(qc) == 1; } + +static INLINE int check_base_neighbor(tran_low_t qc) { + return abs(qc) <= 1 + NUM_BASE_LEVELS; +} + +static INLINE int check_br_neighbor(tran_low_t qc) { + return abs(qc) > BR_MAG_OFFSET; +} + +// TODO(angiebird): add static to this function once it's called +int try_level_down(int coeff_idx, const TxbCache *txb_cache, + const TxbProbs *txb_probs, TxbInfo *txb_info, + int (*cost_map)[COST_MAP_SIZE]) { + if (cost_map) { + for (int i = 0; i < COST_MAP_SIZE; ++i) av1_zero(cost_map[i]); + } + + tran_low_t qc = txb_info->qcoeff[coeff_idx]; + tran_low_t low_coeff; + if (qc == 0) return 0; + int accu_cost_diff = 0; + + const int16_t *iscan = txb_info->scan_order->iscan; + const int eob = txb_info->eob; + const int scan_idx = iscan[coeff_idx]; + if (scan_idx < eob) { + const int cost_diff = try_self_level_down(&low_coeff, coeff_idx, txb_cache, + txb_probs, txb_info); + if (cost_map) + cost_map[0 + COST_MAP_OFFSET][0 + COST_MAP_OFFSET] = cost_diff; + accu_cost_diff += cost_diff; + } + + const int row = coeff_idx >> txb_info->bwl; + const int col = coeff_idx - (row << txb_info->bwl); + if (check_nz_neighbor(qc)) { + for (int i = 0; i < SIG_REF_OFFSET_NUM; ++i) { + const int nb_row = row - sig_ref_offset[i][0]; + const int nb_col = col - sig_ref_offset[i][1]; + const int nb_coeff_idx = nb_row * txb_info->stride + nb_col; + const int nb_scan_idx = iscan[nb_coeff_idx]; + if (nb_scan_idx < eob && nb_row >= 0 && nb_col >= 0 && + nb_row < txb_info->stride && nb_col < txb_info->stride) { + const int cost_diff = try_neighbor_level_down_nz( + nb_coeff_idx, coeff_idx, txb_cache, txb_probs, txb_info); + if (cost_map) + cost_map[nb_row - row + COST_MAP_OFFSET] + [nb_col - col + COST_MAP_OFFSET] += cost_diff; + accu_cost_diff += cost_diff; + } + } + } + + if (check_base_neighbor(qc)) { + for (int i = 0; i < BASE_CONTEXT_POSITION_NUM; ++i) { + const int nb_row = row - base_ref_offset[i][0]; + const int nb_col = col - base_ref_offset[i][1]; + const int nb_coeff_idx = nb_row * txb_info->stride + nb_col; + const int nb_scan_idx = iscan[nb_coeff_idx]; + if (nb_scan_idx < eob && nb_row >= 0 && nb_col >= 0 && + nb_row < txb_info->stride && nb_col < txb_info->stride) { + const int cost_diff = try_neighbor_level_down_base( + nb_coeff_idx, coeff_idx, txb_cache, txb_probs, txb_info); + if (cost_map) + cost_map[nb_row - row + COST_MAP_OFFSET] + [nb_col - col + COST_MAP_OFFSET] += cost_diff; + accu_cost_diff += cost_diff; + } + } + } + + if (check_br_neighbor(qc)) { + for (int i = 0; i < BR_CONTEXT_POSITION_NUM; ++i) { + const int nb_row = row - br_ref_offset[i][0]; + const int nb_col = col - br_ref_offset[i][1]; + const int nb_coeff_idx = nb_row * txb_info->stride + nb_col; + const int nb_scan_idx = iscan[nb_coeff_idx]; + if (nb_scan_idx < eob && nb_row >= 0 && nb_col >= 0 && + nb_row < txb_info->stride && nb_col < txb_info->stride) { + const int cost_diff = try_neighbor_level_down_br( + nb_coeff_idx, coeff_idx, txb_cache, txb_probs, txb_info); + if (cost_map) + cost_map[nb_row - row + COST_MAP_OFFSET] + [nb_col - col + COST_MAP_OFFSET] += cost_diff; + accu_cost_diff += cost_diff; + } + } + } + + return accu_cost_diff; +} + static int get_coeff_cost(tran_low_t qc, int scan_idx, TxbInfo *txb_info, TxbProbs *txb_probs) { const TXB_CTX *txb_ctx = txb_info->txb_ctx;