Remove dead code of lv_map Change-Id: I0e0926119599197af0720470d4bb16156b679954
diff --git a/av1/encoder/encodetxb.c b/av1/encoder/encodetxb.c index 5042512..f406fb8 100644 --- a/av1/encoder/encodetxb.c +++ b/av1/encoder/encodetxb.c
@@ -23,8 +23,6 @@ #include "av1/encoder/rdopt.h" #include "av1/encoder/tokenize.h" -#define TEST_OPTIMIZE_TXB 0 - static int hbt_needs_init = 1; static CRC_CALCULATOR crc_calculator; static const int HBT_EOB = 16; // also the length in opt_qcoeff @@ -63,21 +61,6 @@ } LevelDownStats; void av1_alloc_txb_buf(AV1_COMP *cpi) { -#if 0 - AV1_COMMON *cm = &cpi->common; - const int num_planes = av1_num_planes(cm); - int mi_block_size = 1 << MI_SIZE_LOG2; - // TODO(angiebird): Make sure cm->subsampling_x/y is set correctly, and then - // use precise buffer size according to cm->subsampling_x/y - int pixel_stride = mi_block_size * cm->mi_cols; - int pixel_height = mi_block_size * cm->mi_rows; - int i; - for (i = 0; i < num_planes; ++i) { - CHECK_MEM_ERROR( - cm, cpi->tcoeff_buf[i], - aom_malloc(sizeof(*cpi->tcoeff_buf[i]) * pixel_stride * pixel_height)); - } -#else AV1_COMMON *cm = &cpi->common; int size = ((cm->mi_rows >> cm->seq_params.mib_size_log2) + 1) * ((cm->mi_cols >> cm->seq_params.mib_size_log2) + 1); @@ -86,21 +69,9 @@ // TODO(jingning): This should be further reduced. CHECK_MEM_ERROR(cm, cpi->coeff_buffer_base, aom_malloc(sizeof(*cpi->coeff_buffer_base) * size)); -#endif } -void av1_free_txb_buf(AV1_COMP *cpi) { -#if 0 - int i; - const AV1_COMMON *const cm = &cpi->common; - const int num_planes = av1_num_planes(cm); - for (i = 0; i < num_planes; ++i) { - aom_free(cpi->tcoeff_buf[i]); - } -#else - aom_free(cpi->coeff_buffer_base); -#endif -} +void av1_free_txb_buf(AV1_COMP *cpi) { aom_free(cpi->coeff_buffer_base); } void av1_set_coeff_buffer(const AV1_COMP *const cpi, MACROBLOCK *const x, int mi_row, int mi_col) { @@ -732,547 +703,11 @@ } } -void gen_txb_cache(TxbCache *txb_cache, TxbInfo *txb_info) { - // gen_nz_count_arr - const int16_t *const scan = txb_info->scan_order->scan; - const int bwl = txb_info->bwl; - const int height = txb_info->height; - const tran_low_t *const qcoeff = txb_info->qcoeff; - const uint8_t *const levels = txb_info->levels; - const BASE_CTX_TABLE *base_ctx_table = - txb_info->coeff_ctx_table->base_ctx_table; - for (int c = 0; c < txb_info->eob; ++c) { - const int coeff_idx = scan[c]; // raster order - const int row = coeff_idx >> bwl; - const int col = coeff_idx - (row << bwl); - - txb_cache->nz_count_arr[coeff_idx] = - get_nz_count(levels + get_padded_idx(coeff_idx, bwl), bwl, - tx_type_to_class[txb_info->tx_type]); - - txb_cache->nz_ctx_arr[coeff_idx] = - get_nz_map_ctx_from_stats(0, coeff_idx, bwl, txb_info->tx_size, - tx_type_to_class[txb_info->tx_type]); - - // gen_base_count_mag_arr - if (!has_base(qcoeff[coeff_idx], 0)) continue; - int *base_mag = txb_cache->base_mag_arr[coeff_idx]; - int count[NUM_BASE_LEVELS]; - get_base_count_mag(base_mag, count, qcoeff, bwl, height, row, col); - - for (int i = 0; i < NUM_BASE_LEVELS; ++i) { - if (!has_base(qcoeff[coeff_idx], i)) break; - txb_cache->base_count_arr[i][coeff_idx] = count[i]; - const int level = i + 1; - txb_cache->base_ctx_arr[i][coeff_idx] = - base_ctx_table[row != 0][col != 0][base_mag[0] > level][count[i]]; - } - - // gen_br_count_mag_arr - if (!has_br(qcoeff[coeff_idx])) continue; - int *br_count = txb_cache->br_count_arr + coeff_idx; - int *br_mag = txb_cache->br_mag_arr[coeff_idx]; - *br_count = get_br_count_mag(br_mag, qcoeff, bwl, height, row, col, - NUM_BASE_LEVELS); - txb_cache->br_ctx_arr[coeff_idx] = - get_br_ctx_from_count_mag(row, col, *br_count, br_mag[0]); - } -} - -static INLINE const int *get_level_prob(int level, int coeff_idx, - const TxbCache *txb_cache, - const LV_MAP_COEFF_COST *txb_costs) { - if (level < 1 + NUM_BASE_LEVELS) { - const int ctx = txb_cache->nz_ctx_arr[coeff_idx]; - return &txb_costs->base_cost[ctx][level]; - } else if (level >= 1 + NUM_BASE_LEVELS && - level < 1 + NUM_BASE_LEVELS + COEFF_BASE_RANGE) { - const int ctx = txb_cache->br_ctx_arr[coeff_idx]; - return txb_costs->lps_cost[ctx]; - } else if (level >= 1 + NUM_BASE_LEVELS + COEFF_BASE_RANGE) { - // printf("get_level_prob does not support golomb\n"); - assert(0); - return 0; - } else { - assert(0); - return 0; - } -} - -static INLINE void update_mag_arr(int *mag_arr, int abs_qc) { - if (mag_arr[0] == abs_qc) { - mag_arr[1] -= 1; - assert(mag_arr[1] >= 0); - } -} - -static INLINE int get_mag_from_mag_arr(const int *mag_arr) { - int mag; - if (mag_arr[1] > 0) { - mag = mag_arr[0]; - } else if (mag_arr[0] > 0) { - mag = mag_arr[0] - 1; - } else { - // no neighbor - assert(mag_arr[0] == 0 && mag_arr[1] == 0); - mag = 0; - } - return mag; -} - -static int neighbor_level_down_update(int *new_count, int *new_mag, int count, - const int *mag, int coeff_idx, - tran_low_t abs_nb_coeff, int nb_coeff_idx, - int level, const TxbInfo *txb_info) { - *new_count = count; - *new_mag = get_mag_from_mag_arr(mag); - - int update = 0; - // check if br_count changes - if (abs_nb_coeff == level) { - update = 1; - *new_count -= 1; - assert(*new_count >= 0); - } - const int row = coeff_idx >> txb_info->bwl; - const int col = coeff_idx - (row << txb_info->bwl); - const int nb_row = nb_coeff_idx >> txb_info->bwl; - const int nb_col = nb_coeff_idx - (nb_row << txb_info->bwl); - - // check if mag changes - if (nb_row >= row && nb_col >= col) { - if (abs_nb_coeff == mag[0]) { - assert(mag[1] > 0); - if (mag[1] == 1) { - // the nb is the only qc with max mag - *new_mag -= 1; - assert(*new_mag >= 0); - update = 1; - } - } - } - return update; -} - -static int try_neighbor_level_down_br(int coeff_idx, int nb_coeff_idx, - const TxbCache *txb_cache, - const LV_MAP_COEFF_COST *txb_costs, - const TxbInfo *txb_info) { - const tran_low_t qc = txb_info->qcoeff[coeff_idx]; - const tran_low_t abs_qc = abs(qc); - const int level = NUM_BASE_LEVELS + 1; - if (abs_qc < level) return 0; - - const tran_low_t nb_coeff = txb_info->qcoeff[nb_coeff_idx]; - const tran_low_t abs_nb_coeff = abs(nb_coeff); - const int count = txb_cache->br_count_arr[coeff_idx]; - const int *mag = txb_cache->br_mag_arr[coeff_idx]; - int new_count; - int new_mag; - const int update = - neighbor_level_down_update(&new_count, &new_mag, count, mag, coeff_idx, - abs_nb_coeff, nb_coeff_idx, level, txb_info); - if (update) { - const int row = coeff_idx >> txb_info->bwl; - const int col = coeff_idx - (row << txb_info->bwl); - const int ctx = txb_cache->br_ctx_arr[coeff_idx]; - const int org_cost = get_br_cost(abs_qc, ctx, txb_costs->lps_cost[ctx]); - - const int new_ctx = get_br_ctx_from_count_mag(row, col, new_count, new_mag); - const int new_cost = - get_br_cost(abs_qc, new_ctx, txb_costs->lps_cost[new_ctx]); - const int cost_diff = -org_cost + new_cost; - return cost_diff; - } else { - return 0; - } -} - -static int try_neighbor_level_down_base(int coeff_idx, int nb_coeff_idx, - const TxbCache *txb_cache, - const LV_MAP_COEFF_COST *txb_costs, - const TxbInfo *txb_info) { - // TODO(olah): not implemented yet - (void)coeff_idx; - (void)nb_coeff_idx; - (void)txb_cache; - (void)txb_costs; - (void)txb_info; - return 0; -} - -static int try_neighbor_level_down_nz(int coeff_idx, int nb_coeff_idx, - const TxbCache *txb_cache, - const LV_MAP_COEFF_COST *txb_costs, - TxbInfo *txb_info) { - // assume eob doesn't change - const tran_low_t qc = txb_info->qcoeff[coeff_idx]; - const tran_low_t abs_qc = abs(qc); - const tran_low_t nb_coeff = txb_info->qcoeff[nb_coeff_idx]; - const tran_low_t abs_nb_coeff = abs(nb_coeff); - if (abs_nb_coeff != 1) return 0; - const int16_t *const iscan = txb_info->scan_order->iscan; - const int scan_idx = iscan[coeff_idx]; - if (scan_idx == txb_info->seg_eob) return 0; - const int nb_scan_idx = iscan[nb_coeff_idx]; - if (nb_scan_idx < scan_idx) { - const int count = txb_cache->nz_count_arr[coeff_idx]; - (void)count; - assert(count > 0); - update_qcoeff(nb_coeff_idx, get_lower_coeff(nb_coeff), txb_info); - const int new_ctx = get_nz_map_ctx_from_stats( - 0, coeff_idx, txb_info->bwl, txb_info->tx_size, - tx_type_to_class[txb_info->tx_type]); - update_qcoeff(nb_coeff_idx, nb_coeff, txb_info); - const int ctx = txb_cache->nz_ctx_arr[coeff_idx]; - const int org_cost = txb_costs->base_cost[ctx][AOMMIN(abs_qc, 3)]; - const int new_cost = txb_costs->base_cost[new_ctx][AOMMIN(abs_qc, 3)]; - const int cost_diff = new_cost - org_cost; - return cost_diff; - } else { - return 0; - } -} - -static int try_self_level_down(tran_low_t *low_coeff, int coeff_idx, - const TxbCache *txb_cache, - const LV_MAP_COEFF_COST *txb_costs, - TxbInfo *txb_info) { - const tran_low_t qc = txb_info->qcoeff[coeff_idx]; - if (qc == 0) { - *low_coeff = 0; - return 0; - } - const tran_low_t abs_qc = abs(qc); - *low_coeff = get_lower_coeff(qc); - int cost_diff; - if (*low_coeff == 0) { - const int scan_idx = txb_info->scan_order->iscan[coeff_idx]; - const int *level_cost = - get_level_prob(abs_qc, coeff_idx, txb_cache, txb_costs); - const int *low_level_cost = - get_level_prob(abs(*low_coeff), coeff_idx, txb_cache, txb_costs); - - if (scan_idx < txb_info->eob - 1) { - // When level-0, we code the binary of abs_qc > level - // but when level-k k > 0 we code the binary of abs_qc == level - // That's why wee need this special treatment for level-0 map - // TODO(angiebird): make leve-0 consistent to other levels - cost_diff = -level_cost[1] + low_level_cost[0] - low_level_cost[1]; - } else { - cost_diff = -level_cost[1]; - } - - const int sign_cost = get_sign_bit_cost( - qc, coeff_idx, txb_costs->dc_sign_cost, txb_info->txb_ctx->dc_sign_ctx); - cost_diff -= sign_cost; - } else if (abs_qc <= NUM_BASE_LEVELS) { - const int *level_cost = - get_level_prob(abs_qc, coeff_idx, txb_cache, txb_costs); - const int *low_level_cost = - get_level_prob(abs(*low_coeff), coeff_idx, txb_cache, txb_costs); - cost_diff = -level_cost[1] + low_level_cost[1] - low_level_cost[0]; - } else if (abs_qc == NUM_BASE_LEVELS + 1) { - const int *level_cost = - get_level_prob(abs_qc, coeff_idx, txb_cache, txb_costs); - const int *low_level_cost = - get_level_prob(abs(*low_coeff), coeff_idx, txb_cache, txb_costs); - cost_diff = -level_cost[0] + low_level_cost[1] - low_level_cost[0]; - } else if (abs_qc < 1 + NUM_BASE_LEVELS + COEFF_BASE_RANGE) { - const int *level_cost = - get_level_prob(abs_qc, coeff_idx, txb_cache, txb_costs); - const int *low_level_cost = - get_level_prob(abs(*low_coeff), coeff_idx, txb_cache, txb_costs); - - cost_diff = -level_cost[abs_qc - 1 - NUM_BASE_LEVELS] + - low_level_cost[abs(*low_coeff) - 1 - NUM_BASE_LEVELS]; - } else if (abs_qc == 1 + NUM_BASE_LEVELS + COEFF_BASE_RANGE) { - const int *low_level_cost = - get_level_prob(abs(*low_coeff), coeff_idx, txb_cache, txb_costs); - cost_diff = -get_golomb_cost(abs_qc) - low_level_cost[COEFF_BASE_RANGE] + - low_level_cost[COEFF_BASE_RANGE - 1]; - } else { - assert(abs_qc > 1 + NUM_BASE_LEVELS + COEFF_BASE_RANGE); - const tran_low_t abs_low_coeff = abs(*low_coeff); - cost_diff = -get_golomb_cost(abs_qc) + get_golomb_cost(abs_low_coeff); - } - return cost_diff; -} - -#define COST_MAP_SIZE 5 -#define COST_MAP_OFFSET 2 - -static INLINE int check_nz_neighbor(tran_low_t qc) { return abs(qc) == 1; } - -static INLINE int check_base_neighbor(tran_low_t qc) { - return abs(qc) <= 1 + NUM_BASE_LEVELS; -} - -static INLINE int check_br_neighbor(tran_low_t qc) { - return abs(qc) > BR_MAG_OFFSET; -} - -#define FAST_OPTIMIZE_TXB 1 - -#if FAST_OPTIMIZE_TXB -#define ALNB_REF_OFFSET_NUM 2 -static const int alnb_ref_offset[ALNB_REF_OFFSET_NUM][2] = { - { -1, 0 }, - { 0, -1 }, -}; -#define NB_REF_OFFSET_NUM 4 -static const int nb_ref_offset[NB_REF_OFFSET_NUM][2] = { - { -1, 0 }, - { 0, -1 }, - { 1, 0 }, - { 0, 1 }, -}; -#endif // FAST_OPTIMIZE_TXB - -// TODO(angiebird): add static to this function once it's called -int try_level_down(int coeff_idx, const TxbCache *txb_cache, - const LV_MAP_COEFF_COST *txb_costs, TxbInfo *txb_info, - int (*cost_map)[COST_MAP_SIZE], int fast_mode) { -#if !FAST_OPTIMIZE_TXB - (void)fast_mode; -#endif - if (cost_map) { - for (int i = 0; i < COST_MAP_SIZE; ++i) av1_zero(cost_map[i]); - } - - tran_low_t qc = txb_info->qcoeff[coeff_idx]; - tran_low_t low_coeff; - if (qc == 0) return 0; - int accu_cost_diff = 0; - - const int16_t *const iscan = txb_info->scan_order->iscan; - const int eob = txb_info->eob; - const int scan_idx = iscan[coeff_idx]; - if (scan_idx < eob) { - const int cost_diff = try_self_level_down(&low_coeff, coeff_idx, txb_cache, - txb_costs, txb_info); - if (cost_map) - cost_map[0 + COST_MAP_OFFSET][0 + COST_MAP_OFFSET] = cost_diff; - accu_cost_diff += cost_diff; - } - - const int row = coeff_idx >> txb_info->bwl; - const int col = coeff_idx - (row << txb_info->bwl); - if (check_nz_neighbor(qc)) { -#if FAST_OPTIMIZE_TXB - const int(*ref_offset)[2]; - int ref_num; - if (fast_mode) { - ref_offset = alnb_ref_offset; - ref_num = ALNB_REF_OFFSET_NUM; - } else { - ref_offset = sig_ref_offset; - ref_num = SIG_REF_OFFSET_NUM; - } -#else - const int(*ref_offset)[2] = sig_ref_offset; - const int ref_num = SIG_REF_OFFSET_NUM; -#endif - for (int i = 0; i < ref_num; ++i) { - const int nb_row = row - ref_offset[i][0]; - const int nb_col = col - ref_offset[i][1]; - const int nb_coeff_idx = nb_row * txb_info->width + nb_col; - - if (nb_row < 0 || nb_col < 0 || nb_row >= txb_info->height || - nb_col >= txb_info->width) - continue; - - const int nb_scan_idx = iscan[nb_coeff_idx]; - if (nb_scan_idx < eob) { - const int cost_diff = try_neighbor_level_down_nz( - nb_coeff_idx, coeff_idx, txb_cache, txb_costs, txb_info); - if (cost_map) - cost_map[nb_row - row + COST_MAP_OFFSET] - [nb_col - col + COST_MAP_OFFSET] += cost_diff; - accu_cost_diff += cost_diff; - } - } - } - - if (check_base_neighbor(qc)) { -#if FAST_OPTIMIZE_TXB - const int(*ref_offset)[2]; - int ref_num; - if (fast_mode) { - ref_offset = nb_ref_offset; - ref_num = NB_REF_OFFSET_NUM; - } else { - ref_offset = base_ref_offset; - ref_num = BASE_CONTEXT_POSITION_NUM; - } -#else - const int(*ref_offset)[2] = base_ref_offset; - int ref_num = BASE_CONTEXT_POSITION_NUM; -#endif - for (int i = 0; i < ref_num; ++i) { - const int nb_row = row - ref_offset[i][0]; - const int nb_col = col - ref_offset[i][1]; - const int nb_coeff_idx = nb_row * txb_info->width + nb_col; - - if (nb_row < 0 || nb_col < 0 || nb_row >= txb_info->height || - nb_col >= txb_info->width) - continue; - - const int nb_scan_idx = iscan[nb_coeff_idx]; - if (nb_scan_idx < eob) { - const int cost_diff = try_neighbor_level_down_base( - nb_coeff_idx, coeff_idx, txb_cache, txb_costs, txb_info); - if (cost_map) - cost_map[nb_row - row + COST_MAP_OFFSET] - [nb_col - col + COST_MAP_OFFSET] += cost_diff; - accu_cost_diff += cost_diff; - } - } - } - - if (check_br_neighbor(qc)) { -#if FAST_OPTIMIZE_TXB - const int(*ref_offset)[2]; - int ref_num; - if (fast_mode) { - ref_offset = nb_ref_offset; - ref_num = NB_REF_OFFSET_NUM; - } else { - ref_offset = br_ref_offset; - ref_num = BR_CONTEXT_POSITION_NUM; - } -#else - const int(*ref_offset)[2] = br_ref_offset; - const int ref_num = BR_CONTEXT_POSITION_NUM; -#endif - for (int i = 0; i < ref_num; ++i) { - const int nb_row = row - ref_offset[i][0]; - const int nb_col = col - ref_offset[i][1]; - const int nb_coeff_idx = nb_row * txb_info->width + nb_col; - - if (nb_row < 0 || nb_col < 0 || nb_row >= txb_info->height || - nb_col >= txb_info->width) - continue; - - const int nb_scan_idx = iscan[nb_coeff_idx]; - if (nb_scan_idx < eob) { - const int cost_diff = try_neighbor_level_down_br( - nb_coeff_idx, coeff_idx, txb_cache, txb_costs, txb_info); - if (cost_map) - cost_map[nb_row - row + COST_MAP_OFFSET] - [nb_col - col + COST_MAP_OFFSET] += cost_diff; - accu_cost_diff += cost_diff; - } - } - } - - return accu_cost_diff; -} - static INLINE void set_eob(TxbInfo *txb_info, int eob) { txb_info->eob = eob; txb_info->seg_eob = av1_get_max_eob(txb_info->tx_size); } -// TODO(angiebird): add static to this function it's called -void update_level_down(const int coeff_idx, TxbCache *const txb_cache, - TxbInfo *const txb_info) { - const tran_low_t qc = txb_info->qcoeff[coeff_idx]; - const int abs_qc = abs(qc); - if (qc == 0) return; - const tran_low_t low_coeff = get_lower_coeff(qc); - update_coeff(coeff_idx, low_coeff, txb_info); - - const int row = coeff_idx >> txb_info->bwl; - const int col = coeff_idx - (row << txb_info->bwl); - const int eob = txb_info->eob; - const int16_t *const iscan = txb_info->scan_order->iscan; - for (int i = 0; i < SIG_REF_OFFSET_NUM; ++i) { - const int nb_row = row - sig_ref_offset[i][0]; - const int nb_col = col - sig_ref_offset[i][1]; - - if (!(nb_row >= 0 && nb_col >= 0 && nb_row < txb_info->height && - nb_col < txb_info->width)) - continue; - - const int nb_coeff_idx = nb_row * txb_info->width + nb_col; - const int nb_scan_idx = iscan[nb_coeff_idx]; - if (nb_scan_idx < eob) { - const int scan_idx = iscan[coeff_idx]; - if (scan_idx < nb_scan_idx) { - const int level = 1; - if (abs_qc == level) { - txb_cache->nz_count_arr[nb_coeff_idx] -= 1; - assert(txb_cache->nz_count_arr[nb_coeff_idx] >= 0); - } - txb_cache->nz_ctx_arr[nb_coeff_idx] = get_nz_map_ctx_from_stats( - 0, nb_coeff_idx, txb_info->bwl, txb_info->tx_size, - tx_type_to_class[txb_info->tx_type]); - } - } - } - - const BASE_CTX_TABLE *base_ctx_table = - txb_info->coeff_ctx_table->base_ctx_table; - for (int i = 0; i < BASE_CONTEXT_POSITION_NUM; ++i) { - const int nb_row = row - base_ref_offset[i][0]; - const int nb_col = col - base_ref_offset[i][1]; - const int nb_coeff_idx = nb_row * txb_info->width + nb_col; - - if (!(nb_row >= 0 && nb_col >= 0 && nb_row < txb_info->height && - nb_col < txb_info->width)) - continue; - - const tran_low_t nb_coeff = txb_info->qcoeff[nb_coeff_idx]; - if (!has_base(nb_coeff, 0)) continue; - const int nb_scan_idx = iscan[nb_coeff_idx]; - if (nb_scan_idx < eob) { - if (row >= nb_row && col >= nb_col) - update_mag_arr(txb_cache->base_mag_arr[nb_coeff_idx], abs_qc); - const int mag = - get_mag_from_mag_arr(txb_cache->base_mag_arr[nb_coeff_idx]); - for (int base_idx = 0; base_idx < NUM_BASE_LEVELS; ++base_idx) { - if (!has_base(nb_coeff, base_idx)) continue; - const int level = base_idx + 1; - if (abs_qc == level) { - txb_cache->base_count_arr[base_idx][nb_coeff_idx] -= 1; - assert(txb_cache->base_count_arr[base_idx][nb_coeff_idx] >= 0); - } - const int count = txb_cache->base_count_arr[base_idx][nb_coeff_idx]; - txb_cache->base_ctx_arr[base_idx][nb_coeff_idx] = - base_ctx_table[nb_row != 0][nb_col != 0][mag > level][count]; - } - } - } - - for (int i = 0; i < BR_CONTEXT_POSITION_NUM; ++i) { - const int nb_row = row - br_ref_offset[i][0]; - const int nb_col = col - br_ref_offset[i][1]; - const int nb_coeff_idx = nb_row * txb_info->width + nb_col; - - if (!(nb_row >= 0 && nb_col >= 0 && nb_row < txb_info->height && - nb_col < txb_info->width)) - continue; - - const int nb_scan_idx = iscan[nb_coeff_idx]; - const tran_low_t nb_coeff = txb_info->qcoeff[nb_coeff_idx]; - if (!has_br(nb_coeff)) continue; - if (nb_scan_idx < eob) { - const int level = 1 + NUM_BASE_LEVELS; - if (abs_qc == level) { - txb_cache->br_count_arr[nb_coeff_idx] -= 1; - assert(txb_cache->br_count_arr[nb_coeff_idx] >= 0); - } - if (row >= nb_row && col >= nb_col) - update_mag_arr(txb_cache->br_mag_arr[nb_coeff_idx], abs_qc); - const int count = txb_cache->br_count_arr[nb_coeff_idx]; - const int mag = get_mag_from_mag_arr(txb_cache->br_mag_arr[nb_coeff_idx]); - txb_cache->br_ctx_arr[nb_coeff_idx] = - get_br_ctx_from_count_mag(nb_row, nb_col, count, mag); - } - } -} - static int get_coeff_cost(const tran_low_t qc, const int scan_idx, const int is_eob, const TxbInfo *const txb_info, const LV_MAP_COEFF_COST *const txb_costs, @@ -1303,146 +738,6 @@ return cost; } -#if TEST_OPTIMIZE_TXB -#define ALL_REF_OFFSET_NUM 17 -static const int all_ref_offset[ALL_REF_OFFSET_NUM][2] = { - { 0, 0 }, { -2, -1 }, { -2, 0 }, { -2, 1 }, { -1, -2 }, { -1, -1 }, - { -1, 0 }, { -1, 1 }, { 0, -2 }, { 0, -1 }, { 1, -2 }, { 1, -1 }, - { 1, 0 }, { 2, 0 }, { 0, 1 }, { 0, 2 }, { 1, 1 }, -}; - -static int try_level_down_ref(int coeff_idx, const LV_MAP_COEFF_COST *txb_costs, - TxbInfo *txb_info, - int (*cost_map)[COST_MAP_SIZE]) { - if (cost_map) { - for (int i = 0; i < COST_MAP_SIZE; ++i) av1_zero(cost_map[i]); - } - tran_low_t qc = txb_info->qcoeff[coeff_idx]; - if (qc == 0) return 0; - int row = coeff_idx >> txb_info->bwl; - int col = coeff_idx - (row << txb_info->bwl); - int org_cost = 0; - for (int i = 0; i < ALL_REF_OFFSET_NUM; ++i) { - int nb_row = row - all_ref_offset[i][0]; - int nb_col = col - all_ref_offset[i][1]; - int nb_coeff_idx = nb_row * txb_info->width + nb_col; - int nb_scan_idx = txb_info->scan_order->iscan[nb_coeff_idx]; - if (nb_scan_idx < txb_info->eob && nb_row >= 0 && nb_col >= 0 && - nb_row < txb_info->height && nb_col < txb_info->width) { - const tran_low_t nb_coeff = txb_info->qcoeff[nb_coeff_idx]; - const int coeff_ctx = get_nz_map_ctx( - txb_info->levels, nb_coeff_idx, txb_info->bwl, txb_info->height, - nb_scan_idx, is_eob, txb_info->tx_size, txb_info->tx_type); - const int cost = get_coeff_cost(nb_coeff, nb_scan_idx, is_eob, txb_info, - txb_costs, coeff_ctx); - if (cost_map) - cost_map[nb_row - row + COST_MAP_OFFSET] - [nb_col - col + COST_MAP_OFFSET] -= cost; - org_cost += cost; - } - } - update_qcoeff(coeff_idx, get_lower_coeff(qc), txb_info); - int new_cost = 0; - for (int i = 0; i < ALL_REF_OFFSET_NUM; ++i) { - int nb_row = row - all_ref_offset[i][0]; - int nb_col = col - all_ref_offset[i][1]; - int nb_coeff_idx = nb_row * txb_info->width + nb_col; - int nb_scan_idx = txb_info->scan_order->iscan[nb_coeff_idx]; - if (nb_scan_idx < txb_info->eob && nb_row >= 0 && nb_col >= 0 && - nb_row < txb_info->height && nb_col < txb_info->width) { - const tran_low_t nb_coeff = txb_info->qcoeff[nb_coeff_idx]; - const int coeff_ctx = get_nz_map_ctx( - txb_info->levels, nb_coeff_idx, txb_info->bwl, txb_info->height, - nb_scan_idx, is_eob, txb_info->tx_size, txb_info->tx_type); - const int cost = get_coeff_cost(nb_coeff, nb_scan_idx, is_eob, txb_info, - txb_costs, coeff_ctx); - if (cost_map) - cost_map[nb_row - row + COST_MAP_OFFSET] - [nb_col - col + COST_MAP_OFFSET] += cost; - new_cost += cost; - } - } - update_qcoeff(coeff_idx, qc, txb_info); - return new_cost - org_cost; -} - -static void test_level_down(int coeff_idx, const TxbCache *txb_cache, - const LV_MAP_COEFF_COST *txb_costs, - TxbInfo *txb_info) { - int cost_map[COST_MAP_SIZE][COST_MAP_SIZE]; - int ref_cost_map[COST_MAP_SIZE][COST_MAP_SIZE]; - const int cost_diff = - try_level_down(coeff_idx, txb_cache, txb_costs, txb_info, cost_map, 0); - const int cost_diff_ref = - try_level_down_ref(coeff_idx, txb_costs, txb_info, ref_cost_map); - if (cost_diff != cost_diff_ref) { - printf("qc %d cost_diff %d cost_diff_ref %d\n", txb_info->qcoeff[coeff_idx], - cost_diff, cost_diff_ref); - for (int r = 0; r < COST_MAP_SIZE; ++r) { - for (int c = 0; c < COST_MAP_SIZE; ++c) { - printf("%d:%d ", cost_map[r][c], ref_cost_map[r][c]); - } - printf("\n"); - } - } -} -#endif - -// TODO(angiebird): make this static once it's called -int get_txb_cost(TxbInfo *txb_info, const LV_MAP_COEFF_COST *txb_costs) { - int cost = 0; - const int txb_skip_ctx = txb_info->txb_ctx->txb_skip_ctx; - const int16_t *const scan = txb_info->scan_order->scan; - if (txb_info->eob == 0) { - cost = txb_costs->txb_skip_cost[txb_skip_ctx][1]; - return cost; - } - cost = txb_costs->txb_skip_cost[txb_skip_ctx][0]; - for (int c = 0; c < txb_info->eob; ++c) { - const int pos = scan[c]; - const tran_low_t qc = txb_info->qcoeff[pos]; - const int coeff_ctx = get_nz_map_ctx( - txb_info->levels, pos, txb_info->bwl, txb_info->height, c, - c == txb_info->eob - 1, txb_info->tx_size, txb_info->tx_type); - const int coeff_cost = get_coeff_cost(qc, c, c == txb_info->eob - 1, - txb_info, txb_costs, coeff_ctx); - cost += coeff_cost; - } - return cost; -} - -#if TEST_OPTIMIZE_TXB -void test_try_change_eob(TxbInfo *txb_info, const LV_MAP_COEFF_COST *txb_costs, - TxbCache *txb_cache) { - const int eob = txb_info->eob; - const int16_t *const scan = txb_info->scan_order->scan; - if (eob > 0) { - int last_si = eob - 1; - int last_ci = scan[last_si]; - int last_coeff = txb_info->qcoeff[last_ci]; - if (abs(last_coeff) == 1) { - int new_eob; - int cost_diff = - try_change_eob(&new_eob, last_ci, txb_cache, txb_costs, txb_info, 0); - int org_eob = txb_info->eob; - int cost = get_txb_cost(txb_info, txb_costs); - - update_qcoeff(last_ci, get_lower_coeff(last_coeff), txb_info); - set_eob(txb_info, new_eob); - int new_cost = get_txb_cost(txb_info, txb_costs); - set_eob(txb_info, org_eob); - update_qcoeff(last_ci, last_coeff, txb_info); - - int ref_cost_diff = -cost + new_cost; - if (cost_diff != ref_cost_diff) - printf("org_eob %d new_eob %d cost_diff %d ref_cost_diff %d\n", org_eob, - new_eob, cost_diff, ref_cost_diff); - } - } -} -#endif - -#if 1 static int optimize_txb(TxbInfo *txb_info, const LV_MAP_COEFF_COST *txb_costs, const LV_MAP_EOB_COST *txb_eob_costs, TxbCache *txb_cache, int dry_run, int fast_mode, @@ -1454,14 +749,6 @@ if (txb_info->eob == 0) return update; const int max_eob = av1_get_max_eob(txb_info->tx_size); -#if TEST_OPTIMIZE_TXB - int64_t sse; - int64_t org_dist = - av1_block_error_c(txb_info->tcoeff, txb_info->dqcoeff, max_eob, &sse) * - (1 << (2 * txb_info->shift)); - int org_cost = get_txb_cost(txb_info, txb_probs); -#endif - tran_low_t *org_qcoeff = txb_info->qcoeff; tran_low_t *org_dqcoeff = txb_info->dqcoeff; uint8_t *const org_levels = txb_info->levels; @@ -1603,23 +890,6 @@ ? zero_blk_rate : accu_rate + non_zero_blk_rate; -#if TEST_OPTIMIZE_TXB - int cost_diff = 0; - int64_t dist_diff = 0; - int64_t rd_diff = 0; - int64_t new_dist = - av1_block_error_c(txb_info->tcoeff, txb_info->dqcoeff, max_eob, &sse) * - (1 << (2 * txb_info->shift)); - int new_cost = get_txb_cost(txb_info, txb_probs); - int64_t ref_dist_diff = new_dist - org_dist; - int ref_cost_diff = new_cost - org_cost; - if (cost_diff != ref_cost_diff || dist_diff != ref_dist_diff) - printf( - "overall rd_diff %ld\ncost_diff %d ref_cost_diff%d\ndist_diff %ld " - "ref_dist_diff %ld\neob %d new_eob %d\n\n", - rd_diff, cost_diff, ref_cost_diff, dist_diff, ref_dist_diff, org_eob, - txb_info->eob); -#endif if (dry_run) { txb_info->qcoeff = org_qcoeff; txb_info->dqcoeff = org_dqcoeff; @@ -1629,122 +899,6 @@ return update; } -#else -static int optimize_txb(TxbInfo *txb_info, const LV_MAP_COEFF_COST *txb_costs, - TxbCache *txb_cache, int dry_run, int fast_mode, - int *rate_cost) { - int update = 0; - if (txb_info->eob == 0) return update; - int cost_diff = 0; - int64_t dist_diff = 0; - int64_t rd_diff = 0; - const int max_eob = av1_get_max_eob(txb_info->tx_size); - -#if TEST_OPTIMIZE_TXB - int64_t sse; - int64_t org_dist = - av1_block_error_c(txb_info->tcoeff, txb_info->dqcoeff, max_eob, &sse) * - (1 << (2 * txb_info->shift)); - int org_cost = get_txb_cost(txb_info, txb_costs); -#endif - - tran_low_t *org_qcoeff = txb_info->qcoeff; - tran_low_t *org_dqcoeff = txb_info->dqcoeff; - uint8_t *const org_levels = txb_info->levels; - - tran_low_t tmp_qcoeff[MAX_TX_SQUARE]; - tran_low_t tmp_dqcoeff[MAX_TX_SQUARE]; - uint8_t tmp_levels_buf[TX_PAD_2D]; - uint8_t *const tmp_levels = set_levels(tmp_levels_buf, txb_info->width); - const int org_eob = txb_info->eob; - if (dry_run) { - const int stride = txb_info->width + TX_PAD_HOR; - const int levels_size = - - (stride * (txb_info->height + TX_PAD_VER) + TX_PAD_END); - memcpy(tmp_qcoeff, org_qcoeff, sizeof(org_qcoeff[0]) * max_eob); - memcpy(tmp_dqcoeff, org_dqcoeff, sizeof(org_dqcoeff[0]) * max_eob); - memcpy(tmp_levels, org_levels - TX_PAD_TOP * stride, - sizeof(org_levels[0]) * levels_size); - txb_info->qcoeff = tmp_qcoeff; - txb_info->dqcoeff = tmp_dqcoeff; - txb_info->levels = tmp_levels; - } - - const int16_t *const scan = txb_info->scan_order->scan; - - // forward optimize the nz_map - const int cur_eob = txb_info->eob; - for (int si = 0; si < cur_eob; ++si) { - const int coeff_idx = scan[si]; - tran_low_t qc = txb_info->qcoeff[coeff_idx]; - if (abs(qc) == 1) { - LevelDownStats stats; - try_level_down_facade(&stats, si, txb_cache, txb_costs, txb_info, - fast_mode); - if (stats.update) { - update = 1; - cost_diff += stats.cost_diff; - dist_diff += stats.dist_diff; - rd_diff += stats.rd_diff; - update_level_down(coeff_idx, txb_cache, txb_info); - set_eob(txb_info, stats.new_eob); - } - } - } - - // backward optimize the level-k map - int eob_fix = 0; - for (int si = txb_info->eob - 1; si >= 0; --si) { - const int coeff_idx = scan[si]; - if (eob_fix == 1 && txb_info->qcoeff[coeff_idx] == 1) { - // when eob is fixed, there is not need to optimize again when - // abs(qc) == 1 - continue; - } - LevelDownStats stats; - try_level_down_facade(&stats, si, txb_cache, txb_costs, txb_info, - fast_mode); - if (stats.update) { -#if TEST_OPTIMIZE_TXB -// printf("si %d low_qc %d cost_diff %d dist_diff %ld rd_diff %ld eob %d new_eob -// %d\n", si, stats.low_qc, stats.cost_diff, stats.dist_diff, stats.rd_diff, -// txb_info->eob, stats.new_eob); -#endif - update = 1; - cost_diff += stats.cost_diff; - dist_diff += stats.dist_diff; - rd_diff += stats.rd_diff; - update_level_down(coeff_idx, txb_cache, txb_info); - set_eob(txb_info, stats.new_eob); - } - if (eob_fix == 0 && txb_info->qcoeff[coeff_idx] != 0) eob_fix = 1; - if (si > txb_info->eob) si = txb_info->eob; - } -#if TEST_OPTIMIZE_TXB - int64_t new_dist = - av1_block_error_c(txb_info->tcoeff, txb_info->dqcoeff, max_eob, &sse) * - (1 << (2 * txb_info->shift)); - int new_cost = get_txb_cost(txb_info, txb_costs); - int64_t ref_dist_diff = new_dist - org_dist; - int ref_cost_diff = new_cost - org_cost; - if (cost_diff != ref_cost_diff || dist_diff != ref_dist_diff) - printf( - "overall rd_diff %ld\ncost_diff %d ref_cost_diff%d\ndist_diff %ld " - "ref_dist_diff %ld\neob %d new_eob %d\n\n", - rd_diff, cost_diff, ref_cost_diff, dist_diff, ref_dist_diff, org_eob, - txb_info->eob); -#endif - if (dry_run) { - txb_info->qcoeff = org_qcoeff; - txb_info->dqcoeff = org_dqcoeff; - txb_info->levels = org_levels; - set_eob(txb_info, org_eob); - } - return update; -} -#endif - // These numbers are empirically obtained. static const int plane_rd_mult[REF_TYPES][PLANE_TYPES] = { { 17, 13 },