Remove dead code of lv_map
Change-Id: I0e0926119599197af0720470d4bb16156b679954
diff --git a/av1/encoder/encodetxb.c b/av1/encoder/encodetxb.c
index 5042512..f406fb8 100644
--- a/av1/encoder/encodetxb.c
+++ b/av1/encoder/encodetxb.c
@@ -23,8 +23,6 @@
#include "av1/encoder/rdopt.h"
#include "av1/encoder/tokenize.h"
-#define TEST_OPTIMIZE_TXB 0
-
static int hbt_needs_init = 1;
static CRC_CALCULATOR crc_calculator;
static const int HBT_EOB = 16; // also the length in opt_qcoeff
@@ -63,21 +61,6 @@
} LevelDownStats;
void av1_alloc_txb_buf(AV1_COMP *cpi) {
-#if 0
- AV1_COMMON *cm = &cpi->common;
- const int num_planes = av1_num_planes(cm);
- int mi_block_size = 1 << MI_SIZE_LOG2;
- // TODO(angiebird): Make sure cm->subsampling_x/y is set correctly, and then
- // use precise buffer size according to cm->subsampling_x/y
- int pixel_stride = mi_block_size * cm->mi_cols;
- int pixel_height = mi_block_size * cm->mi_rows;
- int i;
- for (i = 0; i < num_planes; ++i) {
- CHECK_MEM_ERROR(
- cm, cpi->tcoeff_buf[i],
- aom_malloc(sizeof(*cpi->tcoeff_buf[i]) * pixel_stride * pixel_height));
- }
-#else
AV1_COMMON *cm = &cpi->common;
int size = ((cm->mi_rows >> cm->seq_params.mib_size_log2) + 1) *
((cm->mi_cols >> cm->seq_params.mib_size_log2) + 1);
@@ -86,21 +69,9 @@
// TODO(jingning): This should be further reduced.
CHECK_MEM_ERROR(cm, cpi->coeff_buffer_base,
aom_malloc(sizeof(*cpi->coeff_buffer_base) * size));
-#endif
}
-void av1_free_txb_buf(AV1_COMP *cpi) {
-#if 0
- int i;
- const AV1_COMMON *const cm = &cpi->common;
- const int num_planes = av1_num_planes(cm);
- for (i = 0; i < num_planes; ++i) {
- aom_free(cpi->tcoeff_buf[i]);
- }
-#else
- aom_free(cpi->coeff_buffer_base);
-#endif
-}
+void av1_free_txb_buf(AV1_COMP *cpi) { aom_free(cpi->coeff_buffer_base); }
void av1_set_coeff_buffer(const AV1_COMP *const cpi, MACROBLOCK *const x,
int mi_row, int mi_col) {
@@ -732,547 +703,11 @@
}
}
-void gen_txb_cache(TxbCache *txb_cache, TxbInfo *txb_info) {
- // gen_nz_count_arr
- const int16_t *const scan = txb_info->scan_order->scan;
- const int bwl = txb_info->bwl;
- const int height = txb_info->height;
- const tran_low_t *const qcoeff = txb_info->qcoeff;
- const uint8_t *const levels = txb_info->levels;
- const BASE_CTX_TABLE *base_ctx_table =
- txb_info->coeff_ctx_table->base_ctx_table;
- for (int c = 0; c < txb_info->eob; ++c) {
- const int coeff_idx = scan[c]; // raster order
- const int row = coeff_idx >> bwl;
- const int col = coeff_idx - (row << bwl);
-
- txb_cache->nz_count_arr[coeff_idx] =
- get_nz_count(levels + get_padded_idx(coeff_idx, bwl), bwl,
- tx_type_to_class[txb_info->tx_type]);
-
- txb_cache->nz_ctx_arr[coeff_idx] =
- get_nz_map_ctx_from_stats(0, coeff_idx, bwl, txb_info->tx_size,
- tx_type_to_class[txb_info->tx_type]);
-
- // gen_base_count_mag_arr
- if (!has_base(qcoeff[coeff_idx], 0)) continue;
- int *base_mag = txb_cache->base_mag_arr[coeff_idx];
- int count[NUM_BASE_LEVELS];
- get_base_count_mag(base_mag, count, qcoeff, bwl, height, row, col);
-
- for (int i = 0; i < NUM_BASE_LEVELS; ++i) {
- if (!has_base(qcoeff[coeff_idx], i)) break;
- txb_cache->base_count_arr[i][coeff_idx] = count[i];
- const int level = i + 1;
- txb_cache->base_ctx_arr[i][coeff_idx] =
- base_ctx_table[row != 0][col != 0][base_mag[0] > level][count[i]];
- }
-
- // gen_br_count_mag_arr
- if (!has_br(qcoeff[coeff_idx])) continue;
- int *br_count = txb_cache->br_count_arr + coeff_idx;
- int *br_mag = txb_cache->br_mag_arr[coeff_idx];
- *br_count = get_br_count_mag(br_mag, qcoeff, bwl, height, row, col,
- NUM_BASE_LEVELS);
- txb_cache->br_ctx_arr[coeff_idx] =
- get_br_ctx_from_count_mag(row, col, *br_count, br_mag[0]);
- }
-}
-
-static INLINE const int *get_level_prob(int level, int coeff_idx,
- const TxbCache *txb_cache,
- const LV_MAP_COEFF_COST *txb_costs) {
- if (level < 1 + NUM_BASE_LEVELS) {
- const int ctx = txb_cache->nz_ctx_arr[coeff_idx];
- return &txb_costs->base_cost[ctx][level];
- } else if (level >= 1 + NUM_BASE_LEVELS &&
- level < 1 + NUM_BASE_LEVELS + COEFF_BASE_RANGE) {
- const int ctx = txb_cache->br_ctx_arr[coeff_idx];
- return txb_costs->lps_cost[ctx];
- } else if (level >= 1 + NUM_BASE_LEVELS + COEFF_BASE_RANGE) {
- // printf("get_level_prob does not support golomb\n");
- assert(0);
- return 0;
- } else {
- assert(0);
- return 0;
- }
-}
-
-static INLINE void update_mag_arr(int *mag_arr, int abs_qc) {
- if (mag_arr[0] == abs_qc) {
- mag_arr[1] -= 1;
- assert(mag_arr[1] >= 0);
- }
-}
-
-static INLINE int get_mag_from_mag_arr(const int *mag_arr) {
- int mag;
- if (mag_arr[1] > 0) {
- mag = mag_arr[0];
- } else if (mag_arr[0] > 0) {
- mag = mag_arr[0] - 1;
- } else {
- // no neighbor
- assert(mag_arr[0] == 0 && mag_arr[1] == 0);
- mag = 0;
- }
- return mag;
-}
-
-static int neighbor_level_down_update(int *new_count, int *new_mag, int count,
- const int *mag, int coeff_idx,
- tran_low_t abs_nb_coeff, int nb_coeff_idx,
- int level, const TxbInfo *txb_info) {
- *new_count = count;
- *new_mag = get_mag_from_mag_arr(mag);
-
- int update = 0;
- // check if br_count changes
- if (abs_nb_coeff == level) {
- update = 1;
- *new_count -= 1;
- assert(*new_count >= 0);
- }
- const int row = coeff_idx >> txb_info->bwl;
- const int col = coeff_idx - (row << txb_info->bwl);
- const int nb_row = nb_coeff_idx >> txb_info->bwl;
- const int nb_col = nb_coeff_idx - (nb_row << txb_info->bwl);
-
- // check if mag changes
- if (nb_row >= row && nb_col >= col) {
- if (abs_nb_coeff == mag[0]) {
- assert(mag[1] > 0);
- if (mag[1] == 1) {
- // the nb is the only qc with max mag
- *new_mag -= 1;
- assert(*new_mag >= 0);
- update = 1;
- }
- }
- }
- return update;
-}
-
-static int try_neighbor_level_down_br(int coeff_idx, int nb_coeff_idx,
- const TxbCache *txb_cache,
- const LV_MAP_COEFF_COST *txb_costs,
- const TxbInfo *txb_info) {
- const tran_low_t qc = txb_info->qcoeff[coeff_idx];
- const tran_low_t abs_qc = abs(qc);
- const int level = NUM_BASE_LEVELS + 1;
- if (abs_qc < level) return 0;
-
- const tran_low_t nb_coeff = txb_info->qcoeff[nb_coeff_idx];
- const tran_low_t abs_nb_coeff = abs(nb_coeff);
- const int count = txb_cache->br_count_arr[coeff_idx];
- const int *mag = txb_cache->br_mag_arr[coeff_idx];
- int new_count;
- int new_mag;
- const int update =
- neighbor_level_down_update(&new_count, &new_mag, count, mag, coeff_idx,
- abs_nb_coeff, nb_coeff_idx, level, txb_info);
- if (update) {
- const int row = coeff_idx >> txb_info->bwl;
- const int col = coeff_idx - (row << txb_info->bwl);
- const int ctx = txb_cache->br_ctx_arr[coeff_idx];
- const int org_cost = get_br_cost(abs_qc, ctx, txb_costs->lps_cost[ctx]);
-
- const int new_ctx = get_br_ctx_from_count_mag(row, col, new_count, new_mag);
- const int new_cost =
- get_br_cost(abs_qc, new_ctx, txb_costs->lps_cost[new_ctx]);
- const int cost_diff = -org_cost + new_cost;
- return cost_diff;
- } else {
- return 0;
- }
-}
-
-static int try_neighbor_level_down_base(int coeff_idx, int nb_coeff_idx,
- const TxbCache *txb_cache,
- const LV_MAP_COEFF_COST *txb_costs,
- const TxbInfo *txb_info) {
- // TODO(olah): not implemented yet
- (void)coeff_idx;
- (void)nb_coeff_idx;
- (void)txb_cache;
- (void)txb_costs;
- (void)txb_info;
- return 0;
-}
-
-static int try_neighbor_level_down_nz(int coeff_idx, int nb_coeff_idx,
- const TxbCache *txb_cache,
- const LV_MAP_COEFF_COST *txb_costs,
- TxbInfo *txb_info) {
- // assume eob doesn't change
- const tran_low_t qc = txb_info->qcoeff[coeff_idx];
- const tran_low_t abs_qc = abs(qc);
- const tran_low_t nb_coeff = txb_info->qcoeff[nb_coeff_idx];
- const tran_low_t abs_nb_coeff = abs(nb_coeff);
- if (abs_nb_coeff != 1) return 0;
- const int16_t *const iscan = txb_info->scan_order->iscan;
- const int scan_idx = iscan[coeff_idx];
- if (scan_idx == txb_info->seg_eob) return 0;
- const int nb_scan_idx = iscan[nb_coeff_idx];
- if (nb_scan_idx < scan_idx) {
- const int count = txb_cache->nz_count_arr[coeff_idx];
- (void)count;
- assert(count > 0);
- update_qcoeff(nb_coeff_idx, get_lower_coeff(nb_coeff), txb_info);
- const int new_ctx = get_nz_map_ctx_from_stats(
- 0, coeff_idx, txb_info->bwl, txb_info->tx_size,
- tx_type_to_class[txb_info->tx_type]);
- update_qcoeff(nb_coeff_idx, nb_coeff, txb_info);
- const int ctx = txb_cache->nz_ctx_arr[coeff_idx];
- const int org_cost = txb_costs->base_cost[ctx][AOMMIN(abs_qc, 3)];
- const int new_cost = txb_costs->base_cost[new_ctx][AOMMIN(abs_qc, 3)];
- const int cost_diff = new_cost - org_cost;
- return cost_diff;
- } else {
- return 0;
- }
-}
-
-static int try_self_level_down(tran_low_t *low_coeff, int coeff_idx,
- const TxbCache *txb_cache,
- const LV_MAP_COEFF_COST *txb_costs,
- TxbInfo *txb_info) {
- const tran_low_t qc = txb_info->qcoeff[coeff_idx];
- if (qc == 0) {
- *low_coeff = 0;
- return 0;
- }
- const tran_low_t abs_qc = abs(qc);
- *low_coeff = get_lower_coeff(qc);
- int cost_diff;
- if (*low_coeff == 0) {
- const int scan_idx = txb_info->scan_order->iscan[coeff_idx];
- const int *level_cost =
- get_level_prob(abs_qc, coeff_idx, txb_cache, txb_costs);
- const int *low_level_cost =
- get_level_prob(abs(*low_coeff), coeff_idx, txb_cache, txb_costs);
-
- if (scan_idx < txb_info->eob - 1) {
- // When level-0, we code the binary of abs_qc > level
- // but when level-k k > 0 we code the binary of abs_qc == level
- // That's why wee need this special treatment for level-0 map
- // TODO(angiebird): make leve-0 consistent to other levels
- cost_diff = -level_cost[1] + low_level_cost[0] - low_level_cost[1];
- } else {
- cost_diff = -level_cost[1];
- }
-
- const int sign_cost = get_sign_bit_cost(
- qc, coeff_idx, txb_costs->dc_sign_cost, txb_info->txb_ctx->dc_sign_ctx);
- cost_diff -= sign_cost;
- } else if (abs_qc <= NUM_BASE_LEVELS) {
- const int *level_cost =
- get_level_prob(abs_qc, coeff_idx, txb_cache, txb_costs);
- const int *low_level_cost =
- get_level_prob(abs(*low_coeff), coeff_idx, txb_cache, txb_costs);
- cost_diff = -level_cost[1] + low_level_cost[1] - low_level_cost[0];
- } else if (abs_qc == NUM_BASE_LEVELS + 1) {
- const int *level_cost =
- get_level_prob(abs_qc, coeff_idx, txb_cache, txb_costs);
- const int *low_level_cost =
- get_level_prob(abs(*low_coeff), coeff_idx, txb_cache, txb_costs);
- cost_diff = -level_cost[0] + low_level_cost[1] - low_level_cost[0];
- } else if (abs_qc < 1 + NUM_BASE_LEVELS + COEFF_BASE_RANGE) {
- const int *level_cost =
- get_level_prob(abs_qc, coeff_idx, txb_cache, txb_costs);
- const int *low_level_cost =
- get_level_prob(abs(*low_coeff), coeff_idx, txb_cache, txb_costs);
-
- cost_diff = -level_cost[abs_qc - 1 - NUM_BASE_LEVELS] +
- low_level_cost[abs(*low_coeff) - 1 - NUM_BASE_LEVELS];
- } else if (abs_qc == 1 + NUM_BASE_LEVELS + COEFF_BASE_RANGE) {
- const int *low_level_cost =
- get_level_prob(abs(*low_coeff), coeff_idx, txb_cache, txb_costs);
- cost_diff = -get_golomb_cost(abs_qc) - low_level_cost[COEFF_BASE_RANGE] +
- low_level_cost[COEFF_BASE_RANGE - 1];
- } else {
- assert(abs_qc > 1 + NUM_BASE_LEVELS + COEFF_BASE_RANGE);
- const tran_low_t abs_low_coeff = abs(*low_coeff);
- cost_diff = -get_golomb_cost(abs_qc) + get_golomb_cost(abs_low_coeff);
- }
- return cost_diff;
-}
-
-#define COST_MAP_SIZE 5
-#define COST_MAP_OFFSET 2
-
-static INLINE int check_nz_neighbor(tran_low_t qc) { return abs(qc) == 1; }
-
-static INLINE int check_base_neighbor(tran_low_t qc) {
- return abs(qc) <= 1 + NUM_BASE_LEVELS;
-}
-
-static INLINE int check_br_neighbor(tran_low_t qc) {
- return abs(qc) > BR_MAG_OFFSET;
-}
-
-#define FAST_OPTIMIZE_TXB 1
-
-#if FAST_OPTIMIZE_TXB
-#define ALNB_REF_OFFSET_NUM 2
-static const int alnb_ref_offset[ALNB_REF_OFFSET_NUM][2] = {
- { -1, 0 },
- { 0, -1 },
-};
-#define NB_REF_OFFSET_NUM 4
-static const int nb_ref_offset[NB_REF_OFFSET_NUM][2] = {
- { -1, 0 },
- { 0, -1 },
- { 1, 0 },
- { 0, 1 },
-};
-#endif // FAST_OPTIMIZE_TXB
-
-// TODO(angiebird): add static to this function once it's called
-int try_level_down(int coeff_idx, const TxbCache *txb_cache,
- const LV_MAP_COEFF_COST *txb_costs, TxbInfo *txb_info,
- int (*cost_map)[COST_MAP_SIZE], int fast_mode) {
-#if !FAST_OPTIMIZE_TXB
- (void)fast_mode;
-#endif
- if (cost_map) {
- for (int i = 0; i < COST_MAP_SIZE; ++i) av1_zero(cost_map[i]);
- }
-
- tran_low_t qc = txb_info->qcoeff[coeff_idx];
- tran_low_t low_coeff;
- if (qc == 0) return 0;
- int accu_cost_diff = 0;
-
- const int16_t *const iscan = txb_info->scan_order->iscan;
- const int eob = txb_info->eob;
- const int scan_idx = iscan[coeff_idx];
- if (scan_idx < eob) {
- const int cost_diff = try_self_level_down(&low_coeff, coeff_idx, txb_cache,
- txb_costs, txb_info);
- if (cost_map)
- cost_map[0 + COST_MAP_OFFSET][0 + COST_MAP_OFFSET] = cost_diff;
- accu_cost_diff += cost_diff;
- }
-
- const int row = coeff_idx >> txb_info->bwl;
- const int col = coeff_idx - (row << txb_info->bwl);
- if (check_nz_neighbor(qc)) {
-#if FAST_OPTIMIZE_TXB
- const int(*ref_offset)[2];
- int ref_num;
- if (fast_mode) {
- ref_offset = alnb_ref_offset;
- ref_num = ALNB_REF_OFFSET_NUM;
- } else {
- ref_offset = sig_ref_offset;
- ref_num = SIG_REF_OFFSET_NUM;
- }
-#else
- const int(*ref_offset)[2] = sig_ref_offset;
- const int ref_num = SIG_REF_OFFSET_NUM;
-#endif
- for (int i = 0; i < ref_num; ++i) {
- const int nb_row = row - ref_offset[i][0];
- const int nb_col = col - ref_offset[i][1];
- const int nb_coeff_idx = nb_row * txb_info->width + nb_col;
-
- if (nb_row < 0 || nb_col < 0 || nb_row >= txb_info->height ||
- nb_col >= txb_info->width)
- continue;
-
- const int nb_scan_idx = iscan[nb_coeff_idx];
- if (nb_scan_idx < eob) {
- const int cost_diff = try_neighbor_level_down_nz(
- nb_coeff_idx, coeff_idx, txb_cache, txb_costs, txb_info);
- if (cost_map)
- cost_map[nb_row - row + COST_MAP_OFFSET]
- [nb_col - col + COST_MAP_OFFSET] += cost_diff;
- accu_cost_diff += cost_diff;
- }
- }
- }
-
- if (check_base_neighbor(qc)) {
-#if FAST_OPTIMIZE_TXB
- const int(*ref_offset)[2];
- int ref_num;
- if (fast_mode) {
- ref_offset = nb_ref_offset;
- ref_num = NB_REF_OFFSET_NUM;
- } else {
- ref_offset = base_ref_offset;
- ref_num = BASE_CONTEXT_POSITION_NUM;
- }
-#else
- const int(*ref_offset)[2] = base_ref_offset;
- int ref_num = BASE_CONTEXT_POSITION_NUM;
-#endif
- for (int i = 0; i < ref_num; ++i) {
- const int nb_row = row - ref_offset[i][0];
- const int nb_col = col - ref_offset[i][1];
- const int nb_coeff_idx = nb_row * txb_info->width + nb_col;
-
- if (nb_row < 0 || nb_col < 0 || nb_row >= txb_info->height ||
- nb_col >= txb_info->width)
- continue;
-
- const int nb_scan_idx = iscan[nb_coeff_idx];
- if (nb_scan_idx < eob) {
- const int cost_diff = try_neighbor_level_down_base(
- nb_coeff_idx, coeff_idx, txb_cache, txb_costs, txb_info);
- if (cost_map)
- cost_map[nb_row - row + COST_MAP_OFFSET]
- [nb_col - col + COST_MAP_OFFSET] += cost_diff;
- accu_cost_diff += cost_diff;
- }
- }
- }
-
- if (check_br_neighbor(qc)) {
-#if FAST_OPTIMIZE_TXB
- const int(*ref_offset)[2];
- int ref_num;
- if (fast_mode) {
- ref_offset = nb_ref_offset;
- ref_num = NB_REF_OFFSET_NUM;
- } else {
- ref_offset = br_ref_offset;
- ref_num = BR_CONTEXT_POSITION_NUM;
- }
-#else
- const int(*ref_offset)[2] = br_ref_offset;
- const int ref_num = BR_CONTEXT_POSITION_NUM;
-#endif
- for (int i = 0; i < ref_num; ++i) {
- const int nb_row = row - ref_offset[i][0];
- const int nb_col = col - ref_offset[i][1];
- const int nb_coeff_idx = nb_row * txb_info->width + nb_col;
-
- if (nb_row < 0 || nb_col < 0 || nb_row >= txb_info->height ||
- nb_col >= txb_info->width)
- continue;
-
- const int nb_scan_idx = iscan[nb_coeff_idx];
- if (nb_scan_idx < eob) {
- const int cost_diff = try_neighbor_level_down_br(
- nb_coeff_idx, coeff_idx, txb_cache, txb_costs, txb_info);
- if (cost_map)
- cost_map[nb_row - row + COST_MAP_OFFSET]
- [nb_col - col + COST_MAP_OFFSET] += cost_diff;
- accu_cost_diff += cost_diff;
- }
- }
- }
-
- return accu_cost_diff;
-}
-
static INLINE void set_eob(TxbInfo *txb_info, int eob) {
txb_info->eob = eob;
txb_info->seg_eob = av1_get_max_eob(txb_info->tx_size);
}
-// TODO(angiebird): add static to this function it's called
-void update_level_down(const int coeff_idx, TxbCache *const txb_cache,
- TxbInfo *const txb_info) {
- const tran_low_t qc = txb_info->qcoeff[coeff_idx];
- const int abs_qc = abs(qc);
- if (qc == 0) return;
- const tran_low_t low_coeff = get_lower_coeff(qc);
- update_coeff(coeff_idx, low_coeff, txb_info);
-
- const int row = coeff_idx >> txb_info->bwl;
- const int col = coeff_idx - (row << txb_info->bwl);
- const int eob = txb_info->eob;
- const int16_t *const iscan = txb_info->scan_order->iscan;
- for (int i = 0; i < SIG_REF_OFFSET_NUM; ++i) {
- const int nb_row = row - sig_ref_offset[i][0];
- const int nb_col = col - sig_ref_offset[i][1];
-
- if (!(nb_row >= 0 && nb_col >= 0 && nb_row < txb_info->height &&
- nb_col < txb_info->width))
- continue;
-
- const int nb_coeff_idx = nb_row * txb_info->width + nb_col;
- const int nb_scan_idx = iscan[nb_coeff_idx];
- if (nb_scan_idx < eob) {
- const int scan_idx = iscan[coeff_idx];
- if (scan_idx < nb_scan_idx) {
- const int level = 1;
- if (abs_qc == level) {
- txb_cache->nz_count_arr[nb_coeff_idx] -= 1;
- assert(txb_cache->nz_count_arr[nb_coeff_idx] >= 0);
- }
- txb_cache->nz_ctx_arr[nb_coeff_idx] = get_nz_map_ctx_from_stats(
- 0, nb_coeff_idx, txb_info->bwl, txb_info->tx_size,
- tx_type_to_class[txb_info->tx_type]);
- }
- }
- }
-
- const BASE_CTX_TABLE *base_ctx_table =
- txb_info->coeff_ctx_table->base_ctx_table;
- for (int i = 0; i < BASE_CONTEXT_POSITION_NUM; ++i) {
- const int nb_row = row - base_ref_offset[i][0];
- const int nb_col = col - base_ref_offset[i][1];
- const int nb_coeff_idx = nb_row * txb_info->width + nb_col;
-
- if (!(nb_row >= 0 && nb_col >= 0 && nb_row < txb_info->height &&
- nb_col < txb_info->width))
- continue;
-
- const tran_low_t nb_coeff = txb_info->qcoeff[nb_coeff_idx];
- if (!has_base(nb_coeff, 0)) continue;
- const int nb_scan_idx = iscan[nb_coeff_idx];
- if (nb_scan_idx < eob) {
- if (row >= nb_row && col >= nb_col)
- update_mag_arr(txb_cache->base_mag_arr[nb_coeff_idx], abs_qc);
- const int mag =
- get_mag_from_mag_arr(txb_cache->base_mag_arr[nb_coeff_idx]);
- for (int base_idx = 0; base_idx < NUM_BASE_LEVELS; ++base_idx) {
- if (!has_base(nb_coeff, base_idx)) continue;
- const int level = base_idx + 1;
- if (abs_qc == level) {
- txb_cache->base_count_arr[base_idx][nb_coeff_idx] -= 1;
- assert(txb_cache->base_count_arr[base_idx][nb_coeff_idx] >= 0);
- }
- const int count = txb_cache->base_count_arr[base_idx][nb_coeff_idx];
- txb_cache->base_ctx_arr[base_idx][nb_coeff_idx] =
- base_ctx_table[nb_row != 0][nb_col != 0][mag > level][count];
- }
- }
- }
-
- for (int i = 0; i < BR_CONTEXT_POSITION_NUM; ++i) {
- const int nb_row = row - br_ref_offset[i][0];
- const int nb_col = col - br_ref_offset[i][1];
- const int nb_coeff_idx = nb_row * txb_info->width + nb_col;
-
- if (!(nb_row >= 0 && nb_col >= 0 && nb_row < txb_info->height &&
- nb_col < txb_info->width))
- continue;
-
- const int nb_scan_idx = iscan[nb_coeff_idx];
- const tran_low_t nb_coeff = txb_info->qcoeff[nb_coeff_idx];
- if (!has_br(nb_coeff)) continue;
- if (nb_scan_idx < eob) {
- const int level = 1 + NUM_BASE_LEVELS;
- if (abs_qc == level) {
- txb_cache->br_count_arr[nb_coeff_idx] -= 1;
- assert(txb_cache->br_count_arr[nb_coeff_idx] >= 0);
- }
- if (row >= nb_row && col >= nb_col)
- update_mag_arr(txb_cache->br_mag_arr[nb_coeff_idx], abs_qc);
- const int count = txb_cache->br_count_arr[nb_coeff_idx];
- const int mag = get_mag_from_mag_arr(txb_cache->br_mag_arr[nb_coeff_idx]);
- txb_cache->br_ctx_arr[nb_coeff_idx] =
- get_br_ctx_from_count_mag(nb_row, nb_col, count, mag);
- }
- }
-}
-
static int get_coeff_cost(const tran_low_t qc, const int scan_idx,
const int is_eob, const TxbInfo *const txb_info,
const LV_MAP_COEFF_COST *const txb_costs,
@@ -1303,146 +738,6 @@
return cost;
}
-#if TEST_OPTIMIZE_TXB
-#define ALL_REF_OFFSET_NUM 17
-static const int all_ref_offset[ALL_REF_OFFSET_NUM][2] = {
- { 0, 0 }, { -2, -1 }, { -2, 0 }, { -2, 1 }, { -1, -2 }, { -1, -1 },
- { -1, 0 }, { -1, 1 }, { 0, -2 }, { 0, -1 }, { 1, -2 }, { 1, -1 },
- { 1, 0 }, { 2, 0 }, { 0, 1 }, { 0, 2 }, { 1, 1 },
-};
-
-static int try_level_down_ref(int coeff_idx, const LV_MAP_COEFF_COST *txb_costs,
- TxbInfo *txb_info,
- int (*cost_map)[COST_MAP_SIZE]) {
- if (cost_map) {
- for (int i = 0; i < COST_MAP_SIZE; ++i) av1_zero(cost_map[i]);
- }
- tran_low_t qc = txb_info->qcoeff[coeff_idx];
- if (qc == 0) return 0;
- int row = coeff_idx >> txb_info->bwl;
- int col = coeff_idx - (row << txb_info->bwl);
- int org_cost = 0;
- for (int i = 0; i < ALL_REF_OFFSET_NUM; ++i) {
- int nb_row = row - all_ref_offset[i][0];
- int nb_col = col - all_ref_offset[i][1];
- int nb_coeff_idx = nb_row * txb_info->width + nb_col;
- int nb_scan_idx = txb_info->scan_order->iscan[nb_coeff_idx];
- if (nb_scan_idx < txb_info->eob && nb_row >= 0 && nb_col >= 0 &&
- nb_row < txb_info->height && nb_col < txb_info->width) {
- const tran_low_t nb_coeff = txb_info->qcoeff[nb_coeff_idx];
- const int coeff_ctx = get_nz_map_ctx(
- txb_info->levels, nb_coeff_idx, txb_info->bwl, txb_info->height,
- nb_scan_idx, is_eob, txb_info->tx_size, txb_info->tx_type);
- const int cost = get_coeff_cost(nb_coeff, nb_scan_idx, is_eob, txb_info,
- txb_costs, coeff_ctx);
- if (cost_map)
- cost_map[nb_row - row + COST_MAP_OFFSET]
- [nb_col - col + COST_MAP_OFFSET] -= cost;
- org_cost += cost;
- }
- }
- update_qcoeff(coeff_idx, get_lower_coeff(qc), txb_info);
- int new_cost = 0;
- for (int i = 0; i < ALL_REF_OFFSET_NUM; ++i) {
- int nb_row = row - all_ref_offset[i][0];
- int nb_col = col - all_ref_offset[i][1];
- int nb_coeff_idx = nb_row * txb_info->width + nb_col;
- int nb_scan_idx = txb_info->scan_order->iscan[nb_coeff_idx];
- if (nb_scan_idx < txb_info->eob && nb_row >= 0 && nb_col >= 0 &&
- nb_row < txb_info->height && nb_col < txb_info->width) {
- const tran_low_t nb_coeff = txb_info->qcoeff[nb_coeff_idx];
- const int coeff_ctx = get_nz_map_ctx(
- txb_info->levels, nb_coeff_idx, txb_info->bwl, txb_info->height,
- nb_scan_idx, is_eob, txb_info->tx_size, txb_info->tx_type);
- const int cost = get_coeff_cost(nb_coeff, nb_scan_idx, is_eob, txb_info,
- txb_costs, coeff_ctx);
- if (cost_map)
- cost_map[nb_row - row + COST_MAP_OFFSET]
- [nb_col - col + COST_MAP_OFFSET] += cost;
- new_cost += cost;
- }
- }
- update_qcoeff(coeff_idx, qc, txb_info);
- return new_cost - org_cost;
-}
-
-static void test_level_down(int coeff_idx, const TxbCache *txb_cache,
- const LV_MAP_COEFF_COST *txb_costs,
- TxbInfo *txb_info) {
- int cost_map[COST_MAP_SIZE][COST_MAP_SIZE];
- int ref_cost_map[COST_MAP_SIZE][COST_MAP_SIZE];
- const int cost_diff =
- try_level_down(coeff_idx, txb_cache, txb_costs, txb_info, cost_map, 0);
- const int cost_diff_ref =
- try_level_down_ref(coeff_idx, txb_costs, txb_info, ref_cost_map);
- if (cost_diff != cost_diff_ref) {
- printf("qc %d cost_diff %d cost_diff_ref %d\n", txb_info->qcoeff[coeff_idx],
- cost_diff, cost_diff_ref);
- for (int r = 0; r < COST_MAP_SIZE; ++r) {
- for (int c = 0; c < COST_MAP_SIZE; ++c) {
- printf("%d:%d ", cost_map[r][c], ref_cost_map[r][c]);
- }
- printf("\n");
- }
- }
-}
-#endif
-
-// TODO(angiebird): make this static once it's called
-int get_txb_cost(TxbInfo *txb_info, const LV_MAP_COEFF_COST *txb_costs) {
- int cost = 0;
- const int txb_skip_ctx = txb_info->txb_ctx->txb_skip_ctx;
- const int16_t *const scan = txb_info->scan_order->scan;
- if (txb_info->eob == 0) {
- cost = txb_costs->txb_skip_cost[txb_skip_ctx][1];
- return cost;
- }
- cost = txb_costs->txb_skip_cost[txb_skip_ctx][0];
- for (int c = 0; c < txb_info->eob; ++c) {
- const int pos = scan[c];
- const tran_low_t qc = txb_info->qcoeff[pos];
- const int coeff_ctx = get_nz_map_ctx(
- txb_info->levels, pos, txb_info->bwl, txb_info->height, c,
- c == txb_info->eob - 1, txb_info->tx_size, txb_info->tx_type);
- const int coeff_cost = get_coeff_cost(qc, c, c == txb_info->eob - 1,
- txb_info, txb_costs, coeff_ctx);
- cost += coeff_cost;
- }
- return cost;
-}
-
-#if TEST_OPTIMIZE_TXB
-void test_try_change_eob(TxbInfo *txb_info, const LV_MAP_COEFF_COST *txb_costs,
- TxbCache *txb_cache) {
- const int eob = txb_info->eob;
- const int16_t *const scan = txb_info->scan_order->scan;
- if (eob > 0) {
- int last_si = eob - 1;
- int last_ci = scan[last_si];
- int last_coeff = txb_info->qcoeff[last_ci];
- if (abs(last_coeff) == 1) {
- int new_eob;
- int cost_diff =
- try_change_eob(&new_eob, last_ci, txb_cache, txb_costs, txb_info, 0);
- int org_eob = txb_info->eob;
- int cost = get_txb_cost(txb_info, txb_costs);
-
- update_qcoeff(last_ci, get_lower_coeff(last_coeff), txb_info);
- set_eob(txb_info, new_eob);
- int new_cost = get_txb_cost(txb_info, txb_costs);
- set_eob(txb_info, org_eob);
- update_qcoeff(last_ci, last_coeff, txb_info);
-
- int ref_cost_diff = -cost + new_cost;
- if (cost_diff != ref_cost_diff)
- printf("org_eob %d new_eob %d cost_diff %d ref_cost_diff %d\n", org_eob,
- new_eob, cost_diff, ref_cost_diff);
- }
- }
-}
-#endif
-
-#if 1
static int optimize_txb(TxbInfo *txb_info, const LV_MAP_COEFF_COST *txb_costs,
const LV_MAP_EOB_COST *txb_eob_costs,
TxbCache *txb_cache, int dry_run, int fast_mode,
@@ -1454,14 +749,6 @@
if (txb_info->eob == 0) return update;
const int max_eob = av1_get_max_eob(txb_info->tx_size);
-#if TEST_OPTIMIZE_TXB
- int64_t sse;
- int64_t org_dist =
- av1_block_error_c(txb_info->tcoeff, txb_info->dqcoeff, max_eob, &sse) *
- (1 << (2 * txb_info->shift));
- int org_cost = get_txb_cost(txb_info, txb_probs);
-#endif
-
tran_low_t *org_qcoeff = txb_info->qcoeff;
tran_low_t *org_dqcoeff = txb_info->dqcoeff;
uint8_t *const org_levels = txb_info->levels;
@@ -1603,23 +890,6 @@
? zero_blk_rate
: accu_rate + non_zero_blk_rate;
-#if TEST_OPTIMIZE_TXB
- int cost_diff = 0;
- int64_t dist_diff = 0;
- int64_t rd_diff = 0;
- int64_t new_dist =
- av1_block_error_c(txb_info->tcoeff, txb_info->dqcoeff, max_eob, &sse) *
- (1 << (2 * txb_info->shift));
- int new_cost = get_txb_cost(txb_info, txb_probs);
- int64_t ref_dist_diff = new_dist - org_dist;
- int ref_cost_diff = new_cost - org_cost;
- if (cost_diff != ref_cost_diff || dist_diff != ref_dist_diff)
- printf(
- "overall rd_diff %ld\ncost_diff %d ref_cost_diff%d\ndist_diff %ld "
- "ref_dist_diff %ld\neob %d new_eob %d\n\n",
- rd_diff, cost_diff, ref_cost_diff, dist_diff, ref_dist_diff, org_eob,
- txb_info->eob);
-#endif
if (dry_run) {
txb_info->qcoeff = org_qcoeff;
txb_info->dqcoeff = org_dqcoeff;
@@ -1629,122 +899,6 @@
return update;
}
-#else
-static int optimize_txb(TxbInfo *txb_info, const LV_MAP_COEFF_COST *txb_costs,
- TxbCache *txb_cache, int dry_run, int fast_mode,
- int *rate_cost) {
- int update = 0;
- if (txb_info->eob == 0) return update;
- int cost_diff = 0;
- int64_t dist_diff = 0;
- int64_t rd_diff = 0;
- const int max_eob = av1_get_max_eob(txb_info->tx_size);
-
-#if TEST_OPTIMIZE_TXB
- int64_t sse;
- int64_t org_dist =
- av1_block_error_c(txb_info->tcoeff, txb_info->dqcoeff, max_eob, &sse) *
- (1 << (2 * txb_info->shift));
- int org_cost = get_txb_cost(txb_info, txb_costs);
-#endif
-
- tran_low_t *org_qcoeff = txb_info->qcoeff;
- tran_low_t *org_dqcoeff = txb_info->dqcoeff;
- uint8_t *const org_levels = txb_info->levels;
-
- tran_low_t tmp_qcoeff[MAX_TX_SQUARE];
- tran_low_t tmp_dqcoeff[MAX_TX_SQUARE];
- uint8_t tmp_levels_buf[TX_PAD_2D];
- uint8_t *const tmp_levels = set_levels(tmp_levels_buf, txb_info->width);
- const int org_eob = txb_info->eob;
- if (dry_run) {
- const int stride = txb_info->width + TX_PAD_HOR;
- const int levels_size =
-
- (stride * (txb_info->height + TX_PAD_VER) + TX_PAD_END);
- memcpy(tmp_qcoeff, org_qcoeff, sizeof(org_qcoeff[0]) * max_eob);
- memcpy(tmp_dqcoeff, org_dqcoeff, sizeof(org_dqcoeff[0]) * max_eob);
- memcpy(tmp_levels, org_levels - TX_PAD_TOP * stride,
- sizeof(org_levels[0]) * levels_size);
- txb_info->qcoeff = tmp_qcoeff;
- txb_info->dqcoeff = tmp_dqcoeff;
- txb_info->levels = tmp_levels;
- }
-
- const int16_t *const scan = txb_info->scan_order->scan;
-
- // forward optimize the nz_map
- const int cur_eob = txb_info->eob;
- for (int si = 0; si < cur_eob; ++si) {
- const int coeff_idx = scan[si];
- tran_low_t qc = txb_info->qcoeff[coeff_idx];
- if (abs(qc) == 1) {
- LevelDownStats stats;
- try_level_down_facade(&stats, si, txb_cache, txb_costs, txb_info,
- fast_mode);
- if (stats.update) {
- update = 1;
- cost_diff += stats.cost_diff;
- dist_diff += stats.dist_diff;
- rd_diff += stats.rd_diff;
- update_level_down(coeff_idx, txb_cache, txb_info);
- set_eob(txb_info, stats.new_eob);
- }
- }
- }
-
- // backward optimize the level-k map
- int eob_fix = 0;
- for (int si = txb_info->eob - 1; si >= 0; --si) {
- const int coeff_idx = scan[si];
- if (eob_fix == 1 && txb_info->qcoeff[coeff_idx] == 1) {
- // when eob is fixed, there is not need to optimize again when
- // abs(qc) == 1
- continue;
- }
- LevelDownStats stats;
- try_level_down_facade(&stats, si, txb_cache, txb_costs, txb_info,
- fast_mode);
- if (stats.update) {
-#if TEST_OPTIMIZE_TXB
-// printf("si %d low_qc %d cost_diff %d dist_diff %ld rd_diff %ld eob %d new_eob
-// %d\n", si, stats.low_qc, stats.cost_diff, stats.dist_diff, stats.rd_diff,
-// txb_info->eob, stats.new_eob);
-#endif
- update = 1;
- cost_diff += stats.cost_diff;
- dist_diff += stats.dist_diff;
- rd_diff += stats.rd_diff;
- update_level_down(coeff_idx, txb_cache, txb_info);
- set_eob(txb_info, stats.new_eob);
- }
- if (eob_fix == 0 && txb_info->qcoeff[coeff_idx] != 0) eob_fix = 1;
- if (si > txb_info->eob) si = txb_info->eob;
- }
-#if TEST_OPTIMIZE_TXB
- int64_t new_dist =
- av1_block_error_c(txb_info->tcoeff, txb_info->dqcoeff, max_eob, &sse) *
- (1 << (2 * txb_info->shift));
- int new_cost = get_txb_cost(txb_info, txb_costs);
- int64_t ref_dist_diff = new_dist - org_dist;
- int ref_cost_diff = new_cost - org_cost;
- if (cost_diff != ref_cost_diff || dist_diff != ref_dist_diff)
- printf(
- "overall rd_diff %ld\ncost_diff %d ref_cost_diff%d\ndist_diff %ld "
- "ref_dist_diff %ld\neob %d new_eob %d\n\n",
- rd_diff, cost_diff, ref_cost_diff, dist_diff, ref_dist_diff, org_eob,
- txb_info->eob);
-#endif
- if (dry_run) {
- txb_info->qcoeff = org_qcoeff;
- txb_info->dqcoeff = org_dqcoeff;
- txb_info->levels = org_levels;
- set_eob(txb_info, org_eob);
- }
- return update;
-}
-#endif
-
// These numbers are empirically obtained.
static const int plane_rd_mult[REF_TYPES][PLANE_TYPES] = {
{ 17, 13 },
