Remove hash-based trellis optimization Using a hash table for trellis optimization does not seem to provide any benefit and this speed feature is never enabled. This CL removes the related codes as part of the codebase cleanup. BUG=aomedia:2694 Change-Id: Iae5b6c3e078e309dd181e81cef64412335ba9581
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c index 7cbecb1..3e2de04 100644 --- a/av1/encoder/encoder.c +++ b/av1/encoder/encoder.c
@@ -1775,9 +1775,6 @@ #endif // CONFIG_INTERNAL_STATS av1_remove_common(cm); -#if CONFIG_HTB_TRELLIS - if (cpi->sf.use_hash_based_trellis) hbt_destroy(); -#endif // CONFIG_HTB_TRELLIS av1_free_ref_frame_buffers(cm->buffer_pool); aom_free(cpi);
diff --git a/av1/encoder/encodetxb.c b/av1/encoder/encodetxb.c index c74e37d..fd327ad 100644 --- a/av1/encoder/encodetxb.c +++ b/av1/encoder/encodetxb.c
@@ -23,27 +23,6 @@ #include "av1/encoder/rdopt.h" #include "av1/encoder/tokenize.h" -#if CONFIG_HTB_TRELLIS -static int hbt_needs_init = 1; -static CRC32C crc_calculator; -static const int HBT_EOB = 16; // also the length in opt_qcoeff -static const int HBT_TABLE_SIZE = 65536; // 16 bit: holds 65536 'arrays' -static const int HBT_ARRAY_LENGTH = 256; // 8 bit: 256 entries -// If removed in hbt_create_hashes or increased beyond int8_t, widen deltas type -static const int HBT_KICKOUT = 3; - -typedef struct OptTxbQcoeff { - // Use larger type if larger/no kickout value is used in hbt_create_hashes - int8_t deltas[16]; - uint32_t hbt_qc_hash; - uint32_t hbt_ctx_hash; - int init; - int rate_cost; -} OptTxbQcoeff; - -OptTxbQcoeff *hbt_hash_table; -#endif // CONFIG_HTB_TRELLIS - typedef struct LevelDownStats { int update; tran_low_t low_qc; @@ -100,23 +79,6 @@ for (i = length - 1; i >= 0; --i) aom_write_bit(w, (x >> i) & 0x01); } -static INLINE tran_low_t get_lower_coeff(tran_low_t qc) { - if (qc == 0) { - return 0; - } - return qc > 0 ? qc - 1 : qc + 1; -} - -static INLINE tran_low_t qcoeff_to_dqcoeff(tran_low_t qc, int coeff_idx, - int dqv, int shift, - const qm_val_t *iqmatrix) { - int sign = qc < 0 ? -1 : 1; - if (iqmatrix != NULL) - dqv = - ((iqmatrix[coeff_idx] * dqv) + (1 << (AOM_QM_BITS - 1))) >> AOM_QM_BITS; - return sign * ((abs(qc) * dqv) >> shift); -} - static INLINE int64_t get_coeff_dist(tran_low_t tcoeff, tran_low_t dqcoeff, int shift) { const int64_t diff = (tcoeff - dqcoeff) * (1 << shift); @@ -265,16 +227,6 @@ return eob_cost; } -static INLINE int get_sign_bit_cost(tran_low_t qc, int coeff_idx, - const int (*dc_sign_cost)[2], - int dc_sign_ctx) { - if (coeff_idx == 0) { - const int sign = (qc < 0) ? 1 : 0; - return dc_sign_cost[dc_sign_ctx][sign]; - } - return av1_cost_literal(1); -} - static const int golomb_bits_cost[32] = { 0, 512, 512 * 3, 512 * 3, 512 * 5, 512 * 5, 512 * 5, 512 * 5, 512 * 7, 512 * 7, 512 * 7, 512 * 7, 512 * 7, 512 * 7, 512 * 7, 512 * 7, @@ -321,35 +273,6 @@ return coeff_lps[base_range] + get_golomb_cost(level); } -static int get_coeff_cost(const tran_low_t qc, const int scan_idx, - const int is_eob, const TxbInfo *const txb_info, - const LV_MAP_COEFF_COST *const txb_costs, - const int coeff_ctx, const TX_CLASS tx_class) { - const TXB_CTX *const txb_ctx = txb_info->txb_ctx; - const int is_nz = (qc != 0); - const tran_low_t abs_qc = abs(qc); - int cost = 0; - const int16_t *const scan = txb_info->scan_order->scan; - const int pos = scan[scan_idx]; - - if (is_eob) { - cost += txb_costs->base_eob_cost[coeff_ctx][AOMMIN(abs_qc, 3) - 1]; - } else { - cost += txb_costs->base_cost[coeff_ctx][AOMMIN(abs_qc, 3)]; - } - if (is_nz) { - cost += get_sign_bit_cost(qc, scan_idx, txb_costs->dc_sign_cost, - txb_ctx->dc_sign_ctx); - - if (abs_qc > NUM_BASE_LEVELS) { - const int ctx = - get_br_ctx(txb_info->levels, pos, txb_info->bwl, tx_class); - cost += get_br_cost(abs_qc, txb_costs->lps_cost[ctx]); - } - } - return cost; -} - static INLINE int get_nz_map_ctx(const uint8_t *const levels, const int coeff_idx, const int bwl, const int height, const int scan_idx, @@ -366,111 +289,6 @@ return get_nz_map_ctx_from_stats(stats, coeff_idx, bwl, tx_size, tx_class); } -static void get_dist_cost_stats(LevelDownStats *const stats, const int scan_idx, - const int is_eob, - const LV_MAP_COEFF_COST *const txb_costs, - const TxbInfo *const txb_info, - const TX_CLASS tx_class) { - const int16_t *const scan = txb_info->scan_order->scan; - const int coeff_idx = scan[scan_idx]; - const tran_low_t qc = txb_info->qcoeff[coeff_idx]; - const uint8_t *const levels = txb_info->levels; - stats->new_eob = -1; - stats->update = 0; - stats->rd_low = 0; - stats->rd = 0; - stats->nz_rd = 0; - stats->dist_low = 0; - stats->rate_low = 0; - stats->low_qc = 0; - - const tran_low_t tqc = txb_info->tcoeff[coeff_idx]; - const int dqv = txb_info->dequant[coeff_idx != 0]; - const int coeff_ctx = - get_nz_map_ctx(levels, coeff_idx, txb_info->bwl, txb_info->height, - scan_idx, is_eob, txb_info->tx_size, tx_class); - const int qc_cost = get_coeff_cost(qc, scan_idx, is_eob, txb_info, txb_costs, - coeff_ctx, tx_class); - assert(qc != 0); - const tran_low_t dqc = qcoeff_to_dqcoeff(qc, coeff_idx, dqv, txb_info->shift, - txb_info->iqmatrix); - const int64_t dqc_dist = get_coeff_dist(tqc, dqc, txb_info->shift); - - // distortion difference when coefficient is quantized to 0 - const tran_low_t dqc0 = - qcoeff_to_dqcoeff(0, coeff_idx, dqv, txb_info->shift, txb_info->iqmatrix); - - stats->dist0 = get_coeff_dist(tqc, dqc0, txb_info->shift); - stats->dist = dqc_dist - stats->dist0; - stats->rate = qc_cost; - - stats->rd = RDCOST(txb_info->rdmult, stats->rate, stats->dist); - - stats->low_qc = get_lower_coeff(qc); - - if (is_eob && stats->low_qc == 0) { - stats->rd_low = stats->rd; // disable selection of low_qc in this case. - } else { - if (stats->low_qc == 0) { - stats->dist_low = 0; - } else { - stats->low_dqc = qcoeff_to_dqcoeff(stats->low_qc, coeff_idx, dqv, - txb_info->shift, txb_info->iqmatrix); - const int64_t low_dqc_dist = - get_coeff_dist(tqc, stats->low_dqc, txb_info->shift); - stats->dist_low = low_dqc_dist - stats->dist0; - } - const int low_qc_cost = - get_coeff_cost(stats->low_qc, scan_idx, is_eob, txb_info, txb_costs, - coeff_ctx, tx_class); - stats->rate_low = low_qc_cost; - stats->rd_low = RDCOST(txb_info->rdmult, stats->rate_low, stats->dist_low); - } -} - -static void get_dist_cost_stats_with_eob( - LevelDownStats *const stats, const int scan_idx, - const LV_MAP_COEFF_COST *const txb_costs, const TxbInfo *const txb_info, - const TX_CLASS tx_class) { - const int is_eob = 0; - get_dist_cost_stats(stats, scan_idx, is_eob, txb_costs, txb_info, tx_class); - - const int16_t *const scan = txb_info->scan_order->scan; - const int coeff_idx = scan[scan_idx]; - const tran_low_t qc = txb_info->qcoeff[coeff_idx]; - const int coeff_ctx_temp = get_nz_map_ctx( - txb_info->levels, coeff_idx, txb_info->bwl, txb_info->height, scan_idx, 1, - txb_info->tx_size, tx_class); - const int qc_eob_cost = get_coeff_cost(qc, scan_idx, 1, txb_info, txb_costs, - coeff_ctx_temp, tx_class); - int64_t rd_eob = RDCOST(txb_info->rdmult, qc_eob_cost, stats->dist); - if (stats->low_qc != 0) { - const int low_qc_eob_cost = - get_coeff_cost(stats->low_qc, scan_idx, 1, txb_info, txb_costs, - coeff_ctx_temp, tx_class); - int64_t rd_eob_low = - RDCOST(txb_info->rdmult, low_qc_eob_cost, stats->dist_low); - rd_eob = (rd_eob > rd_eob_low) ? rd_eob_low : rd_eob; - } - - stats->nz_rd = AOMMIN(stats->rd_low, stats->rd) - rd_eob; -} - -static INLINE void update_qcoeff(const int coeff_idx, const tran_low_t qc, - const TxbInfo *const txb_info) { - txb_info->qcoeff[coeff_idx] = qc; - txb_info->levels[get_padded_idx(coeff_idx, txb_info->bwl)] = - (uint8_t)clamp(abs(qc), 0, INT8_MAX); -} - -static INLINE void update_coeff(const int coeff_idx, const tran_low_t qc, - const TxbInfo *const txb_info) { - update_qcoeff(coeff_idx, qc, txb_info); - const int dqv = txb_info->dequant[coeff_idx != 0]; - txb_info->dqcoeff[coeff_idx] = qcoeff_to_dqcoeff( - qc, coeff_idx, dqv, txb_info->shift, txb_info->iqmatrix); -} - void av1_txb_init_levels_c(const tran_low_t *const coeff, const int width, const int height, uint8_t *const levels) { const int stride = width + TX_PAD_HOR; @@ -972,443 +790,6 @@ tx_type, tx_class, reduced_tx_set_used); } -static int optimize_txb(TxbInfo *txb_info, const LV_MAP_COEFF_COST *txb_costs, - const LV_MAP_EOB_COST *txb_eob_costs, int *rate_cost) { - int update = 0; - if (txb_info->eob == 0) return update; - const int16_t *const scan = txb_info->scan_order->scan; - // forward optimize the nz_map` - const int init_eob = txb_info->eob; - const TX_CLASS tx_class = tx_type_to_class[txb_info->tx_type]; - const int eob_cost = - get_eob_cost(init_eob, txb_eob_costs, txb_costs, tx_class); - - // backward optimize the level-k map - int accu_rate = eob_cost; - int64_t accu_dist = 0; - int64_t prev_eob_rd_cost = INT64_MAX; - int64_t cur_eob_rd_cost = 0; - - { - const int si = init_eob - 1; - const int coeff_idx = scan[si]; - LevelDownStats stats; - get_dist_cost_stats(&stats, si, si == init_eob - 1, txb_costs, txb_info, - tx_class); - if ((stats.rd_low < stats.rd) && (stats.low_qc != 0)) { - update = 1; - update_coeff(coeff_idx, stats.low_qc, txb_info); - accu_rate += stats.rate_low; - accu_dist += stats.dist_low; - } else { - accu_rate += stats.rate; - accu_dist += stats.dist; - } - } - - int si = init_eob - 2; - int8_t has_nz_tail = 0; - // eob is not fixed - for (; si >= 0 && has_nz_tail < 2; --si) { - assert(si != init_eob - 1); - const int coeff_idx = scan[si]; - tran_low_t qc = txb_info->qcoeff[coeff_idx]; - - if (qc == 0) { - const int coeff_ctx = - get_lower_levels_ctx(txb_info->levels, coeff_idx, txb_info->bwl, - txb_info->tx_size, tx_class); - accu_rate += txb_costs->base_cost[coeff_ctx][0]; - } else { - LevelDownStats stats; - get_dist_cost_stats_with_eob(&stats, si, txb_costs, txb_info, tx_class); - // check if it is better to make this the last significant coefficient - int cur_eob_rate = - get_eob_cost(si + 1, txb_eob_costs, txb_costs, tx_class); - cur_eob_rd_cost = RDCOST(txb_info->rdmult, cur_eob_rate, 0); - prev_eob_rd_cost = - RDCOST(txb_info->rdmult, accu_rate, accu_dist) + stats.nz_rd; - if (cur_eob_rd_cost <= prev_eob_rd_cost) { - update = 1; - for (int j = si + 1; j < txb_info->eob; j++) { - const int coeff_pos_j = scan[j]; - update_coeff(coeff_pos_j, 0, txb_info); - } - txb_info->eob = si + 1; - - // rerun cost calculation due to change of eob - accu_rate = cur_eob_rate; - accu_dist = 0; - get_dist_cost_stats(&stats, si, 1, txb_costs, txb_info, tx_class); - if ((stats.rd_low < stats.rd) && (stats.low_qc != 0)) { - update = 1; - update_coeff(coeff_idx, stats.low_qc, txb_info); - accu_rate += stats.rate_low; - accu_dist += stats.dist_low; - } else { - accu_rate += stats.rate; - accu_dist += stats.dist; - } - - // reset non zero tail when new eob is found - has_nz_tail = 0; - } else { - int bUpdCoeff = 0; - if (stats.rd_low < stats.rd) { - if ((si < txb_info->eob - 1)) { - bUpdCoeff = 1; - update = 1; - } - } else { - ++has_nz_tail; - } - - if (bUpdCoeff) { - update_coeff(coeff_idx, stats.low_qc, txb_info); - accu_rate += stats.rate_low; - accu_dist += stats.dist_low; - } else { - accu_rate += stats.rate; - accu_dist += stats.dist; - } - } - } - } // for (si) - - // eob is fixed - for (; si >= 0; --si) { - assert(si != init_eob - 1); - const int coeff_idx = scan[si]; - tran_low_t qc = txb_info->qcoeff[coeff_idx]; - - if (qc == 0) { - const int coeff_ctx = - get_lower_levels_ctx(txb_info->levels, coeff_idx, txb_info->bwl, - txb_info->tx_size, tx_class); - accu_rate += txb_costs->base_cost[coeff_ctx][0]; - } else { - LevelDownStats stats; - get_dist_cost_stats(&stats, si, 0, txb_costs, txb_info, tx_class); - - int bUpdCoeff = 0; - if (stats.rd_low < stats.rd) { - if ((si < txb_info->eob - 1)) { - bUpdCoeff = 1; - update = 1; - } - } - if (bUpdCoeff) { - update_coeff(coeff_idx, stats.low_qc, txb_info); - accu_rate += stats.rate_low; - accu_dist += stats.dist_low; - } else { - accu_rate += stats.rate; - accu_dist += stats.dist; - } - } - } // for (si) - - int non_zero_blk_rate = - txb_costs->txb_skip_cost[txb_info->txb_ctx->txb_skip_ctx][0]; - prev_eob_rd_cost = - RDCOST(txb_info->rdmult, accu_rate + non_zero_blk_rate, accu_dist); - - int zero_blk_rate = - txb_costs->txb_skip_cost[txb_info->txb_ctx->txb_skip_ctx][1]; - int64_t zero_blk_rd_cost = RDCOST(txb_info->rdmult, zero_blk_rate, 0); - if (zero_blk_rd_cost <= prev_eob_rd_cost) { - update = 1; - for (int j = 0; j < txb_info->eob; j++) { - const int coeff_pos_j = scan[j]; - update_coeff(coeff_pos_j, 0, txb_info); - } - txb_info->eob = 0; - } - - // record total rate cost - *rate_cost = zero_blk_rd_cost <= prev_eob_rd_cost - ? zero_blk_rate - : accu_rate + non_zero_blk_rate; - - if (txb_info->eob > 0) { - *rate_cost += txb_info->tx_type_cost; - } - - return update; -} - -#if CONFIG_HTB_TRELLIS -static void hbt_init() { - hbt_hash_table = - aom_malloc(sizeof(OptTxbQcoeff) * HBT_TABLE_SIZE * HBT_ARRAY_LENGTH); - memset(hbt_hash_table, 0, - sizeof(OptTxbQcoeff) * HBT_TABLE_SIZE * HBT_ARRAY_LENGTH); - av1_crc32c_calculator_init(&crc_calculator); // 31 bit: qc & ctx - - hbt_needs_init = 0; -} - -void hbt_destroy() { aom_free(hbt_hash_table); } - -static int hbt_hash_miss(uint32_t hbt_ctx_hash, uint32_t hbt_qc_hash, - TxbInfo *txb_info, const LV_MAP_COEFF_COST *txb_costs, - const LV_MAP_EOB_COST *txb_eob_costs, - const struct macroblock_plane *p, int block, - int fast_mode, int *rate_cost) { - (void)fast_mode; - const int16_t *scan = txb_info->scan_order->scan; - int prev_eob = txb_info->eob; - assert(HBT_EOB <= 16); // Lengthen array if allowing longer eob. - int32_t prev_coeff[16]; - for (int i = 0; i < prev_eob; i++) { - prev_coeff[i] = txb_info->qcoeff[scan[i]]; - } - for (int i = prev_eob; i < HBT_EOB; i++) { - prev_coeff[i] = 0; // For compiler piece of mind. - } - - av1_txb_init_levels(txb_info->qcoeff, txb_info->width, txb_info->height, - txb_info->levels); - - const int update = - optimize_txb(txb_info, txb_costs, txb_eob_costs, rate_cost); - - // Overwrite old entry - uint16_t hbt_table_index = hbt_ctx_hash % HBT_TABLE_SIZE; - uint16_t hbt_array_index = hbt_qc_hash % HBT_ARRAY_LENGTH; - hbt_hash_table[hbt_table_index * HBT_ARRAY_LENGTH + hbt_array_index] - .rate_cost = *rate_cost; - hbt_hash_table[hbt_table_index * HBT_ARRAY_LENGTH + hbt_array_index].init = 1; - hbt_hash_table[hbt_table_index * HBT_ARRAY_LENGTH + hbt_array_index] - .hbt_qc_hash = hbt_qc_hash; - hbt_hash_table[hbt_table_index * HBT_ARRAY_LENGTH + hbt_array_index] - .hbt_ctx_hash = hbt_ctx_hash; - assert(prev_eob >= txb_info->eob); // eob can't get longer - for (int i = 0; i < txb_info->eob; i++) { - // Record how coeff changed. Convention: towards zero is negative. - if (txb_info->qcoeff[scan[i]] > 0) - hbt_hash_table[hbt_table_index * HBT_ARRAY_LENGTH + hbt_array_index] - .deltas[i] = txb_info->qcoeff[scan[i]] - prev_coeff[i]; - else - hbt_hash_table[hbt_table_index * HBT_ARRAY_LENGTH + hbt_array_index] - .deltas[i] = prev_coeff[i] - txb_info->qcoeff[scan[i]]; - } - for (int i = txb_info->eob; i < prev_eob; i++) { - // If eob got shorter, record that all after it changed to zero. - if (prev_coeff[i] > 0) - hbt_hash_table[hbt_table_index * HBT_ARRAY_LENGTH + hbt_array_index] - .deltas[i] = -prev_coeff[i]; - else - hbt_hash_table[hbt_table_index * HBT_ARRAY_LENGTH + hbt_array_index] - .deltas[i] = prev_coeff[i]; - } - for (int i = prev_eob; i < HBT_EOB; i++) { - // Record 'no change' after optimized coefficients run out. - hbt_hash_table[hbt_table_index * HBT_ARRAY_LENGTH + hbt_array_index] - .deltas[i] = 0; - } - - if (update) { - p->eobs[block] = txb_info->eob; - p->txb_entropy_ctx[block] = av1_get_txb_entropy_context( - txb_info->qcoeff, txb_info->scan_order, txb_info->eob); - } - return txb_info->eob; -} - -static int hbt_hash_hit(uint32_t hbt_table_index, int hbt_array_index, - TxbInfo *txb_info, const struct macroblock_plane *p, - int block, int *rate_cost) { - const int16_t *scan = txb_info->scan_order->scan; - int new_eob = 0; - int update = 0; - - for (int i = 0; i < txb_info->eob; i++) { - // Delta convention is negatives go towards zero, so only apply those ones. - if (hbt_hash_table[hbt_table_index * HBT_ARRAY_LENGTH + hbt_array_index] - .deltas[i] < 0) { - if (txb_info->qcoeff[scan[i]] > 0) - txb_info->qcoeff[scan[i]] += - hbt_hash_table[hbt_table_index * HBT_ARRAY_LENGTH + hbt_array_index] - .deltas[i]; - else - txb_info->qcoeff[scan[i]] -= - hbt_hash_table[hbt_table_index * HBT_ARRAY_LENGTH + hbt_array_index] - .deltas[i]; - - update = 1; - update_coeff(scan[i], txb_info->qcoeff[scan[i]], txb_info); - } - if (txb_info->qcoeff[scan[i]]) new_eob = i + 1; - } - - // Rate_cost can be calculated here instead (av1_cost_coeffs_txb), but - // it is expensive and gives little benefit as long as qc_hash is high bit - *rate_cost = - hbt_hash_table[hbt_table_index * HBT_ARRAY_LENGTH + hbt_array_index] - .rate_cost; - - if (update) { - txb_info->eob = new_eob; - p->eobs[block] = txb_info->eob; - p->txb_entropy_ctx[block] = av1_get_txb_entropy_context( - txb_info->qcoeff, txb_info->scan_order, txb_info->eob); - } - - return txb_info->eob; -} - -static int hbt_search_match(uint32_t hbt_ctx_hash, uint32_t hbt_qc_hash, - TxbInfo *txb_info, - const LV_MAP_COEFF_COST *txb_costs, - const LV_MAP_EOB_COST *txb_eob_costs, - const struct macroblock_plane *p, int block, - int fast_mode, int *rate_cost) { - // Check for qcoeff match - int hbt_array_index = hbt_qc_hash % HBT_ARRAY_LENGTH; - int hbt_table_index = hbt_ctx_hash % HBT_TABLE_SIZE; - - if (hbt_hash_table[hbt_table_index * HBT_ARRAY_LENGTH + hbt_array_index] - .hbt_qc_hash == hbt_qc_hash && - hbt_hash_table[hbt_table_index * HBT_ARRAY_LENGTH + hbt_array_index] - .hbt_ctx_hash == hbt_ctx_hash && - hbt_hash_table[hbt_table_index * HBT_ARRAY_LENGTH + hbt_array_index] - .init) { - return hbt_hash_hit(hbt_table_index, hbt_array_index, txb_info, p, block, - rate_cost); - } else { - return hbt_hash_miss(hbt_ctx_hash, hbt_qc_hash, txb_info, txb_costs, - txb_eob_costs, p, block, fast_mode, rate_cost); - } -} - -static int hbt_create_hashes(TxbInfo *txb_info, - const LV_MAP_COEFF_COST *txb_costs, - const LV_MAP_EOB_COST *txb_eob_costs, - const struct macroblock_plane *p, int block, - int fast_mode, int *rate_cost) { - // Initialize hash table if needed. - if (hbt_needs_init) { - hbt_init(); - } - - //// Hash creation - uint8_t txb_hash_data[256]; // Asserts below to ensure enough space. - const int16_t *scan = txb_info->scan_order->scan; - uint8_t chunk = 0; - int hash_data_index = 0; - - // Make qc_hash. - int packing_index = 0; // needed for packing. - for (int i = 0; i < txb_info->eob; i++) { - tran_low_t prechunk = txb_info->qcoeff[scan[i]]; - - // Softening: Improves speed. Aligns with signed deltas. - if (prechunk < 0) prechunk *= -1; - - // Early kick out: Don't apply feature if there are large coeffs: - // If this kickout value is removed or raised beyond int8_t, - // widen deltas type in OptTxbQcoeff struct. - assert((int8_t)HBT_KICKOUT == HBT_KICKOUT); // If not, widen types. - if (prechunk > HBT_KICKOUT) { - av1_txb_init_levels(txb_info->qcoeff, txb_info->width, txb_info->height, - txb_info->levels); - - const int update = - optimize_txb(txb_info, txb_costs, txb_eob_costs, rate_cost); - - if (update) { - p->eobs[block] = txb_info->eob; - p->txb_entropy_ctx[block] = av1_get_txb_entropy_context( - txb_info->qcoeff, txb_info->scan_order, txb_info->eob); - } - return txb_info->eob; - } - - // Since coeffs are 0 to 3, only 2 bits are needed: pack into bytes - if (packing_index == 0) txb_hash_data[hash_data_index] = 0; - chunk = prechunk << packing_index; - packing_index += 2; - txb_hash_data[hash_data_index] |= chunk; - - // Full byte: - if (packing_index == 8) { - packing_index = 0; - hash_data_index++; - } - } - // Needed when packing_index != 0, to include final byte. - hash_data_index++; - assert(hash_data_index <= 64); - // 31 bit qc_hash: index to array - uint32_t hbt_qc_hash = - av1_get_crc32c_value(&crc_calculator, txb_hash_data, hash_data_index); - - // Make ctx_hash. - hash_data_index = 0; - tran_low_t prechunk; - - for (int i = 0; i < txb_info->eob; i++) { - // Save as magnitudes towards or away from zero. - if (txb_info->tcoeff[scan[i]] >= 0) - prechunk = txb_info->tcoeff[scan[i]] - txb_info->dqcoeff[scan[i]]; - else - prechunk = txb_info->dqcoeff[scan[i]] - txb_info->tcoeff[scan[i]]; - - chunk = prechunk & 0xff; - txb_hash_data[hash_data_index++] = chunk; - } - - // Extra ctx data: - // Include dequants. - txb_hash_data[hash_data_index++] = txb_info->dequant[0] & 0xff; - txb_hash_data[hash_data_index++] = txb_info->dequant[1] & 0xff; - chunk = txb_info->txb_ctx->txb_skip_ctx & 0xff; - txb_hash_data[hash_data_index++] = chunk; - chunk = txb_info->txb_ctx->dc_sign_ctx & 0xff; - txb_hash_data[hash_data_index++] = chunk; - // eob - chunk = txb_info->eob & 0xff; - txb_hash_data[hash_data_index++] = chunk; - // rdmult (int64) - chunk = txb_info->rdmult & 0xff; - txb_hash_data[hash_data_index++] = chunk; - // tx_type - chunk = txb_info->tx_type & 0xff; - txb_hash_data[hash_data_index++] = chunk; - // base_eob_cost - for (int i = 1; i < 3; i++) { // i = 0 are softened away - for (int j = 0; j < SIG_COEF_CONTEXTS_EOB; j++) { - chunk = (txb_costs->base_eob_cost[j][i] & 0xff00) >> 8; - txb_hash_data[hash_data_index++] = chunk; - } - } - // eob_cost - for (int i = 0; i < 11; i++) { - for (int j = 0; j < 2; j++) { - chunk = (txb_eob_costs->eob_cost[j][i] & 0xff00) >> 8; - txb_hash_data[hash_data_index++] = chunk; - } - } - // dc_sign_cost - for (int i = 0; i < 2; i++) { - for (int j = 0; j < DC_SIGN_CONTEXTS; j++) { - chunk = (txb_costs->dc_sign_cost[j][i] & 0xff00) >> 8; - txb_hash_data[hash_data_index++] = chunk; - } - } - - assert(hash_data_index <= 256); - // 31 bit ctx_hash: used to index table - uint32_t hbt_ctx_hash = - av1_get_crc32c_value(&crc_calculator, txb_hash_data, hash_data_index); - //// End hash creation - - return hbt_search_match(hbt_ctx_hash, hbt_qc_hash, txb_info, txb_costs, - txb_eob_costs, p, block, fast_mode, rate_cost); -} -#endif // CONFIG_HTB_TRELLIS - static AOM_FORCE_INLINE int get_two_coeff_cost_simple( int ci, tran_low_t abs_qc, int coeff_ctx, const LV_MAP_COEFF_COST *txb_costs, int bwl, TX_CLASS tx_class, @@ -1904,82 +1285,6 @@ return eob; } -// This function is deprecated, but we keep it here because hash trellis -// is not integrated with av1_optimize_txb_new yet -int av1_optimize_txb(const struct AV1_COMP *cpi, MACROBLOCK *x, int plane, - int blk_row, int blk_col, int block, TX_SIZE tx_size, - TXB_CTX *txb_ctx, int fast_mode, int *rate_cost) { - const AV1_COMMON *cm = &cpi->common; - const int reduced_tx_set_used = cm->features.reduced_tx_set_used; - MACROBLOCKD *const xd = &x->e_mbd; - const PLANE_TYPE plane_type = get_plane_type(plane); - const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size); - const TX_TYPE tx_type = av1_get_tx_type(xd, plane_type, blk_row, blk_col, - tx_size, reduced_tx_set_used); - const MB_MODE_INFO *mbmi = xd->mi[0]; - const struct macroblock_plane *p = &x->plane[plane]; - const int eob = p->eobs[block]; - const int block_offset = BLOCK_OFFSET(block); - tran_low_t *qcoeff = p->qcoeff + block_offset; - tran_low_t *dqcoeff = p->dqcoeff + block_offset; - const tran_low_t *tcoeff = p->coeff + block_offset; - const int16_t *dequant = p->dequant_QTX; - const int seg_eob = av1_get_max_eob(tx_size); - const int bwl = get_txb_bwl(tx_size); - const int width = get_txb_wide(tx_size); - const int height = get_txb_high(tx_size); - const int is_inter = is_inter_block(mbmi); - const SCAN_ORDER *const scan_order = get_scan(tx_size, tx_type); - const CoeffCosts *coeff_costs = &x->coeff_costs; - const LV_MAP_COEFF_COST *txb_costs = - &coeff_costs->coeff_costs[txs_ctx][plane_type]; - const int eob_multi_size = txsize_log2_minus4[tx_size]; - const LV_MAP_EOB_COST txb_eob_costs = - coeff_costs->eob_costs[eob_multi_size][plane_type]; - - const int shift = av1_get_tx_scale(tx_size); - const int64_t rdmult = - (((int64_t)x->rdmult * plane_rd_mult[is_inter][plane_type] - << (2 * (xd->bd - 8))) + - 2) >> - 2; - uint8_t levels_buf[TX_PAD_2D]; - uint8_t *const levels = set_levels(levels_buf, width); - const qm_val_t *iqmatrix = - av1_get_iqmatrix(&cpi->common.quant_params, xd, plane, tx_size, tx_type); - assert(width == (1 << bwl)); - const int tx_type_cost = - get_tx_type_cost(x, xd, plane, tx_size, tx_type, reduced_tx_set_used); - TxbInfo txb_info = { - qcoeff, levels, dqcoeff, tcoeff, dequant, shift, tx_size, - txs_ctx, tx_type, bwl, width, height, eob, seg_eob, - scan_order, txb_ctx, rdmult, iqmatrix, tx_type_cost, - }; - -#if CONFIG_HTB_TRELLIS - // Hash based trellis (hbt) speed feature: avoid expensive optimize_txb calls - // by storing the coefficient deltas in a hash table. - // Currently disabled in speedfeatures.c - if (eob <= HBT_EOB && eob > 0 && cpi->sf.use_hash_based_trellis) { - return hbt_create_hashes(&txb_info, txb_costs, &txb_eob_costs, p, block, - fast_mode, rate_cost); - } -#else - (void)fast_mode; -#endif // CONFIG_HTB_TRELLIS - av1_txb_init_levels(qcoeff, width, height, levels); - - const int update = - optimize_txb(&txb_info, txb_costs, &txb_eob_costs, rate_cost); - - if (update) { - p->eobs[block] = txb_info.eob; - p->txb_entropy_ctx[block] = - av1_get_txb_entropy_context(qcoeff, scan_order, txb_info.eob); - } - return txb_info.eob; -} - uint8_t av1_get_txb_entropy_context(const tran_low_t *qcoeff, const SCAN_ORDER *scan_order, int eob) { const int16_t *const scan = scan_order->scan;
diff --git a/av1/encoder/encodetxb.h b/av1/encoder/encodetxb.h index 391d242..686de75 100644 --- a/av1/encoder/encodetxb.h +++ b/av1/encoder/encodetxb.h
@@ -302,11 +302,6 @@ void av1_update_and_record_txb_context(int plane, int block, int blk_row, int blk_col, BLOCK_SIZE plane_bsize, TX_SIZE tx_size, void *arg); -/*!\cond */ -#if CONFIG_HTB_TRELLIS -void hbt_destroy(); -#endif // CONFIG_HTB_TRELLIS -/*!\endcond */ /*!\brief Adjust the magnitude of quantized coefficients to achieve better * rate-distortion (RD) trade-off.
diff --git a/av1/encoder/speed_features.c b/av1/encoder/speed_features.c index 39ae43f..940f074 100644 --- a/av1/encoder/speed_features.c +++ b/av1/encoder/speed_features.c
@@ -1139,7 +1139,6 @@ rd_sf->simple_model_rd_from_var = 0; rd_sf->tx_domain_dist_level = 0; rd_sf->tx_domain_dist_thres_level = 0; - rd_sf->use_hash_based_trellis = 0; rd_sf->perform_coeff_opt = 0; rd_sf->perform_coeff_opt_based_on_satd = 0; }
diff --git a/av1/encoder/speed_features.h b/av1/encoder/speed_features.h index d118a96..0f7df4a 100644 --- a/av1/encoder/speed_features.h +++ b/av1/encoder/speed_features.h
@@ -828,10 +828,6 @@ // Trellis (dynamic programming) optimization of quantized values TRELLIS_OPT_TYPE optimize_coefficients; - // Use a hash table to store previously computed optimized qcoeffs from - // expensive calls to optimize_txb. - int use_hash_based_trellis; - // Use hash table to store macroblock RD search results // to avoid repeated search on the same residue signal. int use_mb_rd_hash;
diff --git a/build/cmake/aom_config_defaults.cmake b/build/cmake/aom_config_defaults.cmake index 37c779d..3db4f53 100644 --- a/build/cmake/aom_config_defaults.cmake +++ b/build/cmake/aom_config_defaults.cmake
@@ -126,8 +126,6 @@ "Collect encoding component timing information.") set_aom_config_var(CONFIG_LPF_MASK 0 "Enable the use loop filter bitmasks for optimizations.") -set_aom_config_var(CONFIG_HTB_TRELLIS 0 - "Enable the use of hash table for trellis optimizations.") set_aom_config_var(CONFIG_REALTIME_ONLY 0 "Build for RTC-only to reduce binary size.") set_aom_config_var(CONFIG_AV1_HIGHBITDEPTH 1