Pad extra rows in txb levels and signs
This helps the removal of vertical availability check.
Change-Id: Ie9204e3f2aacd86c8e19f1db0e40949e437a500c
diff --git a/av1/encoder/encodetxb.c b/av1/encoder/encodetxb.c
index 205387e..fa31e03 100644
--- a/av1/encoder/encodetxb.c
+++ b/av1/encoder/encodetxb.c
@@ -256,7 +256,14 @@
}
static INLINE void av1_txb_init_levels(const tran_low_t *const coeff,
- const int size, uint8_t *const levels) {
+ const int width, const int size,
+ uint8_t *const levels) {
+ const int stride = width + TX_PAD_HOR;
+
+ memset(levels - TX_PAD_TOP * stride, 0,
+ sizeof(*levels) * TX_PAD_TOP * stride);
+ memset(levels + size, 0, sizeof(*levels) * TX_PAD_BOTTOM * stride);
+
for (int i = 0; i < size; i++) {
levels[i] = (uint8_t)clamp(abs(coeff[i]), 0, UINT8_MAX);
}
@@ -276,10 +283,12 @@
const int seg_eob = tx_size_2d[tx_size];
int c;
const int bwl = b_width_log2_lookup[txsize_to_bsize[tx_size]] + 2;
+ const int width = tx_size_wide[tx_size];
const int height = tx_size_high[tx_size];
int update_eob = -1;
FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
- uint8_t levels[MAX_TX_SQUARE];
+ uint8_t levels_buf[TX_PAD_2D];
+ uint8_t *const levels = set_levels(levels_buf, width);
(void)blk_row;
(void)blk_col;
@@ -289,7 +298,7 @@
if (eob == 0) return;
- av1_txb_init_levels(tcoeff, seg_eob, levels);
+ av1_txb_init_levels(tcoeff, width, seg_eob, levels);
#if CONFIG_TXK_SEL
av1_write_tx_type(cm, xd, blk_row, blk_col, block, plane,
@@ -372,7 +381,7 @@
if (level <= i) continue;
- ctx = get_base_ctx(levels, scan[c], bwl, height, i + 1);
+ ctx = get_base_ctx(levels, scan[c], bwl, i + 1);
if (level == i + 1) {
aom_write_bin(w, 1, ec_ctx->coeff_base_cdf[txs_ctx][plane_type][i][ctx],
@@ -415,7 +424,7 @@
if (level <= NUM_BASE_LEVELS) continue;
// level is above 1.
- ctx = get_br_ctx(levels, scan[c], bwl, height);
+ ctx = get_br_ctx(levels, scan[c], bwl);
int base_range = level - 1 - NUM_BASE_LEVELS;
int br_set_idx = 0;
@@ -487,7 +496,7 @@
static INLINE void get_base_ctx_set(const uint8_t *const levels,
const int c, // raster order
- const int bwl, const int height,
+ const int bwl,
int ctx_set[NUM_BASE_LEVELS]) {
const int row = c >> bwl;
const int col = c - (row << bwl);
@@ -502,8 +511,7 @@
int ref_col = col + base_ref_offset[idx][1];
int pos = (ref_row << bwl) + ref_col;
- if (ref_row < 0 || ref_col < 0 || ref_row >= height || ref_col >= stride)
- continue;
+ if (ref_col < 0 || ref_col >= stride) continue;
const uint8_t abs_coeff = levels[pos];
@@ -567,11 +575,13 @@
int txb_skip_ctx = txb_ctx->txb_skip_ctx;
const int bwl = b_width_log2_lookup[txsize_to_bsize[tx_size]] + 2;
+ const int width = tx_size_wide[tx_size];
const int height = tx_size_high[tx_size];
const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, mbmi);
const int16_t *scan = scan_order->scan;
- uint8_t levels[MAX_TX_SQUARE];
+ uint8_t levels_buf[TX_PAD_2D];
+ uint8_t *const levels = set_levels(levels_buf, width);
LV_MAP_COEFF_COST *coeff_costs = &x->coeff_costs[txs_ctx][plane_type];
@@ -583,7 +593,7 @@
}
cost = coeff_costs->txb_skip_cost[txb_skip_ctx][0];
- av1_txb_init_levels(qcoeff, tx_size_2d[tx_size], levels);
+ av1_txb_init_levels(qcoeff, width, tx_size_2d[tx_size], levels);
#if CONFIG_TXK_SEL
cost += av1_tx_type_cost(cm, x, xd, mbmi->sb_type, plane, tx_size, tx_type);
@@ -634,7 +644,7 @@
if (is_k == 0) break;
}
#else
- get_base_ctx_set(levels, scan[c], bwl, height, ctx_ls);
+ get_base_ctx_set(levels, scan[c], bwl, ctx_ls);
int i;
for (i = 0; i < NUM_BASE_LEVELS; ++i) {
@@ -650,7 +660,7 @@
if (level > NUM_BASE_LEVELS) {
int ctx;
- ctx = get_br_ctx(levels, scan[c], bwl, height);
+ ctx = get_br_ctx(levels, scan[c], bwl);
int base_range = level - 1 - NUM_BASE_LEVELS;
if (base_range < COEFF_BASE_RANGE) {
cost += coeff_costs->lps_cost[ctx][base_range];
@@ -732,8 +742,8 @@
const int row = coeff_idx >> bwl;
const int col = coeff_idx - (row << bwl);
- txb_cache->nz_count_arr[coeff_idx] = get_nz_count(
- levels, bwl, height, row, col, get_tx_class(txb_info->tx_type));
+ txb_cache->nz_count_arr[coeff_idx] =
+ get_nz_count(levels, bwl, row, col, get_tx_class(txb_info->tx_type));
const int nz_count = txb_cache->nz_count_arr[coeff_idx];
txb_cache->nz_ctx_arr[coeff_idx] =
@@ -1433,8 +1443,7 @@
}
#else
int ctx_ls[NUM_BASE_LEVELS] = { 0 };
- get_base_ctx_set(txb_info->levels, scan[scan_idx], txb_info->bwl,
- txb_info->height, ctx_ls);
+ get_base_ctx_set(txb_info->levels, scan[scan_idx], txb_info->bwl, ctx_ls);
int i;
for (i = 0; i < NUM_BASE_LEVELS; ++i) {
@@ -1444,8 +1453,7 @@
#endif
if (abs_qc > NUM_BASE_LEVELS) {
- int ctx = get_br_ctx(txb_info->levels, scan[scan_idx], txb_info->bwl,
- txb_info->height);
+ int ctx = get_br_ctx(txb_info->levels, scan[scan_idx], txb_info->bwl);
cost += get_br_cost(abs_qc, ctx, txb_costs->lps_cost[ctx]);
cost += get_golomb_cost(abs_qc);
}
@@ -1649,12 +1657,15 @@
tran_low_t tmp_qcoeff[MAX_TX_SQUARE];
tran_low_t tmp_dqcoeff[MAX_TX_SQUARE];
- uint8_t tmp_levels[MAX_TX_SQUARE];
+ uint8_t tmp_levels_buf[TX_PAD_2D];
+ uint8_t *const tmp_levels = set_levels(tmp_levels_buf, txb_info->stride);
const int org_eob = txb_info->eob;
if (dry_run) {
+ const int stride = txb_info->stride + TX_PAD_HOR;
memcpy(tmp_qcoeff, org_qcoeff, sizeof(org_qcoeff[0]) * max_eob);
memcpy(tmp_dqcoeff, org_dqcoeff, sizeof(org_dqcoeff[0]) * max_eob);
- memcpy(tmp_levels, org_levels, sizeof(org_levels[0]) * max_eob);
+ memcpy(tmp_levels, org_levels - TX_PAD_TOP * stride,
+ sizeof(org_levels[0]) * stride * (txb_info->height + TX_PAD_VER));
txb_info->qcoeff = tmp_qcoeff;
txb_info->dqcoeff = tmp_dqcoeff;
txb_info->levels = tmp_levels;
@@ -1788,12 +1799,15 @@
tran_low_t tmp_qcoeff[MAX_TX_SQUARE];
tran_low_t tmp_dqcoeff[MAX_TX_SQUARE];
- uint8_t tmp_levels[MAX_TX_SQUARE];
+ uint8_t tmp_levels_buf[TX_PAD_2D];
+ uint8_t *const tmp_levels = set_levels(tmp_levels_buf, txb_info->stride);
const int org_eob = txb_info->eob;
if (dry_run) {
+ const int stride = txb_info->stride + TX_PAD_HOR;
memcpy(tmp_qcoeff, org_qcoeff, sizeof(org_qcoeff[0]) * max_eob);
memcpy(tmp_dqcoeff, org_dqcoeff, sizeof(org_dqcoeff[0]) * max_eob);
- memcpy(tmp_levels, org_levels, sizeof(org_levels[0]) * max_eob);
+ memcpy(tmp_levels, org_levels - TX_PAD_TOP * stride,
+ sizeof(org_levels[0]) * stride * (txb_info->height + TX_PAD_VER));
txb_info->qcoeff = tmp_qcoeff;
txb_info->dqcoeff = tmp_dqcoeff;
txb_info->levels = tmp_levels;
@@ -1897,6 +1911,7 @@
const int seg_eob = tx_size_2d[tx_size];
const int bwl = b_width_log2_lookup[txsize_to_bsize[tx_size]] + 2;
const int stride = 1 << bwl;
+ const int width = tx_size_wide[tx_size];
const int height = tx_size_high[tx_size];
const int is_inter = is_inter_block(mbmi);
const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, mbmi);
@@ -1905,7 +1920,8 @@
const int shift = av1_get_tx_scale(tx_size);
const int64_t rdmult =
(x->rdmult * plane_rd_mult[is_inter][plane_type] + 2) >> 2;
- uint8_t levels[MAX_TX_SQUARE];
+ uint8_t levels_buf[TX_PAD_2D];
+ uint8_t *const levels = set_levels(levels_buf, width);
TxbInfo txb_info = {
qcoeff, levels, dqcoeff, tcoeff, dequant, shift,
@@ -1913,7 +1929,7 @@
eob, seg_eob, scan_order, txb_ctx, rdmult, &cm->coeff_ctx_table
};
- av1_txb_init_levels(qcoeff, tx_size_2d[tx_size], levels);
+ av1_txb_init_levels(qcoeff, width, tx_size_2d[tx_size], levels);
const int update = optimize_txb(&txb_info, &txb_costs, NULL, 0, fast_mode);
@@ -1988,8 +2004,10 @@
get_txb_ctx(plane_bsize, tx_size, plane, pd->above_context + blk_col,
pd->left_context + blk_row, &txb_ctx);
const int bwl = b_width_log2_lookup[txsize_to_bsize[tx_size]] + 2;
+ const int width = tx_size_wide[tx_size];
const int height = tx_size_high[tx_size];
- uint8_t levels[MAX_TX_SQUARE];
+ uint8_t levels_buf[TX_PAD_2D];
+ uint8_t *const levels = set_levels(levels_buf, width);
const uint8_t allow_update_cdf = args->allow_update_cdf;
TX_SIZE txsize_ctx = get_txsize_context(tx_size);
@@ -2010,7 +2028,7 @@
return;
}
- av1_txb_init_levels(tcoeff, tx_size_2d[tx_size], levels);
+ av1_txb_init_levels(tcoeff, width, tx_size_2d[tx_size], levels);
#if CONFIG_TXK_SEL
av1_update_tx_type_count(cm, xd, blk_row, blk_col, block, plane,
@@ -2073,7 +2091,7 @@
if (level <= i) continue;
- ctx = get_base_ctx(levels, scan[c], bwl, height, i + 1);
+ ctx = get_base_ctx(levels, scan[c], bwl, i + 1);
if (level == i + 1) {
++td->counts->coeff_base[txsize_ctx][plane_type][i][ctx][1];
@@ -2112,7 +2130,7 @@
if (level <= NUM_BASE_LEVELS) continue;
// level is above 1.
- ctx = get_br_ctx(levels, scan[c], bwl, height);
+ ctx = get_br_ctx(levels, scan[c], bwl);
int base_range = level - 1 - NUM_BASE_LEVELS;
int br_set_idx = base_range < COEFF_BASE_RANGE
