Pad extra rows in txb levels and signs
This helps the removal of vertical availability check.
Change-Id: Ie9204e3f2aacd86c8e19f1db0e40949e437a500c
diff --git a/av1/common/enums.h b/av1/common/enums.h
index 404a5e9..b368701 100644
--- a/av1/common/enums.h
+++ b/av1/common/enums.h
@@ -243,6 +243,14 @@
#define MIN_TX_SIZE (1 << MIN_TX_SIZE_LOG2)
#define MAX_TX_SQUARE (MAX_TX_SIZE * MAX_TX_SIZE)
+#define TX_PAD_HOR 0
+// Pad 6 extra rows (2 on top and 4 on bottom) to remove vertical availability
+// check.
+#define TX_PAD_TOP 2
+#define TX_PAD_BOTTOM 4
+#define TX_PAD_VER (TX_PAD_TOP + TX_PAD_BOTTOM)
+#define TX_PAD_2D ((MAX_TX_SIZE + TX_PAD_HOR) * (MAX_TX_SIZE + TX_PAD_VER))
+
// Number of maxium size transform blocks in the maximum size superblock
#define MAX_TX_BLOCKS_IN_MAX_SB_LOG2 ((MAX_SB_SIZE_LOG2 - MAX_TX_SIZE_LOG2) * 2)
#define MAX_TX_BLOCKS_IN_MAX_SB (1 << MAX_TX_BLOCKS_IN_MAX_SB_LOG2)
diff --git a/av1/common/txb_common.h b/av1/common/txb_common.h
index b5b3461..6d0d7bd 100644
--- a/av1/common/txb_common.h
+++ b/av1/common/txb_common.h
@@ -38,6 +38,7 @@
0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3,
};
+// Note: TX_PAD_2D is dependent to this offset table.
static const int base_ref_offset[BASE_CONTEXT_POSITION_NUM][2] = {
/* clang-format off*/
{ -2, 0 }, { -1, -1 }, { -1, 0 }, { -1, 1 }, { 0, -2 }, { 0, -1 }, { 0, 1 },
@@ -75,18 +76,23 @@
}
}
-static INLINE int get_level_count_mag(
- int *const mag, const uint8_t *const levels, const int bwl,
- const int height, const int row, const int col, const int level,
- const int (*nb_offset)[2], const int nb_num) {
+static INLINE uint8_t *set_levels(uint8_t *const levels_buf, const int width) {
+ return levels_buf + TX_PAD_TOP * (width + TX_PAD_HOR);
+}
+
+static INLINE int get_level_count_mag(int *const mag,
+ const uint8_t *const levels,
+ const int bwl, const int row,
+ const int col, const int level,
+ const int (*nb_offset)[2],
+ const int nb_num) {
const int stride = 1 << bwl;
int count = 0;
for (int idx = 0; idx < nb_num; ++idx) {
const int ref_row = row + nb_offset[idx][0];
const int ref_col = col + nb_offset[idx][1];
- if (ref_row < 0 || ref_col < 0 || ref_row >= height || ref_col >= stride)
- continue;
+ if (ref_col < 0 || ref_col >= stride) continue;
const int pos = (ref_row << bwl) + ref_col;
count += levels[pos] > level;
if (nb_offset[idx][0] == 0 && nb_offset[idx][1] == 1) mag[0] = levels[pos];
@@ -162,15 +168,14 @@
static INLINE int get_base_ctx(const uint8_t *const levels,
const int c, // raster order
- const int bwl, const int height,
- const int level) {
+ const int bwl, const int level) {
const int row = c >> bwl;
const int col = c - (row << bwl);
const int level_minus_1 = level - 1;
int mag_count = 0;
int nb_mag[3] = { 0 };
const int count =
- get_level_count_mag(nb_mag, levels, bwl, height, row, col, level_minus_1,
+ get_level_count_mag(nb_mag, levels, bwl, row, col, level_minus_1,
base_ref_offset, BASE_CONTEXT_POSITION_NUM);
for (int idx = 0; idx < 3; ++idx) mag_count += nb_mag[idx] > level;
@@ -180,6 +185,7 @@
}
#define BR_CONTEXT_POSITION_NUM 8 // Base range coefficient context
+// Note: TX_PAD_2D is dependent to this offset table.
static const int br_ref_offset[BR_CONTEXT_POSITION_NUM][2] = {
/* clang-format off*/
{ -1, -1 }, { -1, 0 }, { -1, 1 }, { 0, -1 },
@@ -263,14 +269,14 @@
static INLINE int get_br_ctx(const uint8_t *const levels,
const int c, // raster order
- const int bwl, const int height) {
+ const int bwl) {
const int row = c >> bwl;
const int col = c - (row << bwl);
const int level_minus_1 = NUM_BASE_LEVELS;
int mag = 0;
int nb_mag[3] = { 0 };
const int count =
- get_level_count_mag(nb_mag, levels, bwl, height, row, col, level_minus_1,
+ get_level_count_mag(nb_mag, levels, bwl, row, col, level_minus_1,
br_ref_offset, BR_CONTEXT_POSITION_NUM);
for (int idx = 0; idx < 3; ++idx) mag = AOMMAX(mag, nb_mag[idx]);
const int ctx = get_br_ctx_from_count_mag(row, col, count, mag);
@@ -279,6 +285,7 @@
#define SIG_REF_OFFSET_NUM 7
+// Note: TX_PAD_2D is dependent to these offset tables.
static const int sig_ref_offset[SIG_REF_OFFSET_NUM][2] = {
{ 0, 1 }, { 1, 0 }, { 1, 1 }, { 0, 2 }, { 2, 0 }, { 1, 2 }, { 2, 1 },
};
@@ -293,9 +300,8 @@
#if USE_CAUSAL_BASE_CTX
static INLINE int get_nz_count_mag(const uint8_t *const levels, const int bwl,
- const int height, const int row,
- const int col, const TX_CLASS tx_class,
- int *const mag) {
+ const int row, const int col,
+ const TX_CLASS tx_class, int *const mag) {
int count = 0;
*mag = 0;
for (int idx = 0; idx < SIG_REF_OFFSET_NUM; ++idx) {
@@ -311,9 +317,7 @@
: sig_ref_offset_horiz[idx][1]));
const int ref_row = row + row_offset;
const int ref_col = col + col_offset;
- if (ref_row < 0 || ref_col < 0 || ref_row >= height ||
- ref_col >= (1 << bwl))
- continue;
+ if (ref_col >= (1 << bwl)) continue;
const int nb_pos = (ref_row << bwl) + ref_col;
const int level = levels[nb_pos];
count += (level != 0);
@@ -327,7 +331,7 @@
}
#endif
static INLINE int get_nz_count(const uint8_t *const levels, const int bwl,
- const int height, const int row, const int col,
+ const int row, const int col,
const TX_CLASS tx_class) {
int count = 0;
for (int idx = 0; idx < SIG_REF_OFFSET_NUM; ++idx) {
@@ -341,9 +345,7 @@
: ((tx_class == TX_CLASS_VERT)
? sig_ref_offset_vert[idx][1]
: sig_ref_offset_horiz[idx][1]));
- if (ref_row < 0 || ref_col < 0 || ref_row >= height ||
- ref_col >= (1 << bwl))
- continue;
+ if (ref_col >= (1 << bwl)) continue;
const int nb_pos = (ref_row << bwl) + ref_col;
count += (levels[nb_pos] != 0);
}
@@ -436,10 +438,10 @@
int tx_class = get_tx_class(tx_type);
#if USE_CAUSAL_BASE_CTX
int mag = 0;
- int count = get_nz_count_mag(levels, bwl, height, row, col, tx_class, &mag);
+ int count = get_nz_count_mag(levels, bwl, row, col, tx_class, &mag);
return get_nz_map_ctx_from_count(count, coeff_idx, bwl, height, mag, tx_type);
#else
- int count = get_nz_count(levels, bwl, height, row, col, tx_class);
+ int count = get_nz_count(levels, bwl, row, col, tx_class);
return get_nz_map_ctx_from_count(count, coeff_idx, bwl, height, tx_type);
#endif
}
diff --git a/av1/decoder/decodetxb.c b/av1/decoder/decodetxb.c
index 0afba17..f0bba6b 100644
--- a/av1/decoder/decodetxb.c
+++ b/av1/decoder/decodetxb.c
@@ -69,9 +69,11 @@
xd->plane[plane].seg_dequant_QTX[mbmi->segment_id];
const int shift = av1_get_tx_scale(tx_size);
const int bwl = b_width_log2_lookup[txsize_to_bsize[tx_size]] + 2;
+ const int width = tx_size_wide[tx_size];
const int height = tx_size_high[tx_size];
int cul_level = 0;
- uint8_t levels[MAX_TX_SQUARE];
+ uint8_t levels_buf[TX_PAD_2D];
+ uint8_t *const levels = set_levels(levels_buf, width);
int8_t signs[MAX_TX_SQUARE];
memset(tcoeffs, 0, sizeof(*tcoeffs) * seg_eob);
@@ -91,8 +93,9 @@
return 0;
}
- memset(levels, 0, sizeof(levels[0]) * seg_eob);
- memset(signs, 0, sizeof(signs[0]) * seg_eob);
+ memset(levels_buf, 0,
+ sizeof(*levels_buf) * (seg_eob + TX_PAD_VER * (width + TX_PAD_HOR)));
+ memset(signs, 0, sizeof(*signs) * seg_eob);
(void)blk_row;
(void)blk_col;
@@ -209,7 +212,7 @@
if (*level <= i) continue;
- ctx = get_base_ctx(levels, scan[c], bwl, height, i + 1);
+ ctx = get_base_ctx(levels, scan[c], bwl, i + 1);
if (av1_read_record_bin(
counts, r, ec_ctx->coeff_base_cdf[txs_ctx][plane_type][i][ctx], 2,
@@ -256,7 +259,7 @@
if (*level <= NUM_BASE_LEVELS) continue;
- ctx = get_br_ctx(levels, scan[c], bwl, height);
+ ctx = get_br_ctx(levels, scan[c], bwl);
for (idx = 0; idx < BASE_RANGE_SETS; ++idx) {
if (av1_read_record_bin(
diff --git a/av1/encoder/encodetxb.c b/av1/encoder/encodetxb.c
index 205387e..fa31e03 100644
--- a/av1/encoder/encodetxb.c
+++ b/av1/encoder/encodetxb.c
@@ -256,7 +256,14 @@
}
static INLINE void av1_txb_init_levels(const tran_low_t *const coeff,
- const int size, uint8_t *const levels) {
+ const int width, const int size,
+ uint8_t *const levels) {
+ const int stride = width + TX_PAD_HOR;
+
+ memset(levels - TX_PAD_TOP * stride, 0,
+ sizeof(*levels) * TX_PAD_TOP * stride);
+ memset(levels + size, 0, sizeof(*levels) * TX_PAD_BOTTOM * stride);
+
for (int i = 0; i < size; i++) {
levels[i] = (uint8_t)clamp(abs(coeff[i]), 0, UINT8_MAX);
}
@@ -276,10 +283,12 @@
const int seg_eob = tx_size_2d[tx_size];
int c;
const int bwl = b_width_log2_lookup[txsize_to_bsize[tx_size]] + 2;
+ const int width = tx_size_wide[tx_size];
const int height = tx_size_high[tx_size];
int update_eob = -1;
FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
- uint8_t levels[MAX_TX_SQUARE];
+ uint8_t levels_buf[TX_PAD_2D];
+ uint8_t *const levels = set_levels(levels_buf, width);
(void)blk_row;
(void)blk_col;
@@ -289,7 +298,7 @@
if (eob == 0) return;
- av1_txb_init_levels(tcoeff, seg_eob, levels);
+ av1_txb_init_levels(tcoeff, width, seg_eob, levels);
#if CONFIG_TXK_SEL
av1_write_tx_type(cm, xd, blk_row, blk_col, block, plane,
@@ -372,7 +381,7 @@
if (level <= i) continue;
- ctx = get_base_ctx(levels, scan[c], bwl, height, i + 1);
+ ctx = get_base_ctx(levels, scan[c], bwl, i + 1);
if (level == i + 1) {
aom_write_bin(w, 1, ec_ctx->coeff_base_cdf[txs_ctx][plane_type][i][ctx],
@@ -415,7 +424,7 @@
if (level <= NUM_BASE_LEVELS) continue;
// level is above 1.
- ctx = get_br_ctx(levels, scan[c], bwl, height);
+ ctx = get_br_ctx(levels, scan[c], bwl);
int base_range = level - 1 - NUM_BASE_LEVELS;
int br_set_idx = 0;
@@ -487,7 +496,7 @@
static INLINE void get_base_ctx_set(const uint8_t *const levels,
const int c, // raster order
- const int bwl, const int height,
+ const int bwl,
int ctx_set[NUM_BASE_LEVELS]) {
const int row = c >> bwl;
const int col = c - (row << bwl);
@@ -502,8 +511,7 @@
int ref_col = col + base_ref_offset[idx][1];
int pos = (ref_row << bwl) + ref_col;
- if (ref_row < 0 || ref_col < 0 || ref_row >= height || ref_col >= stride)
- continue;
+ if (ref_col < 0 || ref_col >= stride) continue;
const uint8_t abs_coeff = levels[pos];
@@ -567,11 +575,13 @@
int txb_skip_ctx = txb_ctx->txb_skip_ctx;
const int bwl = b_width_log2_lookup[txsize_to_bsize[tx_size]] + 2;
+ const int width = tx_size_wide[tx_size];
const int height = tx_size_high[tx_size];
const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, mbmi);
const int16_t *scan = scan_order->scan;
- uint8_t levels[MAX_TX_SQUARE];
+ uint8_t levels_buf[TX_PAD_2D];
+ uint8_t *const levels = set_levels(levels_buf, width);
LV_MAP_COEFF_COST *coeff_costs = &x->coeff_costs[txs_ctx][plane_type];
@@ -583,7 +593,7 @@
}
cost = coeff_costs->txb_skip_cost[txb_skip_ctx][0];
- av1_txb_init_levels(qcoeff, tx_size_2d[tx_size], levels);
+ av1_txb_init_levels(qcoeff, width, tx_size_2d[tx_size], levels);
#if CONFIG_TXK_SEL
cost += av1_tx_type_cost(cm, x, xd, mbmi->sb_type, plane, tx_size, tx_type);
@@ -634,7 +644,7 @@
if (is_k == 0) break;
}
#else
- get_base_ctx_set(levels, scan[c], bwl, height, ctx_ls);
+ get_base_ctx_set(levels, scan[c], bwl, ctx_ls);
int i;
for (i = 0; i < NUM_BASE_LEVELS; ++i) {
@@ -650,7 +660,7 @@
if (level > NUM_BASE_LEVELS) {
int ctx;
- ctx = get_br_ctx(levels, scan[c], bwl, height);
+ ctx = get_br_ctx(levels, scan[c], bwl);
int base_range = level - 1 - NUM_BASE_LEVELS;
if (base_range < COEFF_BASE_RANGE) {
cost += coeff_costs->lps_cost[ctx][base_range];
@@ -732,8 +742,8 @@
const int row = coeff_idx >> bwl;
const int col = coeff_idx - (row << bwl);
- txb_cache->nz_count_arr[coeff_idx] = get_nz_count(
- levels, bwl, height, row, col, get_tx_class(txb_info->tx_type));
+ txb_cache->nz_count_arr[coeff_idx] =
+ get_nz_count(levels, bwl, row, col, get_tx_class(txb_info->tx_type));
const int nz_count = txb_cache->nz_count_arr[coeff_idx];
txb_cache->nz_ctx_arr[coeff_idx] =
@@ -1433,8 +1443,7 @@
}
#else
int ctx_ls[NUM_BASE_LEVELS] = { 0 };
- get_base_ctx_set(txb_info->levels, scan[scan_idx], txb_info->bwl,
- txb_info->height, ctx_ls);
+ get_base_ctx_set(txb_info->levels, scan[scan_idx], txb_info->bwl, ctx_ls);
int i;
for (i = 0; i < NUM_BASE_LEVELS; ++i) {
@@ -1444,8 +1453,7 @@
#endif
if (abs_qc > NUM_BASE_LEVELS) {
- int ctx = get_br_ctx(txb_info->levels, scan[scan_idx], txb_info->bwl,
- txb_info->height);
+ int ctx = get_br_ctx(txb_info->levels, scan[scan_idx], txb_info->bwl);
cost += get_br_cost(abs_qc, ctx, txb_costs->lps_cost[ctx]);
cost += get_golomb_cost(abs_qc);
}
@@ -1649,12 +1657,15 @@
tran_low_t tmp_qcoeff[MAX_TX_SQUARE];
tran_low_t tmp_dqcoeff[MAX_TX_SQUARE];
- uint8_t tmp_levels[MAX_TX_SQUARE];
+ uint8_t tmp_levels_buf[TX_PAD_2D];
+ uint8_t *const tmp_levels = set_levels(tmp_levels_buf, txb_info->stride);
const int org_eob = txb_info->eob;
if (dry_run) {
+ const int stride = txb_info->stride + TX_PAD_HOR;
memcpy(tmp_qcoeff, org_qcoeff, sizeof(org_qcoeff[0]) * max_eob);
memcpy(tmp_dqcoeff, org_dqcoeff, sizeof(org_dqcoeff[0]) * max_eob);
- memcpy(tmp_levels, org_levels, sizeof(org_levels[0]) * max_eob);
+ memcpy(tmp_levels, org_levels - TX_PAD_TOP * stride,
+ sizeof(org_levels[0]) * stride * (txb_info->height + TX_PAD_VER));
txb_info->qcoeff = tmp_qcoeff;
txb_info->dqcoeff = tmp_dqcoeff;
txb_info->levels = tmp_levels;
@@ -1788,12 +1799,15 @@
tran_low_t tmp_qcoeff[MAX_TX_SQUARE];
tran_low_t tmp_dqcoeff[MAX_TX_SQUARE];
- uint8_t tmp_levels[MAX_TX_SQUARE];
+ uint8_t tmp_levels_buf[TX_PAD_2D];
+ uint8_t *const tmp_levels = set_levels(tmp_levels_buf, txb_info->stride);
const int org_eob = txb_info->eob;
if (dry_run) {
+ const int stride = txb_info->stride + TX_PAD_HOR;
memcpy(tmp_qcoeff, org_qcoeff, sizeof(org_qcoeff[0]) * max_eob);
memcpy(tmp_dqcoeff, org_dqcoeff, sizeof(org_dqcoeff[0]) * max_eob);
- memcpy(tmp_levels, org_levels, sizeof(org_levels[0]) * max_eob);
+ memcpy(tmp_levels, org_levels - TX_PAD_TOP * stride,
+ sizeof(org_levels[0]) * stride * (txb_info->height + TX_PAD_VER));
txb_info->qcoeff = tmp_qcoeff;
txb_info->dqcoeff = tmp_dqcoeff;
txb_info->levels = tmp_levels;
@@ -1897,6 +1911,7 @@
const int seg_eob = tx_size_2d[tx_size];
const int bwl = b_width_log2_lookup[txsize_to_bsize[tx_size]] + 2;
const int stride = 1 << bwl;
+ const int width = tx_size_wide[tx_size];
const int height = tx_size_high[tx_size];
const int is_inter = is_inter_block(mbmi);
const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, mbmi);
@@ -1905,7 +1920,8 @@
const int shift = av1_get_tx_scale(tx_size);
const int64_t rdmult =
(x->rdmult * plane_rd_mult[is_inter][plane_type] + 2) >> 2;
- uint8_t levels[MAX_TX_SQUARE];
+ uint8_t levels_buf[TX_PAD_2D];
+ uint8_t *const levels = set_levels(levels_buf, width);
TxbInfo txb_info = {
qcoeff, levels, dqcoeff, tcoeff, dequant, shift,
@@ -1913,7 +1929,7 @@
eob, seg_eob, scan_order, txb_ctx, rdmult, &cm->coeff_ctx_table
};
- av1_txb_init_levels(qcoeff, tx_size_2d[tx_size], levels);
+ av1_txb_init_levels(qcoeff, width, tx_size_2d[tx_size], levels);
const int update = optimize_txb(&txb_info, &txb_costs, NULL, 0, fast_mode);
@@ -1988,8 +2004,10 @@
get_txb_ctx(plane_bsize, tx_size, plane, pd->above_context + blk_col,
pd->left_context + blk_row, &txb_ctx);
const int bwl = b_width_log2_lookup[txsize_to_bsize[tx_size]] + 2;
+ const int width = tx_size_wide[tx_size];
const int height = tx_size_high[tx_size];
- uint8_t levels[MAX_TX_SQUARE];
+ uint8_t levels_buf[TX_PAD_2D];
+ uint8_t *const levels = set_levels(levels_buf, width);
const uint8_t allow_update_cdf = args->allow_update_cdf;
TX_SIZE txsize_ctx = get_txsize_context(tx_size);
@@ -2010,7 +2028,7 @@
return;
}
- av1_txb_init_levels(tcoeff, tx_size_2d[tx_size], levels);
+ av1_txb_init_levels(tcoeff, width, tx_size_2d[tx_size], levels);
#if CONFIG_TXK_SEL
av1_update_tx_type_count(cm, xd, blk_row, blk_col, block, plane,
@@ -2073,7 +2091,7 @@
if (level <= i) continue;
- ctx = get_base_ctx(levels, scan[c], bwl, height, i + 1);
+ ctx = get_base_ctx(levels, scan[c], bwl, i + 1);
if (level == i + 1) {
++td->counts->coeff_base[txsize_ctx][plane_type][i][ctx][1];
@@ -2112,7 +2130,7 @@
if (level <= NUM_BASE_LEVELS) continue;
// level is above 1.
- ctx = get_br_ctx(levels, scan[c], bwl, height);
+ ctx = get_br_ctx(levels, scan[c], bwl);
int base_range = level - 1 - NUM_BASE_LEVELS;
int br_set_idx = base_range < COEFF_BASE_RANGE
