Pad extra rows in txb levels and signs This helps the removal of vertical availability check. Change-Id: Ie9204e3f2aacd86c8e19f1db0e40949e437a500c
diff --git a/av1/common/enums.h b/av1/common/enums.h index 404a5e9..b368701 100644 --- a/av1/common/enums.h +++ b/av1/common/enums.h
@@ -243,6 +243,14 @@ #define MIN_TX_SIZE (1 << MIN_TX_SIZE_LOG2) #define MAX_TX_SQUARE (MAX_TX_SIZE * MAX_TX_SIZE) +#define TX_PAD_HOR 0 +// Pad 6 extra rows (2 on top and 4 on bottom) to remove vertical availability +// check. +#define TX_PAD_TOP 2 +#define TX_PAD_BOTTOM 4 +#define TX_PAD_VER (TX_PAD_TOP + TX_PAD_BOTTOM) +#define TX_PAD_2D ((MAX_TX_SIZE + TX_PAD_HOR) * (MAX_TX_SIZE + TX_PAD_VER)) + // Number of maxium size transform blocks in the maximum size superblock #define MAX_TX_BLOCKS_IN_MAX_SB_LOG2 ((MAX_SB_SIZE_LOG2 - MAX_TX_SIZE_LOG2) * 2) #define MAX_TX_BLOCKS_IN_MAX_SB (1 << MAX_TX_BLOCKS_IN_MAX_SB_LOG2)
diff --git a/av1/common/txb_common.h b/av1/common/txb_common.h index b5b3461..6d0d7bd 100644 --- a/av1/common/txb_common.h +++ b/av1/common/txb_common.h
@@ -38,6 +38,7 @@ 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, }; +// Note: TX_PAD_2D is dependent to this offset table. static const int base_ref_offset[BASE_CONTEXT_POSITION_NUM][2] = { /* clang-format off*/ { -2, 0 }, { -1, -1 }, { -1, 0 }, { -1, 1 }, { 0, -2 }, { 0, -1 }, { 0, 1 }, @@ -75,18 +76,23 @@ } } -static INLINE int get_level_count_mag( - int *const mag, const uint8_t *const levels, const int bwl, - const int height, const int row, const int col, const int level, - const int (*nb_offset)[2], const int nb_num) { +static INLINE uint8_t *set_levels(uint8_t *const levels_buf, const int width) { + return levels_buf + TX_PAD_TOP * (width + TX_PAD_HOR); +} + +static INLINE int get_level_count_mag(int *const mag, + const uint8_t *const levels, + const int bwl, const int row, + const int col, const int level, + const int (*nb_offset)[2], + const int nb_num) { const int stride = 1 << bwl; int count = 0; for (int idx = 0; idx < nb_num; ++idx) { const int ref_row = row + nb_offset[idx][0]; const int ref_col = col + nb_offset[idx][1]; - if (ref_row < 0 || ref_col < 0 || ref_row >= height || ref_col >= stride) - continue; + if (ref_col < 0 || ref_col >= stride) continue; const int pos = (ref_row << bwl) + ref_col; count += levels[pos] > level; if (nb_offset[idx][0] == 0 && nb_offset[idx][1] == 1) mag[0] = levels[pos]; @@ -162,15 +168,14 @@ static INLINE int get_base_ctx(const uint8_t *const levels, const int c, // raster order - const int bwl, const int height, - const int level) { + const int bwl, const int level) { const int row = c >> bwl; const int col = c - (row << bwl); const int level_minus_1 = level - 1; int mag_count = 0; int nb_mag[3] = { 0 }; const int count = - get_level_count_mag(nb_mag, levels, bwl, height, row, col, level_minus_1, + get_level_count_mag(nb_mag, levels, bwl, row, col, level_minus_1, base_ref_offset, BASE_CONTEXT_POSITION_NUM); for (int idx = 0; idx < 3; ++idx) mag_count += nb_mag[idx] > level; @@ -180,6 +185,7 @@ } #define BR_CONTEXT_POSITION_NUM 8 // Base range coefficient context +// Note: TX_PAD_2D is dependent to this offset table. static const int br_ref_offset[BR_CONTEXT_POSITION_NUM][2] = { /* clang-format off*/ { -1, -1 }, { -1, 0 }, { -1, 1 }, { 0, -1 }, @@ -263,14 +269,14 @@ static INLINE int get_br_ctx(const uint8_t *const levels, const int c, // raster order - const int bwl, const int height) { + const int bwl) { const int row = c >> bwl; const int col = c - (row << bwl); const int level_minus_1 = NUM_BASE_LEVELS; int mag = 0; int nb_mag[3] = { 0 }; const int count = - get_level_count_mag(nb_mag, levels, bwl, height, row, col, level_minus_1, + get_level_count_mag(nb_mag, levels, bwl, row, col, level_minus_1, br_ref_offset, BR_CONTEXT_POSITION_NUM); for (int idx = 0; idx < 3; ++idx) mag = AOMMAX(mag, nb_mag[idx]); const int ctx = get_br_ctx_from_count_mag(row, col, count, mag); @@ -279,6 +285,7 @@ #define SIG_REF_OFFSET_NUM 7 +// Note: TX_PAD_2D is dependent to these offset tables. static const int sig_ref_offset[SIG_REF_OFFSET_NUM][2] = { { 0, 1 }, { 1, 0 }, { 1, 1 }, { 0, 2 }, { 2, 0 }, { 1, 2 }, { 2, 1 }, }; @@ -293,9 +300,8 @@ #if USE_CAUSAL_BASE_CTX static INLINE int get_nz_count_mag(const uint8_t *const levels, const int bwl, - const int height, const int row, - const int col, const TX_CLASS tx_class, - int *const mag) { + const int row, const int col, + const TX_CLASS tx_class, int *const mag) { int count = 0; *mag = 0; for (int idx = 0; idx < SIG_REF_OFFSET_NUM; ++idx) { @@ -311,9 +317,7 @@ : sig_ref_offset_horiz[idx][1])); const int ref_row = row + row_offset; const int ref_col = col + col_offset; - if (ref_row < 0 || ref_col < 0 || ref_row >= height || - ref_col >= (1 << bwl)) - continue; + if (ref_col >= (1 << bwl)) continue; const int nb_pos = (ref_row << bwl) + ref_col; const int level = levels[nb_pos]; count += (level != 0); @@ -327,7 +331,7 @@ } #endif static INLINE int get_nz_count(const uint8_t *const levels, const int bwl, - const int height, const int row, const int col, + const int row, const int col, const TX_CLASS tx_class) { int count = 0; for (int idx = 0; idx < SIG_REF_OFFSET_NUM; ++idx) { @@ -341,9 +345,7 @@ : ((tx_class == TX_CLASS_VERT) ? sig_ref_offset_vert[idx][1] : sig_ref_offset_horiz[idx][1])); - if (ref_row < 0 || ref_col < 0 || ref_row >= height || - ref_col >= (1 << bwl)) - continue; + if (ref_col >= (1 << bwl)) continue; const int nb_pos = (ref_row << bwl) + ref_col; count += (levels[nb_pos] != 0); } @@ -436,10 +438,10 @@ int tx_class = get_tx_class(tx_type); #if USE_CAUSAL_BASE_CTX int mag = 0; - int count = get_nz_count_mag(levels, bwl, height, row, col, tx_class, &mag); + int count = get_nz_count_mag(levels, bwl, row, col, tx_class, &mag); return get_nz_map_ctx_from_count(count, coeff_idx, bwl, height, mag, tx_type); #else - int count = get_nz_count(levels, bwl, height, row, col, tx_class); + int count = get_nz_count(levels, bwl, row, col, tx_class); return get_nz_map_ctx_from_count(count, coeff_idx, bwl, height, tx_type); #endif }
diff --git a/av1/decoder/decodetxb.c b/av1/decoder/decodetxb.c index 0afba17..f0bba6b 100644 --- a/av1/decoder/decodetxb.c +++ b/av1/decoder/decodetxb.c
@@ -69,9 +69,11 @@ xd->plane[plane].seg_dequant_QTX[mbmi->segment_id]; const int shift = av1_get_tx_scale(tx_size); const int bwl = b_width_log2_lookup[txsize_to_bsize[tx_size]] + 2; + const int width = tx_size_wide[tx_size]; const int height = tx_size_high[tx_size]; int cul_level = 0; - uint8_t levels[MAX_TX_SQUARE]; + uint8_t levels_buf[TX_PAD_2D]; + uint8_t *const levels = set_levels(levels_buf, width); int8_t signs[MAX_TX_SQUARE]; memset(tcoeffs, 0, sizeof(*tcoeffs) * seg_eob); @@ -91,8 +93,9 @@ return 0; } - memset(levels, 0, sizeof(levels[0]) * seg_eob); - memset(signs, 0, sizeof(signs[0]) * seg_eob); + memset(levels_buf, 0, + sizeof(*levels_buf) * (seg_eob + TX_PAD_VER * (width + TX_PAD_HOR))); + memset(signs, 0, sizeof(*signs) * seg_eob); (void)blk_row; (void)blk_col; @@ -209,7 +212,7 @@ if (*level <= i) continue; - ctx = get_base_ctx(levels, scan[c], bwl, height, i + 1); + ctx = get_base_ctx(levels, scan[c], bwl, i + 1); if (av1_read_record_bin( counts, r, ec_ctx->coeff_base_cdf[txs_ctx][plane_type][i][ctx], 2, @@ -256,7 +259,7 @@ if (*level <= NUM_BASE_LEVELS) continue; - ctx = get_br_ctx(levels, scan[c], bwl, height); + ctx = get_br_ctx(levels, scan[c], bwl); for (idx = 0; idx < BASE_RANGE_SETS; ++idx) { if (av1_read_record_bin(
diff --git a/av1/encoder/encodetxb.c b/av1/encoder/encodetxb.c index 205387e..fa31e03 100644 --- a/av1/encoder/encodetxb.c +++ b/av1/encoder/encodetxb.c
@@ -256,7 +256,14 @@ } static INLINE void av1_txb_init_levels(const tran_low_t *const coeff, - const int size, uint8_t *const levels) { + const int width, const int size, + uint8_t *const levels) { + const int stride = width + TX_PAD_HOR; + + memset(levels - TX_PAD_TOP * stride, 0, + sizeof(*levels) * TX_PAD_TOP * stride); + memset(levels + size, 0, sizeof(*levels) * TX_PAD_BOTTOM * stride); + for (int i = 0; i < size; i++) { levels[i] = (uint8_t)clamp(abs(coeff[i]), 0, UINT8_MAX); } @@ -276,10 +283,12 @@ const int seg_eob = tx_size_2d[tx_size]; int c; const int bwl = b_width_log2_lookup[txsize_to_bsize[tx_size]] + 2; + const int width = tx_size_wide[tx_size]; const int height = tx_size_high[tx_size]; int update_eob = -1; FRAME_CONTEXT *ec_ctx = xd->tile_ctx; - uint8_t levels[MAX_TX_SQUARE]; + uint8_t levels_buf[TX_PAD_2D]; + uint8_t *const levels = set_levels(levels_buf, width); (void)blk_row; (void)blk_col; @@ -289,7 +298,7 @@ if (eob == 0) return; - av1_txb_init_levels(tcoeff, seg_eob, levels); + av1_txb_init_levels(tcoeff, width, seg_eob, levels); #if CONFIG_TXK_SEL av1_write_tx_type(cm, xd, blk_row, blk_col, block, plane, @@ -372,7 +381,7 @@ if (level <= i) continue; - ctx = get_base_ctx(levels, scan[c], bwl, height, i + 1); + ctx = get_base_ctx(levels, scan[c], bwl, i + 1); if (level == i + 1) { aom_write_bin(w, 1, ec_ctx->coeff_base_cdf[txs_ctx][plane_type][i][ctx], @@ -415,7 +424,7 @@ if (level <= NUM_BASE_LEVELS) continue; // level is above 1. - ctx = get_br_ctx(levels, scan[c], bwl, height); + ctx = get_br_ctx(levels, scan[c], bwl); int base_range = level - 1 - NUM_BASE_LEVELS; int br_set_idx = 0; @@ -487,7 +496,7 @@ static INLINE void get_base_ctx_set(const uint8_t *const levels, const int c, // raster order - const int bwl, const int height, + const int bwl, int ctx_set[NUM_BASE_LEVELS]) { const int row = c >> bwl; const int col = c - (row << bwl); @@ -502,8 +511,7 @@ int ref_col = col + base_ref_offset[idx][1]; int pos = (ref_row << bwl) + ref_col; - if (ref_row < 0 || ref_col < 0 || ref_row >= height || ref_col >= stride) - continue; + if (ref_col < 0 || ref_col >= stride) continue; const uint8_t abs_coeff = levels[pos]; @@ -567,11 +575,13 @@ int txb_skip_ctx = txb_ctx->txb_skip_ctx; const int bwl = b_width_log2_lookup[txsize_to_bsize[tx_size]] + 2; + const int width = tx_size_wide[tx_size]; const int height = tx_size_high[tx_size]; const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, mbmi); const int16_t *scan = scan_order->scan; - uint8_t levels[MAX_TX_SQUARE]; + uint8_t levels_buf[TX_PAD_2D]; + uint8_t *const levels = set_levels(levels_buf, width); LV_MAP_COEFF_COST *coeff_costs = &x->coeff_costs[txs_ctx][plane_type]; @@ -583,7 +593,7 @@ } cost = coeff_costs->txb_skip_cost[txb_skip_ctx][0]; - av1_txb_init_levels(qcoeff, tx_size_2d[tx_size], levels); + av1_txb_init_levels(qcoeff, width, tx_size_2d[tx_size], levels); #if CONFIG_TXK_SEL cost += av1_tx_type_cost(cm, x, xd, mbmi->sb_type, plane, tx_size, tx_type); @@ -634,7 +644,7 @@ if (is_k == 0) break; } #else - get_base_ctx_set(levels, scan[c], bwl, height, ctx_ls); + get_base_ctx_set(levels, scan[c], bwl, ctx_ls); int i; for (i = 0; i < NUM_BASE_LEVELS; ++i) { @@ -650,7 +660,7 @@ if (level > NUM_BASE_LEVELS) { int ctx; - ctx = get_br_ctx(levels, scan[c], bwl, height); + ctx = get_br_ctx(levels, scan[c], bwl); int base_range = level - 1 - NUM_BASE_LEVELS; if (base_range < COEFF_BASE_RANGE) { cost += coeff_costs->lps_cost[ctx][base_range]; @@ -732,8 +742,8 @@ const int row = coeff_idx >> bwl; const int col = coeff_idx - (row << bwl); - txb_cache->nz_count_arr[coeff_idx] = get_nz_count( - levels, bwl, height, row, col, get_tx_class(txb_info->tx_type)); + txb_cache->nz_count_arr[coeff_idx] = + get_nz_count(levels, bwl, row, col, get_tx_class(txb_info->tx_type)); const int nz_count = txb_cache->nz_count_arr[coeff_idx]; txb_cache->nz_ctx_arr[coeff_idx] = @@ -1433,8 +1443,7 @@ } #else int ctx_ls[NUM_BASE_LEVELS] = { 0 }; - get_base_ctx_set(txb_info->levels, scan[scan_idx], txb_info->bwl, - txb_info->height, ctx_ls); + get_base_ctx_set(txb_info->levels, scan[scan_idx], txb_info->bwl, ctx_ls); int i; for (i = 0; i < NUM_BASE_LEVELS; ++i) { @@ -1444,8 +1453,7 @@ #endif if (abs_qc > NUM_BASE_LEVELS) { - int ctx = get_br_ctx(txb_info->levels, scan[scan_idx], txb_info->bwl, - txb_info->height); + int ctx = get_br_ctx(txb_info->levels, scan[scan_idx], txb_info->bwl); cost += get_br_cost(abs_qc, ctx, txb_costs->lps_cost[ctx]); cost += get_golomb_cost(abs_qc); } @@ -1649,12 +1657,15 @@ tran_low_t tmp_qcoeff[MAX_TX_SQUARE]; tran_low_t tmp_dqcoeff[MAX_TX_SQUARE]; - uint8_t tmp_levels[MAX_TX_SQUARE]; + uint8_t tmp_levels_buf[TX_PAD_2D]; + uint8_t *const tmp_levels = set_levels(tmp_levels_buf, txb_info->stride); const int org_eob = txb_info->eob; if (dry_run) { + const int stride = txb_info->stride + TX_PAD_HOR; memcpy(tmp_qcoeff, org_qcoeff, sizeof(org_qcoeff[0]) * max_eob); memcpy(tmp_dqcoeff, org_dqcoeff, sizeof(org_dqcoeff[0]) * max_eob); - memcpy(tmp_levels, org_levels, sizeof(org_levels[0]) * max_eob); + memcpy(tmp_levels, org_levels - TX_PAD_TOP * stride, + sizeof(org_levels[0]) * stride * (txb_info->height + TX_PAD_VER)); txb_info->qcoeff = tmp_qcoeff; txb_info->dqcoeff = tmp_dqcoeff; txb_info->levels = tmp_levels; @@ -1788,12 +1799,15 @@ tran_low_t tmp_qcoeff[MAX_TX_SQUARE]; tran_low_t tmp_dqcoeff[MAX_TX_SQUARE]; - uint8_t tmp_levels[MAX_TX_SQUARE]; + uint8_t tmp_levels_buf[TX_PAD_2D]; + uint8_t *const tmp_levels = set_levels(tmp_levels_buf, txb_info->stride); const int org_eob = txb_info->eob; if (dry_run) { + const int stride = txb_info->stride + TX_PAD_HOR; memcpy(tmp_qcoeff, org_qcoeff, sizeof(org_qcoeff[0]) * max_eob); memcpy(tmp_dqcoeff, org_dqcoeff, sizeof(org_dqcoeff[0]) * max_eob); - memcpy(tmp_levels, org_levels, sizeof(org_levels[0]) * max_eob); + memcpy(tmp_levels, org_levels - TX_PAD_TOP * stride, + sizeof(org_levels[0]) * stride * (txb_info->height + TX_PAD_VER)); txb_info->qcoeff = tmp_qcoeff; txb_info->dqcoeff = tmp_dqcoeff; txb_info->levels = tmp_levels; @@ -1897,6 +1911,7 @@ const int seg_eob = tx_size_2d[tx_size]; const int bwl = b_width_log2_lookup[txsize_to_bsize[tx_size]] + 2; const int stride = 1 << bwl; + const int width = tx_size_wide[tx_size]; const int height = tx_size_high[tx_size]; const int is_inter = is_inter_block(mbmi); const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, mbmi); @@ -1905,7 +1920,8 @@ const int shift = av1_get_tx_scale(tx_size); const int64_t rdmult = (x->rdmult * plane_rd_mult[is_inter][plane_type] + 2) >> 2; - uint8_t levels[MAX_TX_SQUARE]; + uint8_t levels_buf[TX_PAD_2D]; + uint8_t *const levels = set_levels(levels_buf, width); TxbInfo txb_info = { qcoeff, levels, dqcoeff, tcoeff, dequant, shift, @@ -1913,7 +1929,7 @@ eob, seg_eob, scan_order, txb_ctx, rdmult, &cm->coeff_ctx_table }; - av1_txb_init_levels(qcoeff, tx_size_2d[tx_size], levels); + av1_txb_init_levels(qcoeff, width, tx_size_2d[tx_size], levels); const int update = optimize_txb(&txb_info, &txb_costs, NULL, 0, fast_mode); @@ -1988,8 +2004,10 @@ get_txb_ctx(plane_bsize, tx_size, plane, pd->above_context + blk_col, pd->left_context + blk_row, &txb_ctx); const int bwl = b_width_log2_lookup[txsize_to_bsize[tx_size]] + 2; + const int width = tx_size_wide[tx_size]; const int height = tx_size_high[tx_size]; - uint8_t levels[MAX_TX_SQUARE]; + uint8_t levels_buf[TX_PAD_2D]; + uint8_t *const levels = set_levels(levels_buf, width); const uint8_t allow_update_cdf = args->allow_update_cdf; TX_SIZE txsize_ctx = get_txsize_context(tx_size); @@ -2010,7 +2028,7 @@ return; } - av1_txb_init_levels(tcoeff, tx_size_2d[tx_size], levels); + av1_txb_init_levels(tcoeff, width, tx_size_2d[tx_size], levels); #if CONFIG_TXK_SEL av1_update_tx_type_count(cm, xd, blk_row, blk_col, block, plane, @@ -2073,7 +2091,7 @@ if (level <= i) continue; - ctx = get_base_ctx(levels, scan[c], bwl, height, i + 1); + ctx = get_base_ctx(levels, scan[c], bwl, i + 1); if (level == i + 1) { ++td->counts->coeff_base[txsize_ctx][plane_type][i][ctx][1]; @@ -2112,7 +2130,7 @@ if (level <= NUM_BASE_LEVELS) continue; // level is above 1. - ctx = get_br_ctx(levels, scan[c], bwl, height); + ctx = get_br_ctx(levels, scan[c], bwl); int base_range = level - 1 - NUM_BASE_LEVELS; int br_set_idx = base_range < COEFF_BASE_RANGE