Make level map coding system support rectangular tx_size This commit makes the level map coding system support the transform coefficients from rectangular transform block sizes. Change-Id: I5cd6c71d12e41938f942adc98cc1e1f286336f12
diff --git a/av1/common/txb_common.h b/av1/common/txb_common.h index 9a51a13..d3c0664 100644 --- a/av1/common/txb_common.h +++ b/av1/common/txb_common.h
@@ -37,14 +37,14 @@ }; static INLINE int get_level_count(const tran_low_t *tcoeffs, int stride, - int row, int col, int level, + int height, int row, int col, int level, int (*nb_offset)[2], int nb_num) { int count = 0; for (int idx = 0; idx < nb_num; ++idx) { const int ref_row = row + nb_offset[idx][0]; const int ref_col = col + nb_offset[idx][1]; const int pos = ref_row * stride + ref_col; - if (ref_row < 0 || ref_col < 0 || ref_row >= stride || ref_col >= stride) + if (ref_row < 0 || ref_col < 0 || ref_row >= height || ref_col >= stride) continue; tran_low_t abs_coeff = abs(tcoeffs[pos]); count += abs_coeff > level; @@ -53,14 +53,15 @@ } static INLINE void get_mag(int *mag, const tran_low_t *tcoeffs, int stride, - int row, int col, int (*nb_offset)[2], int nb_num) { + int height, int row, int col, int (*nb_offset)[2], + int nb_num) { mag[0] = 0; mag[1] = 0; for (int idx = 0; idx < nb_num; ++idx) { const int ref_row = row + nb_offset[idx][0]; const int ref_col = col + nb_offset[idx][1]; const int pos = ref_row * stride + ref_col; - if (ref_row < 0 || ref_col < 0 || ref_row >= stride || ref_col >= stride) + if (ref_row < 0 || ref_col < 0 || ref_row >= height || ref_col >= stride) continue; tran_low_t abs_coeff = abs(tcoeffs[pos]); if (nb_offset[idx][0] >= 0 && nb_offset[idx][1] >= 0) { @@ -74,15 +75,16 @@ } } static INLINE int get_level_count_mag(int *mag, const tran_low_t *tcoeffs, - int stride, int row, int col, int level, - int (*nb_offset)[2], int nb_num) { + int stride, int height, int row, int col, + int level, int (*nb_offset)[2], + int nb_num) { int count = 0; *mag = 0; for (int idx = 0; idx < nb_num; ++idx) { const int ref_row = row + nb_offset[idx][0]; const int ref_col = col + nb_offset[idx][1]; const int pos = ref_row * stride + ref_col; - if (ref_row < 0 || ref_col < 0 || ref_row >= stride || ref_col >= stride) + if (ref_row < 0 || ref_col < 0 || ref_row >= height || ref_col >= stride) continue; tran_low_t abs_coeff = abs(tcoeffs[pos]); count += abs_coeff > level; @@ -115,15 +117,16 @@ static INLINE int get_base_ctx(const tran_low_t *tcoeffs, int c, // raster order - const int bwl, const int level) { + const int bwl, const int height, + const int level) { const int stride = 1 << bwl; const int row = c >> bwl; const int col = c - (row << bwl); const int level_minus_1 = level - 1; int mag; - int count = - get_level_count_mag(&mag, tcoeffs, stride, row, col, level_minus_1, - base_ref_offset, BASE_CONTEXT_POSITION_NUM); + int count = get_level_count_mag(&mag, tcoeffs, stride, height, row, col, + level_minus_1, base_ref_offset, + BASE_CONTEXT_POSITION_NUM); int ctx_idx = get_base_ctx_from_count_mag(row, col, count, mag, level); return ctx_idx; } @@ -173,15 +176,15 @@ static INLINE int get_br_ctx(const tran_low_t *tcoeffs, const int c, // raster order - const int bwl) { + const int bwl, const int height) { const int stride = 1 << bwl; const int row = c >> bwl; const int col = c - (row << bwl); const int level_minus_1 = NUM_BASE_LEVELS; int mag; - const int count = - get_level_count_mag(&mag, tcoeffs, stride, row, col, level_minus_1, - br_ref_offset, BR_CONTEXT_POSITION_NUM); + const int count = get_level_count_mag(&mag, tcoeffs, stride, height, row, col, + level_minus_1, br_ref_offset, + BR_CONTEXT_POSITION_NUM); const int ctx = get_br_ctx_from_count_mag(row, col, count, mag); return ctx; } @@ -195,7 +198,7 @@ static INLINE int get_nz_map_ctx(const tran_low_t *tcoeffs, const uint8_t *txb_mask, const int coeff_idx, // raster order - const int bwl) { + const int bwl, const int height) { const int row = coeff_idx >> bwl; const int col = coeff_idx - (row << bwl); int ctx = 0; @@ -228,7 +231,7 @@ int ref_col = col + sig_ref_offset[idx][1]; int pos; - if (ref_row < 0 || ref_col < 0 || ref_row >= stride || ref_col >= stride) + if (ref_row < 0 || ref_col < 0 || ref_row >= height || ref_col >= stride) continue; pos = (ref_row << bwl) + ref_col;
diff --git a/av1/decoder/decodetxb.c b/av1/decoder/decodetxb.c index 74693b5..98e4db1 100644 --- a/av1/decoder/decodetxb.c +++ b/av1/decoder/decodetxb.c
@@ -58,6 +58,7 @@ const int16_t *const dequant = xd->plane[plane].seg_dequant[mbmi->segment_id]; const int shift = av1_get_tx_scale(tx_size); const int bwl = b_width_log2_lookup[txsize_to_bsize[tx_size]] + 2; + const int height = tx_size_high[tx_size]; int cul_level = 0; unsigned int(*nz_map_count)[SIG_COEF_CONTEXTS][2]; uint8_t txb_mask[32 * 32] = { 0 }; @@ -87,7 +88,7 @@ for (c = 0; c < seg_eob; ++c) { int is_nz; - int coeff_ctx = get_nz_map_ctx(tcoeffs, txb_mask, scan[c], bwl); + int coeff_ctx = get_nz_map_ctx(tcoeffs, txb_mask, scan[c], bwl, height); int eob_ctx = get_eob_ctx(tcoeffs, scan[c], txs_ctx); if (c < seg_eob - 1) @@ -128,7 +129,7 @@ if (*v <= i) continue; - ctx = get_base_ctx(tcoeffs, scan[c], bwl, i + 1); + ctx = get_base_ctx(tcoeffs, scan[c], bwl, height, i + 1); if (aom_read(r, coeff_base[ctx], tx_size)) { *v = i + 1; @@ -170,7 +171,7 @@ sign = aom_read_bit(r, ACCT_STR); } - ctx = get_br_ctx(tcoeffs, scan[c], bwl); + ctx = get_br_ctx(tcoeffs, scan[c], bwl, height); if (cm->fc->coeff_lps[txs_ctx][plane_type][ctx] == 0) exit(0);
diff --git a/av1/encoder/encodetxb.c b/av1/encoder/encodetxb.c index c21b2d0..1fd462a 100644 --- a/av1/encoder/encodetxb.c +++ b/av1/encoder/encodetxb.c
@@ -86,6 +86,7 @@ int c; int is_nz; const int bwl = b_width_log2_lookup[txsize_to_bsize[tx_size]] + 2; + const int height = tx_size_high[tx_size]; const int seg_eob = tx_size_2d[tx_size]; uint8_t txb_mask[32 * 32] = { 0 }; uint16_t update_eob = 0; @@ -101,7 +102,7 @@ eob_flag = cm->fc->eob_flag[txs_ctx][plane_type]; for (c = 0; c < eob; ++c) { - int coeff_ctx = get_nz_map_ctx(tcoeff, txb_mask, scan[c], bwl); + int coeff_ctx = get_nz_map_ctx(tcoeff, txb_mask, scan[c], bwl, height); int eob_ctx = get_eob_ctx(tcoeff, scan[c], txs_ctx); tran_low_t v = tcoeff[scan[c]]; @@ -130,7 +131,7 @@ if (level <= i) continue; - ctx = get_base_ctx(tcoeff, scan[c], bwl, i + 1); + ctx = get_base_ctx(tcoeff, scan[c], bwl, height, i + 1); if (level == i + 1) { aom_write(w, 1, coeff_base[ctx]); @@ -162,7 +163,7 @@ } // level is above 1. - ctx = get_br_ctx(tcoeff, scan[c], bwl); + ctx = get_br_ctx(tcoeff, scan[c], bwl, height); for (idx = 0; idx < COEFF_BASE_RANGE; ++idx) { if (level == (idx + 1 + NUM_BASE_LEVELS)) { aom_write(w, 1, cm->fc->coeff_lps[txs_ctx][plane_type][ctx]); @@ -217,7 +218,7 @@ static INLINE void get_base_ctx_set(const tran_low_t *tcoeffs, int c, // raster order - const int bwl, + const int bwl, const int height, int ctx_set[NUM_BASE_LEVELS]) { const int row = c >> bwl; const int col = c - (row << bwl); @@ -232,7 +233,7 @@ int ref_col = col + base_ref_offset[idx][1]; int pos = (ref_row << bwl) + ref_col; - if (ref_row < 0 || ref_col < 0 || ref_row >= stride || ref_col >= stride) + if (ref_row < 0 || ref_col < 0 || ref_row >= height || ref_col >= stride) continue; abs_coeff = abs(tcoeffs[pos]); @@ -303,6 +304,8 @@ aom_prob *nz_map = xd->fc->nz_map[txs_ctx][plane_type]; const int bwl = b_width_log2_lookup[txsize_to_bsize[tx_size]] + 2; + const int height = tx_size_high[tx_size]; + // txb_mask is only initialized for once here. After that, it will be set when // coding zero map and then reset when coding level 1 info. uint8_t txb_mask[32 * 32] = { 0 }; @@ -332,7 +335,7 @@ int level = abs(v); if (c < seg_eob) { - int coeff_ctx = get_nz_map_ctx(qcoeff, txb_mask, scan[c], bwl); + int coeff_ctx = get_nz_map_ctx(qcoeff, txb_mask, scan[c], bwl, height); cost += av1_cost_bit(nz_map[coeff_ctx], is_nz); } @@ -349,7 +352,7 @@ cost += av1_cost_bit(128, sign); } - get_base_ctx_set(qcoeff, scan[c], bwl, ctx_ls); + get_base_ctx_set(qcoeff, scan[c], bwl, height, ctx_ls); int i; for (i = 0; i < NUM_BASE_LEVELS; ++i) { @@ -366,7 +369,7 @@ int idx; int ctx; - ctx = get_br_ctx(qcoeff, scan[c], bwl); + ctx = get_br_ctx(qcoeff, scan[c], bwl, height); for (idx = 0; idx < COEFF_BASE_RANGE; ++idx) { if (level == (idx + 1 + NUM_BASE_LEVELS)) { @@ -416,20 +419,20 @@ static void gen_base_count_mag_arr(int (*base_count_arr)[MAX_TX_SQUARE], int (*base_mag_arr)[2], const tran_low_t *qcoeff, int stride, - int eob, const int16_t *scan) { + int height, int eob, const int16_t *scan) { for (int c = 0; c < eob; ++c) { const int coeff_idx = scan[c]; // raster order if (!has_base(qcoeff[coeff_idx], 0)) continue; const int row = coeff_idx / stride; const int col = coeff_idx % stride; int *mag = base_mag_arr[coeff_idx]; - get_mag(mag, qcoeff, stride, row, col, base_ref_offset, + get_mag(mag, qcoeff, stride, height, row, col, base_ref_offset, BASE_CONTEXT_POSITION_NUM); for (int i = 0; i < NUM_BASE_LEVELS; ++i) { if (!has_base(qcoeff[coeff_idx], i)) continue; int *count = base_count_arr[i] + coeff_idx; - *count = get_level_count(qcoeff, stride, row, col, i, base_ref_offset, - BASE_CONTEXT_POSITION_NUM); + *count = get_level_count(qcoeff, stride, height, row, col, i, + base_ref_offset, BASE_CONTEXT_POSITION_NUM); } } } @@ -486,8 +489,8 @@ } static void gen_br_count_mag_arr(int *br_count_arr, int (*br_mag_arr)[2], - const tran_low_t *qcoeff, int stride, int eob, - const int16_t *scan) { + const tran_low_t *qcoeff, int stride, + int height, int eob, const int16_t *scan) { for (int c = 0; c < eob; ++c) { const int coeff_idx = scan[c]; // raster order if (!has_br(qcoeff[coeff_idx])) continue; @@ -495,9 +498,9 @@ const int col = coeff_idx % stride; int *count = br_count_arr + coeff_idx; int *mag = br_mag_arr[coeff_idx]; - *count = get_level_count(qcoeff, stride, row, col, NUM_BASE_LEVELS, + *count = get_level_count(qcoeff, stride, height, row, col, NUM_BASE_LEVELS, br_ref_offset, BR_CONTEXT_POSITION_NUM); - get_mag(mag, qcoeff, stride, row, col, br_ref_offset, + get_mag(mag, qcoeff, stride, height, row, col, br_ref_offset, BR_CONTEXT_POSITION_NUM); } } @@ -556,13 +559,14 @@ txb_info->qcoeff, txb_info->bwl, txb_info->eob, txb_info->scan_order); gen_base_count_mag_arr(txb_cache->base_count_arr, txb_cache->base_mag_arr, - txb_info->qcoeff, txb_info->stride, txb_info->eob, - scan); + txb_info->qcoeff, txb_info->stride, txb_info->height, + txb_info->eob, scan); gen_base_ctx_arr(txb_cache->base_ctx_arr, txb_cache->base_count_arr, txb_cache->base_mag_arr, txb_info->qcoeff, txb_info->stride, txb_info->eob, scan); gen_br_count_mag_arr(txb_cache->br_count_arr, txb_cache->br_mag_arr, - txb_info->qcoeff, txb_info->stride, txb_info->eob, scan); + txb_info->qcoeff, txb_info->stride, txb_info->height, + txb_info->eob, scan); gen_br_ctx_arr(txb_cache->br_ctx_arr, txb_cache->br_count_arr, txb_cache->br_mag_arr, txb_info->qcoeff, txb_info->stride, txb_info->eob, scan); @@ -1131,7 +1135,8 @@ txb_ctx->dc_sign_ctx); int ctx_ls[NUM_BASE_LEVELS] = { 0 }; - get_base_ctx_set(txb_info->qcoeff, scan[scan_idx], txb_info->bwl, ctx_ls); + get_base_ctx_set(txb_info->qcoeff, scan[scan_idx], txb_info->bwl, + txb_info->height, ctx_ls); int i; for (i = 0; i < NUM_BASE_LEVELS; ++i) { @@ -1139,7 +1144,8 @@ } if (abs_qc > NUM_BASE_LEVELS) { - int ctx = get_br_ctx(txb_info->qcoeff, scan[scan_idx], txb_info->bwl); + int ctx = get_br_ctx(txb_info->qcoeff, scan[scan_idx], txb_info->bwl, + txb_info->height); cost += get_br_cost(abs_qc, ctx, txb_probs->coeff_lps); cost += get_golomb_cost(abs_qc); } @@ -1557,6 +1563,7 @@ get_txb_ctx(plane_bsize, tx_size, plane, pd->above_context + blk_col, pd->left_context + blk_row, &txb_ctx); const int bwl = b_width_log2_lookup[txsize_to_bsize[tx_size]] + 2; + const int height = tx_size_high[tx_size]; int cul_level = 0; unsigned int(*nz_map_count)[SIG_COEF_CONTEXTS][2]; uint8_t txb_mask[32 * 32] = { 0 }; @@ -1585,7 +1592,7 @@ for (c = 0; c < eob; ++c) { tran_low_t v = qcoeff[scan[c]]; int is_nz = (v != 0); - int coeff_ctx = get_nz_map_ctx(tcoeff, txb_mask, scan[c], bwl); + int coeff_ctx = get_nz_map_ctx(tcoeff, txb_mask, scan[c], bwl, height); int eob_ctx = get_eob_ctx(tcoeff, scan[c], txsize_ctx); if (c == seg_eob - 1) break; @@ -1608,7 +1615,7 @@ if (level <= i) continue; - ctx = get_base_ctx(tcoeff, scan[c], bwl, i + 1); + ctx = get_base_ctx(tcoeff, scan[c], bwl, height, i + 1); if (level == i + 1) { ++td->counts->coeff_base[txsize_ctx][plane_type][i][ctx][1]; @@ -1643,7 +1650,7 @@ } // level is above 1. - ctx = get_br_ctx(tcoeff, scan[c], bwl); + ctx = get_br_ctx(tcoeff, scan[c], bwl, height); for (idx = 0; idx < COEFF_BASE_RANGE; ++idx) { if (level == (idx + 1 + NUM_BASE_LEVELS)) { ++td->counts->coeff_lps[txsize_ctx][plane_type][ctx][1];