Rework loop filter tx size selection Update and capture the effective transform block size per color plane. Change-Id: Ib6e0e7abb3973db6b8d511ee7c9948aaab048788
diff --git a/av1/common/alloccommon.c b/av1/common/alloccommon.c index c1b596b..80f6b09 100644 --- a/av1/common/alloccommon.c +++ b/av1/common/alloccommon.c
@@ -125,6 +125,11 @@ #if CONFIG_VAR_TX aom_free(cm->above_txfm_context); cm->above_txfm_context = NULL; + + for (i = 0; i < MAX_MB_PLANE; ++i) { + aom_free(cm->top_txfm_context[i]); + cm->top_txfm_context[i] = NULL; + } #endif } @@ -170,6 +175,14 @@ cm->above_txfm_context = (TXFM_CONTEXT *)aom_calloc( aligned_mi_cols << TX_UNIT_WIDE_LOG2, sizeof(*cm->above_txfm_context)); if (!cm->above_txfm_context) goto fail; + + for (i = 0; i < MAX_MB_PLANE; ++i) { + aom_free(cm->top_txfm_context[i]); + cm->top_txfm_context[i] = + (TXFM_CONTEXT *)aom_calloc(aligned_mi_cols << TX_UNIT_WIDE_LOG2, + sizeof(*cm->top_txfm_context[0])); + if (!cm->top_txfm_context[i]) goto fail; + } #endif cm->above_context_alloc_cols = aligned_mi_cols;
diff --git a/av1/common/av1_loopfilter.c b/av1/common/av1_loopfilter.c index f40de33..1f0cb3b 100644 --- a/av1/common/av1_loopfilter.c +++ b/av1/common/av1_loopfilter.c
@@ -1363,7 +1363,7 @@ // the non420 case). // Note: 'row_masks_ptr' and/or 'col_masks_ptr' can be passed NULL. static void get_filter_level_and_masks_non420( - AV1_COMMON *const cm, const struct macroblockd_plane *const plane, + AV1_COMMON *const cm, const struct macroblockd_plane *const plane, int pl, MODE_INFO **mib, int mi_row, int mi_col, int idx_r, uint8_t *const lfl_r, unsigned int *const mask_4x4_int_r_ptr, unsigned int *const mask_4x4_int_c_ptr, FilterMasks *const row_masks_ptr, @@ -1400,10 +1400,6 @@ (num_4x4_blocks_high_lookup[sb_type] > 1) ? !blk_row : 1; const int skip_this_r = skip_this && !block_edge_above; -#if CONFIG_VAR_TX - const TX_SIZE mb_tx_size = mbmi->inter_tx_size[blk_row][blk_col]; -#endif - TX_SIZE tx_size = (plane->plane_type == PLANE_TYPE_UV) ? get_uv_tx_size(mbmi, plane) : mbmi->tx_size; @@ -1420,8 +1416,15 @@ #if CONFIG_VAR_TX if (is_inter_block(mbmi) && !mbmi->skip) { + const int tx_row_idx = + (blk_row * mi_size_high[BLOCK_8X8] << TX_UNIT_HIGH_LOG2) >> 1; + const int tx_col_idx = + (blk_col * mi_size_wide[BLOCK_8X8] << TX_UNIT_WIDE_LOG2) >> 1; + const BLOCK_SIZE bsize = + AOMMAX(BLOCK_4X4, get_plane_block_size(mbmi->sb_type, plane)); + const TX_SIZE mb_tx_size = mbmi->inter_tx_size[tx_row_idx][tx_col_idx]; tx_size = (plane->plane_type == PLANE_TYPE_UV) - ? uv_txsize_lookup[sb_type][mb_tx_size][ss_x][ss_y] + ? uv_txsize_lookup[bsize][mb_tx_size][0][0] : mb_tx_size; } #endif @@ -1434,18 +1437,29 @@ #endif #if CONFIG_VAR_TX - TX_SIZE tx_size_r = AOMMIN( - tx_size, cm->above_txfm_context[(mi_col + c) << TX_UNIT_WIDE_LOG2]); - TX_SIZE tx_size_c = - AOMMIN(tx_size, cm->left_txfm_context[((mi_row + r) & MAX_MIB_MASK) - << TX_UNIT_HIGH_LOG2]); + TX_SIZE tx_size_r, tx_size_c; - cm->above_txfm_context[(mi_col + c) << TX_UNIT_WIDE_LOG2] = tx_size; - cm->left_txfm_context[((mi_row + r) & MAX_MIB_MASK) << TX_UNIT_HIGH_LOG2] = - tx_size; + const int tx_wide = + AOMMIN(tx_size_wide[tx_size], + tx_size_wide[cm->top_txfm_context[pl][(mi_col + idx_c) + << TX_UNIT_WIDE_LOG2]]); + const int tx_high = AOMMIN( + tx_size_high[tx_size], + tx_size_high[cm->left_txfm_context[pl][((mi_row + idx_r) & MAX_MIB_MASK) + << TX_UNIT_HIGH_LOG2]]); + + tx_size_c = get_sqr_tx_size(tx_wide); + tx_size_r = get_sqr_tx_size(tx_high); + + memset(cm->top_txfm_context[pl] + ((mi_col + idx_c) << TX_UNIT_WIDE_LOG2), + tx_size, mi_size_wide[BLOCK_8X8] << TX_UNIT_WIDE_LOG2); + memset(cm->left_txfm_context[pl] + + (((mi_row + idx_r) & MAX_MIB_MASK) << TX_UNIT_HIGH_LOG2), + tx_size, mi_size_high[BLOCK_8X8] << TX_UNIT_HIGH_LOG2); #else TX_SIZE tx_size_c = txsize_horz_map[tx_size]; TX_SIZE tx_size_r = txsize_vert_map[tx_size]; + (void)pl; #endif // CONFIG_VAR_TX if (tx_size_c == TX_32X32) @@ -1530,8 +1544,8 @@ void av1_filter_block_plane_non420_ver(AV1_COMMON *const cm, struct macroblockd_plane *plane, - MODE_INFO **mib, int mi_row, - int mi_col) { + MODE_INFO **mib, int mi_row, int mi_col, + int pl) { const int ss_y = plane->subsampling_y; const int row_step = mi_size_high[BLOCK_8X8] << ss_y; struct buf_2d *const dst = &plane->dst; @@ -1544,7 +1558,7 @@ unsigned int mask_4x4_int; FilterMasks col_masks; const int r = idx_r >> mi_height_log2_lookup[BLOCK_8X8]; - get_filter_level_and_masks_non420(cm, plane, mib, mi_row, mi_col, idx_r, + get_filter_level_and_masks_non420(cm, plane, pl, mib, mi_row, mi_col, idx_r, &lfl[r][0], NULL, &mask_4x4_int, NULL, &col_masks); @@ -1579,8 +1593,8 @@ void av1_filter_block_plane_non420_hor(AV1_COMMON *const cm, struct macroblockd_plane *plane, - MODE_INFO **mib, int mi_row, - int mi_col) { + MODE_INFO **mib, int mi_row, int mi_col, + int pl) { const int ss_y = plane->subsampling_y; const int row_step = mi_size_high[BLOCK_8X8] << ss_y; struct buf_2d *const dst = &plane->dst; @@ -1592,7 +1606,7 @@ for (idx_r = 0; idx_r < cm->mib_size && mi_row + idx_r < cm->mi_rows; idx_r += row_step) { const int r = idx_r >> mi_height_log2_lookup[BLOCK_8X8]; - get_filter_level_and_masks_non420(cm, plane, mib, mi_row, mi_col, idx_r, + get_filter_level_and_masks_non420(cm, plane, pl, mib, mi_row, mi_col, idx_r, &lfl[r][0], mask_4x4_int + r, NULL, row_masks_array + r, NULL); } @@ -2192,12 +2206,15 @@ int mi_row, mi_col; #if CONFIG_VAR_TX - memset(cm->above_txfm_context, TX_SIZES, cm->mi_cols << TX_UNIT_WIDE_LOG2); + for (int i = 0; i < MAX_MB_PLANE; ++i) + memset(cm->top_txfm_context[i], TX_32X32, cm->mi_cols << TX_UNIT_WIDE_LOG2); #endif // CONFIG_VAR_TX for (mi_row = start; mi_row < stop; mi_row += cm->mib_size) { MODE_INFO **mi = cm->mi_grid_visible + mi_row * cm->mi_stride; #if CONFIG_VAR_TX - memset(cm->left_txfm_context, TX_SIZES, MAX_MIB_SIZE << TX_UNIT_WIDE_LOG2); + for (int i = 0; i < MAX_MB_PLANE; ++i) + memset(cm->left_txfm_context[i], TX_32X32, MAX_MIB_SIZE + << TX_UNIT_WIDE_LOG2); #endif // CONFIG_VAR_TX for (mi_col = 0; mi_col < cm->mi_cols; mi_col += cm->mib_size) { int plane; @@ -2206,9 +2223,9 @@ for (plane = 0; plane < num_planes; ++plane) { av1_filter_block_plane_non420_ver(cm, &planes[plane], mi + mi_col, - mi_row, mi_col); + mi_row, mi_col, plane); av1_filter_block_plane_non420_hor(cm, &planes[plane], mi + mi_col, - mi_row, mi_col); + mi_row, mi_col, plane); } } } @@ -2282,9 +2299,9 @@ break; case LF_PATH_SLOW: av1_filter_block_plane_non420_ver(cm, &planes[plane], mi + mi_col, - mi_row, mi_col); + mi_row, mi_col, plane); av1_filter_block_plane_non420_hor(cm, &planes[plane], mi + mi_col, - mi_row, mi_col); + mi_row, mi_col, plane); break; }
diff --git a/av1/common/av1_loopfilter.h b/av1/common/av1_loopfilter.h index 8ac5d99..424df24 100644 --- a/av1/common/av1_loopfilter.h +++ b/av1/common/av1_loopfilter.h
@@ -115,11 +115,11 @@ void av1_filter_block_plane_non420_ver(struct AV1Common *const cm, struct macroblockd_plane *plane, MODE_INFO **mi_8x8, int mi_row, - int mi_col); + int mi_col, int pl); void av1_filter_block_plane_non420_hor(struct AV1Common *const cm, struct macroblockd_plane *plane, MODE_INFO **mi_8x8, int mi_row, - int mi_col); + int mi_col, int pl); void av1_loop_filter_init(struct AV1Common *cm);
diff --git a/av1/common/onyxc_int.h b/av1/common/onyxc_int.h index 795af30..38750c4 100644 --- a/av1/common/onyxc_int.h +++ b/av1/common/onyxc_int.h
@@ -386,7 +386,8 @@ ENTROPY_CONTEXT *above_context[MAX_MB_PLANE]; #if CONFIG_VAR_TX TXFM_CONTEXT *above_txfm_context; - TXFM_CONTEXT left_txfm_context[2 * MAX_MIB_SIZE]; + TXFM_CONTEXT *top_txfm_context[MAX_MB_PLANE]; + TXFM_CONTEXT left_txfm_context[MAX_MB_PLANE][2 * MAX_MIB_SIZE]; #endif int above_context_alloc_cols; @@ -928,6 +929,21 @@ for (i = 0; i < bw; ++i) above_ctx[i] = txw; } +static INLINE TX_SIZE get_sqr_tx_size(int tx_dim) { + TX_SIZE tx_size; + switch (tx_dim) { +#if CONFIG_EXT_PARTITION + case 128: +#endif + case 64: + case 32: tx_size = TX_32X32; break; + case 16: tx_size = TX_16X16; break; + case 8: tx_size = TX_8X8; break; + default: tx_size = TX_4X4; + } + return tx_size; +} + static INLINE int txfm_partition_context(TXFM_CONTEXT *above_ctx, TXFM_CONTEXT *left_ctx, BLOCK_SIZE bsize, TX_SIZE tx_size) { @@ -935,22 +951,13 @@ const uint8_t txh = tx_size_high[tx_size]; const int above = *above_ctx < txw; const int left = *left_ctx < txh; - TX_SIZE max_tx_size = max_txsize_lookup[bsize]; int category = TXFM_PARTITION_CONTEXTS - 1; // dummy return, not used by others. if (tx_size <= TX_4X4) return 0; - switch (AOMMAX(block_size_wide[bsize], block_size_high[bsize])) { -#if CONFIG_EXT_PARTITION - case 128: -#endif - case 64: - case 32: max_tx_size = TX_32X32; break; - case 16: max_tx_size = TX_16X16; break; - case 8: max_tx_size = TX_8X8; break; - default: assert(max_tx_size == max_txsize_lookup[bsize]); assert(0); - } + TX_SIZE max_tx_size = + get_sqr_tx_size(AOMMAX(block_size_wide[bsize], block_size_high[bsize])); if (max_tx_size >= TX_8X8) { category = (tx_size != max_tx_size && max_tx_size > TX_8X8) +
diff --git a/av1/common/thread_common.c b/av1/common/thread_common.c index ca8b1b3..d96a71a 100644 --- a/av1/common/thread_common.c +++ b/av1/common/thread_common.c
@@ -113,7 +113,7 @@ break; case LF_PATH_SLOW: av1_filter_block_plane_non420_ver(cm, &planes[plane], mi, mi_row, - mi_col); + mi_col, plane); break; } } @@ -135,7 +135,7 @@ break; case LF_PATH_SLOW: av1_filter_block_plane_non420_hor(cm, &planes[plane], mi, mi_row, - mi_col); + mi_col, plane); break; } } @@ -168,7 +168,7 @@ #if CONFIG_EXT_PARTITION_TYPES for (plane = 0; plane < num_planes; ++plane) av1_filter_block_plane_non420_ver(lf_data->cm, &lf_data->planes[plane], - mi + mi_col, mi_row, mi_col); + mi + mi_col, mi_row, mi_col, plane); #else for (plane = 0; plane < num_planes; ++plane) @@ -213,7 +213,7 @@ #if CONFIG_EXT_PARTITION_TYPES for (plane = 0; plane < num_planes; ++plane) av1_filter_block_plane_non420_hor(lf_data->cm, &lf_data->planes[plane], - mi + mi_col, mi_row, mi_col); + mi + mi_col, mi_row, mi_col, plane); #else for (plane = 0; plane < num_planes; ++plane) loop_filter_block_plane_hor(lf_data->cm, lf_data->planes, plane, @@ -263,9 +263,9 @@ #if CONFIG_EXT_PARTITION_TYPES for (plane = 0; plane < num_planes; ++plane) { av1_filter_block_plane_non420_ver(lf_data->cm, &lf_data->planes[plane], - mi + mi_col, mi_row, mi_col); + mi + mi_col, mi_row, mi_col, plane); av1_filter_block_plane_non420_hor(lf_data->cm, &lf_data->planes[plane], - mi + mi_col, mi_row, mi_col); + mi + mi_col, mi_row, mi_col, plane); } #else av1_setup_mask(lf_data->cm, mi_row, mi_col, mi + mi_col,