Fix u/v plane filter level bugs (1). To get filter level for u/v plane, we need to access the odd position of mbmi. (2). Separate u/v vertical and horizontal filter levels. Store them separately since we need to update them in building bitmasks. Change-Id: I1df97d1f06afaac991cfb4b716278427a496d56d
diff --git a/av1/common/av1_loopfilter.c b/av1/common/av1_loopfilter.c index d1659bc..86e7ae9 100644 --- a/av1/common/av1_loopfilter.c +++ b/av1/common/av1_loopfilter.c
@@ -763,9 +763,11 @@ else lfm->lfl_y_hor[row][col] = level; } else if (plane == 1) { - lfm->lfl_u[row][col] = level; + lfm->lfl_u_ver[row][col] = level; + lfm->lfl_u_hor[row][col] = level; } else { - lfm->lfl_v[row][col] = level; + lfm->lfl_v_ver[row][col] = level; + lfm->lfl_v_hor[row][col] = level; } } } @@ -1053,11 +1055,13 @@ } else if (plane == 1) { av1_zero(lfm->left_u); av1_zero(lfm->above_u); - av1_zero(lfm->lfl_u); + av1_zero(lfm->lfl_u_ver); + av1_zero(lfm->lfl_u_hor); } else { av1_zero(lfm->left_v); av1_zero(lfm->above_v); - av1_zero(lfm->lfl_v); + av1_zero(lfm->lfl_v_ver); + av1_zero(lfm->lfl_v_hor); } } } @@ -1469,6 +1473,7 @@ for (int r = 0; (r << MI_SIZE_LOG2) < plane_ptr->dst.height; r += row_step) { const int mi_row = r << subsampling_y; const int row = mi_row % MI_SIZE_64X64; + const int row_uv = row | subsampling_y; int index = 0; const int shift = get_index_shift(0, row, &index); @@ -1482,13 +1487,14 @@ const int x = (c + col_in_unit) << MI_SIZE_LOG2; if (x >= plane_ptr->dst.width) break; const int col = col_in_unit << subsampling_x; + const int col_uv = col | subsampling_x; const uint64_t mask = ((uint64_t)1 << (shift | col)); skip = lfm->skip.bits[index] & mask; is_coding_block_border = lfm->is_vert_border.bits[index] & mask; switch (plane) { - case 0: level = lfm->lfl_y_ver[row][col]; break; - case 1: level = lfm->lfl_u[row][col]; break; - case 2: level = lfm->lfl_v[row][col]; break; + case 0: level = lfm->lfl_y_ver[row_uv][col_uv]; break; + case 1: level = lfm->lfl_u_ver[row_uv][col_uv]; break; + case 2: level = lfm->lfl_v_ver[row_uv][col_uv]; break; default: assert(plane >= 0 && plane <= 2); return; } for (TX_SIZE ts = TX_4X4; ts <= TX_64X64; ++ts) { @@ -1502,9 +1508,7 @@ (!prev_skip || !skip || is_coding_block_border)) { const TX_SIZE min_tx_size = AOMMIN(TX_16X16, AOMMIN(tx_size, prev_tx_size)); - const int tmp_row = (mi_row | subsampling_y) % MI_SIZE_64X64; - const int tmp_col = (col | subsampling_x) % MI_SIZE_64X64; - const int shift_1 = get_index_shift(tmp_col, tmp_row, &index); + const int shift_1 = get_index_shift(col_uv, row_uv, &index); const uint64_t mask_1 = ((uint64_t)1 << shift_1); switch (plane) { case 0: lfm->left_y[min_tx_size].bits[index] |= mask_1; break; @@ -1514,9 +1518,9 @@ } if (level == 0 && prev_level != 0) { switch (plane) { - case 0: lfm->lfl_y_ver[tmp_row][tmp_col] = prev_level; break; - case 1: lfm->lfl_u[tmp_row][tmp_col] = prev_level; break; - case 2: lfm->lfl_v[tmp_row][tmp_col] = prev_level; break; + case 0: lfm->lfl_y_ver[row_uv][col_uv] = prev_level; break; + case 1: lfm->lfl_u_ver[row_uv][col_uv] = prev_level; break; + case 2: lfm->lfl_v_ver[row_uv][col_uv] = prev_level; break; default: assert(plane >= 0 && plane <= 2); return; } } @@ -1548,6 +1552,7 @@ for (int c = 0; (c << MI_SIZE_LOG2) < plane_ptr->dst.width; c += col_step) { const int mi_col = c << subsampling_x; const int col = mi_col % MI_SIZE_64X64; + const int col_uv = col | subsampling_x; for (int r = 0; (r << MI_SIZE_LOG2) < plane_ptr->dst.height; r += (tx_size_high_unit[TX_64X64] >> subsampling_y)) { @@ -1559,15 +1564,16 @@ const int y = (r + r_in_unit) << MI_SIZE_LOG2; if (y >= plane_ptr->dst.height) break; const int row = r_in_unit << subsampling_y; + const int row_uv = row | subsampling_y; int index = 0; const int shift = get_index_shift(col, row, &index); const uint64_t mask = ((uint64_t)1 << shift); skip = lfm->skip.bits[index] & mask; is_coding_block_border = lfm->is_horz_border.bits[index] & mask; switch (plane) { - case 0: level = lfm->lfl_y_hor[row][col]; break; - case 1: level = lfm->lfl_u[row][col]; break; - case 2: level = lfm->lfl_v[row][col]; break; + case 0: level = lfm->lfl_y_hor[row_uv][col_uv]; break; + case 1: level = lfm->lfl_u_hor[row_uv][col_uv]; break; + case 2: level = lfm->lfl_v_hor[row_uv][col_uv]; break; default: assert(plane >= 0 && plane <= 2); return; } for (TX_SIZE ts = TX_4X4; ts <= TX_64X64; ++ts) { @@ -1581,9 +1587,7 @@ (!prev_skip || !skip || is_coding_block_border)) { const TX_SIZE min_tx_size = AOMMIN(TX_16X16, AOMMIN(tx_size, prev_tx_size)); - const int tmp_row = (row | subsampling_y) % MI_SIZE_64X64; - const int tmp_col = (mi_col | subsampling_x) % MI_SIZE_64X64; - const int shift_1 = get_index_shift(tmp_col, tmp_row, &index); + const int shift_1 = get_index_shift(col_uv, row_uv, &index); const uint64_t mask_1 = ((uint64_t)1 << shift_1); switch (plane) { @@ -1594,9 +1598,9 @@ } if (level == 0 && prev_level != 0) { switch (plane) { - case 0: lfm->lfl_y_ver[tmp_row][tmp_col] = prev_level; break; - case 1: lfm->lfl_u[tmp_row][tmp_col] = prev_level; break; - case 2: lfm->lfl_v[tmp_row][tmp_col] = prev_level; break; + case 0: lfm->lfl_y_hor[row_uv][col_uv] = prev_level; break; + case 1: lfm->lfl_u_hor[row_uv][col_uv] = prev_level; break; + case 2: lfm->lfl_v_hor[row_uv][col_uv] = prev_level; break; default: assert(plane >= 0 && plane <= 2); return; } } @@ -1644,6 +1648,7 @@ const int shift = get_index_shift(col, row, &index); int index_next = 0; const int shift_next = get_index_shift(col, row_next, &index_next); + const int has_next_row = row_next < cm->mi_rows; switch (pl) { case 0: mask_16x16 = lfm->left_y[TX_16X16].bits[index]; @@ -1656,15 +1661,15 @@ mask_16x16 = lfm->left_u[TX_16X16].bits[index]; mask_8x8 = lfm->left_u[TX_8X8].bits[index]; mask_4x4 = lfm->left_u[TX_4X4].bits[index]; - lfl = &lfm->lfl_u[row][col]; - lfl2 = &lfm->lfl_u[row_next][col]; + lfl = &lfm->lfl_u_ver[row][col]; + lfl2 = &lfm->lfl_u_ver[row_next][col]; break; case 2: mask_16x16 = lfm->left_v[TX_16X16].bits[index]; mask_8x8 = lfm->left_v[TX_8X8].bits[index]; mask_4x4 = lfm->left_v[TX_4X4].bits[index]; - lfl = &lfm->lfl_v[row][col]; - lfl2 = &lfm->lfl_v[row_next][col]; + lfl = &lfm->lfl_v_ver[row][col]; + lfl2 = &lfm->lfl_v_ver[row_next][col]; break; default: assert(pl >= 0 && pl <= 2); return; } @@ -1674,6 +1679,11 @@ uint64_t mask_16x16_1 = (mask_16x16 >> shift_next) & mask_cutoff; uint64_t mask_8x8_1 = (mask_8x8 >> shift_next) & mask_cutoff; uint64_t mask_4x4_1 = (mask_4x4 >> shift_next) & mask_cutoff; + if (!has_next_row) { + mask_16x16_1 = 0; + mask_8x8_1 = 0; + mask_4x4_1 = 0; + } if (cm->seq_params.use_highbitdepth) highbd_filter_selectively_vert_row2( @@ -1728,13 +1738,13 @@ mask_16x16 = lfm->above_u[TX_16X16].bits[index]; mask_8x8 = lfm->above_u[TX_8X8].bits[index]; mask_4x4 = lfm->above_u[TX_4X4].bits[index]; - lfl = &lfm->lfl_u[row][col]; + lfl = &lfm->lfl_u_hor[row][col]; break; case 2: mask_16x16 = lfm->above_v[TX_16X16].bits[index]; mask_8x8 = lfm->above_v[TX_8X8].bits[index]; mask_4x4 = lfm->above_v[TX_4X4].bits[index]; - lfl = &lfm->lfl_v[row][col]; + lfl = &lfm->lfl_v_hor[row][col]; break; default: assert(pl >= 0 && pl <= 2); return; } @@ -1772,10 +1782,10 @@ uint8_t *lfl2; // filter two rows at a time - for (r = 0; r < cm->seq_params.sb_size && + for (r = 0; r < cm->seq_params.mib_size && ((mi_row + r) << MI_SIZE_LOG2 < cm->height); r += r_step) { - for (c = 0; c < cm->seq_params.sb_size && + for (c = 0; c < cm->seq_params.mib_size && ((mi_col + c) << MI_SIZE_LOG2 < cm->width); c += MI_SIZE_64X64) { dst->buf += ((c << MI_SIZE_LOG2) >> ssx); @@ -1802,15 +1812,15 @@ mask_16x16 = lfm->left_u[TX_16X16].bits[index]; mask_8x8 = lfm->left_u[TX_8X8].bits[index]; mask_4x4 = lfm->left_u[TX_4X4].bits[index]; - lfl = &lfm->lfl_u[row][col]; - lfl2 = &lfm->lfl_u[row_next][col]; + lfl = &lfm->lfl_u_ver[row][col]; + lfl2 = &lfm->lfl_u_ver[row_next][col]; break; case 2: mask_16x16 = lfm->left_v[TX_16X16].bits[index]; mask_8x8 = lfm->left_v[TX_8X8].bits[index]; mask_4x4 = lfm->left_v[TX_4X4].bits[index]; - lfl = &lfm->lfl_v[row][col]; - lfl2 = &lfm->lfl_v[row_next][col]; + lfl = &lfm->lfl_v_ver[row][col]; + lfl2 = &lfm->lfl_v_ver[row_next][col]; break; default: assert(pl >= 0 && pl <= 2); return; } @@ -1851,10 +1861,10 @@ uint64_t mask_4x4 = 0; uint8_t *lfl; - for (r = 0; r < cm->seq_params.sb_size && + for (r = 0; r < cm->seq_params.mib_size && ((mi_row + r) << MI_SIZE_LOG2 < cm->height); r += r_step) { - for (c = 0; c < cm->seq_params.sb_size && + for (c = 0; c < cm->seq_params.mib_size && ((mi_col + c) << MI_SIZE_LOG2 < cm->width); c += MI_SIZE_64X64) { if (mi_row + r == 0) continue; @@ -1877,13 +1887,13 @@ mask_16x16 = lfm->above_u[TX_16X16].bits[index]; mask_8x8 = lfm->above_u[TX_8X8].bits[index]; mask_4x4 = lfm->above_u[TX_4X4].bits[index]; - lfl = &lfm->lfl_u[row][col]; + lfl = &lfm->lfl_u_hor[row][col]; break; case 2: mask_16x16 = lfm->above_v[TX_16X16].bits[index]; mask_8x8 = lfm->above_v[TX_8X8].bits[index]; mask_4x4 = lfm->above_v[TX_4X4].bits[index]; - lfl = &lfm->lfl_v[row][col]; + lfl = &lfm->lfl_v_hor[row][col]; break; default: assert(pl >= 0 && pl <= 2); return; } @@ -2347,27 +2357,24 @@ av1_setup_dst_planes(pd, cm->seq_params.sb_size, frame_buffer, 0, 0, plane, plane + 1); - av1_filter_block_plane_vert_test(cm, xd, plane, &pd[plane], 0, 0); - av1_filter_block_plane_horz_test(cm, xd, plane, &pd[plane], 0, 0); - av1_build_bitmask_vert_info(cm, &pd[plane], plane); av1_build_bitmask_horz_info(cm, &pd[plane], plane); // apply loop filtering which only goes through buffer once for (mi_row = start; mi_row < stop; mi_row += MI_SIZE_64X64) { for (mi_col = col_start; mi_col < col_end; mi_col += MI_SIZE_64X64) { - av1_setup_dst_planes(pd, MI_SIZE_64X64, frame_buffer, mi_row, mi_col, + av1_setup_dst_planes(pd, BLOCK_64X64, frame_buffer, mi_row, mi_col, plane, plane + 1); av1_filter_block_plane_bitmask_vert(cm, &pd[plane], plane, mi_row, mi_col); if (mi_col - MI_SIZE_64X64 >= 0) { - av1_setup_dst_planes(pd, MI_SIZE_64X64, frame_buffer, mi_row, + av1_setup_dst_planes(pd, BLOCK_64X64, frame_buffer, mi_row, mi_col - MI_SIZE_64X64, plane, plane + 1); av1_filter_block_plane_bitmask_horz(cm, &pd[plane], plane, mi_row, mi_col - MI_SIZE_64X64); } } - av1_setup_dst_planes(pd, MI_SIZE_64X64, frame_buffer, mi_row, + av1_setup_dst_planes(pd, BLOCK_64X64, frame_buffer, mi_row, mi_col - MI_SIZE_64X64, plane, plane + 1); av1_filter_block_plane_bitmask_horz(cm, &pd[plane], plane, mi_row, mi_col - MI_SIZE_64X64);
diff --git a/av1/common/av1_loopfilter.h b/av1/common/av1_loopfilter.h index 80ac611..afe0c44 100644 --- a/av1/common/av1_loopfilter.h +++ b/av1/common/av1_loopfilter.h
@@ -61,10 +61,12 @@ uint8_t lfl_y_ver[MI_SIZE_64X64][MI_SIZE_64X64]; // U plane filter level - uint8_t lfl_u[MI_SIZE_64X64][MI_SIZE_64X64]; + uint8_t lfl_u_ver[MI_SIZE_64X64][MI_SIZE_64X64]; + uint8_t lfl_u_hor[MI_SIZE_64X64][MI_SIZE_64X64]; // V plane filter level - uint8_t lfl_v[MI_SIZE_64X64][MI_SIZE_64X64]; + uint8_t lfl_v_ver[MI_SIZE_64X64][MI_SIZE_64X64]; + uint8_t lfl_v_hor[MI_SIZE_64X64][MI_SIZE_64X64]; // other info FilterMask skip;
diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c index ba0f7f2..604729b 100644 --- a/av1/decoder/decodeframe.c +++ b/av1/decoder/decodeframe.c
@@ -1561,9 +1561,13 @@ sizeof(uint8_t) * mi_size_wide[bsize]); memset(&lfm->lfl_y_hor[row][col_start], level_horz_y, sizeof(uint8_t) * mi_size_wide[bsize]); - memset(&lfm->lfl_u[row][col_start], level_u, + memset(&lfm->lfl_u_ver[row][col_start], level_u, sizeof(uint8_t) * mi_size_wide[bsize]); - memset(&lfm->lfl_v[row][col_start], level_v, + memset(&lfm->lfl_u_hor[row][col_start], level_u, + sizeof(uint8_t) * mi_size_wide[bsize]); + memset(&lfm->lfl_v_ver[row][col_start], level_v, + sizeof(uint8_t) * mi_size_wide[bsize]); + memset(&lfm->lfl_v_hor[row][col_start], level_v, sizeof(uint8_t) * mi_size_wide[bsize]); } }