[non-normative] Fix CfL prediction for semi-lossless frames
When some segment IDs use lossless mode and others do not,
it is possible for the luma pixel buffer used by CfL to be
incorrectly updated, leading to bad predictions.
Fix this by more accurately calculating which blocks need
to store luma pixels.
BUG=aomedia:1707
Change-Id: Ia4c50b95841e6bd11e8e50f334081af489175a25
diff --git a/av1/common/cfl.c b/av1/common/cfl.c
index 41433b0..36561cb 100644
--- a/av1/common/cfl.c
+++ b/av1/common/cfl.c
@@ -447,7 +447,6 @@
uint8_t *dst =
&pd->dst.buf[(row * pd->dst.stride + col) << tx_size_wide_log2[0]];
- assert(is_cfl_allowed(xd));
if (block_size_high[bsize] == 4 || block_size_wide[bsize] == 4) {
// Only dimensions of size 4 can have an odd offset.
assert(!((col & 1) && tx_size_wide[tx_size] != 4));
@@ -464,7 +463,6 @@
int row = 0;
int col = 0;
- assert(is_cfl_allowed(xd));
if (block_size_high[bsize] == 4 || block_size_wide[bsize] == 4) {
sub8x8_adjust_offset(cfl, &row, &col);
}
diff --git a/av1/common/cfl.h b/av1/common/cfl.h
index 2850c02..a2250bc 100644
--- a/av1/common/cfl.h
+++ b/av1/common/cfl.h
@@ -13,7 +13,9 @@
#define AV1_COMMON_CFL_H_
#include "av1/common/blockd.h"
+#include "av1/common/onyxc_int.h"
+// Can we use CfL for the current block?
static INLINE CFL_ALLOWED_TYPE is_cfl_allowed(const MACROBLOCKD *xd) {
const MB_MODE_INFO *mbmi = xd->mi[0];
const BLOCK_SIZE bsize = mbmi->sb_type;
@@ -30,6 +32,29 @@
block_size_high[bsize] <= 32);
}
+// Do we need to save the luma pixels from the current block,
+// for a possible future CfL prediction?
+static inline CFL_ALLOWED_TYPE store_cfl_required(const AV1_COMMON *cm,
+ const MACROBLOCKD *xd) {
+ const MB_MODE_INFO *mbmi = xd->mi[0];
+
+ if (cm->seq_params.monochrome) return CFL_DISALLOWED;
+
+ if (!xd->cfl.is_chroma_reference) {
+ // For non-chroma-reference blocks, we should always store the luma pixels,
+ // in case the corresponding chroma-reference block uses CfL.
+ // Note that this can only happen for block sizes which are <8 on
+ // their shortest side, as otherwise they would be chroma reference
+ // blocks.
+ return CFL_ALLOWED;
+ }
+
+ // If this block has chroma information, we know whether we're
+ // actually going to perform a CfL prediction
+ return (CFL_ALLOWED_TYPE)(!is_inter_block(mbmi) &&
+ mbmi->uv_mode == UV_CFL_PRED);
+}
+
static INLINE int get_scaled_luma_q0(int alpha_q3, int16_t pred_buf_q3) {
int scaled_luma_q6 = alpha_q3 * pred_buf_q3;
return ROUND_POWER_OF_TWO_SIGNED(scaled_luma_q6, 6);
diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c
index fed1ca6..cf64add 100644
--- a/av1/decoder/decodeframe.c
+++ b/av1/decoder/decodeframe.c
@@ -178,7 +178,7 @@
max_scan_line, eob, cm->reduced_tx_set_used);
}
}
- if (plane == AOM_PLANE_Y && xd->cfl.store_y && is_cfl_allowed(xd)) {
+ if (plane == AOM_PLANE_Y && xd->cfl.store_y) {
cfl_store_tx(xd, row, col, tx_size, mbmi->sb_type);
}
}
@@ -504,10 +504,8 @@
}
}
}
- }
- if (mbmi->uv_mode != UV_CFL_PRED) {
- if (!cfl->is_chroma_reference && is_inter_block(mbmi) &&
- is_cfl_allowed(xd)) {
+
+ if (xd->cfl.store_y) {
cfl_store_block(xd, mbmi->sb_type, mbmi->tx_size);
}
}
diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c
index ad22aad..3326be0 100644
--- a/av1/decoder/decodemv.c
+++ b/av1/decoder/decodemv.c
@@ -799,9 +799,6 @@
read_intra_mode_uv(ec_ctx, r, is_cfl_allowed(xd), mbmi->mode);
if (mbmi->uv_mode == UV_CFL_PRED) {
mbmi->cfl_alpha_idx = read_cfl_alphas(ec_ctx, r, &mbmi->cfl_alpha_signs);
- xd->cfl.store_y = 1;
- } else {
- xd->cfl.store_y = 0;
}
mbmi->angle_delta[PLANE_TYPE_UV] =
(use_angle_delta && av1_is_directional_mode(get_uv_mode(mbmi->uv_mode)))
@@ -812,8 +809,8 @@
// Avoid decoding angle_info if there is is no chroma prediction
mbmi->uv_mode = UV_DC_PRED;
xd->cfl.is_chroma_reference = 0;
- xd->cfl.store_y = 1;
}
+ xd->cfl.store_y = store_cfl_required(cm, xd);
if (av1_allow_palette(cm->allow_screen_content_tools, bsize))
read_palette_mode_info(cm, xd, mi_row, mi_col, r);
@@ -1057,17 +1054,16 @@
use_angle_delta && av1_is_directional_mode(mbmi->mode)
? read_angle_delta(r, ec_ctx->angle_delta_cdf[mbmi->mode - V_PRED])
: 0;
- if (!cm->seq_params.monochrome &&
+ const int has_chroma =
is_chroma_reference(mi_row, mi_col, bsize, xd->plane[1].subsampling_x,
- xd->plane[1].subsampling_y)) {
+ xd->plane[1].subsampling_y);
+ xd->cfl.is_chroma_reference = has_chroma;
+ if (!cm->seq_params.monochrome && has_chroma) {
mbmi->uv_mode =
read_intra_mode_uv(ec_ctx, r, is_cfl_allowed(xd), mbmi->mode);
if (mbmi->uv_mode == UV_CFL_PRED) {
mbmi->cfl_alpha_idx =
read_cfl_alphas(xd->tile_ctx, r, &mbmi->cfl_alpha_signs);
- xd->cfl.store_y = 1;
- } else {
- xd->cfl.store_y = 0;
}
mbmi->angle_delta[PLANE_TYPE_UV] =
use_angle_delta && av1_is_directional_mode(get_uv_mode(mbmi->uv_mode))
@@ -1077,9 +1073,8 @@
} else {
// Avoid decoding angle_info if there is is no chroma prediction
mbmi->uv_mode = UV_DC_PRED;
- xd->cfl.is_chroma_reference = 0;
- xd->cfl.store_y = 1;
}
+ xd->cfl.store_y = store_cfl_required(cm, xd);
mbmi->palette_mode_info.palette_size[0] = 0;
mbmi->palette_mode_info.palette_size[1] = 0;
@@ -1503,6 +1498,10 @@
}
}
+ xd->cfl.is_chroma_reference = is_chroma_reference(
+ mi_row, mi_col, bsize, cm->subsampling_x, cm->subsampling_y);
+ xd->cfl.store_y = store_cfl_required(cm, xd);
+
#if DEC_MISMATCH_DEBUG
dec_dump_logs(cm, mi, mi_row, mi_col, mode_ctx);
#endif // DEC_MISMATCH_DEBUG
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 503376d..a7dd1e3 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -4635,7 +4635,9 @@
const int is_inter = is_inter_block(mbmi);
if (!is_inter) {
- xd->cfl.store_y = 1;
+ xd->cfl.is_chroma_reference = is_chroma_reference(
+ mi_row, mi_col, bsize, cm->subsampling_x, cm->subsampling_y);
+ xd->cfl.store_y = store_cfl_required(cm, xd);
mbmi->skip = 1;
for (int plane = 0; plane < num_planes; ++plane) {
av1_encode_intra_block_plane(cpi, x, bsize, plane,
diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c
index 681220f..afb1f68 100644
--- a/av1/encoder/encodemb.c
+++ b/av1/encoder/encodemb.c
@@ -555,7 +555,7 @@
// very expensive.
*(args->skip) = 0;
- if (plane == AOM_PLANE_Y && xd->cfl.store_y && is_cfl_allowed(xd)) {
+ if (plane == AOM_PLANE_Y && xd->cfl.store_y) {
cfl_store_tx(xd, blk_row, blk_col, tx_size, plane_bsize);
}
}
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 5f1c77c..f33ba84 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -356,6 +356,28 @@
return av1_cost_literal(l);
}
+// Similar to store_cfl_required(), but for use during the RDO process,
+// where we haven't yet determined whether this block uses CfL.
+static inline CFL_ALLOWED_TYPE store_cfl_required_rdo(const AV1_COMMON *cm,
+ const MACROBLOCK *x) {
+ const MACROBLOCKD *xd = &x->e_mbd;
+
+ if (cm->seq_params.monochrome || x->skip_chroma_rd) return CFL_DISALLOWED;
+
+ if (!xd->cfl.is_chroma_reference) {
+ // For non-chroma-reference blocks, we should always store the luma pixels,
+ // in case the corresponding chroma-reference block uses CfL.
+ // Note that this can only happen for block sizes which are <8 on
+ // their shortest side, as otherwise they would be chroma reference
+ // blocks.
+ return CFL_ALLOWED;
+ }
+
+ // For chroma reference blocks, we should store data in the encoder iff we're
+ // allowed to try out CfL.
+ return is_cfl_allowed(xd);
+}
+
// constants for prune 1 and prune 2 decision boundaries
#define FAST_EXT_TX_CORR_MID 0.0
#define FAST_EXT_TX_EDST_MID 0.1
@@ -2265,7 +2287,7 @@
a, l, 0, args->use_fast_coef_costing,
args->best_rd - args->this_rd, &this_rd_stats);
- if (plane == AOM_PLANE_Y && xd->cfl.store_y && is_cfl_allowed(xd)) {
+ if (plane == AOM_PLANE_Y && xd->cfl.store_y) {
assert(!is_inter_block(mbmi) || plane_bsize < BLOCK_8X8);
cfl_store_tx(xd, blk_row, blk_col, tx_size, plane_bsize);
}
@@ -5171,8 +5193,11 @@
int *rate_uv, int *rate_uv_tokenonly,
int64_t *dist_uv, int *skip_uv,
UV_PREDICTION_MODE *mode_uv) {
+ const AV1_COMMON *const cm = &cpi->common;
MACROBLOCKD *xd = &x->e_mbd;
MB_MODE_INFO *mbmi = xd->mi[0];
+ const int mi_row = -xd->mb_to_top_edge >> (3 + MI_SIZE_LOG2);
+ const int mi_col = -xd->mb_to_left_edge >> (3 + MI_SIZE_LOG2);
// Use an estimated rd for uv_intra based on DC_PRED if the
// appropriate speed flag is set.
init_sbuv_mode(mbmi);
@@ -5184,15 +5209,15 @@
*mode_uv = UV_DC_PRED;
return;
}
+ xd->cfl.is_chroma_reference = is_chroma_reference(
+ mi_row, mi_col, bsize, cm->subsampling_x, cm->subsampling_y);
bsize = scale_chroma_bsize(bsize, xd->plane[AOM_PLANE_U].subsampling_x,
xd->plane[AOM_PLANE_U].subsampling_y);
// Only store reconstructed luma when there's chroma RDO. When there's no
// chroma RDO, the reconstructed luma will be stored in encode_superblock().
- xd->cfl.store_y = !x->skip_chroma_rd;
+ xd->cfl.store_y = store_cfl_required_rdo(cm, x);
if (xd->cfl.store_y) {
// Restore reconstructed luma values.
- const int mi_row = -xd->mb_to_top_edge >> (3 + MI_SIZE_LOG2);
- const int mi_col = -xd->mb_to_left_edge >> (3 + MI_SIZE_LOG2);
av1_encode_intra_block_plane(cpi, x, mbmi->sb_type, AOM_PLANE_Y,
cpi->optimize_seg_arr[mbmi->segment_id],
mi_row, mi_col);
@@ -8136,7 +8161,9 @@
if (intra_yrd < best_rd) {
// Only store reconstructed luma when there's chroma RDO. When there's no
// chroma RDO, the reconstructed luma will be stored in encode_superblock().
- xd->cfl.store_y = !x->skip_chroma_rd;
+ xd->cfl.is_chroma_reference = is_chroma_reference(
+ mi_row, mi_col, bsize, cm->subsampling_x, cm->subsampling_y);
+ xd->cfl.store_y = store_cfl_required_rdo(cm, x);
if (xd->cfl.store_y) {
// Restore reconstructed luma values.
memcpy(x->blk_skip, ctx->blk_skip,
