rtc: Fix is_key_frame setting in variance partition
On resized frame, or if LAST is NULL, the frame is treated
as key frame in variance partitioning (lines ~1406),
but in fill_variance_tree_leaves() the is_key_frame is
set again and does not include logic from lines ~1406.
This leads to worse quality on resized frame, and
causes a crash for resizing with 10bit input.
Fix is to pass the is_key_frame into the
fill_variance_tree_leaves().
Change-Id: Ibe4162de54cf31739667796a9666880293bd54ad
diff --git a/av1/encoder/var_based_part.c b/av1/encoder/var_based_part.c
index 2adec67..62601b4 100644
--- a/av1/encoder/var_based_part.c
+++ b/av1/encoder/var_based_part.c
@@ -1013,10 +1013,10 @@
AV1_COMP *cpi, MACROBLOCK *x, VP128x128 *vt, VP16x16 *vt2,
PART_EVAL_STATUS *force_split, int avg_16x16[][4], int maxvar_16x16[][4],
int minvar_16x16[][4], int *variance4x4downsample, int64_t *thresholds,
- const uint8_t *src, int src_stride, const uint8_t *dst, int dst_stride) {
+ const uint8_t *src, int src_stride, const uint8_t *dst, int dst_stride,
+ const bool is_key_frame) {
AV1_COMMON *cm = &cpi->common;
MACROBLOCKD *xd = &x->e_mbd;
- const bool is_key_frame = frame_is_intra_only(cm);
const bool is_small_sb = (cm->seq_params->sb_size == BLOCK_64X64);
const int num_64x64_blocks = is_small_sb ? 1 : 4;
// TODO(kyslov) Bring back compute_minmax_variance with content type detection
@@ -1484,7 +1484,7 @@
fill_variance_tree_leaves(cpi, x, vt, vt2, force_split, avg_16x16,
maxvar_16x16, minvar_16x16, variance4x4downsample,
thresholds, src_buf, src_stride, dst_buf,
- dst_stride);
+ dst_stride, is_key_frame);
avg_64x64 = 0;
for (int blk64_idx = 0; blk64_idx < num_64x64_blocks; ++blk64_idx) {
