av1: Fix temporal filter SEGV on cancellation Add defensive checks in av1_tf_info_filtering and encode_strategy.c to ensure frames exist in the lookahead buffer before attempting temporal filtering. This prevents a NULL pointer dereference when the encoder is forced to flush early during second-pass encoding. Bug: b/465818130 Change-Id: I51e517e46434059196999ce593aac4f3b632326f
diff --git a/av1/encoder/encode_strategy.c b/av1/encoder/encode_strategy.c index 6f9715c..3cf86a2 100644 --- a/av1/encoder/encode_strategy.c +++ b/av1/encoder/encode_strategy.c
@@ -835,14 +835,17 @@ // Right now, we are still using tf_buf_second_arf due to // implementation complexity. // TODO(angiebird): Reuse tf_info->tf_buf here. - av1_temporal_filter(cpi, arf_src_index, cpi->gf_frame_index, &frame_diff, - tf_buf_second_arf); - show_existing_alt_ref = - av1_check_show_filtered_frame(tf_buf_second_arf, &frame_diff, q_index, - cm->seq_params->bit_depth, 1, 1); - if (show_existing_alt_ref) { - aom_extend_frame_borders(tf_buf_second_arf, av1_num_planes(cm)); - frame_input->source = tf_buf_second_arf; + if (av1_lookahead_peek(cpi->ppi->lookahead, arf_src_index, + cpi->compressor_stage)) { + av1_temporal_filter(cpi, arf_src_index, cpi->gf_frame_index, + &frame_diff, tf_buf_second_arf); + show_existing_alt_ref = av1_check_show_filtered_frame( + tf_buf_second_arf, &frame_diff, q_index, cm->seq_params->bit_depth, + 1, 1); + if (show_existing_alt_ref) { + aom_extend_frame_borders(tf_buf_second_arf, av1_num_planes(cm)); + frame_input->source = tf_buf_second_arf; + } } // Currently INTNL_ARF_UPDATE only do show_existing. cpi->common.showable_frame |= 1;
diff --git a/av1/encoder/temporal_filter.c b/av1/encoder/temporal_filter.c index 97be8fc..9498caf 100644 --- a/av1/encoder/temporal_filter.c +++ b/av1/encoder/temporal_filter.c
@@ -1724,6 +1724,10 @@ int buf_idx = gf_group->frame_type[gf_index] == INTER_FRAME; int lookahead_idx = gf_group->arf_src_offset[gf_index] + gf_group->cur_frame_idx[gf_index]; + if (!av1_lookahead_peek(cpi->ppi->lookahead, lookahead_idx, + cpi->compressor_stage)) { + continue; + } // This function is designed to be called multiple times after // av1_tf_info_reset(). It will only generate the filtered frame that does // not exist yet.
diff --git a/test/cancellation_test.cc b/test/cancellation_test.cc new file mode 100644 index 0000000..e0a6f99 --- /dev/null +++ b/test/cancellation_test.cc
@@ -0,0 +1,111 @@ +/* + * Copyright (c) 2026, Alliance for Open Media. All rights reserved. + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include <memory> + +#include "gtest/gtest.h" +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/util.h" +#include "test/video_source.h" + +namespace { + +class CancellableVideoSource : public ::libaom_test::VideoSource { + public: + explicit CancellableVideoSource(::libaom_test::VideoSource *source) + : source_(source), cancel_after_(-1), current_pass_(0) {} + + void Begin() override { + source_->Begin(); + current_pass_++; + } + + void Next() override { source_->Next(); } + + aom_image_t *img() const override { + if (current_pass_ == 2 && cancel_after_ >= 0 && + static_cast<int>(source_->frame()) >= cancel_after_) { + return nullptr; + } + return source_->img(); + } + + aom_codec_pts_t pts() const override { return source_->pts(); } + unsigned long duration() const override { return source_->duration(); } + aom_rational_t timebase() const override { return source_->timebase(); } + unsigned int frame() const override { return source_->frame(); } + unsigned int limit() const override { return source_->limit(); } + + void set_cancel_after(int frame) { cancel_after_ = frame; } + + private: + ::libaom_test::VideoSource *source_; + int cancel_after_; + int current_pass_; +}; + +class CancellationTest + : public ::libaom_test::EncoderTest, + public ::libaom_test::CodecTestWithParam< ::libaom_test::TestMode> { + protected: + CancellationTest() + : EncoderTest(GET_PARAM(0)), tile_columns_(0), tile_rows_(0) {} + ~CancellationTest() override = default; + + void SetUp() override { InitializeConfig(GET_PARAM(1)); } + + void PreEncodeFrameHook(::libaom_test::VideoSource *video, + ::libaom_test::Encoder *encoder) override { + if (video->frame() == 0) { + encoder->Control(AOME_SET_CPUUSED, 4); // speed 4 + encoder->Control(AV1E_SET_TILE_COLUMNS, tile_columns_); + encoder->Control(AV1E_SET_TILE_ROWS, tile_rows_); + } + } + + void DoSimulateCancellationPass2(int threads, int tile_columns, + int tile_rows) { + cfg_.g_threads = threads; + tile_columns_ = tile_columns; + tile_rows_ = tile_rows; + + ::libaom_test::RandomVideoSource raw_video; + raw_video.SetSize(256, 256); + raw_video.set_limit(40); // 40 frames total + + CancellableVideoSource video(&raw_video); + video.set_cancel_after(20); // Cancel after 20 frames in pass 2 + + // This should not crash + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + } + + private: + int tile_columns_; + int tile_rows_; +}; + +TEST_P(CancellationTest, SimulateCancellationPass2) { + DoSimulateCancellationPass2(/*threads=*/1, /*tile_columns=*/0, + /*tile_rows=*/0); +} + +TEST_P(CancellationTest, SimulateCancellationPass2Threads4) { + DoSimulateCancellationPass2(/*threads=*/4, /*tile_columns=*/1, + /*tile_rows=*/1); +} + +// Test 2-pass good quality +AV1_INSTANTIATE_TEST_SUITE(CancellationTest, + ::testing::Values(::libaom_test::kTwoPassGood)); + +} // namespace
diff --git a/test/test.cmake b/test/test.cmake index 84ba918..655268a 100644 --- a/test/test.cmake +++ b/test/test.cmake
@@ -65,6 +65,7 @@ "${AOM_ROOT}/test/av1_external_partition_test.cc" "${AOM_ROOT}/test/avif_progressive_test.cc" "${AOM_ROOT}/test/borders_test.cc" + "${AOM_ROOT}/test/cancellation_test.cc" "${AOM_ROOT}/test/cpu_speed_test.cc" "${AOM_ROOT}/test/cpu_used_firstpass_test.cc" "${AOM_ROOT}/test/datarate_test.cc" @@ -126,6 +127,7 @@ "${AOM_ROOT}/test/av1_external_partition_test.cc" "${AOM_ROOT}/test/avif_progressive_test.cc" "${AOM_ROOT}/test/borders_test.cc" + "${AOM_ROOT}/test/cancellation_test.cc" "${AOM_ROOT}/test/cpu_speed_test.cc" "${AOM_ROOT}/test/cpu_used_firstpass_test.cc" "${AOM_ROOT}/test/deltaq_mode_test.cc"