Unit test for rps=1 verify frames output in ascending display_order_hint order

The test verifies that when a switch frame with restricted_prediction_switch=1 is decoded, eligible frames in ref_frame_map are output in ascending display_order_hint order.

It encodes with RA pyramid coding and uses a test-only encoder flag to defer output of inter frames so they accumulate in ref_frame_map. When the switch frame is decoded, the restricted_prediction_switch path outputs these deferred frames via av2_output_frame_buffers().

The decoder's built-in conformance check (check_and_update_output_doh) within av2_output_frame_buffers() verifies that output frames have strictly ascending display_order_hint per layer, and triggers a fatal AVM_CODEC_UNSUP_BITSTREAM error on any violation. The test simply asserts that decoding completes successfully, which implicitly confirms correct output ordering.

Related MR: !3424

Related Issue: https://gitlab.com/AOMediaCodec/avm/-/work_items/1466
diff --git a/av2/av2_cx_iface.c b/av2/av2_cx_iface.c
index 31cce3c..309af34 100644
--- a/av2/av2_cx_iface.c
+++ b/av2/av2_cx_iface.c
@@ -244,6 +244,7 @@
   int use_buffer_refresh_multi_layers_test;
   int buffer_refresh_multi_layers_test[REF_FRAMES];
   int multi_layers_lag_test;
+  int force_deferred_frames_for_ras_test;
 };
 
 // Example subgop configs. Currently not used by default.
@@ -572,6 +573,7 @@
   0,  // use_buffer_refresh_multi_layers_test
   { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, // buffer_refresh_multi_layers_test
   0,      // multi_layers_test for nozero lag
+  0,      // force_deferred_frames_for_ras_test
 };
 // clang-format on
 
@@ -1734,6 +1736,8 @@
          sizeof(oxcf->target_seq_level_idx));
   oxcf->tier_mask = extra_cfg->tier_mask;
   oxcf->unit_test_cfg.multi_layers_lag_test = extra_cfg->multi_layers_lag_test;
+  oxcf->unit_test_cfg.force_deferred_frames_for_ras_test =
+      extra_cfg->force_deferred_frames_for_ras_test;
 
   if (update_config) {
     update_encoder_config(&cfg->encoder_cfg, extra_cfg);
@@ -2740,6 +2744,14 @@
   return update_extra_cfg(ctx, &extra_cfg);
 }
 
+static avm_codec_err_t ctrl_set_force_deferred_frames_for_ras_test(
+    avm_codec_alg_priv_t *ctx, va_list args) {
+  struct av2_extracfg extra_cfg = ctx->extra_cfg;
+  extra_cfg.force_deferred_frames_for_ras_test =
+      CAST(AV2E_SET_FORCE_DEFERRED_FRAMES_FOR_RAS_TEST, args);
+  return update_extra_cfg(ctx, &extra_cfg);
+}
+
 static avm_codec_err_t create_stats_buffer(FIRSTPASS_STATS **frame_stats_buffer,
                                            STATS_BUFFER_CTX *stats_buf_context,
                                            int num_lap_buffers) {
@@ -4691,6 +4703,8 @@
     ctrl_set_enable_flag_multi_layer_lag_test },
   { AV2E_SET_ADD_SEF_FOR_HIDDEN_FRAMES, ctrl_set_add_sef_for_hidden_frames },
   { AV2E_SET_MONOTONIC_OUTPUT_ORDER, ctrl_set_monotonic_output_order },
+  { AV2E_SET_FORCE_DEFERRED_FRAMES_FOR_RAS_TEST,
+    ctrl_set_force_deferred_frames_for_ras_test },
 
   // Getters
   { AVME_GET_LAST_QUANTIZER, ctrl_get_quantizer },
diff --git a/av2/encoder/bitstream.c b/av2/encoder/bitstream.c
index 18920ef..134926c 100644
--- a/av2/encoder/bitstream.c
+++ b/av2/encoder/bitstream.c
@@ -5259,6 +5259,20 @@
       }
     }
 
+    // Test-only: defer output of non-KEY/non-S inter frames so they
+    // accumulate in ref_frame_map for the restricted_prediction_switch path.
+    const int test_modified =
+        cpi->oxcf.unit_test_cfg.force_deferred_frames_for_ras_test &&
+        cm->immediate_output_picture &&
+        cm->current_frame.frame_type != KEY_FRAME &&
+        cm->current_frame.frame_type != S_FRAME &&
+        !cm->bridge_frame_info.is_bridge_frame &&
+        !seq_params->monotonic_output_order_flag;
+    if (test_modified) {
+      cm->immediate_output_picture = 0;
+      cm->implicit_output_picture = 1;
+    }
+
     if (cm->bridge_frame_info.is_bridge_frame) {
       if (cm->immediate_output_picture) {
         avm_internal_error(&cm->error, AVM_CODEC_UNSUP_BITSTREAM,
@@ -5284,6 +5298,12 @@
     cm->cur_frame->immediate_output_picture = cm->immediate_output_picture;
     cm->cur_frame->implicit_output_picture = cm->implicit_output_picture;
 
+    // Test-only: restore encoder state so subsequent encoding is unaffected.
+    if (test_modified) {
+      cm->immediate_output_picture = 1;
+      cm->implicit_output_picture = 0;
+    }
+
     assert(!seq_params->monotonic_output_order_flag ||
            !cm->implicit_output_picture);
   }
diff --git a/av2/encoder/encoder.h b/av2/encoder/encoder.h
index 0ec1d96..e76dcec 100644
--- a/av2/encoder/encoder.h
+++ b/av2/encoder/encoder.h
@@ -698,6 +698,9 @@
   int buffer_refresh_multi_layers_test[REF_FRAMES];
   // Changes needed for multi layers tests with nonzero lag.
   int multi_layers_lag_test;
+  // Test-only: defer output of non-KEY/non-S frames to exercise the
+  // restricted_prediction_switch output ordering path.
+  int force_deferred_frames_for_ras_test;
   // Signal one sequence header for the entire sequence.
   uint8_t single_seq_header_for_all_test;
 } UnitTestCfg;
diff --git a/avm/avmcx.h b/avm/avmcx.h
index 4c10b9e..fcc5138 100644
--- a/avm/avmcx.h
+++ b/avm/avmcx.h
@@ -1245,6 +1245,10 @@
    */
   AV2E_SET_MONOTONIC_OUTPUT_ORDER = 184,
 
+  /*!\brief Test-only: defer output of non-KEY/non-S frames to exercise
+   * the restricted_prediction_switch output ordering path.
+   */
+  AV2E_SET_FORCE_DEFERRED_FRAMES_FOR_RAS_TEST = 185,
 };
 
 /*!\brief avm 1-D scaling mode
@@ -1769,6 +1773,9 @@
 
 AVM_CTRL_USE_TYPE(AV2E_SET_MONOTONIC_OUTPUT_ORDER, int)
 #define AVME_CTRL_AV2E_SET_MONOTONIC_OUTPUT_ORDER
+
+AVM_CTRL_USE_TYPE(AV2E_SET_FORCE_DEFERRED_FRAMES_FOR_RAS_TEST, int)
+#define AVME_CTRL_AV2E_SET_FORCE_DEFERRED_FRAMES_FOR_RAS_TEST
 /*!\endcond */
 /*! @} - end defgroup avm_encoder */
 #ifdef __cplusplus
diff --git a/test/output_order_test.cc b/test/output_order_test.cc
new file mode 100644
index 0000000..d048b42
--- /dev/null
+++ b/test/output_order_test.cc
@@ -0,0 +1,122 @@
+/*
+ * Copyright (c) 2026, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 3-Clause Clear License
+ * and the Alliance for Open Media Patent License 1.0. If the BSD 3-Clause Clear
+ * License was not distributed with this source code in the LICENSE file, you
+ * can obtain it at aomedia.org/license/software-license/bsd-3-c-c/.  If the
+ * Alliance for Open Media Patent License 1.0 was not distributed with this
+ * source code in the PATENTS file, you can obtain it at
+ * aomedia.org/license/patent-license/.
+ */
+
+// Test for restricted_prediction_switch output ordering.
+//
+// Verifies that when a switch frame with restricted_prediction_switch=1
+// is decoded, eligible frames in ref_frame_map are output in ascending
+// display_order_hint order.
+//
+// Encodes with RA pyramid coding (which stores frames in non-display
+// order in ref_frame_map) and uses AV2E_SET_FORCE_DEFERRED_FRAMES_FOR_RAS_TEST
+// to defer output of inter frames so they accumulate in ref_frame_map.
+// When the switch frame is decoded, the restricted_prediction_switch
+// path outputs these deferred frames.
+//
+// The decoder's av2_output_frame_buffers() conformance check
+// (check_and_update_output_doh) will cause a decode error if frames
+// are output out of order, so the test simply verifies that decoding
+// succeeds.
+
+#include <vector>
+
+#include "third_party/googletest/src/googletest/include/gtest/gtest.h"
+#include "test/codec_factory.h"
+#include "test/encode_test_driver.h"
+#include "test/y4m_video_source.h"
+#include "test/util.h"
+#include "av2/common/enums.h"
+
+namespace {
+
+class OutputOrderTest : public ::libavm_test::CodecTestWithParam<int>,
+                        public ::libavm_test::EncoderTest {
+ protected:
+  OutputOrderTest() : EncoderTest(GET_PARAM(0)), speed_(GET_PARAM(1)) {}
+
+  void SetUp() override {
+    InitializeConfig();
+    passes_ = 1;
+    cfg_.rc_end_usage = AVM_Q;
+    cfg_.g_threads = 1;
+    cfg_.g_lag_in_frames = 9;
+    cfg_.kf_min_dist = 65;
+    cfg_.kf_max_dist = 65;
+    cfg_.use_fixed_qp_offsets = 1;
+    cfg_.sframe_dist = 4;
+    cfg_.sframe_mode = 0;
+    cfg_.sframe_type = 1;
+  }
+
+  void PreEncodeFrameHook(::libavm_test::VideoSource *video,
+                          ::libavm_test::Encoder *encoder) override {
+    if (video->frame() == 0) {
+      encoder->Control(AVME_SET_CPUUSED, speed_);
+      encoder->Control(AVME_SET_QP, 210);
+      encoder->Control(AV2E_SET_ENABLE_KEYFRAME_FILTERING, 0);
+      encoder->Control(AV2E_SET_MIN_GF_INTERVAL, 4);
+      encoder->Control(AV2E_SET_MAX_GF_INTERVAL, 4);
+      encoder->Control(AV2E_SET_GF_MIN_PYRAMID_HEIGHT, 2);
+      encoder->Control(AV2E_SET_GF_MAX_PYRAMID_HEIGHT, 2);
+      encoder->Control(AVME_SET_ENABLEAUTOALTREF, 1);
+      encoder->Control(AV2E_SET_DELTAQ_MODE, 0);
+      encoder->Control(AV2E_SET_ENABLE_TPL_MODEL, 0);
+      encoder->Control(AV2E_SET_ENABLE_EXPLICIT_REF_FRAME_MAP, 1);
+      encoder->SetOption("enable-intrabc-ext", "2");
+      encoder->Control(AV2E_SET_FORCE_DEFERRED_FRAMES_FOR_RAS_TEST, 1);
+    }
+  }
+
+  bool DoDecode() const override { return false; }
+
+  void FramePktHook(const avm_codec_cx_pkt_t *pkt,
+                    ::libavm_test::DxDataIterator *dec_iter) override {
+    (void)dec_iter;
+    if (pkt->kind != AVM_CODEC_CX_FRAME_PKT) return;
+    const uint8_t *buf = static_cast<const uint8_t *>(pkt->data.frame.buf);
+    packets_.emplace_back(buf, buf + pkt->data.frame.sz);
+  }
+
+  int speed_;
+  std::vector<std::vector<uint8_t>> packets_;
+};
+
+TEST_P(OutputOrderTest, VerifyOutputOrderAtRAS) {
+  ::libavm_test::Y4mVideoSource video("park_joy_90p_8_420.y4m", 0, 9);
+  ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
+  ASSERT_FALSE(packets_.empty());
+
+  // Decode all packets. The decoder's conformance check in
+  // av2_output_frame_buffers() will trigger a fatal error if output
+  // frames are not in ascending display_order_hint order.
+  avm_codec_dec_cfg_t c = {};
+  c.threads = 1;
+  libavm_test::AV2Decoder dec(c);
+
+  for (const auto &p : packets_) {
+    ASSERT_EQ(dec.DecodeFrame(p.data(), p.size()), AVM_CODEC_OK);
+    libavm_test::DxDataIterator it = dec.GetDxData();
+    while (it.Next()) {
+    }
+  }
+  // Flush the decoder.
+  ASSERT_EQ(dec.DecodeFrame(NULL, 0), AVM_CODEC_OK);
+  {
+    libavm_test::DxDataIterator it = dec.GetDxData();
+    while (it.Next()) {
+    }
+  }
+}
+
+AV2_INSTANTIATE_TEST_SUITE(OutputOrderTest, ::testing::Values(9));
+
+}  // namespace
diff --git a/test/test.cmake b/test/test.cmake
index 79ce171..ada5e2f 100644
--- a/test/test.cmake
+++ b/test/test.cmake
@@ -139,6 +139,7 @@
       "${AVM_ROOT}/test/segment_binarization_sync.cc"
       "${AVM_ROOT}/test/sframe_test.cc"
       "${AVM_ROOT}/test/ras_frame_pruning_test.cc"
+      "${AVM_ROOT}/test/output_order_test.cc"
       "${AVM_ROOT}/test/still_picture_test.cc"
       "${AVM_ROOT}/test/sub_bitstream_extraction_test.cc"
       "${AVM_ROOT}/test/subgop_test.cc"