rtc: Set costs on frames that skip CDF updates

With nonrd pick mode the coefficient, mode and mv costs are updated
at the start of each superblock or superblock row, but not in frames
that don't update their CDFs. The frame level update is skipped too,
unless the update level is tile. So every thread codes such a frame
with the costs the main thread had at the end of an earlier frame.
With row mt the superblock the main thread coded last depends on
thread timing, so the output changes from run to run.

The default build hits this in real time mode, since its default
cost update frequency is per superblock. The realtime only build
defaults to off and is only hit when the cost update controls are
set.

Set these costs at the frame level when the frame doesn't update its
CDFs and the update level isn't off. The CDFs of such a frame stay at
the frame context, so these are the costs the per row update would
compute.

Checked on main at speed 11 with 4 threads, row mt, 2 tile columns
and the default build. On two screen captures of 150 frames, 3 runs
each, there were 2 or 3 different streams before and 1 after, in
both 4:2:0 and 4:4:4. niklas_640_480_30 with 300 frames gave 6
streams in 6 runs before and 1 after. The realtime only build output
is unchanged, also for 10 bit 4:2:0 and 4:4:4. The new
AVxEncoderThreadRTCostUpdTest fails on main in both builds.

At 1 thread the default build output changes a little on the two
screen captures:

  4:2:0: 0.21% to 0.39% smaller, PSNR 0.07 dB lower to 0.03 dB
  higher, encode time 1.6% faster to 0.5% slower
  4:4:4: up to 1.11% smaller, PSNR up to 0.10 dB lower, encode
  time unchanged to 1.2% slower

The time changes are within run to run noise.

Change-Id: I446b973cb941ed5e70e2c9fd93062dde67e6c154

av1/encoder/rd.c

diff --git a/av1/encoder/rd.c b/av1/encoder/rd.c
index 9fafdaf..9abd00c 100644
--- a/av1/encoder/rd.c
+++ b/av1/encoder/rd.c
@@ -754,7 +754,10 @@
       frame_is_intra_only(cm) ||
       (use_nonrd_pick_mode ? frames_since_key < 2
                            : (cm->current_frame.frame_number & 0x07) == 1);
-  return ((!use_nonrd_pick_mode && cost_upd_level != INTERNAL_COST_UPD_OFF) ||
+  // Nonrd frames without CDF updates get no per superblock cost update.
+  const int update_per_frame =
+      !use_nonrd_pick_mode || cm->features.disable_cdf_update;
+  return ((update_per_frame && cost_upd_level != INTERNAL_COST_UPD_OFF) ||
           cost_upd_level == INTERNAL_COST_UPD_TILE || fill_costs);
 }
 

test/ethread_test.cc

diff --git a/test/ethread_test.cc b/test/ethread_test.cc
index b0940ba..c432fdd 100644
--- a/test/ethread_test.cc
+++ b/test/ethread_test.cc
@@ -221,7 +221,7 @@
       : EncoderTest(GET_PARAM(0)), encoder_initialized_(false),
         encoding_mode_(GET_PARAM(1)), set_cpu_used_(GET_PARAM(2)),
         tile_cols_(GET_PARAM(3)), tile_rows_(GET_PARAM(4)),
-        row_mt_(GET_PARAM(5)) {
+        row_mt_(GET_PARAM(5)), frame_limit_(26) {
     init_flags_ = AOM_CODEC_USE_PSNR;
     aom_codec_dec_cfg_t cfg = aom_codec_dec_cfg_t();
     cfg.w = 1280;
@@ -324,8 +324,9 @@
   }
 
   void DoTest() {
-    ::libaom_test::YUVVideoSource video(
-        "niklas_640_480_30.yuv", AOM_IMG_FMT_I420, 640, 480, 30, 1, 15, 26);
+    ::libaom_test::YUVVideoSource video("niklas_640_480_30.yuv",
+                                        AOM_IMG_FMT_I420, 640, 480, 30, 1, 15,
+                                        frame_limit_);
     cfg_.rc_target_bitrate = 1000;
 
     if (row_mt_ == 0) {
@@ -445,6 +446,7 @@
   int tile_cols_;
   int tile_rows_;
   int row_mt_;
+  int frame_limit_;
   ::libaom_test::Decoder *decoder_;
   std::vector<size_t> size_enc_;
   std::vector<std::string> md5_enc_;
@@ -567,6 +569,35 @@
                            ::testing::Values(7, 9), ::testing::Values(1),
                            ::testing::Values(1), ::testing::Values(0, 1));
 
+// Per superblock cost updates must give the same output for any thread count.
+class AVxEncoderThreadRTCostUpdTest : public AVxEncoderThreadTest {
+ protected:
+  AVxEncoderThreadRTCostUpdTest() { frame_limit_ = 60; }
+
+  void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+                          ::libaom_test::Encoder *encoder) override {
+    const bool first_frame = !encoder_initialized_;
+    AVxEncoderThreadTest::PreEncodeFrameHook(video, encoder);
+    if (first_frame) {
+      encoder->Control(AV1E_SET_COEFF_COST_UPD_FREQ, 0);
+      encoder->Control(AV1E_SET_MODE_COST_UPD_FREQ, 0);
+      encoder->Control(AV1E_SET_MV_COST_UPD_FREQ, 0);
+      encoder->Control(AV1E_SET_CDF_UPDATE_MODE, 2);
+    }
+  }
+};
+
+TEST_P(AVxEncoderThreadRTCostUpdTest, EncoderResultTest) {
+  cfg_.large_scale_tile = 0;
+  decoder_->Control(AV1_SET_TILE_MODE, 0);
+  DoTest();
+}
+
+AV1_INSTANTIATE_TEST_SUITE(AVxEncoderThreadRTCostUpdTest,
+                           ::testing::Values(::libaom_test::kRealTime),
+                           ::testing::Values(7, 10), ::testing::Values(0, 1),
+                           ::testing::Values(0, 1), ::testing::Values(1));
+
 #if !CONFIG_REALTIME_ONLY
 
 // The AVxEncoderThreadTestLarge takes up ~14% of total run-time of the