blob: 2c1392cacdb1bc35aaeb765126587221e8d8a0c1 [file] [log] [blame]
Adrian Grangecc017ca2012-10-02 12:16:27 -07001/*
Frank Galligan38536f62013-12-12 08:36:34 -08002 * Copyright (c) 2013 The WebM project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
Deb Mukherjee01cafaa2013-01-15 06:43:35 -080010
Adrian Grangecc017ca2012-10-02 12:16:27 -070011#include "third_party/googletest/src/include/gtest/gtest.h"
John Koleszar706cafe2013-01-18 11:51:12 -080012#include "test/codec_factory.h"
Adrian Grangecc017ca2012-10-02 12:16:27 -070013#include "test/encode_test_driver.h"
14#include "test/i420_video_source.h"
John Koleszar706cafe2013-01-18 11:51:12 -080015#include "test/util.h"
Adrian Grangecc017ca2012-10-02 12:16:27 -070016
17namespace {
18
Marco Paniconif61b9622014-02-24 18:14:50 -080019const int kMaxErrorFrames = 12;
20const int kMaxDroppableFrames = 12;
Deb Mukherjee01cafaa2013-01-15 06:43:35 -080021
clang-format3a826f12016-08-11 17:46:05 -070022class ErrorResilienceTestLarge
23 : public ::libvpx_test::EncoderTest,
24 public ::libvpx_test::CodecTestWithParam<libvpx_test::TestMode> {
Adrian Grangecc017ca2012-10-02 12:16:27 -070025 protected:
Jim Bankoskia0b5ed62014-03-12 08:13:16 -070026 ErrorResilienceTestLarge()
clang-format3a826f12016-08-11 17:46:05 -070027 : EncoderTest(GET_PARAM(0)), psnr_(0.0), nframes_(0), mismatch_psnr_(0.0),
28 mismatch_nframes_(0), encoding_mode_(GET_PARAM(1)) {
Deb Mukherjee01cafaa2013-01-15 06:43:35 -080029 Reset();
30 }
John Koleszar706cafe2013-01-18 11:51:12 -080031
Jim Bankoskia0b5ed62014-03-12 08:13:16 -070032 virtual ~ErrorResilienceTestLarge() {}
Adrian Grangecc017ca2012-10-02 12:16:27 -070033
Deb Mukherjee01cafaa2013-01-15 06:43:35 -080034 void Reset() {
35 error_nframes_ = 0;
36 droppable_nframes_ = 0;
Marco2c6d9c52015-01-11 15:26:44 -080037 pattern_switch_ = 0;
Deb Mukherjee01cafaa2013-01-15 06:43:35 -080038 }
39
Adrian Grangecc017ca2012-10-02 12:16:27 -070040 virtual void SetUp() {
41 InitializeConfig();
42 SetMode(encoding_mode_);
43 }
44
45 virtual void BeginPassHook(unsigned int /*pass*/) {
46 psnr_ = 0.0;
47 nframes_ = 0;
Deb Mukherjee01cafaa2013-01-15 06:43:35 -080048 mismatch_psnr_ = 0.0;
49 mismatch_nframes_ = 0;
Adrian Grangecc017ca2012-10-02 12:16:27 -070050 }
51
Adrian Grangecc017ca2012-10-02 12:16:27 -070052 virtual void PSNRPktHook(const vpx_codec_cx_pkt_t *pkt) {
53 psnr_ += pkt->data.psnr.psnr[0];
54 nframes_++;
55 }
56
Marco8fd55252014-10-29 15:34:18 -070057 virtual void PreEncodeFrameHook(libvpx_test::VideoSource *video,
James Zerncffef112016-02-11 18:27:00 -080058 ::libvpx_test::Encoder * /*encoder*/) {
clang-format3a826f12016-08-11 17:46:05 -070059 frame_flags_ &=
60 ~(VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF);
James Zerncc73e1f2016-08-08 15:09:30 -070061 if (droppable_nframes_ > 0 &&
62 (cfg_.g_pass == VPX_RC_LAST_PASS || cfg_.g_pass == VPX_RC_ONE_PASS)) {
Deb Mukherjee01cafaa2013-01-15 06:43:35 -080063 for (unsigned int i = 0; i < droppable_nframes_; ++i) {
Deb Mukherjee0d8723f2013-08-19 14:16:26 -070064 if (droppable_frames_[i] == video->frame()) {
clang-format3a826f12016-08-11 17:46:05 -070065 std::cout << "Encoding droppable frame: " << droppable_frames_[i]
66 << "\n";
67 frame_flags_ |= (VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_GF |
James Zerncc73e1f2016-08-08 15:09:30 -070068 VP8_EFLAG_NO_UPD_ARF);
69 return;
Deb Mukherjee01cafaa2013-01-15 06:43:35 -080070 }
71 }
72 }
73 }
74
Adrian Grangecc017ca2012-10-02 12:16:27 -070075 double GetAveragePsnr() const {
clang-format3a826f12016-08-11 17:46:05 -070076 if (nframes_) return psnr_ / nframes_;
Adrian Grangecc017ca2012-10-02 12:16:27 -070077 return 0.0;
78 }
79
Deb Mukherjee01cafaa2013-01-15 06:43:35 -080080 double GetAverageMismatchPsnr() const {
clang-format3a826f12016-08-11 17:46:05 -070081 if (mismatch_nframes_) return mismatch_psnr_ / mismatch_nframes_;
Deb Mukherjee01cafaa2013-01-15 06:43:35 -080082 return 0.0;
83 }
84
85 virtual bool DoDecode() const {
86 if (error_nframes_ > 0 &&
87 (cfg_.g_pass == VPX_RC_LAST_PASS || cfg_.g_pass == VPX_RC_ONE_PASS)) {
88 for (unsigned int i = 0; i < error_nframes_; ++i) {
89 if (error_frames_[i] == nframes_ - 1) {
90 std::cout << " Skipping decoding frame: "
91 << error_frames_[i] << "\n";
92 return 0;
93 }
94 }
95 }
96 return 1;
97 }
98
clang-format3a826f12016-08-11 17:46:05 -070099 virtual void MismatchHook(const vpx_image_t *img1, const vpx_image_t *img2) {
Deb Mukherjee01cafaa2013-01-15 06:43:35 -0800100 double mismatch_psnr = compute_psnr(img1, img2);
101 mismatch_psnr_ += mismatch_psnr;
102 ++mismatch_nframes_;
103 // std::cout << "Mismatch frame psnr: " << mismatch_psnr << "\n";
Geza Loree0dcab92016-05-09 13:19:34 +0100104 ::libvpx_test::EncoderTest::MismatchHook(img1, img2);
Deb Mukherjee01cafaa2013-01-15 06:43:35 -0800105 }
106
107 void SetErrorFrames(int num, unsigned int *list) {
108 if (num > kMaxErrorFrames)
109 num = kMaxErrorFrames;
110 else if (num < 0)
111 num = 0;
112 error_nframes_ = num;
113 for (unsigned int i = 0; i < error_nframes_; ++i)
114 error_frames_[i] = list[i];
115 }
116
117 void SetDroppableFrames(int num, unsigned int *list) {
118 if (num > kMaxDroppableFrames)
119 num = kMaxDroppableFrames;
120 else if (num < 0)
121 num = 0;
122 droppable_nframes_ = num;
123 for (unsigned int i = 0; i < droppable_nframes_; ++i)
124 droppable_frames_[i] = list[i];
125 }
126
clang-format3a826f12016-08-11 17:46:05 -0700127 unsigned int GetMismatchFrames() { return mismatch_nframes_; }
Deb Mukherjee01cafaa2013-01-15 06:43:35 -0800128
clang-format3a826f12016-08-11 17:46:05 -0700129 void SetPatternSwitch(int frame_switch) { pattern_switch_ = frame_switch; }
Marco2c6d9c52015-01-11 15:26:44 -0800130
Adrian Grangecc017ca2012-10-02 12:16:27 -0700131 private:
132 double psnr_;
133 unsigned int nframes_;
Deb Mukherjee01cafaa2013-01-15 06:43:35 -0800134 unsigned int error_nframes_;
135 unsigned int droppable_nframes_;
Marco2c6d9c52015-01-11 15:26:44 -0800136 unsigned int pattern_switch_;
Deb Mukherjee01cafaa2013-01-15 06:43:35 -0800137 double mismatch_psnr_;
138 unsigned int mismatch_nframes_;
139 unsigned int error_frames_[kMaxErrorFrames];
140 unsigned int droppable_frames_[kMaxDroppableFrames];
Adrian Grangecc017ca2012-10-02 12:16:27 -0700141 libvpx_test::TestMode encoding_mode_;
142};
143
Jim Bankoskia0b5ed62014-03-12 08:13:16 -0700144TEST_P(ErrorResilienceTestLarge, OnVersusOff) {
Adrian Grangecc017ca2012-10-02 12:16:27 -0700145 const vpx_rational timebase = { 33333333, 1000000000 };
146 cfg_.g_timebase = timebase;
147 cfg_.rc_target_bitrate = 2000;
Deb Mukherjee0d8723f2013-08-19 14:16:26 -0700148 cfg_.g_lag_in_frames = 10;
Adrian Grangecc017ca2012-10-02 12:16:27 -0700149
150 init_flags_ = VPX_CODEC_USE_PSNR;
151
152 libvpx_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288,
153 timebase.den, timebase.num, 0, 30);
154
155 // Error resilient mode OFF.
156 cfg_.g_error_resilient = 0;
157 ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
158 const double psnr_resilience_off = GetAveragePsnr();
159 EXPECT_GT(psnr_resilience_off, 25.0);
160
161 // Error resilient mode ON.
162 cfg_.g_error_resilient = 1;
163 ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
164 const double psnr_resilience_on = GetAveragePsnr();
165 EXPECT_GT(psnr_resilience_on, 25.0);
166
167 // Test that turning on error resilient mode hurts by 10% at most.
168 if (psnr_resilience_off > 0.0) {
169 const double psnr_ratio = psnr_resilience_on / psnr_resilience_off;
170 EXPECT_GE(psnr_ratio, 0.9);
171 EXPECT_LE(psnr_ratio, 1.1);
172 }
173}
174
Marco Paniconif61b9622014-02-24 18:14:50 -0800175// Check for successful decoding and no encoder/decoder mismatch
176// if we lose (i.e., drop before decoding) a set of droppable
177// frames (i.e., frames that don't update any reference buffers).
178// Check both isolated and consecutive loss.
Jim Bankoskia0b5ed62014-03-12 08:13:16 -0700179TEST_P(ErrorResilienceTestLarge, DropFramesWithoutRecovery) {
Deb Mukherjee01cafaa2013-01-15 06:43:35 -0800180 const vpx_rational timebase = { 33333333, 1000000000 };
181 cfg_.g_timebase = timebase;
John Koleszarc0490a52013-05-07 12:58:32 -0700182 cfg_.rc_target_bitrate = 500;
Deb Mukherjee0d8723f2013-08-19 14:16:26 -0700183 // FIXME(debargha): Fix this to work for any lag.
184 // Currently this test only works for lag = 0
185 cfg_.g_lag_in_frames = 0;
Deb Mukherjee01cafaa2013-01-15 06:43:35 -0800186
187 init_flags_ = VPX_CODEC_USE_PSNR;
188
189 libvpx_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288,
Marco Paniconif61b9622014-02-24 18:14:50 -0800190 timebase.den, timebase.num, 0, 40);
Deb Mukherjee01cafaa2013-01-15 06:43:35 -0800191
192 // Error resilient mode ON.
193 cfg_.g_error_resilient = 1;
Marco Paniconif61b9622014-02-24 18:14:50 -0800194 cfg_.kf_mode = VPX_KF_DISABLED;
Deb Mukherjee01cafaa2013-01-15 06:43:35 -0800195
Marco Paniconif61b9622014-02-24 18:14:50 -0800196 // Set an arbitrary set of error frames same as droppable frames.
197 // In addition to isolated loss/drop, add a long consecutive series
198 // (of size 9) of dropped frames.
199 unsigned int num_droppable_frames = 11;
clang-format3a826f12016-08-11 17:46:05 -0700200 unsigned int droppable_frame_list[] = { 5, 16, 22, 23, 24, 25,
201 26, 27, 28, 29, 30 };
Deb Mukherjee01cafaa2013-01-15 06:43:35 -0800202 SetDroppableFrames(num_droppable_frames, droppable_frame_list);
203 SetErrorFrames(num_droppable_frames, droppable_frame_list);
204 ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
205 // Test that no mismatches have been found
clang-format3a826f12016-08-11 17:46:05 -0700206 std::cout << " Mismatch frames: " << GetMismatchFrames() << "\n";
207 EXPECT_EQ(GetMismatchFrames(), (unsigned int)0);
Deb Mukherjee01cafaa2013-01-15 06:43:35 -0800208
Marco Paniconif61b9622014-02-24 18:14:50 -0800209 // Reset previously set of error/droppable frames.
Deb Mukherjee01cafaa2013-01-15 06:43:35 -0800210 Reset();
211
John Koleszar9fba0342013-05-07 13:02:45 -0700212#if 0
213 // TODO(jkoleszar): This test is disabled for the time being as too
214 // sensitive. It's not clear how to set a reasonable threshold for
215 // this behavior.
216
Deb Mukherjee01cafaa2013-01-15 06:43:35 -0800217 // Now set an arbitrary set of error frames that are non-droppable
218 unsigned int num_error_frames = 3;
219 unsigned int error_frame_list[] = {3, 10, 20};
220 SetErrorFrames(num_error_frames, error_frame_list);
221 ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
John Koleszar9fba0342013-05-07 13:02:45 -0700222
Deb Mukherjee01cafaa2013-01-15 06:43:35 -0800223 // Test that dropping an arbitrary set of inter frames does not hurt too much
224 // Note the Average Mismatch PSNR is the average of the PSNR between
225 // decoded frame and encoder's version of the same frame for all frames
226 // with mismatch.
227 const double psnr_resilience_mismatch = GetAverageMismatchPsnr();
228 std::cout << " Mismatch PSNR: "
229 << psnr_resilience_mismatch << "\n";
230 EXPECT_GT(psnr_resilience_mismatch, 20.0);
John Koleszar9fba0342013-05-07 13:02:45 -0700231#endif
Deb Mukherjee01cafaa2013-01-15 06:43:35 -0800232}
233
James Zerncc73e1f2016-08-08 15:09:30 -0700234VP10_INSTANTIATE_TEST_CASE(ErrorResilienceTestLarge, ONE_PASS_TEST_MODES);
Adrian Grangecc017ca2012-10-02 12:16:27 -0700235} // namespace