video_analyzer.h 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320
  1. /*
  2. * Copyright 2018 The WebRTC project authors. All Rights Reserved.
  3. *
  4. * Use of this source code is governed by a BSD-style license
  5. * that can be found in the LICENSE file in the root of the source
  6. * tree. An additional intellectual property rights grant can be found
  7. * in the file PATENTS. All contributing project authors may
  8. * be found in the AUTHORS file in the root of the source tree.
  9. */
  10. #ifndef VIDEO_VIDEO_ANALYZER_H_
  11. #define VIDEO_VIDEO_ANALYZER_H_
  12. #include <deque>
  13. #include <map>
  14. #include <memory>
  15. #include <string>
  16. #include <vector>
  17. #include "api/task_queue/task_queue_base.h"
  18. #include "api/video/video_source_interface.h"
  19. #include "modules/rtp_rtcp/source/rtp_packet.h"
  20. #include "modules/rtp_rtcp/source/video_rtp_depacketizer.h"
  21. #include "rtc_base/event.h"
  22. #include "rtc_base/numerics/running_statistics.h"
  23. #include "rtc_base/platform_thread.h"
  24. #include "rtc_base/synchronization/mutex.h"
  25. #include "rtc_base/time_utils.h"
  26. #include "test/layer_filtering_transport.h"
  27. #include "test/rtp_file_writer.h"
  28. #include "test/testsupport/perf_test.h"
  29. namespace webrtc {
  30. class VideoAnalyzer : public PacketReceiver,
  31. public Transport,
  32. public rtc::VideoSinkInterface<VideoFrame> {
  33. public:
  34. using Statistics = webrtc_impl::RunningStatistics<double>;
  35. VideoAnalyzer(test::LayerFilteringTransport* transport,
  36. const std::string& test_label,
  37. double avg_psnr_threshold,
  38. double avg_ssim_threshold,
  39. int duration_frames,
  40. TimeDelta test_duration,
  41. FILE* graph_data_output_file,
  42. const std::string& graph_title,
  43. uint32_t ssrc_to_analyze,
  44. uint32_t rtx_ssrc_to_analyze,
  45. size_t selected_stream,
  46. int selected_sl,
  47. int selected_tl,
  48. bool is_quick_test_enabled,
  49. Clock* clock,
  50. std::string rtp_dump_name,
  51. TaskQueueBase* task_queue);
  52. ~VideoAnalyzer();
  53. virtual void SetReceiver(PacketReceiver* receiver);
  54. void SetSource(rtc::VideoSourceInterface<VideoFrame>* video_source,
  55. bool respect_sink_wants);
  56. void SetCall(Call* call);
  57. void SetSendStream(VideoSendStream* stream);
  58. void SetReceiveStream(VideoReceiveStream* stream);
  59. void SetAudioReceiveStream(AudioReceiveStream* recv_stream);
  60. rtc::VideoSinkInterface<VideoFrame>* InputInterface();
  61. rtc::VideoSourceInterface<VideoFrame>* OutputInterface();
  62. DeliveryStatus DeliverPacket(MediaType media_type,
  63. rtc::CopyOnWriteBuffer packet,
  64. int64_t packet_time_us) override;
  65. void PreEncodeOnFrame(const VideoFrame& video_frame);
  66. void PostEncodeOnFrame(size_t stream_id, uint32_t timestamp);
  67. bool SendRtp(const uint8_t* packet,
  68. size_t length,
  69. const PacketOptions& options) override;
  70. bool SendRtcp(const uint8_t* packet, size_t length) override;
  71. void OnFrame(const VideoFrame& video_frame) override;
  72. void Wait();
  73. void StartMeasuringCpuProcessTime();
  74. void StopMeasuringCpuProcessTime();
  75. void StartExcludingCpuThreadTime() RTC_LOCKS_EXCLUDED(cpu_measurement_lock_);
  76. void StopExcludingCpuThreadTime() RTC_LOCKS_EXCLUDED(cpu_measurement_lock_);
  77. double GetCpuUsagePercent() RTC_LOCKS_EXCLUDED(cpu_measurement_lock_);
  78. test::LayerFilteringTransport* const transport_;
  79. PacketReceiver* receiver_;
  80. private:
  81. struct FrameComparison {
  82. FrameComparison();
  83. FrameComparison(const VideoFrame& reference,
  84. const VideoFrame& render,
  85. bool dropped,
  86. int64_t input_time_ms,
  87. int64_t send_time_ms,
  88. int64_t recv_time_ms,
  89. int64_t render_time_ms,
  90. size_t encoded_frame_size);
  91. FrameComparison(bool dropped,
  92. int64_t input_time_ms,
  93. int64_t send_time_ms,
  94. int64_t recv_time_ms,
  95. int64_t render_time_ms,
  96. size_t encoded_frame_size);
  97. absl::optional<VideoFrame> reference;
  98. absl::optional<VideoFrame> render;
  99. bool dropped;
  100. int64_t input_time_ms;
  101. int64_t send_time_ms;
  102. int64_t recv_time_ms;
  103. int64_t render_time_ms;
  104. size_t encoded_frame_size;
  105. };
  106. struct Sample {
  107. Sample(int dropped,
  108. int64_t input_time_ms,
  109. int64_t send_time_ms,
  110. int64_t recv_time_ms,
  111. int64_t render_time_ms,
  112. size_t encoded_frame_size,
  113. double psnr,
  114. double ssim);
  115. int dropped;
  116. int64_t input_time_ms;
  117. int64_t send_time_ms;
  118. int64_t recv_time_ms;
  119. int64_t render_time_ms;
  120. size_t encoded_frame_size;
  121. double psnr;
  122. double ssim;
  123. };
  124. // Implements VideoSinkInterface to receive captured frames from a
  125. // FrameGeneratorCapturer. Implements VideoSourceInterface to be able to act
  126. // as a source to VideoSendStream.
  127. // It forwards all input frames to the VideoAnalyzer for later comparison and
  128. // forwards the captured frames to the VideoSendStream.
  129. class CapturedFrameForwarder : public rtc::VideoSinkInterface<VideoFrame>,
  130. public rtc::VideoSourceInterface<VideoFrame> {
  131. public:
  132. CapturedFrameForwarder(VideoAnalyzer* analyzer,
  133. Clock* clock,
  134. int frames_to_capture,
  135. TimeDelta test_duration);
  136. void SetSource(rtc::VideoSourceInterface<VideoFrame>* video_source);
  137. private:
  138. void OnFrame(const VideoFrame& video_frame)
  139. RTC_LOCKS_EXCLUDED(lock_) override;
  140. // Called when |send_stream_.SetSource()| is called.
  141. void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
  142. const rtc::VideoSinkWants& wants)
  143. RTC_LOCKS_EXCLUDED(lock_) override;
  144. // Called by |send_stream_| when |send_stream_.SetSource()| is called.
  145. void RemoveSink(rtc::VideoSinkInterface<VideoFrame>* sink)
  146. RTC_LOCKS_EXCLUDED(lock_) override;
  147. VideoAnalyzer* const analyzer_;
  148. Mutex lock_;
  149. rtc::VideoSinkInterface<VideoFrame>* send_stream_input_
  150. RTC_GUARDED_BY(lock_);
  151. VideoSourceInterface<VideoFrame>* video_source_;
  152. Clock* clock_;
  153. int captured_frames_ RTC_GUARDED_BY(lock_);
  154. const int frames_to_capture_;
  155. const Timestamp test_end_;
  156. };
  157. struct FrameWithPsnr {
  158. double psnr;
  159. VideoFrame frame;
  160. };
  161. bool IsInSelectedSpatialAndTemporalLayer(const RtpPacket& rtp_packet);
  162. void AddFrameComparison(const VideoFrame& reference,
  163. const VideoFrame& render,
  164. bool dropped,
  165. int64_t render_time_ms)
  166. RTC_EXCLUSIVE_LOCKS_REQUIRED(lock_);
  167. void PollStats() RTC_LOCKS_EXCLUDED(comparison_lock_);
  168. static void FrameComparisonThread(void* obj);
  169. bool CompareFrames();
  170. bool PopComparison(FrameComparison* comparison);
  171. // Increment counter for number of frames received for comparison.
  172. void FrameRecorded() RTC_EXCLUSIVE_LOCKS_REQUIRED(comparison_lock_);
  173. // Returns true if all frames to be compared have been taken from the queue.
  174. bool AllFramesRecorded() RTC_LOCKS_EXCLUDED(comparison_lock_);
  175. bool AllFramesRecordedLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(comparison_lock_);
  176. // Increase count of number of frames processed. Returns true if this was the
  177. // last frame to be processed.
  178. bool FrameProcessed() RTC_LOCKS_EXCLUDED(comparison_lock_);
  179. void PrintResults() RTC_LOCKS_EXCLUDED(lock_, comparison_lock_);
  180. void PerformFrameComparison(const FrameComparison& comparison)
  181. RTC_LOCKS_EXCLUDED(comparison_lock_);
  182. void PrintResult(const char* result_type,
  183. Statistics stats,
  184. const char* unit,
  185. webrtc::test::ImproveDirection improve_direction);
  186. void PrintResultWithExternalMean(
  187. const char* result_type,
  188. double mean,
  189. Statistics stats,
  190. const char* unit,
  191. webrtc::test::ImproveDirection improve_direction);
  192. void PrintSamplesToFile(void) RTC_LOCKS_EXCLUDED(comparison_lock_);
  193. void AddCapturedFrameForComparison(const VideoFrame& video_frame)
  194. RTC_LOCKS_EXCLUDED(lock_, comparison_lock_);
  195. Call* call_;
  196. VideoSendStream* send_stream_;
  197. VideoReceiveStream* receive_stream_;
  198. AudioReceiveStream* audio_receive_stream_;
  199. CapturedFrameForwarder captured_frame_forwarder_;
  200. const std::string test_label_;
  201. FILE* const graph_data_output_file_;
  202. const std::string graph_title_;
  203. const uint32_t ssrc_to_analyze_;
  204. const uint32_t rtx_ssrc_to_analyze_;
  205. const size_t selected_stream_;
  206. const int selected_sl_;
  207. const int selected_tl_;
  208. Mutex comparison_lock_;
  209. std::vector<Sample> samples_ RTC_GUARDED_BY(comparison_lock_);
  210. Statistics sender_time_ RTC_GUARDED_BY(comparison_lock_);
  211. Statistics receiver_time_ RTC_GUARDED_BY(comparison_lock_);
  212. Statistics network_time_ RTC_GUARDED_BY(comparison_lock_);
  213. Statistics psnr_ RTC_GUARDED_BY(comparison_lock_);
  214. Statistics ssim_ RTC_GUARDED_BY(comparison_lock_);
  215. Statistics end_to_end_ RTC_GUARDED_BY(comparison_lock_);
  216. Statistics rendered_delta_ RTC_GUARDED_BY(comparison_lock_);
  217. Statistics encoded_frame_size_ RTC_GUARDED_BY(comparison_lock_);
  218. Statistics encode_frame_rate_ RTC_GUARDED_BY(comparison_lock_);
  219. Statistics encode_time_ms_ RTC_GUARDED_BY(comparison_lock_);
  220. Statistics encode_usage_percent_ RTC_GUARDED_BY(comparison_lock_);
  221. double mean_decode_time_ms_ RTC_GUARDED_BY(comparison_lock_);
  222. Statistics decode_time_ms_ RTC_GUARDED_BY(comparison_lock_);
  223. Statistics decode_time_max_ms_ RTC_GUARDED_BY(comparison_lock_);
  224. Statistics media_bitrate_bps_ RTC_GUARDED_BY(comparison_lock_);
  225. Statistics fec_bitrate_bps_ RTC_GUARDED_BY(comparison_lock_);
  226. Statistics send_bandwidth_bps_ RTC_GUARDED_BY(comparison_lock_);
  227. Statistics memory_usage_ RTC_GUARDED_BY(comparison_lock_);
  228. Statistics audio_expand_rate_ RTC_GUARDED_BY(comparison_lock_);
  229. Statistics audio_accelerate_rate_ RTC_GUARDED_BY(comparison_lock_);
  230. Statistics audio_jitter_buffer_ms_ RTC_GUARDED_BY(comparison_lock_);
  231. Statistics pixels_ RTC_GUARDED_BY(comparison_lock_);
  232. // Rendered frame with worst PSNR is saved for further analysis.
  233. absl::optional<FrameWithPsnr> worst_frame_ RTC_GUARDED_BY(comparison_lock_);
  234. // Freeze metrics.
  235. Statistics time_between_freezes_ RTC_GUARDED_BY(comparison_lock_);
  236. uint32_t freeze_count_ RTC_GUARDED_BY(comparison_lock_);
  237. uint32_t total_freezes_duration_ms_ RTC_GUARDED_BY(comparison_lock_);
  238. uint32_t total_frames_duration_ms_ RTC_GUARDED_BY(comparison_lock_);
  239. double sum_squared_frame_durations_ RTC_GUARDED_BY(comparison_lock_);
  240. double decode_frame_rate_ RTC_GUARDED_BY(comparison_lock_);
  241. double render_frame_rate_ RTC_GUARDED_BY(comparison_lock_);
  242. size_t last_fec_bytes_;
  243. Mutex lock_ RTC_ACQUIRED_BEFORE(comparison_lock_)
  244. RTC_ACQUIRED_BEFORE(cpu_measurement_lock_);
  245. const int frames_to_process_;
  246. const Timestamp test_end_;
  247. int frames_recorded_ RTC_GUARDED_BY(comparison_lock_);
  248. int frames_processed_ RTC_GUARDED_BY(comparison_lock_);
  249. int captured_frames_ RTC_GUARDED_BY(comparison_lock_);
  250. int dropped_frames_ RTC_GUARDED_BY(comparison_lock_);
  251. int dropped_frames_before_first_encode_ RTC_GUARDED_BY(lock_);
  252. int dropped_frames_before_rendering_ RTC_GUARDED_BY(lock_);
  253. int64_t last_render_time_ RTC_GUARDED_BY(comparison_lock_);
  254. int64_t last_render_delta_ms_ RTC_GUARDED_BY(comparison_lock_);
  255. int64_t last_unfreeze_time_ms_ RTC_GUARDED_BY(comparison_lock_);
  256. uint32_t rtp_timestamp_delta_ RTC_GUARDED_BY(lock_);
  257. Mutex cpu_measurement_lock_;
  258. int64_t cpu_time_ RTC_GUARDED_BY(cpu_measurement_lock_);
  259. int64_t wallclock_time_ RTC_GUARDED_BY(cpu_measurement_lock_);
  260. std::deque<VideoFrame> frames_ RTC_GUARDED_BY(lock_);
  261. absl::optional<VideoFrame> last_rendered_frame_ RTC_GUARDED_BY(lock_);
  262. rtc::TimestampWrapAroundHandler wrap_handler_ RTC_GUARDED_BY(lock_);
  263. std::map<int64_t, int64_t> send_times_ RTC_GUARDED_BY(lock_);
  264. std::map<int64_t, int64_t> recv_times_ RTC_GUARDED_BY(lock_);
  265. std::map<int64_t, size_t> encoded_frame_sizes_ RTC_GUARDED_BY(lock_);
  266. absl::optional<uint32_t> first_encoded_timestamp_ RTC_GUARDED_BY(lock_);
  267. absl::optional<uint32_t> first_sent_timestamp_ RTC_GUARDED_BY(lock_);
  268. const double avg_psnr_threshold_;
  269. const double avg_ssim_threshold_;
  270. bool is_quick_test_enabled_;
  271. std::vector<rtc::PlatformThread*> comparison_thread_pool_;
  272. rtc::Event comparison_available_event_;
  273. std::deque<FrameComparison> comparisons_ RTC_GUARDED_BY(comparison_lock_);
  274. bool quit_ RTC_GUARDED_BY(comparison_lock_);
  275. rtc::Event done_;
  276. std::unique_ptr<VideoRtpDepacketizer> vp8_depacketizer_;
  277. std::unique_ptr<VideoRtpDepacketizer> vp9_depacketizer_;
  278. std::unique_ptr<test::RtpFileWriter> rtp_file_writer_;
  279. Clock* const clock_;
  280. const int64_t start_ms_;
  281. TaskQueueBase* task_queue_;
  282. };
  283. } // namespace webrtc
  284. #endif // VIDEO_VIDEO_ANALYZER_H_