/* * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #ifndef VIDEO_VIDEO_RECEIVE_STREAM_H_ #define VIDEO_VIDEO_RECEIVE_STREAM_H_ #include #include #include "api/task_queue/task_queue_factory.h" #include "api/video/recordable_encoded_frame.h" #include "call/rtp_packet_sink_interface.h" #include "call/syncable.h" #include "call/video_receive_stream.h" #include "modules/rtp_rtcp/include/flexfec_receiver.h" #include "modules/rtp_rtcp/source/source_tracker.h" #include "modules/video_coding/frame_buffer2.h" #include "modules/video_coding/video_receiver2.h" #include "rtc_base/synchronization/mutex.h" #include "rtc_base/synchronization/sequence_checker.h" #include "rtc_base/system/no_unique_address.h" #include "rtc_base/task_queue.h" #include "system_wrappers/include/clock.h" #include "video/receive_statistics_proxy.h" #include "video/rtp_streams_synchronizer.h" #include "video/rtp_video_stream_receiver.h" #include "video/transport_adapter.h" #include "video/video_stream_decoder.h" namespace webrtc { class CallStats; class ProcessThread; class RtpStreamReceiverInterface; class RtpStreamReceiverControllerInterface; class RtxReceiveStream; class VCMTiming; namespace internal { class VideoReceiveStream : public webrtc::VideoReceiveStream, public rtc::VideoSinkInterface, public NackSender, public video_coding::OnCompleteFrameCallback, public Syncable, public CallStatsObserver { public: // The default number of milliseconds to pass before re-requesting a key frame // to be sent. static constexpr int kMaxWaitForKeyFrameMs = 200; VideoReceiveStream(TaskQueueFactory* task_queue_factory, RtpStreamReceiverControllerInterface* receiver_controller, int num_cpu_cores, PacketRouter* packet_router, VideoReceiveStream::Config config, ProcessThread* process_thread, CallStats* call_stats, Clock* clock, VCMTiming* timing); VideoReceiveStream(TaskQueueFactory* task_queue_factory, RtpStreamReceiverControllerInterface* receiver_controller, int num_cpu_cores, PacketRouter* packet_router, VideoReceiveStream::Config config, ProcessThread* process_thread, CallStats* call_stats, Clock* clock); ~VideoReceiveStream() override; const Config& config() const { return config_; } void SignalNetworkState(NetworkState state); bool DeliverRtcp(const uint8_t* packet, size_t length); void SetSync(Syncable* audio_syncable); // Implements webrtc::VideoReceiveStream. void Start() override; void Stop() override; webrtc::VideoReceiveStream::Stats GetStats() const override; void AddSecondarySink(RtpPacketSinkInterface* sink) override; void RemoveSecondarySink(const RtpPacketSinkInterface* sink) override; // SetBaseMinimumPlayoutDelayMs and GetBaseMinimumPlayoutDelayMs are called // from webrtc/api level and requested by user code. For e.g. blink/js layer // in Chromium. bool SetBaseMinimumPlayoutDelayMs(int delay_ms) override; int GetBaseMinimumPlayoutDelayMs() const override; void SetFrameDecryptor( rtc::scoped_refptr frame_decryptor) override; void SetDepacketizerToDecoderFrameTransformer( rtc::scoped_refptr frame_transformer) override; // Implements rtc::VideoSinkInterface. void OnFrame(const VideoFrame& video_frame) override; // Implements NackSender. // For this particular override of the interface, // only (buffering_allowed == true) is acceptable. void SendNack(const std::vector& sequence_numbers, bool buffering_allowed) override; // Implements video_coding::OnCompleteFrameCallback. void OnCompleteFrame( std::unique_ptr frame) override; // Implements CallStatsObserver::OnRttUpdate void OnRttUpdate(int64_t avg_rtt_ms, int64_t max_rtt_ms) override; // Implements Syncable. uint32_t id() const override; absl::optional GetInfo() const override; bool GetPlayoutRtpTimestamp(uint32_t* rtp_timestamp, int64_t* time_ms) const override; void SetEstimatedPlayoutNtpTimestampMs(int64_t ntp_timestamp_ms, int64_t time_ms) override; // SetMinimumPlayoutDelay is only called by A/V sync. bool SetMinimumPlayoutDelay(int delay_ms) override; std::vector GetSources() const override; RecordingState SetAndGetRecordingState(RecordingState state, bool generate_key_frame) override; void GenerateKeyFrame() override; private: int64_t GetWaitMs() const; void StartNextDecode() RTC_RUN_ON(decode_queue_); void HandleEncodedFrame(std::unique_ptr frame) RTC_RUN_ON(decode_queue_); void HandleFrameBufferTimeout() RTC_RUN_ON(decode_queue_); void UpdatePlayoutDelays() const RTC_EXCLUSIVE_LOCKS_REQUIRED(playout_delay_lock_); void RequestKeyFrame(int64_t timestamp_ms) RTC_RUN_ON(decode_queue_); void HandleKeyFrameGeneration(bool received_frame_is_keyframe, int64_t now_ms) RTC_RUN_ON(decode_queue_); bool IsReceivingKeyFrame(int64_t timestamp_ms) const RTC_RUN_ON(decode_queue_); void UpdateHistograms(); RTC_NO_UNIQUE_ADDRESS SequenceChecker worker_sequence_checker_; RTC_NO_UNIQUE_ADDRESS SequenceChecker module_process_sequence_checker_; RTC_NO_UNIQUE_ADDRESS SequenceChecker network_sequence_checker_; TaskQueueFactory* const task_queue_factory_; TransportAdapter transport_adapter_; const VideoReceiveStream::Config config_; const int num_cpu_cores_; ProcessThread* const process_thread_; Clock* const clock_; CallStats* const call_stats_; bool decoder_running_ RTC_GUARDED_BY(worker_sequence_checker_) = false; bool decoder_stopped_ RTC_GUARDED_BY(decode_queue_) = true; SourceTracker source_tracker_; ReceiveStatisticsProxy stats_proxy_; // Shared by media and rtx stream receivers, since the latter has no RtpRtcp // module of its own. const std::unique_ptr rtp_receive_statistics_; std::unique_ptr timing_; // Jitter buffer experiment. VideoReceiver2 video_receiver_; std::unique_ptr> incoming_video_stream_; RtpVideoStreamReceiver rtp_video_stream_receiver_; std::unique_ptr video_stream_decoder_; RtpStreamsSynchronizer rtp_stream_sync_; // TODO(nisse, philipel): Creation and ownership of video encoders should be // moved to the new VideoStreamDecoder. std::vector> video_decoders_; // Members for the new jitter buffer experiment. std::unique_ptr frame_buffer_; std::unique_ptr media_receiver_; std::unique_ptr rtx_receive_stream_; std::unique_ptr rtx_receiver_; // Whenever we are in an undecodable state (stream has just started or due to // a decoding error) we require a keyframe to restart the stream. bool keyframe_required_ = true; // If we have successfully decoded any frame. bool frame_decoded_ = false; int64_t last_keyframe_request_ms_ = 0; int64_t last_complete_frame_time_ms_ = 0; // Keyframe request intervals are configurable through field trials. const int max_wait_for_keyframe_ms_; const int max_wait_for_frame_ms_; mutable Mutex playout_delay_lock_; // All of them tries to change current min_playout_delay on |timing_| but // source of the change request is different in each case. Among them the // biggest delay is used. -1 means use default value from the |timing_|. // // Minimum delay as decided by the RTP playout delay extension. int frame_minimum_playout_delay_ms_ RTC_GUARDED_BY(playout_delay_lock_) = -1; // Minimum delay as decided by the setLatency function in "webrtc/api". int base_minimum_playout_delay_ms_ RTC_GUARDED_BY(playout_delay_lock_) = -1; // Minimum delay as decided by the A/V synchronization feature. int syncable_minimum_playout_delay_ms_ RTC_GUARDED_BY(playout_delay_lock_) = -1; // Maximum delay as decided by the RTP playout delay extension. int frame_maximum_playout_delay_ms_ RTC_GUARDED_BY(playout_delay_lock_) = -1; // Function that is triggered with encoded frames, if not empty. std::function encoded_frame_buffer_function_ RTC_GUARDED_BY(decode_queue_); // Set to true while we're requesting keyframes but not yet received one. bool keyframe_generation_requested_ RTC_GUARDED_BY(decode_queue_) = false; // Defined last so they are destroyed before all other members. rtc::TaskQueue decode_queue_; }; } // namespace internal } // namespace webrtc #endif // VIDEO_VIDEO_RECEIVE_STREAM_H_