#include "pch.h" #include "api/video/video_frame.h" #include "cap_interface.h" #include "../common/comm.h" #include "../common/iobuffer.h" #include "../common/remote_def.h" #include "remote.pb.h" #include #include "peer_connection.h" #include "socket_remote.h" #include "remote_window.h" #include "message_queue.h" #include "libyuv/convert_argb.h" #include "libyuv/planar_functions.h" CRemoteWindow::CRemoteWindow(EgoType type, CMessageQueue* q, RenderPosition pos, IRender* render):_pos(pos),_message(q), _render(render),_type(type) { } void CRemoteWindow::OnNotifyRep(int32_t peer) { int32_t width = 1280; int32_t height = 720; //ChannelType channel = (ChannelType)(_pos - RenderPosition::FRONT); _peerconnection = std::make_unique(_message->GetRemoteClient()); SetRenderWindow(width, height); InitPeerConnection(peer); _peerconnection->CreateOffer(); } void CRemoteWindow::OnNotifyAnswer(CIOBuffer* pBuffer) { AnswerDesc* desc = (AnswerDesc*)pBuffer->Buffer; const char* type = desc->type; const char* sdp = desc->sdp; _peerconnection->SetRemoteDescription(type, sdp); pBuffer->Release(__FILE__, __LINE__); // DelayDataChannel(); } void CRemoteWindow::SetRenderWindow(int32_t width, int32_t height) { ZeroMemory(&bmi_, sizeof(bmi_)); bmi_.bmiHeader.biSize = sizeof(BITMAPINFOHEADER); bmi_.bmiHeader.biPlanes = 1; bmi_.bmiHeader.biBitCount = 32; bmi_.bmiHeader.biCompression = BI_RGB; /*if (_pos == RenderPosition::LMIRROR || _pos == RenderPosition::RMIRROR) { bmi_.bmiHeader.biWidth = height; bmi_.bmiHeader.biHeight = -width; } else { bmi_.bmiHeader.biWidth = width; bmi_.bmiHeader.biHeight = -height; } */ bmi_.bmiHeader.biSizeImage = width * height * (bmi_.bmiHeader.biBitCount >> 3); image_.reset(new uint8_t[bmi_.bmiHeader.biSizeImage]); // if (_pos == RenderPosition::LMIRROR || _pos == RenderPosition::RMIRROR) // _rotate.reset(new uint8_t[bmi_.bmiHeader.biSizeImage]); } void CRemoteWindow::SetSize(int32_t width, int32_t height) { if (width == bmi_.bmiHeader.biWidth && height == -bmi_.bmiHeader.biHeight) { return; } // if (m_Pos == CameraPosition::CAR_LEFT || m_Pos == CameraPosition::CAR_RIGHT) // { // bmi_.bmiHeader.biWidth = height; // bmi_.bmiHeader.biHeight = -width; // } // else { bmi_.bmiHeader.biWidth = width; bmi_.bmiHeader.biHeight = -height; } bmi_.bmiHeader.biSizeImage = width * height * (bmi_.bmiHeader.biBitCount >> 3); image_.reset(new uint8_t[bmi_.bmiHeader.biSizeImage]); } void CRemoteWindow::InitPeerConnection(int32_t peer) { if (_peerconnection != nullptr) { _peerconnection->Initialize(peer,_pos); // if(_local->IsAR()==false) // _peerconnection->AddLocalArgb32VideoFrameReady(&CEgoWindow::LocalFrameCallback, this); if(_type==EgoType::Monitor) _peerconnection->AddRemoteVideoFrameReady(&CRemoteWindow::RemoteFrameCallback, this); else { _video = _peerconnection->CreateOverlayVideo(); _peerconnection->AddLocalVideoTrack(_video); } } _start = true; } void CRemoteWindow::OnNotifyReq(int32_t index) { assert(_type == EgoType::User); _peerconnection = std::make_unique(_message->GetRemoteClient()); _message->GetRemoteClient()->WriteVideoRep(_peer, remote::VideoDesc::OK, index); } void CRemoteWindow::OnNotifyCandidate(CIOBuffer* pBuffer) { CandidateDesc* desc = (CandidateDesc*)pBuffer->Buffer; auto candidate = desc->candidate; auto sdp_mid = desc->sdp_mid; auto sdp_mline_index = desc->sdp_mline_index; _peerconnection->AddIceCandidate(candidate, sdp_mline_index, sdp_mid); pBuffer->Release(__FILE__, __LINE__); } void CRemoteWindow::OnNotifyOffer(CIOBuffer* pBuffer) { assert(_type == EgoType::User); OfferDesc* desc = (OfferDesc*)pBuffer->Buffer; auto type = desc->type; auto sdp = desc->sdp; InitPeerConnection(_peer); _peerconnection->SetRemoteDescription(type, sdp); _peerconnection->CreateAnswer(); pBuffer->Release(__FILE__, __LINE__); } void CRemoteWindow::OnAskVideoReq() { Sleep(200); _message->GetRemoteClient()->WriteVideoReq(_peer, _pos); } void CRemoteWindow::RemoteFrameCallback(void* user_data, const webrtc::VideoFrame& frame) { auto ptr = static_cast(user_data); rtc::scoped_refptr buffer( frame.video_frame_buffer()); const int width = frame.width(); const int height = frame.height(); if (buffer->type() != webrtc::VideoFrameBuffer::Type::kI420A) { rtc::scoped_refptr i420_buffer = buffer->ToI420(); const uint8_t* yptr = i420_buffer->DataY(); const uint8_t* uptr = i420_buffer->DataU(); const uint8_t* vptr = i420_buffer->DataV(); const uint8_t* aptr = nullptr; ptr->OnArgb32FrameReady(yptr, i420_buffer->StrideY(), uptr, i420_buffer->StrideU(), vptr, i420_buffer->StrideV(), width * 4, width, height); } else { const webrtc::I420ABufferInterface* i420a_buffer = buffer->GetI420A(); const uint8_t* yptr = i420a_buffer->DataY(); const uint8_t* uptr = i420a_buffer->DataU(); const uint8_t* vptr = i420a_buffer->DataV(); const uint8_t* aptr = i420a_buffer->DataA(); ptr->OnArgb32FrameReady(yptr, i420a_buffer->StrideY(), uptr, i420a_buffer->StrideU(), vptr, i420a_buffer->StrideV(), width * 4, width, height); } } void CRemoteWindow::OnArgb32FrameReady(const uint8_t* yptr, int32_t strideY, const uint8_t* uptr, int32_t strideU, const uint8_t* vptr, int32_t strideV, const int32_t stride, const int width, const int height) { { // int32_t*pixels = (int32_t*)data; //std::lock_guard < std::mutex> l(buffer_lock_); if (!_start) return; { SetSize(width, height); libyuv::I420ToARGB(yptr, strideY, uptr, strideU, vptr, strideV, image_.get(), stride, width, height); } } _render->OnRender(image_, width, height); } void CRemoteWindow::OverlayVideo(const webrtc::VideoFrame& frame) { assert(_type == EgoType::User); if (_start&&_video != nullptr) _video->OnVideoData(frame); } void CRemoteWindow::OnNotifyLeave() { _start = false; // _channelcreated = false; _peer = -1; /*if (_pos == RenderPosition::FRONT) { _peerconnection->RemoveLocalAudioTrack(); } */ if (_peerconnection != nullptr) { _peerconnection->Close(); _peerconnection.reset(); _peerconnection = nullptr; } image_ = nullptr; if (_type==EgoType::Monitor&& _render != nullptr) _render->OnRender(image_, 0, 0); else if (_type == EgoType::User) _video = nullptr; } void CRemoteWindow::SetPeer(int32_t peer) { _peer = peer; }