api.cpp 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566
  1. #include "pch.h"
  2. #include "../common/comm.h"
  3. #include "api.h"
  4. #include "callback.h"
  5. #include "api/create_peerconnection_factory.h"
  6. #include "video_frame_observer.h"
  7. #include "data_channel_observer.h"
  8. #include "audio_frame_observer.h"
  9. #include "peer_connection.h"
  10. #include "sdp_utils.h"
  11. #include "p2p/client/basic_port_allocator.h"
  12. #include <json/json.h>
  13. #include <fstream>
  14. #ifdef WEBRTC_LINUX
  15. //#include <opencv2/opencv.hpp>
  16. //#include <opencv2/core/core.hpp>
  17. //#include <opencv2/opencv.hpp>
  18. #include "capture_op.h"
  19. #include "gsml_capturer.h"
  20. #endif
  21. #ifdef WEBRTC_LINUX
  22. #include "sanitize_string.h"
  23. #include "capture_op.h"
  24. #include "gsml_capturer.h"
  25. #include "VideoScaler.h"
  26. #include "VideoFilter.h"
  27. #endif
  28. struct mrsEnumerator {
  29. virtual ~mrsEnumerator() = default;
  30. virtual void dispose() = 0;
  31. };
  32. rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface>
  33. g_peer_connection_factory;
  34. #ifdef WEBRTC_LINUX
  35. std::unique_ptr<webrtc::VideoDecoderFactory> g_video_decode_factory;
  36. #endif
  37. /// WebRTC worker thread.
  38. std::unique_ptr<rtc::Thread> g_worker_thread;
  39. // #ifdef WIN32
  40. // //rtc::Win32SocketServer w32_ss;
  41. // std::unique_ptr<rtc::Thread> g_network_thread;
  42. // #else
  43. // rtc::AutoSocketServerThread g_network_thread;
  44. // #endif
  45. /// WebRTC signaling thread.
  46. std::unique_ptr<rtc::Thread> g_signaling_thread;
  47. std::unordered_map<
  48. PeerConnectionHandle,
  49. rtc::scoped_refptr<PeerConnection>>
  50. g_peer_connection_map;
  51. /// Predefined name of the local video track.
  52. const std::string kLocalVideoLabel("local_video");
  53. /// Predefined name of the local audio track.
  54. const std::string kLocalAudioLabel("local_audio");
  55. const std::string kLocalDataChannel("local_channel");
  56. /// WebRTC worker thread.
  57. const std::unique_ptr<rtc::Thread>& GetWorkerThread() {
  58. return g_worker_thread;
  59. }
  60. /// WebRTC signaling thread.
  61. const std::unique_ptr<rtc::Thread>& GetSignalingThread() {
  62. return g_signaling_thread;
  63. }
  64. void mrsCloseEnum(mrsEnumHandle* handleRef) {
  65. if (handleRef) {
  66. if (auto& handle = *handleRef) {
  67. handle->dispose();
  68. delete handle;
  69. handle = nullptr;
  70. }
  71. }
  72. }
  73. void mrsEnumVideoCaptureDevicesAsync(
  74. mrsVideoCaptureDeviceEnumCallback callback,
  75. void* userData,
  76. mrsVideoCaptureDeviceEnumCompletedCallback completedCallback,
  77. void* completedCallbackUserData) {
  78. if (!callback) {
  79. return;
  80. }
  81. std::unique_ptr<webrtc::VideoCaptureModule::DeviceInfo> info(
  82. webrtc::VideoCaptureFactory::CreateDeviceInfo());
  83. if (!info) {
  84. if (completedCallback) {
  85. (*completedCallback)(completedCallbackUserData);
  86. }
  87. }
  88. int num_devices = info->NumberOfDevices();
  89. for (int i = 0; i < num_devices; ++i) {
  90. constexpr uint32_t kSize = 256;
  91. char name[kSize] = { 0 };
  92. char id[kSize] = { 0 };
  93. if (info->GetDeviceName(i, name, kSize, id, kSize) != -1) {
  94. (*callback)(id, name, userData);
  95. }
  96. }
  97. if (completedCallback) {
  98. (*completedCallback)(completedCallbackUserData);
  99. }
  100. }
  101. std::unique_ptr<webrtc::VideoEncoderFactory> CreateEncoderFactory(bool bNullCodec)
  102. {
  103. std::unique_ptr<webrtc::VideoEncoderFactory> factory;
  104. /*
  105. if (bNullCodec) {
  106. // factory=webrtc::CreateBuiltinVideoEncoderFactory();
  107. #ifdef WEBRTC_LINUX
  108. factory= std::make_unique<VideoEncoderFactory>();
  109. #endif
  110. }
  111. else {
  112. */
  113. factory=webrtc::CreateBuiltinVideoEncoderFactory();
  114. //}
  115. return factory;
  116. }
  117. std::unique_ptr<webrtc::VideoDecoderFactory> CreateDecoderFactory(bool nullCodec) {
  118. std::unique_ptr<webrtc::VideoDecoderFactory> factory;
  119. /*
  120. if (nullCodec) {
  121. #ifdef WEBRTC_LINUX
  122. //factory= webrtc::CreateBuiltinVideoDecoderFactory();
  123. factory = std::make_unique<VideoDecoderFactory>();
  124. #endif
  125. }
  126. else
  127. {
  128. factory= webrtc::CreateBuiltinVideoDecoderFactory();
  129. }
  130. */
  131. factory = webrtc::CreateBuiltinVideoDecoderFactory();
  132. return factory;
  133. }
  134. bool mrsWebrtcCreateFactory(bool nullCodec)
  135. {
  136. if (g_peer_connection_factory == nullptr) {
  137. g_worker_thread=(rtc::Thread::Create());
  138. g_worker_thread->Start();
  139. g_signaling_thread=(rtc::Thread::Create());
  140. g_signaling_thread->Start();
  141. #ifdef WEBRTC_LINUX
  142. g_video_decode_factory=CreateDecoderFactory(nullCodec);
  143. #endif
  144. g_peer_connection_factory = webrtc::CreatePeerConnectionFactory(
  145. nullptr, g_worker_thread.get(), g_signaling_thread.get(),
  146. nullptr, webrtc::CreateBuiltinAudioEncoderFactory(),
  147. webrtc::CreateBuiltinAudioDecoderFactory(),
  148. CreateEncoderFactory(nullCodec),
  149. /*
  150. std::unique_ptr<webrtc::VideoEncoderFactory>(
  151. new webrtc::MultiplexEncoderFactory(
  152. #ifdef WEBRTC_AARCH
  153. std::make_unique<webrtc::InternalEncoderFactory>()
  154. #else
  155. std::make_unique<webrtc::NvVideoEncoderFactory>()
  156. #endif
  157. )),
  158. */
  159. CreateDecoderFactory(nullCodec),
  160. /*
  161. std::unique_ptr<webrtc::VideoDecoderFactory>(
  162. new webrtc::MultiplexDecoderFactory(
  163. std::make_unique<webrtc::InternalDecoderFactory>())),*/
  164. nullptr, nullptr);
  165. }
  166. if (!g_peer_connection_factory.get()) {
  167. return false;
  168. }
  169. return true;
  170. }
  171. PeerConnectionHandle mrsPeerConnectionCreate() {
  172. // Ensure the factory exists
  173. // Setup the connection configuration
  174. webrtc::PeerConnectionInterface::RTCConfiguration config;
  175. config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan;
  176. /*
  177. Json::Value root;
  178. Json::Reader jsonReader;
  179. std::ifstream ifile("Config.json");
  180. int32_t Udpupper_port;
  181. int32_t Udplower_port;
  182. if (jsonReader.parse(ifile, root))
  183. {
  184. Udpupper_port = root["Udpupper_port"].asInt();
  185. Udplower_port = root["Udplower_port"].asInt();
  186. }
  187. */
  188. config.enable_dtls_srtp = false;
  189. config.enable_rtp_data_channel = true;
  190. webrtc::PeerConnectionInterface::IceServer server;
  191. server.uri = "stun:stun1.l.google.com:19302";
  192. config.servers.push_back(server);
  193. server.uri = "turn:58.34.98.12:3478";
  194. server.username = "ubuntu12";
  195. server.password = "ubuntu12@123";
  196. config.servers.push_back(server);
  197. config.combined_audio_video_bwe=true;
  198. //add-wfg
  199. std::unique_ptr<cricket::PortAllocator> port_allocator(new cricket::BasicPortAllocator(new rtc::BasicNetworkManager()));
  200. //port_allocator->SetPortRange(Udpupper_port, Udplower_port);
  201. port_allocator->SetPortRange(27504, 27525);
  202. config.enable_dtls_srtp = false; //< TODO - Should be true/unset for security
  203. //config.type = webrtc::PeerConnectionInterface::IceTransportsType::kAll;
  204. // Create the new peer connection
  205. rtc::scoped_refptr<PeerConnection> peer =
  206. new rtc::RefCountedObject<PeerConnection>();
  207. /* webrtc::PeersConnectionDependencies dependencies(peer);*/
  208. //add
  209. webrtc::PeerConnectionDependencies dependencies(std::move(peer));
  210. //×¢ÊÍ
  211. /*rtc::scoped_refptr<webrtc::PeerConnectionInterface> peer_connection =
  212. g_peer_connection_factory->CreatePeerConnection(config, nullptr, nullptr,
  213. peer.get());*/
  214. //add
  215. rtc::scoped_refptr<webrtc::PeerConnectionInterface> peer_connection =
  216. g_peer_connection_factory->CreatePeerConnection(config, std::move(port_allocator), nullptr, peer.get());
  217. if (peer_connection.get() == nullptr)
  218. return {};
  219. peer->SetPeerImpl(peer_connection);
  220. const PeerConnectionHandle handle{ peer.get() };
  221. g_peer_connection_map.insert({ handle, std::move(peer) });
  222. return handle;
  223. }
  224. void mrsPeerConnectionRegisterConnectedCallback(
  225. PeerConnectionHandle peerHandle,
  226. PeerConnectionConnectedCallback callback,
  227. void* user_data) {
  228. if (auto peer = static_cast<PeerConnection*>(peerHandle)) {
  229. peer->RegisterConnectedCallback(Callback<>{callback, user_data});
  230. }
  231. }
  232. void mrsPeerConnectionRegisterLocalSdpReadytoSendCallback(
  233. PeerConnectionHandle peerHandle,
  234. int32_t pid,
  235. int32_t index,
  236. PeerConnectionLocalSdpReadytoSendCallback callback,
  237. void* user_data) {
  238. if (auto peer = static_cast<PeerConnection*>(peerHandle)) {
  239. peer->RegisterLocalSdpReadytoSendCallback(
  240. Callback<int32_t,int32_t,const char*, const char*>{callback,user_data,pid,index});
  241. }
  242. }
  243. void mrsPeerConnectionRegisterIceCandidateReadytoSendCallback(
  244. PeerConnectionHandle peerHandle,
  245. int32_t pid,
  246. int32_t view,
  247. PeerConnectionIceCandidateReadytoSendCallback callback,
  248. void* user_data) {
  249. if (auto peer = static_cast<PeerConnection*>(peerHandle)) {
  250. peer->RegisterIceCandidateReadytoSendCallback(
  251. Callback<int32_t,int32_t,const char*, int, const char*>{callback, user_data,pid,view});
  252. }
  253. }
  254. void mrsPeerConnectionRegisterARGBLocalVideoFrameCallback(
  255. PeerConnectionHandle peerHandle,
  256. PeerConnectionARGBVideoFrameCallback callback,
  257. void* user_data) {
  258. if (auto peer = static_cast<PeerConnection*>(peerHandle)) {
  259. peer->RegisterLocalVideoFrameCallback(
  260. ARGBFrameReadyCallback{callback, user_data });
  261. }
  262. }
  263. void mrsPeerConnectionRegisterARGBRemoteVideoFrameCallback(
  264. PeerConnectionHandle peerHandle,
  265. PeerConnectionARGBVideoFrameCallback callback,
  266. void* user_data) {
  267. if (auto peer = static_cast<PeerConnection*>(peerHandle)) {
  268. peer->RegisterRemoteVideoFrameCallback(
  269. ARGBFrameReadyCallback{callback, user_data });
  270. }
  271. }
  272. void mrsPeerConnectionRegisterChannelCallback(PeerConnectionHandle peerHandle,
  273. PeerConnectionDataChannelMessageCallback message_callback, void* message_user_data,
  274. PeerConnectionDataChannelBufferingCallback buffering_callback, void* buffering_user_data,
  275. PeerConnectionDataChannelStateCallback state_callback,
  276. void* state_user_data) {
  277. if (auto peer = static_cast<PeerConnection*>(peerHandle))
  278. {
  279. peer->RegisterDataChannelCallback(
  280. DataChannelMessageCallback{ message_callback, message_user_data },
  281. DataChannelBufferingCallback{ buffering_callback, buffering_user_data },
  282. DataChannelStateCallback{ state_callback, state_user_data });
  283. }
  284. }
  285. bool mrsPeerConnectionAddIceCandidate(PeerConnectionHandle peerHandle,
  286. const char* sdp,
  287. const int sdp_mline_index,
  288. const char* sdp_mid) {
  289. if (auto peer = static_cast<PeerConnection*>(peerHandle)) {
  290. return peer->AddIceCandidate(sdp_mid, sdp_mline_index, sdp);
  291. }
  292. return false;
  293. }
  294. bool mrsPeerConnectionCreateOffer(PeerConnectionHandle peerHandle) {
  295. if (auto peer = static_cast<PeerConnection*>(peerHandle)) {
  296. return peer->CreateOffer();
  297. }
  298. return false;
  299. }
  300. bool mrsPeerConnectionCreateAnswer(PeerConnectionHandle peerHandle) {
  301. if (auto peer = static_cast<PeerConnection*>(peerHandle)) {
  302. return peer->CreateAnswer();
  303. }
  304. return false;
  305. }
  306. bool mrsPeerConnectionSetRemoteDescription(PeerConnectionHandle peerHandle,
  307. const char* type,
  308. const char* sdp) {
  309. if (auto peer = static_cast<PeerConnection*>(peerHandle)) {
  310. return peer->SetRemoteDescription(type, sdp);
  311. }
  312. return false;
  313. }
  314. void mrsPeerConnectionClose(PeerConnectionHandle* peerHandlePtr) {
  315. if (peerHandlePtr) {
  316. if (auto peer = static_cast<PeerConnection*>(*peerHandlePtr)) {
  317. auto it = g_peer_connection_map.find(peer);
  318. if (it != g_peer_connection_map.end()) {
  319. // This generally removes the last reference to the PeerConnection and
  320. // leads to its destruction, unless some background running task is
  321. // still using the connection.
  322. g_peer_connection_map.erase(it);
  323. if (g_peer_connection_map.empty()) {
  324. // Release the factory so that the threads are stopped and the DLL can
  325. // be unloaded. This is mandatory to be able to unload/reload in the
  326. // Unity Editor and be able to Play/Stop multiple times per Editor
  327. // process run.
  328. g_peer_connection_factory = nullptr;
  329. g_signaling_thread.reset();
  330. g_worker_thread.reset();
  331. }
  332. }
  333. }
  334. *peerHandlePtr = nullptr;
  335. }
  336. }
  337. #ifdef WEBRTC_LINUX
  338. bool mrsPeerConnectionAddLocalVideoTrack( PeerConnectionHandle peerHandle,
  339. RenderPosition type,
  340. int32_t index)
  341. {
  342. if(auto peer=static_cast<PeerConnection *>(peerHandle))
  343. {
  344. if (!g_peer_connection_factory)
  345. return false;
  346. std::unique_ptr<CaptureOp > op=std::make_unique<CaptureOp>(type,index);
  347. auto ptr=op.get();
  348. peer->RegisterCaptureOp(op);
  349. rtc::scoped_refptr<webrtc::VideoTrackSourceInterface> video_source = OpenGSMLCapture(ptr);
  350. if(!video_source) return false;
  351. rtc::scoped_refptr<webrtc::VideoTrackInterface> video_track =
  352. g_peer_connection_factory->CreateVideoTrack(kLocalVideoLabel+std::to_string(type),
  353. video_source);
  354. if (!video_track) {
  355. return false;
  356. }
  357. return peer->AddLocalVideoTrack(std::move(video_track), kLocalVideoLabel+std::to_string(type));
  358. }
  359. return false;
  360. }
  361. bool mrsPeerConnectionSwitchCapture(
  362. PeerConnectionHandle peerHandle,
  363. bool front
  364. )
  365. {
  366. if(auto peer=static_cast<PeerConnection *>(peerHandle))
  367. {
  368. peer->SwitchCapture(front);
  369. return true;
  370. }
  371. return false;
  372. }
  373. bool mrsPeerConnectionSetCtx(PeerConnectionHandle peerHandle,void * data)
  374. {
  375. if(auto peer=static_cast<PeerConnection *>(peerHandle))
  376. {
  377. peer->SetOtherCtx(data);
  378. return true;
  379. }
  380. return false;
  381. }
  382. void * mrsPeerConnectionCurrentCtx(PeerConnectionHandle peerHandle)
  383. {
  384. if(auto peer=static_cast<PeerConnection *>(peerHandle))
  385. {
  386. return peer->GetCurrentCtx();
  387. }
  388. return nullptr;
  389. }
  390. #endif
  391. bool mrsPeerConnectionAddLocalAudioTrack(PeerConnectionHandle peerHandle) {
  392. if (auto peer = static_cast<PeerConnection*>(peerHandle)) {
  393. if (!g_peer_connection_factory)
  394. return false;
  395. rtc::scoped_refptr<webrtc::AudioSourceInterface> audio_source =
  396. g_peer_connection_factory->CreateAudioSource(cricket::AudioOptions());
  397. if (!audio_source)
  398. return false;
  399. rtc::scoped_refptr<webrtc::AudioTrackInterface> audio_track =
  400. g_peer_connection_factory->CreateAudioTrack(kLocalAudioLabel,
  401. audio_source);
  402. if (!audio_track)
  403. return false;
  404. return peer->AddLocalAudioTrack(std::move(audio_track));
  405. }
  406. return false;
  407. }
  408. mrsResult mrsPeerConnectionAddDataChannel(
  409. PeerConnectionHandle peerHandle,
  410. bool ordered,
  411. bool reliable//,
  412. // PeerConnectionDataChannelMessageCallback message_callback,
  413. // void* message_user_data,
  414. // PeerConnectionDataChannelBufferingCallback buffering_callback,
  415. // void* buffering_user_data,
  416. // PeerConnectionDataChannelStateCallback state_callback,
  417. // void* state_user_data
  418. )
  419. {
  420. if (auto peer = static_cast<PeerConnection*>(peerHandle)) {
  421. return peer->AddDataChannel(kLocalDataChannel.c_str(), ordered, reliable);
  422. // DataChannelMessageCallback{ message_callback, message_user_data },
  423. // DataChannelBufferingCallback{ buffering_callback, buffering_user_data },
  424. // DataChannelStateCallback{ state_callback, state_user_data });
  425. }
  426. return MRS_E_INVALID_PEER_HANDLE;
  427. }
  428. void mrsPeerConnectionRemoveLocalVideoTrack(
  429. PeerConnectionHandle peerHandle) {
  430. if (auto peer = static_cast<PeerConnection*>(peerHandle)) {
  431. peer->RemoveLocalVideoTrack();
  432. }
  433. }
  434. void mrsPeerConnectionRemoveLocalAudioTrack(
  435. PeerConnectionHandle peerHandle) {
  436. if (auto peer = static_cast<PeerConnection*>(peerHandle)) {
  437. peer->RemoveLocalAudioTrack();
  438. }
  439. }
  440. bool mrsPeerConnectionRemoveDataChannel(PeerConnectionHandle peerHandle) {
  441. if (auto peer = static_cast<PeerConnection*>(peerHandle)) {
  442. return peer->RemoveDataChannel();
  443. }
  444. return false;
  445. }
  446. bool mrsPeerConnectionSendDataChannelMessage(PeerConnectionHandle peerHandle,
  447. const void* data,
  448. uint64_t size)
  449. {
  450. if (auto peer = static_cast<PeerConnection*>(peerHandle)) {
  451. return peer->SendDataChannelMessage(data, size);
  452. }
  453. return false;
  454. }
  455. bool mrsSdpForceCodecs(const char* message,
  456. const char* audio_codec_name,
  457. const char* video_codec_name,
  458. char* buffer,
  459. size_t* buffer_size) {
  460. RTC_CHECK(message);
  461. RTC_CHECK(buffer);
  462. RTC_CHECK(buffer_size);
  463. std::string message_str(message);
  464. std::string audio_codec_name_str;
  465. std::string video_codec_name_str;
  466. if (audio_codec_name) {
  467. audio_codec_name_str.assign(audio_codec_name);
  468. }
  469. if (video_codec_name) {
  470. video_codec_name_str.assign(video_codec_name);
  471. }
  472. std::string out_message =
  473. SdpForceCodecs(message_str, audio_codec_name_str, video_codec_name_str);
  474. const size_t capacity = *buffer_size;
  475. const size_t size = out_message.size();
  476. *buffer_size = size + 1;
  477. if (capacity < size + 1) {
  478. return false;
  479. }
  480. memcpy(buffer, out_message.c_str(), size);
  481. buffer[size] = '\0';
  482. return true;
  483. }
  484. void mrsMemCpy(void* dst, const void* src, size_t size) {
  485. memcpy(dst, src, size);
  486. }
  487. void mrsMemCpyStride(void* dst,
  488. int dst_stride,
  489. const void* src,
  490. int src_stride,
  491. int elem_size,
  492. int elem_count) {
  493. RTC_CHECK(dst);
  494. RTC_CHECK(dst_stride >= elem_size);
  495. RTC_CHECK(src);
  496. RTC_CHECK(src_stride >= elem_size);
  497. if ((dst_stride == elem_size) && (src_stride == elem_size)) {
  498. // If tightly packed, do a single memcpy() for performance
  499. const size_t total_size = (size_t)elem_size * elem_count;
  500. memcpy(dst, src, total_size);
  501. }
  502. else {
  503. // Otherwise, copy row by row
  504. for (int i = 0; i < elem_count; ++i) {
  505. memcpy(dst, src, elem_size);
  506. dst = (char*)dst + dst_stride;
  507. src = (const char*)src + src_stride;
  508. }
  509. }
  510. }