jetson_nv_encoder.cpp 42 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129
  1. // #include "jetson_nv_encoder.h"
  2. // #include <limits>
  3. // #include <string>
  4. // #include "third_party/openh264/src/codec/api/svc/codec_api.h"
  5. // #include "third_party/openh264/src/codec/api/svc/codec_app_def.h"
  6. // #include "third_party/openh264/src/codec/api/svc/codec_def.h"
  7. // #include "third_party/openh264/src/codec/api/svc/codec_ver.h"
  8. // #include "absl/strings/match.h"
  9. // #include "common_video/h264/h264_common.h"
  10. // #include "common_video/libyuv/include/webrtc_libyuv.h"
  11. // #include "modules/video_coding/utility/simulcast_rate_allocator.h"
  12. // #include "modules/video_coding/utility/simulcast_utility.h"
  13. // #include "rtc_base/checks.h"
  14. // #include "rtc_base/logging.h"
  15. // #include "rtc_base/time_utils.h"
  16. // #include "system_wrappers/include/metrics.h"
  17. // #include "third_party/libyuv/include/libyuv/convert.h"
  18. // #include "third_party/libyuv/include/libyuv/scale.h"
  19. // namespace webrtc {
  20. // namespace {
  21. // const bool kOpenH264EncoderDetailedLogging = false;
  22. // // QP scaling thresholds.
  23. // static const int kLowH264QpThreshold = 24;
  24. // static const int kHighH264QpThreshold = 37;
  25. // // Used by histograms. Values of entries should not be changed.
  26. // enum NvVideoEncoderEvent
  27. // {
  28. // kH264EncoderEventInit = 0,
  29. // kH264EncoderEventError = 1,
  30. // kH264EncoderEventMax = 16,
  31. // };
  32. // int NumberOfThreads(int width, int height, int number_of_cores)
  33. // {
  34. // // TODO(hbos): In Chromium, multiple threads do not work with sandbox on Mac,
  35. // // see crbug.com/583348. Until further investigated, only use one thread.
  36. // // if (width * height >= 1920 * 1080 && number_of_cores > 8) {
  37. // // return 8; // 8 threads for 1080p on high perf machines.
  38. // // } else if (width * height > 1280 * 960 && number_of_cores >= 6) {
  39. // // return 3; // 3 threads for 1080p.
  40. // // } else if (width * height > 640 * 480 && number_of_cores >= 3) {
  41. // // return 2; // 2 threads for qHD/HD.
  42. // // } else {
  43. // // return 1; // 1 thread for VGA or less.
  44. // // }
  45. // // TODO(sprang): Also check sSliceArgument.uiSliceNum om GetEncoderPrams(),
  46. // // before enabling multithreading here.
  47. // return 1;
  48. // }
  49. // VideoFrameType ConvertToVideoFrameType(EVideoFrameType type) {
  50. // switch (type) {
  51. // case videoFrameTypeIDR:
  52. // return VideoFrameType::kVideoFrameKey;
  53. // case videoFrameTypeSkip:
  54. // case videoFrameTypeI:
  55. // case videoFrameTypeP:
  56. // case videoFrameTypeIPMixed:
  57. // return VideoFrameType::kVideoFrameDelta;
  58. // case videoFrameTypeInvalid:
  59. // break;
  60. // }
  61. // RTC_NOTREACHED() << "Unexpected/invalid frame type: " << type;
  62. // return VideoFrameType::kEmptyFrame;
  63. // }
  64. // } // namespace
  65. // static void RtpFragmentize(EncodedImage* encoded_image,
  66. // const VideoFrameBuffer& frame_buffer,
  67. // std::vector<uint8_t>& frame_packet,
  68. // RTPFragmentationHeader* frag_header)
  69. // {
  70. // size_t required_capacity = 0;
  71. // encoded_image->set_size(0);
  72. // required_capacity = frame_packet.size();
  73. // encoded_image->SetEncodedData(EncodedImageBuffer::Create(required_capacity));
  74. // // TODO(nisse): Use a cache or buffer pool to avoid allocation?
  75. // encoded_image->SetEncodedData(EncodedImageBuffer::Create(required_capacity));
  76. // memcpy(encoded_image->data(), &frame_packet[0], frame_packet.size());
  77. // std::vector<webrtc::H264::NaluIndex> nalus = webrtc::H264::FindNaluIndices(
  78. // encoded_image->data(), encoded_image->size());
  79. // size_t fragments_count = nalus.size();
  80. // frag_header->VerifyAndAllocateFragmentationHeader(fragments_count);
  81. // for (size_t i = 0; i < nalus.size(); i++) {
  82. // frag_header->fragmentationOffset[i] = nalus[i].payload_start_offset;
  83. // frag_header->fragmentationLength[i] = nalus[i].payload_size;
  84. // }
  85. // }
  86. // NvEncoder::NvEncoder(const cricket::VideoCodec& codec)
  87. // : packetization_mode_(H264PacketizationMode::SingleNalUnit),
  88. // max_payload_size_(0),
  89. // number_of_cores_(0),
  90. // encoded_image_callback_(nullptr),
  91. // has_reported_init_(false),
  92. // has_reported_error_(false),
  93. // num_temporal_layers_(1),
  94. // tl0sync_limit_(0)
  95. // {
  96. // RTC_CHECK(absl::EqualsIgnoreCase(codec.name, cricket::kH264CodecName));
  97. // std::string packetization_mode_string;
  98. // if (codec.GetParam(cricket::kH264FmtpPacketizationMode, &packetization_mode_string)
  99. // && packetization_mode_string == "1") {
  100. // packetization_mode_ = H264PacketizationMode::NonInterleaved;
  101. // }
  102. // encoded_images_.reserve(kMaxSimulcastStreams);
  103. // nv_encoders_.reserve(kMaxSimulcastStreams);
  104. // configurations_.reserve(kMaxSimulcastStreams);
  105. // image_buffer_ = nullptr;
  106. // }
  107. // NvEncoder::~NvEncoder()
  108. // {
  109. // Release();
  110. // }
  111. // int32_t NvEncoder::InitEncode(const VideoCodec* inst,
  112. // int32_t number_of_cores,
  113. // size_t max_payload_size)
  114. // {
  115. // ReportInit();
  116. // if (!inst || inst->codecType != kVideoCodecH264) {
  117. // ReportError();
  118. // return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
  119. // }
  120. // if (inst->maxFramerate == 0) {
  121. // ReportError();
  122. // return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
  123. // }
  124. // if (inst->width < 1 || inst->height < 1) {
  125. // ReportError();
  126. // return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
  127. // }
  128. // int32_t release_ret = Release();
  129. // if (release_ret != WEBRTC_VIDEO_CODEC_OK) {
  130. // ReportError();
  131. // return release_ret;
  132. // }
  133. // int number_of_streams = SimulcastUtility::NumberOfSimulcastStreams(*inst);
  134. // bool doing_simulcast = (number_of_streams > 1);
  135. // if (doing_simulcast && !SimulcastUtility::ValidSimulcastParameters(*inst, number_of_streams)) {
  136. // return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED;
  137. // }
  138. // assert(number_of_streams == 1);
  139. // encoded_images_.resize(number_of_streams);
  140. // nv_encoders_.resize(number_of_streams);
  141. // configurations_.resize(number_of_streams);
  142. // number_of_cores_ = number_of_cores;
  143. // max_payload_size_ = max_payload_size;
  144. // codec_ = *inst;
  145. // // Code expects simulcastStream resolutions to be correct, make sure they are
  146. // // filled even when there are no simulcast layers.
  147. // if (codec_.numberOfSimulcastStreams == 0) {
  148. // codec_.simulcastStream[0].width = codec_.width;
  149. // codec_.simulcastStream[0].height = codec_.height;
  150. // }
  151. // num_temporal_layers_ = codec_.H264()->numberOfTemporalLayers;
  152. // for (int i = 0, idx = number_of_streams - 1; i < number_of_streams; ++i, --idx) {
  153. // // Store nvidia encoder.
  154. // xop::NvidiaD3D11Encoder* nv_encoder = new xop::NvidiaD3D11Encoder();
  155. // nv_encoders_[i] = nv_encoder;
  156. // // Set internal settings from codec_settings
  157. // configurations_[i].simulcast_idx = idx;
  158. // configurations_[i].sending = false;
  159. // configurations_[i].width = codec_.simulcastStream[idx].width;
  160. // configurations_[i].height = codec_.simulcastStream[idx].height;
  161. // configurations_[i].max_frame_rate = static_cast<float>(codec_.maxFramerate);
  162. // configurations_[i].frame_dropping_on = codec_.H264()->frameDroppingOn;
  163. // configurations_[i].key_frame_interval = codec_.H264()->keyFrameInterval;
  164. // // Codec_settings uses kbits/second; encoder uses bits/second.
  165. // configurations_[i].max_bps = codec_.maxBitrate * 1000;
  166. // configurations_[i].target_bps = codec_.maxBitrate * 1000 / 2;
  167. // nv_encoder->SetOption(xop::VE_OPT_WIDTH, configurations_[i].width);
  168. // nv_encoder->SetOption(xop::VE_OPT_HEIGHT, configurations_[i].height);
  169. // nv_encoder->SetOption(xop::VE_OPT_FRAME_RATE, static_cast<int>(configurations_[i].max_frame_rate));
  170. // nv_encoder->SetOption(xop::VE_OPT_GOP, configurations_[i].key_frame_interval);
  171. // nv_encoder->SetOption(xop::VE_OPT_CODEC, xop::VE_OPT_CODEC_H264);
  172. // nv_encoder->SetOption(xop::VE_OPT_BITRATE_KBPS, configurations_[i].target_bps / 1000);
  173. // nv_encoder->SetOption(xop::VE_OPT_TEXTURE_FORMAT, xop::VE_OPT_FORMAT_B8G8R8A8);
  174. // if (!nv_encoder->Init()) {
  175. // Release();
  176. // ReportError();
  177. // return WEBRTC_VIDEO_CODEC_ERROR;
  178. // }
  179. // image_buffer_.reset(new uint8_t[configurations_[i].width * configurations_[i].height * 10]);
  180. // // TODO(pbos): Base init params on these values before submitting.
  181. // video_format_ = EVideoFormatType::videoFormatI420;
  182. // // Initialize encoded image. Default buffer size: size of unencoded data.
  183. // const size_t new_capacity = CalcBufferSize(VideoType::kI420,
  184. // codec_.simulcastStream[idx].width, codec_.simulcastStream[idx].height);
  185. // encoded_images_[i].SetEncodedData(EncodedImageBuffer::Create(new_capacity));
  186. // encoded_images_[i]._completeFrame = true;
  187. // encoded_images_[i]._encodedWidth = codec_.simulcastStream[idx].width;
  188. // encoded_images_[i]._encodedHeight = codec_.simulcastStream[idx].height;
  189. // encoded_images_[i].set_size(0);
  190. // }
  191. // SimulcastRateAllocator init_allocator(codec_);
  192. // VideoBitrateAllocation allocation = init_allocator.GetAllocation(
  193. // codec_.maxBitrate * 1000 / 2, codec_.maxFramerate);
  194. // SetRates(RateControlParameters(allocation, codec_.maxFramerate));
  195. // return WEBRTC_VIDEO_CODEC_OK;
  196. // }
  197. // int32_t NvEncoder::Release()
  198. // {
  199. // while (!nv_encoders_.empty()) {
  200. // xop::NvidiaD3D11Encoder* nv_encoder = reinterpret_cast<xop::NvidiaD3D11Encoder*>(nv_encoders_.back());
  201. // if (nv_encoder) {
  202. // nv_encoder->Destroy();
  203. // delete nv_encoder;
  204. // }
  205. // nv_encoders_.pop_back();
  206. // }
  207. // configurations_.clear();
  208. // encoded_images_.clear();
  209. // return WEBRTC_VIDEO_CODEC_OK;
  210. // }
  211. // int32_t NvEncoder::RegisterEncodeCompleteCallback(EncodedImageCallback* callback)
  212. // {
  213. // encoded_image_callback_ = callback;
  214. // return WEBRTC_VIDEO_CODEC_OK;
  215. // }
  216. // void NvEncoder::SetRates(const RateControlParameters& parameters)
  217. // {
  218. // if (parameters.bitrate.get_sum_bps() == 0) {
  219. // // Encoder paused, turn off all encoding.
  220. // for (size_t i = 0; i < configurations_.size(); ++i)
  221. // configurations_[i].SetStreamState(false);
  222. // return;
  223. // }
  224. // // At this point, bitrate allocation should already match codec settings.
  225. // if (codec_.maxBitrate > 0)
  226. // RTC_DCHECK_LE(parameters.bitrate.get_sum_kbps(), codec_.maxBitrate);
  227. // RTC_DCHECK_GE(parameters.bitrate.get_sum_kbps(), codec_.minBitrate);
  228. // if (codec_.numberOfSimulcastStreams > 0)
  229. // RTC_DCHECK_GE(parameters.bitrate.get_sum_kbps(), codec_.simulcastStream[0].minBitrate);
  230. // codec_.maxFramerate = static_cast<uint32_t>(parameters.framerate_fps);
  231. // size_t stream_idx = nv_encoders_.size() - 1;
  232. // for (size_t i = 0; i < nv_encoders_.size(); ++i, --stream_idx) {
  233. // configurations_[i].target_bps = parameters.bitrate.GetSpatialLayerSum(stream_idx);
  234. // configurations_[i].max_frame_rate = static_cast<float>(parameters.framerate_fps);
  235. // if (configurations_[i].target_bps) {
  236. // configurations_[i].SetStreamState(true);
  237. // if (nv_encoders_[i]) {
  238. // xop::NvidiaD3D11Encoder* nv_encoder = reinterpret_cast<xop::NvidiaD3D11Encoder*>(nv_encoders_[i]);
  239. // nv_encoder->SetEvent(xop::VE_EVENT_RESET_BITRATE_KBPS, configurations_[i].target_bps/1000);
  240. // nv_encoder->SetEvent(xop::VE_EVENT_RESET_FRAME_RATE, static_cast<int>(configurations_[i].max_frame_rate));
  241. // }
  242. // else {
  243. // configurations_[i].SetStreamState(false);
  244. // }
  245. // }
  246. // }
  247. // }
  248. // int32_t NvEncoder::Encode(const VideoFrame& input_frame,
  249. // const std::vector<VideoFrameType>* frame_types)
  250. // {
  251. // if (nv_encoders_.empty()) {
  252. // ReportError();
  253. // return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
  254. // }
  255. // if (!encoded_image_callback_) {
  256. // RTC_LOG(LS_WARNING)
  257. // << "InitEncode() has been called, but a callback function "
  258. // << "has not been set with RegisterEncodeCompleteCallback()";
  259. // ReportError();
  260. // return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
  261. // }
  262. // rtc::scoped_refptr<const I420BufferInterface> frame_buffer = input_frame.video_frame_buffer()->ToI420();
  263. // bool send_key_frame = false;
  264. // for (size_t i = 0; i < configurations_.size(); ++i) {
  265. // if (configurations_[i].key_frame_request && configurations_[i].sending) {
  266. // send_key_frame = true;
  267. // break;
  268. // }
  269. // }
  270. // if (!send_key_frame && frame_types) {
  271. // for (size_t i = 0; i < frame_types->size() && i < configurations_.size(); ++i) {
  272. // if ((*frame_types)[i] == VideoFrameType::kVideoFrameKey && configurations_[i].sending) {
  273. // send_key_frame = true;
  274. // break;
  275. // }
  276. // }
  277. // }
  278. // RTC_DCHECK_EQ(configurations_[0].width, frame_buffer->width());
  279. // RTC_DCHECK_EQ(configurations_[0].height, frame_buffer->height());
  280. // // Encode image for each layer.
  281. // for (size_t i = 0; i < nv_encoders_.size(); ++i) {
  282. // if (!configurations_[i].sending) {
  283. // continue;
  284. // }
  285. // if (frame_types != nullptr) {
  286. // // Skip frame?
  287. // if ((*frame_types)[i] == VideoFrameType::kEmptyFrame) {
  288. // continue;
  289. // }
  290. // }
  291. // if (send_key_frame) {
  292. // if (!nv_encoders_.empty() && nv_encoders_[i]) {
  293. // xop::NvidiaD3D11Encoder* nv_encoder = reinterpret_cast<xop::NvidiaD3D11Encoder*>(nv_encoders_[i]);
  294. // nv_encoder->SetEvent(xop::VE_EVENT_FORCE_IDR, 1);
  295. // }
  296. // configurations_[i].key_frame_request = false;
  297. // }
  298. // // EncodeFrame output.
  299. // SFrameBSInfo info;
  300. // memset(&info, 0, sizeof(SFrameBSInfo));
  301. // std::vector<uint8_t> frame_packet;
  302. // EncodeFrame((int)i, input_frame, frame_packet);
  303. // if (frame_packet.size() == 0) {
  304. // return WEBRTC_VIDEO_CODEC_OK;
  305. // }
  306. // else {
  307. // if ((frame_packet[4] & 0x1f) == 0x07) {
  308. // info.eFrameType = videoFrameTypeIDR;
  309. // }
  310. // else if ((frame_packet[4] & 0x1f) == 0x01) {
  311. // info.eFrameType = videoFrameTypeP;
  312. // }
  313. // else {
  314. // return WEBRTC_VIDEO_CODEC_OK;
  315. // }
  316. // }
  317. // encoded_images_[i]._encodedWidth = configurations_[i].width;
  318. // encoded_images_[i]._encodedHeight = configurations_[i].height;
  319. // encoded_images_[i].SetTimestamp(input_frame.timestamp());
  320. // encoded_images_[i].ntp_time_ms_ = input_frame.ntp_time_ms();
  321. // encoded_images_[i].capture_time_ms_ = input_frame.render_time_ms();
  322. // encoded_images_[i].rotation_ = input_frame.rotation();
  323. // encoded_images_[i].SetColorSpace(input_frame.color_space());
  324. // encoded_images_[i].content_type_ = (codec_.mode == VideoCodecMode::kScreensharing)
  325. // ? VideoContentType::SCREENSHARE
  326. // : VideoContentType::UNSPECIFIED;
  327. // encoded_images_[i].timing_.flags = VideoSendTiming::kInvalid;
  328. // encoded_images_[i]._frameType = ConvertToVideoFrameType(info.eFrameType);
  329. // encoded_images_[i].SetSpatialIndex(configurations_[i].simulcast_idx);
  330. // // Split encoded image up into fragments. This also updates
  331. // // |encoded_image_|.
  332. // RTPFragmentationHeader frag_header;
  333. // RtpFragmentize(&encoded_images_[i], *frame_buffer, frame_packet, &frag_header);
  334. // // Encoder can skip frames to save bandwidth in which case
  335. // // |encoded_images_[i]._length| == 0.
  336. // if (encoded_images_[i].size() > 0) {
  337. // // Parse QP.
  338. // h264_bitstream_parser_.ParseBitstream(encoded_images_[i].data(),
  339. // encoded_images_[i].size());
  340. // h264_bitstream_parser_.GetLastSliceQp(&encoded_images_[i].qp_);
  341. // // Deliver encoded image.
  342. // CodecSpecificInfo codec_specific;
  343. // codec_specific.codecType = kVideoCodecH264;
  344. // codec_specific.codecSpecific.H264.packetization_mode = packetization_mode_;
  345. // codec_specific.codecSpecific.H264.temporal_idx = kNoTemporalIdx;
  346. // codec_specific.codecSpecific.H264.idr_frame = (info.eFrameType == videoFrameTypeIDR);
  347. // codec_specific.codecSpecific.H264.base_layer_sync = false;
  348. // encoded_image_callback_->OnEncodedImage(encoded_images_[i], &codec_specific, &frag_header);
  349. // }
  350. // }
  351. // return WEBRTC_VIDEO_CODEC_OK;
  352. // }
  353. // void NvEncoder::ReportInit()
  354. // {
  355. // if (has_reported_init_)
  356. // return;
  357. // RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.NvEncoder.Event",
  358. // kH264EncoderEventInit, kH264EncoderEventMax);
  359. // has_reported_init_ = true;
  360. // }
  361. // void NvEncoder::ReportError()
  362. // {
  363. // if (has_reported_error_)
  364. // return;
  365. // RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.NvEncoder.Event",
  366. // kH264EncoderEventError, kH264EncoderEventMax);
  367. // has_reported_error_ = true;
  368. // }
  369. // VideoEncoder::EncoderInfo NvEncoder::GetEncoderInfo() const
  370. // {
  371. // EncoderInfo info;
  372. // info.supports_native_handle = false;
  373. // info.implementation_name = "NvEncoder";
  374. // info.scaling_settings = VideoEncoder::ScalingSettings(kLowH264QpThreshold, kHighH264QpThreshold);
  375. // info.is_hardware_accelerated = true;
  376. // info.has_internal_source = false;
  377. // return info;
  378. // }
  379. // void NvEncoder::LayerConfig::SetStreamState(bool send_stream)
  380. // {
  381. // if (send_stream && !sending) {
  382. // // Need a key frame if we have not sent this stream before.
  383. // key_frame_request = true;
  384. // }
  385. // sending = send_stream;
  386. // }
  387. // bool NvEncoder::EncodeFrame(int index, const VideoFrame& input_frame,
  388. // std::vector<uint8_t>& frame_packet)
  389. // {
  390. // frame_packet.clear();
  391. // if (nv_encoders_.empty() || !nv_encoders_[index]) {
  392. // return false;
  393. // }
  394. // if (video_format_ == EVideoFormatType::videoFormatI420) {
  395. // if (image_buffer_ != nullptr) {
  396. // if (webrtc::ConvertFromI420(input_frame, webrtc::VideoType::kARGB, 0,
  397. // image_buffer_.get()) < 0) {
  398. // return false;
  399. // }
  400. // }
  401. // else {
  402. // return false;
  403. // }
  404. // }
  405. // int width = input_frame.width();
  406. // int height = input_frame.height();
  407. // int image_size = width * height * 4; // argb
  408. // xop::NvidiaD3D11Encoder* nv_encoder = reinterpret_cast<xop::NvidiaD3D11Encoder*>(nv_encoders_[index]);
  409. // if (nv_encoder) {
  410. // int frame_size = nv_encoder->Encode(std::vector<uint8_t>(image_buffer_.get(), image_buffer_.get() + image_size) ,frame_packet);
  411. // if (frame_size < 0) {
  412. // return false;
  413. // }
  414. // }
  415. // return true;
  416. // }
  417. // } // namespace webrtc
  418. /*
  419. * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
  420. *
  421. * Use of this source code is governed by a BSD-style license
  422. * that can be found in the LICENSE file in the root of the source
  423. * tree. An additional intellectual property rights grant can be found
  424. * in the file PATENTS. All contributing project authors may
  425. * be found in the AUTHORS file in the root of the source tree.
  426. *
  427. */
  428. // Everything declared/defined in this header is only required when WebRTC is
  429. // build with H264 support, please do not move anything out of the
  430. // #ifdef unless needed and tested.
  431. // #ifdef WEBRTC_USE_H264
  432. #include "modules/video_coding/codecs/h264/h264_encoder_impl.h"
  433. #include <limits>
  434. #include <string>
  435. #include "absl/strings/match.h"
  436. #include "common_video/libyuv/include/webrtc_libyuv.h"
  437. #include "modules/video_coding/utility/simulcast_rate_allocator.h"
  438. #include "modules/video_coding/utility/simulcast_utility.h"
  439. #include "rtc_base/checks.h"
  440. #include "rtc_base/logging.h"
  441. #include "rtc_base/time_utils.h"
  442. #include "system_wrappers/include/metrics.h"
  443. #include "third_party/libyuv/include/libyuv/convert.h"
  444. #include "third_party/libyuv/include/libyuv/scale.h"
  445. #include "third_party/openh264/src/codec/api/svc/codec_api.h"
  446. #include "third_party/openh264/src/codec/api/svc/codec_app_def.h"
  447. #include "third_party/openh264/src/codec/api/svc/codec_def.h"
  448. #include "third_party/openh264/src/codec/api/svc/codec_ver.h"
  449. #include "jetson_nv_encoder.h"
  450. namespace webrtc {
  451. namespace {
  452. const bool kOpenH264EncoderDetailedLogging = false;
  453. // QP scaling thresholds.
  454. static const int kLowH264QpThreshold = 24;
  455. static const int kHighH264QpThreshold = 37;
  456. // Used by histograms. Values of entries should not be changed.
  457. enum H264EncoderImplEvent {
  458. kH264EncoderEventInit = 0,
  459. kH264EncoderEventError = 1,
  460. kH264EncoderEventMax = 16,
  461. };
  462. int NumberOfThreads(int width, int height, int number_of_cores) {
  463. // TODO(hbos): In Chromium, multiple threads do not work with sandbox on Mac,
  464. // see crbug.com/583348. Until further investigated, only use one thread.
  465. // if (width * height >= 1920 * 1080 && number_of_cores > 8) {
  466. // return 8; // 8 threads for 1080p on high perf machines.
  467. // } else if (width * height > 1280 * 960 && number_of_cores >= 6) {
  468. // return 3; // 3 threads for 1080p.
  469. // } else if (width * height > 640 * 480 && number_of_cores >= 3) {
  470. // return 2; // 2 threads for qHD/HD.
  471. // } else {
  472. // return 1; // 1 thread for VGA or less.
  473. // }
  474. // TODO(sprang): Also check sSliceArgument.uiSliceNum om GetEncoderPrams(),
  475. // before enabling multithreading here.
  476. return 1;
  477. }
  478. VideoFrameType ConvertToVideoFrameType(EVideoFrameType type) {
  479. switch (type) {
  480. case videoFrameTypeIDR:
  481. return VideoFrameType::kVideoFrameKey;
  482. case videoFrameTypeSkip:
  483. case videoFrameTypeI:
  484. case videoFrameTypeP:
  485. case videoFrameTypeIPMixed:
  486. return VideoFrameType::kVideoFrameDelta;
  487. case videoFrameTypeInvalid:
  488. break;
  489. }
  490. RTC_NOTREACHED() << "Unexpected/invalid frame type: " << type;
  491. return VideoFrameType::kEmptyFrame;
  492. }
  493. } // namespace
  494. // Helper method used by H264EncoderImpl::Encode.
  495. // Copies the encoded bytes from |info| to |encoded_image|. The
  496. // |encoded_image->_buffer| may be deleted and reallocated if a bigger buffer is
  497. // required.
  498. //
  499. // After OpenH264 encoding, the encoded bytes are stored in |info| spread out
  500. // over a number of layers and "NAL units". Each NAL unit is a fragment starting
  501. // with the four-byte start code {0,0,0,1}. All of this data (including the
  502. // start codes) is copied to the |encoded_image->_buffer|.
  503. static void RtpFragmentize(EncodedImage* encoded_image, SFrameBSInfo* info) {
  504. // Calculate minimum buffer size required to hold encoded data.
  505. size_t required_capacity = 0;
  506. size_t fragments_count = 0;
  507. for (int layer = 0; layer < info->iLayerNum; ++layer) {
  508. const SLayerBSInfo& layerInfo = info->sLayerInfo[layer];
  509. for (int nal = 0; nal < layerInfo.iNalCount; ++nal, ++fragments_count) {
  510. RTC_CHECK_GE(layerInfo.pNalLengthInByte[nal], 0);
  511. // Ensure |required_capacity| will not overflow.
  512. RTC_CHECK_LE(layerInfo.pNalLengthInByte[nal],
  513. std::numeric_limits<size_t>::max() - required_capacity);
  514. required_capacity += layerInfo.pNalLengthInByte[nal];
  515. }
  516. }
  517. // TODO(nisse): Use a cache or buffer pool to avoid allocation?
  518. auto buffer = EncodedImageBuffer::Create(required_capacity);
  519. encoded_image->SetEncodedData(buffer);
  520. // Iterate layers and NAL units, note each NAL unit as a fragment and copy
  521. // the data to |encoded_image->_buffer|.
  522. const uint8_t start_code[4] = {0, 0, 0, 1};
  523. size_t frag = 0;
  524. encoded_image->set_size(0);
  525. for (int layer = 0; layer < info->iLayerNum; ++layer) {
  526. const SLayerBSInfo& layerInfo = info->sLayerInfo[layer];
  527. // Iterate NAL units making up this layer, noting fragments.
  528. size_t layer_len = 0;
  529. for (int nal = 0; nal < layerInfo.iNalCount; ++nal, ++frag) {
  530. // Because the sum of all layer lengths, |required_capacity|, fits in a
  531. // |size_t|, we know that any indices in-between will not overflow.
  532. RTC_DCHECK_GE(layerInfo.pNalLengthInByte[nal], 4);
  533. RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len + 0], start_code[0]);
  534. RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len + 1], start_code[1]);
  535. RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len + 2], start_code[2]);
  536. RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len + 3], start_code[3]);
  537. layer_len += layerInfo.pNalLengthInByte[nal];
  538. }
  539. // Copy the entire layer's data (including start codes).
  540. memcpy(buffer->data() + encoded_image->size(), layerInfo.pBsBuf, layer_len);
  541. encoded_image->set_size(encoded_image->size() + layer_len);
  542. }
  543. }
  544. H264EncoderImpl_::H264EncoderImpl_(const cricket::VideoCodec& codec)
  545. : packetization_mode_(H264PacketizationMode::SingleNalUnit),
  546. max_payload_size_(0),
  547. number_of_cores_(0),
  548. encoded_image_callback_(nullptr),
  549. has_reported_init_(false),
  550. has_reported_error_(false) {
  551. RTC_CHECK(absl::EqualsIgnoreCase(codec.name, cricket::kH264CodecName));
  552. std::string packetization_mode_string;
  553. if (codec.GetParam(cricket::kH264FmtpPacketizationMode,
  554. &packetization_mode_string) &&
  555. packetization_mode_string == "1") {
  556. packetization_mode_ = H264PacketizationMode::NonInterleaved;
  557. }
  558. downscaled_buffers_.reserve(kMaxSimulcastStreams - 1);
  559. encoded_images_.reserve(kMaxSimulcastStreams);
  560. encoders_.reserve(kMaxSimulcastStreams);
  561. configurations_.reserve(kMaxSimulcastStreams);
  562. tl0sync_limit_.reserve(kMaxSimulcastStreams);
  563. }
  564. H264EncoderImpl_::~H264EncoderImpl_() {
  565. Release();
  566. }
  567. int32_t H264EncoderImpl_::InitEncode(const VideoCodec* inst,
  568. const VideoEncoder::Settings& settings) {
  569. ReportInit();
  570. if (!inst || inst->codecType != kVideoCodecH264) {
  571. ReportError();
  572. return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
  573. }
  574. if (inst->maxFramerate == 0) {
  575. ReportError();
  576. return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
  577. }
  578. if (inst->width < 1 || inst->height < 1) {
  579. ReportError();
  580. return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
  581. }
  582. int32_t release_ret = Release();
  583. if (release_ret != WEBRTC_VIDEO_CODEC_OK) {
  584. ReportError();
  585. return release_ret;
  586. }
  587. int number_of_streams = SimulcastUtility::NumberOfSimulcastStreams(*inst);
  588. bool doing_simulcast = (number_of_streams > 1);
  589. if (doing_simulcast &&
  590. !SimulcastUtility::ValidSimulcastParameters(*inst, number_of_streams)) {
  591. return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED;
  592. }
  593. downscaled_buffers_.resize(number_of_streams - 1);
  594. encoded_images_.resize(number_of_streams);
  595. encoders_.resize(number_of_streams);
  596. pictures_.resize(number_of_streams);
  597. configurations_.resize(number_of_streams);
  598. tl0sync_limit_.resize(number_of_streams);
  599. number_of_cores_ = settings.number_of_cores;
  600. max_payload_size_ = settings.max_payload_size;
  601. codec_ = *inst;
  602. // Code expects simulcastStream resolutions to be correct, make sure they are
  603. // filled even when there are no simulcast layers.
  604. if (codec_.numberOfSimulcastStreams == 0) {
  605. codec_.simulcastStream[0].width = codec_.width;
  606. codec_.simulcastStream[0].height = codec_.height;
  607. }
  608. for (int i = 0, idx = number_of_streams - 1; i < number_of_streams;
  609. ++i, --idx) {
  610. ISVCEncoder* openh264_encoder;
  611. // Create encoder.
  612. if (WelsCreateSVCEncoder(&openh264_encoder) != 0) {
  613. // Failed to create encoder.
  614. RTC_LOG(LS_ERROR) << "Failed to create OpenH264 encoder";
  615. RTC_DCHECK(!openh264_encoder);
  616. Release();
  617. ReportError();
  618. return WEBRTC_VIDEO_CODEC_ERROR;
  619. }
  620. RTC_DCHECK(openh264_encoder);
  621. if (kOpenH264EncoderDetailedLogging) {
  622. int trace_level = WELS_LOG_DETAIL;
  623. openh264_encoder->SetOption(ENCODER_OPTION_TRACE_LEVEL, &trace_level);
  624. }
  625. // else WELS_LOG_DEFAULT is used by default.
  626. // Store h264 encoder.
  627. encoders_[i] = openh264_encoder;
  628. // Set internal settings from codec_settings
  629. configurations_[i].simulcast_idx = idx;
  630. configurations_[i].sending = false;
  631. configurations_[i].width = codec_.simulcastStream[idx].width;
  632. configurations_[i].height = codec_.simulcastStream[idx].height;
  633. configurations_[i].max_frame_rate = static_cast<float>(codec_.maxFramerate);
  634. configurations_[i].frame_dropping_on = codec_.H264()->frameDroppingOn;
  635. configurations_[i].key_frame_interval = codec_.H264()->keyFrameInterval;
  636. configurations_[i].num_temporal_layers =
  637. codec_.simulcastStream[idx].numberOfTemporalLayers;
  638. printf("4567890-=09876567890-98765467890-986754567890978654567890-985645678908654567890654567890\n");
  639. // Create downscaled image buffers.
  640. if (i > 0) {
  641. downscaled_buffers_[i - 1] = I420Buffer::Create(
  642. configurations_[i].width, configurations_[i].height,
  643. configurations_[i].width, configurations_[i].width / 2,
  644. configurations_[i].width / 2);
  645. }
  646. // Codec_settings uses kbits/second; encoder uses bits/second.
  647. configurations_[i].max_bps = codec_.maxBitrate * 1000;
  648. configurations_[i].target_bps = codec_.startBitrate * 1000;
  649. // Create encoder parameters based on the layer configuration.
  650. SEncParamExt encoder_params = CreateEncoderParams(i);
  651. // Initialize.
  652. if (openh264_encoder->InitializeExt(&encoder_params) != 0) {
  653. RTC_LOG(LS_ERROR) << "Failed to initialize OpenH264 encoder";
  654. Release();
  655. ReportError();
  656. return WEBRTC_VIDEO_CODEC_ERROR;
  657. }
  658. // TODO(pbos): Base init params on these values before submitting.
  659. int video_format = EVideoFormatType::videoFormatI420;
  660. openh264_encoder->SetOption(ENCODER_OPTION_DATAFORMAT, &video_format);
  661. // Initialize encoded image. Default buffer size: size of unencoded data.
  662. const size_t new_capacity =
  663. CalcBufferSize(VideoType::kI420, codec_.simulcastStream[idx].width,
  664. codec_.simulcastStream[idx].height);
  665. encoded_images_[i].SetEncodedData(EncodedImageBuffer::Create(new_capacity));
  666. encoded_images_[i]._completeFrame = true;
  667. encoded_images_[i]._encodedWidth = codec_.simulcastStream[idx].width;
  668. encoded_images_[i]._encodedHeight = codec_.simulcastStream[idx].height;
  669. encoded_images_[i].set_size(0);
  670. tl0sync_limit_[i] = configurations_[i].num_temporal_layers;
  671. }
  672. SimulcastRateAllocator init_allocator(codec_);
  673. VideoBitrateAllocation allocation =
  674. init_allocator.Allocate(VideoBitrateAllocationParameters(
  675. DataRate::KilobitsPerSec(codec_.startBitrate), codec_.maxFramerate));
  676. SetRates(RateControlParameters(allocation, codec_.maxFramerate));
  677. return WEBRTC_VIDEO_CODEC_OK;
  678. }
  679. int32_t H264EncoderImpl_::Release() {
  680. while (!encoders_.empty()) {
  681. ISVCEncoder* openh264_encoder = encoders_.back();
  682. if (openh264_encoder) {
  683. RTC_CHECK_EQ(0, openh264_encoder->Uninitialize());
  684. WelsDestroySVCEncoder(openh264_encoder);
  685. }
  686. encoders_.pop_back();
  687. }
  688. downscaled_buffers_.clear();
  689. configurations_.clear();
  690. encoded_images_.clear();
  691. pictures_.clear();
  692. tl0sync_limit_.clear();
  693. return WEBRTC_VIDEO_CODEC_OK;
  694. }
  695. int32_t H264EncoderImpl_::RegisterEncodeCompleteCallback(
  696. EncodedImageCallback* callback) {
  697. encoded_image_callback_ = callback;
  698. return WEBRTC_VIDEO_CODEC_OK;
  699. }
  700. void H264EncoderImpl_::SetRates(const RateControlParameters& parameters) {
  701. if (encoders_.empty()) {
  702. RTC_LOG(LS_WARNING) << "SetRates() while uninitialized.";
  703. return;
  704. }
  705. if (parameters.framerate_fps < 1.0) {
  706. RTC_LOG(LS_WARNING) << "Invalid frame rate: " << parameters.framerate_fps;
  707. return;
  708. }
  709. if (parameters.bitrate.get_sum_bps() == 0) {
  710. // Encoder paused, turn off all encoding.
  711. for (size_t i = 0; i < configurations_.size(); ++i) {
  712. configurations_[i].SetStreamState(false);
  713. }
  714. return;
  715. }
  716. codec_.maxFramerate = static_cast<uint32_t>(parameters.framerate_fps);
  717. size_t stream_idx = encoders_.size() - 1;
  718. for (size_t i = 0; i < encoders_.size(); ++i, --stream_idx) {
  719. // Update layer config.
  720. configurations_[i].target_bps =
  721. parameters.bitrate.GetSpatialLayerSum(stream_idx);
  722. configurations_[i].max_frame_rate = parameters.framerate_fps;
  723. if (configurations_[i].target_bps) {
  724. configurations_[i].SetStreamState(true);
  725. // Update h264 encoder.
  726. SBitrateInfo target_bitrate;
  727. memset(&target_bitrate, 0, sizeof(SBitrateInfo));
  728. target_bitrate.iLayer = SPATIAL_LAYER_ALL,
  729. target_bitrate.iBitrate = configurations_[i].target_bps;
  730. encoders_[i]->SetOption(ENCODER_OPTION_BITRATE, &target_bitrate);
  731. encoders_[i]->SetOption(ENCODER_OPTION_FRAME_RATE,
  732. &configurations_[i].max_frame_rate);
  733. } else {
  734. configurations_[i].SetStreamState(false);
  735. }
  736. }
  737. }
  738. int32_t H264EncoderImpl_::Encode(
  739. const VideoFrame& input_frame,
  740. const std::vector<VideoFrameType>* frame_types) {
  741. if (encoders_.empty()) {
  742. ReportError();
  743. return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
  744. }
  745. if (!encoded_image_callback_) {
  746. RTC_LOG(LS_WARNING)
  747. << "InitEncode() has been called, but a callback function "
  748. "has not been set with RegisterEncodeCompleteCallback()";
  749. ReportError();
  750. return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
  751. }
  752. rtc::scoped_refptr<const I420BufferInterface> frame_buffer =
  753. input_frame.video_frame_buffer()->ToI420();
  754. bool send_key_frame = false;
  755. for (size_t i = 0; i < configurations_.size(); ++i) {
  756. if (configurations_[i].key_frame_request && configurations_[i].sending) {
  757. send_key_frame = true;
  758. break;
  759. }
  760. }
  761. if (!send_key_frame && frame_types) {
  762. for (size_t i = 0; i < configurations_.size(); ++i) {
  763. const size_t simulcast_idx =
  764. static_cast<size_t>(configurations_[i].simulcast_idx);
  765. if (configurations_[i].sending && simulcast_idx < frame_types->size() &&
  766. (*frame_types)[simulcast_idx] == VideoFrameType::kVideoFrameKey) {
  767. send_key_frame = true;
  768. break;
  769. }
  770. }
  771. }
  772. RTC_DCHECK_EQ(configurations_[0].width, frame_buffer->width());
  773. RTC_DCHECK_EQ(configurations_[0].height, frame_buffer->height());
  774. // Encode image for each layer.
  775. for (size_t i = 0; i < encoders_.size(); ++i) {
  776. // EncodeFrame input.
  777. pictures_[i] = {0};
  778. pictures_[i].iPicWidth = configurations_[i].width;
  779. pictures_[i].iPicHeight = configurations_[i].height;
  780. pictures_[i].iColorFormat = EVideoFormatType::videoFormatI420;
  781. pictures_[i].uiTimeStamp = input_frame.ntp_time_ms();
  782. // Downscale images on second and ongoing layers.
  783. if (i == 0) {
  784. pictures_[i].iStride[0] = frame_buffer->StrideY();
  785. pictures_[i].iStride[1] = frame_buffer->StrideU();
  786. pictures_[i].iStride[2] = frame_buffer->StrideV();
  787. pictures_[i].pData[0] = const_cast<uint8_t*>(frame_buffer->DataY());
  788. pictures_[i].pData[1] = const_cast<uint8_t*>(frame_buffer->DataU());
  789. pictures_[i].pData[2] = const_cast<uint8_t*>(frame_buffer->DataV());
  790. } else {
  791. pictures_[i].iStride[0] = downscaled_buffers_[i - 1]->StrideY();
  792. pictures_[i].iStride[1] = downscaled_buffers_[i - 1]->StrideU();
  793. pictures_[i].iStride[2] = downscaled_buffers_[i - 1]->StrideV();
  794. pictures_[i].pData[0] =
  795. const_cast<uint8_t*>(downscaled_buffers_[i - 1]->DataY());
  796. pictures_[i].pData[1] =
  797. const_cast<uint8_t*>(downscaled_buffers_[i - 1]->DataU());
  798. pictures_[i].pData[2] =
  799. const_cast<uint8_t*>(downscaled_buffers_[i - 1]->DataV());
  800. // Scale the image down a number of times by downsampling factor.
  801. libyuv::I420Scale(pictures_[i - 1].pData[0], pictures_[i - 1].iStride[0],
  802. pictures_[i - 1].pData[1], pictures_[i - 1].iStride[1],
  803. pictures_[i - 1].pData[2], pictures_[i - 1].iStride[2],
  804. configurations_[i - 1].width,
  805. configurations_[i - 1].height, pictures_[i].pData[0],
  806. pictures_[i].iStride[0], pictures_[i].pData[1],
  807. pictures_[i].iStride[1], pictures_[i].pData[2],
  808. pictures_[i].iStride[2], configurations_[i].width,
  809. configurations_[i].height, libyuv::kFilterBilinear);
  810. }
  811. if (!configurations_[i].sending) {
  812. continue;
  813. }
  814. if (frame_types != nullptr) {
  815. // Skip frame?
  816. if ((*frame_types)[i] == VideoFrameType::kEmptyFrame) {
  817. continue;
  818. }
  819. }
  820. if (send_key_frame) {
  821. // API doc says ForceIntraFrame(false) does nothing, but calling this
  822. // function forces a key frame regardless of the |bIDR| argument's value.
  823. // (If every frame is a key frame we get lag/delays.)
  824. encoders_[i]->ForceIntraFrame(true);
  825. configurations_[i].key_frame_request = false;
  826. }
  827. // EncodeFrame output.
  828. SFrameBSInfo info;
  829. memset(&info, 0, sizeof(SFrameBSInfo));
  830. // Encode!
  831. int enc_ret = encoders_[i]->EncodeFrame(&pictures_[i], &info);
  832. if (enc_ret != 0) {
  833. RTC_LOG(LS_ERROR)
  834. << "OpenH264 frame encoding failed, EncodeFrame returned " << enc_ret
  835. << ".";
  836. ReportError();
  837. return WEBRTC_VIDEO_CODEC_ERROR;
  838. }
  839. encoded_images_[i]._encodedWidth = configurations_[i].width;
  840. encoded_images_[i]._encodedHeight = configurations_[i].height;
  841. encoded_images_[i].SetTimestamp(input_frame.timestamp());
  842. encoded_images_[i]._frameType = ConvertToVideoFrameType(info.eFrameType);
  843. encoded_images_[i].SetSpatialIndex(configurations_[i].simulcast_idx);
  844. // Split encoded image up into fragments. This also updates
  845. // |encoded_image_|.
  846. RtpFragmentize(&encoded_images_[i], &info);
  847. // Encoder can skip frames to save bandwidth in which case
  848. // |encoded_images_[i]._length| == 0.
  849. if (encoded_images_[i].size() > 0) {
  850. // Parse QP.
  851. h264_bitstream_parser_.ParseBitstream(encoded_images_[i].data(),
  852. encoded_images_[i].size());
  853. h264_bitstream_parser_.GetLastSliceQp(&encoded_images_[i].qp_);
  854. // Deliver encoded image.
  855. CodecSpecificInfo codec_specific;
  856. codec_specific.codecType = kVideoCodecH264;
  857. codec_specific.codecSpecific.H264.packetization_mode =
  858. packetization_mode_;
  859. codec_specific.codecSpecific.H264.temporal_idx = kNoTemporalIdx;
  860. codec_specific.codecSpecific.H264.idr_frame =
  861. info.eFrameType == videoFrameTypeIDR;
  862. codec_specific.codecSpecific.H264.base_layer_sync = false;
  863. if (configurations_[i].num_temporal_layers > 1) {
  864. const uint8_t tid = info.sLayerInfo[0].uiTemporalId;
  865. codec_specific.codecSpecific.H264.temporal_idx = tid;
  866. codec_specific.codecSpecific.H264.base_layer_sync =
  867. tid > 0 && tid < tl0sync_limit_[i];
  868. if (codec_specific.codecSpecific.H264.base_layer_sync) {
  869. tl0sync_limit_[i] = tid;
  870. }
  871. if (tid == 0) {
  872. tl0sync_limit_[i] = configurations_[i].num_temporal_layers;
  873. }
  874. }
  875. encoded_image_callback_->OnEncodedImage(encoded_images_[i],
  876. &codec_specific);
  877. }
  878. }
  879. return WEBRTC_VIDEO_CODEC_OK;
  880. }
  881. // Initialization parameters.
  882. // There are two ways to initialize. There is SEncParamBase (cleared with
  883. // memset(&p, 0, sizeof(SEncParamBase)) used in Initialize, and SEncParamExt
  884. // which is a superset of SEncParamBase (cleared with GetDefaultParams) used
  885. // in InitializeExt.
  886. SEncParamExt H264EncoderImpl_::CreateEncoderParams(size_t i) const {
  887. SEncParamExt encoder_params;
  888. encoders_[i]->GetDefaultParams(&encoder_params);
  889. if (codec_.mode == VideoCodecMode::kRealtimeVideo) {
  890. encoder_params.iUsageType = CAMERA_VIDEO_REAL_TIME;
  891. } else if (codec_.mode == VideoCodecMode::kScreensharing) {
  892. encoder_params.iUsageType = SCREEN_CONTENT_REAL_TIME;
  893. } else {
  894. RTC_NOTREACHED();
  895. }
  896. encoder_params.iPicWidth = configurations_[i].width;
  897. encoder_params.iPicHeight = configurations_[i].height;
  898. encoder_params.iTargetBitrate = configurations_[i].target_bps;
  899. // Keep unspecified. WebRTC's max codec bitrate is not the same setting
  900. // as OpenH264's iMaxBitrate. More details in https://crbug.com/webrtc/11543
  901. encoder_params.iMaxBitrate = UNSPECIFIED_BIT_RATE;
  902. // Rate Control mode
  903. encoder_params.iRCMode = RC_BITRATE_MODE;
  904. encoder_params.fMaxFrameRate = configurations_[i].max_frame_rate;
  905. // The following parameters are extension parameters (they're in SEncParamExt,
  906. // not in SEncParamBase).
  907. encoder_params.bEnableFrameSkip = configurations_[i].frame_dropping_on;
  908. // |uiIntraPeriod| - multiple of GOP size
  909. // |keyFrameInterval| - number of frames
  910. encoder_params.uiIntraPeriod = configurations_[i].key_frame_interval;
  911. encoder_params.uiMaxNalSize = 0;
  912. // Threading model: use auto.
  913. // 0: auto (dynamic imp. internal encoder)
  914. // 1: single thread (default value)
  915. // >1: number of threads
  916. encoder_params.iMultipleThreadIdc = NumberOfThreads(
  917. encoder_params.iPicWidth, encoder_params.iPicHeight, number_of_cores_);
  918. // The base spatial layer 0 is the only one we use.
  919. encoder_params.sSpatialLayers[0].iVideoWidth = encoder_params.iPicWidth;
  920. encoder_params.sSpatialLayers[0].iVideoHeight = encoder_params.iPicHeight;
  921. encoder_params.sSpatialLayers[0].fFrameRate = encoder_params.fMaxFrameRate;
  922. encoder_params.sSpatialLayers[0].iSpatialBitrate =
  923. encoder_params.iTargetBitrate;
  924. encoder_params.sSpatialLayers[0].iMaxSpatialBitrate =
  925. encoder_params.iMaxBitrate;
  926. encoder_params.iTemporalLayerNum = configurations_[i].num_temporal_layers;
  927. if (encoder_params.iTemporalLayerNum > 1) {
  928. encoder_params.iNumRefFrame = 1;
  929. }
  930. RTC_LOG(INFO) << "OpenH264 version is " << OPENH264_MAJOR << "."
  931. << OPENH264_MINOR;
  932. switch (packetization_mode_) {
  933. case H264PacketizationMode::SingleNalUnit:
  934. // Limit the size of the packets produced.
  935. encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceNum = 1;
  936. encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceMode =
  937. SM_SIZELIMITED_SLICE;
  938. encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceSizeConstraint =
  939. static_cast<unsigned int>(max_payload_size_);
  940. RTC_LOG(INFO) << "Encoder is configured with NALU constraint: "
  941. << max_payload_size_ << " bytes";
  942. break;
  943. case H264PacketizationMode::NonInterleaved:
  944. // When uiSliceMode = SM_FIXEDSLCNUM_SLICE, uiSliceNum = 0 means auto
  945. // design it with cpu core number.
  946. // TODO(sprang): Set to 0 when we understand why the rate controller borks
  947. // when uiSliceNum > 1.
  948. encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceNum = 1;
  949. encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceMode =
  950. SM_FIXEDSLCNUM_SLICE;
  951. break;
  952. }
  953. return encoder_params;
  954. }
  955. void H264EncoderImpl_::ReportInit() {
  956. if (has_reported_init_)
  957. return;
  958. RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.H264EncoderImpl.Event",
  959. kH264EncoderEventInit, kH264EncoderEventMax);
  960. has_reported_init_ = true;
  961. }
  962. void H264EncoderImpl_::ReportError() {
  963. if (has_reported_error_)
  964. return;
  965. RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.H264EncoderImpl.Event",
  966. kH264EncoderEventError, kH264EncoderEventMax);
  967. has_reported_error_ = true;
  968. }
  969. VideoEncoder::EncoderInfo H264EncoderImpl_::GetEncoderInfo() const {
  970. EncoderInfo info;
  971. info.supports_native_handle = false;
  972. info.implementation_name = "OpenH264";
  973. info.scaling_settings =
  974. VideoEncoder::ScalingSettings(kLowH264QpThreshold, kHighH264QpThreshold);
  975. info.is_hardware_accelerated = false;
  976. info.has_internal_source = false;
  977. info.supports_simulcast = true;
  978. return info;
  979. }
  980. void H264EncoderImpl_::LayerConfig::SetStreamState(bool send_stream) {
  981. if (send_stream && !sending) {
  982. // Need a key frame if we have not sent this stream before.
  983. key_frame_request = true;
  984. }
  985. sending = send_stream;
  986. }
  987. } // namespace webrtc
  988. // #endif // WEBRTC_USE_H264