openh264_imp.cpp 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632
  1. /*
  2. * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
  3. *
  4. * Use of this source code is governed by a BSD-style license
  5. * that can be found in the LICENSE file in the root of the source
  6. * tree. An additional intellectual property rights grant can be found
  7. * in the file PATENTS. All contributing project authors may
  8. * be found in the AUTHORS file in the root of the source tree.
  9. *
  10. */
  11. // Everything declared/defined in this header is only required when WebRTC is
  12. // build with H264 support, please do not move anything out of the
  13. // #ifdef unless needed and tested.
  14. // #include "modules/video_coding/codecs/h264/h264_encoder_impl.h"
  15. #include <limits>
  16. #include <string>
  17. #include "absl/strings/match.h"
  18. #include "common_video/libyuv/include/webrtc_libyuv.h"
  19. #include "modules/video_coding/utility/simulcast_rate_allocator.h"
  20. #include "modules/video_coding/utility/simulcast_utility.h"
  21. #include "rtc_base/checks.h"
  22. #include "rtc_base/logging.h"
  23. #include "rtc_base/time_utils.h"
  24. #include "system_wrappers/include/metrics.h"
  25. #include "third_party/libyuv/include/libyuv/convert.h"
  26. #include "third_party/libyuv/include/libyuv/scale.h"
  27. #include "third_party/openh264/src/codec/api/svc/codec_api.h"
  28. #include "third_party/openh264/src/codec/api/svc/codec_app_def.h"
  29. #include "third_party/openh264/src/codec/api/svc/codec_def.h"
  30. #include "third_party/openh264/src/codec/api/svc/codec_ver.h"
  31. #include "openh264_imp.h"
  32. namespace webrtc {
  33. namespace {
  34. const bool kOpenH264EncoderDetailedLogging = false;
  35. // QP scaling thresholds.
  36. static const int kLowH264QpThreshold = 24;
  37. static const int kHighH264QpThreshold = 37;
  38. // Used by histograms. Values of entries should not be changed.
  39. enum H264EncoderImplEvent {
  40. kH264EncoderEventInit = 0,
  41. kH264EncoderEventError = 1,
  42. kH264EncoderEventMax = 16,
  43. };
  44. int NumberOfThreads(int width, int height, int number_of_cores) {
  45. // TODO(hbos): In Chromium, multiple threads do not work with sandbox on Mac,
  46. // see crbug.com/583348. Until further investigated, only use one thread.
  47. // if (width * height >= 1920 * 1080 && number_of_cores > 8) {
  48. // return 8; // 8 threads for 1080p on high perf machines.
  49. // } else if (width * height > 1280 * 960 && number_of_cores >= 6) {
  50. // return 3; // 3 threads for 1080p.
  51. // } else if (width * height > 640 * 480 && number_of_cores >= 3) {
  52. // return 2; // 2 threads for qHD/HD.
  53. // } else {
  54. // return 1; // 1 thread for VGA or less.
  55. // }
  56. // TODO(sprang): Also check sSliceArgument.uiSliceNum om GetEncoderPrams(),
  57. // before enabling multithreading here.
  58. return 1;
  59. }
  60. VideoFrameType ConvertToVideoFrameType(EVideoFrameType type) {
  61. switch (type) {
  62. case videoFrameTypeIDR:
  63. return VideoFrameType::kVideoFrameKey;
  64. case videoFrameTypeSkip:
  65. case videoFrameTypeI:
  66. case videoFrameTypeP:
  67. case videoFrameTypeIPMixed:
  68. return VideoFrameType::kVideoFrameDelta;
  69. case videoFrameTypeInvalid:
  70. break;
  71. }
  72. RTC_NOTREACHED() << "Unexpected/invalid frame type: " << type;
  73. return VideoFrameType::kEmptyFrame;
  74. }
  75. } // namespace
  76. // Helper method used by H264EncoderImpl::Encode.
  77. // Copies the encoded bytes from |info| to |encoded_image|. The
  78. // |encoded_image->_buffer| may be deleted and reallocated if a bigger buffer is
  79. // required.
  80. //
  81. // After OpenH264 encoding, the encoded bytes are stored in |info| spread out
  82. // over a number of layers and "NAL units". Each NAL unit is a fragment starting
  83. // with the four-byte start code {0,0,0,1}. All of this data (including the
  84. // start codes) is copied to the |encoded_image->_buffer|.
  85. static void RtpFragmentize(EncodedImage* encoded_image, SFrameBSInfo* info) {
  86. // Calculate minimum buffer size required to hold encoded data.
  87. size_t required_capacity = 0;
  88. size_t fragments_count = 0;
  89. for (int layer = 0; layer < info->iLayerNum; ++layer) {
  90. const SLayerBSInfo& layerInfo = info->sLayerInfo[layer];
  91. for (int nal = 0; nal < layerInfo.iNalCount; ++nal, ++fragments_count) {
  92. RTC_CHECK_GE(layerInfo.pNalLengthInByte[nal], 0);
  93. // Ensure |required_capacity| will not overflow.
  94. RTC_CHECK_LE(layerInfo.pNalLengthInByte[nal],
  95. std::numeric_limits<size_t>::max() - required_capacity);
  96. required_capacity += layerInfo.pNalLengthInByte[nal];
  97. }
  98. }
  99. // TODO(nisse): Use a cache or buffer pool to avoid allocation?
  100. auto buffer = EncodedImageBuffer::Create(required_capacity);
  101. encoded_image->SetEncodedData(buffer);
  102. // Iterate layers and NAL units, note each NAL unit as a fragment and copy
  103. // the data to |encoded_image->_buffer|.
  104. const uint8_t start_code[4] = {0, 0, 0, 1};
  105. size_t frag = 0;
  106. encoded_image->set_size(0);
  107. for (int layer = 0; layer < info->iLayerNum; ++layer) {
  108. const SLayerBSInfo& layerInfo = info->sLayerInfo[layer];
  109. // Iterate NAL units making up this layer, noting fragments.
  110. size_t layer_len = 0;
  111. for (int nal = 0; nal < layerInfo.iNalCount; ++nal, ++frag) {
  112. // Because the sum of all layer lengths, |required_capacity|, fits in a
  113. // |size_t|, we know that any indices in-between will not overflow.
  114. RTC_DCHECK_GE(layerInfo.pNalLengthInByte[nal], 4);
  115. RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len + 0], start_code[0]);
  116. RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len + 1], start_code[1]);
  117. RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len + 2], start_code[2]);
  118. RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len + 3], start_code[3]);
  119. layer_len += layerInfo.pNalLengthInByte[nal];
  120. }
  121. // Copy the entire layer's data (including start codes).
  122. memcpy(buffer->data() + encoded_image->size(), layerInfo.pBsBuf, layer_len);
  123. encoded_image->set_size(encoded_image->size() + layer_len);
  124. }
  125. }
  126. H264EncoderImpl::H264EncoderImpl(const cricket::VideoCodec& codec)
  127. : packetization_mode_(H264PacketizationMode::SingleNalUnit),
  128. max_payload_size_(0),
  129. number_of_cores_(0),
  130. encoded_image_callback_(nullptr),
  131. has_reported_init_(false),
  132. has_reported_error_(false) {
  133. RTC_CHECK(absl::EqualsIgnoreCase(codec.name, cricket::kH264CodecName));
  134. std::string packetization_mode_string;
  135. if (codec.GetParam(cricket::kH264FmtpPacketizationMode,
  136. &packetization_mode_string) &&
  137. packetization_mode_string == "1") {
  138. packetization_mode_ = H264PacketizationMode::NonInterleaved;
  139. }
  140. downscaled_buffers_.reserve(kMaxSimulcastStreams - 1);
  141. encoded_images_.reserve(kMaxSimulcastStreams);
  142. encoders_.reserve(kMaxSimulcastStreams);
  143. configurations_.reserve(kMaxSimulcastStreams);
  144. tl0sync_limit_.reserve(kMaxSimulcastStreams);
  145. }
  146. H264EncoderImpl::~H264EncoderImpl() {
  147. Release();
  148. }
  149. int32_t H264EncoderImpl::InitEncode(const VideoCodec* inst,
  150. const VideoEncoder::Settings& settings) {
  151. ReportInit();
  152. if (!inst || inst->codecType != kVideoCodecH264) {
  153. ReportError();
  154. return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
  155. }
  156. if (inst->maxFramerate == 0) {
  157. ReportError();
  158. return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
  159. }
  160. if (inst->width < 1 || inst->height < 1) {
  161. ReportError();
  162. return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
  163. }
  164. int32_t release_ret = Release();
  165. if (release_ret != WEBRTC_VIDEO_CODEC_OK) {
  166. ReportError();
  167. return release_ret;
  168. }
  169. int number_of_streams = SimulcastUtility::NumberOfSimulcastStreams(*inst);
  170. bool doing_simulcast = (number_of_streams > 1);
  171. if (doing_simulcast &&
  172. !SimulcastUtility::ValidSimulcastParameters(*inst, number_of_streams)) {
  173. return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED;
  174. }
  175. downscaled_buffers_.resize(number_of_streams - 1);
  176. encoded_images_.resize(number_of_streams);
  177. encoders_.resize(number_of_streams);
  178. pictures_.resize(number_of_streams);
  179. configurations_.resize(number_of_streams);
  180. tl0sync_limit_.resize(number_of_streams);
  181. number_of_cores_ = settings.number_of_cores;
  182. max_payload_size_ = settings.max_payload_size;
  183. codec_ = *inst;
  184. // Code expects simulcastStream resolutions to be correct, make sure they are
  185. // filled even when there are no simulcast layers.
  186. if (codec_.numberOfSimulcastStreams == 0) {
  187. codec_.simulcastStream[0].width = codec_.width;
  188. codec_.simulcastStream[0].height = codec_.height;
  189. }
  190. for (int i = 0, idx = number_of_streams - 1; i < number_of_streams;
  191. ++i, --idx) {
  192. ISVCEncoder* openh264_encoder;
  193. // Create encoder.
  194. if (WelsCreateSVCEncoder(&openh264_encoder) != 0) {
  195. // Failed to create encoder.
  196. RTC_LOG(LS_ERROR) << "Failed to create OpenH264 encoder";
  197. RTC_DCHECK(!openh264_encoder);
  198. Release();
  199. ReportError();
  200. return WEBRTC_VIDEO_CODEC_ERROR;
  201. }
  202. RTC_DCHECK(openh264_encoder);
  203. if (kOpenH264EncoderDetailedLogging) {
  204. int trace_level = WELS_LOG_DETAIL;
  205. openh264_encoder->SetOption(ENCODER_OPTION_TRACE_LEVEL, &trace_level);
  206. }
  207. // else WELS_LOG_DEFAULT is used by default.
  208. // Store h264 encoder.
  209. encoders_[i] = openh264_encoder;
  210. // Set internal settings from codec_settings
  211. configurations_[i].simulcast_idx = idx;
  212. configurations_[i].sending = false;
  213. configurations_[i].width = codec_.simulcastStream[idx].width;
  214. configurations_[i].height = codec_.simulcastStream[idx].height;
  215. configurations_[i].max_frame_rate = static_cast<float>(codec_.maxFramerate);
  216. configurations_[i].frame_dropping_on = codec_.H264()->frameDroppingOn;
  217. configurations_[i].key_frame_interval = codec_.H264()->keyFrameInterval;
  218. configurations_[i].num_temporal_layers =
  219. codec_.simulcastStream[idx].numberOfTemporalLayers;
  220. // Create downscaled image buffers.
  221. if (i > 0) {
  222. downscaled_buffers_[i - 1] = I420Buffer::Create(
  223. configurations_[i].width, configurations_[i].height,
  224. configurations_[i].width, configurations_[i].width / 2,
  225. configurations_[i].width / 2);
  226. }
  227. // Codec_settings uses kbits/second; encoder uses bits/second.
  228. configurations_[i].max_bps = codec_.maxBitrate * 1000;
  229. configurations_[i].target_bps = codec_.startBitrate * 1000;
  230. // Create encoder parameters based on the layer configuration.
  231. SEncParamExt encoder_params = CreateEncoderParams(i);
  232. // Initialize.
  233. if (openh264_encoder->InitializeExt(&encoder_params) != 0) {
  234. RTC_LOG(LS_ERROR) << "Failed to initialize OpenH264 encoder";
  235. Release();
  236. ReportError();
  237. return WEBRTC_VIDEO_CODEC_ERROR;
  238. }
  239. // TODO(pbos): Base init params on these values before submitting.
  240. int video_format = EVideoFormatType::videoFormatI420;
  241. openh264_encoder->SetOption(ENCODER_OPTION_DATAFORMAT, &video_format);
  242. // Initialize encoded image. Default buffer size: size of unencoded data.
  243. const size_t new_capacity =
  244. CalcBufferSize(VideoType::kI420, codec_.simulcastStream[idx].width,
  245. codec_.simulcastStream[idx].height);
  246. encoded_images_[i].SetEncodedData(EncodedImageBuffer::Create(new_capacity));
  247. encoded_images_[i]._completeFrame = true;
  248. encoded_images_[i]._encodedWidth = codec_.simulcastStream[idx].width;
  249. encoded_images_[i]._encodedHeight = codec_.simulcastStream[idx].height;
  250. encoded_images_[i].set_size(0);
  251. tl0sync_limit_[i] = configurations_[i].num_temporal_layers;
  252. }
  253. SimulcastRateAllocator init_allocator(codec_);
  254. VideoBitrateAllocation allocation =
  255. init_allocator.Allocate(VideoBitrateAllocationParameters(
  256. DataRate::KilobitsPerSec(codec_.startBitrate), codec_.maxFramerate));
  257. SetRates(RateControlParameters(allocation, codec_.maxFramerate));
  258. return WEBRTC_VIDEO_CODEC_OK;
  259. }
  260. int32_t H264EncoderImpl::Release() {
  261. while (!encoders_.empty()) {
  262. ISVCEncoder* openh264_encoder = encoders_.back();
  263. if (openh264_encoder) {
  264. RTC_CHECK_EQ(0, openh264_encoder->Uninitialize());
  265. WelsDestroySVCEncoder(openh264_encoder);
  266. }
  267. encoders_.pop_back();
  268. }
  269. downscaled_buffers_.clear();
  270. configurations_.clear();
  271. encoded_images_.clear();
  272. pictures_.clear();
  273. tl0sync_limit_.clear();
  274. return WEBRTC_VIDEO_CODEC_OK;
  275. }
  276. int32_t H264EncoderImpl::RegisterEncodeCompleteCallback(
  277. EncodedImageCallback* callback) {
  278. encoded_image_callback_ = callback;
  279. return WEBRTC_VIDEO_CODEC_OK;
  280. }
  281. void H264EncoderImpl::SetRates(const RateControlParameters& parameters) {
  282. if (encoders_.empty()) {
  283. RTC_LOG(LS_WARNING) << "SetRates() while uninitialized.";
  284. return;
  285. }
  286. if (parameters.framerate_fps < 1.0) {
  287. RTC_LOG(LS_WARNING) << "Invalid frame rate: " << parameters.framerate_fps;
  288. return;
  289. }
  290. if (parameters.bitrate.get_sum_bps() == 0) {
  291. // Encoder paused, turn off all encoding.
  292. for (size_t i = 0; i < configurations_.size(); ++i) {
  293. configurations_[i].SetStreamState(false);
  294. }
  295. return;
  296. }
  297. codec_.maxFramerate = static_cast<uint32_t>(parameters.framerate_fps);
  298. size_t stream_idx = encoders_.size() - 1;
  299. for (size_t i = 0; i < encoders_.size(); ++i, --stream_idx) {
  300. // Update layer config.
  301. configurations_[i].target_bps =
  302. parameters.bitrate.GetSpatialLayerSum(stream_idx);
  303. configurations_[i].max_frame_rate = parameters.framerate_fps;
  304. if (configurations_[i].target_bps) {
  305. configurations_[i].SetStreamState(true);
  306. // Update h264 encoder.
  307. SBitrateInfo target_bitrate;
  308. memset(&target_bitrate, 0, sizeof(SBitrateInfo));
  309. target_bitrate.iLayer = SPATIAL_LAYER_ALL,
  310. target_bitrate.iBitrate = configurations_[i].target_bps;
  311. encoders_[i]->SetOption(ENCODER_OPTION_BITRATE, &target_bitrate);
  312. encoders_[i]->SetOption(ENCODER_OPTION_FRAME_RATE,
  313. &configurations_[i].max_frame_rate);
  314. } else {
  315. configurations_[i].SetStreamState(false);
  316. }
  317. }
  318. }
  319. int32_t H264EncoderImpl::Encode(
  320. const VideoFrame& input_frame,
  321. const std::vector<VideoFrameType>* frame_types) {
  322. if (encoders_.empty()) {
  323. ReportError();
  324. return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
  325. }
  326. if (!encoded_image_callback_) {
  327. RTC_LOG(LS_WARNING)
  328. << "InitEncode() has been called, but a callback function "
  329. "has not been set with RegisterEncodeCompleteCallback()";
  330. ReportError();
  331. return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
  332. }
  333. rtc::scoped_refptr<const I420BufferInterface> frame_buffer =
  334. input_frame.video_frame_buffer()->ToI420();
  335. bool send_key_frame = false;
  336. for (size_t i = 0; i < configurations_.size(); ++i) {
  337. if (configurations_[i].key_frame_request && configurations_[i].sending) {
  338. send_key_frame = true;
  339. break;
  340. }
  341. }
  342. if (!send_key_frame && frame_types) {
  343. for (size_t i = 0; i < configurations_.size(); ++i) {
  344. const size_t simulcast_idx =
  345. static_cast<size_t>(configurations_[i].simulcast_idx);
  346. if (configurations_[i].sending && simulcast_idx < frame_types->size() &&
  347. (*frame_types)[simulcast_idx] == VideoFrameType::kVideoFrameKey) {
  348. send_key_frame = true;
  349. break;
  350. }
  351. }
  352. }
  353. RTC_DCHECK_EQ(configurations_[0].width, frame_buffer->width());
  354. RTC_DCHECK_EQ(configurations_[0].height, frame_buffer->height());
  355. // Encode image for each layer.
  356. for (size_t i = 0; i < encoders_.size(); ++i) {
  357. // EncodeFrame input.
  358. pictures_[i] = {0};
  359. pictures_[i].iPicWidth = configurations_[i].width;
  360. pictures_[i].iPicHeight = configurations_[i].height;
  361. pictures_[i].iColorFormat = EVideoFormatType::videoFormatI420;
  362. pictures_[i].uiTimeStamp = input_frame.ntp_time_ms();
  363. // Downscale images on second and ongoing layers.
  364. if (i == 0) {
  365. pictures_[i].iStride[0] = frame_buffer->StrideY();
  366. pictures_[i].iStride[1] = frame_buffer->StrideU();
  367. pictures_[i].iStride[2] = frame_buffer->StrideV();
  368. pictures_[i].pData[0] = const_cast<uint8_t*>(frame_buffer->DataY());
  369. pictures_[i].pData[1] = const_cast<uint8_t*>(frame_buffer->DataU());
  370. pictures_[i].pData[2] = const_cast<uint8_t*>(frame_buffer->DataV());
  371. } else {
  372. pictures_[i].iStride[0] = downscaled_buffers_[i - 1]->StrideY();
  373. pictures_[i].iStride[1] = downscaled_buffers_[i - 1]->StrideU();
  374. pictures_[i].iStride[2] = downscaled_buffers_[i - 1]->StrideV();
  375. pictures_[i].pData[0] =
  376. const_cast<uint8_t*>(downscaled_buffers_[i - 1]->DataY());
  377. pictures_[i].pData[1] =
  378. const_cast<uint8_t*>(downscaled_buffers_[i - 1]->DataU());
  379. pictures_[i].pData[2] =
  380. const_cast<uint8_t*>(downscaled_buffers_[i - 1]->DataV());
  381. // Scale the image down a number of times by downsampling factor.
  382. libyuv::I420Scale(pictures_[i - 1].pData[0], pictures_[i - 1].iStride[0],
  383. pictures_[i - 1].pData[1], pictures_[i - 1].iStride[1],
  384. pictures_[i - 1].pData[2], pictures_[i - 1].iStride[2],
  385. configurations_[i - 1].width,
  386. configurations_[i - 1].height, pictures_[i].pData[0],
  387. pictures_[i].iStride[0], pictures_[i].pData[1],
  388. pictures_[i].iStride[1], pictures_[i].pData[2],
  389. pictures_[i].iStride[2], configurations_[i].width,
  390. configurations_[i].height, libyuv::kFilterBilinear);
  391. }
  392. if (!configurations_[i].sending) {
  393. continue;
  394. }
  395. if (frame_types != nullptr) {
  396. // Skip frame?
  397. if ((*frame_types)[i] == VideoFrameType::kEmptyFrame) {
  398. continue;
  399. }
  400. }
  401. if (send_key_frame) {
  402. // API doc says ForceIntraFrame(false) does nothing, but calling this
  403. // function forces a key frame regardless of the |bIDR| argument's value.
  404. // (If every frame is a key frame we get lag/delays.)
  405. encoders_[i]->ForceIntraFrame(true);
  406. configurations_[i].key_frame_request = false;
  407. }
  408. // EncodeFrame output.
  409. SFrameBSInfo info;
  410. memset(&info, 0, sizeof(SFrameBSInfo));
  411. // Encode!
  412. int enc_ret = encoders_[i]->EncodeFrame(&pictures_[i], &info);
  413. if (enc_ret != 0) {
  414. RTC_LOG(LS_ERROR)
  415. << "OpenH264 frame encoding failed, EncodeFrame returned " << enc_ret
  416. << ".";
  417. ReportError();
  418. return WEBRTC_VIDEO_CODEC_ERROR;
  419. }
  420. encoded_images_[i]._encodedWidth = configurations_[i].width;
  421. encoded_images_[i]._encodedHeight = configurations_[i].height;
  422. encoded_images_[i].SetTimestamp(input_frame.timestamp());
  423. encoded_images_[i]._frameType = ConvertToVideoFrameType(info.eFrameType);
  424. encoded_images_[i].SetSpatialIndex(configurations_[i].simulcast_idx);
  425. // printf("frametype:%d",encoded_images_[i]._frameType );
  426. // Split encoded image up into fragments. This also updates
  427. // |encoded_image_|.
  428. RtpFragmentize(&encoded_images_[i], &info);
  429. // Encoder can skip frames to save bandwidth in which case
  430. // |encoded_images_[i]._length| == 0.
  431. if (encoded_images_[i].size() > 0) {
  432. // Parse QP.
  433. h264_bitstream_parser_.ParseBitstream(encoded_images_[i].data(),
  434. encoded_images_[i].size());
  435. h264_bitstream_parser_.GetLastSliceQp(&encoded_images_[i].qp_);
  436. // Deliver encoded image.
  437. CodecSpecificInfo codec_specific;
  438. codec_specific.codecType = kVideoCodecH264;
  439. codec_specific.codecSpecific.H264.packetization_mode =
  440. packetization_mode_;
  441. codec_specific.codecSpecific.H264.temporal_idx = kNoTemporalIdx;
  442. codec_specific.codecSpecific.H264.idr_frame =
  443. info.eFrameType == videoFrameTypeIDR;
  444. codec_specific.codecSpecific.H264.base_layer_sync = false;
  445. if (configurations_[i].num_temporal_layers > 1) {
  446. const uint8_t tid = info.sLayerInfo[0].uiTemporalId;
  447. codec_specific.codecSpecific.H264.temporal_idx = tid;
  448. codec_specific.codecSpecific.H264.base_layer_sync =
  449. tid > 0 && tid < tl0sync_limit_[i];
  450. if (codec_specific.codecSpecific.H264.base_layer_sync) {
  451. tl0sync_limit_[i] = tid;
  452. }
  453. if (tid == 0) {
  454. tl0sync_limit_[i] = configurations_[i].num_temporal_layers;
  455. }
  456. }
  457. encoded_image_callback_->OnEncodedImage(encoded_images_[i],
  458. &codec_specific);
  459. }
  460. }
  461. return WEBRTC_VIDEO_CODEC_OK;
  462. }
  463. // Initialization parameters.
  464. // There are two ways to initialize. There is SEncParamBase (cleared with
  465. // memset(&p, 0, sizeof(SEncParamBase)) used in Initialize, and SEncParamExt
  466. // which is a superset of SEncParamBase (cleared with GetDefaultParams) used
  467. // in InitializeExt.
  468. SEncParamExt H264EncoderImpl::CreateEncoderParams(size_t i) const {
  469. SEncParamExt encoder_params;
  470. encoders_[i]->GetDefaultParams(&encoder_params);
  471. if (codec_.mode == VideoCodecMode::kRealtimeVideo) {
  472. encoder_params.iUsageType = CAMERA_VIDEO_REAL_TIME;
  473. } else if (codec_.mode == VideoCodecMode::kScreensharing) {
  474. encoder_params.iUsageType = SCREEN_CONTENT_REAL_TIME;
  475. } else {
  476. RTC_NOTREACHED();
  477. }
  478. encoder_params.iPicWidth = configurations_[i].width;
  479. encoder_params.iPicHeight = configurations_[i].height;
  480. encoder_params.iTargetBitrate = configurations_[i].target_bps;
  481. // Keep unspecified. WebRTC's max codec bitrate is not the same setting
  482. // as OpenH264's iMaxBitrate. More details in https://crbug.com/webrtc/11543
  483. encoder_params.iMaxBitrate = UNSPECIFIED_BIT_RATE;
  484. // Rate Control mode
  485. encoder_params.iRCMode = RC_BITRATE_MODE;
  486. encoder_params.fMaxFrameRate = configurations_[i].max_frame_rate;
  487. // The following parameters are extension parameters (they're in SEncParamExt,
  488. // not in SEncParamBase).
  489. encoder_params.bEnableFrameSkip = configurations_[i].frame_dropping_on;
  490. // |uiIntraPeriod| - multiple of GOP size
  491. // |keyFrameInterval| - number of frames
  492. encoder_params.uiIntraPeriod = configurations_[i].key_frame_interval;
  493. encoder_params.uiMaxNalSize = 0;
  494. // Threading model: use auto.
  495. // 0: auto (dynamic imp. internal encoder)
  496. // 1: single thread (default value)
  497. // >1: number of threads
  498. encoder_params.iMultipleThreadIdc = NumberOfThreads(
  499. encoder_params.iPicWidth, encoder_params.iPicHeight, number_of_cores_);
  500. // The base spatial layer 0 is the only one we use.
  501. encoder_params.sSpatialLayers[0].iVideoWidth = encoder_params.iPicWidth;
  502. encoder_params.sSpatialLayers[0].iVideoHeight = encoder_params.iPicHeight;
  503. encoder_params.sSpatialLayers[0].fFrameRate = encoder_params.fMaxFrameRate;
  504. encoder_params.sSpatialLayers[0].iSpatialBitrate =
  505. encoder_params.iTargetBitrate;
  506. encoder_params.sSpatialLayers[0].iMaxSpatialBitrate =
  507. encoder_params.iMaxBitrate;
  508. encoder_params.iTemporalLayerNum = configurations_[i].num_temporal_layers;
  509. if (encoder_params.iTemporalLayerNum > 1) {
  510. encoder_params.iNumRefFrame = 1;
  511. }
  512. RTC_LOG(INFO) << "OpenH264 version is " << OPENH264_MAJOR << "."
  513. << OPENH264_MINOR;
  514. switch (packetization_mode_) {
  515. case H264PacketizationMode::SingleNalUnit:
  516. // Limit the size of the packets produced.
  517. encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceNum = 1;
  518. encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceMode =
  519. SM_SIZELIMITED_SLICE;
  520. encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceSizeConstraint =
  521. static_cast<unsigned int>(max_payload_size_);
  522. RTC_LOG(INFO) << "Encoder is configured with NALU constraint: "
  523. << max_payload_size_ << " bytes";
  524. break;
  525. case H264PacketizationMode::NonInterleaved:
  526. // When uiSliceMode = SM_FIXEDSLCNUM_SLICE, uiSliceNum = 0 means auto
  527. // design it with cpu core number.
  528. // TODO(sprang): Set to 0 when we understand why the rate controller borks
  529. // when uiSliceNum > 1.
  530. encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceNum = 1;
  531. encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceMode =
  532. SM_FIXEDSLCNUM_SLICE;
  533. break;
  534. }
  535. return encoder_params;
  536. }
  537. void H264EncoderImpl::ReportInit() {
  538. if (has_reported_init_)
  539. return;
  540. RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.H264EncoderImpl.Event",
  541. kH264EncoderEventInit, kH264EncoderEventMax);
  542. has_reported_init_ = true;
  543. }
  544. void H264EncoderImpl::ReportError() {
  545. if (has_reported_error_)
  546. return;
  547. RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.H264EncoderImpl.Event",
  548. kH264EncoderEventError, kH264EncoderEventMax);
  549. has_reported_error_ = true;
  550. }
  551. VideoEncoder::EncoderInfo H264EncoderImpl::GetEncoderInfo() const {
  552. EncoderInfo info;
  553. info.supports_native_handle = false;
  554. info.implementation_name = "OpenH264";
  555. info.scaling_settings =
  556. VideoEncoder::ScalingSettings(kLowH264QpThreshold, kHighH264QpThreshold);
  557. info.is_hardware_accelerated = false;
  558. info.has_internal_source = false;
  559. info.supports_simulcast = true;
  560. return info;
  561. }
  562. void H264EncoderImpl::LayerConfig::SetStreamState(bool send_stream) {
  563. if (send_stream && !sending) {
  564. // Need a key frame if we have not sent this stream before.
  565. key_frame_request = true;
  566. }
  567. sending = send_stream;
  568. }
  569. } // namespace webrtc