/* * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. * */ // Everything declared/defined in this header is only required when WebRTC is // build with H264 support, please do not move anything out of the // #ifdef unless needed and tested. #include #include #include "absl/strings/match.h" #include "common_video/libyuv/include/webrtc_libyuv.h" #include "modules/video_coding/utility/simulcast_rate_allocator.h" #include "modules/video_coding/utility/simulcast_utility.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" #include "rtc_base/time_utils.h" #include "system_wrappers/include/metrics.h" #include "third_party/libyuv/include/libyuv/convert.h" #include "third_party/libyuv/include/libyuv/scale.h" #include "third_party/openh264/src/codec/api/svc/codec_api.h" #include "third_party/openh264/src/codec/api/svc/codec_app_def.h" #include "third_party/openh264/src/codec/api/svc/codec_def.h" #include "third_party/openh264/src/codec/api/svc/codec_ver.h" #include "/home/nvidia/devdata/ZJ_PRO_JET/webrtcinterop/jetson_h264_encode.h" #include "/home/nvidia/devdata/ZJ_PRO_JET/webrtcinterop/include/NvUtils.h" using namespace std; #define IS_DIGIT(c) (c >= '0' && c <= '9') #define MICROSECOND_UNIT 1000000 namespace webrtc { namespace { const bool kOpenH264EncoderDetailedLogging = false; // QP scaling thresholds. static const int kLowH264QpThreshold = 24; static const int kHighH264QpThreshold = 37; // Used by histograms. Values of entries should not be changed. enum H264EncoderImplEvent { kH264EncoderEventInit = 0, kH264EncoderEventError = 1, kH264EncoderEventMax = 16, }; // // void copyFrame(AVFrame *frame, const webrtc::I420BufferInterface *buffer) { // frame->width = buffer->width(); // frame->height = buffer->height(); // frame->format = AV_PIX_FMT_YUV420P; // frame->data[kYPlaneIndex] = const_cast(buffer->DataY()); // frame->data[kUPlaneIndex] = const_cast(buffer->DataU()); // frame->data[kVPlaneIndex] = const_cast(buffer->DataV()); // } int NumberOfThreads(int width, int height, int number_of_cores) { // TODO(hbos): In Chromium, multiple threads do not work with sandbox on Mac, // see crbug.com/583348. Until further investigated, only use one thread. // if (width * height >= 1920 * 1080 && number_of_cores > 8) { // return 8; // 8 threads for 1080p on high perf machines. // } else if (width * height > 1280 * 960 && number_of_cores >= 6) { // return 3; // 3 threads for 1080p. // } else if (width * height > 640 * 480 && number_of_cores >= 3) { // return 2; // 2 threads for qHD/HD. // } else { // return 1; // 1 thread for VGA or less. // } // TODO(sprang): Also check sSliceArgument.uiSliceNum om GetEncoderPrams(), // before enabling multithreading here. return 1; } /** * Abort on error. * * @param ctx : Encoder context */ static void abort(context_enc_t *ctx) { ctx->got_error = true; ctx->enc->abort(); } static Crc* InitCrc(unsigned int CrcPolynomial) { unsigned short int i; unsigned short int j; unsigned int tempcrc; Crc *phCrc; phCrc = (Crc*) malloc (sizeof(Crc)); if (phCrc == NULL) { cerr << "Mem allocation failed for Init CRC" < 0; j--) { if (tempcrc & 1) { tempcrc = (tempcrc >> 1) ^ CrcPolynomial; } else { tempcrc >>= 1; } } phCrc->CRCTable[i] = tempcrc; } phCrc->CrcValue = 0; return phCrc; } // static void set_defaults(context_enc_t & ctx) // { // memset(ctx, 0, sizeof(context_enc_t)); // // ctx->in_file_path = "/home/nvidia/Desktop/env_enc/jetson_enc/build/output.yuv"; // // ctx->out_file_path = "test.h264"; // // ctx->width = 1280; // // ctx->height = 720; // ctx->encoder_pixfmt = V4L2_PIX_FMT_H264; // ctx->raw_pixfmt = V4L2_PIX_FMT_YUV420M; // ctx->bitrate = 4 * 1024 * 1024; // ctx->peak_bitrate = 0; // ctx->profile = V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE; // ctx->ratecontrol = V4L2_MPEG_VIDEO_BITRATE_MODE_CBR; // ctx->iframe_interval = 30; // ctx->externalRPS = false; // ctx->enableGDR = false; // ctx->enableROI = false; // ctx->bnoIframe = false; // ctx->bGapsInFrameNumAllowed = false; // ctx->bReconCrc = false; // ctx->enableLossless = false; // ctx->nH264FrameNumBits = 0; // ctx->nH265PocLsbBits = 0; // ctx->idr_interval = 256; // ctx->level = -1; // ctx->fps_n = 30; // ctx->fps_d = 1; // ctx->gdr_start_frame_number = 0xffffffff; // ctx->gdr_num_frames = 0xffffffff; // ctx->gdr_out_frame_number = 0xffffffff; // ctx->num_b_frames = (uint32_t) -1; // ctx->nMinQpI = (uint32_t)QP_RETAIN_VAL; // ctx->nMaxQpI = (uint32_t)QP_RETAIN_VAL; // ctx->nMinQpP = (uint32_t)QP_RETAIN_VAL; // ctx->nMaxQpP = (uint32_t)QP_RETAIN_VAL; // ctx->nMinQpB = (uint32_t)QP_RETAIN_VAL; // ctx->nMaxQpB = (uint32_t)QP_RETAIN_VAL; // ctx->use_gold_crc = false; // ctx->pBitStreamCrc = NULL; // ctx->externalRCHints = false; // ctx->input_metadata = false; // ctx->sMaxQp = 51; // ctx->stats = false; // ctx->stress_test = 1; // ctx->output_memory_type = V4L2_MEMORY_DMABUF; // ctx->capture_memory_type = V4L2_MEMORY_MMAP; // ctx->cs = V4L2_COLORSPACE_SMPTE170M; // ctx->copy_timestamp = false; // ctx->sar_width = 0; // ctx->sar_height = 0; // ctx->start_ts = 0; // ctx->max_perf = 0; // ctx->blocking_mode = 1; // ctx->startf = 0; // ctx->endf = 0; // ctx->num_output_buffers = 6; // ctx->num_frames_to_encode = -1; // ctx->poc_type = 0; // ctx->chroma_format_idc = -1; // ctx->bit_depth = 8; // ctx->is_semiplanar = false; // ctx->enable_initQP = false; // ctx->IinitQP = 0; // ctx->PinitQP = 0; // ctx->BinitQP = 0; // ctx->enable_ratecontrol = true; // ctx->enable_av1tile = false; // ctx->log2_num_av1rows = 0; // ctx->log2_num_av1cols = 0; // ctx->enable_av1ssimrdo = (uint8_t)-1; // ctx->disable_av1cdfupdate = (uint8_t)-1; // ctx->ppe_init_params.enable_ppe = false; // ctx->ppe_init_params.wait_time_ms = -1; // ctx->ppe_init_params.feature_flags = V4L2_PPE_FEATURE_NONE; // ctx->ppe_init_params.enable_profiler = 0; // ctx->ppe_init_params.taq_max_qp_delta = 5; // /* TAQ for B-frames is enabled by default */ // ctx->ppe_init_params.taq_b_frame_mode = 1; // } static void CalculateCrc(Crc *phCrc, unsigned char *buffer, uint32_t count) { unsigned char *p; unsigned int temp1; unsigned int temp2; unsigned int crc = phCrc->CrcValue; unsigned int *CRCTable = phCrc->CRCTable; if(!count) return; p = (unsigned char *) buffer; while (count-- != 0) { temp1 = (crc >> 8) & 0x00FFFFFFL; temp2 = CRCTable[((unsigned int) crc ^ *p++) & 0xFF]; crc = temp1 ^ temp2; } phCrc->CrcValue = crc; } /** * Encoder polling thread loop function. * * @param args : void arguments */ static void *encoder_pollthread_fcn(void *arg) { context_enc_t *ctx = (context_enc_t *) arg; v4l2_ctrl_video_device_poll devicepoll; cout << "Starting Device Poll Thread " << endl; memset(&devicepoll, 0, sizeof(v4l2_ctrl_video_device_poll)); /* wait here until signalled to issue the Poll call. Check if the abort status is set , if so exit Else issue the Poll on the encoder and block. When the Poll returns, signal the encoder thread to continue. */ while (!ctx->got_error && !ctx->enc->isInError()) { sem_wait(&ctx->pollthread_sema); if (ctx->got_eos) { cout << "Got eos, exiting poll thread \n"; return NULL; } devicepoll.req_events = POLLIN | POLLOUT | POLLERR | POLLPRI; /* This call shall wait in the v4l2 encoder library */ ctx->enc->DevicePoll(&devicepoll); /* Can check the devicepoll.resp_events bitmask to see which events are set. */ sem_post(&ctx->encoderthread_sema); } return NULL; } static int write_encoder_output_frame(ofstream * stream, NvBuffer * buffer) { stream->write((char *) buffer->planes[0].data, buffer->planes[0].bytesused); return 0; } static bool encoder_capture_plane_dq_callback(struct v4l2_buffer *v4l2_buf, NvBuffer * buffer, NvBuffer * shared_buffer, void *arg) { printf("------------------------------------\n"); context_enc_t *ctx = (context_enc_t *) arg; NvVideoEncoder *enc = ctx->enc; pthread_setname_np(pthread_self(), "EncCapPlane"); uint32_t frame_num = ctx->enc->capture_plane.getTotalDequeuedBuffers() - 1; uint32_t ReconRef_Y_CRC = 0; uint32_t ReconRef_U_CRC = 0; uint32_t ReconRef_V_CRC = 0; static uint32_t num_encoded_frames = 1; struct v4l2_event ev; int ret = 0; if (v4l2_buf == NULL) { cout << "Error while dequeing buffer from output plane" << endl; abort(ctx); return false; } if (ctx->b_use_enc_cmd) { if(v4l2_buf->flags & V4L2_BUF_FLAG_LAST) { memset(&ev,0,sizeof(struct v4l2_event)); ret = ctx->enc->dqEvent(ev,1000); if (ret < 0) cout << "Error in dqEvent" << endl; if(ev.type == V4L2_EVENT_EOS) return false; } } /* Received EOS from encoder. Stop dqthread. */ if (buffer->planes[0].bytesused == 0) { cout << "Got 0 size buffer in capture \n"; return false; } /* Computing CRC with each frame */ if(ctx->pBitStreamCrc) CalculateCrc (ctx->pBitStreamCrc, buffer->planes[0].data, buffer->planes[0].bytesused); if (!ctx->stats) write_encoder_output_frame(ctx->out_file, buffer); /* Accounting for the first frame as it is only sps+pps */ if (ctx->gdr_out_frame_number != 0xFFFFFFFF) if ( (ctx->enableGDR) && (ctx->GDR_out_file_path) && (num_encoded_frames >= ctx->gdr_out_frame_number+1)) write_encoder_output_frame(ctx->gdr_out_file, buffer); num_encoded_frames++; if (ctx->report_metadata) { v4l2_ctrl_videoenc_outputbuf_metadata enc_metadata; if (ctx->enc->getMetadata(v4l2_buf->index, enc_metadata) == 0) { if (ctx->bReconCrc && enc_metadata.bValidReconCRC) { /* CRC for Recon frame */ cout << "Frame: " << frame_num << endl; cout << "ReconFrame_Y_CRC " << enc_metadata.ReconFrame_Y_CRC << " ReconFrame_U_CRC " << enc_metadata.ReconFrame_U_CRC << " ReconFrame_V_CRC " << enc_metadata.ReconFrame_V_CRC << endl; if (!ctx->recon_Ref_file->eof()) { string recon_ref_YUV_data[4]; parse_csv_recon_file(ctx->recon_Ref_file, recon_ref_YUV_data); ReconRef_Y_CRC = stoul(recon_ref_YUV_data[0]); ReconRef_U_CRC = stoul(recon_ref_YUV_data[1]); ReconRef_V_CRC = stoul(recon_ref_YUV_data[2]); } if ((ReconRef_Y_CRC != enc_metadata.ReconFrame_Y_CRC) || (ReconRef_U_CRC != enc_metadata.ReconFrame_U_CRC) || (ReconRef_V_CRC != enc_metadata.ReconFrame_V_CRC)) { cout << "Recon CRC FAIL" << endl; cout << "ReconRef_Y_CRC " << ReconRef_Y_CRC << " ReconRef_U_CRC " << ReconRef_U_CRC << " ReconRef_V_CRC " << ReconRef_V_CRC << endl; abort(ctx); return false; } cout << "Recon CRC PASS for frame : " << frame_num << endl; } else if (ctx->externalRPS && enc_metadata.bRPSFeedback_status) { /* RPS Feedback */ ctx->rps_par.nActiveRefFrames = enc_metadata.nActiveRefFrames; cout << "Frame: " << frame_num << endl; cout << "nCurrentRefFrameId " << enc_metadata.nCurrentRefFrameId << " nActiveRefFrames " << enc_metadata.nActiveRefFrames << endl; for (uint32_t i = 0; i < enc_metadata.nActiveRefFrames; i++) { /* Update RPS List */ ctx->rps_par.rps_list[i].nFrameId = enc_metadata.RPSList[i].nFrameId; ctx->rps_par.rps_list[i].bLTRefFrame = enc_metadata.RPSList[i].bLTRefFrame; cout << "FrameId " << enc_metadata.RPSList[i].nFrameId << " IdrFrame " << (int) enc_metadata.RPSList[i].bIdrFrame << " LTRefFrame " << (int) enc_metadata.RPSList[i].bLTRefFrame << " PictureOrderCnt " << enc_metadata.RPSList[i].nPictureOrderCnt << " FrameNum " << enc_metadata.RPSList[i].nFrameNum << " LTFrameIdx " << enc_metadata.RPSList[i].nLTRFrameIdx << endl; } } else if (ctx->externalRCHints) { /* Rate Control Feedback */ cout << "Frame: " << frame_num << endl; cout << "EncodedBits " << enc_metadata.EncodedFrameBits << " MinQP " << enc_metadata.FrameMinQP << " MaxQP " << enc_metadata.FrameMaxQP << endl; } else { cout << "Frame " << frame_num << ": isKeyFrame=" << (int) enc_metadata.KeyFrame << " AvgQP=" << enc_metadata.AvgQP << " MinQP=" << enc_metadata.FrameMinQP << " MaxQP=" << enc_metadata.FrameMaxQP << " EncodedBits=" << enc_metadata.EncodedFrameBits << endl; } } } if (ctx->dump_mv) { /* Get motion vector parameters of the frames from encoder */ v4l2_ctrl_videoenc_outputbuf_metadata_MV enc_mv_metadata; if (ctx->enc->getMotionVectors(v4l2_buf->index, enc_mv_metadata) == 0) { uint32_t numMVs = enc_mv_metadata.bufSize / sizeof(MVInfo); MVInfo *pInfo = enc_mv_metadata.pMVInfo; cout << "Frame " << frame_num << ": Num MVs=" << numMVs << endl; for (uint32_t i = 0; i < numMVs; i++, pInfo++) { cout << i << ": mv_x=" << pInfo->mv_x << " mv_y=" << pInfo->mv_y << " weight=" << pInfo->weight << endl; } } } if (ctx->blocking_mode && ctx->RPS_threeLayerSvc) { sem_post(&ctx->rps_par.sema); } /* encoder qbuffer for capture plane */ if (enc->capture_plane.qBuffer(*v4l2_buf, NULL) < 0) { cerr << "Error while Qing buffer at capture plane" << endl; abort(ctx); return false; } return true; } static int setup_output_dmabuf(context_enc_t *ctx, uint32_t num_buffers ) { int ret=0; NvBufSurf::NvCommonAllocateParams cParams; int fd; ret = ctx->enc->output_plane.reqbufs(V4L2_MEMORY_DMABUF,num_buffers); if(ret) { cerr << "reqbufs failed for output plane V4L2_MEMORY_DMABUF" << endl; return ret; } for (uint32_t i = 0; i < ctx->enc->output_plane.getNumBuffers(); i++) { cParams.width = ctx->width; cParams.height = ctx->height; cParams.layout = NVBUF_LAYOUT_PITCH; switch (ctx->cs) { case V4L2_COLORSPACE_REC709: cParams.colorFormat = ctx->enable_extended_colorformat ? NVBUF_COLOR_FORMAT_YUV420_709_ER : NVBUF_COLOR_FORMAT_YUV420_709; break; case V4L2_COLORSPACE_SMPTE170M: default: cParams.colorFormat = ctx->enable_extended_colorformat ? NVBUF_COLOR_FORMAT_YUV420_ER : NVBUF_COLOR_FORMAT_YUV420; } if (ctx->is_semiplanar) { cParams.colorFormat = NVBUF_COLOR_FORMAT_NV12; } if (ctx->encoder_pixfmt == V4L2_PIX_FMT_H264) { if (ctx->enableLossless) { if (ctx->is_semiplanar) cParams.colorFormat = NVBUF_COLOR_FORMAT_NV24; else cParams.colorFormat = NVBUF_COLOR_FORMAT_YUV444; } } else if (ctx->encoder_pixfmt == V4L2_PIX_FMT_H265) { if (ctx->chroma_format_idc == 3) { if (ctx->is_semiplanar) cParams.colorFormat = NVBUF_COLOR_FORMAT_NV24; else cParams.colorFormat = NVBUF_COLOR_FORMAT_YUV444; if (ctx->bit_depth == 10) cParams.colorFormat = NVBUF_COLOR_FORMAT_NV24_10LE; } if (ctx->profile == V4L2_MPEG_VIDEO_H265_PROFILE_MAIN10 && (ctx->bit_depth == 10)) { cParams.colorFormat = NVBUF_COLOR_FORMAT_NV12_10LE; } } cParams.memtag = NvBufSurfaceTag_VIDEO_ENC; cParams.memType = NVBUF_MEM_SURFACE_ARRAY; /* Create output plane fd for DMABUF io-mode */ ret = NvBufSurf::NvAllocate(&cParams, 1, &fd); if(ret < 0) { cerr << "Failed to create NvBuffer" << endl; return ret; } ctx->output_plane_fd[i]=fd; } return ret; } static int setup_capture_dmabuf(context_enc_t *ctx, uint32_t num_buffers ) { NvBufSurfaceAllocateParams cParams = {{0}}; NvBufSurface *surface = 0; int ret=0; ret = ctx->enc->capture_plane.reqbufs(V4L2_MEMORY_DMABUF,num_buffers); if(ret) { cerr << "reqbufs failed for capture plane V4L2_MEMORY_DMABUF" << endl; return ret; } for (uint32_t i = 0; i < ctx->enc->capture_plane.getNumBuffers(); i++) { ret = ctx->enc->capture_plane.queryBuffer(i); if (ret) { cerr << "Error in querying for " << i << "th buffer plane" << endl; return ret; } NvBuffer *buffer = ctx->enc->capture_plane.getNthBuffer(i); cParams.params.memType = NVBUF_MEM_HANDLE; cParams.params.size = buffer->planes[0].length; cParams.memtag = NvBufSurfaceTag_VIDEO_ENC; ret = NvBufSurfaceAllocate(&surface, 1, &cParams); if(ret < 0) { cerr << "Failed to create NvBuffer" << endl; return ret; } surface->numFilled = 1; ctx->capture_plane_fd[i] = surface->surfaceList[0].bufferDesc; } return ret; } static int get_next_parsed_pair(context_enc_t *ctx, char *id, uint32_t *value) { char charval; *ctx->runtime_params_str >> *id; if (ctx->runtime_params_str->eof()) { return -1; } charval = ctx->runtime_params_str->peek(); if (!IS_DIGIT(charval)) { return -1; } *ctx->runtime_params_str >> *value; *ctx->runtime_params_str >> charval; if (ctx->runtime_params_str->eof()) { return 0; } return charval; } static int get_next_runtime_param_change_frame(context_enc_t *ctx) { char charval; int ret; ret = get_next_parsed_pair(ctx, &charval, &ctx->next_param_change_frame); if(ret == 0) { return 0; } // TEST_PARSE_ERROR((ret != ';' && ret != ',') || charval != 'f', err); return 0; // err: // cerr << "Skipping further runtime parameter changes" <runtime_params_str; // ctx->runtime_params_str = NULL; // return -1; } VideoFrameType ConvertToVideoFrameType(EVideoFrameType type) { switch (type) { case videoFrameTypeIDR: return VideoFrameType::kVideoFrameKey; case videoFrameTypeSkip: case videoFrameTypeI: case videoFrameTypeP: case videoFrameTypeIPMixed: return VideoFrameType::kVideoFrameDelta; case videoFrameTypeInvalid: break; } RTC_NOTREACHED() << "Unexpected/invalid frame type: " << type; return VideoFrameType::kEmptyFrame; } } // namespace // Helper method used by H264EncoderImpl::Encode. // Copies the encoded bytes from |info| to |encoded_image|. The // |encoded_image->_buffer| may be deleted and reallocated if a bigger buffer is // required. // // After OpenH264 encoding, the encoded bytes are stored in |info| spread out // over a number of layers and "NAL units". Each NAL unit is a fragment starting // with the four-byte start code {0,0,0,1}. All of this data (including the // start codes) is copied to the |encoded_image->_buffer|. static void RtpFragmentize(EncodedImage* encoded_image, SFrameBSInfo* info) { // Calculate minimum buffer size required to hold encoded data. size_t required_capacity = 0; size_t fragments_count = 0; for (int layer = 0; layer < info->iLayerNum; ++layer) { const SLayerBSInfo& layerInfo = info->sLayerInfo[layer]; for (int nal = 0; nal < layerInfo.iNalCount; ++nal, ++fragments_count) { RTC_CHECK_GE(layerInfo.pNalLengthInByte[nal], 0); // Ensure |required_capacity| will not overflow. RTC_CHECK_LE(layerInfo.pNalLengthInByte[nal], std::numeric_limits::max() - required_capacity); required_capacity += layerInfo.pNalLengthInByte[nal]; } } // TODO(nisse): Use a cache or buffer pool to avoid allocation? auto buffer = EncodedImageBuffer::Create(required_capacity); encoded_image->SetEncodedData(buffer); // Iterate layers and NAL units, note each NAL unit as a fragment and copy // the data to |encoded_image->_buffer|. const uint8_t start_code[4] = {0, 0, 0, 1}; size_t frag = 0; encoded_image->set_size(0); for (int layer = 0; layer < info->iLayerNum; ++layer) { const SLayerBSInfo& layerInfo = info->sLayerInfo[layer]; // Iterate NAL units making up this layer, noting fragments. size_t layer_len = 0; for (int nal = 0; nal < layerInfo.iNalCount; ++nal, ++frag) { // Because the sum of all layer lengths, |required_capacity|, fits in a // |size_t|, we know that any indices in-between will not overflow. RTC_DCHECK_GE(layerInfo.pNalLengthInByte[nal], 4); RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len + 0], start_code[0]); RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len + 1], start_code[1]); RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len + 2], start_code[2]); RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len + 3], start_code[3]); layer_len += layerInfo.pNalLengthInByte[nal]; } // Copy the entire layer's data (including start codes). memcpy(buffer->data() + encoded_image->size(), layerInfo.pBsBuf, layer_len); encoded_image->set_size(encoded_image->size() + layer_len); } } JetH264Encoder::JetH264Encoder(const cricket::VideoCodec& codec) : packetization_mode_(H264PacketizationMode::SingleNalUnit), max_payload_size_(0), number_of_cores_(0), encoded_image_callback_(nullptr), has_reported_init_(false), has_reported_error_(false) { RTC_CHECK(absl::EqualsIgnoreCase(codec.name, cricket::kH264CodecName)); std::string packetization_mode_string; if (codec.GetParam(cricket::kH264FmtpPacketizationMode, &packetization_mode_string) && packetization_mode_string == "1") { packetization_mode_ = H264PacketizationMode::NonInterleaved; } downscaled_buffers_.reserve(kMaxSimulcastStreams - 1); encoded_images_.reserve(kMaxSimulcastStreams); encoders_.reserve(kMaxSimulcastStreams); configurations_.reserve(kMaxSimulcastStreams); tl0sync_limit_.reserve(kMaxSimulcastStreams); } JetH264Encoder::~JetH264Encoder() { Release(); } // //增加 bool JetH264Encoder::OpenEncoder(context_enc_t *ctx, JetH264Encoder::LayerConfig &config) { int ret = 0; int error = 0; bool eos = false; memset(ctx, 0, sizeof(context_enc_t)); // ctx->in_file_path = "/home/nvidia/Desktop/env_enc/jetson_enc/build/output.yuv"; // ctx->out_file_path = "test.h264"; // ctx->width = 1280; // ctx->height = 720; ctx->encoder_pixfmt = V4L2_PIX_FMT_H264; ctx->raw_pixfmt = V4L2_PIX_FMT_YUV420M; ctx->bitrate = 4 * 1024 * 1024; ctx->peak_bitrate = 0; ctx->profile = V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE; ctx->ratecontrol = V4L2_MPEG_VIDEO_BITRATE_MODE_CBR; ctx->iframe_interval = 30; ctx->externalRPS = false; ctx->enableGDR = false; ctx->enableROI = false; ctx->bnoIframe = false; ctx->bGapsInFrameNumAllowed = false; ctx->bReconCrc = false; ctx->enableLossless = false; ctx->nH264FrameNumBits = 0; ctx->nH265PocLsbBits = 0; ctx->idr_interval = 256; ctx->level = -1; ctx->fps_n = 30; ctx->fps_d = 1; ctx->gdr_start_frame_number = 0xffffffff; ctx->gdr_num_frames = 0xffffffff; ctx->gdr_out_frame_number = 0xffffffff; ctx->num_b_frames = (uint32_t) -1; ctx->nMinQpI = (uint32_t)QP_RETAIN_VAL; ctx->nMaxQpI = (uint32_t)QP_RETAIN_VAL; ctx->nMinQpP = (uint32_t)QP_RETAIN_VAL; ctx->nMaxQpP = (uint32_t)QP_RETAIN_VAL; ctx->nMinQpB = (uint32_t)QP_RETAIN_VAL; ctx->nMaxQpB = (uint32_t)QP_RETAIN_VAL; ctx->use_gold_crc = false; ctx->pBitStreamCrc = NULL; ctx->externalRCHints = false; ctx->input_metadata = false; ctx->sMaxQp = 51; ctx->stats = false; ctx->stress_test = 1; ctx->output_memory_type = V4L2_MEMORY_DMABUF; ctx->capture_memory_type = V4L2_MEMORY_MMAP; ctx->cs = V4L2_COLORSPACE_SMPTE170M; ctx->copy_timestamp = false; ctx->sar_width = 0; ctx->sar_height = 0; ctx->start_ts = 0; ctx->max_perf = 0; ctx->blocking_mode = 1; ctx->startf = 0; ctx->endf = 0; ctx->num_output_buffers = 6; ctx->num_frames_to_encode = -1; ctx->poc_type = 0; ctx->chroma_format_idc = -1; ctx->bit_depth = 8; ctx->is_semiplanar = false; ctx->enable_initQP = false; ctx->IinitQP = 0; ctx->PinitQP = 0; ctx->BinitQP = 0; ctx->enable_ratecontrol = true; ctx->enable_av1tile = false; ctx->log2_num_av1rows = 0; ctx->log2_num_av1cols = 0; ctx->enable_av1ssimrdo = (uint8_t)-1; ctx->disable_av1cdfupdate = (uint8_t)-1; ctx->ppe_init_params.enable_ppe = false; ctx->ppe_init_params.wait_time_ms = -1; ctx->ppe_init_params.feature_flags = V4L2_PPE_FEATURE_NONE; ctx->ppe_init_params.enable_profiler = 0; ctx->ppe_init_params.taq_max_qp_delta = 5; /* TAQ for B-frames is enabled by default */ ctx->ppe_init_params.taq_b_frame_mode = 1; /* Set default values for encoder context members. */ // set_defaults(&ctx); /* Parse application command line options. */ // ret = parse_csv_args(&ctx, argc, argv); // TEST_ERROR(ret < 0, "Error parsing commandline arguments", cleanup); // std ::cout << ctx.encoder_pixfmt << std::endl; /* Set thread name for encoder Output Plane thread. */ pthread_setname_np(pthread_self(),"EncOutPlane"); // /* Get the parsed encoder runtime parameters */ // if (ctx->runtime_params_str) // { // get_next_runtime_param_change_frame(&ctx); // } if (ctx->endf) { ctx->num_frames_to_encode = ctx->endf - ctx->startf + 1; } if (ctx->use_gold_crc) { /* CRC specific initializetion if gold_crc flag is set */ ctx->pBitStreamCrc = InitCrc(CRC32_POLYNOMIAL); } /* Open input file for raw yuv */ ctx->in_file = new ifstream(ctx->in_file_path); if (!ctx->stats) { /* Open output file for encoded bitstream */ ctx->out_file = new ofstream(ctx->out_file_path); } if (ctx->ROI_Param_file_path) { /* Open Region of Intreset(ROI) parameter file when ROI feature enabled */ ctx->roi_Param_file = new ifstream(ctx->ROI_Param_file_path); } if (ctx->Recon_Ref_file_path) { /* Open Reconstructed CRC reference file when ReconCRC feature enabled */ ctx->recon_Ref_file = new ifstream(ctx->Recon_Ref_file_path); } if (ctx->RPS_Param_file_path) { /* Open Reference Picture set(RPS) specififc reference file when Dynamic RPS feature enabled */ ctx->rps_Param_file = new ifstream(ctx->RPS_Param_file_path); } if (ctx->GDR_Param_file_path) { /* Open Gradual Decoder Refresh(GDR) parameters reference file when GDR feature enabled */ ctx->gdr_Param_file = new ifstream(ctx->GDR_Param_file_path); } if (ctx->GDR_out_file_path) { /* Open Gradual Decoder Refresh(GDR) output parameters reference file when GDR feature enabled */ ctx->gdr_out_file = new ofstream(ctx->GDR_out_file_path); } if (ctx->hints_Param_file_path) { /* Open external hints parameters file for when external rate control feature enabled */ ctx->hints_Param_file = new ifstream(ctx->hints_Param_file_path); } /* Create NvVideoEncoder object for blocking or non-blocking I/O mode. */ //进入 if (ctx->blocking_mode) { cout << "Creating Encoder in blocking mode \n"; ctx->enc = NvVideoEncoder::createVideoEncoder("enc0"); } else { cout << "Creating Encoder in non-blocking mode \n"; ctx->enc = NvVideoEncoder::createVideoEncoder("enc0", O_NONBLOCK); } if (ctx->stats) { ctx->enc->enableProfiling(); } /* Set encoder capture plane format. NOTE: It is necessary that Capture Plane format be set before Output Plane format. It is necessary to set width and height on the capture plane as well */ ret = ctx->enc->setCapturePlaneFormat(ctx->encoder_pixfmt, ctx->width, ctx->height, 2 * 1024 * 1024); if (ctx->encoder_pixfmt == V4L2_PIX_FMT_H265) { switch (ctx->profile) { case V4L2_MPEG_VIDEO_H265_PROFILE_MAIN10: { ctx->raw_pixfmt = V4L2_PIX_FMT_P010M; ctx->is_semiplanar = true; /* To keep previous execution commands working */ ctx->bit_depth = 10; break; } case V4L2_MPEG_VIDEO_H265_PROFILE_MAIN: { if (ctx->is_semiplanar) ctx->raw_pixfmt = V4L2_PIX_FMT_NV12M; else ctx->raw_pixfmt = V4L2_PIX_FMT_YUV420M; if (ctx->chroma_format_idc == 3) { if (ctx->bit_depth == 10 && ctx->is_semiplanar) ctx->raw_pixfmt = V4L2_PIX_FMT_NV24_10LE; if (ctx->bit_depth == 8) { if (ctx->is_semiplanar) ctx->raw_pixfmt = V4L2_PIX_FMT_NV24M; else ctx->raw_pixfmt = V4L2_PIX_FMT_YUV444M; } } } break; default: ctx->raw_pixfmt = V4L2_PIX_FMT_YUV420M; } } if (ctx->encoder_pixfmt == V4L2_PIX_FMT_H264) { if (ctx->enableLossless && ctx->profile == V4L2_MPEG_VIDEO_H264_PROFILE_HIGH_444_PREDICTIVE) { if (ctx->is_semiplanar) ctx->raw_pixfmt = V4L2_PIX_FMT_NV24M; else ctx->raw_pixfmt = V4L2_PIX_FMT_YUV444M; } else if ((ctx->enableLossless && ctx->profile != V4L2_MPEG_VIDEO_H264_PROFILE_HIGH_444_PREDICTIVE) || (!ctx->enableLossless && ctx->profile == V4L2_MPEG_VIDEO_H264_PROFILE_HIGH_444_PREDICTIVE)) { cerr << "Lossless encoding is supported only for high444 profile\n"; error = 1; // goto cleanup; } else { if (ctx->is_semiplanar) ctx->raw_pixfmt = V4L2_PIX_FMT_NV12M; else ctx->raw_pixfmt = V4L2_PIX_FMT_YUV420M; } } /* Set encoder output plane format */ ret = ctx->enc->setOutputPlaneFormat(ctx->raw_pixfmt, ctx->width, ctx->height); if (ctx->num_frames_to_encode) { ret = ctx->enc->setFramesToEncode(ctx->num_frames_to_encode); } ret = ctx->enc->setBitrate(ctx->bitrate); if (ctx->encoder_pixfmt == V4L2_PIX_FMT_H264) { /* Set encoder profile for H264 format */ ret = ctx->enc->setProfile(ctx->profile); if (ctx->level == (uint32_t)-1) { ctx->level = (uint32_t)V4L2_MPEG_VIDEO_H264_LEVEL_5_1; } ret = ctx->enc->setLevel(ctx->level); } else if (ctx->encoder_pixfmt == V4L2_PIX_FMT_H265) { ret = ctx->enc->setProfile(ctx->profile); if (ctx->level != (uint32_t)-1) { ret = ctx->enc->setLevel(ctx->level); } if (ctx->chroma_format_idc != (uint8_t)-1) { ret = ctx->enc->setChromaFactorIDC(ctx->chroma_format_idc); } } if (ctx->enable_initQP) { ret = ctx->enc->setInitQP(ctx->IinitQP, ctx->PinitQP, ctx->BinitQP); } if (ctx->enableLossless) { ret = ctx->enc->setLossless(ctx->enableLossless); } else if (!ctx->enable_ratecontrol) { /* Set constant QP configuration by disabling rate control */ ret = ctx->enc->setConstantQp(ctx->enable_ratecontrol); } else { /* Set rate control mode for encoder */ ret = ctx->enc->setRateControlMode(ctx->ratecontrol); if (ctx->ratecontrol == V4L2_MPEG_VIDEO_BITRATE_MODE_VBR) { uint32_t peak_bitrate; if (ctx->peak_bitrate < ctx->bitrate) peak_bitrate = 1.2f * ctx->bitrate; else peak_bitrate = ctx->peak_bitrate; /* Set peak bitrate value for variable bitrate mode for encoder */ ret = ctx->enc->setPeakBitrate(peak_bitrate); } } if (ctx->poc_type) { ret = ctx->enc->setPocType(ctx->poc_type); } /* Set IDR frame interval for encoder */ ret = ctx->enc->setIDRInterval(ctx->idr_interval); /* Set I frame interval for encoder */ ret = ctx->enc->setIFrameInterval(ctx->iframe_interval); /* Set framerate for encoder */ ret = ctx->enc->setFrameRate(ctx->fps_n, ctx->fps_d); if (ctx->temporal_tradeoff_level) { /* Set temporal tradeoff level value for encoder */ ret = ctx->enc->setTemporalTradeoff(ctx->temporal_tradeoff_level); } if (ctx->slice_length) { /* Set slice length value for encoder */ ret = ctx->enc->setSliceLength(ctx->slice_length_type, ctx->slice_length); } if (ctx->enable_slice_level_encode) { /* Enable slice level encode for encoder */ ret = ctx->enc->setSliceLevelEncode(true); } if (ctx->hw_preset_type) { /* Set hardware preset value for encoder */ ret = ctx->enc->setHWPresetType(ctx->hw_preset_type); } if (ctx->virtual_buffer_size) { /* Set virtual buffer size value for encoder */ ret = ctx->enc->setVirtualBufferSize(ctx->virtual_buffer_size); } if (ctx->slice_intrarefresh_interval) { /* Set slice intra refresh interval value for encoder */ ret = ctx->enc->setSliceIntrarefresh(ctx->slice_intrarefresh_interval); } if (ctx->insert_sps_pps_at_idr) { /* Enable insert of SPSPPS at IDR frames */ ret = ctx->enc->setInsertSpsPpsAtIdrEnabled(true); } if (ctx->disable_cabac) { /* Disable CABAC entropy encoding */ ret = ctx->enc->setCABAC(false); } if (ctx->sar_width) { /* Set SAR width */ ret = ctx->enc->setSampleAspectRatioWidth(ctx->sar_width); } if (ctx->sar_height) { /* Set SAR width */ ret = ctx->enc->setSampleAspectRatioHeight(ctx->sar_height); } if (ctx->insert_vui) { /* Enable insert of VUI parameters */ ret = ctx->enc->setInsertVuiEnabled(true); } if (ctx->enable_extended_colorformat) { /* Enable extnded colorformat for encoder */ ret = ctx->enc->setExtendedColorFormat(true); } if (ctx->insert_aud) { /* Enable insert of AUD parameters */ ret = ctx->enc->setInsertAudEnabled(true); } if (ctx->alliframes) { /* Enable all I-frame encode */ ret = ctx->enc->setAlliFramesEncode(true); } if (ctx->num_b_frames != (uint32_t) -1) { /* Set number of B-frames to to be used by encoder */ ret = ctx->enc->setNumBFrames(ctx->num_b_frames); } if ((ctx->nMinQpI != (uint32_t)QP_RETAIN_VAL) || (ctx->nMaxQpI != (uint32_t)QP_RETAIN_VAL) || (ctx->nMinQpP != (uint32_t)QP_RETAIN_VAL) || (ctx->nMaxQpP != (uint32_t)QP_RETAIN_VAL) || (ctx->nMinQpB != (uint32_t)QP_RETAIN_VAL) || (ctx->nMaxQpB != (uint32_t)QP_RETAIN_VAL)) { /* Set Min & Max qp range values for I/P/B-frames to be used by encoder */ ret = ctx->enc->setQpRange(ctx->nMinQpI, ctx->nMaxQpI, ctx->nMinQpP, ctx->nMaxQpP, ctx->nMinQpB, ctx->nMaxQpB); } if (ctx->max_perf) { /* Enable maximum performance mode by disabling internal DFS logic. NOTE: This enables encoder to run at max clocks */ ret = ctx->enc->setMaxPerfMode(ctx->max_perf); } if (ctx->dump_mv) { /* Enable dumping of motion vectors report from encoder */ ret = ctx->enc->enableMotionVectorReporting(); } if (ctx->bnoIframe) { ctx->iframe_interval = ((1<<31) + 1); /* TODO: how can we do this properly */ ret = ctx->enc->setIFrameInterval(ctx->iframe_interval); } if (ctx->enableROI) { v4l2_enc_enable_roi_param VEnc_enable_ext_roi_ctrl; VEnc_enable_ext_roi_ctrl.bEnableROI = ctx->enableROI; /* Enable region of intrest configuration for encoder */ ret = ctx->enc->enableROI(VEnc_enable_ext_roi_ctrl); } if (ctx->bReconCrc) { v4l2_enc_enable_reconcrc_param VEnc_enable_recon_crc_ctrl; VEnc_enable_recon_crc_ctrl.bEnableReconCRC = ctx->bReconCrc; /* Enable reconstructed CRC configuration for encoder */ ret = ctx->enc->enableReconCRC(VEnc_enable_recon_crc_ctrl); } if (ctx->externalRPS) { v4l2_enc_enable_ext_rps_ctr VEnc_enable_ext_rps_ctrl; VEnc_enable_ext_rps_ctrl.bEnableExternalRPS = ctx->externalRPS; if (ctx->encoder_pixfmt == V4L2_PIX_FMT_H264) { VEnc_enable_ext_rps_ctrl.bGapsInFrameNumAllowed = ctx->bGapsInFrameNumAllowed; VEnc_enable_ext_rps_ctrl.nH264FrameNumBits = ctx->nH264FrameNumBits; } if (ctx->encoder_pixfmt == V4L2_PIX_FMT_H265) { VEnc_enable_ext_rps_ctrl.nH265PocLsbBits = ctx->nH265PocLsbBits; } /* Enable external reference picture set configuration for encoder */ ret = ctx->enc->enableExternalRPS(VEnc_enable_ext_rps_ctrl); } if (ctx->num_reference_frames) { /* Set number of reference frame configuration value for encoder */ ret = ctx->enc->setNumReferenceFrames(ctx->num_reference_frames); } if (ctx->externalRCHints) { v4l2_enc_enable_ext_rate_ctr VEnc_enable_ext_rate_ctrl; VEnc_enable_ext_rate_ctrl.bEnableExternalPictureRC = ctx->externalRCHints; VEnc_enable_ext_rate_ctrl.nsessionMaxQP = ctx->sMaxQp; /* Enable external rate control configuration for encoder */ ret = ctx->enc->enableExternalRC(VEnc_enable_ext_rate_ctrl); } if (ctx->encoder_pixfmt == V4L2_PIX_FMT_AV1) { if (ctx->enable_av1tile) { v4l2_enc_av1_tile_config VEnc_av1_tile_config; VEnc_av1_tile_config.bEnableTile = ctx->enable_av1tile; VEnc_av1_tile_config.nLog2RowTiles = ctx->log2_num_av1rows; VEnc_av1_tile_config.nLog2ColTiles = ctx->log2_num_av1cols; /* Enable tile configuration for encoder */ ret = ctx->enc->enableAV1Tile(VEnc_av1_tile_config); } if (ctx->enable_av1ssimrdo != (uint8_t) -1) { ret = ctx->enc->setAV1SsimRdo(ctx->enable_av1ssimrdo); } if (ctx->disable_av1cdfupdate != (uint8_t) -1) { ret = ctx->enc->setAV1DisableCDFUpdate(ctx->disable_av1cdfupdate); } } /* Query, Export and Map the output plane buffers so that we can read raw data into the buffers */ switch(ctx->output_memory_type) { case V4L2_MEMORY_MMAP: ret = ctx->enc->output_plane.setupPlane(V4L2_MEMORY_MMAP, 10, true, false); break; case V4L2_MEMORY_USERPTR: ret = ctx->enc->output_plane.setupPlane(V4L2_MEMORY_USERPTR, 10, false, true); break; case V4L2_MEMORY_DMABUF: ret = setup_output_dmabuf(ctx,10); break; default : break; } /* Query, Export and Map the capture plane buffers so that we can write encoded bitstream data into the buffers */ switch(ctx->capture_memory_type) { case V4L2_MEMORY_MMAP: ret = ctx->enc->capture_plane.setupPlane(V4L2_MEMORY_MMAP, ctx->num_output_buffers, true, false); // TEST_ERROR(ret < 0, "Could not setup capture plane", cleanup); break; case V4L2_MEMORY_DMABUF: ret = setup_capture_dmabuf(ctx,ctx->num_output_buffers); break; default : break; } /* Subscibe for End Of Stream event */ ret = ctx->enc->subscribeEvent(V4L2_EVENT_EOS,0,0); if (ctx->b_use_enc_cmd) { /* Send v4l2 command for encoder start */ ret = ctx->enc->setEncoderCommand(V4L2_ENC_CMD_START, 0); } else { /* set encoder output plane STREAMON */ ret = ctx->enc->output_plane.setStreamStatus(true); /* set encoder capture plane STREAMON */ ret = ctx->enc->capture_plane.setStreamStatus(true); } if (ctx->blocking_mode) //进入 { if (ctx->RPS_threeLayerSvc) { sem_init(&ctx->rps_par.sema, 0, 0); } /* Set encoder capture plane dq thread callback for blocking io mode */ ctx->enc->capture_plane.setDQThreadCallback(encoder_capture_plane_dq_callback); /* startDQThread starts a thread internally which calls the encoder_capture_plane_dq_callback whenever a buffer is dequeued on the plane */ ctx->enc->capture_plane.startDQThread(&ctx); } else { sem_init(&ctx->pollthread_sema, 0, 0); sem_init(&ctx->encoderthread_sema, 0, 0); /* Set encoder poll thread for non-blocking io mode */ pthread_create(&ctx->enc_pollthread, NULL, encoder_pollthread_fcn, &ctx); pthread_setname_np(ctx->enc_pollthread, "EncPollThread"); cout << "Created the PollThread and Encoder Thread \n"; } /* Enqueue all the empty capture plane buffers. */ //将一组缓冲区排队到编码器的捕获平面 for (uint32_t i = 0; i < ctx->enc->capture_plane.getNumBuffers(); i++) { struct v4l2_buffer v4l2_buf; struct v4l2_plane planes[MAX_PLANES]; memset(&v4l2_buf, 0, sizeof(v4l2_buf)); memset(planes, 0, MAX_PLANES * sizeof(struct v4l2_plane)); v4l2_buf.index = i; v4l2_buf.m.planes = planes; if(ctx->capture_memory_type == V4L2_MEMORY_DMABUF) { v4l2_buf.m.planes[0].m.fd = ctx->capture_plane_fd[i]; /* Map capture plane buffer for memory type DMABUF. */ ret = ctx->enc->capture_plane.mapOutputBuffers(v4l2_buf, ctx->capture_plane_fd[i]); // if (ret < 0) // { // cerr << "Error while mapping buffer at capture plane" << endl; // abort(&ctx); // goto cleanup; // } } ret = ctx->enc->capture_plane.qBuffer(v4l2_buf, NULL); // if (ret < 0) // { // cerr << "Error while queueing buffer at capture plane" << endl; // abort(&ctx); // goto cleanup; // } } if (ctx->copy_timestamp) { /* Set user provided timestamp when copy timestamp is enabled */ ctx->timestamp = (ctx->start_ts * MICROSECOND_UNIT); ctx->timestampincr = (MICROSECOND_UNIT * 16) / ((uint32_t) (ctx->fps_n * 16)); } if(ctx->ppe_init_params.enable_ppe) { ret = ctx->enc->setPPEInitParams(ctx->ppe_init_params); if (ret < 0){ cerr << "Error calling setPPEInitParams" << endl; } } config.target_bps = config.max_bps; return true; } int32_t JetH264Encoder::InitEncode(const VideoCodec* inst, const VideoEncoder::Settings& settings) { printf("init ----------------------------------------\n"); ReportInit(); if (!inst || inst->codecType != kVideoCodecH264) { ReportError(); return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } if (inst->maxFramerate == 0) { ReportError(); return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } if (inst->width < 1 || inst->height < 1) { ReportError(); return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } int32_t release_ret = Release(); if (release_ret != WEBRTC_VIDEO_CODEC_OK) { ReportError(); return release_ret; } int number_of_streams = SimulcastUtility::NumberOfSimulcastStreams(*inst); bool doing_simulcast = (number_of_streams > 1); if (doing_simulcast && !SimulcastUtility::ValidSimulcastParameters(*inst, number_of_streams)) { return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED; } downscaled_buffers_.resize(number_of_streams - 1); encoded_images_.resize(number_of_streams); encoders_.resize(number_of_streams); pictures_.resize(number_of_streams);// configurations_.resize(number_of_streams); tl0sync_limit_.resize(number_of_streams); for (int i = 0; i < number_of_streams; i++){ encoders_[i] = new context_enc_t(); } number_of_cores_ = settings.number_of_cores; max_payload_size_ = settings.max_payload_size; codec_ = *inst; // Code expects simulcastStream resolutions to be correct, make sure they are // filled even when there are no simulcast layers. if (codec_.numberOfSimulcastStreams == 0) { codec_.simulcastStream[0].width = codec_.width; codec_.simulcastStream[0].height = codec_.height; } for (int i = 0, idx = number_of_streams - 1; i < number_of_streams; ++i, --idx) { // Set internal settings from codec_settings configurations_[i].simulcast_idx = idx; configurations_[i].sending = false; configurations_[i].width = codec_.simulcastStream[idx].width; configurations_[i].height = codec_.simulcastStream[idx].height; configurations_[i].max_frame_rate = static_cast(codec_.maxFramerate); configurations_[i].frame_dropping_on = codec_.H264()->frameDroppingOn; configurations_[i].key_frame_interval = codec_.H264()->keyFrameInterval; configurations_[i].num_temporal_layers = codec_.simulcastStream[idx].numberOfTemporalLayers; // Create downscaled image buffers. if (i > 0) { downscaled_buffers_[i - 1] = I420Buffer::Create( configurations_[i].width, configurations_[i].height, configurations_[i].width, configurations_[i].width / 2, configurations_[i].width / 2); } // Codec_settings uses kbits/second; encoder uses bits/second. configurations_[i].max_bps = codec_.maxBitrate * 1000; //2500000 configurations_[i].target_bps = codec_.startBitrate * 1000; //2000000 if (!OpenEncoder(encoders_[i], configurations_[i])) { Release(); ReportError(); return WEBRTC_VIDEO_CODEC_ERROR; } //初始化编码图片大小  默认是 使用未编码数据大小 // Initialize encoded image. Default buffer size: size of unencoded data. const size_t new_capacity = CalcBufferSize(VideoType::kI420, codec_.simulcastStream[idx].width, codec_.simulcastStream[idx].height); encoded_images_[i].SetEncodedData(EncodedImageBuffer::Create(new_capacity)); encoded_images_[i]._completeFrame = true; encoded_images_[i]._encodedWidth = codec_.simulcastStream[idx].width; encoded_images_[i]._encodedHeight = codec_.simulcastStream[idx].height; encoded_images_[i].set_size(0); tl0sync_limit_[i] = configurations_[i].num_temporal_layers; } SimulcastRateAllocator init_allocator(codec_); VideoBitrateAllocation allocation = init_allocator.Allocate(VideoBitrateAllocationParameters( DataRate::KilobitsPerSec(codec_.startBitrate), codec_.maxFramerate)); // SetRates(RateControlParameters(allocation, codec_.maxFramerate)); return WEBRTC_VIDEO_CODEC_OK; } // void JetH264Encoder::copyFrame(AVFrame *frame, const webrtc::I420BufferInterface *buffer) { // frame->width = buffer->width(); // frame->height = buffer->height(); // frame->format = AV_PIX_FMT_YUV420P; // frame->data[kYPlaneIndex] = const_cast(buffer->DataY()); // frame->data[kUPlaneIndex] = const_cast(buffer->DataU()); // frame->data[kVPlaneIndex] = const_cast(buffer->DataV()); // } // int32_t JetH264Encoder::Release() { // while (!encoders_.empty()) { // // ISVCEncoder* openh264_encoder = encoders_.back(); // context_enc_t* encoders_ = encoders_.back(); // if (encoders_) { // RTC_CHECK_EQ(0, openh264_encoder->Uninitialize()); // WelsDestroySVCEncoder(openh264_encoder); // } // encoders_.pop_back(); // } // downscaled_buffers_.clear(); // configurations_.clear(); // encoded_images_.clear(); // pictures_.clear(); // tl0sync_limit_.clear(); // return WEBRTC_VIDEO_CODEC_OK; // } int32_t JetH264Encoder::RegisterEncodeCompleteCallback( EncodedImageCallback* callback) { encoded_image_callback_ = callback; return WEBRTC_VIDEO_CODEC_OK; } // void JetH264Encoder::SetRates(const RateControlParameters& parameters) { // if (encoders_.empty()) { // RTC_LOG(LS_WARNING) << "SetRates() while uninitialized."; // return; // } // if (parameters.framerate_fps < 1.0) { // RTC_LOG(LS_WARNING) << "Invalid frame rate: " << parameters.framerate_fps; // return; // } // if (parameters.bitrate.get_sum_bps() == 0) { // // Encoder paused, turn off all encoding. // for (size_t i = 0; i < configurations_.size(); ++i) { // configurations_[i].SetStreamState(false); // } // return; // } // codec_.maxFramerate = static_cast(parameters.framerate_fps); // size_t stream_idx = encoders_.size() - 1; // for (size_t i = 0; i < encoders_.size(); ++i, --stream_idx) { // // Update layer config. // configurations_[i].target_bps = // parameters.bitrate.GetSpatialLayerSum(stream_idx); // configurations_[i].max_frame_rate = parameters.framerate_fps; // if (configurations_[i].target_bps) { // configurations_[i].SetStreamState(true); // // Update h264 encoder. // SBitrateInfo target_bitrate; // memset(&target_bitrate, 0, sizeof(SBitrateInfo)); // target_bitrate.iLayer = SPATIAL_LAYER_ALL, // target_bitrate.iBitrate = configurations_[i].target_bps; // encoders_[i]->SetOption(ENCODER_OPTION_BITRATE, &target_bitrate); // encoders_[i]->SetOption(ENCODER_OPTION_FRAME_RATE, // &configurations_[i].max_frame_rate); // } else { // configurations_[i].SetStreamState(false); // } // } // } int32_t JetH264Encoder::Encode( const VideoFrame& input_frame, const std::vector* frame_types) { if (encoders_.empty()) { ReportError(); return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } if (!encoded_image_callback_) { RTC_LOG(LS_WARNING) << "InitEncode() has been called, but a callback function " "has not been set with RegisterEncodeCompleteCallback()"; ReportError(); return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } rtc::scoped_refptr frame_buffer = input_frame.video_frame_buffer()->ToI420(); bool send_key_frame = false; for (size_t i = 0; i < configurations_.size(); ++i) { if (configurations_[i].key_frame_request && configurations_[i].sending) { send_key_frame = true; break; } } if (!send_key_frame && frame_types) { for (size_t i = 0; i < configurations_.size(); ++i) { const size_t simulcast_idx = static_cast(configurations_[i].simulcast_idx); if (configurations_[i].sending && simulcast_idx < frame_types->size() && (*frame_types)[simulcast_idx] == VideoFrameType::kVideoFrameKey) { send_key_frame = true; break; } } } RTC_DCHECK_EQ(configurations_[0].width, frame_buffer->width()); RTC_DCHECK_EQ(configurations_[0].height, frame_buffer->height()); // Encode image for each layer. for (size_t i = 0; i < encoders_.size(); ++i) { // EncodeFrame input. // copyFrame(encoders_[i]->frame, frame_buffer); pictures_[i] = {0}; pictures_[i].iPicWidth = configurations_[i].width; pictures_[i].iPicHeight = configurations_[i].height; pictures_[i].iColorFormat = EVideoFormatType::videoFormatI420; pictures_[i].uiTimeStamp = input_frame.ntp_time_ms(); // Downscale images on second and ongoing layers. if (i == 0) { pictures_[i].iStride[0] = frame_buffer->StrideY(); pictures_[i].iStride[1] = frame_buffer->StrideU(); pictures_[i].iStride[2] = frame_buffer->StrideV(); pictures_[i].pData[0] = const_cast(frame_buffer->DataY()); pictures_[i].pData[1] = const_cast(frame_buffer->DataU()); pictures_[i].pData[2] = const_cast(frame_buffer->DataV()); } else { pictures_[i].iStride[0] = downscaled_buffers_[i - 1]->StrideY(); pictures_[i].iStride[1] = downscaled_buffers_[i - 1]->StrideU(); pictures_[i].iStride[2] = downscaled_buffers_[i - 1]->StrideV(); pictures_[i].pData[0] = const_cast(downscaled_buffers_[i - 1]->DataY()); pictures_[i].pData[1] = const_cast(downscaled_buffers_[i - 1]->DataU()); pictures_[i].pData[2] = const_cast(downscaled_buffers_[i - 1]->DataV()); // Scale the image down a number of times by downsampling factor. libyuv::I420Scale(pictures_[i - 1].pData[0], pictures_[i - 1].iStride[0], pictures_[i - 1].pData[1], pictures_[i - 1].iStride[1], pictures_[i - 1].pData[2], pictures_[i - 1].iStride[2], configurations_[i - 1].width, configurations_[i - 1].height, pictures_[i].pData[0], pictures_[i].iStride[0], pictures_[i].pData[1], pictures_[i].iStride[1], pictures_[i].pData[2], pictures_[i].iStride[2], configurations_[i].width, configurations_[i].height, libyuv::kFilterBilinear); } if (!configurations_[i].sending) { continue; } if (frame_types != nullptr) { // Skip frame? if ((*frame_types)[i] == VideoFrameType::kEmptyFrame) { continue; } } if (send_key_frame) { // API doc says ForceIntraFrame(false) does nothing, but calling this // function forces a key frame regardless of the |bIDR| argument's value. // (If every frame is a key frame we get lag/delays.) encoders_[i]->ForceIntraFrame(true); configurations_[i].key_frame_request = false; } // EncodeFrame output. SFrameBSInfo info; memset(&info, 0, sizeof(SFrameBSInfo)); // Encode! int enc_ret = encoders_[i]->EncodeFrame(&pictures_[i], &info); if (enc_ret != 0) { RTC_LOG(LS_ERROR) << "OpenH264 frame encoding failed, EncodeFrame returned " << enc_ret << "."; ReportError(); return WEBRTC_VIDEO_CODEC_ERROR; } encoded_images_[i]._encodedWidth = configurations_[i].width; encoded_images_[i]._encodedHeight = configurations_[i].height; encoded_images_[i].SetTimestamp(input_frame.timestamp()); encoded_images_[i]._frameType = ConvertToVideoFrameType(info.eFrameType); encoded_images_[i].SetSpatialIndex(configurations_[i].simulcast_idx); // Split encoded image up into fragments. This also updates // |encoded_image_|. RtpFragmentize(&encoded_images_[i], &info); // Encoder can skip frames to save bandwidth in which case // |encoded_images_[i]._length| == 0. //进行编码帧的输出 if (encoded_images_[i].size() > 0) { // Parse QP. h264_bitstream_parser_.ParseBitstream(encoded_images_[i].data(), encoded_images_[i].size()); h264_bitstream_parser_.GetLastSliceQp(&encoded_images_[i].qp_); // Deliver encoded image. CodecSpecificInfo codec_specific; codec_specific.codecType = kVideoCodecH264; codec_specific.codecSpecific.H264.packetization_mode = packetization_mode_; codec_specific.codecSpecific.H264.temporal_idx = kNoTemporalIdx; codec_specific.codecSpecific.H264.idr_frame = info.eFrameType == videoFrameTypeIDR; codec_specific.codecSpecific.H264.base_layer_sync = false; if (configurations_[i].num_temporal_layers > 1) { const uint8_t tid = info.sLayerInfo[0].uiTemporalId; codec_specific.codecSpecific.H264.temporal_idx = tid; codec_specific.codecSpecific.H264.base_layer_sync = tid > 0 && tid < tl0sync_limit_[i]; if (codec_specific.codecSpecific.H264.base_layer_sync) { tl0sync_limit_[i] = tid; } if (tid == 0) { tl0sync_limit_[i] = configurations_[i].num_temporal_layers; } } encoded_image_callback_->OnEncodedImage(encoded_images_[i], &codec_specific); } } return WEBRTC_VIDEO_CODEC_OK; } // // Initialization parameters. // // There are two ways to initialize. There is SEncParamBase (cleared with // // memset(&p, 0, sizeof(SEncParamBase)) used in Initialize, and SEncParamExt // // which is a superset of SEncParamBase (cleared with GetDefaultParams) used // // in InitializeExt. // SEncParamExt JetH264Encoder::CreateEncoderParams(size_t i) const { // SEncParamExt encoder_params; // encoders_[i]->GetDefaultParams(&encoder_params); // if (codec_.mode == VideoCodecMode::kRealtimeVideo) { // encoder_params.iUsageType = CAMERA_VIDEO_REAL_TIME; // } else if (codec_.mode == VideoCodecMode::kScreensharing) { // encoder_params.iUsageType = SCREEN_CONTENT_REAL_TIME; // } else { // printf("other"); // RTC_NOTREACHED(); // } // encoder_params.iPicWidth = configurations_[i].width; // encoder_params.iPicHeight = configurations_[i].height; // encoder_params.iTargetBitrate = configurations_[i].target_bps; //目标码率即开始码率 // // Keep unspecified. WebRTC's max codec bitrate is not the same setting // // as OpenH264's iMaxBitrate. More details in https://crbug.com/webrtc/11543 // encoder_params.iMaxBitrate = UNSPECIFIED_BIT_RATE; //最大码率.默认 // // Rate Control mode // encoder_params.iRCMode = RC_BITRATE_MODE; //速率控制模式 // encoder_params.fMaxFrameRate = configurations_[i].max_frame_rate;//最大帧率 // // The following parameters are extension parameters (they're in SEncParamExt, // // not in SEncParamBase). // encoder_params.bEnableFrameSkip = configurations_[i].frame_dropping_on; //是否开启抽帧 // // |uiIntraPeriod| - multiple of GOP size // // |keyFrameInterval| - number of frames // encoder_params.uiIntraPeriod = configurations_[i].key_frame_interval;//关键帧间隔 // encoder_params.uiMaxNalSize = 0; // // Threading model: use auto. // // 0: auto (dynamic imp. internal encoder) // // 1: single thread (default value) // // >1: number of threads // encoder_params.iMultipleThreadIdc = NumberOfThreads( // encoder_params.iPicWidth, encoder_params.iPicHeight, number_of_cores_); // // The base spatial layer 0 is the only one we use. 空间层配置 // encoder_params.sSpatialLayers[0].iVideoWidth = encoder_params.iPicWidth; // encoder_params.sSpatialLayers[0].iVideoHeight = encoder_params.iPicHeight; // encoder_params.sSpatialLayers[0].fFrameRate = encoder_params.fMaxFrameRate; // encoder_params.sSpatialLayers[0].iSpatialBitrate = // encoder_params.iTargetBitrate; // encoder_params.sSpatialLayers[0].iMaxSpatialBitrate = // encoder_params.iMaxBitrate; // encoder_params.iTemporalLayerNum = configurations_[i].num_temporal_layers; // if (encoder_params.iTemporalLayerNum > 1) { // encoder_params.iNumRefFrame = 1; // } // RTC_LOG(INFO) << "OpenH264 version is " << OPENH264_MAJOR << "." // << OPENH264_MINOR; // //打包模式 // switch (packetization_mode_) { // case H264PacketizationMode::SingleNalUnit: // // Limit the size of the packets produced. // encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceNum = 1; // encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceMode = // SM_SIZELIMITED_SLICE; // encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceSizeConstraint = // static_cast(max_payload_size_); // RTC_LOG(INFO) << "Encoder is configured with NALU constraint: " // << max_payload_size_ << " bytes"; // break; // case H264PacketizationMode::NonInterleaved: // // When uiSliceMode = SM_FIXEDSLCNUM_SLICE, uiSliceNum = 0 means auto // // design it with cpu core number. // // TODO(sprang): Set to 0 when we understand why the rate controller borks // // when uiSliceNum > 1. // encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceNum = 1; // encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceMode = // SM_FIXEDSLCNUM_SLICE; // break; // } // return encoder_params; // } void JetH264Encoder::ReportInit() { if (has_reported_init_) return; RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.H264EncoderImpl.Event", kH264EncoderEventInit, kH264EncoderEventMax); has_reported_init_ = true; } void JetH264Encoder::ReportError() { if (has_reported_error_) return; RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.H264EncoderImpl.Event", kH264EncoderEventError, kH264EncoderEventMax); has_reported_error_ = true; } VideoEncoder::EncoderInfo JetH264Encoder::GetEncoderInfo() const { EncoderInfo info; info.supports_native_handle = false; info.implementation_name = "OpenH264"; info.scaling_settings = VideoEncoder::ScalingSettings(kLowH264QpThreshold, kHighH264QpThreshold); info.is_hardware_accelerated = false; info.has_internal_source = false; info.supports_simulcast = true; return info; } void JetH264Encoder::LayerConfig::SetStreamState(bool send_stream) { if (send_stream && !sending) { // Need a key frame if we have not sent this stream before. key_frame_request = true; } sending = send_stream; } } // namespace webrtc