python及C++的yolov5检测的代码及环境结果说明文件更新

yolov5onnxC++/enviroment.txt

@@ -0,0 +1,22 @@
+库需求：
+openCV
+onnxruntime-gpu
+
+onnxruntime版本需求为：1.16.0
+
+存在问题：onnxruntimeC++安装问题
+onnxruntime安装步骤参考为：
+https://blog.csdn.net/LABLENET/article/details/136615836?ops_request_misc=%257B%2522request%255Fid%2522%253A%25226E86B7C2-BF96-4D15-A4B3-5E636040D088%2522%252C%2522scm%2522%253A%252220140713.130102334..%2522%257D&request_id=6E86B7C2-BF96-4D15-A4B3-5E636040D088&biz_id=0&utm_medium=distribute.pc_search_result.none-task-blog-2~all~sobaiduend~default-2-136615836-null-null.142^v100^pc_search_result_base4&utm_term=onnxruntime-gpu%E5%AE%89%E8%A3%85linux&spm=1018.2226.3001.4187
+
+此环境与AD10基本类似，但实际编译存在问题：
+-- SHA1 hash of
+does not match expected value
+expected: 'ee201b07085203ea7bd8eb97cbcb31b07cfa3efb'
+actual: '7fdb7133e38f370a71c38e82f03458c2b638bd29'
+-- Hash mismatch, removing...
+CMake Error at eigen-subbuild/eigen-populate-prefix/src/eigen-populate-stamp/download-eigen-populate.cmake:170 (message):
+Each download failed!
+
+测试结果：
+1、windows机上，cpu运行速度与python的cpu运行速度一致
+2、AD10上：需先解决onnxruntimeC++版本的安装问题

yolov5onnxC++/yolov5onnxC++.cpp

@@ -0,0 +1,280 @@
+#include <array>
+#include <algorithm>
+#include <iostream>
+#include <opencv2/opencv.hpp>
+#include <onnxruntime_cxx_api.h>
+#include <vector>
+#include <string>
+#include <chrono>
+#include <filesystem>
+
+using cv::Mat;
+using std::cout;
+using std::endl;
+using std::string;
+using std::vector;
+namespace fs = std::filesystem;
+
+static const vector<string> class_name = { "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light",
+                      "fire hydrant","stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow",
+                      "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee",
+                      "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard",
+                      "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple",
+                      "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch",
+                      "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", "cell phone",
+                      "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear",
+                      "hair drier", "toothbrush" };
+
+// 筛选置信度低的
+vector<vector<float>> get_info(const float* pdata, int total, float conf = 0.5, int len_data = 85)
+{
+    vector<vector<float>> info;
+    for (int i = 0; i < total / len_data; i++)
+    {
+        if (pdata[4] >= conf)
+        {
+            vector<float> line_data(pdata, pdata + len_data);
+            info.push_back(line_data);
+        }
+        pdata += len_data;
+    }
+    return info;
+}
+
+void info_simplify(vector<vector<float>>& info)
+{
+    for (int i = 0; i < info.size(); i++)
+    {
+        // 找出类别的索引
+        auto max_pos = std::max_element(info[i].cbegin() + 5, info[i].cend());
+        int class_id = std::distance(info[i].cbegin() + 5, max_pos);
+
+        // 仅保留类别为 "person" 的检测
+        if (class_id == 0)
+        {
+            info[i][5] = class_id;
+            info[i].resize(6);
+
+            float x = info[i][0];
+            float y = info[i][1];
+            float w = info[i][2];
+            float h = info[i][3];
+            info[i][0] = x - w / 2.0;
+            info[i][1] = y - h / 2.0;
+            info[i][2] = x + w / 2.0;
+            info[i][3] = y + h / 2.0;
+        }
+        else
+        {
+            info.erase(info.begin() + i);
+            i--;
+        }
+    }
+}
+
+void nms(vector<vector<float>>& info, float iou = 0.4)
+{
+    int counter = 0;
+    vector<vector<float>> return_info;
+    while (counter < info.size())
+    {
+        return_info.clear();
+        float x1 = 0;
+        float x2 = 0;
+        float y1 = 0;
+        float y2 = 0;
+        // 按照置信度排序
+        std::sort(info.begin(), info.end(), [](vector<float> p1, vector<float> p2) {
+            return p1[4] > p2[4];
+            });
+        for (auto i = 0; i < info.size(); i++)
+        {
+            if (i < counter)
+            {
+                return_info.push_back(info[i]);
+                continue;
+            }
+            if (i == counter)
+            {
+                x1 = info[i][0];
+                y1 = info[i][1];
+                x2 = info[i][2];
+                y2 = info[i][3];
+                return_info.push_back(info[i]);
+                continue;
+            }
+            if (info[i][0] > x2 || info[i][2] < x1 || info[i][1] > y2 || info[i][3] < y1)
+            {
+                return_info.push_back(info[i]);
+            }
+            else
+            {
+                float over_x1 = std::max(x1, info[i][0]);
+                float over_y1 = std::max(y1, info[i][1]);
+                float over_x2 = std::min(x2, info[i][2]);
+                float over_y2 = std::min(y2, info[i][3]);
+                float s_over = (over_x2 - over_x1) * (over_y2 - over_y1);
+                float s_total = (x2 - x1) * (y2 - y1) + (info[i][0] - info[i][2]) * (info[i][1] - info[i][3]) - s_over;
+                if (s_over / s_total < iou)
+                {
+                    return_info.push_back(info[i]);
+                }
+            }
+        }
+        info = return_info;
+        counter += 1;
+    }
+}
+
+void draw_box(Mat& img, const vector<vector<float>>& info)
+{
+    for (int i = 0; i < info.size(); i++)
+    {
+        if (static_cast<int>(info[i][5]) == 0)
+        {
+            cv::Point topLeft(info[i][0], info[i][1]);
+            cv::Point bottomRight(info[i][2], info[i][3]);
+            int thickness = 2;
+            cv::Scalar color(0, 255, 0);
+            int lineType = cv::LINE_8;
+            const int cornerRadius = 5;
+            cv::rectangle(img, topLeft, bottomRight, color, thickness, lineType);
+            string label = class_name[0] + " " + std::to_string(info[i][4]); // 仅显示 "person" 标签
+            cv::Size textSize = cv::getTextSize(label, cv::FONT_HERSHEY_SIMPLEX, 0.6, 1, nullptr);
+            cv::Rect textBgRect(topLeft.x, topLeft.y - textSize.height - 5, textSize.width + 10, textSize.height + 5);
+            cv::rectangle(img, textBgRect, color, cv::FILLED);
+            cv::putText(img, label, cv::Point(topLeft.x + 5, topLeft.y - 5), cv::FONT_HERSHEY_SIMPLEX, 0.6, CV_RGB(255, 255, 255), 2);
+        }
+    }
+}
+
+int main()
+{
+    // 定义ONNX模型路径
+    const wchar_t* model_path = L"D://Thework//testrelease//yolov5s.onnx";
+
+    // 初始化 ONNX 运行环境和内存信息
+    Ort::Env env(ORT_LOGGING_LEVEL_WARNING, "ONNXRuntime");
+    auto memory_info = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU);
+
+    // 配置会话选项
+    Ort::SessionOptions session_options;
+    session_options.SetIntraOpNumThreads(5);
+    session_options.SetGraphOptimizationLevel(ORT_ENABLE_ALL);
+
+    // 定义输入输出的名称
+    const char* input_names[] = { "images" }; // 根据你的模型输入名调整
+    const char* output_names[] = { "output" }; // 根据你的模型输出名调整
+
+    try {
+        // 创建 ONNX 会话
+        Ort::Session session(env, model_path, session_options);
+
+        // 打开摄像头
+        //cv::VideoCapture cap(0);
+        //if (!cap.isOpened())
+        //{
+        //    cout << "Error: Cannot open the camera" << endl;
+        //    return -1;
+        //}
+        cv::VideoCapture cap(7, cv::CAP_V4L2);
+        if (!cap.isOpened())
+        {
+            cout << "Error: Cannot open the camera" << endl;
+            return -1;
+        }
+
+        cap.set(cv::CAP_PROP_FRAME_WIDTH, 1280);
+        cap.set(cv::CAP_PROP_FRAME_HEIGHT, 720);
+
+        // 创建保存图像的文件夹
+        std::filesystem::path save_dir = "saved_images";
+        std::filesystem::create_directory(save_dir);
+
+        int image_count = 0; // 用于记录已保存的图像数量
+        auto start_time = std::chrono::steady_clock::now();
+
+        while (image_count < 5)
+        {
+            cv::Mat img;
+            cap >> img;
+            if (img.empty()) break;
+
+            double start = cv::getTickCount();
+
+            // 获取原始图像尺寸
+            cv::Size originalSize = img.size();
+
+            // 将图像调整为模型要求的尺寸
+            cv::Mat resized_img;
+            cv::resize(img, resized_img, cv::Size(640, 640));
+
+            cv::Mat blob = cv::dnn::blobFromImage(resized_img, 1.0 / 255.0, cv::Size(640, 640), cv::Scalar(), true);
+
+            // 定义输入Tensor
+            std::array<int64_t, 4> input_shape = { 1, 3, 640, 640 };
+            Ort::Value input_tensor = Ort::Value::CreateTensor<float>(memory_info, blob.ptr<float>(), blob.total(), input_shape.data(), input_shape.size());
+
+            // 定义输出Tensor的形状
+            std::array<int64_t, 3> output_shape{ 1, 25200, 85 }; // 根据实际模型输出调整
+            std::vector<float> output_tensor_values(1 * 25200 * 85); // 根据实际模型输出调整
+            Ort::Value output_tensor = Ort::Value::CreateTensor<float>(memory_info, output_tensor_values.data(), output_tensor_values.size(), output_shape.data(), output_shape.size());
+
+            // 推理
+            session.Run(Ort::RunOptions{ nullptr }, input_names, &input_tensor, 1, output_names, &output_tensor, 1);
+
+            // 处理输出
+            float* output_data = output_tensor.GetTensorMutableData<float>();
+            int output_size = 25200 * 85; // 根据实际模型输出调整
+
+            vector<vector<float>> info = get_info(output_data, output_size, 0.5);
+            info_simplify(info);
+            nms(info);
+
+            // 将检测框重新映射回原始图像尺寸
+            for (auto& box : info)
+            {
+                box[0] *= (float)originalSize.width / 640.0;
+                box[1] *= (float)originalSize.height / 640.0;
+                box[2] *= (float)originalSize.width / 640.0;
+                box[3] *= (float)originalSize.height / 640.0;
+            }
+
+            // 在原始图像上绘制检测框
+            draw_box(img, info);
+
+            double end = cv::getTickCount();
+            double timeSec = (end - start) / cv::getTickFrequency();
+            cout << "Frame time: " << timeSec << " seconds" << endl;
+
+            // 每隔3秒保存一张图像
+            auto now = std::chrono::steady_clock::now();
+            if (std::chrono::duration_cast<std::chrono::seconds>(now - start_time).count() >= 3)
+            {
+                string filename = (save_dir / ("image_" + std::to_string(image_count) + ".jpg")).string();
+                cv::imwrite(filename, img);
+                cout << "Saved " << filename << endl;
+                start_time = now; // 重置计时器
+                image_count++; // 增加计数器
+            }
+
+            // 等待1毫秒
+            std::this_thread::sleep_for(std::chrono::milliseconds(1));
+        }
+
+        // 释放摄像头
+        cap.release();
+        cv::destroyAllWindows();
+    }
+    catch (const Ort::Exception& e) {
+        std::cerr << "ONNX Runtime 异常: " << e.what() << std::endl;
+    }
+    catch (const std::exception& e) {
+        std::cerr << "标准异常: " << e.what() << std::endl;
+    }
+    catch (...) {
+        std::cerr << "未知异常." << std::endl;
+    }
+
+    return 0;
+}

yolov5onnxtest/enviroment.txt

@@ -1,25 +1,6 @@
 python==3.8
 onnx==1.11
+onnxruntime-gpu==1.16.0
 
-CPU时常：130-140ms。GPU待解决
-CPU使用情况：总的为600%
-onnxruntime-gpu版本待定均有报错：
-1、1.11及1.10版本可以使用CPU但无法使用cuda，报错信息为：
-2024-09-04 15:33:24.369159907 [E:onnxruntime:Default, provider_bridge_ort.cc:1022 Get] Failed to load library libonnxruntime_providers_cuda.so with error: libcublas.so.10: cannot open shared object file: No such file or directory
-2024-09-04 15:33:24.369221636 [W:onnxruntime:Default, onnxruntime_pybind_state.cc:552 CreateExecutionProviderInstance] Failed to create CUDAExecutionProvider. Please reference https://onnxruntime.ai/docs/reference/execution-providers/CUDA-ExecutionProvider.html#requirements to ensure all dependencies are met.
-
-2、1.12.1及1.17（测试了两个版本）
-不再有libcublas.so.10: cannot open shared object file: No such file or directory的错误，但是有新的报错：
-Model correct
-Using CUDA for inference.
-2024-09-04 16:15:58.913593544 [E:onnxruntime:, sequential_executor.cc:368 Execute] Non-zero status code returned while running Conv node. Name:'/model.1/conv/Conv' Status Message: :0: cudaFuncSetAttribute(kernel_entry, cudaFuncAttributeMaxDynamicSharedMemorySize, integer_cast<int32_t>(launch_configs[0].smemSizeInBytes)): invalid device function
-Traceback (most recent call last):
-  File "testonnxvideo.py", line 263, in <module>
-    main()
-  File "testonnxvideo.py", line 242, in main
-    output, org_img = model.inference(frame)
-  File "testonnxvideo.py", line 86, in inference
-    pred = self.onnx_session.run(None, input_feed)[0]
-  File "/usr/local/lib/python3.8/dist-packages/onnxruntime/capi/onnxruntime_inference_collection.py", line 200, in run
-    return self._sess.run(output_names, input_feed, run_options)
-onnxruntime.capi.onnxruntime_pybind11_state.RuntimeException: [ONNXRuntimeError] : 6 : RUNTIME_EXCEPTION : Non-zero status code returned while running Conv node. Name:'/model.1/conv/Conv' Status Message: :0: cudaFuncSetAttribute(kernel_entry, cudaFuncAttributeMaxDynamicSharedMemorySize, integer_cast<int32_t>(launch_configs[0].smemSizeInBytes)): invalid device function
+仅CPU情况下：每秒1帧，速度为130-140ms。每秒一帧CPU使用率可以降到130%
+CUDA加速（仅推理加速，前后处理未加速）：每秒检测1帧，检测速度为30ms，CPU利用率为70%

yolov5onnxtest/testonnxvideo-1.py

@@ -0,0 +1,228 @@
+import os
+import sys
+import onnx
+import onnxruntime as ort
+import cv2
+import numpy as np
+import time
+
+CLASSES = ['person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic light',
+           'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow',
+           'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee',
+           'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard',
+           'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple',
+           'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch',
+           'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone',
+           'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear',
+           'hair drier', 'toothbrush']
+
+
+class Yolov5ONNX(object):
+    def __init__(self, onnx_path):
+        onnx_model = onnx.load(onnx_path)
+        try:
+            onnx.checker.check_model(onnx_model)
+        except Exception:
+            print("Model incorrect")
+        else:
+            print("Model correct")
+
+        self.onnx_session = ort.InferenceSession(onnx_path, providers=['CUDAExecutionProvider', 'CPUExecutionProvider'])
+
+        providers = self.onnx_session.get_providers()
+        if 'CUDAExecutionProvider' in providers:
+            print("Using CUDA for inference.")
+        else:
+            print("CUDA is not available, using CPU for inference.")
+
+        self.input_name = self.get_input_name()
+        self.output_name = self.get_output_name()
+        self.input_size = (640, 640)
+
+    def get_input_name(self):
+        input_name = []
+        for node in self.onnx_session.get_inputs():
+            input_name.append(node.name)
+        return input_name
+
+    def get_output_name(self):
+        output_name = []
+        for node in self.onnx_session.get_outputs():
+            output_name.append(node.name)
+        return output_name
+
+    def get_input_feed(self, image_numpy):
+        input_feed = {}
+        for name in self.input_name:
+            input_feed[name] = image_numpy
+        return input_feed
+
+    def inference(self, img):
+        h, w, _ = img.shape
+        new_w, new_h = self.input_size
+
+        scale = min(new_w / w, new_h / h)
+        new_w = int(w * scale)
+        new_h = int(h * scale)
+
+        img_resized = cv2.resize(img, (new_w, new_h))
+
+        padded_img = np.zeros((self.input_size[1], self.input_size[0], 3), dtype=np.uint8)
+        pad_x = (self.input_size[0] - new_w) // 2
+        pad_y = (self.input_size[1] - new_h) // 2
+        padded_img[pad_y:pad_y + new_h, pad_x:pad_x + new_w] = img_resized
+
+        img_rgb = cv2.cvtColor(padded_img, cv2.COLOR_BGR2RGB).transpose(2, 0, 1)
+        img_rgb = img_rgb.astype(np.float32)
+        img_rgb /= 255.0
+        img_rgb = np.expand_dims(img_rgb, axis=0)
+
+        input_feed = self.get_input_feed(img_rgb)
+
+        start_time = time.time()
+        pred = self.onnx_session.run(None, input_feed)[0]
+        end_time = time.time()
+
+        inference_time = end_time - start_time
+        print(f"Inference time: {inference_time:.4f} seconds")
+
+        return pred, padded_img
+
+
+def nms(dets, thresh):
+    x1 = dets[:, 0]
+    y1 = dets[:, 1]
+    x2 = dets[:, 2]
+    y2 = dets[:, 3]
+
+    areas = (y2 - y1 + 1) * (x2 - x1 + 1)
+    scores = dets[:, 4]
+
+    keep = []
+    index = scores.argsort()[::-1]
+
+    while index.size > 0:
+        i = index[0]
+        keep.append(i)
+
+        x11 = np.maximum(x1[i], x1[index[1:]])
+        y11 = np.maximum(y1[i], y1[index[1:]])
+        x22 = np.minimum(x2[i], x2[index[1:]])
+        y22 = np.minimum(y2[i], y2[index[1:]])
+
+        w = np.maximum(0, x22 - x11 + 1)
+        h = np.maximum(0, y22 - y11 + 1)
+
+        overlaps = w * h
+        ious = overlaps / (areas[i] + areas[index[1:]] - overlaps)
+
+        idx = np.where(ious <= thresh)[0]
+        index = index[idx + 1]
+    return keep
+
+
+def xywh2xyxy(x):
+    y = np.copy(x)
+    y[:, 0] = x[:, 0] - x[:, 2] / 2
+    y[:, 1] = x[:, 1] - x[:, 3] / 2
+    y[:, 2] = x[:, 0] + x[:, 2] / 2
+    y[:, 3] = x[:, 1] + x[:, 3] / 2
+
+    return y
+
+
+def filter_box(org_box, conf_thres, iou_thres):
+    org_box = np.squeeze(org_box)
+    conf = org_box[..., 4] > conf_thres
+    box = org_box[conf == True]
+
+    if box.size == 0:
+        return np.array([])
+
+    cls_cinf = box[..., 5:]
+    cls = [int(np.argmax(cls_cinf[i])) for i in range(len(cls_cinf))]
+
+    person_boxes = [box[i] for i in range(len(cls)) if cls[i] == 0]
+
+    if len(person_boxes) == 0:
+        return np.array([])
+
+    person_boxes = np.array(person_boxes)
+    person_boxes = xywh2xyxy(person_boxes)
+
+    person_out_box = nms(person_boxes, iou_thres)
+    output = [person_boxes[k] for k in person_out_box]
+
+    return np.array(output)
+
+
+def draw(image, box_data):
+    if box_data.size == 0:
+        return image
+
+    boxes = box_data[..., :4].astype(np.int32)
+    scores = box_data[..., 4]
+    classes = box_data[..., 5].astype(np.int32)
+    for box, score, cl in zip(boxes, scores, classes):
+        top, left, right, bottom = box
+
+        cv2.rectangle(image, (top, left), (right, bottom), (255, 0, 0), 2)
+        cv2.putText(image, '{0} {1:.2f}'.format(CLASSES[cl], score),
+                    (top, left),
+                    cv2.FONT_HERSHEY_SIMPLEX,
+                    0.6, (0, 0, 255), 2)
+    return image
+
+
+def main():
+    onnx_path = 'yolov5s.onnx'
+    model = Yolov5ONNX(onnx_path)
+
+    cap = cv2.VideoCapture(7, cv2.CAP_V4L2)
+    cap.set(cv2.CAP_PROP_FRAME_WIDTH, 1280)
+    cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 720)
+    if not cap.isOpened():
+        print("无法打开摄像头")
+        sys.exit(0)
+
+    last_save_time = time.time()
+    frame_count = 0
+    output_dir = "saved_images"
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+
+    # 控制帧率
+    frame_rate = 1  # 每秒最多处理10帧
+    prev_time = time.time()
+
+    while True:
+        ret, frame = cap.read()
+        if not ret:
+            print("无法读取摄像头图像")
+            break
+
+        current_time = time.time()
+        if current_time - prev_time >= 1.0 / frame_rate:
+            prev_time = current_time
+
+            output, org_img = model.inference(frame)
+            outbox = filter_box(output, 0.5, 0.5)
+            org_img = draw(org_img, outbox)
+
+            if time.time() - last_save_time >= 2:
+                frame_count += 1
+                image_path = os.path.join(output_dir, f'result_{frame_count}.jpg')
+                cv2.imwrite(image_path, org_img)
+                print(f"Image saved: {image_path}")
+                last_save_time = time.time()
+
+            if frame_count >= 5:
+                print("保存了5张图片，程序退出")
+                break
+
+    cap.release()
+    cv2.destroyAllWindows()
+
+
+if __name__ == "__main__":
+    main()