yolov8转openvino并进行INT8量化(python)

使用openvino对yolov8模型进行部署，以下为模型转换和验证测试代码

1.开发环境

win11、python3.10、openvino2023.1.0、ultralytics8.3.55

2.模型转换及INT8量化

import argparse
from openvino.tools import mo
from openvino.runtime import serialize
from openvino.runtime import Core
import os
import nncf #nncf（神经网络压缩框架）用于压缩神经网络,它提供了模型剪枝、量化、稀疏化等压缩方法的工具和技术。

from utils.datasets import create_dataloader #原YOLOv5的函数
# from utils.general import check_dataset
from ultralytics.data.utils import check_det_dataset #函数接口修改

def create_data_source(dataset_yaml,image_size):
    data = check_det_dataset(dataset_yaml)
    val_dataloader = create_dataloader(data['train'], imgsz=image_size, batch_size=1, stride=32, pad=0.5, workers=1)[0]
    return val_dataloader

def transform_fn(data_item):
    # unpack input images tensor
    images = data_item[0]
    # convert input tensor into float format
    images = images.float()
    # scale input
    images = images / 255
    # convert torch tensor to numpy array
    images = images.cpu().detach().numpy()
    return images

def quant_nncf(fp32_path,dataset_yaml,nncf_int8_path,image_size):
    core = Core()
    ov_model = core.read_model(fp32_path)
    subset_size = 120
    preset = nncf.QuantizationPreset.MIXED
    # preset 是量化的预设配置，subset_size 则是用于量化校准的子集大小。
    data_source = create_data_source(dataset_yaml,image_size)
    nncf_calibration_dataset = nncf.Dataset(data_source, transform_func=transform_fn)
    print('wrap data source into nncf.dataset object. ')
    quantized_model = nncf.quantize(
        ov_model, nncf_calibration_dataset,preset=preset,subset_size=subset_size
    )
    serialize(quantized_model, nncf_int8_path)

def onnx2opnvino(onnx_path,fp32_path,fp16_path):
    # fp32 IR model
    print(f"Export ONNX to OpenVINO FP32 to: {fp32_path}")
    model = mo.convert_model(onnx_path)
    serialize(model, fp32_path)

    print(f"Export ONNX to OpenVINO FP16 to: {fp16_path}")
    model = mo.convert_model(onnx_path, compress_to_fp16=True)
    serialize(model, fp16_path)

def parse_opt():
    parser = argparse.ArgumentParser()
    parser.add_argument('--model',type=str,default=r'runs\detect\train6\weights\last.onnx',help='model path')
    parser.add_argument('--Int8',type=bool,default="True",help='export int8 model')
    parser.add_argument('--dataset',type=str,default=r'D:\project\YOLO\2024_12_16SKF_conver\ultralytics-main\ultralytics\cfg\datasets\my_detdata_all.yaml',help='dataset yaml file path')
    parser.add_argument('--image_size',type=int,default=640, help='model input image size')

    opt = parser.parse_args()
    return opt

if __name__ == "__main__":
    args = parse_opt()
    model_path=args.model
    sep=os.sep
    model=os.path.split(model_path)[-1]
    model_name=model.split('.')[0]
    model_type=model.split('.')[1]
    model_save_dir=os.path.join(os.path.split(model_path)[0],"opnvino_model")
    fp32_path = f"{model_save_dir}{sep}FP32_openvino_model{sep}{model_name}_fp32.xml"
    fp16_path = f"{model_save_dir}{sep}FP16_openvino_model{sep}{model_name}_fp16.xml"
    onnx2opnvino(args.model,fp32_path,fp16_path)

    #使用nccf进行Int8量化
    if(args.Int8):
        nncf_int8_path = f"{model_save_dir}{sep}NNCF_INT8_openvino_model{sep}{model_name}_int8.xml"
        print("start nccf qua")
        quant_nncf(fp32_path,args.dataset,nncf_int8_path,args.image_size)
        print("nccf qua done")

注意：进行INT8量化后的量化数据的加载，使用了原YOLOv5项目中的datasets.py,该python文件在utils文佳夹中

3.转换后的模型测试

from openvino.runtime import Core
import cv2
import numpy as np

# 加载OpenVINO模型
core = Core()
model_ir = core.read_model(r'D:\project\YOLO\2024_12_16SKF_conver\ultralytics-main\runs\detect\train6\weights\opnvino_model\NNCF_INT8_openvino_model\last_int8.xml')
# model_ir = core.read_model(r'D:\project\YOLO\2024_12_16SKF_conver\ultralytics-main\runs\detect\train6\weights\sim_v8n_640_cls4_all.xml')
compiled_model = core.compile_model(model_ir, device_name="CPU")
img = cv2.imread(r'conver_defect2_1.jpg')
# 获取输入和输出
input_tensor = compiled_model.input(0)
output_tensor = compiled_model.output(0)
#resize image
def letterbox(im, new_shape=(640, 640), color=(0, 0, 0)):
    shape = im.shape[:2]
    if isinstance(new_shape, int):
        new_shape = (new_shape, new_shape)
    r = min(new_shape[0] / shape[0], new_shape[1] / shape[1])
    ratio = r, r
    new_unpad = int(round(shape[1] * r)), int(round(shape[0] * r))
    dw, dh = new_shape[1] - new_unpad[0], new_shape[0] - new_unpad[1]  # wh padding
    dw /= 2
    dh /= 2
    if shape[::-1] != new_unpad:
        im = cv2.resize(im, new_unpad, interpolation=cv2.INTER_LINEAR)
    top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1))
    left, right = int(round(dw - 0.1)), int(round(dw + 0.1))
    im = cv2.copyMakeBorder(im, top, bottom, left, right, cv2.BORDER_CONSTANT, value=color)
    return im, ratio, (dw, dh)
#前处理
def preprocess_data(img):
    image, _, _ = letterbox(img, (640, 640))  # 根据模型输入要求进行调整
    input_data = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    # image=cv2.resize(image,(640,640))
    input_data = np.transpose(input_data, (2, 0, 1)).astype(np.float32)  # 转换为模型所需格式
    input_data = input_data / 255.0
    input_data = np.expand_dims(input_data, axis=0)
    return input_data,image

# 后处理：处理模型的输出
def postprocess(predictions, conf_threshold=0.5, iou_threshold=0.25):
    # 获取边界框（坐标）、置信度和类别
    predictions=np.array(predictions).squeeze().transpose(1,0)
    boxes = predictions[:, :4]  # 边界框
    scores=predictions[:,4:]
    scores_id =np.argmax(scores,axis=1)  # 置信度下标
    scores_value=np.max(scores,axis=1)   # 置信度

    # 只保留置信度大于阈值的预测框
    mask = scores_value > conf_threshold
    boxes = boxes[mask]
    # center_x,center_y,w,h to x,y,w,h
    boxes[:,0]=boxes[:,0]-boxes[:,2]/2
    boxes[:,1]=boxes[:,1]-boxes[:,3]/2

    scores = scores_value[mask]
    class_ids = scores_id[mask]

    # 进行NMS（非极大值抑制）以去除冗余框
    indices = cv2.dnn.NMSBoxes(boxes.tolist(), scores.tolist(), score_threshold=conf_threshold,
                               nms_threshold=iou_threshold)
    return boxes[indices], scores[indices], class_ids[indices]

if __name__ == "__main__":
    input_data,image = preprocess_data(img)
    results = compiled_model([input_data])
    print(results[0].shape)
    boxes, scores, class_ids = postprocess(results[0])

    # 绘制检测框
    for i in range(len(boxes)):
        x1, y1, x2, y2 = boxes[i]
        color = (0, 0, 255)  # 设置框的颜色（绿色）
        cv2.rectangle(image, (int(x1), int(y1)), (int(x1+x2), int(y1+y2)), color, 2)  # 绘制矩形框
        cv2.putText(image, f'Class {class_ids[i]}: {scores[i]:.2f}', (int(x1), int(y1) - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, color, 2)

    # 展示结果
    cv2.imshow('Detection Result', image)
    cv2.waitKey(0)
    cv2.destroyAllWindows()

4.c#推理

补充c#推理：.netframwork4.8,需要使用nuget安装opencv和openvino，目标平台一定要改成X64，否则会报错。

using OpenCvSharp.Dnn;
using OpenCvSharp;
using Sdcb.OpenVINO;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Xml.Linq;
using System.Xml.XPath;
using Sdcb.OpenVINO.Extensions.OpenCvSharp4;

namespace YOLOv8Det
{
    internal class YOLOv8Det
    {
        private Model m = null;
        private Mat ImageFloat = null;
        private CompiledModel cm = null;
        public float Confidence { get; set; }
        public float NmsThresh { get; set; }
        public int ImageSize { get; set; }
        private Scalar padColor = new Scalar(114, 114, 114);
        public long total_run_time { get; set; }
        public string[] Labels = new string[80];

        // 用于标识资源是否已经被释放
        private bool disposed = false;

        public YOLOv8Det(string model_path, float confidence, float nmsthres)
        {
            Model rawModel = OVCore.Shared.ReadModel(model_path);
            PrePostProcessor pp = rawModel.CreatePrePostProcessor();
            using (PreProcessInputInfo inputInfo = pp.Inputs.Primary)
            {
                inputInfo.TensorInfo.Layout = Layout.NHWC;
                inputInfo.ModelInfo.Layout = Layout.NCHW;
            }
            m = pp.BuildModel();
            cm = OVCore.Shared.CompileModel(m, "CPU");
            Confidence = confidence;
            NmsThresh = nmsthres;
            ImageSize = (int)m.Inputs.Primary.Shape.Dimensions[2];
            ImageFloat = new Mat();
            var dicts = XDocument.Load(model_path).XPathSelectElement(@"/net/rt_info/framework/names").Attribute("value").Value;
            Labels = ParseValueToStringArray(dicts);

        }
        public DetectionResult[] Detect(Mat img)
        {
            Stopwatch sw = new Stopwatch();
            float ratio = 0.0f;
            Point diff1 = new Point();
            Point diff2 = new Point();
            DetectionResult[] results = null;

            InferRequest ir = cm.CreateInferRequest();
            sw.Restart();

            using (var letterimg = Letterbox(img.Clone(), new Size(ImageSize, ImageSize), padColor, out ratio, out diff1, out diff2, auto: false, scaleFill: false))
            {
                sw.Stop();
                Console.WriteLine("pre time:" + sw.ElapsedMilliseconds);

                letterimg.ConvertTo(ImageFloat, MatType.CV_32FC3, 1.0 / 255);
                using (Tensor input = ImageFloat.AsTensor())
                {
                    ir.Inputs.Primary = input;
                }
                sw.Restart();
                ir.Run();
                sw.Stop();
                Console.WriteLine("infer time:" + sw.ElapsedMilliseconds);

                Tensor outputs = ir.Outputs.Primary;
                ReadOnlySpan<float> data = outputs.GetData<float>();
                sw.Restart();
                results = Postprocessing(data, outputs.Shape, ratio, diff1, Labels);
                sw.Stop();
                Console.WriteLine("post time:" + sw.ElapsedMilliseconds);

            }
            return results;
        }

        /// <summary>
        /// 后处理
        /// </summary>
        /// <param name="tensorData"></param>
        /// <param name="shape"></param>
        /// <param name="sizeRatio"></param>
        /// <param name="padding_size"></param>
        /// <param name="dicts"></param>
        /// <returns></returns>
        /// <exception cref="ArgumentException"></exception>
        public DetectionResult[] Postprocessing(ReadOnlySpan<float> tensorData, Shape shape, float sizeRatio, Point padding_size, string[] dicts)
        {
            // tensorData: 1x84x8400
            float[] t = Transpose(tensorData, shape[1], shape[2]);
            List<DetectionResult> detResults = new List<DetectionResult>();

            int objectCount = shape[2];
            int clsRowCount = shape[1];
            if (dicts.Length != clsRowCount - 4) throw new ArgumentException($"dicts length {dicts.Length} does not match shape cls row count{clsRowCount}.");
            for (int i = 0; i < objectCount; i++)
            {
                var rec = GetSlice(t, i * clsRowCount, 4);
                ReadOnlySpan<float> rectData = rec.AsSpan();
                var conf = GetSlice(t, i * clsRowCount + 4, dicts.Length);
                ReadOnlySpan<float> confidenceInfo = conf.AsSpan();
                int maxConfidenceClsId = IndexOfMax(confidenceInfo);
                float confidence = confidenceInfo[maxConfidenceClsId];

                int centerX = (int)((rectData[0] - padding_size.X) / sizeRatio);
                int centerY = (int)((rectData[1] - padding_size.Y) / sizeRatio);
                int width = (int)(rectData[2] / sizeRatio);
                int height = (int)(rectData[3] / sizeRatio);
                detResults.Add(new DetectionResult
                {
                    ClassId = maxConfidenceClsId,
                    Class = dicts[maxConfidenceClsId],
                    Rect = new Rect(centerX - width / 2, centerY - height / 2, width, height),
                    Confidence = confidence
                });
            }

            CvDnn.NMSBoxes(detResults.Select(x => x.Rect), detResults.Select(x => x.Confidence), scoreThreshold: Confidence, nmsThreshold: NmsThresh, out int[] indices);
            return detResults.Where((x, i) => indices.Contains(i)).ToArray();
        }

        static int IndexOfMax(ReadOnlySpan<float> data)
        {
            if (data.Length == 0) throw new ArgumentException("The provided data span is null or empty.");

            // 初始化最大值及其索引
            int maxIndex = 0;
            float maxValue = data[0];

            // 遍历跨度查找最大值及其索引
            for (int i = 1; i < data.Length; i++)
            {
                if (data[i] > maxValue)
                {
                    maxValue = data[i];
                    maxIndex = i;
                }
            }
            // 返回最大值索引
            return maxIndex;
        }


        static float[] Transpose(ReadOnlySpan<float> tensorData, int rows, int cols)
        {
            float[] transposedTensorData = new float[tensorData.Length];

            for (int i = 0; i < rows; i++)
            {
                for (int j = 0; j < cols; j++)
                {
                    // Index in the original tensor
                    int index = i * cols + j;

                    // Index in the transposed tensor
                    int transposedIndex = j * rows + i;

                    transposedTensorData[transposedIndex] = tensorData[index];
                }
            }

            return transposedTensorData;
        }
        static float[] GetSlice(float[] source, int start, int length)
        {
            float[] slice = new float[length];
            Array.Copy(source, start, slice, 0, length);
            return slice;
        }

        public static Mat Letterbox(Mat img, Size sz, Scalar color, out float ratio, out Point diff, out Point diff2,
                                    bool auto = false, bool scaleFill = false, bool scaleup = true)
        {
            Mat newImage = new Mat();
            Cv2.CvtColor(img, newImage, ColorConversionCodes.BGR2RGB);
            ratio = Math.Min((float)sz.Width / newImage.Width, (float)sz.Height / newImage.Height);
            if (!scaleup)
            {
                ratio = Math.Min(ratio, 1.0f);
            }
            var newUnpad = new OpenCvSharp.Size((int)Math.Round(newImage.Width * ratio),
                (int)Math.Round(newImage.Height * ratio));
            var dW = sz.Width - newUnpad.Width;
            var dH = sz.Height - newUnpad.Height;

            var tensor_ratio = sz.Height / (float)sz.Width;
            var input_ratio = img.Height / (float)img.Width;
            if (auto && tensor_ratio != input_ratio)
            {
                dW %= 32;
                dH %= 32;
            }
            else if (scaleFill)
            {
                dW = 0;
                dH = 0;
                newUnpad = sz;
            }
            var dW_h = (int)Math.Round((float)dW / 2);
            var dH_h = (int)Math.Round((float)dH / 2);
            var dw2 = 0;
            var dh2 = 0;
            if (dW_h * 2 != dW)
            {
                dw2 = dW - dW_h * 2;
            }
            if (dH_h * 2 != dH)
            {
                dh2 = dH - dH_h * 2;
            }

            if (newImage.Width != newUnpad.Width || newImage.Height != newUnpad.Height)
            {
                Cv2.Resize(newImage, newImage, newUnpad);
            }
            Cv2.CopyMakeBorder(newImage, newImage, dH_h + dh2, dH_h, dW_h + dw2, dW_h, BorderTypes.Constant, color);
            Cv2.Resize(newImage, newImage, sz);
            diff = new OpenCvSharp.Point(dW_h, dH_h);
            diff2 = new OpenCvSharp.Point(dw2, dh2);
            return newImage;
        }
        /// <summary>
        /// 解析xml文件 value 属性为 string[]，忽略序号。获取类别标签。
        /// </summary>
        /// <param name="value"></param>
        /// <returns></returns>
        public static string[] ParseValueToStringArray(string value)
        {
            if (!string.IsNullOrEmpty(value))
            {
                // 去除大括号
                value = value.Trim('{', '}');

                // 分割键值对并提取值，忽略序号
                var names = value
                    .Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries)
                    .Select(pair => pair.Split(':')[1].Trim('\'', ' '))
                    .ToArray();

                return names;
            }
            return new string[0];
        }
        /// <summary>
        /// 绘制检测框
        /// </summary>
        /// <param name="img"></param>
        /// <param name="results"></param>
        /// <param name="is_usePositiveSample"></param>
        /// <param name="positiveSampleNum"></param>
        public void draw(Mat img, DetectionResult[] results)
        {
            int width = img.Width;
            int height = img.Height;
            foreach (DetectionResult r in results)
            {
                Cv2.Rectangle(img, r.Rect, Scalar.Red, thickness: 2);
                var (x, y) = (r.Rect.X, r.Rect.Y);
                var message = r.Class + ":" + Math.Round(r.Confidence, 2).ToString();
                Cv2.PutText(img, message, new Point(x, y - 5), HersheyFonts.HersheyPlain, 1, Scalar.Red, 2);
            }
        }

        public void Dispose()
        {
            // 调用 Dispose 方法来释放托管和非托管资源
            Dispose(true);
            // 防止垃圾回收器调用析构函数
            GC.SuppressFinalize(this);
        }

        protected virtual void Dispose(bool disposing)
        {
            // 避免重复释放资源
            if (!disposed)
            {
                if (disposing)
                {
                    // 释放托管资源
                    if (ImageFloat != null)
                    {
                        ImageFloat.Dispose();
                        ImageFloat = null;
                    }

                    if (cm != null)
                    {
                        cm.Dispose();
                        cm = null;
                    }

                    if (m != null)
                    {
                        m.Dispose();
                        m = null;
                    }
                }
                // 标记资源已被释放
                disposed = true;
            }
        }
    }
    /// <summary>
    /// 模型检测结果数据结构定义
    /// </summary>
    public class DetectionResult
    {
        public int ClassId { get; set; }
        public string Class { get; set; }
        public Rect Rect { get; set; }
        public float Confidence { get; set; }
    }
}