diff --git a/DH.Commons/DH.Commons.csproj b/DH.Commons/DH.Commons.csproj index 1c2b199..9d1c41f 100644 --- a/DH.Commons/DH.Commons.csproj +++ b/DH.Commons/DH.Commons.csproj @@ -4,7 +4,7 @@ net8.0 enable enable - AnyCPU;X64 + AnyCPU;x64 diff --git a/DH.Devices.Camera/DH.Devices.Camera.csproj b/DH.Devices.Camera/DH.Devices.Camera.csproj index af2ea5b..ea384fe 100644 --- a/DH.Devices.Camera/DH.Devices.Camera.csproj +++ b/DH.Devices.Camera/DH.Devices.Camera.csproj @@ -19,7 +19,7 @@ - ..\X64\Debug\DVPCameraCS64.dll + ..\x64\Debug\DVPCameraCS64.dll diff --git a/DH.Devices.PLC/DH.Devices.PLC.csproj b/DH.Devices.PLC/DH.Devices.PLC.csproj index 15e4ceb..74bc23a 100644 --- a/DH.Devices.PLC/DH.Devices.PLC.csproj +++ b/DH.Devices.PLC/DH.Devices.PLC.csproj @@ -22,7 +22,7 @@ - ..\X64\Debug\HslCommunication.dll + ..\x64\Debug\HslCommunication.dll diff --git a/DH.Devices.Vision/DH.Devices.Vision.csproj b/DH.Devices.Vision/DH.Devices.Vision.csproj index 6473332..9ff2894 100644 --- a/DH.Devices.Vision/DH.Devices.Vision.csproj +++ b/DH.Devices.Vision/DH.Devices.Vision.csproj @@ -11,6 +11,9 @@ AnyCPU;x64 + + + diff --git a/DH.Devices.Vision/DetectionConfig.cs b/DH.Devices.Vision/DetectionConfig.cs new file mode 100644 index 0000000..9cea844 --- /dev/null +++ b/DH.Devices.Vision/DetectionConfig.cs @@ -0,0 +1,666 @@ +using OpenCvSharp; +using System.ComponentModel; +using System.Drawing; +using static OpenCvSharp.AgastFeatureDetector; +using System.Text.RegularExpressions; +using System.Text; +using System.Drawing.Design; + +namespace DH.Devices.Vision +{ + public enum MLModelType + { + [Description("图像分类")] + ImageClassification = 1, + [Description("目标检测")] + ObjectDetection = 2, + //[Description("图像分割")] + //ImageSegmentation = 3 + [Description("语义分割")] + SemanticSegmentation = 3, + [Description("实例分割")] + InstanceSegmentation = 4, + [Description("目标检测GPU")] + ObjectGPUDetection = 5 + } + public class ModelLabel + { + public string LabelId { get; set; } + + + [Category("模型标签")] + [DisplayName("模型标签索引")] + [Description("模型识别的标签索引")] + public int LabelIndex { get; set; } + + + [Category("模型标签")] + [DisplayName("模型标签")] + [Description("模型识别的标签名称")] + public string LabelName { get; set; } + + + + + //[Category("模型配置")] + //[DisplayName("模型参数配置")] + //[Description("模型参数配置集合")] + + //public ModelParamSetting ModelParamSetting { get; set; } = new ModelParamSetting(); + + + public string GetDisplayText() + { + return $"{LabelId}-{LabelName}"; + } + } + public class MLRequest + { + public int ImageChannels = 3; + public Mat mImage; + public int ResizeWidth; + public int ResizeHeight; + + public float confThreshold; + + public float iouThreshold; + + //public int ImageResizeCount; + public bool IsCLDetection; + public int ProCount; + public string in_node_name; + + public string out_node_name; + + public string in_lable_path; + + public int ResizeImageSize; + public int segmentWidth; + public int ImageWidth; + + // public List OkClassTxtList; + + + public List LabelNames; + + + + } + public enum ResultState + { + + [Description("检测NG")] + DetectNG = -3, + + //[Description("检测不足TBD")] + // ShortageTBD = -2, + [Description("检测结果TBD")] + ResultTBD = -1, + [Description("OK")] + OK = 1, + // [Description("NG")] + // NG = 2, + //统计结果 + [Description("A类NG")] + A_NG = 25, + [Description("B类NG")] + B_NG = 26, + [Description("C类NG")] + C_NG = 27, + } + /// + /// 深度学习 识别结果明细 面向业务：detect 面向深度学习：Recongnition、Inference + /// + public class DetectionResultDetail + { + public string LabelBGR { get; set; }//识别到对象的标签BGR + + + public int LabelNo { get; set; } // 识别到对象的标签索引 + + public string LabelName { get; set; }//识别到对象的标签名称 + + public double Score { get; set; }//识别目标结果的可能性、得分 + + public string LabelDisplay { get; set; }//识别到对象的 显示信息 + + public double Area { get; set; }//识别目标的区域面积 + + public Rectangle Rect { get; set; }//识别目标的外接矩形 + + public RotatedRect MinRect { get; set; }//识别目标的最小外接矩形(带角度) + + public ResultState InferenceResult { get; set; }//只是模型推理 label的结果 + + public double DistanceToImageCenter { get; set; } //计算矩形框到图像中心的距离 + + + + public ResultState FinalResult { get; set; }//模型推理+其他视觉、逻辑判断后 label结果 + } + public class MLResult + { + public bool IsSuccess = false; + public string ResultMessage; + public Bitmap ResultMap; + public List ResultDetails = new List(); + } + public class MLInit + { + public string ModelFile; + public string InferenceDevice; + + + public int InferenceWidth; + public int InferenceHeight; + + public string InputNodeName; + + + public int SizeModel; + + public bool bReverse;//尺寸测量正反面 + //目标检测Gpu + public bool IsGPU; + public int GPUId; + public float Score_thre; + public MLInit(string modelFile, bool isGPU, int gpuId, float score_thre) + { + ModelFile = modelFile; + IsGPU = isGPU; + GPUId = gpuId; + Score_thre = score_thre; + } + + public MLInit(string modelFile, string inputNodeName, string inferenceDevice, int inferenceWidth, int inferenceHeight) + { + ModelFile = modelFile; + InferenceDevice = inferenceDevice; + + InferenceWidth = inferenceWidth; + InferenceHeight = inferenceHeight; + InputNodeName = inputNodeName; + + + } + } + public class DetectStationResult + { + public string Pid { get; set; } + + public string TempPid { get; set; } + + /// + /// 检测工位名称 + /// + public string DetectName { get; set; } + + + /// + /// 深度学习 检测结果 + /// + public List DetectDetails = new List(); + + + /// + /// 工位检测结果 + /// + public ResultState ResultState { get; set; } = ResultState.ResultTBD; + + + public double FinalResultfScore { get; set; } = 0.0; + + + public string ResultLabel { get; set; } = "";// 多个ng时，根据label优先级，设定当前检测项的label + + public string ResultLabelCategoryId { get; set; } = "";// 多个ng时，根据label优先级，设定当前检测项的label + + public int PreTreatState { get; set; } + public bool IsPreTreatDone { get; set; } = true; + + public bool IsAfterTreatDone { get; set; } = true; + + public bool IsMLDetectDone { get; set; } = true; + + /// + /// 预处理阶段已经NG + /// + public bool IsPreTreatNG { get; set; } = false; + + /// + /// 目标检测NG + /// + public bool IsObjectDetectNG { get; set; } = false; + + public DateTime EndTime { get; set; } + + public int StationDetectElapsed { get; set; } + public static string NormalizeAndClean(string input) + { + if (input == null) return null; + + // Step 1: 标准化字符编码为 Form C (规范组合) + string normalizedString = input.Normalize(NormalizationForm.FormC); + + // Step 2: 移除所有空白字符，包括制表符和换行符 + string withoutWhitespace = Regex.Replace(normalizedString, @"\s+", ""); + + // Step 3: 移除控制字符 (Unicode 控制字符，范围 \u0000 - \u001F 和 \u007F) + string withoutControlChars = Regex.Replace(withoutWhitespace, @"[\u0000-\u001F\u007F]+", ""); + + // Step 4: 移除特殊的不可见字符（如零宽度空格等） + string cleanedString = Regex.Replace(withoutControlChars, @"[\u200B\u200C\u200D\uFEFF]+", ""); + + return cleanedString; + } + + } + public class RelatedCamera + { + + [Category("关联相机")] + [DisplayName("关联相机")] + [Description("关联相机描述")] + + //[TypeConverter(typeof(CollectionCountConvert))] + public string CameraSourceId { get; set; } = ""; + + public RelatedCamera() + { + + } + public RelatedCamera(string cameraSourceId) + { + CameraSourceId = cameraSourceId; + + } + } + public class DetectionConfig + { + [ReadOnly(true)] + public string Id { get; set; } = Guid.NewGuid().ToString(); + + + [Category("检测配置")] + [DisplayName("检测配置名称")] + [Description("检测配置名称")] + public string Name { get; set; } + + [Category("关联相机")] + [DisplayName("关联相机")] + [Description("关联相机描述")] + + + public string CameraSourceId { get; set; } = ""; + + + [Category("关联相机集合")] + [DisplayName("关联相机集合")] + [Description("关联相机描述")] + //[TypeConverter(typeof(DeviceIdSelectorConverter))] + + public List CameraCollects { get; set; } = new List(); + + + [Category("启用配置")] + [DisplayName("是否启用GPU检测")] + [Description("是否启用GPU检测")] + public bool IsEnableGPU { get; set; } = false; + + [Category("启用配置")] + [DisplayName("是否混料模型")] + [Description("是否混料模型")] + public bool IsMixModel { get; set; } = false; + + + + [Category("启用配置")] + [DisplayName("是否启用该检测")] + [Description("是否启用该检测")] + public bool IsEnabled { get; set; } + + [Category("启用配置")] + [DisplayName("是否加入检测工位")] + [Description("是否加入检测工位")] + public bool IsAddStation { get; set; } = true; + + [Category("2.中检测（深度学习）")] + [DisplayName("中检测-模型类型")] + [Description("模型类型：ImageClassification-图片分类；ObjectDetection：目标检测；Segmentation-图像分割")] + //[TypeConverter(typeof(EnumDescriptionConverter))] + public MLModelType ModelType { get; set; } = MLModelType.ObjectDetection; + + //[Category("2.中检测（深度学习）")] + //[DisplayName("中检测-GPU索引")] + //[Description("GPU索引")] + //public int GPUIndex { get; set; } = 0; + + [Category("2.中检测（深度学习）")] + [DisplayName("中检测-模型文件路径")] + [Description("中处理 深度学习模型文件路径，路径中不可含有中文字符，一般情况可以只配置中检测模型，当需要先用预检测过滤一次时，请先配置好与预检测相关配置")] + + public string ModelPath { get; set; } + + [Category("2.中检测（深度学习）")] + [DisplayName("中检测-模型宽度")] + [Description("中处理-模型宽度")] + + public int ModelWidth { get; set; } = 640; + + [Category("2.中检测（深度学习）")] + [DisplayName("中检测-模型高度")] + [Description("中处理-模型高度")] + + public int ModelHeight { get; set; } = 640; + + [Category("2.中检测（深度学习）")] + [DisplayName("中检测-模型节点名称")] + [Description("中处理-模型节点名称")] + + public string ModeloutNodeName { get; set; } = "output0"; + + [Category("2.中检测（深度学习）")] + [DisplayName("中检测-模型置信度")] + [Description("中处理-模型置信度")] + + public float ModelconfThreshold { get; set; } = 0.5f; + + [Category("2.中检测（深度学习）")] + [DisplayName("中检测-模型标签路径")] + [Description("中处理-模型标签路径")] + + public string in_lable_path { get; set; } + + [Category("4.最终过滤（逻辑过滤）")] + [DisplayName("过滤器集合")] + [Description("最后的逻辑过滤：可根据 识别出对象的 宽度、高度、面积、得分来设置最终检测结果，同一识别目标同一判定，多项过滤器之间为“或”关系")] + + public List DetectionFilterList { get; set; } = new List(); + + //[Category("深度学习配置")] + //[DisplayName("检测配置标签")] + //[Description("检测配置标签关联")] + + //public List DetectConfigLabelList { get; set; } = new List(); + + + public DetectionConfig() + { + + } + + public DetectionConfig(string name, MLModelType modelType, string modelPath, bool isEnableGPU,string sCameraSourceId) + { + ModelPath = modelPath ?? string.Empty; + Name = name; + ModelType = modelType; + IsEnableGPU = isEnableGPU; + Id = Guid.NewGuid().ToString(); + CameraSourceId = sCameraSourceId; + + } + } + /// + /// 识别目标定义 class：分类信息 Detection Segmentation：要识别的对象 + /// + public class RecongnitionLabel //: IComplexDisplay + { + [Category("检测标签定义")] + [Description("检测标签编码")] + [ReadOnly(true)] + public string Id { get; set; } = Guid.NewGuid().ToString(); + + [Category("检测标签定义")] + [DisplayName("检测标签名称")] + [Description("检测标签名称")] + public string LabelName { get; set; } = ""; + + [Category("检测标签定义")] + [DisplayName("检测标签描述")] + [Description("检测标签描述，中文描述")] + public string LabelDescription { get; set; } = ""; + + [Category("检测标签定义")] + [DisplayName("检测标签分类")] + [Description("检测标签分类id")] + //[TypeConverter(typeof(LabelCategoryConverter))] + public string LabelCategory { get; set; } = ""; + + + + + } + + /// + /// 检测项识别对象 + /// + public class DetectConfigLabel //: IComplexDisplay + { + [Category("检测项标签")] + [DisplayName("检测项标签")] + [Description("检测标签Id")] + //[TypeConverter(typeof(DetectionLabelConverter))] + public string LabelId { get; set; } + + [Browsable(false)] + //public string LabelName { get => GetLabelName(); } + + [Category("检测项标签")] + [DisplayName("检测标签优先级")] + [Description("检测标签优先级，值越小，优先级越高")] + public int LabelPriority { get; set; } = 0; + + //[Category("检测项标签")] + //[DisplayName("标签BGR值")] + //[Description("检测标签BGR值，例如：0,128,0")] + //public string LabelBGR { get; set; } + + //[Category("模型配置")] + //[DisplayName("模型参数配置")] + //[Description("模型参数配置集合")] + //[TypeConverter(typeof(ComplexObjectConvert))] + //[Editor(typeof(PropertyObjectEditor), typeof(UITypeEditor))] + //public ModelParamSetting ModelParamSetting { get; set; } = new ModelParamSetting(); + + //public string GetDisplayText() + //{ + // string dName = ""; + // if (!string.IsNullOrWhiteSpace(LabelId)) + // { + // using (var scope = GlobalVar.Container.BeginLifetimeScope()) + // { + // IProcessConfig config = scope.Resolve(); + + // var mlBase = config.DeviceConfigs.FirstOrDefault(c => c is VisionEngineInitialConfigBase) as VisionEngineInitialConfigBase; + // if (mlBase != null) + // { + // var targetLabel = mlBase.RecongnitionLabelList.FirstOrDefault(u => u.Id == LabelId); + // if (targetLabel != null) + // { + // dName = targetLabel.GetDisplayText(); + // } + // } + // } + // } + // return dName; + //} + //public string GetLabelName() + //{ + // var name = ""; + + + // var mlBase = iConfig.DeviceConfigs.FirstOrDefault(c => c is VisionEngineInitialConfigBase) as VisionEngineInitialConfigBase; + // if (mlBase != null) + // { + // var label = mlBase.RecongnitionLabelList.FirstOrDefault(u => u.Id == LabelId); + // if (label != null) + // { + // name = label.LabelName; + // } + // } + + + // return name; + //} + } + + /// + /// 识别对象定义分类信息 A类B类 + /// + public class RecongnitionLabelCategory //: IComplexDisplay + { + [Category("检测标签分类")] + [Description("检测标签分类")] + [ReadOnly(true)] + public string Id { get; set; } = Guid.NewGuid().ToString(); + + [Category("检测标签分类")] + [DisplayName("检测标签分类名称")] + [Description("检测标签分类名称")] + public string CategoryName { get; set; } = "A-NG"; + + [Category("检测标签分类")] + [DisplayName("检测标签分类优先级")] + [Description("检测标签分类优先级，值越小，优先级越高")] + public int CategoryPriority { get; set; } = 0; + + public string GetDisplayText() + { + return CategoryPriority + ":" + CategoryName; + } + } + + /// + /// 检测过滤 + /// + public class DetectionFilter ///: IComplexDisplay + { + [Category("过滤器基础信息")] + [DisplayName("检测标签")] + [Description("检测标签信息")] + //[TypeConverter(typeof(DetectionLabelConverter))] + public string LabelId { get; set; } + + // [Browsable(false)] + public string LabelName { get; set; } + + [Category("过滤器基础信息")] + [DisplayName("是否启用过滤器")] + [Description("是否启用过滤器")] + public bool IsEnabled { get; set; } + + [Category("过滤器判定信息")] + [DisplayName("判定结果")] + [Description("过滤器默认判定结果")] + public ResultState ResultState { get; set; } = ResultState.ResultTBD; + + [Category("过滤条件")] + [DisplayName("过滤条件集合")] + [Description("过滤条件集合，集合之间为“且”关系")] + //[TypeConverter(typeof(CollectionCountConvert))] + // [Editor(typeof(ComplexCollectionEditor), typeof(UITypeEditor))] + public List FilterConditionsCollection { get; set; } = new List(); + + + + public bool FilterOperation(DetectionResultDetail recongnitionResult) + { + return FilterConditionsCollection.All(u => + { + return u.FilterConditionCollection.Any(c => + { + double compareValue = 0; + + switch (c.FilterPropperty) + { + case DetectionFilterProperty.Width: + compareValue = recongnitionResult.Rect.Width; + break; + case DetectionFilterProperty.Height: + compareValue = recongnitionResult.Rect.Height; + break; + case DetectionFilterProperty.Area: + compareValue = recongnitionResult.Area; + break; + case DetectionFilterProperty.Score: + compareValue = recongnitionResult.Score; + break; + //case RecongnitionTargetFilterProperty.Uncertainty: + // compareValue = 0; + // //defect.Uncertainty; + // break; + } + + return compareValue >= c.MinValue && compareValue <= c.MaxValue; + }); + }); + } + } + + public class FilterConditions //: IComplexDisplay + { + [Category("过滤条件")] + [DisplayName("过滤条件集合")] + [Description("过滤条件集合，集合之间为“或”关系")] + //[TypeConverter(typeof(CollectionCountConvert))] + //[Editor(typeof(ComplexCollectionEditor), typeof(UITypeEditor))] + public List FilterConditionCollection { get; set; } = new List(); + + //public string GetDisplayText() + //{ + // if (FilterConditionCollection.Count == 0) + // { + // return "空"; + // } + // else + // { + // var desc = string.Join(" OR ", FilterConditionCollection.Select(u => u.GetDisplayText())); + + // if (FilterConditionCollection.Count > 1) + // { + // desc = $"({desc})"; + // } + + // return desc; + // } + //} + } + + public class FilterCondition //: IComplexDisplay + { + [Category("识别目标属性")] + [DisplayName("过滤属性")] + [Description("识别目标过滤针对的属性")] + //[TypeConverter(typeof(EnumDescriptionConverter))] + public DetectionFilterProperty FilterPropperty { get; set; } = DetectionFilterProperty.Width; + + [Category("过滤值")] + [DisplayName("最小值")] + [Description("最小值")] + public double MinValue { get; set; } = 1; + + [Category("过滤值")] + [DisplayName("最大值")] + [Description("最大值")] + public double MaxValue { get; set; } = 99999999; + + //public string GetDisplayText() + //{ + // return $"{FilterPropperty.GetEnumDescription()}:{MinValue}-{MaxValue}"; + //} + } + + public enum DetectionFilterProperty + { + [Description("宽度")] + Width = 1, + [Description("高度")] + Height = 2, + [Description("面积")] + Area = 3, + [Description("得分")] + Score = 4, + //[Description("不确定性")] + //Uncertainty = 5, + } +} diff --git a/DH.Devices.Vision/SimboDetection.cs b/DH.Devices.Vision/SimboDetection.cs new file mode 100644 index 0000000..d579cf8 --- /dev/null +++ b/DH.Devices.Vision/SimboDetection.cs @@ -0,0 +1,244 @@ +#define USE_MULTI_THREAD + +using OpenCvSharp; +using OpenCvSharp.Extensions; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Drawing; +using System.Linq; +using System.Runtime.ExceptionServices; +using System.Threading; +using System.Threading.Tasks; +using System.Security.Cryptography.Xml; +using System.Runtime.InteropServices; +using Newtonsoft.Json; + + + +namespace DH.Devices.Vision +{ + + + /// + /// 目标检测 GPU + /// + public class SimboDetection : SimboVisionMLBase + { + + public override bool Load(MLInit mLInit) + { + bool res = false; + try + { + Model = MLGPUEngine.InitModel(mLInit.ModelFile, 1, mLInit.Score_thre, mLInit.GPUId, 3, 8); + + //Model = MLEngine.InitModel(mLInit.ModelFile, 1, 0.45f, 0, 3); + + res = true; + +#if USE_MULTI_THREAD + IsCreated = true; + if (IsCreated) + { + _runHandleBefore ??= new AutoResetEvent(false); + _runHandleAfter ??= new ManualResetEvent(false); + + _runTask ??= Task.Factory.StartNew(() => + { + while (IsCreated) + { + _runHandleBefore.WaitOne(); + + if (IsCreated) + { + _result = RunInferenceFixed(_req); + _runHandleAfter.Set(); + } + } + }, TaskCreationOptions.LongRunning); + } +#endif + } + catch (Exception ex) + { + throw ex; + } + return res; + } +#if USE_MULTI_THREAD + MLRequest _req = null; + MLResult _result = null; + public bool IsCreated { get; set; } = false; + Task _runTask = null; + AutoResetEvent _runHandleBefore = new AutoResetEvent(false); + ManualResetEvent _runHandleAfter = new ManualResetEvent(false); + object _runLock = new object(); +#endif + [HandleProcessCorruptedStateExceptions] + public override MLResult RunInference(MLRequest req) + { +#if USE_MULTI_THREAD + MLResult mlResult = null; + lock (_runLock) + { + _result = new MLResult(); + + _req = req; + + _runHandleAfter.Reset(); + _runHandleBefore.Set(); + _runHandleAfter.WaitOne(); + + mlResult = _result; + } + + return mlResult; +#else + return RunInferenceFixed(req); +#endif + } + private void ConvertJsonResult(string json, ref MLResult result) + { + // json = "{\"FastDetResult\":[{\"cls_id\":0,\"cls\":\"liewen\",\"fScore\":0.654843,\"rect\":[175,99,110,594]},{\"cls_id\":0,\"cls\":\"liewen\",\"fScore\":0.654589,\"rect\":[2608,19,104,661]},{\"cls_id\":0,\"cls\":\"liewen\",\"fScore\":0.654285,\"rect\":[1275,19,104,662]},{\"cls_id\":0,\"cls\":\"liewen\",\"fScore\":0.620762,\"rect\":[1510,95,107,600]},{\"cls_id\":0,\"cls\":\"liewen\",\"fScore\":0.617812,\"rect\":[2844,93,106,602]}]}"; + // + Console.WriteLine("检测结果JSON：" + json); + HYoloResult detResult = JsonConvert.DeserializeObject(json); + if (detResult == null) + { + return; + } + + int iNum = detResult.HYolo.Count; + int IokNum = 0; + for (int ix = 0; ix < iNum; ix++) + { + var det = detResult.HYolo[ix]; + + var rect = det.rect; + DetectionResultDetail detectionResultDetail = new DetectionResultDetail(); + // detectionResultDetail.LabelNo = det.classId; + //todo: 标签名相对应 + detectionResultDetail.LabelDisplay = det.classname; + detectionResultDetail.Rect = new Rectangle(rect[0], rect[1], rect[2], rect[3]); + detectionResultDetail.Score = det.fScore; + detectionResultDetail.LabelName = det.classname; + detectionResultDetail.Area = rect[2] * rect[3]; + detectionResultDetail.InferenceResult = ResultState.DetectNG; + + result.ResultDetails.Add(detectionResultDetail); + + + } + + } + + [HandleProcessCorruptedStateExceptions] + public MLResult RunInferenceFixed(MLRequest req) + { + MLResult mlResult = new MLResult(); + Mat originMat = new Mat(); + Mat detectMat = new Mat(); + + try + { + if (req.mImage == null) + { + mlResult.IsSuccess = false; + mlResult.ResultMessage = "异常：mat为null，无法执行推理！"; + return mlResult; + } + + // resize + detectMat = req.mImage;//1ms + + + + int iWidth = detectMat.Cols; + int iHeight = detectMat.Rows; + + // 如果是单通道图像，转换为三通道 RGB 格式 + if (detectMat.Channels() == 1) + { + // 将灰度图像转换为RGB格式（三通道） + + Cv2.CvtColor(detectMat, originMat, ColorConversionCodes.GRAY2BGR); + + } + else if (detectMat.Channels() == 3) + { + // 如果已经是三通道（BGR），则直接转换为RGB + + Cv2.CvtColor(detectMat, originMat, ColorConversionCodes.BGR2RGB); + + } + + //输入数据转化为字节 + var inputByte = new byte[originMat.Total() * 3];//这里必须乘以通道数，不然数组越界，也可以用w*h*c，差不多 + Marshal.Copy(originMat.Data, inputByte, 0, inputByte.Length); + + byte[] labellist = new byte[40960]; //新建字节数组：label1_str label2_str + + byte[] outputByte = new byte[originMat.Total() * 3]; + + Stopwatch sw = new Stopwatch(); + sw.Start(); + + //mlResult.IsSuccess = true; + unsafe + { + //mlResult.IsSuccess = MLGPUEngine.Inference(Model, inputByte, iWidth, iHeight, 3, req.in_lable_path, ref outputByte[0], ref labellist[0]); + + mlResult.IsSuccess = MLGPUEngine.Inference2(Model, inputByte, iWidth, iHeight, 3, req.in_lable_path, ref labellist[0]); + } + + sw.Stop(); + + + if (mlResult.IsSuccess) + { + mlResult.ResultMessage = $"深度学习推理成功，耗时：{sw.ElapsedMilliseconds} ms"; + + //将字节数组转换为字符串 + mlResult.ResultMap = originMat.ToBitmap();//4ms + string strGet = System.Text.Encoding.Default.GetString(labellist, 0, labellist.Length); + + if (strGet == null) + { + mlResult.ResultMessage = $"异常：深度学习执行推理失败！"; + return mlResult; + } + + ConvertJsonResult(strGet, ref mlResult); + + return mlResult; + } + else + { + mlResult.ResultMessage = $"异常：深度学习执行推理失败！"; + return mlResult; + } + } + catch (Exception ex) + { + mlResult.ResultMessage = $"深度学习执行推理异常"; + return mlResult; + } + finally + { + + originMat?.Dispose(); + originMat = null; + //maskMat?.Dispose(); + // maskMat = null; + detectMat?.Dispose(); + detectMat = null; + // maskWeighted?.Dispose(); + // maskWeighted = null; + // GC.Collect(); + } + } + + + } +} diff --git a/DH.Devices.Vision/SimboInstanceSegmentation.cs b/DH.Devices.Vision/SimboInstanceSegmentation.cs new file mode 100644 index 0000000..f0a751d --- /dev/null +++ b/DH.Devices.Vision/SimboInstanceSegmentation.cs @@ -0,0 +1,264 @@ +//#define USE_MULTI_THREAD + +using OpenCvSharp; +using OpenCvSharp.Extensions; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Drawing; +using System.Linq; +using System.Runtime.ExceptionServices; +using System.Threading; +using System.Threading.Tasks; +using System.Runtime.InteropServices; +using Newtonsoft.Json; + + +namespace DH.Devices.Vision +{ + + /// + /// 实例分割 maskrcnn + /// + public class SimboInstanceSegmentation : SimboVisionMLBase + { + public override bool Load(MLInit mLInit) + { + bool res = false; + try + { + + + Model = MLEngine.InitModel(mLInit.ModelFile, + mLInit.InferenceDevice, + mLInit.InputNodeName, + 1, 3, + mLInit.InferenceWidth, + mLInit.InferenceHeight,5); + res = true; + +#if USE_MULTI_THREAD + + IsCreated = true; + if (IsCreated) + { + if (_runHandleBefore == null) + { + _runHandleBefore = new AutoResetEvent(false); + } + + if (_runHandleAfter == null) + { + _runHandleAfter = new ManualResetEvent(false); + } + + if (_runTask == null) + { + _runTask = Task.Factory.StartNew(() => + { + while (IsCreated) + { + _runHandleBefore.WaitOne(); + + if (IsCreated) + { + _result = RunInferenceFixed(_req); + _runHandleAfter.Set(); + } + } + }, TaskCreationOptions.LongRunning); + } + } +#endif + } + catch (Exception ex) + { + throw ex; + } + return res; + } + + + +#if USE_MULTI_THREAD + MLRequest _req = null; + MLResult _result = null; + + + public bool IsCreated { get; set; } = false; + Task _runTask = null; + AutoResetEvent _runHandleBefore = new AutoResetEvent(false); + ManualResetEvent _runHandleAfter = new ManualResetEvent(false); + object _runLock = new object(); +#endif + + [HandleProcessCorruptedStateExceptions] + public override MLResult RunInference(MLRequest req) + { +#if USE_MULTI_THREAD + MLResult mlResult = null; + lock (_runLock) + { + _result = new MLResult(); + + _req = req; + + _runHandleAfter.Reset(); + _runHandleBefore.Set(); + _runHandleAfter.WaitOne(); + + mlResult = _result; + } + + return mlResult; +#else + return RunInferenceFixed(req); +#endif + + + + + } + + + private void ConvertJsonResult(string json, ref MLResult result) + { + // json = "{\"FastDetResult\":[{\"cls_id\":0,\"cls\":\"liewen\",\"fScore\":0.654843,\"rect\":[175,99,110,594]},{\"cls_id\":0,\"cls\":\"liewen\",\"fScore\":0.654589,\"rect\":[2608,19,104,661]},{\"cls_id\":0,\"cls\":\"liewen\",\"fScore\":0.654285,\"rect\":[1275,19,104,662]},{\"cls_id\":0,\"cls\":\"liewen\",\"fScore\":0.620762,\"rect\":[1510,95,107,600]},{\"cls_id\":0,\"cls\":\"liewen\",\"fScore\":0.617812,\"rect\":[2844,93,106,602]}]}"; + // + Console.WriteLine("检测结果JSON：" + json); + SegResult detResult = JsonConvert.DeserializeObject(json); + if (detResult == null) + { + return; + } + + int iNum = detResult.SegmentResult.Count; + int IokNum = 0; + for (int ix = 0; ix < iNum; ix++) + { + var det = detResult.SegmentResult[ix]; + + var rect = det.rect; + DetectionResultDetail detectionResultDetail = new DetectionResultDetail(); + detectionResultDetail.LabelNo = det.classId; + //todo: 标签名相对应 + detectionResultDetail.LabelDisplay = det.classname; + detectionResultDetail.Rect = new Rectangle(rect[0], rect[1], rect[2], rect[3]); + detectionResultDetail.Score = det.fScore; + detectionResultDetail.LabelName = det.classname; + detectionResultDetail.Area = det.area; + detectionResultDetail.InferenceResult = ResultState.DetectNG; + + result.ResultDetails.Add(detectionResultDetail); + + + } + + } + + + + + [HandleProcessCorruptedStateExceptions] + public MLResult RunInferenceFixed(MLRequest req) + { + MLResult mlResult = new MLResult(); + Mat originMat = new Mat(); + Mat detectMat = new Mat(); + + try + { + if (req.mImage == null) + { + mlResult.IsSuccess = false; + mlResult.ResultMessage = "异常：mat为null，无法执行推理！"; + return mlResult; + } + + // resize + detectMat = req.mImage;//1ms + + + + int iWidth = detectMat.Cols; + int iHeight = detectMat.Rows; + + // 如果是单通道图像，转换为三通道 RGB 格式 + if (detectMat.Channels() == 1) + { + // 将灰度图像转换为RGB格式（三通道） + + Cv2.CvtColor(detectMat, originMat, ColorConversionCodes.GRAY2BGR); + + } + else if (detectMat.Channels() == 3) + { + // 如果已经是三通道（BGR），则直接转换为RGB + + Cv2.CvtColor(detectMat, originMat, ColorConversionCodes.BGR2RGB); + + } + + //输入数据转化为字节 + var inputByte = new byte[originMat.Total() * 3];//这里必须乘以通道数，不然数组越界，也可以用w*h*c，差不多 + Marshal.Copy(originMat.Data, inputByte, 0, inputByte.Length); + + byte[] labellist = new byte[40960]; //新建字节数组：label1_str label2_str + + byte[] outputByte = new byte[originMat.Total() * 3]; + + Stopwatch sw = new Stopwatch(); + sw.Start(); + unsafe + { + + mlResult.IsSuccess = MLEngine.seg_ModelPredict(Model, inputByte, iWidth, iHeight, 3, + req.in_lable_path, req.confThreshold, req.iouThreshold, req.confThreshold, req.segmentWidth, ref outputByte[0], ref labellist[0]); + //mlResult.IsSuccess = true; + } + sw.Stop(); + + if (mlResult.IsSuccess) + { + mlResult.ResultMessage = $"深度学习推理成功，耗时：{sw.ElapsedMilliseconds} ms"; + + //将字节数组转换为字符串 + mlResult.ResultMap = originMat.ToBitmap();//4ms + string strGet = System.Text.Encoding.Default.GetString(labellist, 0, labellist.Length); + + + Console.WriteLine("strGet:", strGet); + + + ConvertJsonResult(strGet, ref mlResult); + + + + //解析json字符串 + + return mlResult; + } + else + { + mlResult.ResultMessage = $"异常：深度学习执行推理失败！"; + return mlResult; + } + } + catch (Exception ex) + { + mlResult.ResultMessage = $"深度学习执行推理异常"; + return mlResult; + } + finally + { + + originMat?.Dispose(); + originMat = null; + + + // GC.Collect(); + } + } + + } +} diff --git a/DH.Devices.Vision/SimboObjectDetection.cs b/DH.Devices.Vision/SimboObjectDetection.cs index 21cf262..9ef5e57 100644 --- a/DH.Devices.Vision/SimboObjectDetection.cs +++ b/DH.Devices.Vision/SimboObjectDetection.cs @@ -17,23 +17,7 @@ using Newtonsoft.Json; namespace DH.Devices.Vision { - //public class SegResult - //{ - // public List SegmentResult; - // public class Result - // { - // public double fScore; - // public int classId; - // public string classname; - - // public double area; - // public List rect; - - - // } - - //} @@ -157,7 +141,6 @@ namespace DH.Devices.Vision } int iNum = detResult.SegmentResult.Count; - int IokNum = 0; for (int ix = 0; ix < iNum; ix++) { var det = detResult.SegmentResult[ix]; @@ -188,7 +171,7 @@ namespace DH.Devices.Vision { MLResult mlResult = new MLResult(); Mat originMat=new Mat() ; - Mat tempMat; + Mat detectMat; try { if (req.mImage == null) @@ -199,26 +182,26 @@ namespace DH.Devices.Vision } // resize - tempMat = req.mImage;//1ms + detectMat = req.mImage;//1ms - int iWidth = tempMat.Cols; - int iHeight = tempMat.Rows; + int iWidth = detectMat.Cols; + int iHeight = detectMat.Rows; // 如果是单通道图像，转换为三通道 RGB 格式 - if (tempMat.Channels() == 1) + if (detectMat.Channels() == 1) { // 将灰度图像转换为RGB格式（三通道） - Cv2.CvtColor( tempMat,originMat, ColorConversionCodes.GRAY2BGR); + Cv2.CvtColor( detectMat,originMat, ColorConversionCodes.GRAY2BGR); } - else if (tempMat.Channels() == 3) + else if (detectMat.Channels() == 3) { // 如果已经是三通道（BGR），则直接转换为RGB - Cv2.CvtColor( tempMat,originMat, ColorConversionCodes.BGR2RGB); + Cv2.CvtColor( detectMat,originMat, ColorConversionCodes.BGR2RGB); } @@ -250,9 +233,6 @@ namespace DH.Devices.Vision { mlResult.ResultMessage = $"深度学习推理成功，耗时：{sw.ElapsedMilliseconds} ms"; - //Mat maskWeighted = new Mat(iHeight, iWidth, MatType.CV_8UC3, outputByte); - - //mlResult.ResultMap = BitmapConverter.ToBitmap(maskWeighted);//4ms //将字节数组转换为字符串 mlResult.ResultMap = originMat.ToBitmap();//4ms string strGet = System.Text.Encoding.Default.GetString(labellist, 0, labellist.Length); @@ -261,9 +241,6 @@ namespace DH.Devices.Vision ConvertJsonResult(strGet, ref mlResult); - //maskWeighted?.Dispose(); - //maskWeighted = null; - // 解析json字符串 return mlResult; } diff --git a/DH.Devices.Vision/SimboVisionDriver.cs b/DH.Devices.Vision/SimboVisionDriver.cs new file mode 100644 index 0000000..378cecf --- /dev/null +++ b/DH.Devices.Vision/SimboVisionDriver.cs @@ -0,0 +1,18 @@ +using OpenCvSharp; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Runtime.ExceptionServices; +using System.Text; +using System.Threading.Tasks; +using System.Xml.Linq; + +namespace DH.Devices.Vision +{ + public class SimboVisionDriver + { + + + } +} diff --git a/DH.Devices.Vision/SimboVisionMLBase.cs b/DH.Devices.Vision/SimboVisionMLBase.cs index 1f76fe1..0164094 100644 --- a/DH.Devices.Vision/SimboVisionMLBase.cs +++ b/DH.Devices.Vision/SimboVisionMLBase.cs @@ -44,6 +44,31 @@ namespace DH.Devices.Vision // ColorLut = new Mat(1, 256, MatType.CV_8UC3, ColorMap); } } + public class HYoloResult + { + //{ + // "HYolo": [{ + // "fScore": "0.687012", + // "classId": 0, + // "classname": "quejiao", + // "rect": [421, 823, 6, 8] + // }] + //} + public List HYolo; + public class Result + { + + public double fScore; + public int classId; + public string classname; + + //public double area; + public List rect; + + + } + + } public class SegResult { public List SegmentResult; diff --git a/DH.Devices.Vision/VisionEngine.cs b/DH.Devices.Vision/VisionEngine.cs deleted file mode 100644 index 8556658..0000000 --- a/DH.Devices.Vision/VisionEngine.cs +++ /dev/null @@ -1,299 +0,0 @@ -using OpenCvSharp; -using System.ComponentModel; -using System.Drawing; -using static OpenCvSharp.AgastFeatureDetector; -using System.Text.RegularExpressions; -using System.Text; - -namespace DH.Devices.Vision -{ - public enum MLModelType - { - [Description("图像分类")] - ImageClassification = 1, - [Description("目标检测")] - ObjectDetection = 2, - //[Description("图像分割")] - //ImageSegmentation = 3 - [Description("语义分割")] - SemanticSegmentation = 3, - [Description("实例分割")] - InstanceSegmentation = 4, - [Description("目标检测GPU")] - ObjectGPUDetection = 5 - } - public class MLRequest - { - public int ImageChannels = 3; - public Mat mImage; - public int ResizeWidth; - public int ResizeHeight; - - public float confThreshold; - - public float iouThreshold; - - //public int ImageResizeCount; - public bool IsCLDetection; - public int ProCount; - public string in_node_name; - - public string out_node_name; - - public string in_lable_path; - - public int ResizeImageSize; - public int segmentWidth; - public int ImageWidth; - - // public List OkClassTxtList; - - - // public List LabelNames; - - public float Score; - - } - public enum ResultState - { - - [Description("检测NG")] - DetectNG = -3, - - //[Description("检测不足TBD")] - // ShortageTBD = -2, - [Description("检测结果TBD")] - ResultTBD = -1, - [Description("OK")] - OK = 1, - // [Description("NG")] - // NG = 2, - //统计结果 - [Description("A类NG")] - A_NG = 25, - [Description("B类NG")] - B_NG = 26, - [Description("C类NG")] - C_NG = 27, - } - /// - /// 深度学习 识别结果明细 面向业务：detect 面向深度学习：Recongnition、Inference - /// - public class DetectionResultDetail - { - public string LabelBGR { get; set; }//识别到对象的标签BGR - - - public int LabelNo { get; set; } // 识别到对象的标签索引 - - public string LabelName { get; set; }//识别到对象的标签名称 - - public double Score { get; set; }//识别目标结果的可能性、得分 - - public string LabelDisplay { get; set; }//识别到对象的 显示信息 - - public double Area { get; set; }//识别目标的区域面积 - - public Rectangle Rect { get; set; }//识别目标的外接矩形 - - public RotatedRect MinRect { get; set; }//识别目标的最小外接矩形(带角度) - - public ResultState InferenceResult { get; set; }//只是模型推理 label的结果 - - public double DistanceToImageCenter { get; set; } //计算矩形框到图像中心的距离 - - - - public ResultState FinalResult { get; set; }//模型推理+其他视觉、逻辑判断后 label结果 - } - public class MLResult - { - public bool IsSuccess = false; - public string ResultMessage; - public Bitmap ResultMap; - public List ResultDetails = new List(); - } - public class MLInit - { - public string ModelFile; - public string InferenceDevice; - - - public int InferenceWidth; - public int InferenceHeight; - - public string InputNodeName; - - - public int SizeModel; - - public bool bReverse;//尺寸测量正反面 - //目标检测Gpu - public bool IsGPU; - public int GPUId; - public float Score_thre; - public MLInit(string modelFile, bool isGPU, int gpuId, float score_thre) - { - ModelFile = modelFile; - IsGPU = isGPU; - GPUId = gpuId; - Score_thre = score_thre; - } - - public MLInit(string modelFile, string inputNodeName, string inferenceDevice, int inferenceWidth, int inferenceHeight) - { - ModelFile = modelFile; - InferenceDevice = inferenceDevice; - - InferenceWidth = inferenceWidth; - InferenceHeight = inferenceHeight; - InputNodeName = inputNodeName; - - - } - } - public class DetectStationResult - { - public string Pid { get; set; } - - public string TempPid { get; set; } - - /// - /// 检测工位名称 - /// - public string DetectName { get; set; } - - - /// - /// 深度学习 检测结果 - /// - public List DetectDetails = new List(); - - - /// - /// 工位检测结果 - /// - public ResultState ResultState { get; set; } = ResultState.ResultTBD; - - - public double FinalResultfScore { get; set; } = 0.0; - - - public string ResultLabel { get; set; } = "";// 多个ng时，根据label优先级，设定当前检测项的label - - public string ResultLabelCategoryId { get; set; } = "";// 多个ng时，根据label优先级，设定当前检测项的label - - public int PreTreatState { get; set; } - public bool IsPreTreatDone { get; set; } = true; - - public bool IsAfterTreatDone { get; set; } = true; - - public bool IsMLDetectDone { get; set; } = true; - - /// - /// 预处理阶段已经NG - /// - public bool IsPreTreatNG { get; set; } = false; - - /// - /// 目标检测NG - /// - public bool IsObjectDetectNG { get; set; } = false; - - public DateTime EndTime { get; set; } - - public int StationDetectElapsed { get; set; } - public static string NormalizeAndClean(string input) - { - if (input == null) return null; - - // Step 1: 标准化字符编码为 Form C (规范组合) - string normalizedString = input.Normalize(NormalizationForm.FormC); - - // Step 2: 移除所有空白字符，包括制表符和换行符 - string withoutWhitespace = Regex.Replace(normalizedString, @"\s+", ""); - - // Step 3: 移除控制字符 (Unicode 控制字符，范围 \u0000 - \u001F 和 \u007F) - string withoutControlChars = Regex.Replace(withoutWhitespace, @"[\u0000-\u001F\u007F]+", ""); - - // Step 4: 移除特殊的不可见字符（如零宽度空格等） - string cleanedString = Regex.Replace(withoutControlChars, @"[\u200B\u200C\u200D\uFEFF]+", ""); - - return cleanedString; - } - - } - public class RelatedCamera - { - - [Category("关联相机")] - [DisplayName("关联相机")] - [Description("关联相机描述")] - - //[TypeConverter(typeof(CollectionCountConvert))] - public string CameraSourceId { get; set; } = ""; - - - - } - public class VisionEngine - { - [ReadOnly(true)] - public string Id { get; set; } = Guid.NewGuid().ToString(); - - - [Category("检测配置")] - [DisplayName("检测配置名称")] - [Description("检测配置名称")] - public string Name { get; set; } - - [Category("关联相机")] - [DisplayName("关联相机")] - [Description("关联相机描述")] - - - public string CameraSourceId { get; set; } = ""; - - - [Category("关联相机集合")] - [DisplayName("关联相机集合")] - [Description("关联相机描述")] - //[TypeConverter(typeof(DeviceIdSelectorConverter))] - - public List CameraCollects { get; set; } = new List(); - - - [Category("启用配置")] - [DisplayName("是否启用GPU检测")] - [Description("是否启用GPU检测")] - public bool IsEnableGPU { get; set; } = false; - - [Category("2.中检测（深度学习）")] - [DisplayName("中检测-模型类型")] - [Description("模型类型：ImageClassification-图片分类；ObjectDetection：目标检测；Segmentation-图像分割")] - //[TypeConverter(typeof(EnumDescriptionConverter))] - public MLModelType ModelType { get; set; } = MLModelType.ObjectDetection; - - //[Category("2.中检测（深度学习）")] - //[DisplayName("中检测-GPU索引")] - //[Description("GPU索引")] - //public int GPUIndex { get; set; } = 0; - - [Category("2.中检测（深度学习）")] - [DisplayName("中检测-模型文件路径")] - [Description("中处理 深度学习模型文件路径，路径中不可含有中文字符，一般情况可以只配置中检测模型，当需要先用预检测过滤一次时，请先配置好与预检测相关配置")] - - public string ModelPath { get; set; } - - public VisionEngine(string name, MLModelType modelType, string modelPath, bool isEnableGPU,string sCameraSourceId) - { - ModelPath = modelPath ?? string.Empty; - Name = name; - ModelType = modelType; - IsEnableGPU = isEnableGPU; - Id = Guid.NewGuid().ToString(); - CameraSourceId = sCameraSourceId; - - } - } -} diff --git a/DHSoftware.sln b/DHSoftware.sln index b48051f..500edbf 100644 --- a/DHSoftware.sln +++ b/DHSoftware.sln @@ -17,60 +17,60 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Commons", "Commons", "{0AB4 EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DH.Commons", "DH.Commons\DH.Commons.csproj", "{027373EC-C5CB-4161-8D43-AB6009371FDE}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DH.Devices.Vision", "DH.Devices.Vision\DH.Devices.Vision.csproj", "{97B55FCF-54A3-449E-8437-735E65C35291}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DH.Devices.Vision", "DH.Devices.Vision\DH.Devices.Vision.csproj", "{97B55FCF-54A3-449E-8437-735E65C35291}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DH.Devices.Camera", "DH.Devices.Camera\DH.Devices.Camera.csproj", "{1378A932-1C25-40EF-BA31-A3463B23F4E5}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DH.Devices.Camera", "DH.Devices.Camera\DH.Devices.Camera.csproj", "{1378A932-1C25-40EF-BA31-A3463B23F4E5}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DH.Devices.PLC", "DH.Devices.PLC\DH.Devices.PLC.csproj", "{458B2CF6-6F1B-4B8B-BB85-C6FD7F453A5D}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DH.Devices.PLC", "DH.Devices.PLC\DH.Devices.PLC.csproj", "{458B2CF6-6F1B-4B8B-BB85-C6FD7F453A5D}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU - Debug|X64 = Debug|X64 + Debug|x64 = Debug|x64 Release|Any CPU = Release|Any CPU - Release|X64 = Release|X64 + Release|x64 = Release|x64 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {17CC10DC-9132-4A03-AADA-2D1070418C9B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {17CC10DC-9132-4A03-AADA-2D1070418C9B}.Debug|Any CPU.Build.0 = Debug|Any CPU - {17CC10DC-9132-4A03-AADA-2D1070418C9B}.Debug|X64.ActiveCfg = Debug|X64 - {17CC10DC-9132-4A03-AADA-2D1070418C9B}.Debug|X64.Build.0 = Debug|X64 + {17CC10DC-9132-4A03-AADA-2D1070418C9B}.Debug|x64.ActiveCfg = Debug|x64 + {17CC10DC-9132-4A03-AADA-2D1070418C9B}.Debug|x64.Build.0 = Debug|x64 {17CC10DC-9132-4A03-AADA-2D1070418C9B}.Release|Any CPU.ActiveCfg = Release|Any CPU {17CC10DC-9132-4A03-AADA-2D1070418C9B}.Release|Any CPU.Build.0 = Release|Any CPU - {17CC10DC-9132-4A03-AADA-2D1070418C9B}.Release|X64.ActiveCfg = Release|X64 - {17CC10DC-9132-4A03-AADA-2D1070418C9B}.Release|X64.Build.0 = Release|X64 + {17CC10DC-9132-4A03-AADA-2D1070418C9B}.Release|x64.ActiveCfg = Release|x64 + {17CC10DC-9132-4A03-AADA-2D1070418C9B}.Release|x64.Build.0 = Release|x64 {027373EC-C5CB-4161-8D43-AB6009371FDE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {027373EC-C5CB-4161-8D43-AB6009371FDE}.Debug|Any CPU.Build.0 = Debug|Any CPU - {027373EC-C5CB-4161-8D43-AB6009371FDE}.Debug|X64.ActiveCfg = Debug|X64 - {027373EC-C5CB-4161-8D43-AB6009371FDE}.Debug|X64.Build.0 = Debug|X64 + {027373EC-C5CB-4161-8D43-AB6009371FDE}.Debug|x64.ActiveCfg = Debug|x64 + {027373EC-C5CB-4161-8D43-AB6009371FDE}.Debug|x64.Build.0 = Debug|x64 {027373EC-C5CB-4161-8D43-AB6009371FDE}.Release|Any CPU.ActiveCfg = Release|Any CPU {027373EC-C5CB-4161-8D43-AB6009371FDE}.Release|Any CPU.Build.0 = Release|Any CPU - {027373EC-C5CB-4161-8D43-AB6009371FDE}.Release|X64.ActiveCfg = Release|X64 - {027373EC-C5CB-4161-8D43-AB6009371FDE}.Release|X64.Build.0 = Release|X64 + {027373EC-C5CB-4161-8D43-AB6009371FDE}.Release|x64.ActiveCfg = Release|x64 + {027373EC-C5CB-4161-8D43-AB6009371FDE}.Release|x64.Build.0 = Release|x64 {97B55FCF-54A3-449E-8437-735E65C35291}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {97B55FCF-54A3-449E-8437-735E65C35291}.Debug|Any CPU.Build.0 = Debug|Any CPU - {97B55FCF-54A3-449E-8437-735E65C35291}.Debug|X64.ActiveCfg = Debug|X64 - {97B55FCF-54A3-449E-8437-735E65C35291}.Debug|X64.Build.0 = Debug|X64 + {97B55FCF-54A3-449E-8437-735E65C35291}.Debug|x64.ActiveCfg = Debug|x64 + {97B55FCF-54A3-449E-8437-735E65C35291}.Debug|x64.Build.0 = Debug|x64 {97B55FCF-54A3-449E-8437-735E65C35291}.Release|Any CPU.ActiveCfg = Release|Any CPU {97B55FCF-54A3-449E-8437-735E65C35291}.Release|Any CPU.Build.0 = Release|Any CPU - {97B55FCF-54A3-449E-8437-735E65C35291}.Release|X64.ActiveCfg = Release|X64 - {97B55FCF-54A3-449E-8437-735E65C35291}.Release|X64.Build.0 = Release|X64 + {97B55FCF-54A3-449E-8437-735E65C35291}.Release|x64.ActiveCfg = Release|x64 + {97B55FCF-54A3-449E-8437-735E65C35291}.Release|x64.Build.0 = Release|x64 {1378A932-1C25-40EF-BA31-A3463B23F4E5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {1378A932-1C25-40EF-BA31-A3463B23F4E5}.Debug|Any CPU.Build.0 = Debug|Any CPU - {1378A932-1C25-40EF-BA31-A3463B23F4E5}.Debug|X64.ActiveCfg = Debug|x64 - {1378A932-1C25-40EF-BA31-A3463B23F4E5}.Debug|X64.Build.0 = Debug|x64 + {1378A932-1C25-40EF-BA31-A3463B23F4E5}.Debug|x64.ActiveCfg = Debug|x64 + {1378A932-1C25-40EF-BA31-A3463B23F4E5}.Debug|x64.Build.0 = Debug|x64 {1378A932-1C25-40EF-BA31-A3463B23F4E5}.Release|Any CPU.ActiveCfg = Release|Any CPU {1378A932-1C25-40EF-BA31-A3463B23F4E5}.Release|Any CPU.Build.0 = Release|Any CPU - {1378A932-1C25-40EF-BA31-A3463B23F4E5}.Release|X64.ActiveCfg = Release|x64 - {1378A932-1C25-40EF-BA31-A3463B23F4E5}.Release|X64.Build.0 = Release|x64 + {1378A932-1C25-40EF-BA31-A3463B23F4E5}.Release|x64.ActiveCfg = Release|x64 + {1378A932-1C25-40EF-BA31-A3463B23F4E5}.Release|x64.Build.0 = Release|x64 {458B2CF6-6F1B-4B8B-BB85-C6FD7F453A5D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {458B2CF6-6F1B-4B8B-BB85-C6FD7F453A5D}.Debug|Any CPU.Build.0 = Debug|Any CPU - {458B2CF6-6F1B-4B8B-BB85-C6FD7F453A5D}.Debug|X64.ActiveCfg = Debug|x64 - {458B2CF6-6F1B-4B8B-BB85-C6FD7F453A5D}.Debug|X64.Build.0 = Debug|x64 + {458B2CF6-6F1B-4B8B-BB85-C6FD7F453A5D}.Debug|x64.ActiveCfg = Debug|x64 + {458B2CF6-6F1B-4B8B-BB85-C6FD7F453A5D}.Debug|x64.Build.0 = Debug|x64 {458B2CF6-6F1B-4B8B-BB85-C6FD7F453A5D}.Release|Any CPU.ActiveCfg = Release|Any CPU {458B2CF6-6F1B-4B8B-BB85-C6FD7F453A5D}.Release|Any CPU.Build.0 = Release|Any CPU - {458B2CF6-6F1B-4B8B-BB85-C6FD7F453A5D}.Release|X64.ActiveCfg = Release|x64 - {458B2CF6-6F1B-4B8B-BB85-C6FD7F453A5D}.Release|X64.Build.0 = Release|x64 + {458B2CF6-6F1B-4B8B-BB85-C6FD7F453A5D}.Release|x64.ActiveCfg = Release|x64 + {458B2CF6-6F1B-4B8B-BB85-C6FD7F453A5D}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/DHSoftware/DHSoftware.csproj b/DHSoftware/DHSoftware.csproj index 2b98a95..dbb3345 100644 --- a/DHSoftware/DHSoftware.csproj +++ b/DHSoftware/DHSoftware.csproj @@ -25,7 +25,7 @@ - ..\X64\Debug\DVPCameraCS64.dll + ..\x64\Debug\DVPCameraCS64.dll diff --git a/DHSoftware/MainWindow.cs b/DHSoftware/MainWindow.cs index dd1a0db..1a26d3e 100644 --- a/DHSoftware/MainWindow.cs +++ b/DHSoftware/MainWindow.cs @@ -13,6 +13,7 @@ using System; using System.CodeDom; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics; using System.Drawing; using System.Linq; using System.Runtime.InteropServices; @@ -258,41 +259,111 @@ namespace DHSoftware public volatile int ProductNum_Total = 0; public volatile int ProductNum_OK = 0; private readonly object _cameraSummaryLock = new object(); + List detectionList = new List(); + public List RecongnitionLabelList { get; set; } = new List(); public DateTime sraerttime; private void HandleStartButton() { CurrentMachine = true; - List detectionList = new List(); - detectionList.Add(new VisionEngine("相机1", MLModelType.ObjectDetection, @"D:\DHSoftware\DHSoftware\Models\yolov3.cfg", false, "Cam1")); - detectionList.Add(new VisionEngine("相机2", MLModelType.ObjectDetection, @"D:\DHSoftware\DHSoftware\Models\yolov3.cfg", false, "Cam2")); - detectionList.Add(new VisionEngine("相机3", MLModelType.ObjectDetection, @"D:\DHSoftware\DHSoftware\Models\yolov3.cfg", false, "Cam3")); - detectionList.Add(new VisionEngine("相机4", MLModelType.ObjectDetection, @"D:\DHSoftware\DHSoftware\Models\yolov3.cfg", false, "Cam4")); + //[Category("深度学习检测配置")] + //[DisplayName("检测标签定义集合")] + //[Description("定义检测标签的集合，例如：Seg/Detection模式：断裂、油污、划伤...；Class模式：ok、ng、上面、下面、套环、正常...")] + //[TypeConverter(typeof(CollectionCountConvert))] + //[Editor(typeof(ComplexCollectionEditor), typeof(UITypeEditor))] + RecongnitionLabel recongnition=new RecongnitionLabel + { + LabelName="youwu", + LabelDescription="油污", + LabelCategory="A_NG" + }; + RecongnitionLabel recongnition2 = new RecongnitionLabel + { + LabelName = "youwu", + LabelDescription = "油污", + LabelCategory = "A_NG" + }; + RecongnitionLabel recongnition3 = new RecongnitionLabel + { + LabelName = "youwu", + LabelDescription = "油污", + LabelCategory = "A_NG" + }; + RecongnitionLabelList.Add(recongnition); + RecongnitionLabelList.Add(recongnition2); + RecongnitionLabelList.Add(recongnition3); + var det1 = new DetectionConfig("相机1", MLModelType.ObjectDetection, @"D:\DHSoftware\DHSoftware\Models\yolov3.cfg", false, "Cam1"); + var det2 = new DetectionConfig("相机2", MLModelType.ObjectDetection, @"D:\DHSoftware\DHSoftware\Models\yolov3.cfg", false, "Cam2"); + var det3 = new DetectionConfig("相机3", MLModelType.ObjectDetection, @"D:\DHSoftware\DHSoftware\Models\yolov3.cfg", false, "Cam3"); + var det4 = new DetectionConfig("相机4", MLModelType.ObjectDetection, @"D:\DHSoftware\DHSoftware\Models\yolov3.cfg", false, "Cam4"); + List CameraCollects=new List(); + CameraCollects.Add(new RelatedCamera("Cam1")); + List CameraCollects2 = new List(); + CameraCollects2.Add(new RelatedCamera("Cam2")); + List CameraCollects3 = new List(); + CameraCollects3.Add(new RelatedCamera("Cam3")); + List CameraCollects4 = new List(); + CameraCollects4.Add(new RelatedCamera("Cam4")); + List CameraCollects5 = new List(); + CameraCollects5.Add(new RelatedCamera("Cam5")); + float Conf = 0.5f; + det1.CameraCollects = CameraCollects; + det1.ModelconfThreshold = Conf; + det1.ModelWidth = 640; + det1.ModelHeight = 640; + det1.in_lable_path = " D:\\PROJECTS\\MaodingTest1\\Vision\\cam1.txt"; + + + det2.CameraCollects = CameraCollects2; + det1.ModelconfThreshold = Conf; + det1.ModelWidth = 640; + det1.ModelHeight = 640; + det1.in_lable_path = " D:\\PROJECTS\\MaodingTest1\\Vision\\cam2.txt"; + + det3.CameraCollects = CameraCollects3; + det1.ModelconfThreshold = Conf; + det1.ModelWidth = 640; + det1.ModelHeight = 640; + det1.in_lable_path = " D:\\PROJECTS\\MaodingTest1\\Vision\\cam3.txt"; + + det4.CameraCollects = CameraCollects4; + det1.ModelconfThreshold = Conf; + det1.ModelWidth = 640; + det1.ModelHeight = 640; + det1.in_lable_path = " D:\\PROJECTS\\MaodingTest1\\Vision\\cam4.txt"; + + detectionList.Add(det1); + detectionList.Add(det2); + detectionList.Add(det3); + detectionList.Add(det4); + Cameras.Clear(); + Dectection.Clear(); _cameraRelatedDetectionDict = new(); + detectionList.ForEach(detection => { - // detection.CameraCollects.ForEach(cam => + detection.CameraCollects.ForEach(cam => { List Dets = new List { detection.Id }; - if (!_cameraRelatedDetectionDict.ContainsKey(detection.CameraSourceId)) + if (!_cameraRelatedDetectionDict.ContainsKey(cam.CameraSourceId)) { - _cameraRelatedDetectionDict.Add(detection.CameraSourceId, Dets); + _cameraRelatedDetectionDict.Add(cam.CameraSourceId, Dets); } else { - _cameraRelatedDetectionDict[detection.CameraSourceId].Add(detection.Id); + _cameraRelatedDetectionDict[cam.CameraSourceId].Add(detection.Id); } } - //); + ); }); //Add the code for the "启动" button click here @@ -310,10 +381,7 @@ namespace DHSoftware do3ThinkCamera2.CameraConnect(); do3ThinkCamera1.OnHImageOutput += OnCameraHImageOutput; do3ThinkCamera2.OnHImageOutput += OnCameraHImageOutput; - var simbo1 = new SimboObjectDetection - { - - }; + var simbo1 = new SimboObjectDetection(); MLInit mLInit; string inferenceDevice = "CPU"; @@ -321,12 +389,11 @@ namespace DHSoftware simbo1.Load(mLInit); - Dectection.Add(do3ThinkCamera1.CameraName, simbo1); + - var simbo2 = new SimboObjectDetection - { + Dectection.Add(det1.Id, simbo1); - }; + var simbo2 = new SimboObjectDetection(); MLInit mLInit2; string inferenceDevice2 = "CPU"; @@ -334,7 +401,11 @@ namespace DHSoftware simbo2.Load(mLInit2); - Dectection.Add(do3ThinkCamera2.CameraName, simbo2); + for(int i = 0;i { - this.BeginInvoke(new MethodInvoker(delegate () { richTextBox1.AppendText("入料成功" + PieceCount); })); - - }); + int index = PieceNumberToIndex(pieceNumber); // productDatas.Add(pData); //转盘2 的物料是不是重新覆盖之前的pDta @@ -387,6 +455,11 @@ namespace DHSoftware ProductData pData = new ProductData("", pieceNumber, ProductBaseCount); _productLists[index][pieceNumber] = pData; } + string logStr = $"时间：{DateTime.Now} 轴{axisIndex}新产品{pieceNumber}加入队列{index}----入料计数{PieceCount}\n"; + Task.Run(() => { + this.BeginInvoke(new MethodInvoker(delegate () { richTextBox1.AppendText(logStr); })); + + }); DateTime dtNow = DateTime.Now; UpdateCT(null, (float)(dtNow - _ctTime).TotalSeconds); _ctTime = dtNow; @@ -472,28 +545,173 @@ namespace DHSoftware for (int i = 0; i < detectionDict.Count; i++) { - string d = detectionDict[i]; + string detectionId = detectionDict[i]; try { + DetectionConfig detectConfig = null; + //找到对应的配置 + if (!string.IsNullOrWhiteSpace(detectionId)) + { + detectConfig = detectionList.FirstOrDefault(u => u.Id == detectionId); + } + else + { + detectConfig = detectionList.FirstOrDefault(u => u.CameraSourceId == camera.CameraName); + } - // LogAsync(DateTime.Now, LogLevel.Information, $"{camera.Name} 推理进度1.3，产品{productNumber}"); + if (detectConfig == null) + { + + //未能获得检测配置 + return ; + } + + + #region 1.预处理 + #endregion + #region 2.深度学习推理 var req = new MLRequest(); req.mImage = imageSet.Clone(); - req.ResizeWidth = 640; - req.ResizeHeight = 640; - req.confThreshold = 0.5f; + req.ResizeWidth = detectConfig.ModelWidth; + req.ResizeHeight = detectConfig.ModelHeight; + req.confThreshold = detectConfig.ModelconfThreshold; req.iouThreshold = 0.3f; - req.out_node_name = "output0"; - req.in_lable_path = "D:\\PROJECTS\\MaodingTest1\\Vision\\cam1.txt"; + req.out_node_name = detectConfig.ModeloutNodeName; + req.in_lable_path = detectConfig.in_lable_path; //req.LabelNames = dc.GetLabelNames(); - req.Score = 0.5f; + //HOperatorSet.WriteImage(req.HImage, "png", 0, @"D:\\666.png"); - var result = Dectection[camera.CameraName].RunInference(req); + Stopwatch + sw = new Stopwatch(); + sw.Start(); + + + var result = Dectection[detectionId].RunInference(req); + sw.Stop(); + //LogAsync(DateTime.Now, LogLevel.Information, $"{camera.Name} 推理进度1.1，产品{productNumber}，耗时{sw.ElapsedMilliseconds}ms"); + #endregion + #region 3.后处理 + DetectStationResult detectResult = new DetectStationResult(); + if (result == null || (result != null && !result.IsSuccess)) + { + detectResult.IsMLDetectDone = false; + } + if (result != null && result.IsSuccess) + { + detectResult.DetectDetails = result.ResultDetails; + if (detectResult.DetectDetails != null) + { + } + else + { + detectResult.IsMLDetectDone = false; + } + } + #endregion + #region 3.后处理 + #endregion + //根据那些得分大于阈值的推理结果，判断产品是否成功 + #region 4.最终过滤（逻辑过滤） + detectResult.DetectDetails?.ForEach(d => + { - this.BeginInvoke(new MethodInvoker(delegate () { - pictureBox1.Image = result.ResultMap; richTextBox1.AppendText("推理成功" + productNumber+ result.IsSuccess+ "\n"); })); + //当前检测项的 过滤条件 + //var conditionList = detectConfig.DetectionFilterList + // .Where(u => u.IsEnabled && u.LabelName == d.LabelName) + // .GroupBy(u => u.ResultState) + // .OrderBy(u => u.Key) + // .ToList(); + //当前检测项的 过滤条件 + var conditionList = detectConfig.DetectionFilterList + .Where(u => u.IsEnabled && u.LabelName == d.LabelName) + .GroupBy(u => u.ResultState) + .OrderBy(u => u.Key) + .ToList(); + + if (conditionList.Count == 0) + { + + if (d.LabelName.ToLower() == "ok") + { + d.FinalResult = d.InferenceResult = ResultState.OK; + } + else + { + d.FinalResult = d.InferenceResult = ResultState.DetectNG; + } + } + else + { + if (detectConfig.IsMixModel) + { + d.FinalResult = d.InferenceResult = ResultState.A_NG; + } + else + { + //将所有已将筛选出来的缺陷进行过滤 + d.FinalResult = d.InferenceResult = ResultState.OK; + } + + + } + + + foreach (IGrouping group in conditionList) + { + bool b = group.ToList().Any(f => + { + return f.FilterOperation(d); + }); + + + if (b) + { + d.FinalResult = group.Key; + break; + } + //else + //{ + // d.FinalResult = d.InferenceResult = ResultState.OK; + //} + } + }); + #endregion + #region 5.统计缺陷过滤结果或预处理直接NG + if (detectResult.DetectDetails?.Count > 0) + { + detectResult.ResultState = detectResult.DetectDetails.GroupBy(u => u.FinalResult).OrderBy(u => u.Key).First().First().FinalResult; + detectResult.ResultLabel = detectResult.ResultLabel; + detectResult.ResultLabelCategoryId = detectResult.ResultLabel;//TODO:设置优先级 + //////根据优先级设置ResultLabel + //if (detectionLabels.Count > 0) + //{ + // foreach (var l in detectionLabels) + // { + // var isExist = DetectDetails.Any(o => NormalizeAndClean(o.LabelName) == NormalizeAndClean(l.LabelName) && o.FinalResult == ResultState.DetectNG); + // if (isExist) + // { + + // ResultLabelCategoryId = l.LabelCategoryId; + // break; + // } + // } + //} + + return; + } + #endregion + + + resultStates.Add(detectResult.ResultState); + + product.ResultCollection.Add(detectResult); + this.BeginInvoke(new MethodInvoker(delegate () + { + pictureBox1.Image = result.ResultMap; richTextBox1.AppendText($"推理成功{productNumber}，{result.IsSuccess} 推理耗时{sw.ElapsedMilliseconds}ms，总推理耗时\n"); + + })); //DetectStationResult temp; ////LogAsync(DateTime.Now, LogLevel.Information, $"{camera.Name} 推理进度1.4，产品{productNumber}"); //// 检测结果 @@ -541,10 +759,7 @@ namespace DHSoftware if (!product.InferenceFinished()) { - //if (!(camera.Name == "Cam8")) - //{ - // return; - //} + return; } ProductNum_Total++; @@ -559,6 +774,30 @@ namespace DHSoftware richTextBox1.SelectionStart = richTextBox1.TextLength; richTextBox1.ScrollToCaret(); })); + #region 6.统计产品结果 + if (product.ResultCollection.Any(u => u.ResultState != ResultState.OK)) + { + //检测结果TBD + // CurTrigger = TriggerSettings.FirstOrDefault(u => u.TriggerType == TriggerType.B_NG); + product.ProductResult = ResultState.B_NG; + product.ProductLabelCategory = ResultState.B_NG.GetEnumDescription(); + product.ProductLabel = ResultState.B_NG.GetEnumDescription(); + + + } + else + { + // CurTrigger = TriggerSettings.FirstOrDefault(u => u.TriggerType == TriggerType.OK); + product.ProductResult = ResultState.OK; + product.ProductLabelCategory = ResultState.OK.GetEnumDescription(); + product.ProductLabel = ResultState.OK.GetEnumDescription(); + } + #endregion + #region 7.产品吹气 + + #endregion + + //LogAsync(DateTime.Now, LogLevel.Information, $"推理完成，产品{product.PieceNumber}获取结果"); @@ -661,9 +900,19 @@ namespace DHSoftware if (isSuccess) { - // LogAsync(DateTime.Now, LogLevel.Assist, $"产品{productNumber}出列成功：{isSuccess}，" + - //$"产品结果：{temp.ProductResult.GetEnumDescription()}，" + - //$"当前队列产品数量：{tmpDic.Count}"); + string logStr =$"{DateTime.Now}产品{productNumber}出列成功：{isSuccess}，" + + $"产品结果：{temp.ProductResult.GetEnumDescription()}，" + + $"当前队列产品数量：{tmpDic.Count}"; + this.BeginInvoke(new MethodInvoker(delegate () { + + int currentScrollPosition = richTextBox1.GetPositionFromCharIndex(richTextBox1.TextLength).Y; + + richTextBox1.AppendText(logStr); + + // 设置回原来的滚动位置 + richTextBox1.SelectionStart = richTextBox1.TextLength; + richTextBox1.ScrollToCaret(); + })); } tryTimes--; Thread.Sleep(1); @@ -703,6 +952,29 @@ namespace DHSoftware } }); } + public void SetResult() + { + + + + + + //// detectResult.IsPreTreatDone = detectResult.VisionImageSet.PreTreatedFlag + ////2024-02-29 目标检测不能全是NG + //if (IsPreTreatNG || IsObjectDetectNG) + //{ + // return; + //} + + + + //if (IsPreTreatDone && IsMLDetectDone && IsAfterTreatDone) + //{ + // ResultState = ResultState.OK; + // ResultLabel = ResultState.OK.GetEnumDescription(); + //} + + } private void HandleStopButton() { Cameras.Clear();