DHDHSoftware/DH.Devices.Vision/SimboVisionDriver.cs
2025-03-24 15:21:16 +08:00

657 lines
25 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

using DH.Commons.Base;
using DH.Commons.Enums;
using DH.UI.Model.Winform;
using HalconDotNet;
using OpenCvSharp;
using OpenCvSharp.Extensions;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Runtime.ExceptionServices;
using System.Text;
using System.Threading.Tasks;
using System.Windows.Forms;
using System.Xml.Linq;
using XKRS.UI.Model.Winform;
using static DH.Commons.Enums.EnumHelper;
using ResultState = DH.Commons.Base.ResultState;
namespace DH.Devices.Vision
{
public class SimboVisionDriver : VisionEngineBase
{
public Dictionary<string, HDevEngineTool> HalconToolDict = new Dictionary<string, HDevEngineTool>();
public List<SimboStationMLEngineSet> SimboStationMLEngineList = new List<SimboStationMLEngineSet>();
public void Init()
{
//InitialQueue();
InitialHalconTools();
InitialSimboMLEnginesAsync();
// ImageSaveHelper.OnImageSaveExceptionRaised -= ImageSaveHelper_OnImageSaveExceptionRaised;
// ImageSaveHelper.OnImageSaveExceptionRaised += ImageSaveHelper_OnImageSaveExceptionRaised;
// base.Init();
}
//private void ImageSaveHelper_OnImageSaveExceptionRaised(DateTime dt, string msg)
//{
// LogAsync(new LogMsg(dt, LogLevel.Error, msg));
//}
public override DetectStationResult RunInference(Mat originImgSet, string detectionId = null)
{
DetectStationResult detectResult = new DetectStationResult();
DetectionConfig detectConfig = null;
//找到对应的配置
if (!string.IsNullOrWhiteSpace(detectionId))
{
detectConfig = DetectionConfigs.FirstOrDefault(u => u.Id == detectionId);
}
else
{
//detectConfig = DetectionConfigs.FirstOrDefault(u => u.CameraSourceId == camera.CameraName);
}
if (detectConfig == null)
{
//未能获得检测配置
return detectResult;
}
#region 1.
using (Mat PreTMat = originImgSet.Clone())
{
PreTreated(detectConfig, detectResult, PreTMat);
}
#endregion
if (detectResult.IsPreTreatNG)
{
detectResult.ResultState = ResultState.DetectNG;
detectResult.IsPreTreatDone = true;
detectResult.IsMLDetectDone = false;
return detectResult;
}
if (!string.IsNullOrWhiteSpace(detectConfig.ModelPath) && detectConfig.IsEnabled)
{
SimboStationMLEngineSet mlSet = null;
mlSet = SimboStationMLEngineList.FirstOrDefault(t => t.DetectionId == detectConfig.Id);
if (mlSet == null)
{
// LogAsync(DateTime.Now, LogLevel.Exception, $"异常:{detectConfig.Name}未能获取对应配置的模型检测工具");
detectResult.IsMLDetectDone = false;
//HandleDetectDone(detectResult, detectConfig);
return detectResult;
}
#region 2.
//LogAsync(DateTime.Now, LogLevel.Information, $"{detectConfig.Name} 产品{detectResult.TempPid} 模型检测执行");
if (!string.IsNullOrWhiteSpace(detectConfig.ModelPath))
{
Stopwatch mlWatch = new Stopwatch();
var req = new MLRequest();
//之前的检测图片都是相机存储成HImage
req.ResizeWidth = (int)detectConfig.ModelWidth;
req.ResizeHeight = (int)detectConfig.ModelHeight;
// req.LabelNames = detectConfig.GetLabelNames();
// req.Score = IIConfig.Score;
req.mImage = originImgSet.Clone();
req.in_lable_path = detectConfig.In_lable_path;
req.confThreshold = detectConfig.ModelconfThreshold;
req.iouThreshold = 0.3f;
req.segmentWidth = 320;
req.out_node_name = "output0";
switch (detectConfig.ModelType)
{
case ModelType.:
break;
case ModelType.:
break;
case ModelType.:
break;
case ModelType.:
break;
case ModelType.GPU:
break;
default:
break;
}
// LogAsync(DateTime.Now, LogLevel.Information, $"{detectConfig.Name} 产品{detectResult.TempPid} RunInference BEGIN");
mlWatch.Start();
//20230802改成多线程推理 RunInferenceFixed
var result = mlSet.StationMLEngine.RunInference(req);
// var result = mlSet.StationMLEngine.RunInferenceFixed(req);
mlWatch.Stop();
// LogAsync(DateTime.Now, LogLevel.Information, $"{detectConfig.Name} 产品{detectResult.TempPid} RunInference END");
// var req = new MLRequest();
//req.mImage = inferenceImage;
//req.ResizeWidth = detectConfig.ModelWidth;
//req.ResizeHeight = detectConfig.ModelHeight;
//req.confThreshold = detectConfig.ModelconfThreshold;
//req.iouThreshold = 0.3f;
//req.out_node_name = "output0";
//req.in_lable_path = detectConfig.in_lable_path;
//Stopwatch sw = Stopwatch.StartNew();
//var result = Dectection[detectionId].RunInference(req);
//sw.Stop();
//LogAsync(DateTime.Now, LogLevel.Information, $"{camera.Name} 推理进度1.1,产品{productNumber},耗时{sw.ElapsedMilliseconds}ms");
//this.BeginInvoke(new MethodInvoker(delegate ()
//{
// // pictureBox1.Image?.Dispose(); // 释放旧图像
// // pictureBox1.Image = result.ResultMap;
// richTextBox1.AppendText($"推理成功 {productNumber}, {result.IsSuccess}相机名字{camera.CameraName} 耗时 {mlWatch.ElapsedMilliseconds}ms\n");
//}));
//req.mImage?.Dispose();
if (result == null || (result != null && !result.IsSuccess))
{
detectResult.IsMLDetectDone = false;
}
if (result != null && result.IsSuccess)
{
detectResult.DetectDetails = result.ResultDetails;
if (detectResult.DetectDetails != null)
{
}
else
{
detectResult.IsMLDetectDone = false;
}
}
}
#endregion
#region 3.
#endregion
//根据那些得分大于阈值的推理结果,判断产品是否成功
#region 4.
detectResult.DetectDetails?.ForEach(d =>
{
// 当前检测项的 过滤条件
var conditionList = detectConfig.DetectionLableList
.Where(u=>u.LabelName == d.LabelName)
.GroupBy(u => u.ResultState)
.OrderBy(u => u.Key)
.ToList();
if (conditionList.Count == 0)
{
d.FinalResult = d.LabelName.ToLower() == "ok"
? ResultState.OK
: ResultState.DetectNG;
}
else
{
d.FinalResult = detectConfig.IsMixModel
? ResultState.A_NG
: ResultState.OK;
}
foreach (IGrouping<ResultState, DetectionFilter> group in conditionList)
{
bool b = group.ToList().Any(f =>
{
return f.FilterOperation(d);
});
if (b)
{
d.FinalResult = group.Key;
break;
}
}
});
#endregion
#region 5.NG
//if (detectResult.DetectDetails?.Count > 0)
//{
// detectResult.ResultState = detectResult.DetectDetails.GroupBy(u => u.FinalResult).OrderBy(u => u.Key).First().First().FinalResult;
// detectResult.ResultLabel = detectResult.ResultLabel;
// detectResult.ResultLabelCategoryId = detectResult.ResultLabel;//TODO:设置优先级
//}
detectResult.ResultState = detectResult.DetectDetails?
.GroupBy(u => u.FinalResult)
.OrderBy(u => u.Key)
.FirstOrDefault()?.Key ?? ResultState.OK;
detectResult.ResultLabel = detectResult.ResultLabel;
detectResult.ResultLabelCategoryId = detectResult.ResultLabel;//TODO:设置优先级
#endregion
DisplayDetectionResult(detectResult, originImgSet.Clone(), detectionId);
}
return detectResult;
}
/// <summary>
/// 初始化深度学习工具
/// </summary>
private bool InitialSimboMLEnginesAsync()
{
//深度学习 模型加载
var resultOK = MLLoadModel();
return resultOK;
}
/// <summary>
/// 深度学习 模型加载
/// </summary>
/// <returns></returns>
private bool MLLoadModel()
{
bool resultOK = false;
try
{
// SimboStationMLEngineList = new List<SimboStationMLEngineSet>();
// _cameraRelatedDetectionDict = IConfig.DetectionConfigs.Select(t => t.ModelPath).Distinct().ToList();
DetectionConfigs.ForEach(dc =>
//_cameraRelatedDetectionDict.ForEach(dc =>
{
if (dc.IsEnabled && !string.IsNullOrWhiteSpace(dc.ModelPath))
{
if (dc.IsEnableGPU)
{
//if (IIConfig.IsLockGPU)
//{
//foreach (var validGPU in ValidGPUList2)
//{
// if (validGPU.DetectionIds.Contains(dc.Id))
// {
var engine = SingleMLLoadModel(dc, true, 0);
SimboStationMLEngineList.Add(engine);
// }
//}
//}
//else
//{
// foreach (var validGPU in ValidGPUList)
// {
// //var validGPU = ValidGPUList.FirstOrDefault(u => u.DetectionIds.Contains(dc.Id));
// if (validGPU.DetectionId == dc.Id)
// {
// var engine = SingleMLLoadModel(dc, true, validGPU.GPUNo);
// SimboStationMLEngineList.Add(engine);
// }
// }
//}
}
else
{
//for (int i = 0; i < IConfig.CPUNums; i++)
for (int i = 0; i < 1; i++)
{
//var engine = SingleMLLoadModel(dc, false, i);
var engine = SingleMLLoadModel(dc, false, i);
SimboStationMLEngineList.Add(engine);
}
}
}
});
resultOK = true;
}
catch (Exception ex)
{
// LogAsync(DateTime.Now, LogLevel.Exception, $"异常:模型并发加载异常:{ex.GetExceptionMessage()}");
resultOK = false;
}
return resultOK;
}
/// <summary>
/// 单个模型加载
/// </summary>
/// <param name="dc"></param>
/// <param name="gpuNum"></param>
/// <returns></returns>
private SimboStationMLEngineSet SingleMLLoadModel(DetectionConfig dc, bool isGPU, int coreInx)
{
SimboStationMLEngineSet mLEngineSet = new SimboStationMLEngineSet();
try
{
mLEngineSet.IsUseGPU = isGPU;
if (isGPU)
{
mLEngineSet.GPUNo = coreInx;
}
else
{
mLEngineSet.CPUNo = coreInx;
}
mLEngineSet.DetectionId = dc.Id;
mLEngineSet.DetectionName = dc.Name;
if (!string.IsNullOrWhiteSpace(dc.ModelPath))
{
// 根据算法类型创建不同的实例
switch (dc.ModelType)
{
case ModelType.:
break;
case ModelType.:
mLEngineSet.StationMLEngine = new SimboObjectDetection();
break;
case ModelType.:
break;
case ModelType.:
mLEngineSet.StationMLEngine = new SimboInstanceSegmentation();
break;
case ModelType.GPU:
mLEngineSet.StationMLEngine = new SimboDetection();
break;
default:
break;
}
MLInit mLInit;
string inferenceDevice = "CPU";
if (dc.IsEnableGPU)
{
inferenceDevice = "GPU";
mLInit = new MLInit(dc.ModelPath, isGPU, coreInx, dc.ModelconfThreshold);
}
else
{
mLInit = new MLInit(dc.ModelPath, "images", inferenceDevice, (int)dc.ModelWidth, (int)dc.ModelHeight);
}
bool isSuccess = mLEngineSet.StationMLEngine.Load(mLInit);
if (!isSuccess)
{
// throw new ProcessException("异常:模型加载异常", null);
}
//LogAsync(DateTime.Now, LogLevel.Information, $"模型加载成功是否GPU:{isGPU} CoreInx:{coreInx} - {dc.Name}" + $" {dc.ModelType.GetEnumDescription()}:{dc.ModelPath}");
}
}
catch (Exception ex)
{
//throw new ProcessException($"异常是否GPU:{isGPU} CoreInx:{coreInx} - {dc.Name}模型加载异常:{ex.GetExceptionMessage()}");
}
return mLEngineSet;
}
private void InitialHalconTools()
{
HOperatorSet.SetSystem("parallelize_operators", "true");
HOperatorSet.SetSystem("reentrant", "true");
HOperatorSet.SetSystem("global_mem_cache", "exclusive");
HalconToolDict = new Dictionary<string, HDevEngineTool>();
DetectionConfigs.ForEach(c =>
{
if (!c.IsEnabled)
return;
if (c.HalconAlgorithemPath_Pre != null)
LoadHalconTool(c.HalconAlgorithemPath_Pre);
});
}
private void LoadHalconTool(string path)
{
if (!HalconToolDict.ContainsKey(path))
{
string algorithemPath = path;
if (string.IsNullOrWhiteSpace(algorithemPath))
return;
string directoryPath = Path.GetDirectoryName(algorithemPath);
string fileName = Path.GetFileNameWithoutExtension(algorithemPath);
HDevEngineTool tool = new HDevEngineTool(directoryPath);
tool.LoadProcedure(fileName);
HalconToolDict[path] = tool;
}
}
/// <summary>
/// 预处理
/// </summary>
/// <param name="detectConfig"></param>
/// <param name="detectResult"></param>
public void PreTreated(DetectionConfig detectConfig, DetectStationResult detectResult, Mat MhImage)
{
try
{
// detectResult.VisionImageSet.DetectionOriginImage = detectResult.VisionImageSet.HImage.ConvertHImageToBitmap();
//detectResult.VisionImageSet.PreTreatedBitmap = detectResult.VisionImageSet.HImage.ConvertHImageToBitmap();
//detectResult.VisionImageSet.DetectionResultImage = detectResult.VisionImageSet.PreTreatedBitmap?.CopyBitmap();
if (!string.IsNullOrWhiteSpace(detectConfig.HalconAlgorithemPath_Pre))
{
HObject obj = OpenCVHelper.MatToHImage(MhImage);
HImage hImage = HalconHelper.ConvertHObjectToHImage(obj);
string toolKey = detectConfig.HalconAlgorithemPath_Pre;
if (!HalconToolDict.ContainsKey(toolKey))
{
// LogAsync(DateTime.Now, LogLevel.Exception, $"{detectConfig.Name}未获取预处理算法");
return;
}
//Mean_Thre Deviation_Thre Mean_standard Deviation_standard
var tool = HalconToolDict[toolKey];
////tool.InputTupleDic["Mean_Thre"] = 123;
for (int i = 0; i < detectConfig.PreTreatParams.Count; i++)
{
var param = detectConfig.PreTreatParams[i];
tool.InputTupleDic[param.Name] = double.Parse(param.Value);
}
// tool.InputTupleDic["fCricularity"] = 200;
tool.InputImageDic["INPUT_Image"] = hImage;
if (!tool.RunProcedure(out string errorMsg, out _))
{
// detectResult.PreTreatedFlag = false;
detectResult.IsPreTreatDone = false;
return;
}
var preTreatRet = tool.GetResultTuple("OUTPUT_Flag").I;
//var fRCricularity = tool.GetResultTuple("fRCricularity");
// detectResult.IsPreTreatDone = detectResult.VisionImageSet.PreTreatedFlag = preTreatRet == 1;
//detectResult.IsPreTreatDone = detectResult.VisionImageSet.PreTreatedFlag = true;
// detectResult.VisionImageSet.PreTreatedTime = DateTime.Now;
for (int i = 0; i < detectConfig.OUTPreTreatParams.Count; i++)
{
var param = detectConfig.OUTPreTreatParams[i];
tool.InputTupleDic[param.Name] = double.Parse(param.Value);
}
// 2023/10/16 新增预处理结果反馈如果预处理结果为NG直接返回
if (preTreatRet != 0)
{
detectResult.ResultState = ResultState.DetectNG;
detectResult.IsPreTreatNG = true;
// if (detectResult.VisionImageSet.PreTreatedFlag)
{
//detectResult.VisionImageSet.MLImage = tool.GetResultObject("OUTPUT_PreTreatedImage");
//DetectionResultImage
// detectResult.VisionImageSet.DetectionResultImage = detectResult.VisionImageSet.MLImage.ConvertHImageToBitmap();
}
}
else
{
// detectResult.VisionImageSet.DetectionResultImage = detectResult.VisionImageSet.MLImage.ConvertHImageToBitmap();
}
}
}
catch (Exception ex)
{
}
finally
{
//detectResult.VisionImageSet.HImage?.Dispose();
//detectResult.VisionImageSet.HImage = null;
// MhImage?.Dispose();
//MhImage = null;
}
}
/// <summary>
/// 显示检测结果
/// </summary>
/// <param name="detectResult"></param>
private void DisplayDetectionResult(DetectStationResult detectResult,Mat result,string DetectionId)
{
//结果显示上传
Task.Run(() =>
{
try
{
string displayTxt = detectResult.ResultState.ToString() + "\r\n";
if (detectResult.DetectDetails != null && detectResult.DetectDetails?.Count > 0)
{
detectResult.DetectDetails.ForEach(d =>
{
displayTxt +=
$"{d.LabelName} score:{d.Score.ToString("f2")} area:{d.Area.ToString("f2")}\r\n";
});
}
//if (detectResult.realSpecs != null && detectResult.realSpecs?.Count > 0)
//{
// detectResult.realSpecs.ForEach(d =>
// {
// displayTxt +=
// $"{d.Code} :{d.ActualValue} \r\n";
// });
//}
Bitmap resultMask=result.ToBitmap();
//if (detectResult.VisionImageSet.DetectionResultImage == null && detectResult.VisionImageSet.SizeResultImage == null)
//{
// return;
//}
//else if (detectResult.VisionImageSet.DetectionResultImage == null && detectResult.VisionImageSet.SizeResultImage != null)
//{
// detectResult.VisionImageSet.DetectionResultImage = detectResult.VisionImageSet.SizeResultImage.CopyBitmap();
// resultMask = detectResult.VisionImageSet.DetectionResultImage.CopyBitmap();
//}
//else if (detectResult.VisionImageSet.DetectionResultImage != null && detectResult.VisionImageSet.SizeResultImage != null)
//{
// Mat img1 = ConvertBitmapToMat(detectResult.VisionImageSet.SizeResultImage.CopyBitmap()); // 第一张图片,已经带框
// Mat img2 = ConvertBitmapToMat(detectResult.VisionImageSet.DetectionResultImage.CopyBitmap()); // 第二张图片,已经带框
// // 合成两张图像:可以选择叠加或拼接
// Mat resultImg = new Mat();
// Cv2.AddWeighted(img1, 0.5, img2, 0.5, 0, resultImg); // 使用加权平均法合成图像
// resultMask = resultImg.ToBitmap();
//}
//else
//{
// resultMask = detectResult.VisionImageSet.DetectionResultImage.CopyBitmap();
//}
List<IShapeElement> detectionResultShapes =
new List<IShapeElement>(detectResult.DetectionResultShapes);
DetectResultDisplay resultDisplay = new DetectResultDisplay(detectResult, resultMask, displayTxt);
detectionResultShapes.Add(resultDisplay);
List<IShapeElement> detectionResultShapesClone = new List<IShapeElement>(detectionResultShapes);
DetectionDone(DetectionId, resultMask, detectionResultShapes);
//SaveDetectResultImageAsync(detectResult);
// SaveDetectResultCSVAsync(detectResult);
}
catch (Exception ex)
{
// LogAsync(DateTime.Now, LogLevel.Exception,
// $"{Name}显示{detectResult.DetectName}检测结果异常,{ex.GetExceptionMessage()}");
}
finally
{
}
});
}
}
}