DHDHSoftware/DH.Devices.Vision/SimboVisionDriver.cs

using DH.Commons.Base;
using DH.Commons.Enums;
using DH.UI.Model.Winform;
using HalconDotNet;
using OpenCvSharp;
using OpenCvSharp.Extensions;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Runtime.ExceptionServices;
using System.Text;
using System.Threading.Tasks;
using System.Windows.Forms;
using System.Xml.Linq;
using XKRS.UI.Model.Winform;
using static DH.Commons.Enums.EnumHelper;
using ResultState = DH.Commons.Base.ResultState;


namespace DH.Devices.Vision
{
    public class SimboVisionDriver : VisionEngineBase
    {
      public  Dictionary<string, HDevEngineTool> HalconToolDict = new Dictionary<string, HDevEngineTool>();
    
      public  List<SimboStationMLEngineSet> SimboStationMLEngineList = new List<SimboStationMLEngineSet>();

        public void Init()
        {
            //InitialQueue();
            InitialHalconTools();
            InitialSimboMLEnginesAsync();

          //  ImageSaveHelper.OnImageSaveExceptionRaised -= ImageSaveHelper_OnImageSaveExceptionRaised;
          //  ImageSaveHelper.OnImageSaveExceptionRaised += ImageSaveHelper_OnImageSaveExceptionRaised;
          //  base.Init();
        }

        //private void ImageSaveHelper_OnImageSaveExceptionRaised(DateTime dt, string msg)
        //{
        //    LogAsync(new LogMsg(dt, LogLevel.Error, msg));
        //}
        public override DetectStationResult RunInference(Mat originImgSet, string detectionId = null)
        {
            DetectStationResult detectResult = new DetectStationResult();
            DetectionConfig detectConfig = null;
            //找到对应的配置
            if (!string.IsNullOrWhiteSpace(detectionId))
            {
                detectConfig = DetectionConfigs.FirstOrDefault(u => u.Id == detectionId);
            }
            else
            {
                //detectConfig = DetectionConfigs.FirstOrDefault(u => u.CameraSourceId == camera.CameraName);
            }

            if (detectConfig == null)
            {

                //未能获得检测配置
                return detectResult;
            }
            #region 1.预处理

            using (Mat PreTMat = originImgSet.Clone())
            {
                PreTreated(detectConfig, detectResult, PreTMat);
            }



            #endregion
            if (detectResult.IsPreTreatNG)
            {
                detectResult.ResultState = ResultState.DetectNG;
                detectResult.IsPreTreatDone = true;
                detectResult.IsMLDetectDone = false;
                return detectResult;

            }



            if (!string.IsNullOrWhiteSpace(detectConfig.ModelPath) && detectConfig.IsEnabled)
            {


                SimboStationMLEngineSet mlSet = null;
                mlSet = SimboStationMLEngineList.FirstOrDefault(t => t.DetectionId == detectConfig.Id);
                if (mlSet == null)
                {
                    // LogAsync(DateTime.Now, LogLevel.Exception, $"异常：{detectConfig.Name}未能获取对应配置的模型检测工具");
                    detectResult.IsMLDetectDone = false;

                    //HandleDetectDone(detectResult, detectConfig);
                    return detectResult;
                }

                #region 2.深度学习推理 
                //LogAsync(DateTime.Now, LogLevel.Information, $"{detectConfig.Name} 产品{detectResult.TempPid} 模型检测执行");

                if (!string.IsNullOrWhiteSpace(detectConfig.ModelPath))
                {
                    Stopwatch mlWatch = new Stopwatch();
                    var req = new MLRequest();
                    //之前的检测图片都是相机存储成HImage


                    req.ResizeWidth = (int)detectConfig.ModelWidth;
                    req.ResizeHeight = (int)detectConfig.ModelHeight;
                    // req.LabelNames = detectConfig.GetLabelNames();
                    // req.Score = IIConfig.Score;
                    req.mImage = originImgSet.Clone();

                    req.in_lable_path = detectConfig.In_lable_path;

                    req.confThreshold = detectConfig.ModelconfThreshold;
                    req.iouThreshold = 0.3f;
                    req.segmentWidth = 320;
                    req.out_node_name = "output0";
                    switch (detectConfig.ModelType)
                    {
                        case ModelType.图像分类:
                            break;
                        case ModelType.目标检测:

                            break;
                        case ModelType.语义分割:
                            break;
                        case ModelType.实例分割:
                            break;
                        case ModelType.目标检测GPU:

                            break;
                        default:
                            break;
                    }

                    //  LogAsync(DateTime.Now, LogLevel.Information, $"{detectConfig.Name} 产品{detectResult.TempPid} RunInference BEGIN");
                    mlWatch.Start();
                    //20230802改成多线程推理 RunInferenceFixed

                    var result = mlSet.StationMLEngine.RunInference(req);
                    // var result = mlSet.StationMLEngine.RunInferenceFixed(req);
                    mlWatch.Stop();
                    // LogAsync(DateTime.Now, LogLevel.Information, $"{detectConfig.Name} 产品{detectResult.TempPid} RunInference END");






                    //            var req = new MLRequest();

                    //req.mImage = inferenceImage;

                    //req.ResizeWidth = detectConfig.ModelWidth;
                    //req.ResizeHeight = detectConfig.ModelHeight;
                    //req.confThreshold = detectConfig.ModelconfThreshold;
                    //req.iouThreshold = 0.3f;
                    //req.out_node_name = "output0";
                    //req.in_lable_path = detectConfig.in_lable_path;

                    //Stopwatch sw = Stopwatch.StartNew();
                    //var result =   Dectection[detectionId].RunInference(req);
                    //sw.Stop();
                    //LogAsync(DateTime.Now, LogLevel.Information, $"{camera.Name} 推理进度1.1，产品{productNumber}，耗时{sw.ElapsedMilliseconds}ms");

                    //this.BeginInvoke(new MethodInvoker(delegate ()
                    //{
                    //    // pictureBox1.Image?.Dispose(); // 释放旧图像
                    //    // pictureBox1.Image = result.ResultMap;
                    //    richTextBox1.AppendText($"推理成功 {productNumber}, {result.IsSuccess}相机名字{camera.CameraName} 耗时 {mlWatch.ElapsedMilliseconds}ms\n");
                    //}));
                    //req.mImage?.Dispose();




                    if (result == null || (result != null && !result.IsSuccess))
                    {
                        detectResult.IsMLDetectDone = false;
                    }
                    if (result != null && result.IsSuccess)
                    {
                        detectResult.DetectDetails = result.ResultDetails;
                        if (detectResult.DetectDetails != null)
                        {
                        }
                        else
                        {
                            detectResult.IsMLDetectDone = false;
                        }
                    }
                }
                #endregion



                #region 3.后处理
                #endregion
                //根据那些得分大于阈值的推理结果，判断产品是否成功
                #region 4.最终过滤（逻辑过滤）
                detectResult.DetectDetails?.ForEach(d =>
                {


                    // 当前检测项的 过滤条件
                    var conditionList = detectConfig.DetectionLableList
                        .Where(u=>u.LabelName == d.LabelName)
                        .GroupBy(u => u.ResultState)
                        .OrderBy(u => u.Key)
                        .ToList();
               

                    if (conditionList.Count == 0)
                    {

                        d.FinalResult = d.LabelName.ToLower() == "ok"
                       ? ResultState.OK
                       : ResultState.DetectNG;
                    }
                    else
                    {
                        d.FinalResult = detectConfig.IsMixModel
                       ? ResultState.A_NG
                       : ResultState.OK;


                    }


                    foreach (IGrouping<ResultState, DetectionFilter> group in conditionList)
                    {
                        bool b = group.ToList().Any(f =>
                        {
                            return f.FilterOperation(d);
                        });


                        if (b)
                        {
                            d.FinalResult = group.Key;
                            break;
                        }

                       
                    }
                });
                #endregion
                #region 5.统计缺陷过滤结果或预处理直接NG
                //if (detectResult.DetectDetails?.Count > 0)
                //{
                //    detectResult.ResultState = detectResult.DetectDetails.GroupBy(u => u.FinalResult).OrderBy(u => u.Key).First().First().FinalResult;
                //    detectResult.ResultLabel = detectResult.ResultLabel;
                //    detectResult.ResultLabelCategoryId = detectResult.ResultLabel;//TODO:设置优先级


                //}
                detectResult.ResultState = detectResult.DetectDetails?
           .GroupBy(u => u.FinalResult)
           .OrderBy(u => u.Key)
           .FirstOrDefault()?.Key ?? ResultState.OK;
                detectResult.ResultLabel = detectResult.ResultLabel;
                detectResult.ResultLabelCategoryId = detectResult.ResultLabel;//TODO:设置优先级
                #endregion





                DisplayDetectionResult(detectResult, originImgSet.Clone(), detectionId);




            }
            return detectResult;

        }
        /// <summary>
        /// 初始化深度学习工具
        /// </summary>
        private bool InitialSimboMLEnginesAsync()
        {
            //深度学习 模型加载
            var resultOK = MLLoadModel();
            return resultOK;
        }
        /// <summary>
        /// 深度学习 模型加载
        /// </summary>
        /// <returns></returns>
        private bool MLLoadModel()
        {
            bool resultOK = false;
            try
            {
               // SimboStationMLEngineList = new List<SimboStationMLEngineSet>();
                // _cameraRelatedDetectionDict = IConfig.DetectionConfigs.Select(t => t.ModelPath).Distinct().ToList();
                DetectionConfigs.ForEach(dc =>
                //_cameraRelatedDetectionDict.ForEach(dc =>
                {

                    if (dc.IsEnabled && !string.IsNullOrWhiteSpace(dc.ModelPath))
                    {
                        if (dc.IsEnableGPU)
                        {
                            //if (IIConfig.IsLockGPU)
                            //{
                            //foreach (var validGPU in ValidGPUList2)
                            //{
                            //    if (validGPU.DetectionIds.Contains(dc.Id))
                            //    {
                            var engine = SingleMLLoadModel(dc, true, 0);
                            SimboStationMLEngineList.Add(engine);
                            //    }
                            //}
                            //}
                            //else
                            //{
                            //    foreach (var validGPU in ValidGPUList)
                            //    {
                            //        //var validGPU = ValidGPUList.FirstOrDefault(u => u.DetectionIds.Contains(dc.Id));
                            //        if (validGPU.DetectionId == dc.Id)
                            //        {
                            //            var engine = SingleMLLoadModel(dc, true, validGPU.GPUNo);
                            //            SimboStationMLEngineList.Add(engine);
                            //        }
                            //    }
                            //}

                        }
                        else
                        {
                            //for (int i = 0; i < IConfig.CPUNums; i++)
                            for (int i = 0; i < 1; i++)
                            {
                                //var engine = SingleMLLoadModel(dc, false, i);
                                var engine = SingleMLLoadModel(dc, false, i);
                                SimboStationMLEngineList.Add(engine);
                            }
                        }
                    }
                });
                resultOK = true;
            }
            catch (Exception ex)
            {
                // LogAsync(DateTime.Now, LogLevel.Exception, $"异常：模型并发加载异常:{ex.GetExceptionMessage()}");
                resultOK = false;
            }

            return resultOK;
        }
        /// <summary>
        /// 单个模型加载
        /// </summary>
        /// <param name="dc"></param>
        /// <param name="gpuNum"></param>
        /// <returns></returns>
        private SimboStationMLEngineSet SingleMLLoadModel(DetectionConfig dc, bool isGPU, int coreInx)
        {
            SimboStationMLEngineSet mLEngineSet = new SimboStationMLEngineSet();
            try
            {
                mLEngineSet.IsUseGPU = isGPU;
                if (isGPU)
                {
                    mLEngineSet.GPUNo = coreInx;
                }
                else
                {
                    mLEngineSet.CPUNo = coreInx;
                }
                mLEngineSet.DetectionId = dc.Id;
                mLEngineSet.DetectionName = dc.Name;

                if (!string.IsNullOrWhiteSpace(dc.ModelPath))
                {
                    // 根据算法类型创建不同的实例
                    switch (dc.ModelType)
                    {
                        case ModelType.图像分类:
                            break;
                        case ModelType.目标检测:
                            mLEngineSet.StationMLEngine = new SimboObjectDetection();
                            break;
                        case ModelType.语义分割:

                            break;
                        case ModelType.实例分割:
                            mLEngineSet.StationMLEngine = new SimboInstanceSegmentation();
                            break;
                        case ModelType.目标检测GPU:
                            mLEngineSet.StationMLEngine = new SimboDetection();
                            break;
                        default:
                            break;
                    }
                    MLInit mLInit;
                    string inferenceDevice = "CPU";
                    if (dc.IsEnableGPU)
                    {
                        inferenceDevice = "GPU";
                        mLInit = new MLInit(dc.ModelPath, isGPU, coreInx, dc.ModelconfThreshold);
                    }
                    else
                    {
                        mLInit = new MLInit(dc.ModelPath, "images", inferenceDevice, (int)dc.ModelWidth, (int)dc.ModelHeight);

                    }

                    bool isSuccess = mLEngineSet.StationMLEngine.Load(mLInit);
                    if (!isSuccess)
                    {
                        // throw new ProcessException("异常：模型加载异常", null);
                    }
                    //LogAsync(DateTime.Now, LogLevel.Information, $"模型加载成功；是否GPU:{isGPU} CoreInx:{coreInx} - {dc.Name}" + $" {dc.ModelType.GetEnumDescription()}:{dc.ModelPath}");
                }
            }
            catch (Exception ex)
            {
                //throw new ProcessException($"异常：是否GPU:{isGPU} CoreInx:{coreInx} - {dc.Name}模型加载异常：{ex.GetExceptionMessage()}");
            }
            return mLEngineSet;
        }
        private void InitialHalconTools()
        {
            HOperatorSet.SetSystem("parallelize_operators", "true");
            HOperatorSet.SetSystem("reentrant", "true");
            HOperatorSet.SetSystem("global_mem_cache", "exclusive");

            HalconToolDict = new Dictionary<string, HDevEngineTool>();

            DetectionConfigs.ForEach(c =>
            {
                if (!c.IsEnabled)
                    return;

                if (c.HalconAlgorithemPath_Pre != null)
                    LoadHalconTool(c.HalconAlgorithemPath_Pre);

            });
        }

        private void LoadHalconTool(string path)
        {
            if (!HalconToolDict.ContainsKey(path))
            {


                string algorithemPath = path;

                if (string.IsNullOrWhiteSpace(algorithemPath))
                    return;

                string directoryPath = Path.GetDirectoryName(algorithemPath);
                string fileName = Path.GetFileNameWithoutExtension(algorithemPath);

                HDevEngineTool tool = new HDevEngineTool(directoryPath);
                tool.LoadProcedure(fileName);

                HalconToolDict[path] = tool;
            }
        }

        /// <summary>
        /// 预处理
        /// </summary>
        /// <param name="detectConfig"></param>
        /// <param name="detectResult"></param>
        public void PreTreated(DetectionConfig detectConfig, DetectStationResult detectResult, Mat MhImage)
        {
            try
            {
                // detectResult.VisionImageSet.DetectionOriginImage = detectResult.VisionImageSet.HImage.ConvertHImageToBitmap();
                //detectResult.VisionImageSet.PreTreatedBitmap = detectResult.VisionImageSet.HImage.ConvertHImageToBitmap();
                //detectResult.VisionImageSet.DetectionResultImage = detectResult.VisionImageSet.PreTreatedBitmap?.CopyBitmap();
                if (!string.IsNullOrWhiteSpace(detectConfig.HalconAlgorithemPath_Pre))
                {
                    HObject obj = OpenCVHelper.MatToHImage(MhImage);
                    HImage hImage = HalconHelper.ConvertHObjectToHImage(obj);
                    string toolKey = detectConfig.HalconAlgorithemPath_Pre;
                    if (!HalconToolDict.ContainsKey(toolKey))
                    {
                        // LogAsync(DateTime.Now, LogLevel.Exception, $"{detectConfig.Name}未获取预处理算法");
                        return;
                    }
                    //Mean_Thre Deviation_Thre  Mean_standard Deviation_standard
                    var tool = HalconToolDict[toolKey];

                    ////tool.InputTupleDic["Mean_Thre"] = 123;
                    for (int i = 0; i < detectConfig.PreTreatParams.Count; i++)
                    {
                        var param = detectConfig.PreTreatParams[i];
                        tool.InputTupleDic[param.Name] = double.Parse(param.Value);
                    }

                    // tool.InputTupleDic["fCricularity"] = 200;

                    tool.InputImageDic["INPUT_Image"] = hImage;


                    if (!tool.RunProcedure(out string errorMsg, out _))
                    {
                        //  detectResult.PreTreatedFlag = false;

                        detectResult.IsPreTreatDone = false;


                        return;
                    }

                    var preTreatRet = tool.GetResultTuple("OUTPUT_Flag").I;

                    //var fRCricularity = tool.GetResultTuple("fRCricularity");


                    // detectResult.IsPreTreatDone = detectResult.VisionImageSet.PreTreatedFlag = preTreatRet == 1;
                    //detectResult.IsPreTreatDone = detectResult.VisionImageSet.PreTreatedFlag = true;
                    //  detectResult.VisionImageSet.PreTreatedTime = DateTime.Now;

                    for (int i = 0; i < detectConfig.OUTPreTreatParams.Count; i++)
                    {
                        var param = detectConfig.OUTPreTreatParams[i];
                        tool.InputTupleDic[param.Name] = double.Parse(param.Value);
                    }




                    // 2023/10/16 新增预处理结果反馈，如果预处理结果为NG，直接返回
                    if (preTreatRet != 0)
                    {
                        detectResult.ResultState = ResultState.DetectNG;

                        detectResult.IsPreTreatNG = true;



                        // if (detectResult.VisionImageSet.PreTreatedFlag)
                        {
                            //detectResult.VisionImageSet.MLImage = tool.GetResultObject("OUTPUT_PreTreatedImage");
                            //DetectionResultImage
                            // detectResult.VisionImageSet.DetectionResultImage = detectResult.VisionImageSet.MLImage.ConvertHImageToBitmap();

                        }

                    }
                    else
                    {
                        // detectResult.VisionImageSet.DetectionResultImage = detectResult.VisionImageSet.MLImage.ConvertHImageToBitmap();

                    }
                }
            }
            catch (Exception ex)
            {

            }
            finally
            {
                //detectResult.VisionImageSet.HImage?.Dispose();
                //detectResult.VisionImageSet.HImage = null;
                // MhImage?.Dispose();
                //MhImage = null;
            }

        }

        /// <summary>
        /// 显示检测结果
        /// </summary>
        /// <param name="detectResult"></param>
        private void DisplayDetectionResult(DetectStationResult detectResult,Mat result,string DetectionId)
        {
            //结果显示上传
            Task.Run(() =>
            {
                try
                {


                    string displayTxt = detectResult.ResultState.ToString() + "\r\n";
                    if (detectResult.DetectDetails != null && detectResult.DetectDetails?.Count > 0)
                    {
                        detectResult.DetectDetails.ForEach(d =>
                        {
                            displayTxt +=
                                $"{d.LabelName} score:{d.Score.ToString("f2")} area:{d.Area.ToString("f2")}\r\n";
                        });
                    }

                    //if (detectResult.realSpecs != null && detectResult.realSpecs?.Count > 0)
                    //{
                    //    detectResult.realSpecs.ForEach(d =>
                    //    {
                    //        displayTxt +=
                    //            $"{d.Code} :{d.ActualValue} \r\n";
                    //    });
                    //}
                    Bitmap resultMask=result.ToBitmap();
                    //if (detectResult.VisionImageSet.DetectionResultImage == null && detectResult.VisionImageSet.SizeResultImage == null)
                    //{
                    //    return;
                    //}
                    //else if (detectResult.VisionImageSet.DetectionResultImage == null && detectResult.VisionImageSet.SizeResultImage != null)
                    //{
                    //    detectResult.VisionImageSet.DetectionResultImage = detectResult.VisionImageSet.SizeResultImage.CopyBitmap();
                    //    resultMask = detectResult.VisionImageSet.DetectionResultImage.CopyBitmap();
                    //}
                    //else if (detectResult.VisionImageSet.DetectionResultImage != null && detectResult.VisionImageSet.SizeResultImage != null)
                    //{
                    //    Mat img1 = ConvertBitmapToMat(detectResult.VisionImageSet.SizeResultImage.CopyBitmap()); // 第一张图片，已经带框
                    //    Mat img2 = ConvertBitmapToMat(detectResult.VisionImageSet.DetectionResultImage.CopyBitmap()); // 第二张图片，已经带框

                    //    // 合成两张图像：可以选择叠加或拼接
                    //    Mat resultImg = new Mat();
                    //    Cv2.AddWeighted(img1, 0.5, img2, 0.5, 0, resultImg); // 使用加权平均法合成图像

                    //    resultMask = resultImg.ToBitmap();
                    //}
                    //else
                    //{
                    //    resultMask = detectResult.VisionImageSet.DetectionResultImage.CopyBitmap();
                    //}

                    List<IShapeElement> detectionResultShapes =
                        new List<IShapeElement>(detectResult.DetectionResultShapes);

                    DetectResultDisplay resultDisplay = new DetectResultDisplay(detectResult, resultMask, displayTxt);
                    detectionResultShapes.Add(resultDisplay);
                    List<IShapeElement> detectionResultShapesClone = new List<IShapeElement>(detectionResultShapes);

                    DetectionDone(DetectionId, resultMask, detectionResultShapes);

                    //SaveDetectResultImageAsync(detectResult);
                   // SaveDetectResultCSVAsync(detectResult);
                }
                catch (Exception ex)
                {
                   // LogAsync(DateTime.Now, LogLevel.Exception,
                      //  $"{Name}显示{detectResult.DetectName}检测结果异常,{ex.GetExceptionMessage()}");
                }
                finally
                {


                }
            });
        }

    }
}