SIC-12/App/Modules/SicModules/PMs/RecipeExecutions/Process.cs

using System;
using System.Collections.Generic;
using Aitex.Core.Common;
using Aitex.Core.Common.DeviceData.IoDevice;
using Aitex.Core.RT.DataCenter;
using Aitex.Core.RT.Device;
using Aitex.Core.RT.Device.Devices;
using Aitex.Core.RT.Device.PmDevices;
using Aitex.Core.RT.Event;
using Aitex.Core.RT.Log;
using Aitex.Core.RT.OperationCenter;
using Aitex.Core.RT.Routine;
using Aitex.Core.RT.SCCore;
using Aitex.Core.RT.Tolerance;
using Aitex.Core.Util;
using MECF.Framework.Common.DBCore;
using MECF.Framework.Common.Equipment;
using MECF.Framework.Common.Gem;
using MECF.Framework.Common.SubstrateTrackings;
using SicModules.PMs.Routines.Base;
using MECF.Framework.Common.MECF.Framework.Common.RecipeCenter.Recipe;
using static Aitex.Core.RT.Device.PmDevices.DicMode;

namespace SicModules.PMs.RecipeExecutions
{
    public enum RecipeContinueMode
    {
        None,
        WaferReturnAndJobStop,
        RecipeCompleted,
        StepContinue,
        StepRestart,
        RecipeRestart,
        NextStep,
    }

    public partial class Process : PMBaseRoutine
    {
        enum RoutineStep
        {
            WaitProcess,
        }

        enum RecipeRunningState
        {
            Error,
            RecipeCompleted,
            ExecStep,
            TimeWait,
            ConditionWait,
            StepCompleted,
            Paused,
        }

        private object _recipeLocker = new object();
        private bool _hasRecordRunTime = false;

        private RecipeRunningState _state = RecipeRunningState.ExecStep;
        private RecipeRunningState _pausedState = RecipeRunningState.ExecStep;
        private DeviceTimer _estimatedTimeCalcTimer = new DeviceTimer(); //用于定时计算工艺程序估计的结束时间

        private double _curStepElpasedTimeBeforePaused;

        private double _curStepElpasedTimeBeforePaused2;
        private List<int> _lstSkipSteps = new List<int>();

        public RecipeContinueMode ContinueAction { get; set; }

        public DateTime _recipeStartTime { get; private set; }

        public string CurrentRecipeContent { get; private set; }


        private int _currentStepNumber;

        private int _dummyStepCount;

        public int CurStepTotalLoopCount { get; private set; }

        public double CurStepTotalTime
        {
            get
            {
                if (PmDevice.RecipeRunningInfo.RecipeStepList == null ||
                    PmDevice.RecipeRunningInfo.RecipeStepList.Count == 0 ||
                    _state == RecipeRunningState.RecipeCompleted || _state == RecipeRunningState.Error)
                    return 0;
                return PmDevice.RecipeRunningInfo.RecipeStepList[_currentStepNumber].StepTime * 1000;
            }
        }


        public int CurrentLoopCount { get; private set; }

        private DeviceTimer _stepTimer = new DeviceTimer();
        private DeviceTimer _stepTimer2 = new DeviceTimer();
        private DeviceTimer _recipeTimer = new DeviceTimer();

        public bool IsPaused { private set; get; }


        private double CurStepLeftTime
        {
            get { return _stepTimer.GetTotalTime() - _stepTimer.GetElapseTime(); }
        }

        public double EstimatedTotalLeftTime { get; private set; }

        private RecipeDBCallback _dbCallback;
        private Fdc _fdc;

        private bool _isDryRun;
        private int _delayTimeDryRun;

        private double _tempOffset;

        private int _currentStepIndex = 99;
        private IoInterLock _pmInterLock;

        private SicServo _pmSicServo = null;

        private bool bAlarm = false; //True-Alarm, False-Warning
        private double fThreshold = 3.0f; //阈值，从配置文件里取
        private int iTimes = -1; //超限次数，设定值
        private int iT = 0; //超限次数计数，实际值

        private double _pressureDifferenceUpperLimit;

        #region Parse

        private bool _isPSUHeaterJumpMode;
        private bool _isSCRHeaterJumpMode;
        private bool _isMFCJumpMode;

        #endregion


        #region Check

        private PeriodicJob _thread;
        private PeriodicJob _threadRotationAlarm;

        private DeviceTimer _rampCalcTimer = new DeviceTimer(); //用于定时获取PC的Ramp

        protected ToleranceChecker[] _mfcGapChecker = new ToleranceChecker[32];
        protected R_TRIG[] _mfcTrig = new R_TRIG[32];

        protected ToleranceChecker[] _pcGapChecker = new ToleranceChecker[7];
        protected R_TRIG[] _pcTrig = new R_TRIG[7];
        protected R_TRIG[] _pcTrig2 = new R_TRIG[7];

        protected double[] _pressurePrevious = new double[7];

        #endregion

        public Process(ModuleName module, PMModule pm1) : base(module, pm1)
        {
            Module = module.ToString();
            Name = "Process";

            _dbCallback = new RecipeDBCallback();
            _fdc = new Fdc(Module);
            _pmInterLock = DEVICE.GetDevice<IoInterLock>($"{Module}.PMInterLock");
            this._pmSicServo = DEVICE.GetDevice<SicServo>($"{Module}.PMServo");

            if (this._pmSicServo != null)
            {
                this.bAlarm = SC.GetValue<bool>($"PM.{Module}.RotationAlarm.WarningOrAlarm");
                this.fThreshold = SC.GetValue<double>($"PM.{Module}.RotationAlarm.Threshold");
                this.iTimes = SC.GetValue<int>($"PM.{Module}.RotationAlarm.Times");
            }


            for (int i = 0; i < _mfcGapChecker.Length; i++)
            {
                _mfcGapChecker[i] = new ToleranceChecker();
                _mfcTrig[i] = new R_TRIG();
            }

            for (int i = 0; i < _pcGapChecker.Length; i++)
            {
                _pcGapChecker[i] = new ToleranceChecker();
                _pcTrig[i] = new R_TRIG();
                _pcTrig2[i] = new R_TRIG();
            }


            Initialize();

            Calculte();

            SC.RegisterValueChangedCallback($"PM.{Module}.PT1PT2PressureDifferenceUpperLimit", (obj) => { _pressureDifferenceUpperLimit = (double)obj; });

            _thread = new PeriodicJob(10 * 1000, Calculte, "Calculte Standard Deviation", false);
            _threadRotationAlarm =
                new PeriodicJob(1000, CalculteRotationAlarm, "Calculte Rotation Alarm Deviation", false);
        }

        public override Result Start(params object[] param)
        {
            Reset();
            _hasRecordRunTime = false;
            if (!_pmInterLock.SetPMProcessRunning(true, out string reason))
            {
                EV.PostAlarmLog(Module, $"can not run Process, {reason}");
                return Result.FAIL;
            }

            _stepTimer = new DeviceTimer();
            _curStepElpasedTimeBeforePaused = 0;
            //_stepTimer2 = new DeviceTimer();
            //_recipeTimer = new DeviceTimer();

            //_stepTimer.Start(0);

            _lstSkipSteps = new List<int>();

            _currentStepIndex = 99;
            _currentStepNumber = CurStepTotalLoopCount = 0;
            _dummyStepCount = 0;

            _estimatedTimeCalcTimer.Start(1000);
            _rampCalcTimer.Start(1000);

            PmDevice.RecipeRunningInfo.InnerId = Guid.NewGuid();
            PmDevice.RecipeRunningInfo.BeginTime = DateTime.Now;
            PmDevice.RecipeRunningInfo.TotalTime = CalcRecipeTime();
            PmDevice.RecipeRunningInfo.IsRoutineAbort = false;

            PmDevice.RecipeRunningInfo.StepNumber = 1;
            PmDevice.RecipeRunningInfo.StepName =
                PmDevice.RecipeRunningInfo.RecipeStepList[_currentStepNumber].StepName;

            _state = RecipeRunningState.ExecStep;

            _recipeTimer.Start(int.MaxValue);

            _dbCallback.RecipeStart(PmDevice.Module, 0, PmDevice.RecipeRunningInfo.InnerId.ToString(),
                PmDevice.RecipeRunningInfo.RecipeName);
            _dbCallback.RecipeUpdateStatus(PmDevice.RecipeRunningInfo.InnerId.ToString(), "InProcess");

            //手动工艺时LotID和SequenceID可能是string.Empty
            //补全路径
            string recipeID = PmDevice.RecipeRunningInfo.RecipeName + ".rcp";
            string squenceID = $"Sequence\\{PmDevice.GetWaferSequenceID()}.seq";

            GemManager.Instance.Equipment?.TriggerEvent("PMRecipeStart", new string[] { "ChamberID", "RecipeID", "LotID", "SequenceID" }, new object[] { PmDevice.Module, recipeID, PmDevice.GetWaferLotID(), squenceID });

            WaferManager.Instance.UpdateWaferProcessStatus(ModuleHelper.Converter(Module), 0,
                WaferProcessStatus.InProcess);
            WaferManager.Instance.GetWafer(ModuleHelper.Converter(Module), 0).TrayProcessCount--;

            _fdc.Reset();

            _isDryRun = SC.GetValue<bool>($"PM.{Module}.DryRun.IsDryRun");
            _delayTimeDryRun = SC.GetValue<int>($"PM.{Module}.DryRun.DryRunDelayTime");
            _tempOffset = SC.GetValue<double>($"PM.{Module}.Process.TempOffset");

            _pressureDifferenceUpperLimit = SC.GetValue<double>($"PM.{Module}.PT1PT2PressureDifferenceUpperLimit");

            ResetHeaterResCheckResult();

            _thread.Start();
            _threadRotationAlarm.Start();

            Notify($"Start");
            return Result.RUN;
        }

        #region Heater Resistance Monitor

        private readonly R_TRIG _trigHeaterResOutOfRange = new();
        private readonly R_TRIG _trigHeaterResMonitorBegin = new();
        private readonly R_TRIG _trigHeaterResMonitorEnd = new();
        private readonly R_TRIG _trigPressureDifference = new();


        private void ResetHeaterResCheckResult()
        {
            _trigHeaterResOutOfRange.RST = true;
            _trigHeaterResMonitorBegin.RST = true;
            _trigHeaterResMonitorEnd.RST = true;
            _trigPressureDifference.RST = true;
        }

        private void MonitorHeaterResistance()
        {
            var reason = "";
            var totalElapseTime = PmDevice.RecipeRunningInfo.TotalElapseTime;//Recipe已经执行的时间
            var totalTime = PmDevice.RecipeRunningInfo.TotalTime;//Recipe总时间

            if (totalElapseTime == 0) //当开始执行后再去判断时间和电阻
                return;

            // 工艺刚开始和快结束的一段时间内，不需要检测电阻值。
            var ignoredDurationSec = SC.GetValue<double>($"PM.{Module}.Heater.InProcessResistanceMonitorIgnoreDuration");

            // 开始监测和节结束监测电阻时，输出信息，方便调试。
            _trigHeaterResMonitorBegin.CLK = totalElapseTime >= ignoredDurationSec;
            _trigHeaterResMonitorEnd.CLK = totalTime - totalElapseTime < ignoredDurationSec;
            if (_trigHeaterResMonitorBegin.Q)
                Notify("Begin Heater Resistance Monitor");
            if (_trigHeaterResMonitorEnd.Q)
                Notify("End Heater Resistance Monitor");

            // 工艺刚开始和快结束的一段时间内，不需要检测电阻值。
            if (!_trigHeaterResMonitorBegin.M || _trigHeaterResMonitorEnd.M)
                return;

            // 如果在工艺中，检测Heater电阻
            for (var i = 1; i <= 3; i++) //检测出所有的加热丝是否有断开的,不要提前结束循环
            {
                var ioPsuData = (IoPsuData)DATA.Poll($"{Module}.PSU{i}.DeviceData");
                if (ioPsuData != null)
                {
                    if (ioPsuData.IsResistanceOutOfRange)//PSU加热丝断开,后续可能增加条件判断
                        reason += $"PSU{i} resistance:{ioPsuData.Resistance} is out of range\r\n ";
                }
                
                var ioScrData = (IoPsuData)DATA.Poll($"{Module}.SCR{i}.DeviceData");
                if (ioScrData != null)
                {
                    if (ioScrData.IsResistanceOutOfRange)//SCR加热丝断开,后续可能增加条件判断
                        reason += $"SCR{i} resistance:{ioScrData.Resistance} is out of range\r\n ";
                }
            }

            if (reason.Length > 0)
            {
                // 某个加热器电阻超标
                _trigHeaterResOutOfRange.CLK = true;
                if (_trigHeaterResOutOfRange.Q)
                {
                    var alarmLevel = SC.SafeGetStringValue($"PM.{Module}.Heater.InProcessResistanceFailAlarmLevel", "Alarm");
                    if (alarmLevel == "Alarm")
                        Stop(reason);
                    else
                        EV.PostWarningLog(Module,reason);
                }
            }
        }

        private void PressureDifferenceDetection()
        {
            double pt1 = PmDevice.GetChamberPressure();

            double pt2 = PmDevice.GetForelinePressure();

            double d = Math.Abs(pt1 - pt2);
            if (d >= _pressureDifferenceUpperLimit)
                _trigPressureDifference.CLK = true;

            if (_trigPressureDifference.Q)
            {
                string reason = $"PT1={pt1}  PT2={pt2},Difference {d}  over  DifferenceMax={_pressureDifferenceUpperLimit} ";
                var alarmLevel = SC.SafeGetStringValue($"PM.{Module}.PT1PT2PressureDifferenceMaxAlarmLevel", "Alarm");
                if (alarmLevel == "Alarm")
                    Stop(reason);
                else
                    EV.PostWarningLog(Module, reason);
            }
        }

        #endregion

        public override Result Monitor()
        {
            if (!PmDevice.CheckEnableRunProcess(out string reason))
            {
                EV.PostAlarmLog(Module, reason);
                return Result.FAIL;
            }

            if (_isDryRun) // 空跑工艺
            {
                try
                {
                    DryRunProcess((int)RoutineStep.WaitProcess, $"Chamber:{Name}:WaitProcess", _delayTimeDryRun);
                }
                catch (RoutineBreakException)
                {
                    return Result.RUN;
                }
                catch (RoutineFaildException)
                {
                    return Result.FAIL;
                }

                Notify("End");

                return Result.DONE;
            }
            else
            {
                MonitorRecipeEndTime();

                MonitorRecipeRunInfo();

                MonitorHeaterResistance(); //加热丝断开检测

                PressureDifferenceDetection();//腔体和管道压力检测

                lock (_recipeLocker)
                {
                    try
                    {
                        switch (_state)
                        {
                            case RecipeRunningState.ExecStep:
                            {
                                PmDevice.ResetToleranceChecker();

                                //int stepTime = (int)PMDevice.RecipeRunningInfo.RecipeStepList[_currentStepNumber].StepTime * 1000;
                                if (ContinueAction != RecipeContinueMode.StepContinue)
                                {
                                    _curStepElpasedTimeBeforePaused = 0;
                                    _curStepElpasedTimeBeforePaused2 = 0;
                                }

                                ContinueAction = RecipeContinueMode.None;

                                if (PmDevice.RecipeRunningInfo.RecipeStepList[_currentStepNumber].IsLoopStartStep)
                                {
                                    CurStepTotalLoopCount = PmDevice.RecipeRunningInfo
                                        .RecipeStepList[_currentStepNumber].LoopCount;
                                    if (CurStepTotalLoopCount == 0)
                                    {
                                        CurrentLoopCount = 0;
                                    }
                                    else
                                    {
                                        CurrentLoopCount++;
                                    }
                                }

                                //stepTime = (int)(PMDevice.RecipeRunningInfo.RecipeStepList[_currentStepNumber].StepTime - _curStepElpasedTimeBeforePaused / 1000);
                                _stepTimer.Start(
                                    PmDevice.RecipeRunningInfo.RecipeStepList[_currentStepNumber].StepTime * 1000 -
                                    _curStepElpasedTimeBeforePaused);

                                if (!_stepTimer2.IsIdle())
                                {
                                    _curStepElpasedTimeBeforePaused2 += _stepTimer2.GetElapseTime();
                                    _stepTimer2.Stop();
                                }

                                _stepTimer2.Start(int.MaxValue);

                                Notify($"Running step {_currentStepNumber + 1} : {PmDevice.RecipeRunningInfo.RecipeStepList[_currentStepNumber].StepName}");

                                GemManager.Instance?.TriggerEvent($"{Module}RecipeStepStart", new string[] { $"{Module}.SelectedRecipeName", $"{Module}.RecipeStepNumber",$"{Module}.RecipeStepName" },
                                        new object[] { PmDevice.RecipeRunningInfo.RecipeName, _currentStepNumber + 1, 
                                        PmDevice.RecipeRunningInfo.RecipeStepList[_currentStepNumber].StepName });

                                if (PmDevice.RecipeRunningInfo.RecipeStepList[_currentStepNumber].IsDummyStep)
                                {
                                    _dummyStepCount++;
                                }

                                //执行工艺程序命令
                                foreach (var recipeCmd in PmDevice.RecipeRunningInfo.RecipeStepList[_currentStepNumber]
                                             .RecipeCommands.Keys)
                                {
                                    if (recipeCmd == "SusHeaterSetMode")
                                    {
                                        if (PmDevice.RecipeRunningInfo.RecipeStepList[_currentStepNumber]
                                                .RecipeCommands[recipeCmd] == "Jump")
                                            _isPSUHeaterJumpMode = true;
                                        else
                                            _isPSUHeaterJumpMode = false;

                                        continue;
                                    }

                                    if (recipeCmd == "WWHeaterSetMode")
                                    {
                                        if (PmDevice.RecipeRunningInfo.RecipeStepList[_currentStepNumber]
                                                .RecipeCommands[recipeCmd] == "Jump")
                                            _isSCRHeaterJumpMode = true;
                                        else
                                            _isSCRHeaterJumpMode = false;

                                        continue;
                                    }

                                    if (recipeCmd == "FlowSetMode")
                                    {
                                        if (PmDevice.RecipeRunningInfo.RecipeStepList[_currentStepNumber]
                                                .RecipeCommands[recipeCmd] == "Jump")
                                            _isMFCJumpMode = true;
                                        else
                                            _isMFCJumpMode = false;

                                        continue;
                                    }

                                    if (IsCmdSkip(recipeCmd)) // 不是注册的方法，需要跳过
                                        continue;

                                    if (!OP.CanDoOperation($"{Module}.{recipeCmd}", out reason,
                                            PmDevice.RecipeRunningInfo.RecipeStepList[_currentStepNumber]
                                                .RecipeCommands[recipeCmd]))
                                    {
                                        EV.PostAlarmLog(Module, $"Can not execute {recipeCmd}, {reason}");
                                        return Result.FAIL;
                                    }
                                    else
                                    {
                                        int time = (int)(PmDevice.RecipeRunningInfo.RecipeStepList[_currentStepNumber]
                                                             .StepTime * 1000 -
                                                         _curStepElpasedTimeBeforePaused); // (int)PMDevice.RecipeRunningInfo.RecipeStepList[_currentStepNumber].StepTime * 1000;

                                        if (recipeCmd.StartsWith("TC1") && _isPSUHeaterJumpMode)
                                        {
                                            time = 1;
                                        }

                                        if (recipeCmd.StartsWith("TC2") && _isSCRHeaterJumpMode)
                                        {
                                            time = 1;
                                        }

                                        if (recipeCmd.StartsWith("Mfc") && recipeCmd.EndsWith(".Ramp") &&
                                            !PmDevice.RecipeRunningInfo.RecipeStepList[_currentStepNumber].IsDummyStep)
                                        {
                                            if (_isMFCJumpMode)
                                            {
                                                time = 1;
                                            }
                                        }

                                        if ((recipeCmd == "TV.SetPressure"
                                             || recipeCmd == "PMServo.SetActualSpeed"
                                             || recipeCmd.StartsWith("Pressure") && recipeCmd.EndsWith(".Ramp")
                                             || recipeCmd.StartsWith("Mfc") && recipeCmd.EndsWith(".Ramp"))
                                            && !PmDevice.RecipeRunningInfo.RecipeStepList[_currentStepNumber]
                                                .IsDummyStep)
                                        {
                                            if (_currentStepNumber >= 1)
                                            {
                                                int previousStepNumber = _currentStepNumber - 1;

                                                if (PmDevice.RecipeRunningInfo.RecipeStepList[previousStepNumber]
                                                    .IsDummyStep)
                                                    previousStepNumber = _currentStepNumber - 2;

                                                if (PmDevice.RecipeRunningInfo.RecipeStepList[previousStepNumber]
                                                    .RecipeCommands.ContainsKey(recipeCmd))
                                                {
                                                    string previousValue = PmDevice.RecipeRunningInfo
                                                        .RecipeStepList[previousStepNumber].RecipeCommands[recipeCmd];
                                                    string currentValue = PmDevice.RecipeRunningInfo
                                                        .RecipeStepList[_currentStepNumber].RecipeCommands[recipeCmd];
                                                    if (previousValue == currentValue)
                                                    {
                                                        if (_lstSkipSteps.Count > 0 &&
                                                            _lstSkipSteps.Contains(
                                                                previousStepNumber)) //上一步是跳步过来的，这一步和上一步的值是相同的不用设置
                                                        {
                                                            continue;
                                                        }

                                                        time = 1;
                                                    }
                                                }
                                            }
                                        }

                                        OP.DoOperation($"{Module}.{recipeCmd}", out string reason1, time,
                                            PmDevice.RecipeRunningInfo.RecipeStepList[_currentStepNumber]
                                                .RecipeCommands[recipeCmd]);
                                    }
                                }

                                if (PmDevice.RecipeRunningInfo.RecipeStepList[_currentStepNumber].EndBy ==
                                    EnumEndByCondition.ByTime)
                                    _state = RecipeRunningState.TimeWait;
                                else
                                    _state = RecipeRunningState.ConditionWait;

                                //ResetChecker();

                                _dbCallback.RecipeStepStart(PmDevice.RecipeRunningInfo.InnerId.ToString(),
                                    _currentStepNumber,
                                    PmDevice.RecipeRunningInfo.RecipeStepList[_currentStepNumber].StepName,
                                    (float)PmDevice.RecipeRunningInfo.RecipeStepList[_currentStepNumber].StepTime);
                                _fdc.Start(PmDevice.RecipeRunningInfo.RecipeStepList[_currentStepNumber]
                                    .RecipeCommands);
                            }
                                break;

                            case RecipeRunningState.TimeWait:

                                if (IsPaused)
                                {
                                    _state = RecipeRunningState.Paused;
                                }

                                if (_stepTimer.IsTimeout())
                                {
                                    _state = RecipeRunningState.StepCompleted;
                                    Grow();
                                }

                                if (PmDevice.RecipeRunningInfo.NeedReloadRecipe)
                                {
                                    if (string.IsNullOrEmpty(PmDevice.RecipeRunningInfo.XmlRecipeToReload))
                                    {
                                        EV.PostWarningLog(Module, "Recipe is required to be reloaded but no new recipe is received");
                                        PmDevice.RecipeRunningInfo.NeedReloadRecipe = false;
                                    }
                                    else
                                    {   //重新加载后面的Recipe内容
                                        if (RecipeParser.ParseXmlString(PmDevice.RecipeRunningInfo.XmlRecipeToReload, Module, out var recipeHead, out var recipeSteps, out var reason1))
                                        {
                                            //记录高亮部分
                                            foreach (var item in PmDevice.RecipeRunningInfo.HighlightDic)
                                            {
                                                var s = recipeSteps.Find(i => i.StepUid == item.Key);
                                                if (s != null)
                                                {
                                                    s.HighLightList = item.Value;
                                                }
                                            }
                                            PmDevice.RecipeRunningInfo.RecipeStepList = recipeSteps;
                                            PmDevice.RecipeRunningInfo.TotalTime = CalcRecipeTime();
                                            PmDevice.RecipeRunningInfo.NeedReloadRecipe = false;
                                        }
                                        else
                                        {
                                            EV.PostWarningLog(Module, $"Reloading Recipe failed, {reason1}");
                                        }
                                    }

                                }

                                //ToleranceChecker();

                                SkipStepForHeat();

                                break;

                            case RecipeRunningState.ConditionWait:
                            {
                                if (_stepTimer.IsTimeout())
                                {
                                    _state = RecipeRunningState.StepCompleted;
                                    Grow();
                                }
                            }
                                break;

                            case RecipeRunningState.Paused:
                                PmDevice.PauseRecipe(out reason);
                                if (!_stepTimer.IsIdle())
                                {
                                    _curStepElpasedTimeBeforePaused += _stepTimer.GetElapseTime();
                                    _stepTimer.Stop();
                                }

                                switch (ContinueAction)
                                {
                                    case RecipeContinueMode.None:
                                        break;
                                    case RecipeContinueMode.WaferReturnAndJobStop:
                                        //Singleton<RouteManager>.Instance.CheckToPostMessage((int)RouteManager.MSG.StopJob);
                                        _state = RecipeRunningState.Error;
                                        break;
                                    case RecipeContinueMode.RecipeCompleted:
                                        _state = RecipeRunningState.RecipeCompleted;
                                        break;
                                    case RecipeContinueMode.StepContinue:
                                        _state = RecipeRunningState.ExecStep;
                                        break;
                                    case RecipeContinueMode.StepRestart:
                                        _state = RecipeRunningState.ExecStep;
                                        break;
                                    case RecipeContinueMode.RecipeRestart:
                                        _currentStepNumber = 0;
                                        _state = RecipeRunningState.ExecStep;
                                        break;
                                    case RecipeContinueMode.NextStep:
                                        _state = RecipeRunningState.StepCompleted;
                                        break;
                                }

                                break;

                            case RecipeRunningState.StepCompleted:
                            {
                                    //放在前面，stepnumber后面会被更新
                                 GemManager.Instance?.TriggerEvent($"{Module}RecipeStepComplete", 
                                     new string[] { $"{Module}.SelectedRecipeName", $"{Module}.RecipeStepNumber", $"{Module}.RecipeStepName" },
                                          new object[] { PmDevice.RecipeRunningInfo.RecipeName, _currentStepNumber + 1,PmDevice.RecipeRunningInfo.RecipeStepList[_currentStepNumber].StepName });

                                    _stepTimer2.Stop();

                                _dbCallback.RecipeStepEnd(PmDevice.RecipeRunningInfo.InnerId.ToString(),
                                    _currentStepNumber, _fdc.DataList);
                                _fdc.Stop();

                                if (PmDevice.RecipeRunningInfo.RecipeStepList[_currentStepNumber].IsLoopEndStep)
                                {
                                    //重新读取循环的设定次数
                                    for (int nn = _currentStepNumber; nn >= 0; nn--)
                                    {
                                        if (PmDevice.RecipeRunningInfo.RecipeStepList[nn].IsLoopStartStep)
                                        {
                                            CurStepTotalLoopCount =
                                                PmDevice.RecipeRunningInfo.RecipeStepList[nn].LoopCount;
                                            break;
                                        }
                                    }

                                    if (CurrentLoopCount >= CurStepTotalLoopCount)
                                    {
                                        CurrentLoopCount = CurStepTotalLoopCount = 0;
                                        _currentStepNumber++;
                                    }
                                    else
                                    {
                                        int n = _currentStepNumber - 1;
                                        int next = -1;
                                        while (n >= 0)
                                        {
                                            if (PmDevice.RecipeRunningInfo.RecipeStepList[n].IsLoopStartStep)
                                            {
                                                next = n;
                                                break;
                                            }

                                            n--;
                                        }

                                        if (next == -1)
                                            throw new Exception("Loop End control error");
                                        _currentStepNumber = next;
                                    }
                                }
                                else
                                {
                                    _currentStepNumber++;
                                }

                                if (_currentStepNumber >= PmDevice.RecipeRunningInfo.RecipeStepList.Count)
                                {
                                    _currentStepNumber = PmDevice.RecipeRunningInfo.RecipeStepList.Count - 1;
                                    _state = RecipeRunningState.RecipeCompleted;
                                }
                                else
                                {
                                    _state = RecipeRunningState.ExecStep;
                                }
                            }


                                break;

                            case RecipeRunningState.RecipeCompleted:
                                {

                                    //更新PM的Runtime
                                    if (!_hasRecordRunTime)
                                    {
                                        _hasRecordRunTime = true;
                                        RuntimeDataRecorder.UpdateElapseTimePM(Module,
                                            (int)_recipeTimer.GetElapseTime() / 1000);
                                    }

                                    _recipeTimer.Stop();
                                    Notify("Finished");
                                    GrowCheck();

                                    //手动工艺时LotID和SequenceID可能是string.Empty
                                    //补全路径
                                    string recipeID = PmDevice.RecipeRunningInfo.RecipeName + ".rcp";
                                    string squenceID = $"Sequence\\{PmDevice.GetWaferSequenceID()}.seq";

                                    GemManager.Instance.Equipment?.TriggerEvent("PMRecipeComplete", new string[] { "ChamberID", "RecipeID", "LotID", "SequenceID" }, new object[] { PmDevice.Module, recipeID, PmDevice.GetWaferLotID(), squenceID });

                                    return Result.DONE;
                                }

                            case RecipeRunningState.Error:
                            {
                                //更新PM的Runtime
                                if (!_hasRecordRunTime)
                                {
                                    _hasRecordRunTime = true;
                                        RuntimeDataRecorder.UpdateElapseTimePM(Module,
                                            (int)_recipeTimer.GetElapseTime() / 1000);
                                }
                                GrowCheck();
                                return Result.DONE;
                            }
                            default:
                                break;
                        }
                    }
                    catch (Exception ex)
                    {
                        PmDevice.SetHeaterStopRamp();
                        PmDevice.SetMfcStopRamp(PmDevice.GetMfcListByGroupName(MfcGroupName.All));
                        PmDevice.SetRotationStopRamp();

                        LOG.Write(ex);
                        return Result.FAIL;
                    }
                }

                return Result.RUN;
            }
        }

        

        private void MonitorRecipeRunInfo()
        {
            try
            {
                PmDevice.RecipeRunningInfo.StepNumber =
                    _currentStepNumber + 1 - _dummyStepCount; //CurStepNum start from 0, ignore dummy step
                PmDevice.RecipeRunningInfo.StepName =
                    PmDevice.RecipeRunningInfo.RecipeStepList[_currentStepNumber].StepName;
                PmDevice.RecipeRunningInfo.StepTime =
                    PmDevice.RecipeRunningInfo.RecipeStepList[_currentStepNumber].StepTime;
                PmDevice.RecipeRunningInfo.GrowthRate =
                    PmDevice.RecipeRunningInfo.RecipeStepList[_currentStepNumber].GrowthRate;
                PmDevice.RecipeRunningInfo.StepElapseTime = _stepTimer.IsIdle()
                    ? _curStepElpasedTimeBeforePaused / 1000
                    : (_stepTimer.GetElapseTime() + _curStepElpasedTimeBeforePaused) / 1000;

                PmDevice.RecipeRunningInfo.TotalElapseTime = CalcElapseRecipeTime();

                PmDevice.RecipeRunningInfo.StepElapseTime2 = _stepTimer2.IsIdle()
                    ? _curStepElpasedTimeBeforePaused2 / 1000
                    : (_stepTimer2.GetElapseTime() + _curStepElpasedTimeBeforePaused2) / 1000;
                PmDevice.RecipeRunningInfo.TotalElapseTime2 = _recipeTimer.GetElapseTime() / 1000;

                if (PmDevice.RecipeRunningInfo.RecipeStepList[_currentStepNumber].RecipeCommands
                    .ContainsKey("SHArH2Switch.SetValve"))
                {
                    PmDevice.RecipeRunningInfo.ArH2Switch = PmDevice.RecipeRunningInfo
                        .RecipeStepList[_currentStepNumber].RecipeCommands["SHArH2Switch.SetValve"];
                }

                if (PmDevice.RecipeRunningInfo.RecipeStepList[_currentStepNumber].RecipeCommands
                    .ContainsKey("N2Dilution.SetValve"))
                {
                    PmDevice.RecipeRunningInfo.N2FlowMode = PmDevice.RecipeRunningInfo
                        .RecipeStepList[_currentStepNumber].RecipeCommands["N2Dilution.SetValve"];
                }

                if (_currentStepIndex != PmDevice.RecipeRunningInfo.StepNumber)
                {
                    Notify(
                        $"Start Recipe: Step:{PmDevice.RecipeRunningInfo.StepNumber}  Name:{PmDevice.RecipeRunningInfo.StepName} ");
                    _currentStepIndex = PmDevice.RecipeRunningInfo.StepNumber;
                }
            }
            catch (Exception ex)
            {

            }
        }

        public override void Abort()
        {
            //Abort后设置加热比列为0
            PmDevice.TC1.SetRatio(0, 0, 0);
            PmDevice.TC2.SetRatio(0, 0, 0);

            //更新PM的Runtime
            PmDevice.RecipeRunningInfo.IsRoutineAbort = true;
            if (!_hasRecordRunTime)
            {
                _hasRecordRunTime = true;
                RuntimeDataRecorder.UpdateElapseTimePM(Module,(int)_recipeTimer.GetElapseTime() / 1000);
            }

            _state = RecipeRunningState.RecipeCompleted;

            _dbCallback.RecipeFailed(PmDevice.RecipeRunningInfo.InnerId.ToString());
            _fdc.Stop();

            PmDevice.AbortRunProcess(out string reason);

            PmDevice.RecipeRunningInfo.StepName = string.Empty;
            PmDevice.RecipeRunningInfo.StepNumber = 0;
            PmDevice.RecipeRunningInfo.StepTime = 0;
            PmDevice.RecipeRunningInfo.StepElapseTime = 0;
            PmDevice.RecipeRunningInfo.TotalTime = 0;
            PmDevice.RecipeRunningInfo.TotalElapseTime = 0;

            //手动工艺时LotID和SequenceID可能是string.Empty
            //补全路径
            string recipeID = PmDevice.RecipeRunningInfo.RecipeName + ".rcp";
            string squenceID = $"Sequence\\{PmDevice.GetWaferSequenceID()}.seq";

            GemManager.Instance.Equipment?.TriggerEvent("PMRecipeAbort", new string[] { "ChamberID", "RecipeID", "LotID", "SequenceID" }, new object[] { PmDevice.Module, recipeID, PmDevice.GetWaferLotID(), squenceID });

            //PMDevice.Rf.SetPowerOnOff(false, out _);
            //PMDevice.Microwave.SetPowerOnOff(false, out _);

            //PMDevice.GasLine1.SetFlow(out _, 0, 0);
            //PMDevice.GasLine2.SetFlow(out _, 0, 0);
            //PMDevice.GasLine3.SetFlow(out _, 0, 0);
        }

        public void ExitProcess()
        {
            if (_state == RecipeRunningState.RecipeCompleted)
            {
                _dbCallback.RecipeComplete(PmDevice.RecipeRunningInfo.InnerId.ToString());
                _fdc.Stop();
            }
            else
            {
                _dbCallback.RecipeFailed(PmDevice.RecipeRunningInfo.InnerId.ToString());
                _fdc.Stop();
            }

            _thread.Stop();
        }


        public void PauseRecipe()
        {
            if (_state != RecipeRunningState.TimeWait && _state != RecipeRunningState.ConditionWait)
                return;

            if (!IsPaused)
            {
                IsPaused = true;
                _pausedState = _state;
                _state = RecipeRunningState.Paused;
            }
        }


        public void SkipCurrentRecipeStep()
        {
            if (_state == RecipeRunningState.ConditionWait || _state == RecipeRunningState.TimeWait)
            {
                _lstSkipSteps.Add(_currentStepNumber);
                _state = RecipeRunningState.StepCompleted;
            }
        }


        protected int CalcRecipeTime()
        {
            double total = 0;
            for (int i = 0; i < PmDevice.RecipeRunningInfo.RecipeStepList.Count; i++)
            {
                if (!PmDevice.RecipeRunningInfo.RecipeStepList[i].IsDummyStep) // 不统计虚拟Step的时间
                {
                    total += PmDevice.RecipeRunningInfo.RecipeStepList[i].StepTime;
                }
            }

            return (int)total;
        }

        protected double CalcElapseRecipeTime()
        {
            double total = 0;
            for (int i = 0; i < _currentStepNumber; i++)
            {
                if (!PmDevice.RecipeRunningInfo.RecipeStepList[i].IsDummyStep) // 不统计虚拟Step的时间
                {
                    total += PmDevice.RecipeRunningInfo.RecipeStepList[i].StepTime;
                }
            }

            total += PmDevice.RecipeRunningInfo.StepElapseTime;

            return total;
        }

        protected void MonitorRecipeEndTime()
        {
            try
            {
                if (!_estimatedTimeCalcTimer.IsTimeout())
                    return;
                _estimatedTimeCalcTimer.Start(1000);

                EstimatedTotalLeftTime = 0;

                if (_state == RecipeRunningState.RecipeCompleted)
                    return;

                if (!(_currentStepNumber >= 0 &&
                      _currentStepNumber <= PmDevice.RecipeRunningInfo.RecipeStepList.Count - 1))
                    return;

                if (CurStepLeftTime > 0)
                {
                    EstimatedTotalLeftTime = CurStepLeftTime;
                }

                int nextBegin = _currentStepNumber;
                bool IsInLoop = false;
                int iNum1 = 0;
                int iNum2 = 0;
                //int j=i;
                for (int j = _currentStepNumber; j < PmDevice.RecipeRunningInfo.RecipeStepList.Count; j++)
                {
                    if (PmDevice.RecipeRunningInfo.RecipeStepList[j].IsLoopEndStep)
                    {
                        iNum2 = j;
                        IsInLoop = true;
                        break;
                    }
                    else if (j > _currentStepNumber && PmDevice.RecipeRunningInfo.RecipeStepList[j].IsLoopStartStep)
                    {
                        IsInLoop = false;
                        break;
                    }
                }

                if (IsInLoop)
                {
                    iNum1 = _currentStepNumber;
                    for (int j = _currentStepNumber; j >= 0; j--)
                    {
                        if (PmDevice.RecipeRunningInfo.RecipeStepList[j].IsLoopStartStep)
                        {
                            iNum1 = j;
                            break;
                        }
                    }

                    for (int j = _currentStepNumber + 1; j <= iNum2; j++)
                    {
                        EstimatedTotalLeftTime += PmDevice.RecipeRunningInfo.RecipeStepList[j].StepTime * 1000 *
                                                  (PmDevice.RecipeRunningInfo.RecipeStepList[iNum1].LoopCount -
                                                      CurrentLoopCount + 1);
                    }

                    for (int j = iNum1; j <= _currentStepNumber; j++)
                    {
                        EstimatedTotalLeftTime += PmDevice.RecipeRunningInfo.RecipeStepList[j].StepTime * 1000 *
                                                  (PmDevice.RecipeRunningInfo.RecipeStepList[iNum1].LoopCount -
                                                   CurrentLoopCount);
                    }

                    nextBegin = iNum2 + 1;
                }
                else
                {
                    nextBegin++;
                }

                for (int j = nextBegin; j < PmDevice.RecipeRunningInfo.RecipeStepList.Count; j++)
                {
                    if (PmDevice.RecipeRunningInfo.RecipeStepList[j].IsLoopStartStep)
                    {
                        //j=i;
                        iNum1 = j;
                        iNum2 = j + 1;
                        double lr1 = 0;
                        for (int m = j; m < PmDevice.RecipeRunningInfo.RecipeStepList.Count; m++)
                        {
                            lr1 += PmDevice.RecipeRunningInfo.RecipeStepList[m].StepTime * 1000;
                            if (PmDevice.RecipeRunningInfo.RecipeStepList[m].IsLoopEndStep)
                            {
                                iNum2 = m;
                                break;
                            }
                        }

                        EstimatedTotalLeftTime += lr1 * PmDevice.RecipeRunningInfo.RecipeStepList[iNum1].LoopCount;
                        j = iNum2;
                    }
                    else
                    {
                        EstimatedTotalLeftTime += PmDevice.RecipeRunningInfo.RecipeStepList[j].StepTime * 1000;
                    }
                }
            }
            catch (Exception ex)
            {
                LOG.Write(ex);
            }
        }

        public void SetContinue(string continueMode)
        {
            switch (continueMode)
            {
                case "Step continue":
                    ContinueAction = RecipeContinueMode.StepContinue;
                    break;
                case "Step restart":
                    ContinueAction = RecipeContinueMode.StepRestart;
                    break;
                case "Next step":
                    ContinueAction = RecipeContinueMode.NextStep;
                    break;
                case "Recipe restart":
                    ContinueAction = RecipeContinueMode.RecipeRestart;
                    break;
                case "Recipe complete":
                    ContinueAction = RecipeContinueMode.RecipeCompleted;
                    break;
                case "Wafer return and job stop":
                    ContinueAction = RecipeContinueMode.WaferReturnAndJobStop;
                    break;
            }

            IsPaused = false;

            PmDevice.ResetToleranceChecker();
        }

        public void DryRunProcess(int id, string stepName, int delayTime)
        {
            Tuple<bool, Result> ret = Delay(id, () =>
            {
                Notify($"Dry run process {delayTime} seconds");

                _stepSpan = new TimeSpan(0, 0, 0, (int)delayTime);
                _stepStartTime = DateTime.Now;
                _stepName = stepName;

                return true;
            }, delayTime * 1000);

            if (ret.Item1)
            {
                if (ret.Item2 == Result.FAIL)
                {
                    throw (new RoutineFaildException());
                }

                throw new RoutineBreakException();
            }
        }

        public void ToleranceChecker()
        {
            #region MFC

            for (int i = 0; i < _mfcGapChecker.Length; i++)
            {
                _mfcGapChecker[i].Monitor(PmDevice._mfcList[i].FeedBack, PmDevice._mfcList[i].SetPoint * (1 - 2 / 100),
                    PmDevice._mfcList[i].SetPoint * (1 + 2 / 100), 5 * 60);

                if (_mfcGapChecker[i].Trig)
                {
                    EV.PostWarningLog(Module, $"{PmDevice._mfcList[i].Name} flow gap > 2%, over 5 minites");
                }
            }

            for (int i = 0; i < _mfcTrig.Length; i++)
            {
                double gap = Convert.ToDouble(dbMfcGapResults[i].StdDev);
                double setPoint = Convert.ToDouble(dbMfcSetPointResults[i].Mean);

                _mfcTrig[i].CLK = gap / setPoint > 1 / 100;

                if (_mfcTrig[i].Q)
                {
                    EV.PostWarningLog(Module,
                        $"{PmDevice._mfcList[i].Name} flow standard deviation > 1%, for 5 minites");
                }
            }

            #endregion

            #region PC

            for (int i = 0; i < _pcGapChecker.Length; i++)
            {
                _pcGapChecker[i].Monitor(PmDevice._pcList[i].FeedBack, PmDevice._pcList[i].SetPoint - 60,
                    PmDevice._pcList[i].SetPoint + 60, 5 * 60);

                if (_pcGapChecker[i].Trig)
                {
                    EV.PostWarningLog(Module, $"{PmDevice._pcList[i].Name} flow gap > 60 mbar over 5 minites");
                }
            }

            for (int i = 0; i < _pcTrig.Length; i++)
            {
                double gap = Convert.ToDouble(dbPcGapResults[i].StdDev);
                //double setPoint = Convert.ToDouble(dbPcSetPointResults[i].StdDev);

                _pcTrig[i].CLK = gap > 10;

                if (_pcTrig[i].Q)
                {
                    EV.PostWarningLog(Module,
                        $"{PmDevice._mfcList[i].Name} flow standard deviation > 1%, for 5 minites");
                }
            }

            if (_rampCalcTimer.IsTimeout())
            {
                for (int i = 0; i < _pcTrig2.Length; i++)
                {
                    _pcTrig2[i].CLK = Math.Abs(PmDevice._pcList[i].FeedBack - _pressurePrevious[i]) > 90;

                    if (_pcTrig2[i].Q)
                    {
                        EV.PostWarningLog(Module, $"{PmDevice._mfcList[i].Name} pressure ramp > 90 mbar");
                    }

                    _pressurePrevious[i] = PmDevice._pcList[i].FeedBack;
                }

                _rampCalcTimer.Start(1000);
            }

            #endregion

        }

        public void ResetChecker()
        {
            #region MFC

            for (int i = 0; i < _mfcGapChecker.Length; i++)
            {
                _mfcGapChecker[i].Reset(5 * 60);
            }

            for (int i = 0; i < _mfcTrig.Length; i++)
            {
                _mfcTrig[i].RST = true;
            }

            #endregion

            #region PC

            for (int i = 0; i < _pcGapChecker.Length; i++)
            {
                _pcGapChecker[i].Reset(5 * 60);
            }

            for (int i = 0; i < _pcTrig.Length; i++)
            {
                _pcTrig[i].RST = true;
            }

            #endregion
        }

        public void SkipStepForHeat()
        {
            int stepCount = PmDevice.RecipeRunningInfo.RecipeStepList.Count;

            if (_currentStepNumber + 1 < stepCount)
            {
                var currentStep = PmDevice.RecipeRunningInfo.RecipeStepList[_currentStepNumber];
                var nextStep = PmDevice.RecipeRunningInfo.RecipeStepList[_currentStepNumber + 1];

                if (currentStep.IsDummyStep)
                    return;

                if (nextStep.IsDummyStep)
                    nextStep = PmDevice.RecipeRunningInfo.RecipeStepList[_currentStepNumber + 2];

                if (!currentStep.RecipeCommands.ContainsKey("TC1.SetHeaterMode") ||
                    !nextStep.RecipeCommands.ContainsKey("TC1.SetHeaterMode"))
                    return;

                var currentPSUMode = (HeatStrategy)Enum.Parse(typeof(HeatStrategy),
                    currentStep.RecipeCommands["TC1.SetHeaterMode"]);
                var nextPSUMode = (HeatStrategy)Enum.Parse(typeof(HeatStrategy),
                    nextStep.RecipeCommands["TC1.SetHeaterMode"]);

                if (currentPSUMode == HeatStrategy.Power && nextPSUMode != HeatStrategy.Power)
                {
                    float PSUL2InputTemp =
                        (float)DATA.Poll(
                            $"{Module}.TC1.L2InputTempSetPoint"); // (float)DATA.Poll($"{Module}.TC1.L1PVFeedBack"); 测试用

                    float nextL2Temp = Convert.ToSingle(nextStep.RecipeCommands["TC1.SetL2TargetSP"]);

                    if (nextL2Temp > PSUL2InputTemp && nextL2Temp - PSUL2InputTemp < Math.Abs(_tempOffset))
                    {
                        SkipCurrentRecipeStep();

                        EV.PostInfoLog(Module,
                            $"Current PSU middle temperature is {PSUL2InputTemp}℃ and Power mode, next step PSU middle temperature setpoint is {nextL2Temp}℃ and {nextPSUMode} mode." +
                            $" TempOffset is {_tempOffset}℃. Need jump step!");

                    }
                }
            }
        }

        public void Grow()
        {
            try
            {
                var wi = WaferManager.Instance.GetWafer(ModuleHelper.Converter(Module), 0);
                OP.DoOperation($"HistoryCoatingManager.Grow",
                    new object[] {
                (wi.TrayInnerID).ToString(),
                PmDevice.RecipeRunningInfo.GrowthRate,
                PmDevice.RecipeRunningInfo.StepTime ,
                Module });
            }
            catch (Exception ex)
            {

                LOG.Error(ex.Message, ex);
            }
            
        }
        public void GrowCheck()
        {
            try
            {
                var wi = WaferManager.Instance.GetWafer(ModuleHelper.Converter(Module), 0);
                OP.DoOperation($"HistoryCoatingManager.GrowCheck",
                    new object[] {
                (wi.WaferInnerID).ToString(),
                Module });
            }
            catch (Exception ex)
            {

                LOG.Error(ex.Message, ex);
            }

        }
    }
}