Index: /trunk/FACT++/src/smartfact.cc =================================================================== --- /trunk/FACT++/src/smartfact.cc (revision 14033) +++ /trunk/FACT++/src/smartfact.cc (revision 14034) @@ -374,6 +374,6 @@ vector fFadControlDrsRuns; - float fFtmControlTriggerRateCam; deque fFtmControlTriggerRateHist; + bool fFtmControlNewRunStarted; float fFtmPatchThresholdMed; @@ -387,4 +387,7 @@ uint8_t fRateScanBoard; deque fRateScanDataHist[41]; + + set fErrorList; + deque fErrorHist; Sun fSun; @@ -397,6 +400,4 @@ // --------------------------- File header ---------------------------- - - int fHasError; Time fAudioTime; @@ -438,5 +439,5 @@ DimDescribedState fDimRateControl; DimDescribedState fDimRateScan; - DimDescribedState fDimChatServer; + DimDescribedState fDimChat; DimDescribedState fDimSkypeClient; @@ -605,4 +606,19 @@ } + void HandleFscControlStateChange() + { + const int32_t &last = fDimFscControl.last.second; + const int32_t &state = fDimFscControl.state(); + + if (last==DimState::kOffline || state==DimState::kOffline) + return; + + if (last=-2) SetAudio("losticks"); + + if (d.GetQoS()==MCP::State::kTakingData) + { + fMcpConfigurationRunStart = Time(); + SetAudio("losticks"); + } fMcpConfigurationState = d.GetQoS(); @@ -627,7 +651,4 @@ fMcpConfigurationMaxEvents = d.Get(8); fMcpConfigurationName = d.Ptr(16); - - if (d.GetQoS()==MCP::State::kTakingData) - fMcpConfigurationRunStart = Time(); return GetCurrentState(); @@ -779,7 +800,7 @@ out << setprecision(5); out << fDriveControlPointingZd << '\n'; - out << az << '\n'; - - ofstream(fPath+"/drive-pointing.data") << out.str(); + out << az << '\t' << fDriveControlPointingAz << '\n'; + + ofstream(fPath+"/pointing.data") << out.str(); return GetCurrentState(); @@ -1167,9 +1188,13 @@ return GetCurrentState(); + const double crate = d.Get(20); // Camera rate + // New run started - if (d.Get(20)<0) + if (crate<0) + { + fFtmControlNewRunStarted = true; return GetCurrentState(); - - fFtmControlTriggerRateCam = d.Get(20); + } + fFtmControlNewRunStarted = false; const float *brates = d.Ptr(24); // Board rate @@ -1177,5 +1202,5 @@ // Store a history of the last 60 entries - fFtmControlTriggerRateHist.push_back(fFtmControlTriggerRateCam); + fFtmControlTriggerRateHist.push_back(crate); if (fFtmControlTriggerRateHist.size()>60) fFtmControlTriggerRateHist.pop_front(); @@ -1193,5 +1218,5 @@ out << setprecision(3); out << d.GetJavaDate() << '\n'; - out << HTML::kWhite << '\t' << fFtmControlTriggerRateCam << '\n'; + out << HTML::kWhite << '\t' << crate << '\n'; ofstream(fPath+"/trigger.data") << out.str(); @@ -1571,5 +1596,5 @@ Out() << fDimTngWeather << endl; Out() << fDimRateScan << endl; - Out() << fDimChatServer << endl; + Out() << fDimChat << endl; Out() << fDimSkypeClient << endl; @@ -1599,5 +1624,5 @@ //return msg.str(); - if (rc.index<1) + if (rc.index<0) return HTML::kWhite + "\t—\n"; @@ -1609,4 +1634,29 @@ return col + '\t' + rc.name + '\n'; + } + + bool SetError(bool b, const string &err) + { + if (!b) + { + fErrorList.erase(err); + return 0; + } + + const bool isnew = fErrorList.insert(err).second; + + if (isnew) + { + ostringstream msg; + msg << "
" << Time().GetAsStr("%m-%d %H:%M") << "
<->" << err << ""; + if (find(fErrorHist.begin(), fErrorHist.end(), msg.str())==fErrorHist.end()) + { + fErrorHist.push_front(msg.str()); + if (fErrorHist.size()>80) + fErrorHist.pop_back(); + } + } + + return isnew; } @@ -1632,52 +1682,66 @@ fDimMcp.state()==MCP::State::kTakingData; - ostringstream msg; - - if (fHasError==2) - msg << "SmartFACT backend initializing." << endl; - if (!fDimDNS.online()) - msg << "DIM network not available.
"; - if (fDimDriveControl.state()>0xff && data_taking) - msg << "Drive in ERROR state during data-taking
"; - if (fDimBiasControl.state()"; - if (fDimBiasControl.state()==BIAS::State::kOverCurrent) - msg << "BIAS channels in OverCurrent
"; - if (fDimBiasControl.state()==BIAS::State::kNotReferenced) - msg << "BIAS voltage not at reference
"; - if (fFeedbackCalibration.size()>0) - { - if (fBiasControlCurrentMed>75) - msg << "Median current exceeds 75µA/pix
"; - if (fBiasControlCurrentMax>90) - msg << "Maximum current exceeds 90µA/pix
"; - } - if (fMagicWeatherHist[kHum].size()>0 && fMagicWeatherHist[kHum].back()>98 && data_taking) - msg << "Outside humidity exceeds 98% during data-taking
"; - if (fMagicWeatherHist[kGusts].size()>0 && fMagicWeatherHist[kGusts].back()>98 && fDimDriveControl.state()==Drive::State::kTracking) - msg << "Wind gusts exceed 50km/h during tracking
"; - if (fFscControlTemperatureHist.size()>0 && fFscControlTemperatureHist.back()>7) - msg << "Sensor temperature exceeds outside temperature by more than 7°C
"; - if (fFtmControlTriggerRateCam<0.01 && data_taking) - msg << "Trigger rate below 10mHz during data taking
"; - if (fFscControlHumidityAvg>60) - msg << "Average camera humidity exceed 60%
"; - if (!fDimControl.online()) - msg << "dimctrl offline
"; - - if (fDriveControlMoonDist>155) - msg << "Moon within the field-of-view of the cones
"; - if (fDriveControlMoonDist>=0 && fDriveControlMoonDist<3) - msg << "Moon within the field-of-view of the camera
"; - - if (fDimTimeCheck.state()==2) - msg << "Warning NTP time difference of drive PC exceeds 1s.
"; - - - if (fDimFeedback.state()!=Feedback::State::kCalibrating && - fDimBiasControl.state()==BIAS::State::kVoltageOn && - fBiasControlVoltageMed>3 && - fFeedbackCalibration.size()==0) - msg << "bias voltage switched on, but bias crate not calibrated."; + const bool data_run = + fMcpConfigurationName=="data" || + fMcpConfigurationName=="data-rt"; + + const bool haderr = fErrorList.size()>0; + + bool newerr = false; + + newerr |= SetError(!fDimDNS.online(), + "<#darkred>DIM network not available"); + newerr |= SetError(!fDimControl.online(), + "dimctrl offline"); + + //newerr |= SetError(fDimDriveControl.state()==Drive::State::kLocked, + // "<#darkred>Drive in LOCKED state, drive was automatically parked"); + + newerr |= SetError(fDimDriveControl.state()>0xff && data_taking && data_run, + "Drive in ERROR state during data-taking"); + newerr |= SetError(fDriveControlMoonDist>155, + "Moon within the field-of-view of the cones"); + newerr |= SetError(fDriveControlMoonDist>=0 && fDriveControlMoonDist<3, + "Moon within the field-of-view of the camera"); + + newerr |= SetError(fDimBiasControl.state()0 && fBiasControlCurrentMed>80, + "Median current exceeds 80µA/pix"); + newerr |= SetError(fFeedbackCalibration.size()>0 && fBiasControlCurrentMax>100, + "Maximum current exceeds 100µA/pix"); + + newerr |= SetError(fFscControlHumidityAvg>60, + "Average camera humidity exceed 60%"); + + newerr |= SetError(fMagicWeatherHist[kHum].size()>0 && fMagicWeatherHist[kHum].back()>98 && data_taking, + "Outside humidity exceeds 98% during data-taking"); + newerr |= SetError(fMagicWeatherHist[kGusts].size()>0 && fMagicWeatherHist[kGusts].back()>98 && fDimDriveControl.state()==Drive::State::kTracking, + "Wind gusts exceed 50km/h during tracking"); + + newerr |= SetError(fFscControlTemperatureHist.size()>0 && fFscControlTemperatureHist.back()>8, + "Sensor temperature exceeds outside temperature by more than 8°C"); + + if (fFtmControlTriggerRateHist.size()>0 && !fFtmControlNewRunStarted) + { + newerr |= SetError(fFtmControlTriggerRateHist.size()>0 && fFtmControlTriggerRateHist.back()<0.01 && fDimMcp.state()==MCP::State::kTakingData, + "Trigger rate below 10mHz during data taking"); + } + + newerr |= SetError(fDimTimeCheck.state()==1, + "Warning NTP time difference of drive PC exceeds 1s"); + newerr |= SetError(fDimTimeCheck.state()<1, + "Warning timecheck not running"); + + newerr |= SetError(fDimFeedback.state()!=Feedback::State::kCalibrating && + fDimBiasControl.state()==BIAS::State::kVoltageOn && + fBiasControlVoltageMed>3 && + fFeedbackCalibration.size()==0, + "Bias voltage switched on, but bias crate not calibrated"); // FTM in Connected instead of Idle --> power cyclen @@ -1696,5 +1760,5 @@ Out() << fDimMagicWeather << endl; Out() << fDimRateScan << endl; - Out() << fDimChatServer << endl; + Out() << fDimChat << endl; */ @@ -1703,28 +1767,38 @@ // -------------------------------------------------------------- - - const bool haserror = msg.str().size()>0; - ostringstream out; - out << Header(now) << '\t' << haserror << '\t' << (fDimControl.state()>-3) << '\n'; + + if (newerr) + { + SetAudio("sound_7"); + + out << now.JavaDate() << '\n'; + out << HTML::kWhite << '\t'; + for (auto it=fErrorHist.begin(); it!=fErrorHist.end(); it++) + out << *it << "
"; + out << '\n'; + + ofstream(fPath+"/errorhist.data") << out.str(); + } + + out.str(""); + out << Header(now) << '\t' << (fErrorList.size()>0) << '\t' << (fDimControl.state()>-3) << '\n'; out << setprecision(3); - out << HTML::kWhite << '\t' << msg.str() << '\n'; - - if (haserror || fHasError) + out << HTML::kWhite << '\t'; + for (auto it=fErrorList.begin(); it!=fErrorList.end(); it++) + out << *it << "
"; + out << '\n'; + + if (haderr || fErrorList.size()>0) ofstream(fPath+"/error.data") << out.str(); - fHasError = haserror; - - if (!fDimDNS.online()) - return kStateDimNetworkNA; - // ============================================================== out.str(""); - out << Header(now) << '\t' << fHasError << '\t' << (fDimControl.state()>-3) << '\n'; + out << Header(now) << '\t' << (fErrorList.size()>0) << '\t' << (fDimControl.state()>-3) << '\n'; out << setprecision(3); // -------------- System status -------------- - if (fDimMcp.state()>=MCP::State::kIdle) // Idle + if (fDimDNS.online() && fDimMcp.state()>=MCP::State::kIdle) // Idle { string col = HTML::kBlue; @@ -1843,5 +1917,5 @@ // ------------------ Drive ----------------- - if (fDimDriveControl.state()>=Drive::State::kArmed) // Armed, Moving, Tracking + if (fDimDNS.online() && fDimDriveControl.state()>=Drive::State::kArmed) // Armed, Moving, Tracking { const double dev = fDriveControlTrackingDevHist.size()>0 ? fDriveControlTrackingDevHist.back() : 0; @@ -1859,4 +1933,6 @@ col = HTML::kRed; } + if (rc.index==0x100) + col = HTML::kRed; out << col << '\t'; @@ -1896,5 +1972,5 @@ // ------------------- FSC ------------------ - if (fDimFscControl.state()>FSC::State::kDisconnected && fFscControlTemperatureHist.size()>0) + if (fDimDNS.online() && fDimFscControl.state()>FSC::State::kDisconnected && fFscControlTemperatureHist.size()>0) { out << HTML::kGreen << '\t' << fFscControlTemperatureHist.back() << '\n'; @@ -1904,5 +1980,5 @@ // --------------- MagicWeather ------------- - if (fDimMagicWeather.state()==MagicWeather::State::kReceiving && fMagicWeatherHist[kWeatherBegin].size()>0) + if (fDimDNS.online() && fDimMagicWeather.state()==MagicWeather::State::kReceiving && fMagicWeatherHist[kWeatherBegin].size()>0) { /* @@ -1933,13 +2009,17 @@ // --------------- FtmControl ------------- - if (fDimFtmControl.state()==FTM::State::kTriggerOn) + if (fDimDNS.online() && fDimFtmControl.state()==FTM::State::kTriggerOn) { string col = HTML::kGreen; - if (fFtmControlTriggerRateCam<15) - col = HTML::kYellow; - if (fFtmControlTriggerRateCam>100) - col = HTML::kRed; - - out << col << '\t' << fFtmControlTriggerRateCam << " Hz"; + if (fFtmControlTriggerRateHist.size()>0 && !fFtmControlNewRunStarted) + { + if (fFtmControlTriggerRateHist.back()<15) + col = HTML::kYellow; + if (fFtmControlTriggerRateHist.back()>100) + col = HTML::kRed; + + out << col << '\t' << fFtmControlTriggerRateHist.back() << " Hz"; + } + if (fDimBiasControl.state()==BIAS::State::kVoltageOn) out << " (" << fFtmPatchThresholdMed << ')'; @@ -1950,8 +2030,9 @@ // --------------- BiasControl ------------- - if (fDimBiasControl.state()==BIAS::State::kRamping || - fDimBiasControl.state()==BIAS::State::kOverCurrent || - fDimBiasControl.state()==BIAS::State::kVoltageOn || - fDimBiasControl.state()==BIAS::State::kVoltageOff) + if (fDimDNS.online() && + (fDimBiasControl.state()==BIAS::State::kRamping || + fDimBiasControl.state()==BIAS::State::kOverCurrent || + fDimBiasControl.state()==BIAS::State::kVoltageOn || + fDimBiasControl.state()==BIAS::State::kVoltageOff)) { const bool off = fDimBiasControl.state()==BIAS::State::kVoltageOff; @@ -1959,7 +2040,7 @@ string col = fBiasControlVoltageMed>3?HTML::kGreen:HTML::kWhite; - if (fBiasControlCurrentMax>65) + if (fBiasControlCurrentMed>60 || fBiasControlCurrentMax>80) col = HTML::kYellow; - if (fBiasControlCurrentMax>80) + if (fBiasControlCurrentMed>70 || fBiasControlCurrentMax>90) col = HTML::kRed; @@ -2015,5 +2096,5 @@ out.str(""); - out << Header(now) << '\t' << fHasError << '\t' << (fDimControl.state()>-3) << '\n'; + out << Header(now) << '\t' << (fErrorList.size()>0) << '\t' << (fDimControl.state()>-3) << '\n'; if (!fDimDNS.online()) @@ -2040,5 +2121,5 @@ out << GetStateHtml(fDimTngWeather, 2); out << GetStateHtml(fDimRateScan, 4); - out << GetStateHtml(fDimChatServer, 1); + out << GetStateHtml(fDimChat, 0); out << GetStateHtml(fDimSkypeClient, 1); @@ -2048,5 +2129,5 @@ ofstream(fPath+"/status.data") << out.str(); - return kStateRunning; + return fDimDNS.online() ? kStateRunning : kStateDimNetworkNA; } @@ -2056,5 +2137,5 @@ fPath("www/smartfact/data"), fControlScriptDepth(0), - fMcpConfigurationState(-256), + fMcpConfigurationState(DimState::kOffline), fMcpConfigurationMaxTime(0), fMcpConfigurationMaxEvents(0), @@ -2067,8 +2148,6 @@ fFadControlNumEvents(0), fFadControlDrsRuns(3), - fFtmControlTriggerRateCam(0), fRateScanDataId(0), fRateScanBoard(0), - fHasError(2), // --- fDimMcp ("MCP"), @@ -2085,5 +2164,5 @@ fDimRateControl ("RATE_CONTROL"), fDimRateScan ("RATE_SCAN"), - fDimChatServer ("CHAT_SERVER"), + fDimChat ("CHAT"), fDimSkypeClient ("SKYPE_CLIENT") { @@ -2103,7 +2182,8 @@ fDimRateControl.Subscribe(*this); fDimRateScan.Subscribe(*this); - fDimChatServer.Subscribe(*this); + fDimChat.Subscribe(*this); fDimSkypeClient.Subscribe(*this); + fDimFscControl.SetCallback(bind(&StateMachineSmartFACT::HandleFscControlStateChange, this)); fDimControl.SetCallback(bind(&StateMachineSmartFACT::HandleControlStateChange, this, placeholders::_1)); fDimControl.AddCallback("dotest.dim", bind(&StateMachineSmartFACT::HandleDoTest, this, placeholders::_1)); @@ -2189,4 +2269,10 @@ fPath = conf.Get("path"); + + ostringstream out; + out << Time().JavaDate() << '\n'; + + ofstream(fPath+"/errorhist.data") << out.str(); + ofstream(fPath+"/error.data") << out.str(); return -1;