/********************************************************************\ Alarm handler of the Evidence Control System - Checks periodically if all required servers are up - Listens to the 'Message' service of each server and generates new service for each observed server indicating the maximum Severity in the past. - Maximum severity may be reset by a command 'Alarm/ResetAlarm' for a server. - A text describing the current state of all servers is published as DIM service. The states are described in LevelStr[]. - A master alarm (indicating most severe of individual alarms) is published. A mutex is used because UpdateAlarmSummary() may be called from DIM handler thread and from main thread. Oliver Grimm, June 2010 \********************************************************************/ #define SERVER_NAME "Alarm" #include "Evidence.h" #include using namespace std; const char* LevelStr[] = {"OK", "WARN", "ERROR", "FATAL", "UNAVAILABLE"}; // // Data handling class // class AlarmHandler: public DimClient, public EvidenceServer { DimCommand *Command; DimService *Summary, *Master; char *AlarmText; int MasterAlarm; pthread_mutex_t Mutex; void infoHandler(); void commandHandler(); public: AlarmHandler(); ~AlarmHandler(); struct Item { string Server; string Email; DimStampedInfo *Subscription; DimService *AlarmLevel; int WarnedLevel; int Level; }; vector List; void UpdateAlarmSummary(); }; // Constructor AlarmHandler::AlarmHandler(): EvidenceServer(SERVER_NAME) { struct Item N; static int InitLevel = -1; // static for DIM service below // Initialise MasterAlarm = 0; AlarmText = NULL; if (pthread_mutex_init(&Mutex, NULL) != 0) { Message(FATAL, "pthread_mutex_init failed"); } // Handling of servies will only start after start() autoStartOff(); // Create DIM services Summary = new DimService(SERVER_NAME"/Summary", (char *) "not yet available"); Master = new DimService(SERVER_NAME"/MasterAlarm", MasterAlarm); // Get DIM servers to observe vector Token = Tokenize(GetConfig("servers")); for (int i=0; i A = Tokenize(Token[i], ":"); N.Server = A[0]; if (A.size() == 2) N.Email = A[1]; else N.Email = string(); // DIS_DNS has no Message service if (N.Server == "DIS_DNS") N.Subscription = NULL; else N.Subscription = new DimStampedInfo((N.Server+"/Message").c_str(), NO_LINK, this); // Alarm service for server (reference to variable will be updated in UpdateAlarmSummary()) N.WarnedLevel = 0; N.Level = -1; N.AlarmLevel = new DimService((N.Server+"/AlarmLevel").c_str(), InitLevel); List.push_back(N); } // Provide command to reset Level Command = new DimCommand("Alarm/ResetAlarm", (char *) "C", this); // List set up, can start handling start(SERVER_NAME); } // Destructor AlarmHandler::~AlarmHandler() { delete Command; for (int i=0; igetInt() > List[i].Level) List[i].Level = getInfo()->getInt(); } UpdateAlarmSummary(); } // Reset alarm level of given server void AlarmHandler::commandHandler() { DimCommand *C = getCommand(); // Check for valid command parameter if (C != Command) return; if (C->getSize() == 0) return; if (*((char *) C->getData() + C->getSize() - 1) != '\0') return; // Reset alarm level and publish/log action for (int i=0; igetString()) { Message(INFO, "Alarm level of server %s reset by %s (ID %d)", C->getString(), getClientName(), getClientId()); List[i].Level = 0; List[i].WarnedLevel = 0; } UpdateAlarmSummary(); } // Update alarm status summary void AlarmHandler::UpdateAlarmSummary() { ostringstream Buf; int Alarm, Ret; // Lock because access can be from main thread and DIM handler thread if ((Ret = pthread_mutex_lock(&Mutex)) != 0) { Message(FATAL, "pthread_mutex_lock() failed (%s)", strerror(Ret)); } for (int i=0; i=0 && List[i].Level<=4 ? LevelStr[List[i].Level] : "unknown"); Buf << " (" << List[i].Level << ")" << endl; // Adjust master alarm and update server alarm level if (List[i].Level > Alarm) Alarm = List[i].Level; List[i].AlarmLevel->updateService(List[i].Level); // Check if alarm level raised, then send alarm message once if (List[i].WarnedLevel < List[i].Level && !List[i].Email.empty()) { List[i].WarnedLevel = List[i].Level; // Prepare email message char *Text; time_t Time = time(NULL); if (asprintf(&Text, "echo \"Server alarm level '%s' at %s\"|" "mail -s \"Evidence Alarm for '%s'\" %s", List[i].Level>=0 && List[i].Level<=4 ? LevelStr[List[i].Level] : "unknown", ctime(&Time), List[i].Server.c_str(), List[i].Email.c_str()) != -1) { system(Text); // Return value depending on OS free(Text); } else Message(ERROR, "Could not send alarm email, asprintf() failed"); } } // Update master alarm services MasterAlarm = Alarm; Master->updateService(); // Update alarm description (DIM requires variables to be valid until update) char *Tmp = new char[Buf.str().size()+1]; strcpy(Tmp, Buf.str().c_str()); Summary->updateService(Tmp); delete[] AlarmText; AlarmText = Tmp; // Unlock if ((Ret = pthread_mutex_unlock(&Mutex)) != 0) { Message(FATAL, "pthread_mutex_unlock() failed (%s)", strerror(Ret)); } } // // Main program // int main() { DimBrowser Browser; char *Server, *Node; bool Exist; // Static declaration ensures calling of destructor by exit() static AlarmHandler Alarm; // Check periodically if servers are up while(!Alarm.ExitRequest) { for (int i=0; i