source: Evidence/Alarm.cc@ 7462

/********************************************************************\

  Alarm handler of the Evidence Control System

  - Checks periodically if all required servers are up
  - Listens to the 'Message' service of each server and generates new service for
    each observed server indicating the maximum Severity in the past.
  - Maximum severity may be reset by a command 'Alarm/ResetAlarm' for a server.
  - A text describing the current state of all servers is published as DIM service.
    The states are described in LevelStr[].
  - A master alarm (indicating most severe of individual alarms) is published. 
    
  A mutex is used because UpdateAlarmSummary() may be called from DIM handler thread and
  from main thread.

  Oliver Grimm, June 2010

\********************************************************************/

#define SERVER_NAME "Alarm"
#include "Evidence.h"

#include <sstream>

using namespace std;

const char* LevelStr[] = {"OK", "WARN", "ERROR", "FATAL", "UNAVAILABLE"};

//
// Class declaration
//
class AlarmHandler: public DimClient, public EvidenceServer {
    
        DimCommand *Command;
        DimService *Summary, *Master;
        char *AlarmText;
        int MasterAlarm;

    void infoHandler();
        void commandHandler();

  public:
    AlarmHandler();
    ~AlarmHandler();

        struct Item {
          string Server;
          string Email;
          DimStampedInfo *Subscription;
          DimService *AlarmLevel;
          int WarnedLevel;
          int Level;
        };
        vector<struct Item> List;

        void UpdateAlarmSummary();
}; 

// Constructor
AlarmHandler::AlarmHandler(): EvidenceServer(SERVER_NAME) {

  struct Item N;
  static int InitLevel = -1; // static for DIM service below

  // Initialise
  MasterAlarm = 0;
  AlarmText = NULL;
  
  // Handling of servies will only start after start()
  autoStartOff();

  // Create DIM services
  Summary = new DimService(SERVER_NAME"/Summary", (char *) "not yet available");
  Master = new DimService(SERVER_NAME"/MasterAlarm", MasterAlarm);

  // Get DIM servers to observe
  vector<string> Token = Tokenize(GetConfig("servers"));

  for (int i=0; i<Token.size(); i++) {
        // Extract server name and email
        vector<string> A = Tokenize(Token[i], ":");
        N.Server = A[0];
        if (A.size() == 2) N.Email = A[1];
        else N.Email = string();

        // DIS_DNS has no Message service
        if (N.Server == "DIS_DNS") N.Subscription = NULL;
        else N.Subscription = new DimStampedInfo((N.Server+"/Message").c_str(), NO_LINK, this);

        // Alarm service for server (reference to variable will be updated in UpdateAlarmSummary())
        N.WarnedLevel = 0;
        N.Level = -1;
        N.AlarmLevel = new DimService((N.Server+"/AlarmLevel").c_str(), InitLevel);

        List.push_back(N);
  }

  // Provide command to reset Level   
  Command = new DimCommand("ResetAlarm", (char *) "C", this);
  
  // List set up, can start handling
  start(SERVER_NAME);
}


// Destructor
AlarmHandler::~AlarmHandler() {

  delete Command;

  for (int i=0; i<List.size(); i++) {
    delete List[i].Subscription;
    delete List[i].AlarmLevel;
  }     
  delete Master;
  delete Summary;
  delete[] AlarmText;
}


// Print messages of status changes to screen and update status string
void AlarmHandler::infoHandler() {

  // Identify status service
  for (int i=0; i<List.size(); i++) if (getInfo() == List[i].Subscription) {
        // Update level: unavailable or current severity of status (safely extracted)  
        if (!ServiceOK(getInfo())) List[i].Level = 4;
        else {
          int Severity = atoi(ToString(getInfo()->getFormat(), getInfo()->getData(), getInfo()->getSize()).c_str());
          if (Severity > List[i].Level) List[i].Level = Severity;
        }
  }

  UpdateAlarmSummary();
}


// Reset alarm level of given server
void AlarmHandler::commandHandler() {

  // Safety check
  string Server = ToString((char *) "C", getCommand()->getData(), getCommand()->getSize());
  if (getCommand() != Command || Server.empty()) return;
 
  // Reset alarm level, publish/log action and reset server message severity
  for (int i=0; i<List.size(); i++) if (List[i].Server == Server) {
    Message(INFO, "Alarm level of server %s reset by %s (ID %d)", Server.c_str(), getClientName(), getClientId());
        List[i].Level = 0;
        List[i].WarnedLevel = 0;
        sendCommandNB((Server+"/ResetMessage").c_str(), (int) 0);
  }
  
  UpdateAlarmSummary();
}


// Update alarm status summary (locking since access can be from main thread and DIM handler threads)
void AlarmHandler::UpdateAlarmSummary() {

  ostringstream Buf;
  int Alarm = 0, Ret;  

  Lock();

  for (int i=0; i<List.size(); i++) {
        // Alarm level description
        Buf << List[i].Server << ": " << (List[i].Level>=0 && List[i].Level<=4 ? LevelStr[List[i].Level] : "unknown");
        Buf << " (" << List[i].Level << ")" << endl;

        // Adjust master alarm and update server alarm level
        if (List[i].Level > Alarm) Alarm = List[i].Level;
        List[i].AlarmLevel->updateService(List[i].Level);

        // Check if alarm level raised, then send alarm message once
        if (List[i].WarnedLevel < List[i].Level && !List[i].Email.empty()) {
          List[i].WarnedLevel = List[i].Level;
          
          // Prepare email message
          char *Text;
          time_t Time = time(NULL);
          if (asprintf(&Text, "echo \"Server alarm level '%s' at %s\"|"
                        "mail -s \"Evidence Alarm for '%s'\" %s",
                        List[i].Level>=0 && List[i].Level<=4 ? LevelStr[List[i].Level] : "unknown",
                        ctime(&Time), List[i].Server.c_str(), List[i].Email.c_str()) != -1) {
                system(Text); // Return value depending on OS
                free(Text);
          }
          else Message(ERROR, "Could not send alarm email, asprintf() failed");
        }
  }
  
  // Update master alarm services
  MasterAlarm = Alarm;   
  Master->updateService();
  
  // Update alarm description (DIM requires variables to be valid until update)
  char *Tmp = new char[Buf.str().size()+1];
  strcpy(Tmp, Buf.str().c_str());  
  Summary->updateService(Tmp);

  delete[] AlarmText;
  AlarmText = Tmp;
  
  Unlock();
}

//          
// Main program
//
int main() {
    
  DimBrowser Browser;
  char *Server, *Node;
  bool Exist;
  
  // Static declaration ensures calling of destructor by exit()
  static AlarmHandler Alarm; 
  
  // Check periodically if servers are up
  while(!Alarm.ExitRequest) {

    for (int i=0; i<Alarm.List.size(); i++) {
      Exist = false;
      Browser.getServers();
      while (Browser.getNextServer(Server, Node) == 1) {
        if (Alarm.List[i].Server == Server) Exist = true;
      }
          if (!Exist) Alarm.List[i].Level = 4;
          else if (Alarm.List[i].Level == -1) Alarm.List[i].Level = 0;
    }
    
    Alarm.UpdateAlarmSummary();
    sleep(atoi(Alarm.GetConfig("period").c_str()));
  }
}