Changeset 14659 for fact/tools

Timestamp:

11/19/12 12:10:01 (12 years ago)

Author:

neise

Message:

long time since last checkin

File:

• fact/tools/PyDimCtrl/ScriptsForPyDimCtrl.py (modified) (20 diffs)

fact/tools/PyDimCtrl/ScriptsForPyDimCtrl.py

@@ -6 +6 @@
 import types
 import sys
+import threading
+import Queue
+
+import logging
+# create logger with 'spam_application'
+logger = logging.getLogger('PyDimCtrl')
+logger.setLevel(logging.DEBUG)
+# create file handler which logs even debug messages
+fh = logging.FileHandler('PyDimCtrl.log')
+fh.setLevel(logging.DEBUG)
+# create console handler with a higher log level
+ch = logging.StreamHandler()
+ch.setLevel(logging.WARNING)
+# create formatter and add it to the handlers
+formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+fh.setFormatter(formatter)
+ch.setFormatter(formatter)
+# add the handlers to the logger
+logger.addHandler(fh)
+logger.addHandler(ch)
 
 last_drive_kwargs = {}
@@ -11 +31 @@
 
 bias_calibration = {}
+
+bias_current = {}
 
 report_length = 0
@@ -26 +48 @@
   ["IC 310",        0.6, -18, -18+180 ],
   ["PKS 2155-304",  0.6,  90, -90 ] ]
+  
+crazyPatches = [66,191,193]
   
 sourcedict = {}
@@ -117 +141 @@
     print "connecting to crate", cratenum
     for i in range(cratenum*10, (cratenum+1)*10 ):
-      print 'disconnecting from i', time.time()
+      print 'disconnecting from ', i
       fad_control.disconnect(i)
     print "... done"
@@ -310 +334 @@
 
 
+
 def Take( time, num_events, runtype, verbose=False):
   """ more general version of e.g. TakePedestalOnRun
@@ -337 +362 @@
   """
   if Bias not in [None, 'On', 'Off']:
-    raise ValueError('Bias needs to be one of [None, 'On', 'Off']')
+    raise ValueError('Bias needs to be one of [None, "On", "Off"]')
+    return False
+
+  # Sanity check
+  # Drive can be:
+  # Drive = 'sourcename'
+  # Drive = ('sourcename', wobble_pos = 1 || 2)
+  # Drive = None
+  if type(Drive) not in [type(''), type((0,0)), type(None) ]:
+    raise ValueError('Bias needs to be one of [None, "On", "Off"]')
     return False
 
@@ -380 +414 @@
   #
   # In case Drive is None no commands will be send to DRIVE_CONTROL at allow
+  
+  # Drive can be:
+  # Drive = 'sourcename'
+  # Drive = ('sourcename', wobble_pos = 1 || 2)
+  # Drive = None
   
   # RATE_CTRL
@@ -430 +469 @@
   duration = 20.
   if Bias == 'On':
-    if (bias_control.voltage()[0:320].mean() > 60) and 
-      ( (bias_control.stn == 5) or (bias_control.stn == 9) )
+    if ( (bias_control.voltage()[0:320].mean() > 60) and 
+            ( (bias_control.stn == 5) or 
+              (bias_control.stn == 9) ) ):
+      pass 
       
   elif Bias == 'Off':
-    
+    pass
 
   if verbose:
@@ -456 +497 @@
   # take a Pedestal run
   IsReadyForDataTaking()
-  print 'taking External Light Pulser with BIAS on 1000 ...'
+  print 'taking pedestal run with 1000 events'
   Take(-1, 1000, 'pedestal')
   
@@ -541 +582 @@
 
 def FirstDrsCalib( SkipCurrentCalib=False ):
+  logger = logging.getLogger('PyDimCtrl.FDC')
   """ performs the everything, which is done in the FirstDrsCalib Script as well . 
   """
@@ -549 +591 @@
   # Since we do not know, what the feedback program is doing at the moment, we should as well,
   # tell it to keep its mouth shut ... just to be sure, we know whats going on
-  print "stopping feedback"
+  logger.debug("stopping feedback")
   feedback.stop()
 
   time.sleep(2)
   # stopping should always be possible, and end in state 'Connected'(6)
-  print " ...waiting for FEEDBACK to be in state 6: Connected"
+  
+  logger.debug(" ...waiting for FEEDBACK to be in state 6: Connected")
+  print 
   feedback.wait(6)
   print "..done"
@@ -628 +672 @@
   # and a second pedestal run, with the same ROI as our next data will be, i.e. ROI=300 in this case
   print "taking DRS:Pedestal 1000 ..."
-  print "==================================================="
-  print "OPERATOR: "
-  print "observe Events tab and make sure there are no patches "
-  print "with strange behaviour, which can be caused "
-  print "by DRS-CHIP Problems"
-  print "==================================================="
+  #print "==================================================="
+  #print "OPERATOR: "
+  #print "observe Events tab and make sure there are no patches "
+  #print "with strange behaviour, which can be caused "
+  #print "by DRS-CHIP Problems"
+  #print "==================================================="
 
   fad_control.start_drs_calibration()
@@ -669 +713 @@
   # the feedback should be in state 'CurrentControl'(12) now
   # the feedback should be in state 'CurrentCtrlIdle'(9) now since 30.05.12
-  print "...waiting for FEEDBACK to be in state 9: CurrentCtrlIdle"
-  feedback.wait(9)
+  print "...waiting for FEEDBACK to be in state 12: CurrentControl"
+  feedback.wait(12)
   print "... done"
   print "switching on bias"
@@ -874 +918 @@
     return False
 
-
-
-
 def GetBiasCalibration():
   cali = feedback.calibration()
@@ -882 +923 @@
   bias_calibration['Calibration'] = cali
 
-def GetBiasCurrent(verbose = False):
+def _GetBiasCurrent_Raw(verbose = False):
   """ return median, std, max and min current
   """
@@ -907 +948 @@
 #  print 'GUI feedback:', II[0] - V[0]/r[0]*1e6
 
+  # GUII means current(I) as it is written in the GUI :-)
+  # read: GUI-I :-)
   GUII = II-V/r*1e6
   if verbose:
@@ -916 +959 @@
     print 'min', GUII.min()
 
-  return GUII.mean(), GUII.std(), GUII.max(), GUII.min()
-
-  
+  bias_current['Time'] = time.time()
+  bias_current['raw_patch_current'] = GUII
+  return GUII
+
+def GetPatchCurrents(crazyPatches=crazyPatches ,verbose = False, max_age_sec=1. ):
+  
+  if 'Time' in bias_current:
+    if time.time() - bias_current['Time'] > max_age_sec:
+      raw_cur = _GetBiasCurrent_Raw()
+    else:
+      raw_cur = bias_current['raw_patch_current']
+  else:
+    raw_cur = _GetBiasCurrent_Raw()
+    
+  if crazyPatches is None:
+    return raw_cur
+  else:
+    try: 
+      crazyPatches = list(crazyPatches)
+    except:
+        raise
+    goodPatches = [x for x in range(320) if x not in crazyPatches]
+    return raw_cur[goodPatches]
+
+def GetPixelCurrents(crazyPatches=crazyPatches ,verbose = False, max_age_sec=1. ):
+  
+  if 'Time' in bias_current:
+    if time.time() - bias_current['Time'] > max_age_sec:
+      raw_cur = _GetBiasCurrent_Raw()
+    else:
+      raw_cur = bias_current['raw_patch_current']
+  else:
+    raw_cur = _GetBiasCurrent_Raw()
+
+  pixel_cur = np.zeros(len(raw_cur))
+  for i in range(len(raw_cur)):
+    if i/2:
+      pixel_cur[i] = raw_cur[i]/5.
+    else:
+      pixel_cur[i] = raw_cur[i]/4.
+
+  if crazyPatches is None:
+    return pixel_cur
+  else:
+    try: 
+      crazyPatches = list(crazyPatches)
+    except:
+        raise
+    goodPatches = [x for x in range(320) if x not in crazyPatches]
+    return pixel_cur[goodPatches]
+
+def PrintPixelCurrents( delay = 4.0):
+  while True:
+    time.sleep( delay )
+    current = GetPixelCurrents()
+    interesting =  current.min(), np.median(current), current.mean(), current.max()
+    print time.strftime('%d %b %Y %H:%M:%S UTC', time.gmtime()), interesting
 
 
 def TrackSourceWobbleX( sourcename, wobble_pos ):
-  """ general Tracking function
-  """
-  wp = int(wobble_pos)
-  if wp != 1 and wp != 2:
-    raise ValueError('wobble_pos *must* be 1 or 2')
-  
-  if sourcename not in sourcedict:
-    print sourcedict.keys()
-    raise ValueError('sourcename: '+ sourcename +' must be in sourcedict.')
-
-  print 'moving telescope to wobble position ', wp, 'of ', sourcename
-  print '...waiting for DRIVE_CONTROL'
-  print '   to be in state 6: Armed'
-  drive_control.stop()
-  drive_control.wait(6) #Armed
-  print 'DRIVE: ARMED'
-  time.sleep(5.)
-
-  wobble_offset = sourcedict[sourcename]['wobble_offset']
-  wobble_angle = sourcedict[sourcename]['wobble_angle'+str(wp)]
-  sourcename += '\0'
-  
-  last_drive_method = drive_control.track_source
-  last_drive_kwargs = { 'wobble_offset' : wobble_offset,
+    """ general Tracking function
+    """
+    wp = int(wobble_pos)
+    if wp != 1 and wp != 2:
+        raise ValueError('wobble_pos *must* be 1 or 2')
+  
+    if sourcename not in sourcedict:
+        print sourcedict.keys()
+        raise ValueError('sourcename: '+ sourcename +' must be in sourcedict.')
+    
+    logger.debug('TrackSourceWobbleX: sending drive_control.stop()')
+    drive_control.stop()
+    logger.debug('TrackSourceWobbleX: waiting for DRIVE_CONTROL to be "Armed"')
+    drive_control.wait(6) #Armed
+    
+    logger.info('moving telescope to wobble position %s of %s' % str(wp), sourcename)
+
+    
+    wobble_offset = sourcedict[sourcename]['wobble_offset']
+    wobble_angle = sourcedict[sourcename]['wobble_angle'+str(wp)]
+    sourcename += '\0'
+  
+    # this is just book keeping. I store the method, which is used 
+    # in this case drive_control.track_source()
+    # and the arguments, in some global vars .. in order to be able to 
+    # resend the command if necessary.
+    last_drive_method = drive_control.track_source
+    last_drive_kwargs = { 'wobble_offset' : wobble_offset,
                           'wobble_angle' : wobble_angle,
                           'source_name' : sourcename }
-  kwa = last_drive_kwargs
-  last_drive_method(kwa['wobble_offset'], kwa['wobble_angle'], kwa['source_name'])
-  
-  print '... done'
+    kwa = last_drive_kwargs
+    # Here I really send the command
+    last_drive_method(kwa['wobble_offset'], kwa['wobble_angle'], kwa['source_name'])
+
+    logger.debug('TrackSourceWobbleX: waiting for DRIVE_CONTROL to be "Tracking"')
+    drive_control.wait(8) #Tracking
+    
+    return True
 
 def TakeSource( name ):
@@ -976 +1081 @@
   StopTracking()
 
-
-
-
-
+def set_file_format_to_FITS():
+    fad_control.set_file_format(2)
+
+# (Time , Drive, Bias, DAQ, call-before-run)
+class RunType(object):
+    drs_pedestal = (-1, 1000, 'drs-pedestal')
+    drs_gain = (-1, 1000, 'drs-gain')
+    drs_time = (-1, 1000, 'drs-time')
+    drs_time_upshifted = (-1, 1000, 'drs-gain-upshifted')
+    pedestal = (-1, 1000, 'pedestal')
+    lightpulser = (-1, 1000, 'light-pulser-ext')
+    data = (300, -1, 'data')
+
+class Schedule(object):
+    drs_calib_crab = [
+      ( None , ('Crab',1) , 'Off', RunType.drs_pedestal , fad_control.start_drs_calibration ),
+      ( None , ('Crab',1) , 'Off', RunType.drs_gain     , None ),
+      ( None , ('Crab',1) , 'Off', RunType.drs_pedestal , None ),
+      ( None , ('Crab',1) , 'Off', RunType.drs_pedestal , set_file_format_to_FITS ),
+      ( None , ('Crab',1) , 'Off', RunType.drs_time     , None ),
+      ( None , ('Crab',1) , 'Off', RunType.drs_time_upshifted , None ),
+      ( None , ('Crab',1) , 'Off', RunType.pedestal     , fad_control.reset_secondary_drs_baseline ),
+      ( None , ('Crab',1) , 'Off', RunType.pedestal     , None )
+    ]
+
+    drs_calib_not_moving = [
+      ( None , None , 'Off', RunType.drs_pedestal , fad_control.start_drs_calibration ),
+      ( None , None , 'Off', RunType.drs_gain     , None ),
+      ( None , None , 'Off', RunType.drs_pedestal , None ),
+      ( None , None , 'Off', RunType.drs_pedestal , set_file_format_to_FITS ),
+      ( None , None , 'Off', RunType.drs_time     , None ),
+      ( None , None , 'Off', RunType.drs_time_upshifted , None ),
+      ( None , None , 'Off', RunType.pedestal     , fad_control.reset_secondary_drs_baseline ),
+      ( None , None , 'Off', RunType.pedestal     , None )
+    ]
+
+    data_crab_Wobble_1 = [
+      ( None , ('Crab',1) , 'On', RunType.pedestal    , None ),
+      ( None , ('Crab',1) , 'On', RunType.lightpulser , None ),
+      ( None , ('Crab',1) , 'On', RunType.data        , None ),
+      ( None , ('Crab',1) , 'On', RunType.data        , None ),
+      ( None , ('Crab',1) , 'On', RunType.data        , None ),
+      ( None , ('Crab',1) , 'On', RunType.data        , None ),
+    ]
+    data_crab_Wobble_2 = [
+      ( None , ('Crab',2) , 'On', RunType.pedestal    , None ),
+      ( None , ('Crab',2) , 'On', RunType.lightpulser , None ),
+      ( None , ('Crab',2) , 'On', RunType.data        , None ),
+      ( None , ('Crab',2) , 'On', RunType.data        , None ),
+      ( None , ('Crab',2) , 'On', RunType.data        , None ),
+      ( None , ('Crab',2) , 'On', RunType.data        , None ),
+    ]
+
+
+
+class ScheduledDataTakingThread( threading.Thread ):
+    """ 
+        
+    """
+    def __init__( self , delay = 5):
+        threading.Thread.__init__(self) 
+        self.stoprequest = threading.Event()
+        self.warning_state = 'Unknown'
+        self.state_change = time.time()
+        self.logger = logging.getLogger('PyDimCtrl.ScheduledDataTakingThread')
+        
+        self.delay = delay
+        self.queue = Queue.Queue()
+
+    def run(self):
+        while not self.stoprequest.isSet():
+
+            task = self.queue.get()
+            
+            start_time = task[0]
+            drive_task = task[1]
+            bias_task = task[2]
+            daq_task = task[3]
+            pre_action = task[4]
+                
+            if drive_task is not None:
+                TrackSourceWobbleX( *drive_task )
+
+            #    if start_time is None or start_time > time.gmtime():
+            #        # do it
+            #        if drive_task is not None:
+            #            TrackSourceWobbleX( *drive_task ) # this blocks until 'Tracking'
+            #            # TODO:
+            #            # spawn a new DriveMonitorThread here and start it.
+                    
+            if bias_task is not None:
+                if bias_task == 'On':
+                    Make_Bias_On()
+                else:
+                    Make_Bias_On()
+                    # TODO:
+                    # spawn a new BiasMonitorThread here and start it.
+                    
+            if daq_task is not None:
+                if pre_action is not None:
+                    pre_action()
+                MakeReadyForDataTaking()
+                Take( *daq_task )
+                # TODO:
+                # spawn a new DaqMonitorThread here and start it.
+                        
+                        
+            self.queue.task_done() 
+            time.sleep( self.delay )
+            
+    def join( self, timeout=None):
+        self.stoprequest.set()        
+        self.logger.info('was requested to die with timeout=%s' % str(timeout) )
+        #super(CheckBiasThread, self).join()
+        threading.Thread.join(self) 
+
+
+
+
+def MakeReadyForDataTaking( delay=1 ):
+    while fad_control.stn != 4 or ftm_control.stn != 3: # Connected & Idle
+        logger.info('MakeReadyForDataTaking FTM trigger is still on')
+        if ftm_control.stn == 4: # Trigger is on
+            ftm_control.stop_trigger()
+    
+        logger.warning('MakeReadyForDataTaking FAD: still in WritingData')
+        if fad_control.stn == 8: # Writing Data
+            fad_control.close_all_open_files()
+        
+        logger.warning('MakeReadyForDataTaking FAD: still in one of the Configure(ing) states')
+        if fad_control.stn in [5,6,7]: # one of the Configure(ing) states
+            fad_control.reset_configure()
+            
+        logger.warning('MakeReadyForDataTaking FTM: still in one of the Configure(ing) states')
+        if ftm_control.stn in [5,6,7]: # one of the Configure(ing) states
+            ftm_control.reset_configure()
+
+        time.sleep(delay)
+
+    return True
 
 def PrintCurrents( x = 1.0):
   while True:
     time.sleep( x )
-    print time.strftime('%d %b %Y %H:%M:%S UTC', time.gmtime()), GetBiasCurrent()
-
+    GUII = GetBiasCurrent()
+    interesting =  GUII.mean(), GUII.std(), GUII.max(), GUII.min()
+    print time.strftime('%d %b %Y %H:%M:%S UTC', time.gmtime()), interesting
 
 
@@ -1003 +1245 @@
   return conn
   
-  
-  
+def GetFadBadConnections( good = 0x42 ):
+  """ returns list of bad FAD IDs
+  """
+  conn = GetFadConnections()
+  bads = np.where( conn != good) [0]
+  return bads
+
+
+def StartUp():
+  fad_control.start()
+  bias_control.reconnect()
+  bias_control.set_zero_voltage()
+  while bias_control.stn == 6 : # OverCurrent
+    bias_control.set_zero_voltage()
+    bias_control.reset_over_current_status()
+  
+  ftm_control.reconnect()
+  
+  number_of_loops = 3
+  while True:
+    bad = GetFadBadConnections() 
+    if len(bad) == 0:
+      break
+      
+    for b in set( bad/10 ): # set(b/10) contains the crate ids, which have bad FADs
+      ResetCrate( b )
+      
+    number_of_loops -= 1
+    if not number_of_loops:
+      print 'StartUp aborted while trying to get the FADs into Connected state ... '
+      return False
+  
+  
+  if not MakeDriveCtrlReady():
+    return False
+  
+  return True
+  
+def Shutdown():
+    fad_control.stop()
+    ftm_control.disconnect()
+    feedback.stop()
+    bias_control.set_zero_voltage()
+    
+
+
+def MakeDriveCtrlReady():
+  """ Tries to get Drive ctrl from 
+      Disconnected, locked or Error to Armed
+      after a few tries it gives up and returns False
+  """
+  outer_counter = 3
+  while drive_control.stn != 6: #Armed
+#    print 'MakeDriveCtrlReady outer_counter', outer_counter
+  
+    counter = 3
+    while drive_control.stn == 1: #Disconnected
+#      print 'MakeDriveCtrlReady in while 1, counter:', counter
+      drive_control.reconnect()
+      time.sleep(0.5)
+      counter -= 1
+      if not counter:
+        print 'count not connect drive control to cosy ... please check if cosy is running'
+        return False
+    
+    counter = 3
+    while drive_control.stn == 3: # Locked
+#      print 'MakeDriveCtrlReady in while 2, counter:', counter
+      drive_control.unlock()
+      time.sleep(0.5)
+      counter -= 1
+      if not counter:
+        print 'could not unlock drive control WTF?'
+        return False
+
+    counter = 3
+    while drive_control.stn == 256: # ERROR
+#      print 'MakeDriveCtrlReady in while 3, counter:', counter
+      drive_control.stop()
+      time.sleep(0.5)
+      counter -= 1
+      if not counter:
+        print 'could not unlock drive control WTF?'
+        return False
+
+    outer_counter -= 1
+    if not outer_counter:
+      print 'ERROR while trying to get Drive control ready'
+      return False
+
+  return True
+  
+feedback_calibration_done = None # Bool... 
+
+def Make_Bias_Off():
+
+    bias_control.set_zero_voltage()
+    bias_control.wait(7)
+
+
+def Make_Bias_On():
+    """ Tries to get the Bias, and Feedback into a defined safe Off state
+        While beeing as ready as possible to go into On state
+    """
+    global feedback_calibration_done
+    # Feedback states:
+    #List of available states:
+    #-[-1]:    NotReady (State machine not ready, events are ignored.)
+    #-[0]:     Ready (State machine ready to receive events.)
+    #-[1]:     DimNetworkNotAvailable (The Dim DNS is not reachable.)
+    #-[2]:     Disconnected (The Dim DNS is reachable, but the required subsystems are not available.)
+    #-[3]:     Connecting (Only biasctrl is available and connected with its hardware.)
+    #-[4]:     ConnectedFSC (biasctrl and fscctrl are available and connected with their hardware.)
+    #-[5]:     ConnectedFAD (biasctrl and fadctrl are available and connected with their hardware.)
+    #-[6]:     Connected (biasctrl, fadctrl and fscctrl are available and connected with their hardware.)
+    #-[7]:     TempCtrlIdle (Temperature control activated, but voltage output disabled.)
+    #-[8]:     FeedbackIdle (Feedback control activated, but voltage output disabled.)
+    #-[9]:     CurrentCtrlIdle (Current control activated, but voltage output disabled.)
+    #-[10]:    TempControl (Temperature control activated and voltage output enabled.)
+    #-[11]:    FeedbackControl (Feedback control activated and voltage output enabled.)
+    #-[12]:    CurrentControl (Current/Temp control activated and voltage output enabled.)
+    #-[13]:    Calibrating (Calibrating current offsets.)
+    #-[256]:   ERROR (Common error state.)
+    #-[65535]: FATAL (A fatal error occured, the eventloop is stopped.)
+
+
+    while not  ( (feedback.stn == 12) and
+                 (bias_control.stn == 9) and
+                 (np.array(bias_control.voltage()[0:320]).mean() > 65.) ):
+
+        # for security
+        feedback.enable_output(0)
+
+        # try to get bias_control into state: VoltageOn
+        while not bias_control.stn == 9:
+            
+            if bias_control.stn == 1: #Disconnected
+                bias_control.reconnect()
+            if bias_control.stn == 2: #Connecting
+                time.sleep(1)
+            if bias_control.stn == 3: #Initializing
+                time.sleep(1)
+            if bias_control.stn == 4: #Connected
+                # this seems never to happen ... 
+                time.sleep(1)
+                #bias_control.set_global_dac(1)
+            if bias_control.stn == 5: #Ramping
+                time.sleep(1)
+            if bias_control.stn == 6: #OverCurrent
+                bias_control.set_zero_voltage()
+                time.sleep(1)
+                bias_control.reset_over_current_status()
+            if bias_control.stn == 7: #VoltageOff
+                bias_control.set_global_dac(1)
+            if bias_control.stn == 8: #NotReferenced
+                bias_control.set_zero_voltage()
+            if bias_control.stn == 9: #VoltageOn
+                pass
+            if bias_control.stn > 9:
+                print "BIAS control is in strange status... don't know what to do"
+                time.sleep(1)
+            time.sleep(1)
+
+
+        # Try to get Feedback into CurrentControl
+        while not (feedback.stn == 12):
+            
+            if feedback.stn < 6:    # not properly connected.
+                # State 6 is 'Connected' but this normaly hardly shows up
+                # Its either voltageOff or any other meaningful state...
+                print "FEEDBACK not properly connected..."
+                time.sleep(1)
+
+            if feedback.stn == 13: #Calibrating
+                time.sleep(1)
+                continue
+                
+            if not feedback_calibration_done:
+                if feedback.calibration() is None:
+                    print "FEEDBACK has not CURRENT CALIBRATION"
+                    print "TRYING TO CALIBRATE"
+                    feedback.calibrate_currents()
+                    time.sleep(0.5)
+                    continue
+                else:
+                    feedback_calibration_done = True
+            
+            if feedback.stn == 6: #Connected
+                feedback.start_current_control(0.0)
+            
+            if feedback.stn in [7,8]: #TempCtrlIdle , FeedbackIdle
+                feedback.stop()
+                time.sleep(0.5)
+                feedback.start_current_control(0.0)
+                
+            if feedback.stn == 9: #CurrentCtrlIdle
+                feedback.enable_output(1)
+            
+            if feedback.stn in [10,11]: #TempControl, FeedbackControl
+                feedback.stop()
+                time.sleep(0.5)
+                feedback.start_current_control(0.0)
+                
+            if feedback.stn == 12: # This is where we want ot be
+                pass
+                
+            
+            time.sleep(1)
+        # Bias States
+        #List of available states:
+        #-[-1]:    NotReady (State machine not ready, events are ignored.)
+        #-[0]:     Ready (State machine ready to receive events.)
+        #-[1]:     Disconnected (Bias-power supply not connected via USB.)
+        #-[2]:     Connecting (Trying to establish USB connection to bias-power supply.)
+        #-[3]:     Initializing (USB connection to bias-power supply established, synchronizing USB stream.)
+        #-[4]:     Connected (USB connection to bias-power supply established.)
+        #-[5]:     Ramping (Voltage ramping in progress.)
+        #-[6]:     OverCurrent (At least one channel is in over current state.)
+        #-[7]:     VoltageOff (All voltages are supposed to be switched off.)
+        #-[8]:     NotReferenced (Internal reference voltage does not match last sent voltage.)
+        #-[9]:     VoltageOn (At least one voltage is switched on and all are at reference.)
+        #-[10]:    ExpertMode (Special (risky!) mode to directly send command to the bias-power supply.)
+        #-[256]:   ERROR (Common error state.)
+        #-[65535]: FATAL (A fatal error occured, the eventloop is stopped.)
+                
+
+class CheckBiasThread( threading.Thread ):
+    """ Thread, which continously monitors the Bias_Control status every
+        *delay* seconds.
+        in case of too high currents it will ... 
+        .... do nothing :-) yet.
+        So far it will just change its internal state from 'Normal'
+        to 'PreWarning' or 'Serious'.
+        If the currents remain "high" for more than
+        *serious_action_after* seconds. some action would be taken.
+        I our case, just a WARNING will be logged.
+        
+        The caller thread should check the *warning_state* from outside this 
+        thread and take apropriate actions, when 'PreWarning' is reached...
+        
+        
+    """
+    def __init__( self , delay=5, serious_action_after = 60 ):
+        #super(CheckBiasThread, self).__init__()
+        threading.Thread.__init__(self) 
+        self.stoprequest = threading.Event()
+        self.warning_state = 'Unknown'
+        self.state_change = time.time()
+        self.delay = delay
+        self.serious_action_after = serious_action_after
+        self.logger = logging.getLogger('PyDimCtrl.CheckBiasThread')
+
+    def run(self):
+        while not self.stoprequest.isSet():
+            
+            if  (self.warning_state == 'Serious' and 
+                    time.time()-self.state_change > self.serious_action_after):
+                # Remergency Ramp Down
+                self.logger.info('At this moment, I would ramp down the voltage')
+                self.logger.warning('G-APD currents are too high, since %d seconds' % self.serious_action_after)
+                self.logger.warning('Emergency voltage shutdown [not yet implemented]')
+
+            currents = GetPixelCurrents()
+            if ( (np.median(currents) > 70) or (currents.max() > 90) ):
+                if self.warning_state != 'Serious':
+                    self.logger.info('trasit into Serious Warning')
+                    self.warning_state = 'Serious'
+                    self.state_change = time.time()
+                
+            elif ( (np.median(currents) > 60) or (currents.max() > 80) ):
+                if self.warning_state != 'PreWarning':
+                    self.logger.info('transit into Pre Warning')
+                    self.warning_state = 'PreWarning'
+                    self.state_change = time.time()
+                
+            elif ( (np.median(currents) < 60) or (currents.max() < 80) ):
+                if self.warning_state != 'Normal':
+                    self.logger.info('transit into Normal Situation')
+                    self.warning_state = 'Normal'
+                    self.state_change = time.time()
+                
+            if bias_control.stn == 6: #Overcurrent
+                if self.warning_state != 'OverCurrent':
+                    self.logger.warning('trasit into Overcurrent State during data Taking')
+                    self.warning_state = 'OverCurrent'
+                    self.state_change = time.time()
+                    
+            if bias_control.stn == 1: #Disconnected
+                if self.warning_state != 'Disconnected':
+                    self.logger.warning('trasit into Disconnected state during data Taking')
+                    self.warning_state = 'Disconnected'
+                    self.state_change = time.time()
+                
+                
+            
+            time.sleep( self.delay )
+            
+    def join( self, timeout=None):
+        self.stoprequest.set()
+        
+        self.logger.info('was requested to die with timeout=%s' % str(timeout) )
+        #super(CheckBiasThread, self).join()
+        threading.Thread.join(self) 
+
+
+class CheckDriveThread( threading.Thread ):
+    """ Thread, which continously monitors the Drive_Control status every
+        *delay* seconds.
+        in case Drive_Control goes into Error State it will 
+        send:
+              * STOP
+              * and then resend the last tracking command
+              
+        In case sending STOP is not bringing the Drive_Control back to
+        state 'Armed' it will try again. Until the *try_counter* is zero.
+        Then it will totally Freak out.
+        
+        .... does nothing really! :-) yet.
+        
+    """
+    def __init__( self , try_counter = 5, timeout = 30, delay=5 ):
+        threading.Thread.__init__(self) 
+        self.stoprequest = threading.Event()
+        
+        self.error_counter = 0
+        self.try_counter = try_counter
+        self.timeout = timeout
+        self.delay = delay
+        
+        self.warning_state = 'Unknown'
+        self.state_change = time.time()
+
+        self.logger = logging.getLogger('PyDimCtrl.CheckDriveThread')
+
+    def run(self):
+        while not self.stoprequest.isSet():
+            
+            if drive_control.stn == 256: #Error
+                self.error_counter += 1
+                self.state_change = time.time()
+                self.warning_state = 'ErrorFound'
+                self.logger.info('found Error State ... starting to react on it.')
+                self.logger.warning(' Error State ')
+                
+                start_of_Error_reaction = time.time()
+                while not drive.stn == 8: #Tracking
+                    if drive_control.stn == 256: #Error
+                        if try_counter:
+                            self.logger.info('sending STOP')
+                            drive_control.stop()
+                            try_counter -= 1
+                            time.sleep(1)
+
+                    elif drive_control.stn == 6: #Armed
+                        kwa = last_drive_kwargs
+                        self.logger.info('resending last tracking command')
+                        last_drive_method(kwa['wobble_offset'], 
+                                        kwa['wobble_angle'], 
+                                        kwa['source_name'])
+                        time.sleep(2)
+                    else:
+                        self.logger.error('found Error, but now neither Error state nor Armed state?!')
+                        self.logger.error('what should I do?')
+                        self.logger.error('thread aborting')
+                        self.warning_state = 'Critical'
+                        self.stoprequest.set()
+                        threading.Thread.join(self)
+                        
+                    if time.time() - start_of_Error_reaction > self.timeout:
+                        self.logger.error('timed out while trying to react on Drive Control Error')
+                        self.logger.error('what should I do?')
+                        self.logger.error('thread aborting')
+                        self.warning_state = 'Critical'
+                        #self.stoprequest.set()
+                        #threading.Thread.join(self)
+                        self.join()
+                        
+            else:
+                if self.warning_state != 'Normal':
+                    self.logger.info('transit into Normal Situation')
+                    self.warning_state = 'Normal'
+                    self.state_change = time.time()
+                                    
+            time.sleep( self.delay )
+            
+    def join( self, timeout=None):
+        self.stoprequest.set()
+        self.logger.info('was requested to die with timeout=%s' % str(timeout))
+        threading.Thread.join(self) 
+
+
+class BiasThread(threading.Thread):
+  """ A worker thread that takes directory names from a queue, finds all
+      files in them recursively and reports the result.
+
+      Input is done by setting the *requested_state*:
+      ['On', 'Off']
+      Output is done by setting the *readyness*:
+      ['Ready', 'NotYet', 'CantDoIt']
+      
+      Ask the thread to stop by calling its join() method.
+  """
+  requests = ['On', 'Off']
+  readynesses = ['Ready', 'NotYet', 'CantDoIt']
+  requested_state = None
+  readyness = 'NotYet'
+  
+  def __init__(self, start_state = 'Off'):
+    super(BiasThread, self).__init__()
+    BiasThread.requested_state = start_state
+    self.stoprequest = threading.Event()
+
+  def run(self):
+    # As long as we weren't asked to stop, try to find out, if we are in 
+    # the requested state, if not, try to reach the state...
+    while not self.stoprequest.isSet():
+        
+        if not self.InRequestedState():
+          print 'not in Requested State', BiasThread.requested_state
+          BiasThread.readyness = 'NotYet'
+      
+        try:
+            dirname = self.dir_q.get(True, 0.05)
+            filenames = list(self._files_in_dir(dirname))
+            self.result_q.put((self.name, dirname, filenames))
+        except Queue.Empty:
+            continue
+
+  def join(self, timeout=None):
+    self.stoprequest.set()
+    super(BiasThread, self).join(timeout)
+  
+  def InRequestedState():
+    """
+    """
+    if BiasThread.requested_state == 'On':
+      pass
+    elif BiasThread.requested_state == 'Off':
+      pass
+    else: # Houston we have a problem!
+      pass
+
+
+
+
+class CheckDaqThread( threading.Thread ):
+    """ Thread, which continously monitors the Rate
+        
+        in case the rate is < 1Hz --> Bing bing
+        
+    """
+    def __init__( self , delay=5 ):
+        threading.Thread.__init__(self) 
+        self.stoprequest = threading.Event()
+        
+        self.delay = delay
+        
+        self.warning_state = 'Unknown'
+        self.state_change = time.time()
+
+        self.logger = logging.getLogger('PyDimCtrl.CheckDaqThread')
+
+    def run(self):
+        while not self.stoprequest.isSet():
+            
+            rate = ftm_control.trigger_rates()[3]
+            if rate < 1:
+                if self.warning_state != 'SmallRate':
+                    self.logger.warning('SmallRate detected!')
+                    self.warning_state = 'SmallRate'
+                    self.state_change = time.time()
+            else:
+                if self.warning_state != 'Normal':
+                    self.logger.info('transit to Normal')
+                    self.warning_state = 'Normal'
+                    self.state_change = time.time()
+                
+            time.sleep( self.delay )
+            
+    def join( self, timeout=None):
+        self.stoprequest.set()
+        self.logger.info('was requested to die with timeout=%s' % str(timeout))
+        threading.Thread.join(self) 
+
+  
+
+def off():
+    for m in monis:
+        m.join()
+    
+monis = []
 if __name__ == '__main__':
   print 'Welcome to PyDimCtrl'
@@ -1017 +1748 @@
   print
 
-
+  #bm = CheckBiasThread()
+  #dm = CheckDriveThread()
+  #rm = CheckDaqThread()
+  #monis = [bm, dm, rm]
+#  bm.start()
+#  dm.start()
+#  rm.start()
+  #print
+  #print 'monis created'