Index: fact/tools/PyDimCtrl/ScriptsForPyDimCtrl.py =================================================================== --- fact/tools/PyDimCtrl/ScriptsForPyDimCtrl.py (revision 14466) +++ fact/tools/PyDimCtrl/ScriptsForPyDimCtrl.py (revision 14659) @@ -6,4 +6,24 @@ import types import sys +import threading +import Queue + +import logging +# create logger with 'spam_application' +logger = logging.getLogger('PyDimCtrl') +logger.setLevel(logging.DEBUG) +# create file handler which logs even debug messages +fh = logging.FileHandler('PyDimCtrl.log') +fh.setLevel(logging.DEBUG) +# create console handler with a higher log level +ch = logging.StreamHandler() +ch.setLevel(logging.WARNING) +# create formatter and add it to the handlers +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +fh.setFormatter(formatter) +ch.setFormatter(formatter) +# add the handlers to the logger +logger.addHandler(fh) +logger.addHandler(ch) last_drive_kwargs = {} @@ -11,4 +31,6 @@ bias_calibration = {} + +bias_current = {} report_length = 0 @@ -26,4 +48,6 @@ ["IC 310", 0.6, -18, -18+180 ], ["PKS 2155-304", 0.6, 90, -90 ] ] + +crazyPatches = [66,191,193] sourcedict = {} @@ -117,5 +141,5 @@ print "connecting to crate", cratenum for i in range(cratenum*10, (cratenum+1)*10 ): - print 'disconnecting from i', time.time() + print 'disconnecting from ', i fad_control.disconnect(i) print "... done" @@ -310,4 +334,5 @@ + def Take( time, num_events, runtype, verbose=False): """ more general version of e.g. TakePedestalOnRun @@ -337,5 +362,14 @@ """ if Bias not in [None, 'On', 'Off']: - raise ValueError('Bias needs to be one of [None, 'On', 'Off']') + raise ValueError('Bias needs to be one of [None, "On", "Off"]') + return False + + # Sanity check + # Drive can be: + # Drive = 'sourcename' + # Drive = ('sourcename', wobble_pos = 1 || 2) + # Drive = None + if type(Drive) not in [type(''), type((0,0)), type(None) ]: + raise ValueError('Bias needs to be one of [None, "On", "Off"]') return False @@ -380,4 +414,9 @@ # # In case Drive is None no commands will be send to DRIVE_CONTROL at allow + + # Drive can be: + # Drive = 'sourcename' + # Drive = ('sourcename', wobble_pos = 1 || 2) + # Drive = None # RATE_CTRL @@ -430,9 +469,11 @@ duration = 20. if Bias == 'On': - if (bias_control.voltage()[0:320].mean() > 60) and - ( (bias_control.stn == 5) or (bias_control.stn == 9) ) + if ( (bias_control.voltage()[0:320].mean() > 60) and + ( (bias_control.stn == 5) or + (bias_control.stn == 9) ) ): + pass elif Bias == 'Off': - + pass if verbose: @@ -456,5 +497,5 @@ # take a Pedestal run IsReadyForDataTaking() - print 'taking External Light Pulser with BIAS on 1000 ...' + print 'taking pedestal run with 1000 events' Take(-1, 1000, 'pedestal') @@ -541,4 +582,5 @@ def FirstDrsCalib( SkipCurrentCalib=False ): + logger = logging.getLogger('PyDimCtrl.FDC') """ performs the everything, which is done in the FirstDrsCalib Script as well . """ @@ -549,10 +591,12 @@ # Since we do not know, what the feedback program is doing at the moment, we should as well, # tell it to keep its mouth shut ... just to be sure, we know whats going on - print "stopping feedback" + logger.debug("stopping feedback") feedback.stop() time.sleep(2) # stopping should always be possible, and end in state 'Connected'(6) - print " ...waiting for FEEDBACK to be in state 6: Connected" + + logger.debug(" ...waiting for FEEDBACK to be in state 6: Connected") + print feedback.wait(6) print "..done" @@ -628,10 +672,10 @@ # and a second pedestal run, with the same ROI as our next data will be, i.e. ROI=300 in this case print "taking DRS:Pedestal 1000 ..." - print "===================================================" - print "OPERATOR: " - print "observe Events tab and make sure there are no patches " - print "with strange behaviour, which can be caused " - print "by DRS-CHIP Problems" - print "===================================================" + #print "===================================================" + #print "OPERATOR: " + #print "observe Events tab and make sure there are no patches " + #print "with strange behaviour, which can be caused " + #print "by DRS-CHIP Problems" + #print "===================================================" fad_control.start_drs_calibration() @@ -669,6 +713,6 @@ # the feedback should be in state 'CurrentControl'(12) now # the feedback should be in state 'CurrentCtrlIdle'(9) now since 30.05.12 - print "...waiting for FEEDBACK to be in state 9: CurrentCtrlIdle" - feedback.wait(9) + print "...waiting for FEEDBACK to be in state 12: CurrentControl" + feedback.wait(12) print "... done" print "switching on bias" @@ -874,7 +918,4 @@ return False - - - def GetBiasCalibration(): cali = feedback.calibration() @@ -882,5 +923,5 @@ bias_calibration['Calibration'] = cali -def GetBiasCurrent(verbose = False): +def _GetBiasCurrent_Raw(verbose = False): """ return median, std, max and min current """ @@ -907,4 +948,6 @@ # print 'GUI feedback:', II[0] - V[0]/r[0]*1e6 + # GUII means current(I) as it is written in the GUI :-) + # read: GUI-I :-) GUII = II-V/r*1e6 if verbose: @@ -916,40 +959,102 @@ print 'min', GUII.min() - return GUII.mean(), GUII.std(), GUII.max(), GUII.min() - - + bias_current['Time'] = time.time() + bias_current['raw_patch_current'] = GUII + return GUII + +def GetPatchCurrents(crazyPatches=crazyPatches ,verbose = False, max_age_sec=1. ): + + if 'Time' in bias_current: + if time.time() - bias_current['Time'] > max_age_sec: + raw_cur = _GetBiasCurrent_Raw() + else: + raw_cur = bias_current['raw_patch_current'] + else: + raw_cur = _GetBiasCurrent_Raw() + + if crazyPatches is None: + return raw_cur + else: + try: + crazyPatches = list(crazyPatches) + except: + raise + goodPatches = [x for x in range(320) if x not in crazyPatches] + return raw_cur[goodPatches] + +def GetPixelCurrents(crazyPatches=crazyPatches ,verbose = False, max_age_sec=1. ): + + if 'Time' in bias_current: + if time.time() - bias_current['Time'] > max_age_sec: + raw_cur = _GetBiasCurrent_Raw() + else: + raw_cur = bias_current['raw_patch_current'] + else: + raw_cur = _GetBiasCurrent_Raw() + + pixel_cur = np.zeros(len(raw_cur)) + for i in range(len(raw_cur)): + if i/2: + pixel_cur[i] = raw_cur[i]/5. + else: + pixel_cur[i] = raw_cur[i]/4. + + if crazyPatches is None: + return pixel_cur + else: + try: + crazyPatches = list(crazyPatches) + except: + raise + goodPatches = [x for x in range(320) if x not in crazyPatches] + return pixel_cur[goodPatches] + +def PrintPixelCurrents( delay = 4.0): + while True: + time.sleep( delay ) + current = GetPixelCurrents() + interesting = current.min(), np.median(current), current.mean(), current.max() + print time.strftime('%d %b %Y %H:%M:%S UTC', time.gmtime()), interesting def TrackSourceWobbleX( sourcename, wobble_pos ): - """ general Tracking function - """ - wp = int(wobble_pos) - if wp != 1 and wp != 2: - raise ValueError('wobble_pos *must* be 1 or 2') - - if sourcename not in sourcedict: - print sourcedict.keys() - raise ValueError('sourcename: '+ sourcename +' must be in sourcedict.') - - print 'moving telescope to wobble position ', wp, 'of ', sourcename - print '...waiting for DRIVE_CONTROL' - print ' to be in state 6: Armed' - drive_control.stop() - drive_control.wait(6) #Armed - print 'DRIVE: ARMED' - time.sleep(5.) - - wobble_offset = sourcedict[sourcename]['wobble_offset'] - wobble_angle = sourcedict[sourcename]['wobble_angle'+str(wp)] - sourcename += '\0' - - last_drive_method = drive_control.track_source - last_drive_kwargs = { 'wobble_offset' : wobble_offset, + """ general Tracking function + """ + wp = int(wobble_pos) + if wp != 1 and wp != 2: + raise ValueError('wobble_pos *must* be 1 or 2') + + if sourcename not in sourcedict: + print sourcedict.keys() + raise ValueError('sourcename: '+ sourcename +' must be in sourcedict.') + + logger.debug('TrackSourceWobbleX: sending drive_control.stop()') + drive_control.stop() + logger.debug('TrackSourceWobbleX: waiting for DRIVE_CONTROL to be "Armed"') + drive_control.wait(6) #Armed + + logger.info('moving telescope to wobble position %s of %s' % str(wp), sourcename) + + + wobble_offset = sourcedict[sourcename]['wobble_offset'] + wobble_angle = sourcedict[sourcename]['wobble_angle'+str(wp)] + sourcename += '\0' + + # this is just book keeping. I store the method, which is used + # in this case drive_control.track_source() + # and the arguments, in some global vars .. in order to be able to + # resend the command if necessary. + last_drive_method = drive_control.track_source + last_drive_kwargs = { 'wobble_offset' : wobble_offset, 'wobble_angle' : wobble_angle, 'source_name' : sourcename } - kwa = last_drive_kwargs - last_drive_method(kwa['wobble_offset'], kwa['wobble_angle'], kwa['source_name']) - - print '... done' + kwa = last_drive_kwargs + # Here I really send the command + last_drive_method(kwa['wobble_offset'], kwa['wobble_angle'], kwa['source_name']) + + logger.debug('TrackSourceWobbleX: waiting for DRIVE_CONTROL to be "Tracking"') + drive_control.wait(8) #Tracking + + return True def TakeSource( name ): @@ -976,14 +1081,151 @@ StopTracking() - - - - +def set_file_format_to_FITS(): + fad_control.set_file_format(2) + +# (Time , Drive, Bias, DAQ, call-before-run) +class RunType(object): + drs_pedestal = (-1, 1000, 'drs-pedestal') + drs_gain = (-1, 1000, 'drs-gain') + drs_time = (-1, 1000, 'drs-time') + drs_time_upshifted = (-1, 1000, 'drs-gain-upshifted') + pedestal = (-1, 1000, 'pedestal') + lightpulser = (-1, 1000, 'light-pulser-ext') + data = (300, -1, 'data') + +class Schedule(object): + drs_calib_crab = [ + ( None , ('Crab',1) , 'Off', RunType.drs_pedestal , fad_control.start_drs_calibration ), + ( None , ('Crab',1) , 'Off', RunType.drs_gain , None ), + ( None , ('Crab',1) , 'Off', RunType.drs_pedestal , None ), + ( None , ('Crab',1) , 'Off', RunType.drs_pedestal , set_file_format_to_FITS ), + ( None , ('Crab',1) , 'Off', RunType.drs_time , None ), + ( None , ('Crab',1) , 'Off', RunType.drs_time_upshifted , None ), + ( None , ('Crab',1) , 'Off', RunType.pedestal , fad_control.reset_secondary_drs_baseline ), + ( None , ('Crab',1) , 'Off', RunType.pedestal , None ) + ] + + drs_calib_not_moving = [ + ( None , None , 'Off', RunType.drs_pedestal , fad_control.start_drs_calibration ), + ( None , None , 'Off', RunType.drs_gain , None ), + ( None , None , 'Off', RunType.drs_pedestal , None ), + ( None , None , 'Off', RunType.drs_pedestal , set_file_format_to_FITS ), + ( None , None , 'Off', RunType.drs_time , None ), + ( None , None , 'Off', RunType.drs_time_upshifted , None ), + ( None , None , 'Off', RunType.pedestal , fad_control.reset_secondary_drs_baseline ), + ( None , None , 'Off', RunType.pedestal , None ) + ] + + data_crab_Wobble_1 = [ + ( None , ('Crab',1) , 'On', RunType.pedestal , None ), + ( None , ('Crab',1) , 'On', RunType.lightpulser , None ), + ( None , ('Crab',1) , 'On', RunType.data , None ), + ( None , ('Crab',1) , 'On', RunType.data , None ), + ( None , ('Crab',1) , 'On', RunType.data , None ), + ( None , ('Crab',1) , 'On', RunType.data , None ), + ] + data_crab_Wobble_2 = [ + ( None , ('Crab',2) , 'On', RunType.pedestal , None ), + ( None , ('Crab',2) , 'On', RunType.lightpulser , None ), + ( None , ('Crab',2) , 'On', RunType.data , None ), + ( None , ('Crab',2) , 'On', RunType.data , None ), + ( None , ('Crab',2) , 'On', RunType.data , None ), + ( None , ('Crab',2) , 'On', RunType.data , None ), + ] + + + +class ScheduledDataTakingThread( threading.Thread ): + """ + + """ + def __init__( self , delay = 5): + threading.Thread.__init__(self) + self.stoprequest = threading.Event() + self.warning_state = 'Unknown' + self.state_change = time.time() + self.logger = logging.getLogger('PyDimCtrl.ScheduledDataTakingThread') + + self.delay = delay + self.queue = Queue.Queue() + + def run(self): + while not self.stoprequest.isSet(): + + task = self.queue.get() + + start_time = task[0] + drive_task = task[1] + bias_task = task[2] + daq_task = task[3] + pre_action = task[4] + + if drive_task is not None: + TrackSourceWobbleX( *drive_task ) + + # if start_time is None or start_time > time.gmtime(): + # # do it + # if drive_task is not None: + # TrackSourceWobbleX( *drive_task ) # this blocks until 'Tracking' + # # TODO: + # # spawn a new DriveMonitorThread here and start it. + + if bias_task is not None: + if bias_task == 'On': + Make_Bias_On() + else: + Make_Bias_On() + # TODO: + # spawn a new BiasMonitorThread here and start it. + + if daq_task is not None: + if pre_action is not None: + pre_action() + MakeReadyForDataTaking() + Take( *daq_task ) + # TODO: + # spawn a new DaqMonitorThread here and start it. + + + self.queue.task_done() + time.sleep( self.delay ) + + def join( self, timeout=None): + self.stoprequest.set() + self.logger.info('was requested to die with timeout=%s' % str(timeout) ) + #super(CheckBiasThread, self).join() + threading.Thread.join(self) + + + + +def MakeReadyForDataTaking( delay=1 ): + while fad_control.stn != 4 or ftm_control.stn != 3: # Connected & Idle + logger.info('MakeReadyForDataTaking FTM trigger is still on') + if ftm_control.stn == 4: # Trigger is on + ftm_control.stop_trigger() + + logger.warning('MakeReadyForDataTaking FAD: still in WritingData') + if fad_control.stn == 8: # Writing Data + fad_control.close_all_open_files() + + logger.warning('MakeReadyForDataTaking FAD: still in one of the Configure(ing) states') + if fad_control.stn in [5,6,7]: # one of the Configure(ing) states + fad_control.reset_configure() + + logger.warning('MakeReadyForDataTaking FTM: still in one of the Configure(ing) states') + if ftm_control.stn in [5,6,7]: # one of the Configure(ing) states + ftm_control.reset_configure() + + time.sleep(delay) + + return True def PrintCurrents( x = 1.0): while True: time.sleep( x ) - print time.strftime('%d %b %Y %H:%M:%S UTC', time.gmtime()), GetBiasCurrent() - + GUII = GetBiasCurrent() + interesting = GUII.mean(), GUII.std(), GUII.max(), GUII.min() + print time.strftime('%d %b %Y %H:%M:%S UTC', time.gmtime()), interesting @@ -1003,6 +1245,495 @@ return conn - - +def GetFadBadConnections( good = 0x42 ): + """ returns list of bad FAD IDs + """ + conn = GetFadConnections() + bads = np.where( conn != good) [0] + return bads + + +def StartUp(): + fad_control.start() + bias_control.reconnect() + bias_control.set_zero_voltage() + while bias_control.stn == 6 : # OverCurrent + bias_control.set_zero_voltage() + bias_control.reset_over_current_status() + + ftm_control.reconnect() + + number_of_loops = 3 + while True: + bad = GetFadBadConnections() + if len(bad) == 0: + break + + for b in set( bad/10 ): # set(b/10) contains the crate ids, which have bad FADs + ResetCrate( b ) + + number_of_loops -= 1 + if not number_of_loops: + print 'StartUp aborted while trying to get the FADs into Connected state ... ' + return False + + + if not MakeDriveCtrlReady(): + return False + + return True + +def Shutdown(): + fad_control.stop() + ftm_control.disconnect() + feedback.stop() + bias_control.set_zero_voltage() + + + +def MakeDriveCtrlReady(): + """ Tries to get Drive ctrl from + Disconnected, locked or Error to Armed + after a few tries it gives up and returns False + """ + outer_counter = 3 + while drive_control.stn != 6: #Armed +# print 'MakeDriveCtrlReady outer_counter', outer_counter + + counter = 3 + while drive_control.stn == 1: #Disconnected +# print 'MakeDriveCtrlReady in while 1, counter:', counter + drive_control.reconnect() + time.sleep(0.5) + counter -= 1 + if not counter: + print 'count not connect drive control to cosy ... please check if cosy is running' + return False + + counter = 3 + while drive_control.stn == 3: # Locked +# print 'MakeDriveCtrlReady in while 2, counter:', counter + drive_control.unlock() + time.sleep(0.5) + counter -= 1 + if not counter: + print 'could not unlock drive control WTF?' + return False + + counter = 3 + while drive_control.stn == 256: # ERROR +# print 'MakeDriveCtrlReady in while 3, counter:', counter + drive_control.stop() + time.sleep(0.5) + counter -= 1 + if not counter: + print 'could not unlock drive control WTF?' + return False + + outer_counter -= 1 + if not outer_counter: + print 'ERROR while trying to get Drive control ready' + return False + + return True + +feedback_calibration_done = None # Bool... + +def Make_Bias_Off(): + + bias_control.set_zero_voltage() + bias_control.wait(7) + + +def Make_Bias_On(): + """ Tries to get the Bias, and Feedback into a defined safe Off state + While beeing as ready as possible to go into On state + """ + global feedback_calibration_done + # Feedback states: + #List of available states: + #-[-1]: NotReady (State machine not ready, events are ignored.) + #-[0]: Ready (State machine ready to receive events.) + #-[1]: DimNetworkNotAvailable (The Dim DNS is not reachable.) + #-[2]: Disconnected (The Dim DNS is reachable, but the required subsystems are not available.) + #-[3]: Connecting (Only biasctrl is available and connected with its hardware.) + #-[4]: ConnectedFSC (biasctrl and fscctrl are available and connected with their hardware.) + #-[5]: ConnectedFAD (biasctrl and fadctrl are available and connected with their hardware.) + #-[6]: Connected (biasctrl, fadctrl and fscctrl are available and connected with their hardware.) + #-[7]: TempCtrlIdle (Temperature control activated, but voltage output disabled.) + #-[8]: FeedbackIdle (Feedback control activated, but voltage output disabled.) + #-[9]: CurrentCtrlIdle (Current control activated, but voltage output disabled.) + #-[10]: TempControl (Temperature control activated and voltage output enabled.) + #-[11]: FeedbackControl (Feedback control activated and voltage output enabled.) + #-[12]: CurrentControl (Current/Temp control activated and voltage output enabled.) + #-[13]: Calibrating (Calibrating current offsets.) + #-[256]: ERROR (Common error state.) + #-[65535]: FATAL (A fatal error occured, the eventloop is stopped.) + + + while not ( (feedback.stn == 12) and + (bias_control.stn == 9) and + (np.array(bias_control.voltage()[0:320]).mean() > 65.) ): + + # for security + feedback.enable_output(0) + + # try to get bias_control into state: VoltageOn + while not bias_control.stn == 9: + + if bias_control.stn == 1: #Disconnected + bias_control.reconnect() + if bias_control.stn == 2: #Connecting + time.sleep(1) + if bias_control.stn == 3: #Initializing + time.sleep(1) + if bias_control.stn == 4: #Connected + # this seems never to happen ... + time.sleep(1) + #bias_control.set_global_dac(1) + if bias_control.stn == 5: #Ramping + time.sleep(1) + if bias_control.stn == 6: #OverCurrent + bias_control.set_zero_voltage() + time.sleep(1) + bias_control.reset_over_current_status() + if bias_control.stn == 7: #VoltageOff + bias_control.set_global_dac(1) + if bias_control.stn == 8: #NotReferenced + bias_control.set_zero_voltage() + if bias_control.stn == 9: #VoltageOn + pass + if bias_control.stn > 9: + print "BIAS control is in strange status... don't know what to do" + time.sleep(1) + time.sleep(1) + + + # Try to get Feedback into CurrentControl + while not (feedback.stn == 12): + + if feedback.stn < 6: # not properly connected. + # State 6 is 'Connected' but this normaly hardly shows up + # Its either voltageOff or any other meaningful state... + print "FEEDBACK not properly connected..." + time.sleep(1) + + if feedback.stn == 13: #Calibrating + time.sleep(1) + continue + + if not feedback_calibration_done: + if feedback.calibration() is None: + print "FEEDBACK has not CURRENT CALIBRATION" + print "TRYING TO CALIBRATE" + feedback.calibrate_currents() + time.sleep(0.5) + continue + else: + feedback_calibration_done = True + + if feedback.stn == 6: #Connected + feedback.start_current_control(0.0) + + if feedback.stn in [7,8]: #TempCtrlIdle , FeedbackIdle + feedback.stop() + time.sleep(0.5) + feedback.start_current_control(0.0) + + if feedback.stn == 9: #CurrentCtrlIdle + feedback.enable_output(1) + + if feedback.stn in [10,11]: #TempControl, FeedbackControl + feedback.stop() + time.sleep(0.5) + feedback.start_current_control(0.0) + + if feedback.stn == 12: # This is where we want ot be + pass + + + time.sleep(1) + # Bias States + #List of available states: + #-[-1]: NotReady (State machine not ready, events are ignored.) + #-[0]: Ready (State machine ready to receive events.) + #-[1]: Disconnected (Bias-power supply not connected via USB.) + #-[2]: Connecting (Trying to establish USB connection to bias-power supply.) + #-[3]: Initializing (USB connection to bias-power supply established, synchronizing USB stream.) + #-[4]: Connected (USB connection to bias-power supply established.) + #-[5]: Ramping (Voltage ramping in progress.) + #-[6]: OverCurrent (At least one channel is in over current state.) + #-[7]: VoltageOff (All voltages are supposed to be switched off.) + #-[8]: NotReferenced (Internal reference voltage does not match last sent voltage.) + #-[9]: VoltageOn (At least one voltage is switched on and all are at reference.) + #-[10]: ExpertMode (Special (risky!) mode to directly send command to the bias-power supply.) + #-[256]: ERROR (Common error state.) + #-[65535]: FATAL (A fatal error occured, the eventloop is stopped.) + + +class CheckBiasThread( threading.Thread ): + """ Thread, which continously monitors the Bias_Control status every + *delay* seconds. + in case of too high currents it will ... + .... do nothing :-) yet. + So far it will just change its internal state from 'Normal' + to 'PreWarning' or 'Serious'. + If the currents remain "high" for more than + *serious_action_after* seconds. some action would be taken. + I our case, just a WARNING will be logged. + + The caller thread should check the *warning_state* from outside this + thread and take apropriate actions, when 'PreWarning' is reached... + + + """ + def __init__( self , delay=5, serious_action_after = 60 ): + #super(CheckBiasThread, self).__init__() + threading.Thread.__init__(self) + self.stoprequest = threading.Event() + self.warning_state = 'Unknown' + self.state_change = time.time() + self.delay = delay + self.serious_action_after = serious_action_after + self.logger = logging.getLogger('PyDimCtrl.CheckBiasThread') + + def run(self): + while not self.stoprequest.isSet(): + + if (self.warning_state == 'Serious' and + time.time()-self.state_change > self.serious_action_after): + # Remergency Ramp Down + self.logger.info('At this moment, I would ramp down the voltage') + self.logger.warning('G-APD currents are too high, since %d seconds' % self.serious_action_after) + self.logger.warning('Emergency voltage shutdown [not yet implemented]') + + currents = GetPixelCurrents() + if ( (np.median(currents) > 70) or (currents.max() > 90) ): + if self.warning_state != 'Serious': + self.logger.info('trasit into Serious Warning') + self.warning_state = 'Serious' + self.state_change = time.time() + + elif ( (np.median(currents) > 60) or (currents.max() > 80) ): + if self.warning_state != 'PreWarning': + self.logger.info('transit into Pre Warning') + self.warning_state = 'PreWarning' + self.state_change = time.time() + + elif ( (np.median(currents) < 60) or (currents.max() < 80) ): + if self.warning_state != 'Normal': + self.logger.info('transit into Normal Situation') + self.warning_state = 'Normal' + self.state_change = time.time() + + if bias_control.stn == 6: #Overcurrent + if self.warning_state != 'OverCurrent': + self.logger.warning('trasit into Overcurrent State during data Taking') + self.warning_state = 'OverCurrent' + self.state_change = time.time() + + if bias_control.stn == 1: #Disconnected + if self.warning_state != 'Disconnected': + self.logger.warning('trasit into Disconnected state during data Taking') + self.warning_state = 'Disconnected' + self.state_change = time.time() + + + + time.sleep( self.delay ) + + def join( self, timeout=None): + self.stoprequest.set() + + self.logger.info('was requested to die with timeout=%s' % str(timeout) ) + #super(CheckBiasThread, self).join() + threading.Thread.join(self) + + +class CheckDriveThread( threading.Thread ): + """ Thread, which continously monitors the Drive_Control status every + *delay* seconds. + in case Drive_Control goes into Error State it will + send: + * STOP + * and then resend the last tracking command + + In case sending STOP is not bringing the Drive_Control back to + state 'Armed' it will try again. Until the *try_counter* is zero. + Then it will totally Freak out. + + .... does nothing really! :-) yet. + + """ + def __init__( self , try_counter = 5, timeout = 30, delay=5 ): + threading.Thread.__init__(self) + self.stoprequest = threading.Event() + + self.error_counter = 0 + self.try_counter = try_counter + self.timeout = timeout + self.delay = delay + + self.warning_state = 'Unknown' + self.state_change = time.time() + + self.logger = logging.getLogger('PyDimCtrl.CheckDriveThread') + + def run(self): + while not self.stoprequest.isSet(): + + if drive_control.stn == 256: #Error + self.error_counter += 1 + self.state_change = time.time() + self.warning_state = 'ErrorFound' + self.logger.info('found Error State ... starting to react on it.') + self.logger.warning(' Error State ') + + start_of_Error_reaction = time.time() + while not drive.stn == 8: #Tracking + if drive_control.stn == 256: #Error + if try_counter: + self.logger.info('sending STOP') + drive_control.stop() + try_counter -= 1 + time.sleep(1) + + elif drive_control.stn == 6: #Armed + kwa = last_drive_kwargs + self.logger.info('resending last tracking command') + last_drive_method(kwa['wobble_offset'], + kwa['wobble_angle'], + kwa['source_name']) + time.sleep(2) + else: + self.logger.error('found Error, but now neither Error state nor Armed state?!') + self.logger.error('what should I do?') + self.logger.error('thread aborting') + self.warning_state = 'Critical' + self.stoprequest.set() + threading.Thread.join(self) + + if time.time() - start_of_Error_reaction > self.timeout: + self.logger.error('timed out while trying to react on Drive Control Error') + self.logger.error('what should I do?') + self.logger.error('thread aborting') + self.warning_state = 'Critical' + #self.stoprequest.set() + #threading.Thread.join(self) + self.join() + + else: + if self.warning_state != 'Normal': + self.logger.info('transit into Normal Situation') + self.warning_state = 'Normal' + self.state_change = time.time() + + time.sleep( self.delay ) + + def join( self, timeout=None): + self.stoprequest.set() + self.logger.info('was requested to die with timeout=%s' % str(timeout)) + threading.Thread.join(self) + + +class BiasThread(threading.Thread): + """ A worker thread that takes directory names from a queue, finds all + files in them recursively and reports the result. + + Input is done by setting the *requested_state*: + ['On', 'Off'] + Output is done by setting the *readyness*: + ['Ready', 'NotYet', 'CantDoIt'] + + Ask the thread to stop by calling its join() method. + """ + requests = ['On', 'Off'] + readynesses = ['Ready', 'NotYet', 'CantDoIt'] + requested_state = None + readyness = 'NotYet' + + def __init__(self, start_state = 'Off'): + super(BiasThread, self).__init__() + BiasThread.requested_state = start_state + self.stoprequest = threading.Event() + + def run(self): + # As long as we weren't asked to stop, try to find out, if we are in + # the requested state, if not, try to reach the state... + while not self.stoprequest.isSet(): + + if not self.InRequestedState(): + print 'not in Requested State', BiasThread.requested_state + BiasThread.readyness = 'NotYet' + + try: + dirname = self.dir_q.get(True, 0.05) + filenames = list(self._files_in_dir(dirname)) + self.result_q.put((self.name, dirname, filenames)) + except Queue.Empty: + continue + + def join(self, timeout=None): + self.stoprequest.set() + super(BiasThread, self).join(timeout) + + def InRequestedState(): + """ + """ + if BiasThread.requested_state == 'On': + pass + elif BiasThread.requested_state == 'Off': + pass + else: # Houston we have a problem! + pass + + + + +class CheckDaqThread( threading.Thread ): + """ Thread, which continously monitors the Rate + + in case the rate is < 1Hz --> Bing bing + + """ + def __init__( self , delay=5 ): + threading.Thread.__init__(self) + self.stoprequest = threading.Event() + + self.delay = delay + + self.warning_state = 'Unknown' + self.state_change = time.time() + + self.logger = logging.getLogger('PyDimCtrl.CheckDaqThread') + + def run(self): + while not self.stoprequest.isSet(): + + rate = ftm_control.trigger_rates()[3] + if rate < 1: + if self.warning_state != 'SmallRate': + self.logger.warning('SmallRate detected!') + self.warning_state = 'SmallRate' + self.state_change = time.time() + else: + if self.warning_state != 'Normal': + self.logger.info('transit to Normal') + self.warning_state = 'Normal' + self.state_change = time.time() + + time.sleep( self.delay ) + + def join( self, timeout=None): + self.stoprequest.set() + self.logger.info('was requested to die with timeout=%s' % str(timeout)) + threading.Thread.join(self) + + + +def off(): + for m in monis: + m.join() + +monis = [] if __name__ == '__main__': print 'Welcome to PyDimCtrl' @@ -1017,3 +1748,11 @@ print - + #bm = CheckBiasThread() + #dm = CheckDriveThread() + #rm = CheckDaqThread() + #monis = [bm, dm, rm] +# bm.start() +# dm.start() +# rm.start() + #print + #print 'monis created'