Index: /fact/tools/pyscripts/examples/result_collection/area_vs_size_with_can.py =================================================================== --- /fact/tools/pyscripts/examples/result_collection/area_vs_size_with_can.py (revision 13113) +++ /fact/tools/pyscripts/examples/result_collection/area_vs_size_with_can.py (revision 13113) @@ -0,0 +1,124 @@ +#!/usr/bin/python -i +# +# Dominik Neise +# +# copy of the /tools/cleaning/avea_vs_size.py script which was originally meant +# as an example for the AmplitudeCleaner. +# +# This script is an example how one can use the python dict, for +# book keeping. +# Note, this script is not yet runnable, since one needs to change +# quite some things in the classes... +# this is just meant to be entertaining + +from pyfact_rename import * +import os.path +import matplotlib.pyplot as plt +import numpy as np +from fir_filter import * +from extractor import * +from drs_spikes import * +from plotters import * +import time as t +from cleaners import AmplitudeCleaner +confirm_next_step = False# this is for user interaction + +import sys +import pickle + +# this little can will hold +# * all the settings, +# * all results and intermediate results +# * as well as this script itself +# and then it will be written, to an output file +can = {} # the FACT can +can['settings'] = {} # a subspace for holdind all settings +can['results'] = {} # a subspace for holding all results... or similar stuff +can['logbook'] = [] # a list of statements ... human readable logbook +can['src'] = [] # this script and other sources + +# store this scipt in the list of sources. +can['src'].append( open(sys.argv[0], 'r').read() ) + +# declare filenames to work with: +can['setting']['data_file_names'] = ['/media/daten_platte/FACT/data/20120229_144.fits.gz'] +can['setting']['calib_file_names'] = ['/media/daten_platte/FACT/data/20120229_132.drs.fits.gz'] +# we even tell the can, where we are going to store it +can['setting']['outfile_name'] = ['./can.pkl'] + +# FileChecker checks if all files exist, and appends this action to the logbook +# if one File does not exist, it will cause this script to end. (...well ... maybe) +FileChecker(can) + +# put the settings for all the extractors and similar data processors +# into the settings +can['settings']['SlidingAverage'] = [8] +can['settings']['DRSSpikes'] = [] +can['settings']['GlobalMaxFinder'] = [40,200] +can['settings']['AmplitudeCleaner'] = [45,18,'return_bool_mask=False'] +# then call the __init__s of the processors and give them the can... +# they will see if they can find their own init parameters ... if not they take +# the default +# in addition, they will add some lines to the logbook telling the reader +# that they were created, and what parameters they found/used. +event = RawDataFeeder(can) +despike = DRSSpikes(can) +smooth = SlidingAverage(can) +extract = GlobalMaxFinder(can) +cleaner = AmplitudeCleaner(can) + +# prepare some space in the can, for the results of this script +can['results']['areas'] = [] +can['results']['sizes'] = [] +# and make some shortcuts +areas = can['results']['areas'] +sizes = can['results']['sizes'] + +# this loop will loop over all events, out of all data files in the can, +# so the analysis loop doesn't even know, there are more files involved. +# of course in the logbook the opening and closing will be stored +for data,startcell,tt in event: + if tt==4: + data = despike(data) + data = smooth(data) + amplitude, time_of_max = extract(data) + survivors = cleaner(amplitude) + + size = 0 + for pixel in survivors: + size += amplitude[pixel] + + if len(survivors) > 0: + areas.append( len(survivors) ) + sizes.append( size ) + # we suddenly realize, that we would like to store + # the event IDs of events, which are have no survivors + # we should scroll up ... before the loop and declare an empty list + # like: can['results']['no_survivor_ids'] = [] + # but imagine we are writing a function, which is beeing called inside the loop + # and this function has no possibility to declare that ... + # well then this function can still put new things inside the can! + else: + if 'no_survivor_ids' not in can['results']: + can['results']['no_survivor_ids'] = [event.event_id.value] + else: + can['results']['no_survivor_ids'].append(event.event_id.value) + + if confirm_next_step: + user_input = raw_input("'q'-quit, 'r'-run, anything else goes one step") + number=None + try: + number=int(user_input) + except: + number=None + if user_input.find('q') != -1: + sys.exit(0) + elif user_input.find('r') != -1: + confirm_next_step = False + elif number!=None: + run += number + +# Now our analysis is done and we save the can +output = open(can['settings']['outfile_name'], 'wb') +pickle.dump(can, output) +output.close() Index: /fact/tools/pyscripts/examples/result_collection/can.pkl =================================================================== --- /fact/tools/pyscripts/examples/result_collection/can.pkl (revision 13113) +++ /fact/tools/pyscripts/examples/result_collection/can.pkl (revision 13113) @@ -0,0 +1,238 @@ +(dp0 +S'src' +p1 +(lp2 +S'#!/usr/bin/python -i\nimport sys\nimport pickle\nimport random\nconfirm_next_step = True\n\n# this little can will hold \n# * all the settings, \n# * all results and intermediate results\n# * as well as this script itself\n# and then it will be written, to an output file\ncan = {} # the FACT can \ncan[\'settings\'] = {} # a subspace for holdind all settings\ncan[\'results\'] = {} # a subspace for holding all results... or similar stuff\ncan[\'logbook\'] = [] # a list of statements ... human readable logbook\ncan[\'src\'] = [] # this script and other sources\n\n# store this scipt in the list of sources.\ncan[\'src\'].append( open(sys.argv[0], \'r\').read() )\n\n# declare filenames to work with:\ncan[\'settings\'][\'data_file_names\'] = [\'/media/daten_platte/FACT/data/20120229_144.fits.gz\']\ncan[\'settings\'][\'calib_file_names\'] = [\'/media/daten_platte/FACT/data/20120229_132.drs.fits.gz\']\n# we even tell the can, where we are going to store it\ncan[\'settings\'][\'outfile_name\'] = \'./can.pkl\' # <-- no list ... simple string\n\n\n# put the settings for all the extractors and similar data processors \n# into the settings\ncan[\'settings\'][\'SlidingAverage\'] = [8]\ncan[\'settings\'][\'DRSSpikes\'] = []\ncan[\'settings\'][\'GlobalMaxFinder\'] = [40,200]\ncan[\'settings\'][\'AmplitudeCleaner\'] = [45,18,\'return_bool_mask=False\']\n\n# prepare some space in the can, for the results of this script\ncan[\'results\'][\'areas\'] = []\ncan[\'results\'][\'sizes\'] = []\n# and make some shortcuts\nareas = can[\'results\'][\'areas\']\nsizes = can[\'results\'][\'sizes\']\n\n# this loop will loop over all events, out of all data files in the can,\n# so the analysis loop doesn\'t even know, there are more files involved.\n# of course in the logbook the opening and closing will be stored\nfor data in range(100):\n if True:\n\n size = 0\n for i in range(data):\n size += data\n \n area = random.randint(0,data)\n if area > 4:\n areas.append( area )\n sizes.append( size )\n # we suddenly realize, that we would like to store\n # the event IDs of events, which are have no survivors\n # we should scroll up ... before the loop and declare an empty list\n # like: can[\'results\'][\'no_survivor_ids\'] = []\n # but imagine we are writing a function, which is beeing called inside the loop\n # and this function has no possibility to declare that ...\n # well then this function can still put new things inside the can!\n else:\n if \'no_survivor_ids\' not in can[\'results\']:\n can[\'results\'][\'no_survivor_ids\'] = [data]\n else:\n can[\'results\'][\'no_survivor_ids\'].append(data)\n \n if confirm_next_step:\n user_input = raw_input("\'q\'-quit, \'r\'-run, anything else goes one step")\n number=None\n try:\n number=int(user_input)\n except:\n number=None\n if user_input.find(\'q\') != -1:\n sys.exit(0)\n elif user_input.find(\'r\') != -1:\n confirm_next_step = False\n elif number!=None:\n run += number\n\n# Now our analysis is done and we save the can\noutput = open(can[\'settings\'][\'outfile_name\'], \'wb\')\npickle.dump(can, output)\noutput.close()' +p3 +asS'logbook' +p4 +(lp5 +sS'results' +p6 +(dp7 +S'sizes' +p8 +(lp9 +I64 +aI121 +aI196 +aI225 +aI256 +aI289 +aI361 +aI400 +aI441 +aI484 +aI625 +aI676 +aI729 +aI784 +aI841 +aI900 +aI961 +aI1024 +aI1089 +aI1156 +aI1225 +aI1296 +aI1369 +aI1444 +aI1521 +aI1600 +aI1681 +aI1849 +aI1936 +aI2025 +aI2116 +aI2209 +aI2304 +aI2401 +aI2704 +aI2809 +aI2916 +aI3025 +aI3136 +aI3364 +aI3481 +aI3600 +aI3721 +aI3844 +aI3969 +aI4096 +aI4225 +aI4356 +aI4489 +aI4624 +aI4761 +aI4900 +aI5041 +aI5184 +aI5329 +aI5476 +aI5625 +aI5776 +aI5929 +aI6084 +aI6241 +aI6400 +aI6561 +aI6889 +aI7056 +aI7225 +aI7396 +aI7569 +aI7744 +aI7921 +aI8100 +aI8281 +aI8464 +aI8649 +aI8836 +aI9025 +aI9216 +aI9409 +aI9604 +aI9801 +asS'no_survivor_ids' +p10 +(lp11 +I0 +aI1 +aI2 +aI3 +aI4 +aI5 +aI6 +aI7 +aI9 +aI10 +aI12 +aI13 +aI18 +aI23 +aI24 +aI42 +aI50 +aI51 +aI57 +aI82 +asS'areas' +p12 +(lp13 +I5 +aI10 +aI10 +aI13 +aI15 +aI6 +aI10 +aI20 +aI13 +aI5 +aI22 +aI20 +aI21 +aI18 +aI14 +aI29 +aI25 +aI21 +aI13 +aI10 +aI12 +aI16 +aI10 +aI28 +aI32 +aI36 +aI12 +aI9 +aI23 +aI43 +aI7 +aI35 +aI42 +aI30 +aI49 +aI26 +aI16 +aI42 +aI10 +aI32 +aI17 +aI13 +aI40 +aI49 +aI24 +aI29 +aI61 +aI65 +aI14 +aI7 +aI37 +aI53 +aI24 +aI57 +aI20 +aI30 +aI19 +aI67 +aI21 +aI75 +aI57 +aI77 +aI35 +aI76 +aI66 +aI55 +aI11 +aI15 +aI73 +aI47 +aI15 +aI81 +aI12 +aI11 +aI16 +aI53 +aI28 +aI86 +aI68 +aI39 +assS'settings' +p14 +(dp15 +S'AmplitudeCleaner' +p16 +(lp17 +I45 +aI18 +aS'return_bool_mask=False' +p18 +asS'outfile_name' +p19 +S'./can.pkl' +p20 +sS'data_file_names' +p21 +(lp22 +S'/media/daten_platte/FACT/data/20120229_144.fits.gz' +p23 +asS'SlidingAverage' +p24 +(lp25 +I8 +asS'calib_file_names' +p26 +(lp27 +S'/media/daten_platte/FACT/data/20120229_132.drs.fits.gz' +p28 +asS'DRSSpikes' +p29 +(lp30 +sS'GlobalMaxFinder' +p31 +(lp32 +I40 +aI200 +ass. Index: /fact/tools/pyscripts/examples/result_collection/canread.py =================================================================== --- /fact/tools/pyscripts/examples/result_collection/canread.py (revision 13113) +++ /fact/tools/pyscripts/examples/result_collection/canread.py (revision 13113) @@ -0,0 +1,31 @@ +#!/usr/bin/python + +# Please read first: +# * readme.1st and +# * area_vs_hist_with_can.py +# * fake_analysis.py +# +# this script is reading the outout of fake analysis.py +# and displaying it +import sys +import pickle + +pklfile = open('can.pkl', 'rb') +can = pickle.load(pklfile) + +# now we display some things +print 'can contents' +for key in can.keys(): + print key, type(can[key]) + if type(can[key]) == type(dict()): + print '-->' + for k in can[key].keys(): + print ' ',k, type(can[key][k]), + if type(can[key][k])==type(list()): + print len(can[key][k]) + else: + print + + print '<--' + + Index: /fact/tools/pyscripts/examples/result_collection/fake_analysis.py =================================================================== --- /fact/tools/pyscripts/examples/result_collection/fake_analysis.py (revision 13113) +++ /fact/tools/pyscripts/examples/result_collection/fake_analysis.py (revision 13113) @@ -0,0 +1,97 @@ +#!/usr/bin/python -i + +# Please read first: +# * readme.1st and +# * area_vs_hist_with_can.py +# +# this scipt is just a funny test, how such a can, might be filled during an +# analysis ... +# it is possible to execute this script +# +# the output can be read in, and will be displayed by +# canread.py +import sys +import pickle +import random +confirm_next_step = False + +# this little can will hold +# * all the settings, +# * all results and intermediate results +# * as well as this script itself +# and then it will be written, to an output file +can = {} # the FACT can +can['settings'] = {} # a subspace for holdind all settings +can['results'] = {} # a subspace for holding all results... or similar stuff +can['logbook'] = [] # a list of statements ... human readable logbook +can['src'] = [] # this script and other sources + +# store this scipt in the list of sources. +can['src'].append( open(sys.argv[0], 'r').read() ) + +# declare filenames to work with: +can['settings']['data_file_names'] = ['/media/daten_platte/FACT/data/20120229_144.fits.gz'] +can['settings']['calib_file_names'] = ['/media/daten_platte/FACT/data/20120229_132.drs.fits.gz'] +# we even tell the can, where we are going to store it +can['settings']['outfile_name'] = './can.pkl' # <-- no list ... simple string + + +# put the settings for all the extractors and similar data processors +# into the settings +can['settings']['SlidingAverage'] = [8] +can['settings']['DRSSpikes'] = [] +can['settings']['GlobalMaxFinder'] = [40,200] +can['settings']['AmplitudeCleaner'] = [45,18,'return_bool_mask=False'] + +# prepare some space in the can, for the results of this script +can['results']['areas'] = [] +can['results']['sizes'] = [] +# and make some shortcuts +areas = can['results']['areas'] +sizes = can['results']['sizes'] + +# this loop will loop over all events, out of all data files in the can, +# so the analysis loop doesn't even know, there are more files involved. +# of course in the logbook the opening and closing will be stored +for data in range(100): + if True: + + size = 0 + for i in range(data): + size += data + + area = random.randint(0,data) + if area > 4: + areas.append( area ) + sizes.append( size ) + # we suddenly realize, that we would like to store + # the event IDs of events, which are have no survivors + # we should scroll up ... before the loop and declare an empty list + # like: can['results']['no_survivor_ids'] = [] + # but imagine we are writing a function, which is beeing called inside the loop + # and this function has no possibility to declare that ... + # well then this function can still put new things inside the can! + else: + if 'no_survivor_ids' not in can['results']: + can['results']['no_survivor_ids'] = [data] + else: + can['results']['no_survivor_ids'].append(data) + + if confirm_next_step: + user_input = raw_input("'q'-quit, 'r'-run, anything else goes one step") + number=None + try: + number=int(user_input) + except: + number=None + if user_input.find('q') != -1: + sys.exit(0) + elif user_input.find('r') != -1: + confirm_next_step = False + elif number!=None: + run += number + +# Now our analysis is done and we save the can +output = open(can['settings']['outfile_name'], 'wb') +pickle.dump(can, output) +output.close() Index: /fact/tools/pyscripts/examples/result_collection/readme.1st =================================================================== --- /fact/tools/pyscripts/examples/result_collection/readme.1st (revision 13113) +++ /fact/tools/pyscripts/examples/result_collection/readme.1st (revision 13113) @@ -0,0 +1,66 @@ +what is this about? + +Werner mentioned at some point, that we need some way to collect the results of +our analysis transparently, while the analysis in on going. +I do not mean only the analysis you all know, which aims +for getting some theta^2 plot or some light curve, but I mean all different kinds +of information, which might be used for understanding some features. + +Let me give a small example: +Assume we want to anlyse some time of data, say 4 runs. +Usually you have then the following files: + * N data files + * 1 pedestal file + * 1 LP file + * DRS amplitude calibration files + * DRS time calibration files + * several slow control files + * 1 closed shutter run, for gain analysis +but depending on what kind of analysis you do, some if these files might miss. +You might not even have physics data, since you analyse LP data at the moment. + +And using this comparably large and complicated number of files containing +different information, you plan to retrieve different kinds of information +such as: + * the behaviour of the baseline, using chunks of data of say 500 events + * the behaviour of the interleaved LP data + * image parameters from the physics events + - using the information from interleaved ped and LP events *and* + - using the information of the dedicated ped and LP runs + * ... + +And in order to start your analysis, you might apply some of the basic classes +we currently have at hand. +Assume you: + * apply DRS-amplitude calibration & spike removal + * do some sort of filtering of the data + * test some sort of signal extraction, which might need some + special information from the dedicated pedestal run for example. + +And assume you aim to test, how the signal extraction behaves, if the +way you calculated the pedestal changes. +I guess, at this point you would be glad to have some transparent way of +treating all these intermediate results. +And in the end, you might want to print some sort of + "what files and classes did I use, in order to generate this plot" + +If you could get this, without noting it in your laboratory notebook, +but rather get it automatically, because the results, and the way you got them +store themselves somewhere, I guess you would be glad. + +At the time beeing(14.03.12) we are a bit bound to do all these analyses apart from +each other and using the results of say some baseline analysis steps is not yet +included nicely into analysis steps, which might follow. + +And doing a special and maybe complicated analysis in several small steps, +is not a bad thing. So it would be nice, to have some means of tracking what +was done in the entire analysis, and also to have some way of collecting all the +results and also intermediate results. Which are needed for debugging of course. + + +So I made up my mind, and I think there is nothing which has to be written +or developed. The normal standard dict class of python is all we need. +It is simple to use. Flexible enough for storing whatever one likes. +And can be stored to a file using the pickle and unpickle classes. + +