Index: fact/tools/pyscripts/pyfact/pyfact.py =================================================================== --- fact/tools/pyscripts/pyfact/pyfact.py (revision 13504) +++ fact/tools/pyscripts/pyfact/pyfact.py (revision 13505) @@ -6,4 +6,5 @@ from ctypes import * import numpy as np +import pprint # for SlowData from scipy import signal @@ -380,4 +381,318 @@ # ----------------------------------------------------------------------------- + +class SlowData( fits ): + """ -Fact Fits File- + A Python wrapper for the fits-class implemented in pyfits.h + provides easy access to the fits file meta data. + * dictionary of file metadata - self.meta + * dict of table metadata - self.columns + * variable table column access, thus possibly increased speed while looping + """ + def __init__(self, path): + """ creates meta and columns dictionaries + """ + self.path = path + try: + fits.__init__(self,path) + except IOError: + print 'problem accessing data file: ', data_file_name + raise # stop ! no data + + self.meta = self._make_meta_dict() + self.columns = self._make_columns_dict() + + self.treat_meta_dict() + + + # list of columns, which are already registered + # see method register() + self._registered_cols = [] + # dict of column data, this is used, in order to be able to remove + # the ctypes of + self._table_cols = {} + + # I need to count the rows, since the normal loop mechanism seems not to work. + self._current_row = 0 + + self.stacked_cols = {} + + def _make_meta_dict(self): + """ This method retrieves meta information about the fits file and + stores this information in a dict + return: dict + key: string - all capital letters + value: tuple( numerical value, string comment) + """ + # intermediate variables for file metadata dict generation + keys=self.GetPy_KeyKeys() + values=self.GetPy_KeyValues() + comments=self.GetPy_KeyComments() + types=self.GetPy_KeyTypes() + + if len(keys) != len(values): + raise TypeError('len(keys)',len(keys),' != len(values)', len(values)) + if len(keys) != len(types): + raise TypeError('len(keys)',len(keys),' != len(types)', len(types)) + if len(keys) != len(comments): + raise TypeError('len(keys)',len(keys),' != len(comments)', len(comments)) + + meta_dict = {} + for i in range(len(keys)): + type = types[i] + if type == 'I': + value = int(values[i]) + elif type == 'F': + value = float(values[i]) + elif type == 'B': + if values[i] == 'T': + value = True + elif values[i] == 'F': + value = False + else: + raise TypeError("meta-type is 'B', but meta-value is neither 'T' nor 'F'. meta-value:",values[i]) + elif type == 'T': + value = values[i] + else: + raise TypeError("unknown meta-type: known meta types are: I,F,B and T. meta-type:",type) + meta_dict[keys[i]]=(value, comments[i]) + return meta_dict + + + def _make_columns_dict(self): + """ This method retrieves information about the columns + stored inside the fits files internal binary table. + returns: dict + key: string column name -- all capital letters + values: tuple( + number of elements in table field - integer + size of element in bytes -- this is not really interesting for any user + might be ommited in future versions + type - a single character code -- should be translated into + a comrehensible word + unit - string like 'mV' or 'ADC count' + """ + # intermediate variables for file table-metadata dict generation + keys=self.GetPy_ColumnKeys() + #offsets=self.GetPy_ColumnOffsets() #not needed on python level... + nums=self.GetPy_ColumnNums() + sizes=self.GetPy_ColumnSizes() + types=self.GetPy_ColumnTypes() + units=self.GetPy_ColumnUnits() + + # zip the values + values = zip(nums,sizes,types,units) + # create the columns dictionary + columns = dict(zip(keys ,values)) + return columns + + def stack(self, on=True): + self.next() + for col in self._registered_cols: + if isinstance( self.dict[col], type(np.array('')) ): + self.stacked_cols[col] = self.dict[col] + else: +# elif isinstance(self.dict[col], ctypes._SimpleCData): + self.stacked_cols[col] = np.array(self.dict[col]) +# else: +# raise TypeError("I don't know how to stack "+col+". It is of type: "+str(type(self.dict[col]))) + + def register(self, input_str): + columns = self.columns + if input_str.lower() == 'all': + for col in columns: + self._register(col) + else: + #check if colname is in columns: + if input_str not in columns: + error_msg = 'colname:'+ input_str +' is not a column in the binary table.\n' + error_msg+= 'possible colnames are\n' + for key in columns: + error_msg += key+'\n' + raise KeyError(error_msg) + else: + self._register(input_str) + + # 'private' method, do not use + def _register( self, colname): + columns = self.columns + local = None + + number_of_elements = int(columns[colname][0]) + size_of_elements_in_bytes = int(columns[colname][1]) + ctypecode_of_elements = columns[colname][2] + physical_unit_of_elements = columns[colname][3] + + # snippet from the C++ source code, or header file to be precise: + #case 'L': gLog << "bool(8)"; break; + #case 'B': gLog << "byte(8)"; break; + #case 'I': gLog << "short(16)"; break; + #case 'J': gLog << "int(32)"; break; + #case 'K': gLog << "int(64)"; break; + #case 'E': gLog << "float(32)"; break; + #case 'D': gLog << "double(64)"; break; + + + + # the fields inside the columns can either contain single numbers, + # or whole arrays of numbers as well. + # we treat single elements differently... + if number_of_elements == 1: + # allocate some memory for a single number according to its type + if ctypecode_of_elements == 'J': # J is for a 4byte int, i.e. an unsigned long + local = ctypes.c_ulong() + un_c_type = long + elif ctypecode_of_elements == 'I': # I is for a 2byte int, i.e. an unsinged int + local = ctypes.c_ushort() + un_c_type = int + elif ctypecode_of_elements == 'B': # B is for a byte + local = ctypes.c_ubyte() + un_c_type = int + elif ctypecode_of_elements == 'D': + local = ctypes.c_double() + un_c_type = float + elif ctypecode_of_elements == 'E': + local = ctypes.c_float() + un_c_type = float + elif ctypecode_of_elements == 'A': + local = ctypes.c_uchar() + un_c_type = chr + elif ctypecode_of_elements == 'K': + local = ctypes.c_ulonglong() + un_c_type = long + else: + raise TypeError('unknown ctypecode_of_elements:',ctypecode_of_elements) + else: + if ctypecode_of_elements == 'B': # B is for a byte + nptype = np.int8 + elif ctypecode_of_elements == 'A': # A is for a char .. but I don't know how to handle it + nptype = np.int8 + elif ctypecode_of_elements == 'I': # I is for a 2byte int + nptype = np.int16 + elif ctypecode_of_elements == 'J': # J is for a 4byte int + nptype = np.int32 + elif ctypecode_of_elements == 'K': # B is for a byte + nptype = np.int64 + elif ctypecode_of_elements == 'E': # B is for a byte + nptype = np.float32 + elif ctypecode_of_elements == 'D': # B is for a byte + nptype = np.float64 + else: + raise TypeError('unknown ctypecode_of_elements:',ctypecode_of_elements) + local = np.zeros( number_of_elements, nptype) + + # Set the Pointer Address + self.SetPtrAddress(colname, local) + self._table_cols[colname] = local + if number_of_elements > 1: + self.__dict__[colname] = local + self.dict[colname] = local + else: + # remove any traces of ctypes: + self.__dict__[colname] = local.value + self.dict[colname] = local.value + self._registered_cols.append(colname) + + + def treat_meta_dict(self): + """make 'interesting' meta information available like normal members. + non interesting are: + TFORM, TUNIT, and TTYPE + since these are available via the columns dict. + """ + + self.number_of_rows = self.meta['NAXIS2'][0] + self.number_of_columns = self.meta['TFIELDS'][0] + + # there are some information in the meta dict, which are alsways there: + # there are regarded as not interesting: + uninteresting_meta = {} + uninteresting_meta['arraylike'] = {} + uninteresting = ['NAXIS', 'NAXIS1', 'NAXIS2', + 'TFIELDS', + 'XTENSION','EXTNAME','EXTREL', + 'BITPIX', 'PCOUNT', 'GCOUNT', + 'ORIGIN', + 'PACKAGE', 'COMPILED', 'CREATOR', + 'TELESCOP','TIMESYS','TIMEUNIT','VERSION'] + for key in uninteresting: + if key in self.meta: + uninteresting_meta[key]=self.meta[key] + del self.meta[key] + + # the table meta data contains + + + # shortcut to access the meta dict. But this needs to + # be cleaned up quickly!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + meta = self.meta + + # loop over keys: + # * try to find array-like keys + arraylike = {} + singlelike = [] + for key in self.meta: + stripped = key.rstrip('1234567890') + if stripped == key: + singlelike.append(key) + else: + if stripped not in arraylike: + arraylike[stripped] = 0 + else: + arraylike[stripped] += 1 + newmeta = {} + for key in singlelike: + newmeta[key.lower()] = meta[key] + for key in arraylike: + uninteresting_meta['arraylike'][key.lower()] = [] + for i in range(arraylike[key]+1): + if key+str(i) in meta: + uninteresting_meta['arraylike'][key.lower()].append(meta[key+str(i)]) + self.ui_meta = uninteresting_meta + # make newmeta self + for key in newmeta: + self.__dict__[key]=newmeta[key] + + dict = self.__dict__.copy() + del dict['meta'] + del dict['ui_meta'] + self.dict = dict + + def __iter__(self): + """ iterator """ + return self + + def next(self): + """ used by __iter__ """ + # Here one might check, if looping makes any sense, and if not + # one could stop looping or so... + # like this: + # + # if len(self._registered_cols) == 0: + # print 'warning: looping without any registered columns' + if self._current_row < self.number_of_rows: + if self.GetNextRow() == False: + raise StopIteration + for col in self._registered_cols: + if isinstance(self._table_cols[col], ctypes._SimpleCData): + self.__dict__[col] = self._table_cols[col].value + self.dict[col] = self._table_cols[col].value + + for col in self.stacked_cols: + if isinstance(self.dict[col], type(np.array(''))): + self.stacked_cols[col] = np.vstack( (self.stacked_cols[col],self.dict[col]) ) + else: + self.stacked_cols[col] = np.vstack( (self.stacked_cols[col],np.array(self.dict[col])) ) + self._current_row += 1 + else: + raise StopIteration + return self + + def show(self): + pprint.pprint(self.dict) + + + + class fnames( object ): """ organize file names of a FACT data run @@ -455,4 +770,27 @@ # end of class definition: fnames( object ) +def _test_SlowData( filename ): + file = FactFits(filename) + print '-'*70 + print "opened :", filename, " as 'file'" + print + print '-'*70 + print 'type file.show() to look at its contents' + print "type file.register( columnname ) or file.register('all') in order to register columns" + print + print " due column-registration you declare, that you would like to retrieve the contents of one of the columns" + print " after column-registration, the 'file' has new member variables, they are named like the columns" + print " PLEASE NOTE: immediatly after registration, the members exist, but they are empty." + print " the values are assigned only, when you call file.next() or when you loop over the 'file'" + print + print "in order to loop over it, just go like this:" + print "for row in file:" + print " print row.columnname_one, row.columnname_two" + print + print "" + print '-'*70 + + + def _test_iter( nevents ): """ test for function __iter__ """ @@ -471,4 +809,11 @@ if __name__ == '__main__': """ tests """ - - _test_iter(10) + import sys + if len(sys.argv) == 1: + print 'showing test of iterator of RawData class' + print 'in order to test the SlowData classe please use:', sys.argv[0], 'fits-file-name' + _test_iter(10) + else: + print 'showing test of SlowData class' + print 'in case you wanted to test the RawData class, please give no commandline arguments' + _test_SlowData(sys.argv[1])