Changeset 13505 for fact/tools/pyscripts/pyfact/pyfact.py

Timestamp:

05/02/12 12:53:13 (13 years ago)

Author:

neise

Message:

added class SlowData. This class was formerly known as FactFits in module factfits.py

File:

• fact/tools/pyscripts/pyfact/pyfact.py (modified) (4 diffs)

fact/tools/pyscripts/pyfact/pyfact.py

@@ -6 +6 @@
 from ctypes import *
 import numpy as np
+import pprint # for SlowData
 from scipy import signal
 
@@ -380 +381 @@
         
 # -----------------------------------------------------------------------------
+
+class SlowData( fits ):
+    """ -Fact Fits File-
+        A Python wrapper for the fits-class implemented in pyfits.h
+        provides easy access to the fits file meta data.
+        * dictionary of file metadata - self.meta
+        * dict of table metadata - self.columns
+        * variable table column access, thus possibly increased speed while looping
+    """
+    def __init__(self, path):
+        """ creates meta and columns dictionaries
+        """
+        self.path = path
+        try:
+            fits.__init__(self,path)
+        except IOError:
+            print 'problem accessing data file: ', data_file_name
+            raise  # stop ! no data
+        
+        self.meta = self._make_meta_dict()
+        self.columns = self._make_columns_dict()
+        
+        self.treat_meta_dict()
+        
+        
+        # list of columns, which are already registered
+        # see method register()
+        self._registered_cols = []
+        # dict of column data, this is used, in order to be able to remove
+        # the ctypes of 
+        self._table_cols = {}
+        
+        # I need to count the rows, since the normal loop mechanism seems not to work.
+        self._current_row = 0
+        
+        self.stacked_cols = {}
+
+    def _make_meta_dict(self):
+        """ This method retrieves meta information about the fits file and 
+            stores this information in a dict
+            return: dict
+                key: string - all capital letters
+                value: tuple( numerical value, string comment)
+        """
+        # intermediate variables for file metadata dict generation
+        keys=self.GetPy_KeyKeys()
+        values=self.GetPy_KeyValues()
+        comments=self.GetPy_KeyComments()
+        types=self.GetPy_KeyTypes()
+        
+        if len(keys) != len(values):
+            raise TypeError('len(keys)',len(keys),' != len(values)', len(values))
+        if len(keys) != len(types):
+            raise TypeError('len(keys)',len(keys),' != len(types)', len(types))
+        if len(keys) != len(comments):
+            raise TypeError('len(keys)',len(keys),' != len(comments)', len(comments))
+        
+        meta_dict = {}
+        for i in range(len(keys)):
+            type = types[i]
+            if type == 'I':
+                value = int(values[i])
+            elif type == 'F':
+                value = float(values[i])
+            elif type == 'B':
+                if values[i] == 'T':
+                    value = True
+                elif values[i] == 'F':
+                    value = False
+                else:
+                    raise TypeError("meta-type is 'B', but meta-value is neither 'T' nor 'F'. meta-value:",values[i])
+            elif type == 'T':
+                value = values[i]
+            else:
+                raise TypeError("unknown meta-type: known meta types are: I,F,B and T. meta-type:",type)
+            meta_dict[keys[i]]=(value, comments[i])
+        return meta_dict
+
+
+    def _make_columns_dict(self):
+        """ This method retrieves information about the columns
+            stored inside the fits files internal binary table.
+            returns: dict
+                key:    string column name -- all capital letters
+                values: tuple(
+                    number of elements in table field - integer
+                    size of element in bytes -- this is not really interesting for any user
+                            might be ommited in future versions
+                    type - a single character code -- should be translated into 
+                            a comrehensible word
+                    unit - string like 'mV' or 'ADC count'
+        """
+        # intermediate variables for file table-metadata dict generation
+        keys=self.GetPy_ColumnKeys()
+        #offsets=self.GetPy_ColumnOffsets() #not needed on python level...
+        nums=self.GetPy_ColumnNums()
+        sizes=self.GetPy_ColumnSizes()
+        types=self.GetPy_ColumnTypes()
+        units=self.GetPy_ColumnUnits()
+    
+        # zip the values
+        values = zip(nums,sizes,types,units)
+        # create the columns dictionary
+        columns = dict(zip(keys ,values))
+        return columns
+
+    def stack(self, on=True):
+        self.next()
+        for col in self._registered_cols:
+            if isinstance( self.dict[col], type(np.array('')) ):
+                self.stacked_cols[col] = self.dict[col]
+            else:
+#            elif isinstance(self.dict[col], ctypes._SimpleCData):
+                self.stacked_cols[col] = np.array(self.dict[col])
+#            else:
+#                raise TypeError("I don't know how to stack "+col+". It is of type: "+str(type(self.dict[col])))
+    
+    def register(self, input_str):
+        columns = self.columns
+        if input_str.lower() == 'all':
+            for col in columns:
+                self._register(col)
+        else:
+            #check if colname is in columns:
+            if input_str not in columns:
+                error_msg = 'colname:'+ input_str +' is not a column in the binary table.\n'
+                error_msg+= 'possible colnames are\n'
+                for key in columns:
+                    error_msg += key+'\n'
+                raise KeyError(error_msg)
+            else:
+                self._register(input_str)
+
+    # 'private' method, do not use
+    def _register( self, colname):
+        columns = self.columns
+        local = None
+        
+        number_of_elements = int(columns[colname][0])
+        size_of_elements_in_bytes = int(columns[colname][1])
+        ctypecode_of_elements = columns[colname][2]
+        physical_unit_of_elements = columns[colname][3]
+        
+        # snippet from the C++ source code, or header file to be precise:
+        #case 'L': gLog << "bool(8)";    break;
+        #case 'B': gLog << "byte(8)";    break;
+        #case 'I': gLog << "short(16)";  break;
+        #case 'J': gLog << "int(32)";    break;
+        #case 'K': gLog << "int(64)";    break;
+        #case 'E': gLog << "float(32)";  break;
+        #case 'D': gLog << "double(64)"; break;
+
+        
+        
+        # the fields inside the columns can either contain single numbers,
+        # or whole arrays of numbers as well.
+        # we treat single elements differently...
+        if number_of_elements == 1:
+            # allocate some memory for a single number according to its type
+            if ctypecode_of_elements == 'J':  # J is for a 4byte int, i.e. an unsigned long
+                local = ctypes.c_ulong()
+                un_c_type = long
+            elif ctypecode_of_elements == 'I':  # I is for a 2byte int, i.e. an unsinged int
+                local = ctypes.c_ushort()
+                un_c_type = int
+            elif ctypecode_of_elements == 'B':  # B is for a byte
+                local = ctypes.c_ubyte()
+                un_c_type = int
+            elif ctypecode_of_elements == 'D':
+                local = ctypes.c_double()
+                un_c_type = float
+            elif ctypecode_of_elements == 'E':
+                local = ctypes.c_float()
+                un_c_type = float
+            elif ctypecode_of_elements == 'A':
+                local = ctypes.c_uchar()
+                un_c_type = chr
+            elif ctypecode_of_elements == 'K':
+                local = ctypes.c_ulonglong()
+                un_c_type = long
+            else:
+                raise TypeError('unknown ctypecode_of_elements:',ctypecode_of_elements)
+        else:
+            if ctypecode_of_elements == 'B':  # B is for a byte
+                nptype = np.int8
+            elif ctypecode_of_elements == 'A':  # A is for a char .. but I don't know how to handle it
+                nptype = np.int8
+            elif ctypecode_of_elements == 'I':  # I is for a 2byte int
+                nptype = np.int16
+            elif ctypecode_of_elements == 'J':  # J is for a 4byte int
+                nptype = np.int32
+            elif ctypecode_of_elements == 'K':  # B is for a byte
+                nptype = np.int64
+            elif ctypecode_of_elements == 'E':  # B is for a byte
+                nptype = np.float32
+            elif ctypecode_of_elements == 'D':  # B is for a byte
+                nptype = np.float64
+            else:
+                raise TypeError('unknown ctypecode_of_elements:',ctypecode_of_elements)
+            local = np.zeros( number_of_elements, nptype)
+        
+        # Set the Pointer Address 
+        self.SetPtrAddress(colname, local)
+        self._table_cols[colname] = local
+        if number_of_elements > 1:
+            self.__dict__[colname] = local
+            self.dict[colname] = local
+        else:
+            # remove any traces of ctypes:
+            self.__dict__[colname] = local.value
+            self.dict[colname] = local.value
+        self._registered_cols.append(colname)
+        
+    
+    def treat_meta_dict(self):
+        """make 'interesting' meta information available like normal members.
+            non interesting are:
+            TFORM, TUNIT, and TTYPE
+            since these are available via the columns dict.
+        """
+        
+        self.number_of_rows = self.meta['NAXIS2'][0]
+        self.number_of_columns = self.meta['TFIELDS'][0]
+
+        # there are some information in the meta dict, which are alsways there:
+        # there are regarded as not interesting:
+        uninteresting_meta = {}
+        uninteresting_meta['arraylike'] = {}
+        uninteresting = ['NAXIS', 'NAXIS1', 'NAXIS2',
+                        'TFIELDS',
+                        'XTENSION','EXTNAME','EXTREL',
+                        'BITPIX', 'PCOUNT', 'GCOUNT',
+                        'ORIGIN',
+                        'PACKAGE', 'COMPILED', 'CREATOR',
+                        'TELESCOP','TIMESYS','TIMEUNIT','VERSION']
+        for key in uninteresting:
+            if key in self.meta:
+                uninteresting_meta[key]=self.meta[key]
+                del self.meta[key]
+        
+        # the table meta data contains 
+        
+        
+        # shortcut to access the meta dict. But this needs to 
+        # be cleaned up quickly!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+        meta = self.meta
+        
+        # loop over keys:
+        #   * try to find array-like keys
+        arraylike = {}
+        singlelike = []
+        for key in self.meta:
+            stripped = key.rstrip('1234567890')
+            if stripped == key:
+                singlelike.append(key)
+            else:
+                if stripped not in arraylike:
+                    arraylike[stripped] = 0
+                else:
+                    arraylike[stripped] += 1
+        newmeta = {}
+        for key in singlelike:
+            newmeta[key.lower()] = meta[key]
+        for key in arraylike:
+            uninteresting_meta['arraylike'][key.lower()] = []
+            for i in range(arraylike[key]+1):
+                if key+str(i) in meta:
+                    uninteresting_meta['arraylike'][key.lower()].append(meta[key+str(i)])
+        self.ui_meta = uninteresting_meta
+        # make newmeta self
+        for key in newmeta:
+            self.__dict__[key]=newmeta[key]
+        
+        dict = self.__dict__.copy()
+        del dict['meta']
+        del dict['ui_meta']
+        self.dict = dict
+
+    def __iter__(self):
+        """ iterator """
+        return self
+
+    def next(self):
+        """ used by __iter__ """
+        # Here one might check, if looping makes any sense, and if not
+        # one could stop looping or so...
+        # like this:
+        #
+        # if len(self._registered_cols) == 0:
+        #   print 'warning: looping without any registered columns'
+        if self._current_row < self.number_of_rows:
+            if self.GetNextRow() == False:
+                raise StopIteration
+            for col in self._registered_cols:
+                if isinstance(self._table_cols[col], ctypes._SimpleCData):
+                    self.__dict__[col] = self._table_cols[col].value
+                    self.dict[col] = self._table_cols[col].value
+                    
+            for col in self.stacked_cols:
+                if isinstance(self.dict[col], type(np.array(''))):
+                    self.stacked_cols[col] = np.vstack( (self.stacked_cols[col],self.dict[col]) ) 
+                else:
+                    self.stacked_cols[col] = np.vstack( (self.stacked_cols[col],np.array(self.dict[col])) )
+            self._current_row += 1
+        else: 
+            raise StopIteration
+        return self
+
+    def show(self):
+        pprint.pprint(self.dict)
+
+
+
+
 class fnames( object ):
     """ organize file names of a FACT data run
@@ -455 +770 @@
 # end of class definition: fnames( object )
 
+def _test_SlowData( filename ):
+    file = FactFits(filename)
+    print '-'*70
+    print "opened :", filename, " as 'file'"
+    print 
+    print '-'*70
+    print 'type file.show() to look at its contents'
+    print "type file.register( columnname ) or file.register('all') in order to register columns"
+    print 
+    print "   due column-registration you declare, that you would like to retrieve the contents of one of the columns"
+    print "   after column-registration, the 'file' has new member variables, they are named like the columns"
+    print "   PLEASE NOTE: immediatly after registration, the members exist, but they are empty."
+    print "   the values are assigned only, when you call file.next() or when you loop over the 'file'"
+    print
+    print "in order to loop over it, just go like this:"
+    print "for row in file:"
+    print "    print row.columnname_one, row.columnname_two"
+    print
+    print ""
+    print '-'*70
+    
+
+
 def _test_iter( nevents ):
     """ test for function __iter__ """
@@ -471 +809 @@
 if __name__ == '__main__':
     """ tests  """
-
-    _test_iter(10)
+    import sys
+    if len(sys.argv) == 1:
+        print 'showing test of iterator of RawData class'
+        print 'in order to test the SlowData classe please use:', sys.argv[0], 'fits-file-name'
+        _test_iter(10)
+    else:
+        print 'showing test of SlowData class'
+        print 'in case you wanted to test the RawData class, please give no commandline arguments'
+        _test_SlowData(sys.argv[1])