#ifndef MARS_fits #define MARS_fits #ifdef __CINT__ #define int8_t Char_t #define int16_t Short_t #define int32_t Int_t #define int64_t Long64_t #define uint8_t UChar_t #define uint16_t UShort_t #define uint32_t UInt_t #define uint64_t ULong64_t #else #include #endif #include #include #include #include #ifdef __EXCEPTIONS #include #endif #ifdef __CINT__ #define off_t size_t #else #include #endif #ifndef __MARS__ #include #include #include #define gLog cerr #define ___err___ "" #define ___warn___ "" #define ___all___ "" #else #include "MLog.h" #include "MLogManip.h" #define ___err___ err #define ___warn___ warn #define ___all___ all #endif #ifdef HAVE_ZLIB #include "izstream.h" #else #include #define izstream ifstream #warning Support for zipped FITS files disabled. #endif #include "checksum.h" #ifndef __MARS__ namespace std { #else using namespace std; #endif class fits : public izstream { public: struct Entry { char type; string value; string comment; template T Get() const { T t; istringstream str(value); str >> t; return t; } }; struct Table { off_t offset; string name; size_t bytes_per_row; size_t num_rows; size_t num_cols; struct Column { size_t offset; size_t num; size_t size; char type; string unit; }; typedef map Keys; typedef map Columns; Columns cols; Keys keys; int64_t datasum; string Trim(const string &str, char c=' ') const { // Trim Both leading and trailing spaces const size_t pstart = str.find_first_not_of(c); // Find the first character position after excluding leading blank spaces const size_t pend = str.find_last_not_of(c); // Find the first character position from reverse af // if all spaces or empty return an empty string if (string::npos==pstart || string::npos==pend) return string(); return str.substr(pstart, pend-pstart+1); } bool Check(const string &key, char type, const string &value="") const { const Keys::const_iterator it = keys.find(key); if (it==keys.end()) { ostringstream str; str << "Key '" << key << "' not found."; #ifdef __EXCEPTIONS throw runtime_error(str.str()); #else gLog << ___err___ << "ERROR - " << str.str() << endl; return false; #endif } if (it->second.type!=type) { ostringstream str; str << "Wrong type for key '" << key << "': expected " << type << ", found " << it->second.type << "."; #ifdef __EXCEPTIONS throw runtime_error(str.str()); #else gLog << ___err___ << "ERROR - " << str.str() << endl; return false; #endif } if (!value.empty() && it->second.value!=value) { ostringstream str; str << "Wrong value for key '" << key << "': expected " << value << ", found " << it->second.value << "."; #ifdef __EXCEPTIONS throw runtime_error(str.str()); #else gLog << ___err___ << "ERROR - " << str.str() << endl; return false; #endif } return true; } Keys ParseBlock(const vector &vec) const { map rc; for (unsigned int i=0; i &vec, off_t off) : offset(off), keys(ParseBlock(vec)) { if (!Check("XTENSION", 'T', "BINTABLE") || !Check("NAXIS", 'I', "2") || !Check("BITPIX", 'I', "8") || !Check("PCOUNT", 'I', "0") || !Check("GCOUNT", 'I', "1") || !Check("EXTNAME", 'T') || !Check("NAXIS1", 'I') || !Check("NAXIS2", 'I') || !Check("TFIELDS", 'I')) return; bytes_per_row = Get("NAXIS1"); num_rows = Get("NAXIS2"); num_cols = Get("TFIELDS"); datasum = Get("DATASUM", -1); size_t bytes = 0; for (size_t i=1; i<=num_cols; i++) { ostringstream num; num << i; if (!Check("TTYPE"+num.str(), 'T') || !Check("TFORM"+num.str(), 'T')) return; const string id = Get("TTYPE"+num.str()); const string fmt = Get("TFORM"+num.str()); const string unit = Get("TUNIT"+num.str(), ""); istringstream sin(fmt); int n = 0; sin >> n; if (!sin) n = 1; const char type = fmt[fmt.length()-1]; size_t size = 0; switch (type) { // We could use negative values to mark floats // otheriwse we could just cast them to int64_t? case 'L': // logical case 'A': // char case 'B': size = 1; break; // byte case 'I': size = 2; break; // short case 'J': size = 4; break; // int case 'K': size = 8; break; // long long case 'E': size = 4; break; // float case 'D': size = 8; break; // double // case 'X': size = n; break; // bits (n=number of bytes needed to contain all bits) // case 'C': size = 8; break; // complex float // case 'M': size = 16; break; // complex double // case 'P': size = 8; break; // array descriptor (32bit) // case 'Q': size = 16; break; // array descriptor (64bit) default: { ostringstream str; str << "FITS format TFORM='" << fmt << "' not yet supported."; #ifdef __EXCEPTIONS throw runtime_error(str.str()); #else gLog << ___err___ << "ERROR - " << str.str() << endl; return; #endif } } const Table::Column col = { bytes, n, size, type, unit }; cols[id] = col; bytes += n*size; } if (bytes!=bytes_per_row) { #ifdef __EXCEPTIONS throw runtime_error("Column size mismatch"); #else gLog << ___err___ << "ERROR - Column size mismatch" << endl; return; #endif } name = Get("EXTNAME"); } void PrintKeys(bool display_all=false) const { for (Keys::const_iterator it=keys.begin(); it!=keys.end(); it++) { if (!display_all && (it->first.substr(0, 6)=="TTYPE" || it->first.substr(0, 6)=="TFORM" || it->first.substr(0, 6)=="TUNIT" || it->first=="TFIELDS" || it->first=="XTENSION" || it->first=="NAXIS" || it->first=="BITPIX" || it->first=="PCOUNT" || it->first=="GCOUNT") ) continue; gLog << ___all___ << setw(2) << it->second.type << '|' << it->first << '=' << it->second.value << '/' << it->second.comment << '|' << endl; }} void PrintColumns() const { typedef map, Column> Sorted; Sorted sorted; for (Columns::const_iterator it=cols.begin(); it!=cols.end(); it++) sorted[make_pair(it->second.offset, it->first)] = it->second; for (Sorted::const_iterator it=sorted.begin(); it!=sorted.end(); it++) { gLog << ___all___ << setw(6) << it->second.offset << "| "; gLog << it->second.num << 'x'; switch (it->second.type) { case 'A': gLog << "char(8)"; break; case 'L': gLog << "bool(8)"; break; case 'B': gLog << "byte(8)"; break; case 'I': gLog << "short(16)"; break; case 'J': gLog << "int(32)"; break; case 'K': gLog << "int(64)"; break; case 'E': gLog << "float(32)"; break; case 'D': gLog << "double(64)"; break; } gLog << ": " << it->first.second << " [" << it->second.unit << "]" << endl; } } operator bool() const { return !name.empty(); } bool HasKey(const string &key) const { return keys.find(key)!=keys.end(); } bool HasColumn(const string& col) const { return cols.find(col)!=cols.end(); } const Columns &GetColumns() const { return cols; } const Keys &GetKeys() const { return keys; } // Values of keys are always signed template T Get(const string &key) const { const map::const_iterator it = keys.find(key); if (it==keys.end()) { ostringstream str; str << "Key '" << key << "' not found." << endl; #ifdef __EXCEPTIONS throw runtime_error(str.str()); #else gLog << ___err___ << "ERROR - " << str.str() << endl; return T(); #endif } return it->second.Get(); } // Values of keys are always signed template T Get(const string &key, const T &deflt) const { const map::const_iterator it = keys.find(key); return it==keys.end() ? deflt :it->second.Get(); } size_t GetN(const string &key) const { const Columns::const_iterator it = cols.find(key); return it==cols.end() ? 0 : it->second.num; } }; private: Table fTable; typedef pair Address; typedef vector
Addresses; //map fAddresses; Addresses fAddresses; #ifdef __MARS__ typedef map Pointers; #else typedef unordered_map Pointers; #endif Pointers fPointers; vector> fGarbage; vector fBufferRow; vector fBufferDat; size_t fRow; Checksum fChkHeader; Checksum fChkData; bool ReadBlock(vector &vec) { int endtag = 0; for (int i=0; i<36; i++) { char c[81]; c[80] = 0; read(c, 80); if (!good()) break; fChkHeader.add(c, 80); // if (c[0]==0) // return vector(); string str(c); // if (!str.empty()) // cout << setw(2) << i << "|" << str << "|" << (endtag?'-':'+') << endl; if (endtag==2 || str=="END ") { endtag = 2; // valid END tag found continue; } if (endtag==1 || str==" ") { endtag = 1; // end tag not found, but expected to be there continue; } vec.push_back(str); } // Make sure that no empty vector is returned if (endtag && vec.size()%36==0) vec.push_back(string("END = '' / ")); return endtag==2; } string Compile(const string &key, int16_t i=-1) { if (i<0) return key; ostringstream str; str << key << i; return str.str(); } public: fits(const string &fname, bool force=false) : izstream(fname.c_str()) { char simple[10]; read(simple, 10); if (!good()) return; if (memcmp(simple, "SIMPLE = ", 10)) { clear(rdstate()|ios::badbit); #ifdef __EXCEPTIONS throw runtime_error("File is not a FITS file."); #else gLog << ___err___ << "ERROR - File is not a FITS file." << endl; return; #endif } seekg(0); while (good()) { vector block; while (1) { // FIXME: Set limit on memory consumption const int rc = ReadBlock(block); if (!good()) { clear(rdstate()|ios::badbit); #ifdef __EXCEPTIONS throw runtime_error("FITS file corrupted."); #else gLog << ___err___ << "ERROR - FITS file corrupted." << endl; return; #endif } if (block.size()%36) { if (!rc && !force) { clear(rdstate()|ios::badbit); #ifdef __EXCEPTIONS throw runtime_error("END keyword missing in FITS header."); #else gLog << ___err___ << "ERROR - END keyword missing in FITS file... file might be corrupted." << endl; return; #endif } break; } } if (block.size()==0) break; if (block[0].substr(0, 9)=="SIMPLE =") { fChkHeader.reset(); continue; } if (block[0].substr(0, 9)=="XTENSION=") { // FIXME: Check for table name fTable = Table(block, tellg()); fRow = (size_t)-1; //fTable.PrintKeys(); if (!fTable) { clear(rdstate()|ios::badbit); return; } fBufferRow.resize(fTable.bytes_per_row + 8-fTable.bytes_per_row%4); fBufferDat.resize(fTable.bytes_per_row); /* // Next table should start at: const size_t size = fTable.bytes_per_row*fTable.num_rows; const size_t blks = size/(36*80); const size_t rest = size%(36*80); seekg((blks+(rest>0?1:0))*(36*80), ios::cur); if (!good()) gLog << ___err___ << "File seems to be incomplete (less data than expected from header)." << endl; fRow = fTable.num_rows; */ break; } } } uint8_t ReadRow(size_t row) { // if (row!=fRow+1) // Fast seeking is ensured by izstream seekg(fTable.offset+row*fTable.bytes_per_row); // For the checksum we need everything to be correctly aligned const uint8_t offset = (row*fTable.bytes_per_row)%4; auto ib = fBufferRow.begin(); auto ie = fBufferRow.end(); *ib++ = 0; *ib++ = 0; *ib++ = 0; *ib = 0; *--ie = 0; *--ie = 0; *--ie = 0; *--ie = 0; *--ie = 0; *--ie = 0; *--ie = 0; *--ie = 0; read(fBufferRow.data()+offset, fTable.bytes_per_row); //fin.clear(fin.rdstate()&~ios::eofbit); if (row==fRow+1) fChkData.add(fBufferRow); fRow = row; return offset; } template void revcpy(char *dest, const char *src, int num) { const char *pend = src + num*N; for (const char *ptr = src; ptr=fTable.num_rows) return false; const uint8_t offset = ReadRow(row); if (!good()) return good(); const char *ptr = fBufferRow.data() + offset; for (Addresses::const_iterator it=fAddresses.begin(); it!=fAddresses.end(); it++) { const Table::Column &c = it->second; const char *src = ptr + c.offset; char *dest = reinterpret_cast(it->first); // Let the compiler do some optimization by // knowing the we only have 1, 2, 4 and 8 switch (c.size) { case 1: memcpy (dest, src, c.num*c.size); break; case 2: revcpy<2>(dest, src, c.num); break; case 4: revcpy<4>(dest, src, c.num); break; case 8: revcpy<8>(dest, src, c.num); break; } } return good(); } bool GetNextRow(bool check=true) { return GetRow(fRow+1, check); } bool SkipNextRow() { seekg(fTable.offset+(++fRow)*fTable.bytes_per_row); return good(); } static bool Compare(const Address &p1, const Address &p2) { return p1.first>p2.first; } template const T &GetAs(const string &name) { return *reinterpret_cast(fPointers[name]); } void *SetPtrAddress(const string &name) { if (fTable.cols.count(name)==0) { ostringstream str; str <<"SetPtrAddress('" << name << "') - Column not found." << endl; #ifdef __EXCEPTIONS throw runtime_error(str.str()); #else gLog << ___err___ << "ERROR - " << str.str() << endl; return NULL; #endif } Pointers::const_iterator it = fPointers.find(name); if (it!=fPointers.end()) return it->second; fGarbage.push_back(vector(fTable.cols[name].size*fTable.cols[name].num)); void *ptr = fGarbage.back().data(); fPointers[name] = ptr; fAddresses.push_back(make_pair(ptr, fTable.cols[name])); sort(fAddresses.begin(), fAddresses.end(), Compare); return ptr; } template bool SetPtrAddress(const string &name, T *ptr, size_t cnt) { if (fTable.cols.count(name)==0) { ostringstream str; str << "SetPtrAddress('" << name << "') - Column not found." << endl; #ifdef __EXCEPTIONS throw runtime_error(str.str()); #else gLog << ___err___ << "ERROR - " << str.str() << endl; return false; #endif } if (sizeof(T)!=fTable.cols[name].size) { ostringstream str; str << "SetPtrAddress('" << name << "') - Element size mismatch: expected " << fTable.cols[name].size << " from header, got " << sizeof(T) << endl; #ifdef __EXCEPTIONS throw runtime_error(str.str()); #else gLog << ___err___ << "ERROR - " << str.str() << endl; return false; #endif } if (cnt!=fTable.cols[name].num) { ostringstream str; str << "SetPtrAddress('" << name << "') - Element count mismatch: expected " << fTable.cols[name].num << " from header, got " << cnt << endl; #ifdef __EXCEPTIONS throw runtime_error(str.str()); #else gLog << ___err___ << "ERROR - " << str.str() << endl; return false; #endif } // if (fAddresses.count(ptr)>0) // gLog << warn << "SetPtrAddress('" << name << "') - Pointer " << ptr << " already assigned." << endl; //fAddresses[ptr] = fTable.cols[name]; fPointers[name] = ptr; fAddresses.push_back(make_pair(ptr, fTable.cols[name])); sort(fAddresses.begin(), fAddresses.end(), Compare); return true; } template bool SetRefAddress(const string &name, T &ptr) { return SetPtrAddress(name, &ptr, sizeof(ptr)/sizeof(T)); } template bool SetVecAddress(const string &name, vector &vec) { return SetPtrAddress(name, vec.data(), vec.size()); } template T Get(const string &key) const { return fTable.Get(key); } template T Get(const string &key, const string &deflt) const { return fTable.Get(key, deflt); } bool SetPtrAddress(const string &name, void *ptr) { if (fTable.cols.count(name)==0) { ostringstream str; str <<"SetPtrAddress('" << name << "') - Column not found." << endl; #ifdef __EXCEPTIONS throw runtime_error(str.str()); #else gLog << ___err___ << "ERROR - " << str.str() << endl; return false; #endif } // if (fAddresses.count(ptr)>0) // gLog << warn << "SetPtrAddress('" << name << "') - Pointer " << ptr << " already assigned." << endl; //fAddresses[ptr] = fTable.cols[name]; fPointers[name] = ptr; fAddresses.push_back(make_pair(ptr, fTable.cols[name])); sort(fAddresses.begin(), fAddresses.end(), Compare); return true; } bool HasKey(const string &key) const { return fTable.HasKey(key); } bool HasColumn(const string& col) const { return fTable.HasColumn(col);} const Table::Columns &GetColumns() const { return fTable.GetColumns();} const Table::Keys &GetKeys() const { return fTable.GetKeys();} int64_t GetInt(const string &key) const { return fTable.Get(key); } uint64_t GetUInt(const string &key) const { return fTable.Get(key); } double GetFloat(const string &key) const { return fTable.Get(key); } string GetStr(const string &key) const { return fTable.Get(key); } size_t GetN(const string &key) const { return fTable.GetN(key); } size_t GetNumRows() const { return fTable.num_rows; } size_t GetRow() const { return fRow==(size_t)-1 ? 0 : fRow; } operator bool() const { return fTable && fTable.offset!=0; } void PrintKeys() const { fTable.PrintKeys(); } void PrintColumns() const { fTable.PrintColumns(); } bool IsHeaderOk() const { return fTable.datasum<0?false:(fChkHeader+Checksum(fTable.datasum)).valid(); } bool IsFileOk() const { return (fChkHeader+fChkData).valid(); } }; #ifndef __MARS__ }; #endif #endif