Context Navigation

source: fact/tools/pyscripts/pyfact/fits.h@ 17689

Visit:

Last change on this file since 17689 was 17687, checked in by dneise, 10 years ago
differs from Mars/mcore only in special python accessors like GetPy_KeyKeys and similar.
Property svn:executable set to ``*
File size: 35.1 KB

Line
1	#ifndef MARS_fits
2	#define MARS_fits
3
4	#include <stdint.h>
5
6	#include <map>
7	#include <string>
8	#include <fstream>
9	#include <sstream>
10	#include <algorithm>
11	#include <stdexcept>
12
13	#define GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__)
14
15	#ifndef __CINT__
16	#include <unordered_map>
17	#else
18	#define off_t size_t
19	namespace std
20	{
21	template<class T, class S> class unordered_map<T, S>;
22	}
23	#endif
24
25	#ifndef __MARS__
26	#include <vector>
27	#include <iomanip>
28	#include <iostream>
29	#ifndef gLog
30	#define gLog std::cerr
31	#define ___err___ ""
32	#define ___warn___ ""
33	#define ___all___ ""
34	#endif
35	#else
36	#include "MLog.h"
37	#include "MLogManip.h"
38	#define ___err___ err
39	#define ___warn___ warn
40	#define ___all___ all
41	#endif
42
43	#define HAVE_ZLIB
44	#if defined(HAVE_ZLIB) \|\| defined(__CINT__)
45	#include "extern_Mars_mcore/izstream.h"
46	#else
47	#include <fstream>
48	#define izstream ifstream
49	#warning Support for zipped FITS files disabled.
50	#endif
51
52	#include "extern_Mars_mcore/checksum.h"
53
54	class fits : public izstream
55	{
56	public:
57	//I know I know, you're going to yiell that this does not belong here.
58	//It will belong in the global scope eventually, and it makes the coding of zfits much simpler this way.
59	enum Compression_t
60	{
61	kCompUnknown,
62	kCompFACT
63	};
64
65	struct Entry
66	{
67	char type;
68	std::string value;
69	std::string comment;
70	std::string fitsString;
71
72	template<typename T>
73	T Get() const
74	{
75	T t;
76
77	std::istringstream str(value);
78	str >> t;
79
80	return t;
81	}
82	};
83
84	struct Table
85	{
86	off_t offset;
87
88	bool is_compressed;
89
90	std::string name;
91	size_t bytes_per_row;
92	size_t num_rows;
93	size_t num_cols;
94	size_t total_bytes; // NAXIS1*NAXIS2
95
96	struct Column
97	{
98	size_t offset;
99	size_t num;
100	size_t size;
101	size_t bytes; // num*size
102	char type;
103	std::string unit;
104	Compression_t comp;
105	};
106
107	typedef std::map<std::string, Entry> Keys;
108	typedef std::map<std::string, Column> Columns;
109	typedef std::vector<Column> SortedColumns;
110
111	Columns cols;
112	SortedColumns sorted_cols;
113	Keys keys;
114
115	//------------ vectors containing the same info as cols and keys------
116	// They are filled at the end of the contstuctor
117	// They are used py python, for retrieving the contents of the maps
118	// I did not find out how to return a map of <string, struct> via ROOT to python
119	// So I use this ugly hack. I guess its no problem, its just ugly
120	//
121	// btw: The way these variables are named, is imho as ugly as possible,
122	// but I will stick to it.
123	// usually a std::map contains key-value pairs
124	// that means the e.g. map<string, Entry> contains keys of type string
125	// and values of type Entry.
126	// but now the map itself was called 'keys', which means one should say:
127	// the keys of 'keys' are of type string
128	// and
129	// the value of 'keys' are of type Entry.
130	// not to forget that:
131	// one field of a value of 'keys' is named 'value'
132
133	// vectors for Keys keys;
134	vector<string> Py_KeyKeys;
135	vector<string> Py_KeyValues;
136	vector<string> Py_KeyComments;
137	vector<char> Py_KeyTypes;
138
139	// vectors for Columns cols;
140	vector<string> Py_ColumnKeys;
141	vector<size_t> Py_ColumnOffsets;
142	vector<size_t> Py_ColumnNums;
143	vector<size_t> Py_ColumnSizes;
144	vector<char> Py_ColumnTypes;
145	vector<string> Py_ColumnUnits;
146	//-end of----- vectors containing the same info as cols and keys------
147
148
149	int64_t datasum;
150
151	std::string Trim(const std::string &str, char c=' ') const
152	{
153	// Trim Both leading and trailing spaces
154	const size_t pstart = str.find_first_not_of(c); // Find the first character position after excluding leading blank spaces
155	const size_t pend = str.find_last_not_of(c); // Find the first character position from reverse af
156
157	// if all spaces or empty return an empty string
158	if (std::string::npos==pstart \|\| std::string::npos==pend)
159	return std::string();
160
161	return str.substr(pstart, pend-pstart+1);
162	}
163
164	bool Check(const std::string &key, char type, const std::string &value="") const
165	{
166	const Keys::const_iterator it = keys.find(key);
167	if (it==keys.end())
168	{
169	std::ostringstream str;
170	str << "Key '" << key << "' not found.";
171	#ifdef __EXCEPTIONS
172	throw std::runtime_error(str.str());
173	#else
174	gLog << ___err___ << "ERROR - " << str.str() << std::endl;
175	return false;
176	#endif
177	}
178
179	if (it->second.type!=type)
180	{
181	std::ostringstream str;
182	str << "Wrong type for key '" << key << "': expected " << type << ", found " << it->second.type << ".";
183	#ifdef __EXCEPTIONS
184	throw std::runtime_error(str.str());
185	#else
186	gLog << ___err___ << "ERROR - " << str.str() << std::endl;
187	return false;
188	#endif
189	}
190
191	if (!value.empty() && it->second.value!=value)
192	{
193	std::ostringstream str;
194	str << "Wrong value for key '" << key << "': expected " << value << ", found " << it->second.value << ".";
195	#ifdef __EXCEPTIONS
196	throw std::runtime_error(str.str());
197	#else
198	gLog << ___err___ << "ERROR - " << str.str() << std::endl;
199	return false;
200	#endif
201	}
202
203	return true;
204	}
205
206	Keys ParseBlock(const std::vector<std::string> &vec) const
207	{
208	std::map<std::string,Entry> rc;
209
210	for (unsigned int i=0; i<vec.size(); i++)
211	{
212	const std::string key = Trim(vec[i].substr(0,8));
213	// Keywords without a value, like COMMENT / HISTORY
214	if (vec[i].substr(8,2)!="= ")
215	continue;
216
217	char type = 0;
218
219	std::string com;
220	std::string val = Trim(vec[i].substr(10));
221
222	if (val[0]=='\'')
223	{
224	// First skip all '' in the string
225	size_t p = 1;
226	while (1)
227	{
228	const size_t pp = val.find_first_of('\'', p);
229	if (pp==std::string::npos)
230	break;
231
232	p = val[pp+1]=='\'' ? pp+2 : pp+1;
233	}
234
235	// Now find the comment
236	const size_t ppp = val.find_first_of('/', p);
237
238	// Set value, comment and type
239	// comments could be just spaces. take care of this.
240	if (ppp!=std::string::npos && val.size() != ppp+1)
241	com = Trim(val.substr(ppp+1));
242
243	val = Trim(val.substr(1, p-2));
244	type = 'T';
245	}
246	else
247	{
248	const size_t p = val.find_first_of('/');
249
250	if (val.size() != p+1)
251	com = Trim(val.substr(p+2));
252
253	val = Trim(val.substr(0, p));
254
255	if (val.empty() \|\| val.find_first_of('T')!=std::string::npos \|\| val.find_first_of('F')!=std::string::npos)
256	type = 'B';
257	else
258	type = val.find_last_of('.')==std::string::npos ? 'I' : 'F';
259	}
260
261	const Entry e = { type, val, com, vec[i] };
262
263	rc[key] = e;
264	}
265
266	return rc;
267	}
268
269	Table() : offset(0), is_compressed(false) { }
270	Table(const std::vector<std::string> &vec, off_t off) : offset(off),
271	keys(ParseBlock(vec))
272	{
273	is_compressed = HasKey("ZTABLE") && Check("ZTABLE", 'B', "T");
274
275	if (!Check("XTENSION", 'T', "BINTABLE") \|\|
276	!Check("NAXIS", 'I', "2") \|\|
277	!Check("BITPIX", 'I', "8") \|\|
278	!Check("GCOUNT", 'I', "1") \|\|
279	!Check("EXTNAME", 'T') \|\|
280	!Check("NAXIS1", 'I') \|\|
281	!Check("NAXIS2", 'I') \|\|
282	!Check("TFIELDS", 'I'))
283	return;
284
285	if (is_compressed)
286	{
287	if (!Check("ZNAXIS1", 'I') \|\|
288	!Check("ZNAXIS2", 'I') \|\|
289	!Check("ZPCOUNT", 'I', "0"))
290	return;
291	}
292	else
293	{
294	if (!Check("PCOUNT", 'I', "0"))
295	return;
296	}
297
298	total_bytes = Get<size_t>("NAXIS1")*Get<size_t>("NAXIS2");
299	bytes_per_row = is_compressed ? Get<size_t>("ZNAXIS1") : Get<size_t>("NAXIS1");
300	num_rows = is_compressed ? Get<size_t>("ZNAXIS2") : Get<size_t>("NAXIS2");
301	num_cols = Get<size_t>("TFIELDS");
302	datasum = is_compressed ? Get<int64_t>("DATASUM", -1) : Get<int64_t>("DATASUM", -1);
303	//cout << "IS COMPRESSED =-========= " << is_compressed << " " << Get<size_t>("NAXIS1") << endl;
304	size_t bytes = 0;
305
306	const std::string tFormName = is_compressed ? "ZFORM" : "TFORM";
307	for (long long unsigned int i=1; i<=num_cols; i++)
308	{
309	const std::string num(std::to_string(i));
310
311	if (!Check("TTYPE"+num, 'T') \|\|
312	!Check(tFormName+num, 'T'))
313	return;
314
315	const std::string id = Get<std::string>("TTYPE"+num);
316	const std::string fmt = Get<std::string>(tFormName+num);
317	const std::string unit = Get<std::string>("TUNIT"+num, "");
318	const std::string comp = Get<std::string>("ZCTYP"+num, "");
319
320	Compression_t compress = kCompUnknown;
321	if (comp == "FACT")
322	compress = kCompFACT;
323
324	std::istringstream sin(fmt);
325	int n = 0;
326	sin >> n;
327	if (!sin)
328	n = 1;
329
330	const char type = fmt[fmt.length()-1];
331
332	size_t size = 0;
333	switch (type)
334	{
335	// We could use negative values to mark floats
336	// otheriwse we could just cast them to int64_t?
337	case 'L': // logical
338	case 'A': // char
339	case 'B': size = 1; break; // byte
340	case 'I': size = 2; break; // short
341	case 'J': size = 4; break; // int
342	case 'K': size = 8; break; // long long
343	case 'E': size = 4; break; // float
344	case 'D': size = 8; break; // double
345	// case 'X': size = n; break; // bits (n=number of bytes needed to contain all bits)
346	// case 'C': size = 8; break; // complex float
347	// case 'M': size = 16; break; // complex double
348	// case 'P': size = 8; break; // array descriptor (32bit)
349	// case 'Q': size = 16; break; // array descriptor (64bit)
350	default:
351	{
352	std::ostringstream str;
353	str << "FITS format TFORM='" << fmt << "' not yet supported.";
354	#ifdef __EXCEPTIONS
355	throw std::runtime_error(str.str());
356	#else
357	gLog << ___err___ << "ERROR - " << str.str() << std::endl;
358	return;
359	#endif
360	}
361	}
362
363	const Table::Column col = { bytes, n, size, n*size, type, unit, compress};
364
365	cols[id] = col;
366	sorted_cols.push_back(col);
367	bytes += n*size;
368	}
369
370	if (bytes!=bytes_per_row)
371	{
372	#ifdef __EXCEPTIONS
373	throw std::runtime_error("Column size mismatch");
374	#else
375	gLog << ___err___ << "ERROR - Column size mismatch" << std::endl;
376	return;
377	#endif
378	}
379
380	name = Get<std::string>("EXTNAME");
381
382	Fill_Py_Vectors();
383	}
384
385	void Fill_Py_Vectors(void)
386	{
387	// Fill the Py_ vectors see line: 90ff
388	// vectors for Keys keys;
389	for (Keys::const_iterator it=keys.begin(); it!=keys.end(); it++)
390	{
391	Py_KeyKeys.push_back(it->first);
392	Py_KeyValues.push_back(it->second.value);
393	Py_KeyComments.push_back(it->second.comment);
394	Py_KeyTypes.push_back(it->second.type);
395	}
396
397	// vectors for Columns cols;
398	for (Columns::const_iterator it=cols.begin(); it!=cols.end(); it++)
399	{
400	Py_ColumnKeys.push_back(it->first);
401	Py_ColumnTypes.push_back(it->second.type);
402	Py_ColumnOffsets.push_back(it->second.offset);
403	Py_ColumnNums.push_back(it->second.num);
404	Py_ColumnSizes.push_back(it->second.size);
405	Py_ColumnUnits.push_back(it->second.unit);
406	}
407	}
408
409	void PrintKeys(bool display_all=false) const
410	{
411	for (Keys::const_iterator it=keys.cbegin(); it!=keys.cend(); it++)
412	{
413	if (!display_all &&
414	(it->first.substr(0, 6)=="TTYPE" \|\|
415	it->first.substr(0, 6)=="TFORM" \|\|
416	it->first.substr(0, 6)=="TUNIT" \|\|
417	it->first=="TFIELDS" \|\|
418	it->first=="XTENSION" \|\|
419	it->first=="NAXIS" \|\|
420	it->first=="BITPIX" \|\|
421	it->first=="PCOUNT" \|\|
422	it->first=="GCOUNT")
423	)
424	continue;
425
426	gLog << ___all___ << std::setw(2) << it->second.type << '\|' << it->first << '=' << it->second.value << '/' << it->second.comment << '\|' << std::endl;
427	}}
428
429	void PrintColumns() const
430	{
431	typedef std::map<std::pair<size_t, std::string>, Column> Sorted;
432
433	Sorted sorted;
434
435	for (Columns::const_iterator it=cols.cbegin(); it!=cols.cend(); it++)
436	sorted[std::make_pair(it->second.offset, it->first)] = it->second;
437
438	for (Sorted::const_iterator it=sorted.cbegin(); it!=sorted.cend(); it++)
439	{
440	gLog << ___all___ << std::setw(6) << it->second.offset << "\| ";
441	gLog << it->second.num << 'x';
442	switch (it->second.type)
443	{
444	case 'A': gLog << "char(8)"; break;
445	case 'L': gLog << "bool(8)"; break;
446	case 'B': gLog << "byte(8)"; break;
447	case 'I': gLog << "short(16)"; break;
448	case 'J': gLog << "int(32)"; break;
449	case 'K': gLog << "int(64)"; break;
450	case 'E': gLog << "float(32)"; break;
451	case 'D': gLog << "double(64)"; break;
452	}
453	gLog << ": " << it->first.second << " [" << it->second.unit << "]" << std::endl;
454	}
455	}
456
457	operator bool() const { return !name.empty(); }
458
459	bool HasKey(const std::string &key) const
460	{
461	return keys.find(key)!=keys.end();
462	}
463
464	bool HasColumn(const std::string& col) const
465	{
466	return cols.find(col)!=cols.end();
467	}
468
469	const Columns &GetColumns() const
470	{
471	return cols;
472	}
473
474	const Keys &GetKeys() const
475	{
476	return keys;
477	}
478
479	// Values of keys are always signed
480	template<typename T>
481	T Get(const std::string &key) const
482	{
483	const std::map<std::string,Entry>::const_iterator it = keys.find(key);
484	if (it==keys.end())
485	{
486	std::ostringstream str;
487	str << "Key '" << key << "' not found.";
488	#ifdef __EXCEPTIONS
489	throw std::runtime_error(str.str());
490	#else
491	gLog << ___err___ << "ERROR - " << str.str() << std::endl;
492	return T();
493	#endif
494	}
495	return it->second.Get<T>();
496	}
497
498	// Values of keys are always signed
499	template<typename T>
500	T Get(const std::string &key, const T &deflt) const
501	{
502	const std::map<std::string,Entry>::const_iterator it = keys.find(key);
503	return it==keys.end() ? deflt :it->second.Get<T>();
504	}
505
506	size_t GetN(const std::string &key) const
507	{
508	const Columns::const_iterator it = cols.find(key);
509	return it==cols.end() ? 0 : it->second.num;
510	}
511
512
513
514	// There may be a gap between the main table and the start of the heap:
515	// this computes the offset
516	streamoff GetHeapShift() const
517	{
518	if (!HasKey("ZHEAPPTR"))
519	return 0;
520
521	const size_t shift = Get<size_t>("ZHEAPPTR");
522	return shift <= total_bytes ? 0 : shift - total_bytes;
523	}
524
525	// return total number of bytes 'all inclusive'
526	streamoff GetTotalBytes() const
527	{
528	//get offset of special data area from start of main table
529	const streamoff shift = GetHeapShift();
530
531	//and special data area size
532	const streamoff size = HasKey("PCOUNT") ? Get<streamoff>("PCOUNT") : 0;
533
534	// Get the total size
535	const streamoff total = total_bytes + size + shift;
536
537	// check for padding
538	if (total%2880==0)
539	return total;
540
541	// padding necessary
542	return total + (2880 - (total%2880));
543	}
544	};
545
546	protected:
547	std::ofstream fCopy;
548	std::vector<std::string> fListOfTables; // List of skipped tables. Last table is open table
549
550	Table fTable;
551
552	typedef std::pair<void*, Table::Column> Address;
553	typedef std::vector<Address> Addresses;
554	//map<void*, Table::Column> fAddresses;
555	Addresses fAddresses;
556
557	typedef std::unordered_map<std::string, void*> Pointers;
558	Pointers fPointers;
559
560	std::vector<std::vector<char>> fGarbage;
561
562	std::vector<char> fBufferRow;
563	std::vector<char> fBufferDat;
564
565	size_t fRow;
566
567	Checksum fChkHeader;
568	Checksum fChkData;
569
570	bool ReadBlock(std::vector<std::string> &vec)
571	{
572	int endtag = 0;
573	for (int i=0; i<36; i++)
574	{
575	char c[81];
576	c[80] = 0;
577	read(c, 80);
578	if (!good())
579	break;
580
581	fChkHeader.add(c, 80);
582
583	// if (c[0]==0)
584	// return vector<string>();
585
586	std::string str(c);
587
588	// if (!str.empty())
589	// cout << setw(2) << i << "\|" << str << "\|" << (endtag?'-':'+') << endl;
590
591	if (endtag==2 \|\| str=="END ")
592	{
593	endtag = 2; // valid END tag found
594	continue;
595	}
596
597	if (endtag==1 \|\| str==" ")
598	{
599	endtag = 1; // end tag not found, but expected to be there
600	continue;
601	}
602
603	vec.push_back(str);
604	}
605
606	// Make sure that no empty vector is returned
607	if (endtag && vec.size()%36==0)
608	vec.emplace_back("END = '' / ");
609
610	return endtag==2;
611	}
612
613	std::string Compile(const std::string &key, int16_t i=-1) const
614	{
615	#if GCC_VERSION < 40603
616	return i<0 ? key : key+std::to_string((long long int)(i));
617	#else
618	return i<0 ? key : key+std::to_string(i);
619	#endif
620	}
621
622	void Constructor(const std::string &fname, std::string fout="", const std::string& tableName="", bool force=false)
623	{
624	char simple[10];
625	read(simple, 10);
626	if (!good())
627	return;
628
629	if (memcmp(simple, "SIMPLE = ", 10))
630	{
631	clear(rdstate()\|std::ios::badbit);
632	#ifdef __EXCEPTIONS
633	throw std::runtime_error("File is not a FITS file.");
634	#else
635	gLog << ___err___ << "ERROR - File is not a FITS file." << std::endl;
636	return;
637	#endif
638	}
639
640	seekg(0);
641
642	while (good())
643	{
644	std::vector<std::string> block;
645	while (1)
646	{
647	// If we search for a table, we implicitly assume that
648	// not finding the table is not an error. The user
649	// can easily check that by eof() && !bad()
650	peek();
651	if (eof() && !bad() && !tableName.empty())
652	{
653	break;
654	}
655	// FIXME: Set limit on memory consumption
656	const int rc = ReadBlock(block);
657	if (!good())
658	{
659	clear(rdstate()\|std::ios::badbit);
660	#ifdef __EXCEPTIONS
661	throw std::runtime_error("FITS file corrupted.");
662	#else
663	gLog << ___err___ << "ERROR - FITS file corrupted." << std::endl;
664	return;
665	#endif
666	}
667
668	if (block.size()%36)
669	{
670	if (!rc && !force)
671	{
672	clear(rdstate()\|std::ios::badbit);
673	#ifdef __EXCEPTIONS
674	throw std::runtime_error("END keyword missing in FITS header.");
675	#else
676	gLog << ___err___ << "ERROR - END keyword missing in FITS file... file might be corrupted." << std::endl;
677	return;
678	#endif
679	}
680	break;
681	}
682	}
683
684	if (block.empty())
685	break;
686
687	if (block[0].substr(0, 9)=="SIMPLE =")
688	{
689	fChkHeader.reset();
690	continue;
691	}
692
693	if (block[0].substr(0, 9)=="XTENSION=")
694	{
695	fTable = Table(block, tellg());
696	fRow = (size_t)-1;
697
698	if (!fTable)
699	{
700	clear(rdstate()\|std::ios::badbit);
701	return;
702	}
703
704	const std::string &tname = fTable.Get<std::string>("EXTNAME");
705
706	fListOfTables.emplace_back(tname);
707
708	// Check for table name. Skip until eof or requested table are found.
709	// skip the current table?
710	if ((!tableName.empty() && tableName!=tname) \|\| (tableName.empty() && "ZDrsCellOffsets"==tname))
711	{
712	const streamoff skip = fTable.GetTotalBytes();
713	seekg(skip, std::ios_base::cur);
714
715	fChkHeader.reset();
716
717	continue;
718	}
719
720	fBufferRow.resize(fTable.bytes_per_row + 8-fTable.bytes_per_row%4);
721	fBufferDat.resize(fTable.bytes_per_row);
722
723	break;
724	}
725	}
726
727	if (fout.empty())
728	return;
729
730	if (*fout.rbegin()=='/')
731	{
732	const size_t p = fname.find_last_of('/');
733	fout.append(fname.substr(p+1));
734	}
735
736	fCopy.open(fout);
737	if (!fCopy)
738	{
739	clear(rdstate()\|std::ios::badbit);
740	#ifdef __EXCEPTIONS
741	throw std::runtime_error("Could not open output file.");
742	#else
743	gLog << ___err___ << "ERROR - Failed to open output file." << std::endl;
744	return;
745	#endif
746	}
747
748	const streampos p = tellg();
749	seekg(0);
750
751	std::vector<char> buf(p);
752	read(buf.data(), p);
753
754	fCopy.write(buf.data(), p);
755	if (!fCopy)
756	clear(rdstate()\|std::ios::badbit);
757	}
758
759	public:
760	fits(const std::string &fname, const std::string& tableName="", bool force=false) : izstream(fname.c_str())
761	{
762	Constructor(fname, "", tableName, force);
763	if ((fTable.is_compressed \|\|fTable.name=="ZDrsCellOffsets") && !force)
764	{
765	#ifdef __EXCEPTIONS
766	throw std::runtime_error("You are trying to read a compressed fits with the base fits class. Please use factfits instead.");
767	#else
768	gLog << ___err___ << "ERROR - You are trying to read a compressed fits with the base fits class. Please use factfits instead." << std::endl;
769	#endif
770	clear(rdstate()\|std::ios::badbit);
771	}
772	}
773
774	fits(const std::string &fname, const std::string &fout, const std::string& tableName, bool force=false) : izstream(fname.c_str())
775	{
776	Constructor(fname, fout, tableName, force);
777	if ((fTable.is_compressed \|\| fTable.name=="ZDrsCellOffsets") && !force)
778	{
779	#ifdef __EXCEPTIONS
780	throw std::runtime_error("You are trying to read a compressed fits with the base fits class. Please use factfits instead.");
781	#else
782	gLog << ___err___ << "ERROR - You are trying to read a compressed fits with the base fits class. Please use factfits instead." << std::endl;
783	#endif
784	clear(rdstate()\|std::ios::badbit);
785	}
786	}
787
788	fits() : izstream()
789	{
790
791	}
792
793	~fits()
794	{
795	std::copy(std::istreambuf_iterator<char>(*this),
796	std::istreambuf_iterator<char>(),
797	std::ostreambuf_iterator<char>(fCopy));
798	}
799
800	virtual void StageRow(size_t row, char* dest)
801	{
802	// if (row!=fRow+1) // Fast seeking is ensured by izstream
803	seekg(fTable.offset+row*fTable.bytes_per_row);
804	read(dest, fTable.bytes_per_row);
805	//fin.clear(fin.rdstate()&~ios::eofbit);
806	}
807
808	virtual void WriteRowToCopyFile(size_t row)
809	{
810	if (row==fRow+1)
811	{
812	const uint8_t offset = (row*fTable.bytes_per_row)%4;
813
814	fChkData.add(fBufferRow);
815	if (fCopy.is_open() && fCopy.good())
816	fCopy.write(fBufferRow.data()+offset, fTable.bytes_per_row);
817	if (!fCopy)
818	clear(rdstate()\|std::ios::badbit);
819	}
820	else
821	if (fCopy.is_open())
822	clear(rdstate()\|std::ios::badbit);
823	}
824
825	void ZeroBufferForChecksum(std::vector<char>& vec, const uint64_t extraZeros=0)
826	{
827	auto ib = vec.begin();
828	auto ie = vec.end();
829
830	*ib++ = 0;
831	*ib++ = 0;
832	*ib++ = 0;
833	*ib = 0;
834
835	for (uint64_t i=0;i<extraZeros+8;i++)
836	*--ie = 0;
837	}
838
839	uint8_t ReadRow(size_t row)
840	{
841	// For the checksum we need everything to be correctly aligned
842	const uint8_t offset = (row*fTable.bytes_per_row)%4;
843
844	ZeroBufferForChecksum(fBufferRow);
845
846	StageRow(row, fBufferRow.data()+offset);
847
848	WriteRowToCopyFile(row);
849
850	fRow = row;
851
852	return offset;
853	}
854
855	template<size_t N>
856	void revcpy(char dest, const char src, const int &num)
857	{
858	const char pend = src + numN;
859	for (const char *ptr = src; ptr<pend; ptr+=N, dest+=N)
860	std::reverse_copy(ptr, ptr+N, dest);
861	}
862
863	virtual void MoveColumnDataToUserSpace(char dest, const char src, const Table::Column& c)
864	{
865	// Let the compiler do some optimization by
866	// knowing that we only have 1, 2, 4 and 8
867	switch (c.size)
868	{
869	case 1: memcpy (dest, src, c.bytes); break;
870	case 2: revcpy<2>(dest, src, c.num); break;
871	case 4: revcpy<4>(dest, src, c.num); break;
872	case 8: revcpy<8>(dest, src, c.num); break;
873	}
874	}
875
876	virtual bool GetRow(size_t row, bool check=true)
877	{
878	if (check && row>=fTable.num_rows)
879	return false;
880
881	const uint8_t offset = ReadRow(row);
882	if (!good())
883	return good();
884
885	const char *ptr = fBufferRow.data() + offset;
886
887	for (Addresses::const_iterator it=fAddresses.cbegin(); it!=fAddresses.cend(); it++)
888	{
889	const Table::Column &c = it->second;
890
891	const char *src = ptr + c.offset;
892	char dest = reinterpret_cast<char>(it->first);
893
894	MoveColumnDataToUserSpace(dest, src, c);
895	}
896
897	return good();
898	}
899
900	bool GetNextRow(bool check=true)
901	{
902	return GetRow(fRow+1, check);
903	}
904
905	virtual bool SkipNextRow()
906	{
907	seekg(fTable.offset+(++fRow)*fTable.bytes_per_row);
908	return good();
909	}
910
911	static bool Compare(const Address &p1, const Address &p2)
912	{
913	return p1.first>p2.first;
914	}
915
916	template<class T, class S>
917	const T &GetAs(const std::string &name)
918	{
919	return reinterpret_cast<S>(fPointers[name]);
920	}
921
922	void *SetPtrAddress(const std::string &name)
923	{
924	if (fTable.cols.count(name)==0)
925	{
926	std::ostringstream str;
927	str << "SetPtrAddress('" << name << "') - Column not found.";
928	#ifdef __EXCEPTIONS
929	throw std::runtime_error(str.str());
930	#else
931	gLog << ___err___ << "ERROR - " << str.str() << std::endl;
932	return NULL;
933	#endif
934	}
935
936	Pointers::const_iterator it = fPointers.find(name);
937	if (it!=fPointers.end())
938	return it->second;
939
940	fGarbage.emplace_back(fTable.cols[name].bytes);
941
942	void *ptr = fGarbage.back().data();
943
944	fPointers[name] = ptr;
945	fAddresses.emplace_back(ptr, fTable.cols[name]);
946	sort(fAddresses.begin(), fAddresses.end(), Compare);
947	return ptr;
948	}
949
950	template<typename T>
951	bool SetPtrAddress(const std::string &name, T *ptr, size_t cnt)
952	{
953	if (fTable.cols.count(name)==0)
954	{
955	std::ostringstream str;
956	str << "SetPtrAddress('" << name << "') - Column not found.";
957	#ifdef __EXCEPTIONS
958	throw std::runtime_error(str.str());
959	#else
960	gLog << ___err___ << "ERROR - " << str.str() << std::endl;
961	return false;
962	#endif
963	}
964
965	if (sizeof(T)!=fTable.cols[name].size)
966	{
967	std::ostringstream str;
968	str << "SetPtrAddress('" << name << "') - Element size mismatch: expected "
969	<< fTable.cols[name].size << " from header, got " << sizeof(T);
970	#ifdef __EXCEPTIONS
971	throw std::runtime_error(str.str());
972	#else
973	gLog << ___err___ << "ERROR - " << str.str() << std::endl;
974	return false;
975	#endif
976	}
977
978	if (cnt!=fTable.cols[name].num)
979	{
980	std::ostringstream str;
981	str << "SetPtrAddress('" << name << "') - Element count mismatch: expected "
982	<< fTable.cols[name].num << " from header, got " << cnt;
983	#ifdef __EXCEPTIONS
984	throw std::runtime_error(str.str());
985	#else
986	gLog << ___err___ << "ERROR - " << str.str() << std::endl;
987	return false;
988	#endif
989	}
990
991	// if (fAddresses.count(ptr)>0)
992	// gLog << warn << "SetPtrAddress('" << name << "') - Pointer " << ptr << " already assigned." << endl;
993
994	//fAddresses[ptr] = fTable.cols[name];
995	fPointers[name] = ptr;
996	fAddresses.emplace_back(ptr, fTable.cols[name]);
997	sort(fAddresses.begin(), fAddresses.end(), Compare);
998	return true;
999	}
1000
1001	template<class T>
1002	bool SetRefAddress(const std::string &name, T &ptr)
1003	{
1004	return SetPtrAddress(name, &ptr, sizeof(ptr)/sizeof(T));
1005	}
1006
1007	template<typename T>
1008	bool SetVecAddress(const std::string &name, std::vector<T> &vec)
1009	{
1010	return SetPtrAddress(name, vec.data(), vec.size());
1011	}
1012
1013	template<typename T>
1014	T Get(const std::string &key) const
1015	{
1016	return fTable.Get<T>(key);
1017	}
1018
1019	template<typename T>
1020	T Get(const std::string &key, const std::string &deflt) const
1021	{
1022	return fTable.Get<T>(key, deflt);
1023	}
1024
1025	bool SetPtrAddress(const std::string &name, void *ptr, size_t cnt=0)
1026	{
1027	if (fTable.cols.count(name)==0)
1028	{
1029	std::ostringstream str;
1030	str <<"SetPtrAddress('" << name << "') - Column not found.";
1031	#ifdef __EXCEPTIONS
1032	throw std::runtime_error(str.str());
1033	#else
1034	gLog << ___err___ << "ERROR - " << str.str() << std::endl;
1035	return false;
1036	#endif
1037	}
1038
1039	if (cnt && cnt!=fTable.cols[name].num)
1040	{
1041	std::ostringstream str;
1042	str << "SetPtrAddress('" << name << "') - Element count mismatch: expected "
1043	<< fTable.cols[name].num << " from header, got " << cnt;
1044	#ifdef __EXCEPTIONS
1045	throw std::runtime_error(str.str());
1046	#else
1047	gLog << ___err___ << "ERROR - " << str.str() << std::endl;
1048	return false;
1049	#endif
1050	}
1051
1052	// if (fAddresses.count(ptr)>0)
1053	// gLog << warn << "SetPtrAddress('" << name << "') - Pointer " << ptr << " already assigned." << endl;
1054
1055	//fAddresses[ptr] = fTable.cols[name];
1056	fPointers[name] = ptr;
1057	fAddresses.emplace_back(ptr, fTable.cols[name]);
1058	sort(fAddresses.begin(), fAddresses.end(), Compare);
1059	return true;
1060	}
1061
1062	bool HasKey(const std::string &key) const { return fTable.HasKey(key); }
1063	bool HasColumn(const std::string& col) const { return fTable.HasColumn(col);}
1064	const Table::Columns &GetColumns() const { return fTable.GetColumns();}
1065	const Table::SortedColumns& GetSortedColumns() const { return fTable.sorted_cols;}
1066	const Table::Keys &GetKeys() const { return fTable.GetKeys();}
1067
1068	int64_t GetInt(const std::string &key) const { return fTable.Get<int64_t>(key); }
1069	uint64_t GetUInt(const std::string &key) const { return fTable.Get<uint64_t>(key); }
1070	double GetFloat(const std::string &key) const { return fTable.Get<double>(key); }
1071	std::string GetStr(const std::string &key) const { return fTable.Get<std::string>(key); }
1072
1073	size_t GetN(const std::string &key) const
1074	{
1075	return fTable.GetN(key);
1076	}
1077
1078	// size_t GetNumRows() const { return fTable.num_rows; }
1079	size_t GetRow() const { return fRow==(size_t)-1 ? 0 : fRow; }
1080
1081	operator bool() const { return fTable && fTable.offset!=0; }
1082
1083	void PrintKeys() const { fTable.PrintKeys(); }
1084	void PrintColumns() const { fTable.PrintColumns(); }
1085
1086	// 'Wrappers' for the Table's Getters of the Py Vectors
1087	vector<string> GetPy_KeyKeys() {return fTable.Py_KeyKeys; }
1088	vector<string> GetPy_KeyValues() {return fTable.Py_KeyValues; }
1089	vector<string> GetPy_KeyComments() {return fTable.Py_KeyComments; }
1090	vector<char> GetPy_KeyTypes() {return fTable.Py_KeyTypes; }
1091
1092	vector<string> GetPy_ColumnKeys() {return fTable.Py_ColumnKeys; }
1093	vector<size_t> GetPy_ColumnOffsets() {return fTable.Py_ColumnOffsets; }
1094	vector<size_t> GetPy_ColumnNums() {return fTable.Py_ColumnNums; }
1095	vector<size_t> GetPy_ColumnSizes() {return fTable.Py_ColumnSizes;}
1096	vector<char> GetPy_ColumnTypes() {return fTable.Py_ColumnTypes;}
1097	vector<string> GetPy_ColumnUnits() {return fTable.Py_ColumnUnits;}
1098
1099	bool IsHeaderOk() const { return fTable.datasum<0?false:(fChkHeader+Checksum(fTable.datasum)).valid(); }
1100	virtual bool IsFileOk() const { return (fChkHeader+fChkData).valid(); }
1101
1102	bool IsCompressedFITS() const { return fTable.is_compressed;}
1103
1104	virtual size_t GetNumRows() const
1105	{
1106	return fTable.Get<size_t>("NAXIS2");
1107	}
1108
1109	virtual size_t GetBytesPerRow() const
1110	{
1111	return fTable.Get<size_t>("NAXIS1");
1112	}
1113
1114	const std::vector<std::string> &GetTables() const
1115	{
1116	return fListOfTables;
1117	}
1118	};
1119
1120	#endif

Note: See TracBrowser for help on using the repository browser.

Download in other formats: