source: trunk/FACT++/src/fitsloader.cc @ 10795

Last change on this file since 10795 was 10795, checked in by tbretz, 8 years ago
Replaced a stringstream by an ostringstream
File size: 22.5 KB
Line 
1//****************************************************************
2/** @class FitsLoader
3
4  @brief Load a given Fits file and table, and dump selected columns if requested.
5
6  It derives from StateMachineDim. the first parent is here to enforce
7  a state machine behaviour
8  The possible states and transitions of the machine are:
9  \dot
10  digraph FitsLoader {
11          node [shape=record, fontname=Helvetica, fontsize=10];
12      e [label="Error" color="red"];
13   r [label="Ready"]
14   d [label="FileLoaded"]
15
16  e -> r
17  r -> e
18  r -> d
19  d -> r
20   }
21  \enddot
22 */
23 //****************************************************************
24#include "FACT.h"
25#include "Event.h"
26#include "StateMachineDim.h"
27#include "WindowLog.h"
28#include "Configuration.h"
29#include "LocalControl.h"
30#include "Description.h"
31
32
33#include <boost/bind.hpp>
34#if BOOST_VERSION < 104400
35#if (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 4))
36#undef BOOST_HAS_RVALUE_REFS
37#endif
38#endif
39#include <boost/thread.hpp>
40
41#include <iostream>
42
43#include <CCfits/CCfits>
44
45class FitsLoader : public StateMachineDim
46{
47
48public:
49    enum
50    {
51        kSM_FileLoaded = 20,
52    } localstates_t;
53
54    FitsLoader(ostream& out);
55    ~FitsLoader();
56
57    ///Define command names
58    static const char* fLoadFits;
59    static const char* fUnloadFits;
60    static const char* fListColumns;
61    static const char* fDumpColumns;
62    static const char* fClearDumpList;
63    static const char* fDoDump;
64    static const char* fConfigFileName;
65    static const char* fConfigTableName;
66    static const char* fConfigPrecName;
67    static const char* fConfigFileOutName;
68
69private:
70    ///Name of the fits file to load
71    string fFileName;
72    ///Name of the table to load from the  file
73    string fTableName;
74    ///FITS pointer
75    CCfits::FITS* fFile;
76    ///Table pointer
77    CCfits::Table* fTable;
78    ///Precision of the ofstream. Used to output a given number of significant digits for floats or doubles
79    int fStreamPrecision;
80    ///Name of the output file
81    string fFileOut;
82    ///map between the column names and their CCfits objects
83    map<string, CCfits::Column*> fColMap;
84    ///List of the column names to be dumped
85    vector<string> fDumpList;
86    ///Transition from ready to fileLoaded.
87    int LoadPlease();
88    ///Transition from fileLoaded to ready
89    int UnloadPlease();
90    ///Lists the loaded column names
91    int ListColumnsPlease(const Event&);
92    ///Add a column name to the dump list
93    int AddDumpColumnsPlease(const Event&);
94    ///Clear the dump list
95    int ClearDumpListPlease(const Event&);
96    ///Perform the dumping, based on the current dump list
97    int DoDumpPlease(const Event&);
98    ///Set the name of the Fits file to be loaded
99    int ConfigFileNamePlease(const Event&);
100    ///Set the name of the table to be loaded
101    int ConfigTableNamePlease(const Event&);
102    ///Set the ofstream precision
103    int SetOFStreamPrecisionPlease(const Event&);
104    ///Set the name of the output file
105    int SetFileOutPlease(const Event&);
106    ///Calculate the buffer size required to read a row of the fits table, as well as the offsets to each column
107    vector<int> CalculateBufferSize();
108    ///Write a single row of the selected data
109    void writeValuesFromFits(vector<int>& offsets,ofstream& targetFile, unsigned char* fitsBuffer);
110
111public:
112    ///Configures the fitsLoader from the config file and/or command arguments.
113    void SetupConfig(Configuration& conf);
114};
115
116const char* FitsLoader::fLoadFits = "load";
117const char* FitsLoader::fUnloadFits = "unload";
118const char* FitsLoader::fListColumns = "list_columns";
119const char* FitsLoader::fDumpColumns = "add_dump";
120const char* FitsLoader::fClearDumpList = "clear_dump";
121const char* FitsLoader::fDoDump = "dump";
122const char* FitsLoader::fConfigFileName = "set_file";
123const char* FitsLoader::fConfigTableName = "set_table";
124const char* FitsLoader::fConfigPrecName = "set_prec";
125const char* FitsLoader::fConfigFileOutName = "set_outfile";
126
127// --------------------------------------------------------------------------
128//
129//! Set the name of the output file
130//! @param evt
131//!        the event transporting the file name
132//
133int FitsLoader::SetFileOutPlease(const Event& evt)
134{
135    fFileOut = string(evt.GetText());
136    stringstream str;
137    str << "Output file is now " << fFileOut;
138    Message(str.str());
139    return 0;
140}
141// --------------------------------------------------------------------------
142//
143//! Set the precision of the ofstream. So that an appropriate number of significant digits are outputted.
144//! @param evt
145//!        the event transporting the precision
146//
147int FitsLoader::SetOFStreamPrecisionPlease(const Event& evt)
148{
149    fStreamPrecision = evt.GetInt();
150    stringstream str;
151    str << "OFStream precision is now " << fStreamPrecision;
152    Message(str.str());
153    return 0;
154}
155// --------------------------------------------------------------------------
156//
157//! Writes a single row of the selected FITS data to the output file.
158//! @param offsets
159//!         a vector containing the offsets to the columns (in bytes)
160//! @param targetFile
161//!         the ofstream where to write to
162//! @param fitsBuffer
163//!         the memory were the row has been loaded by cfitsio
164//
165void FitsLoader::writeValuesFromFits(vector<int>& offsets,ofstream& targetFile, unsigned char* fitsBuffer)
166{
167    targetFile.precision(fStreamPrecision);
168    map<string, CCfits::Column*>::iterator it;
169   for (it=fColMap.begin(); it != fColMap.end(); it++)
170    {
171        bool found = false;
172        for (vector<string>::iterator jt=fDumpList.begin(); jt != fDumpList.end(); jt++)
173        {
174            if (it->first == *jt)
175            {
176                found = true;
177                break;
178            }
179        }
180        if (!found)
181            continue;
182       int offset = offsets[it->second->index()-1];
183       const char* charSrc = reinterpret_cast<char*>(&fitsBuffer[offset]);
184        unsigned char copyBuffer[30];//max size of a single variable
185        for (int width = 0; width<it->second->width(); width++)
186        {
187            switch (it->second->type())
188            {
189            case CCfits::Tbyte:
190                targetFile << *charSrc;
191                charSrc += sizeof(char);
192            break;
193            case CCfits::Tushort:
194                targetFile << *reinterpret_cast<const unsigned short*>(charSrc);
195                charSrc += sizeof(char);
196            break;
197            case CCfits::Tshort:
198                targetFile << *reinterpret_cast<const short*>(charSrc);
199                charSrc += sizeof(char);
200            break;
201            case CCfits::Tuint:
202                reverse_copy(charSrc, charSrc+sizeof(unsigned int), copyBuffer);
203                //warning suppressed in gcc4.0.2
204                targetFile << *reinterpret_cast<unsigned int*>(copyBuffer);
205                charSrc += sizeof(int);
206            break;
207            case CCfits::Tint:
208                reverse_copy(charSrc, charSrc+sizeof(int), copyBuffer);
209                targetFile << *reinterpret_cast<int*>(copyBuffer);
210                charSrc += sizeof(int);
211            break;
212            case CCfits::Tulong:
213                reverse_copy(charSrc, charSrc+sizeof(unsigned long), copyBuffer);
214                targetFile << *reinterpret_cast<unsigned long*>(copyBuffer);
215                charSrc += sizeof(int);
216            break;
217            case CCfits::Tlong:
218                reverse_copy(charSrc, charSrc+sizeof(long), copyBuffer);
219                targetFile << *reinterpret_cast<long*>(copyBuffer);
220                charSrc += sizeof(int);
221            break;
222            case CCfits::Tlonglong:
223                reverse_copy(charSrc, charSrc+sizeof(long long), copyBuffer);
224                targetFile << *reinterpret_cast<long long*>(copyBuffer);
225                charSrc += sizeof(long long);
226            break;
227            case CCfits::Tfloat:
228                reverse_copy(charSrc, charSrc+sizeof(float), copyBuffer);
229                targetFile << *reinterpret_cast<float*>(copyBuffer);
230                charSrc += sizeof(float);
231            break;
232            case CCfits::Tdouble:
233                reverse_copy(charSrc, charSrc+sizeof(double), copyBuffer);
234                targetFile << *reinterpret_cast<double*>(copyBuffer);
235                charSrc += sizeof(double);
236            break;
237            case CCfits::Tnull:
238            case CCfits::Tbit:
239            case CCfits::Tlogical:
240            case CCfits::Tstring:
241            case CCfits::Tcomplex:
242            case CCfits::Tdblcomplex:
243            case CCfits::VTbit:
244            case CCfits::VTbyte:
245            case CCfits::VTlogical:
246            case CCfits::VTushort:
247            case CCfits::VTshort:
248            case CCfits::VTuint:
249            case CCfits::VTint:
250            case CCfits::VTulong:
251            case CCfits::VTlong:
252            case CCfits::VTlonglong:
253            case CCfits::VTfloat:
254            case CCfits::VTdouble:
255            case CCfits::VTcomplex:
256            case CCfits::VTdblcomplex:
257                Error("Data type not implemented yet.");
258                return;
259            break;
260            default:
261                Error("THIS SHOULD NEVER BE REACHED");
262                return;
263            }//switch
264            targetFile << " ";
265        }//width loop
266    }//iterator over the columns
267    targetFile << endl;
268}
269
270// --------------------------------------------------------------------------
271//
272//! Calculates the required buffer size for reading one row of the current table.
273//! Also calculates the offsets to all the columns
274//
275vector<int> FitsLoader::CalculateBufferSize()
276{
277    vector<int> result;
278    map<int,int> sizes;
279    int size = 0;
280
281    for (map<string, CCfits::Column*>::iterator it=fColMap.begin(); it != fColMap.end(); it++)
282    {
283        int width = it->second->width();
284        switch (it->second->type())
285        {
286        case CCfits::Tbyte:
287        case CCfits::Tushort:
288        case CCfits::Tshort:
289            Message("short");
290            sizes[it->second->index()] =  sizeof(char)*width;
291        break;
292        case CCfits::Tuint:
293        case CCfits::Tint:
294            Message("int");
295            sizes[it->second->index()] =  sizeof(int)*width;
296        break;
297        case CCfits::Tulong:
298        case CCfits::Tlong:
299            Message("long");
300            sizes[it->second->index()] = sizeof(int)*width;
301        break;
302        case CCfits::Tlonglong:
303            Message("longlong");
304            sizes[it->second->index()] =  sizeof(long long)*width;
305        break;
306        case CCfits::Tfloat:
307            Message("float");
308            sizes[it->second->index()] =  sizeof(float)*width;
309        break;
310        case CCfits::Tdouble:
311            Message("double");
312            sizes[it->second->index()] =  sizeof(double)*width;
313        break;
314        case CCfits::Tnull:
315        case CCfits::Tbit:
316        case CCfits::Tlogical:
317        case CCfits::Tstring:
318        case CCfits::Tcomplex:
319        case CCfits::Tdblcomplex:
320        case CCfits::VTbit:
321        case CCfits::VTbyte:
322        case CCfits::VTlogical:
323        case CCfits::VTushort:
324        case CCfits::VTshort:
325        case CCfits::VTuint:
326        case CCfits::VTint:
327        case CCfits::VTulong:
328        case CCfits::VTlong:
329        case CCfits::VTlonglong:
330        case CCfits::VTfloat:
331        case CCfits::VTdouble:
332        case CCfits::VTcomplex:
333        case CCfits::VTdblcomplex:
334            Error("Data type not implemented yet.");
335            return vector<int>();
336        break;
337        default:
338            Error("THIS SHOULD NEVER BE REACHED");
339            return vector<int>();
340        }
341    }
342    //calculate the offsets in the vector.
343    int checkIndex = 1;
344    for (map<int,int>::iterator it=sizes.begin(); it != sizes.end(); it++)
345    {
346        result.push_back(size);
347        size += it->second;
348        if (it->first != checkIndex)
349        {
350            stringstream str;
351            str << "Expected index " << checkIndex << " found " << it->first;
352            Error(str.str());
353        }
354        checkIndex++;
355    }
356    result.push_back(size);
357    return result;
358}
359// --------------------------------------------------------------------------
360//
361//! Constructor
362//! @param out
363//!        the ostream where to redirect the outputs
364//
365FitsLoader::FitsLoader(ostream& out) : StateMachineDim(out, "FITS_LOADER")
366{
367    //Add the existing states
368    AddStateName(kSM_FileLoaded,  "FileLoaded", "A Fits file has been loaded");
369
370    //Add the possible transitions
371    AddEvent(kSM_FileLoaded, fLoadFits, kSM_Ready)
372            (boost::bind(&FitsLoader::LoadPlease, this))
373            ("Loads the given Fits file");
374    AddEvent(kSM_Ready, fUnloadFits, kSM_FileLoaded)
375            (boost::bind(&FitsLoader::UnloadPlease, this))
376            ("Unloads the given Fits file");
377
378    //Add the possible configurations
379    AddEvent(fListColumns, "", kSM_FileLoaded)
380            (boost::bind(&FitsLoader::ListColumnsPlease, this, _1))
381            ("List the columns that were loaded from that file");
382    AddEvent(fDumpColumns, "C", kSM_FileLoaded)
383            (boost::bind(&FitsLoader::AddDumpColumnsPlease, this, _1))
384            ("Add a given column to the dumping list");
385    AddEvent(fClearDumpList, "", kSM_FileLoaded)
386            (boost::bind(&FitsLoader::ClearDumpListPlease, this, _1))
387            ("Clear the dumping list");
388    AddEvent(fDoDump, "", kSM_FileLoaded)
389            (boost::bind(&FitsLoader::DoDumpPlease, this, _1))
390            ("Perform the dump of columns data, based on the to dump list");
391    AddEvent(fConfigFileName, "C", kSM_Ready, kSM_FileLoaded)
392            (boost::bind(&FitsLoader::ConfigFileNamePlease, this, _1))
393            ("Gives the name of the Fits file to be loaded");
394    AddEvent(fConfigTableName, "C", kSM_Ready, kSM_FileLoaded)
395            (boost::bind(&FitsLoader::ConfigTableNamePlease, this, _1))
396            ("Gives the name of the Table to be loaded");
397    AddEvent(fConfigPrecName, "I", kSM_Ready, kSM_FileLoaded)
398            (boost::bind(&FitsLoader::SetOFStreamPrecisionPlease, this, _1))
399            ("Set the precision of the ofstream, i.e. the number of significant digits being outputted");
400    AddEvent(fConfigFileOutName, "C", kSM_Ready, kSM_FileLoaded)
401            (boost::bind(&FitsLoader::SetFileOutPlease, this, _1))
402            ("Set the name of the outputted file.");
403
404    fFile = NULL;
405    fStreamPrecision = 20;
406
407}
408// --------------------------------------------------------------------------
409//
410//! Destructor
411//
412FitsLoader::~FitsLoader()
413{
414    if (fFile)
415        delete fFile;
416    fFile = NULL;
417}
418// --------------------------------------------------------------------------
419//
420//! Loads the fits file based on the current parameters
421//
422int FitsLoader::LoadPlease()
423{
424    ostringstream str;
425    try
426    {
427        fFile = new CCfits::FITS(fFileName);
428    }
429    catch (CCfits::FitsException e)
430     {
431         str << "Could not open FITS file " << fFileName << " reason: " << e.message();
432         Error(str);
433         return kSM_Ready;
434     }
435    str.str("");
436    const multimap< string, CCfits::ExtHDU * > extMap = fFile->extension();
437    if (extMap.find(fTableName) == extMap.end())
438    {
439        str.str("");
440        str << "Could not open table " << fTableName << ". Tables in file are: ";
441        for (std::multimap<string, CCfits::ExtHDU*>::const_iterator it=extMap.begin(); it != extMap.end(); it++)
442            str << it->first << " ";
443        Error(str.str());
444        return kSM_Ready;
445    }
446    else
447        fTable = dynamic_cast<CCfits::Table*>(extMap.find(fTableName)->second);
448    int numRows = fTable->rows();
449    str.str("");
450    str << "Loaded table has " << numRows << " rows";
451    Message(str.str());
452
453    fColMap = fTable->column();
454    if (fDumpList.size() != 0)
455    {
456        bool should_clear = false;
457        for (vector<string>::iterator it=fDumpList.begin(); it!= fDumpList.end(); it++)
458        {
459            if (fColMap.find(*it) == fColMap.end())
460            {
461                should_clear = true;
462                Error("Config-given dump list contains invalid entry " + *it + " clearing the list");
463            }
464        }
465        if (should_clear)
466            fDumpList.clear();
467    }
468    return kSM_FileLoaded;
469}
470// --------------------------------------------------------------------------
471//
472//! Unloads the Fits file
473//
474int FitsLoader::UnloadPlease()
475{
476    if (fFile)
477        delete fFile;
478    else
479        Error("Error: Fits file is  NULL while it should not have been");
480    fFile = NULL;
481    return kSM_Ready;
482}
483// --------------------------------------------------------------------------
484//
485//! List the columns that are in the loaded Fits table
486//
487int FitsLoader::ListColumnsPlease(const Event&)
488{
489    Message("Columns in the loaded table are:");
490    map<string, CCfits::Column*>::iterator it;
491    for (it=fColMap.begin(); it != fColMap.end(); it++)
492        Message(it->first);
493    return GetCurrentState();
494}
495// --------------------------------------------------------------------------
496//
497//! Add a given column name  to the list of columns to dump
498//! @param evt
499//!        the event transporting the column name
500//
501int FitsLoader::AddDumpColumnsPlease(const Event& evt)
502{
503    string evtText(evt.GetText());
504    //TODO check that this column indeed exist in the file
505    if (fColMap.find(evtText) != fColMap.end())
506        fDumpList.push_back(evtText);
507    else
508        Error("Could not find column " + evtText + " int table");
509    Message("New dump list:");
510    for (vector<string>::iterator it=fDumpList.begin(); it != fDumpList.end(); it++)
511        Message(*it);
512    return GetCurrentState();
513}
514// --------------------------------------------------------------------------
515//
516//! Clear the list of columns to dump
517//
518int FitsLoader::ClearDumpListPlease(const Event&)
519{
520    fDumpList.clear();
521    Message("Dump list is now empty");
522    return GetCurrentState();
523}
524// --------------------------------------------------------------------------
525//
526//! Perform the actual dump, based on the current parameters
527//
528int FitsLoader::DoDumpPlease(const Event&)
529{
530    fTable->makeThisCurrent();
531    vector<int> offsets = CalculateBufferSize();
532    int size = offsets[offsets.size()-1];
533    offsets.pop_back();
534    unsigned char* fitsBuffer = new unsigned char[size];
535
536    ofstream targetFile(fFileOut);
537    int status = 0;
538
539    for (int i=1;i<=fTable->rows(); i++)
540    {
541        fits_read_tblbytes(fFile->fitsPointer(), i, 1, size, fitsBuffer, &status);
542        if (status)
543        {
544            stringstream str;
545            str << "An error occurred while reading fits row #" << i << " error code: " << status;
546            Error(str.str());
547            str.str("");
548            for (unsigned int j=0;j<offsets.size(); j++)
549                str << offsets[j] << " ";
550            Error(str.str());
551        }
552        writeValuesFromFits(offsets, targetFile, fitsBuffer);
553    }
554    delete[] fitsBuffer;
555    return GetCurrentState();
556}
557// --------------------------------------------------------------------------
558//
559//! Set the name of the intput Fits file
560//! @param evt
561//!        the event transporting the file name
562//
563int FitsLoader::ConfigFileNamePlease(const Event& evt)
564{
565    fFileName = string(evt.GetText());
566    Message("New Fits file: " + fFileName);
567    return GetCurrentState();
568}
569// --------------------------------------------------------------------------
570//
571//! Set the name of the input table
572//! @param evt
573//!        the event transporting the table name
574//
575int FitsLoader::ConfigTableNamePlease(const Event& evt)
576{
577    fTableName = string(evt.GetText());
578    Message("New Fits table: " + fTableName);
579    return GetCurrentState();
580}
581// --------------------------------------------------------------------------
582//
583//! Retrieves the configuration parameters
584//! @param conf
585//!             the configuration object
586//
587void FitsLoader::SetupConfig(Configuration& conf)
588{
589    if (conf.Has("outfile"))
590    {
591        this->fFileOut = conf.Get<string>("outfile");
592        Message("Output file is: " + fFileOut);
593    }
594    if (conf.Has("fitsfile"))
595    {
596        this->fFileName = conf.Get<string>("fitsfile");
597        Message("Input fits is: " + fFileName);
598    }
599    if (conf.Has("tablename"))
600    {
601        this->fTableName = conf.Get<string>("tablename");
602        Message("Input Table is: " + fTableName);
603    }
604    if (conf.Has("dump"))
605    {
606        this->fDumpList = conf.Get<vector<string>>("dump");
607        Message("Dump list is:");
608        for (vector<string>::iterator it=fDumpList.begin(); it != fDumpList.end(); it++)
609            Message(*it);
610    }
611    if (conf.Has("precision"))
612    {
613        this->fStreamPrecision = conf.Get<int>("precision");
614        stringstream str;
615        str << "OFStream precision is: " << fStreamPrecision;
616        Message(str.str());
617    }
618}
619void RunThread(FitsLoader* loader)
620{
621    loader->Run(true);
622    Readline::Stop();
623}
624template<class T>
625int RunShell(Configuration& conf)
626{
627    static T shell(conf.GetName().c_str(), conf.Get<int>("console")!=1);
628
629    WindowLog& wout = shell.GetStreamOut();
630
631    FitsLoader loader(wout);
632    loader.SetupConfig(conf);
633    shell.SetReceiver(loader);
634
635    boost::thread t(boost::bind(RunThread, &loader));
636
637    shell.Run();
638
639    loader.Stop();
640
641    t.join();
642
643    return 0;
644}
645void PrintUsage()
646{
647    cout << "This is a usage. to be completed" << endl;
648}
649void PrintHelp()
650{
651    cout << "This is the help. I know, not so helpfull at the moment..." << endl;
652}
653void SetupConfiguration(Configuration& conf)
654{
655    po::options_description configp("Programm options");
656    configp.add_options()
657            ("console,c", var<int>(), "Use console (0=shell, 1=simple buffered, X=simple unbuffered)");
658
659    po::options_description configs("Fits Loader options");
660    configs.add_options()
661            ("outfile,o", var<string>(), "Output file")
662            ("fitsfile,f", var<string>(), "Input Fits file")
663            ("tablename,t", var<string>(), "Input Table")
664            ("dump,d", vars<string>(), "List of columns to dump")
665            ("precision,p", var<int>(), "Precision of ofstream")
666            ;
667//    conf.AddEnv("dns", "DIM_DNS_NODE");
668
669    conf.AddOptions(configp);
670    conf.AddOptions(configs);
671}
672int main(int argc, const char** argv)
673{
674    Configuration conf(argv[0]);
675    conf.SetPrintUsage(PrintUsage);
676    SetupConfiguration(conf);
677
678    po::variables_map vm;
679    try
680    {
681        vm = conf.Parse(argc, argv);
682    }
683    catch (exception& e)
684    {
685        cout << "Sorry, could not properly parse the program's arguments. returning" << endl;
686        cout << e.what() << endl;
687        return -1;
688    }
689
690    if (conf.HasPrint())
691            return -1;
692    if (conf.HasVersion())
693    {
694        FACT::PrintVersion(argv[0]);
695        return -1;
696    }
697
698    if (conf.HasHelp())
699    {
700        PrintHelp();
701        return -1;
702    }
703
704//    if (!conf.Has("console"))
705//        return Run(conf);
706    if (conf.Get<int>("console")==0)
707        return RunShell<LocalShell>(conf);
708    else
709        return RunShell<LocalConsole>(conf);
710
711    return 0;
712}
Note: See TracBrowser for help on using the repository browser.