source: trunk/FACT++/src/fitsloader.cc @ 10804

Last change on this file since 10804 was 10804, checked in by tbretz, 8 years ago
Simplified program options handling.
File size: 22.6 KB
Line 
1//****************************************************************
2/** @class FitsLoader
3
4  @brief Load a given Fits file and table, and dump selected columns if requested.
5
6  It derives from StateMachineDim. the first parent is here to enforce
7  a state machine behaviour
8  The possible states and transitions of the machine are:
9  \dot
10  digraph FitsLoader {
11          node [shape=record, fontname=Helvetica, fontsize=10];
12      e [label="Error" color="red"];
13   r [label="Ready"]
14   d [label="FileLoaded"]
15
16  e -> r
17  r -> e
18  r -> d
19  d -> r
20   }
21  \enddot
22 */
23 //****************************************************************
24#include "Event.h"
25#include "StateMachineDim.h"
26#include "WindowLog.h"
27#include "Configuration.h"
28#include "LocalControl.h"
29#include "Description.h"
30
31
32#include <boost/bind.hpp>
33#if BOOST_VERSION < 104400
34#if (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 4))
35#undef BOOST_HAS_RVALUE_REFS
36#endif
37#endif
38#include <boost/thread.hpp>
39
40#include <iostream>
41
42#include <CCfits/CCfits>
43
44class FitsLoader : public StateMachineDim
45{
46
47public:
48    enum
49    {
50        kSM_FileLoaded = 20,
51    } localstates_t;
52
53    FitsLoader(ostream& out);
54    ~FitsLoader();
55
56    ///Define command names
57    static const char* fLoadFits;
58    static const char* fUnloadFits;
59    static const char* fListColumns;
60    static const char* fDumpColumns;
61    static const char* fClearDumpList;
62    static const char* fDoDump;
63    static const char* fConfigFileName;
64    static const char* fConfigTableName;
65    static const char* fConfigPrecName;
66    static const char* fConfigFileOutName;
67
68private:
69    ///Name of the fits file to load
70    string fFileName;
71    ///Name of the table to load from the  file
72    string fTableName;
73    ///FITS pointer
74    CCfits::FITS* fFile;
75    ///Table pointer
76    CCfits::Table* fTable;
77    ///Precision of the ofstream. Used to output a given number of significant digits for floats or doubles
78    int fStreamPrecision;
79    ///Name of the output file
80    string fFileOut;
81    ///map between the column names and their CCfits objects
82    map<string, CCfits::Column*> fColMap;
83    ///List of the column names to be dumped
84    vector<string> fDumpList;
85    ///Transition from ready to fileLoaded.
86    int LoadPlease();
87    ///Transition from fileLoaded to ready
88    int UnloadPlease();
89    ///Lists the loaded column names
90    int ListColumnsPlease(const Event&);
91    ///Add a column name to the dump list
92    int AddDumpColumnsPlease(const Event&);
93    ///Clear the dump list
94    int ClearDumpListPlease(const Event&);
95    ///Perform the dumping, based on the current dump list
96    int DoDumpPlease(const Event&);
97    ///Set the name of the Fits file to be loaded
98    int ConfigFileNamePlease(const Event&);
99    ///Set the name of the table to be loaded
100    int ConfigTableNamePlease(const Event&);
101    ///Set the ofstream precision
102    int SetOFStreamPrecisionPlease(const Event&);
103    ///Set the name of the output file
104    int SetFileOutPlease(const Event&);
105    ///Calculate the buffer size required to read a row of the fits table, as well as the offsets to each column
106    vector<int> CalculateBufferSize();
107    ///Write a single row of the selected data
108    void writeValuesFromFits(vector<int>& offsets,ofstream& targetFile, unsigned char* fitsBuffer);
109
110public:
111    ///Configures the fitsLoader from the config file and/or command arguments.
112    void SetupConfig(Configuration& conf);
113};
114
115const char* FitsLoader::fLoadFits = "load";
116const char* FitsLoader::fUnloadFits = "unload";
117const char* FitsLoader::fListColumns = "list_columns";
118const char* FitsLoader::fDumpColumns = "add_dump";
119const char* FitsLoader::fClearDumpList = "clear_dump";
120const char* FitsLoader::fDoDump = "dump";
121const char* FitsLoader::fConfigFileName = "set_file";
122const char* FitsLoader::fConfigTableName = "set_table";
123const char* FitsLoader::fConfigPrecName = "set_prec";
124const char* FitsLoader::fConfigFileOutName = "set_outfile";
125
126// --------------------------------------------------------------------------
127//
128//! Set the name of the output file
129//! @param evt
130//!        the event transporting the file name
131//
132int FitsLoader::SetFileOutPlease(const Event& evt)
133{
134    fFileOut = string(evt.GetText());
135    stringstream str;
136    str << "Output file is now " << fFileOut;
137    Message(str.str());
138    return 0;
139}
140// --------------------------------------------------------------------------
141//
142//! Set the precision of the ofstream. So that an appropriate number of significant digits are outputted.
143//! @param evt
144//!        the event transporting the precision
145//
146int FitsLoader::SetOFStreamPrecisionPlease(const Event& evt)
147{
148    fStreamPrecision = evt.GetInt();
149    stringstream str;
150    str << "OFStream precision is now " << fStreamPrecision;
151    Message(str.str());
152    return 0;
153}
154// --------------------------------------------------------------------------
155//
156//! Writes a single row of the selected FITS data to the output file.
157//! @param offsets
158//!         a vector containing the offsets to the columns (in bytes)
159//! @param targetFile
160//!         the ofstream where to write to
161//! @param fitsBuffer
162//!         the memory were the row has been loaded by cfitsio
163//
164void FitsLoader::writeValuesFromFits(vector<int>& offsets,ofstream& targetFile, unsigned char* fitsBuffer)
165{
166    targetFile.precision(fStreamPrecision);
167    map<string, CCfits::Column*>::iterator it;
168   for (it=fColMap.begin(); it != fColMap.end(); it++)
169    {
170        bool found = false;
171        for (vector<string>::iterator jt=fDumpList.begin(); jt != fDumpList.end(); jt++)
172        {
173            if (it->first == *jt)
174            {
175                found = true;
176                break;
177            }
178        }
179        if (!found)
180            continue;
181       int offset = offsets[it->second->index()-1];
182       const char* charSrc = reinterpret_cast<char*>(&fitsBuffer[offset]);
183        unsigned char copyBuffer[30];//max size of a single variable
184        for (int width = 0; width<it->second->width(); width++)
185        {
186            switch (it->second->type())
187            {
188            case CCfits::Tbyte:
189                targetFile << *charSrc;
190                charSrc += sizeof(char);
191            break;
192            case CCfits::Tushort:
193                targetFile << *reinterpret_cast<const unsigned short*>(charSrc);
194                charSrc += sizeof(char);
195            break;
196            case CCfits::Tshort:
197                targetFile << *reinterpret_cast<const short*>(charSrc);
198                charSrc += sizeof(char);
199            break;
200            case CCfits::Tuint:
201                reverse_copy(charSrc, charSrc+sizeof(unsigned int), copyBuffer);
202                //warning suppressed in gcc4.0.2
203                targetFile << *reinterpret_cast<unsigned int*>(copyBuffer);
204                charSrc += sizeof(int);
205            break;
206            case CCfits::Tint:
207                reverse_copy(charSrc, charSrc+sizeof(int), copyBuffer);
208                targetFile << *reinterpret_cast<int*>(copyBuffer);
209                charSrc += sizeof(int);
210            break;
211            case CCfits::Tulong:
212                reverse_copy(charSrc, charSrc+sizeof(unsigned long), copyBuffer);
213                targetFile << *reinterpret_cast<unsigned long*>(copyBuffer);
214                charSrc += sizeof(int);
215            break;
216            case CCfits::Tlong:
217                reverse_copy(charSrc, charSrc+sizeof(long), copyBuffer);
218                targetFile << *reinterpret_cast<long*>(copyBuffer);
219                charSrc += sizeof(int);
220            break;
221            case CCfits::Tlonglong:
222                reverse_copy(charSrc, charSrc+sizeof(long long), copyBuffer);
223                targetFile << *reinterpret_cast<long long*>(copyBuffer);
224                charSrc += sizeof(long long);
225            break;
226            case CCfits::Tfloat:
227                reverse_copy(charSrc, charSrc+sizeof(float), copyBuffer);
228                targetFile << *reinterpret_cast<float*>(copyBuffer);
229                charSrc += sizeof(float);
230            break;
231            case CCfits::Tdouble:
232                reverse_copy(charSrc, charSrc+sizeof(double), copyBuffer);
233                targetFile << *reinterpret_cast<double*>(copyBuffer);
234                charSrc += sizeof(double);
235            break;
236            case CCfits::Tnull:
237            case CCfits::Tbit:
238            case CCfits::Tlogical:
239            case CCfits::Tstring:
240            case CCfits::Tcomplex:
241            case CCfits::Tdblcomplex:
242            case CCfits::VTbit:
243            case CCfits::VTbyte:
244            case CCfits::VTlogical:
245            case CCfits::VTushort:
246            case CCfits::VTshort:
247            case CCfits::VTuint:
248            case CCfits::VTint:
249            case CCfits::VTulong:
250            case CCfits::VTlong:
251            case CCfits::VTlonglong:
252            case CCfits::VTfloat:
253            case CCfits::VTdouble:
254            case CCfits::VTcomplex:
255            case CCfits::VTdblcomplex:
256                Error("Data type not implemented yet.");
257                return;
258            break;
259            default:
260                Error("THIS SHOULD NEVER BE REACHED");
261                return;
262            }//switch
263            targetFile << " ";
264        }//width loop
265    }//iterator over the columns
266    targetFile << endl;
267}
268
269// --------------------------------------------------------------------------
270//
271//! Calculates the required buffer size for reading one row of the current table.
272//! Also calculates the offsets to all the columns
273//
274vector<int> FitsLoader::CalculateBufferSize()
275{
276    vector<int> result;
277    map<int,int> sizes;
278    int size = 0;
279
280    for (map<string, CCfits::Column*>::iterator it=fColMap.begin(); it != fColMap.end(); it++)
281    {
282        int width = it->second->width();
283        switch (it->second->type())
284        {
285        case CCfits::Tbyte:
286        case CCfits::Tushort:
287        case CCfits::Tshort:
288            Message("short");
289            sizes[it->second->index()] =  sizeof(char)*width;
290        break;
291        case CCfits::Tuint:
292        case CCfits::Tint:
293            Message("int");
294            sizes[it->second->index()] =  sizeof(int)*width;
295        break;
296        case CCfits::Tulong:
297        case CCfits::Tlong:
298            Message("long");
299            sizes[it->second->index()] = sizeof(int)*width;
300        break;
301        case CCfits::Tlonglong:
302            Message("longlong");
303            sizes[it->second->index()] =  sizeof(long long)*width;
304        break;
305        case CCfits::Tfloat:
306            Message("float");
307            sizes[it->second->index()] =  sizeof(float)*width;
308        break;
309        case CCfits::Tdouble:
310            Message("double");
311            sizes[it->second->index()] =  sizeof(double)*width;
312        break;
313        case CCfits::Tnull:
314        case CCfits::Tbit:
315        case CCfits::Tlogical:
316        case CCfits::Tstring:
317        case CCfits::Tcomplex:
318        case CCfits::Tdblcomplex:
319        case CCfits::VTbit:
320        case CCfits::VTbyte:
321        case CCfits::VTlogical:
322        case CCfits::VTushort:
323        case CCfits::VTshort:
324        case CCfits::VTuint:
325        case CCfits::VTint:
326        case CCfits::VTulong:
327        case CCfits::VTlong:
328        case CCfits::VTlonglong:
329        case CCfits::VTfloat:
330        case CCfits::VTdouble:
331        case CCfits::VTcomplex:
332        case CCfits::VTdblcomplex:
333            Error("Data type not implemented yet.");
334            return vector<int>();
335        break;
336        default:
337            Error("THIS SHOULD NEVER BE REACHED");
338            return vector<int>();
339        }
340    }
341    //calculate the offsets in the vector.
342    int checkIndex = 1;
343    for (map<int,int>::iterator it=sizes.begin(); it != sizes.end(); it++)
344    {
345        result.push_back(size);
346        size += it->second;
347        if (it->first != checkIndex)
348        {
349            stringstream str;
350            str << "Expected index " << checkIndex << " found " << it->first;
351            Error(str.str());
352        }
353        checkIndex++;
354    }
355    result.push_back(size);
356    return result;
357}
358// --------------------------------------------------------------------------
359//
360//! Constructor
361//! @param out
362//!        the ostream where to redirect the outputs
363//
364FitsLoader::FitsLoader(ostream& out) : StateMachineDim(out, "FITS_LOADER")
365{
366    //Add the existing states
367    AddStateName(kSM_FileLoaded,  "FileLoaded", "A Fits file has been loaded");
368
369    //Add the possible transitions
370    AddEvent(kSM_FileLoaded, fLoadFits, kSM_Ready)
371            (boost::bind(&FitsLoader::LoadPlease, this))
372            ("Loads the given Fits file");
373    AddEvent(kSM_Ready, fUnloadFits, kSM_FileLoaded)
374            (boost::bind(&FitsLoader::UnloadPlease, this))
375            ("Unloads the given Fits file");
376
377    //Add the possible configurations
378    AddEvent(fListColumns, "", kSM_FileLoaded)
379            (boost::bind(&FitsLoader::ListColumnsPlease, this, _1))
380            ("List the columns that were loaded from that file");
381    AddEvent(fDumpColumns, "C", kSM_FileLoaded)
382            (boost::bind(&FitsLoader::AddDumpColumnsPlease, this, _1))
383            ("Add a given column to the dumping list");
384    AddEvent(fClearDumpList, "", kSM_FileLoaded)
385            (boost::bind(&FitsLoader::ClearDumpListPlease, this, _1))
386            ("Clear the dumping list");
387    AddEvent(fDoDump, "", kSM_FileLoaded)
388            (boost::bind(&FitsLoader::DoDumpPlease, this, _1))
389            ("Perform the dump of columns data, based on the to dump list");
390    AddEvent(fConfigFileName, "C", kSM_Ready, kSM_FileLoaded)
391            (boost::bind(&FitsLoader::ConfigFileNamePlease, this, _1))
392            ("Gives the name of the Fits file to be loaded");
393    AddEvent(fConfigTableName, "C", kSM_Ready, kSM_FileLoaded)
394            (boost::bind(&FitsLoader::ConfigTableNamePlease, this, _1))
395            ("Gives the name of the Table to be loaded");
396    AddEvent(fConfigPrecName, "I", kSM_Ready, kSM_FileLoaded)
397            (boost::bind(&FitsLoader::SetOFStreamPrecisionPlease, this, _1))
398            ("Set the precision of the ofstream, i.e. the number of significant digits being outputted");
399    AddEvent(fConfigFileOutName, "C", kSM_Ready, kSM_FileLoaded)
400            (boost::bind(&FitsLoader::SetFileOutPlease, this, _1))
401            ("Set the name of the outputted file.");
402
403    fFile = NULL;
404    fStreamPrecision = 20;
405
406}
407// --------------------------------------------------------------------------
408//
409//! Destructor
410//
411FitsLoader::~FitsLoader()
412{
413    if (fFile)
414        delete fFile;
415    fFile = NULL;
416}
417// --------------------------------------------------------------------------
418//
419//! Loads the fits file based on the current parameters
420//
421int FitsLoader::LoadPlease()
422{
423    ostringstream str;
424    try
425    {
426        fFile = new CCfits::FITS(fFileName);
427    }
428    catch (CCfits::FitsException e)
429     {
430         str << "Could not open FITS file " << fFileName << " reason: " << e.message();
431         Error(str);
432         return kSM_Ready;
433     }
434    str.str("");
435    const multimap< string, CCfits::ExtHDU * > extMap = fFile->extension();
436    if (extMap.find(fTableName) == extMap.end())
437    {
438        str.str("");
439        str << "Could not open table " << fTableName << ". Tables in file are: ";
440        for (std::multimap<string, CCfits::ExtHDU*>::const_iterator it=extMap.begin(); it != extMap.end(); it++)
441            str << it->first << " ";
442        Error(str.str());
443        return kSM_Ready;
444    }
445    else
446        fTable = dynamic_cast<CCfits::Table*>(extMap.find(fTableName)->second);
447    int numRows = fTable->rows();
448    str.str("");
449    str << "Loaded table has " << numRows << " rows";
450    Message(str.str());
451
452    fColMap = fTable->column();
453    if (fDumpList.size() != 0)
454    {
455        bool should_clear = false;
456        for (vector<string>::iterator it=fDumpList.begin(); it!= fDumpList.end(); it++)
457        {
458            if (fColMap.find(*it) == fColMap.end())
459            {
460                should_clear = true;
461                Error("Config-given dump list contains invalid entry " + *it + " clearing the list");
462            }
463        }
464        if (should_clear)
465            fDumpList.clear();
466    }
467    return kSM_FileLoaded;
468}
469// --------------------------------------------------------------------------
470//
471//! Unloads the Fits file
472//
473int FitsLoader::UnloadPlease()
474{
475    if (fFile)
476        delete fFile;
477    else
478        Error("Error: Fits file is  NULL while it should not have been");
479    fFile = NULL;
480    return kSM_Ready;
481}
482// --------------------------------------------------------------------------
483//
484//! List the columns that are in the loaded Fits table
485//
486int FitsLoader::ListColumnsPlease(const Event&)
487{
488    Message("Columns in the loaded table are:");
489    map<string, CCfits::Column*>::iterator it;
490    for (it=fColMap.begin(); it != fColMap.end(); it++)
491        Message(it->first);
492    return GetCurrentState();
493}
494// --------------------------------------------------------------------------
495//
496//! Add a given column name  to the list of columns to dump
497//! @param evt
498//!        the event transporting the column name
499//
500int FitsLoader::AddDumpColumnsPlease(const Event& evt)
501{
502    string evtText(evt.GetText());
503    //TODO check that this column indeed exist in the file
504    if (fColMap.find(evtText) != fColMap.end())
505        fDumpList.push_back(evtText);
506    else
507        Error("Could not find column " + evtText + " int table");
508    Message("New dump list:");
509    for (vector<string>::iterator it=fDumpList.begin(); it != fDumpList.end(); it++)
510        Message(*it);
511    return GetCurrentState();
512}
513// --------------------------------------------------------------------------
514//
515//! Clear the list of columns to dump
516//
517int FitsLoader::ClearDumpListPlease(const Event&)
518{
519    fDumpList.clear();
520    Message("Dump list is now empty");
521    return GetCurrentState();
522}
523// --------------------------------------------------------------------------
524//
525//! Perform the actual dump, based on the current parameters
526//
527int FitsLoader::DoDumpPlease(const Event&)
528{
529    fTable->makeThisCurrent();
530    vector<int> offsets = CalculateBufferSize();
531    int size = offsets[offsets.size()-1];
532    offsets.pop_back();
533    unsigned char* fitsBuffer = new unsigned char[size];
534
535    ofstream targetFile(fFileOut);
536    int status = 0;
537
538    for (int i=1;i<=fTable->rows(); i++)
539    {
540        fits_read_tblbytes(fFile->fitsPointer(), i, 1, size, fitsBuffer, &status);
541        if (status)
542        {
543            stringstream str;
544            str << "An error occurred while reading fits row #" << i << " error code: " << status;
545            Error(str.str());
546            str.str("");
547            for (unsigned int j=0;j<offsets.size(); j++)
548                str << offsets[j] << " ";
549            Error(str.str());
550        }
551        writeValuesFromFits(offsets, targetFile, fitsBuffer);
552    }
553    delete[] fitsBuffer;
554    return GetCurrentState();
555}
556// --------------------------------------------------------------------------
557//
558//! Set the name of the intput Fits file
559//! @param evt
560//!        the event transporting the file name
561//
562int FitsLoader::ConfigFileNamePlease(const Event& evt)
563{
564    fFileName = string(evt.GetText());
565    Message("New Fits file: " + fFileName);
566    return GetCurrentState();
567}
568// --------------------------------------------------------------------------
569//
570//! Set the name of the input table
571//! @param evt
572//!        the event transporting the table name
573//
574int FitsLoader::ConfigTableNamePlease(const Event& evt)
575{
576    fTableName = string(evt.GetText());
577    Message("New Fits table: " + fTableName);
578    return GetCurrentState();
579}
580// --------------------------------------------------------------------------
581//
582//! Retrieves the configuration parameters
583//! @param conf
584//!             the configuration object
585//
586void FitsLoader::SetupConfig(Configuration& conf)
587{
588    if (conf.Has("outfile"))
589    {
590        this->fFileOut = conf.Get<string>("outfile");
591        Message("Output file is: " + fFileOut);
592    }
593    if (conf.Has("fitsfile"))
594    {
595        this->fFileName = conf.Get<string>("fitsfile");
596        Message("Input fits is: " + fFileName);
597    }
598    if (conf.Has("tablename"))
599    {
600        this->fTableName = conf.Get<string>("tablename");
601        Message("Input Table is: " + fTableName);
602    }
603    if (conf.Has("dump"))
604    {
605        this->fDumpList = conf.Get<vector<string>>("dump");
606        Message("Dump list is:");
607        for (vector<string>::iterator it=fDumpList.begin(); it != fDumpList.end(); it++)
608            Message(*it);
609    }
610    if (conf.Has("precision"))
611    {
612        this->fStreamPrecision = conf.Get<int>("precision");
613        stringstream str;
614        str << "OFStream precision is: " << fStreamPrecision;
615        Message(str.str());
616    }
617}
618void RunThread(FitsLoader* loader)
619{
620    loader->Run(true);
621    Readline::Stop();
622}
623template<class T>
624int RunShell(Configuration& conf)
625{
626    static T shell(conf.GetName().c_str(), conf.Get<int>("console")!=1);
627
628    WindowLog& wout = shell.GetStreamOut();
629
630    FitsLoader loader(wout);
631    loader.SetupConfig(conf);
632    shell.SetReceiver(loader);
633
634    boost::thread t(boost::bind(RunThread, &loader));
635
636    shell.Run();
637
638    loader.Stop();
639
640    t.join();
641
642    return 0;
643}
644void PrintUsage()
645{
646    cout << "This is a usage. to be completed" << endl;
647}
648void PrintHelp()
649{
650    cout << "This is the help. I know, not so helpfull at the moment..." << endl;
651}
652void SetupConfiguration(Configuration& conf)
653{
654    po::options_description configp("Programm options");
655    configp.add_options()
656            ("console,c", var<int>(), "Use console (0=shell, 1=simple buffered, X=simple unbuffered)");
657
658    po::options_description configs("Fits Loader options");
659    configs.add_options()
660            ("outfile,o", var<string>(), "Output file")
661            ("fitsfile,f", var<string>(), "Input Fits file")
662            ("tablename,t", var<string>(), "Input Table")
663            ("dump,d", vars<string>(), "List of columns to dump")
664            ("precision,p", var<int>(), "Precision of ofstream")
665            ;
666//    conf.AddEnv("dns", "DIM_DNS_NODE");
667
668    conf.AddOptions(configp);
669    conf.AddOptions(configs);
670}
671int main(int argc, const char** argv)
672{
673    Configuration conf(argv[0]);
674    conf.SetPrintUsage(PrintUsage);
675    SetupConfiguration(conf);
676
677    po::variables_map vm;
678    try
679    {
680        vm = conf.Parse(argc, argv);
681    }
682#if BOOST_VERSION > 104000
683    catch (po::multiple_occurrences &e)
684    {
685        cerr << "Program options invalid due to: " << e.what() << " of '" << e.get_option_name() << "'." << endl;
686        return -1;
687    }
688#endif
689    catch (exception& e)
690    {
691        cerr << "Program options invalid due to: " << e.what() << endl;
692        return -1;
693    }
694
695    if (conf.HasVersion() || conf.HasPrint())
696        return -1;
697
698    if (conf.HasHelp())
699    {
700        PrintHelp();
701        return -1;
702    }
703
704//    if (!conf.Has("console"))
705//        return Run(conf);
706    if (conf.Get<int>("console")==0)
707        return RunShell<LocalShell>(conf);
708    else
709        return RunShell<LocalConsole>(conf);
710
711    return 0;
712}
Note: See TracBrowser for help on using the repository browser.