Index: /trunk/FACT++/CMakeLists.txt =================================================================== --- /trunk/FACT++/CMakeLists.txt (revision 19791) +++ /trunk/FACT++/CMakeLists.txt (revision 19792) @@ -249,5 +249,5 @@ IF (NOT TOOLS_ONLY AND NOT VIEWER_ONLY) # SET(ROOT_CONFIG_DEBUG 1) - FIND_PACKAGE(ROOT OPTIONAL_COMPONENTS GQt) + FIND_PACKAGE(ROOT OPTIONAL_COMPONENTS GQt TreePlayer ) ENDIF() @@ -603,4 +603,8 @@ MANPAGE(root2sql "FACT++ - root2sql - Fill contents of a root-tree into a MySQL database") +ADD_EXECUTABLE(root2csv src/root2csv.cc) +TARGET_LINK_LIBRARIES(root2csv ${HELP++LIBS} ${ROOT_LIBRARIES}) +MANPAGE(root2csv "FACT++ - root2csv - Convert a root-tree to a csv file") + ADD_EXECUTABLE(fits2sql src/fits2sql.cc) TARGET_LINK_LIBRARIES(fits2sql ${HELP++LIBS} ZLIB::ZLIB) @@ -1011,4 +1015,5 @@ INSTALL(TARGETS fits2sql DESTINATION "${CMAKE_INSTALL_BINDIR}") INSTALL(TARGETS root2sql DESTINATION "${CMAKE_INSTALL_BINDIR}") +INSTALL(TARGETS root2csv DESTINATION "${CMAKE_INSTALL_BINDIR}") INSTALL(TARGETS fitsdump DESTINATION "${CMAKE_INSTALL_BINDIR}") INSTALL(TARGETS zfits DESTINATION "${CMAKE_INSTALL_BINDIR}") Index: /trunk/FACT++/src/root2csv.cc =================================================================== --- /trunk/FACT++/src/root2csv.cc (revision 19792) +++ /trunk/FACT++/src/root2csv.cc (revision 19792) @@ -0,0 +1,947 @@ +#include + +#include +#include +#include + +#include "tools.h" +#include "Time.h" +#include "Configuration.h" + +#include +#include +#include +#include +#include +#include +#include + +using namespace std; +namespace fs = boost::filesystem; + +// ------------------------------------------------------------------------ + +struct Map : pair +{ + Map() { } +}; + +std::istream &operator>>(std::istream &in, Map &m) +{ + const istreambuf_iterator eos; + string txt(istreambuf_iterator(in), eos); + + const boost::regex expr("((.*)[^\\\\])/(.*)"); + boost::smatch match; + if (!boost::regex_match(txt, match, expr)) + throw runtime_error("Could not evaluate map argument: "+txt); + + m.first = match[1].str(); + m.second = match[3].str(); + + return in; +} + +void SetupConfiguration(Configuration &conf) +{ + po::options_description control("Root to SQL"); + control.add_options() + ("file", vars()->required(),"The root files to read from") + ("out,o", var()->required(), "Output file name") + ("force,f", po_switch(), "Force overwrite if output file already exists.") + ("append,a", po_switch(), "Append to an existing file (not check for the format is done!)") + ("create", po_switch(), "Create the database if not existing") + ("tree,t", var("Events"), "Name of the root tree to convert") + ("ignore", vars(), "Ignore the given leaf, if the given regular expression matches") + ("alias.*", var(), "Define an alias") + ("auto-alias", vars(), "Regular expression to define aliases from the branch names automatically") + ("add.*", var(), "Define an additional column") + ("selector", var("1"), "Define a selector for the columns (colums where this evaluates to a value <=0 are discarded)") + ("skip", po_switch(), "Discards all default leaves and writes only the columns defined by --add.*") + ("first", var(int64_t(0)), "First event to start with (default: 0), mainly for test purpose") + ("max", var(int64_t(0)), "Maximum number of events to process (0: all), mainly for test purpose") + ("const.*", var(), "Insert a constant number into the given column (--const.mycolumn=5). A special case is `/.../.../`") + ("dry-run", po_switch(), "Do not create or manipulate any output file") + ; + + po::options_description split("Splitting options"); + split.add_options() + ("split-sequence,S", vars(), "Split data sequentially into several trees/files (e.g. 1, 1, 2)") + ("split-quantile,Q", vars(), "Split data randomly into several trees/files (e.g. 0.5, 1)") + ("seed", var(mt19937_64::default_seed), "Seed value in case of random split") + ; + + po::options_description debug("Debug options"); + debug.add_options() + ("print-ls", po_switch(), "Calls TFile::ls()") + ("print-branches", po_switch(), "Print the branches found in the tree") + ("print-leaves", po_switch(), "Print the leaves found in the tree (this is what is processed)") + ("verbose,v", var(1), "Verbosity (0: quiet, 1: default, 2: more, 3, ...)") + ; + + po::positional_options_description p; + p.add("file", -1); // All positional options + + conf.AddOptions(control); + conf.AddOptions(split); + conf.AddOptions(debug); + conf.SetArgumentPositions(p); +} + +void PrintUsage() +{ + cout << + "root2csv - Reads data from a root tree and writes a csv file\n" + "\n" + "For convenience, this documentation uses the extended version of the options, " + "refer to the output below to get the abbreviations.\n" + "\n" + "This is a general purpose tool to fill the contents of a root file into a database " + "as long as this is technically possible and makes sense. Note that root can even " + "write complex data like a TH1F into a database, this is not the purpose of this " + "program.\n" + "\n" + "Each root tree has branches and leaves (the basic data types). These leaves can " + "be read independently of the classes which were used to write the root file. " + "The default tree to read from is 'Events' but the name can be overwritten " + "using --tree. The default table name to fill the data into is identical to " + "the tree name. It can be overwritten using --table.\n" + "\n" + "To get a list of the contents (keys and trees) of a root file, you can use --print-ls. " + "The name of each column to which data is filled from a leave is obtained from " + "the leaves' names. The leave names can be checked using --print-leaves. " + "A --print-branches exists for convenience to print only the high-level branches. " + "Sometimes these names might be quite unconvenient like MTime.fTime.fMilliSec or " + "just MHillas.fWidth. To allow to simplify column names, regular expressions " + "(using boost's regex) can be defined to change the names. Note that these regular " + "expressions are applied one by one on each leaf's name. A valid expression could " + "be:\n" + " --map=MHillas\\.f/\n" + "which would remove all occurances of 'MHillas.f'. This option can be used more than " + "once. They are applied in sequence. A single match does not stop the sequence.\n" + "\n" + "Sometimes it might also be convenient to skip a leaf. This can be done with " + "the --ignore resource. If the given regular expresion yields a match, the " + "leaf will be ignored. Note that the regular expression works on the raw-name " + "of the leaf not the readily mapped SQL column names. Example:\n" + " --ignore=ThetaSq\\..*\n" + "will skip all leaved which start with 'ThetaSq.'. This option can be used" + "more than once.\n" + "\n" + "The data type of each column is kept as close as possible to the leaves' data " + "types. If for some reason this is not wanted, the data type of the SQL column " + "can be overwritten with --sql-type sql-column/sql-ytpe, for example:\n" + " --sql-type=FileId/UNSIGNED INT\n" + "while the first argument of the name of the SQL column to which the data type " + "should be applied. The second column is the basic SQL data type. The option can " + "be given more than once.\n" + "\n" + "Database interaction:\n" + "\n" + "To drop an existing table, --drop can be used.\n" + "\n" + "To create a table according to theSQL column names and data types, --create " + "can be used. The query used can be printed with --print-create even --create " + "has not been specified.\n" + "\n" + "To choose the columns which should become primary keys, use --primary, " + "for example:\n" + " --primary=col1\n" + "To define more than one column as primary key, the option can be given more than " + "once. Note that the combination of these columns must be unique.\n" + "\n" + "All columns are created as NOT NULL as default. To force a database engine " + "and/or a storage format, use --engine and --row-format.\n" + "\n" + "Usually, the INSERT query would fail if the PRIMARY key exists already. " + "This can be avoided using the 'ON DUPLICATE KEY UPDATE' directive. With the " + "--duplicate, you can specify what should be updated in case of a duplicate key. " + "To keep the row untouched, you can just update the primary key " + "with the identical primary key, e.g. --duplicate='MyPrimary=VALUES(MyPrimary)'. " + "The --duplicate resource can be specified more than once to add more expressions " + "to the assignment_list. For more details, see the MySQL manual.\n" + "\n" + "For debugging purpose, or to just create or drop a table, the final insert " + "query can be skipped using --no-insert. Note that for performance reason, " + "all data is collected in memory and a single INSERT query is issued at the " + "end.\n" + "\n" + "Another possibility is to add the IGNORE keyword to the INSERT query by " + "--ignore-errors, which essentially ignores all errors and turns them into " + "warnings which are printed after the query succeeded.\n" + "\n" + "Using a higher verbosity level (-v), an overview of the written columns or all " + "processed leaves is printed depending on the verbosity level. The output looks " + "like the following\n" + " Leaf name [root data type] (SQL name)\n" + "for example\n" + " MTime.fTime.fMilliSec [Long64_t] (MilliSec)\n" + "which means that the leaf MTime.fTime.fMilliSec is detected to be a Long64_t " + "which is filled into a column called MilliSec. Leaves with non basic data types " + "are ignored automatically and are marked as (-n/a-). User ignored columns " + "are marked as (-ignored-).\n" + "\n" + "A constant value for the given file can be inserted by using the --const directive. " + "For example --const.mycolumn=42 would insert 42 into a column called mycolumn. " + "The column is created as INT UNSIGNED as default which can be altered by " + "--sql-type. A special case is a value of the form `/regex/format/`. Here, the given " + "regular expression is applied to the filename and it is newly formated with " + "the new format string. Uses the standard formatting rules to replace matches " + "(those used by ECMAScript's replace method).\n" + "\n" + "Usually the previously defined constant values are helpful to create an index " + "which relates unambiguously the inserted data to the file. It might be useful " + "to delete all data which belongs to this particular file before new data is " + "entered. This can be achieved with the `--delete` directive. It deletes all " + "data from the table before inserting new data which fulfills the condition " + "defined by the `--const` directives.\n" + "\n" + "The constant values can also be used for a conditional execution (--conditional). " + "If any row with the given constant values are found, the execution is stopped " + "(note that this happend after the table drop/create but before the delete/insert.\n" + "\n" + "To ensure efficient access for a conditonal execution, it makes sense to have " + "an index created for those columns. This can be done during table creation " + "with the --index option.\n" + "\n" + "To create the index as a UNIQUE INDEX, you can use the --unique option which " + "implies --index.\n" + "\n" + "If a query failed, the query is printed to stderr together with the error message. " + "For the main INSERT query, this is only true if the verbosity level is at least 2 " + "or the query has less than 80*25 bytes.\n" + "\n" + "In case of success, 0 is returned, a value>0 otherwise.\n" + "\n" + "Usage: root2sql [options] -uri URI rootfile.root\n" + "\n" + ; + cout << endl; +} + +enum BasicType_t +{ + kNone = 0, + kConst, + kFloat, + kDouble, + kInt16, + kUInt16, + kInt32, + kUInt32, + kInt64, + kUInt64, +}; + +static const map> ConvRoot = +{ + { "Float_t", { kFloat, "FLOAT" } }, + { "Double_t", { kDouble, "DOUBLE" } }, + { "ULong64_t", { kUInt64, "BIGINT UNSIGNED" } }, + { "Long64_t", { kInt64, "BIGINT" } }, + { "UInt_t", { kUInt32, "INT UNSIGNED" } }, + { "Int_t", { kInt32, "INT" } }, + { "UShort_t", { kUInt16, "SMALLINT UNSIGNED" } }, + { "Short_t", { kInt16, "SMALLINT" } }, +}; + +struct Container +{ + static map counter; + + string branch; // branch name + string column; // column name + BasicType_t type; + size_t num; + void *ptr; + + Container(const string &b, const string &c, const BasicType_t &t, const size_t n=1) : branch(b), column(c), type(t), num(n), ptr(0) + { + switch (t) + { + case kFloat: ptr = new Float_t[n]; break; + case kDouble: ptr = new Double_t[n]; break; + case kInt16: ptr = new Short_t[n]; break; + case kUInt16: ptr = new UShort_t[n]; break; + case kInt32: ptr = new Int_t[n]; break; + case kUInt32: ptr = new UInt_t[n]; break; + case kInt64: ptr = new Long64_t[n]; break; + case kUInt64: ptr = new ULong64_t[n]; break; + case kConst: + case kNone: + break; + } + counter[ptr]++; + } + Container(const string &c, const string &value) : branch(value), column(c), type(kConst), num(1), ptr(0) + { + } + + Container(const Container &c) : branch(c.branch), column(c.column), type(c.type), num(c.num), ptr(c.ptr) + { + counter[ptr]++; + } + + ~Container() + { + counter[ptr]--; + if (counter[ptr]==0) + ::operator delete[](ptr); // It seems root is deleting it already + } + + string fmt(const size_t &index) const + { + ostringstream str; + + switch (type) + { + case kFloat: str << setprecision(8) << reinterpret_cast(ptr)[index]; break; + case kDouble: str << setprecision(16) << reinterpret_cast(ptr)[index]; break; + case kInt16: str << reinterpret_cast(ptr)[index]; break; + case kUInt16: str << reinterpret_cast(ptr)[index]; break; + case kInt32: str << reinterpret_cast(ptr)[index]; break; + case kUInt32: str << reinterpret_cast(ptr)[index]; break; + case kInt64: str << reinterpret_cast(ptr)[index]; break; + case kUInt64: str << reinterpret_cast(ptr)[index]; break; + case kConst: str << branch; break; + case kNone: + break; + } + + //if (str.str()=="nan" || str.str()=="-nan" || str.str()=="inf" || str.str()=="-inf") + // return "NULL"; + + return str.str(); + } +}; + +map Container::counter; + +void ErrorHandlerAll(Int_t level, Bool_t abort, const char *location, const char *msg) +{ + if (string(msg).substr(0,24)=="no dictionary for class ") + return; + if (string(msg).substr(0,15)=="unknown branch ") + return; + + DefaultErrorHandler(level, abort, location, msg); +} + +// --------------------------- Write Header -------------------------------- +void WriteHeader(ostream &out, const vector &vec, const vector &form, bool skip) +{ + out << "#"; + + if (!skip) + { + for (auto v=vec.cbegin(); v!=vec.cend(); v++) + { + const size_t N = v->num; + for (size_t i=0; icolumn; + if (N!=1) + out << "["+to_string(i)+"]"; + } + } + } + + for (auto v=form.cbegin(); v!=form.cend(); v++) + out << " " << (*v)->GetName(); + + out << "\n"; +} + +int CheckFile(TString &path, bool force, int verbose) +{ + gSystem->ExpandPathName(path); + + FileStat_t stat; + const Int_t exist = !gSystem->GetPathInfo(path, stat); + const Bool_t _write = !gSystem->AccessPathName(path, kWritePermission) && R_ISREG(stat.fMode); + + if (exist) + { + if (!_write) + { + cerr << "File '" << path << "' is not writable." << endl; + return 2; + } + + if (!force) + { + cerr << "File '" << path << "' already exists." << endl; + return 3; + } + else + { + if (verbose>0) + cerr << "File '" << path << "' will be overwritten." << endl; + } + } + return exist ? 0 : -1; +} + +void GetLeaves(vector &list, const TTreeFormula &f) +{ + int i=0; + while (1) + { + const auto l = f.GetLeaf(i++); + if (!l) + return; + list.emplace_back(l->GetName()); + } +} + +int main(int argc, const char* argv[]) +{ + Time start; + + gROOT->SetBatch(); + SetErrorHandler(ErrorHandlerAll); + + Configuration conf(argv[0]); + conf.SetPrintUsage(PrintUsage); + SetupConfiguration(conf); + + if (!conf.DoParse(argc, argv)) + return 127; + + // ----------------------------- Evaluate options -------------------------- + const vector files = conf.Vec("file"); + const string out = conf.Get("out"); + const string tree = conf.Get("tree"); + + const bool force = conf.Get("force"); + const bool append = conf.Get("append"); + const bool dryrun = conf.Get("dry-run"); + const bool skip = conf.Get("skip"); + + const uint16_t verbose = conf.Get("verbose"); + const int64_t first = conf.Get("first"); + const int64_t max = conf.Get("max"); + + const bool print_ls = conf.Get("print-ls"); + const bool print_branches = conf.Get("print-branches"); + const bool print_leaves = conf.Get("print-leaves"); + + const vector _ignore = conf.Vec("ignore"); + const vector autoalias = conf.Vec("auto-alias"); + + // ----------------------------- Setup splitting --------------------------- + + vector split_seq = conf.Vec("split-sequence"); + vector split_quant = conf.Vec("split-quantile"); + + if (!split_seq.empty() && !split_quant.empty()) + throw runtime_error("Only splitting by --split-sequence or --split-quantile is allowed."); + + const size_t num_split = split_seq.size()+split_quant.size()==0 ? 0 : + ::max(split_seq.size(), split_quant.size()+1); + + map split_lut; + for (size_t i=0; i=1) + throw runtime_error("Splitting quantiles must be in the range [0;1)"); + + for (size_t i=1; i distribution(0,1); + mt19937_64 generator; + generator.seed(conf.Get("seed")); + auto rndm = bind(distribution, generator); + + // ------------------------------------------------------------------------- + + if (verbose>0) + { + cout << "\n-------------------------- Evaluating input ------------------------\n"; + cout << "Start Time: " << Time::sql << Time(Time::local) << endl; + } + + if (verbose>0) + cout << "Processing Tree: " << tree << endl; + + TChain c(tree.c_str()); + + uint64_t cnt = 0; + for (const auto &file : files) + { + const auto add = c.Add(file.c_str()); + if (verbose>0) + cout << file << ": " << add << " file(s) added." << endl; + cnt += add; + } + + if (cnt==0) + { + cerr << "No files found." << endl; + return 1; + } + + if (verbose>0) + cout << cnt << " file(s) found." << endl; + + if (print_ls) + { + cout << '\n'; + c.ls(); + cout << '\n'; + } + + c.SetMakeClass(1); + + TObjArray *branches = c.GetListOfBranches(); + TObjArray *leaves = c.GetListOfLeaves(); + + if (print_branches) + { + cout << '\n'; + branches->Print(); + } + + const auto entries = c.GetEntries(); + + if (verbose>0) + cout << branches->GetEntries() << " branches found." << endl; + + if (print_leaves) + { + cout << '\n'; + leaves->Print(); + } + if (verbose>0) + { + cout << leaves->GetEntries() << " leaves found." << endl; + cout << entries << " events found." << endl; + } + + // ---------------------------------------------------------------------- + + if (verbose>0) + cout << "\n-------------------------- Evaluating output -----------------------" << endl; + + vector vec; + +/* + const auto fixed = conf.GetWildcardOptions("const.*"); + + string where; + vector vindex; + for (auto it=fixed.cbegin(); it!=fixed.cend(); it++) + { + const string name = it->substr(6); + string val = conf.Get(*it); + + boost::smatch match; + if (boost::regex_match(val, match, boost::regex("\\/(.+)(?0) + { + cout << "Regular expression detected for constant column `" << *it << "`\n"; + cout << "Filename converted with /" << reg << "/ to /" << fmt << "/\n"; + cout << "Filename: " << file << '\n'; + cout << "Result: " << val << endl; + } + } + + if (verbose>2) + cout << "\n" << val << " [-const-]"; + if (verbose>1) + cout << " (" << name << ")"; + + string sqltype = "INT UNSIGNED"; + + for (auto m=sqltypes.cbegin(); m!=sqltypes.cend(); m++) + if (m->first==name) + sqltype = m->second; + + if (!vec.empty()) + query += ",\n"; + query += " `"+name+"` "+sqltype+" NOT NULL COMMENT '--user--'"; + + vec.emplace_back(name, val); + where += " AND `"+name+"`="+val; + vindex.emplace_back(name); + } + */ + + if (autoalias.size()) + { + TIter Next(leaves); + TObject *o = 0; + while ((o=Next())) + { + TLeaf *L = c.GetLeaf(o->GetName()); + if (L->GetLenStatic()!=L->GetLen()) + continue; + + string name = o->GetName(); +/* + bool found = false; + for (auto b=_ignore.cbegin(); b!=_ignore.cend(); b++) + { + if (boost::regex_match(name, boost::regex(*b))) + { + found = true; + break; + } + } + if (found) + continue; +*/ + const string tn = L->GetTypeName(); + auto it = ConvRoot.find(tn); + if (it==ConvRoot.end()) + continue; + + for (auto m=autoalias.cbegin(); m!=autoalias.cend(); m++) + name = boost::regex_replace(name, boost::regex(m->first), m->second); + + if (name==o->GetName()) + continue; + + if (verbose>0) + cout << "Auto-alias: " << name << " = " << o->GetName() << endl; + + if (!c.SetAlias(name.c_str(), o->GetName())) + cout << "WARNING - Alias could not be established!" << endl; + } + } + + // ------------------------ Configure Aliases ----------------------------- + + const auto valiases = conf.GetWildcardOptions("alias.*"); + if (verbose>0 && valiases.size()>0) + cout << '\n'; + for (auto it=valiases.cbegin(); it!=valiases.cend(); it++) + { + const string name = it->substr(6); + const string val = conf.Get(*it); + + if (verbose>0) + cout << "Alias: " << name << " = " << val << endl; + + if (!c.SetAlias(name.c_str(), val.c_str())) + { + cerr << "Alias could not be established!" << endl; + return 2; + } + } + + // -------------------------- Configure Selector -------------------------- + + vector leaflist; + c.SetBranchStatus("*", 1); + + TTreeFormulaManager *manager = new TTreeFormulaManager; + + if (verbose>0) + cout << "\nSelector: " << conf.Get("selector") << endl; + + TTreeFormula selector("Selector", conf.Get("selector").c_str(), &c); + if (selector.GetNdim()==0) + { + cerr << "Compilation of Selector failed!" << endl; + return 3; + } + selector.SetQuickLoad(kTRUE); + manager->Add(&selector); + GetLeaves(leaflist, selector); + + // -------------------- Configure additional columns ---------------------- + + vector formulas; + + const auto vform = conf.GetWildcardOptions("add.*"); + if (verbose>0 && vform.size()>0) + cout << '\n'; + for (auto it=vform.cbegin(); it!=vform.cend(); it++) + { + const string name = it->substr(4); + const string val = conf.Get(*it); + + if (verbose>0) + cout << "Column: " << name << " = " << val << endl; + + TTreeFormula *form = new TTreeFormula(name.c_str(), val.c_str(), &c); + if (form->GetNdim()==0) + { + cerr << "Compilation of Column failed!" << endl; + return 4; + } + form->SetQuickLoad(kTRUE); + formulas.emplace_back(form); + manager->Add(form); + GetLeaves(leaflist, *form); + } + manager->Sync(); + + if (verbose>0) + cout << '\n' << formulas.size() << " additional columns setup for writing." << endl; + + // ------------------------- Setup all branches in tree ------------------- + + if (!skip) + { + TIter Next(leaves); + TObject *o = 0; + while ((o=Next())) + { + TLeaf *L = c.GetLeaf(o->GetName()); + + if (verbose>2) + cout << '\n' << L->GetTitle() << " {" << L->GetTypeName() << "}"; + + if (L->GetLenStatic()!=L->GetLen()) + { + if (verbose>2) + cout << " (-skipped-)"; + continue; + } + + string name = o->GetName(); + + bool found = false; + for (auto b=_ignore.cbegin(); b!=_ignore.cend(); b++) + { + if (boost::regex_match(name, boost::regex(*b))) + { + found = true; + if (verbose>2) + cout << " (-ignored-)"; + break; + } + } + if (found) + continue; + + const string tn = L->GetTypeName(); + + auto it = ConvRoot.find(tn); + if (it==ConvRoot.end()) + { + if (verbose>2) + cout << " (-n/a-)"; + continue; + } + + if (verbose==2) + cout << '\n' << L->GetTitle() << " {" << L->GetTypeName() << "}"; + + if (verbose>1) + cout << " (" << name << ")"; + + vec.emplace_back(o->GetTitle(), name, it->second.first, L->GetLenStatic()); + c.SetBranchAddress(o->GetTitle(), vec.back().ptr); + } + } + + if (verbose>0) + cout << vec.size() << " default leaf/leaves setup for reading." << endl; + + // --------------------- Setup all branches in formulas ------------------- + + for (auto l=leaflist.cbegin(); l!=leaflist.cend(); l++) + { + // Branch address already set + if (c.GetBranch(l->c_str())->GetAddress()) + continue; + + TLeaf *L = c.GetLeaf(l->c_str()); + + if (verbose>2) + cout << '\n' << L->GetTitle() << " {" << L->GetTypeName() << "}"; + + if (L->GetLenStatic()!=L->GetLen()) + { + if (verbose>2) + cout << " (-skipped-)"; + continue; + } + + const string tn = L->GetTypeName(); + + auto it = ConvRoot.find(tn); + if (it==ConvRoot.end()) + { + if (verbose>2) + cout << " (-n/a-)"; + continue; + } + + if (verbose==2) + cout << '\n' << L->GetTitle() << " {" << L->GetTypeName() << "}"; + + if (verbose>1) + cout << " (" << *l << ")"; + + vec.emplace_back(l->c_str(), l->c_str(), it->second.first, L->GetLenStatic()); + c.SetBranchAddress(l->c_str(), vec.back().ptr); + } + if (verbose>1) + cout << '\n'; + + // ------------------------- Enable branch reading ------------------------ + + UInt_t datatype = 0; + const bool has_datatype = c.SetBranchAddress("DataType.fVal", &datatype) >= 0; + + // Seting up branch status (must be after all SetBranchAddress) + c.SetBranchStatus("*", 0); + for (auto v=vec.cbegin(); v!=vec.cend(); v++) + if (v->type!=kConst) + c.SetBranchStatus(v->branch.c_str(), 1); + + if (has_datatype) + { + c.SetBranchStatus("DataType.fVal", 1); + if (verbose>0) + cout << "Rows with DataType.fVal!=1 will be skipped." << endl; + } + + // ------------------------------------------------------------------------- + + if (num_split) + { + cout << "\nSplitting configured " << (split_seq.empty()?"randomly":"in sequence") << " into " << num_split << " files." << endl; + if (!split_quant.empty()) + cout << "Seed value configured as " << conf.Get("seed") << "." << endl; + } + + if (dryrun) + { + cout << "\nDry run: file output skipped!" << endl; + return 0; + } + + if (verbose>0) + cout << "\n-------------------------- Converting file -------------------------" << endl; + + vector outfiles; + + if (num_split==0) + { + TString path(out.c_str()); + const int rc = CheckFile(path, force, verbose); + if (rc>0) + return rc; + + outfiles.emplace_back(path.Data(), append ? ios::app : ios::trunc); + if (rc==-1 || (force && rc==0 && !append)) + WriteHeader(outfiles.back(), vec, formulas, skip); + } + else + { + for (size_t i=0; i0) + return rc; + outfiles.emplace_back(path.Data(), append ? ios::app : ios::trunc); + if (rc==-1 || (force && rc==0 && !append)) + WriteHeader(outfiles.back(), vec, formulas, skip); + } + } + + // ---------------------------- Write Body -------------------------------- + size_t count = 0; + vector ncount(num_split?num_split:1); + + auto itree = c.GetTreeNumber(); + + const size_t num = max>0 && (max-first)UpdateFormulaLeaves(); + itree = c.GetTreeNumber(); + } + + if (selector.GetNdim() && selector.EvalInstance(0)<=0) + continue; + + size_t index = 0; + if (!split_lut.empty()) + index = split_lut[count % split_lut.size()]; + if (!split_quant.empty()) + { + const float r = rndm(); + for (; r>=split_quant[index]; index++) + if (index==split_quant.size()) + break; + } + + if (!skip) + { + for (auto v=vec.cbegin(); v!=vec.cend(); v++) + { + const size_t N = v->num; + for (size_t i=0; i0) + outfiles[index] << " "; + + outfiles[index] << v->fmt(i); + } + } + } + + for (auto v=formulas.cbegin(); v!=formulas.cend(); v++) + { + if (v!=formulas.cbegin() || (vec.size()>0 && !skip)) + outfiles[index] << " "; + + outfiles[index] << setprecision(8) << (*v)->EvalInstance(0); + } + + outfiles[index] << "\n"; + + count ++; + ncount[index] ++; + } + + if (verbose>0) + { + cout << "\nTotal: N=" << count << " out of " << num << " row(s) written [N=" << first << ".." << num-1 << "]." << endl; + for (int i=0; i