Changeset 19802

Timestamp:

10/27/19 11:11:01 (5 years ago)

Author:

tbretz

Message:

Moved the common Splitting algorithm to a common class 'Splitting' (added) and added the possibility to rename columns (csv2root)

Location:

trunk/FACT++/src

Files:

• Splitting.h (added)
• csv2root.cc (modified) (14 diffs)
• root2csv.cc (modified) (12 diffs)
• rootifysql.cc (modified) (11 diffs)

trunk/FACT++/src/csv2root.cc

@@ -1 @@
-#include <random>
-
 #include <boost/regex.hpp>
 #include <boost/filesystem.hpp>
@@ -7 +5 @@
 #include "tools.h"
 #include "Time.h"
-#include "Configuration.h"
+#include "Splitting.h"
 
 #include <TROOT.h>
@@ -18 +16 @@
 using namespace std;
 namespace fs = boost::filesystem;
+
+// ------------------------------------------------------------------------
+
 
 // ------------------------------------------------------------------------
@@ -32 +33 @@
         ("compression,c",  var<uint16_t>(1),          "zlib compression level for the root file")
         ("no-header,n",    po_switch(),               "Use if the first line contains no header")
+        ("rename.*",       var<string>(),             "Can be used to rename a column")
         ("dry-run",        po_switch(),               "Do not create or manipulate any output file")
-        ;
-
-    po::options_description split("Splitting options");
-    split.add_options()
-        ("split-sequence,S", vars<uint16_t>(),            "Split data sequentially into several trees/files (e.g. 1, 1, 2)")
-        ("split-quantile,Q", vars<double>(),              "Split data randomly into several trees/files (e.g. 0.5, 1)")
-        ("seed", var<uint64_t>(mt19937_64::default_seed), "Seed value in case of random split")
-        ;
-
-    po::options_description debug("Debug options");
-    debug.add_options()
         ("verbose,v",      var<uint16_t>(1),          "Verbosity (0: quiet, 1: default, 2: more, 3, ...)")
         ;
@@ -53 +44 @@
 
     conf.AddOptions(control);
-    conf.AddOptions(split);
-    conf.AddOptions(debug);
+    conf.AddOptions(Tools::Splitting::options());
     conf.SetArgumentPositions(p);
 }
@@ -90 +80 @@
         "with --compression.\n"
         "\n"
-        "For several purposes, it might be convenient to split the output to several "
-        "different root-treess. This can be done using the --split-sequence (-S) "
-        "and the --split-quantile (-Q) options. If a split sequence is defined as "
-        "-S 1 -S 2 -S 1 the events are split by 1:2:1 in this sequence order. If "
-        "quantiles are given as -Q 0.5 -Q 0.6, the first tree will contain 50% of "
-        "the second one 10% and the third one 40%. The corresponding seed value can "
-        "be set with --seed.\n"
+        "Columns can be renamed with --rename.new=old\n"
+        "\n"
+        << Tools::Splitting::usage() <<
         "\n"
         "In case of success, 0 is returned, a value>0 otherwise.\n"
@@ -178 +164 @@
     }
 
-    // ----------------------------- Setup splitting ---------------------------
-
-    vector<uint16_t> split_seq   = conf.Vec<uint16_t>("split-sequence");
-    vector<double>   split_quant = conf.Vec<double>("split-quantile");
-
-    if (!split_seq.empty() && !split_quant.empty())
-        throw runtime_error("Only splitting by --split-sequence or --split-quantile is allowed.");
-
-    const size_t num_split = split_seq.size()+split_quant.size()==0 ? 0 :
-        ::max(split_seq.size(), split_quant.size()+1);
-
-    map<size_t, size_t> split_lut;
-    for (size_t i=0; i<split_seq.size(); i++)
-    {
-        const size_t sz = split_lut.size();
-        for (size_t j=0; j<split_seq[i]; j++)
-            split_lut.emplace(j+sz, i);
-    }
-
-    for (size_t i=0; i<split_quant.size(); i++)
-        if (split_quant[i]<0 || split_quant[i]>=1)
-            throw runtime_error("Splitting quantiles must be in the range [0;1)");
-
-    for (size_t i=1; i<split_quant.size(); i++)
-    {
-        if (split_quant[i]<=split_quant[i-1])
-            throw runtime_error("Splitting quantiles must be in increasing order.");
-    }
-
     // -------------------------------------------------------------------------
 
-    const uniform_real_distribution<double> distribution(0,1);
-    mt19937_64 generator;
-    generator.seed(conf.Get<uint64_t>("seed"));
-    auto rndm = bind(distribution, generator);
-
-    // -------------------------------------------------------------------------
+    /*const*/ Tools::Splitting split(conf);
 
     if (verbose>0)
@@ -300 +252 @@
         cout << "Opened root file '" << path << "'.\n";
         cout << "Writing to tree: " << tree << ".\n";
+        split.print();
     }
 
@@ -306 +259 @@
 
     size_t entries = 0;
-    if (num_split==0)
+    if (split.empty())
     {
         if (AddTree(ttree, tfile, tree, update, verbose))
@@ -318 +271 @@
     {
         bool found = false;
-        for (size_t i=0; i<num_split; i++)
+        for (size_t i=0; i<split.size(); i++)
             found |= AddTree(ttree, tfile, tree+"["+to_string(i)+"]", update, verbose);
 
@@ -328 +281 @@
     }
 
+    const auto rename = conf.GetWildcardOptions("rename.*");
+
     vector<float> vec(numcol);
     for (int i=0; i<numcol; i++)
@@ -333 +288 @@
         string col = noheader ? Tools::Form("col%d", i) : title->At(i)->GetName();
 
+        if (verbose>1)
+            cout << "Column: " << col;
+
         boost::regex rexpr(":");
         col = boost::regex_replace(col, rexpr, "");
 
         if (verbose>1)
-            cout << "Column: " << col << '\n';
+            cout << " -> " << col;
+
+        for (auto it=rename.cbegin(); it!=rename.cend(); it++)
+        {
+            if (col!=it->substr(7))
+                continue;
+
+            col = conf.Get<string>(*it);
+            if (verbose>1)
+                cout << " -> " << col;
+            break;
+        }
+        if (verbose>1)
+            cout << endl;
 
         for (auto it=ttree.begin(); it!=ttree.end(); it++)
@@ -361 +332 @@
         if (buf.IsNull() || buf[0]=='#')
             continue;
-
-        valid++;
 
         TObjArray *arr = buf.Tokenize(" ");
@@ -386 +355 @@
         delete arr;
 
-
-        size_t index = 0;
-        if (!split_lut.empty())
-            index = split_lut[line % split_lut.size()];
-        if (!split_quant.empty())
-        {
-            const float r = rndm();
-            for (; r>=split_quant[index]; index++)
-                if (index==split_quant.size())
-                    break;
-        }
+        const size_t index = split.index(valid++);
 
         // Fill only branches for which an adress was set

trunk/FACT++/src/root2csv.cc

@@ -1 @@
-#include <random>
-
 #include <boost/regex.hpp>
 #include <boost/filesystem.hpp>
@@ -7 +5 @@
 #include "tools.h"
 #include "Time.h"
-#include "Configuration.h"
+#include "Splitting.h"
 
 #include <TROOT.h>
@@ -65 +63 @@
         ;
 
-    po::options_description split("Splitting options");
-    split.add_options()
-        ("split-sequence,S", vars<uint16_t>(),            "Split data sequentially into several trees/files (e.g. 1, 1, 2)")
-        ("split-quantile,Q", vars<double>(),              "Split data randomly into several trees/files (e.g. 0.5, 1)")
-        ("seed", var<uint64_t>(mt19937_64::default_seed), "Seed value in case of random split")
-        ;
-
     po::options_description debug("Debug options");
     debug.add_options()
@@ -84 +75 @@
 
     conf.AddOptions(control);
-    conf.AddOptions(split);
+    conf.AddOptions(Tools::Splitting::options());
     conf.AddOptions(debug);
     conf.SetArgumentPositions(p);
@@ -169 +160 @@
         "and all negative values are considered 'fales' (discard the entry).\n"
         "\n"
-        "For several purposes, it might be convenient to split the output to several "
-        "files. This can be achieved using the --split-sequence (-S) "
-        "and the --split-quantile (-Q) options. If a split sequence is defined as "
-        "-S 1 -S 2 -S 1 the events are split by 1:2:1 in this sequence order. If "
-        "quantiles are given as -Q 0.5 -Q 0.6, the first tree will contain 50% of "
-        "the second one 10% and the third one 40%. The corresponding seed value can "
-        "be set with --seed. Filenames are then created by adding an index after(!) "
-        "the extension, e.g. file.csv-0, file.csv-1, ...\n"
+        << Tools::Splitting::usage() <<
         "\n"
         "In case of success, 0 is returned, a value>0 otherwise.\n"
@@ -403 +387 @@
     const vector<Map> autoalias  = conf.Vec<Map>("auto-alias");
 
-    // ----------------------------- Setup splitting ---------------------------
-
-    vector<uint16_t> split_seq   = conf.Vec<uint16_t>("split-sequence");
-    vector<double>   split_quant = conf.Vec<double>("split-quantile");
-
-    if (!split_seq.empty() && !split_quant.empty())
-        throw runtime_error("Only splitting by --split-sequence or --split-quantile is allowed.");
-
-    const size_t num_split = split_seq.size()+split_quant.size()==0 ? 0 :
-        ::max(split_seq.size(), split_quant.size()+1);
-
-    map<size_t, size_t> split_lut;
-    for (size_t i=0; i<split_seq.size(); i++)
-    {
-        const size_t sz = split_lut.size();
-        for (size_t j=0; j<split_seq[i]; j++)
-            split_lut.emplace(j+sz, i);
-    }
-
-    for (size_t i=0; i<split_quant.size(); i++)
-        if (split_quant[i]<0 || split_quant[i]>=1)
-            throw runtime_error("Splitting quantiles must be in the range [0;1)");
-
-    for (size_t i=1; i<split_quant.size(); i++)
-    {
-        if (split_quant[i]<=split_quant[i-1])
-            throw runtime_error("Splitting quantiles must be in increasing order.");
-    }
-
     // -------------------------------------------------------------------------
 
-    const uniform_real_distribution<double> distribution(0,1);
-    mt19937_64 generator;
-    generator.seed(conf.Get<uint64_t>("seed"));
-    auto rndm = bind(distribution, generator);
-
-    // -------------------------------------------------------------------------
+    /*const*/ Tools::Splitting split(conf);
 
     if (verbose>0)
@@ -780 +730 @@
     // -------------------------------------------------------------------------
 
-    if (num_split)
-    {
-        cout << "\nSplitting configured " << (split_seq.empty()?"randomly":"in sequence") << " into " << num_split << " files." << endl;
-        if (!split_quant.empty())
-            cout << "Seed value configured as " << conf.Get<uint64_t>("seed") << "." << endl;
+    if (verbose>0)
+    {
+        cout << '\n';
+        split.print();
     }
 
@@ -798 +747 @@
     vector<ofstream> outfiles;
 
-    if (num_split==0)
+    if (split,empty())
     {
         TString path(out.c_str());
@@ -811 +760 @@
     else
     {
-        for (size_t i=0; i<num_split; i++)
+        for (size_t i=0; i<split.size(); i++)
         {
             TString path(out.c_str());
@@ -828 +777 @@
     // ---------------------------- Write Body --------------------------------
     size_t count = 0;
-    vector<size_t> ncount(num_split?num_split:1);
+    vector<size_t> ncount(split.empty()?1:split.size());
 
     auto itree = c.GetTreeNumber();
@@ -848 +797 @@
             continue;
 
-        size_t index = 0;
-        if (!split_lut.empty())
-            index = split_lut[count % split_lut.size()];
-        if (!split_quant.empty())
-        {
-            const float r = rndm();
-            for (; r>=split_quant[index]; index++)
-                if (index==split_quant.size())
-                    break;
-        }
+        const size_t index = split.index(count++);
+        ncount[index]++;
 
         vector<string> join;
@@ -875 +816 @@
 
         outfiles[index] << boost::join(join, " ") << "\n";
-
-        count ++;
-        ncount[index] ++;
     }
 

trunk/FACT++/src/rootifysql.cc

@@ -1 +1 @@
 #include "Database.h"
-
-#include <random>
 
 #include <boost/regex.hpp>
@@ -10 +8 @@
 #include "tools.h"
 #include "Time.h"
-#include "Configuration.h"
+#include "Splitting.h"
 
 #include <TROOT.h>
@@ -62 +60 @@
         ;
 
-    po::options_description split("Splitting options");
-    split.add_options()
-        ("split-sequence,S", vars<uint16_t>(),            "Split data sequentially into several trees/files (e.g. 1, 1, 2)")
-        ("split-quantile,Q", vars<double>(),              "Split data randomly into several trees/files (e.g. 0.5, 1)")
-        ("seed", var<uint64_t>(mt19937_64::default_seed), "Seed value in case of random split")
-        ;
-
     po::positional_options_description p;
     p.add("file", 1); // The 1st positional options (n=1)
@@ -76 +67 @@
     conf.AddOptions(ascii);
     conf.AddOptions(root);
-    conf.AddOptions(split);
+    conf.AddOptions(Tools::Splitting::options());
     conf.SetArgumentPositions(p);
 }
@@ -125 +116 @@
         "/*comment*/ or introduced with # (shell script style) or -- (SQL style).\n"
         "\n"
-        "For several purposes, it might be convenient to split the output to several "
-        "different root-trees or ascii files. This can be done using the --split-sequence (-S) "
-        "and the --split-quantile (-Q) options. If a split sequence is defined as "
-        "-S 1 -S 2 -S 1 the events are split by 1:2:1 in this sequence order. If "
-        "quantiled are given as -Q 0.5 -Q 0.6, the first tree will contain 50% of "
-        "the second one 10% and the third one 40%. The corresponding seed value can "
-        "be set with --seed.\n"
+        << Tools::Splitting::usage() <<
         "\n"
         "In case of success, 0 is returned, a value>0 otherwise.\n"
@@ -522 +507 @@
     // ----------------------- Setup splitting ---------------------------------
 
-    vector<uint16_t> split_seq   = conf.Vec<uint16_t>("split-sequence");
-    vector<double>   split_quant = conf.Vec<double>("split-quantile");
-
-    if (!split_seq.empty() && !split_quant.empty())
-        throw runtime_error("Only splitting by --split-sequence or --split-quantile is allowed.");
-
-    const size_t num_split = split_seq.size()+split_quant.size()==0 ? 0 :
-        ::max(split_seq.size(), split_quant.size()+1);
-
-    map<size_t, size_t> split_lut;
-    for (size_t i=0; i<split_seq.size(); i++)
-    {
-        const size_t sz = split_lut.size();
-        for (size_t j=0; j<split_seq[i]; j++)
-            split_lut.emplace(j+sz, i);
-    }
-
-    for (size_t i=0; i<split_quant.size(); i++)
-        if (split_quant[i]<0 || split_quant[i]>=1)
-            throw runtime_error("Splitting quantiles must be in the range [0;1)");
-
-    for (size_t i=1; i<split_quant.size(); i++)
-    {
-        if (split_quant[i]<=split_quant[i-1])
-            throw runtime_error("Splitting quantiles must be in increasing order.");
-    }
-
-    // -------------------------------------------------------------------------
-
     const auto vars = conf.GetWildcardOptions("var.*");
 
@@ -558 +514 @@
 
     // -------------------------------------------------------------------------
+
+    /*const*/ Tools::Splitting split(conf);
 
     if (verbose>0)
@@ -826 +784 @@
         cout << "Opening file '" << path << "' [compression=" << compression << "]...\n";
         cout << "Writing data to tree '" << tree << "'" << (nofill?" (--skipped--)":"") << endl;
-        if (num_split)
-        {
-            cout << "Splitting configured " << (split_seq.empty()?"randomly":"in sequence") << " into " << num_split << " branches." << endl;
-            if (!split_quant.empty())
-                cout << "Seed value configured as " << conf.Get<uint64_t>("seed") << "." << endl;
-        }
+        split.print();
     }
 
@@ -875 +828 @@
     vector<TTree*> ttree;
 
-    if (num_split==0)
+    if (split.empty())
         ttree.emplace_back(new TTree(tree.c_str(), query.c_str()));
     else
-        for (size_t i=0; i<num_split; i++)
+        for (size_t i=0; i<split.size(); i++)
             ttree.emplace_back(new TTree((tree+"["+to_string(i)+"]").c_str(), query.c_str()));
 
@@ -955 +908 @@
     {
         vector<string> names;
-        if (num_split==0)
+        if (split.empty())
             names.emplace_back(write);
         else
-            for (size_t i=0; i<num_split; i++)
+            for (size_t i=0; i<split.size(); i++)
                 names.emplace_back(write+"-"+to_string(i));
 
@@ -1017 +970 @@
     // ---------------------- Fill TTree with DB data --------------------------
 
-    const uniform_real_distribution<double> distribution(0,1);
-    mt19937_64 generator;
-    generator.seed(conf.Get<uint64_t>("seed"));
-    auto rndm = bind(distribution, generator);
-
     size_t count = 0;
     size_t skip  = 0;
     do
     {
-        size_t index = 0;
-        if (!split_lut.empty())
-            index = split_lut[count % split_lut.size()];
-        if (!split_quant.empty())
-        {
-            const float r = rndm();
-            for (; r>=split_quant[index]; index++)
-                if (index==split_quant.size())
-                    break;
-        }
-
-        count++;
+        size_t index = split.index(count++);
 
         ostringstream rtxt;