source: trunk/FACT++/src/root2csv.cc@ 19794

#include <random>

#include <boost/regex.hpp>
#include <boost/filesystem.hpp>
#include <boost/algorithm/string/join.hpp>

#include "tools.h"
#include "Time.h"
#include "Configuration.h"

#include <TROOT.h>
#include <TSystem.h>
#include <TChain.h>
#include <TLeaf.h>
#include <TError.h>
#include <TTreeFormula.h>
#include <TTreeFormulaManager.h>

using namespace std;
namespace fs = boost::filesystem;

// ------------------------------------------------------------------------

struct Map : pair<string, string>
{
    Map() { }
};

std::istream &operator>>(std::istream &in, Map &m)
{
    const istreambuf_iterator<char> eos;
    string txt(istreambuf_iterator<char>(in), eos);

    const boost::regex expr("((.*)[^\\\\])/(.*)");
    boost::smatch match;
    if (!boost::regex_match(txt, match, expr))
        throw runtime_error("Could not evaluate map argument: "+txt);

    m.first  = match[1].str();
    m.second = match[3].str();

    return in;
}

void SetupConfiguration(Configuration &conf)
{
    po::options_description control("Root to SQL");
    control.add_options()
        ("file",           vars<string>()->required(),"The root files to read from")
        ("out,o",          var<string>()->required(), "Output file name")
        ("force,f",        po_switch(),               "Force overwrite if output file already exists.")
        ("append,a",       po_switch(),               "Append to an existing file (not check for the format is done!)")
        ("tree,t",         var<string>("Events"),     "Name of the root tree to convert")
        ("ignore",         vars<string>(),            "Ignore the given leaf, if the given regular expression matches")
        ("alias.*",        var<string>(),             "Define an alias")
        ("auto-alias",     vars<Map>(),               "Regular expression to define aliases from the branch names automatically")
        ("header",         var<uint16_t>(uint16_t(0)),"Type of header line (0: preceeding #, 1: without preceeding #, 2: none)")
        ("add.*",          var<string>(),             "Define an additional column")
        ("selector",       var<string>("1"),          "Define a selector for the columns (colums where this evaluates to a value <=0 are discarded)")
        ("skip",           po_switch(),               "Discards all default leaves and writes only the columns defined by --add.*")
        ("first",          var<int64_t>(int64_t(0)),  "First event to start with (default: 0), mainly for test purpose")
        ("max",            var<int64_t>(int64_t(0)),  "Maximum number of events to process (0: all), mainly for test purpose")
        //("const.*",        var<string>(),             "Insert a constant number into the given column (--const.mycolumn=5). A special case is `/.../.../`")
        ("dry-run",        po_switch(),               "Do not create or manipulate any output file")
        ;

    po::options_description split("Splitting options");
    split.add_options()
        ("split-sequence,S", vars<uint16_t>(),            "Split data sequentially into several trees/files (e.g. 1, 1, 2)")
        ("split-quantile,Q", vars<double>(),              "Split data randomly into several trees/files (e.g. 0.5, 1)")
        ("seed", var<uint64_t>(mt19937_64::default_seed), "Seed value in case of random split")
        ;

    po::options_description debug("Debug options");
    debug.add_options()
        ("print-ls",       po_switch(),               "Calls TFile::ls()")
        ("print-branches", po_switch(),               "Print the branches found in the tree")
        ("print-leaves",   po_switch(),               "Print the leaves found in the tree (this is what is processed)")
        ("verbose,v",      var<uint16_t>(1),          "Verbosity (0: quiet, 1: default, 2: more, 3, ...)")
        ;

    po::positional_options_description p;
    p.add("file", -1); // All positional options

    conf.AddOptions(control);
    conf.AddOptions(split);
    conf.AddOptions(debug);
    conf.SetArgumentPositions(p);
}

void PrintUsage()
{
    cout <<
        "root2csv - Reads data from a root tree and writes a csv file\n"
        "\n"
        "For convenience, this documentation uses the extended version of the options, "
        "refer to the output below to get the abbreviations.\n"
        "\n"
        "This is a general purpose tool to fill the contents of a root file into a database "
        "as long as this is technically possible and makes sense. Note that root can even "
        "write complex data like a TH1F into a database, this is not the purpose of this "
        "program.\n"
        "\n"
        "Each root tree has branches and leaves (the basic data types). These leaves can "
        "be read independently of the classes which were used to write the root file. "
        "The default tree to read from is 'Events' but the name can be overwritten "
        "using --tree. The default table name to fill the data into is identical to "
        "the tree name. It can be overwritten using --table.\n"
        "\n"
        "To get a list of the contents (keys and trees) of a root file, you can use --print-ls. "
        "The name of each column to which data is filled from a leave is obtained from "
        "the leaves' names. The leave names can be checked using --print-leaves. "
        "A --print-branches exists for convenience to print only the high-level branches. "
        "Sometimes these names might be quite unconvenient like MTime.fTime.fMilliSec or "
        "just MHillas.fWidth. To allow to simplify column names, regular expressions "
        "(using boost's regex) can be defined to change the names. Note that these regular "
        "expressions are applied one by one on each leaf's name. A valid expression could "
        "be:\n"
        "   --map=MHillas\\.f/\n"
        "which would remove all occurances of 'MHillas.f'. This option can be used more than "
        "once. They are applied in sequence. A single match does not stop the sequence.\n"
        "\n"
        "Sometimes it might also be convenient to skip a leaf. This can be done with "
        "the --ignore resource. If the given regular expresion yields a match, the "
        "leaf will be ignored. Note that the regular expression works on the raw-name "
        "of the leaf not the readily mapped SQL column names. Example:\n"
        "   --ignore=ThetaSq\\..*\n"
        "will skip all leaved which start with 'ThetaSq.'. This option can be used"
        "more than once.\n"
        "\n"
        "The data type of each column is kept as close as possible to the leaves' data "
        "types. If for some reason this is not wanted, the data type of the SQL column "
        "can be overwritten with --sql-type sql-column/sql-ytpe, for example:\n"
        "   --sql-type=FileId/UNSIGNED INT\n"
        "while the first argument of the name of the SQL column to which the data type "
        "should be applied. The second column is the basic SQL data type. The option can "
        "be given more than once.\n"
        "\n"
        "Database interaction:\n"
        "\n"
        "To drop an existing table, --drop can be used.\n"
        "\n"
        "To create a table according to theSQL  column names and data types, --create "
        "can be used. The query used can be printed with --print-create even --create "
        "has not been specified.\n"
        "\n"
        "To choose the columns which should become primary keys, use --primary, "
        "for example:\n"
        "   --primary=col1\n"
        "To define more than one column as primary key, the option can be given more than "
        "once. Note that the combination of these columns must be unique.\n"
        "\n"
        "All columns are created as NOT NULL as default. To force a database engine "
        "and/or a storage format, use --engine and --row-format.\n"
        "\n"
        "Usually, the INSERT query would fail if the PRIMARY key exists already. "
        "This can be avoided using the 'ON DUPLICATE KEY UPDATE' directive. With the "
        "--duplicate, you can specify what should be updated in case of a duplicate key. "
        "To keep the row untouched, you can just update the primary key "
        "with the identical primary key, e.g. --duplicate='MyPrimary=VALUES(MyPrimary)'. "
        "The --duplicate resource can be specified more than once to add more expressions "
        "to the assignment_list. For more details, see the MySQL manual.\n"
        "\n"
        "For debugging purpose, or to just create or drop a table, the final insert "
        "query can be skipped using --no-insert. Note that for performance reason, "
        "all data is collected in memory and a single INSERT query is issued at the "
        "end.\n"
        "\n"
        "Another possibility is to add the IGNORE keyword to the INSERT query by "
        "--ignore-errors, which essentially ignores all errors and turns them into "
        "warnings which are printed after the query succeeded.\n"
        "\n"
        "Using a higher verbosity level (-v), an overview of the written columns or all "
        "processed leaves is printed depending on the verbosity level. The output looks "
        "like the following\n"
        "   Leaf name [root data type] (SQL name)\n"
        "for example\n"
        "   MTime.fTime.fMilliSec [Long64_t] (MilliSec)\n"
        "which means that the leaf MTime.fTime.fMilliSec is detected to be a Long64_t "
        "which is filled into a column called MilliSec. Leaves with non basic data types "
        "are ignored automatically and are marked as (-n/a-). User ignored columns "
        "are marked as (-ignored-).\n"
        "\n"
        "A constant value for the given file can be inserted by using the --const directive. "
        "For example --const.mycolumn=42 would insert 42 into a column called mycolumn. "
        "The column is created as INT UNSIGNED as default which can be altered by "
        "--sql-type. A special case is a value of the form `/regex/format/`. Here, the given "
        "regular expression is applied to the filename and it is newly formated with "
        "the new format string. Uses the standard formatting rules to replace matches "
        "(those used by ECMAScript's replace method).\n"
        "\n"
        "Usually the previously defined constant values are helpful to create an index "
        "which relates unambiguously the inserted data to the file. It might be useful "
        "to delete all data which belongs to this particular file before new data is "
        "entered. This can be achieved with the `--delete` directive. It deletes all "
        "data from the table before inserting new data which fulfills the condition "
        "defined by the `--const` directives.\n"
        "\n"
        "The constant values can also be used for a conditional execution (--conditional). "
        "If any row with the given constant values are found, the execution is stopped "
        "(note that this happend after the table drop/create but before the delete/insert.\n"
        "\n"
        "To ensure efficient access for a conditonal execution, it makes sense to have "
        "an index created for those columns. This can be done during table creation "
        "with the --index option.\n"
        "\n"
        "To create the index as a UNIQUE INDEX, you can use the --unique option which "
        "implies --index.\n"
        "\n"
        "If a query failed, the query is printed to stderr together with the error message. "
        "For the main INSERT query, this is only true if the verbosity level is at least 2 "
        "or the query has less than 80*25 bytes.\n"
        "\n"
        "In case of success, 0 is returned, a value>0 otherwise.\n"
        "\n"
        "Usage: root2sql [options] -uri URI rootfile.root\n"
        "\n"
        ;
    cout << endl;
}

enum BasicType_t
{
    kNone = 0,
    kConst,
    kFloat,
    kDouble,
    kInt16,
    kUInt16,
    kInt32,
    kUInt32,
    kInt64,
    kUInt64,
};

static const map<string, pair<BasicType_t, string>> ConvRoot =
{
    { "Float_t",   { kFloat,  "FLOAT"             } },
    { "Double_t",  { kDouble, "DOUBLE"            } },
    { "ULong64_t", { kUInt64, "BIGINT UNSIGNED"   } },
    { "Long64_t",  { kInt64,  "BIGINT"            } },
    { "UInt_t",    { kUInt32, "INT UNSIGNED"      } },
    { "Int_t",     { kInt32,  "INT"               } },
    { "UShort_t",  { kUInt16, "SMALLINT UNSIGNED" } },
    { "Short_t",   { kInt16,  "SMALLINT"          } },
};

struct Container
{
    static map<void*, size_t> counter;

    string branch; // branch name
    string column; // column name
    BasicType_t type;
    size_t num;
    void *ptr;

    Container(const string &b, const string &c, const BasicType_t &t, const size_t n=1) : branch(b), column(c), type(t), num(n), ptr(0)
    {
        switch (t)
        {
        case kFloat:  ptr = new Float_t[n];   break;
        case kDouble: ptr = new Double_t[n];  break;
        case kInt16:  ptr = new Short_t[n];   break;
        case kUInt16: ptr = new UShort_t[n];  break;
        case kInt32:  ptr = new Int_t[n];     break;
        case kUInt32: ptr = new UInt_t[n];    break;
        case kInt64:  ptr = new Long64_t[n];  break;
        case kUInt64: ptr = new ULong64_t[n]; break;
        case kConst:
        case kNone:
            break;
        }
        counter[ptr]++;
    }
    Container(const string &c, const string &value) : branch(value), column(c), type(kConst), num(1), ptr(0)
    {
    }

    Container(const Container &c) : branch(c.branch), column(c.column), type(c.type), num(c.num), ptr(c.ptr)
    {
        counter[ptr]++;
    }

    ~Container()
    {
        counter[ptr]--;
        if (counter[ptr]==0)
            ::operator delete[](ptr); // It seems root is deleting it already
    }

    string fmt(const size_t &index) const
    {
        ostringstream str;

        switch (type)
        {
        case kFloat:   str << setprecision(8) << reinterpret_cast<Float_t*>(ptr)[index];  break;
        case kDouble:  str << setprecision(16) << reinterpret_cast<Double_t*>(ptr)[index]; break;
        case kInt16:   str << reinterpret_cast<Short_t*>(ptr)[index]; break;
        case kUInt16:  str << reinterpret_cast<UShort_t*>(ptr)[index]; break;
        case kInt32:   str << reinterpret_cast<Int_t*>(ptr)[index]; break;
        case kUInt32:  str << reinterpret_cast<UInt_t*>(ptr)[index]; break;
        case kInt64:   str << reinterpret_cast<Long64_t*>(ptr)[index]; break;
        case kUInt64:  str << reinterpret_cast<ULong64_t*>(ptr)[index]; break;
        case kConst:   str << branch; break;
        case kNone:
            break;
        }

        //if (str.str()=="nan" || str.str()=="-nan" || str.str()=="inf" || str.str()=="-inf")
        //    return "NULL";

        return str.str();
    }
};

map<void*, size_t> Container::counter;

void ErrorHandlerAll(Int_t level, Bool_t abort, const char *location, const char *msg)
{
    if (string(msg).substr(0,24)=="no dictionary for class ")
        return;
    if (string(msg).substr(0,15)=="unknown branch ")
        return;

    DefaultErrorHandler(level, abort, location, msg);
}

// --------------------------- Write Header --------------------------------
void WriteHeader(ostream &out, const vector<Container> &vec, const vector<TTreeFormula*> &form, bool skip, uint16_t header)
{
    if (header>1)
        return;
    if (header==0)
        out << "# ";

    vector<string> join;

    if (!skip)
    {
        for (auto v=vec.cbegin(); v!=vec.cend(); v++)
        {
            const size_t N = v->num;
            for (size_t i=0; i<N; i++)
            {
                string name = v->column;
                if (N!=1)
                    name += "["+to_string(i)+"]";
                join.emplace_back(name);
            }
        }
    }

    for (auto v=form.cbegin(); v!=form.cend(); v++)
        join.emplace_back((*v)->GetName());

    out << boost::join(join, " ") << "\n";
}

int CheckFile(TString &path, bool force, int verbose)
{
    gSystem->ExpandPathName(path);

    FileStat_t stat;
    const Int_t  exist  = !gSystem->GetPathInfo(path, stat);
    const Bool_t _write = !gSystem->AccessPathName(path,  kWritePermission) && R_ISREG(stat.fMode);

    if (exist)
    {
        if (!_write)
        {
            cerr << "File '" << path << "' is not writable." << endl;
            return 2;
        }

        if (!force)
        {
            cerr << "File '" << path << "' already exists." << endl;
            return 3;
        }
        else
        {
            if (verbose>0)
                cerr << "File '" << path << "' will be overwritten." << endl;
        }
    }
    return exist ? 0 : -1;
}

void GetLeaves(vector<string> &list, const TTreeFormula &f)
{
    int i=0;
    while (1)
    {
        const auto l = f.GetLeaf(i++);
        if (!l)
            return;
        list.emplace_back(l->GetName());
    }
}

int main(int argc, const char* argv[])
{
    Time start;

    gROOT->SetBatch();
    SetErrorHandler(ErrorHandlerAll);

    Configuration conf(argv[0]);
    conf.SetPrintUsage(PrintUsage);
    SetupConfiguration(conf);

    if (!conf.DoParse(argc, argv))
        return 127;

    // ----------------------------- Evaluate options --------------------------
    const vector<string> files   = conf.Vec<string>("file");
    const string out             = conf.Get<string>("out");
    const string tree            = conf.Get<string>("tree");

    const bool force             = conf.Get<bool>("force");
    const bool append            = conf.Get<bool>("append");
    const bool dryrun            = conf.Get<bool>("dry-run");
    const bool skip              = conf.Get<bool>("skip");

    const uint16_t verbose       = conf.Get<uint16_t>("verbose");
    const int64_t  first         = conf.Get<int64_t>("first");
    const int64_t  max           = conf.Get<int64_t>("max");
    const uint16_t header        = conf.Get<uint16_t>("header");

    const bool print_ls          = conf.Get<bool>("print-ls");
    const bool print_branches    = conf.Get<bool>("print-branches");
    const bool print_leaves      = conf.Get<bool>("print-leaves");

    const vector<string> _ignore = conf.Vec<string>("ignore");
    const vector<Map> autoalias  = conf.Vec<Map>("auto-alias");

    // ----------------------------- Setup splitting ---------------------------

    vector<uint16_t> split_seq   = conf.Vec<uint16_t>("split-sequence");
    vector<double>   split_quant = conf.Vec<double>("split-quantile");

    if (!split_seq.empty() && !split_quant.empty())
        throw runtime_error("Only splitting by --split-sequence or --split-quantile is allowed.");

    const size_t num_split = split_seq.size()+split_quant.size()==0 ? 0 :
        ::max(split_seq.size(), split_quant.size()+1);

    map<size_t, size_t> split_lut;
    for (size_t i=0; i<split_seq.size(); i++)
    {
        const size_t sz = split_lut.size();
        for (size_t j=0; j<split_seq[i]; j++)
            split_lut.emplace(j+sz, i);
    }

    for (size_t i=0; i<split_quant.size(); i++)
        if (split_quant[i]<0 || split_quant[i]>=1)
            throw runtime_error("Splitting quantiles must be in the range [0;1)");

    for (size_t i=1; i<split_quant.size(); i++)
    {
        if (split_quant[i]<=split_quant[i-1])
            throw runtime_error("Splitting quantiles must be in increasing order.");
    }

    // -------------------------------------------------------------------------

    const uniform_real_distribution<double> distribution(0,1);
    mt19937_64 generator;
    generator.seed(conf.Get<uint64_t>("seed"));
    auto rndm = bind(distribution, generator);

    // -------------------------------------------------------------------------

    if (verbose>0)
    {
        cout << "\n-------------------------- Evaluating input ------------------------\n";
        cout << "Start Time: " << Time::sql << Time(Time::local) << endl;
    }

    if (verbose>0)
        cout << "Processing Tree: " << tree << endl;

    TChain c(tree.c_str());

    uint64_t cnt = 0;
    for (const auto &file : files)
    {
        const auto add = c.Add(file.c_str(), 0);
        if (verbose>0)
            cout << file << ": " << add << " file(s) added." << endl;
        cnt += add;
    }

    if (cnt==0)
    {
        cerr << "No files found." << endl;
        return 1;
    }

    if (verbose>0)
        cout << cnt << " file(s) found." << endl;

    if (print_ls)
    {
        cout << '\n';
        c.ls();
        cout << '\n';
    }

    c.SetMakeClass(1);

    TObjArray *branches = c.GetListOfBranches();
    TObjArray *leaves   = c.GetListOfLeaves();

    if (print_branches)
    {
        cout << '\n';
        branches->Print();
    }

    const auto entries = c.GetEntriesFast();

    if (verbose>0)
        cout << branches->GetEntries() << " branches found." << endl;

    if (print_leaves)
    {
        cout << '\n';
        leaves->Print();
    }
    if (verbose>0)
    {
        cout << leaves->GetEntries() << " leaves found." << endl;
        cout << entries << " events found." << endl;
    }

    // ----------------------------------------------------------------------

    if (verbose>0)
        cout << "\n-------------------------- Evaluating output -----------------------" << endl;

    vector<Container> vec;

/*
    const auto fixed = conf.GetWildcardOptions("const.*");

    string where;
    vector<string> vindex;
    for (auto it=fixed.cbegin(); it!=fixed.cend(); it++)
    {
        const string name = it->substr(6);
        string val  = conf.Get<string>(*it);

        boost::smatch match;
        if (boost::regex_match(val, match, boost::regex("\\/(.+)(?<!\\\\)\\/(.*)(?<!\\\\)\\/")))
        {
            const string reg = match[1];
            const string fmt = match[2];

            val = boost::regex_replace(file, boost::regex(reg), fmt.empty()?"$0":fmt,
                                       boost::regex_constants::format_default|boost::regex_constants::format_no_copy);

            if (verbose>0)
            {
                cout << "Regular expression detected for constant column `" << *it << "`\n";
                cout << "Filename converted with /" << reg << "/ to /" << fmt << "/\n";
                cout << "Filename: " << file << '\n';
                cout << "Result: " << val << endl;
            }
        }

        if (verbose>2)
            cout << "\n" << val << " [-const-]";
        if (verbose>1)
            cout << " (" << name << ")";

        string sqltype = "INT UNSIGNED";

        for (auto m=sqltypes.cbegin(); m!=sqltypes.cend(); m++)
            if (m->first==name)
                sqltype = m->second;

        if (!vec.empty())
            query += ",\n";
        query += "   `"+name+"` "+sqltype+" NOT NULL COMMENT '--user--'";

        vec.emplace_back(name, val);
        where += " AND `"+name+"`="+val;
        vindex.emplace_back(name);
    }
    */

    // ------------------------- Setup all branches in tree -------------------

        TIter Next(leaves);
        TObject *o = 0;
        while ((o=Next()))
        {
            TLeaf *L = c.GetLeaf(o->GetName());

            string name = o->GetName();

            for (auto m=autoalias.cbegin(); m!=autoalias.cend(); m++)
                name = boost::regex_replace(name, boost::regex(m->first), m->second);

            if (name!=o->GetName())
            {
                if (verbose>0)
                    cout << "Auto-alias: " << name << " = " << o->GetName() << endl;
                if (!c.SetAlias(name.c_str(), o->GetName()))
                    cout << "WARNING - Alias could not be established!" << endl;
            }

            if (skip)
                continue;

            if (verbose>2)
                cout << '\n' << L->GetTitle() << " {" << L->GetTypeName() << "}";

            if (L->GetLenStatic()!=L->GetLen())
            {
                if (verbose>2)
                    cout << " (-skipped-)";
                continue;
            }

            bool found = false;
            for (auto b=_ignore.cbegin(); b!=_ignore.cend(); b++)
            {
                if (boost::regex_match(o->GetName(), boost::regex(*b)))
                {
                    found = true;
                    if (verbose>2)
                        cout << " (-ignored-)";
                    break;
                }
            }

            if (found)
                continue;

            const string tn = L->GetTypeName();

            auto it = ConvRoot.find(tn);
            if (it==ConvRoot.end())
            {
                if (verbose>2)
                    cout << " (-n/a-)";
                continue;
            }

            if (verbose==2)
                cout << '\n' << L->GetTitle() << " {" << L->GetTypeName() << "}";

            if (verbose>1)
                cout << " (" << name << ")";

            vec.emplace_back(o->GetTitle(), name, it->second.first, L->GetLenStatic());
            c.SetBranchAddress(o->GetTitle(), vec.back().ptr);
        }

        if (verbose>0)
        {
            if (skip)
                cout << "Default columns skipped: ";
            cout << vec.size() << " default leaf/leaves setup for reading." << endl;
        }


    // ------------------- Configure manual aliases ----------------------------

    const auto valiases = conf.GetWildcardOptions("alias.*");
    if (verbose>0 && valiases.size()>0)
        cout << '\n';
    for (auto it=valiases.cbegin(); it!=valiases.cend(); it++)
    {
        const string name = it->substr(6);
        const string val  = conf.Get<string>(*it);

        if (verbose>0)
            cout << "Alias: " << name << " = " << val << endl;

        if (!c.SetAlias(name.c_str(), val.c_str()))
        {
            cerr << "Alias could not be established!" << endl;
            return 2;
        }
    }

    // -------------------------- Configure Selector --------------------------

    vector<string> leaflist;
    c.SetBranchStatus("*", 1);

    TTreeFormulaManager *manager = new TTreeFormulaManager;

    if (verbose>0)
        cout << "\nSelector: " <<  conf.Get<string>("selector") << endl;

    TTreeFormula selector("Selector", conf.Get<string>("selector").c_str(), &c);
    if (selector.GetNdim()==0)
    {
        cerr << "Compilation of Selector failed!" << endl;
        return 3;
    }
    selector.SetQuickLoad(kTRUE);
    manager->Add(&selector);
    GetLeaves(leaflist, selector);

    // -------------------- Configure additional columns ----------------------

    vector<TTreeFormula*> formulas;

    const auto vform = conf.GetWildcardOptions("add.*");
    if (verbose>0 && vform.size()>0)
        cout << '\n';
    for (auto it=vform.cbegin(); it!=vform.cend(); it++)
    {
        const string name = it->substr(4);
        const string val  = conf.Get<string>(*it);

        if (verbose>0)
            cout << "Adding column: " << name << " = " << val << endl;

        TTreeFormula *form = new TTreeFormula(name.c_str(), val.c_str(), &c);
        if (form->GetNdim()==0)
        {
            cerr << "Compilation of Column failed!" << endl;
            return 4;
        }
        form->SetQuickLoad(kTRUE);
        formulas.emplace_back(form);
        manager->Add(form);
        GetLeaves(leaflist, *form);
    }
    manager->Sync();

    if (verbose>0)
        cout << '\n' << formulas.size() << " additional columns setup for writing." << endl;

    // --------------------- Setup all branches in formulas -------------------

    for (auto l=leaflist.cbegin(); l!=leaflist.cend(); l++)
    {
        // Branch address already set
        if (c.GetBranch(l->c_str())->GetAddress())
            continue;

        TLeaf *L = c.GetLeaf(l->c_str());

        if (verbose>2)
            cout << '\n' << L->GetTitle() << " {" << L->GetTypeName() << "}";

        if (L->GetLenStatic()!=L->GetLen())
        {
            if (verbose>2)
                cout << " (-skipped-)";
            continue;
        }

        const string tn = L->GetTypeName();

        auto it = ConvRoot.find(tn);
        if (it==ConvRoot.end())
        {
            if (verbose>2)
                cout << " (-n/a-)";
            continue;
        }

        if (verbose==2)
            cout << '\n' << L->GetTitle() << " {" << L->GetTypeName() << "}";

        if (verbose>1)
            cout << " (" << *l << ")";

        vec.emplace_back(l->c_str(), l->c_str(), it->second.first, L->GetLenStatic());
        c.SetBranchAddress(l->c_str(), vec.back().ptr);
    }
    if (verbose>1)
        cout << '\n';

    // ------------------------- Enable branch reading ------------------------

    UInt_t datatype = 0;
    const bool has_datatype = c.SetBranchAddress("DataType.fVal", &datatype) >= 0;

    // Seting up branch status (must be after all SetBranchAddress)
    c.SetBranchStatus("*", 0);
    for (auto v=vec.cbegin(); v!=vec.cend(); v++)
        if (v->type!=kConst)
            c.SetBranchStatus(v->branch.c_str(), 1);

    if (has_datatype)
    {
        c.SetBranchStatus("DataType.fVal", 1);
        if (verbose>0)
            cout << "Rows with DataType.fVal!=1 will be skipped." << endl;
    }

    // -------------------------------------------------------------------------

    if (num_split)
    {
        cout << "\nSplitting configured " << (split_seq.empty()?"randomly":"in sequence") << " into " << num_split << " files." << endl;
        if (!split_quant.empty())
            cout << "Seed value configured as " << conf.Get<uint64_t>("seed") << "." << endl;
    }

    if (dryrun)
    {
        cout << "\nDry run: file output skipped!" << endl;
        return 0;
    }

    if (verbose>0)
        cout << "\n-------------------------- Converting file -------------------------" << endl;

    vector<ofstream> outfiles;

    if (num_split==0)
    {
        TString path(out.c_str());
        const int rc = CheckFile(path, force, verbose);
        if (rc>0)
            return rc;

        outfiles.emplace_back(path.Data(), append ? ios::app : ios::trunc);
        if (rc==-1 || (force && rc==0 && !append))
            WriteHeader(outfiles.back(), vec, formulas, skip, header);
    }
    else
    {
        for (size_t i=0; i<num_split; i++)
        {
            TString path(out.c_str());
            path += "-";
            path += i;

            const int rc = CheckFile(path, force, verbose);
            if (rc>0)
                return rc;
            outfiles.emplace_back(path.Data(), append ? ios::app : ios::trunc);
            if (rc==-1 || (force && rc==0 && !append))
                WriteHeader(outfiles.back(), vec, formulas, skip, header);
        }
    }

    // ---------------------------- Write Body --------------------------------
    size_t count = 0;
    vector<size_t> ncount(num_split?num_split:1);

    auto itree = c.GetTreeNumber();

    const size_t num = max>0 && (max-first)<entries ? (max-first) : entries;
    for (size_t j=first; j<num; j++)
    {
        c.GetEntry(j);
        if (has_datatype && datatype!=1)
            continue;

        if (itree != c.GetTreeNumber())
        {
            manager->UpdateFormulaLeaves();
            itree = c.GetTreeNumber();
        }

        if (selector.GetNdim() && selector.EvalInstance(0)<=0)
            continue;

        size_t index = 0;
        if (!split_lut.empty())
            index = split_lut[count % split_lut.size()];
        if (!split_quant.empty())
        {
            const float r = rndm();
            for (; r>=split_quant[index]; index++)
                if (index==split_quant.size())
                    break;
        }

        vector<string> join;

        if (!skip)
        {
            for (auto v=vec.cbegin(); v!=vec.cend(); v++)
            {
                const size_t N = v->num;
                for (size_t i=0; i<N; i++)
                    join.emplace_back(v->fmt(i));
            }
        }

        for (auto v=formulas.cbegin(); v!=formulas.cend(); v++)
            join.emplace_back(to_string((*v)->EvalInstance(0)));

        outfiles[index] << boost::join(join, " ") << "\n";

        count ++;
        ncount[index] ++;
    }

    if (verbose>0)
    {
        cout << "\nTotal: N=" << count << " out of " << num << " row(s) written [N=" << first << ".." << num-1 << "]." << endl;
        for (int i=0; i<num_split; i++)
            cout << "File " << i << ": nrows=" << ncount[i] << '\n';
        cout << '\n';
    }

    if (verbose>0)
    {
        cout << "Total execution time: " << Time().UnixTime()-start.UnixTime() << "s.\n";
        cout << "Success!\n" << endl;
    }
    return 0;
}