source: trunk/FACT++/src/rootifysql.cc@ 19124

Last change on this file since 19124 was 19121, checked in by tbretz, 6 years ago
Do not print a warning if no ascii file should have been written, print execution time per row.
File size: 29.9 KB
Line 
1#include "Database.h"
2
3#include <regex>
4
5#include <boost/algorithm/string/join.hpp>
6#include <boost/tokenizer.hpp>
7#include <boost/algorithm/string.hpp>
8
9#include "tools.h"
10#include "Time.h"
11#include "Configuration.h"
12
13#include <TROOT.h>
14#include <TSystem.h>
15#include <TFile.h>
16#include <TTree.h>
17
18using namespace std;
19
20// ------------------------------------------------------------------------
21
22void SetupConfiguration(Configuration &conf)
23{
24 po::options_description control("Rootify SQL");
25 control.add_options()
26 ("uri,u", var<string>()->required(), "Database link as in\n\tuser:password@server[:port]/database.")
27 ("query,q", var<string>(""), "MySQL query (overwrites --file)")
28 ("file", var<string>("rootify.sql"), "An ASCII file with the MySQL query (overwrites --query)")
29 ("ignore-null,i", po_switch(), "Do not skip rows containing any NULL field")
30 ("out,o", var<string>("rootify.root"), "Output root file name")
31 ("force,f", po_switch(), "Force overwriting an existing root file ('RECREATE')")
32 ("update", po_switch(), "Update an existing root file with the new tree ('UPDATE')")
33 ("compression,c", var<uint16_t>(1), "zlib compression level for the root file")
34 ("tree,t", var<string>("Result"), "Name of the root tree")
35 ("ignore", vars<string>(), "Ignore the given columns")
36 ("display,d", po_switch(), "Displays contents on the screen (most usefull in combination with mysql statements as SHOW or EXPLAIN)")
37 ("write", var<string>(""), "Write output to an ascii file")
38 ("null,n", po_switch(), "Redirect the output file to /dev/null (mainly for debugging purposes, e.g. performance studies)")
39 ("no-fill", po_switch(), "Do not fill events into the root file (mainly for debugging purposes, e.g. performance studies)")
40 ("delimiter", var<string>(""), "The delimiter used if contents are displayed with --display (default=\\t)")
41 ("explain", po_switch(), "Requests an EXPLAIN from the server (shows the server optimized query)\nsee also https://dev.mysql.com/doc/refman/explain-output.html")
42 ("profiling", po_switch(), "Turn on profiling and print profile")
43 ("var.*", var<string>(), "Predefined SQL user variables (@VAR)")
44 ("env.*", vars<string>(), "Predefined environment for substitutions in the query ($ENV)")
45 ("list.*", var<string>(), "Predefined environment for substitutions in the query ($ENV). The list is read from the given file (one list entry per line)")
46 ("verbose,v", var<uint16_t>(1), "Verbosity (0: quiet, 1: default, 2: more, 3, ...)")
47 ;
48
49 po::positional_options_description p;
50 p.add("file", 1); // The 1st positional options (n=1)
51 p.add("out", 1); // The 2nd positional options (n=1)
52
53 conf.AddOptions(control);
54 conf.SetArgumentPositions(p);
55}
56
57void PrintUsage()
58{
59 cout <<
60 "rootifysql - Converts the result of a mysql query into a root file\n"
61 "\n"
62 "For convenience, this documentation uses the extended version of the options, "
63 "refer to the output below to get the abbreviations.\n"
64 "\n"
65 "Writes the result of a mysql query into a root file. For each column, a branch is "
66 "created of type double with the field name as name. This is usually the column name "
67 "if not specified otherwise by the AS mysql directive.\n"
68 "\n"
69 "Columns with CHAR or VARCHAR as field type are ignored. DATETIME, DATE and TIME "
70 "columns are converted to unix time (time_t). Rows containing any file which is "
71 "NULL are skipped if not suppressed by the --ignore-null option. Ideally, the query "
72 "is compiled in a way that no NULL field is returned. With the --display option the "
73 "result of the request is printed on the screen (NULL skipping still in action). "
74 "This can be useful to create an ascii file or to show results as 'SHOW DATABASES' "
75 "or 'EXPLAIN table'. To redirect the contents into an ascii file, the option -v0 "
76 "is useful. To suppredd writing to an output file --null can be used.\n"
77 "\n"
78 "The default is to read the query from a file called rootify.sql. Except if a different "
79 "filename is specified by the --file option or a query is given with --query.\n"
80 "\n"
81 "As a trick, the rootify.sql file can be made excutable (chmod u+x rootify.sql). "
82 "If the first line contains '#!rootifysql', the script can be executed directly.\n"
83 "\n"
84 "Columns whose name start with @ are skipped. If you want them in your output file "
85 "give them a name using AS, e.g. 'SELECT @A:=5 AS A'.\n"
86 "\n"
87 "You can use variables in your sql query like @MyVar and define them on the "
88 "command line. In this example with --var.MyVar=5\n"
89 "\n"
90 "You can use environment definitions for substitutions in your SQL query. "
91 "For example --env.TEST=5 would replace $TEST or ${TEST} in your query by 5."
92 "If you specifiy one environmentvariable more than once, a list is created. "
93 "For example --env.TEST=1 --env.TEST=2 --env.TEST=3 would substitute "
94 "$TEST or ${TEST} by '1, 2, 3'. This is useful for the SQL `IN` keyword. "
95 "You can also read the values for an enviroment substitution from a file "
96 "(one element per line), e.g. --env.TEST=file.txt. Lines starting with a # "
97 "are skipped.\n"
98 "\n"
99 "Comments in the query-file can be placed according to the SQL standard inline "
100 "/*comment*/ or introduced with # (shell script style) or -- (SQL style).\n"
101 "\n"
102 "In case of succes, 0 is returned, a value>0 otherwise.\n"
103 "\n"
104 "Usage: rootifysql [rootify.sql [rootify.root]] [-u URI] [-q query|-f file] [-i] [-o out] [-f] [-cN] [-t tree] [-vN]\n"
105 "\n"
106 ;
107 cout << endl;
108}
109
110struct ExplainParser
111{
112 string sql;
113
114 vector<string> vec;
115
116 string substitute(string _str, const regex &expr)
117 {
118 smatch match;
119 while (regex_search(_str, match, expr, regex_constants::format_first_only))
120 {
121 const auto &len = match.length(0);
122 const auto &pos = match.position(0);
123 const auto &str = match.str(0);
124
125 const auto it = find(vec.cbegin(), vec.cend(), str);
126 const size_t id = it==vec.cend() ? vec.size() : it-vec.cbegin();
127
128 _str.replace(pos, len, "{"+to_string(id)+"}");
129
130 if (it==vec.cend())
131 vec.push_back(str);//.substr(1, str.size()-2));
132 }
133
134 return _str;
135 }
136
137 string substitute(const string &str, const string &expr)
138 {
139 return substitute(str, regex(expr));
140 }
141
142 vector<string> queries;
143
144 string resub(string str)
145 {
146 // search for "KEYWORD expression"
147 regex reg("\\{[0-9]+\\}");
148
149 //smatch match;
150 smatch match;
151 while (regex_search(str, match, reg, regex_constants::format_first_only))
152 {
153 const auto &len = match.length(0);
154 const auto &pos = match.position(0);
155 const auto &arg = match.str(0); // Argument
156
157 const auto idx = atoi(arg.c_str()+1);
158
159 str.replace(pos, len, resub(vec[idx]));
160 }
161
162 return str;
163 }
164
165 void expression(string expr, size_t indent=0)
166 {
167 if (expr[0]=='{')
168 {
169 const auto idx = atoi(expr.c_str()+1);
170
171 // This is a subquery
172 if (vec[idx].substr(0,3)=="(/*")
173 {
174 cout << setw(indent) << ' ' << "(\n";
175 find_tokens(vec[idx], indent+4);
176 cout << setw(indent) << ' ' << ") ";
177 }
178 else
179 // This is just something to substitute back
180 if (vec[idx].substr(0,2)=="({")
181 {
182 cout << setw(indent) << ' ' << "(" << resub(vec[idx]) << ") ";
183 }
184 else
185 {
186 if (indent>0)
187 cout << setw(indent) << ' ';
188 cout << resub(vec[idx]);
189 }
190 }
191 else
192 {
193 if (indent>0)
194 cout << setw(indent) << ' ';
195 cout << resub(expr);
196 }
197 }
198
199 void find_tokens(string str, size_t indent=0)
200 {
201 // ( COMMENT )?( TOKEN )?(( {NNN} | NNN )( AS|ON ( {NNN}) ))?(,)?)
202 //regex reg("(\\/\\*\\ select\\#[0-9]+\\ \\*\\/\\ *)?([a-zA-Z ]+)?((\\{[0-9]+\\}|[0-9]+)(\\ ?([Aa][Ss]|[Oo][Nn])\\ ?(\\{[0-9]+\\}))?(,)?)");
203
204 const string _com = "\\/\\*\\ select\\#[0-9]+\\ \\*\\/\\ *";
205
206 const string _tok = "[a-zA-Z_ ]+";
207
208 const string _nnn = "\\{[0-9]+\\}|[0-9]+";
209
210 const string _as = "\\ ?([Aa][Ss])\\ ?";
211
212 // ( _nnn ) ( _as ( _nnn ))?(,)? // can also match noting in between two {NNN}
213 const string _exp = "("+_nnn+")" + "("+_as+"("+_nnn+"))?(,)?";
214
215 // Matche: ( _com )? ( ( _tok )? ( _exp ) | ( _tok ) )
216 regex reg("("+_com+")?" + "(" + "("+_tok+")?"+"("+_exp+")" + "|" + "("+_tok+")" + ")");
217
218 smatch match;
219 while (regex_search(str, match, reg, regex_constants::format_first_only))
220 {
221
222 const auto &com = match.str(1); // comment
223 const auto &tok1 = Tools::Trim(match.str(3)); // token with expression
224 const auto &arg1 = match.str(5); // argument 1
225 const auto &as = match.str(7); // as
226 const auto &arg2 = match.str(8); // argument 2
227 const auto &comma = match.str(9); // comma
228 const auto &tok2 = Tools::Trim(match.str(10)); // token without expression
229
230 if (!com.empty())
231 cout << setw(indent) << ' ' << "\033[34m" << com << "\033[0m" << '\n';
232
233 if (!tok1.empty())
234 cout << setw(indent) << ' ' << "\033[32m" << tok1 << "\033[0m" << '\n';
235 if (!tok2.empty())
236 cout << setw(indent) << ' ' << "\033[32m" << tok2 << "\033[0m" << '\n';
237
238 if (!arg1.empty())
239 {
240 expression(arg1, indent+4);
241
242 if (!as.empty())
243 cout << " \033[33m" << as << "\033[0m ";
244
245 if (!arg2.empty())
246 expression(arg2);
247
248 if (!comma.empty())
249 cout << ',';
250
251 cout << '\n';
252 }
253
254 str = str.substr(match.position(0)+match.length(0));
255 }
256 }
257
258
259 ExplainParser(const string &_sql) : sql(_sql)
260 {
261 // substitute all strings
262 sql = substitute(sql, "'[^']*'");
263
264 // substitute all escaped sequences (`something`.`something-else`)
265 sql = substitute(sql, "`[^`]*`(\\.`[^`]*`)*");
266
267 // substitute all paranthesis
268 sql = substitute(sql, "[a-zA-Z0-9_]*\\([^\\(\\)]*\\)");
269
270 //cout << sql << "\n\n";
271 find_tokens(sql);
272 cout << endl;
273 }
274};
275
276// Remove queries...
277void format(string sql)
278{
279 ExplainParser p(sql);
280
281 /*
282
283 SELECT
284 [ALL | DISTINCT | DISTINCTROW ]
285 [HIGH_PRIORITY]
286 [STRAIGHT_JOIN]
287 [SQL_SMALL_RESULT] [SQL_BIG_RESULT] [SQL_BUFFER_RESULT]
288 [SQL_CACHE | SQL_NO_CACHE] [SQL_CALC_FOUND_ROWS]
289 select_expr [, select_expr ...]
290 [FROM table_references
291 [PARTITION partition_list]
292 [WHERE where_condition]
293 [GROUP BY {col_name | expr | position}, ... [WITH ROLLUP]]
294 [HAVING where_condition]
295 [WINDOW window_name AS (window_spec)
296 [, window_name AS (window_spec)] ...]
297 [ORDER BY {col_name | expr | position}
298 [ASC | DESC], ... [WITH ROLLUP]]
299 [LIMIT {[offset,] row_count | row_count OFFSET offset}]
300 [INTO OUTFILE 'file_name'
301 [CHARACTER SET charset_name]
302 export_options
303 | INTO DUMPFILE 'file_name'
304 | INTO var_name [, var_name]]
305 [FOR {UPDATE | SHARE} [OF tbl_name [, tbl_name] ...] [NOWAIT | SKIP LOCKED]
306 | LOCK IN SHARE MODE]]
307 */
308
309 /*
310table_references:
311 escaped_table_reference [, escaped_table_reference] ...
312
313escaped_table_reference:
314 table_reference
315 | { OJ table_reference }
316
317table_reference:
318 table_factor
319 | join_table
320
321table_factor:
322 tbl_name [PARTITION (partition_names)]
323 [[AS] alias] [index_hint_list]
324 | table_subquery [AS] alias [(col_list)]
325 | ( table_references )
326
327join_table:
328 table_reference [INNER | CROSS] JOIN table_factor [join_condition]
329 | table_reference STRAIGHT_JOIN table_factor
330 | table_reference STRAIGHT_JOIN table_factor ON conditional_expr
331 | table_reference {LEFT|RIGHT} [OUTER] JOIN table_reference join_condition
332 | table_reference NATURAL [INNER | {LEFT|RIGHT} [OUTER]] JOIN table_factor
333
334join_condition:
335 ON conditional_expr
336 | USING (column_list)
337
338index_hint_list:
339 index_hint [, index_hint] ...
340
341index_hint:
342 USE {INDEX|KEY}
343 [FOR {JOIN|ORDER BY|GROUP BY}] ([index_list])
344 | IGNORE {INDEX|KEY}
345 [FOR {JOIN|ORDER BY|GROUP BY}] (index_list)
346 | FORCE {INDEX|KEY}
347 [FOR {JOIN|ORDER BY|GROUP BY}] (index_list)
348
349index_list:
350 index_name [, index_name] ...
351 */
352
353}
354
355int main(int argc, const char* argv[])
356{
357 Time start;
358
359 gROOT->SetBatch();
360
361 Configuration conf(argv[0]);
362 conf.SetPrintUsage(PrintUsage);
363 SetupConfiguration(conf);
364
365 if (!conf.DoParse(argc, argv))
366 return 127;
367
368 // ----------------------------- Evaluate options --------------------------
369 const string uri = conf.Get<string>("uri");
370 const string out = conf.Get<string>("out");
371 const string file = conf.Get<string>("file");
372 const string tree = conf.Get<string>("tree");
373 const bool force = conf.Get<bool>("force");
374 const bool ignorenull = conf.Get<bool>("ignore-null");
375 const bool update = conf.Get<bool>("update");
376 const bool display = conf.Get<bool>("display");
377 const string write = conf.Get<string>("write");
378 const bool noout = conf.Get<bool>("null");
379 const bool nofill = conf.Get<bool>("no-fill");
380 const bool explain = conf.Get<bool>("explain");
381 const bool profiling = conf.Get<bool>("profiling");
382 const uint16_t verbose = conf.Get<uint16_t>("verbose");
383 const uint16_t compression = conf.Get<uint16_t>("compression");
384 const string delimiter = conf.Get<string>("delimiter");
385 const vector<string> _ignore = conf.Vec<string>("ignore");
386 //const vector<Map> mymap = conf.Vec<Map>("map");
387
388 // -------------------------------------------------------------------------
389
390 const auto vars = conf.GetWildcardOptions("var.*");
391
392 vector<string> variables;
393 for (const auto &var : vars)
394 variables.emplace_back('@'+var.substr(4)+":="+Tools::Trim(conf.Get<string>(var)));
395
396 // -------------------------------------------------------------------------
397
398 if (verbose>0)
399 cout << "\n------------------------ Rootify SQL -------------------------" << endl;
400
401 string query = conf.Get<string>("query");
402 if (query.empty())
403 {
404 if (verbose>0)
405 cout << "Reading query from file '" << file << "'." << endl;
406
407 ifstream fin(file);
408 if (!fin)
409 {
410 cerr << "Could not open query in '" << file << "': " << strerror(errno) << endl;
411 return 1;
412 }
413 getline(fin, query, (char)fin.eof());
414 }
415
416 if (query.empty())
417 {
418 cerr << "No query specified." << endl;
419 return 2;
420 }
421
422 // -------------------------------------------------------------------------
423
424 map<string, vector<string>> envs;
425
426 for (const auto &env : conf.GetWildcardOptions("env.*"))
427 envs[env.substr(4)] = conf.Vec<string>(env);
428
429 for (const auto &env : conf.GetWildcardOptions("list.*"))
430 {
431 const string fname = conf.Get<string>(env);
432 const string &ident = env.substr(5);
433
434 ifstream fin(fname);
435 if (!fin)
436 {
437 cerr << "Could not open environment in '" << fname << "' for ${" << ident << "}: " << strerror(errno) << endl;
438 return 3;
439 }
440
441 for (string line; getline(fin, line); )
442 if (Tools::Trim(line)[0]!='#')
443 envs[ident].push_back(line);
444
445 if (verbose>0)
446 cout << "Found " << envs[ident].size() << " list element(s) for ${" << ident << "}" << endl;
447 }
448
449 for (const auto &env : envs)
450 {
451 regex rexpr("\\$(\\{"+env.first+"\\}|"+env.first+"\\b)");
452 query = regex_replace(query, rexpr, boost::join(env.second, ", "));
453 }
454
455 // -------------------------- Check for file permssion ---------------------
456 // Strictly speaking, checking for write permission and existance is not necessary,
457 // but it is convenient that the user does not find out that it failed after
458 // waiting long for the query result
459 //
460 // I am using root here instead of boost to be
461 // consistent with the access pattern by TFile
462 TString path(noout?"/dev/null":out.c_str());
463 gSystem->ExpandPathName(path);
464
465 if (!noout)
466 {
467 FileStat_t stat;
468 const Int_t exist = !gSystem->GetPathInfo(path, stat);
469 const Bool_t _write = !gSystem->AccessPathName(path, kWritePermission) && R_ISREG(stat.fMode);
470
471 if ((update && !exist) || (update && exist && !_write) || (force && exist && !_write))
472 {
473 cerr << "File '" << path << "' is not writable." << endl;
474 return 3;
475 }
476
477 if (!update && !force && exist)
478 {
479 cerr << "File '" << path << "' already exists." << endl;
480 return 4;
481 }
482 }
483
484 Time start2;
485
486 // --------------------------- Connect to database -------------------------------------------------
487
488 if (query.back()!='\n')
489 query += '\n';
490
491 if (verbose>0)
492 cout << "Connecting to database..." << endl;
493
494
495 Database connection(uri); // Keep alive while fetching rows
496
497 try
498 {
499 if (profiling)
500 connection.query("SET PROFILING=1").execute();
501 }
502 catch (const exception &e)
503 {
504 cerr << "\nSET profiling=1\n\n";
505 cerr << "SQL query failed:\n" << e.what() << endl;
506 return 6;
507 }
508
509 // -------------------------- Set user defined variables -------------------
510 if (variables.size()>0)
511 {
512 if (verbose>0)
513 cout << "Setting user defined variables..." << endl;
514
515 const string varset =
516 "SET\n "+boost::algorithm::join(variables, ",\n ");
517
518 try
519 {
520 connection.query(varset).execute();
521 }
522 catch (const exception &e)
523 {
524 cerr << '\n' << varset << "\n\n";
525 cerr << "SQL query failed:\n" << e.what() << endl;
526 return 7;
527 }
528
529 if (verbose>2)
530 cout << '\n' << varset << '\n' << endl;
531 }
532
533 // ------------------------- Explain query if requested --------------------
534
535 if (explain)
536 {
537 try
538 {
539 const auto res0 =
540 connection.query("EXPLAIN FORMAT=JSON "+query).store();
541
542 cout << res0[0][0] << endl;
543 cout << endl;
544
545 const mysqlpp::StoreQueryResult res1 =
546 connection.query("EXPLAIN "+query).store();
547
548 for (size_t i=0; i<res1.num_rows(); i++)
549 {
550 const mysqlpp::Row &row = res1[i];
551
552 cout << "\nid : " << row["id"];
553 cout << "\nselect type : " << row["select_type"];
554
555 if (!row["table"].is_null())
556 cout << "\ntable : " << row["table"];
557
558 if (!row["partitions"].is_null())
559 cout << "\npartitions : " << row["partitions"];
560
561 if (!row["key"].is_null())
562 cout << "\nselected key : " << row["key"] << " [len=" << row["key_len"] << "] out of (" << row["possible_keys"] << ")";
563
564 if (!row["type"].is_null())
565 cout << "\njoin type : " << row["type"];
566
567 //if (!row["possible_keys"].is_null())
568 // cout << "\npossible_keys: " << row["possible_keys"];
569
570 //if (!row["key_len"].is_null())
571 // cout << "\nkey_len : " << row["key_len"];
572
573 if (!row["ref"].is_null())
574 cout << "\nref : (" << row["ref"] << ") compared to the index";
575
576 if (!row["rows"].is_null())
577 cout << "\nrows : " << row["rows"];
578
579 if (!row["filtered"].is_null())
580 cout << "\nfiltered : " << row["filtered"];
581
582 if (!row["extra"].is_null())
583 cout << "\nExtra : " << row["extra"];
584
585 cout << endl;
586 }
587
588 cout << endl;
589
590 const mysqlpp::StoreQueryResult res2 =
591 connection.query("SHOW WARNINGS").store();
592
593 for (size_t i=0; i<res2.num_rows(); i++)
594 {
595 const mysqlpp::Row &row = res2[i];
596
597 // 1003 //
598 cout << row["Level"] << '[' << row["Code"] << "]:\n";
599 if (uint32_t(row["Code"])==1003)
600 format(row["Message"].c_str());
601 else
602 cout << row["Message"] << '\n' << endl;
603
604 }
605
606 }
607 catch (const exception &e)
608 {
609 cerr << '\n' << query << "\n\n";
610 cerr << "SQL query failed:\n" << e.what() << endl;
611 return 8;
612 }
613
614 return 0;
615 }
616
617 // -------------------------- Request data from database -------------------
618 if (verbose>0)
619 cout << "Requesting data..." << endl;
620
621 if (verbose>2)
622 cout << '\n' << query << endl;
623
624 const mysqlpp::UseQueryResult res =
625 connection.query(query).use();
626
627 // -------------------------------------------------------------------------
628
629 if (verbose>0)
630 cout << "Opening file '" << path << "' [compression=" << compression << "]..." << endl;
631
632 // ----------------------------- Open output file --------------------------
633 TFile tfile(path, update?"UPDATE":(force?"RECREATE":"CREATE"), "Rootify SQL", compression);
634 if (tfile.IsZombie())
635 return 9;
636
637 // -------------------------------------------------------------------------
638
639 // get the first row to get the field description
640 mysqlpp::Row row = res.fetch_row();
641 if (!row)
642 {
643 cerr << "Empty set returned... nothing to write." << endl;
644 return 10;
645 }
646
647 if (verbose>0)
648 cout << "Trying to setup " << row.size() << " branches..." << endl;
649
650 if (verbose>1)
651 cout << endl;
652
653 const mysqlpp::FieldNames &l = *row.field_list().list;
654
655 vector<double> buf(l.size());
656 vector<uint8_t> typ(l.size(),'n'); // n=number [double], d is used for DateTime
657
658 UInt_t cols = 0;
659
660
661 // -------------------- Configure branches of TTree ------------------------
662 TTree *ttree = new TTree(tree.c_str(), query.c_str());
663
664 size_t skipat = 0;
665 size_t skipreg = 0;
666 for (size_t i=0; i<l.size(); i++)
667 {
668 const string t = row[i].type().sql_name();
669
670 if (t.find("DATETIME")!=string::npos)
671 typ[i] = 'd';
672 else
673 if (t.find("DATE")!=string::npos)
674 typ[i] = 'D';
675 else
676 if (t.find("TIME")!=string::npos)
677 typ[i] = 'T';
678 else
679 if (t.find("VARCHAR")!=string::npos)
680 typ[i] = 'V';
681 else
682 if (t.find("CHAR")!=string::npos)
683 typ[i] = 'C';
684
685 bool found = false;
686 for (const auto &pattern: _ignore)
687 {
688 if (regex_match(l[i], regex(pattern)))
689 {
690 found = true;
691 typ[i] = '-';
692 skipreg++;
693 break;
694 }
695 }
696
697 if (l[i][0]=='@')
698 {
699 typ[i] = '@';
700 skipat++;
701 }
702
703 const bool use = l[i][0]!='@' && typ[i]!='V' && typ[i]!='C' && !found;
704
705 if (verbose>1)
706 cout << (use?" + ":" - ") << l[i].c_str() << " [" << t << "] {" << typ[i] << "}\n";
707
708 if (use)
709 {
710 // string name = l[i];
711 // for (const auto &m: mymap)
712 // name = boost::regex_replace(l[i], boost::regex(m.first), m.second);
713
714 ttree->Branch(l[i].c_str(), buf.data()+i);
715 cols++;
716 }
717 }
718 // -------------------------------------------------------------------------
719
720 if (verbose>1)
721 cout << endl;
722 if (verbose>0)
723 {
724 if (skipreg)
725 cout << skipreg << " branches skipped due to ignore list." << endl;
726 if (skipat)
727 cout << skipat << " branches skipped due to name starting with @." << endl;
728 cout << "Configured " << cols << " branches.\nFilling branches..." << endl;
729 }
730
731 ofstream fout(write);
732 if (!write.empty() && !fout)
733 cout << "WARNING: Writing to '" << write << "' failed: " << strerror(errno) << endl;
734
735 if (display)
736 {
737 cout << endl;
738 cout << "#";
739 for (size_t i=0; i<l.size(); i++)
740 cout << ' ' << l[i].c_str();
741 cout << endl;
742 }
743
744 if (!write.empty())
745 {
746 fout << "#";
747 for (size_t i=0; i<l.size(); i++)
748 fout << ' ' << l[i].c_str();
749 fout << endl;
750 }
751
752 // ---------------------- Fill TTree with DB data --------------------------
753 size_t count = 0;
754 size_t skip = 0;
755 do
756 {
757 count++;
758
759 ostringstream sout;
760
761 size_t idx=0;
762 for (auto col=row.begin(); col!=row.end(); col++, idx++)
763 {
764 if (display || !write.empty())
765 {
766 if (idx>0)
767 sout << (delimiter.empty()?"\t":delimiter);
768 sout << col->c_str();
769 }
770
771 if (!ignorenull && col->is_null())
772 {
773 skip++;
774 break;
775 }
776
777 switch (typ[idx])
778 {
779 case 'd':
780 buf[idx] = time_t((mysqlpp::DateTime)(*col));
781 break;
782
783 case 'D':
784 buf[idx] = time_t((mysqlpp::Date)(*col));
785 break;
786
787 case 'T':
788 buf[idx] = time_t((mysqlpp::Time)(*col));
789 break;
790
791 case 'V':
792 case 'C':
793 case '-':
794 case '@':
795 break;
796
797 default:
798 buf[idx] = atof(col->c_str());
799 }
800 }
801
802 if (idx==row.size())
803 {
804 if (!nofill)
805 ttree->Fill();
806
807 if (display)
808 cout << sout.str() << endl;
809 if (!write.empty())
810 fout << sout.str() << '\n';
811 }
812
813 row = res.fetch_row();
814
815
816 } while (row);
817
818 // -------------------------------------------------------------------------
819
820 if (display)
821 cout << '\n' << endl;
822
823 if (verbose>0)
824 {
825 cout << count << " rows fetched." << endl;
826 if (skip>0)
827 cout << skip << " rows skipped due to NULL field." << endl;
828
829 cout << ttree->GetEntries() << " rows filled into tree." << endl;
830 }
831
832 ttree->Write();
833 tfile.Close();
834
835 if (verbose>0)
836 {
837 const auto sec = Time().UnixTime()-start.UnixTime();
838
839 cout << Tools::Scientific(tfile.GetSize()) << "B written to disk.\n";
840 cout << "File closed.\n";
841 cout << "Execution time: " << sec << "s ";
842 cout << "(" << Tools::Fractional(sec/count) << "s/row)\n";
843 cout << "--------------------------------------------------------------" << endl;
844
845 try
846 {
847 const auto resw =
848 connection.query("SHOW WARNINGS").store();
849
850 if (resw.num_rows()>0)
851 cout << "\n" << resw.num_rows() << " Warning(s) issued:\n\n";
852
853 for (size_t i=0; i<resw.num_rows(); i++)
854 {
855 const mysqlpp::Row &roww = resw[i];
856
857 cout << roww["Level"] << '[' << roww["Code"] << "]: ";
858 cout << roww["Message"] << '\n';
859 }
860 cout << endl;
861
862 }
863 catch (const exception &e)
864 {
865 cerr << "\nSHOW WARNINGS\n\n";
866 cerr << "SQL query failed:\n" << e.what() << endl;
867 return 6;
868 }
869 }
870
871 if (profiling)
872 {
873 try
874 {
875 const auto N =
876 connection.query("SHOW PROFILES").store().num_rows();
877
878 const auto resp =
879 connection.query("SHOW PROFILE ALL FOR QUERY "+to_string(verbose?N-1:N)).store();
880
881 cout << '\n';
882 cout << left;
883 cout << setw(26) << "Status" << ' ';
884 cout << right;
885 cout << setw(11) << "Duration" << ' ';
886 cout << setw(11) << "CPU User" << ' ';
887 cout << setw(11) << "CPU System" << '\n';
888 cout << "--------------------------------------------------------------\n";
889 for (size_t i=0; i<resp.num_rows(); i++)
890 {
891 const mysqlpp::Row &rowp = resp[i];
892
893 cout << left;
894 cout << setw(26) << rowp["Status"] << ' ';
895 cout << right;
896 cout << setw(11) << rowp["Duration"] << ' ';
897 cout << setw(11) << rowp["CPU_user"] << ' ';
898 cout << setw(11) << rowp["CPU_system"] << '\n';
899 }
900 cout << "--------------------------------------------------------------\n";
901 cout << endl;
902 }
903 catch (const exception &e)
904 {
905 cerr << "\nSHOW PROFILE ALL\n\n";
906 cerr << "SQL query failed:\n" << e.what() << '\n' <<endl;
907 return 11;
908 }
909 }
910
911 return 0;
912}
Note: See TracBrowser for help on using the repository browser.