Changeset 19147

Timestamp:

08/10/18 22:56:13 (6 years ago)

Author:

tbretz

Message:

Added support for a user defined column with a constant number, allow to extract that number from the file name

File:

• trunk/FACT++/src/fits2sql.cc (modified) (13 diffs)

trunk/FACT++/src/fits2sql.cc

@@ -66 +66 @@
         ("duplicate",      vars<string>(),            "Specifies an assignment_list for an 'ON DUPLICATE KEY UPDATE' expression")
         ("ignore-errors",  po_switch(),               "Adds the IGNORE keyword to the INSERT query (turns errors into warnings, ignores rows with errors)")
+        ("const.*",        var<string>(),             "Insert a constant number into the given column (--const.mycolumn=5). A special case is `/..../N/`")
         ;
 
@@ -176 +177 @@
         "are marked as (-ignored-).\n"
         "\n"
+        "A constant value for the given file can be inserted by using the --const directive. "
+        "For example --const.mycolumn=42 would insert 42 into a column called mycolumn. "
+        "The column is created as INT UNSIGNED as default which can be altered by "
+        "--sql-type. A special case is a value of the form `/regex/N/`. Here, the given "
+        "regular expression is applied to the filename and N specifies the N-th "
+        "sub-sequence which matches. To debug what matches, verbosity can be set to 3.\n"
+        "\n"
         "If a query failed, the query is printed to stderr together with the error message. "
         "For the main INSERT query, this is only true if the verbosity level is at least 2 "
@@ -191 +199 @@
 {
     kNone = 0,
+    kConst,
     kVarchar,
     kBool,
@@ -238 +247 @@
     {
     }
+    Container(const string &c, const string &value) : branch(value), column(c), type(kConst), num(1), ptr(0)
+    {
+    }
     ~Container()
     {
@@ -262 +274 @@
         case kInt64:   str << reinterpret_cast<int64_t*>(ptr)[index]; break;
         case kUInt64:  str << reinterpret_cast<uint64_t*>(ptr)[index]; break;
+        case kConst:   str << branch; break;
         case kNone:
             break;
@@ -379 +392 @@
     vector<Container> vec;
 
-    for (const auto &ic : cols)
-    {
-        const auto &col = ic.second;
+    const auto fixed = conf.GetWildcardOptions("const.*");
+
+    for (auto it=fixed.cbegin(); it!=fixed.cend(); it++)
+    {
+        const string name = it->substr(6);
+        string val = conf.Get<string>(*it);
+
+        boost::smatch match;
+        if (boost::regex_match(val, match, boost::regex("\\/(.+)\\/([0-9]*)\\/")))
+        {
+            string s = match[1];
+            size_t r = atoi(match[2].str().c_str());
+
+            if (boost::regex_search(file, match, boost::regex(s)))
+            {
+                if (verbose>2)
+                    for (size_t i=0; i<match.size(); i++)
+                        cout << "Regex match " << setw(2) << i << ": `" << match[i] << "`" << endl;
+
+                val = match[r];
+            }
+        }
+
+        if (verbose==2)
+            cout << "\n" << val << " [-const-]";
+        if (verbose>1)
+            cout << " (" << name << ")";
+
+        string sqltype = "INT UNSIGNED";
+
+        for (auto m=sqltypes.cbegin(); m!=sqltypes.cend(); m++)
+            if (m->first==name)
+                sqltype = m->second;
+
+        if (!vec.empty())
+            query += ",\n";
+        query += "   `"+name+"` "+sqltype+" NOT NULL COMMENT '--user--'";
+
+        vec.emplace_back(name, val);
+    }
+
+    const size_t nvec = vec.size();
+
+    for (auto ic=cols.cbegin(); ic!=cols.cend(); ic++)
+    {
+        const auto &col = ic->second;
 
         if (verbose>2)
-            cout << '\n' << col.type << " " << ic.first << "[" << col.num << "]";
-
-        string name = ic.first;
+            cout << '\n' << col.type << " " << ic->first << "[" << col.num << "]";
+
+        string name = ic->first;
 
         bool found = false;
@@ -402 +458 @@
             continue;
 
-        const char tn = find(notsigned.cbegin(), notsigned.cend(), ic.first)!=notsigned.cend() ?
+        const char tn = find(notsigned.cbegin(), notsigned.cend(), ic->first)!=notsigned.cend() ?
             tolower(col.type) : toupper(col.type);
 
@@ -436 +492 @@
             query += "   `"+name;
             if (N>1)
-                query += "["+to_string(i)+"]";
+                query += "["+to_string((long long int)i)+"]";
             query += "` "+sqltype;
             if (col.type=='A')
-                query += '('+to_string(col.num)+')';
-            query += " NOT NULL COMMENT '"+ic.first;
+                query += '('+to_string((long long int)col.num)+')';
+            query += " NOT NULL COMMENT '"+ic->first;
             if (!col.unit.empty())
                 query += "["+col.unit+"]";
@@ -454 +510 @@
             kMJD :*/ it->second.first;
 
-        vec.emplace_back(ic.first, name, bt, col.num/*, mjdref*/);
-        vec.back().ptr = f.SetPtrAddress(ic.first);
+        vec.emplace_back(ic->first, name, bt, col.num/*, mjdref*/);
+        vec.back().ptr = f.SetPtrAddress(ic->first);
     }
 
@@ -461 +517 @@
         cout << "\n\n";
     if (verbose>0)
-        cout << vec.size() << " columns setup for reading." << endl;
+    {
+        if (nvec>0)
+            cout << nvec << " constant value column(s) configured." << endl;
+        cout << vec.size()-nvec << " FITS columns setup for reading." << endl;
+    }
 
     // -------------------------------------------------------------------------
@@ -509 +569 @@
 
     if (!primary.empty())
-        query += ",\n   PRIMARY KEY USING BTREE (`"+boost::algorithm::join(primary, "`, `")+"`)\n";
+        query += ",\n   PRIMARY KEY USING BTREE (`"+boost::algorithm::join(primary, "`, `")+"`)";
 
     query +=
-        ")\n"
+        "\n)\n"
         "DEFAULT CHARSET=latin1 COLLATE=latin1_general_ci\n";
     if (!engine.empty())
@@ -573 +633 @@
     for (auto c=vec.cbegin(); c!=vec.cend(); c++)
     {
+        if (c!=vec.cbegin())
+            query += ",\n";
+
         const size_t N = c->type==kVarchar ? 1 : c->num;
         for (size_t i=0; i<N; i++)
         {
-            if (c!=vec.cbegin())
-                query += ",\n";
-
             if (N==1)
                 query += "   `"+c->column+"`";
             else
-                query += "   `"+c->column+"["+to_string(i)+"]`";
+                query += "   `"+c->column+"["+to_string((long long int)i)+"]`";
 
             if (N>1 && i!=N-1)
@@ -607 +667 @@
         for (auto c=vec.cbegin(); c!=vec.cend(); c++)
         {
+            if (c!=vec.cbegin())
+                query += ",\n";
+
             const size_t N = c->type==kVarchar ? 1 : c->num;
             for (size_t i=0; i<N; i++)
             {
-                if (c!=vec.cbegin())
-                    query += ",\n";
-
                 if (c->type==kVarchar)
                     query += "   '"+c->fmt(i)+"'";