Changeset 18232 for branches


Ignore:
Timestamp:
06/18/15 14:14:51 (9 years ago)
Author:
dneise
Message:
made library
File:
1 edited

Legend:

Unmodified
Added
Removed

  • branches/FACT++_scripts_refactoring/takeRun.js

    r18174 r18232  
    11'use strict';
    22
    3 // ================================================================
    4 //  Code related to monitoring the fad system
    5 // ================================================================
    6 
    7 var incomplete = 0;
    8 
    9 sub_incomplete.onchange = function(evt)
    10 {
    11     if (!evt.data)
    12         return;
    13 
    14     var inc = evt.obj['incomplete'];
    15     if (!inc || inc>0xffffffffff)
    16         return;
    17 
    18     if (incomplete>0)
    19         return;
    20 
    21     if (dim.state("MCP").name!="TakingData")
    22         return;
    23 
    24     console.out("");
    25     dim.log("Incomplete event ["+inc+","+incomplete+"] detected, sending MCP/STOP");
    26 
    27     incomplete = inc;
    28     dim.send("MCP/STOP");
     3if (!("takeRun" in this)){
     4    var takeRun = {
     5
     6        // ================================================================
     7        //  Code related to monitoring the fad system
     8        // ================================================================
     9
     10        attachFadControlIncompleteOnChange : function()
     11        {
     12            var sub_incomplete = Subscription("FAD_CONTROL/INCOMPLETE");
     13            if (!sub_incomplete)
     14                sub_incomplete = new Subscription("FAD_CONTROL/INCOMPLETE");
     15
     16            attachFadControlIncompleteOnChange.incomplete = 0;
     17
     18            sub_incomplete.onchange = function(evt)
     19            {
     20                if (!evt.data)
     21                    return;
     22
     23                var inc = evt.obj['incomplete'];
     24                if (!inc || inc>0xffffffffff)
     25                    return;
     26
     27                if (attachFadControlIncompleteOnChange.incomplete>0)
     28                    return;
     29
     30                if (dim.state("MCP").name!="TakingData")
     31                    return;
     32
     33                console.out("");
     34                dim.log("Incomplete event ["+inc+","+attachFadControlIncompleteOnChange.incomplete+"] detected, sending MCP/STOP");
     35
     36                attachFadControlIncompleteOnChange.incomplete = inc;
     37                dim.send("MCP/STOP");
     38            }
     39        },
     40
     41        // ================================================================
     42        //  Code related to taking data
     43        // ================================================================
     44
     45        /**
     46         * reconnect to problematic FADs
     47         *
     48         * Dis- and Reconnects to FADs, found to be problematic by call-back function
     49         * onchange() to have a different CONNECTION value than 66 or 67.
     50         *
     51         * @returns
     52         *      a boolean is returned.
     53         *      reconnect returns true if:
     54         *          * nothing needed to be reset --> no problems found by onchange()
     55         *          * the reconnection went fine.
     56         *     
     57         *      reconnect *never returns false* so far.
     58         *
     59         * @example
     60         *      if (!sub_connections.reconnect())
     61         *          exit();
     62         */
     63        reconnect :function(list, txt)
     64        {
     65            /*
     66            var reset = [ ];
     67
     68            for (var i=0; i<list.length; i++)
     69                {
     70                    console.out("  FAD %2d".$(list[i])+" lost during "+txt);
     71                    reset.push(parseInt(list[i]/10));
     72                }
     73
     74            reset = reset.filter(function(elem,pos){return reset.indexOf(elem)==pos;});
     75
     76            console.out("");
     77            console.out("  FADs belong to crate(s): "+reset);
     78            console.out("");
     79            */
     80            console.out("");
     81            dim.log("Trying automatic reconnect ["+txt+",n="+list.length+"]...");
     82
     83            if (list.length>3)
     84                throw new Error("Too many boards to be reconnected. Please check what happened.");
     85
     86            for (var i=0; i<list.length; i++)
     87            {
     88                console.out("   ...disconnect "+list[i]);
     89                dim.send("FAD_CONTROL/DISCONNECT", list[i]);
     90            }
     91
     92            console.out("   ...waiting for 3s");
     93            v8.sleep(3000);
     94
     95            for (var i=0; i<list.length; i++)
     96            {
     97                console.out("   ...reconnect "+list[i]);
     98                dim.send("FAD_CONTROL/CONNECT", list[i]);
     99            }
     100
     101            console.out("   ...waiting for 1s");
     102
     103            // Wait for one second to bridge possible pending connects
     104            v8.sleep(1000);
     105
     106            console.out("   ...checking connection");
     107
     108            // Wait for FAD_CONTROL to realize that all boards are connected
     109            // FIXME: Wait for '40' boards being connected instead
     110            try
     111            {
     112                dim.wait("FAD_CONTROL", "Connected", 3000);
     113            }
     114            catch (e)
     115            {
     116                if (dim.state("FAD_CONTROL").name!="Connecting")
     117                {
     118                    console.out("");
     119                    console.out(" + FAD_CONTROL: "+dim.state("FAD_CONTROL").name);
     120                    console.out("");
     121                    throw e;
     122                }
     123
     124                var crates = [];
     125                for (var i=0; i<list.length; i++)
     126                    crates[list[i]/4] = true;
     127
     128                include('scripts/crateReset.js');
     129                crateReset(crates);
     130            }
     131
     132            // Wait also for MCP to have all boards connected again
     133            dim.wait("MCP", "Idle", 3000);
     134
     135            dim.log("Automatic reconnect successfull.");
     136            console.out("");
     137        },
     138
     139        takeRun: function(type, count, time)
     140        {
     141            if (!count)
     142                count = -1;
     143            if (!time)
     144                time = -1;
     145
     146            var custom = typeof(type)=="function";
     147
     148            var nextrun = sub_startrun.get().obj['next'];
     149            dim.log("Take run %3d".$(nextrun)+": N="+count+" T="+time+"s ["+(custom?"custom":type)+"]");
     150
     151            // FIXME: Replace by callback?
     152            //
     153            // DN: I believe instead of waiting for 'TakingData' one could split this
     154            // up into two checks with an extra condition:
     155            //  if type == 'data':
     156            //      wait until ThresholdCalibration starts:
     157            //          --> this time should be pretty identical for each run
     158            //      if this takes longer than say 3s:
     159            //          there might be a problem with one/more FADs
     160            //   
     161            //      wait until "TakingData":
     162            //          --> this seems to take even some minutes sometimes...
     163            //              (might be optimized rather soon, but still in the moment...)
     164            //      if this takes way too long:
     165            //          there might be something broken,
     166            //          so maybe a very high time limit is ok here.
     167            //          I think there is not much that can go wrong,
     168            //          when the Thr-Calib has already started. Still it might be nice
     169            //          If in the future RateControl is written so to find out that
     170            //          in case the threshold finding algorithm does
     171            //          *not converge as usual*
     172            //          it can complain, and in this way give a hint, that the weather
     173            //          might be a little bit too bad.
     174            //  else:
     175            //      wait until "TakingData":
     176            //          --> in a non-data run this time should be pretty short again
     177            //      if this takes longer than say 3s:
     178            //          there might be a problem with one/more FADs
     179            // 
     180
     181            // Use this if you use the rate control to calibrate by rates
     182            //if (!dim.wait("MCP", "TakingData", -300000) )
     183            //{
     184            //    throw new Error("MCP took longer than 5 minutes to start TakingData"+
     185            //                    "maybe this idicates a problem with one of the FADs?");
     186            //}
     187
     188            // ================================================================
     189            //  Function for Critical voltage
     190            // ================================================================
     191
     192            // INSTALL a watchdog... send FAD_CONTROL/CLOSE_OPEN_FILES
     193            // could send MCP/RESET as well but would result in a timeout
     194            var callback = dim.onchange['FEEDBACK'];
     195            dim.onchange['FEEDBACK'] = function(state)
     196            {
     197                if (callback)
     198                    callback.call(this, state);
     199
     200                if ((state.name=="Critical" || state.name=="OnStandby") &&
     201                    (this.last!="Critical"  && this.last!="OnStandby"))
     202                {
     203                    console.out("Feedback state changed from "+this.last+" to "+state.name+" [takeRun.js]");
     204
     205                    // Includes FAD_CONTROL/CLOSE_ALL_OPEN_FILES
     206                    dim.send("MCP/STOP");
     207                }
     208
     209                this.last=state.name;
     210            }
     211
     212            // Here we could check and handle fad losses
     213
     214            attachFadControlIncompleteOnChange.incomplete = 0;
     215
     216            var start = true;
     217
     218            for (var n=0; n<3; n++)
     219            {
     220                if (start)
     221                {
     222                    dim.send("MCP/START", time, count, custom?"custom":type);
     223                    if (custom)
     224                        type();
     225                }
     226
     227                try
     228                {
     229                    dim.wait("MCP", "TakingData", 15000);
     230                    break;
     231                }
     232                catch (e)
     233                {
     234                    if (dim.state("MCP").name=="TriggerOn" &&
     235                        dim.state("FAD_CONTROL").name=="Connected" &&
     236                        dim.state("FTM_CONTROL").name=="TriggerOn")
     237                    {
     238                        console.out("");
     239                        console.out("Waiting for TakingData timed out. Everything looks ok, but file not yet open... waiting once more.");
     240                        start = false;
     241                        continue;
     242                    }
     243
     244                    start = true;
     245
     246                    console.out("");
     247                    console.out(" + MCP:         "+dim.state("MCP").name);
     248                    console.out(" + FAD_CONTROL: "+dim.state("FAD_CONTROL").name);
     249                    console.out(" + FTM_CONTROL: "+dim.state("FTM_CONTROL").name);
     250                    console.out("");
     251
     252                    if (dim.state("MCP").name!="Configuring3" ||
     253                        (dim.state("FAD_CONTROL").name!="Configuring1" &&
     254                         dim.state("FAD_CONTROL").name!="Configuring2"))
     255                        throw e;
     256
     257                    console.out("");
     258                    console.out("Waiting for fadctrl to get configured timed out... checking for in-run FAD loss.");
     259
     260                    var con  = sub_connections.get();
     261                    var stat = con.obj['status'];
     262
     263                    console.out("Sending MCP/RESET");
     264                    dim.send("MCP/RESET");
     265
     266                    dim.wait("FTM_CONTROL", "Valid",     3000);
     267                    dim.wait("FAD_CONTROL", "Connected", 3000);
     268                    dim.wait("MCP",         "Idle",      3000);
     269
     270                    var list = [];
     271                    for (var i=0; i<40; i++)
     272                        if (stat[i]!=0x43)
     273                            list.push(i);
     274
     275                    this.reconnect(list, "configuration");
     276
     277                    if (n==2)
     278                        throw e;
     279
     280                    //dim.wait("MCP", "Idle", 3000);
     281                }
     282            }
     283
     284            // This is to check if we have missed the event. This can happen as
     285            // a race condition when the MCP/STOP is sent by the event handler
     286            // but the run was not yet fully configured.
     287            var statefb = dim.state("FEEDBACK").name;
     288            if (statefb=="Critical" || statefb=="OnStandby")
     289            {
     290                console.out("Run started by FEEDBACK in state "+statefb);
     291                dim.send("MCP/STOP"); // Includes FAD_CONTROL/CLOSE_ALL_OPEN_FILES
     292
     293                dim.onchange['FEEDBACK'] = callback;
     294
     295                return true;
     296            }
     297
     298            dim.wait("MCP", "Idle", time>0 ? time*1250 : undefined); // run time plus 25%
     299
     300            // REMOVE watchdog
     301            dim.onchange['FEEDBACK'] = callback;
     302
     303            if (incomplete)
     304            {
     305                console.out("");
     306                console.out(" - MCP:         "+dim.state("MCP").name);
     307                console.out(" - FAD_CONTROL: "+dim.state("FAD_CONTROL").name);
     308                console.out(" - FTM_CONTROL: "+dim.state("FTM_CONTROL").name);
     309
     310                dim.wait("FTM_CONTROL", "Valid",     3000);
     311                dim.wait("FAD_CONTROL", "Connected", 3000);
     312                dim.wait("MCP",         "Idle",      3000);
     313
     314                var str = attachFadControlIncompleteOnChange.incomplete.toString(2);
     315                var len = str.length;
     316
     317                var list = [];
     318                for (var i=0; i<str.length; i++)
     319                    if (str[str.length-i-1]=='1')
     320                        list.push(i);
     321
     322                this.reconnect(list, "data taking");
     323
     324                return false;
     325            }
     326
     327            // FIXME: What if the ext1 is not enabled in the configuration?
     328            if (type=="data")
     329            {
     330                var dim_trg = new Subscription("FAD_CONTROL/TRIGGER_COUNTER");
     331                var counter = dim_trg.get(3000);
     332
     333                // The check on physics and pedestal triggers is to ensure that
     334                // there was at least a chance to receive any event (e.g. in case
     335                // of an interrupt this might not be the case)
     336                if (counter.qos!=111 &&
     337                    (counter.data['N_trg']>1000 || counter.data['N_ped']>5) &&
     338                    counter.data['N_ext1']==0) // 'o' for open
     339                    throw new Error("No ext1 triggers received during data taking... please check the reason and report in the logbook.");
     340                dim_trg.close();
     341            }
     342
     343            return true;
     344        },
     345
     346    };
    29347}
    30 
    31 // ================================================================
    32 //  Code related to taking data
    33 // ================================================================
    34 
    35 /**
    36  * reconnect to problematic FADs
    37  *
    38  * Dis- and Reconnects to FADs, found to be problematic by call-back function
    39  * onchange() to have a different CONNECTION value than 66 or 67.
    40  *
    41  * @returns
    42  *      a boolean is returned.
    43  *      reconnect returns true if:
    44  *          * nothing needed to be reset --> no problems found by onchange()
    45  *          * the reconnection went fine.
    46  *     
    47  *      reconnect *never returns false* so far.
    48  *
    49  * @example
    50  *      if (!sub_connections.reconnect())
    51  *          exit();
    52  */
    53 function reconnect(list, txt)
    54 { /*
    55     var reset = [ ];
    56 
    57     for (var i=0; i<list.length; i++)
    58         {
    59             console.out("  FAD %2d".$(list[i])+" lost during "+txt);
    60             reset.push(parseInt(list[i]/10));
    61         }
    62 
    63     reset = reset.filter(function(elem,pos){return reset.indexOf(elem)==pos;});
    64 
    65     console.out("");
    66     console.out("  FADs belong to crate(s): "+reset);
    67     console.out("");
    68 */
    69     console.out("");
    70     dim.log("Trying automatic reconnect ["+txt+",n="+list.length+"]...");
    71 
    72     if (list.length>3)
    73         throw new Error("Too many boards to be reconnected. Please check what happened.");
    74 
    75     for (var i=0; i<list.length; i++)
    76     {
    77         console.out("   ...disconnect "+list[i]);
    78         dim.send("FAD_CONTROL/DISCONNECT", list[i]);
    79     }
    80 
    81     console.out("   ...waiting for 3s");
    82     v8.sleep(3000);
    83 
    84     for (var i=0; i<list.length; i++)
    85     {
    86         console.out("   ...reconnect "+list[i]);
    87         dim.send("FAD_CONTROL/CONNECT", list[i]);
    88     }
    89 
    90     console.out("   ...waiting for 1s");
    91 
    92     // Wait for one second to bridge possible pending connects
    93     v8.sleep(1000);
    94 
    95     console.out("   ...checking connection");
    96 
    97     // Wait for FAD_CONTROL to realize that all boards are connected
    98     // FIXME: Wait for '40' boards being connected instead
    99     try
    100     {
    101         dim.wait("FAD_CONTROL", "Connected", 3000);
    102     }
    103     catch (e)
    104     {
    105         if (dim.state("FAD_CONTROL").name!="Connecting")
    106         {
    107             console.out("");
    108             console.out(" + FAD_CONTROL: "+dim.state("FAD_CONTROL").name);
    109             console.out("");
    110             throw e;
    111         }
    112 
    113         var crates = [];
    114         for (var i=0; i<list.length; i++)
    115             crates[list[i]/4] = true;
    116 
    117         include('scripts/crateReset.js');
    118         crateReset(crates);
    119     }
    120 
    121     // Wait also for MCP to have all boards connected again
    122     dim.wait("MCP", "Idle", 3000);
    123 
    124     dim.log("Automatic reconnect successfull.");
    125     console.out("");
     348else{
     349    console.out("multiple include of 'takeRun.js'");
    126350}
    127351
    128 function takeRun(type, count, time)
    129 {
    130     if (!count)
    131         count = -1;
    132     if (!time)
    133         time = -1;
    134 
    135     var custom = typeof(type)=="function";
    136 
    137     var nextrun = sub_startrun.get().obj['next'];
    138     dim.log("Take run %3d".$(nextrun)+": N="+count+" T="+time+"s ["+(custom?"custom":type)+"]");
    139 
    140     // FIXME: Replace by callback?
    141     //
    142     // DN: I believe instead of waiting for 'TakingData' one could split this
    143     // up into two checks with an extra condition:
    144     //  if type == 'data':
    145     //      wait until ThresholdCalibration starts:
    146     //          --> this time should be pretty identical for each run
    147     //      if this takes longer than say 3s:
    148     //          there might be a problem with one/more FADs
    149     //   
    150     //      wait until "TakingData":
    151     //          --> this seems to take even some minutes sometimes...
    152     //              (might be optimized rather soon, but still in the moment...)
    153     //      if this takes way too long:
    154     //          there might be something broken,
    155     //          so maybe a very high time limit is ok here.
    156     //          I think there is not much that can go wrong,
    157     //          when the Thr-Calib has already started. Still it might be nice
    158     //          If in the future RateControl is written so to find out that
    159     //          in case the threshold finding algorithm does
    160     //          *not converge as usual*
    161     //          it can complain, and in this way give a hint, that the weather
    162     //          might be a little bit too bad.
    163     //  else:
    164     //      wait until "TakingData":
    165     //          --> in a non-data run this time should be pretty short again
    166     //      if this takes longer than say 3s:
    167     //          there might be a problem with one/more FADs
    168     // 
    169 
    170     // Use this if you use the rate control to calibrate by rates
    171     //if (!dim.wait("MCP", "TakingData", -300000) )
    172     //{
    173     //    throw new Error("MCP took longer than 5 minutes to start TakingData"+
    174     //                    "maybe this idicates a problem with one of the FADs?");
    175     //}
    176 
    177     // ================================================================
    178     //  Function for Critical voltage
    179     // ================================================================
    180 
    181     // INSTALL a watchdog... send FAD_CONTROL/CLOSE_OPEN_FILES
    182     // could send MCP/RESET as well but would result in a timeout
    183     var callback = dim.onchange['FEEDBACK'];
    184     dim.onchange['FEEDBACK'] = function(state)
    185     {
    186         if (callback)
    187             callback.call(this, state);
    188 
    189         if ((state.name=="Critical" || state.name=="OnStandby") &&
    190             (this.last!="Critical"  && this.last!="OnStandby"))
    191         {
    192             console.out("Feedback state changed from "+this.last+" to "+state.name+" [takeRun.js]");
    193 
    194             // Includes FAD_CONTROL/CLOSE_ALL_OPEN_FILES
    195             dim.send("MCP/STOP");
    196         }
    197 
    198         this.last=state.name;
    199     }
    200 
    201     // Here we could check and handle fad losses
    202 
    203     incomplete = 0;
    204 
    205     var start = true;
    206 
    207     for (var n=0; n<3; n++)
    208     {
    209         if (start)
    210         {
    211             dim.send("MCP/START", time, count, custom?"custom":type);
    212             if (custom)
    213                 type();
    214         }
    215 
    216         try
    217         {
    218             dim.wait("MCP", "TakingData", 15000);
    219             break;
    220         }
    221         catch (e)
    222         {
    223             if (dim.state("MCP").name=="TriggerOn" &&
    224                 dim.state("FAD_CONTROL").name=="Connected" &&
    225                 dim.state("FTM_CONTROL").name=="TriggerOn")
    226             {
    227                 console.out("");
    228                 console.out("Waiting for TakingData timed out. Everything looks ok, but file not yet open... waiting once more.");
    229                 start = false;
    230                 continue;
    231             }
    232 
    233             start = true;
    234 
    235             console.out("");
    236             console.out(" + MCP:         "+dim.state("MCP").name);
    237             console.out(" + FAD_CONTROL: "+dim.state("FAD_CONTROL").name);
    238             console.out(" + FTM_CONTROL: "+dim.state("FTM_CONTROL").name);
    239             console.out("");
    240 
    241             if (dim.state("MCP").name!="Configuring3" ||
    242                 (dim.state("FAD_CONTROL").name!="Configuring1" &&
    243                  dim.state("FAD_CONTROL").name!="Configuring2"))
    244                 throw e;
    245 
    246             console.out("");
    247             console.out("Waiting for fadctrl to get configured timed out... checking for in-run FAD loss.");
    248 
    249             var con  = sub_connections.get();
    250             var stat = con.obj['status'];
    251 
    252             console.out("Sending MCP/RESET");
    253             dim.send("MCP/RESET");
    254 
    255             dim.wait("FTM_CONTROL", "Valid",     3000);
    256             dim.wait("FAD_CONTROL", "Connected", 3000);
    257             dim.wait("MCP",         "Idle",      3000);
    258 
    259             var list = [];
    260             for (var i=0; i<40; i++)
    261                 if (stat[i]!=0x43)
    262                     list.push(i);
    263 
    264             reconnect(list, "configuration");
    265 
    266             if (n==2)
    267                 throw e;
    268 
    269             //dim.wait("MCP", "Idle", 3000);
    270         }
    271     }
    272 
    273     // This is to check if we have missed the event. This can happen as
    274     // a race condition when the MCP/STOP is sent by the event handler
    275     // but the run was not yet fully configured.
    276     var statefb = dim.state("FEEDBACK").name;
    277     if (statefb=="Critical" || statefb=="OnStandby")
    278     {
    279         console.out("Run started by FEEDBACK in state "+statefb);
    280         dim.send("MCP/STOP"); // Includes FAD_CONTROL/CLOSE_ALL_OPEN_FILES
    281 
    282         dim.onchange['FEEDBACK'] = callback;
    283 
    284         return true;
    285     }
    286 
    287     dim.wait("MCP", "Idle", time>0 ? time*1250 : undefined); // run time plus 25%
    288 
    289     // REMOVE watchdog
    290     dim.onchange['FEEDBACK'] = callback;
    291 
    292     if (incomplete)
    293     {
    294         console.out("");
    295         console.out(" - MCP:         "+dim.state("MCP").name);
    296         console.out(" - FAD_CONTROL: "+dim.state("FAD_CONTROL").name);
    297         console.out(" - FTM_CONTROL: "+dim.state("FTM_CONTROL").name);
    298 
    299         dim.wait("FTM_CONTROL", "Valid",     3000);
    300         dim.wait("FAD_CONTROL", "Connected", 3000);
    301         dim.wait("MCP",         "Idle",      3000);
    302 
    303         var str = incomplete.toString(2);
    304         var len = str.length;
    305 
    306         var list = [];
    307         for (var i=0; i<str.length; i++)
    308             if (str[str.length-i-1]=='1')
    309                 list.push(i);
    310 
    311         reconnect(list, "data taking");
    312 
    313         return false;
    314     }
    315 
    316     // FIXME: What if the ext1 is not enabled in the configuration?
    317     if (type=="data")
    318     {
    319         var dim_trg = new Subscription("FAD_CONTROL/TRIGGER_COUNTER");
    320         var counter = dim_trg.get(3000);
    321 
    322         // The check on physics and pedestal triggers is to ensure that
    323         // there was at least a chance to receive any event (e.g. in case
    324         // of an interrupt this might not be the case)
    325         if (counter.qos!=111 &&
    326             (counter.data['N_trg']>1000 || counter.data['N_ped']>5) &&
    327             counter.data['N_ext1']==0) // 'o' for open
    328             throw new Error("No ext1 triggers received during data taking... please check the reason and report in the logbook.");
    329         dim_trg.close();
    330     }
    331 
    332     return true;
    333 }
     352
     353
     354
     355
     356       
Note: See TracChangeset for help on using the changeset viewer.