source: trunk/FACT++/scripts/takeRun.js@ 19711

Last change on this file since 19711 was 18731, checked in by tbretz, 8 years ago
Implemented the possibility to define a custom setup in a callback function.
File size: 10.2 KB
Line 
1'use strict';
2
3// ================================================================
4// Code related to monitoring the fad system
5// ================================================================
6
7var incomplete = 0;
8
9sub_incomplete.onchange = function(evt)
10{
11 if (!evt.data)
12 return;
13
14 var inc = evt.obj['incomplete'];
15 if (!inc || inc>0xffffffffff)
16 return;
17
18 if (incomplete>0)
19 return;
20
21 if (dim.state("MCP").name!="TakingData")
22 return;
23
24 console.out("");
25 dim.log("Incomplete event ["+inc+","+incomplete+"] detected, sending MCP/STOP");
26
27 incomplete = inc;
28 dim.send("MCP/STOP");
29}
30
31// ================================================================
32// Code related to taking data
33// ================================================================
34
35/**
36 * reconnect to problematic FADs
37 *
38 * Dis- and Reconnects to FADs, found to be problematic by call-back function
39 * onchange() to have a different CONNECTION value than 66 or 67.
40 *
41 * @returns
42 * a boolean is returned.
43 * reconnect returns true if:
44 * * nothing needed to be reset --> no problems found by onchange()
45 * * the reconnection went fine.
46 *
47 * reconnect *never returns false* so far.
48 *
49 * @example
50 * if (!sub_connections.reconnect())
51 * exit();
52 */
53function reconnect(list, txt)
54{ /*
55 var reset = [ ];
56
57 for (var i=0; i<list.length; i++)
58 {
59 console.out(" FAD %2d".$(list[i])+" lost during "+txt);
60 reset.push(parseInt(list[i]/10));
61 }
62
63 reset = reset.filter(function(elem,pos){return reset.indexOf(elem)==pos;});
64
65 console.out("");
66 console.out(" FADs belong to crate(s): "+reset);
67 console.out("");
68*/
69 console.out("");
70 dim.log("Trying automatic reconnect ["+txt+",n="+list.length+"]...");
71
72 if (list.length>3)
73 throw new Error("Too many boards to be reconnected. Please check what happened.");
74
75 for (var i=0; i<list.length; i++)
76 {
77 console.out(" ...disconnect "+list[i]);
78 dim.send("FAD_CONTROL/DISCONNECT", list[i]);
79 }
80
81 console.out(" ...waiting for 3s");
82 v8.sleep(3000);
83
84 for (var i=0; i<list.length; i++)
85 {
86 console.out(" ...reconnect "+list[i]);
87 dim.send("FAD_CONTROL/CONNECT", list[i]);
88 }
89
90 console.out(" ...waiting for 1s");
91
92 // Wait for one second to bridge possible pending connects
93 v8.sleep(1000);
94
95 console.out(" ...checking connection");
96
97 // Wait for FAD_CONTROL to realize that all boards are connected
98 // FIXME: Wait for '40' boards being connected instead
99 try
100 {
101 dim.wait("FAD_CONTROL", "Connected", 3000);
102 }
103 catch (e)
104 {
105 if (dim.state("FAD_CONTROL").name!="Connecting")
106 {
107 console.out("");
108 console.out(" + FAD_CONTROL: "+dim.state("FAD_CONTROL").name);
109 console.out("");
110 throw e;
111 }
112
113 var crates = [];
114 for (var i=0; i<list.length; i++)
115 crates[list[i]/10] = true;
116
117 include('scripts/crateReset.js');
118 crateReset(crates);
119 }
120
121 // Wait also for MCP to have all boards connected again
122 dim.wait("MCP", "Idle", 3000);
123
124 dim.log("Automatic reconnect successfull.");
125 console.out("");
126}
127
128function takeRun(type, count, time, func)
129{
130 if (!count)
131 count = -1;
132 if (!time)
133 time = -1;
134
135 var nextrun = sub_startrun.get().obj['next'];
136 dim.log("Take run %3d".$(nextrun)+": N="+count+" T="+time+"s ["+type+"]");
137
138 // FIXME: Replace by callback?
139 //
140 // DN: I believe instead of waiting for 'TakingData' one could split this
141 // up into two checks with an extra condition:
142 // if type == 'data':
143 // wait until ThresholdCalibration starts:
144 // --> this time should be pretty identical for each run
145 // if this takes longer than say 3s:
146 // there might be a problem with one/more FADs
147 //
148 // wait until "TakingData":
149 // --> this seems to take even some minutes sometimes...
150 // (might be optimized rather soon, but still in the moment...)
151 // if this takes way too long:
152 // there might be something broken,
153 // so maybe a very high time limit is ok here.
154 // I think there is not much that can go wrong,
155 // when the Thr-Calib has already started. Still it might be nice
156 // If in the future RateControl is written so to find out that
157 // in case the threshold finding algorithm does
158 // *not converge as usual*
159 // it can complain, and in this way give a hint, that the weather
160 // might be a little bit too bad.
161 // else:
162 // wait until "TakingData":
163 // --> in a non-data run this time should be pretty short again
164 // if this takes longer than say 3s:
165 // there might be a problem with one/more FADs
166 //
167
168 // Use this if you use the rate control to calibrate by rates
169 //if (!dim.wait("MCP", "TakingData", -300000) )
170 //{
171 // throw new Error("MCP took longer than 5 minutes to start TakingData"+
172 // "maybe this idicates a problem with one of the FADs?");
173 //}
174
175 // ================================================================
176 // Function for Critical voltage
177 // ================================================================
178
179 // INSTALL a watchdog... send FAD_CONTROL/CLOSE_OPEN_FILES
180 // could send MCP/RESET as well but would result in a timeout
181 var callback = dim.onchange['FEEDBACK'];
182 dim.onchange['FEEDBACK'] = function(state)
183 {
184 if (callback)
185 callback.call(this, state);
186
187 if ((state.name=="Critical" || state.name=="OnStandby") &&
188 (this.last!="Critical" && this.last!="OnStandby"))
189 {
190 console.out("Feedback state changed from "+this.last+" to "+state.name+" [takeRun.js]");
191
192 // Includes FAD_CONTROL/CLOSE_ALL_OPEN_FILES
193 dim.send("MCP/STOP");
194 }
195
196 this.last=state.name;
197 }
198
199 // Here we could check and handle fad losses
200
201 incomplete = 0;
202
203 var start = true;
204
205 for (var n=0; n<3; n++)
206 {
207 if (start)
208 {
209 dim.send("MCP/START", time, count, type);
210 if (typeof(func)=="function")
211 func();
212 }
213
214 try
215 {
216 dim.wait("MCP", "TakingData", 15000);
217 break;
218 }
219 catch (e)
220 {
221 if (dim.state("MCP").name=="TriggerOn" &&
222 dim.state("FAD_CONTROL").name=="Connected" &&
223 dim.state("FTM_CONTROL").name=="TriggerOn")
224 {
225 console.out("");
226 console.out("Waiting for TakingData timed out. Everything looks ok, but file not yet open... waiting once more.");
227 start = false;
228 continue;
229 }
230
231 start = true;
232
233 console.out("");
234 console.out(" + MCP: "+dim.state("MCP").name);
235 console.out(" + FAD_CONTROL: "+dim.state("FAD_CONTROL").name);
236 console.out(" + FTM_CONTROL: "+dim.state("FTM_CONTROL").name);
237 console.out("");
238
239 if (dim.state("MCP").name!="Configuring3" ||
240 (dim.state("FAD_CONTROL").name!="Configuring1" &&
241 dim.state("FAD_CONTROL").name!="Configuring2"))
242 throw e;
243
244 console.out("");
245 console.out("Waiting for fadctrl to get configured timed out... checking for in-run FAD loss.");
246
247 var con = sub_connections.get();
248 var stat = con.obj['status'];
249
250 console.out("Sending MCP/RESET");
251 dim.send("MCP/RESET");
252
253 dim.wait("FTM_CONTROL", "Valid", 3000);
254 dim.wait("FAD_CONTROL", "Connected", 3000);
255 dim.wait("MCP", "Idle", 3000);
256
257 var list = [];
258 for (var i=0; i<40; i++)
259 if (stat[i]!=0x43)
260 list.push(i);
261
262 reconnect(list, "configuration");
263
264 if (n==2)
265 throw e;
266
267 //dim.wait("MCP", "Idle", 3000);
268 }
269 }
270
271 // This is to check if we have missed the event. This can happen as
272 // a race condition when the MCP/STOP is sent by the event handler
273 // but the run was not yet fully configured.
274 var statefb = dim.state("FEEDBACK").name;
275 if (statefb=="Critical" || statefb=="OnStandby")
276 {
277 console.out("Run started by FEEDBACK in state "+statefb);
278 dim.send("MCP/STOP"); // Includes FAD_CONTROL/CLOSE_ALL_OPEN_FILES
279
280 dim.onchange['FEEDBACK'] = callback;
281
282 return true;
283 }
284
285 dim.wait("MCP", "Idle", time>0 ? time*1250 : undefined); // run time plus 25%
286
287 // REMOVE watchdog
288 dim.onchange['FEEDBACK'] = callback;
289
290 if (incomplete)
291 {
292 console.out("");
293 console.out(" - MCP: "+dim.state("MCP").name);
294 console.out(" - FAD_CONTROL: "+dim.state("FAD_CONTROL").name);
295 console.out(" - FTM_CONTROL: "+dim.state("FTM_CONTROL").name);
296
297 dim.wait("FTM_CONTROL", "Valid", 3000);
298 dim.wait("FAD_CONTROL", "Connected", 3000);
299 dim.wait("MCP", "Idle", 3000);
300
301 var str = incomplete.toString(2);
302 var len = str.length;
303
304 var list = [];
305 for (var i=0; i<str.length; i++)
306 if (str[str.length-i-1]=='1')
307 list.push(i);
308
309 reconnect(list, "data taking");
310
311 return false;
312 }
313
314 // FIXME: What if the ext1 is not enabled in the configuration?
315 if (type=="data")
316 {
317 var dim_trg = new Subscription("FAD_CONTROL/TRIGGER_COUNTER");
318 var counter = dim_trg.get(3000);
319
320 // The check on physics and pedestal triggers is to ensure that
321 // there was at least a chance to receive any event (e.g. in case
322 // of an interrupt this might not be the case)
323 if (counter.qos!=111 &&
324 (counter.data['N_trg']>1000 || counter.data['N_ped']>5) &&
325 counter.data['N_ext1']==0) // 'o' for open
326 throw new Error("No ext1 triggers received during data taking... please check the reason and report in the logbook.");
327 dim_trg.close();
328 }
329
330 return true;
331}
Note: See TracBrowser for help on using the repository browser.