source: branches/FACT++_scripts_refactoring/takeRun.js@ 18374

Last change on this file since 18374 was 18232, checked in by dneise, 10 years ago
made library
File size: 13.0 KB
Line 
1'use strict';
2
3if (!("takeRun" in this)){
4 var takeRun = {
5
6 // ================================================================
7 // Code related to monitoring the fad system
8 // ================================================================
9
10 attachFadControlIncompleteOnChange : function()
11 {
12 var sub_incomplete = Subscription("FAD_CONTROL/INCOMPLETE");
13 if (!sub_incomplete)
14 sub_incomplete = new Subscription("FAD_CONTROL/INCOMPLETE");
15
16 attachFadControlIncompleteOnChange.incomplete = 0;
17
18 sub_incomplete.onchange = function(evt)
19 {
20 if (!evt.data)
21 return;
22
23 var inc = evt.obj['incomplete'];
24 if (!inc || inc>0xffffffffff)
25 return;
26
27 if (attachFadControlIncompleteOnChange.incomplete>0)
28 return;
29
30 if (dim.state("MCP").name!="TakingData")
31 return;
32
33 console.out("");
34 dim.log("Incomplete event ["+inc+","+attachFadControlIncompleteOnChange.incomplete+"] detected, sending MCP/STOP");
35
36 attachFadControlIncompleteOnChange.incomplete = inc;
37 dim.send("MCP/STOP");
38 }
39 },
40
41 // ================================================================
42 // Code related to taking data
43 // ================================================================
44
45 /**
46 * reconnect to problematic FADs
47 *
48 * Dis- and Reconnects to FADs, found to be problematic by call-back function
49 * onchange() to have a different CONNECTION value than 66 or 67.
50 *
51 * @returns
52 * a boolean is returned.
53 * reconnect returns true if:
54 * * nothing needed to be reset --> no problems found by onchange()
55 * * the reconnection went fine.
56 *
57 * reconnect *never returns false* so far.
58 *
59 * @example
60 * if (!sub_connections.reconnect())
61 * exit();
62 */
63 reconnect :function(list, txt)
64 {
65 /*
66 var reset = [ ];
67
68 for (var i=0; i<list.length; i++)
69 {
70 console.out(" FAD %2d".$(list[i])+" lost during "+txt);
71 reset.push(parseInt(list[i]/10));
72 }
73
74 reset = reset.filter(function(elem,pos){return reset.indexOf(elem)==pos;});
75
76 console.out("");
77 console.out(" FADs belong to crate(s): "+reset);
78 console.out("");
79 */
80 console.out("");
81 dim.log("Trying automatic reconnect ["+txt+",n="+list.length+"]...");
82
83 if (list.length>3)
84 throw new Error("Too many boards to be reconnected. Please check what happened.");
85
86 for (var i=0; i<list.length; i++)
87 {
88 console.out(" ...disconnect "+list[i]);
89 dim.send("FAD_CONTROL/DISCONNECT", list[i]);
90 }
91
92 console.out(" ...waiting for 3s");
93 v8.sleep(3000);
94
95 for (var i=0; i<list.length; i++)
96 {
97 console.out(" ...reconnect "+list[i]);
98 dim.send("FAD_CONTROL/CONNECT", list[i]);
99 }
100
101 console.out(" ...waiting for 1s");
102
103 // Wait for one second to bridge possible pending connects
104 v8.sleep(1000);
105
106 console.out(" ...checking connection");
107
108 // Wait for FAD_CONTROL to realize that all boards are connected
109 // FIXME: Wait for '40' boards being connected instead
110 try
111 {
112 dim.wait("FAD_CONTROL", "Connected", 3000);
113 }
114 catch (e)
115 {
116 if (dim.state("FAD_CONTROL").name!="Connecting")
117 {
118 console.out("");
119 console.out(" + FAD_CONTROL: "+dim.state("FAD_CONTROL").name);
120 console.out("");
121 throw e;
122 }
123
124 var crates = [];
125 for (var i=0; i<list.length; i++)
126 crates[list[i]/4] = true;
127
128 include('scripts/crateReset.js');
129 crateReset(crates);
130 }
131
132 // Wait also for MCP to have all boards connected again
133 dim.wait("MCP", "Idle", 3000);
134
135 dim.log("Automatic reconnect successfull.");
136 console.out("");
137 },
138
139 takeRun: function(type, count, time)
140 {
141 if (!count)
142 count = -1;
143 if (!time)
144 time = -1;
145
146 var custom = typeof(type)=="function";
147
148 var nextrun = sub_startrun.get().obj['next'];
149 dim.log("Take run %3d".$(nextrun)+": N="+count+" T="+time+"s ["+(custom?"custom":type)+"]");
150
151 // FIXME: Replace by callback?
152 //
153 // DN: I believe instead of waiting for 'TakingData' one could split this
154 // up into two checks with an extra condition:
155 // if type == 'data':
156 // wait until ThresholdCalibration starts:
157 // --> this time should be pretty identical for each run
158 // if this takes longer than say 3s:
159 // there might be a problem with one/more FADs
160 //
161 // wait until "TakingData":
162 // --> this seems to take even some minutes sometimes...
163 // (might be optimized rather soon, but still in the moment...)
164 // if this takes way too long:
165 // there might be something broken,
166 // so maybe a very high time limit is ok here.
167 // I think there is not much that can go wrong,
168 // when the Thr-Calib has already started. Still it might be nice
169 // If in the future RateControl is written so to find out that
170 // in case the threshold finding algorithm does
171 // *not converge as usual*
172 // it can complain, and in this way give a hint, that the weather
173 // might be a little bit too bad.
174 // else:
175 // wait until "TakingData":
176 // --> in a non-data run this time should be pretty short again
177 // if this takes longer than say 3s:
178 // there might be a problem with one/more FADs
179 //
180
181 // Use this if you use the rate control to calibrate by rates
182 //if (!dim.wait("MCP", "TakingData", -300000) )
183 //{
184 // throw new Error("MCP took longer than 5 minutes to start TakingData"+
185 // "maybe this idicates a problem with one of the FADs?");
186 //}
187
188 // ================================================================
189 // Function for Critical voltage
190 // ================================================================
191
192 // INSTALL a watchdog... send FAD_CONTROL/CLOSE_OPEN_FILES
193 // could send MCP/RESET as well but would result in a timeout
194 var callback = dim.onchange['FEEDBACK'];
195 dim.onchange['FEEDBACK'] = function(state)
196 {
197 if (callback)
198 callback.call(this, state);
199
200 if ((state.name=="Critical" || state.name=="OnStandby") &&
201 (this.last!="Critical" && this.last!="OnStandby"))
202 {
203 console.out("Feedback state changed from "+this.last+" to "+state.name+" [takeRun.js]");
204
205 // Includes FAD_CONTROL/CLOSE_ALL_OPEN_FILES
206 dim.send("MCP/STOP");
207 }
208
209 this.last=state.name;
210 }
211
212 // Here we could check and handle fad losses
213
214 attachFadControlIncompleteOnChange.incomplete = 0;
215
216 var start = true;
217
218 for (var n=0; n<3; n++)
219 {
220 if (start)
221 {
222 dim.send("MCP/START", time, count, custom?"custom":type);
223 if (custom)
224 type();
225 }
226
227 try
228 {
229 dim.wait("MCP", "TakingData", 15000);
230 break;
231 }
232 catch (e)
233 {
234 if (dim.state("MCP").name=="TriggerOn" &&
235 dim.state("FAD_CONTROL").name=="Connected" &&
236 dim.state("FTM_CONTROL").name=="TriggerOn")
237 {
238 console.out("");
239 console.out("Waiting for TakingData timed out. Everything looks ok, but file not yet open... waiting once more.");
240 start = false;
241 continue;
242 }
243
244 start = true;
245
246 console.out("");
247 console.out(" + MCP: "+dim.state("MCP").name);
248 console.out(" + FAD_CONTROL: "+dim.state("FAD_CONTROL").name);
249 console.out(" + FTM_CONTROL: "+dim.state("FTM_CONTROL").name);
250 console.out("");
251
252 if (dim.state("MCP").name!="Configuring3" ||
253 (dim.state("FAD_CONTROL").name!="Configuring1" &&
254 dim.state("FAD_CONTROL").name!="Configuring2"))
255 throw e;
256
257 console.out("");
258 console.out("Waiting for fadctrl to get configured timed out... checking for in-run FAD loss.");
259
260 var con = sub_connections.get();
261 var stat = con.obj['status'];
262
263 console.out("Sending MCP/RESET");
264 dim.send("MCP/RESET");
265
266 dim.wait("FTM_CONTROL", "Valid", 3000);
267 dim.wait("FAD_CONTROL", "Connected", 3000);
268 dim.wait("MCP", "Idle", 3000);
269
270 var list = [];
271 for (var i=0; i<40; i++)
272 if (stat[i]!=0x43)
273 list.push(i);
274
275 this.reconnect(list, "configuration");
276
277 if (n==2)
278 throw e;
279
280 //dim.wait("MCP", "Idle", 3000);
281 }
282 }
283
284 // This is to check if we have missed the event. This can happen as
285 // a race condition when the MCP/STOP is sent by the event handler
286 // but the run was not yet fully configured.
287 var statefb = dim.state("FEEDBACK").name;
288 if (statefb=="Critical" || statefb=="OnStandby")
289 {
290 console.out("Run started by FEEDBACK in state "+statefb);
291 dim.send("MCP/STOP"); // Includes FAD_CONTROL/CLOSE_ALL_OPEN_FILES
292
293 dim.onchange['FEEDBACK'] = callback;
294
295 return true;
296 }
297
298 dim.wait("MCP", "Idle", time>0 ? time*1250 : undefined); // run time plus 25%
299
300 // REMOVE watchdog
301 dim.onchange['FEEDBACK'] = callback;
302
303 if (incomplete)
304 {
305 console.out("");
306 console.out(" - MCP: "+dim.state("MCP").name);
307 console.out(" - FAD_CONTROL: "+dim.state("FAD_CONTROL").name);
308 console.out(" - FTM_CONTROL: "+dim.state("FTM_CONTROL").name);
309
310 dim.wait("FTM_CONTROL", "Valid", 3000);
311 dim.wait("FAD_CONTROL", "Connected", 3000);
312 dim.wait("MCP", "Idle", 3000);
313
314 var str = attachFadControlIncompleteOnChange.incomplete.toString(2);
315 var len = str.length;
316
317 var list = [];
318 for (var i=0; i<str.length; i++)
319 if (str[str.length-i-1]=='1')
320 list.push(i);
321
322 this.reconnect(list, "data taking");
323
324 return false;
325 }
326
327 // FIXME: What if the ext1 is not enabled in the configuration?
328 if (type=="data")
329 {
330 var dim_trg = new Subscription("FAD_CONTROL/TRIGGER_COUNTER");
331 var counter = dim_trg.get(3000);
332
333 // The check on physics and pedestal triggers is to ensure that
334 // there was at least a chance to receive any event (e.g. in case
335 // of an interrupt this might not be the case)
336 if (counter.qos!=111 &&
337 (counter.data['N_trg']>1000 || counter.data['N_ped']>5) &&
338 counter.data['N_ext1']==0) // 'o' for open
339 throw new Error("No ext1 triggers received during data taking... please check the reason and report in the logbook.");
340 dim_trg.close();
341 }
342
343 return true;
344 },
345
346 };
347}
348else{
349 console.out("multiple include of 'takeRun.js'");
350}
351
352
353
354
355
356
Note: See TracBrowser for help on using the repository browser.