Changeset 18232 for branches/FACT++_scripts_refactoring/takeRun.js
- Timestamp:
- 06/18/15 14:14:51 (9 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
branches/FACT++_scripts_refactoring/takeRun.js
r18174 r18232 1 1 'use strict'; 2 2 3 // ================================================================ 4 // Code related to monitoring the fad system 5 // ================================================================ 6 7 var incomplete = 0; 8 9 sub_incomplete.onchange = function(evt) 10 { 11 if (!evt.data) 12 return; 13 14 var inc = evt.obj['incomplete']; 15 if (!inc || inc>0xffffffffff) 16 return; 17 18 if (incomplete>0) 19 return; 20 21 if (dim.state("MCP").name!="TakingData") 22 return; 23 24 console.out(""); 25 dim.log("Incomplete event ["+inc+","+incomplete+"] detected, sending MCP/STOP"); 26 27 incomplete = inc; 28 dim.send("MCP/STOP"); 3 if (!("takeRun" in this)){ 4 var takeRun = { 5 6 // ================================================================ 7 // Code related to monitoring the fad system 8 // ================================================================ 9 10 attachFadControlIncompleteOnChange : function() 11 { 12 var sub_incomplete = Subscription("FAD_CONTROL/INCOMPLETE"); 13 if (!sub_incomplete) 14 sub_incomplete = new Subscription("FAD_CONTROL/INCOMPLETE"); 15 16 attachFadControlIncompleteOnChange.incomplete = 0; 17 18 sub_incomplete.onchange = function(evt) 19 { 20 if (!evt.data) 21 return; 22 23 var inc = evt.obj['incomplete']; 24 if (!inc || inc>0xffffffffff) 25 return; 26 27 if (attachFadControlIncompleteOnChange.incomplete>0) 28 return; 29 30 if (dim.state("MCP").name!="TakingData") 31 return; 32 33 console.out(""); 34 dim.log("Incomplete event ["+inc+","+attachFadControlIncompleteOnChange.incomplete+"] detected, sending MCP/STOP"); 35 36 attachFadControlIncompleteOnChange.incomplete = inc; 37 dim.send("MCP/STOP"); 38 } 39 }, 40 41 // ================================================================ 42 // Code related to taking data 43 // ================================================================ 44 45 /** 46 * reconnect to problematic FADs 47 * 48 * Dis- and Reconnects to FADs, found to be problematic by call-back function 49 * onchange() to have a different CONNECTION value than 66 or 67. 50 * 51 * @returns 52 * a boolean is returned. 53 * reconnect returns true if: 54 * * nothing needed to be reset --> no problems found by onchange() 55 * * the reconnection went fine. 56 * 57 * reconnect *never returns false* so far. 58 * 59 * @example 60 * if (!sub_connections.reconnect()) 61 * exit(); 62 */ 63 reconnect :function(list, txt) 64 { 65 /* 66 var reset = [ ]; 67 68 for (var i=0; i<list.length; i++) 69 { 70 console.out(" FAD %2d".$(list[i])+" lost during "+txt); 71 reset.push(parseInt(list[i]/10)); 72 } 73 74 reset = reset.filter(function(elem,pos){return reset.indexOf(elem)==pos;}); 75 76 console.out(""); 77 console.out(" FADs belong to crate(s): "+reset); 78 console.out(""); 79 */ 80 console.out(""); 81 dim.log("Trying automatic reconnect ["+txt+",n="+list.length+"]..."); 82 83 if (list.length>3) 84 throw new Error("Too many boards to be reconnected. Please check what happened."); 85 86 for (var i=0; i<list.length; i++) 87 { 88 console.out(" ...disconnect "+list[i]); 89 dim.send("FAD_CONTROL/DISCONNECT", list[i]); 90 } 91 92 console.out(" ...waiting for 3s"); 93 v8.sleep(3000); 94 95 for (var i=0; i<list.length; i++) 96 { 97 console.out(" ...reconnect "+list[i]); 98 dim.send("FAD_CONTROL/CONNECT", list[i]); 99 } 100 101 console.out(" ...waiting for 1s"); 102 103 // Wait for one second to bridge possible pending connects 104 v8.sleep(1000); 105 106 console.out(" ...checking connection"); 107 108 // Wait for FAD_CONTROL to realize that all boards are connected 109 // FIXME: Wait for '40' boards being connected instead 110 try 111 { 112 dim.wait("FAD_CONTROL", "Connected", 3000); 113 } 114 catch (e) 115 { 116 if (dim.state("FAD_CONTROL").name!="Connecting") 117 { 118 console.out(""); 119 console.out(" + FAD_CONTROL: "+dim.state("FAD_CONTROL").name); 120 console.out(""); 121 throw e; 122 } 123 124 var crates = []; 125 for (var i=0; i<list.length; i++) 126 crates[list[i]/4] = true; 127 128 include('scripts/crateReset.js'); 129 crateReset(crates); 130 } 131 132 // Wait also for MCP to have all boards connected again 133 dim.wait("MCP", "Idle", 3000); 134 135 dim.log("Automatic reconnect successfull."); 136 console.out(""); 137 }, 138 139 takeRun: function(type, count, time) 140 { 141 if (!count) 142 count = -1; 143 if (!time) 144 time = -1; 145 146 var custom = typeof(type)=="function"; 147 148 var nextrun = sub_startrun.get().obj['next']; 149 dim.log("Take run %3d".$(nextrun)+": N="+count+" T="+time+"s ["+(custom?"custom":type)+"]"); 150 151 // FIXME: Replace by callback? 152 // 153 // DN: I believe instead of waiting for 'TakingData' one could split this 154 // up into two checks with an extra condition: 155 // if type == 'data': 156 // wait until ThresholdCalibration starts: 157 // --> this time should be pretty identical for each run 158 // if this takes longer than say 3s: 159 // there might be a problem with one/more FADs 160 // 161 // wait until "TakingData": 162 // --> this seems to take even some minutes sometimes... 163 // (might be optimized rather soon, but still in the moment...) 164 // if this takes way too long: 165 // there might be something broken, 166 // so maybe a very high time limit is ok here. 167 // I think there is not much that can go wrong, 168 // when the Thr-Calib has already started. Still it might be nice 169 // If in the future RateControl is written so to find out that 170 // in case the threshold finding algorithm does 171 // *not converge as usual* 172 // it can complain, and in this way give a hint, that the weather 173 // might be a little bit too bad. 174 // else: 175 // wait until "TakingData": 176 // --> in a non-data run this time should be pretty short again 177 // if this takes longer than say 3s: 178 // there might be a problem with one/more FADs 179 // 180 181 // Use this if you use the rate control to calibrate by rates 182 //if (!dim.wait("MCP", "TakingData", -300000) ) 183 //{ 184 // throw new Error("MCP took longer than 5 minutes to start TakingData"+ 185 // "maybe this idicates a problem with one of the FADs?"); 186 //} 187 188 // ================================================================ 189 // Function for Critical voltage 190 // ================================================================ 191 192 // INSTALL a watchdog... send FAD_CONTROL/CLOSE_OPEN_FILES 193 // could send MCP/RESET as well but would result in a timeout 194 var callback = dim.onchange['FEEDBACK']; 195 dim.onchange['FEEDBACK'] = function(state) 196 { 197 if (callback) 198 callback.call(this, state); 199 200 if ((state.name=="Critical" || state.name=="OnStandby") && 201 (this.last!="Critical" && this.last!="OnStandby")) 202 { 203 console.out("Feedback state changed from "+this.last+" to "+state.name+" [takeRun.js]"); 204 205 // Includes FAD_CONTROL/CLOSE_ALL_OPEN_FILES 206 dim.send("MCP/STOP"); 207 } 208 209 this.last=state.name; 210 } 211 212 // Here we could check and handle fad losses 213 214 attachFadControlIncompleteOnChange.incomplete = 0; 215 216 var start = true; 217 218 for (var n=0; n<3; n++) 219 { 220 if (start) 221 { 222 dim.send("MCP/START", time, count, custom?"custom":type); 223 if (custom) 224 type(); 225 } 226 227 try 228 { 229 dim.wait("MCP", "TakingData", 15000); 230 break; 231 } 232 catch (e) 233 { 234 if (dim.state("MCP").name=="TriggerOn" && 235 dim.state("FAD_CONTROL").name=="Connected" && 236 dim.state("FTM_CONTROL").name=="TriggerOn") 237 { 238 console.out(""); 239 console.out("Waiting for TakingData timed out. Everything looks ok, but file not yet open... waiting once more."); 240 start = false; 241 continue; 242 } 243 244 start = true; 245 246 console.out(""); 247 console.out(" + MCP: "+dim.state("MCP").name); 248 console.out(" + FAD_CONTROL: "+dim.state("FAD_CONTROL").name); 249 console.out(" + FTM_CONTROL: "+dim.state("FTM_CONTROL").name); 250 console.out(""); 251 252 if (dim.state("MCP").name!="Configuring3" || 253 (dim.state("FAD_CONTROL").name!="Configuring1" && 254 dim.state("FAD_CONTROL").name!="Configuring2")) 255 throw e; 256 257 console.out(""); 258 console.out("Waiting for fadctrl to get configured timed out... checking for in-run FAD loss."); 259 260 var con = sub_connections.get(); 261 var stat = con.obj['status']; 262 263 console.out("Sending MCP/RESET"); 264 dim.send("MCP/RESET"); 265 266 dim.wait("FTM_CONTROL", "Valid", 3000); 267 dim.wait("FAD_CONTROL", "Connected", 3000); 268 dim.wait("MCP", "Idle", 3000); 269 270 var list = []; 271 for (var i=0; i<40; i++) 272 if (stat[i]!=0x43) 273 list.push(i); 274 275 this.reconnect(list, "configuration"); 276 277 if (n==2) 278 throw e; 279 280 //dim.wait("MCP", "Idle", 3000); 281 } 282 } 283 284 // This is to check if we have missed the event. This can happen as 285 // a race condition when the MCP/STOP is sent by the event handler 286 // but the run was not yet fully configured. 287 var statefb = dim.state("FEEDBACK").name; 288 if (statefb=="Critical" || statefb=="OnStandby") 289 { 290 console.out("Run started by FEEDBACK in state "+statefb); 291 dim.send("MCP/STOP"); // Includes FAD_CONTROL/CLOSE_ALL_OPEN_FILES 292 293 dim.onchange['FEEDBACK'] = callback; 294 295 return true; 296 } 297 298 dim.wait("MCP", "Idle", time>0 ? time*1250 : undefined); // run time plus 25% 299 300 // REMOVE watchdog 301 dim.onchange['FEEDBACK'] = callback; 302 303 if (incomplete) 304 { 305 console.out(""); 306 console.out(" - MCP: "+dim.state("MCP").name); 307 console.out(" - FAD_CONTROL: "+dim.state("FAD_CONTROL").name); 308 console.out(" - FTM_CONTROL: "+dim.state("FTM_CONTROL").name); 309 310 dim.wait("FTM_CONTROL", "Valid", 3000); 311 dim.wait("FAD_CONTROL", "Connected", 3000); 312 dim.wait("MCP", "Idle", 3000); 313 314 var str = attachFadControlIncompleteOnChange.incomplete.toString(2); 315 var len = str.length; 316 317 var list = []; 318 for (var i=0; i<str.length; i++) 319 if (str[str.length-i-1]=='1') 320 list.push(i); 321 322 this.reconnect(list, "data taking"); 323 324 return false; 325 } 326 327 // FIXME: What if the ext1 is not enabled in the configuration? 328 if (type=="data") 329 { 330 var dim_trg = new Subscription("FAD_CONTROL/TRIGGER_COUNTER"); 331 var counter = dim_trg.get(3000); 332 333 // The check on physics and pedestal triggers is to ensure that 334 // there was at least a chance to receive any event (e.g. in case 335 // of an interrupt this might not be the case) 336 if (counter.qos!=111 && 337 (counter.data['N_trg']>1000 || counter.data['N_ped']>5) && 338 counter.data['N_ext1']==0) // 'o' for open 339 throw new Error("No ext1 triggers received during data taking... please check the reason and report in the logbook."); 340 dim_trg.close(); 341 } 342 343 return true; 344 }, 345 346 }; 29 347 } 30 31 // ================================================================ 32 // Code related to taking data 33 // ================================================================ 34 35 /** 36 * reconnect to problematic FADs 37 * 38 * Dis- and Reconnects to FADs, found to be problematic by call-back function 39 * onchange() to have a different CONNECTION value than 66 or 67. 40 * 41 * @returns 42 * a boolean is returned. 43 * reconnect returns true if: 44 * * nothing needed to be reset --> no problems found by onchange() 45 * * the reconnection went fine. 46 * 47 * reconnect *never returns false* so far. 48 * 49 * @example 50 * if (!sub_connections.reconnect()) 51 * exit(); 52 */ 53 function reconnect(list, txt) 54 { /* 55 var reset = [ ]; 56 57 for (var i=0; i<list.length; i++) 58 { 59 console.out(" FAD %2d".$(list[i])+" lost during "+txt); 60 reset.push(parseInt(list[i]/10)); 61 } 62 63 reset = reset.filter(function(elem,pos){return reset.indexOf(elem)==pos;}); 64 65 console.out(""); 66 console.out(" FADs belong to crate(s): "+reset); 67 console.out(""); 68 */ 69 console.out(""); 70 dim.log("Trying automatic reconnect ["+txt+",n="+list.length+"]..."); 71 72 if (list.length>3) 73 throw new Error("Too many boards to be reconnected. Please check what happened."); 74 75 for (var i=0; i<list.length; i++) 76 { 77 console.out(" ...disconnect "+list[i]); 78 dim.send("FAD_CONTROL/DISCONNECT", list[i]); 79 } 80 81 console.out(" ...waiting for 3s"); 82 v8.sleep(3000); 83 84 for (var i=0; i<list.length; i++) 85 { 86 console.out(" ...reconnect "+list[i]); 87 dim.send("FAD_CONTROL/CONNECT", list[i]); 88 } 89 90 console.out(" ...waiting for 1s"); 91 92 // Wait for one second to bridge possible pending connects 93 v8.sleep(1000); 94 95 console.out(" ...checking connection"); 96 97 // Wait for FAD_CONTROL to realize that all boards are connected 98 // FIXME: Wait for '40' boards being connected instead 99 try 100 { 101 dim.wait("FAD_CONTROL", "Connected", 3000); 102 } 103 catch (e) 104 { 105 if (dim.state("FAD_CONTROL").name!="Connecting") 106 { 107 console.out(""); 108 console.out(" + FAD_CONTROL: "+dim.state("FAD_CONTROL").name); 109 console.out(""); 110 throw e; 111 } 112 113 var crates = []; 114 for (var i=0; i<list.length; i++) 115 crates[list[i]/4] = true; 116 117 include('scripts/crateReset.js'); 118 crateReset(crates); 119 } 120 121 // Wait also for MCP to have all boards connected again 122 dim.wait("MCP", "Idle", 3000); 123 124 dim.log("Automatic reconnect successfull."); 125 console.out(""); 348 else{ 349 console.out("multiple include of 'takeRun.js'"); 126 350 } 127 351 128 function takeRun(type, count, time) 129 { 130 if (!count) 131 count = -1; 132 if (!time) 133 time = -1; 134 135 var custom = typeof(type)=="function"; 136 137 var nextrun = sub_startrun.get().obj['next']; 138 dim.log("Take run %3d".$(nextrun)+": N="+count+" T="+time+"s ["+(custom?"custom":type)+"]"); 139 140 // FIXME: Replace by callback? 141 // 142 // DN: I believe instead of waiting for 'TakingData' one could split this 143 // up into two checks with an extra condition: 144 // if type == 'data': 145 // wait until ThresholdCalibration starts: 146 // --> this time should be pretty identical for each run 147 // if this takes longer than say 3s: 148 // there might be a problem with one/more FADs 149 // 150 // wait until "TakingData": 151 // --> this seems to take even some minutes sometimes... 152 // (might be optimized rather soon, but still in the moment...) 153 // if this takes way too long: 154 // there might be something broken, 155 // so maybe a very high time limit is ok here. 156 // I think there is not much that can go wrong, 157 // when the Thr-Calib has already started. Still it might be nice 158 // If in the future RateControl is written so to find out that 159 // in case the threshold finding algorithm does 160 // *not converge as usual* 161 // it can complain, and in this way give a hint, that the weather 162 // might be a little bit too bad. 163 // else: 164 // wait until "TakingData": 165 // --> in a non-data run this time should be pretty short again 166 // if this takes longer than say 3s: 167 // there might be a problem with one/more FADs 168 // 169 170 // Use this if you use the rate control to calibrate by rates 171 //if (!dim.wait("MCP", "TakingData", -300000) ) 172 //{ 173 // throw new Error("MCP took longer than 5 minutes to start TakingData"+ 174 // "maybe this idicates a problem with one of the FADs?"); 175 //} 176 177 // ================================================================ 178 // Function for Critical voltage 179 // ================================================================ 180 181 // INSTALL a watchdog... send FAD_CONTROL/CLOSE_OPEN_FILES 182 // could send MCP/RESET as well but would result in a timeout 183 var callback = dim.onchange['FEEDBACK']; 184 dim.onchange['FEEDBACK'] = function(state) 185 { 186 if (callback) 187 callback.call(this, state); 188 189 if ((state.name=="Critical" || state.name=="OnStandby") && 190 (this.last!="Critical" && this.last!="OnStandby")) 191 { 192 console.out("Feedback state changed from "+this.last+" to "+state.name+" [takeRun.js]"); 193 194 // Includes FAD_CONTROL/CLOSE_ALL_OPEN_FILES 195 dim.send("MCP/STOP"); 196 } 197 198 this.last=state.name; 199 } 200 201 // Here we could check and handle fad losses 202 203 incomplete = 0; 204 205 var start = true; 206 207 for (var n=0; n<3; n++) 208 { 209 if (start) 210 { 211 dim.send("MCP/START", time, count, custom?"custom":type); 212 if (custom) 213 type(); 214 } 215 216 try 217 { 218 dim.wait("MCP", "TakingData", 15000); 219 break; 220 } 221 catch (e) 222 { 223 if (dim.state("MCP").name=="TriggerOn" && 224 dim.state("FAD_CONTROL").name=="Connected" && 225 dim.state("FTM_CONTROL").name=="TriggerOn") 226 { 227 console.out(""); 228 console.out("Waiting for TakingData timed out. Everything looks ok, but file not yet open... waiting once more."); 229 start = false; 230 continue; 231 } 232 233 start = true; 234 235 console.out(""); 236 console.out(" + MCP: "+dim.state("MCP").name); 237 console.out(" + FAD_CONTROL: "+dim.state("FAD_CONTROL").name); 238 console.out(" + FTM_CONTROL: "+dim.state("FTM_CONTROL").name); 239 console.out(""); 240 241 if (dim.state("MCP").name!="Configuring3" || 242 (dim.state("FAD_CONTROL").name!="Configuring1" && 243 dim.state("FAD_CONTROL").name!="Configuring2")) 244 throw e; 245 246 console.out(""); 247 console.out("Waiting for fadctrl to get configured timed out... checking for in-run FAD loss."); 248 249 var con = sub_connections.get(); 250 var stat = con.obj['status']; 251 252 console.out("Sending MCP/RESET"); 253 dim.send("MCP/RESET"); 254 255 dim.wait("FTM_CONTROL", "Valid", 3000); 256 dim.wait("FAD_CONTROL", "Connected", 3000); 257 dim.wait("MCP", "Idle", 3000); 258 259 var list = []; 260 for (var i=0; i<40; i++) 261 if (stat[i]!=0x43) 262 list.push(i); 263 264 reconnect(list, "configuration"); 265 266 if (n==2) 267 throw e; 268 269 //dim.wait("MCP", "Idle", 3000); 270 } 271 } 272 273 // This is to check if we have missed the event. This can happen as 274 // a race condition when the MCP/STOP is sent by the event handler 275 // but the run was not yet fully configured. 276 var statefb = dim.state("FEEDBACK").name; 277 if (statefb=="Critical" || statefb=="OnStandby") 278 { 279 console.out("Run started by FEEDBACK in state "+statefb); 280 dim.send("MCP/STOP"); // Includes FAD_CONTROL/CLOSE_ALL_OPEN_FILES 281 282 dim.onchange['FEEDBACK'] = callback; 283 284 return true; 285 } 286 287 dim.wait("MCP", "Idle", time>0 ? time*1250 : undefined); // run time plus 25% 288 289 // REMOVE watchdog 290 dim.onchange['FEEDBACK'] = callback; 291 292 if (incomplete) 293 { 294 console.out(""); 295 console.out(" - MCP: "+dim.state("MCP").name); 296 console.out(" - FAD_CONTROL: "+dim.state("FAD_CONTROL").name); 297 console.out(" - FTM_CONTROL: "+dim.state("FTM_CONTROL").name); 298 299 dim.wait("FTM_CONTROL", "Valid", 3000); 300 dim.wait("FAD_CONTROL", "Connected", 3000); 301 dim.wait("MCP", "Idle", 3000); 302 303 var str = incomplete.toString(2); 304 var len = str.length; 305 306 var list = []; 307 for (var i=0; i<str.length; i++) 308 if (str[str.length-i-1]=='1') 309 list.push(i); 310 311 reconnect(list, "data taking"); 312 313 return false; 314 } 315 316 // FIXME: What if the ext1 is not enabled in the configuration? 317 if (type=="data") 318 { 319 var dim_trg = new Subscription("FAD_CONTROL/TRIGGER_COUNTER"); 320 var counter = dim_trg.get(3000); 321 322 // The check on physics and pedestal triggers is to ensure that 323 // there was at least a chance to receive any event (e.g. in case 324 // of an interrupt this might not be the case) 325 if (counter.qos!=111 && 326 (counter.data['N_trg']>1000 || counter.data['N_ped']>5) && 327 counter.data['N_ext1']==0) // 'o' for open 328 throw new Error("No ext1 triggers received during data taking... please check the reason and report in the logbook."); 329 dim_trg.close(); 330 } 331 332 return true; 333 } 352 353 354 355 356
Note:
See TracChangeset
for help on using the changeset viewer.