source: trunk/FACT++/scripts/takeRun.js@ 16761

// ================================================================
//  Code related to monitoring the fad system
// ================================================================

var incomplete = 0;

sub_incomplete.onchange = function(evt)
{
    if (!evt.data)
        return;

    var inc = evt.obj['incomplete'];
    if (!inc || inc>0xffffffffff)
        return;

    if (incomplete>0)
        return;

    if (dim.state("MCP").name!="TakingData")
        return;

    console.out("", "Incomplete event ["+inc+","+incomplete+"] detected, sending MCP/STOP");

    incomplete = inc;
    dim.send("MCP/STOP");
}

// ================================================================
//  Code related to taking data
// ================================================================

/**
 * reconnect to problematic FADs
 *
 * Dis- and Reconnects to FADs, found to be problematic by call-back function
 * onchange() to have a different CONNECTION value than 66 or 67. 
 * 
 * @returns
 *      a boolean is returned. 
 *      reconnect returns true if:
 *          * nothing needed to be reset --> no problems found by onchange()
 *          * the reconnection went fine.
 *      
 *      reconnect *never returns false* so far.
 *
 * @example
 *      if (!sub_connections.reconnect())
 *          exit();
 */
function reconnect(list, txt)
{ /*
    var reset = [ ];

    for (var i=0; i<list.length; i++)
        {
            console.out("  FAD %2d".$(list[i])+" lost during "+txt);
            reset.push(parseInt(list[i]/10));
        }

    reset = reset.filter(function(elem,pos){return reset.indexOf(elem)==pos;});

    console.out("");
    console.out("  FADs belong to crate(s): "+reset);
    console.out("");
*/
    console.out("");
    console.out("Trying automatic reconnect ["+txt+",n="+list.length+"]...");
    console.out(new Date().toUTCString());

    if (list.length>3)
        throw new Error("Too many boards to be reconnected. Please check what happened.");

    for (var i=0; i<list.length; i++)
    {
        console.out("   ...disconnect "+list[i]);
        dim.send("FAD_CONTROL/DISCONNECT", list[i]);
    }

    console.out("   ...waiting for 3s");
    v8.sleep(3000);

    for (var i=0; i<list.length; i++)
    {
        console.out("   ...reconnect "+list[i]);
        dim.send("FAD_CONTROL/CONNECT", list[i]);
    }

    console.out("   ...waiting for 1s");

    // Wait for one second to bridge possible pending connects
    v8.sleep(1000);

    console.out("   ...checking connection");

    // Wait for FAD_CONTROL to realize that all boards are connected
    // FIXME: Wait for '40' boards being connected instead
    try
    {
        dim.wait("FAD_CONTROL", "Connected", 3000);
    }
    catch (e)
    {
        console.out("");
        console.out(" + FAD_CONTROL: "+dim.state("FAD_CONTROL").name);
        console.out("");
        throw e;
    }

    // Wait also for MCP to have all boards connected again
    dim.wait("MCP", "Idle", 3000);
    console.out("");
}

function takeRun(type, count, time)
{
    if (!count)
        count = -1;
    if (!time)
        time = -1;

    var nextrun = sub_startrun.get().obj['next'];
    console.out("  "+new Date().toUTCString()+": Take run %3d".$(nextrun)+": N="+count+" T="+time+"s ["+type+"]");

    // FIXME: Replace by callback?
    //
    // DN: I believe instead of waiting for 'TakingData' one could split this
    // up into two checks with an extra condition:
    //  if type == 'data':
    //      wait until ThresholdCalibration starts:
    //          --> this time should be pretty identical for each run
    //      if this takes longer than say 3s:
    //          there might be a problem with one/more FADs
    //    
    //      wait until "TakingData":
    //          --> this seems to take even some minutes sometimes... 
    //              (might be optimized rather soon, but still in the moment...)
    //      if this takes way too long: 
    //          there might be something broken, 
    //          so maybe a very high time limit is ok here.
    //          I think there is not much that can go wrong, 
    //          when the Thr-Calib has already started. Still it might be nice 
    //          If in the future RateControl is written so to find out that 
    //          in case the threshold finding algorithm does 
    //          *not converge as usual*
    //          it can complain, and in this way give a hint, that the weather
    //          might be a little bit too bad.
    //  else:
    //      wait until "TakingData":
    //          --> in a non-data run this time should be pretty short again
    //      if this takes longer than say 3s:
    //          there might be a problem with one/more FADs
    //  

    // Use this if you use the rate control to calibrate by rates
    //if (!dim.wait("MCP", "TakingData", -300000) )
    //{
    //    throw new Error("MCP took longer than 5 minutes to start TakingData"+
    //                    "maybe this idicates a problem with one of the FADs?");
    //}

    // Here we could check and handle fad losses

    incomplete = 0;

    var start = true;

    for (var n=0; n<3; n++)
    {
        if (start)
            dim.send("MCP/START", time, count, type);

        try
        {
            dim.wait("MCP", "TakingData", 15000);
            break;
        }
        catch (e)
        {
            if (dim.state("MCP").name=="TriggerOn" &&
                dim.state("FAD_CONTROL").name=="Connected" &&
                dim.state("FTM_CONTROL").name=="TriggerOn")
            {
                console.out("");
                console.out("Waiting for TakingData timed out. Everything looks ok, but file not yet open... waiting once more.");
                start = false;
                continue;
            }

            start = true;

            console.out("");
            console.out(" + MCP:         "+dim.state("MCP").name);
            console.out(" + FAD_CONTROL: "+dim.state("FAD_CONTROL").name);
            console.out(" + FTM_CONTROL: "+dim.state("FTM_CONTROL").name);
            console.out("");

            if (dim.state("MCP").name!="Configuring3" ||
                (dim.state("FAD_CONTROL").name!="Configuring1" &&
                 dim.state("FAD_CONTROL").name!="Configuring2"))
                throw e;

            console.out("");
            console.out("Waiting for fadctrl to get configured timed out... checking for in-run FAD loss.");

            var con  = sub_connections.get();
            var stat = con.obj['status'];

            console.out("Sending MCP/RESET");
            dim.send("MCP/RESET");

            dim.wait("FTM_CONTROL", "Idle",      3000);
            dim.wait("FAD_CONTROL", "Connected", 3000);
            dim.wait("MCP",         "Idle",      3000);

            var list = [];
            for (var i=0; i<40; i++)
                if (stat[i]!=0x43)
                    list.push(i);

            reconnect(list, "configuration");

            if (n==2)
                throw e;

            dim.wait("MCP", "Idle", 3000);
        }
    }

    dim.wait("MCP", "Idle", time>0 ? time*1250 : undefined); // run time plus 25%

    if (incomplete)
    {
        console.out("");
        console.out(" - MCP:         "+dim.state("MCP").name);
        console.out(" - FAD_CONTROL: "+dim.state("FAD_CONTROL").name);
        console.out(" - FTM_CONTROL: "+dim.state("FTM_CONTROL").name);

        dim.wait("FTM_CONTROL", "Idle",      3000);
        dim.wait("FAD_CONTROL", "Connected", 3000);
        dim.wait("MCP",         "Idle",      3000);

        var list = [];
        for (var i=0; i<40; i++)
            if (incomplete&(1<<i))
                list.push(i);

        reconnect(list, "data taking");

        return false;
    }

    return true;
}

undefined;