source: branches/FACT++_scripts_refactoring/CheckUnderflow.js@ 18374

'use strict';

include('scripts/CheckStates.js');
include('scripts/Hist2D.js');
include('scripts/Func.js');
include('scripts/takeRun.js');
include('scripts/tools.js');

if (!("CheckUnderflow" in this)){
    var CheckUnderflow = {

        check_power_on_time_and_get_drs_runs : function(){
            console.out("Checking power on time");

            var service_drs = new Subscription("FAD_CONTROL/DRS_RUNS");
            var runs = service_drs.get(5000, false);
            var power = dim.state("AGILENT_CONTROL_50V").time;
            var now   = new Date();

            console.out(" * Now:                "+now);
            console.out(" * Last power cycle:   "+power);
            console.out(" * Last DRS calib set: "+(runs.data?runs.time:"none"));

            service_drs.close();
            return runs;
        },

        // find a GOOD name! 
        check_states_one : function(){
            var table = [
             [ "MCP",                 [ "Idle"      ] ],
             [ "AGILENT_CONTROL_24V", [ "VoltageOn" ] ],
             [ "AGILENT_CONTROL_50V", [ "VoltageOn" ] ],
             [ "AGILENT_CONTROL_80V", [ "VoltageOn" ] ],
             [ "FTM_CONTROL",         [ "Valid"     ] ],
             [ "FAD_CONTROL",         [ "Connected",    "RunInProgress"   ] ],
             [ "BIAS_CONTROL",        [ "Disconnected", "VoltageOff"      ] ],
             [ "DATA_LOGGER",         [ "WaitForRun",   "NightlyFileOpen", "Logging" ] ],
            ];

            console.out("Checking states.");
            if (!CheckStates.checkStates(table))
            {
                throw new Error("Something unexpected has happened. One of the servers",
                        "is in a state in which it should not be. Please,",
                        "try to find out what happened...");
            }
        },

        prepare_drs_calibration : function(){
            dim.send("FAD_CONTROL/START_DRS_CALIBRATION");
            dim.send("FAD_CONTROL/SET_FILE_FORMAT", 0);

            var sub_runs = new Subscription("FAD_CONTROL/RUNS");
            var sruns = sub_runs.get(5000, false);

            if (dim.state("FAD_CONTROL").name=="RunInProgress" || sruns.qos==1)
            {
                dim.send("FAD_CONTROL/CLOSE_OPEN_FILES");
                dim.wait("FAD_CONTROL", "Connected", 3000);

                console.out("Waiting for open files to be closed...");
                v8.timeout(60000, function() { if (sub_runs.get(0, false).qos==0) return true; });

                // Although the file should be closed now, the processing might still be on-going
                // and delayed events might be received. The only fix for that issue is to
                // add the run number to the data we are waiting for
                v8.sleep(5000);
            }

            sub_runs.close();
        },

        take_at_least_one_drs_gain_events : function(){
            console.out("Starting drs-gain... waiting for new event");

            var fad_event_data_service = new Subscription("FAD_CONTROL/EVENT_DATA");
            var sub_startrun = new Subscription("FAD_CONTROL/START_RUN");
            var sub_incomplete = new Subscription("FAD_CONTROL/INCOMPLETE");
            var sub_connections = new Subscription("FAD_CONTROL/CONNECTIONS");
            sub_connections.get(5000);
            sub_startrun.get(5000);
            takeRun.attachFadControlIncompleteOnChange();

            while (1)
            {
                var event_counter_before_run = fad_event_data_service.get(10000, false).counter;

                var stop_run_after_first_events_recieved = function ()
                {
                    while (1)
                    {
                        var current_event_counter = fad_event_data_service.get(0, false).counter;
                        if (dim.state("MCP").name=="TakingData" && current_event_counter > event_counter_before_run)
                        {
                            dim.send("MCP/STOP");
                            console.out("Sent MCP/STOP.");
                            return;
                        }
                        v8.sleep(100);
                    }
                }

                var thread = new Thread(250, stop_run_after_first_events_recieved);

                var run_was_okay = takeRun.takeRun("drs-gain");

                thread.kill();

                if (run_was_okay)
                    break;
            }

            console.out("Event received.");

            sub_incomplete.close();
            sub_connections.close();
            sub_startrun.close();


            // FIXME: Restore DRS calibration in case of failure!!
            //        FAD Re-connect in case of failure?
            //        MCP/RESET in case of failure?
            //        Proper error reporting!

            var event = fad_event_data_service.get(3000);
            fad_event_data_service.close();

            console.out("Run stopped.");

            dim.send("RATE_CONTROL/STOP"); // GlobalThresholdSet -> Connected
            dim.wait("MCP", "Idle", 3000);

            return event;
        },

        get_max_drs_run_number : function(drs_runs)
        {
            if ( drs_runs.data 
                    && drs_runs.data.length > 0 
                    && drs_runs.obj['roi'] > 0 ) 
                return drs_runs.obj['run'].reduce(Func.max):
            else
                return -1;
        },

        stop_trigger : function(){
            if (dim.state("FTM_CONTROL").name=="TriggerOn")
            {
                dim.send("FTM_CONTROL/STOP_TRIGGER");
                dim.wait("FTM_CONTROL", "Valid");
            }
        },

        path_from_night_n_run : function(night_int, run_number, extension){

            var yy =  night_int/10000;
            var mm = (night_int/100)%100;
            var dd =  night_int%100;

            var filefmt = "/loc_data/raw/%d/%02d/%02d/%8d_%03d."+extension;
            var path = filefmt.$(yy, mm, dd, night_int, run_number);
            return path;
        },

        drs_path_from_night_n_run : function(night_int, run_number){
            return path_from_night_n_run(night_int, run_number, "drs.fits"):
        },

        restore_drs_calibration : function(drs_runs){
            var max_drs_run_number = this.get_max_drs_run_number(drs_runs);
            if (max_drs_run_number > 0)
            {
                var night = drs_runs.obj['night'];

                dim.log("Trying to restore last DRS calibration #"+max_drs_run_number+"  ["+drs_runs.time+"; "+night+"]");

                // FIXME: Timeout
                var service_drs = Subscription("FAD_CONTROL/DRS_RUNS");
                if (!service_drs)
                    service_drs = new Subscription("FAD_CONTROL/DRS_RUNS");
                var drs_counter = service_drs.get(0, false).counter;
                dim.send("FAD_CONTROL/LOAD_DRS_CALIBRATION", drs_path_from_night_n_run(night, max_drs_run_number));

                try
                {
                    var now = new Date();
                    v8.timeout(3000, function() { if (service_drs.get(0, false).counter>drs_counter) return true; });
                    dim.log("Last DRS calibration restored ["+(new Date()-now)/1000+"s]");
                }
                catch (e)
                {
                    console.warn("Restoring last DRS calibration failed.");
                }

                service_drs.close();
            }
        },

        analyse_drs_gain_event : function(event){
            var data = event.obj;

            var hist = Hist2D.Hist2D(16, -2048.5, 2048.5, 11, -10, 100);
            for (var i=0; i<1440; i++)
                hist.fill(data.avg[i], isNaN(data.rms[i])?-1:data.rms[i]);

            hist.print();

            var stat0 = Func.stat(data.avg, function(val, idx) { if (val<600) console.out(" PIX[hw="+idx+"]="+val); return val<600; });
            var stat1 = Func.stat(data.rms);

            console.out("Avg[min]=%.1f".$(stat0.min));
            console.out("Avg[avg]=%.1f +- %.1f".$(stat0.avg, stat0.rms));
            console.out("Avg[max]=%.1f".$(+stat0.max));
            console.out("Avg[cnt]="+stat0.count);
            console.out("");
            console.out("Rms[min]=%.1f".$(stat1.min));
            console.out("Rms[avg]=%.1f +- %.1f".$(stat1.avg, stat1.rms));
            console.out("Rms[max]=%.1f".$(stat1.max));
            console.out(tools.line_of_equal_signs(78));

            //      OK                            UNDERFLOW
            // ------------------------------------------------------
            // Avg[min]=722.0                Avg[min]=-380.0
            // Avg[avg]=815.9 +- 45.9        Avg[avg]= 808.0 +- 102.0
            // Avg[max]=930.5                Avg[max]= 931.1
            // Avg[cnt]=0                    Avg[cnt]= 9

            // Rms[min]=14.0                 Rms[min]=13.9
            // Rms[avg]=16.5 +- 1.6          Rms[avg]=18.8 +- 26.8
            // Rms[max]=44.0                 Rms[max]=382.1

            if (stat0.count>0)
            {
                if (stat0.count>8)
                    throw new Error("Underflow condition detected in about "+parseInt(stat0.count/9+.5)+" DRS.");

                console.warn("There is probably an underflow condition in one DRS... please check manually.");
            }
        },

        checkUnderflow : function(){
            console.out( tools.line_of_equal_signs(78) );            
            this.stop_trigger();
            this.check_states_one();
            var drs_runs = this.check_power_on_time_and_get_drs_runs();
            CheckStates.checkSend(["FAD_CONTROL", "MCP", "RATE_CONTROL"], 5000, true);            
            this.prepare_drs_calibration();
            var event = this.take_at_least_one_drs_gain_events();
            this.restore_drs_calibration(drs_runs);
            this.analyse_drs_gain_event(event):
        },
    };
}
else{
    console.out("multiple include of 'CheckUnderflow.js'")
}