Index: trunk/DataCheck/Archive/ENDfixer.cpp =================================================================== --- trunk/DataCheck/Archive/ENDfixer.cpp (revision 12871) +++ trunk/DataCheck/Archive/ENDfixer.cpp (revision 12871) @@ -0,0 +1,133 @@ +/* + * fitsHacker.cc + * + * Created on: Sep 8, 2011 + * Author: lyard + */ + +#include +#include +#include +#include + +using namespace std; +/* + * Usage: fitsHacker (optionnal) + * + * + * + */ + +enum ModeT {seekingHDU, + foundHDU, + fixedEND, + reachedHeaderEnd}; + +int main(int argc, char** argv) +{ + if (argc < 2) + return 0; + +/* ENDfixer */ + fstream file(argv[1]); + + char c[81]; + c[80] = 0; + int seeking=0; + + ModeT mode = seekingHDU; + + bool reallyFixedEnd = false; + int endAddress = 0; + + while (mode != fixedEND) + { + file.read(c, 80); + if (!file.good()) { + cout << 0; + return 0; + } + string str(c); +// cout << c << endl; + if (str.substr(0, 9) == "XTENSION=") + mode = foundHDU; + + if (mode == foundHDU && str=="END ") + { + mode = fixedEND; + endAddress = seeking; +// cout << "found END at " << endAddress << endl; + } + if (mode == foundHDU && str ==" ") + { + file.seekp(seeking); + file.put('E'); + file.put('N'); + file.put('D'); + mode = fixedEND; + reallyFixedEnd = true; + endAddress = seeking; +// cout << "added END at " << endAddress << endl; + } + + seeking+=80; + } + + file.seekp(seeking-1); + while (mode != reachedHeaderEnd) + { + file.read(c, 80); + if (!file.good()) { + cout << 0; + return 0; + } + string str(c); + + if (str ==" ") + seeking+=80; + else + mode = reachedHeaderEnd; + } + + file.close(); + + if (seeking % 2880 != 0) + { + cout << "Error: header length not acceptable" << endl; + return 0; + } + + if (((seeking - endAddress)/80) > 36) + { + cout << "Error: too much header space after END keyword" << endl; + return 0; + } + + cout << seeking; + + return seeking; + +/* FITS HACKER + file.get(data, shift); + + for (int i=0;i + +using namespace std; + + +int main(int argc, const char** argv) +{ + if (argc != 2) + { + cout << "Error: only one argument is accepted" << endl; + return -1; + } + + double MjD = atof(argv[1]); + + Time t(MjD); + + cout << t.Iso() << endl; + +} Index: trunk/DataCheck/Archive/README =================================================================== --- trunk/DataCheck/Archive/README (revision 12871) +++ trunk/DataCheck/Archive/README (revision 12871) @@ -0,0 +1,13 @@ +This is the FACT construction data ingest and verification scripts README + +In order to ingest data, use either auxIngest.sh or rawIngest.sh appropriately. + +It takes source and dest folders as argument, the last arg being the string to append to log filenames. + +You will need the programs which source is included in this folder: please compile them appropriately + +The ingested data headers can be checked with checkForHeaderUpdatesAux/Raw.sh + +while the actual data can be compared with checkRawFioleIntegrityAndContent.sh + +This script will not work on aux data because the time column may have been updated. hence different files. Index: trunk/DataCheck/Archive/RowChecker.cpp =================================================================== --- trunk/DataCheck/Archive/RowChecker.cpp (revision 12871) +++ trunk/DataCheck/Archive/RowChecker.cpp (revision 12871) @@ -0,0 +1,114 @@ +/* + * RowChecker.cc + * + * Created on: Dec 20, 2011 + * Author: lyard + */ + +#include +#include +#include +#include +#include + +using namespace std; + + +//usage RowChecker +int main(int argc, char** argv) +{ + + if (argc < 6) + return 0; + + fstream file(argv[1]); + + int headLen = atoi(argv[2]); + int lineWidth = atoi(argv[3]); + double mjdRef = atof(argv[4]); + int numLines = atoi(argv[5]); + + int totalBytes = headLen; + file.seekp(headLen); + + char* buf = new char[lineWidth]; + + double currentTime = 0; + char timeBuf[16]; + int realNumRows = 0; + + while (file.read(buf, lineWidth)) + { + timeBuf[0] = buf[7]; + timeBuf[1] = buf[6]; + timeBuf[2] = buf[5]; + timeBuf[3] = buf[4]; + timeBuf[4] = buf[3]; + timeBuf[5] = buf[2]; + timeBuf[6] = buf[1]; + timeBuf[7] = buf[0]; + currentTime = reinterpret_cast(timeBuf)[0]; + + if (realNumRows >= numLines) + { + if (currentTime + mjdRef > 60000 || currentTime + mjdRef < 10000) + break; + if (currentTime + mjdRef > 20000 && currentTime + mjdRef < 50000) + break; + } +//fix the time column if required. + if (currentTime > 50000 && currentTime < 60000) + { + currentTime -= 40587; + reinterpret_cast(timeBuf)[0] = currentTime; + file.seekp(totalBytes); + file.put(timeBuf[7]); + file.put(timeBuf[6]); + file.put(timeBuf[5]); + file.put(timeBuf[4]); + file.put(timeBuf[3]); + file.put(timeBuf[2]); + file.put(timeBuf[1]); + file.put(timeBuf[0]); + file.seekp(totalBytes + lineWidth); + } + + realNumRows++; + totalBytes += lineWidth; + } + //now update the number of lines of the file + file.close(); + file.open(argv[1]); + file.seekp(2880); + delete[] buf; + buf = new char[81]; + buf[80] = 0; + bool changeDone = false; + int seeked = 2880; + if (realNumRows == numLines) + changeDone = true; + + while (file.good() && !changeDone) + { + file.read(buf, 80); + string str(buf); + + if (str.substr(0,9) == "NAXIS2 =") + { + ostringstream ss; + ss << realNumRows; + file.seekp(seeked + 30 - ss.str().size()); + for (int i=0;i>> $newBaseFolder" + fi + baseFolder=$newBaseFolder +done + +rawFolder=$baseFolder"/raw/"$year"/"$month"/"$day +auxFolder=$baseFolder"/aux/"$year"/"$month"/"$day +tempFile="./tempFile.txt" +tempFits="./tempFits.fits" +#input aux files +trackingTable="DRIVE_CONTROL_TRACKING_POSITION" +trackingFile=$auxFolder"/"$year$month$day"."$trackingTable".fits" +triggerTable="FTM_CONTROL_TRIGGER_RATES" +triggerFile=$auxFolder"/"$year$month$day"."$triggerTable".fits" +staticTable="FTM_CONTROL_STATIC_DATA" +staticFile=$auxFolder"/"$year$month$day"."$staticTable".fits" +voltagesTable="BIAS_CONTROL_VOLTAGE" +voltagesFile=$auxFolder"/"$year$month$day"."$voltagesTable".fits" +currentsTable="BIAS_CONTROL_CURRENT" +currentFile=$auxFolder"/"$year$month$day"."$currentsTable".fits" + +basePathAux=$auxFolder"/"$year$month$day + +auxTable="BIAS_CONTROL_NOMINAL" +auxDesc="Some_Description" +auxFile=$basePathAux"."$auxTable".fits" + +########################################### +# +# WRITE EXTRA AUX FILE +# Writes an extra entry to the data file +# Also checks for existence of said file +# +########################################## +function writeExtraAuxFile() { + + auxFile=$basePathAux"."$auxTable".fits" + if ! [ -e $auxFile ] + then + dummy=3 + #echo "WARNING: Optionnal aux file "$auxFile" not found." | tee -a $logfile 2>&1 + else + echo $auxTable" BINTABLE URL ../../../"$auxFile" 1 1 "$auxDesc >> $dataFile + fi +} + +currentEntry="" +roi=-1 +roiTM=-1 +runnumber=-1 +doNotDoThisEntry="false" +########################################### +# +# WRITE HEADER KEYS TO INPUT FILE +# Extract and write the header keywords to the above devined output text files. +# These text files are then meant to feed the fits creation tool fcreate +# Large parts of this function code were retaken from Daniela's scripts +# +########################################### +function writeHeaderKeysToInputFile () { +#echo "Writing header keys for "$currentEntry + #get all the missing data (everything appart from roi and roiTM) + runtype=`/opt/FACT++/fitsdump -h -t Events $currentEntry 2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"` + numevents=`/opt/FACT++/fitsdump -h -t Events $currentEntry 2>/dev/null | grep Events | grep -E -o '[0-9]+'` + numphys=`/opt/FACT++/fitsdump -h -t Events $currentEntry 2>/dev/null | grep 'NTRG ' | grep -E -o '[0-9]+'` + numext1=`/opt/FACT++/fitsdump -h -t Events $currentEntry 2>/dev/null | grep 'NTRGEXT1' | grep -E -o "'[0-9]+'" | grep -E -o '[0-9]+'` + numext2=`/opt/FACT++/fitsdump -h -t Events $currentEntry 2>/dev/null | grep 'NTRGEXT2' | grep -E -o "'[0-9]+'" | grep -E -o '[0-9]+'` + numelp=`/opt/FACT++/fitsdump -h -t Events $currentEntry 2>/dev/null | grep 'NTRGLPE' | grep -E -o '[0-9]+'` + numilp=`/opt/FACT++/fitsdump -h -t Events $currentEntry 2>/dev/null | grep 'NTRGLPI' | grep -E -o '[0-9]+'` + numoth=`/opt/FACT++/fitsdump -h -t Events $currentEntry 2>/dev/null | grep 'NTRGMISC' | grep -E -o '[0-9]+'` + numped=`/opt/FACT++/fitsdump -h -t Events $currentEntry 2>/dev/null | grep 'NTRGPED' | grep -E -o '[0-9]+'` + numtime=`/opt/FACT++/fitsdump -h -t Events $currentEntry 2>/dev/null | grep 'NTRGTIM' | grep -E -o '[0-9]+'` + dateRaw=$year$month$day + + if [ "$runtype" == "" ] + then + runtype="N/A" + fi + if [ "$numevents" == "" ] + then + numevents="0" + fi + +# dateRaw=`echo $currentEntry | grep -E -o '20[1-3][0-9][01][0-9][012][0-9]'` +#echo "dateRaw: "$dateRaw" yearmonthday: "$year$month$day + + # in newest data start time is in DATE-OBS + # in older data start time is in TSTART + # in the beginning TSTART was empty + runstart=`/opt/FACT++/fitsdump -h -t Events $currentEntry 2>/dev/null | grep DATE-OBS | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` + runstart2=`/opt/FACT++/fitsdump -h -t Events $currentEntry 2>/dev/null | grep TSTART | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` + if [ "$runstart" == "" ] + then + if [ "$runstart2" == "" ] + then + runstart=`/opt/FACT++/fitsdump -h -t Events $currentEntry 2>/dev/null | grep DATE | grep -v 'DATE-' | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` + else + runstart=$runstart2 + fi + fi + # in newest data start time is in DATE-END + # in older data start time is in TSTOP + # in the beginning TSTOP was empty + runstop=`/opt/FACT++/fitsdump -h -t Events $currentEntry 2>/dev/null | grep DATE-END | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` + runstop2=`/opt/FACT++/fitsdump -h -t Events $currentEntry 2>/dev/null | grep TSTOP | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{6}'` + if [ "$runstop" == "" ] + then + if [ "$runstop2" == "" ] + then + runstop=`stat $currentEntry 2>/dev/null | grep Modify | grep -E -o '20[1-9][0-9][\-][01][0-9][\-][0-3][0-9][ ][0-2][0-9]:[0-6][0-9]:[0-6][0-9][.][0-9]{9}'` + else + runstop=$runstop2 + fi + fi + + # set runtype to 'unknown' if no runtype could be queried + if [ "$runtype" == "" ] + then + runtype="n/a" + fi + # on 15.11.2011 the runtypes had different names + if [ "$date" == "2011/11/15" ] + then + if [ "$runtype" == "drs-calib" ] + then + runtype="drs-gain" + fi + if [ "$runtype" == "drs-time-calib" ] + then + runtype="drs-time" + fi + if [ "$runtype" == "pedestal" ] + then + runtype="drs-pedestal" + fi + if [ "$runtype" == "light-pulser" ] + then + runtype="light-pulser-ext" + fi + if [ "$runtype" == "pedestal-on" ] + then + runtype="pedestal" + fi + fi + #now take care of the slow control data. + #first get the start and stop time of the raw file + + mjdref=`/opt/FACT++/fitsdump -h -t Events $currentEntry 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'` + tstarti=`/opt/FACT++/fitsdump -h -t Events $currentEntry 2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'` + tstartf=`/opt/FACT++/fitsdump -h -t Events $currentEntry 2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'` + tstopi=`/opt/FACT++/fitsdump -h -t Events $currentEntry 2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'` + tstopf=`/opt/FACT++/fitsdump -h -t Events $currentEntry 2>/dev/null | grep 'TSTOPF' | grep -E -o '0[.][0-9]+'` + doNotDoThisEntry="false" + if [ "$tstarti" == "0" ] || [ "$tstopi" == "0" ] || [ "$mjdref" == "0" ] + then + echo "ERROR: "$currentEntry" has tstart, tstop or mjdref = 0" + fi + + if [ "$tstartf" == "" ] + then + doNotDoThisEntry="true" + echo $currentEntry >> problemWithTSTART.txt + echo "WARNING: "$currentEntry" Has problems with Dates" + fi + if [ "$tstarti" == "" ] + then + tstarti="0" + fi + if [ "$tstopi" == "" ] + then + tstopi="0" + fi + if [ "$mjdref" == "" ] + then + mjdref="0" + fi + if [ "$tstartf" == "" ] + then + tstartf="0" + fi + if [ "$tstopf" == "" ] + then + tstopf="0" + fi + + tstart=`echo " $tstarti + $mjdref + $tstartf " | bc -l` + tstop=`echo " $tstopi + $mjdref + $tstopf " | bc -l` + exposure=`echo "$tstop - $tstart " | bc -l` + exposure=`echo "$exposure * 86400" | bc -l ` + + #now get relevant data from daily files + #first get mjdref for the aux file and adapt start time accordingly + auxmjdref=`/opt/FACT++/fitsdump -h -t $trackingTable $trackingFile 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'` + if [ "$auxmjdref" == "" ] + then + echo "ERROR: "$trackingFile" has no mjdref available. Aborting script" + exit + fi + + tstartaux=`echo " $tstart - $auxmjdref " | bc -l` + tstopaux=`echo " $tstop - $auxmjdref " | bc -l` + ftcopy $trackingFile'[Time> '${tstartaux}' && Time< '${tstopaux}'][col Ra;Dec;Zd;Az;Time]' history=NO !$tempFits + /opt/FACT++/fitsdump $tempFits -c Ra --stat -o $tempFile 2>/dev/null + ramin=`grep 'Min' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + ramax=`grep 'Max' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + ramean=`grep 'Mea' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + if [ "$ramin" != "" ] + then + ramin=`echo "$ramin * 15" | bc -l` + ramax=`echo "$ramax * 15" | bc -l` + ramean=`echo "$ramean * 15" | bc -l` + fi + /opt/FACT++/fitsdump $tempFits -c Dec --stat -o $tempFile 2>/dev/null + decmin=`grep 'Min' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + decmax=`grep 'Max' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + decmean=`grep 'Mea' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + /opt/FACT++/fitsdump $tempFits -c Zd --stat -o $tempFile 2>/dev/null + zdmin=`grep 'Min' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + zdmax=`grep 'Max' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + zdmean=`grep 'Mea' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + /opt/FACT++/fitsdump $tempFits -c Az --stat -o $tempFile 2>/dev/null + azmin=`grep 'Min' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + azmax=`grep 'Max' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + azmean=`grep 'Mea' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + + auxmjdref=`/opt/FACT++/fitsdump -h -t $triggerTable $triggerFile 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'` + if [ "$auxmjdref" == "" ] + then + echo "ERROR: "$triggerFile" has no mjdref available. Aborting script" + exit + fi + tstartaux=`echo " $tstart - $auxmjdref " | bc -l` + tstopaux=`echo " $tstop - $auxmjdref " | bc -l` + ftcopy $triggerFile'[Time> '${tstartaux}' && Time< '${tstopaux}'][col TriggerRate;Time]' history=NO !$tempFits + /opt/FACT++/fitsdump $tempFits -c TriggerRate --stat -o $tempFile 2>/dev/null + ratemin=`grep 'Min' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + ratemax=`grep 'Max' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + ratemean=`grep 'Mea' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + ratemedian=`grep 'Med' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + + auxmjdref=`/opt/FACT++/fitsdump -h -t $staticTable $staticFile 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'` + if [ "$auxmjdref" == "" ] + then + echo "ERROR: "$staticFile" has no mjdref available. Aborting script" + exit + fi + iter=0 + timeShift=0 + threshmin="" + while [ "$threshmin" == "" ] && [ "$iter" != "10" ] + do + tstartaux=`echo " $tstart - $auxmjdref - $timeShift " | bc -l` + tstopaux=`echo " $tstop - $auxmjdref " | bc -l` + ftcopy $staticFile'[Time> '${tstartaux}' && Time< '${tstopaux}'][col PatchThresh;Time]' history=NO !$tempFits + /opt/FACT++/fitsdump $tempFits -c PatchThresh --stat -o $tempFile 2>/dev/null + threshmin=`grep 'Min' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + threshmax=`grep 'Max' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + threshmean=`grep 'Mea' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + threshmedian=`grep 'Med' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + + timeShift=`echo " $timeShift + 0.00011574 " | bc -l` + iter=`echo " $iter + 1 " | bc -l` + done + + ######################################################################################################################################################### + auxmjdref=`/opt/FACT++/fitsdump -h -t $voltagesTable $voltagesFile 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'` + if [ "$auxmjdref" == "" ] + then + echo "ERROR: "$voltagesFile" has no mjdref available. Aborting script" + exit + fi + iter=0 + timeShift=0 + biasvoltmin="" + while [ "$biasvoltmin" == "" ] && [ "$iter" != "10" ] + do + tstartaux=`echo " $tstart - $auxmjdref " | bc -l` + tstopaux=`echo " $tstop - $auxmjdref " | bc -l` + ftcopy $voltagesFile'[Time> '${tstartaux}' && Time< '${tstopaux}'][col U;Time]' !$tempFits + /opt/FACT++/fitsdump $tempFits -c U[0:319] --stat -o $tempFile 2>/dev/null + biasvoltmin=`grep 'Min' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + biasvoltmax=`grep 'Max' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + biasvoltmean=`grep 'Mea' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + biasvoltmedian=`grep 'Med' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + timeShift=`echo " $timeShift + 0.00011574 " | bc -l` + iter=`echo " $iter + 1 " | bc -l` + + done + + auxmjdref=`/opt/FACT++/fitsdump -h -t $currentsTable $currentFile 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'` + if [ "$auxmjdref" == "" ] + then + echo "ERROR: "$currentFile" has no mjdref available. Aborting script" + exit + fi + iter=0 + timeShift=0 + biascurrentmin="" + while [ "$biascurrentmin" == "" ] && [ "$iter" != "10" ] + do + tstartaux=`echo " $tstart - $auxmjdref " | bc -l` + tstopaux=`echo " $tstop - $auxmjdref " | bc -l` + ftcopy $currentFile'[Time> '${tstartaux}' && Time< '${tstopaux}'][col I;Time]' !$tempFits + /opt/FACT++/fitsdump $tempFits -c I[0:319] --stat -o $tempFile 2>/dev/null + biascurrentmin=`grep 'Min' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + biascurrentmax=`grep 'Max' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + biascurrentmean=`grep 'Mea' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + biascurrentmedian=`grep 'Med' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + timeShift=`echo " $timeShift + 0.00011574 " | bc -l` + iter=`echo " $iter + 1 " | bc -l` + +#echo "biascurrent min, max. mean, median: "$biascurrentmin" "$biascurrentmax" "$biascurrentmean" "$biascurrentmedian + done + + #write the retrieved data to the input file for creating fits afterwards + echo "RUNTYPE "$runtype" / run type" >> $keywordsFile + echo "ROI "$roi" / region of interest" >> $keywordsFile + echo "ROITM "$roiTM" / Roi for time markers" >> $keywordsFile + echo "NUMEVENT "$numevents" / number of events" >> $keywordsFile + echo "NUMPHYS "$numphys" / num of physical triggers" >> $keywordsFile + echo "NUMEXT1 "$numext1" / num of Light pulser 1 triggers" >> $keywordsFile + echo "NUMEXT2 "$numext2" / num of Light pulser 2 triggers" >> $keywordsFile + echo "NUMELP "$numelp" / num of external Light pulser triggers" >> $keywordsFile + echo "NUMILP "$numilp" / num of internal Light pulser triggers" >> $keywordsFile + echo "NUMOTH "$numoth" / num of other triggers" >> $keywordsFile + echo "NUMPED "$numped" / num of pedestal triggers" >> $keywordsFile + echo "NUMTIME "$numtime" / num of time calibration triggers" >> $keywordsFile + echo "DATEOBS "$runstart" / start of observation" >> $keywordsFile + echo "MJDREF "$mjdref" / reference of MjD values" >> $keywordsFile + echo "EXPOSURE "$exposure" /duration of run" >> $keywordsFile + echo "TSTARTI "$tstarti" / first event arrival time (int)" >> $keywordsFile + echo "TSTARTF "$tstartf" / first event arrival time (frac)" >> $keywordsFile + echo "TSTOPI "$tstopi" / last event arrival time (int)" >> $keywordsFile + echo "TSTOPF "$tstopf" / last event arrival time (frac)" >> $keywordsFile + echo "RAMIN "$ramin" / min value of right ascension" >> $keywordsFile + echo "RAMAX "$ramax" / max value of right ascension" >> $keywordsFile + echo "RAMEAN "$ramean" / mean value of right ascension" >> $keywordsFile + echo "DECMIN "$decmin" / min value of declination" >> $keywordsFile + echo "DECMAX "$decmax" / max value of declination" >> $keywordsFile + echo "DECMEAN "$decmean" / mean value of declination" >> $keywordsFile + echo "ZDMIN "$zdmin" / min value of zenith distance" >> $keywordsFile + echo "ZDMAX "$zdmax" / max value of zenith distance" >> $keywordsFile + echo "ZDMEAN "$zdmean" / mean value of zenith distance" >> $keywordsFile + echo "AZMIN "$azmin" / min value of azimuth" >> $keywordsFile + echo "AZMAX "$azmax" / max value of azimuth" >> $keywordsFile + echo "AZMEAN "$azmean" / mean value of azimuth" >> $keywordsFile + echo "RATEMIN "$ratemin" / min value of trigger rates" >> $keywordsFile + echo "RATEMAX "$ratemax" / max value of trigger rates" >> $keywordsFile + echo "RATEMEAN "$ratemean" / mean value of trigger rates" >> $keywordsFile + echo "RATEMED "$ratemedian" / median value of trigger rates" >> $keywordsFile + echo "THRESMIN "$threshmin" / min threshold value" >> $keywordsFile + echo "THRESMAX "$threshmax" / max threshold value" >> $keywordsFile + echo "THRESMEA "$threshmean" / mean threshold value" >> $keywordsFile + echo "THRESMED "$threshmedian" / max threshold value" >> $keywordsFile + echo "BIASVMIN "$biasvoltmin" / min bias voltage (V)" >> $keywordsFile + echo "BIASVMAX "$biasvoltmax" / max bias voltage (V)" >> $keywordsFile + echo "BIASVMEA "$biasvoltmean" / mean bias voltage (V)" >> $keywordsFile + echo "BIASVMED "$biasvoltmedian" / median bias voltage (V)" >> $keywordsFile + echo "BIASAMIN "$biascurrentmin" / min bias current (uA)" >> $keywordsFile + echo "BIASAMAX "$biascurrentmax" / max bias current (uA)" >> $keywordsFile + echo "BIASAMEA "$biascurrentmean" / mean bias current (uA)" >> $keywordsFile + echo "BIASAMED "$biascurrentmedian" / median bias current (uA)" >> $keywordsFile + + + echo "EXTNAME GROUPING / grouping table" >> $keywordsFile + echo "GRPNAME FACT-RAW / name of group" >> $keywordsFile + echo "RUNNUM "$runnumber" / run number" >> $keywordsFile + echo "RUNID "$dateRaw"_"$runnumber" / Run Id" >> $keywordsFile + +} + +#today=`date +%F` +#alias alsoToLog='tee -a $logfile 2>&1' + +#echo "" | tee -a $logfile 2>&1 +#echo "" | tee -a $logfile 2>&1 +#echo "" | tee -a $logfile 2>&1 + +#cleanup logfile +rm -f $logfile + +#echo `date`" executing "$0 | tee -a $logfile 2>&1 + + + +#first of all, let's make sure that required slow control files are present +if ! [ -e $trackingFile ] +then + echo "ERROR: Required aux file "$trackingFile" not found. Aborting script" | tee -a $logfile 2>&1 + exit +fi +if ! [ -e $triggerFile ] +then + echo "ERROR: Required aux file "$triggerFile" not found. Aborting script" | tee -a $logfile 2>&1 + exit +fi +if ! [ -e $staticFile ] +then + echo "ERROR: Required aux file "$staticFile" not found. Aborting script" | tee -a $logfile 2>&1 + exit +fi +if ! [ -e $voltagesFile ] +then + echo "ERROR: Required aux file "$voltagesFile" not found. Aborting script" | tee -a $logfile 2>&1 + exit +fi +if ! [ -e $currentFile ] +then + echo "ERROR: Required aux file "$currentFile" not found. Aborting script" | tee -a $logfile 2>&1 + exit +fi + +#next define the format of the grouping file + +rm -f $colDescFile +echo "MEMBER_NAME 60A" >> $colDescFile +echo "MEMBER_XTENSION 8A" >> $colDescFile +echo "MEMBER_URI_TYPE 3A" >> $colDescFile +echo "MEMBER_LOCATION 256A" >> $colDescFile +echo "MEMBER_VERSION 1J" >> $colDescFile +echo "MEMBER_POSITION 1J" >> $colDescFile +echo "DATA_TYPE 60A" >> $colDescFile + +entries=`find $rawFolder -type f -name '*.fits.gz' | sort` +calibs="" +calibFiles="" +calibDrsFiles="" +calibDrsPedestalFiles="" +calibsRoi="" +calibsRoiTM="" + +#data="" +#correspondingCalib="" +numCalibs=0 +#numData=0 + +if [ "$year" == "" ] +then + echo "Missing year argument" + exit +fi + +if [ "$month" == "" ] +then + echo "Missing month argument" + exit +fi + +if [ "$day" == "" ] +then + echo "Missing day argument" + exit +fi + +numEntries=0 +for item in ${entries[@]} +do + numEntries=`expr $numEntries + 1` +done + +echo "There are "$numEntries" entries to examine in folder "$rawFolder | tee -a $logfile 2>&1 + +#echo "Will now erase entries generated previously..." +#toDelete=`find $targetFolder -type f -name '*_raw.fits' | sort` +#for delete in ${toDelete[@]} +#do +# echo "removing "$delete +# rm $delete +#done +#toDelete=`find $targetFolder -type f -name '*_raw.txt' | sort` +#for delete in ${toDelete[@]} +#do +# echo "removing $delete" +# rm $delete +#done +#echo "done" + +for entry in ${entries[@]} +do + filename=`echo $entry | grep -E -o '20[1-9][0-9][01][0-9][0-3][0-9]_[0-9]{3}'` + filedate=`echo $filename | cut -d_ -f1` + runnumberInter=`echo $filename | cut -d_ -f2 | sed -e 's/^0//g' -e 's/^0//g'` + runnumber=`printf %03d $runnumberInter` + #figure out whether this is a drs calib + if [ "`echo $entry | grep drs`" != "" ] + then + #check if this is the pedestal run. for this, grep the associated raw data file and check for previous calibrations (drs pedestal and drs gain) + minusOne=`expr $runnumber - 1` + minusOne=`printf %03d $minusOne` + minusTwo=`expr $runnumber - 2` + minusTwo=`printf %03d $minusTwo` + pedestalRaw=$rawFolder/$filename.fits.gz + DrsGainRaw=$rawFolder/$filedate"_"$minusOne.fits.gz + DrsPedestalRaw=$rawFolder/$filedate"_"$minusTwo.fits.gz + DrsGain=$rawFolder/$filedate"_"$minusOne.drs.fits.gz + DrsPedestal=$rawFolder/$filedate"_"$minusTwo.drs.fits.gz + pedestalkey="" + drsGainKey="" + drsPedestalKey="" + if [ -f $pedestalRaw ] + then + pedestalKey=`/opt/FACT++/fitsdump -h $pedestalRaw 2>/dev/null | grep "'pedestal'"` + fi + if [ "$pedestalKey" == "" ] + then #the current drs file is NOT a pedestal. Continue + continue + fi +# echo "Found Pedestal entry "$entry + if [ -f $DrsGainRaw ] && [ -f $DrsGain ] + then + drsGainKey=`/opt/FACT++/fitsdump -h $DrsGainRaw 2>/dev/null | grep "'drs-gain'"` + else + DrsGain="" + fi + if [ -f $DrsPedestalRaw ] && [ -f $DrsPedestal ] + then + drsPedestalKey=`/opt/FACT++/fitsdump -h $DrsPedestalRaw 2>/dev/null | grep "'drs-pedestal'"` + else + DrsPedestal="" + fi + + if [ "$drsPedestalKey" != "" ] + then + calibtstarti=`/opt/FACT++/fitsdump -h $DrsPedestalRaw 2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'` + calibtstartf=`/opt/FACT++/fitsdump -h $DrsPedestalRaw 2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'` + else + if [ "$drsGainKey" != "" ] + then + calibtstarti=`/opt/FACT++/fitsdump -h $DrsGainRaw 2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'` + calibtstartf=`/opt/FACT++/fitsdump -h $DrsGainRaw 2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'` + else + calibtstarti=`/opt/FACT++/fitsdump -h $pedestalRaw 2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'` + calibtstartf=`/opt/FACT++/fitsdump -h $pedestalRaw 2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'` + fi + fi + calibtstopi=`/opt/FACT++/fitsdump -h $pedestalRaw 2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'` + calibtstopf=`/opt/FACT++/fitsdump -h $pedestalRaw 2>/dev/null | grep 'TSTOPF' | grep -E -o '0[.][0-9]+'` + + auxstarti=`/opt/FACT++/fitsdump -h $voltagesFile 2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'` + auxstartf=`/opt/FACT++/fitsdump -h $voltagesFile 2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'` + + if [ "$auxstarti" == "" ] + then + auxstarti="0" + fi + if [ "$auxstartf" == "" ] + then + auxstartf="0" + fi + if [ "$calibtstarti" == "" ] + then + calibtstarti="0" + fi + if [ "$calibtstartf" == "" ] + then + calibtstartf="0" + fi + if [ "$calibtstopi" == "" ] + then + calibtstopi="0" + fi + if [ "$calibtstopf" == "" ] + then + calibtstopf="0" + fi + + auxstart=`echo " $auxstarti + 40587 + $auxstartf " | bc -l` + calibtstart=`echo " $calibtstarti + 40587 + $calibtstartf " | bc -l` + calibtstop=`echo " $calibtstopi + 40587 + $calibtstopf " | bc -l` + iter=0 + timeShift=0 + biasvoltmean="" + tstartaux=`echo " $calibtstart - 40587 - $timeShift" | bc -l` + compare=`echo " $tstartaux >= $auxstart " | bc -l` + + while [ "$biasvoltmean" == "" ] && [ "$iter" != "10" ] && [ "$compare" == "1" ] + do + tstartaux=`echo " $calibtstart - 40587 - $timeShift" | bc -l` + compare=`echo " $tstartaux >= $auxstart " | bc -l` + tstopaux=`echo " $calibtstop - 40587 " | bc -l` + ftcopy $voltagesFile'[Time> '${tstartaux}' && Time< '${tstopaux}'][col U;Time]' !$tempFits + /opt/FACT++/fitsdump $tempFits -c U[0:319] --stat -o $tempFile 2>/dev/null + biasvoltmean=`grep 'Mea' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + timeShift=`echo " $timeShift + 0.000011574 " | bc -l` + iter=`echo " $iter + 1 " | bc -l` + done + auxstarti=`/opt/FACT++/fitsdump -h $currentFile 2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'` + auxstartf=`/opt/FACT++/fitsdump -h $currentFile 2>/dev/null | grep 'TSTARTF' | grep -E -o '0[.][0-9]+'` + auxstart=`echo " $auxstarti + $auxstartf " | bc -l` + iter=0 + timeShift=0 + biascurrentmean="" + tstartaux=`echo " $calibtstart - 40587 - $timeShift" | bc -l` + compare=`echo " $tstartaux >= $auxstart " | bc -l` + while [ "$biascurrentmean" == "" ] && [ "$iter" != "10" ] && [ "$compare" == "1" ] + do + tstartaux=`echo " $calibtstart - 40587 - $timeShift " | bc -l` + compare=`echo " $tstartaux >= $auxstart " | bc -l` + tstopaux=`echo " $calibtstop - 40587 " | bc -l` + ftcopy $currentFile'[Time> '${tstartaux}' && Time< '${tstopaux}'][col I;Time]' !$tempFits + /opt/FACT++/fitsdump $tempFits -c I[0:319] --stat -o $tempFile 2>/dev/null + biascurrentmean=`grep 'Mea' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + timeShift=`echo " $timeShift + 0.000011574 " | bc -l` + iter=`echo " $iter + 1 " | bc -l` + done + + if [ "$biasvoltmean" == "" ] + then + biasvoltmean=0 + fi +# echo "Bias voltage: "$biasvoltmean + if [ "$biascurrentmean" == "" ] + then + biascurrentmean=0 + fi + compare=`echo " $biascurrentmean < 5 " | bc -l` + if [ "$pedestalKey" != "" ] && [ "$drsGainKey" != "" ] && [ "$drsPedestalKey" != "" ] && [ "$biasvoltmean" == "0" ] && [ "$compare" == "1" ] + then + calibFiles[$numCalibs]=$entry + calibs[$numCalibs]=$runnumber + calibDrsFiles[$numCalibs]=$DrsGain + calibDrsPedestalFiles[$numCalibs]=$DrsPedestal + calibsRoi[$numCalibs]=`/opt/FACT++/fitsdump -h -t DrsCalibration $entry 2>/dev/null | grep NROI | grep -v NROITM | grep -E -o '[0-9]{1,4}'` + calibsRoiTM[$numCalibs]=`/opt/FACT++/fitsdump -h -t DrsCalibration $entry 2>/dev/null | grep NTM | grep -E -o '[0-9]{1,4}'` + #echo "Calib file #"$runnumber" found. Roi=${calibsRoi[$numCalibs]} RoiTM="${calibsRoiTM[$numCalibs]} + numCalibs=`expr $numCalibs + 1` + fi + continue + fi + + #if not calib, then look if this is a calibration run (i.e. run that created the drs calib) + correspondingDrs=$rawFolder/$filename.drs.fits.gz + if [ -f $correspondingDrs ] + then + continue + fi + #if regular run, add its run number, and figure out a proper calibration file + data[$numData]=$runnumber + calibFound="false" + partialCalibFound="false" + calibIndex=`expr $numCalibs - 1` + + #if there is no calib available, report error + if [ "$numCalibs" == "0" ] + then + calibFound="notFound" + fi + while [ "$calibFound" == "false" ] + do + roi=`/opt/FACT++/fitsdump -h -t Events $entry 2>/dev/null | grep NROI | grep -v NROITM | grep -E -o '[0-9]{1,4}'` + roiTM=`/opt/FACT++/fitsdump -h -t Events $entry 2>/dev/null | grep NROITM | grep -E -o '[0-9]{1,4}'` + if [ "$roi" == "${calibsRoi[$calibIndex]}" ] && [ "$roiTM" == "${calibsRoiTM[$calibIndex]}" ] + then + if [ "${calibDrsFiles[$calibIndex]}" != "" ] && [ "${calibDrsPedestalFiles[$calibIndex]}" != "" ] + then + calibFound="true" + break + else + if [ "$partialCalibFound" == "false" ] + then + partialCalibFound=$calibIndex + fi + fi + fi + calibIndex=`expr $calibIndex - 1` + if [ "$calibIndex" == "-1" ] + then + calibFound="notFound" + fi + done + + if [ "$partialCalibFound" != "false" ] && [ "$calibFound" != "true" ] + then + calibFound="" + calibIndex=$partialCalibFound + fi + + if [ "$calibFound" == "true" ] + then + calibFound="complete" + fi + + if [ "$calibFound" == "notFound" ] + then + echo "No suitable calibration file could be found for run $runnumber" | tee -a $logfile 2>&1 + calibFileString="NULL" + calibDrsString="NULL" + calibDrsPedestalString="NULL" + else + echo "Found $calibFound calibration "${calibs[$calibIndex]}" for run "$runnumber" with Roi="$roi" and roiTM="$roiTM | tee -a $logfile 2>&1 + calibFileString=${calibFiles[$calibIndex]} + calibDrsString=${calibDrsFiles[$calibIndex]} + calibDrsPedestalString=${calibDrsPedestalFiles[$calibIndex]} + if [ "$calibDrsString" == "" ] + then + calibDrsString="NULL" + fi + if [ "$calibDrsPedestalString" == "" ] + then + calibDrsPedestalString="NULL" + fi + fi + + #File is valid. get its related informations + currentEntry=$entry + rm -f $keywordsFile + rm -f $dataFile + writeHeaderKeysToInputFile + if [ "$doNotDoThisEntry" == "true" ] + then + continue + fi + #Header keys written for raw data. do the same for related aux data + echo "Events BINTABLE URL ../../../"$entry" 1 1 Events" >> $dataFile + if [ "$calibFileString" != "NULL" ] + then + echo "DrsCalibration BINTABLE URL ../../../"$calibFileString" 1 1 Pedestal" >> $dataFile + fi + if [ "$calibDrsString" != "NULL" ] + then + echo "DrsCalibration BINTABLE URL ../../../"$calibDrsString" 1 1 Drs_Gain" >> $dataFile + fi + if [ "$calibDrsPedestalString" != "NULL" ] + then + echo "DrsCalibration BINTABLE URL ../../../"$calibDrsPedestalString" 1 1 Drs_Pedestal" >> $dataFile + fi + echo "DRIVE_CONTROL_TRACKING_POSITION BINTABLE URL ../../../"$trackingFile" 1 1 Tracking_Position" >> $dataFile + echo "FTM_CONTROL_TRIGGER_RATE BINTABLE URL ../../../"$triggerFile" 1 1 Trigger_Rate" >> $dataFile + echo "FTM_CONTROL_STATIC_DATA BINTABLE URL ../../../"$staticFile" 1 1 Thresholds" >> $dataFile + echo "BIAS_CONTROL_VOLTAGE BINTABLE URL ../../../"$voltagesFile" 1 1 Voltages" >> $dataFile + echo "BIAS_CONTROL_CURRENT BINTABLE URL ../../../"$currentFile" 1 1 Currents" >> $dataFile + + if [ "$calibFound" != "notFound" ] + then + #write info to Werner's file + if [ "$runtype" == "custom" ] + then + runtype="custom____________" + fi + if [ "$runtype" == "data" ] + then + runtype="data______________" + fi + if [ "$runtype" == "drs-gain" ] + then + runtype="drs-gain__________" + fi + if [ "$runtype" == "drs-gain-ext" ] + then + runtype="drs-gain-ext______" + fi + if [ "$runtype" == "drs-pedestal" ] + then + runtype="drs-pedestal______" + fi + if [ "$runtype" == "drs-pedestal-ext" ] + then + runtype="drs-pedestal-ext__" + fi + if [ "$runtype" == "drs-time" ] + then + runtype="drs-time__________" + fi + if [ "$runtype" == "drs-time-delay15" ] + then + runtype="drs-time-delay15__" + fi + if [ "$runtype" == "drs-time-delay05" ] + then + runtype="drs-time-delay05__" + fi + if [ "$runtype" == "drs-time-delay20" ] + then + runtype="drs-time-delay20__" + fi + if [ "$runtype" == "drs-time-upshifted" ] + then + runtype="drs-time-upshifted" + fi + if [ "$runtype" == "light-pulser-ext" ] + then + runtype="light-pulser-ext__" + fi + if [ "$runtype" == "n/a" ] + then + runtype="n/a_______________" + fi + if [ "$runtype" == "ped-and-lp-ext" ] + then + runtype="ped-and-lp-ex_____" + fi + if [ "$runtype" == "pedestal" ] + then + runtype="pedestal__________" + fi + targetWerner=$targetFolder"/"$year"/"$month + if [ ! -d $targetWerner ] + then + mkdir -p $targetWerner + fi + wernerFile=$targetWerner"/"$year$month"_001.txt" + if [ ! -f $wernerFile ] + then + echo "creating "$wernerFile + echo "# DRS DAT DD MM YYYY TYPE " >> $wernerFile + fi + echo " "${calibs[$calibIndex]}" "$runnumber" "$day" "$month" "$year" "$runtype >> $wernerFile + fi + + #now add the "other" slow control files, i.e. the ones that are not mandatory for analysis + auxTable="BIAS_CONTROL_NOMINAL" + auxDesc="Bias_Control" + writeExtraAuxFile + auxTable="BIAS_CONTROL_STATE" + auxDesc="Bias_State" + writeExtraAuxFile + auxTable="DATA_LOGGER_FILENAME_NIGHTLY" + auxDesc="Logger_Filename_Night" + writeExtraAuxFile + auxTable="DATA_LOGGER_FILENAME_RUN" + auxDesc="Logger_Filename_Run" + writeExtraAuxFile + auxTable="DATA_LOGGER_NUM_SUBS" + auxDesc="Logger_num_subs" + writeExtraAuxFile + auxTable="DATA_LOGGER_STATE" + auxDesc="Logger_State" + writeExtraAuxFile + auxTable="DATA_LOGGER_STATS" + auxDesc="Logger_Statistics" + writeExtraAuxFile + auxTable="DRIVE_CONTROL_POINTING_POSITION" + auxDesc="Pointing_Position" + writeExtraAuxFile + auxTable="DRIVE_CONTROL_STATE" + auxDesc="Drive_State" + writeExtraAuxFile + auxTable="DRIVE_CONTROL_STATUS" + auxDesc="Drive_Status" + writeExtraAuxFile + auxTable="FAD_CONTROL_DAC" + auxDesc="FAD_DAC" + writeExtraAuxFile + auxTable="FAD_CONTROL_DNA" + auxDesc="FAD_DNA" + writeExtraAuxFile + auxTable="FAD_CONTROL_DRS_CALIBRATION" + auxDesc="FAD_Drs_Calibration" + writeExtraAuxFile + auxTable="FAD_CONTROL_EVENTS" + auxDesc="FAD_Events" + writeExtraAuxFile + auxTable="FAD_CONTROL_FEEDBACK_DATA" + auxDesc="FAD_Feedback_Data" + writeExtraAuxFile + auxTable="FAD_CONTROL_FILE_FORMAT" + auxDesc="FAD_File_Format" + writeExtraAuxFile + auxTable="FAD_CONTROL_FIRMWARE_VERSION" + auxDesc="FAD_Firmware_Version" + writeExtraAuxFile + auxTable="FAD_CONTROL_PRESCALER" + auxDesc="FAD_Prescaler" + writeExtraAuxFile + auxTable="FAD_CONTROL_REFERENCE_CLOCK" + auxDesc="FAD_Reference_Clock" + writeExtraAuxFile + auxTable="FAD_CONTROL_REGION_OF_INTEREST" + auxDesc="FAD_ROI" + writeExtraAuxFile + auxTable="FAD_CONTROL_RUN_NUMBER" + auxDesc="FAD_Run_Number" + writeExtraAuxFile + auxTable="FAD_CONTROL_RUNS" + auxDesc="FAD_Runs" + writeExtraAuxFile + auxTable="FAD_CONTROL_START_RUN" + auxDesc="FAD_Start_Run" + writeExtraAuxFile + auxTable="FAD_CONTROL_STATE" + auxDesc="FAD_State" + writeExtraAuxFile + auxTable="FAD_CONTROL_STATISTICS1" + auxDesc="FAD_Statistics_1" + writeExtraAuxFile + auxTable="FAD_CONTROL_STATISTICS2" + auxDesc="FAD_Statistics_2" + writeExtraAuxFile + auxTable="FAD_CONTROL_STATS" + auxDesc="FAD_Stats" + writeExtraAuxFile + auxTable="FAD_CONTROL_STATUS" + auxDesc="FAD_Status" + writeExtraAuxFile + auxTable="FAD_CONTROL_TEMPERATURE" + auxDesc="FAD_Temperatures" + writeExtraAuxFile + auxTable="FEEDBACK_DEVIATION" + auxDesc="Feedback_Deviation" + writeExtraAuxFile + auxTable="FEEDBACK_STATE" + auxDesc="Feedback_State" + writeExtraAuxFile + auxTable="FSC_CONTROL_HUMIDITY" + auxDesc="FSC_Humidity" + writeExtraAuxFile + auxTable="FSC_CONTROL_STATE" + auxDesc="FSC_State" + writeExtraAuxFile + auxTable="FSC_CONTROL_TEMPERATURE" + auxDesc="FSC_Temperature" + writeExtraAuxFile + auxTable="FTM_CONTROL_COUNTER" + auxDesc="FTM_Counter" + writeExtraAuxFile + auxTable="FTM_CONTROL_DYNAMIC_DATA" + auxDesc="FTM_Dynamic_Data" + writeExtraAuxFile + auxTable="FTM_CONTROL_FTU_LIST" + auxDesc="FTM_FTU_List" + writeExtraAuxFile + auxTable="FTM_CONTROL_PASSPORT" + auxDesc="FTM_Passeport" + writeExtraAuxFile + auxTable="FTM_CONTROL_STATE" + auxDesc="FTM_State" + writeExtraAuxFile + auxTable="MAGIC_WEATHER_DATA" + auxDesc="MAGIC_Weather_Data" + writeExtraAuxFile + auxTable="MAGIC_WEATHER_STATE" + auxDesc="MAGIC_Weather_State" + writeExtraAuxFile + auxTable="MCP_STATE" + auxDesc="MCP_State" + writeExtraAuxFile + auxTable="RATE_CONTROL_STATE" + auxDesc="Rate_Control_State" + writeExtraAuxFile + #create the fits file + targetNormal=$targetFolder"/"$year"/"$month"/"$day + if [ ! -d $targetNormal ] + then + mkdir -p $targetNormal + fi + targetFile=$targetNormal"/"$year$month$day"_"$runnumber"_001.fits" + if [ -f $targetFile ] + then + rm $targetFile + fi + fcreate $colDescFile $dataFile $targetFile "headfile="$keywordsFile + echo "Created "$targetFile | tee -a $logfile 2>&1 + +done Index: trunk/DataCheck/Archive/auxIngest.sh =================================================================== --- trunk/DataCheck/Archive/auxIngest.sh (revision 12750) +++ trunk/DataCheck/Archive/auxIngest.sh (revision 12871) @@ -3,8 +3,9 @@ sourceFolder=$1 destFolder=$2 +suffix=$3 -if [ "$#" != "2" ] +if [ "$#" != "3" ] then - echo "Please specify source and dest folders. Aborting" + echo "Please specify source and dest folders and suffix. Aborting" exit fi @@ -85,5 +86,5 @@ if [ -a $targetFile ] then - echo "File $targetFile already exist. Skipping it" >> report.txt + echo "File $targetFile already exist. Skipping it" >> report_$suffix.txt continue fi @@ -98,13 +99,13 @@ # then - repairAuxFile.sh $targetFile ENDerrors.txt MJDerror.txt report.txt processErrors.txt + repairAuxFile.sh $targetFile ENDerrors_$suffix.txt MJDerror_$suffix.txt report_$suffix.txt processErrors_$suffix.txt if [ -a $targetFile ] then - fixAuxKeyWords.sh $targetFile reportTwo.txt processErrors.txt + fixAuxKeyWords.sh $targetFile reportTwo_$suffix.txt processErrors_$suffix.txt result=`fverify $targetFile 2>/dev/null | grep '0 error(s)'` if [ "$result" == "" ] then - echo "$targetFile" >> stillHasProblems.txt + echo "$targetFile" >> stillHasProblems_$suffix.txt rm $targetFile fi Index: trunk/DataCheck/Archive/checkForHeaderUpdatesAux.sh =================================================================== --- trunk/DataCheck/Archive/checkForHeaderUpdatesAux.sh (revision 12871) +++ trunk/DataCheck/Archive/checkForHeaderUpdatesAux.sh (revision 12871) @@ -0,0 +1,134 @@ +#!/bin/bash + +month=$1 + +if [ "$month" == "" ] +then + echo "Please give the month to look for as parameter" + exit +fi + +archive="/archive/fact/rev_1/aux/2011/"$month +tempFile="tempFile$month.txt" +tempFile2="tempFile2$month.txt" +entries=`find $archive -type f -name '*.fits' | sort` + +for entry in ${entries[@]} +do + if [ -f $tempFile ] + then + rm $tempFile + fi + if [ -f $tempFile2 ] + then + rm $tempFile2 + fi + + tstartf=1 + tstartf2=2 + + result=`/opt/FACT++/fitsdump -h $entry > $tempFile` + origEntry=`echo $entry | sed -e 's/archive\/fact\/rev_1/data00\/fact-construction/g'` + if [ ! -f $origEntry ] + then + echo $entry >> headerReportAux_$month.txt + echo "Name was changed" >> headerReportAux_$month.txt + continue + fi + result2=`/opt/FACT++/fitsdump -h $origEntry > $tempFile2` + + tstartf=`grep 'TSTARTF' $tempFile | grep -E -o '0\.[0-9]{7}' ` + tstartf2=`grep 'TSTARTF' $tempFile2 | grep -E -o '0\.[0-9]{7}' ` + if [ "$tstartf" == "" ] + then + tstartf=`grep 'TSTARTF' $tempFile | grep -E -o '0'` + fi + if [ "$tstartf2" == "" ] + then + tstartf2=`grep 'TSTARTF' $tempFile2 | grep -E -o '0'` + fi + + tstarti=`grep 'TSTARTI' $tempFile | grep -E -o '[0-9]{5}' ` + tstarti2=`grep 'TSTARTI' $tempFile2 | grep -E -o '[0-9]{5}' ` + tstopf=`grep 'TSTOPF' $tempFile | grep -E -o '0\.[0-9]{7}'` + tstopf2=`grep 'TSTOPF' $tempFile2 | grep -E -o '0\.[0-9]{7}'` + if [ "$tstopf" == "" ] + then + tstopf=`grep 'TSTOPF' $tempFile | grep -E -o '0'` + fi + if [ "$tstopf2" == "" ] + then + tstopf2=`grep 'TSTOPF' $tempFile2 | grep -E -o '0'` + fi + tstopi=`grep 'TSTOPI' $tempFile | grep -E -o '[0-9]{5}'` + tstopi2=`grep 'TSTOPI' $tempFile2 | grep -E -o '[0-9]{5}'` + dateobs=`grep 'DATE-OBS' $tempFile | grep -E -o 201[1-3]-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{4}` + dateobs2=`grep 'DATE-OBS' $tempFile2 | grep -E -o 201[1-3]-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{4}` + dateend=`grep 'DATE-END' $tempFile | grep -E -o 201[1-3]-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{4} ` + dateend2=`grep 'DATE-END' $tempFile2 | grep -E -o 201[1-3]-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{4}` + telescop=`grep 'TELESCOP' $tempFile` + telescop2=`grep 'TELESCOP' $tempFile2` + package=`grep 'PACKAGE' $tempFile` + package2=`grep 'PACKAGE' $tempFile2` + origin=`grep 'ORIGIN' $tempFile` + origin2=`grep 'ORIGIN' $tempFile2` + timeunit=`grep 'TIMEUNIT' $tempFile` + timeunit2=`grep 'TIMEUNIT' $tempFile2` + mjdref=`grep -E '## +MJDREF' $tempFile` + mjdref2=`grep -E '## +MJDREF' $tempFile2` + timesys=`grep 'TIMESYS' $tempFile` + timesys2=`grep 'TIMESYS' $tempFile2` + + echo $entry >> headerReportAux_$month.txt + echo $entry + + if [ "$tstarti" != "$tstarti2" ] + then + echo "TSTARTI. new: "$tstarti" old: "$tstarti2 >> headerReportAux_$month.txt + fi + if [ "$tstartf" != "$tstartf2" ] + then + echo "TSTARTF. new: "$tstartf" old: "$tstartf2 >> headerReportAux_$month.txt + fi + if [ "$tstopi" != "$tstopi2" ] + then + echo "TSTOPI. new: "$tstopi" old: "$tstopi2 >> headerReportAux_$month.txt + fi + if [ "$tstopf" != "$tstopf2" ] + then + echo "TSTOPF. new: "$tstopf" old: "$tstopf2 >> headerReportAux_$month.txt + fi + if [ "$dateobs" != "$dateobs2" ] + then + echo "DATEOBS. new: "$dateobs" old: "$dateobs2 >> headerReportAux_$month.txt + fi + if [ "$dateend" != "$dateend2" ] + then + echo "DATEEND. new: "$dateend" old: "$dateend2 >> headerReportAux_$month.txt + fi + if [ "$telescop" != "$telescop2" ] + then + echo "TELESCOP. new: "$telescop" old: "$telescop2 >> headerReportAux_$month.txt + fi + if [ "$package" != "$package2" ] + then + echo "PACKAGE. new: "$package" old: "$package2 >> headerReportAux_$month.txt + fi + if [ "$origin" != "$origin2" ] + then + echo "ORIGIN. new: "$origin" old: "$origin2 >> headerReportAux_$month.txt + fi + if [ "$timeunit" != "$timeunit2" ] + then + echo "TIMEUNIT. new: "$timeunit" old: "$timeunit2 >> headerReportAux_$month.txt + fi + if [ "$mjdref" != "$mjdref2" ] + then + echo "MJDREF. new: "$mjdref" old: "$mjdref2 >> headerReportAux_$month.txt + fi + if [ "$timesys" != "$timesys2" ] + then + echo "TIMESYS. new: "$timesys" old: "$timesys2 >> headerReportAux_$month.txt + fi +done + Index: trunk/DataCheck/Archive/checkForHeaderUpdatesRaw.sh =================================================================== --- trunk/DataCheck/Archive/checkForHeaderUpdatesRaw.sh (revision 12871) +++ trunk/DataCheck/Archive/checkForHeaderUpdatesRaw.sh (revision 12871) @@ -0,0 +1,139 @@ +#!/bin/bash + +month=$1 + +if [ "$month" == "" ] +then + echo "Please give the month to look for as parameter" + exit +fi + +archive="/archive/fact/rev_1/raw/2011/"$month +tempFile="tempFile$month.txt" +tempFile2="tempFile2$month.txt" +entries=`find $archive -type f -name '*.fits.gz' | sort` + +for entry in ${entries[@]} +do + res=`echo $entry | grep 'drs'` + if [ "$res" != "" ] + then + continue + fi + if [ -f $tempFile ] + then + rm $tempFile + fi + if [ -f $tempFile2 ] + then + rm $tempFile2 + fi + + tstartf=1 + tstartf2=2 + + result=`/opt/FACT++/fitsdump -h $entry > $tempFile` + origEntry=`echo $entry | sed -e 's/archive\/fact\/rev_1/data00\/fact-construction/g'` + if [ ! -f $origEntry ] + then + echo $entry >> headerReportRaw_$month.txt + echo "Name was changed" >> headerReportRaw_$month.txt + continue + fi + result2=`/opt/FACT++/fitsdump -h $origEntry > $tempFile2` + + tstartf=`grep 'TSTARTF' $tempFile | grep -E -o '0\.[0-9]{7}' ` + tstartf2=`grep 'TSTARTF' $tempFile2 | grep -E -o '0\.[0-9]{7}' ` + if [ "$tstartf" == "" ] + then + tstartf=`grep 'TSTARTF' $tempFile | grep -E -o '0'` + fi + if [ "$tstartf2" == "" ] + then + tstartf2=`grep 'TSTARTF' $tempFile2 | grep -E -o '0'` + fi + + tstarti=`grep 'TSTARTI' $tempFile | grep -E -o '[0-9]{5}' ` + tstarti2=`grep 'TSTARTI' $tempFile2 | grep -E -o '[0-9]{5}' ` + tstopf=`grep 'TSTOPF' $tempFile | grep -E -o '0\.[0-9]{7}'` + tstopf2=`grep 'TSTOPF' $tempFile2 | grep -E -o '0\.[0-9]{7}'` + if [ "$tstopf" == "" ] + then + tstopf=`grep 'TSTOPF' $tempFile | grep -E -o '0'` + fi + if [ "$tstopf2" == "" ] + then + tstopf2=`grep 'TSTOPF' $tempFile2 | grep -E -o '0'` + fi + tstopi=`grep 'TSTOPI' $tempFile | grep -E -o '[0-9]{5}'` + tstopi2=`grep 'TSTOPI' $tempFile2 | grep -E -o '[0-9]{5}'` + dateobs=`grep 'DATE-OBS' $tempFile | grep -E -o 201[1-3]-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{4}` + dateobs2=`grep 'DATE-OBS' $tempFile2 | grep -E -o 201[1-3]-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{4}` + dateend=`grep 'DATE-END' $tempFile | grep -E -o 201[1-3]-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{4} ` + dateend2=`grep 'DATE-END' $tempFile2 | grep -E -o 201[1-3]-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{4}` + telescop=`grep 'TELESCOP' $tempFile` + telescop2=`grep 'TELESCOP' $tempFile2` + package=`grep 'PACKAGE' $tempFile` + package2=`grep 'PACKAGE' $tempFile2` + origin=`grep 'ORIGIN' $tempFile` + origin2=`grep 'ORIGIN' $tempFile2` + timeunit=`grep 'TIMEUNIT' $tempFile` + timeunit2=`grep 'TIMEUNIT' $tempFile2` + mjdref=`grep -E '## +MJDREF' $tempFile` + mjdref2=`grep -E '## +MJDREF' $tempFile2` + timesys=`grep 'TIMESYS' $tempFile` + timesys2=`grep 'TIMESYS' $tempFile2` + + echo $entry >> headerReportRaw_$month.txt + echo $entry + + if [ "$tstarti" != "$tstarti2" ] + then + echo "TSTARTI. new: "$tstarti" old: "$tstarti2 >> headerReportRaw_$month.txt + fi + if [ "$tstartf" != "$tstartf2" ] + then + echo "TSTARTF. new: "$tstartf" old: "$tstartf2 >> headerReportRaw_$month.txt + fi + if [ "$tstopi" != "$tstopi2" ] + then + echo "TSTOPI. new: "$tstopi" old: "$tstopi2 >> headerReportRaw_$month.txt + fi + if [ "$tstopf" != "$tstopf2" ] + then + echo "TSTOPF. new: "$tstopf" old: "$tstopf2 >> headerReportRaw_$month.txt + fi + if [ "$dateobs" != "$dateobs2" ] + then + echo "DATEOBS. new: "$dateobs" old: "$dateobs2 >> headerReportRaw_$month.txt + fi + if [ "$dateend" != "$dateend2" ] + then + echo "DATEEND. new: "$dateend" old: "$dateend2 >> headerReportRaw_$month.txt + fi + if [ "$telescop" != "$telescop2" ] + then + echo "TELESCOP. new: "$telescop" old: "$telescop2 >> headerReportRaw_$month.txt + fi + if [ "$package" != "$package2" ] + then + echo "PACKAGE. new: "$package" old: "$package2 >> headerReportRaw_$month.txt + fi + if [ "$origin" != "$origin2" ] + then + echo "ORIGIN. new: "$origin" old: "$origin2 >> headerReportRaw_$month.txt + fi + if [ "$timeunit" != "$timeunit2" ] + then + echo "TIMEUNIT. new: "$timeunit" old: "$timeunit2 >> headerReportRaw_$month.txt + fi + if [ "$mjdref" != "$mjdref2" ] + then + echo "MJDREF. new: "$mjdref" old: "$mjdref2 >> headerReportRaw_$month.txt + fi + if [ "$timesys" != "$timesys2" ] + then + echo "TIMESYS. new: "$timesys" old: "$timesys2 >> headerReportRaw_$month.txt + fi +done + Index: trunk/DataCheck/Archive/checkRawFileIntegrityAndContent.sh =================================================================== --- trunk/DataCheck/Archive/checkRawFileIntegrityAndContent.sh (revision 12871) +++ trunk/DataCheck/Archive/checkRawFileIntegrityAndContent.sh (revision 12871) @@ -0,0 +1,100 @@ +#!/bin/bash + +sourceFolder=$1 +archiveFolder=$2 +suffix=$3 + +#are the arguments subfolders of /data00/fact-construction and /archive/fact ? +sourceOk=`echo $sourceFolder | sed "s/data00\/fact-construction\/raw/OK/"` +archiveOk=`echo $archiveFolder | sed "s/archive\/fact\/rev_1\/raw/OK/"` + +sourceOk=`echo $sourceOk | grep OK` +archiveOk=`echo $archiveOk | grep OK` + +if [ "$sourceOk" == "" ] +then + echo "source folder not good." + exit +fi + +if [ "$archiveOk" == "" ] +then + echo "archive folder not good" + exit +fi + +uncompressedFolder=`echo $sourceFolder | sed "s/data00/data03/"` + +#do the listing of the files to be checked +entries=`find $sourceFolder -type f -name '*.fits.gz' | sort` + +for entry in ${entries[@]} +do + echo "$entry" >> GGfileSequence$suffix.txt + + uncompressedEntry=`echo $entry | sed "s/data00/data03/"` + uncompressedEntry=`echo $uncompressedEntry | sed "s/.gz//"` + archiveEntry=`echo $entry | sed "s/data00\/fact-construction/archive\/fact\/rev_1/"` + corruptedFile="0" + if [ -f $uncompressedEntry ] + then + result=`fverify $uncompressedEntry 2>/dev/null | grep '0 error(s)'` + if [ "$result" == "" ] + then + if [ -f $archiveEntry ] + then + echo "$entry" >> GGcorruptFiles$suffix.txt + fi + corruptedFile="1" + fi + else + echo "$entry" >> GGmissingUncompressed$suffix.txt + echo "$entry is missing uncompressed" + continue + fi + + if [ -f $archiveEntry ] + then + result=`/home_nfs/isdc/lyard/FACT++/fitsCompare $entry $archiveEntry` + if [ "$result" == "0" ] + then + echo "$entry" >> GGidenticalFiles$suffix.txt + echo "$entry is fine" + continue + fi + if [ "$result" == "1" ] + then + echo "$entry" >> GGdifferentFiles$suffix.txt + echo "$entry differs from its archived version" + continue + fi + if [ "$result" == "2" ] + then + echo "$entry" >> GGunexpectedErrors$suffix.txt + echo "$entry encountered an unexpected error" + continue + fi + if [ "$result" == "3" ] + then + echo "$entry" >> GGfineWithMoreRows$suffix.txt + echo "$entry is fine (with more rows !" + continue + fi + if [ "$corryuptedFile" == "1"] + then + echo "$entry is corrupted and produced unkown error $result" + else + echo "$entry produced unkown error $result" + fi + echo "$entry $result" >> GGunknownErrors$suffix.txt + else + if [ "$corruptedFile" == "1" ] + then + echo "$entry" >> GGreallymessedup$suffix.txt + echo "$entry is really messed up" + else + echo "$entry" >> GGmissingArchive$suffix.txt + echo "$entry is missing archive" + fi + fi +done Index: trunk/DataCheck/Archive/correctFileName.cpp =================================================================== --- trunk/DataCheck/Archive/correctFileName.cpp (revision 12871) +++ trunk/DataCheck/Archive/correctFileName.cpp (revision 12871) @@ -0,0 +1,111 @@ +#include +#include + +using namespace std; + +bool isNumber(char c) +{ + switch (c) { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + return true; + break; + default: + return false; + }; +} + +int main(int argc, char** argv) +{ + if (argc != 2) + return 0; + + string c(argv[1]); + + //get the year, month and day from the path. + string year, month, day; + string runnumber; + string serviceName; + int count = c.size()-5; + + while (count >= 0) + { + if (c[count+0] == '2' && + c[count+1] == '0' && + c[count+2] == '1' && + c[count+3] == '1' && + c[count+4] == '/') + { + year = "2011"; + month = c.substr(count+5, 2); + day = c.substr(count+8, 2); + break; + } + count--; + } + + //separate the runnumber from the rest of the filename + count = c.size()-6; + while (count >= 0) + { + if (c[count] == '_') + { + if (isNumber(c[count+1]) && + isNumber(c[count+2]) && + isNumber(c[count+3])) + { + runnumber=c.substr(count+1, 3); + break; + } + } + else + { + if (c[count+0] != '0' && + c[count+1] == '0' && + c[count+2] == '0' && + c[count+3] == '0' && + c[count+4] == '0' && + c[count+5] == '0') + { + runnumber=c.substr(count+6, 3); + break; + } + } + count--; + } + + //figure out the service name + count = c.size()-3; + while (count > 0) + { + if ((c[count] == '_' || c[count] == '.') && + !isNumber(c[count+1]) && + isNumber(c[count-1])) + { + serviceName = c.substr(count+1, c.size()-(count+6)); + } + count--; + } + if (serviceName == "fits") + serviceName = ""; + + + cout << year << month << day; + if (runnumber != "") cout << "_"; + cout << runnumber; + if (serviceName != "") cout << "."; + cout << serviceName << ".fits"; +// cout << "year: " << year << " month: " << month << " day: " << day << endl; +// cout << "run number: " << runnumber << " service name: " << serviceName << endl; + +return 0; + +} Index: trunk/DataCheck/Archive/fitsCompare.cpp =================================================================== --- trunk/DataCheck/Archive/fitsCompare.cpp (revision 12871) +++ trunk/DataCheck/Archive/fitsCompare.cpp (revision 12871) @@ -0,0 +1,214 @@ +//compile with: +//g++44 -std=c++0x -o fitsCompare -DHAVE_ZLIB=1 -lz dataChecker.cpp +#include +#include +#include +#include + +using namespace std; +//#include + +#include "externals/fits.h" + +using namespace std; + + /* + * Usage: program-name + * + * + * + * + * + * + */ + + fits* file1; + fits* file2; + char* file1Data; + char* file2Data; + + int customReturn(int code) + { + if (file1) + { + file1->close(); + delete file1; + } + if (file2) + { + file2->close(); + delete file2; + } + if (file1Data) + { + delete[] file1Data; + } + if (file2Data) + { + delete[] file2Data; + } + exit(code); + } + +int main(int argc, char** argv) +{ + if (argc < 3) + return -1; + + bool verbose=false; + + if (argc > 3 && + !strcmp(argv[3], "-v")) + verbose=true; + + string filename1(argv[1]); + string filename2(argv[2]); + file1=NULL; + file2=NULL; + file1Data=NULL; + file2Data=NULL; + try + { + file1 = new fits(filename1); + file2 = new fits(filename2); + } + catch (std::runtime_error e) + { + if (verbose) + cout << "Could not open at least one of the two files." << endl; + else + cout << 2 << endl; + return -1; + } + + //get the columns in the file. + fits::Table::Columns columns1=file1->GetColumns(); + fits::Table::Columns columns2=file2->GetColumns(); + + if (columns1.size() != columns2.size()) + { + if (verbose) + cout << "Different number of columns" << endl; + else + cout << "1" << endl; + customReturn(-1); + } + + long totalSize=0; + for (auto it=columns1.begin(), jt=columns2.begin(); it != columns1.end(); it++, jt++) + { + if (it->first != jt->first) + { + if (verbose) + cout << "Different column names" << endl; + else + cout << "1" << endl; + customReturn(-1); + } + if ((it->second.offset != jt->second.offset) || + (it->second.num != jt->second.num) || + (it->second.size != jt->second.size) || + (it->second.type != jt->second.type) || + (it->second.unit != jt->second.unit)) + { + if (verbose) + cout << "Different column def" << endl; + else + cout << "1" << endl; + customReturn(-1); + } + + totalSize += it->second.size * it->second.num; + + } + + char* file1Data = new char[totalSize]; + char* file2Data = new char[totalSize]; + + if ((sizeof(long) != 8) || + (sizeof(int) != 4) || + (sizeof(short) != 2)) + { + if (verbose) + cout << "OS IS NOT SUITABLE (32bits ?) please use a 64 bits system" << endl; + else + cout << "2" << endl; + customReturn(-1); + } + + for (auto it=columns1.begin(); it != columns1.end(); it++) + { + switch (it->second.size) { + case 1: + file1->SetPtrAddress(it->first, &file1Data[it->second.offset], it->second.num); + file2->SetPtrAddress(it->first, &file2Data[it->second.offset], it->second.num); + break; + case 2: + file1->SetPtrAddress(it->first, (short*)(&file1Data[it->second.offset]), it->second.num); + file2->SetPtrAddress(it->first, (short*)(&file2Data[it->second.offset]), it->second.num); + break; + case 4: + file1->SetPtrAddress(it->first, (int*)(&file1Data[it->second.offset]), it->second.num); + file2->SetPtrAddress(it->first, (int*)(&file2Data[it->second.offset]), it->second.num); + break; + case 8: + file1->SetPtrAddress(it->first, (long*)(&file1Data[it->second.offset]), it->second.num); + file2->SetPtrAddress(it->first, (long*)(&file2Data[it->second.offset]), it->second.num); + break; + default: + if (verbose) + cout << "Unkown column element size: " << it->second.size << endl; + else + cout << "2" << endl; + customReturn(-1); + }; + } + + int numRows1 = file1->GetInt("NAXIS2"); + int numRows2 = file2->GetInt("NAXIS2"); + if (numRows1 > numRows2) + { + if (verbose) + cout << "looks like the files has different number of rows: " << numRows1 << " vs " << numRows2 << endl; + else + cout << "1" << endl; + customReturn(0); + } + int row=0; + if (verbose) + cout << "files have " << numRows1 << " rows" << endl << endl; + while (file1->GetNextRow() && + file2->GetNextRow() && + row < numRows1) + { + if (verbose) + { + cout << "\rrow: " << row; + cout.flush(); + } + for (int i=0;i/dev/null` +result=`/home_nfs/isdc/lyard/FACT++/fitsdump $file -h -o $tempFile 2>/dev/null` if [ -a $tempFile ] @@ -42,5 +42,5 @@ #retrieve the start and stop time from the data itself -result=`/home/isdc/lyard/FACT++/fitsdump $file -c UnixTimeUTC --minmax --nozero -o $tempFile 2>/dev/null` +result=`/home_nfs/isdc/lyard/FACT++/fitsdump $file -c UnixTimeUTC --minmax --nozero -o $tempFile 2>/dev/null` if [ -a $tempFile ] @@ -56,5 +56,5 @@ else #let's give it a shot with PCTime - result=`/home/isdc/lyard/FACT++/fitsdump $file -c PCTime --minmax --nozero -o $tempFile 2>/dev/null` + result=`/home_nfs/isdc/lyard/FACT++/fitsdump $file -c PCTime --minmax --nozero -o $tempFile 2>/dev/null` if [ -a $tempFile ] then @@ -124,12 +124,23 @@ fi #adapt the start and stop to the mjdref (it is raw unix time in the data) -tstarti2=`echo "$tstarti2 - $mjdref" | bc -l` -tstopi2=`echo "$tstopi2 - $mjdref" | bc -l` -if [ "$tstarti2" != "$tstarti" ] || [ "$tstartf2" != "$tstartf" ] +if [ "$tstarti2" != "0" ] && [ "$tstarti2" != "" ] +then + tstarti2=`echo "$tstarti2 - $mjdref" | bc -l` + tstopi2=`echo "$tstopi2 - $mjdref" | bc -l` +else + tstarti2=0 + tstopi2=0 +fi + +#give latitude for 10-6 precision in tstart and tstop +tfcompare=`echo $tstartf | grep -E -o '0\.[0-9]{6}'` +tfcompare2=`echo $tstartf2 | grep -E -o '0\.[0-9]{6}'` + +if [ "$tstarti2" != "$tstarti" ] || [ "$tfcompare" != "$tfcompare2" ] then echo "TSTARTI "$tstarti2" / Time when first event received (integral part)" >> $tempFile echo "TSTARTF "$tstartf2" / Time when first event received (fractional part)" >> $tempFile date_obs2=`echo "$tstarti2 + $tstartf2 + $mjdref" | bc -l` - date_obs2=`/home/isdc/lyard/FACT++/MjDtoISO $date_obs2` + date_obs2=`/home_nfs/isdc/lyard/FACT++/MjDtoISO $date_obs2` echo "DATE-OBS "$date_obs2" / Time when first event was received" >> $tempFile modified="true" @@ -137,10 +148,15 @@ fi -if [ "$tstopi2" != "$tstopi" ] || [ "$tstopf2" != "$tstopf" ] +tfcompare=`echo $tstopf | grep -E -o '0\.[0-9]{6}'` +tfcompare2=`echo $tstopf2 | grep -E -o '0\.[0-9]{6}'` + +if [ "$tstopi2" != "$tstopi" ] || [ "$tfcompare" != "$tfcompare2" ] then + + echo "first: $tfcompare ||| second: $tfcompare2||" echo "TSTOPI "$tstopi2" / Time when last event received (integral part)" >> $tempFile echo "TSTOPF "$tstopf2" / Time when last event received (fractional part)" >> $tempFile date_end2=`echo "$tstopi2 + $tstopf2 + $mjdref" | bc -l` - date_end2=`/home/isdc/lyard/FACT++/MjDtoISO $date_end2` + date_end2=`/home_nfs/isdc/lyard/FACT++/MjDtoISO $date_end2` echo "DATE-END "$date_end2" / Time when last event was received" >> $tempFile modified="true" @@ -150,5 +166,8 @@ if [ "$modified" == "true" ] then - echo "INGEST v0.1 Version of Etienne ingest script" >> $tempFile + echo "INGEST v0.2 Version of Etienne ingest script" >> $tempFile + echo $file" header has been modified" >> $reportFile +else + echo $file" header has NOT been modified" >> $reportFile fi Index: trunk/DataCheck/Archive/rawIngest.sh =================================================================== --- trunk/DataCheck/Archive/rawIngest.sh (revision 12750) +++ trunk/DataCheck/Archive/rawIngest.sh (revision 12871) @@ -4,7 +4,7 @@ destFolder=$2 -if [ "$#" != "2" ] +if [ "$#" != "3" ] then - echo "Please specify source and dest folders. Aborting" + echo "Please specify source and dest folders. and an identifier for the log files Aborting" exit fi @@ -19,4 +19,10 @@ then echo "Dest folder is empty. Aborting" + exit +fi + +if [ $3 == "" ] +then + echo "Identifier for log files empty. Aborting" exit fi @@ -61,4 +67,5 @@ destFolder=${destFolder%/} echo "Will start ingesting files from "$sourceFolder" to "$destFolder +echo "Will start ingesting files from "$sourceFolder" to "$destFolder >> Rawreport$3.txt #list all the files in sourceFolder, and copy then with the same structure to destfolder @@ -87,5 +94,5 @@ if [ -a $targetFile".gz" ] then - echo "File $targetFile already exist. Skipping it" >> Rawreport.txt + echo "File $targetFile already exist. Skipping it" >> Rawreport$3.txt continue fi @@ -100,16 +107,16 @@ # then - repairRawFile.sh $interFile RawENDerrors.txt RawMJDerror.txt Rawreport.txt RawprocessErrors.txt + repairRawFile.sh $interFile RawENDerrors$3.txt RawMJDerror$3.txt Rawreport$3.txt RawprocessErrors$3.txt if [ -a $interFile ] then - fixRawKeyWords.sh $interFile RawreportTwo.txt RawprocessErrors.txt + fixRawKeyWords.sh $interFile RawreportTwo$3.txt RawprocessErrors$3.txt result=`fverify $interFile 2>/dev/null | grep '0 error(s)'` if [ "$result" == "" ] then - echo "$interFile" >> RawstillHasProblems.txt + echo "$interFile" >> RawstillHasProblems$3.txt rm $interFile else - gzip $interFile + gzip -1 $interFile cp $interFile".gz" $targetFile".gz" rm $interFile".gz" Index: trunk/DataCheck/Archive/repairRawFile.sh =================================================================== --- trunk/DataCheck/Archive/repairRawFile.sh (revision 12750) +++ trunk/DataCheck/Archive/repairRawFile.sh (revision 12871) @@ -30,5 +30,5 @@ if [ "$result" != "" ] then - mjdref=`/home/isdc/lyard/FACT++/fitsdump $entry -h 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]+'` + mjdref=`/home_nfs/isdc/lyard/FACT++/fitsdump $entry -h 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]+'` if [ "$mjdref" != 40587 ] then @@ -53,5 +53,5 @@ filesize=`stat -c%s $entry` # echo $headerSize" "$filesize - numrows=`/home/isdc/lyard/FACT++/fitsdump $entry -h 2>/dev/null | grep 'NAXIS2' | awk '{ print $4 }' | grep -E -o '[0-9]+'` + numrows=`/home_nfs/isdc/lyard/FACT++/fitsdump $entry -h 2>/dev/null | grep 'NAXIS2' | awk '{ print $4 }' | grep -E -o '[0-9]+'` if [ "$numrows" == "" ] then @@ -60,5 +60,5 @@ exit fi - rowWidth=`/home/isdc/lyard/FACT++/fitsdump $entry -h 2>/dev/null | grep 'NAXIS1' | awk '{ print $4 }' | grep -E -o '[0-9]+'` + rowWidth=`/home_nfs/isdc/lyard/FACT++/fitsdump $entry -h 2>/dev/null | grep 'NAXIS1' | awk '{ print $4 }' | grep -E -o '[0-9]+'` totSize=`echo " $headerSize + $numrows * $rowWidth " | bc -l` @@ -67,7 +67,11 @@ # echo $headerSize" "$numrows" "$rowWidth" "$totSize" "$fitsSize # echo " FileSize: "$filesize" should be "$fitsSize - truncate -s $totSize $entry 2>/dev/null - truncate -s $fitsSize $entry 2>/dev/null - echo "Resized $entry" >> $reportFile + if [ $filesize > $fitsSize ] + then + truncate -s $totSize $entry 2>/dev/null + truncate -s $fitsSize $entry 2>/dev/null + echo "Resized $entry" >> $reportFile + fi + fi fi