Index: /trunk/DataCheck/Archive/ENDfixer.cxx =================================================================== --- /trunk/DataCheck/Archive/ENDfixer.cxx (revision 12750) +++ /trunk/DataCheck/Archive/ENDfixer.cxx (revision 12750) @@ -0,0 +1,133 @@ +/* + * fitsHacker.cc + * + * Created on: Sep 8, 2011 + * Author: lyard + */ + +#include +#include +#include +#include + +using namespace std; +/* + * Usage: fitsHacker (optionnal) + * + * + * + */ + +enum ModeT {seekingHDU, + foundHDU, + fixedEND, + reachedHeaderEnd}; + +int main(int argc, char** argv) +{ + if (argc < 2) + return 0; + +/* ENDfixer */ + fstream file(argv[1]); + + char c[81]; + c[80] = 0; + int seeking=0; + + ModeT mode = seekingHDU; + + bool reallyFixedEnd = false; + int endAddress = 0; + + while (mode != fixedEND) + { + file.read(c, 80); + if (!file.good()) { + cout << 0; + return 0; + } + string str(c); +// cout << c << endl; + if (str.substr(0, 9) == "XTENSION=") + mode = foundHDU; + + if (mode == foundHDU && str=="END ") + { + mode = fixedEND; + endAddress = seeking; +// cout << "found END at " << endAddress << endl; + } + if (mode == foundHDU && str ==" ") + { + file.seekp(seeking); + file.put('E'); + file.put('N'); + file.put('D'); + mode = fixedEND; + reallyFixedEnd = true; + endAddress = seeking; +// cout << "added END at " << endAddress << endl; + } + + seeking+=80; + } + + file.seekp(seeking-1); + while (mode != reachedHeaderEnd) + { + file.read(c, 80); + if (!file.good()) { + cout << 0; + return 0; + } + string str(c); + + if (str ==" ") + seeking+=80; + else + mode = reachedHeaderEnd; + } + + file.close(); + + if (seeking % 2880 != 0) + { + cout << "Error: header length not acceptable" << endl; + return 0; + } + + if (((seeking - endAddress)/80) > 36) + { + cout << "Error: too much header space after END keyword" << endl; + return 0; + } + + cout << seeking; + + return seeking; + +/* FITS HACKER + file.get(data, shift); + + for (int i=0;i +#include +#include +#include +#include + +using namespace std; + + +//usage RowChecker +int main(int argc, char** argv) +{ + + if (argc < 6) + return 0; + + fstream file(argv[1]); + + int headLen = atoi(argv[2]); + int lineWidth = atoi(argv[3]); + double mjdRef = atof(argv[4]); + int numLines = atoi(argv[5]); + + int totalBytes = headLen; + file.seekp(headLen); + + char* buf = new char[lineWidth]; + + double currentTime = 0; + char timeBuf[16]; + int realNumRows = 0; + + while (file.read(buf, lineWidth)) + { + timeBuf[0] = buf[7]; + timeBuf[1] = buf[6]; + timeBuf[2] = buf[5]; + timeBuf[3] = buf[4]; + timeBuf[4] = buf[3]; + timeBuf[5] = buf[2]; + timeBuf[6] = buf[1]; + timeBuf[7] = buf[0]; + currentTime = reinterpret_cast(timeBuf)[0]; + + if (realNumRows >= numLines) + { + if (currentTime + mjdRef > 60000 || currentTime + mjdRef < 10000) + break; + if (currentTime + mjdRef > 20000 && currentTime + mjdRef < 50000) + break; + } +//fix the time column if required. + if (currentTime > 50000 && currentTime < 60000) + { + currentTime -= 40587; + reinterpret_cast(timeBuf)[0] = currentTime; + file.seekp(totalBytes); + file.put(timeBuf[7]); + file.put(timeBuf[6]); + file.put(timeBuf[5]); + file.put(timeBuf[4]); + file.put(timeBuf[3]); + file.put(timeBuf[2]); + file.put(timeBuf[1]); + file.put(timeBuf[0]); + file.seekp(totalBytes + lineWidth); + } + + realNumRows++; + totalBytes += lineWidth; + } + //now update the number of lines of the file + file.close(); + file.open(argv[1]); + file.seekp(2880); + delete[] buf; + buf = new char[81]; + buf[80] = 0; + bool changeDone = false; + int seeked = 2880; + if (realNumRows == numLines) + changeDone = true; + + while (file.good() && !changeDone) + { + file.read(buf, 80); + string str(buf); + + if (str.substr(0,9) == "NAXIS2 =") + { + ostringstream ss; + ss << realNumRows; + file.seekp(seeked + 30 - ss.str().size()); + for (int i=0;i/dev/null + if [ -d $2"/etiennetest" ] + then + rm -rf $2"/etiennetest" + destFolder=$2 + else + echo "Dest folder is not writable. Aborting" + exit + fi +else + echo "Dest folder does not exist. Aborting" + exit +fi + +#files=`ls $destFolder` +#if [ "$files" != "" ] +#then +# echo "Dest folder is not empty. Aborting" +# exit +#fi +sourceFolder=${sourceFolder%/} +destFolder=${destFolder%/} +echo "Will start ingesting files from "$sourceFolder" to "$destFolder + +#list all the files in sourceFolder, and copy then with the same structure to destfolder + +entries=`find $sourceFolder -type f -name '*.fits' | sort` + +for entry in ${entries[@]} +do + #first construct the correct file name + targetFileName=`correctFileName $entry` + #second construct the destination path. + filenameonly=${entry##*/} + pathonly=${entry%$filenameonly} + extrapathonly=${pathonly#$sourceFolder/} + targetFolder=$destFolder"/"$extrapathonly + if [ ! -d $targetFolder ] + then + mkdir -p $targetFolder + fi + + #check if the file already exist there + targetFile=$targetFolder"/"$targetFileName + echo "$targetFile" + if [ -a $targetFile ] + then + echo "File $targetFile already exist. Skipping it" >> report.txt + continue + fi + cp $entry $targetFile + #if not, do the copying, fixing and checking + +# grouping=`/home/isdc/lyard/FACT++/fitsdump $targetFile -h 2>/dev/null | grep GROUPING` + +# grouping=`grep 'GROUPING' "temp.txt"` + +# if [ "$grouping" == "" ] +# then + + repairAuxFile.sh $targetFile ENDerrors.txt MJDerror.txt report.txt processErrors.txt + + if [ -a $targetFile ] + then + fixAuxKeyWords.sh $targetFile reportTwo.txt processErrors.txt + result=`fverify $targetFile 2>/dev/null | grep '0 error(s)'` + if [ "$result" == "" ] + then + echo "$targetFile" >> stillHasProblems.txt + rm $targetFile + fi + fi +done + +#set the correct permissions +find $destFolder -type f -exec chmod 640 {} \; +find $destFolder -type d -exec chmod 750 {} \; +find $destFolder -exec chgrp fact {} \; + + + + + + + + + + + + + + + + + + + + + + Index: /trunk/DataCheck/Archive/correctFileName.cxx =================================================================== --- /trunk/DataCheck/Archive/correctFileName.cxx (revision 12750) +++ /trunk/DataCheck/Archive/correctFileName.cxx (revision 12750) @@ -0,0 +1,111 @@ +#include +#include + +using namespace std; + +bool isNumber(char c) +{ + switch (c) { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + return true; + break; + default: + return false; + }; +} + +int main(int argc, char** argv) +{ + if (argc != 2) + return 0; + + string c(argv[1]); + + //get the year, month and day from the path. + string year, month, day; + string runnumber; + string serviceName; + int count = c.size()-5; + + while (count >= 0) + { + if (c[count+0] == '2' && + c[count+1] == '0' && + c[count+2] == '1' && + c[count+3] == '1' && + c[count+4] == '/') + { + year = "2011"; + month = c.substr(count+5, 2); + day = c.substr(count+8, 2); + break; + } + count--; + } + + //separate the runnumber from the rest of the filename + count = c.size()-6; + while (count >= 0) + { + if (c[count] == '_') + { + if (isNumber(c[count+1]) && + isNumber(c[count+2]) && + isNumber(c[count+3])) + { + runnumber=c.substr(count+1, 3); + break; + } + } + else + { + if (c[count+0] != '0' && + c[count+1] == '0' && + c[count+2] == '0' && + c[count+3] == '0' && + c[count+4] == '0' && + c[count+5] == '0') + { + runnumber=c.substr(count+6, 3); + break; + } + } + count--; + } + + //figure out the service name + count = c.size()-3; + while (count > 0) + { + if ((c[count] == '_' || c[count] == '.') && + !isNumber(c[count+1]) && + isNumber(c[count-1])) + { + serviceName = c.substr(count+1, c.size()-(count+6)); + } + count--; + } + if (serviceName == "fits") + serviceName = ""; + + + cout << year << month << day; + if (runnumber != "") cout << "_"; + cout << runnumber; + if (serviceName != "") cout << "."; + cout << serviceName << ".fits"; +// cout << "year: " << year << " month: " << month << " day: " << day << endl; +// cout << "run number: " << runnumber << " service name: " << serviceName << endl; + +return 0; + +} Index: /trunk/DataCheck/Archive/fixAuxKeyWords.sh =================================================================== --- /trunk/DataCheck/Archive/fixAuxKeyWords.sh (revision 12750) +++ /trunk/DataCheck/Archive/fixAuxKeyWords.sh (revision 12750) @@ -0,0 +1,147 @@ +#!/bin/bash + +if [ "$#" != 3 ] +then + echo "Error: fixAuxKeyWords.sh should be called with 2 arguments. Aborting." + exit +fi + +file=$1 +tempFile="temp.txt" +reportFile=$2 +processErrors=$3 + +if [ -a $tempFile ] +then + rm $tempFile 2>/dev/null +fi + +#get current keywords value +result=`/home/isdc/lyard/FACT++/fitsdump $file -h -o $tempFile 2>/dev/null` + +if [ -a $tempFile ] +then + timesys=`grep 'TIMESYS' $tempFile | grep -E -o 'UTC'` + mjdref=`grep 'MJDREF' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + tstarti=`grep 'TSTARTI' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + tstartf=`grep 'TSTARTF' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + tstopi=`grep 'TSTOPI' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + tstopf=`grep 'TSTOPF' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + date_obs=`grep 'DATE-OBS' $tempFile` + date_end=`grep 'DATE-END' $tempFile` + telescope=`grep 'TELESCOP' $tempFile` + package=`grep 'PACKAGE' $tempFile` + origin=`grep 'ORIGIN' $tempFile` + timeunit=`grep 'TIMEUNIT' $tempFile` + +else + echo "Could not list keywords in $file" >> $processErrors + exit +fi + +rm $tempFile + +#retrieve the start and stop time from the data itself +result=`/home/isdc/lyard/FACT++/fitsdump $file -c Time --minmax --nozero -o $tempFile 2>/dev/null` + +if [ -a $tempFile ] +then + + tstart=`grep 'min' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + tstop=`grep 'max' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + tstarti2=`echo $tstart | grep -E -o '[-]*[0-9]*[.]' | grep -E -o '[-]*[0-9]*'` + tstartf2=`echo $tstart | grep -E -o '[.][0-9]+'` + if [ "$tstarti2" == "" ] #no decimal part + then + tstarti2=$tstart + fi + tstartf2="0"$tstartf2 + tstopi2=`echo $tstop | grep -E -o '[-]*[0-9]*[.]' | grep -E -o '[-]*[0-9]*'` + tstopf2=`echo $tstop | grep -E -o '[.][0-9]+'` + if [ "$tstopi2" == "" ] #no decimal part + then + tstopi2=$tstop + fi + tstopf2="0"$tstopf2 + +else +# echo "Could not minmax $file" >> $processErrors + exit +fi + +rm $tempFile +#output the values to be added/updated to temp text files +modified="false" +if [ "$telescope" == "" ] +then + echo "TELESCOP FACT / Telescope that acquired this data" >> $tempFile + modified="true" + echo "TELESCOP in $file" >> $2 +fi +if [ "$package" == "" ] +then + echo "PACKAGE FACT++ / Package name" >> $tempFile + modified="true" + echo "PACKAGE in $file" >> $2 +fi +if [ "$origin" == "" ] +then + echo "ORIGIN FACT / Institution that wrote the file" >> $tempFile + modified="true" + echo "ORIGIN in $file" >> $2 +fi +if [ "$timeunit" == "" ] +then + echo "TIMEUNIT d / Time given in days w.r.t. to MJDREF" >> $tempFile + modified="true" + echo "TIMEUNIT in $file" >> $2 +fi +if [ "$mjdref" != "40587" ] +then + mjdref="40587" + echo "MJDREF "$mjdref" / Store times in UNIX time (sec from 1970ish)" >> $tempFile + modified="true" + echo "MJDREF in $file" >> $2 +fi +if [ "$timesys" != "UTC" ] +then + timesys="UTC" + echo "TIMESYS "$timesys" / Time system" >> $tempFile + modified="true" + echo "TIMESYS in $file" >> $2 +fi +if [ "$tstarti2" != "$tstarti" ] || [ "$tstartf2" != "$tstartf" ] +then + echo "TSTARTI "$tstarti2" / Time when first event received (integral part)" >> $tempFile + echo "TSTARTF "$tstartf2" / Time when first event received (fractional part)" >> $tempFile + date_obs2=`echo "$tstarti2 + $tstartf2 + $mjdref" | bc -l` + date_obs2=`/home/isdc/lyard/FACT++/MjDtoISO $date_obs2` + echo "DATE-OBS "$date_obs2" / Time when first event was received" >> $tempFile + modified="true" + echo "TSTART in $file" >> $2 +fi + +if [ "$tstopi2" != "$tstopi" ] || [ "$tstopf2" != "$tstopf" ] +then + echo "TSTOPI "$tstopi2" / Time when last event received (integral part)" >> $tempFile + echo "TSTOPF "$tstopf2" / Time when last event received (fractional part)" >> $tempFile + date_end2=`echo "$tstopi2 + $tstopf2 + $mjdref" | bc -l` + date_end2=`/home/isdc/lyard/FACT++/MjDtoISO $date_end2` + echo "DATE-END "$date_end2" / Time when last event was received" >> $tempFile + modified="true" + echo "TSTOP in $file" >> $2 +fi + +if [ "$modified" == "true" ] +then + echo "INGEST v0.1 Version of Etienne ingest script" >> $tempFile +fi + +if [ -a $tempFile ] +then + fmodhead $file $tempFile 2>&1 1>/dev/null +fi + +fchecksum $file update+ 2>&1 1>/dev/null + +rm $tempFile 2>/dev/null Index: /trunk/DataCheck/Archive/fixRawKeyWords.sh =================================================================== --- /trunk/DataCheck/Archive/fixRawKeyWords.sh (revision 12750) +++ /trunk/DataCheck/Archive/fixRawKeyWords.sh (revision 12750) @@ -0,0 +1,162 @@ +#!/bin/bash + +if [ "$#" != 3 ] +then + echo "Error: fixAuxKeyWords.sh should be called with 2 arguments. Aborting." + exit +fi + +file=$1 +tempFile="tempRaw.txt" +reportFile=$2 +processErrors=$3 + +if [ -a $tempFile ] +then + rm $tempFile 2>/dev/null +fi + +#get current keywords value +result=`/home/isdc/lyard/FACT++/fitsdump $file -h -o $tempFile 2>/dev/null` + +if [ -a $tempFile ] +then + timesys=`grep 'TIMESYS' $tempFile | grep -E -o 'UTC'` + mjdref=`grep 'MJDREF' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + tstarti=`grep 'TSTARTI' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + tstartf=`grep 'TSTARTF' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + tstopi=`grep 'TSTOPI' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + tstopf=`grep 'TSTOPF' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + date_obs=`grep 'DATE-OBS' $tempFile` + date_end=`grep 'DATE-END' $tempFile` + telescope=`grep 'TELESCOP' $tempFile` + package=`grep 'PACKAGE' $tempFile` + origin=`grep 'ORIGIN' $tempFile` + timeunit=`grep 'TIMEUNIT' $tempFile` +else + echo "Could not list keywords in $file" >> $processErrors + exit +fi + +rm $tempFile + +#retrieve the start and stop time from the data itself +result=`/home/isdc/lyard/FACT++/fitsdump $file -c UnixTimeUTC --minmax --nozero -o $tempFile 2>/dev/null` + +if [ -a $tempFile ] +then + tstart=`grep 'min' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + tstop=`grep 'max' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + tstarti2=`echo $tstart | grep -E -o '[-]*[0-9]*[.]' | grep -E -o '[-]*[0-9]*'` + tstartf2=`echo $tstart | grep -E -o '[.][0-9]+'` + tstartf2="0"$tstartf2 + tstopi2=`echo $tstop | grep -E -o '[-]*[0-9]*[.]' | grep -E -o '[-]*[0-9]*'` + tstopf2=`echo $tstop | grep -E -o '[.][0-9]+'` + tstopf2="0"$tstopf2 +else + #let's give it a shot with PCTime + result=`/home/isdc/lyard/FACT++/fitsdump $file -c PCTime --minmax --nozero -o $tempFile 2>/dev/null` + if [ -a $tempFile ] + then + tstart=`grep 'min' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + tstop=`grep 'max' $tempFile | grep -E -o '[-]*[0-9]*[.]*[0-9]+'` + tstarti2=`echo $tstart | grep -E -o '[-]*[0-9]*[.]' | grep -E -o '[-]*[0-9]*'` + tstartf2=`echo $tstart | grep -E -o '[.][0-9]+'` + if [ "$tstarti2" == "" ] #no decimal part + then + tstarti2=$tstart + fi + tstartf2="0"$tstartf2 + tstopi2=`echo $tstop | grep -E -o '[-]*[0-9]*[.]' | grep -E -o '[-]*[0-9]*'` + tstopf2=`echo $tstop | grep -E -o '[.][0-9]+'` + if [ "$tstopi2" == "" ] #no decimal part + then + tstopi2=$tstop + fi + tstopf2="0"$tstopf2 + else +# echo "Could not minmax $file" >> $processErrors + exit + fi +fi + +rm $tempFile +#output the values to be added/updated to temp text files +modified="false" +if [ "$telescope" == "" ] +then + echo "TELESCOP FACT / Telescope that acquired this data" >> $tempFile + modified="true" + echo "TELESCOP in $file" >> $2 +fi +if [ "$package" == "" ] +then + echo "PACKAGE FACT++ / Package name" >> $tempFile + modified="true" + echo "PACKAGE in $file" >> $2 +fi +if [ "$origin" == "" ] +then + echo "ORIGIN FACT / Institution that wrote the file" >> $tempFile + modified="true" + echo "ORIGIN in $file" >> $2 +fi +if [ "$timeunit" == "" ] +then + echo "TIMEUNIT d / Time given in days w.r.t. to MJDREF" >> $tempFile + modified="true" + echo "TIMEUNIT in $file" >> $2 +fi +if [ "$mjdref" == "" ] +then +# mjdref="0" + mjdref="40587" + echo "MJDREF "$mjdref" / Store times in UNIX time (sec from 1970ish)" >> $tempFile + modified="true" + echo "MJDREF in $file" >> $2 +fi +if [ "$timesys" != "UTC" ] +then + timesys="UTC" + echo "TIMESYS "$timesys" / Time system" >> $tempFile + modified="true" + echo "TIMESYS in $file" >> $2 +fi +#adapt the start and stop to the mjdref (it is raw unix time in the data) +tstarti2=`echo "$tstarti2 - $mjdref" | bc -l` +tstopi2=`echo "$tstopi2 - $mjdref" | bc -l` +if [ "$tstarti2" != "$tstarti" ] || [ "$tstartf2" != "$tstartf" ] +then + echo "TSTARTI "$tstarti2" / Time when first event received (integral part)" >> $tempFile + echo "TSTARTF "$tstartf2" / Time when first event received (fractional part)" >> $tempFile + date_obs2=`echo "$tstarti2 + $tstartf2 + $mjdref" | bc -l` + date_obs2=`/home/isdc/lyard/FACT++/MjDtoISO $date_obs2` + echo "DATE-OBS "$date_obs2" / Time when first event was received" >> $tempFile + modified="true" + echo "TSTART in $file" >> $2 +fi + +if [ "$tstopi2" != "$tstopi" ] || [ "$tstopf2" != "$tstopf" ] +then + echo "TSTOPI "$tstopi2" / Time when last event received (integral part)" >> $tempFile + echo "TSTOPF "$tstopf2" / Time when last event received (fractional part)" >> $tempFile + date_end2=`echo "$tstopi2 + $tstopf2 + $mjdref" | bc -l` + date_end2=`/home/isdc/lyard/FACT++/MjDtoISO $date_end2` + echo "DATE-END "$date_end2" / Time when last event was received" >> $tempFile + modified="true" + echo "TSTOP in $file" >> $2 +fi + +if [ "$modified" == "true" ] +then + echo "INGEST v0.1 Version of Etienne ingest script" >> $tempFile +fi + +if [ -a $tempFile ] +then + fmodhead $file $tempFile 2>&1 1>/dev/null +fi + +fchecksum $file update+ 2>&1 1>/dev/null + +rm $tempFile 2>/dev/null Index: /trunk/DataCheck/Archive/ingest.sh =================================================================== --- /trunk/DataCheck/Archive/ingest.sh (revision 12750) +++ /trunk/DataCheck/Archive/ingest.sh (revision 12750) @@ -0,0 +1,49 @@ +sourceAux="/data00/fact-construction/aux" +destAux="/archive/fact/rev_1/aux" + +sourceRaw="/data03/fact-construction/raw" +destRaw="/archive/fact/rev_1/raw" + +ok="true" + +if [ -d $sourceAux ] +then + echo "Source aux: "$sourceAux +else + echo "Source aux DOES NOT EXIST" + ok="false" +fi + +if [ -d $destAux ] +then + echo "Dest aux: "$destAux +else + echo "Dest aux DOES NOT EXIST" + ok="false" +fi + +if [ -d $sourceRaw ] +then + echo "Source raw: "$sourceRaw +else + echo "Source raw DOES NOT EXIST" + ok="false" +fi + +if [ -d $destRaw ] +then + echo "Dest raw: "$destRaw +else + echo "Dest raw DOES NOT EXIST" + ok="false" +fi + +if [ "$ok" == "true" ] +then + #echo "Here I should be starting both scripts" + auxIngest.sh $sourceAux $destAux + rawIngest.sh $sourceRaw $destRaw +else + echo "Something went wrong with folders. Please check them" +fi + Index: /trunk/DataCheck/Archive/rawIngest.sh =================================================================== --- /trunk/DataCheck/Archive/rawIngest.sh (revision 12750) +++ /trunk/DataCheck/Archive/rawIngest.sh (revision 12750) @@ -0,0 +1,143 @@ +#!/bin/bash + +sourceFolder=$1 +destFolder=$2 + +if [ "$#" != "2" ] +then + echo "Please specify source and dest folders. Aborting" + exit +fi + +if [ $1 == "" ] +then + echo "Source folder is empty. Aborting" + exit +fi + +if [ $2 == "" ] +then + echo "Dest folder is empty. Aborting" + exit +fi + +#first let's make sure that source and dest folders do exist, and that dest is writable +if [ -d $1 ] +then + sourceFolder=$1 +else + echo "Source folder "$1" does not exist (or cannnot be read.) Aborting" + exit +fi + +if [ -d $2 ] +then + if [ -d $2"/etiennetest" ] + then + echo "Test folder already exist. Aborting" + exit + fi + mkdir $2"/etiennetest" 2>/dev/null + if [ -d $2"/etiennetest" ] + then + rm -rf $2"/etiennetest" + destFolder=$2 + else + echo "Dest folder is not writable. Aborting" + exit + fi +else + echo "Dest folder does not exist. Aborting" + exit +fi + +#files=`ls $destFolder` +#if [ "$files" != "" ] +#then +# echo "Dest folder is not empty. Aborting" +# exit +#fi +sourceFolder=${sourceFolder%/} +destFolder=${destFolder%/} +echo "Will start ingesting files from "$sourceFolder" to "$destFolder + +#list all the files in sourceFolder, and copy then with the same structure to destfolder + +entries=`find $sourceFolder -type f -name '*.fits' | sort` + +for entry in ${entries[@]} +do + #second construct the destination path. + filenameonly=${entry##*/} + #first construct the correct file name + targetFileName=$filenameonly + pathonly=${entry%$filenameonly} + extrapathonly=${pathonly#$sourceFolder/} + targetFolder=$destFolder"/"$extrapathonly + if [ ! -d $targetFolder ] + then + mkdir -p $targetFolder + fi + + #check if the file already exist there + targetFile=$targetFolder"/"$targetFileName + interFile="/scratch/"$targetFileName + echo "$targetFile" +# echo "$interFile" + if [ -a $targetFile".gz" ] + then + echo "File $targetFile already exist. Skipping it" >> Rawreport.txt + continue + fi + cp $entry $interFile + #if not, do the copying, fixing and checking + +# grouping=`/home/isdc/lyard/FACT++/fitsdump $targetFile -h 2>/dev/null | grep GROUPING` + +# grouping=`grep 'GROUPING' "temp.txt"` + +# if [ "$grouping" == "" ] +# then + + repairRawFile.sh $interFile RawENDerrors.txt RawMJDerror.txt Rawreport.txt RawprocessErrors.txt + + if [ -a $interFile ] + then + fixRawKeyWords.sh $interFile RawreportTwo.txt RawprocessErrors.txt + result=`fverify $interFile 2>/dev/null | grep '0 error(s)'` + if [ "$result" == "" ] + then + echo "$interFile" >> RawstillHasProblems.txt + rm $interFile + else + gzip $interFile + cp $interFile".gz" $targetFile".gz" + rm $interFile".gz" + fi + fi +done + +#set the correct permissions +find $destFolder -type f -exec chmod 640 {} \; +find $destFolder -type d -exec chmod 750 {} \; +find $destFolder -exec chgrp fact {} \; + + + + + + + + + + + + + + + + + + + + Index: /trunk/DataCheck/Archive/repairAuxFile.sh =================================================================== --- /trunk/DataCheck/Archive/repairAuxFile.sh (revision 12750) +++ /trunk/DataCheck/Archive/repairAuxFile.sh (revision 12750) @@ -0,0 +1,81 @@ +#!/bin/bash +#sourceFolder="./backup" +#destFolder="./fixed" + +#entries=`find $sourceFolder -type f -name '*.fits' | sort` + +#for entry in ${entries[@]} +#do +# echo $entry" "$destFolder +# cp $entry $destFolder +#done + +#entries=`find $destFolder -type f -name '*.fits' | sort` + +#for entry in ${entries[@]} +#do + +if [ "$#" != "5" ] +then + echo "Error: repairAuxFile.sh should be called with 4 arguments please" + exit +fi +entry=$1 +errorFile=$2 +wrongMjdFile=$3 +reportFile=$4 +processErrorFile=$5 + #verify file with fverify + result=`fverify $entry 2>/dev/null | grep '0 error(s)'` + if [ "$result" != "" ] + then + mjdref=`/home/isdc/lyard/FACT++/fitsdump $entry -h 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]+'` + if [ "$mjdref" != 40587 ] + then + echo "$entry" >> $wrongMjdFile + result="" + fi + else + echo "$entry" >> $errorFile + fi + + if [ "$result" == "" ] + then + #fix it ! +# echo $entry + headerSize=`ENDfixer $entry 2>/dev/null` + echo "Fixed END $entry" >> $reportFile + if [ "$headerSize" == "Error: header length not acceptable" ] || [ "$headerSize" == "Error: too much header space after END keyword" ] + then + echo "File "$entry" looks really messed up: "$headerSize >> $errorFile + exit + else + filesize=`stat -c%s $entry` +# echo $headerSize" "$filesize + numrows=`/home/isdc/lyard/FACT++/fitsdump $entry -h 2>/dev/null | grep 'NAXIS2' | awk '{ print $4 }' | grep -E -o '[0-9]+'` + if [ "$numrows" == "" ] + then + echo "Cannot fitsdump $entry" >> $processErrorFile + rm $entry + exit + fi + rowWidth=`/home/isdc/lyard/FACT++/fitsdump $entry -h 2>/dev/null | grep 'NAXIS1' | awk '{ print $4 }' | grep -E -o '[0-9]+'` + + #is there any extra row that we can keep ? + #let RowCheck examine the time markers + numrows2=`RowChecker $entry $headerSize $rowWidth 0 $numrows 2>/dev/null` + if [ $numrows2 -gt $numrows ] + then + numrows=$numrows2 + totSize=`echo " $headerSize + $numrows * $rowWidth " | bc -l` + fitsSize=`echo " 2880 - ($totSize % 2880) " | bc` + fitsSize=`echo " $totSize + $fitsSize " | bc -l` +# echo $headerSize" "$numrows" "$rowWidth" "$totSize" "$fitsSize +# echo " FileSize: "$filesize" should be "$fitsSize + truncate -s $totSize $entry 2>/dev/null + truncate -s $fitsSize $entry 2>/dev/null + echo "Resized $entry" >> $reportFile + fi + fi + fi +#done Index: /trunk/DataCheck/Archive/repairRawFile.sh =================================================================== --- /trunk/DataCheck/Archive/repairRawFile.sh (revision 12750) +++ /trunk/DataCheck/Archive/repairRawFile.sh (revision 12750) @@ -0,0 +1,74 @@ +#!/bin/bash +#sourceFolder="./backup" +#destFolder="./fixed" + +#entries=`find $sourceFolder -type f -name '*.fits' | sort` + +#for entry in ${entries[@]} +#do +# echo $entry" "$destFolder +# cp $entry $destFolder +#done + +#entries=`find $destFolder -type f -name '*.fits' | sort` + +#for entry in ${entries[@]} +#do + +if [ "$#" != "5" ] +then + echo "Error: repairAuxFile.sh should be called with 4 arguments please" + exit +fi +entry=$1 +errorFile=$2 +wrongMjdFile=$3 +reportFile=$4 +processErrorFile=$5 + #verify file with fverify + result=`fverify $entry 2>/dev/null | grep '0 error(s)'` + if [ "$result" != "" ] + then + mjdref=`/home/isdc/lyard/FACT++/fitsdump $entry -h 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]+'` + if [ "$mjdref" != 40587 ] + then + echo "$entry" >> $wrongMjdFile + result="" + fi + else + echo "$entry" >> $errorFile + fi + + if [ "$result" == "" ] + then + #fix it ! +# echo $entry + headerSize=`ENDfixer $entry 2>/dev/null` + echo "Fixed END $entry" >> $reportFile + if [ "$headerSize" == "Error: header length not acceptable" ] || [ "$headerSize" == "Error: too much header space after END keyword" ] + then + echo "File "$entry" looks really messed up: "$headerSize >> $errorFile + exit + else + filesize=`stat -c%s $entry` +# echo $headerSize" "$filesize + numrows=`/home/isdc/lyard/FACT++/fitsdump $entry -h 2>/dev/null | grep 'NAXIS2' | awk '{ print $4 }' | grep -E -o '[0-9]+'` + if [ "$numrows" == "" ] + then + echo "Cannot fitsdump $entry" >> $processErrorFile + rm $entry + exit + fi + rowWidth=`/home/isdc/lyard/FACT++/fitsdump $entry -h 2>/dev/null | grep 'NAXIS1' | awk '{ print $4 }' | grep -E -o '[0-9]+'` + + totSize=`echo " $headerSize + $numrows * $rowWidth " | bc -l` + fitsSize=`echo " 2880 - ($totSize % 2880) " | bc` + fitsSize=`echo " $totSize + $fitsSize " | bc -l` +# echo $headerSize" "$numrows" "$rowWidth" "$totSize" "$fitsSize +# echo " FileSize: "$filesize" should be "$fitsSize + truncate -s $totSize $entry 2>/dev/null + truncate -s $fitsSize $entry 2>/dev/null + echo "Resized $entry" >> $reportFile + fi + fi +#done Index: /trunk/DataCheck/Archive/testNaming.sh =================================================================== --- /trunk/DataCheck/Archive/testNaming.sh (revision 12750) +++ /trunk/DataCheck/Archive/testNaming.sh (revision 12750) @@ -0,0 +1,20 @@ +#!/bin/bash + +filename=$1 + +filenameonly=${filename##*/} + +filenameonly=${filenameonly%%.fits} + +echo $filenameonly + + + #first construct the correct file name +# filenameonly=${entry##*/} +# year="2011" +# month=${filenameonly} + #second construct the destination path. +# filenameonly=${entry##*/} +# pathonly=${entry%$filenameonly} +# extrapathonly=${pathonly#$sourceFolder/} +# targetFolder=$destFolder"/"$extrapathonly