Ignore:
Timestamp:
02/08/12 12:33:48 (13 years ago)
Author:
lyard
Message:
added more scripts
Location:
trunk/DataCheck/Archive
Files:
10 added
4 edited

Legend:

Unmodified
Added
Removed

  • trunk/DataCheck/Archive/auxIngest.sh

    r12750 r12871  
    33sourceFolder=$1
    44destFolder=$2
     5suffix=$3
    56
    6 if [ "$#" != "2" ]
     7if [ "$#" != "3" ]
    78then
    8         echo "Please specify source and dest folders. Aborting"
     9        echo "Please specify source and dest folders and suffix. Aborting"
    910        exit
    1011fi
    … …  
    8586        if [ -a $targetFile ]
    8687        then
    87                 echo "File $targetFile already exist. Skipping it" >> report.txt
     88                echo "File $targetFile already exist. Skipping it" >> report_$suffix.txt
    8889                continue
    8990        fi
    … …  
    9899#       then
    99100               
    100         repairAuxFile.sh $targetFile ENDerrors.txt MJDerror.txt report.txt processErrors.txt
     101        repairAuxFile.sh $targetFile ENDerrors_$suffix.txt MJDerror_$suffix.txt report_$suffix.txt processErrors_$suffix.txt
    101102       
    102103        if [ -a $targetFile ]
    103104        then
    104                 fixAuxKeyWords.sh $targetFile reportTwo.txt processErrors.txt
     105                fixAuxKeyWords.sh $targetFile reportTwo_$suffix.txt processErrors_$suffix.txt
    105106                result=`fverify $targetFile 2>/dev/null | grep '0 error(s)'`
    106107                if [ "$result" == "" ]
    107108                then
    108                         echo "$targetFile" >> stillHasProblems.txt
     109                        echo "$targetFile" >> stillHasProblems_$suffix.txt
    109110                        rm $targetFile
    110111                fi

  • trunk/DataCheck/Archive/fixRawKeyWords.sh

    r12750 r12871  
    88
    99file=$1
    10 tempFile="tempRaw.txt"
     10tempFile="/scratch/tempRaw.txt"
    1111reportFile=$2
    1212processErrors=$3
    … …  
    1818
    1919#get current keywords value
    20 result=`/home/isdc/lyard/FACT++/fitsdump $file -h -o $tempFile 2>/dev/null`
     20result=`/home_nfs/isdc/lyard/FACT++/fitsdump $file -h -o $tempFile 2>/dev/null`
    2121
    2222if [ -a $tempFile ]
    … …  
    4242
    4343#retrieve the start and stop time from the data itself
    44 result=`/home/isdc/lyard/FACT++/fitsdump $file -c UnixTimeUTC --minmax --nozero -o $tempFile 2>/dev/null`
     44result=`/home_nfs/isdc/lyard/FACT++/fitsdump $file -c UnixTimeUTC --minmax --nozero -o $tempFile 2>/dev/null`
    4545
    4646if [ -a $tempFile ]
    … …  
    5656else
    5757        #let's give it a shot with PCTime
    58         result=`/home/isdc/lyard/FACT++/fitsdump $file -c PCTime --minmax --nozero -o $tempFile 2>/dev/null`
     58        result=`/home_nfs/isdc/lyard/FACT++/fitsdump $file -c PCTime --minmax --nozero -o $tempFile 2>/dev/null`
    5959        if [ -a $tempFile ]
    6060        then
    … …  
    124124fi
    125125#adapt the start and stop to the mjdref (it is raw unix time in the data)
    126 tstarti2=`echo "$tstarti2 - $mjdref" | bc -l`
    127 tstopi2=`echo "$tstopi2 - $mjdref" | bc -l`
    128 if [ "$tstarti2" != "$tstarti" ] || [ "$tstartf2" != "$tstartf" ]
     126if [ "$tstarti2" != "0" ] && [ "$tstarti2" != "" ]
     127then
     128        tstarti2=`echo "$tstarti2 - $mjdref" | bc -l`
     129        tstopi2=`echo "$tstopi2 - $mjdref" | bc -l`
     130else
     131        tstarti2=0
     132        tstopi2=0
     133fi
     134
     135#give latitude for 10-6 precision in tstart and tstop
     136tfcompare=`echo $tstartf | grep -E -o '0\.[0-9]{6}'`
     137tfcompare2=`echo $tstartf2 | grep -E -o '0\.[0-9]{6}'`
     138
     139if [ "$tstarti2" != "$tstarti" ] || [ "$tfcompare" != "$tfcompare2" ]
    129140then
    130141        echo "TSTARTI "$tstarti2" / Time when first event received (integral part)" >> $tempFile
    131142        echo "TSTARTF "$tstartf2" / Time when first event received (fractional part)" >> $tempFile
    132143        date_obs2=`echo "$tstarti2 + $tstartf2 + $mjdref" | bc -l`
    133         date_obs2=`/home/isdc/lyard/FACT++/MjDtoISO $date_obs2`
     144        date_obs2=`/home_nfs/isdc/lyard/FACT++/MjDtoISO $date_obs2`
    134145        echo "DATE-OBS "$date_obs2" / Time when first event was received" >> $tempFile
    135146        modified="true"
    … …  
    137148fi
    138149
    139 if [ "$tstopi2" != "$tstopi" ] || [ "$tstopf2" != "$tstopf" ]
     150tfcompare=`echo $tstopf | grep -E -o '0\.[0-9]{6}'`
     151tfcompare2=`echo $tstopf2 | grep -E -o '0\.[0-9]{6}'`
     152
     153if [ "$tstopi2" != "$tstopi" ] || [ "$tfcompare" != "$tfcompare2" ]
    140154then
     155
     156        echo "first: $tfcompare |||  second: $tfcompare2||"
    141157        echo "TSTOPI "$tstopi2" / Time when last event received (integral part)" >> $tempFile
    142158        echo "TSTOPF "$tstopf2" / Time when last event received (fractional part)" >> $tempFile
    143159        date_end2=`echo "$tstopi2 + $tstopf2 + $mjdref" | bc -l`
    144         date_end2=`/home/isdc/lyard/FACT++/MjDtoISO $date_end2`
     160        date_end2=`/home_nfs/isdc/lyard/FACT++/MjDtoISO $date_end2`
    145161        echo "DATE-END "$date_end2" / Time when last event was received" >> $tempFile
    146162        modified="true"
    … …  
    150166if [ "$modified" == "true" ]
    151167then
    152         echo "INGEST v0.1 Version of Etienne ingest script" >> $tempFile
     168        echo "INGEST v0.2 Version of Etienne ingest script" >> $tempFile
     169        echo $file" header has been modified" >> $reportFile
     170else
     171        echo $file" header has NOT been modified" >> $reportFile
    153172fi
    154173

  • trunk/DataCheck/Archive/rawIngest.sh

    r12750 r12871  
    44destFolder=$2
    55
    6 if [ "$#" != "2" ]
     6if [ "$#" != "3" ]
    77then
    8         echo "Please specify source and dest folders. Aborting"
     8        echo "Please specify source and dest folders. and an identifier for the log files Aborting"
    99        exit
    1010fi
    … …  
    1919then
    2020        echo "Dest folder is empty. Aborting"
     21        exit
     22fi
     23
     24if [ $3 == "" ]
     25then
     26        echo "Identifier for log files empty. Aborting"
    2127        exit
    2228fi
    … …  
    6167destFolder=${destFolder%/}
    6268echo "Will start ingesting files from "$sourceFolder" to "$destFolder
     69echo "Will start ingesting files from "$sourceFolder" to "$destFolder >> Rawreport$3.txt
    6370
    6471#list all the files in sourceFolder, and copy then with the same structure to destfolder
    … …  
    8794        if [ -a $targetFile".gz" ]
    8895        then
    89                 echo "File $targetFile already exist. Skipping it" >> Rawreport.txt
     96                echo "File $targetFile already exist. Skipping it" >> Rawreport$3.txt
    9097                continue
    9198        fi
    … …  
    100107#       then
    101108               
    102         repairRawFile.sh $interFile RawENDerrors.txt RawMJDerror.txt Rawreport.txt RawprocessErrors.txt
     109        repairRawFile.sh $interFile RawENDerrors$3.txt RawMJDerror$3.txt Rawreport$3.txt RawprocessErrors$3.txt
    103110       
    104111        if [ -a $interFile ]
    105112        then
    106                 fixRawKeyWords.sh $interFile RawreportTwo.txt RawprocessErrors.txt
     113                fixRawKeyWords.sh $interFile RawreportTwo$3.txt RawprocessErrors$3.txt
    107114                result=`fverify $interFile 2>/dev/null | grep '0 error(s)'`
    108115                if [ "$result" == "" ]
    109116                then
    110                         echo "$interFile" >> RawstillHasProblems.txt
     117                        echo "$interFile" >> RawstillHasProblems$3.txt
    111118                        rm $interFile
    112119                else
    113                         gzip $interFile
     120                        gzip -1 $interFile
    114121                        cp $interFile".gz" $targetFile".gz"
    115122                        rm $interFile".gz"

  • trunk/DataCheck/Archive/repairRawFile.sh

    r12750 r12871  
    3030        if [ "$result" != "" ]
    3131        then
    32                 mjdref=`/home/isdc/lyard/FACT++/fitsdump $entry -h 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]+'`
     32                mjdref=`/home_nfs/isdc/lyard/FACT++/fitsdump $entry -h 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]+'`
    3333                if [ "$mjdref" != 40587 ]
    3434                then
    … …  
    5353                        filesize=`stat -c%s $entry`
    5454#                       echo $headerSize" "$filesize
    55                         numrows=`/home/isdc/lyard/FACT++/fitsdump $entry -h 2>/dev/null | grep 'NAXIS2' | awk '{ print $4 }' | grep -E -o '[0-9]+'`
     55                        numrows=`/home_nfs/isdc/lyard/FACT++/fitsdump $entry -h 2>/dev/null | grep 'NAXIS2' | awk '{ print $4 }' | grep -E -o '[0-9]+'`
    5656                        if [ "$numrows" == "" ]
    5757                        then
    … …  
    6060                                exit
    6161                        fi
    62                         rowWidth=`/home/isdc/lyard/FACT++/fitsdump $entry -h 2>/dev/null | grep 'NAXIS1' | awk '{ print $4 }' | grep -E -o '[0-9]+'`
     62                        rowWidth=`/home_nfs/isdc/lyard/FACT++/fitsdump $entry -h 2>/dev/null | grep 'NAXIS1' | awk '{ print $4 }' | grep -E -o '[0-9]+'`
    6363
    6464                        totSize=`echo " $headerSize + $numrows * $rowWidth " | bc -l`
    … …  
    6767#                               echo $headerSize" "$numrows" "$rowWidth" "$totSize" "$fitsSize
    6868#                               echo " FileSize: "$filesize" should be "$fitsSize
    69                         truncate -s $totSize $entry 2>/dev/null
    70                         truncate -s $fitsSize $entry 2>/dev/null
    71                         echo "Resized   $entry" >> $reportFile
     69                        if [ $filesize > $fitsSize ]
     70                        then
     71                                truncate -s $totSize $entry 2>/dev/null
     72                                truncate -s $fitsSize $entry 2>/dev/null
     73                                echo "Resized   $entry" >> $reportFile
     74                        fi
     75                       
    7276                fi
    7377        fi
Note: See TracChangeset for help on using the changeset viewer.