Changeset 12871 for trunk/DataCheck

Timestamp:

02/08/12 12:33:48 (13 years ago)

Author:

lyard

Message:

added more scripts

Location:

trunk/DataCheck/Archive

Files:

• ENDfixer.cpp (added)
• MjDtoISO.cpp (added)
• README (added)
• RowChecker.cpp (added)
• associateRunsAndCalibs.sh (added)
• auxIngest.sh (modified) (3 diffs)
• checkForHeaderUpdatesAux.sh (added)
• checkForHeaderUpdatesRaw.sh (added)
• checkRawFileIntegrityAndContent.sh (added)
• correctFileName.cpp (added)
• fitsCompare.cpp (added)
• fixRawKeyWords.sh (modified) (7 diffs)
• rawIngest.sh (modified) (5 diffs)
• repairRawFile.sh (modified) (4 diffs)

trunk/DataCheck/Archive/auxIngest.sh

@@ -3 +3 @@
 sourceFolder=$1 
 destFolder=$2 
+suffix=$3
 
-if [ "$#" != "2" ]
+if [ "$#" != "3" ]
 then
-        echo "Please specify source and dest folders. Aborting"
+        echo "Please specify source and dest folders and suffix. Aborting"
         exit
 fi
@@ -85 +86 @@
         if [ -a $targetFile ]
         then
-                echo "File $targetFile already exist. Skipping it" >> report.txt
+                echo "File $targetFile already exist. Skipping it" >> report_$suffix.txt
                 continue
         fi
@@ -98 +99 @@
 #       then
                 
-        repairAuxFile.sh $targetFile ENDerrors.txt MJDerror.txt report.txt processErrors.txt
+        repairAuxFile.sh $targetFile ENDerrors_$suffix.txt MJDerror_$suffix.txt report_$suffix.txt processErrors_$suffix.txt
         
         if [ -a $targetFile ]
         then
-                fixAuxKeyWords.sh $targetFile reportTwo.txt processErrors.txt
+                fixAuxKeyWords.sh $targetFile reportTwo_$suffix.txt processErrors_$suffix.txt
                 result=`fverify $targetFile 2>/dev/null | grep '0 error(s)'`
                 if [ "$result" == "" ]
                 then
-                        echo "$targetFile" >> stillHasProblems.txt
+                        echo "$targetFile" >> stillHasProblems_$suffix.txt
                         rm $targetFile
                 fi

trunk/DataCheck/Archive/fixRawKeyWords.sh

@@ -8 +8 @@
 
 file=$1
-tempFile="tempRaw.txt"
+tempFile="/scratch/tempRaw.txt"
 reportFile=$2
 processErrors=$3
@@ -18 +18 @@
 
 #get current keywords value
-result=`/home/isdc/lyard/FACT++/fitsdump $file -h -o $tempFile 2>/dev/null`
+result=`/home_nfs/isdc/lyard/FACT++/fitsdump $file -h -o $tempFile 2>/dev/null`
 
 if [ -a $tempFile ]
@@ -42 +42 @@
 
 #retrieve the start and stop time from the data itself
-result=`/home/isdc/lyard/FACT++/fitsdump $file -c UnixTimeUTC --minmax --nozero -o $tempFile 2>/dev/null`
+result=`/home_nfs/isdc/lyard/FACT++/fitsdump $file -c UnixTimeUTC --minmax --nozero -o $tempFile 2>/dev/null`
 
 if [ -a $tempFile ]
@@ -56 +56 @@
 else
         #let's give it a shot with PCTime
-        result=`/home/isdc/lyard/FACT++/fitsdump $file -c PCTime --minmax --nozero -o $tempFile 2>/dev/null`
+        result=`/home_nfs/isdc/lyard/FACT++/fitsdump $file -c PCTime --minmax --nozero -o $tempFile 2>/dev/null`
         if [ -a $tempFile ]
         then
@@ -124 +124 @@
 fi
 #adapt the start and stop to the mjdref (it is raw unix time in the data)
-tstarti2=`echo "$tstarti2 - $mjdref" | bc -l`
-tstopi2=`echo "$tstopi2 - $mjdref" | bc -l`
-if [ "$tstarti2" != "$tstarti" ] || [ "$tstartf2" != "$tstartf" ]
+if [ "$tstarti2" != "0" ] && [ "$tstarti2" != "" ]
+then
+        tstarti2=`echo "$tstarti2 - $mjdref" | bc -l`
+        tstopi2=`echo "$tstopi2 - $mjdref" | bc -l`
+else
+        tstarti2=0
+        tstopi2=0
+fi
+
+#give latitude for 10-6 precision in tstart and tstop
+tfcompare=`echo $tstartf | grep -E -o '0\.[0-9]{6}'`
+tfcompare2=`echo $tstartf2 | grep -E -o '0\.[0-9]{6}'`
+
+if [ "$tstarti2" != "$tstarti" ] || [ "$tfcompare" != "$tfcompare2" ] 
 then
         echo "TSTARTI "$tstarti2" / Time when first event received (integral part)" >> $tempFile
         echo "TSTARTF "$tstartf2" / Time when first event received (fractional part)" >> $tempFile
         date_obs2=`echo "$tstarti2 + $tstartf2 + $mjdref" | bc -l`
-        date_obs2=`/home/isdc/lyard/FACT++/MjDtoISO $date_obs2`
+        date_obs2=`/home_nfs/isdc/lyard/FACT++/MjDtoISO $date_obs2`
         echo "DATE-OBS "$date_obs2" / Time when first event was received" >> $tempFile
         modified="true"
@@ -137 +148 @@
 fi
 
-if [ "$tstopi2" != "$tstopi" ] || [ "$tstopf2" != "$tstopf" ]
+tfcompare=`echo $tstopf | grep -E -o '0\.[0-9]{6}'`
+tfcompare2=`echo $tstopf2 | grep -E -o '0\.[0-9]{6}'`
+
+if [ "$tstopi2" != "$tstopi" ] || [ "$tfcompare" != "$tfcompare2" ]
 then
+
+        echo "first: $tfcompare |||  second: $tfcompare2||"
         echo "TSTOPI "$tstopi2" / Time when last event received (integral part)" >> $tempFile
         echo "TSTOPF "$tstopf2" / Time when last event received (fractional part)" >> $tempFile
         date_end2=`echo "$tstopi2 + $tstopf2 + $mjdref" | bc -l`
-        date_end2=`/home/isdc/lyard/FACT++/MjDtoISO $date_end2`
+        date_end2=`/home_nfs/isdc/lyard/FACT++/MjDtoISO $date_end2`
         echo "DATE-END "$date_end2" / Time when last event was received" >> $tempFile
         modified="true"
@@ -150 +166 @@
 if [ "$modified" == "true" ]
 then
-        echo "INGEST v0.1 Version of Etienne ingest script" >> $tempFile
+        echo "INGEST v0.2 Version of Etienne ingest script" >> $tempFile
+        echo $file" header has been modified" >> $reportFile
+else
+        echo $file" header has NOT been modified" >> $reportFile
 fi
 

trunk/DataCheck/Archive/rawIngest.sh

@@ -4 +4 @@
 destFolder=$2 
 
-if [ "$#" != "2" ]
+if [ "$#" != "3" ]
 then
-        echo "Please specify source and dest folders. Aborting"
+        echo "Please specify source and dest folders. and an identifier for the log files Aborting"
         exit
 fi
@@ -19 +19 @@
 then
         echo "Dest folder is empty. Aborting"
+        exit
+fi
+
+if [ $3 == "" ]
+then 
+        echo "Identifier for log files empty. Aborting"
         exit
 fi
@@ -61 +67 @@
 destFolder=${destFolder%/}
 echo "Will start ingesting files from "$sourceFolder" to "$destFolder
+echo "Will start ingesting files from "$sourceFolder" to "$destFolder >> Rawreport$3.txt
 
 #list all the files in sourceFolder, and copy then with the same structure to destfolder
@@ -87 +94 @@
         if [ -a $targetFile".gz" ]
         then
-                echo "File $targetFile already exist. Skipping it" >> Rawreport.txt
+                echo "File $targetFile already exist. Skipping it" >> Rawreport$3.txt
                 continue
         fi
@@ -100 +107 @@
 #       then
                 
-        repairRawFile.sh $interFile RawENDerrors.txt RawMJDerror.txt Rawreport.txt RawprocessErrors.txt
+        repairRawFile.sh $interFile RawENDerrors$3.txt RawMJDerror$3.txt Rawreport$3.txt RawprocessErrors$3.txt
         
         if [ -a $interFile ]
         then
-                fixRawKeyWords.sh $interFile RawreportTwo.txt RawprocessErrors.txt
+                fixRawKeyWords.sh $interFile RawreportTwo$3.txt RawprocessErrors$3.txt
                 result=`fverify $interFile 2>/dev/null | grep '0 error(s)'`
                 if [ "$result" == "" ]
                 then
-                        echo "$interFile" >> RawstillHasProblems.txt
+                        echo "$interFile" >> RawstillHasProblems$3.txt
                         rm $interFile
                 else
-                        gzip $interFile
+                        gzip -1 $interFile
                         cp $interFile".gz" $targetFile".gz"
                         rm $interFile".gz"

trunk/DataCheck/Archive/repairRawFile.sh

@@ -30 +30 @@
         if [ "$result" != "" ]
         then
-                mjdref=`/home/isdc/lyard/FACT++/fitsdump $entry -h 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]+'`
+                mjdref=`/home_nfs/isdc/lyard/FACT++/fitsdump $entry -h 2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]+'`
                 if [ "$mjdref" != 40587 ]
                 then
@@ -53 +53 @@
                         filesize=`stat -c%s $entry`
 #                       echo $headerSize" "$filesize
-                        numrows=`/home/isdc/lyard/FACT++/fitsdump $entry -h 2>/dev/null | grep 'NAXIS2' | awk '{ print $4 }' | grep -E -o '[0-9]+'`
+                        numrows=`/home_nfs/isdc/lyard/FACT++/fitsdump $entry -h 2>/dev/null | grep 'NAXIS2' | awk '{ print $4 }' | grep -E -o '[0-9]+'`
                         if [ "$numrows" == "" ]
                         then
@@ -60 +60 @@
                                 exit
                         fi
-                        rowWidth=`/home/isdc/lyard/FACT++/fitsdump $entry -h 2>/dev/null | grep 'NAXIS1' | awk '{ print $4 }' | grep -E -o '[0-9]+'`
+                        rowWidth=`/home_nfs/isdc/lyard/FACT++/fitsdump $entry -h 2>/dev/null | grep 'NAXIS1' | awk '{ print $4 }' | grep -E -o '[0-9]+'`
 
                         totSize=`echo " $headerSize + $numrows * $rowWidth " | bc -l`
@@ -67 +67 @@
 #                               echo $headerSize" "$numrows" "$rowWidth" "$totSize" "$fitsSize
 #                               echo " FileSize: "$filesize" should be "$fitsSize
-                        truncate -s $totSize $entry 2>/dev/null
-                        truncate -s $fitsSize $entry 2>/dev/null
-                        echo "Resized   $entry" >> $reportFile
+                        if [ $filesize > $fitsSize ]
+                        then
+                                truncate -s $totSize $entry 2>/dev/null
+                                truncate -s $fitsSize $entry 2>/dev/null
+                                echo "Resized   $entry" >> $reportFile
+                        fi
+                        
                 fi
         fi