Changeset 19201 for trunk/DataCheck


Ignore:
Timestamp:
08/31/18 17:24:07 (6 years ago)
Author:
Daniela Dorner
Message:
usage of check_file_avail function
Location:
trunk/DataCheck/Processing
Files:
9 edited

Legend:

Unmodified
Added
Removed

  • trunk/DataCheck/Processing/FillAuxContTemp.sh

    r18708 r19201  
    8282   # get daily fits files
    8383   conttempfile=$auxdir/$runnumber.TEMPERATURE_DATA.fits
    84    if ! [ -e $conttempfile ]
     84   #if ! [ -e $conttempfile ]
     85   if ! check_file_avail $conttempfile
    8586   then
    86       printprocesslog "WARN "$conttempfile" not found."
     87      #printprocesslog "WARN "$conttempfile" not found."
    8788      continue
    8889   else
    … …  
    101102      # get information from rawfile
    102103      rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.fz
    103       if ! [ -e $rawfile ]
     104      #if ! [ -e $rawfile ]
     105      if ! check_file_avail $rawfile
    104106      then
    105          if [ $runnumber -lt $checknight ]
    106          then
    107             printprocesslog "WARN "$rawfile" not found."
    108          else
    109             printprocesslog "INFO "$rawfile" not found."
    110          fi
     107         #if [ $runnumber -lt $checknight ]
     108         #then
     109         #   printprocesslog "WARN "$rawfile" not found."
     110         #else
     111         #   printprocesslog "INFO "$rawfile" not found."
     112         #fi
    111113         continue
    112114      fi

  • trunk/DataCheck/Processing/FillAuxCtrDev.sh

    r18708 r19201  
    8080   # get daily fits files
    8181   trackingfile=$auxdir/$runnumber.DRIVE_CONTROL_TRACKING_POSITION.fits
    82    if ! [ -e $trackingfile ]
     82   #if ! [ -e $trackingfile ]
     83   if ! check_file_avail $trackingfile
    8384   then
    84       printprocesslog "WARN "$trackingfile" not found."
    85       #echo "WARN "$trackingfile" not found."
     85      #printprocesslog "WARN "$trackingfile" not found."
    8686      continue
    8787   else
    … …  
    100100      # get information from rawfile
    101101      rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.fz
    102       if ! [ -e $rawfile ]
     102      #if ! [ -e $rawfile ]
     103      if ! check_file_avail $rawfile
    103104      then
    104          if [ $runnumber -lt $checknight ]
    105          then
    106             printprocesslog "WARN "$rawfile" not found."
    107          else
    108             printprocesslog "INFO "$rawfile" not found."
    109          fi
     105         #if [ $runnumber -lt $checknight ]
     106         #then
     107         #   printprocesslog "WARN "$rawfile" not found."
     108         #else
     109         #   printprocesslog "INFO "$rawfile" not found."
     110         #fi
    110111         continue
    111112      fi

  • trunk/DataCheck/Processing/FillAuxCurrents.sh

    r18708 r19201  
    8080
    8181   biasvoltagefile=$auxdir/$runnumber.BIAS_CONTROL_VOLTAGE.fits
    82    if ! [ -e $biasvoltagefile ]
    83    then
    84       printprocesslog "WARN "$biasvoltagefile" not found."
     82   #if ! [ -e $biasvoltagefile ]
     83   if ! check_file_avail $biasvoltagefile
     84   then
     85      #printprocesslog "WARN "$biasvoltagefile" not found."
    8586      continue
    8687   else
    … …  
    9394
    9495   biascurrentfile=$auxdir/$runnumber.BIAS_CONTROL_CURRENT.fits
    95    if ! [ -e $biascurrentfile ]
    96    then
    97       printprocesslog "WARN "$biascurrentfile" not found."
     96   #if ! [ -e $biascurrentfile ]
     97   if ! check_file_avail $biascurrentfile
     98   then
     99      #printprocesslog "WARN "$biascurrentfile" not found."
    98100      continue
    99101   else
    … …  
    107109   # this file is needed for the calibration of the currents
    108110   feedbackcalfile=$auxdir/$runnumber.FEEDBACK_CALIBRATION.fits
    109    if ! [ -e $feedbackcalfile ]
    110    then
    111       printprocesslog "WARN "$feedbackcalfile" not found."
     111   #if ! [ -e $feedbackcalfile ]
     112   if ! check_file_avail $feedbackcalfile
     113   then
     114      #printprocesslog "WARN "$feedbackcalfile" not found."
    112115      continue
    113116   else
    … …  
    120123
    121124   calcurrentsfile=$auxdir/$runnumber.FEEDBACK_CALIBRATED_CURRENTS.fits
    122    if ! [ -e $calcurrentsfile ]
     125   #if ! [ -e $calcurrentsfile ]
     126   if ! check_file_avail $calcurrentsfile
    123127   then
    124128      calcurrentsfile=/gpfs/scratch/fact/calibrated_currents/$runnumber.CALIBRATED_CURRENTS.fits
    … …  
    134138   #calcurrentsfile=/scratch_nfs/calibrated_currents/$runnumber.CALIBRATED_CURRENTS.fits
    135139   #calcurrentsfile=/gpfs/scratch/fact/calibrated_currents/$runnumber.CALIBRATED_CURRENTS.fits
    136    if ! [ -e $calcurrentsfile ]
    137    then
    138       printprocesslog "WARN "$calcurrentsfile" not found."
     140   #if ! [ -e $calcurrentsfile ]
     141   if ! check_file_avail $calcurrentsfile
     142   then
     143      #printprocesslog "WARN "$calcurrentsfile" not found."
    139144      continue
    140145   else
    … …  
    153158      # get information from rawfile
    154159      rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.fz
    155       if ! [ -e $rawfile ]
    156       then
    157          if [ $runnumber -lt $checknight ]
    158          then
    159             printprocesslog "WARN "$rawfile" not found."
    160          else
    161             printprocesslog "INFO "$rawfile" not found."
    162          fi
     160      #if ! [ -e $rawfile ]
     161      if ! check_file_avail $rawfile
     162      then
     163#         if [ $runnumber -lt $checknight ]
     164#         then
     165#            printprocesslog "WARN "$rawfile" not found."
     166#         else
     167#            printprocesslog "INFO "$rawfile" not found."
     168#         fi
    163169         continue
    164170      fi

  • trunk/DataCheck/Processing/FillAuxDust.sh

    r18748 r19201  
    7979
    8080   dustfile=$auxdir/$runnumber.TNG_WEATHER_DUST.fits
    81    if ! [ -e $dustfile ]
     81   #if ! [ -e $dustfile ]
     82   if ! check_file_avail $dustfile
    8283   then
    83       printprocesslog "WARN "$dustfile" not found."
     84      #printprocesslog "WARN "$dustfile" not found."
     85      continue
    8486   else
    8587      dustnumerrors=`fverify $dustfile 2>/dev/null | grep -o '[0-9][ ]error(s)'  | grep -E -o '[0-9]'`
    … …  
    9799      # get information from rawfile
    98100      rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.fz
    99       if ! [ -e $rawfile ]
     101      #if ! [ -e $rawfile ]
     102      if ! check_file_avail $rawfile
    100103      then
    101          if [ $runnumber -lt $checknight ]
    102          then
    103             printprocesslog "WARN "$rawfile" not found."
    104          else
    105             printprocesslog "INFO "$rawfile" not found."
    106          fi
     104         #if [ $runnumber -lt $checknight ]
     105         #then
     106         #   printprocesslog "WARN "$rawfile" not found."
     107         #else
     108         #   printprocesslog "INFO "$rawfile" not found."
     109         #fi
    107110         continue
    108111      fi

  • trunk/DataCheck/Processing/FillAuxLidar.sh

    r18768 r19201  
    7979
    8080   lidarfile=$auxdir/$runnumber.MAGIC_LIDAR_DATA.fits
    81    if ! [ -e $lidarfile ]
     81   #if ! [ -e $lidarfile ]
     82   if ! check_file_avail $lidarfile
    8283   then
    83       printprocesslog "WARN "$lidarfile" not found."
     84      #printprocesslog "WARN "$lidarfile" not found."
     85      continue
    8486   else
    8587      lidarnumerrors=`fverify $lidarfile 2>/dev/null | grep -o '[0-9][ ]error(s)'  | grep -E -o '[0-9]'`
    … …  
    9799      # get information from rawfile
    98100      rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.fz
    99       if ! [ -e $rawfile ]
     101      #if ! [ -e $rawfile ]
     102      if ! check_file_avail $rawfile
    100103      then
    101          if [ $runnumber -lt $checknight ]
    102          then
    103             printprocesslog "WARN "$rawfile" not found."
    104          else
    105             printprocesslog "INFO "$rawfile" not found."
    106          fi
     104         #if [ $runnumber -lt $checknight ]
     105         #then
     106         #   printprocesslog "WARN "$rawfile" not found."
     107         #else
     108         #   printprocesslog "INFO "$rawfile" not found."
     109         #fi
    107110         continue
    108111      fi

  • trunk/DataCheck/Processing/FillAuxMagic.sh

    r18944 r19201  
    106106            printprocesslog "INFO "$rawfile" not found."
    107107         fi
    108          continue
     108         printprocesslog "INFO getting information from DB for "$rawfile
     109         # get information from DB
     110         query="SELECT fRunTypeName FROM RunInfo LEFT JOIN RunType USING(fRunTypeKey) WHERE fNight="$runnumber" AND fRunID="$filenum
     111         runtype=`sendquery`
     112         query="SELECT Mjd(fRunStart)-40587 FROM RunInfo WHERE fNight="$runnumber" AND fRunID="$filenum
     113         tstart=`sendquery`
     114         query="SELECT Mjd(fRunStop)-40587 FROM RunInfo WHERE fNight="$runnumber" AND fRunID="$filenum
     115         tstop=`sendquery`
     116         query="SELECT Mjd(fRunStart)-40587-0.00011574 FROM RunInfo WHERE fNight="$runnumber" AND fRunID="$filenum
     117         tstart2=`sendquery`
     118         query="SELECT Mjd(fRunStop)-40587-0.00011574 FROM RunInfo WHERE fNight="$runnumber" AND fRunID="$filenum
     119         tstop2=`sendquery`
     120      else
     121         #checkfitsfile=`fverify $rawfile  2>/dev/null | grep '0 error(s)'`
     122         #if [ "$checkfitsfile" == "" ]
     123         #then
     124         #   numfitserrors=1
     125         #   printprocesslog "WARN: "$rawfile" probably corrupted."
     126         #   continue
     127         #fi
     128         runtype=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"`
     129         mjdrefraw=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
     130         tstarti=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
     131         tstartf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
     132         tstopi=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
     133         tstopf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
     134         if [ "$tstarti" == "" ] || [ "$tstopi" == "" ] || [ "$tstartf" == "" ] || [ "$tstopf" == "" ]
     135         then
     136            printprocesslog "WARN: "$rawfile": one of the following keywords is empty or 0: TSTARTI TSTARTF TSTOPI TSTOPF "
     137            continue
     138         fi
     139         # assuming that at least TSTARTI and TSTOPI are consistent
     140         if [ $tstarti -gt 30000 ]
     141         then
     142            tstart=`echo " $tstarti + $tstartf - 40587 " | bc -l`
     143            tstart2=`echo " $tstarti + $tstartf - 40587 - 0.00011574 " | bc -l`  # 10 sec
     144            #tstart2=`echo " $tstarti + $tstartf - 40587 - 0.000023148 " | bc -l` # 2 sec
     145            tstop=`echo " $tstopi + $tstopf - 40587 " | bc -l`
     146         else
     147            tstart=`echo " $tstarti + $tstartf " | bc -l`
     148            tstart2=`echo " $tstarti + $tstartf - 0.00011574 " | bc -l`  # 10 sec
     149            #tstart2=`echo " $tstarti + $tstartf - 0.000023148 " | bc -l` # 2 sec
     150            tstop=`echo " $tstopi + $tstopf " | bc -l`
     151         fi
     152         #echo  `date`": processing file number "$runnumber"_"`printf %03d $filenum`" "$runtype
    109153      fi
    110       #checkfitsfile=`fverify $rawfile  2>/dev/null | grep '0 error(s)'`
    111       #if [ "$checkfitsfile" == "" ]
    112       #then
    113       #   numfitserrors=1
    114       #   printprocesslog "WARN: "$rawfile" probably corrupted."
    115       #   continue
    116       #fi
    117       runtype=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"`
    118       mjdrefraw=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
    119       tstarti=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
    120       tstartf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
    121       tstopi=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
    122       tstopf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
    123       if [ "$tstarti" == "" ] || [ "$tstopi" == "" ] || [ "$tstartf" == "" ] || [ "$tstopf" == "" ]
    124       then
    125          printprocesslog "WARN: "$rawfile": one of the following keywords is empty or 0: TSTARTI TSTARTF TSTOPI TSTOPF "
    126          continue
    127       fi
    128       # assuming that at least TSTARTI and TSTOPI are consistent
    129       if [ $tstarti -gt 30000 ]
    130       then
    131          tstart=`echo " $tstarti + $tstartf - 40587 " | bc -l`
    132          tstart2=`echo " $tstarti + $tstartf - 40587 - 0.00011574 " | bc -l`  # 10 sec
    133          #tstart2=`echo " $tstarti + $tstartf - 40587 - 0.000023148 " | bc -l` # 2 sec
    134          tstop=`echo " $tstopi + $tstopf - 40587 " | bc -l`
    135       else
    136          tstart=`echo " $tstarti + $tstartf " | bc -l`
    137          tstart2=`echo " $tstarti + $tstartf - 0.00011574 " | bc -l`  # 10 sec
    138          #tstart2=`echo " $tstarti + $tstartf - 0.000023148 " | bc -l` # 2 sec
    139          tstop=`echo " $tstopi + $tstopf " | bc -l`
    140       fi
    141       #echo  `date`": processing file number "$runnumber"_"`printf %03d $filenum`" "$runtype
    142154
    143155      # build query to update runinfo in DB

  • trunk/DataCheck/Processing/FillAuxThresholds.sh

    r18708 r19201  
    7070   then
    7171      printprocesslog "INFO no data available in "$auxdir
     72      echo "INFO no data available in "$auxdir
    7273      continue
    7374   else
    … …  
    7879   printprocesslog "INFO processing "$thresholdfile
    7980   echo "INFO processing "$thresholdfile >> $logfile 2>&1
    80    if ! [ -e $thresholdfile ]
     81   #if ! [ -e $thresholdfile ]
     82   if ! check_file_avail $thresholdfile
    8183   then
    82       printprocesslog "WARN "$thresholdfile" not found."
     84      #printprocesslog "WARN "$thresholdfile" not found."
    8385      continue
    8486   else
    … …  
    9193
    9294   thresholdfile2=$auxdir/$runnumber.FTM_CONTROL_STATIC_DATA.fits
    93    if ! [ -e $thresholdfile2 ]
     95   #if ! [ -e $thresholdfile2 ]
     96   if ! check_file_avail $thresholdfile2
    9497   then
    95       printprocesslog "WARN "$thresholdfile2" not found."
     98      #printprocesslog "WARN "$thresholdfile2" not found."
    9699      continue
    97100   else
    … …  
    110113      # get information from rawfile
    111114      rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.fz
    112       if ! [ -e $rawfile ]
    113       then
    114          if [ $runnumber -lt $checknight ]
    115          then
    116             printprocesslog "WARN "$rawfile" not found."
    117          else
    118             printprocesslog "INFO "$rawfile" not found."
    119          fi
     115      #if ! [ -e $rawfile ]
     116      if ! check_file_avail $rawfile
     117      then
     118         #if [ $runnumber -lt $checknight ]
     119         #then
     120         #   printprocesslog "WARN "$rawfile" not found."
     121         #else
     122         #   printprocesslog "INFO "$rawfile" not found."
     123         #fi
    120124         continue
    121125      fi

  • trunk/DataCheck/Processing/FillDrsTemp.sh

    r18708 r19201  
    7979
    8080   drstempfile=$auxdir/$runnumber.FAD_CONTROL_TEMPERATURE.fits
    81    if ! [ -e $drstempfile ]
     81   #if ! [ -e $drstempfile ]
     82   if ! check_file_avail $drstempfile
    8283   then
    83       printprocesslog "WARN "$drstempfile" not found."
    84       #echo "WARN "$drstempfile" not found."
     84      #printprocesslog "WARN "$drstempfile" not found."
     85      continue
    8586   else
    8687      tempnumerrors=`fverify $drstempfile 2>/dev/null | grep -o '[0-9][ ]error(s)'  | grep -E -o '[0-9]'`
    … …  
    9899      # get information from rawfile
    99100      rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.fz
    100       if ! [ -e $rawfile ]
     101      #if ! [ -e $rawfile ]
     102      if ! check_file_avail $rawfile
    101103      then
    102          if [ $runnumber -lt $checknight ]
    103          then
    104             printprocesslog "WARN "$rawfile" not found. "$night" "$checknight
    105          else
    106             printprocesslog "INFO "$rawfile" not found."
    107          fi
     104         #if [ $runnumber -lt $checknight ]
     105         #then
     106         #   printprocesslog "WARN "$rawfile" not found. "$night" "$checknight
     107         #else
     108         #   printprocesslog "INFO "$rawfile" not found."
     109         #fi
    108110         continue
    109111      fi

  • trunk/DataCheck/Processing/FillEffectiveOn.sh

    r18708 r19201  
    7575   
    7676   ftmcontrolfile=$auxdir/$runnumber.FTM_CONTROL_TRIGGER_RATES.fits
    77    if ! [ -e $ftmcontrolfile ]
     77   #if ! [ -e $ftmcontrolfile ]
     78   if ! check_file_avail $ftmcontrolfile
    7879   then
    79       printprocesslog "WARN "$ftmcontrolfile" not found."
     80      #printprocesslog "WARN "$ftmcontrolfile" not found."
    8081      continue
    8182   else
    … …  
    9495      # get information from rawfile
    9596      rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.fz
    96       if ! [ -e $rawfile ]
    97       then
    98          if [ $runnumber -lt $checknight ]
    99          then
    100             printprocesslog "WARN "$rawfile" not found."
    101          else
    102             printprocesslog "INFO "$rawfile" not found."
    103          fi
     97      #if ! [ -e $rawfile ]
     98      if ! check_file_avail $rawfile
     99      then
     100         #if [ $runnumber -lt $checknight ]
     101         #then
     102         #   printprocesslog "WARN "$rawfile" not found."
     103         #else
     104         #   printprocesslog "INFO "$rawfile" not found."
     105         #fi
    104106         continue
    105107      fi
Note: See TracChangeset for help on using the changeset viewer.