Index: trunk/DataCheck/Processing/FillAuxContTemp.sh
===================================================================
--- trunk/DataCheck/Processing/FillAuxContTemp.sh	(revision 19200)
+++ trunk/DataCheck/Processing/FillAuxContTemp.sh	(revision 19201)
@@ -82,7 +82,8 @@
    # get daily fits files 
    conttempfile=$auxdir/$runnumber.TEMPERATURE_DATA.fits
-   if ! [ -e $conttempfile ]
+   #if ! [ -e $conttempfile ]
+   if ! check_file_avail $conttempfile
    then 
-      printprocesslog "WARN "$conttempfile" not found."
+      #printprocesslog "WARN "$conttempfile" not found."
       continue
    else
@@ -101,12 +102,13 @@
       # get information from rawfile
       rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.fz
-      if ! [ -e $rawfile ]
+      #if ! [ -e $rawfile ]
+      if ! check_file_avail $rawfile
       then 
-         if [ $runnumber -lt $checknight ] 
-         then 
-            printprocesslog "WARN "$rawfile" not found."
-         else
-            printprocesslog "INFO "$rawfile" not found."
-         fi
+         #if [ $runnumber -lt $checknight ] 
+         #then 
+         #   printprocesslog "WARN "$rawfile" not found."
+         #else
+         #   printprocesslog "INFO "$rawfile" not found."
+         #fi
          continue
       fi
Index: trunk/DataCheck/Processing/FillAuxCtrDev.sh
===================================================================
--- trunk/DataCheck/Processing/FillAuxCtrDev.sh	(revision 19200)
+++ trunk/DataCheck/Processing/FillAuxCtrDev.sh	(revision 19201)
@@ -80,8 +80,8 @@
    # get daily fits files 
    trackingfile=$auxdir/$runnumber.DRIVE_CONTROL_TRACKING_POSITION.fits
-   if ! [ -e $trackingfile ]
+   #if ! [ -e $trackingfile ]
+   if ! check_file_avail $trackingfile
    then 
-      printprocesslog "WARN "$trackingfile" not found."
-      #echo "WARN "$trackingfile" not found."
+      #printprocesslog "WARN "$trackingfile" not found."
       continue
    else
@@ -100,12 +100,13 @@
       # get information from rawfile
       rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.fz
-      if ! [ -e $rawfile ]
+      #if ! [ -e $rawfile ]
+      if ! check_file_avail $rawfile
       then 
-         if [ $runnumber -lt $checknight ] 
-         then 
-            printprocesslog "WARN "$rawfile" not found."
-         else
-            printprocesslog "INFO "$rawfile" not found."
-         fi
+         #if [ $runnumber -lt $checknight ] 
+         #then 
+         #   printprocesslog "WARN "$rawfile" not found."
+         #else
+         #   printprocesslog "INFO "$rawfile" not found."
+         #fi
          continue
       fi
Index: trunk/DataCheck/Processing/FillAuxCurrents.sh
===================================================================
--- trunk/DataCheck/Processing/FillAuxCurrents.sh	(revision 19200)
+++ trunk/DataCheck/Processing/FillAuxCurrents.sh	(revision 19201)
@@ -80,7 +80,8 @@
 
    biasvoltagefile=$auxdir/$runnumber.BIAS_CONTROL_VOLTAGE.fits
-   if ! [ -e $biasvoltagefile ]
-   then 
-      printprocesslog "WARN "$biasvoltagefile" not found."
+   #if ! [ -e $biasvoltagefile ]
+   if ! check_file_avail $biasvoltagefile 
+   then 
+      #printprocesslog "WARN "$biasvoltagefile" not found."
       continue
    else
@@ -93,7 +94,8 @@
 
    biascurrentfile=$auxdir/$runnumber.BIAS_CONTROL_CURRENT.fits
-   if ! [ -e $biascurrentfile ]
-   then 
-      printprocesslog "WARN "$biascurrentfile" not found."
+   #if ! [ -e $biascurrentfile ]
+   if ! check_file_avail $biascurrentfile
+   then 
+      #printprocesslog "WARN "$biascurrentfile" not found."
       continue
    else
@@ -107,7 +109,8 @@
    # this file is needed for the calibration of the currents
    feedbackcalfile=$auxdir/$runnumber.FEEDBACK_CALIBRATION.fits
-   if ! [ -e $feedbackcalfile ]
-   then 
-      printprocesslog "WARN "$feedbackcalfile" not found."
+   #if ! [ -e $feedbackcalfile ]
+   if ! check_file_avail $feedbackcalfile
+   then 
+      #printprocesslog "WARN "$feedbackcalfile" not found."
       continue
    else
@@ -120,5 +123,6 @@
 
    calcurrentsfile=$auxdir/$runnumber.FEEDBACK_CALIBRATED_CURRENTS.fits
-   if ! [ -e $calcurrentsfile ]
+   #if ! [ -e $calcurrentsfile ]
+   if ! check_file_avail $calcurrentsfile
    then 
       calcurrentsfile=/gpfs/scratch/fact/calibrated_currents/$runnumber.CALIBRATED_CURRENTS.fits
@@ -134,7 +138,8 @@
    #calcurrentsfile=/scratch_nfs/calibrated_currents/$runnumber.CALIBRATED_CURRENTS.fits
    #calcurrentsfile=/gpfs/scratch/fact/calibrated_currents/$runnumber.CALIBRATED_CURRENTS.fits
-   if ! [ -e $calcurrentsfile ]
-   then 
-      printprocesslog "WARN "$calcurrentsfile" not found."
+   #if ! [ -e $calcurrentsfile ]
+   if ! check_file_avail $calcurrentsfile
+   then 
+      #printprocesslog "WARN "$calcurrentsfile" not found."
       continue
    else
@@ -153,12 +158,13 @@
       # get information from rawfile
       rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.fz
-      if ! [ -e $rawfile ]
-      then 
-         if [ $runnumber -lt $checknight ] 
-         then 
-            printprocesslog "WARN "$rawfile" not found."
-         else
-            printprocesslog "INFO "$rawfile" not found."
-         fi
+      #if ! [ -e $rawfile ]
+      if ! check_file_avail $rawfile
+      then 
+#         if [ $runnumber -lt $checknight ] 
+#         then 
+#            printprocesslog "WARN "$rawfile" not found."
+#         else
+#            printprocesslog "INFO "$rawfile" not found."
+#         fi
          continue
       fi
Index: trunk/DataCheck/Processing/FillAuxDust.sh
===================================================================
--- trunk/DataCheck/Processing/FillAuxDust.sh	(revision 19200)
+++ trunk/DataCheck/Processing/FillAuxDust.sh	(revision 19201)
@@ -79,7 +79,9 @@
 
    dustfile=$auxdir/$runnumber.TNG_WEATHER_DUST.fits
-   if ! [ -e $dustfile ]
+   #if ! [ -e $dustfile ]
+   if ! check_file_avail $dustfile
    then 
-      printprocesslog "WARN "$dustfile" not found."
+      #printprocesslog "WARN "$dustfile" not found."
+      continue
    else
       dustnumerrors=`fverify $dustfile 2>/dev/null | grep -o '[0-9][ ]error(s)'  | grep -E -o '[0-9]'`
@@ -97,12 +99,13 @@
       # get information from rawfile
       rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.fz
-      if ! [ -e $rawfile ]
+      #if ! [ -e $rawfile ]
+      if ! check_file_avail $rawfile
       then 
-         if [ $runnumber -lt $checknight ] 
-         then 
-            printprocesslog "WARN "$rawfile" not found."
-         else
-            printprocesslog "INFO "$rawfile" not found."
-         fi
+         #if [ $runnumber -lt $checknight ] 
+         #then 
+         #   printprocesslog "WARN "$rawfile" not found."
+         #else
+         #   printprocesslog "INFO "$rawfile" not found."
+         #fi
          continue
       fi
Index: trunk/DataCheck/Processing/FillAuxLidar.sh
===================================================================
--- trunk/DataCheck/Processing/FillAuxLidar.sh	(revision 19200)
+++ trunk/DataCheck/Processing/FillAuxLidar.sh	(revision 19201)
@@ -79,7 +79,9 @@
 
    lidarfile=$auxdir/$runnumber.MAGIC_LIDAR_DATA.fits
-   if ! [ -e $lidarfile ]
+   #if ! [ -e $lidarfile ]
+   if ! check_file_avail $lidarfile
    then 
-      printprocesslog "WARN "$lidarfile" not found."
+      #printprocesslog "WARN "$lidarfile" not found."
+      continue
    else
       lidarnumerrors=`fverify $lidarfile 2>/dev/null | grep -o '[0-9][ ]error(s)'  | grep -E -o '[0-9]'`
@@ -97,12 +99,13 @@
       # get information from rawfile
       rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.fz
-      if ! [ -e $rawfile ]
+      #if ! [ -e $rawfile ]
+      if ! check_file_avail $rawfile
       then 
-         if [ $runnumber -lt $checknight ] 
-         then 
-            printprocesslog "WARN "$rawfile" not found."
-         else
-            printprocesslog "INFO "$rawfile" not found."
-         fi
+         #if [ $runnumber -lt $checknight ] 
+         #then 
+         #   printprocesslog "WARN "$rawfile" not found."
+         #else
+         #   printprocesslog "INFO "$rawfile" not found."
+         #fi
          continue
       fi
Index: trunk/DataCheck/Processing/FillAuxMagic.sh
===================================================================
--- trunk/DataCheck/Processing/FillAuxMagic.sh	(revision 19200)
+++ trunk/DataCheck/Processing/FillAuxMagic.sh	(revision 19201)
@@ -106,38 +106,50 @@
             printprocesslog "INFO "$rawfile" not found."
          fi
-         continue
+         printprocesslog "INFO getting information from DB for "$rawfile
+         # get information from DB
+         query="SELECT fRunTypeName FROM RunInfo LEFT JOIN RunType USING(fRunTypeKey) WHERE fNight="$runnumber" AND fRunID="$filenum
+         runtype=`sendquery`
+         query="SELECT Mjd(fRunStart)-40587 FROM RunInfo WHERE fNight="$runnumber" AND fRunID="$filenum
+         tstart=`sendquery`
+         query="SELECT Mjd(fRunStop)-40587 FROM RunInfo WHERE fNight="$runnumber" AND fRunID="$filenum
+         tstop=`sendquery`
+         query="SELECT Mjd(fRunStart)-40587-0.00011574 FROM RunInfo WHERE fNight="$runnumber" AND fRunID="$filenum
+         tstart2=`sendquery`
+         query="SELECT Mjd(fRunStop)-40587-0.00011574 FROM RunInfo WHERE fNight="$runnumber" AND fRunID="$filenum
+         tstop2=`sendquery`
+      else
+         #checkfitsfile=`fverify $rawfile  2>/dev/null | grep '0 error(s)'`
+         #if [ "$checkfitsfile" == "" ]
+         #then
+         #   numfitserrors=1
+         #   printprocesslog "WARN: "$rawfile" probably corrupted."
+         #   continue
+         #fi
+         runtype=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"`
+         mjdrefraw=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
+         tstarti=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
+         tstartf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
+         tstopi=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
+         tstopf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
+         if [ "$tstarti" == "" ] || [ "$tstopi" == "" ] || [ "$tstartf" == "" ] || [ "$tstopf" == "" ]
+         then
+            printprocesslog "WARN: "$rawfile": one of the following keywords is empty or 0: TSTARTI TSTARTF TSTOPI TSTOPF "
+            continue
+         fi
+         # assuming that at least TSTARTI and TSTOPI are consistent
+         if [ $tstarti -gt 30000 ]
+         then
+            tstart=`echo " $tstarti + $tstartf - 40587 " | bc -l`
+            tstart2=`echo " $tstarti + $tstartf - 40587 - 0.00011574 " | bc -l`  # 10 sec
+            #tstart2=`echo " $tstarti + $tstartf - 40587 - 0.000023148 " | bc -l` # 2 sec
+            tstop=`echo " $tstopi + $tstopf - 40587 " | bc -l`
+         else
+            tstart=`echo " $tstarti + $tstartf " | bc -l`
+            tstart2=`echo " $tstarti + $tstartf - 0.00011574 " | bc -l`  # 10 sec
+            #tstart2=`echo " $tstarti + $tstartf - 0.000023148 " | bc -l` # 2 sec
+            tstop=`echo " $tstopi + $tstopf " | bc -l`
+         fi
+         #echo  `date`": processing file number "$runnumber"_"`printf %03d $filenum`" "$runtype
       fi
-      #checkfitsfile=`fverify $rawfile  2>/dev/null | grep '0 error(s)'`
-      #if [ "$checkfitsfile" == "" ]
-      #then
-      #   numfitserrors=1
-      #   printprocesslog "WARN: "$rawfile" probably corrupted."
-      #   continue
-      #fi
-      runtype=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep RUNTYPE | grep -E -o "['][a-z-]+[']" | sed -e "s/'//g"`
-      mjdrefraw=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'MJDREF' | grep -E -o '[0-9]{5}'`
-      tstarti=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTI' | grep -E -o '[0-9]{5}'`
-      tstartf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTARTF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
-      tstopi=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPI' | grep -E -o '[0-9]{5}'`
-      tstopf=`$factpath/fitsdump -h $rawfile  2>/dev/null | grep 'TSTOPF' | grep -E -o '[0-9][.][0-9]+([E][\-][0-9][0-9])?' | sed -e 's/[E]+*/\\*10\\^/'`
-      if [ "$tstarti" == "" ] || [ "$tstopi" == "" ] || [ "$tstartf" == "" ] || [ "$tstopf" == "" ]
-      then 
-         printprocesslog "WARN: "$rawfile": one of the following keywords is empty or 0: TSTARTI TSTARTF TSTOPI TSTOPF "
-         continue
-      fi
-      # assuming that at least TSTARTI and TSTOPI are consistent
-      if [ $tstarti -gt 30000 ]
-      then 
-         tstart=`echo " $tstarti + $tstartf - 40587 " | bc -l`
-         tstart2=`echo " $tstarti + $tstartf - 40587 - 0.00011574 " | bc -l`  # 10 sec
-         #tstart2=`echo " $tstarti + $tstartf - 40587 - 0.000023148 " | bc -l` # 2 sec
-         tstop=`echo " $tstopi + $tstopf - 40587 " | bc -l`
-      else
-         tstart=`echo " $tstarti + $tstartf " | bc -l`
-         tstart2=`echo " $tstarti + $tstartf - 0.00011574 " | bc -l`  # 10 sec
-         #tstart2=`echo " $tstarti + $tstartf - 0.000023148 " | bc -l` # 2 sec
-         tstop=`echo " $tstopi + $tstopf " | bc -l`
-      fi
-      #echo  `date`": processing file number "$runnumber"_"`printf %03d $filenum`" "$runtype
 
       # build query to update runinfo in DB
Index: trunk/DataCheck/Processing/FillAuxThresholds.sh
===================================================================
--- trunk/DataCheck/Processing/FillAuxThresholds.sh	(revision 19200)
+++ trunk/DataCheck/Processing/FillAuxThresholds.sh	(revision 19201)
@@ -70,4 +70,5 @@
    then
       printprocesslog "INFO no data available in "$auxdir
+      echo "INFO no data available in "$auxdir
       continue
    else
@@ -78,7 +79,8 @@
    printprocesslog "INFO processing "$thresholdfile
    echo "INFO processing "$thresholdfile >> $logfile 2>&1
-   if ! [ -e $thresholdfile ]
+   #if ! [ -e $thresholdfile ]
+   if ! check_file_avail $thresholdfile
    then 
-      printprocesslog "WARN "$thresholdfile" not found."
+      #printprocesslog "WARN "$thresholdfile" not found."
       continue
    else
@@ -91,7 +93,8 @@
 
    thresholdfile2=$auxdir/$runnumber.FTM_CONTROL_STATIC_DATA.fits
-   if ! [ -e $thresholdfile2 ]
+   #if ! [ -e $thresholdfile2 ]
+   if ! check_file_avail $thresholdfile2
    then 
-      printprocesslog "WARN "$thresholdfile2" not found."
+      #printprocesslog "WARN "$thresholdfile2" not found."
       continue
    else
@@ -110,12 +113,13 @@
       # get information from rawfile
       rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.fz
-      if ! [ -e $rawfile ]
-      then 
-         if [ $runnumber -lt $checknight ] 
-         then 
-            printprocesslog "WARN "$rawfile" not found."
-         else
-            printprocesslog "INFO "$rawfile" not found."
-         fi
+      #if ! [ -e $rawfile ]
+      if ! check_file_avail $rawfile
+      then 
+         #if [ $runnumber -lt $checknight ] 
+         #then 
+         #   printprocesslog "WARN "$rawfile" not found."
+         #else
+         #   printprocesslog "INFO "$rawfile" not found."
+         #fi
          continue
       fi
Index: trunk/DataCheck/Processing/FillDrsTemp.sh
===================================================================
--- trunk/DataCheck/Processing/FillDrsTemp.sh	(revision 19200)
+++ trunk/DataCheck/Processing/FillDrsTemp.sh	(revision 19201)
@@ -79,8 +79,9 @@
 
    drstempfile=$auxdir/$runnumber.FAD_CONTROL_TEMPERATURE.fits
-   if ! [ -e $drstempfile ]
+   #if ! [ -e $drstempfile ]
+   if ! check_file_avail $drstempfile
    then 
-      printprocesslog "WARN "$drstempfile" not found."
-      #echo "WARN "$drstempfile" not found."
+      #printprocesslog "WARN "$drstempfile" not found."
+      continue
    else
       tempnumerrors=`fverify $drstempfile 2>/dev/null | grep -o '[0-9][ ]error(s)'  | grep -E -o '[0-9]'`
@@ -98,12 +99,13 @@
       # get information from rawfile
       rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.fz
-      if ! [ -e $rawfile ]
+      #if ! [ -e $rawfile ]
+      if ! check_file_avail $rawfile
       then 
-         if [ $runnumber -lt $checknight ] 
-         then 
-            printprocesslog "WARN "$rawfile" not found. "$night" "$checknight
-         else
-            printprocesslog "INFO "$rawfile" not found."
-         fi
+         #if [ $runnumber -lt $checknight ] 
+         #then 
+         #   printprocesslog "WARN "$rawfile" not found. "$night" "$checknight
+         #else
+         #   printprocesslog "INFO "$rawfile" not found."
+         #fi
          continue
       fi
Index: trunk/DataCheck/Processing/FillEffectiveOn.sh
===================================================================
--- trunk/DataCheck/Processing/FillEffectiveOn.sh	(revision 19200)
+++ trunk/DataCheck/Processing/FillEffectiveOn.sh	(revision 19201)
@@ -75,7 +75,8 @@
    
    ftmcontrolfile=$auxdir/$runnumber.FTM_CONTROL_TRIGGER_RATES.fits
-   if ! [ -e $ftmcontrolfile ]
+   #if ! [ -e $ftmcontrolfile ]
+   if ! check_file_avail $ftmcontrolfile
    then 
-      printprocesslog "WARN "$ftmcontrolfile" not found."
+      #printprocesslog "WARN "$ftmcontrolfile" not found."
       continue
    else
@@ -94,12 +95,13 @@
       # get information from rawfile
       rawfile=$ziprawdata/$date/$runnumber"_"`printf %03d $filenum`.fits.fz
-      if ! [ -e $rawfile ]
-      then 
-         if [ $runnumber -lt $checknight ] 
-         then 
-            printprocesslog "WARN "$rawfile" not found."
-         else
-            printprocesslog "INFO "$rawfile" not found."
-         fi
+      #if ! [ -e $rawfile ]
+      if ! check_file_avail $rawfile
+      then 
+         #if [ $runnumber -lt $checknight ] 
+         #then 
+         #   printprocesslog "WARN "$rawfile" not found."
+         #else
+         #   printprocesslog "INFO "$rawfile" not found."
+         #fi
          continue
       fi