source: trunk/DataCheck/Monitoring/CheckSwiftSchedule.sh@ 19547

#!/bin/bash


source `dirname $0`/../Sourcefile.sh
printprocesslog "INFO starting $0"

# list of sources
sources=( "MRK 421" "Mrk 421" "Mkn 421" "Mkn421" \
          "1959" "V404" \
          "1H 0323+342" "RXJ0324.6+3410" "RGBJ0324+3410" "SWIFTJ0325.3-5916" 
          "Crab" "CRAB" \
          "Mrk 501" "Mkn 501" \
          "1ES 2344+514" "1ES2344+514" )
          
# target ids: 
# http://www.swift.ac.uk/swift_live/index.php

# 0323: 90415, 36533, 34137, 35372, 35630 
# http://www.swift.ac.uk/swift_live/doSimpleSearch.php?catname=swiftmastr&searchpos=1H+0323%2B342&searchrad=12&searchepoch=2000&getWhat=1&sortBy=_r&dandr=on&coordType=0&retEpoch=2000&retType=0&retWhere=1

# Mrk421: 30352, 35014, 215769, 31202, 80050, 31540, 91137, 34228, tbc
# http://www.swift.ac.uk/swift_live/doSimpleSearch.php?catname=swiftmastr&searchpos=Mrk+421&searchrad=12&searchepoch=2000&getWhat=1&sortBy=_r&dandr=on&coordType=0&retEpoch=2000&retType=0&retWhere=2

# get dates
if [ "$certaindate" != "" ]
then
   getdates $certaindate
else
   if [ "$1" = "" ]
   then 
      # get next 3 night
      getdates +3
   else
      getdates $1
   fi
fi


function getschedule()
{
   text=$( { links -width 300 -ssl.certificates 0 -dump https://www.swift.psu.edu/operations/obsSchedule.php?d=$1 | grep $1'\|was not found\|Connection' >> $schedulefile ; } 2>&1 )
   #text=`links -width 300 -ssl.certificates 0 -dump https://www.swift.psu.edu/operations/obsSchedule.php?d=$1 | grep $1 >> $schedulefile `
   check1=$?
   if [ "$text" != "" ] || [ $check1 -gt 0 ]
   then
      printprocesslog "links -width 300 -ssl.certificates 0 -dump https://www.swift.psu.edu/operations/obsSchedule.php?d="$1" | grep "$1" >> "$schedulefile 
      printprocesslog "WARN could not get Swift schedule ("$text", return code: "$check1")"
      echo "WARN could not get Swift schedule ("$text", return code: "$check1")"
      if [ "$text" = "Connection timed out" ] || [ "$text" = "Connection refused" ]
      then
         printprocesslog "WARN could not connect to https://www.swift.psu.edu/operations/obsSchedule.php?d="$1
         finish
      fi
   fi
}


for date in ${dates[@]}
do 
   printprocesslog "INFO getting Swift schedule for the night of "$date
   #echo "INFO getting Swift schedule for the night of "$date
   # paths and files needed to store schedule
   date2=`echo $date | sed -e 's/\//-/g'`
   schedulepath=/home/factwww/scheduling/$date
   if [ ! -d $schedulepath ]
   then
      printprocesslog "WARNING "$schedulingpath" does not exist."
      echo "WARNING "$schedulingpath" does not exist."
      continue
   fi
   schedulefile=$schedulepath/swift.schedule
   schedulefile2=$schedulepath/swift.schedule.formated
   schedulefile2old=$schedulepath/swift.schedule.formated.old
   schedulefile3=$schedulepath/swift.schedule.txt
   schedulefile4=$schedulepath/swift.schedule-cut.txt
   
   # overwrite already existing file
   echo "Schedule retrieved at "`date` > $schedulefile
   # Get Swift Data
   #   for the date of sunset
   printprocesslog "INFO Checking "$date2
   #lynx -width 200 -nolist -dump https://www.swift.psu.edu/operations/obsSchedule.php?d=$date2 | grep $date2 >> $schedulefile
   #links -width 300 -ssl.certificates 0 -dump https://www.swift.psu.edu/operations/obsSchedule.php?d=$date2 | grep $date2 >> $schedulefile
   getschedule $date2
   #   for the date of sunrise
   nextday=`date +%Y-%m-%d --date=$date2"+24hour"`
   printprocesslog "INFO Checking "$nextday
   #lynx -width 200 -nolist -dump https://www.swift.psu.edu/operations/obsSchedule.php?d=$nextday | grep $nextday >> $schedulefile
   #links -width 300 -ssl.certificates 0 -dump https://www.swift.psu.edu/operations/obsSchedule.php?d=$nextday | grep $nextday >> $schedulefile
   getschedule $nextday

   # check if schedule is empty
   #checkcontent=`cat $schedulefile`
   #if [ "$checkcontent" = "" ]
   #then
   #   printprocesslog "INFO No Schedule found for the night of "$date
   #   continue
   #fi
   numnotfound=`grep -c 'was not found' $schedulefile`
   if [ $numnotfound -gt 1 ]
   then 
      printprocesslog "INFO No Schedule found for the night of "$date
      #echo "INFO No Schedule found for the night of "$date
      continue
   fi
   if [ $numnotfound -eq 1 ]
   then 
      printprocesslog "INFO No Schedule found half of the night ("$date")"
      #echo "INFO No Schedule found half of the night ("$date")"
   fi
   
   # copy of formatted schedulefile for comparison
   if [ -e $schedulefile2 ]
   then
      mv $schedulefile2 $schedulefile2old
   fi
   # grep only the columns which are needed
   #cat $schedulefile | sed -r -e 's/[0-9]{5,6}[\ ]{1,4}[0-9]{1,3}[\ ]{1,6}//' | grep -o -E '^[\ ]{1,20}20[1-9][0-9]-[01][0-9]-[0-3][0-9]\ [0-9][0-9]:[0-9][0-9]:[0-9][0-9]\ \ \ 20[1-9][0-9]-[01][0-9]-[0-3][0-9]\ [0-9][0-9]:[0-9][0-9]:[0-9][0-9]\ \ \ [A-Z1-4]([A-Za-z0-9][\ \.\+\-]?){3,20}' > $schedulefile2
   cat $schedulefile | awk -v d1=$date2 -v d2=$nextday '{ if (($1 ~ d1 && $2 ~ /^18|^19|^20|^21|^22|^23/) || ( $1 ~ d2 && $2 ~ /^0/)) print $1" "$2" "$3" "$4" "$7" "$8" "$9}' | sed -r -e 's/[-]?[0-9]{1,3}[.][0-9]{5}$//' | sed -r -e 's/[-]?[0-9]{1,3}[.][0-9]{5}\ $//' > $schedulefile2
   
   # check if there is an update in the schedule
   # return values of diff
   #   0: no differences found
   #   1: differences found
   #  >1: error
   upd=""
   if [ -e $schedulefile2old ]
   then
      diff $schedulefile2 $schedulefile2old >/dev/null
      checkdiff=$?
      if [ $checkdiff -eq 0 ]
      then
         printprocesslog "INFO Schedule did not change."
         continue
      else
         update="(upd)"
      fi
   else
      update="(new)"
   fi
   
   # output list of sources
   echo -n "Searched the schedule for the following source names: " >$schedulefile3
   for (( i=0; i<${#sources[@]}; i++ ))
   do
      echo -n ${sources[$i]} >>$schedulefile3
      i2=`echo " $i + 1" | bc -l`
      if [ $i2 -lt ${#sources[@]} ]
      then
         echo -n ", " >>$schedulefile3
      else
         echo "" >>$schedulefile3
      fi
   done
   
   printprocesslog "INFO searching the Swift schedule for the following sources: "${sources[@]}
   # search for source names in schedule file
   for (( i=0; i<${#sources[@]}; i++ ))
   do
      grep "${sources[$i]}" $schedulefile2  >>$schedulefile3
   done
   
   # check if a source was found
   numfound=`cat $schedulefile3 | wc -l`
   if [ $numfound -eq 1 ]
   then
      echo -e "\nNone of the sources was found in the Swift schedule." >> $schedulefile3
   else
      echo "night of "$date": "$update && cat $schedulefile3 | grep -v Searched | sort
   fi

   # output link to full schedule
   echo -e "\nThe full schedule can be found <a href='../scheduling/"$date"/"`basename $schedulefile2`"'>here</a>\n" >>$schedulefile3
   
   printprocesslog "INFO schedule output stored to "$schedulefile3
done

finish