Return to tdsynop CVS log

Up to [Development] / JSOC / proj / timed / scripts

modified for JSOC release 8.7

#!/bin/csh -f
# script to run a single instance of trackings plus analyses in the 
#   time-distance synoptic series for a given time center
# This script should be run every 8 hours

set TW
@ TEST = 0	# set to 1 for production in namespace hmi_test
if ("$1" =~ '-test') then
  @ TEST = 1
  set TW = test
  shift
endif

if ($#argv < 2) then
  echo "usage: $0 [-test] YYMMDD HR"
  exit
endif
@ YYMMDD = `echo $1 | sed  -e "s/^0*//"`
@ HR = `echo $2 | sed  -e "s/^0*//"`

@ SKIPTRK = 0
@ SKIPFIT = 0
set PROG = `basename $0`
if ($PROG !~ '*tdsynop*') then
  if ($?JOB_NAME) then
    set PROG = $JOB_NAME
  endif
endif
if ($PROG =~ "*_ttim") @ SKIPTRK = 1
if ($PROG =~ "*_inv") @ SKIPFIT = 1

# modules and programs
set COVER = rdcover
set GENTG = gentargs
set TRACK = mtrack
set TTFIT = travel_times
set TDINV = invert_td_hr

if ($TEST) then
  set TSTOPT = ~rick/hmi/timed/testoptions
  if (!(-r $TSTOPT)) then
    echo Error: unable to read file $TSTOPT for test options
    exit
  endif
  set NMSP = hmi_test
  grep -q "^GENTG =" $TSTOPT
  if ($status == 0) set GENTG = `grep "^GENTG =" $TSTOPT | awk '{print $3}'`
  grep -q "^TRACK =" $TSTOPT
  if ($status == 0) set TRACK = `grep "^TRACK =" $TSTOPT | awk '{print $3}'`
  grep -q "^TTFIT =" $TSTOPT
  if ($status == 0) set TTFIT = `grep "^TTFIT =" $TSTOPT | awk '{print $3}'`
  grep -q "^TDINV =" $TSTOPT
  if ($status == 0) set TDINV = `grep "^TDINV =" $TSTOPT | awk '{print $3}'`
else
  set NMSP = hmi
endif

#  check that required scripts and programs are in path or rooted
foreach CMD ($GENTG $TRACK $TTFIT)
  which $CMD > /dev/null
  if ($status) then
    if (!(-x $CMD)) then
      echo Error: command $CMD not in path\; script will fail
      exit
    endif
  endif
end

# add flags
set GENTG = "$GENTG grid= timed24+"
set TRACK = "$TRACK -vo"
set TTFIT = "$TTFIT -v"
set TDINV = "$TDINV -v"

# base naming convention for I/O series and logs
set BASE = tdV
set SUFX = _synopHC
set PLID = hmitdV
set TYPE = $NMSP.$BASE$SUFX

# parameters
@ LNGTH = 640
@ SIZE = 512
set SCALE = 0.06
set QMASK = 0x80004000
@ MMISS = 5000
@ MINCV = 900
@ MMINCV = 500
set QUEUE = j.q

# output series
set TSER = $NMSP.$BASE\track$SUFX
set OSER = $NMSP.$BASE\times$SUFX
set RSER = $NMSP.$BASE\invrt$SUFX

# input series
set ISER = hmi.V_45s
set PSER = hmi.tdpixlist
set BCKG = hmi.V_avg120
set KSER = hmi.TDKernels

# auxiliary files
set NOTE = ~rick/hmi/notify
set RJCT = ~rick/hmi/qual/reject.V

if ($TEST) then
  grep -q "^ISER =" $TSTOPT
  if ($status == 0) set ISER = `grep "^ISER =" $TSTOPT | awk '{print $3}'`
  grep -q "^PSER =" $TSTOPT
  if ($status == 0) set MSER = `grep "^PSER =" $TSTOPT | awk '{print $3}'`
  grep -q "^BCKG =" $TSTOPT
  if ($status == 0) set BCKG = `grep "^BCKG =" $TSTOPT | awk '{print $3}'`
  grep -q "^KSER =" $TSTOPT
  if ($status == 0) set GSER = `grep "^KSER =" $TSTOPT | awk '{print $3}'`
  grep -q "^TSER =" $TSTOPT
  if ($status == 0) set TSER = `grep "^TSER =" $TSTOPT | awk '{print $3}'`
  grep -q "^OSER =" $TSTOPT
  if ($status == 0) set OSER = `grep "^OSER =" $TSTOPT | awk '{print $3}'`
  grep -q "^RSER =" $TSTOPT
  if ($status == 0) set RSER = `grep "^RSER =" $TSTOPT | awk '{print $3}'`
  grep -q "^RJCT =" $TSTOPT
  if ($status == 0) then
    set RJCT = `grep "^RJCT =" $TSTOPT | awk '{print $3}'`
    if ($RJCT == "") set RJCT = "Not Specified"
  endif
endif

#  check that input and product series exist
foreach SER ($ISER $PSER $BCKG $KSER $TSER $OSER $RSER)
  show_info -j ds= $SER >& /dev/null
  if ($status) then
    echo Error: series $SER does not exist\; full script will fail
    exit
  endif
end

set PLOC = $PSER'"[]"'

# ensure target times at uniform intervals
if ($YYMMDD < 500000) @ YYMMDD += 20000000
if ($YYMMDD < 1000000) @ YYMMDD += 19000000
@ YR = $YYMMDD / 10000
@ MO = ($YYMMDD % 10000) / 100
@ DA = $YYMMDD % 100
if ($HR < 8) then
  @ HH = 4
else if ($HR < 16) then
  @ HH = 12
else
  @ HH = 20
endif
set TMID = $YR.`printf %02d $MO`.`printf %02d $DA`_`printf %02d $HH`:00_TAI
@ YY = $YR % 100
set CTIME = `printf %02d $YY``printf %02d $MO``printf %02d $DA`:`printf %02d $HH`

# determine root location of scratch and temp directory files
set TMPFSLOC = ~rick/hmi/tmpfs
set TMPARSE = ~rick/bin/tmplocs
set ALTDIR = ~rick/hmi
if (-x $TMPARSE) then
  set XXLOCS = (`$TMPARSE $TMPFSLOC $ALTDIR`)
  set TEMP = $XXLOCS[1]
  set SCRT = $XXLOCS[2]
else
  set TEMP = /tmp
  set SCRT = /tmp
endif

set HOSTID = `hostname | awk -F. '{print $1}'`
set PROCID = $HOSTID\_$$

set TMPB = $TEMP/$PLID$SUFX
if ($TEST) set TMPB = $TMPB.$TW
if (!(-d $TMPB)) mkdir $TMPB
set TMPD = $TMPB/scripts
if (!(-d $TMPD)) mkdir $TMPD
set MSGS = $TEMP/msgs
if (!(-d $MSGS)) mkdir $MSGS
set SCRB = $SCRT/$PLID
if ($TEST) set SCRB = $SCRB.$TW
if (!(-d $SCRB)) mkdir $SCRB
set LOGF = $SCRB/runlog$SUFX
set RLOG = $TMPB/$CTIME
if (-e $RLOG) mv $RLOG $RLOG.bck
set MMSG = $MSGS/msg.$PROCID
set TLIST = $TMPB/td_inf.$$
set TIMES = $SCRB/latency
set DTIME = $SCRB/synop
set QRES = $TEMP/qres
if (!(-d $QRES)) mkdir $QRES

set NOW = `date +%Y.%m.%d_%T`
set STARTED = $NOW
onintr INTERRUPT

# check gatekeeper for permission to proceed
set TURN = ~rick/hmi/pikewatch
set PIKE = ~rick/hmi/pike
if ($TEST) then
  set RES = (`$TURN $PIKE -test`)
else
  set RES = (`$TURN $PIKE`)
endif
@ NOGO = $RES[1]
if ($NOGO) goto TURNOUT

# check coverage
$COVER ds= $ISER tmid= $TMID length= $LNGTH \
    max_miss= $MMISS qmask= $QMASK reject= $RJCT > $TLIST
@ LINES = `wc -l $TLIST |  awk '{print $1}'`
if ($LINES < 2) then
  @ CVG = 0
else
  set ECVG = `head -2 $TLIST | tail -1 | awk '{print $4}'`
  @ CVG = `echo $ECVG | awk -F. '{print $1}'`
  @ CVG *= 1000
  @ CVGFRAC = `echo $ECVG | awk -F. '{print $2}' | sed  -e "s/^0*//"`
  @ CVG += $CVGFRAC
endif
set CVRG = 0.$CVG
if ($CVG >= 1000) set CVRG = 1.000

set TTIME = '*'
set PTIME = '*'
set VTIME = '*'
@ STRTSTOP = 1
if ($STRTSTOP) @ PBEGIN = `date +%s`

# set up logging, notification
set SUBJ = "HMI time-distance $TW processing for $CTIME"
if ($SKIPTRK || $SKIPFIT) then
  echo "$NOW resume $CTIME" > $MMSG
  @ STRTSTOP = 0
else
  echo "$NOW begin $CTIME" > $MMSG
  @ STRTSTOP = 1
endif
if ($CVG < $MINCV) then
  if (($CVG < $MMINCV) && ($TEST == 0)) then
    echo "Notice: effective coverage < 0.$MMINCV" >> $MMSG
    echo "        processing not attempted" >> $MMSG
    cat $TLIST >> $MMSG
    echo "  ($CTIME) Effective coverage < 0.$MMINCV; processing abandoned" >> $LOGF
    set SUBJ = "HMI time-distance $TW processing abandoned for $CTIME"
    @ STRTSTOP = 0
    set STARTED = "not attempted"
    goto CLEANUP
  endif
  echo "Warning: effective coverage < 0.$MINCV" >> $MMSG
  cat $MMSG >> $LOGF
  cat $TLIST >> $MMSG
else
  cat $MMSG >> $LOGF
endif
rm $TLIST

$GENTG time= $TMID > $TLIST
set LATS = (`head -1 $TLIST`)
set LONS = (`tail -1 $TLIST`)
@ RGNCT = $#LATS
if ($#LONS > $RGNCT) @ RGNCT = $#LONS

set LATLIST
set LONLIST
@ N = 0
while ($N < $#LATS)
  @ N++
  set LATLIST = "$LATLIST, $LATS[$N]"
end
@ N = 0
while ($N < $#LONS)
  @ N++
  set LONLIST = "$LONLIST, $LONS[$N]"
end
rm $TLIST

if ($SKIPFIT) goto SKIP_TIMES
if ($SKIPTRK) goto SKIP_TRACK

# check for background average
set LINE = (`sunephem $TMID | head -3 | tail -1`)
@ CROT = `echo $LINE[7] | awk -F. '{print $1}'`
@ CLON = `echo $LINE[2] | awk -F. '{print $1}'`
@ CR60 = $CROT
if ($CLON > 330) then
  @ CL60 = 360
else if ($CLON > 270) then
  @ CL60 = 300
else if ($CLON > 210) then
  @ CL60 = 240
else if ($CLON > 150) then
  @ CL60 = 180
else if ($CLON > 90) then
  @ CL60 = 120
else if ($CLON > 30) then
  @ CL60 = 60
else
  @ CR60++
  @ CL60 = 360
endif
set BCK = $BCKG"[$CR60][$CL60]"
@ BCKCT = `show_info -c ds= "$BCK" | awk '{print $1}'`
if ($BCKCT < 1) then
  echo "Warning: Data average record $BCK does not exist"  >> $MMSG
  echo "         Background subtraction skipped"  >> $MMSG
  set BCK = "Not Specified"
endif

@ BEGUN = `date +%s`
$TRACK out= $TSER in= $ISER tmid= $TMID length= $LNGTH \
  bckgn= "$BCK" qmask= $QMASK max_miss= $MMISS reject= $RJCT \
  scale= $SCALE cols= $SIZE rows= $SIZE \
  tobs_key= T_REC \
  lat= "{$LATLIST}" lon= "{$LONLIST}" >>& $RLOG
if ($status) then
  set SUBJ = "$SUBJ incomplete"
  echo "  tracking failure on `hostname`"  >> $MMSG
  echo "  run log is on $RLOG"  >> $MMSG
  echo "  tracking failure on `hostname`"  >> $LOGF
  @ STRTSTOP = 0
  goto CLEANUP
else
  @ FINIS = `date +%s`
  @ ELAPS = $FINIS - $BEGUN
  @ ELAPS /= 60
  echo "  time for tracking $RGNCT regions on `hostname`: $ELAPS min"  >> $MMSG
  echo "    tracked data in $TSER""[$TMID]" >> $MMSG
  echo "  ($CTIME) time for tracking $RGNCT regions on `hostname`: $ELAPS min"  >> $LOGF
  set TTIME = $ELAPS
endif

SKIP_TRACK:

# check gatekeeper for permission to proceed
if ($TEST) then
  set RES = (`$TURN $PIKE -test`)
else
  set RES = (`$TURN $PIKE`)
endif
@ NOGO = $RES[1]
if ($NOGO) goto TURNOUT

# prepare script for batch processing of travel time fitting
@ BEGUN = `date +%s`
$GENTG -c time= $TMID > $TLIST
set LATS = (`head -1 $TLIST`)
set LONS = (`tail -1 $TLIST`)
@ RGNCT = `show_info -cq ds= $TSER"[$TMID]"`
if ($#LATS != $RGNCT || $#LONS != $RGNCT) then
  set SUBJ = "$SUBJ incomplete"
  echo "  inconsistent region count" >> $MMSG
  echo "  ($CTIME) inconsistent region count" >> $LOGF
  @ STRTSTOP = 0
  goto CLEANUP
endif
# set noglob
# set TSERINPS = `show_info -iq ds= $TSER\[$TMID]`
set FILE = td`echo $CTIME | tr ':' '.'`
set SCRIPT = $TMPD/$FILE
echo "#\!/bin/csh -f" > $SCRIPT
# echo "limit vm 3000M" >> $SCRIPT
# if there is a limit it should be at least 3500M
# echo "set noglob" >> $SCRIPT
# echo 'set TSERINPS =  ('$TSERINPS')' >> $SCRIPT
# unset noglob
# echo 'set TSERINP = "$TSERINPS[$SGE_TASK_ID]"' >> $SCRIPT
echo 'set LATS =  ('"$LATS"')' >> $SCRIPT
echo 'set LONS =  ('"$LONS"')' >> $SCRIPT
echo 'set LAT = $LATS[$SGE_TASK_ID]' >> $SCRIPT
echo 'set LON = $LONS[$SGE_TASK_ID]' >> $SCRIPT
echo "$TTFIT \" >> $SCRIPT
# echo '  in= "$TSERINP" \' >> $SCRIPT
echo '  in= '$TSER'"['$TMID'][][$LAT][$LON]" \' >> $SCRIPT
echo '  pxloc= '$PLOC' \' >> $SCRIPT
echo "  out= $OSER "'>>&' $RLOG >> $SCRIPT
qsub -sync yes -t 1-$RGNCT -q $QUEUE -e $QRES -o $QRES $SCRIPT >>& $RLOG
@ QJOK = `grep "exited with exit code 0." $RLOG | wc -l`
@ FAIL = $RGNCT - $QJOK
@ FDCT = `show_info -cq ds= $OSER"[$TMID]"`
if ($FAIL || $FDCT != $RGNCT) then
  set SUBJ = "$SUBJ incomplete"
  echo "  $FAIL travel-time failures on cluster" >> $MMSG
  echo "  $FDCT records in $OSER""[$TMID]" >> $MMSG
  echo "  ($CTIME) $FAIL travel-time failures on cluster" >> $LOGF
  echo "  qsub script is $SCRIPT" >> $MMSG
  echo "  qsub script is $SCRIPT" >> $LOGF
  @ STRTSTOP = 0
  goto CLEANUP
else
  @ FINIS = `date +%s`
  @ ELAPS = $FINIS - $BEGUN
  @ ELAPS += 30
  @ ELAPS /= 60
  echo "  time for $RGNCT travel times on the cluster: $ELAPS min"  >> $MMSG
  echo "    travel-time fits in $OSER""[$TMID]" >> $MMSG
  echo "  ($CTIME) time for $RGNCT travel times on the cluster: $ELAPS min"  >> $LOGF
  set PTIME = $ELAPS
endif
if (!($FAIL)) rm $SCRIPT

SKIP_TIMES:

# check gatekeeper for permission to proceed
if ($TEST) then
  set RES = (`$TURN $PIKE -test`)
else
  set RES = (`$TURN $PIKE`)
endif
@ NOGO = $RES[1]
if ($NOGO) goto TURNOUT

setenv OMP_NUM_THREADS 1
@ BEGUN = `date +%s`
foreach KTYP (Raypath Born)
  set KSET = $KSER"[$KTYP][1100]"
  foreach FTYP (Gabor GizonBirch)
    $TDINV in= $OSER"[$TMID]" kernels= "$KSET" out= $RSER fitting= $FTYP >>& $RLOG
    if ($status) then
      set SUBJ = "$SUBJ incomplete"
      echo "  inversion failure on `hostname`"  >> $MMSG
      echo "  run log is on $RLOG"  >> $MMSG
      echo "  inversion failure on `hostname`"  >> $LOGF
      @ STRTSTOP = 0
      goto CLEANUP
    endif
  end
end
@ RPCT = `show_info -cq ds= $RSER"[$TMID]"`
@ FINIS = `date +%s`
@ ELAPS = $FINIS - $BEGUN
@ ELAPS /= 60
echo "  time for inverting $RPCT regions on `hostname`: $ELAPS min"  >> $MMSG
echo "    inversions in $RSER""[$TMID]" >> $MMSG
echo "  ($CTIME) time for inverting $RPCT regions on `hostname`: $ELAPS min"  >> $LOGF
set VTIME = $ELAPS
goto CLEANUP

TURNOUT:
set NOW = `date +%Y.%m.%d_%T`
set SUBJ = "$SUBJ aborted"
set TYPE = misc
echo "-> $NOW Script $PROG aborted! <-" >> $MMSG
echo "$CTIME $STARTED  aborted" >> $TIMES
echo "$CTIME	$TTIME	$PTIME	$VTIME aborted" >> $DTIME
if ($NOGO < 0) then
  echo "   watcher script $TURN called with invalid argument" >> $MMSG
  echo "   or file $PIKE has invalid format" >> $MMSG
else
  @ HRCT = $NOGO / 3600
  @ NOGO -= $HRCT * 3600
  @ MNCT = $NOGO / 60
  @ NOGO -= $MNCT * 60
  echo -n "   file $PIKE contains NoGo signal for $HRCT" >> $MMSG
  echo `printf :%02d:%02d $MNCT $NOGO` >> $MMSG
  if ($#RES > 1) then
    echo "   Reason: $RES[2-]" >> $MMSG
  endif
endif
set RGNCT = '?'
@ STRTSTOP = 0
goto CLEANUP

INTERRUPT:
set SUBJ = "$SUBJ interrupted"
set TYPE = misc
echo "-> Script "$PROG" or module aborted! <-" >> $MMSG
echo "$CTIME $STARTED  aborted" >> $TIMES
echo "$CTIME	$TTIME	$PTIME	$VTIME	aborted" >> $DTIME
set RGNCT = '?'
@ STRTSTOP = 0

CLEANUP:
set NOW = `date +%Y.%m.%d_%T`
set NOTIFY = `grep ^$TYPE $NOTE | awk '{print $2}'`
if ($STRTSTOP) then
  @ ELAPS = $FINIS - $PBEGIN
  @ ELAPS += 30
  @ ELAPS /= 60
  @ HREL = $ELAPS / 60
  @ MNEL = $ELAPS % 60
else
  @ HREL = 0
  @ MNEL = 0
  echo -n "# " >> $TIMES
endif
echo "$CTIME $STARTED  $HREL`printf :%02d $MNEL`" >> $TIMES
echo "$CTIME	$TTIME	$PTIME	$VTIME" >> $DTIME
echo "$NOW finish $CTIME" >> $MMSG
if ($STRTSTOP) then
  echo "  End-to-end latency = $HREL`printf :%02d $MNEL`" >> $MMSG
endif
echo $SUBJ > $MSGS/subj.$PROCID
if ($#NOTIFY) then
  echo "$NOTIFY" > $MSGS/list.$PROCID
else
  echo rick > $MSGS/list.$PROCID
endif
echo "$NOW finish $CTIME ($RGNCT regions)" >> $LOGF
if (-e $TLIST) rm $TLIST