Return to dodscopyn CVS log

Up to [Development] / JSOC / proj / globalhs / scripts

script that parallels doretilen, copies datasets

#! /bin/tcsh -f

setenv PATH $JSOCROOT/bin/$JSOC_MACHINE':'$JSOCROOT/proj/globalhs/scripts':'$PATH

# number of jobs will be ntimechunks = (totalsecs / chunksecs) rounded up

# submit next batch of jobs when total number of retile jobs drops below jobthreshold
if (! $?GLOBALHS_RETJOBTHRESHOLD) then
  set jobthreshold=2
else
  set jobthreshold=$GLOBALHS_RETJOBTHRESHOLD
endif

# if this script inherited a label in its environment, use it as the suffix for job names
if (! $?GLOBALHS_LABEL) then
  set suff=''
else
  set suff=.$GLOBALHS_LABEL
endif

set qsubtmp=/tmp27/$USER/qsubtmp
mkdir -p $qsubtmp
if (! $?GLOBALHS_RETQUEUE) then
  set q=j.q
else
  set q=$GLOBALHS_RETQUEUE
endif

if ($q == a.q) then
  alias qsub qsub2
  alias qstat qstat2
  alias waittosubmit waittosubmit2
endif

@ i = 1
while ($i <= $#argv )
  set $argv[$i]
@ i++
end

if (! $?starttime) then
  echo must specify parameter starttime
  exit 1
endif

if (! $?totaltime) then
  echo must specify parameter totaltime
  exit 1
endif

if (! $?lchunkin) then
  echo must specify parameter lchunkin
  exit 1
endif

set totalsecs = `durcon $totaltime`
if ($status) then
  echo incorrect format for parameter totaltime
  exit 1
endif

if (! $?timechunk) set timechunk = 12d
set chunksecs = `durcon $timechunk`
if ($status) then
  echo incorrect format for parameter timechunk
  exit 1
endif

set ntimechunks = `echo "$totalsecs / $chunksecs" | bc`

if (! $?lmin) set lmin = 0
if (! $?lmax) set lmax = 300
if (! $?lchunkout) set lchunkout = $lchunkin

if (($lchunkin % $lchunkout && $lchunkin < $lmax + 1) && ($lchunkout % $lchunkin && $lchunkout < $lmax + 1)) then
  echo lchunkout must evenly divide lchunkin or lchunkin must evenly divide lchunkout
  exit 1
endif

date
echo $PWD
echo $0 $argv

if (! -es ret.parms.blank) then
  echo parameter file blank missing: ret.parms.blank is required
  exit 1
endif

grep -v "^[[:space:]]*#"  ret.parms.blank | sed s@"[[:space:]]"@"\n"@g | grep -v '^[[:space:]]*$'  > parms.tmp
grep    "^[[:space:]]*##" ret.parms.blank | awk -F '##' '{print $2}' > script.parms

set vars = `cat script.parms`
@ i = 1
while ($i <= $#vars )
  set $vars[$i]
@ i++
end

if (! $?copyto) then
  echo copyto must be supplied as a script parameter in ret.parms.blank
  exit 1
endif

set tagclause = `cat parms.tmp | grep ^in= | cut -d= -f2- | sed s/"'"/""/g | sed s@"\["@"\n"@g | sed s@]@"\n"@g | grep TAG`
#set in = `cat parms.tmp | grep ^in= | sed s/\'//g | sed s/\"//g | awk -F '[=[]' '{print $2}'`
set out = `cat parms.tmp | grep ^out= | cut -d= -f2`
set epoch = `show_info -j $out | grep -i '^keyword:t_start_epoch' | cut -d, -f5`
set epochsecs = `time_convert o=jsoc time=$epoch`
set tstartstep = `show_info -j $out | grep -i '^keyword:t_start_step' | cut -d, -f5`
# the following assumes t_start_step is given in seconds
set indexchunk = `echo "$chunksecs / $tstartstep" | bc`

set tchunk = `cat parms.tmp | grep ^TCHUNK= | cut -d= -f2`
set chunksecs2 = `durcon $tchunk`
set cadence = `show_info -j $out | grep -i '^keyword:t_step' | cut -d, -f5`
set ndt = `echo $chunksecs2 / $cadence | bc`

durcon $starttime >& /dev/null
if ($status) then
  set startsecs = `time_convert o=jsoc time=$starttime`
else
  set startsecs = `durcon $starttime` 
  set startsecs = `echo "$startsecs + $epochsecs" | bc`
endif

set firstindex = `echo "($startsecs - $epochsecs)/$tstartstep" | bc`
set nextfirstindex = `echo "($startsecs + $totalsecs - $epochsecs)/$tstartstep" | bc`

@ lchunkfirst = $lmin / $lchunkin
@ lchunklast  = $lmax / $lchunkin

set i=0
while ($i < $ntimechunks)
@ index = $firstindex + $i * $indexchunk
  set subfile = subr.$firstindex.$index.$lmin-$lmax$suff
  echo '#\!/bin/csh' > $subfile
  echo 'setenv PATH' $JSOCROOT'/bin/$JSOC_MACHINE' >> $subfile
  echo 'cd' $PWD >> $subfile
  set interval = '#'$index'/'$indexchunk
@ lc = $lchunkfirst
  while ($lc <= $lchunklast)
  @ lfirst = $lc * $lchunkin 
  @ llast  = $lfirst + $lchunkin - 1
    if ($lfirst < $lmin) set lfirst = $lmin
    if ($llast  > $lmax) set llast  = $lmax

    echo \(time dscp dsout=$copyto rsin=$out\"\[$interval]\[$lfirst]\[$llast]\[$ndt]\[$tagclause]\"\) '>&' dscp.log.$index.$lfirst-$llast >> $subfile
    echo echo \$status '>&' dscp.exitstatus.$index.$lfirst-$llast >> $subfile

  @ lc++
  end
@ i++
end

@ index += $indexchunk
if ($index < $nextfirstindex) then
@ indexchunk = $nextfirstindex - $index
  set chunksecs = `echo "$indexchunk * $tstartstep" | bc`
  set subfile = subr.$firstindex.$index.$lmin-$lmax$suff
  echo '#\!/bin/csh' > $subfile
  echo 'setenv PATH' $JSOCROOT'/bin/$JSOC_MACHINE' >> $subfile
  echo 'cd' $PWD >> $subfile
  set interval = '#'$index'/'$indexchunk
@ lc = $lchunkfirst
  while ($lc <= $lchunklast)
  @ lfirst = $lc * $lchunkin 
  @ llast  = $lfirst + $lchunkin - 1
    if ($lfirst < $lmin) set lfirst = $lmin
    if ($llast  > $lmax) set llast  = $lmax

    echo \(time dscp dsout=$copyto rsin=$out\"\[$interval]\[$lfirst]\[$llast]\[$ndt]\[$tagclause]\"\) '>&' dscp.log.$index.$lfirst-$llast >> $subfile
    echo echo \$status '>&' dscp.exitstatus.$index.$lfirst-$llast >> $subfile

  @ lc++
  end
@ ntimechunks++
  set chunksecs = `durcon $timechunk`
  set indexchunk = `echo "$chunksecs / $tstartstep" | bc`
endif

echo $ntimechunks job scripts created

waittosubmit $jobthreshold subr
set i=0
while ($i < $ntimechunks)
@ index = $firstindex + $i * $indexchunk
  set subfile = subr.$firstindex.$index.$lmin-$lmax$suff
  qsub -q $q -e $qsubtmp -o $qsubtmp $subfile
@ i++
end

echo jobs submitted, start waiting

set njobsrunning = `qstat -r -u $USER | grep "Full jobname:" | grep subr.$firstindex | grep $lmin-$lmax | grep "$suff" | wc -l`
while($njobsrunning > 0)
  sleep 60
  set njobsrunning = `qstat -r -u $USER | grep "Full jobname:" | grep subr.$firstindex | grep $lmin-$lmax | grep "$suff" | wc -l`
end

#check for errors here

set errlist1 = `grep -Hv 0 dscp.exitstatus.* | awk -F '[.:]' '{print $3"."$4}'`
set errlist2 = `grep -Hic error dscp.log.* | grep -v :0 | awk -F '[.:]' '{print $3"."$4}'`
set errlist = `echo $errlist1 $errlist2 | sed s/" "/"\n"/g | sort -n | uniq`

if ($#errlist) then
  echo some jobs fail: $errlist
  exit 1
endif

echo successful completion

rm parms.tmp script.parms

exit 0