(file) Return to doylmsht CVS log (file) (dir) Up to [Development] / JSOC / proj / globalhs / scripts

  1 tplarson 1.1 #!/bin/tcsh -f
  2              
  3              set jobthreshold=50
  4              set njobs=10
  5              #set dl=6
  6              #set dm=15
  7              #set ndt=60
  8              
  9              set qsubtmp=/tmp27/$USER/qsubtmp
 10              mkdir -p $qsubtmp
 11              
 12              @ i = 1
 13              while ($i <= $#argv )
 14                set $argv[$i]
 15              @ i++
 16              end
 17              
 18              if (! $?deltal) set deltal = 6
 19              if (! $?xoffset) set xoffset = 0.0
 20              if (! $?yoffset) set yoffset = 0.0
 21              if (! $?pangle) set pangle = 0.0
 22 tplarson 1.1 if (! $?bangle) set bangle = 0.0
 23              if (! $?obsdist) set obsdist = 1.000
 24              if (! $?segment) set segment = vradsum
 25              
 26              echo `date` on $HOST
 27              echo $PWD
 28              echo $0 $argv
 29              
 30              # if this script inherited a label in its environment, use it as the suffix for job names
 31              if (! $?GLOBALHS_LABEL) then
 32                set lab=''
 33                set suff=''
 34              else
 35                set lab=$GLOBALHS_LABEL
 36                set suff=.$GLOBALHS_LABEL
 37              endif
 38              
 39              if (! -es v2t.parms.blank) then
 40                echo parameter file blank missing: v2t.parms.blank is required
 41                exit 1
 42              endif
 43 tplarson 1.1 
 44              grep -v '^[[:space:]]*$' v2t.parms.blank | grep -v "^[[:space:]]*#" | sed s@"[[:space:]]"@"\n"@g > parms.tmp
 45              set tagblank = `cat parms.tmp | grep TAG= | cut -d= -f2`
 46              set tagsuff = $xoffset'_'$yoffset'_'$pangle'_'$bangle'_'$obsdist'_'$segment
 47              set tag = `echo $tagblank | sed s/TTTT/$tagsuff/`
 48              
 49              set in = `cat parms.tmp | grep in= | sed s/"'"/""/g | awk -F '[=[]' '{print $2}'`
 50              set trecstep = `show_info -j $in | grep -i '^keyword:t_rec_step' | cut -d, -f5`
 51              set ttotal = `cat parms.tmp | grep TTOTAL= | cut -d= -f2`
 52              set totalsecs = `durcon $ttotal`
 53              set ndt = `echo "$totalsecs/$trecstep" | bc`
 54              
 55              if (! -es ylm.parms.blank) then
 56                echo parameter file blank missing: ylm.parms.blank is required
 57                exit 1
 58              endif
 59              
 60              set tstart = `cat ylm.parms.blank | grep -v "^[[:space:]]*#" | grep TSTART= | cut -d= -f2`
 61              if ($#tstart == 0) then
 62              @ line = `mkylms -H | grep -n "TSTART (time)" | cut -d: -f1` + 1
 63                set tstart = `mkylms -H | sed -n $line'p' | awk '{print $3}'`
 64 tplarson 1.1 endif
 65              set startsecs = `time_convert o=jsoc time=$tstart`
 66              
 67              set file = `cat ylm.parms.blank | grep -v "^[[:space:]]*#" | grep MODELIST= | cut -d= -f2`
 68              if (! -es $file) then
 69                echo MODELIST file $file is missing
 70                exit 1
 71              endif
 72              
 73              set nim=`cat $file | wc -l`
 74              @ nchunks = ( ( $nim - 1 ) / $ndt ) + 1
 75              #not necessary if while is replaced by foreach below
 76              #if ($njobs > $nchunks) set njobs=$nchunks
 77              
 78              set i=0
 79              while ($i < $nchunks)
 80                set isecs = `echo "$startsecs + ($i * $ndt * $trecstep)" | bc`
 81                set istart = `time_convert o=cal zone=tai s=$isecs`
 82                @ index = ($i * $ndt) + 1
 83                @ index2 = $index + $ndt - 1
 84                if ($index2 > $nim) set index2 = $nim
 85 tplarson 1.1   set lmin = `sed -n $index'p' $file | awk '{print $1}'`
 86                @ lmin = $lmin - $deltal
 87                if ($lmin < 0) set lmin=0
 88                set lmax = `sed -n $index2'p' $file | awk '{print $1}'`
 89                @ lmax = $lmax + $deltal
 90                cat parms.tmp | sed s@RRRR@$istart/$ndt'm'@ | sed s/MMMM/$lmax/ | sed s/NNNN/$lmin/ | sed s/SSSS/$istart/ | sed s/XXXX/$xoffset/g | sed s/YYYY/$yoffset/g | sed s/PPPP/$pangle/g | sed s/BBBB/$bangle/g | sed s/DDDD/$obsdist/g | sed s/GGGG/$segment/g | sed s/TTTT/$tagsuff/g  > v2t.parms.$i.$segment
 91                @ r = $i % $njobs
 92                set subfile = subt.$r$suff._$tag
 93                if ($i < $njobs) then
 94                  echo '#\!/bin/csh' > $subfile
 95                  echo 'cd' $PWD >> $subfile
 96                endif
 97                echo \(time jv2ts @v2t.parms.$i.$segment\) '>&' v2t.log.$i.$segment >> $subfile
 98                echo echo \$status '>&' jv2ts.exitstatus.$i.$segment >> $subfile
 99                @ i++
100              end
101              
102              waittosubmit $jobthreshold subt
103              set q=j.q
104              set sublist=`/bin/ls subt.*._$tag`
105              foreach sub ($sublist)
106 tplarson 1.1   qsub -q $q -e $qsubtmp -o $qsubtmp $sub
107              end
108              
109              #set i=0
110              #while ($i < $njobs)
111              #  qsub -q $q -e $qsubtmp -o $qsubtmp subt.$i$suff
112              #@ i++
113              #end
114              
115              echo jobs submitted, start waiting
116              
117              set njobsrunning = `qstat -r -u $USER | grep "Full jobname:" | grep subt | grep "$lab._$tag" | wc -l`
118              while($njobsrunning > 0)
119                sleep 60
120                set njobsrunning = `qstat -r -u $USER | grep "Full jobname:" | grep subt | grep "$lab._$tag" | wc -l`
121              end
122              
123              set expectedlist = `/bin/ls v2t.parms.[0-9]*.$segment | cut -d'.' -f 3`
124              set ranlist   = `/bin/ls v2t.log.*.$segment | cut -d'.' -f 3`
125              set rerunlist = `echo $expectedlist $ranlist | sed s/" "/"\n"/g | sort -n | uniq -u`
126              
127 tplarson 1.1 set errlist1 = `grep -Hv 0 jv2ts.exitstatus.*.$segment | awk -F '[.:]' '{print $3}'`
128              set errlist2 = `grep -Hc "successful completion" v2t.log.*.$segment | grep -v :1 | awk -F '[.:]' '{print $3}'`
129              set errlist = `echo $errlist1 $errlist2 | sed s/" "/"\n"/g | sort -n | uniq`
130              
131              set list = `echo $rerunlist $errlist | sed s/" "/"\n"/g | sort -n`
132              if ($#list) then
133                echo some jobs fail, rerunning
134                echo rerunlist: $rerunlist > faillog.$segment
135                echo errlist1: $errlist1 >> faillog.$segment
136                echo errlist2: $errlist2 >> faillog.$segment
137                mkdir -p savesubt
138                mv subt.*._$tag savesubt
139              
140                #now have to start with 1 because that is the first index of list
141                set i=1
142                while ($i <= $#list)
143                  @ r = $i % $njobs
144                  set subfile = subt.$r$suff._$tag
145                  if ($i <= $njobs) then
146                    echo '#\!/bin/csh' > $subfile
147                    echo 'cd' $PWD >> $subfile
148 tplarson 1.1     endif
149                  echo \(time jv2ts @v2t.parms.$list[$i].$segment\) '>&' v2t.log.$list[$i].$segment >> $subfile
150                  echo echo \$status '>&' jv2ts.exitstatus.$list[$i].$segment >> $subfile
151                  @ i++
152                end
153              
154                waittosubmit $jobthreshold subt
155                set sublist=`/bin/ls subt.*._$tag`
156                foreach sub ($sublist)
157                  qsub -q $q -e $qsubtmp -o $qsubtmp $sub
158                end
159              
160                set njobs = `qstat -r -u $USER | grep "Full jobname:" | grep subt | grep "$lab._$tag" | wc -l`
161                while($njobs > 0)
162                  sleep 60
163                  set njobs = `qstat -r -u $USER | grep "Full jobname:" | grep subt | grep "$lab._$tag" | wc -l`
164                end
165              
166                set expectedlist = `/bin/ls v2t.parms.*.$segment | cut -d'.' -f 3`
167                set ranlist   = `/bin/ls v2t.log.*.$segment | cut -d'.' -f 3`
168                set rerunlist = `echo $expectedlist $ranlist | sed s/" "/"\n"/g | sort -n | uniq -u`
169 tplarson 1.1 
170                set errlist1 = `grep -Hv 0 jv2ts.exitstatus.*.$segment | awk -F '[.:]' '{print $3}'`
171                set errlist2 = `grep -Hc "successful completion" v2t.log.*.$segment | grep -v :1 | awk -F '[.:]' '{print $3}'`
172                set errlist = `echo $errlist1 $errlist2 | sed s/" "/"\n"/g | sort -n | uniq`
173              
174                if ($#rerunlist || $#errlist) then
175                  echo some jobs still fail, i give up
176                  exit 1
177                endif
178              endif
179              
180              rm parms.tmp
181              
182              echo successful completion
183              exit 0

Karen Tian
Powered by
ViewCVS 0.9.4