(file) Return to doylmsht CVS log (file) (dir) Up to [Development] / JSOC / proj / globalhs / scripts

  1 tplarson 1.1 #!/bin/tcsh -f
  2              
  3 tplarson 1.2 setenv PATH $JSOCROOT/bin/$JSOC_MACHINE':'$JSOCROOT/proj/globalhs/scripts':'$PATH
  4              
  5              # submit njobs jobs at once
  6              if (! $?GLOBALHS_V2TNJOBS) then
  7                set njobs=20
  8              else
  9                set njobs=$GLOBALHS_V2TNJOBS
 10              endif
 11              
 12              # submit next batch of jobs when total number of sht jobs drops below jobthreshold
 13              if (! $?GLOBALHS_V2TJOBTHRESHOLD) then
 14                set jobthreshold=40
 15              else
 16                set jobthreshold=$GLOBALHS_V2TJOBTHRESHOLD
 17              endif
 18              
 19              # if this script inherited a label in its environment, use it as the suffix for job names
 20              if (! $?GLOBALHS_LABEL) then
 21                set suff=''
 22              else
 23                set suff=.$GLOBALHS_LABEL
 24 tplarson 1.2 endif
 25 tplarson 1.1 
 26              set qsubtmp=/tmp27/$USER/qsubtmp
 27              mkdir -p $qsubtmp
 28 tplarson 1.2 if (! $?GLOBALHS_V2TQUEUE) then
 29                set q=k.q
 30              else
 31                set q=$GLOBALHS_V2TQUEUE
 32              endif
 33              
 34              if ($q == k.q) then
 35                alias qsub qsub2
 36                alias qstat qstat2
 37                alias waittosubmit waittosubmit2
 38              endif
 39 tplarson 1.1 
 40              @ i = 1
 41              while ($i <= $#argv )
 42                set $argv[$i]
 43              @ i++
 44              end
 45              
 46              if (! $?deltal) set deltal = 6
 47              if (! $?xoffset) set xoffset = 0.0
 48              if (! $?yoffset) set yoffset = 0.0
 49              if (! $?pangle) set pangle = 0.0
 50              if (! $?bangle) set bangle = 0.0
 51              if (! $?obsdist) set obsdist = 1.000
 52              if (! $?segment) set segment = vradsum
 53 tplarson 1.2 set tag='x'$xoffset'_y'$yoffset'_p'$pangle'_b'$bangle'_d'$obsdist'_'$segment
 54 tplarson 1.1 
 55              echo `date` on $HOST
 56              echo $PWD
 57              echo $0 $argv
 58              
 59              if (! -es v2t.parms.blank) then
 60                echo parameter file blank missing: v2t.parms.blank is required
 61                exit 1
 62              endif
 63              
 64 tplarson 1.2 grep -v "^[[:space:]]*#" v2t.parms.blank | sed s@"[[:space:]]"@"\n"@g | grep -v '^[[:space:]]*$' > parms.tmp
 65              #set tagblank = `cat parms.tmp | grep TAG= | cut -d= -f2`
 66              #set tagsuff = $xoffset'_'$yoffset'_'$pangle'_'$bangle'_'$obsdist'_'$segment
 67              #set tag = `echo $tagblank | sed s/TTTT/$tagsuff/`
 68 tplarson 1.1 
 69              set in = `cat parms.tmp | grep in= | sed s/"'"/""/g | awk -F '[=[]' '{print $2}'`
 70              set trecstep = `show_info -j $in | grep -i '^keyword:t_rec_step' | cut -d, -f5`
 71              set ttotal = `cat parms.tmp | grep TTOTAL= | cut -d= -f2`
 72              set totalsecs = `durcon $ttotal`
 73              set ndt = `echo "$totalsecs/$trecstep" | bc`
 74              
 75 tplarson 1.2 if (! -es ../ylm.parms.blank) then
 76 tplarson 1.1   echo parameter file blank missing: ylm.parms.blank is required
 77                exit 1
 78              endif
 79              
 80 tplarson 1.2 set tstart = `cat ../ylm.parms.blank | grep -v "^[[:space:]]*#" | grep TSTART= | cut -d= -f2`
 81 tplarson 1.1 if ($#tstart == 0) then
 82              @ line = `mkylms -H | grep -n "TSTART (time)" | cut -d: -f1` + 1
 83                set tstart = `mkylms -H | sed -n $line'p' | awk '{print $3}'`
 84              endif
 85              set startsecs = `time_convert o=jsoc time=$tstart`
 86              
 87 tplarson 1.2 set file = `cat ../ylm.parms.blank | grep -v "^[[:space:]]*#" | grep MODELIST= | cut -d= -f2`
 88              if (! -es ../$file) then
 89 tplarson 1.1   echo MODELIST file $file is missing
 90                exit 1
 91              endif
 92              
 93 tplarson 1.2 set nim=`cat ../$file | wc -l`
 94 tplarson 1.1 @ nchunks = ( ( $nim - 1 ) / $ndt ) + 1
 95              #not necessary if while is replaced by foreach below
 96              #if ($njobs > $nchunks) set njobs=$nchunks
 97              
 98              set i=0
 99              while ($i < $nchunks)
100                set isecs = `echo "$startsecs + ($i * $ndt * $trecstep)" | bc`
101                set istart = `time_convert o=cal zone=tai s=$isecs`
102                @ index = ($i * $ndt) + 1
103                @ index2 = $index + $ndt - 1
104                if ($index2 > $nim) set index2 = $nim
105 tplarson 1.2   set lmin = `sed -n $index'p' ../$file | awk '{print $1}'`
106 tplarson 1.1   @ lmin = $lmin - $deltal
107                if ($lmin < 0) set lmin=0
108 tplarson 1.2   set lmax = `sed -n $index2'p' ../$file | awk '{print $1}'`
109 tplarson 1.1   @ lmax = $lmax + $deltal
110 tplarson 1.2   cat parms.tmp | sed s@RRRR@$istart/$ndt'm'@ | sed s/MMMM/$lmax/ | sed s/NNNN/$lmin/ | sed s/SSSS/$istart/ | sed s/XXXX/$xoffset/g | sed s/YYYY/$yoffset/g | sed s/PPPP/$pangle/g | sed s/BBBB/$bangle/g | sed s/DDDD/$obsdist/g | sed s/GGGG/$segment/g | sed s/TTTT/$tag/g  > v2t.parms.$i
111 tplarson 1.1   @ r = $i % $njobs
112 tplarson 1.2   set subfile = subt.$r$suff'_'$tag
113 tplarson 1.1   if ($i < $njobs) then
114                  echo '#\!/bin/csh' > $subfile
115                  echo 'cd' $PWD >> $subfile
116                endif
117 tplarson 1.2   echo \(time jv2ts @v2t.parms.$i\) '>&' v2t.log.$i >> $subfile
118                echo echo \$status '>&' jv2ts.exitstatus.$i >> $subfile
119 tplarson 1.1   @ i++
120              end
121              
122              waittosubmit $jobthreshold subt
123 tplarson 1.2 set sublist=`/bin/ls subt.*_$tag`
124 tplarson 1.1 foreach sub ($sublist)
125                qsub -q $q -e $qsubtmp -o $qsubtmp $sub
126              end
127              
128              #set i=0
129              #while ($i < $njobs)
130              #  qsub -q $q -e $qsubtmp -o $qsubtmp subt.$i$suff
131              #@ i++
132              #end
133              
134              echo jobs submitted, start waiting
135              
136 tplarson 1.2 set njobsrunning = `qstat -r -u $USER | grep "Full jobname:" | grep subt | grep $suff'_'$tag | wc -l`
137 tplarson 1.1 while($njobsrunning > 0)
138                sleep 60
139 tplarson 1.2   set njobsrunning = `qstat -r -u $USER | grep "Full jobname:" | grep subt | grep $suff'_'$tag | wc -l`
140 tplarson 1.1 end
141              
142 tplarson 1.2 set expectedlist = `/bin/ls v2t.parms.[0-9]* | cut -d'.' -f 3`
143              set ranlist   = `/bin/ls v2t.log.* | cut -d'.' -f 3`
144 tplarson 1.1 set rerunlist = `echo $expectedlist $ranlist | sed s/" "/"\n"/g | sort -n | uniq -u`
145              
146 tplarson 1.2 set errlist1 = `grep -Hv 0 jv2ts.exitstatus.* | awk -F '[.:]' '{print $3}'`
147              set errlist2 = `grep -Hc "successful completion" v2t.log.* | grep -v :1 | awk -F '[.:]' '{print $3}'`
148 tplarson 1.1 set errlist = `echo $errlist1 $errlist2 | sed s/" "/"\n"/g | sort -n | uniq`
149              
150              set list = `echo $rerunlist $errlist | sed s/" "/"\n"/g | sort -n`
151              if ($#list) then
152                echo some jobs fail, rerunning
153                echo rerunlist: $rerunlist > faillog.$segment
154                echo errlist1: $errlist1 >> faillog.$segment
155                echo errlist2: $errlist2 >> faillog.$segment
156                mkdir -p savesubt
157 tplarson 1.2   mv subt.*_$tag savesubt
158 tplarson 1.1 
159                #now have to start with 1 because that is the first index of list
160                set i=1
161                while ($i <= $#list)
162                  @ r = $i % $njobs
163 tplarson 1.2     set subfile = subt.$r$suff'_'$tag
164 tplarson 1.1     if ($i <= $njobs) then
165                    echo '#\!/bin/csh' > $subfile
166                    echo 'cd' $PWD >> $subfile
167                  endif
168 tplarson 1.2     echo \(time jv2ts @v2t.parms.$list[$i]\) '>&' v2t.log.$list[$i] >> $subfile
169                  echo echo \$status '>&' jv2ts.exitstatus.$list[$i] >> $subfile
170 tplarson 1.1     @ i++
171                end
172              
173                waittosubmit $jobthreshold subt
174 tplarson 1.2   set sublist=`/bin/ls subt.*_$tag`
175 tplarson 1.1   foreach sub ($sublist)
176                  qsub -q $q -e $qsubtmp -o $qsubtmp $sub
177                end
178              
179 tplarson 1.2   set njobs = `qstat -r -u $USER | grep "Full jobname:" | grep subt | grep $suff'_'$tag | wc -l`
180 tplarson 1.1   while($njobs > 0)
181                  sleep 60
182 tplarson 1.2     set njobs = `qstat -r -u $USER | grep "Full jobname:" | grep subt | grep $suff'_'$tag | wc -l`
183 tplarson 1.1   end
184              
185 tplarson 1.2   set expectedlist = `/bin/ls v2t.parms.* | cut -d'.' -f 3`
186                set ranlist   = `/bin/ls v2t.log.* | cut -d'.' -f 3`
187 tplarson 1.1   set rerunlist = `echo $expectedlist $ranlist | sed s/" "/"\n"/g | sort -n | uniq -u`
188              
189 tplarson 1.2   set errlist1 = `grep -Hv 0 jv2ts.exitstatus.* | awk -F '[.:]' '{print $3}'`
190                set errlist2 = `grep -Hc "successful completion" v2t.log.* | grep -v :1 | awk -F '[.:]' '{print $3}'`
191 tplarson 1.1   set errlist = `echo $errlist1 $errlist2 | sed s/" "/"\n"/g | sort -n | uniq`
192              
193                if ($#rerunlist || $#errlist) then
194                  echo some jobs still fail, i give up
195                  exit 1
196                endif
197              endif
198              
199              rm parms.tmp
200              
201              echo successful completion
202              exit 0

Karen Tian
Powered by
ViewCVS 0.9.4