(file) Return to doylmsht CVS log (file) (dir) Up to [Development] / JSOC / proj / globalhs / scripts

  1 tplarson 1.1 #!/bin/tcsh -f
  2              
  3 tplarson 1.2 setenv PATH $JSOCROOT/bin/$JSOC_MACHINE':'$JSOCROOT/proj/globalhs/scripts':'$PATH
  4              
  5              # submit njobs jobs at once
  6              if (! $?GLOBALHS_V2TNJOBS) then
  7                set njobs=20
  8              else
  9                set njobs=$GLOBALHS_V2TNJOBS
 10              endif
 11              
 12              # submit next batch of jobs when total number of sht jobs drops below jobthreshold
 13              if (! $?GLOBALHS_V2TJOBTHRESHOLD) then
 14                set jobthreshold=40
 15              else
 16                set jobthreshold=$GLOBALHS_V2TJOBTHRESHOLD
 17              endif
 18              
 19              # if this script inherited a label in its environment, use it as the suffix for job names
 20              if (! $?GLOBALHS_LABEL) then
 21                set suff=''
 22              else
 23                set suff=.$GLOBALHS_LABEL
 24 tplarson 1.2 endif
 25 tplarson 1.1 
 26              set qsubtmp=/tmp27/$USER/qsubtmp
 27              mkdir -p $qsubtmp
 28 tplarson 1.2 if (! $?GLOBALHS_V2TQUEUE) then
 29                set q=k.q
 30              else
 31                set q=$GLOBALHS_V2TQUEUE
 32              endif
 33              
 34              if ($q == k.q) then
 35                alias qsub qsub2
 36                alias qstat qstat2
 37                alias waittosubmit waittosubmit2
 38              endif
 39 tplarson 1.1 
 40              @ i = 1
 41              while ($i <= $#argv )
 42                set $argv[$i]
 43              @ i++
 44              end
 45              
 46              if (! $?deltal) set deltal = 6
 47              if (! $?xoffset) set xoffset = 0.0
 48              if (! $?yoffset) set yoffset = 0.0
 49              if (! $?pangle) set pangle = 0.0
 50              if (! $?bangle) set bangle = 0.0
 51              if (! $?obsdist) set obsdist = 1.000
 52              if (! $?segment) set segment = vradsum
 53 tplarson 1.2 set tag='x'$xoffset'_y'$yoffset'_p'$pangle'_b'$bangle'_d'$obsdist'_'$segment
 54 tplarson 1.1 
 55              echo `date` on $HOST
 56              echo $PWD
 57              echo $0 $argv
 58              
 59              if (! -es v2t.parms.blank) then
 60                echo parameter file blank missing: v2t.parms.blank is required
 61                exit 1
 62              endif
 63              
 64 tplarson 1.2 grep -v "^[[:space:]]*#" v2t.parms.blank | sed s@"[[:space:]]"@"\n"@g | grep -v '^[[:space:]]*$' > parms.tmp
 65              #set tagblank = `cat parms.tmp | grep TAG= | cut -d= -f2`
 66              #set tagsuff = $xoffset'_'$yoffset'_'$pangle'_'$bangle'_'$obsdist'_'$segment
 67              #set tag = `echo $tagblank | sed s/TTTT/$tagsuff/`
 68 tplarson 1.1 
 69              set in = `cat parms.tmp | grep in= | sed s/"'"/""/g | awk -F '[=[]' '{print $2}'`
 70              set trecstep = `show_info -j $in | grep -i '^keyword:t_rec_step' | cut -d, -f5`
 71              set ttotal = `cat parms.tmp | grep TTOTAL= | cut -d= -f2`
 72              set totalsecs = `durcon $ttotal`
 73              set ndt = `echo "$totalsecs/$trecstep" | bc`
 74              
 75 tplarson 1.2 if (! -es ../ylm.parms.blank) then
 76 tplarson 1.1   echo parameter file blank missing: ylm.parms.blank is required
 77                exit 1
 78              endif
 79              
 80 tplarson 1.2 set tstart = `cat ../ylm.parms.blank | grep -v "^[[:space:]]*#" | grep TSTART= | cut -d= -f2`
 81 tplarson 1.1 if ($#tstart == 0) then
 82              @ line = `mkylms -H | grep -n "TSTART (time)" | cut -d: -f1` + 1
 83                set tstart = `mkylms -H | sed -n $line'p' | awk '{print $3}'`
 84              endif
 85              set startsecs = `time_convert o=jsoc time=$tstart`
 86              
 87 tplarson 1.2 set file = `cat ../ylm.parms.blank | grep -v "^[[:space:]]*#" | grep MODELIST= | cut -d= -f2`
 88              if (! -es ../$file) then
 89 tplarson 1.1   echo MODELIST file $file is missing
 90                exit 1
 91              endif
 92              
 93 tplarson 1.2 set nim=`cat ../$file | wc -l`
 94 tplarson 1.1 @ nchunks = ( ( $nim - 1 ) / $ndt ) + 1
 95              #not necessary if while is replaced by foreach below
 96              #if ($njobs > $nchunks) set njobs=$nchunks
 97              
 98              set i=0
 99              while ($i < $nchunks)
100                set isecs = `echo "$startsecs + ($i * $ndt * $trecstep)" | bc`
101                set istart = `time_convert o=cal zone=tai s=$isecs`
102                @ index = ($i * $ndt) + 1
103                @ index2 = $index + $ndt - 1
104                if ($index2 > $nim) set index2 = $nim
105 tplarson 1.2   set lmin = `sed -n $index'p' ../$file | awk '{print $1}'`
106 tplarson 1.1   @ lmin = $lmin - $deltal
107                if ($lmin < 0) set lmin=0
108 tplarson 1.2   set lmax = `sed -n $index2'p' ../$file | awk '{print $1}'`
109 tplarson 1.1   @ lmax = $lmax + $deltal
110 tplarson 1.2   cat parms.tmp | sed s@RRRR@$istart/$ndt'm'@ | sed s/MMMM/$lmax/ | sed s/NNNN/$lmin/ | sed s/SSSS/$istart/ | sed s/XXXX/$xoffset/g | sed s/YYYY/$yoffset/g | sed s/PPPP/$pangle/g | sed s/BBBB/$bangle/g | sed s/DDDD/$obsdist/g | sed s/GGGG/$segment/g | sed s/TTTT/$tag/g  > v2t.parms.$i
111 tplarson 1.1   @ r = $i % $njobs
112 tplarson 1.2   set subfile = subt.$r$suff'_'$tag
113 tplarson 1.1   if ($i < $njobs) then
114                  echo '#\!/bin/csh' > $subfile
115 tplarson 1.4     echo 'setenv PATH' $JSOCROOT'/bin/$JSOC_MACHINE' >> $subfile
116 tplarson 1.1     echo 'cd' $PWD >> $subfile
117                endif
118 tplarson 1.2   echo \(time jv2ts @v2t.parms.$i\) '>&' v2t.log.$i >> $subfile
119                echo echo \$status '>&' jv2ts.exitstatus.$i >> $subfile
120 tplarson 1.1   @ i++
121              end
122              
123              waittosubmit $jobthreshold subt
124 tplarson 1.2 set sublist=`/bin/ls subt.*_$tag`
125 tplarson 1.1 foreach sub ($sublist)
126                qsub -q $q -e $qsubtmp -o $qsubtmp $sub
127              end
128              
129              #set i=0
130              #while ($i < $njobs)
131              #  qsub -q $q -e $qsubtmp -o $qsubtmp subt.$i$suff
132              #@ i++
133              #end
134              
135              echo jobs submitted, start waiting
136              
137 tplarson 1.2 set njobsrunning = `qstat -r -u $USER | grep "Full jobname:" | grep subt | grep $suff'_'$tag | wc -l`
138 tplarson 1.1 while($njobsrunning > 0)
139                sleep 60
140 tplarson 1.2   set njobsrunning = `qstat -r -u $USER | grep "Full jobname:" | grep subt | grep $suff'_'$tag | wc -l`
141 tplarson 1.1 end
142              
143 tplarson 1.2 set expectedlist = `/bin/ls v2t.parms.[0-9]* | cut -d'.' -f 3`
144              set ranlist   = `/bin/ls v2t.log.* | cut -d'.' -f 3`
145 tplarson 1.1 set rerunlist = `echo $expectedlist $ranlist | sed s/" "/"\n"/g | sort -n | uniq -u`
146              
147 tplarson 1.2 set errlist1 = `grep -Hv 0 jv2ts.exitstatus.* | awk -F '[.:]' '{print $3}'`
148              set errlist2 = `grep -Hc "successful completion" v2t.log.* | grep -v :1 | awk -F '[.:]' '{print $3}'`
149 tplarson 1.1 set errlist = `echo $errlist1 $errlist2 | sed s/" "/"\n"/g | sort -n | uniq`
150              
151              set list = `echo $rerunlist $errlist | sed s/" "/"\n"/g | sort -n`
152              if ($#list) then
153                echo some jobs fail, rerunning
154                echo rerunlist: $rerunlist > faillog.$segment
155                echo errlist1: $errlist1 >> faillog.$segment
156                echo errlist2: $errlist2 >> faillog.$segment
157                mkdir -p savesubt
158 tplarson 1.2   mv subt.*_$tag savesubt
159 tplarson 1.1 
160                #now have to start with 1 because that is the first index of list
161                set i=1
162                while ($i <= $#list)
163                  @ r = $i % $njobs
164 tplarson 1.2     set subfile = subt.$r$suff'_'$tag
165 tplarson 1.1     if ($i <= $njobs) then
166                    echo '#\!/bin/csh' > $subfile
167 tplarson 1.4       echo 'setenv PATH' $JSOCROOT'/bin/$JSOC_MACHINE' >> $subfile
168 tplarson 1.1       echo 'cd' $PWD >> $subfile
169                  endif
170 tplarson 1.2     echo \(time jv2ts @v2t.parms.$list[$i]\) '>&' v2t.log.$list[$i] >> $subfile
171                  echo echo \$status '>&' jv2ts.exitstatus.$list[$i] >> $subfile
172 tplarson 1.1     @ i++
173                end
174              
175                waittosubmit $jobthreshold subt
176 tplarson 1.2   set sublist=`/bin/ls subt.*_$tag`
177 tplarson 1.1   foreach sub ($sublist)
178                  qsub -q $q -e $qsubtmp -o $qsubtmp $sub
179                end
180              
181 tplarson 1.3   set njobsrunning = `qstat -r -u $USER | grep "Full jobname:" | grep subt | grep $suff'_'$tag | wc -l`
182                while($njobsrunning > 0)
183 tplarson 1.1     sleep 60
184 tplarson 1.3     set njobsrunning = `qstat -r -u $USER | grep "Full jobname:" | grep subt | grep $suff'_'$tag | wc -l`
185 tplarson 1.1   end
186              
187 tplarson 1.2   set expectedlist = `/bin/ls v2t.parms.* | cut -d'.' -f 3`
188                set ranlist   = `/bin/ls v2t.log.* | cut -d'.' -f 3`
189 tplarson 1.1   set rerunlist = `echo $expectedlist $ranlist | sed s/" "/"\n"/g | sort -n | uniq -u`
190              
191 tplarson 1.2   set errlist1 = `grep -Hv 0 jv2ts.exitstatus.* | awk -F '[.:]' '{print $3}'`
192                set errlist2 = `grep -Hc "successful completion" v2t.log.* | grep -v :1 | awk -F '[.:]' '{print $3}'`
193 tplarson 1.1   set errlist = `echo $errlist1 $errlist2 | sed s/" "/"\n"/g | sort -n | uniq`
194              
195                if ($#rerunlist || $#errlist) then
196                  echo some jobs still fail, i give up
197                  exit 1
198                endif
199              endif
200              
201              rm parms.tmp
202              
203              echo successful completion
204              exit 0

Karen Tian
Powered by
ViewCVS 0.9.4