1 phil 1.1 #! /bin/csh -f
2 # set echo
3
4 # gatekeeper
5
|
6 phil 1.4
7 set CADENCE = 10
|
8 phil 1.1 set FASTCADENCE = $CADENCE
9 set SLOWCADENCE = 60
10
11 if ($?WORKFLOW_ROOT) then
12 set WFDIR = $WORKFLOW_DATA
13 set WFCODE = $WORKFLOW_ROOT
14 else
15 echo Need WORKFLOW_ROOT variable to be set.
16 exit 1
17 endif
18
19 cd $WFDIR
|
20 phil 1.5 echo $$ >> restart.log
|
21 phil 1.4
22 echo $USER > Gatekeeper_owner
23 echo $HOST'.'$$ > Keep_running
|
24 phil 1.1 set keep_running = 1
25
|
26 arta 1.6 while ($keep_running > 0) # KEEPRUNNING LOOP
|
27 phil 1.1
28 echo " "
29 echo " "
30 echo " "
|
31 phil 1.5 echo `/bin/date +%Y.%m.%d_%H:%M:%S`
|
32 phil 1.1 echo "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX "
33
34 # gatekkeper run mode management
35 cd $WFDIR
36
37 if (-e GATEKEEPER_VERBOSE) then
38 set verbosemode = `cat GATEKEEPER_VERBOSE`
39 else
40 set verbosemode = 1
41 endif
42
43 if (-e GATEKEEPER_DEBUG) then
44 set debugmode = `cat GATEKEEPER_DEBUG`
45 else
46 set debugmode = 1
47 endif
48
|
49 jeneen 1.8 $WFCODE/scripts/checkDRMSnSUMS.csh
50 if ($?) then
51 echo "GATEKEEPER, DRMS and/or SUMS is down, try again in a minute."
52 set CADENCE = $SLOWCADENCE
53 goto ALL_GATES_DONE
54 else
|
55 phil 1.1 set CADENCE = $FASTCADENCE
|
56 jeneen 1.8 endif
|
57 phil 1.1
58 touch GATEKEEPERBUSY
59
60 set nowtxt = `date +%Y.%m.%d_%H:%M:%S -u`
61 set now = `time_convert time=$nowtxt`
62 # echo set now = `time_convert time=$nowtxt`
63
64 # inspect each gate and deal with all tickets at that gate
|
65 arta 1.6 foreach gate (`/bin/ls gates`) # GATE LOOP
|
66 phil 1.1 cd $WFDIR/gates/$gate
|
67 phil 1.2 # get gate information attributes
|
68 phil 1.1 if (-e statusbusy) then
|
69 phil 1.2 if ($verbosemode) echo "GATEKEEPER statustask running, skip gate for now"
|
70 arta 1.6 continue # NEXT GATE
|
71 phil 1.1 endif
72 set nextupdate = `cat nextupdate`
73 set updatedelta = `cat updatedelta`
74 set type = `cat type`
75 set actiontask = `cat actiontask`
76 set gatestatus = `cat gatestatus`
77 set product = `cat product`
|
78 phil 1.2 set statustask = `cat statustask`
|
79 phil 1.1 if ($gatestatus == "HOLD") then
80 if ($verbosemode) echo "GATEKEEPER Gate: $gate on HOLD, skip this gate"
|
81 arta 1.6 continue # NEXT GATE - This makes sense, since this gate is busy - go on to the next gate.
|
82 phil 1.1 endif
83 echo starting $gate
84 # Do general update if it is time
85 if ($verbosemode) echo "GATEKEEPER Gate: $gate, Starting to check for update times expired"
86 # echo set nextupdatetime = `time_convert time=$nextupdate`
87 set nextupdatetime = `time_convert time=$nextupdate`
88 if ($?) set nextupdatetime = now
89 if ($verbosemode) echo "GATEKEEPER Gate: $gate, now = $now"
90 if ($now >= $nextupdatetime) then
91 @ nextupdatetime = $now + $updatedelta
92 time_convert s=$nextupdatetime > nextupdate
93 echo $nowtxt > lastupdate
94 touch statusbusy
95 if ($gate == "clock_gate") then
|
96 phil 1.2 $WFCODE/$statustask $gate # do clock_gate inline
|
97 phil 1.1 else
|
98 phil 1.2 $WFCODE/$statustask $gate &
|
99 arta 1.6 continue # NEXT GATE
100 # stop with this gate until update is done
|
101 phil 1.1 endif
102 endif
103 set low = `cat low`
104 set high = `cat high`
105 if ($verbosemode) echo "GATEKEEPER Gate: $gate, Set updated low=$low, high=$high"
106 if ($#low == 0 || $low == "NaN" || $#high == 0 || $high == "NaN") then
107 if ($verbosemode) echo "GATEKEEPER Gate $gate not properly initialized, try to init"
108 touch statusbusy
|
109 phil 1.2 $WFCODE/$statustask $gate &
|
110 arta 1.6 continue # NEXT GATE
111 # stop with this gate until update is done
|
112 phil 1.1 endif
|
113 phil 1.3 # if ($low == -1) then
114 # echo "GATEKEEPER DEBUG $gate low is -1, reset"
115 # ls active_tickets
116 # echo NaN >low
117 # touch statusbusy
118 # $WFCODE/$statustask $gate &
119 # continue
120 # endif
|
121 phil 1.1 if ($type == "time") then
122 set low_t = `time_convert time=$low`
123 # echo set low_t = `time_convert time=$low`
124 set high_t = `time_convert time=$high`
125 # echo set high_t = `time_convert time=$high`
126 else
127 set low_t = $low
128 set high_t = $high
129 endif
130
131 # inspect all new tickets and disposition
|
132 arta 1.6 foreach ticket (`/bin/ls new_tickets`) # TICKET LOOP
|
133 phil 1.1 if ($verbosemode) echo "GATEKEEPER Start processing new ticket $ticket"
134 echo $nowtxt > $WFDIR/LAST_NEWTICKET
135 # get ticket key values
|
136 arta 1.6 foreach key (ACTION WANTLOW WANTHIGH EXPIRES) # KEY LOOP
|
137 phil 1.1 # set setval = `grep ^$key new_tickets/$ticket`
138 set setval = `grep $key new_tickets/$ticket`
139 if ($#setval == 1) set $setval
|
140 arta 1.6 end # KEY LOOP
|
141 phil 1.1
142 if ($type == "time") then
143 set WANTLOW_t = `time_convert time=$WANTLOW`
144 # echo set WANTLOW_t = `time_convert time=$WANTLOW`
145 set WANTHIGH_t = `time_convert time=$WANTHIGH`
146 # echo set WANTHIGH_t = `time_convert time=$WANTHIGH`
147 else
148 set WANTLOW_t = $WANTLOW
149 set WANTHIGH_t = $WANTHIGH
150 endif
151
152 # check for old stale ticket
153 set expirestime = `time_convert time=$EXPIRES`
154 # echo set expirestime = `time_convert time=$EXPIRES`
155 if ($expirestime < $now) then
156 if ($verbosemode) echo GATEKEEPER TIMEOUT of new ticket, $expirestime "<" $now
157 echo "STATUS=4" >> new_tickets/$ticket
158 mv new_tickets/$ticket active_tickets
|
159 arta 1.6 continue # NEXT TICKET
|
160 phil 1.1 endif
161
162 # disposition new tickets
163 # tickets that simply ask for status are completed and returned
164 # any request that will take processing time will have that processing started then
165 # the tickets are moved to the active ticket queue
166 # and are waited for in the next section.
167 if ($verbosemode) echo "GATEKEEPER examine new ticket $ticket for action = $ACTION"
|
168 phil 1.4
|
169 phil 1.1 if ($ACTION == 1) then # get low and high range for gate.
170 echo "GATELOW=$low" >> new_tickets/$ticket
171 echo "GATEHIGH=$high" >> new_tickets/$ticket
172 echo "STATUS=0" >> new_tickets/$ticket
173 mv new_tickets/$ticket active_tickets
|
174 phil 1.4
|
175 phil 1.1 else if ($ACTION == 2) then # check for wanted range, answer yes or no
176 if ($low_t > $WANTLOW_t || $high_t < $WANTHIGH_t) then
177 if ($low_t > $WANTLOW_t) then
178 echo "GATELOW=$low" >> new_tickets/$ticket
179 else
180 echo "GATELOW=$WANTLOW" >> new_tickets/$ticket
181 endif
182 if ($high_t < $WANTHIGH_t) then
183 echo "GATEHIGH=$high" >> new_tickets/$ticket
184 else
185 echo "GATEHIGH=$WANTHIGH" >> new_tickets/$ticket
186 endif
187 echo "STATUS=1" >> new_tickets/$ticket
188 else
189 echo "GATELOW=$WANTLOW" >> new_tickets/$ticket
190 echo "GATEHIGH=$WANTHIGH" >> new_tickets/$ticket
191 echo "STATUS=0" >> new_tickets/$ticket
192 endif
193 mv new_tickets/$ticket active_tickets
|
194 phil 1.4
195 else if ($ACTION == 3) then # wait for data, set status=2 and wait for later if data is not ready
|
196 phil 1.1 echo "STATUS=2" >> new_tickets/$ticket
197 mv new_tickets/$ticket active_tickets
|
198 phil 1.4
|
199 phil 1.1 else if ($ACTION == 4 || $ACTION == 5) then # start actiontask to do 4 and 5
200 if ($verbosemode) echo GATEKEEPER ACTION=5 started for ticket $ticket
201 echo "STATUS=3" >> new_tickets/$ticket
202 mv new_tickets/$ticket active_tickets # must happen before actiontask is started
|
203 phil 1.4
|
204 phil 1.1 set pending = `cat $WFDIR/tasks/$actiontask/state`
205 @ pending = $pending + 1
206 echo $pending > $WFDIR/tasks/$actiontask/state
|
207 phil 1.4
|
208 phil 1.2 set TASKMANAGER = `cat $WFDIR/tasks/$actiontask/manager`
|
209 phil 1.4 mkdir $WFDIR/tasks/$actiontask/logs/$ticket
210 $WFCODE/$TASKMANAGER gate=$gate task=$actiontask ticket=$ticket >>&$WFDIR/tasks/$actiontask/logs/$ticket/manager.log &
211 echo $! > $WFDIR/tasks/$actiontask/logs/$ticket/manager.pid
212
|
213 phil 1.1 else if ($ACTION == 6) then # make coverage map, may be slow
214 # ACTION == 6, Generate COVERAGE map
215 echo "NaN" > low
216 echo "NaN" > high
217 echo "STATUS=2" >> new_tickets/$ticket
218 mv new_tickets/$ticket active_tickets # must happen before statustask
219 touch statusbusy
|
220 phil 1.2 $WFCODE/$statustask $gate low=$low high=$high &
|
221 phil 1.1 endif
222 if ($verbosemode) echo GATEKEEPER moved new_tickets/$ticket to active_tickets with action=$ACTION
|
223 arta 1.6 end # TICKET LOOP
|
224 phil 1.1
225 if ($verbosemode) echo GATEKEEPER done with new_tickets, examine this gates action task done list
226
227 # PROCESS EXISTING TICKETS
228 # First, since only the gatekeeper may change the content of a ticket, examine this gate's
229 # action task to see any tickets are in the done queue. If they are, update their status
230 # and return them to their parent task.
231 # actiontask is the task associated with this gate, the task that the ticket is sent to.
232 # TASKID is the parent task instance of the ticket.
233 # task is the taskname of the parent task -the one that wanted the processing.
234 # The tickets in the task instances in the done directory have just finished
235 # processing at this gate.
236 # look to see if the active ticket is one that has its processing finished.
237 set thisgatedir = $cwd
238 cd $WFDIR/tasks/$actiontask/done/
239 if ($verbosemode) echo GATEKEEPER change to $cwd
|
240 arta 1.6 foreach donetask ( `/bin/ls` ) # DONE TASK LOOP
|
241 phil 1.1 # see if there is a ticket in this gate's actiontask done queue
242 # get info about this ticket.
243 if ($verbosemode) echo "GATEKEEPER look at done task $donetask"
|
244 phil 1.5 set lookroot = $donetask:s/-/./:e
245 if ($lookroot == root) then
246 echo found done task for $actiontask
247 set root_done = 1
248 else
249 set root_done = 0
250 endif
|
251 arta 1.6 foreach key (TICKET GATE TASKID) # KEY LOOP
|
252 phil 1.1 set setval = `grep $key $donetask/ticket`
|
253 phil 1.5 if ($#setval) then
254 set $setval
255 else
256 set $key = VOID
257 endif
|
258 arta 1.6 end # DONE KEY LOOP
|
259 phil 1.5 if ($TASKID == VOID) then
260 set TASKID = $donetask
261 endif
262 # set parenttask = `echo $TASKID | sed -e 's/-.*//'`
263 set parenttask = $TASKID:s/-/./:r
264 if ($TICKET != VOID) rm -f ../logs/$TICKET/manager.pid
|
265 phil 1.1 if ($verbosemode) echo "GATEKEEPER parent task is $parenttask"
266 ex - $donetask/ticket <<!
267 /STATUS/d
268 w
269 q
270 !
|
271 phil 1.5 if ($debugmode) ls -l $donetask/ticket
|
272 phil 1.1 set taskstate = `cat $donetask/state`
273 if ($taskstate == 0) then
274 echo "STATUS=0" >> $donetask/ticket
275 mv $donetask ../archive/ok
276 else
277 echo "STATUS=5" >> $donetask/ticket
278 mv $donetask ../archive/failed
|
279 phil 1.5 echo " " >> $WFDIR/FAILED_TASKS
280 echo -n "$actiontask FAILED for " >>$WFDIR/FAILED_TASKS
281 echo -n `grep WANTLOW $WFDIR/tasks/$actiontask/archive/failed/$donetask/ticket` >> $WFDIR/FAILED_TASKS
282 echo -n " " >> $WFDIR/FAILED_TASKS
283 echo -n `grep WANTHIGH $WFDIR/tasks/$actiontask/archive/failed/$donetask/ticket` >> $WFDIR/FAILED_TASKS
284 echo -n " at " >> $WFDIR/FAILED_TASKS
285 date >> $WFDIR/FAILED_TASKS
286 echo " See:" $WFDIR/tasks/$actiontask/archive/failed/$donetask >> $WFDIR/FAILED_TASKS
|
287 phil 1.1 endif
|
288 phil 1.5 if (-e ../logs/$TICKET) mv ../logs/$TICKET ../archive/logs
|
289 phil 1.1 if ($verbosemode) echo "GATEKEEPER done queue, move $GATE/active_tickets/$TICKET to /$parenttask/active/$TASKID/ticket_return"
290 if (-e $WFDIR/gates/$GATE/active_tickets/$TICKET) then
291 mv $WFDIR/gates/$GATE/active_tickets/$TICKET $WFDIR/tasks/$parenttask/active/$TASKID/ticket_return
292 endif
293 if (-e $WFDIR/tasks/$parenttask/active/$TASKID/pending_tickets/$TICKET) then
294 rm $WFDIR/tasks/$parenttask/active/$TASKID/pending_tickets/$TICKET
295 else if (-e $WFDIR/tasks/$parenttask/archive/ok/$TASKID/pending_tickets/$TICKET) then
|
296 phil 1.5 # rm $WFDIR/tasks/$parenttask/active/$TASKID/pending_tickets/$TICKET
|
297 phil 1.1 else
298 echo "GATEKEEPER done queue, FAILED to rm $WFDIR/tasks/$parenttask/active/$TASKID/pending_tickets/$TICKET"
299 endif
|
300 arta 1.6 end # DONE TASK
301 # processing this gate's action task done list for donetask
|
302 phil 1.1 cd $thisgatedir
303
304 if ($verbosemode) echo "GATEKEEPER DONE with done queue, start at $gate active_tickets"
305 # examine all existing tickets, examine status then do action
|
306 arta 1.6 foreach ticket (`/bin/ls active_tickets`) # TICKET LOOP
|
307 phil 1.1 # get ticket attributes and check for timeout
308 echo $gate existing ticket $ticket
309 if ($verbosemode) echo "GATEKEEPER Start processing active ticket $ticket"
|
310 arta 1.6 foreach key (ACTION STATUS TASKID EXPIRES WANTLOW WANTHIGH) # KEY LOOP
|
311 phil 1.1 set setval = `grep $key active_tickets/$ticket`
312 if ($#setval == 1) set $setval
|
313 arta 1.6 end # KEY LOOP
|
314 phil 1.1 set EXPIRES_t = `time_convert time=$EXPIRES`
315 # echo set EXPIRES_t = `time_convert time=$EXPIRES`
316 if ($type == "time") then
317 set WANTLOW_t = `time_convert time=$WANTLOW`
318 # echo set WANTLOW_t = `time_convert time=$WANTLOW`
319 set WANTHIGH_t = `time_convert time=$WANTHIGH`
320 # echo set WANTHIGH_t = `time_convert time=$WANTHIGH`
321 else
322 set WANTLOW_t = $WANTLOW
323 set WANTHIGH_t = $WANTHIGH
324 endif
325 if ($verbosemode) echo "GATEKEEPER check for timeout"
326
327 # check expire time on working tickets
328 if ( ($STATUS == 3 || $STATUS == 2) && $EXPIRES_t < $now) then
329 if ($verbosemode) echo GATEKEEPER TIMEOUT of ticket $ticket
330 ex - active_tickets/$ticket <<!
331 /STATUS/d
332 w
333 q
334 !
335 phil 1.1 echo "STATUS=4" >> active_tickets/$ticket
336 set STATUS=4
337 set task = `echo $TASKID | sed -e 's/-.*//'`
338 if (-e WFDIR/tasks/$task/active/$TASKID) then
339 mv active_tickets/$ticket $WFDIR/tasks/$task/active/$TASKID/ticket_return
340 rm $WFDIR/tasks/$task/active/$TASKID/pending_tickets/$ticket
341 else if (-e WFDIR/tasks/$task/archive/ok/$TASKID) then
342 mv active_tickets/$ticket $WFDIR/tasks/$task/archive/ok/$TASKID/ticket_return
343 rm $WFDIR/tasks/$task/archive/ok/$TASKID/pending_tickets/$ticket
344 else
345 echo "GATEKEEPER can not do mv active_tickets/$ticket $WFDIR/tasks/$task/active/$TASKID/ticket_return"
346 endif
347 endif
348
349 # if new slow ticket type, initiate if new, else check status and wait
350 if ($STATUS == 2) then # means getting coverage map or wait passively
351 if ($verbosemode) echo "GATEKEEPER STATUS is 2"
352 if (-e statusbusy) then
353 if ($verbosemode) echo "GATEKEEPER StatusCommand is running"
|
354 arta 1.6 continue # NEXT TICKET
|
355 phil 1.1 endif
356 set low = `cat low`
357 if ($#low == 0) then
358 echo XXXX got null, sleeping
359 sleep 1
360 set low = `cat low`
361 endif
362 set high = `cat high`
363 if ($#high == 0) then
364 echo XXXX got null, sleeping
365 sleep 1
366 set high = `cat high`
367 endif
368 # if ($#low == 0 || $low == "NaN" || $#high == 0 || $high == "NaN") then
369 # if ($verbosemode) echo "GATEKEEPER Gate $gate not properly initialized"
370 # exit 1
371 # endif
372 if ($type == "time") then
|
373 phil 1.3 # echo XXXXX
374 # cat active_tickets/$ticket
375 # echo XXXXX low=$low,
376 # echo XXXXX set low_t = `time_convert time=$low`
|
377 phil 1.1 set low_t = `time_convert time=$low`
378
|
379 phil 1.3 # echo xxxxx high=$high,
380 # echo xxxxx set high_t = `time_convert time=$high`
|
381 phil 1.1 set high_t = `time_convert time=$high`
382 else
383 set low_t = $low
384 set high_t = $high
385 endif
386 if ($ACTION == 3 && $WANTLOW_t >= $low_t && $WANTHIGH_t <= $high_t ) then
387 if ( $verbosemode ) echo "GATEKEEPER ACTION = 3, waiting done"
388 ex - active_tickets/$ticket << !
389 /STATUS/d
390 w
391 q
392 !
393 echo "GATELOW=$WANTLOW" >> active_tickets/$ticket
394 echo "GATEHIGH=$WANTHIGH" >> active_tickets/$ticket
395 echo "STATUS=0" >> active_tickets/$ticket
396 set STATUS = 0
397 else if ($ACTION == 6) then
398 # ACTION == 6
399 if ($verbosemode) echo "GATEKEEPER ACTION = 6"
400 # XXXXX this logic will need to change when coverage map implemented
401 if ($low != "NaN" && $high != "NaN") then # Coverage map must be complete now
402 phil 1.1 ex - active_tickets/$ticket <<!
403 /STATUS/d
404 w
405 q
406 !
407 echo "GATELOW=$low" >> active_tickets/$ticket
408 echo "GATEHIGH=$high" >> active_tickets/$ticket
409 echo "STATUS=0" >> active_tickets/$ticket
410 set STATUS = 0
411 endif
412 endif
413 # Action was 3 or 6
414 endif
415 # STATUS == 2
416
417 # if ticket processing is complete, return ticket to sender
418 set task = `echo $TASKID | sed -e 's/-.*//'`
419 if ($STATUS != 3 && $STATUS != 2) then # 2,3 means still owned by some task, else return ticket
420 if ($verbosemode) echo "GATEKEEPER Ticket is complete, return to owner"
421 # XXXXX need to update coverage here with want range in done ticket. For now call get coverage.
|
422 phil 1.4 echo Start statustask
|
423 phil 1.1 touch statusbusy
|
424 phil 1.2 $WFCODE/$statustask $gate # wait for this to complete.
|
425 phil 1.4 echo done statustask
|
426 phil 1.1 # ticket should have been already removed from the target task's pending list
427 # but check anyway
428 if (-e $WFDIR/tasks/$task/active/$TASKID/ticket_return) then
429 if ($verbosemode) echo "GATEKEEPER active ticket exam, move active_tickets/$ticket to $task/active/$TASKID/ticket_return"
430 mv active_tickets/$ticket $WFDIR/tasks/$task/active/$TASKID/ticket_return
431 rm $WFDIR/tasks/$task/active/$TASKID/pending_tickets/$ticket
432 endif
433 mv active_tickets/$ticket $WFDIR/tickets_done
434 endif
|
435 arta 1.6 end # TICKET LOOP
436 # done with process existing tickets
437 end # GATE LOOP
438 # done with tour of all gates
|
439 phil 1.1
440 # gatekeeper loop management
441 ALL_GATES_DONE:
442 cd $WFDIR
443 rm -f GATEKEEPERBUSY
444 if (!(-e Keep_running)) then
445 set keep_running = 0
446 else
|
447 phil 1.4 echo "XXXXXXXXXXXXXXXXX Start Sleeping XXXXXXXXXXXXXXXX"
|
448 phil 1.1 sleep $CADENCE
449 endif
450
|
451 arta 1.6 end # KEEPRUNNING LOOP
|
452 phil 1.1
453 echo Gate_Keeper Exit
|
454 phil 1.5 exit 0
|
455 phil 1.1
|
Return to
gatekeeper.csh
CVS log
Up to [Development] / JSOC / proj / workflow