(file) Return to gatekeeper.csh CVS log (file) (dir) Up to [Development] / JSOC / proj / workflow

  1 phil  1.1 #! /bin/csh -f
  2           # set echo
  3           
  4           # gatekeeper
  5           
  6 phil  1.4 
  7           set CADENCE = 10
  8 phil  1.1 set FASTCADENCE = $CADENCE
  9           set SLOWCADENCE = 60
 10           
 11           if ($?WORKFLOW_ROOT) then
 12             set WFDIR = $WORKFLOW_DATA
 13             set WFCODE = $WORKFLOW_ROOT
 14           else
 15             echo Need WORKFLOW_ROOT variable to be set.
 16             exit 1
 17           endif
 18           
 19           cd $WFDIR
 20 phil  1.5 echo $$ >> restart.log
 21 phil  1.4 
 22           echo $USER > Gatekeeper_owner
 23           echo $HOST'.'$$ > Keep_running
 24 phil  1.1 set keep_running = 1
 25           
 26 arta  1.6 while ($keep_running > 0) # KEEPRUNNING LOOP
 27 phil  1.1 
 28               echo " "
 29               echo " "
 30               echo " "
 31 phil  1.5     echo `/bin/date +%Y.%m.%d_%H:%M:%S`
 32 phil  1.1     echo "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX "
 33           
 34               # gatekkeper run mode management
 35               cd $WFDIR
 36           
 37               if (-e GATEKEEPER_VERBOSE) then
 38                 set verbosemode = `cat GATEKEEPER_VERBOSE`
 39               else
 40                 set verbosemode = 1
 41               endif
 42               
 43               if (-e GATEKEEPER_DEBUG) then
 44                 set debugmode = `cat GATEKEEPER_DEBUG`
 45               else
 46                 set debugmode = 1
 47               endif
 48           
 49 jeneen 1.8    $WFCODE/scripts/checkDRMSnSUMS.csh
 50               if ($?) then 
 51                  echo "GATEKEEPER,  DRMS and/or SUMS is down, try again in a minute."
 52                  set CADENCE = $SLOWCADENCE
 53                  goto ALL_GATES_DONE
 54               else
 55 phil   1.1       set CADENCE = $FASTCADENCE
 56 jeneen 1.8    endif
 57 phil   1.1 
 58               touch GATEKEEPERBUSY
 59            
 60               set nowtxt = `date +%Y.%m.%d_%H:%M:%S -u` 
 61               set now = `time_convert time=$nowtxt`
 62            # echo set now = `time_convert time=$nowtxt`
 63            
 64               # inspect each gate and deal with all tickets at that gate
 65 arta   1.6 foreach gate (`/bin/ls gates`) # GATE LOOP
 66 phil   1.1 	cd $WFDIR/gates/$gate
 67 phil   1.2         # get gate information attributes
 68 phil   1.1         if (-e statusbusy) then
 69 phil   1.2 		if ($verbosemode) echo "GATEKEEPER statustask running, skip gate for now"
 70 arta   1.6 continue # NEXT GATE 
 71 phil   1.1         endif
 72                    set nextupdate = `cat nextupdate`
 73                    set updatedelta = `cat updatedelta`
 74                    set type = `cat type`
 75                    set actiontask = `cat actiontask`
 76                    set gatestatus = `cat gatestatus`
 77                    set product = `cat product`
 78 phil   1.2         set statustask = `cat statustask`
 79 phil   1.1         if ($gatestatus == "HOLD") then
 80                        if ($verbosemode) echo "GATEKEEPER Gate: $gate on HOLD, skip this gate"
 81 arta   1.6 continue # NEXT GATE - This makes sense, since this gate is busy - go on to the next gate.
 82 phil   1.1         endif
 83            echo starting $gate
 84                    # Do general update if it is time
 85            	if ($verbosemode) echo "GATEKEEPER Gate: $gate, Starting to check for update times expired"
 86            # echo set nextupdatetime = `time_convert time=$nextupdate`
 87                    set nextupdatetime = `time_convert time=$nextupdate`
 88                    if ($?) set nextupdatetime = now
 89            	if ($verbosemode) echo "GATEKEEPER Gate: $gate, now = $now"
 90                    if ($now >= $nextupdatetime) then
 91            		@ nextupdatetime = $now + $updatedelta
 92            		time_convert s=$nextupdatetime > nextupdate
 93            		echo $nowtxt > lastupdate
 94                            touch statusbusy
 95                            if ($gate == "clock_gate") then
 96 phil   1.2                         $WFCODE/$statustask $gate # do clock_gate inline
 97 phil   1.1                 else
 98 phil   1.2                         $WFCODE/$statustask $gate & 
 99 arta   1.6 continue # NEXT GATE 
100                     # stop with this gate until update is done
101 phil   1.1                 endif
102            	endif
103                    set low = `cat low`
104                    set high = `cat high`
105            	if ($verbosemode) echo "GATEKEEPER Gate: $gate, Set updated low=$low, high=$high"
106            	if ($#low == 0 || $low == "NaN" || $#high == 0 || $high == "NaN") then
107            		if ($verbosemode) echo "GATEKEEPER Gate $gate not properly initialized, try to init"
108            		touch statusbusy
109 phil   1.2                 $WFCODE/$statustask $gate & 
110 arta   1.6 continue # NEXT GATE
111                     # stop with this gate until update is done
112 phil   1.1 	endif
113 phil   1.3 # if ($low == -1) then
114            # echo "GATEKEEPER DEBUG $gate low is -1, reset"
115            # ls active_tickets
116            # echo NaN >low
117            # touch statusbusy
118            # $WFCODE/$statustask $gate & 
119            # continue
120            # endif
121 phil   1.1         if ($type == "time") then
122            		set low_t = `time_convert time=$low`
123            # echo set low_t = `time_convert time=$low`
124            		set high_t = `time_convert time=$high`
125            # echo set high_t = `time_convert time=$high`
126                    else
127            		set low_t = $low
128            		set high_t = $high
129                    endif
130            
131                    # inspect all new tickets and disposition
132 arta   1.6 foreach ticket (`/bin/ls new_tickets`) # TICKET LOOP
133 phil   1.1                 if ($verbosemode) echo "GATEKEEPER Start processing new ticket $ticket"
134                            echo $nowtxt > $WFDIR/LAST_NEWTICKET
135                            # get ticket key values
136 arta   1.6 foreach key (ACTION WANTLOW WANTHIGH EXPIRES) # KEY LOOP
137 phil   1.1 			# set setval = `grep ^$key new_tickets/$ticket`
138            			set setval = `grep $key new_tickets/$ticket`
139                                    if ($#setval == 1) set $setval
140 arta   1.6 end # KEY LOOP
141 phil   1.1 
142                    	if ($type == "time") then
143            			set WANTLOW_t = `time_convert time=$WANTLOW`
144            # echo set WANTLOW_t = `time_convert time=$WANTLOW`
145            			set WANTHIGH_t = `time_convert time=$WANTHIGH`
146            # echo set WANTHIGH_t = `time_convert time=$WANTHIGH`
147            		else
148            			set WANTLOW_t = $WANTLOW
149            			set WANTHIGH_t = $WANTHIGH
150            		endif
151            
152                            # check for old stale ticket
153                            set expirestime = `time_convert time=$EXPIRES`
154            # echo set expirestime = `time_convert time=$EXPIRES`
155            		if ($expirestime < $now) then
156                                    if ($verbosemode) echo GATEKEEPER TIMEOUT of new ticket, $expirestime "<" $now
157            			echo "STATUS=4" >> new_tickets/$ticket
158            			mv new_tickets/$ticket active_tickets
159 arta   1.6 continue # NEXT TICKET
160 phil   1.1 		endif
161            
162                            # disposition new tickets
163            		# tickets that simply ask for status are completed and returned
164            		# any request that will take processing time will have that processing started then
165            		# the tickets are moved to the active ticket queue
166            		# and are waited for in the next section.
167                            if ($verbosemode) echo "GATEKEEPER examine new ticket $ticket for action = $ACTION"
168 phil   1.4 
169 phil   1.1 		if ($ACTION == 1) then  # get low and high range for gate.
170            			echo "GATELOW=$low" >> new_tickets/$ticket
171            			echo "GATEHIGH=$high" >> new_tickets/$ticket
172            			echo "STATUS=0" >> new_tickets/$ticket
173            			mv new_tickets/$ticket active_tickets
174 phil   1.4 
175 phil   1.1 		else if ($ACTION == 2) then # check for wanted range, answer yes or no
176            			if ($low_t > $WANTLOW_t || $high_t < $WANTHIGH_t) then
177            				if ($low_t > $WANTLOW_t) then
178            					echo "GATELOW=$low" >> new_tickets/$ticket
179            				else
180            					echo "GATELOW=$WANTLOW" >> new_tickets/$ticket
181            				endif
182            				if ($high_t < $WANTHIGH_t) then
183            					echo "GATEHIGH=$high" >> new_tickets/$ticket
184            				else
185            					echo "GATEHIGH=$WANTHIGH" >> new_tickets/$ticket
186            				endif
187            				echo "STATUS=1" >> new_tickets/$ticket
188            			else
189            				echo "GATELOW=$WANTLOW" >> new_tickets/$ticket
190            				echo "GATEHIGH=$WANTHIGH" >> new_tickets/$ticket
191            				echo "STATUS=0" >> new_tickets/$ticket
192            			endif
193            			mv new_tickets/$ticket active_tickets
194 phil   1.4 
195            		else if ($ACTION == 3) then # wait for data, set status=2 and wait for later if data is not ready
196 phil   1.1 			echo "STATUS=2" >> new_tickets/$ticket
197            			mv new_tickets/$ticket active_tickets
198 phil   1.4 
199 phil   1.1 		else if ($ACTION == 4 || $ACTION == 5) then # start actiontask to do 4 and 5
200                                    if ($verbosemode) echo GATEKEEPER ACTION=5 started for ticket $ticket
201            			echo "STATUS=3" >> new_tickets/$ticket
202            			mv new_tickets/$ticket active_tickets # must happen before actiontask is started
203 phil   1.4 
204 phil   1.1                         set pending = `cat $WFDIR/tasks/$actiontask/state`
205            			@ pending = $pending + 1
206                                    echo $pending >  $WFDIR/tasks/$actiontask/state
207 phil   1.4 
208 phil   1.2 			set TASKMANAGER = `cat $WFDIR/tasks/$actiontask/manager`
209 phil   1.4                         mkdir $WFDIR/tasks/$actiontask/logs/$ticket
210            			$WFCODE/$TASKMANAGER gate=$gate task=$actiontask ticket=$ticket >>&$WFDIR/tasks/$actiontask/logs/$ticket/manager.log  &
211            			echo $! > $WFDIR/tasks/$actiontask/logs/$ticket/manager.pid
212            
213 phil   1.1 		else if ($ACTION == 6) then # make coverage map, may be slow
214            			# ACTION == 6, Generate COVERAGE map
215            			echo "NaN" > low
216            			echo "NaN" > high
217            			echo "STATUS=2" >> new_tickets/$ticket
218            			mv new_tickets/$ticket active_tickets # must happen before statustask
219                                    touch statusbusy
220 phil   1.2                         $WFCODE/$statustask $gate low=$low high=$high &
221 phil   1.1 		endif
222            		if ($verbosemode) echo GATEKEEPER moved new_tickets/$ticket to active_tickets with action=$ACTION
223 arta   1.6 	end # TICKET LOOP
224 phil   1.1 
225            		if ($verbosemode) echo GATEKEEPER done with new_tickets, examine this gates action task done list
226            
227            # PROCESS EXISTING TICKETS
228                    # First, since only the gatekeeper may change the content of a ticket, examine this gate's
229                    # action task to see any tickets are in the done queue.  If they are, update their status
230                    # and return them to their parent task.
231            	#     actiontask is the task associated with this gate, the task that the ticket is sent to.
232            	#     TASKID is the parent task instance of the ticket.
233            	#     task is the taskname of the parent task -the one that wanted the processing.
234            	# The tickets in the task instances in the done directory have just finished
235                    # processing at this gate.
236            	# look to see if the active ticket is one that has its processing finished.
237            	set thisgatedir = $cwd
238            	cd $WFDIR/tasks/$actiontask/done/
239                    if ($verbosemode) echo GATEKEEPER change to $cwd
240 arta   1.6 foreach donetask ( `/bin/ls` ) # DONE TASK LOOP
241 phil   1.1             # see if there is a ticket in this gate's actiontask done queue
242                        # get info about this ticket.
243            	    if ($verbosemode) echo "GATEKEEPER look at done task $donetask"
244 phil   1.5             set lookroot = $donetask:s/-/./:e
245                        if ($lookroot == root) then
246                           echo found done task for $actiontask
247                           set root_done = 1
248                        else
249                           set root_done = 0
250            	    endif 
251 arta   1.6 foreach key (TICKET GATE TASKID) # KEY LOOP
252 phil   1.1 		set setval = `grep $key $donetask/ticket`
253 phil   1.5                 if ($#setval) then
254            		    set $setval
255                            else
256                                set $key = VOID
257            	        endif
258 arta   1.6 end # DONE KEY LOOP
259 phil   1.5             if ($TASKID == VOID) then
260            		set TASKID = $donetask
261            	    endif
262            	    # set parenttask = `echo $TASKID | sed -e 's/-.*//'`
263                        set parenttask = $TASKID:s/-/./:r
264                        if ($TICKET != VOID) rm -f ../logs/$TICKET/manager.pid
265 phil   1.1             if ($verbosemode) echo "GATEKEEPER parent task is $parenttask"
266            	    ex - $donetask/ticket <<!
267            /STATUS/d
268            w
269            q
270            !
271 phil   1.5 if ($debugmode) ls -l $donetask/ticket
272 phil   1.1 	    set taskstate = `cat $donetask/state`
273            	    if ($taskstate == 0) then
274                            echo "STATUS=0" >> $donetask/ticket
275            		mv $donetask ../archive/ok
276            	    else 
277                            echo "STATUS=5" >> $donetask/ticket
278            		mv $donetask ../archive/failed
279 phil   1.5 		echo " " >> $WFDIR/FAILED_TASKS
280            		echo -n "$actiontask FAILED for " >>$WFDIR/FAILED_TASKS
281            		echo -n `grep WANTLOW $WFDIR/tasks/$actiontask/archive/failed/$donetask/ticket` >> $WFDIR/FAILED_TASKS
282            		echo -n "  " >> $WFDIR/FAILED_TASKS
283            		echo -n   `grep WANTHIGH $WFDIR/tasks/$actiontask/archive/failed/$donetask/ticket` >> $WFDIR/FAILED_TASKS
284            		echo -n "  at " >> $WFDIR/FAILED_TASKS
285            		date >> $WFDIR/FAILED_TASKS
286            		echo "     See:" $WFDIR/tasks/$actiontask/archive/failed/$donetask >> $WFDIR/FAILED_TASKS
287 phil   1.1 	    endif
288 phil   1.5             if (-e ../logs/$TICKET) mv ../logs/$TICKET ../archive/logs
289 phil   1.1 if ($verbosemode) echo "GATEKEEPER done queue, move $GATE/active_tickets/$TICKET to /$parenttask/active/$TASKID/ticket_return"
290                        if (-e $WFDIR/gates/$GATE/active_tickets/$TICKET) then
291            	      mv $WFDIR/gates/$GATE/active_tickets/$TICKET $WFDIR/tasks/$parenttask/active/$TASKID/ticket_return
292                        endif
293                        if (-e $WFDIR/tasks/$parenttask/active/$TASKID/pending_tickets/$TICKET) then
294            	      rm $WFDIR/tasks/$parenttask/active/$TASKID/pending_tickets/$TICKET
295                        else if (-e $WFDIR/tasks/$parenttask/archive/ok/$TASKID/pending_tickets/$TICKET) then
296 phil   1.5               # rm $WFDIR/tasks/$parenttask/active/$TASKID/pending_tickets/$TICKET
297 phil   1.1             else
298            echo "GATEKEEPER done queue, FAILED to rm $WFDIR/tasks/$parenttask/active/$TASKID/pending_tickets/$TICKET"
299                        endif
300 arta   1.6         end # DONE TASK
301                        # processing this gate's action task done list for donetask
302 phil   1.1 	cd $thisgatedir
303            
304            	if ($verbosemode) echo "GATEKEEPER DONE with done queue, start at $gate active_tickets"
305                    # examine all existing tickets, examine status then do action
306 arta   1.6 foreach ticket (`/bin/ls active_tickets`) # TICKET LOOP
307 phil   1.1 		# get ticket attributes and check for timeout
308            echo $gate existing ticket  $ticket
309                            if ($verbosemode) echo "GATEKEEPER Start processing active ticket $ticket"
310 arta   1.6 foreach key (ACTION STATUS TASKID EXPIRES WANTLOW WANTHIGH) # KEY LOOP
311 phil   1.1 			set setval = `grep $key active_tickets/$ticket`
312            			if ($#setval == 1) set $setval
313 arta   1.6 end # KEY LOOP
314 phil   1.1 		set EXPIRES_t = `time_convert time=$EXPIRES`
315            # echo set EXPIRES_t = `time_convert time=$EXPIRES`
316                    	if ($type == "time") then
317            			set WANTLOW_t = `time_convert time=$WANTLOW`
318            # echo set WANTLOW_t = `time_convert time=$WANTLOW`
319            			set WANTHIGH_t = `time_convert time=$WANTHIGH`
320            # echo set WANTHIGH_t = `time_convert time=$WANTHIGH`
321            		else
322            			set WANTLOW_t = $WANTLOW
323            			set WANTHIGH_t = $WANTHIGH
324            		endif
325                            if ($verbosemode) echo "GATEKEEPER check for timeout"
326            
327                            # check expire time on working tickets
328            		if ( ($STATUS == 3 || $STATUS == 2) && $EXPIRES_t < $now) then
329                                    if ($verbosemode) echo GATEKEEPER TIMEOUT of ticket $ticket
330                                    ex - active_tickets/$ticket <<!
331            /STATUS/d
332            w
333            q
334            !
335 phil   1.1 			echo "STATUS=4" >> active_tickets/$ticket
336                                    set STATUS=4
337            			set task = `echo $TASKID | sed -e 's/-.*//'`
338                                    if (-e WFDIR/tasks/$task/active/$TASKID) then
339            			  mv active_tickets/$ticket $WFDIR/tasks/$task/active/$TASKID/ticket_return
340            			  rm $WFDIR/tasks/$task/active/$TASKID/pending_tickets/$ticket
341                                    else if (-e WFDIR/tasks/$task/archive/ok/$TASKID) then
342            			  mv active_tickets/$ticket $WFDIR/tasks/$task/archive/ok/$TASKID/ticket_return
343            			  rm $WFDIR/tasks/$task/archive/ok/$TASKID/pending_tickets/$ticket
344                                    else
345            echo "GATEKEEPER can not do mv active_tickets/$ticket $WFDIR/tasks/$task/active/$TASKID/ticket_return"
346                                    endif
347            		endif
348                            
349                            # if new slow ticket type, initiate if new, else check status and wait
350            		if ($STATUS == 2) then  # means getting coverage map or wait passively
351            			if ($verbosemode) echo "GATEKEEPER STATUS is 2"
352                                    if (-e statusbusy) then
353                                            if ($verbosemode) echo "GATEKEEPER StatusCommand is running"
354 arta   1.6 continue # NEXT TICKET
355 phil   1.1                         endif
356            			set low = `cat low`
357            if ($#low == 0) then
358            echo XXXX got null, sleeping
359            sleep 1
360            set low = `cat low`
361            endif
362            			set high = `cat high`
363            if ($#high == 0) then
364            echo XXXX got null, sleeping
365            sleep 1
366            set high = `cat high`
367            endif
368            			# if ($#low == 0 || $low == "NaN" || $#high == 0 || $high == "NaN") then
369            				# if ($verbosemode) echo "GATEKEEPER Gate $gate not properly initialized"
370            				# exit 1
371                                    # endif
372            			if ($type == "time") then
373 phil   1.3 # echo XXXXX
374            # cat active_tickets/$ticket
375            # echo XXXXX low=$low,
376            # echo XXXXX set low_t = `time_convert time=$low`
377 phil   1.1 				set low_t = `time_convert time=$low`
378            
379 phil   1.3 # echo xxxxx high=$high,
380            # echo xxxxx set high_t = `time_convert time=$high`
381 phil   1.1 				set high_t = `time_convert time=$high`
382            			else
383            				set low_t = $low
384            				set high_t = $high
385            			endif
386            			if ($ACTION == 3  && $WANTLOW_t >= $low_t && $WANTHIGH_t <= $high_t ) then
387            				if ( $verbosemode ) echo "GATEKEEPER ACTION = 3, waiting done"
388                                            ex - active_tickets/$ticket << !
389            /STATUS/d
390            w
391            q
392            !
393            				echo "GATELOW=$WANTLOW" >> active_tickets/$ticket
394            				echo "GATEHIGH=$WANTHIGH" >> active_tickets/$ticket
395            				echo "STATUS=0" >> active_tickets/$ticket
396            				set STATUS = 0
397            			else if ($ACTION == 6) then
398            				# ACTION == 6
399            				if ($verbosemode) echo "GATEKEEPER ACTION = 6"
400            				# XXXXX this logic will need to change when coverage map implemented
401            				if ($low != "NaN" && $high != "NaN") then # Coverage map must be complete now
402 phil   1.1                                         ex - active_tickets/$ticket <<!
403            /STATUS/d
404            w
405            q
406            !
407            					echo "GATELOW=$low" >> active_tickets/$ticket
408            					echo "GATEHIGH=$high" >> active_tickets/$ticket
409            					echo "STATUS=0" >> active_tickets/$ticket
410            					set STATUS = 0
411            				endif
412            			endif
413            			# Action was 3 or 6
414            		endif
415            		# STATUS == 2
416            
417                            # if ticket processing is complete, return ticket to sender
418            		set task = `echo $TASKID | sed -e 's/-.*//'`
419            		if ($STATUS != 3 && $STATUS != 2) then   # 2,3 means still owned by some task, else return ticket
420            			if ($verbosemode) echo "GATEKEEPER Ticket is complete, return to owner"
421            			# XXXXX need to update coverage here with want range in done ticket.  For now call get coverage.
422 phil   1.4 echo Start statustask
423 phil   1.1 			touch statusbusy
424 phil   1.2 			$WFCODE/$statustask $gate  # wait for this to complete.
425 phil   1.4 echo done statustask
426 phil   1.1                         # ticket should have been already removed from the target task's pending list
427                                    # but check anyway
428                                    if (-e $WFDIR/tasks/$task/active/$TASKID/ticket_return) then
429            if ($verbosemode) echo "GATEKEEPER active ticket exam, move active_tickets/$ticket to $task/active/$TASKID/ticket_return"
430            			  mv active_tickets/$ticket $WFDIR/tasks/$task/active/$TASKID/ticket_return
431            			  rm $WFDIR/tasks/$task/active/$TASKID/pending_tickets/$ticket
432                                    endif
433            			mv active_tickets/$ticket $WFDIR/tickets_done
434            		endif
435 arta   1.6 	end # TICKET LOOP
436                    # done with process existing tickets
437                end # GATE LOOP
438                    # done with tour of all gates
439 phil   1.1 
440                # gatekeeper loop management
441            ALL_GATES_DONE:
442                cd $WFDIR
443                rm -f GATEKEEPERBUSY
444                if (!(-e Keep_running)) then
445            	set keep_running = 0
446                else
447 phil   1.4         echo "XXXXXXXXXXXXXXXXX Start Sleeping XXXXXXXXXXXXXXXX"
448 phil   1.1         sleep $CADENCE
449                endif
450            
451 arta   1.6 end # KEEPRUNNING LOOP
452 phil   1.1 
453            echo Gate_Keeper Exit
454 phil   1.5 exit 0
455 phil   1.1 

Karen Tian
Powered by
ViewCVS 0.9.4