1 arta 1.29 #!/usr/bin/env python
|
2 arta 1.1
|
3 arta 1.13 # When run with the -s flag, localize.py configures the SUMS-server component of NetDRMS.
|
4 arta 1.1 import sys
5 import getopt
6 import re
|
7 arta 1.6 import os
8 import stat
|
9 arta 1.13 import filecmp
|
10 arta 1.6 import xml.etree.ElementTree as ET
|
11 arta 1.1 from subprocess import check_output, CalledProcessError
12
|
13 arta 1.24
|
14 arta 1.1 # Constants
15 VERS_FILE = 'jsoc_version.h'
16 SDP_CFG = 'configsdp.txt'
17 NET_CFG = 'config.local'
18 NET_CFGMAP = 'config.local.map'
19 RET_SUCCESS = 0
20 RET_NOTDRMS = 1
21
|
22 arta 1.2 PREFIX = """# This file was auto-generated by localize.py. Please do not edit it directly (running
23 # configure will run localize.py, which will then overwrite any edits manually performed).
24 """
|
25 arta 1.1
|
26 arta 1.2 C_PREFIX = """/* This file was auto-generated by localize.py. Please do not edit it directly (running
27 * configure will run localize.py, which will then overwrite any edits manually performed). */
|
28 arta 1.1 """
29
|
30 arta 1.2 PERL_BINPATH = '#!/usr/bin/perl\n'
31
32 PERL_INTIAL = """package drmsparams;
|
33 arta 1.28
|
34 arta 1.2 use warnings;
35 use strict;
36 """
37
38 PERL_FXNS_A = """sub new
|
39 arta 1.1 {
40 my($clname) = shift;
|
41 arta 1.28
42 my($self) =
|
43 arta 1.1 {
44 _paramsH => undef
45 };
|
46 arta 1.28
|
47 arta 1.1 bless($self, $clname);
|
48 arta 1.9 $self->{_paramsH} = {};
49 $self->initialize();
|
50 arta 1.28
|
51 arta 1.1 return $self;
52 }
|
53 arta 1.28
|
54 arta 1.1 sub DESTROY
55 {
56 my($self) = shift;
57 }
|
58 arta 1.2 """
59
60 PERL_FXNS_B = """sub get
61 {
62 my($self) = shift;
63 my($name) = shift;
64 my($rv);
|
65 arta 1.28
|
66 arta 1.2 if (exists($self->{_paramsH}->{$name}))
67 {
68 return $self->{_paramsH}->{$name};
69 }
70 else
71 {
72 return undef;
73 }
74 }
75 1;"""
|
76 arta 1.1
|
77 arta 1.29 PY_BINPATH = '#!/usr/bin/env python\n'
78
79 PY_ERROR_CLASSES = """
80 class DPError(Exception):
81 def __init__(self):
82 self._msg = 'DRMS Parameter Error: '
83
84 class DPMissingParameterError(DPError):
85 def __init__(self, parameter):
86 super().__init__()
87 self._msg += 'missing DRMS parameter ' + parameter
88
89 def __str__(self):
90 return self._msg
91 """
|
92 arta 1.18
93 PY_FXNS_A = """
|
94 arta 1.23 class DRMSParams(object):
|
95 arta 1.18 def __init__(self):
96 self.params = {}
97 self.initialize()
|
98 arta 1.28
|
99 arta 1.18 def __del__(self):
100 del self.params
|
101 arta 1.28
|
102 arta 1.18 def initialize(self):
103 """
104
105 PY_FXNS_B = """ def get(self, name):
106 if name in self.params:
|
107 arta 1.20 return self.params[name]
|
108 arta 1.18 else:
109 return None
110 """
111
|
112 arta 1.29 PY_FXNS_C = """ def get_required(self, name):
113 try:
114 value = self.params[name]
115 except:
116 raise DPMissingParameterError(name)
117
118 return value
119
120 def getBool(self, name):
|
121 arta 1.22 if name in self.params:
122 return bool(self.params[name] == '1')
123 else:
124 return None
|
125 arta 1.29
126 def __getattr__(self, name):
127 # only called if object.__getattribute__(self, name) raises; and if that is true, then we want
128 # to look in self.params for it, and set the instance attribute if it does exist in self.params
129 if name in self.params:
130 attr = self.params[name]
131 self.__setattr__(name, attr)
132 else:
133 attr = None
134
135 return attr
136
137 def __setattr__(self, name, value):
138 # call neither __setattr__ nor __getattr__
139 try:
140 params = object.__getattr__(self, 'params')
141
142 # put into self.params dict, overwriting if necessary
143 params[name] = value
144 except:
145 pass
146 arta 1.29
147 # store in instance dict as well
148 object.__setattr__(self, name, value)
|
149 arta 1.22 """
150
|
151 arta 1.25 SH_BINPATH = '#!/bin/bash\n'
152
|
153 arta 1.22
|
154 arta 1.13 SUMRM_COMMENT = """# This is the configuration file for the sum_rm program. It was auto-generated by the DRMS master configure script.
155 # It controls the behavior of the sum_rm program, and is loaded each time sum_rm runs. To change the
156 # parameter values in this configuration file, modify config.local, then re-run configure. This configuration
157 # file will be updated only if parameters affecting it are modified. If such changes are made to config.local,
158 # please make sure that the sum_rm service is disabled while configure in running.
159 """
160
|
161 arta 1.28 SUMRM_DOC = """# sum_rm removes end-of-life SUMS data files to prevent disk-partitions from becomming 100% full.
|
162 arta 1.13 # sum_svc, the main SUMS service, starts this daemon when it is launched. Within an infinite loop, sum_rm sleeps
163 # for SLEEP number of seconds (defined below). When it awakes, it loads the sum_rm configuration file. For each SU,
164 # it then examines the 'effective_date' column within the sum_partn_alloc table of the SUMS database. It does so by
165 # sorting all SUs by effective date (this date is actually an expiration date - the SU is good until the end of that day),
166 # and then, starting with the SU with the oldest effective date, it deletes the SUs. It continues deleting SUs until one
167 # of three conditions is met:
168 #
169 # (a) The disk has at least PART_PERCENT_FREE percent free space.
170 # (b) All SUs have effective_date values in the future.
171 # (c) At least 600 SUs have been deleted (this is defined in the LIMIT statement in the file SUMLIB_RmDoX.pgc).
172 #
173 # After sum_rm stops deleteing SUs, it then sleeps for SLEEP seconds, completing the first iteration of the
174 # infinite loop.
175 """
176
177 SUMRM_PARTN_PERCENT_FREE = """
178 # This is the percentage at which all disk partitions are to be kept free.
179 # If not specified, this defaults to 3. For example, setting PART_PERCENT_FREE = 5 will allow all partitions to
180 # fill to 95% full. Dividing the number of unused blocks by the total number of blocks, and rounding up,
181 # will result in the number specified by PART_PERCENT_FREE.
182 #
183 arta 1.13 # NOTE : This behavior was previously controlled by the MAX_FREE_{n} family of parameters. {n} referred to the
184 # disk-partition number and the value of parameter MAX_FREE_{n} was the MINIMUM number of free MB in the
185 # partition [No clue why the word MAX was used]. As of NetDRMS 7.0, MAX_FREE_{n} has no effect."""
186
187 SUMRM_SLEEP = """
188 # The value is the number of seconds to sleep between iterations of the main loop in sum_rm."""
189
190 SUMRM_LOG = """
191 # The value is the log file (opened only at sum_rm startup; the sum_rm pid is appended to this file name)."""
192
193 SUMRM_MAIL = """
194 # The value is the email address of the recipient to be notified in case of a problem."""
195
196 SUMRM_NOOP = """
197 # If the value is set to anything other than 0, then sum_rm is rendered inactive. Otherwise, sum_rm is active."""
198
199 SUMRM_USER = """
200 # The value designates the linux user who is allowed to run sum_rm."""
201
202 SUMRM_NORUN = """
203 # This pair of paramters defines a window of time, during which sum_rm will become torpid - in this
204 arta 1.13 # window, sum_rm will not scan for SUs to delete, not will it delete any SUs. Each value is an integer, 0-23, that
205 # represents an hour of the day. For example, if NORUN_START=8 and NORUN_STOP=10, then between 8am and 10am
206 # local time, sum_rm will be dormant. To disable this behavior, set both parameters to the same value."""
|
207 arta 1.6
208 RULESPREFIX = """# Standard things
209 sp := $(sp).x
210 dirstack_$(sp) := $(d)
211 d := $(dir)
212 """
213
214 RULESSUFFIX = """dir := $(d)/example
215 -include $(SRCDIR)/$(dir)/Rules.mk
216 dir := $(d)/cookbook
217 -include $(SRCDIR)/$(dir)/Rules.mk
218 dir := $(d)/myproj
219 -include $(SRCDIR)/$(dir)/Rules.mk
220
221 # Standard things
222 d := $(dirstack_$(sp))
223 sp := $(basename $(sp))
224 """
225
226 TARGETPREFIX = """$(PROJOBJDIR):\n\t+@[ -d $@ ] || mkdir -p $@"""
227
|
228 arta 1.3 ICC_MAJOR = 9
229 ICC_MINOR = 0
230 GCC_MAJOR = 3
231 GCC_MINOR = 0
232 IFORT_MAJOR = 9
233 IFORT_MINOR = 0
234 GFORT_MAJOR = 4
235 GFORT_MINOR = 2
236
237
|
238 arta 1.1 # Read arguments
239 # d - localization directory
240 # b - base name of all parameter files (e.g., -b drmsparams --> drmsparams.h, drmsparams.mk, drmsparams.pm, etc.)
|
241 arta 1.13 # s - configure a NetDRMS server (i.e., SUMS server). Otherwise, do client configuration only. A server
242 # configuration will modify something that only the production user has access to. An example is the sum_rm.cfg
243 # file. To modify that file, the user running configure must be running configure on the SUMS-server host, and
244 # must have write permission on sum_rm.cfg.
|
245 arta 1.1 def GetArgs(args):
246 rv = bool(0)
247 optD = {}
|
248 arta 1.28
|
249 arta 1.1 try:
|
250 arta 1.13 opts, remainder = getopt.getopt(args, "hd:b:s",["dir=", "base="])
|
251 arta 1.1 except getopt.GetoptError:
252 print('Usage:')
253 print('localize.py [-h] -d <localization directory> -b <parameter file base>')
254 rv = bool(1)
|
255 arta 1.28
|
256 arta 1.1 if rv == bool(0):
257 for opt, arg in opts:
258 if opt == '-h':
259 print('localize.py [-h] -d <localization directory> -b <parameter file base>')
260 elif opt in ("-d", "--dir"):
261 regexp = re.compile(r"(\S+)/?")
262 matchobj = regexp.match(arg)
263 if matchobj is None:
264 rv = bool(1)
265 else:
266 optD['dir'] = matchobj.group(1)
267 elif opt in ("-b", "--base"):
268 optD['base'] = arg
|
269 arta 1.13 elif opt in ("-s"):
270 optD['server'] = ""
|
271 arta 1.1 else:
272 optD[opt] = arg
|
273 arta 1.28
|
274 arta 1.1 return optD
275
276 def createMacroStr(key, val, keyColLen, status):
277 if keyColLen < len(key):
278 status = bool(1)
279 return None
280 else:
281 nsp = keyColLen - len(key)
282 spaces = str()
283 for isp in range(nsp):
284 spaces += ' '
285 status = bool(0)
286 return '#define ' + key + spaces + val + '\n'
287
288 def createPerlConst(key, val, keyColLen, status):
289 if keyColLen < len(key):
290 status = bool(1)
291 return None
292 else:
293 nsp = keyColLen - len(key)
294 spaces = str()
295 arta 1.1 for isp in range(nsp):
296 spaces += ' '
297 status = bool(0)
298 return 'use constant ' + key + ' => ' + spaces + val + ';\n'
299
|
300 arta 1.18 def createPyConst(key, val, keyColLen, status):
301 if keyColLen < len(key):
302 status = bool(1)
303 return None
304 else:
305 nsp = keyColLen - len(key)
306 spaces = str()
307 for isp in range(nsp):
308 spaces += ' '
309 status = bool(0)
310 return key + ' = ' + spaces + val + '\n'
|
311 arta 1.28
|
312 arta 1.25 def createShConst(key, val, status):
313 status = bool(0)
314 return key + '=' + val + '\n'
|
315 arta 1.28
|
316 arta 1.18
|
317 arta 1.3 def isSupportedPlat(plat):
318 regexp = re.compile(r"\s*(^x86_64|^ia32|^ia64|^avx)", re.IGNORECASE)
319 matchobj = regexp.match(plat);
|
320 arta 1.28
|
321 arta 1.3 if not matchobj is None:
322 return bool(1);
323 else:
324 return bool(0);
325
|
326 arta 1.6 def processMakeParam(mDefs, key, val, platDict, machDict):
|
327 arta 1.3 varMach = None
328 regexp = re.compile(r"(\S+):(\S+)")
329 matchobj = regexp.match(key)
330 if not matchobj is None:
331 varName = matchobj.group(1)
332 varMach = matchobj.group(2)
333 else:
334 varName = key
|
335 arta 1.28
|
336 arta 1.3 varValu = val
|
337 arta 1.28
|
338 arta 1.3 if varMach is None:
339 mDefs.extend(list('\n' + varName + ' = ' + varValu))
340 else:
341 if isSupportedPlat(varMach):
342 # The guard will compare varValu to $JSOC_MACHINE.
343 if not varMach in platDict:
344 platDict[varMach] = {}
345 platDict[varMach][varName] = varValu
346 else:
347 # The guard will compare varValu to $MACHINETYPE (this is just the hostname of the machine on which localize.py is running).
348 if not varMach in machDict:
349 machDict[varMach] = {}
350 machDict[varMach][varName] = varValu
351
|
352 arta 1.25 def processParam(cfgfile, line, regexpQuote, regexp, keymap, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection, platDict, machDict, section):
|
353 arta 1.1 status = 0
|
354 arta 1.28
|
355 arta 1.1 if ''.join(section) == 'defs' or not cfgfile:
356 matchobj = regexp.match(line)
357 if not matchobj is None:
358 # We have a key-value line
359 keyCfgSp = matchobj.group(1)
360 val = matchobj.group(2)
|
361 arta 1.28
|
362 arta 1.1 # Must map the indirect name to the actual name
363 if keymap:
364 # Map to actual name only if the keymap is not empty (which signifies NA).
365 if keyCfgSp in keymap:
366 key = keymap[keyCfgSp]
367 elif keyCfgSp == 'LOCAL_CONFIG_SET' or keyCfgSp == 'DRMS_SAMPLE_NAMESPACE':
|
368 arta 1.6 # Ignore parameters that are not useful and shouldn't have been there in the first place. But
369 # they have been released to the world, so we have to account for them.
|
370 arta 1.1 return bool(0)
371 elif not cfgfile:
372 # Should not be doing mapping for addenda
373 key = keyCfgSp
374 else:
375 raise Exception('badKeyMapKey', keyCfgSp)
376 else:
377 key = keyCfgSp
|
378 arta 1.28
|
379 arta 1.1 matchobj = regexpQuote.match(key)
380 if not matchobj is None:
381 quote = matchobj.group(1)
382 key = matchobj.group(2)
383
384 # master defs dictionary
385 defs[key] = val
|
386 arta 1.28
|
387 arta 1.1 # C header file
388 if quote == "q":
389 # Add double-quotes
390 cDefs.extend(list(createMacroStr(key, '"' + val + '"', 40, status)))
391 elif quote == "p":
392 # Add parentheses
393 cDefs.extend(list(createMacroStr(key, '(' + val + ')', 40, status)))
394 elif quote == "a":
395 # Leave as-is
396 cDefs.extend(list(createMacroStr(key, val, 40, status)))
397 else:
398 # Unknown quote type
399 raise Exception('badQuoteQual', key)
|
400 arta 1.28
|
401 arta 1.1 if status:
402 raise Exception('paramNameTooLong', key)
|
403 arta 1.28
|
404 arta 1.1 # Make file - val should never be quoted; just use as is
|
405 arta 1.4 mDefsGen.extend(list('\n' + key + ' = ' + val))
|
406 arta 1.28
|
407 arta 1.1 # Perl file - val should ALWAYS be single-quote quoted
408 # Save const info to a string
409 perlConstSection.extend(list(createPerlConst(key, "'" + val + "'", 40, status)))
|
410 arta 1.28
|
411 arta 1.18 # Python file
412 pyConstSection.extend(list(createPyConst(key, "'" + val + "'", 40, status)))
|
413 arta 1.28
|
414 arta 1.25 # Shell source file
415 shConstSection.extend(list(createShConst(key, "'" + val + "'", status)))
|
416 arta 1.28
|
417 arta 1.1 if status:
418 raise Exception('paramNameTooLong', key)
|
419 arta 1.28
|
420 arta 1.1 # Save initialization information as a string. Now that we've defined
|
421 arta 1.28 # constants (the names of which are the parameter names)
|
422 arta 1.1 # we can refer to those in the init section. The key variable holds the
423 # name of the constant.
|
424 arta 1.9 perlInitSection.extend(list("\n $self->{_paramsH}->{'" + key + "'} = " + key + ';'))
|
425 arta 1.28
|
426 arta 1.18 # The amount of indenting matters! This is Python.
427 pyInitSection.extend(list(" self.params['" + key + "'] = " + key + '\n'))
|
428 arta 1.1 else:
429 # No quote qualifier
430 raise Exception('missingQuoteQual', key)
|
431 arta 1.3 elif ''.join(section) == 'make' and cfgfile:
432 # Configure the remaining make variables defined in the __MAKE__ section of the configuration file. Third-party
433 # library make variables are specified in the __MAKE__ section.
434 matchobj = regexp.match(line)
435 if not matchobj is None:
436 # We have a key-value line
437 key = matchobj.group(1)
438 val = matchobj.group(2)
439
440 # This information is for making make variables only. We do not need to worry about quoting any values
441 defs[key] = val
|
442 arta 1.4 processMakeParam(mDefsMake, key, val, platDict, machDict)
|
443 arta 1.28
|
444 arta 1.1 return bool(0)
445
446 # We have some extraneous line or a newline - ignore.
447
|
448 arta 1.6 def processXML(xml, projRules, projTarget):
449 rv = bool(0)
|
450 arta 1.28
|
451 arta 1.6 # <projects>
452 root = ET.fromstring(xml)
|
453 arta 1.28
|
454 arta 1.6 # Iterate through each proj child.
455 for proj in root.iter('proj'):
456 # Rules.mk
457 nameElem = proj.find('name')
458 rulesStr = 'dir := $(d)/' + nameElem.text + '\n-include $(SRCDIR)/$(dir)/Rules.mk\n'
|
459 arta 1.28
460 # make doesn't support logical operations in ifeq conditionals (you can't do ifeq (A AND B)),
|
461 arta 1.6 # so we need to write:
462 # ifeq (A)
463 # ifeq (B)
464 # <do something>
465 # endif
466 # endif
|
467 arta 1.28
|
468 arta 1.6 rulesPref = '';
469 rulesSuff = '';
|
470 arta 1.28
|
471 arta 1.6 filters = proj.find('filters')
472 if filters is not None:
473 for filter in filters.findall('filter'):
474 rulesPref += 'ifeq ($(' + filter.find('name').text + '),' + filter.find('value').text + ')\n'
475 rulesSuff += 'endif\n'
|
476 arta 1.28
|
477 arta 1.6 if len(rulesPref) > 0 and len(rulesSuff) > 0:
478 projRules.extend(list(rulesPref))
479 projRules.extend(list(rulesStr))
480 projRules.extend(list(rulesSuff))
481 else:
482 projRules.extend(list(rulesStr))
|
483 arta 1.28
|
484 arta 1.6 # target.mk
485 subdirs = proj.find('subdirs')
486 if subdirs is not None:
487 for subdir in subdirs.findall('subdir'):
488 targetStr = '\n\t+@[ -d $@/' + nameElem.text + '/' + subdir.text + ' ] || mkdir -p $@/' + nameElem.text + '/' + subdir.text
489 projTarget.extend(list(targetStr))
490
491 return rv
492
|
493 arta 1.27 def determineSection(line, regexpStyle, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg):
494 matchobj = regexpStyle.match(line)
495 if matchobj:
496 return 'style'
497
|
498 arta 1.6 matchobj = regexpDefs.match(line)
499 if not matchobj is None:
500 return 'defs'
|
501 arta 1.28
|
502 arta 1.6 matchobj = regexpMake.match(line)
503 if not matchobj is None:
504 return 'make'
|
505 arta 1.28
|
506 arta 1.6 matchobj = regexpProjMkRules.match(line)
507 if not matchobj is None:
508 return 'projmkrules'
|
509 arta 1.28
|
510 arta 1.6 matchobj = regexpProj.match(line)
511 if not matchobj is None:
512 return 'proj'
|
513 arta 1.28
|
514 arta 1.6 matchobj = regexpProjCfg.match(line)
515 if not matchobj is None:
516 return 'projcfg'
517
518 return None
519
|
520 arta 1.1 # defs is a dictionary containing all parameters (should they be needed in this script)
|
521 arta 1.6 # projCfg is the list containing the configure script content.
522 # projMkRules is the list containing the make_basic.mk content.
523 # projRules is the list containing the Rules.mk content.
524 # projTargert is the list containing the target.mk content.
|
525 arta 1.25 def parseConfig(fin, keymap, addenda, defs, cDefs, mDefsGen, mDefsMake, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection):
|
526 arta 1.1 rv = bool(0)
|
527 arta 1.28
|
528 arta 1.1 # Open required config file (config.local)
529 try:
530 # Examine each line, looking for key=value pairs.
|
531 arta 1.27 regexpStyle = re.compile(r"^__STYLE__")
|
532 arta 1.1 regexpDefs = re.compile(r"^__DEFS__")
533 regexpMake = re.compile(r"^__MAKE__")
|
534 arta 1.6 regexpProjMkRules = re.compile(r"__PROJ_MK_RULES__")
535 regexpProj = re.compile(r"^__PROJ__")
536 regexpProjCfg = re.compile(r"^__PROJCFG__")
|
537 arta 1.1 regexpComm = re.compile(r"^\s*#")
|
538 arta 1.6 regexpSp = re.compile(r"^s*$")
|
539 arta 1.1 regexpQuote = re.compile(r"^\s*(\w):(.+)")
|
540 arta 1.6 regexpCustMkBeg = re.compile(r"^_CUST_")
541 regexpCustMkEnd = re.compile(r"^_ENDCUST_")
542 regexpDiv = re.compile(r"^__")
|
543 arta 1.11 regexp = re.compile(r"^\s*(\S+)\s+(\S.*)")
|
544 arta 1.27
545 ignoreKeymap = False
|
546 arta 1.3 platDict = {}
547 machDict = {}
|
548 arta 1.28
|
549 arta 1.6 xml = None
|
550 arta 1.28
|
551 arta 1.1 # Process the parameters in the configuration file
552 if not fin is None:
553 for line in fin:
|
554 arta 1.6 matchobj = regexpComm.match(line)
555 if not matchobj is None:
556 # Skip comment line
557 continue
|
558 arta 1.28
|
559 arta 1.6 matchobj = regexpSp.match(line)
560 if not matchobj is None:
561 # Skip whitespace line
562 continue
|
563 arta 1.28
|
564 arta 1.27 newSection = determineSection(line, regexpStyle, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg)
|
565 arta 1.6 if not newSection is None:
566 section = newSection
|
567 arta 1.28
|
568 arta 1.27 if not section:
569 raise Exception('invalidConfigFile', 'line ' + line.strip() + ' is not in any section')
|
570 arta 1.28
|
571 arta 1.27 if section == 'style':
572 # if the config.local file has new in the __STYLE__ section, then ignore the keymap and treat config.local like configsdp.txt;
573 # do not map from NetDRMS config.local parameter names to configsdp.txt names
574 for line in fin:
575 matchobj = regexpDiv.match(line)
576 if matchobj:
577 break;
578 if line.strip(' \n').lower() == 'new' and keymap:
579 ignoreKeymap = True
580
581 newSection = determineSection(line, regexpStyle, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg)
582 if not newSection is None:
583 section = newSection
584 continue
585 elif section == 'make':
|
586 arta 1.10
|
587 arta 1.28 # There are some blocks of lines in the __MAKE__ section that must be copied ver batim to the output make file.
|
588 arta 1.6 # The blocks are defined by _CUST_/_ENDCUST_ tags.
589 matchobj = regexpCustMkBeg.match(line)
|
590 arta 1.28
|
591 arta 1.6 if not matchobj is None:
592 mDefsMake.extend(list('\n'))
593 for line in fin:
594 matchobj = regexpCustMkEnd.match(line)
595 if not matchobj is None:
596 break;
597 mDefsMake.extend(list(line))
|
598 arta 1.27 newSection = determineSection(line, regexpStyle, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg)
|
599 arta 1.6 if not newSection is None:
600 section = newSection
601 continue
602 # Intentional fall through to next if statement
603 if section == 'defs' or section == 'make':
604 iscfg = bool(1)
|
605 arta 1.27 if ignoreKeymap:
606 keymapActual = None
607 else:
608 keymapActual = keymap
609 ppRet = processParam(iscfg, line, regexpQuote, regexp, keymapActual, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection, platDict, machDict, section)
|
610 arta 1.28
|
611 arta 1.6 if ppRet:
|
612 arta 1.10 break
|
613 arta 1.6 elif section == 'projcfg':
614 # Copy the line ver batim to the projCfg list (configure)
615 for line in fin:
616 matchobj = regexpDiv.match(line)
617 if not matchobj is None:
618 break;
619 projCfg.extend(list(line))
|
620 arta 1.27 newSection = determineSection(line, regexpStyle, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg)
|
621 arta 1.6 if not newSection is None:
622 section = newSection
623 continue
624 elif section == 'projmkrules':
625 # Copy the line ver batim to the projMkRules list (make_basic.mk)
626 for line in fin:
627 matchobj = regexpDiv.match(line)
628 if not matchobj is None:
629 break;
630 projMkRules.extend(list(line))
|
631 arta 1.27 newSection = determineSection(line, regexpStyle, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg)
|
632 arta 1.6 if not newSection is None:
633 section = newSection
634 continue
635 elif section == 'proj':
636 # Must parse xml and use the project-specific information to populate the Rules.mk and target.mk files.
637 # Collect all xml lines for now, then process after file-read loop.
638 if xml is None:
|
639 arta 1.21 # The first time through this section, line is the config.local div, __PROJ__. Discard that.
640 xml = ''
641 continue
|
642 arta 1.6 else:
643 xml += line
644 else:
|
645 arta 1.10 # Unknown section
|
646 arta 1.6 raise Exception('unknownSection', section)
|
647 arta 1.1 except Exception as exc:
|
648 arta 1.17 if len(exc.args) >= 2:
649 msg = exc.args[0]
650 else:
651 # re-raise the exception
652 raise
|
653 arta 1.28
|
654 arta 1.27 if msg == 'invalidConfigFile':
655 violator = exc.args[1]
656 print(violator, file=sys.stderr)
657 rv = bool(1)
658 elif msg == 'badKeyMapKey':
|
659 arta 1.1 # If we are here, then there was a non-empty keymap, and the parameter came from
660 # the configuration file.
|
661 arta 1.6 violator = exc.args[1]
|
662 arta 1.17 print('Unknown parameter name ' + "'" + violator + "'" + ' in ' + fin.name + '.', file=sys.stderr)
|
663 arta 1.1 rv = bool(1)
664 elif msg == 'badQuoteQual':
665 # The bad quote qualifier came from the configuration file, not the addenda, since
666 # we will have fixed any bad qualifiers in the addenda (which is populated by code).
|
667 arta 1.6 violator = exc.args[1]
|
668 arta 1.17 print('Unknown quote qualifier ' + "'" + violator + "'" + ' in ' + fin.name + '.', file=sys.stderr)
|
669 arta 1.1 rv = bool(1)
670 elif msg == 'missingQuoteQual':
|
671 arta 1.6 violator = exc.args[1]
|
672 arta 1.17 print('Missing quote qualifier for parameter ' + "'" + violator + "'" + ' in ' + fin.name + '.', file=sys.stderr)
|
673 arta 1.1 rv = bool(1)
674 elif msg == 'paramNameTooLong':
|
675 arta 1.6 violator = exc.args[1]
|
676 arta 1.1 print('Macro name ' + "'" + violator + "' is too long.", file=sys.stderr)
677 rv = bool(1)
|
678 arta 1.6 elif msg == 'unknownSection':
679 violator = exc.args[1]
680 print('Unknown section ' + "'" + violator + "' in configuration file.", file=sys.stderr)
681 rv = bool(1)
|
682 arta 1.1 else:
683 # re-raise the exception
684 raise
685
|
686 arta 1.6 if not rv:
687 if not xml is None:
688 # Process xml.
689 rv = processXML(xml, projRules, projTarget)
690
|
691 arta 1.28 # Process addenda - these are parameters that are not configurable and must be set in the
|
692 arta 1.6 # NetDRMS build.
693 if not rv:
694 iscfg = bool(0)
695 for key in addenda:
696 item = key + ' ' + addenda[key]
|
697 arta 1.27 if ignoreKeymap:
698 keymapActual = None
699 else:
700 keymapActual = keymap
701 ppRet = processParam(iscfg, item, regexpQuote, regexp, keymapActual, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection, platDict, machDict, 'defs')
|
702 arta 1.6 if ppRet:
703 break;
704
|
705 arta 1.4 # Put information collected in platDict and machDict into mDefs. Must do this here, and not in processParam, since
706 # we need to parse all platform-specific make variables before grouping them into platform categories.
|
707 arta 1.3 if not rv:
708 for plat in platDict:
|
709 arta 1.4 mDefsMake.extend(list('\nifeq ($(JSOC_MACHINE), linux_' + plat.lower() + ')'))
|
710 arta 1.3 for var in platDict[plat]:
|
711 arta 1.4 mDefsMake.extend(list('\n' + var + ' = ' + platDict[plat][var]))
712 mDefsMake.extend(list('\nendif\n'))
|
713 arta 1.28
|
714 arta 1.3 if not rv:
715 for mach in machDict:
|
716 arta 1.4 mDefsMake.extend(list('\nifeq ($(MACHTYPE), ' + mach + ')'))
|
717 arta 1.7 for var in machDict[mach]:
718 mDefsMake.extend(list('\n' + var + ' = ' + machDict[mach][var]))
|
719 arta 1.4 mDefsMake.extend(list('\nendif\n'))
|
720 arta 1.1 return rv
721
722 def getMgrUIDLine(defs, uidParam):
723 rv = bool(0)
|
724 arta 1.28
|
725 arta 1.1 cmd = 'id -u ' + defs['SUMS_MANAGER']
726 try:
727 ret = check_output(cmd, shell=True)
728 uidParam['q:SUMS_MANAGER_UID'] = ret.decode("utf-8")
729 except ValueError:
730 print('Unable to run cmd: ' + cmd + '.')
731 rv = bool(1)
732 except CalledProcessError:
733 print('Command ' + "'" + cmd + "'" + ' ran improperly.')
734 rv = bool(1)
735
736 return rv
737
|
738 arta 1.3 def isVersion(maj, min, majDef, minDef):
739 res = 0
|
740 arta 1.28
|
741 arta 1.3 if maj > majDef or (maj == majDef and min >= minDef):
742 res = 1
|
743 arta 1.28
|
744 arta 1.3 return res
745
746 def configureComps(defs, mDefs):
747 rv = bool(0)
|
748 arta 1.6 autoConfig = bool(1)
749
750 if 'AUTOSELCOMP' in defs:
751 autoConfig = (not defs['AUTOSELCOMP'] == '0')
|
752 arta 1.28
|
753 arta 1.3 if autoConfig:
754 hasicc = bool(0)
755 hasgcc = bool(0)
756 hasifort = bool(0)
757 hasgfort = bool(0)
|
758 arta 1.28
|
759 arta 1.3 # Try icc.
|
760 arta 1.12 cmd = 'icc --version 2>&1'
|
761 arta 1.3 try:
762 ret = check_output(cmd, shell=True)
763 ret = ret.decode("utf-8")
764 except CalledProcessError:
765 print('Command ' + "'" + cmd + "'" + ' ran improperly.')
766 rv = bool(1)
|
767 arta 1.28
|
768 arta 1.6 if not rv:
|
769 arta 1.12 regexp = re.compile(r"\s*\S+\s+\S+\s+(\d+)[.](\d+)", re.DOTALL)
|
770 arta 1.3 matchobj = regexp.match(ret)
771 if matchobj is None:
772 raise Exception('unexpectedIccRet', ret)
773 else:
774 major = matchobj.group(1)
775 minor = matchobj.group(2)
776 if isVersion(int(major), int(minor), ICC_MAJOR, ICC_MINOR):
777 hasicc = bool(1)
|
778 arta 1.19
|
779 arta 1.3 # Try gcc.
780 if not hasicc:
|
781 arta 1.19 rv = bool(0)
|
782 arta 1.3 cmd = 'gcc -v 2>&1'
783 try:
784 ret = check_output(cmd, shell=True)
785 ret = ret.decode("utf-8")
786 except CalledProcessError:
787 print('Command ' + "'" + cmd + "'" + ' ran improperly.')
788 rv = bool(1)
789
|
790 arta 1.19 if not rv:
|
791 arta 1.3 regexp = re.compile(r".+gcc\s+version\s+(\d+)\.(\d+)", re.DOTALL)
792 matchobj = regexp.match(ret)
793 if matchobj is None:
794 raise Exception('unexpectedGccRet', ret)
795 else:
796 major = matchobj.group(1)
797 minor = matchobj.group(2)
798 if isVersion(int(major), int(minor), GCC_MAJOR, GCC_MINOR):
799 hasgcc = bool(1)
800
801 # Try ifort.
|
802 arta 1.19 rv = bool(0)
|
803 arta 1.12 cmd = 'ifort --version 2>&1'
|
804 arta 1.3 try:
805 ret = check_output(cmd, shell=True)
806 ret = ret.decode("utf-8")
807 except CalledProcessError:
808 print('Command ' + "'" + cmd + "'" + ' ran improperly.')
809 rv = bool(1)
810
|
811 arta 1.6 if not rv:
|
812 arta 1.12 regexp = re.compile(r"\s*\S+\s+\S+\s+(\d+)\.(\d+)", re.DOTALL)
|
813 arta 1.3 matchobj = regexp.match(ret)
814 if matchobj is None:
815 raise Exception('unexpectedIfortRet', ret)
816 else:
817 major = matchobj.group(1)
818 minor = matchobj.group(2)
819 if isVersion(int(major), int(minor), IFORT_MAJOR, IFORT_MINOR):
820 hasifort = bool(1)
|
821 arta 1.28
|
822 arta 1.3 # Try gfortran
823 if not hasifort:
|
824 arta 1.19 rv = bool(0)
|
825 arta 1.3 cmd = 'gfortran -v 2>&1'
826 try:
827 ret = check_output(cmd, shell=True)
828 ret = ret.decode("utf-8")
829 except CalledProcessError:
830 print('Command ' + "'" + cmd + "'" + ' ran improperly.')
831 rv = bool(1)
832
|
833 arta 1.19 if not rv:
|
834 arta 1.3 regexp = re.compile(r".+gcc\s+version\s+(\d+)\.(\d+)", re.DOTALL)
835 matchobj = regexp.match(ret)
836 if matchobj is None:
837 raise Exception('unexpectedGfortranRet', ret)
838 else:
839 major = matchobj.group(1)
840 minor = matchobj.group(2)
841 if isVersion(int(major), int(minor), GFORT_MAJOR, GFORT_MINOR):
842 hasgfort = bool(1)
|
843 arta 1.28
|
844 arta 1.19 # Append the compiler make variables to the make file
845 rv = bool(0)
|
846 arta 1.28
|
847 arta 1.3 if not hasicc and not hasgcc:
848 print('Fatal error: Acceptable C compiler not found! You will be unable to build the DRMS library.', file=sys.stderr)
849 rv = bool(1)
850 elif hasicc:
851 mDefs.extend(list('\nCOMPILER = icc'))
|
852 arta 1.28 # mDefs.extend(list('\nICC_VERSION = blah'))
|
853 arta 1.3 else:
854 mDefs.extend(list('\nCOMPILER = gcc'))
855
856 if not hasifort and not hasgfort:
857 print('Warning: Acceptable Fortran compiler not found! Fortran interface will not be built, and you will be unable to build Fortran modules.', file=sys.stderr)
858 elif hasifort:
859 mDefs.extend(list('\nFCOMPILER = ifort'))
860 else:
861 mDefs.extend(list('\nFCOMPILER = gfortran'))
|
862 arta 1.28
|
863 arta 1.3 # Environment overrides. These get written, regardless of the disposition of auto-configuration.
|
864 arta 1.9 mDefs.extend(list('\nifneq ($(JSOC_COMPILER),)\n COMPILER = $(JSOC_COMPILER)\nendif'))
865 mDefs.extend(list('\nifneq ($(JSOC_FCOMPILER),)\n FCOMPILER = $(JSOC_FCOMPILER)\nendif'))
|
866 arta 1.3
867 return rv
868
|
869 arta 1.25 def writeParamsFiles(base, cfile, mfile, pfile, pyfile, shfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection):
|
870 arta 1.1 rv = bool(0)
|
871 arta 1.4
872 # Merge mDefsGen, mDefsMake, and mDefsComps into a single string with compiler configuration first, general parameters next, then
873 # make-specific make variables (e.g., third-party library information) last.
874 mDefs = '\n# Compiler Selection\n' + ''.join(mDefsComps) + '\n\n# General Parameters\n' + ''.join(mDefsGen) + '\n\n# Parameters to Configure make\n' + ''.join(mDefsMake)
|
875 arta 1.28
|
876 arta 1.1 try:
|
877 arta 1.25 with open(cfile, 'w') as cout, open(mfile, 'w') as mout, open(pfile, 'w') as pout, open(pyfile, 'w') as pyout, open(shfile, 'w') as shout:
|
878 arta 1.1 # C file of macros
|
879 arta 1.2 print(C_PREFIX, file=cout)
880 print('/* This file contains a set of preprocessor macros - one for each configuration parameter. */\n', file=cout)
|
881 arta 1.1 buf = '__' + base.upper() + '_H'
882 print('#ifndef ' + buf, file=cout)
883 print('#define ' + buf, file=cout)
884 print(''.join(cDefs), file=cout)
885 print('#endif', file=cout)
|
886 arta 1.28
|
887 arta 1.1 # Make file of make variables
|
888 arta 1.2 print(PREFIX, file=mout)
|
889 arta 1.4 print('# This file contains a set of make-variable values. The first section contains compiler-selection variables, the second contains general configuration variables, and the third section contains variables that configure how make is run.', file=mout)
890 print(mDefs, file=mout)
|
891 arta 1.28
|
892 arta 1.2 # Perl module
893 print(PERL_BINPATH, file=pout)
894 print(PREFIX, file=pout)
895 print('# This file contains a set of constants - one for each configuration parameter.\n', file=pout)
896 print(PERL_INTIAL, file=pout)
|
897 arta 1.1 print(''.join(perlConstSection), file=pout)
|
898 arta 1.2 print(PERL_FXNS_A, file=pout)
|
899 arta 1.1 print('sub initialize', file=pout)
900 print('{', file=pout)
|
901 arta 1.2 print(' my($self) = shift;', file=pout, end='')
|
902 arta 1.1 print('', file=pout)
903 print(''.join(perlInitSection), file=pout)
|
904 arta 1.2 print('}\n', file=pout)
905 print(PERL_FXNS_B, file=pout)
|
906 arta 1.18
907 # Python module
908 print(PY_BINPATH, file=pyout)
909 print(PREFIX, file=pyout)
910 print('# This file contains a set of constants - one for each configuration parameter.\n', file=pyout)
911 print(''.join(pyConstSection), file=pyout)
|
912 arta 1.29
913 print(PY_ERROR_CLASSES, file=pyout)
|
914 arta 1.18 print(PY_FXNS_A, file=pyout, end='')
915 print(''.join(pyInitSection), file=pyout)
916 print(PY_FXNS_B, file=pyout)
|
917 arta 1.22 print(PY_FXNS_C, file=pyout)
|
918 arta 1.28
|
919 arta 1.25 # Shell (bash) source file
920 print(SH_BINPATH, file=shout)
921 print(PREFIX, file=shout)
922 print('# This file contains a set of variable assignments - one for each configuration parameter.\n', file=shout)
923 print(''.join(shConstSection), file=shout)
|
924 arta 1.18
|
925 arta 1.1 except IOError as exc:
|
926 arta 1.6 type, value, traceback = sys.exc_info()
927 print(exc.strerror, file=sys.stderr)
|
928 arta 1.28 print('Unable to open ' + "'" + value.filename + "'.", file=sys.stderr)
|
929 arta 1.6 rv = bool(1)
930
931 return rv
932
933 def writeProjFiles(pCfile, pMfile, pRfile, pTfile, projCfg, projMkRules, projRules, projTarget):
934 rv = bool(0)
935
936 try:
937 if projCfg:
938 with open(pCfile, 'w') as cout:
939 # configure
940 print(''.join(projCfg), file=cout)
|
941 arta 1.28
|
942 arta 1.6 if projMkRules:
943 with open(pMfile, 'w') as mout:
944 # make_basic.mk
945 print(PREFIX, file=mout)
946 print(''.join(projMkRules), file=mout)
|
947 arta 1.28
|
948 arta 1.6 if projRules:
949 with open(pRfile, 'w') as rout:
950 # Rules.mk
951 print(PREFIX, file=rout)
952 print(''.join(projRules), file=rout)
|
953 arta 1.28
|
954 arta 1.6 if projTarget:
955 with open(pTfile, 'w') as tout:
956 # target.mk
957 print(PREFIX, file=tout)
|
958 arta 1.10 print(''.join(projTarget), file=tout)
|
959 arta 1.6 except IOError as exc:
960 type, value, traceback = sys.exc_info()
961 print(exc.strerror, file=sys.stderr)
962 print('Unable to open ' + "'" + value.filename + "'.", file=sys.stderr)
|
963 arta 1.1 rv = bool(1)
964
|
965 arta 1.6 if not rv:
|
966 arta 1.10 if os.path.exists(pCfile):
967 try:
968 os.chmod(pCfile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IROTH)
969 except OSError as exc:
970 type, value, traceback = sys.exc_info()
971 print('Unable to chmod file ' + "'" + value.filename + "'.", file=sys.stderr)
972 print(exc.strerror, file=sys.stderr)
973 rv = bool(1)
|
974 arta 1.6
|
975 arta 1.1 return rv
|
976 arta 1.3
|
977 arta 1.13 def generateSumRmCfg(defs):
978 rv = bool(0)
979 # ACK! Remember that Rick renamed these parameters. The ones in config.local are the aliases - do not use those.
980 # Use the ones that those map to (defined in config.local.map).
981 cFileTmp = defs['SUMLOG_BASEDIR'] + '/' + '.sum_rm.cfg.tmp'
982 cFile = defs['SUMLOG_BASEDIR'] + '/' + 'sum_rm.cfg'
983
984 # Write a temporary file sum_rm configuration file.
985 try:
986 with open(cFileTmp, 'w') as fout:
987 # Print comment at the top of the configuration file.
988 print(SUMRM_COMMENT, file=fout)
989 print(SUMRM_DOC, file=fout)
990 print(SUMRM_PARTN_PERCENT_FREE, file=fout)
991 if 'SUMRM_PART_PERCENT_FREE' in defs:
992 print('PART_PERCENT_FREE=' + defs['SUMRM_PART_PERCENT_FREE'], file=fout)
993 else:
994 print('PART_PERCENT_FREE=3', file=fout)
995
996 print(SUMRM_SLEEP, file=fout)
997 if 'SUMRM_SLEEP' in defs:
998 arta 1.13 print('SLEEP=' + defs['SUMRM_SLEEP'], file=fout)
999 else:
1000 print('SLEEP=300', file=fout)
1001
1002 print(SUMRM_LOG, file=fout)
1003 if 'SUMRM_LOG' in defs:
1004 print('LOG=' + defs['SUMRM_LOG'], file=fout)
1005 else:
1006 print('LOG=/tmp/sum_rm.log', file=fout)
1007
1008 print(SUMRM_MAIL, file=fout)
1009 # No default for mail - don't send nothing to nobody unless the operator has asked for notifications.
1010 if 'SUMRM_MAIL' in defs:
1011 print('MAIL=' + defs['SUMRM_MAIL'], file=fout)
1012 else:
1013 print('# MAIL=president@whitehouse.gov', file=fout)
1014
1015 print(SUMRM_NOOP, file=fout)
1016 if 'SUMRM_NOOP' in defs:
1017 print('NOOP=' + defs['SUMRM_NOOP'], file=fout)
1018 else:
1019 arta 1.13 print('NOOP=0', file=fout)
1020
1021 print(SUMRM_USER, file=fout)
1022 if 'SUMRM_USER' in defs:
1023 print('USER=' + defs['SUMRM_USER'], file=fout)
1024 else:
1025 print('USER=production', file=fout)
1026
1027 print(SUMRM_NORUN, file=fout)
1028 # Default norun window is to have no such window. This can be accomplished by simply not providing either argument.
1029 if 'SUMRM_NORUN_START' in defs or 'SUMRM_NORUN_STOP' in defs:
1030 if 'SUMRM_NORUN_START' in defs:
1031 print('NORUN_START=' + defs['SUMRM_NORUN_START'], file=fout)
1032 else:
1033 print('NORUN_START=0', file=fout)
1034 if 'SUMRM_NORUN_STOP' in defs:
1035 print('NORUN_STOP=' + defs['SUMRM_NORUN_STOP'], file=fout)
1036 else:
1037 print('NORUN_STOP=0', file=fout)
1038 else:
1039 print('# NORUN_START=0', file=fout)
1040 arta 1.13 print('# NORUN_STOP=0', file=fout)
|
1041 arta 1.28
|
1042 arta 1.13 except OSError:
1043 print('Unable to open sum_rm temporary configuration file ' + cFileTmp + 'for writing.', file=sys.stderr)
1044 rv = bool(1)
1045
1046 # If the content of the temporary file differs from the content of the existing configuration file, then overwrite
1047 # the original file. Otherwise, delete the temporary file
1048 if not rv:
1049 try:
1050 if filecmp.cmp(cFile, cFileTmp):
1051 # Files identical - delete temporary file
1052 try:
1053 os.remove(cFileTmp)
|
1054 arta 1.28
|
1055 arta 1.13 except OSError as exc:
1056 print('Unable to remove temporary file ' + exc.filename + '.', file=sys.stderr)
1057 print(exc.strerr, file=sys.stderr)
1058 else:
1059 # Replace original with temporary file
1060 try:
1061 os.rename(cFileTmp, cFile)
1062
1063 except OSError as exc:
1064 print('Unable to update sum_rm configuration file ' + cFile + '.', file=sys.stderr)
1065 print(exc.strerr, file=sys.stderr)
1066 rv = bool(1)
1067 except OSError as exc:
1068 # One of the files doesn't exist.
1069 if exc.filename == cFile:
1070 # We are ok - there might be no configuration file yet.
1071 # Replace original with temporary file
1072 try:
1073 os.rename(cFileTmp, cFile)
1074
1075 except OSError as exc:
1076 arta 1.13 print('Unable to update sum_rm configuration file ' + cFile + '.', file=sys.stderr)
1077 print(exc.strerr, file=sys.stderr)
1078 rv = bool(1)
1079 else:
1080 # There is a problem with the temp file - bail.
1081 print('Unable to update sum_rm configuration file ' + cFile + '.', file=sys.stderr)
1082 print(exc.strerr, file=sys.stderr)
1083 rv = bool(1)
1084
1085 return rv
1086
|
1087 arta 1.25 def configureNet(cfgfile, cfile, mfile, pfile, pyfile, shfile, pCfile, pMfile, pRfile, pTfile, base, keymap, createSumRmCfg):
|
1088 arta 1.1 rv = bool(0)
|
1089 arta 1.28
|
1090 arta 1.1 defs = {}
1091 cDefs = list()
|
1092 arta 1.4 mDefsGen = list()
1093 mDefsMake = list()
1094 mDefsComps = list()
|
1095 arta 1.6 projCfg = list()
1096 projMkRules = list()
1097 projRules = list()
1098 projTarget = list()
|
1099 arta 1.1 perlConstSection = list()
1100 perlInitSection = list()
|
1101 arta 1.18 pyConstSection = list()
1102 pyInitSection = list()
|
1103 arta 1.25 shConstSection = list()
|
1104 arta 1.2 addenda = {}
|
1105 arta 1.28
|
1106 arta 1.6 # There are three parameters that were not included in the original config.local parameter set, for some reason.
1107 # Due to this omission, then are not configurable, and must be set in the script.
|
1108 arta 1.2 addenda['a:USER'] = 'NULL'
1109 addenda['a:PASSWD'] = 'NULL'
1110 addenda['p:DSDS_SUPPORT'] = '0'
|
1111 arta 1.28
|
1112 arta 1.6 # This parameter is not configurable. BUILD_TYPE is used to distinguish between a NetDRMS and an JSOC-SDP build.
|
1113 arta 1.2 addenda['a:BUILD_TYPE'] = 'NETDRMS' # Means a non-Stanford build. This will set two additional macros used by make:
1114 # __LOCALIZED_DEFS__ and NETDRMS_BUILD. The former is to support legacy code
|
1115 arta 1.28 # which incorrectly used this macro, and the latter is for future use.
|
1116 arta 1.2 # __LOCALIZED_DEFS__ is deprecated and should not be used in new code.
|
1117 arta 1.1
1118 try:
1119 with open(cfgfile, 'r') as fin:
|
1120 arta 1.3 # Process configuration parameters
|
1121 arta 1.26
1122 # Always create a Rules.mk and target.mk, even if no proj XML is provided. All builds should have the proj/example and
1123 # proj/cookbook directories. The required make information is in RULESPREFIX, TARGETPREFIX, and RULESSUFFIX. RULESSUFFIX
1124 # must be added after the xml has been parsed.
1125 projRules.extend(list(RULESPREFIX))
1126 projTarget.extend(list(TARGETPREFIX))
1127
|
1128 arta 1.25 rv = parseConfig(fin, keymap, addenda, defs, cDefs, mDefsGen, mDefsMake, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
|
1129 arta 1.6 if not rv:
|
1130 arta 1.26 projRules.extend(RULESSUFFIX)
|
1131 arta 1.28
|
1132 arta 1.3 # Must add a parameter for the SUMS_MANAGER UID (for some reason). This must be done after the
1133 # config file is processed since an input to getMgrUIDLine() is one of the config file's
1134 # parameter values.
|
1135 arta 1.1 uidParam = {}
1136 rv = getMgrUIDLine(defs, uidParam)
1137 if rv == bool(0):
|
1138 arta 1.25 rv = parseConfig(None, keymap, uidParam, defs, cDefs, mDefsGen, None, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
|
1139 arta 1.3 # Configure the compiler-selection make variables.
|
1140 arta 1.6 if not rv:
|
1141 arta 1.4 rv = configureComps(defs, mDefsComps)
|
1142 arta 1.1
|
1143 arta 1.3 # Write out the parameter files.
|
1144 arta 1.6 if not rv:
|
1145 arta 1.25 rv = writeParamsFiles(base, cfile, mfile, pfile, pyfile, shfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
|
1146 arta 1.13
|
1147 arta 1.6 # Write out the project-specific make files (make_basic.mk, Rules.mk, and target.mk).
1148 if not rv:
1149 rv = writeProjFiles(pCfile, pMfile, pRfile, pTfile, projCfg, projMkRules, projRules, projTarget)
|
1150 arta 1.13
1151 # Write out the sum_rm.cfg file.
1152 if not rv and createSumRmCfg:
1153 rv = generateSumRmCfg(defs)
|
1154 arta 1.1 except IOError as exc:
|
1155 arta 1.6 print(exc.strerror, file=sys.stderr)
1156 print('Unable to read configuration file ' + cfgfile + '.', file=sys.stderr)
|
1157 arta 1.3 except Exception as exc:
|
1158 arta 1.16 if len(exc.args) >= 2:
|
1159 arta 1.14 type, msg = exc.args
1160 else:
1161 # re-raise the exception
1162 raise
1163
|
1164 arta 1.3 if type == 'unexpectedIccRet':
1165 print('icc -V returned this unexpected message:\n' + msg, file=sys.stderr)
1166 rv = bool(1)
1167 elif type == 'unexpectedGccRet':
1168 print('gcc -v returned this unexpected message:\n' + msg, file=sys.stderr)
1169 rv = bool(1)
1170 elif type == 'unexpectedIfortRet':
1171 print('ifort -V returned this unexpected message:\n' + msg, file=sys.stderr)
1172 rv = bool(1)
1173 elif type == 'unexpectedGfortranRet':
1174 print('gfortran -v returned this unexpected message:\n' + msg, file=sys.stderr)
1175 rv = bool(1)
1176 else:
1177 # re-raise the exception
1178 raise
|
1179 arta 1.28
|
1180 arta 1.2 return rv
|
1181 arta 1.1
|
1182 arta 1.25 def configureSdp(cfgfile, cfile, mfile, pfile, pyfile, shfile, pCfile, pMfile, pRfile, pTfile, base):
|
1183 arta 1.1 rv = bool(0)
|
1184 arta 1.28
|
1185 arta 1.1 defs = {}
1186 cDefs = list()
|
1187 arta 1.5 mDefsGen = list()
1188 mDefsMake = list()
|
1189 arta 1.6 projCfg = list()
1190 projMkRules = list()
1191 projRules = list()
1192 projTarget = list()
|
1193 arta 1.5 mDefsComps = list()
|
1194 arta 1.1 perlConstSection = list()
1195 perlInitSection = list()
|
1196 arta 1.18 pyConstSection = list()
1197 pyInitSection = list()
|
1198 arta 1.25 shConstSection = list()
|
1199 arta 1.18
|
1200 arta 1.2 addenda = {}
|
1201 arta 1.28
|
1202 arta 1.6 # There are three parameters that were not included in the original config.local parameter set, for some reason.
1203 # Due to this omission, then are not configurable, and must be set in the script.
|
1204 arta 1.5 addenda['a:USER'] = 'NULL'
1205 addenda['a:PASSWD'] = 'NULL'
1206 addenda['p:DSDS_SUPPORT'] = '1'
|
1207 arta 1.28
|
1208 arta 1.6 # This parameter is not configurable. BUILD_TYPE is used to distinguish between a NetDRMS and an JSOC-SDP build.
|
1209 arta 1.5 addenda['a:BUILD_TYPE'] = 'JSOC_SDP' # Means a Stanford build. This will set one additional macro used by make: JSOC_SDP_BUILD.
|
1210 arta 1.28
|
1211 arta 1.1 try:
1212 with open(cfgfile, 'r') as fin:
|
1213 arta 1.28
|
1214 arta 1.26 # Always create a Rules.mk and target.mk, even if no proj XML is provided. All builds should have the proj/example and
1215 # proj/cookbook directories. The required make information is in RULESPREFIX, TARGETPREFIX, and RULESSUFFIX. RULESSUFFIX
1216 # must be added after the xml has been parsed.
1217 projRules.extend(list(RULESPREFIX))
1218 projTarget.extend(list(TARGETPREFIX))
|
1219 arta 1.28
|
1220 arta 1.25 rv = parseConfig(fin, None, addenda, defs, cDefs, mDefsGen, mDefsMake, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
|
1221 arta 1.28
|
1222 arta 1.6 if not rv:
|
1223 arta 1.26 projRules.extend(RULESSUFFIX)
|
1224 arta 1.28
|
1225 arta 1.2 # Must add a parameter for the SUMS_MANAGER UID (for some reason)
1226 uidParam = {}
1227 rv = getMgrUIDLine(defs, uidParam)
|
1228 arta 1.6 if not rv:
|
1229 arta 1.25 rv = parseConfig(None, None, uidParam, defs, cDefs, mDefsGen, None, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
|
1230 arta 1.6
|
1231 arta 1.5 # Configure the compiler-selection make variables.
|
1232 arta 1.6 if not rv:
|
1233 arta 1.5 rv = configureComps(defs, mDefsComps)
|
1234 arta 1.28
|
1235 arta 1.6 # Write out the parameter files.
1236 if not rv:
|
1237 arta 1.25 rv = writeParamsFiles(base, cfile, mfile, pfile, pyfile, shfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
|
1238 arta 1.6
1239 # Write out the project-specific make files (make_basic.mk, Rules.mk, and target.mk).
1240 if not rv:
1241 rv = writeProjFiles(pCfile, pMfile, pRfile, pTfile, projCfg, projMkRules, projRules, projTarget)
|
1242 arta 1.13
1243 # At Stanford, skip the creation of the sum_rm configuration file. config.local will still
1244 # have the SUMRM parameters, but they will not be used.
|
1245 arta 1.1 except IOError as exc:
|
1246 arta 1.6 print(exc.strerror, file=sys.stderr)
1247 print('Unable to read configuration file ' + cfgfile + '.', file=sys.stderr)
|
1248 arta 1.5 except Exception as exc:
|
1249 arta 1.17 if len(exc.args) >= 2:
1250 type = exc.args[0]
1251 else:
1252 # re-raise the exception
1253 raise
|
1254 arta 1.28
|
1255 arta 1.5 if type == 'unexpectedIccRet':
|
1256 arta 1.6 msg = exc.args[1]
|
1257 arta 1.5 print('icc -V returned this unexpected message:\n' + msg, file=sys.stderr)
1258 rv = bool(1)
1259 elif type == 'unexpectedGccRet':
|
1260 arta 1.6 msg = exc.args[1]
|
1261 arta 1.5 print('gcc -v returned this unexpected message:\n' + msg, file=sys.stderr)
1262 rv = bool(1)
1263 elif type == 'unexpectedIfortRet':
|
1264 arta 1.6 msg = exc.args[1]
|
1265 arta 1.5 print('ifort -V returned this unexpected message:\n' + msg, file=sys.stderr)
1266 rv = bool(1)
1267 elif type == 'unexpectedGfortranRet':
|
1268 arta 1.6 msg = exc.args[1]
|
1269 arta 1.5 print('gfortran -v returned this unexpected message:\n' + msg, file=sys.stderr)
1270 rv = bool(1)
1271 else:
1272 # re-raise the exception
1273 raise
|
1274 arta 1.28
|
1275 arta 1.2 return rv
1276
|
1277 arta 1.1 # Beginning of program
1278 rv = RET_SUCCESS
1279 net = bool(1)
1280
1281 # Parse arguments
1282 if __name__ == "__main__":
1283 optD = GetArgs(sys.argv[1:])
1284
1285 if not(optD is None):
1286 # Ensure we are configuring a DRMS tree
1287 cdir = os.path.realpath(os.getcwd())
1288 versfile = cdir + '/base/' + VERS_FILE
1289
1290 if not os.path.isfile(versfile):
1291 rv = RET_NOTDRMS
1292
1293 # Determine whether we are localizing a Stanford build, or a NetDRMS build. If configsdp.txt exists, then
1294 # it is a Stanford build, otherwise it is a NetDRMS build.
1295 if rv == RET_SUCCESS:
1296 stanfordFile = cdir + '/' + SDP_CFG
1297 if os.path.isfile(stanfordFile):
1298 arta 1.1 net = bool(0)
|
1299 arta 1.28
|
1300 arta 1.1 cfile = optD['dir'] + '/' + optD['base'] + '.h'
1301 mfile = optD['dir'] + '/' + optD['base'] + '.mk'
1302 pfile = optD['dir'] + '/' + optD['base'] + '.pm'
|
1303 arta 1.18 pyfile = optD['dir'] + '/' + optD['base'] + '.py'
|
1304 arta 1.25 shfile = optD['dir'] + '/' + optD['base'] + '.sh'
|
1305 arta 1.6 pCfile = optD['dir'] + '/configure'
1306 pMfile = optD['dir'] + '/make_basic.mk'
1307 pRfile = optD['dir'] + '/Rules.mk'
1308 pTfile = optD['dir'] + '/target.mk'
|
1309 arta 1.1
1310 if net:
1311 try:
1312 with open(NET_CFGMAP, 'r') as fin:
1313 regexpComm = re.compile(r"^\s*#")
1314 regexp = re.compile(r"^\s*(\S+)\s+(\w:\S+)")
1315 # Must map from config.local namespace to DRMS namespace (e.g., the names used for the C macros)
1316 keymap = {}
1317 for line in fin:
1318 matchobj = regexpComm.match(line)
1319 if not matchobj is None:
1320 # Skip comment line
1321 continue
1322
1323 matchobj = regexp.match(line)
1324 if not(matchobj is None):
1325 # We have a key-value line
1326 key = matchobj.group(1)
1327 val = matchobj.group(2)
1328 keymap[key] = val
1329 except OSError:
1330 arta 1.1 sys.stderr.write('Unable to read configuration map-file ' + NET_CFGMAP + '.')
1331 rv = bool(1)
1332
1333 # We also need to set the UID of the SUMS manager. We have the name of the
1334 # SUMS manager (it is in the configuration file)
|
1335 arta 1.25 configureNet(NET_CFG, cfile, mfile, pfile, pyfile, shfile, pCfile, pMfile, pRfile, pTfile, optD['base'], keymap, 'server' in optD)
|
1336 arta 1.1 else:
|
1337 arta 1.28 # A Stanford user can override the parameters in configsdp.txt by copying that file to config.local,
|
1338 arta 1.8 # and then editing config.local. So, if config.local exists, use that.
1339 if os.path.isfile(cdir + '/' + NET_CFG):
|
1340 arta 1.25 configureSdp(NET_CFG, cfile, mfile, pfile, pyfile, shfile, pCfile, pMfile, pRfile, pTfile, optD['base'])
|
1341 arta 1.8 else:
|
1342 arta 1.25 configureSdp(SDP_CFG, cfile, mfile, pfile, pyfile, shfile, pCfile, pMfile, pRfile, pTfile, optD['base'])
|
Return to
localize.py
CVS log
Up to [Development] / JSOC