1 arta 1.15 #!/usr/bin/python
|
2 arta 1.1
|
3 arta 1.13 # When run with the -s flag, localize.py configures the SUMS-server component of NetDRMS.
|
4 arta 1.15 from __future__ import print_function
|
5 arta 1.1 import sys
6 import getopt
7 import re
|
8 arta 1.6 import os
9 import stat
|
10 arta 1.13 import filecmp
|
11 arta 1.6 import xml.etree.ElementTree as ET
|
12 arta 1.1 from subprocess import check_output, CalledProcessError
13
|
14 arta 1.24 if sys.version_info < (2, 7):
15 raise Exception("You must run the 2.7 release, or a more recent release, of Python.")
16
|
17 arta 1.1 # Constants
18 VERS_FILE = 'jsoc_version.h'
19 SDP_CFG = 'configsdp.txt'
20 NET_CFG = 'config.local'
21 NET_CFGMAP = 'config.local.map'
22 RET_SUCCESS = 0
23 RET_NOTDRMS = 1
24
|
25 arta 1.2 PREFIX = """# This file was auto-generated by localize.py. Please do not edit it directly (running
26 # configure will run localize.py, which will then overwrite any edits manually performed).
27 """
|
28 arta 1.1
|
29 arta 1.2 C_PREFIX = """/* This file was auto-generated by localize.py. Please do not edit it directly (running
30 * configure will run localize.py, which will then overwrite any edits manually performed). */
|
31 arta 1.1 """
32
|
33 arta 1.2 PERL_BINPATH = '#!/usr/bin/perl\n'
34
35 PERL_INTIAL = """package drmsparams;
36
37 use warnings;
38 use strict;
39 """
40
41 PERL_FXNS_A = """sub new
|
42 arta 1.1 {
43 my($clname) = shift;
44
45 my($self) =
46 {
47 _paramsH => undef
48 };
49
50 bless($self, $clname);
|
51 arta 1.9 $self->{_paramsH} = {};
52 $self->initialize();
|
53 arta 1.1
54 return $self;
55 }
56
57 sub DESTROY
58 {
59 my($self) = shift;
60 }
|
61 arta 1.2 """
62
63 PERL_FXNS_B = """sub get
64 {
65 my($self) = shift;
66 my($name) = shift;
67 my($rv);
|
68 arta 1.1
|
69 arta 1.2 if (exists($self->{_paramsH}->{$name}))
70 {
71 return $self->{_paramsH}->{$name};
72 }
73 else
74 {
75 return undef;
76 }
77 }
78 1;"""
|
79 arta 1.1
|
80 arta 1.25 PY_BINPATH = '#!/usr/bin/python\n'
|
81 arta 1.18
82 PY_FXNS_A = """
|
83 arta 1.23 class DRMSParams(object):
|
84 arta 1.18 def __init__(self):
85 self.params = {}
86 self.initialize()
87
88 def __del__(self):
89 del self.params
90
91 def initialize(self):
92 """
93
94 PY_FXNS_B = """ def get(self, name):
95 if name in self.params:
|
96 arta 1.20 return self.params[name]
|
97 arta 1.18 else:
98 return None
99 """
100
|
101 arta 1.22 PY_FXNS_C = """ def getBool(self, name):
102 if name in self.params:
103 return bool(self.params[name] == '1')
104 else:
105 return None
106 """
107
|
108 arta 1.25 SH_BINPATH = '#!/bin/bash\n'
109
|
110 arta 1.22
|
111 arta 1.13 SUMRM_COMMENT = """# This is the configuration file for the sum_rm program. It was auto-generated by the DRMS master configure script.
112 # It controls the behavior of the sum_rm program, and is loaded each time sum_rm runs. To change the
113 # parameter values in this configuration file, modify config.local, then re-run configure. This configuration
114 # file will be updated only if parameters affecting it are modified. If such changes are made to config.local,
115 # please make sure that the sum_rm service is disabled while configure in running.
116 """
117
118 SUMRM_DOC = """# sum_rm removes end-of-life SUMS data files to prevent disk-partitions from becomming 100% full.
119 # sum_svc, the main SUMS service, starts this daemon when it is launched. Within an infinite loop, sum_rm sleeps
120 # for SLEEP number of seconds (defined below). When it awakes, it loads the sum_rm configuration file. For each SU,
121 # it then examines the 'effective_date' column within the sum_partn_alloc table of the SUMS database. It does so by
122 # sorting all SUs by effective date (this date is actually an expiration date - the SU is good until the end of that day),
123 # and then, starting with the SU with the oldest effective date, it deletes the SUs. It continues deleting SUs until one
124 # of three conditions is met:
125 #
126 # (a) The disk has at least PART_PERCENT_FREE percent free space.
127 # (b) All SUs have effective_date values in the future.
128 # (c) At least 600 SUs have been deleted (this is defined in the LIMIT statement in the file SUMLIB_RmDoX.pgc).
129 #
130 # After sum_rm stops deleteing SUs, it then sleeps for SLEEP seconds, completing the first iteration of the
131 # infinite loop.
132 arta 1.13 """
133
134 SUMRM_PARTN_PERCENT_FREE = """
135 # This is the percentage at which all disk partitions are to be kept free.
136 # If not specified, this defaults to 3. For example, setting PART_PERCENT_FREE = 5 will allow all partitions to
137 # fill to 95% full. Dividing the number of unused blocks by the total number of blocks, and rounding up,
138 # will result in the number specified by PART_PERCENT_FREE.
139 #
140 # NOTE : This behavior was previously controlled by the MAX_FREE_{n} family of parameters. {n} referred to the
141 # disk-partition number and the value of parameter MAX_FREE_{n} was the MINIMUM number of free MB in the
142 # partition [No clue why the word MAX was used]. As of NetDRMS 7.0, MAX_FREE_{n} has no effect."""
143
144 SUMRM_SLEEP = """
145 # The value is the number of seconds to sleep between iterations of the main loop in sum_rm."""
146
147 SUMRM_LOG = """
148 # The value is the log file (opened only at sum_rm startup; the sum_rm pid is appended to this file name)."""
149
150 SUMRM_MAIL = """
151 # The value is the email address of the recipient to be notified in case of a problem."""
152
153 arta 1.13 SUMRM_NOOP = """
154 # If the value is set to anything other than 0, then sum_rm is rendered inactive. Otherwise, sum_rm is active."""
155
156 SUMRM_USER = """
157 # The value designates the linux user who is allowed to run sum_rm."""
158
159 SUMRM_NORUN = """
160 # This pair of paramters defines a window of time, during which sum_rm will become torpid - in this
161 # window, sum_rm will not scan for SUs to delete, not will it delete any SUs. Each value is an integer, 0-23, that
162 # represents an hour of the day. For example, if NORUN_START=8 and NORUN_STOP=10, then between 8am and 10am
163 # local time, sum_rm will be dormant. To disable this behavior, set both parameters to the same value."""
|
164 arta 1.6
165 RULESPREFIX = """# Standard things
166 sp := $(sp).x
167 dirstack_$(sp) := $(d)
168 d := $(dir)
169 """
170
171 RULESSUFFIX = """dir := $(d)/example
172 -include $(SRCDIR)/$(dir)/Rules.mk
173 dir := $(d)/cookbook
174 -include $(SRCDIR)/$(dir)/Rules.mk
175 dir := $(d)/myproj
176 -include $(SRCDIR)/$(dir)/Rules.mk
177
178 # Standard things
179 d := $(dirstack_$(sp))
180 sp := $(basename $(sp))
181 """
182
183 TARGETPREFIX = """$(PROJOBJDIR):\n\t+@[ -d $@ ] || mkdir -p $@"""
184
|
185 arta 1.3 ICC_MAJOR = 9
186 ICC_MINOR = 0
187 GCC_MAJOR = 3
188 GCC_MINOR = 0
189 IFORT_MAJOR = 9
190 IFORT_MINOR = 0
191 GFORT_MAJOR = 4
192 GFORT_MINOR = 2
193
194
|
195 arta 1.1 # Read arguments
196 # d - localization directory
197 # b - base name of all parameter files (e.g., -b drmsparams --> drmsparams.h, drmsparams.mk, drmsparams.pm, etc.)
|
198 arta 1.13 # s - configure a NetDRMS server (i.e., SUMS server). Otherwise, do client configuration only. A server
199 # configuration will modify something that only the production user has access to. An example is the sum_rm.cfg
200 # file. To modify that file, the user running configure must be running configure on the SUMS-server host, and
201 # must have write permission on sum_rm.cfg.
|
202 arta 1.1 def GetArgs(args):
203 rv = bool(0)
204 optD = {}
205
206 try:
|
207 arta 1.13 opts, remainder = getopt.getopt(args, "hd:b:s",["dir=", "base="])
|
208 arta 1.1 except getopt.GetoptError:
209 print('Usage:')
210 print('localize.py [-h] -d <localization directory> -b <parameter file base>')
211 rv = bool(1)
212
213 if rv == bool(0):
214 for opt, arg in opts:
215 if opt == '-h':
216 print('localize.py [-h] -d <localization directory> -b <parameter file base>')
217 elif opt in ("-d", "--dir"):
218 regexp = re.compile(r"(\S+)/?")
219 matchobj = regexp.match(arg)
220 if matchobj is None:
221 rv = bool(1)
222 else:
223 optD['dir'] = matchobj.group(1)
224 elif opt in ("-b", "--base"):
225 optD['base'] = arg
|
226 arta 1.13 elif opt in ("-s"):
227 optD['server'] = ""
|
228 arta 1.1 else:
229 optD[opt] = arg
230
231 return optD
232
233 def createMacroStr(key, val, keyColLen, status):
234 if keyColLen < len(key):
235 status = bool(1)
236 return None
237 else:
238 nsp = keyColLen - len(key)
239 spaces = str()
240 for isp in range(nsp):
241 spaces += ' '
242 status = bool(0)
243 return '#define ' + key + spaces + val + '\n'
244
245 def createPerlConst(key, val, keyColLen, status):
246 if keyColLen < len(key):
247 status = bool(1)
248 return None
249 arta 1.1 else:
250 nsp = keyColLen - len(key)
251 spaces = str()
252 for isp in range(nsp):
253 spaces += ' '
254 status = bool(0)
255 return 'use constant ' + key + ' => ' + spaces + val + ';\n'
256
|
257 arta 1.18 def createPyConst(key, val, keyColLen, status):
258 if keyColLen < len(key):
259 status = bool(1)
260 return None
261 else:
262 nsp = keyColLen - len(key)
263 spaces = str()
264 for isp in range(nsp):
265 spaces += ' '
266 status = bool(0)
267 return key + ' = ' + spaces + val + '\n'
|
268 arta 1.25
269 def createShConst(key, val, status):
270 status = bool(0)
271 return key + '=' + val + '\n'
272
|
273 arta 1.18
|
274 arta 1.3 def isSupportedPlat(plat):
275 regexp = re.compile(r"\s*(^x86_64|^ia32|^ia64|^avx)", re.IGNORECASE)
276 matchobj = regexp.match(plat);
277
278 if not matchobj is None:
279 return bool(1);
280 else:
281 return bool(0);
282
|
283 arta 1.6 def processMakeParam(mDefs, key, val, platDict, machDict):
|
284 arta 1.3 varMach = None
285 regexp = re.compile(r"(\S+):(\S+)")
286 matchobj = regexp.match(key)
287 if not matchobj is None:
288 varName = matchobj.group(1)
289 varMach = matchobj.group(2)
290 else:
291 varName = key
292
293 varValu = val
294
295 if varMach is None:
296 mDefs.extend(list('\n' + varName + ' = ' + varValu))
297 else:
298 if isSupportedPlat(varMach):
299 # The guard will compare varValu to $JSOC_MACHINE.
300 if not varMach in platDict:
301 platDict[varMach] = {}
302 platDict[varMach][varName] = varValu
303 else:
304 # The guard will compare varValu to $MACHINETYPE (this is just the hostname of the machine on which localize.py is running).
305 arta 1.3 if not varMach in machDict:
306 machDict[varMach] = {}
307 machDict[varMach][varName] = varValu
308
|
309 arta 1.25 def processParam(cfgfile, line, regexpQuote, regexp, keymap, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection, platDict, machDict, section):
|
310 arta 1.1 status = 0
311
312 if ''.join(section) == 'defs' or not cfgfile:
313 matchobj = regexp.match(line)
314 if not matchobj is None:
315 # We have a key-value line
316 keyCfgSp = matchobj.group(1)
317 val = matchobj.group(2)
318
319 # Must map the indirect name to the actual name
320 if keymap:
321 # Map to actual name only if the keymap is not empty (which signifies NA).
322 if keyCfgSp in keymap:
323 key = keymap[keyCfgSp]
324 elif keyCfgSp == 'LOCAL_CONFIG_SET' or keyCfgSp == 'DRMS_SAMPLE_NAMESPACE':
|
325 arta 1.6 # Ignore parameters that are not useful and shouldn't have been there in the first place. But
326 # they have been released to the world, so we have to account for them.
|
327 arta 1.1 return bool(0)
328 elif not cfgfile:
329 # Should not be doing mapping for addenda
330 key = keyCfgSp
331 else:
332 raise Exception('badKeyMapKey', keyCfgSp)
333 else:
334 key = keyCfgSp
335
336 matchobj = regexpQuote.match(key)
337 if not matchobj is None:
338 quote = matchobj.group(1)
339 key = matchobj.group(2)
340
341 # master defs dictionary
342 defs[key] = val
343
344 # C header file
345 if quote == "q":
346 # Add double-quotes
347 cDefs.extend(list(createMacroStr(key, '"' + val + '"', 40, status)))
348 arta 1.1 elif quote == "p":
349 # Add parentheses
350 cDefs.extend(list(createMacroStr(key, '(' + val + ')', 40, status)))
351 elif quote == "a":
352 # Leave as-is
353 cDefs.extend(list(createMacroStr(key, val, 40, status)))
354 else:
355 # Unknown quote type
356 raise Exception('badQuoteQual', key)
357
358 if status:
359 raise Exception('paramNameTooLong', key)
360
361 # Make file - val should never be quoted; just use as is
|
362 arta 1.4 mDefsGen.extend(list('\n' + key + ' = ' + val))
|
363 arta 1.1
364 # Perl file - val should ALWAYS be single-quote quoted
365 # Save const info to a string
366 perlConstSection.extend(list(createPerlConst(key, "'" + val + "'", 40, status)))
367
|
368 arta 1.18 # Python file
369 pyConstSection.extend(list(createPyConst(key, "'" + val + "'", 40, status)))
370
|
371 arta 1.25 # Shell source file
372 shConstSection.extend(list(createShConst(key, "'" + val + "'", status)))
373
|
374 arta 1.1 if status:
375 raise Exception('paramNameTooLong', key)
376
377 # Save initialization information as a string. Now that we've defined
378 # constants (the names of which are the parameter names)
379 # we can refer to those in the init section. The key variable holds the
380 # name of the constant.
|
381 arta 1.9 perlInitSection.extend(list("\n $self->{_paramsH}->{'" + key + "'} = " + key + ';'))
|
382 arta 1.18
383 # The amount of indenting matters! This is Python.
384 pyInitSection.extend(list(" self.params['" + key + "'] = " + key + '\n'))
|
385 arta 1.1 else:
386 # No quote qualifier
387 raise Exception('missingQuoteQual', key)
|
388 arta 1.3 elif ''.join(section) == 'make' and cfgfile:
389 # Configure the remaining make variables defined in the __MAKE__ section of the configuration file. Third-party
390 # library make variables are specified in the __MAKE__ section.
391 matchobj = regexp.match(line)
392 if not matchobj is None:
393 # We have a key-value line
394 key = matchobj.group(1)
395 val = matchobj.group(2)
396
397 # This information is for making make variables only. We do not need to worry about quoting any values
398 defs[key] = val
|
399 arta 1.4 processMakeParam(mDefsMake, key, val, platDict, machDict)
|
400 arta 1.1
401 return bool(0)
402
403 # We have some extraneous line or a newline - ignore.
404
|
405 arta 1.6 def processXML(xml, projRules, projTarget):
406 rv = bool(0)
407
408 # <projects>
409 root = ET.fromstring(xml)
410
411 # Iterate through each proj child.
412 for proj in root.iter('proj'):
413 # Rules.mk
414 nameElem = proj.find('name')
415 rulesStr = 'dir := $(d)/' + nameElem.text + '\n-include $(SRCDIR)/$(dir)/Rules.mk\n'
416
417 # make doesn't support logical operations in ifeq conditionals (you can't do ifeq (A AND B)),
418 # so we need to write:
419 # ifeq (A)
420 # ifeq (B)
421 # <do something>
422 # endif
423 # endif
424
425 rulesPref = '';
426 arta 1.6 rulesSuff = '';
427
428 filters = proj.find('filters')
429 if filters is not None:
430 for filter in filters.findall('filter'):
431 rulesPref += 'ifeq ($(' + filter.find('name').text + '),' + filter.find('value').text + ')\n'
432 rulesSuff += 'endif\n'
433
434 if len(rulesPref) > 0 and len(rulesSuff) > 0:
435 projRules.extend(list(rulesPref))
436 projRules.extend(list(rulesStr))
437 projRules.extend(list(rulesSuff))
438 else:
439 projRules.extend(list(rulesStr))
440
441 # target.mk
442 subdirs = proj.find('subdirs')
443 if subdirs is not None:
444 for subdir in subdirs.findall('subdir'):
445 targetStr = '\n\t+@[ -d $@/' + nameElem.text + '/' + subdir.text + ' ] || mkdir -p $@/' + nameElem.text + '/' + subdir.text
446 projTarget.extend(list(targetStr))
447 arta 1.6
448 return rv
449
|
450 arta 1.27 def determineSection(line, regexpStyle, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg):
451 matchobj = regexpStyle.match(line)
452 if matchobj:
453 return 'style'
454
|
455 arta 1.6 matchobj = regexpDefs.match(line)
456 if not matchobj is None:
457 return 'defs'
458
459 matchobj = regexpMake.match(line)
460 if not matchobj is None:
461 return 'make'
462
463 matchobj = regexpProjMkRules.match(line)
464 if not matchobj is None:
465 return 'projmkrules'
466
467 matchobj = regexpProj.match(line)
468 if not matchobj is None:
469 return 'proj'
470
471 matchobj = regexpProjCfg.match(line)
472 if not matchobj is None:
473 return 'projcfg'
474
475 return None
476 arta 1.6
|
477 arta 1.1 # defs is a dictionary containing all parameters (should they be needed in this script)
|
478 arta 1.6 # projCfg is the list containing the configure script content.
479 # projMkRules is the list containing the make_basic.mk content.
480 # projRules is the list containing the Rules.mk content.
481 # projTargert is the list containing the target.mk content.
|
482 arta 1.25 def parseConfig(fin, keymap, addenda, defs, cDefs, mDefsGen, mDefsMake, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection):
|
483 arta 1.1 rv = bool(0)
484
485 # Open required config file (config.local)
486 try:
487 # Examine each line, looking for key=value pairs.
|
488 arta 1.27 regexpStyle = re.compile(r"^__STYLE__")
|
489 arta 1.1 regexpDefs = re.compile(r"^__DEFS__")
490 regexpMake = re.compile(r"^__MAKE__")
|
491 arta 1.6 regexpProjMkRules = re.compile(r"__PROJ_MK_RULES__")
492 regexpProj = re.compile(r"^__PROJ__")
493 regexpProjCfg = re.compile(r"^__PROJCFG__")
|
494 arta 1.1 regexpComm = re.compile(r"^\s*#")
|
495 arta 1.6 regexpSp = re.compile(r"^s*$")
|
496 arta 1.1 regexpQuote = re.compile(r"^\s*(\w):(.+)")
|
497 arta 1.6 regexpCustMkBeg = re.compile(r"^_CUST_")
498 regexpCustMkEnd = re.compile(r"^_ENDCUST_")
499 regexpDiv = re.compile(r"^__")
|
500 arta 1.11 regexp = re.compile(r"^\s*(\S+)\s+(\S.*)")
|
501 arta 1.27
502 ignoreKeymap = False
|
503 arta 1.3 platDict = {}
504 machDict = {}
|
505 arta 1.6
506 xml = None
|
507 arta 1.1
508 # Process the parameters in the configuration file
509 if not fin is None:
510 for line in fin:
|
511 arta 1.6 matchobj = regexpComm.match(line)
512 if not matchobj is None:
513 # Skip comment line
514 continue
515
516 matchobj = regexpSp.match(line)
517 if not matchobj is None:
518 # Skip whitespace line
519 continue
|
520 arta 1.1
|
521 arta 1.27 newSection = determineSection(line, regexpStyle, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg)
|
522 arta 1.6 if not newSection is None:
523 section = newSection
|
524 arta 1.27
525 if not section:
526 raise Exception('invalidConfigFile', 'line ' + line.strip() + ' is not in any section')
|
527 arta 1.6
|
528 arta 1.27 if section == 'style':
529 # if the config.local file has new in the __STYLE__ section, then ignore the keymap and treat config.local like configsdp.txt;
530 # do not map from NetDRMS config.local parameter names to configsdp.txt names
531 for line in fin:
532 matchobj = regexpDiv.match(line)
533 if matchobj:
534 break;
535 if line.strip(' \n').lower() == 'new' and keymap:
536 ignoreKeymap = True
537
538 newSection = determineSection(line, regexpStyle, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg)
539 if not newSection is None:
540 section = newSection
541 continue
542 elif section == 'make':
|
543 arta 1.10
|
544 arta 1.6 # There are some blocks of lines in the __MAKE__ section that must be copied ver batim to the output make file.
545 # The blocks are defined by _CUST_/_ENDCUST_ tags.
546 matchobj = regexpCustMkBeg.match(line)
547
548 if not matchobj is None:
549 mDefsMake.extend(list('\n'))
550 for line in fin:
551 matchobj = regexpCustMkEnd.match(line)
552 if not matchobj is None:
553 break;
554 mDefsMake.extend(list(line))
|
555 arta 1.27 newSection = determineSection(line, regexpStyle, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg)
|
556 arta 1.6 if not newSection is None:
557 section = newSection
558 continue
559 # Intentional fall through to next if statement
560 if section == 'defs' or section == 'make':
561 iscfg = bool(1)
|
562 arta 1.27 if ignoreKeymap:
563 keymapActual = None
564 else:
565 keymapActual = keymap
566 ppRet = processParam(iscfg, line, regexpQuote, regexp, keymapActual, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection, platDict, machDict, section)
|
567 arta 1.10
|
568 arta 1.6 if ppRet:
|
569 arta 1.10 break
|
570 arta 1.6 elif section == 'projcfg':
571 # Copy the line ver batim to the projCfg list (configure)
572 for line in fin:
573 matchobj = regexpDiv.match(line)
574 if not matchobj is None:
575 break;
576 projCfg.extend(list(line))
|
577 arta 1.27 newSection = determineSection(line, regexpStyle, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg)
|
578 arta 1.6 if not newSection is None:
579 section = newSection
580 continue
581 elif section == 'projmkrules':
582 # Copy the line ver batim to the projMkRules list (make_basic.mk)
583 for line in fin:
584 matchobj = regexpDiv.match(line)
585 if not matchobj is None:
586 break;
587 projMkRules.extend(list(line))
|
588 arta 1.27 newSection = determineSection(line, regexpStyle, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg)
|
589 arta 1.6 if not newSection is None:
590 section = newSection
591 continue
592 elif section == 'proj':
593 # Must parse xml and use the project-specific information to populate the Rules.mk and target.mk files.
594 # Collect all xml lines for now, then process after file-read loop.
595 if xml is None:
|
596 arta 1.21 # The first time through this section, line is the config.local div, __PROJ__. Discard that.
597 xml = ''
598 continue
|
599 arta 1.6 else:
600 xml += line
601 else:
|
602 arta 1.10 # Unknown section
|
603 arta 1.6 raise Exception('unknownSection', section)
|
604 arta 1.1 except Exception as exc:
|
605 arta 1.17 if len(exc.args) >= 2:
606 msg = exc.args[0]
607 else:
608 # re-raise the exception
609 raise
|
610 arta 1.27
611 if msg == 'invalidConfigFile':
612 violator = exc.args[1]
613 print(violator, file=sys.stderr)
614 rv = bool(1)
615 elif msg == 'badKeyMapKey':
|
616 arta 1.1 # If we are here, then there was a non-empty keymap, and the parameter came from
617 # the configuration file.
|
618 arta 1.6 violator = exc.args[1]
|
619 arta 1.17 print('Unknown parameter name ' + "'" + violator + "'" + ' in ' + fin.name + '.', file=sys.stderr)
|
620 arta 1.1 rv = bool(1)
621 elif msg == 'badQuoteQual':
622 # The bad quote qualifier came from the configuration file, not the addenda, since
623 # we will have fixed any bad qualifiers in the addenda (which is populated by code).
|
624 arta 1.6 violator = exc.args[1]
|
625 arta 1.17 print('Unknown quote qualifier ' + "'" + violator + "'" + ' in ' + fin.name + '.', file=sys.stderr)
|
626 arta 1.1 rv = bool(1)
627 elif msg == 'missingQuoteQual':
|
628 arta 1.6 violator = exc.args[1]
|
629 arta 1.17 print('Missing quote qualifier for parameter ' + "'" + violator + "'" + ' in ' + fin.name + '.', file=sys.stderr)
|
630 arta 1.1 rv = bool(1)
631 elif msg == 'paramNameTooLong':
|
632 arta 1.6 violator = exc.args[1]
|
633 arta 1.1 print('Macro name ' + "'" + violator + "' is too long.", file=sys.stderr)
634 rv = bool(1)
|
635 arta 1.6 elif msg == 'unknownSection':
636 violator = exc.args[1]
637 print('Unknown section ' + "'" + violator + "' in configuration file.", file=sys.stderr)
638 rv = bool(1)
|
639 arta 1.1 else:
640 # re-raise the exception
641 raise
642
|
643 arta 1.6 if not rv:
644 if not xml is None:
645 # Process xml.
646 rv = processXML(xml, projRules, projTarget)
647
648 # Process addenda - these are parameters that are not configurable and must be set in the
649 # NetDRMS build.
650 if not rv:
651 iscfg = bool(0)
652 for key in addenda:
653 item = key + ' ' + addenda[key]
|
654 arta 1.27 if ignoreKeymap:
655 keymapActual = None
656 else:
657 keymapActual = keymap
658 ppRet = processParam(iscfg, item, regexpQuote, regexp, keymapActual, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection, platDict, machDict, 'defs')
|
659 arta 1.6 if ppRet:
660 break;
661
|
662 arta 1.4 # Put information collected in platDict and machDict into mDefs. Must do this here, and not in processParam, since
663 # we need to parse all platform-specific make variables before grouping them into platform categories.
|
664 arta 1.3 if not rv:
665 for plat in platDict:
|
666 arta 1.4 mDefsMake.extend(list('\nifeq ($(JSOC_MACHINE), linux_' + plat.lower() + ')'))
|
667 arta 1.3 for var in platDict[plat]:
|
668 arta 1.4 mDefsMake.extend(list('\n' + var + ' = ' + platDict[plat][var]))
669 mDefsMake.extend(list('\nendif\n'))
|
670 arta 1.3
671 if not rv:
672 for mach in machDict:
|
673 arta 1.4 mDefsMake.extend(list('\nifeq ($(MACHTYPE), ' + mach + ')'))
|
674 arta 1.7 for var in machDict[mach]:
675 mDefsMake.extend(list('\n' + var + ' = ' + machDict[mach][var]))
|
676 arta 1.4 mDefsMake.extend(list('\nendif\n'))
|
677 arta 1.1 return rv
678
679 def getMgrUIDLine(defs, uidParam):
680 rv = bool(0)
681
682 cmd = 'id -u ' + defs['SUMS_MANAGER']
683 try:
684 ret = check_output(cmd, shell=True)
685 uidParam['q:SUMS_MANAGER_UID'] = ret.decode("utf-8")
686 except ValueError:
687 print('Unable to run cmd: ' + cmd + '.')
688 rv = bool(1)
689 except CalledProcessError:
690 print('Command ' + "'" + cmd + "'" + ' ran improperly.')
691 rv = bool(1)
692
693 return rv
694
|
695 arta 1.3 def isVersion(maj, min, majDef, minDef):
696 res = 0
697
698 if maj > majDef or (maj == majDef and min >= minDef):
699 res = 1
700
701 return res
702
703 def configureComps(defs, mDefs):
704 rv = bool(0)
|
705 arta 1.6 autoConfig = bool(1)
706
707 if 'AUTOSELCOMP' in defs:
708 autoConfig = (not defs['AUTOSELCOMP'] == '0')
|
709 arta 1.3
710 if autoConfig:
711 hasicc = bool(0)
712 hasgcc = bool(0)
713 hasifort = bool(0)
714 hasgfort = bool(0)
715
716 # Try icc.
|
717 arta 1.12 cmd = 'icc --version 2>&1'
|
718 arta 1.3 try:
719 ret = check_output(cmd, shell=True)
720 ret = ret.decode("utf-8")
721 except CalledProcessError:
722 print('Command ' + "'" + cmd + "'" + ' ran improperly.')
723 rv = bool(1)
724
|
725 arta 1.6 if not rv:
|
726 arta 1.12 regexp = re.compile(r"\s*\S+\s+\S+\s+(\d+)[.](\d+)", re.DOTALL)
|
727 arta 1.3 matchobj = regexp.match(ret)
728 if matchobj is None:
729 raise Exception('unexpectedIccRet', ret)
730 else:
731 major = matchobj.group(1)
732 minor = matchobj.group(2)
733 if isVersion(int(major), int(minor), ICC_MAJOR, ICC_MINOR):
734 hasicc = bool(1)
|
735 arta 1.19
|
736 arta 1.3 # Try gcc.
737 if not hasicc:
|
738 arta 1.19 rv = bool(0)
|
739 arta 1.3 cmd = 'gcc -v 2>&1'
740 try:
741 ret = check_output(cmd, shell=True)
742 ret = ret.decode("utf-8")
743 except CalledProcessError:
744 print('Command ' + "'" + cmd + "'" + ' ran improperly.')
745 rv = bool(1)
746
|
747 arta 1.19 if not rv:
|
748 arta 1.3 regexp = re.compile(r".+gcc\s+version\s+(\d+)\.(\d+)", re.DOTALL)
749 matchobj = regexp.match(ret)
750 if matchobj is None:
751 raise Exception('unexpectedGccRet', ret)
752 else:
753 major = matchobj.group(1)
754 minor = matchobj.group(2)
755 if isVersion(int(major), int(minor), GCC_MAJOR, GCC_MINOR):
756 hasgcc = bool(1)
757
758 # Try ifort.
|
759 arta 1.19 rv = bool(0)
|
760 arta 1.12 cmd = 'ifort --version 2>&1'
|
761 arta 1.3 try:
762 ret = check_output(cmd, shell=True)
763 ret = ret.decode("utf-8")
764 except CalledProcessError:
765 print('Command ' + "'" + cmd + "'" + ' ran improperly.')
766 rv = bool(1)
767
|
768 arta 1.6 if not rv:
|
769 arta 1.12 regexp = re.compile(r"\s*\S+\s+\S+\s+(\d+)\.(\d+)", re.DOTALL)
|
770 arta 1.3 matchobj = regexp.match(ret)
771 if matchobj is None:
772 raise Exception('unexpectedIfortRet', ret)
773 else:
774 major = matchobj.group(1)
775 minor = matchobj.group(2)
776 if isVersion(int(major), int(minor), IFORT_MAJOR, IFORT_MINOR):
777 hasifort = bool(1)
778
779 # Try gfortran
780 if not hasifort:
|
781 arta 1.19 rv = bool(0)
|
782 arta 1.3 cmd = 'gfortran -v 2>&1'
783 try:
784 ret = check_output(cmd, shell=True)
785 ret = ret.decode("utf-8")
786 except CalledProcessError:
787 print('Command ' + "'" + cmd + "'" + ' ran improperly.')
788 rv = bool(1)
789
|
790 arta 1.19 if not rv:
|
791 arta 1.3 regexp = re.compile(r".+gcc\s+version\s+(\d+)\.(\d+)", re.DOTALL)
792 matchobj = regexp.match(ret)
793 if matchobj is None:
794 raise Exception('unexpectedGfortranRet', ret)
795 else:
796 major = matchobj.group(1)
797 minor = matchobj.group(2)
798 if isVersion(int(major), int(minor), GFORT_MAJOR, GFORT_MINOR):
799 hasgfort = bool(1)
800
|
801 arta 1.19 # Append the compiler make variables to the make file
802 rv = bool(0)
803
|
804 arta 1.3 if not hasicc and not hasgcc:
805 print('Fatal error: Acceptable C compiler not found! You will be unable to build the DRMS library.', file=sys.stderr)
806 rv = bool(1)
807 elif hasicc:
808 mDefs.extend(list('\nCOMPILER = icc'))
809 else:
810 mDefs.extend(list('\nCOMPILER = gcc'))
811
812 if not hasifort and not hasgfort:
813 print('Warning: Acceptable Fortran compiler not found! Fortran interface will not be built, and you will be unable to build Fortran modules.', file=sys.stderr)
814 elif hasifort:
815 mDefs.extend(list('\nFCOMPILER = ifort'))
816 else:
817 mDefs.extend(list('\nFCOMPILER = gfortran'))
818
819 # Environment overrides. These get written, regardless of the disposition of auto-configuration.
|
820 arta 1.9 mDefs.extend(list('\nifneq ($(JSOC_COMPILER),)\n COMPILER = $(JSOC_COMPILER)\nendif'))
821 mDefs.extend(list('\nifneq ($(JSOC_FCOMPILER),)\n FCOMPILER = $(JSOC_FCOMPILER)\nendif'))
|
822 arta 1.3
823 return rv
824
|
825 arta 1.25 def writeParamsFiles(base, cfile, mfile, pfile, pyfile, shfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection):
|
826 arta 1.1 rv = bool(0)
|
827 arta 1.4
828 # Merge mDefsGen, mDefsMake, and mDefsComps into a single string with compiler configuration first, general parameters next, then
829 # make-specific make variables (e.g., third-party library information) last.
830 mDefs = '\n# Compiler Selection\n' + ''.join(mDefsComps) + '\n\n# General Parameters\n' + ''.join(mDefsGen) + '\n\n# Parameters to Configure make\n' + ''.join(mDefsMake)
|
831 arta 1.1
832 try:
|
833 arta 1.25 with open(cfile, 'w') as cout, open(mfile, 'w') as mout, open(pfile, 'w') as pout, open(pyfile, 'w') as pyout, open(shfile, 'w') as shout:
|
834 arta 1.1 # C file of macros
|
835 arta 1.2 print(C_PREFIX, file=cout)
836 print('/* This file contains a set of preprocessor macros - one for each configuration parameter. */\n', file=cout)
|
837 arta 1.1 buf = '__' + base.upper() + '_H'
838 print('#ifndef ' + buf, file=cout)
839 print('#define ' + buf, file=cout)
840 print(''.join(cDefs), file=cout)
841 print('#endif', file=cout)
842
843 # Make file of make variables
|
844 arta 1.2 print(PREFIX, file=mout)
|
845 arta 1.4 print('# This file contains a set of make-variable values. The first section contains compiler-selection variables, the second contains general configuration variables, and the third section contains variables that configure how make is run.', file=mout)
846 print(mDefs, file=mout)
|
847 arta 1.1
|
848 arta 1.2 # Perl module
849 print(PERL_BINPATH, file=pout)
850 print(PREFIX, file=pout)
851 print('# This file contains a set of constants - one for each configuration parameter.\n', file=pout)
852 print(PERL_INTIAL, file=pout)
|
853 arta 1.1 print(''.join(perlConstSection), file=pout)
|
854 arta 1.2 print(PERL_FXNS_A, file=pout)
|
855 arta 1.1 print('sub initialize', file=pout)
856 print('{', file=pout)
|
857 arta 1.2 print(' my($self) = shift;', file=pout, end='')
|
858 arta 1.1 print('', file=pout)
859 print(''.join(perlInitSection), file=pout)
|
860 arta 1.2 print('}\n', file=pout)
861 print(PERL_FXNS_B, file=pout)
|
862 arta 1.18
863 # Python module
864 print(PY_BINPATH, file=pyout)
865 print(PREFIX, file=pyout)
866 print('# This file contains a set of constants - one for each configuration parameter.\n', file=pyout)
867 print(''.join(pyConstSection), file=pyout)
868 print(PY_FXNS_A, file=pyout, end='')
869 print(''.join(pyInitSection), file=pyout)
870 print(PY_FXNS_B, file=pyout)
|
871 arta 1.22 print(PY_FXNS_C, file=pyout)
|
872 arta 1.25
873 # Shell (bash) source file
874 print(SH_BINPATH, file=shout)
875 print(PREFIX, file=shout)
876 print('# This file contains a set of variable assignments - one for each configuration parameter.\n', file=shout)
877 print(''.join(shConstSection), file=shout)
|
878 arta 1.18
|
879 arta 1.1 except IOError as exc:
|
880 arta 1.6 type, value, traceback = sys.exc_info()
881 print(exc.strerror, file=sys.stderr)
882 print('Unable to open ' + "'" + value.filename + "'.", file=sys.stderr)
883 rv = bool(1)
884
885 return rv
886
887 def writeProjFiles(pCfile, pMfile, pRfile, pTfile, projCfg, projMkRules, projRules, projTarget):
888 rv = bool(0)
889
890 try:
891 if projCfg:
892 with open(pCfile, 'w') as cout:
893 # configure
894 print(''.join(projCfg), file=cout)
895
896 if projMkRules:
897 with open(pMfile, 'w') as mout:
898 # make_basic.mk
899 print(PREFIX, file=mout)
900 print(''.join(projMkRules), file=mout)
901 arta 1.6
902 if projRules:
903 with open(pRfile, 'w') as rout:
904 # Rules.mk
905 print(PREFIX, file=rout)
906 print(''.join(projRules), file=rout)
907
908 if projTarget:
909 with open(pTfile, 'w') as tout:
910 # target.mk
911 print(PREFIX, file=tout)
|
912 arta 1.10 print(''.join(projTarget), file=tout)
|
913 arta 1.6 except IOError as exc:
914 type, value, traceback = sys.exc_info()
915 print(exc.strerror, file=sys.stderr)
916 print('Unable to open ' + "'" + value.filename + "'.", file=sys.stderr)
|
917 arta 1.1 rv = bool(1)
918
|
919 arta 1.6 if not rv:
|
920 arta 1.10 if os.path.exists(pCfile):
921 try:
922 os.chmod(pCfile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IROTH)
923 except OSError as exc:
924 type, value, traceback = sys.exc_info()
925 print('Unable to chmod file ' + "'" + value.filename + "'.", file=sys.stderr)
926 print(exc.strerror, file=sys.stderr)
927 rv = bool(1)
|
928 arta 1.6
|
929 arta 1.1 return rv
|
930 arta 1.3
|
931 arta 1.13 def generateSumRmCfg(defs):
932 rv = bool(0)
933 # ACK! Remember that Rick renamed these parameters. The ones in config.local are the aliases - do not use those.
934 # Use the ones that those map to (defined in config.local.map).
935 cFileTmp = defs['SUMLOG_BASEDIR'] + '/' + '.sum_rm.cfg.tmp'
936 cFile = defs['SUMLOG_BASEDIR'] + '/' + 'sum_rm.cfg'
937
938 # Write a temporary file sum_rm configuration file.
939 try:
940 with open(cFileTmp, 'w') as fout:
941 # Print comment at the top of the configuration file.
942 print(SUMRM_COMMENT, file=fout)
943 print(SUMRM_DOC, file=fout)
944 print(SUMRM_PARTN_PERCENT_FREE, file=fout)
945 if 'SUMRM_PART_PERCENT_FREE' in defs:
946 print('PART_PERCENT_FREE=' + defs['SUMRM_PART_PERCENT_FREE'], file=fout)
947 else:
948 print('PART_PERCENT_FREE=3', file=fout)
949
950 print(SUMRM_SLEEP, file=fout)
951 if 'SUMRM_SLEEP' in defs:
952 arta 1.13 print('SLEEP=' + defs['SUMRM_SLEEP'], file=fout)
953 else:
954 print('SLEEP=300', file=fout)
955
956 print(SUMRM_LOG, file=fout)
957 if 'SUMRM_LOG' in defs:
958 print('LOG=' + defs['SUMRM_LOG'], file=fout)
959 else:
960 print('LOG=/tmp/sum_rm.log', file=fout)
961
962 print(SUMRM_MAIL, file=fout)
963 # No default for mail - don't send nothing to nobody unless the operator has asked for notifications.
964 if 'SUMRM_MAIL' in defs:
965 print('MAIL=' + defs['SUMRM_MAIL'], file=fout)
966 else:
967 print('# MAIL=president@whitehouse.gov', file=fout)
968
969 print(SUMRM_NOOP, file=fout)
970 if 'SUMRM_NOOP' in defs:
971 print('NOOP=' + defs['SUMRM_NOOP'], file=fout)
972 else:
973 arta 1.13 print('NOOP=0', file=fout)
974
975 print(SUMRM_USER, file=fout)
976 if 'SUMRM_USER' in defs:
977 print('USER=' + defs['SUMRM_USER'], file=fout)
978 else:
979 print('USER=production', file=fout)
980
981 print(SUMRM_NORUN, file=fout)
982 # Default norun window is to have no such window. This can be accomplished by simply not providing either argument.
983 if 'SUMRM_NORUN_START' in defs or 'SUMRM_NORUN_STOP' in defs:
984 if 'SUMRM_NORUN_START' in defs:
985 print('NORUN_START=' + defs['SUMRM_NORUN_START'], file=fout)
986 else:
987 print('NORUN_START=0', file=fout)
988 if 'SUMRM_NORUN_STOP' in defs:
989 print('NORUN_STOP=' + defs['SUMRM_NORUN_STOP'], file=fout)
990 else:
991 print('NORUN_STOP=0', file=fout)
992 else:
993 print('# NORUN_START=0', file=fout)
994 arta 1.13 print('# NORUN_STOP=0', file=fout)
995
996 except OSError:
997 print('Unable to open sum_rm temporary configuration file ' + cFileTmp + 'for writing.', file=sys.stderr)
998 rv = bool(1)
999
1000 # If the content of the temporary file differs from the content of the existing configuration file, then overwrite
1001 # the original file. Otherwise, delete the temporary file
1002 if not rv:
1003 try:
1004 if filecmp.cmp(cFile, cFileTmp):
1005 # Files identical - delete temporary file
1006 try:
1007 os.remove(cFileTmp)
1008
1009 except OSError as exc:
1010 print('Unable to remove temporary file ' + exc.filename + '.', file=sys.stderr)
1011 print(exc.strerr, file=sys.stderr)
1012 else:
1013 # Replace original with temporary file
1014 try:
1015 arta 1.13 os.rename(cFileTmp, cFile)
1016
1017 except OSError as exc:
1018 print('Unable to update sum_rm configuration file ' + cFile + '.', file=sys.stderr)
1019 print(exc.strerr, file=sys.stderr)
1020 rv = bool(1)
1021 except OSError as exc:
1022 # One of the files doesn't exist.
1023 if exc.filename == cFile:
1024 # We are ok - there might be no configuration file yet.
1025 # Replace original with temporary file
1026 try:
1027 os.rename(cFileTmp, cFile)
1028
1029 except OSError as exc:
1030 print('Unable to update sum_rm configuration file ' + cFile + '.', file=sys.stderr)
1031 print(exc.strerr, file=sys.stderr)
1032 rv = bool(1)
1033 else:
1034 # There is a problem with the temp file - bail.
1035 print('Unable to update sum_rm configuration file ' + cFile + '.', file=sys.stderr)
1036 arta 1.13 print(exc.strerr, file=sys.stderr)
1037 rv = bool(1)
1038
1039 return rv
1040
|
1041 arta 1.25 def configureNet(cfgfile, cfile, mfile, pfile, pyfile, shfile, pCfile, pMfile, pRfile, pTfile, base, keymap, createSumRmCfg):
|
1042 arta 1.1 rv = bool(0)
1043
1044 defs = {}
1045 cDefs = list()
|
1046 arta 1.4 mDefsGen = list()
1047 mDefsMake = list()
1048 mDefsComps = list()
|
1049 arta 1.6 projCfg = list()
1050 projMkRules = list()
1051 projRules = list()
1052 projTarget = list()
|
1053 arta 1.1 perlConstSection = list()
1054 perlInitSection = list()
|
1055 arta 1.18 pyConstSection = list()
1056 pyInitSection = list()
|
1057 arta 1.25 shConstSection = list()
|
1058 arta 1.2 addenda = {}
1059
|
1060 arta 1.6 # There are three parameters that were not included in the original config.local parameter set, for some reason.
1061 # Due to this omission, then are not configurable, and must be set in the script.
|
1062 arta 1.2 addenda['a:USER'] = 'NULL'
1063 addenda['a:PASSWD'] = 'NULL'
1064 addenda['p:DSDS_SUPPORT'] = '0'
|
1065 arta 1.6
1066 # This parameter is not configurable. BUILD_TYPE is used to distinguish between a NetDRMS and an JSOC-SDP build.
|
1067 arta 1.2 addenda['a:BUILD_TYPE'] = 'NETDRMS' # Means a non-Stanford build. This will set two additional macros used by make:
1068 # __LOCALIZED_DEFS__ and NETDRMS_BUILD. The former is to support legacy code
1069 # which incorrectly used this macro, and the latter is for future use.
1070 # __LOCALIZED_DEFS__ is deprecated and should not be used in new code.
|
1071 arta 1.1
1072 try:
1073 with open(cfgfile, 'r') as fin:
|
1074 arta 1.3 # Process configuration parameters
|
1075 arta 1.26
1076 # Always create a Rules.mk and target.mk, even if no proj XML is provided. All builds should have the proj/example and
1077 # proj/cookbook directories. The required make information is in RULESPREFIX, TARGETPREFIX, and RULESSUFFIX. RULESSUFFIX
1078 # must be added after the xml has been parsed.
1079 projRules.extend(list(RULESPREFIX))
1080 projTarget.extend(list(TARGETPREFIX))
1081
|
1082 arta 1.25 rv = parseConfig(fin, keymap, addenda, defs, cDefs, mDefsGen, mDefsMake, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
|
1083 arta 1.6 if not rv:
|
1084 arta 1.26 projRules.extend(RULESSUFFIX)
1085
|
1086 arta 1.3 # Must add a parameter for the SUMS_MANAGER UID (for some reason). This must be done after the
1087 # config file is processed since an input to getMgrUIDLine() is one of the config file's
1088 # parameter values.
|
1089 arta 1.1 uidParam = {}
1090 rv = getMgrUIDLine(defs, uidParam)
1091 if rv == bool(0):
|
1092 arta 1.25 rv = parseConfig(None, keymap, uidParam, defs, cDefs, mDefsGen, None, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
|
1093 arta 1.3 # Configure the compiler-selection make variables.
|
1094 arta 1.6 if not rv:
|
1095 arta 1.4 rv = configureComps(defs, mDefsComps)
|
1096 arta 1.1
|
1097 arta 1.3 # Write out the parameter files.
|
1098 arta 1.6 if not rv:
|
1099 arta 1.25 rv = writeParamsFiles(base, cfile, mfile, pfile, pyfile, shfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
|
1100 arta 1.13
|
1101 arta 1.6 # Write out the project-specific make files (make_basic.mk, Rules.mk, and target.mk).
1102 if not rv:
1103 rv = writeProjFiles(pCfile, pMfile, pRfile, pTfile, projCfg, projMkRules, projRules, projTarget)
|
1104 arta 1.13
1105 # Write out the sum_rm.cfg file.
1106 if not rv and createSumRmCfg:
1107 rv = generateSumRmCfg(defs)
|
1108 arta 1.1 except IOError as exc:
|
1109 arta 1.6 print(exc.strerror, file=sys.stderr)
1110 print('Unable to read configuration file ' + cfgfile + '.', file=sys.stderr)
|
1111 arta 1.3 except Exception as exc:
|
1112 arta 1.16 if len(exc.args) >= 2:
|
1113 arta 1.14 type, msg = exc.args
1114 else:
1115 # re-raise the exception
1116 raise
1117
|
1118 arta 1.3 if type == 'unexpectedIccRet':
1119 print('icc -V returned this unexpected message:\n' + msg, file=sys.stderr)
1120 rv = bool(1)
1121 elif type == 'unexpectedGccRet':
1122 print('gcc -v returned this unexpected message:\n' + msg, file=sys.stderr)
1123 rv = bool(1)
1124 elif type == 'unexpectedIfortRet':
1125 print('ifort -V returned this unexpected message:\n' + msg, file=sys.stderr)
1126 rv = bool(1)
1127 elif type == 'unexpectedGfortranRet':
1128 print('gfortran -v returned this unexpected message:\n' + msg, file=sys.stderr)
1129 rv = bool(1)
1130 else:
1131 # re-raise the exception
1132 raise
|
1133 arta 1.1
|
1134 arta 1.2 return rv
|
1135 arta 1.1
|
1136 arta 1.25 def configureSdp(cfgfile, cfile, mfile, pfile, pyfile, shfile, pCfile, pMfile, pRfile, pTfile, base):
|
1137 arta 1.1 rv = bool(0)
1138
1139 defs = {}
1140 cDefs = list()
|
1141 arta 1.5 mDefsGen = list()
1142 mDefsMake = list()
|
1143 arta 1.6 projCfg = list()
1144 projMkRules = list()
1145 projRules = list()
1146 projTarget = list()
|
1147 arta 1.5 mDefsComps = list()
|
1148 arta 1.1 perlConstSection = list()
1149 perlInitSection = list()
|
1150 arta 1.18 pyConstSection = list()
1151 pyInitSection = list()
|
1152 arta 1.25 shConstSection = list()
|
1153 arta 1.18
|
1154 arta 1.2 addenda = {}
1155
|
1156 arta 1.6 # There are three parameters that were not included in the original config.local parameter set, for some reason.
1157 # Due to this omission, then are not configurable, and must be set in the script.
|
1158 arta 1.5 addenda['a:USER'] = 'NULL'
1159 addenda['a:PASSWD'] = 'NULL'
1160 addenda['p:DSDS_SUPPORT'] = '1'
|
1161 arta 1.6
1162 # This parameter is not configurable. BUILD_TYPE is used to distinguish between a NetDRMS and an JSOC-SDP build.
|
1163 arta 1.5 addenda['a:BUILD_TYPE'] = 'JSOC_SDP' # Means a Stanford build. This will set one additional macro used by make: JSOC_SDP_BUILD.
|
1164 arta 1.1
1165 try:
1166 with open(cfgfile, 'r') as fin:
|
1167 arta 1.26
1168 # Always create a Rules.mk and target.mk, even if no proj XML is provided. All builds should have the proj/example and
1169 # proj/cookbook directories. The required make information is in RULESPREFIX, TARGETPREFIX, and RULESSUFFIX. RULESSUFFIX
1170 # must be added after the xml has been parsed.
1171 projRules.extend(list(RULESPREFIX))
1172 projTarget.extend(list(TARGETPREFIX))
1173
|
1174 arta 1.25 rv = parseConfig(fin, None, addenda, defs, cDefs, mDefsGen, mDefsMake, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
|
1175 arta 1.6
1176 if not rv:
|
1177 arta 1.26 projRules.extend(RULESSUFFIX)
1178
|
1179 arta 1.2 # Must add a parameter for the SUMS_MANAGER UID (for some reason)
1180 uidParam = {}
1181 rv = getMgrUIDLine(defs, uidParam)
|
1182 arta 1.6 if not rv:
|
1183 arta 1.25 rv = parseConfig(None, None, uidParam, defs, cDefs, mDefsGen, None, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
|
1184 arta 1.6
|
1185 arta 1.5 # Configure the compiler-selection make variables.
|
1186 arta 1.6 if not rv:
|
1187 arta 1.5 rv = configureComps(defs, mDefsComps)
|
1188 arta 1.3
|
1189 arta 1.6 # Write out the parameter files.
1190 if not rv:
|
1191 arta 1.25 rv = writeParamsFiles(base, cfile, mfile, pfile, pyfile, shfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
|
1192 arta 1.6
1193 # Write out the project-specific make files (make_basic.mk, Rules.mk, and target.mk).
1194 if not rv:
1195 rv = writeProjFiles(pCfile, pMfile, pRfile, pTfile, projCfg, projMkRules, projRules, projTarget)
|
1196 arta 1.13
1197 # At Stanford, skip the creation of the sum_rm configuration file. config.local will still
1198 # have the SUMRM parameters, but they will not be used.
|
1199 arta 1.1 except IOError as exc:
|
1200 arta 1.6 print(exc.strerror, file=sys.stderr)
1201 print('Unable to read configuration file ' + cfgfile + '.', file=sys.stderr)
|
1202 arta 1.5 except Exception as exc:
|
1203 arta 1.17 if len(exc.args) >= 2:
1204 type = exc.args[0]
1205 else:
1206 # re-raise the exception
1207 raise
1208
|
1209 arta 1.5 if type == 'unexpectedIccRet':
|
1210 arta 1.6 msg = exc.args[1]
|
1211 arta 1.5 print('icc -V returned this unexpected message:\n' + msg, file=sys.stderr)
1212 rv = bool(1)
1213 elif type == 'unexpectedGccRet':
|
1214 arta 1.6 msg = exc.args[1]
|
1215 arta 1.5 print('gcc -v returned this unexpected message:\n' + msg, file=sys.stderr)
1216 rv = bool(1)
1217 elif type == 'unexpectedIfortRet':
|
1218 arta 1.6 msg = exc.args[1]
|
1219 arta 1.5 print('ifort -V returned this unexpected message:\n' + msg, file=sys.stderr)
1220 rv = bool(1)
1221 elif type == 'unexpectedGfortranRet':
|
1222 arta 1.6 msg = exc.args[1]
|
1223 arta 1.5 print('gfortran -v returned this unexpected message:\n' + msg, file=sys.stderr)
1224 rv = bool(1)
1225 else:
1226 # re-raise the exception
1227 raise
1228
|
1229 arta 1.2 return rv
1230
|
1231 arta 1.1 # Beginning of program
1232 rv = RET_SUCCESS
1233 net = bool(1)
1234
1235 # Parse arguments
1236 if __name__ == "__main__":
1237 optD = GetArgs(sys.argv[1:])
1238
1239 if not(optD is None):
1240 # Ensure we are configuring a DRMS tree
1241 cdir = os.path.realpath(os.getcwd())
1242 versfile = cdir + '/base/' + VERS_FILE
1243
1244 if not os.path.isfile(versfile):
1245 rv = RET_NOTDRMS
1246
1247 # Determine whether we are localizing a Stanford build, or a NetDRMS build. If configsdp.txt exists, then
1248 # it is a Stanford build, otherwise it is a NetDRMS build.
1249 if rv == RET_SUCCESS:
1250 stanfordFile = cdir + '/' + SDP_CFG
1251 if os.path.isfile(stanfordFile):
1252 arta 1.1 net = bool(0)
1253
1254 cfile = optD['dir'] + '/' + optD['base'] + '.h'
1255 mfile = optD['dir'] + '/' + optD['base'] + '.mk'
1256 pfile = optD['dir'] + '/' + optD['base'] + '.pm'
|
1257 arta 1.18 pyfile = optD['dir'] + '/' + optD['base'] + '.py'
|
1258 arta 1.25 shfile = optD['dir'] + '/' + optD['base'] + '.sh'
|
1259 arta 1.6 pCfile = optD['dir'] + '/configure'
1260 pMfile = optD['dir'] + '/make_basic.mk'
1261 pRfile = optD['dir'] + '/Rules.mk'
1262 pTfile = optD['dir'] + '/target.mk'
|
1263 arta 1.1
1264 if net:
1265 try:
1266 with open(NET_CFGMAP, 'r') as fin:
1267 regexpComm = re.compile(r"^\s*#")
1268 regexp = re.compile(r"^\s*(\S+)\s+(\w:\S+)")
1269 # Must map from config.local namespace to DRMS namespace (e.g., the names used for the C macros)
1270 keymap = {}
1271 for line in fin:
1272 matchobj = regexpComm.match(line)
1273 if not matchobj is None:
1274 # Skip comment line
1275 continue
1276
1277 matchobj = regexp.match(line)
1278 if not(matchobj is None):
1279 # We have a key-value line
1280 key = matchobj.group(1)
1281 val = matchobj.group(2)
1282 keymap[key] = val
1283 except OSError:
1284 arta 1.1 sys.stderr.write('Unable to read configuration map-file ' + NET_CFGMAP + '.')
1285 rv = bool(1)
1286
1287 # We also need to set the UID of the SUMS manager. We have the name of the
1288 # SUMS manager (it is in the configuration file)
|
1289 arta 1.25 configureNet(NET_CFG, cfile, mfile, pfile, pyfile, shfile, pCfile, pMfile, pRfile, pTfile, optD['base'], keymap, 'server' in optD)
|
1290 arta 1.1 else:
|
1291 arta 1.8 # A Stanford user can override the parameters in configsdp.txt by copying that file to config.local,
1292 # and then editing config.local. So, if config.local exists, use that.
1293 if os.path.isfile(cdir + '/' + NET_CFG):
|
1294 arta 1.25 configureSdp(NET_CFG, cfile, mfile, pfile, pyfile, shfile, pCfile, pMfile, pRfile, pTfile, optD['base'])
|
1295 arta 1.8 else:
|
1296 arta 1.25 configureSdp(SDP_CFG, cfile, mfile, pfile, pyfile, shfile, pCfile, pMfile, pRfile, pTfile, optD['base'])
|
Return to
localize.py
CVS log
Up to [Development] / JSOC