1 arta 1.29 #!/usr/bin/env python
|
2 arta 1.1
|
3 arta 1.13 # When run with the -s flag, localize.py configures the SUMS-server component of NetDRMS.
|
4 arta 1.1 import sys
5 import getopt
6 import re
|
7 arta 1.6 import os
8 import stat
|
9 arta 1.13 import filecmp
|
10 arta 1.1 from subprocess import check_output, CalledProcessError
|
11 arta 1.33 import shlex
|
12 arta 1.1
|
13 arta 1.24
|
14 arta 1.1 # Constants
15 VERS_FILE = 'jsoc_version.h'
16 SDP_CFG = 'configsdp.txt'
17 NET_CFG = 'config.local'
18 NET_CFGMAP = 'config.local.map'
19 RET_SUCCESS = 0
20 RET_NOTDRMS = 1
21
|
22 arta 1.2 PREFIX = """# This file was auto-generated by localize.py. Please do not edit it directly (running
23 # configure will run localize.py, which will then overwrite any edits manually performed).
24 """
|
25 arta 1.1
|
26 arta 1.2 C_PREFIX = """/* This file was auto-generated by localize.py. Please do not edit it directly (running
27 * configure will run localize.py, which will then overwrite any edits manually performed). */
|
28 arta 1.1 """
29
|
30 arta 1.2 PERL_BINPATH = '#!/usr/bin/perl\n'
31
32 PERL_INTIAL = """package drmsparams;
|
33 arta 1.28
|
34 arta 1.2 use warnings;
35 use strict;
36 """
37
38 PERL_FXNS_A = """sub new
|
39 arta 1.1 {
40 my($clname) = shift;
|
41 arta 1.28
42 my($self) =
|
43 arta 1.1 {
44 _paramsH => undef
45 };
|
46 arta 1.28
|
47 arta 1.1 bless($self, $clname);
|
48 arta 1.9 $self->{_paramsH} = {};
49 $self->initialize();
|
50 arta 1.28
|
51 arta 1.1 return $self;
52 }
|
53 arta 1.28
|
54 arta 1.1 sub DESTROY
55 {
56 my($self) = shift;
57 }
|
58 arta 1.2 """
59
60 PERL_FXNS_B = """sub get
61 {
62 my($self) = shift;
63 my($name) = shift;
64 my($rv);
|
65 arta 1.28
|
66 arta 1.2 if (exists($self->{_paramsH}->{$name}))
67 {
68 return $self->{_paramsH}->{$name};
69 }
70 else
71 {
72 return undef;
73 }
74 }
75 1;"""
|
76 arta 1.1
|
77 arta 1.30 PY_BINPATH = '#!/usr/bin/env python3\n'
78
79 PY_ALL = f"__all__ = [ 'DPError', 'DPMissingParameterError', 'DRMSParams' ]"
|
80 arta 1.29
81 PY_ERROR_CLASSES = """
82 class DPError(Exception):
83 def __init__(self):
84 self._msg = 'DRMS Parameter Error: '
85
86 class DPMissingParameterError(DPError):
87 def __init__(self, parameter):
88 super().__init__()
89 self._msg += 'missing DRMS parameter ' + parameter
90
91 def __str__(self):
92 return self._msg
93 """
|
94 arta 1.18
95 PY_FXNS_A = """
|
96 arta 1.23 class DRMSParams(object):
|
97 arta 1.18 def __init__(self):
98 self.params = {}
99 self.initialize()
|
100 arta 1.28
|
101 arta 1.18 def __del__(self):
102 del self.params
|
103 arta 1.28
|
104 arta 1.18 def initialize(self):
105 """
106
107 PY_FXNS_B = """ def get(self, name):
108 if name in self.params:
|
109 arta 1.20 return self.params[name]
|
110 arta 1.18 else:
111 return None
112 """
113
|
114 arta 1.29 PY_FXNS_C = """ def get_required(self, name):
115 try:
116 value = self.params[name]
117 except:
118 raise DPMissingParameterError(name)
119
120 return value
121
122 def getBool(self, name):
|
123 arta 1.22 if name in self.params:
124 return bool(self.params[name] == '1')
125 else:
126 return None
|
127 arta 1.29
128 def __getattr__(self, name):
129 # only called if object.__getattribute__(self, name) raises; and if that is true, then we want
130 # to look in self.params for it, and set the instance attribute if it does exist in self.params
131 if name in self.params:
132 attr = self.params[name]
133 self.__setattr__(name, attr)
134 else:
135 attr = None
136
137 return attr
138
139 def __setattr__(self, name, value):
140 # call neither __setattr__ nor __getattr__
141 try:
142 params = object.__getattr__(self, 'params')
143
144 # put into self.params dict, overwriting if necessary
145 params[name] = value
146 except:
147 pass
148 arta 1.29
149 # store in instance dict as well
150 object.__setattr__(self, name, value)
|
151 arta 1.22 """
152
|
153 arta 1.25 SH_BINPATH = '#!/bin/bash\n'
154
|
155 arta 1.22
|
156 arta 1.13 SUMRM_COMMENT = """# This is the configuration file for the sum_rm program. It was auto-generated by the DRMS master configure script.
157 # It controls the behavior of the sum_rm program, and is loaded each time sum_rm runs. To change the
158 # parameter values in this configuration file, modify config.local, then re-run configure. This configuration
159 # file will be updated only if parameters affecting it are modified. If such changes are made to config.local,
160 # please make sure that the sum_rm service is disabled while configure in running.
161 """
162
|
163 arta 1.28 SUMRM_DOC = """# sum_rm removes end-of-life SUMS data files to prevent disk-partitions from becomming 100% full.
|
164 arta 1.13 # sum_svc, the main SUMS service, starts this daemon when it is launched. Within an infinite loop, sum_rm sleeps
165 # for SLEEP number of seconds (defined below). When it awakes, it loads the sum_rm configuration file. For each SU,
166 # it then examines the 'effective_date' column within the sum_partn_alloc table of the SUMS database. It does so by
167 # sorting all SUs by effective date (this date is actually an expiration date - the SU is good until the end of that day),
168 # and then, starting with the SU with the oldest effective date, it deletes the SUs. It continues deleting SUs until one
169 # of three conditions is met:
170 #
171 # (a) The disk has at least PART_PERCENT_FREE percent free space.
172 # (b) All SUs have effective_date values in the future.
173 # (c) At least 600 SUs have been deleted (this is defined in the LIMIT statement in the file SUMLIB_RmDoX.pgc).
174 #
175 # After sum_rm stops deleteing SUs, it then sleeps for SLEEP seconds, completing the first iteration of the
176 # infinite loop.
177 """
178
179 SUMRM_PARTN_PERCENT_FREE = """
180 # This is the percentage at which all disk partitions are to be kept free.
181 # If not specified, this defaults to 3. For example, setting PART_PERCENT_FREE = 5 will allow all partitions to
182 # fill to 95% full. Dividing the number of unused blocks by the total number of blocks, and rounding up,
183 # will result in the number specified by PART_PERCENT_FREE.
184 #
185 arta 1.13 # NOTE : This behavior was previously controlled by the MAX_FREE_{n} family of parameters. {n} referred to the
186 # disk-partition number and the value of parameter MAX_FREE_{n} was the MINIMUM number of free MB in the
187 # partition [No clue why the word MAX was used]. As of NetDRMS 7.0, MAX_FREE_{n} has no effect."""
188
189 SUMRM_SLEEP = """
190 # The value is the number of seconds to sleep between iterations of the main loop in sum_rm."""
191
192 SUMRM_LOG = """
193 # The value is the log file (opened only at sum_rm startup; the sum_rm pid is appended to this file name)."""
194
195 SUMRM_MAIL = """
196 # The value is the email address of the recipient to be notified in case of a problem."""
197
198 SUMRM_NOOP = """
199 # If the value is set to anything other than 0, then sum_rm is rendered inactive. Otherwise, sum_rm is active."""
200
201 SUMRM_USER = """
202 # The value designates the linux user who is allowed to run sum_rm."""
203
204 SUMRM_NORUN = """
205 # This pair of paramters defines a window of time, during which sum_rm will become torpid - in this
206 arta 1.13 # window, sum_rm will not scan for SUs to delete, not will it delete any SUs. Each value is an integer, 0-23, that
207 # represents an hour of the day. For example, if NORUN_START=8 and NORUN_STOP=10, then between 8am and 10am
208 # local time, sum_rm will be dormant. To disable this behavior, set both parameters to the same value."""
|
209 arta 1.6
210 RULESPREFIX = """# Standard things
211 sp := $(sp).x
212 dirstack_$(sp) := $(d)
213 d := $(dir)
214 """
215
|
216 arta 1.33 RULESSUFFIX = """# Standard things
|
217 arta 1.6 d := $(dirstack_$(sp))
218 sp := $(basename $(sp))
219 """
220
|
221 arta 1.3 ICC_MAJOR = 9
222 ICC_MINOR = 0
223 GCC_MAJOR = 3
224 GCC_MINOR = 0
225 IFORT_MAJOR = 9
226 IFORT_MINOR = 0
227 GFORT_MAJOR = 4
228 GFORT_MINOR = 2
229
230
|
231 arta 1.1 # Read arguments
232 # d - localization directory
233 # b - base name of all parameter files (e.g., -b drmsparams --> drmsparams.h, drmsparams.mk, drmsparams.pm, etc.)
|
234 arta 1.13 # s - configure a NetDRMS server (i.e., SUMS server). Otherwise, do client configuration only. A server
235 # configuration will modify something that only the production user has access to. An example is the sum_rm.cfg
236 # file. To modify that file, the user running configure must be running configure on the SUMS-server host, and
237 # must have write permission on sum_rm.cfg.
|
238 arta 1.1 def GetArgs(args):
239 rv = bool(0)
240 optD = {}
|
241 arta 1.28
|
242 arta 1.1 try:
|
243 arta 1.13 opts, remainder = getopt.getopt(args, "hd:b:s",["dir=", "base="])
|
244 arta 1.1 except getopt.GetoptError:
245 print('Usage:')
246 print('localize.py [-h] -d <localization directory> -b <parameter file base>')
247 rv = bool(1)
|
248 arta 1.28
|
249 arta 1.1 if rv == bool(0):
250 for opt, arg in opts:
251 if opt == '-h':
252 print('localize.py [-h] -d <localization directory> -b <parameter file base>')
253 elif opt in ("-d", "--dir"):
254 regexp = re.compile(r"(\S+)/?")
255 matchobj = regexp.match(arg)
256 if matchobj is None:
257 rv = bool(1)
258 else:
259 optD['dir'] = matchobj.group(1)
260 elif opt in ("-b", "--base"):
261 optD['base'] = arg
|
262 arta 1.13 elif opt in ("-s"):
263 optD['server'] = ""
|
264 arta 1.1 else:
265 optD[opt] = arg
|
266 arta 1.28
|
267 arta 1.1 return optD
268
269 def createMacroStr(key, val, keyColLen, status):
270 if keyColLen < len(key):
271 status = bool(1)
272 return None
273 else:
274 nsp = keyColLen - len(key)
275 spaces = str()
276 for isp in range(nsp):
277 spaces += ' '
278 status = bool(0)
279 return '#define ' + key + spaces + val + '\n'
280
281 def createPerlConst(key, val, keyColLen, status):
282 if keyColLen < len(key):
283 status = bool(1)
284 return None
285 else:
286 nsp = keyColLen - len(key)
287 spaces = str()
288 arta 1.1 for isp in range(nsp):
289 spaces += ' '
290 status = bool(0)
291 return 'use constant ' + key + ' => ' + spaces + val + ';\n'
292
|
293 arta 1.18 def createPyConst(key, val, keyColLen, status):
294 if keyColLen < len(key):
295 status = bool(1)
296 return None
297 else:
298 nsp = keyColLen - len(key)
299 spaces = str()
300 for isp in range(nsp):
301 spaces += ' '
302 status = bool(0)
303 return key + ' = ' + spaces + val + '\n'
|
304 arta 1.28
|
305 arta 1.25 def createShConst(key, val, status):
306 status = bool(0)
307 return key + '=' + val + '\n'
|
308 arta 1.28
|
309 arta 1.18
|
310 arta 1.3 def isSupportedPlat(plat):
311 regexp = re.compile(r"\s*(^x86_64|^ia32|^ia64|^avx)", re.IGNORECASE)
312 matchobj = regexp.match(plat);
|
313 arta 1.28
|
314 arta 1.3 if not matchobj is None:
315 return bool(1);
316 else:
317 return bool(0);
318
|
319 arta 1.6 def processMakeParam(mDefs, key, val, platDict, machDict):
|
320 arta 1.3 varMach = None
321 regexp = re.compile(r"(\S+):(\S+)")
322 matchobj = regexp.match(key)
323 if not matchobj is None:
324 varName = matchobj.group(1)
325 varMach = matchobj.group(2)
326 else:
327 varName = key
|
328 arta 1.28
|
329 arta 1.3 varValu = val
|
330 arta 1.28
|
331 arta 1.3 if varMach is None:
332 mDefs.extend(list('\n' + varName + ' = ' + varValu))
333 else:
334 if isSupportedPlat(varMach):
335 # The guard will compare varValu to $JSOC_MACHINE.
336 if not varMach in platDict:
337 platDict[varMach] = {}
338 platDict[varMach][varName] = varValu
339 else:
340 # The guard will compare varValu to $MACHINETYPE (this is just the hostname of the machine on which localize.py is running).
341 if not varMach in machDict:
342 machDict[varMach] = {}
343 machDict[varMach][varName] = varValu
344
|
345 arta 1.25 def processParam(cfgfile, line, regexpQuote, regexp, keymap, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection, platDict, machDict, section):
|
346 arta 1.1 status = 0
|
347 arta 1.33 parameter_added = {}
348 keyCfgSp = None
349 val = None
|
350 arta 1.28
|
351 arta 1.1 if ''.join(section) == 'defs' or not cfgfile:
|
352 arta 1.33 if type(line) is str:
353 matchobj = regexp.match(line)
354
355 if not matchobj is None:
356 # We have a key-value line
357 keyCfgSp = matchobj.group(1)
358 val = matchobj.group(2)
359 print(f'herer, key={keyCfgSp}, val={val}')
360 else:
361 keyCfgSp, val = list(line.items())
362
363 if keyCfgSp is not None and val is not None:
|
364 arta 1.1 # We have a key-value line
365 keyCfgSp = matchobj.group(1)
366 val = matchobj.group(2)
|
367 arta 1.28
|
368 arta 1.1 # Must map the indirect name to the actual name
369 if keymap:
370 # Map to actual name only if the keymap is not empty (which signifies NA).
371 if keyCfgSp in keymap:
372 key = keymap[keyCfgSp]
373 elif keyCfgSp == 'LOCAL_CONFIG_SET' or keyCfgSp == 'DRMS_SAMPLE_NAMESPACE':
|
374 arta 1.6 # Ignore parameters that are not useful and shouldn't have been there in the first place. But
375 # they have been released to the world, so we have to account for them.
|
376 arta 1.33 return None
|
377 arta 1.1 elif not cfgfile:
378 # Should not be doing mapping for addenda
379 key = keyCfgSp
380 else:
381 raise Exception('badKeyMapKey', keyCfgSp)
382 else:
383 key = keyCfgSp
|
384 arta 1.28
|
385 arta 1.1 matchobj = regexpQuote.match(key)
386 if not matchobj is None:
387 quote = matchobj.group(1)
388 key = matchobj.group(2)
389
390 # master defs dictionary
391 defs[key] = val
|
392 arta 1.28
|
393 arta 1.33 parameter_added[key] = val
394
|
395 arta 1.1 # C header file
396 if quote == "q":
397 # Add double-quotes
398 cDefs.extend(list(createMacroStr(key, '"' + val + '"', 40, status)))
399 elif quote == "p":
400 # Add parentheses
401 cDefs.extend(list(createMacroStr(key, '(' + val + ')', 40, status)))
402 elif quote == "a":
403 # Leave as-is
404 cDefs.extend(list(createMacroStr(key, val, 40, status)))
405 else:
406 # Unknown quote type
407 raise Exception('badQuoteQual', key)
|
408 arta 1.28
|
409 arta 1.1 if status:
410 raise Exception('paramNameTooLong', key)
|
411 arta 1.28
|
412 arta 1.1 # Make file - val should never be quoted; just use as is
|
413 arta 1.4 mDefsGen.extend(list('\n' + key + ' = ' + val))
|
414 arta 1.28
|
415 arta 1.1 # Perl file - val should ALWAYS be single-quote quoted
416 # Save const info to a string
417 perlConstSection.extend(list(createPerlConst(key, "'" + val + "'", 40, status)))
|
418 arta 1.28
|
419 arta 1.18 # Python file
420 pyConstSection.extend(list(createPyConst(key, "'" + val + "'", 40, status)))
|
421 arta 1.28
|
422 arta 1.25 # Shell source file
423 shConstSection.extend(list(createShConst(key, "'" + val + "'", status)))
|
424 arta 1.28
|
425 arta 1.1 if status:
426 raise Exception('paramNameTooLong', key)
|
427 arta 1.28
|
428 arta 1.1 # Save initialization information as a string. Now that we've defined
|
429 arta 1.28 # constants (the names of which are the parameter names)
|
430 arta 1.1 # we can refer to those in the init section. The key variable holds the
431 # name of the constant.
|
432 arta 1.9 perlInitSection.extend(list("\n $self->{_paramsH}->{'" + key + "'} = " + key + ';'))
|
433 arta 1.28
|
434 arta 1.18 # The amount of indenting matters! This is Python.
435 pyInitSection.extend(list(" self.params['" + key + "'] = " + key + '\n'))
|
436 arta 1.1 else:
437 # No quote qualifier
438 raise Exception('missingQuoteQual', key)
|
439 arta 1.3 elif ''.join(section) == 'make' and cfgfile:
440 # Configure the remaining make variables defined in the __MAKE__ section of the configuration file. Third-party
441 # library make variables are specified in the __MAKE__ section.
442 matchobj = regexp.match(line)
443 if not matchobj is None:
444 # We have a key-value line
445 key = matchobj.group(1)
446 val = matchobj.group(2)
447
448 # This information is for making make variables only. We do not need to worry about quoting any values
449 defs[key] = val
|
450 arta 1.4 processMakeParam(mDefsMake, key, val, platDict, machDict)
|
451 arta 1.28
|
452 arta 1.33 return parameter_added
|
453 arta 1.1
454 # We have some extraneous line or a newline - ignore.
455
|
456 arta 1.33 def process_project_repos(project_includes):
457 print(f'[ process_project_repos ]')
458 error = False
459 ordered_files = []
460
461 # iterate through all proj subdirectories
462 for subdirectory in os.listdir('proj'):
463 stripped_subdirectory = subdirectory.strip()
464 path = os.path.join('proj', stripped_subdirectory)
|
465 arta 1.28
|
466 arta 1.33 # skip any dir in proj that does not have a Rules.mk file
467 if os.path.isfile(os.path.join(path, 'Rules.mk')):
468 if os.path.isdir(path):
469 ordered_files.append(stripped_subdirectory)
|
470 arta 1.28
|
471 arta 1.33 ordered_files.sort()
|
472 arta 1.28
|
473 arta 1.33 for stripped_subdirectory in ordered_files:
474 project_includes.append(f'dir := $(d)/{stripped_subdirectory}')
|
475 arta 1.6
|
476 arta 1.33 return error
|
477 arta 1.6
|
478 arta 1.33 def determineSection(line, regexpStyle, regexpDefs, regexpMake):
|
479 arta 1.27 matchobj = regexpStyle.match(line)
480 if matchobj:
481 return 'style'
482
|
483 arta 1.6 matchobj = regexpDefs.match(line)
484 if not matchobj is None:
485 return 'defs'
|
486 arta 1.28
|
487 arta 1.6 matchobj = regexpMake.match(line)
488 if not matchobj is None:
489 return 'make'
|
490 arta 1.28
|
491 arta 1.6 return None
492
|
493 arta 1.1 # defs is a dictionary containing all parameters (should they be needed in this script)
|
494 arta 1.33 def parseConfig(fin, keymap, addenda, defs, cDefs, mDefsGen, mDefsMake, project_includes, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection):
495 error = False
496
497 print(f'addenda is {str(addenda)}')
|
498 arta 1.28
|
499 arta 1.1 # Open required config file (config.local)
500 try:
501 # Examine each line, looking for key=value pairs.
|
502 arta 1.27 regexpStyle = re.compile(r"^__STYLE__")
|
503 arta 1.1 regexpDefs = re.compile(r"^__DEFS__")
504 regexpMake = re.compile(r"^__MAKE__")
505 regexpComm = re.compile(r"^\s*#")
|
506 arta 1.6 regexpSp = re.compile(r"^s*$")
|
507 arta 1.1 regexpQuote = re.compile(r"^\s*(\w):(.+)")
|
508 arta 1.6 regexpCustMkBeg = re.compile(r"^_CUST_")
509 regexpCustMkEnd = re.compile(r"^_ENDCUST_")
510 regexpDiv = re.compile(r"^__")
|
511 arta 1.11 regexp = re.compile(r"^\s*(\S+)\s+(\S.*)")
|
512 arta 1.27
513 ignoreKeymap = False
|
514 arta 1.3 platDict = {}
515 machDict = {}
|
516 arta 1.28
|
517 arta 1.1 # Process the parameters in the configuration file
518 if not fin is None:
519 for line in fin:
|
520 arta 1.6 matchobj = regexpComm.match(line)
521 if not matchobj is None:
522 # Skip comment line
523 continue
|
524 arta 1.28
|
525 arta 1.6 matchobj = regexpSp.match(line)
526 if not matchobj is None:
527 # Skip whitespace line
528 continue
|
529 arta 1.28
|
530 arta 1.33 newSection = determineSection(line, regexpStyle, regexpDefs, regexpMake)
|
531 arta 1.6 if not newSection is None:
532 section = newSection
|
533 arta 1.28
|
534 arta 1.27 if not section:
535 raise Exception('invalidConfigFile', 'line ' + line.strip() + ' is not in any section')
|
536 arta 1.28
|
537 arta 1.27 if section == 'style':
538 # if the config.local file has new in the __STYLE__ section, then ignore the keymap and treat config.local like configsdp.txt;
539 # do not map from NetDRMS config.local parameter names to configsdp.txt names
540 for line in fin:
541 matchobj = regexpDiv.match(line)
542 if matchobj:
543 break;
544 if line.strip(' \n').lower() == 'new' and keymap:
545 ignoreKeymap = True
546
|
547 arta 1.33 newSection = determineSection(line, regexpStyle, regexpDefs, regexpMake)
|
548 arta 1.27 if not newSection is None:
549 section = newSection
550 continue
551 elif section == 'make':
|
552 arta 1.10
|
553 arta 1.28 # There are some blocks of lines in the __MAKE__ section that must be copied ver batim to the output make file.
|
554 arta 1.6 # The blocks are defined by _CUST_/_ENDCUST_ tags.
555 matchobj = regexpCustMkBeg.match(line)
|
556 arta 1.28
|
557 arta 1.6 if not matchobj is None:
558 mDefsMake.extend(list('\n'))
559 for line in fin:
560 matchobj = regexpCustMkEnd.match(line)
561 if not matchobj is None:
562 break;
563 mDefsMake.extend(list(line))
|
564 arta 1.33 newSection = determineSection(line, regexpStyle, regexpDefs, regexpMake)
|
565 arta 1.6 if not newSection is None:
566 section = newSection
567 continue
568 # Intentional fall through to next if statement
569 if section == 'defs' or section == 'make':
570 iscfg = bool(1)
|
571 arta 1.27 if ignoreKeymap:
572 keymapActual = None
573 else:
574 keymapActual = keymap
575 ppRet = processParam(iscfg, line, regexpQuote, regexp, keymapActual, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection, platDict, machDict, section)
|
576 arta 1.6 else:
|
577 arta 1.10 # Unknown section
|
578 arta 1.6 raise Exception('unknownSection', section)
|
579 arta 1.1 except Exception as exc:
|
580 arta 1.17 if len(exc.args) >= 2:
581 msg = exc.args[0]
582 else:
583 # re-raise the exception
584 raise
|
585 arta 1.28
|
586 arta 1.27 if msg == 'invalidConfigFile':
587 violator = exc.args[1]
588 print(violator, file=sys.stderr)
|
589 arta 1.33 error = True
|
590 arta 1.27 elif msg == 'badKeyMapKey':
|
591 arta 1.1 # If we are here, then there was a non-empty keymap, and the parameter came from
592 # the configuration file.
|
593 arta 1.6 violator = exc.args[1]
|
594 arta 1.17 print('Unknown parameter name ' + "'" + violator + "'" + ' in ' + fin.name + '.', file=sys.stderr)
|
595 arta 1.33 error = True
|
596 arta 1.1 elif msg == 'badQuoteQual':
597 # The bad quote qualifier came from the configuration file, not the addenda, since
598 # we will have fixed any bad qualifiers in the addenda (which is populated by code).
|
599 arta 1.6 violator = exc.args[1]
|
600 arta 1.17 print('Unknown quote qualifier ' + "'" + violator + "'" + ' in ' + fin.name + '.', file=sys.stderr)
|
601 arta 1.33 error = True
|
602 arta 1.1 elif msg == 'missingQuoteQual':
|
603 arta 1.6 violator = exc.args[1]
|
604 arta 1.17 print('Missing quote qualifier for parameter ' + "'" + violator + "'" + ' in ' + fin.name + '.', file=sys.stderr)
|
605 arta 1.33 error = True
|
606 arta 1.1 elif msg == 'paramNameTooLong':
|
607 arta 1.6 violator = exc.args[1]
|
608 arta 1.1 print('Macro name ' + "'" + violator + "' is too long.", file=sys.stderr)
|
609 arta 1.33 error = True
|
610 arta 1.6 elif msg == 'unknownSection':
611 violator = exc.args[1]
612 print('Unknown section ' + "'" + violator + "' in configuration file.", file=sys.stderr)
|
613 arta 1.33 error = True
|
614 arta 1.1 else:
615 # re-raise the exception
616 raise
617
|
618 arta 1.33 if not error:
619 if project_includes is not None:
620 error = process_project_repos(project_includes)
|
621 arta 1.6
|
622 arta 1.28 # Process addenda - these are parameters that are not configurable and must be set in the
|
623 arta 1.6 # NetDRMS build.
|
624 arta 1.33 if not error:
|
625 arta 1.6 iscfg = bool(0)
|
626 arta 1.33 for key, val in addenda.items():
627 item = f'{key} {val}'
628 print(f'addenda item {item}')
|
629 arta 1.27 if ignoreKeymap:
630 keymapActual = None
631 else:
632 keymapActual = keymap
633 ppRet = processParam(iscfg, item, regexpQuote, regexp, keymapActual, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection, platDict, machDict, 'defs')
|
634 arta 1.6
|
635 arta 1.4 # Put information collected in platDict and machDict into mDefs. Must do this here, and not in processParam, since
636 # we need to parse all platform-specific make variables before grouping them into platform categories.
|
637 arta 1.33 if not error:
|
638 arta 1.3 for plat in platDict:
|
639 arta 1.4 mDefsMake.extend(list('\nifeq ($(JSOC_MACHINE), linux_' + plat.lower() + ')'))
|
640 arta 1.3 for var in platDict[plat]:
|
641 arta 1.4 mDefsMake.extend(list('\n' + var + ' = ' + platDict[plat][var]))
642 mDefsMake.extend(list('\nendif\n'))
|
643 arta 1.28
|
644 arta 1.33 if not error:
|
645 arta 1.3 for mach in machDict:
|
646 arta 1.4 mDefsMake.extend(list('\nifeq ($(MACHTYPE), ' + mach + ')'))
|
647 arta 1.7 for var in machDict[mach]:
648 mDefsMake.extend(list('\n' + var + ' = ' + machDict[mach][var]))
|
649 arta 1.4 mDefsMake.extend(list('\nendif\n'))
|
650 arta 1.33 return error
|
651 arta 1.1
|
652 arta 1.33 def getMgrUIDLine(sums_manager, uidParam):
653 error = False
|
654 arta 1.28
|
655 arta 1.33 if sums_manager and len(sums_manager) > 0:
656 cmd = [ 'id', '-u', shlex.quote(sums_manager) ]
657 try:
658 ret = check_output(cmd)
659 uidParam['q:SUMS_MANAGER_UID'] = ret.decode("utf-8")
660 except ValueError:
661 print('Unable to run cmd: ' + cmd + '.')
662 error = True
663 except CalledProcessError:
664 print('Command ' + "'" + cmd + "'" + ' ran improperly.')
665 error = True
|
666 arta 1.1
|
667 arta 1.33 return error
|
668 arta 1.1
|
669 arta 1.3 def isVersion(maj, min, majDef, minDef):
670 res = 0
|
671 arta 1.28
|
672 arta 1.3 if maj > majDef or (maj == majDef and min >= minDef):
673 res = 1
|
674 arta 1.28
|
675 arta 1.3 return res
676
677 def configureComps(defs, mDefs):
678 rv = bool(0)
|
679 arta 1.6 autoConfig = bool(1)
680
681 if 'AUTOSELCOMP' in defs:
682 autoConfig = (not defs['AUTOSELCOMP'] == '0')
|
683 arta 1.28
|
684 arta 1.3 if autoConfig:
685 hasicc = bool(0)
686 hasgcc = bool(0)
687 hasifort = bool(0)
688 hasgfort = bool(0)
|
689 arta 1.28
|
690 arta 1.3 # Try icc.
|
691 arta 1.12 cmd = 'icc --version 2>&1'
|
692 arta 1.3 try:
693 ret = check_output(cmd, shell=True)
694 ret = ret.decode("utf-8")
695 except CalledProcessError:
696 print('Command ' + "'" + cmd + "'" + ' ran improperly.')
697 rv = bool(1)
|
698 arta 1.28
|
699 arta 1.6 if not rv:
|
700 arta 1.12 regexp = re.compile(r"\s*\S+\s+\S+\s+(\d+)[.](\d+)", re.DOTALL)
|
701 arta 1.3 matchobj = regexp.match(ret)
702 if matchobj is None:
703 raise Exception('unexpectedIccRet', ret)
704 else:
705 major = matchobj.group(1)
706 minor = matchobj.group(2)
707 if isVersion(int(major), int(minor), ICC_MAJOR, ICC_MINOR):
708 hasicc = bool(1)
|
709 arta 1.19
|
710 arta 1.3 # Try gcc.
711 if not hasicc:
|
712 arta 1.19 rv = bool(0)
|
713 arta 1.3 cmd = 'gcc -v 2>&1'
714 try:
715 ret = check_output(cmd, shell=True)
716 ret = ret.decode("utf-8")
717 except CalledProcessError:
718 print('Command ' + "'" + cmd + "'" + ' ran improperly.')
719 rv = bool(1)
720
|
721 arta 1.19 if not rv:
|
722 arta 1.3 regexp = re.compile(r".+gcc\s+version\s+(\d+)\.(\d+)", re.DOTALL)
723 matchobj = regexp.match(ret)
724 if matchobj is None:
725 raise Exception('unexpectedGccRet', ret)
726 else:
727 major = matchobj.group(1)
728 minor = matchobj.group(2)
729 if isVersion(int(major), int(minor), GCC_MAJOR, GCC_MINOR):
730 hasgcc = bool(1)
731
732 # Try ifort.
|
733 arta 1.19 rv = bool(0)
|
734 arta 1.12 cmd = 'ifort --version 2>&1'
|
735 arta 1.3 try:
736 ret = check_output(cmd, shell=True)
737 ret = ret.decode("utf-8")
738 except CalledProcessError:
739 print('Command ' + "'" + cmd + "'" + ' ran improperly.')
740 rv = bool(1)
741
|
742 arta 1.6 if not rv:
|
743 arta 1.12 regexp = re.compile(r"\s*\S+\s+\S+\s+(\d+)\.(\d+)", re.DOTALL)
|
744 arta 1.3 matchobj = regexp.match(ret)
745 if matchobj is None:
746 raise Exception('unexpectedIfortRet', ret)
747 else:
748 major = matchobj.group(1)
749 minor = matchobj.group(2)
750 if isVersion(int(major), int(minor), IFORT_MAJOR, IFORT_MINOR):
751 hasifort = bool(1)
|
752 arta 1.28
|
753 arta 1.3 # Try gfortran
754 if not hasifort:
|
755 arta 1.19 rv = bool(0)
|
756 arta 1.3 cmd = 'gfortran -v 2>&1'
757 try:
758 ret = check_output(cmd, shell=True)
759 ret = ret.decode("utf-8")
760 except CalledProcessError:
761 print('Command ' + "'" + cmd + "'" + ' ran improperly.')
762 rv = bool(1)
763
|
764 arta 1.19 if not rv:
|
765 arta 1.3 regexp = re.compile(r".+gcc\s+version\s+(\d+)\.(\d+)", re.DOTALL)
766 matchobj = regexp.match(ret)
767 if matchobj is None:
768 raise Exception('unexpectedGfortranRet', ret)
769 else:
770 major = matchobj.group(1)
771 minor = matchobj.group(2)
772 if isVersion(int(major), int(minor), GFORT_MAJOR, GFORT_MINOR):
773 hasgfort = bool(1)
|
774 arta 1.28
|
775 arta 1.19 # Append the compiler make variables to the make file
776 rv = bool(0)
|
777 arta 1.28
|
778 arta 1.3 if not hasicc and not hasgcc:
779 print('Fatal error: Acceptable C compiler not found! You will be unable to build the DRMS library.', file=sys.stderr)
|
780 arta 1.32 rv = bool(0) # Art - don't bail, we might be using drmsparams.py and not care about the rest of DRMS
|
781 arta 1.3 elif hasicc:
782 mDefs.extend(list('\nCOMPILER = icc'))
|
783 arta 1.28 # mDefs.extend(list('\nICC_VERSION = blah'))
|
784 arta 1.3 else:
785 mDefs.extend(list('\nCOMPILER = gcc'))
786
787 if not hasifort and not hasgfort:
788 print('Warning: Acceptable Fortran compiler not found! Fortran interface will not be built, and you will be unable to build Fortran modules.', file=sys.stderr)
789 elif hasifort:
790 mDefs.extend(list('\nFCOMPILER = ifort'))
791 else:
792 mDefs.extend(list('\nFCOMPILER = gfortran'))
|
793 arta 1.28
|
794 arta 1.3 # Environment overrides. These get written, regardless of the disposition of auto-configuration.
|
795 arta 1.9 mDefs.extend(list('\nifneq ($(JSOC_COMPILER),)\n COMPILER = $(JSOC_COMPILER)\nendif'))
796 mDefs.extend(list('\nifneq ($(JSOC_FCOMPILER),)\n FCOMPILER = $(JSOC_FCOMPILER)\nendif'))
|
797 arta 1.3
798 return rv
799
|
800 arta 1.25 def writeParamsFiles(base, cfile, mfile, pfile, pyfile, shfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection):
|
801 arta 1.1 rv = bool(0)
|
802 arta 1.4
803 # Merge mDefsGen, mDefsMake, and mDefsComps into a single string with compiler configuration first, general parameters next, then
804 # make-specific make variables (e.g., third-party library information) last.
805 mDefs = '\n# Compiler Selection\n' + ''.join(mDefsComps) + '\n\n# General Parameters\n' + ''.join(mDefsGen) + '\n\n# Parameters to Configure make\n' + ''.join(mDefsMake)
|
806 arta 1.28
|
807 arta 1.1 try:
|
808 arta 1.25 with open(cfile, 'w') as cout, open(mfile, 'w') as mout, open(pfile, 'w') as pout, open(pyfile, 'w') as pyout, open(shfile, 'w') as shout:
|
809 arta 1.1 # C file of macros
|
810 arta 1.2 print(C_PREFIX, file=cout)
811 print('/* This file contains a set of preprocessor macros - one for each configuration parameter. */\n', file=cout)
|
812 arta 1.1 buf = '__' + base.upper() + '_H'
813 print('#ifndef ' + buf, file=cout)
814 print('#define ' + buf, file=cout)
815 print(''.join(cDefs), file=cout)
816 print('#endif', file=cout)
|
817 arta 1.28
|
818 arta 1.1 # Make file of make variables
|
819 arta 1.2 print(PREFIX, file=mout)
|
820 arta 1.4 print('# This file contains a set of make-variable values. The first section contains compiler-selection variables, the second contains general configuration variables, and the third section contains variables that configure how make is run.', file=mout)
821 print(mDefs, file=mout)
|
822 arta 1.28
|
823 arta 1.2 # Perl module
824 print(PERL_BINPATH, file=pout)
825 print(PREFIX, file=pout)
826 print('# This file contains a set of constants - one for each configuration parameter.\n', file=pout)
827 print(PERL_INTIAL, file=pout)
|
828 arta 1.1 print(''.join(perlConstSection), file=pout)
|
829 arta 1.2 print(PERL_FXNS_A, file=pout)
|
830 arta 1.1 print('sub initialize', file=pout)
831 print('{', file=pout)
|
832 arta 1.2 print(' my($self) = shift;', file=pout, end='')
|
833 arta 1.1 print('', file=pout)
834 print(''.join(perlInitSection), file=pout)
|
835 arta 1.2 print('}\n', file=pout)
836 print(PERL_FXNS_B, file=pout)
|
837 arta 1.18
838 # Python module
839 print(PY_BINPATH, file=pyout)
840 print(PREFIX, file=pyout)
841 print('# This file contains a set of constants - one for each configuration parameter.\n', file=pyout)
|
842 arta 1.30 print(PY_ALL, file=pyout)
|
843 arta 1.18 print(''.join(pyConstSection), file=pyout)
|
844 arta 1.29
845 print(PY_ERROR_CLASSES, file=pyout)
|
846 arta 1.18 print(PY_FXNS_A, file=pyout, end='')
847 print(''.join(pyInitSection), file=pyout)
848 print(PY_FXNS_B, file=pyout)
|
849 arta 1.22 print(PY_FXNS_C, file=pyout)
|
850 arta 1.28
|
851 arta 1.25 # Shell (bash) source file
852 print(SH_BINPATH, file=shout)
853 print(PREFIX, file=shout)
854 print('# This file contains a set of variable assignments - one for each configuration parameter.\n', file=shout)
855 print(''.join(shConstSection), file=shout)
|
856 arta 1.18
|
857 arta 1.1 except IOError as exc:
|
858 arta 1.6 type, value, traceback = sys.exc_info()
859 print(exc.strerror, file=sys.stderr)
|
860 arta 1.28 print('Unable to open ' + "'" + value.filename + "'.", file=sys.stderr)
|
861 arta 1.6 rv = bool(1)
862
863 return rv
864
|
865 arta 1.33 def write_project_includes(project_includes_file, project_includes):
866 error = False
|
867 arta 1.6
|
868 arta 1.33 if project_includes_file is not None:
869 try:
870 with open(project_includes_file, 'w') as file_out:
871 # Rules.mk
872 print(PREFIX, file=file_out)
873 print(RULESPREFIX, file=file_out)
|
874 arta 1.28
|
875 arta 1.33 for dir_variable in project_includes:
876 print(dir_variable, file=file_out)
877 print(f'-include $(SRCDIR)/$(dir)/Rules.mk', file=file_out)
|
878 arta 1.28
|
879 arta 1.33 print('', file=file_out)
880 print(RULESSUFFIX, file=file_out)
|
881 arta 1.1
|
882 arta 1.33 except IOError as exc:
883 print(f'unable to open {project_includes_file} for writing ({str(exc)})', file=sys.stderr)
884 error = True
|
885 arta 1.6
|
886 arta 1.33 return error
|
887 arta 1.3
|
888 arta 1.13 def generateSumRmCfg(defs):
889 rv = bool(0)
890 # ACK! Remember that Rick renamed these parameters. The ones in config.local are the aliases - do not use those.
891 # Use the ones that those map to (defined in config.local.map).
892 cFileTmp = defs['SUMLOG_BASEDIR'] + '/' + '.sum_rm.cfg.tmp'
893 cFile = defs['SUMLOG_BASEDIR'] + '/' + 'sum_rm.cfg'
894
895 # Write a temporary file sum_rm configuration file.
896 try:
897 with open(cFileTmp, 'w') as fout:
898 # Print comment at the top of the configuration file.
899 print(SUMRM_COMMENT, file=fout)
900 print(SUMRM_DOC, file=fout)
901 print(SUMRM_PARTN_PERCENT_FREE, file=fout)
902 if 'SUMRM_PART_PERCENT_FREE' in defs:
903 print('PART_PERCENT_FREE=' + defs['SUMRM_PART_PERCENT_FREE'], file=fout)
904 else:
905 print('PART_PERCENT_FREE=3', file=fout)
906
907 print(SUMRM_SLEEP, file=fout)
908 if 'SUMRM_SLEEP' in defs:
909 arta 1.13 print('SLEEP=' + defs['SUMRM_SLEEP'], file=fout)
910 else:
911 print('SLEEP=300', file=fout)
912
913 print(SUMRM_LOG, file=fout)
914 if 'SUMRM_LOG' in defs:
915 print('LOG=' + defs['SUMRM_LOG'], file=fout)
916 else:
917 print('LOG=/tmp/sum_rm.log', file=fout)
918
919 print(SUMRM_MAIL, file=fout)
920 # No default for mail - don't send nothing to nobody unless the operator has asked for notifications.
921 if 'SUMRM_MAIL' in defs:
922 print('MAIL=' + defs['SUMRM_MAIL'], file=fout)
923 else:
924 print('# MAIL=president@whitehouse.gov', file=fout)
925
926 print(SUMRM_NOOP, file=fout)
927 if 'SUMRM_NOOP' in defs:
928 print('NOOP=' + defs['SUMRM_NOOP'], file=fout)
929 else:
930 arta 1.13 print('NOOP=0', file=fout)
931
932 print(SUMRM_USER, file=fout)
933 if 'SUMRM_USER' in defs:
934 print('USER=' + defs['SUMRM_USER'], file=fout)
935 else:
936 print('USER=production', file=fout)
937
938 print(SUMRM_NORUN, file=fout)
939 # Default norun window is to have no such window. This can be accomplished by simply not providing either argument.
940 if 'SUMRM_NORUN_START' in defs or 'SUMRM_NORUN_STOP' in defs:
941 if 'SUMRM_NORUN_START' in defs:
942 print('NORUN_START=' + defs['SUMRM_NORUN_START'], file=fout)
943 else:
944 print('NORUN_START=0', file=fout)
945 if 'SUMRM_NORUN_STOP' in defs:
946 print('NORUN_STOP=' + defs['SUMRM_NORUN_STOP'], file=fout)
947 else:
948 print('NORUN_STOP=0', file=fout)
949 else:
950 print('# NORUN_START=0', file=fout)
951 arta 1.13 print('# NORUN_STOP=0', file=fout)
|
952 arta 1.28
|
953 arta 1.13 except OSError:
954 print('Unable to open sum_rm temporary configuration file ' + cFileTmp + 'for writing.', file=sys.stderr)
955 rv = bool(1)
956
957 # If the content of the temporary file differs from the content of the existing configuration file, then overwrite
958 # the original file. Otherwise, delete the temporary file
959 if not rv:
960 try:
961 if filecmp.cmp(cFile, cFileTmp):
962 # Files identical - delete temporary file
963 try:
964 os.remove(cFileTmp)
|
965 arta 1.28
|
966 arta 1.13 except OSError as exc:
967 print('Unable to remove temporary file ' + exc.filename + '.', file=sys.stderr)
968 print(exc.strerr, file=sys.stderr)
969 else:
970 # Replace original with temporary file
971 try:
972 os.rename(cFileTmp, cFile)
973
974 except OSError as exc:
975 print('Unable to update sum_rm configuration file ' + cFile + '.', file=sys.stderr)
976 print(exc.strerr, file=sys.stderr)
977 rv = bool(1)
978 except OSError as exc:
979 # One of the files doesn't exist.
980 if exc.filename == cFile:
981 # We are ok - there might be no configuration file yet.
982 # Replace original with temporary file
983 try:
984 os.rename(cFileTmp, cFile)
985
986 except OSError as exc:
987 arta 1.13 print('Unable to update sum_rm configuration file ' + cFile + '.', file=sys.stderr)
988 print(exc.strerr, file=sys.stderr)
989 rv = bool(1)
990 else:
991 # There is a problem with the temp file - bail.
992 print('Unable to update sum_rm configuration file ' + cFile + '.', file=sys.stderr)
993 print(exc.strerr, file=sys.stderr)
994 rv = bool(1)
995
996 return rv
997
|
998 arta 1.33 def configureNet(cfgfile, cfile, mfile, pfile, pyfile, shfile, project_includes_file, base, keymap, createSumRmCfg):
999 error = False
|
1000 arta 1.28
|
1001 arta 1.1 defs = {}
1002 cDefs = list()
|
1003 arta 1.4 mDefsGen = list()
1004 mDefsMake = list()
1005 mDefsComps = list()
|
1006 arta 1.33 project_includes = []
|
1007 arta 1.1 perlConstSection = list()
1008 perlInitSection = list()
|
1009 arta 1.18 pyConstSection = list()
1010 pyInitSection = list()
|
1011 arta 1.25 shConstSection = list()
|
1012 arta 1.2 addenda = {}
|
1013 arta 1.28
|
1014 arta 1.6 # There are three parameters that were not included in the original config.local parameter set, for some reason.
1015 # Due to this omission, then are not configurable, and must be set in the script.
|
1016 arta 1.2 addenda['a:USER'] = 'NULL'
1017 addenda['a:PASSWD'] = 'NULL'
1018 addenda['p:DSDS_SUPPORT'] = '0'
|
1019 arta 1.28
|
1020 arta 1.6 # This parameter is not configurable. BUILD_TYPE is used to distinguish between a NetDRMS and an JSOC-SDP build.
|
1021 arta 1.2 addenda['a:BUILD_TYPE'] = 'NETDRMS' # Means a non-Stanford build. This will set two additional macros used by make:
1022 # __LOCALIZED_DEFS__ and NETDRMS_BUILD. The former is to support legacy code
|
1023 arta 1.28 # which incorrectly used this macro, and the latter is for future use.
|
1024 arta 1.2 # __LOCALIZED_DEFS__ is deprecated and should not be used in new code.
|
1025 arta 1.1
1026 try:
1027 with open(cfgfile, 'r') as fin:
|
1028 arta 1.3 # Process configuration parameters
|
1029 arta 1.26
|
1030 arta 1.33 error = parseConfig(fin, keymap, addenda, defs, cDefs, mDefsGen, mDefsMake, project_includes, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
1031 if not error:
|
1032 arta 1.3 # Must add a parameter for the SUMS_MANAGER UID (for some reason). This must be done after the
1033 # config file is processed since an input to getMgrUIDLine() is one of the config file's
1034 # parameter values.
|
1035 arta 1.31
1036 # CANNOT run this unless sunroom2 home directories are accessible
|
1037 arta 1.1 uidParam = {}
|
1038 arta 1.33 error = getMgrUIDLine(defs.get('SUMS_MANAGER', None), uidParam)
1039 if not error and len(uidParam) > 0:
1040 error = parseConfig(None, keymap, uidParam, defs, cDefs, mDefsGen, None, None, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
|
1041 arta 1.31
1042 # ignore the error where the SUMS manager UID cannot be determined
|
1043 arta 1.33 error = False
|
1044 arta 1.31
|
1045 arta 1.3 # Configure the compiler-selection make variables.
|
1046 arta 1.33 if not error:
1047 error = configureComps(defs, mDefsComps)
|
1048 arta 1.1
|
1049 arta 1.3 # Write out the parameter files.
|
1050 arta 1.33 if not error:
1051 error = writeParamsFiles(base, cfile, mfile, pfile, pyfile, shfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
|
1052 arta 1.13
|
1053 arta 1.33 if not error:
1054 error = write_project_includes(project_includes_file, project_includes)
|
1055 arta 1.13
1056 # Write out the sum_rm.cfg file.
|
1057 arta 1.33 if not error and createSumRmCfg:
1058 error = generateSumRmCfg(defs)
|
1059 arta 1.1 except IOError as exc:
|
1060 arta 1.6 print(exc.strerror, file=sys.stderr)
1061 print('Unable to read configuration file ' + cfgfile + '.', file=sys.stderr)
|
1062 arta 1.3 except Exception as exc:
|
1063 arta 1.16 if len(exc.args) >= 2:
|
1064 arta 1.14 type, msg = exc.args
1065 else:
1066 # re-raise the exception
1067 raise
1068
|
1069 arta 1.3 if type == 'unexpectedIccRet':
1070 print('icc -V returned this unexpected message:\n' + msg, file=sys.stderr)
|
1071 arta 1.33 error = True
|
1072 arta 1.3 elif type == 'unexpectedGccRet':
1073 print('gcc -v returned this unexpected message:\n' + msg, file=sys.stderr)
|
1074 arta 1.33 error = True
|
1075 arta 1.3 elif type == 'unexpectedIfortRet':
1076 print('ifort -V returned this unexpected message:\n' + msg, file=sys.stderr)
|
1077 arta 1.33 error = True
|
1078 arta 1.3 elif type == 'unexpectedGfortranRet':
1079 print('gfortran -v returned this unexpected message:\n' + msg, file=sys.stderr)
|
1080 arta 1.33 error = True
|
1081 arta 1.3 else:
1082 # re-raise the exception
1083 raise
|
1084 arta 1.28
|
1085 arta 1.33 return error
|
1086 arta 1.1
|
1087 arta 1.33 def configureSdp(cfgfile, cfile, mfile, pfile, pyfile, shfile, project_includes_file, base):
1088 error = False
|
1089 arta 1.28
|
1090 arta 1.1 defs = {}
1091 cDefs = list()
|
1092 arta 1.5 mDefsGen = list()
1093 mDefsMake = list()
1094 mDefsComps = list()
|
1095 arta 1.33 project_includes = []
|
1096 arta 1.1 perlConstSection = list()
1097 perlInitSection = list()
|
1098 arta 1.18 pyConstSection = list()
1099 pyInitSection = list()
|
1100 arta 1.25 shConstSection = list()
|
1101 arta 1.18
|
1102 arta 1.2 addenda = {}
|
1103 arta 1.28
|
1104 arta 1.6 # There are three parameters that were not included in the original config.local parameter set, for some reason.
1105 # Due to this omission, then are not configurable, and must be set in the script.
|
1106 arta 1.5 addenda['a:USER'] = 'NULL'
1107 addenda['a:PASSWD'] = 'NULL'
1108 addenda['p:DSDS_SUPPORT'] = '1'
|
1109 arta 1.28
|
1110 arta 1.6 # This parameter is not configurable. BUILD_TYPE is used to distinguish between a NetDRMS and an JSOC-SDP build.
|
1111 arta 1.5 addenda['a:BUILD_TYPE'] = 'JSOC_SDP' # Means a Stanford build. This will set one additional macro used by make: JSOC_SDP_BUILD.
|
1112 arta 1.28
|
1113 arta 1.1 try:
1114 with open(cfgfile, 'r') as fin:
|
1115 arta 1.33 error = parseConfig(fin, None, addenda, defs, cDefs, mDefsGen, mDefsMake, project_includes, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
|
1116 arta 1.28
|
1117 arta 1.33 if not error:
|
1118 arta 1.2 # Must add a parameter for the SUMS_MANAGER UID (for some reason)
1119 uidParam = {}
|
1120 arta 1.33 error = getMgrUIDLine(defs.get('SUMS_MANAGER', None), uidParam)
1121 if not error and len(uidParam) > 0:
1122 error = parseConfig(None, None, uidParam, defs, cDefs, mDefsGen, None, None, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
|
1123 arta 1.6
|
1124 arta 1.31 # ignore the error where the SUMS manager UID cannot be determined
|
1125 arta 1.33 error = False
|
1126 arta 1.31
|
1127 arta 1.5 # Configure the compiler-selection make variables.
|
1128 arta 1.33 if not error:
1129 error = configureComps(defs, mDefsComps)
|
1130 arta 1.28
|
1131 arta 1.6 # Write out the parameter files.
|
1132 arta 1.33 if not error:
1133 error = writeParamsFiles(base, cfile, mfile, pfile, pyfile, shfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
|
1134 arta 1.6
|
1135 arta 1.33 if not error:
1136 error = write_project_includes(project_includes_file, project_includes)
|
1137 arta 1.13
1138 # At Stanford, skip the creation of the sum_rm configuration file. config.local will still
1139 # have the SUMRM parameters, but they will not be used.
|
1140 arta 1.1 except IOError as exc:
|
1141 arta 1.6 print(exc.strerror, file=sys.stderr)
1142 print('Unable to read configuration file ' + cfgfile + '.', file=sys.stderr)
|
1143 arta 1.5 except Exception as exc:
|
1144 arta 1.17 if len(exc.args) >= 2:
1145 type = exc.args[0]
1146 else:
1147 # re-raise the exception
1148 raise
|
1149 arta 1.28
|
1150 arta 1.5 if type == 'unexpectedIccRet':
|
1151 arta 1.6 msg = exc.args[1]
|
1152 arta 1.5 print('icc -V returned this unexpected message:\n' + msg, file=sys.stderr)
|
1153 arta 1.33 error = True
|
1154 arta 1.5 elif type == 'unexpectedGccRet':
|
1155 arta 1.6 msg = exc.args[1]
|
1156 arta 1.5 print('gcc -v returned this unexpected message:\n' + msg, file=sys.stderr)
|
1157 arta 1.33 error = True
|
1158 arta 1.5 elif type == 'unexpectedIfortRet':
|
1159 arta 1.6 msg = exc.args[1]
|
1160 arta 1.5 print('ifort -V returned this unexpected message:\n' + msg, file=sys.stderr)
|
1161 arta 1.33 error = True
|
1162 arta 1.5 elif type == 'unexpectedGfortranRet':
|
1163 arta 1.6 msg = exc.args[1]
|
1164 arta 1.5 print('gfortran -v returned this unexpected message:\n' + msg, file=sys.stderr)
|
1165 arta 1.33 error = True
|
1166 arta 1.5 else:
1167 # re-raise the exception
1168 raise
|
1169 arta 1.28
|
1170 arta 1.33 return error
|
1171 arta 1.2
|
1172 arta 1.1 # Beginning of program
1173 rv = RET_SUCCESS
1174 net = bool(1)
1175
1176 # Parse arguments
1177 if __name__ == "__main__":
1178 optD = GetArgs(sys.argv[1:])
1179
1180 if not(optD is None):
1181 # Ensure we are configuring a DRMS tree
1182 cdir = os.path.realpath(os.getcwd())
1183 versfile = cdir + '/base/' + VERS_FILE
1184
1185 if not os.path.isfile(versfile):
1186 rv = RET_NOTDRMS
1187
1188 # Determine whether we are localizing a Stanford build, or a NetDRMS build. If configsdp.txt exists, then
1189 # it is a Stanford build, otherwise it is a NetDRMS build.
1190 if rv == RET_SUCCESS:
1191 stanfordFile = cdir + '/' + SDP_CFG
1192 if os.path.isfile(stanfordFile):
1193 arta 1.1 net = bool(0)
|
1194 arta 1.28
|
1195 arta 1.1 cfile = optD['dir'] + '/' + optD['base'] + '.h'
1196 mfile = optD['dir'] + '/' + optD['base'] + '.mk'
1197 pfile = optD['dir'] + '/' + optD['base'] + '.pm'
|
1198 arta 1.18 pyfile = optD['dir'] + '/' + optD['base'] + '.py'
|
1199 arta 1.25 shfile = optD['dir'] + '/' + optD['base'] + '.sh'
|
1200 arta 1.33 project_includes_file = os.path.join(optD['dir'], 'includes.mk')
|
1201 arta 1.1
1202 if net:
1203 try:
1204 with open(NET_CFGMAP, 'r') as fin:
1205 regexpComm = re.compile(r"^\s*#")
1206 regexp = re.compile(r"^\s*(\S+)\s+(\w:\S+)")
1207 # Must map from config.local namespace to DRMS namespace (e.g., the names used for the C macros)
1208 keymap = {}
1209 for line in fin:
1210 matchobj = regexpComm.match(line)
1211 if not matchobj is None:
1212 # Skip comment line
1213 continue
1214
1215 matchobj = regexp.match(line)
1216 if not(matchobj is None):
1217 # We have a key-value line
1218 key = matchobj.group(1)
1219 val = matchobj.group(2)
1220 keymap[key] = val
1221 except OSError:
1222 arta 1.1 sys.stderr.write('Unable to read configuration map-file ' + NET_CFGMAP + '.')
1223 rv = bool(1)
1224
1225 # We also need to set the UID of the SUMS manager. We have the name of the
1226 # SUMS manager (it is in the configuration file)
|
1227 arta 1.33 configureNet(NET_CFG, cfile, mfile, pfile, pyfile, shfile, project_includes_file, optD['base'], keymap, 'server' in optD)
|
1228 arta 1.1 else:
|
1229 arta 1.28 # A Stanford user can override the parameters in configsdp.txt by copying that file to config.local,
|
1230 arta 1.8 # and then editing config.local. So, if config.local exists, use that.
1231 if os.path.isfile(cdir + '/' + NET_CFG):
|
1232 arta 1.33 configureSdp(NET_CFG, cfile, mfile, pfile, pyfile, shfile, project_includes_file, optD['base'])
|
1233 arta 1.8 else:
|
1234 arta 1.33 configureSdp(SDP_CFG, cfile, mfile, pfile, pyfile, shfile, project_includes_file, optD['base'])
|
Return to
localize.py
CVS log
Up to [Development] / JSOC