1 arta 1.1 #!/home/jsoc/bin/linux_x86_64/activepython
2
3 import sys
4 import getopt
5 import re
|
6 arta 1.6 import os
7 import stat
8 import xml.etree.ElementTree as ET
|
9 arta 1.1 from subprocess import check_output, CalledProcessError
10
11 # Constants
12 VERS_FILE = 'jsoc_version.h'
13 SDP_CFG = 'configsdp.txt'
14 NET_CFG = 'config.local'
15 NET_CFGMAP = 'config.local.map'
16 RET_SUCCESS = 0
17 RET_NOTDRMS = 1
18
|
19 arta 1.2 PREFIX = """# This file was auto-generated by localize.py. Please do not edit it directly (running
20 # configure will run localize.py, which will then overwrite any edits manually performed).
21 """
|
22 arta 1.1
|
23 arta 1.2 C_PREFIX = """/* This file was auto-generated by localize.py. Please do not edit it directly (running
24 * configure will run localize.py, which will then overwrite any edits manually performed). */
|
25 arta 1.1 """
26
|
27 arta 1.2 PERL_BINPATH = '#!/usr/bin/perl\n'
28
29 PERL_INTIAL = """package drmsparams;
30
31 use warnings;
32 use strict;
33 """
34
35 PERL_FXNS_A = """sub new
|
36 arta 1.1 {
37 my($clname) = shift;
38
39 my($self) =
40 {
41 _paramsH => undef
42 };
43
44 bless($self, $clname);
|
45 arta 1.9 $self->{_paramsH} = {};
46 $self->initialize();
|
47 arta 1.1
48 return $self;
49 }
50
51 sub DESTROY
52 {
53 my($self) = shift;
54 }
|
55 arta 1.2 """
56
57 PERL_FXNS_B = """sub get
58 {
59 my($self) = shift;
60 my($name) = shift;
61 my($rv);
|
62 arta 1.1
|
63 arta 1.2 if (exists($self->{_paramsH}->{$name}))
64 {
65 return $self->{_paramsH}->{$name};
66 }
67 else
68 {
69 return undef;
70 }
71 }
72 1;"""
|
73 arta 1.1
|
74 arta 1.6
75 RULESPREFIX = """# Standard things
76 sp := $(sp).x
77 dirstack_$(sp) := $(d)
78 d := $(dir)
79 """
80
81 RULESSUFFIX = """dir := $(d)/example
82 -include $(SRCDIR)/$(dir)/Rules.mk
83 dir := $(d)/cookbook
84 -include $(SRCDIR)/$(dir)/Rules.mk
85 dir := $(d)/myproj
86 -include $(SRCDIR)/$(dir)/Rules.mk
87
88 # Standard things
89 d := $(dirstack_$(sp))
90 sp := $(basename $(sp))
91 """
92
93 TARGETPREFIX = """$(PROJOBJDIR):\n\t+@[ -d $@ ] || mkdir -p $@"""
94
|
95 arta 1.3 ICC_MAJOR = 9
96 ICC_MINOR = 0
97 GCC_MAJOR = 3
98 GCC_MINOR = 0
99 IFORT_MAJOR = 9
100 IFORT_MINOR = 0
101 GFORT_MAJOR = 4
102 GFORT_MINOR = 2
103
104
|
105 arta 1.1 # Read arguments
106 # d - localization directory
107 # b - base name of all parameter files (e.g., -b drmsparams --> drmsparams.h, drmsparams.mk, drmsparams.pm, etc.)
108 # m - make file
109 def GetArgs(args):
110 rv = bool(0)
111 optD = {}
112
113 try:
114 opts, remainder = getopt.getopt(args, "hd:b:",["dir=", "base="])
115 except getopt.GetoptError:
116 print('Usage:')
117 print('localize.py [-h] -d <localization directory> -b <parameter file base>')
118 rv = bool(1)
119
120 if rv == bool(0):
121 for opt, arg in opts:
122 if opt == '-h':
123 print('localize.py [-h] -d <localization directory> -b <parameter file base>')
124 elif opt in ("-d", "--dir"):
125 regexp = re.compile(r"(\S+)/?")
126 arta 1.1 matchobj = regexp.match(arg)
127 if matchobj is None:
128 rv = bool(1)
129 else:
130 optD['dir'] = matchobj.group(1)
131 elif opt in ("-b", "--base"):
132 optD['base'] = arg
133 else:
134 optD[opt] = arg
135
136 return optD
137
138 def createMacroStr(key, val, keyColLen, status):
139 if keyColLen < len(key):
140 status = bool(1)
141 return None
142 else:
143 nsp = keyColLen - len(key)
144 spaces = str()
145 for isp in range(nsp):
146 spaces += ' '
147 arta 1.1 status = bool(0)
148 return '#define ' + key + spaces + val + '\n'
149
150 def createPerlConst(key, val, keyColLen, status):
151 if keyColLen < len(key):
152 status = bool(1)
153 return None
154 else:
155 nsp = keyColLen - len(key)
156 spaces = str()
157 for isp in range(nsp):
158 spaces += ' '
159 status = bool(0)
160 return 'use constant ' + key + ' => ' + spaces + val + ';\n'
161
|
162 arta 1.3 def isSupportedPlat(plat):
163 regexp = re.compile(r"\s*(^x86_64|^ia32|^ia64|^avx)", re.IGNORECASE)
164 matchobj = regexp.match(plat);
165
166 if not matchobj is None:
167 return bool(1);
168 else:
169 return bool(0);
170
|
171 arta 1.6 def processMakeParam(mDefs, key, val, platDict, machDict):
|
172 arta 1.3 varMach = None
173 regexp = re.compile(r"(\S+):(\S+)")
174 matchobj = regexp.match(key)
175 if not matchobj is None:
176 varName = matchobj.group(1)
177 varMach = matchobj.group(2)
178 else:
179 varName = key
180
181 varValu = val
182
183 if varMach is None:
184 mDefs.extend(list('\n' + varName + ' = ' + varValu))
185 else:
186 if isSupportedPlat(varMach):
187 # The guard will compare varValu to $JSOC_MACHINE.
188 if not varMach in platDict:
189 platDict[varMach] = {}
190 platDict[varMach][varName] = varValu
191 else:
192 # The guard will compare varValu to $MACHINETYPE (this is just the hostname of the machine on which localize.py is running).
193 arta 1.3 if not varMach in machDict:
194 machDict[varMach] = {}
195 machDict[varMach][varName] = varValu
196
|
197 arta 1.6 def processParam(cfgfile, line, regexpQuote, regexp, keymap, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, platDict, machDict, section):
|
198 arta 1.1 status = 0
199
200 if ''.join(section) == 'defs' or not cfgfile:
201 matchobj = regexp.match(line)
202 if not matchobj is None:
203 # We have a key-value line
204 keyCfgSp = matchobj.group(1)
205 val = matchobj.group(2)
206
207 # Must map the indirect name to the actual name
208 if keymap:
209 # Map to actual name only if the keymap is not empty (which signifies NA).
210 if keyCfgSp in keymap:
211 key = keymap[keyCfgSp]
212 elif keyCfgSp == 'LOCAL_CONFIG_SET' or keyCfgSp == 'DRMS_SAMPLE_NAMESPACE':
|
213 arta 1.6 # Ignore parameters that are not useful and shouldn't have been there in the first place. But
214 # they have been released to the world, so we have to account for them.
|
215 arta 1.1 return bool(0)
216 elif not cfgfile:
217 # Should not be doing mapping for addenda
218 key = keyCfgSp
219 else:
220 raise Exception('badKeyMapKey', keyCfgSp)
221 else:
222 key = keyCfgSp
223
224 matchobj = regexpQuote.match(key)
225 if not matchobj is None:
226 quote = matchobj.group(1)
227 key = matchobj.group(2)
228
229 # master defs dictionary
230 defs[key] = val
231
232 # C header file
233 if quote == "q":
234 # Add double-quotes
235 cDefs.extend(list(createMacroStr(key, '"' + val + '"', 40, status)))
236 arta 1.1 elif quote == "p":
237 # Add parentheses
238 cDefs.extend(list(createMacroStr(key, '(' + val + ')', 40, status)))
239 elif quote == "a":
240 # Leave as-is
241 cDefs.extend(list(createMacroStr(key, val, 40, status)))
242 else:
243 # Unknown quote type
244 raise Exception('badQuoteQual', key)
245
246 if status:
247 raise Exception('paramNameTooLong', key)
248
249 # Make file - val should never be quoted; just use as is
|
250 arta 1.4 mDefsGen.extend(list('\n' + key + ' = ' + val))
|
251 arta 1.1
252 # Perl file - val should ALWAYS be single-quote quoted
253 # Save const info to a string
254 perlConstSection.extend(list(createPerlConst(key, "'" + val + "'", 40, status)))
255
256 if status:
257 raise Exception('paramNameTooLong', key)
258
259 # Save initialization information as a string. Now that we've defined
260 # constants (the names of which are the parameter names)
261 # we can refer to those in the init section. The key variable holds the
262 # name of the constant.
|
263 arta 1.9 perlInitSection.extend(list("\n $self->{_paramsH}->{'" + key + "'} = " + key + ';'))
|
264 arta 1.1 else:
265 # No quote qualifier
266 raise Exception('missingQuoteQual', key)
|
267 arta 1.3 elif ''.join(section) == 'make' and cfgfile:
268 # Configure the remaining make variables defined in the __MAKE__ section of the configuration file. Third-party
269 # library make variables are specified in the __MAKE__ section.
270 matchobj = regexp.match(line)
271 if not matchobj is None:
272 # We have a key-value line
273 key = matchobj.group(1)
274 val = matchobj.group(2)
275
276 # This information is for making make variables only. We do not need to worry about quoting any values
277 defs[key] = val
|
278 arta 1.4 processMakeParam(mDefsMake, key, val, platDict, machDict)
|
279 arta 1.1
280 return bool(0)
281
282 # We have some extraneous line or a newline - ignore.
283
|
284 arta 1.6 def processXML(xml, projRules, projTarget):
285 rv = bool(0)
286
287 # <projects>
288 root = ET.fromstring(xml)
289
290 # Iterate through each proj child.
291 for proj in root.iter('proj'):
292 # Rules.mk
293 nameElem = proj.find('name')
294 rulesStr = 'dir := $(d)/' + nameElem.text + '\n-include $(SRCDIR)/$(dir)/Rules.mk\n'
295
296 # make doesn't support logical operations in ifeq conditionals (you can't do ifeq (A AND B)),
297 # so we need to write:
298 # ifeq (A)
299 # ifeq (B)
300 # <do something>
301 # endif
302 # endif
303
304 rulesPref = '';
305 arta 1.6 rulesSuff = '';
306
307 filters = proj.find('filters')
308 if filters is not None:
309 for filter in filters.findall('filter'):
310 rulesPref += 'ifeq ($(' + filter.find('name').text + '),' + filter.find('value').text + ')\n'
311 rulesSuff += 'endif\n'
312
313 if len(rulesPref) > 0 and len(rulesSuff) > 0:
314 projRules.extend(list(rulesPref))
315 projRules.extend(list(rulesStr))
316 projRules.extend(list(rulesSuff))
317 else:
318 projRules.extend(list(rulesStr))
319
320 # target.mk
321 subdirs = proj.find('subdirs')
322 if subdirs is not None:
323 for subdir in subdirs.findall('subdir'):
324 targetStr = '\n\t+@[ -d $@/' + nameElem.text + '/' + subdir.text + ' ] || mkdir -p $@/' + nameElem.text + '/' + subdir.text
325 projTarget.extend(list(targetStr))
326 arta 1.6
327 return rv
328
329 def determineSection(line, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg):
330 matchobj = regexpDefs.match(line)
331 if not matchobj is None:
332 return 'defs'
333
334 matchobj = regexpMake.match(line)
335 if not matchobj is None:
336 return 'make'
337
338 matchobj = regexpProjMkRules.match(line)
339 if not matchobj is None:
340 return 'projmkrules'
341
342 matchobj = regexpProj.match(line)
343 if not matchobj is None:
344 return 'proj'
345
346 matchobj = regexpProjCfg.match(line)
347 arta 1.6 if not matchobj is None:
348 return 'projcfg'
349
350 return None
351
|
352 arta 1.1 # defs is a dictionary containing all parameters (should they be needed in this script)
|
353 arta 1.6 # projCfg is the list containing the configure script content.
354 # projMkRules is the list containing the make_basic.mk content.
355 # projRules is the list containing the Rules.mk content.
356 # projTargert is the list containing the target.mk content.
357 def parseConfig(fin, keymap, addenda, defs, cDefs, mDefsGen, mDefsMake, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection):
|
358 arta 1.1 rv = bool(0)
359
360 # Open required config file (config.local)
361 try:
362 # Examine each line, looking for key=value pairs.
363 regexpDefs = re.compile(r"^__DEFS__")
364 regexpMake = re.compile(r"^__MAKE__")
|
365 arta 1.6 regexpProjMkRules = re.compile(r"__PROJ_MK_RULES__")
366 regexpProj = re.compile(r"^__PROJ__")
367 regexpProjCfg = re.compile(r"^__PROJCFG__")
|
368 arta 1.1 regexpComm = re.compile(r"^\s*#")
|
369 arta 1.6 regexpSp = re.compile(r"^s*$")
|
370 arta 1.1 regexpQuote = re.compile(r"^\s*(\w):(.+)")
|
371 arta 1.6 regexpCustMkBeg = re.compile(r"^_CUST_")
372 regexpCustMkEnd = re.compile(r"^_ENDCUST_")
373 regexpDiv = re.compile(r"^__")
|
374 arta 1.11 regexp = re.compile(r"^\s*(\S+)\s+(\S.*)")
|
375 arta 1.1
|
376 arta 1.3 platDict = {}
377 machDict = {}
|
378 arta 1.6
379 xml = None
|
380 arta 1.1
381 # Process the parameters in the configuration file
382 if not fin is None:
383 for line in fin:
|
384 arta 1.6 matchobj = regexpComm.match(line)
385 if not matchobj is None:
386 # Skip comment line
387 continue
388
389 matchobj = regexpSp.match(line)
390 if not matchobj is None:
391 # Skip whitespace line
392 continue
|
393 arta 1.1
|
394 arta 1.6 newSection = determineSection(line, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg)
395 if not newSection is None:
396 section = newSection
397
398 if section == 'make':
|
399 arta 1.10
|
400 arta 1.6 # There are some blocks of lines in the __MAKE__ section that must be copied ver batim to the output make file.
401 # The blocks are defined by _CUST_/_ENDCUST_ tags.
402 matchobj = regexpCustMkBeg.match(line)
403
404 if not matchobj is None:
405 mDefsMake.extend(list('\n'))
406 for line in fin:
407 matchobj = regexpCustMkEnd.match(line)
408 if not matchobj is None:
409 break;
410 mDefsMake.extend(list(line))
411 newSection = determineSection(line, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg)
412 if not newSection is None:
413 section = newSection
414 continue
415 # Intentional fall through to next if statement
416 if section == 'defs' or section == 'make':
417 iscfg = bool(1)
418 ppRet = processParam(iscfg, line, regexpQuote, regexp, keymap, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, platDict, machDict, section)
|
419 arta 1.10
|
420 arta 1.6 if ppRet:
|
421 arta 1.10 break
|
422 arta 1.6 elif section == 'projcfg':
423 # Copy the line ver batim to the projCfg list (configure)
424 for line in fin:
425 matchobj = regexpDiv.match(line)
426 if not matchobj is None:
427 break;
428 projCfg.extend(list(line))
429 newSection = determineSection(line, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg)
430 if not newSection is None:
431 section = newSection
432 continue
433 elif section == 'projmkrules':
434 # Copy the line ver batim to the projMkRules list (make_basic.mk)
435 for line in fin:
436 matchobj = regexpDiv.match(line)
437 if not matchobj is None:
438 break;
439 projMkRules.extend(list(line))
440 newSection = determineSection(line, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg)
441 if not newSection is None:
442 section = newSection
443 arta 1.6 continue
444 elif section == 'proj':
445 # Must parse xml and use the project-specific information to populate the Rules.mk and target.mk files.
446 # Collect all xml lines for now, then process after file-read loop.
447 if xml is None:
448 xml = line
449 else:
450 xml += line
451 else:
|
452 arta 1.10 # Unknown section
|
453 arta 1.6 raise Exception('unknownSection', section)
|
454 arta 1.1 except Exception as exc:
|
455 arta 1.6 msg = exc.args[0]
|
456 arta 1.1 if msg == 'badKeyMapKey':
457 # If we are here, then there was a non-empty keymap, and the parameter came from
458 # the configuration file.
|
459 arta 1.6 violator = exc.args[1]
|
460 arta 1.1 print('Unknown parameter name ' + "'" + violator + "'" + ' in ' + cfgfile + '.', file=sys.stderr)
461 rv = bool(1)
462 elif msg == 'badQuoteQual':
463 # The bad quote qualifier came from the configuration file, not the addenda, since
464 # we will have fixed any bad qualifiers in the addenda (which is populated by code).
|
465 arta 1.6 violator = exc.args[1]
|
466 arta 1.1 print('Unknown quote qualifier ' + "'" + violator + "'" + ' in ' + cfgfile + '.', file=sys.stderr)
467 rv = bool(1)
468 elif msg == 'missingQuoteQual':
|
469 arta 1.6 violator = exc.args[1]
|
470 arta 1.1 print('Missing quote qualifier for parameter ' + "'" + violator + "'" + ' in ' + cfgfile + '.', file=sys.stderr)
471 rv = bool(1)
472 elif msg == 'paramNameTooLong':
|
473 arta 1.6 violator = exc.args[1]
|
474 arta 1.1 print('Macro name ' + "'" + violator + "' is too long.", file=sys.stderr)
475 rv = bool(1)
|
476 arta 1.6 elif msg == 'unknownSection':
477 violator = exc.args[1]
478 print('Unknown section ' + "'" + violator + "' in configuration file.", file=sys.stderr)
479 rv = bool(1)
|
480 arta 1.1 else:
481 # re-raise the exception
482 raise
483
|
484 arta 1.6 if not rv:
485 if not xml is None:
486 # Process xml.
487 projRules.extend(list(RULESPREFIX))
488 projTarget.extend(list(TARGETPREFIX))
489 rv = processXML(xml, projRules, projTarget)
490 projRules.extend(RULESSUFFIX)
491
492 # Process addenda - these are parameters that are not configurable and must be set in the
493 # NetDRMS build.
494 if not rv:
495 iscfg = bool(0)
496 for key in addenda:
497 item = key + ' ' + addenda[key]
498 ppRet = processParam(iscfg, item, regexpQuote, regexp, keymap, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, platDict, machDict, 'defs')
499 if ppRet:
500 break;
501
|
502 arta 1.4 # Put information collected in platDict and machDict into mDefs. Must do this here, and not in processParam, since
503 # we need to parse all platform-specific make variables before grouping them into platform categories.
|
504 arta 1.3 if not rv:
505 for plat in platDict:
|
506 arta 1.4 mDefsMake.extend(list('\nifeq ($(JSOC_MACHINE), linux_' + plat.lower() + ')'))
|
507 arta 1.3 for var in platDict[plat]:
|
508 arta 1.4 mDefsMake.extend(list('\n' + var + ' = ' + platDict[plat][var]))
509 mDefsMake.extend(list('\nendif\n'))
|
510 arta 1.3
511 if not rv:
512 for mach in machDict:
|
513 arta 1.4 mDefsMake.extend(list('\nifeq ($(MACHTYPE), ' + mach + ')'))
|
514 arta 1.7 for var in machDict[mach]:
515 mDefsMake.extend(list('\n' + var + ' = ' + machDict[mach][var]))
|
516 arta 1.4 mDefsMake.extend(list('\nendif\n'))
|
517 arta 1.1 return rv
518
519 def getMgrUIDLine(defs, uidParam):
520 rv = bool(0)
521
522 cmd = 'id -u ' + defs['SUMS_MANAGER']
523 try:
524 ret = check_output(cmd, shell=True)
525 uidParam['q:SUMS_MANAGER_UID'] = ret.decode("utf-8")
526 except ValueError:
527 print('Unable to run cmd: ' + cmd + '.')
528 rv = bool(1)
529 except CalledProcessError:
530 print('Command ' + "'" + cmd + "'" + ' ran improperly.')
531 rv = bool(1)
532
533 return rv
534
|
535 arta 1.3 def isVersion(maj, min, majDef, minDef):
536 res = 0
537
538 if maj > majDef or (maj == majDef and min >= minDef):
539 res = 1
540
541 return res
542
543 def configureComps(defs, mDefs):
544 rv = bool(0)
|
545 arta 1.6 autoConfig = bool(1)
546
547 if 'AUTOSELCOMP' in defs:
548 autoConfig = (not defs['AUTOSELCOMP'] == '0')
|
549 arta 1.3
550 if autoConfig:
551 hasicc = bool(0)
552 hasgcc = bool(0)
553 hasifort = bool(0)
554 hasgfort = bool(0)
555
556 # Try icc.
|
557 arta 1.12 cmd = 'icc --version 2>&1'
|
558 arta 1.3 try:
559 ret = check_output(cmd, shell=True)
560 ret = ret.decode("utf-8")
561 except CalledProcessError:
562 print('Command ' + "'" + cmd + "'" + ' ran improperly.')
563 rv = bool(1)
564
|
565 arta 1.6 if not rv:
|
566 arta 1.12 regexp = re.compile(r"\s*\S+\s+\S+\s+(\d+)[.](\d+)", re.DOTALL)
|
567 arta 1.3 matchobj = regexp.match(ret)
568 if matchobj is None:
569 raise Exception('unexpectedIccRet', ret)
570 else:
571 major = matchobj.group(1)
572 minor = matchobj.group(2)
573 if isVersion(int(major), int(minor), ICC_MAJOR, ICC_MINOR):
574 hasicc = bool(1)
575
576 # Try gcc.
577 if not hasicc:
578 cmd = 'gcc -v 2>&1'
579 try:
580 ret = check_output(cmd, shell=True)
581 ret = ret.decode("utf-8")
582 except CalledProcessError:
583 print('Command ' + "'" + cmd + "'" + ' ran improperly.')
584 rv = bool(1)
585
586 if rv == bool(0):
587 regexp = re.compile(r".+gcc\s+version\s+(\d+)\.(\d+)", re.DOTALL)
588 arta 1.3 matchobj = regexp.match(ret)
589 if matchobj is None:
590 raise Exception('unexpectedGccRet', ret)
591 else:
592 major = matchobj.group(1)
593 minor = matchobj.group(2)
594 if isVersion(int(major), int(minor), GCC_MAJOR, GCC_MINOR):
595 hasgcc = bool(1)
596
597 # Try ifort.
|
598 arta 1.12 cmd = 'ifort --version 2>&1'
|
599 arta 1.3 try:
600 ret = check_output(cmd, shell=True)
601 ret = ret.decode("utf-8")
602 except CalledProcessError:
603 print('Command ' + "'" + cmd + "'" + ' ran improperly.')
604 rv = bool(1)
605
|
606 arta 1.6 if not rv:
|
607 arta 1.12 regexp = re.compile(r"\s*\S+\s+\S+\s+(\d+)\.(\d+)", re.DOTALL)
|
608 arta 1.3 matchobj = regexp.match(ret)
609 if matchobj is None:
610 raise Exception('unexpectedIfortRet', ret)
611 else:
612 major = matchobj.group(1)
613 minor = matchobj.group(2)
614 if isVersion(int(major), int(minor), IFORT_MAJOR, IFORT_MINOR):
615 hasifort = bool(1)
616
617 # Try gfortran
618 if not hasifort:
619 cmd = 'gfortran -v 2>&1'
620 try:
621 ret = check_output(cmd, shell=True)
622 ret = ret.decode("utf-8")
623 except CalledProcessError:
624 print('Command ' + "'" + cmd + "'" + ' ran improperly.')
625 rv = bool(1)
626
627 if rv == bool(0):
628 regexp = re.compile(r".+gcc\s+version\s+(\d+)\.(\d+)", re.DOTALL)
629 arta 1.3 matchobj = regexp.match(ret)
630 if matchobj is None:
631 raise Exception('unexpectedGfortranRet', ret)
632 else:
633 major = matchobj.group(1)
634 minor = matchobj.group(2)
635 if isVersion(int(major), int(minor), GFORT_MAJOR, GFORT_MINOR):
636 hasgfort = bool(1)
637
638 # Append the compiler make variables to the make file
639 if not hasicc and not hasgcc:
640 print('Fatal error: Acceptable C compiler not found! You will be unable to build the DRMS library.', file=sys.stderr)
641 rv = bool(1)
642 elif hasicc:
643 mDefs.extend(list('\nCOMPILER = icc'))
644 else:
645 mDefs.extend(list('\nCOMPILER = gcc'))
646
647 if not hasifort and not hasgfort:
648 print('Warning: Acceptable Fortran compiler not found! Fortran interface will not be built, and you will be unable to build Fortran modules.', file=sys.stderr)
649 elif hasifort:
650 arta 1.3 mDefs.extend(list('\nFCOMPILER = ifort'))
651 else:
652 mDefs.extend(list('\nFCOMPILER = gfortran'))
653
654 # Environment overrides. These get written, regardless of the disposition of auto-configuration.
|
655 arta 1.9 mDefs.extend(list('\nifneq ($(JSOC_COMPILER),)\n COMPILER = $(JSOC_COMPILER)\nendif'))
656 mDefs.extend(list('\nifneq ($(JSOC_FCOMPILER),)\n FCOMPILER = $(JSOC_FCOMPILER)\nendif'))
|
657 arta 1.3
658 return rv
659
|
660 arta 1.6 def writeParamsFiles(base, cfile, mfile, pfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection):
|
661 arta 1.1 rv = bool(0)
|
662 arta 1.4
663 # Merge mDefsGen, mDefsMake, and mDefsComps into a single string with compiler configuration first, general parameters next, then
664 # make-specific make variables (e.g., third-party library information) last.
665 mDefs = '\n# Compiler Selection\n' + ''.join(mDefsComps) + '\n\n# General Parameters\n' + ''.join(mDefsGen) + '\n\n# Parameters to Configure make\n' + ''.join(mDefsMake)
|
666 arta 1.1
667 try:
668 with open(cfile, 'w') as cout, open(mfile, 'w') as mout, open(pfile, 'w') as pout:
669 # C file of macros
|
670 arta 1.2 print(C_PREFIX, file=cout)
671 print('/* This file contains a set of preprocessor macros - one for each configuration parameter. */\n', file=cout)
|
672 arta 1.1 buf = '__' + base.upper() + '_H'
673 print('#ifndef ' + buf, file=cout)
674 print('#define ' + buf, file=cout)
675 print(''.join(cDefs), file=cout)
676 print('#endif', file=cout)
677
678 # Make file of make variables
|
679 arta 1.2 print(PREFIX, file=mout)
|
680 arta 1.4 print('# This file contains a set of make-variable values. The first section contains compiler-selection variables, the second contains general configuration variables, and the third section contains variables that configure how make is run.', file=mout)
681 print(mDefs, file=mout)
|
682 arta 1.1
|
683 arta 1.2 # Perl module
684 print(PERL_BINPATH, file=pout)
685 print(PREFIX, file=pout)
686 print('# This file contains a set of constants - one for each configuration parameter.\n', file=pout)
687 print(PERL_INTIAL, file=pout)
|
688 arta 1.1 print(''.join(perlConstSection), file=pout)
|
689 arta 1.2 print(PERL_FXNS_A, file=pout)
|
690 arta 1.1 print('sub initialize', file=pout)
691 print('{', file=pout)
|
692 arta 1.2 print(' my($self) = shift;', file=pout, end='')
|
693 arta 1.1 print('', file=pout)
694 print(''.join(perlInitSection), file=pout)
|
695 arta 1.2 print('}\n', file=pout)
696 print(PERL_FXNS_B, file=pout)
|
697 arta 1.1 except IOError as exc:
|
698 arta 1.6 type, value, traceback = sys.exc_info()
699 print(exc.strerror, file=sys.stderr)
700 print('Unable to open ' + "'" + value.filename + "'.", file=sys.stderr)
701 rv = bool(1)
702
703 return rv
704
705 def writeProjFiles(pCfile, pMfile, pRfile, pTfile, projCfg, projMkRules, projRules, projTarget):
706 rv = bool(0)
707
708 try:
709 if projCfg:
710 with open(pCfile, 'w') as cout:
711 # configure
712 print(''.join(projCfg), file=cout)
713
714 if projMkRules:
715 with open(pMfile, 'w') as mout:
716 # make_basic.mk
717 print(PREFIX, file=mout)
718 print(''.join(projMkRules), file=mout)
719 arta 1.6
720 if projRules:
721 with open(pRfile, 'w') as rout:
722 # Rules.mk
723 print(PREFIX, file=rout)
724 print(''.join(projRules), file=rout)
725
726 if projTarget:
727 with open(pTfile, 'w') as tout:
728 # target.mk
729 print(PREFIX, file=tout)
|
730 arta 1.10 print(''.join(projTarget), file=tout)
|
731 arta 1.6 except IOError as exc:
732 type, value, traceback = sys.exc_info()
733 print(exc.strerror, file=sys.stderr)
734 print('Unable to open ' + "'" + value.filename + "'.", file=sys.stderr)
|
735 arta 1.1 rv = bool(1)
736
|
737 arta 1.6 if not rv:
|
738 arta 1.10 if os.path.exists(pCfile):
739 try:
740 os.chmod(pCfile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IROTH)
741 except OSError as exc:
742 type, value, traceback = sys.exc_info()
743 print('Unable to chmod file ' + "'" + value.filename + "'.", file=sys.stderr)
744 print(exc.strerror, file=sys.stderr)
745 rv = bool(1)
|
746 arta 1.6
|
747 arta 1.1 return rv
|
748 arta 1.3
|
749 arta 1.6 def configureNet(cfgfile, cfile, mfile, pfile, pCfile, pMfile, pRfile, pTfile, base, keymap):
|
750 arta 1.1 rv = bool(0)
751
752 defs = {}
753 cDefs = list()
|
754 arta 1.4 mDefsGen = list()
755 mDefsMake = list()
756 mDefsComps = list()
|
757 arta 1.6 projCfg = list()
758 projMkRules = list()
759 projRules = list()
760 projTarget = list()
|
761 arta 1.1 perlConstSection = list()
762 perlInitSection = list()
|
763 arta 1.2 addenda = {}
764
|
765 arta 1.6 # There are three parameters that were not included in the original config.local parameter set, for some reason.
766 # Due to this omission, then are not configurable, and must be set in the script.
|
767 arta 1.2 addenda['a:USER'] = 'NULL'
768 addenda['a:PASSWD'] = 'NULL'
769 addenda['p:DSDS_SUPPORT'] = '0'
|
770 arta 1.6
771 # This parameter is not configurable. BUILD_TYPE is used to distinguish between a NetDRMS and an JSOC-SDP build.
|
772 arta 1.2 addenda['a:BUILD_TYPE'] = 'NETDRMS' # Means a non-Stanford build. This will set two additional macros used by make:
773 # __LOCALIZED_DEFS__ and NETDRMS_BUILD. The former is to support legacy code
774 # which incorrectly used this macro, and the latter is for future use.
775 # __LOCALIZED_DEFS__ is deprecated and should not be used in new code.
|
776 arta 1.1
777 try:
778 with open(cfgfile, 'r') as fin:
|
779 arta 1.3 # Process configuration parameters
|
780 arta 1.6 rv = parseConfig(fin, keymap, addenda, defs, cDefs, mDefsGen, mDefsMake, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection)
781 if not rv:
|
782 arta 1.3 # Must add a parameter for the SUMS_MANAGER UID (for some reason). This must be done after the
783 # config file is processed since an input to getMgrUIDLine() is one of the config file's
784 # parameter values.
|
785 arta 1.1 uidParam = {}
786 rv = getMgrUIDLine(defs, uidParam)
787 if rv == bool(0):
|
788 arta 1.6 rv = parseConfig(None, keymap, uidParam, defs, cDefs, mDefsGen, None, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection)
|
789 arta 1.3 # Configure the compiler-selection make variables.
|
790 arta 1.6 if not rv:
|
791 arta 1.4 rv = configureComps(defs, mDefsComps)
|
792 arta 1.1
|
793 arta 1.3 # Write out the parameter files.
|
794 arta 1.6 if not rv:
795 rv = writeParamsFiles(base, cfile, mfile, pfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection)
796 # Write out the project-specific make files (make_basic.mk, Rules.mk, and target.mk).
797 if not rv:
798 rv = writeProjFiles(pCfile, pMfile, pRfile, pTfile, projCfg, projMkRules, projRules, projTarget)
|
799 arta 1.1 except IOError as exc:
|
800 arta 1.6 print(exc.strerror, file=sys.stderr)
801 print('Unable to read configuration file ' + cfgfile + '.', file=sys.stderr)
|
802 arta 1.3 except Exception as exc:
803 type, msg = exc.args
804 if type == 'unexpectedIccRet':
805 print('icc -V returned this unexpected message:\n' + msg, file=sys.stderr)
806 rv = bool(1)
807 elif type == 'unexpectedGccRet':
808 print('gcc -v returned this unexpected message:\n' + msg, file=sys.stderr)
809 rv = bool(1)
810 elif type == 'unexpectedIfortRet':
811 print('ifort -V returned this unexpected message:\n' + msg, file=sys.stderr)
812 rv = bool(1)
813 elif type == 'unexpectedGfortranRet':
814 print('gfortran -v returned this unexpected message:\n' + msg, file=sys.stderr)
815 rv = bool(1)
816 else:
817 # re-raise the exception
818 raise
|
819 arta 1.1
|
820 arta 1.2 return rv
|
821 arta 1.1
|
822 arta 1.6 def configureSdp(cfgfile, cfile, mfile, pfile, pCfile, pMfile, pRfile, pTfile, base):
|
823 arta 1.1 rv = bool(0)
824
825 defs = {}
826 cDefs = list()
|
827 arta 1.5 mDefsGen = list()
828 mDefsMake = list()
|
829 arta 1.6 projCfg = list()
830 projMkRules = list()
831 projRules = list()
832 projTarget = list()
|
833 arta 1.5 mDefsComps = list()
|
834 arta 1.1 perlConstSection = list()
835 perlInitSection = list()
|
836 arta 1.2 addenda = {}
837
|
838 arta 1.6 # There are three parameters that were not included in the original config.local parameter set, for some reason.
839 # Due to this omission, then are not configurable, and must be set in the script.
|
840 arta 1.5 addenda['a:USER'] = 'NULL'
841 addenda['a:PASSWD'] = 'NULL'
842 addenda['p:DSDS_SUPPORT'] = '1'
|
843 arta 1.6
844 # This parameter is not configurable. BUILD_TYPE is used to distinguish between a NetDRMS and an JSOC-SDP build.
|
845 arta 1.5 addenda['a:BUILD_TYPE'] = 'JSOC_SDP' # Means a Stanford build. This will set one additional macro used by make: JSOC_SDP_BUILD.
|
846 arta 1.1
847 try:
848 with open(cfgfile, 'r') as fin:
|
849 arta 1.6 rv = parseConfig(fin, None, addenda, defs, cDefs, mDefsGen, mDefsMake, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection)
850
851 if not rv:
|
852 arta 1.2 # Must add a parameter for the SUMS_MANAGER UID (for some reason)
853 uidParam = {}
854 rv = getMgrUIDLine(defs, uidParam)
|
855 arta 1.6 if not rv:
856 rv = parseConfig(None, None, uidParam, defs, cDefs, mDefsGen, None, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection)
857
|
858 arta 1.5 # Configure the compiler-selection make variables.
|
859 arta 1.6 if not rv:
|
860 arta 1.5 rv = configureComps(defs, mDefsComps)
|
861 arta 1.3
|
862 arta 1.6 # Write out the parameter files.
863 if not rv:
864 rv = writeParamsFiles(base, cfile, mfile, pfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection)
865
866 # Write out the project-specific make files (make_basic.mk, Rules.mk, and target.mk).
867 if not rv:
868 rv = writeProjFiles(pCfile, pMfile, pRfile, pTfile, projCfg, projMkRules, projRules, projTarget)
|
869 arta 1.1 except IOError as exc:
|
870 arta 1.6 print(exc.strerror, file=sys.stderr)
871 print('Unable to read configuration file ' + cfgfile + '.', file=sys.stderr)
|
872 arta 1.5 except Exception as exc:
|
873 arta 1.6 type = exc.args[0]
|
874 arta 1.5 if type == 'unexpectedIccRet':
|
875 arta 1.6 msg = exc.args[1]
|
876 arta 1.5 print('icc -V returned this unexpected message:\n' + msg, file=sys.stderr)
877 rv = bool(1)
878 elif type == 'unexpectedGccRet':
|
879 arta 1.6 msg = exc.args[1]
|
880 arta 1.5 print('gcc -v returned this unexpected message:\n' + msg, file=sys.stderr)
881 rv = bool(1)
882 elif type == 'unexpectedIfortRet':
|
883 arta 1.6 msg = exc.args[1]
|
884 arta 1.5 print('ifort -V returned this unexpected message:\n' + msg, file=sys.stderr)
885 rv = bool(1)
886 elif type == 'unexpectedGfortranRet':
|
887 arta 1.6 msg = exc.args[1]
|
888 arta 1.5 print('gfortran -v returned this unexpected message:\n' + msg, file=sys.stderr)
889 rv = bool(1)
890 else:
891 # re-raise the exception
892 raise
893
|
894 arta 1.2 return rv
895
|
896 arta 1.1 # Beginning of program
897 rv = RET_SUCCESS
898 net = bool(1)
899
900 # Parse arguments
901 if __name__ == "__main__":
902 optD = GetArgs(sys.argv[1:])
903
904 if not(optD is None):
905 # Ensure we are configuring a DRMS tree
906 cdir = os.path.realpath(os.getcwd())
907 versfile = cdir + '/base/' + VERS_FILE
908
909 if not os.path.isfile(versfile):
910 rv = RET_NOTDRMS
911
912 # Determine whether we are localizing a Stanford build, or a NetDRMS build. If configsdp.txt exists, then
913 # it is a Stanford build, otherwise it is a NetDRMS build.
914 if rv == RET_SUCCESS:
915 stanfordFile = cdir + '/' + SDP_CFG
916 if os.path.isfile(stanfordFile):
917 arta 1.1 net = bool(0)
918
919 cfile = optD['dir'] + '/' + optD['base'] + '.h'
920 mfile = optD['dir'] + '/' + optD['base'] + '.mk'
921 pfile = optD['dir'] + '/' + optD['base'] + '.pm'
|
922 arta 1.6 pCfile = optD['dir'] + '/configure'
923 pMfile = optD['dir'] + '/make_basic.mk'
924 pRfile = optD['dir'] + '/Rules.mk'
925 pTfile = optD['dir'] + '/target.mk'
|
926 arta 1.1
927 if net:
928 try:
929 with open(NET_CFGMAP, 'r') as fin:
930 regexpComm = re.compile(r"^\s*#")
931 regexp = re.compile(r"^\s*(\S+)\s+(\w:\S+)")
932 # Must map from config.local namespace to DRMS namespace (e.g., the names used for the C macros)
933 keymap = {}
934 for line in fin:
935 matchobj = regexpComm.match(line)
936 if not matchobj is None:
937 # Skip comment line
938 continue
939
940 matchobj = regexp.match(line)
941 if not(matchobj is None):
942 # We have a key-value line
943 key = matchobj.group(1)
944 val = matchobj.group(2)
945 keymap[key] = val
946 except OSError:
947 arta 1.1 sys.stderr.write('Unable to read configuration map-file ' + NET_CFGMAP + '.')
948 rv = bool(1)
949
950 # We also need to set the UID of the SUMS manager. We have the name of the
951 # SUMS manager (it is in the configuration file)
|
952 arta 1.6 configureNet(NET_CFG, cfile, mfile, pfile, pCfile, pMfile, pRfile, pTfile, optD['base'], keymap)
|
953 arta 1.1 else:
|
954 arta 1.8 # A Stanford user can override the parameters in configsdp.txt by copying that file to config.local,
955 # and then editing config.local. So, if config.local exists, use that.
956 if os.path.isfile(cdir + '/' + NET_CFG):
957 configureSdp(NET_CFG, cfile, mfile, pfile, pCfile, pMfile, pRfile, pTfile, optD['base'])
958 else:
959 configureSdp(SDP_CFG, cfile, mfile, pfile, pCfile, pMfile, pRfile, pTfile, optD['base'])
|
Return to
localize.py
CVS log
Up to [Development] / JSOC