Return to
localize.py
CVS log
Up to [Development] / JSOC
|
Return to
localize.py
CVS log
|
Up to [Development] / JSOC
|
|
File: [Development] / JSOC / localize.py
(download)
/
(as text)
Revision: 1.29, Fri Aug 14 16:41:25 2020 UTC (2 years, 9 months ago) by arta Branch: MAIN CVS Tags: Ver_9-41, NetDRMS_Ver_9-41 Changes since 1.28: +51 -3 lines add get_required() - raises new exception if arg is missing; add code to set parameters in DRMSParams instance object |
#!/usr/bin/env python
# When run with the -s flag, localize.py configures the SUMS-server component of NetDRMS.
import sys
import getopt
import re
import os
import stat
import filecmp
import xml.etree.ElementTree as ET
from subprocess import check_output, CalledProcessError
# Constants
VERS_FILE = 'jsoc_version.h'
SDP_CFG = 'configsdp.txt'
NET_CFG = 'config.local'
NET_CFGMAP = 'config.local.map'
RET_SUCCESS = 0
RET_NOTDRMS = 1
PREFIX = """# This file was auto-generated by localize.py. Please do not edit it directly (running
# configure will run localize.py, which will then overwrite any edits manually performed).
"""
C_PREFIX = """/* This file was auto-generated by localize.py. Please do not edit it directly (running
* configure will run localize.py, which will then overwrite any edits manually performed). */
"""
PERL_BINPATH = '#!/usr/bin/perl\n'
PERL_INTIAL = """package drmsparams;
use warnings;
use strict;
"""
PERL_FXNS_A = """sub new
{
my($clname) = shift;
my($self) =
{
_paramsH => undef
};
bless($self, $clname);
$self->{_paramsH} = {};
$self->initialize();
return $self;
}
sub DESTROY
{
my($self) = shift;
}
"""
PERL_FXNS_B = """sub get
{
my($self) = shift;
my($name) = shift;
my($rv);
if (exists($self->{_paramsH}->{$name}))
{
return $self->{_paramsH}->{$name};
}
else
{
return undef;
}
}
1;"""
PY_BINPATH = '#!/usr/bin/env python\n'
PY_ERROR_CLASSES = """
class DPError(Exception):
def __init__(self):
self._msg = 'DRMS Parameter Error: '
class DPMissingParameterError(DPError):
def __init__(self, parameter):
super().__init__()
self._msg += 'missing DRMS parameter ' + parameter
def __str__(self):
return self._msg
"""
PY_FXNS_A = """
class DRMSParams(object):
def __init__(self):
self.params = {}
self.initialize()
def __del__(self):
del self.params
def initialize(self):
"""
PY_FXNS_B = """ def get(self, name):
if name in self.params:
return self.params[name]
else:
return None
"""
PY_FXNS_C = """ def get_required(self, name):
try:
value = self.params[name]
except:
raise DPMissingParameterError(name)
return value
def getBool(self, name):
if name in self.params:
return bool(self.params[name] == '1')
else:
return None
def __getattr__(self, name):
# only called if object.__getattribute__(self, name) raises; and if that is true, then we want
# to look in self.params for it, and set the instance attribute if it does exist in self.params
if name in self.params:
attr = self.params[name]
self.__setattr__(name, attr)
else:
attr = None
return attr
def __setattr__(self, name, value):
# call neither __setattr__ nor __getattr__
try:
params = object.__getattr__(self, 'params')
# put into self.params dict, overwriting if necessary
params[name] = value
except:
pass
# store in instance dict as well
object.__setattr__(self, name, value)
"""
SH_BINPATH = '#!/bin/bash\n'
SUMRM_COMMENT = """# This is the configuration file for the sum_rm program. It was auto-generated by the DRMS master configure script.
# It controls the behavior of the sum_rm program, and is loaded each time sum_rm runs. To change the
# parameter values in this configuration file, modify config.local, then re-run configure. This configuration
# file will be updated only if parameters affecting it are modified. If such changes are made to config.local,
# please make sure that the sum_rm service is disabled while configure in running.
"""
SUMRM_DOC = """# sum_rm removes end-of-life SUMS data files to prevent disk-partitions from becomming 100% full.
# sum_svc, the main SUMS service, starts this daemon when it is launched. Within an infinite loop, sum_rm sleeps
# for SLEEP number of seconds (defined below). When it awakes, it loads the sum_rm configuration file. For each SU,
# it then examines the 'effective_date' column within the sum_partn_alloc table of the SUMS database. It does so by
# sorting all SUs by effective date (this date is actually an expiration date - the SU is good until the end of that day),
# and then, starting with the SU with the oldest effective date, it deletes the SUs. It continues deleting SUs until one
# of three conditions is met:
#
# (a) The disk has at least PART_PERCENT_FREE percent free space.
# (b) All SUs have effective_date values in the future.
# (c) At least 600 SUs have been deleted (this is defined in the LIMIT statement in the file SUMLIB_RmDoX.pgc).
#
# After sum_rm stops deleteing SUs, it then sleeps for SLEEP seconds, completing the first iteration of the
# infinite loop.
"""
SUMRM_PARTN_PERCENT_FREE = """
# This is the percentage at which all disk partitions are to be kept free.
# If not specified, this defaults to 3. For example, setting PART_PERCENT_FREE = 5 will allow all partitions to
# fill to 95% full. Dividing the number of unused blocks by the total number of blocks, and rounding up,
# will result in the number specified by PART_PERCENT_FREE.
#
# NOTE : This behavior was previously controlled by the MAX_FREE_{n} family of parameters. {n} referred to the
# disk-partition number and the value of parameter MAX_FREE_{n} was the MINIMUM number of free MB in the
# partition [No clue why the word MAX was used]. As of NetDRMS 7.0, MAX_FREE_{n} has no effect."""
SUMRM_SLEEP = """
# The value is the number of seconds to sleep between iterations of the main loop in sum_rm."""
SUMRM_LOG = """
# The value is the log file (opened only at sum_rm startup; the sum_rm pid is appended to this file name)."""
SUMRM_MAIL = """
# The value is the email address of the recipient to be notified in case of a problem."""
SUMRM_NOOP = """
# If the value is set to anything other than 0, then sum_rm is rendered inactive. Otherwise, sum_rm is active."""
SUMRM_USER = """
# The value designates the linux user who is allowed to run sum_rm."""
SUMRM_NORUN = """
# This pair of paramters defines a window of time, during which sum_rm will become torpid - in this
# window, sum_rm will not scan for SUs to delete, not will it delete any SUs. Each value is an integer, 0-23, that
# represents an hour of the day. For example, if NORUN_START=8 and NORUN_STOP=10, then between 8am and 10am
# local time, sum_rm will be dormant. To disable this behavior, set both parameters to the same value."""
RULESPREFIX = """# Standard things
sp := $(sp).x
dirstack_$(sp) := $(d)
d := $(dir)
"""
RULESSUFFIX = """dir := $(d)/example
-include $(SRCDIR)/$(dir)/Rules.mk
dir := $(d)/cookbook
-include $(SRCDIR)/$(dir)/Rules.mk
dir := $(d)/myproj
-include $(SRCDIR)/$(dir)/Rules.mk
# Standard things
d := $(dirstack_$(sp))
sp := $(basename $(sp))
"""
TARGETPREFIX = """$(PROJOBJDIR):\n\t+@[ -d $@ ] || mkdir -p $@"""
ICC_MAJOR = 9
ICC_MINOR = 0
GCC_MAJOR = 3
GCC_MINOR = 0
IFORT_MAJOR = 9
IFORT_MINOR = 0
GFORT_MAJOR = 4
GFORT_MINOR = 2
# Read arguments
# d - localization directory
# b - base name of all parameter files (e.g., -b drmsparams --> drmsparams.h, drmsparams.mk, drmsparams.pm, etc.)
# s - configure a NetDRMS server (i.e., SUMS server). Otherwise, do client configuration only. A server
# configuration will modify something that only the production user has access to. An example is the sum_rm.cfg
# file. To modify that file, the user running configure must be running configure on the SUMS-server host, and
# must have write permission on sum_rm.cfg.
def GetArgs(args):
rv = bool(0)
optD = {}
try:
opts, remainder = getopt.getopt(args, "hd:b:s",["dir=", "base="])
except getopt.GetoptError:
print('Usage:')
print('localize.py [-h] -d <localization directory> -b <parameter file base>')
rv = bool(1)
if rv == bool(0):
for opt, arg in opts:
if opt == '-h':
print('localize.py [-h] -d <localization directory> -b <parameter file base>')
elif opt in ("-d", "--dir"):
regexp = re.compile(r"(\S+)/?")
matchobj = regexp.match(arg)
if matchobj is None:
rv = bool(1)
else:
optD['dir'] = matchobj.group(1)
elif opt in ("-b", "--base"):
optD['base'] = arg
elif opt in ("-s"):
optD['server'] = ""
else:
optD[opt] = arg
return optD
def createMacroStr(key, val, keyColLen, status):
if keyColLen < len(key):
status = bool(1)
return None
else:
nsp = keyColLen - len(key)
spaces = str()
for isp in range(nsp):
spaces += ' '
status = bool(0)
return '#define ' + key + spaces + val + '\n'
def createPerlConst(key, val, keyColLen, status):
if keyColLen < len(key):
status = bool(1)
return None
else:
nsp = keyColLen - len(key)
spaces = str()
for isp in range(nsp):
spaces += ' '
status = bool(0)
return 'use constant ' + key + ' => ' + spaces + val + ';\n'
def createPyConst(key, val, keyColLen, status):
if keyColLen < len(key):
status = bool(1)
return None
else:
nsp = keyColLen - len(key)
spaces = str()
for isp in range(nsp):
spaces += ' '
status = bool(0)
return key + ' = ' + spaces + val + '\n'
def createShConst(key, val, status):
status = bool(0)
return key + '=' + val + '\n'
def isSupportedPlat(plat):
regexp = re.compile(r"\s*(^x86_64|^ia32|^ia64|^avx)", re.IGNORECASE)
matchobj = regexp.match(plat);
if not matchobj is None:
return bool(1);
else:
return bool(0);
def processMakeParam(mDefs, key, val, platDict, machDict):
varMach = None
regexp = re.compile(r"(\S+):(\S+)")
matchobj = regexp.match(key)
if not matchobj is None:
varName = matchobj.group(1)
varMach = matchobj.group(2)
else:
varName = key
varValu = val
if varMach is None:
mDefs.extend(list('\n' + varName + ' = ' + varValu))
else:
if isSupportedPlat(varMach):
# The guard will compare varValu to $JSOC_MACHINE.
if not varMach in platDict:
platDict[varMach] = {}
platDict[varMach][varName] = varValu
else:
# The guard will compare varValu to $MACHINETYPE (this is just the hostname of the machine on which localize.py is running).
if not varMach in machDict:
machDict[varMach] = {}
machDict[varMach][varName] = varValu
def processParam(cfgfile, line, regexpQuote, regexp, keymap, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection, platDict, machDict, section):
status = 0
if ''.join(section) == 'defs' or not cfgfile:
matchobj = regexp.match(line)
if not matchobj is None:
# We have a key-value line
keyCfgSp = matchobj.group(1)
val = matchobj.group(2)
# Must map the indirect name to the actual name
if keymap:
# Map to actual name only if the keymap is not empty (which signifies NA).
if keyCfgSp in keymap:
key = keymap[keyCfgSp]
elif keyCfgSp == 'LOCAL_CONFIG_SET' or keyCfgSp == 'DRMS_SAMPLE_NAMESPACE':
# Ignore parameters that are not useful and shouldn't have been there in the first place. But
# they have been released to the world, so we have to account for them.
return bool(0)
elif not cfgfile:
# Should not be doing mapping for addenda
key = keyCfgSp
else:
raise Exception('badKeyMapKey', keyCfgSp)
else:
key = keyCfgSp
matchobj = regexpQuote.match(key)
if not matchobj is None:
quote = matchobj.group(1)
key = matchobj.group(2)
# master defs dictionary
defs[key] = val
# C header file
if quote == "q":
# Add double-quotes
cDefs.extend(list(createMacroStr(key, '"' + val + '"', 40, status)))
elif quote == "p":
# Add parentheses
cDefs.extend(list(createMacroStr(key, '(' + val + ')', 40, status)))
elif quote == "a":
# Leave as-is
cDefs.extend(list(createMacroStr(key, val, 40, status)))
else:
# Unknown quote type
raise Exception('badQuoteQual', key)
if status:
raise Exception('paramNameTooLong', key)
# Make file - val should never be quoted; just use as is
mDefsGen.extend(list('\n' + key + ' = ' + val))
# Perl file - val should ALWAYS be single-quote quoted
# Save const info to a string
perlConstSection.extend(list(createPerlConst(key, "'" + val + "'", 40, status)))
# Python file
pyConstSection.extend(list(createPyConst(key, "'" + val + "'", 40, status)))
# Shell source file
shConstSection.extend(list(createShConst(key, "'" + val + "'", status)))
if status:
raise Exception('paramNameTooLong', key)
# Save initialization information as a string. Now that we've defined
# constants (the names of which are the parameter names)
# we can refer to those in the init section. The key variable holds the
# name of the constant.
perlInitSection.extend(list("\n $self->{_paramsH}->{'" + key + "'} = " + key + ';'))
# The amount of indenting matters! This is Python.
pyInitSection.extend(list(" self.params['" + key + "'] = " + key + '\n'))
else:
# No quote qualifier
raise Exception('missingQuoteQual', key)
elif ''.join(section) == 'make' and cfgfile:
# Configure the remaining make variables defined in the __MAKE__ section of the configuration file. Third-party
# library make variables are specified in the __MAKE__ section.
matchobj = regexp.match(line)
if not matchobj is None:
# We have a key-value line
key = matchobj.group(1)
val = matchobj.group(2)
# This information is for making make variables only. We do not need to worry about quoting any values
defs[key] = val
processMakeParam(mDefsMake, key, val, platDict, machDict)
return bool(0)
# We have some extraneous line or a newline - ignore.
def processXML(xml, projRules, projTarget):
rv = bool(0)
# <projects>
root = ET.fromstring(xml)
# Iterate through each proj child.
for proj in root.iter('proj'):
# Rules.mk
nameElem = proj.find('name')
rulesStr = 'dir := $(d)/' + nameElem.text + '\n-include $(SRCDIR)/$(dir)/Rules.mk\n'
# make doesn't support logical operations in ifeq conditionals (you can't do ifeq (A AND B)),
# so we need to write:
# ifeq (A)
# ifeq (B)
# <do something>
# endif
# endif
rulesPref = '';
rulesSuff = '';
filters = proj.find('filters')
if filters is not None:
for filter in filters.findall('filter'):
rulesPref += 'ifeq ($(' + filter.find('name').text + '),' + filter.find('value').text + ')\n'
rulesSuff += 'endif\n'
if len(rulesPref) > 0 and len(rulesSuff) > 0:
projRules.extend(list(rulesPref))
projRules.extend(list(rulesStr))
projRules.extend(list(rulesSuff))
else:
projRules.extend(list(rulesStr))
# target.mk
subdirs = proj.find('subdirs')
if subdirs is not None:
for subdir in subdirs.findall('subdir'):
targetStr = '\n\t+@[ -d $@/' + nameElem.text + '/' + subdir.text + ' ] || mkdir -p $@/' + nameElem.text + '/' + subdir.text
projTarget.extend(list(targetStr))
return rv
def determineSection(line, regexpStyle, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg):
matchobj = regexpStyle.match(line)
if matchobj:
return 'style'
matchobj = regexpDefs.match(line)
if not matchobj is None:
return 'defs'
matchobj = regexpMake.match(line)
if not matchobj is None:
return 'make'
matchobj = regexpProjMkRules.match(line)
if not matchobj is None:
return 'projmkrules'
matchobj = regexpProj.match(line)
if not matchobj is None:
return 'proj'
matchobj = regexpProjCfg.match(line)
if not matchobj is None:
return 'projcfg'
return None
# defs is a dictionary containing all parameters (should they be needed in this script)
# projCfg is the list containing the configure script content.
# projMkRules is the list containing the make_basic.mk content.
# projRules is the list containing the Rules.mk content.
# projTargert is the list containing the target.mk content.
def parseConfig(fin, keymap, addenda, defs, cDefs, mDefsGen, mDefsMake, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection):
rv = bool(0)
# Open required config file (config.local)
try:
# Examine each line, looking for key=value pairs.
regexpStyle = re.compile(r"^__STYLE__")
regexpDefs = re.compile(r"^__DEFS__")
regexpMake = re.compile(r"^__MAKE__")
regexpProjMkRules = re.compile(r"__PROJ_MK_RULES__")
regexpProj = re.compile(r"^__PROJ__")
regexpProjCfg = re.compile(r"^__PROJCFG__")
regexpComm = re.compile(r"^\s*#")
regexpSp = re.compile(r"^s*$")
regexpQuote = re.compile(r"^\s*(\w):(.+)")
regexpCustMkBeg = re.compile(r"^_CUST_")
regexpCustMkEnd = re.compile(r"^_ENDCUST_")
regexpDiv = re.compile(r"^__")
regexp = re.compile(r"^\s*(\S+)\s+(\S.*)")
ignoreKeymap = False
platDict = {}
machDict = {}
xml = None
# Process the parameters in the configuration file
if not fin is None:
for line in fin:
matchobj = regexpComm.match(line)
if not matchobj is None:
# Skip comment line
continue
matchobj = regexpSp.match(line)
if not matchobj is None:
# Skip whitespace line
continue
newSection = determineSection(line, regexpStyle, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg)
if not newSection is None:
section = newSection
if not section:
raise Exception('invalidConfigFile', 'line ' + line.strip() + ' is not in any section')
if section == 'style':
# if the config.local file has new in the __STYLE__ section, then ignore the keymap and treat config.local like configsdp.txt;
# do not map from NetDRMS config.local parameter names to configsdp.txt names
for line in fin:
matchobj = regexpDiv.match(line)
if matchobj:
break;
if line.strip(' \n').lower() == 'new' and keymap:
ignoreKeymap = True
newSection = determineSection(line, regexpStyle, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg)
if not newSection is None:
section = newSection
continue
elif section == 'make':
# There are some blocks of lines in the __MAKE__ section that must be copied ver batim to the output make file.
# The blocks are defined by _CUST_/_ENDCUST_ tags.
matchobj = regexpCustMkBeg.match(line)
if not matchobj is None:
mDefsMake.extend(list('\n'))
for line in fin:
matchobj = regexpCustMkEnd.match(line)
if not matchobj is None:
break;
mDefsMake.extend(list(line))
newSection = determineSection(line, regexpStyle, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg)
if not newSection is None:
section = newSection
continue
# Intentional fall through to next if statement
if section == 'defs' or section == 'make':
iscfg = bool(1)
if ignoreKeymap:
keymapActual = None
else:
keymapActual = keymap
ppRet = processParam(iscfg, line, regexpQuote, regexp, keymapActual, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection, platDict, machDict, section)
if ppRet:
break
elif section == 'projcfg':
# Copy the line ver batim to the projCfg list (configure)
for line in fin:
matchobj = regexpDiv.match(line)
if not matchobj is None:
break;
projCfg.extend(list(line))
newSection = determineSection(line, regexpStyle, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg)
if not newSection is None:
section = newSection
continue
elif section == 'projmkrules':
# Copy the line ver batim to the projMkRules list (make_basic.mk)
for line in fin:
matchobj = regexpDiv.match(line)
if not matchobj is None:
break;
projMkRules.extend(list(line))
newSection = determineSection(line, regexpStyle, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg)
if not newSection is None:
section = newSection
continue
elif section == 'proj':
# Must parse xml and use the project-specific information to populate the Rules.mk and target.mk files.
# Collect all xml lines for now, then process after file-read loop.
if xml is None:
# The first time through this section, line is the config.local div, __PROJ__. Discard that.
xml = ''
continue
else:
xml += line
else:
# Unknown section
raise Exception('unknownSection', section)
except Exception as exc:
if len(exc.args) >= 2:
msg = exc.args[0]
else:
# re-raise the exception
raise
if msg == 'invalidConfigFile':
violator = exc.args[1]
print(violator, file=sys.stderr)
rv = bool(1)
elif msg == 'badKeyMapKey':
# If we are here, then there was a non-empty keymap, and the parameter came from
# the configuration file.
violator = exc.args[1]
print('Unknown parameter name ' + "'" + violator + "'" + ' in ' + fin.name + '.', file=sys.stderr)
rv = bool(1)
elif msg == 'badQuoteQual':
# The bad quote qualifier came from the configuration file, not the addenda, since
# we will have fixed any bad qualifiers in the addenda (which is populated by code).
violator = exc.args[1]
print('Unknown quote qualifier ' + "'" + violator + "'" + ' in ' + fin.name + '.', file=sys.stderr)
rv = bool(1)
elif msg == 'missingQuoteQual':
violator = exc.args[1]
print('Missing quote qualifier for parameter ' + "'" + violator + "'" + ' in ' + fin.name + '.', file=sys.stderr)
rv = bool(1)
elif msg == 'paramNameTooLong':
violator = exc.args[1]
print('Macro name ' + "'" + violator + "' is too long.", file=sys.stderr)
rv = bool(1)
elif msg == 'unknownSection':
violator = exc.args[1]
print('Unknown section ' + "'" + violator + "' in configuration file.", file=sys.stderr)
rv = bool(1)
else:
# re-raise the exception
raise
if not rv:
if not xml is None:
# Process xml.
rv = processXML(xml, projRules, projTarget)
# Process addenda - these are parameters that are not configurable and must be set in the
# NetDRMS build.
if not rv:
iscfg = bool(0)
for key in addenda:
item = key + ' ' + addenda[key]
if ignoreKeymap:
keymapActual = None
else:
keymapActual = keymap
ppRet = processParam(iscfg, item, regexpQuote, regexp, keymapActual, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection, platDict, machDict, 'defs')
if ppRet:
break;
# Put information collected in platDict and machDict into mDefs. Must do this here, and not in processParam, since
# we need to parse all platform-specific make variables before grouping them into platform categories.
if not rv:
for plat in platDict:
mDefsMake.extend(list('\nifeq ($(JSOC_MACHINE), linux_' + plat.lower() + ')'))
for var in platDict[plat]:
mDefsMake.extend(list('\n' + var + ' = ' + platDict[plat][var]))
mDefsMake.extend(list('\nendif\n'))
if not rv:
for mach in machDict:
mDefsMake.extend(list('\nifeq ($(MACHTYPE), ' + mach + ')'))
for var in machDict[mach]:
mDefsMake.extend(list('\n' + var + ' = ' + machDict[mach][var]))
mDefsMake.extend(list('\nendif\n'))
return rv
def getMgrUIDLine(defs, uidParam):
rv = bool(0)
cmd = 'id -u ' + defs['SUMS_MANAGER']
try:
ret = check_output(cmd, shell=True)
uidParam['q:SUMS_MANAGER_UID'] = ret.decode("utf-8")
except ValueError:
print('Unable to run cmd: ' + cmd + '.')
rv = bool(1)
except CalledProcessError:
print('Command ' + "'" + cmd + "'" + ' ran improperly.')
rv = bool(1)
return rv
def isVersion(maj, min, majDef, minDef):
res = 0
if maj > majDef or (maj == majDef and min >= minDef):
res = 1
return res
def configureComps(defs, mDefs):
rv = bool(0)
autoConfig = bool(1)
if 'AUTOSELCOMP' in defs:
autoConfig = (not defs['AUTOSELCOMP'] == '0')
if autoConfig:
hasicc = bool(0)
hasgcc = bool(0)
hasifort = bool(0)
hasgfort = bool(0)
# Try icc.
cmd = 'icc --version 2>&1'
try:
ret = check_output(cmd, shell=True)
ret = ret.decode("utf-8")
except CalledProcessError:
print('Command ' + "'" + cmd + "'" + ' ran improperly.')
rv = bool(1)
if not rv:
regexp = re.compile(r"\s*\S+\s+\S+\s+(\d+)[.](\d+)", re.DOTALL)
matchobj = regexp.match(ret)
if matchobj is None:
raise Exception('unexpectedIccRet', ret)
else:
major = matchobj.group(1)
minor = matchobj.group(2)
if isVersion(int(major), int(minor), ICC_MAJOR, ICC_MINOR):
hasicc = bool(1)
# Try gcc.
if not hasicc:
rv = bool(0)
cmd = 'gcc -v 2>&1'
try:
ret = check_output(cmd, shell=True)
ret = ret.decode("utf-8")
except CalledProcessError:
print('Command ' + "'" + cmd + "'" + ' ran improperly.')
rv = bool(1)
if not rv:
regexp = re.compile(r".+gcc\s+version\s+(\d+)\.(\d+)", re.DOTALL)
matchobj = regexp.match(ret)
if matchobj is None:
raise Exception('unexpectedGccRet', ret)
else:
major = matchobj.group(1)
minor = matchobj.group(2)
if isVersion(int(major), int(minor), GCC_MAJOR, GCC_MINOR):
hasgcc = bool(1)
# Try ifort.
rv = bool(0)
cmd = 'ifort --version 2>&1'
try:
ret = check_output(cmd, shell=True)
ret = ret.decode("utf-8")
except CalledProcessError:
print('Command ' + "'" + cmd + "'" + ' ran improperly.')
rv = bool(1)
if not rv:
regexp = re.compile(r"\s*\S+\s+\S+\s+(\d+)\.(\d+)", re.DOTALL)
matchobj = regexp.match(ret)
if matchobj is None:
raise Exception('unexpectedIfortRet', ret)
else:
major = matchobj.group(1)
minor = matchobj.group(2)
if isVersion(int(major), int(minor), IFORT_MAJOR, IFORT_MINOR):
hasifort = bool(1)
# Try gfortran
if not hasifort:
rv = bool(0)
cmd = 'gfortran -v 2>&1'
try:
ret = check_output(cmd, shell=True)
ret = ret.decode("utf-8")
except CalledProcessError:
print('Command ' + "'" + cmd + "'" + ' ran improperly.')
rv = bool(1)
if not rv:
regexp = re.compile(r".+gcc\s+version\s+(\d+)\.(\d+)", re.DOTALL)
matchobj = regexp.match(ret)
if matchobj is None:
raise Exception('unexpectedGfortranRet', ret)
else:
major = matchobj.group(1)
minor = matchobj.group(2)
if isVersion(int(major), int(minor), GFORT_MAJOR, GFORT_MINOR):
hasgfort = bool(1)
# Append the compiler make variables to the make file
rv = bool(0)
if not hasicc and not hasgcc:
print('Fatal error: Acceptable C compiler not found! You will be unable to build the DRMS library.', file=sys.stderr)
rv = bool(1)
elif hasicc:
mDefs.extend(list('\nCOMPILER = icc'))
# mDefs.extend(list('\nICC_VERSION = blah'))
else:
mDefs.extend(list('\nCOMPILER = gcc'))
if not hasifort and not hasgfort:
print('Warning: Acceptable Fortran compiler not found! Fortran interface will not be built, and you will be unable to build Fortran modules.', file=sys.stderr)
elif hasifort:
mDefs.extend(list('\nFCOMPILER = ifort'))
else:
mDefs.extend(list('\nFCOMPILER = gfortran'))
# Environment overrides. These get written, regardless of the disposition of auto-configuration.
mDefs.extend(list('\nifneq ($(JSOC_COMPILER),)\n COMPILER = $(JSOC_COMPILER)\nendif'))
mDefs.extend(list('\nifneq ($(JSOC_FCOMPILER),)\n FCOMPILER = $(JSOC_FCOMPILER)\nendif'))
return rv
def writeParamsFiles(base, cfile, mfile, pfile, pyfile, shfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection):
rv = bool(0)
# Merge mDefsGen, mDefsMake, and mDefsComps into a single string with compiler configuration first, general parameters next, then
# make-specific make variables (e.g., third-party library information) last.
mDefs = '\n# Compiler Selection\n' + ''.join(mDefsComps) + '\n\n# General Parameters\n' + ''.join(mDefsGen) + '\n\n# Parameters to Configure make\n' + ''.join(mDefsMake)
try:
with open(cfile, 'w') as cout, open(mfile, 'w') as mout, open(pfile, 'w') as pout, open(pyfile, 'w') as pyout, open(shfile, 'w') as shout:
# C file of macros
print(C_PREFIX, file=cout)
print('/* This file contains a set of preprocessor macros - one for each configuration parameter. */\n', file=cout)
buf = '__' + base.upper() + '_H'
print('#ifndef ' + buf, file=cout)
print('#define ' + buf, file=cout)
print(''.join(cDefs), file=cout)
print('#endif', file=cout)
# Make file of make variables
print(PREFIX, file=mout)
print('# This file contains a set of make-variable values. The first section contains compiler-selection variables, the second contains general configuration variables, and the third section contains variables that configure how make is run.', file=mout)
print(mDefs, file=mout)
# Perl module
print(PERL_BINPATH, file=pout)
print(PREFIX, file=pout)
print('# This file contains a set of constants - one for each configuration parameter.\n', file=pout)
print(PERL_INTIAL, file=pout)
print(''.join(perlConstSection), file=pout)
print(PERL_FXNS_A, file=pout)
print('sub initialize', file=pout)
print('{', file=pout)
print(' my($self) = shift;', file=pout, end='')
print('', file=pout)
print(''.join(perlInitSection), file=pout)
print('}\n', file=pout)
print(PERL_FXNS_B, file=pout)
# Python module
print(PY_BINPATH, file=pyout)
print(PREFIX, file=pyout)
print('# This file contains a set of constants - one for each configuration parameter.\n', file=pyout)
print(''.join(pyConstSection), file=pyout)
print(PY_ERROR_CLASSES, file=pyout)
print(PY_FXNS_A, file=pyout, end='')
print(''.join(pyInitSection), file=pyout)
print(PY_FXNS_B, file=pyout)
print(PY_FXNS_C, file=pyout)
# Shell (bash) source file
print(SH_BINPATH, file=shout)
print(PREFIX, file=shout)
print('# This file contains a set of variable assignments - one for each configuration parameter.\n', file=shout)
print(''.join(shConstSection), file=shout)
except IOError as exc:
type, value, traceback = sys.exc_info()
print(exc.strerror, file=sys.stderr)
print('Unable to open ' + "'" + value.filename + "'.", file=sys.stderr)
rv = bool(1)
return rv
def writeProjFiles(pCfile, pMfile, pRfile, pTfile, projCfg, projMkRules, projRules, projTarget):
rv = bool(0)
try:
if projCfg:
with open(pCfile, 'w') as cout:
# configure
print(''.join(projCfg), file=cout)
if projMkRules:
with open(pMfile, 'w') as mout:
# make_basic.mk
print(PREFIX, file=mout)
print(''.join(projMkRules), file=mout)
if projRules:
with open(pRfile, 'w') as rout:
# Rules.mk
print(PREFIX, file=rout)
print(''.join(projRules), file=rout)
if projTarget:
with open(pTfile, 'w') as tout:
# target.mk
print(PREFIX, file=tout)
print(''.join(projTarget), file=tout)
except IOError as exc:
type, value, traceback = sys.exc_info()
print(exc.strerror, file=sys.stderr)
print('Unable to open ' + "'" + value.filename + "'.", file=sys.stderr)
rv = bool(1)
if not rv:
if os.path.exists(pCfile):
try:
os.chmod(pCfile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IROTH)
except OSError as exc:
type, value, traceback = sys.exc_info()
print('Unable to chmod file ' + "'" + value.filename + "'.", file=sys.stderr)
print(exc.strerror, file=sys.stderr)
rv = bool(1)
return rv
def generateSumRmCfg(defs):
rv = bool(0)
# ACK! Remember that Rick renamed these parameters. The ones in config.local are the aliases - do not use those.
# Use the ones that those map to (defined in config.local.map).
cFileTmp = defs['SUMLOG_BASEDIR'] + '/' + '.sum_rm.cfg.tmp'
cFile = defs['SUMLOG_BASEDIR'] + '/' + 'sum_rm.cfg'
# Write a temporary file sum_rm configuration file.
try:
with open(cFileTmp, 'w') as fout:
# Print comment at the top of the configuration file.
print(SUMRM_COMMENT, file=fout)
print(SUMRM_DOC, file=fout)
print(SUMRM_PARTN_PERCENT_FREE, file=fout)
if 'SUMRM_PART_PERCENT_FREE' in defs:
print('PART_PERCENT_FREE=' + defs['SUMRM_PART_PERCENT_FREE'], file=fout)
else:
print('PART_PERCENT_FREE=3', file=fout)
print(SUMRM_SLEEP, file=fout)
if 'SUMRM_SLEEP' in defs:
print('SLEEP=' + defs['SUMRM_SLEEP'], file=fout)
else:
print('SLEEP=300', file=fout)
print(SUMRM_LOG, file=fout)
if 'SUMRM_LOG' in defs:
print('LOG=' + defs['SUMRM_LOG'], file=fout)
else:
print('LOG=/tmp/sum_rm.log', file=fout)
print(SUMRM_MAIL, file=fout)
# No default for mail - don't send nothing to nobody unless the operator has asked for notifications.
if 'SUMRM_MAIL' in defs:
print('MAIL=' + defs['SUMRM_MAIL'], file=fout)
else:
print('# MAIL=president@whitehouse.gov', file=fout)
print(SUMRM_NOOP, file=fout)
if 'SUMRM_NOOP' in defs:
print('NOOP=' + defs['SUMRM_NOOP'], file=fout)
else:
print('NOOP=0', file=fout)
print(SUMRM_USER, file=fout)
if 'SUMRM_USER' in defs:
print('USER=' + defs['SUMRM_USER'], file=fout)
else:
print('USER=production', file=fout)
print(SUMRM_NORUN, file=fout)
# Default norun window is to have no such window. This can be accomplished by simply not providing either argument.
if 'SUMRM_NORUN_START' in defs or 'SUMRM_NORUN_STOP' in defs:
if 'SUMRM_NORUN_START' in defs:
print('NORUN_START=' + defs['SUMRM_NORUN_START'], file=fout)
else:
print('NORUN_START=0', file=fout)
if 'SUMRM_NORUN_STOP' in defs:
print('NORUN_STOP=' + defs['SUMRM_NORUN_STOP'], file=fout)
else:
print('NORUN_STOP=0', file=fout)
else:
print('# NORUN_START=0', file=fout)
print('# NORUN_STOP=0', file=fout)
except OSError:
print('Unable to open sum_rm temporary configuration file ' + cFileTmp + 'for writing.', file=sys.stderr)
rv = bool(1)
# If the content of the temporary file differs from the content of the existing configuration file, then overwrite
# the original file. Otherwise, delete the temporary file
if not rv:
try:
if filecmp.cmp(cFile, cFileTmp):
# Files identical - delete temporary file
try:
os.remove(cFileTmp)
except OSError as exc:
print('Unable to remove temporary file ' + exc.filename + '.', file=sys.stderr)
print(exc.strerr, file=sys.stderr)
else:
# Replace original with temporary file
try:
os.rename(cFileTmp, cFile)
except OSError as exc:
print('Unable to update sum_rm configuration file ' + cFile + '.', file=sys.stderr)
print(exc.strerr, file=sys.stderr)
rv = bool(1)
except OSError as exc:
# One of the files doesn't exist.
if exc.filename == cFile:
# We are ok - there might be no configuration file yet.
# Replace original with temporary file
try:
os.rename(cFileTmp, cFile)
except OSError as exc:
print('Unable to update sum_rm configuration file ' + cFile + '.', file=sys.stderr)
print(exc.strerr, file=sys.stderr)
rv = bool(1)
else:
# There is a problem with the temp file - bail.
print('Unable to update sum_rm configuration file ' + cFile + '.', file=sys.stderr)
print(exc.strerr, file=sys.stderr)
rv = bool(1)
return rv
def configureNet(cfgfile, cfile, mfile, pfile, pyfile, shfile, pCfile, pMfile, pRfile, pTfile, base, keymap, createSumRmCfg):
rv = bool(0)
defs = {}
cDefs = list()
mDefsGen = list()
mDefsMake = list()
mDefsComps = list()
projCfg = list()
projMkRules = list()
projRules = list()
projTarget = list()
perlConstSection = list()
perlInitSection = list()
pyConstSection = list()
pyInitSection = list()
shConstSection = list()
addenda = {}
# There are three parameters that were not included in the original config.local parameter set, for some reason.
# Due to this omission, then are not configurable, and must be set in the script.
addenda['a:USER'] = 'NULL'
addenda['a:PASSWD'] = 'NULL'
addenda['p:DSDS_SUPPORT'] = '0'
# This parameter is not configurable. BUILD_TYPE is used to distinguish between a NetDRMS and an JSOC-SDP build.
addenda['a:BUILD_TYPE'] = 'NETDRMS' # Means a non-Stanford build. This will set two additional macros used by make:
# __LOCALIZED_DEFS__ and NETDRMS_BUILD. The former is to support legacy code
# which incorrectly used this macro, and the latter is for future use.
# __LOCALIZED_DEFS__ is deprecated and should not be used in new code.
try:
with open(cfgfile, 'r') as fin:
# Process configuration parameters
# Always create a Rules.mk and target.mk, even if no proj XML is provided. All builds should have the proj/example and
# proj/cookbook directories. The required make information is in RULESPREFIX, TARGETPREFIX, and RULESSUFFIX. RULESSUFFIX
# must be added after the xml has been parsed.
projRules.extend(list(RULESPREFIX))
projTarget.extend(list(TARGETPREFIX))
rv = parseConfig(fin, keymap, addenda, defs, cDefs, mDefsGen, mDefsMake, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
if not rv:
projRules.extend(RULESSUFFIX)
# Must add a parameter for the SUMS_MANAGER UID (for some reason). This must be done after the
# config file is processed since an input to getMgrUIDLine() is one of the config file's
# parameter values.
uidParam = {}
rv = getMgrUIDLine(defs, uidParam)
if rv == bool(0):
rv = parseConfig(None, keymap, uidParam, defs, cDefs, mDefsGen, None, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
# Configure the compiler-selection make variables.
if not rv:
rv = configureComps(defs, mDefsComps)
# Write out the parameter files.
if not rv:
rv = writeParamsFiles(base, cfile, mfile, pfile, pyfile, shfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
# Write out the project-specific make files (make_basic.mk, Rules.mk, and target.mk).
if not rv:
rv = writeProjFiles(pCfile, pMfile, pRfile, pTfile, projCfg, projMkRules, projRules, projTarget)
# Write out the sum_rm.cfg file.
if not rv and createSumRmCfg:
rv = generateSumRmCfg(defs)
except IOError as exc:
print(exc.strerror, file=sys.stderr)
print('Unable to read configuration file ' + cfgfile + '.', file=sys.stderr)
except Exception as exc:
if len(exc.args) >= 2:
type, msg = exc.args
else:
# re-raise the exception
raise
if type == 'unexpectedIccRet':
print('icc -V returned this unexpected message:\n' + msg, file=sys.stderr)
rv = bool(1)
elif type == 'unexpectedGccRet':
print('gcc -v returned this unexpected message:\n' + msg, file=sys.stderr)
rv = bool(1)
elif type == 'unexpectedIfortRet':
print('ifort -V returned this unexpected message:\n' + msg, file=sys.stderr)
rv = bool(1)
elif type == 'unexpectedGfortranRet':
print('gfortran -v returned this unexpected message:\n' + msg, file=sys.stderr)
rv = bool(1)
else:
# re-raise the exception
raise
return rv
def configureSdp(cfgfile, cfile, mfile, pfile, pyfile, shfile, pCfile, pMfile, pRfile, pTfile, base):
rv = bool(0)
defs = {}
cDefs = list()
mDefsGen = list()
mDefsMake = list()
projCfg = list()
projMkRules = list()
projRules = list()
projTarget = list()
mDefsComps = list()
perlConstSection = list()
perlInitSection = list()
pyConstSection = list()
pyInitSection = list()
shConstSection = list()
addenda = {}
# There are three parameters that were not included in the original config.local parameter set, for some reason.
# Due to this omission, then are not configurable, and must be set in the script.
addenda['a:USER'] = 'NULL'
addenda['a:PASSWD'] = 'NULL'
addenda['p:DSDS_SUPPORT'] = '1'
# This parameter is not configurable. BUILD_TYPE is used to distinguish between a NetDRMS and an JSOC-SDP build.
addenda['a:BUILD_TYPE'] = 'JSOC_SDP' # Means a Stanford build. This will set one additional macro used by make: JSOC_SDP_BUILD.
try:
with open(cfgfile, 'r') as fin:
# Always create a Rules.mk and target.mk, even if no proj XML is provided. All builds should have the proj/example and
# proj/cookbook directories. The required make information is in RULESPREFIX, TARGETPREFIX, and RULESSUFFIX. RULESSUFFIX
# must be added after the xml has been parsed.
projRules.extend(list(RULESPREFIX))
projTarget.extend(list(TARGETPREFIX))
rv = parseConfig(fin, None, addenda, defs, cDefs, mDefsGen, mDefsMake, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
if not rv:
projRules.extend(RULESSUFFIX)
# Must add a parameter for the SUMS_MANAGER UID (for some reason)
uidParam = {}
rv = getMgrUIDLine(defs, uidParam)
if not rv:
rv = parseConfig(None, None, uidParam, defs, cDefs, mDefsGen, None, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
# Configure the compiler-selection make variables.
if not rv:
rv = configureComps(defs, mDefsComps)
# Write out the parameter files.
if not rv:
rv = writeParamsFiles(base, cfile, mfile, pfile, pyfile, shfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
# Write out the project-specific make files (make_basic.mk, Rules.mk, and target.mk).
if not rv:
rv = writeProjFiles(pCfile, pMfile, pRfile, pTfile, projCfg, projMkRules, projRules, projTarget)
# At Stanford, skip the creation of the sum_rm configuration file. config.local will still
# have the SUMRM parameters, but they will not be used.
except IOError as exc:
print(exc.strerror, file=sys.stderr)
print('Unable to read configuration file ' + cfgfile + '.', file=sys.stderr)
except Exception as exc:
if len(exc.args) >= 2:
type = exc.args[0]
else:
# re-raise the exception
raise
if type == 'unexpectedIccRet':
msg = exc.args[1]
print('icc -V returned this unexpected message:\n' + msg, file=sys.stderr)
rv = bool(1)
elif type == 'unexpectedGccRet':
msg = exc.args[1]
print('gcc -v returned this unexpected message:\n' + msg, file=sys.stderr)
rv = bool(1)
elif type == 'unexpectedIfortRet':
msg = exc.args[1]
print('ifort -V returned this unexpected message:\n' + msg, file=sys.stderr)
rv = bool(1)
elif type == 'unexpectedGfortranRet':
msg = exc.args[1]
print('gfortran -v returned this unexpected message:\n' + msg, file=sys.stderr)
rv = bool(1)
else:
# re-raise the exception
raise
return rv
# Beginning of program
rv = RET_SUCCESS
net = bool(1)
# Parse arguments
if __name__ == "__main__":
optD = GetArgs(sys.argv[1:])
if not(optD is None):
# Ensure we are configuring a DRMS tree
cdir = os.path.realpath(os.getcwd())
versfile = cdir + '/base/' + VERS_FILE
if not os.path.isfile(versfile):
rv = RET_NOTDRMS
# Determine whether we are localizing a Stanford build, or a NetDRMS build. If configsdp.txt exists, then
# it is a Stanford build, otherwise it is a NetDRMS build.
if rv == RET_SUCCESS:
stanfordFile = cdir + '/' + SDP_CFG
if os.path.isfile(stanfordFile):
net = bool(0)
cfile = optD['dir'] + '/' + optD['base'] + '.h'
mfile = optD['dir'] + '/' + optD['base'] + '.mk'
pfile = optD['dir'] + '/' + optD['base'] + '.pm'
pyfile = optD['dir'] + '/' + optD['base'] + '.py'
shfile = optD['dir'] + '/' + optD['base'] + '.sh'
pCfile = optD['dir'] + '/configure'
pMfile = optD['dir'] + '/make_basic.mk'
pRfile = optD['dir'] + '/Rules.mk'
pTfile = optD['dir'] + '/target.mk'
if net:
try:
with open(NET_CFGMAP, 'r') as fin:
regexpComm = re.compile(r"^\s*#")
regexp = re.compile(r"^\s*(\S+)\s+(\w:\S+)")
# Must map from config.local namespace to DRMS namespace (e.g., the names used for the C macros)
keymap = {}
for line in fin:
matchobj = regexpComm.match(line)
if not matchobj is None:
# Skip comment line
continue
matchobj = regexp.match(line)
if not(matchobj is None):
# We have a key-value line
key = matchobj.group(1)
val = matchobj.group(2)
keymap[key] = val
except OSError:
sys.stderr.write('Unable to read configuration map-file ' + NET_CFGMAP + '.')
rv = bool(1)
# We also need to set the UID of the SUMS manager. We have the name of the
# SUMS manager (it is in the configuration file)
configureNet(NET_CFG, cfile, mfile, pfile, pyfile, shfile, pCfile, pMfile, pRfile, pTfile, optD['base'], keymap, 'server' in optD)
else:
# A Stanford user can override the parameters in configsdp.txt by copying that file to config.local,
# and then editing config.local. So, if config.local exists, use that.
if os.path.isfile(cdir + '/' + NET_CFG):
configureSdp(NET_CFG, cfile, mfile, pfile, pyfile, shfile, pCfile, pMfile, pRfile, pTfile, optD['base'])
else:
configureSdp(SDP_CFG, cfile, mfile, pfile, pyfile, shfile, pCfile, pMfile, pRfile, pTfile, optD['base'])
| Karen Tian |
Powered by ViewCVS 0.9.4 |