(file) Return to localize.py CVS log (file) (dir) Up to [Development] / JSOC

Diff for /JSOC/localize.py between version 1.6 and 1.33

version 1.6, 2013/11/13 17:26:27 version 1.33, 2022/09/14 15:59:13
Line 1 
Line 1 
 #!/home/jsoc/bin/linux_x86_64/activepython  #!/usr/bin/env python
  
   # When run with the -s flag, localize.py configures the SUMS-server component of NetDRMS.
 import sys import sys
 import getopt import getopt
 import re import re
 import os import os
 import stat import stat
 import xml.etree.ElementTree as ET  import filecmp
 from subprocess import check_output, CalledProcessError from subprocess import check_output, CalledProcessError
   import shlex
   
  
 # Constants # Constants
 VERS_FILE = 'jsoc_version.h' VERS_FILE = 'jsoc_version.h'
Line 42  PERL_FXNS_A = """sub new
Line 45  PERL_FXNS_A = """sub new
     };     };
  
     bless($self, $clname);     bless($self, $clname);
     $self->{_paramsH} = $self->initialize();      $self->{_paramsH} = {};
       $self->initialize();
  
     return $self;     return $self;
 } }
Line 70  PERL_FXNS_B = """sub get
Line 74  PERL_FXNS_B = """sub get
 } }
 1;""" 1;"""
  
   PY_BINPATH = '#!/usr/bin/env python3\n'
   
   PY_ALL = f"__all__ = [ 'DPError', 'DPMissingParameterError', 'DRMSParams' ]"
   
   PY_ERROR_CLASSES = """
   class DPError(Exception):
       def __init__(self):
           self._msg = 'DRMS Parameter Error: '
   
   class DPMissingParameterError(DPError):
       def __init__(self, parameter):
           super().__init__()
           self._msg += 'missing DRMS parameter ' + parameter
   
       def __str__(self):
           return self._msg
   """
   
   PY_FXNS_A = """
   class DRMSParams(object):
       def __init__(self):
           self.params = {}
           self.initialize()
   
       def __del__(self):
           del self.params
   
       def initialize(self):
   """
   
   PY_FXNS_B = """    def get(self, name):
           if name in self.params:
               return self.params[name]
           else:
               return None
   """
   
   PY_FXNS_C = """    def get_required(self, name):
           try:
               value = self.params[name]
           except:
               raise DPMissingParameterError(name)
   
           return value
   
       def getBool(self, name):
           if name in self.params:
               return bool(self.params[name] == '1')
           else:
               return None
   
       def __getattr__(self, name):
           # only called if object.__getattribute__(self, name) raises; and if that is true, then we want
           # to look in self.params for it, and set the instance attribute if it does exist in self.params
           if name in self.params:
               attr = self.params[name]
               self.__setattr__(name, attr)
           else:
               attr = None
   
           return attr
   
       def __setattr__(self, name, value):
           # call neither __setattr__ nor __getattr__
           try:
               params = object.__getattr__(self, 'params')
   
               # put into self.params dict, overwriting if necessary
               params[name] = value
           except:
               pass
   
           # store in instance dict as well
           object.__setattr__(self, name, value)
   """
   
   SH_BINPATH = '#!/bin/bash\n'
   
   
   SUMRM_COMMENT = """# This is the configuration file for the sum_rm program. It was auto-generated by the DRMS master configure script.
   # It controls the behavior of the sum_rm program, and is loaded each time sum_rm runs. To change the
   # parameter values in this configuration file, modify config.local, then re-run configure. This configuration
   # file will be updated only if parameters affecting it are modified. If such changes are made to config.local,
   # please make sure that the sum_rm service is disabled while configure in running.
   """
   
   SUMRM_DOC = """# sum_rm removes end-of-life SUMS data files to prevent disk-partitions from becomming 100% full.
   # sum_svc, the main SUMS service, starts this daemon when it is launched. Within an infinite loop, sum_rm sleeps
   # for SLEEP number of seconds (defined below). When it awakes, it loads the sum_rm configuration file. For each SU,
   # it then examines the 'effective_date' column within the sum_partn_alloc table of the SUMS database. It does so by
   # sorting all SUs by effective date (this date is actually an expiration date - the SU is good until the end of that day),
   # and then, starting with the SU with the oldest effective date, it deletes the SUs. It continues deleting SUs until one
   # of three conditions is met:
   #
   #   (a) The disk has at least PART_PERCENT_FREE percent free space.
   #   (b) All SUs have effective_date values in the future.
   #   (c) At least 600 SUs have been deleted (this is defined in the LIMIT statement in the file SUMLIB_RmDoX.pgc).
   #
   # After sum_rm stops deleteing SUs, it then sleeps for SLEEP seconds, completing the first iteration of the
   # infinite loop.
   """
   
   SUMRM_PARTN_PERCENT_FREE = """
   # This is the percentage at which all disk partitions are to be kept free.
   # If not specified, this defaults to 3. For example, setting PART_PERCENT_FREE = 5 will allow all partitions to
   # fill to 95% full. Dividing the number of unused blocks by the total number of blocks, and rounding up,
   # will result in the number specified by PART_PERCENT_FREE.
   #
   # NOTE : This behavior was previously controlled by the MAX_FREE_{n} family of parameters. {n} referred to the
   # disk-partition number and the value of parameter MAX_FREE_{n} was the MINIMUM number of free MB in the
   # partition [No clue why the word MAX was used]. As of NetDRMS 7.0, MAX_FREE_{n} has no effect."""
   
   SUMRM_SLEEP = """
   # The value is the number of seconds to sleep between iterations of the main loop in sum_rm."""
   
   SUMRM_LOG = """
   # The value is the log file (opened only at sum_rm startup; the sum_rm pid is appended to this file name)."""
   
   SUMRM_MAIL = """
   # The value is the email address of the recipient to be notified in case of a problem."""
   
   SUMRM_NOOP = """
   # If the value is set to anything other than 0, then sum_rm is rendered inactive. Otherwise, sum_rm is active."""
   
   SUMRM_USER = """
   # The value designates the linux user who is allowed to run sum_rm."""
   
   SUMRM_NORUN = """
   # This pair of paramters defines a window of time, during which sum_rm will become torpid - in this
   # window, sum_rm will not scan for SUs to delete, not will it delete any SUs. Each value is an integer, 0-23, that
   # represents an hour of the day. For example, if NORUN_START=8 and NORUN_STOP=10, then between 8am and 10am
   # local time, sum_rm will be dormant. To disable this behavior, set both parameters to the same value."""
  
 RULESPREFIX = """# Standard things RULESPREFIX = """# Standard things
 sp              := $(sp).x sp              := $(sp).x
Line 77  dirstack_$(sp) := $(d)
Line 213  dirstack_$(sp) := $(d)
 d               := $(dir) d               := $(dir)
 """ """
  
 RULESSUFFIX = """dir    := $(d)/example  RULESSUFFIX = """# Standard things
 -include                $(SRCDIR)/$(dir)/Rules.mk  
 dir     := $(d)/cookbook  
 -include                $(SRCDIR)/$(dir)/Rules.mk  
 dir     := $(d)/myproj  
 -include                $(SRCDIR)/$(dir)/Rules.mk  
   
 # Standard things  
 d               := $(dirstack_$(sp)) d               := $(dirstack_$(sp))
 sp              := $(basename $(sp)) sp              := $(basename $(sp))
 """ """
  
 TARGETPREFIX = """$(PROJOBJDIR):\n\t+@[ -d $@ ] || mkdir -p $@"""  
   
 ICC_MAJOR = 9 ICC_MAJOR = 9
 ICC_MINOR = 0 ICC_MINOR = 0
 GCC_MAJOR = 3 GCC_MAJOR = 3
Line 104  GFORT_MINOR = 2
Line 231  GFORT_MINOR = 2
 # Read arguments # Read arguments
 #  d - localization directory #  d - localization directory
 #  b - base name of all parameter files (e.g., -b drmsparams --> drmsparams.h, drmsparams.mk, drmsparams.pm, etc.) #  b - base name of all parameter files (e.g., -b drmsparams --> drmsparams.h, drmsparams.mk, drmsparams.pm, etc.)
 #  m - make file  #  s - configure a NetDRMS server (i.e., SUMS server). Otherwise, do client configuration only. A server
   #      configuration will modify something that only the production user has access to. An example is the sum_rm.cfg
   #      file. To modify that file, the user running configure must be running configure on the SUMS-server host, and
   #      must have write permission on sum_rm.cfg.
 def GetArgs(args): def GetArgs(args):
     rv = bool(0)     rv = bool(0)
     optD = {}     optD = {}
  
     try:     try:
         opts, remainder = getopt.getopt(args, "hd:b:",["dir=", "base="])          opts, remainder = getopt.getopt(args, "hd:b:s",["dir=", "base="])
     except getopt.GetoptError:     except getopt.GetoptError:
         print('Usage:')         print('Usage:')
         print('localize.py [-h] -d <localization directory> -b <parameter file base>')         print('localize.py [-h] -d <localization directory> -b <parameter file base>')
Line 129  def GetArgs(args):
Line 259  def GetArgs(args):
                     optD['dir'] = matchobj.group(1)                     optD['dir'] = matchobj.group(1)
             elif opt in ("-b", "--base"):             elif opt in ("-b", "--base"):
                 optD['base'] = arg                 optD['base'] = arg
               elif opt in ("-s"):
                   optD['server'] = ""
             else:             else:
                 optD[opt] = arg                 optD[opt] = arg
  
Line 158  def createPerlConst(key, val, keyColLen,
Line 290  def createPerlConst(key, val, keyColLen,
         status = bool(0)         status = bool(0)
         return 'use constant ' + key + ' => ' + spaces + val + ';\n'         return 'use constant ' + key + ' => ' + spaces + val + ';\n'
  
   def createPyConst(key, val, keyColLen, status):
       if keyColLen < len(key):
           status = bool(1)
           return None
       else:
           nsp = keyColLen - len(key)
           spaces = str()
           for isp in range(nsp):
               spaces += ' '
           status = bool(0)
           return key + ' = ' + spaces + val + '\n'
   
   def createShConst(key, val, status):
       status = bool(0)
       return key + '=' + val + '\n'
   
   
 def isSupportedPlat(plat): def isSupportedPlat(plat):
     regexp = re.compile(r"\s*(^x86_64|^ia32|^ia64|^avx)", re.IGNORECASE)     regexp = re.compile(r"\s*(^x86_64|^ia32|^ia64|^avx)", re.IGNORECASE)
     matchobj = regexp.match(plat);     matchobj = regexp.match(plat);
Line 193  def processMakeParam(mDefs, key, val, pl
Line 342  def processMakeParam(mDefs, key, val, pl
                 machDict[varMach] = {}                 machDict[varMach] = {}
             machDict[varMach][varName] = varValu             machDict[varMach][varName] = varValu
  
 def processParam(cfgfile, line, regexpQuote, regexp, keymap, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, platDict, machDict, section):  def processParam(cfgfile, line, regexpQuote, regexp, keymap, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection, platDict, machDict, section):
     status = 0     status = 0
       parameter_added = {}
       keyCfgSp  = None
       val = None
  
     if ''.join(section) == 'defs' or not cfgfile:     if ''.join(section) == 'defs' or not cfgfile:
           if type(line) is str:
         matchobj = regexp.match(line)         matchobj = regexp.match(line)
   
         if not matchobj is None:         if not matchobj is None:
             # We have a key-value line             # We have a key-value line
             keyCfgSp = matchobj.group(1)             keyCfgSp = matchobj.group(1)
             val = matchobj.group(2)             val = matchobj.group(2)
                   print(f'herer, key={keyCfgSp}, val={val}')
           else:
               keyCfgSp, val = list(line.items())
   
           if keyCfgSp is not None and val is not None:
               # We have a key-value line
               keyCfgSp = matchobj.group(1)
               val = matchobj.group(2)
  
             # Must map the indirect name to the actual name             # Must map the indirect name to the actual name
             if keymap:             if keymap:
Line 211  def processParam(cfgfile, line, regexpQu
Line 373  def processParam(cfgfile, line, regexpQu
                 elif keyCfgSp == 'LOCAL_CONFIG_SET' or keyCfgSp == 'DRMS_SAMPLE_NAMESPACE':                 elif keyCfgSp == 'LOCAL_CONFIG_SET' or keyCfgSp == 'DRMS_SAMPLE_NAMESPACE':
                     # Ignore parameters that are not useful and shouldn't have been there in the first place. But                     # Ignore parameters that are not useful and shouldn't have been there in the first place. But
                     # they have been released to the world, so we have to account for them.                     # they have been released to the world, so we have to account for them.
                     return bool(0)                      return None
                 elif not cfgfile:                 elif not cfgfile:
                     # Should not be doing mapping for addenda                     # Should not be doing mapping for addenda
                     key = keyCfgSp                     key = keyCfgSp
Line 228  def processParam(cfgfile, line, regexpQu
Line 390  def processParam(cfgfile, line, regexpQu
                 # master defs dictionary                 # master defs dictionary
                 defs[key] = val                 defs[key] = val
  
                   parameter_added[key] = val
   
                 # C header file                 # C header file
                 if quote == "q":                 if quote == "q":
                     # Add double-quotes                     # Add double-quotes
Line 252  def processParam(cfgfile, line, regexpQu
Line 416  def processParam(cfgfile, line, regexpQu
                 # Save const info to a string                 # Save const info to a string
                 perlConstSection.extend(list(createPerlConst(key, "'" + val + "'", 40, status)))                 perlConstSection.extend(list(createPerlConst(key, "'" + val + "'", 40, status)))
  
                   # Python file
                   pyConstSection.extend(list(createPyConst(key, "'" + val + "'", 40, status)))
   
                   # Shell source file
                   shConstSection.extend(list(createShConst(key, "'" + val + "'", status)))
   
                 if status:                 if status:
                     raise Exception('paramNameTooLong', key)                     raise Exception('paramNameTooLong', key)
  
Line 259  def processParam(cfgfile, line, regexpQu
Line 429  def processParam(cfgfile, line, regexpQu
                 # constants (the names of which are the parameter names)                 # constants (the names of which are the parameter names)
                 # we can refer to those in the init section. The key variable holds the                 # we can refer to those in the init section. The key variable holds the
                 # name of the constant.                 # name of the constant.
                 perlInitSection.extend(list('\n  $self->{_paramsH}->{' + key + '} = ' + "'" + val + "';"))                  perlInitSection.extend(list("\n  $self->{_paramsH}->{'" + key + "'} = " + key + ';'))
   
                   # The amount of indenting matters! This is Python.
                   pyInitSection.extend(list("        self.params['" + key + "'] = " + key + '\n'))
             else:             else:
                 # No quote qualifier                 # No quote qualifier
                 raise Exception('missingQuoteQual', key)                 raise Exception('missingQuoteQual', key)
Line 276  def processParam(cfgfile, line, regexpQu
Line 449  def processParam(cfgfile, line, regexpQu
             defs[key] = val             defs[key] = val
             processMakeParam(mDefsMake, key, val, platDict, machDict)             processMakeParam(mDefsMake, key, val, platDict, machDict)
  
     return bool(0)      return parameter_added
  
 # We have some extraneous line or a newline - ignore. # We have some extraneous line or a newline - ignore.
  
 def processXML(xml, projRules, projTarget):  def process_project_repos(project_includes):
     rv = bool(0)      print(f'[ process_project_repos ]')
       error = False
     # <projects>      ordered_files = []
     root = ET.fromstring(xml)  
       # iterate through all proj subdirectories
     # Iterate through each proj child.      for subdirectory in os.listdir('proj'):
     for proj in root.iter('proj'):          stripped_subdirectory = subdirectory.strip()
         # Rules.mk          path = os.path.join('proj', stripped_subdirectory)
         nameElem = proj.find('name')  
         rulesStr = 'dir     := $(d)/' + nameElem.text + '\n-include          $(SRCDIR)/$(dir)/Rules.mk\n'          # skip any dir in proj that does not have a Rules.mk file
           if os.path.isfile(os.path.join(path, 'Rules.mk')):
         # make doesn't support logical operations in ifeq conditionals (you can't do ifeq (A AND B)),              if os.path.isdir(path):
         # so we need to write:                  ordered_files.append(stripped_subdirectory)
         #   ifeq (A)  
         #      ifeq (B)      ordered_files.sort()
         #        <do something>  
         #      endif      for stripped_subdirectory in ordered_files:
         #   endif          project_includes.append(f'dir     := $(d)/{stripped_subdirectory}')
   
         rulesPref = '';      return error
         rulesSuff = '';  
   def determineSection(line, regexpStyle, regexpDefs, regexpMake):
         filters = proj.find('filters')      matchobj = regexpStyle.match(line)
         if filters is not None:      if matchobj:
             for filter in filters.findall('filter'):          return 'style'
                 rulesPref += 'ifeq ($(' + filter.find('name').text + '),' + filter.find('value').text + ')\n'  
                 rulesSuff += 'endif\n'  
   
         if len(rulesPref) > 0 and len(rulesSuff) > 0:  
             projRules.extend(list(rulesPref))  
             projRules.extend(list(rulesStr))  
             projRules.extend(list(rulesSuff))  
         else:  
             projRules.extend(list(rulesStr))  
   
         # target.mk  
         subdirs = proj.find('subdirs')  
         if subdirs is not None:  
             for subdir in subdirs.findall('subdir'):  
                 targetStr = '\n\t+@[ -d $@/' + nameElem.text + '/' + subdir.text + ' ] || mkdir -p $@/' + nameElem.text + '/' + subdir.text  
                 projTarget.extend(list(targetStr))  
   
     return rv  
  
 def determineSection(line, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg):  
     matchobj = regexpDefs.match(line)     matchobj = regexpDefs.match(line)
     if not matchobj is None:     if not matchobj is None:
         return 'defs'         return 'defs'
Line 334  def determineSection(line, regexpDefs, r
Line 488  def determineSection(line, regexpDefs, r
     if not matchobj is None:     if not matchobj is None:
         return 'make'         return 'make'
  
     matchobj = regexpProjMkRules.match(line)  
     if not matchobj is None:  
         return 'projmkrules'  
   
     matchobj = regexpProj.match(line)  
     if not matchobj is None:  
         return 'proj'  
   
     matchobj = regexpProjCfg.match(line)  
     if not matchobj is None:  
         return 'projcfg'  
   
     return None     return None
  
 # defs is a dictionary containing all parameters (should they be needed in this script) # defs is a dictionary containing all parameters (should they be needed in this script)
 # projCfg is the list containing the configure script content.  def parseConfig(fin, keymap, addenda, defs, cDefs, mDefsGen, mDefsMake, project_includes, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection):
 # projMkRules is the list containing the make_basic.mk content.      error = False
 # projRules is the list containing the Rules.mk content.  
 # projTargert is the list containing the target.mk content.      print(f'addenda is {str(addenda)}')
 def parseConfig(fin, keymap, addenda, defs, cDefs, mDefsGen, mDefsMake, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection):  
     rv = bool(0)  
  
     # Open required config file (config.local)     # Open required config file (config.local)
     try:     try:
         # Examine each line, looking for key=value pairs.         # Examine each line, looking for key=value pairs.
           regexpStyle = re.compile(r"^__STYLE__")
         regexpDefs = re.compile(r"^__DEFS__")         regexpDefs = re.compile(r"^__DEFS__")
         regexpMake = re.compile(r"^__MAKE__")         regexpMake = re.compile(r"^__MAKE__")
         regexpProjMkRules = re.compile(r"__PROJ_MK_RULES__")  
         regexpProj = re.compile(r"^__PROJ__")  
         regexpProjCfg = re.compile(r"^__PROJCFG__")  
         regexpComm = re.compile(r"^\s*#")         regexpComm = re.compile(r"^\s*#")
         regexpSp = re.compile(r"^s*$")         regexpSp = re.compile(r"^s*$")
         regexpQuote = re.compile(r"^\s*(\w):(.+)")         regexpQuote = re.compile(r"^\s*(\w):(.+)")
         regexpCustMkBeg = re.compile(r"^_CUST_")         regexpCustMkBeg = re.compile(r"^_CUST_")
         regexpCustMkEnd = re.compile(r"^_ENDCUST_")         regexpCustMkEnd = re.compile(r"^_ENDCUST_")
         regexpDiv = re.compile(r"^__")         regexpDiv = re.compile(r"^__")
         regexp = re.compile(r"^\s*(\S+)\s+(\S+)")          regexp = re.compile(r"^\s*(\S+)\s+(\S.*)")
  
           ignoreKeymap = False
         platDict = {}         platDict = {}
         machDict = {}         machDict = {}
  
         xml = None  
   
         # Process the parameters in the configuration file         # Process the parameters in the configuration file
         if not fin is None:         if not fin is None:
             for line in fin:             for line in fin:
Line 390  def parseConfig(fin, keymap, addenda, de
Line 527  def parseConfig(fin, keymap, addenda, de
                     # Skip whitespace line                     # Skip whitespace line
                     continue                     continue
  
                 newSection = determineSection(line, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg)                  newSection = determineSection(line, regexpStyle, regexpDefs, regexpMake)
                   if not newSection is None:
                       section = newSection
   
                   if not section:
                       raise Exception('invalidConfigFile', 'line ' + line.strip() + ' is not in any section')
   
                   if section == 'style':
                       # if the config.local file has new in the __STYLE__ section, then ignore the keymap and treat config.local like configsdp.txt;
                       # do not map from NetDRMS config.local parameter names to configsdp.txt names
                       for line in fin:
                           matchobj = regexpDiv.match(line)
                           if matchobj:
                               break;
                           if line.strip(' \n').lower() == 'new' and keymap:
                               ignoreKeymap = True
   
                       newSection = determineSection(line, regexpStyle, regexpDefs, regexpMake)
                 if not newSection is None:                 if not newSection is None:
                     section = newSection                     section = newSection
                       continue
                   elif section == 'make':
  
                 if section == 'make':  
                     # There are some blocks of lines in the __MAKE__ section that must be copied ver batim to the output make file.                     # There are some blocks of lines in the __MAKE__ section that must be copied ver batim to the output make file.
                     # The blocks are defined by _CUST_/_ENDCUST_ tags.                     # The blocks are defined by _CUST_/_ENDCUST_ tags.
                     matchobj = regexpCustMkBeg.match(line)                     matchobj = regexpCustMkBeg.match(line)
Line 406  def parseConfig(fin, keymap, addenda, de
Line 561  def parseConfig(fin, keymap, addenda, de
                             if not matchobj is None:                             if not matchobj is None:
                                 break;                                 break;
                             mDefsMake.extend(list(line))                             mDefsMake.extend(list(line))
                         newSection = determineSection(line, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg)                          newSection = determineSection(line, regexpStyle, regexpDefs, regexpMake)
                         if not newSection is None:                         if not newSection is None:
                             section = newSection                             section = newSection
                         continue                         continue
                     # Intentional fall through to next if statement                     # Intentional fall through to next if statement
                 if section == 'defs' or section == 'make':                 if section == 'defs' or section == 'make':
                     iscfg = bool(1)                     iscfg = bool(1)
                     ppRet = processParam(iscfg, line, regexpQuote, regexp, keymap, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, platDict, machDict, section)                      if ignoreKeymap:
                           keymapActual = None
                     if ppRet:  
                         break;  
                 elif section == 'projcfg':  
                     # Copy the line ver batim to the projCfg list (configure)  
                     for line in fin:  
                         matchobj = regexpDiv.match(line)  
                         if not matchobj is None:  
                             break;  
                         projCfg.extend(list(line))  
                     newSection = determineSection(line, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg)  
                     if not newSection is None:  
                         section = newSection  
                     continue  
                 elif section == 'projmkrules':  
                     # Copy the line ver batim to the projMkRules list (make_basic.mk)  
                     for line in fin:  
                         matchobj = regexpDiv.match(line)  
                         if not matchobj is None:  
                             break;  
                         projMkRules.extend(list(line))  
                     newSection = determineSection(line, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg)  
                     if not newSection is None:  
                         section = newSection  
                     continue  
                 elif section == 'proj':  
                     # Must parse xml and use the project-specific information to populate the Rules.mk and target.mk files.  
                     # Collect all xml lines for now, then process after file-read loop.  
                     if xml is None:  
                         xml = line  
                     else:                     else:
                         xml += line                          keymapActual = keymap
                       ppRet = processParam(iscfg, line, regexpQuote, regexp, keymapActual, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection, platDict, machDict, section)
                 else:                 else:
                     # Unknown section                     # Unknown section
                     raise Exception('unknownSection', section)                     raise Exception('unknownSection', section)
     except Exception as exc:     except Exception as exc:
           if len(exc.args) >= 2:
         msg = exc.args[0]         msg = exc.args[0]
         if msg == 'badKeyMapKey':          else:
               # re-raise the exception
               raise
   
           if msg == 'invalidConfigFile':
               violator = exc.args[1]
               print(violator, file=sys.stderr)
               error = True
           elif msg == 'badKeyMapKey':
             # If we are here, then there was a non-empty keymap, and the parameter came from             # If we are here, then there was a non-empty keymap, and the parameter came from
             # the configuration file.             # the configuration file.
             violator = exc.args[1]             violator = exc.args[1]
             print('Unknown parameter name ' + "'" + violator + "'" + ' in ' + cfgfile + '.', file=sys.stderr)              print('Unknown parameter name ' + "'" + violator + "'" + ' in ' + fin.name + '.', file=sys.stderr)
             rv = bool(1)              error = True
         elif msg == 'badQuoteQual':         elif msg == 'badQuoteQual':
             # The bad quote qualifier came from the configuration file, not the addenda, since             # The bad quote qualifier came from the configuration file, not the addenda, since
             # we will have fixed any bad qualifiers in the addenda (which is populated by code).             # we will have fixed any bad qualifiers in the addenda (which is populated by code).
             violator = exc.args[1]             violator = exc.args[1]
             print('Unknown quote qualifier ' + "'" + violator + "'" + ' in ' + cfgfile + '.', file=sys.stderr)              print('Unknown quote qualifier ' + "'" + violator + "'" + ' in ' + fin.name + '.', file=sys.stderr)
             rv = bool(1)              error = True
         elif msg == 'missingQuoteQual':         elif msg == 'missingQuoteQual':
             violator = exc.args[1]             violator = exc.args[1]
             print('Missing quote qualifier for parameter ' + "'" + violator + "'" + ' in ' + cfgfile + '.', file=sys.stderr)              print('Missing quote qualifier for parameter ' + "'" + violator + "'" + ' in ' + fin.name + '.', file=sys.stderr)
             rv = bool(1)              error = True
         elif msg == 'paramNameTooLong':         elif msg == 'paramNameTooLong':
             violator = exc.args[1]             violator = exc.args[1]
             print('Macro name ' + "'" + violator + "' is too long.", file=sys.stderr)             print('Macro name ' + "'" + violator + "' is too long.", file=sys.stderr)
             rv = bool(1)              error = True
         elif msg == 'unknownSection':         elif msg == 'unknownSection':
             violator = exc.args[1]             violator = exc.args[1]
             print('Unknown section ' + "'" + violator + "' in configuration file.", file=sys.stderr)             print('Unknown section ' + "'" + violator + "' in configuration file.", file=sys.stderr)
             rv = bool(1)              error = True
         else:         else:
             # re-raise the exception             # re-raise the exception
             raise             raise
  
     if not rv:      if not error:
         if not xml is None:          if project_includes is not None:
             # Process xml.              error = process_project_repos(project_includes)
             projRules.extend(list(RULESPREFIX))  
             projTarget.extend(list(TARGETPREFIX))  
             rv = processXML(xml, projRules, projTarget)  
             projRules.extend(RULESSUFFIX)  
  
     # Process addenda - these are parameters that are not configurable and must be set in the     # Process addenda - these are parameters that are not configurable and must be set in the
     # NetDRMS build.     # NetDRMS build.
     if not rv:      if not error:
         iscfg = bool(0)         iscfg = bool(0)
         for key in addenda:          for key, val in addenda.items():
             item = key + ' ' + addenda[key]              item = f'{key} {val}'
             ppRet = processParam(iscfg, item, regexpQuote, regexp, keymap, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, platDict, machDict, 'defs')              print(f'addenda item {item}')
             if ppRet:              if ignoreKeymap:
                 break;                  keymapActual = None
               else:
                   keymapActual = keymap
               ppRet = processParam(iscfg, item, regexpQuote, regexp, keymapActual, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection, platDict, machDict, 'defs')
  
     # Put information collected in platDict and machDict into mDefs. Must do this here, and not in processParam, since     # Put information collected in platDict and machDict into mDefs. Must do this here, and not in processParam, since
     # we need to parse all platform-specific make variables before grouping them into platform categories.     # we need to parse all platform-specific make variables before grouping them into platform categories.
     if not rv:      if not error:
         for plat in platDict:         for plat in platDict:
             mDefsMake.extend(list('\nifeq ($(JSOC_MACHINE), linux_' + plat.lower() + ')'))             mDefsMake.extend(list('\nifeq ($(JSOC_MACHINE), linux_' + plat.lower() + ')'))
             for var in platDict[plat]:             for var in platDict[plat]:
                 mDefsMake.extend(list('\n' + var + ' = ' + platDict[plat][var]))                 mDefsMake.extend(list('\n' + var + ' = ' + platDict[plat][var]))
             mDefsMake.extend(list('\nendif\n'))             mDefsMake.extend(list('\nendif\n'))
  
     if not rv:      if not error:
         for mach in machDict:         for mach in machDict:
             mDefsMake.extend(list('\nifeq ($(MACHTYPE), ' + mach + ')'))             mDefsMake.extend(list('\nifeq ($(MACHTYPE), ' + mach + ')'))
             for var in platDict[plat]:              for var in machDict[mach]:
                 mDefsMake.extend(list('\n' + var + ' = ' + platDict[plat][var]))                  mDefsMake.extend(list('\n' + var + ' = ' + machDict[mach][var]))
             mDefsMake.extend(list('\nendif\n'))             mDefsMake.extend(list('\nendif\n'))
       return error
  
     return rv  def getMgrUIDLine(sums_manager, uidParam):
       error = False
 def getMgrUIDLine(defs, uidParam):  
     rv = bool(0)  
  
     cmd = 'id -u ' + defs['SUMS_MANAGER']      if sums_manager and len(sums_manager) > 0:
           cmd = [ 'id', '-u', shlex.quote(sums_manager) ]
     try:     try:
         ret = check_output(cmd, shell=True)              ret = check_output(cmd)
         uidParam['q:SUMS_MANAGER_UID'] = ret.decode("utf-8")         uidParam['q:SUMS_MANAGER_UID'] = ret.decode("utf-8")
     except ValueError:     except ValueError:
         print('Unable to run cmd: ' + cmd + '.')         print('Unable to run cmd: ' + cmd + '.')
         rv = bool(1)              error = True
     except CalledProcessError:     except CalledProcessError:
         print('Command ' + "'" + cmd + "'" + ' ran improperly.')         print('Command ' + "'" + cmd + "'" + ' ran improperly.')
         rv = bool(1)              error = True
  
     return rv      return error
  
 def isVersion(maj, min, majDef, minDef): def isVersion(maj, min, majDef, minDef):
     res = 0     res = 0
Line 553  def configureComps(defs, mDefs):
Line 688  def configureComps(defs, mDefs):
         hasgfort = bool(0)         hasgfort = bool(0)
  
         # Try icc.         # Try icc.
         cmd = 'icc -V 2>&1'          cmd = 'icc --version 2>&1'
         try:         try:
             ret = check_output(cmd, shell=True)             ret = check_output(cmd, shell=True)
             ret = ret.decode("utf-8")             ret = ret.decode("utf-8")
Line 562  def configureComps(defs, mDefs):
Line 697  def configureComps(defs, mDefs):
             rv = bool(1)             rv = bool(1)
  
         if not rv:         if not rv:
             regexp = re.compile(r".+Version\s+(\d+)[.](\d+)", re.DOTALL)              regexp = re.compile(r"\s*\S+\s+\S+\s+(\d+)[.](\d+)", re.DOTALL)
             matchobj = regexp.match(ret)             matchobj = regexp.match(ret)
             if matchobj is None:             if matchobj is None:
                 raise Exception('unexpectedIccRet', ret)                 raise Exception('unexpectedIccRet', ret)
Line 574  def configureComps(defs, mDefs):
Line 709  def configureComps(defs, mDefs):
  
         # Try gcc.         # Try gcc.
         if not hasicc:         if not hasicc:
               rv = bool(0)
             cmd = 'gcc -v 2>&1'             cmd = 'gcc -v 2>&1'
             try:             try:
                 ret = check_output(cmd, shell=True)                 ret = check_output(cmd, shell=True)
Line 582  def configureComps(defs, mDefs):
Line 718  def configureComps(defs, mDefs):
                 print('Command ' + "'" + cmd + "'" + ' ran improperly.')                 print('Command ' + "'" + cmd + "'" + ' ran improperly.')
                 rv = bool(1)                 rv = bool(1)
  
             if rv == bool(0):              if not rv:
                 regexp = re.compile(r".+gcc\s+version\s+(\d+)\.(\d+)", re.DOTALL)                 regexp = re.compile(r".+gcc\s+version\s+(\d+)\.(\d+)", re.DOTALL)
                 matchobj = regexp.match(ret)                 matchobj = regexp.match(ret)
                 if matchobj is None:                 if matchobj is None:
Line 594  def configureComps(defs, mDefs):
Line 730  def configureComps(defs, mDefs):
                         hasgcc = bool(1)                         hasgcc = bool(1)
  
         # Try ifort.         # Try ifort.
         cmd = 'ifort -V 2>&1'          rv = bool(0)
           cmd = 'ifort --version 2>&1'
         try:         try:
             ret = check_output(cmd, shell=True)             ret = check_output(cmd, shell=True)
             ret = ret.decode("utf-8")             ret = ret.decode("utf-8")
Line 603  def configureComps(defs, mDefs):
Line 740  def configureComps(defs, mDefs):
             rv = bool(1)             rv = bool(1)
  
         if not rv:         if not rv:
             regexp = re.compile(r".+Version\s+(\d+)\.(\d+)", re.DOTALL)              regexp = re.compile(r"\s*\S+\s+\S+\s+(\d+)\.(\d+)", re.DOTALL)
             matchobj = regexp.match(ret)             matchobj = regexp.match(ret)
             if matchobj is None:             if matchobj is None:
                 raise Exception('unexpectedIfortRet', ret)                 raise Exception('unexpectedIfortRet', ret)
Line 615  def configureComps(defs, mDefs):
Line 752  def configureComps(defs, mDefs):
  
         # Try gfortran         # Try gfortran
         if not hasifort:         if not hasifort:
               rv = bool(0)
             cmd = 'gfortran -v 2>&1'             cmd = 'gfortran -v 2>&1'
             try:             try:
                 ret = check_output(cmd, shell=True)                 ret = check_output(cmd, shell=True)
Line 623  def configureComps(defs, mDefs):
Line 761  def configureComps(defs, mDefs):
                 print('Command ' + "'" + cmd + "'" + ' ran improperly.')                 print('Command ' + "'" + cmd + "'" + ' ran improperly.')
                 rv = bool(1)                 rv = bool(1)
  
             if rv == bool(0):              if not rv:
                 regexp = re.compile(r".+gcc\s+version\s+(\d+)\.(\d+)", re.DOTALL)                 regexp = re.compile(r".+gcc\s+version\s+(\d+)\.(\d+)", re.DOTALL)
                 matchobj = regexp.match(ret)                 matchobj = regexp.match(ret)
                 if matchobj is None:                 if matchobj is None:
Line 635  def configureComps(defs, mDefs):
Line 773  def configureComps(defs, mDefs):
                         hasgfort = bool(1)                         hasgfort = bool(1)
  
         # Append the compiler make variables to the make file         # Append the compiler make variables to the make file
           rv = bool(0)
   
         if not hasicc and not hasgcc:         if not hasicc and not hasgcc:
             print('Fatal error: Acceptable C compiler not found! You will be unable to build the DRMS library.', file=sys.stderr)             print('Fatal error: Acceptable C compiler not found! You will be unable to build the DRMS library.', file=sys.stderr)
             rv = bool(1)              rv = bool(0) # Art - don't bail, we might be using drmsparams.py and not care about the rest of DRMS
         elif hasicc:         elif hasicc:
             mDefs.extend(list('\nCOMPILER = icc'))             mDefs.extend(list('\nCOMPILER = icc'))
               # mDefs.extend(list('\nICC_VERSION = blah'))
         else:         else:
             mDefs.extend(list('\nCOMPILER = gcc'))             mDefs.extend(list('\nCOMPILER = gcc'))
  
Line 651  def configureComps(defs, mDefs):
Line 792  def configureComps(defs, mDefs):
             mDefs.extend(list('\nFCOMPILER = gfortran'))             mDefs.extend(list('\nFCOMPILER = gfortran'))
  
         # Environment overrides. These get written, regardless of the disposition of auto-configuration.         # Environment overrides. These get written, regardless of the disposition of auto-configuration.
         mDefs.extend(list('\nifneq $(JSOC_COMPILER,)\n  COMPILER = $(JSOC_COMPILER)\nendif'))          mDefs.extend(list('\nifneq ($(JSOC_COMPILER),)\n  COMPILER = $(JSOC_COMPILER)\nendif'))
         mDefs.extend(list('\nifneq $(JSOC_FCOMPILER,)\n  FCOMPILER = $(JSOC_FCOMPILER)\nendif'))          mDefs.extend(list('\nifneq ($(JSOC_FCOMPILER),)\n  FCOMPILER = $(JSOC_FCOMPILER)\nendif'))
  
     return rv     return rv
  
 def writeParamsFiles(base, cfile, mfile, pfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection):  def writeParamsFiles(base, cfile, mfile, pfile, pyfile, shfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection):
     rv = bool(0)     rv = bool(0)
  
     # Merge mDefsGen, mDefsMake, and mDefsComps into a single string with compiler configuration first, general parameters next, then     # Merge mDefsGen, mDefsMake, and mDefsComps into a single string with compiler configuration first, general parameters next, then
Line 664  def writeParamsFiles(base, cfile, mfile,
Line 805  def writeParamsFiles(base, cfile, mfile,
     mDefs = '\n# Compiler Selection\n' + ''.join(mDefsComps) + '\n\n# General Parameters\n' + ''.join(mDefsGen) + '\n\n# Parameters to Configure make\n' + ''.join(mDefsMake)     mDefs = '\n# Compiler Selection\n' + ''.join(mDefsComps) + '\n\n# General Parameters\n' + ''.join(mDefsGen) + '\n\n# Parameters to Configure make\n' + ''.join(mDefsMake)
  
     try:     try:
         with open(cfile, 'w') as cout, open(mfile, 'w') as mout, open(pfile, 'w') as pout:          with open(cfile, 'w') as cout, open(mfile, 'w') as mout, open(pfile, 'w') as pout, open(pyfile, 'w') as pyout, open(shfile, 'w') as shout:
             # C file of macros             # C file of macros
             print(C_PREFIX, file=cout)             print(C_PREFIX, file=cout)
             print('/* This file contains a set of preprocessor macros - one for each configuration parameter. */\n', file=cout)             print('/* This file contains a set of preprocessor macros - one for each configuration parameter. */\n', file=cout)
Line 693  def writeParamsFiles(base, cfile, mfile,
Line 834  def writeParamsFiles(base, cfile, mfile,
             print(''.join(perlInitSection), file=pout)             print(''.join(perlInitSection), file=pout)
             print('}\n', file=pout)             print('}\n', file=pout)
             print(PERL_FXNS_B, file=pout)             print(PERL_FXNS_B, file=pout)
   
               # Python module
               print(PY_BINPATH, file=pyout)
               print(PREFIX, file=pyout)
               print('# This file contains a set of constants - one for each configuration parameter.\n', file=pyout)
               print(PY_ALL, file=pyout)
               print(''.join(pyConstSection), file=pyout)
   
               print(PY_ERROR_CLASSES, file=pyout)
               print(PY_FXNS_A, file=pyout, end='')
               print(''.join(pyInitSection), file=pyout)
               print(PY_FXNS_B, file=pyout)
               print(PY_FXNS_C, file=pyout)
   
               # Shell (bash) source file
               print(SH_BINPATH, file=shout)
               print(PREFIX, file=shout)
               print('# This file contains a set of variable assignments - one for each configuration parameter.\n', file=shout)
               print(''.join(shConstSection), file=shout)
   
     except IOError as exc:     except IOError as exc:
         type, value, traceback = sys.exc_info()         type, value, traceback = sys.exc_info()
         print(exc.strerror, file=sys.stderr)         print(exc.strerror, file=sys.stderr)
Line 701  def writeParamsFiles(base, cfile, mfile,
Line 862  def writeParamsFiles(base, cfile, mfile,
  
     return rv     return rv
  
 def writeProjFiles(pCfile, pMfile, pRfile, pTfile, projCfg, projMkRules, projRules, projTarget):  def write_project_includes(project_includes_file, project_includes):
     rv = bool(0)      error = False
  
       if project_includes_file is not None:
     try:     try:
         if projCfg:              with open(project_includes_file, 'w') as file_out:
             with open(pCfile, 'w') as cout:  
                 # configure  
                 print(''.join(projCfg), file=cout)  
   
         if projMkRules:  
             with open(pMfile, 'w') as mout:  
                 # make_basic.mk  
                 print(PREFIX, file=mout)  
                 print(''.join(projMkRules), file=mout)  
   
         if projRules:  
             with open(pRfile, 'w') as rout:  
                 # Rules.mk                 # Rules.mk
                 print(PREFIX, file=rout)                  print(PREFIX, file=file_out)
                 print(''.join(projRules), file=rout)                  print(RULESPREFIX, file=file_out)
   
                   for dir_variable in project_includes:
                       print(dir_variable, file=file_out)
                       print(f'-include          $(SRCDIR)/$(dir)/Rules.mk', file=file_out)
   
                   print('', file=file_out)
                   print(RULESSUFFIX, file=file_out)
  
         if projTarget:  
             with open(pTfile, 'w') as tout:  
                 # target.mk  
                 print(PREFIX, file=tout)  
                 print(''.join(projTarget), file=tout)  
     except IOError as exc:     except IOError as exc:
         type, value, traceback = sys.exc_info()              print(f'unable to open {project_includes_file} for writing ({str(exc)})', file=sys.stderr)
         print(exc.strerror, file=sys.stderr)              error = True
         print('Unable to open ' + "'" + value.filename + "'.", file=sys.stderr)  
       return error
   
   def generateSumRmCfg(defs):
       rv = bool(0)
       # ACK! Remember that Rick renamed these parameters. The ones in config.local are the aliases - do not use those.
       # Use the ones that those map to (defined in config.local.map).
       cFileTmp = defs['SUMLOG_BASEDIR'] + '/' + '.sum_rm.cfg.tmp'
       cFile = defs['SUMLOG_BASEDIR'] + '/' + 'sum_rm.cfg'
   
       # Write a temporary file sum_rm configuration file.
       try:
           with open(cFileTmp, 'w') as fout:
               # Print comment at the top of the configuration file.
               print(SUMRM_COMMENT, file=fout)
               print(SUMRM_DOC, file=fout)
               print(SUMRM_PARTN_PERCENT_FREE, file=fout)
               if 'SUMRM_PART_PERCENT_FREE' in defs:
                   print('PART_PERCENT_FREE=' + defs['SUMRM_PART_PERCENT_FREE'], file=fout)
               else:
                   print('PART_PERCENT_FREE=3', file=fout)
   
               print(SUMRM_SLEEP, file=fout)
               if 'SUMRM_SLEEP' in defs:
                   print('SLEEP=' + defs['SUMRM_SLEEP'], file=fout)
               else:
                   print('SLEEP=300', file=fout)
   
               print(SUMRM_LOG, file=fout)
               if 'SUMRM_LOG' in defs:
                   print('LOG=' + defs['SUMRM_LOG'], file=fout)
               else:
                   print('LOG=/tmp/sum_rm.log', file=fout)
   
               print(SUMRM_MAIL, file=fout)
               # No default for mail - don't send nothing to nobody unless the operator has asked for notifications.
               if 'SUMRM_MAIL' in defs:
                   print('MAIL=' + defs['SUMRM_MAIL'], file=fout)
               else:
                   print('# MAIL=president@whitehouse.gov', file=fout)
   
               print(SUMRM_NOOP, file=fout)
               if 'SUMRM_NOOP' in defs:
                   print('NOOP=' + defs['SUMRM_NOOP'], file=fout)
               else:
                   print('NOOP=0', file=fout)
   
               print(SUMRM_USER, file=fout)
               if 'SUMRM_USER' in defs:
                   print('USER=' + defs['SUMRM_USER'], file=fout)
               else:
                   print('USER=production', file=fout)
   
               print(SUMRM_NORUN, file=fout)
               # Default norun window is to have no such window. This can be accomplished by simply not providing either argument.
               if 'SUMRM_NORUN_START' in defs or 'SUMRM_NORUN_STOP' in defs:
                   if 'SUMRM_NORUN_START' in defs:
                       print('NORUN_START=' + defs['SUMRM_NORUN_START'], file=fout)
                   else:
                       print('NORUN_START=0', file=fout)
                   if 'SUMRM_NORUN_STOP' in defs:
                       print('NORUN_STOP=' + defs['SUMRM_NORUN_STOP'], file=fout)
                   else:
                       print('NORUN_STOP=0', file=fout)
               else:
                   print('# NORUN_START=0', file=fout)
                   print('# NORUN_STOP=0', file=fout)
   
       except OSError:
           print('Unable to open sum_rm temporary configuration file ' + cFileTmp + 'for writing.', file=sys.stderr)
         rv = bool(1)         rv = bool(1)
  
       # If the content of the temporary file differs from the content of the existing configuration file, then overwrite
       # the original file. Otherwise, delete the temporary file
     if not rv:     if not rv:
         try:         try:
             os.chmod(pCfile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IROTH)              if filecmp.cmp(cFile, cFileTmp):
                   # Files identical - delete temporary file
                   try:
                       os.remove(cFileTmp)
   
         except OSError as exc:         except OSError as exc:
             print(exc.strerror, file=sys.stderr)                      print('Unable to remove temporary file ' + exc.filename + '.', file=sys.stderr)
                       print(exc.strerr, file=sys.stderr)
               else:
                   # Replace original with temporary file
                   try:
                       os.rename(cFileTmp, cFile)
   
                   except OSError as exc:
                       print('Unable to update sum_rm configuration file ' + cFile + '.', file=sys.stderr)
                       print(exc.strerr, file=sys.stderr)
                       rv = bool(1)
           except OSError as exc:
               # One of the files doesn't exist.
               if exc.filename == cFile:
                   # We are ok - there might be no configuration file yet.
                   # Replace original with temporary file
                   try:
                       os.rename(cFileTmp, cFile)
   
                   except OSError as exc:
                       print('Unable to update sum_rm configuration file ' + cFile + '.', file=sys.stderr)
                       print(exc.strerr, file=sys.stderr)
                       rv = bool(1)
               else:
                   # There is a problem with the temp file - bail.
                   print('Unable to update sum_rm configuration file ' + cFile + '.', file=sys.stderr)
                   print(exc.strerr, file=sys.stderr)
             rv = bool(1)             rv = bool(1)
  
     return rv     return rv
  
 def configureNet(cfgfile, cfile, mfile, pfile, pCfile, pMfile, pRfile, pTfile, base, keymap):  def configureNet(cfgfile, cfile, mfile, pfile, pyfile, shfile, project_includes_file, base, keymap, createSumRmCfg):
     rv = bool(0)      error = False
  
     defs = {}     defs = {}
     cDefs = list()     cDefs = list()
     mDefsGen = list()     mDefsGen = list()
     mDefsMake = list()     mDefsMake = list()
     mDefsComps = list()     mDefsComps = list()
     projCfg = list()      project_includes = []
     projMkRules = list()  
     projRules = list()  
     projTarget = list()  
     perlConstSection = list()     perlConstSection = list()
     perlInitSection = list()     perlInitSection = list()
       pyConstSection = list()
       pyInitSection = list()
       shConstSection = list()
     addenda = {}     addenda = {}
  
     # There are three parameters that were not included in the original config.local parameter set, for some reason.     # There are three parameters that were not included in the original config.local parameter set, for some reason.
Line 773  def configureNet(cfgfile, cfile, mfile,
Line 1026  def configureNet(cfgfile, cfile, mfile,
     try:     try:
         with open(cfgfile, 'r') as fin:         with open(cfgfile, 'r') as fin:
             # Process configuration parameters             # Process configuration parameters
             rv = parseConfig(fin, keymap, addenda, defs, cDefs, mDefsGen, mDefsMake, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection)  
             if not rv:              error = parseConfig(fin, keymap, addenda, defs, cDefs, mDefsGen, mDefsMake, project_includes, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
               if not error:
                 # Must add a parameter for the SUMS_MANAGER UID (for some reason). This must be done after the                 # Must add a parameter for the SUMS_MANAGER UID (for some reason). This must be done after the
                 # config file is processed since an input to getMgrUIDLine() is one of the config file's                 # config file is processed since an input to getMgrUIDLine() is one of the config file's
                 # parameter values.                 # parameter values.
   
                   # CANNOT run this unless sunroom2 home directories are accessible
                 uidParam = {}                 uidParam = {}
                 rv = getMgrUIDLine(defs, uidParam)                  error = getMgrUIDLine(defs.get('SUMS_MANAGER', None), uidParam)
                 if rv == bool(0):                  if not error and len(uidParam) > 0:
                     rv = parseConfig(None, keymap, uidParam, defs, cDefs, mDefsGen, None, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection)                      error = parseConfig(None, keymap, uidParam, defs, cDefs, mDefsGen, None, None, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
   
                   # ignore the error where the SUMS manager UID cannot be determined
                   error = False
  
             # Configure the compiler-selection make variables.             # Configure the compiler-selection make variables.
             if not rv:              if not error:
                 rv = configureComps(defs, mDefsComps)                  error = configureComps(defs, mDefsComps)
  
             # Write out the parameter files.             # Write out the parameter files.
             if not rv:              if not error:
                 rv = writeParamsFiles(base, cfile, mfile, pfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection)                  error = writeParamsFiles(base, cfile, mfile, pfile, pyfile, shfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
  
             # Write out the project-specific make files (make_basic.mk, Rules.mk, and target.mk).              if not error:
             if not rv:                  error = write_project_includes(project_includes_file, project_includes)
                 rv = writeProjFiles(pCfile, pMfile, pRfile, pTfile, projCfg, projMkRules, projRules, projTarget)  
  
               # Write out the sum_rm.cfg file.
               if not error and createSumRmCfg:
                   error = generateSumRmCfg(defs)
     except IOError as exc:     except IOError as exc:
         print(exc.strerror, file=sys.stderr)         print(exc.strerror, file=sys.stderr)
         print('Unable to read configuration file ' + cfgfile + '.', file=sys.stderr)         print('Unable to read configuration file ' + cfgfile + '.', file=sys.stderr)
     except Exception as exc:     except Exception as exc:
           if len(exc.args) >= 2:
         type, msg = exc.args         type, msg = exc.args
           else:
               # re-raise the exception
               raise
   
         if type == 'unexpectedIccRet':         if type == 'unexpectedIccRet':
             print('icc -V returned this unexpected message:\n' + msg, file=sys.stderr)             print('icc -V returned this unexpected message:\n' + msg, file=sys.stderr)
             rv = bool(1)              error = True
         elif type == 'unexpectedGccRet':         elif type == 'unexpectedGccRet':
             print('gcc -v returned this unexpected message:\n' + msg, file=sys.stderr)             print('gcc -v returned this unexpected message:\n' + msg, file=sys.stderr)
             rv = bool(1)              error = True
         elif type == 'unexpectedIfortRet':         elif type == 'unexpectedIfortRet':
             print('ifort -V returned this unexpected message:\n' + msg, file=sys.stderr)             print('ifort -V returned this unexpected message:\n' + msg, file=sys.stderr)
             rv = bool(1)              error = True
         elif type == 'unexpectedGfortranRet':         elif type == 'unexpectedGfortranRet':
             print('gfortran -v returned this unexpected message:\n' + msg, file=sys.stderr)             print('gfortran -v returned this unexpected message:\n' + msg, file=sys.stderr)
             rv = bool(1)              error = True
         else:         else:
             # re-raise the exception             # re-raise the exception
             raise             raise
  
     return rv      return error
  
 def configureSdp(cfgfile, cfile, mfile, pfile, pCfile, pMfile, pRfile, pTfile, base):  def configureSdp(cfgfile, cfile, mfile, pfile, pyfile, shfile, project_includes_file, base):
     rv = bool(0)      error = False
  
     defs = {}     defs = {}
     cDefs = list()     cDefs = list()
     mDefsGen = list()     mDefsGen = list()
     mDefsMake = list()     mDefsMake = list()
     projCfg = list()  
     projMkRules = list()  
     projRules = list()  
     projTarget = list()  
     mDefsComps = list()     mDefsComps = list()
       project_includes = []
     perlConstSection = list()     perlConstSection = list()
     perlInitSection = list()     perlInitSection = list()
       pyConstSection = list()
       pyInitSection = list()
       shConstSection = list()
   
     addenda = {}     addenda = {}
  
     # There are three parameters that were not included in the original config.local parameter set, for some reason.     # There are three parameters that were not included in the original config.local parameter set, for some reason.
Line 845  def configureSdp(cfgfile, cfile, mfile,
Line 1112  def configureSdp(cfgfile, cfile, mfile,
  
     try:     try:
         with open(cfgfile, 'r') as fin:         with open(cfgfile, 'r') as fin:
             rv = parseConfig(fin, None, addenda, defs, cDefs, mDefsGen, mDefsMake, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection)              error = parseConfig(fin, None, addenda, defs, cDefs, mDefsGen, mDefsMake, project_includes, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
  
             if not rv:              if not error:
                 # Must add a parameter for the SUMS_MANAGER UID (for some reason)                 # Must add a parameter for the SUMS_MANAGER UID (for some reason)
                 uidParam = {}                 uidParam = {}
                 rv = getMgrUIDLine(defs, uidParam)                  error = getMgrUIDLine(defs.get('SUMS_MANAGER', None), uidParam)
                 if not rv:                  if not error and len(uidParam) > 0:
                     rv = parseConfig(None, None, uidParam, defs, cDefs, mDefsGen, None, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection)                      error = parseConfig(None, None, uidParam, defs, cDefs, mDefsGen, None, None, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
   
                   # ignore the error where the SUMS manager UID cannot be determined
                   error = False
  
                 # Configure the compiler-selection make variables.                 # Configure the compiler-selection make variables.
                 if not rv:                  if not error:
                     rv = configureComps(defs, mDefsComps)                      error = configureComps(defs, mDefsComps)
  
                 # Write out the parameter files.                 # Write out the parameter files.
                 if not rv:                  if not error:
                     rv = writeParamsFiles(base, cfile, mfile, pfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection)                      error = writeParamsFiles(base, cfile, mfile, pfile, pyfile, shfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
  
                 # Write out the project-specific make files (make_basic.mk, Rules.mk, and target.mk).                  if not error:
                 if not rv:                      error = write_project_includes(project_includes_file, project_includes)
                     rv = writeProjFiles(pCfile, pMfile, pRfile, pTfile, projCfg, projMkRules, projRules, projTarget)  
                   # At Stanford, skip the creation of the sum_rm configuration file. config.local will still
                   # have the SUMRM parameters, but they will not be used.
     except IOError as exc:     except IOError as exc:
         print(exc.strerror, file=sys.stderr)         print(exc.strerror, file=sys.stderr)
         print('Unable to read configuration file ' + cfgfile + '.', file=sys.stderr)         print('Unable to read configuration file ' + cfgfile + '.', file=sys.stderr)
     except Exception as exc:     except Exception as exc:
           if len(exc.args) >= 2:
         type = exc.args[0]         type = exc.args[0]
           else:
               # re-raise the exception
               raise
   
         if type == 'unexpectedIccRet':         if type == 'unexpectedIccRet':
             msg = exc.args[1]             msg = exc.args[1]
             print('icc -V returned this unexpected message:\n' + msg, file=sys.stderr)             print('icc -V returned this unexpected message:\n' + msg, file=sys.stderr)
             rv = bool(1)              error = True
         elif type == 'unexpectedGccRet':         elif type == 'unexpectedGccRet':
             msg = exc.args[1]             msg = exc.args[1]
             print('gcc -v returned this unexpected message:\n' + msg, file=sys.stderr)             print('gcc -v returned this unexpected message:\n' + msg, file=sys.stderr)
             rv = bool(1)              error = True
         elif type == 'unexpectedIfortRet':         elif type == 'unexpectedIfortRet':
             msg = exc.args[1]             msg = exc.args[1]
             print('ifort -V returned this unexpected message:\n' + msg, file=sys.stderr)             print('ifort -V returned this unexpected message:\n' + msg, file=sys.stderr)
             rv = bool(1)              error = True
         elif type == 'unexpectedGfortranRet':         elif type == 'unexpectedGfortranRet':
             msg = exc.args[1]             msg = exc.args[1]
             print('gfortran -v returned this unexpected message:\n' + msg, file=sys.stderr)             print('gfortran -v returned this unexpected message:\n' + msg, file=sys.stderr)
             rv = bool(1)              error = True
         else:         else:
             # re-raise the exception             # re-raise the exception
             raise             raise
  
     return rv      return error
  
 # Beginning of program # Beginning of program
 rv = RET_SUCCESS rv = RET_SUCCESS
Line 918  if rv == RET_SUCCESS:
Line 1195  if rv == RET_SUCCESS:
     cfile = optD['dir'] + '/' + optD['base'] + '.h'     cfile = optD['dir'] + '/' + optD['base'] + '.h'
     mfile = optD['dir'] + '/' + optD['base'] + '.mk'     mfile = optD['dir'] + '/' + optD['base'] + '.mk'
     pfile = optD['dir'] + '/' + optD['base'] + '.pm'     pfile = optD['dir'] + '/' + optD['base'] + '.pm'
     pCfile = optD['dir'] + '/configure'      pyfile = optD['dir'] + '/' + optD['base'] + '.py'
     pMfile = optD['dir'] + '/make_basic.mk'      shfile = optD['dir'] + '/' + optD['base'] + '.sh'
     pRfile = optD['dir'] + '/Rules.mk'      project_includes_file = os.path.join(optD['dir'], 'includes.mk')
     pTfile = optD['dir'] + '/target.mk'  
  
     if net:     if net:
         try:         try:
Line 948  if rv == RET_SUCCESS:
Line 1224  if rv == RET_SUCCESS:
  
         # We also need to set the UID of the SUMS manager. We have the name of the         # We also need to set the UID of the SUMS manager. We have the name of the
         # SUMS manager (it is in the configuration file)         # SUMS manager (it is in the configuration file)
         configureNet(NET_CFG, cfile, mfile, pfile, pCfile, pMfile, pRfile, pTfile, optD['base'], keymap)          configureNet(NET_CFG, cfile, mfile, pfile, pyfile, shfile, project_includes_file, optD['base'], keymap, 'server' in optD)
     else:     else:
         configureSdp(SDP_CFG, cfile, mfile, pfile, pCfile, pMfile, pRfile, pTfile, optD['base'])          # A Stanford user can override the parameters in configsdp.txt by copying that file to config.local,
           # and then editing config.local. So, if config.local exists, use that.
           if os.path.isfile(cdir + '/' + NET_CFG):
               configureSdp(NET_CFG, cfile, mfile, pfile, pyfile, shfile, project_includes_file, optD['base'])
           else:
               configureSdp(SDP_CFG, cfile, mfile, pfile, pyfile, shfile, project_includes_file, optD['base'])

Legend:
Removed from v.1.6  
changed lines
  Added in v.1.33
Karen Tian
 Powered by
ViewCVS 0.9.4