(file) Return to localize.py CVS log (file) (dir) Up to [Development] / JSOC

Diff for /JSOC/localize.py between version 1.19 and 1.33

version 1.19, 2014/05/22 23:01:34 version 1.33, 2022/09/14 15:59:13
Line 1 
Line 1 
 #!/usr/bin/python  #!/usr/bin/env python
  
 # When run with the -s flag, localize.py configures the SUMS-server component of NetDRMS. # When run with the -s flag, localize.py configures the SUMS-server component of NetDRMS.
 from __future__ import print_function  
 import sys import sys
 import getopt import getopt
 import re import re
 import os import os
 import stat import stat
 import filecmp import filecmp
 import xml.etree.ElementTree as ET  
 from subprocess import check_output, CalledProcessError from subprocess import check_output, CalledProcessError
   import shlex
   
  
 # Constants # Constants
 VERS_FILE = 'jsoc_version.h' VERS_FILE = 'jsoc_version.h'
Line 74  PERL_FXNS_B = """sub get
Line 74  PERL_FXNS_B = """sub get
 } }
 1;""" 1;"""
  
 PY_BINPATH = '#/usr/bin/python\n'  PY_BINPATH = '#!/usr/bin/env python3\n'
   
   PY_ALL = f"__all__ = [ 'DPError', 'DPMissingParameterError', 'DRMSParams' ]"
   
   PY_ERROR_CLASSES = """
   class DPError(Exception):
       def __init__(self):
           self._msg = 'DRMS Parameter Error: '
   
   class DPMissingParameterError(DPError):
       def __init__(self, parameter):
           super().__init__()
           self._msg += 'missing DRMS parameter ' + parameter
   
       def __str__(self):
           return self._msg
   """
  
 PY_FXNS_A = """ PY_FXNS_A = """
 class DRMSParams:  class DRMSParams(object):
     def __init__(self):     def __init__(self):
         self.params = {}         self.params = {}
         self.initialize()         self.initialize()
Line 90  class DRMSParams:
Line 106  class DRMSParams:
  
 PY_FXNS_B = """    def get(self, name): PY_FXNS_B = """    def get(self, name):
         if name in self.params:         if name in self.params:
             return self.parms[name]              return self.params[name]
           else:
               return None
   """
   
   PY_FXNS_C = """    def get_required(self, name):
           try:
               value = self.params[name]
           except:
               raise DPMissingParameterError(name)
   
           return value
   
       def getBool(self, name):
           if name in self.params:
               return bool(self.params[name] == '1')
         else:         else:
             return None             return None
   
       def __getattr__(self, name):
           # only called if object.__getattribute__(self, name) raises; and if that is true, then we want
           # to look in self.params for it, and set the instance attribute if it does exist in self.params
           if name in self.params:
               attr = self.params[name]
               self.__setattr__(name, attr)
           else:
               attr = None
   
           return attr
   
       def __setattr__(self, name, value):
           # call neither __setattr__ nor __getattr__
           try:
               params = object.__getattr__(self, 'params')
   
               # put into self.params dict, overwriting if necessary
               params[name] = value
           except:
               pass
   
           # store in instance dict as well
           object.__setattr__(self, name, value)
 """ """
  
   SH_BINPATH = '#!/bin/bash\n'
   
   
 SUMRM_COMMENT = """# This is the configuration file for the sum_rm program. It was auto-generated by the DRMS master configure script. SUMRM_COMMENT = """# This is the configuration file for the sum_rm program. It was auto-generated by the DRMS master configure script.
 # It controls the behavior of the sum_rm program, and is loaded each time sum_rm runs. To change the # It controls the behavior of the sum_rm program, and is loaded each time sum_rm runs. To change the
 # parameter values in this configuration file, modify config.local, then re-run configure. This configuration # parameter values in this configuration file, modify config.local, then re-run configure. This configuration
Line 155  dirstack_$(sp) := $(d)
Line 213  dirstack_$(sp) := $(d)
 d               := $(dir) d               := $(dir)
 """ """
  
 RULESSUFFIX = """dir    := $(d)/example  RULESSUFFIX = """# Standard things
 -include                $(SRCDIR)/$(dir)/Rules.mk  
 dir     := $(d)/cookbook  
 -include                $(SRCDIR)/$(dir)/Rules.mk  
 dir     := $(d)/myproj  
 -include                $(SRCDIR)/$(dir)/Rules.mk  
   
 # Standard things  
 d               := $(dirstack_$(sp)) d               := $(dirstack_$(sp))
 sp              := $(basename $(sp)) sp              := $(basename $(sp))
 """ """
  
 TARGETPREFIX = """$(PROJOBJDIR):\n\t+@[ -d $@ ] || mkdir -p $@"""  
   
 ICC_MAJOR = 9 ICC_MAJOR = 9
 ICC_MINOR = 0 ICC_MINOR = 0
 GCC_MAJOR = 3 GCC_MAJOR = 3
Line 253  def createPyConst(key, val, keyColLen, s
Line 302  def createPyConst(key, val, keyColLen, s
         status = bool(0)         status = bool(0)
         return key + ' = ' + spaces + val + '\n'         return key + ' = ' + spaces + val + '\n'
  
   def createShConst(key, val, status):
       status = bool(0)
       return key + '=' + val + '\n'
   
   
 def isSupportedPlat(plat): def isSupportedPlat(plat):
     regexp = re.compile(r"\s*(^x86_64|^ia32|^ia64|^avx)", re.IGNORECASE)     regexp = re.compile(r"\s*(^x86_64|^ia32|^ia64|^avx)", re.IGNORECASE)
     matchobj = regexp.match(plat);     matchobj = regexp.match(plat);
Line 288  def processMakeParam(mDefs, key, val, pl
Line 342  def processMakeParam(mDefs, key, val, pl
                 machDict[varMach] = {}                 machDict[varMach] = {}
             machDict[varMach][varName] = varValu             machDict[varMach][varName] = varValu
  
 def processParam(cfgfile, line, regexpQuote, regexp, keymap, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, pyConstSection, pyInitSection, platDict, machDict, section):  def processParam(cfgfile, line, regexpQuote, regexp, keymap, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection, platDict, machDict, section):
     status = 0     status = 0
       parameter_added = {}
       keyCfgSp  = None
       val = None
  
     if ''.join(section) == 'defs' or not cfgfile:     if ''.join(section) == 'defs' or not cfgfile:
           if type(line) is str:
         matchobj = regexp.match(line)         matchobj = regexp.match(line)
   
         if not matchobj is None:         if not matchobj is None:
             # We have a key-value line             # We have a key-value line
             keyCfgSp = matchobj.group(1)             keyCfgSp = matchobj.group(1)
             val = matchobj.group(2)             val = matchobj.group(2)
                   print(f'herer, key={keyCfgSp}, val={val}')
           else:
               keyCfgSp, val = list(line.items())
   
           if keyCfgSp is not None and val is not None:
               # We have a key-value line
               keyCfgSp = matchobj.group(1)
               val = matchobj.group(2)
  
             # Must map the indirect name to the actual name             # Must map the indirect name to the actual name
             if keymap:             if keymap:
Line 306  def processParam(cfgfile, line, regexpQu
Line 373  def processParam(cfgfile, line, regexpQu
                 elif keyCfgSp == 'LOCAL_CONFIG_SET' or keyCfgSp == 'DRMS_SAMPLE_NAMESPACE':                 elif keyCfgSp == 'LOCAL_CONFIG_SET' or keyCfgSp == 'DRMS_SAMPLE_NAMESPACE':
                     # Ignore parameters that are not useful and shouldn't have been there in the first place. But                     # Ignore parameters that are not useful and shouldn't have been there in the first place. But
                     # they have been released to the world, so we have to account for them.                     # they have been released to the world, so we have to account for them.
                     return bool(0)                      return None
                 elif not cfgfile:                 elif not cfgfile:
                     # Should not be doing mapping for addenda                     # Should not be doing mapping for addenda
                     key = keyCfgSp                     key = keyCfgSp
Line 323  def processParam(cfgfile, line, regexpQu
Line 390  def processParam(cfgfile, line, regexpQu
                 # master defs dictionary                 # master defs dictionary
                 defs[key] = val                 defs[key] = val
  
                   parameter_added[key] = val
   
                 # C header file                 # C header file
                 if quote == "q":                 if quote == "q":
                     # Add double-quotes                     # Add double-quotes
Line 350  def processParam(cfgfile, line, regexpQu
Line 419  def processParam(cfgfile, line, regexpQu
                 # Python file                 # Python file
                 pyConstSection.extend(list(createPyConst(key, "'" + val + "'", 40, status)))                 pyConstSection.extend(list(createPyConst(key, "'" + val + "'", 40, status)))
  
                   # Shell source file
                   shConstSection.extend(list(createShConst(key, "'" + val + "'", status)))
   
                 if status:                 if status:
                     raise Exception('paramNameTooLong', key)                     raise Exception('paramNameTooLong', key)
  
Line 377  def processParam(cfgfile, line, regexpQu
Line 449  def processParam(cfgfile, line, regexpQu
             defs[key] = val             defs[key] = val
             processMakeParam(mDefsMake, key, val, platDict, machDict)             processMakeParam(mDefsMake, key, val, platDict, machDict)
  
     return bool(0)      return parameter_added
  
 # We have some extraneous line or a newline - ignore. # We have some extraneous line or a newline - ignore.
  
 def processXML(xml, projRules, projTarget):  def process_project_repos(project_includes):
     rv = bool(0)      print(f'[ process_project_repos ]')
       error = False
     # <projects>      ordered_files = []
     root = ET.fromstring(xml)  
       # iterate through all proj subdirectories
     # Iterate through each proj child.      for subdirectory in os.listdir('proj'):
     for proj in root.iter('proj'):          stripped_subdirectory = subdirectory.strip()
         # Rules.mk          path = os.path.join('proj', stripped_subdirectory)
         nameElem = proj.find('name')  
         rulesStr = 'dir     := $(d)/' + nameElem.text + '\n-include          $(SRCDIR)/$(dir)/Rules.mk\n'          # skip any dir in proj that does not have a Rules.mk file
           if os.path.isfile(os.path.join(path, 'Rules.mk')):
         # make doesn't support logical operations in ifeq conditionals (you can't do ifeq (A AND B)),              if os.path.isdir(path):
         # so we need to write:                  ordered_files.append(stripped_subdirectory)
         #   ifeq (A)  
         #      ifeq (B)      ordered_files.sort()
         #        <do something>  
         #      endif      for stripped_subdirectory in ordered_files:
         #   endif          project_includes.append(f'dir     := $(d)/{stripped_subdirectory}')
   
         rulesPref = '';      return error
         rulesSuff = '';  
   def determineSection(line, regexpStyle, regexpDefs, regexpMake):
         filters = proj.find('filters')      matchobj = regexpStyle.match(line)
         if filters is not None:      if matchobj:
             for filter in filters.findall('filter'):          return 'style'
                 rulesPref += 'ifeq ($(' + filter.find('name').text + '),' + filter.find('value').text + ')\n'  
                 rulesSuff += 'endif\n'  
   
         if len(rulesPref) > 0 and len(rulesSuff) > 0:  
             projRules.extend(list(rulesPref))  
             projRules.extend(list(rulesStr))  
             projRules.extend(list(rulesSuff))  
         else:  
             projRules.extend(list(rulesStr))  
   
         # target.mk  
         subdirs = proj.find('subdirs')  
         if subdirs is not None:  
             for subdir in subdirs.findall('subdir'):  
                 targetStr = '\n\t+@[ -d $@/' + nameElem.text + '/' + subdir.text + ' ] || mkdir -p $@/' + nameElem.text + '/' + subdir.text  
                 projTarget.extend(list(targetStr))  
   
     return rv  
  
 def determineSection(line, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg):  
     matchobj = regexpDefs.match(line)     matchobj = regexpDefs.match(line)
     if not matchobj is None:     if not matchobj is None:
         return 'defs'         return 'defs'
Line 435  def determineSection(line, regexpDefs, r
Line 488  def determineSection(line, regexpDefs, r
     if not matchobj is None:     if not matchobj is None:
         return 'make'         return 'make'
  
     matchobj = regexpProjMkRules.match(line)  
     if not matchobj is None:  
         return 'projmkrules'  
   
     matchobj = regexpProj.match(line)  
     if not matchobj is None:  
         return 'proj'  
   
     matchobj = regexpProjCfg.match(line)  
     if not matchobj is None:  
         return 'projcfg'  
   
     return None     return None
  
 # defs is a dictionary containing all parameters (should they be needed in this script) # defs is a dictionary containing all parameters (should they be needed in this script)
 # projCfg is the list containing the configure script content.  def parseConfig(fin, keymap, addenda, defs, cDefs, mDefsGen, mDefsMake, project_includes, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection):
 # projMkRules is the list containing the make_basic.mk content.      error = False
 # projRules is the list containing the Rules.mk content.  
 # projTargert is the list containing the target.mk content.      print(f'addenda is {str(addenda)}')
 def parseConfig(fin, keymap, addenda, defs, cDefs, mDefsGen, mDefsMake, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection, pyConstSection, pyInitSection):  
     rv = bool(0)  
  
     # Open required config file (config.local)     # Open required config file (config.local)
     try:     try:
         # Examine each line, looking for key=value pairs.         # Examine each line, looking for key=value pairs.
           regexpStyle = re.compile(r"^__STYLE__")
         regexpDefs = re.compile(r"^__DEFS__")         regexpDefs = re.compile(r"^__DEFS__")
         regexpMake = re.compile(r"^__MAKE__")         regexpMake = re.compile(r"^__MAKE__")
         regexpProjMkRules = re.compile(r"__PROJ_MK_RULES__")  
         regexpProj = re.compile(r"^__PROJ__")  
         regexpProjCfg = re.compile(r"^__PROJCFG__")  
         regexpComm = re.compile(r"^\s*#")         regexpComm = re.compile(r"^\s*#")
         regexpSp = re.compile(r"^s*$")         regexpSp = re.compile(r"^s*$")
         regexpQuote = re.compile(r"^\s*(\w):(.+)")         regexpQuote = re.compile(r"^\s*(\w):(.+)")
Line 473  def parseConfig(fin, keymap, addenda, de
Line 510  def parseConfig(fin, keymap, addenda, de
         regexpDiv = re.compile(r"^__")         regexpDiv = re.compile(r"^__")
         regexp = re.compile(r"^\s*(\S+)\s+(\S.*)")         regexp = re.compile(r"^\s*(\S+)\s+(\S.*)")
  
           ignoreKeymap = False
         platDict = {}         platDict = {}
         machDict = {}         machDict = {}
  
         xml = None  
   
         # Process the parameters in the configuration file         # Process the parameters in the configuration file
         if not fin is None:         if not fin is None:
             for line in fin:             for line in fin:
Line 491  def parseConfig(fin, keymap, addenda, de
Line 527  def parseConfig(fin, keymap, addenda, de
                     # Skip whitespace line                     # Skip whitespace line
                     continue                     continue
  
                 newSection = determineSection(line, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg)                  newSection = determineSection(line, regexpStyle, regexpDefs, regexpMake)
                 if not newSection is None:                 if not newSection is None:
                     section = newSection                     section = newSection
  
                 if section == 'make':                  if not section:
                       raise Exception('invalidConfigFile', 'line ' + line.strip() + ' is not in any section')
   
                   if section == 'style':
                       # if the config.local file has new in the __STYLE__ section, then ignore the keymap and treat config.local like configsdp.txt;
                       # do not map from NetDRMS config.local parameter names to configsdp.txt names
                       for line in fin:
                           matchobj = regexpDiv.match(line)
                           if matchobj:
                               break;
                           if line.strip(' \n').lower() == 'new' and keymap:
                               ignoreKeymap = True
   
                       newSection = determineSection(line, regexpStyle, regexpDefs, regexpMake)
                       if not newSection is None:
                           section = newSection
                       continue
                   elif section == 'make':
  
                     # There are some blocks of lines in the __MAKE__ section that must be copied ver batim to the output make file.                     # There are some blocks of lines in the __MAKE__ section that must be copied ver batim to the output make file.
                     # The blocks are defined by _CUST_/_ENDCUST_ tags.                     # The blocks are defined by _CUST_/_ENDCUST_ tags.
Line 508  def parseConfig(fin, keymap, addenda, de
Line 561  def parseConfig(fin, keymap, addenda, de
                             if not matchobj is None:                             if not matchobj is None:
                                 break;                                 break;
                             mDefsMake.extend(list(line))                             mDefsMake.extend(list(line))
                         newSection = determineSection(line, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg)                          newSection = determineSection(line, regexpStyle, regexpDefs, regexpMake)
                         if not newSection is None:                         if not newSection is None:
                             section = newSection                             section = newSection
                         continue                         continue
                     # Intentional fall through to next if statement                     # Intentional fall through to next if statement
                 if section == 'defs' or section == 'make':                 if section == 'defs' or section == 'make':
                     iscfg = bool(1)                     iscfg = bool(1)
                     ppRet = processParam(iscfg, line, regexpQuote, regexp, keymap, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, pyConstSection, pyInitSection, platDict, machDict, section)                      if ignoreKeymap:
                           keymapActual = None
                     if ppRet:  
                         break  
                 elif section == 'projcfg':  
                     # Copy the line ver batim to the projCfg list (configure)  
                     for line in fin:  
                         matchobj = regexpDiv.match(line)  
                         if not matchobj is None:  
                             break;  
                         projCfg.extend(list(line))  
                     newSection = determineSection(line, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg)  
                     if not newSection is None:  
                         section = newSection  
                     continue  
                 elif section == 'projmkrules':  
                     # Copy the line ver batim to the projMkRules list (make_basic.mk)  
                     for line in fin:  
                         matchobj = regexpDiv.match(line)  
                         if not matchobj is None:  
                             break;  
                         projMkRules.extend(list(line))  
                     newSection = determineSection(line, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg)  
                     if not newSection is None:  
                         section = newSection  
                     continue  
                 elif section == 'proj':  
                     # Must parse xml and use the project-specific information to populate the Rules.mk and target.mk files.  
                     # Collect all xml lines for now, then process after file-read loop.  
                     if xml is None:  
                         xml = line  
                     else:                     else:
                         xml += line                          keymapActual = keymap
                       ppRet = processParam(iscfg, line, regexpQuote, regexp, keymapActual, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection, platDict, machDict, section)
                 else:                 else:
                     # Unknown section                     # Unknown section
                     raise Exception('unknownSection', section)                     raise Exception('unknownSection', section)
Line 557  def parseConfig(fin, keymap, addenda, de
Line 582  def parseConfig(fin, keymap, addenda, de
         else:         else:
             # re-raise the exception             # re-raise the exception
             raise             raise
         if msg == 'badKeyMapKey':  
           if msg == 'invalidConfigFile':
               violator = exc.args[1]
               print(violator, file=sys.stderr)
               error = True
           elif msg == 'badKeyMapKey':
             # If we are here, then there was a non-empty keymap, and the parameter came from             # If we are here, then there was a non-empty keymap, and the parameter came from
             # the configuration file.             # the configuration file.
             violator = exc.args[1]             violator = exc.args[1]
             print('Unknown parameter name ' + "'" + violator + "'" + ' in ' + fin.name + '.', file=sys.stderr)             print('Unknown parameter name ' + "'" + violator + "'" + ' in ' + fin.name + '.', file=sys.stderr)
             rv = bool(1)              error = True
         elif msg == 'badQuoteQual':         elif msg == 'badQuoteQual':
             # The bad quote qualifier came from the configuration file, not the addenda, since             # The bad quote qualifier came from the configuration file, not the addenda, since
             # we will have fixed any bad qualifiers in the addenda (which is populated by code).             # we will have fixed any bad qualifiers in the addenda (which is populated by code).
             violator = exc.args[1]             violator = exc.args[1]
             print('Unknown quote qualifier ' + "'" + violator + "'" + ' in ' + fin.name + '.', file=sys.stderr)             print('Unknown quote qualifier ' + "'" + violator + "'" + ' in ' + fin.name + '.', file=sys.stderr)
             rv = bool(1)              error = True
         elif msg == 'missingQuoteQual':         elif msg == 'missingQuoteQual':
             violator = exc.args[1]             violator = exc.args[1]
             print('Missing quote qualifier for parameter ' + "'" + violator + "'" + ' in ' + fin.name + '.', file=sys.stderr)             print('Missing quote qualifier for parameter ' + "'" + violator + "'" + ' in ' + fin.name + '.', file=sys.stderr)
             rv = bool(1)              error = True
         elif msg == 'paramNameTooLong':         elif msg == 'paramNameTooLong':
             violator = exc.args[1]             violator = exc.args[1]
             print('Macro name ' + "'" + violator + "' is too long.", file=sys.stderr)             print('Macro name ' + "'" + violator + "' is too long.", file=sys.stderr)
             rv = bool(1)              error = True
         elif msg == 'unknownSection':         elif msg == 'unknownSection':
             violator = exc.args[1]             violator = exc.args[1]
             print('Unknown section ' + "'" + violator + "' in configuration file.", file=sys.stderr)             print('Unknown section ' + "'" + violator + "' in configuration file.", file=sys.stderr)
             rv = bool(1)              error = True
         else:         else:
             # re-raise the exception             # re-raise the exception
             raise             raise
  
     if not rv:      if not error:
         if not xml is None:          if project_includes is not None:
             # Process xml.              error = process_project_repos(project_includes)
             projRules.extend(list(RULESPREFIX))  
             projTarget.extend(list(TARGETPREFIX))  
             rv = processXML(xml, projRules, projTarget)  
             projRules.extend(RULESSUFFIX)  
  
     # Process addenda - these are parameters that are not configurable and must be set in the     # Process addenda - these are parameters that are not configurable and must be set in the
     # NetDRMS build.     # NetDRMS build.
     if not rv:      if not error:
         iscfg = bool(0)         iscfg = bool(0)
         for key in addenda:          for key, val in addenda.items():
             item = key + ' ' + addenda[key]              item = f'{key} {val}'
             ppRet = processParam(iscfg, item, regexpQuote, regexp, keymap, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, pyConstSection, pyInitSection, platDict, machDict, 'defs')              print(f'addenda item {item}')
             if ppRet:              if ignoreKeymap:
                 break;                  keymapActual = None
               else:
                   keymapActual = keymap
               ppRet = processParam(iscfg, item, regexpQuote, regexp, keymapActual, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection, platDict, machDict, 'defs')
  
     # Put information collected in platDict and machDict into mDefs. Must do this here, and not in processParam, since     # Put information collected in platDict and machDict into mDefs. Must do this here, and not in processParam, since
     # we need to parse all platform-specific make variables before grouping them into platform categories.     # we need to parse all platform-specific make variables before grouping them into platform categories.
     if not rv:      if not error:
         for plat in platDict:         for plat in platDict:
             mDefsMake.extend(list('\nifeq ($(JSOC_MACHINE), linux_' + plat.lower() + ')'))             mDefsMake.extend(list('\nifeq ($(JSOC_MACHINE), linux_' + plat.lower() + ')'))
             for var in platDict[plat]:             for var in platDict[plat]:
                 mDefsMake.extend(list('\n' + var + ' = ' + platDict[plat][var]))                 mDefsMake.extend(list('\n' + var + ' = ' + platDict[plat][var]))
             mDefsMake.extend(list('\nendif\n'))             mDefsMake.extend(list('\nendif\n'))
  
     if not rv:      if not error:
         for mach in machDict:         for mach in machDict:
             mDefsMake.extend(list('\nifeq ($(MACHTYPE), ' + mach + ')'))             mDefsMake.extend(list('\nifeq ($(MACHTYPE), ' + mach + ')'))
             for var in machDict[mach]:             for var in machDict[mach]:
                 mDefsMake.extend(list('\n' + var + ' = ' + machDict[mach][var]))                 mDefsMake.extend(list('\n' + var + ' = ' + machDict[mach][var]))
             mDefsMake.extend(list('\nendif\n'))             mDefsMake.extend(list('\nendif\n'))
     return rv      return error
  
 def getMgrUIDLine(defs, uidParam):  def getMgrUIDLine(sums_manager, uidParam):
     rv = bool(0)      error = False
  
     cmd = 'id -u ' + defs['SUMS_MANAGER']      if sums_manager and len(sums_manager) > 0:
           cmd = [ 'id', '-u', shlex.quote(sums_manager) ]
     try:     try:
         ret = check_output(cmd, shell=True)              ret = check_output(cmd)
         uidParam['q:SUMS_MANAGER_UID'] = ret.decode("utf-8")         uidParam['q:SUMS_MANAGER_UID'] = ret.decode("utf-8")
     except ValueError:     except ValueError:
         print('Unable to run cmd: ' + cmd + '.')         print('Unable to run cmd: ' + cmd + '.')
         rv = bool(1)              error = True
     except CalledProcessError:     except CalledProcessError:
         print('Command ' + "'" + cmd + "'" + ' ran improperly.')         print('Command ' + "'" + cmd + "'" + ' ran improperly.')
         rv = bool(1)              error = True
  
     return rv      return error
  
 def isVersion(maj, min, majDef, minDef): def isVersion(maj, min, majDef, minDef):
     res = 0     res = 0
Line 747  def configureComps(defs, mDefs):
Line 777  def configureComps(defs, mDefs):
  
         if not hasicc and not hasgcc:         if not hasicc and not hasgcc:
             print('Fatal error: Acceptable C compiler not found! You will be unable to build the DRMS library.', file=sys.stderr)             print('Fatal error: Acceptable C compiler not found! You will be unable to build the DRMS library.', file=sys.stderr)
             rv = bool(1)              rv = bool(0) # Art - don't bail, we might be using drmsparams.py and not care about the rest of DRMS
         elif hasicc:         elif hasicc:
             mDefs.extend(list('\nCOMPILER = icc'))             mDefs.extend(list('\nCOMPILER = icc'))
               # mDefs.extend(list('\nICC_VERSION = blah'))
         else:         else:
             mDefs.extend(list('\nCOMPILER = gcc'))             mDefs.extend(list('\nCOMPILER = gcc'))
  
Line 766  def configureComps(defs, mDefs):
Line 797  def configureComps(defs, mDefs):
  
     return rv     return rv
  
 def writeParamsFiles(base, cfile, mfile, pfile, pyfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection, pyConstSection, pyInitSection):  def writeParamsFiles(base, cfile, mfile, pfile, pyfile, shfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection):
     rv = bool(0)     rv = bool(0)
  
     # Merge mDefsGen, mDefsMake, and mDefsComps into a single string with compiler configuration first, general parameters next, then     # Merge mDefsGen, mDefsMake, and mDefsComps into a single string with compiler configuration first, general parameters next, then
Line 774  def writeParamsFiles(base, cfile, mfile,
Line 805  def writeParamsFiles(base, cfile, mfile,
     mDefs = '\n# Compiler Selection\n' + ''.join(mDefsComps) + '\n\n# General Parameters\n' + ''.join(mDefsGen) + '\n\n# Parameters to Configure make\n' + ''.join(mDefsMake)     mDefs = '\n# Compiler Selection\n' + ''.join(mDefsComps) + '\n\n# General Parameters\n' + ''.join(mDefsGen) + '\n\n# Parameters to Configure make\n' + ''.join(mDefsMake)
  
     try:     try:
         with open(cfile, 'w') as cout, open(mfile, 'w') as mout, open(pfile, 'w') as pout, open(pyfile, 'w') as pyout:          with open(cfile, 'w') as cout, open(mfile, 'w') as mout, open(pfile, 'w') as pout, open(pyfile, 'w') as pyout, open(shfile, 'w') as shout:
             # C file of macros             # C file of macros
             print(C_PREFIX, file=cout)             print(C_PREFIX, file=cout)
             print('/* This file contains a set of preprocessor macros - one for each configuration parameter. */\n', file=cout)             print('/* This file contains a set of preprocessor macros - one for each configuration parameter. */\n', file=cout)
Line 808  def writeParamsFiles(base, cfile, mfile,
Line 839  def writeParamsFiles(base, cfile, mfile,
             print(PY_BINPATH, file=pyout)             print(PY_BINPATH, file=pyout)
             print(PREFIX, file=pyout)             print(PREFIX, file=pyout)
             print('# This file contains a set of constants - one for each configuration parameter.\n', file=pyout)             print('# This file contains a set of constants - one for each configuration parameter.\n', file=pyout)
               print(PY_ALL, file=pyout)
             print(''.join(pyConstSection), file=pyout)             print(''.join(pyConstSection), file=pyout)
   
               print(PY_ERROR_CLASSES, file=pyout)
             print(PY_FXNS_A, file=pyout, end='')             print(PY_FXNS_A, file=pyout, end='')
             print(''.join(pyInitSection), file=pyout)             print(''.join(pyInitSection), file=pyout)
             print(PY_FXNS_B, file=pyout)             print(PY_FXNS_B, file=pyout)
               print(PY_FXNS_C, file=pyout)
   
               # Shell (bash) source file
               print(SH_BINPATH, file=shout)
               print(PREFIX, file=shout)
               print('# This file contains a set of variable assignments - one for each configuration parameter.\n', file=shout)
               print(''.join(shConstSection), file=shout)
  
     except IOError as exc:     except IOError as exc:
         type, value, traceback = sys.exc_info()         type, value, traceback = sys.exc_info()
Line 821  def writeParamsFiles(base, cfile, mfile,
Line 862  def writeParamsFiles(base, cfile, mfile,
  
     return rv     return rv
  
 def writeProjFiles(pCfile, pMfile, pRfile, pTfile, projCfg, projMkRules, projRules, projTarget):  def write_project_includes(project_includes_file, project_includes):
     rv = bool(0)      error = False
  
       if project_includes_file is not None:
     try:     try:
         if projCfg:              with open(project_includes_file, 'w') as file_out:
             with open(pCfile, 'w') as cout:  
                 # configure  
                 print(''.join(projCfg), file=cout)  
   
         if projMkRules:  
             with open(pMfile, 'w') as mout:  
                 # make_basic.mk  
                 print(PREFIX, file=mout)  
                 print(''.join(projMkRules), file=mout)  
   
         if projRules:  
             with open(pRfile, 'w') as rout:  
                 # Rules.mk                 # Rules.mk
                 print(PREFIX, file=rout)                  print(PREFIX, file=file_out)
                 print(''.join(projRules), file=rout)                  print(RULESPREFIX, file=file_out)
  
         if projTarget:                  for dir_variable in project_includes:
             with open(pTfile, 'w') as tout:                      print(dir_variable, file=file_out)
                 # target.mk                      print(f'-include          $(SRCDIR)/$(dir)/Rules.mk', file=file_out)
                 print(PREFIX, file=tout)  
                 print(''.join(projTarget), file=tout)  
     except IOError as exc:  
         type, value, traceback = sys.exc_info()  
         print(exc.strerror, file=sys.stderr)  
         print('Unable to open ' + "'" + value.filename + "'.", file=sys.stderr)  
         rv = bool(1)  
  
     if not rv:                  print('', file=file_out)
         if os.path.exists(pCfile):                  print(RULESSUFFIX, file=file_out)
             try:  
                 os.chmod(pCfile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IROTH)  
             except OSError as exc:  
                 type, value, traceback = sys.exc_info()  
                 print('Unable to chmod file ' + "'" + value.filename + "'.", file=sys.stderr)  
                 print(exc.strerror, file=sys.stderr)  
                 rv = bool(1)  
  
     return rv          except IOError as exc:
               print(f'unable to open {project_includes_file} for writing ({str(exc)})', file=sys.stderr)
               error = True
   
       return error
  
 def generateSumRmCfg(defs): def generateSumRmCfg(defs):
     rv = bool(0)     rv = bool(0)
Line 975  def generateSumRmCfg(defs):
Line 995  def generateSumRmCfg(defs):
  
     return rv     return rv
  
 def configureNet(cfgfile, cfile, mfile, pfile, pyfile, pCfile, pMfile, pRfile, pTfile, base, keymap, createSumRmCfg):  def configureNet(cfgfile, cfile, mfile, pfile, pyfile, shfile, project_includes_file, base, keymap, createSumRmCfg):
     rv = bool(0)      error = False
  
     defs = {}     defs = {}
     cDefs = list()     cDefs = list()
     mDefsGen = list()     mDefsGen = list()
     mDefsMake = list()     mDefsMake = list()
     mDefsComps = list()     mDefsComps = list()
     projCfg = list()      project_includes = []
     projMkRules = list()  
     projRules = list()  
     projTarget = list()  
     perlConstSection = list()     perlConstSection = list()
     perlInitSection = list()     perlInitSection = list()
     pyConstSection = list()     pyConstSection = list()
     pyInitSection = list()     pyInitSection = list()
       shConstSection = list()
     addenda = {}     addenda = {}
  
     # There are three parameters that were not included in the original config.local parameter set, for some reason.     # There are three parameters that were not included in the original config.local parameter set, for some reason.
Line 1008  def configureNet(cfgfile, cfile, mfile,
Line 1026  def configureNet(cfgfile, cfile, mfile,
     try:     try:
         with open(cfgfile, 'r') as fin:         with open(cfgfile, 'r') as fin:
             # Process configuration parameters             # Process configuration parameters
             rv = parseConfig(fin, keymap, addenda, defs, cDefs, mDefsGen, mDefsMake, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection, pyConstSection, pyInitSection)  
             if not rv:              error = parseConfig(fin, keymap, addenda, defs, cDefs, mDefsGen, mDefsMake, project_includes, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
               if not error:
                 # Must add a parameter for the SUMS_MANAGER UID (for some reason). This must be done after the                 # Must add a parameter for the SUMS_MANAGER UID (for some reason). This must be done after the
                 # config file is processed since an input to getMgrUIDLine() is one of the config file's                 # config file is processed since an input to getMgrUIDLine() is one of the config file's
                 # parameter values.                 # parameter values.
   
                   # CANNOT run this unless sunroom2 home directories are accessible
                 uidParam = {}                 uidParam = {}
                 rv = getMgrUIDLine(defs, uidParam)                  error = getMgrUIDLine(defs.get('SUMS_MANAGER', None), uidParam)
                 if rv == bool(0):                  if not error and len(uidParam) > 0:
                     rv = parseConfig(None, keymap, uidParam, defs, cDefs, mDefsGen, None, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection, pyConstSection, pyInitSection)                      error = parseConfig(None, keymap, uidParam, defs, cDefs, mDefsGen, None, None, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
   
                   # ignore the error where the SUMS manager UID cannot be determined
                   error = False
   
             # Configure the compiler-selection make variables.             # Configure the compiler-selection make variables.
             if not rv:              if not error:
                 rv = configureComps(defs, mDefsComps)                  error = configureComps(defs, mDefsComps)
  
             # Write out the parameter files.             # Write out the parameter files.
             if not rv:              if not error:
                 rv = writeParamsFiles(base, cfile, mfile, pfile, pyfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection, pyConstSection, pyInitSection)                  error = writeParamsFiles(base, cfile, mfile, pfile, pyfile, shfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
  
             # Write out the project-specific make files (make_basic.mk, Rules.mk, and target.mk).              if not error:
             if not rv:                  error = write_project_includes(project_includes_file, project_includes)
                 rv = writeProjFiles(pCfile, pMfile, pRfile, pTfile, projCfg, projMkRules, projRules, projTarget)  
  
             # Write out the sum_rm.cfg file.             # Write out the sum_rm.cfg file.
             if not rv and createSumRmCfg:              if not error and createSumRmCfg:
                 rv = generateSumRmCfg(defs)                  error = generateSumRmCfg(defs)
     except IOError as exc:     except IOError as exc:
         print(exc.strerror, file=sys.stderr)         print(exc.strerror, file=sys.stderr)
         print('Unable to read configuration file ' + cfgfile + '.', file=sys.stderr)         print('Unable to read configuration file ' + cfgfile + '.', file=sys.stderr)
Line 1044  def configureNet(cfgfile, cfile, mfile,
Line 1068  def configureNet(cfgfile, cfile, mfile,
  
         if type == 'unexpectedIccRet':         if type == 'unexpectedIccRet':
             print('icc -V returned this unexpected message:\n' + msg, file=sys.stderr)             print('icc -V returned this unexpected message:\n' + msg, file=sys.stderr)
             rv = bool(1)              error = True
         elif type == 'unexpectedGccRet':         elif type == 'unexpectedGccRet':
             print('gcc -v returned this unexpected message:\n' + msg, file=sys.stderr)             print('gcc -v returned this unexpected message:\n' + msg, file=sys.stderr)
             rv = bool(1)              error = True
         elif type == 'unexpectedIfortRet':         elif type == 'unexpectedIfortRet':
             print('ifort -V returned this unexpected message:\n' + msg, file=sys.stderr)             print('ifort -V returned this unexpected message:\n' + msg, file=sys.stderr)
             rv = bool(1)              error = True
         elif type == 'unexpectedGfortranRet':         elif type == 'unexpectedGfortranRet':
             print('gfortran -v returned this unexpected message:\n' + msg, file=sys.stderr)             print('gfortran -v returned this unexpected message:\n' + msg, file=sys.stderr)
             rv = bool(1)              error = True
         else:         else:
             # re-raise the exception             # re-raise the exception
             raise             raise
  
     return rv      return error
  
 def configureSdp(cfgfile, cfile, mfile, pfile, pyfile, pCfile, pMfile, pRfile, pTfile, base):  def configureSdp(cfgfile, cfile, mfile, pfile, pyfile, shfile, project_includes_file, base):
     rv = bool(0)      error = False
  
     defs = {}     defs = {}
     cDefs = list()     cDefs = list()
     mDefsGen = list()     mDefsGen = list()
     mDefsMake = list()     mDefsMake = list()
     projCfg = list()  
     projMkRules = list()  
     projRules = list()  
     projTarget = list()  
     mDefsComps = list()     mDefsComps = list()
       project_includes = []
     perlConstSection = list()     perlConstSection = list()
     perlInitSection = list()     perlInitSection = list()
     pyConstSection = list()     pyConstSection = list()
     pyInitSection = list()     pyInitSection = list()
       shConstSection = list()
  
     addenda = {}     addenda = {}
  
Line 1090  def configureSdp(cfgfile, cfile, mfile,
Line 1112  def configureSdp(cfgfile, cfile, mfile,
  
     try:     try:
         with open(cfgfile, 'r') as fin:         with open(cfgfile, 'r') as fin:
             rv = parseConfig(fin, None, addenda, defs, cDefs, mDefsGen, mDefsMake, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection, pyConstSection, pyInitSection)              error = parseConfig(fin, None, addenda, defs, cDefs, mDefsGen, mDefsMake, project_includes, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
  
             if not rv:              if not error:
                 # Must add a parameter for the SUMS_MANAGER UID (for some reason)                 # Must add a parameter for the SUMS_MANAGER UID (for some reason)
                 uidParam = {}                 uidParam = {}
                 rv = getMgrUIDLine(defs, uidParam)                  error = getMgrUIDLine(defs.get('SUMS_MANAGER', None), uidParam)
                 if not rv:                  if not error and len(uidParam) > 0:
                     rv = parseConfig(None, None, uidParam, defs, cDefs, mDefsGen, None, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection, pyConstSection, pyInitSection)                      error = parseConfig(None, None, uidParam, defs, cDefs, mDefsGen, None, None, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
   
                   # ignore the error where the SUMS manager UID cannot be determined
                   error = False
  
                 # Configure the compiler-selection make variables.                 # Configure the compiler-selection make variables.
                 if not rv:                  if not error:
                     rv = configureComps(defs, mDefsComps)                      error = configureComps(defs, mDefsComps)
  
                 # Write out the parameter files.                 # Write out the parameter files.
                 if not rv:                  if not error:
                     rv = writeParamsFiles(base, cfile, mfile, pfile, pyfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection, pyConstSection, pyInitSection)                      error = writeParamsFiles(base, cfile, mfile, pfile, pyfile, shfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection, pyConstSection, pyInitSection, shConstSection)
  
                 # Write out the project-specific make files (make_basic.mk, Rules.mk, and target.mk).                  if not error:
                 if not rv:                      error = write_project_includes(project_includes_file, project_includes)
                     rv = writeProjFiles(pCfile, pMfile, pRfile, pTfile, projCfg, projMkRules, projRules, projTarget)  
  
                 # At Stanford, skip the creation of the sum_rm configuration file. config.local will still                 # At Stanford, skip the creation of the sum_rm configuration file. config.local will still
                 # have the SUMRM parameters, but they will not be used.                 # have the SUMRM parameters, but they will not be used.
Line 1126  def configureSdp(cfgfile, cfile, mfile,
Line 1150  def configureSdp(cfgfile, cfile, mfile,
         if type == 'unexpectedIccRet':         if type == 'unexpectedIccRet':
             msg = exc.args[1]             msg = exc.args[1]
             print('icc -V returned this unexpected message:\n' + msg, file=sys.stderr)             print('icc -V returned this unexpected message:\n' + msg, file=sys.stderr)
             rv = bool(1)              error = True
         elif type == 'unexpectedGccRet':         elif type == 'unexpectedGccRet':
             msg = exc.args[1]             msg = exc.args[1]
             print('gcc -v returned this unexpected message:\n' + msg, file=sys.stderr)             print('gcc -v returned this unexpected message:\n' + msg, file=sys.stderr)
             rv = bool(1)              error = True
         elif type == 'unexpectedIfortRet':         elif type == 'unexpectedIfortRet':
             msg = exc.args[1]             msg = exc.args[1]
             print('ifort -V returned this unexpected message:\n' + msg, file=sys.stderr)             print('ifort -V returned this unexpected message:\n' + msg, file=sys.stderr)
             rv = bool(1)              error = True
         elif type == 'unexpectedGfortranRet':         elif type == 'unexpectedGfortranRet':
             msg = exc.args[1]             msg = exc.args[1]
             print('gfortran -v returned this unexpected message:\n' + msg, file=sys.stderr)             print('gfortran -v returned this unexpected message:\n' + msg, file=sys.stderr)
             rv = bool(1)              error = True
         else:         else:
             # re-raise the exception             # re-raise the exception
             raise             raise
  
     return rv      return error
  
 # Beginning of program # Beginning of program
 rv = RET_SUCCESS rv = RET_SUCCESS
Line 1172  if rv == RET_SUCCESS:
Line 1196  if rv == RET_SUCCESS:
     mfile = optD['dir'] + '/' + optD['base'] + '.mk'     mfile = optD['dir'] + '/' + optD['base'] + '.mk'
     pfile = optD['dir'] + '/' + optD['base'] + '.pm'     pfile = optD['dir'] + '/' + optD['base'] + '.pm'
     pyfile = optD['dir'] + '/' + optD['base'] + '.py'     pyfile = optD['dir'] + '/' + optD['base'] + '.py'
     pCfile = optD['dir'] + '/configure'      shfile = optD['dir'] + '/' + optD['base'] + '.sh'
     pMfile = optD['dir'] + '/make_basic.mk'      project_includes_file = os.path.join(optD['dir'], 'includes.mk')
     pRfile = optD['dir'] + '/Rules.mk'  
     pTfile = optD['dir'] + '/target.mk'  
  
     if net:     if net:
         try:         try:
Line 1202  if rv == RET_SUCCESS:
Line 1224  if rv == RET_SUCCESS:
  
         # We also need to set the UID of the SUMS manager. We have the name of the         # We also need to set the UID of the SUMS manager. We have the name of the
         # SUMS manager (it is in the configuration file)         # SUMS manager (it is in the configuration file)
         configureNet(NET_CFG, cfile, mfile, pfile, pyfile, pCfile, pMfile, pRfile, pTfile, optD['base'], keymap, 'server' in optD)          configureNet(NET_CFG, cfile, mfile, pfile, pyfile, shfile, project_includes_file, optD['base'], keymap, 'server' in optD)
     else:     else:
         # A Stanford user can override the parameters in configsdp.txt by copying that file to config.local,         # A Stanford user can override the parameters in configsdp.txt by copying that file to config.local,
         # and then editing config.local. So, if config.local exists, use that.         # and then editing config.local. So, if config.local exists, use that.
         if os.path.isfile(cdir + '/' + NET_CFG):         if os.path.isfile(cdir + '/' + NET_CFG):
             configureSdp(NET_CFG, cfile, mfile, pfile, pyfile, pCfile, pMfile, pRfile, pTfile, optD['base'])              configureSdp(NET_CFG, cfile, mfile, pfile, pyfile, shfile, project_includes_file, optD['base'])
         else:         else:
             configureSdp(SDP_CFG, cfile, mfile, pfile, pyfile, pCfile, pMfile, pRfile, pTfile, optD['base'])              configureSdp(SDP_CFG, cfile, mfile, pfile, pyfile, shfile, project_includes_file, optD['base'])

Legend:
Removed from v.1.19  
changed lines
  Added in v.1.33
Karen Tian
 Powered by
ViewCVS 0.9.4