(file) Return to localize.py CVS log (file) (dir) Up to [Development] / JSOC

Diff for /JSOC/localize.py between version 1.2 and 1.14

version 1.2, 2013/11/07 21:03:04 version 1.14, 2014/04/17 19:11:05
Line 1 
Line 1 
 #!/home/jsoc/bin/linux_x86_64/activepython #!/home/jsoc/bin/linux_x86_64/activepython
  
 import os.path  # When run with the -s flag, localize.py configures the SUMS-server component of NetDRMS.
   
 import sys import sys
 import getopt import getopt
 import re import re
   import os
   import stat
   import filecmp
   import xml.etree.ElementTree as ET
 from subprocess import check_output, CalledProcessError from subprocess import check_output, CalledProcessError
  
 # Constants # Constants
Line 40  PERL_FXNS_A = """sub new
Line 45  PERL_FXNS_A = """sub new
     };     };
  
     bless($self, $clname);     bless($self, $clname);
     $self->{_paramsH} = $self->initialize();      $self->{_paramsH} = {};
       $self->initialize();
  
     return $self;     return $self;
 } }
Line 68  PERL_FXNS_B = """sub get
Line 74  PERL_FXNS_B = """sub get
 } }
 1;""" 1;"""
  
   SUMRM_COMMENT = """# This is the configuration file for the sum_rm program. It was auto-generated by the DRMS master configure script.
   # It controls the behavior of the sum_rm program, and is loaded each time sum_rm runs. To change the
   # parameter values in this configuration file, modify config.local, then re-run configure. This configuration
   # file will be updated only if parameters affecting it are modified. If such changes are made to config.local,
   # please make sure that the sum_rm service is disabled while configure in running.
   """
   
   SUMRM_DOC = """# sum_rm removes end-of-life SUMS data files to prevent disk-partitions from becomming 100% full.
   # sum_svc, the main SUMS service, starts this daemon when it is launched. Within an infinite loop, sum_rm sleeps
   # for SLEEP number of seconds (defined below). When it awakes, it loads the sum_rm configuration file. For each SU,
   # it then examines the 'effective_date' column within the sum_partn_alloc table of the SUMS database. It does so by
   # sorting all SUs by effective date (this date is actually an expiration date - the SU is good until the end of that day),
   # and then, starting with the SU with the oldest effective date, it deletes the SUs. It continues deleting SUs until one
   # of three conditions is met:
   #
   #   (a) The disk has at least PART_PERCENT_FREE percent free space.
   #   (b) All SUs have effective_date values in the future.
   #   (c) At least 600 SUs have been deleted (this is defined in the LIMIT statement in the file SUMLIB_RmDoX.pgc).
   #
   # After sum_rm stops deleteing SUs, it then sleeps for SLEEP seconds, completing the first iteration of the
   # infinite loop.
   """
   
   SUMRM_PARTN_PERCENT_FREE = """
   # This is the percentage at which all disk partitions are to be kept free.
   # If not specified, this defaults to 3. For example, setting PART_PERCENT_FREE = 5 will allow all partitions to
   # fill to 95% full. Dividing the number of unused blocks by the total number of blocks, and rounding up,
   # will result in the number specified by PART_PERCENT_FREE.
   #
   # NOTE : This behavior was previously controlled by the MAX_FREE_{n} family of parameters. {n} referred to the
   # disk-partition number and the value of parameter MAX_FREE_{n} was the MINIMUM number of free MB in the
   # partition [No clue why the word MAX was used]. As of NetDRMS 7.0, MAX_FREE_{n} has no effect."""
   
   SUMRM_SLEEP = """
   # The value is the number of seconds to sleep between iterations of the main loop in sum_rm."""
   
   SUMRM_LOG = """
   # The value is the log file (opened only at sum_rm startup; the sum_rm pid is appended to this file name)."""
   
   SUMRM_MAIL = """
   # The value is the email address of the recipient to be notified in case of a problem."""
   
   SUMRM_NOOP = """
   # If the value is set to anything other than 0, then sum_rm is rendered inactive. Otherwise, sum_rm is active."""
   
   SUMRM_USER = """
   # The value designates the linux user who is allowed to run sum_rm."""
   
   SUMRM_NORUN = """
   # This pair of paramters defines a window of time, during which sum_rm will become torpid - in this
   # window, sum_rm will not scan for SUs to delete, not will it delete any SUs. Each value is an integer, 0-23, that
   # represents an hour of the day. For example, if NORUN_START=8 and NORUN_STOP=10, then between 8am and 10am
   # local time, sum_rm will be dormant. To disable this behavior, set both parameters to the same value."""
   
   RULESPREFIX = """# Standard things
   sp              := $(sp).x
   dirstack_$(sp)  := $(d)
   d               := $(dir)
   """
   
   RULESSUFFIX = """dir    := $(d)/example
   -include                $(SRCDIR)/$(dir)/Rules.mk
   dir     := $(d)/cookbook
   -include                $(SRCDIR)/$(dir)/Rules.mk
   dir     := $(d)/myproj
   -include                $(SRCDIR)/$(dir)/Rules.mk
   
   # Standard things
   d               := $(dirstack_$(sp))
   sp              := $(basename $(sp))
   """
   
   TARGETPREFIX = """$(PROJOBJDIR):\n\t+@[ -d $@ ] || mkdir -p $@"""
   
   ICC_MAJOR = 9
   ICC_MINOR = 0
   GCC_MAJOR = 3
   GCC_MINOR = 0
   IFORT_MAJOR = 9
   IFORT_MINOR = 0
   GFORT_MAJOR = 4
   GFORT_MINOR = 2
   
   
 # Read arguments # Read arguments
 #  d - localization directory #  d - localization directory
 #  b - base name of all parameter files (e.g., -b drmsparams --> drmsparams.h, drmsparams.mk, drmsparams.pm, etc.) #  b - base name of all parameter files (e.g., -b drmsparams --> drmsparams.h, drmsparams.mk, drmsparams.pm, etc.)
 #  m - make file  #  s - configure a NetDRMS server (i.e., SUMS server). Otherwise, do client configuration only. A server
   #      configuration will modify something that only the production user has access to. An example is the sum_rm.cfg
   #      file. To modify that file, the user running configure must be running configure on the SUMS-server host, and
   #      must have write permission on sum_rm.cfg.
 def GetArgs(args): def GetArgs(args):
     rv = bool(0)     rv = bool(0)
     optD = {}     optD = {}
  
     try:     try:
         opts, remainder = getopt.getopt(args, "hd:b:",["dir=", "base="])          opts, remainder = getopt.getopt(args, "hd:b:s",["dir=", "base="])
     except getopt.GetoptError:     except getopt.GetoptError:
         print('Usage:')         print('Usage:')
         print('localize.py [-h] -d <localization directory> -b <parameter file base>')         print('localize.py [-h] -d <localization directory> -b <parameter file base>')
Line 96  def GetArgs(args):
Line 189  def GetArgs(args):
                     optD['dir'] = matchobj.group(1)                     optD['dir'] = matchobj.group(1)
             elif opt in ("-b", "--base"):             elif opt in ("-b", "--base"):
                 optD['base'] = arg                 optD['base'] = arg
               elif opt in ("-s"):
                   optD['server'] = ""
             else:             else:
                 optD[opt] = arg                 optD[opt] = arg
  
Line 125  def createPerlConst(key, val, keyColLen,
Line 220  def createPerlConst(key, val, keyColLen,
         status = bool(0)         status = bool(0)
         return 'use constant ' + key + ' => ' + spaces + val + ';\n'         return 'use constant ' + key + ' => ' + spaces + val + ';\n'
  
 def processParam(cfgfile, line, regexpComm, regexpDefs, regexpMake, regexpQuote, regexp, keymap, defs, cDefs, mDefs, perlConstSection, perlInitSection, section):  def isSupportedPlat(plat):
     status = 0      regexp = re.compile(r"\s*(^x86_64|^ia32|^ia64|^avx)", re.IGNORECASE)
       matchobj = regexp.match(plat);
  
     matchobj = regexpComm.match(line)  
     if not matchobj is None:     if not matchobj is None:
         # Skip comment line          return bool(1);
         return bool(0)      else:
           return bool(0);
  
     matchobj = regexpDefs.match(line)  def processMakeParam(mDefs, key, val, platDict, machDict):
       varMach = None
       regexp = re.compile(r"(\S+):(\S+)")
       matchobj = regexp.match(key)
     if not matchobj is None:     if not matchobj is None:
         section.extend(list('defs'))          varName = matchobj.group(1)
         return bool(0)          varMach = matchobj.group(2)
       else:
           varName = key
  
     matchobj = regexpMake.match(line)      varValu = val
     if not matchobj is None:  
         section = 'make'      if varMach is None:
         return bool(1)          mDefs.extend(list('\n' + varName + ' = ' + varValu))
       else:
           if isSupportedPlat(varMach):
               # The guard will compare varValu to $JSOC_MACHINE.
               if not varMach in platDict:
                   platDict[varMach] = {}
               platDict[varMach][varName] = varValu
           else:
               # The guard will compare varValu to $MACHINETYPE (this is just the hostname of the machine on which localize.py is running).
               if not varMach in machDict:
                   machDict[varMach] = {}
               machDict[varMach][varName] = varValu
   
   def processParam(cfgfile, line, regexpQuote, regexp, keymap, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, platDict, machDict, section):
       status = 0
  
     if ''.join(section) == 'defs' or not cfgfile:     if ''.join(section) == 'defs' or not cfgfile:
         matchobj = regexp.match(line)         matchobj = regexp.match(line)
Line 156  def processParam(cfgfile, line, regexpCo
Line 271  def processParam(cfgfile, line, regexpCo
                 if keyCfgSp in keymap:                 if keyCfgSp in keymap:
                     key = keymap[keyCfgSp]                     key = keymap[keyCfgSp]
                 elif keyCfgSp == 'LOCAL_CONFIG_SET' or keyCfgSp == 'DRMS_SAMPLE_NAMESPACE':                 elif keyCfgSp == 'LOCAL_CONFIG_SET' or keyCfgSp == 'DRMS_SAMPLE_NAMESPACE':
                     # Ignore parameters that are not useful and shouldn't have been there in the first place                      # Ignore parameters that are not useful and shouldn't have been there in the first place. But
                       # they have been released to the world, so we have to account for them.
                     return bool(0)                     return bool(0)
                 elif not cfgfile:                 elif not cfgfile:
                     # Should not be doing mapping for addenda                     # Should not be doing mapping for addenda
Line 192  def processParam(cfgfile, line, regexpCo
Line 308  def processParam(cfgfile, line, regexpCo
                     raise Exception('paramNameTooLong', key)                     raise Exception('paramNameTooLong', key)
  
                 # Make file - val should never be quoted; just use as is                 # Make file - val should never be quoted; just use as is
                 mDefs.extend(list('\n' + key + ' = ' + val))                  mDefsGen.extend(list('\n' + key + ' = ' + val))
  
                 # Perl file - val should ALWAYS be single-quote quoted                 # Perl file - val should ALWAYS be single-quote quoted
                 # Save const info to a string                 # Save const info to a string
Line 205  def processParam(cfgfile, line, regexpCo
Line 321  def processParam(cfgfile, line, regexpCo
                 # constants (the names of which are the parameter names)                 # constants (the names of which are the parameter names)
                 # we can refer to those in the init section. The key variable holds the                 # we can refer to those in the init section. The key variable holds the
                 # name of the constant.                 # name of the constant.
                 perlInitSection.extend(list('\n  $self->{_paramsH}->{' + key + '} = ' + "'" + val + "';"))                  perlInitSection.extend(list("\n  $self->{_paramsH}->{'" + key + "'} = " + key + ';'))
             else:             else:
                 # No quote qualifier                 # No quote qualifier
                 raise Exception('missingQuoteQual', key)                 raise Exception('missingQuoteQual', key)
       elif ''.join(section) == 'make' and cfgfile:
           # Configure the remaining make variables defined in the __MAKE__ section of the configuration file. Third-party
           # library make variables are specified in the __MAKE__ section.
           matchobj = regexp.match(line)
           if not matchobj is None:
               # We have a key-value line
               key = matchobj.group(1)
               val = matchobj.group(2)
   
               # This information is for making make variables only. We do not need to worry about quoting any values
               defs[key] = val
               processMakeParam(mDefsMake, key, val, platDict, machDict)
  
     return bool(0)     return bool(0)
  
 # We have some extraneous line or a newline - ignore. # We have some extraneous line or a newline - ignore.
  
   def processXML(xml, projRules, projTarget):
       rv = bool(0)
   
       # <projects>
       root = ET.fromstring(xml)
   
       # Iterate through each proj child.
       for proj in root.iter('proj'):
           # Rules.mk
           nameElem = proj.find('name')
           rulesStr = 'dir     := $(d)/' + nameElem.text + '\n-include          $(SRCDIR)/$(dir)/Rules.mk\n'
   
           # make doesn't support logical operations in ifeq conditionals (you can't do ifeq (A AND B)),
           # so we need to write:
           #   ifeq (A)
           #      ifeq (B)
           #        <do something>
           #      endif
           #   endif
   
           rulesPref = '';
           rulesSuff = '';
   
           filters = proj.find('filters')
           if filters is not None:
               for filter in filters.findall('filter'):
                   rulesPref += 'ifeq ($(' + filter.find('name').text + '),' + filter.find('value').text + ')\n'
                   rulesSuff += 'endif\n'
   
           if len(rulesPref) > 0 and len(rulesSuff) > 0:
               projRules.extend(list(rulesPref))
               projRules.extend(list(rulesStr))
               projRules.extend(list(rulesSuff))
           else:
               projRules.extend(list(rulesStr))
   
           # target.mk
           subdirs = proj.find('subdirs')
           if subdirs is not None:
               for subdir in subdirs.findall('subdir'):
                   targetStr = '\n\t+@[ -d $@/' + nameElem.text + '/' + subdir.text + ' ] || mkdir -p $@/' + nameElem.text + '/' + subdir.text
                   projTarget.extend(list(targetStr))
   
       return rv
   
   def determineSection(line, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg):
       matchobj = regexpDefs.match(line)
       if not matchobj is None:
           return 'defs'
   
       matchobj = regexpMake.match(line)
       if not matchobj is None:
           return 'make'
   
       matchobj = regexpProjMkRules.match(line)
       if not matchobj is None:
           return 'projmkrules'
   
       matchobj = regexpProj.match(line)
       if not matchobj is None:
           return 'proj'
   
       matchobj = regexpProjCfg.match(line)
       if not matchobj is None:
           return 'projcfg'
   
       return None
   
 # defs is a dictionary containing all parameters (should they be needed in this script) # defs is a dictionary containing all parameters (should they be needed in this script)
 def parseConfig(fin, keymap, addenda, defs, cDefs, mDefs, perlConstSection, perlInitSection):  # projCfg is the list containing the configure script content.
   # projMkRules is the list containing the make_basic.mk content.
   # projRules is the list containing the Rules.mk content.
   # projTargert is the list containing the target.mk content.
   def parseConfig(fin, keymap, addenda, defs, cDefs, mDefsGen, mDefsMake, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection):
     rv = bool(0)     rv = bool(0)
  
     # Open required config file (config.local)     # Open required config file (config.local)
Line 223  def parseConfig(fin, keymap, addenda, de
Line 423  def parseConfig(fin, keymap, addenda, de
         # Examine each line, looking for key=value pairs.         # Examine each line, looking for key=value pairs.
         regexpDefs = re.compile(r"^__DEFS__")         regexpDefs = re.compile(r"^__DEFS__")
         regexpMake = re.compile(r"^__MAKE__")         regexpMake = re.compile(r"^__MAKE__")
           regexpProjMkRules = re.compile(r"__PROJ_MK_RULES__")
           regexpProj = re.compile(r"^__PROJ__")
           regexpProjCfg = re.compile(r"^__PROJCFG__")
         regexpComm = re.compile(r"^\s*#")         regexpComm = re.compile(r"^\s*#")
           regexpSp = re.compile(r"^s*$")
         regexpQuote = re.compile(r"^\s*(\w):(.+)")         regexpQuote = re.compile(r"^\s*(\w):(.+)")
         regexp = re.compile(r"^\s*(\S+)\s+(.+)")          regexpCustMkBeg = re.compile(r"^_CUST_")
           regexpCustMkEnd = re.compile(r"^_ENDCUST_")
           regexpDiv = re.compile(r"^__")
           regexp = re.compile(r"^\s*(\S+)\s+(\S.*)")
   
           platDict = {}
           machDict = {}
  
         section = list()          xml = None
  
         # Process the parameters in the configuration file         # Process the parameters in the configuration file
         iscfg = bool(1)  
         if not fin is None:         if not fin is None:
             for line in fin:             for line in fin:
                 ppRet = processParam(iscfg, line, regexpComm, regexpDefs, regexpMake, regexpQuote, regexp, keymap, defs, cDefs, mDefs, perlConstSection, perlInitSection, section)                  matchobj = regexpComm.match(line)
                 if ppRet:                  if not matchobj is None:
                       # Skip comment line
                       continue
   
                   matchobj = regexpSp.match(line)
                   if not matchobj is None:
                       # Skip whitespace line
                       continue
   
                   newSection = determineSection(line, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg)
                   if not newSection is None:
                       section = newSection
   
                   if section == 'make':
   
                       # There are some blocks of lines in the __MAKE__ section that must be copied ver batim to the output make file.
                       # The blocks are defined by _CUST_/_ENDCUST_ tags.
                       matchobj = regexpCustMkBeg.match(line)
   
                       if not matchobj is None:
                           mDefsMake.extend(list('\n'))
                           for line in fin:
                               matchobj = regexpCustMkEnd.match(line)
                               if not matchobj is None:
                     break;                     break;
                               mDefsMake.extend(list(line))
                           newSection = determineSection(line, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg)
                           if not newSection is None:
                               section = newSection
                           continue
                       # Intentional fall through to next if statement
                   if section == 'defs' or section == 'make':
                       iscfg = bool(1)
                       ppRet = processParam(iscfg, line, regexpQuote, regexp, keymap, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, platDict, machDict, section)
  
         # Process addenda - these are parameters that are not configurable and must be set in the  
         # NetDRMS build.  
         iscfg = bool(0)  
         for key in addenda:  
             item = key + ' ' + addenda[key]  
             ppRet = processParam(iscfg, item, regexpComm, regexpDefs, regexpMake, regexpQuote, regexp, keymap, defs, cDefs, mDefs, perlConstSection, perlInitSection, section)  
             if ppRet:             if ppRet:
                           break
                   elif section == 'projcfg':
                       # Copy the line ver batim to the projCfg list (configure)
                       for line in fin:
                           matchobj = regexpDiv.match(line)
                           if not matchobj is None:
                 break;                 break;
                           projCfg.extend(list(line))
                       newSection = determineSection(line, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg)
                       if not newSection is None:
                           section = newSection
                       continue
                   elif section == 'projmkrules':
                       # Copy the line ver batim to the projMkRules list (make_basic.mk)
                       for line in fin:
                           matchobj = regexpDiv.match(line)
                           if not matchobj is None:
                               break;
                           projMkRules.extend(list(line))
                       newSection = determineSection(line, regexpDefs, regexpMake, regexpProjMkRules, regexpProj, regexpProjCfg)
                       if not newSection is None:
                           section = newSection
                       continue
                   elif section == 'proj':
                       # Must parse xml and use the project-specific information to populate the Rules.mk and target.mk files.
                       # Collect all xml lines for now, then process after file-read loop.
                       if xml is None:
                           xml = line
                       else:
                           xml += line
                   else:
                       # Unknown section
                       raise Exception('unknownSection', section)
     except Exception as exc:     except Exception as exc:
         msg, violator = exc.args          msg = exc.args[0]
         if msg == 'badKeyMapKey':         if msg == 'badKeyMapKey':
             # If we are here, then there was a non-empty keymap, and the parameter came from             # If we are here, then there was a non-empty keymap, and the parameter came from
             # the configuration file.             # the configuration file.
               violator = exc.args[1]
             print('Unknown parameter name ' + "'" + violator + "'" + ' in ' + cfgfile + '.', file=sys.stderr)             print('Unknown parameter name ' + "'" + violator + "'" + ' in ' + cfgfile + '.', file=sys.stderr)
             rv = bool(1)             rv = bool(1)
         elif msg == 'badQuoteQual':         elif msg == 'badQuoteQual':
             # The bad quote qualifier came from the configuration file, not the addenda, since             # The bad quote qualifier came from the configuration file, not the addenda, since
             # we will have fixed any bad qualifiers in the addenda (which is populated by code).             # we will have fixed any bad qualifiers in the addenda (which is populated by code).
               violator = exc.args[1]
             print('Unknown quote qualifier ' + "'" + violator + "'" + ' in ' + cfgfile + '.', file=sys.stderr)             print('Unknown quote qualifier ' + "'" + violator + "'" + ' in ' + cfgfile + '.', file=sys.stderr)
             rv = bool(1)             rv = bool(1)
         elif msg == 'missingQuoteQual':         elif msg == 'missingQuoteQual':
               violator = exc.args[1]
             print('Missing quote qualifier for parameter ' + "'" + violator + "'" + ' in ' + cfgfile + '.', file=sys.stderr)             print('Missing quote qualifier for parameter ' + "'" + violator + "'" + ' in ' + cfgfile + '.', file=sys.stderr)
             rv = bool(1)             rv = bool(1)
         elif msg == 'paramNameTooLong':         elif msg == 'paramNameTooLong':
               violator = exc.args[1]
             print('Macro name ' + "'" + violator + "' is too long.", file=sys.stderr)             print('Macro name ' + "'" + violator + "' is too long.", file=sys.stderr)
             rv = bool(1)             rv = bool(1)
           elif msg == 'unknownSection':
               violator = exc.args[1]
               print('Unknown section ' + "'" + violator + "' in configuration file.", file=sys.stderr)
               rv = bool(1)
         else:         else:
             # re-raise the exception             # re-raise the exception
             raise             raise
  
       if not rv:
           if not xml is None:
               # Process xml.
               projRules.extend(list(RULESPREFIX))
               projTarget.extend(list(TARGETPREFIX))
               rv = processXML(xml, projRules, projTarget)
               projRules.extend(RULESSUFFIX)
   
       # Process addenda - these are parameters that are not configurable and must be set in the
       # NetDRMS build.
       if not rv:
           iscfg = bool(0)
           for key in addenda:
               item = key + ' ' + addenda[key]
               ppRet = processParam(iscfg, item, regexpQuote, regexp, keymap, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, platDict, machDict, 'defs')
               if ppRet:
                   break;
   
       # Put information collected in platDict and machDict into mDefs. Must do this here, and not in processParam, since
       # we need to parse all platform-specific make variables before grouping them into platform categories.
       if not rv:
           for plat in platDict:
               mDefsMake.extend(list('\nifeq ($(JSOC_MACHINE), linux_' + plat.lower() + ')'))
               for var in platDict[plat]:
                   mDefsMake.extend(list('\n' + var + ' = ' + platDict[plat][var]))
               mDefsMake.extend(list('\nendif\n'))
   
       if not rv:
           for mach in machDict:
               mDefsMake.extend(list('\nifeq ($(MACHTYPE), ' + mach + ')'))
               for var in machDict[mach]:
                   mDefsMake.extend(list('\n' + var + ' = ' + machDict[mach][var]))
               mDefsMake.extend(list('\nendif\n'))
     return rv     return rv
  
 def getMgrUIDLine(defs, uidParam): def getMgrUIDLine(defs, uidParam):
Line 286  def getMgrUIDLine(defs, uidParam):
Line 593  def getMgrUIDLine(defs, uidParam):
  
     return rv     return rv
  
 def writeFiles(base, cfile, mfile, pfile, cDefs, mDefs, perlConstSection, perlInitSection):  def isVersion(maj, min, majDef, minDef):
       res = 0
   
       if maj > majDef or (maj == majDef and min >= minDef):
           res = 1
   
       return res
   
   def configureComps(defs, mDefs):
     rv = bool(0)     rv = bool(0)
       autoConfig = bool(1)
   
       if 'AUTOSELCOMP' in defs:
           autoConfig = (not defs['AUTOSELCOMP'] == '0')
   
       if autoConfig:
           hasicc = bool(0)
           hasgcc = bool(0)
           hasifort = bool(0)
           hasgfort = bool(0)
   
           # Try icc.
           cmd = 'icc --version 2>&1'
           try:
               ret = check_output(cmd, shell=True)
               ret = ret.decode("utf-8")
           except CalledProcessError:
               print('Command ' + "'" + cmd + "'" + ' ran improperly.')
               rv = bool(1)
   
           if not rv:
               regexp = re.compile(r"\s*\S+\s+\S+\s+(\d+)[.](\d+)", re.DOTALL)
               matchobj = regexp.match(ret)
               if matchobj is None:
                   raise Exception('unexpectedIccRet', ret)
               else:
                   major = matchobj.group(1)
                   minor = matchobj.group(2)
                   if isVersion(int(major), int(minor), ICC_MAJOR, ICC_MINOR):
                       hasicc = bool(1)
   
           # Try gcc.
           if not hasicc:
               cmd = 'gcc -v 2>&1'
               try:
                   ret = check_output(cmd, shell=True)
                   ret = ret.decode("utf-8")
               except CalledProcessError:
                   print('Command ' + "'" + cmd + "'" + ' ran improperly.')
                   rv = bool(1)
   
               if rv == bool(0):
                   regexp = re.compile(r".+gcc\s+version\s+(\d+)\.(\d+)", re.DOTALL)
                   matchobj = regexp.match(ret)
                   if matchobj is None:
                       raise Exception('unexpectedGccRet', ret)
                   else:
                       major = matchobj.group(1)
                       minor = matchobj.group(2)
                       if isVersion(int(major), int(minor), GCC_MAJOR, GCC_MINOR):
                           hasgcc = bool(1)
   
           # Try ifort.
           cmd = 'ifort --version 2>&1'
           try:
               ret = check_output(cmd, shell=True)
               ret = ret.decode("utf-8")
           except CalledProcessError:
               print('Command ' + "'" + cmd + "'" + ' ran improperly.')
               rv = bool(1)
   
           if not rv:
               regexp = re.compile(r"\s*\S+\s+\S+\s+(\d+)\.(\d+)", re.DOTALL)
               matchobj = regexp.match(ret)
               if matchobj is None:
                   raise Exception('unexpectedIfortRet', ret)
               else:
                   major = matchobj.group(1)
                   minor = matchobj.group(2)
                   if isVersion(int(major), int(minor), IFORT_MAJOR, IFORT_MINOR):
                       hasifort = bool(1)
   
           # Try gfortran
           if not hasifort:
               cmd = 'gfortran -v 2>&1'
               try:
                   ret = check_output(cmd, shell=True)
                   ret = ret.decode("utf-8")
               except CalledProcessError:
                   print('Command ' + "'" + cmd + "'" + ' ran improperly.')
                   rv = bool(1)
   
               if rv == bool(0):
                   regexp = re.compile(r".+gcc\s+version\s+(\d+)\.(\d+)", re.DOTALL)
                   matchobj = regexp.match(ret)
                   if matchobj is None:
                       raise Exception('unexpectedGfortranRet', ret)
                   else:
                       major = matchobj.group(1)
                       minor = matchobj.group(2)
                       if isVersion(int(major), int(minor), GFORT_MAJOR, GFORT_MINOR):
                           hasgfort = bool(1)
   
           # Append the compiler make variables to the make file
           if not hasicc and not hasgcc:
               print('Fatal error: Acceptable C compiler not found! You will be unable to build the DRMS library.', file=sys.stderr)
               rv = bool(1)
           elif hasicc:
               mDefs.extend(list('\nCOMPILER = icc'))
           else:
               mDefs.extend(list('\nCOMPILER = gcc'))
   
           if not hasifort and not hasgfort:
               print('Warning: Acceptable Fortran compiler not found! Fortran interface will not be built, and you will be unable to build Fortran modules.', file=sys.stderr)
           elif hasifort:
               mDefs.extend(list('\nFCOMPILER = ifort'))
           else:
               mDefs.extend(list('\nFCOMPILER = gfortran'))
   
           # Environment overrides. These get written, regardless of the disposition of auto-configuration.
           mDefs.extend(list('\nifneq ($(JSOC_COMPILER),)\n  COMPILER = $(JSOC_COMPILER)\nendif'))
           mDefs.extend(list('\nifneq ($(JSOC_FCOMPILER),)\n  FCOMPILER = $(JSOC_FCOMPILER)\nendif'))
   
       return rv
   
   def writeParamsFiles(base, cfile, mfile, pfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection):
       rv = bool(0)
   
       # Merge mDefsGen, mDefsMake, and mDefsComps into a single string with compiler configuration first, general parameters next, then
       # make-specific make variables (e.g., third-party library information) last.
       mDefs = '\n# Compiler Selection\n' + ''.join(mDefsComps) + '\n\n# General Parameters\n' + ''.join(mDefsGen) + '\n\n# Parameters to Configure make\n' + ''.join(mDefsMake)
  
     try:     try:
         with open(cfile, 'w') as cout, open(mfile, 'w') as mout, open(pfile, 'w') as pout:         with open(cfile, 'w') as cout, open(mfile, 'w') as mout, open(pfile, 'w') as pout:
Line 302  def writeFiles(base, cfile, mfile, pfile
Line 738  def writeFiles(base, cfile, mfile, pfile
  
             # Make file of make variables             # Make file of make variables
             print(PREFIX, file=mout)             print(PREFIX, file=mout)
             print('# This file contains a set of make-variable values - one for each configuration parameter.', file=mout)              print('# This file contains a set of make-variable values. The first section contains compiler-selection variables, the second contains general configuration variables, and the third section contains variables that configure how make is run.', file=mout)
             print(''.join(mDefs), file=mout)              print(mDefs, file=mout)
  
             # Perl module             # Perl module
             print(PERL_BINPATH, file=pout)             print(PERL_BINPATH, file=pout)
Line 320  def writeFiles(base, cfile, mfile, pfile
Line 756  def writeFiles(base, cfile, mfile, pfile
             print('}\n', file=pout)             print('}\n', file=pout)
             print(PERL_FXNS_B, file=pout)             print(PERL_FXNS_B, file=pout)
     except IOError as exc:     except IOError as exc:
         sys.stderr.write(exc.strerror)          type, value, traceback = sys.exc_info()
         sys.stderr.write('Unable to open a parameter vile.')          print(exc.strerror, file=sys.stderr)
           print('Unable to open ' + "'" + value.filename + "'.", file=sys.stderr)
           rv = bool(1)
   
       return rv
   
   def writeProjFiles(pCfile, pMfile, pRfile, pTfile, projCfg, projMkRules, projRules, projTarget):
       rv = bool(0)
   
       try:
           if projCfg:
               with open(pCfile, 'w') as cout:
                   # configure
                   print(''.join(projCfg), file=cout)
   
           if projMkRules:
               with open(pMfile, 'w') as mout:
                   # make_basic.mk
                   print(PREFIX, file=mout)
                   print(''.join(projMkRules), file=mout)
   
           if projRules:
               with open(pRfile, 'w') as rout:
                   # Rules.mk
                   print(PREFIX, file=rout)
                   print(''.join(projRules), file=rout)
   
           if projTarget:
               with open(pTfile, 'w') as tout:
                   # target.mk
                   print(PREFIX, file=tout)
                   print(''.join(projTarget), file=tout)
       except IOError as exc:
           type, value, traceback = sys.exc_info()
           print(exc.strerror, file=sys.stderr)
           print('Unable to open ' + "'" + value.filename + "'.", file=sys.stderr)
           rv = bool(1)
   
       if not rv:
           if os.path.exists(pCfile):
               try:
                   os.chmod(pCfile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IROTH)
               except OSError as exc:
                   type, value, traceback = sys.exc_info()
                   print('Unable to chmod file ' + "'" + value.filename + "'.", file=sys.stderr)
                   print(exc.strerror, file=sys.stderr)
                   rv = bool(1)
   
       return rv
   
   def generateSumRmCfg(defs):
       rv = bool(0)
       # ACK! Remember that Rick renamed these parameters. The ones in config.local are the aliases - do not use those.
       # Use the ones that those map to (defined in config.local.map).
       cFileTmp = defs['SUMLOG_BASEDIR'] + '/' + '.sum_rm.cfg.tmp'
       cFile = defs['SUMLOG_BASEDIR'] + '/' + 'sum_rm.cfg'
   
       # Write a temporary file sum_rm configuration file.
       try:
           with open(cFileTmp, 'w') as fout:
               # Print comment at the top of the configuration file.
               print(SUMRM_COMMENT, file=fout)
               print(SUMRM_DOC, file=fout)
               print(SUMRM_PARTN_PERCENT_FREE, file=fout)
               if 'SUMRM_PART_PERCENT_FREE' in defs:
                   print('PART_PERCENT_FREE=' + defs['SUMRM_PART_PERCENT_FREE'], file=fout)
               else:
                   print('PART_PERCENT_FREE=3', file=fout)
   
               print(SUMRM_SLEEP, file=fout)
               if 'SUMRM_SLEEP' in defs:
                   print('SLEEP=' + defs['SUMRM_SLEEP'], file=fout)
               else:
                   print('SLEEP=300', file=fout)
   
               print(SUMRM_LOG, file=fout)
               if 'SUMRM_LOG' in defs:
                   print('LOG=' + defs['SUMRM_LOG'], file=fout)
               else:
                   print('LOG=/tmp/sum_rm.log', file=fout)
   
               print(SUMRM_MAIL, file=fout)
               # No default for mail - don't send nothing to nobody unless the operator has asked for notifications.
               if 'SUMRM_MAIL' in defs:
                   print('MAIL=' + defs['SUMRM_MAIL'], file=fout)
               else:
                   print('# MAIL=president@whitehouse.gov', file=fout)
   
               print(SUMRM_NOOP, file=fout)
               if 'SUMRM_NOOP' in defs:
                   print('NOOP=' + defs['SUMRM_NOOP'], file=fout)
               else:
                   print('NOOP=0', file=fout)
   
               print(SUMRM_USER, file=fout)
               if 'SUMRM_USER' in defs:
                   print('USER=' + defs['SUMRM_USER'], file=fout)
               else:
                   print('USER=production', file=fout)
   
               print(SUMRM_NORUN, file=fout)
               # Default norun window is to have no such window. This can be accomplished by simply not providing either argument.
               if 'SUMRM_NORUN_START' in defs or 'SUMRM_NORUN_STOP' in defs:
                   if 'SUMRM_NORUN_START' in defs:
                       print('NORUN_START=' + defs['SUMRM_NORUN_START'], file=fout)
                   else:
                       print('NORUN_START=0', file=fout)
                   if 'SUMRM_NORUN_STOP' in defs:
                       print('NORUN_STOP=' + defs['SUMRM_NORUN_STOP'], file=fout)
                   else:
                       print('NORUN_STOP=0', file=fout)
               else:
                   print('# NORUN_START=0', file=fout)
                   print('# NORUN_STOP=0', file=fout)
   
       except OSError:
           print('Unable to open sum_rm temporary configuration file ' + cFileTmp + 'for writing.', file=sys.stderr)
           rv = bool(1)
   
       # If the content of the temporary file differs from the content of the existing configuration file, then overwrite
       # the original file. Otherwise, delete the temporary file
       if not rv:
           try:
               if filecmp.cmp(cFile, cFileTmp):
                   # Files identical - delete temporary file
                   try:
                       os.remove(cFileTmp)
   
                   except OSError as exc:
                       print('Unable to remove temporary file ' + exc.filename + '.', file=sys.stderr)
                       print(exc.strerr, file=sys.stderr)
               else:
                   # Replace original with temporary file
                   try:
                       os.rename(cFileTmp, cFile)
   
                   except OSError as exc:
                       print('Unable to update sum_rm configuration file ' + cFile + '.', file=sys.stderr)
                       print(exc.strerr, file=sys.stderr)
                       rv = bool(1)
           except OSError as exc:
               # One of the files doesn't exist.
               if exc.filename == cFile:
                   # We are ok - there might be no configuration file yet.
                   # Replace original with temporary file
                   try:
                       os.rename(cFileTmp, cFile)
   
                   except OSError as exc:
                       print('Unable to update sum_rm configuration file ' + cFile + '.', file=sys.stderr)
                       print(exc.strerr, file=sys.stderr)
                       rv = bool(1)
               else:
                   # There is a problem with the temp file - bail.
                   print('Unable to update sum_rm configuration file ' + cFile + '.', file=sys.stderr)
                   print(exc.strerr, file=sys.stderr)
         rv = bool(1)         rv = bool(1)
  
     return rv     return rv
  
 def configureNet(cfgfile, cfile, mfile, pfile, base, keymap):  def configureNet(cfgfile, cfile, mfile, pfile, pCfile, pMfile, pRfile, pTfile, base, keymap, createSumRmCfg):
     rv = bool(0)     rv = bool(0)
  
     defs = {}     defs = {}
     cDefs = list()     cDefs = list()
     mDefs = list()      mDefsGen = list()
       mDefsMake = list()
       mDefsComps = list()
       projCfg = list()
       projMkRules = list()
       projRules = list()
       projTarget = list()
     perlConstSection = list()     perlConstSection = list()
     perlInitSection = list()     perlInitSection = list()
     addenda = {}     addenda = {}
  
     # There are three parameters that are not configurable and must be set.      # There are three parameters that were not included in the original config.local parameter set, for some reason.
       # Due to this omission, then are not configurable, and must be set in the script.
     addenda['a:USER'] = 'NULL'     addenda['a:USER'] = 'NULL'
     addenda['a:PASSWD'] = 'NULL'     addenda['a:PASSWD'] = 'NULL'
     addenda['p:DSDS_SUPPORT'] = '0'     addenda['p:DSDS_SUPPORT'] = '0'
   
       # This parameter is not configurable. BUILD_TYPE is used to distinguish between a NetDRMS and an JSOC-SDP build.
     addenda['a:BUILD_TYPE'] = 'NETDRMS' # Means a non-Stanford build. This will set two additional macros used by make:     addenda['a:BUILD_TYPE'] = 'NETDRMS' # Means a non-Stanford build. This will set two additional macros used by make:
                                         #   __LOCALIZED_DEFS__ and NETDRMS_BUILD. The former is to support legacy code                                         #   __LOCALIZED_DEFS__ and NETDRMS_BUILD. The former is to support legacy code
                                         #   which incorrectly used this macro, and the latter is for future use.                                         #   which incorrectly used this macro, and the latter is for future use.
Line 347  def configureNet(cfgfile, cfile, mfile,
Line 947  def configureNet(cfgfile, cfile, mfile,
  
     try:     try:
         with open(cfgfile, 'r') as fin:         with open(cfgfile, 'r') as fin:
             rv = parseConfig(fin, keymap, addenda, defs, cDefs, mDefs, perlConstSection, perlInitSection)              # Process configuration parameters
             if rv == bool(0):              rv = parseConfig(fin, keymap, addenda, defs, cDefs, mDefsGen, mDefsMake, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection)
                 # Must add a parameter for the SUMS_MANAGER UID (for some reason)              if not rv:
                   # Must add a parameter for the SUMS_MANAGER UID (for some reason). This must be done after the
                   # config file is processed since an input to getMgrUIDLine() is one of the config file's
                   # parameter values.
                 uidParam = {}                 uidParam = {}
                 rv = getMgrUIDLine(defs, uidParam)                 rv = getMgrUIDLine(defs, uidParam)
                 if rv == bool(0):                 if rv == bool(0):
                     rv = parseConfig(None, keymap, uidParam, defs, cDefs, mDefs, perlConstSection, perlInitSection)                      rv = parseConfig(None, keymap, uidParam, defs, cDefs, mDefsGen, None, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection)
               # Configure the compiler-selection make variables.
                 if rv == bool(0):              if not rv:
                     rv = writeFiles(base, cfile, mfile, pfile, cDefs, mDefs, perlConstSection, perlInitSection)                  rv = configureComps(defs, mDefsComps)
   
               # Write out the parameter files.
               if not rv:
                   rv = writeParamsFiles(base, cfile, mfile, pfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection)
   
               # Write out the project-specific make files (make_basic.mk, Rules.mk, and target.mk).
               if not rv:
                   rv = writeProjFiles(pCfile, pMfile, pRfile, pTfile, projCfg, projMkRules, projRules, projTarget)
   
               # Write out the sum_rm.cfg file.
               if not rv and createSumRmCfg:
                   rv = generateSumRmCfg(defs)
     except IOError as exc:     except IOError as exc:
         sys.stderr.write(exc.strerror)          print(exc.strerror, file=sys.stderr)
         sys.stderr.write('Unable to read configuration file ' + cfgfile + '.')          print('Unable to read configuration file ' + cfgfile + '.', file=sys.stderr)
       except Exception as exc:
           if len(exc.args >= 2):
               type, msg = exc.args
           else:
               # re-raise the exception
               raise
   
           if type == 'unexpectedIccRet':
               print('icc -V returned this unexpected message:\n' + msg, file=sys.stderr)
               rv = bool(1)
           elif type == 'unexpectedGccRet':
               print('gcc -v returned this unexpected message:\n' + msg, file=sys.stderr)
               rv = bool(1)
           elif type == 'unexpectedIfortRet':
               print('ifort -V returned this unexpected message:\n' + msg, file=sys.stderr)
               rv = bool(1)
           elif type == 'unexpectedGfortranRet':
               print('gfortran -v returned this unexpected message:\n' + msg, file=sys.stderr)
               rv = bool(1)
           else:
               # re-raise the exception
               raise
  
     return rv     return rv
  
 def configureSdp(cfgfile, cfile, mfile, pfile, base, keymap):  def configureSdp(cfgfile, cfile, mfile, pfile, pCfile, pMfile, pRfile, pTfile, base):
     rv = bool(0)     rv = bool(0)
  
     defs = {}     defs = {}
     cDefs = list()     cDefs = list()
     mDefs = list()      mDefsGen = list()
       mDefsMake = list()
       projCfg = list()
       projMkRules = list()
       projRules = list()
       projTarget = list()
       mDefsComps = list()
     perlConstSection = list()     perlConstSection = list()
     perlInitSection = list()     perlInitSection = list()
     addenda = {}     addenda = {}
  
     addenda['a:BUILD_TYPE'] = 'JSOC_SDP'      # There are three parameters that were not included in the original config.local parameter set, for some reason.
       # Due to this omission, then are not configurable, and must be set in the script.
       addenda['a:USER'] = 'NULL'
       addenda['a:PASSWD'] = 'NULL'
       addenda['p:DSDS_SUPPORT'] = '1'
   
       # This parameter is not configurable. BUILD_TYPE is used to distinguish between a NetDRMS and an JSOC-SDP build.
       addenda['a:BUILD_TYPE'] = 'JSOC_SDP' # Means a Stanford build. This will set one additional macro used by make: JSOC_SDP_BUILD.
  
     try:     try:
         with open(cfgfile, 'r') as fin:         with open(cfgfile, 'r') as fin:
             rv = parseConfig(cfgfile, keymap, addenda, defs, cDefs, mDefs, perlConstSection, perlInitSection)              rv = parseConfig(fin, None, addenda, defs, cDefs, mDefsGen, mDefsMake, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection)
             if rv == bool(0):  
               if not rv:
                 # Must add a parameter for the SUMS_MANAGER UID (for some reason)                 # Must add a parameter for the SUMS_MANAGER UID (for some reason)
                 uidParam = {}                 uidParam = {}
                 rv = getMgrUIDLine(defs, uidParam)                 rv = getMgrUIDLine(defs, uidParam)
                 if rv == bool(0):                  if not rv:
                     rv = parseConfig(None, keymap, uidParam, defs, cDefs, mDefs, perlConstSection, perlInitSection)                      rv = parseConfig(None, None, uidParam, defs, cDefs, mDefsGen, None, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection)
   
                   # Configure the compiler-selection make variables.
                   if not rv:
                       rv = configureComps(defs, mDefsComps)
   
                   # Write out the parameter files.
                   if not rv:
                       rv = writeParamsFiles(base, cfile, mfile, pfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection)
   
                   # Write out the project-specific make files (make_basic.mk, Rules.mk, and target.mk).
                   if not rv:
                       rv = writeProjFiles(pCfile, pMfile, pRfile, pTfile, projCfg, projMkRules, projRules, projTarget)
  
                 rv = writeFiles(base, cfile, mfile, pfile, cDefs, mDefs, perlConstSection, perlInitSection)                  # At Stanford, skip the creation of the sum_rm configuration file. config.local will still
                   # have the SUMRM parameters, but they will not be used.
     except IOError as exc:     except IOError as exc:
         sys.stderr.write(exc.strerror)          print(exc.strerror, file=sys.stderr)
         sys.stderr.write('Unable to read configuration file ' + cfgfile + '.')          print('Unable to read configuration file ' + cfgfile + '.', file=sys.stderr)
       except Exception as exc:
           type = exc.args[0]
           if type == 'unexpectedIccRet':
               msg = exc.args[1]
               print('icc -V returned this unexpected message:\n' + msg, file=sys.stderr)
               rv = bool(1)
           elif type == 'unexpectedGccRet':
               msg = exc.args[1]
               print('gcc -v returned this unexpected message:\n' + msg, file=sys.stderr)
               rv = bool(1)
           elif type == 'unexpectedIfortRet':
               msg = exc.args[1]
               print('ifort -V returned this unexpected message:\n' + msg, file=sys.stderr)
               rv = bool(1)
           elif type == 'unexpectedGfortranRet':
               msg = exc.args[1]
               print('gfortran -v returned this unexpected message:\n' + msg, file=sys.stderr)
               rv = bool(1)
           else:
               # re-raise the exception
               raise
  
     return rv     return rv
  
Line 418  if rv == RET_SUCCESS:
Line 1103  if rv == RET_SUCCESS:
     cfile = optD['dir'] + '/' + optD['base'] + '.h'     cfile = optD['dir'] + '/' + optD['base'] + '.h'
     mfile = optD['dir'] + '/' + optD['base'] + '.mk'     mfile = optD['dir'] + '/' + optD['base'] + '.mk'
     pfile = optD['dir'] + '/' + optD['base'] + '.pm'     pfile = optD['dir'] + '/' + optD['base'] + '.pm'
       pCfile = optD['dir'] + '/configure'
       pMfile = optD['dir'] + '/make_basic.mk'
       pRfile = optD['dir'] + '/Rules.mk'
       pTfile = optD['dir'] + '/target.mk'
  
     if net:     if net:
         try:         try:
Line 442  if rv == RET_SUCCESS:
Line 1131  if rv == RET_SUCCESS:
             sys.stderr.write('Unable to read configuration map-file ' + NET_CFGMAP + '.')             sys.stderr.write('Unable to read configuration map-file ' + NET_CFGMAP + '.')
             rv = bool(1)             rv = bool(1)
  
   
   
         # We also need to set the UID of the SUMS manager. We have the name of the         # We also need to set the UID of the SUMS manager. We have the name of the
         # SUMS manager (it is in the configuration file)         # SUMS manager (it is in the configuration file)
         configureNet(NET_CFG, cfile, mfile, pfile, optD['base'], keymap)          configureNet(NET_CFG, cfile, mfile, pfile, pCfile, pMfile, pRfile, pTfile, optD['base'], keymap, 'server' in optD)
     else:     else:
         configureSdp(SDP_CFG, cfile, mfile, pfile, optD['base'], {})          # A Stanford user can override the parameters in configsdp.txt by copying that file to config.local,
           # and then editing config.local. So, if config.local exists, use that.
           if os.path.isfile(cdir + '/' + NET_CFG):
               configureSdp(NET_CFG, cfile, mfile, pfile, pCfile, pMfile, pRfile, pTfile, optD['base'])
           else:
               configureSdp(SDP_CFG, cfile, mfile, pfile, pCfile, pMfile, pRfile, pTfile, optD['base'])


Legend:
Removed from v.1.2  
changed lines
  Added in v.1.14

Karen Tian
Powered by
ViewCVS 0.9.4