(file) Return to localize.py CVS log (file) (dir) Up to [Development] / JSOC

Diff for /JSOC/localize.py between version 1.9 and 1.17

version 1.9, 2013/11/15 17:05:49 version 1.17, 2014/04/18 19:14:54
Line 1 
Line 1 
 #!/home/jsoc/bin/linux_x86_64/activepython  #!/usr/bin/python
  
   # When run with the -s flag, localize.py configures the SUMS-server component of NetDRMS.
   from __future__ import print_function
 import sys import sys
 import getopt import getopt
 import re import re
 import os import os
 import stat import stat
   import filecmp
 import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
 from subprocess import check_output, CalledProcessError from subprocess import check_output, CalledProcessError
  
Line 71  PERL_FXNS_B = """sub get
Line 74  PERL_FXNS_B = """sub get
 } }
 1;""" 1;"""
  
   SUMRM_COMMENT = """# This is the configuration file for the sum_rm program. It was auto-generated by the DRMS master configure script.
   # It controls the behavior of the sum_rm program, and is loaded each time sum_rm runs. To change the
   # parameter values in this configuration file, modify config.local, then re-run configure. This configuration
   # file will be updated only if parameters affecting it are modified. If such changes are made to config.local,
   # please make sure that the sum_rm service is disabled while configure in running.
   """
   
   SUMRM_DOC = """# sum_rm removes end-of-life SUMS data files to prevent disk-partitions from becomming 100% full.
   # sum_svc, the main SUMS service, starts this daemon when it is launched. Within an infinite loop, sum_rm sleeps
   # for SLEEP number of seconds (defined below). When it awakes, it loads the sum_rm configuration file. For each SU,
   # it then examines the 'effective_date' column within the sum_partn_alloc table of the SUMS database. It does so by
   # sorting all SUs by effective date (this date is actually an expiration date - the SU is good until the end of that day),
   # and then, starting with the SU with the oldest effective date, it deletes the SUs. It continues deleting SUs until one
   # of three conditions is met:
   #
   #   (a) The disk has at least PART_PERCENT_FREE percent free space.
   #   (b) All SUs have effective_date values in the future.
   #   (c) At least 600 SUs have been deleted (this is defined in the LIMIT statement in the file SUMLIB_RmDoX.pgc).
   #
   # After sum_rm stops deleteing SUs, it then sleeps for SLEEP seconds, completing the first iteration of the
   # infinite loop.
   """
   
   SUMRM_PARTN_PERCENT_FREE = """
   # This is the percentage at which all disk partitions are to be kept free.
   # If not specified, this defaults to 3. For example, setting PART_PERCENT_FREE = 5 will allow all partitions to
   # fill to 95% full. Dividing the number of unused blocks by the total number of blocks, and rounding up,
   # will result in the number specified by PART_PERCENT_FREE.
   #
   # NOTE : This behavior was previously controlled by the MAX_FREE_{n} family of parameters. {n} referred to the
   # disk-partition number and the value of parameter MAX_FREE_{n} was the MINIMUM number of free MB in the
   # partition [No clue why the word MAX was used]. As of NetDRMS 7.0, MAX_FREE_{n} has no effect."""
   
   SUMRM_SLEEP = """
   # The value is the number of seconds to sleep between iterations of the main loop in sum_rm."""
   
   SUMRM_LOG = """
   # The value is the log file (opened only at sum_rm startup; the sum_rm pid is appended to this file name)."""
   
   SUMRM_MAIL = """
   # The value is the email address of the recipient to be notified in case of a problem."""
   
   SUMRM_NOOP = """
   # If the value is set to anything other than 0, then sum_rm is rendered inactive. Otherwise, sum_rm is active."""
   
   SUMRM_USER = """
   # The value designates the linux user who is allowed to run sum_rm."""
   
   SUMRM_NORUN = """
   # This pair of paramters defines a window of time, during which sum_rm will become torpid - in this
   # window, sum_rm will not scan for SUs to delete, not will it delete any SUs. Each value is an integer, 0-23, that
   # represents an hour of the day. For example, if NORUN_START=8 and NORUN_STOP=10, then between 8am and 10am
   # local time, sum_rm will be dormant. To disable this behavior, set both parameters to the same value."""
  
 RULESPREFIX = """# Standard things RULESPREFIX = """# Standard things
 sp              := $(sp).x sp              := $(sp).x
Line 105  GFORT_MINOR = 2
Line 161  GFORT_MINOR = 2
 # Read arguments # Read arguments
 #  d - localization directory #  d - localization directory
 #  b - base name of all parameter files (e.g., -b drmsparams --> drmsparams.h, drmsparams.mk, drmsparams.pm, etc.) #  b - base name of all parameter files (e.g., -b drmsparams --> drmsparams.h, drmsparams.mk, drmsparams.pm, etc.)
 #  m - make file  #  s - configure a NetDRMS server (i.e., SUMS server). Otherwise, do client configuration only. A server
   #      configuration will modify something that only the production user has access to. An example is the sum_rm.cfg
   #      file. To modify that file, the user running configure must be running configure on the SUMS-server host, and
   #      must have write permission on sum_rm.cfg.
 def GetArgs(args): def GetArgs(args):
     rv = bool(0)     rv = bool(0)
     optD = {}     optD = {}
  
     try:     try:
         opts, remainder = getopt.getopt(args, "hd:b:",["dir=", "base="])          opts, remainder = getopt.getopt(args, "hd:b:s",["dir=", "base="])
     except getopt.GetoptError:     except getopt.GetoptError:
         print('Usage:')         print('Usage:')
         print('localize.py [-h] -d <localization directory> -b <parameter file base>')         print('localize.py [-h] -d <localization directory> -b <parameter file base>')
Line 130  def GetArgs(args):
Line 189  def GetArgs(args):
                     optD['dir'] = matchobj.group(1)                     optD['dir'] = matchobj.group(1)
             elif opt in ("-b", "--base"):             elif opt in ("-b", "--base"):
                 optD['base'] = arg                 optD['base'] = arg
               elif opt in ("-s"):
                   optD['server'] = ""
             else:             else:
                 optD[opt] = arg                 optD[opt] = arg
  
Line 371  def parseConfig(fin, keymap, addenda, de
Line 432  def parseConfig(fin, keymap, addenda, de
         regexpCustMkBeg = re.compile(r"^_CUST_")         regexpCustMkBeg = re.compile(r"^_CUST_")
         regexpCustMkEnd = re.compile(r"^_ENDCUST_")         regexpCustMkEnd = re.compile(r"^_ENDCUST_")
         regexpDiv = re.compile(r"^__")         regexpDiv = re.compile(r"^__")
         regexp = re.compile(r"^\s*(\S+)\s+(\S+)")          regexp = re.compile(r"^\s*(\S+)\s+(\S.*)")
  
         platDict = {}         platDict = {}
         machDict = {}         machDict = {}
Line 396  def parseConfig(fin, keymap, addenda, de
Line 457  def parseConfig(fin, keymap, addenda, de
                     section = newSection                     section = newSection
  
                 if section == 'make':                 if section == 'make':
   
                     # There are some blocks of lines in the __MAKE__ section that must be copied ver batim to the output make file.                     # There are some blocks of lines in the __MAKE__ section that must be copied ver batim to the output make file.
                     # The blocks are defined by _CUST_/_ENDCUST_ tags.                     # The blocks are defined by _CUST_/_ENDCUST_ tags.
                     matchobj = regexpCustMkBeg.match(line)                     matchobj = regexpCustMkBeg.match(line)
Line 417  def parseConfig(fin, keymap, addenda, de
Line 479  def parseConfig(fin, keymap, addenda, de
                     ppRet = processParam(iscfg, line, regexpQuote, regexp, keymap, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, platDict, machDict, section)                     ppRet = processParam(iscfg, line, regexpQuote, regexp, keymap, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, platDict, machDict, section)
  
                     if ppRet:                     if ppRet:
                         break;                          break
                 elif section == 'projcfg':                 elif section == 'projcfg':
                     # Copy the line ver batim to the projCfg list (configure)                     # Copy the line ver batim to the projCfg list (configure)
                     for line in fin:                     for line in fin:
Line 451  def parseConfig(fin, keymap, addenda, de
Line 513  def parseConfig(fin, keymap, addenda, de
                     # Unknown section                     # Unknown section
                     raise Exception('unknownSection', section)                     raise Exception('unknownSection', section)
     except Exception as exc:     except Exception as exc:
           if len(exc.args) >= 2:
         msg = exc.args[0]         msg = exc.args[0]
           else:
               # re-raise the exception
               raise
         if msg == 'badKeyMapKey':         if msg == 'badKeyMapKey':
             # If we are here, then there was a non-empty keymap, and the parameter came from             # If we are here, then there was a non-empty keymap, and the parameter came from
             # the configuration file.             # the configuration file.
             violator = exc.args[1]             violator = exc.args[1]
             print('Unknown parameter name ' + "'" + violator + "'" + ' in ' + cfgfile + '.', file=sys.stderr)              print('Unknown parameter name ' + "'" + violator + "'" + ' in ' + fin.name + '.', file=sys.stderr)
             rv = bool(1)             rv = bool(1)
         elif msg == 'badQuoteQual':         elif msg == 'badQuoteQual':
             # The bad quote qualifier came from the configuration file, not the addenda, since             # The bad quote qualifier came from the configuration file, not the addenda, since
             # we will have fixed any bad qualifiers in the addenda (which is populated by code).             # we will have fixed any bad qualifiers in the addenda (which is populated by code).
             violator = exc.args[1]             violator = exc.args[1]
             print('Unknown quote qualifier ' + "'" + violator + "'" + ' in ' + cfgfile + '.', file=sys.stderr)              print('Unknown quote qualifier ' + "'" + violator + "'" + ' in ' + fin.name + '.', file=sys.stderr)
             rv = bool(1)             rv = bool(1)
         elif msg == 'missingQuoteQual':         elif msg == 'missingQuoteQual':
             violator = exc.args[1]             violator = exc.args[1]
             print('Missing quote qualifier for parameter ' + "'" + violator + "'" + ' in ' + cfgfile + '.', file=sys.stderr)              print('Missing quote qualifier for parameter ' + "'" + violator + "'" + ' in ' + fin.name + '.', file=sys.stderr)
             rv = bool(1)             rv = bool(1)
         elif msg == 'paramNameTooLong':         elif msg == 'paramNameTooLong':
             violator = exc.args[1]             violator = exc.args[1]
Line 513  def parseConfig(fin, keymap, addenda, de
Line 579  def parseConfig(fin, keymap, addenda, de
             for var in machDict[mach]:             for var in machDict[mach]:
                 mDefsMake.extend(list('\n' + var + ' = ' + machDict[mach][var]))                 mDefsMake.extend(list('\n' + var + ' = ' + machDict[mach][var]))
             mDefsMake.extend(list('\nendif\n'))             mDefsMake.extend(list('\nendif\n'))
   
     return rv     return rv
  
 def getMgrUIDLine(defs, uidParam): def getMgrUIDLine(defs, uidParam):
Line 554  def configureComps(defs, mDefs):
Line 619  def configureComps(defs, mDefs):
         hasgfort = bool(0)         hasgfort = bool(0)
  
         # Try icc.         # Try icc.
         cmd = 'icc -V 2>&1'          cmd = 'icc --version 2>&1'
         try:         try:
             ret = check_output(cmd, shell=True)             ret = check_output(cmd, shell=True)
             ret = ret.decode("utf-8")             ret = ret.decode("utf-8")
Line 563  def configureComps(defs, mDefs):
Line 628  def configureComps(defs, mDefs):
             rv = bool(1)             rv = bool(1)
  
         if not rv:         if not rv:
             regexp = re.compile(r".+Version\s+(\d+)[.](\d+)", re.DOTALL)              regexp = re.compile(r"\s*\S+\s+\S+\s+(\d+)[.](\d+)", re.DOTALL)
             matchobj = regexp.match(ret)             matchobj = regexp.match(ret)
             if matchobj is None:             if matchobj is None:
                 raise Exception('unexpectedIccRet', ret)                 raise Exception('unexpectedIccRet', ret)
Line 595  def configureComps(defs, mDefs):
Line 660  def configureComps(defs, mDefs):
                         hasgcc = bool(1)                         hasgcc = bool(1)
  
         # Try ifort.         # Try ifort.
         cmd = 'ifort -V 2>&1'          cmd = 'ifort --version 2>&1'
         try:         try:
             ret = check_output(cmd, shell=True)             ret = check_output(cmd, shell=True)
             ret = ret.decode("utf-8")             ret = ret.decode("utf-8")
Line 604  def configureComps(defs, mDefs):
Line 669  def configureComps(defs, mDefs):
             rv = bool(1)             rv = bool(1)
  
         if not rv:         if not rv:
             regexp = re.compile(r".+Version\s+(\d+)\.(\d+)", re.DOTALL)              regexp = re.compile(r"\s*\S+\s+\S+\s+(\d+)\.(\d+)", re.DOTALL)
             matchobj = regexp.match(ret)             matchobj = regexp.match(ret)
             if matchobj is None:             if matchobj is None:
                 raise Exception('unexpectedIfortRet', ret)                 raise Exception('unexpectedIfortRet', ret)
Line 735  def writeProjFiles(pCfile, pMfile, pRfil
Line 800  def writeProjFiles(pCfile, pMfile, pRfil
         rv = bool(1)         rv = bool(1)
  
     if not rv:     if not rv:
           if os.path.exists(pCfile):
         try:         try:
             os.chmod(pCfile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IROTH)             os.chmod(pCfile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IROTH)
         except OSError as exc:         except OSError as exc:
                   type, value, traceback = sys.exc_info()
                   print('Unable to chmod file ' + "'" + value.filename + "'.", file=sys.stderr)
             print(exc.strerror, file=sys.stderr)             print(exc.strerror, file=sys.stderr)
             rv = bool(1)             rv = bool(1)
  
     return rv     return rv
  
 def configureNet(cfgfile, cfile, mfile, pfile, pCfile, pMfile, pRfile, pTfile, base, keymap):  def generateSumRmCfg(defs):
       rv = bool(0)
       # ACK! Remember that Rick renamed these parameters. The ones in config.local are the aliases - do not use those.
       # Use the ones that those map to (defined in config.local.map).
       cFileTmp = defs['SUMLOG_BASEDIR'] + '/' + '.sum_rm.cfg.tmp'
       cFile = defs['SUMLOG_BASEDIR'] + '/' + 'sum_rm.cfg'
   
       # Write a temporary file sum_rm configuration file.
       try:
           with open(cFileTmp, 'w') as fout:
               # Print comment at the top of the configuration file.
               print(SUMRM_COMMENT, file=fout)
               print(SUMRM_DOC, file=fout)
               print(SUMRM_PARTN_PERCENT_FREE, file=fout)
               if 'SUMRM_PART_PERCENT_FREE' in defs:
                   print('PART_PERCENT_FREE=' + defs['SUMRM_PART_PERCENT_FREE'], file=fout)
               else:
                   print('PART_PERCENT_FREE=3', file=fout)
   
               print(SUMRM_SLEEP, file=fout)
               if 'SUMRM_SLEEP' in defs:
                   print('SLEEP=' + defs['SUMRM_SLEEP'], file=fout)
               else:
                   print('SLEEP=300', file=fout)
   
               print(SUMRM_LOG, file=fout)
               if 'SUMRM_LOG' in defs:
                   print('LOG=' + defs['SUMRM_LOG'], file=fout)
               else:
                   print('LOG=/tmp/sum_rm.log', file=fout)
   
               print(SUMRM_MAIL, file=fout)
               # No default for mail - don't send nothing to nobody unless the operator has asked for notifications.
               if 'SUMRM_MAIL' in defs:
                   print('MAIL=' + defs['SUMRM_MAIL'], file=fout)
               else:
                   print('# MAIL=president@whitehouse.gov', file=fout)
   
               print(SUMRM_NOOP, file=fout)
               if 'SUMRM_NOOP' in defs:
                   print('NOOP=' + defs['SUMRM_NOOP'], file=fout)
               else:
                   print('NOOP=0', file=fout)
   
               print(SUMRM_USER, file=fout)
               if 'SUMRM_USER' in defs:
                   print('USER=' + defs['SUMRM_USER'], file=fout)
               else:
                   print('USER=production', file=fout)
   
               print(SUMRM_NORUN, file=fout)
               # Default norun window is to have no such window. This can be accomplished by simply not providing either argument.
               if 'SUMRM_NORUN_START' in defs or 'SUMRM_NORUN_STOP' in defs:
                   if 'SUMRM_NORUN_START' in defs:
                       print('NORUN_START=' + defs['SUMRM_NORUN_START'], file=fout)
                   else:
                       print('NORUN_START=0', file=fout)
                   if 'SUMRM_NORUN_STOP' in defs:
                       print('NORUN_STOP=' + defs['SUMRM_NORUN_STOP'], file=fout)
                   else:
                       print('NORUN_STOP=0', file=fout)
               else:
                   print('# NORUN_START=0', file=fout)
                   print('# NORUN_STOP=0', file=fout)
   
       except OSError:
           print('Unable to open sum_rm temporary configuration file ' + cFileTmp + 'for writing.', file=sys.stderr)
           rv = bool(1)
   
       # If the content of the temporary file differs from the content of the existing configuration file, then overwrite
       # the original file. Otherwise, delete the temporary file
       if not rv:
           try:
               if filecmp.cmp(cFile, cFileTmp):
                   # Files identical - delete temporary file
                   try:
                       os.remove(cFileTmp)
   
                   except OSError as exc:
                       print('Unable to remove temporary file ' + exc.filename + '.', file=sys.stderr)
                       print(exc.strerr, file=sys.stderr)
               else:
                   # Replace original with temporary file
                   try:
                       os.rename(cFileTmp, cFile)
   
                   except OSError as exc:
                       print('Unable to update sum_rm configuration file ' + cFile + '.', file=sys.stderr)
                       print(exc.strerr, file=sys.stderr)
                       rv = bool(1)
           except OSError as exc:
               # One of the files doesn't exist.
               if exc.filename == cFile:
                   # We are ok - there might be no configuration file yet.
                   # Replace original with temporary file
                   try:
                       os.rename(cFileTmp, cFile)
   
                   except OSError as exc:
                       print('Unable to update sum_rm configuration file ' + cFile + '.', file=sys.stderr)
                       print(exc.strerr, file=sys.stderr)
                       rv = bool(1)
               else:
                   # There is a problem with the temp file - bail.
                   print('Unable to update sum_rm configuration file ' + cFile + '.', file=sys.stderr)
                   print(exc.strerr, file=sys.stderr)
                   rv = bool(1)
   
       return rv
   
   def configureNet(cfgfile, cfile, mfile, pfile, pCfile, pMfile, pRfile, pTfile, base, keymap, createSumRmCfg):
     rv = bool(0)     rv = bool(0)
  
     defs = {}     defs = {}
Line 783  def configureNet(cfgfile, cfile, mfile,
Line 961  def configureNet(cfgfile, cfile, mfile,
                 rv = getMgrUIDLine(defs, uidParam)                 rv = getMgrUIDLine(defs, uidParam)
                 if rv == bool(0):                 if rv == bool(0):
                     rv = parseConfig(None, keymap, uidParam, defs, cDefs, mDefsGen, None, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection)                     rv = parseConfig(None, keymap, uidParam, defs, cDefs, mDefsGen, None, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection)
   
             # Configure the compiler-selection make variables.             # Configure the compiler-selection make variables.
             if not rv:             if not rv:
                 rv = configureComps(defs, mDefsComps)                 rv = configureComps(defs, mDefsComps)
Line 796  def configureNet(cfgfile, cfile, mfile,
Line 973  def configureNet(cfgfile, cfile, mfile,
             if not rv:             if not rv:
                 rv = writeProjFiles(pCfile, pMfile, pRfile, pTfile, projCfg, projMkRules, projRules, projTarget)                 rv = writeProjFiles(pCfile, pMfile, pRfile, pTfile, projCfg, projMkRules, projRules, projTarget)
  
               # Write out the sum_rm.cfg file.
               if not rv and createSumRmCfg:
                   rv = generateSumRmCfg(defs)
     except IOError as exc:     except IOError as exc:
         print(exc.strerror, file=sys.stderr)         print(exc.strerror, file=sys.stderr)
         print('Unable to read configuration file ' + cfgfile + '.', file=sys.stderr)         print('Unable to read configuration file ' + cfgfile + '.', file=sys.stderr)
     except Exception as exc:     except Exception as exc:
           if len(exc.args) >= 2:
         type, msg = exc.args         type, msg = exc.args
           else:
               # re-raise the exception
               raise
   
         if type == 'unexpectedIccRet':         if type == 'unexpectedIccRet':
             print('icc -V returned this unexpected message:\n' + msg, file=sys.stderr)             print('icc -V returned this unexpected message:\n' + msg, file=sys.stderr)
             rv = bool(1)             rv = bool(1)
Line 866  def configureSdp(cfgfile, cfile, mfile,
Line 1051  def configureSdp(cfgfile, cfile, mfile,
                 # Write out the project-specific make files (make_basic.mk, Rules.mk, and target.mk).                 # Write out the project-specific make files (make_basic.mk, Rules.mk, and target.mk).
                 if not rv:                 if not rv:
                     rv = writeProjFiles(pCfile, pMfile, pRfile, pTfile, projCfg, projMkRules, projRules, projTarget)                     rv = writeProjFiles(pCfile, pMfile, pRfile, pTfile, projCfg, projMkRules, projRules, projTarget)
   
                   # At Stanford, skip the creation of the sum_rm configuration file. config.local will still
                   # have the SUMRM parameters, but they will not be used.
     except IOError as exc:     except IOError as exc:
         print(exc.strerror, file=sys.stderr)         print(exc.strerror, file=sys.stderr)
         print('Unable to read configuration file ' + cfgfile + '.', file=sys.stderr)         print('Unable to read configuration file ' + cfgfile + '.', file=sys.stderr)
     except Exception as exc:     except Exception as exc:
           if len(exc.args) >= 2:
         type = exc.args[0]         type = exc.args[0]
           else:
               # re-raise the exception
               raise
   
         if type == 'unexpectedIccRet':         if type == 'unexpectedIccRet':
             msg = exc.args[1]             msg = exc.args[1]
             print('icc -V returned this unexpected message:\n' + msg, file=sys.stderr)             print('icc -V returned this unexpected message:\n' + msg, file=sys.stderr)
Line 949  if rv == RET_SUCCESS:
Line 1142  if rv == RET_SUCCESS:
  
         # We also need to set the UID of the SUMS manager. We have the name of the         # We also need to set the UID of the SUMS manager. We have the name of the
         # SUMS manager (it is in the configuration file)         # SUMS manager (it is in the configuration file)
         configureNet(NET_CFG, cfile, mfile, pfile, pCfile, pMfile, pRfile, pTfile, optD['base'], keymap)          configureNet(NET_CFG, cfile, mfile, pfile, pCfile, pMfile, pRfile, pTfile, optD['base'], keymap, 'server' in optD)
     else:     else:
         # A Stanford user can override the parameters in configsdp.txt by copying that file to config.local,         # A Stanford user can override the parameters in configsdp.txt by copying that file to config.local,
         # and then editing config.local. So, if config.local exists, use that.         # and then editing config.local. So, if config.local exists, use that.

Legend:
Removed from v.1.9  
changed lines
  Added in v.1.17
Karen Tian
 Powered by
ViewCVS 0.9.4