(file) Return to localize.py CVS log (file) (dir) Up to [Development] / JSOC

Diff for /JSOC/localize.py between version 1.6 and 1.13

version 1.6, 2013/11/13 17:26:27 version 1.13, 2014/02/12 19:25:42
Line 1 
Line 1 
 #!/home/jsoc/bin/linux_x86_64/activepython #!/home/jsoc/bin/linux_x86_64/activepython
  
   # When run with the -s flag, localize.py configures the SUMS-server component of NetDRMS.
   
 import sys import sys
 import getopt import getopt
 import re import re
 import os import os
 import stat import stat
   import filecmp
 import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
 from subprocess import check_output, CalledProcessError from subprocess import check_output, CalledProcessError
  
Line 42  PERL_FXNS_A = """sub new
Line 45  PERL_FXNS_A = """sub new
     };     };
  
     bless($self, $clname);     bless($self, $clname);
     $self->{_paramsH} = $self->initialize();      $self->{_paramsH} = {};
       $self->initialize();
  
     return $self;     return $self;
 } }
Line 70  PERL_FXNS_B = """sub get
Line 74  PERL_FXNS_B = """sub get
 } }
 1;""" 1;"""
  
   SUMRM_COMMENT = """# This is the configuration file for the sum_rm program. It was auto-generated by the DRMS master configure script.
   # It controls the behavior of the sum_rm program, and is loaded each time sum_rm runs. To change the
   # parameter values in this configuration file, modify config.local, then re-run configure. This configuration
   # file will be updated only if parameters affecting it are modified. If such changes are made to config.local,
   # please make sure that the sum_rm service is disabled while configure in running.
   """
   
   SUMRM_DOC = """# sum_rm removes end-of-life SUMS data files to prevent disk-partitions from becomming 100% full.
   # sum_svc, the main SUMS service, starts this daemon when it is launched. Within an infinite loop, sum_rm sleeps
   # for SLEEP number of seconds (defined below). When it awakes, it loads the sum_rm configuration file. For each SU,
   # it then examines the 'effective_date' column within the sum_partn_alloc table of the SUMS database. It does so by
   # sorting all SUs by effective date (this date is actually an expiration date - the SU is good until the end of that day),
   # and then, starting with the SU with the oldest effective date, it deletes the SUs. It continues deleting SUs until one
   # of three conditions is met:
   #
   #   (a) The disk has at least PART_PERCENT_FREE percent free space.
   #   (b) All SUs have effective_date values in the future.
   #   (c) At least 600 SUs have been deleted (this is defined in the LIMIT statement in the file SUMLIB_RmDoX.pgc).
   #
   # After sum_rm stops deleteing SUs, it then sleeps for SLEEP seconds, completing the first iteration of the
   # infinite loop.
   """
   
   SUMRM_PARTN_PERCENT_FREE = """
   # This is the percentage at which all disk partitions are to be kept free.
   # If not specified, this defaults to 3. For example, setting PART_PERCENT_FREE = 5 will allow all partitions to
   # fill to 95% full. Dividing the number of unused blocks by the total number of blocks, and rounding up,
   # will result in the number specified by PART_PERCENT_FREE.
   #
   # NOTE : This behavior was previously controlled by the MAX_FREE_{n} family of parameters. {n} referred to the
   # disk-partition number and the value of parameter MAX_FREE_{n} was the MINIMUM number of free MB in the
   # partition [No clue why the word MAX was used]. As of NetDRMS 7.0, MAX_FREE_{n} has no effect."""
   
   SUMRM_SLEEP = """
   # The value is the number of seconds to sleep between iterations of the main loop in sum_rm."""
   
   SUMRM_LOG = """
   # The value is the log file (opened only at sum_rm startup; the sum_rm pid is appended to this file name)."""
   
   SUMRM_MAIL = """
   # The value is the email address of the recipient to be notified in case of a problem."""
   
   SUMRM_NOOP = """
   # If the value is set to anything other than 0, then sum_rm is rendered inactive. Otherwise, sum_rm is active."""
   
   SUMRM_USER = """
   # The value designates the linux user who is allowed to run sum_rm."""
   
   SUMRM_NORUN = """
   # This pair of paramters defines a window of time, during which sum_rm will become torpid - in this
   # window, sum_rm will not scan for SUs to delete, not will it delete any SUs. Each value is an integer, 0-23, that
   # represents an hour of the day. For example, if NORUN_START=8 and NORUN_STOP=10, then between 8am and 10am
   # local time, sum_rm will be dormant. To disable this behavior, set both parameters to the same value."""
  
 RULESPREFIX = """# Standard things RULESPREFIX = """# Standard things
 sp              := $(sp).x sp              := $(sp).x
Line 104  GFORT_MINOR = 2
Line 161  GFORT_MINOR = 2
 # Read arguments # Read arguments
 #  d - localization directory #  d - localization directory
 #  b - base name of all parameter files (e.g., -b drmsparams --> drmsparams.h, drmsparams.mk, drmsparams.pm, etc.) #  b - base name of all parameter files (e.g., -b drmsparams --> drmsparams.h, drmsparams.mk, drmsparams.pm, etc.)
 #  m - make file  #  s - configure a NetDRMS server (i.e., SUMS server). Otherwise, do client configuration only. A server
   #      configuration will modify something that only the production user has access to. An example is the sum_rm.cfg
   #      file. To modify that file, the user running configure must be running configure on the SUMS-server host, and
   #      must have write permission on sum_rm.cfg.
 def GetArgs(args): def GetArgs(args):
     rv = bool(0)     rv = bool(0)
     optD = {}     optD = {}
  
     try:     try:
         opts, remainder = getopt.getopt(args, "hd:b:",["dir=", "base="])          opts, remainder = getopt.getopt(args, "hd:b:s",["dir=", "base="])
     except getopt.GetoptError:     except getopt.GetoptError:
         print('Usage:')         print('Usage:')
         print('localize.py [-h] -d <localization directory> -b <parameter file base>')         print('localize.py [-h] -d <localization directory> -b <parameter file base>')
Line 129  def GetArgs(args):
Line 189  def GetArgs(args):
                     optD['dir'] = matchobj.group(1)                     optD['dir'] = matchobj.group(1)
             elif opt in ("-b", "--base"):             elif opt in ("-b", "--base"):
                 optD['base'] = arg                 optD['base'] = arg
               elif opt in ("-s"):
                   optD['server'] = ""
             else:             else:
                 optD[opt] = arg                 optD[opt] = arg
  
Line 259  def processParam(cfgfile, line, regexpQu
Line 321  def processParam(cfgfile, line, regexpQu
                 # constants (the names of which are the parameter names)                 # constants (the names of which are the parameter names)
                 # we can refer to those in the init section. The key variable holds the                 # we can refer to those in the init section. The key variable holds the
                 # name of the constant.                 # name of the constant.
                 perlInitSection.extend(list('\n  $self->{_paramsH}->{' + key + '} = ' + "'" + val + "';"))                  perlInitSection.extend(list("\n  $self->{_paramsH}->{'" + key + "'} = " + key + ';'))
             else:             else:
                 # No quote qualifier                 # No quote qualifier
                 raise Exception('missingQuoteQual', key)                 raise Exception('missingQuoteQual', key)
Line 370  def parseConfig(fin, keymap, addenda, de
Line 432  def parseConfig(fin, keymap, addenda, de
         regexpCustMkBeg = re.compile(r"^_CUST_")         regexpCustMkBeg = re.compile(r"^_CUST_")
         regexpCustMkEnd = re.compile(r"^_ENDCUST_")         regexpCustMkEnd = re.compile(r"^_ENDCUST_")
         regexpDiv = re.compile(r"^__")         regexpDiv = re.compile(r"^__")
         regexp = re.compile(r"^\s*(\S+)\s+(\S+)")          regexp = re.compile(r"^\s*(\S+)\s+(\S.*)")
  
         platDict = {}         platDict = {}
         machDict = {}         machDict = {}
Line 395  def parseConfig(fin, keymap, addenda, de
Line 457  def parseConfig(fin, keymap, addenda, de
                     section = newSection                     section = newSection
  
                 if section == 'make':                 if section == 'make':
   
                     # There are some blocks of lines in the __MAKE__ section that must be copied ver batim to the output make file.                     # There are some blocks of lines in the __MAKE__ section that must be copied ver batim to the output make file.
                     # The blocks are defined by _CUST_/_ENDCUST_ tags.                     # The blocks are defined by _CUST_/_ENDCUST_ tags.
                     matchobj = regexpCustMkBeg.match(line)                     matchobj = regexpCustMkBeg.match(line)
Line 416  def parseConfig(fin, keymap, addenda, de
Line 479  def parseConfig(fin, keymap, addenda, de
                     ppRet = processParam(iscfg, line, regexpQuote, regexp, keymap, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, platDict, machDict, section)                     ppRet = processParam(iscfg, line, regexpQuote, regexp, keymap, defs, cDefs, mDefsGen, mDefsMake, perlConstSection, perlInitSection, platDict, machDict, section)
  
                     if ppRet:                     if ppRet:
                         break;                          break
                 elif section == 'projcfg':                 elif section == 'projcfg':
                     # Copy the line ver batim to the projCfg list (configure)                     # Copy the line ver batim to the projCfg list (configure)
                     for line in fin:                     for line in fin:
Line 509  def parseConfig(fin, keymap, addenda, de
Line 572  def parseConfig(fin, keymap, addenda, de
     if not rv:     if not rv:
         for mach in machDict:         for mach in machDict:
             mDefsMake.extend(list('\nifeq ($(MACHTYPE), ' + mach + ')'))             mDefsMake.extend(list('\nifeq ($(MACHTYPE), ' + mach + ')'))
             for var in platDict[plat]:              for var in machDict[mach]:
                 mDefsMake.extend(list('\n' + var + ' = ' + platDict[plat][var]))                  mDefsMake.extend(list('\n' + var + ' = ' + machDict[mach][var]))
             mDefsMake.extend(list('\nendif\n'))             mDefsMake.extend(list('\nendif\n'))
   
     return rv     return rv
  
 def getMgrUIDLine(defs, uidParam): def getMgrUIDLine(defs, uidParam):
Line 553  def configureComps(defs, mDefs):
Line 615  def configureComps(defs, mDefs):
         hasgfort = bool(0)         hasgfort = bool(0)
  
         # Try icc.         # Try icc.
         cmd = 'icc -V 2>&1'          cmd = 'icc --version 2>&1'
         try:         try:
             ret = check_output(cmd, shell=True)             ret = check_output(cmd, shell=True)
             ret = ret.decode("utf-8")             ret = ret.decode("utf-8")
Line 562  def configureComps(defs, mDefs):
Line 624  def configureComps(defs, mDefs):
             rv = bool(1)             rv = bool(1)
  
         if not rv:         if not rv:
             regexp = re.compile(r".+Version\s+(\d+)[.](\d+)", re.DOTALL)              regexp = re.compile(r"\s*\S+\s+\S+\s+(\d+)[.](\d+)", re.DOTALL)
             matchobj = regexp.match(ret)             matchobj = regexp.match(ret)
             if matchobj is None:             if matchobj is None:
                 raise Exception('unexpectedIccRet', ret)                 raise Exception('unexpectedIccRet', ret)
Line 594  def configureComps(defs, mDefs):
Line 656  def configureComps(defs, mDefs):
                         hasgcc = bool(1)                         hasgcc = bool(1)
  
         # Try ifort.         # Try ifort.
         cmd = 'ifort -V 2>&1'          cmd = 'ifort --version 2>&1'
         try:         try:
             ret = check_output(cmd, shell=True)             ret = check_output(cmd, shell=True)
             ret = ret.decode("utf-8")             ret = ret.decode("utf-8")
Line 603  def configureComps(defs, mDefs):
Line 665  def configureComps(defs, mDefs):
             rv = bool(1)             rv = bool(1)
  
         if not rv:         if not rv:
             regexp = re.compile(r".+Version\s+(\d+)\.(\d+)", re.DOTALL)              regexp = re.compile(r"\s*\S+\s+\S+\s+(\d+)\.(\d+)", re.DOTALL)
             matchobj = regexp.match(ret)             matchobj = regexp.match(ret)
             if matchobj is None:             if matchobj is None:
                 raise Exception('unexpectedIfortRet', ret)                 raise Exception('unexpectedIfortRet', ret)
Line 651  def configureComps(defs, mDefs):
Line 713  def configureComps(defs, mDefs):
             mDefs.extend(list('\nFCOMPILER = gfortran'))             mDefs.extend(list('\nFCOMPILER = gfortran'))
  
         # Environment overrides. These get written, regardless of the disposition of auto-configuration.         # Environment overrides. These get written, regardless of the disposition of auto-configuration.
         mDefs.extend(list('\nifneq $(JSOC_COMPILER,)\n  COMPILER = $(JSOC_COMPILER)\nendif'))          mDefs.extend(list('\nifneq ($(JSOC_COMPILER),)\n  COMPILER = $(JSOC_COMPILER)\nendif'))
         mDefs.extend(list('\nifneq $(JSOC_FCOMPILER,)\n  FCOMPILER = $(JSOC_FCOMPILER)\nendif'))          mDefs.extend(list('\nifneq ($(JSOC_FCOMPILER),)\n  FCOMPILER = $(JSOC_FCOMPILER)\nendif'))
  
     return rv     return rv
  
Line 734  def writeProjFiles(pCfile, pMfile, pRfil
Line 796  def writeProjFiles(pCfile, pMfile, pRfil
         rv = bool(1)         rv = bool(1)
  
     if not rv:     if not rv:
           if os.path.exists(pCfile):
         try:         try:
             os.chmod(pCfile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IROTH)             os.chmod(pCfile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IROTH)
         except OSError as exc:         except OSError as exc:
                   type, value, traceback = sys.exc_info()
                   print('Unable to chmod file ' + "'" + value.filename + "'.", file=sys.stderr)
             print(exc.strerror, file=sys.stderr)             print(exc.strerror, file=sys.stderr)
             rv = bool(1)             rv = bool(1)
  
     return rv     return rv
  
 def configureNet(cfgfile, cfile, mfile, pfile, pCfile, pMfile, pRfile, pTfile, base, keymap):  def generateSumRmCfg(defs):
       rv = bool(0)
       # ACK! Remember that Rick renamed these parameters. The ones in config.local are the aliases - do not use those.
       # Use the ones that those map to (defined in config.local.map).
       cFileTmp = defs['SUMLOG_BASEDIR'] + '/' + '.sum_rm.cfg.tmp'
       cFile = defs['SUMLOG_BASEDIR'] + '/' + 'sum_rm.cfg'
   
       # Write a temporary file sum_rm configuration file.
       try:
           with open(cFileTmp, 'w') as fout:
               # Print comment at the top of the configuration file.
               print(SUMRM_COMMENT, file=fout)
               print(SUMRM_DOC, file=fout)
               print(SUMRM_PARTN_PERCENT_FREE, file=fout)
               if 'SUMRM_PART_PERCENT_FREE' in defs:
                   print('PART_PERCENT_FREE=' + defs['SUMRM_PART_PERCENT_FREE'], file=fout)
               else:
                   print('PART_PERCENT_FREE=3', file=fout)
   
               print(SUMRM_SLEEP, file=fout)
               if 'SUMRM_SLEEP' in defs:
                   print('SLEEP=' + defs['SUMRM_SLEEP'], file=fout)
               else:
                   print('SLEEP=300', file=fout)
   
               print(SUMRM_LOG, file=fout)
               if 'SUMRM_LOG' in defs:
                   print('LOG=' + defs['SUMRM_LOG'], file=fout)
               else:
                   print('LOG=/tmp/sum_rm.log', file=fout)
   
               print(SUMRM_MAIL, file=fout)
               # No default for mail - don't send nothing to nobody unless the operator has asked for notifications.
               if 'SUMRM_MAIL' in defs:
                   print('MAIL=' + defs['SUMRM_MAIL'], file=fout)
               else:
                   print('# MAIL=president@whitehouse.gov', file=fout)
   
               print(SUMRM_NOOP, file=fout)
               if 'SUMRM_NOOP' in defs:
                   print('NOOP=' + defs['SUMRM_NOOP'], file=fout)
               else:
                   print('NOOP=0', file=fout)
   
               print(SUMRM_USER, file=fout)
               if 'SUMRM_USER' in defs:
                   print('USER=' + defs['SUMRM_USER'], file=fout)
               else:
                   print('USER=production', file=fout)
   
               print(SUMRM_NORUN, file=fout)
               # Default norun window is to have no such window. This can be accomplished by simply not providing either argument.
               if 'SUMRM_NORUN_START' in defs or 'SUMRM_NORUN_STOP' in defs:
                   if 'SUMRM_NORUN_START' in defs:
                       print('NORUN_START=' + defs['SUMRM_NORUN_START'], file=fout)
                   else:
                       print('NORUN_START=0', file=fout)
                   if 'SUMRM_NORUN_STOP' in defs:
                       print('NORUN_STOP=' + defs['SUMRM_NORUN_STOP'], file=fout)
                   else:
                       print('NORUN_STOP=0', file=fout)
               else:
                   print('# NORUN_START=0', file=fout)
                   print('# NORUN_STOP=0', file=fout)
   
       except OSError:
           print('Unable to open sum_rm temporary configuration file ' + cFileTmp + 'for writing.', file=sys.stderr)
           rv = bool(1)
   
       # If the content of the temporary file differs from the content of the existing configuration file, then overwrite
       # the original file. Otherwise, delete the temporary file
       if not rv:
           try:
               if filecmp.cmp(cFile, cFileTmp):
                   # Files identical - delete temporary file
                   try:
                       os.remove(cFileTmp)
   
                   except OSError as exc:
                       print('Unable to remove temporary file ' + exc.filename + '.', file=sys.stderr)
                       print(exc.strerr, file=sys.stderr)
               else:
                   # Replace original with temporary file
                   try:
                       os.rename(cFileTmp, cFile)
   
                   except OSError as exc:
                       print('Unable to update sum_rm configuration file ' + cFile + '.', file=sys.stderr)
                       print(exc.strerr, file=sys.stderr)
                       rv = bool(1)
           except OSError as exc:
               # One of the files doesn't exist.
               if exc.filename == cFile:
                   # We are ok - there might be no configuration file yet.
                   # Replace original with temporary file
                   try:
                       os.rename(cFileTmp, cFile)
   
                   except OSError as exc:
                       print('Unable to update sum_rm configuration file ' + cFile + '.', file=sys.stderr)
                       print(exc.strerr, file=sys.stderr)
                       rv = bool(1)
               else:
                   # There is a problem with the temp file - bail.
                   print('Unable to update sum_rm configuration file ' + cFile + '.', file=sys.stderr)
                   print(exc.strerr, file=sys.stderr)
                   rv = bool(1)
   
       return rv
   
   def configureNet(cfgfile, cfile, mfile, pfile, pCfile, pMfile, pRfile, pTfile, base, keymap, createSumRmCfg):
     rv = bool(0)     rv = bool(0)
  
     defs = {}     defs = {}
Line 782  def configureNet(cfgfile, cfile, mfile,
Line 957  def configureNet(cfgfile, cfile, mfile,
                 rv = getMgrUIDLine(defs, uidParam)                 rv = getMgrUIDLine(defs, uidParam)
                 if rv == bool(0):                 if rv == bool(0):
                     rv = parseConfig(None, keymap, uidParam, defs, cDefs, mDefsGen, None, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection)                     rv = parseConfig(None, keymap, uidParam, defs, cDefs, mDefsGen, None, projCfg, projMkRules, projRules, projTarget, perlConstSection, perlInitSection)
   
             # Configure the compiler-selection make variables.             # Configure the compiler-selection make variables.
             if not rv:             if not rv:
                 rv = configureComps(defs, mDefsComps)                 rv = configureComps(defs, mDefsComps)
Line 795  def configureNet(cfgfile, cfile, mfile,
Line 969  def configureNet(cfgfile, cfile, mfile,
             if not rv:             if not rv:
                 rv = writeProjFiles(pCfile, pMfile, pRfile, pTfile, projCfg, projMkRules, projRules, projTarget)                 rv = writeProjFiles(pCfile, pMfile, pRfile, pTfile, projCfg, projMkRules, projRules, projTarget)
  
               # Write out the sum_rm.cfg file.
               if not rv and createSumRmCfg:
                   rv = generateSumRmCfg(defs)
     except IOError as exc:     except IOError as exc:
         print(exc.strerror, file=sys.stderr)         print(exc.strerror, file=sys.stderr)
         print('Unable to read configuration file ' + cfgfile + '.', file=sys.stderr)         print('Unable to read configuration file ' + cfgfile + '.', file=sys.stderr)
Line 865  def configureSdp(cfgfile, cfile, mfile,
Line 1042  def configureSdp(cfgfile, cfile, mfile,
                 # Write out the project-specific make files (make_basic.mk, Rules.mk, and target.mk).                 # Write out the project-specific make files (make_basic.mk, Rules.mk, and target.mk).
                 if not rv:                 if not rv:
                     rv = writeProjFiles(pCfile, pMfile, pRfile, pTfile, projCfg, projMkRules, projRules, projTarget)                     rv = writeProjFiles(pCfile, pMfile, pRfile, pTfile, projCfg, projMkRules, projRules, projTarget)
   
                   # At Stanford, skip the creation of the sum_rm configuration file. config.local will still
                   # have the SUMRM parameters, but they will not be used.
     except IOError as exc:     except IOError as exc:
         print(exc.strerror, file=sys.stderr)         print(exc.strerror, file=sys.stderr)
         print('Unable to read configuration file ' + cfgfile + '.', file=sys.stderr)         print('Unable to read configuration file ' + cfgfile + '.', file=sys.stderr)
Line 948  if rv == RET_SUCCESS:
Line 1128  if rv == RET_SUCCESS:
  
         # We also need to set the UID of the SUMS manager. We have the name of the         # We also need to set the UID of the SUMS manager. We have the name of the
         # SUMS manager (it is in the configuration file)         # SUMS manager (it is in the configuration file)
         configureNet(NET_CFG, cfile, mfile, pfile, pCfile, pMfile, pRfile, pTfile, optD['base'], keymap)          configureNet(NET_CFG, cfile, mfile, pfile, pCfile, pMfile, pRfile, pTfile, optD['base'], keymap, 'server' in optD)
       else:
           # A Stanford user can override the parameters in configsdp.txt by copying that file to config.local,
           # and then editing config.local. So, if config.local exists, use that.
           if os.path.isfile(cdir + '/' + NET_CFG):
               configureSdp(NET_CFG, cfile, mfile, pfile, pCfile, pMfile, pRfile, pTfile, optD['base'])
     else:     else:
         configureSdp(SDP_CFG, cfile, mfile, pfile, pCfile, pMfile, pRfile, pTfile, optD['base'])         configureSdp(SDP_CFG, cfile, mfile, pfile, pCfile, pMfile, pRfile, pTfile, optD['base'])
   
   
   

Legend:
Removed from v.1.6  
changed lines
  Added in v.1.13
Karen Tian
 Powered by
ViewCVS 0.9.4