JSOC/localize.py - diff

Return to localize.py CVS log

Up to [Development] / JSOC

Diff for /JSOC/localize.py between version 1.12 and 1.14

version 1.12, 2013/11/22 01:58:25

version 1.14, 2014/04/17 19:11:05

Line 1

#!/home/jsoc/bin/linux_x86_64/activepython

# When run with the -s flag, localize.py configures the SUMS-server component of NetDRMS.

import sys

import getopt

import re

import os

import stat

import filecmp

import xml.etree.ElementTree as ET

from subprocess import check_output, CalledProcessError

Line 71 PERL_FXNS_B = """sub get

Line 74 PERL_FXNS_B = """sub get

}

1;"""

SUMRM_COMMENT = """# This is the configuration file for the sum_rm program. It was auto-generated by the DRMS master configure script.

# It controls the behavior of the sum_rm program, and is loaded each time sum_rm runs. To change the

# parameter values in this configuration file, modify config.local, then re-run configure. This configuration

# file will be updated only if parameters affecting it are modified. If such changes are made to config.local,

# please make sure that the sum_rm service is disabled while configure in running.

"""

SUMRM_DOC = """# sum_rm removes end-of-life SUMS data files to prevent disk-partitions from becomming 100% full.

# sum_svc, the main SUMS service, starts this daemon when it is launched. Within an infinite loop, sum_rm sleeps

# for SLEEP number of seconds (defined below). When it awakes, it loads the sum_rm configuration file. For each SU,

# it then examines the 'effective_date' column within the sum_partn_alloc table of the SUMS database. It does so by

# sorting all SUs by effective date (this date is actually an expiration date - the SU is good until the end of that day),

# and then, starting with the SU with the oldest effective date, it deletes the SUs. It continues deleting SUs until one

# of three conditions is met:

# (a) The disk has at least PART_PERCENT_FREE percent free space.

# (b) All SUs have effective_date values in the future.

# (c) At least 600 SUs have been deleted (this is defined in the LIMIT statement in the file SUMLIB_RmDoX.pgc).

# After sum_rm stops deleteing SUs, it then sleeps for SLEEP seconds, completing the first iteration of the

# infinite loop.

"""

SUMRM_PARTN_PERCENT_FREE = """

# This is the percentage at which all disk partitions are to be kept free.

# If not specified, this defaults to 3. For example, setting PART_PERCENT_FREE = 5 will allow all partitions to

# fill to 95% full. Dividing the number of unused blocks by the total number of blocks, and rounding up,

# will result in the number specified by PART_PERCENT_FREE.

# NOTE : This behavior was previously controlled by the MAX_FREE_{n} family of parameters. {n} referred to the

# disk-partition number and the value of parameter MAX_FREE_{n} was the MINIMUM number of free MB in the

# partition [No clue why the word MAX was used]. As of NetDRMS 7.0, MAX_FREE_{n} has no effect."""

SUMRM_SLEEP = """

# The value is the number of seconds to sleep between iterations of the main loop in sum_rm."""

SUMRM_LOG = """

# The value is the log file (opened only at sum_rm startup; the sum_rm pid is appended to this file name)."""

SUMRM_MAIL = """

# The value is the email address of the recipient to be notified in case of a problem."""

SUMRM_NOOP = """

# If the value is set to anything other than 0, then sum_rm is rendered inactive. Otherwise, sum_rm is active."""

SUMRM_USER = """

# The value designates the linux user who is allowed to run sum_rm."""

SUMRM_NORUN = """

# This pair of paramters defines a window of time, during which sum_rm will become torpid - in this

# window, sum_rm will not scan for SUs to delete, not will it delete any SUs. Each value is an integer, 0-23, that

# represents an hour of the day. For example, if NORUN_START=8 and NORUN_STOP=10, then between 8am and 10am

# local time, sum_rm will be dormant. To disable this behavior, set both parameters to the same value."""

RULESPREFIX = """# Standard things

sp := $(sp).x

Line 105 GFORT_MINOR = 2

Line 161 GFORT_MINOR = 2

# Read arguments

# d - localization directory

# b - base name of all parameter files (e.g., -b drmsparams --> drmsparams.h, drmsparams.mk, drmsparams.pm, etc.)

# m - make file

# s - configure a NetDRMS server (i.e., SUMS server). Otherwise, do client configuration only. A server

# configuration will modify something that only the production user has access to. An example is the sum_rm.cfg

# file. To modify that file, the user running configure must be running configure on the SUMS-server host, and

# must have write permission on sum_rm.cfg.

def GetArgs(args):

rv = bool(0)

optD = {}

try:

opts, remainder = getopt.getopt(args, "hd:b:",["dir=", "base="])

opts, remainder = getopt.getopt(args, "hd:b:s",["dir=", "base="])

except getopt.GetoptError:

print('Usage:')

print('localize.py [-h] -d <localization directory> -b <parameter file base>')

Line 130 def GetArgs(args):

Line 189 def GetArgs(args):

optD['dir'] = matchobj.group(1)

elif opt in ("-b", "--base"):

optD['base'] = arg

elif opt in ("-s"):

optD['server'] = ""

else:

optD[opt] = arg

Line 746 def writeProjFiles(pCfile, pMfile, pRfil

Line 807 def writeProjFiles(pCfile, pMfile, pRfil

return rv

def configureNet(cfgfile, cfile, mfile, pfile, pCfile, pMfile, pRfile, pTfile, base, keymap):

def generateSumRmCfg(defs):

rv = bool(0)

# ACK! Remember that Rick renamed these parameters. The ones in config.local are the aliases - do not use those.

# Use the ones that those map to (defined in config.local.map).

cFileTmp = defs['SUMLOG_BASEDIR'] + '/' + '.sum_rm.cfg.tmp'

cFile = defs['SUMLOG_BASEDIR'] + '/' + 'sum_rm.cfg'

# Write a temporary file sum_rm configuration file.

try:

with open(cFileTmp, 'w') as fout:

# Print comment at the top of the configuration file.

print(SUMRM_COMMENT, file=fout)

print(SUMRM_DOC, file=fout)

print(SUMRM_PARTN_PERCENT_FREE, file=fout)

if 'SUMRM_PART_PERCENT_FREE' in defs:

print('PART_PERCENT_FREE=' + defs['SUMRM_PART_PERCENT_FREE'], file=fout)

else:

print('PART_PERCENT_FREE=3', file=fout)

print(SUMRM_SLEEP, file=fout)

if 'SUMRM_SLEEP' in defs:

print('SLEEP=' + defs['SUMRM_SLEEP'], file=fout)

else:

print('SLEEP=300', file=fout)

print(SUMRM_LOG, file=fout)

if 'SUMRM_LOG' in defs:

print('LOG=' + defs['SUMRM_LOG'], file=fout)

else:

print('LOG=/tmp/sum_rm.log', file=fout)

print(SUMRM_MAIL, file=fout)

# No default for mail - don't send nothing to nobody unless the operator has asked for notifications.

if 'SUMRM_MAIL' in defs:

print('MAIL=' + defs['SUMRM_MAIL'], file=fout)

else:

print('# MAIL=president@whitehouse.gov', file=fout)

print(SUMRM_NOOP, file=fout)

if 'SUMRM_NOOP' in defs:

print('NOOP=' + defs['SUMRM_NOOP'], file=fout)

else:

print('NOOP=0', file=fout)

print(SUMRM_USER, file=fout)

if 'SUMRM_USER' in defs:

print('USER=' + defs['SUMRM_USER'], file=fout)

else:

print('USER=production', file=fout)

print(SUMRM_NORUN, file=fout)

# Default norun window is to have no such window. This can be accomplished by simply not providing either argument.

if 'SUMRM_NORUN_START' in defs or 'SUMRM_NORUN_STOP' in defs:

if 'SUMRM_NORUN_START' in defs:

print('NORUN_START=' + defs['SUMRM_NORUN_START'], file=fout)

else:

print('NORUN_START=0', file=fout)

if 'SUMRM_NORUN_STOP' in defs:

print('NORUN_STOP=' + defs['SUMRM_NORUN_STOP'], file=fout)

else:

print('NORUN_STOP=0', file=fout)

else:

print('# NORUN_START=0', file=fout)

print('# NORUN_STOP=0', file=fout)

except OSError:

print('Unable to open sum_rm temporary configuration file ' + cFileTmp + 'for writing.', file=sys.stderr)

rv = bool(1)

# If the content of the temporary file differs from the content of the existing configuration file, then overwrite

# the original file. Otherwise, delete the temporary file

if not rv:

try:

if filecmp.cmp(cFile, cFileTmp):

# Files identical - delete temporary file

try:

os.remove(cFileTmp)

except OSError as exc:

print('Unable to remove temporary file ' + exc.filename + '.', file=sys.stderr)

print(exc.strerr, file=sys.stderr)

else:

# Replace original with temporary file

try:

os.rename(cFileTmp, cFile)

except OSError as exc:

print('Unable to update sum_rm configuration file ' + cFile + '.', file=sys.stderr)

print(exc.strerr, file=sys.stderr)

rv = bool(1)

except OSError as exc:

# One of the files doesn't exist.

if exc.filename == cFile:

# We are ok - there might be no configuration file yet.

# Replace original with temporary file

try:

os.rename(cFileTmp, cFile)

except OSError as exc:

print('Unable to update sum_rm configuration file ' + cFile + '.', file=sys.stderr)

print(exc.strerr, file=sys.stderr)

rv = bool(1)

else:

# There is a problem with the temp file - bail.

print('Unable to update sum_rm configuration file ' + cFile + '.', file=sys.stderr)

print(exc.strerr, file=sys.stderr)

rv = bool(1)

return rv

def configureNet(cfgfile, cfile, mfile, pfile, pCfile, pMfile, pRfile, pTfile, base, keymap, createSumRmCfg):

rv = bool(0)

defs = {}

Line 793 def configureNet(cfgfile, cfile, mfile,

Line 964 def configureNet(cfgfile, cfile, mfile,

# Write out the parameter files.

if not rv:

rv = writeParamsFiles(base, cfile, mfile, pfile, cDefs, mDefsGen, mDefsMake, mDefsComps, perlConstSection, perlInitSection)

# Write out the project-specific make files (make_basic.mk, Rules.mk, and target.mk).

if not rv:

rv = writeProjFiles(pCfile, pMfile, pRfile, pTfile, projCfg, projMkRules, projRules, projTarget)

# Write out the sum_rm.cfg file.

if not rv and createSumRmCfg:

rv = generateSumRmCfg(defs)

except IOError as exc:

print(exc.strerror, file=sys.stderr)

print('Unable to read configuration file ' + cfgfile + '.', file=sys.stderr)

except Exception as exc:

if len(exc.args >= 2):

type, msg = exc.args

else:

# re-raise the exception

raise

if type == 'unexpectedIccRet':

print('icc -V returned this unexpected message:\n' + msg, file=sys.stderr)

rv = bool(1)

Line 866 def configureSdp(cfgfile, cfile, mfile,

Line 1047 def configureSdp(cfgfile, cfile, mfile,

# Write out the project-specific make files (make_basic.mk, Rules.mk, and target.mk).

if not rv:

rv = writeProjFiles(pCfile, pMfile, pRfile, pTfile, projCfg, projMkRules, projRules, projTarget)

# At Stanford, skip the creation of the sum_rm configuration file. config.local will still

# have the SUMRM parameters, but they will not be used.

except IOError as exc:

print(exc.strerror, file=sys.stderr)

print('Unable to read configuration file ' + cfgfile + '.', file=sys.stderr)

Line 949 if rv == RET_SUCCESS:

Line 1133 if rv == RET_SUCCESS:

# We also need to set the UID of the SUMS manager. We have the name of the

# SUMS manager (it is in the configuration file)

configureNet(NET_CFG, cfile, mfile, pfile, pCfile, pMfile, pRfile, pTfile, optD['base'], keymap)

configureNet(NET_CFG, cfile, mfile, pfile, pCfile, pMfile, pRfile, pTfile, optD['base'], keymap, 'server' in optD)

else:

# A Stanford user can override the parameters in configsdp.txt by copying that file to config.local,

# and then editing config.local. So, if config.local exists, use that.

Legend:

Removed from v.1.12
changed lines
	Added in v.1.14

Karen Tian