Return to localize.py CVS log

Up to [Development] / JSOC

Add support for auto-selection of compilers, merge the format of the Stanford and NetDRMS configuration files, add support for determining the SUMS manager UID from the SUMS manager account name, add support for parsing the config-file make variables sections (including those needed for locating third-party libraries.)

#!/home/jsoc/bin/linux_x86_64/activepython

import os.path
import sys
import getopt
import re
from subprocess import check_output, CalledProcessError

# Constants
VERS_FILE = 'jsoc_version.h'
SDP_CFG = 'configsdp.txt'
NET_CFG = 'config.local'
NET_CFGMAP = 'config.local.map'
RET_SUCCESS = 0
RET_NOTDRMS = 1

PREFIX = """# This file was auto-generated by localize.py. Please do not edit it directly (running
# configure will run localize.py, which will then overwrite any edits manually performed).
"""

C_PREFIX = """/* This file was auto-generated by localize.py. Please do not edit it directly (running
 * configure will run localize.py, which will then overwrite any edits manually performed). */
"""

PERL_BINPATH = '#!/usr/bin/perl\n'

PERL_INTIAL = """package drmsparams;
    
use warnings;
use strict;
"""

PERL_FXNS_A = """sub new
{
    my($clname) = shift;
    
    my($self) = 
    {
        _paramsH => undef
    };
    
    bless($self, $clname);
    $self->{_paramsH} = $self->initialize();
    
    return $self;
}
    
sub DESTROY
{
    my($self) = shift;
}
"""

PERL_FXNS_B = """sub get
{
    my($self) = shift;
    my($name) = shift;
    my($rv);
    
    if (exists($self->{_paramsH}->{$name}))
    {
        return $self->{_paramsH}->{$name};
    }
    else
    {
        return undef;
    }
}
1;"""

ICC_MAJOR = 9
ICC_MINOR = 0
GCC_MAJOR = 3
GCC_MINOR = 0
IFORT_MAJOR = 9
IFORT_MINOR = 0
GFORT_MAJOR = 4
GFORT_MINOR = 2


# Read arguments
#  d - localization directory
#  b - base name of all parameter files (e.g., -b drmsparams --> drmsparams.h, drmsparams.mk, drmsparams.pm, etc.)
#  m - make file
def GetArgs(args):
    rv = bool(0)
    optD = {}
    
    try:
        opts, remainder = getopt.getopt(args, "hd:b:",["dir=", "base="])
    except getopt.GetoptError:
        print('Usage:')
        print('localize.py [-h] -d <localization directory> -b <parameter file base>')
        rv = bool(1)
    
    if rv == bool(0):
        for opt, arg in opts:
            if opt == '-h':
                print('localize.py [-h] -d <localization directory> -b <parameter file base>')
            elif opt in ("-d", "--dir"):
                regexp = re.compile(r"(\S+)/?")
                matchobj = regexp.match(arg)
                if matchobj is None:
                    rv = bool(1)
                else:
                    optD['dir'] = matchobj.group(1)
            elif opt in ("-b", "--base"):
                optD['base'] = arg
            else:
                optD[opt] = arg
	
    return optD

def createMacroStr(key, val, keyColLen, status):
    if keyColLen < len(key):
        status = bool(1)
        return None
    else:
        nsp = keyColLen - len(key)
        spaces = str()
        for isp in range(nsp):
            spaces += ' '
        status = bool(0)
        return '#define ' + key + spaces + val + '\n'

def createPerlConst(key, val, keyColLen, status):
    if keyColLen < len(key):
        status = bool(1)
        return None
    else:
        nsp = keyColLen - len(key)
        spaces = str()
        for isp in range(nsp):
            spaces += ' '
        status = bool(0)
        return 'use constant ' + key + ' => ' + spaces + val + ';\n'

def isSupportedPlat(plat):
    regexp = re.compile(r"\s*(^x86_64|^ia32|^ia64|^avx)", re.IGNORECASE)
    matchobj = regexp.match(plat);
        
    if not matchobj is None:
        return bool(1);
    else:
        return bool(0);

def processMakeParam(mDefs, key, val, platDict, matchDict):
    varMach = None
    regexp = re.compile(r"(\S+):(\S+)")
    matchobj = regexp.match(key)
    if not matchobj is None:
        varName = matchobj.group(1)
        varMach = matchobj.group(2)
    else:
        varName = key
    
    varValu = val
        
    if varMach is None:
        mDefs.extend(list('\n' + varName + ' = ' + varValu))
    else:
        if isSupportedPlat(varMach):
            # The guard will compare varValu to $JSOC_MACHINE.
            if not varMach in platDict:
                platDict[varMach] = {}
            platDict[varMach][varName] = varValu
        else:
            # The guard will compare varValu to $MACHINETYPE (this is just the hostname of the machine on which localize.py is running).
            if not varMach in machDict:
                machDict[varMach] = {}
            machDict[varMach][varName] = varValu

def processParam(cfgfile, line, regexpComm, regexpDefs, regexpMake, regexpQuote, regexp, keymap, defs, cDefs, mDefs, perlConstSection, perlInitSection, platDict, machDict, section):
    status = 0
    
    matchobj = regexpComm.match(line)
    if not matchobj is None:
        # Skip comment line
        return bool(0)
    
    matchobj = regexpDefs.match(line)
    if not matchobj is None:
        del section[:]
        section.extend(list('defs'))
        return bool(0)
    
    matchobj = regexpMake.match(line)
    if not matchobj is None:
        del section[:]
        section.extend(list('make'))
        return bool(0)
            
    if ''.join(section) == 'defs' or not cfgfile:
        matchobj = regexp.match(line)
        if not matchobj is None:
            # We have a key-value line
            keyCfgSp = matchobj.group(1)
            val = matchobj.group(2)
            
            # Must map the indirect name to the actual name
            if keymap:
                # Map to actual name only if the keymap is not empty (which signifies NA).
                if keyCfgSp in keymap:
                    key = keymap[keyCfgSp]
                elif keyCfgSp == 'LOCAL_CONFIG_SET' or keyCfgSp == 'DRMS_SAMPLE_NAMESPACE':
                    # Ignore parameters that are not useful and shouldn't have been there in the first place
                    return bool(0)
                elif not cfgfile:
                    # Should not be doing mapping for addenda
                    key = keyCfgSp
                else:
                    raise Exception('badKeyMapKey', keyCfgSp)
            else:
                key = keyCfgSp
            
            matchobj = regexpQuote.match(key)
            if not matchobj is None:
                quote = matchobj.group(1)
                key = matchobj.group(2)

                # master defs dictionary
                defs[key] = val
                
                # C header file
                if quote == "q":
                    # Add double-quotes
                    cDefs.extend(list(createMacroStr(key, '"' + val + '"', 40, status)))
                elif quote == "p":
                    # Add parentheses
                    cDefs.extend(list(createMacroStr(key, '(' + val + ')', 40, status)))
                elif quote == "a":
                    # Leave as-is
                    cDefs.extend(list(createMacroStr(key, val, 40, status)))
                else:
                    # Unknown quote type
                    raise Exception('badQuoteQual', key)
                
                if status:
                    raise Exception('paramNameTooLong', key)
                
                # Make file - val should never be quoted; just use as is
                mDefs.extend(list('\n' + key + ' = ' + val))
                
                # Perl file - val should ALWAYS be single-quote quoted
                # Save const info to a string
                perlConstSection.extend(list(createPerlConst(key, "'" + val + "'", 40, status)))
                
                if status:
                    raise Exception('paramNameTooLong', key)
                
                # Save initialization information as a string. Now that we've defined
                # constants (the names of which are the parameter names) 
                # we can refer to those in the init section. The key variable holds the
                # name of the constant.
                perlInitSection.extend(list('\n  $self->{_paramsH}->{' + key + '} = ' + "'" + val + "';"))
            else:
                # No quote qualifier
                raise Exception('missingQuoteQual', key)
    elif ''.join(section) == 'make' and cfgfile:
        # Configure the remaining make variables defined in the __MAKE__ section of the configuration file. Third-party
        # library make variables are specified in the __MAKE__ section.
        matchobj = regexp.match(line)
        if not matchobj is None:
            # We have a key-value line
            key = matchobj.group(1)
            val = matchobj.group(2)

            # This information is for making make variables only. We do not need to worry about quoting any values
            defs[key] = val
            processMakeParam(mDefs, key, val, platDict, machDict)
    
    return bool(0)

# We have some extraneous line or a newline - ignore.

# defs is a dictionary containing all parameters (should they be needed in this script)
def parseConfig(fin, keymap, addenda, defs, cDefs, mDefs, perlConstSection, perlInitSection):
    rv = bool(0)
    
    # Open required config file (config.local)
    try:
        # Examine each line, looking for key=value pairs.
        regexpDefs = re.compile(r"^__DEFS__")
        regexpMake = re.compile(r"^__MAKE__")
        regexpComm = re.compile(r"^\s*#")
        regexpQuote = re.compile(r"^\s*(\w):(.+)")
        regexp = re.compile(r"^\s*(\S+)\s+(\S+)")
        
        platDict = {}
        machDict = {}
        section = list()
        
        # Process the parameters in the configuration file
        iscfg = bool(1)
        if not fin is None:
            for line in fin:
                ppRet = processParam(iscfg, line, regexpComm, regexpDefs, regexpMake, regexpQuote, regexp, keymap, defs, cDefs, mDefs, perlConstSection, perlInitSection, platDict, machDict, section)
                if ppRet:
                    break;
            
        # Process addenda - these are parameters that are not configurable and must be set in the 
        # NetDRMS build.
        iscfg = bool(0)
        for key in addenda:
            item = key + ' ' + addenda[key]
            ppRet = processParam(iscfg, item, regexpComm, regexpDefs, regexpMake, regexpQuote, regexp, keymap, defs, cDefs, mDefs, perlConstSection, perlInitSection, platDict, machDict, section)
            if ppRet:
                break;
    except Exception as exc:
        msg, violator = exc.args
        if msg == 'badKeyMapKey':
            # If we are here, then there was a non-empty keymap, and the parameter came from
            # the configuration file.
            print('Unknown parameter name ' + "'" + violator + "'" + ' in ' + cfgfile + '.', file=sys.stderr)
            rv = bool(1)
        elif msg == 'badQuoteQual':
            # The bad quote qualifier came from the configuration file, not the addenda, since
            # we will have fixed any bad qualifiers in the addenda (which is populated by code).
            print('Unknown quote qualifier ' + "'" + violator + "'" + ' in ' + cfgfile + '.', file=sys.stderr)
            rv = bool(1)
        elif msg == 'missingQuoteQual':
            print('Missing quote qualifier for parameter ' + "'" + violator + "'" + ' in ' + cfgfile + '.', file=sys.stderr)
            rv = bool(1)
        elif msg == 'paramNameTooLong':
            print('Macro name ' + "'" + violator + "' is too long.", file=sys.stderr)
            rv = bool(1)
        else:
            # re-raise the exception
            raise

    # Put information collected in platDict and machDict into mDefs
    if not rv:
        for plat in platDict:
            mDefs.extend(list('\nifeq ($(JSOC_MACHINE), linux_' + plat.lower() + ')'))
            for var in platDict[plat]:
                mDefs.extend(list('\n' + var + ' = ' + platDict[plat][var]))
            mDefs.extend(list('\nendif\n'))
                             
    if not rv:
        for mach in machDict:
            mDefs.extend(list('\nifeq ($(MACHTYPE), ' + mach + ')'))
            for var in platDict[plat]:
                mDefs.extend(list('\n' + var + ' = ' + platDict[plat][var]))
            mDefs.extend(list('\nendif\n'))
                             
    return rv

def getMgrUIDLine(defs, uidParam):
    rv = bool(0)
    
    cmd = 'id -u ' + defs['SUMS_MANAGER']
    try:
        ret = check_output(cmd, shell=True)
        uidParam['q:SUMS_MANAGER_UID'] = ret.decode("utf-8")
    except ValueError:
        print('Unable to run cmd: ' + cmd + '.')
        rv = bool(1)
    except CalledProcessError:
        print('Command ' + "'" + cmd + "'" + ' ran improperly.')
        rv = bool(1)

    return rv

def isVersion(maj, min, majDef, minDef):
    res = 0
    
    if maj > majDef or (maj == majDef and min >= minDef):
        res = 1
    
    return res

def configureComps(defs, mDefs):
    rv = bool(0)
    autoConfig = (not defs['AUTOSELCOMP'] == '0')
    
    if autoConfig:
        hasicc = bool(0)
        hasgcc = bool(0)
        hasifort = bool(0)
        hasgfort = bool(0)
        
        # Try icc.
        cmd = 'icc -V 2>&1'
        try:
            ret = check_output(cmd, shell=True)
            ret = ret.decode("utf-8")
        except CalledProcessError:
            print('Command ' + "'" + cmd + "'" + ' ran improperly.')
            rv = bool(1)
        
        if rv == bool(0):
            regexp = re.compile(r".+Version\s+(\d+)[.](\d+)", re.DOTALL)
            matchobj = regexp.match(ret)
            if matchobj is None:
                raise Exception('unexpectedIccRet', ret)
            else:
                major = matchobj.group(1)
                minor = matchobj.group(2)
                if isVersion(int(major), int(minor), ICC_MAJOR, ICC_MINOR):
                    hasicc = bool(1)
            
        # Try gcc.
        if not hasicc:
            cmd = 'gcc -v 2>&1'
            try:
                ret = check_output(cmd, shell=True)
                ret = ret.decode("utf-8")
            except CalledProcessError:
                print('Command ' + "'" + cmd + "'" + ' ran improperly.')
                rv = bool(1)

            if rv == bool(0):
                regexp = re.compile(r".+gcc\s+version\s+(\d+)\.(\d+)", re.DOTALL)
                matchobj = regexp.match(ret)
                if matchobj is None:
                    raise Exception('unexpectedGccRet', ret)
                else:
                    major = matchobj.group(1)
                    minor = matchobj.group(2)
                    if isVersion(int(major), int(minor), GCC_MAJOR, GCC_MINOR):
                        hasgcc = bool(1)

        # Try ifort.
        cmd = 'ifort -V 2>&1'
        try:
            ret = check_output(cmd, shell=True)
            ret = ret.decode("utf-8")
        except CalledProcessError:
            print('Command ' + "'" + cmd + "'" + ' ran improperly.')
            rv = bool(1)

        if rv == bool(0):
            regexp = re.compile(r".+Version\s+(\d+)\.(\d+)", re.DOTALL)
            matchobj = regexp.match(ret)
            if matchobj is None:
                raise Exception('unexpectedIfortRet', ret)
            else:
                major = matchobj.group(1)
                minor = matchobj.group(2)
                if isVersion(int(major), int(minor), IFORT_MAJOR, IFORT_MINOR):
                    hasifort = bool(1)
        
        # Try gfortran
        if not hasifort:
            cmd = 'gfortran -v 2>&1'
            try:
                ret = check_output(cmd, shell=True)
                ret = ret.decode("utf-8")
            except CalledProcessError:
                print('Command ' + "'" + cmd + "'" + ' ran improperly.')
                rv = bool(1)

            if rv == bool(0):
                regexp = re.compile(r".+gcc\s+version\s+(\d+)\.(\d+)", re.DOTALL)
                matchobj = regexp.match(ret)
                if matchobj is None:
                    raise Exception('unexpectedGfortranRet', ret)
                else:
                    major = matchobj.group(1)
                    minor = matchobj.group(2)
                    if isVersion(int(major), int(minor), GFORT_MAJOR, GFORT_MINOR):
                        hasgfort = bool(1)
        
        # Append the compiler make variables to the make file 
        if not hasicc and not hasgcc:
            print('Fatal error: Acceptable C compiler not found! You will be unable to build the DRMS library.', file=sys.stderr)
            rv = bool(1)
        elif hasicc:
            mDefs.extend(list('\nCOMPILER = icc'))
        else:
            mDefs.extend(list('\nCOMPILER = gcc'))

        if not hasifort and not hasgfort:
            print('Warning: Acceptable Fortran compiler not found! Fortran interface will not be built, and you will be unable to build Fortran modules.', file=sys.stderr)
        elif hasifort:
            mDefs.extend(list('\nFCOMPILER = ifort'))
        else:
            mDefs.extend(list('\nFCOMPILER = gfortran'))
    
        # Environment overrides. These get written, regardless of the disposition of auto-configuration.
        mDefs.extend(list('\nifneq $(JSOC_COMPILER,)\n  COMPILER = $(JSOC_COMPILER)\nendif'))
        mDefs.extend(list('\nifneq $(JSOC_FCOMPILER,)\n  FCOMPILER = $(JSOC_FCOMPILER)\nendif'))

    return rv

def writeFiles(base, cfile, mfile, pfile, cDefs, mDefs, perlConstSection, perlInitSection):
    rv = bool(0)
    
    try:
        with open(cfile, 'w') as cout, open(mfile, 'w') as mout, open(pfile, 'w') as pout:
            # C file of macros
            print(C_PREFIX, file=cout)
            print('/* This file contains a set of preprocessor macros - one for each configuration parameter. */\n', file=cout)
            buf = '__' + base.upper() + '_H'
            print('#ifndef ' + buf, file=cout)
            print('#define ' + buf, file=cout)
            print(''.join(cDefs), file=cout)
            print('#endif', file=cout)
            
            # Make file of make variables
            print(PREFIX, file=mout)
            print('# This file contains a set of make-variable values - one for each configuration parameter.', file=mout)
            print(''.join(mDefs), file=mout)
            
            # Perl module
            print(PERL_BINPATH, file=pout)
            print(PREFIX, file=pout)
            print('# This file contains a set of constants - one for each configuration parameter.\n', file=pout)
            print(PERL_INTIAL, file=pout)
            print(''.join(perlConstSection), file=pout)
            print(PERL_FXNS_A, file=pout)
            print('sub initialize', file=pout)
            print('{', file=pout)
            print('  my($self) = shift;', file=pout, end='')
            print('', file=pout)
            print(''.join(perlInitSection), file=pout)
            print('}\n', file=pout)
            print(PERL_FXNS_B, file=pout)
    except IOError as exc:
        sys.stderr.write(exc.strerror)
        sys.stderr.write('Unable to open a parameter vile.')
        rv = bool(1)

    return rv

def configureNet(cfgfile, cfile, mfile, pfile, base, keymap):
    rv = bool(0)
    
    defs = {}
    cDefs = list()
    mDefs = list()
    perlConstSection = list()
    perlInitSection = list()
    addenda = {}
    
    # There are three parameters that are not configurable and must be set.
    addenda['a:USER'] = 'NULL'
    addenda['a:PASSWD'] = 'NULL'
    addenda['p:DSDS_SUPPORT'] = '0'
    addenda['a:BUILD_TYPE'] = 'NETDRMS' # Means a non-Stanford build. This will set two additional macros used by make:
                                        #   __LOCALIZED_DEFS__ and NETDRMS_BUILD. The former is to support legacy code
                                        #   which incorrectly used this macro, and the latter is for future use. 
                                        #   __LOCALIZED_DEFS__ is deprecated and should not be used in new code.

    try:
        with open(cfgfile, 'r') as fin:
            # Process configuration parameters
            rv = parseConfig(fin, keymap, addenda, defs, cDefs, mDefs, perlConstSection, perlInitSection)
            if rv == bool(0):
                # Must add a parameter for the SUMS_MANAGER UID (for some reason). This must be done after the
                # config file is processed since an input to getMgrUIDLine() is one of the config file's
                # parameter values.
                uidParam = {}
                rv = getMgrUIDLine(defs, uidParam)
                if rv == bool(0):
                    rv = parseConfig(None, keymap, uidParam, defs, cDefs, mDefs, perlConstSection, perlInitSection)
                
            # Configure the compiler-selection make variables.
            if rv == bool(0):
                rv = configureComps(defs, mDefs)

            # Write out the parameter files.
            if rv == bool(0):
                rv = writeFiles(base, cfile, mfile, pfile, cDefs, mDefs, perlConstSection, perlInitSection)
    except IOError as exc:
        sys.stderr.write(exc.strerror)
        sys.stderr.write('Unable to read configuration file ' + cfgfile + '.')
    except Exception as exc:
        type, msg = exc.args
        if type == 'unexpectedIccRet':
            print('icc -V returned this unexpected message:\n' + msg, file=sys.stderr)
            rv = bool(1)
        elif type == 'unexpectedGccRet':
            print('gcc -v returned this unexpected message:\n' + msg, file=sys.stderr)
            rv = bool(1)
        elif type == 'unexpectedIfortRet':
            print('ifort -V returned this unexpected message:\n' + msg, file=sys.stderr)
            rv = bool(1)
        elif type == 'unexpectedGfortranRet':
            print('gfortran -v returned this unexpected message:\n' + msg, file=sys.stderr)
            rv = bool(1)
        else:
            # re-raise the exception
            raise
        
    return rv

def configureSdp(cfgfile, cfile, mfile, pfile, base, keymap):
    rv = bool(0)
    
    defs = {}
    cDefs = list()
    mDefs = list()
    perlConstSection = list()
    perlInitSection = list()
    addenda = {}
    
    addenda['a:BUILD_TYPE'] = 'JSOC_SDP'
    
    try:
        with open(cfgfile, 'r') as fin:
            rv = parseConfig(cfgfile, keymap, addenda, defs, cDefs, mDefs, perlConstSection, perlInitSection)
            if rv == bool(0):
                # Must add a parameter for the SUMS_MANAGER UID (for some reason)
                uidParam = {}
                rv = getMgrUIDLine(defs, uidParam)
                if rv == bool(0):
                    rv = parseConfig(None, keymap, uidParam, defs, cDefs, mDefs, perlConstSection, perlInitSection)
                
                
                # So, we need to rename __MAKE__ in configsdp.txt to something else (__PROJ_MK_RULES__).
                # The only code that currently reads this file is configproj.pl, so we change that logic to use __PROJ_MK_RULES__
                # for the make rules. 
                if rv == bool(0):
                    rv = writeFiles(base, cfile, mfile, pfile, cDefs, mDefs, perlConstSection, perlInitSection)
    except IOError as exc:
        sys.stderr.write(exc.strerror)
        sys.stderr.write('Unable to read configuration file ' + cfgfile + '.')

    return rv

# Beginning of program
rv = RET_SUCCESS
net = bool(1)

# Parse arguments
if __name__ == "__main__":
    optD = GetArgs(sys.argv[1:])

if not(optD is None):
    # Ensure we are configuring a DRMS tree
    cdir = os.path.realpath(os.getcwd())
    versfile = cdir + '/base/' + VERS_FILE

    if not os.path.isfile(versfile):
        rv = RET_NOTDRMS

# Determine whether we are localizing a Stanford build, or a NetDRMS build. If configsdp.txt exists, then
# it is a Stanford build, otherwise it is a NetDRMS build.
if rv == RET_SUCCESS:
    stanfordFile = cdir + '/' + SDP_CFG
    if os.path.isfile(stanfordFile):
        net = bool(0)
    
    cfile = optD['dir'] + '/' + optD['base'] + '.h'
    mfile = optD['dir'] + '/' + optD['base'] + '.mk'
    pfile = optD['dir'] + '/' + optD['base'] + '.pm'

    if net:
        try:
            with open(NET_CFGMAP, 'r') as fin:
                regexpComm = re.compile(r"^\s*#")
                regexp = re.compile(r"^\s*(\S+)\s+(\w:\S+)")
                # Must map from config.local namespace to DRMS namespace (e.g., the names used for the C macros)
                keymap = {}
                for line in fin:
                    matchobj = regexpComm.match(line)
                    if not matchobj is None:
                        # Skip comment line
                        continue

                    matchobj = regexp.match(line)
                    if not(matchobj is None):
                        # We have a key-value line
                        key = matchobj.group(1)
                        val = matchobj.group(2)
                        keymap[key] = val
        except OSError:
            sys.stderr.write('Unable to read configuration map-file ' + NET_CFGMAP + '.')
            rv = bool(1)
            
        

        # We also need to set the UID of the SUMS manager. We have the name of the
        # SUMS manager (it is in the configuration file)
        configureNet(NET_CFG, cfile, mfile, pfile, optD['base'], keymap)
    else:
        configureSdp(SDP_CFG, cfile, mfile, pfile, optD['base'], {})