| # Gaudi Framework, version v21r10p1 |

Home |  | Generated: 29 Jul 2010

# ProcessJobOptions.py

Go to the documentation of this file.

    00001 import os, sys, re
    00002
    00003 import logging
    00004 _log = logging.getLogger(__name__)
    00005
    00006 class LogFormatter(logging.Formatter):
    00007     def __init__(self, fmt=None, datefmt=None, prefix = "# "):
    00008         logging.Formatter.__init__(self, fmt, datefmt)
    00009         self.prefix = prefix
    00010     def format(self, record):
    00011         fmsg = logging.Formatter.format(self, record)
    00012         prefix = self.prefix
    00013         if record.levelno >= logging.WARNING:
    00014             prefix += record.levelname + ": "
    00015         s = "\n".join([ prefix + line
    00016                         for line in fmsg.splitlines() ])
    00017         return s
    00018
    00019 class LogFilter(logging.Filter):
    00020     def __init__(self, name = ""):
    00021         logging.Filter.__init__(self, name)
    00022         self.printing_level = 0
    00023         self.enabled = True
    00024         self.threshold = logging.WARNING
    00025     def filter(self, record):
    00026         return record.levelno >= self.threshold or (self.enabled and self.printing_level <= 0)
    00027     def printOn(self, step = 1, force = False):
    00028         """
    00029         Decrease the printing_level of 'step' units. ( >0 means no print)
    00030         The level cannot go below 0, unless the force flag is set to True.
    00031         A negative value of the threshold disables subsequent "PrintOff"s.
    00032         """
    00033         if force:
    00034             self.printing_level -= step
    00035         else:
    00036             if self.printing_level > step:
    00037                 self.printing_level -= step
    00038             else:
    00039                 self.printing_level = 0
    00040     def printOff(self, step = 1):
    00041         """
    00042         Increase the printing_level of 'step' units. ( >0 means no print)
    00043         """
    00044         self.printing_level += step
    00045     def disable(self, allowed = logging.WARNING):
    00046         self.enabled = False
    00047         self.threshold = allowed
    00048     def enable(self, allowed = logging.WARNING):
    00049         self.enabled = True
    00050         self.threshold = allowed
    00051
    00052 class ConsoleHandler(logging.StreamHandler):
    00053     def __init__(self, strm = None, prefix = None):
    00054         if strm is None:
    00055             strm = sys.stdout
    00056         logging.StreamHandler.__init__(self, strm = strm)
    00057         if prefix is None:
    00058             prefix = "# "
    00059         self._filter = LogFilter(_log.name)
    00060         self._formatter = LogFormatter(prefix = prefix)
    00061         self.setFormatter(self._formatter)
    00062         self.addFilter(self._filter)
    00063     def setPrefix(self, prefix):
    00064         self._formatter.prefix = prefix
    00065     def printOn(self, step = 1, force = False):
    00066         """
    00067         Decrease the printing_level of 'step' units. ( >0 means no print)
    00068         The level cannot go below 0, unless the force flag is set to True.
    00069         A negative value of the threshold disables subsequent "PrintOff"s.
    00070         """
    00071         self._filter.printOn(step, force)
    00072     def printOff(self, step = 1):
    00073         """
    00074         Increase the printing_level of 'step' units. ( >0 means no print)
    00075         """
    00076         self._filter.printOff(step)
    00077     def disable(self, allowed = logging.WARNING):
    00078         self._filter.disable(allowed)
    00079     def enable(self, allowed = logging.WARNING):
    00080         self._filter.enable(allowed)
    00081
    00082 _consoleHandler = None
    00083 def GetConsoleHandler(prefix = None, strm = None):
    00084     global _consoleHandler
    00085     if _consoleHandler is None:
    00086         _consoleHandler = ConsoleHandler(prefix = prefix, strm = strm)
    00087     elif prefix is not None:
    00088         _consoleHandler.setPrefix(prefix)
    00089     return _consoleHandler
    00090
    00091 def InstallRootLoggingHandler(prefix = None, level = None, strm = None):
    00092     root_logger = logging.getLogger()
    00093     if not root_logger.handlers:
    00094         root_logger.addHandler(GetConsoleHandler(prefix, strm))
    00095         root_logger.setLevel(logging.WARNING)
    00096     if level is not None:
    00097         root_logger.setLevel(level)
    00098
    00099 def PrintOn(step = 1, force = False):
    00100     GetConsoleHandler().printOn(step, force)
    00101 def PrintOff(step = 1):
    00102     GetConsoleHandler().printOff(step)
    00103
    00104 class ParserError(RuntimeError):
    00105     pass
    00106
    00107 def _find_file(f):
    00108     # expand environment variables in the filename
    00109     f = os.path.expandvars(f)
    00110     if os.path.isfile(f):
    00111         return os.path.realpath(f)
    00112
    00113     path = os.environ.get('JOBOPTSEARCHPATH','').split(os.pathsep)
    00114     # find the full path to the option file
    00115     candidates = [d for d in path if os.path.isfile(os.path.join(d,f))]
    00116     if not candidates:
    00117         raise ParserError("Cannot find '%s' in %s" % (f,path))
    00118     return os.path.realpath(os.path.join(candidates[0],f))
    00119
    00120 _included_files = set()
    00121 def _to_be_included(f):
    00122     if f in _included_files:
    00123         _log.warning("file '%s' already included, ignored.", f)
    00124         return False
    00125     _included_files.add(f)
    00126     return True
    00127
    00128 class JobOptsParser:
    00129     comment = re.compile(r'(//.*)$')
    00130     # non-perfect R-E to check if '//' is inside a string
    00131     # (a tokenizer would be better)
    00132     comment_in_string = re.compile(r'(["\']).*//.*\1')
    00133     directive = re.compile(r'^\s*#\s*([\w!]+)\s*(.*)\s*$')
    00134     comment_ml = ( re.compile(r'/\*'), re.compile(r'\*/') )
    00135     statement_sep = ";"
    00136     reference = re.compile(r'^@([\w.]*)$')
    00137
    00138     def __init__(self):
    00139         # parser level states
    00140         self.units = {}
    00141         self.defines = {}
    00142         if sys.platform != 'win32':
    00143             self.defines[ "WIN32" ] = True
    00144
    00145     def _include(self,file,function):
    00146         file = _find_file(file)
    00147         if _to_be_included(file):
    00148             _log.info("--> Including file '%s'", file)
    00149             function(file)
    00150             _log.info("<-- End of file '%s'", file)
    00151
    00152     def parse(self,file):
    00153         # states for the "translation unit"
    00154         statement = ""
    00155
    00156         ifdef_level = 0
    00157         ifdef_skipping = False
    00158         ifdef_skipping_level = 0
    00159
    00160         f = open(_find_file(file))
    00161         l = f.readline()
    00162         if l.startswith("#!"):
    00163             # Skip the first line if it starts with "#!".
    00164             # It allows to use options files as scripts.
    00165             l = f.readline()
    00166
    00167         while l:
    00168             l = l.rstrip()+'\n' # normalize EOL chars (to avoid problems with DOS new-line on Unix)
    00169
    00170             # single line comment
    00171             m = self.comment.search(l)
    00172             if m:
    00173                 # check if the '//' is part of a string
    00174                 m2 = self.comment_in_string.search(l)
    00175                 # the '//' is part of a string if we find the quotes around it
    00176                 # and they are not part of the comment itself
    00177                 if not ( m2 and m2.start() < m.start() ):
    00178                     # if it is not the case, we can remove the comment from the
    00179                     # statement
    00180                     l = l[:m.start()]+l[m.end():]
    00181             # process directives
    00182             m = self.directive.search(l)
    00183             if m:
    00184                 directive_name = m.group(1)
    00185                 directive_arg = m.group(2).strip()
    00186                 if directive_name == "include":
    00187                     included_file = directive_arg.strip("'\"")
    00188                     importOptions(included_file)
    00189                 elif directive_name == "units":
    00190                     units_file = directive_arg.strip("'\"")
    00191                     self._include(units_file,self._parse_units)
    00192                 elif directive_name in [ "ifdef", "ifndef"]:
    00193                     ifdef_skipping_level = ifdef_level
    00194                     ifdef_level += 1
    00195                     if directive_arg in self.defines:
    00196                         ifdef_skipping = directive_name == "ifndef"
    00197                     else:
    00198                         ifdef_skipping = directive_name == "ifdef"
    00199                 elif directive_name == "else":
    00200                     ifdef_skipping = not ifdef_skipping
    00201                 elif directive_name == "endif":
    00202                     ifdef_level -= 1
    00203                     if ifdef_skipping and ifdef_skipping_level == ifdef_level:
    00204                         ifdef_skipping = False
    00205                 elif directive_name == "pragma":
    00206                     if not directive_arg:
    00207                         l = f.readline()
    00208                         continue
    00209                     pragma = directive_arg.split()
    00210                     if pragma[0] == "print":
    00211                         if len(pragma) > 1:
    00212                             if pragma[1].upper() in [ "ON", "TRUE", "1" ]:
    00213                                 PrintOn()
    00214                             else:
    00215                                 PrintOff()
    00216                 else:
    00217                     _log.warning("unknown directive '%s'", directive_name)
    00218                 l = f.readline()
    00219                 continue
    00220
    00221             if ifdef_skipping:
    00222                 l = f.readline()
    00223                 continue
    00224
    00225             # multi-line comment
    00226             m = self.comment_ml[0].search(l)
    00227             if m:
    00228                 l,l1 = l[:m.start()],l[m.end():]
    00229                 m = self.comment_ml[1].search(l1)
    00230                 while not m:
    00231                     l1 = f.readline()
    00232                     if not l1:
    00233                         break # EOF
    00234                     m = self.comment_ml[1].search(l1)
    00235                 if not l1 and not m:
    00236                     raise ParserError("End Of File reached before end of multi-line comment")
    00237                 l += l1[m.end():]
    00238
    00239             if self.statement_sep in l:
    00240                 i = l.index(self.statement_sep)
    00241                 statement += l[:i]
    00242                 self._eval_statement(statement.replace("\n","").strip())
    00243                 statement = l[i+1:]
    00244                 # it may happen (bug #37479) that the rest of the statement
    00245                 # contains a comment.
    00246                 if statement.lstrip().startswith("//"):
    00247                     statement = ""
    00248             else:
    00249                 statement += l
    00250
    00251             l = f.readline()
    00252
    00253     def _parse_units(self,file):
    00254         for line in open(file):
    00255             if '//' in line:
    00256                 line = line[:line.index('//')]
    00257             line = line.strip()
    00258             if not line:
    00259                 continue
    00260             nunit, value = line.split('=')
    00261             factor, unit = nunit.split()
    00262             value = eval(value)/eval(factor)
    00263             self.units[unit] = value
    00264
    00265     def _eval_statement(self,statement):
    00266         from GaudiKernel.Proxy.Configurable import (ConfigurableGeneric,
    00267                                                     Configurable,
    00268                                                     PropertyReference)
    00269         #statement = statement.replace("\n","").strip()
    00270         _log.info("%s%s", statement, self.statement_sep)
    00271
    00272         property,value = statement.split("=",1)
    00273
    00274         inc = None
    00275         if property[-1] in [ "+", "-" ]:
    00276             inc = property[-1]
    00277             property = property[:-1]
    00278
    00279         property = property.strip()
    00280         value = value.strip()
    00281
    00282         ## find the configurable to apply the property to
    00283         #parent_cfg = None
    00284         #while '.' in property:
    00285         #    component, property = property.split('.',1)
    00286         #    if parent_cfg:
    00287         #        if hasattr(parent_cfg,component):
    00288         #            cfg = getattr(parent_cfg,component)
    00289         #        else:
    00290         #            cfg = ConfigurableGeneric(component)
    00291         #            setattr(parent_cfg,component,cfg)
    00292         #    else:
    00293         #        cfg = ConfigurableGeneric(component)
    00294         #    parent_cfg = cfg
    00295
    00296         # remove spaces around dots
    00297         property = '.'.join([w.strip() for w in property.split('.')])
    00298         component, property = property.rsplit('.',1)
    00299         if component in Configurable.allConfigurables:
    00300             cfg = Configurable.allConfigurables[component]
    00301         else:
    00302             cfg = ConfigurableGeneric(component)
    00303
    00304         #value = os.path.expandvars(value)
    00305         value = value.replace('true','True').replace('false','False')
    00306         if value[0] == '{' :
    00307             # Try to guess if the values looks like a dictionary
    00308             if ':' in value and not ( value[:value.index(':')].count('"')%2 or value[:value.index(':')].count("'")%2 ) :
    00309                 # for dictionaries, keep the surrounding {}
    00310                 value = '{'+value[1:-1].replace('{','[').replace('}',']')+'}'
    00311             else : # otherwise replace all {} with []
    00312                 value = value.replace('{','[').replace('}',']')
    00313
    00314         # We must escape '\' because eval tends to interpret them
    00315         value = value.replace('\\','\\\\')
    00316         # Replace literal '\n' and '\t' with spaces (bug #47258)
    00317         value = value.replace("\\n", " ").replace("\\t", " ")
    00318
    00319         # interprete the @ operator
    00320         m = self.reference.match(value)
    00321         if m:
    00322             # this allows late binding of references
    00323             value = PropertyReference(m.group(1))
    00324         else:
    00325             value = eval(value,self.units)
    00326
    00327         #if type(value) is str    : value = os.path.expandvars(value)
    00328         #elif type(value) is list : value = [ type(item) is str and os.path.expandvars(item) or item for item in value ]
    00329
    00330         if property not in cfg.__slots__ and not hasattr(cfg,property):
    00331             # check if the case of the property is wrong (old options are case insensitive)
    00332             lprop = property.lower()
    00333             for p in cfg.__slots__:
    00334                 if lprop == p.lower():
    00335                     _log.warning("property '%s' was requested for %s, but the correct spelling is '%s'", property, cfg.name(), p)
    00336                     property = p
    00337                     break
    00338
    00339         # consider the += and -=
    00340         if inc == "+":
    00341             if hasattr(cfg,property):
    00342                 prop = getattr(cfg,property)
    00343                 if type(prop) == dict:
    00344                     for k in value:
    00345                         prop[k] = value[k]
    00346                 else:
    00347                     prop += value
    00348             else:
    00349                 setattr(cfg,property,value)
    00350         elif inc == "-":
    00351             if hasattr(cfg,property):
    00352                 prop = getattr(cfg,property)
    00353                 if type(prop) is dict:
    00354                     for k in value:
    00355                         if k in prop:
    00356                             del prop[k]
    00357                         else:
    00358                             _log.warning("key '%s' not in %s.%s", k, cfg.name(), property)
    00359                 else:
    00360                     for k in value:
    00361                         if k in prop:
    00362                             prop.remove(k)
    00363                         else:
    00364                             _log.warning("value '%s' not in %s.%s", k, cfg.name(), property)
    00365         else:
    00366             setattr(cfg,property,value)
    00367
    00368 class _TempSysPath:
    00369     def __init__(self, new_path):
    00370         self.old_path = sys.path
    00371         sys.path = new_path
    00372     def __del__(self):
    00373         sys.path = self.old_path
    00374
    00375 _parser = JobOptsParser()
    00376
    00377 def _import_python(file):
    00378     execfile(file, {})
    00379
    00380 def _import_pickle(file):
    00381     import pickle
    00382     input = open(file, 'rb')
    00383     catalog = pickle.load(input)
    00384     _log.info('Unpickled %d configurables', len(catalog))
    00385
    00386 def _import_opts(file):
    00387     _parser.parse(file)
    00388
    00389 _import_function_mapping = {
    00390                              ".py"   : _import_python,
    00391                              ".pkl"  : _import_pickle,
    00392                              ".opts" : _import_opts,
    00393                             }
    00394
    00395 def importOptions( optsfile ) :
    00396     # expand environment variables before checking the extension
    00397     optsfile = os.path.expandvars(optsfile)
    00398     # check the file type (extension)
    00399     dummy, ext = os.path.splitext(optsfile)
    00400     if ext in _import_function_mapping:
    00401         # check if the file has been already included
    00402         optsfile = _find_file(optsfile)
    00403         if _to_be_included(optsfile):
    00404             _log.info("--> Including file '%s'", optsfile)
    00405             # include the file
    00406             _import_function_mapping[ext](optsfile)
    00407             _log.info("<-- End of file '%s'", optsfile)
    00408     else:
    00409         raise ParserError("Unknown file type '%s' ('%s')" % (ext,optsfile))
    00410
    00411 ## Import a file containing declaration of units.
    00412 #  It is equivalent to:
    00413 #
    00414 #  #units "unitsfile.opts"
    00415 #
    00416 def importUnits(unitsfile):
    00417     # expand environment variables
    00418     unitsfile = os.path.expandvars(unitsfile)
    00419     # we do not need to check the file type (extension) because it must be a
    00420     # units file
    00421     _parser._include(unitsfile, _parser._parse_units)