| The Gaudi Framework v28r2-patches (bc4ad17a)

- Main Page
- Related Pages
- Modules
- Namespaces
- Classes
- Files

- File List
- File Members

ProcessJobOptions.py
Go to the documentation of this file.
1 import os, sys, re
2 import time
3
4 import logging
5 _log = logging.getLogger(__name__)
6
7 class LogFormatter(logging.Formatter):
8 def __init__(self, fmt=None, datefmt=None, prefix = "# ", with_time = False):
9 logging.Formatter.__init__(self, fmt, datefmt)
10 self.prefix = prefix
11 self.with_time = with_time
12 def format(self, record):
13 fmsg = logging.Formatter.format(self, record)
14 prefix = self.prefix
15 if self.with_time:
16 prefix += '%f ' % time.time()
17 if record.levelno >= logging.WARNING:
18 prefix += record.levelname + ": "
19 s = "\n".join([ prefix + line
20 for line in fmsg.splitlines() ])
21 return s
22
23 class LogFilter(logging.Filter):
24 def __init__(self, name = ""):
25 logging.Filter.__init__(self, name)
26 self.printing_level = 0
27 self.enabled = True
28 self.threshold = logging.WARNING
29 def filter(self, record):
30 return record.levelno >= self.threshold or (self.enabled and self.printing_level <= 0)
31 def printOn(self, step = 1, force = False):
32 """
33 Decrease the printing_level of 'step' units. ( >0 means no print)
34 The level cannot go below 0, unless the force flag is set to True.
35 A negative value of the threshold disables subsequent "PrintOff"s.
36 """
37 if force:
38 self.printing_level -= step
39 else:
40 if self.printing_level > step:
41 self.printing_level -= step
42 else:
43 self.printing_level = 0
44 def printOff(self, step = 1):
45 """
46 Increase the printing_level of 'step' units. ( >0 means no print)
47 """
48 self.printing_level += step
49 def disable(self, allowed = logging.WARNING):
50 self.enabled = False
51 self.threshold = allowed
52 def enable(self, allowed = logging.WARNING):
53 self.enabled = True
54 self.threshold = allowed
55
56 class ConsoleHandler(logging.StreamHandler):
57 def __init__(self, stream = None, prefix = None, with_time = False):
58 if stream is None:
59 stream = sys.stdout
60 logging.StreamHandler.__init__(self, stream)
61 if prefix is None:
62 prefix = "# "
63 self._filter = LogFilter(_log.name)
64 self._formatter = LogFormatter(prefix = prefix, with_time = with_time)
65 self.setFormatter(self._formatter)
66 self.addFilter(self._filter)
67 def setPrefix(self, prefix):
68 self._formatter.prefix = prefix
69 def printOn(self, step = 1, force = False):
70 """
71 Decrease the printing_level of 'step' units. ( >0 means no print)
72 The level cannot go below 0, unless the force flag is set to True.
73 A negative value of the threshold disables subsequent "PrintOff"s.
74 """
75 self._filter.printOn(step, force)
76 def printOff(self, step = 1):
77 """
78 Increase the printing_level of 'step' units. ( >0 means no print)
79 """
80 self._filter.printOff(step)
81 def disable(self, allowed = logging.WARNING):
82 self._filter.disable(allowed)
83 def enable(self, allowed = logging.WARNING):
84 self._filter.enable(allowed)
85
86 _consoleHandler = None
87 def GetConsoleHandler(prefix = None, stream = None, with_time = False):
88 global _consoleHandler
89 if _consoleHandler is None:
90 _consoleHandler = ConsoleHandler(prefix = prefix, stream = stream, with_time = with_time)
91 elif prefix is not None:
92 _consoleHandler.setPrefix(prefix)
93 return _consoleHandler
94
95 def InstallRootLoggingHandler(prefix = None, level = None, stream = None, with_time = False):
96 root_logger = logging.getLogger()
97 if not root_logger.handlers:
98 root_logger.addHandler(GetConsoleHandler(prefix, stream, with_time))
99 root_logger.setLevel(logging.WARNING)
100 if level is not None:
101 root_logger.setLevel(level)
102
103 def PrintOn(step = 1, force = False):
104 GetConsoleHandler().printOn(step, force)
105 def PrintOff(step = 1):
106 GetConsoleHandler().printOff(step)
107
108 class ParserError(RuntimeError):
109 pass
110
111 def _find_file(f):
112 # expand environment variables in the filename
113 f = os.path.expandvars(f)
114 if os.path.isfile(f):
115 return os.path.realpath(f)
116
117 path = os.environ.get('JOBOPTSEARCHPATH','').split(os.pathsep)
118 # find the full path to the option file
119 candidates = [d for d in path if os.path.isfile(os.path.join(d,f))]
120 if not candidates:
121 raise ParserError("Cannot find '%s' in %s" % (f,path))
122 return os.path.realpath(os.path.join(candidates[0],f))
123
124 _included_files = set()
125 def _to_be_included(f):
126 if f in _included_files:
127 _log.warning("file '%s' already included, ignored.", f)
128 return False
129 _included_files.add(f)
130 return True
131
132 class JobOptsParser:
133 comment = re.compile(r'(//.*)$')
134 # non-perfect R-E to check if '//' is inside a string
135 # (a tokenizer would be better)
136 comment_in_string = re.compile(r'(["\']).*//.*\1')
137 directive = re.compile(r'^\s*#\s*([\w!]+)\s*(.*)\s*$')
138 comment_ml = ( re.compile(r'/\*'), re.compile(r'\*/') )
139 statement_sep = ";"
140 reference = re.compile(r'^@([\w.]*)$')
141
142 def __init__(self):
143 # parser level states
144 self.units = {}
145 self.defines = {}
146 if sys.platform != 'win32':
147 self.defines[ "WIN32" ] = True
148
149 def _include(self,file,function):
150 file = _find_file(file)
151 if _to_be_included(file):
152 _log.info("--> Including file '%s'", file)
153 function(file)
154 _log.info("<-- End of file '%s'", file)
155
156 def parse(self,file):
157 # states for the "translation unit"
158 statement = ""
159
160 ifdef_level = 0
161 ifdef_skipping = False
162 ifdef_skipping_level = 0
163
164 in_string = False
165
166 f = open(_find_file(file))
167 l = f.readline()
168 if l.startswith("#!"):
169 # Skip the first line if it starts with "#!".
170 # It allows to use options files as scripts.
171 l = f.readline()
172
173 while l:
174 l = l.rstrip()+'\n' # normalize EOL chars (to avoid problems with DOS new-line on Unix)
175
176 # single line comment
177 m = self.comment.search(l)
178 if m:
179 # check if the '//' is part of a string
180 m2 = self.comment_in_string.search(l)
181 # the '//' is part of a string if we find the quotes around it
182 # and they are not part of the comment itself
183 if not ( m2 and m2.start() < m.start() ):
184 # if it is not the case, we can remove the comment from the
185 # statement
186 l = l[:m.start()]+l[m.end():]
187 # process directives
188 m = self.directive.search(l)
189 if m:
190 directive_name = m.group(1)
191 directive_arg = m.group(2).strip()
192 if directive_name == "include":
193 included_file = directive_arg.strip("'\"")
194 importOptions(included_file)
195 elif directive_name == "units":
196 units_file = directive_arg.strip("'\"")
197 self._include(units_file,self._parse_units)
198 elif directive_name in [ "ifdef", "ifndef"]:
199 ifdef_skipping_level = ifdef_level
200 ifdef_level += 1
201 if directive_arg in self.defines:
202 ifdef_skipping = directive_name == "ifndef"
203 else:
204 ifdef_skipping = directive_name == "ifdef"
205 elif directive_name == "else":
206 ifdef_skipping = not ifdef_skipping
207 elif directive_name == "endif":
208 ifdef_level -= 1
209 if ifdef_skipping and ifdef_skipping_level == ifdef_level:
210 ifdef_skipping = False
211 elif directive_name == "pragma":
212 if not directive_arg:
213 l = f.readline()
214 continue
215 pragma = directive_arg.split()
216 if pragma[0] == "print":
217 if len(pragma) > 1:
218 if pragma[1].upper() in [ "ON", "TRUE", "1" ]:
219 PrintOn()
220 else:
221 PrintOff()
222 else:
223 _log.warning("unknown directive '%s'", directive_name)
224 l = f.readline()
225 continue
226
227 if ifdef_skipping:
228 l = f.readline()
229 continue
230
231 # multi-line comment
232 m = self.comment_ml[0].search(l)
233 if m:
234 l,l1 = l[:m.start()],l[m.end():]
235 m = self.comment_ml[1].search(l1)
236 while not m:
237 l1 = f.readline()
238 if not l1:
239 break # EOF
240 m = self.comment_ml[1].search(l1)
241 if not l1 and not m:
242 raise ParserError("End Of File reached before end of multi-line comment")
243 l += l1[m.end():]
244
245 # if we are in a multiline string, we add to the statement
246 # everything until the next '"'
247 if in_string:
248 string_end = l.find('"')
249 if string_end >= 0:
250 statement += l[:string_end+1]
251 l = l[string_end+1:]
252 in_string = False # the string ends here
253 else:
254 statement += l
255 l = ''
256 else: # check if we have a string
257 string_start = l.find('"')
258 if string_start >= 0:
259 string_end = l.find('"', string_start + 1)
260 if string_end >= 0:
261 # the string is opened and closed
262 statement += l[:string_end+1]
263 l = l[string_end+1:]
264 else:
265 # the string is only opened
266 statement += l
267 in_string = True
268 l = f.readline()
269 continue
270
271 if self.statement_sep in l:
272 i = l.index(self.statement_sep)
273 statement += l[:i]
274 self._eval_statement(statement.strip().replace("\n","\\n"))
275 statement = l[i+1:]
276 # it may happen (bug #37479) that the rest of the statement
277 # contains a comment.
278 if statement.lstrip().startswith("//"):
279 statement = ""
280 else:
281 statement += l
282
283 l = f.readline()
284
285 def _parse_units(self,file):
286 for line in open(file):
287 if '//' in line:
288 line = line[:line.index('//')]
289 line = line.strip()
290 if not line:
291 continue
292 nunit, value = line.split('=')
293 factor, unit = nunit.split()
294 value = eval(value)/eval(factor)
295 self.units[unit] = value
296
297 def _eval_statement(self,statement):
298 from GaudiKernel.Proxy.Configurable import (ConfigurableGeneric,
299 Configurable,
300 PropertyReference)
301 #statement = statement.replace("\n","").strip()
302 _log.info("%s%s", statement, self.statement_sep)
303
304 property,value = statement.split("=",1)
305
306 inc = None
307 if property[-1] in [ "+", "-" ]:
308 inc = property[-1]
309 property = property[:-1]
310
311 property = property.strip()
312 value = value.strip()
313
314 ## find the configurable to apply the property to
315 #parent_cfg = None
316 #while '.' in property:
317 # component, property = property.split('.',1)
318 # if parent_cfg:
319 # if hasattr(parent_cfg,component):
320 # cfg = getattr(parent_cfg,component)
321 # else:
322 # cfg = ConfigurableGeneric(component)
323 # setattr(parent_cfg,component,cfg)
324 # else:
325 # cfg = ConfigurableGeneric(component)
326 # parent_cfg = cfg
327
328 # remove spaces around dots
329 property = '.'.join([w.strip() for w in property.split('.')])
330 component, property = property.rsplit('.',1)
331 if component in Configurable.allConfigurables:
332 cfg = Configurable.allConfigurables[component]
333 else:
334 cfg = ConfigurableGeneric(component)
335
336 #value = os.path.expandvars(value)
337 value = value.replace('true','True').replace('false','False')
338 if value[0] == '{' :
339 # Try to guess if the values looks like a dictionary
340 if ':' in value and not ( value[:value.index(':')].count('"')%2 or value[:value.index(':')].count("'")%2 ) :
341 # for dictionaries, keep the surrounding {}
342 value = '{'+value[1:-1].replace('{','[').replace('}',']')+'}'
343 else : # otherwise replace all {} with []
344 value = value.replace('{','[').replace('}',']')
345
346 # We must escape '\' because eval tends to interpret them
347 value = value.replace('\\', '\\\\')
348 # Restore special cases ('\n', '\t' and '\"') (see GAUDI-1001)
349 value = (value.replace(r"\\n", r"\n")
350 .replace(r"\\t", r"\t")
351 .replace(r'\\"', r'\"'))
352 # replace r'\n' and r'\t' that are outside double quoted strings
353 value = '"'.join([(v if i % 2 else re.sub(r'\\[nt]', ' ', v))
354 for i, v in enumerate(value.split('"'))])
355
356 # interprete the @ operator
357 m = self.reference.match(value)
358 if m:
359 # this allows late binding of references
360 value = PropertyReference(m.group(1))
361 else:
362 value = eval(value,self.units)
363
364 #if type(value) is str : value = os.path.expandvars(value)
365 #elif type(value) is list : value = [ type(item) is str and os.path.expandvars(item) or item for item in value ]
366
367 if property not in cfg.__slots__ and not hasattr(cfg,property):
368 # check if the case of the property is wrong (old options are case insensitive)
369 lprop = property.lower()
370 for p in cfg.__slots__:
371 if lprop == p.lower():
372 _log.warning("property '%s' was requested for %s, but the correct spelling is '%s'", property, cfg.name(), p)
373 property = p
374 break
375
376 # consider the += and -=
377 if inc == "+":
378 if hasattr(cfg,property):
379 prop = getattr(cfg,property)
380 if type(prop) == dict:
381 for k in value:
382 prop[k] = value[k]
383 else:
384 prop += value
385 else:
386 setattr(cfg,property,value)
387 elif inc == "-":
388 if hasattr(cfg,property):
389 prop = getattr(cfg,property)
390 if type(prop) is dict:
391 for k in value:
392 if k in prop:
393 del prop[k]
394 else:
395 _log.warning("key '%s' not in %s.%s", k, cfg.name(), property)
396 else:
397 for k in value:
398 if k in prop:
399 prop.remove(k)
400 else:
401 _log.warning("value '%s' not in %s.%s", k, cfg.name(), property)
402 else:
403 setattr(cfg,property,value)
404
405 class _TempSysPath:
406 def __init__(self, new_path):
407 self.old_path = sys.path
408 sys.path = new_path
409 def __del__(self):
410 sys.path = self.old_path
411
412 _parser = JobOptsParser()
413
414 def _import_python(file):
415 execfile(file, {})
416
417 def _import_pickle(file):
418 import pickle
419 input = open(file, 'rb')
420 catalog = pickle.load(input)
421 _log.info('Unpickled %d configurables', len(catalog))
422
423 def _import_opts(file):
424 _parser.parse(file)
425
426 _import_function_mapping = {
427 ".py" : _import_python,
428 ".pkl" : _import_pickle,
429 ".opts" : _import_opts,
430 }
431
432 def importOptions( optsfile ) :
433 # expand environment variables before checking the extension
434 optsfile = os.path.expandvars(optsfile)
435 # check the file type (extension)
436 dummy, ext = os.path.splitext(optsfile)
437 if ext in _import_function_mapping:
438 # check if the file has been already included
439 optsfile = _find_file(optsfile)
440 if _to_be_included(optsfile):
441 _log.info("--> Including file '%s'", optsfile)
442 # include the file
443 _import_function_mapping[ext](optsfile)
444 _log.info("<-- End of file '%s'", optsfile)
445 else:
446 raise ParserError("Unknown file type '%s' ('%s')" % (ext,optsfile))
447
448 ## Import a file containing declaration of units.
449 # It is equivalent to:
450 #
451 # #units "unitsfile.opts"
452 #
453 def importUnits(unitsfile):
454 # expand environment variables
455 unitsfile = os.path.expandvars(unitsfile)
456 # we do not need to check the file type (extension) because it must be a
457 # units file
458 _parser._include(unitsfile, _parser._parse_units)
GaudiKernel.ProcessJobOptions.ConsoleHandler.disable
def disable(self, allowed=logging.WARNING)
Definition: ProcessJobOptions.py:81
GaudiKernel.ProcessJobOptions.JobOptsParser.__init__
def __init__(self)
Definition: ProcessJobOptions.py:142
GaudiKernel.ProcessJobOptions.PrintOff
def PrintOff(step=1)
Definition: ProcessJobOptions.py:105
GaudiKernel.ProcessJobOptions._import_opts
def _import_opts(file)
Definition: ProcessJobOptions.py:423
GaudiKernel.ProcessJobOptions._to_be_included
def _to_be_included(f)
Definition: ProcessJobOptions.py:125
GaudiKernel.ProcessJobOptions.ParserError
Definition: ProcessJobOptions.py:108
GaudiKernel.ProcessJobOptions.LogFormatter.with_time
with_time
Definition: ProcessJobOptions.py:11
GaudiKernel.ProcessJobOptions.ConsoleHandler.__init__
def __init__(self, stream=None, prefix=None, with_time=False)
Definition: ProcessJobOptions.py:57
GaudiKernel.ProcessJobOptions._TempSysPath.__del__
def __del__(self)
Definition: ProcessJobOptions.py:409
GaudiKernel.ProcessJobOptions.JobOptsParser._parse_units
def _parse_units(self, file)
Definition: ProcessJobOptions.py:285
GaudiKernel.ProcessJobOptions._find_file
def _find_file(f)
Definition: ProcessJobOptions.py:111
GaudiKernel.ProcessJobOptions.GetConsoleHandler
def GetConsoleHandler(prefix=None, stream=None, with_time=False)
Definition: ProcessJobOptions.py:87
GaudiKernel.ProcessJobOptions.JobOptsParser._eval_statement
def _eval_statement(self, statement)
Definition: ProcessJobOptions.py:297
GaudiKernel.ProcessJobOptions.LogFilter.filter
def filter(self, record)
Definition: ProcessJobOptions.py:29
GaudiKernel.ProcessJobOptions._TempSysPath.old_path
old_path
Definition: ProcessJobOptions.py:407
GaudiKernel.ProcessJobOptions.JobOptsParser.statement_sep
string statement_sep
Definition: ProcessJobOptions.py:139
GaudiKernel.ProcessJobOptions.JobOptsParser.comment_ml
tuple comment_ml
Definition: ProcessJobOptions.py:138
GaudiKernel.ProcessJobOptions.ConsoleHandler.enable
def enable(self, allowed=logging.WARNING)
Definition: ProcessJobOptions.py:83
GaudiKernel.ProcessJobOptions._TempSysPath.__init__
def __init__(self, new_path)
Definition: ProcessJobOptions.py:406
GaudiKernel.ProcessJobOptions.ConsoleHandler.printOn
def printOn(self, step=1, force=False)
Definition: ProcessJobOptions.py:69
GaudiKernel.ProcessJobOptions.LogFilter.threshold
threshold
Definition: ProcessJobOptions.py:28
GaudiKernel.ProcessJobOptions.LogFormatter.__init__
def __init__(self, fmt=None, datefmt=None, prefix="# ", with_time=False)
Definition: ProcessJobOptions.py:8
GaudiKernel.ProcessJobOptions.JobOptsParser.defines
defines
Definition: ProcessJobOptions.py:145
GaudiKernel.ProcessJobOptions.LogFilter
Definition: ProcessJobOptions.py:23
GaudiKernel.ProcessJobOptions.LogFormatter
Definition: ProcessJobOptions.py:7
GaudiKernel.ProcessJobOptions.ConsoleHandler._filter
_filter
Definition: ProcessJobOptions.py:63
GaudiKernel.ProcessJobOptions.InstallRootLoggingHandler
def InstallRootLoggingHandler(prefix=None, level=None, stream=None, with_time=False)
Definition: ProcessJobOptions.py:95
GaudiKernel.ProcessJobOptions.LogFilter.printOff
def printOff(self, step=1)
Definition: ProcessJobOptions.py:44
GaudiKernel.ProcessJobOptions.LogFilter.printing_level
printing_level
Definition: ProcessJobOptions.py:26
GaudiKernel.ProcessJobOptions.LogFilter.__init__
def __init__(self, name="")
Definition: ProcessJobOptions.py:24
GaudiKernel.ProcessJobOptions._TempSysPath
Definition: ProcessJobOptions.py:405
gaudirun.type
type
Definition: gaudirun.py:125
GaudiKernel.ProcessJobOptions.JobOptsParser._include
def _include(self, file, function)
Definition: ProcessJobOptions.py:149
GaudiKernel.ProcessJobOptions.importOptions
def importOptions(optsfile)
Definition: ProcessJobOptions.py:432
GaudiKernel.ProcessJobOptions.LogFormatter.prefix
prefix
Definition: ProcessJobOptions.py:10
GaudiKernel.ProcessJobOptions.ConsoleHandler._formatter
_formatter
Definition: ProcessJobOptions.py:64
GaudiKernel.ProcessJobOptions.LogFilter.enabled
enabled
Definition: ProcessJobOptions.py:27
GaudiKernel.ProcessJobOptions._import_python
def _import_python(file)
Definition: ProcessJobOptions.py:414
GaudiKernel.ProcessJobOptions.JobOptsParser.units
units
Definition: ProcessJobOptions.py:144
GaudiKernel.ProcessJobOptions.LogFilter.printOn
def printOn(self, step=1, force=False)
Definition: ProcessJobOptions.py:31
GaudiKernel.ProcessJobOptions.ConsoleHandler.setPrefix
def setPrefix(self, prefix)
Definition: ProcessJobOptions.py:67
GaudiKernel.ProcessJobOptions.LogFormatter.format
def format(self, record)
Definition: ProcessJobOptions.py:12
GaudiKernel.ProcessJobOptions.ConsoleHandler
Definition: ProcessJobOptions.py:56
GaudiKernel.ProcessJobOptions.PrintOn
def PrintOn(step=1, force=False)
Definition: ProcessJobOptions.py:103
GaudiKernel.ProcessJobOptions.JobOptsParser
Definition: ProcessJobOptions.py:132
GaudiKernel.ProcessJobOptions.ConsoleHandler.printOff
def printOff(self, step=1)
Definition: ProcessJobOptions.py:76
GaudiKernel.ProcessJobOptions._import_pickle
def _import_pickle(file)
Definition: ProcessJobOptions.py:417
GaudiKernel.ProcessJobOptions.JobOptsParser.parse
def parse(self, file)
Definition: ProcessJobOptions.py:156
GaudiKernel.ProcessJobOptions.LogFilter.enable
def enable(self, allowed=logging.WARNING)
Definition: ProcessJobOptions.py:52
GaudiKernel.ProcessJobOptions.LogFilter.disable
def disable(self, allowed=logging.WARNING)
Definition: ProcessJobOptions.py:49
GaudiKernel.ProcessJobOptions.importUnits
def importUnits(unitsfile)
Import a file containing declaration of units.
Definition: ProcessJobOptions.py:453