| The Gaudi Framework v29r0 (ff2e7097)

- Main Page
- Related Pages
- Modules
- Namespaces
- Classes
- Files

- File List
- File Members

ProcessJobOptions.py
Go to the documentation of this file.
1 import os
2 import sys
3 import re
4 import time
5
6 import logging
7 _log = logging.getLogger(__name__)
8
9
10 class LogFormatter(logging.Formatter):
11 def __init__(self, fmt=None, datefmt=None, prefix="# ", with_time=False):
12 logging.Formatter.__init__(self, fmt, datefmt)
13 self.prefix = prefix
14 self.with_time = with_time
15
16 def format(self, record):
17 fmsg = logging.Formatter.format(self, record)
18 prefix = self.prefix
19 if self.with_time:
20 prefix += '%f ' % time.time()
21 if record.levelno >= logging.WARNING:
22 prefix += record.levelname + ": "
23 s = "\n".join([prefix + line
24 for line in fmsg.splitlines()])
25 return s
26
27
28 class LogFilter(logging.Filter):
29 def __init__(self, name=""):
30 logging.Filter.__init__(self, name)
31 self.printing_level = 0
32 self.enabled = True
33 self.threshold = logging.WARNING
34
35 def filter(self, record):
36 return record.levelno >= self.threshold or (self.enabled and self.printing_level <= 0)
37
38 def printOn(self, step=1, force=False):
39 """
40 Decrease the printing_level of 'step' units. ( >0 means no print)
41 The level cannot go below 0, unless the force flag is set to True.
42 A negative value of the threshold disables subsequent "PrintOff"s.
43 """
44 if force:
45 self.printing_level -= step
46 else:
47 if self.printing_level > step:
48 self.printing_level -= step
49 else:
50 self.printing_level = 0
51
52 def printOff(self, step=1):
53 """
54 Increase the printing_level of 'step' units. ( >0 means no print)
55 """
56 self.printing_level += step
57
58 def disable(self, allowed=logging.WARNING):
59 self.enabled = False
60 self.threshold = allowed
61
62 def enable(self, allowed=logging.WARNING):
63 self.enabled = True
64 self.threshold = allowed
65
66
67 class ConsoleHandler(logging.StreamHandler):
68 def __init__(self, stream=None, prefix=None, with_time=False):
69 if stream is None:
70 stream = sys.stdout
71 logging.StreamHandler.__init__(self, stream)
72 if prefix is None:
73 prefix = "# "
74 self._filter = LogFilter(_log.name)
75 self._formatter = LogFormatter(prefix=prefix, with_time=with_time)
76 self.setFormatter(self._formatter)
77 self.addFilter(self._filter)
78
79 def setPrefix(self, prefix):
80 self._formatter.prefix = prefix
81
82 def printOn(self, step=1, force=False):
83 """
84 Decrease the printing_level of 'step' units. ( >0 means no print)
85 The level cannot go below 0, unless the force flag is set to True.
86 A negative value of the threshold disables subsequent "PrintOff"s.
87 """
88 self._filter.printOn(step, force)
89
90 def printOff(self, step=1):
91 """
92 Increase the printing_level of 'step' units. ( >0 means no print)
93 """
94 self._filter.printOff(step)
95
96 def disable(self, allowed=logging.WARNING):
97 self._filter.disable(allowed)
98
99 def enable(self, allowed=logging.WARNING):
100 self._filter.enable(allowed)
101
102
103 _consoleHandler = None
104
105
106 def GetConsoleHandler(prefix=None, stream=None, with_time=False):
107 global _consoleHandler
108 if _consoleHandler is None:
109 _consoleHandler = ConsoleHandler(
110 prefix=prefix, stream=stream, with_time=with_time)
111 elif prefix is not None:
112 _consoleHandler.setPrefix(prefix)
113 return _consoleHandler
114
115
116 def InstallRootLoggingHandler(prefix=None, level=None, stream=None, with_time=False):
117 root_logger = logging.getLogger()
118 if not root_logger.handlers:
119 root_logger.addHandler(GetConsoleHandler(prefix, stream, with_time))
120 root_logger.setLevel(logging.WARNING)
121 if level is not None:
122 root_logger.setLevel(level)
123
124
125 def PrintOn(step=1, force=False):
126 GetConsoleHandler().printOn(step, force)
127
128
129 def PrintOff(step=1):
130 GetConsoleHandler().printOff(step)
131
132
133 class ParserError(RuntimeError):
134 pass
135
136
137 def _find_file(f):
138 # expand environment variables in the filename
139 f = os.path.expandvars(f)
140 if os.path.isfile(f):
141 return os.path.realpath(f)
142
143 path = os.environ.get('JOBOPTSEARCHPATH', '').split(os.pathsep)
144 # find the full path to the option file
145 candidates = [d for d in path if os.path.isfile(os.path.join(d, f))]
146 if not candidates:
147 raise ParserError("Cannot find '%s' in %s" % (f, path))
148 return os.path.realpath(os.path.join(candidates[0], f))
149
150
151 _included_files = set()
152
153
154 def _to_be_included(f):
155 if f in _included_files:
156 _log.warning("file '%s' already included, ignored.", f)
157 return False
158 _included_files.add(f)
159 return True
160
161
162 class JobOptsParser:
163 comment = re.compile(r'(//.*)$')
164 # non-perfect R-E to check if '//' is inside a string
165 # (a tokenizer would be better)
166 comment_in_string = re.compile(r'(["\']).*//.*\1')
167 directive = re.compile(r'^\s*#\s*([\w!]+)\s*(.*)\s*$')
168 comment_ml = (re.compile(r'/\*'), re.compile(r'\*/'))
169 statement_sep = ";"
170 reference = re.compile(r'^@([\w.]*)$')
171
172 def __init__(self):
173 # parser level states
174 self.units = {}
175 self.defines = {}
176 if sys.platform != 'win32':
177 self.defines["WIN32"] = True
178
179 def _include(self, file, function):
180 file = _find_file(file)
181 if _to_be_included(file):
182 _log.info("--> Including file '%s'", file)
183 function(file)
184 _log.info("<-- End of file '%s'", file)
185
186 def parse(self, file):
187 # states for the "translation unit"
188 statement = ""
189
190 ifdef_level = 0
191 ifdef_skipping = False
192 ifdef_skipping_level = 0
193
194 in_string = False
195
196 f = open(_find_file(file))
197 l = f.readline()
198 if l.startswith("#!"):
199 # Skip the first line if it starts with "#!".
200 # It allows to use options files as scripts.
201 l = f.readline()
202
203 while l:
204 l = l.rstrip() + '\n' # normalize EOL chars (to avoid problems with DOS new-line on Unix)
205
206 # single line comment
207 m = self.comment.search(l)
208 if m:
209 # check if the '//' is part of a string
210 m2 = self.comment_in_string.search(l)
211 # the '//' is part of a string if we find the quotes around it
212 # and they are not part of the comment itself
213 if not (m2 and m2.start() < m.start()):
214 # if it is not the case, we can remove the comment from the
215 # statement
216 l = l[:m.start()] + l[m.end():]
217 # process directives
218 m = self.directive.search(l)
219 if m:
220 directive_name = m.group(1)
221 directive_arg = m.group(2).strip()
222 if directive_name == "include":
223 included_file = directive_arg.strip("'\"")
224 importOptions(included_file)
225 elif directive_name == "units":
226 units_file = directive_arg.strip("'\"")
227 self._include(units_file, self._parse_units)
228 elif directive_name in ["ifdef", "ifndef"]:
229 ifdef_skipping_level = ifdef_level
230 ifdef_level += 1
231 if directive_arg in self.defines:
232 ifdef_skipping = directive_name == "ifndef"
233 else:
234 ifdef_skipping = directive_name == "ifdef"
235 elif directive_name == "else":
236 ifdef_skipping = not ifdef_skipping
237 elif directive_name == "endif":
238 ifdef_level -= 1
239 if ifdef_skipping and ifdef_skipping_level == ifdef_level:
240 ifdef_skipping = False
241 elif directive_name == "pragma":
242 if not directive_arg:
243 l = f.readline()
244 continue
245 pragma = directive_arg.split()
246 if pragma[0] == "print":
247 if len(pragma) > 1:
248 if pragma[1].upper() in ["ON", "TRUE", "1"]:
249 PrintOn()
250 else:
251 PrintOff()
252 else:
253 _log.warning("unknown directive '%s'", directive_name)
254 l = f.readline()
255 continue
256
257 if ifdef_skipping:
258 l = f.readline()
259 continue
260
261 # multi-line comment
262 m = self.comment_ml[0].search(l)
263 if m:
264 l, l1 = l[:m.start()], l[m.end():]
265 m = self.comment_ml[1].search(l1)
266 while not m:
267 l1 = f.readline()
268 if not l1:
269 break # EOF
270 m = self.comment_ml[1].search(l1)
271 if not l1 and not m:
272 raise ParserError(
273 "End Of File reached before end of multi-line comment")
274 l += l1[m.end():]
275
276 # if we are in a multiline string, we add to the statement
277 # everything until the next '"'
278 if in_string:
279 string_end = l.find('"')
280 if string_end >= 0:
281 statement += l[:string_end + 1]
282 l = l[string_end + 1:]
283 in_string = False # the string ends here
284 else:
285 statement += l
286 l = ''
287 else: # check if we have a string
288 string_start = l.find('"')
289 if string_start >= 0:
290 string_end = l.find('"', string_start + 1)
291 if string_end >= 0:
292 # the string is opened and closed
293 statement += l[:string_end + 1]
294 l = l[string_end + 1:]
295 else:
296 # the string is only opened
297 statement += l
298 in_string = True
299 l = f.readline()
300 continue
301
302 if self.statement_sep in l:
303 i = l.index(self.statement_sep)
304 statement += l[:i]
305 self._eval_statement(statement.strip().replace("\n", "\\n"))
306 statement = l[i + 1:]
307 # it may happen (bug #37479) that the rest of the statement
308 # contains a comment.
309 if statement.lstrip().startswith("//"):
310 statement = ""
311 else:
312 statement += l
313
314 l = f.readline()
315
316 def _parse_units(self, file):
317 for line in open(file):
318 if '//' in line:
319 line = line[:line.index('//')]
320 line = line.strip()
321 if not line:
322 continue
323 nunit, value = line.split('=')
324 factor, unit = nunit.split()
325 value = eval(value) / eval(factor)
326 self.units[unit] = value
327
328 def _eval_statement(self, statement):
329 from GaudiKernel.Proxy.Configurable import (ConfigurableGeneric,
330 Configurable,
331 PropertyReference)
332 #statement = statement.replace("\n","").strip()
333 _log.info("%s%s", statement, self.statement_sep)
334
335 property, value = statement.split("=", 1)
336
337 inc = None
338 if property[-1] in ["+", "-"]:
339 inc = property[-1]
340 property = property[:-1]
341
342 property = property.strip()
343 value = value.strip()
344
345 # find the configurable to apply the property to
346 #parent_cfg = None
347 # while '.' in property:
348 # component, property = property.split('.',1)
349 # if parent_cfg:
350 # if hasattr(parent_cfg,component):
351 # cfg = getattr(parent_cfg,component)
352 # else:
353 # cfg = ConfigurableGeneric(component)
354 # setattr(parent_cfg,component,cfg)
355 # else:
356 # cfg = ConfigurableGeneric(component)
357 # parent_cfg = cfg
358
359 # remove spaces around dots
360 property = '.'.join([w.strip() for w in property.split('.')])
361 component, property = property.rsplit('.', 1)
362 if component in Configurable.allConfigurables:
363 cfg = Configurable.allConfigurables[component]
364 else:
365 cfg = ConfigurableGeneric(component)
366
367 #value = os.path.expandvars(value)
368 value = value.replace('true', 'True').replace('false', 'False')
369 if value[0] == '{':
370 # Try to guess if the values looks like a dictionary
371 if ':' in value and not (value[:value.index(':')].count('"') % 2 or value[:value.index(':')].count("'") % 2):
372 # for dictionaries, keep the surrounding {}
373 value = '{' + \
374 value[1:-1].replace('{', '[').replace('}', ']') + '}'
375 else: # otherwise replace all {} with []
376 value = value.replace('{', '[').replace('}', ']')
377
378 # We must escape '\' because eval tends to interpret them
379 value = value.replace('\\', '\\\\')
380 # Restore special cases ('\n', '\t' and '\"') (see GAUDI-1001)
381 value = (value.replace(r"\\n", r"\n")
382 .replace(r"\\t", r"\t")
383 .replace(r'\\"', r'\"'))
384 # replace r'\n' and r'\t' that are outside double quoted strings
385 value = '"'.join([(v if i % 2 else re.sub(r'\\[nt]', ' ', v))
386 for i, v in enumerate(value.split('"'))])
387
388 # interprete the @ operator
389 m = self.reference.match(value)
390 if m:
391 # this allows late binding of references
392 value = PropertyReference(m.group(1))
393 else:
394 value = eval(value, self.units)
395
396 #if type(value) is str : value = os.path.expandvars(value)
397 # elif type(value) is list : value = [ type(item) is str and os.path.expandvars(item) or item for item in value ]
398
399 if property not in cfg.__slots__ and not hasattr(cfg, property):
400 # check if the case of the property is wrong (old options are case insensitive)
401 lprop = property.lower()
402 for p in cfg.__slots__:
403 if lprop == p.lower():
404 _log.warning(
405 "property '%s' was requested for %s, but the correct spelling is '%s'", property, cfg.name(), p)
406 property = p
407 break
408
409 # consider the += and -=
410 if inc == "+":
411 if hasattr(cfg, property):
412 prop = getattr(cfg, property)
413 if type(prop) == dict:
414 for k in value:
415 prop[k] = value[k]
416 else:
417 prop += value
418 else:
419 setattr(cfg, property, value)
420 elif inc == "-":
421 if hasattr(cfg, property):
422 prop = getattr(cfg, property)
423 if type(prop) is dict:
424 for k in value:
425 if k in prop:
426 del prop[k]
427 else:
428 _log.warning("key '%s' not in %s.%s",
429 k, cfg.name(), property)
430 else:
431 for k in value:
432 if k in prop:
433 prop.remove(k)
434 else:
435 _log.warning("value '%s' not in %s.%s",
436 k, cfg.name(), property)
437 else:
438 setattr(cfg, property, value)
439
440
441 class _TempSysPath:
442 def __init__(self, new_path):
443 self.old_path = sys.path
444 sys.path = new_path
445
446 def __del__(self):
447 sys.path = self.old_path
448
449
450 _parser = JobOptsParser()
451
452
453 def _import_python(file):
454 execfile(file, {})
455
456
457 def _import_pickle(file):
458 import pickle
459 input = open(file, 'rb')
460 catalog = pickle.load(input)
461 _log.info('Unpickled %d configurables', len(catalog))
462
463
464 def _import_opts(file):
465 _parser.parse(file)
466
467
468 _import_function_mapping = {
469 ".py": _import_python,
470 ".pkl": _import_pickle,
471 ".opts": _import_opts,
472 }
473
474
475 def importOptions(optsfile):
476 # expand environment variables before checking the extension
477 optsfile = os.path.expandvars(optsfile)
478 # check the file type (extension)
479 dummy, ext = os.path.splitext(optsfile)
480 if ext in _import_function_mapping:
481 # check if the file has been already included
482 optsfile = _find_file(optsfile)
483 if _to_be_included(optsfile):
484 _log.info("--> Including file '%s'", optsfile)
485 # include the file
486 _import_function_mapping[ext](optsfile)
487 _log.info("<-- End of file '%s'", optsfile)
488 else:
489 raise ParserError("Unknown file type '%s' ('%s')" % (ext, optsfile))
490
491 # Import a file containing declaration of units.
492 # It is equivalent to:
493 #
494 # #units "unitsfile.opts"
495 #
496
497
498 def importUnits(unitsfile):
499 # expand environment variables
500 unitsfile = os.path.expandvars(unitsfile)
501 # we do not need to check the file type (extension) because it must be a
502 # units file
503 _parser._include(unitsfile, _parser._parse_units)
GaudiKernel.ProcessJobOptions.ConsoleHandler.disable
def disable(self, allowed=logging.WARNING)
Definition: ProcessJobOptions.py:96
GaudiKernel.ProcessJobOptions.JobOptsParser.__init__
def __init__(self)
Definition: ProcessJobOptions.py:172
GaudiKernel.ProcessJobOptions.PrintOff
def PrintOff(step=1)
Definition: ProcessJobOptions.py:129
GaudiKernel.ProcessJobOptions._import_opts
def _import_opts(file)
Definition: ProcessJobOptions.py:464
GaudiKernel.ProcessJobOptions._to_be_included
def _to_be_included(f)
Definition: ProcessJobOptions.py:154
GaudiKernel.ProcessJobOptions.ParserError
Definition: ProcessJobOptions.py:133
GaudiKernel.ProcessJobOptions.LogFormatter.with_time
with_time
Definition: ProcessJobOptions.py:14
GaudiKernel.ProcessJobOptions.ConsoleHandler.__init__
def __init__(self, stream=None, prefix=None, with_time=False)
Definition: ProcessJobOptions.py:68
GaudiKernel.ProcessJobOptions._TempSysPath.__del__
def __del__(self)
Definition: ProcessJobOptions.py:446
GaudiKernel.ProcessJobOptions.JobOptsParser._parse_units
def _parse_units(self, file)
Definition: ProcessJobOptions.py:316
GaudiKernel.ProcessJobOptions._find_file
def _find_file(f)
Definition: ProcessJobOptions.py:137
GaudiKernel.ProcessJobOptions.GetConsoleHandler
def GetConsoleHandler(prefix=None, stream=None, with_time=False)
Definition: ProcessJobOptions.py:106
GaudiKernel.ProcessJobOptions.JobOptsParser._eval_statement
def _eval_statement(self, statement)
Definition: ProcessJobOptions.py:328
GaudiKernel.ProcessJobOptions.LogFilter.filter
def filter(self, record)
Definition: ProcessJobOptions.py:35
GaudiKernel.ProcessJobOptions._TempSysPath.old_path
old_path
Definition: ProcessJobOptions.py:443
GaudiKernel.ProcessJobOptions.JobOptsParser.statement_sep
string statement_sep
Definition: ProcessJobOptions.py:169
GaudiKernel.ProcessJobOptions.JobOptsParser.comment_ml
tuple comment_ml
Definition: ProcessJobOptions.py:168
GaudiKernel.ProcessJobOptions.ConsoleHandler.enable
def enable(self, allowed=logging.WARNING)
Definition: ProcessJobOptions.py:99
GaudiKernel.ProcessJobOptions._TempSysPath.__init__
def __init__(self, new_path)
Definition: ProcessJobOptions.py:442
GaudiKernel.ProcessJobOptions.ConsoleHandler.printOn
def printOn(self, step=1, force=False)
Definition: ProcessJobOptions.py:82
GaudiKernel.ProcessJobOptions.LogFilter.threshold
threshold
Definition: ProcessJobOptions.py:33
GaudiKernel.ProcessJobOptions.LogFormatter.__init__
def __init__(self, fmt=None, datefmt=None, prefix="# ", with_time=False)
Definition: ProcessJobOptions.py:11
GaudiKernel.ProcessJobOptions.JobOptsParser.defines
defines
Definition: ProcessJobOptions.py:175
GaudiKernel.ProcessJobOptions.LogFilter
Definition: ProcessJobOptions.py:28
GaudiKernel.ProcessJobOptions.LogFormatter
Definition: ProcessJobOptions.py:10
GaudiKernel.ProcessJobOptions.ConsoleHandler._filter
_filter
Definition: ProcessJobOptions.py:74
GaudiKernel.ProcessJobOptions.InstallRootLoggingHandler
def InstallRootLoggingHandler(prefix=None, level=None, stream=None, with_time=False)
Definition: ProcessJobOptions.py:116
GaudiKernel.ProcessJobOptions.LogFilter.printOff
def printOff(self, step=1)
Definition: ProcessJobOptions.py:52
GaudiKernel.ProcessJobOptions.LogFilter.printing_level
printing_level
Definition: ProcessJobOptions.py:31
GaudiKernel.ProcessJobOptions.LogFilter.__init__
def __init__(self, name="")
Definition: ProcessJobOptions.py:29
GaudiKernel.ProcessJobOptions._TempSysPath
Definition: ProcessJobOptions.py:441
gaudirun.type
type
Definition: gaudirun.py:133
GaudiKernel.ProcessJobOptions.JobOptsParser._include
def _include(self, file, function)
Definition: ProcessJobOptions.py:179
GaudiKernel.ProcessJobOptions.importOptions
def importOptions(optsfile)
Definition: ProcessJobOptions.py:475
GaudiKernel.ProcessJobOptions.LogFormatter.prefix
prefix
Definition: ProcessJobOptions.py:13
GaudiKernel.ProcessJobOptions.ConsoleHandler._formatter
_formatter
Definition: ProcessJobOptions.py:75
GaudiKernel.ProcessJobOptions.LogFilter.enabled
enabled
Definition: ProcessJobOptions.py:32
GaudiKernel.ProcessJobOptions._import_python
def _import_python(file)
Definition: ProcessJobOptions.py:453
GaudiKernel.ProcessJobOptions.JobOptsParser.units
units
Definition: ProcessJobOptions.py:174
GaudiKernel.ProcessJobOptions.LogFilter.printOn
def printOn(self, step=1, force=False)
Definition: ProcessJobOptions.py:38
GaudiKernel.ProcessJobOptions.ConsoleHandler.setPrefix
def setPrefix(self, prefix)
Definition: ProcessJobOptions.py:79
GaudiKernel.ProcessJobOptions.LogFormatter.format
def format(self, record)
Definition: ProcessJobOptions.py:16
GaudiKernel.ProcessJobOptions.ConsoleHandler
Definition: ProcessJobOptions.py:67
GaudiKernel.ProcessJobOptions.PrintOn
def PrintOn(step=1, force=False)
Definition: ProcessJobOptions.py:125
GaudiKernel.ProcessJobOptions.JobOptsParser
Definition: ProcessJobOptions.py:162
GaudiKernel.ProcessJobOptions.ConsoleHandler.printOff
def printOff(self, step=1)
Definition: ProcessJobOptions.py:90
GaudiKernel.ProcessJobOptions._import_pickle
def _import_pickle(file)
Definition: ProcessJobOptions.py:457
GaudiKernel.ProcessJobOptions.JobOptsParser.parse
def parse(self, file)
Definition: ProcessJobOptions.py:186
GaudiKernel.ProcessJobOptions.LogFilter.enable
def enable(self, allowed=logging.WARNING)
Definition: ProcessJobOptions.py:62
GaudiKernel.ProcessJobOptions.LogFilter.disable
def disable(self, allowed=logging.WARNING)
Definition: ProcessJobOptions.py:58
GaudiKernel.ProcessJobOptions.importUnits
def importUnits(unitsfile)
Definition: ProcessJobOptions.py:498