| The Gaudi Framework v32r1 (f65d50dc)

ProcessJobOptions.py
Go to the documentation of this file.
1 import os
2 import sys
3 import re
4 import time
5
6 import logging
7 _log = logging.getLogger(__name__)
8
9
10 class LogFormatter(logging.Formatter):
11 def __init__(self, fmt=None, datefmt=None, prefix="# ", with_time=False):
12 logging.Formatter.__init__(self, fmt, datefmt)
13 self.prefix = prefix
14 self.with_time = with_time
15
16 def format(self, record):
17 fmsg = logging.Formatter.format(self, record)
18 prefix = self.prefix
19 if self.with_time:
20 prefix += '%f ' % time.time()
21 if record.levelno >= logging.WARNING:
22 prefix += record.levelname + ": "
23 s = "\n".join([prefix + line for line in fmsg.splitlines()])
24 return s
25
26
27 class LogFilter(logging.Filter):
28 def __init__(self, name=""):
29 logging.Filter.__init__(self, name)
30 self.printing_level = 0
31 self.enabled = True
32 self.threshold = logging.WARNING
33
34 def filter(self, record):
35 return record.levelno >= self.threshold or (self.enabled and
36 self.printing_level <= 0)
37
38 def printOn(self, step=1, force=False):
39 """
40 Decrease the printing_level of 'step' units. ( >0 means no print)
41 The level cannot go below 0, unless the force flag is set to True.
42 A negative value of the threshold disables subsequent "PrintOff"s.
43 """
44 if force:
45 self.printing_level -= step
46 else:
47 if self.printing_level > step:
48 self.printing_level -= step
49 else:
50 self.printing_level = 0
51
52 def printOff(self, step=1):
53 """
54 Increase the printing_level of 'step' units. ( >0 means no print)
55 """
56 self.printing_level += step
57
58 def disable(self, allowed=logging.WARNING):
59 self.enabled = False
60 self.threshold = allowed
61
62 def enable(self, allowed=logging.WARNING):
63 self.enabled = True
64 self.threshold = allowed
65
66
67 class ConsoleHandler(logging.StreamHandler):
68 def __init__(self, stream=None, prefix=None, with_time=False):
69 if stream is None:
70 stream = sys.stdout
71 logging.StreamHandler.__init__(self, stream)
72 if prefix is None:
73 prefix = "# "
74 self._filter = LogFilter(_log.name)
75 self._formatter = LogFormatter(prefix=prefix, with_time=with_time)
76 self.setFormatter(self._formatter)
77 self.addFilter(self._filter)
78
79 def setPrefix(self, prefix):
80 self._formatter.prefix = prefix
81
82 def printOn(self, step=1, force=False):
83 """
84 Decrease the printing_level of 'step' units. ( >0 means no print)
85 The level cannot go below 0, unless the force flag is set to True.
86 A negative value of the threshold disables subsequent "PrintOff"s.
87 """
88 self._filter.printOn(step, force)
89
90 def printOff(self, step=1):
91 """
92 Increase the printing_level of 'step' units. ( >0 means no print)
93 """
94 self._filter.printOff(step)
95
96 def disable(self, allowed=logging.WARNING):
97 self._filter.disable(allowed)
98
99 def enable(self, allowed=logging.WARNING):
100 self._filter.enable(allowed)
101
102
103 _consoleHandler = None
104
105
106 def GetConsoleHandler(prefix=None, stream=None, with_time=False):
107 global _consoleHandler
108 if _consoleHandler is None:
109 _consoleHandler = ConsoleHandler(
110 prefix=prefix, stream=stream, with_time=with_time)
111 elif prefix is not None:
112 _consoleHandler.setPrefix(prefix)
113 return _consoleHandler
114
115
116 def InstallRootLoggingHandler(prefix=None,
117 level=None,
118 stream=None,
119 with_time=False):
120 root_logger = logging.getLogger()
121 if not root_logger.handlers:
122 root_logger.addHandler(GetConsoleHandler(prefix, stream, with_time))
123 root_logger.setLevel(logging.WARNING)
124 if level is not None:
125 root_logger.setLevel(level)
126
127
128 def PrintOn(step=1, force=False):
129 GetConsoleHandler().printOn(step, force)
130
131
132 def PrintOff(step=1):
133 GetConsoleHandler().printOff(step)
134
135
136 class ParserError(RuntimeError):
137 pass
138
139
140 def _find_file(f):
141 # expand environment variables in the filename
142 f = os.path.expandvars(f)
143 if os.path.isfile(f):
144 return os.path.realpath(f)
145
146 path = os.environ.get('JOBOPTSEARCHPATH', '').split(os.pathsep)
147 # find the full path to the option file
148 candidates = [d for d in path if os.path.isfile(os.path.join(d, f))]
149 if not candidates:
150 raise ParserError("Cannot find '%s' in %s" % (f, path))
151 return os.path.realpath(os.path.join(candidates[0], f))
152
153
154 _included_files = set()
155
156
157 def _to_be_included(f):
158 if f in _included_files:
159 _log.warning("file '%s' already included, ignored.", f)
160 return False
161 _included_files.add(f)
162 return True
163
164
165 class JobOptsParser:
166 comment = re.compile(r'(//.*)$')
167 # non-perfect R-E to check if '//' is inside a string
168 # (a tokenizer would be better)
169 comment_in_string = re.compile(r'(["\']).*//.*\1')
170 directive = re.compile(r'^\s*#\s*([\w!]+)\s*(.*)\s*$')
171 comment_ml = (re.compile(r'/\*'), re.compile(r'\*/'))
172 statement_sep = ";"
173 reference = re.compile(r'^@([\w.]*)$')
174
175 def __init__(self):
176 # parser level states
177 self.units = {}
178 self.defines = {}
179 if sys.platform != 'win32':
180 self.defines["WIN32"] = True
181
182 def _include(self, file, function):
183 file = _find_file(file)
184 if _to_be_included(file):
185 _log.info("--> Including file '%s'", file)
186 function(file)
187 _log.info("<-- End of file '%s'", file)
188
189 def parse(self, file):
190 # states for the "translation unit"
191 statement = ""
192
193 ifdef_level = 0
194 ifdef_skipping = False
195 ifdef_skipping_level = 0
196
197 in_string = False
198
199 f = open(_find_file(file))
200 l = f.readline()
201 if l.startswith("#!"):
202 # Skip the first line if it starts with "#!".
203 # It allows to use options files as scripts.
204 l = f.readline()
205
206 while l:
207 l = l.rstrip(
208 ) + '\n' # normalize EOL chars (to avoid problems with DOS new-line on Unix)
209
210 # single line comment
211 m = self.comment.search(l)
212 if m:
213 # check if the '//' is part of a string
214 m2 = self.comment_in_string.search(l)
215 # the '//' is part of a string if we find the quotes around it
216 # and they are not part of the comment itself
217 if not (m2 and m2.start() < m.start()):
218 # if it is not the case, we can remove the comment from the
219 # statement
220 l = l[:m.start()] + l[m.end():]
221 # process directives
222 m = self.directive.search(l)
223 if m:
224 directive_name = m.group(1)
225 directive_arg = m.group(2).strip()
226 if directive_name == "include":
227 included_file = directive_arg.strip("'\"")
228 importOptions(included_file)
229 elif directive_name == "units":
230 units_file = directive_arg.strip("'\"")
231 self._include(units_file, self._parse_units)
232 elif directive_name in ["ifdef", "ifndef"]:
233 ifdef_skipping_level = ifdef_level
234 ifdef_level += 1
235 if directive_arg in self.defines:
236 ifdef_skipping = directive_name == "ifndef"
237 else:
238 ifdef_skipping = directive_name == "ifdef"
239 elif directive_name == "else":
240 ifdef_skipping = not ifdef_skipping
241 elif directive_name == "endif":
242 ifdef_level -= 1
243 if ifdef_skipping and ifdef_skipping_level == ifdef_level:
244 ifdef_skipping = False
245 elif directive_name == "pragma":
246 if not directive_arg:
247 l = f.readline()
248 continue
249 pragma = directive_arg.split()
250 if pragma[0] == "print":
251 if len(pragma) > 1:
252 if pragma[1].upper() in ["ON", "TRUE", "1"]:
253 PrintOn()
254 else:
255 PrintOff()
256 else:
257 _log.warning("unknown directive '%s'", directive_name)
258 l = f.readline()
259 continue
260
261 if ifdef_skipping:
262 l = f.readline()
263 continue
264
265 # multi-line comment
266 m = self.comment_ml[0].search(l)
267 if m:
268 l, l1 = l[:m.start()], l[m.end():]
269 m = self.comment_ml[1].search(l1)
270 while not m:
271 l1 = f.readline()
272 if not l1:
273 break # EOF
274 m = self.comment_ml[1].search(l1)
275 if not l1 and not m:
276 raise ParserError(
277 "End Of File reached before end of multi-line comment")
278 l += l1[m.end():]
279
280 # if we are in a multiline string, we add to the statement
281 # everything until the next '"'
282 if in_string:
283 string_end = l.find('"')
284 if string_end >= 0:
285 statement += l[:string_end + 1]
286 l = l[string_end + 1:]
287 in_string = False # the string ends here
288 else:
289 statement += l
290 l = ''
291 else: # check if we have a string
292 string_start = l.find('"')
293 if string_start >= 0:
294 string_end = l.find('"', string_start + 1)
295 if string_end >= 0:
296 # the string is opened and closed
297 statement += l[:string_end + 1]
298 l = l[string_end + 1:]
299 else:
300 # the string is only opened
301 statement += l
302 in_string = True
303 l = f.readline()
304 continue
305
306 if self.statement_sep in l:
307 i = l.index(self.statement_sep)
308 statement += l[:i]
309 self._eval_statement(statement.strip().replace("\n", "\\n"))
310 statement = l[i + 1:]
311 # it may happen (bug #37479) that the rest of the statement
312 # contains a comment.
313 if statement.lstrip().startswith("//"):
314 statement = ""
315 else:
316 statement += l
317
318 l = f.readline()
319
320 def _parse_units(self, file):
321 for line in open(file):
322 if '//' in line:
323 line = line[:line.index('//')]
324 line = line.strip()
325 if not line:
326 continue
327 nunit, value = line.split('=')
328 factor, unit = nunit.split()
329 value = eval(value) / eval(factor)
330 self.units[unit] = value
331
332 def _eval_statement(self, statement):
333 from GaudiKernel.Proxy.Configurable import (
334 ConfigurableGeneric, Configurable, PropertyReference)
335 #statement = statement.replace("\n","").strip()
336 _log.info("%s%s", statement, self.statement_sep)
337
338 property, value = statement.split("=", 1)
339
340 inc = None
341 if property[-1] in ["+", "-"]:
342 inc = property[-1]
343 property = property[:-1]
344
345 property = property.strip()
346 value = value.strip()
347
348 # find the configurable to apply the property to
349 #parent_cfg = None
350 # while '.' in property:
351 # component, property = property.split('.',1)
352 # if parent_cfg:
353 # if hasattr(parent_cfg,component):
354 # cfg = getattr(parent_cfg,component)
355 # else:
356 # cfg = ConfigurableGeneric(component)
357 # setattr(parent_cfg,component,cfg)
358 # else:
359 # cfg = ConfigurableGeneric(component)
360 # parent_cfg = cfg
361
362 # remove spaces around dots
363 property = '.'.join([w.strip() for w in property.split('.')])
364 component, property = property.rsplit('.', 1)
365 if component in Configurable.allConfigurables:
366 cfg = Configurable.allConfigurables[component]
367 else:
368 cfg = ConfigurableGeneric(component)
369
370 #value = os.path.expandvars(value)
371 value = value.replace('true', 'True').replace('false', 'False')
372 if value[0] == '{':
373 # Try to guess if the values looks like a dictionary
374 if ':' in value and not (value[:value.index(':')].count('"') % 2 or
375 value[:value.index(':')].count("'") % 2):
376 # for dictionaries, keep the surrounding {}
377 value = '{' + \
378 value[1:-1].replace('{', '[').replace('}', ']') + '}'
379 else: # otherwise replace all {} with []
380 value = value.replace('{', '[').replace('}', ']')
381
382 # We must escape '\' because eval tends to interpret them
383 value = value.replace('\\', '\\\\')
384 # Restore special cases ('\n', '\t' and '\"') (see GAUDI-1001)
385 value = (value.replace(r"\\n", r"\n").replace(r"\\t", r"\t").replace(
386 r'\\"', r'\"'))
387 # replace r'\n' and r'\t' that are outside double quoted strings
388 value = '"'.join([(v if i % 2 else re.sub(r'\\[nt]', ' ', v))
389 for i, v in enumerate(value.split('"'))])
390
391 # interprete the @ operator
392 m = self.reference.match(value)
393 if m:
394 # this allows late binding of references
395 value = PropertyReference(m.group(1))
396 else:
397 value = eval(value, self.units)
398
399 #if type(value) is str : value = os.path.expandvars(value)
400 # elif type(value) is list : value = [ type(item) is str and os.path.expandvars(item) or item for item in value ]
401
402 if property not in cfg.__slots__ and not hasattr(cfg, property):
403 # check if the case of the property is wrong (old options are case insensitive)
404 lprop = property.lower()
405 for p in cfg.__slots__:
406 if lprop == p.lower():
407 _log.warning(
408 "property '%s' was requested for %s, but the correct spelling is '%s'",
409 property, cfg.name(), p)
410 property = p
411 break
412
413 # consider the += and -=
414 if inc == "+":
415 if hasattr(cfg, property):
416 prop = getattr(cfg, property)
417 if type(prop) == dict:
418 for k in value:
419 prop[k] = value[k]
420 else:
421 prop += value
422 else:
423 setattr(cfg, property, value)
424 elif inc == "-":
425 if hasattr(cfg, property):
426 prop = getattr(cfg, property)
427 if type(prop) is dict:
428 for k in value:
429 if k in prop:
430 del prop[k]
431 else:
432 _log.warning("key '%s' not in %s.%s", k,
433 cfg.name(), property)
434 else:
435 for k in value:
436 if k in prop:
437 prop.remove(k)
438 else:
439 _log.warning("value '%s' not in %s.%s", k,
440 cfg.name(), property)
441 else:
442 setattr(cfg, property, value)
443
444
445 class _TempSysPath:
446 def __init__(self, new_path):
447 self.old_path = sys.path
448 sys.path = new_path
449
450 def __del__(self):
451 sys.path = self.old_path
452
453
454 _parser = JobOptsParser()
455
456
457 def _import_python(file):
458 execfile(file, {})
459
460
461 def _import_pickle(file):
462 import pickle
463 input = open(file, 'rb')
464 catalog = pickle.load(input)
465 _log.info('Unpickled %d configurables', len(catalog))
466
467
468 def _import_opts(file):
469 _parser.parse(file)
470
471
472 _import_function_mapping = {
473 ".py": _import_python,
474 ".pkl": _import_pickle,
475 ".opts": _import_opts,
476 }
477
478
479 def importOptions(optsfile):
480 # expand environment variables before checking the extension
481 optsfile = os.path.expandvars(optsfile)
482 # check the file type (extension)
483 dummy, ext = os.path.splitext(optsfile)
484 if ext in _import_function_mapping:
485 # check if the file has been already included
486 optsfile = _find_file(optsfile)
487 if _to_be_included(optsfile):
488 _log.info("--> Including file '%s'", optsfile)
489 # include the file
490 _import_function_mapping[ext](optsfile)
491 _log.info("<-- End of file '%s'", optsfile)
492 else:
493 raise ParserError("Unknown file type '%s' ('%s')" % (ext, optsfile))
494
495
496 # Import a file containing declaration of units.
497 # It is equivalent to:
498 #
499 # #units "unitsfile.opts"
500 #
501
502
503 def importUnits(unitsfile):
504 # expand environment variables
505 unitsfile = os.path.expandvars(unitsfile)
506 # we do not need to check the file type (extension) because it must be a
507 # units file
508 _parser._include(unitsfile, _parser._parse_units)
GaudiKernel.ProcessJobOptions.ConsoleHandler.disable
def disable(self, allowed=logging.WARNING)
Definition: ProcessJobOptions.py:96
GaudiKernel.ProcessJobOptions.JobOptsParser.__init__
def __init__(self)
Definition: ProcessJobOptions.py:175
GaudiKernel.ProcessJobOptions.JobOptsParser.reference
reference
Definition: ProcessJobOptions.py:173
GaudiKernel.ProcessJobOptions.PrintOff
def PrintOff(step=1)
Definition: ProcessJobOptions.py:132
GaudiKernel.ProcessJobOptions._import_opts
def _import_opts(file)
Definition: ProcessJobOptions.py:468
GaudiKernel.ProcessJobOptions._to_be_included
def _to_be_included(f)
Definition: ProcessJobOptions.py:157
GaudiKernel.ProcessJobOptions.ParserError
Definition: ProcessJobOptions.py:136
GaudiKernel.ProcessJobOptions.LogFormatter.with_time
with_time
Definition: ProcessJobOptions.py:14
GaudiKernel.ProcessJobOptions.ConsoleHandler.__init__
def __init__(self, stream=None, prefix=None, with_time=False)
Definition: ProcessJobOptions.py:68
GaudiKernel.ProcessJobOptions._TempSysPath.__del__
def __del__(self)
Definition: ProcessJobOptions.py:450
GaudiKernel.ProcessJobOptions.JobOptsParser._parse_units
def _parse_units(self, file)
Definition: ProcessJobOptions.py:320
GaudiKernel.ProcessJobOptions._find_file
def _find_file(f)
Definition: ProcessJobOptions.py:140
GaudiKernel.ProcessJobOptions.GetConsoleHandler
def GetConsoleHandler(prefix=None, stream=None, with_time=False)
Definition: ProcessJobOptions.py:106
GaudiKernel.ProcessJobOptions.JobOptsParser._eval_statement
def _eval_statement(self, statement)
Definition: ProcessJobOptions.py:332
GaudiKernel.ProcessJobOptions.LogFilter.filter
def filter(self, record)
Definition: ProcessJobOptions.py:34
GaudiKernel.ProcessJobOptions._TempSysPath.old_path
old_path
Definition: ProcessJobOptions.py:447
GaudiKernel.ProcessJobOptions.JobOptsParser.statement_sep
string statement_sep
Definition: ProcessJobOptions.py:172
GaudiKernel.ProcessJobOptions.JobOptsParser.comment_ml
tuple comment_ml
Definition: ProcessJobOptions.py:171
GaudiKernel.ProcessJobOptions.ConsoleHandler.enable
def enable(self, allowed=logging.WARNING)
Definition: ProcessJobOptions.py:99
GaudiKernel.ProcessJobOptions.JobOptsParser.directive
directive
Definition: ProcessJobOptions.py:170
GaudiKernel.ProcessJobOptions._TempSysPath.__init__
def __init__(self, new_path)
Definition: ProcessJobOptions.py:446
GaudiKernel.ProcessJobOptions.ConsoleHandler.printOn
def printOn(self, step=1, force=False)
Definition: ProcessJobOptions.py:82
GaudiKernel.ProcessJobOptions.LogFilter.threshold
threshold
Definition: ProcessJobOptions.py:32
GaudiKernel.ProcessJobOptions.LogFormatter.__init__
def __init__(self, fmt=None, datefmt=None, prefix="# ", with_time=False)
Definition: ProcessJobOptions.py:11
GaudiKernel.ProcessJobOptions.JobOptsParser.defines
defines
Definition: ProcessJobOptions.py:178
GaudiKernel.ProcessJobOptions.LogFilter
Definition: ProcessJobOptions.py:27
GaudiKernel.ProcessJobOptions.LogFormatter
Definition: ProcessJobOptions.py:10
GaudiKernel.ProcessJobOptions.ConsoleHandler._filter
_filter
Definition: ProcessJobOptions.py:74
GaudiKernel.ProcessJobOptions.InstallRootLoggingHandler
def InstallRootLoggingHandler(prefix=None, level=None, stream=None, with_time=False)
Definition: ProcessJobOptions.py:116
GaudiKernel.ProcessJobOptions.LogFilter.printOff
def printOff(self, step=1)
Definition: ProcessJobOptions.py:52
GaudiKernel.ProcessJobOptions.LogFilter.printing_level
printing_level
Definition: ProcessJobOptions.py:30
GaudiKernel.ProcessJobOptions.LogFilter.__init__
def __init__(self, name="")
Definition: ProcessJobOptions.py:28
GaudiKernel.ProcessJobOptions._TempSysPath
Definition: ProcessJobOptions.py:445
gaudirun.type
type
Definition: gaudirun.py:142
GaudiKernel.ProcessJobOptions.JobOptsParser._include
def _include(self, file, function)
Definition: ProcessJobOptions.py:182
GaudiKernel.ProcessJobOptions.importOptions
def importOptions(optsfile)
Definition: ProcessJobOptions.py:479
GaudiKernel.ProcessJobOptions.LogFormatter.prefix
prefix
Definition: ProcessJobOptions.py:13
GaudiKernel.ProcessJobOptions.ConsoleHandler._formatter
_formatter
Definition: ProcessJobOptions.py:75
GaudiKernel.ProcessJobOptions.LogFilter.enabled
enabled
Definition: ProcessJobOptions.py:31
GaudiKernel.ProcessJobOptions._import_python
def _import_python(file)
Definition: ProcessJobOptions.py:457
GaudiKernel.ProcessJobOptions.JobOptsParser.comment
comment
Definition: ProcessJobOptions.py:166
GaudiKernel.ProcessJobOptions.JobOptsParser.units
units
Definition: ProcessJobOptions.py:177
GaudiKernel.ProcessJobOptions.LogFilter.printOn
def printOn(self, step=1, force=False)
Definition: ProcessJobOptions.py:38
GaudiKernel.ProcessJobOptions.ConsoleHandler.setPrefix
def setPrefix(self, prefix)
Definition: ProcessJobOptions.py:79
GaudiKernel.ProcessJobOptions.LogFormatter.format
def format(self, record)
Definition: ProcessJobOptions.py:16
GaudiKernel.ProcessJobOptions.ConsoleHandler
Definition: ProcessJobOptions.py:67
GaudiKernel.ProcessJobOptions.PrintOn
def PrintOn(step=1, force=False)
Definition: ProcessJobOptions.py:128
GaudiKernel.ProcessJobOptions.JobOptsParser
Definition: ProcessJobOptions.py:165
GaudiKernel.ProcessJobOptions.ConsoleHandler.printOff
def printOff(self, step=1)
Definition: ProcessJobOptions.py:90
GaudiKernel.ProcessJobOptions._import_pickle
def _import_pickle(file)
Definition: ProcessJobOptions.py:461
GaudiKernel.ProcessJobOptions.JobOptsParser.comment_in_string
comment_in_string
Definition: ProcessJobOptions.py:169
GaudiKernel.ProcessJobOptions.JobOptsParser.parse
def parse(self, file)
Definition: ProcessJobOptions.py:189
GaudiKernel.ProcessJobOptions.LogFilter.enable
def enable(self, allowed=logging.WARNING)
Definition: ProcessJobOptions.py:62
GaudiKernel.ProcessJobOptions.LogFilter.disable
def disable(self, allowed=logging.WARNING)
Definition: ProcessJobOptions.py:58
GaudiKernel.ProcessJobOptions.importUnits
def importUnits(unitsfile)
Definition: ProcessJobOptions.py:503