Loading [MathJax]/jax/input/TeX/config.js
The Gaudi Framework  v31r0 (aeb156f0)
All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Modules Pages
ProcessJobOptions.py
Go to the documentation of this file.
1 import os
2 import sys
3 import re
4 import time
5 
6 import logging
7 _log = logging.getLogger(__name__)
8 
9 
10 class LogFormatter(logging.Formatter):
11  def __init__(self, fmt=None, datefmt=None, prefix="# ", with_time=False):
12  logging.Formatter.__init__(self, fmt, datefmt)
13  self.prefix = prefix
14  self.with_time = with_time
15 
16  def format(self, record):
17  fmsg = logging.Formatter.format(self, record)
18  prefix = self.prefix
19  if self.with_time:
20  prefix += '%f ' % time.time()
21  if record.levelno >= logging.WARNING:
22  prefix += record.levelname + ": "
23  s = "\n".join([prefix + line for line in fmsg.splitlines()])
24  return s
25 
26 
27 class LogFilter(logging.Filter):
28  def __init__(self, name=""):
29  logging.Filter.__init__(self, name)
30  self.printing_level = 0
31  self.enabled = True
32  self.threshold = logging.WARNING
33 
34  def filter(self, record):
35  return record.levelno >= self.threshold or (self.enabled and
36  self.printing_level <= 0)
37 
38  def printOn(self, step=1, force=False):
39  """
40  Decrease the printing_level of 'step' units. ( >0 means no print)
41  The level cannot go below 0, unless the force flag is set to True.
42  A negative value of the threshold disables subsequent "PrintOff"s.
43  """
44  if force:
45  self.printing_level -= step
46  else:
47  if self.printing_level > step:
48  self.printing_level -= step
49  else:
50  self.printing_level = 0
51 
52  def printOff(self, step=1):
53  """
54  Increase the printing_level of 'step' units. ( >0 means no print)
55  """
56  self.printing_level += step
57 
58  def disable(self, allowed=logging.WARNING):
59  self.enabled = False
60  self.threshold = allowed
61 
62  def enable(self, allowed=logging.WARNING):
63  self.enabled = True
64  self.threshold = allowed
65 
66 
67 class ConsoleHandler(logging.StreamHandler):
68  def __init__(self, stream=None, prefix=None, with_time=False):
69  if stream is None:
70  stream = sys.stdout
71  logging.StreamHandler.__init__(self, stream)
72  if prefix is None:
73  prefix = "# "
74  self._filter = LogFilter(_log.name)
75  self._formatter = LogFormatter(prefix=prefix, with_time=with_time)
76  self.setFormatter(self._formatter)
77  self.addFilter(self._filter)
78 
79  def setPrefix(self, prefix):
80  self._formatter.prefix = prefix
81 
82  def printOn(self, step=1, force=False):
83  """
84  Decrease the printing_level of 'step' units. ( >0 means no print)
85  The level cannot go below 0, unless the force flag is set to True.
86  A negative value of the threshold disables subsequent "PrintOff"s.
87  """
88  self._filter.printOn(step, force)
89 
90  def printOff(self, step=1):
91  """
92  Increase the printing_level of 'step' units. ( >0 means no print)
93  """
94  self._filter.printOff(step)
95 
96  def disable(self, allowed=logging.WARNING):
97  self._filter.disable(allowed)
98 
99  def enable(self, allowed=logging.WARNING):
100  self._filter.enable(allowed)
101 
102 
103 _consoleHandler = None
104 
105 
106 def GetConsoleHandler(prefix=None, stream=None, with_time=False):
107  global _consoleHandler
108  if _consoleHandler is None:
109  _consoleHandler = ConsoleHandler(
110  prefix=prefix, stream=stream, with_time=with_time)
111  elif prefix is not None:
112  _consoleHandler.setPrefix(prefix)
113  return _consoleHandler
114 
115 
116 def InstallRootLoggingHandler(prefix=None,
117  level=None,
118  stream=None,
119  with_time=False):
120  root_logger = logging.getLogger()
121  if not root_logger.handlers:
122  root_logger.addHandler(GetConsoleHandler(prefix, stream, with_time))
123  root_logger.setLevel(logging.WARNING)
124  if level is not None:
125  root_logger.setLevel(level)
126 
127 
128 def PrintOn(step=1, force=False):
129  GetConsoleHandler().printOn(step, force)
130 
131 
132 def PrintOff(step=1):
133  GetConsoleHandler().printOff(step)
134 
135 
136 class ParserError(RuntimeError):
137  pass
138 
139 
140 def _find_file(f):
141  # expand environment variables in the filename
142  f = os.path.expandvars(f)
143  if os.path.isfile(f):
144  return os.path.realpath(f)
145 
146  path = os.environ.get('JOBOPTSEARCHPATH', '').split(os.pathsep)
147  # find the full path to the option file
148  candidates = [d for d in path if os.path.isfile(os.path.join(d, f))]
149  if not candidates:
150  raise ParserError("Cannot find '%s' in %s" % (f, path))
151  return os.path.realpath(os.path.join(candidates[0], f))
152 
153 
154 _included_files = set()
155 
156 
158  if f in _included_files:
159  _log.warning("file '%s' already included, ignored.", f)
160  return False
161  _included_files.add(f)
162  return True
163 
164 
166  comment = re.compile(r'(//.*)$')
167  # non-perfect R-E to check if '//' is inside a string
168  # (a tokenizer would be better)
169  comment_in_string = re.compile(r'(["\']).*//.*\1')
170  directive = re.compile(r'^\s*#\s*([\w!]+)\s*(.*)\s*$')
171  comment_ml = (re.compile(r'/\*'), re.compile(r'\*/'))
172  statement_sep = ";"
173  reference = re.compile(r'^@([\w.]*)$')
174 
175  def __init__(self):
176  # parser level states
177  self.units = {}
178  self.defines = {}
179  if sys.platform != 'win32':
180  self.defines["WIN32"] = True
181 
182  def _include(self, file, function):
183  file = _find_file(file)
184  if _to_be_included(file):
185  _log.info("--> Including file '%s'", file)
186  function(file)
187  _log.info("<-- End of file '%s'", file)
188 
189  def parse(self, file):
190  # states for the "translation unit"
191  statement = ""
192 
193  ifdef_level = 0
194  ifdef_skipping = False
195  ifdef_skipping_level = 0
196 
197  in_string = False
198 
199  f = open(_find_file(file))
200  l = f.readline()
201  if l.startswith("#!"):
202  # Skip the first line if it starts with "#!".
203  # It allows to use options files as scripts.
204  l = f.readline()
205 
206  while l:
207  l = l.rstrip(
208  ) + '\n' # normalize EOL chars (to avoid problems with DOS new-line on Unix)
209 
210  # single line comment
211  m = self.comment.search(l)
212  if m:
213  # check if the '//' is part of a string
214  m2 = self.comment_in_string.search(l)
215  # the '//' is part of a string if we find the quotes around it
216  # and they are not part of the comment itself
217  if not (m2 and m2.start() < m.start()):
218  # if it is not the case, we can remove the comment from the
219  # statement
220  l = l[:m.start()] + l[m.end():]
221  # process directives
222  m = self.directive.search(l)
223  if m:
224  directive_name = m.group(1)
225  directive_arg = m.group(2).strip()
226  if directive_name == "include":
227  included_file = directive_arg.strip("'\"")
228  importOptions(included_file)
229  elif directive_name == "units":
230  units_file = directive_arg.strip("'\"")
231  self._include(units_file, self._parse_units)
232  elif directive_name in ["ifdef", "ifndef"]:
233  ifdef_skipping_level = ifdef_level
234  ifdef_level += 1
235  if directive_arg in self.defines:
236  ifdef_skipping = directive_name == "ifndef"
237  else:
238  ifdef_skipping = directive_name == "ifdef"
239  elif directive_name == "else":
240  ifdef_skipping = not ifdef_skipping
241  elif directive_name == "endif":
242  ifdef_level -= 1
243  if ifdef_skipping and ifdef_skipping_level == ifdef_level:
244  ifdef_skipping = False
245  elif directive_name == "pragma":
246  if not directive_arg:
247  l = f.readline()
248  continue
249  pragma = directive_arg.split()
250  if pragma[0] == "print":
251  if len(pragma) > 1:
252  if pragma[1].upper() in ["ON", "TRUE", "1"]:
253  PrintOn()
254  else:
255  PrintOff()
256  else:
257  _log.warning("unknown directive '%s'", directive_name)
258  l = f.readline()
259  continue
260 
261  if ifdef_skipping:
262  l = f.readline()
263  continue
264 
265  # multi-line comment
266  m = self.comment_ml[0].search(l)
267  if m:
268  l, l1 = l[:m.start()], l[m.end():]
269  m = self.comment_ml[1].search(l1)
270  while not m:
271  l1 = f.readline()
272  if not l1:
273  break # EOF
274  m = self.comment_ml[1].search(l1)
275  if not l1 and not m:
276  raise ParserError(
277  "End Of File reached before end of multi-line comment")
278  l += l1[m.end():]
279 
280  # if we are in a multiline string, we add to the statement
281  # everything until the next '"'
282  if in_string:
283  string_end = l.find('"')
284  if string_end >= 0:
285  statement += l[:string_end + 1]
286  l = l[string_end + 1:]
287  in_string = False # the string ends here
288  else:
289  statement += l
290  l = ''
291  else: # check if we have a string
292  string_start = l.find('"')
293  if string_start >= 0:
294  string_end = l.find('"', string_start + 1)
295  if string_end >= 0:
296  # the string is opened and closed
297  statement += l[:string_end + 1]
298  l = l[string_end + 1:]
299  else:
300  # the string is only opened
301  statement += l
302  in_string = True
303  l = f.readline()
304  continue
305 
306  if self.statement_sep in l:
307  i = l.index(self.statement_sep)
308  statement += l[:i]
309  self._eval_statement(statement.strip().replace("\n", "\\n"))
310  statement = l[i + 1:]
311  # it may happen (bug #37479) that the rest of the statement
312  # contains a comment.
313  if statement.lstrip().startswith("//"):
314  statement = ""
315  else:
316  statement += l
317 
318  l = f.readline()
319 
320  def _parse_units(self, file):
321  for line in open(file):
322  if '//' in line:
323  line = line[:line.index('//')]
324  line = line.strip()
325  if not line:
326  continue
327  nunit, value = line.split('=')
328  factor, unit = nunit.split()
329  value = eval(value) / eval(factor)
330  self.units[unit] = value
331 
332  def _eval_statement(self, statement):
333  from GaudiKernel.Proxy.Configurable import (
334  ConfigurableGeneric, Configurable, PropertyReference)
335  #statement = statement.replace("\n","").strip()
336  _log.info("%s%s", statement, self.statement_sep)
337 
338  property, value = statement.split("=", 1)
339 
340  inc = None
341  if property[-1] in ["+", "-"]:
342  inc = property[-1]
343  property = property[:-1]
344 
345  property = property.strip()
346  value = value.strip()
347 
348  # find the configurable to apply the property to
349  #parent_cfg = None
350  # while '.' in property:
351  # component, property = property.split('.',1)
352  # if parent_cfg:
353  # if hasattr(parent_cfg,component):
354  # cfg = getattr(parent_cfg,component)
355  # else:
356  # cfg = ConfigurableGeneric(component)
357  # setattr(parent_cfg,component,cfg)
358  # else:
359  # cfg = ConfigurableGeneric(component)
360  # parent_cfg = cfg
361 
362  # remove spaces around dots
363  property = '.'.join([w.strip() for w in property.split('.')])
364  component, property = property.rsplit('.', 1)
365  if component in Configurable.allConfigurables:
366  cfg = Configurable.allConfigurables[component]
367  else:
368  cfg = ConfigurableGeneric(component)
369 
370  #value = os.path.expandvars(value)
371  value = value.replace('true', 'True').replace('false', 'False')
372  if value[0] == '{':
373  # Try to guess if the values looks like a dictionary
374  if ':' in value and not (value[:value.index(':')].count('"') % 2 or
375  value[:value.index(':')].count("'") % 2):
376  # for dictionaries, keep the surrounding {}
377  value = '{' + \
378  value[1:-1].replace('{', '[').replace('}', ']') + '}'
379  else: # otherwise replace all {} with []
380  value = value.replace('{', '[').replace('}', ']')
381 
382  # We must escape '\' because eval tends to interpret them
383  value = value.replace('\\', '\\\\')
384  # Restore special cases ('\n', '\t' and '\"') (see GAUDI-1001)
385  value = (value.replace(r"\\n", r"\n").replace(r"\\t", r"\t").replace(
386  r'\\"', r'\"'))
387  # replace r'\n' and r'\t' that are outside double quoted strings
388  value = '"'.join([(v if i % 2 else re.sub(r'\\[nt]', ' ', v))
389  for i, v in enumerate(value.split('"'))])
390 
391  # interprete the @ operator
392  m = self.reference.match(value)
393  if m:
394  # this allows late binding of references
395  value = PropertyReference(m.group(1))
396  else:
397  value = eval(value, self.units)
398 
399  #if type(value) is str : value = os.path.expandvars(value)
400  # elif type(value) is list : value = [ type(item) is str and os.path.expandvars(item) or item for item in value ]
401 
402  if property not in cfg.__slots__ and not hasattr(cfg, property):
403  # check if the case of the property is wrong (old options are case insensitive)
404  lprop = property.lower()
405  for p in cfg.__slots__:
406  if lprop == p.lower():
407  _log.warning(
408  "property '%s' was requested for %s, but the correct spelling is '%s'",
409  property, cfg.name(), p)
410  property = p
411  break
412 
413  # consider the += and -=
414  if inc == "+":
415  if hasattr(cfg, property):
416  prop = getattr(cfg, property)
417  if type(prop) == dict:
418  for k in value:
419  prop[k] = value[k]
420  else:
421  prop += value
422  else:
423  setattr(cfg, property, value)
424  elif inc == "-":
425  if hasattr(cfg, property):
426  prop = getattr(cfg, property)
427  if type(prop) is dict:
428  for k in value:
429  if k in prop:
430  del prop[k]
431  else:
432  _log.warning("key '%s' not in %s.%s", k,
433  cfg.name(), property)
434  else:
435  for k in value:
436  if k in prop:
437  prop.remove(k)
438  else:
439  _log.warning("value '%s' not in %s.%s", k,
440  cfg.name(), property)
441  else:
442  setattr(cfg, property, value)
443 
444 
446  def __init__(self, new_path):
447  self.old_path = sys.path
448  sys.path = new_path
449 
450  def __del__(self):
451  sys.path = self.old_path
452 
453 
454 _parser = JobOptsParser()
455 
456 
457 def _import_python(file):
458  execfile(file, {})
459 
460 
461 def _import_pickle(file):
462  import pickle
463  input = open(file, 'rb')
464  catalog = pickle.load(input)
465  _log.info('Unpickled %d configurables', len(catalog))
466 
467 
468 def _import_opts(file):
469  _parser.parse(file)
470 
471 
472 _import_function_mapping = {
473  ".py": _import_python,
474  ".pkl": _import_pickle,
475  ".opts": _import_opts,
476 }
477 
478 
479 def importOptions(optsfile):
480  # expand environment variables before checking the extension
481  optsfile = os.path.expandvars(optsfile)
482  # check the file type (extension)
483  dummy, ext = os.path.splitext(optsfile)
484  if ext in _import_function_mapping:
485  # check if the file has been already included
486  optsfile = _find_file(optsfile)
487  if _to_be_included(optsfile):
488  _log.info("--> Including file '%s'", optsfile)
489  # include the file
490  _import_function_mapping[ext](optsfile)
491  _log.info("<-- End of file '%s'", optsfile)
492  else:
493  raise ParserError("Unknown file type '%s' ('%s')" % (ext, optsfile))
494 
495 
496 # Import a file containing declaration of units.
497 # It is equivalent to:
498 #
499 # #units "unitsfile.opts"
500 #
501 
502 
503 def importUnits(unitsfile):
504  # expand environment variables
505  unitsfile = os.path.expandvars(unitsfile)
506  # we do not need to check the file type (extension) because it must be a
507  # units file
508  _parser._include(unitsfile, _parser._parse_units)
def disable(self, allowed=logging.WARNING)
def __init__(self, stream=None, prefix=None, with_time=False)
def GetConsoleHandler(prefix=None, stream=None, with_time=False)
def enable(self, allowed=logging.WARNING)
def printOn(self, step=1, force=False)
def __init__(self, fmt=None, datefmt=None, prefix="# ", with_time=False)
def InstallRootLoggingHandler(prefix=None, level=None, stream=None, with_time=False)
def printOn(self, step=1, force=False)
def PrintOn(step=1, force=False)
def enable(self, allowed=logging.WARNING)
def disable(self, allowed=logging.WARNING)