ProcessJobOptions.py
Go to the documentation of this file.
1 import os, sys, re
2 import time
3 
4 import logging
5 _log = logging.getLogger(__name__)
6 
7 class LogFormatter(logging.Formatter):
8  def __init__(self, fmt=None, datefmt=None, prefix = "# ", with_time = False):
9  logging.Formatter.__init__(self, fmt, datefmt)
10  self.prefix = prefix
11  self.with_time = with_time
12  def format(self, record):
13  fmsg = logging.Formatter.format(self, record)
14  prefix = self.prefix
15  if self.with_time:
16  prefix += '%f ' % time.time()
17  if record.levelno >= logging.WARNING:
18  prefix += record.levelname + ": "
19  s = "\n".join([ prefix + line
20  for line in fmsg.splitlines() ])
21  return s
22 
23 class LogFilter(logging.Filter):
24  def __init__(self, name = ""):
25  logging.Filter.__init__(self, name)
26  self.printing_level = 0
27  self.enabled = True
28  self.threshold = logging.WARNING
29  def filter(self, record):
30  return record.levelno >= self.threshold or (self.enabled and self.printing_level <= 0)
31  def printOn(self, step = 1, force = False):
32  """
33  Decrease the printing_level of 'step' units. ( >0 means no print)
34  The level cannot go below 0, unless the force flag is set to True.
35  A negative value of the threshold disables subsequent "PrintOff"s.
36  """
37  if force:
38  self.printing_level -= step
39  else:
40  if self.printing_level > step:
41  self.printing_level -= step
42  else:
43  self.printing_level = 0
44  def printOff(self, step = 1):
45  """
46  Increase the printing_level of 'step' units. ( >0 means no print)
47  """
48  self.printing_level += step
49  def disable(self, allowed = logging.WARNING):
50  self.enabled = False
51  self.threshold = allowed
52  def enable(self, allowed = logging.WARNING):
53  self.enabled = True
54  self.threshold = allowed
55 
56 class ConsoleHandler(logging.StreamHandler):
57  def __init__(self, stream = None, prefix = None, with_time = False):
58  if stream is None:
59  stream = sys.stdout
60  logging.StreamHandler.__init__(self, stream)
61  if prefix is None:
62  prefix = "# "
63  self._filter = LogFilter(_log.name)
64  self._formatter = LogFormatter(prefix = prefix, with_time = with_time)
65  self.setFormatter(self._formatter)
66  self.addFilter(self._filter)
67  def setPrefix(self, prefix):
68  self._formatter.prefix = prefix
69  def printOn(self, step = 1, force = False):
70  """
71  Decrease the printing_level of 'step' units. ( >0 means no print)
72  The level cannot go below 0, unless the force flag is set to True.
73  A negative value of the threshold disables subsequent "PrintOff"s.
74  """
75  self._filter.printOn(step, force)
76  def printOff(self, step = 1):
77  """
78  Increase the printing_level of 'step' units. ( >0 means no print)
79  """
80  self._filter.printOff(step)
81  def disable(self, allowed = logging.WARNING):
82  self._filter.disable(allowed)
83  def enable(self, allowed = logging.WARNING):
84  self._filter.enable(allowed)
85 
86 _consoleHandler = None
87 def GetConsoleHandler(prefix = None, stream = None, with_time = False):
88  global _consoleHandler
89  if _consoleHandler is None:
90  _consoleHandler = ConsoleHandler(prefix = prefix, stream = stream, with_time = with_time)
91  elif prefix is not None:
92  _consoleHandler.setPrefix(prefix)
93  return _consoleHandler
94 
95 def InstallRootLoggingHandler(prefix = None, level = None, stream = None, with_time = False):
96  root_logger = logging.getLogger()
97  if not root_logger.handlers:
98  root_logger.addHandler(GetConsoleHandler(prefix, stream, with_time))
99  root_logger.setLevel(logging.WARNING)
100  if level is not None:
101  root_logger.setLevel(level)
102 
103 def PrintOn(step = 1, force = False):
104  GetConsoleHandler().printOn(step, force)
105 def PrintOff(step = 1):
106  GetConsoleHandler().printOff(step)
107 
108 class ParserError(RuntimeError):
109  pass
110 
111 def _find_file(f):
112  # expand environment variables in the filename
113  f = os.path.expandvars(f)
114  if os.path.isfile(f):
115  return os.path.realpath(f)
116 
117  path = os.environ.get('JOBOPTSEARCHPATH','').split(os.pathsep)
118  # find the full path to the option file
119  candidates = [d for d in path if os.path.isfile(os.path.join(d,f))]
120  if not candidates:
121  raise ParserError("Cannot find '%s' in %s" % (f,path))
122  return os.path.realpath(os.path.join(candidates[0],f))
123 
124 _included_files = set()
126  if f in _included_files:
127  _log.warning("file '%s' already included, ignored.", f)
128  return False
129  _included_files.add(f)
130  return True
131 
133  comment = re.compile(r'(//.*)$')
134  # non-perfect R-E to check if '//' is inside a string
135  # (a tokenizer would be better)
136  comment_in_string = re.compile(r'(["\']).*//.*\1')
137  directive = re.compile(r'^\s*#\s*([\w!]+)\s*(.*)\s*$')
138  comment_ml = ( re.compile(r'/\*'), re.compile(r'\*/') )
139  statement_sep = ";"
140  reference = re.compile(r'^@([\w.]*)$')
141 
142  def __init__(self):
143  # parser level states
144  self.units = {}
145  self.defines = {}
146  if sys.platform != 'win32':
147  self.defines[ "WIN32" ] = True
148 
149  def _include(self,file,function):
150  file = _find_file(file)
151  if _to_be_included(file):
152  _log.info("--> Including file '%s'", file)
153  function(file)
154  _log.info("<-- End of file '%s'", file)
155 
156  def parse(self,file):
157  # states for the "translation unit"
158  statement = ""
159 
160  ifdef_level = 0
161  ifdef_skipping = False
162  ifdef_skipping_level = 0
163 
164  in_string = False
165 
166  f = open(_find_file(file))
167  l = f.readline()
168  if l.startswith("#!"):
169  # Skip the first line if it starts with "#!".
170  # It allows to use options files as scripts.
171  l = f.readline()
172 
173  while l:
174  l = l.rstrip()+'\n' # normalize EOL chars (to avoid problems with DOS new-line on Unix)
175 
176  # single line comment
177  m = self.comment.search(l)
178  if m:
179  # check if the '//' is part of a string
180  m2 = self.comment_in_string.search(l)
181  # the '//' is part of a string if we find the quotes around it
182  # and they are not part of the comment itself
183  if not ( m2 and m2.start() < m.start() ):
184  # if it is not the case, we can remove the comment from the
185  # statement
186  l = l[:m.start()]+l[m.end():]
187  # process directives
188  m = self.directive.search(l)
189  if m:
190  directive_name = m.group(1)
191  directive_arg = m.group(2).strip()
192  if directive_name == "include":
193  included_file = directive_arg.strip("'\"")
194  importOptions(included_file)
195  elif directive_name == "units":
196  units_file = directive_arg.strip("'\"")
197  self._include(units_file,self._parse_units)
198  elif directive_name in [ "ifdef", "ifndef"]:
199  ifdef_skipping_level = ifdef_level
200  ifdef_level += 1
201  if directive_arg in self.defines:
202  ifdef_skipping = directive_name == "ifndef"
203  else:
204  ifdef_skipping = directive_name == "ifdef"
205  elif directive_name == "else":
206  ifdef_skipping = not ifdef_skipping
207  elif directive_name == "endif":
208  ifdef_level -= 1
209  if ifdef_skipping and ifdef_skipping_level == ifdef_level:
210  ifdef_skipping = False
211  elif directive_name == "pragma":
212  if not directive_arg:
213  l = f.readline()
214  continue
215  pragma = directive_arg.split()
216  if pragma[0] == "print":
217  if len(pragma) > 1:
218  if pragma[1].upper() in [ "ON", "TRUE", "1" ]:
219  PrintOn()
220  else:
221  PrintOff()
222  else:
223  _log.warning("unknown directive '%s'", directive_name)
224  l = f.readline()
225  continue
226 
227  if ifdef_skipping:
228  l = f.readline()
229  continue
230 
231  # multi-line comment
232  m = self.comment_ml[0].search(l)
233  if m:
234  l,l1 = l[:m.start()],l[m.end():]
235  m = self.comment_ml[1].search(l1)
236  while not m:
237  l1 = f.readline()
238  if not l1:
239  break # EOF
240  m = self.comment_ml[1].search(l1)
241  if not l1 and not m:
242  raise ParserError("End Of File reached before end of multi-line comment")
243  l += l1[m.end():]
244 
245  # if we are in a multiline string, we add to the statement
246  # everything until the next '"'
247  if in_string:
248  string_end = l.find('"')
249  if string_end >= 0:
250  statement += l[:string_end+1]
251  l = l[string_end+1:]
252  in_string = False # the string ends here
253  else:
254  statement += l
255  l = ''
256  else: # check if we have a string
257  string_start = l.find('"')
258  if string_start >= 0:
259  string_end = l.find('"', string_start + 1)
260  if string_end >= 0:
261  # the string is opened and closed
262  statement += l[:string_end+1]
263  l = l[string_end+1:]
264  else:
265  # the string is only opened
266  statement += l
267  in_string = True
268  l = f.readline()
269  continue
270 
271  if self.statement_sep in l:
272  i = l.index(self.statement_sep)
273  statement += l[:i]
274  self._eval_statement(statement.strip().replace("\n","\\n"))
275  statement = l[i+1:]
276  # it may happen (bug #37479) that the rest of the statement
277  # contains a comment.
278  if statement.lstrip().startswith("//"):
279  statement = ""
280  else:
281  statement += l
282 
283  l = f.readline()
284 
285  def _parse_units(self,file):
286  for line in open(file):
287  if '//' in line:
288  line = line[:line.index('//')]
289  line = line.strip()
290  if not line:
291  continue
292  nunit, value = line.split('=')
293  factor, unit = nunit.split()
294  value = eval(value)/eval(factor)
295  self.units[unit] = value
296 
297  def _eval_statement(self,statement):
298  from GaudiKernel.Proxy.Configurable import (ConfigurableGeneric,
299  Configurable,
300  PropertyReference)
301  #statement = statement.replace("\n","").strip()
302  _log.info("%s%s", statement, self.statement_sep)
303 
304  property,value = statement.split("=",1)
305 
306  inc = None
307  if property[-1] in [ "+", "-" ]:
308  inc = property[-1]
309  property = property[:-1]
310 
311  property = property.strip()
312  value = value.strip()
313 
314  ## find the configurable to apply the property to
315  #parent_cfg = None
316  #while '.' in property:
317  # component, property = property.split('.',1)
318  # if parent_cfg:
319  # if hasattr(parent_cfg,component):
320  # cfg = getattr(parent_cfg,component)
321  # else:
322  # cfg = ConfigurableGeneric(component)
323  # setattr(parent_cfg,component,cfg)
324  # else:
325  # cfg = ConfigurableGeneric(component)
326  # parent_cfg = cfg
327 
328  # remove spaces around dots
329  property = '.'.join([w.strip() for w in property.split('.')])
330  component, property = property.rsplit('.',1)
331  if component in Configurable.allConfigurables:
332  cfg = Configurable.allConfigurables[component]
333  else:
334  cfg = ConfigurableGeneric(component)
335 
336  #value = os.path.expandvars(value)
337  value = value.replace('true','True').replace('false','False')
338  if value[0] == '{' :
339  # Try to guess if the values looks like a dictionary
340  if ':' in value and not ( value[:value.index(':')].count('"')%2 or value[:value.index(':')].count("'")%2 ) :
341  # for dictionaries, keep the surrounding {}
342  value = '{'+value[1:-1].replace('{','[').replace('}',']')+'}'
343  else : # otherwise replace all {} with []
344  value = value.replace('{','[').replace('}',']')
345 
346  # We must escape '\' because eval tends to interpret them
347  value = value.replace('\\', '\\\\')
348  # Restore special cases ('\n', '\t' and '\"') (see GAUDI-1001)
349  value = (value.replace(r"\\n", r"\n")
350  .replace(r"\\t", r"\t")
351  .replace(r'\\"', r'\"'))
352  # replace r'\n' and r'\t' that are outside double quoted strings
353  value = '"'.join([(v if i % 2 else re.sub(r'\\[nt]', ' ', v))
354  for i, v in enumerate(value.split('"'))])
355 
356  # interprete the @ operator
357  m = self.reference.match(value)
358  if m:
359  # this allows late binding of references
360  value = PropertyReference(m.group(1))
361  else:
362  value = eval(value,self.units)
363 
364  #if type(value) is str : value = os.path.expandvars(value)
365  #elif type(value) is list : value = [ type(item) is str and os.path.expandvars(item) or item for item in value ]
366 
367  if property not in cfg.__slots__ and not hasattr(cfg,property):
368  # check if the case of the property is wrong (old options are case insensitive)
369  lprop = property.lower()
370  for p in cfg.__slots__:
371  if lprop == p.lower():
372  _log.warning("property '%s' was requested for %s, but the correct spelling is '%s'", property, cfg.name(), p)
373  property = p
374  break
375 
376  # consider the += and -=
377  if inc == "+":
378  if hasattr(cfg,property):
379  prop = getattr(cfg,property)
380  if type(prop) == dict:
381  for k in value:
382  prop[k] = value[k]
383  else:
384  prop += value
385  else:
386  setattr(cfg,property,value)
387  elif inc == "-":
388  if hasattr(cfg,property):
389  prop = getattr(cfg,property)
390  if type(prop) is dict:
391  for k in value:
392  if k in prop:
393  del prop[k]
394  else:
395  _log.warning("key '%s' not in %s.%s", k, cfg.name(), property)
396  else:
397  for k in value:
398  if k in prop:
399  prop.remove(k)
400  else:
401  _log.warning("value '%s' not in %s.%s", k, cfg.name(), property)
402  else:
403  setattr(cfg,property,value)
404 
406  def __init__(self, new_path):
407  self.old_path = sys.path
408  sys.path = new_path
409  def __del__(self):
410  sys.path = self.old_path
411 
412 _parser = JobOptsParser()
413 
414 def _import_python(file):
415  execfile(file, {})
416 
417 def _import_pickle(file):
418  import pickle
419  input = open(file, 'rb')
420  catalog = pickle.load(input)
421  _log.info('Unpickled %d configurables', len(catalog))
422 
423 def _import_opts(file):
424  _parser.parse(file)
425 
426 _import_function_mapping = {
427  ".py" : _import_python,
428  ".pkl" : _import_pickle,
429  ".opts" : _import_opts,
430  }
431 
432 def importOptions( optsfile ) :
433  # expand environment variables before checking the extension
434  optsfile = os.path.expandvars(optsfile)
435  # check the file type (extension)
436  dummy, ext = os.path.splitext(optsfile)
437  if ext in _import_function_mapping:
438  # check if the file has been already included
439  optsfile = _find_file(optsfile)
440  if _to_be_included(optsfile):
441  _log.info("--> Including file '%s'", optsfile)
442  # include the file
443  _import_function_mapping[ext](optsfile)
444  _log.info("<-- End of file '%s'", optsfile)
445  else:
446  raise ParserError("Unknown file type '%s' ('%s')" % (ext,optsfile))
447 
448 ## Import a file containing declaration of units.
449 # It is equivalent to:
450 #
451 # #units "unitsfile.opts"
452 #
453 def importUnits(unitsfile):
454  # expand environment variables
455  unitsfile = os.path.expandvars(unitsfile)
456  # we do not need to check the file type (extension) because it must be a
457  # units file
458  _parser._include(unitsfile, _parser._parse_units)
def disable(self, allowed=logging.WARNING)
def __init__(self, stream=None, prefix=None, with_time=False)
def GetConsoleHandler(prefix=None, stream=None, with_time=False)
def enable(self, allowed=logging.WARNING)
def printOn(self, step=1, force=False)
def __init__(self, fmt=None, datefmt=None, prefix="# ", with_time=False)
def InstallRootLoggingHandler(prefix=None, level=None, stream=None, with_time=False)
def printOn(self, step=1, force=False)
def PrintOn(step=1, force=False)
def enable(self, allowed=logging.WARNING)
def disable(self, allowed=logging.WARNING)
def importUnits(unitsfile)
Import a file containing declaration of units.