Loading [MathJax]/extensions/tex2jax.js
The Gaudi Framework  v31r0 (aeb156f0)
All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Modules Pages
Main.py
Go to the documentation of this file.
1 import sys
2 import os
3 from time import time
4 from Gaudi import Configuration
5 import logging
6 
7 log = logging.getLogger(__name__)
8 
9 
10 class BootstrapHelper(object):
11  class StatusCode(object):
12  def __init__(self, value):
13  self.value = value
14 
15  def __bool__(self):
16  return self.value
17 
18  __nonzero__ = __bool__
19 
20  def isSuccess(self):
21  return self.value
22 
23  def isFailure(self):
24  return not self.value
25 
26  def ignore(self):
27  pass
28 
29  class Property(object):
30  def __init__(self, value):
31  self.value = value
32 
33  def __str__(self):
34  return str(self.value)
35 
36  toString = __str__
37 
38  class AppMgr(object):
39  def __init__(self, ptr, lib):
40  self.ptr = ptr
41  self.lib = lib
42  self._as_parameter_ = ptr
43 
44  def configure(self):
46  self.lib.py_bootstrap_fsm_configure(self.ptr))
47 
48  def initialize(self):
50  self.lib.py_bootstrap_fsm_initialize(self.ptr))
51 
52  def start(self):
54  self.lib.py_bootstrap_fsm_start(self.ptr))
55 
56  def run(self, nevt):
58  self.lib.py_bootstrap_app_run(self.ptr, nevt))
59 
60  def stop(self):
62  self.lib.py_bootstrap_fsm_stop(self.ptr))
63 
64  def finalize(self):
66  self.lib.py_bootstrap_fsm_finalize(self.ptr))
67 
68  def terminate(self):
70  self.lib.py_bootstrap_fsm_terminate(self.ptr))
71 
72  def getService(self, name):
73  return self.lib.py_bootstrap_getService(self.ptr, name)
74 
75  def setProperty(self, name, value):
77  self.lib.py_bootstrap_setProperty(self.ptr, name, value))
78 
79  def getProperty(self, name):
81  self.lib.py_bootstrap_getProperty(self.ptr, name))
82 
83  def printAlgsSequences(self):
84  return self.lib.py_helper_printAlgsSequences(self.ptr)
85 
86  def __init__(self):
87  from ctypes import (PyDLL, util, c_void_p, c_bool, c_char_p, c_int,
88  RTLD_GLOBAL)
89 
90  # Helper class to avoid void* to int conversion
91  # (see http://stackoverflow.com/questions/17840144)
92 
93  class IInterface_p(c_void_p):
94  def __repr__(self):
95  return "IInterface_p(0x%x)" % (0 if self.value is None else
96  self.value)
97 
98  self.log = logging.getLogger('BootstrapHelper')
99  libname = util.find_library('GaudiKernel') or 'libGaudiKernel.so'
100  self.log.debug('loading GaudiKernel (%s)', libname)
101 
102  # FIXME: note that we need PyDLL instead of CDLL if the calls to
103  # Python functions are not protected with the GIL.
104  self.lib = gkl = PyDLL(libname, mode=RTLD_GLOBAL)
105 
106  functions = [
107  ('createApplicationMgr', IInterface_p, []),
108  ('getService', IInterface_p, [IInterface_p, c_char_p]),
109  ('setProperty', c_bool, [IInterface_p, c_char_p, c_char_p]),
110  ('getProperty', c_char_p, [IInterface_p, c_char_p]),
111  ('addPropertyToCatalogue', c_bool,
112  [IInterface_p, c_char_p, c_char_p, c_char_p]),
113  ('ROOT_VERSION_CODE', c_int, []),
114  ]
115 
116  for name, restype, argtypes in functions:
117  f = getattr(gkl, 'py_bootstrap_%s' % name)
118  f.restype, f.argtypes = restype, argtypes
119  # create a delegate method if not already present
120  # (we do not want to use hasattr because it calls "properties")
121  if name not in self.__class__.__dict__:
122  setattr(self, name, f)
123 
124  for name in ('configure', 'initialize', 'start', 'stop', 'finalize',
125  'terminate'):
126  f = getattr(gkl, 'py_bootstrap_fsm_%s' % name)
127  f.restype, f.argtypes = c_bool, [IInterface_p]
128  gkl.py_bootstrap_app_run.restype = c_bool
129  gkl.py_bootstrap_app_run.argtypes = [IInterface_p, c_int]
130 
131  gkl.py_helper_printAlgsSequences.restype = None
132  gkl.py_helper_printAlgsSequences.argtypes = [IInterface_p]
133 
135  ptr = self.lib.py_bootstrap_createApplicationMgr()
136  return self.AppMgr(ptr, self.lib)
137 
138  @property
139  def ROOT_VERSION_CODE(self):
140  return self.lib.py_bootstrap_ROOT_VERSION_CODE()
141 
142  @property
143  def ROOT_VERSION(self):
144  root_version_code = self.ROOT_VERSION_CODE
145  a = root_version_code >> 16 & 0xff
146  b = root_version_code >> 8 & 0xff
147  c = root_version_code & 0xff
148  return (a, b, c)
149 
150 
151 _bootstrap = None
152 
153 
154 def toOpt(value):
155  '''
156  Helper to convert values to old .opts format.
157 
158  >>> print toOpt('some "text"')
159  "some \\"text\\""
160  >>> print toOpt('first\\nsecond')
161  "first
162  second"
163  >>> print toOpt({'a': [1, 2, '3']})
164  {"a": [1, 2, "3"]}
165  '''
166  if isinstance(value, basestring):
167  return '"{0}"'.format(value.replace('"', '\\"'))
168  elif isinstance(value, dict):
169  return '{{{0}}}'.format(', '.join(
170  '{0}: {1}'.format(toOpt(k), toOpt(v))
171  for k, v in value.iteritems()))
172  elif hasattr(value, '__iter__'):
173  return '[{0}]'.format(', '.join(map(toOpt, value)))
174  else:
175  return repr(value)
176 
177 
178 class gaudimain(object):
179  # main loop implementation, None stands for the default
180  mainLoop = None
181 
182  def __init__(self):
183  from Configurables import ApplicationMgr
184  appMgr = ApplicationMgr()
185  if "GAUDIAPPNAME" in os.environ:
186  appMgr.AppName = str(os.environ["GAUDIAPPNAME"])
187  if "GAUDIAPPVERSION" in os.environ:
188  appMgr.AppVersion = str(os.environ["GAUDIAPPVERSION"])
189  self.log = logging.getLogger(__name__)
190  self.printsequence = False
191 
193  # ---------------------------------------------------
194  # set up Logging
195  # ----------------
196  # from multiprocessing import enableLogging, getLogger
197  import multiprocessing
198  # preliminaries for handlers/output files, etc.
199  from time import ctime
200  datetime = ctime()
201  datetime = datetime.replace(' ', '_')
202  outfile = open('gaudirun-%s.log' % (datetime), 'w')
203  # two handlers, one for a log file, one for terminal
204  streamhandler = logging.StreamHandler(stream=outfile)
205  console = logging.StreamHandler()
206  # create formatter : the params in parentheses are variable names available via logging
207  formatter = logging.Formatter(
208  "%(asctime)s - %(name)s - %(levelname)s - %(message)s")
209  # add formatter to Handler
210  streamhandler.setFormatter(formatter)
211  console.setFormatter(formatter)
212  # now, configure the logger
213  # enableLogging( level=0 )
214  # self.log = getLogger()
215  self.log = multiprocessing.log_to_stderr()
216  self.log.setLevel(logging.INFO)
217  self.log.name = 'Gaudi/Main.py Logger'
218  self.log.handlers = []
219  # add handlers to logger : one for output to a file, one for console output
220  self.log.addHandler(streamhandler)
221  self.log.addHandler(console)
222  self.log.removeHandler(console)
223  # set level!!
224  self.log.setLevel = logging.INFO
225  # ---------------------------------------------------
226 
227  def generatePyOutput(self, all=False):
228  from pprint import pformat
229  conf_dict = Configuration.configurationDict(all)
230  return pformat(conf_dict)
231 
232  def generateOptsOutput(self, all=False):
233  from pprint import pformat
234  conf_dict = Configuration.configurationDict(all)
235  out = []
236  names = conf_dict.keys()
237  names.sort()
238  for n in names:
239  props = conf_dict[n].keys()
240  props.sort()
241  for p in props:
242  out.append('%s.%s = %s;' % (n, p, toOpt(conf_dict[n][p])))
243  return "\n".join(out)
244 
245  def _writepickle(self, filename):
246  # --- Lets take the first file input file as the name of the pickle file
247  import pickle
248  output = open(filename, 'wb')
249  # Dump only the the configurables that make sense to dump (not User ones)
250  from GaudiKernel.Proxy.Configurable import getNeededConfigurables
251  to_dump = {}
252  for n in getNeededConfigurables():
253  to_dump[n] = Configuration.allConfigurables[n]
254  pickle.dump(to_dump, output, -1)
255  output.close()
256 
257  def printconfig(self, old_format=False, all=False):
258  msg = 'Dumping all configurables and properties'
259  if not all:
260  msg += ' (different from default)'
261  log.info(msg)
262  conf_dict = Configuration.configurationDict(all)
263  if old_format:
264  print self.generateOptsOutput(all)
265  else:
266  print self.generatePyOutput(all)
267 
268  def writeconfig(self, filename, all=False):
269  write = {".pkl": lambda filename, all: self._writepickle(filename),
270  ".py": lambda filename, all: open(filename, "w").write(self.generatePyOutput(all) + "\n"),
271  ".opts": lambda filename, all: open(filename, "w").write(self.generateOptsOutput(all) + "\n"),
272  }
273  from os.path import splitext
274  ext = splitext(filename)[1]
275  if ext in write:
276  write[ext](filename, all)
277  else:
278  log.error("Unknown file type '%s'. Must be any of %r.", ext,
279  write.keys())
280  sys.exit(1)
281 
282  # Instantiate and run the application.
283  # Depending on the number of CPUs (ncpus) specified, it start
284  def run(self, attach_debugger, ncpus=None):
285  if not ncpus:
286  # Standard sequential mode
287  result = self.runSerial(attach_debugger)
288  else:
289  # Otherwise, run with the specified number of cpus
290  result = self.runParallel(ncpus)
291  return result
292 
293  def hookDebugger(self, debugger='gdb'):
294  import os
295  self.log.info('attaching debugger to PID ' + str(os.getpid()))
296  pid = os.spawnvp(os.P_NOWAIT, debugger,
297  [debugger, '-q', 'python',
298  str(os.getpid())])
299 
300  # give debugger some time to attach to the python process
301  import time
302  time.sleep(5)
303 
304  # verify the process' existence (will raise OSError if failed)
305  os.waitpid(pid, os.WNOHANG)
306  os.kill(pid, 0)
307  return
308 
309  def basicInit(self):
310  '''
311  Bootstrap the application with minimal use of Python bindings.
312  '''
313  try:
314  from GaudiKernel.Proxy.Configurable import expandvars
315  except ImportError:
316  # pass-through implementation if expandvars is not defined (AthenaCommon)
317  def expandvars(data):
318  return data
319 
320  from GaudiKernel.Proxy.Configurable import Configurable, getNeededConfigurables
321 
322  global _bootstrap
323  if _bootstrap is None:
324  _bootstrap = BootstrapHelper()
325 
326  self.log.debug('basicInit: instantiate ApplicationMgr')
327  self.ip = self.g = _bootstrap.createApplicationMgr()
328 
329  self.log.debug('basicInit: apply options')
330 
331  # set ApplicationMgr properties
332  comp = 'ApplicationMgr'
333  props = Configurable.allConfigurables.get(comp, {})
334  if props:
335  props = expandvars(props.getValuedProperties())
336  for p, v in props.items() + [('JobOptionsType', 'NONE')]:
337  if not self.g.setProperty(p, str(v)):
338  self.log.error('Cannot set property %s.%s to %s', comp, p, v)
339  sys.exit(10)
340  # issue with missing dictionary with ROOT < 6.2.7
341  if _bootstrap.ROOT_VERSION < (6, 2, 7):
342  # we need to load GaudiPython
343  import GaudiPython
344 
345  self.g.configure()
346 
347  # set MessageSvc properties
348  comp = 'MessageSvc'
349  msp = self.g.getService(comp)
350  if not msp:
351  self.log.error('Cannot get service %s', comp)
352  sys.exit(10)
353  props = Configurable.allConfigurables.get(comp, {})
354  if props:
355  props = expandvars(props.getValuedProperties())
356  for p, v in props.items():
357  if not _bootstrap.setProperty(msp, p, str(v)):
358  self.log.error('Cannot set property %s.%s to %s', comp, p, v)
359  sys.exit(10)
360 
361  # feed JobOptionsSvc
362  comp = 'JobOptionsSvc'
363  jos = self.g.getService(comp)
364  if not jos:
365  self.log.error('Cannot get service %s', comp)
366  sys.exit(10)
367  for n in getNeededConfigurables():
368  c = Configurable.allConfigurables[n]
369  if n in ['ApplicationMgr', 'MessageSvc']:
370  continue # These are already done
371  for p, v in c.getValuedProperties().items():
372  v = expandvars(v)
373  # Note: AthenaCommon.Configurable does not have Configurable.PropertyReference
374  if hasattr(Configurable, "PropertyReference") and type(
375  v) == Configurable.PropertyReference:
376  # this is done in "getFullName", but the exception is ignored,
377  # so we do it again to get it
378  v = v.__resolve__()
379  if type(v) == str:
380  v = '"%s"' % v # need double quotes
381  elif type(v) == long:
382  v = '%d' % v # prevent pending 'L'
383  _bootstrap.addPropertyToCatalogue(jos, n, p, str(v))
384  if hasattr(Configurable, "_configurationLocked"):
385  Configurable._configurationLocked = True
386  self.log.debug('basicInit: done')
387 
388  def gaudiPythonInit(self):
389  '''
390  Initialize the application with full Python bindings.
391  '''
392  self.log.debug('gaudiPythonInit: import GaudiPython')
393  import GaudiPython
394  self.log.debug('gaudiPythonInit: instantiate ApplicationMgr')
395  self.g = GaudiPython.AppMgr()
396  self.ip = self.g._ip
397  self.log.debug('gaudiPythonInit: done')
398 
399  def runSerial(self, attach_debugger):
400  # --- Instantiate the ApplicationMgr------------------------------
401  if (self.mainLoop or os.environ.get('GAUDIRUN_USE_GAUDIPYTHON')):
402  self.gaudiPythonInit()
403  else:
404  self.basicInit()
405 
406  self.log.debug('-' * 80)
407  self.log.debug('%s: running in serial mode', __name__)
408  self.log.debug('-' * 80)
409  sysStart = time()
410 
411  if self.mainLoop:
412  runner = self.mainLoop
413  else:
414 
415  def runner(app, nevt):
416  self.log.debug('initialize')
417  sc = app.initialize()
418  if sc.isSuccess():
419  if self.printsequence:
420  app.printAlgsSequences()
421  self.log.debug('start')
422  sc = app.start()
423  if sc.isSuccess():
424  self.log.debug('run(%d)', nevt)
425  sc = app.run(nevt)
426  self.log.debug('stop')
427  app.stop().ignore()
428  self.log.debug('finalize')
429  app.finalize().ignore()
430  self.log.debug('terminate')
431  sc1 = app.terminate()
432  if sc.isSuccess():
433  sc = sc1
434  else:
435  sc1.ignore()
436  self.log.debug('status code: %s',
437  'SUCCESS' if sc.isSuccess() else 'FAILURE')
438  return sc
439 
440  if (attach_debugger == True):
441  self.hookDebugger()
442 
443  try:
444  statuscode = runner(self.g,
445  int(self.ip.getProperty('EvtMax').toString()))
446  except SystemError:
447  # It may not be 100% correct, but usually it means a segfault in C++
448  self.ip.setProperty('ReturnCode', str(128 + 11))
449  statuscode = False
450  except Exception, x:
451  print 'Exception:', x
452  # for other exceptions, just set a generic error code
453  self.ip.setProperty('ReturnCode', '1')
454  statuscode = False
455  if hasattr(statuscode, "isSuccess"):
456  success = statuscode.isSuccess()
457  else:
458  success = statuscode
459  #success = self.g.exit().isSuccess() and success
460  if not success and self.ip.getProperty('ReturnCode').toString() == '0':
461  # ensure that the return code is correctly set
462  self.ip.setProperty('ReturnCode', '1')
463  sysTime = time() - sysStart
464  self.log.debug('-' * 80)
465  self.log.debug('%s: serial system finished, time taken: %5.4fs',
466  __name__, sysTime)
467  self.log.debug('-' * 80)
468  return int(self.ip.getProperty('ReturnCode').toString())
469 
470  def runParallel(self, ncpus):
471  if self.mainLoop:
472  self.log.fatal(
473  "Cannot use custom main loop in multi-process mode, check your options"
474  )
475  return 1
476  self.setupParallelLogging()
477  from Gaudi.Configuration import Configurable
478  import GaudiMP.GMPBase as gpp
479  c = Configurable.allConfigurables
480  self.log.info('-' * 80)
481  self.log.info('%s: Parallel Mode : %i ', __name__, ncpus)
482  for name, value in [
483  ('platrofm', ' '.join(os.uname())),
484  ('config', os.environ.get('BINARY_TAG')
485  or os.environ.get('CMTCONFIG')),
486  ('app. name', os.environ.get('GAUDIAPPNAME')),
487  ('app. version', os.environ.get('GAUDIAPPVERSION')),
488  ]:
489  self.log.info('%s: %30s : %s ', __name__, name, value
490  or 'Undefined')
491  try:
492  events = str(c['ApplicationMgr'].EvtMax)
493  except:
494  events = "Undetermined"
495  self.log.info('%s: Events Specified : %s ', __name__, events)
496  self.log.info('-' * 80)
497  # Parall = gpp.Coordinator(ncpus, shared, c, self.log)
498  Parall = gpp.Coord(ncpus, c, self.log)
499  sysStart = time()
500  sc = Parall.Go()
501  self.log.info('MAIN.PY : received %s from Coordinator' % (sc))
502  if sc.isFailure():
503  return 1
504  sysTime = time() - sysStart
505  self.log.name = 'Gaudi/Main.py Logger'
506  self.log.info('-' * 80)
507  self.log.info('%s: parallel system finished, time taken: %5.4fs',
508  __name__, sysTime)
509  self.log.info('-' * 80)
510  return 0
def run(self, attach_debugger, ncpus=None)
Definition: Main.py:284
def setupParallelLogging(self)
Definition: Main.py:192
def basicInit(self)
Definition: Main.py:309
GAUDI_API std::string format(const char *,...)
MsgStream format utility "a la sprintf(...)".
Definition: MsgStream.cpp:109
def toOpt(value)
Definition: Main.py:154
def _writepickle(self, filename)
Definition: Main.py:245
def generateOptsOutput(self, all=False)
Definition: Main.py:232
def gaudiPythonInit(self)
Definition: Main.py:388
getNeededConfigurables
Definition: Proxy.py:21
def runParallel(self, ncpus)
Definition: Main.py:470
def hookDebugger(self, debugger='gdb')
Definition: Main.py:293
def generatePyOutput(self, all=False)
Definition: Main.py:227
def __init__(self, ptr, lib)
Definition: Main.py:39
struct GAUDI_API map
Parametrisation class for map-like implementation.
def getProperty(self, name)
Definition: Main.py:79
def getService(self, name)
Definition: Main.py:72
def createApplicationMgr(self)
Definition: Main.py:134
def __init__(self, value)
Definition: Main.py:30
The Application Manager class.
def ROOT_VERSION_CODE(self)
Definition: Main.py:139
def __init__(self)
Definition: Main.py:182
def __init__(self)
Definition: Main.py:86
def printconfig(self, old_format=False, all=False)
Definition: Main.py:257
def writeconfig(self, filename, all=False)
Definition: Main.py:268
def ROOT_VERSION(self)
Definition: Main.py:143
std::string toString(const Type &)
def setProperty(self, name, value)
Definition: Main.py:75
def runSerial(self, attach_debugger)
Definition: Main.py:399