| The Gaudi Framework v32r1 (f65d50dc)

Main.py
Go to the documentation of this file.
1 import sys
2 import os
3 from time import time
4 from Gaudi import Configuration
5 import logging
6
7 log = logging.getLogger(__name__)
8
9
10 class BootstrapHelper(object):
11 class StatusCode(object):
12 def __init__(self, value):
13 self.value = value
14
15 def __bool__(self):
16 return self.value
17
18 __nonzero__ = __bool__
19
20 def isSuccess(self):
21 return self.value
22
23 def isFailure(self):
24 return not self.value
25
26 def ignore(self):
27 pass
28
29 class Property(object):
30 def __init__(self, value):
31 self.value = value
32
33 def __str__(self):
34 return str(self.value)
35
36 toString = __str__
37
38 class AppMgr(object):
39 def __init__(self, ptr, lib):
40 self.ptr = ptr
41 self.lib = lib
42 self._as_parameter_ = ptr
43
44 def configure(self):
45 return BootstrapHelper.StatusCode(
46 self.lib.py_bootstrap_fsm_configure(self.ptr))
47
48 def initialize(self):
49 return BootstrapHelper.StatusCode(
50 self.lib.py_bootstrap_fsm_initialize(self.ptr))
51
52 def start(self):
53 return BootstrapHelper.StatusCode(
54 self.lib.py_bootstrap_fsm_start(self.ptr))
55
56 def run(self, nevt):
57 return BootstrapHelper.StatusCode(
58 self.lib.py_bootstrap_app_run(self.ptr, nevt))
59
60 def stop(self):
61 return BootstrapHelper.StatusCode(
62 self.lib.py_bootstrap_fsm_stop(self.ptr))
63
64 def finalize(self):
65 return BootstrapHelper.StatusCode(
66 self.lib.py_bootstrap_fsm_finalize(self.ptr))
67
68 def terminate(self):
69 return BootstrapHelper.StatusCode(
70 self.lib.py_bootstrap_fsm_terminate(self.ptr))
71
72 def getService(self, name):
73 return self.lib.py_bootstrap_getService(self.ptr, name)
74
75 def setProperty(self, name, value):
76 return BootstrapHelper.StatusCode(
77 self.lib.py_bootstrap_setProperty(self.ptr, name, value))
78
79 def getProperty(self, name):
80 return BootstrapHelper.Property(
81 self.lib.py_bootstrap_getProperty(self.ptr, name))
82
83 def printAlgsSequences(self):
84 return self.lib.py_helper_printAlgsSequences(self.ptr)
85
86 def __init__(self):
87 from ctypes import (PyDLL, util, c_void_p, c_bool, c_char_p, c_int,
88 RTLD_GLOBAL)
89
90 # Helper class to avoid void* to int conversion
91 # (see http://stackoverflow.com/questions/17840144)
92
93 class IInterface_p(c_void_p):
94 def __repr__(self):
95 return "IInterface_p(0x%x)" % (0 if self.value is None else
96 self.value)
97
98 self.log = logging.getLogger('BootstrapHelper')
99 libname = util.find_library('GaudiKernel') or 'libGaudiKernel.so'
100 self.log.debug('loading GaudiKernel (%s)', libname)
101
102 # FIXME: note that we need PyDLL instead of CDLL if the calls to
103 # Python functions are not protected with the GIL.
104 self.lib = gkl = PyDLL(libname, mode=RTLD_GLOBAL)
105
106 functions = [
107 ('createApplicationMgr', IInterface_p, []),
108 ('getService', IInterface_p, [IInterface_p, c_char_p]),
109 ('setProperty', c_bool, [IInterface_p, c_char_p, c_char_p]),
110 ('getProperty', c_char_p, [IInterface_p, c_char_p]),
111 ('addPropertyToCatalogue', c_bool,
112 [IInterface_p, c_char_p, c_char_p, c_char_p]),
113 ('ROOT_VERSION_CODE', c_int, []),
114 ]
115
116 for name, restype, argtypes in functions:
117 f = getattr(gkl, 'py_bootstrap_%s' % name)
118 f.restype, f.argtypes = restype, argtypes
119 # create a delegate method if not already present
120 # (we do not want to use hasattr because it calls "properties")
121 if name not in self.__class__.__dict__:
122 setattr(self, name, f)
123
124 for name in ('configure', 'initialize', 'start', 'stop', 'finalize',
125 'terminate'):
126 f = getattr(gkl, 'py_bootstrap_fsm_%s' % name)
127 f.restype, f.argtypes = c_bool, [IInterface_p]
128 gkl.py_bootstrap_app_run.restype = c_bool
129 gkl.py_bootstrap_app_run.argtypes = [IInterface_p, c_int]
130
131 gkl.py_helper_printAlgsSequences.restype = None
132 gkl.py_helper_printAlgsSequences.argtypes = [IInterface_p]
133
134 def createApplicationMgr(self):
135 ptr = self.lib.py_bootstrap_createApplicationMgr()
136 return self.AppMgr(ptr, self.lib)
137
138 @property
139 def ROOT_VERSION_CODE(self):
140 return self.lib.py_bootstrap_ROOT_VERSION_CODE()
141
142 @property
143 def ROOT_VERSION(self):
144 root_version_code = self.ROOT_VERSION_CODE
145 a = root_version_code >> 16 & 0xff
146 b = root_version_code >> 8 & 0xff
147 c = root_version_code & 0xff
148 return (a, b, c)
149
150
151 _bootstrap = None
152
153
154 def toOpt(value):
155 '''
156 Helper to convert values to old .opts format.
157
158 >>> print toOpt('some "text"')
159 "some \\"text\\""
160 >>> print toOpt('first\\nsecond')
161 "first
162 second"
163 >>> print toOpt({'a': [1, 2, '3']})
164 {"a": [1, 2, "3"]}
165 '''
166 if isinstance(value, basestring):
167 return '"{0}"'.format(value.replace('"', '\\"'))
168 elif isinstance(value, dict):
169 return '{{{0}}}'.format(', '.join(
170 '{0}: {1}'.format(toOpt(k), toOpt(v))
171 for k, v in value.iteritems()))
172 elif hasattr(value, '__iter__'):
173 return '[{0}]'.format(', '.join(map(toOpt, value)))
174 else:
175 return repr(value)
176
177
178 class gaudimain(object):
179 def __init__(self):
180 from Configurables import ApplicationMgr
181 appMgr = ApplicationMgr()
182 if "GAUDIAPPNAME" in os.environ:
183 appMgr.AppName = str(os.environ["GAUDIAPPNAME"])
184 if "GAUDIAPPVERSION" in os.environ:
185 appMgr.AppVersion = str(os.environ["GAUDIAPPVERSION"])
186 self.log = logging.getLogger(__name__)
187 self.printsequence = False
188 self.application = 'Gaudi::Application'
189
190 def setupParallelLogging(self):
191 # ---------------------------------------------------
192 # set up Logging
193 # ----------------
194 # from multiprocessing import enableLogging, getLogger
195 import multiprocessing
196 # preliminaries for handlers/output files, etc.
197 from time import ctime
198 datetime = ctime()
199 datetime = datetime.replace(' ', '_')
200 outfile = open('gaudirun-%s.log' % (datetime), 'w')
201 # two handlers, one for a log file, one for terminal
202 streamhandler = logging.StreamHandler(stream=outfile)
203 console = logging.StreamHandler()
204 # create formatter : the params in parentheses are variable names available via logging
205 formatter = logging.Formatter(
206 "%(asctime)s - %(name)s - %(levelname)s - %(message)s")
207 # add formatter to Handler
208 streamhandler.setFormatter(formatter)
209 console.setFormatter(formatter)
210 # now, configure the logger
211 # enableLogging( level=0 )
212 # self.log = getLogger()
213 self.log = multiprocessing.log_to_stderr()
214 self.log.setLevel(logging.INFO)
215 self.log.name = 'Gaudi/Main.py Logger'
216 self.log.handlers = []
217 # add handlers to logger : one for output to a file, one for console output
218 self.log.addHandler(streamhandler)
219 self.log.addHandler(console)
220 self.log.removeHandler(console)
221 # set level!!
222 self.log.setLevel = logging.INFO
223 # ---------------------------------------------------
224
225 def generatePyOutput(self, all=False):
226 from pprint import pformat
227 conf_dict = Configuration.configurationDict(all)
228 return pformat(conf_dict)
229
230 def generateOptsOutput(self, all=False):
231 from pprint import pformat
232 conf_dict = Configuration.configurationDict(all)
233 out = []
234 names = conf_dict.keys()
235 names.sort()
236 for n in names:
237 props = conf_dict[n].keys()
238 props.sort()
239 for p in props:
240 out.append('%s.%s = %s;' % (n, p, toOpt(conf_dict[n][p])))
241 return "\n".join(out)
242
243 def _writepickle(self, filename):
244 # --- Lets take the first file input file as the name of the pickle file
245 import pickle
246 output = open(filename, 'wb')
247 # Dump only the the configurables that make sense to dump (not User ones)
248 from GaudiKernel.Proxy.Configurable import getNeededConfigurables
249 to_dump = {}
250 for n in getNeededConfigurables():
251 to_dump[n] = Configuration.allConfigurables[n]
252 pickle.dump(to_dump, output, -1)
253 output.close()
254
255 def printconfig(self, old_format=False, all=False):
256 msg = 'Dumping all configurables and properties'
257 if not all:
258 msg += ' (different from default)'
259 log.info(msg)
260 conf_dict = Configuration.configurationDict(all)
261 if old_format:
262 print self.generateOptsOutput(all)
263 else:
264 print self.generatePyOutput(all)
265
266 def writeconfig(self, filename, all=False):
267 write = {".pkl": lambda filename, all: self._writepickle(filename),
268 ".py": lambda filename, all: open(filename, "w").write(self.generatePyOutput(all) + "\n"),
269 ".opts": lambda filename, all: open(filename, "w").write(self.generateOptsOutput(all) + "\n"),
270 }
271 from os.path import splitext
272 ext = splitext(filename)[1]
273 if ext in write:
274 write[ext](filename, all)
275 else:
276 log.error("Unknown file type '%s'. Must be any of %r.", ext,
277 write.keys())
278 sys.exit(1)
279
280 # Instantiate and run the application.
281 # Depending on the number of CPUs (ncpus) specified, it start
282 def run(self, attach_debugger, ncpus=None):
283 if not ncpus:
284 # Standard sequential mode
285 result = self.runSerial(attach_debugger)
286 else:
287 # Otherwise, run with the specified number of cpus
288 result = self.runParallel(ncpus)
289 return result
290
291 def hookDebugger(self, debugger='gdb'):
292 import os
293 self.log.info('attaching debugger to PID ' + str(os.getpid()))
294 pid = os.spawnvp(os.P_NOWAIT, debugger,
295 [debugger, '-q', 'python',
296 str(os.getpid())])
297
298 # give debugger some time to attach to the python process
299 import time
300 time.sleep(5)
301
302 # verify the process' existence (will raise OSError if failed)
303 os.waitpid(pid, os.WNOHANG)
304 os.kill(pid, 0)
305 return
306
307 def runSerial(self, attach_debugger):
308 try:
309 from GaudiKernel.Proxy.Configurable import expandvars
310 except ImportError:
311 # pass-through implementation if expandvars is not defined (AthenaCommon)
312 def expandvars(data):
313 return data
314
315 from GaudiKernel.Proxy.Configurable import Configurable, getNeededConfigurables
316
317 self.log.debug('runSerial: apply options')
318 conf_dict = {'ApplicationMgr.JobOptionsType': '"NONE"'}
319
320 # FIXME: this is to make sure special properties are correctly
321 # expanded before we fill conf_dict
322 for c in Configurable.allConfigurables.values():
323 if hasattr(c, 'getValuedProperties'):
324 c.getValuedProperties()
325
326 for n in getNeededConfigurables():
327 c = Configurable.allConfigurables[n]
328 for p, v in c.getValuedProperties().items():
329 v = expandvars(v)
330 # Note: AthenaCommon.Configurable does not have Configurable.PropertyReference
331 if hasattr(Configurable, "PropertyReference") and type(
332 v) == Configurable.PropertyReference:
333 # this is done in "getFullName", but the exception is ignored,
334 # so we do it again to get it
335 v = v.__resolve__()
336 if type(v) == str:
337 # properly escape quotes in the string
338 v = '"%s"' % v.replace('"', '\\"')
339 elif type(v) == long:
340 v = '%d' % v # prevent pending 'L'
341 conf_dict['{}.{}'.format(n, p)] = str(v)
342
343 if self.printsequence:
344 conf_dict['ApplicationMgr.PrintAlgsSequence'] = 'true'
345
346 if hasattr(Configurable, "_configurationLocked"):
347 Configurable._configurationLocked = True
348
349 if attach_debugger:
350 self.hookDebugger()
351
352 self.log.debug('-' * 80)
353 self.log.debug('%s: running in serial mode', __name__)
354 self.log.debug('-' * 80)
355 sysStart = time()
356
357 import Gaudi
358 app = Gaudi.Application.create(self.application, conf_dict)
359 retcode = app.run()
360
361 sysTime = time() - sysStart
362 self.log.debug('-' * 80)
363 self.log.debug('%s: serial system finished, time taken: %5.4fs',
364 __name__, sysTime)
365 self.log.debug('-' * 80)
366
367 return retcode
368
369 def runParallel(self, ncpus):
370 self.setupParallelLogging()
371 from Gaudi.Configuration import Configurable
372 import GaudiMP.GMPBase as gpp
373 c = Configurable.allConfigurables
374 self.log.info('-' * 80)
375 self.log.info('%s: Parallel Mode : %i ', __name__, ncpus)
376 for name, value in [
377 ('platrofm', ' '.join(os.uname())),
378 ('config', os.environ.get('BINARY_TAG')
379 or os.environ.get('CMTCONFIG')),
380 ('app. name', os.environ.get('GAUDIAPPNAME')),
381 ('app. version', os.environ.get('GAUDIAPPVERSION')),
382 ]:
383 self.log.info('%s: %30s : %s ', __name__, name, value
384 or 'Undefined')
385 try:
386 events = str(c['ApplicationMgr'].EvtMax)
387 except:
388 events = "Undetermined"
389 self.log.info('%s: Events Specified : %s ', __name__, events)
390 self.log.info('-' * 80)
391 # Parall = gpp.Coordinator(ncpus, shared, c, self.log)
392 Parall = gpp.Coord(ncpus, c, self.log)
393 sysStart = time()
394 sc = Parall.Go()
395 self.log.info('MAIN.PY : received %s from Coordinator' % (sc))
396 if sc.isFailure():
397 return 1
398 sysTime = time() - sysStart
399 self.log.name = 'Gaudi/Main.py Logger'
400 self.log.info('-' * 80)
401 self.log.info('%s: parallel system finished, time taken: %5.4fs',
402 __name__, sysTime)
403 self.log.info('-' * 80)
404 return 0
plotSpeedupsPyRoot.time
def time
Definition: plotSpeedupsPyRoot.py:152
Gaudi.Main.BootstrapHelper.AppMgr.printAlgsSequences
def printAlgsSequences(self)
Definition: Main.py:83
Gaudi.Main.gaudimain.run
def run(self, attach_debugger, ncpus=None)
Definition: Main.py:282
Gaudi.Main.BootstrapHelper.lib
lib
Definition: Main.py:104
Gaudi.Main.gaudimain.setupParallelLogging
def setupParallelLogging(self)
Definition: Main.py:190
Gaudi.Main.BootstrapHelper.StatusCode.isSuccess
def isSuccess(self)
Definition: Main.py:20
format
GAUDI_API std::string format(const char *,...)
MsgStream format utility "a la sprintf(...)".
Definition: MsgStream.cpp:109
Gaudi.Main.toOpt
def toOpt(value)
Definition: Main.py:154
Gaudi.Main.gaudimain.log
log
Definition: Main.py:186
Gaudi.Main.gaudimain._writepickle
def _writepickle(self, filename)
Definition: Main.py:243
Gaudi.Main.gaudimain.generateOptsOutput
def generateOptsOutput(self, all=False)
Definition: Main.py:230
Gaudi.Main.BootstrapHelper.Property
Definition: Main.py:29
Gaudi.Main.BootstrapHelper.StatusCode.__bool__
def __bool__(self)
Definition: Main.py:15
Gaudi.Main.BootstrapHelper.AppMgr.lib
lib
Definition: Main.py:41
GaudiKernel.Proxy.getNeededConfigurables
getNeededConfigurables
Definition: Proxy.py:21
Gaudi.Main.BootstrapHelper.StatusCode.ignore
def ignore(self)
Definition: Main.py:26
Gaudi.Main.BootstrapHelper.AppMgr._as_parameter_
_as_parameter_
Definition: Main.py:42
Gaudi.Main.BootstrapHelper.StatusCode.value
value
Definition: Main.py:13
Gaudi.Configuration
Definition: Configuration.py:1
Gaudi.Main.BootstrapHelper.AppMgr.stop
def stop(self)
Definition: Main.py:60
Gaudi.Main.gaudimain.runParallel
def runParallel(self, ncpus)
Definition: Main.py:369
Gaudi.Main.gaudimain.hookDebugger
def hookDebugger(self, debugger='gdb')
Definition: Main.py:291
Gaudi.Main.BootstrapHelper.Property.__str__
def __str__(self)
Definition: Main.py:33
Gaudi.Main.BootstrapHelper
Definition: Main.py:10
Gaudi.Main.gaudimain.generatePyOutput
def generatePyOutput(self, all=False)
Definition: Main.py:225
Gaudi.Main.gaudimain.printsequence
printsequence
Definition: Main.py:187
Gaudi.Main.BootstrapHelper.StatusCode.isFailure
def isFailure(self)
Definition: Main.py:23
Gaudi.Main.BootstrapHelper.AppMgr.start
def start(self)
Definition: Main.py:52
Gaudi.Main.BootstrapHelper.AppMgr.__init__
def __init__(self, ptr, lib)
Definition: Main.py:39
py_helper_printAlgsSequences
void py_helper_printAlgsSequences(IInterface *app)
Helper to call printAlgsSequences from Pyhton ctypes.
Definition: PrintAlgsSequences.cpp:47
Containers::map
struct GAUDI_API map
Parametrisation class for map-like implementation.
Definition: KeyedObjectManager.h:25
Gaudi.Main.gaudimain.application
application
Definition: Main.py:188
gaudirun.type
type
Definition: gaudirun.py:142
Gaudi.Main.BootstrapHelper.AppMgr.getProperty
def getProperty(self, name)
Definition: Main.py:79
Gaudi.Main.BootstrapHelper.AppMgr.getService
def getService(self, name)
Definition: Main.py:72
Gaudi.Main.BootstrapHelper.createApplicationMgr
def createApplicationMgr(self)
Definition: Main.py:134
Gaudi.Main.BootstrapHelper.AppMgr.finalize
def finalize(self)
Definition: Main.py:64
Gaudi.Application.create
def create(cls, appType, opts)
Definition: __init__.py:83
Gaudi.Main.BootstrapHelper.log
log
Definition: Main.py:98
Gaudi.Main.BootstrapHelper.Property.__init__
def __init__(self, value)
Definition: Main.py:30
ApplicationMgr
The Application Manager class.
Definition: ApplicationMgr.h:45
Gaudi.Main.BootstrapHelper.ROOT_VERSION_CODE
def ROOT_VERSION_CODE(self)
Definition: Main.py:139
Gaudi.Main.BootstrapHelper.StatusCode.__init__
def __init__(self, value)
Definition: Main.py:12
Gaudi.Main.BootstrapHelper.AppMgr.run
def run(self, nevt)
Definition: Main.py:56
Gaudi.Main.BootstrapHelper.AppMgr
Definition: Main.py:38
Gaudi.Main.BootstrapHelper.StatusCode
Definition: Main.py:11
Gaudi.Main.gaudimain.__init__
def __init__(self)
Definition: Main.py:179
Gaudi.Main.BootstrapHelper.AppMgr.ptr
ptr
Definition: Main.py:40
GaudiKernel.Configurable.expandvars
def expandvars(data)
Definition: Configurable.py:35
Gaudi.Main.BootstrapHelper.AppMgr.terminate
def terminate(self)
Definition: Main.py:68
Gaudi.Main.BootstrapHelper.__init__
def __init__(self)
Definition: Main.py:86
GaudiMP.GMPBase
Definition: GMPBase.py:1
Gaudi.Main.BootstrapHelper.Property.value
value
Definition: Main.py:31
Gaudi.Main.gaudimain.printconfig
def printconfig(self, old_format=False, all=False)
Definition: Main.py:255
Gaudi.Main.gaudimain.writeconfig
def writeconfig(self, filename, all=False)
Definition: Main.py:266
Gaudi.Main.BootstrapHelper.ROOT_VERSION
def ROOT_VERSION(self)
Definition: Main.py:143
GaudiPython.HistoUtils.__repr__
__repr__
Definition: HistoUtils.py:508
Gaudi.Main.BootstrapHelper.AppMgr.configure
def configure(self)
Definition: Main.py:44
Gaudi.Main.BootstrapHelper.AppMgr.setProperty
def setProperty(self, name, value)
Definition: Main.py:75
Gaudi.Main.gaudimain.runSerial
def runSerial(self, attach_debugger)
Definition: Main.py:307
Gaudi.Main.BootstrapHelper.AppMgr.initialize
def initialize(self)
Definition: Main.py:48
Gaudi.Main.gaudimain
Definition: Main.py:178