| The Gaudi Framework v28r2-patches (bc4ad17a)

- Main Page
- Related Pages
- Modules
- Namespaces
- Classes
- Files

- File List
- File Members

gaudirun.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2
3 import os
4 import sys
5 from tempfile import mkstemp
6
7 def getArgsWithoutoProfilerInfo(args):
8 """
9 Remove from the arguments the presence of the profiler and its output in
10 order to relaunch the script w/o infinite loops.
11
12 >>> getArgsWithoutoProfilerInfo(['--profilerName', 'igprof', 'myopts.py'])
13 ['myopts.py']
14
15 >>> getArgsWithoutoProfilerInfo(['--profilerName=igprof', 'myopts.py'])
16 ['myopts.py']
17
18 >>> getArgsWithoutoProfilerInfo(['--profilerName', 'igprof', '--profilerExtraOptions', 'a b c', 'myopts.py'])
19 ['myopts.py']
20
21 >>> getArgsWithoutoProfilerInfo(['--profilerName', 'igprof', '--options', 'a b c', 'myopts.py'])
22 ['--options', 'a b c', 'myopts.py']
23 """
24 newargs = []
25 args = list(args) # make a temp copy
26 while args:
27 o = args.pop(0)
28 if o.startswith('--profile'):
29 if '=' not in o:
30 args.pop(0)
31 else:
32 newargs.append(o)
33 return newargs
34
35 def setLibraryPreload(newpreload):
36 ''' Adds a list of libraries to LD_PRELOAD '''
37 preload = os.environ.get("LD_PRELOAD", "")
38 if preload:
39 preload = preload.replace(" ", ":").split(":")
40 else:
41 preload = []
42
43 for libname in set(preload).intersection(newpreload):
44 logging.warning("Ignoring preload of library %s because it is "
45 "already in LD_PRELOAD.", libname)
46
47 to_load = [libname
48 for libname in newpreload
49 if libname not in set(preload)]
50
51 if to_load:
52 preload += to_load
53 preload = ":".join(preload)
54 os.environ["LD_PRELOAD"] = preload
55 logging.info("Setting LD_PRELOAD='%s'", preload)
56
57 return to_load
58
59 def rationalizepath(path):
60 '''
61 Convert the given path to a real path if the pointed file exists, otherwise
62 just normalize it.
63 '''
64 path = os.path.normpath(os.path.expandvars(path))
65 if os.path.exists(path):
66 path = os.path.realpath(path)
67 return path
68
69 # variable used to keep alive the temporary option files extracted from the .qmt
70 _qmt_tmp_opt_files = []
71 def getArgsFromQmt(qmtfile):
72 '''
73 Given a .qmt file, return the command line arguments of the corresponding
74 test.
75 '''
76 from xml.etree import ElementTree as ET
77 global _qmt_tmp_opt_files
78 # parse the .qmt file and extract args and options
79 qmt = ET.parse(qmtfile)
80 args = [a.text for a in qmt.findall("argument[@name='args']//text")]
81 options = qmt.find("argument[@name='options']/text")
82
83 if options is not None: # options need to be dumped in a temporary file
84 from tempfile import NamedTemporaryFile
85 import re
86 if re.search(r"from\s+Gaudi.Configuration\s+import\s+\*"
87 r"|from\s+Configurables\s+import", options.text):
88 tmp_opts = NamedTemporaryFile(suffix='.py')
89 else:
90 tmp_opts = NamedTemporaryFile(suffix='.opts')
91 tmp_opts.write(options.text)
92 tmp_opts.flush()
93 args.append(tmp_opts.name)
94 _qmt_tmp_opt_files.append(tmp_opts)
95
96 # relative paths in a .qmt are rooted in the qmtest directory, so
97 # - find where the .qmt lives
98 qmtfile = os.path.abspath(qmtfile)
99 if 'qmtest' in qmtfile.split(os.path.sep):
100 # this return the path up to the 'qmtest' entry in qmtfile
101 testdir = qmtfile
102 while os.path.basename(testdir) != 'qmtest':
103 testdir = os.path.dirname(testdir)
104 else:
105 testdir = '.'
106 # - temporarily switch to that directory and rationalize the paths
107 old_cwd = os.getcwd()
108 os.chdir(testdir)
109 args = map(rationalizepath, args)
110 os.chdir(old_cwd)
111
112 return args
113
114 #---------------------------------------------------------------------
115 if __name__ == "__main__":
116 # ensure that we (and the subprocesses) use the C standard localization
117 if os.environ.get('LC_ALL') != 'C':
118 print '# setting LC_ALL to "C"'
119 os.environ['LC_ALL'] = 'C'
120
121 from optparse import OptionParser
122 parser = OptionParser(usage = "%prog [options] <opts_file> ...")
123 parser.add_option("-n","--dry-run", action="store_true",
124 help="do not run the application, just parse option files")
125 parser.add_option("-p","--pickle-output", action="store", type="string",
126 metavar = "FILE",
127 help="DEPRECATED: use '--output file.pkl' instead. Write "
128 "the parsed options as a pickle file (static option "
129 "file)")
130 parser.add_option("-v","--verbose", action="store_true",
131 help="print the parsed options")
132 parser.add_option("--old-opts", action="store_true",
133 help="format printed options in old option files style")
134 parser.add_option("--all-opts", action="store_true",
135 help="print all the option (even if equal to default)")
136 # GaudiPython Parallel Mode Option
137 # Argument must be an integer in range [ -1, sys_cpus ]
138 # -1 : All available cpus
139 # 0 : Serial Mode (traditional gaudirun)
140 # n>0 : parallel with n cpus (n <= sys_cpus)
141 parser.add_option("--ncpus", action="store", type="int", default=0,
142 help="start the application in parallel mode using NCPUS processes. "
143 "0 => serial mode (default), -1 => use all CPUs")
144
145 def option_cb(option, opt, value, parser):
146 """Add the option line to a list together with its position in the
147 argument list.
148 """
149 parser.values.options.append((len(parser.largs), value))
150 parser.add_option("--option", action="callback", callback=option_cb,
151 type = "string", nargs = 1,
152 help="add a single line (Python) option to the configuration. "
153 "All options lines are executed, one after the other, in "
154 "the same context.")
155 parser.add_option("--no-conf-user-apply", action="store_true",
156 help="disable the automatic application of configurable "
157 "users (for backward compatibility)")
158 parser.add_option("--old-conf-user-apply", action="store_true",
159 help="use the old logic when applying ConfigurableUsers "
160 "(with bug #103803) [default]")
161 parser.add_option("--new-conf-user-apply", action="store_false",
162 dest="old_conf_user_apply",
163 help="use the new (correct) logic when applying "
164 "ConfigurableUsers (fixed bug #103803), can be "
165 "turned on also with the environment variable "
166 "GAUDI_FIXED_APPLY_CONF")
167 parser.add_option("-o", "--output", action = "store", type = "string",
168 help ="dump the configuration to a file. The format of "
169 "the options is determined by the extension of the "
170 "file name: .pkl = pickle, .py = python, .opts = "
171 "old style options. The python format cannot be "
172 "used to run the application and it contains the "
173 "same dictionary printed with -v")
174 parser.add_option("--post-option", action="append", type="string",
175 dest="post_options",
176 help="Python options to be executed after the ConfigurableUser "
177 "are applied. "
178 "All options lines are executed, one after the other, in "
179 "the same context.")
180 parser.add_option("--debug", action="store_true",
181 help="enable some debug print-out")
182 parser.add_option("--gdb", action="store_true",
183 help="attach gdb")
184 parser.add_option("--printsequence", action="store_true",
185 help="print the sequence")
186 if not sys.platform.startswith("win"):
187 # These options can be used only on unix platforms
188 parser.add_option("-T", "--tcmalloc", action="store_true",
189 help="Use the Google malloc replacement. The environment "
190 "variable TCMALLOCLIB can be used to specify a different "
191 "name for the library (the default is libtcmalloc.so)")
192 parser.add_option("--preload", action="append",
193 help="Allow pre-loading of special libraries (e.g. Google "
194 "profiling libraries).")
195 # Option to use a profiler
196 parser.add_option("--profilerName", type="string",
197 help="Select one profiler among: igprofPerf, igprofMem and valgrind<toolname>")
198
199 # Option to specify the filename where to collect the profiler's output
200 parser.add_option("--profilerOutput", type="string",
201 help="Specify the name of the output file for the profiler output")
202
203 # Option to specify the filename where to collect the profiler's output
204 parser.add_option("--profilerExtraOptions", type="string",
205 help="Specify additional options for the profiler. The '--' string should be expressed as '__' (--my-opt becomes __my-opt)")
206
207 parser.add_option('--use-temp-opts', action='store_true',
208 help='when this option is enabled, the options are parsed'
209 ' and stored in a temporary file, then the job is '
210 'restarted using that file as input (to save '
211 'memory)')
212 parser.add_option("--run-info-file", type="string",
213 help="Save gaudi process information to the file specified (in JSON format)")
214
215 parser.set_defaults(options = [],
216 tcmalloc = False,
217 profilerName = '',
218 profilerOutput = '',
219 profilerExtraOptions = '',
220 preload = [],
221 ncpus = None,
222 # the old logic can be turned off with an env variable
223 old_conf_user_apply='GAUDI_FIXED_APPLY_CONF' not in os.environ,
224 run_info_file = None)
225
226 # replace .qmt files in the command line with their contained args
227 argv = []
228 for a in sys.argv[1:]:
229 if a.endswith('.qmt') and os.path.exists(a):
230 argv.extend(getArgsFromQmt(a))
231 else:
232 argv.append(a)
233 if argv != sys.argv[1:]:
234 print '# Running', sys.argv[0], 'with arguments', argv
235
236 opts, args = parser.parse_args(args=argv)
237
238 # Check consistency of options
239
240 # Parallel Option ---------------------------------------------------------
241 if opts.ncpus:
242 from multiprocessing import cpu_count
243 sys_cpus = cpu_count()
244 if opts.ncpus > sys_cpus:
245 s = "Invalid value : --ncpus : only %i cpus available" % sys_cpus
246 parser.error(s)
247 elif opts.ncpus < -1 :
248 s = "Invalid value : --ncpus must be integer >= -1"
249 parser.error(s)
250 else:
251 # FIXME: is it really needed to set it to None if it is 0 or False?
252 opts.ncpus = None
253
254 # configure the logging
255 import logging
256 from GaudiKernel.ProcessJobOptions import (InstallRootLoggingHandler,
257 PrintOff)
258
259 if opts.old_opts: prefix = "// "
260 else: prefix = "# "
261 level = logging.INFO
262 if opts.debug:
263 level = logging.DEBUG
264 InstallRootLoggingHandler(prefix, level = level, with_time = opts.debug)
265 root_logger = logging.getLogger()
266
267 # tcmalloc support
268 if opts.tcmalloc:
269 opts.preload.insert(0, os.environ.get("TCMALLOCLIB", "libtcmalloc.so"))
270 # allow preloading of libraries
271 if opts.preload:
272 preload = os.environ.get("LD_PRELOAD", "")
273 if preload:
274 preload = preload.replace(" ", ":").split(":")
275 else:
276 preload = []
277 for libname in set(preload).intersection(opts.preload):
278 logging.warning("Ignoring preload of library %s because it is "
279 "already in LD_PRELOAD.", libname)
280 to_load = [libname
281 for libname in opts.preload
282 if libname not in set(preload)]
283 if to_load:
284 preload += to_load
285 preload = ":".join(preload)
286 os.environ["LD_PRELOAD"] = preload
287 logging.info("Restarting with LD_PRELOAD='%s'", preload)
288 # remove the --tcmalloc option from the arguments
289 # FIXME: the --preload arguments will issue a warning but it's tricky to remove them
290 args = [ a for a in sys.argv if a != '-T' and not '--tcmalloc'.startswith(a) ]
291 os.execv(sys.executable, [sys.executable] + args)
292
293 # Profiler Support ------
294 if opts.profilerName:
295 profilerName = opts.profilerName
296 profilerExecName = ""
297 profilerOutput = opts.profilerOutput or (profilerName + ".output")
298
299 # To restart the application removing the igprof option and prepending the string
300 args = getArgsWithoutoProfilerInfo(sys.argv)
301
302 igprofPerfOptions = "-d -pp -z -o igprof.pp.gz".split()
303
304 profilerOptions = ""
305 if profilerName == "igprof":
306 if not opts.profilerOutput:
307 profilerOutput += ".profile.gz"
308 profilerOptions = "-d -z -o %s" % profilerOutput
309 profilerExecName = "igprof"
310
311 elif profilerName == "igprofPerf":
312 if not opts.profilerOutput:
313 profilerOutput += ".pp.gz"
314 profilerOptions = "-d -pp -z -o %s" % profilerOutput
315 profilerExecName = "igprof"
316
317 elif profilerName == "igprofMem":
318 if not opts.profilerOutput:
319 profilerOutput += ".mp.gz"
320 profilerOptions = "-d -mp -z -o %s" % profilerOutput
321 profilerExecName = "igprof"
322
323 elif "valgrind" in profilerName:
324 # extract the tool
325 if not opts.profilerOutput:
326 profilerOutput += ".log"
327 toolname = profilerName.replace('valgrind','')
328 outoption = "--log-file"
329 if toolname in ("massif", "callgrind", "cachegrind"):
330 outoption = "--%s-out-file" % toolname
331 profilerOptions = "--tool=%s %s=%s" % (toolname, outoption, profilerOutput)
332 profilerExecName = "valgrind"
333
334 elif profilerName == "jemalloc":
335 opts.preload.insert(0, os.environ.get("JEMALLOCLIB", "libjemalloc.so"))
336 os.environ['MALLOC_CONF'] = "prof:true,prof_leak:true"
337 else:
338 root_logger.warning("Profiler %s not recognized!" % profilerName)
339
340 # Add potential extra options
341 if opts.profilerExtraOptions!="":
342 profilerExtraOptions = opts.profilerExtraOptions
343 profilerExtraOptions = profilerExtraOptions.replace("__","--")
344 profilerOptions += " %s" % profilerExtraOptions
345
346 # now we look for the full path of the profiler: is it really there?
347 if profilerExecName:
348 import distutils.spawn
349 profilerPath = distutils.spawn.find_executable(profilerExecName)
350 if not profilerPath:
351 root_logger.error("Cannot locate profiler %s" % profilerExecName)
352 sys.exit(1)
353
354 root_logger.info("------ Profiling options are on ------ \n"\
355 " o Profiler: %s\n"\
356 " o Options: '%s'.\n"\
357 " o Output: %s" % (profilerExecName or profilerName, profilerOptions, profilerOutput))
358
359 # allow preloading of libraries
360 # That code need to be acsracted from above
361 to_reload = []
362 if opts.preload:
363 to_reload = setLibraryPreload(opts.preload)
364
365 if profilerExecName:
366 # We profile python
367 profilerOptions += " python"
368
369 # now we have all the ingredients to prepare our command
370 arglist = [profilerPath] + profilerOptions.split() + args
371 arglist = [ a for a in arglist if a!='' ]
372 #print profilerPath
373 #for arg in arglist:
374 #print arg
375 os.execv(profilerPath, arglist)
376 else:
377 arglist = [a for a in sys.argv if not a.startswith("--profiler")]
378 os.execv(sys.executable, [sys.executable] + arglist)
379
380 # End Profiler Support ------
381
382 if opts.pickle_output:
383 if opts.output:
384 root_logger.error("Conflicting options: use only --pickle-output or --output")
385 sys.exit(1)
386 else:
387 root_logger.warning("--pickle-output is deprecated, use --output instead")
388 opts.output = opts.pickle_output
389
390 from Gaudi.Main import gaudimain
391 c = gaudimain()
392
393 # Prepare the "configuration script" to parse (like this it is easier than
394 # having a list with files and python commands, with an if statements that
395 # decides to do importOptions or exec)
396 options = [ "importOptions(%r)" % f for f in args ]
397 # The option lines are inserted into the list of commands using their
398 # position on the command line
399 optlines = list(opts.options)
400 optlines.reverse() # this allows to avoid to have to care about corrections of the positions
401 for pos, l in optlines:
402 options.insert(pos,l)
403
404 # prevent the usage of GaudiPython
405 class FakeModule(object):
406 def __init__(self, exception):
407 self.exception = exception
408 def __getattr__(self, *args, **kwargs):
409 raise self.exception
410 sys.modules["GaudiPython"] = FakeModule(RuntimeError("GaudiPython cannot be used in option files"))
411
412 # when the special env GAUDI_TEMP_OPTS_FILE is set, it overrides any
413 # option(file) on the command line
414 if 'GAUDI_TEMP_OPTS_FILE' in os.environ:
415 options = ['importOptions(%r)' % os.environ['GAUDI_TEMP_OPTS_FILE']]
416 PrintOff(100)
417
418 # "execute" the configuration script generated (if any)
419 if options:
420 g = {}
421 l = {}
422 exec "from Gaudi.Configuration import *" in g, l
423 for o in options:
424 logging.debug(o)
425 exec o in g, l
426
427 import GaudiKernel.Proxy.Configurable
428 if opts.no_conf_user_apply:
429 logging.info("Disabling automatic apply of ConfigurableUser")
430 # pretend that they have been already applied
431 GaudiKernel.Proxy.Configurable._appliedConfigurableUsers_ = True
432
433 # This need to be done before dumping
434 if opts.old_conf_user_apply:
435 from GaudiKernel.Proxy.Configurable import applyConfigurableUsers_old as applyConfigurableUsers
436 else:
437 from GaudiKernel.Proxy.Configurable import applyConfigurableUsers
438 applyConfigurableUsers()
439
440 # Options to be processed after applyConfigurableUsers
441 if opts.post_options:
442 g = {}
443 l = {}
444 exec "from Gaudi.Configuration import *" in g, l
445 for o in opts.post_options:
446 logging.debug(o)
447 exec o in g, l
448
449 if 'GAUDI_TEMP_OPTS_FILE' in os.environ:
450 os.remove(os.environ['GAUDI_TEMP_OPTS_FILE'])
451 opts.use_temp_opts = False
452
453 if opts.verbose and not opts.use_temp_opts:
454 c.printconfig(opts.old_opts, opts.all_opts)
455 if opts.output:
456 c.writeconfig(opts.output, opts.all_opts)
457
458 if opts.use_temp_opts:
459 fd, tmpfile = mkstemp('.opts')
460 os.close(fd)
461 c.writeconfig(tmpfile, opts.all_opts)
462 os.environ['GAUDI_TEMP_OPTS_FILE'] = tmpfile
463 logging.info('Restarting from pre-parsed options')
464 os.execv(sys.executable, [sys.executable] + sys.argv)
465
466 c.printsequence = opts.printsequence
467 if opts.printsequence:
468 if opts.ncpus:
469 logging.warning("--printsequence not supported with --ncpus: ignored")
470 elif opts.dry_run:
471 logging.warning("--printsequence not supported with --dry-run: ignored")
472
473 # re-enable the GaudiPython module
474 del sys.modules["GaudiPython"]
475
476 if not opts.dry_run:
477 # Do the real processing
478 retcode = c.run(opts.gdb,opts.ncpus)
479
480 # Now saving the run information pid, retcode and executable path to
481 # a file is requested
482 if opts.run_info_file:
483 import os, json
484 run_info = {}
485 run_info["pid"] = os.getpid()
486 run_info["retcode"] = retcode
487 if os.path.exists('/proc/self/exe'):
488 # These options can be used only on unix platforms
489 run_info["exe"] = os.readlink('/proc/self/exe')
490
491 logging.info("Saving run info to: %s" % opts.run_info_file)
492 with open(opts.run_info_file, "w") as f:
493 json.dump(run_info, f)
494
495 sys.exit(retcode)
GaudiKernel.ProcessJobOptions.PrintOff
def PrintOff(step=1)
Definition: ProcessJobOptions.py:105
gaudirun.FakeModule.exception
exception
Definition: gaudirun.py:407
gaudirun.FakeModule.__init__
def __init__(self, exception)
Definition: gaudirun.py:406
gaudirun.option_cb
def option_cb(option, opt, value, parser)
Definition: gaudirun.py:145
gaudirun.rationalizepath
def rationalizepath(path)
Definition: gaudirun.py:59
GaudiKernel.Configurable.applyConfigurableUsers
def applyConfigurableUsers()
Definition: Configurable.py:1395
gaudirun.getArgsFromQmt
def getArgsFromQmt(qmtfile)
Definition: gaudirun.py:71
GaudiKernel.ProcessJobOptions
Definition: ProcessJobOptions.py:1
GaudiKernel.ProcessJobOptions.InstallRootLoggingHandler
def InstallRootLoggingHandler(prefix=None, level=None, stream=None, with_time=False)
Definition: ProcessJobOptions.py:95
Containers::map
struct GAUDI_API map
Parametrisation class for map-like implementation.
Definition: KeyedObjectManager.h:25
gaudirun.FakeModule
Definition: gaudirun.py:405
gaudirun.getArgsWithoutoProfilerInfo
def getArgsWithoutoProfilerInfo(args)
Definition: gaudirun.py:7
Gaudi.Main
Definition: Main.py:1
gaudirun.FakeModule.__getattr__
def __getattr__(self, args, kwargs)
Definition: gaudirun.py:408
Gaudi.Main.gaudimain
Definition: Main.py:139
gaudirun.setLibraryPreload
def setLibraryPreload(newpreload)
Definition: gaudirun.py:35