| The Gaudi Framework v32r1 (f65d50dc)

gaudirun.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2
3 import os
4 import sys
5 from tempfile import mkstemp
6
7
8 def getArgsWithoutoProfilerInfo(args):
9 """
10 Remove from the arguments the presence of the profiler and its output in
11 order to relaunch the script w/o infinite loops.
12
13 >>> getArgsWithoutoProfilerInfo(['--profilerName', 'igprof', 'myopts.py'])
14 ['myopts.py']
15
16 >>> getArgsWithoutoProfilerInfo(['--profilerName=igprof', 'myopts.py'])
17 ['myopts.py']
18
19 >>> getArgsWithoutoProfilerInfo(['--profilerName', 'igprof', '--profilerExtraOptions', 'a b c', 'myopts.py'])
20 ['myopts.py']
21
22 >>> getArgsWithoutoProfilerInfo(['--profilerName', 'igprof', '--options', 'a b c', 'myopts.py'])
23 ['--options', 'a b c', 'myopts.py']
24 """
25 newargs = []
26 args = list(args) # make a temp copy
27 while args:
28 o = args.pop(0)
29 if o.startswith('--profile'):
30 if '=' not in o:
31 args.pop(0)
32 else:
33 newargs.append(o)
34 return newargs
35
36
37 def setLibraryPreload(newpreload):
38 ''' Adds a list of libraries to LD_PRELOAD '''
39 preload = os.environ.get("LD_PRELOAD", "")
40 if preload:
41 preload = preload.replace(" ", ":").split(":")
42 else:
43 preload = []
44
45 for libname in set(preload).intersection(newpreload):
46 logging.warning(
47 "Ignoring preload of library %s because it is "
48 "already in LD_PRELOAD.", libname)
49
50 to_load = [
51 libname for libname in newpreload if libname not in set(preload)
52 ]
53
54 if to_load:
55 preload += to_load
56 preload = ":".join(preload)
57 os.environ["LD_PRELOAD"] = preload
58 logging.info("Setting LD_PRELOAD='%s'", preload)
59
60 return to_load
61
62
63 def rationalizepath(path):
64 '''
65 Convert the given path to a real path if the pointed file exists, otherwise
66 just normalize it.
67 '''
68 path = os.path.normpath(os.path.expandvars(path))
69 if os.path.exists(path):
70 path = os.path.realpath(path)
71 return path
72
73
74 # variable used to keep alive the temporary option files extracted
75 # from the .qmt
76 _qmt_tmp_opt_files = []
77
78
79 def getArgsFromQmt(qmtfile):
80 '''
81 Given a .qmt file, return the command line arguments of the corresponding
82 test.
83 '''
84 from xml.etree import ElementTree as ET
85 global _qmt_tmp_opt_files
86 # parse the .qmt file and extract args and options
87 qmt = ET.parse(qmtfile)
88 args = [a.text for a in qmt.findall("argument[@name='args']//text")]
89 options = qmt.find("argument[@name='options']/text")
90
91 if options is not None: # options need to be dumped in a temporary file
92 from tempfile import NamedTemporaryFile
93 import re
94 if re.search(
95 r"from\s+Gaudi.Configuration\s+import\s+\*"
96 r"|from\s+Configurables\s+import", options.text):
97 tmp_opts = NamedTemporaryFile(suffix='.py')
98 else:
99 tmp_opts = NamedTemporaryFile(suffix='.opts')
100 tmp_opts.write(options.text)
101 tmp_opts.flush()
102 args.append(tmp_opts.name)
103 _qmt_tmp_opt_files.append(tmp_opts)
104
105 # relative paths in a .qmt are rooted in the qmtest directory, so
106 # - find where the .qmt lives
107 qmtfile = os.path.abspath(qmtfile)
108 if 'qmtest' in qmtfile.split(os.path.sep):
109 # this return the path up to the 'qmtest' entry in qmtfile
110 testdir = qmtfile
111 while os.path.basename(testdir) != 'qmtest':
112 testdir = os.path.dirname(testdir)
113 else:
114 testdir = '.'
115 # - temporarily switch to that directory and rationalize the paths
116 old_cwd = os.getcwd()
117 os.chdir(testdir)
118 args = map(rationalizepath, args)
119 os.chdir(old_cwd)
120
121 return args
122
123
124 # ---------------------------------------------------------------------
125 if __name__ == "__main__":
126 # ensure that we (and the subprocesses) use the C standard localization
127 if os.environ.get('LC_ALL') != 'C':
128 print '# setting LC_ALL to "C"'
129 os.environ['LC_ALL'] = 'C'
130
131 from optparse import OptionParser
132 parser = OptionParser(usage="%prog [options] <opts_file> ...")
133 parser.add_option(
134 "-n",
135 "--dry-run",
136 action="store_true",
137 help="do not run the application, just parse option files")
138 parser.add_option(
139 "-p",
140 "--pickle-output",
141 action="store",
142 type="string",
143 metavar="FILE",
144 help="DEPRECATED: use '--output file.pkl' instead. Write "
145 "the parsed options as a pickle file (static option "
146 "file)")
147 parser.add_option(
148 "-v",
149 "--verbose",
150 action="store_true",
151 help="print the parsed options")
152 parser.add_option(
153 "--old-opts",
154 action="store_true",
155 help="format printed options in old option files style")
156 parser.add_option(
157 "--all-opts",
158 action="store_true",
159 help="print all the option (even if equal to default)")
160 # GaudiPython Parallel Mode Option
161 # Argument must be an integer in range [ -1, sys_cpus ]
162 # -1 : All available cpus
163 # 0 : Serial Mode (traditional gaudirun)
164 # n>0 : parallel with n cpus (n <= sys_cpus)
165 parser.add_option(
166 "--ncpus",
167 action="store",
168 type="int",
169 default=0,
170 help="start the application in parallel mode using NCPUS processes. "
171 "0 => serial mode (default), -1 => use all CPUs")
172
173 def option_cb(option, opt, value, parser):
174 """Add the option line to a list together with its position in the
175 argument list.
176 """
177 parser.values.options.append((len(parser.largs), value))
178
179 parser.add_option(
180 "--option",
181 action="callback",
182 callback=option_cb,
183 type="string",
184 nargs=1,
185 help="add a single line (Python) option to the configuration. "
186 "All options lines are executed, one after the other, in "
187 "the same context.")
188 parser.add_option(
189 "--no-conf-user-apply",
190 action="store_true",
191 help="disable the automatic application of configurable "
192 "users (for backward compatibility)")
193 parser.add_option(
194 "--old-conf-user-apply",
195 action="store_true",
196 help="use the old logic when applying ConfigurableUsers "
197 "(with bug #103803) [default]")
198 parser.add_option(
199 "--new-conf-user-apply",
200 action="store_false",
201 dest="old_conf_user_apply",
202 help="use the new (correct) logic when applying "
203 "ConfigurableUsers (fixed bug #103803), can be "
204 "turned on also with the environment variable "
205 "GAUDI_FIXED_APPLY_CONF")
206 parser.add_option(
207 "-o",
208 "--output",
209 action="store",
210 type="string",
211 help="dump the configuration to a file. The format of "
212 "the options is determined by the extension of the "
213 "file name: .pkl = pickle, .py = python, .opts = "
214 "old style options. The python format cannot be "
215 "used to run the application and it contains the "
216 "same dictionary printed with -v")
217 parser.add_option(
218 "--post-option",
219 action="append",
220 type="string",
221 dest="post_options",
222 help="Python options to be executed after the ConfigurableUser "
223 "are applied. "
224 "All options lines are executed, one after the other, in "
225 "the same context.")
226 parser.add_option(
227 "--debug", action="store_true", help="enable some debug print-out")
228 parser.add_option("--gdb", action="store_true", help="attach gdb")
229 parser.add_option(
230 "--printsequence", action="store_true", help="print the sequence")
231 if not sys.platform.startswith("win"):
232 # These options can be used only on unix platforms
233 parser.add_option(
234 "-T",
235 "--tcmalloc",
236 action="store_true",
237 help="Use the Google malloc replacement. The environment "
238 "variable TCMALLOCLIB can be used to specify a different "
239 "name for the library (the default is libtcmalloc.so)")
240 parser.add_option(
241 "--preload",
242 action="append",
243 help="Allow pre-loading of special libraries (e.g. Google "
244 "profiling libraries).")
245 # Option to use a profiler
246 parser.add_option(
247 "--profilerName",
248 type="string",
249 help=
250 "Select one profiler among: igprofPerf, igprofMem and valgrind<toolname>"
251 )
252
253 # Option to specify the filename where to collect the profiler's output
254 parser.add_option(
255 "--profilerOutput",
256 type="string",
257 help="Specify the name of the output file for the profiler output")
258
259 # Option to specify the filename where to collect the profiler's output
260 parser.add_option(
261 "--profilerExtraOptions",
262 type="string",
263 help=
264 "Specify additional options for the profiler. The '--' string should be expressed as '__' (--my-opt becomes __my-opt)"
265 )
266
267 parser.add_option(
268 '--use-temp-opts',
269 action='store_true',
270 help='when this option is enabled, the options are parsed'
271 ' and stored in a temporary file, then the job is '
272 'restarted using that file as input (to save '
273 'memory)')
274 parser.add_option(
275 "--run-info-file",
276 type="string",
277 help=
278 "Save gaudi process information to the file specified (in JSON format)"
279 )
280 parser.add_option(
281 '--application',
282 help='name of the Gaudi::Application to use [default: %default]')
283
284 parser.set_defaults(
285 options=[],
286 tcmalloc=False,
287 profilerName='',
288 profilerOutput='',
289 profilerExtraOptions='',
290 preload=[],
291 ncpus=None,
292 # the old logic can be turned off with an env variable
293 old_conf_user_apply='GAUDI_FIXED_APPLY_CONF' not in os.environ,
294 run_info_file=None,
295 application='Gaudi::Application')
296
297 # replace .qmt files in the command line with their contained args
298 argv = []
299 for a in sys.argv[1:]:
300 if a.endswith('.qmt') and os.path.exists(a):
301 argv.extend(getArgsFromQmt(a))
302 else:
303 argv.append(a)
304 if argv != sys.argv[1:]:
305 print '# Running', sys.argv[0], 'with arguments', argv
306
307 opts, args = parser.parse_args(args=argv)
308
309 # Check consistency of options
310
311 # Parallel Option ---------------------------------------------------------
312 if opts.ncpus:
313 from multiprocessing import cpu_count
314 sys_cpus = cpu_count()
315 if opts.ncpus > sys_cpus:
316 s = "Invalid value : --ncpus : only %i cpus available" % sys_cpus
317 parser.error(s)
318 elif opts.ncpus < -1:
319 s = "Invalid value : --ncpus must be integer >= -1"
320 parser.error(s)
321 else:
322 # FIXME: is it really needed to set it to None if it is 0 or False?
323 opts.ncpus = None
324
325 # configure the logging
326 import logging
327 from GaudiKernel.ProcessJobOptions import (InstallRootLoggingHandler,
328 PrintOff)
329
330 if opts.old_opts:
331 prefix = "// "
332 else:
333 prefix = "# "
334 level = logging.INFO
335 if opts.debug:
336 level = logging.DEBUG
337 InstallRootLoggingHandler(prefix, level=level, with_time=opts.debug)
338 root_logger = logging.getLogger()
339
340 # Sanitizer support
341 sanitizer = os.environ.get("PRELOAD_SANITIZER_LIB", "")
342 if sanitizer:
343 if sanitizer not in os.environ.get("LD_PRELOAD", ""):
344 opts.preload.insert(0, sanitizer)
345 os.environ["PRELOAD_SANITIZER_LIB"] = ""
346
347 # tcmalloc support
348 if opts.tcmalloc:
349 # Disable tcmalloc if sanitizer is selected
350 if sanitizer:
351 logging.warning("tcmalloc preload disabled when using a sanitizer")
352 else:
353 opts.preload.insert(
354 0, os.environ.get("TCMALLOCLIB", "libtcmalloc.so"))
355
356 # allow preloading of libraries
357 if opts.preload:
358 preload = os.environ.get("LD_PRELOAD", "")
359 if preload:
360 preload = preload.replace(" ", ":").split(":")
361 else:
362 preload = []
363 for libname in set(preload).intersection(opts.preload):
364 logging.warning(
365 "Ignoring preload of library %s because it is "
366 "already in LD_PRELOAD.", libname)
367 to_load = [
368 libname for libname in opts.preload if libname not in set(preload)
369 ]
370 if to_load:
371 preload += to_load
372 preload = ":".join(preload)
373 os.environ["LD_PRELOAD"] = preload
374 logging.info("Restarting with LD_PRELOAD='%s'", preload)
375 # remove the --tcmalloc option from the arguments
376 # FIXME: the --preload arguments will issue a warning but it's tricky to remove them
377 args = [
378 a for a in sys.argv
379 if a != '-T' and not '--tcmalloc'.startswith(a)
380 ]
381 os.execv(sys.executable, [sys.executable] + args)
382
383 # Profiler Support ------
384 if opts.profilerName:
385 profilerName = opts.profilerName
386 profilerExecName = ""
387 profilerOutput = opts.profilerOutput or (profilerName + ".output")
388
389 # To restart the application removing the igprof option and prepending the string
390 args = getArgsWithoutoProfilerInfo(sys.argv)
391
392 igprofPerfOptions = "-d -pp -z -o igprof.pp.gz".split()
393
394 profilerOptions = ""
395 if profilerName == "igprof":
396 if not opts.profilerOutput:
397 profilerOutput += ".profile.gz"
398 profilerOptions = "-d -z -o %s" % profilerOutput
399 profilerExecName = "igprof"
400
401 elif profilerName == "igprofPerf":
402 if not opts.profilerOutput:
403 profilerOutput += ".pp.gz"
404 profilerOptions = "-d -pp -z -o %s" % profilerOutput
405 profilerExecName = "igprof"
406
407 elif profilerName == "igprofMem":
408 if not opts.profilerOutput:
409 profilerOutput += ".mp.gz"
410 profilerOptions = "-d -mp -z -o %s" % profilerOutput
411 profilerExecName = "igprof"
412
413 elif "valgrind" in profilerName:
414 # extract the tool
415 if not opts.profilerOutput:
416 profilerOutput += ".log"
417 toolname = profilerName.replace('valgrind', '')
418 outoption = "--log-file"
419 if toolname in ("massif", "callgrind", "cachegrind"):
420 outoption = "--%s-out-file" % toolname
421 profilerOptions = "--tool=%s %s=%s" % (toolname, outoption,
422 profilerOutput)
423 profilerExecName = "valgrind"
424
425 elif profilerName == "jemalloc":
426 opts.preload.insert(
427 0, os.environ.get("JEMALLOCLIB", "libjemalloc.so"))
428 os.environ['MALLOC_CONF'] = "prof:true,prof_leak:true"
429 else:
430 root_logger.warning("Profiler %s not recognized!" % profilerName)
431
432 # Add potential extra options
433 if opts.profilerExtraOptions != "":
434 profilerExtraOptions = opts.profilerExtraOptions
435 profilerExtraOptions = profilerExtraOptions.replace("__", "--")
436 profilerOptions += " %s" % profilerExtraOptions
437
438 # now we look for the full path of the profiler: is it really there?
439 if profilerExecName:
440 import distutils.spawn
441 profilerPath = distutils.spawn.find_executable(profilerExecName)
442 if not profilerPath:
443 root_logger.error(
444 "Cannot locate profiler %s" % profilerExecName)
445 sys.exit(1)
446
447 root_logger.info("------ Profiling options are on ------ \n"
448 " o Profiler: %s\n"
449 " o Options: '%s'.\n"
450 " o Output: %s" % (profilerExecName or profilerName,
451 profilerOptions, profilerOutput))
452
453 # allow preloading of libraries
454 # That code need to be acsracted from above
455 to_reload = []
456 if opts.preload:
457 to_reload = setLibraryPreload(opts.preload)
458
459 if profilerExecName:
460 # We profile python
461 profilerOptions += " python"
462
463 # now we have all the ingredients to prepare our command
464 arglist = [profilerPath] + profilerOptions.split() + args
465 arglist = [a for a in arglist if a != '']
466 # print profilerPath
467 # for arg in arglist:
468 # print arg
469 os.execv(profilerPath, arglist)
470 else:
471 arglist = [a for a in sys.argv if not a.startswith("--profiler")]
472 os.execv(sys.executable, [sys.executable] + arglist)
473
474 # End Profiler Support ------
475
476 if opts.pickle_output:
477 if opts.output:
478 root_logger.error(
479 "Conflicting options: use only --pickle-output or --output")
480 sys.exit(1)
481 else:
482 root_logger.warning(
483 "--pickle-output is deprecated, use --output instead")
484 opts.output = opts.pickle_output
485
486 from Gaudi.Main import gaudimain
487 c = gaudimain()
488
489 # Prepare the "configuration script" to parse (like this it is easier than
490 # having a list with files and python commands, with an if statements that
491 # decides to do importOptions or exec)
492 options = ["importOptions(%r)" % f for f in args]
493 # The option lines are inserted into the list of commands using their
494 # position on the command line
495 optlines = list(opts.options)
496 # this allows to avoid to have to care about corrections of the positions
497 optlines.reverse()
498 for pos, l in optlines:
499 options.insert(pos, l)
500
501 # prevent the usage of GaudiPython
502 class FakeModule(object):
503 def __init__(self, exception):
504 self.exception = exception
505
506 def __getattr__(self, *args, **kwargs):
507 raise self.exception
508
509 sys.modules["GaudiPython"] = FakeModule(
510 RuntimeError("GaudiPython cannot be used in option files"))
511
512 # when the special env GAUDI_TEMP_OPTS_FILE is set, it overrides any
513 # option(file) on the command line
514 if 'GAUDI_TEMP_OPTS_FILE' in os.environ:
515 options = ['importOptions(%r)' % os.environ['GAUDI_TEMP_OPTS_FILE']]
516 PrintOff(100)
517
518 # "execute" the configuration script generated (if any)
519 if options:
520 g = {}
521 l = {}
522 exec ("from Gaudi.Configuration import *", g, l)
523 for o in options:
524 logging.debug(o)
525 exec (o, g, l)
526
527 import GaudiKernel.Proxy.Configurable
528 if opts.no_conf_user_apply:
529 logging.info("Disabling automatic apply of ConfigurableUser")
530 # pretend that they have been already applied
531 GaudiKernel.Proxy.Configurable._appliedConfigurableUsers_ = True
532
533 # This need to be done before dumping
534 if opts.old_conf_user_apply:
535 from GaudiKernel.Proxy.Configurable import applyConfigurableUsers_old as applyConfigurableUsers
536 else:
537 from GaudiKernel.Proxy.Configurable import applyConfigurableUsers
538 applyConfigurableUsers()
539
540 # Options to be processed after applyConfigurableUsers
541 if opts.post_options:
542 g = {}
543 l = {}
544 exec ("from Gaudi.Configuration import *", g, l)
545 for o in opts.post_options:
546 logging.debug(o)
547 exec (o, g, l)
548
549 if 'GAUDI_TEMP_OPTS_FILE' in os.environ:
550 os.remove(os.environ['GAUDI_TEMP_OPTS_FILE'])
551 opts.use_temp_opts = False
552
553 if opts.verbose and not opts.use_temp_opts:
554 c.printconfig(opts.old_opts, opts.all_opts)
555 if opts.output:
556 c.writeconfig(opts.output, opts.all_opts)
557
558 if opts.use_temp_opts:
559 fd, tmpfile = mkstemp('.opts')
560 os.close(fd)
561 c.writeconfig(tmpfile, opts.all_opts)
562 os.environ['GAUDI_TEMP_OPTS_FILE'] = tmpfile
563 logging.info('Restarting from pre-parsed options')
564 os.execv(sys.executable, [sys.executable] + sys.argv)
565
566 c.printsequence = opts.printsequence
567 if opts.printsequence:
568 if opts.ncpus:
569 logging.warning(
570 "--printsequence not supported with --ncpus: ignored")
571 elif opts.dry_run:
572 logging.warning(
573 "--printsequence not supported with --dry-run: ignored")
574
575 c.application = opts.application
576
577 # re-enable the GaudiPython module
578 del sys.modules["GaudiPython"]
579
580 if not opts.dry_run:
581 # Do the real processing
582 retcode = c.run(opts.gdb, opts.ncpus)
583
584 # Now saving the run information pid, retcode and executable path to
585 # a file is requested
586 if opts.run_info_file:
587 import os
588 import json
589 run_info = {}
590 run_info["pid"] = os.getpid()
591 run_info["retcode"] = retcode
592 if os.path.exists('/proc/self/exe'):
593 # These options can be used only on unix platforms
594 run_info["exe"] = os.readlink('/proc/self/exe')
595
596 logging.info("Saving run info to: %s" % opts.run_info_file)
597 with open(opts.run_info_file, "w") as f:
598 json.dump(run_info, f)
599
600 sys.exit(retcode)
GaudiKernel.ProcessJobOptions.PrintOff
def PrintOff(step=1)
Definition: ProcessJobOptions.py:132
gaudirun.FakeModule.exception
exception
Definition: gaudirun.py:504
gaudirun.FakeModule.__init__
def __init__(self, exception)
Definition: gaudirun.py:503
gaudirun.option_cb
def option_cb(option, opt, value, parser)
Definition: gaudirun.py:173
gaudirun.rationalizepath
def rationalizepath(path)
Definition: gaudirun.py:63
GaudiKernel.Configurable.applyConfigurableUsers
def applyConfigurableUsers()
Definition: Configurable.py:1519
gaudirun.getArgsFromQmt
def getArgsFromQmt(qmtfile)
Definition: gaudirun.py:79
GaudiKernel.ProcessJobOptions
Definition: ProcessJobOptions.py:1
GaudiKernel.ProcessJobOptions.InstallRootLoggingHandler
def InstallRootLoggingHandler(prefix=None, level=None, stream=None, with_time=False)
Definition: ProcessJobOptions.py:116
Containers::map
struct GAUDI_API map
Parametrisation class for map-like implementation.
Definition: KeyedObjectManager.h:25
gaudirun.FakeModule
Definition: gaudirun.py:502
gaudirun.getArgsWithoutoProfilerInfo
def getArgsWithoutoProfilerInfo(args)
Definition: gaudirun.py:8
Gaudi.Main
Definition: Main.py:1
gaudirun.FakeModule.__getattr__
def __getattr__(self, *args, **kwargs)
Definition: gaudirun.py:506
Gaudi.Main.gaudimain
Definition: Main.py:178
gaudirun.setLibraryPreload
def setLibraryPreload(newpreload)
Definition: gaudirun.py:37