5 from tempfile
import mkstemp
10 Remove from the arguments the presence of the profiler and its output in 11 order to relaunch the script w/o infinite loops. 13 >>> getArgsWithoutoProfilerInfo(['--profilerName', 'igprof', 'myopts.py']) 16 >>> getArgsWithoutoProfilerInfo(['--profilerName=igprof', 'myopts.py']) 19 >>> getArgsWithoutoProfilerInfo(['--profilerName', 'igprof', '--profilerExtraOptions', 'a b c', 'myopts.py']) 22 >>> getArgsWithoutoProfilerInfo(['--profilerName', 'igprof', '--options', 'a b c', 'myopts.py']) 23 ['--options', 'a b c', 'myopts.py'] 29 if o.startswith(
'--profile'):
38 ''' Adds a list of libraries to LD_PRELOAD ''' 39 preload = os.environ.get(
"LD_PRELOAD",
"")
41 preload = preload.replace(
" ",
":").split(
":")
45 for libname
in set(preload).intersection(newpreload):
47 "Ignoring preload of library %s because it is " 48 "already in LD_PRELOAD.", libname)
51 libname
for libname
in newpreload
if libname
not in set(preload)
56 preload =
":".join(preload)
57 os.environ[
"LD_PRELOAD"] = preload
58 logging.info(
"Setting LD_PRELOAD='%s'", preload)
65 Convert the given path to a real path if the pointed file exists, otherwise 68 path = os.path.normpath(os.path.expandvars(path))
69 if os.path.exists(path):
70 path = os.path.realpath(path)
76 _qmt_tmp_opt_files = []
81 Given a .qmt file, return the command line arguments of the corresponding 84 from xml.etree
import ElementTree
as ET
85 global _qmt_tmp_opt_files
87 qmt = ET.parse(qmtfile)
88 args = [a.text
for a
in qmt.findall(
"argument[@name='args']//text")]
89 options = qmt.find(
"argument[@name='options']/text")
91 if options
is not None:
92 from tempfile
import NamedTemporaryFile
95 r"from\s+Gaudi.Configuration\s+import\s+\*" 96 r"|from\s+Configurables\s+import", options.text):
97 tmp_opts = NamedTemporaryFile(suffix=
'.py')
99 tmp_opts = NamedTemporaryFile(suffix=
'.opts')
100 tmp_opts.write(options.text)
102 args.append(tmp_opts.name)
103 _qmt_tmp_opt_files.append(tmp_opts)
107 qmtfile = os.path.abspath(qmtfile)
108 if 'qmtest' in qmtfile.split(os.path.sep):
111 while os.path.basename(testdir) !=
'qmtest':
112 testdir = os.path.dirname(testdir)
116 old_cwd = os.getcwd()
118 args =
map(rationalizepath, args)
125 if __name__ ==
"__main__":
127 if os.environ.get(
'LC_ALL') !=
'C':
128 print '# setting LC_ALL to "C"' 129 os.environ[
'LC_ALL'] =
'C' 131 from optparse
import OptionParser
132 parser = OptionParser(usage=
"%prog [options] <opts_file> ...")
137 help=
"do not run the application, just parse option files")
144 help=
"DEPRECATED: use '--output file.pkl' instead. Write " 145 "the parsed options as a pickle file (static option " 151 help=
"print the parsed options")
155 help=
"format printed options in old option files style")
159 help=
"print all the option (even if equal to default)")
170 help=
"start the application in parallel mode using NCPUS processes. " 171 "0 => serial mode (default), -1 => use all CPUs")
174 """Add the option line to a list together with its position in the 177 parser.values.options.append((len(parser.largs), value))
185 help=
"add a single line (Python) option to the configuration. " 186 "All options lines are executed, one after the other, in " 189 "--no-conf-user-apply",
191 help=
"disable the automatic application of configurable " 192 "users (for backward compatibility)")
194 "--old-conf-user-apply",
196 help=
"use the old logic when applying ConfigurableUsers " 197 "(with bug #103803) [default]")
199 "--new-conf-user-apply",
200 action=
"store_false",
201 dest=
"old_conf_user_apply",
202 help=
"use the new (correct) logic when applying " 203 "ConfigurableUsers (fixed bug #103803), can be " 204 "turned on also with the environment variable " 205 "GAUDI_FIXED_APPLY_CONF")
211 help=
"dump the configuration to a file. The format of " 212 "the options is determined by the extension of the " 213 "file name: .pkl = pickle, .py = python, .opts = " 214 "old style options. The python format cannot be " 215 "used to run the application and it contains the " 216 "same dictionary printed with -v")
222 help=
"Python options to be executed after the ConfigurableUser " 224 "All options lines are executed, one after the other, in " 227 "--debug", action=
"store_true", help=
"enable some debug print-out")
228 parser.add_option(
"--gdb", action=
"store_true", help=
"attach gdb")
230 "--printsequence", action=
"store_true", help=
"print the sequence")
231 if not sys.platform.startswith(
"win"):
237 help=
"Use the Google malloc replacement. The environment " 238 "variable TCMALLOCLIB can be used to specify a different " 239 "name for the library (the default is libtcmalloc.so)")
243 help=
"Allow pre-loading of special libraries (e.g. Google " 244 "profiling libraries).")
250 "Select one profiler among: igprofPerf, igprofMem and valgrind<toolname>" 257 help=
"Specify the name of the output file for the profiler output")
261 "--profilerExtraOptions",
264 "Specify additional options for the profiler. The '--' string should be expressed as '__' (--my-opt becomes __my-opt)" 270 help=
'when this option is enabled, the options are parsed' 271 ' and stored in a temporary file, then the job is ' 272 'restarted using that file as input (to save ' 278 "Save gaudi process information to the file specified (in JSON format)" 286 profilerExtraOptions=
'',
290 old_conf_user_apply=
'GAUDI_FIXED_APPLY_CONF' not in os.environ,
295 for a
in sys.argv[1:]:
296 if a.endswith(
'.qmt')
and os.path.exists(a):
300 if argv != sys.argv[1:]:
301 print '# Running', sys.argv[0],
'with arguments', argv
303 opts, args = parser.parse_args(args=argv)
309 from multiprocessing
import cpu_count
310 sys_cpus = cpu_count()
311 if opts.ncpus > sys_cpus:
312 s =
"Invalid value : --ncpus : only %i cpus available" % sys_cpus
314 elif opts.ncpus < -1:
315 s =
"Invalid value : --ncpus must be integer >= -1" 332 level = logging.DEBUG
334 root_logger = logging.getLogger()
337 sanitizer = os.environ.get(
"PRELOAD_SANITIZER_LIB",
"")
339 if sanitizer
not in os.environ.get(
"LD_PRELOAD",
""):
340 opts.preload.insert(0, sanitizer)
341 os.environ[
"PRELOAD_SANITIZER_LIB"] =
"" 347 logging.warning(
"tcmalloc preload disabled when using a sanitizer")
350 0, os.environ.get(
"TCMALLOCLIB",
"libtcmalloc.so"))
354 preload = os.environ.get(
"LD_PRELOAD",
"")
356 preload = preload.replace(
" ",
":").split(
":")
359 for libname
in set(preload).intersection(opts.preload):
361 "Ignoring preload of library %s because it is " 362 "already in LD_PRELOAD.", libname)
364 libname
for libname
in opts.preload
if libname
not in set(preload)
368 preload =
":".join(preload)
369 os.environ[
"LD_PRELOAD"] = preload
370 logging.info(
"Restarting with LD_PRELOAD='%s'", preload)
375 if a !=
'-T' and not '--tcmalloc'.startswith(a)
377 os.execv(sys.executable, [sys.executable] + args)
380 if opts.profilerName:
381 profilerName = opts.profilerName
382 profilerExecName =
"" 383 profilerOutput = opts.profilerOutput
or (profilerName +
".output")
388 igprofPerfOptions =
"-d -pp -z -o igprof.pp.gz".split()
391 if profilerName ==
"igprof":
392 if not opts.profilerOutput:
393 profilerOutput +=
".profile.gz" 394 profilerOptions =
"-d -z -o %s" % profilerOutput
395 profilerExecName =
"igprof" 397 elif profilerName ==
"igprofPerf":
398 if not opts.profilerOutput:
399 profilerOutput +=
".pp.gz" 400 profilerOptions =
"-d -pp -z -o %s" % profilerOutput
401 profilerExecName =
"igprof" 403 elif profilerName ==
"igprofMem":
404 if not opts.profilerOutput:
405 profilerOutput +=
".mp.gz" 406 profilerOptions =
"-d -mp -z -o %s" % profilerOutput
407 profilerExecName =
"igprof" 409 elif "valgrind" in profilerName:
411 if not opts.profilerOutput:
412 profilerOutput +=
".log" 413 toolname = profilerName.replace(
'valgrind',
'')
414 outoption =
"--log-file" 415 if toolname
in (
"massif",
"callgrind",
"cachegrind"):
416 outoption =
"--%s-out-file" % toolname
417 profilerOptions =
"--tool=%s %s=%s" % (toolname, outoption,
419 profilerExecName =
"valgrind" 421 elif profilerName ==
"jemalloc":
423 0, os.environ.get(
"JEMALLOCLIB",
"libjemalloc.so"))
424 os.environ[
'MALLOC_CONF'] =
"prof:true,prof_leak:true" 426 root_logger.warning(
"Profiler %s not recognized!" % profilerName)
429 if opts.profilerExtraOptions !=
"":
430 profilerExtraOptions = opts.profilerExtraOptions
431 profilerExtraOptions = profilerExtraOptions.replace(
"__",
"--")
432 profilerOptions +=
" %s" % profilerExtraOptions
436 import distutils.spawn
437 profilerPath = distutils.spawn.find_executable(profilerExecName)
440 "Cannot locate profiler %s" % profilerExecName)
443 root_logger.info(
"------ Profiling options are on ------ \n" 445 " o Options: '%s'.\n" 446 " o Output: %s" % (profilerExecName
or profilerName,
447 profilerOptions, profilerOutput))
457 profilerOptions +=
" python" 460 arglist = [profilerPath] + profilerOptions.split() + args
461 arglist = [a
for a
in arglist
if a !=
'']
465 os.execv(profilerPath, arglist)
467 arglist = [a
for a
in sys.argv
if not a.startswith(
"--profiler")]
468 os.execv(sys.executable, [sys.executable] + arglist)
472 if opts.pickle_output:
475 "Conflicting options: use only --pickle-output or --output")
479 "--pickle-output is deprecated, use --output instead")
480 opts.output = opts.pickle_output
488 options = [
"importOptions(%r)" % f
for f
in args]
491 optlines = list(opts.options)
494 for pos, l
in optlines:
495 options.insert(pos, l)
506 RuntimeError(
"GaudiPython cannot be used in option files"))
510 if 'GAUDI_TEMP_OPTS_FILE' in os.environ:
511 options = [
'importOptions(%r)' % os.environ[
'GAUDI_TEMP_OPTS_FILE']]
518 exec
"from Gaudi.Configuration import *" in g, l
523 import GaudiKernel.Proxy.Configurable
524 if opts.no_conf_user_apply:
525 logging.info(
"Disabling automatic apply of ConfigurableUser")
527 GaudiKernel.Proxy.Configurable._appliedConfigurableUsers_ =
True 530 if opts.old_conf_user_apply:
531 from GaudiKernel.Proxy.Configurable
import applyConfigurableUsers_old
as applyConfigurableUsers
533 from GaudiKernel.Proxy.Configurable
import applyConfigurableUsers
537 if opts.post_options:
540 exec
"from Gaudi.Configuration import *" in g, l
541 for o
in opts.post_options:
545 if 'GAUDI_TEMP_OPTS_FILE' in os.environ:
546 os.remove(os.environ[
'GAUDI_TEMP_OPTS_FILE'])
547 opts.use_temp_opts =
False 549 if opts.verbose
and not opts.use_temp_opts:
550 c.printconfig(opts.old_opts, opts.all_opts)
552 c.writeconfig(opts.output, opts.all_opts)
554 if opts.use_temp_opts:
555 fd, tmpfile = mkstemp(
'.opts')
557 c.writeconfig(tmpfile, opts.all_opts)
558 os.environ[
'GAUDI_TEMP_OPTS_FILE'] = tmpfile
559 logging.info(
'Restarting from pre-parsed options')
560 os.execv(sys.executable, [sys.executable] + sys.argv)
562 c.printsequence = opts.printsequence
563 if opts.printsequence:
566 "--printsequence not supported with --ncpus: ignored")
569 "--printsequence not supported with --dry-run: ignored")
572 del sys.modules[
"GaudiPython"]
576 retcode = c.run(opts.gdb, opts.ncpus)
580 if opts.run_info_file:
584 run_info[
"pid"] = os.getpid()
585 run_info[
"retcode"] = retcode
586 if os.path.exists(
'/proc/self/exe'):
588 run_info[
"exe"] = os.readlink(
'/proc/self/exe')
590 logging.info(
"Saving run info to: %s" % opts.run_info_file)
591 with open(opts.run_info_file,
"w")
as f:
592 json.dump(run_info, f)
def __init__(self, exception)
def option_cb(option, opt, value, parser)
def rationalizepath(path)
def applyConfigurableUsers()
def getArgsFromQmt(qmtfile)
def InstallRootLoggingHandler(prefix=None, level=None, stream=None, with_time=False)
struct GAUDI_API map
Parametrisation class for map-like implementation.
def getArgsWithoutoProfilerInfo(args)
def __getattr__(self, args, kwargs)
def setLibraryPreload(newpreload)