| # Gaudi Framework, version v25r0 |

Home |  | Generated: Mon Feb 17 2014

- Main Page
- Related Pages
- Modules
- Namespaces
- Classes
- Files

- File List
- File Members

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
gaudirun.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2
3 import os
4 import sys
5
6 def getArgsWithoutoProfilerInfo(args):
7 """
8 Remove from the arguments the presence of the profiler and its output in
9 order to relaunch the script w/o infinite loops.
10
11 >>> getArgsWithoutoProfilerInfo(['--profilerName', 'igprof', 'myopts.py'])
12 ['myopts.py']
13
14 >>> getArgsWithoutoProfilerInfo(['--profilerName=igprof', 'myopts.py'])
15 ['myopts.py']
16
17 >>> getArgsWithoutoProfilerInfo(['--profilerName', 'igprof', '--profilerExtraOptions', 'a b c', 'myopts.py'])
18 ['myopts.py']
19
20 >>> getArgsWithoutoProfilerInfo(['--profilerName', 'igprof', '--options', 'a b c', 'myopts.py'])
21 ['--options', 'a b c', 'myopts.py']
22 """
23 newargs = []
24 args = list(args) # make a temp copy
25 while args:
26 o = args.pop(0)
27 if o.startswith('--profile'):
28 if '=' not in o:
29 args.pop(0)
30 else:
31 newargs.append(o)
32 return newargs
33
34 def rationalizepath(path):
35 '''
36 Convert the given path to a real path if the pointed file exists, otherwise
37 just normalize it.
38 '''
39 path = os.path.normpath(os.path.expandvars(path))
40 if os.path.exists(path):
41 path = os.path.realpath(path)
42 return path
43
44 # variable used to keep alive the temporary option files extracted from the .qmt
45 _qmt_tmp_opt_files = []
46 def getArgsFromQmt(qmtfile):
47 '''
48 Given a .qmt file, return the command line arguments of the corresponding
49 test.
50 '''
51 from xml.etree import ElementTree as ET
52 global _qmt_tmp_opt_files
53 # parse the .qmt file and extract args and options
54 qmt = ET.parse(qmtfile)
55 args = [a.text for a in qmt.findall("argument[@name='args']//text")]
56 options = qmt.find("argument[@name='options']/text")
57
58 if options is not None: # options need to be dumped in a temporary file
59 from tempfile import NamedTemporaryFile
60 import re
61 if re.search(r"from\s+Gaudi.Configuration\s+import\s+\*"
62 r"|from\s+Configurables\s+import", options.text):
63 tmp_opts = NamedTemporaryFile(suffix='.py')
64 else:
65 tmp_opts = NamedTemporaryFile(suffix='.opts')
66 tmp_opts.write(options.text)
67 tmp_opts.flush()
68 args.append(tmp_opts.name)
69 _qmt_tmp_opt_files.append(tmp_opts)
70
71 # relative paths in a .qmt are rooted in the qmtest directory, so
72 # - find where the .qmt lives
73 qmtfile = os.path.abspath(qmtfile)
74 if 'qmtest' in qmtfile.split(os.path.sep):
75 # this return the path up to the 'qmtest' entry in qmtfile
76 testdir = qmtfile
77 while os.path.basename(testdir) != 'qmtest':
78 testdir = os.path.dirname(testdir)
79 else:
80 testdir = '.'
81 # - temporarily switch to that directory and rationalize the paths
82 old_cwd = os.getcwd()
83 os.chdir(testdir)
84 args = map(rationalizepath, args)
85 os.chdir(old_cwd)
86
87 return args
88
89 #---------------------------------------------------------------------
90 if __name__ == "__main__":
91 # ensure that we (and the subprocesses) use the C standard localization
92 if os.environ.get('LC_ALL') != 'C':
93 print '# setting LC_ALL to "C"'
94 os.environ['LC_ALL'] = 'C'
95
96 from optparse import OptionParser
97 parser = OptionParser(usage = "%prog [options] <opts_file> ...")
98 parser.add_option("-n","--dry-run", action="store_true",
99 help="do not run the application, just parse option files")
100 parser.add_option("-p","--pickle-output", action="store", type="string",
101 metavar = "FILE",
102 help="DEPRECATED: use '--output file.pkl' instead. Write "
103 "the parsed options as a pickle file (static option "
104 "file)")
105 parser.add_option("-v","--verbose", action="store_true",
106 help="print the parsed options")
107 parser.add_option("--old-opts", action="store_true",
108 help="format printed options in old option files style")
109 parser.add_option("--all-opts", action="store_true",
110 help="print all the option (even if equal to default)")
111 # GaudiPython Parallel Mode Option
112 # Argument must be an integer in range [ -1, sys_cpus ]
113 # -1 : All available cpus
114 # 0 : Serial Mode (traditional gaudirun)
115 # n>0 : parallel with n cpus (n <= sys_cpus)
116 parser.add_option("--ncpus", action="store", type="int", default=0,
117 help="start the application in parallel mode using NCPUS processes. "
118 "0 => serial mode (default), -1 => use all CPUs")
119
120 def option_cb(option, opt, value, parser):
121 """Add the option line to a list together with its position in the
122 argument list.
123 """
124 parser.values.options.append((len(parser.largs), value))
125 parser.add_option("--option", action="callback", callback=option_cb,
126 type = "string", nargs = 1,
127 help="add a single line (Python) option to the configuration. "
128 "All options lines are executed, one after the other, in "
129 "the same context.")
130 parser.add_option("--no-conf-user-apply", action="store_true",
131 help="disable the automatic application of configurable "
132 "users (for backward compatibility)")
133 parser.add_option("--old-conf-user-apply", action="store_true",
134 help="use the old logic when applying ConfigurableUsers "
135 "(with bug #103803) [default]")
136 parser.add_option("--new-conf-user-apply", action="store_false",
137 dest="old_conf_user_apply",
138 help="use the new (correct) logic when applying "
139 "ConfigurableUsers (fixed bug #103803), can be "
140 "turned on also with the environment variable "
141 "GAUDI_FIXED_APPLY_CONF")
142 parser.add_option("-o", "--output", action = "store", type = "string",
143 help ="dump the configuration to a file. The format of "
144 "the options is determined by the extension of the "
145 "file name: .pkl = pickle, .py = python, .opts = "
146 "old style options. The python format cannot be "
147 "used to run the application and it contains the "
148 "same dictionary printed with -v")
149 parser.add_option("--post-option", action="append", type="string",
150 dest="post_options",
151 help="Python options to be executed after the ConfigurableUser "
152 "are applied. "
153 "All options lines are executed, one after the other, in "
154 "the same context.")
155 parser.add_option("--debug", action="store_true",
156 help="enable some debug print-out")
157 parser.add_option("--printsequence", action="store_true",
158 help="print the sequence")
159 if not sys.platform.startswith("win"):
160 # These options can be used only on unix platforms
161 parser.add_option("-T", "--tcmalloc", action="store_true",
162 help="Use the Google malloc replacement. The environment "
163 "variable TCMALLOCLIB can be used to specify a different "
164 "name for the library (the default is libtcmalloc.so)")
165 parser.add_option("--preload", action="append",
166 help="Allow pre-loading of special libraries (e.g. Google "
167 "profiling libraries).")
168
169 # Option to use a profiler
170 parser.add_option("--profilerName", type="string",
171 help="Select one profiler among: igprofPerf, igprofMem and valgrind<toolname>")
172
173 # Option to specify the filename where to collect the profiler's output
174 parser.add_option("--profilerOutput", type="string",
175 help="Specify the name of the output file for the profiler output")
176
177 # Option to specify the filename where to collect the profiler's output
178 parser.add_option("--profilerExtraOptions", type="string",
179 help="Specify additional options for the profiler. The '--' string should be expressed as '__' (--my-opt becomes __my-opt)")
180
181 parser.set_defaults(options = [],
182 tcmalloc = False,
183 profilerName = '',
184 profilerOutput = '',
185 profilerExtraOptions = '',
186 preload = [],
187 ncpus = None,
188 # the old logic can be turned off with an env variable
189 old_conf_user_apply='GAUDI_FIXED_APPLY_CONF' not in os.environ)
190
191 # replace .qmt files in the command line with their contained args
192 argv = []
193 for a in sys.argv[1:]:
194 if a.endswith('.qmt') and os.path.exists(a):
195 argv.extend(getArgsFromQmt(a))
196 else:
197 argv.append(a)
198 if argv != sys.argv[1:]:
199 print '# Running', sys.argv[0], 'with arguments', argv
200
201 opts, args = parser.parse_args(args=argv)
202
203 # Check consistency of options
204
205 # Parallel Option ---------------------------------------------------------
206 if opts.ncpus:
207 from multiprocessing import cpu_count
208 sys_cpus = cpu_count()
209 if opts.ncpus > sys_cpus:
210 s = "Invalid value : --ncpus : only %i cpus available" % sys_cpus
211 parser.error(s)
212 elif opts.ncpus < -1 :
213 s = "Invalid value : --ncpus must be integer >= -1"
214 parser.error(s)
215 else:
216 # FIXME: is it really needed to set it to None if it is 0 or False?
217 opts.ncpus = None
218
219 # configure the logging
220 import logging
221 from GaudiKernel.ProcessJobOptions import InstallRootLoggingHandler
222
223 if opts.old_opts: prefix = "// "
224 else: prefix = "# "
225 level = logging.INFO
226 if opts.debug:
227 level = logging.DEBUG
228 InstallRootLoggingHandler(prefix, level = level)
229 root_logger = logging.getLogger()
230
231 # tcmalloc support
232 if opts.tcmalloc:
233 opts.preload.insert(0, os.environ.get("TCMALLOCLIB", "libtcmalloc.so"))
234 # allow preloading of libraries
235 if opts.preload:
236 preload = os.environ.get("LD_PRELOAD", "")
237 if preload:
238 preload = preload.replace(" ", ":").split(":")
239 else:
240 preload = []
241 for libname in set(preload).intersection(opts.preload):
242 logging.warning("Ignoring preload of library %s because it is "
243 "already in LD_PRELOAD.", libname)
244 to_load = [libname
245 for libname in opts.preload
246 if libname not in set(preload)]
247 if to_load:
248 preload += to_load
249 preload = ":".join(preload)
250 os.environ["LD_PRELOAD"] = preload
251 logging.info("Restarting with LD_PRELOAD='%s'", preload)
252 # remove the --tcmalloc option from the arguments
253 # FIXME: the --preload arguments will issue a warning but it's tricky to remove them
254 args = [ a for a in sys.argv if a != '-T' and not '--tcmalloc'.startswith(a) ]
255 os.execv(sys.executable, [sys.executable] + args)
256
257 # Profiler Support ------
258 if opts.profilerName:
259 profilerName = opts.profilerName
260 profilerExecName = ""
261 profilerOutput = opts.profilerOutput or (profilerName + ".output")
262
263 # To restart the application removing the igprof option and prepending the string
264 args = getArgsWithoutoProfilerInfo(sys.argv)
265
266 igprofPerfOptions = "-d -pp -z -o igprof.pp.gz".split()
267
268 profilerOptions = ""
269 if profilerName == "igprof":
270 if not opts.profilerOutput:
271 profilerOutput += ".profile.gz"
272 profilerOptions = "-d -z -o %s" % profilerOutput
273 profilerExecName = "igprof"
274
275 elif profilerName == "igprofPerf":
276 if not opts.profilerOutput:
277 profilerOutput += ".pp.gz"
278 profilerOptions = "-d -pp -z -o %s" % profilerOutput
279 profilerExecName = "igprof"
280
281 elif profilerName == "igprofMem":
282 if not opts.profilerOutput:
283 profilerOutput += ".mp.gz"
284 profilerOptions = "-d -mp -z -o %s" % profilerOutput
285 profilerExecName = "igprof"
286
287 elif "valgrind" in profilerName:
288 # extract the tool
289 if not opts.profilerOutput:
290 profilerOutput += ".log"
291 toolname = profilerName.replace('valgrind','')
292 outoption = "--log-file"
293 if toolname in ("massif", "callgrind", "cachegrind"):
294 outoption = "--%s-out-file" % toolname
295 profilerOptions = "--tool=%s %s=%s" % (toolname, outoption, profilerOutput)
296 profilerExecName = "valgrind"
297
298 else:
299 root_logger.warning("Profiler %s not recognized!" % profilerName)
300
301 # Add potential extra options
302 if opts.profilerExtraOptions!="":
303 profilerExtraOptions = opts.profilerExtraOptions
304 profilerExtraOptions = profilerExtraOptions.replace("__","--")
305 profilerOptions += " %s" % profilerExtraOptions
306
307 # now we look for the full path of the profiler: is it really there?
308 import distutils.spawn
309 profilerPath = distutils.spawn.find_executable(profilerExecName)
310 if not profilerPath:
311 root_logger.error("Cannot locate profiler %s" % profilerExecName)
312 sys.exit(1)
313
314 root_logger.info("------ Profiling options are on ------ \n"\
315 " o Profiler: %s\n"\
316 " o Options: '%s'.\n"\
317 " o Output: %s" % (profilerExecName, profilerOptions, profilerOutput))
318
319 # We profile python
320 profilerOptions += " python"
321
322 # now we have all the ingredients to prepare our command
323 arglist = [profilerPath] + profilerOptions.split() + args
324 arglist = [ a for a in arglist if a!='' ]
325 #print profilerPath
326 #for arg in arglist:
327 #print arg
328 os.execv(profilerPath, arglist)
329
330 # End Profiler Support ------
331
332 if opts.pickle_output:
333 if opts.output:
334 root_logger.error("Conflicting options: use only --pickle-output or --output")
335 sys.exit(1)
336 else:
337 root_logger.warning("--pickle-output is deprecated, use --output instead")
338 opts.output = opts.pickle_output
339
340 from Gaudi.Main import gaudimain
341 c = gaudimain()
342
343 # Prepare the "configuration script" to parse (like this it is easier than
344 # having a list with files and python commands, with an if statements that
345 # decides to do importOptions or exec)
346 options = [ "importOptions(%r)" % f for f in args ]
347 # The option lines are inserted into the list of commands using their
348 # position on the command line
349 optlines = list(opts.options)
350 optlines.reverse() # this allows to avoid to have to care about corrections of the positions
351 for pos, l in optlines:
352 options.insert(pos,l)
353
354 # prevent the usage of GaudiPython
355 class FakeModule(object):
356 def __init__(self, exception):
357 self.exception = exception
358 def __getattr__(self, *args, **kwargs):
359 raise self.exception
360 sys.modules["GaudiPython"] = FakeModule(RuntimeError("GaudiPython cannot be used in option files"))
361
362 # "execute" the configuration script generated (if any)
363 if options:
364 g = {}
365 l = {}
366 exec "from Gaudi.Configuration import *" in g, l
367 for o in options:
368 logging.debug(o)
369 exec o in g, l
370
371 import GaudiKernel.Proxy.Configurable
372 if opts.no_conf_user_apply:
373 logging.info("Disabling automatic apply of ConfigurableUser")
374 # pretend that they have been already applied
375 GaudiKernel.Proxy.Configurable._appliedConfigurableUsers_ = True
376
377 # This need to be done before dumping
378 if opts.old_conf_user_apply:
379 from GaudiKernel.Proxy.Configurable import applyConfigurableUsers_old as applyConfigurableUsers
380 else:
381 from GaudiKernel.Proxy.Configurable import applyConfigurableUsers
382 applyConfigurableUsers()
383
384 # Options to be processed after applyConfigurableUsers
385 if opts.post_options:
386 g = {}
387 l = {}
388 exec "from Gaudi.Configuration import *" in g, l
389 for o in opts.post_options:
390 logging.debug(o)
391 exec o in g, l
392
393 if opts.verbose:
394 c.printconfig(opts.old_opts, opts.all_opts)
395 if opts.output:
396 c.writeconfig(opts.output, opts.all_opts)
397
398 c.printsequence = opts.printsequence
399 if opts.printsequence:
400 if opts.ncpus:
401 logging.warning("--printsequence not supported with --ncpus: ignored")
402 elif opts.dry_run:
403 logging.warning("--printsequence not supported with --dry-run: ignored")
404
405 # re-enable the GaudiPython module
406 del sys.modules["GaudiPython"]
407
408 if not opts.dry_run:
409 # Do the real processing
410 sys.exit(c.run(opts.ncpus))