All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
gaudirun.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 import os
4 import sys
5 
7  """
8  Remove from the arguments the presence of the profiler and its output in
9  order to relaunch the script w/o infinite loops.
10 
11  >>> getArgsWithoutoProfilerInfo(['--profilerName', 'igprof', 'myopts.py'])
12  ['myopts.py']
13 
14  >>> getArgsWithoutoProfilerInfo(['--profilerName=igprof', 'myopts.py'])
15  ['myopts.py']
16 
17  >>> getArgsWithoutoProfilerInfo(['--profilerName', 'igprof', '--profilerExtraOptions', 'a b c', 'myopts.py'])
18  ['myopts.py']
19 
20  >>> getArgsWithoutoProfilerInfo(['--profilerName', 'igprof', '--options', 'a b c', 'myopts.py'])
21  ['--options', 'a b c', 'myopts.py']
22  """
23  newargs = []
24  args = list(args) # make a temp copy
25  while args:
26  o = args.pop(0)
27  if o.startswith('--profile'):
28  if '=' not in o:
29  args.pop(0)
30  else:
31  newargs.append(o)
32  return newargs
33 
34 def rationalizepath(path):
35  '''
36  Convert the given path to a real path if the pointed file exists, otherwise
37  just normalize it.
38  '''
39  path = os.path.normpath(os.path.expandvars(path))
40  if os.path.exists(path):
41  path = os.path.realpath(path)
42  return path
43 
44 # variable used to keep alive the temporary option files extracted from the .qmt
45 _qmt_tmp_opt_files = []
46 def getArgsFromQmt(qmtfile):
47  '''
48  Given a .qmt file, return the command line arguments of the corresponding
49  test.
50  '''
51  from xml.etree import ElementTree as ET
52  global _qmt_tmp_opt_files
53  # parse the .qmt file and extract args and options
54  qmt = ET.parse(qmtfile)
55  args = [a.text for a in qmt.findall("argument[@name='args']//text")]
56  options = qmt.find("argument[@name='options']/text")
57 
58  if options is not None: # options need to be dumped in a temporary file
59  from tempfile import NamedTemporaryFile
60  import re
61  if re.search(r"from\s+Gaudi.Configuration\s+import\s+\*"
62  r"|from\s+Configurables\s+import", options.text):
63  tmp_opts = NamedTemporaryFile(suffix='.py')
64  else:
65  tmp_opts = NamedTemporaryFile(suffix='.opts')
66  tmp_opts.write(options.text)
67  tmp_opts.flush()
68  args.append(tmp_opts.name)
69  _qmt_tmp_opt_files.append(tmp_opts)
70 
71  # relative paths in a .qmt are rooted in the qmtest directory, so
72  # - find where the .qmt lives
73  qmtfile = os.path.abspath(qmtfile)
74  if 'qmtest' in qmtfile.split(os.path.sep):
75  # this return the path up to the 'qmtest' entry in qmtfile
76  testdir = qmtfile
77  while os.path.basename(testdir) != 'qmtest':
78  testdir = os.path.dirname(testdir)
79  else:
80  testdir = '.'
81  # - temporarily switch to that directory and rationalize the paths
82  old_cwd = os.getcwd()
83  os.chdir(testdir)
84  args = map(rationalizepath, args)
85  os.chdir(old_cwd)
86 
87  return args
88 
89 #---------------------------------------------------------------------
90 if __name__ == "__main__":
91  # ensure that we (and the subprocesses) use the C standard localization
92  if os.environ.get('LC_ALL') != 'C':
93  print '# setting LC_ALL to "C"'
94  os.environ['LC_ALL'] = 'C'
95 
96  from optparse import OptionParser
97  parser = OptionParser(usage = "%prog [options] <opts_file> ...")
98  parser.add_option("-n","--dry-run", action="store_true",
99  help="do not run the application, just parse option files")
100  parser.add_option("-p","--pickle-output", action="store", type="string",
101  metavar = "FILE",
102  help="DEPRECATED: use '--output file.pkl' instead. Write "
103  "the parsed options as a pickle file (static option "
104  "file)")
105  parser.add_option("-v","--verbose", action="store_true",
106  help="print the parsed options")
107  parser.add_option("--old-opts", action="store_true",
108  help="format printed options in old option files style")
109  parser.add_option("--all-opts", action="store_true",
110  help="print all the option (even if equal to default)")
111  # GaudiPython Parallel Mode Option
112  # Argument must be an integer in range [ -1, sys_cpus ]
113  # -1 : All available cpus
114  # 0 : Serial Mode (traditional gaudirun)
115  # n>0 : parallel with n cpus (n <= sys_cpus)
116  parser.add_option("--ncpus", action="store", type="int", default=0,
117  help="start the application in parallel mode using NCPUS processes. "
118  "0 => serial mode (default), -1 => use all CPUs")
119 
120  def option_cb(option, opt, value, parser):
121  """Add the option line to a list together with its position in the
122  argument list.
123  """
124  parser.values.options.append((len(parser.largs), value))
125  parser.add_option("--option", action="callback", callback=option_cb,
126  type = "string", nargs = 1,
127  help="add a single line (Python) option to the configuration. "
128  "All options lines are executed, one after the other, in "
129  "the same context.")
130  parser.add_option("--no-conf-user-apply", action="store_true",
131  help="disable the automatic application of configurable "
132  "users (for backward compatibility)")
133  parser.add_option("--old-conf-user-apply", action="store_true",
134  help="use the old logic when applying ConfigurableUsers "
135  "(with bug #103803) [default]")
136  parser.add_option("--new-conf-user-apply", action="store_false",
137  dest="old_conf_user_apply",
138  help="use the new (correct) logic when applying "
139  "ConfigurableUsers (fixed bug #103803), can be "
140  "turned on also with the environment variable "
141  "GAUDI_FIXED_APPLY_CONF")
142  parser.add_option("-o", "--output", action = "store", type = "string",
143  help ="dump the configuration to a file. The format of "
144  "the options is determined by the extension of the "
145  "file name: .pkl = pickle, .py = python, .opts = "
146  "old style options. The python format cannot be "
147  "used to run the application and it contains the "
148  "same dictionary printed with -v")
149  parser.add_option("--post-option", action="append", type="string",
150  dest="post_options",
151  help="Python options to be executed after the ConfigurableUser "
152  "are applied. "
153  "All options lines are executed, one after the other, in "
154  "the same context.")
155  parser.add_option("--debug", action="store_true",
156  help="enable some debug print-out")
157  parser.add_option("--printsequence", action="store_true",
158  help="print the sequence")
159  if not sys.platform.startswith("win"):
160  # These options can be used only on unix platforms
161  parser.add_option("-T", "--tcmalloc", action="store_true",
162  help="Use the Google malloc replacement. The environment "
163  "variable TCMALLOCLIB can be used to specify a different "
164  "name for the library (the default is libtcmalloc.so)")
165  parser.add_option("--preload", action="append",
166  help="Allow pre-loading of special libraries (e.g. Google "
167  "profiling libraries).")
168 
169  # Option to use a profiler
170  parser.add_option("--profilerName", type="string",
171  help="Select one profiler among: igprofPerf, igprofMem and valgrind<toolname>")
172 
173  # Option to specify the filename where to collect the profiler's output
174  parser.add_option("--profilerOutput", type="string",
175  help="Specify the name of the output file for the profiler output")
176 
177  # Option to specify the filename where to collect the profiler's output
178  parser.add_option("--profilerExtraOptions", type="string",
179  help="Specify additional options for the profiler. The '--' string should be expressed as '__' (--my-opt becomes __my-opt)")
180 
181  parser.set_defaults(options = [],
182  tcmalloc = False,
183  profilerName = '',
184  profilerOutput = '',
185  profilerExtraOptions = '',
186  preload = [],
187  ncpus = None,
188  # the old logic can be turned off with an env variable
189  old_conf_user_apply='GAUDI_FIXED_APPLY_CONF' not in os.environ)
190 
191  # replace .qmt files in the command line with their contained args
192  argv = []
193  for a in sys.argv[1:]:
194  if a.endswith('.qmt') and os.path.exists(a):
195  argv.extend(getArgsFromQmt(a))
196  else:
197  argv.append(a)
198  if argv != sys.argv[1:]:
199  print '# Running', sys.argv[0], 'with arguments', argv
200 
201  opts, args = parser.parse_args(args=argv)
202 
203  # Check consistency of options
204 
205  # Parallel Option ---------------------------------------------------------
206  if opts.ncpus:
207  from multiprocessing import cpu_count
208  sys_cpus = cpu_count()
209  if opts.ncpus > sys_cpus:
210  s = "Invalid value : --ncpus : only %i cpus available" % sys_cpus
211  parser.error(s)
212  elif opts.ncpus < -1 :
213  s = "Invalid value : --ncpus must be integer >= -1"
214  parser.error(s)
215  else:
216  # FIXME: is it really needed to set it to None if it is 0 or False?
217  opts.ncpus = None
218 
219  # configure the logging
220  import logging
221  from GaudiKernel.ProcessJobOptions import InstallRootLoggingHandler
222 
223  if opts.old_opts: prefix = "// "
224  else: prefix = "# "
225  level = logging.INFO
226  if opts.debug:
227  level = logging.DEBUG
228  InstallRootLoggingHandler(prefix, level = level)
229  root_logger = logging.getLogger()
230 
231  # tcmalloc support
232  if opts.tcmalloc:
233  opts.preload.insert(0, os.environ.get("TCMALLOCLIB", "libtcmalloc.so"))
234  # allow preloading of libraries
235  if opts.preload:
236  preload = os.environ.get("LD_PRELOAD", "")
237  if preload:
238  preload = preload.replace(" ", ":").split(":")
239  else:
240  preload = []
241  for libname in set(preload).intersection(opts.preload):
242  logging.warning("Ignoring preload of library %s because it is "
243  "already in LD_PRELOAD.", libname)
244  to_load = [libname
245  for libname in opts.preload
246  if libname not in set(preload)]
247  if to_load:
248  preload += to_load
249  preload = ":".join(preload)
250  os.environ["LD_PRELOAD"] = preload
251  logging.info("Restarting with LD_PRELOAD='%s'", preload)
252  # remove the --tcmalloc option from the arguments
253  # FIXME: the --preload arguments will issue a warning but it's tricky to remove them
254  args = [ a for a in sys.argv if a != '-T' and not '--tcmalloc'.startswith(a) ]
255  os.execv(sys.executable, [sys.executable] + args)
256 
257  # Profiler Support ------
258  if opts.profilerName:
259  profilerName = opts.profilerName
260  profilerExecName = ""
261  profilerOutput = opts.profilerOutput or (profilerName + ".output")
262 
263  # To restart the application removing the igprof option and prepending the string
264  args = getArgsWithoutoProfilerInfo(sys.argv)
265 
266  igprofPerfOptions = "-d -pp -z -o igprof.pp.gz".split()
267 
268  profilerOptions = ""
269  if profilerName == "igprof":
270  if not opts.profilerOutput:
271  profilerOutput += ".profile.gz"
272  profilerOptions = "-d -z -o %s" % profilerOutput
273  profilerExecName = "igprof"
274 
275  elif profilerName == "igprofPerf":
276  if not opts.profilerOutput:
277  profilerOutput += ".pp.gz"
278  profilerOptions = "-d -pp -z -o %s" % profilerOutput
279  profilerExecName = "igprof"
280 
281  elif profilerName == "igprofMem":
282  if not opts.profilerOutput:
283  profilerOutput += ".mp.gz"
284  profilerOptions = "-d -mp -z -o %s" % profilerOutput
285  profilerExecName = "igprof"
286 
287  elif "valgrind" in profilerName:
288  # extract the tool
289  if not opts.profilerOutput:
290  profilerOutput += ".log"
291  toolname = profilerName.replace('valgrind','')
292  outoption = "--log-file"
293  if toolname in ("massif", "callgrind", "cachegrind"):
294  outoption = "--%s-out-file" % toolname
295  profilerOptions = "--tool=%s %s=%s" % (toolname, outoption, profilerOutput)
296  profilerExecName = "valgrind"
297 
298  else:
299  root_logger.warning("Profiler %s not recognized!" % profilerName)
300 
301  # Add potential extra options
302  if opts.profilerExtraOptions!="":
303  profilerExtraOptions = opts.profilerExtraOptions
304  profilerExtraOptions = profilerExtraOptions.replace("__","--")
305  profilerOptions += " %s" % profilerExtraOptions
306 
307  # now we look for the full path of the profiler: is it really there?
308  import distutils.spawn
309  profilerPath = distutils.spawn.find_executable(profilerExecName)
310  if not profilerPath:
311  root_logger.error("Cannot locate profiler %s" % profilerExecName)
312  sys.exit(1)
313 
314  root_logger.info("------ Profiling options are on ------ \n"\
315  " o Profiler: %s\n"\
316  " o Options: '%s'.\n"\
317  " o Output: %s" % (profilerExecName, profilerOptions, profilerOutput))
318 
319  # We profile python
320  profilerOptions += " python"
321 
322  # now we have all the ingredients to prepare our command
323  arglist = [profilerPath] + profilerOptions.split() + args
324  arglist = [ a for a in arglist if a!='' ]
325  #print profilerPath
326  #for arg in arglist:
327  #print arg
328  os.execv(profilerPath, arglist)
329 
330  # End Profiler Support ------
331 
332  if opts.pickle_output:
333  if opts.output:
334  root_logger.error("Conflicting options: use only --pickle-output or --output")
335  sys.exit(1)
336  else:
337  root_logger.warning("--pickle-output is deprecated, use --output instead")
338  opts.output = opts.pickle_output
339 
340  from Gaudi.Main import gaudimain
341  c = gaudimain()
342 
343  # Prepare the "configuration script" to parse (like this it is easier than
344  # having a list with files and python commands, with an if statements that
345  # decides to do importOptions or exec)
346  options = [ "importOptions(%r)" % f for f in args ]
347  # The option lines are inserted into the list of commands using their
348  # position on the command line
349  optlines = list(opts.options)
350  optlines.reverse() # this allows to avoid to have to care about corrections of the positions
351  for pos, l in optlines:
352  options.insert(pos,l)
353 
354  # prevent the usage of GaudiPython
355  class FakeModule(object):
356  def __init__(self, exception):
357  self.exception = exception
358  def __getattr__(self, *args, **kwargs):
359  raise self.exception
360  sys.modules["GaudiPython"] = FakeModule(RuntimeError("GaudiPython cannot be used in option files"))
361 
362  # "execute" the configuration script generated (if any)
363  if options:
364  g = {}
365  l = {}
366  exec "from Gaudi.Configuration import *" in g, l
367  for o in options:
368  logging.debug(o)
369  exec o in g, l
370 
371  import GaudiKernel.Proxy.Configurable
372  if opts.no_conf_user_apply:
373  logging.info("Disabling automatic apply of ConfigurableUser")
374  # pretend that they have been already applied
375  GaudiKernel.Proxy.Configurable._appliedConfigurableUsers_ = True
376 
377  # This need to be done before dumping
378  if opts.old_conf_user_apply:
379  from GaudiKernel.Proxy.Configurable import applyConfigurableUsers_old as applyConfigurableUsers
380  else:
381  from GaudiKernel.Proxy.Configurable import applyConfigurableUsers
383 
384  # Options to be processed after applyConfigurableUsers
385  if opts.post_options:
386  g = {}
387  l = {}
388  exec "from Gaudi.Configuration import *" in g, l
389  for o in opts.post_options:
390  logging.debug(o)
391  exec o in g, l
392 
393  if opts.verbose:
394  c.printconfig(opts.old_opts, opts.all_opts)
395  if opts.output:
396  c.writeconfig(opts.output, opts.all_opts)
397 
398  c.printsequence = opts.printsequence
399  if opts.printsequence:
400  if opts.ncpus:
401  logging.warning("--printsequence not supported with --ncpus: ignored")
402  elif opts.dry_run:
403  logging.warning("--printsequence not supported with --dry-run: ignored")
404 
405  # re-enable the GaudiPython module
406  del sys.modules["GaudiPython"]
407 
408  if not opts.dry_run:
409  # Do the real processing
410  sys.exit(c.run(opts.ncpus))
def getArgsWithoutoProfilerInfo
Definition: gaudirun.py:6
struct GAUDI_API map
Parametrisation class for map-like implementation.
def option_cb
Definition: gaudirun.py:120
def getArgsFromQmt
Definition: gaudirun.py:46
def rationalizepath
Definition: gaudirun.py:34