Loading [MathJax]/extensions/tex2jax.js
The Gaudi Framework  v31r0 (aeb156f0)
All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Modules Pages
gaudirun.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 import os
4 import sys
5 from tempfile import mkstemp
6 
7 
9  """
10  Remove from the arguments the presence of the profiler and its output in
11  order to relaunch the script w/o infinite loops.
12 
13  >>> getArgsWithoutoProfilerInfo(['--profilerName', 'igprof', 'myopts.py'])
14  ['myopts.py']
15 
16  >>> getArgsWithoutoProfilerInfo(['--profilerName=igprof', 'myopts.py'])
17  ['myopts.py']
18 
19  >>> getArgsWithoutoProfilerInfo(['--profilerName', 'igprof', '--profilerExtraOptions', 'a b c', 'myopts.py'])
20  ['myopts.py']
21 
22  >>> getArgsWithoutoProfilerInfo(['--profilerName', 'igprof', '--options', 'a b c', 'myopts.py'])
23  ['--options', 'a b c', 'myopts.py']
24  """
25  newargs = []
26  args = list(args) # make a temp copy
27  while args:
28  o = args.pop(0)
29  if o.startswith('--profile'):
30  if '=' not in o:
31  args.pop(0)
32  else:
33  newargs.append(o)
34  return newargs
35 
36 
37 def setLibraryPreload(newpreload):
38  ''' Adds a list of libraries to LD_PRELOAD '''
39  preload = os.environ.get("LD_PRELOAD", "")
40  if preload:
41  preload = preload.replace(" ", ":").split(":")
42  else:
43  preload = []
44 
45  for libname in set(preload).intersection(newpreload):
46  logging.warning(
47  "Ignoring preload of library %s because it is "
48  "already in LD_PRELOAD.", libname)
49 
50  to_load = [
51  libname for libname in newpreload if libname not in set(preload)
52  ]
53 
54  if to_load:
55  preload += to_load
56  preload = ":".join(preload)
57  os.environ["LD_PRELOAD"] = preload
58  logging.info("Setting LD_PRELOAD='%s'", preload)
59 
60  return to_load
61 
62 
63 def rationalizepath(path):
64  '''
65  Convert the given path to a real path if the pointed file exists, otherwise
66  just normalize it.
67  '''
68  path = os.path.normpath(os.path.expandvars(path))
69  if os.path.exists(path):
70  path = os.path.realpath(path)
71  return path
72 
73 
74 # variable used to keep alive the temporary option files extracted
75 # from the .qmt
76 _qmt_tmp_opt_files = []
77 
78 
79 def getArgsFromQmt(qmtfile):
80  '''
81  Given a .qmt file, return the command line arguments of the corresponding
82  test.
83  '''
84  from xml.etree import ElementTree as ET
85  global _qmt_tmp_opt_files
86  # parse the .qmt file and extract args and options
87  qmt = ET.parse(qmtfile)
88  args = [a.text for a in qmt.findall("argument[@name='args']//text")]
89  options = qmt.find("argument[@name='options']/text")
90 
91  if options is not None: # options need to be dumped in a temporary file
92  from tempfile import NamedTemporaryFile
93  import re
94  if re.search(
95  r"from\s+Gaudi.Configuration\s+import\s+\*"
96  r"|from\s+Configurables\s+import", options.text):
97  tmp_opts = NamedTemporaryFile(suffix='.py')
98  else:
99  tmp_opts = NamedTemporaryFile(suffix='.opts')
100  tmp_opts.write(options.text)
101  tmp_opts.flush()
102  args.append(tmp_opts.name)
103  _qmt_tmp_opt_files.append(tmp_opts)
104 
105  # relative paths in a .qmt are rooted in the qmtest directory, so
106  # - find where the .qmt lives
107  qmtfile = os.path.abspath(qmtfile)
108  if 'qmtest' in qmtfile.split(os.path.sep):
109  # this return the path up to the 'qmtest' entry in qmtfile
110  testdir = qmtfile
111  while os.path.basename(testdir) != 'qmtest':
112  testdir = os.path.dirname(testdir)
113  else:
114  testdir = '.'
115  # - temporarily switch to that directory and rationalize the paths
116  old_cwd = os.getcwd()
117  os.chdir(testdir)
118  args = map(rationalizepath, args)
119  os.chdir(old_cwd)
120 
121  return args
122 
123 
124 # ---------------------------------------------------------------------
125 if __name__ == "__main__":
126  # ensure that we (and the subprocesses) use the C standard localization
127  if os.environ.get('LC_ALL') != 'C':
128  print '# setting LC_ALL to "C"'
129  os.environ['LC_ALL'] = 'C'
130 
131  from optparse import OptionParser
132  parser = OptionParser(usage="%prog [options] <opts_file> ...")
133  parser.add_option(
134  "-n",
135  "--dry-run",
136  action="store_true",
137  help="do not run the application, just parse option files")
138  parser.add_option(
139  "-p",
140  "--pickle-output",
141  action="store",
142  type="string",
143  metavar="FILE",
144  help="DEPRECATED: use '--output file.pkl' instead. Write "
145  "the parsed options as a pickle file (static option "
146  "file)")
147  parser.add_option(
148  "-v",
149  "--verbose",
150  action="store_true",
151  help="print the parsed options")
152  parser.add_option(
153  "--old-opts",
154  action="store_true",
155  help="format printed options in old option files style")
156  parser.add_option(
157  "--all-opts",
158  action="store_true",
159  help="print all the option (even if equal to default)")
160  # GaudiPython Parallel Mode Option
161  # Argument must be an integer in range [ -1, sys_cpus ]
162  # -1 : All available cpus
163  # 0 : Serial Mode (traditional gaudirun)
164  # n>0 : parallel with n cpus (n <= sys_cpus)
165  parser.add_option(
166  "--ncpus",
167  action="store",
168  type="int",
169  default=0,
170  help="start the application in parallel mode using NCPUS processes. "
171  "0 => serial mode (default), -1 => use all CPUs")
172 
173  def option_cb(option, opt, value, parser):
174  """Add the option line to a list together with its position in the
175  argument list.
176  """
177  parser.values.options.append((len(parser.largs), value))
178 
179  parser.add_option(
180  "--option",
181  action="callback",
182  callback=option_cb,
183  type="string",
184  nargs=1,
185  help="add a single line (Python) option to the configuration. "
186  "All options lines are executed, one after the other, in "
187  "the same context.")
188  parser.add_option(
189  "--no-conf-user-apply",
190  action="store_true",
191  help="disable the automatic application of configurable "
192  "users (for backward compatibility)")
193  parser.add_option(
194  "--old-conf-user-apply",
195  action="store_true",
196  help="use the old logic when applying ConfigurableUsers "
197  "(with bug #103803) [default]")
198  parser.add_option(
199  "--new-conf-user-apply",
200  action="store_false",
201  dest="old_conf_user_apply",
202  help="use the new (correct) logic when applying "
203  "ConfigurableUsers (fixed bug #103803), can be "
204  "turned on also with the environment variable "
205  "GAUDI_FIXED_APPLY_CONF")
206  parser.add_option(
207  "-o",
208  "--output",
209  action="store",
210  type="string",
211  help="dump the configuration to a file. The format of "
212  "the options is determined by the extension of the "
213  "file name: .pkl = pickle, .py = python, .opts = "
214  "old style options. The python format cannot be "
215  "used to run the application and it contains the "
216  "same dictionary printed with -v")
217  parser.add_option(
218  "--post-option",
219  action="append",
220  type="string",
221  dest="post_options",
222  help="Python options to be executed after the ConfigurableUser "
223  "are applied. "
224  "All options lines are executed, one after the other, in "
225  "the same context.")
226  parser.add_option(
227  "--debug", action="store_true", help="enable some debug print-out")
228  parser.add_option("--gdb", action="store_true", help="attach gdb")
229  parser.add_option(
230  "--printsequence", action="store_true", help="print the sequence")
231  if not sys.platform.startswith("win"):
232  # These options can be used only on unix platforms
233  parser.add_option(
234  "-T",
235  "--tcmalloc",
236  action="store_true",
237  help="Use the Google malloc replacement. The environment "
238  "variable TCMALLOCLIB can be used to specify a different "
239  "name for the library (the default is libtcmalloc.so)")
240  parser.add_option(
241  "--preload",
242  action="append",
243  help="Allow pre-loading of special libraries (e.g. Google "
244  "profiling libraries).")
245  # Option to use a profiler
246  parser.add_option(
247  "--profilerName",
248  type="string",
249  help=
250  "Select one profiler among: igprofPerf, igprofMem and valgrind<toolname>"
251  )
252 
253  # Option to specify the filename where to collect the profiler's output
254  parser.add_option(
255  "--profilerOutput",
256  type="string",
257  help="Specify the name of the output file for the profiler output")
258 
259  # Option to specify the filename where to collect the profiler's output
260  parser.add_option(
261  "--profilerExtraOptions",
262  type="string",
263  help=
264  "Specify additional options for the profiler. The '--' string should be expressed as '__' (--my-opt becomes __my-opt)"
265  )
266 
267  parser.add_option(
268  '--use-temp-opts',
269  action='store_true',
270  help='when this option is enabled, the options are parsed'
271  ' and stored in a temporary file, then the job is '
272  'restarted using that file as input (to save '
273  'memory)')
274  parser.add_option(
275  "--run-info-file",
276  type="string",
277  help=
278  "Save gaudi process information to the file specified (in JSON format)"
279  )
280 
281  parser.set_defaults(
282  options=[],
283  tcmalloc=False,
284  profilerName='',
285  profilerOutput='',
286  profilerExtraOptions='',
287  preload=[],
288  ncpus=None,
289  # the old logic can be turned off with an env variable
290  old_conf_user_apply='GAUDI_FIXED_APPLY_CONF' not in os.environ,
291  run_info_file=None)
292 
293  # replace .qmt files in the command line with their contained args
294  argv = []
295  for a in sys.argv[1:]:
296  if a.endswith('.qmt') and os.path.exists(a):
297  argv.extend(getArgsFromQmt(a))
298  else:
299  argv.append(a)
300  if argv != sys.argv[1:]:
301  print '# Running', sys.argv[0], 'with arguments', argv
302 
303  opts, args = parser.parse_args(args=argv)
304 
305  # Check consistency of options
306 
307  # Parallel Option ---------------------------------------------------------
308  if opts.ncpus:
309  from multiprocessing import cpu_count
310  sys_cpus = cpu_count()
311  if opts.ncpus > sys_cpus:
312  s = "Invalid value : --ncpus : only %i cpus available" % sys_cpus
313  parser.error(s)
314  elif opts.ncpus < -1:
315  s = "Invalid value : --ncpus must be integer >= -1"
316  parser.error(s)
317  else:
318  # FIXME: is it really needed to set it to None if it is 0 or False?
319  opts.ncpus = None
320 
321  # configure the logging
322  import logging
323  from GaudiKernel.ProcessJobOptions import (InstallRootLoggingHandler,
324  PrintOff)
325 
326  if opts.old_opts:
327  prefix = "// "
328  else:
329  prefix = "# "
330  level = logging.INFO
331  if opts.debug:
332  level = logging.DEBUG
333  InstallRootLoggingHandler(prefix, level=level, with_time=opts.debug)
334  root_logger = logging.getLogger()
335 
336  # Sanitizer support
337  sanitizer = os.environ.get("PRELOAD_SANITIZER_LIB", "")
338  if sanitizer:
339  if sanitizer not in os.environ.get("LD_PRELOAD", ""):
340  opts.preload.insert(0, sanitizer)
341  os.environ["PRELOAD_SANITIZER_LIB"] = ""
342 
343  # tcmalloc support
344  if opts.tcmalloc:
345  # Disable tcmalloc if sanitizer is selected
346  if sanitizer:
347  logging.warning("tcmalloc preload disabled when using a sanitizer")
348  else:
349  opts.preload.insert(
350  0, os.environ.get("TCMALLOCLIB", "libtcmalloc.so"))
351 
352  # allow preloading of libraries
353  if opts.preload:
354  preload = os.environ.get("LD_PRELOAD", "")
355  if preload:
356  preload = preload.replace(" ", ":").split(":")
357  else:
358  preload = []
359  for libname in set(preload).intersection(opts.preload):
360  logging.warning(
361  "Ignoring preload of library %s because it is "
362  "already in LD_PRELOAD.", libname)
363  to_load = [
364  libname for libname in opts.preload if libname not in set(preload)
365  ]
366  if to_load:
367  preload += to_load
368  preload = ":".join(preload)
369  os.environ["LD_PRELOAD"] = preload
370  logging.info("Restarting with LD_PRELOAD='%s'", preload)
371  # remove the --tcmalloc option from the arguments
372  # FIXME: the --preload arguments will issue a warning but it's tricky to remove them
373  args = [
374  a for a in sys.argv
375  if a != '-T' and not '--tcmalloc'.startswith(a)
376  ]
377  os.execv(sys.executable, [sys.executable] + args)
378 
379  # Profiler Support ------
380  if opts.profilerName:
381  profilerName = opts.profilerName
382  profilerExecName = ""
383  profilerOutput = opts.profilerOutput or (profilerName + ".output")
384 
385  # To restart the application removing the igprof option and prepending the string
386  args = getArgsWithoutoProfilerInfo(sys.argv)
387 
388  igprofPerfOptions = "-d -pp -z -o igprof.pp.gz".split()
389 
390  profilerOptions = ""
391  if profilerName == "igprof":
392  if not opts.profilerOutput:
393  profilerOutput += ".profile.gz"
394  profilerOptions = "-d -z -o %s" % profilerOutput
395  profilerExecName = "igprof"
396 
397  elif profilerName == "igprofPerf":
398  if not opts.profilerOutput:
399  profilerOutput += ".pp.gz"
400  profilerOptions = "-d -pp -z -o %s" % profilerOutput
401  profilerExecName = "igprof"
402 
403  elif profilerName == "igprofMem":
404  if not opts.profilerOutput:
405  profilerOutput += ".mp.gz"
406  profilerOptions = "-d -mp -z -o %s" % profilerOutput
407  profilerExecName = "igprof"
408 
409  elif "valgrind" in profilerName:
410  # extract the tool
411  if not opts.profilerOutput:
412  profilerOutput += ".log"
413  toolname = profilerName.replace('valgrind', '')
414  outoption = "--log-file"
415  if toolname in ("massif", "callgrind", "cachegrind"):
416  outoption = "--%s-out-file" % toolname
417  profilerOptions = "--tool=%s %s=%s" % (toolname, outoption,
418  profilerOutput)
419  profilerExecName = "valgrind"
420 
421  elif profilerName == "jemalloc":
422  opts.preload.insert(
423  0, os.environ.get("JEMALLOCLIB", "libjemalloc.so"))
424  os.environ['MALLOC_CONF'] = "prof:true,prof_leak:true"
425  else:
426  root_logger.warning("Profiler %s not recognized!" % profilerName)
427 
428  # Add potential extra options
429  if opts.profilerExtraOptions != "":
430  profilerExtraOptions = opts.profilerExtraOptions
431  profilerExtraOptions = profilerExtraOptions.replace("__", "--")
432  profilerOptions += " %s" % profilerExtraOptions
433 
434  # now we look for the full path of the profiler: is it really there?
435  if profilerExecName:
436  import distutils.spawn
437  profilerPath = distutils.spawn.find_executable(profilerExecName)
438  if not profilerPath:
439  root_logger.error(
440  "Cannot locate profiler %s" % profilerExecName)
441  sys.exit(1)
442 
443  root_logger.info("------ Profiling options are on ------ \n"
444  " o Profiler: %s\n"
445  " o Options: '%s'.\n"
446  " o Output: %s" % (profilerExecName or profilerName,
447  profilerOptions, profilerOutput))
448 
449  # allow preloading of libraries
450  # That code need to be acsracted from above
451  to_reload = []
452  if opts.preload:
453  to_reload = setLibraryPreload(opts.preload)
454 
455  if profilerExecName:
456  # We profile python
457  profilerOptions += " python"
458 
459  # now we have all the ingredients to prepare our command
460  arglist = [profilerPath] + profilerOptions.split() + args
461  arglist = [a for a in arglist if a != '']
462  # print profilerPath
463  # for arg in arglist:
464  # print arg
465  os.execv(profilerPath, arglist)
466  else:
467  arglist = [a for a in sys.argv if not a.startswith("--profiler")]
468  os.execv(sys.executable, [sys.executable] + arglist)
469 
470  # End Profiler Support ------
471 
472  if opts.pickle_output:
473  if opts.output:
474  root_logger.error(
475  "Conflicting options: use only --pickle-output or --output")
476  sys.exit(1)
477  else:
478  root_logger.warning(
479  "--pickle-output is deprecated, use --output instead")
480  opts.output = opts.pickle_output
481 
482  from Gaudi.Main import gaudimain
483  c = gaudimain()
484 
485  # Prepare the "configuration script" to parse (like this it is easier than
486  # having a list with files and python commands, with an if statements that
487  # decides to do importOptions or exec)
488  options = ["importOptions(%r)" % f for f in args]
489  # The option lines are inserted into the list of commands using their
490  # position on the command line
491  optlines = list(opts.options)
492  # this allows to avoid to have to care about corrections of the positions
493  optlines.reverse()
494  for pos, l in optlines:
495  options.insert(pos, l)
496 
497  # prevent the usage of GaudiPython
498  class FakeModule(object):
499  def __init__(self, exception):
500  self.exception = exception
501 
502  def __getattr__(self, *args, **kwargs):
503  raise self.exception
504 
505  sys.modules["GaudiPython"] = FakeModule(
506  RuntimeError("GaudiPython cannot be used in option files"))
507 
508  # when the special env GAUDI_TEMP_OPTS_FILE is set, it overrides any
509  # option(file) on the command line
510  if 'GAUDI_TEMP_OPTS_FILE' in os.environ:
511  options = ['importOptions(%r)' % os.environ['GAUDI_TEMP_OPTS_FILE']]
512  PrintOff(100)
513 
514  # "execute" the configuration script generated (if any)
515  if options:
516  g = {}
517  l = {}
518  exec "from Gaudi.Configuration import *" in g, l
519  for o in options:
520  logging.debug(o)
521  exec o in g, l
522 
523  import GaudiKernel.Proxy.Configurable
524  if opts.no_conf_user_apply:
525  logging.info("Disabling automatic apply of ConfigurableUser")
526  # pretend that they have been already applied
527  GaudiKernel.Proxy.Configurable._appliedConfigurableUsers_ = True
528 
529  # This need to be done before dumping
530  if opts.old_conf_user_apply:
531  from GaudiKernel.Proxy.Configurable import applyConfigurableUsers_old as applyConfigurableUsers
532  else:
533  from GaudiKernel.Proxy.Configurable import applyConfigurableUsers
535 
536  # Options to be processed after applyConfigurableUsers
537  if opts.post_options:
538  g = {}
539  l = {}
540  exec "from Gaudi.Configuration import *" in g, l
541  for o in opts.post_options:
542  logging.debug(o)
543  exec o in g, l
544 
545  if 'GAUDI_TEMP_OPTS_FILE' in os.environ:
546  os.remove(os.environ['GAUDI_TEMP_OPTS_FILE'])
547  opts.use_temp_opts = False
548 
549  if opts.verbose and not opts.use_temp_opts:
550  c.printconfig(opts.old_opts, opts.all_opts)
551  if opts.output:
552  c.writeconfig(opts.output, opts.all_opts)
553 
554  if opts.use_temp_opts:
555  fd, tmpfile = mkstemp('.opts')
556  os.close(fd)
557  c.writeconfig(tmpfile, opts.all_opts)
558  os.environ['GAUDI_TEMP_OPTS_FILE'] = tmpfile
559  logging.info('Restarting from pre-parsed options')
560  os.execv(sys.executable, [sys.executable] + sys.argv)
561 
562  c.printsequence = opts.printsequence
563  if opts.printsequence:
564  if opts.ncpus:
565  logging.warning(
566  "--printsequence not supported with --ncpus: ignored")
567  elif opts.dry_run:
568  logging.warning(
569  "--printsequence not supported with --dry-run: ignored")
570 
571  # re-enable the GaudiPython module
572  del sys.modules["GaudiPython"]
573 
574  if not opts.dry_run:
575  # Do the real processing
576  retcode = c.run(opts.gdb, opts.ncpus)
577 
578  # Now saving the run information pid, retcode and executable path to
579  # a file is requested
580  if opts.run_info_file:
581  import os
582  import json
583  run_info = {}
584  run_info["pid"] = os.getpid()
585  run_info["retcode"] = retcode
586  if os.path.exists('/proc/self/exe'):
587  # These options can be used only on unix platforms
588  run_info["exe"] = os.readlink('/proc/self/exe')
589 
590  logging.info("Saving run info to: %s" % opts.run_info_file)
591  with open(opts.run_info_file, "w") as f:
592  json.dump(run_info, f)
593 
594  sys.exit(retcode)
def __init__(self, exception)
Definition: gaudirun.py:499
def option_cb(option, opt, value, parser)
Definition: gaudirun.py:173
def rationalizepath(path)
Definition: gaudirun.py:63
def getArgsFromQmt(qmtfile)
Definition: gaudirun.py:79
def InstallRootLoggingHandler(prefix=None, level=None, stream=None, with_time=False)
struct GAUDI_API map
Parametrisation class for map-like implementation.
def getArgsWithoutoProfilerInfo(args)
Definition: gaudirun.py:8
def __getattr__(self, args, kwargs)
Definition: gaudirun.py:502
def setLibraryPreload(newpreload)
Definition: gaudirun.py:37