11 from __future__
import print_function
20 from Gaudi
import Configuration
22 log = logging.getLogger(__name__)
33 __nonzero__ = __bool__
44 class Property(object):
50 return bytes(self.
value).decode(
"utf-8")
62 self.
lib.py_bootstrap_fsm_configure(self.
ptr)
67 self.
lib.py_bootstrap_fsm_initialize(self.
ptr)
83 self.
lib.py_bootstrap_fsm_finalize(self.
ptr)
88 self.
lib.py_bootstrap_fsm_terminate(self.
ptr)
92 return self.
lib.py_bootstrap_getService(self.
ptr, name.encode(
"ascii"))
96 self.
lib.py_bootstrap_setProperty(
97 self.
ptr, name.encode(
"ascii"), value.encode(
"ascii")
103 self.
lib.py_bootstrap_getProperty(self.
ptr, name.encode(
"ascii"))
110 self.
lib.py_bootstrap_setOption(
111 self.
ptr, key.encode(
"ascii"), value.encode(
"ascii")
115 from ctypes
import RTLD_GLOBAL, PyDLL, c_bool, c_char_p, c_int, c_void_p, util
120 class IInterface_p(c_void_p):
122 return "IInterface_p(0x%x)" % (0
if self.value
is None else self.value)
124 self.
log = logging.getLogger(
"BootstrapHelper")
125 libname = util.find_library(
"GaudiKernel")
or "libGaudiKernel.so"
126 self.
log.debug(
"loading GaudiKernel (%s)", libname)
130 self.
lib = gkl = PyDLL(libname, mode=RTLD_GLOBAL)
133 (
"createApplicationMgr", IInterface_p, []),
134 (
"getService", IInterface_p, [IInterface_p, c_char_p]),
135 (
"setProperty", c_bool, [IInterface_p, c_char_p, c_char_p]),
136 (
"getProperty", c_char_p, [IInterface_p, c_char_p]),
137 (
"setOption",
None, [IInterface_p, c_char_p, c_char_p]),
138 (
"ROOT_VERSION_CODE", c_int, []),
141 for name, restype, argtypes
in functions:
142 f = getattr(gkl,
"py_bootstrap_%s" % name)
143 f.restype, f.argtypes = restype, argtypes
146 if name
not in self.__class__.__dict__:
147 setattr(self, name, f)
157 f = getattr(gkl,
"py_bootstrap_fsm_%s" % name)
158 f.restype, f.argtypes = c_bool, [IInterface_p]
159 gkl.py_bootstrap_app_run.restype = c_bool
160 gkl.py_bootstrap_app_run.argtypes = [IInterface_p, c_int]
162 gkl.py_helper_printAlgsSequences.restype =
None
163 gkl.py_helper_printAlgsSequences.argtypes = [IInterface_p]
166 ptr = self.
lib.py_bootstrap_createApplicationMgr()
171 return self.
lib.py_bootstrap_ROOT_VERSION_CODE()
176 a = root_version_code >> 16 & 0xFF
177 b = root_version_code >> 8 & 0xFF
178 c = root_version_code & 0xFF
184 Return all options from the old configuration system as a dictionary.
186 If explicit_defaults is true, include default values of unset properties in the dictionary.
188 from itertools
import chain
190 from GaudiKernel.Proxy.Configurable
import Configurable, getNeededConfigurables
200 while count != new_count:
203 new_count = len(needed_conf)
204 for n
in needed_conf:
205 c = Configurable.allConfigurables[n]
206 if hasattr(c,
"getValuedProperties"):
207 c.getValuedProperties()
209 for n
in needed_conf:
210 c = Configurable.allConfigurables[n]
212 chain(c.getDefaultProperties().
items(), c.getValuedProperties().
items())
214 else c.getValuedProperties().
items()
219 hasattr(Configurable,
"PropertyReference")
220 and type(v) == Configurable.PropertyReference
225 if isinstance(v, str):
227 v =
'"%s"' % v.replace(
'"',
'\\"')
228 elif sys.version_info < (3,)
and isinstance(v, long):
230 elif hasattr(v,
"__opt_value__"):
231 v = v.__opt_value__()
232 old_opts[
".".join((n, p))] = str(v)
239 Return all options from the old and new configuration system as a dictionary.
241 If explicit_defaults is true, include default values of unset properties in the dictionary.
248 opts = GaudiConfig2.all_options(
False)
250 conflicts = [n
for n
in set(opts).intersection(old_opts)
if opts[n] != old_opts[n]]
253 log.error(
"Some properties are set in old and new style configuration")
254 log.warning(
"name: old -> new")
256 log.warning(
"%s: %s -> %s", n, old_opts[n], opts[n])
259 opts.update(old_opts)
261 if explicit_defaults:
265 all_opts.update(GaudiConfig2.all_options(
True))
269 all_opts.update(opts)
277 Helper to convert values to old .opts format.
279 >>> print(toOpt('some "text"'))
281 >>> print(toOpt('first\\nsecond'))
284 >>> print(toOpt({'a': [1, 2, '3']}))
287 if isinstance(value, six.string_types):
288 return '"{0}"'.
format(value.replace(
'"',
'\\"'))
289 elif isinstance(value, dict):
293 elif hasattr(value,
"__iter__"):
294 return "[{0}]".
format(
", ".join(
map(toOpt, value)))
301 Helper to parse option strings to Python values.
303 Ideally it should just be "eval", but the string parser of Gaudi
304 is different from the Python one, so we get string options that
305 cannot be just evaluated.
307 >>> print(parseOpt('123'))
309 >>> print(parseOpt('"some\\n\\\\"text\\\\""'))
312 >>> print(parseOpt(''))
319 quoted_string = re.compile(
r'^"(.*)"$', re.DOTALL)
324 m = quoted_string.match(s)
326 return m.group(1).replace(
'\\"',
'"')
332 from Configurables
import ApplicationMgr
335 if os.environ.get(
"GAUDIAPPNAME"):
336 appMgr.AppName = str(os.environ[
"GAUDIAPPNAME"])
337 if os.environ.get(
"GAUDIAPPVERSION"):
338 appMgr.AppVersion = str(os.environ[
"GAUDIAPPVERSION"])
339 self.
log = logging.getLogger(__name__)
348 import multiprocessing
351 from time
import ctime
354 datetime = datetime.replace(
" ",
"_")
355 outfile = open(
"gaudirun-%s.log" % (datetime),
"w")
357 streamhandler = logging.StreamHandler(stream=outfile)
358 console = logging.StreamHandler()
360 formatter = logging.Formatter(
361 "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
364 streamhandler.setFormatter(formatter)
365 console.setFormatter(formatter)
369 self.
log = multiprocessing.log_to_stderr()
370 self.
log.setLevel(logging.INFO)
371 self.
log.name =
"Gaudi/Main.py Logger"
372 self.
log.handlers = []
374 self.
log.addHandler(streamhandler)
375 self.
log.addHandler(console)
376 self.
log.removeHandler(console)
378 self.
log.setLevel = logging.INFO
382 from collections
import defaultdict
383 from pprint
import pformat
385 optDict = defaultdict(dict)
388 c, p = key.rsplit(
".", 1)
389 optDict[c][p] =
parseOpt(allOpts[key])
390 formatted = pformat(dict(optDict))
398 return re.sub(
r'"\n +"',
"", formatted, flags=re.MULTILINE)
411 output = open(filename,
"wb")
413 from GaudiKernel.Proxy.Configurable
import getNeededConfigurables
417 to_dump[n] = Configuration.allConfigurables[n]
418 pickle.dump(to_dump, output, -1)
422 msg =
"Dumping all configurables and properties"
424 msg +=
" (different from default)"
436 ".pkl":
lambda filename, all: self.
_writepickle(filename),
437 ".py":
lambda filename, all: open(filename,
"w").write(
440 ".opts":
lambda filename, all: open(filename,
"w").write(
443 ".json":
lambda filename, all: json.dump(
444 getAllOpts(all), open(filename,
"w"), indent=2, sort_keys=
True
450 write[
".yaml"] =
lambda filename, all: yaml.safe_dump(
453 write[
".yml"] = write[
".yaml"]
457 from os.path
import splitext
459 ext = splitext(filename)[1]
461 write[ext](filename, all)
464 "Unknown file type '%s'. Must be any of %r.", ext, sorted(write.keys())
470 def run(self, attach_debugger, ncpus=None):
482 self.
log.info(
"attaching debugger to PID " + str(os.getpid()))
484 os.P_NOWAIT, debugger, [debugger,
"-q",
"python", str(os.getpid())]
493 os.waitpid(pid, os.WNOHANG)
499 from GaudiKernel.Proxy.Configurable
import expandvars
505 from GaudiKernel.Proxy.Configurable
import Configurable
507 self.
log.debug(
"runSerial: apply options")
509 conf_dict[
"ApplicationMgr.JobOptionsType"] =
'"NONE"'
512 conf_dict[
"ApplicationMgr.PrintAlgsSequence"] =
"true"
514 if hasattr(Configurable,
"_configurationLocked"):
515 Configurable._configurationLocked =
True
520 self.
log.debug(
"-" * 80)
521 self.
log.debug(
"%s: running in serial mode", __name__)
522 self.
log.debug(
"-" * 80)
530 sysTime =
time() - sysStart
531 self.
log.debug(
"-" * 80)
533 "%s: serial system finished, time taken: %5.4fs", __name__, sysTime
535 self.
log.debug(
"-" * 80)
544 c = Configurable.allConfigurables
545 self.
log.info(
"-" * 80)
546 self.
log.info(
"%s: Parallel Mode : %i ", __name__, ncpus)
548 (
"platform",
" ".join(os.uname())),
549 (
"config", os.environ.get(
"BINARY_TAG")
or os.environ.get(
"CMTCONFIG")),
550 (
"app. name", os.environ.get(
"GAUDIAPPNAME")),
551 (
"app. version", os.environ.get(
"GAUDIAPPVERSION")),
553 self.
log.info(
"%s: %30s : %s ", __name__, name, value
or "Undefined")
555 events = str(c[
"ApplicationMgr"].EvtMax)
557 events =
"Undetermined"
558 self.
log.info(
"%s: Events Specified : %s ", __name__, events)
559 self.
log.info(
"-" * 80)
561 Parall = gpp.Coord(ncpus, c, self.
log)
564 self.
log.info(
"MAIN.PY : received %s from Coordinator" % (sc))
567 sysTime =
time() - sysStart
568 self.
log.name =
"Gaudi/Main.py Logger"
569 self.
log.info(
"-" * 80)
571 "%s: parallel system finished, time taken: %5.4fs", __name__, sysTime
573 self.
log.info(
"-" * 80)