The Gaudi Framework  master (42b00024)
All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Modules Pages
genconfuser.py
Go to the documentation of this file.
1 #!/usr/bin/env python3
2 
12 """
13 Generate _confDb.py files for ConfigurableUser classes.
14 """
15 
16 import logging
17 import os
18 import sys
19 import time
20 from glob import glob
21 from pprint import pformat
22 
24 from GaudiKernel.ConfigurableDb import cfgDb
25 
26 logging.VERBOSE = (logging.INFO + logging.DEBUG) // 2
27 logging.addLevelName(logging.VERBOSE, "VERBOSE")
28 logging.verbose = lambda msg, *args, **kwargs: logging.log(
29  logging.VERBOSE, msg, *args, **kwargs
30 )
31 
32 
33 def _inheritsfrom(derived, basenames):
34  """
35  Check if any of the class names in 'basenames' is anywhere in the base
36  classes of the class 'derived'.
37  If 'derived' _is_ one in 'basenames', returns False.
38 
39  'basenames' can be a string or an iterable (of strings).
40  """
41  if isinstance(basenames, str):
42  basenames = (basenames,)
43  for b in derived.__bases__:
44  if b.__name__ in basenames:
45  return True
46  else:
47  if _inheritsfrom(b, basenames):
48  return True
49  return False
50 
51 
52 def loadConfigurableDb(build_dir=None, project_name=None):
53  """
54  Equivalent to GaudiKernel.ConfigurableDb.loadConfigurableDb(), but does a
55  deep search and executes the '*.confdb' files instead of importing them.
56  """
57  log = GaudiKernel.ConfigurableDb.log
58  from os.path import join as path_join
59 
60  # look for the confdb files in all the reasonable places
61  # - CMake builds
62  confDbFiles = []
63  for path in sys.path:
64  confDbFiles += [
65  f for f in glob(path_join(path, "*", "*.confdb")) if os.path.isfile(f)
66  ]
67  # - new-style CMake builds (look for all .confdb in the build tree)
68  if build_dir:
69  for root, _, files in os.walk(build_dir):
70  confDbFiles += [
71  os.path.join(root, f) for f in files if f.endswith(".confdb")
72  ]
73  # - used projects and local merged file
74  pathlist = os.getenv("GAUDI_PLUGIN_PATH", "").split(os.pathsep) + os.getenv(
75  "LD_LIBRARY_PATH", ""
76  ).split(os.pathsep)
77  for path in filter(os.path.isdir, pathlist):
78  confDbFiles += [
79  f
80  for f in [
81  path_join(path, f) for f in os.listdir(path) if f.endswith(".confdb")
82  ]
83  ]
84  # - get the list of ignored files
85  ignored_files = set(os.environ.get("CONFIGURABLE_DB_IGNORE", "").split(","))
86  # - load the confdb files
87  for confDb in set(confDbFiles):
88  if confDb in ignored_files or (
89  project_name
90  and os.path.basename(confDb) == ("{}.confdb".format(project_name))
91  ):
92  # skip ignored files and the project's own confdb
93  log.debug("\t-ignoring [%s]", confDb)
94  # make sure we count also <project_name>.confdb for GaudiKernel
95  ignored_files.add(confDb)
96  continue
97  log.debug("\t-loading [%s]...", confDb)
98  try:
99  cfgDb._loadModule(confDb)
100  except Exception as err:
101  # It may happen that the file is found but not completely
102  # written, usually during parallel builds, but we do not care.
103  log.warning("Could not load file [%s] !", confDb)
104  log.warning("Reason: %s", err)
105  # top up with the regular merged confDb (for the used projects)
106  # (make sure GaudiKernel gets the right exclusions)
107  os.environ["CONFIGURABLE_DB_IGNORE"] = ",".join(ignored_files)
109 
110 
111 def getConfigurableUsers(modulename, root, mayNotExist=False):
112  """
113  Find in the module 'modulename' all the classes that derive from ConfigurableUser.
114  Return the list of the names.
115  The flag mayNotExist is used to choose the level of the logging message in case
116  the requested module does not exist.
117  """
118  # remember the old system path
119  oldpath = list(sys.path)
120  # we need to hack the sys.path to add the first part of the module name after root
121  moduleelements = modulename.split(".")
122  if len(moduleelements) > 1:
123  moddir = os.sep.join([root] + moduleelements[:-1])
124  else:
125  moddir = root
126  # this is the name of the submodule to import
127  shortmodname = moduleelements[-1]
128  # check if the module file actually exists
129  if not os.path.exists(os.path.join(moddir, shortmodname) + ".py"):
130  msg = "Module %s does not exist" % modulename
131  if mayNotExist:
132  logging.verbose(msg)
133  else:
134  logging.error(msg)
135  # no file -> do not try to import
136  return []
137  # prepend moddir to the path
138  sys.path.insert(0, moddir)
139  logging.verbose("sys.path prepended with %r", sys.path[0])
140 
141  logging.info("Looking for ConfigurableUser in %r", modulename)
142  g, l = {}, {}
143  try:
144  logging.verbose("importing %s", shortmodname)
145  exec("import %s as mod" % shortmodname, g, l)
146  finally:
147  # restore old sys.path
148  logging.verbose("restoring old sys.path")
149  sys.path = oldpath
150  mod = l["mod"]
151  if "__all__" in dir(mod) and mod.__all__:
152  all = mod.__all__
153  else:
154  all = [n for n in dir(mod) if not n.startswith("_")]
155  result = []
156  for name in all:
157  cfg = cfgDb.get(name)
158  if cfg and cfg["module"] != modulename:
159  # This name comes from another module
160  logging.verbose("Object %r already found in module %r", name, cfg["module"])
161  continue
162  t = getattr(mod, name)
163  if isinstance(t, type) and _inheritsfrom(
164  t, ("ConfigurableUser", "SuperAlgorithm")
165  ):
166  result.append(name)
167  logging.verbose("Found %r", result)
168  return result
169 
170 
171 def main():
172  from optparse import OptionParser
173 
174  parser = OptionParser(
175  prog=os.path.basename(sys.argv[0]),
176  usage="%prog [options] <PackageName> [<Module1> ...]",
177  )
178  parser.add_option(
179  "-o",
180  "--output",
181  action="store",
182  type="string",
183  help="output file for confDb data [default = '../genConfDir/<PackageName>_user_confDb.py'].",
184  )
185  parser.add_option(
186  "-r",
187  "--root",
188  action="store",
189  type="string",
190  help="root directory of the python modules [default = '../python'].",
191  )
192  parser.add_option(
193  "-v", "--verbose", action="store_true", help="print some debugging information"
194  )
195  parser.add_option(
196  "--debug", action="store_true", help="print more debugging information"
197  )
198  parser.add_option(
199  "--build-dir",
200  action="store",
201  help="build directory where to look for .confdb files (search all subdirectories)",
202  )
203  parser.add_option(
204  "--project-name",
205  action="store",
206  help="name of the current project (used to exclude spurious versions of the .confdb file for the current project)",
207  )
208  parser.set_defaults(root=os.path.join("..", "python"))
209 
210  opts, args = parser.parse_args()
211 
212  if opts.debug:
213  log_level = logging.DEBUG
214  elif opts.verbose:
215  log_level = logging.VERBOSE
216  else:
217  log_level = logging.INFO if os.environ.get("VERBOSE") else logging.WARNING
218  logging.basicConfig(
219  format="%(levelname)s: %(message)s", stream=sys.stdout, level=log_level
220  )
221 
222  if len(args) < 1:
223  parser.error("PackageName is required")
224 
225  package_name = args.pop(0)
226 
227  usingConvention = False
228  if not args:
229  # use the conventional module name <package>.Configuration
230  args = [package_name + ".Configuration"]
231  usingConvention = True
232 
233  genConfDir = os.path.join("..", os.environ.get("CMTCONFIG", ""), "genConfDir")
234  if not os.path.exists(genConfDir):
235  genConfDir = os.path.join("..", "genConfDir")
236 
237  if not opts.output:
238  outputfile = os.path.join(genConfDir, package_name + "_user.confdb")
239  else:
240  outputfile = opts.output
241 
242  # We can disable the error on missing configurables only if we can import Gaudi.Configurables
243  # It must be done at this point because it may conflict with logging.basicConfig
244  try:
245  import Gaudi.Configurables
246 
247  Gaudi.Configurables.ignoreMissingConfigurables = True
248  except ImportError:
249  pass
250  # load configurables database to avoid fake duplicates
251  loadConfigurableDb(opts.build_dir, opts.project_name)
252  # ensure that local configurables are in the database
253  try:
254  # Add the local python directories to the python path to be able to import the local
255  # configurables
256  sys.path.insert(0, genConfDir)
257  sys.path.insert(0, os.path.join("..", "python"))
258  localConfDb = os.path.join(genConfDir, package_name, package_name + ".confdb")
259  if os.path.exists(localConfDb):
260  cfgDb._loadModule(localConfDb)
261  # Extend the search path of the package module to find the configurables
262  package_module = __import__(package_name)
263  package_module.__path__.insert(0, os.path.join(genConfDir, package_name))
264  except Exception:
265  pass # ignore failures (not important)
266 
267  # Collecting ConfigurableUser specializations
268  cus = {}
269  for mod in args:
270  lst = None
271  try:
272  lst = getConfigurableUsers(mod, root=opts.root, mayNotExist=usingConvention)
273  except ImportError:
274  import traceback
275 
276  logging.error(
277  "Cannot import module %r:\n%s", mod, traceback.format_exc().rstrip()
278  ) # I remove the trailing '\n'
279  return 2
280  if lst:
281  cus[mod] = lst
282  # Add the configurables to the database as fake entries to avoid duplicates
283  for m in lst:
284  cfgDb.add(configurable=m, package="None", module="None", lib="None")
285  elif not usingConvention:
286  logging.warning(
287  "Specified module %r does not contain ConfigurableUser specializations",
288  mod,
289  )
290 
291  if cus:
292  logging.info("ConfigurableUser found:\n%s", pformat(cus))
293  # header
294  output = """## -*- ascii -*-
295 # db file automatically generated by %s on: %s
296 """ % (
297  parser.prog,
298  time.asctime(),
299  )
300 
301  for mod in cus:
302  for cu in cus[mod]:
303  output += "%s %s %s\n" % (mod, "None", cu)
304 
305  # trailer
306  output += "## %s\n" % package_name
307  elif usingConvention:
308  logging.info("No ConfigurableUser found")
309  output = (
310  "# db file automatically generated by %s on: %s\n"
311  "# No ConfigurableUser specialization in %s\n"
312  ) % (parser.prog, time.asctime(), package_name)
313  else:
314  logging.error("No ConfigurableUser specialization found")
315  return 1
316 
317  # create the destination directory if not there
318  output_dir = os.path.dirname(outputfile)
319  try:
320  logging.info("Creating directory %r", output_dir)
321  os.makedirs(output_dir, 0o755)
322  except OSError as err:
323  import errno
324 
325  if err.errno == errno.EEXIST:
326  # somebody already - perhaps concurrently - created that dir.
327  pass
328  else:
329  raise
330 
331  # write output to file
332  logging.verbose("Writing confDb data to %r", outputfile)
333  open(outputfile, "w").write(output)
334  return 0
335 
336 
337 if __name__ == "__main__":
338  retcode = main()
339  sys.exit(retcode)
GaudiKernel.ConfigurableDb.loadConfigurableDb
def loadConfigurableDb()
Definition: ConfigurableDb.py:113
genconfuser._inheritsfrom
def _inheritsfrom(derived, basenames)
Definition: genconfuser.py:33
format
GAUDI_API std::string format(const char *,...)
MsgStream format utility "a la sprintf(...)".
Definition: MsgStream.cpp:119
genconfuser.main
def main()
Definition: genconfuser.py:171
genconfuser.getConfigurableUsers
def getConfigurableUsers(modulename, root, mayNotExist=False)
Definition: genconfuser.py:111
genconfuser.loadConfigurableDb
def loadConfigurableDb(build_dir=None, project_name=None)
Definition: genconfuser.py:52
GaudiKernel.ConfigurableDb
Definition: ConfigurableDb.py:1