All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
GaudiTest Namespace Reference

Classes

class  BasicOutputValidator
 Output Validation Classes. More...
 
class  BlockSkipper
 
class  CMT
 
class  FilePreprocessor
 
class  FilePreprocessorSequence
 
class  GaudiExeTest
 
class  GaudiFilterExecutable
 
class  HTMLResultStream
 
class  LineSkipper
 
class  LineSorter
 Special preprocessor sorting the list of strings (whitespace separated) that follow a signature on a single line. More...
 
class  ReferenceFileValidator
 
class  RegexpReplacer
 
class  TempDir
 
class  TempFile
 
class  TemporaryEnvironment
 Utility Classes. More...
 
class  XMLResultStream
 

Functions

def ROOT6WorkAroundEnabled
 
def total_seconds_replacement
 
def which
 Locates an executable in the executables path ($PATH) and returns the full path to it. More...
 
def rationalizepath
 
def hexreplace
 
def hexConvert
 
def convert_xml_illegal_chars
 
def escape_xml_illegal_chars
 
def findReferenceBlock
 
def countErrorLines
 
def _parseTTreeSummary
 
def findTTreeSummaries
 
def cmpTreesDicts
 
def getCmpFailingValues
 
def parseHistosSummary
 
def findHistosSummaries
 

Variables

string __author__ = 'Marco Clemencic CERN/PH-LBC'
 File: GaudiTest.py Author: Marco Clemencic CERN/PH-LBC. More...
 
tuple _illegal_xml_chars_RE = re.compile(u'[\x00-\x08\x0b\x0c\x0e-\x1F\uD800-\uDFFF\uFFFE\uFFFF]')
 
tuple maskPointers = RegexpReplacer("0x[0-9a-fA-F]{4,16}","0x########")
 
tuple normalizeDate
 
tuple normalizeEOL = FilePreprocessor()
 
tuple skipEmptyLines = FilePreprocessor()
 
 normalizeExamples = maskPointers+normalizeDate
 
tuple lineSkipper
 
tuple h_count_re = re.compile(r"^(.*)SUCCESS\s+Booked (\d+) Histogram\(s\) :\s+(.*)")
 

Function Documentation

def GaudiTest._parseTTreeSummary (   lines,
  pos 
)
private
Parse the TTree summary table in lines, starting from pos.
Returns a tuple with the dictionary with the digested informations and the
position of the first line after the summary.

Definition at line 703 of file GaudiTest.py.

704 def _parseTTreeSummary(lines, pos):
705  """
706  Parse the TTree summary table in lines, starting from pos.
707  Returns a tuple with the dictionary with the digested informations and the
708  position of the first line after the summary.
709  """
710  result = {}
711  i = pos + 1 # first line is a sequence of '*'
712  count = len(lines)
713 
714  splitcols = lambda l: [ f.strip() for f in l.strip("*\n").split(':',2) ]
715  def parseblock(ll):
716  r = {}
717  cols = splitcols(ll[0])
718  r["Name"], r["Title"] = cols[1:]
719 
720  cols = splitcols(ll[1])
721  r["Entries"] = int(cols[1])
722 
723  sizes = cols[2].split()
724  r["Total size"] = int(sizes[2])
725  if sizes[-1] == "memory":
726  r["File size"] = 0
727  else:
728  r["File size"] = int(sizes[-1])
729 
730  cols = splitcols(ll[2])
731  sizes = cols[2].split()
732  if cols[0] == "Baskets":
733  r["Baskets"] = int(cols[1])
734  r["Basket size"] = int(sizes[2])
735  r["Compression"] = float(sizes[-1])
736  return r
737 
738  if i < (count - 3) and lines[i].startswith("*Tree"):
739  result = parseblock(lines[i:i+3])
740  result["Branches"] = {}
741  i += 4
742  while i < (count - 3) and lines[i].startswith("*Br"):
743  if i < (count - 2) and lines[i].startswith("*Branch "):
744  # skip branch header
745  i += 3
746  continue
747  branch = parseblock(lines[i:i+3])
748  result["Branches"][branch["Name"]] = branch
749  i += 4
750 
751  return (result, i)
def _parseTTreeSummary
Definition: GaudiTest.py:703
def GaudiTest.cmpTreesDicts (   reference,
  to_check,
  ignore = None 
)
Check that all the keys in reference are in to_check too, with the same value.
If the value is a dict, the function is called recursively. to_check can
contain more keys than reference, that will not be tested.
The function returns at the first difference found.

Definition at line 773 of file GaudiTest.py.

774 def cmpTreesDicts(reference, to_check, ignore = None):
775  """
776  Check that all the keys in reference are in to_check too, with the same value.
777  If the value is a dict, the function is called recursively. to_check can
778  contain more keys than reference, that will not be tested.
779  The function returns at the first difference found.
780  """
781  fail_keys = []
782  # filter the keys in the reference dictionary
783  if ignore:
784  ignore_re = re.compile(ignore)
785  keys = [ key for key in reference if not ignore_re.match(key) ]
786  else:
787  keys = reference.keys()
788  # loop over the keys (not ignored) in the reference dictionary
789  for k in keys:
790  if k in to_check: # the key must be in the dictionary to_check
791  if (type(reference[k]) is dict) and (type(to_check[k]) is dict):
792  # if both reference and to_check values are dictionaries, recurse
793  failed = fail_keys = cmpTreesDicts(reference[k], to_check[k], ignore)
794  else:
795  # compare the two values
796  failed = to_check[k] != reference[k]
797  else: # handle missing keys in the dictionary to check (i.e. failure)
798  to_check[k] = None
799  failed = True
800  if failed:
801  fail_keys.insert(0, k)
802  break # exit from the loop at the first failure
803  return fail_keys # return the list of keys bringing to the different values
string type
Definition: gaudirun.py:126
def cmpTreesDicts
Definition: GaudiTest.py:773
def GaudiTest.convert_xml_illegal_chars (   val)

Definition at line 337 of file GaudiTest.py.

339  return _illegal_xml_chars_RE.sub(hexreplace, val)
def convert_xml_illegal_chars
Definition: GaudiTest.py:337
def GaudiTest.countErrorLines (   expected = {'ERROR':0,
  FATAL 
)

Definition at line 667 of file GaudiTest.py.

668 def countErrorLines(expected = {'ERROR':0, 'FATAL':0}, **kwargs):
669  """
670  Count the number of messages with required severity (by default ERROR and FATAL)
671  and check if their numbers match the expected ones (0 by default).
672  The dictionary "expected" can be used to tune the number of errors and fatals
673  allowed, or to limit the number of expected warnings etc.
674  """
675  stdout = kwargs["stdout"]
676  result = kwargs["result"]
677  causes = kwargs["causes"]
678 
679  # prepare the dictionary to record the extracted lines
680  errors = {}
681  for sev in expected:
682  errors[sev] = []
683 
684  outlines = stdout.splitlines()
685  from math import log10
686  fmt = "%%%dd - %%s" % (int(log10(len(outlines))+1))
687 
688  linecount = 0
689  for l in outlines:
690  linecount += 1
691  words = l.split()
692  if len(words) >= 2 and words[1] in errors:
693  errors[words[1]].append(fmt%(linecount,l.rstrip()))
694 
695  for e in errors:
696  if len(errors[e]) != expected[e]:
697  causes.append('%s(%d)'%(e,len(errors[e])))
698  result["GaudiTest.lines.%s"%e] = result.Quote('\n'.join(errors[e]))
699  result["GaudiTest.lines.%s.expected#"%e] = result.Quote(str(expected[e]))
700 
701  return causes
702 
def countErrorLines
Definition: GaudiTest.py:667
def GaudiTest.escape_xml_illegal_chars (   val,
  replacement = '?' 
)
Filter out characters that are illegal in XML.
Looks for any character in val that is not allowed in XML
and replaces it with replacement ('?' by default).

Definition at line 340 of file GaudiTest.py.

341 def escape_xml_illegal_chars(val, replacement='?'):
342  """Filter out characters that are illegal in XML.
343  Looks for any character in val that is not allowed in XML
344  and replaces it with replacement ('?' by default).
345 
346  """
347  return _illegal_xml_chars_RE.sub(replacement, val)
def escape_xml_illegal_chars
Definition: GaudiTest.py:340
def GaudiTest.findHistosSummaries (   stdout)
Scan stdout to find ROOT TTree summaries and digest them.

Definition at line 880 of file GaudiTest.py.

881 def findHistosSummaries(stdout):
882  """
883  Scan stdout to find ROOT TTree summaries and digest them.
884  """
885  outlines = stdout.splitlines()
886  nlines = len(outlines) - 1
887  summaries = {}
888  global h_count_re
889 
890  pos = 0
891  while pos < nlines:
892  summ = {}
893  # find first line of block:
894  match = h_count_re.search(outlines[pos])
895  while pos < nlines and not match:
896  pos += 1
897  match = h_count_re.search(outlines[pos])
898  if match:
899  summ, pos = parseHistosSummary(outlines, pos)
900  summaries.update(summ)
901  return summaries
def findHistosSummaries
Definition: GaudiTest.py:880
def parseHistosSummary
Definition: GaudiTest.py:817
def GaudiTest.findReferenceBlock (   reference,
  stdout,
  result,
  causes,
  signature_offset = 0,
  signature = None,
  id = None 
)
Given a block of text, tries to find it in the output.
The block had to be identified by a signature line. By default, the first
line is used as signature, or the line pointed to by signature_offset. If
signature_offset points outside the block, a signature line can be passed as
signature argument. Note: if 'signature' is None (the default), a negative
signature_offset is interpreted as index in a list (e.g. -1 means the last
line), otherwise the it is interpreted as the number of lines before the
first one of the block the signature must appear.
The parameter 'id' allow to distinguish between different calls to this
function in the same validation code.

Definition at line 621 of file GaudiTest.py.

622  id = None):
623  """
624  Given a block of text, tries to find it in the output.
625  The block had to be identified by a signature line. By default, the first
626  line is used as signature, or the line pointed to by signature_offset. If
627  signature_offset points outside the block, a signature line can be passed as
628  signature argument. Note: if 'signature' is None (the default), a negative
629  signature_offset is interpreted as index in a list (e.g. -1 means the last
630  line), otherwise the it is interpreted as the number of lines before the
631  first one of the block the signature must appear.
632  The parameter 'id' allow to distinguish between different calls to this
633  function in the same validation code.
634  """
635  # split reference file, sanitize EOLs and remove empty lines
636  reflines = filter(None,map(lambda s: s.rstrip(), reference.splitlines()))
637  if not reflines:
638  raise RuntimeError("Empty (or null) reference")
639  # the same on standard output
640  outlines = filter(None,map(lambda s: s.rstrip(), stdout.splitlines()))
641 
642  res_field = "GaudiTest.RefBlock"
643  if id:
644  res_field += "_%s" % id
645 
646  if signature is None:
647  if signature_offset < 0:
648  signature_offset = len(reference)+signature_offset
649  signature = reflines[signature_offset]
650  # find the reference block in the output file
651  try:
652  pos = outlines.index(signature)
653  outlines = outlines[pos-signature_offset:pos+len(reflines)-signature_offset]
654  if reflines != outlines:
655  msg = "standard output"
656  # I do not want 2 messages in causes if teh function is called twice
657  if not msg in causes:
658  causes.append(msg)
659  result[res_field + ".observed"] = result.Quote("\n".join(outlines))
660  except ValueError:
661  causes.append("missing signature")
662  result[res_field + ".signature"] = result.Quote(signature)
663  if len(reflines) > 1 or signature != reflines[0]:
664  result[res_field + ".expected"] = result.Quote("\n".join(reflines))
665 
666  return causes
struct GAUDI_API map
Parametrisation class for map-like implementation.
def GaudiTest.findTTreeSummaries (   stdout)
Scan stdout to find ROOT TTree summaries and digest them.

Definition at line 752 of file GaudiTest.py.

753 def findTTreeSummaries(stdout):
754  """
755  Scan stdout to find ROOT TTree summaries and digest them.
756  """
757  stars = re.compile(r"^\*+$")
758  outlines = stdout.splitlines()
759  nlines = len(outlines)
760  trees = {}
761 
762  i = 0
763  while i < nlines: #loop over the output
764  # look for
765  while i < nlines and not stars.match(outlines[i]):
766  i += 1
767  if i < nlines:
768  tree, i = _parseTTreeSummary(outlines, i)
769  if tree:
770  trees[tree["Name"]] = tree
771 
772  return trees
def _parseTTreeSummary
Definition: GaudiTest.py:703
def findTTreeSummaries
Definition: GaudiTest.py:752
def GaudiTest.getCmpFailingValues (   reference,
  to_check,
  fail_path 
)

Definition at line 804 of file GaudiTest.py.

805 def getCmpFailingValues(reference, to_check, fail_path):
806  c = to_check
807  r = reference
808  for k in fail_path:
809  c = c.get(k,None)
810  r = r.get(k,None)
811  if c is None or r is None:
812  break # one of the dictionaries is not deep enough
813  return (fail_path, r, c)
814 
# signature of the print-out of the histograms
def getCmpFailingValues
Definition: GaudiTest.py:804
def GaudiTest.hexConvert (   char)

Definition at line 335 of file GaudiTest.py.

336 def hexConvert(char):
return hex(ord(char))
def hexConvert
Definition: GaudiTest.py:335
def GaudiTest.hexreplace (   match)

Definition at line 331 of file GaudiTest.py.

332 def hexreplace( match ):
333  "Return the hex string "
334  return "".join(map(hexConvert,match.group()))
struct GAUDI_API map
Parametrisation class for map-like implementation.
def hexreplace
Definition: GaudiTest.py:331
def GaudiTest.parseHistosSummary (   lines,
  pos 
)
Extract the histograms infos from the lines starting at pos.
Returns the position of the first line after the summary block.

Definition at line 817 of file GaudiTest.py.

818 def parseHistosSummary(lines, pos):
819  """
820  Extract the histograms infos from the lines starting at pos.
821  Returns the position of the first line after the summary block.
822  """
823  global h_count_re
824  h_table_head = re.compile(r'SUCCESS\s+List of booked (1D|2D|3D|1D profile|2D profile) histograms in directory\s+"(\w*)"')
825  h_short_summ = re.compile(r"ID=([^\"]+)\s+\"([^\"]+)\"\s+(.*)")
826 
827  nlines = len(lines)
828 
829  # decode header
830  m = h_count_re.search(lines[pos])
831  name = m.group(1).strip()
832  total = int(m.group(2))
833  header = {}
834  for k, v in [ x.split("=") for x in m.group(3).split() ]:
835  header[k] = int(v)
836  pos += 1
837  header["Total"] = total
838 
839  summ = {}
840  while pos < nlines:
841  m = h_table_head.search(lines[pos])
842  if m:
843  t, d = m.groups(1) # type and directory
844  t = t.replace(" profile", "Prof")
845  pos += 1
846  if pos < nlines:
847  l = lines[pos]
848  else:
849  l = ""
850  cont = {}
851  if l.startswith(" | ID"):
852  # table format
853  titles = [ x.strip() for x in l.split("|")][1:]
854  pos += 1
855  while pos < nlines and lines[pos].startswith(" |"):
856  l = lines[pos]
857  values = [ x.strip() for x in l.split("|")][1:]
858  hcont = {}
859  for i in range(len(titles)):
860  hcont[titles[i]] = values[i]
861  cont[hcont["ID"]] = hcont
862  pos += 1
863  elif l.startswith(" ID="):
864  while pos < nlines and lines[pos].startswith(" ID="):
865  values = [ x.strip() for x in h_short_summ.search(lines[pos]).groups() ]
866  cont[values[0]] = values
867  pos += 1
868  else: # not interpreted
869  raise RuntimeError("Cannot understand line %d: '%s'" % (pos, l))
870  if not d in summ:
871  summ[d] = {}
872  summ[d][t] = cont
873  summ[d]["header"] = header
874  else:
875  break
876  if not summ:
877  # If the full table is not present, we use only the header
878  summ[name] = {"header": header}
879  return summ, pos
def parseHistosSummary
Definition: GaudiTest.py:817
NamedRange_< CONTAINER > range(const CONTAINER &cnt, const std::string &name)
simple function to create the named range form arbitrary container
Definition: NamedRange.h:133
def GaudiTest.rationalizepath (   p)

Definition at line 309 of file GaudiTest.py.

310 def rationalizepath(p):
311  np = os.path.normpath(os.path.expandvars(p))
312  if os.path.exists(np):
313  p = os.path.realpath(np)
314  return p
315 
# XML Escaping character
def rationalizepath
Definition: GaudiTest.py:309
def GaudiTest.ROOT6WorkAroundEnabled (   id = None)

Definition at line 25 of file GaudiTest.py.

25 
26  def ROOT6WorkAroundEnabled(id=None):
27  # dummy implementation
28  return False
29 
30 # ensure the preferred locale
31 os.environ['LC_ALL'] = 'C'
32 
33 # Needed for the XML wrapper
try:
def ROOT6WorkAroundEnabled
Definition: GaudiTest.py:25
def GaudiTest.total_seconds_replacement (   timedelta)

Definition at line 39 of file GaudiTest.py.

39 
40 def total_seconds_replacement(timedelta) :
41  return timedelta.days*86400 + timedelta.seconds + timedelta.microseconds/1000000
42 
def total_seconds_replacement
Definition: GaudiTest.py:39
def GaudiTest.which (   executable)

Locates an executable in the executables path ($PATH) and returns the full path to it.

If the executable cannot be found, None is returned

Locates an executable in the executables path ($PATH) and returns the full
path to it.  An application is looked for with or without the '.exe' suffix.
If the executable cannot be found, None is returned

Definition at line 289 of file GaudiTest.py.

290 def which(executable):
291  """
292  Locates an executable in the executables path ($PATH) and returns the full
293  path to it. An application is looked for with or without the '.exe' suffix.
294  If the executable cannot be found, None is returned
295  """
296  if os.path.isabs(executable):
297  if not os.path.exists(executable):
298  if executable.endswith('.exe'):
299  if os.path.exists(executable[:-4]):
300  return executable[:-4]
301  return executable
302  for d in os.environ.get("PATH").split(os.pathsep):
303  fullpath = os.path.join(d, executable)
304  if os.path.exists(fullpath):
305  return fullpath
306  if executable.endswith('.exe'):
307  return which(executable[:-4])
308  return None
def which
Locates an executable in the executables path ($PATH) and returns the full path to it...
Definition: GaudiTest.py:289

Variable Documentation

string GaudiTest.__author__ = 'Marco Clemencic CERN/PH-LBC'

File: GaudiTest.py Author: Marco Clemencic CERN/PH-LBC.

Definition at line 5 of file GaudiTest.py.

tuple GaudiTest._illegal_xml_chars_RE = re.compile(u'[\x00-\x08\x0b\x0c\x0e-\x1F\uD800-\uDFFF\uFFFE\uFFFF]')

Definition at line 329 of file GaudiTest.py.

tuple GaudiTest.h_count_re = re.compile(r"^(.*)SUCCESS\s+Booked (\d+) Histogram\(s\) :\s+(.*)")

Definition at line 815 of file GaudiTest.py.

tuple GaudiTest.lineSkipper

Definition at line 528 of file GaudiTest.py.

tuple GaudiTest.maskPointers = RegexpReplacer("0x[0-9a-fA-F]{4,16}","0x########")

Definition at line 483 of file GaudiTest.py.

tuple GaudiTest.normalizeDate
Initial value:
1 = RegexpReplacer("[0-2]?[0-9]:[0-5][0-9]:[0-5][0-9] [0-9]{4}[-/][01][0-9][-/][0-3][0-9] *(CES?T)?",
2  "00:00:00 1970-01-01")

Definition at line 484 of file GaudiTest.py.

tuple GaudiTest.normalizeEOL = FilePreprocessor()

Definition at line 486 of file GaudiTest.py.

tuple GaudiTest.normalizeExamples = maskPointers+normalizeDate

Definition at line 509 of file GaudiTest.py.

tuple GaudiTest.skipEmptyLines = FilePreprocessor()

Definition at line 489 of file GaudiTest.py.