All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
GaudiTest Namespace Reference

Classes

class  BasicOutputValidator
 Output Validation Classes. More...
 
class  BlockSkipper
 
class  CMT
 
class  FilePreprocessor
 
class  FilePreprocessorSequence
 
class  GaudiExeTest
 
class  GaudiFilterExecutable
 
class  HTMLResultStream
 
class  LineSkipper
 
class  LineSorter
 Special preprocessor sorting the list of strings (whitespace separated) that follow a signature on a single line. More...
 
class  ReferenceFileValidator
 
class  RegexpReplacer
 
class  TempDir
 
class  TempFile
 
class  TemporaryEnvironment
 Utility Classes. More...
 
class  XMLResultStream
 

Functions

def ROOT6WorkAroundEnabled
 
def total_seconds_replacement
 
def which
 Locates an executable in the executables path ($PATH) and returns the full path to it. More...
 
def rationalizepath
 
def hexreplace
 
def hexConvert
 
def convert_xml_illegal_chars
 
def escape_xml_illegal_chars
 
def findReferenceBlock
 
def countErrorLines
 
def _parseTTreeSummary
 
def findTTreeSummaries
 
def cmpTreesDicts
 
def getCmpFailingValues
 
def parseHistosSummary
 
def findHistosSummaries
 

Variables

string __author__ = 'Marco Clemencic CERN/PH-LBC'
 File: GaudiTest.py Author: Marco Clemencic CERN/PH-LBC. More...
 
tuple _illegal_xml_chars_RE = re.compile(u'[\x00-\x08\x0b\x0c\x0e-\x1F\uD800-\uDFFF\uFFFE\uFFFF]')
 
tuple maskPointers = RegexpReplacer("0x[0-9a-fA-F]{4,16}","0x########")
 
tuple normalizeDate
 
tuple normalizeEOL = FilePreprocessor()
 
tuple skipEmptyLines = FilePreprocessor()
 
 normalizeExamples = maskPointers+normalizeDate
 
tuple lineSkipper
 
tuple h_count_re = re.compile(r"^(.*)SUCCESS\s+Booked (\d+) Histogram\(s\) :\s+(.*)")
 

Function Documentation

def GaudiTest._parseTTreeSummary (   lines,
  pos 
)
private
Parse the TTree summary table in lines, starting from pos.
Returns a tuple with the dictionary with the digested informations and the
position of the first line after the summary.

Definition at line 705 of file GaudiTest.py.

706 def _parseTTreeSummary(lines, pos):
707  """
708  Parse the TTree summary table in lines, starting from pos.
709  Returns a tuple with the dictionary with the digested informations and the
710  position of the first line after the summary.
711  """
712  result = {}
713  i = pos + 1 # first line is a sequence of '*'
714  count = len(lines)
715 
716  splitcols = lambda l: [ f.strip() for f in l.strip("*\n").split(':',2) ]
717  def parseblock(ll):
718  r = {}
719  cols = splitcols(ll[0])
720  r["Name"], r["Title"] = cols[1:]
721 
722  cols = splitcols(ll[1])
723  r["Entries"] = int(cols[1])
724 
725  sizes = cols[2].split()
726  r["Total size"] = int(sizes[2])
727  if sizes[-1] == "memory":
728  r["File size"] = 0
729  else:
730  r["File size"] = int(sizes[-1])
731 
732  cols = splitcols(ll[2])
733  sizes = cols[2].split()
734  if cols[0] == "Baskets":
735  r["Baskets"] = int(cols[1])
736  r["Basket size"] = int(sizes[2])
737  r["Compression"] = float(sizes[-1])
738  return r
739 
740  if i < (count - 3) and lines[i].startswith("*Tree"):
741  result = parseblock(lines[i:i+3])
742  result["Branches"] = {}
743  i += 4
744  while i < (count - 3) and lines[i].startswith("*Br"):
745  if i < (count - 2) and lines[i].startswith("*Branch "):
746  # skip branch header
747  i += 3
748  continue
749  branch = parseblock(lines[i:i+3])
750  result["Branches"][branch["Name"]] = branch
751  i += 4
752 
753  return (result, i)
def _parseTTreeSummary
Definition: GaudiTest.py:705
def GaudiTest.cmpTreesDicts (   reference,
  to_check,
  ignore = None 
)
Check that all the keys in reference are in to_check too, with the same value.
If the value is a dict, the function is called recursively. to_check can
contain more keys than reference, that will not be tested.
The function returns at the first difference found.

Definition at line 775 of file GaudiTest.py.

776 def cmpTreesDicts(reference, to_check, ignore = None):
777  """
778  Check that all the keys in reference are in to_check too, with the same value.
779  If the value is a dict, the function is called recursively. to_check can
780  contain more keys than reference, that will not be tested.
781  The function returns at the first difference found.
782  """
783  fail_keys = []
784  # filter the keys in the reference dictionary
785  if ignore:
786  ignore_re = re.compile(ignore)
787  keys = [ key for key in reference if not ignore_re.match(key) ]
788  else:
789  keys = reference.keys()
790  # loop over the keys (not ignored) in the reference dictionary
791  for k in keys:
792  if k in to_check: # the key must be in the dictionary to_check
793  if (type(reference[k]) is dict) and (type(to_check[k]) is dict):
794  # if both reference and to_check values are dictionaries, recurse
795  failed = fail_keys = cmpTreesDicts(reference[k], to_check[k], ignore)
796  else:
797  # compare the two values
798  failed = to_check[k] != reference[k]
799  else: # handle missing keys in the dictionary to check (i.e. failure)
800  to_check[k] = None
801  failed = True
802  if failed:
803  fail_keys.insert(0, k)
804  break # exit from the loop at the first failure
805  return fail_keys # return the list of keys bringing to the different values
string type
Definition: gaudirun.py:126
def cmpTreesDicts
Definition: GaudiTest.py:775
def GaudiTest.convert_xml_illegal_chars (   val)

Definition at line 337 of file GaudiTest.py.

339  return _illegal_xml_chars_RE.sub(hexreplace, val)
def convert_xml_illegal_chars
Definition: GaudiTest.py:337
def GaudiTest.countErrorLines (   expected = {'ERROR':0,
  FATAL 
)

Definition at line 669 of file GaudiTest.py.

670 def countErrorLines(expected = {'ERROR':0, 'FATAL':0}, **kwargs):
671  """
672  Count the number of messages with required severity (by default ERROR and FATAL)
673  and check if their numbers match the expected ones (0 by default).
674  The dictionary "expected" can be used to tune the number of errors and fatals
675  allowed, or to limit the number of expected warnings etc.
676  """
677  stdout = kwargs["stdout"]
678  result = kwargs["result"]
679  causes = kwargs["causes"]
680 
681  # prepare the dictionary to record the extracted lines
682  errors = {}
683  for sev in expected:
684  errors[sev] = []
685 
686  outlines = stdout.splitlines()
687  from math import log10
688  fmt = "%%%dd - %%s" % (int(log10(len(outlines))+1))
689 
690  linecount = 0
691  for l in outlines:
692  linecount += 1
693  words = l.split()
694  if len(words) >= 2 and words[1] in errors:
695  errors[words[1]].append(fmt%(linecount,l.rstrip()))
696 
697  for e in errors:
698  if len(errors[e]) != expected[e]:
699  causes.append('%s(%d)'%(e,len(errors[e])))
700  result["GaudiTest.lines.%s"%e] = result.Quote('\n'.join(errors[e]))
701  result["GaudiTest.lines.%s.expected#"%e] = result.Quote(str(expected[e]))
702 
703  return causes
704 
def countErrorLines
Definition: GaudiTest.py:669
def GaudiTest.escape_xml_illegal_chars (   val,
  replacement = '?' 
)
Filter out characters that are illegal in XML.
Looks for any character in val that is not allowed in XML
and replaces it with replacement ('?' by default).

Definition at line 340 of file GaudiTest.py.

341 def escape_xml_illegal_chars(val, replacement='?'):
342  """Filter out characters that are illegal in XML.
343  Looks for any character in val that is not allowed in XML
344  and replaces it with replacement ('?' by default).
345 
346  """
347  return _illegal_xml_chars_RE.sub(replacement, val)
def escape_xml_illegal_chars
Definition: GaudiTest.py:340
def GaudiTest.findHistosSummaries (   stdout)
Scan stdout to find ROOT TTree summaries and digest them.

Definition at line 882 of file GaudiTest.py.

883 def findHistosSummaries(stdout):
884  """
885  Scan stdout to find ROOT TTree summaries and digest them.
886  """
887  outlines = stdout.splitlines()
888  nlines = len(outlines) - 1
889  summaries = {}
890  global h_count_re
891 
892  pos = 0
893  while pos < nlines:
894  summ = {}
895  # find first line of block:
896  match = h_count_re.search(outlines[pos])
897  while pos < nlines and not match:
898  pos += 1
899  match = h_count_re.search(outlines[pos])
900  if match:
901  summ, pos = parseHistosSummary(outlines, pos)
902  summaries.update(summ)
903  return summaries
def findHistosSummaries
Definition: GaudiTest.py:882
def parseHistosSummary
Definition: GaudiTest.py:819
def GaudiTest.findReferenceBlock (   reference,
  stdout,
  result,
  causes,
  signature_offset = 0,
  signature = None,
  id = None 
)
Given a block of text, tries to find it in the output.
The block had to be identified by a signature line. By default, the first
line is used as signature, or the line pointed to by signature_offset. If
signature_offset points outside the block, a signature line can be passed as
signature argument. Note: if 'signature' is None (the default), a negative
signature_offset is interpreted as index in a list (e.g. -1 means the last
line), otherwise the it is interpreted as the number of lines before the
first one of the block the signature must appear.
The parameter 'id' allow to distinguish between different calls to this
function in the same validation code.

Definition at line 623 of file GaudiTest.py.

624  id = None):
625  """
626  Given a block of text, tries to find it in the output.
627  The block had to be identified by a signature line. By default, the first
628  line is used as signature, or the line pointed to by signature_offset. If
629  signature_offset points outside the block, a signature line can be passed as
630  signature argument. Note: if 'signature' is None (the default), a negative
631  signature_offset is interpreted as index in a list (e.g. -1 means the last
632  line), otherwise the it is interpreted as the number of lines before the
633  first one of the block the signature must appear.
634  The parameter 'id' allow to distinguish between different calls to this
635  function in the same validation code.
636  """
637  # split reference file, sanitize EOLs and remove empty lines
638  reflines = filter(None,map(lambda s: s.rstrip(), reference.splitlines()))
639  if not reflines:
640  raise RuntimeError("Empty (or null) reference")
641  # the same on standard output
642  outlines = filter(None,map(lambda s: s.rstrip(), stdout.splitlines()))
643 
644  res_field = "GaudiTest.RefBlock"
645  if id:
646  res_field += "_%s" % id
647 
648  if signature is None:
649  if signature_offset < 0:
650  signature_offset = len(reference)+signature_offset
651  signature = reflines[signature_offset]
652  # find the reference block in the output file
653  try:
654  pos = outlines.index(signature)
655  outlines = outlines[pos-signature_offset:pos+len(reflines)-signature_offset]
656  if reflines != outlines:
657  msg = "standard output"
658  # I do not want 2 messages in causes if teh function is called twice
659  if not msg in causes:
660  causes.append(msg)
661  result[res_field + ".observed"] = result.Quote("\n".join(outlines))
662  except ValueError:
663  causes.append("missing signature")
664  result[res_field + ".signature"] = result.Quote(signature)
665  if len(reflines) > 1 or signature != reflines[0]:
666  result[res_field + ".expected"] = result.Quote("\n".join(reflines))
667 
668  return causes
struct GAUDI_API map
Parametrisation class for map-like implementation.
def GaudiTest.findTTreeSummaries (   stdout)
Scan stdout to find ROOT TTree summaries and digest them.

Definition at line 754 of file GaudiTest.py.

755 def findTTreeSummaries(stdout):
756  """
757  Scan stdout to find ROOT TTree summaries and digest them.
758  """
759  stars = re.compile(r"^\*+$")
760  outlines = stdout.splitlines()
761  nlines = len(outlines)
762  trees = {}
763 
764  i = 0
765  while i < nlines: #loop over the output
766  # look for
767  while i < nlines and not stars.match(outlines[i]):
768  i += 1
769  if i < nlines:
770  tree, i = _parseTTreeSummary(outlines, i)
771  if tree:
772  trees[tree["Name"]] = tree
773 
774  return trees
def _parseTTreeSummary
Definition: GaudiTest.py:705
def findTTreeSummaries
Definition: GaudiTest.py:754
def GaudiTest.getCmpFailingValues (   reference,
  to_check,
  fail_path 
)

Definition at line 806 of file GaudiTest.py.

807 def getCmpFailingValues(reference, to_check, fail_path):
808  c = to_check
809  r = reference
810  for k in fail_path:
811  c = c.get(k,None)
812  r = r.get(k,None)
813  if c is None or r is None:
814  break # one of the dictionaries is not deep enough
815  return (fail_path, r, c)
816 
# signature of the print-out of the histograms
def getCmpFailingValues
Definition: GaudiTest.py:806
def GaudiTest.hexConvert (   char)

Definition at line 335 of file GaudiTest.py.

336 def hexConvert(char):
return hex(ord(char))
def hexConvert
Definition: GaudiTest.py:335
def GaudiTest.hexreplace (   match)

Definition at line 331 of file GaudiTest.py.

332 def hexreplace( match ):
333  "Return the hex string "
334  return "".join(map(hexConvert,match.group()))
struct GAUDI_API map
Parametrisation class for map-like implementation.
def hexreplace
Definition: GaudiTest.py:331
def GaudiTest.parseHistosSummary (   lines,
  pos 
)
Extract the histograms infos from the lines starting at pos.
Returns the position of the first line after the summary block.

Definition at line 819 of file GaudiTest.py.

820 def parseHistosSummary(lines, pos):
821  """
822  Extract the histograms infos from the lines starting at pos.
823  Returns the position of the first line after the summary block.
824  """
825  global h_count_re
826  h_table_head = re.compile(r'SUCCESS\s+List of booked (1D|2D|3D|1D profile|2D profile) histograms in directory\s+"(\w*)"')
827  h_short_summ = re.compile(r"ID=([^\"]+)\s+\"([^\"]+)\"\s+(.*)")
828 
829  nlines = len(lines)
830 
831  # decode header
832  m = h_count_re.search(lines[pos])
833  name = m.group(1).strip()
834  total = int(m.group(2))
835  header = {}
836  for k, v in [ x.split("=") for x in m.group(3).split() ]:
837  header[k] = int(v)
838  pos += 1
839  header["Total"] = total
840 
841  summ = {}
842  while pos < nlines:
843  m = h_table_head.search(lines[pos])
844  if m:
845  t, d = m.groups(1) # type and directory
846  t = t.replace(" profile", "Prof")
847  pos += 1
848  if pos < nlines:
849  l = lines[pos]
850  else:
851  l = ""
852  cont = {}
853  if l.startswith(" | ID"):
854  # table format
855  titles = [ x.strip() for x in l.split("|")][1:]
856  pos += 1
857  while pos < nlines and lines[pos].startswith(" |"):
858  l = lines[pos]
859  values = [ x.strip() for x in l.split("|")][1:]
860  hcont = {}
861  for i in range(len(titles)):
862  hcont[titles[i]] = values[i]
863  cont[hcont["ID"]] = hcont
864  pos += 1
865  elif l.startswith(" ID="):
866  while pos < nlines and lines[pos].startswith(" ID="):
867  values = [ x.strip() for x in h_short_summ.search(lines[pos]).groups() ]
868  cont[values[0]] = values
869  pos += 1
870  else: # not interpreted
871  raise RuntimeError("Cannot understand line %d: '%s'" % (pos, l))
872  if not d in summ:
873  summ[d] = {}
874  summ[d][t] = cont
875  summ[d]["header"] = header
876  else:
877  break
878  if not summ:
879  # If the full table is not present, we use only the header
880  summ[name] = {"header": header}
881  return summ, pos
def parseHistosSummary
Definition: GaudiTest.py:819
NamedRange_< CONTAINER > range(const CONTAINER &cnt, const std::string &name)
simple function to create the named range form arbitrary container
Definition: NamedRange.h:133
def GaudiTest.rationalizepath (   p)

Definition at line 309 of file GaudiTest.py.

310 def rationalizepath(p):
311  np = os.path.normpath(os.path.expandvars(p))
312  if os.path.exists(np):
313  p = os.path.realpath(np)
314  return p
315 
# XML Escaping character
def rationalizepath
Definition: GaudiTest.py:309
def GaudiTest.ROOT6WorkAroundEnabled (   id = None)

Definition at line 25 of file GaudiTest.py.

25 
26  def ROOT6WorkAroundEnabled(id=None):
27  # dummy implementation
28  return False
29 
30 # ensure the preferred locale
31 os.environ['LC_ALL'] = 'C'
32 
33 # Needed for the XML wrapper
try:
def ROOT6WorkAroundEnabled
Definition: GaudiTest.py:25
def GaudiTest.total_seconds_replacement (   timedelta)

Definition at line 39 of file GaudiTest.py.

39 
40 def total_seconds_replacement(timedelta) :
41  return timedelta.days*86400 + timedelta.seconds + timedelta.microseconds/1000000
42 
def total_seconds_replacement
Definition: GaudiTest.py:39
def GaudiTest.which (   executable)

Locates an executable in the executables path ($PATH) and returns the full path to it.

If the executable cannot be found, None is returned

Locates an executable in the executables path ($PATH) and returns the full
path to it.  An application is looked for with or without the '.exe' suffix.
If the executable cannot be found, None is returned

Definition at line 289 of file GaudiTest.py.

290 def which(executable):
291  """
292  Locates an executable in the executables path ($PATH) and returns the full
293  path to it. An application is looked for with or without the '.exe' suffix.
294  If the executable cannot be found, None is returned
295  """
296  if os.path.isabs(executable):
297  if not os.path.exists(executable):
298  if executable.endswith('.exe'):
299  if os.path.exists(executable[:-4]):
300  return executable[:-4]
301  return executable
302  for d in os.environ.get("PATH").split(os.pathsep):
303  fullpath = os.path.join(d, executable)
304  if os.path.exists(fullpath):
305  return fullpath
306  if executable.endswith('.exe'):
307  return which(executable[:-4])
308  return None
def which
Locates an executable in the executables path ($PATH) and returns the full path to it...
Definition: GaudiTest.py:289

Variable Documentation

string GaudiTest.__author__ = 'Marco Clemencic CERN/PH-LBC'

File: GaudiTest.py Author: Marco Clemencic CERN/PH-LBC.

Definition at line 5 of file GaudiTest.py.

tuple GaudiTest._illegal_xml_chars_RE = re.compile(u'[\x00-\x08\x0b\x0c\x0e-\x1F\uD800-\uDFFF\uFFFE\uFFFF]')

Definition at line 329 of file GaudiTest.py.

tuple GaudiTest.h_count_re = re.compile(r"^(.*)SUCCESS\s+Booked (\d+) Histogram\(s\) :\s+(.*)")

Definition at line 817 of file GaudiTest.py.

tuple GaudiTest.lineSkipper

Definition at line 528 of file GaudiTest.py.

tuple GaudiTest.maskPointers = RegexpReplacer("0x[0-9a-fA-F]{4,16}","0x########")

Definition at line 483 of file GaudiTest.py.

tuple GaudiTest.normalizeDate
Initial value:
1 = RegexpReplacer("[0-2]?[0-9]:[0-5][0-9]:[0-5][0-9] [0-9]{4}[-/][01][0-9][-/][0-3][0-9] *(CES?T)?",
2  "00:00:00 1970-01-01")

Definition at line 484 of file GaudiTest.py.

tuple GaudiTest.normalizeEOL = FilePreprocessor()

Definition at line 486 of file GaudiTest.py.

tuple GaudiTest.normalizeExamples = maskPointers+normalizeDate

Definition at line 509 of file GaudiTest.py.

tuple GaudiTest.skipEmptyLines = FilePreprocessor()

Definition at line 489 of file GaudiTest.py.