GaudiTest Namespace Reference

Classes
class	TemporaryEnvironment
class	TempDir
class	TempFile
class	CMT
class	BasicOutputValidator
class	FilePreprocessor
class	FilePreprocessorSequence
class	LineSkipper
class	BlockSkipper
class	RegexpReplacer
class	LineSorter
	Special preprocessor sorting the list of strings (whitespace separated) that follow a signature on a single line. More...
class	ReferenceFileValidator
class	GaudiFilterExecutable
class	GaudiExeTest
class	HTMLResultStream
Functions
def	which
	Locates an executable in the executables path ($PATH) and returns the full path to it.
def	rationalizepath
def	findReferenceBlock
def	countErrorLines
def	_parseTTreeSummary
def	findTTreeSummaries
def	cmpTreesDicts
def	getCmpFailingValues
def	parseHistosSummary
def	findHistosSummaries
Variables
string	__author__ = 'Marco Clemencic CERN/PH-LBC'
tuple	maskPointers = RegexpReplacer("0x[0-9a-fA-F]{4,16}","0x########")
tuple	normalizeDate
tuple	normalizeEOL = FilePreprocessor()
tuple	skipEmptyLines = FilePreprocessor()
	normalizeExamples = maskPointers+normalizeDate
tuple	h_count_re = re.compile(r"^(.*)SUCCESS\s+Booked (\d+) Histogram\(s\) :\s+(.*)")

---

Function Documentation

def GaudiTest::_parseTTreeSummary	(		lines,
			pos
	)			[private]

Parse the TTree summary table in lines, starting from pos.
Returns a tuple with the dictionary with the digested informations and the
position of the first line after the summary.

Definition at line 616 of file GaudiTest.py.

00616                                   :
00617     """
00618     Parse the TTree summary table in lines, starting from pos.
00619     Returns a tuple with the dictionary with the digested informations and the
00620     position of the first line after the summary.
00621     """
00622     result = {}
00623     i = pos + 1 # first line is a sequence of '*'
00624     count = len(lines)
00625 
00626     splitcols = lambda l: [ f.strip() for f in l.strip("*\n").split(':',2) ]
00627     def parseblock(ll):
00628         r = {}
00629         cols = splitcols(ll[0])
00630         r["Name"], r["Title"] = cols[1:]
00631 
00632         cols = splitcols(ll[1])
00633         r["Entries"] = int(cols[1])
00634 
00635         sizes = cols[2].split()
00636         r["Total size"] = int(sizes[2])
00637         if sizes[-1] == "memory":
00638             r["File size"] = 0
00639         else:
00640             r["File size"] = int(sizes[-1])
00641 
00642         cols = splitcols(ll[2])
00643         sizes = cols[2].split()
00644         if cols[0] == "Baskets":
00645             r["Baskets"] = int(cols[1])
00646             r["Basket size"] = int(sizes[2])
00647         r["Compression"] = float(sizes[-1])
00648         return r
00649 
00650     if i < (count - 3) and lines[i].startswith("*Tree"):
00651         result = parseblock(lines[i:i+3])
00652         result["Branches"] = {}
00653         i += 4
00654         while i < (count - 3) and lines[i].startswith("*Br"):
00655             branch = parseblock(lines[i:i+3])
00656             result["Branches"][branch["Name"]] = branch
00657             i += 4
00658 
00659     return (result, i)
00660 
def findTTreeSummaries(stdout):

def GaudiTest::cmpTreesDicts	(		reference,
			to_check,
			ignore = None
	)

Check that all the keys in reference are in to_check too, with the same value.
If the value is a dict, the function is called recursively. to_check can
contain more keys than reference, that will not be tested.
The function returns at the first difference found.

Definition at line 682 of file GaudiTest.py.

00682                                                      :
00683     """
00684     Check that all the keys in reference are in to_check too, with the same value.
00685     If the value is a dict, the function is called recursively. to_check can
00686     contain more keys than reference, that will not be tested.
00687     The function returns at the first difference found.
00688     """
00689     fail_keys = []
00690     # filter the keys in the reference dictionary
00691     if ignore:
00692         ignore_re = re.compile(ignore)
00693         keys = [ key for key in reference if not ignore_re.match(key) ]
00694     else:
00695         keys = reference.keys()
00696     # loop over the keys (not ignored) in the reference dictionary
00697     for k in keys:
00698         if k in to_check: # the key must be in the dictionary to_check
00699             if (type(reference[k]) is dict) and (type(to_check[k]) is dict):
00700                 # if both reference and to_check values are dictionaries, recurse
00701                 failed = fail_keys = cmpTreesDicts(reference[k], to_check[k], ignore)
00702             else:
00703                 # compare the two values
00704                 failed = to_check[k] != reference[k]
00705         else: # handle missing keys in the dictionary to check (i.e. failure)
00706             to_check[k] = None
00707             failed = True
00708         if failed:
00709             fail_keys.insert(0, k)
00710             break # exit from the loop at the first failure
00711     return fail_keys # return the list of keys bringing to the different values
00712 
def getCmpFailingValues(reference, to_check, fail_path):

def GaudiTest::countErrorLines	(		expected = {'ERROR':0,
			FATAL
	)

Definition at line 580 of file GaudiTest.py.

                               {'ERROR':0, 'FATAL':0}, **kwargs):
    """
    Count the number of messages with required severity (by default ERROR and FATAL)
    and check if their numbers match the expected ones (0 by default).
    The dictionary "expected" can be used to tune the number of errors and fatals
    allowed, or to limit the number of expected warnings etc.
    """
    stdout = kwargs["stdout"]
    result = kwargs["result"]
    causes = kwargs["causes"]

    # prepare the dictionary to record the extracted lines
    errors = {}
    for sev in expected:
        errors[sev] = []

    outlines = stdout.splitlines()
    from math import log10
    fmt = "%%%dd - %%s" % (int(log10(len(outlines))+1))

    linecount = 0
    for l in outlines:
        linecount += 1
        words = l.split()
        if len(words) >= 2 and words[1] in errors:
            errors[words[1]].append(fmt%(linecount,l.rstrip()))

    for e in errors:
        if len(errors[e]) != expected[e]:
            causes.append('%s(%d)'%(e,len(errors[e])))
            result["GaudiTest.lines.%s"%e] = result.Quote('\n'.join(errors[e]))
            result["GaudiTest.lines.%s.expected#"%e] = result.Quote(str(expected[e]))

    return causes

def GaudiTest::findHistosSummaries

(

stdout

)

Scan stdout to find ROOT TTree summaries and digest them.

Definition at line 789 of file GaudiTest.py.

00789                                :
00790     """
00791     Scan stdout to find ROOT TTree summaries and digest them.
00792     """
00793     outlines = stdout.splitlines()
00794     nlines = len(outlines) - 1
00795     summaries = {}
00796     global h_count_re
00797 
00798     pos = 0
00799     while pos < nlines:
00800         summ = {}
00801         # find first line of block:
00802         match = h_count_re.search(outlines[pos])
00803         while pos < nlines and not match:
00804             pos += 1
00805             match = h_count_re.search(outlines[pos])
00806         if match:
00807             summ, pos = parseHistosSummary(outlines, pos)
00808         summaries.update(summ)
00809     return summaries
00810 
class GaudiFilterExecutable(qm.executable.Filter):

def GaudiTest::findReferenceBlock	(		reference,
			stdout,
			result,
			causes,
			signature_offset = 0,
			signature = None,
			id = None
	)

Given a block of text, tries to find it in the output.
The block had to be identified by a signature line. By default, the first
line is used as signature, or the line pointed to by signature_offset. If
signature_offset points outside the block, a signature line can be passed as
signature argument. Note: if 'signature' is None (the default), a negative
signature_offset is interpreted as index in a list (e.g. -1 means the last
line), otherwise the it is interpreted as the number of lines before the
first one of the block the signature must appear.
The parameter 'id' allow to distinguish between different calls to this
function in the same validation code.

Definition at line 533 of file GaudiTest.py.

00534                                  :
00535     """
00536     Given a block of text, tries to find it in the output.
00537     The block had to be identified by a signature line. By default, the first
00538     line is used as signature, or the line pointed to by signature_offset. If
00539     signature_offset points outside the block, a signature line can be passed as
00540     signature argument. Note: if 'signature' is None (the default), a negative
00541     signature_offset is interpreted as index in a list (e.g. -1 means the last
00542     line), otherwise the it is interpreted as the number of lines before the
00543     first one of the block the signature must appear.
00544     The parameter 'id' allow to distinguish between different calls to this
00545     function in the same validation code.
00546     """
00547     # split reference file, sanitize EOLs and remove empty lines
00548     reflines = filter(None,map(lambda s: s.rstrip(), reference.splitlines()))
00549     if not reflines:
00550         raise RuntimeError("Empty (or null) reference")
00551     # the same on standard output
00552     outlines = filter(None,map(lambda s: s.rstrip(), stdout.splitlines()))
00553 
00554     res_field = "GaudiTest.RefBlock"
00555     if id:
00556         res_field += "_%s" % id
00557 
00558     if signature is None:
00559         if signature_offset < 0:
00560             signature_offset = len(reference)+signature_offset
00561         signature = reflines[signature_offset]
00562     # find the reference block in the output file
00563     try:
00564         pos = outlines.index(signature)
00565         outlines = outlines[pos-signature_offset:pos+len(reflines)-signature_offset]
00566         if reflines != outlines:
00567             msg = "standard output"
00568             # I do not want 2 messages in causes if teh function is called twice
00569             if not msg in causes:
00570                 causes.append(msg)
00571             result[res_field + ".observed"] = result.Quote("\n".join(outlines))
00572     except ValueError:
00573         causes.append("missing signature")
00574     result[res_field + ".signature"] = result.Quote(signature)
00575     if len(reflines) > 1 or signature != reflines[0]:
00576         result[res_field + ".expected"] = result.Quote("\n".join(reflines))
00577 
00578     return causes
00579 
def countErrorLines(expected = {'ERROR':0, 'FATAL':0}, **kwargs):

def GaudiTest::findTTreeSummaries

(

stdout

)

Scan stdout to find ROOT TTree summaries and digest them.

Definition at line 661 of file GaudiTest.py.

00661                               :
00662     """
00663     Scan stdout to find ROOT TTree summaries and digest them.
00664     """
00665     stars = re.compile(r"^\*+$")
00666     outlines = stdout.splitlines()
00667     nlines = len(outlines)
00668     trees = {}
00669 
00670     i = 0
00671     while i < nlines: #loop over the output
00672         # look for
00673         while i < nlines and not stars.match(outlines[i]):
00674             i += 1
00675         if i < nlines:
00676             tree, i = _parseTTreeSummary(outlines, i)
00677             if tree:
00678                 trees[tree["Name"]] = tree
00679 
00680     return trees
00681 
def cmpTreesDicts(reference, to_check, ignore = None):

def GaudiTest::getCmpFailingValues	(		reference,
			to_check,
			fail_path
	)

Definition at line 713 of file GaudiTest.py.

00713                                                        :
00714     c = to_check
00715     r = reference
00716     for k in fail_path:
00717         c = c.get(k,None)
00718         r = r.get(k,None)
00719         if c is None or r is None:
00720             break # one of the dictionaries is not deep enough
00721     return (fail_path, r, c)
00722 
00723 # signature of the print-out of the histograms
h_count_re = re.compile(r"^(.*)SUCCESS\s+Booked (\d+) Histogram\(s\) :\s+(.*)")

def GaudiTest::parseHistosSummary	(		lines,
			pos
	)

Extract the histograms infos from the lines starting at pos.
Returns the position of the first line after the summary block.

Definition at line 726 of file GaudiTest.py.

00726                                   :
00727     """
00728     Extract the histograms infos from the lines starting at pos.
00729     Returns the position of the first line after the summary block.
00730     """
00731     global h_count_re
00732     h_table_head = re.compile(r'SUCCESS\s+List of booked (1D|2D|3D|1D profile|2D profile) histograms in directory\s+"(\w*)"')
00733     h_short_summ = re.compile(r"ID=([^\"]+)\s+\"([^\"]+)\"\s+(.*)")
00734 
00735     nlines = len(lines)
00736 
00737     # decode header
00738     m = h_count_re.search(lines[pos])
00739     name = m.group(1).strip()
00740     total = int(m.group(2))
00741     header = {}
00742     for k, v in [ x.split("=") for x in  m.group(3).split() ]:
00743         header[k] = int(v)
00744     pos += 1
00745     header["Total"] = total
00746 
00747     summ = {}
00748     while pos < nlines:
00749         m = h_table_head.search(lines[pos])
00750         if m:
00751             t, d = m.groups(1) # type and directory
00752             t = t.replace(" profile", "Prof")
00753             pos += 1
00754             if pos < nlines:
00755                 l = lines[pos]
00756             else:
00757                 l = ""
00758             cont = {}
00759             if l.startswith(" | ID"):
00760                 # table format
00761                 titles = [ x.strip() for x in l.split("|")][1:]
00762                 pos += 1
00763                 while pos < nlines and lines[pos].startswith(" |"):
00764                     l = lines[pos]
00765                     values = [ x.strip() for x in l.split("|")][1:]
00766                     hcont = {}
00767                     for i in range(len(titles)):
00768                         hcont[titles[i]] = values[i]
00769                     cont[hcont["ID"]] = hcont
00770                     pos += 1
00771             elif l.startswith(" ID="):
00772                 while pos < nlines and lines[pos].startswith(" ID="):
00773                     values = [ x.strip() for x in  h_short_summ.search(lines[pos]).groups() ]
00774                     cont[values[0]] = values
00775                     pos += 1
00776             else: # not interpreted
00777                 raise RuntimeError("Cannot understand line %d: '%s'" % (pos, l))
00778             if not d in summ:
00779                 summ[d] = {}
00780             summ[d][t] = cont
00781             summ[d]["header"] = header
00782         else:
00783             break
00784     if not summ:
00785         # If the full table is not present, we use only the header
00786         summ[name] = {"header": header}
00787     return summ, pos
00788 
def findHistosSummaries(stdout):

def GaudiTest::rationalizepath

(

p

)

Definition at line 272 of file GaudiTest.py.

00272                       :
00273     p = os.path.normpath(os.path.expandvars(p))
00274     if os.path.exists(p):
00275         p = os.path.realpath(p)
00276     return p
00277 
########################################################################

def GaudiTest::which

(

executable

)

Locates an executable in the executables path ($PATH) and returns the full path to it.

If the executable cannot be found, None is returned

Definition at line 263 of file GaudiTest.py.

00263                      :
00264     if os.path.isabs(executable):
00265         return executable
00266     for d in os.environ.get("PATH").split(os.pathsep):
00267         fullpath = os.path.join(d,executable)
00268         if os.path.exists(fullpath):
00269             return fullpath
00270     return None
00271 
def rationalizepath(p):

---

Variable Documentation

string GaudiTest::__author__ = 'Marco Clemencic CERN/PH-LBC'

Definition at line 5 of file GaudiTest.py.

tuple GaudiTest::h_count_re = re.compile(r"^(.*)SUCCESS\s+Booked (\d+) Histogram\(s\) :\s+(.*)")

Definition at line 724 of file GaudiTest.py.

tuple GaudiTest::maskPointers = RegexpReplacer("0x[0-9a-fA-F]{4,16}","0x########")

Definition at line 407 of file GaudiTest.py.

tuple GaudiTest::normalizeDate

Initial value:

RegexpReplacer("[0-2]?[0-9]:[0-5][0-9]:[0-5][0-9] [0-9]{4}[-/][01][0-9][-/][0-3][0-9] *(CES?T)?",
                               "00:00:00 1970-01-01")

Definition at line 408 of file GaudiTest.py.

tuple GaudiTest::normalizeEOL = FilePreprocessor()

Definition at line 410 of file GaudiTest.py.

tuple GaudiTest::normalizeExamples = maskPointers+normalizeDate

Definition at line 433 of file GaudiTest.py.

tuple GaudiTest::skipEmptyLines = FilePreprocessor()

Definition at line 413 of file GaudiTest.py.