GaudiTest Namespace Reference

Classes
class	BasicOutputValidator
	Output Validation Classes. More...
class	BlockSkipper
class	CMT
class	FilePreprocessor
class	FilePreprocessorSequence
class	GaudiExeTest
class	GaudiFilterExecutable
class	HTMLResultStream
class	LineSkipper
class	LineSorter
class	ReferenceFileValidator
class	RegexpReplacer
class	TempDir
class	TempFile
class	TemporaryEnvironment
	Utility Classes. More...
class	XMLResultStream

Functions
def	ROOT6WorkAroundEnabled (id=None)
def	total_seconds_replacement (timedelta)
def	which (executable)
def	rationalizepath (p)
def	hexreplace (match)
def	hexConvert (char)
def	convert_xml_illegal_chars (val)
def	escape_xml_illegal_chars (val, replacement='?')
def	findReferenceBlock (reference, stdout, result, causes, signature_offset=0, signature=None, id=None)
def	countErrorLines (expected={ 'ERROR':0, 'FATAL':0}, **kwargs)
def	_parseTTreeSummary (lines, pos)
def	findTTreeSummaries (stdout)
def	cmpTreesDicts (reference, to_check, ignore=None)
def	getCmpFailingValues (reference, to_check, fail_path)
def	parseHistosSummary (lines, pos)
def	findHistosSummaries (stdout)

Variables
string	__author__ = 'Marco Clemencic CERN/PH-LBC'
	File: GaudiTest.py Author: Marco Clemencic CERN/PH-LBC. More...
	_illegal_xml_chars_RE
	maskPointers = RegexpReplacer("0x[0-9a-fA-F]{4,16}", "0x########")
	normalizeDate
	normalizeEOL = FilePreprocessor()
	__processLine__
	skipEmptyLines = FilePreprocessor()
	normalizeExamples = maskPointers + normalizeDate
	lineSkipper
	regexps
	h_count_re = re.compile(r"^(.*)SUCCESS\s+Booked (\d+) Histogram\(s\) :\s+(.*)")

Function Documentation

_parseTTreeSummary()

def GaudiTest._parseTTreeSummary (   lines,   pos  )

private

Parse the TTree summary table in lines, starting from pos.
Returns a tuple with the dictionary with the digested informations and the
position of the first line after the summary.

Definition at line 812 of file GaudiTest.py.

def _parseTTreeSummary(lines, pos):
    """
    Parse the TTree summary table in lines, starting from pos.
    Returns a tuple with the dictionary with the digested informations and the
    position of the first line after the summary.
    """
    result = {}
    i = pos + 1  # first line is a sequence of '*'
    count = len(lines)

    def splitcols(l):
        return [f.strip() for f in l.strip("*\n").split(':', 2)]

    def parseblock(ll):
        r = {}
        cols = splitcols(ll[0])
        r["Name"], r["Title"] = cols[1:]

        cols = splitcols(ll[1])
        r["Entries"] = int(cols[1])

        sizes = cols[2].split()
        r["Total size"] = int(sizes[2])
        if sizes[-1] == "memory":
            r["File size"] = 0
        else:
            r["File size"] = int(sizes[-1])

        cols = splitcols(ll[2])
        sizes = cols[2].split()
        if cols[0] == "Baskets":
            r["Baskets"] = int(cols[1])
            r["Basket size"] = int(sizes[2])
        r["Compression"] = float(sizes[-1])
        return r

    if i < (count - 3) and lines[i].startswith("*Tree"):
        result = parseblock(lines[i:i + 3])
        result["Branches"] = {}
        i += 4
        while i < (count - 3) and lines[i].startswith("*Br"):
            if i < (count - 2) and lines[i].startswith("*Branch "):
                # skip branch header
                i += 3
                continue
            branch = parseblock(lines[i:i + 3])
            result["Branches"][branch["Name"]] = branch
            i += 4

    return (result, i)

cmpTreesDicts()

def GaudiTest.cmpTreesDicts	(		reference,
			to_check,
			ignore = None
	)

Check that all the keys in reference are in to_check too, with the same value.
If the value is a dict, the function is called recursively. to_check can
contain more keys than reference, that will not be tested.
The function returns at the first difference found.

Definition at line 886 of file GaudiTest.py.

def cmpTreesDicts(reference, to_check, ignore=None):
    """
    Check that all the keys in reference are in to_check too, with the same value.
    If the value is a dict, the function is called recursively. to_check can
    contain more keys than reference, that will not be tested.
    The function returns at the first difference found.
    """
    fail_keys = []
    # filter the keys in the reference dictionary
    if ignore:
        ignore_re = re.compile(ignore)
        keys = [key for key in reference if not ignore_re.match(key)]
    else:
        keys = reference.keys()
    # loop over the keys (not ignored) in the reference dictionary
    for k in keys:
        if k in to_check:  # the key must be in the dictionary to_check
            if (type(reference[k]) is dict) and (type(to_check[k]) is dict):
                # if both reference and to_check values are dictionaries, recurse
                failed = fail_keys = cmpTreesDicts(reference[k], to_check[k],
                                                   ignore)
            else:
                # compare the two values
                failed = to_check[k] != reference[k]
        else:  # handle missing keys in the dictionary to check (i.e. failure)
            to_check[k] = None
            failed = True
        if failed:
            fail_keys.insert(0, k)
            break  # exit from the loop at the first failure
    return fail_keys  # return the list of keys bringing to the different values

convert_xml_illegal_chars()

def GaudiTest.convert_xml_illegal_chars

(

val

)

Definition at line 375 of file GaudiTest.py.

def convert_xml_illegal_chars(val):
    return _illegal_xml_chars_RE.sub(hexreplace, val)

countErrorLines()

def GaudiTest.countErrorLines	(		expected = {'ERROR': 0, 'FATAL': 0},
		**	kwargs
	)

Count the number of messages with required severity (by default ERROR and FATAL)
and check if their numbers match the expected ones (0 by default).
The dictionary "expected" can be used to tune the number of errors and fatals
allowed, or to limit the number of expected warnings etc.

Definition at line 774 of file GaudiTest.py.

def countErrorLines(expected={'ERROR': 0, 'FATAL': 0}, **kwargs):
    """
    Count the number of messages with required severity (by default ERROR and FATAL)
    and check if their numbers match the expected ones (0 by default).
    The dictionary "expected" can be used to tune the number of errors and fatals
    allowed, or to limit the number of expected warnings etc.
    """
    stdout = kwargs["stdout"]
    result = kwargs["result"]
    causes = kwargs["causes"]

    # prepare the dictionary to record the extracted lines
    errors = {}
    for sev in expected:
        errors[sev] = []

    outlines = stdout.splitlines()
    from math import log10
    fmt = "%%%dd - %%s" % (int(log10(len(outlines)) + 1))

    linecount = 0
    for l in outlines:
        linecount += 1
        words = l.split()
        if len(words) >= 2 and words[1] in errors:
            errors[words[1]].append(fmt % (linecount, l.rstrip()))

    for e in errors:
        if len(errors[e]) != expected[e]:
            causes.append('%s(%d)' % (e, len(errors[e])))
            result["GaudiTest.lines.%s" % e] = result.Quote('\n'.join(
                errors[e]))
            result["GaudiTest.lines.%s.expected#" % e] = result.Quote(
                str(expected[e]))

    return causes

escape_xml_illegal_chars()

def GaudiTest.escape_xml_illegal_chars	(		val,
			replacement = '?'
	)

Filter out characters that are illegal in XML.
Looks for any character in val that is not allowed in XML
and replaces it with replacement ('?' by default).

Definition at line 379 of file GaudiTest.py.

def escape_xml_illegal_chars(val, replacement='?'):
    """Filter out characters that are illegal in XML.
    Looks for any character in val that is not allowed in XML
    and replaces it with replacement ('?' by default).

    """
    return _illegal_xml_chars_RE.sub(replacement, val)

findHistosSummaries()

def GaudiTest.findHistosSummaries

(

stdout

)

Scan stdout to find ROOT TTree summaries and digest them.

Definition at line 1004 of file GaudiTest.py.

def findHistosSummaries(stdout):
    """
    Scan stdout to find ROOT TTree summaries and digest them.
    """
    outlines = stdout.splitlines()
    nlines = len(outlines) - 1
    summaries = {}
    global h_count_re

    pos = 0
    while pos < nlines:
        summ = {}
        # find first line of block:
        match = h_count_re.search(outlines[pos])
        while pos < nlines and not match:
            pos += 1
            match = h_count_re.search(outlines[pos])
        if match:
            summ, pos = parseHistosSummary(outlines, pos)
        summaries.update(summ)
    return summaries

findReferenceBlock()

def GaudiTest.findReferenceBlock	(		reference,
			stdout,
			result,
			causes,
			signature_offset = 0,
			signature = None,
			id = None
	)

Given a block of text, tries to find it in the output.
The block had to be identified by a signature line. By default, the first
line is used as signature, or the line pointed to by signature_offset. If
signature_offset points outside the block, a signature line can be passed as
signature argument. Note: if 'signature' is None (the default), a negative
signature_offset is interpreted as index in a list (e.g. -1 means the last
line), otherwise the it is interpreted as the number of lines before the
first one of the block the signature must appear.
The parameter 'id' allow to distinguish between different calls to this
function in the same validation code.

Definition at line 720 of file GaudiTest.py.

def findReferenceBlock(reference,
                       stdout,
                       result,
                       causes,
                       signature_offset=0,
                       signature=None,
                       id=None):
    """
    Given a block of text, tries to find it in the output.
    The block had to be identified by a signature line. By default, the first
    line is used as signature, or the line pointed to by signature_offset. If
    signature_offset points outside the block, a signature line can be passed as
    signature argument. Note: if 'signature' is None (the default), a negative
    signature_offset is interpreted as index in a list (e.g. -1 means the last
    line), otherwise the it is interpreted as the number of lines before the
    first one of the block the signature must appear.
    The parameter 'id' allow to distinguish between different calls to this
    function in the same validation code.
    """
    # split reference file, sanitize EOLs and remove empty lines
    reflines = filter(None, map(lambda s: s.rstrip(), reference.splitlines()))
    if not reflines:
        raise RuntimeError("Empty (or null) reference")
    # the same on standard output
    outlines = filter(None, map(lambda s: s.rstrip(), stdout.splitlines()))

    res_field = "GaudiTest.RefBlock"
    if id:
        res_field += "_%s" % id

    if signature is None:
        if signature_offset < 0:
            signature_offset = len(reference) + signature_offset
        signature = reflines[signature_offset]
    # find the reference block in the output file
    try:
        pos = outlines.index(signature)
        outlines = outlines[pos - signature_offset:pos + len(reflines) -
                            signature_offset]
        if reflines != outlines:
            msg = "standard output"
            # I do not want 2 messages in causes if teh function is called twice
            if not msg in causes:
                causes.append(msg)
            result[res_field + ".observed"] = result.Quote("\n".join(outlines))
    except ValueError:
        causes.append("missing signature")
    result[res_field + ".signature"] = result.Quote(signature)
    if len(reflines) > 1 or signature != reflines[0]:
        result[res_field + ".expected"] = result.Quote("\n".join(reflines))

    return causes

findTTreeSummaries()

def GaudiTest.findTTreeSummaries

(

stdout

)

Scan stdout to find ROOT TTree summaries and digest them.

Definition at line 864 of file GaudiTest.py.

def findTTreeSummaries(stdout):
    """
    Scan stdout to find ROOT TTree summaries and digest them.
    """
    stars = re.compile(r"^\*+$")
    outlines = stdout.splitlines()
    nlines = len(outlines)
    trees = {}

    i = 0
    while i < nlines:  # loop over the output
        # look for
        while i < nlines and not stars.match(outlines[i]):
            i += 1
        if i < nlines:
            tree, i = _parseTTreeSummary(outlines, i)
            if tree:
                trees[tree["Name"]] = tree

    return trees

getCmpFailingValues()

def GaudiTest.getCmpFailingValues	(		reference,
			to_check,
			fail_path
	)

Definition at line 919 of file GaudiTest.py.

def getCmpFailingValues(reference, to_check, fail_path):
    c = to_check
    r = reference
    for k in fail_path:
        c = c.get(k, None)
        r = r.get(k, None)
        if c is None or r is None:
            break  # one of the dictionaries is not deep enough
    return (fail_path, r, c)


# signature of the print-out of the histograms

hexConvert()

def GaudiTest.hexConvert

(

char

)

Definition at line 371 of file GaudiTest.py.

def hexConvert(char):
    return hex(ord(char))

hexreplace()

def GaudiTest.hexreplace

(

match

)

Definition at line 366 of file GaudiTest.py.

def hexreplace(match):
    "Return the hex string "
    return "".join(map(hexConvert, match.group()))

parseHistosSummary()

def GaudiTest.parseHistosSummary	(		lines,
			pos
	)

Extract the histograms infos from the lines starting at pos.
Returns the position of the first line after the summary block.

Definition at line 934 of file GaudiTest.py.

def parseHistosSummary(lines, pos):
    """
    Extract the histograms infos from the lines starting at pos.
    Returns the position of the first line after the summary block.
    """
    global h_count_re
    h_table_head = re.compile(
        r'SUCCESS\s+(1D|2D|3D|1D profile|2D profile) histograms in directory\s+"(\w*)"'
    )
    h_short_summ = re.compile(r"ID=([^\"]+)\s+\"([^\"]+)\"\s+(.*)")

    nlines = len(lines)

    # decode header
    m = h_count_re.search(lines[pos])
    name = m.group(1).strip()
    total = int(m.group(2))
    header = {}
    for k, v in [x.split("=") for x in m.group(3).split()]:
        header[k] = int(v)
    pos += 1
    header["Total"] = total

    summ = {}
    while pos < nlines:
        m = h_table_head.search(lines[pos])
        if m:
            t, d = m.groups(1)  # type and directory
            t = t.replace(" profile", "Prof")
            pos += 1
            if pos < nlines:
                l = lines[pos]
            else:
                l = ""
            cont = {}
            if l.startswith(" | ID"):
                # table format
                titles = [x.strip() for x in l.split("|")][1:]
                pos += 1
                while pos < nlines and lines[pos].startswith(" |"):
                    l = lines[pos]
                    values = [x.strip() for x in l.split("|")][1:]
                    hcont = {}
                    for i in range(len(titles)):
                        hcont[titles[i]] = values[i]
                    cont[hcont["ID"]] = hcont
                    pos += 1
            elif l.startswith(" ID="):
                while pos < nlines and lines[pos].startswith(" ID="):
                    values = [
                        x.strip()
                        for x in h_short_summ.search(lines[pos]).groups()
                    ]
                    cont[values[0]] = values
                    pos += 1
            else:  # not interpreted
                raise RuntimeError(
                    "Cannot understand line %d: '%s'" % (pos, l))
            if not d in summ:
                summ[d] = {}
            summ[d][t] = cont
            summ[d]["header"] = header
        else:
            break
    if not summ:
        # If the full table is not present, we use only the header
        summ[name] = {"header": header}
    return summ, pos

rationalizepath()

def GaudiTest.rationalizepath

(

p

)

Definition at line 341 of file GaudiTest.py.

def rationalizepath(p):
    np = os.path.normpath(os.path.expandvars(p))
    if os.path.exists(np):
        p = os.path.realpath(np)
    return p


# XML Escaping character

ROOT6WorkAroundEnabled()

def GaudiTest.ROOT6WorkAroundEnabled

(

id = None

)

Definition at line 37 of file GaudiTest.py.

    def ROOT6WorkAroundEnabled(id=None):
        # dummy implementation
        return False


# ensure the preferred locale
os.environ['LC_ALL'] = 'C'

# Needed for the XML wrapper
try:

total_seconds_replacement()

def GaudiTest.total_seconds_replacement

(

timedelta

)

Definition at line 54 of file GaudiTest.py.

def total_seconds_replacement(timedelta):
    return timedelta.days * 86400 + timedelta.seconds + timedelta.microseconds / 1000000

which()

def GaudiTest.which

(

executable

)

Locates an executable in the executables path ($PATH) and returns the full
path to it.  An application is looked for with or without the '.exe' suffix.
If the executable cannot be found, None is returned

Definition at line 320 of file GaudiTest.py.

def which(executable):
    """
    Locates an executable in the executables path ($PATH) and returns the full
    path to it.  An application is looked for with or without the '.exe' suffix.
    If the executable cannot be found, None is returned
    """
    if os.path.isabs(executable):
        if not os.path.exists(executable):
            if executable.endswith('.exe'):
                if os.path.exists(executable[:-4]):
                    return executable[:-4]
        return executable
    for d in os.environ.get("PATH").split(os.pathsep):
        fullpath = os.path.join(d, executable)
        if os.path.exists(fullpath):
            return fullpath
    if executable.endswith('.exe'):
        return which(executable[:-4])
    return None

Variable Documentation

__author__

string GaudiTest.__author__ = 'Marco Clemencic CERN/PH-LBC'

private

File: GaudiTest.py Author: Marco Clemencic CERN/PH-LBC.

Definition at line 16 of file GaudiTest.py.

__processLine__

GaudiTest.__processLine__

private

Definition at line 554 of file GaudiTest.py.

_illegal_xml_chars_RE

GaudiTest._illegal_xml_chars_RE

private

Initial value:

=  re.compile(
    u'[\x00-\x08\x0b\x0c\x0e-\x1F\uD800-\uDFFF\uFFFE\uFFFF]')

Definition at line 362 of file GaudiTest.py.

h_count_re

GaudiTest.h_count_re = re.compile(r"^(.*)SUCCESS\s+Booked (\d+) Histogram\(s\) :\s+(.*)")

Definition at line 931 of file GaudiTest.py.

lineSkipper

GaudiTest.lineSkipper

Definition at line 607 of file GaudiTest.py.

maskPointers

GaudiTest.maskPointers = RegexpReplacer("0x[0-9a-fA-F]{4,16}", "0x########")

Definition at line 549 of file GaudiTest.py.

normalizeDate

GaudiTest.normalizeDate

Initial value:

=  RegexpReplacer(
    "[0-2]?[0-9]:[0-5][0-9]:[0-5][0-9] [0-9]{4}[-/][01][0-9][-/][0-3][0-9] *(CES?T)?",
    "00:00:00 1970-01-01")

Definition at line 550 of file GaudiTest.py.

normalizeEOL

GaudiTest.normalizeEOL = FilePreprocessor()

Definition at line 553 of file GaudiTest.py.

normalizeExamples

tuple GaudiTest.normalizeExamples = maskPointers + normalizeDate

Definition at line 580 of file GaudiTest.py.

regexps

GaudiTest.regexps

Definition at line 672 of file GaudiTest.py.

skipEmptyLines

GaudiTest.skipEmptyLines = FilePreprocessor()

Definition at line 556 of file GaudiTest.py.