GaudiTesting.BaseTest.BaseTest Class Reference

Inheritance diagram for GaudiTesting.BaseTest.BaseTest:

Collaboration diagram for GaudiTesting.BaseTest.BaseTest:

Public Member Functions
def	__init__ (self)
def	run (self)
def	ValidateOutput (self, stdout, stderr, result)
def	findReferenceBlock (self, reference=None, stdout=None, result=None, causes=None, signature_offset=0, signature=None, id=None)
def	countErrorLines (self, expected={"ERROR":0, "FATAL":0}, stdout=None, result=None, causes=None)
def	CheckTTreesSummaries (self, stdout=None, result=None, causes=None, trees_dict=None, ignore=r"Basket|.*size|Compression")
def	CheckHistosSummaries (self, stdout=None, result=None, causes=None, dict=None, ignore=None)
def	validateWithReference (self, stdout=None, stderr=None, result=None, causes=None, preproc=None)

Public Attributes
	program
	args
	reference
	error_reference
	options
	stderr
	timeout
	exit_code
	environment
	unsupported_platforms
	signal
	workdir
	use_temp_dir
	status
	name
	causes
	result
	returnedCode
	out
	err
	proc
	stack_trace
	basedir

Private Member Functions
def	_expandReferenceFileName (self, reffile)

Static Private Attributes
	_common_tmpdir = None

Detailed Description

Definition at line 102 of file BaseTest.py.

Constructor & Destructor Documentation

__init__()

def GaudiTesting.BaseTest.BaseTest.__init__

(

self

)

Definition at line 106 of file BaseTest.py.

    def __init__(self):
        self.program = ""
        self.args = []
        self.reference = ""
        self.error_reference = ""
        self.options = ""
        self.stderr = ""
        self.timeout = 600
        self.exit_code = None
        self.environment = dict(os.environ)
        self.unsupported_platforms = []
        self.signal = None
        self.workdir = os.curdir
        self.use_temp_dir = False
        # Variables not for users
        self.status = None
        self.name = ""
        self.causes = []
        self.result = Result(self)
        self.returnedCode = 0
        self.out = ""
        self.err = ""
        self.proc = None
        self.stack_trace = None
        self.basedir = os.getcwd()
 

Member Function Documentation

_expandReferenceFileName()

def GaudiTesting.BaseTest.BaseTest._expandReferenceFileName (   self,   reffile  )

private

Definition at line 616 of file BaseTest.py.

    def _expandReferenceFileName(self, reffile):
        # if no file is passed, do nothing
        if not reffile:
            return ""
 
        # function to split an extension in constituents parts
        def platformSplit(p):
            import re
 
            delim = re.compile("-" in p and r"[-+]" or r"_")
            return set(delim.split(p))
 
        reference = os.path.normpath(
            os.path.join(self.basedir, os.path.expandvars(reffile))
        )
 
        # old-style platform-specific reference name
        spec_ref = reference[:-3] + GetPlatform(self)[0:3] + reference[-3:]
        if os.path.isfile(spec_ref):
            reference = spec_ref
        else:  # look for new-style platform specific reference files:
            # get all the files whose name start with the reference filename
            dirname, basename = os.path.split(reference)
            if not dirname:
                dirname = "."
            head = basename + "."
            head_len = len(head)
            platform = platformSplit(GetPlatform(self))
            if "do0" in platform:
                platform.add("dbg")
            candidates = []
            for f in os.listdir(dirname):
                if f.startswith(head):
                    req_plat = platformSplit(f[head_len:])
                    if platform.issuperset(req_plat):
                        candidates.append((len(req_plat), f))
            if candidates:  # take the one with highest matching
                # FIXME: it is not possible to say if x86_64-slc5-gcc43-dbg
                #        has to use ref.x86_64-gcc43 or ref.slc5-dbg
                candidates.sort()
                reference = os.path.join(dirname, candidates[-1][1])
        return reference
 
 
# ======= GAUDI TOOLS =======
 

CheckHistosSummaries()

def GaudiTesting.BaseTest.BaseTest.CheckHistosSummaries	(		self,
			stdout = None,
			result = None,
			causes = None,
			dict = None,
			ignore = None
	)

Compare the TTree summaries in stdout with the ones in trees_dict or in
the reference file. By default ignore the size, compression and basket
fields.
The presence of TTree summaries when none is expected is not a failure.

Definition at line 491 of file BaseTest.py.

    def CheckHistosSummaries(
        self, stdout=None, result=None, causes=None, dict=None, ignore=None
    ):
        """
        Compare the TTree summaries in stdout with the ones in trees_dict or in
        the reference file. By default ignore the size, compression and basket
        fields.
        The presence of TTree summaries when none is expected is not a failure.
        """
        if stdout is None:
            stdout = self.out
        if result is None:
            result = self.result
        if causes is None:
            causes = self.causes
 
        if dict is None:
            lreference = self._expandReferenceFileName(self.reference)
            # call the validator if the file exists
            if lreference and os.path.isfile(lreference):
                dict = findHistosSummaries(open(lreference).read())
            else:
                dict = {}
 
        from pprint import PrettyPrinter
 
        pp = PrettyPrinter()
        if dict:
            result["GaudiTest.Histos.expected"] = result.Quote(pp.pformat(dict))
            if ignore:
                result["GaudiTest.Histos.ignore"] = result.Quote(ignore)
 
        histos = findHistosSummaries(stdout)
        failed = cmpTreesDicts(dict, histos, ignore)
        if failed:
            causes.append("histos summaries")
            msg = "%s: %s != %s" % getCmpFailingValues(dict, histos, failed)
            result["GaudiTest.Histos.failure_on"] = result.Quote(msg)
            result["GaudiTest.Histos.found"] = result.Quote(pp.pformat(histos))
 
        return causes
 

CheckTTreesSummaries()

def GaudiTesting.BaseTest.BaseTest.CheckTTreesSummaries	(		self,
			stdout = None,
			result = None,
			causes = None,
			trees_dict = None,
			ignore = r"Basket|.*size|Compression"
	)

Compare the TTree summaries in stdout with the ones in trees_dict or in
the reference file. By default ignore the size, compression and basket
fields.
The presence of TTree summaries when none is expected is not a failure.

Definition at line 445 of file BaseTest.py.

    def CheckTTreesSummaries(
        self,
        stdout=None,
        result=None,
        causes=None,
        trees_dict=None,
        ignore=r"Basket|.*size|Compression",
    ):
        """
        Compare the TTree summaries in stdout with the ones in trees_dict or in
        the reference file. By default ignore the size, compression and basket
        fields.
        The presence of TTree summaries when none is expected is not a failure.
        """
        if stdout is None:
            stdout = self.out
        if result is None:
            result = self.result
        if causes is None:
            causes = self.causes
        if trees_dict is None:
            lreference = self._expandReferenceFileName(self.reference)
            # call the validator if the file exists
            if lreference and os.path.isfile(lreference):
                trees_dict = findTTreeSummaries(open(lreference).read())
            else:
                trees_dict = {}
 
        from pprint import PrettyPrinter
 
        pp = PrettyPrinter()
        if trees_dict:
            result["GaudiTest.TTrees.expected"] = result.Quote(pp.pformat(trees_dict))
            if ignore:
                result["GaudiTest.TTrees.ignore"] = result.Quote(ignore)
 
        trees = findTTreeSummaries(stdout)
        failed = cmpTreesDicts(trees_dict, trees, ignore)
        if failed:
            causes.append("trees summaries")
            msg = "%s: %s != %s" % getCmpFailingValues(trees_dict, trees, failed)
            result["GaudiTest.TTrees.failure_on"] = result.Quote(msg)
            result["GaudiTest.TTrees.found"] = result.Quote(pp.pformat(trees))
 
        return causes
 

countErrorLines()

def GaudiTesting.BaseTest.BaseTest.countErrorLines	(		self,
			expected = {"ERROR": 0, "FATAL": 0},
			stdout = None,
			result = None,
			causes = None
	)

Count the number of messages with required severity (by default ERROR and FATAL)
and check if their numbers match the expected ones (0 by default).
The dictionary "expected" can be used to tune the number of errors and fatals
allowed, or to limit the number of expected warnings etc.

Definition at line 401 of file BaseTest.py.

    def countErrorLines(
        self, expected={"ERROR": 0, "FATAL": 0}, stdout=None, result=None, causes=None
    ):
        """
        Count the number of messages with required severity (by default ERROR and FATAL)
        and check if their numbers match the expected ones (0 by default).
        The dictionary "expected" can be used to tune the number of errors and fatals
        allowed, or to limit the number of expected warnings etc.
        """
 
        if stdout is None:
            stdout = self.out
        if result is None:
            result = self.result
        if causes is None:
            causes = self.causes
 
        # prepare the dictionary to record the extracted lines
        errors = {}
        for sev in expected:
            errors[sev] = []
 
        outlines = stdout.splitlines()
        from math import log10
 
        fmt = "%%%dd - %%s" % (int(log10(len(outlines) + 1)))
 
        linecount = 0
        for l in outlines:
            linecount += 1
            words = l.split()
            if len(words) >= 2 and words[1] in errors:
                errors[words[1]].append(fmt % (linecount, l.rstrip()))
 
        for e in errors:
            if len(errors[e]) != expected[e]:
                causes.append("%s(%d)" % (e, len(errors[e])))
                result["GaudiTest.lines.%s" % e] = result.Quote("\n".join(errors[e]))
                result["GaudiTest.lines.%s.expected#" % e] = result.Quote(
                    str(expected[e])
                )
 
        return causes
 

findReferenceBlock()

def GaudiTesting.BaseTest.BaseTest.findReferenceBlock	(		self,
			reference = None,
			stdout = None,
			result = None,
			causes = None,
			signature_offset = 0,
			signature = None,
			id = None
	)

Given a block of text, tries to find it in the output. The block had to be identified by a signature line. By default, the first line is used as signature, or the line pointed to by signature_offset. If signature_offset points outside the block, a signature line can be passed as signature argument. Note: if 'signature' is None (the default), a negative signature_offset is interpreted as index in a list (e.g. -1 means the last line), otherwise the it is interpreted as the number of lines before the first one of the block the signature must appear. The parameter 'id' allow to distinguish between different calls to this function in the same validation code.

Definition at line 344 of file BaseTest.py.

    def findReferenceBlock(
        self,
        reference=None,
        stdout=None,
        result=None,
        causes=None,
        signature_offset=0,
        signature=None,
        id=None,
    ):
        """
        Given a block of text, tries to find it in the output. The block had to be identified by a signature line. By default, the first line is used as signature, or the line pointed to by signature_offset. If signature_offset points outside the block, a signature line can be passed as signature argument. Note: if 'signature' is None (the default), a negative signature_offset is interpreted as index in a list (e.g. -1 means the last line), otherwise the it is interpreted as the number of lines before the first one of the block the signature must appear. The parameter 'id' allow to distinguish between different calls to this function in the same validation code.
        """
 
        if reference is None:
            reference = self.reference
        if stdout is None:
            stdout = self.out
        if result is None:
            result = self.result
        if causes is None:
            causes = self.causes
 
        reflines = list(filter(None, map(lambda s: s.rstrip(), reference.splitlines())))
        if not reflines:
            raise RuntimeError("Empty (or null) reference")
        # the same on standard output
        outlines = list(filter(None, map(lambda s: s.rstrip(), stdout.splitlines())))
 
        res_field = "GaudiTest.RefBlock"
        if id:
            res_field += "_%s" % id
 
        if signature is None:
            if signature_offset < 0:
                signature_offset = len(reference) + signature_offset
            signature = reflines[signature_offset]
        # find the reference block in the output file
        try:
            pos = outlines.index(signature)
            outlines = outlines[
                pos - signature_offset : pos + len(reflines) - signature_offset
            ]
            if reflines != outlines:
                msg = "standard output"
                # I do not want 2 messages in causes if the function is called
                # twice
                if not msg in causes:
                    causes.append(msg)
                result[res_field + ".observed"] = result.Quote("\n".join(outlines))
        except ValueError:
            causes.append("missing signature")
        result[res_field + ".signature"] = result.Quote(signature)
        if len(reflines) > 1 or signature != reflines[0]:
            result[res_field + ".expected"] = result.Quote("\n".join(reflines))
        return causes
 

run()

def GaudiTesting.BaseTest.BaseTest.run

(

self

)

Definition at line 132 of file BaseTest.py.

    def run(self):
        logging.debug("running test %s", self.name)
 
        self.result = Result(
            {
                "CAUSE": None,
                "EXCEPTION": None,
                "RESOURCE": None,
                "TARGET": None,
                "TRACEBACK": None,
                "START_TIME": None,
                "END_TIME": None,
                "TIMEOUT_DETAIL": None,
            }
        )
 
        if self.options:
            if re.search(
                r"from\s+Gaudi.Configuration\s+import\s+\*|"
                "from\s+Configurables\s+import",
                self.options,
            ):
                suffix, lang = ".py", "python"
            else:
                suffix, lang = ".opts", "c++"
            self.result["Options"] = '<code lang="{}"><pre>{}</pre></code>'.format(
                lang, escape_for_html(self.options)
            )
            optionFile = tempfile.NamedTemporaryFile(suffix=suffix)
            optionFile.file.write(self.options.encode("utf-8"))
            optionFile.seek(0)
            self.args.append(RationalizePath(optionFile.name))
 
        platform_id = (
            self.environment.get("BINARY_TAG")
            or self.environment.get("CMTCONFIG")
            or platform.platform()
        )
        # If at least one regex matches we skip the test.
        skip_test = bool(
            [
                None
                for prex in self.unsupported_platforms
                if re.search(prex, platform_id)
            ]
        )
 
        if not skip_test:
            # handle working/temporary directory options
            workdir = self.workdir
            if self.use_temp_dir:
                if self._common_tmpdir:
                    workdir = self._common_tmpdir
                else:
                    workdir = tempfile.mkdtemp()
 
            # prepare the command to execute
            prog = ""
            if self.program != "":
                prog = self.program
            elif "GAUDIEXE" in self.environment:
                prog = self.environment["GAUDIEXE"]
            else:
                prog = "Gaudi.exe"
 
            prog_ext = os.path.splitext(prog)[1]
            if prog_ext not in [".exe", ".py", ".bat"]:
                prog += ".exe"
                prog_ext = ".exe"
 
            prog = which(prog) or prog
 
            args = list(map(RationalizePath, self.args))
 
            if prog_ext == ".py":
                params = ["python", RationalizePath(prog)] + args
            else:
                params = [RationalizePath(prog)] + args
 
            # we need to switch directory because the validator expects to run
            # in the same dir as the program
            os.chdir(workdir)
 
            # launching test in a different thread to handle timeout exception
            def target():
                logging.debug("executing %r in %s", params, workdir)
                self.proc = Popen(
                    params, stdout=PIPE, stderr=PIPE, env=self.environment
                )
                logging.debug("(pid: %d)", self.proc.pid)
                out, err = self.proc.communicate()
                self.out = out.decode("utf-8", errors="backslashreplace")
                self.err = err.decode("utf-8", errors="backslashreplace")
 
            thread = threading.Thread(target=target)
            thread.start()
            # catching timeout
            thread.join(self.timeout)
 
            if thread.is_alive():
                logging.debug("time out in test %s (pid %d)", self.name, self.proc.pid)
                # get the stack trace of the stuck process
                cmd = [
                    "gdb",
                    "--pid",
                    str(self.proc.pid),
                    "--batch",
                    "--eval-command=thread apply all backtrace",
                ]
                gdb = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=STDOUT)
                self.stack_trace = gdb.communicate()[0].decode(
                    "utf-8", errors="backslashreplace"
                )
 
                kill_tree(self.proc.pid, signal.SIGTERM)
                thread.join(60)
                if thread.is_alive():
                    kill_tree(self.proc.pid, signal.SIGKILL)
                self.causes.append("timeout")
            else:
                self.returnedCode = self.proc.returncode
                if self.returnedCode != SKIP_RETURN_CODE:
                    logging.debug(
                        f"completed test {self.name} with returncode = {self.returnedCode}"
                    )
                    logging.debug("validating test...")
                    self.result, self.causes = self.ValidateOutput(
                        stdout=self.out, stderr=self.err, result=self.result
                    )
                else:
                    logging.debug(f"skipped test {self.name}")
                    self.status = "skipped"
 
            # remove the temporary directory if we created it
            if self.use_temp_dir and not self._common_tmpdir:
                shutil.rmtree(workdir, True)
 
            os.chdir(self.basedir)
 
            if self.status != "skipped":
                # handle application exit code
                if self.signal is not None:
                    if int(self.returnedCode) != -int(self.signal):
                        self.causes.append("exit code")
 
                elif self.exit_code is not None:
                    if int(self.returnedCode) != int(self.exit_code):
                        self.causes.append("exit code")
 
                elif self.returnedCode != 0:
                    self.causes.append("exit code")
 
                if self.causes:
                    self.status = "failed"
                else:
                    self.status = "passed"
 
        else:
            self.status = "skipped"
 
        logging.debug("%s: %s", self.name, self.status)
        field_mapping = {
            "Exit Code": "returnedCode",
            "stderr": "err",
            "Arguments": "args",
            "Runtime Environment": "environment",
            "Status": "status",
            "stdout": "out",
            "Program Name": "program",
            "Name": "name",
            "Validator": "validator",
            "Output Reference File": "reference",
            "Error Reference File": "error_reference",
            "Causes": "causes",
            # 'Validator Result': 'result.annotations',
            "Unsupported Platforms": "unsupported_platforms",
            "Stack Trace": "stack_trace",
        }
        resultDict = [
            (key, getattr(self, attr))
            for key, attr in field_mapping.items()
            if getattr(self, attr)
        ]
        resultDict.append(
            (
                "Working Directory",
                RationalizePath(os.path.join(os.getcwd(), self.workdir)),
            )
        )
        # print(dict(resultDict).keys())
        resultDict.extend(self.result.annotations.items())
        # print(self.result.annotations.keys())
        resultDict = dict(resultDict)
 
        # Special cases
        if "Validator" in resultDict:
            resultDict["Validator"] = '<code lang="{}"><pre>{}</pre></code>'.format(
                "python", escape_for_html(resultDict["Validator"])
            )
        return resultDict
 

ValidateOutput()

def GaudiTesting.BaseTest.BaseTest.ValidateOutput	(		self,
			stdout,
			stderr,
			result
	)

Reimplemented in GaudiTesting.QMTTest.QMTTest.

Definition at line 337 of file BaseTest.py.

    def ValidateOutput(self, stdout, stderr, result):
        if not self.stderr:
            self.validateWithReference(stdout, stderr, result, self.causes)
        elif stderr.strip() != self.stderr.strip():
            self.causes.append("standard error")
        return result, self.causes
 

validateWithReference()

def GaudiTesting.BaseTest.BaseTest.validateWithReference	(		self,
			stdout = None,
			stderr = None,
			result = None,
			causes = None,
			preproc = None
	)

Default validation acti*on: compare standard output and error to the
reference files.

Definition at line 533 of file BaseTest.py.

    def validateWithReference(
        self, stdout=None, stderr=None, result=None, causes=None, preproc=None
    ):
        """
        Default validation acti*on: compare standard output and error to the
        reference files.
        """
 
        if stdout is None:
            stdout = self.out
        if stderr is None:
            stderr = self.err
        if result is None:
            result = self.result
        if causes is None:
            causes = self.causes
 
        # set the default output preprocessor
        if preproc is None:
            preproc = normalizeExamples
        # check standard output
        lreference = self._expandReferenceFileName(self.reference)
        # call the validator if the file exists
        if lreference and os.path.isfile(lreference):
            causes += ReferenceFileValidator(
                lreference, "standard output", "Output Diff", preproc=preproc
            )(stdout, result)
        elif lreference:
            causes += ["missing reference file"]
        # Compare TTree summaries
        causes = self.CheckTTreesSummaries(stdout, result, causes)
        causes = self.CheckHistosSummaries(stdout, result, causes)
        if causes and lreference:  # Write a new reference file for stdout
            try:
                cnt = 0
                newrefname = ".".join([lreference, "new"])
                while os.path.exists(newrefname):
                    cnt += 1
                    newrefname = ".".join([lreference, "~%d~" % cnt, "new"])
                newref = open(newrefname, "w")
                # sanitize newlines
                for l in stdout.splitlines():
                    newref.write(l.rstrip() + "\n")
                del newref  # flush and close
                result["New Output Reference File"] = os.path.relpath(
                    newrefname, self.basedir
                )
            except IOError:
                # Ignore IO errors when trying to update reference files
                # because we may be in a read-only filesystem
                pass
 
        # check standard error
        lreference = self._expandReferenceFileName(self.error_reference)
        # call the validator if we have a file to use
        if lreference:
            if os.path.isfile(lreference):
                newcauses = ReferenceFileValidator(
                    lreference, "standard error", "Error Diff", preproc=preproc
                )(stderr, result)
            else:
                newcauses = ["missing error reference file"]
            causes += newcauses
            if newcauses and lreference:  # Write a new reference file for stdedd
                cnt = 0
                newrefname = ".".join([lreference, "new"])
                while os.path.exists(newrefname):
                    cnt += 1
                    newrefname = ".".join([lreference, "~%d~" % cnt, "new"])
                newref = open(newrefname, "w")
                # sanitize newlines
                for l in stderr.splitlines():
                    newref.write(l.rstrip() + "\n")
                del newref  # flush and close
                result["New Error Reference File"] = os.path.relpath(
                    newrefname, self.basedir
                )
        else:
            causes += BasicOutputValidator(
                lreference, "standard error", "ExecTest.expected_stderr"
            )(stderr, result)
        return causes
 

Member Data Documentation

_common_tmpdir

GaudiTesting.BaseTest.BaseTest._common_tmpdir = None

staticprivate

Definition at line 104 of file BaseTest.py.

args

GaudiTesting.BaseTest.BaseTest.args

Definition at line 108 of file BaseTest.py.

basedir

GaudiTesting.BaseTest.BaseTest.basedir

Definition at line 130 of file BaseTest.py.

causes

GaudiTesting.BaseTest.BaseTest.causes

Definition at line 123 of file BaseTest.py.

environment

GaudiTesting.BaseTest.BaseTest.environment

Definition at line 115 of file BaseTest.py.

err

GaudiTesting.BaseTest.BaseTest.err

Definition at line 127 of file BaseTest.py.

error_reference

GaudiTesting.BaseTest.BaseTest.error_reference

Definition at line 110 of file BaseTest.py.

exit_code

GaudiTesting.BaseTest.BaseTest.exit_code

Definition at line 114 of file BaseTest.py.

name

GaudiTesting.BaseTest.BaseTest.name

Definition at line 122 of file BaseTest.py.

options

GaudiTesting.BaseTest.BaseTest.options

Definition at line 111 of file BaseTest.py.

out

GaudiTesting.BaseTest.BaseTest.out

Definition at line 126 of file BaseTest.py.

proc

GaudiTesting.BaseTest.BaseTest.proc

Definition at line 128 of file BaseTest.py.

program

GaudiTesting.BaseTest.BaseTest.program

Definition at line 107 of file BaseTest.py.

reference

GaudiTesting.BaseTest.BaseTest.reference

Definition at line 109 of file BaseTest.py.

result

GaudiTesting.BaseTest.BaseTest.result

Definition at line 124 of file BaseTest.py.

returnedCode

GaudiTesting.BaseTest.BaseTest.returnedCode

Definition at line 125 of file BaseTest.py.

signal

GaudiTesting.BaseTest.BaseTest.signal

Definition at line 117 of file BaseTest.py.

stack_trace

GaudiTesting.BaseTest.BaseTest.stack_trace

Definition at line 129 of file BaseTest.py.

status

GaudiTesting.BaseTest.BaseTest.status

Definition at line 121 of file BaseTest.py.

stderr

GaudiTesting.BaseTest.BaseTest.stderr

Definition at line 112 of file BaseTest.py.

timeout

GaudiTesting.BaseTest.BaseTest.timeout

Definition at line 113 of file BaseTest.py.

unsupported_platforms

GaudiTesting.BaseTest.BaseTest.unsupported_platforms

Definition at line 116 of file BaseTest.py.

use_temp_dir

GaudiTesting.BaseTest.BaseTest.use_temp_dir

Definition at line 119 of file BaseTest.py.

workdir

GaudiTesting.BaseTest.BaseTest.workdir

Definition at line 118 of file BaseTest.py.

---

The documentation for this class was generated from the following file:

• GaudiPolicy/python/GaudiTesting/BaseTest.py