BaseTest.py

Go to the documentation of this file.

# -*- coding: utf-8 -*-

import subprocess
import time
from subprocess import *
import os
import signal
import threading
import sys
import platform
import tempfile
import inspect
import re
import xml.sax.saxutils as XSS

#-------------------------------------------------------------------------#
class BaseTest :
    def __init__(self):
        self.program=''
        self.args=[]
        self.reference=''
        self.error_reference=''
        self.options=''
        self.stderr=''
        self.timeout=120
        self.exit_code=None
        self.environment=None
        self.target='local'
        self.traceback=True
        self.unsupported_platforms=[]
        self.signal=None
        #Variables not for users
        self.status=None
        self.name=''
        self.causes = []
        self.timeOut=False
        self.result = Result(self)
        self.returnedCode=0
        self.out = ''
        self.err = ''
        self.proc=None

    def validator(self, stdout='',stderr=''):
        pass

    def runTest (self):
        #Setting time
        begining = int(time.time())
        DOB = time.localtime()
        dateOfBegining = str(DOB[1])+"/"+str(DOB[2])+"/"+str(DOB[0])+" "+str(DOB[3])+":"+str(DOB[4])+":"+str(DOB[5])


        if self.options != '' :
            if self.options.startswith("import") or self.options.startswith("from") or self.options.startswith("\nfrom") or self.options.startswith("\nimport") or (self.options.startswith("\n#") and not self.options.startswith('\n#include')):
                optionFile = tempfile.NamedTemporaryFile(suffix='.py')
            else :
                optionFile = tempfile.NamedTemporaryFile(suffix='.opts')
            optionFile.file.write(self.options)
            optionFile.seek(0)
            self.args.append(RationalizePath(optionFile.name))

        #If not specified, setting the environment
        if self.environment is None : self.environment = os.environ
        else : self.environment=dict(self.environment.items()+os.environ.items())

        #launching test in a different thread to handle timeout exception
        def run (self=self):
            def target (self=self) :
                prog=''
                if self.program != '' :
                    prog = RationalizePath(self.program)
                elif "GAUDIEXE" in os.environ :
                    prog = os.environ["GAUDIEXE"]
                else :
                    prog = "Gaudi.exe"

                dummy, prog_ext = os.path.splitext(prog)
                if prog_ext not in [ ".exe", ".py", ".bat" ]:
                    prog += ".exe"
                    prog_ext = ".exe"

                prog = which(prog) or prog

                if prog_ext == ".py" :
                    params = ["/afs/cern.ch/user/v/valentin/workspace/Gaudi/build.x86_64-slc6-gcc48-opt/run","/afs/cern.ch/sw/lcg/external/Python/2.7.3/x86_64-slc6-gcc48-opt/bin/python",RationalizePath(prog)]+self.args
                else :
                    params = ["/afs/cern.ch/user/v/valentin/workspace/Gaudi/build.x86_64-slc6-gcc48-opt/run", RationalizePath(prog)]+self.args

                print self.name
                print params
                self.proc= Popen(params,stdout=PIPE,stderr=PIPE, env=self.environment)
                self.proc.wait()

            thread = threading.Thread(target=target)
            thread.start()
            #catching timeout
            thread.join(self.timeout)

            if thread.is_alive():
                self.proc.send_signal(signal.SIGABRT)
                print "abortion"
                thread.join()
                self.timeOut = True
            self.out = self.proc.stdout.read()
            if self.traceback:
                self.err = self.proc.stderr.read()
            #Getting the error code
            self.returnedCode = self.proc.returncode

        unsupPlat = True
        for p in self.unsupported_platforms :
            if re.search(p,platform.platform()):
                unsupPlat=False

        if unsupPlat :
            run()

        #End time
        end=time.time()
        #Time lasted
        lasted = end-begining
        #Getting date and local time
        DOE = time.localtime()
        dateOfEnding = str(DOE[1])+"/"+str(DOE[2])+"/"+str(DOE[0])+" "+str(DOE[3])+":"+str(DOE[4])+":"+str(DOE[5])

        if unsupPlat :
            validatorRes = Result({'CAUSE':None,'EXCEPTION':None,'RESOURCE':None,'TARGET':None,'TRACEBACK':None,'START_TIME':None,'END_TIME':None,'TIMEOUT_DETAIL':None})
            self.result=validatorRes

            if self.timeOut:
                self.result["Abort cause"]=self.result.Quote("Event Timeout")

            self.result,self.causes=self.ValidateOutput(stdout=self.out,stderr=self.err,result=validatorRes)

            #Setting status
            self.status = "Failed"
            if self.signal is not None :
                if (int(self.returnedCode) - int(self.signal) - 128)!=0:
                    self.causes.append('wrong return code')
            if self.exit_code is not None:
                if int(self.returnedCode) != int(self.exit_code) :
                    self.causes.append('wrong return code')
            if self.returnedCode!=0 and self.exit_code is None and self.signal is None:
                self.causes.append("Return code !=0")
            if self.causes == []:
                self.status = "Passed"
        else :
            self.status="SKIPPED"

        resultDic = {'Execution Time':lasted, 'Exit code':self.returnedCode, 'Start Time':dateOfBegining, 'End Time':dateOfEnding, 'Stderr':self.err,'Arguments':self.args, 'Environment':self.environment, 'Expected stderr':self.stderr, 'Status':self.status,  'Measurement':self.out, 'Program Name':self.program,'Name':self.name, 'Validator':self.validator,'Reference file':self.reference,'Error reference file':self.error_reference,'Causes':self.causes,'Validator results':self.result.annotations,'Unsupported platforms':self.unsupported_platforms}
        return resultDic


    #-------------------------------------------------#
    #----------------Validating tool------------------#
    #-------------------------------------------------#

    def ValidateOutput(self, stdout, stderr, result):
        causes = self.causes
        reference =self.reference
        error_reference=self.error_reference
        #checking if default validation or not
        if inspect.getsource(self.validator)!="""    def validator(self, stdout='',stderr=''):
        pass
""" :
            self.validator(stdout, stderr, result, causes, reference, error_reference)
        else:
            if self.stderr=='':
                self.validateWithReference(stdout, stderr, result, causes)
            elif stderr!=self.stderr:
                self.causes.append("DIFFERENT STDERR THAN EXPECTED")


        return result,causes



    def findReferenceBlock(self,reference=None, stdout=None, result=None, causes=None, signature_offset=0, signature=None, id = None):
        """
            Given a block of text, tries to find it in the output. The block had to be identified by a signature line. By default, the first line is used as signature, or the line pointed to by signature_offset. If signature_offset points outside the block, a signature line can be passed as signature argument. Note: if 'signature' is None (the default), a negative signature_offset is interpreted as index in a list (e.g. -1 means the last line), otherwise the it is interpreted as the number of lines before the first one of the block the signature must appear. The parameter 'id' allow to distinguish between different calls to this function in the same validation code.
            """

        if reference is None : reference=self.reference
        if stdout is None : stdout=self.out
        if result is None : result=self.result
        if causes is None : causes=self.causes

        reflines = filter(None,map(lambda s: s.rstrip(), reference.splitlines()))
        if not reflines:
            raise RuntimeError("Empty (or null) reference")
        # the same on standard output
        outlines = filter(None,map(lambda s: s.rstrip(), stdout.splitlines()))

        res_field = "GaudiTest.RefBlock"
        if id:
            res_field += "_%s" % id

        if signature is None:
            if signature_offset < 0:
                signature_offset = len(reference)+signature_offset
            signature = reflines[signature_offset]
        # find the reference block in the output file
        try:
            pos = outlines.index(signature)
            outlines = outlines[pos-signature_offset:pos+len(reflines)-signature_offset]
            if reflines != outlines:
                msg = "standard output"
                # I do not want 2 messages in causes if teh function is called twice
                if not msg in causes:
                    causes.append(msg)
                result[res_field + ".observed"] = result.Quote("\n".join(outlines))
        except ValueError:
            causes.append("missing signature")
        result[res_field + ".signature"] = result.Quote(signature)
        if len(reflines) > 1 or signature != reflines[0]:
            result[res_field + ".expected"] = result.Quote("\n".join(reflines))
        return causes

    def countErrorLines(self, expected = {'ERROR':0, 'FATAL':0}, stdout=None, result=None,causes=None):
        """
            Count the number of messages with required severity (by default ERROR and FATAL)
            and check if their numbers match the expected ones (0 by default).
            The dictionary "expected" can be used to tune the number of errors and fatals
            allowed, or to limit the number of expected warnings etc.
            """

        if stdout is None : stdout=self.out
        if result is None : result=self.result
        if causes is None : causes=self.causes

        # prepare the dictionary to record the extracted lines
        errors = {}
        for sev in expected:
            errors[sev] = []

        outlines = stdout.splitlines()
        from math import log10
        fmt = "%%%dd - %%s" % (int(log10(len(outlines)+1)))

        linecount = 0
        for l in outlines:
            linecount += 1
            words = l.split()
            if len(words) >= 2 and words[1] in errors:
                errors[words[1]].append(fmt%(linecount,l.rstrip()))

        for e in errors:
            if len(errors[e]) != expected[e]:
                causes.append('%s(%d)'%(e,len(errors[e])))
                result["GaudiTest.lines.%s"%e] = result.Quote('\n'.join(errors[e]))
                result["GaudiTest.lines.%s.expected#"%e] = result.Quote(str(expected[e]))

        return causes

    def CheckTTreesSummaries(self, stdout=None, result=None, causes=None,
                             trees_dict = None,
                             ignore = r"Basket|.*size|Compression"):
        """
            Compare the TTree summaries in stdout with the ones in trees_dict or in
            the reference file. By default ignore the size, compression and basket
            fields.
            The presence of TTree summaries when none is expected is not a failure.
            """
        if stdout is None : stdout=self.out
        if result is None : result=self.result
        if causes is None : causes=self.causes
        if trees_dict is None:
            lreference = _expandReferenceFileName(self,self.reference)
            # call the validator if the file exists
            if lreference and os.path.isfile(lreference):
                trees_dict = findTTreeSummaries(open(lreference).read())
            else:
                trees_dict = {}

        from pprint import PrettyPrinter
        pp = PrettyPrinter()
        if trees_dict:
            result["GaudiTest.TTrees.expected"] = result.Quote(pp.pformat(trees_dict))
            if ignore:
                result["GaudiTest.TTrees.ignore"] = result.Quote(ignore)

        trees = findTTreeSummaries(stdout)
        failed = cmpTreesDicts(trees_dict, trees, ignore)
        if failed:
            causes.append("trees summaries")
            msg = "%s: %s != %s" % getCmpFailingValues(trees_dict, trees, failed)
            result["GaudiTest.TTrees.failure_on"] = result.Quote(msg)
            result["GaudiTest.TTrees.found"] = result.Quote(pp.pformat(trees))

        return causes

    def CheckHistosSummaries(self, stdout=None, result=None, causes=None,
                             dict = None,
                             ignore = None):
        """
            Compare the TTree summaries in stdout with the ones in trees_dict or in
            the reference file. By default ignore the size, compression and basket
            fields.
            The presence of TTree summaries when none is expected is not a failure.
            """
        if stdout is None : stdout=self.out
        if result is None : result=self.result
        if causes is None : causes=self.causes

        if dict is None:
            lreference = _expandReferenceFileName(self,self.reference)
            # call the validator if the file exists
            if lreference and os.path.isfile(lreference):
                dict = findHistosSummaries(open(lreference).read())
            else:
                dict = {}

        from pprint import PrettyPrinter
        pp = PrettyPrinter()
        if dict:
            result["GaudiTest.Histos.expected"] = result.Quote(pp.pformat(dict))
            if ignore:
                result["GaudiTest.Histos.ignore"] = result.Quote(ignore)

        histos = findHistosSummaries(stdout)
        failed = cmpTreesDicts(dict, histos, ignore)
        if failed:
            causes.append("histos summaries")
            msg = "%s: %s != %s" % getCmpFailingValues(dict, histos, failed)
            result["GaudiTest.Histos.failure_on"] = result.Quote(msg)
            result["GaudiTest.Histos.found"] = result.Quote(pp.pformat(histos))

        return causes

    def validateWithReference(self, stdout=None, stderr=None, result=None, causes=None, preproc = None):
        """
            Default validation action: compare standard output and error to the
            reference files.
            """

        if stdout is None : stdout=self.out
        if stderr is None : stderr=self.err
        if result is None : result=self.result
        if causes is None : causes=self.causes

        # set the default output preprocessor
        if preproc is None:
            preproc = normalizeExamples
        # check standard output
        lreference = _expandReferenceFileName(self,self.reference)
        # call the validator if the file exists
        if lreference and os.path.isfile(lreference):
            result["GaudiTest.output_reference"] = lreference
            causes += ReferenceFileValidator(lreference,
                                             "standard output",
                                             "GaudiTest.output_diff",
                                             preproc = preproc)(stdout, result)
        # Compare TTree summaries
        causes = self.CheckTTreesSummaries(stdout, result, causes)
        causes = self.CheckHistosSummaries(stdout, result, causes)
        if causes: # Write a new reference file for stdout
            try:
                newref = open(lreference + ".new","w")
                # sanitize newlines
                for l in stdout.splitlines():
                    newref.write(l.rstrip() + '\n')
                del newref # flush and close
            except IOError:
                # Ignore IO errors when trying to update reference files
                # because we may be in a read-only filesystem
                pass

        # check standard error
        lreference = _expandReferenceFileName(self,self.error_reference)
        # call the validator if we have a file to use
        if lreference and os.path.isfile(self.error_reference):
            result["GaudiTest.error_reference"] = self.error_reference
            newcauses = ReferenceFileValidator(lreference, "standard error", "GaudiTest.error_diff", preproc = preproc)(stderr, result)
            causes += newcauses
            if newcauses: # Write a new reference file for stdedd
                newref = open(self.reference + ".new","w")
                # sanitize newlines
                for l in stderr.splitlines():
                    newref.write(l.rstrip() + '\n')
                del newref # flush and close
        else:
            causes += BasicOutputValidator(lreference, "standard error", "ExecTest.expected_stderr")(stderr, result)
        return causes

#---------------------------------------------------------------------------------------------------#
#---------------------------------------------------------------------------------------------------#
#-----------------------------------------GAUDI TOOLS-----------------------------------------------------#
#---------------------------------------------------------------------------------------------------#
#---------------------------------------------------------------------------------------------------#

import os
import shutil
import string
import difflib
import calendar
from subprocess import Popen, PIPE, STDOUT
import re
import xml.etree.ElementTree as ET

try:
    from GaudiKernel import ROOT6WorkAroundEnabled
except ImportError:
    def ROOT6WorkAroundEnabled(id=None):
        # dummy implementation
        return False

if sys.platform == "win32":
    import msvcrt
    import pywintypes
    from   threading import *
    import win32api
    import win32con
    import win32event
    import win32file
    import win32pipe
    import win32process
else:
    import cPickle
    import fcntl
    import select


#--------------------------------- TOOLS ---------------------------------#

def RationalizePath(p) :
    """
    Function used to normalize the used path
    """
    newPath = os.path.normpath(os.path.expandvars(p))
    if os.path.exists(newPath) :
        p = os.path.realpath(newPath)
    p = os.path.realpath(newPath)
    return p


def which(executable):
    """
    Locates an executable in the executables path ($PATH) and returns the full
    path to it.  An application is looked for with or without the '.exe' suffix.
    If the executable cannot be found, None is returned
    """
    if os.path.isabs(executable):
        if not os.path.exists(executable):
            if executable.endswith('.exe'):
                if os.path.exists(executable[:-4]):
                    return executable[:-4]
            else :
                head,executable = os.path.split(executable)
        else :
            return executable
    for d in os.environ.get("PATH").split(os.pathsep):
        fullpath = os.path.join(d, executable)
        if os.path.exists(fullpath):
            return fullpath
    if executable.endswith('.exe'):
        return which(executable[:-4])
    return None



#-------------------------------------------------------------------------#
#----------------------------- Result Classe -----------------------------#
#-------------------------------------------------------------------------#
import types

class Result:

    PASS='PASS'
    FAIL='FAIL'
    ERROR='ERROR'
    UNTESTED='UNTESTED'

    EXCEPTION = ""
    RESOURCE = ""
    TARGET = ""
    TRACEBACK = ""
    START_TIME = ""
    END_TIME = ""
    TIMEOUT_DETAIL = ""

    def __init__(self,kind=None,id=None,outcome=PASS,annotations={}):
        self.annotations = annotations.copy()

    def __getitem__(self,key):
        assert type(key) in types.StringTypes
        return self.annotations[key]

    def __setitem__(self,key,value):
        assert type(key) in types.StringTypes
        assert type(value) in types.StringTypes
        self.annotations[key]=value

    def Quote(self,string):
        return string


#-------------------------------------------------------------------------#
#--------------------------- Validator Classes ---------------------------#
#-------------------------------------------------------------------------#

#Basic implementation of an option validator for Gaudi test. This implementation is based on the standard (LCG) validation functions used in QMTest.


class BasicOutputValidator:

    def __init__(self,ref,cause,result_key):
        self.ref=ref
        self.cause=cause
        self.result_key=result_key

    def __call__(self,out,result):
        """Validate the output of the program.
            'stdout' -- A string containing the data written to the standard output
            stream.
            'stderr' -- A string containing the data written to the standard error
            stream.
            'result' -- A 'Result' object. It may be used to annotate
            the outcome according to the content of stderr.
            returns -- A list of strings giving causes of failure."""

        causes=[]
        #Check the output
        if not self.__CompareText(out,self.ref):
            causes.append(self.cause)
            result[self.result_key] =result.Quote(self.ref)



        return causes

    def __CompareText(self, s1, s2):
        """Compare 's1' and 's2', ignoring line endings.
            's1' -- A string.
            's2' -- A string.
            returns -- True if 's1' and 's2' are the same, ignoring
            differences in line endings."""
        if ROOT6WorkAroundEnabled('ReadRootmapCheck'):
            # FIXME: (MCl) Hide warnings from new rootmap sanity check until we can fix them
            to_ignore = re.compile(r'Warning in <TInterpreter::ReadRootmapFile>: .* is already in .*')
            keep_line = lambda l: not to_ignore.match(l)
            return filter(keep_line, s1.splitlines()) == filter(keep_line, s2.splitlines())
        else:
            return s1.splitlines() == s2.splitlines()



#------------------------ Preprocessor elements ------------------------#
class FilePreprocessor:
    """ Base class for a callable that takes a file and returns a modified
        version of it."""
    def __processLine__(self, line):
        return line
    def __call__(self, input):
        if hasattr(input,"__iter__"):
            lines = input
            mergeback = False
        else:
            lines = input.splitlines()
            mergeback = True
        output = []
        for l in lines:
            l = self.__processLine__(l)
            if l: output.append(l)
        if mergeback: output = '\n'.join(output)
        return output
    def __add__(self, rhs):
        return FilePreprocessorSequence([self,rhs])

class FilePreprocessorSequence(FilePreprocessor):
    def __init__(self, members = []):
        self.members = members
    def __add__(self, rhs):
        return FilePreprocessorSequence(self.members + [rhs])
    def __call__(self, input):
        output = input
        for pp in self.members:
            output = pp(output)
        return output

class LineSkipper(FilePreprocessor):
    def __init__(self, strings = [], regexps = []):
        import re
        self.strings = strings
        self.regexps = map(re.compile,regexps)

    def __processLine__(self, line):
        for s in self.strings:
            if line.find(s) >= 0: return None
        for r in self.regexps:
            if r.search(line): return None
        return line

class BlockSkipper(FilePreprocessor):
    def __init__(self, start, end):
        self.start = start
        self.end = end
        self._skipping = False

    def __processLine__(self, line):
        if self.start in line:
            self._skipping = True
            return None
        elif self.end in line:
            self._skipping = False
        elif self._skipping:
            return None
        return line

class RegexpReplacer(FilePreprocessor):
    def __init__(self, orig, repl = "", when = None):
        if when:
            when = re.compile(when)
        self._operations = [ (when, re.compile(orig), repl) ]
    def __add__(self,rhs):
        if isinstance(rhs, RegexpReplacer):
            res = RegexpReplacer("","",None)
            res._operations = self._operations + rhs._operations
        else:
            res = FilePreprocessor.__add__(self, rhs)
        return res
    def __processLine__(self, line):
        for w,o,r in self._operations:
            if w is None or w.search(line):
                line = o.sub(r, line)
        return line

# Common preprocessors
maskPointers  = RegexpReplacer("0x[0-9a-fA-F]{4,16}","0x########")
normalizeDate = RegexpReplacer("[0-2]?[0-9]:[0-5][0-9]:[0-5][0-9] [0-9]{4}[-/][01][0-9][-/][0-3][0-9] *(CES?T)?",
                               "00:00:00 1970-01-01")
normalizeEOL = FilePreprocessor()
normalizeEOL.__processLine__ = lambda line: str(line).rstrip() + '\n'

skipEmptyLines = FilePreprocessor()
# FIXME: that's ugly
skipEmptyLines.__processLine__ = lambda line: (line.strip() and line) or None

## Special preprocessor sorting the list of strings (whitespace separated)
#  that follow a signature on a single line
class LineSorter(FilePreprocessor):
    def __init__(self, signature):
        self.signature = signature
        self.siglen = len(signature)
    def __processLine__(self, line):
        pos = line.find(self.signature)
        if pos >=0:
            line = line[:(pos+self.siglen)]
            lst = line[(pos+self.siglen):].split()
            lst.sort()
            line += " ".join(lst)
        return line

# Preprocessors for GaudiExamples
normalizeExamples = maskPointers + normalizeDate
for w,o,r in [
              #("TIMER.TIMER",r"[0-9]", "0"), # Normalize time output
              ("TIMER.TIMER",r"\s+[+-]?[0-9]+[0-9.]*", " 0"), # Normalize time output
              ("release all pending",r"^.*/([^/]*:.*)",r"\1"),
              ("0x########",r"\[.*/([^/]*.*)\]",r"[\1]"),
              ("^#.*file",r"file '.*[/\\]([^/\\]*)$",r"file '\1"),
              ("^JobOptionsSvc.*options successfully read in from",r"read in from .*[/\\]([^/\\]*)$",r"file \1"), # normalize path to options
              # Normalize UUID, except those ending with all 0s (i.e. the class IDs)
              (None,r"[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}(?!-0{12})-[0-9A-Fa-f]{12}","00000000-0000-0000-0000-000000000000"),
              # Absorb a change in ServiceLocatorHelper
              ("ServiceLocatorHelper::", "ServiceLocatorHelper::(create|locate)Service", "ServiceLocatorHelper::service"),
              # Remove the leading 0 in Windows' exponential format
              (None, r"e([-+])0([0-9][0-9])", r"e\1\2"),
              # Output line changed in Gaudi v24
              (None, r'Service reference count check:', r'Looping over all active services...'),
              ]: #[ ("TIMER.TIMER","[0-9]+[0-9.]*", "") ]
    normalizeExamples += RegexpReplacer(o,r,w)

lineSkipper = LineSkipper(["//GP:",
                           "JobOptionsSvc        INFO # ",
                           "JobOptionsSvc     WARNING # ",
                           "Time User",
                           "Welcome to",
                           "This machine has a speed",
                           "TIME:",
                           "running on",
                           "ToolSvc.Sequenc...   INFO",
                           "DataListenerSvc      INFO XML written to file:",
                           "[INFO]","[WARNING]",
                           "DEBUG No writable file catalog found which contains FID:",
                           "0 local", # hack for ErrorLogExample
                           "DEBUG Service base class initialized successfully", # changed between v20 and v21
                           "DEBUG Incident  timing:", # introduced with patch #3487
                           "INFO  'CnvServices':[", # changed the level of the message from INFO to DEBUG
                           # The signal handler complains about SIGXCPU not defined on some platforms
                           'SIGXCPU',
                           ],regexps = [
                                        r"^JobOptionsSvc        INFO *$",
                                        r"^#", # Ignore python comments
                                        r"(Always|SUCCESS)\s*(Root f|[^ ]* F)ile version:", # skip the message reporting the version of the root file
                                        r"0x[0-9a-fA-F#]+ *Algorithm::sysInitialize\(\) *\[", # hack for ErrorLogExample
                                        r"0x[0-9a-fA-F#]* *__gxx_personality_v0 *\[", # hack for ErrorLogExample
                                        r"File '.*.xml' does not exist",
                                        r"INFO Refer to dataset .* by its file ID:",
                                        r"INFO Referring to dataset .* by its file ID:",
                                        r"INFO Disconnect from dataset",
                                        r"INFO Disconnected from dataset",
                                        r"INFO Disconnected data IO:",
                                        r"IncidentSvc\s*(DEBUG (Adding|Removing)|VERBOSE Calling)",
                                        # I want to ignore the header of the unchecked StatusCode report
                                        r"^StatusCodeSvc.*listing all unchecked return codes:",
                                        r"^StatusCodeSvc\s*INFO\s*$",
                                        r"Num\s*\|\s*Function\s*\|\s*Source Library",
                                        r"^[-+]*\s*$",
                                        # Hide the fake error message coming from POOL/ROOT (ROOT 5.21)
                                        r"ERROR Failed to modify file: .* Errno=2 No such file or directory",
                                        # Hide unckeched StatusCodes  from dictionaries
                                        r"^ +[0-9]+ \|.*ROOT",
                                        r"^ +[0-9]+ \|.*\|.*Dict",
                                        # Remove ROOT TTree summary table, which changes from one version to the other
                                        r"^\*.*\*$",
                                        # Remove Histos Summaries
                                        r"SUCCESS\s*Booked \d+ Histogram\(s\)",
                                        r"^ \|",
                                        r"^ ID=",
                                        ] )

if ROOT6WorkAroundEnabled('ReadRootmapCheck'):
    # FIXME: (MCl) Hide warnings from new rootmap sanity check until we can fix them
    lineSkipper += LineSkipper(regexps = [
        r'Warning in <TInterpreter::ReadRootmapFile>: .* is already in .*',
        ])

normalizeExamples = (lineSkipper + normalizeExamples + skipEmptyLines +
                     normalizeEOL + LineSorter("Services to release : "))


#--------------------- Validation functions/classes ---------------------#

class ReferenceFileValidator:
    def __init__(self,reffile, cause, result_key, preproc=normalizeExamples):
        self.reffile = os.path.expandvars(reffile)
        self.cause=cause
        self.result_key = result_key
        self.preproc = preproc

    def __call__(self,stdout, result) :
        causes=[]
        if os.path.isfile(self.reffile):
            orig=open(self.reffile).xreadlines()
            if self.preproc:
                orig = self.preproc(orig)
        else:
            orig = []
        new = stdout.splitlines()
        if self.preproc:
            new = self.preproc(new)

        diffs = difflib.ndiff(orig,new,charjunk=difflib.IS_CHARACTER_JUNK)
        filterdiffs = map(lambda x: x.strip(),filter(lambda x: x[0] != " ",diffs))
        if filterdiffs:
            result[self.result_key] = result.Quote("\n".join(filterdiffs))
            result[self.result_key] += result.Quote("""
                Legend:
                -) reference file
                +) standard output of the test""")
            causes.append(self.cause)
        return causes

def findTTreeSummaries(stdout):
    """
        Scan stdout to find ROOT TTree summaries and digest them.
        """
    stars = re.compile(r"^\*+$")
    outlines = stdout.splitlines()
    nlines = len(outlines)
    trees = {}

    i = 0
    while i < nlines: #loop over the output
        # look for
        while i < nlines and not stars.match(outlines[i]):
            i += 1
        if i < nlines:
            tree, i = _parseTTreeSummary(outlines, i)
            if tree:
                trees[tree["Name"]] = tree

    return trees

def cmpTreesDicts(reference, to_check, ignore = None):
    """
        Check that all the keys in reference are in to_check too, with the same value.
        If the value is a dict, the function is called recursively. to_check can
        contain more keys than reference, that will not be tested.
        The function returns at the first difference found.
        """
    fail_keys = []
    # filter the keys in the reference dictionary
    if ignore:
        ignore_re = re.compile(ignore)
        keys = [ key for key in reference if not ignore_re.match(key) ]
    else:
        keys = reference.keys()
    # loop over the keys (not ignored) in the reference dictionary
    for k in keys:
        if k in to_check: # the key must be in the dictionary to_check
            if (type(reference[k]) is dict) and (type(to_check[k]) is dict):
                # if both reference and to_check values are dictionaries, recurse
                failed = fail_keys = cmpTreesDicts(reference[k], to_check[k], ignore)
            else:
                # compare the two values
                failed = to_check[k] != reference[k]
        else: # handle missing keys in the dictionary to check (i.e. failure)
            to_check[k] = None
            failed = True
        if failed:
            fail_keys.insert(0, k)
            break # exit from the loop at the first failure
    return fail_keys # return the list of keys bringing to the different values

def getCmpFailingValues(reference, to_check, fail_path):
    c = to_check
    r = reference
    for k in fail_path:
        c = c.get(k,None)
        r = r.get(k,None)
        if c is None or r is None:
            break # one of the dictionaries is not deep enough
    return (fail_path, r, c)

# signature of the print-out of the histograms
h_count_re = re.compile(r"^(.*)SUCCESS\s+Booked (\d+) Histogram\(s\) :\s+(.*)")


def _parseTTreeSummary(lines, pos):
    """
    Parse the TTree summary table in lines, starting from pos.
    Returns a tuple with the dictionary with the digested informations and the
    position of the first line after the summary.
    """
    result = {}
    i = pos + 1 # first line is a sequence of '*'
    count = len(lines)

    splitcols = lambda l: [ f.strip() for f in l.strip("*\n").split(':',2) ]
    def parseblock(ll):
        r = {}
        cols = splitcols(ll[0])
        r["Name"], r["Title"] = cols[1:]

        cols = splitcols(ll[1])
        r["Entries"] = int(cols[1])

        sizes = cols[2].split()
        r["Total size"] = int(sizes[2])
        if sizes[-1] == "memory":
            r["File size"] = 0
        else:
            r["File size"] = int(sizes[-1])

        cols = splitcols(ll[2])
        sizes = cols[2].split()
        if cols[0] == "Baskets":
            r["Baskets"] = int(cols[1])
            r["Basket size"] = int(sizes[2])
        r["Compression"] = float(sizes[-1])
        return r

    if i < (count - 3) and lines[i].startswith("*Tree"):
        result = parseblock(lines[i:i+3])
        result["Branches"] = {}
        i += 4
        while i < (count - 3) and lines[i].startswith("*Br"):
            if i < (count - 2) and lines[i].startswith("*Branch "):
                # skip branch header
                i += 3
                continue
            branch = parseblock(lines[i:i+3])
            result["Branches"][branch["Name"]] = branch
            i += 4

    return (result, i)

def parseHistosSummary(lines, pos):
    """
        Extract the histograms infos from the lines starting at pos.
        Returns the position of the first line after the summary block.
        """
    global h_count_re
    h_table_head = re.compile(r'SUCCESS\s+List of booked (1D|2D|3D|1D profile|2D profile) histograms in directory\s+"(\w*)"')
    h_short_summ = re.compile(r"ID=([^\"]+)\s+\"([^\"]+)\"\s+(.*)")

    nlines = len(lines)

    # decode header
    m = h_count_re.search(lines[pos])
    name = m.group(1).strip()
    total = int(m.group(2))
    header = {}
    for k, v in [ x.split("=") for x in  m.group(3).split() ]:
        header[k] = int(v)
    pos += 1
    header["Total"] = total

    summ = {}
    while pos < nlines:
        m = h_table_head.search(lines[pos])
        if m:
            t, d = m.groups(1) # type and directory
            t = t.replace(" profile", "Prof")
            pos += 1
            if pos < nlines:
                l = lines[pos]
            else:
                l = ""
            cont = {}
            if l.startswith(" | ID"):
                # table format
                titles = [ x.strip() for x in l.split("|")][1:]
                pos += 1
                while pos < nlines and lines[pos].startswith(" |"):
                    l = lines[pos]
                    values = [ x.strip() for x in l.split("|")][1:]
                    hcont = {}
                    for i in range(len(titles)):
                        hcont[titles[i]] = values[i]
                    cont[hcont["ID"]] = hcont
                    pos += 1
            elif l.startswith(" ID="):
                while pos < nlines and lines[pos].startswith(" ID="):
                    values = [ x.strip() for x in  h_short_summ.search(lines[pos]).groups() ]
                    cont[values[0]] = values
                    pos += 1
            else: # not interpreted
                raise RuntimeError("Cannot understand line %d: '%s'" % (pos, l))
            if not d in summ:
                summ[d] = {}
            summ[d][t] = cont
            summ[d]["header"] = header
        else:
            break
    if not summ:
        # If the full table is not present, we use only the header
        summ[name] = {"header": header}
    return summ, pos



def findHistosSummaries(stdout):
    """
        Scan stdout to find ROOT TTree summaries and digest them.
        """
    outlines = stdout.splitlines()
    nlines = len(outlines) - 1
    summaries = {}
    global h_count_re

    pos = 0
    while pos < nlines:
        summ = {}
        # find first line of block:
        match = h_count_re.search(outlines[pos])
        while pos < nlines and not match:
            pos += 1
            match = h_count_re.search(outlines[pos])
        if match:
            summ, pos = parseHistosSummary(outlines, pos)
        summaries.update(summ)
    return summaries

def PlatformIsNotSupported(self, context, result):
    platform = GetPlatform(self)
    unsupported = [ re.compile(x) for x in [ str(y).strip() for y in unsupported_platforms ] if x]
    for p_re in unsupported :
       if p_re.search(platform):
           result.SetOutcome(result.UNTESTED)
           result[result.CAUSE] = 'Platform not supported.'
           return True
    return False

def GetPlatform(self):
   """
       Return the platform Id defined in CMTCONFIG or SCRAM_ARCH.
       """
   arch = "None"
   # check architecture name
   if "CMTCONFIG" in os.environ:
       arch = os.environ["CMTCONFIG"]
   elif "SCRAM_ARCH" in os.environ:
       arch = os.environ["SCRAM_ARCH"]
   return arch

def isWinPlatform(self):
   """
       Return True if the current platform is Windows.

       This function was needed because of the change in the CMTCONFIG format,
       from win32_vc71_dbg to i686-winxp-vc9-dbg.
       """
   platform = GetPlatform(self)
   return "winxp" in platform or platform.startswith("win")


def _expandReferenceFileName(self, reffile):
    # if no file is passed, do nothing
    if not reffile:
        return ""

    # function to split an extension in constituents parts
    platformSplit = lambda p: set(p.split('-' in p and '-' or '_'))

    reference = os.path.normpath(os.path.expandvars(reffile))
    # old-style platform-specific reference name
    spec_ref = reference[:-3] + GetPlatform(self)[0:3] + reference[-3:]
    if os.path.isfile(spec_ref):
        reference = spec_ref
    else: # look for new-style platform specific reference files:
        # get all the files whose name start with the reference filename
        dirname, basename = os.path.split(reference)
        if not dirname: dirname = '.'
        head = basename + "."
        head_len = len(head)
        platform = platformSplit(GetPlatform(self))
        candidates = []
        for f in os.listdir(dirname):
            if f.startswith(head):
                req_plat = platformSplit(f[head_len:])
                if platform.issuperset(req_plat):
                    candidates.append( (len(req_plat), f) )
        if candidates: # take the one with highest matching
            # FIXME: it is not possible to say if x86_64-slc5-gcc43-dbg
            #        has to use ref.x86_64-gcc43 or ref.slc5-dbg
            candidates.sort()
            reference = os.path.join(dirname, candidates[-1][1])
    return reference


#-------------------------------------------------------------------------#