preprocessors.py

Go to the documentation of this file.

import re
 
 
class FilePreprocessor:
    """Base class for a callable that takes a file and returns a modified
    version of it."""
 
    def __processLine__(self, line):
        return line
 
    def __processFile__(self, lines):
        output = []
        for l in lines:
            l = self.__processLine__(l)
            if l:
                output.append(l)
        return output
 
    def __call__(self, input):
        if not isinstance(input, str):
            lines = input
            mergeback = False
        else:
            lines = input.splitlines()
            mergeback = True
        output = self.__processFile__(lines)
        if mergeback:
            output = "\n".join(output)
        return output
 
    def __add__(self, rhs):
        return FilePreprocessorSequence([self, rhs])
 
 
class FilePreprocessorSequence(FilePreprocessor):
    def __init__(self, members=[]):
        self.members = members
 
    def __add__(self, rhs):
        return FilePreprocessorSequence(self.members + [rhs])
 
    def __call__(self, input):
        output = input
        for pp in self.members:
            output = pp(output)
        return output
 
 
def _search_pattern(regexp):
    """Drop a redundant leading ".*", which makes re.search quadratic in line length."""
    if isinstance(regexp, str):
        return re.sub(r"^\.\*(?![*+?{])", "", regexp)
    return regexp
 
 
class LineSkipper(FilePreprocessor):
    def __init__(self, strings=[], regexps=[]):
        import re
 
        self.strings = strings
        self.regexps = [re.compile(_search_pattern(r)) for r in regexps]
 
    def __processLine__(self, line):
        for s in self.strings:
            if line.find(s) >= 0:
                return None
        for r in self.regexps:
            if r.search(line):
                return None
        return line
 
 
class BlockSkipper(FilePreprocessor):
    def __init__(self, start, end):
        self.start = start
        self.end = end
        self._skipping = False
 
    def __processLine__(self, line):
        if self.start in line:
            self._skipping = True
            return None
        elif self.end in line:
            self._skipping = False
        elif self._skipping:
            return None
        return line
 
 
class RootObjectAutoRegistrationSkipper(FilePreprocessor):
    """Ignore ROOT's environment-dependent registration message and its color reset."""
 
    def __processFile__(self, lines):
        return re.sub(
            r"^TROOT[ \t]+INFO Object auto registration disabled using the environment variable ROOT_OBJECT_AUTO_REGISTRATION$"
            r"(?:\n\x1b\‍[m$)?",
            "",
            "\n".join(lines),
            flags=re.MULTILINE,
        ).splitlines()
 
 
class RegexpReplacer(FilePreprocessor):
    def __init__(self, orig, repl="", when=None):
        if when:
            when = re.compile(when)
        self._operations = [(when, re.compile(orig), repl)]
 
    def __add__(self, rhs):
        if isinstance(rhs, RegexpReplacer):
            res = RegexpReplacer("", "", None)
            res._operations = self._operations + rhs._operations
        else:
            res = FilePreprocessor.__add__(self, rhs)
        return res
 
    def __processLine__(self, line):
        for w, o, r in self._operations:
            if w is None or w.search(line):
                line = o.sub(r, line)
        return line
 
 
# Common preprocessors
maskPointers = RegexpReplacer("0x[0-9a-fA-F]{4,16}", "0x########")
normalizeDate = RegexpReplacer(
    "[0-2]?[0-9]:[0-5][0-9]:[0-5][0-9] [0-9]{4}[-/][01][0-9][-/][0-3][0-9][ A-Z]*",
    "00:00:00 1970-01-01",
)
normalizeEOL = FilePreprocessor()
normalizeEOL.__processLine__ = lambda line: str(line).rstrip() + "\n"
 
skipEmptyLines = FilePreprocessor()
# FIXME: that's ugly
skipEmptyLines.__processLine__ = lambda line: (line.strip() and line) or None
 
# Special preprocessor sorting the list of strings (whitespace separated)
#  that follow a signature on a single line
 
 
class LineSorter(FilePreprocessor):
    def __init__(self, signature):
        self.signature = signature
        self.siglen = len(signature)
 
    def __processLine__(self, line):
        pos = line.find(self.signature)
        if pos >= 0:
            line = line[: (pos + self.siglen)]
            lst = line[(pos + self.siglen) :].split()
            lst.sort()
            line += " ".join(lst)
        return line
 
 
class SortGroupOfLines(FilePreprocessor):
    """
    Sort group of lines matching a regular expression
    """
 
    def __init__(self, exp):
        self.exp = exp if hasattr(exp, "match") else re.compile(exp)
 
    def __processFile__(self, lines):
        match = self.exp.match
        output = []
        group = []
        for l in lines:
            if match(l):
                group.append(l)
            else:
                if group:
                    group.sort()
                    output.extend(group)
                    group = []
                output.append(l)
        return output
 
 
# Preprocessors for GaudiTestSuite
normalizeTestSuite = maskPointers + normalizeDate
for w, o, r in [
    ("TIMER", r"\s+[+-]?[0-9]+[0-9.e+-]*", " 0"),  # Normalize time output
    ("release all pending", r"^.*/([^/]*:.*)", r"\1"),
    ("^#.*file", r"file '.*[/\\]([^/\\]*)$", r"file '\1"),
    (
        "^JobOptionsSvc.*options successfully read in from",
        r"read in from .*[/\\]([^/\\]*)$",
        r"file \1",
    ),  # normalize path to options
    # Normalize UUID, except those ending with all 0s (i.e. the class IDs)
    (
        None,
        r"[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}(?!-0{12})-[0-9A-Fa-f]{12}",
        "00000000-0000-0000-0000-000000000000",
    ),
    # Ignore count of declared properties (anyway they are all printed)
    (
        None,
        r"^(.*(DEBUG|SUCCESS) List of ALL properties of .*#properties = )\d+",
        r"\1NN",
    ),
    (r"Property \‍['Name': Value\‍]", r"( =  '[^']+':)'(.*)'", r"\1\2"),
    ("TimelineSvc", "to file  'TimelineFile':", "to file "),
    ("DataObjectHandleBase", r'DataObjectHandleBase\‍("([^"]*)"\‍)', r"'\1'"),
]:
    normalizeTestSuite += RegexpReplacer(o, r, w)
 
lineSkipper = LineSkipper(
    [
        "Time User",
        "Welcome to",
        "running on",
        "[INFO]",
        "[WARNING]",
        "DEBUG No writable file catalog found which contains FID:",
        "DEBUG Service base class initialized successfully",
        # changed between v20 and v21
        "DEBUG Incident  timing:",
        # message removed because could be printed in constructor
        "DEBUG  'CnvServices':[",
        # The signal handler complains about SIGXCPU not
        # defined on some platforms
        "SIGXCPU",
        # Ignore warnings for properties case mismatch
        "mismatching case for property name:",
        # Message added in gaudi/Gaudi!577
        "Properties are dumped into",
        # Messages changed in gaudi/Gaudi!1426
        "WARNING no ROOT output file name",
        "INFO Writing ROOT histograms to:",
        "INFO Completed update of ROOT histograms in:",
        "INFO Disconnect from dataset",
        "INFO Disconnected from dataset",
        "INFO Disconnected data IO:",
        # absorb changes in data dependencies reports (https://gitlab.cern.ch/gaudi/Gaudi/-/merge_requests/1348)
        "Data Deps for ",
        "data dependencies:",
    ],
    regexps=[
        # Job property printouts
        r"^JobOptionsSvc\s+INFO # ",
        r"^JobOptionsSvc\s+WARNING # ",
        r"^JobOptionsSvc\s+INFO *$",
        r"^# ",  # Ignore python comments
        # skip the message reporting the version of the root file
        r"(Always|SUCCESS)\s*(Root f|[^ ]* F)ile version:",
        r"File '.*.xml' does not exist",
        r"INFO Referring to dataset .* by its file ID:",
        r"IncidentSvc\s*(DEBUG (Adding|Removing)|VERBOSE Calling)",
        # skip '---'
        r"^[-+]*\s*$",
        # Hide EventLoopMgr total timing report
        r"EventLoopMgr.*---> Loop Finished",
        r"HiveSlimEventLo.*---> Loop Finished",
        # Remove ROOT TTree summary table, which changes from one version to the other
        r"^\*.*\*$",
        # Remove Histos Summaries
        r"SUCCESS\s*Booked \d+ Histogram\‍(s\‍)",
        r"^ \|",
        r"^ ID=",
        # ignore uninteresting/obsolete messages
        r"Property update for OutputLevel : new value =",
    ],
)
 
normalizeTestSuite = (
    RootObjectAutoRegistrationSkipper()
    + lineSkipper
    + normalizeTestSuite
    + skipEmptyLines
    + normalizeEOL
    + LineSorter("Services to release : ")
    + SortGroupOfLines(r"^\S+\s+(DEBUG|SUCCESS) Property \‍[\'Name\':")
)