| # Gaudi Framework, version v21r8 |

Home |  | Generated: 17 Mar 2010

# GaudiTest.py

Go to the documentation of this file.

    00001 ########################################################################
    00002 # File:   GaudiTest.py
    00003 # Author: Marco Clemencic CERN/PH-LBC
    00004 ########################################################################
    00005 __author__  = 'Marco Clemencic CERN/PH-LBC'
    00006 __version__ = "$Revision: 1.52 $"
    00007 __tag__     = "$Name:  $"
    00008 ########################################################################
    00009 # Imports
    00010 ########################################################################
    00011 import os
    00012 import sys
    00013 import re
    00014 import tempfile
    00015 import shutil
    00016 import string
    00017 import difflib
    00018 from subprocess import Popen, PIPE, STDOUT
    00019
    00020 import qm
    00021 from qm.test.classes.command import ExecTestBase
    00022 from qm.test.result_stream import ResultStream
    00023
    00024 ### Needed by the re-implementation of TimeoutExecutable
    00025 import qm.executable
    00026 import time, signal
    00027 # The classes in this module are implemented differently depending on
    00028 # the operating system in use.
    00029 if sys.platform == "win32":
    00030     import msvcrt
    00031     import pywintypes
    00032     from   threading import *
    00033     import win32api
    00034     import win32con
    00035     import win32event
    00036     import win32file
    00037     import win32pipe
    00038     import win32process
    00039 else:
    00040     import cPickle
    00041     import fcntl
    00042     import select
    00043     import qm.sigmask
    00044
    00045 ########################################################################
    00046 # Utility Classes
    00047 ########################################################################
    00048 class TemporaryEnvironment:
    00049     """
    00050     Class to changes the environment temporarily.
    00051     """
    00052     def __init__(self, orig = os.environ, keep_same = False):
    00053         """
    00054         Create a temporary environment on top of the one specified
    00055         (it can be another TemporaryEnvironment instance).
    00056         """
    00057         #print "New environment"
    00058         self.old_values = {}
    00059         self.env = orig
    00060         self._keep_same = keep_same
    00061
    00062     def __setitem__(self,key,value):
    00063         """
    00064         Set an environment variable recording the previous value.
    00065         """
    00066         if key not in self.old_values :
    00067             if key in self.env :
    00068                 if not self._keep_same or self.env[key] != value:
    00069                     self.old_values[key] = self.env[key]
    00070             else:
    00071                 self.old_values[key] = None
    00072         self.env[key] = value
    00073
    00074     def __getitem__(self,key):
    00075         """
    00076         Get an environment variable.
    00077         Needed to provide the same interface as os.environ.
    00078         """
    00079         return self.env[key]
    00080
    00081     def __delitem__(self,key):
    00082         """
    00083         Unset an environment variable.
    00084         Needed to provide the same interface as os.environ.
    00085         """
    00086         if key not in self.env :
    00087             raise KeyError(key)
    00088         self.old_values[key] = self.env[key]
    00089         del self.env[key]
    00090
    00091     def keys(self):
    00092         """
    00093         Return the list of defined environment variables.
    00094         Needed to provide the same interface as os.environ.
    00095         """
    00096         return self.env.keys()
    00097
    00098     def items(self):
    00099         """
    00100         Return the list of (name,value) pairs for the defined environment variables.
    00101         Needed to provide the same interface as os.environ.
    00102         """
    00103         return self.env.items()
    00104
    00105     def __contains__(self,key):
    00106         """
    00107         Operator 'in'.
    00108         Needed to provide the same interface as os.environ.
    00109         """
    00110         return key in self.env
    00111
    00112     def restore(self):
    00113         """
    00114         Revert all the changes done to the orignal environment.
    00115         """
    00116         for key,value in self.old_values.items():
    00117             if value is None:
    00118                 del self.env[key]
    00119             else:
    00120                 self.env[key] = value
    00121         self.old_values = {}
    00122
    00123     def __del__(self):
    00124         """
    00125         Revert the changes on destruction.
    00126         """
    00127         #print "Restoring the environment"
    00128         self.restore()
    00129
    00130     def gen_script(self,shell_type):
    00131         """
    00132         Generate a shell script to reproduce the changes in the environment.
    00133         """
    00134         shells = [ 'csh', 'sh', 'bat' ]
    00135         if shell_type not in shells:
    00136             raise RuntimeError("Shell type '%s' unknown. Available: %s"%(shell_type,shells))
    00137         out = ""
    00138         for key,value in self.old_values.items():
    00139             if key not in self.env:
    00140                 # unset variable
    00141                 if shell_type == 'csh':
    00142                     out += 'unsetenv %s\n'%key
    00143                 elif shell_type == 'sh':
    00144                     out += 'unset %s\n'%key
    00145                 elif shell_type == 'bat':
    00146                     out += 'set %s=\n'%key
    00147             else:
    00148                 # set variable
    00149                 if shell_type == 'csh':
    00150                     out += 'setenv %s "%s"\n'%(key,self.env[key])
    00151                 elif shell_type == 'sh':
    00152                     out += 'export %s="%s"\n'%(key,self.env[key])
    00153                 elif shell_type == 'bat':
    00154                     out += 'set %s=%s\n'%(key,self.env[key])
    00155         return out
    00156
    00157 class TempDir:
    00158     """Small class for temporary directories.
    00159     When instantiated, it creates a temporary directory and the instance
    00160     behaves as the string containing the directory name.
    00161     When the instance goes out of scope, it removes all the content of
    00162     the temporary directory (automatic clean-up).
    00163     """
    00164     def __init__(self, keep = False, chdir = False):
    00165         self.name = tempfile.mkdtemp()
    00166         self._keep = keep
    00167         self._origdir = None
    00168         if chdir:
    00169             self._origdir = os.getcwd()
    00170             os.chdir(self.name)
    00171
    00172     def __str__(self):
    00173         return self.name
    00174
    00175     def __del__(self):
    00176         if self._origdir:
    00177             os.chdir(self._origdir)
    00178         if self.name and not self._keep:
    00179             shutil.rmtree(self.name)
    00180
    00181     def __getattr__(self,attr):
    00182         return getattr(self.name,attr)
    00183
    00184 class TempFile:
    00185     """Small class for temporary files.
    00186     When instantiated, it creates a temporary directory and the instance
    00187     behaves as the string containing the directory name.
    00188     When the instance goes out of scope, it removes all the content of
    00189     the temporary directory (automatic clean-up).
    00190     """
    00191     def __init__(self, suffix='', prefix='tmp', dir=None, text=False, keep = False):
    00192         self.file = None
    00193         self.name = None
    00194         self._keep = keep
    00195
    00196         self._fd, self.name = tempfile.mkstemp(suffix,prefix,dir,text)
    00197         self.file = os.fdopen(self._fd,"r+")
    00198
    00199     def __str__(self):
    00200         return self.name
    00201
    00202     def __del__(self):
    00203         if self.file:
    00204             self.file.close()
    00205         if self.name and not self._keep:
    00206             os.remove(self.name)
    00207
    00208     def __getattr__(self,attr):
    00209         return getattr(self.file,attr)
    00210
    00211 class CMT:
    00212     """Small wrapper to call CMT.
    00213     """
    00214     def __init__(self,path=None):
    00215         if path is None:
    00216             path = os.getcwd()
    00217         self.path = path
    00218
    00219     def _run_cmt(self,command,args):
    00220         # prepare command line
    00221         if type(args) is str:
    00222             args = [args]
    00223         cmd = "cmt %s"%command
    00224         for arg in args:
    00225             cmd += ' "%s"'%arg
    00226
    00227         # go to the execution directory
    00228         olddir = os.getcwd()
    00229         os.chdir(self.path)
    00230         # run cmt
    00231         result = os.popen4(cmd)[1].read()
    00232         # return to the old directory
    00233         os.chdir(olddir)
    00234         return result
    00235
    00236     def __getattr__(self,attr):
    00237         return lambda args=[]: self._run_cmt(attr, args)
    00238
    00239     def runtime_env(self,env = None):
    00240         """Returns a dictionary containing the runtime environment produced by CMT.
    00241         If a dictionary is passed a modified instance of it is returned.
    00242         """
    00243         if env is None:
    00244             env = {}
    00245         for l in self.setup("-csh").splitlines():
    00246             l = l.strip()
    00247             if l.startswith("setenv"):
    00248                 dummy,name,value = l.split(None,3)
    00249                 env[name] = value.strip('"')
    00250             elif l.startswith("unsetenv"):
    00251                 dummy,name = l.split(None,2)
    00252                 if name in env:
    00253                     del env[name]
    00254         return env
    00255     def show_macro(self,k):
    00256         r = self.show(["macro",k])
    00257         if r.find("CMT> Error: symbol not found") >= 0:
    00258             return None
    00259         else:
    00260             return self.show(["macro_value",k]).strip()
    00261
    00262 ## Locates an executable in the executables path ($PATH) and returns the full
    00263 #  path to it.
    00264 #  If the excutable cannot be found, None is returned
    00265 def which(executable):
    00266     for d in os.environ.get("PATH").split(os.pathsep):
    00267         fullpath = os.path.join(d,executable)
    00268         if os.path.exists(fullpath):
    00269             return fullpath
    00270     return None
    00271
    00272 def rationalizepath(p):
    00273     p = os.path.normpath(os.path.expandvars(p))
    00274     if os.path.exists(p):
    00275         p = os.path.realpath(p)
    00276     return p
    00277
    00278 ########################################################################
    00279 # Output Validation Classes
    00280 ########################################################################
    00281 class BasicOutputValidator:
    00282     """Basic implementation of an option validator for Gaudi tests.
    00283     This implementation is based on the standard (LCG) validation functions
    00284     used in QMTest.
    00285     """
    00286     def __init__(self,ref,cause,result_key):
    00287         self.reference = ref
    00288         self.cause = cause
    00289         self.result_key = result_key
    00290
    00291     def __call__(self, out, result):
    00292         """Validate the output of the program.
    00293
    00294         'stdout' -- A string containing the data written to the standard output
    00295         stream.
    00296
    00297         'stderr' -- A string containing the data written to the standard error
    00298         stream.
    00299
    00300         'result' -- A 'Result' object. It may be used to annotate
    00301         the outcome according to the content of stderr.
    00302
    00303         returns -- A list of strings giving causes of failure."""
    00304
    00305         causes = []
    00306         # Check to see if theoutput matches.
    00307         if not self.__CompareText(out, self.reference):
    00308             causes.append(self.cause)
    00309             result[self.result_key] = result.Quote(self.reference)
    00310
    00311         return causes
    00312
    00313     def __CompareText(self, s1, s2):
    00314         """Compare 's1' and 's2', ignoring line endings.
    00315
    00316         's1' -- A string.
    00317
    00318         's2' -- A string.
    00319
    00320         returns -- True if 's1' and 's2' are the same, ignoring
    00321         differences in line endings."""
    00322
    00323         # The "splitlines" method works independently of the line ending
    00324         # convention in use.
    00325         return s1.splitlines() == s2.splitlines()
    00326
    00327 class FilePreprocessor:
    00328     """ Base class for a callable that takes a file and returns a modified
    00329     version of it."""
    00330     def __processLine__(self, line):
    00331         return line
    00332     def __call__(self, input):
    00333         if hasattr(input,"__iter__"):
    00334             lines = input
    00335             mergeback = False
    00336         else:
    00337             lines = input.splitlines()
    00338             mergeback = True
    00339         output = []
    00340         for l in lines:
    00341             l = self.__processLine__(l)
    00342             if l: output.append(l)
    00343         if mergeback: output = '\n'.join(output)
    00344         return output
    00345     def __add__(self, rhs):
    00346         return FilePreprocessorSequence([self,rhs])
    00347
    00348 class FilePreprocessorSequence(FilePreprocessor):
    00349     def __init__(self, members = []):
    00350         self.members = members
    00351     def __add__(self, rhs):
    00352         return FilePreprocessorSequence(self.members + [rhs])
    00353     def __call__(self, input):
    00354         output = input
    00355         for pp in self.members:
    00356             output = pp(output)
    00357         return output
    00358
    00359 class LineSkipper(FilePreprocessor):
    00360     def __init__(self, strings = [], regexps = []):
    00361         import re
    00362         self.strings = strings
    00363         self.regexps = map(re.compile,regexps)
    00364
    00365     def __processLine__(self, line):
    00366         for s in self.strings:
    00367             if line.find(s) >= 0: return None
    00368         for r in self.regexps:
    00369             if r.search(line): return None
    00370         return line
    00371
    00372 class BlockSkipper(FilePreprocessor):
    00373     def __init__(self, start, end):
    00374         self.start = start
    00375         self.end = end
    00376         self._skipping = False
    00377
    00378     def __processLine__(self, line):
    00379         if self.start in line:
    00380             self._skipping = True
    00381             return None
    00382         elif self.end in line:
    00383             self._skipping = False
    00384         elif self._skipping:
    00385             return None
    00386         return line
    00387
    00388 class RegexpReplacer(FilePreprocessor):
    00389     def __init__(self, orig, repl = "", when = None):
    00390         if when:
    00391             when = re.compile(when)
    00392         self._operations = [ (when, re.compile(orig), repl) ]
    00393     def __add__(self,rhs):
    00394         if isinstance(rhs, RegexpReplacer):
    00395             res = RegexpReplacer("","",None)
    00396             res._operations = self._operations + rhs._operations
    00397         else:
    00398             res = FilePreprocessor.__add__(self, rhs)
    00399         return res
    00400     def __processLine__(self, line):
    00401         for w,o,r in self._operations:
    00402             if w is None or w.search(line):
    00403                 line = o.sub(r, line)
    00404         return line
    00405
    00406 # Common preprocessors
    00407 maskPointers  = RegexpReplacer("0x[0-9a-fA-F]{4,16}","0x########")
    00408 normalizeDate = RegexpReplacer("[0-2]?[0-9]:[0-5][0-9]:[0-5][0-9] [0-9]{4}[-/][01][0-9][-/][0-3][0-9] *(CES?T)?",
    00409                                "00:00:00 1970-01-01")
    00410 normalizeEOL = FilePreprocessor()
    00411 normalizeEOL.__processLine__ = lambda line: str(line).rstrip() + '\n'
    00412
    00413 skipEmptyLines = FilePreprocessor()
    00414 # FIXME: that's ugly
    00415 skipEmptyLines.__processLine__ = lambda line: (line.strip() and line) or None
    00416
    00417 ## Special preprocessor sorting the list of strings (whitespace separated)
    00418 #  that follow a signature on a single line
    00419 class LineSorter(FilePreprocessor):
    00420     def __init__(self, signature):
    00421         self.signature = signature
    00422         self.siglen = len(signature)
    00423     def __processLine__(self, line):
    00424         pos = line.find(self.signature)
    00425         if pos >=0:
    00426             line = line[:(pos+self.siglen)]
    00427             lst = line[(pos+self.siglen):].split()
    00428             lst.sort()
    00429             line += " ".join(lst)
    00430         return line
    00431
    00432 # Preprocessors for GaudiExamples
    00433 normalizeExamples = maskPointers + normalizeDate
    00434 for w,o,r in [
    00435               #("TIMER.TIMER",r"[0-9]", "0"), # Normalize time output
    00436               ("TIMER.TIMER",r"\s+[+-]?[0-9]+[0-9.]*", " 0"), # Normalize time output
    00437               ("release all pending",r"^.*/([^/]*:.*)",r"\1"),
    00438               ("0x########",r"\[.*/([^/]*.*)\]",r"[\1]"),
    00439               ("^#.*file",r"file '.*[/\\]([^/\\]*)$",r"file '\1"),
    00440               ("^JobOptionsSvc.*options successfully read in from",r"read in from .*[/\\]([^/\\]*)$",r"file \1"), # normalize path to options
    00441               # Normalize UUID, except those ending with all 0s (i.e. the class IDs)
    00442               (None,r"[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}(?!-0{12})-[0-9A-Fa-f]{12}","00000000-0000-0000-0000-000000000000"),
    00443               # Absorb a change in ServiceLocatorHelper
    00444               ("ServiceLocatorHelper::", "ServiceLocatorHelper::(create|locate)Service", "ServiceLocatorHelper::service"),
    00445               # Remove the leading 0 in Windows' exponential format
    00446               (None, r"e([-+])0([0-9][0-9])", r"e\1\2"),
    00447               ]: #[ ("TIMER.TIMER","[0-9]+[0-9.]*", "") ]
    00448     normalizeExamples += RegexpReplacer(o,r,w)
    00449 normalizeExamples = LineSkipper(["//GP:",
    00450                                  "Time User",
    00451                                  "Welcome to",
    00452                                  "This machine has a speed",
    00453                                  "TIME:",
    00454                                  "running on",
    00455                                  "ToolSvc.Sequenc...   INFO",
    00456                                  "DataListenerSvc      INFO XML written to file:",
    00457                                  "[INFO]","[WARNING]",
    00458                                  "DEBUG No writable file catalog found which contains FID:",
    00459                                  "0 local", # hack for ErrorLogExample
    00460                                  "DEBUG Service base class initialized successfully", # changed between v20 and v21
    00461                                  "DEBUG Incident  timing:", # introduced with patch #3487
    00462                                  # This comes from ROOT, when using GaudiPython
    00463                                  'Note: (file "(tmpfile)", line 2) File "set" already loaded',
    00464                                  ],regexps = [
    00465                                  r"^#", # Ignore python comments
    00466                                  r"(Always|SUCCESS)\s*(Root f|[^ ]* F)ile version:", # skip the message reporting the version of the root file
    00467                                  r"0x[0-9a-fA-F#]+ *Algorithm::sysInitialize\(\) *\[", # hack for ErrorLogExample
    00468                                  r"0x[0-9a-fA-F#]* *__gxx_personality_v0 *\[", # hack for ErrorLogExample
    00469                                  r"File '.*.xml' does not exist",
    00470                                  r"INFO Refer to dataset .* by its file ID:",
    00471                                  r"INFO Referring to dataset .* by its file ID:",
    00472                                  r"INFO Disconnect from dataset",
    00473                                  r"INFO Disconnected from dataset",
    00474                                  r"INFO Disconnected data IO:",
    00475                                  r"IncidentSvc\s*(DEBUG (Adding|Removing)|VERBOSE Calling)",
    00476                                  # I want to ignore the header of the unchecked StatusCode report
    00477                                  r"^StatusCodeSvc.*listing all unchecked return codes:",
    00478                                  r"^StatusCodeSvc\s*INFO\s*$",
    00479                                  r"Num\s*|\s*Function\s*|\s*Source Library",
    00480                                  r"^[-+]*\s*$",
    00481                                  # Hide the fake error message coming from POOL/ROOT (ROOT 5.21)
    00482                                  r"ERROR Failed to modify file: .* Errno=2 No such file or directory",
    00483                                  # Hide unckeched StatusCodes  from dictionaries
    00484                                  r"^ +[0-9]+ \|.*ROOT",
    00485                                  r"^ +[0-9]+ \|.*\|.*Dict",
    00486                                  # Remove ROOT TTree summary table, which changes from one version to the other
    00487                                  r"^\*.*\*$",
    00488                                  # Remove Histos Summaries
    00489                                  r"SUCCESS\s*Booked \d+ Histogram\(s\)",
    00490                                  r"^ \|",
    00491                                  r"^ ID=",
    00492                                  ] ) + normalizeExamples + skipEmptyLines + \
    00493                                   normalizeEOL + \
    00494                                   LineSorter("Services to release : ")
    00495
    00496 class ReferenceFileValidator:
    00497     def __init__(self, reffile, cause, result_key, preproc = normalizeExamples):
    00498         self.reffile = os.path.expandvars(reffile)
    00499         self.cause = cause
    00500         self.result_key = result_key
    00501         self.preproc = preproc
    00502     def __call__(self, stdout, result):
    00503         causes = []
    00504         if os.path.isfile(self.reffile):
    00505             orig = open(self.reffile).xreadlines()
    00506             if self.preproc:
    00507                 orig = self.preproc(orig)
    00508         else:
    00509             orig = []
    00510
    00511         new = stdout.splitlines()
    00512         if self.preproc:
    00513             new = self.preproc(new)
    00514         #open(self.reffile + ".test","w").writelines(new)
    00515         diffs = difflib.ndiff(orig,new,charjunk=difflib.IS_CHARACTER_JUNK)
    00516         filterdiffs = map(lambda x: x.strip(),filter(lambda x: x[0] != " ",diffs))
    00517         #filterdiffs = [x.strip() for x in diffs]
    00518         if filterdiffs:
    00519             result[self.result_key] = result.Quote("\n".join(filterdiffs))
    00520             result[self.result_key] += result.Quote("""
    00521 Legend:
    00522         -) reference file
    00523         +) standard output of the test""")
    00524             causes.append(self.cause)
    00525
    00526         return causes
    00527
    00528 ########################################################################
    00529 # Useful validation functions
    00530 ########################################################################
    00531 def findReferenceBlock(reference, stdout, result, causes, signature_offset=0, signature=None,
    00532                        id = None):
    00533     """
    00534     Given a block of text, tries to find it in the output.
    00535     The block had to be identified by a signature line. By default, the first
    00536     line is used as signature, or the line pointed to by signature_offset. If
    00537     signature_offset points outside the block, a signature line can be passed as
    00538     signature argument. Note: if 'signature' is None (the default), a negative
    00539     signature_offset is interpreted as index in a list (e.g. -1 means the last
    00540     line), otherwise the it is interpreted as the number of lines before the
    00541     first one of the block the signature must appear.
    00542     The parameter 'id' allow to distinguish between different calls to this
    00543     function in the same validation code.
    00544     """
    00545     # split reference file, sanitize EOLs and remove empty lines
    00546     reflines = filter(None,map(lambda s: s.rstrip(), reference.splitlines()))
    00547     if not reflines:
    00548         raise RuntimeError("Empty (or null) reference")
    00549     # the same on standard output
    00550     outlines = filter(None,map(lambda s: s.rstrip(), stdout.splitlines()))
    00551
    00552     res_field = "GaudiTest.RefBlock"
    00553     if id:
    00554         res_field += "_%s" % id
    00555
    00556     if signature is None:
    00557         if signature_offset < 0:
    00558             signature_offset = len(reference)+signature_offset
    00559         signature = reflines[signature_offset]
    00560     # find the reference block in the output file
    00561     try:
    00562         pos = outlines.index(signature)
    00563         outlines = outlines[pos-signature_offset:pos+len(reflines)-signature_offset]
    00564         if reflines != outlines:
    00565             msg = "standard output"
    00566             # I do not want 2 messages in causes if teh function is called twice
    00567             if not msg in causes:
    00568                 causes.append(msg)
    00569             result[res_field + ".observed"] = result.Quote("\n".join(outlines))
    00570     except ValueError:
    00571         causes.append("missing signature")
    00572     result[res_field + ".signature"] = result.Quote(signature)
    00573     if len(reflines) > 1 or signature != reflines[0]:
    00574         result[res_field + ".expected"] = result.Quote("\n".join(reflines))
    00575
    00576     return causes
    00577
    00578 def countErrorLines(expected = {'ERROR':0, 'FATAL':0}, **kwargs):
    00579     """
    00580     Count the number of messages with required severity (by default ERROR and FATAL)
    00581     and check if their numbers match the expected ones (0 by default).
    00582     The dictionary "expected" can be used to tune the number of errors and fatals
    00583     allowed, or to limit the number of expected warnings etc.
    00584     """
    00585     stdout = kwargs["stdout"]
    00586     result = kwargs["result"]
    00587     causes = kwargs["causes"]
    00588
    00589     # prepare the dictionary to record the extracted lines
    00590     errors = {}
    00591     for sev in expected:
    00592         errors[sev] = []
    00593
    00594     outlines = stdout.splitlines()
    00595     from math import log10
    00596     fmt = "%%%dd - %%s" % (int(log10(len(outlines))+1))
    00597
    00598     linecount = 0
    00599     for l in outlines:
    00600         linecount += 1
    00601         words = l.split()
    00602         if len(words) >= 2 and words[1] in errors:
    00603             errors[words[1]].append(fmt%(linecount,l.rstrip()))
    00604
    00605     for e in errors:
    00606         if len(errors[e]) != expected[e]:
    00607             causes.append('%s(%d)'%(e,len(errors[e])))
    00608             result["GaudiTest.lines.%s"%e] = result.Quote('\n'.join(errors[e]))
    00609             result["GaudiTest.lines.%s.expected#"%e] = result.Quote(str(expected[e]))
    00610
    00611     return causes
    00612
    00613
    00614 def _parseTTreeSummary(lines, pos):
    00615     """
    00616     Parse the TTree summary table in lines, starting from pos.
    00617     Returns a tuple with the dictionary with the digested informations and the
    00618     position of the first line after the summary.
    00619     """
    00620     result = {}
    00621     i = pos + 1 # first line is a sequence of '*'
    00622     count = len(lines)
    00623
    00624     splitcols = lambda l: [ f.strip() for f in l.strip("*\n").split(':',2) ]
    00625     def parseblock(ll):
    00626         r = {}
    00627         cols = splitcols(ll[0])
    00628         r["Name"], r["Title"] = cols[1:]
    00629
    00630         cols = splitcols(ll[1])
    00631         r["Entries"] = int(cols[1])
    00632
    00633         sizes = cols[2].split()
    00634         r["Total size"] = int(sizes[2])
    00635         if sizes[-1] == "memory":
    00636             r["File size"] = 0
    00637         else:
    00638             r["File size"] = int(sizes[-1])
    00639
    00640         cols = splitcols(ll[2])
    00641         sizes = cols[2].split()
    00642         if cols[0] == "Baskets":
    00643             r["Baskets"] = int(cols[1])
    00644             r["Basket size"] = int(sizes[2])
    00645         r["Compression"] = float(sizes[-1])
    00646         return r
    00647
    00648     if i < (count - 3) and lines[i].startswith("*Tree"):
    00649         result = parseblock(lines[i:i+3])
    00650         result["Branches"] = {}
    00651         i += 4
    00652         while i < (count - 3) and lines[i].startswith("*Br"):
    00653             branch = parseblock(lines[i:i+3])
    00654             result["Branches"][branch["Name"]] = branch
    00655             i += 4
    00656
    00657     return (result, i)
    00658
    00659 def findTTreeSummaries(stdout):
    00660     """
    00661     Scan stdout to find ROOT TTree summaries and digest them.
    00662     """
    00663     stars = re.compile(r"^\*+$")
    00664     outlines = stdout.splitlines()
    00665     nlines = len(outlines)
    00666     trees = {}
    00667
    00668     i = 0
    00669     while i < nlines: #loop over the output
    00670         # look for
    00671         while i < nlines and not stars.match(outlines[i]):
    00672             i += 1
    00673         if i < nlines:
    00674             tree, i = _parseTTreeSummary(outlines, i)
    00675             if tree:
    00676                 trees[tree["Name"]] = tree
    00677
    00678     return trees
    00679
    00680 def cmpTreesDicts(reference, to_check, ignore = None):
    00681     """
    00682     Check that all the keys in reference are in to_check too, with the same value.
    00683     If the value is a dict, the function is called recursively. to_check can
    00684     contain more keys than reference, that will not be tested.
    00685     The function returns at the first difference found.
    00686     """
    00687     fail_keys = []
    00688     # filter the keys in the reference dictionary
    00689     if ignore:
    00690         ignore_re = re.compile(ignore)
    00691         keys = [ key for key in reference if not ignore_re.match(key) ]
    00692     else:
    00693         keys = reference.keys()
    00694     # loop over the keys (not ignored) in the reference dictionary
    00695     for k in keys:
    00696         if k in to_check: # the key must be in the dictionary to_check
    00697             if (type(reference[k]) is dict) and (type(to_check[k]) is dict):
    00698                 # if both reference and to_check values are dictionaries, recurse
    00699                 failed = fail_keys = cmpTreesDicts(reference[k], to_check[k], ignore)
    00700             else:
    00701                 # compare the two values
    00702                 failed = to_check[k] != reference[k]
    00703         else: # handle missing keys in the dictionary to check (i.e. failure)
    00704             to_check[k] = None
    00705             failed = True
    00706         if failed:
    00707             fail_keys.insert(0, k)
    00708             break # exit from the loop at the first failure
    00709     return fail_keys # return the list of keys bringing to the different values
    00710
    00711 def getCmpFailingValues(reference, to_check, fail_path):
    00712     c = to_check
    00713     r = reference
    00714     for k in fail_path:
    00715         c = c.get(k,None)
    00716         r = r.get(k,None)
    00717         if c is None or r is None:
    00718             break # one of the dictionaries is not deep enough
    00719     return (fail_path, r, c)
    00720
    00721 # signature of the print-out of the histograms
    00722 h_count_re = re.compile(r"^(.*)SUCCESS\s+Booked (\d+) Histogram\(s\) :\s+(.*)")
    00723
    00724 def parseHistosSummary(lines, pos):
    00725     """
    00726     Extract the histograms infos from the lines starting at pos.
    00727     Returns the position of the first line after the summary block.
    00728     """
    00729     global h_count_re
    00730     h_table_head = re.compile(r'SUCCESS\s+List of booked (1D|2D|3D|1D profile|2D profile) histograms in directory\s+"(\w*)"')
    00731     h_short_summ = re.compile(r"ID=([^\"]+)\s+\"([^\"]+)\"\s+(.*)")
    00732
    00733     nlines = len(lines)
    00734
    00735     # decode header
    00736     m = h_count_re.search(lines[pos])
    00737     name = m.group(1).strip()
    00738     total = int(m.group(2))
    00739     header = {}
    00740     for k, v in [ x.split("=") for x in  m.group(3).split() ]:
    00741         header[k] = int(v)
    00742     pos += 1
    00743     header["Total"] = total
    00744
    00745     summ = {}
    00746     while pos < nlines:
    00747         m = h_table_head.search(lines[pos])
    00748         if m:
    00749             t, d = m.groups(1) # type and directory
    00750             t = t.replace(" profile", "Prof")
    00751             pos += 1
    00752             if pos < nlines:
    00753                 l = lines[pos]
    00754             else:
    00755                 l = ""
    00756             cont = {}
    00757             if l.startswith(" | ID"):
    00758                 # table format
    00759                 titles = [ x.strip() for x in l.split("|")][1:]
    00760                 pos += 1
    00761                 while pos < nlines and lines[pos].startswith(" |"):
    00762                     l = lines[pos]
    00763                     values = [ x.strip() for x in l.split("|")][1:]
    00764                     hcont = {}
    00765                     for i in range(len(titles)):
    00766                         hcont[titles[i]] = values[i]
    00767                     cont[hcont["ID"]] = hcont
    00768                     pos += 1
    00769             elif l.startswith(" ID="):
    00770                 while pos < nlines and lines[pos].startswith(" ID="):
    00771                     values = [ x.strip() for x in  h_short_summ.search(lines[pos]).groups() ]
    00772                     cont[values[0]] = values
    00773                     pos += 1
    00774             else: # not interpreted
    00775                 raise RuntimeError("Cannot understand line %d: '%s'" % (pos, l))
    00776             if not d in summ:
    00777                 summ[d] = {}
    00778             summ[d][t] = cont
    00779             summ[d]["header"] = header
    00780         else:
    00781             break
    00782     if not summ:
    00783         # If the full table is not present, we use only the header
    00784         summ[name] = {"header": header}
    00785     return summ, pos
    00786
    00787 def findHistosSummaries(stdout):
    00788     """
    00789     Scan stdout to find ROOT TTree summaries and digest them.
    00790     """
    00791     outlines = stdout.splitlines()
    00792     nlines = len(outlines) - 1
    00793     summaries = {}
    00794     global h_count_re
    00795
    00796     pos = 0
    00797     while pos < nlines:
    00798         summ = {}
    00799         # find first line of block:
    00800         match = h_count_re.search(outlines[pos])
    00801         while pos < nlines and not match:
    00802             pos += 1
    00803             match = h_count_re.search(outlines[pos])
    00804         if match:
    00805             summ, pos = parseHistosSummary(outlines, pos)
    00806         summaries.update(summ)
    00807     return summaries
    00808
    00809 class GaudiFilterExecutable(qm.executable.Filter):
    00810     def __init__(self, input, timeout = -1):
    00811         """Create a new 'Filter'.
    00812
    00813         'input' -- The string containing the input to provide to the
    00814         child process.
    00815
    00816         'timeout' -- As for 'TimeoutExecutable.__init__'."""
    00817
    00818         super(GaudiFilterExecutable, self).__init__(input, timeout)
    00819         self.__input = input
    00820         self.__timeout = timeout
    00821         self.stack_trace_file = None
    00822         # Temporary file to pass the stack trace from one process to the other
    00823         # The file must be closed and reopened when needed to avoid conflicts
    00824         # between the processes
    00825         tmpf = tempfile.mkstemp()
    00826         os.close(tmpf[0])
    00827         self.stack_trace_file = tmpf[1] # remember only the name
    00828
    00829     def __UseSeparateProcessGroupForChild(self):
    00830         """Copied from TimeoutExecutable to allow the re-implementation of
    00831            _HandleChild.
    00832         """
    00833         if sys.platform == "win32":
    00834             # In Windows 2000 (or later), we should use "jobs" by
    00835             # analogy with UNIX process groups.  However, that
    00836             # functionality is not (yet) provided by the Python Win32
    00837             # extensions.
    00838             return 0
    00839
    00840         return self.__timeout >= 0 or self.__timeout == -2
    00841     ##
    00842     # Needs to replace the ones from RedirectedExecutable and TimeoutExecutable
    00843     def _HandleChild(self):
    00844         """Code copied from both FilterExecutable and TimeoutExecutable.
    00845         """
    00846         # Close the pipe ends that we do not need.
    00847         if self._stdin_pipe:
    00848             self._ClosePipeEnd(self._stdin_pipe[0])
    00849         if self._stdout_pipe:
    00850             self._ClosePipeEnd(self._stdout_pipe[1])
    00851         if self._stderr_pipe:
    00852             self._ClosePipeEnd(self._stderr_pipe[1])
    00853
    00854         # The pipes created by 'RedirectedExecutable' must be closed
    00855         # before the monitor process (created by 'TimeoutExecutable')
    00856         # is created.  Otherwise, if the child process dies, 'select'
    00857         # in the parent will not return if the monitor process may
    00858         # still have one of the file descriptors open.
    00859
    00860         super(qm.executable.TimeoutExecutable, self)._HandleChild()
    00861
    00862         if self.__UseSeparateProcessGroupForChild():
    00863             # Put the child into its own process group.  This step is
    00864             # performed in both the parent and the child; therefore both
    00865             # processes can safely assume that the creation of the process
    00866             # group has taken place.
    00867             child_pid = self._GetChildPID()
    00868             try:
    00869                 os.setpgid(child_pid, child_pid)
    00870             except:
    00871                 # The call to setpgid may fail if the child has exited,
    00872                 # or has already called 'exec'.  In that case, we are
    00873                 # guaranteed that the child has already put itself in the
    00874                 # desired process group.
    00875                 pass
    00876             # Create the monitoring process.
    00877             #
    00878             # If the monitoring process is in parent's process group and
    00879             # kills the child after waitpid has returned in the parent, we
    00880             # may end up trying to kill a process group other than the one
    00881             # that we intend to kill.  Therefore, we put the monitoring
    00882             # process in the same process group as the child; that ensures
    00883             # that the process group will persist until the monitoring
    00884             # process kills it.
    00885             self.__monitor_pid = os.fork()
    00886             if self.__monitor_pid != 0:
    00887                 # Make sure that the monitoring process is placed into the
    00888                 # child's process group before the parent process calls
    00889                 # 'waitpid'.  In this way, we are guaranteed that the process
    00890                 # group as the child
    00891                 os.setpgid(self.__monitor_pid, child_pid)
    00892             else:
    00893                 # Put the monitoring process into the child's process
    00894                 # group.  We know the process group still exists at
    00895                 # this point because either (a) we are in the process
    00896                 # group, or (b) the parent has not yet called waitpid.
    00897                 os.setpgid(0, child_pid)
    00898
    00899                 # Close all open file descriptors.  They are not needed
    00900                 # in the monitor process.  Furthermore, when the parent
    00901                 # closes the write end of the stdin pipe to the child,
    00902                 # we do not want the pipe to remain open; leaving the
    00903                 # pipe open in the monitor process might cause the child
    00904                 # to block waiting for additional input.
    00905                 try:
    00906                     max_fds = os.sysconf("SC_OPEN_MAX")
    00907                 except:
    00908                     max_fds = 256
    00909                 for fd in xrange(max_fds):
    00910                     try:
    00911                         os.close(fd)
    00912                     except:
    00913                         pass
    00914                 try:
    00915                     if self.__timeout >= 0:
    00916                         # Give the child time to run.
    00917                         time.sleep (self.__timeout)
    00918                         #######################################################
    00919                         ### This is the interesting part: dump the stack trace to a file
    00920                         if sys.platform == "linux2": # we should be have /proc and gdb
    00921                             cmd = ["gdb",
    00922                                    os.path.join("/proc", str(child_pid), "exe"),
    00923                                    str(child_pid),
    00924                                    "-batch", "-n", "-x",
    00925                                    "'%s'" % os.path.join(os.path.dirname(__file__), "stack-trace.gdb")]
    00926                             # FIXME: I wanted to use subprocess.Popen, but it doesn't want to work
    00927                             #        in this context.
    00928                             o = os.popen(" ".join(cmd)).read()
    00929                             open(self.stack_trace_file,"w").write(o)
    00930                         #######################################################
    00931
    00932                         # Kill all processes in the child process group.
    00933                         os.kill(0, signal.SIGKILL)
    00934                     else:
    00935                         # This call to select will never terminate.
    00936                         select.select ([], [], [])
    00937                 finally:
    00938                     # Exit.  This code is in a finally clause so that
    00939                     # we are guaranteed to get here no matter what.
    00940                     os._exit(0)
    00941         elif self.__timeout >= 0 and sys.platform == "win32":
    00942             # Create a monitoring thread.
    00943             self.__monitor_thread = Thread(target = self.__Monitor)
    00944             self.__monitor_thread.start()
    00945
    00946     if sys.platform == "win32":
    00947
    00948         def __Monitor(self):
    00949             """Code copied from FilterExecutable.
    00950             Kill the child if the timeout expires.
    00951
    00952             This function is run in the monitoring thread."""
    00953
    00954             # The timeout may be expressed as a floating-point value
    00955             # on UNIX, but it must be an integer number of
    00956             # milliseconds when passed to WaitForSingleObject.
    00957             timeout = int(self.__timeout * 1000)
    00958             # Wait for the child process to terminate or for the
    00959             # timer to expire.
    00960             result = win32event.WaitForSingleObject(self._GetChildPID(),
    00961                                                     timeout)
    00962             # If the timeout occurred, kill the child process.
    00963             if result == win32con.WAIT_TIMEOUT:
    00964                 self.Kill()
    00965
    00966 ########################################################################
    00967 # Test Classes
    00968 ########################################################################
    00969 class GaudiExeTest(ExecTestBase):
    00970     """Standard Gaudi test.
    00971     """
    00972     arguments = [
    00973         qm.fields.TextField(
    00974             name="program",
    00975             title="Program",
    00976             not_empty_text=1,
    00977             description="""The path to the program.
    00978
    00979             This field indicates the path to the program.  If it is not
    00980             an absolute path, the value of the 'PATH' environment
    00981             variable will be used to search for the program.
    00982             If not specified, $GAUDIEXE or Gaudi.exe are used.
    00983             """
    00984             ),
    00985         qm.fields.SetField(qm.fields.TextField(
    00986             name="args",
    00987             title="Argument List",
    00988             description="""The command-line arguments.
    00989
    00990             If this field is left blank, the program is run without any
    00991             arguments.
    00992
    00993             Use this field to specify the option files.
    00994
    00995             An implicit 0th argument (the path to the program) is added
    00996             automatically."""
    00997             )),
    00998         qm.fields.TextField(
    00999             name="options",
    01000             title="Options",
    01001             description="""Options to be passed to the application.
    01002
    01003             This field allows to pass a list of options to the main program
    01004             without the need of a separate option file.
    01005
    01006             The content of the field is written to a temporary file which name
    01007             is passed the the application as last argument (appended to the
    01008             field "Argument List".
    01009             """,
    01010             verbatim="true",
    01011             multiline="true",
    01012             default_value=""
    01013             ),
    01014         qm.fields.TextField(
    01015             name="workdir",
    01016             title="Working Directory",
    01017             description="""Path to the working directory.
    01018
    01019             If this field is left blank, the program will be run from the qmtest
    01020             directory, otherwise from the directory specified.""",
    01021             default_value=""
    01022             ),
    01023         qm.fields.TextField(
    01024             name="reference",
    01025             title="Reference Output",
    01026             description="""Path to the file containing the reference output.
    01027
    01028             If this field is left blank, any standard output will be considered
    01029             valid.
    01030
    01031             If the reference file is specified, any output on standard error is
    01032             ignored."""
    01033             ),
    01034         qm.fields.TextField(
    01035             name="error_reference",
    01036             title="Reference for standard error",
    01037             description="""Path to the file containing the reference for the standard error.
    01038
    01039             If this field is left blank, any standard output will be considered
    01040             valid.
    01041
    01042             If the reference file is specified, any output on standard error is
    01043             ignored."""
    01044             ),
    01045         qm.fields.SetField(qm.fields.TextField(
    01046             name = "unsupported_platforms",
    01047             title = "Unsupported Platforms",
    01048             description = """Platform on which the test must not be run.
    01049
    01050             List of regular expressions identifying the platforms on which the
    01051             test is not run and the result is set to UNTESTED."""
    01052             )),
    01053
    01054         qm.fields.TextField(
    01055             name = "validator",
    01056             title = "Validator",
    01057             description = """Function to validate the output of the test.
    01058
    01059             If defined, the function is used to validate the products of the
    01060             test.
    01061             The function is called passing as arguments:
    01062               self:   the test class instance
    01063               stdout: the standard output of the executed test
    01064               stderr: the standard error of the executed test
    01065               result: the Result objects to fill with messages
    01066             The function must return a list of causes for the failure.
    01067             If specified, overrides standard output, standard error and
    01068             reference files.
    01069             """,
    01070             verbatim="true",
    01071             multiline="true",
    01072             default_value=""
    01073             ),
    01074
    01075         qm.fields.BooleanField(
    01076             name = "use_temp_dir",
    01077             title = "Use temporary directory",
    01078             description = """Use temporary directory.
    01079
    01080             If set to true, use a temporary directory as working directory.
    01081             """,
    01082             default_value="false"
    01083             ),
    01084         ]
    01085
    01086     def PlatformIsNotSupported(self, context, result):
    01087         platform = self.GetPlatform()
    01088         unsupported = [ re.compile(x)
    01089                         for x in [ str(y).strip()
    01090                                    for y in self.unsupported_platforms ]
    01091                         if x
    01092                        ]
    01093         for p_re in unsupported:
    01094             if p_re.search(platform):
    01095                 result.SetOutcome(result.UNTESTED)
    01096                 result[result.CAUSE] = 'Platform not supported.'
    01097                 return True
    01098         return False
    01099
    01100     def GetPlatform(self):
    01101         """
    01102         Return the platform Id defined in CMTCONFIG or SCRAM_ARCH.
    01103         """
    01104         arch = "None"
    01105         # check architecture name
    01106         if "CMTCONFIG" in os.environ:
    01107             arch = os.environ["CMTCONFIG"]
    01108         elif "SCRAM_ARCH" in os.environ:
    01109             arch = os.environ["SCRAM_ARCH"]
    01110         return arch
    01111
    01112     def _expandReferenceFileName(self, reffile):
    01113         # if no file is passed, do nothing
    01114         if not reffile:
    01115             return ""
    01116
    01117         reference = os.path.normpath(os.path.expandvars(reffile))
    01118         # old-style platform-specific reference name
    01119         spec_ref = reference[:-3] + self.GetPlatform()[0:3] + reference[-3:]
    01120         if os.path.isfile(spec_ref):
    01121             reference = spec_ref
    01122         else: # look for new-style platform specific reference files:
    01123             # get all the files whose name start with the reference filename
    01124             dirname, basename = os.path.split(reference)
    01125             if not dirname: dirname = '.'
    01126             head = basename + "."
    01127             head_len = len(head)
    01128             platform = self.GetPlatform()
    01129             candidates = []
    01130             for f in os.listdir(dirname):
    01131                 if f.startswith(head) and platform.startswith(f[head_len:]):
    01132                     candidates.append( (len(f) - head_len, f) )
    01133             if candidates: # take the one with highest matching
    01134                 candidates.sort()
    01135                 reference = os.path.join(dirname, candidates[-1][1])
    01136         return reference
    01137
    01138     def CheckTTreesSummaries(self, stdout, result, causes,
    01139                              trees_dict = None,
    01140                              ignore = r"Basket|.*size|Compression"):
    01141         """
    01142         Compare the TTree summaries in stdout with the ones in trees_dict or in
    01143         the reference file. By default ignore the size, compression and basket
    01144         fields.
    01145         The presence of TTree summaries when none is expected is not a failure.
    01146         """
    01147         if trees_dict is None:
    01148             reference = self._expandReferenceFileName(self.reference)
    01149             # call the validator if the file exists
    01150             if reference and os.path.isfile(reference):
    01151                 trees_dict = findTTreeSummaries(open(reference).read())
    01152             else:
    01153                 trees_dict = {}
    01154
    01155         from pprint import PrettyPrinter
    01156         pp = PrettyPrinter()
    01157         if trees_dict:
    01158             result["GaudiTest.TTrees.expected"] = result.Quote(pp.pformat(trees_dict))
    01159             if ignore:
    01160                 result["GaudiTest.TTrees.ignore"] = result.Quote(ignore)
    01161
    01162         trees = findTTreeSummaries(stdout)
    01163         failed = cmpTreesDicts(trees_dict, trees, ignore)
    01164         if failed:
    01165             causes.append("trees summaries")
    01166             msg = "%s: %s != %s" % getCmpFailingValues(trees_dict, trees, failed)
    01167             result["GaudiTest.TTrees.failure_on"] = result.Quote(msg)
    01168             result["GaudiTest.TTrees.found"] = result.Quote(pp.pformat(trees))
    01169
    01170         return causes
    01171
    01172     def CheckHistosSummaries(self, stdout, result, causes,
    01173                              dict = None,
    01174                              ignore = None):
    01175         """
    01176         Compare the TTree summaries in stdout with the ones in trees_dict or in
    01177         the reference file. By default ignore the size, compression and basket
    01178         fields.
    01179         The presence of TTree summaries when none is expected is not a failure.
    01180         """
    01181         if dict is None:
    01182             reference = self._expandReferenceFileName(self.reference)
    01183             # call the validator if the file exists
    01184             if reference and os.path.isfile(reference):
    01185                 dict = findHistosSummaries(open(reference).read())
    01186             else:
    01187                 dict = {}
    01188
    01189         from pprint import PrettyPrinter
    01190         pp = PrettyPrinter()
    01191         if dict:
    01192             result["GaudiTest.Histos.expected"] = result.Quote(pp.pformat(dict))
    01193             if ignore:
    01194                 result["GaudiTest.Histos.ignore"] = result.Quote(ignore)
    01195
    01196         histos = findHistosSummaries(stdout)
    01197         failed = cmpTreesDicts(dict, histos, ignore)
    01198         if failed:
    01199             causes.append("histos summaries")
    01200             msg = "%s: %s != %s" % getCmpFailingValues(dict, histos, failed)
    01201             result["GaudiTest.Histos.failure_on"] = result.Quote(msg)
    01202             result["GaudiTest.Histos.found"] = result.Quote(pp.pformat(histos))
    01203
    01204         return causes
    01205
    01206     def ValidateWithReference(self, stdout, stderr, result, causes, preproc = None):
    01207         """
    01208         Default validation action: compare standard output and error to the
    01209         reference files.
    01210         """
    01211         # set the default output preprocessor
    01212         if preproc is None:
    01213             preproc = normalizeExamples
    01214         # check standard output
    01215         reference = self._expandReferenceFileName(self.reference)
    01216         # call the validator if the file exists
    01217         if reference and os.path.isfile(reference):
    01218             result["GaudiTest.output_reference"] = reference
    01219             causes += ReferenceFileValidator(reference,
    01220                                              "standard output",
    01221                                              "GaudiTest.output_diff",
    01222                                              preproc = preproc)(stdout, result)
    01223
    01224         # Compare TTree summaries
    01225         causes = self.CheckTTreesSummaries(stdout, result, causes)
    01226         causes = self.CheckHistosSummaries(stdout, result, causes)
    01227
    01228         if causes: # Write a new reference file for stdout
    01229             try:
    01230                 newref = open(reference + ".new","w")
    01231                 # sanitize newlines
    01232                 for l in stdout.splitlines():
    01233                     newref.write(l.rstrip() + '\n')
    01234                 del newref # flush and close
    01235             except IOError:
    01236                 # Ignore IO errors when trying to update reference files
    01237                 # because we may be in a read-only filesystem
    01238                 pass
    01239
    01240         # check standard error
    01241         reference = self._expandReferenceFileName(self.error_reference)
    01242         # call the validator if we have a file to use
    01243         if reference and os.path.isfile(reference):
    01244             result["GaudiTest.error_reference"] = reference
    01245             newcauses = ReferenceFileValidator(reference,
    01246                                                "standard error",
    01247                                                "GaudiTest.error_diff",
    01248                                                preproc = preproc)(stderr, result)
    01249             causes += newcauses
    01250             if newcauses: # Write a new reference file for stdedd
    01251                 newref = open(reference + ".new","w")
    01252                 # sanitize newlines
    01253                 for l in stderr.splitlines():
    01254                     newref.write(l.rstrip() + '\n')
    01255                 del newref # flush and close
    01256         else:
    01257             causes += BasicOutputValidator(self.stderr,
    01258                                            "standard error",
    01259                                            "ExecTest.expected_stderr")(stderr, result)
    01260
    01261         return causes
    01262
    01263     def ValidateOutput(self, stdout, stderr, result):
    01264         causes = []
    01265         # if the test definition contains a custom validator, use it
    01266         if self.validator.strip() != "":
    01267             class CallWrapper(object):
    01268                 """
    01269                 Small wrapper class to dynamically bind some default arguments
    01270                 to a callable.
    01271                 """
    01272                 def __init__(self, callable, extra_args = {}):
    01273                     self.callable = callable
    01274                     self.extra_args = extra_args
    01275                     # get the list of names of positional arguments
    01276                     from inspect import getargspec
    01277                     self.args_order = getargspec(callable)[0]
    01278                     # Remove "self" from the list of positional arguments
    01279                     # since it is added automatically
    01280                     if self.args_order[0] == "self":
    01281                         del self.args_order[0]
    01282                 def __call__(self, *args, **kwargs):
    01283                     # Check which positional arguments are used
    01284                     positional = self.args_order[:len(args)]
    01285
    01286                     kwargs = dict(kwargs) # copy the arguments dictionary
    01287                     for a in self.extra_args:
    01288                         # use "extra_args" for the arguments not specified as
    01289                         # positional or keyword
    01290                         if a not in positional and a not in kwargs:
    01291                             kwargs[a] = self.extra_args[a]
    01292                     return apply(self.callable, args, kwargs)
    01293             # local names to be exposed in the script
    01294             exported_symbols = {"self":self,
    01295                                 "stdout":stdout,
    01296                                 "stderr":stderr,
    01297                                 "result":result,
    01298                                 "causes":causes,
    01299                                 "findReferenceBlock":
    01300                                     CallWrapper(findReferenceBlock, {"stdout":stdout,
    01301                                                                      "result":result,
    01302                                                                      "causes":causes}),
    01303                                 "validateWithReference":
    01304                                     CallWrapper(self.ValidateWithReference, {"stdout":stdout,
    01305                                                                              "stderr":stderr,
    01306                                                                              "result":result,
    01307                                                                              "causes":causes}),
    01308                                 "countErrorLines":
    01309                                     CallWrapper(countErrorLines, {"stdout":stdout,
    01310                                                                   "result":result,
    01311                                                                   "causes":causes}),
    01312                                 "checkTTreesSummaries":
    01313                                     CallWrapper(self.CheckTTreesSummaries, {"stdout":stdout,
    01314                                                                             "result":result,
    01315                                                                             "causes":causes}),
    01316                                 "checkHistosSummaries":
    01317                                     CallWrapper(self.CheckHistosSummaries, {"stdout":stdout,
    01318                                                                             "result":result,
    01319                                                                             "causes":causes}),
    01320
    01321                                 }
    01322             exec self.validator in globals(), exported_symbols
    01323         else:
    01324             self.ValidateWithReference(stdout, stderr, result, causes)
    01325
    01326         return causes
    01327
    01328     def DumpEnvironment(self, result):
    01329         """
    01330         Add the content of the environment to the result object.
    01331
    01332         Copied from the QMTest class of COOL.
    01333         """
    01334         vars = os.environ.keys()
    01335         vars.sort()
    01336         result['GaudiTest.environment'] = \
    01337             result.Quote('\n'.join(["%s=%s"%(v,os.environ[v]) for v in vars]))
    01338
    01339     def _find_program(self,prog):
    01340         # check if it is an absolute path or the file can be found
    01341         # from the local directory, otherwise search for it in PATH
    01342         if not os.path.isabs(prog) and not os.path.isfile(prog):
    01343             for d in os.environ["PATH"].split(os.pathsep):
    01344                 p = os.path.join(d,prog)
    01345                 if os.path.isfile(p):
    01346                     return p
    01347         return prog
    01348
    01349     def Run(self, context, result):
    01350         """Run the test.
    01351
    01352         'context' -- A 'Context' giving run-time parameters to the
    01353         test.
    01354
    01355         'result' -- A 'Result' object.  The outcome will be
    01356         'Result.PASS' when this method is called.  The 'result' may be
    01357         modified by this method to indicate outcomes other than
    01358         'Result.PASS' or to add annotations."""
    01359
    01360         # Check if the platform is supported
    01361         if self.PlatformIsNotSupported(context, result):
    01362             return
    01363
    01364         # Prepare program name and arguments (expanding variables, and converting to absolute)
    01365         if self.program:
    01366             prog = rationalizepath(self.program)
    01367         elif "GAUDIEXE" in os.environ:
    01368             prog = os.environ["GAUDIEXE"]
    01369         else:
    01370             prog = "Gaudi.exe"
    01371         self.program = prog
    01372
    01373         dummy, prog_ext = os.path.splitext(prog)
    01374         if prog_ext not in [ ".exe", ".py", ".bat" ] and self.GetPlatform()[0:3] == "win":
    01375             prog += ".exe"
    01376             prog_ext = ".exe"
    01377
    01378         prog = self._find_program(prog)
    01379
    01380         # Convert paths to absolute paths in arguments and reference files
    01381         args = map(rationalizepath, self.args)
    01382         self.reference = rationalizepath(self.reference)
    01383         self.error_reference = rationalizepath(self.error_reference)
    01384
    01385
    01386         # check if the user provided inline options
    01387         tmpfile = None
    01388         if self.options.strip():
    01389             ext = ".opts"
    01390             if re.search(r"from\s*Gaudi.Configuration\s*import\s*\*", self.options):
    01391                 ext = ".py"
    01392             tmpfile = TempFile(ext)
    01393             tmpfile.writelines("\n".join(self.options.splitlines()))
    01394             tmpfile.flush()
    01395             args.append(tmpfile.name)
    01396             result["GaudiTest.options"] = result.Quote(self.options)
    01397
    01398         # if the program is a python file, execute it through python
    01399         if prog_ext == ".py":
    01400             args.insert(0,prog)
    01401             if self.GetPlatform()[0:3] == "win":
    01402                 prog = self._find_program("python.exe")
    01403             else:
    01404                 prog = self._find_program("python")
    01405
    01406         # Change to the working directory if specified or to the default temporary
    01407         origdir = os.getcwd()
    01408         if self.workdir:
    01409             os.chdir(str(os.path.normpath(os.path.expandvars(self.workdir))))
    01410         elif self.use_temp_dir == "true":
    01411             if "QMTEST_TMPDIR" in os.environ:
    01412                 os.chdir(os.environ["QMTEST_TMPDIR"])
    01413             elif "qmtest.tmpdir" in context:
    01414                 os.chdir(context["qmtest.tmpdir"])
    01415
    01416         if "QMTEST_IGNORE_TIMEOUT" not in os.environ:
    01417             self.timeout = max(self.timeout,600)
    01418         else:
    01419             self.timeout = -1
    01420
    01421         try:
    01422             # Generate eclipse.org debug launcher for the test
    01423             self._CreateEclipseLaunch(prog, args, destdir = origdir)
    01424             # Run the test
    01425             self.RunProgram(prog,
    01426                             [ prog ] + args,
    01427                             context, result)
    01428             # Record the content of the enfironment for failing tests
    01429             if result.GetOutcome() not in [ result.PASS ]:
    01430                 self.DumpEnvironment(result)
    01431         finally:
    01432             # revert to the original directory
    01433             os.chdir(origdir)
    01434
    01435     def RunProgram(self, program, arguments, context, result):
    01436         """Run the 'program'.
    01437
    01438         'program' -- The path to the program to run.
    01439
    01440         'arguments' -- A list of the arguments to the program.  This
    01441         list must contain a first argument corresponding to 'argv[0]'.
    01442
    01443         'context' -- A 'Context' giving run-time parameters to the
    01444         test.
    01445
    01446         'result' -- A 'Result' object.  The outcome will be
    01447         'Result.PASS' when this method is called.  The 'result' may be
    01448         modified by this method to indicate outcomes other than
    01449         'Result.PASS' or to add annotations.
    01450
    01451         @attention: This method has been copied from command.ExecTestBase
    01452                     (QMTest 2.3.0) and modified to keep stdout and stderr
    01453                     for tests that have been terminated by a signal.
    01454                     (Fundamental for debugging in the Application Area)
    01455         """
    01456
    01457         # Construct the environment.
    01458         environment = self.MakeEnvironment(context)
    01459         # Create the executable.
    01460         if self.timeout >= 0:
    01461             timeout = self.timeout
    01462         else:
    01463             # If no timeout was specified, we sill run this process in a
    01464             # separate process group and kill the entire process group
    01465             # when the child is done executing.  That means that
    01466             # orphaned child processes created by the test will be
    01467             # cleaned up.
    01468             timeout = -2
    01469         e = GaudiFilterExecutable(self.stdin, timeout)
    01470         # Run it.
    01471         exit_status = e.Run(arguments, environment, path = program)
    01472         # Get the stack trace from the temporary file (if present)
    01473         if e.stack_trace_file and os.path.exists(e.stack_trace_file):
    01474             stack_trace = open(e.stack_trace_file).read()
    01475             os.remove(e.stack_trace_file)
    01476         else:
    01477             stack_trace = None
    01478         if stack_trace:
    01479             result["ExecTest.stack_trace"] = result.Quote(stack_trace)
    01480
    01481         # If the process terminated normally, check the outputs.
    01482         if sys.platform == "win32" or os.WIFEXITED(exit_status):
    01483             # There are no causes of failure yet.
    01484             causes = []
    01485             # The target program terminated normally.  Extract the
    01486             # exit code, if this test checks it.
    01487             if self.exit_code is None:
    01488                 exit_code = None
    01489             elif sys.platform == "win32":
    01490                 exit_code = exit_status
    01491             else:
    01492                 exit_code = os.WEXITSTATUS(exit_status)
    01493             # Get the output generated by the program.
    01494             stdout = e.stdout
    01495             stderr = e.stderr
    01496             # Record the results.
    01497             result["ExecTest.exit_code"] = str(exit_code)
    01498             result["ExecTest.stdout"] = result.Quote(stdout)
    01499             result["ExecTest.stderr"] = result.Quote(stderr)
    01500             # Check to see if the exit code matches.
    01501             if exit_code != self.exit_code:
    01502                 causes.append("exit_code")
    01503                 result["ExecTest.expected_exit_code"] \
    01504                     = str(self.exit_code)
    01505             # Validate the output.
    01506             causes += self.ValidateOutput(stdout, stderr, result)
    01507             # If anything went wrong, the test failed.
    01508             if causes:
    01509                 result.Fail("Unexpected %s." % string.join(causes, ", "))
    01510         elif os.WIFSIGNALED(exit_status):
    01511             # The target program terminated with a signal.  Construe
    01512             # that as a test failure.
    01513             signal_number = str(os.WTERMSIG(exit_status))
    01514             if not stack_trace:
    01515                 result.Fail("Program terminated by signal.")
    01516             else:
    01517                 # The presence of stack_trace means tha we stopped the job because
    01518                 # of a time-out
    01519                 result.Fail("Exceeded time limit (%ds), terminated." % timeout)
    01520             result["ExecTest.signal_number"] = signal_number
    01521             result["ExecTest.stdout"] = result.Quote(e.stdout)
    01522             result["ExecTest.stderr"] = result.Quote(e.stderr)
    01523         elif os.WIFSTOPPED(exit_status):
    01524             # The target program was stopped.  Construe that as a
    01525             # test failure.
    01526             signal_number = str(os.WSTOPSIG(exit_status))
    01527             if not stack_trace:
    01528                 result.Fail("Program stopped by signal.")
    01529             else:
    01530                 # The presence of stack_trace means tha we stopped the job because
    01531                 # of a time-out
    01532                 result.Fail("Exceeded time limit (%ds), stopped." % timeout)
    01533             result["ExecTest.signal_number"] = signal_number
    01534             result["ExecTest.stdout"] = result.Quote(e.stdout)
    01535             result["ExecTest.stderr"] = result.Quote(e.stderr)
    01536         else:
    01537             # The target program terminated abnormally in some other
    01538             # manner.  (This shouldn't normally happen...)
    01539             result.Fail("Program did not terminate normally.")
    01540
    01541         # Marco Cl.: This is a special trick to fix a "problem" with the output
    01542         # of gaudi jobs when they use colors
    01543         esc = '\x1b'
    01544         repr_esc = '\\x1b'
    01545         result["ExecTest.stdout"] = result["ExecTest.stdout"].replace(esc,repr_esc)
    01546         # TODO: (MCl) improve the hack for colors in standard output
    01547         #             may be converting them to HTML tags
    01548
    01549     def _CreateEclipseLaunch(self, prog, args, destdir = None):
    01550         # Find the project name used in ecplise.
    01551         # The name is in a file called ".project" in one of the parent directories
    01552         projbasedir = os.path.normpath(destdir)
    01553         while not os.path.exists(os.path.join(projbasedir, ".project")):
    01554             oldprojdir = projbasedir
    01555             projbasedir = os.path.normpath(os.path.join(projbasedir, os.pardir))
    01556             # FIXME: the root level is invariant when trying to go up one level,
    01557             #        but it must be cheched on windows
    01558             if oldprojdir == projbasedir:
    01559                 # If we cannot find a .project, so no point in creating a .launch file
    01560                 return
    01561         # Use ElementTree to parse the XML file
    01562         from xml.etree import ElementTree as ET
    01563         t = ET.parse(os.path.join(projbasedir, ".project"))
    01564         projectName = t.find("name").text
    01565
    01566         # prepare the name/path of the generated file
    01567         destfile = "%s.launch" % self._Runnable__id
    01568         if destdir:
    01569             destfile = os.path.join(destdir, destfile)
    01570
    01571         if self.options.strip():
    01572             # this means we have some custom options in the qmt file, so we have
    01573             # to copy them from the temporary file at the end of the arguments
    01574             # in another file
    01575             tempfile = args.pop()
    01576             optsfile = destfile + os.path.splitext(tempfile)[1]
    01577             shutil.copyfile(tempfile, optsfile)
    01578             args.append(optsfile)
    01579
    01580         # prepare the data to insert in the XML file
    01581         from xml.sax.saxutils import quoteattr # useful to quote XML special chars
    01582         data = {}
    01583         # Note: the "quoteattr(k)" is not needed because special chars cannot be part of a variable name,
    01584         # but it doesn't harm.
    01585         data["environment"] = "\n".join(['<mapEntry key=%s value=%s/>' % (quoteattr(k), quoteattr(v))
    01586                                          for k, v in os.environ.iteritems()])
    01587
    01588         data["exec"] = which(prog)
    01589
    01590         data["args"] = "&#10;".join(map(rationalizepath, args))
    01591
    01592         if not self.use_temp_dir:
    01593             data["workdir"] = os.getcwd()
    01594         else:
    01595             # If the test is using a tmporary directory, it is better to run it
    01596             # in the same directory as the .launch file when debugged in eclipse
    01597             data["workdir"] = destdir
    01598
    01599         data["project"] = projectName.strip()
    01600
    01601         # Template for the XML file, based on eclipse 3.4
    01602         xml = """<?xml version="1.0" encoding="UTF-8" standalone="no"?>
    01603 <launchConfiguration type="org.eclipse.cdt.launch.applicationLaunchType">
    01604 <booleanAttribute key="org.eclipse.cdt.debug.mi.core.AUTO_SOLIB" value="true"/>
    01605 <listAttribute key="org.eclipse.cdt.debug.mi.core.AUTO_SOLIB_LIST"/>
    01606 <stringAttribute key="org.eclipse.cdt.debug.mi.core.DEBUG_NAME" value="gdb"/>
    01607 <stringAttribute key="org.eclipse.cdt.debug.mi.core.GDB_INIT" value=".gdbinit"/>
    01608 <listAttribute key="org.eclipse.cdt.debug.mi.core.SOLIB_PATH"/>
    01609 <booleanAttribute key="org.eclipse.cdt.debug.mi.core.STOP_ON_SOLIB_EVENTS" value="false"/>
    01610 <stringAttribute key="org.eclipse.cdt.debug.mi.core.protocol" value="mi"/>
    01611 <stringAttribute key="org.eclipse.cdt.launch.COREFILE_PATH" value=""/>
    01612 <stringAttribute key="org.eclipse.cdt.launch.DEBUGGER_ID" value="org.eclipse.cdt.debug.mi.core.CDebugger"/>
    01613 <stringAttribute key="org.eclipse.cdt.launch.DEBUGGER_REGISTER_GROUPS" value=""/>
    01614 <stringAttribute key="org.eclipse.cdt.launch.DEBUGGER_START_MODE" value="run"/>
    01615 <booleanAttribute key="org.eclipse.cdt.launch.DEBUGGER_STOP_AT_MAIN" value="true"/>
    01616 <stringAttribute key="org.eclipse.cdt.launch.DEBUGGER_STOP_AT_MAIN_SYMBOL" value="main"/>
    01617 <booleanAttribute key="org.eclipse.cdt.launch.ENABLE_REGISTER_BOOKKEEPING" value="false"/>
    01618 <booleanAttribute key="org.eclipse.cdt.launch.ENABLE_VARIABLE_BOOKKEEPING" value="false"/>
    01619 <stringAttribute key="org.eclipse.cdt.launch.FORMAT" value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot; standalone=&quot;no&quot;?&gt;&lt;contentList/&gt;"/>
    01620 <stringAttribute key="org.eclipse.cdt.launch.GLOBAL_VARIABLES" value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot; standalone=&quot;no&quot;?&gt;&#10;&lt;globalVariableList/&gt;&#10;"/>
    01621 <stringAttribute key="org.eclipse.cdt.launch.MEMORY_BLOCKS" value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot; standalone=&quot;no&quot;?&gt;&#10;&lt;memoryBlockExpressionList/&gt;&#10;"/>
    01622 <stringAttribute key="org.eclipse.cdt.launch.PROGRAM_ARGUMENTS" value="%(args)s"/>
    01623 <stringAttribute key="org.eclipse.cdt.launch.PROGRAM_NAME" value="%(exec)s"/>
    01624 <stringAttribute key="org.eclipse.cdt.launch.PROJECT_ATTR" value="%(project)s"/>
    01625 <stringAttribute key="org.eclipse.cdt.launch.PROJECT_BUILD_CONFIG_ID_ATTR" value=""/>
    01626 <stringAttribute key="org.eclipse.cdt.launch.WORKING_DIRECTORY" value="%(workdir)s"/>
    01627 <booleanAttribute key="org.eclipse.cdt.launch.use_terminal" value="true"/>
    01628 <listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">
    01629 <listEntry value="/%(project)s"/>
    01630 </listAttribute>
    01631 <listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">
    01632 <listEntry value="4"/>
    01633 </listAttribute>
    01634 <booleanAttribute key="org.eclipse.debug.core.appendEnvironmentVariables" value="false"/>
    01635 <mapAttribute key="org.eclipse.debug.core.environmentVariables">
    01636 %(environment)s
    01637 </mapAttribute>
    01638 <mapAttribute key="org.eclipse.debug.core.preferred_launchers">
    01639 <mapEntry key="[debug]" value="org.eclipse.cdt.cdi.launch.localCLaunch"/>
    01640 </mapAttribute>
    01641 <listAttribute key="org.eclipse.debug.ui.favoriteGroups">
    01642 <listEntry value="org.eclipse.debug.ui.launchGroup.debug"/>
    01643 </listAttribute>
    01644 </launchConfiguration>
    01645 """ % data
    01646
    01647         # Write the output file
    01648         open(destfile, "w").write(xml)
    01649         #open(destfile + "_copy.xml", "w").write(xml)
    01650
    01651
    01652 try:
    01653     import json
    01654 except ImportError:
    01655     # Hack: json is not available, so I use the version we ship
    01656     sys.path.append(os.path.dirname(__file__))
    01657     import simplejson as json
    01658
    01659 class HTMLResultStream(ResultStream):
    01660     """An 'HTMLResultStream' writes its output to a set of HTML files.
    01661
    01662     The argument 'dir' is used to select the destination directory for the HTML
    01663     report.
    01664     The destination directory may already contain the report from a previous run
    01665     (for example of a different package), in which case it will be extended to
    01666     include the new data.
    01667     """
    01668     arguments = [
    01669         qm.fields.TextField(
    01670             name = "dir",
    01671             title = "Destination Directory",
    01672             description = """The name of the directory.
    01673
    01674             All results will be written to the directory indicated.""",
    01675             verbatim = "true",
    01676             default_value = ""),
    01677     ]
    01678
    01679     def __init__(self, arguments = None, **args):
    01680         """Prepare the destination directory.
    01681
    01682         Creates the destination directory and store in it some preliminary
    01683         annotations and the static files found in the template directory
    01684         'html_report'.
    01685         """
    01686         ResultStream.__init__(self, arguments, **args)
    01687         self._summary = []
    01688         self._summaryFile = os.path.join(self.dir, "summary.json")
    01689         self._annotationsFile = os.path.join(self.dir, "annotations.json")
    01690         # Prepare the destination directory using the template
    01691         templateDir = os.path.join(os.path.dirname(__file__), "html_report")
    01692         if not os.path.isdir(self.dir):
    01693             os.makedirs(self.dir)
    01694         # Copy the files in the template directory excluding the directories
    01695         for f in os.listdir(templateDir):
    01696             src = os.path.join(templateDir, f)
    01697             dst = os.path.join(self.dir, f)
    01698             if not os.path.isdir(src) and not os.path.exists(dst):
    01699                 shutil.copy(src, dst)
    01700         # Add some non-QMTest attributes
    01701         if "CMTCONFIG" in os.environ:
    01702             self.WriteAnnotation("cmt.cmtconfig", os.environ["CMTCONFIG"])
    01703         import socket
    01704         self.WriteAnnotation("hostname", socket.gethostname())
    01705
    01706     def _updateSummary(self):
    01707         """Helper function to extend the global summary file in the destination
    01708         directory.
    01709         """
    01710         if os.path.exists(self._summaryFile):
    01711             oldSummary = json.load(open(self._summaryFile))
    01712         else:
    01713             oldSummary = []
    01714         ids = set([ i["id"] for i in self._summary ])
    01715         newSummary = [ i for i in oldSummary if i["id"] not in ids ]
    01716         newSummary.extend(self._summary)
    01717         json.dump(newSummary, open(self._summaryFile, "w"),
    01718                   sort_keys = True)
    01719
    01720     def WriteAnnotation(self, key, value):
    01721         """Writes the annotation to the annotation file.
    01722         If the key is already present with a different value, the value becomes
    01723         a list and the new value is appended to it, except for start_time and
    01724         end_time.
    01725         """
    01726         # Initialize the annotation dict from the file (if present)
    01727         if os.path.exists(self._annotationsFile):
    01728             annotations = json.load(open(self._annotationsFile))
    01729         else:
    01730             annotations = {}
    01731         # hack because we do not have proper JSON support
    01732         key, value = map(str, [key, value])
    01733         if key == "qmtest.run.start_time":
    01734             # Special handling of the start time:
    01735             # if we are updating a result, we have to keep the original start
    01736             # time, but remove the original end time to mark the report to be
    01737             # in progress.
    01738             if key not in annotations:
    01739                 annotations[key] = value
    01740             if "qmtest.run.end_time" in annotations:
    01741                 del annotations["qmtest.run.end_time"]
    01742         else:
    01743             # All other annotations are added to a list
    01744             if key in annotations:
    01745                 old = annotations[key]
    01746                 if type(old) is list:
    01747                     if value not in old:
    01748                         annotations[key].append(value)
    01749                 elif value != old:
    01750                     annotations[key] = [old, value]
    01751             else:
    01752                 annotations[key] = value
    01753         # Write the new annotations file
    01754         json.dump(annotations, open(self._annotationsFile, "w"),
    01755                   sort_keys = True)
    01756
    01757     def WriteResult(self, result):
    01758         """Prepare the test result directory in the destination directory storing
    01759         into it the result fields.
    01760         A summary of the test result is stored both in a file in the test directory
    01761         and in the global summary file.
    01762         """
    01763         summary = {}
    01764         summary["id"] = result.GetId()
    01765         summary["outcome"] = result.GetOutcome()
    01766         summary["cause"] = result.GetCause()
    01767         summary["fields"] = result.keys()
    01768         summary["fields"].sort()
    01769
    01770         # Since we miss proper JSON support, I hack a bit
    01771         for f in ["id", "outcome", "cause"]:
    01772             summary[f] = str(summary[f])
    01773         summary["fields"] = map(str, summary["fields"])
    01774
    01775         self._summary.append(summary)
    01776
    01777         # format:
    01778         # testname/summary.json
    01779         # testname/field1
    01780         # testname/field2
    01781         testOutDir = os.path.join(self.dir, summary["id"])
    01782         if not os.path.isdir(testOutDir):
    01783             os.makedirs(testOutDir)
    01784         json.dump(summary, open(os.path.join(testOutDir, "summary.json"), "w"),
    01785                   sort_keys = True)
    01786         for f in summary["fields"]:
    01787             open(os.path.join(testOutDir, f), "w").write(result[f])
    01788
    01789         self._updateSummary()
    01790
    01791     def Summarize(self):
    01792         # Not implemented.
    01793         pass