qmtest_summarize.py

Go to the documentation of this file.

#!/usr/bin/env python
#
# Simple script to loop over the packages providing tests and collect
# the result summaries
#
# @author: Marco Clemencic <marco.clemencic@cern.ch>
# @date: 3/11/2007
#
import os, sys, re

def parse_summary(output):
    report = { "results":[], "not_passed":[], "statistics": {} }
    current_block = None
    for l in output.splitlines():
        if l.startswith("--- TEST RESULTS"):
            current_block = "results"
        elif l.startswith("--- TESTS THAT DID NOT PASS"):
            current_block = "not_passed"
        elif l.startswith("--- STATISTICS"):
            current_block = "statistics"
        else:
            if current_block in ["results","not_passed"]:
                report[current_block].append(l)
            elif current_block == "statistics":
                tkns = l.split()
                if tkns:
                    report["statistics"][tkns[-1]] = int(tkns[0])
    
    report["results"] = report["results"][:-1]
    report["not_passed"] = report["not_passed"][:-2]
    if report["not_passed"][-1].strip() == "None.":
        report["not_passed"] = []
    return report

def main():
    # I assume that this script is run from a CMT action, which implies few other
    # things:
    #   - we are in the cmt directory of a package
    #   - CMTCONFIG etc. are correctly defined
    #   - we can call cmt
    
    # If the use specifies a directory where to collect all the results
    # (e.g. because running from a read-only location) we must use it
    if "QMTESTRESULTSDIR" in os.environ:
        outputdir = os.path.normpath(os.path.expandvars(os.environ["QMTESTRESULTSDIR"]))
    else:
        outputdir = None
    
    ## Directory where to store the report
    #if "CMTINSTALLAREA" not in os.environ:
    #    os.environ["CMTINSTALLAREA"] = os.popen("cmt show macro_value CMTINSTALLAREA","r").read().strip()
    
    # Find the packages (their cmt dirs)
    cmt_br_header_re = re.compile(r"^# Now trying.*in (.*) \([0-9]+/[0-9]+\)")
    dirs = []
    
    # Hack to work around a problem with cmt broadcast and CMTUSERCONTEXT.
    old_CMTUSERCONTEXT = None
    if "CMTUSERCONTEXT" in os.environ:
        old_CMTUSERCONTEXT = os.environ["CMTUSERCONTEXT"]
        del os.environ["CMTUSERCONTEXT"]
    
    for l in os.popen("cmt broadcast","r"):
        m = cmt_br_header_re.match(l)
        if m:
            dirs.append(m.group(1))
    
    # Restore original CMTUSERCONTEXT, if it was set. 
    if old_CMTUSERCONTEXT is not None:
        os.environ["CMTUSERCONTEXT"] = old_CMTUSERCONTEXT
    
    # Find the results.qmr files
    if not outputdir:
        results = filter(os.path.isfile,[ os.path.realpath(os.path.join(d,'..',os.environ["CMTCONFIG"],"results.qmr")) for d in dirs ])
    else:
        results = []
        for d in dirs:
            pkg = os.path.basename(os.path.dirname(d))
            r = os.path.join(outputdir, "%s.%s.qmr" % (pkg, os.environ.get("CMTCONFIG", "noConfig")))
            if os.path.isfile(r):
                results.append(r)
    
    # Check if the result files have been found
    if len(results) == 0:
        print "Warning: no result file found! (Did you run the tests?)"
        sys.exit()
    
    # parse and collect the summaries
    # FIXME: (MCl) with QMTest 2.3 we need a test database to be able to run "qmtest summarize", while it
    #              is not needed with QMTest 2.4
    from tempfile import mkdtemp
    from shutil import rmtree
    tmpdir = mkdtemp()
    origdir = os.getcwd()
    report = None
    try:
        os.chdir(tmpdir)
        os.popen("qmtest create-tdb","r").read()
    
        for r in results:
            out = os.popen("qmtest summarize -f brief %s"%r,"r").read()
            rep = parse_summary(out)
            if report is None:
                report = rep
            else:
                report["results"] += rep["results"]
                if rep["not_passed"]:
                    report["not_passed"] += rep["not_passed"]
                for k in rep["statistics"]:
                    if k in report["statistics"]:
                        report["statistics"][k] += rep["statistics"][k]
                    else:
                        report["statistics"][k] = rep["statistics"][k]
    finally:
        os.chdir(origdir)
        rmtree(tmpdir,ignore_errors=True)
    
    if report is None:
        print "Warning: I could not generate the report"
        sys.exit()
    
    # Finalize the report
    report["results"].append('')
    if not report["not_passed"]:
        report["not_passed"] = ['','  None.']
    report["not_passed"] += ['','']
    
    statistics_output = ['--- STATISTICS ---------------------------------------------------------------','']
    # order: total, FAIL, UNTESTED, PASS
    #"       2        tests total"
    tot = report["statistics"]["total"]
    statistics_output.append("%8d        tests total"%(tot))
    success = True 
    for k in [ "ERROR", "FAIL", "UNTESTED", "PASS" ]:
        if k in report["statistics"]:
            n = report["statistics"][k]
            p = round(100. * n / tot)
            statistics_output.append("%8d (%3d%%) tests %s"%(n,p,k))
            if k in ["ERROR", "FAIL"]:
                success = False
    statistics_output.append('')
    
    results_output = ['--- TEST RESULTS -------------------------------------------------------------']
    results_output += report["results"]
    
    not_passed_output = ['--- TESTS THAT DID NOT PASS --------------------------------------------------']
    if not report["not_passed"]:
        not_passed_output 
    not_passed_output += report["not_passed"]
    
    output = statistics_output + not_passed_output + results_output + not_passed_output + statistics_output
    print '\n'.join(output)
    return success

if __name__ == '__main__':
    if not main():
        # failure
        sys.exit(1)