| # Gaudi Framework, version v21r8 |

Home |  | Generated: 17 Mar 2010

# qmtest_summarize.py

Go to the documentation of this file.

    00001 #!/usr/bin/env python
    00002 #
    00003 # Simple script to loop over the packages providing tests and collect
    00004 # the result summaries
    00005 #
    00006 # @author: Marco Clemencic <marco.clemencic@cern.ch>
    00007 # @date: 3/11/2007
    00008 #
    00009 import os, sys, re
    00010
    00011 def parse_summary(output):
    00012     report = { "results":[], "not_passed":[], "statistics": {} }
    00013     current_block = None
    00014     for l in output.splitlines():
    00015         if l.startswith("--- TEST RESULTS"):
    00016             current_block = "results"
    00017         elif l.startswith("--- TESTS THAT DID NOT PASS"):
    00018             current_block = "not_passed"
    00019         elif l.startswith("--- STATISTICS"):
    00020             current_block = "statistics"
    00021         else:
    00022             if current_block in ["results","not_passed"]:
    00023                 report[current_block].append(l)
    00024             elif current_block == "statistics":
    00025                 tkns = l.split()
    00026                 if tkns:
    00027                     report["statistics"][tkns[-1]] = int(tkns[0])
    00028
    00029     report["results"] = report["results"][:-1]
    00030     report["not_passed"] = report["not_passed"][:-2]
    00031     if report["not_passed"][-1].strip() == "None.":
    00032         report["not_passed"] = []
    00033     return report
    00034
    00035 def main():
    00036     # I assume that this script is run from a CMT action, which implies few other
    00037     # things:
    00038     #   - we are in the cmt directory of a package
    00039     #   - CMTCONFIG etc. are correctly defined
    00040     #   - we can call cmt
    00041
    00042     # If the use specifies a directory where to collect all the results
    00043     # (e.g. because running from a read-only location) we must use it
    00044     if "QMTESTRESULTSDIR" in os.environ:
    00045         outputdir = os.path.normpath(os.path.expandvars(os.environ["QMTESTRESULTSDIR"]))
    00046     else:
    00047         outputdir = None
    00048
    00049     ## Directory where to store the report
    00050     #if "CMTINSTALLAREA" not in os.environ:
    00051     #    os.environ["CMTINSTALLAREA"] = os.popen("cmt show macro_value CMTINSTALLAREA","r").read().strip()
    00052
    00053     # Find the packages (their cmt dirs)
    00054     cmt_br_header_re = re.compile(r"^# Now trying.*in (.*) \([0-9]+/[0-9]+\)")
    00055     dirs = []
    00056
    00057     # Hack to work around a problem with cmt broadcast and CMTUSERCONTEXT.
    00058     old_CMTUSERCONTEXT = None
    00059     if "CMTUSERCONTEXT" in os.environ:
    00060         old_CMTUSERCONTEXT = os.environ["CMTUSERCONTEXT"]
    00061         del os.environ["CMTUSERCONTEXT"]
    00062
    00063     for l in os.popen("cmt broadcast","r"):
    00064         m = cmt_br_header_re.match(l)
    00065         if m:
    00066             dirs.append(m.group(1))
    00067
    00068     # Restore original CMTUSERCONTEXT, if it was set.
    00069     if old_CMTUSERCONTEXT is not None:
    00070         os.environ["CMTUSERCONTEXT"] = old_CMTUSERCONTEXT
    00071
    00072     # Find the results.qmr files
    00073     if not outputdir:
    00074         results = filter(os.path.isfile,[ os.path.realpath(os.path.join(d,'..',os.environ["CMTCONFIG"],"results.qmr")) for d in dirs ])
    00075     else:
    00076         results = []
    00077         for d in dirs:
    00078             pkg = os.path.basename(os.path.dirname(d))
    00079             r = os.path.join(outputdir, "%s.%s.qmr" % (pkg, os.environ.get("CMTCONFIG", "noConfig")))
    00080             if os.path.isfile(r):
    00081                 results.append(r)
    00082
    00083     # Check if the result files have been found
    00084     if len(results) == 0:
    00085         print "Warning: no result file found! (Did you run the tests?)"
    00086         sys.exit()
    00087
    00088     # parse and collect the summaries
    00089     # FIXME: (MCl) with QMTest 2.3 we need a test database to be able to run "qmtest summarize", while it
    00090     #              is not needed with QMTest 2.4
    00091     from tempfile import mkdtemp
    00092     from shutil import rmtree
    00093     tmpdir = mkdtemp()
    00094     origdir = os.getcwd()
    00095     report = None
    00096     try:
    00097         os.chdir(tmpdir)
    00098         os.popen("qmtest create-tdb","r").read()
    00099
    00100         for r in results:
    00101             out = os.popen("qmtest summarize -f brief %s"%r,"r").read()
    00102             rep = parse_summary(out)
    00103             if report is None:
    00104                 report = rep
    00105             else:
    00106                 report["results"] += rep["results"]
    00107                 if rep["not_passed"]:
    00108                     report["not_passed"] += rep["not_passed"]
    00109                 for k in rep["statistics"]:
    00110                     if k in report["statistics"]:
    00111                         report["statistics"][k] += rep["statistics"][k]
    00112                     else:
    00113                         report["statistics"][k] = rep["statistics"][k]
    00114     finally:
    00115         os.chdir(origdir)
    00116         rmtree(tmpdir,ignore_errors=True)
    00117
    00118     if report is None:
    00119         print "Warning: I could not generate the report"
    00120         sys.exit()
    00121
    00122     # Finalize the report
    00123     report["results"].append('')
    00124     if not report["not_passed"]:
    00125         report["not_passed"] = ['','  None.']
    00126     report["not_passed"] += ['','']
    00127
    00128     statistics_output = ['--- STATISTICS ---------------------------------------------------------------','']
    00129     # order: total, FAIL, UNTESTED, PASS
    00130     #"       2        tests total"
    00131     tot = report["statistics"]["total"]
    00132     statistics_output.append("%8d        tests total"%(tot))
    00133     success = True
    00134     for k in [ "ERROR", "FAIL", "UNTESTED", "PASS" ]:
    00135         if k in report["statistics"]:
    00136             n = report["statistics"][k]
    00137             p = round(100. * n / tot)
    00138             statistics_output.append("%8d (%3d%%) tests %s"%(n,p,k))
    00139             if k in ["ERROR", "FAIL"]:
    00140                 success = False
    00141     statistics_output.append('')
    00142
    00143     results_output = ['--- TEST RESULTS -------------------------------------------------------------']
    00144     results_output += report["results"]
    00145
    00146     not_passed_output = ['--- TESTS THAT DID NOT PASS --------------------------------------------------']
    00147     if not report["not_passed"]:
    00148         not_passed_output
    00149     not_passed_output += report["not_passed"]
    00150
    00151     output = statistics_output + not_passed_output + results_output + not_passed_output + statistics_output
    00152     print '\n'.join(output)
    00153     return success
    00154
    00155 if __name__ == '__main__':
    00156     if not main():
    00157         # failure
    00158         sys.exit(1)