| The Gaudi Framework v28r2p1 (f1a77ff4)

- Main Page
- Related Pages
- Modules
- Namespaces
- Classes
- Files

- File List
- File Members

compareRootHistos.py
Go to the documentation of this file.
1 #! /usr/bin/env python
2 from optparse import OptionParser
3 import re
4 import sys
5 # Well known trick
6 backupArgv=sys.argv[:]
7 sys.argv=[]
8 from ROOT import TFile
9 sys.argv=backupArgv
10
11 # The list of histograms not to check. Expressed as regexps
12 gRegexBlackList=[]
13 histos = ['TH1D', 'TH1F', 'TH2D', 'TH2F', 'TProfile']
14 ref = 'REFERENCE'
15 test = 'TEST'
16
17 # =================================================================================================
18 # Method : rec( o, path=None, lst=None )
19 #
20 # @param o : a ROOT object
21 # @param path : a string like a transient store path; ie '/stat/CaloPIDs/ECALPIDE'
22 # @param lst : a list to hold (path, object) tuples
23 #
24 # function : recursively pull apart a ROOT file, making a list of (path, TObject) tuples
25 # This is done by GetListOfKeys method, which lets one work down through directories
26 # until you hit the Histo at the end of the path. The list of tuples is returned
27 #
28 def rec( o, path=None, lst=None ) :
29 if not path : path = '/stat' ; lst = []
30 else : path = path + '/' + o.GetName()
31 lst.append( (path,o) )
32 if 'GetListOfKeys' in dir(o) :
33 keys = o.GetListOfKeys()
34 for k in keys :
35 name = k.GetName()
36 rec( o.Get(name), path, lst )
37 else :
38 pass
39 return lst
40 # =================================================================================================
41
42 # =================================================================================================
43 # Method : composition( t )
44 #
45 # @param t : a tuple of ( type, d ) where type is either 'REFERENCE' or 'TEST'
46 # and d is a dictionary of ROOT objects, with each key = ROOT path
47 #
48 # function : deduce the composition, (objects/histos) counts
49 #
50 def composition( t ) :
51 typ, d = t
52 hists = 0 ; objs = 0
53 for k in d.keys() :
54 if d[k].__class__.__name__ in histos : hists += 1
55 else : objs += 1
56 return objs, hists
57 # =================================================================================================
58
59 # =================================================================================================
60 # Method : comparePaths( t1, t2 )
61 #
62 # @param t1, t2 : a tuple of ( type, d ) where type is either 'REFERENCE' or 'TEST'
63 # and d is a dictionary of ROOT objects, with each key = ROOT path
64 #
65 # function : compare the paths between the two histo files. If the files are identical, they
66 # should have the same set of paths. The Test file should definitely have the
67 # same paths as the Reference. Perhaps the Reference file will have some more paths due
68 # to extra histos added as part of Application Sequencer finalisation
69 # Arguments t1 and t2 are checked and the test/reference auto-detected
70 #
71 def comparePaths( t1, t2 ) :
72 if t1[0] == ref : ds = t1[1] ; dp = t2[1]
73 elif t2[0] == ref : ds = t2[1] ; dp = t1[1]
74 else : print 'Neither tuple is Reference Root file reference?' ; return
75
76 dsks = ds.keys() ; dpks = dp.keys()
77 dsks.sort() ; dpks.sort()
78
79 sset = set( dsks )
80 pset = set( dpks )
81 os, hs = composition( (ref, ds) )
82 op, hp = composition( (test, dp) )
83 print '\n' + '='*80
84 print 'Comparison of Paths : Reference vs Test ROOT files'
85 print '-'*80
86 print 'Number of paths in Reference file : %i (objects, histos) = ( %i, %i )'%( len(dsks), os, hs )
87 print 'Number of paths in Test file : %i (objects, histos) = ( %i, %i )'%( len(dpks), op, hp )
88 matching = sset.intersection(pset)
89 matchingHistos = 0
90 for n in matching :
91 if ds[n].__class__.__name__ in histos : matchingHistos += 1
92 print '\nMatching paths : %i'%( len(matching) )
93 uSer = sset - pset
94 # work out histos unique to test file
95 uniqueReferenceHistos = 0
96 for n in uSer :
97 if ds[n].__class__.__name__ in histos : uniqueReferenceHistos += 1
98 print 'Paths unique to Reference file : %i ( %i Histos )'%( len(uSer), uniqueReferenceHistos )
99 if uSer :
100 for n in uSer : print '\t%s : \t%s'%( ds[n], n )
101 uPar = pset - sset
102 uniqueTestHistos = 0
103 for n in uPar :
104 if dp[n].__class__.__name__ in histos : uniqueTestHistos += 1
105 print 'Paths unique to Test file : %i ( %i Histos )'%( len(uPar), uniqueTestHistos )
106 if uPar :
107 for n in uPar : print '\t%s : \t%s'%( dp[n], n )
108 print 'Matching Histos to test : %i'%( matchingHistos )
109 print '='*80 + '\n'
110 return ( ((os,hs),(op,hp)), (uSer, uniqueReferenceHistos), (uPar, uniqueTestHistos), matchingHistos )
111 # =================================================================================================
112
113 # =================================================================================================
114 # Method : bin2binIdentity(h1,h2)
115 #
116 # @param h1, h2 : The two histogtams to compare
117 # function : Return the number of different bins
118
119 def bin2binIdentity(h1,h2):
120 def getNbins(h):
121 biny=h.GetNbinsY()
122 if biny>1:biny+=1
123 binz=h.GetNbinsZ()
124 if binz>1:binz+=1
125 return (h.GetNbinsX()+1)*(biny)*(binz)
126
127 nbins=getNbins(h1)
128 diffbins=0
129 for ibin in xrange(0,nbins):
130 h1bin=h1.GetBinContent(ibin)
131 h2bin=h2.GetBinContent(ibin)
132 diffbins+= (h1bin!=h2bin)
133 return diffbins
134
135
136 # =================================================================================================
137 # Method : compareHistos( t1, t2 )
138 #
139 # @param t1, t2 : a tuple of ( type, d ) where type is either 'REFERENCE' or 'TEST'
140 # and d is a dictionary of ROOT objects, with each key = ROOT path
141 #
142 # function : compare the histograms in Reference/Test ROOT files. First, go through each
143 # dict to collect the histos (ignore TDirectory objects, etc). Then the histos
144 # in the test file (experimental) are compared to their equivalents in the
145 # reference file (definitely correct) using 3 methods.
146 # 1) The entries are checked, they should be equal
147 # 2) If entries are equal, check the Integral(); should be equal
148 # 3) If integrals are equal, check the KolmogorovTest() ; should be 1
149 # 4) If identity flag is there and KS test is performed, perform bin2bin identity test
150 # Arguments t1 and t2 are checked and the test/reference auto-detected
151 #
152 def compareHistos(t1, t2, state, checkBin2BinIdentity) :
153
154 ( ((referenceObjects,referenceHistos),(parallObjects, parallHistos)), (uniqueSerPaths,uniqueSerHistos), (uniqueParPaths,uniqueParHistos), mh ) = state
155
156 # deduce which one is test, which reference
157 if t1[0] == ref : ds = t1[1] ; dp = t2[1]
158 elif t2[0] == ref : ds = t2[1] ; dp = t1[1]
159 else : print 'Neither tuple is Reference Root file reference?' ; return
160
161 # histocount, objectcount for test/reference
162 hcp = 0 ; pHistos = []
163 hcs = 0 ; sHistos = []
164
165 omit = [re.compile(regex) for regex in gRegexBlackList]
166
167 # find the histos in the reference file
168 for k in ds.keys() :
169 if not any(regex.search(k)!=None for regex in omit):
170 if ds[k].__class__.__name__ in histos : hcs += 1 ; sHistos.append( k )
171 # same for test
172 for k in dp.keys() :
173 if not any(regex.search(k)!=None for regex in omit):
174 if dp[k].__class__.__name__ in histos : hcp += 1 ; pHistos.append( k )
175
176
177 cEntries = 0 ; xEntries = 0 ; diffEntries = []
178 cIntegrals = 0 ; xIntegrals = 0 ; diffIntegrals = []
179 passedKol = 0 ; failedKol = 0 ; diffKols = [] ; zeroIntegrals = 0
180 passedIdentity =0 ; failedIdentity=0; diffIdentity=[]; identityDiffBins={}
181 kTested = 0
182 kTestResults={}
183 notfound = 0 ; integralMatch = 0 ; otherTest = 0 ; zeroIntegralMatch = 0
184 for h in sHistos :
185 if h in pHistos :
186 # matching histos to check
187 cEntries += 1
188 sh = ds[h] ; ph = dp[h]
189 # first check entries
190 if sh.GetEntries() != ph.GetEntries() : diffEntries.append(h) ; xEntries += 1 ; continue
191 # check for (non-zero sum of bin error) && (non-zero integrals) for K-Test
192 sBinError = 0.0 ; pBinError = 0.0
193 for i in xrange(sh.GetNbinsX()) : sBinError += sh.GetBinError(i)
194 for i in xrange(ph.GetNbinsX()) : pBinError += ph.GetBinError(i)
195 sint = sh.Integral() ; pint = ph.Integral()
196 doKS=(bool(sint) and bool(pint)) and ( sBinError>0 and pBinError>0 )
197 if checkBin2BinIdentity and doKS:
198 diffBins=bin2binIdentity(sh,ph)
199 if diffBins==0:
200 passedIdentity += 1
201 else:
202 failedIdentity += 1
203 diffIdentity.append(h)
204 identityDiffBins[h]=diffBins
205 if (bool(sint) and bool(pint)) and ( sBinError>0 and pBinError>0 ) :
206 kTested += 1
207 kTest = sh.KolmogorovTest(ph)
208 kTestResults [h] = kTest
209 if int(kTest) : passedKol += 1
210 else : failedKol += 1 ; diffKols.append(h) # ; print 'KTest result : ', kTest
211 else :
212 # try the integral test?
213 otherTest += 1
214 if all((sint, pint)) and (sint==pint) :
215 integralMatch += 1
216 elif (sint==pint) :
217 zeroIntegralMatch += 1
218 else :
219 diffIntegrals.append( h )
220 xIntegrals += 1
221 else :
222 notfound += 1 ; print 'not found? ', h
223
224 # report on Failed Entry-Checks
225 print '\n\n'+'-'*80
226 print 'Summary of histos with different Entries'
227 print '-'*80
228 if diffEntries :
229 diffEntries.sort()
230 for e in diffEntries : print '\t\t\t%s:\t%i != %i'%( e, int(ds[e].GetEntries()), int(dp[e].GetEntries()) )
231 print '-'*80
232
233 # report on Failed Kolmogorov Tests
234 print '\n\n'+'-'*60
235 print 'Summary of histos which failed Kolmogorov Test'
236 print '-'*60
237 if diffKols :
238 diffKols.sort()
239 for e in diffKols :
240 result = kTestResults[e] # DP Calculated twice ARGH!!
241 print '%s\t\t%s :\tK-Test Result :\t %5.16f'%( ds[e].ClassName(), e, result )
242 print '-'*60
243
244 # report on Failed Integral Checks
245 print '\n\n'+'-'*60
246 print 'Summary of histos which failed Integral Check'
247 print '-'*60
248 if diffIntegrals :
249 diffIntegrals.sort()
250 for e in diffIntegrals :
251 diff = dp[e].Integral()-ds[e].Integral()
252 pc = (diff*100)/ds[e].Integral()
253 print '%s\t\t%s:\t Diff = %5.6f\tPercent Diff to Reference : %5.6f '%( ds[e].ClassName(), e, diff, pc )
254 print '-'*60 + '\n'
255 print '='*80 + '\n'
256
257 # Report on failed bin2bin identity
258 if checkBin2BinIdentity:
259 # report on b2b checks
260 print '\n\n'+'-'*80
261 print 'Summary of histos with at least one bin with different Entries'
262 print '-'*80
263 if diffIdentity:
264 diffIdentity.sort()
265 for e in diffIdentity:
266 print '%s\t\t%s: %i different bins'%( ds[e].ClassName(), e, identityDiffBins[e])
267 print '-'*80
268
269 print '\n' + '='*80
270 print 'Comparison : Reference/Test ROOT Histo files'
271 print '\n\t\tReference\tTest'
272 print '\tObjects : %i\t%i\t\t( p-s = %i )'%( referenceObjects, parallObjects, parallObjects-referenceObjects )
273 print '\tHistos : %i\t%i\t\t( p-s = %i )'%( referenceHistos, parallHistos, parallHistos-referenceHistos )
274 print '\t __________'
275 print '\tTotal : %i\t%i\n'%( referenceHistos+referenceObjects, parallHistos+parallObjects )
276 print 'Objects/Histos unique to Reference File : %i / %i'%( len(uniqueSerPaths)-uniqueSerHistos, uniqueSerHistos )
277 print 'Objects/Histos unique to Test File : %i / %i'%( len(uniqueParPaths)-uniqueParHistos, uniqueParHistos )
278 print '\nMatching Histograms valid for Comparison : %i'%( mh )
279 print '\nOmissions\' patterns : '
280 for entry in gRegexBlackList : print '\t%s'%( entry )
281 print '\nHistograms for Comparison (after Omissions) : %i'%( mh-len(gRegexBlackList) )
282 print '\n\tHISTOGRAM TESTS : '
283 print '\t\tKOLMOGOROV TEST : %i'%( kTested )
284 print '\t\tINTEGRAL TEST : %i'%( otherTest )
285 print '\t\tENTRIES TEST : %i'%( xEntries )
286 if checkBin2BinIdentity:
287 print '\t\tBIN2BIN TEST : %i'%( passedIdentity )
288 print '\t\t ____'
289 print '\t\tTested : %i'%( cEntries )
290
291 print '\n\tDISCREPANCIES : '
292 print '\t\tK-Test : %i'%( failedKol )
293 print '\t\tIntegrals : %i'%( xIntegrals )
294 print '\t\tEntries : %i'%( xEntries )
295 retval = failedKol+xIntegrals+xEntries+failedIdentity
296 if retval!=0:
297 print '\nThe two sets of histograms were not identical'
298 print '\n'+'='*80
299 return retval
300
301 # =================================================================================================
302
303 def extractBlacklist(listString):
304 global gRegexBlackList
305 if listString:
306 for blackRegexp in listString.split(","):
307 gRegexBlackList.append(blackRegexp)
308 else:
309 gBlackList =[]
310
311 # =================================================================================================
312
313 if __name__ == '__main__' :
314 usage = "usage: %prog testFile.root referenceFile.root [options]"
315 parser = OptionParser()
316 parser.add_option("-b", dest="blacklist",
317 help='Comma separated list of regexps matching histograms to skip (for example -b"MemoryTool,ProcTime")')
318
319 parser.add_option("-i",
320 action="store_true", dest="bin2bin", default=False,
321 help="Check for bin to bin identity")
322 (options, args) = parser.parse_args()
323
324 if len(args)!=2:
325 print "Wrong number of rootfiles. Usage:"
326 print usage
327 sys.exit(1)
328
329 extractBlacklist(options.blacklist)
330
331 testFile,referenceFile=args
332
333 tfs = TFile( testFile, 'REC' ) ; print 'opening Test File : %s'%( testFile )
334 tfp = TFile( referenceFile, 'REC' ) ; print 'opening Reference File : %s'%( referenceFile )
335
336 # get structure of TFiles in a list of (path, object) tuples
337 lref = rec(tfs) ; ltest = rec(tfp)
338 # make a dictionary of lser and lpar. keys=paths
339 dref = dict( [(n, o) for n, o in lref] )
340 dtest = dict( [(n, o) for n, o in ltest] )
341 # make a tuple of (type, dict) where type is either 'reference' or 'test'
342 ts = ( ref, dref ) ; tp = ( test, dtest )
343
344 # check objs/histos in each file
345 composition( ts ) ; composition( tp )
346
347 # compare paths from each file
348 state = comparePaths( ts, tp )
349
350 # compare histos from each file
351 retval =compareHistos( ts, tp, state, checkBin2BinIdentity=options.bin2bin )
352
353 # finished with TFiles
354 tfs.Close() ; tfp.Close()
355
356 sys.exit(retval)
357
compareRootHistos.rec
def rec(o, path=None, lst=None)
Definition: compareRootHistos.py:28
compareRootHistos.compareHistos
def compareHistos(t1, t2, state, checkBin2BinIdentity)
Definition: compareRootHistos.py:152
compareRootHistos.extractBlacklist
def extractBlacklist(listString)
Definition: compareRootHistos.py:303
compareRootHistos.comparePaths
def comparePaths(t1, t2)
Definition: compareRootHistos.py:71
compareRootHistos.bin2binIdentity
def bin2binIdentity(h1, h2)
Definition: compareRootHistos.py:119
GaudiMath::Integral
GAUDI_API double Integral(const Genfun::AbsFunction &function, const double a, const double b, const GaudiMath::Integration::Type type=GaudiMath::Integration::Adaptive, const GaudiMath::Integration::KronrodRule rule=GaudiMath::Integration::Default, const double epsabs=1.e-10, const double epsrel=1.e-7, const size_t size=1000)
Definition: Integral.cpp:27
compareRootHistos.composition
def composition(t)
Definition: compareRootHistos.py:50