compareRootHistos Namespace Reference

Functions
def	rec
def	composition
def	comparePaths
def	compareHistos
Variables
list	histos = ['TH1D', 'TH2D', 'TProfile']
string	ser = 'SERIAL'
string	par = 'PARALL'
list	pFile = sys.argv[0]
list	sFile = sys.argv[1]
tuple	tfs = TFile( sFile, 'REC' )
tuple	tfp = TFile( pFile, 'REC' )
tuple	lser = rec(tfs)
tuple	dserial = dict( [(n, o) for n, o in lser] )
tuple	dparall = dict( [(n, o) for n, o in lpar] )
tuple	ts = ( ser, dserial )
tuple	state = comparePaths( ts, tp )

Function Documentation

def compareRootHistos::compareHistos	(	t1,
		t2,
		state
	)

Definition at line 121 of file compareRootHistos.py.

00122                                  :
00123   
00124   ( ((serialObjects,serialHistos),(parallObjects, parallHistos)), (uniqueSerPaths,uniqueSerHistos), (uniqueParPaths,uniqueParHistos), mh ) = state
00125  
00126   # deduce which one is parallel, which serial
00127   if   t1[0]  == ser : ds = t1[1] ; dp = t2[1]
00128   elif t2[0]  == ser : ds = t2[1] ; dp = t1[1]
00129   else : print 'Neither tuple is Serial Root file reference?' ; return
00130 
00131   # histocount, objectcount for parallel/serial
00132   hcp = 0 ; pHistos = []
00133   hcs = 0 ; sHistos = []
00134   
00135   omit = ['/stat/Brunel/MemoryTool/Virtual mem, all entries',
00136           '/stat/Brunel/MemoryTool/Virtual mem, downscaled']
00137   omit = []
00138   
00139   # find the histos in the serial file
00140   for k in ds.keys() : 
00141     if k not in omit : 
00142       if ds[k].__class__.__name__ in histos : hcs += 1 ; sHistos.append( k )
00143   # same for parallel
00144   for k in dp.keys() : 
00145     if k not in omit : 
00146       if dp[k].__class__.__name__ in histos : hcp += 1 ; pHistos.append( k )
00147       
00148 
00149   cEntries   = 0 ; xEntries   = 0 ; diffEntries   = []
00150   cIntegrals = 0 ; xIntegrals = 0 ; diffIntegrals = []
00151   passedKol  = 0 ; failedKol  = 0 ; diffKols      = [] ; zeroIntegrals = 0
00152   kTested    = 0
00153   notfound   = 0 ; integralMatch = 0 ; otherTest = 0 ; zeroIntegralMatch = 0
00154   for h in sHistos :
00155     if h in pHistos : 
00156       # matching histos to check
00157       cEntries += 1
00158       sh = ds[h] ; ph = dp[h]
00159       # first check entries
00160       if sh.GetEntries() != ph.GetEntries() : diffEntries.append(h) ; xEntries += 1 ; continue
00161       # check for (non-zero sum of bin error) && (non-zero integrals) for K-Test
00162       sBinError = 0.0 ; pBinError = 0.0
00163       for i in xrange(sh.GetNbinsX()) : sBinError += sh.GetBinError(i)
00164       for i in xrange(ph.GetNbinsX()) : pBinError += ph.GetBinError(i)
00165       sint = sh.Integral() ; pint = ph.Integral()
00166       if (bool(sint) and bool(pint)) and ( sBinError>0 and pBinError>0 ) : 
00167         kTested += 1
00168         kTest = sh.KolmogorovTest(ph)
00169         if int(kTest) : passedKol += 1 
00170         else          : failedKol += 1 ; diffKols.append(h) # ; print 'KTest result : ', kTest        
00171       else : 
00172         # try the integral test?
00173         otherTest += 1
00174         if all((sint, pint)) and (sint==pint) : 
00175           integralMatch += 1
00176         elif (sint==pint) :
00177           zeroIntegralMatch += 1
00178         else : 
00179             diffIntegrals.append( h )
00180             xIntegrals += 1
00181     else :
00182       notfound += 1 ; print 'not found? ', h
00183 
00184   # report on Failed Entry-Checks      
00185   print '\n\n'+'-'*80
00186   print 'Summary of histos with different Entries'
00187   print '-'*80
00188   if diffEntries :
00189     diffEntries.sort()  
00190     for e in diffEntries : print '\t\t\t%s:\t%i != %i'%( e, int(ds[e].GetEntries()), int(dp[e].GetEntries()) )  
00191   print '-'*80
00192   
00193   # report on Failed Kolmogorov Tests    
00194   print '\n\n'+'-'*60
00195   print 'Summary of histos which failed Kolmogorov Test'
00196   print '-'*60
00197   if diffKols :
00198     diffKols.sort()  
00199     for e in diffKols : 
00200       result = ds[e].KolmogorovTest(dp[e])
00201       print '%s\t\t%s :\tK-Test Result :\t %5.16f'%( type(ds[e]), e, result )  
00202   print '-'*60
00203   
00204   # report on Failed Integral Checks
00205   print '\n\n'+'-'*60
00206   print 'Summary of histos which failed Integral Check'
00207   print '-'*60
00208   if diffIntegrals :
00209     diffIntegrals.sort()
00210     for e in diffIntegrals : 
00211       diff = dp[e].Integral()-ds[e].Integral()
00212       pc   = (diff*100)/ds[e].Integral()
00213       print '%s\t\t%s:\t Diff = %5.6f\tPercent Diff to Serial : %5.6f '%( type(ds[e]), e, diff, pc  )
00214   print '-'*60 + '\n' 
00215   print '='*80 + '\n'
00216   
00217   print '\n' + '='*80
00218   print 'Comparison : Serial/Parallel ROOT Histo files'
00219   print '\n\t\tSerial\tParall'
00220   print '\tObjects : %i\t%i\t\t( p-s = %i )'%( serialObjects, parallObjects, parallObjects-serialObjects )  
00221   print '\tHistos  : %i\t%i\t\t( p-s = %i )'%( serialHistos,  parallHistos,  parallHistos-serialHistos  )
00222   print '\t          __________'
00223   print '\tTotal   : %i\t%i\n'%( serialHistos+serialObjects,  parallHistos+parallObjects  )  
00224   print 'Objects/Histos unique to Serial File : %i / %i'%( len(uniqueSerPaths)-uniqueSerHistos, uniqueSerHistos )
00225   print 'Objects/Histos unique to Parall File : %i / %i'%( len(uniqueParPaths)-uniqueParHistos, uniqueParHistos )
00226   print '\nMatching Histograms valid for Comparison : %i'%( mh )
00227   print '\nOmissions : '
00228   for entry in omit : print '\t%s'%( entry )   
00229   print '\nHistograms for Comparison (after Omissions) : %i'%( mh-len(omit) )  
00230   print '\n\tHISTOGRAM TESTS : '
00231   print '\t\tKOLMOGOROV TEST      : %i'%( kTested )
00232   print '\t\tINTEGRAL TEST        : %i'%( otherTest )
00233   print '\t\tENTRIES TEST         : %i'%( xEntries )
00234   print '\t\t                       ____'
00235   print '\t\tTested               : %i'%( cEntries )  
00236 
00237   print '\n\tDISCREPANCIES : '
00238   print '\t\tK-Test      : %i'%( failedKol  )
00239   print '\t\tIntegrals   : %i'%( xIntegrals )    
00240   print '\t\tEntries     : %i'%( xEntries   )  
00241   print '\n'+'='*80  
00242 
00243 # =================================================================================================

def compareRootHistos::comparePaths	(	t1,
		t2
	)

Definition at line 64 of file compareRootHistos.py.

00064                            : 
00065   if   t1[0]  == ser : ds = t1[1] ; dp = t2[1]
00066   elif t2[0]  == ser : ds = t2[1] ; dp = t1[1]
00067   else : print 'Neither tuple is Serial Root file reference?' ; return
00068   
00069   dsks = ds.keys() ; dpks = dp.keys()
00070   dsks.sort()      ; dpks.sort()
00071   
00072   sset = sets.Set( dsks )
00073   pset = sets.Set( dpks )
00074   os, hs = composition( (ser, ds) )
00075   op, hp = composition( (par, dp) )
00076   print '\n' + '='*80
00077   print 'Comparison of Paths : Serial vs Parallel ROOT files'
00078   print '-'*80
00079   print 'Number of paths in Serial file : %i (objects, histos) = ( %i, %i )'%( len(dsks), os, hs )
00080   print 'Number of paths in Parall file : %i (objects, histos) = ( %i, %i )'%( len(dpks), op, hp )
00081   matching = sset.intersection(pset)
00082   matchingHistos = 0
00083   for n in matching :
00084     if ds[n].__class__.__name__ in histos : matchingHistos += 1    
00085   print '\nMatching paths                 : %i'%( len(matching) )
00086   uSer = sset - pset
00087   # work out histos unique to parallel file
00088   uniqueSerialHistos = 0
00089   for n in uSer :
00090     if ds[n].__class__.__name__ in histos : uniqueSerialHistos += 1
00091   print 'Paths unique to Serial file : %i ( %i Histos )'%( len(uSer), uniqueSerialHistos )
00092   if uSer : 
00093     for n in uSer : print '\t%s : \t%s'%( ds[n], n )
00094   uPar = pset - sset
00095   uniqueParallHistos = 0  
00096   for n in uPar :
00097     if dp[n].__class__.__name__ in histos : uniqueParallHistos += 1  
00098   print 'Paths unique to Parall file : %i ( %i Histos )'%( len(uPar), uniqueParallHistos )
00099   if uPar : 
00100     for n in uPar : print '\t%s : \t%s'%( dp[n], n )
00101   print 'Matching Histos to test : %i'%( matchingHistos )
00102   print '='*80 + '\n'
00103   return ( ((os,hs),(op,hp)), (uSer, uniqueSerialHistos), (uPar, uniqueParallHistos), matchingHistos )
00104 # =================================================================================================  
00105 
00106 # =================================================================================================
00107 # Method        : compareHistos( t1, t2 )
00108 # 
00109 # @param t1, t2 : a tuple of ( type, d ) where type is either 'SERIAL' or 'PARALL'
00110 #                 and d is a dictionary of ROOT objects, with each key = ROOT path 
00111 # 
00112 # function      : compare the histograms in Serial/Parallel ROOT files.  First, go through each 
00113 #                 dict to collect the histos (ignore TDirectory objects, etc).  Then the histos
00114 #                 in the parallel file (experimental) are compared to their equivalents in the 
00115 #                 serial file (definitely correct) using 3 methods.
00116 #                 1) The entries are checked, they should be equal
00117 #                 2) If entries are equal, check the Integral(); should be equal
00118 #                 3) If integrals are equal, check the KolmogorovTest() ; should be 1
00119 #                 Arguments t1 and t2 are checked and the parallel/serial auto-detected
00120 #

def compareRootHistos::composition ( t )

Definition at line 42 of file compareRootHistos.py.

00043                      :
00044   typ, d = t 
00045   hists = 0 ; objs = 0
00046   for k in d.keys() : 
00047     if d[k].__class__.__name__ in histos : hists += 1
00048     else                                 : objs  += 1
00049   return objs, hists
00050 # =================================================================================================
00051   
00052 # =================================================================================================
00053 # Method        : comparePaths( t1, t2 )
00054 # 
00055 # @param t1, t2 : a tuple of ( type, d ) where type is either 'SERIAL' or 'PARALL'
00056 #                 and d is a dictionary of ROOT objects, with each key = ROOT path 
00057 # 
00058 # function      : compare the paths between the two histo files.  If the files are identical, they
00059 #                 should have the same set of paths.  The Parallel file should definitely have the
00060 #                 same paths as the Serial.  Perhaps the Serial file will have some more paths due
00061 #                 to extra histos added as part of Application Sequencer finalisation
00062 #                 Arguments t1 and t2 are checked and the parallel/serial auto-detected
00063 #                 Uses sets module for intersections/unions, etc.
#

def compareRootHistos::rec	(	o,
		path = `None`,
		lst = `None`
	)

Definition at line 20 of file compareRootHistos.py.

00021                                   :
00022   if not path : path = '/stat' ; lst = []
00023   else        : path = path + '/' + o.GetName()
00024   lst.append( (path,o) )
00025   if 'GetListOfKeys' in dir(o) : 
00026     keys = o.GetListOfKeys()
00027     for k in keys :
00028       name =  k.GetName()
00029       rec( o.Get(name), path, lst ) 
00030   else :
00031     pass
00032   return lst
00033 # =================================================================================================
00034 
00035 # =================================================================================================
00036 # Method   : composition( t )
00037 # 
00038 # @param t : a tuple of ( type, d ) where type is either 'SERIAL' or 'PARALL'
00039 #            and d is a dictionary of ROOT objects, with each key = ROOT path 
00040 # 
00041 # function : deduce the composition, (objects/histos) counts
#