| # Gaudi Framework, version v23r0 |

Home |  | Generated: Mon Jan 30 2012

Functions | Variables

# compareRootHistos Namespace Reference

## Functions
def | rec
def | composition
def | comparePaths
def | compareHistos
## Variables
list | histos = ['TH1D', 'TH2D', 'TProfile']
string | ser = 'SERIAL'
string | par = 'PARALL'
list | pFile = sys.argv[0]
list | sFile = sys.argv[1]
tuple | tfs = TFile( sFile, 'REC' )
tuple | tfp = TFile( pFile, 'REC' )
tuple | lser = rec(tfs)
tuple | dserial = dict( [(n, o) for n, o in lser] )
tuple | dparall = dict( [(n, o) for n, o in lpar] )
tuple | ts = ( ser, dserial )
tuple | state = comparePaths( ts, tp )

## Function Documentation

def compareRootHistos::compareHistos | ( |  | t1,
|  |  | t2,
|  |  | state
| ) |  |

Definition at line 121 of file compareRootHistos.py.

    00122                                  :
    00123
    00124   ( ((serialObjects,serialHistos),(parallObjects, parallHistos)), (uniqueSerPaths,uniqueSerHistos), (uniqueParPaths,uniqueParHistos), mh ) = state
    00125
    00126   # deduce which one is parallel, which serial
    00127   if   t1[0]  == ser : ds = t1[1] ; dp = t2[1]
    00128   elif t2[0]  == ser : ds = t2[1] ; dp = t1[1]
    00129   else : print 'Neither tuple is Serial Root file reference?' ; return
    00130
    00131   # histocount, objectcount for parallel/serial
    00132   hcp = 0 ; pHistos = []
    00133   hcs = 0 ; sHistos = []
    00134
    00135   omit = ['/stat/Brunel/MemoryTool/Virtual mem, all entries',
    00136           '/stat/Brunel/MemoryTool/Virtual mem, downscaled']
    00137   omit = []
    00138
    00139   # find the histos in the serial file
    00140   for k in ds.keys() :
    00141     if k not in omit :
    00142       if ds[k].__class__.__name__ in histos : hcs += 1 ; sHistos.append( k )
    00143   # same for parallel
    00144   for k in dp.keys() :
    00145     if k not in omit :
    00146       if dp[k].__class__.__name__ in histos : hcp += 1 ; pHistos.append( k )
    00147
    00148
    00149   cEntries   = 0 ; xEntries   = 0 ; diffEntries   = []
    00150   cIntegrals = 0 ; xIntegrals = 0 ; diffIntegrals = []
    00151   passedKol  = 0 ; failedKol  = 0 ; diffKols      = [] ; zeroIntegrals = 0
    00152   kTested    = 0
    00153   notfound   = 0 ; integralMatch = 0 ; otherTest = 0 ; zeroIntegralMatch = 0
    00154   for h in sHistos :
    00155     if h in pHistos :
    00156       # matching histos to check
    00157       cEntries += 1
    00158       sh = ds[h] ; ph = dp[h]
    00159       # first check entries
    00160       if sh.GetEntries() != ph.GetEntries() : diffEntries.append(h) ; xEntries += 1 ; continue
    00161       # check for (non-zero sum of bin error) && (non-zero integrals) for K-Test
    00162       sBinError = 0.0 ; pBinError = 0.0
    00163       for i in xrange(sh.GetNbinsX()) : sBinError += sh.GetBinError(i)
    00164       for i in xrange(ph.GetNbinsX()) : pBinError += ph.GetBinError(i)
    00165       sint = sh.Integral() ; pint = ph.Integral()
    00166       if (bool(sint) and bool(pint)) and ( sBinError>0 and pBinError>0 ) :
    00167         kTested += 1
    00168         kTest = sh.KolmogorovTest(ph)
    00169         if int(kTest) : passedKol += 1
    00170         else          : failedKol += 1 ; diffKols.append(h) # ; print 'KTest result : ', kTest
    00171       else :
    00172         # try the integral test?
    00173         otherTest += 1
    00174         if all((sint, pint)) and (sint==pint) :
    00175           integralMatch += 1
    00176         elif (sint==pint) :
    00177           zeroIntegralMatch += 1
    00178         else :
    00179             diffIntegrals.append( h )
    00180             xIntegrals += 1
    00181     else :
    00182       notfound += 1 ; print 'not found? ', h
    00183
    00184   # report on Failed Entry-Checks
    00185   print '\n\n'+'-'*80
    00186   print 'Summary of histos with different Entries'
    00187   print '-'*80
    00188   if diffEntries :
    00189     diffEntries.sort()
    00190     for e in diffEntries : print '\t\t\t%s:\t%i != %i'%( e, int(ds[e].GetEntries()), int(dp[e].GetEntries()) )
    00191   print '-'*80
    00192
    00193   # report on Failed Kolmogorov Tests
    00194   print '\n\n'+'-'*60
    00195   print 'Summary of histos which failed Kolmogorov Test'
    00196   print '-'*60
    00197   if diffKols :
    00198     diffKols.sort()
    00199     for e in diffKols :
    00200       result = ds[e].KolmogorovTest(dp[e])
    00201       print '%s\t\t%s :\tK-Test Result :\t %5.16f'%( type(ds[e]), e, result )
    00202   print '-'*60
    00203
    00204   # report on Failed Integral Checks
    00205   print '\n\n'+'-'*60
    00206   print 'Summary of histos which failed Integral Check'
    00207   print '-'*60
    00208   if diffIntegrals :
    00209     diffIntegrals.sort()
    00210     for e in diffIntegrals :
    00211       diff = dp[e].Integral()-ds[e].Integral()
    00212       pc   = (diff*100)/ds[e].Integral()
    00213       print '%s\t\t%s:\t Diff = %5.6f\tPercent Diff to Serial : %5.6f '%( type(ds[e]), e, diff, pc  )
    00214   print '-'*60 + '\n'
    00215   print '='*80 + '\n'
    00216
    00217   print '\n' + '='*80
    00218   print 'Comparison : Serial/Parallel ROOT Histo files'
    00219   print '\n\t\tSerial\tParall'
    00220   print '\tObjects : %i\t%i\t\t( p-s = %i )'%( serialObjects, parallObjects, parallObjects-serialObjects )
    00221   print '\tHistos  : %i\t%i\t\t( p-s = %i )'%( serialHistos,  parallHistos,  parallHistos-serialHistos  )
    00222   print '\t          __________'
    00223   print '\tTotal   : %i\t%i\n'%( serialHistos+serialObjects,  parallHistos+parallObjects  )
    00224   print 'Objects/Histos unique to Serial File : %i / %i'%( len(uniqueSerPaths)-uniqueSerHistos, uniqueSerHistos )
    00225   print 'Objects/Histos unique to Parall File : %i / %i'%( len(uniqueParPaths)-uniqueParHistos, uniqueParHistos )
    00226   print '\nMatching Histograms valid for Comparison : %i'%( mh )
    00227   print '\nOmissions : '
    00228   for entry in omit : print '\t%s'%( entry )
    00229   print '\nHistograms for Comparison (after Omissions) : %i'%( mh-len(omit) )
    00230   print '\n\tHISTOGRAM TESTS : '
    00231   print '\t\tKOLMOGOROV TEST      : %i'%( kTested )
    00232   print '\t\tINTEGRAL TEST        : %i'%( otherTest )
    00233   print '\t\tENTRIES TEST         : %i'%( xEntries )
    00234   print '\t\t                       ____'
    00235   print '\t\tTested               : %i'%( cEntries )
    00236
    00237   print '\n\tDISCREPANCIES : '
    00238   print '\t\tK-Test      : %i'%( failedKol  )
    00239   print '\t\tIntegrals   : %i'%( xIntegrals )
    00240   print '\t\tEntries     : %i'%( xEntries   )
    00241   print '\n'+'='*80
    00242
    00243 # =================================================================================================

def compareRootHistos::comparePaths | ( |  | t1,
|  |  | t2
| ) |  |

Definition at line 64 of file compareRootHistos.py.

    00064                            :
    00065   if   t1[0]  == ser : ds = t1[1] ; dp = t2[1]
    00066   elif t2[0]  == ser : ds = t2[1] ; dp = t1[1]
    00067   else : print 'Neither tuple is Serial Root file reference?' ; return
    00068
    00069   dsks = ds.keys() ; dpks = dp.keys()
    00070   dsks.sort()      ; dpks.sort()
    00071
    00072   sset = sets.Set( dsks )
    00073   pset = sets.Set( dpks )
    00074   os, hs = composition( (ser, ds) )
    00075   op, hp = composition( (par, dp) )
    00076   print '\n' + '='*80
    00077   print 'Comparison of Paths : Serial vs Parallel ROOT files'
    00078   print '-'*80
    00079   print 'Number of paths in Serial file : %i (objects, histos) = ( %i, %i )'%( len(dsks), os, hs )
    00080   print 'Number of paths in Parall file : %i (objects, histos) = ( %i, %i )'%( len(dpks), op, hp )
    00081   matching = sset.intersection(pset)
    00082   matchingHistos = 0
    00083   for n in matching :
    00084     if ds[n].__class__.__name__ in histos : matchingHistos += 1
    00085   print '\nMatching paths                 : %i'%( len(matching) )
    00086   uSer = sset - pset
    00087   # work out histos unique to parallel file
    00088   uniqueSerialHistos = 0
    00089   for n in uSer :
    00090     if ds[n].__class__.__name__ in histos : uniqueSerialHistos += 1
    00091   print 'Paths unique to Serial file : %i ( %i Histos )'%( len(uSer), uniqueSerialHistos )
    00092   if uSer :
    00093     for n in uSer : print '\t%s : \t%s'%( ds[n], n )
    00094   uPar = pset - sset
    00095   uniqueParallHistos = 0
    00096   for n in uPar :
    00097     if dp[n].__class__.__name__ in histos : uniqueParallHistos += 1
    00098   print 'Paths unique to Parall file : %i ( %i Histos )'%( len(uPar), uniqueParallHistos )
    00099   if uPar :
    00100     for n in uPar : print '\t%s : \t%s'%( dp[n], n )
    00101   print 'Matching Histos to test : %i'%( matchingHistos )
    00102   print '='*80 + '\n'
    00103   return ( ((os,hs),(op,hp)), (uSer, uniqueSerialHistos), (uPar, uniqueParallHistos), matchingHistos )
    00104 # =================================================================================================
    00105
    00106 # =================================================================================================
    00107 # Method        : compareHistos( t1, t2 )
    00108 #
    00109 # @param t1, t2 : a tuple of ( type, d ) where type is either 'SERIAL' or 'PARALL'
    00110 #                 and d is a dictionary of ROOT objects, with each key = ROOT path
    00111 #
    00112 # function      : compare the histograms in Serial/Parallel ROOT files.  First, go through each
    00113 #                 dict to collect the histos (ignore TDirectory objects, etc).  Then the histos
    00114 #                 in the parallel file (experimental) are compared to their equivalents in the
    00115 #                 serial file (definitely correct) using 3 methods.
    00116 #                 1) The entries are checked, they should be equal
    00117 #                 2) If entries are equal, check the Integral(); should be equal
    00118 #                 3) If integrals are equal, check the KolmogorovTest() ; should be 1
    00119 #                 Arguments t1 and t2 are checked and the parallel/serial auto-detected
    00120 #

def compareRootHistos::composition | ( |  | t ) |

Definition at line 42 of file compareRootHistos.py.

    00043                      :
    00044   typ, d = t
    00045   hists = 0 ; objs = 0
    00046   for k in d.keys() :
    00047     if d[k].__class__.__name__ in histos : hists += 1
    00048     else                                 : objs  += 1
    00049   return objs, hists
    00050 # =================================================================================================
    00051
    00052 # =================================================================================================
    00053 # Method        : comparePaths( t1, t2 )
    00054 #
    00055 # @param t1, t2 : a tuple of ( type, d ) where type is either 'SERIAL' or 'PARALL'
    00056 #                 and d is a dictionary of ROOT objects, with each key = ROOT path
    00057 #
    00058 # function      : compare the paths between the two histo files.  If the files are identical, they
    00059 #                 should have the same set of paths.  The Parallel file should definitely have the
    00060 #                 same paths as the Serial.  Perhaps the Serial file will have some more paths due
    00061 #                 to extra histos added as part of Application Sequencer finalisation
    00062 #                 Arguments t1 and t2 are checked and the parallel/serial auto-detected
    00063 #                 Uses sets module for intersections/unions, etc.
    #

def compareRootHistos::rec | ( |  | o,
|  |  | path = `None`,
|  |  | lst = `None`
| ) |  |

Definition at line 20 of file compareRootHistos.py.

    00021                                   :
    00022   if not path : path = '/stat' ; lst = []
    00023   else        : path = path + '/' + o.GetName()
    00024   lst.append( (path,o) )
    00025   if 'GetListOfKeys' in dir(o) :
    00026     keys = o.GetListOfKeys()
    00027     for k in keys :
    00028       name =  k.GetName()
    00029       rec( o.Get(name), path, lst )
    00030   else :
    00031     pass
    00032   return lst
    00033 # =================================================================================================
    00034
    00035 # =================================================================================================
    00036 # Method   : composition( t )
    00037 #
    00038 # @param t : a tuple of ( type, d ) where type is either 'SERIAL' or 'PARALL'
    00039 #            and d is a dictionary of ROOT objects, with each key = ROOT path
    00040 #
    00041 # function : deduce the composition, (objects/histos) counts
    #

## Variable Documentation

tuple compareRootHistos::dparall = dict( [(n, o) for n, o in lpar] )

Definition at line 261 of file compareRootHistos.py.

tuple compareRootHistos::dserial = dict( [(n, o) for n, o in lser] )

Definition at line 260 of file compareRootHistos.py.

list compareRootHistos::histos = ['TH1D', 'TH2D', 'TProfile']

Definition at line 5 of file compareRootHistos.py.

tuple compareRootHistos::lser = rec(tfs)

Definition at line 258 of file compareRootHistos.py.

string compareRootHistos::par = 'PARALL'

Definition at line 7 of file compareRootHistos.py.

list compareRootHistos::pFile = sys.argv[0]

Definition at line 247 of file compareRootHistos.py.

string compareRootHistos::ser = 'SERIAL'

Definition at line 6 of file compareRootHistos.py.

list compareRootHistos::sFile = sys.argv[1]

Definition at line 248 of file compareRootHistos.py.

tuple compareRootHistos::state = comparePaths( ts, tp )

Definition at line 269 of file compareRootHistos.py.

tuple compareRootHistos::tfp = TFile( pFile, 'REC' )

Definition at line 255 of file compareRootHistos.py.

tuple compareRootHistos::tfs = TFile( sFile, 'REC' )

Definition at line 254 of file compareRootHistos.py.

tuple compareRootHistos::ts = ( ser, dserial )

Definition at line 263 of file compareRootHistos.py.

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Defines