| # Gaudi Framework, version v21r7 |

Home |  | Generated: 22 Jan 2010

# compareOutputFiles.py

Go to the documentation of this file.

    00001 from Gaudi.Configuration import *
    00002 from GaudiPython import AppMgr, gbl
    00003 from ROOT import TFile, TBufferFile, TBuffer
    00004 from processing import Process, Queue
    00005
    00006 import sys, sets
    00007
    00008 #
    00009 # loadFile.py
    00010 # -----------
    00011 # Open a dst file for inspection
    00012 #
    00013
    00014 def checkKeys( name ) :
    00015   # Check the TTree keys in each file
    00016   fname = name[4:] # TFile doesn't need the "PFN:" prefix
    00017   tf = TFile(fname, 'REC')
    00018
    00019
    00020
    00021
    00022
    00023 importOptions('$STDOPTS/LHCbApplication.opts')
    00024 importOptions('$GAUDIPOOLDBROOT/options/GaudiPoolDbRoot.opts')
    00025
    00026
    00027 OutputStream("DstWriter").Output = ''
    00028 HistogramPersistencySvc().OutputFile = ''
    00029 MessageSvc(OutputLevel = ERROR)
    00030 EventSelector().PrintFreq = 100
    00031
    00032 ApplicationMgr( OutputLevel = ERROR,
    00033                 AppName = 'File Check - Serial vs Parallel' )
    00034                 # TopAlg = ['UnpackMCParticle', 'UnpackMCVertex'] )
    00035
    00036 PAR = 'PARALLEL'
    00037 SER = 'SERIAL'
    00038
    00039 def CompareTrees( pname, sname ) :
    00040   pf = TFile(pname, 'REC')
    00041   sf = TFile(sname, 'REC')
    00042   event = '_Event'
    00043   pfks = pf.GetListOfKeys()
    00044   sfks = sf.GetListOfKeys()
    00045   pfkeys = list( [pfk.GetName() for pfk in pfks] ) ; pfkeys.sort()
    00046   sfkeys = list( [sfk.GetName() for sfk in sfks] ) ; sfkeys.sort()
    00047   pMeta = [] ; pEvent = [] ; pOther = []
    00048   for k in pfkeys :
    00049     if   k.startswith(event) : pEvent.append(k)
    00050     elif k.startswith('##')  : pMeta.append(k)
    00051     else : pOther.append(k)
    00052   sMeta = [] ; sEvent = [] ; sOther = []
    00053   for k in sfkeys :
    00054     if   k.startswith(event) : sEvent.append(k)
    00055     elif k.startswith('##')  : sMeta.append(k)
    00056     else : sOther.append(k)
    00057
    00058   if pMeta == sMeta : pass
    00059   else             : print 'Meta Data differs'
    00060
    00061   if pEvent == sEvent : pass
    00062   else               : print 'Event data differs'
    00063
    00064   if pOther != sOther :
    00065     pset = sets.Set(pOther)
    00066     sset = sets.Set(sOther)
    00067     pExtra = pset-sset
    00068     sExtra = sset-pset
    00069     if pExtra : print 'Extra Data in parallel file : ', pExtra
    00070     if sExtra : print 'Extra Data in serial   file : ', sExtra
    00071     if sExtra or pExtra : print 'Files will have different sizes'
    00072   pf.Close()
    00073   sf.Close()
    00074
    00075 def switchDict( d ) :
    00076   # switch a dictionary around ;  make the values the keys, and vice versa
    00077   # only works if all values are unique
    00078   nkeys = len(d.keys())
    00079   vals = d.values()
    00080   nvals = len(vals)
    00081   for v in vals :
    00082     if vals.count(v) > 1 :
    00083       print 'Dictionary cannot be switched, values not unique'
    00084       return None
    00085   print 'Dict has keys/values : %i/%i'%( nkeys, nvals )
    00086   pairs = d.items() # returns (key, val) tuples in a list
    00087   newd = {}
    00088   for k, entry in pairs : newd[entry] = k
    00089   return newd
    00090
    00091
    00092 def printDict( d, name='unspecified' ) :
    00093   # Print out a dictionary in the form
    00094   #
    00095   # Dictionary Name :
    00096   #   key     value
    00097   #   key     value
    00098   #   ...
    00099   #
    00100   print '-'*80
    00101   print 'Dictionary %s : '%(name)
    00102   for k in iter(d.keys()) :
    00103     print '\t', k, '\t', d[k]
    00104   print '-'*80
    00105
    00106
    00107 def Reader( readerType, filename, qacross, qToEngine ) :
    00108   #
    00109   # Process for reading a file
    00110   # One process for reading Serial File, another for Parallel File
    00111   #
    00112   # First the order of events is determined, (parallel != serial, usually)
    00113   #
    00114   # Then the events are run *in order* using AppMgr().runSelectedEvents(pfn, evtNumber)
    00115   # on both Serial-Reader and Parallel-Reader processes.
    00116   #
    00117   # The string repr of everything in the TES is placed in a dictionary and
    00118   # sent to the comparison Process, which compares the two dictionaries
    00119   #
    00120   a = AppMgr()
    00121   sel = a.evtsel()
    00122   evt = a.evtsvc()
    00123
    00124   header = '/Event/Rec/Header'
    00125   sel.open( filename )
    00126   ct = 0
    00127   order = {}
    00128   fname = filename[4:] # runSelectedEvents doesn't need the "PFN:" prefix
    00129
    00130   # determine the ordering
    00131   while True :
    00132     a.run(1)
    00133     if evt[header] :
    00134       eNumber = int(evt[header].evtNumber())
    00135       order[eNumber] = ct
    00136       ct += 1
    00137     else : break
    00138
    00139   if readerType==SER :
    00140     # send the ordering details to the parallel-reader
    00141     order = switchDict( order )
    00142     qacross.put( order )
    00143     qacross.put( None  )
    00144     # changeName
    00145     serOrder = order
    00146   elif readerType==PAR :
    00147     # receive the serial ordering from queue,  and send ordering to SerialReader
    00148     for serOrder in iter(qacross.get, None) : pass
    00149     lsks = len(serOrder.keys())
    00150     lpks = len(order.keys())
    00151     print 'Events in Files (serial/parallel) : %i / %i'%( lsks, lpks )
    00152
    00153   # now run files in the order specified by the serial ordering
    00154   # and send them one by one to the comparison engine
    00155   for i in iter( serOrder.keys() ) :
    00156     if readerType==PAR  : i = order[serOrder[i]]
    00157     a.runSelectedEvents( fname, i )
    00158     lst = evt.getList() ; lst.sort()
    00159     ascii = dict( [ (l, (evt[l].__class__.__name__, evt[l].__repr__())) for l in lst ] )
    00160     qToEngine.put(ascii)
    00161   qToEngine.put(None)
    00162   print '%s Reader Finished'%(readerType)
    00163
    00164 def ComparisonEngine( pQueue, sQueue ) :
    00165    # The Comparison Engine runs on a seperate forked process and receives
    00166    # events in pairs, one each from Serial FileReader and Parallel FileReader
    00167    #
    00168    # The events arrive in Dictionary Format, d[path]=(className, string_repr)
    00169    # and are compared using the compareEvents method
    00170    #
    00171    # Results are stored in an array of bools (PerfectMatch=True, Diff=False)
    00172    #
    00173    results = []
    00174    while True :
    00175      pitem = pQueue.get()
    00176      sitem = sQueue.get()
    00177      if pitem==sitem==None : print 'Termination Signals received ok' ; break
    00178      elif pitem==None      : print 'pitem != sitem : ', pitem, sitem ; break
    00179      elif sitem==None      : print 'pitem != sitem : ', pitem, sitem ; break
    00180      results.append( compareEvents(pitem,sitem) )
    00181    print '='*80
    00182    print 'Comparison Engine Finished'
    00183    print '-'*80
    00184    print 'Total Events Checked : %i'%( len(results) )
    00185    print 'Perfect Matches      : %i'%( sum(results) )
    00186    print 'Errors               : %i'%( len(results)-sum(results) )
    00187    print '='*80
    00188
    00189 def checkForAddressDifference( a, b ) :
    00190   # the __repr__() method for Event Data Objects will return a generic
    00191   # string "DataObject at 0xADDRESS" for non-Pythonised objects
    00192   # If these objects have the same path, they are equal, but this
    00193   # cannot be tested with "==" in Python, as the memory address will
    00194   # be different for the two different DataObjects, so this method
    00195   # will check if the difference is in the address
    00196   #
    00197   # args : a, b two string representations
    00198   ref = 'DataObject at 0x'
    00199   if a[:16]==b[:16]==ref : return True
    00200   else : return False
    00201
    00202
    00203 def compareEvents( s, p ) :
    00204   # events in form of dictionary, with form
    00205   # d[ path ] = tuple( className, string_repr )
    00206
    00207   # check 1 : number of keys (paths)
    00208   sks = s.keys() ; pks = p.keys()
    00209   sks.sort() ; pks.sort()
    00210   if len(sks) == len(pks) : pass
    00211   else                    :
    00212     # There may be extra keys in the parallel file
    00213     # example: DstWriter may ask for /Event/Prev/MC/Header#1
    00214     #          but in TESSerializer, *all* DataObjects will be sent
    00215     #          including /Event/Prev and /Event/Prev/MC
    00216
    00217     # check for extra keys in the parallel file which are just containing DataObjects
    00218     # if found, remove them
    00219
    00220     extras = list(sets.Set(pks) - sets.Set(sks))
    00221     for e in extras :
    00222       if p[e][0] == 'DataObject' : pks.remove(e)
    00223       else : print 'Extra Other thing found!', e, p[e][0] ; return False
    00224
    00225   # check 2 : same paths?
    00226   if sks == pks : pass
    00227   else          : return False
    00228
    00229   # check 3 : check the content
    00230   l = len(sks)
    00231   diffs = []
    00232   for i in xrange(l) :
    00233     key = sks[i]
    00234     # compare class name
    00235     if s[key][0] == p[key][0] : pass
    00236     else : diffs.append(key)
    00237     # compare string representation
    00238     if s[key][1] == p[key][1] : pass
    00239     elif checkForAddressDifference( p[key][1], s[key][1] ) : pass
    00240     else : diffs.append(key)
    00241
    00242   # finish
    00243   if diffs : return False
    00244   else     : return True
    00245
    00246
    00247 if __name__ == '__main__' :
    00248
    00249   args = sys.argv
    00250   args.pop(0) # get rid of script name
    00251   if len(args) != 2 :
    00252     print 'Please supply two arguments : > python loadFile <parallelFile> <serialFile>'
    00253     sys.exit(0)
    00254   else :
    00255     par = 'PFN:'+args[0]
    00256     ser = 'PFN:'+args[1]
    00257     print 'Parallel File to be analysed : %s'%( par )
    00258     print 'Serial   File to be analysed : %s'%( ser )
    00259
    00260
    00261   pname = par[4:] # TFile doesn't need the "PFN:" prefix
    00262   sname = ser[4:]
    00263
    00264
    00265
    00266   qacross = Queue()
    00267   pout    = Queue()
    00268   sout    = Queue()
    00269
    00270   par = Process( target=Reader,           args=( PAR, par, qacross, pout) )
    00271   ser = Process( target=Reader,           args=( SER, ser, qacross, sout) )
    00272   com = Process( target=ComparisonEngine, args=(pout, sout)               )
    00273
    00274   com.start() ; par.start() ; ser.start()
    00275   ser.join()  ; par.join()  ; com.join()
    00276
    00277   CompareTrees( pname, sname )
    00278