| # Gaudi Framework, version v22r2 |

Home |  | Generated: Tue May 10 2011

Functions | Variables

# compareOutputFiles Namespace Reference

## Functions
def | checkKeys
def | CompareTrees
def | switchDict
def | printDict
def | Reader
def | ComparisonEngine
def | checkForAddressDifference
def | compareEvents
## Variables
string | PAR = 'PARALLEL'
string | SER = 'SERIAL'
| args = sys.argv
string | par = 'PFN:'
string | ser = 'PFN:'
list | pname = par[4:]
list | sname = ser[4:]
tuple | qacross = Queue()
tuple | pout = Queue()
tuple | sout = Queue()
tuple | com = Process( target=ComparisonEngine, args=(pout, sout) )

## Function Documentation

def compareOutputFiles::checkForAddressDifference | ( |  | a,
|  |  | b
| ) |  |

Definition at line 189 of file compareOutputFiles.py.

    00189                                       :
    00190   # the __repr__() method for Event Data Objects will return a generic
    00191   # string "DataObject at 0xADDRESS" for non-Pythonised objects
    00192   # If these objects have the same path, they are equal, but this
    00193   # cannot be tested with "==" in Python, as the memory address will
    00194   # be different for the two different DataObjects, so this method
    00195   # will check if the difference is in the address
    00196   #
    00197   # args : a, b two string representations
    00198   ref = 'DataObject at 0x'
    00199   if a[:16]==b[:16]==ref : return True
    00200   else : return False
    00201
    00202

def compareOutputFiles::checkKeys | ( |  | name ) |

Definition at line 14 of file compareOutputFiles.py.

    00014                       :
    00015   # Check the TTree keys in each file
    00016   fname = name[4:] # TFile doesn't need the "PFN:" prefix
    00017   tf = TFile(fname, 'REC')
    00018
    00019
    00020
    00021
    00022
    00023 importOptions('$STDOPTS/LHCbApplication.opts')
    00024 importOptions('$GAUDIPOOLDBROOT/options/GaudiPoolDbRoot.opts')
    00025
    00026
    00027 OutputStream("DstWriter").Output = ''
    00028 HistogramPersistencySvc().OutputFile = ''
    00029 MessageSvc(OutputLevel = ERROR)
    00030 EventSelector().PrintFreq = 100
    00031
    00032 ApplicationMgr( OutputLevel = ERROR,
    00033                 AppName = 'File Check - Serial vs Parallel' )
    00034                 # TopAlg = ['UnpackMCParticle', 'UnpackMCVertex'] )
    00035

def compareOutputFiles::compareEvents | ( |  | s,
|  |  | p
| ) |  |

Definition at line 203 of file compareOutputFiles.py.

    00204                           :
    00205   # events in form of dictionary, with form
    00206   # d[ path ] = tuple( className, string_repr )
    00207
    00208   # check 1 : number of keys (paths)
    00209   sks = s.keys() ; pks = p.keys()
    00210   sks.sort() ; pks.sort()
    00211   if len(sks) == len(pks) : pass
    00212   else                    :
    00213     # There may be extra keys in the parallel file
    00214     # example: DstWriter may ask for /Event/Prev/MC/Header#1
    00215     #          but in TESSerializer, *all* DataObjects will be sent
    00216     #          including /Event/Prev and /Event/Prev/MC
    00217
    00218     # check for extra keys in the parallel file which are just containing DataObjects
    00219     # if found, remove them
    00220
    00221     extras = list(sets.Set(pks) - sets.Set(sks))
    00222     for e in extras :
    00223       if p[e][0] == 'DataObject' : pks.remove(e)
    00224       else : print 'Extra Other thing found!', e, p[e][0] ; return False
    00225
    00226   # check 2 : same paths?
    00227   if sks == pks : pass
    00228   else          : return False
    00229
    00230   # check 3 : check the content
    00231   l = len(sks)
    00232   diffs = []
    00233   for i in xrange(l) :
    00234     key = sks[i]
    00235     # compare class name
    00236     if s[key][0] == p[key][0] : pass
    00237     else : diffs.append(key)
    00238     # compare string representation
    00239     if s[key][1] == p[key][1] : pass
    00240     elif checkForAddressDifference( p[key][1], s[key][1] ) : pass
    00241     else : diffs.append(key)
    00242
    00243   # finish
    00244   if diffs : return False
    00245   else     : return True
    00246

def compareOutputFiles::CompareTrees | ( |  | pname,
|  |  | sname
| ) |  |

Definition at line 39 of file compareOutputFiles.py.

    00039                                  :
    00040   pf = TFile(pname, 'REC')
    00041   sf = TFile(sname, 'REC')
    00042   event = '_Event'
    00043   pfks = pf.GetListOfKeys()
    00044   sfks = sf.GetListOfKeys()
    00045   pfkeys = list( [pfk.GetName() for pfk in pfks] ) ; pfkeys.sort()
    00046   sfkeys = list( [sfk.GetName() for sfk in sfks] ) ; sfkeys.sort()
    00047   pMeta = [] ; pEvent = [] ; pOther = []
    00048   for k in pfkeys :
    00049     if   k.startswith(event) : pEvent.append(k)
    00050     elif k.startswith('##')  : pMeta.append(k)
    00051     else : pOther.append(k)
    00052   sMeta = [] ; sEvent = [] ; sOther = []
    00053   for k in sfkeys :
    00054     if   k.startswith(event) : sEvent.append(k)
    00055     elif k.startswith('##')  : sMeta.append(k)
    00056     else : sOther.append(k)
    00057
    00058   if pMeta == sMeta : pass
    00059   else             : print 'Meta Data differs'
    00060
    00061   if pEvent == sEvent : pass
    00062   else               : print 'Event data differs'
    00063
    00064   if pOther != sOther :
    00065     pset = sets.Set(pOther)
    00066     sset = sets.Set(sOther)
    00067     pExtra = pset-sset
    00068     sExtra = sset-pset
    00069     if pExtra : print 'Extra Data in parallel file : ', pExtra
    00070     if sExtra : print 'Extra Data in serial   file : ', sExtra
    00071     if sExtra or pExtra : print 'Files will have different sizes'
    00072   pf.Close()
    00073   sf.Close()
    00074

def compareOutputFiles::ComparisonEngine | ( |  | pQueue,
|  |  | sQueue
| ) |  |

Definition at line 164 of file compareOutputFiles.py.

    00165                                        :
    00166    # The Comparison Engine runs on a seperate forked process and receives
    00167    # events in pairs, one each from Serial FileReader and Parallel FileReader
    00168    #
    00169    # The events arrive in Dictionary Format, d[path]=(className, string_repr)
    00170    # and are compared using the compareEvents method
    00171    #
    00172    # Results are stored in an array of bools (PerfectMatch=True, Diff=False)
    00173    #
    00174    results = []
    00175    while True :
    00176      pitem = pQueue.get()
    00177      sitem = sQueue.get()
    00178      if pitem==sitem==None : print 'Termination Signals received ok' ; break
    00179      elif pitem==None      : print 'pitem != sitem : ', pitem, sitem ; break
    00180      elif sitem==None      : print 'pitem != sitem : ', pitem, sitem ; break
    00181      results.append( compareEvents(pitem,sitem) )
    00182    print '='*80
    00183    print 'Comparison Engine Finished'
    00184    print '-'*80
    00185    print 'Total Events Checked : %i'%( len(results) )
    00186    print 'Perfect Matches      : %i'%( sum(results) )
    00187    print 'Errors               : %i'%( len(results)-sum(results) )
    00188    print '='*80

def compareOutputFiles::printDict | ( |  | d,
|  |  | name = `'unspecified'`
| ) |  |

Definition at line 92 of file compareOutputFiles.py.

    00093                                        :
    00094   # Print out a dictionary in the form
    00095   #
    00096   # Dictionary Name :
    00097   #   key     value
    00098   #   key     value
    00099   #   ...
    00100   #
    00101   print '-'*80
    00102   print 'Dictionary %s : '%(name)
    00103   for k in iter(d.keys()) :
    00104     print '\t', k, '\t', d[k]
    00105   print '-'*80
    00106

def compareOutputFiles::Reader | ( |  | readerType,
|  |  | filename,
|  |  | qacross,
|  |  | qToEngine
| ) |  |

Definition at line 107 of file compareOutputFiles.py.

    00107                                                        :
    00108   #
    00109   # Process for reading a file
    00110   # One process for reading Serial File, another for Parallel File
    00111   #
    00112   # First the order of events is determined, (parallel != serial, usually)
    00113   #
    00114   # Then the events are run *in order* using AppMgr().runSelectedEvents(pfn, evtNumber)
    00115   # on both Serial-Reader and Parallel-Reader processes.
    00116   #
    00117   # The string repr of everything in the TES is placed in a dictionary and
    00118   # sent to the comparison Process, which compares the two dictionaries
    00119   #
    00120   a = AppMgr()
    00121   sel = a.evtsel()
    00122   evt = a.evtsvc()
    00123
    00124   header = '/Event/Rec/Header'
    00125   sel.open( filename )
    00126   ct = 0
    00127   order = {}
    00128   fname = filename[4:] # runSelectedEvents doesn't need the "PFN:" prefix
    00129
    00130   # determine the ordering
    00131   while True :
    00132     a.run(1)
    00133     if evt[header] :
    00134       eNumber = int(evt[header].evtNumber())
    00135       order[eNumber] = ct
    00136       ct += 1
    00137     else : break
    00138
    00139   if readerType==SER :
    00140     # send the ordering details to the parallel-reader
    00141     order = switchDict( order )
    00142     qacross.put( order )
    00143     qacross.put( None  )
    00144     # changeName
    00145     serOrder = order
    00146   elif readerType==PAR :
    00147     # receive the serial ordering from queue,  and send ordering to SerialReader
    00148     for serOrder in iter(qacross.get, None) : pass
    00149     lsks = len(serOrder.keys())
    00150     lpks = len(order.keys())
    00151     print 'Events in Files (serial/parallel) : %i / %i'%( lsks, lpks )
    00152
    00153   # now run files in the order specified by the serial ordering
    00154   # and send them one by one to the comparison engine
    00155   for i in iter( serOrder.keys() ) :
    00156     if readerType==PAR  : i = order[serOrder[i]]
    00157     a.runSelectedEvents( fname, i )
    00158     lst = evt.getList() ; lst.sort()
    00159     ascii = dict( [ (l, (evt[l].__class__.__name__, evt[l].__repr__())) for l in lst ] )
    00160     qToEngine.put(ascii)
    00161   qToEngine.put(None)
    00162   print '%s Reader Finished'%(readerType)
    00163

def compareOutputFiles::switchDict | ( |  | d ) |

Definition at line 75 of file compareOutputFiles.py.

    00076                     :
    00077   # switch a dictionary around ;  make the values the keys, and vice versa
    00078   # only works if all values are unique
    00079   nkeys = len(d.keys())
    00080   vals = d.values()
    00081   nvals = len(vals)
    00082   for v in vals :
    00083     if vals.count(v) > 1 :
    00084       print 'Dictionary cannot be switched, values not unique'
    00085       return None
    00086   print 'Dict has keys/values : %i/%i'%( nkeys, nvals )
    00087   pairs = d.items() # returns (key, val) tuples in a list
    00088   newd = {}
    00089   for k, entry in pairs : newd[entry] = k
    00090   return newd
    00091

## Variable Documentation

compareOutputFiles::args = sys.argv

Definition at line 249 of file compareOutputFiles.py.

tuple compareOutputFiles::com = Process( target=ComparisonEngine, args=(pout, sout) )

Definition at line 272 of file compareOutputFiles.py.

tuple compareOutputFiles::par = 'PFN:'

Definition at line 255 of file compareOutputFiles.py.

string compareOutputFiles::PAR = 'PARALLEL'

Definition at line 36 of file compareOutputFiles.py.

list compareOutputFiles::pname = par[4:]

Definition at line 261 of file compareOutputFiles.py.

tuple compareOutputFiles::pout = Queue()

Definition at line 267 of file compareOutputFiles.py.

tuple compareOutputFiles::qacross = Queue()

Definition at line 266 of file compareOutputFiles.py.

tuple compareOutputFiles::ser = 'PFN:'

Definition at line 256 of file compareOutputFiles.py.

string compareOutputFiles::SER = 'SERIAL'

Definition at line 37 of file compareOutputFiles.py.

list compareOutputFiles::sname = ser[4:]

Definition at line 262 of file compareOutputFiles.py.

tuple compareOutputFiles::sout = Queue()

Definition at line 268 of file compareOutputFiles.py.

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Defines