compareOutputFiles Namespace Reference

Functions
def	checkKeys
def	CompareTrees
def	switchDict
def	printDict
def	Reader
def	ComparisonEngine
def	checkForAddressDifference
def	compareEvents
Variables
string	PAR = 'PARALLEL'
string	SER = 'SERIAL'
	args = sys.argv
string	par = 'PFN:'
string	ser = 'PFN:'
list	pname = par[4:]
list	sname = ser[4:]
tuple	qacross = Queue()
tuple	pout = Queue()
tuple	sout = Queue()
tuple	com = Process( target=ComparisonEngine, args=(pout, sout) )

---

Function Documentation

def compareOutputFiles::checkForAddressDifference	(		a,
			b
	)

Definition at line 189 of file compareOutputFiles.py.

                                      : 
  # the __repr__() method for Event Data Objects will return a generic
  # string "DataObject at 0xADDRESS" for non-Pythonised objects
  # If these objects have the same path, they are equal, but this 
  # cannot be tested with "==" in Python, as the memory address will
  # be different for the two different DataObjects, so this method
  # will check if the difference is in the address
  #
  # args : a, b two string representations
  ref = 'DataObject at 0x'
  if a[:16]==b[:16]==ref : return True
  else : return False
  
  

def compareOutputFiles::checkKeys

(

name

)

Definition at line 14 of file compareOutputFiles.py.

                      : 
  # Check the TTree keys in each file
  fname = name[4:] # TFile doesn't need the "PFN:" prefix  
  tf = TFile(fname, 'REC')

  



importOptions('$STDOPTS/LHCbApplication.opts')
importOptions('$GAUDIPOOLDBROOT/options/GaudiPoolDbRoot.opts')


OutputStream("DstWriter").Output = ''
HistogramPersistencySvc().OutputFile = ''
MessageSvc(OutputLevel = ERROR)
EventSelector().PrintFreq = 100

ApplicationMgr( OutputLevel = ERROR,
                AppName = 'File Check - Serial vs Parallel' )
                # TopAlg = ['UnpackMCParticle', 'UnpackMCVertex'] )

def compareOutputFiles::compareEvents	(		s,
			p
	)

Definition at line 203 of file compareOutputFiles.py.

00203                           :
00204   # events in form of dictionary, with form
00205   # d[ path ] = tuple( className, string_repr )
00206   
00207   # check 1 : number of keys (paths)
00208   sks = s.keys() ; pks = p.keys()
00209   sks.sort() ; pks.sort()    
00210   if len(sks) == len(pks) : pass
00211   else                    :
00212     # There may be extra keys in the parallel file
00213     # example: DstWriter may ask for /Event/Prev/MC/Header#1
00214     #          but in TESSerializer, *all* DataObjects will be sent
00215     #          including /Event/Prev and /Event/Prev/MC
00216     
00217     # check for extra keys in the parallel file which are just containing DataObjects
00218     # if found, remove them
00219     
00220     extras = list(sets.Set(pks) - sets.Set(sks))
00221     for e in extras :
00222       if p[e][0] == 'DataObject' : pks.remove(e)
00223       else : print 'Extra Other thing found!', e, p[e][0] ; return False
00224       
00225   # check 2 : same paths?
00226   if sks == pks : pass
00227   else          : return False
00228   
00229   # check 3 : check the content
00230   l = len(sks)
00231   diffs = []
00232   for i in xrange(l) : 
00233     key = sks[i]
00234     # compare class name 
00235     if s[key][0] == p[key][0] : pass 
00236     else : diffs.append(key) 
00237     # compare string representation
00238     if s[key][1] == p[key][1] : pass
00239     elif checkForAddressDifference( p[key][1], s[key][1] ) : pass
00240     else : diffs.append(key)
00241     
00242   # finish
00243   if diffs : return False
00244   else     : return True
00245   
00246   
if __name__ == '__main__' :

def compareOutputFiles::CompareTrees	(		pname,
			sname
	)

Definition at line 39 of file compareOutputFiles.py.

                                 : 
  pf = TFile(pname, 'REC')
  sf = TFile(sname, 'REC')
  event = '_Event'
  pfks = pf.GetListOfKeys()
  sfks = sf.GetListOfKeys()
  pfkeys = list( [pfk.GetName() for pfk in pfks] ) ; pfkeys.sort()
  sfkeys = list( [sfk.GetName() for sfk in sfks] ) ; sfkeys.sort()
  pMeta = [] ; pEvent = [] ; pOther = []
  for k in pfkeys : 
    if   k.startswith(event) : pEvent.append(k)
    elif k.startswith('##')  : pMeta.append(k)
    else : pOther.append(k)
  sMeta = [] ; sEvent = [] ; sOther = []
  for k in sfkeys : 
    if   k.startswith(event) : sEvent.append(k)
    elif k.startswith('##')  : sMeta.append(k)
    else : sOther.append(k)    
    
  if pMeta == sMeta : pass
  else             : print 'Meta Data differs'
  
  if pEvent == sEvent : pass
  else               : print 'Event data differs'
  
  if pOther != sOther : 
    pset = sets.Set(pOther)
    sset = sets.Set(sOther)
    pExtra = pset-sset
    sExtra = sset-pset
    if pExtra : print 'Extra Data in parallel file : ', pExtra
    if sExtra : print 'Extra Data in serial   file : ', sExtra
    if sExtra or pExtra : print 'Files will have different sizes'
  pf.Close()
  sf.Close()

def compareOutputFiles::ComparisonEngine	(		pQueue,
			sQueue
	)

Definition at line 164 of file compareOutputFiles.py.

00164                                        :
00165    # The Comparison Engine runs on a seperate forked process and receives
00166    # events in pairs, one each from Serial FileReader and Parallel FileReader
00167    #
00168    # The events arrive in Dictionary Format, d[path]=(className, string_repr)
00169    # and are compared using the compareEvents method
00170    #
00171    # Results are stored in an array of bools (PerfectMatch=True, Diff=False)
00172    #
00173    results = []
00174    while True :
00175      pitem = pQueue.get()
00176      sitem = sQueue.get()
00177      if pitem==sitem==None : print 'Termination Signals received ok' ; break
00178      elif pitem==None      : print 'pitem != sitem : ', pitem, sitem ; break
00179      elif sitem==None      : print 'pitem != sitem : ', pitem, sitem ; break
00180      results.append( compareEvents(pitem,sitem) )
00181    print '='*80
00182    print 'Comparison Engine Finished'
00183    print '-'*80
00184    print 'Total Events Checked : %i'%( len(results) )
00185    print 'Perfect Matches      : %i'%( sum(results) )
00186    print 'Errors               : %i'%( len(results)-sum(results) )
00187    print '='*80  
00188    
def checkForAddressDifference( a, b ) : 

def compareOutputFiles::printDict	(		d,
			name = 'unspecified'
	)

Definition at line 92 of file compareOutputFiles.py.

00092                                        :
00093   # Print out a dictionary in the form 
00094   # 
00095   # Dictionary Name : 
00096   #   key     value
00097   #   key     value
00098   #   ...
00099   #
00100   print '-'*80
00101   print 'Dictionary %s : '%(name)
00102   for k in iter(d.keys()) : 
00103     print '\t', k, '\t', d[k]
00104   print '-'*80
00105 
00106                 
def Reader( readerType, filename, qacross, qToEngine ) : 

def compareOutputFiles::Reader	(		readerType,
			filename,
			qacross,
			qToEngine
	)

Definition at line 107 of file compareOutputFiles.py.

                                                       : 
  #
  # Process for reading a file
  # One process for reading Serial File, another for Parallel File
  #
  # First the order of events is determined, (parallel != serial, usually)
  #
  # Then the events are run *in order* using AppMgr().runSelectedEvents(pfn, evtNumber)
  # on both Serial-Reader and Parallel-Reader processes.
  #
  # The string repr of everything in the TES is placed in a dictionary and 
  # sent to the comparison Process, which compares the two dictionaries
  #
  a = AppMgr()
  sel = a.evtsel()
  evt = a.evtsvc()
  
  header = '/Event/Rec/Header'
  sel.open( filename )
  ct = 0 
  order = {}
  fname = filename[4:] # runSelectedEvents doesn't need the "PFN:" prefix
    
  # determine the ordering
  while True : 
    a.run(1)
    if evt[header] : 
      eNumber = int(evt[header].evtNumber())
      order[eNumber] = ct
      ct += 1
    else : break

  if readerType==SER :  
    # send the ordering details to the parallel-reader
    order = switchDict( order )
    qacross.put( order )
    qacross.put( None  )
    # changeName
    serOrder = order
  elif readerType==PAR : 
    # receive the serial ordering from queue,  and send ordering to SerialReader
    for serOrder in iter(qacross.get, None) : pass
    lsks = len(serOrder.keys())
    lpks = len(order.keys())
    print 'Events in Files (serial/parallel) : %i / %i'%( lsks, lpks )
  
  # now run files in the order specified by the serial ordering
  # and send them one by one to the comparison engine  
  for i in iter( serOrder.keys() ) : 
    if readerType==PAR  : i = order[serOrder[i]]
    a.runSelectedEvents( fname, i )
    lst = evt.getList() ; lst.sort()  
    ascii = dict( [ (l, (evt[l].__class__.__name__, evt[l].__repr__())) for l in lst ] )    
    qToEngine.put(ascii)
  qToEngine.put(None)
  print '%s Reader Finished'%(readerType)

def compareOutputFiles::switchDict

(

d

)

Definition at line 75 of file compareOutputFiles.py.

00075                     :
00076   # switch a dictionary around ;  make the values the keys, and vice versa
00077   # only works if all values are unique
00078   nkeys = len(d.keys())
00079   vals = d.values()
00080   nvals = len(vals)
00081   for v in vals : 
00082     if vals.count(v) > 1 : 
00083       print 'Dictionary cannot be switched, values not unique'
00084       return None
00085   print 'Dict has keys/values : %i/%i'%( nkeys, nvals )
00086   pairs = d.items() # returns (key, val) tuples in a list
00087   newd = {}
00088   for k, entry in pairs : newd[entry] = k
00089   return newd
00090 
00091 
def printDict( d, name='unspecified' ) :

---

Variable Documentation

compareOutputFiles::args = sys.argv

Definition at line 249 of file compareOutputFiles.py.

tuple compareOutputFiles::com = Process( target=ComparisonEngine, args=(pout, sout) )

Definition at line 272 of file compareOutputFiles.py.

tuple compareOutputFiles::par = 'PFN:'

Definition at line 255 of file compareOutputFiles.py.

string compareOutputFiles::PAR = 'PARALLEL'

Definition at line 36 of file compareOutputFiles.py.

list compareOutputFiles::pname = par[4:]

Definition at line 261 of file compareOutputFiles.py.

tuple compareOutputFiles::pout = Queue()

Definition at line 267 of file compareOutputFiles.py.

tuple compareOutputFiles::qacross = Queue()

Definition at line 266 of file compareOutputFiles.py.

tuple compareOutputFiles::ser = 'PFN:'

Definition at line 256 of file compareOutputFiles.py.

string compareOutputFiles::SER = 'SERIAL'

Definition at line 37 of file compareOutputFiles.py.

list compareOutputFiles::sname = ser[4:]

Definition at line 262 of file compareOutputFiles.py.

tuple compareOutputFiles::sout = Queue()

Definition at line 268 of file compareOutputFiles.py.