All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
Functions | Variables
compareOutputFiles Namespace Reference

Functions

def checkKeys
 
def CompareTrees
 
def switchDict
 
def printDict
 
def Reader
 
def ComparisonEngine
 
def checkForAddressDifference
 
def compareEvents
 
def CheckFileRecords
 
def LumiFSR
 
def GetFSRdict
 
def CompareFSR
 

Variables

string PAR = 'PARALLEL'
 
string SER = 'SERIAL'
 
 args = sys.argv
 
string par = 'PFN:'
 
string ser = 'PFN:'
 
list pname = par[4:]
 
list sname = ser[4:]
 
tuple qacross = Queue()
 
tuple pout = Queue()
 
tuple sout = Queue()
 
tuple com = Process( target = ComparisonEngine, args = ( pout, sout ) )
 
tuple sp = Process( target = GetFSRdict, args = ( ser, sout ) )
 
tuple pp = Process( target = GetFSRdict, args = ( par, pout ) )
 
tuple cp = Process( target = CompareFSR, args = ( pout, sout ) )
 

Function Documentation

def compareOutputFiles.CheckFileRecords (   par,
  ser 
)

Definition at line 251 of file compareOutputFiles.py.

251 
252 def CheckFileRecords( par, ser ):
253 
254  print "Checking File Records"
255 
256  parFSR = GetFSRdicts( par )
257  serFSR = GetFSRdicts( ser )
258 
259 
260 
261  diff1 = set( parFSR["TimeSpanFSR"].iteritems() ) - set( serFSR["TimeSpanFSR"].iteritems() )
262  diff2 = set( parFSR["EventCountFSR"].iteritems() ) - set( serFSR["EventCountFSR"].iteritems() )
263 
264  print "\nDifferent entries in TimeSpanFSR: \t" + str( len( diff1 ) ) + "\nDifferent entries in EventCountFSR:\t" + str( len( diff2 ) )
265 
266  for k in ["LumiFSRBeamCrossing", "LumiFSRBeam2", "LumiFSRNoBeam"]:
267  diff3 = set( parFSR[k]["key"] ) - set( serFSR[k]["key"] )
268  diff4 = set( parFSR[k]["incr"] ) - set( serFSR[k]["incr"] )
269  diff5 = set( parFSR[k]["integral"] ) - set( serFSR[k]["integral"] )
270  print "Different entries in " + str( k ) + ": \tkey: " + str( len( diff3 ) ) + " increment: " + str( len( diff4 ) ) + " integral: " + str( len( diff5 ) )
271 
def compareOutputFiles.checkForAddressDifference (   a,
  b 
)

Definition at line 194 of file compareOutputFiles.py.

194 
195 def checkForAddressDifference( a, b ) :
196  # the __repr__() method for Event Data Objects will return a generic
197  # string "DataObject at 0xADDRESS" for non-Pythonised objects
198  # If these objects have the same path, they are equal, but this
199  # cannot be tested with "==" in Python, as the memory address will
200  # be different for the two different DataObjects, so this method
201  # will check if the difference is in the address
202  #
203  # args : a, b two string representations
204  ref = 'DataObject at 0x'
205  if a[:16] == b[:16] == ref : return True
206  else : return False
207 
def compareOutputFiles.checkKeys (   name)

Definition at line 15 of file compareOutputFiles.py.

15 
16 def checkKeys( name ) :
17  # Check the TTree keys in each file
18  fname = name[4:] # TFile doesn't need the "PFN:" prefix
19  tf = TFile( fname, 'REC' )
20 
21 
22 
23 
24 
25 importOptions( '$STDOPTS/LHCbApplication.opts' )
26 #importOptions( '$GAUDIPOOLDBROOT/options/GaudiPoolDbRoot.opts' )
27 importOptions( '$GAUDICNVROOT/options/Setup.opts' )
28 
29 
30 OutputStream( "DstWriter" ).Output = ''
31 HistogramPersistencySvc().OutputFile = ''
32 MessageSvc( OutputLevel = ERROR )
33 EventSelector().PrintFreq = 100
34 
35 ApplicationMgr( OutputLevel = ERROR,
36  AppName = 'File Check - Serial vs Parallel' )
37  # TopAlg = ['UnpackMCParticle', 'UnpackMCVertex'] )
OutputStream
A small to stream Data I/O.
Definition: OutputStream.h:27
HistogramPersistencySvc
HistogramPersistencySvc class implementation definition.
Definition: HistogramPersistencySvc.h:48
ApplicationMgr
The Application Manager class.
Definition: ApplicationMgr.h:46
EventSelector
Definition of class EventSelector.
Definition: EventSelector.h:53
def compareOutputFiles.compareEvents (   s,
  p 
)

Definition at line 208 of file compareOutputFiles.py.

208 
209 def compareEvents( s, p ) :
210  # events in form of dictionary, with form
211  # d[ path ] = tuple( className, string_repr )
212 
213  # check 1 : number of keys (paths)
214  sks = s.keys() ; pks = p.keys()
215  sks.sort() ; pks.sort()
216  if len( sks ) == len( pks ) : pass
217  else :
218  # There may be extra keys in the parallel file
219  # example: DstWriter may ask for /Event/Prev/MC/Header#1
220  # but in TESSerializer, *all* DataObjects will be sent
221  # including /Event/Prev and /Event/Prev/MC
222 
223  # check for extra keys in the parallel file which are just containing DataObjects
224  # if found, remove them
225 
226  extras = list( set( pks ) - set( sks ) )
227  for e in extras :
228  if p[e][0] == 'DataObject' : pks.remove( e )
229  else : print 'Extra Other thing found!', e, p[e][0] ; return False
230 
231  # check 2 : same paths?
232  if sks == pks : pass
233  else : return False
234 
235  # check 3 : check the content
236  l = len( sks )
237  diffs = []
238  for i in xrange( l ) :
239  key = sks[i]
240  # compare class name
241  if s[key][0] == p[key][0] : pass
242  else : diffs.append( key )
243  # compare string representation
244  if s[key][1] == p[key][1] : pass
245  elif checkForAddressDifference( p[key][1], s[key][1] ) : pass
246  else : diffs.append( key )
247 
248  # finish
249  if diffs : return False
250  else : return True
def compareOutputFiles.CompareFSR (   pout,
  sout 
)

Definition at line 356 of file compareOutputFiles.py.

356 
357 def CompareFSR( pout, sout ):
358 
359  parFSR = pout.get()
360  serFSR = sout.get()
361 
362  print "Comparing File Records"
363 
364  diff1 = set( parFSR["TimeSpanFSR"].iteritems() ) - set( serFSR["TimeSpanFSR"].iteritems() )
365  diff2 = set( parFSR["EventCountFSR"].iteritems() ) - set( serFSR["EventCountFSR"].iteritems() )
366 
367  print "\nDifferent entries in TimeSpanFSR: \t" + str( len( diff1 ) ) + "\nDifferent entries in EventCountFSR:\t" + str( len( diff2 ) )
368 
369  for k in ["LumiFSRBeamCrossing", "LumiFSRBeam2", "LumiFSRNoBeam"]:
370  diff3 = set( parFSR[k]['key'] ) - set( serFSR[k]['key'] )
371  diff4 = set( parFSR[k]['incr'] ) - set( serFSR[k]['incr'] )
372  diff5 = set( parFSR[k]['integral'] ) - set( serFSR[k]["integral"] )
373  print "Different entries in " + str( k ) + ": \tkey: " + str( len( diff3 ) ) + " increment: " + str( len( diff4 ) ) + " integral: " + str( len( diff5 ) )
374 
375 
376  print "\nParallel: \n" + str( parFSR )
377  print "\nSerial: \n" + str( serFSR )
def compareOutputFiles.CompareTrees (   pname,
  sname 
)

Definition at line 41 of file compareOutputFiles.py.

41 
42 def CompareTrees( pname, sname ) :
43  pf = TFile( pname, 'REC' )
44  sf = TFile( sname, 'REC' )
45  event = '_Event'
46  pfks = pf.GetListOfKeys()
47  sfks = sf.GetListOfKeys()
48  pfkeys = list( [pfk.GetName() for pfk in pfks] ) ; pfkeys.sort()
49  sfkeys = list( [sfk.GetName() for sfk in sfks] ) ; sfkeys.sort()
50  pMeta = [] ; pEvent = [] ; pOther = []
51  for k in pfkeys :
52  if k.startswith( event ) : pEvent.append( k )
53  elif k.startswith( '##' ) : pMeta.append( k )
54  else : pOther.append( k )
55  sMeta = [] ; sEvent = [] ; sOther = []
56  for k in sfkeys :
57  if k.startswith( event ) : sEvent.append( k )
58  elif k.startswith( '##' ) : sMeta.append( k )
59  else : sOther.append( k )
60 
61  if pMeta == sMeta : pass
62  else : print 'Meta Data differs'
63 
64  if pEvent == sEvent : pass
65  else : print 'Event data differs'
66 
67  if pOther != sOther :
68  pset = set( pOther )
69  sset = set( sOther )
70  pExtra = pset - sset
71  sExtra = sset - pset
72  if pExtra : print 'Extra Data in parallel file : ', pExtra
73  if sExtra : print 'Extra Data in serial file : ', sExtra
74  if sExtra or pExtra : print 'Files will have different sizes'
75  pf.Close()
76  sf.Close()
def compareOutputFiles.ComparisonEngine (   pQueue,
  sQueue 
)

Definition at line 169 of file compareOutputFiles.py.

169 
170 def ComparisonEngine( pQueue, sQueue ) :
171  # The Comparison Engine runs on a seperate forked process and receives
172  # events in pairs, one each from Serial FileReader and Parallel FileReader
173  #
174  # The events arrive in Dictionary Format, d[path]=(className, string_repr)
175  # and are compared using the compareEvents method
176  #
177  # Results are stored in an array of bools (PerfectMatch=True, Diff=False)
178  #
179  results = []
180  while True :
181  pitem = pQueue.get()
182  sitem = sQueue.get()
183  if pitem == sitem == None : print 'Termination Signals received ok' ; break
184  elif pitem == None : print 'pitem != sitem : ', pitem, sitem ; break
185  elif sitem == None : print 'pitem != sitem : ', pitem, sitem ; break
186  results.append( compareEvents( pitem, sitem ) )
187  print '=' * 80
188  print 'Comparison Engine Finished'
189  print '-' * 80
190  print 'Total Events Checked : %i' % ( len( results ) )
191  print 'Perfect Matches : %i' % ( sum( results ) )
192  print 'Errors : %i' % ( len( results ) - sum( results ) )
193  print '=' * 80
sum
double sum(double x, double y, double z)
Definition: 3DoubleFuncTest.cpp:26
def compareOutputFiles.GetFSRdict (   filename,
  queue 
)

Definition at line 300 of file compareOutputFiles.py.

300 
301 def GetFSRdict( filename, queue ):
302 
303 
304  FSR = { "TimeSpanFSR" : {'earliest': 0, 'latest': 0},
305  "LumiFSRBeamCrossing" : {'key': 0, 'incr': 0, 'integral':0},
306  "LumiFSRBeam1" : {'key': 0, 'incr': 0, 'integral':0},
307  "LumiFSRBeam2" : {'key': 0, 'incr': 0, 'integral':0},
308  "LumiFSRNoBeam" : {'key': 0, 'incr': 0, 'integral':0},
309  "EventCountFSR" : {'input': 0, 'output': 0, 'statusFlag': 0}}
310 
311  options = "from LumiAlgs.LumiFsrReaderConf import LumiFsrReaderConf as LumiFsrReader; LumiFsrReader().OutputLevel = INFO; LumiFsrReader().inputFiles = ['%s'] ;" %filename
312  options += "LumiFsrReader().Persistency='ROOT'; LumiFsrReader().EvtMax = 1; from Configurables import LHCbApp; LHCbApp().Persistency='ROOT'; from Configurables import CondDB, DDDBConf;"
313  options += " CondDB().UseLatestTags=['%s']; DDDBConf(DataType='%s');"%( 2011, 2011 )
314  exec options
315  app = AppMgr()
316  app.run( 1 )
317  fsr = app.filerecordsvc()
318 
319  lst = fsr.getHistoNames()
320 
321  if lst :
322  for l in lst :
323 
324  ob = fsr.retrieveObject( l )
325 
326 
327  if "LumiFSR" in l:
328 
329  assert ob.numberOfObjects() == 1
330  k = ob.containedObject( 0 )
331  runs, files, keys, increment, integral = LumiFSR( k )
332 
333  FSR[l[l.rfind( '/' ) + 1:]]['runs'] = runs
334  FSR[l[l.rfind( '/' ) + 1:]]['files'] = files
335  FSR[l[l.rfind( '/' ) + 1:]]['key'] = keys
336  FSR[l[l.rfind( '/' ) + 1:]]['incr'] = increment
337  FSR[l[l.rfind( '/' ) + 1:]]['integral'] = integral
338 
339 
340 
341  if "TimeSpanFSR" in l:
342 
343  FSR["TimeSpanFSR"]['earliest'] = ob.containedObject( 0 ).earliest()
344  FSR["TimeSpanFSR"]['latest'] = ob.containedObject( 0 ).latest()
345 
346  if "EventCountFSR" in l:
347 
348  FSR["EventCountFSR"]['input'] = ob.input()
349  FSR["EventCountFSR"]['output'] = ob.output()
350  FSR["EventCountFSR"]['statusFlag'] = ob.statusFlag()
351 
352  app.stop()
353  app.finalize()
354 
355  queue.put( FSR )
def compareOutputFiles.LumiFSR (   lumi)

Definition at line 272 of file compareOutputFiles.py.

272 
273 def LumiFSR( lumi ):
274 
275  runs = []
276  files = []
277  info = {}
278  keys = []
279 
280  for r in lumi.runNumbers() :
281  runs.append( r )
282 
283  for f in lumi.fileIDs() :
284  files.append( f )
285  s = str( lumi )
286  sa = s.split( "info (key/incr/integral) : " )[-1]
287  sa = sa.split( '/' )[:-1]
288 
289  key = []
290  incr = []
291  integral = []
292  for rec in sa :
293  k, i, t = rec.split()
294  key.append( int( k ) )
295  incr.append( int( i ) )
296  integral.append( int( t ) )
297 
298  return ( runs, files, key, incr, integral )
299 
def compareOutputFiles.printDict (   d,
  name = 'unspecified' 
)

Definition at line 94 of file compareOutputFiles.py.

94 
95 def printDict( d, name = 'unspecified' ) :
96  # Print out a dictionary in the form
97  #
98  # Dictionary Name :
99  # key value
100  # key value
101  # ...
102  #
103  print '-' * 80
104  print 'Dictionary %s : ' % ( name )
105  for k in iter( d.keys() ) :
106  print '\t', k, '\t', d[k]
107  print '-' * 80
108 
def compareOutputFiles.Reader (   readerType,
  filename,
  qacross,
  qToEngine 
)

Definition at line 109 of file compareOutputFiles.py.

109 
110 def Reader( readerType, filename, qacross, qToEngine ) :
111  #
112  # Process for reading a file
113  # One process for reading Serial File, another for Parallel File
114  #
115  # First the order of events is determined, (parallel != serial, usually)
116  #
117  # Then the events are run *in order* using AppMgr().runSelectedEvents(pfn, evtNumber)
118  # on both Serial-Reader and Parallel-Reader processes.
119  #
120  # The string repr of everything in the TES is placed in a dictionary and
121  # sent to the comparison Process, which compares the two dictionaries
122  #
123  a = AppMgr()
124  sel = a.evtsel()
125  evt = a.evtsvc()
126 
127  header = '/Event/Rec/Header'
128  sel.open( filename )
129  ct = 0
130  order = {}
131  fname = filename[4:] # runSelectedEvents doesn't need the "PFN:" prefix
132 
133  # determine the ordering
134  while True :
135  a.run( 1 )
136  if evt[header] :
137  eNumber = int( evt[header].evtNumber() )
138  order[eNumber] = ct
139  ct += 1
140  else : break
141 
142  if readerType == SER :
143  # send the ordering details to the parallel-reader
144  order = switchDict( order )
145  qacross.put( order )
146  qacross.put( None )
147  # changeName
148  serOrder = order
149  elif readerType == PAR :
150  # receive the serial ordering from queue, and send ordering to SerialReader
151  for serOrder in iter( qacross.get, None ) : pass
152  lsks = len( serOrder.keys() )
153  lpks = len( order.keys() )
154  print 'Events in Files (serial/parallel) : %i / %i' % ( lsks, lpks )
155 
156  # now run files in the order specified by the serial ordering
157  # and send them one by one to the comparison engine
158  for i in iter( serOrder.keys() ) :
159  if readerType == PAR : i = order[serOrder[i]]
160 
161  a.runSelectedEvents( fname, i )
162  lst = evt.getList()
163 
164  lst.sort()
165  ascii = dict( [ ( l, ( evt[l].__class__.__name__, evt[l].__repr__() ) ) for l in lst ] )
166  qToEngine.put( ascii )
167  qToEngine.put( None )
168  print '%s Reader Finished' % ( readerType )
def compareOutputFiles.switchDict (   d)

Definition at line 77 of file compareOutputFiles.py.

77 
78 def switchDict( d ) :
79  # switch a dictionary around ; make the values the keys, and vice versa
80  # only works if all values are unique
81  nkeys = len( d.keys() )
82  vals = d.values()
83  nvals = len( vals )
84  for v in vals :
85  if vals.count( v ) > 1 :
86  print 'Dictionary cannot be switched, values not unique'
87  return None
88  print 'Dict has keys/values : %i/%i' % ( nkeys, nvals )
89  pairs = d.items() # returns (key, val) tuples in a list
90  newd = {}
91  for k, entry in pairs : newd[entry] = k
92  return newd
93 

Variable Documentation

compareOutputFiles.args = sys.argv

Definition at line 380 of file compareOutputFiles.py.

tuple compareOutputFiles.com = Process( target = ComparisonEngine, args = ( pout, sout ) )

Definition at line 401 of file compareOutputFiles.py.

tuple compareOutputFiles.cp = Process( target = CompareFSR, args = ( pout, sout ) )

Definition at line 418 of file compareOutputFiles.py.

string compareOutputFiles.PAR = 'PARALLEL'

Definition at line 38 of file compareOutputFiles.py.

list compareOutputFiles.par = 'PFN:'

Definition at line 386 of file compareOutputFiles.py.

list compareOutputFiles.pname = par[4:]

Definition at line 392 of file compareOutputFiles.py.

tuple compareOutputFiles.pout = Queue()

Definition at line 396 of file compareOutputFiles.py.

tuple compareOutputFiles.pp = Process( target = GetFSRdict, args = ( par, pout ) )

Definition at line 417 of file compareOutputFiles.py.

tuple compareOutputFiles.qacross = Queue()

Definition at line 395 of file compareOutputFiles.py.

string compareOutputFiles.SER = 'SERIAL'

Definition at line 39 of file compareOutputFiles.py.

list compareOutputFiles.ser = 'PFN:'

Definition at line 387 of file compareOutputFiles.py.

list compareOutputFiles.sname = ser[4:]

Definition at line 393 of file compareOutputFiles.py.

tuple compareOutputFiles.sout = Queue()

Definition at line 397 of file compareOutputFiles.py.

tuple compareOutputFiles.sp = Process( target = GetFSRdict, args = ( ser, sout ) )

Definition at line 416 of file compareOutputFiles.py.