| # Gaudi Framework, version v25r0 |

Home |  | Generated: Mon Feb 17 2014

- Main Page
- Related Pages
- Modules
- Namespaces
- Classes
- Files

- Namespace List
- Namespace Members

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
Functions | Variables
compareOutputFiles Namespace Reference

## Functions
def | checkKeys
def | CompareTrees
def | switchDict
def | printDict
def | Reader
def | ComparisonEngine
def | checkForAddressDifference
def | compareEvents
def | CheckFileRecords
def | LumiFSR
def | GetFSRdict
def | CompareFSR

## Variables
string | PAR 'PARALLEL'
string | SER 'SERIAL'
| args sys.argv
string | par 'PFN:'
string | ser 'PFN:'
list | pname par[4:]
list | sname ser[4:]
tuple | qacross Queue()
tuple | pout Queue()
tuple | sout Queue()
tuple | com Process( target = ComparisonEngine, args = ( pout, sout ) )
tuple | sp Process( target = GetFSRdict, args = ( ser, sout ) )
tuple | pp Process( target = GetFSRdict, args = ( par, pout ) )
tuple | cp Process( target = CompareFSR, args = ( pout, sout ) )

## Function Documentation

def compareOutputFiles.CheckFileRecords | ( |  | par,
|  |  | ser
| ) |  |

Definition at line 251 of file compareOutputFiles.py.

251
252 def CheckFileRecords( par, ser ):
253
254 print "Checking File Records"
255
256 parFSR = GetFSRdicts( par )
257 serFSR = GetFSRdicts( ser )
258
259
260
261 diff1 = set( parFSR["TimeSpanFSR"].iteritems() ) - set( serFSR["TimeSpanFSR"].iteritems() )
262 diff2 = set( parFSR["EventCountFSR"].iteritems() ) - set( serFSR["EventCountFSR"].iteritems() )
263
264 print "\nDifferent entries in TimeSpanFSR: \t" + str( len( diff1 ) ) + "\nDifferent entries in EventCountFSR:\t" + str( len( diff2 ) )
265
266 for k in ["LumiFSRBeamCrossing", "LumiFSRBeam2", "LumiFSRNoBeam"]:
267 diff3 = set( parFSR[k]["key"] ) - set( serFSR[k]["key"] )
268 diff4 = set( parFSR[k]["incr"] ) - set( serFSR[k]["incr"] )
269 diff5 = set( parFSR[k]["integral"] ) - set( serFSR[k]["integral"] )
270 print "Different entries in " + str( k ) + ": \tkey: " + str( len( diff3 ) ) + " increment: " + str( len( diff4 ) ) + " integral: " + str( len( diff5 ) )
271

def compareOutputFiles.checkForAddressDifference | ( |  | a,
|  |  | b
| ) |  |

Definition at line 194 of file compareOutputFiles.py.

194
195 def checkForAddressDifference( a, b ) :
196 # the __repr__() method for Event Data Objects will return a generic
197 # string "DataObject at 0xADDRESS" for non-Pythonised objects
198 # If these objects have the same path, they are equal, but this
199 # cannot be tested with "==" in Python, as the memory address will
200 # be different for the two different DataObjects, so this method
201 # will check if the difference is in the address
202 #
203 # args : a, b two string representations
204 ref = 'DataObject at 0x'
205 if a[:16] == b[:16] == ref : return True
206 else : return False
207

def compareOutputFiles.checkKeys | ( |  | name | ) |

Definition at line 15 of file compareOutputFiles.py.

15
16 def checkKeys( name ) :
17 # Check the TTree keys in each file
18 fname = name[4:] # TFile doesn't need the "PFN:" prefix
19 tf = TFile( fname, 'REC' )
20
21
22
23
24
25 importOptions( '$STDOPTS/LHCbApplication.opts' )
26 #importOptions( '$GAUDIPOOLDBROOT/options/GaudiPoolDbRoot.opts' )
27 importOptions( '$GAUDICNVROOT/options/Setup.opts' )
28
29
30 OutputStream( "DstWriter" ).Output = ''
31 HistogramPersistencySvc().OutputFile = ''
32 MessageSvc( OutputLevel = ERROR )
33 EventSelector().PrintFreq = 100
34
35 ApplicationMgr( OutputLevel = ERROR,
36 AppName = 'File Check - Serial vs Parallel' )
37 # TopAlg = ['UnpackMCParticle', 'UnpackMCVertex'] )

def compareOutputFiles.compareEvents | ( |  | s,
|  |  | p
| ) |  |

Definition at line 208 of file compareOutputFiles.py.

208
209 def compareEvents( s, p ) :
210 # events in form of dictionary, with form
211 # d[ path ] = tuple( className, string_repr )
212
213 # check 1 : number of keys (paths)
214 sks = s.keys() ; pks = p.keys()
215 sks.sort() ; pks.sort()
216 if len( sks ) == len( pks ) : pass
217 else :
218 # There may be extra keys in the parallel file
219 # example: DstWriter may ask for /Event/Prev/MC/Header#1
220 # but in TESSerializer, *all* DataObjects will be sent
221 # including /Event/Prev and /Event/Prev/MC
222
223 # check for extra keys in the parallel file which are just containing DataObjects
224 # if found, remove them
225
226 extras = list( set( pks ) - set( sks ) )
227 for e in extras :
228 if p[e][0] == 'DataObject' : pks.remove( e )
229 else : print 'Extra Other thing found!', e, p[e][0] ; return False
230
231 # check 2 : same paths?
232 if sks == pks : pass
233 else : return False
234
235 # check 3 : check the content
236 l = len( sks )
237 diffs = []
238 for i in xrange( l ) :
239 key = sks[i]
240 # compare class name
241 if s[key][0] == p[key][0] : pass
242 else : diffs.append( key )
243 # compare string representation
244 if s[key][1] == p[key][1] : pass
245 elif checkForAddressDifference( p[key][1], s[key][1] ) : pass
246 else : diffs.append( key )
247
248 # finish
249 if diffs : return False
250 else : return True

def compareOutputFiles.CompareFSR | ( |  | pout,
|  |  | sout
| ) |  |

Definition at line 356 of file compareOutputFiles.py.

356
357 def CompareFSR( pout, sout ):
358
359 parFSR = pout.get()
360 serFSR = sout.get()
361
362 print "Comparing File Records"
363
364 diff1 = set( parFSR["TimeSpanFSR"].iteritems() ) - set( serFSR["TimeSpanFSR"].iteritems() )
365 diff2 = set( parFSR["EventCountFSR"].iteritems() ) - set( serFSR["EventCountFSR"].iteritems() )
366
367 print "\nDifferent entries in TimeSpanFSR: \t" + str( len( diff1 ) ) + "\nDifferent entries in EventCountFSR:\t" + str( len( diff2 ) )
368
369 for k in ["LumiFSRBeamCrossing", "LumiFSRBeam2", "LumiFSRNoBeam"]:
370 diff3 = set( parFSR[k]['key'] ) - set( serFSR[k]['key'] )
371 diff4 = set( parFSR[k]['incr'] ) - set( serFSR[k]['incr'] )
372 diff5 = set( parFSR[k]['integral'] ) - set( serFSR[k]["integral"] )
373 print "Different entries in " + str( k ) + ": \tkey: " + str( len( diff3 ) ) + " increment: " + str( len( diff4 ) ) + " integral: " + str( len( diff5 ) )
374
375
376 print "\nParallel: \n" + str( parFSR )
377 print "\nSerial: \n" + str( serFSR )

def compareOutputFiles.CompareTrees | ( |  | pname,
|  |  | sname
| ) |  |

Definition at line 41 of file compareOutputFiles.py.

41
42 def CompareTrees( pname, sname ) :
43 pf = TFile( pname, 'REC' )
44 sf = TFile( sname, 'REC' )
45 event = '_Event'
46 pfks = pf.GetListOfKeys()
47 sfks = sf.GetListOfKeys()
48 pfkeys = list( [pfk.GetName() for pfk in pfks] ) ; pfkeys.sort()
49 sfkeys = list( [sfk.GetName() for sfk in sfks] ) ; sfkeys.sort()
50 pMeta = [] ; pEvent = [] ; pOther = []
51 for k in pfkeys :
52 if k.startswith( event ) : pEvent.append( k )
53 elif k.startswith( '##' ) : pMeta.append( k )
54 else : pOther.append( k )
55 sMeta = [] ; sEvent = [] ; sOther = []
56 for k in sfkeys :
57 if k.startswith( event ) : sEvent.append( k )
58 elif k.startswith( '##' ) : sMeta.append( k )
59 else : sOther.append( k )
60
61 if pMeta == sMeta : pass
62 else : print 'Meta Data differs'
63
64 if pEvent == sEvent : pass
65 else : print 'Event data differs'
66
67 if pOther != sOther :
68 pset = set( pOther )
69 sset = set( sOther )
70 pExtra = pset - sset
71 sExtra = sset - pset
72 if pExtra : print 'Extra Data in parallel file : ', pExtra
73 if sExtra : print 'Extra Data in serial file : ', sExtra
74 if sExtra or pExtra : print 'Files will have different sizes'
75 pf.Close()
76 sf.Close()

def compareOutputFiles.ComparisonEngine | ( |  | pQueue,
|  |  | sQueue
| ) |  |

Definition at line 169 of file compareOutputFiles.py.

169
170 def ComparisonEngine( pQueue, sQueue ) :
171 # The Comparison Engine runs on a seperate forked process and receives
172 # events in pairs, one each from Serial FileReader and Parallel FileReader
173 #
174 # The events arrive in Dictionary Format, d[path]=(className, string_repr)
175 # and are compared using the compareEvents method
176 #
177 # Results are stored in an array of bools (PerfectMatch=True, Diff=False)
178 #
179 results = []
180 while True :
181 pitem = pQueue.get()
182 sitem = sQueue.get()
183 if pitem == sitem == None : print 'Termination Signals received ok' ; break
184 elif pitem == None : print 'pitem != sitem : ', pitem, sitem ; break
185 elif sitem == None : print 'pitem != sitem : ', pitem, sitem ; break
186 results.append( compareEvents( pitem, sitem ) )
187 print '=' * 80
188 print 'Comparison Engine Finished'
189 print '-' * 80
190 print 'Total Events Checked : %i' % ( len( results ) )
191 print 'Perfect Matches : %i' % ( sum( results ) )
192 print 'Errors : %i' % ( len( results ) - sum( results ) )
193 print '=' * 80

def compareOutputFiles.GetFSRdict | ( |  | filename,
|  |  | queue
| ) |  |

Definition at line 300 of file compareOutputFiles.py.

300
301 def GetFSRdict( filename, queue ):
302
303
304 FSR = { "TimeSpanFSR" : {'earliest': 0, 'latest': 0},
305 "LumiFSRBeamCrossing" : {'key': 0, 'incr': 0, 'integral':0},
306 "LumiFSRBeam1" : {'key': 0, 'incr': 0, 'integral':0},
307 "LumiFSRBeam2" : {'key': 0, 'incr': 0, 'integral':0},
308 "LumiFSRNoBeam" : {'key': 0, 'incr': 0, 'integral':0},
309 "EventCountFSR" : {'input': 0, 'output': 0, 'statusFlag': 0}}
310
311 options = "from LumiAlgs.LumiFsrReaderConf import LumiFsrReaderConf as LumiFsrReader; LumiFsrReader().OutputLevel = INFO; LumiFsrReader().inputFiles = ['%s'] ;" %filename
312 options += "LumiFsrReader().Persistency='ROOT'; LumiFsrReader().EvtMax = 1; from Configurables import LHCbApp; LHCbApp().Persistency='ROOT'; from Configurables import CondDB, DDDBConf;"
313 options += " CondDB().UseLatestTags=['%s']; DDDBConf(DataType='%s');"%( 2011, 2011 )
314 exec options
315 app = AppMgr()
316 app.run( 1 )
317 fsr = app.filerecordsvc()
318
319 lst = fsr.getHistoNames()
320
321 if lst :
322 for l in lst :
323
324 ob = fsr.retrieveObject( l )
325
326
327 if "LumiFSR" in l:
328
329 assert ob.numberOfObjects() == 1
330 k = ob.containedObject( 0 )
331 runs, files, keys, increment, integral = LumiFSR( k )
332
333 FSR[l[l.rfind( '/' ) + 1:]]['runs'] = runs
334 FSR[l[l.rfind( '/' ) + 1:]]['files'] = files
335 FSR[l[l.rfind( '/' ) + 1:]]['key'] = keys
336 FSR[l[l.rfind( '/' ) + 1:]]['incr'] = increment
337 FSR[l[l.rfind( '/' ) + 1:]]['integral'] = integral
338
339
340
341 if "TimeSpanFSR" in l:
342
343 FSR["TimeSpanFSR"]['earliest'] = ob.containedObject( 0 ).earliest()
344 FSR["TimeSpanFSR"]['latest'] = ob.containedObject( 0 ).latest()
345
346 if "EventCountFSR" in l:
347
348 FSR["EventCountFSR"]['input'] = ob.input()
349 FSR["EventCountFSR"]['output'] = ob.output()
350 FSR["EventCountFSR"]['statusFlag'] = ob.statusFlag()
351
352 app.stop()
353 app.finalize()
354
355 queue.put( FSR )

def compareOutputFiles.LumiFSR | ( |  | lumi | ) |

Definition at line 272 of file compareOutputFiles.py.

272
273 def LumiFSR( lumi ):
274
275 runs = []
276 files = []
277 info = {}
278 keys = []
279
280 for r in lumi.runNumbers() :
281 runs.append( r )
282
283 for f in lumi.fileIDs() :
284 files.append( f )
285 s = str( lumi )
286 sa = s.split( "info (key/incr/integral) : " )[-1]
287 sa = sa.split( '/' )[:-1]
288
289 key = []
290 incr = []
291 integral = []
292 for rec in sa :
293 k, i, t = rec.split()
294 key.append( int( k ) )
295 incr.append( int( i ) )
296 integral.append( int( t ) )
297
298 return ( runs, files, key, incr, integral )
299

def compareOutputFiles.printDict | ( |  | d,
|  |  | name = `'unspecified'`
| ) |  |

Definition at line 94 of file compareOutputFiles.py.

94
95 def printDict( d, name = 'unspecified' ) :
96 # Print out a dictionary in the form
97 #
98 # Dictionary Name :
99 # key value
100 # key value
101 # ...
102 #
103 print '-' * 80
104 print 'Dictionary %s : ' % ( name )
105 for k in iter( d.keys() ) :
106 print '\t', k, '\t', d[k]
107 print '-' * 80
108

def compareOutputFiles.Reader | ( |  | readerType,
|  |  | filename,
|  |  | qacross,
|  |  | qToEngine
| ) |  |

Definition at line 109 of file compareOutputFiles.py.

109
110 def Reader( readerType, filename, qacross, qToEngine ) :
111 #
112 # Process for reading a file
113 # One process for reading Serial File, another for Parallel File
114 #
115 # First the order of events is determined, (parallel != serial, usually)
116 #
117 # Then the events are run *in order* using AppMgr().runSelectedEvents(pfn, evtNumber)
118 # on both Serial-Reader and Parallel-Reader processes.
119 #
120 # The string repr of everything in the TES is placed in a dictionary and
121 # sent to the comparison Process, which compares the two dictionaries
122 #
123 a = AppMgr()
124 sel = a.evtsel()
125 evt = a.evtsvc()
126
127 header = '/Event/Rec/Header'
128 sel.open( filename )
129 ct = 0
130 order = {}
131 fname = filename[4:] # runSelectedEvents doesn't need the "PFN:" prefix
132
133 # determine the ordering
134 while True :
135 a.run( 1 )
136 if evt[header] :
137 eNumber = int( evt[header].evtNumber() )
138 order[eNumber] = ct
139 ct += 1
140 else : break
141
142 if readerType == SER :
143 # send the ordering details to the parallel-reader
144 order = switchDict( order )
145 qacross.put( order )
146 qacross.put( None )
147 # changeName
148 serOrder = order
149 elif readerType == PAR :
150 # receive the serial ordering from queue, and send ordering to SerialReader
151 for serOrder in iter( qacross.get, None ) : pass
152 lsks = len( serOrder.keys() )
153 lpks = len( order.keys() )
154 print 'Events in Files (serial/parallel) : %i / %i' % ( lsks, lpks )
155
156 # now run files in the order specified by the serial ordering
157 # and send them one by one to the comparison engine
158 for i in iter( serOrder.keys() ) :
159 if readerType == PAR : i = order[serOrder[i]]
160
161 a.runSelectedEvents( fname, i )
162 lst = evt.getList()
163
164 lst.sort()
165 ascii = dict( [ ( l, ( evt[l].__class__.__name__, evt[l].__repr__() ) ) for l in lst ] )
166 qToEngine.put( ascii )
167 qToEngine.put( None )
168 print '%s Reader Finished' % ( readerType )

def compareOutputFiles.switchDict | ( |  | d | ) |

Definition at line 77 of file compareOutputFiles.py.

77
78 def switchDict( d ) :
79 # switch a dictionary around ; make the values the keys, and vice versa
80 # only works if all values are unique
81 nkeys = len( d.keys() )
82 vals = d.values()
83 nvals = len( vals )
84 for v in vals :
85 if vals.count( v ) > 1 :
86 print 'Dictionary cannot be switched, values not unique'
87 return None
88 print 'Dict has keys/values : %i/%i' % ( nkeys, nvals )
89 pairs = d.items() # returns (key, val) tuples in a list
90 newd = {}
91 for k, entry in pairs : newd[entry] = k
92 return newd
93

## Variable Documentation

compareOutputFiles.args sys.argv

Definition at line 380 of file compareOutputFiles.py.

tuple compareOutputFiles.com Process( target = ComparisonEngine, args = ( pout, sout ) )

Definition at line 401 of file compareOutputFiles.py.

tuple compareOutputFiles.cp Process( target = CompareFSR, args = ( pout, sout ) )

Definition at line 418 of file compareOutputFiles.py.

string compareOutputFiles.PAR 'PARALLEL'

Definition at line 38 of file compareOutputFiles.py.

list compareOutputFiles.par 'PFN:'

Definition at line 386 of file compareOutputFiles.py.

list compareOutputFiles.pname par[4:]

Definition at line 392 of file compareOutputFiles.py.

tuple compareOutputFiles.pout Queue()

Definition at line 396 of file compareOutputFiles.py.

tuple compareOutputFiles.pp Process( target = GetFSRdict, args = ( par, pout ) )

Definition at line 417 of file compareOutputFiles.py.

tuple compareOutputFiles.qacross Queue()

Definition at line 395 of file compareOutputFiles.py.

string compareOutputFiles.SER 'SERIAL'

Definition at line 39 of file compareOutputFiles.py.

list compareOutputFiles.ser 'PFN:'

Definition at line 387 of file compareOutputFiles.py.

list compareOutputFiles.sname ser[4:]

Definition at line 393 of file compareOutputFiles.py.

tuple compareOutputFiles.sout Queue()

Definition at line 397 of file compareOutputFiles.py.

tuple compareOutputFiles.sp Process( target = GetFSRdict, args = ( ser, sout ) )

Definition at line 416 of file compareOutputFiles.py.