2 from GaudiPython
import AppMgr, gbl
3 from ROOT
import TFile, TBufferFile, TBuffer
4 from multiprocessing
import Process, Queue
5 from Configurables
import LHCbApp
18 tf = TFile(fname,
'REC')
30 ApplicationMgr(OutputLevel=ERROR, AppName=
'File Check - Serial vs Parallel')
38 pf = TFile(pname,
'REC')
39 sf = TFile(sname,
'REC')
41 pfks = pf.GetListOfKeys()
42 sfks = sf.GetListOfKeys()
43 pfkeys = list([pfk.GetName()
for pfk
in pfks])
45 sfkeys = list([sfk.GetName()
for sfk
in sfks])
51 if k.startswith(event):
53 elif k.startswith(
'##'):
61 if k.startswith(event):
63 elif k.startswith(
'##'):
71 print 'Meta Data differs' 76 print 'Event data differs' 84 print 'Extra Data in parallel file : ', pExtra
86 print 'Extra Data in serial file : ', sExtra
88 print 'Files will have different sizes' 100 if vals.count(v) > 1:
101 print 'Dictionary cannot be switched, values not unique' 103 print 'Dict has keys/values : %i/%i' % (nkeys, nvals)
106 for k, entry
in pairs:
120 print 'Dictionary %s : ' % (name)
121 for k
in iter(d.keys()):
122 print '\t', k,
'\t', d[k]
126 def Reader(readerType, filename, qacross, qToEngine):
143 header =
'/Event/Rec/Header' 153 eNumber = int(evt[header].evtNumber())
159 if readerType == SER:
166 elif readerType == PAR:
168 for serOrder
in iter(qacross.get,
None):
170 lsks = len(serOrder.keys())
171 lpks = len(order.keys())
172 print 'Events in Files (serial/parallel) : %i / %i' % (lsks, lpks)
176 for i
in iter(serOrder.keys()):
177 if readerType == PAR:
178 i = order[serOrder[i]]
180 a.runSelectedEvents(fname, i)
185 [(l, (evt[l].__class__.__name__, evt[l].
__repr__()))
for l
in lst])
188 print '%s Reader Finished' % (readerType)
204 if pitem == sitem ==
None:
205 print 'Termination Signals received ok' 208 print 'pitem != sitem : ', pitem, sitem
211 print 'pitem != sitem : ', pitem, sitem
215 print 'Comparison Engine Finished' 217 print 'Total Events Checked : %i' % (len(results))
218 print 'Perfect Matches : %i' % (
sum(results))
219 print 'Errors : %i' % (len(results) -
sum(results))
232 ref =
'DataObject at 0x' 233 if a[:16] == b[:16] == ref:
248 if len(sks) == len(pks):
259 extras = list(set(pks) - set(sks))
261 if p[e][0] ==
'DataObject':
264 print 'Extra Other thing found!', e, p[e][0]
279 if s[key][0] == p[key][0]:
284 if s[key][1] == p[key][1]:
300 print "Checking File Records" 302 parFSR = GetFSRdicts(par)
303 serFSR = GetFSRdicts(ser)
305 diff1 = set(parFSR[
"TimeSpanFSR"].iteritems()) - \
306 set(serFSR[
"TimeSpanFSR"].iteritems())
307 diff2 = set(parFSR[
"EventCountFSR"].iteritems()) - \
308 set(serFSR[
"EventCountFSR"].iteritems())
310 print "\nDifferent entries in TimeSpanFSR: \t" + \
311 str(len(diff1)) +
"\nDifferent entries in EventCountFSR:\t" + str(len(diff2))
313 for k
in [
"LumiFSRBeamCrossing",
"LumiFSRBeam2",
"LumiFSRNoBeam"]:
314 diff3 = set(parFSR[k][
"key"]) - set(serFSR[k][
"key"])
315 diff4 = set(parFSR[k][
"incr"]) - set(serFSR[k][
"incr"])
316 diff5 = set(parFSR[k][
"integral"]) - set(serFSR[k][
"integral"])
317 print "Different entries in " + str(k) +
": \tkey: " + str(
318 len(diff3)) +
" increment: " + str(
319 len(diff4)) +
" integral: " + str(len(diff5))
329 for r
in lumi.runNumbers():
332 for f
in lumi.fileIDs():
335 sa = s.split(
"info (key/incr/integral) : ")[-1]
336 sa = sa.split(
'/')[:-1]
342 k, i, t = rec.split()
345 integral.append(int(t))
347 return (runs, files, key, incr, integral)
357 "LumiFSRBeamCrossing": {
384 options =
"from LumiAlgs.LumiFsrReaderConf import LumiFsrReaderConf as LumiFsrReader; LumiFsrReader().OutputLevel = INFO; LumiFsrReader().inputFiles = ['%s'] ;" % filename
385 options +=
"LumiFsrReader().Persistency='ROOT'; LumiFsrReader().EvtMax = 1; from Configurables import LHCbApp; LHCbApp().Persistency='ROOT'; from Configurables import CondDB, DDDBConf;" 386 options +=
" CondDB().UseLatestTags=['%s']; DDDBConf(DataType='%s');" % (
391 fsr = app.filerecordsvc()
393 lst = fsr.getHistoNames()
398 ob = fsr.retrieveObject(l)
402 assert ob.numberOfObjects() == 1
403 k = ob.containedObject(0)
404 runs, files, keys, increment, integral =
LumiFSR(k)
406 FSR[l[l.rfind(
'/') + 1:]][
'runs'] = runs
407 FSR[l[l.rfind(
'/') + 1:]][
'files'] = files
408 FSR[l[l.rfind(
'/') + 1:]][
'key'] = keys
409 FSR[l[l.rfind(
'/') + 1:]][
'incr'] = increment
410 FSR[l[l.rfind(
'/') + 1:]][
'integral'] = integral
412 if "TimeSpanFSR" in l:
414 FSR[
"TimeSpanFSR"][
'earliest'] = ob.containedObject(
416 FSR[
"TimeSpanFSR"][
'latest'] = ob.containedObject(0).latest()
418 if "EventCountFSR" in l:
420 FSR[
"EventCountFSR"][
'input'] = ob.input()
421 FSR[
"EventCountFSR"][
'output'] = ob.output()
422 FSR[
"EventCountFSR"][
'statusFlag'] = ob.statusFlag()
435 print "Comparing File Records" 437 diff1 = set(parFSR[
"TimeSpanFSR"].iteritems()) - \
438 set(serFSR[
"TimeSpanFSR"].iteritems())
439 diff2 = set(parFSR[
"EventCountFSR"].iteritems()) - \
440 set(serFSR[
"EventCountFSR"].iteritems())
442 print "\nDifferent entries in TimeSpanFSR: \t" + \
443 str(len(diff1)) +
"\nDifferent entries in EventCountFSR:\t" + str(len(diff2))
445 for k
in [
"LumiFSRBeamCrossing",
"LumiFSRBeam2",
"LumiFSRNoBeam"]:
446 diff3 = set(parFSR[k][
'key']) - set(serFSR[k][
'key'])
447 diff4 = set(parFSR[k][
'incr']) - set(serFSR[k][
'incr'])
448 diff5 = set(parFSR[k][
'integral']) - set(serFSR[k][
"integral"])
449 print "Different entries in " + str(k) +
": \tkey: " + str(
450 len(diff3)) +
" increment: " + str(
451 len(diff4)) +
" integral: " + str(len(diff5))
453 print "\nParallel: \n" + str(parFSR)
454 print "\nSerial: \n" + str(serFSR)
457 if __name__ ==
'__main__':
462 print 'Please supply two arguments : > python loadFile <parallelFile> <serialFile>' 465 par =
'PFN:' + args[0]
466 ser =
'PFN:' + args[1]
467 print 'Parallel File to be analysed : %s' % (par)
468 print 'Serial File to be analysed : %s' % (ser)
477 par = Process(target=Reader, args=(PAR, par, qacross, pout))
478 ser = Process(target=Reader, args=(SER, ser, qacross, sout))
479 com = Process(target=ComparisonEngine, args=(pout, sout))
486 print "Check File Records" 494 sp = Process(target=GetFSRdict, args=(ser, sout))
495 pp = Process(target=GetFSRdict, args=(par, pout))
496 cp = Process(target=CompareFSR, args=(pout, sout))
def printDict(d, name='unspecified')
def CheckFileRecords(par, ser)
def CompareFSR(pout, sout)
def ComparisonEngine(pQueue, sQueue)
double sum(double x, double y, double z)
A small to stream Data I/O.
def importOptions(optsfile)
def CompareTrees(pname, sname)
HistogramPersistencySvc class implementation definition.
def checkForAddressDifference(a, b)
def GetFSRdict(filename, queue)
The Application Manager class.
Definition of class EventSelector.
def Reader(readerType, filename, qacross, qToEngine)