compareOutputFiles Namespace Reference

Functions
def	checkKeys (name)
def	CompareTrees (pname, sname)
def	switchDict (d)
def	printDict (d, name='unspecified')
def	Reader (readerType, filename, qacross, qToEngine)
def	ComparisonEngine (pQueue, sQueue)
def	checkForAddressDifference (a, b)
def	compareEvents (s, p)
def	CheckFileRecords (par, ser)
def	LumiFSR (lumi)
def	GetFSRdict (filename, queue)
def	CompareFSR (pout, sout)

Variables
string	PAR = 'PARALLEL'
string	SER = 'SERIAL'
	args = sys.argv
string	par = 'PFN:' + args[0]
string	ser = 'PFN:' + args[1]
string	pname = par[4:]
string	sname = ser[4:]
	qacross = Queue()
	pout = Queue()
	sout = Queue()
	com = Process(target=ComparisonEngine, args=(pout, sout))
	sp = Process(target=GetFSRdict, args=(ser, sout))
	pp = Process(target=GetFSRdict, args=(par, pout))
	cp = Process(target=CompareFSR, args=(pout, sout))

Function Documentation

CheckFileRecords()

def compareOutputFiles.CheckFileRecords	(		par,
			ser
	)

Definition at line 308 of file compareOutputFiles.py.

def CheckFileRecords(par, ser):
 
    print("Checking File Records")
 
    parFSR = GetFSRdicts(par)
    serFSR = GetFSRdicts(ser)
 
    diff1 = set(parFSR["TimeSpanFSR"].iteritems()) - \
        set(serFSR["TimeSpanFSR"].iteritems())
    diff2 = set(parFSR["EventCountFSR"].iteritems()) - \
        set(serFSR["EventCountFSR"].iteritems())
 
    print("\nDifferent entries in TimeSpanFSR:  \t" + \
        str(len(diff1)) + "\nDifferent entries in EventCountFSR:\t" + str(len(diff2)))
 
    for k in ["LumiFSRBeamCrossing", "LumiFSRBeam2", "LumiFSRNoBeam"]:
        diff3 = set(parFSR[k]["key"]) - set(serFSR[k]["key"])
        diff4 = set(parFSR[k]["incr"]) - set(serFSR[k]["incr"])
        diff5 = set(parFSR[k]["integral"]) - set(serFSR[k]["integral"])
        print("Different entries in " + str(k) + ": \tkey: " +
              str(len(diff3)) + " increment: " + str(len(diff4)) +
              " integral: " + str(len(diff5)))
 
 

checkForAddressDifference()

def compareOutputFiles.checkForAddressDifference	(		a,
			b
	)

Definition at line 233 of file compareOutputFiles.py.

def checkForAddressDifference(a, b):
    # the __repr__() method for Event Data Objects will return a generic
    # string "DataObject at 0xADDRESS" for non-Pythonised objects
    # If these objects have the same path, they are equal, but this
    # cannot be tested with "==" in Python, as the memory address will
    # be different for the two different DataObjects, so this method
    # will check if the difference is in the address
    #
    # args : a, b two string representations
    ref = 'DataObject at 0x'
    if a[:16] == b[:16] == ref:
        return True
    else:
        return False
 
 

checkKeys()

def compareOutputFiles.checkKeys

(

name

)

Definition at line 26 of file compareOutputFiles.py.

def checkKeys(name):
    # Check the TTree keys in each file
    fname = name[4:]  # TFile doesn't need the "PFN:" prefix
    tf = TFile(fname, 'REC')
 
 
importOptions('$STDOPTS/LHCbApplication.opts')
importOptions('$ENV_CMAKE_SOURCE_DIR/RootCnv/options/Setup.opts')
 
OutputStream("DstWriter").Output = ''
HistogramPersistencySvc().OutputFile = ''
MessageSvc(OutputLevel=ERROR)
EventSelector().PrintFreq = 100
 
ApplicationMgr(OutputLevel=ERROR, AppName='File Check - Serial vs Parallel')
# TopAlg = ['UnpackMCParticle', 'UnpackMCVertex'] )
 

compareEvents()

def compareOutputFiles.compareEvents	(		s,
			p
	)

Definition at line 249 of file compareOutputFiles.py.

def compareEvents(s, p):
    # events in form of dictionary, with form
    # d[ path ] = tuple( className, string_repr )
 
    # check 1 : number of keys (paths)
    sks = s.keys()
    pks = p.keys()
    sks.sort()
    pks.sort()
    if len(sks) == len(pks):
        pass
    else:
        # There may be extra keys in the parallel file
        # example: DstWriter may ask for /Event/Prev/MC/Header#1
        #          but in TESSerializer, *all* DataObjects will be sent
        #          including /Event/Prev and /Event/Prev/MC
 
        # check for extra keys in the parallel file which are just containing DataObjects
        # if found, remove them
 
        extras = list(set(pks) - set(sks))
        for e in extras:
            if p[e][0] == 'DataObject':
                pks.remove(e)
            else:
                print('Extra Other thing found!', e, p[e][0])
                return False
 
    # check 2 : same paths?
    if sks == pks:
        pass
    else:
        return False
 
    # check 3 : check the content
    l = len(sks)
    diffs = []
    for i in range(l):
        key = sks[i]
        # compare class name
        if s[key][0] == p[key][0]:
            pass
        else:
            diffs.append(key)
        # compare string representation
        if s[key][1] == p[key][1]:
            pass
        elif checkForAddressDifference(p[key][1], s[key][1]):
            pass
        else:
            diffs.append(key)
 
    # finish
    if diffs:
        return False
    else:
        return True
 
 

CompareFSR()

def compareOutputFiles.CompareFSR	(		pout,
			sout
	)

Definition at line 440 of file compareOutputFiles.py.

def CompareFSR(pout, sout):
 
    parFSR = pout.get()
    serFSR = sout.get()
 
    print("Comparing File Records")
 
    diff1 = set(parFSR["TimeSpanFSR"].iteritems()) - \
        set(serFSR["TimeSpanFSR"].iteritems())
    diff2 = set(parFSR["EventCountFSR"].iteritems()) - \
        set(serFSR["EventCountFSR"].iteritems())
 
    print("\nDifferent entries in TimeSpanFSR:  \t" + \
        str(len(diff1)) + "\nDifferent entries in EventCountFSR:\t" + str(len(diff2)))
 
    for k in ["LumiFSRBeamCrossing", "LumiFSRBeam2", "LumiFSRNoBeam"]:
        diff3 = set(parFSR[k]['key']) - set(serFSR[k]['key'])
        diff4 = set(parFSR[k]['incr']) - set(serFSR[k]['incr'])
        diff5 = set(parFSR[k]['integral']) - set(serFSR[k]["integral"])
        print("Different entries in " + str(k) + ": \tkey: " +
              str(len(diff3)) + " increment: " + str(len(diff4)) +
              " integral: " + str(len(diff5)))
 
    print("\nParallel: \n" + str(parFSR))
    print("\nSerial: \n" + str(serFSR))
 
 

CompareTrees()

def compareOutputFiles.CompareTrees	(		pname,
			sname
	)

Definition at line 47 of file compareOutputFiles.py.

def CompareTrees(pname, sname):
    pf = TFile(pname, 'REC')
    sf = TFile(sname, 'REC')
    event = '_Event'
    pfks = pf.GetListOfKeys()
    sfks = sf.GetListOfKeys()
    pfkeys = list([pfk.GetName() for pfk in pfks])
    pfkeys.sort()
    sfkeys = list([sfk.GetName() for sfk in sfks])
    sfkeys.sort()
    pMeta = []
    pEvent = []
    pOther = []
    for k in pfkeys:
        if k.startswith(event):
            pEvent.append(k)
        elif k.startswith('##'):
            pMeta.append(k)
        else:
            pOther.append(k)
    sMeta = []
    sEvent = []
    sOther = []
    for k in sfkeys:
        if k.startswith(event):
            sEvent.append(k)
        elif k.startswith('##'):
            sMeta.append(k)
        else:
            sOther.append(k)
 
    if pMeta == sMeta:
        pass
    else:
        print('Meta Data differs')
 
    if pEvent == sEvent:
        pass
    else:
        print('Event data differs')
 
    if pOther != sOther:
        pset = set(pOther)
        sset = set(sOther)
        pExtra = pset - sset
        sExtra = sset - pset
        if pExtra:
            print('Extra Data in parallel file : ', pExtra)
        if sExtra:
            print('Extra Data in serial   file : ', sExtra)
        if sExtra or pExtra:
            print('Files will have different sizes')
    pf.Close()
    sf.Close()
 
 

ComparisonEngine()

def compareOutputFiles.ComparisonEngine	(		pQueue,
			sQueue
	)

Definition at line 201 of file compareOutputFiles.py.

def ComparisonEngine(pQueue, sQueue):
    # The Comparison Engine runs on a seperate forked process and receives
    # events in pairs, one each from Serial FileReader and Parallel FileReader
    #
    # The events arrive in Dictionary Format, d[path]=(className, string_repr)
    # and are compared using the compareEvents method
    #
    # Results are stored in an array of bools (PerfectMatch=True, Diff=False)
    #
    results = []
    while True:
        pitem = pQueue.get()
        sitem = sQueue.get()
        if pitem == sitem == None:
            print('Termination Signals received ok')
            break
        elif pitem == None:
            print('pitem != sitem : ', pitem, sitem)
            break
        elif sitem == None:
            print('pitem != sitem : ', pitem, sitem)
            break
        results.append(compareEvents(pitem, sitem))
    print('=' * 80)
    print('Comparison Engine Finished')
    print('-' * 80)
    print('Total Events Checked : %i' % (len(results)))
    print('Perfect Matches      : %i' % (sum(results)))
    print('Errors               : %i' % (len(results) - sum(results)))
    print('=' * 80)
 
 

GetFSRdict()

def compareOutputFiles.GetFSRdict	(		filename,
			queue
	)

Definition at line 360 of file compareOutputFiles.py.

def GetFSRdict(filename, queue):
 
    FSR = {
        "TimeSpanFSR": {
            'earliest': 0,
            'latest': 0
        },
        "LumiFSRBeamCrossing": {
            'key': 0,
            'incr': 0,
            'integral': 0
        },
        "LumiFSRBeam1": {
            'key': 0,
            'incr': 0,
            'integral': 0
        },
        "LumiFSRBeam2": {
            'key': 0,
            'incr': 0,
            'integral': 0
        },
        "LumiFSRNoBeam": {
            'key': 0,
            'incr': 0,
            'integral': 0
        },
        "EventCountFSR": {
            'input': 0,
            'output': 0,
            'statusFlag': 0
        }
    }
 
    options = "from LumiAlgs.LumiFsrReaderConf import LumiFsrReaderConf as LumiFsrReader; LumiFsrReader().OutputLevel =  INFO; LumiFsrReader().inputFiles = ['%s'] ;" % filename
    options += "LumiFsrReader().Persistency='ROOT'; LumiFsrReader().EvtMax = 1; from Configurables import LHCbApp; LHCbApp().Persistency='ROOT';  from Configurables import CondDB, DDDBConf;"
    options += " CondDB().UseLatestTags=['%s']; DDDBConf(DataType='%s');" % (
        2011, 2011)
    exec(options)
    app = AppMgr()
    app.run(1)
    fsr = app.filerecordsvc()
 
    lst = fsr.getHistoNames()
 
    if lst:
        for l in lst:
 
            ob = fsr.retrieveObject(l)
 
            if "LumiFSR" in l:
 
                assert ob.numberOfObjects() == 1
                k = ob.containedObject(0)
                runs, files, keys, increment, integral = LumiFSR(k)
 
                FSR[l[l.rfind('/') + 1:]]['runs'] = runs
                FSR[l[l.rfind('/') + 1:]]['files'] = files
                FSR[l[l.rfind('/') + 1:]]['key'] = keys
                FSR[l[l.rfind('/') + 1:]]['incr'] = increment
                FSR[l[l.rfind('/') + 1:]]['integral'] = integral
 
            if "TimeSpanFSR" in l:
 
                FSR["TimeSpanFSR"]['earliest'] = ob.containedObject(
                    0).earliest()
                FSR["TimeSpanFSR"]['latest'] = ob.containedObject(0).latest()
 
            if "EventCountFSR" in l:
 
                FSR["EventCountFSR"]['input'] = ob.input()
                FSR["EventCountFSR"]['output'] = ob.output()
                FSR["EventCountFSR"]['statusFlag'] = ob.statusFlag()
 
    app.stop()
    app.finalize()
 
    queue.put(FSR)
 
 

LumiFSR()

def compareOutputFiles.LumiFSR

(

lumi

)

Definition at line 332 of file compareOutputFiles.py.

def LumiFSR(lumi):
 
    runs = []
    files = []
    info = {}
    keys = []
 
    for r in lumi.runNumbers():
        runs.append(r)
 
    for f in lumi.fileIDs():
        files.append(f)
    s = str(lumi)
    sa = s.split("info (key/incr/integral) : ")[-1]
    sa = sa.split('/')[:-1]
 
    key = []
    incr = []
    integral = []
    for rec in sa:
        k, i, t = rec.split()
        key.append(int(k))
        incr.append(int(i))
        integral.append(int(t))
 
    return (runs, files, key, incr, integral)
 
 

printDict()

def compareOutputFiles.printDict	(		d,
			name = 'unspecified'
	)

Definition at line 121 of file compareOutputFiles.py.

def printDict(d, name='unspecified'):
    # Print out a dictionary in the form
    #
    # Dictionary Name :
    #   key     value
    #   key     value
    #   ...
    #
    print('-' * 80)
    print('Dictionary %s : ' % (name))
    for k in iter(d.keys()):
        print('\t', k, '\t', d[k])
    print('-' * 80)
 
 

Reader()

def compareOutputFiles.Reader	(		readerType,
			filename,
			qacross,
			qToEngine
	)

Definition at line 136 of file compareOutputFiles.py.

def Reader(readerType, filename, qacross, qToEngine):
    #
    # Process for reading a file
    # One process for reading Serial File, another for Parallel File
    #
    # First the order of events is determined, (parallel != serial, usually)
    #
    # Then the events are run *in order* using AppMgr().runSelectedEvents(pfn, evtNumber)
    # on both Serial-Reader and Parallel-Reader processes.
    #
    # The string repr of everything in the TES is placed in a dictionary and
    # sent to the comparison Process, which compares the two dictionaries
    #
    a = AppMgr()
    sel = a.evtsel()
    evt = a.evtsvc()
 
    header = '/Event/Rec/Header'
    sel.open(filename)
    ct = 0
    order = {}
    fname = filename[4:]  # runSelectedEvents doesn't need the "PFN:" prefix
 
    # determine the ordering
    while True:
        a.run(1)
        if evt[header]:
            eNumber = int(evt[header].evtNumber())
            order[eNumber] = ct
            ct += 1
        else:
            break
 
    if readerType == SER:
        # send the ordering details to the parallel-reader
        order = switchDict(order)
        qacross.put(order)
        qacross.put(None)
        # changeName
        serOrder = order
    elif readerType == PAR:
        # receive the serial ordering from queue,  and send ordering to SerialReader
        for serOrder in iter(qacross.get, None):
            pass
        lsks = len(serOrder.keys())
        lpks = len(order.keys())
        print('Events in Files (serial/parallel) : %i / %i' % (lsks, lpks))
 
    # now run files in the order specified by the serial ordering
    # and send them one by one to the comparison engine
    for i in iter(serOrder.keys()):
        if readerType == PAR:
            i = order[serOrder[i]]
 
        a.runSelectedEvents(fname, i)
        lst = evt.getList()
 
        lst.sort()
        ascii = dict(
            [(l, (evt[l].__class__.__name__, evt[l].__repr__())) for l in lst])
        qToEngine.put(ascii)
    qToEngine.put(None)
    print('%s Reader Finished' % (readerType))
 
 

switchDict()

def compareOutputFiles.switchDict

(

d

)

Definition at line 103 of file compareOutputFiles.py.

def switchDict(d):
    # switch a dictionary around ;  make the values the keys, and vice versa
    # only works if all values are unique
    nkeys = len(d.keys())
    vals = d.values()
    nvals = len(vals)
    for v in vals:
        if vals.count(v) > 1:
            print('Dictionary cannot be switched, values not unique')
            return None
    print('Dict has keys/values : %i/%i' % (nkeys, nvals))
    pairs = d.items()  # returns (key, val) tuples in a list
    newd = {}
    for k, entry in pairs:
        newd[entry] = k
    return newd
 
 

Variable Documentation

args

compareOutputFiles.args = sys.argv

Definition at line 469 of file compareOutputFiles.py.

com

compareOutputFiles.com = Process(target=ComparisonEngine, args=(pout, sout))

Definition at line 491 of file compareOutputFiles.py.

cp

compareOutputFiles.cp = Process(target=CompareFSR, args=(pout, sout))

Definition at line 508 of file compareOutputFiles.py.

PAR

string compareOutputFiles.PAR = 'PARALLEL'

Definition at line 43 of file compareOutputFiles.py.

par

compareOutputFiles.par = 'PFN:' + args[0]

Definition at line 477 of file compareOutputFiles.py.

pname

string compareOutputFiles.pname = par[4:]

Definition at line 482 of file compareOutputFiles.py.

pout

compareOutputFiles.pout = Queue()

Definition at line 486 of file compareOutputFiles.py.

pp

compareOutputFiles.pp = Process(target=GetFSRdict, args=(par, pout))

Definition at line 507 of file compareOutputFiles.py.

qacross

compareOutputFiles.qacross = Queue()

Definition at line 485 of file compareOutputFiles.py.

SER

string compareOutputFiles.SER = 'SERIAL'

Definition at line 44 of file compareOutputFiles.py.

ser

compareOutputFiles.ser = 'PFN:' + args[1]

Definition at line 478 of file compareOutputFiles.py.

sname

string compareOutputFiles.sname = ser[4:]

Definition at line 483 of file compareOutputFiles.py.

sout

compareOutputFiles.sout = Queue()

Definition at line 487 of file compareOutputFiles.py.

sp

compareOutputFiles.sp = Process(target=GetFSRdict, args=(ser, sout))

Definition at line 506 of file compareOutputFiles.py.