| # Gaudi Framework, version v22r0 |

Home |  | Generated: 9 Feb 2011

# GaudiMP::pTools::FileRecordsAgent Class Reference

List of all members.

## Public Member Functions
def | __init__
def | localCmp
def | SendFileRecords
def | Receive
def | Rebuild
def | MergeFSRobject
def | ProcessTimeSpanFSR
def | ProcessEventCountFSR
def | MergeLumiFSR
## Public Attributes
| fsr
| q
| log
| objectsIn
| objectsOut
## Private Attributes
| _gmpc

## Detailed Description

Definition at line 203 of file pTools.py.

## Member Function Documentation

def GaudiMP::pTools::FileRecordsAgent::__init__ | ( |  | self,
|  |  | gmpComponent |
| ) |  |  |

Definition at line 204 of file pTools.py.

    00205                                        :
    00206         self._gmpc = gmpComponent
    00207         self.fsr = self._gmpc.fsr
    00208         self.q   = self._gmpc.fq
    00209         self.log = self._gmpc.log
    00210         self.objectsIn  = []   # used for collecting FSR store objects
    00211         self.objectsOut = []

def GaudiMP::pTools::FileRecordsAgent::localCmp | ( |  | self,
|  |  | tupA,
|  |  | tupB |
| ) |  |  |

Definition at line 212 of file pTools.py.

    00213                                      :
    00214         # sort tuples by a particular element
    00215         # for the sort() method
    00216         ind  = 0
    00217         valA = tupA[ind]
    00218         valB = tupB[ind]
    00219         if   valA<valB : return -1
    00220         elif valA>valB : return  1
    00221         else           : return  0
    00222

def GaudiMP::pTools::FileRecordsAgent::MergeFSRobject | ( |  | self,
|  |  | sourceNode,
|  |  | path,
|  |  | ob |
| ) |  |  |

Definition at line 345 of file pTools.py.

    00346                                                      :
    00347         # Merge Non-Empty Keyed Container from Worker>0
    00348         if path == '/FileRecords/TimeSpanFSR' :
    00349             # TimeSpanFSR is a straightforward case
    00350             self.ProcessTimeSpanFSR( path, ob )
    00351         elif path == '/FileRecords/EventCountFSR' :
    00352             # Event Counter is also easy
    00353             self.ProcessEventCountFSR( path, ob )
    00354         # now other cases may not be so easy...
    00355         elif "KeyedContainer" in ob.__class__.__name__ :
    00356             # Keyed Container of LumiFSRs : extract and re-register
    00357             # self.ProcessLumiFSR( path, ob )
    00358             if "LumiFSR" in ob.__class__.__name__ :
    00359                 self.MergeLumiFSR( path, ob )
    00360             else:
    00361                 self.log.info("Skipping Merge of Keyed Container %s for %s"\
    00362                               %(ob.__class__.__name__,path))

def GaudiMP::pTools::FileRecordsAgent::MergeLumiFSR | ( |  | self,
|  |  | path,
|  |  | keyedC |
| ) |  |  |

Definition at line 380 of file pTools.py.

    00381                                            :
    00382         from ROOT import string
    00383         # Fetch the first lumi
    00384         keyedContainer = self.fsr.retrieveObject(path)
    00385         # The LumiFSR KeyedContainers only have one object
    00386         assert keyedContainer.numberOfObjects() == 1
    00387         l = keyedContainer.containedObject(0)
    00388         baseLumi = LumiFSR( l )
    00389         # Now deal with the argument Non-empty Keyed Container of LumiFSRs
    00390         nCont = keyedC.numberOfObjects()
    00391         for i in xrange(nCont) :
    00392             obj = keyedC.containedObject(i)
    00393             nextLumi = LumiFSR( obj )
    00394             baseLumi.merge( nextLumi )
    00395         # Now Rebuild and ReRegister
    00396         newLumi = gbl.LHCb.LumiFSR()
    00397         for r in baseLumi.runs :
    00398             newLumi.addRunNumber( r )
    00399         for f in baseLumi.files :
    00400             newLumi.addFileID( string(f) )
    00401         for k in baseLumi.keys :
    00402             increment, integral = baseLumi.info[k]
    00403             newLumi.addInfo(k, increment, integral)
    00404         # clear existing Keyed Container
    00405         self.fsr[path].clear()
    00406         # Add newly merged lumiFSR
    00407         self.fsr[path].add(newLumi)
    00408         return SUCCESS
    00409
    00410 # =============================================================================

def GaudiMP::pTools::FileRecordsAgent::ProcessEventCountFSR | ( |  | self,
|  |  | path,
|  |  | ob |
| ) |  |  |

Definition at line 376 of file pTools.py.

    00377                                                :
    00378         self.log.debug('Event Count Input Addition')
    00379         self.fsr[path].setInput( self.fsr[path].input()+ob.input() )

def GaudiMP::pTools::FileRecordsAgent::ProcessTimeSpanFSR | ( |  | self,
|  |  | path,
|  |  | ob |
| ) |  |  |

Definition at line 363 of file pTools.py.

    00364                                              :
    00365         if ob.containedObjects().size() :
    00366             sz = ob.containedObjects().size()
    00367             for j in xrange(sz) :
    00368                 cob = ob.containedObjects()[j]
    00369                 self.log.debug('Adding TimeSpanFSR')
    00370                 # this is annoying: it has to be rebuilt, without a key & added
    00371                 tsfsr = gbl.LHCb.TimeSpanFSR()
    00372                 tsfsr.setEarliest( cob.earliest() )
    00373                 tsfsr.setLatest(   cob.latest(  ) )
    00374                 self.fsr[path].add(tsfsr)
    00375                 continue

def GaudiMP::pTools::FileRecordsAgent::Rebuild | ( |  | self | ) |

Definition at line 302 of file pTools.py.

    00303                         :
    00304         # objects is a list of (path, serializedObject) tuples
    00305         for sourceNode, path, serialob in self.objectsIn :
    00306             self.log.debug('Working with %s'%(path))
    00307             ob = pickle.loads(serialob)
    00308             if hasattr( ob, "configureDirectAccess" ) :
    00309                 ob.configureDirectAccess()
    00310                 nCont = ob.numberOfObjects()
    00311                 self.log.debug( '\tcontainedObjects : %i'%(nCont) )
    00312             if sourceNode == 0 :
    00313                 self.log.debug('Registering Object to : %s'%(path))
    00314                 self.fsr.registerObject( path, ob )
    00315             else :
    00316                 self.log.debug('Merging Object to : %s'%(path))
    00317                 self.MergeFSRobject( sourceNode, path, ob )
    00318         # As RecordStream has been split into Worker and Writer parts, the
    00319         # count for output is wrong... fix that here, as every event received
    00320         # by the writer is written (validation testing occurs on the worker)
    00321
    00322         self.log.info('FSR Store Rebuilt.  Correcting EventCountFSR')
    00323         if bool( self.fsr._idp ) :   # There might not be an FSR stream (Gauss)
    00324             ecount  = '/FileRecords/EventCountFSR'
    00325             if self.fsr[ecount] :
    00326                 self.fsr[ecount].setOutput( self._gmpc.nIn )
    00327                 self.log.info( 'Event Counter Output set : %s : %i'\
    00328                                %(ecount, self.fsr[ecount].output()) )
    00329             # Do some reporting
    00330             self.log.debug('FSR store reconstructed!')
    00331             lst = self.fsr.getHistoNames()
    00332             if lst :
    00333                 for l in lst :
    00334                     ob = self.fsr.retrieveObject(l)
    00335                     if hasattr( ob, 'configureDirectAccess' ) :
    00336                         ob.configureDirectAccess()
    00337                     if hasattr( ob, 'containedObjects' ) :
    00338                         # if ob.numberOfObjects() :
    00339                         self.log.debug('\t%s (cont. objects : %i)'\
    00340                                        %(l, ob.numberOfObjects()))
    00341                     else :
    00342                         self.log.debug('\t%s'%(l))
    00343         self.log.info('FSR Store fully rebuilt.')
    00344         return SUCCESS

def GaudiMP::pTools::FileRecordsAgent::Receive | ( |  | self | ) |

Definition at line 282 of file pTools.py.

    00283                         :
    00284         # Receive contents of all Workers FileRecords Transient Stores
    00285         self.log.info('Receiving FSR store data...')
    00286         nc = self._gmpc.nWorkers
    00287         while nc > 0 :
    00288             objects = self.q.get( )
    00289             if objects == 'END_FSR' :
    00290                 nc -= 1
    00291                 continue
    00292                 if nc==0 :
    00293                     break
    00294             # but if it's regular objects...
    00295             for o in objects :
    00296                 self.objectsIn.append(o)
    00297         # Now sort it by which worker it came from
    00298         # an object is : (nodeID, path, pickledObject)
    00299         self.objectsIn.sort(cmp=self.localCmp)
    00300         self.log.info('All FSR data received')
    00301         return SUCCESS

def GaudiMP::pTools::FileRecordsAgent::SendFileRecords | ( |  | self | ) |

Definition at line 223 of file pTools.py.

    00224                                 :
    00225         # send the FileRecords data as part of finalisation
    00226
    00227         # Take Care of FileRecords!
    00228         # There are two main things to consider here
    00229         # 1) The DataObjects in the FileRecords Transient Store
    00230         # 2) The fact that they are Keyed Containers, containing other objects
    00231         #
    00232         # The Lead Worker, nodeID=0, sends everything in the FSR store, as
    00233         #   a completeness guarantee,
    00234         #
    00235         # send in form ( nodeID, path, object)
    00236         self.log.info('Sending FileRecords...')
    00237         lst      = self.fsr.getHistoNames()
    00238
    00239         # Check Validity
    00240         if not lst :
    00241             self.log.info('No FileRecords Data to send to Writer.')
    00242             self.q.put( 'END_FSR' )
    00243             return SUCCESS
    00244
    00245         # no need to send the root node
    00246         if '/FileRecords' in lst : lst.remove('/FileRecords')
    00247
    00248         for l in lst :
    00249             o = self.fsr.retrieveObject( l )
    00250             if hasattr(o, "configureDirectAccess") :
    00251                 o.configureDirectAccess()
    00252             # lead worker sends everything, as completeness guarantee
    00253             if self._gmpc.nodeID == 0 :
    00254                 self.objectsOut.append( (0, l, pickle.dumps(o)) )
    00255             else :
    00256                 # only add the Event Counter
    00257                 # and non-Empty Keyed Containers (ignore empty ones)
    00258                 if l == '/FileRecords/EventCountFSR' :
    00259                     tup = (self._gmpc.nodeID, l, pickle.dumps(o))
    00260                     self.objectsOut.append( tup )
    00261                 else :
    00262                     # It's a Keyed Container
    00263                     assert "KeyedContainer" in o.__class__.__name__
    00264                     nObjects = o.numberOfObjects()
    00265                     if nObjects :
    00266                         self.log.debug("Keyed Container %s with %i objects"\
    00267                                        %(l, nObjects))
    00268                         tup = (self._gmpc.nodeID, l, pickle.dumps(o))
    00269                         self.objectsOut.append( tup )
    00270         self.log.debug('Done with FSR store, just to send to Writer.')
    00271
    00272         if self.objectsOut :
    00273             self.log.debug('%i FSR objects to Writer'%(len(self.objectsOut)))
    00274             for ob in self.objectsOut :
    00275                 self.log.debug('\t%s'%(ob[0]))
    00276             self.q.put( self.objectsOut )
    00277         else :
    00278             self.log.info('Valid FSR Store, but no data to send to Writer')
    00279         self.log.info('SendFSR complete')
    00280         self.q.put( 'END_FSR' )
    00281         return SUCCESS

## Member Data Documentation

GaudiMP::pTools::FileRecordsAgent::_gmpc`[private]`

Definition at line 205 of file pTools.py.

GaudiMP::pTools::FileRecordsAgent::fsr

Definition at line 206 of file pTools.py.

GaudiMP::pTools::FileRecordsAgent::log

Definition at line 208 of file pTools.py.

GaudiMP::pTools::FileRecordsAgent::objectsIn

Definition at line 209 of file pTools.py.

GaudiMP::pTools::FileRecordsAgent::objectsOut

Definition at line 210 of file pTools.py.

GaudiMP::pTools::FileRecordsAgent::q

Definition at line 207 of file pTools.py.

The documentation for this class was generated from the following file:

- /afs/cern.ch/sw/Gaudi/releases/GAUDI/GAUDI_v22r0/GaudiMP/python/GaudiMP/pTools.py

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Defines