| The Gaudi Framework v29r0 (ff2e7097)

- Main Page
- Related Pages
- Modules
- Namespaces
- Classes
- Files

- Class List
- Class Index
- Class Hierarchy
- Class Members

Public Member Functions | Public Attributes | Private Attributes | List of all members
GaudiMP.pTools.FileRecordsAgent Class Reference

## Public Member Functions
def | __init__ (self, gmpComponent)
def | localCmp (self, tupA, tupB)
def | SendFileRecords (self)
def | Receive (self)
def | Rebuild (self)
def | MergeFSRobject (self, sourceNode, path, ob)
def | ProcessTimeSpanFSR (self, path, ob)
def | ProcessEventCountFSR (self, path, ob)
def | MergeLumiFSR (self, path, keyedC)

## Public Attributes
| fsr
| q
| log
| objectsIn
| objectsOut

## Private Attributes
| _gmpc

## Detailed Description

Definition at line 210 of file pTools.py.

## Constructor & Destructor Documentation

def GaudiMP.pTools.FileRecordsAgent.__init__ | ( |  | self,
|  |  | gmpComponent
| ) |  |

Definition at line 211 of file pTools.py.

211 def __init__(self, gmpComponent):
212 self._gmpc = gmpComponent
213 self.fsr = self._gmpc.fsr
214 self.q = self._gmpc.fq
215 self.log = self._gmpc.log
216 self.objectsIn = [] # used for collecting FSR store objects
217 self.objectsOut = []
218
GaudiMP.pTools.FileRecordsAgent.q
q
Definition: pTools.py:214
GaudiMP.pTools.FileRecordsAgent.__init__
def __init__(self, gmpComponent)
Definition: pTools.py:211
GaudiMP.pTools.FileRecordsAgent.log
log
Definition: pTools.py:215
GaudiMP.pTools.FileRecordsAgent.objectsOut
objectsOut
Definition: pTools.py:217
GaudiMP.pTools.FileRecordsAgent.objectsIn
objectsIn
Definition: pTools.py:216
GaudiMP.pTools.FileRecordsAgent._gmpc
_gmpc
Definition: pTools.py:212
GaudiMP.pTools.FileRecordsAgent.fsr
fsr
Definition: pTools.py:213

## Member Function Documentation

def GaudiMP.pTools.FileRecordsAgent.localCmp | ( |  | self,
|  |  | tupA,
|  |  | tupB
| ) |  |

Definition at line 219 of file pTools.py.

219 def localCmp(self, tupA, tupB):
220 # sort tuples by a particular element
221 # for the sort() method
222 ind = 0
223 valA = tupA[ind]
224 valB = tupB[ind]
225 if valA < valB:
226 return -1
227 elif valA > valB:
228 return 1
229 else:
230 return 0
231
GaudiMP.pTools.FileRecordsAgent.localCmp
def localCmp(self, tupA, tupB)
Definition: pTools.py:219

def GaudiMP.pTools.FileRecordsAgent.MergeFSRobject | ( |  | self,
|  |  | sourceNode,
|  |  | path,
|  |  | ob
| ) |  |

Definition at line 357 of file pTools.py.

357 def MergeFSRobject(self, sourceNode, path, ob):
358 # Merge Non-Empty Keyed Container from Worker>0
359 if path == '/FileRecords/TimeSpanFSR':
360 # TimeSpanFSR is a straightforward case
361 self.ProcessTimeSpanFSR(path, ob)
362 elif path == '/FileRecords/EventCountFSR':
363 # Event Counter is also easy
364 self.ProcessEventCountFSR(path, ob)
365 # now other cases may not be so easy...
366 elif "KeyedContainer" in ob.__class__.__name__:
367 # Keyed Container of LumiFSRs : extract and re-register
368 # self.ProcessLumiFSR( path, ob )
369 if "LumiFSR" in ob.__class__.__name__:
370 self.MergeLumiFSR(path, ob)
371 else:
372 self.log.info("Skipping Merge of Keyed Container %s for %s"
373 % (ob.__class__.__name__, path))
374
GaudiMP.pTools.FileRecordsAgent.ProcessEventCountFSR
def ProcessEventCountFSR(self, path, ob)
Definition: pTools.py:397
GaudiMP.pTools.FileRecordsAgent.MergeLumiFSR
def MergeLumiFSR(self, path, keyedC)
Definition: pTools.py:401
GaudiMP.pTools.FileRecordsAgent.ProcessTimeSpanFSR
def ProcessTimeSpanFSR(self, path, ob)
Definition: pTools.py:375
GaudiMP.pTools.FileRecordsAgent.MergeFSRobject
def MergeFSRobject(self, sourceNode, path, ob)
Definition: pTools.py:357

def GaudiMP.pTools.FileRecordsAgent.MergeLumiFSR | ( |  | self,
|  |  | path,
|  |  | keyedC
| ) |  |

Definition at line 401 of file pTools.py.

401 def MergeLumiFSR(self, path, keyedC):
402 from ROOT import string
403 # Fetch the first lumi
404 keyedContainer = self.fsr.retrieveObject(path)
405 # The LumiFSR KeyedContainers only have one object
406 assert keyedContainer.numberOfObjects() == 1
407 l = keyedContainer.containedObject(0)
408 baseLumi = LumiFSR(l)
409 # Now deal with the argument Non-empty Keyed Container of LumiFSRs
410 nCont = keyedC.numberOfObjects()
411 for i in xrange(nCont):
412 obj = keyedC.containedObject(i)
413 nextLumi = LumiFSR(obj)
414 baseLumi.merge(nextLumi)
415 # Now Rebuild and ReRegister
416 newLumi = gbl.LHCb.LumiFSR()
417 for r in baseLumi.runs:
418 newLumi.addRunNumber(r)
419 for f in baseLumi.files:
420 newLumi.addFileID(string(f))
421 for k in baseLumi.keys:
422 increment, integral = baseLumi.info[k]
423 newLumi.addInfo(k, increment, integral)
424 # clear existing Keyed Container
425 self.fsr[path].clear()
426 # Add newly merged lumiFSR
427 self.fsr[path].add(newLumi)
428 return SUCCESS
429
430 # =============================================================================
431
432
GaudiPython.Persistency.add
def add(instance)
Definition: Persistency.py:39
GaudiMP.pTools.FileRecordsAgent.MergeLumiFSR
def MergeLumiFSR(self, path, keyedC)
Definition: pTools.py:401
GaudiMP.pTools.FileRecordsAgent.fsr
fsr
Definition: pTools.py:213
GaudiMP.pTools.LumiFSR
Definition: pTools.py:433

def GaudiMP.pTools.FileRecordsAgent.ProcessEventCountFSR | ( |  | self,
|  |  | path,
|  |  | ob
| ) |  |

Definition at line 397 of file pTools.py.

397 def ProcessEventCountFSR(self, path, ob):
398 self.log.debug('Event Count Input Addition')
399 self.fsr[path].setInput(self.fsr[path].input() + ob.input())
400
GaudiMP.pTools.FileRecordsAgent.ProcessEventCountFSR
def ProcessEventCountFSR(self, path, ob)
Definition: pTools.py:397
GaudiMP.pTools.FileRecordsAgent.fsr
fsr
Definition: pTools.py:213

def GaudiMP.pTools.FileRecordsAgent.ProcessTimeSpanFSR | ( |  | self,
|  |  | path,
|  |  | ob
| ) |  |

Definition at line 375 of file pTools.py.

375 def ProcessTimeSpanFSR(self, path, ob):
376 ob2 = self.fsr.retrieveObject(path)
377 if ob.containedObjects().size():
378 sz = ob.containedObjects().size()
379 cob = ob2.containedObjects()[0]
380 min = cob.earliest()
381 max = cob.latest()
382 for j in xrange(sz):
383 cob = ob.containedObjects()[j]
384 self.log.debug('Adding TimeSpanFSR')
385 if cob.earliest() < min:
386 min = cob.earliest()
387 if cob.latest() > max:
388 max = cob.latest()
389 # this is annoying: it has to be rebuilt, without a key & added
390 continue
391 tsfsr = gbl.LHCb.TimeSpanFSR()
392 tsfsr.setEarliest(min)
393 tsfsr.setLatest(max)
394 self.fsr[path].clear()
395 self.fsr[path].add(tsfsr)
396
GaudiPython.Persistency.add
def add(instance)
Definition: Persistency.py:39
GaudiMP.pTools.FileRecordsAgent.ProcessTimeSpanFSR
def ProcessTimeSpanFSR(self, path, ob)
Definition: pTools.py:375
GaudiMP.pTools.FileRecordsAgent.fsr
fsr
Definition: pTools.py:213

def GaudiMP.pTools.FileRecordsAgent.Rebuild | ( |  | self | ) |

Definition at line 312 of file pTools.py.

312 def Rebuild(self):
313 # objects is a list of (path, serializedObject) tuples
314 for sourceNode, path, serialob in self.objectsIn:
315 self.log.debug('Working with %s' % (path))
316 ob = pickle.loads(serialob)
317 if hasattr(ob, 'update'):
318 ob.update()
319 if hasattr(ob, 'numberOfObjects'):
320 nCont = ob.numberOfObjects()
321 self.log.debug('\t %s has containedObjects : %i' %
322 (type(ob).__name__, nCont))
323 if sourceNode == 0:
324 self.log.debug('Registering Object to : %s' % (path))
325 self.fsr.registerObject(path, ob)
326 else:
327 self.log.debug('Merging Object to : %s' % (path))
328 self.MergeFSRobject(sourceNode, path, ob)
329 # As RecordStream has been split into Worker and Writer parts, the
330 # count for output is wrong... fix that here, as every event received
331 # by the writer is written (validation testing occurs on the worker)
332
333 self.log.info('FSR Store Rebuilt. Correcting EventCountFSR')
334 if bool(self.fsr._idp): # There might not be an FSR stream (Gauss)
335 ecount = '/FileRecords/EventCountFSR'
336 if self.fsr[ecount]:
337 self.fsr[ecount].setOutput(self._gmpc.nIn)
338 self.log.info('Event Counter Output set : %s : %i'
339 % (ecount, self.fsr[ecount].output()))
340 # Do some reporting
341 self.log.debug('FSR store reconstructed!')
342 lst = self.fsr.getHistoNames()
343 if lst:
344 for l in lst:
345 ob = self.fsr.retrieveObject(l)
346 if hasattr(ob, 'configureDirectAccess'):
347 ob.configureDirectAccess()
348 if hasattr(ob, 'containedObjects'):
349 # if ob.numberOfObjects() :
350 self.log.debug('\t%s (cont. objects : %i)'
351 % (l, ob.numberOfObjects()))
352 else:
353 self.log.debug('\t%s' % (l))
354 self.log.info('FSR Store fully rebuilt.')
355 return SUCCESS
356
gaudirun.output
output
Definition: gaudirun.py:404
GaudiMP.pTools.FileRecordsAgent.Rebuild
def Rebuild(self)
Definition: pTools.py:312
gaudirun.type
type
Definition: gaudirun.py:133
GaudiMP.pTools.FileRecordsAgent.objectsIn
objectsIn
Definition: pTools.py:216
GaudiMP.pTools.FileRecordsAgent.MergeFSRobject
def MergeFSRobject(self, sourceNode, path, ob)
Definition: pTools.py:357
GaudiMP.pTools.FileRecordsAgent.fsr
fsr
Definition: pTools.py:213

def GaudiMP.pTools.FileRecordsAgent.Receive | ( |  | self | ) |

Definition at line 292 of file pTools.py.

292 def Receive(self):
293 # Receive contents of all Workers FileRecords Transient Stores
294 self.log.info('Receiving FSR store data...')
295 nc = self._gmpc.nWorkers
296 while nc > 0:
297 objects = self.q.get()
298 if objects == 'END_FSR':
299 nc -= 1
300 continue
301 if nc == 0:
302 break
303 # but if it's regular objects...
304 for o in objects:
305 self.objectsIn.append(o)
306 # Now sort it by which worker it came from
307 # an object is : (nodeID, path, pickledObject)
308 self.objectsIn.sort(cmp=self.localCmp)
309 self.log.info('All FSR data received')
310 return SUCCESS
311
GaudiMP.pTools.FileRecordsAgent.localCmp
def localCmp(self, tupA, tupB)
Definition: pTools.py:219
GaudiMP.pTools.FileRecordsAgent.Receive
def Receive(self)
Definition: pTools.py:292

def GaudiMP.pTools.FileRecordsAgent.SendFileRecords | ( |  | self | ) |

Definition at line 232 of file pTools.py.

232 def SendFileRecords(self):
233 # send the FileRecords data as part of finalisation
234
235 # Take Care of FileRecords!
236 # There are two main things to consider here
237 # 1) The DataObjects in the FileRecords Transient Store
238 # 2) The fact that they are Keyed Containers, containing other objects
239 #
240 # The Lead Worker, nodeID=0, sends everything in the FSR store, as
241 # a completeness guarantee,
242 #
243 # send in form ( nodeID, path, object)
244 self.log.info('Sending FileRecords...')
245 lst = self.fsr.getHistoNames()
246
247 # Check Validity
248 if not lst:
249 self.log.info('No FileRecords Data to send to Writer.')
250 self.q.put('END_FSR')
251 return SUCCESS
252
253 # no need to send the root node
254 if '/FileRecords' in lst:
255 lst.remove('/FileRecords')
256
257 for l in lst:
258 o = self.fsr.retrieveObject(l)
259 if hasattr(o, "configureDirectAccess"):
260 o.configureDirectAccess()
261 # lead worker sends everything, as completeness guarantee
262 if self._gmpc.nodeID == 0:
263 self.objectsOut.append((0, l, pickle.dumps(o)))
264 else:
265 # only add the Event Counter
266 # and non-Empty Keyed Containers (ignore empty ones)
267 if l == '/FileRecords/EventCountFSR':
268 tup = (self._gmpc.nodeID, l, pickle.dumps(o))
269 self.objectsOut.append(tup)
270 else:
271 # It's a Keyed Container
272 assert "KeyedContainer" in o.__class__.__name__
273 nObjects = o.numberOfObjects()
274 if nObjects:
275 self.log.debug("Keyed Container %s with %i objects"
276 % (l, nObjects))
277 tup = (self._gmpc.nodeID, l, pickle.dumps(o))
278 self.objectsOut.append(tup)
279 self.log.debug('Done with FSR store, just to send to Writer.')
280
281 if self.objectsOut:
282 self.log.debug('%i FSR objects to Writer' % (len(self.objectsOut)))
283 for ob in self.objectsOut:
284 self.log.debug('\t%s' % (ob[0]))
285 self.q.put(self.objectsOut)
286 else:
287 self.log.info('Valid FSR Store, but no data to send to Writer')
288 self.log.info('SendFSR complete')
289 self.q.put('END_FSR')
290 return SUCCESS
291
GaudiMP.pTools.FileRecordsAgent.SendFileRecords
def SendFileRecords(self)
Definition: pTools.py:232
GaudiMP.pTools.FileRecordsAgent.objectsOut
objectsOut
Definition: pTools.py:217

## Member Data Documentation

GaudiMP.pTools.FileRecordsAgent._gmpc
| private

Definition at line 212 of file pTools.py.

GaudiMP.pTools.FileRecordsAgent.fsr

Definition at line 213 of file pTools.py.

GaudiMP.pTools.FileRecordsAgent.log

Definition at line 215 of file pTools.py.

GaudiMP.pTools.FileRecordsAgent.objectsIn

Definition at line 216 of file pTools.py.

GaudiMP.pTools.FileRecordsAgent.objectsOut

Definition at line 217 of file pTools.py.

GaudiMP.pTools.FileRecordsAgent.q

Definition at line 214 of file pTools.py.

The documentation for this class was generated from the following file:

- GaudiMP/python/GaudiMP/pTools.py