| The Gaudi Framework v32r1 (f65d50dc)

Public Member Functions | Public Attributes | Private Member Functions | List of all members
GaudiMP.Parallel.WorkManager Class Reference
Inheritance diagram for GaudiMP.Parallel.WorkManager:
[legend]
Collaboration diagram for GaudiMP.Parallel.WorkManager:
[legend]

## Public Member Functions
def | __init__ (self, ncpus='autodetect', ppservers=None)
def | __del__ (self)
def | process (self, task, items, timeout=90000)

## Public Attributes
| ncpus
| ppservers
| sessions
| server
| mode
| pool
| stats

## Private Member Functions
def | _printStatistics (self)
def | _mergeStatistics (self, stat)

## Detailed Description

    Class to in charge of managing the tasks and distributing them to
        the workers. They can be local (using other cores) or remote
        using other nodes in the local cluster

Definition at line 134 of file Parallel.py.

## Constructor & Destructor Documentation

## ◆ __init__()

def GaudiMP.Parallel.WorkManager.__init__ | ( |  | self,
|  |  | ncpus = `'autodetect'`,
|  |  | ppservers = `None`
| ) |  |

Definition at line 139 of file Parallel.py.

139 def __init__(self, ncpus='autodetect', ppservers=None):
140 if ncpus == 'autodetect':
141 self.ncpus = multiprocessing.cpu_count()
142 else:
143 self.ncpus = ncpus
144 if ppservers:
145 import pp
146 self.ppservers = ppservers
147 self.sessions = [SshSession(srv) for srv in ppservers]
148 self.server = pp.Server(ncpus=self.ncpus, ppservers=self.ppservers)
149 self.mode = 'cluster'
150 else:
151 self.pool = multiprocessing.Pool(self.ncpus)
152 self.mode = 'multicore'
153 self.stats = {}
154

## ◆ __del__()

def GaudiMP.Parallel.WorkManager.__del__ | ( |  | self | ) |

Definition at line 155 of file Parallel.py.

155 def __del__(self):
156 if hasattr(self, 'server'):
157 self.server.destroy()
158

## Member Function Documentation

## ◆ _mergeStatistics()

def GaudiMP.Parallel.WorkManager._mergeStatistics | ( |  | self,
|  |  | stat
| ) |  |
| private

Definition at line 201 of file Parallel.py.

201 def _mergeStatistics(self, stat):
202 if stat.name not in self.stats:
203 self.stats[stat.name] = Statistics()
204 s = self.stats[stat.name]
205 s.time += stat.time
206 s.njob += 1
207
208

## ◆ _printStatistics()

def GaudiMP.Parallel.WorkManager._printStatistics | ( |  | self | ) |
| private

Definition at line 190 of file Parallel.py.

190 def _printStatistics(self):
191 njobs = 0
192 for stat in self.stats.values():
193 njobs += stat.njob
194 print 'Job execution statistics:'
195 print 'job count | % of all jobs | job time sum | time per job | job server'
196 for name, stat in self.stats.items():
197 print ' %d | %6.2f | %8.3f | %8.3f | %s' % (
198 stat.njob, 100. * stat.njob / njobs, stat.time,
199 stat.time / stat.njob, name)
200

## ◆ process()

def GaudiMP.Parallel.WorkManager.process | ( |  | self,
|  |  | task,
|  |  | items,
|  |  | timeout = `90000`
| ) |  |

Definition at line 159 of file Parallel.py.

159 def process(self, task, items, timeout=90000):
160 if not isinstance(task, Task):
161 raise TypeError("task argument needs to be an 'Task' instance")
162 # --- Call the Local initialialization
163 task.initializeLocal()
164 # --- Schedule all the jobs ....
165 if self.mode == 'cluster':
166 jobs = [
167 self.server.submit(_prefunction, (_ppfunction, task, item), (),
168 ('GaudiMP.Parallel', 'time'))
169 for item in items
170 ]
171 for job in jobs:
172 result, stat = job()
173 task._mergeResults(result)
174 self._mergeStatistics(stat)
175 self._printStatistics()
176 self.server.print_stats()
177 elif self.mode == 'multicore':
178 start = time.time()
179 jobs = self.pool.map_async(_ppfunction,
180 zip([task for i in items], items))
181 for result, stat in jobs.get(timeout):
182 task._mergeResults(result)
183 self._mergeStatistics(stat)
184 end = time.time()
185 self._printStatistics()
186 print 'Time elapsed since server creation %f' % (end - start)
187 # --- Call the Local Finalize
188 task.finalize()
189

## Member Data Documentation

## ◆ mode

GaudiMP.Parallel.WorkManager.mode

Definition at line 149 of file Parallel.py.

## ◆ ncpus

GaudiMP.Parallel.WorkManager.ncpus

Definition at line 141 of file Parallel.py.

## ◆ pool

GaudiMP.Parallel.WorkManager.pool

Definition at line 151 of file Parallel.py.

## ◆ ppservers

GaudiMP.Parallel.WorkManager.ppservers

Definition at line 146 of file Parallel.py.

## ◆ server

GaudiMP.Parallel.WorkManager.server

Definition at line 148 of file Parallel.py.

## ◆ sessions

GaudiMP.Parallel.WorkManager.sessions

Definition at line 147 of file Parallel.py.

## ◆ stats

GaudiMP.Parallel.WorkManager.stats

Definition at line 153 of file Parallel.py.

The documentation for this class was generated from the following file:

- GaudiMP/python/GaudiMP/Parallel.py