| The Gaudi Framework v29r0 (ff2e7097)

- Main Page
- Related Pages
- Modules
- Namespaces
- Classes
- Files

- Class List
- Class Index
- Class Hierarchy
- Class Members

Public Member Functions | Public Attributes | Private Member Functions | List of all members
GaudiMP.Parallel.WorkManager Class Reference
Inheritance diagram for GaudiMP.Parallel.WorkManager:
[legend]
Collaboration diagram for GaudiMP.Parallel.WorkManager:
[legend]

## Public Member Functions
def | __init__ (self, ncpus='autodetect', ppservers=None)
def | __del__ (self)
def | process (self, task, items, timeout=90000)

## Public Attributes
| ncpus
| ppservers
| sessions
| server
| mode
| pool
| stats

## Private Member Functions
def | _printStatistics (self)
def | _mergeStatistics (self, stat)

## Detailed Description

    Class to in charge of managing the tasks and distributing them to
        the workers. They can be local (using other cores) or remote
        using other nodes in the local cluster

Definition at line 135 of file Parallel.py.

## Constructor & Destructor Documentation

def GaudiMP.Parallel.WorkManager.__init__ | ( |  | self,
|  |  | ncpus = `'autodetect'`,
|  |  | ppservers = `None`
| ) |  |

Definition at line 140 of file Parallel.py.

140 def __init__(self, ncpus='autodetect', ppservers=None):
141 if ncpus == 'autodetect':
142 self.ncpus = multiprocessing.cpu_count()
143 else:
144 self.ncpus = ncpus
145 if ppservers:
146 import pp
147 self.ppservers = ppservers
148 self.sessions = [SshSession(srv) for srv in ppservers]
149 self.server = pp.Server(ncpus=self.ncpus, ppservers=self.ppservers)
150 self.mode = 'cluster'
151 else:
152 self.pool = multiprocessing.Pool(self.ncpus)
153 self.mode = 'multicore'
154 self.stats = {}
155
GaudiMP.Parallel.WorkManager.__init__
def __init__(self, ncpus='autodetect', ppservers=None)
Definition: Parallel.py:140
GaudiMP.Parallel.WorkManager.ncpus
ncpus
Definition: Parallel.py:142
GaudiMP.Parallel.WorkManager.mode
mode
Definition: Parallel.py:150
GaudiMP.Parallel.WorkManager.stats
stats
Definition: Parallel.py:154
GaudiMP.Parallel.WorkManager.server
server
Definition: Parallel.py:149
GaudiMP.Parallel.WorkManager.ppservers
ppservers
Definition: Parallel.py:147
GaudiMP.Parallel.WorkManager.pool
pool
Definition: Parallel.py:152
GaudiMP.Parallel.SshSession
Definition: Parallel.py:205
GaudiMP.Parallel.WorkManager.sessions
sessions
Definition: Parallel.py:148

def GaudiMP.Parallel.WorkManager.__del__ | ( |  | self | ) |

Definition at line 156 of file Parallel.py.

156 def __del__(self):
157 if hasattr(self, 'server'):
158 self.server.destroy()
159
GaudiMP.Parallel.WorkManager.__del__
def __del__(self)
Definition: Parallel.py:156

## Member Function Documentation

def GaudiMP.Parallel.WorkManager._mergeStatistics | ( |  | self,
|  |  | stat
| ) |  |
| private

Definition at line 197 of file Parallel.py.

197 def _mergeStatistics(self, stat):
198 if stat.name not in self.stats:
199 self.stats[stat.name] = Statistics()
200 s = self.stats[stat.name]
201 s.time += stat.time
202 s.njob += 1
203
204
GaudiMP.Parallel.WorkManager._mergeStatistics
def _mergeStatistics(self, stat)
Definition: Parallel.py:197
GaudiMP.Parallel.WorkManager.stats
stats
Definition: Parallel.py:154
GaudiMP.Parallel.Statistics
Definition: Parallel.py:46

def GaudiMP.Parallel.WorkManager._printStatistics | ( |  | self | ) |
| private

Definition at line 188 of file Parallel.py.

188 def _printStatistics(self):
189 njobs = 0
190 for stat in self.stats.values():
191 njobs += stat.njob
192 print 'Job execution statistics:'
193 print 'job count | % of all jobs | job time sum | time per job | job server'
194 for name, stat in self.stats.items():
195 print ' %d | %6.2f | %8.3f | %8.3f | %s' % (stat.njob, 100. * stat.njob / njobs, stat.time, stat.time / stat.njob, name)
196
GaudiMP.Parallel.WorkManager._printStatistics
def _printStatistics(self)
Definition: Parallel.py:188

def GaudiMP.Parallel.WorkManager.process | ( |  | self,
|  |  | task,
|  |  | items,
|  |  | timeout = `90000`
| ) |  |

Definition at line 160 of file Parallel.py.

160 def process(self, task, items, timeout=90000):
161 if not isinstance(task, Task):
162 raise TypeError("task argument needs to be an 'Task' instance")
163 # --- Call the Local initialialization
164 task.initializeLocal()
165 # --- Schedule all the jobs ....
166 if self.mode == 'cluster':
167 jobs = [self.server.submit(
168 _prefunction, (_ppfunction, task, item), (), ('GaudiMP.Parallel', 'time')) for item in items]
169 for job in jobs:
170 result, stat = job()
171 task._mergeResults(result)
172 self._mergeStatistics(stat)
173 self._printStatistics()
174 self.server.print_stats()
175 elif self.mode == 'multicore':
176 start = time.time()
177 jobs = self.pool.map_async(
178 _ppfunction, zip([task for i in items], items))
179 for result, stat in jobs.get(timeout):
180 task._mergeResults(result)
181 self._mergeStatistics(stat)
182 end = time.time()
183 self._printStatistics()
184 print 'Time elapsed since server creation %f' % (end - start)
185 # --- Call the Local Finalize
186 task.finalize()
187
GaudiMP.Parallel.WorkManager._mergeStatistics
def _mergeStatistics(self, stat)
Definition: Parallel.py:197
GaudiMP.Parallel.WorkManager._printStatistics
def _printStatistics(self)
Definition: Parallel.py:188
GaudiMP.Parallel.WorkManager.mode
mode
Definition: Parallel.py:150
GaudiMP.Parallel.WorkManager.process
def process(self, task, items, timeout=90000)
Definition: Parallel.py:160

## Member Data Documentation

GaudiMP.Parallel.WorkManager.mode

Definition at line 150 of file Parallel.py.

GaudiMP.Parallel.WorkManager.ncpus

Definition at line 142 of file Parallel.py.

GaudiMP.Parallel.WorkManager.pool

Definition at line 152 of file Parallel.py.

GaudiMP.Parallel.WorkManager.ppservers

Definition at line 147 of file Parallel.py.

GaudiMP.Parallel.WorkManager.server

Definition at line 149 of file Parallel.py.

GaudiMP.Parallel.WorkManager.sessions

Definition at line 148 of file Parallel.py.

GaudiMP.Parallel.WorkManager.stats

Definition at line 154 of file Parallel.py.

The documentation for this class was generated from the following file:

- GaudiMP/python/GaudiMP/Parallel.py