| The Gaudi Framework v29r0 (ff2e7097)

- Main Page
- Related Pages
- Modules
- Namespaces
- Classes
- Files

- File List
- File Members

precedence.py
Go to the documentation of this file.
1 import os
2 import sys
3 import random
4 import string
5 import json
6 import networkx as nx
7
8 from Gaudi.Configuration import INFO
9 from Configurables import GaudiSequencer, CPUCruncher
10
11
12 def _buildFilePath(filePath):
13
14 if not os.path.exists(filePath):
15 __fullFilePath__ = os.path.realpath(os.path.join(
16 os.environ.get('GAUDIHIVEROOT', ''), "data", filePath))
17 if not os.path.exists(__fullFilePath__):
18 print "\nERROR: invalid file path '%s'. It must be either absolute, or relative to '$GAUDIHIVEROOT/data/'." % filePath
19 sys.exit(1)
20 else:
21 __fullFilePath__ = filePath
22
23 return __fullFilePath__
24
25
26 class UniformTimeValue(object):
27 """A class to manage uniform algorithm timing"""
28
29 def __init__(self, avgRuntime, varRuntime=0):
30
31 self.avgRuntime = avgRuntime
32 self.varRuntime = varRuntime
33
34 def get(self, algoName=''):
35 """Get time and its variance (in a tuple) for a given algorithm name"""
36
37 return self.avgRuntime, self.varRuntime
38
39
40 class RealTimeValue(object):
41 """A class to manage real algorithm timing"""
42
43 def __init__(self, path, defaultTime, factor=1):
44 """
45 defaultTime -- run time, assigned to an algorithm if no time is found in provided timing library
46 (and it will also be scaled by the 'factor' argument)
47 """
48
49 self.path = os.path.realpath(_buildFilePath(path))
50 self.factor = factor
51 self.defaultTime = defaultTime # typically 0.05s
52 self.varRuntime = 0
53
54 self.file = open(self.path)
55 self.timings = json.load(self.file)
56
57 def get(self, algoName=''):
58 """Get time for a given algorithm name"""
59
60 if algoName in self.timings:
61 time = float(self.timings[algoName])
62 else:
63 capAlgoName = algoName[0].upper() + algoName[1:len(algoName)]
64
65 if capAlgoName in self.timings:
66 time = float(self.timings[capAlgoName])
67 else:
68 time = self.defaultTime
69 print "WARNING: TimiNg for %s (or %s) not found in the provided library, using default one: %s" % (algoName, capAlgoName, time)
70
71 time = time * self.factor
72
73 return time, self.varRuntime
74
75
76 class UniformBooleanValue(object):
77
78 def __init__(self, value):
79
80 self.value = value
81
82 def get(self):
83
84 return self.value
85
86
87 class RndBiased10BooleanValue(object):
88 """
89 Provides randomly distributed boolean value with True taking only 10%.
90 The distribution has only 276 values and is reproducible, if no pattern re-generation is requested.
91 """
92
93 # 276 values, biased as 90% to 10%
94 builtinPattern = [True, False, False, True, False, False, False, False, False, False, False, False, False, False, False, False, True, False, False, True, False, False, False, False, False, False, False, False, False, True, False, False, False, False, False, False, False, False, False, False, False, True, False, False, False, False, False, False, False, False, True, False, False, False, False, False, False, False, False, True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, False, True, False, True, False, False, False, False, False, False, False, False, True, False, True, False, False, True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, False, False, False, False, False, False, False, False, False, True, False, False, False, False, True, True, False,
95 False, False, False, False, False, False, False, False, False, True, True, False, True, False, False, False, False, False, False, False, False, False, False, True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, False, False, False, False, False, False, False, False, False, False, False, True, False, False, False, False, False, False, False, False, False, False, False, True, False, False, False, True, False, False, False, False, False, False, False, False, True, False, False, False, False, False, False, False, False, False, False, False, True, False, True]
96 pattern = []
97
98 def __init__(self, useBuiltinPattern=True):
99
100 if useBuiltinPattern:
101 self.pattern = self.builtinPattern
102 else:
103 # 278 values, biased approximately as 90% to 10% (276 precedence graph algorithms, plus two algorithms added manually - DstWriter and Framework)
104 self.pattern = [False for i in range(
105 249)] + [True for i in range(29)]
106 random.shuffle(self.pattern)
107
108 self.generator = self._create_generator(self.pattern)
109
110 def _create_generator(self, pattern):
111
112 for b in pattern:
113 yield b
114
115 def get(self):
116
117 return next(self.generator)
118
119 def get_pattern(self):
120
121 return self.pattern
122
123
124 class CruncherSequence(object):
125 """Constructs the sequence tree of CPUCrunchers with provided control flow and data flow precedence rules."""
126
127 unique_sequencers = []
128 dupl_seqs = {}
129 OR_sequencers = []
130 unique_algos = []
131 dupl_algos = {}
132
133 unique_data_objects = []
134
135 def __init__(self, timeValue, IOboolValue, sleepFraction, cfgPath, dfgPath, topSequencer,
136 showStat=False, timeline=False, outputLevel=INFO):
137 """
138 Keyword arguments:
139 timeValue -- timeValue object to set algorithm execution time
140 IOboolValue -- *BooleanValue object to set whether an algorithm has to experience IO-bound execution
141 cfgPath -- relative to $GAUDIHIVEROOT/data path to GRAPHML file with control flow dependencies
142 dfgPath -- relative to $GAUDIHIVEROOT/data path to GRAPHML file with data flow dependencies
143 showStat -- print out statistics on precedence graph
144 """
145
146 self.timeValue = timeValue
147 self.IOboolValue = IOboolValue
148 self.sleepFraction = sleepFraction
149
150 self.cfg = nx.read_graphml(_buildFilePath(cfgPath))
151 self.dfg = nx.read_graphml(_buildFilePath(dfgPath))
152
153 self.enableTimeline = timeline
154
155 self.outputLevel = outputLevel
156
157 # Generate control flow part
158 self.sequencer = self._generate_sequence(topSequencer)
159
160 if showStat:
161 import pprint
162
163 print "\n===== Statistics on Algorithms ====="
164 print "Total number of algorithm nodes: ", len(self.unique_algos) + sum([self.dupl_algos[i] - 1 for i in self.dupl_algos])
165 print "Number of unique algorithms: ", len(self.unique_algos)
166 print " -->", len(self.dupl_algos), "of them being re-used with the following distribution: ", [self.dupl_algos[i] for i in self.dupl_algos]
167 # pprint.pprint(dupl_algos)
168
169 print "\n===== Statistics on Sequencers ====="
170 print "Total number of sequencers: ", len(self.unique_sequencers) + sum([self.dupl_seqs[i] - 1 for i in self.dupl_seqs])
171 print "Number of unique sequencers: ", len(self.unique_sequencers)
172 print " -->", len(self.dupl_seqs), "of them being re-used with the following distribution: ", [self.dupl_seqs[i] for i in self.dupl_seqs]
173 # pprint.pprint(dupl_seqs)
174 print "Number of OR-sequencers: ", len(self.OR_sequencers)
175
176 print "\n===== Statistics on DataObjects ====="
177 print "Number of unique DataObjects: ", len(self.unique_data_objects)
178 # pprint.pprint(self.unique_data_objects)
179 print
180
181 def get(self):
182
183 return self.sequencer
184
185 def _declare_data_deps(self, algo_name, algo):
186 """ Declare data inputs and outputs for a given algorithm. """
187
188 # Declare data inputs
189 for inNode, outNode in self.dfg.in_edges(algo_name):
190 dataName = inNode
191 if dataName not in self.unique_data_objects:
192 self.unique_data_objects.append(dataName)
193
194 if dataName not in algo.inpKeys:
195 algo.inpKeys.append(dataName)
196
197 # Declare data outputs
198 for inNode, outNode in self.dfg.out_edges(algo_name):
199 dataName = outNode
200 if dataName not in self.unique_data_objects:
201 self.unique_data_objects.append(dataName)
202
203 if dataName not in algo.outKeys:
204 algo.outKeys.append(dataName)
205
206 def _generate_sequence(self, name, seq=None):
207 """ Assemble the tree of sequencers. """
208
209 if not seq:
210 seq = GaudiSequencer(name, ShortCircuit=False)
211
212 for n in self.cfg[name]:
213 if '/' in n:
214 algo_type, algo_name = n.split('/')
215 else:
216 algo_type = 'GaudiAlgorithm'
217 algo_name = n
218
219 if algo_type in ['GaudiSequencer', 'AthSequencer', 'ProcessPhase']:
220 if algo_name in ['RecoITSeq', 'RecoOTSeq', 'RecoTTSeq']:
221 continue
222
223 if n not in self.unique_sequencers:
224 self.unique_sequencers.append(n)
225 else:
226 if n not in self.dupl_seqs:
227 self.dupl_seqs[n] = 2
228 else:
229 self.dupl_seqs[n] += 1
230
231 seq_daughter = GaudiSequencer(algo_name, OutputLevel=INFO)
232 if self.cfg.node[n].get('ModeOR') == 'True':
233 self.OR_sequencers.append(n)
234 seq_daughter.ModeOR = True
235 # if self.cfg.node[n].get('Lazy') == 'False':
236 # print "Non-Lazy - ", n
237 seq_daughter.ShortCircuit = False
238 if seq_daughter not in seq.Members:
239 seq.Members += [seq_daughter]
240 # iterate deeper
241 self._generate_sequence(n, seq_daughter)
242 else:
243 #rndname = ''.join(random.choice(string.lowercase) for i in range(5))
244 #if algo_name in unique_algos: algo_name = algo_name + "-" + rndname
245 if n not in self.unique_algos:
246 self.unique_algos.append(n)
247 else:
248 if n not in self.dupl_algos:
249 self.dupl_algos[n] = 2
250 else:
251 self.dupl_algos[n] += 1
252
253 avgRuntime, varRuntime = self.timeValue.get(algo_name)
254 algo_daughter = CPUCruncher(algo_name,
255 OutputLevel=self.outputLevel,
256 shortCalib=True,
257 varRuntime=varRuntime,
258 avgRuntime=avgRuntime,
259 SleepFraction=self.sleepFraction if self.IOboolValue.get() else 0.,
260 Timeline=self.enableTimeline)
261
262 self._declare_data_deps(algo_name, algo_daughter)
263
264 if algo_daughter not in seq.Members:
265 seq.Members += [algo_daughter]
266
267 return seq
GaudiHive.precedence.RndBiased10BooleanValue.generator
generator
Definition: precedence.py:108
GaudiHive.precedence.RndBiased10BooleanValue._create_generator
def _create_generator(self, pattern)
Definition: precedence.py:110
GaudiHive.precedence.RealTimeValue.__init__
def __init__(self, path, defaultTime, factor=1)
Definition: precedence.py:43
GaudiHive.precedence.CruncherSequence._generate_sequence
def _generate_sequence(self, name, seq=None)
Definition: precedence.py:206
GaudiHive.precedence.UniformTimeValue.__init__
def __init__(self, avgRuntime, varRuntime=0)
Definition: precedence.py:29
GaudiHive.precedence.CruncherSequence.timeValue
timeValue
Definition: precedence.py:146
GaudiHive.precedence.CruncherSequence.OR_sequencers
list OR_sequencers
Definition: precedence.py:129
GaudiHive.precedence.CruncherSequence.get
def get(self)
Definition: precedence.py:181
GaudiHive.precedence.RndBiased10BooleanValue
Definition: precedence.py:87
sum
double sum(double x, double y, double z)
Definition: 3DoubleFuncTest.cpp:25
GaudiHive.precedence.RealTimeValue
Definition: precedence.py:40
GaudiHive.precedence.CruncherSequence.enableTimeline
enableTimeline
Definition: precedence.py:153
Gaudi.Configuration
Definition: Configuration.py:1
GaudiHive.precedence.CruncherSequence.unique_data_objects
list unique_data_objects
Definition: precedence.py:133
GaudiHive.precedence.CruncherSequence.unique_algos
list unique_algos
Definition: precedence.py:130
GaudiHive.precedence.RealTimeValue.get
def get(self, algoName='')
Definition: precedence.py:57
GaudiHive.precedence.UniformBooleanValue.__init__
def __init__(self, value)
Definition: precedence.py:78
GaudiHive.precedence.CruncherSequence.cfg
cfg
Definition: precedence.py:150
GaudiHive.precedence.RndBiased10BooleanValue.get_pattern
def get_pattern(self)
Definition: precedence.py:119
GaudiHive.precedence.CruncherSequence.dfg
dfg
Definition: precedence.py:151
GaudiHive.precedence.UniformTimeValue.get
def get(self, algoName='')
Definition: precedence.py:34
GaudiHive.precedence.RealTimeValue.timings
timings
Definition: precedence.py:55
GaudiHive.precedence.RealTimeValue.file
file
Definition: precedence.py:54
GaudiHive.precedence.CruncherSequence.__init__
def __init__(self, timeValue, IOboolValue, sleepFraction, cfgPath, dfgPath, topSequencer, showStat=False, timeline=False, outputLevel=INFO)
Definition: precedence.py:136
GaudiHive.precedence.UniformBooleanValue
Definition: precedence.py:76
GaudiHive.precedence.RndBiased10BooleanValue.__init__
def __init__(self, useBuiltinPattern=True)
Definition: precedence.py:98
GaudiHive.precedence.CruncherSequence.IOboolValue
IOboolValue
Definition: precedence.py:147
Gaudi::Functional::details::zip::range
decltype(auto) range(Args &&...args)
Zips multiple containers together to form a single range.
Definition: FunctionalDetails.h:85
GaudiHive.precedence.CruncherSequence.unique_sequencers
list unique_sequencers
Definition: precedence.py:127
GaudiHive.precedence.RealTimeValue.varRuntime
varRuntime
Definition: precedence.py:52
GaudiHive.precedence.RndBiased10BooleanValue.get
def get(self)
Definition: precedence.py:115
GaudiHive.precedence.CruncherSequence.outputLevel
outputLevel
Definition: precedence.py:155
GaudiHive.precedence.RealTimeValue.factor
factor
Definition: precedence.py:50
GaudiHive.precedence.UniformTimeValue.varRuntime
varRuntime
Definition: precedence.py:32
GaudiHive.precedence.CruncherSequence.dupl_seqs
dictionary dupl_seqs
Definition: precedence.py:128
GaudiHive.precedence.RealTimeValue.defaultTime
defaultTime
Definition: precedence.py:51
GaudiHive.precedence.CruncherSequence.sequencer
sequencer
Definition: precedence.py:158
GaudiHive.precedence.UniformBooleanValue.get
def get(self)
Definition: precedence.py:82
GaudiHive.precedence.CruncherSequence.dupl_algos
dictionary dupl_algos
Definition: precedence.py:131
GaudiHive.precedence.UniformTimeValue.avgRuntime
avgRuntime
Definition: precedence.py:31
GaudiHive.precedence.CruncherSequence._declare_data_deps
def _declare_data_deps(self, algo_name, algo)
Definition: precedence.py:185
GaudiHive.precedence.CruncherSequence
Definition: precedence.py:124
GaudiHive.precedence._buildFilePath
def _buildFilePath(filePath)
Definition: precedence.py:12
GaudiHive.precedence.RndBiased10BooleanValue.pattern
list pattern
Definition: precedence.py:96
GaudiHive.precedence.RealTimeValue.path
path
Definition: precedence.py:49
GaudiHive.precedence.UniformBooleanValue.value
value
Definition: precedence.py:80
GaudiHive.precedence.UniformTimeValue
Definition: precedence.py:26
GaudiHive.precedence.RndBiased10BooleanValue.builtinPattern
list builtinPattern
Definition: precedence.py:94
GaudiHive.precedence.CruncherSequence.sleepFraction
sleepFraction
Definition: precedence.py:148