The Gaudi Framework master (9cc87df9)
Loading...
Searching...
No Matches
AvalancheSchedulerSvc.h
Go to the documentation of this file.
1/***********************************************************************************\
2* (c) Copyright 1998-2026 CERN for the benefit of the LHCb and ATLAS collaborations *
3* *
4* This software is distributed under the terms of the Apache version 2 licence, *
5* copied verbatim in the file "LICENSE". *
6* *
7* In applying this licence, CERN does not waive the privileges and immunities *
8* granted to it by virtue of its status as an Intergovernmental Organization *
9* or submit itself to any jurisdiction. *
10\***********************************************************************************/
11#pragma once
12
13// Local includes
14#include "AlgsExecutionStates.h"
15#include "EventSlot.h"
16#include "FiberManager.h"
17#include "PrecedenceSvc.h"
18
19// Framework includes
27#include <GaudiKernel/Service.h>
28
29// STL includes
30#include <atomic>
31#include <chrono>
32#include <cstdint>
33#include <functional>
34#include <map>
35#include <memory>
36#include <queue>
37#include <string>
38#include <string_view>
39#include <thread>
40#include <unordered_map>
41#include <vector>
42
43// External libs
44#include <tbb/concurrent_priority_queue.h>
45#include <tbb/concurrent_queue.h>
46#include <tbb/task_arena.h>
47
48class IAlgorithm;
49
50//---------------------------------------------------------------------------
51
118class AvalancheSchedulerSvc : public extends<Service, IScheduler> {
119
120 friend class AlgTask;
121
122public:
124 using extends::extends;
125
127 StatusCode initialize() override;
128
130 StatusCode finalize() override;
131
133 StatusCode pushNewEvent( EventContext* eventContext ) override;
134
135 // Make multiple events available to the scheduler
136 StatusCode pushNewEvents( std::vector<EventContext*>& eventContexts ) override;
137
139 StatusCode popFinishedEvent( EventContext*& eventContext ) override;
140
142 StatusCode tryPopFinishedEvent( EventContext*& eventContext ) override;
143
145 unsigned int freeSlots() override;
146
148 void dumpState() override;
149
151 virtual StatusCode scheduleEventView( const EventContext* sourceContext, const std::string& nodeName,
152 std::unique_ptr<EventContext> viewContext ) override;
153
157 virtual void recordOccupancy( int samplePeriod, std::function<void( OccupancySnapshot )> callback ) override;
158
159private:
161 const std::map<std::string, DataObjIDColl>& outDeps ) const;
162
163private:
165 using action = std::function<StatusCode()>;
166
167 enum ActivationState { INACTIVE = 0, ACTIVE = 1, FAILURE = 2 };
168
169 // Occupancy snapshot data
171 std::chrono::system_clock::time_point m_lastSnapshot = std::chrono::system_clock::now();
172 std::function<void( OccupancySnapshot )> m_snapshotCallback;
173
175 this, "ThreadPoolSize", -1,
176 "Size of the global thread pool initialised by TBB; a value of -1 requests to use"
177 "all available hardware threads; -100 requests to bypass TBB executing "
178 "all algorithms in the scheduler's thread." };
180 this, "maxParallelismExtra", 0,
181 "Allows to add some extra threads to the maximum parallelism set in TBB"
182 "The TBB max parallelism is set as: ThreadPoolSize + maxParallelismExtra + 1" };
183 Gaudi::Property<std::string> m_whiteboardSvcName{ this, "WhiteboardSvc", "EventDataSvc", "The whiteboard name" };
185 this, "SimulateExecution", false,
186 "Flag to perform single-pass simulation of execution flow before the actual execution" };
188 "The following modes are currently available: PCE, COD, DRE, E" };
189 Gaudi::Property<bool> m_dumpIntraEventDynamics{ this, "DumpIntraEventDynamics", false,
190 "Dump intra-event concurrency dynamics to csv file" };
192 this, "NumOffloadThreads", 0,
193 "Number of threads to use for CPU portion of asynchronous algorithms. Asynchronous algorithms must be flagged "
194 "and use Boost Fiber functionality to suspend while waiting for offloaded work." };
195 // Defaults to 16 MiB
196 Gaudi::Property<std::size_t> m_fiberStackSize{ this, "FiberStackSize", 16ULL * 1024 * 1024,
197 "Stack size for fibers (asynchronous algorithms)" };
198 Gaudi::Property<bool> m_checkDeps{ this, "CheckDependencies", false,
199 "Runtime check of Algorithm Input Data Dependencies" };
200 Gaudi::Property<bool> m_checkOutput{ this, "CheckOutputUsage", false,
201 "Runtime check of Algorithm Output Data usage" };
203 this,
204 "CheckOutputUsageIgnoreList",
205 {},
206 "Ignore outputs of the Algorithms of this name when doing the check",
207 "OrderedSet<std::string>" };
208
210 "Attribute unmet input dependencies to this DataLoader Algorithm" };
211
212 Gaudi::Property<bool> m_enableCondSvc{ this, "EnableConditions", false, "Enable ConditionsSvc" };
213
214 Gaudi::Property<bool> m_showDataDeps{ this, "ShowDataDependencies", true,
215 "Show the INPUT and OUTPUT data dependencies of Algorithms" };
216
217 Gaudi::Property<bool> m_showDataFlow{ this, "ShowDataFlow", false,
218 "Show the configuration of DataFlow between Algorithms" };
219
220 Gaudi::Property<bool> m_showControlFlow{ this, "ShowControlFlow", false,
221 "Show the configuration of all Algorithms and Sequences" };
222
223 Gaudi::Property<bool> m_verboseSubSlots{ this, "VerboseSubSlots", false, "Dump algorithm states for all sub-slots" };
224
226 this, "DataDepsGraphFile", "",
227 "Name of the output file (.dot, .md or .graphml extensions allowed) containing the data dependency graph "
228 "for some selected Algorithms" };
229
231 this, "DataDepsGraphAlgPattern", ".*",
232 "Regex pattern for selecting desired Algorithms by name, whose data dependency has to be included in the data "
233 "deps graph" };
234
236 this, "DataDepsGraphObjectPattern", ".*",
237 "Regex pattern for selecting desired input or output by their full key" };
238
239 // Utils and shortcuts ----------------------------------------------------
240
242 void activate();
243
246
249
252
254 inline unsigned int algname2index( const std::string& algoname ) { return m_algname_index_map[algoname]; }
255
258
260 inline const std::string& index2algname( unsigned int index ) { return m_algname_vect[index]; }
261
264
267
270
273
276
278 tbb::concurrent_bounded_queue<EventContext*> m_finishedEvents;
279
282
285
287 unsigned int m_algosInFlight = 0;
288
289 // States management ------------------------------------------------------
290
293
294 // Update algorithm state and, optionally, revise states of other downstream algorithms
295 StatusCode revise( unsigned int iAlgo, EventContext* contextPtr, AState state, bool iterate = false );
296
298 struct TaskSpec;
300 StatusCode signoff( const TaskSpec& );
301
303 bool isStalled( const EventSlot& ) const;
305 void eventFailed( EventContext* eventContext );
306
308 void dumpSchedulerState( int iSlot );
309
310 // Algos Management -------------------------------------------------------
311
314
315 // Actions management -----------------------------------------------------
316
318 tbb::concurrent_bounded_queue<action> m_actionsQueue;
319
321 struct TaskSpec {
324 TaskSpec( IAlgorithm* algPtr, unsigned int algIndex, const std::string& algName, unsigned int algRank,
325 bool asynchronous, int slotIndex, EventContext* eventContext )
326 : algPtr( algPtr )
327 , algIndex( algIndex )
328 , algName( algName )
329 , algRank( algRank )
332 , contextPtr( eventContext ) {}
333
334 TaskSpec( const TaskSpec& ) = default;
336 TaskSpec& operator=( const TaskSpec& ) = delete;
338 TaskSpec( TaskSpec&& ) = default;
340 TaskSpec& operator=( TaskSpec&& ) = default;
341
342 IAlgorithm* algPtr{ nullptr };
343 unsigned int algIndex{ 0 };
344 std::string_view algName;
345 unsigned int algRank{ 0 };
346 bool asynchronous{ false };
347 int slotIndex{ 0 };
349 };
350
353 bool operator()( const TaskSpec& i, const TaskSpec& j ) const { return ( i.algRank < j.algRank ); }
354 };
355
357 tbb::concurrent_priority_queue<TaskSpec, AlgQueueSort> m_scheduledQueue;
358 tbb::concurrent_priority_queue<TaskSpec, AlgQueueSort> m_scheduledAsynchronousQueue;
360
361 // Prompt the scheduler to call updateStates
363
364 // ------------------------------------------------------------------------
365
366 // Service for thread pool initialization
368 tbb::task_arena* m_arena{ nullptr };
370
372
373public:
374 // get next schedule-able TaskSpec
375 bool next( TaskSpec& ts, bool asynchronous ) {
376 if ( asynchronous ) { return m_scheduledAsynchronousQueue.try_pop( ts ); }
377 return m_scheduledQueue.try_pop( ts );
378 }
379};
atomic< int > atomic_int
State
Execution states of the algorithms Must have contiguous integer values 0, 1... N.
StatusCode pushNewEvent(EventContext *eventContext) override
Make an event available to the scheduler.
Gaudi::Property< std::vector< std::string > > m_checkOutputIgnoreList
SmartIF< IThreadPoolSvc > m_threadPoolSvc
Gaudi::Property< std::string > m_useDataLoader
void dumpState() override
Dump scheduler state for all slots.
void activate()
Activate scheduler.
Gaudi::Property< std::string > m_optimizationMode
bool next(TaskSpec &ts, bool asynchronous)
StatusCode popFinishedEvent(EventContext *&eventContext) override
Blocks until an event is available.
Gaudi::Property< bool > m_dumpIntraEventDynamics
std::chrono::system_clock::time_point m_lastSnapshot
std::vector< std::string > m_algname_vect
Vector to bookkeep the information necessary to the index2name conversion.
Gaudi::Property< int > m_threadPoolSize
StatusCode finalize() override
Finalise.
tbb::concurrent_priority_queue< TaskSpec, AlgQueueSort > m_scheduledQueue
Queues for scheduled algorithms.
std::function< void(OccupancySnapshot)> m_snapshotCallback
std::queue< TaskSpec > m_retryQueue
Gaudi::Property< bool > m_verboseSubSlots
tbb::concurrent_bounded_queue< action > m_actionsQueue
Queue where closures are stored and picked for execution.
unsigned int algname2index(const std::string &algoname)
Convert a name to an integer.
SmartIF< ICondSvc > m_condSvc
A shortcut to service for Conditions handling.
AlgsExecutionStates::State AState
bool isStalled(const EventSlot &) const
Check if scheduling in a particular slot is in a stall.
SmartIF< IAlgExecStateSvc > m_algExecStateSvc
Algorithm execution state manager.
StatusCode pushNewEvents(std::vector< EventContext * > &eventContexts) override
StatusCode revise(unsigned int iAlgo, EventContext *contextPtr, AState state, bool iterate=false)
tbb::concurrent_bounded_queue< EventContext * > m_finishedEvents
Queue of finished events.
StatusCode deactivate()
Deactivate scheduler.
unsigned int m_algosInFlight
Number of algorithms presently in flight.
Gaudi::Property< std::string > m_dataDepsGraphObjectPattern
Gaudi::Property< bool > m_showDataFlow
StatusCode schedule(TaskSpec &&)
SmartIF< IPrecedenceSvc > m_precSvc
A shortcut to the Precedence Service.
Gaudi::Property< bool > m_checkDeps
std::chrono::duration< int64_t, std::milli > m_snapshotInterval
Gaudi::Property< std::string > m_dataDepsGraphFile
SmartIF< IAlgResourcePool > m_algResourcePool
Cache for the algorithm resource pool.
Gaudi::Property< bool > m_showControlFlow
Gaudi::Property< bool > m_simulateExecution
Gaudi::Property< std::string > m_whiteboardSvcName
StatusCode tryPopFinishedEvent(EventContext *&eventContext) override
Try to fetch an event from the scheduler.
std::unordered_map< std::string, unsigned int > m_algname_index_map
Map to bookkeep the information necessary to the name2index conversion.
std::atomic< bool > m_needsUpdate
virtual StatusCode scheduleEventView(const EventContext *sourceContext, const std::string &nodeName, std::unique_ptr< EventContext > viewContext) override
Method to inform the scheduler about event views.
Gaudi::Property< int > m_maxParallelismExtra
StatusCode signoff(const TaskSpec &)
The call to this method is triggered only from within the AlgTask.
std::function< StatusCode()> action
Gaudi::Property< std::string > m_dataDepsGraphAlgoPattern
Gaudi::Property< int > m_numOffloadThreads
Gaudi::Property< bool > m_checkOutput
std::atomic< ActivationState > m_isActive
Flag to track if the scheduler is active or not.
StatusCode initialize() override
Initialise.
Gaudi::Property< bool > m_enableCondSvc
virtual void recordOccupancy(int samplePeriod, std::function< void(OccupancySnapshot)> callback) override
Sample occupancy at fixed interval (ms) Negative value to deactivate, 0 to snapshot every change Each...
void eventFailed(EventContext *eventContext)
Method to execute if an event failed.
std::atomic_int m_freeSlots
Atomic to account for asyncronous updates by the scheduler wrt the rest.
StatusCode dumpDataDepsGraphFile(const std::map< std::string, DataObjIDColl > &inDeps, const std::map< std::string, DataObjIDColl > &outDeps) const
unsigned int freeSlots() override
Get free slots number.
void dumpSchedulerState(int iSlot)
Dump the state of the scheduler.
std::unique_ptr< FiberManager > m_fiberManager
SmartIF< IHiveWhiteBoard > m_whiteboard
A shortcut to the whiteboard.
tbb::concurrent_priority_queue< TaskSpec, AlgQueueSort > m_scheduledAsynchronousQueue
Gaudi::Property< std::size_t > m_fiberStackSize
StatusCode iterate()
Loop on all slots to schedule DATAREADY algorithms and sign off ready events.
Gaudi::Property< bool > m_showDataDeps
std::thread m_thread
The thread in which the activate function runs.
const std::string & index2algname(unsigned int index)
Convert an integer to a name.
std::vector< EventSlot > m_eventSlots
Vector of events slots.
This class represents an entry point to all the event specific data.
Implementation of property with value of concrete type.
Definition Property.h:35
The IAlgorithm is the interface implemented by the Algorithm base class.
Definition IAlgorithm.h:37
Small smart pointer class with automatic reference counting for IInterface.
Definition SmartIF.h:28
This class is used for returning status codes from appropriate routines.
Definition StatusCode.h:64
StatusCode()=default
Default constructor.
Base class used to extend a class implementing other interfaces.
Definition extends.h:19
Comparison operator to sort the queues.
bool operator()(const TaskSpec &i, const TaskSpec &j) const
Struct to hold entries in the alg queues.
TaskSpec & operator=(const TaskSpec &)=delete
Assignment operator.
TaskSpec(IAlgorithm *algPtr, unsigned int algIndex, const std::string &algName, unsigned int algRank, bool asynchronous, int slotIndex, EventContext *eventContext)
TaskSpec(const TaskSpec &)=default
Copy constructor (to keep a lambda capturing a TaskSpec storable as a std::function value).
TaskSpec & operator=(TaskSpec &&)=default
Move assignment.
TaskSpec(TaskSpec &&)=default
Move constructor.
Class representing an event slot.
Definition EventSlot.h:28