The Gaudi Framework master (934d2c3c)
Loading...
Searching...
No Matches
ThreadPoolSvc.cpp
Go to the documentation of this file.
1/***********************************************************************************\
2* (c) Copyright 1998-2026 CERN for the benefit of the LHCb and ATLAS collaborations *
3* *
4* This software is distributed under the terms of the Apache version 2 licence, *
5* copied verbatim in the file "LICENSE". *
6* *
7* In applying this licence, CERN does not waive the privileges and immunities *
8* granted to it by virtue of its status as an Intergovernmental Organization *
9* or submit itself to any jurisdiction. *
10\***********************************************************************************/
11
12// Local includes
13#include "ThreadPoolSvc.h"
14#include "ThreadInitTask.h"
15
16// Framework includes
18
19// STL includes
20#include <chrono>
21#include <ios>
22#include <thread>
23
24#define ON_DEBUG if ( msgLevel( MSG::DEBUG ) )
25#define ON_VERBOSE if ( msgLevel( MSG::VERBOSE ) )
26
27namespace Gaudi {
28 namespace Concurrency {
29 extern thread_local bool ThreadInitDone;
30 }
31} // namespace Gaudi
32
34
35//=============================================================================
36
37ThreadPoolSvc::ThreadPoolSvc( const std::string& name, ISvcLocator* svcLoc ) : extends( name, svcLoc ) {
38 declareProperty( "ThreadInitTools", m_threadInitTools, "ToolHandleArray of IThreadInitTools" );
39}
40
41//-----------------------------------------------------------------------------
42
44
45 // Initialise mother class (read properties, ...)
46 if ( Service::initialize().isFailure() ) {
47 warning() << "Base class could not be initialized" << endmsg;
49 }
50
52 error() << "Unable to retrieve ThreadInitTools Array" << endmsg;
53
55 }
56
57 if ( m_threadInitTools.size() != 0 )
58 info() << "retrieved " << m_threadInitTools.size() << " thread init tools" << endmsg;
59 else
60 info() << "no thread init tools attached" << endmsg;
61
63}
64
65//-----------------------------------------------------------------------------
66
68
69 if ( !m_init )
70 warning() << "Looks like the ThreadPoolSvc was created, but thread pool "
71 << "was never initialized" << endmsg;
72
74}
75
76//-----------------------------------------------------------------------------
77
78StatusCode ThreadPoolSvc::initPool( const int& poolSize, const int& maxParallelismExtra ) {
79
80 tbb::spin_mutex::scoped_lock lock( m_initMutex );
81
83
84 ON_DEBUG debug() << "ThreadPoolSvc::initPool() poolSize = " << poolSize << endmsg;
85 // There is a problem in the piece of the code below. if
86 // m_threadPoolSize is set to something negative which is < -1,
87 // algorithm below might not behave as expected. For the time being
88 // I've choosen to create the barrier with the default number of
89 // threads created by the task scheduler init assuming that a
90 // negative value will choose automatic thread creation which will
91 // create default number of threads.
92 // SK
93
94 // -100 prevents the creation of the pool and the scheduler directly
95 // executes the tasks.
96 // -1 means use all available hardware threads
97
98 if ( -100 != m_threadPoolSize ) {
99 ON_DEBUG debug() << "Initialising a thread pool of size " << m_threadPoolSize << endmsg;
100
101 if ( m_threadPoolSize == -1 ) {
102 m_threadPoolSize = std::thread::hardware_concurrency();
103 } else if ( m_threadPoolSize < -1 ) {
104 fatal() << "Unexpected ThreadPoolSize \"" << m_threadPoolSize << "\". Allowed negative values are "
105 << "-1 (use all available cores) and -100 (don't use a thread pool)" << endmsg;
106 return StatusCode::FAILURE;
107 }
108
109 ON_VERBOSE verbose() << "Maximum allowed parallelism before adjusting: "
110 << tbb::global_control::active_value( tbb::global_control::max_allowed_parallelism ) << endmsg;
111
112 // to get the number of threads we need, request one thread more to account for how TBB calculates
113 // its soft limit of the number of threads for the global thread pool
114 m_tbbgc = std::make_unique<tbb::global_control>( tbb::global_control::max_allowed_parallelism,
115 m_threadPoolSize + maxParallelismExtra + 1 );
116
118 // Create the task arena to run all algorithms
119 m_arena.initialize( m_threadPoolSize + 1 );
120
121 // Create the barrier for task synchronization at termination
122 // (here we increase the number of threads by one to account for calling thread)
123 m_barrier = std::make_unique<std::barrier<>>( m_threadPoolSize + 1 );
124
125 } else {
126 // don't use a thread pool
128 m_tbbgc = std::make_unique<tbb::global_control>( tbb::global_control::max_allowed_parallelism, 0 );
129 }
130
131 ON_DEBUG debug() << "Thread Pool initialization complete. Maximum allowed parallelism: "
132 << tbb::global_control::active_value( tbb::global_control::max_allowed_parallelism ) << endmsg;
133
134 m_init = true;
135
136 return StatusCode::SUCCESS;
137}
138
139//-----------------------------------------------------------------------------
140
142 tbb::spin_mutex::scoped_lock lock( m_initMutex );
143
144 ON_DEBUG debug() << "ThreadPoolSvc::terminatePool()" << endmsg;
145
146 if ( !m_init ) {
147 error() << "Trying to terminate uninitialized thread pool!" << endmsg;
148 return StatusCode::FAILURE;
149 }
150
151 // Launch the termination tasks
152 const bool terminate = true;
153 if ( launchTasks( terminate ).isFailure() ) return StatusCode::FAILURE;
154
155 ON_DEBUG debug() << "Thread pool termination complete!" << endmsg;
156
157 return StatusCode::SUCCESS;
158}
159
160//-----------------------------------------------------------------------------
161
163
164 const std::string taskType = terminate ? "termination" : "initialization";
165
166 // If we have a thread pool (via a scheduler), then we want to queue
167 // the tasks in TBB to execute on each thread.
168 if ( tbb::global_control::active_value( tbb::global_control::max_allowed_parallelism ) > 0 ) {
169
170 // Make a warning message if not all threads can be finalised
172 warning() << "Finalising " << m_threadPoolSize << " threads, but " << m_threadInitCount.load()
173 << " threads were initialised" << endmsg;
174 }
175
176 // Create one task for each worker thread in the pool
177 for ( int i = 0; i < m_threadPoolSize; ++i ) {
178 ON_DEBUG debug() << "creating ThreadInitTask " << i << endmsg;
179
180 // Queue the task
184 }
185
186 // Now wait for all the workers to reach the barrier
187 ON_DEBUG debug() << "waiting at barrier for all ThreadInitTool to finish executing" << endmsg;
188 m_barrier->arrive_and_wait();
189
190 // Check to make sure all Tools were invoked.
191 // I'm not sure this mechanism is worthwhile.
192 for ( auto& t : m_threadInitTools ) {
193 // Number of threads initialized but not terminated.
194 int numInit = t->nInit();
195 // Expected number based on the type of task.
196 int expectedNumInit = terminate ? 0 : m_threadPoolSize;
197 if ( numInit != expectedNumInit ) {
198 std::ostringstream ost;
199 ost << "not all threads " << ( terminate ? "terminated" : "initialized" ) << " for tool " << t << " : "
200 << t->nInit() << " out of " << m_threadPoolSize << " are currently active.";
201 if ( terminate ) {
202 // it is likely the case that TBB activated new threads
203 // late in the game, and extra initializations were done
204 info() << ost.str() << endmsg;
205 } else {
206 error() << ost.str() << endmsg;
207 return StatusCode::FAILURE;
208 }
209 }
210 }
211
212 }
213
214 // In single-threaded mode, there is no scheduler, so we simply call
215 // the task wrapper directly in this thread.
216 else {
217 ON_DEBUG debug() << "launching ThreadInitTask " << taskType << "in this thread." << endmsg;
218
219 std::barrier<>* noBarrier = nullptr;
221 }
222
223 // Now, we do some error checking
225 error() << "a ThreadInitTask failed to execute successfully" << endmsg;
226 return StatusCode::FAILURE;
227 }
228
229 return StatusCode::SUCCESS;
230}
231
232//-----------------------------------------------------------------------------
233
234// tbb will actually create more threads than requested, and will sometimes
235// activate them late. This method is used to initialize one of these threads
236// when it is detected
237
239
241 // this should never happen
242 error() << "initThisThread triggered, but thread already initialized" << endmsg;
243 throw GaudiException( "initThisThread triggered, but thread already initialized", name(), StatusCode::FAILURE );
244 }
245
247 std::barrier<>* noBarrier = nullptr;
248 ThreadInitTask( m_threadInitTools, noBarrier, serviceLocator(), false )();
249}
#define ON_VERBOSE
MsgStream & endmsg(MsgStream &s)
MsgStream Modifier: endmsg. Calls the output method of the MsgStream.
Definition MsgStream.h:198
#define ON_DEBUG
#define DECLARE_COMPONENT(type)
duration< int64_t, milli > milliseconds
void lock(_L1 &__l1, _L2 &__l2, _L3 &... __l3)
void sleep_for(const chrono::duration< _Rep, _Period > &__rtime)
__string_type str() const
constexpr pointer get() const noexcept
MsgStream & error() const
shortcut for the method msgStream(MSG::ERROR)
MsgStream & verbose() const
shortcut for the method msgStream(MSG::VERBOSE)
MsgStream & warning() const
shortcut for the method msgStream(MSG::WARNING)
MsgStream & fatal() const
shortcut for the method msgStream(MSG::FATAL)
MsgStream & debug() const
shortcut for the method msgStream(MSG::DEBUG)
MsgStream & info() const
shortcut for the method msgStream(MSG::INFO)
static void setNumThreads(const std::size_t &nT)
Set the number of worker threads (for MT).
GaudiException(std::string Message, std::string Tag, StatusCode Code)
Constructor (1).
size_type size() const
StatusCode retrieve() override
Retrieve all tools.
bool empty() const override
Return whether the list of tools is empty.
The ISvcLocator is the interface implemented by the Service Factory in the Application Manager to loc...
Definition ISvcLocator.h:42
SmartIF< ISvcLocator > & serviceLocator() const override
Retrieve pointer to service locator.
Definition Service.cpp:336
const std::string & name() const override
Retrieve name of the service.
Definition Service.cpp:333
StatusCode terminate() override
Definition Service.h:54
Gaudi::Details::PropertyBase * declareProperty(const std::string &name, ToolHandle< T > &hndl, const std::string &doc="none")
Definition Service.h:91
StatusCode initialize() override
Definition Service.cpp:118
This class is used for returning status codes from appropriate routines.
Definition StatusCode.h:64
bool isFailure() const
Definition StatusCode.h:118
constexpr static const auto SUCCESS
Definition StatusCode.h:99
constexpr static const auto FAILURE
Definition StatusCode.h:100
static bool execFailed()
ThreadInitTask(ToolHandleArray< IThreadInitTool > &tools, std::barrier<> *b, ISvcLocator *svcLocator, bool terminate=false)
A service which initializes a TBB thread pool.
virtual void initThisThread() override
StatusCode initPool(const int &poolSize, const int &maxParallelismExtra) override final
Initialize the thread pool and launch the ThreadInitTasks.
StatusCode launchTasks(bool finalize=false)
Launch tasks to execute the ThreadInitTools.
std::unique_ptr< tbb::global_control > m_tbbgc
TBB global control parameter.
std::atomic< int > m_threadInitCount
Counter for all threads that are initialised.
bool m_init
Was the thread pool initialized?
std::unique_ptr< std::barrier<> > m_barrier
Barrier used to synchronization thread init tasks.
int m_threadPoolSize
Size of the thread pool allocated.
ToolHandleArray< IThreadInitTool > m_threadInitTools
Handle array of thread init tools.
StatusCode terminatePool() override final
Terminate the thread pool and launch thread termination tasks.
int poolSize() const override final
tbb::task_arena m_arena
TBB task arena to run all algorithms.
StatusCode finalize() override final
Finalise.
tbb::spin_mutex m_initMutex
Mutex used to protect the initPool and terminatePool methods.
StatusCode initialize() override final
Initialise.
ThreadPoolSvc(const std::string &name, ISvcLocator *svc)
Constructor.
Base class used to extend a class implementing other interfaces.
Definition extends.h:19
This file provides a Grammar for the type Gaudi::Accumulators::Axis It allows to use that type from p...
Definition __init__.py:1
STL namespace.