A service which initializes a TBB thread pool. More...

#include <src/ThreadPoolSvc.h>

Inheritance diagram for ThreadPoolSvc:

[legend]

Collaboration diagram for ThreadPoolSvc:

[legend]

Public Member Functions
	ThreadPoolSvc (const std::string &name, ISvcLocator *svc)
	Constructor. More...

StatusCode	initialize () override final
	Initialise. More...

StatusCode	finalize () override final
	Finalise. More...

StatusCode	initPool (const int &poolSize) override final
	Initialize the thread pool and launch the ThreadInitTasks. More...

StatusCode	terminatePool () override final
	Terminate the thread pool and launch thread termination tasks. More...

int	poolSize () const override final

virtual bool	isInit () const

virtual void	initThisThread () override

Public Member Functions inherited from extends< Service, IThreadPoolSvc >
void *	i_cast (const InterfaceID &tid) const override
	Implementation of IInterface::i_cast. More...

StatusCode	queryInterface (const InterfaceID &ti, void **pp) override
	Implementation of IInterface::queryInterface. More...

std::vector< std::string >	getInterfaceNames () const override
	Implementation of IInterface::getInterfaceNames. More...

Public Member Functions inherited from Service
const std::string &	name () const override
	Retrieve name of the service. More...

StatusCode	configure () override

StatusCode	initialize () override

StatusCode	start () override

StatusCode	stop () override

StatusCode	finalize () override

StatusCode	terminate () override

Gaudi::StateMachine::State	FSMState () const override

Gaudi::StateMachine::State	targetFSMState () const override

StatusCode	reinitialize () override

StatusCode	restart () override

StatusCode	sysInitialize () override
	Initialize Service. More...

StatusCode	sysStart () override
	Initialize Service. More...

StatusCode	sysStop () override
	Initialize Service. More...

StatusCode	sysFinalize () override
	Finalize Service. More...

StatusCode	sysReinitialize () override
	Re-initialize the Service. More...

StatusCode	sysRestart () override
	Re-initialize the Service. More...

	Service (std::string name, ISvcLocator *svcloc)
	Standard Constructor. More...

SmartIF< ISvcLocator > &	serviceLocator () const override
	Retrieve pointer to service locator. More...

StatusCode	setProperties ()
	Method for setting declared properties to the values specified for the job. More...

template<class T >
StatusCode	service (const std::string &name, const T *&psvc, bool createIf=true) const
	Access a service by name, creating it if it doesn't already exist. More...

template<class T >
StatusCode	service (const std::string &name, T *&psvc, bool createIf=true) const

template<typename IFace = IService>
SmartIF< IFace >	service (const std::string &name, bool createIf=true) const

template<class T >
StatusCode	service (const std::string &svcType, const std::string &svcName, T *&psvc) const
	Access a service by name and type, creating it if it doesn't already exist. More...

template<class T >
StatusCode	declareTool (ToolHandle< T > &handle, std::string toolTypeAndName, bool createIf=true)
	Declare used tool. More...

SmartIF< IAuditorSvc > &	auditorSvc () const
	The standard auditor service.May not be invoked before sysInitialize() has been invoked. More...

Public Member Functions inherited from PropertyHolder< CommonMessaging< implements< IService, IProperty, IStateful > > >
	PropertyHolder ()=default

Gaudi::Details::PropertyBase &	declareProperty (Gaudi::Details::PropertyBase &prop)
	Declare a property. More...

Gaudi::Details::PropertyBase *	declareProperty (const std::string &name, TYPE &value, const std::string &doc="none")
	Helper to wrap a regular data member and use it as a regular property. More...

Gaudi::Details::PropertyBase *	declareProperty (const std::string &name, Gaudi::Property< TYPE, VERIFIER, HANDLERS > &prop, const std::string &doc="none")
	Declare a PropertyBase instance setting name and documentation. More...

Gaudi::Details::PropertyBase *	declareRemoteProperty (const std::string &name, IProperty *rsvc, const std::string &rname="")
	Declare a remote property. More...

StatusCode	setProperty (const Gaudi::Details::PropertyBase &p) override
	set the property form another property More...

StatusCode	setProperty (const std::string &s) override
	set the property from the formatted string More...

StatusCode	setProperty (const std::string &n, const std::string &v) override
	set the property from name and the value More...

StatusCode	setProperty (const std::string &name, const TYPE &value)
	set the property form the value More...

StatusCode	getProperty (Gaudi::Details::PropertyBase *p) const override
	get the property More...

const Gaudi::Details::PropertyBase &	getProperty (const std::string &name) const override
	get the property by name More...

StatusCode	getProperty (const std::string &n, std::string &v) const override
	convert the property to the string More...

const std::vector< Gaudi::Details::PropertyBase * > &	getProperties () const override
	get all properties More...

bool	hasProperty (const std::string &name) const override
	Return true if we have a property with the given name. More...

	PropertyHolder (const PropertyHolder &)=delete

PropertyHolder &	operator= (const PropertyHolder &)=delete

Public Member Functions inherited from CommonMessaging< implements< IService, IProperty, IStateful > >
MSG::Level	msgLevel () const
	get the cached level (originally extracted from the embedded MsgStream) More...

bool	msgLevel (MSG::Level lvl) const
	get the output level from the embedded MsgStream More...

Public Member Functions inherited from CommonMessagingBase
virtual	~CommonMessagingBase ()=default
	Virtual destructor. More...

const SmartIF< IMessageSvc > &	msgSvc () const
	The standard message service. More...

MsgStream &	msgStream () const
	Return an uninitialized MsgStream. More...

MsgStream &	msgStream (const MSG::Level level) const
	Predefined configurable message stream for the efficient printouts. More...

MsgStream &	always () const
	shortcut for the method msgStream(MSG::ALWAYS) More...

MsgStream &	fatal () const
	shortcut for the method msgStream(MSG::FATAL) More...

MsgStream &	err () const
	shortcut for the method msgStream(MSG::ERROR) More...

MsgStream &	error () const
	shortcut for the method msgStream(MSG::ERROR) More...

MsgStream &	warning () const
	shortcut for the method msgStream(MSG::WARNING) More...

MsgStream &	info () const
	shortcut for the method msgStream(MSG::INFO) More...

MsgStream &	debug () const
	shortcut for the method msgStream(MSG::DEBUG) More...

MsgStream &	verbose () const
	shortcut for the method msgStream(MSG::VERBOSE) More...

MsgStream &	msg () const
	shortcut for the method msgStream(MSG::INFO) More...

Private Member Functions
StatusCode	launchTasks (bool finalize=false)
	Launch tasks to execute the ThreadInitTools. More...

Private Attributes
ToolHandleArray< IThreadInitTool >	m_threadInitTools = {this}
	Handle array of thread init tools. More...

bool	m_init = false
	Was the thread pool initialized? More...

int	m_threadPoolSize = 0
	Size of the thread pool allocated. More...

tbb::spin_mutex	m_initMutex
	Mutex used to protect the initPool and terminatePool methods. More...

std::unique_ptr< tbb::task_scheduler_init >	m_tbbSchedInit
	TBB task scheduler initializer. More...

std::unique_ptr< boost::barrier >	m_barrier
	Barrier used to synchronization thread init tasks. More...

std::unique_ptr< tbb::global_control >	m_tbbgc
	TBB global control parameter. More...

Additional Inherited Members
Public Types inherited from extends< Service, IThreadPoolSvc >
using	base_class = extends
	Typedef to this class. More...

using	extend_interfaces_base = extend_interfaces< Interfaces... >
	Typedef to the base of this class. More...

Public Types inherited from Service
using	Factory = Gaudi::PluginService::Factory< IService (const std::string &, ISvcLocator )>

Public Types inherited from PropertyHolder< CommonMessaging< implements< IService, IProperty, IStateful > > >
using	PropertyHolderImpl = PropertyHolder< CommonMessaging< implements< IService, IProperty, IStateful > > >
	Typedef used to refer to this class from derived classes, as in. More...

Public Types inherited from CommonMessaging< implements< IService, IProperty, IStateful > >
using	base_class = CommonMessaging

Public Types inherited from extend_interfaces< Interfaces... >
using	ext_iids = typename Gaudi::interface_list_cat< typename Interfaces::ext_iids... >::type
	take union of the ext_iids of all Interfaces... More...

Protected Member Functions inherited from Service
	~Service () override
	Standard Destructor. More...

int	outputLevel () const
	get the `Service's` output level More...

Protected Member Functions inherited from PropertyHolder< CommonMessaging< implements< IService, IProperty, IStateful > > >
Gaudi::Details::PropertyBase *	property (const std::string &name) const

Protected Member Functions inherited from CommonMessaging< implements< IService, IProperty, IStateful > >
MSG::Level	setUpMessaging () const
	Set up local caches. More...

MSG::Level	resetMessaging ()
	Reinitialize internal states. More...

void	updateMsgStreamOutputLevel (int level)
	Update the output level of the cached MsgStream. More...

Protected Attributes inherited from Service
Gaudi::StateMachine::State	m_state = Gaudi::StateMachine::OFFLINE
	Service state. More...

Gaudi::StateMachine::State	m_targetState = Gaudi::StateMachine::OFFLINE
	Service state. More...

Gaudi::Property< int >	m_outputLevel {this, "OutputLevel", MSG::NIL, "output level"}

Gaudi::Property< bool >	m_auditInit {this, "AuditServices", false, " unused"}

Gaudi::Property< bool >	m_auditorInitialize {this, "AuditInitialize", false, "trigger auditor on initialize()"}

Gaudi::Property< bool >	m_auditorStart {this, "AuditStart", false, "trigger auditor on start()"}

Gaudi::Property< bool >	m_auditorStop {this, "AuditStop", false, "trigger auditor on stop()"}

Gaudi::Property< bool >	m_auditorFinalize {this, "AuditFinalize", false, "trigger auditor on finalize()"}

Gaudi::Property< bool >	m_auditorReinitialize {this, "AuditReinitialize", false, "trigger auditor on reinitialize()"}

Gaudi::Property< bool >	m_auditorRestart {this, "AuditRestart", false, "trigger auditor on restart()"}

SmartIF< IAuditorSvc >	m_pAuditorSvc
	Auditor Service. More...

Detailed Description

A service which initializes a TBB thread pool.

This service can be configured with an array of IThreadInitTools which will each be invoked concurrently on each worker thread. A ThreadInitTask is created for each thread and given the list of tools. A boost::barrier is used to synchronize the calling of each tool concurrently on all threads at the same time.

Definition at line 41 of file ThreadPoolSvc.h.

Constructor & Destructor Documentation

◆ ThreadPoolSvc()

ThreadPoolSvc::ThreadPoolSvc	(	const std::string &	name,
		ISvcLocator *	svc
	)

Constructor.

Definition at line 34 of file ThreadPoolSvc.cpp.

                                                                          : extends( name, svcLoc ) {
   declareProperty( "ThreadInitTools", m_threadInitTools, "ToolHandleArray of IThreadInitTools" );
 }

Member Function Documentation

◆ finalize()

StatusCode ThreadPoolSvc::finalize ( )

finaloverride

Finalise.

Definition at line 64 of file ThreadPoolSvc.cpp.

                                    {
 
   if ( !m_init )
     warning() << "Looks like the ThreadPoolSvc was created, but thread pool "
               << "was never initialized" << endmsg;
 
   return StatusCode::SUCCESS;
 }

◆ initialize()

StatusCode ThreadPoolSvc::initialize ( )

finaloverride

Initialise.

Definition at line 40 of file ThreadPoolSvc.cpp.

                                      {
 
   // Initialise mother class (read properties, ...)
   if ( Service::initialize().isFailure() ) {
     warning() << "Base class could not be initialized" << endmsg;
     return StatusCode::FAILURE;
   }
 
   if ( m_threadInitTools.retrieve().isFailure() ) {
     error() << "Unable to retrieve ThreadInitTools Array" << endmsg;
 
     return StatusCode::FAILURE;
   }
 
   if ( m_threadInitTools.size() != 0 )
     info() << "retrieved " << m_threadInitTools.size() << " thread init tools" << endmsg;
   else
     info() << "no thread init tools attached" << endmsg;
 
   return StatusCode::SUCCESS;
 }

◆ initPool()

StatusCode ThreadPoolSvc::initPool ( const int & poolSize )

finaloverride

Initialize the thread pool and launch the ThreadInitTasks.

Definition at line 75 of file ThreadPoolSvc.cpp.

                                                         {
 
   tbb::spin_mutex::scoped_lock lock( m_initMutex );
 
   m_threadPoolSize = poolSize;
 
   ON_DEBUG debug() << "ThreadPoolSvc::initPool() poolSize = " << poolSize << endmsg;
   // There is a problem in the piece of the code below.  if
   // m_threadPoolSize is set to something negative which is < -1,
   // algorithm below might not behave as expected. For the time being
   // I've choosen to create the barrier with the default number of
   // threads created by the task scheduler init assuming that a
   // negative value will choose automatic thread creation which will
   // create default number of threads.
   // SK
 
   // -100 prevents the creation of the pool and the scheduler directly
   // executes the tasks.
   // -1 means use all available hardware threads
 
   if ( -100 != m_threadPoolSize ) {
     ON_DEBUG debug() << "Initialising a thread pool of size " << m_threadPoolSize << endmsg;
 
     if ( m_threadPoolSize == -1 ) {
       m_threadPoolSize = std::thread::hardware_concurrency();
     } else if ( m_threadPoolSize < -1 ) {
       fatal() << "Unexpected ThreadPoolSize \"" << m_threadPoolSize << "\". Allowed negative values are "
               << "-1 (use all available cores) and -100 (don't use a thread pool)" << endmsg;
       return StatusCode::FAILURE;
     }
 
 #if TBB_INTERFACE_VERSION_MAJOR < 12
     m_tbbSchedInit = std::make_unique<tbb::task_scheduler_init>( m_threadPoolSize + 1 );
 #endif // TBB_INTERFACE_VERSION_MAJOR < 12
 
     ON_VERBOSE verbose() << "Maximum allowed parallelism before adjusting: "
                          << tbb::global_control::active_value( tbb::global_control::max_allowed_parallelism ) << endmsg;
 
     // to get the number of threads we need, request one thread more to account for how TBB calculates
     // its soft limit of the number of threads for the global thread pool
     m_tbbgc =
         std::make_unique<tbb::global_control>( tbb::global_control::max_allowed_parallelism, m_threadPoolSize + 1 );
 
     Gaudi::Concurrency::ConcurrencyFlags::setNumThreads( m_threadPoolSize );
 
     // Create the barrier for task synchronization at termination
     // (here we increase the number of threads by one to account for calling thread)
     m_barrier = std::make_unique<boost::barrier>( m_threadPoolSize + 1 );
 
   } else {
     // don't use a thread pool
     Gaudi::Concurrency::ConcurrencyFlags::setNumThreads( 1 );
     m_tbbgc = std::make_unique<tbb::global_control>( tbb::global_control::max_allowed_parallelism, 0 );
   }
 
   ON_DEBUG debug() << "Thread Pool initialization complete. Maximum allowed parallelism: "
                    << tbb::global_control::active_value( tbb::global_control::max_allowed_parallelism ) << endmsg;
 
   m_init = true;
 
   return StatusCode::SUCCESS;
 }

◆ initThisThread()

void GAUDI_NO_SANITIZE_ADDRESS ThreadPoolSvc::initThisThread ( )

overridevirtual

Definition at line 244 of file ThreadPoolSvc.cpp.

                                                              {
 
   if ( Gaudi::Concurrency::ThreadInitDone ) {
     // this should never happen
     error() << "initThisThread triggered, but thread already initialized" << endmsg;
     throw GaudiException( "initThisThread triggered, but thread already initialized", name(), StatusCode::FAILURE );
   }
 
   boost::barrier* noBarrier = nullptr;
   ThreadInitTask  theTask( m_threadInitTools, noBarrier, serviceLocator(), false );
   theTask.execute();
 }

◆ isInit()

virtual bool ThreadPoolSvc::isInit ( ) const

inlinevirtual

Definition at line 60 of file ThreadPoolSvc.h.

60 { return m_init; }

ThreadPoolSvc::m_init

bool m_init

Was the thread pool initialized?

Definition: ThreadPoolSvc.h:72

◆ launchTasks()

StatusCode ThreadPoolSvc::launchTasks ( bool finalize = false )

private

Launch tasks to execute the ThreadInitTools.

Definition at line 161 of file ThreadPoolSvc.cpp.

                                                       {
 
   const std::string taskType = terminate ? "termination" : "initialization";
 
   // If we have a thread pool (via a scheduler), then we want to queue
   // the tasks in TBB to execute on each thread.
   if ( tbb::global_control::active_value( tbb::global_control::max_allowed_parallelism ) > 0 ) {
 
     // Create one task for each worker thread in the pool
     for ( int i = 0; i < m_threadPoolSize; ++i ) {
 
       ON_DEBUG   debug() << "creating ThreadInitTask " << i << endmsg;
       tbb::task* t = new ( tbb::task::allocate_root() )
           ThreadInitTask( m_threadInitTools, m_barrier.get(), serviceLocator(), terminate );
 
       // Queue the task
       tbb::task::enqueue( *t );
       std::this_thread::sleep_for( std::chrono::milliseconds( 20 ) );
     }
 
     // Now wait for all the workers to reach the barrier
     ON_DEBUG debug() << "waiting at barrier for all ThreadInitTool to finish executing" << endmsg;
     m_barrier->wait();
 
     // Check to make sure all Tools were invoked.
     // I'm not sure this mechanism is worthwhile.
     for ( auto& t : m_threadInitTools ) {
       // Number of threads initialized but not terminated.
       int numInit = t->nInit();
       // Expected number based on the type of task.
       int expectedNumInit = terminate ? 0 : m_threadPoolSize;
       if ( numInit != expectedNumInit ) {
         std::ostringstream ost;
         ost << "not all threads " << ( terminate ? "terminated" : "initialized" ) << " for tool " << t << " : "
             << t->nInit() << " out of " << m_threadPoolSize << " are currently active.";
         if ( terminate ) {
           // it is likely the case that TBB activated new threads
           // late in the game, and extra initializations were done
           info() << ost.str() << endmsg;
         } else {
           error() << ost.str() << endmsg;
           return StatusCode::FAILURE;
         }
       }
     }
 
   }
 
   // In single-threaded mode, there is no scheduler, so we simply call
   // the task wrapper directly in this thread.
   else {
     ON_DEBUG debug() << "launching ThreadInitTask " << taskType << "in this thread." << endmsg;
 
     boost::barrier* noBarrier = nullptr;
     ThreadInitTask  theTask( m_threadInitTools, noBarrier, serviceLocator(), terminate );
     theTask.execute();
   }
 
   // Now, we do some error checking
   if ( ThreadInitTask::execFailed() ) {
     error() << "a ThreadInitTask failed to execute successfully" << endmsg;
     return StatusCode::FAILURE;
   }
 
   return StatusCode::SUCCESS;
 }

◆ poolSize()

int ThreadPoolSvc::poolSize ( ) const

inlinefinaloverride

Definition at line 58 of file ThreadPoolSvc.h.

58 { return m_threadPoolSize; }

ThreadPoolSvc::m_threadPoolSize

int m_threadPoolSize

Size of the thread pool allocated.

Definition: ThreadPoolSvc.h:75

◆ terminatePool()

StatusCode ThreadPoolSvc::terminatePool ( )

finaloverride

Terminate the thread pool and launch thread termination tasks.

Definition at line 140 of file ThreadPoolSvc.cpp.

                                         {
   tbb::spin_mutex::scoped_lock lock( m_initMutex );
 
   ON_DEBUG debug() << "ThreadPoolSvc::terminatePool()" << endmsg;
 
   if ( !m_init ) {
     error() << "Trying to terminate uninitialized thread pool!" << endmsg;
     return StatusCode::FAILURE;
   }
 
   // Launch the termination tasks
   const bool terminate = true;
   if ( launchTasks( terminate ).isFailure() ) return StatusCode::FAILURE;
 
   ON_DEBUG debug() << "Thread pool termination complete!" << endmsg;
 
   return StatusCode::SUCCESS;
 }

Member Data Documentation

◆ m_barrier

std::unique_ptr<boost::barrier> ThreadPoolSvc::m_barrier

private

Barrier used to synchronization thread init tasks.

Definition at line 86 of file ThreadPoolSvc.h.

◆ m_init

bool ThreadPoolSvc::m_init = false

private

Was the thread pool initialized?

Definition at line 72 of file ThreadPoolSvc.h.

◆ m_initMutex

tbb::spin_mutex ThreadPoolSvc::m_initMutex

private

Mutex used to protect the initPool and terminatePool methods.

Definition at line 78 of file ThreadPoolSvc.h.

◆ m_tbbgc

std::unique_ptr<tbb::global_control> ThreadPoolSvc::m_tbbgc

private

TBB global control parameter.

Definition at line 89 of file ThreadPoolSvc.h.

◆ m_tbbSchedInit

std::unique_ptr<tbb::task_scheduler_init> ThreadPoolSvc::m_tbbSchedInit

private

TBB task scheduler initializer.

Definition at line 82 of file ThreadPoolSvc.h.

◆ m_threadInitTools

ToolHandleArray<IThreadInitTool> ThreadPoolSvc::m_threadInitTools = {this}

private

Handle array of thread init tools.

Definition at line 69 of file ThreadPoolSvc.h.

◆ m_threadPoolSize

int ThreadPoolSvc::m_threadPoolSize = 0

private

Size of the thread pool allocated.

Definition at line 75 of file ThreadPoolSvc.h.

The documentation for this class was generated from the following files:

GaudiHive/src/ThreadPoolSvc.h
GaudiHive/src/ThreadPoolSvc.cpp

Public Member Functions

Private Member Functions

Private Attributes

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

◆ ThreadPoolSvc()

Member Function Documentation

◆ finalize()

◆ initialize()

◆ initPool()

◆ initThisThread()

◆ isInit()

◆ launchTasks()

◆ poolSize()

◆ terminatePool()

Member Data Documentation

◆ m_barrier

◆ m_init

◆ m_initMutex

◆ m_tbbgc

◆ m_tbbSchedInit

◆ m_threadInitTools

◆ m_threadPoolSize