A service which initializes a TBB thread pool. More...

#include </builds/gaudi/Gaudi/GaudiHive/src/ThreadPoolSvc.h>

Inheritance diagram for ThreadPoolSvc:

[legend]

Collaboration diagram for ThreadPoolSvc:

[legend]

Public Member Functions
	ThreadPoolSvc (const std::string &name, ISvcLocator *svc)
	Constructor.

StatusCode	initialize () override final
	Initialise.

StatusCode	finalize () override final
	Finalise.

StatusCode	initPool (const int &poolSize, const int &maxParallelismExtra) override final
	Initialize the thread pool and launch the ThreadInitTasks.

StatusCode	terminatePool () override final
	Terminate the thread pool and launch thread termination tasks.

int	poolSize () const override final

virtual bool	isInit () const

virtual void	initThisThread () override

tbb::task_arena *	getArena ()

Public Member Functions inherited from extends< Service, IThreadPoolSvc >
void const *	i_cast (const InterfaceID &tid) const override
	Implementation of IInterface::i_cast.

StatusCode	queryInterface (const InterfaceID &ti, void **pp) override
	Implementation of IInterface::queryInterface.

std::vector< std::string >	getInterfaceNames () const override
	Implementation of IInterface::getInterfaceNames.

Public Member Functions inherited from Service
const std::string &	name () const override
	Retrieve name of the service.

StatusCode	configure () override

StatusCode	initialize () override

StatusCode	start () override

StatusCode	stop () override

StatusCode	finalize () override

StatusCode	terminate () override

Gaudi::StateMachine::State	FSMState () const override

Gaudi::StateMachine::State	targetFSMState () const override

StatusCode	reinitialize () override

StatusCode	restart () override

StatusCode	sysInitialize () override
	Initialize Service.

StatusCode	sysStart () override
	Initialize Service.

StatusCode	sysStop () override
	Initialize Service.

StatusCode	sysFinalize () override
	Finalize Service.

StatusCode	sysReinitialize () override
	Re-initialize the Service.

StatusCode	sysRestart () override
	Re-initialize the Service.

	Service (std::string name, ISvcLocator *svcloc)
	Standard Constructor.

SmartIF< ISvcLocator > &	serviceLocator () const override
	Retrieve pointer to service locator.

template<typename IFace = IService>
SmartIF< IFace >	service (const std::string &name, bool createIf=true) const

template<class T>
Gaudi::Details::PropertyBase *	declareProperty (const std::string &name, ToolHandle< T > &hndl, const std::string &doc="none")

template<class T>
StatusCode	declareTool (ToolHandle< T > &handle, bool createIf=true)

template<class T>
StatusCode	declareTool (ToolHandle< T > &handle, const std::string &toolTypeAndName, bool createIf=true)
	Declare used tool.

template<class T>
Gaudi::Details::PropertyBase *	declareProperty (const std::string &name, ToolHandleArray< T > &hndlArr, const std::string &doc="none")

template<class T>
void	addToolsArray (ToolHandleArray< T > &hndlArr)

const std::vector< IAlgTool * > &	tools () const

SmartIF< IAuditorSvc > &	auditorSvc () const
	The standard auditor service.May not be invoked before sysInitialize() has been invoked.

Public Member Functions inherited from PropertyHolder< CommonMessaging< implements< IService, IProperty, IStateful > > >
StatusCode	setProperty (const Gaudi::Details::PropertyBase &p)
	Set the property from a property.

StatusCode	setProperty (const std::string &name, const char *v)
	Special case for string literals.

StatusCode	setProperty (const std::string &name, const std::string &v)
	Special case for std::string.

StatusCode	setProperty (const std::string &name, const TYPE &value)
	set the property form the value

	PropertyHolder ()=default

Gaudi::Details::PropertyBase &	declareProperty (Gaudi::Details::PropertyBase &prop)
	Declare a property.

Gaudi::Details::PropertyBase *	declareProperty (const std::string &name, TYPE &value, const std::string &doc="none")
	Helper to wrap a regular data member and use it as a regular property.

Gaudi::Details::PropertyBase *	declareProperty (const std::string &name, Gaudi::Property< TYPE, VERIFIER, HANDLERS > &prop, const std::string &doc="none")
	Declare a PropertyBase instance setting name and documentation.

Gaudi::Details::PropertyBase *	declareRemoteProperty (const std::string &name, IProperty *rsvc, const std::string &rname="")
	Declare a remote property.

StatusCode	setProperty (const std::string &name, const Gaudi::Details::PropertyBase &p) override
	set the property from another property with a different name

StatusCode	setProperty (const std::string &s) override
	set the property from the formatted string

StatusCode	setProperty (const Gaudi::Details::PropertyBase &p)
	Set the property from a property.

StatusCode	setProperty (const std::string &name, const char *v)
	Special case for string literals.

StatusCode	setProperty (const std::string &name, const std::string &v)
	Special case for std::string.

StatusCode	setProperty (const std::string &name, const TYPE &value)
	set the property form the value

StatusCode	setPropertyRepr (const std::string &n, const std::string &r) override
	set the property from name and value string representation

StatusCode	getProperty (Gaudi::Details::PropertyBase *p) const override
	get the property

const Gaudi::Details::PropertyBase &	getProperty (std::string_view name) const override
	get the property by name

StatusCode	getProperty (std::string_view n, std::string &v) const override
	convert the property to the string

const std::vector< Gaudi::Details::PropertyBase * > &	getProperties () const override
	get all properties

bool	hasProperty (std::string_view name) const override
	Return true if we have a property with the given name.

Gaudi::Details::PropertyBase *	property (std::string_view name) const
	\fixme property and bindPropertiesTo should be protected

void	bindPropertiesTo (Gaudi::Interfaces::IOptionsSvc &optsSvc)

	PropertyHolder (const PropertyHolder &)=delete

PropertyHolder &	operator= (const PropertyHolder &)=delete

Public Member Functions inherited from CommonMessaging< implements< IService, IProperty, IStateful > >
MSG::Level	msgLevel () const
	get the cached level (originally extracted from the embedded MsgStream)

bool	msgLevel (MSG::Level lvl) const
	get the output level from the embedded MsgStream

Public Member Functions inherited from CommonMessagingBase
virtual	~CommonMessagingBase ()=default
	Virtual destructor.

const SmartIF< IMessageSvc > &	msgSvc () const
	The standard message service.

MsgStream &	msgStream () const
	Return an uninitialized MsgStream.

MsgStream &	msgStream (const MSG::Level level) const
	Predefined configurable message stream for the efficient printouts.

MsgStream &	always () const
	shortcut for the method msgStream(MSG::ALWAYS)

MsgStream &	fatal () const
	shortcut for the method msgStream(MSG::FATAL)

MsgStream &	err () const
	shortcut for the method msgStream(MSG::ERROR)

MsgStream &	error () const
	shortcut for the method msgStream(MSG::ERROR)

MsgStream &	warning () const
	shortcut for the method msgStream(MSG::WARNING)

MsgStream &	info () const
	shortcut for the method msgStream(MSG::INFO)

MsgStream &	debug () const
	shortcut for the method msgStream(MSG::DEBUG)

MsgStream &	verbose () const
	shortcut for the method msgStream(MSG::VERBOSE)

MsgStream &	msg () const
	shortcut for the method msgStream(MSG::INFO)

Private Member Functions
StatusCode	launchTasks (bool finalize=false)
	Launch tasks to execute the ThreadInitTools.

Private Attributes
ToolHandleArray< IThreadInitTool >	m_threadInitTools = { this }
	Handle array of thread init tools.

bool	m_init = false
	Was the thread pool initialized?

int	m_threadPoolSize = 0
	Size of the thread pool allocated.

tbb::spin_mutex	m_initMutex
	Mutex used to protect the initPool and terminatePool methods.

std::unique_ptr< boost::barrier >	m_barrier
	Barrier used to synchronization thread init tasks.

std::unique_ptr< tbb::global_control >	m_tbbgc
	TBB global control parameter.

tbb::task_arena	m_arena
	TBB task arena to run all algorithms.

std::atomic< int >	m_threadInitCount = 0
	Counter for all threads that are initialised.

Additional Inherited Members
Public Types inherited from extends< Service, IThreadPoolSvc >
using	base_class
	Typedef to this class.

using	extend_interfaces_base
	Typedef to the base of this class.

Public Types inherited from Service
using	Factory = Gaudi::PluginService::Factory<IService( const std::string&, ISvcLocator )>

Public Types inherited from PropertyHolder< CommonMessaging< implements< IService, IProperty, IStateful > > >
using	PropertyHolderImpl
	Typedef used to refer to this class from derived classes, as in.

Public Types inherited from CommonMessaging< implements< IService, IProperty, IStateful > >
using	base_class

Public Types inherited from extend_interfaces< Interfaces... >
using	ext_iids
	take union of the ext_iids of all Interfaces...

Protected Member Functions inherited from Service
std::vector< IAlgTool * > &	tools ()

	~Service () override

int	outputLevel () const
	get the `Service's` output level

Protected Member Functions inherited from CommonMessaging< implements< IService, IProperty, IStateful > >
MSG::Level	setUpMessaging () const
	Set up local caches.

MSG::Level	resetMessaging ()
	Reinitialize internal states.

void	updateMsgStreamOutputLevel (int level)
	Update the output level of the cached MsgStream.

Protected Attributes inherited from Service
Gaudi::StateMachine::State	m_state = Gaudi::StateMachine::OFFLINE
	Service state.

Gaudi::StateMachine::State	m_targetState = Gaudi::StateMachine::OFFLINE
	Service state.

Gaudi::Property< int >	m_outputLevel { this, "OutputLevel", MSG::NIL, "output level" }
	flag indicating whether ToolHandle tools have been added to m_tools

Gaudi::Property< bool >	m_auditorInitialize { this, "AuditInitialize", false, "trigger auditor on initialize()" }

Gaudi::Property< bool >	m_auditorStart { this, "AuditStart", false, "trigger auditor on start()" }

Gaudi::Property< bool >	m_auditorStop { this, "AuditStop", false, "trigger auditor on stop()" }

Gaudi::Property< bool >	m_auditorFinalize { this, "AuditFinalize", false, "trigger auditor on finalize()" }

Gaudi::Property< bool >	m_auditorReinitialize { this, "AuditReinitialize", false, "trigger auditor on reinitialize()" }

Gaudi::Property< bool >	m_auditorRestart { this, "AuditRestart", false, "trigger auditor on restart()" }

Gaudi::Property< bool >	m_autoRetrieveTools

Gaudi::Property< bool >	m_checkToolDeps

SmartIF< IAuditorSvc >	m_pAuditorSvc
	Auditor Service.

Detailed Description

A service which initializes a TBB thread pool.

This service can be configured with an array of IThreadInitTools which will each be invoked concurrently on each worker thread. A ThreadInitTask is created for each thread and given the list of tools. A boost::barrier is used to synchronize the calling of each tool concurrently on all threads at the same time.

Definition at line 37 of file ThreadPoolSvc.h.

Constructor & Destructor Documentation

◆ ThreadPoolSvc()

ThreadPoolSvc::ThreadPoolSvc	(	const std::string &	name,
		ISvcLocator *	svc )

Constructor.

Definition at line 32 of file ThreadPoolSvc.cpp.

                                                                         : extends( name, svcLoc ) {
  declareProperty( "ThreadInitTools", m_threadInitTools, "ToolHandleArray of IThreadInitTools" );
}

Member Function Documentation

◆ finalize()

StatusCode ThreadPoolSvc::finalize ( )

finaloverride

Finalise.

Definition at line 62 of file ThreadPoolSvc.cpp.

                                   {
 
  if ( !m_init )
    warning() << "Looks like the ThreadPoolSvc was created, but thread pool "
              << "was never initialized" << endmsg;
 
  return StatusCode::SUCCESS;
}

◆ getArena()

tbb::task_arena * ThreadPoolSvc::getArena ( )

inline

Definition at line 60 of file ThreadPoolSvc.h.

60{ return &m_arena; }

ThreadPoolSvc::m_arena

tbb::task_arena m_arena

TBB task arena to run all algorithms.

Definition ThreadPoolSvc.h:85

◆ initialize()

StatusCode ThreadPoolSvc::initialize ( )

finaloverride

Initialise.

Definition at line 38 of file ThreadPoolSvc.cpp.

                                     {
 
  // Initialise mother class (read properties, ...)
  if ( Service::initialize().isFailure() ) {
    warning() << "Base class could not be initialized" << endmsg;
    return StatusCode::FAILURE;
  }
 
  if ( m_threadInitTools.retrieve().isFailure() ) {
    error() << "Unable to retrieve ThreadInitTools Array" << endmsg;
 
    return StatusCode::FAILURE;
  }
 
  if ( m_threadInitTools.size() != 0 )
    info() << "retrieved " << m_threadInitTools.size() << " thread init tools" << endmsg;
  else
    info() << "no thread init tools attached" << endmsg;
 
  return StatusCode::SUCCESS;
}

◆ initPool()

StatusCode ThreadPoolSvc::initPool	(	const int &	poolSize,
		const int &	maxParallelismExtra )

finaloverride

Initialize the thread pool and launch the ThreadInitTasks.

Definition at line 73 of file ThreadPoolSvc.cpp.

                                                                                        {
 
  tbb::spin_mutex::scoped_lock lock( m_initMutex );
 
  m_threadPoolSize = poolSize;
 
  ON_DEBUG debug() << "ThreadPoolSvc::initPool() poolSize = " << poolSize << endmsg;
  // There is a problem in the piece of the code below.  if
  // m_threadPoolSize is set to something negative which is < -1,
  // algorithm below might not behave as expected. For the time being
  // I've choosen to create the barrier with the default number of
  // threads created by the task scheduler init assuming that a
  // negative value will choose automatic thread creation which will
  // create default number of threads.
  // SK
 
  // -100 prevents the creation of the pool and the scheduler directly
  // executes the tasks.
  // -1 means use all available hardware threads
 
  if ( -100 != m_threadPoolSize ) {
    ON_DEBUG debug() << "Initialising a thread pool of size " << m_threadPoolSize << endmsg;
 
    if ( m_threadPoolSize == -1 ) {
      m_threadPoolSize = std::thread::hardware_concurrency();
    } else if ( m_threadPoolSize < -1 ) {
      fatal() << "Unexpected ThreadPoolSize \"" << m_threadPoolSize << "\". Allowed negative values are "
              << "-1 (use all available cores) and -100 (don't use a thread pool)" << endmsg;
      return StatusCode::FAILURE;
    }
 
    ON_VERBOSE verbose() << "Maximum allowed parallelism before adjusting: "
                         << tbb::global_control::active_value( tbb::global_control::max_allowed_parallelism ) << endmsg;
 
    // to get the number of threads we need, request one thread more to account for how TBB calculates
    // its soft limit of the number of threads for the global thread pool
    m_tbbgc = std::make_unique<tbb::global_control>( tbb::global_control::max_allowed_parallelism,
                                                     m_threadPoolSize + maxParallelismExtra + 1 );
 
    Gaudi::Concurrency::ConcurrencyFlags::setNumThreads( m_threadPoolSize );
 
    // Create the task arena to run all algorithms
    m_arena.initialize( m_threadPoolSize + 1 );
 
    // Create the barrier for task synchronization at termination
    // (here we increase the number of threads by one to account for calling thread)
    m_barrier = std::make_unique<boost::barrier>( m_threadPoolSize + 1 );
 
  } else {
    // don't use a thread pool
    Gaudi::Concurrency::ConcurrencyFlags::setNumThreads( 1 );
    m_tbbgc = std::make_unique<tbb::global_control>( tbb::global_control::max_allowed_parallelism, 0 );
  }
 
  ON_DEBUG debug() << "Thread Pool initialization complete. Maximum allowed parallelism: "
                   << tbb::global_control::active_value( tbb::global_control::max_allowed_parallelism ) << endmsg;
 
  m_init = true;
 
  return StatusCode::SUCCESS;
}

◆ initThisThread()

void ThreadPoolSvc::initThisThread ( )

overridevirtual

Definition at line 234 of file ThreadPoolSvc.cpp.

                                   {
 
  if ( Gaudi::Concurrency::ThreadInitDone ) {
    // this should never happen
    error() << "initThisThread triggered, but thread already initialized" << endmsg;
    throw GaudiException( "initThisThread triggered, but thread already initialized", name(), StatusCode::FAILURE );
  }
 
  m_threadInitCount++;
  boost::barrier* noBarrier = nullptr;
  ThreadInitTask( m_threadInitTools, noBarrier, serviceLocator(), false )();
}

◆ isInit()

virtual bool ThreadPoolSvc::isInit ( ) const

inlinevirtual

Definition at line 56 of file ThreadPoolSvc.h.

56{ return m_init; }

◆ launchTasks()

StatusCode ThreadPoolSvc::launchTasks ( bool finalize = false )

private

Launch tasks to execute the ThreadInitTools.

Definition at line 158 of file ThreadPoolSvc.cpp.

                                                      {
 
  const std::string taskType = terminate ? "termination" : "initialization";
 
  // If we have a thread pool (via a scheduler), then we want to queue
  // the tasks in TBB to execute on each thread.
  if ( tbb::global_control::active_value( tbb::global_control::max_allowed_parallelism ) > 0 ) {
 
    // Make a warning message if not all threads can be finalised
    if ( terminate and m_threadInitCount.load() != m_threadPoolSize and not m_threadInitTools.empty() ) {
      warning() << "Finalising " << m_threadPoolSize << " threads, but " << m_threadInitCount.load()
                << " threads were initialised" << endmsg;
    }
 
    // Create one task for each worker thread in the pool
    for ( int i = 0; i < m_threadPoolSize; ++i ) {
      ON_DEBUG debug() << "creating ThreadInitTask " << i << endmsg;
 
      // Queue the task
      if ( !terminate ) m_threadInitCount++;
      m_arena.enqueue( ThreadInitTask( m_threadInitTools, m_barrier.get(), serviceLocator(), terminate ) );
      std::this_thread::sleep_for( std::chrono::milliseconds( 20 ) );
    }
 
    // Now wait for all the workers to reach the barrier
    ON_DEBUG debug() << "waiting at barrier for all ThreadInitTool to finish executing" << endmsg;
    m_barrier->wait();
 
    // Check to make sure all Tools were invoked.
    // I'm not sure this mechanism is worthwhile.
    for ( auto& t : m_threadInitTools ) {
      // Number of threads initialized but not terminated.
      int numInit = t->nInit();
      // Expected number based on the type of task.
      int expectedNumInit = terminate ? 0 : m_threadPoolSize;
      if ( numInit != expectedNumInit ) {
        std::ostringstream ost;
        ost << "not all threads " << ( terminate ? "terminated" : "initialized" ) << " for tool " << t << " : "
            << t->nInit() << " out of " << m_threadPoolSize << " are currently active.";
        if ( terminate ) {
          // it is likely the case that TBB activated new threads
          // late in the game, and extra initializations were done
          info() << ost.str() << endmsg;
        } else {
          error() << ost.str() << endmsg;
          return StatusCode::FAILURE;
        }
      }
    }
 
  }
 
  // In single-threaded mode, there is no scheduler, so we simply call
  // the task wrapper directly in this thread.
  else {
    ON_DEBUG debug() << "launching ThreadInitTask " << taskType << "in this thread." << endmsg;
 
    boost::barrier* noBarrier = nullptr;
    ThreadInitTask( m_threadInitTools, noBarrier, serviceLocator(), terminate )();
  }
 
  // Now, we do some error checking
  if ( ThreadInitTask::execFailed() ) {
    error() << "a ThreadInitTask failed to execute successfully" << endmsg;
    return StatusCode::FAILURE;
  }
 
  return StatusCode::SUCCESS;
}

◆ poolSize()

int ThreadPoolSvc::poolSize ( ) const

inlinefinaloverride

Definition at line 54 of file ThreadPoolSvc.h.

54{ return m_threadPoolSize; }

◆ terminatePool()

StatusCode ThreadPoolSvc::terminatePool ( )

finaloverride

Terminate the thread pool and launch thread termination tasks.

Definition at line 137 of file ThreadPoolSvc.cpp.

                                        {
  tbb::spin_mutex::scoped_lock lock( m_initMutex );
 
  ON_DEBUG debug() << "ThreadPoolSvc::terminatePool()" << endmsg;
 
  if ( !m_init ) {
    error() << "Trying to terminate uninitialized thread pool!" << endmsg;
    return StatusCode::FAILURE;
  }
 
  // Launch the termination tasks
  const bool terminate = true;
  if ( launchTasks( terminate ).isFailure() ) return StatusCode::FAILURE;
 
  ON_DEBUG debug() << "Thread pool termination complete!" << endmsg;
 
  return StatusCode::SUCCESS;
}

Member Data Documentation

◆ m_arena

tbb::task_arena ThreadPoolSvc::m_arena

private

TBB task arena to run all algorithms.

Definition at line 85 of file ThreadPoolSvc.h.

◆ m_barrier

std::unique_ptr<boost::barrier> ThreadPoolSvc::m_barrier

private

Barrier used to synchronization thread init tasks.

Definition at line 79 of file ThreadPoolSvc.h.

◆ m_init

bool ThreadPoolSvc::m_init = false

private

Was the thread pool initialized?

Definition at line 70 of file ThreadPoolSvc.h.

◆ m_initMutex

tbb::spin_mutex ThreadPoolSvc::m_initMutex

private

Mutex used to protect the initPool and terminatePool methods.

Definition at line 76 of file ThreadPoolSvc.h.

◆ m_tbbgc

std::unique_ptr<tbb::global_control> ThreadPoolSvc::m_tbbgc

private

TBB global control parameter.

Definition at line 82 of file ThreadPoolSvc.h.

◆ m_threadInitCount

std::atomic<int> ThreadPoolSvc::m_threadInitCount = 0

private

Counter for all threads that are initialised.

Definition at line 88 of file ThreadPoolSvc.h.

◆ m_threadInitTools

ToolHandleArray<IThreadInitTool> ThreadPoolSvc::m_threadInitTools = { this }

private

Handle array of thread init tools.

Definition at line 67 of file ThreadPoolSvc.h.

67{ this };

◆ m_threadPoolSize

int ThreadPoolSvc::m_threadPoolSize = 0

private

Size of the thread pool allocated.

Definition at line 73 of file ThreadPoolSvc.h.

The documentation for this class was generated from the following files:

GaudiHive/src/ThreadPoolSvc.h
GaudiHive/src/ThreadPoolSvc.cpp

Public Member Functions

Private Member Functions

Private Attributes

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

◆ ThreadPoolSvc()

Member Function Documentation

◆ finalize()

◆ getArena()

◆ initialize()

◆ initPool()

◆ initThisThread()

◆ isInit()

◆ launchTasks()

◆ poolSize()

◆ terminatePool()

Member Data Documentation

◆ m_arena

◆ m_barrier

◆ m_init

◆ m_initMutex

◆ m_tbbgc

◆ m_threadInitCount

◆ m_threadInitTools

◆ m_threadPoolSize