ThreadPoolSvc Class Reference

A service which initializes a TBB thread pool. More...

#include <src/ThreadPoolSvc.h>

Inheritance diagram for ThreadPoolSvc:

Collaboration diagram for ThreadPoolSvc:

Public Member Functions
	ThreadPoolSvc (const std::string &name, ISvcLocator *svc)
	Constructor. More...
StatusCode	initialize () override final
	Initialise. More...
StatusCode	finalize () override final
	Finalise. More...
StatusCode	initPool (const int &poolSize) override final
	Initialize the thread pool and launch the ThreadInitTasks. More...
StatusCode	terminatePool () override final
	Terminate the thread pool and launch thread termination tasks. More...
int	poolSize () const override final
virtual bool	isInit () const
Public Member Functions inherited from extends< Service, IThreadPoolSvc >
void *	i_cast (const InterfaceID &tid) const override
	Implementation of IInterface::i_cast. More...
StatusCode	queryInterface (const InterfaceID &ti, void **pp) override
	Implementation of IInterface::queryInterface. More...
std::vector< std::string >	getInterfaceNames () const override
	Implementation of IInterface::getInterfaceNames. More...
Public Member Functions inherited from Service
const std::string &	name () const override
	Retrieve name of the service. More...
StatusCode	configure () override
StatusCode	initialize () override
StatusCode	start () override
StatusCode	stop () override
StatusCode	finalize () override
StatusCode	terminate () override
Gaudi::StateMachine::State	FSMState () const override
Gaudi::StateMachine::State	targetFSMState () const override
StatusCode	reinitialize () override
StatusCode	restart () override
StatusCode	sysInitialize () override
	Initialize Service. More...
StatusCode	sysStart () override
	Initialize Service. More...
StatusCode	sysStop () override
	Initialize Service. More...
StatusCode	sysFinalize () override
	Finalize Service. More...
StatusCode	sysReinitialize () override
	Re-initialize the Service. More...
StatusCode	sysRestart () override
	Re-initialize the Service. More...
	Service (std::string name, ISvcLocator *svcloc)
	Standard Constructor. More...
SmartIF< ISvcLocator > &	serviceLocator () const override
	Retrieve pointer to service locator. More...
StatusCode	setProperties ()
	Method for setting declared properties to the values specified for the job. More...
template<class T >
StatusCode	service (const std::string &name, const T *&psvc, bool createIf=true) const
	Access a service by name, creating it if it doesn't already exist. More...
template<class T >
StatusCode	service (const std::string &name, T *&psvc, bool createIf=true) const
template<typename IFace = IService>
SmartIF< IFace >	service (const std::string &name, bool createIf=true) const
template<class T >
StatusCode	service (const std::string &svcType, const std::string &svcName, T *&psvc) const
	Access a service by name and type, creating it if it doesn't already exist. More...
template<class T >
StatusCode	declareTool (ToolHandle< T > &handle, std::string toolTypeAndName, bool createIf=true)
	Declare used tool. More...
SmartIF< IAuditorSvc > &	auditorSvc () const
	The standard auditor service.May not be invoked before sysInitialize() has been invoked. More...
Public Member Functions inherited from PropertyHolder< CommonMessaging< implements< IService, IProperty, IStateful > > >
	PropertyHolder ()=default
Gaudi::Details::PropertyBase &	declareProperty (Gaudi::Details::PropertyBase &prop)
	Declare a property. More...
Gaudi::Details::PropertyBase *	declareProperty (const std::string &name, TYPE &value, const std::string &doc="none")
	Helper to wrap a regular data member and use it as a regular property. More...
Gaudi::Details::PropertyBase *	declareProperty (const std::string &name, Gaudi::Property< TYPE, VERIFIER, HANDLERS > &prop, const std::string &doc="none")
	Declare a PropertyBase instance setting name and documentation. More...
Gaudi::Details::PropertyBase *	declareRemoteProperty (const std::string &name, IProperty *rsvc, const std::string &rname="")
	Declare a remote property. More...
StatusCode	setProperty (const Gaudi::Details::PropertyBase &p) override
	set the property form another property More...
StatusCode	setProperty (const std::string &s) override
	set the property from the formatted string More...
StatusCode	setProperty (const std::string &n, const std::string &v) override
	set the property from name and the value More...
StatusCode	setProperty (const std::string &name, const TYPE &value)
	set the property form the value More...
StatusCode	getProperty (Gaudi::Details::PropertyBase *p) const override
	get the property More...
const Gaudi::Details::PropertyBase &	getProperty (const std::string &name) const override
	get the property by name More...
StatusCode	getProperty (const std::string &n, std::string &v) const override
	convert the property to the string More...
const std::vector< Gaudi::Details::PropertyBase * > &	getProperties () const override
	get all properties More...
bool	hasProperty (const std::string &name) const override
	Return true if we have a property with the given name. More...
	PropertyHolder (const PropertyHolder &)=delete
PropertyHolder &	operator= (const PropertyHolder &)=delete
Public Member Functions inherited from CommonMessagingBase
virtual	~CommonMessagingBase ()=default
	Virtual destructor. More...
const SmartIF< IMessageSvc > &	msgSvc () const
	The standard message service. More...
MsgStream &	msgStream () const
	Return an uninitialized MsgStream. More...
MsgStream &	msgStream (const MSG::Level level) const
	Predefined configurable message stream for the efficient printouts. More...
MsgStream &	always () const
	shortcut for the method msgStream(MSG::ALWAYS) More...
MsgStream &	fatal () const
	shortcut for the method msgStream(MSG::FATAL) More...
MsgStream &	err () const
	shortcut for the method msgStream(MSG::ERROR) More...
MsgStream &	error () const
	shortcut for the method msgStream(MSG::ERROR) More...
MsgStream &	warning () const
	shortcut for the method msgStream(MSG::WARNING) More...
MsgStream &	info () const
	shortcut for the method msgStream(MSG::INFO) More...
MsgStream &	debug () const
	shortcut for the method msgStream(MSG::DEBUG) More...
MsgStream &	verbose () const
	shortcut for the method msgStream(MSG::VERBOSE) More...
MsgStream &	msg () const
	shortcut for the method msgStream(MSG::INFO) More...
MSG::Level	msgLevel () const
	get the cached level (originally extracted from the embedded MsgStream) More...
MSG::Level	outputLevel () const __attribute__((deprecated))
	Backward compatibility function for getting the output level. More...
bool	msgLevel (MSG::Level lvl) const
	get the output level from the embedded MsgStream More...

Private Member Functions
StatusCode	launchTasks (bool finalize=false)
	Launch tasks to execute the ThreadInitTools. More...

Private Attributes
ToolHandleArray< IThreadInitTool >	m_threadInitTools = {this}
	Handle array of thread init tools. More...
bool	m_init = false
	Was the thread pool initialized? More...
int	m_threadPoolSize = 0
	Size of the thread pool allocated. More...
tbb::spin_mutex	m_initMutex
	Mutex used to protect the initPool and terminatePool methods. More...
std::unique_ptr< tbb::task_scheduler_init >	m_tbbSchedInit
	TBB task scheduler initializer. More...
std::unique_ptr< boost::barrier >	m_barrier
	Barrier used to synchronization thread init tasks. More...

Additional Inherited Members
Public Types inherited from extends< Service, IThreadPoolSvc >
using	base_class = extends
	Typedef to this class. More...
using	extend_interfaces_base = extend_interfaces< Interfaces... >
	Typedef to the base of this class. More...
Public Types inherited from Service
typedef Gaudi::PluginService::Factory< IService *, const std::string &, ISvcLocator * >	Factory
Public Types inherited from PropertyHolder< CommonMessaging< implements< IService, IProperty, IStateful > > >
using	PropertyHolderImpl = PropertyHolder< CommonMessaging< implements< IService, IProperty, IStateful > > >
	Typedef used to refer to this class from derived classes, as in. More...
Public Types inherited from CommonMessaging< implements< IService, IProperty, IStateful > >
using	base_class = CommonMessaging
Public Types inherited from extend_interfaces< Interfaces... >
using	ext_iids = typename Gaudi::interface_list_cat< typename Interfaces::ext_iids... >::type
	take union of the ext_iids of all Interfaces... More...
Protected Member Functions inherited from Service
	~Service () override
	Standard Destructor. More...
int	outputLevel () const
	get the Service's output level More...
Protected Member Functions inherited from PropertyHolder< CommonMessaging< implements< IService, IProperty, IStateful > > >
Gaudi::Details::PropertyBase *	property (const std::string &name) const
Protected Member Functions inherited from CommonMessaging< implements< IService, IProperty, IStateful > >
MSG::Level	setUpMessaging ()
	Set up local caches. More...
MSG::Level	resetMessaging ()
	Reinitialize internal states. More...
void	updateMsgStreamOutputLevel (int level)
	Update the output level of the cached MsgStream. More...
Protected Attributes inherited from Service
Gaudi::StateMachine::State	m_state = Gaudi::StateMachine::OFFLINE
	Service state. More...
Gaudi::StateMachine::State	m_targetState = Gaudi::StateMachine::OFFLINE
	Service state. More...
Gaudi::Property< int >	m_outputLevel {this, "OutputLevel", MSG::NIL, "output level"}
Gaudi::Property< bool >	m_auditInit {this, "AuditServices", false, "[[deprecated]] unused"}
Gaudi::Property< bool >	m_auditorInitialize {this, "AuditInitialize", false, "trigger auditor on initialize()"}
Gaudi::Property< bool >	m_auditorStart {this, "AuditStart", false, "trigger auditor on start()"}
Gaudi::Property< bool >	m_auditorStop {this, "AuditStop", false, "trigger auditor on stop()"}
Gaudi::Property< bool >	m_auditorFinalize {this, "AuditFinalize", false, "trigger auditor on finalize()"}
Gaudi::Property< bool >	m_auditorReinitialize {this, "AuditReinitialize", false, "trigger auditor on reinitialize()"}
Gaudi::Property< bool >	m_auditorRestart {this, "AuditRestart", false, "trigger auditor on restart()"}
SmartIF< IAuditorSvc >	m_pAuditorSvc
	Auditor Service. More...

Detailed Description

A service which initializes a TBB thread pool.

This service can be configured with an array of IThreadInitTools which will each be invoked concurrently on each worker thread. A ThreadInitTask is created for each thread and given the list of tools. A boost::barrier is used to synchronize the calling of each tool concurrently on all threads at the same time.

Definition at line 26 of file ThreadPoolSvc.h.

Constructor & Destructor Documentation

ThreadPoolSvc::ThreadPoolSvc	(	const std::string &	name,
		ISvcLocator *	svc
	)

Constructor.

Definition at line 19 of file ThreadPoolSvc.cpp.

                                                                         : extends( name, svcLoc )
{
  declareProperty( "ThreadInitTools", m_threadInitTools, "ToolHandleArray of IThreadInitTools" );
}

Member Function Documentation

StatusCode ThreadPoolSvc::finalize ( )

finaloverride

Finalise.

Definition at line 52 of file ThreadPoolSvc.cpp.

{

  if ( !m_init ) {
    warning() << "Looks like the ThreadPoolSvc was created, but thread pool "
              << "was never initialized" << endmsg;
  }

  return StatusCode::SUCCESS;
}

StatusCode ThreadPoolSvc::initialize ( )

finaloverride

Initialise.

Definition at line 26 of file ThreadPoolSvc.cpp.

{

  // Initialise mother class (read properties, ...)
  StatusCode sc( Service::initialize() );
  if ( !sc.isSuccess() ) {
    warning() << "Base class could not be initialized" << endmsg;
    return StatusCode::FAILURE;
  }

  if ( m_threadInitTools.retrieve().isFailure() ) {
    error() << "Unable to retrieve ThreadInitTools Array" << endmsg;

    return StatusCode::FAILURE;
  }
  if ( m_threadInitTools.size() != 0 ) {
    info() << "retrieved " << m_threadInitTools.size() << " thread init tools" << endmsg;
  } else {
    info() << "no thread init tools attached" << endmsg;
  }

  return StatusCode::SUCCESS;
}

StatusCode ThreadPoolSvc::initPool ( const int &  poolSize )

finaloverride

Initialize the thread pool and launch the ThreadInitTasks.

Definition at line 65 of file ThreadPoolSvc.cpp.

{

  tbb::spin_mutex::scoped_lock lock( m_initMutex );

  m_threadPoolSize = poolSize;

  if ( msgLevel( MSG::DEBUG ) ) debug() << "ThreadPoolSvc::initPool() poolSize = " << poolSize << endmsg;
  // There is a problem in the piece of the code below.  if
  // m_threadPoolSize is set to something negative which is < -1,
  // algorithm below might not behave as expected. For the time being
  // I've choosen to create the barrier with the default number of
  // threads created by the task scheduler init assuming that a
  // negative value will choose automatic thread creation which will
  // create default number of threads.
  // SK

  // -100 prevents the creation of the pool and the scheduler directly
  // executes the tasks.
  if ( -100 != m_threadPoolSize ) {
    if ( msgLevel( MSG::DEBUG ) ) debug() << "Initialising a thread pool of size " << m_threadPoolSize << endmsg;

    // Leave -1 in case selected, increment otherwise
    // - What?
    int thePoolSize = m_threadPoolSize;
    if ( thePoolSize != -1 ) thePoolSize += 1;

    // Create the TBB task scheduler
    m_tbbSchedInit = std::make_unique<tbb::task_scheduler_init>( thePoolSize );
    // Create the barrier for task synchronization
    if ( m_threadPoolSize <= -1 ) thePoolSize = m_tbbSchedInit->default_num_threads();
    if ( msgLevel( MSG::DEBUG ) ) {
      debug() << "creating barrier of size " << thePoolSize << endmsg;
    }
    Gaudi::Concurrency::ConcurrencyFlags::setNumThreads( thePoolSize );

    m_barrier = std::make_unique<boost::barrier>( thePoolSize );

  } else {
    Gaudi::Concurrency::ConcurrencyFlags::setNumThreads( 1 );
  }

  // Launch the init tool tasks
  const bool terminate = false;
  if ( launchTasks( terminate ).isFailure() ) return StatusCode::FAILURE;

  if ( msgLevel( MSG::DEBUG ) ) debug() << "Thread Pool initialization complete!" << endmsg;

  m_init = true;

  return StatusCode::SUCCESS;
}

virtual bool ThreadPoolSvc::isInit ( ) const

inlinevirtual

Definition at line 46 of file ThreadPoolSvc.h.

{ return m_init; }

StatusCode ThreadPoolSvc::launchTasks ( bool  finalize = false )

private

Launch tasks to execute the ThreadInitTools.

Definition at line 141 of file ThreadPoolSvc.cpp.

{

  if ( m_threadInitTools.empty() ) return StatusCode::SUCCESS;

  const std::string taskType = terminate ? "termination" : "initialization";

  // If we have a thread pool (via a scheduler), then we want to queue
  // the tasks in TBB to execute on each thread.
  if ( m_tbbSchedInit ) {

    // Create one task for each worker thread in the pool
    for ( int i = 0; i < m_threadPoolSize; ++i ) {
      if ( msgLevel( MSG::DEBUG ) ) debug() << "creating ThreadInitTask " << i << endmsg;
      tbb::task* t = new ( tbb::task::allocate_root() )
          ThreadInitTask( m_threadInitTools, m_barrier.get(), serviceLocator(), terminate );

      // Queue the task
      tbb::task::enqueue( *t );
      this_tbb_thread::sleep( tbb::tick_count::interval_t( .1 ) );
    }

    // Now wait for all the workers to reach the barrier
    if ( msgLevel( MSG::DEBUG ) ) debug() << "waiting at barrier for all ThreadInitTool to finish executing" << endmsg;
    m_barrier->wait();

    // Check to make sure all Tools were invoked.
    // I'm not sure this mechanism is worthwhile.
    for ( auto& t : m_threadInitTools ) {
      // Number of threads initialized but not terminated.
      int numInit = t->nInit();
      // Expected number based on the type of task.
      int expectedNumInit = terminate ? 0 : m_threadPoolSize;
      if ( numInit != expectedNumInit ) {
        error() << "not all threads " << ( terminate ? "terminated" : "initialized" ) << " for tool " << t << " : "
                << t->nInit() << " out of " << m_threadPoolSize << " are currently active" << endmsg;
        return StatusCode::FAILURE;
      }
    }

  }

  // In single-threaded mode, there is no scheduler, so we simply call
  // the task wrapper directly in this thread.
  else {
    if ( msgLevel( MSG::DEBUG ) ) debug() << "launching ThreadInitTask " << taskType << "in this thread." << endmsg;
    boost::barrier* noBarrier = nullptr;
    ThreadInitTask theTask( m_threadInitTools, noBarrier, serviceLocator(), terminate );
    theTask.execute();
  }

  // Now, we do some error checking
  if ( ThreadInitTask::execFailed() ) {
    error() << "a ThreadInitTask failed to execute successfully" << endmsg;
    return StatusCode::FAILURE;
  }

  return StatusCode::SUCCESS;
}

int ThreadPoolSvc::poolSize ( ) const

inlinefinaloverride

Definition at line 44 of file ThreadPoolSvc.h.

{ return m_threadPoolSize; }

StatusCode ThreadPoolSvc::terminatePool ( )

finaloverride

Terminate the thread pool and launch thread termination tasks.

Definition at line 120 of file ThreadPoolSvc.cpp.

{
  tbb::spin_mutex::scoped_lock lock( m_initMutex );
  if ( msgLevel( MSG::DEBUG ) ) debug() << "ThreadPoolSvc::terminatePool()" << endmsg;

  if ( !m_init ) {
    error() << "Trying to terminate uninitialized thread pool!" << endmsg;
    return StatusCode::FAILURE;
  }

  // Launch the termination tasks
  const bool terminate = true;
  if ( launchTasks( terminate ).isFailure() ) return StatusCode::FAILURE;

  if ( msgLevel( MSG::DEBUG ) ) debug() << "Thread pool termination complete!" << endmsg;

  return StatusCode::SUCCESS;
}

Member Data Documentation

std::unique_ptr<boost::barrier> ThreadPoolSvc::m_barrier

private

Barrier used to synchronization thread init tasks.

Definition at line 68 of file ThreadPoolSvc.h.

bool ThreadPoolSvc::m_init = false

private

Was the thread pool initialized?

Definition at line 56 of file ThreadPoolSvc.h.

tbb::spin_mutex ThreadPoolSvc::m_initMutex

private

Mutex used to protect the initPool and terminatePool methods.

Definition at line 62 of file ThreadPoolSvc.h.

std::unique_ptr<tbb::task_scheduler_init> ThreadPoolSvc::m_tbbSchedInit

private

TBB task scheduler initializer.

Definition at line 65 of file ThreadPoolSvc.h.

ToolHandleArray<IThreadInitTool> ThreadPoolSvc::m_threadInitTools = {this}

private

Handle array of thread init tools.

Definition at line 53 of file ThreadPoolSvc.h.

int ThreadPoolSvc::m_threadPoolSize = 0

private

Size of the thread pool allocated.

Definition at line 59 of file ThreadPoolSvc.h.

---

The documentation for this class was generated from the following files:

• GaudiHive/src/ThreadPoolSvc.h
• GaudiHive/src/ThreadPoolSvc.cpp