HiveDataBrokerSvc Class Reference

#include <src/HiveDataBroker.h>

Inheritance diagram for HiveDataBrokerSvc:

Collaboration diagram for HiveDataBrokerSvc:

Classes
struct	AlgEntry

Public Member Functions
std::vector< Algorithm * >	algorithmsRequiredFor (const DataObjIDColl &requested, const std::vector< std::string > &stoppers={}) const override
std::vector< Algorithm * >	algorithmsRequiredFor (const Gaudi::Utils::TypeNameString &alg, const std::vector< std::string > &stoppers={}) const override
StatusCode	initialize () override
StatusCode	start () override
StatusCode	stop () override
StatusCode	finalize () override
Public Member Functions inherited from extends< Service, IDataBroker >
void *	i_cast (const InterfaceID &tid) const override
	Implementation of IInterface::i_cast. More...
StatusCode	queryInterface (const InterfaceID &ti, void **pp) override
	Implementation of IInterface::queryInterface. More...
std::vector< std::string >	getInterfaceNames () const override
	Implementation of IInterface::getInterfaceNames. More...
Public Member Functions inherited from Service
const std::string &	name () const override
	Retrieve name of the service. More...
StatusCode	configure () override
StatusCode	initialize () override
StatusCode	start () override
StatusCode	stop () override
StatusCode	finalize () override
StatusCode	terminate () override
Gaudi::StateMachine::State	FSMState () const override
Gaudi::StateMachine::State	targetFSMState () const override
StatusCode	reinitialize () override
StatusCode	restart () override
StatusCode	sysInitialize () override
	Initialize Service. More...
StatusCode	sysStart () override
	Initialize Service. More...
StatusCode	sysStop () override
	Initialize Service. More...
StatusCode	sysFinalize () override
	Finalize Service. More...
StatusCode	sysReinitialize () override
	Re-initialize the Service. More...
StatusCode	sysRestart () override
	Re-initialize the Service. More...
	Service (std::string name, ISvcLocator *svcloc)
	Standard Constructor. More...
SmartIF< ISvcLocator > &	serviceLocator () const override
	Retrieve pointer to service locator. More...
StatusCode	setProperties ()
	Method for setting declared properties to the values specified for the job. More...
template<class T >
StatusCode	service (const std::string &name, const T *&psvc, bool createIf=true) const
	Access a service by name, creating it if it doesn't already exist. More...
template<class T >
StatusCode	service (const std::string &name, T *&psvc, bool createIf=true) const
template<typename IFace = IService>
SmartIF< IFace >	service (const std::string &name, bool createIf=true) const
template<class T >
StatusCode	service (const std::string &svcType, const std::string &svcName, T *&psvc) const
	Access a service by name and type, creating it if it doesn't already exist. More...
template<class T >
StatusCode	declareTool (ToolHandle< T > &handle, std::string toolTypeAndName, bool createIf=true)
	Declare used tool. More...
SmartIF< IAuditorSvc > &	auditorSvc () const
	The standard auditor service.May not be invoked before sysInitialize() has been invoked. More...
Public Member Functions inherited from PropertyHolder< CommonMessaging< implements< IService, IProperty, IStateful > > >
	PropertyHolder ()=default
Gaudi::Details::PropertyBase &	declareProperty (Gaudi::Details::PropertyBase &prop)
	Declare a property. More...
Gaudi::Details::PropertyBase *	declareProperty (const std::string &name, TYPE &value, const std::string &doc="none")
	Helper to wrap a regular data member and use it as a regular property. More...
Gaudi::Details::PropertyBase *	declareProperty (const std::string &name, Gaudi::Property< TYPE, VERIFIER, HANDLERS > &prop, const std::string &doc="none")
	Declare a PropertyBase instance setting name and documentation. More...
Gaudi::Details::PropertyBase *	declareRemoteProperty (const std::string &name, IProperty *rsvc, const std::string &rname="")
	Declare a remote property. More...
StatusCode	setProperty (const Gaudi::Details::PropertyBase &p) override
	set the property form another property More...
StatusCode	setProperty (const std::string &s) override
	set the property from the formatted string More...
StatusCode	setProperty (const std::string &n, const std::string &v) override
	set the property from name and the value More...
StatusCode	setProperty (const std::string &name, const TYPE &value)
	set the property form the value More...
StatusCode	getProperty (Gaudi::Details::PropertyBase *p) const override
	get the property More...
const Gaudi::Details::PropertyBase &	getProperty (const std::string &name) const override
	get the property by name More...
StatusCode	getProperty (const std::string &n, std::string &v) const override
	convert the property to the string More...
const std::vector< Gaudi::Details::PropertyBase * > &	getProperties () const override
	get all properties More...
bool	hasProperty (const std::string &name) const override
	Return true if we have a property with the given name. More...
	PropertyHolder (const PropertyHolder &)=delete
PropertyHolder &	operator= (const PropertyHolder &)=delete
Public Member Functions inherited from CommonMessaging< implements< IService, IProperty, IStateful > >
MSG::Level	msgLevel () const
	get the cached level (originally extracted from the embedded MsgStream) More...
bool	msgLevel (MSG::Level lvl) const
	get the output level from the embedded MsgStream More...
MSG::Level	outputLevel () const
	Backward compatibility function for getting the output level. More...
Public Member Functions inherited from CommonMessagingBase
virtual	~CommonMessagingBase ()=default
	Virtual destructor. More...
const SmartIF< IMessageSvc > &	msgSvc () const
	The standard message service. More...
MsgStream &	msgStream () const
	Return an uninitialized MsgStream. More...
MsgStream &	msgStream (const MSG::Level level) const
	Predefined configurable message stream for the efficient printouts. More...
MsgStream &	always () const
	shortcut for the method msgStream(MSG::ALWAYS) More...
MsgStream &	fatal () const
	shortcut for the method msgStream(MSG::FATAL) More...
MsgStream &	err () const
	shortcut for the method msgStream(MSG::ERROR) More...
MsgStream &	error () const
	shortcut for the method msgStream(MSG::ERROR) More...
MsgStream &	warning () const
	shortcut for the method msgStream(MSG::WARNING) More...
MsgStream &	info () const
	shortcut for the method msgStream(MSG::INFO) More...
MsgStream &	debug () const
	shortcut for the method msgStream(MSG::DEBUG) More...
MsgStream &	verbose () const
	shortcut for the method msgStream(MSG::VERBOSE) More...
MsgStream &	msg () const
	shortcut for the method msgStream(MSG::INFO) More...

Private Member Functions
std::vector< AlgEntry >	instantiateAndInitializeAlgorithms (const std::vector< std::string > &names) const
std::map< DataObjID, AlgEntry * >	mapProducers (std::vector< AlgEntry > &algorithms) const

Private Attributes
Gaudi::Property< std::string >	m_dataLoader
Gaudi::Property< std::vector< std::string > >	m_producers
std::vector< AlgEntry >	m_algorithms
std::vector< AlgEntry >	m_cfnodes
std::map< DataObjID, AlgEntry * >	m_dependencies

Additional Inherited Members
Public Types inherited from extends< Service, IDataBroker >
using	base_class = extends
	Typedef to this class. More...
using	extend_interfaces_base = extend_interfaces< Interfaces... >
	Typedef to the base of this class. More...
Public Types inherited from Service
using	Factory = Gaudi::PluginService::Factory< IService *(const std::string &, ISvcLocator *)>
Public Types inherited from PropertyHolder< CommonMessaging< implements< IService, IProperty, IStateful > > >
using	PropertyHolderImpl = PropertyHolder< CommonMessaging< implements< IService, IProperty, IStateful > > >
	Typedef used to refer to this class from derived classes, as in. More...
Public Types inherited from CommonMessaging< implements< IService, IProperty, IStateful > >
using	base_class = CommonMessaging
Public Types inherited from extend_interfaces< Interfaces... >
using	ext_iids = typename Gaudi::interface_list_cat< typename Interfaces::ext_iids... >::type
	take union of the ext_iids of all Interfaces... More...
Protected Member Functions inherited from Service
	~Service () override
	Standard Destructor. More...
int	outputLevel () const
	get the Service's output level More...
Protected Member Functions inherited from PropertyHolder< CommonMessaging< implements< IService, IProperty, IStateful > > >
Gaudi::Details::PropertyBase *	property (const std::string &name) const
Protected Member Functions inherited from CommonMessaging< implements< IService, IProperty, IStateful > >
MSG::Level	setUpMessaging () const
	Set up local caches. More...
MSG::Level	resetMessaging ()
	Reinitialize internal states. More...
void	updateMsgStreamOutputLevel (int level)
	Update the output level of the cached MsgStream. More...
Protected Attributes inherited from Service
Gaudi::StateMachine::State	m_state = Gaudi::StateMachine::OFFLINE
	Service state. More...
Gaudi::StateMachine::State	m_targetState = Gaudi::StateMachine::OFFLINE
	Service state. More...
Gaudi::Property< int >	m_outputLevel {this, "OutputLevel", MSG::NIL, "output level"}
Gaudi::Property< bool >	m_auditInit {this, "AuditServices", false, "[[deprecated]] unused"}
Gaudi::Property< bool >	m_auditorInitialize {this, "AuditInitialize", false, "trigger auditor on initialize()"}
Gaudi::Property< bool >	m_auditorStart {this, "AuditStart", false, "trigger auditor on start()"}
Gaudi::Property< bool >	m_auditorStop {this, "AuditStop", false, "trigger auditor on stop()"}
Gaudi::Property< bool >	m_auditorFinalize {this, "AuditFinalize", false, "trigger auditor on finalize()"}
Gaudi::Property< bool >	m_auditorReinitialize {this, "AuditReinitialize", false, "trigger auditor on reinitialize()"}
Gaudi::Property< bool >	m_auditorRestart {this, "AuditRestart", false, "trigger auditor on restart()"}
SmartIF< IAuditorSvc >	m_pAuditorSvc
	Auditor Service. More...

Detailed Description

Definition at line 7 of file HiveDataBroker.h.

Member Function Documentation

std::vector< Algorithm * > HiveDataBrokerSvc::algorithmsRequiredFor ( const DataObjIDColl &  requested, const std::vector< std::string > &  stoppers = {}  ) const

override

Definition at line 259 of file HiveDataBroker.cpp.

{
  std::vector<Algorithm*> result;

  std::vector<const AlgEntry*> deps;
  deps.reserve( requested.size() );

  // start with seeding from the initial request
  for ( const auto& req : requested ) {
    DataObjID id = req;
    if ( id.key().find( ":" ) != std::string::npos ) {
      warning() << req.key() << " contains alternatives which require resolution...\n";
      auto tokens = boost::tokenizer<boost::char_separator<char>>{id.key(), boost::char_separator<char>{":"}};
      auto itok   = std::find_if( tokens.begin(), tokens.end(),
                                [&]( DataObjID t ) { return m_dependencies.find( t ) != m_dependencies.end(); } );
      if ( itok != tokens.end() ) {
        warning() << "found matching output for " << *itok << " -- updating info\n";
        id.updateKey( *itok );
        warning() << "Please update input to not require alternatives, and "
                     "instead properly configure the dataloader"
                  << endmsg;
      } else {
        error() << "failed to find alternate in global output list"
                << " for id: " << id << endmsg;
      }
    }
    auto i = m_dependencies.find( id );
    if ( i == m_dependencies.end() )
      throw GaudiException( "unknown requested input: " + id.key(), __func__, StatusCode::FAILURE );
    deps.push_back( i->second );
  }
  // insert the (direct) dependencies of 'current' right after 'current', and
  // interate until done...
  for ( auto current = deps.begin(); current != deps.end(); ++current ) {
    if ( std::any_of( std::begin( stoppers ), std::end( stoppers ),
                      [current]( auto& stopper ) { return ( *current )->alg->name() == stopper; } ) ) {
      continue;
    }
    for ( auto* entry : ( *current )->dependsOn ) {
      if ( std::find( std::next( current ), deps.end(), entry ) != deps.end() ) continue; // already there downstream...

      auto dup = std::find( deps.begin(), current, entry );
      // if present upstream, move it downstream. Otherwise, insert
      // downstream...
      current = std::prev( dup != current ? std::rotate( dup, std::next( dup ), std::next( current ) )
                                          : deps.insert( std::next( current ), entry ) );
    }
  }
  auto range = ( deps | ranges::view::transform( []( auto& i ) { return i->alg; } ) | ranges::view::reverse );
  return {begin( range ), end( range )};
}

std::vector< Algorithm * > HiveDataBrokerSvc::algorithmsRequiredFor ( const Gaudi::Utils::TypeNameString &  alg, const std::vector< std::string > &  stoppers = {}  ) const

override

Definition at line 312 of file HiveDataBroker.cpp.

{
  std::vector<Algorithm*> result;

  auto alg = std::find_if( begin( m_cfnodes ), end( m_cfnodes ),
                           [&]( const AlgEntry& ae ) { return ae.alg->name() == requested.name(); } );

  if ( alg != end( m_cfnodes ) && alg->alg->type() != requested.type() ) {
    error() << "requested " << requested << " but have matching name with different type: " << alg->alg->type()
            << endmsg;
  }
  if ( alg == end( m_cfnodes ) ) {
    auto av = instantiateAndInitializeAlgorithms( {requested.type() + '/' + requested.name()} );
    assert( av.size() == 1 );
    m_cfnodes.push_back( std::move( av.front() ) );
    alg = std::next( m_cfnodes.rbegin() ).base();
  }
  assert( alg != end( m_cfnodes ) );
  assert( alg->alg != nullptr );
  if ( std::find_if( std::begin( stoppers ), std::end( stoppers ),
                     [&requested]( auto& stopper ) { return requested.name() == stopper; } ) == std::end( stoppers ) ) {
    result = algorithmsRequiredFor( alg->alg->dataDependencies( AccessMode::Read ), stoppers );
  }
  result.push_back( alg->alg );
  if ( msgLevel( MSG::DEBUG ) ) {
    debug() << std::endl << "requested " << requested << " returning " << std::endl << "  ";
    GaudiUtils::details::ostream_joiner( debug(), result, ",\n  ", []( auto& os, const Algorithm* a ) -> decltype(
                                                                       auto ) { return os << AlgorithmRepr{*a}; } );
    debug() << std::endl << endmsg;
  }
  return result;
}

StatusCode HiveDataBrokerSvc::finalize ( )

override

Definition at line 135 of file HiveDataBroker.cpp.

{
  ranges::for_each( m_algorithms | ranges::view::transform( &AlgEntry::alg ),
                    []( Algorithm* alg ) { alg->sysFinalize(); } );

  ranges::for_each(
      m_algorithms | ranges::view::remove_if( []( const auto& entry ) { return entry.requestCount != 0; } ) |
          ranges::view::transform( &AlgEntry::alg ),
      [&]( Algorithm* alg ) { this->warning() << "Unused algorithm: " << AlgorithmRepr{*alg} << endmsg; } );
  m_algorithms.clear();
  return Service::finalize();
}

StatusCode HiveDataBrokerSvc::initialize ( )

override

Definition at line 57 of file HiveDataBroker.cpp.

{
  auto sc = Service::initialize();
  if ( sc.isFailure() ) return sc;
  // populate m_algorithms
  m_algorithms = instantiateAndInitializeAlgorithms( m_producers );
  if ( sc.isFailure() ) return sc;

  // warn about non-reentrant algorithms
  ranges::for_each( m_algorithms | ranges::view::transform( []( const auto& entry ) { return entry.alg; } ) |
                        ranges::view::remove_if( []( const auto* alg ) { return alg->cardinality() == 0; } ),
                    [&]( const Algorithm* alg ) {
                      this->warning() << "non-reentrant algorithm: " << AlgorithmRepr{*alg} << endmsg;
                    } );
  //== Print the list of the created algorithms
  if ( msgLevel( MSG::DEBUG ) ) {
    MsgStream& msg = debug();
    msg << "Available DataProducers: ";
    GaudiUtils::details::ostream_joiner( msg, m_algorithms, ", ", []( auto& os, const AlgEntry& e ) -> decltype(
                                                                      auto ) { return os << AlgorithmRepr{*e.alg}; } );
    msg << endmsg;
  }

  // populate m_dependencies
  m_dependencies = mapProducers( m_algorithms );

  return sc;
}

std::vector< HiveDataBrokerSvc::AlgEntry > HiveDataBrokerSvc::instantiateAndInitializeAlgorithms ( const std::vector< std::string > &  names ) const

private

Definition at line 150 of file HiveDataBroker.cpp.

{
  std::vector<AlgEntry> algorithms;

  //= Get the Application manager, to see if algorithm exist
  auto appMgr = service<IAlgManager>( "ApplicationMgr" );
  for ( const Gaudi::Utils::TypeNameString item : names ) {
    const std::string& theName = item.name();
    const std::string& theType = item.type();

    //== Check wether the specified algorithm already exists. If not, create it
    SmartIF<IAlgorithm> myIAlg = appMgr->algorithm( item, false ); // do not create it now
    if ( !myIAlg ) {
      myIAlg = createAlgorithm( *appMgr, theType, theName );
    } else {
      // when the algorithm is not created, the ref count is short by one, so we
      // have to fix it.
      myIAlg->addRef();
    }

    if ( !myIAlg ) {
      throw GaudiException{"Failed to create " + boost::lexical_cast<std::string>( item ), __func__,
                           StatusCode::FAILURE};
    }

    // propagate the sub-algorithm into own state.
    StatusCode sc = myIAlg->sysInitialize();
    if ( sc.isFailure() ) {
      throw GaudiException{"Failed to initialize " + boost::lexical_cast<std::string>( item ), __func__,
                           StatusCode::FAILURE};
    }

    algorithms.emplace_back( std::move( myIAlg ) );
  }

  return algorithms;
}

std::map< DataObjID, HiveDataBrokerSvc::AlgEntry * > HiveDataBrokerSvc::mapProducers ( std::vector< AlgEntry > &  algorithms ) const

private

Definition at line 189 of file HiveDataBroker.cpp.

{
  // figure out all outputs
  std::map<DataObjID, AlgEntry*> producers;
  for ( AlgEntry& alg : algorithms ) {
    const auto& output = alg.alg->dataDependencies( AccessMode::Write );
    if ( output.empty() ) {
      error() << AlgorithmRepr{*algorithms.back().alg} << " does not produce any data -- should not appear in list of "
                                                          "producers. Ignoring."
              << endmsg;
      continue;
    }
    for ( auto id : output ) {
      if ( id.key().find( ":" ) != std::string::npos ) {
        error() << " in Alg " << AlgorithmRepr{*alg.alg} << " alternatives are NOT allowed for outputs! id: " << id
                << endmsg;
      }

      auto r = producers.emplace( id, &alg );
      if ( !r.second ) {
        if ( output.size() == 1 ) {
          warning() << "multiple algorithms declare " << id << " as output! -- IGNORING " << AlgorithmRepr{*alg.alg}
                    << endmsg;
        } else {
          error() << "multiple algorithms declate " << id << " as output; given that " << AlgorithmRepr{*alg.alg}
                  << " produces multiple outputs ";
          //<< output <<
          error() << " this could lead to clashes in case any of the other "
                     "items is ever requested"
                  << endmsg;
        }
      }
    }
  }

  // resolve dependencies
  for ( auto& algEntry : algorithms ) {
    auto input = sortedDataObjIDColl( algEntry.alg->dataDependencies( AccessMode::Read ) );
    for ( const DataObjID* idp : input ) {
      DataObjID id = *idp;
      if ( id.key().find( ":" ) != std::string::npos ) {
        warning() << " contains alternatives which require resolution...\n";
        auto tokens = boost::tokenizer<boost::char_separator<char>>{id.key(), boost::char_separator<char>{":"}};
        auto itok   = std::find_if( tokens.begin(), tokens.end(),
                                  [&]( DataObjID t ) { return producers.find( t ) != producers.end(); } );
        if ( itok != tokens.end() ) {
          warning() << "found matching output for " << *itok << " -- updating info\n";
          id.updateKey( *itok );
          warning() << "Please update input to not require alternatives, and "
                       "instead properly configure the dataloader"
                    << endmsg;
        } else {
          error() << "failed to find alternate in global output list"
                  << " for id: " << id << " in Alg " << algEntry.alg << endmsg;
        }
      }
      auto iproducer = producers.find( id );
      if ( iproducer != producers.end() ) {
        algEntry.dependsOn.insert( iproducer->second );
      } else {
        // TODO: assign to dataloader!
        // algEntry.dependsOn.insert(dataloader.alg);
        // dataloader.data.emplace( id ); // TODO: we may ask to much of the
        // dataloader this way...
      }
    }
  }
  return producers;
}

StatusCode HiveDataBrokerSvc::start ( )

override

Definition at line 86 of file HiveDataBroker.cpp.

{

  StatusCode ss = Service::start();
  if ( !ss.isSuccess() ) return ss;

  // sysStart for m_algorithms
  for ( AlgEntry& algEntry : m_algorithms ) {
    ss = algEntry.alg->sysStart();
    if ( ss.isFailure() ) {
      error() << "Unable to start Algorithm: " << algEntry.alg->name() << endmsg;
      return ss;
    }
  }
  // sysStart for m_cfnodes
  for ( AlgEntry& algEntry : m_cfnodes ) {
    ss = algEntry.alg->sysStart();
    if ( ss.isFailure() ) {
      error() << "Unable to start Algorithm: " << algEntry.alg->name() << endmsg;
      return ss;
    }
  }
  return ss;
}

StatusCode HiveDataBrokerSvc::stop ( )

override

Definition at line 111 of file HiveDataBroker.cpp.

{
  StatusCode ss = Service::stop();
  if ( !ss.isSuccess() ) return ss;

  // sysStart for m_algorithms
  for ( AlgEntry& algEntry : m_algorithms ) {
    ss = algEntry.alg->sysStop();
    if ( ss.isFailure() ) {
      error() << "Unable to stop Algorithm: " << algEntry.alg->name() << endmsg;
      return ss;
    }
  }
  // sysStart for m_cfnodes
  for ( AlgEntry& algEntry : m_cfnodes ) {
    ss = algEntry.alg->sysStop();
    if ( ss.isFailure() ) {
      error() << "Unable to stop Algorithm: " << algEntry.alg->name() << endmsg;
      return ss;
    }
  }
  return ss;
}

Member Data Documentation

std::vector<AlgEntry> HiveDataBrokerSvc::m_algorithms

private

Definition at line 47 of file HiveDataBroker.h.

std::vector<AlgEntry> HiveDataBrokerSvc::m_cfnodes

mutableprivate

Definition at line 49 of file HiveDataBroker.h.

Gaudi::Property<std::string> HiveDataBrokerSvc::m_dataLoader

private

Initial value:

{this, "DataLoader", "",
                                            "Attribute any unmet input dependencies to this Algorithm"}

Definition at line 23 of file HiveDataBroker.h.

std::map<DataObjID, AlgEntry*> HiveDataBrokerSvc::m_dependencies

private

Definition at line 53 of file HiveDataBroker.h.

Gaudi::Property<std::vector<std::string> > HiveDataBrokerSvc::m_producers

private

Initial value:

{
      this, "DataProducers", {}, "List of algorithms to be used to resolve data dependencies"}

Definition at line 25 of file HiveDataBroker.h.

---

The documentation for this class was generated from the following files:

• GaudiHive/src/HiveDataBroker.h
• GaudiHive/src/HiveDataBroker.cpp