The Gaudi Framework master (9cc87df9)
Loading...
Searching...
No Matches
HiveDataBroker.cpp
Go to the documentation of this file.
1/***********************************************************************************\
2* (c) Copyright 1998-2026 CERN for the benefit of the LHCb and ATLAS collaborations *
3* *
4* This software is distributed under the terms of the Apache version 2 licence, *
5* copied verbatim in the file "LICENSE". *
6* *
7* In applying this licence, CERN does not waive the privileges and immunities *
8* granted to it by virtue of its status as an Intergovernmental Organization *
9* or submit itself to any jurisdiction. *
10\***********************************************************************************/
11// Local includes
12#include "GraphDumper.h"
13
14// Framework includes
15#include <Gaudi/Algorithm.h>
19#include <GaudiKernel/Service.h>
20#include <GaudiKernel/System.h>
21
22// STL includes
23#include <algorithm>
24#include <iomanip>
25#include <ranges>
26#include <stdexcept>
27
28namespace {
29 struct AlgEntry {
30 size_t index;
31 SmartIF<IAlgorithm> ialg;
32 Gaudi::Algorithm* alg;
33 std::set<AlgEntry const*> dependsOn;
34 std::vector<DataObjID const*> inputs;
35 std::vector<DataObjID const*> outputs;
36
37 friend bool operator<( AlgEntry const& lhs, AlgEntry const& rhs ) { return lhs.index < rhs.index; }
38
39 friend bool operator==( AlgEntry const& lhs, AlgEntry const& rhs ) { return lhs.index == rhs.index; }
40
41 AlgEntry( size_t i, SmartIF<IAlgorithm>&& p )
42 : index{ i }, ialg{ std::move( p ) }, alg{ dynamic_cast<Gaudi::Algorithm*>( ialg.get() ) } {
43 if ( !alg ) throw std::runtime_error( "algorithm pointer == nullptr???" );
44 // gather _all_ the inputs and outputs in a well-defined, reproducible manner,
45 // removing any duplication (sometimes, extra{Out,In}putDeps entries will already appear in {Out,In}putDataObjs)
46 constexpr auto gather = []( auto& c, auto const& in1, auto const& in2 ) {
47 for ( const DataObjID& id : in1 ) c.push_back( &id );
48 for ( const DataObjID& id : in2 ) c.push_back( &id );
49 constexpr auto by_key = []( const DataObjID* id ) { return id->fullKey(); };
50 std::ranges::sort( c, std::less{}, by_key );
51 auto od = std::ranges::unique( c, std::equal_to{}, by_key );
52 c.erase( od.begin(), od.end() );
53 };
54 gather( outputs, alg->outputDataObjs(), alg->extraOutputDeps() );
55 gather( inputs, alg->inputDataObjs(), alg->extraInputDeps() );
56 }
57 };
58
59 template <std::ranges::range R>
60 requires std::common_reference_with<std::ranges::range_reference_t<R>, const AlgEntry&>
61 void dumpDataDepsGraphFile( std::string const& fname, R const& algorithms ) {
62 Gaudi::Hive::Graph g{ fname };
63
64 std::size_t algoIndex = 0ul;
65
66 // define algs and objects
67 std::set<std::size_t> definedObjects;
68
69 // Remember algorithm indices by their name
71
72 // loop over all algorithms to generate indexes
74 for ( const AlgEntry& entry : algorithms ) {
75 std::string algIndex = "Alg_" + std::to_string( algoIndex );
76 indexByName[entry.alg->name()] = algIndex;
77 ++algoIndex;
78 }
79
80 // loop over all algorithms to create nodes and edges
81 for ( const AlgEntry& entry : algorithms ) {
82 // Create algorithm node
83 g.addNode( indexByName[entry.alg->name()], entry.alg->name() );
84
85 // inputs
86 for ( const auto* id : entry.inputs ) {
87 const auto [itr, inserted] = definedObjects.insert( id->hash() );
88 std::string objIndex = "obj_" + std::to_string( id->hash() );
89 if ( inserted ) g.addNode( objIndex, id->key() );
90
91 // the object is the source, the algorithm taking it as an input is the target
92 g.addEdge( objIndex, indexByName[entry.alg->name()] );
93 }
94
95 // outputs
96 for ( const auto* id : entry.outputs ) {
97 const auto [itr, inserted] = definedObjects.insert( id->hash() );
98 std::string objIndex = "obj_" + std::to_string( id->hash() );
99 if ( inserted ) g.addNode( objIndex, id->key() );
100
101 // the algorithm that generated the object is the source, the object is the target
102 g.addEdge( indexByName[entry.alg->name()], objIndex );
103 }
104 }
105 }
106
107 struct AlgorithmRepr {
108 const Gaudi::Algorithm& parent;
109
110 friend std::ostream& operator<<( std::ostream& s, const AlgorithmRepr& a ) {
111 std::string typ = System::typeinfoName( typeid( a.parent ) );
112 s << typ;
113 if ( a.parent.name() != typ ) s << "/" << a.parent.name();
114 return s;
115 }
116 };
117
118 // Used for making debugging dumps.
119 template <typename T>
120 std::vector<const T*> sorted_( const std::set<T*>& s ) {
121 std::vector<const T*> v{ s.begin(), s.end() };
122 std::sort( v.begin(), v.end(), []( const auto* lhs, const auto* rhs ) { return *lhs < *rhs; } );
123 return v;
124 }
125
126 SmartIF<IAlgorithm> createAlgorithm( IAlgManager& am, const std::string& type, const std::string& name ) {
127 // Maybe modify the AppMgr interface to return Algorithm* ??
128 IAlgorithm* tmp = nullptr;
129 StatusCode sc = am.createAlgorithm( type, name, tmp );
130 return sc.isSuccess() ? dynamic_cast<Gaudi::Algorithm*>( tmp ) : nullptr;
131 }
132} // namespace
133
134class HiveDataBrokerSvc final : public extends<Service, IDataBroker> {
135public:
136 using extends::extends;
137
139 const std::vector<std::string>& stoppers = {} ) const override;
141 const std::vector<std::string>& stoppers = {} ) const override;
142
143 StatusCode initialize() override;
144 StatusCode start() override;
145 StatusCode stop() override;
146 StatusCode finalize() override;
147
148private:
150 "Attribute any unmet input dependencies to this Algorithm" };
152 this, "DataProducers", {}, "List of algorithms to be used to resolve data dependencies" };
153
154 Gaudi::Property<std::string> m_dataDepsGraphFile{ this, "DataDepsGraphFile", "",
155 "Name of the output file (.dot, .md or .graphml extensions "
156 "allowed) containing the data dependency graph. "
157 "If empty, no graph is dumped" };
158
160 instantiateAndInitializeAlgorithms( const std::vector<std::string>& names ) const; // algorithms must be fully
161 // initialized first, as
162 // doing so may create
163 // additional data
164 // dependencies...
165
167
169
171
172 void visit( AlgEntry const& alg, std::vector<std::string> const& stoppers, std::vector<Gaudi::Algorithm*>& sorted,
173 std::vector<bool>& visited, std::vector<bool>& visiting ) const;
174};
175
177
179 return Service::initialize().andThen( [&] {
180 // populate m_algorithms
182
183 // warn about non-reentrant algorithms
184 std::ranges::for_each( m_algorithms | std::ranges::views::transform( []( const auto& entry ) -> decltype( auto ) {
185 return entry.second.alg;
186 } ) |
187 std::ranges::views::filter( []( const auto* alg ) { return alg->cardinality() > 0; } ),
188 [&]( const Gaudi::Algorithm* alg ) {
189 this->warning() << "non-reentrant algorithm: " << AlgorithmRepr{ *alg } << endmsg;
190 } );
191 //== Print the list of the created algorithms
192 if ( msgLevel( MSG::DEBUG ) ) {
193 debug() << "Available DataProducers:\n";
194 std::ranges::for_each( m_algorithms | std::ranges::views::transform( []( const auto& entry ) -> decltype( auto ) {
195 return entry.second.alg;
196 } ),
197 [&]( const Gaudi::Algorithm* alg ) {
198 this->debug() << " " << AlgorithmRepr{ *alg } << " " << alg->outputDataObjs() << " "
199 << alg->extraOutputDeps() << endmsg;
200 } );
201 }
202
203 // populate m_dependencies and set AlgEntry::dependsOn
205 } );
206}
207
209
211 if ( !ss.isSuccess() ) return ss;
212
213 // sysStart for m_algorithms
214 for ( auto& [name, algEntry] : m_algorithms ) {
215 ss = algEntry.alg->sysStart();
216 if ( ss.isFailure() ) {
217 error() << "Unable to start Algorithm: " << name << endmsg;
218 return ss;
219 }
220 }
221 return ss;
222}
223
226 if ( !ss.isSuccess() ) return ss;
227
228 // sysStart for m_algorithms
229 for ( auto& [name, algEntry] : m_algorithms ) {
230 ss = algEntry.alg->sysStop();
231 if ( ss.isFailure() ) {
232 error() << "Unable to stop Algorithm: " << name << endmsg;
233 return ss;
234 }
235 }
236 return ss;
237}
238
240 for ( auto& [name, algEntry] : m_algorithms ) {
241 algEntry.alg->sysFinalize().ignore( /* AUTOMATICALLY ADDED FOR gaudi/Gaudi!763 */ );
242 }
243 m_algorithms.clear();
244 return Service::finalize();
245}
246
250
251 //= Get the Application manager, to see if algorithm exist
252 auto appMgr = service<IAlgManager>( "ApplicationMgr" );
253 size_t index = 0;
254 for ( const std::string& item : names ) {
255 const Gaudi::Utils::TypeNameString tn( item );
256
257 //== Check wether the specified algorithm already exists. If not, create it
258 SmartIF<IAlgorithm> myIAlg = appMgr->algorithm( item, false ); // do not create it now
259 if ( !myIAlg ) {
260 myIAlg = createAlgorithm( *appMgr, tn.type(), tn.name() );
261 } else {
262 // when the algorithm is not created, the ref count is short by one, so we
263 // have to fix it.
264 myIAlg->addRef();
265 }
266
267 if ( !myIAlg ) { throw GaudiException{ "Failed to create " + item, __func__, StatusCode::FAILURE }; }
268
269 // propagate the sub-algorithm into own state.
270 StatusCode sc = myIAlg->sysInitialize();
271 if ( sc.isFailure() ) { throw GaudiException{ "Failed to initialize " + item, __func__, StatusCode::FAILURE }; }
272
273 algorithms.emplace( tn.name(), AlgEntry{ index++, std::move( myIAlg ) } );
274 }
275
276 return algorithms;
277}
278
281 if ( msgLevel( MSG::DEBUG ) ) {
282 debug() << "Data Dependencies for Algorithms:";
283 for ( const auto& [name, entry] : m_algorithms ) {
284 debug() << "\n " << name << " :";
285 for ( const auto* id : entry.inputs ) { debug() << "\n o INPUT " << id->key(); }
286 for ( const auto* id : entry.outputs ) { debug() << "\n o OUTPUT " << id->key(); }
287 }
288 debug() << endmsg;
289 }
290
291 // If requested, dump a graph of the data dependencies in a .dot, .md or .graphml file
292 if ( !m_dataDepsGraphFile.empty() ) {
293 info() << "Dumping data dependencies graph to file: " << m_dataDepsGraphFile.value() << endmsg;
294 dumpDataDepsGraphFile( m_dataDepsGraphFile, m_algorithms | std::views::values );
295 }
296
297 // figure out all outputs
299 for ( auto& [name, alg] : algorithms ) {
300 for ( auto id : alg.outputs ) {
301 auto r = producers.emplace( *id, &alg );
302 if ( !r.second ) {
303 throw GaudiException( "multiple algorithms declare " + id->key() + " as output (" + name + " and " +
304 producers[*id]->alg->name() + " at least). This is not allowed",
305 __func__, StatusCode::FAILURE );
306 }
307 }
308 }
309
310 // resolve dependencies
311 for ( auto& [name, algEntry] : algorithms ) {
312 for ( const DataObjID* idp : algEntry.inputs ) {
313 DataObjID id = *idp;
314 auto iproducer = producers.find( id );
315 if ( iproducer != producers.end() ) {
316 algEntry.dependsOn.insert( iproducer->second );
317 } else {
318 std::ostringstream error_message;
319 error_message << "\nUnknown requested input by " << AlgorithmRepr{ *( algEntry.alg ) } << " : "
320 << std::quoted( id.key(), '\'' ) << ".\n";
321 error_message << "You can set the OutputLevel of HiveDataBrokerSvc to DEBUG to get a list of inputs and "
322 "outputs of every registered algorithm.\n";
323 throw GaudiException( error_message.str(), __func__, StatusCode::FAILURE );
324 // TODO: assign to dataloader!
325 // algEntry.dependsOn.insert(dataloader.alg);
326 // dataloader.data.emplace( id ); // TODO: we may ask to much of the
327 // dataloader this way...
328 }
329 }
330 }
331 return producers;
332}
333
335void HiveDataBrokerSvc::visit( AlgEntry const& alg, std::vector<std::string> const& stoppers,
337 std::vector<bool>& visiting ) const {
338 assert( visited.size() == m_algorithms.size() );
339 assert( visiting.size() == m_algorithms.size() );
340 if ( visited[alg.index] ) { return; }
341 if ( visiting[alg.index] ) { throw GaudiException( "Cycle detected ", __func__, StatusCode::FAILURE ); }
342
343 if ( std::none_of( std::begin( stoppers ), std::end( stoppers ),
344 [alg]( auto& stopper ) { return alg.alg->name() == stopper; } ) ) {
345 visiting[alg.index] = true;
346 for ( auto* dep : sorted_( alg.dependsOn ) ) { visit( *dep, stoppers, sorted, visited, visiting ); }
347 visiting[alg.index] = false;
348 }
349
350 visited[alg.index] = true;
351 sorted.push_back( alg.alg );
352}
353
356 const std::vector<std::string>& stoppers ) const {
358
360 deps.reserve( requested.size() );
361
362 // start with seeding from the initial request
363 for ( const auto& id : requested ) {
364 auto i = m_dependencies.find( id );
365 if ( i == m_dependencies.end() )
366 throw GaudiException( "unknown requested input: " + id.key(), __func__, StatusCode::FAILURE );
367 deps.push_back( i->second );
368 }
369 // producers may be responsible for multiple requested DataObjID -- make sure they are only mentioned once
370 std::sort( deps.begin(), deps.end(), []( auto const* lhs, auto const* rhs ) { return *lhs < *rhs; } );
371 deps.erase( std::unique( deps.begin(), deps.end(), []( auto const& lhs, auto const& rhs ) { return *lhs == *rhs; } ),
372 deps.end() );
373
375 std::vector<bool> visiting( m_algorithms.size() );
376 for ( auto* alg : deps ) { visit( *alg, stoppers, result, visited, visiting ); }
377 return result;
378}
379
382 const std::vector<std::string>& stoppers ) const {
384
385 auto it = m_algorithms.find( requested.name() );
386 if ( it == end( m_algorithms ) ) {
387 throw GaudiException{ "No algorithm with name " + requested.name() + " in DataProducers. Type is " +
388 ( requested.haveType() ? requested.type() : "not specified" ),
389 __func__, StatusCode::FAILURE };
390 }
391 auto const& alg = it->second;
392 if ( requested.haveType() && alg.alg->type() != requested.type() ) {
393 error() << "requested " << requested << " but have matching name with different type: " << alg.alg->type()
394 << endmsg;
395 }
396 assert( alg.alg != nullptr );
397
399 std::vector<bool> visiting( m_algorithms.size() );
400 visit( alg, stoppers, result, visited, visiting );
401
402 if ( msgLevel( MSG::DEBUG ) ) {
403 debug() << std::endl << "requested " << requested << " returning " << std::endl << " ";
405 debug(), result, ",\n ",
406 []( auto& os, const Gaudi::Algorithm* a ) -> decltype( auto ) { return os << AlgorithmRepr{ *a }; } );
407 debug() << std::endl << endmsg;
408 }
409 return result;
410}
std::unordered_set< DataObjID, DataObjID_Hasher > DataObjIDColl
Definition DataObjID.h:121
MsgStream & endmsg(MsgStream &s)
MsgStream Modifier: endmsg. Calls the output method of the MsgStream.
Definition MsgStream.h:198
#define DECLARE_COMPONENT(type)
strong_ordering operator(const error_code &__lhs, const error_code &__rhs) noexcept
basic_ostream< char > ostream
constexpr std::remove_reference< _Tp >::type && move(_Tp &&__t) noexcept
constexpr _ForwardIterator unique(_ForwardIterator __first, _ForwardIterator __last)
constexpr bool none_of(_InputIterator __first, _InputIterator __last, _Predicate __pred)
constexpr void sort(_RandomAccessIterator __first, _RandomAccessIterator __last)
basic_string< char > string
basic_ostream< _CharT, _Traits > & endl(basic_ostream< _CharT, _Traits > &__os)
constexpr auto begin(_Container &__cont) -> decltype(__cont.begin())
constexpr auto end(_Container &__cont) -> decltype(__cont.end())
constexpr bool operator==(const basic_string< _CharT, _Traits, _Alloc > &__lhs, const _CharT *__rhs)
__string_type str() const
const_iterator end() const noexcept
size_type size() const noexcept
iterator find(const key_type &__x)
std::pair< iterator, bool > emplace(_Args &&... __args)
void insert(_InputIterator __first, _InputIterator __last)
constexpr void push_back(const value_type &__x)
constexpr const_iterator begin() const noexcept
constexpr iterator erase(const_iterator __first, const_iterator __last)
constexpr void reserve(size_type __n)
constexpr const_iterator end() const noexcept
constexpr size_type size() const noexcept
size_type size() const noexcept
MsgStream & error() const
shortcut for the method msgStream(MSG::ERROR)
MsgStream & warning() const
shortcut for the method msgStream(MSG::WARNING)
MsgStream & debug() const
shortcut for the method msgStream(MSG::DEBUG)
MsgStream & info() const
shortcut for the method msgStream(MSG::INFO)
MSG::Level msgLevel() const
get the cached level (originally extracted from the embedded MsgStream)
Base class from which all concrete algorithm classes should be derived.
Definition Algorithm.h:87
const std::string & name() const override
The identifying name of the algorithm object.
utilities to dump graphs in different formats
Definition GraphDumper.h:30
Implementation of property with value of concrete type.
Definition Property.h:35
const ValueType & value() const
Definition Property.h:246
auto empty() const
Definition Property.h:272
Helper class to parse a string of format "type/name".
const std::string & type() const
const std::string & name() const
Define general base for Gaudi exception.
GaudiException(std::string Message, std::string Tag, StatusCode Code)
Constructor (1).
StatusCode initialize() override
std::map< std::string, AlgEntry > instantiateAndInitializeAlgorithms(const std::vector< std::string > &names) const
Gaudi::Property< std::vector< std::string > > m_producers
std::map< std::string, AlgEntry > m_algorithms
StatusCode stop() override
Gaudi::Property< std::string > m_dataDepsGraphFile
StatusCode finalize() override
Gaudi::Property< std::string > m_dataLoader
std::map< DataObjID, AlgEntry const * > mapProducers(std::map< std::string, AlgEntry > &algorithms) const
void visit(AlgEntry const &alg, std::vector< std::string > const &stoppers, std::vector< Gaudi::Algorithm * > &sorted, std::vector< bool > &visited, std::vector< bool > &visiting) const
Implements DFS topological sorting.
std::vector< Gaudi::Algorithm * > algorithmsRequiredFor(const DataObjIDColl &requested, const std::vector< std::string > &stoppers={}) const override
StatusCode start() override
std::map< DataObjID, AlgEntry const * > m_dependencies
The IAlgManager is the interface implemented by the Algorithm Factory in the Application Manager to s...
Definition IAlgManager.h:34
virtual StatusCode createAlgorithm(const std::string &algtype, const std::string &algname, IAlgorithm *&alg, bool managed=false, bool checkIfExists=true)=0
Create an instance of a algorithm type that has been declared beforehand and assigns to it a name.
The IAlgorithm is the interface implemented by the Algorithm base class.
Definition IAlgorithm.h:37
StatusCode finalize() override
Definition Service.cpp:223
const std::string & name() const override
Retrieve name of the service.
Definition Service.cpp:333
StatusCode stop() override
Definition Service.cpp:181
SmartIF< IFace > service(const std::string &name, bool createIf=true) const
Definition Service.h:79
StatusCode start() override
Definition Service.cpp:187
StatusCode initialize() override
Definition Service.cpp:118
Small smart pointer class with automatic reference counting for IInterface.
Definition SmartIF.h:28
This class is used for returning status codes from appropriate routines.
Definition StatusCode.h:64
bool isFailure() const
Definition StatusCode.h:118
StatusCode andThen(F &&f, ARGS &&... args) const
Chain code blocks making the execution conditional a success result.
Definition StatusCode.h:151
bool isSuccess() const
Definition StatusCode.h:302
constexpr static const auto FAILURE
Definition StatusCode.h:100
Base class used to extend a class implementing other interfaces.
Definition extends.h:19
get
decorate the vector of properties
Definition decorators.py:94
Stream & ostream_joiner(Stream &os, Iterator first, Iterator last, Separator sep, OutputElement output=OutputElement{})
@ DEBUG
Definition IMessageSvc.h:22
GAUDI_API const std::string typeinfoName(const std::type_info &)
Get platform independent information about the class type.
Definition System.cpp:254
dict g
Definition gaudirun.py:590