Gaudi::EventWatchdogAlg Class Reference

Add to the transient store a tracker that detects events that are taking too long. More...

Inheritance diagram for Gaudi::EventWatchdogAlg:

Collaboration diagram for Gaudi::EventWatchdogAlg:

Public Member Functions
	EventWatchdogAlg (const std::string &name, ISvcLocator *pSvcLocator)
StatusCode	initialize () override
PeriodicAction	operator() (EventContext const &ctx) const override

Private Attributes
Gaudi::CheckedProperty< unsigned int >	m_eventTimeout
Gaudi::Property< bool >	m_abortOnTimeout
Gaudi::Property< bool >	m_stackTrace { this, "StackTrace", false, "Whether to print the stack-trace on timeout." }

Detailed Description

Add to the transient store a tracker that detects events that are taking too long.

Definition at line 55 of file EventWatchdogAlg.cpp.

Constructor & Destructor Documentation

EventWatchdogAlg()

Gaudi::EventWatchdogAlg::EventWatchdogAlg ( const std::string &  name, ISvcLocator *  pSvcLocator  )

inline

Definition at line 57 of file EventWatchdogAlg.cpp.

        : Transformer( name, pSvcLocator, KeyValue{ "TimerLocation", fmt::format( ".{}-timer", sanitize( name ) ) } ) {
      // timeout period cannot be smaller than 1
      m_eventTimeout.verifier().setLower( 1 );
    }

Member Function Documentation

initialize()

StatusCode Gaudi::EventWatchdogAlg::initialize ( )

inlineoverride

Definition at line 63 of file EventWatchdogAlg.cpp.

                                     {
      return Transformer::initialize().andThen( [this] {
        if ( !service( "StalledEventMonitor", false, true ) ) {
          // StalledEventMonitor was not instantiated, so we try to steal its options
          auto& opts = serviceLocator()->getOptsSvc();
          opts.bind( "StalledEventMonitor", &m_eventTimeout );
          opts.bind( "StalledEventMonitor", &m_stackTrace );
          if ( opts.isSet( "StalledEventMonitor.MaxTimeoutCount" ) ) {
            Gaudi::Property<int> maxCount{ "MaxTimeoutCount", 0 };
            opts.bind( "StalledEventMonitor", &maxCount );
            m_abortOnTimeout = maxCount.value() >= 0;
          }
        }
      } );
    }

operator()()

PeriodicAction Gaudi::EventWatchdogAlg::operator() ( EventContext const &  ctx ) const

inlineoverride

Definition at line 79 of file EventWatchdogAlg.cpp.

                                                                        {
      using namespace std::chrono_literals;
      // we use a functor because we cannot pass mutable states to a lambda
      struct Action {
        MsgStream          log;
        const bool         doStackTrace;
        const bool         abortOnTimeout;
        const unsigned int timeout;
        const EventContext ctx;
 
        const std::vector<IScheduler*> schedulers;
 
        int counter{ 0 };
 
        const std::chrono::steady_clock::time_point eventStart{ std::chrono::steady_clock::now() };
 
        void operator()() {
          ++counter;
          if ( counter == 1 ) {
            log << MSG::WARNING << fmt::format( "More than {}s since the beginning of the event ({})", timeout, ctx )
                << endmsg;
          } else {
            log << MSG::WARNING << fmt::format( "Another {}s passed since last timeout ({})", timeout, ctx ) << endmsg;
          }
          if ( log.level() <= MSG::INFO ) {
            log << MSG::INFO
                << fmt::format( "Current memory usage is virtual size = {} MB, resident set size = {} MB",
                                System::virtualMemory() / 1024., System::pagedMemory() / 1024. )
                << endmsg;
          }
 
          std::scoped_lock protectReport( s_watchdogReportMutex );
          if ( doStackTrace ) {
            for ( auto scheduler : schedulers ) { scheduler->dumpState(); }
            if ( gSystem ) {
              // TSystem::StackTrace() prints on the standard error, so we do the same
              fmt::print( stderr, "=== Stalled event: current stack trace ({}) ===\n", ctx );
              gSystem->StackTrace();
            }
          }
          if ( abortOnTimeout ) {
            log << MSG::FATAL << fmt::format( "too much time on a single event ({}): aborting process", ctx ) << endmsg;
            std::raise( SIGQUIT );
          }
        }
 
        ~Action() {
          if ( counter ) {
            const std::chrono::duration<float, std::chrono::seconds::period> duration =
                std::chrono::steady_clock::now() - eventStart;
            log << MSG::INFO << fmt::format( "An event ({}) took {:.3f}s", ctx, duration.count() ) << endmsg;
          }
        }
      };
 
      std::vector<IScheduler*> schedulers;
      if ( m_stackTrace ) {
        using namespace ranges;
        // if we ask for the stack trace we are also interested in the state of the scheduler
        schedulers = svcLoc()->getServices() |
                     views::transform( []( auto svc ) { return Gaudi::Cast<IScheduler>( svc ); } ) |
                     views::remove( nullptr ) | to<std::vector>();
      }
 
      Action action{ MsgStream{ msgSvc(), "EventWatchdog" }, m_stackTrace, m_abortOnTimeout, m_eventTimeout,
                     // use a partial copy of the context to avoid copying the optional extension
                     EventContext{ ctx.evt(), ctx.slot(), ctx.subSlot() }, std::move( schedulers ) };
      return PeriodicAction( std::move( action ), std::chrono::seconds{ m_eventTimeout } );
    }

Member Data Documentation

m_abortOnTimeout

Gaudi::Property<bool> Gaudi::EventWatchdogAlg::m_abortOnTimeout

private

Initial value:

{ this, "AbortOnTimeout", false,
                                            "If set to true, the application is killed when we reach the timeout." }

Definition at line 152 of file EventWatchdogAlg.cpp.

m_eventTimeout

Gaudi::CheckedProperty<unsigned int> Gaudi::EventWatchdogAlg::m_eventTimeout

private

Initial value:

{ this, "EventTimeout", 600,
                                                         "Number of seconds allowed to process a single event." }

Definition at line 150 of file EventWatchdogAlg.cpp.

m_stackTrace

Gaudi::Property<bool> Gaudi::EventWatchdogAlg::m_stackTrace { this, "StackTrace", false, "Whether to print the stack-trace on timeout." }

private

Definition at line 154 of file EventWatchdogAlg.cpp.

---

The documentation for this class was generated from the following file:

• GaudiUtils/src/component/EventWatchdogAlg.cpp