| The Gaudi Framework v32r1 (f65d50dc)

StalledEventMonitor.cpp
Go to the documentation of this file.
1 /*
2 * StalledEventMonitor.cpp
3 *
4 * Created on: Apr 19, 2010
5 * Author: Marco Clemencic
6 */
7
8 // Include files
9 #include "StalledEventMonitor.h"
10
11 #include "GaudiKernel/IIncidentSvc.h"
12 #include "GaudiKernel/Memory.h"
13 #include "GaudiKernel/WatchdogThread.h"
14
15 #include "TSystem.h"
16
17 #include <csignal>
18
19 namespace {
21 class EventWatchdog : public WatchdogThread {
22 public:
23 EventWatchdog( const SmartIF<IMessageSvc>& msgSvc, const std::string& name,
24 boost::posix_time::time_duration timeout, bool stackTrace = false, long maxCount = 0,
25 bool autostart = false )
26 : WatchdogThread( timeout, autostart )
27 , log( msgSvc, name )
28 , m_maxCount( maxCount )
29 , m_stackTrace( stackTrace ) {}
30
31 private:
33 MsgStream log;
35 long m_counter = 0;
37 long m_maxCount = 0;
39 bool m_stackTrace = false;
41 void action() override {
42 if ( !m_counter ) {
43 log << MSG::WARNING << "More than " << getTimeout().total_seconds() << "s since the last "
44 << IncidentType::BeginEvent << endmsg;
45 } else {
46 log << MSG::WARNING << "Other " << getTimeout().total_seconds() << "s passed" << endmsg;
47 }
48 log << MSG::INFO
49 << "Current memory usage is"
50 " virtual size = "
51 << System::virtualMemory() / 1024.
52 << " MB"
53 ", resident set size = "
54 << System::pagedMemory() / 1024. << " MB" << endmsg;
55 if ( m_stackTrace && gSystem ) {
56 // TSystem::StackTrace() prints on the standard error, so we
57 std::cerr << "=== Stalled event: current stack trace ===" << std::endl;
58 gSystem->StackTrace();
59 }
60 ++m_counter;
61 if ( m_maxCount > 0 && m_counter >= m_maxCount ) {
62 log << MSG::FATAL << "too much time on a single event: aborting process" << endmsg;
63 std::raise( SIGABRT );
64 }
65 }
66 void onPing() override {
67 if ( m_counter ) {
68 if ( m_counter >= 3 )
69 log << MSG::INFO << "Starting a new event after ~" << m_counter * getTimeout().total_seconds() << "s"
70 << endmsg;
71 m_counter = 0;
72 }
73 }
74 void onStop() override {
75 if ( m_counter >= 3 )
76 log << MSG::INFO << "The last event took ~" << m_counter * getTimeout().total_seconds() << "s" << endmsg;
77 }
78 };
79 } // namespace
80
81 // Initialization of the service.
82 StatusCode StalledEventMonitor::initialize() {
83 StatusCode sc = base_class::initialize();
84 if ( sc.isFailure() ) return sc;
85
86 if ( m_eventTimeout ) {
87 // create the watchdog thread
88 m_watchdog = std::make_unique<EventWatchdog>( msgSvc(), "EventWatchdog",
89 boost::posix_time::seconds( m_eventTimeout.value() ), m_stackTrace,
90 m_maxTimeoutCount );
91
92 // register to the incident service
93 static const std::string serviceName = "IncidentSvc";
94 m_incidentSvc = serviceLocator()->service( serviceName );
95 if ( !m_incidentSvc ) {
96 error() << "Cannot retrieve " << serviceName << endmsg;
97 return StatusCode::FAILURE;
98 }
99 debug() << "Register to the IncidentSvc" << endmsg;
100 m_incidentSvc->addListener( this, IncidentType::BeginEvent );
101 } else {
102 warning() << "StalledEventMonitor/" << name() << " instantiated with 0 time-out: no monitoring performed" << endmsg;
103 }
104
105 return StatusCode::SUCCESS;
106 }
107
108 // Start the monitoring.
109 StatusCode StalledEventMonitor::start() {
110 if ( m_watchdog ) m_watchdog->start();
111 return StatusCode::SUCCESS;
112 }
113
114 // Notify the watchdog that a new event has been started
115 void StalledEventMonitor::handle( const Incident& /* incident */ ) {
116 if ( m_watchdog ) m_watchdog->ping();
117 }
118
119 // Start the monitoring.
120 StatusCode StalledEventMonitor::stop() {
121 if ( m_watchdog ) m_watchdog->stop();
122 return StatusCode::SUCCESS;
123 }
124
125 // Finalization of the service.
126 StatusCode StalledEventMonitor::finalize() {
127 // destroy the watchdog thread (if any)
128 m_watchdog.reset();
129 // unregistering from the IncidentSvc
130 m_incidentSvc->removeListener( this, IncidentType::BeginEvent );
131 m_incidentSvc.reset();
132 return base_class::finalize();
133 }
134
135 // Declaration of the factory
136 DECLARE_COMPONENT( StalledEventMonitor )
MsgStream
Definition of the MsgStream class used to transmit messages.
Definition: MsgStream.h:24
SmartIF< IMessageSvc >
Service::serviceLocator
SmartIF< ISvcLocator > & serviceLocator() const override
Retrieve pointer to service locator.
Definition: Service.cpp:277
StalledEventMonitor::stop
StatusCode stop() override
Stop the watchdog thread (after the event loop).
Definition: StalledEventMonitor.cpp:120
CommonMessagingBase::msgSvc
const SmartIF< IMessageSvc > & msgSvc() const
The standard message service.
Definition: CommonMessaging.h:68
WatchdogThread::start
void start()
Start the watchdog thread.
Definition: WatchdogThread.cpp:23
WatchdogThread::onStop
virtual void onStop()
User implemented function that will be called when stopping.
Definition: WatchdogThread.cpp:87
gaudiComponentHelp.name
name
Definition: gaudiComponentHelp.py:41
CommonMessagingBase::warning
MsgStream & warning() const
shortcut for the method msgStream(MSG::WARNING)
Definition: CommonMessaging.h:101
StalledEventMonitor::m_eventTimeout
Gaudi::Property< unsigned int > m_eventTimeout
Definition: StalledEventMonitor.h:51
StalledEventMonitor::m_watchdog
std::unique_ptr< WatchdogThread > m_watchdog
Pointer to the watchdog thread that checks for the event timeout.
Definition: StalledEventMonitor.h:58
StalledEventMonitor::handle
void handle(const Incident &) override
Notify the watchdog thread for a new event.
Definition: StalledEventMonitor.cpp:115
StalledEventMonitor::m_stackTrace
Gaudi::Property< bool > m_stackTrace
Definition: StalledEventMonitor.h:55
std::endl
T endl(T... args)
StatusCode::SUCCESS
constexpr static const auto SUCCESS
Definition: StatusCode.h:85
StalledEventMonitor::start
StatusCode start() override
Start the watchdog thread (before entering the event loop).
Definition: StalledEventMonitor.cpp:109
WatchdogThread::ping
void ping()
Function to call to notify the watchdog thread that we are still alive.
Definition: WatchdogThread.h:46
AvalancheSchedulerErrorTest.msgSvc
msgSvc
Definition: AvalancheSchedulerErrorTest.py:67
std::string
STL class.
DECLARE_COMPONENT
#define DECLARE_COMPONENT(type)
Definition: PluginServiceV1.h:46
MSG::FATAL
Definition: IMessageSvc.h:15
ISvcLocator::service
StatusCode service(const Gaudi::Utils::TypeNameString &name, T *&svc, bool createIf=true)
Templated method to access a service by name.
Definition: ISvcLocator.h:76
Service::name
const std::string & name() const override
Retrieve name of the service.
Definition: Service.cpp:274
StalledEventMonitor
Service that monitor the time taken by processing of single events using a separate thread.
Definition: StalledEventMonitor.h:29
StalledEventMonitor::m_incidentSvc
SmartIF< IIncidentSvc > m_incidentSvc
Pointer to the incident service.
Definition: StalledEventMonitor.h:61
CommonMessagingBase::error
MsgStream & error() const
shortcut for the method msgStream(MSG::ERROR)
Definition: CommonMessaging.h:98
IIncidentSvc.h
StatusCode
This class is used for returning status codes from appropriate routines.
Definition: StatusCode.h:50
CommonMessagingBase::debug
MsgStream & debug() const
shortcut for the method msgStream(MSG::DEBUG)
Definition: CommonMessaging.h:107
std::unique_ptr::reset
T reset(T... args)
std::raise
T raise(T... args)
WatchdogThread::onPing
virtual void onPing()
User implemented function that will be called when ping is called.
Definition: WatchdogThread.cpp:81
StalledEventMonitor::initialize
StatusCode initialize() override
Initialization of the service.
Definition: StalledEventMonitor.cpp:82
Memory.h
WatchdogThread
Simple class for asynchronous check of time-out.
Definition: WatchdogThread.h:27
WatchdogThread::stop
void stop()
Signal the watchdog thread to stop and wait for it.
Definition: WatchdogThread.cpp:35
Incident
Base class for all Incidents (computing events).
Definition: Incident.h:17
IIncidentSvc::addListener
virtual void addListener(IIncidentListener *lis, const std::string &type="", long priority=0, bool rethrow=false, bool singleShot=false)=0
Add listener.
StatusCode::FAILURE
constexpr static const auto FAILURE
Definition: StatusCode.h:86
System::virtualMemory
GAUDI_API long virtualMemory(MemoryUnit unit=kByte, InfoType fetch=Memory, long pid=-1)
Basic Process Information: priority boost.
Definition: Memory.cpp:218
StalledEventMonitor::finalize
StatusCode finalize() override
Finalization of the service.
Definition: StalledEventMonitor.cpp:126
MSG::WARNING
Definition: IMessageSvc.h:15
SmartIF::reset
void reset(TYPE *ptr=nullptr)
Set the internal pointer to the passed one disposing of the old one.
Definition: SmartIF.h:86
IIncidentSvc::removeListener
virtual void removeListener(IIncidentListener *lis, const std::string &type="")=0
Remove listener.
StatusCode::isFailure
bool isFailure() const
Definition: StatusCode.h:130
WatchdogThread::action
virtual void action()
User implemented function that will be called if the time-out is reached.
Definition: WatchdogThread.cpp:78
StalledEventMonitor.h
MSG::INFO
Definition: IMessageSvc.h:15
std::cerr
System::pagedMemory
GAUDI_API long pagedMemory(MemoryUnit unit=kByte, InfoType fetch=Memory, long pid=-1)
Basic Process Information: Amount of paged memory currently occupied by the process 'pid'.
Definition: Memory.cpp:131
WatchdogThread.h
endmsg
MsgStream & endmsg(MsgStream &s)
MsgStream Modifier: endmsg. Calls the output method of the MsgStream.
Definition: MsgStream.h:192
WatchdogThread::getTimeout
boost::posix_time::time_duration getTimeout() const
Get the current time-out value.
Definition: WatchdogThread.h:56
Gaudi.Configuration.log
log
Definition: Configuration.py:13
StalledEventMonitor::m_maxTimeoutCount
Gaudi::Property< int > m_maxTimeoutCount
Definition: StalledEventMonitor.h:53