| The Gaudi Framework v27r1

- Main Page
- Related Pages
- Modules
- Namespaces
- Classes
- Files

- File List
- File Members

StalledEventMonitor.cpp
Go to the documentation of this file.
1 /*
2 * StalledEventMonitor.cpp
3 *
4 * Created on: Apr 19, 2010
5 * Author: Marco Clemencic
6 */
7
8 // Include files
9 #include "StalledEventMonitor.h"
10
11 #include "GaudiKernel/WatchdogThread.h"
12 #include "GaudiKernel/IIncidentSvc.h"
13 #include "GaudiKernel/Memory.h"
14
15 #include "TSystem.h"
16
17 #include <csignal>
18
19 namespace {
21 class EventWatchdog: public WatchdogThread {
22 public:
23 EventWatchdog(const SmartIF<IMessageSvc> &msgSvc,
24 const std::string &name,
25 boost::posix_time::time_duration timeout,
26 bool stackTrace = false,
27 long maxCount = 0,
28 bool autostart = false):
29 WatchdogThread(timeout, autostart),
30 log(msgSvc, name),
31 m_maxCount(maxCount),
32 m_stackTrace(stackTrace){}
33 ~EventWatchdog() override = default;
34 private:
36 MsgStream log;
38 long m_counter = 0;
40 long m_maxCount = 0;
42 bool m_stackTrace = false;
44 void action() override {
45 if (!m_counter) {
46 log << MSG::WARNING << "More than " << getTimeout().total_seconds()
47 << "s since the last " << IncidentType::BeginEvent << endmsg;
48 } else {
49 log << MSG::WARNING << "Other " << getTimeout().total_seconds()
50 << "s passed" << endmsg;
51 }
52 log << MSG::INFO << "Current memory usage is"
53 " virtual size = " << System::virtualMemory() / 1024. << " MB"
54 ", resident set size = " << System::pagedMemory() / 1024.<< " MB"
55 << endmsg;
56 if (m_stackTrace && gSystem) {
57 // TSystem::StackTrace() prints on the standard error, so we
58 std::cerr << "=== Stalled event: current stack trace ===" << std::endl;
59 gSystem->StackTrace();
60 }
61 ++m_counter;
62 if (m_maxCount > 0 && m_counter >= m_maxCount) {
63 log << MSG::FATAL << "too much time on a single event: aborting process" << endmsg;
64 std::raise(SIGABRT);
65 }
66 }
67 void onPing() override {
68 if (m_counter) {
69 if (m_counter >= 3)
70 log << MSG::INFO << "Starting a new event after ~"
71 << m_counter * getTimeout().total_seconds() << "s" << endmsg;
72 m_counter = 0;
73 }
74 }
75 void onStop() override {
76 if (m_counter >= 3)
77 log << MSG::INFO << "The last event took ~"
78 << m_counter * getTimeout().total_seconds() << "s" << endmsg;
79 }
80 };
81 }
82
83 // Constructor
84 StalledEventMonitor::StalledEventMonitor(const std::string& name, ISvcLocator* svcLoc):
85 base_class(name, svcLoc) {
86
87 declareProperty("EventTimeout", m_eventTimeout = 600,
88 "Number of seconds allowed to process a single event (0 to disable the check).");
89
90 declareProperty("MaxTimeoutCount", m_maxTimeoutCount = 0,
91 "Number timeouts before aborting the execution (0 means never abort).");
92
93 declareProperty("StackTrace", m_stackTrace = false,
94 "Whether to print the stack-trace on timeout.");
95 }
96
97 // Initialization of the service.
98 StatusCode StalledEventMonitor::initialize() {
99 StatusCode sc = base_class::initialize();
100 if (sc.isFailure()) return sc;
101
102
103 if (m_eventTimeout) {
104 // create the watchdog thread
105 m_watchdog.reset( new EventWatchdog(msgSvc(),
106 "EventWatchdog",
107 boost::posix_time::seconds(m_eventTimeout),
108 m_stackTrace,
109 m_maxTimeoutCount));
110
111 // register to the incident service
112 static const std::string serviceName = "IncidentSvc";
113 m_incidentSvc = serviceLocator()->service(serviceName);
114 if ( ! m_incidentSvc ) {
115 error() << "Cannot retrieve " << serviceName << endmsg;
116 return StatusCode::FAILURE;
117 }
118 debug() << "Register to the IncidentSvc" << endmsg;
119 m_incidentSvc->addListener(this, IncidentType::BeginEvent);
120 } else {
121 warning() << "StalledEventMonitor/" << name()
122 << " instantiated with 0 time-out: no monitoring performed" << endmsg;
123 }
124
125 return StatusCode::SUCCESS;
126 }
127
128 // Start the monitoring.
129 StatusCode StalledEventMonitor::start() {
130 if (m_watchdog) m_watchdog->start();
131 return StatusCode::SUCCESS;
132 }
133
134 // Notify the watchdog that a new event has been started
135 void StalledEventMonitor::handle(const Incident& /* incident */) {
136 if (m_watchdog) m_watchdog->ping();
137 }
138
139 // Start the monitoring.
140 StatusCode StalledEventMonitor::stop() {
141 if (m_watchdog) m_watchdog->stop();
142 return StatusCode::SUCCESS;
143 }
144
145 // Finalization of the service.
146 StatusCode StalledEventMonitor::finalize() {
147 // destroy the watchdog thread (if any)
148 m_watchdog.reset();
149 // unregistering from the IncidentSvc
150 m_incidentSvc->removeListener(this, IncidentType::BeginEvent);
151 m_incidentSvc.reset();
152 return base_class::finalize();
153 }
154
155 // Declaration of the factory
156 DECLARE_COMPONENT(StalledEventMonitor)
MsgStream
Definition of the MsgStream class used to transmit messages.
Definition: MsgStream.h:24
SmartIF< IMessageSvc >
Service::serviceLocator
SmartIF< ISvcLocator > & serviceLocator() const override
Retrieve pointer to service locator.
Definition: Service.cpp:324
AnalysisTest.initialize
def initialize()
Definition: AnalysisTest.py:12
ISvcLocator
The ISvcLocator is the interface implemented by the Service Factory in the Application Manager to loc...
Definition: ISvcLocator.h:25
StalledEventMonitor::stop
StatusCode stop() override
Stop the watchdog thread (after the event loop).
Definition: StalledEventMonitor.cpp:140
WatchdogThread::getTimeout
boost::posix_time::time_duration getTimeout() const
Get the current time-out value.
Definition: WatchdogThread.h:56
WatchdogThread::start
void start()
Start the watchdog thread.
Definition: WatchdogThread.cpp:24
WatchdogThread::onStop
virtual void onStop()
User implemented function that will be called when stopping.
Definition: WatchdogThread.cpp:91
gaudiComponentHelp.name
name
Definition: gaudiComponentHelp.py:28
Gaudi.Configuration.log
tuple log
Definition: Configuration.py:13
StalledEventMonitor::m_watchdog
std::unique_ptr< WatchdogThread > m_watchdog
Pointer to the watchdog thread that checks for the event timeout.
Definition: StalledEventMonitor.h:65
StalledEventMonitor::handle
void handle(const Incident &) override
Notify the watchdog thread for a new event.
Definition: StalledEventMonitor.cpp:135
std::endl
T endl(T...args)
StalledEventMonitor::start
StatusCode start() override
Start the watchdog thread (before entering the event loop).
Definition: StalledEventMonitor.cpp:129
WatchdogThread::ping
void ping()
Function to call to notify the watchdog thread that we are still alive.
Definition: WatchdogThread.h:44
merge_files.sc
int sc
Definition: merge_files.py:167
StatusCode::isFailure
bool isFailure() const
Test for a status code of FAILURE.
Definition: StatusCode.h:86
DECLARE_COMPONENT
#define DECLARE_COMPONENT(type)
Definition: PluginService.h:36
std::string
STL class.
MSG::FATAL
Definition: IMessageSvc.h:22
ISvcLocator::service
StatusCode service(const Gaudi::Utils::TypeNameString &name, T *&svc, bool createIf=true)
Templated method to access a service by name.
Definition: ISvcLocator.h:78
Service::name
const std::string & name() const override
Retrieve name of the service.
Definition: Service.cpp:319
CommonMessagingBase::error
MsgStream & error() const
shortcut for the method msgStream(MSG::ERROR)
Definition: CommonMessaging.h:118
StalledEventMonitor
Service that monitor the time taken by processing of single events using a separate thread...
Definition: StalledEventMonitor.h:29
StalledEventMonitor::m_incidentSvc
SmartIF< IIncidentSvc > m_incidentSvc
Pointer to the incident service.
Definition: StalledEventMonitor.h:68
CommonMessagingBase::warning
MsgStream & warning() const
shortcut for the method msgStream(MSG::WARNING)
Definition: CommonMessaging.h:121
StatusCode::FAILURE
Definition: StatusCode.h:29
IIncidentSvc.h
StatusCode::SUCCESS
Definition: StatusCode.h:30
StatusCode
This class is used for returning status codes from appropriate routines.
Definition: StatusCode.h:26
StalledEventMonitor::m_maxTimeoutCount
int m_maxTimeoutCount
Number timeouts before aborting the execution (0 means never abort).
Definition: StalledEventMonitor.h:59
StalledEventMonitor::m_eventTimeout
unsigned int m_eventTimeout
Number of seconds allowed to process a single event.
Definition: StalledEventMonitor.h:56
std::unique_ptr::reset
T reset(T...args)
std::raise
T raise(T...args)
StalledEventMonitor::StalledEventMonitor
StalledEventMonitor(const std::string &name, ISvcLocator *svcLoc)
Constructor.
Definition: StalledEventMonitor.cpp:84
WatchdogThread::onPing
virtual void onPing()
User implemented function that will be called when ping is called.
Definition: WatchdogThread.cpp:83
StalledEventMonitor::initialize
StatusCode initialize() override
Initialization of the service.
Definition: StalledEventMonitor.cpp:98
Memory.h
WatchdogThread
Simple class for asynchronous check of time-out.
Definition: WatchdogThread.h:25
CommonMessagingBase::debug
MsgStream & debug() const
shortcut for the method msgStream(MSG::DEBUG)
Definition: CommonMessaging.h:127
WatchdogThread::stop
void stop()
Signal the watchdog thread to stop and wait for it.
Definition: WatchdogThread.cpp:36
Incident
Base class for all Incidents (computing events).
Definition: Incident.h:17
IIncidentSvc::addListener
virtual void addListener(IIncidentListener *lis, const std::string &type="", long priority=0, bool rethrow=false, bool singleShot=false)=0
Add listener.
CommonMessaging
Definition: CommonMessaging.h:58
CommonMessagingBase::msgSvc
SmartIF< IMessageSvc > & msgSvc() const
The standard message service.
Definition: CommonMessaging.h:73
System::virtualMemory
GAUDI_API long virtualMemory(MemoryUnit unit=kByte, InfoType fetch=Memory, long pid=-1)
Basic Process Information: priority boost.
Definition: Memory.cpp:212
StalledEventMonitor::finalize
StatusCode finalize() override
Finalization of the service.
Definition: StalledEventMonitor.cpp:146
MSG::WARNING
Definition: IMessageSvc.h:20
SmartIF::reset
void reset(TYPE *ptr=nullptr)
Set the internal pointer to the passed one disposing of the old one.
Definition: SmartIF.h:88
IIncidentSvc::removeListener
virtual void removeListener(IIncidentListener *lis, const std::string &type="")=0
Remove listener.
Service::declareProperty
Property * declareProperty(const std::string &name, T &property, const std::string &doc="none") const
Declare the named property.
Definition: Service.h:215
WatchdogThread::action
virtual void action()
User implemented function that will be called if the time-out is reached.
Definition: WatchdogThread.cpp:79
StalledEventMonitor.h
MSG::INFO
Definition: IMessageSvc.h:19
std::cerr
System::pagedMemory
GAUDI_API long pagedMemory(MemoryUnit unit=kByte, InfoType fetch=Memory, long pid=-1)
Basic Process Information: Amount of paged memory currently occupied by the process 'pid'...
Definition: Memory.cpp:121
WatchdogThread.h
endmsg
MsgStream & endmsg(MsgStream &s)
MsgStream Modifier: endmsg. Calls the output method of the MsgStream.
Definition: MsgStream.h:244
ForwardSchedulerSimpleTest.msgSvc
tuple msgSvc
Definition: ForwardSchedulerSimpleTest.py:112