| # Gaudi Framework, version v25r0 |

Home |  | Generated: Mon Feb 17 2014

- Main Page
- Related Pages
- Modules
- Namespaces
- Classes
- Files

- File List
- File Members

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
PerfMonAuditor.cpp
Go to the documentation of this file.
1 #include "GaudiKernel/Auditor.h"
2 #include "GaudiKernel/IToolSvc.h"
3 #include "GaudiKernel/IIncidentListener.h"
4 #include "GaudiKernel/IIncidentSvc.h"
5 #include "GaudiKernel/IToolSvc.h"
6 #include "GaudiKernel/VectorMap.h"
7 #include "GaudiKernel/HashMap.h"
8 #include "GaudiKernel/MsgStream.h"
9
10 /*BEGIN: perfmon*/
11 #include <iostream>
12 #include <string>
13 #include <cstring>
14 #include <fstream>
15 #include <perfmon/pfmlib.h>
16 #include <perfmon/pfmlib_core.h>
17 #include <perfmon/pfmlib_intel_nhm.h>
18 #include <vector>
19 #include <map>
20 #include <utility>
21 #include <sstream>
22
23 #include <perfmon/perfmon.h>
24 #include <perfmon/perfmon_dfl_smpl.h>
25
27
29
30
31
32 #include <sys/types.h>
33 #include <stdio.h>
34 #include <stdlib.h>
35 #include <stdarg.h>
36 #include <errno.h>
37 #include <unistd.h>
38 #include <string.h>
39 #include <signal.h>
40 #include <stdarg.h>
41 #include <stdint.h>
42 #include <getopt.h>
43 #include <time.h>
44 #include <sys/ptrace.h>
45 #include <sys/wait.h>
46 #include <sys/mman.h>
47 #include <sys/time.h>
48 #include <sys/resource.h>
49 #include <unistd.h>
50 #include <fcntl.h>
51 #include <zlib.h>
52
53 #include <algorithm>
54 #include <list>
55 #include <stack>
56 #include <cmath>
57 #include <sys/stat.h>
58 #include "IgHook_IgHookTrace.h"
59
60 // dlopen (link with -ldl)
61 #include <dlfcn.h>
62
63
64 #define MAX_EVT_NAME_LEN 256
65 #define NUM_PMCS PFMLIB_MAX_PMCS
66 #define NUM_PMDS PFMLIB_MAX_PMDS
67 #define FMT_NAME PFM_DFL_SMPL_NAME
68 #define BPL (sizeof(uint64_t)<<3)
69 #define LBPL 6
70
71 #define SYM_NAME_MAX_LENGTH 10000
72 #define MAX_OUTPUT_FILENAME_LENGTH 1024
73 #define MAX_EVENT_NAME_LENGTH 500
74 #define MAX_PREFIX_NAME_LENGTH 1024
75 #define FILENAME_MAX_LENGTH 1024
76
77 #define MAX_NUMBER_OF_PROGRAMMABLE_COUNTERS 4
78
79 #define cpuid(func,ax,bx,cx,dx) __asm__ __volatile__ ("cpuid": "=a" (ax), "=b" (bx), "=c" (cx), "=d" (dx) : "a" (func));
80
81
82 static pfarg_pmd_t pd_smpl[NUM_PMDS];
83 static uint64_t collected_samples, collected_partial;
84 static int ctx_fd;
85 static pfm_dfl_smpl_hdr_t *hdr;
86 static uint64_t ovfl_count;
87 static size_t entry_size;
88 static unsigned int num_smpl_pmds;
89 static std::vector<std::map<std::string, std::map<unsigned long, unsigned int> > > samples(MAX_NUMBER_OF_PROGRAMMABLE_COUNTERS); //a map of modules each containing numbers of samples of their addresses
90 static std::vector<std::map<std::string, std::vector<unsigned long int> > > results(MAX_NUMBER_OF_PROGRAMMABLE_COUNTERS); //a map of modules and their result values across multiple events
91 static uint64_t last_overflow;
92 static uint64_t last_count;
93 static int sp[MAX_NUMBER_OF_PROGRAMMABLE_COUNTERS];
94
95 static std::stack<std::pair<INamedInterface *, std::vector<unsigned long int> > > alg_stack;
96 /*END: perfmon*/
97
98
99
100 namespace {
102 template <typename T>
103 inline T function_cast(void *p) {
104 union {
105 void* object;
106 T function;
107 } caster;
108 caster.object = p;
109 return caster.function;
110 }
111 class PFMon {
112 public:
113 bool loaded;
114 typedef void (*pfm_stop_t)(int);
115 pfm_stop_t pfm_stop;
116 typedef void (*pfm_self_stop_t)(int);
117 pfm_self_stop_t pfm_self_stop;
118 typedef os_err_t (*pfm_restart_t)(int);
119 pfm_restart_t pfm_restart;
120 typedef int (*pfm_read_pmds_t)(int, pfarg_pmd_t*, int);
121 pfm_read_pmds_t pfm_read_pmds;
122 typedef pfm_err_t (*pfm_initialize_t)();
123 pfm_initialize_t pfm_initialize;
124 typedef pfm_err_t (*pfm_find_full_event_t)(const char *, pfmlib_event_t *);
125 pfm_find_full_event_t pfm_find_full_event;
126 typedef pfm_err_t (*pfm_dispatch_events_t)(pfmlib_input_param_t *, void *, pfmlib_output_param_t *, void *);
127 pfm_dispatch_events_t pfm_dispatch_events;
128 typedef os_err_t (*pfm_create_context_t)(pfarg_ctx_t *, char *, void *, size_t);
129 pfm_create_context_t pfm_create_context;
130 typedef os_err_t (*pfm_write_pmcs_t)(int, pfarg_pmc_t *, int);
131 pfm_write_pmcs_t pfm_write_pmcs;
132 typedef os_err_t (*pfm_write_pmds_t)(int, pfarg_pmd_t *, int);
133 pfm_write_pmds_t pfm_write_pmds;
134 typedef os_err_t (*pfm_load_context_t)(int, pfarg_load_t *);
135 pfm_load_context_t pfm_load_context;
136 typedef os_err_t (*pfm_start_t)(int fd, pfarg_start_t *);
137 pfm_start_t pfm_start;
138 typedef char* (*pfm_strerror_t)(int);
139 pfm_strerror_t pfm_strerror;
140 typedef pfm_err_t (*pfm_set_options_t)(pfmlib_options_t *);
141 pfm_set_options_t pfm_set_options;
142 typedef pfm_err_t (*pfm_get_num_counters_t)(unsigned int *);
143 pfm_get_num_counters_t pfm_get_num_counters;
144 static PFMon &instance() {
145 return s_instance;
146 }
147 private:
148 // static void failure() { throw 1; }
149
150 void* handle;
151
152 PFMon() {
153 handle = dlopen("libpfm.so", RTLD_NOW);
154 if (handle) { loaded = true; } else { loaded = false; }
155 if (loaded) {
156 pfm_start = function_cast<pfm_start_t>(dlsym(handle, "pfm_start"));
157 pfm_stop = function_cast<pfm_stop_t>(dlsym(handle, "pfm_stop"));
158 pfm_self_stop = function_cast<pfm_self_stop_t>(dlsym(handle, "pfm_stop")); //it's the same
159 pfm_restart = function_cast<pfm_restart_t>(dlsym(handle, "pfm_restart"));
160 pfm_read_pmds = function_cast<pfm_read_pmds_t>(dlsym(handle, "pfm_read_pmds"));
161 pfm_initialize = function_cast<pfm_initialize_t>(dlsym(handle, "pfm_initialize"));
162 pfm_find_full_event = function_cast<pfm_find_full_event_t>(dlsym(handle, "pfm_find_full_event"));
163 pfm_dispatch_events = function_cast<pfm_dispatch_events_t>(dlsym(handle, "pfm_dispatch_events"));
164 pfm_create_context = function_cast<pfm_create_context_t>(dlsym(handle, "pfm_create_context"));
165 pfm_write_pmcs = function_cast<pfm_write_pmcs_t>(dlsym(handle, "pfm_write_pmcs"));
166 pfm_write_pmds = function_cast<pfm_write_pmds_t>(dlsym(handle, "pfm_write_pmds"));
167 pfm_load_context = function_cast<pfm_load_context_t>(dlsym(handle, "pfm_load_context"));
168 pfm_strerror = function_cast<pfm_strerror_t>(dlsym(handle, "pfm_strerror"));
169 pfm_set_options = function_cast<pfm_set_options_t>(dlsym(handle, "pfm_set_options"));
170 pfm_get_num_counters = function_cast<pfm_get_num_counters_t>(dlsym(handle, "pfm_get_num_counters"));
171 } else {
172 // pfm_start = pfm_stop = pfm_self_stop = pfm_restart = pfm_read_pmds = pfm_initialize = pfm_find_full_event = pfm_dispatch_events = pfm_create_context = pfm_write_pmcs = pfm_write_pmds = pfm_load_context = pfm_strerror = pfm_set_options = pfm_get_num_counters = failure;
173 }
174 }
175 ~PFMon() {
176 if (handle) dlclose(handle);
177 }
178
179 static PFMon s_instance;
180 };
181
182 PFMon PFMon::s_instance;
183 }
184
185
186
187 // ============================================================================
188 // GaudiAlg
189 // ============================================================================
190 // ============================================================================
200 class PerfMonAuditor: virtual public Auditor
201 {
202 public:
203 virtual void before(StandardEventType evt, INamedInterface* alg);
204 virtual void after(StandardEventType evt, INamedInterface* alg, const StatusCode &sc);
205 using Auditor::before;
206 using Auditor::after;
207
208 private:
209 void i_beforeInitialize(INamedInterface* alg);
210 void i_afterInitialize(INamedInterface* alg);
211 void i_beforeExecute(INamedInterface* alg);
212 void i_afterExecute(INamedInterface* alg);
213
214 public:
215 virtual StatusCode initialize();
216 virtual StatusCode finalize();
217 int is_nehalem() {
218 #ifdef __ICC
219 // Disable ICC remark #593: variable "x" was set but never used
220 #pragma warning(push)
221 #pragma warning(disable:593)
222 #endif
223 int a,b,c,d;
224 cpuid(1,a,b,c,d);
225 int sse4_2_mask = 1 << 20;
226 if(c & sse4_2_mask) return 1; else return 0;
227 #ifdef __ICC
228 #pragma warning(pop)
229 #endif
230 }
231
232 private:
233 PFMon &m_pfm;
234 /*
235 typedef void (*pfm_stop_t)(int);
236 pfm_stop_t pfm_stop;
237 typedef void (*pfm_self_stop_t)(int);
238 pfm_self_stop_t pfm_self_stop;
239
240 typedef os_err_t (*pfm_restart_t)(int);
241 pfm_restart_t pfm_restart;
242
243 //typedef int (*pfm_read_pmds_t)(int, pfarg_pmd_t*, int);
244 //pfm_read_pmds_t pfm_read_pmds;
245
246 typedef pfm_err_t (*pfm_initialize_t)();
247 pfm_initialize_t pfm_initialize;
248 typedef pfm_err_t (*pfm_find_full_event_t)(const char *, pfmlib_event_t *);
249 pfm_find_full_event_t pfm_find_full_event;
250 typedef pfm_err_t (*pfm_dispatch_events_t)(pfmlib_input_param_t *, void *, pfmlib_output_param_t *, void *);
251 pfm_dispatch_events_t pfm_dispatch_events;
252 typedef os_err_t (*pfm_create_context_t)(pfarg_ctx_t *, char *, void *, size_t);
253 pfm_create_context_t pfm_create_context;
254 typedef os_err_t (*pfm_write_pmcs_t)(int, pfarg_pmc_t *, int);
255 pfm_write_pmcs_t pfm_write_pmcs;
256 typedef os_err_t (*pfm_write_pmds_t)(int, pfarg_pmd_t *, int);
257 pfm_write_pmds_t pfm_write_pmds;
258 typedef os_err_t (*pfm_load_context_t)(int, pfarg_load_t *);
259 pfm_load_context_t pfm_load_context;
260 typedef os_err_t (*pfm_start_t)(int fd, pfarg_start_t *);
261 pfm_start_t pfm_start;
262 typedef char* (*pfm_strerror_t)(int);
263 pfm_strerror_t pfm_strerror;
264 typedef pfm_err_t (*pfm_set_options_t)(pfmlib_options_t *);
265 pfm_set_options_t pfm_set_options;
266 typedef pfm_err_t (*pfm_get_num_counters_t)(unsigned int *);
267 pfm_get_num_counters_t pfm_get_num_counters;
268 */
269
270
271 public:
272 PerfMonAuditor(const std::string& name, ISvcLocator* pSvc): // standard constructor
273 Auditor(name, pSvc),
274 m_pfm(PFMon::instance()),
275 m_map(),
276 m_indent(0),
277 m_inEvent(false)
278 {
279 is_nehalem_ret = is_nehalem();
280 declareProperty("EVENT0", event_str[0]);
281 declareProperty("EVENT1", event_str[1]);
282 declareProperty("EVENT2", event_str[2]);
283 declareProperty("EVENT3", event_str[3]);
284 declareProperty("FAMILY", family);
285 declareProperty("PREFIX", prefix);
286 declareProperty("INV0", inv[0]);
287 declareProperty("INV1", inv[1]);
288 declareProperty("INV2", inv[2]);
289 declareProperty("INV3", inv[3]);
290 declareProperty("CMASK0", cmask[0]);
291 declareProperty("CMASK1", cmask[1]);
292 declareProperty("CMASK2", cmask[2]);
293 declareProperty("CMASK3", cmask[3]);
294 declareProperty("SP0", sp[0]);
295 declareProperty("SP1", sp[1]);
296 declareProperty("SP2", sp[2]);
297 declareProperty("SP3", sp[3]);
298 declareProperty("SAMPLE", sampling);
299 declareProperty("START_AT_EVENT", start_at_event);
300 declareProperty("IS_NEHALEM", is_nehalem_ret);
301
302 // MsgStream log(msgSvc(), name());
303
305 /*
306 // loading functions from PFM library
307 void* handle = dlopen("libpfm.so", RTLD_NOW);
308 if (!handle) {
309 // log << MSG::ERROR << "Cannot open library: " << dlerror() << endmsg;
310 }
311 typedef void (*hello_t)();
312 hello_t hello = (hello_t) dlsym(handle, "hello");
313 if (!hello) {
314 // log << MSG::ERROR << "Cannot load symbol 'hello': " << dlerror() << endmsg;
315 dlclose(handle);
316 }
317
318 pfm_start = (pfm_start_t) dlsym(handle, "pfm_start");
319 pfm_stop = (pfm_stop_t) dlsym(handle, "pfm_stop");
320 pfm_self_stop = (pfm_self_stop_t) dlsym(handle, "pfm_stop"); //it's the same
321 pfm_restart = (pfm_restart_t) dlsym(handle, "pfm_restart");
322 //pfm_read_pmds = (pfm_read_pmds_t) dlsym(handle, "pfm_read_pmds");
323 pfm_initialize = (pfm_initialize_t) dlsym(handle, "pfm_initialize");
324 pfm_find_full_event = (pfm_find_full_event_t) dlsym(handle, "pfm_find_full_event");
325 pfm_dispatch_events = (pfm_dispatch_events_t) dlsym(handle, "pfm_dispatch_events");
326 pfm_create_context = (pfm_create_context_t) dlsym(handle, "pfm_create_context");
327 pfm_write_pmcs = (pfm_write_pmcs_t) dlsym(handle, "pfm_write_pmcs");
328 pfm_write_pmds = (pfm_write_pmds_t) dlsym(handle, "pfm_write_pmds");
329 pfm_load_context = (pfm_load_context_t) dlsym(handle, "pfm_load_context");
330 pfm_strerror = (pfm_strerror_t) dlsym(handle, "pfm_strerror");
331 pfm_set_options = (pfm_set_options_t) dlsym(handle, "pfm_set_options");
332 pfm_get_num_counters = (pfm_get_num_counters_t) dlsym(handle, "pfm_get_num_counters");
333 // use it to do the calculation
334 // log << MSG::INFO << "Calling hello..." << endmsg;
335 // hello();
336
337 // close the library
338 // log << MSG::INFO << "Closing library..." << endmsg;
339 dlclose(handle);
340 */
341
343
344 }
345
346 virtual ~PerfMonAuditor() {} // virtual destructor
347
348 private:
349 PerfMonAuditor(); // the default constructor is disabled
350 PerfMonAuditor(const PerfMonAuditor&); // copy constructor is disabled
351 PerfMonAuditor& operator=(const PerfMonAuditor&); // assignement operator is disabled
352
353 private:
354 typedef GaudiUtils::VectorMap<const INamedInterface*,int> Map;
355 Map m_map;
356 int m_indent; // indentation level
357 bool m_inEvent; // "In event" flag
358
359 private:
360 int is_nehalem_ret;
361
362 pfmlib_input_param_t inp;
363 pfmlib_output_param_t outp;
364 pfarg_ctx_t ctx;
365 pfarg_pmd_t pd[NUM_PMDS];
366 pfarg_pmc_t pc[NUM_PMCS];
367 pfarg_load_t load_arg;
368 int fd;
369 unsigned int i;
370 int ret;
371 void startpm();
372 void pausepm();
373 void stoppm();
374 void finalizepm();
375 std::string event_str[MAX_NUMBER_OF_PROGRAMMABLE_COUNTERS];
376 std::string prefix;
377 std::string family;
378 char event_cstr[MAX_NUMBER_OF_PROGRAMMABLE_COUNTERS][MAX_EVENT_NAME_LENGTH];
379 char prefix_cstr[MAX_PREFIX_NAME_LENGTH];
380 unsigned int ph_ev_count;
381 bool inv[MAX_NUMBER_OF_PROGRAMMABLE_COUNTERS];
382 unsigned int cmask[MAX_NUMBER_OF_PROGRAMMABLE_COUNTERS];
383 unsigned int start_at_event;
384 pfmlib_core_input_param_t params;
385 pfmlib_nhm_input_param_t nhm_params;
386 int used_counters_number;
387 bool nehalem;
388 bool westmere;
389 bool core;
390
391 bool sampling;
392 int detect_unavail_pmu_regs(int fd, pfmlib_regmask_t *r_pmcs, pfmlib_regmask_t *r_pmds);
393 int detect_unavail_pmcs(int fd, pfmlib_regmask_t *r_pmcs){return detect_unavail_pmu_regs(fd, r_pmcs, NULL);}
394 void pfm_bv_set(uint64_t *bv, uint16_t rnum){bv[rnum>>LBPL] |= 1UL << (rnum&(BPL-1));}
395 int pfm_bv_isset(uint64_t *bv, uint16_t rnum){return bv[rnum>>LBPL] & (1UL <<(rnum&(BPL-1))) ? 1 : 0;}
396 void pfm_bv_copy(uint64_t *d, uint64_t *j, uint16_t n){if(n<=BPL) *d = *j; else {memcpy(d, j, (n>>LBPL)*sizeof(uint64_t));}}
397 static void process_smpl_buf(pfm_dfl_smpl_hdr_t *hdr, size_t entry_size);
398 static void sigio_handler(int, struct siginfo *, struct sigcontext *); // dlopen ==>
399 //void sigio_handler(int, struct siginfo *, struct sigcontext *);
400 void start_smpl();
401 void stop_smpl();
402 void finalize_smpl();
403 pfm_dfl_smpl_arg_t buf_arg;
404 pfarg_load_t load_args;
405 void *buf_addr;
406 unsigned num_counters;
407 unsigned int max_pmd;
408 pfmlib_options_t pfmlib_options;
409
410
411 int level;
412
413 bool first_alg;
414 std::string first_alg_name;
415 bool event_count_reached;
416
417
418 };
419
420 void PerfMonAuditor::startpm()
421 {
422 MsgStream log(msgSvc(), name());
423 memset(&ctx,0, sizeof(ctx));
424 memset(&inp,0, sizeof(inp));
425 memset(&outp,0, sizeof(outp));
426 memset(pd, 0, sizeof(pd));
427 memset(pc, 0, sizeof(pc));
428 memset(&load_arg, 0, sizeof(load_arg));
429 memset(&params, 0, sizeof(params));
430 memset(&nhm_params, 0, sizeof(nhm_params));
431
432 for(int i=0; i<used_counters_number; i++)
433 {
434 ret = m_pfm.pfm_find_full_event(event_cstr[i], &inp.pfp_events[i]);
435 if(ret != PFMLIB_SUCCESS)
436 {
437 log << MSG::ERROR << "ERROR: cannot find event: " << event_cstr[i] << ". Aborting..." << endmsg;
438 }
439 }
440 inp.pfp_dfl_plm = PFM_PLM3;
441 inp.pfp_event_count = 4;
442 for(int i=0; i<used_counters_number; i++)
443 {
444 if(inv[i])
445 {
446 (params.pfp_core_counters[i]).flags |= PFM_CORE_SEL_INV;
447 (nhm_params.pfp_nhm_counters[i]).flags |= PFM_NHM_SEL_INV;
448 }
449 if(cmask[i]>0)
450 {
451 (params.pfp_core_counters[i]).cnt_mask = cmask[i];
452 (nhm_params.pfp_nhm_counters[i]).cnt_mask = cmask[i];
453 }
454 }
455 if(nehalem || westmere)
456 {
457 ret = m_pfm.pfm_dispatch_events(&inp, &nhm_params, &outp, NULL);
458 }
459 else
460 {
461 ret = m_pfm.pfm_dispatch_events(&inp, &params, &outp, NULL);
462 }
463 if(ret != PFMLIB_SUCCESS)
464 {
465 log << MSG::ERROR << "ERROR: cannot dispatch events: " << m_pfm.pfm_strerror(ret) << ". Aborting..." << endmsg;
466 }
467 for(unsigned int i=0; i<outp.pfp_pmc_count; i++)
468 {
469 pc[i].reg_num = outp.pfp_pmcs[i].reg_num;
470 pc[i].reg_value = outp.pfp_pmcs[i].reg_value;
471 }
472 for(unsigned int i=0; i<outp.pfp_pmd_count; i++)
473 {
474 pd[i].reg_num = outp.pfp_pmds[i].reg_num;
475 pd[i].reg_value = 0;
476 }
477 fd = m_pfm.pfm_create_context(&ctx, NULL, 0, 0);
478 if(fd == -1)
479 {
480 log << MSG::ERROR << "ERROR: Context not created. Aborting..." << endmsg;
481 }
482 if(m_pfm.pfm_write_pmcs(fd, pc, outp.pfp_pmc_count) == -1)
483 {
484 log << MSG::ERROR << "ERROR: Could not write pmcs. Aborting..." << endmsg;
485 }
486 if(m_pfm.pfm_write_pmds(fd, pd, outp.pfp_pmd_count) == -1)
487 {
488 log << MSG::ERROR << "ERROR: Could not write pmds. Aborting..." << endmsg;
489 }
490 load_arg.load_pid = getpid();
491 if(m_pfm.pfm_load_context(fd, &load_arg) == -1)
492 {
493 log << MSG::ERROR << "ERROR: Could not load context. Aborting..." << endmsg;
494 // MsgStream log(msgSvc(), name());
495 // log << MSG::ERROR << "Could not read pmds" << endmsg;
496 }
497
498 m_pfm.pfm_start(fd, NULL);
499 }
500
501
502
503
504
505 // stoppm()
506 // const ModuleDescription& desc : description of the module that just finished its execution (we are only interested in its name)
507 // stops the counting calling pfm_stop() and stores the counting results into the "results" map
508 void PerfMonAuditor::stoppm()
509 {
510 MsgStream log(msgSvc(), name());
511 m_pfm.pfm_stop(fd);
512 if(m_pfm.pfm_read_pmds(fd, pd, inp.pfp_event_count) == -1)
513 {
514 log << MSG::ERROR << "Could not read pmds" << endmsg;
515 }
516 for(int i=0; i<used_counters_number; i++)
517 {
518 results[i][(alg_stack.top().first)->name()].push_back(alg_stack.top().second[i] + pd[i].reg_value);
519 }
520
521 close(fd);
522 }
523
524
525 void PerfMonAuditor::pausepm()
526 {
527 MsgStream log(msgSvc(), name());
528 m_pfm.pfm_stop(fd);
529 if(m_pfm.pfm_read_pmds(fd, pd, inp.pfp_event_count) == -1)
530 {
531 log << MSG::ERROR << "Could not read pmds" << endmsg;
532 }
533
534 for(int i=0; i<used_counters_number; i++)
535 {
536 alg_stack.top().second[i] += pd[i].reg_value;
537 }
538
539 close(fd);
540 }
541
542
543 // finalizepm()
544 // called when all the countings of the current event are finished, it dumps the results
545 // into the output file corresponding to the event being counted
546 void PerfMonAuditor::finalizepm()
547 {
548 MsgStream log(msgSvc(), name());
549 log << MSG::INFO << "start of finalizepm ucn:" << used_counters_number << endmsg;
550 char filename[MAX_OUTPUT_FILENAME_LENGTH];
551 char to_cat[50];
552 FILE *outfile;
553 for(int i=0; i<used_counters_number; i++)
554 {
555 bzero(filename, MAX_OUTPUT_FILENAME_LENGTH);
556 sprintf(filename, "%s_%s", prefix_cstr, event_cstr[i]);
557 for(int j=0; j<(int)strlen(filename); j++)
558 {
559 if(filename[j]==':')
560 {
561 filename[j]='-';
562 }
563 }
564 bzero(to_cat, 50);
565 if(inv[i])
566 {
567 strcpy(to_cat, "_INV_1");
568 }
569 if(cmask[i]>0)
570 {
571 sprintf(to_cat, "%s_CMASK_%d", to_cat, cmask[i]);
572 }
573 sprintf(filename, "%s%s.txt", filename, to_cat);
574 log << MSG::INFO << "Filename:" << filename << endmsg;
575 outfile = fopen(filename, "w");
576 if(nehalem)
577 {
578 fprintf(outfile, "NHM ");
579 }
580 else if(westmere)
581 {
582 fprintf(outfile, "WSM ");
583 }
584 else if(core)
585 {
586 fprintf(outfile, "CORE ");
587 }
588 fprintf(outfile, "%s %d %d %d\n", event_cstr[i], cmask[i], inv[i], sp[i]);
589 for(std::map<std::string, std::vector<unsigned long int> >::iterator it=(results[i]).begin(); it!=(results[i]).end(); it++)
590 {
591 fprintf(outfile, "%s\n", (it->first).c_str());
592 for(std::vector<unsigned long int>::iterator j=(it->second).begin(); j!=(it->second).end(); j++)
593 {
594 fprintf(outfile, "%lu\n", *j);
595 }
596 }
597 fclose(outfile);
598 }
599 }
600
601 StatusCode PerfMonAuditor::initialize()
602 {
603 MsgStream log(msgSvc(), name());
604
605 if (!m_pfm.loaded) {
606 log << MSG::ERROR << "pfm library could not be loaded" << endmsg;
607 return false;
608 }
609
610 log << MSG::INFO << "Initializing..." << endmsg;
611 StatusCode sc = Auditor::initialize() ;
612 if(sc.isFailure())
613 {
614 return sc;
615 }
616 used_counters_number = 0;
617 for(int i=0; i<MAX_NUMBER_OF_PROGRAMMABLE_COUNTERS; i++)
618 {
619 if(event_str[i].length()>0) used_counters_number++;
620 }
621 for(int i=0; i<MAX_NUMBER_OF_PROGRAMMABLE_COUNTERS; i++)
622 {
623 strcpy(event_cstr[i], event_str[i].c_str());
624 }
625 strcpy(prefix_cstr, prefix.c_str());
626
627 if(m_pfm.pfm_initialize() != PFMLIB_SUCCESS)
628 {
629 log << MSG::ERROR << "Cannot initialize perfmon!!" << endmsg;
630 }
631 ph_ev_count = 0;
632 first_alg = true;
633 event_count_reached = false;
634 nehalem = false;
635 core = false;
636 westmere = false;
637 if(family.compare("CORE")==0) core = true;
638 else if(family.compare("NEHALEM")==0) nehalem = true;
639 else if(family.compare("WESTMERE")==0) westmere = true;
640 else
641 {
642 log << MSG::ERROR << "ERROR: Unsupported processor family " << family << ". aborting..." << endmsg;
643 }
644
645 log << MSG::INFO << "Initialized!" << endmsg;
646 return StatusCode::SUCCESS ;
647 }
648
649 // process_smpl_buf()
650 // pfm_dfl_smpl_hdr_t *hdr : pointer to header of the buffer containing addresses sampled during the sampling process
651 // size_t entry_size : size of each entry, used to navigate through the various entries
652 // called when the sampling buffer is full, saves the samples into memory ("samples" map)
653 void PerfMonAuditor::process_smpl_buf(pfm_dfl_smpl_hdr_t *hdr, size_t entry_size)
654 {
656 pfm_dfl_smpl_entry_t *ent;
657 size_t pos, count;
658 uint64_t entry;
659 if(hdr->hdr_overflows == last_overflow && hdr->hdr_count == last_count)
660 {
661 printf("skipping identical set of samples...\n");
662 return;
663 }
664 count = hdr->hdr_count;
665 ent = (pfm_dfl_smpl_entry_t *)(hdr+1);
666 pos = (unsigned long)ent;
667 entry = collected_samples;
668 while(count--)
669 {
670 //if(ent->ovfl_pmd>=0 && ent->ovfl_pmd<=3)
671 if(ent->ovfl_pmd<=3)
672 {
673 ((samples[ent->ovfl_pmd])[(alg_stack.top().first)->name()])[(unsigned long)(ent->ip)]++;
674 }
675 pos += entry_size;
676 ent = (pfm_dfl_smpl_entry_t *)pos;
677 entry++;
678 }
679 collected_samples = entry;
680 last_overflow = hdr->hdr_overflows;
681 if (last_count != hdr->hdr_count && (last_count || last_overflow == 0))
682 {
683 collected_partial += hdr->hdr_count;
684 }
685 last_count = hdr->hdr_count;
686 return;
687 }
688
689 // sigio_handler()
690 // int n : signal number of the signal being delivered
691 // struct siginfo *info : pointer to a siginfo_t structure containing info about the signal
692 // struct sigcontext *sc : context of the signal, NULL in our case
693 // signal handler used to catch sampling buffer overflows. When they occur it calls the process_smpl_buf() function
694 void PerfMonAuditor::sigio_handler(int /*n*/, struct siginfo */*info*/, struct sigcontext */*sc*/)
695 {
696 //MsgStream log(msgSvc(), name());
697 PFMon& pfm = PFMon::instance();
698 pfarg_msg_t msg;
699 int fd = ctx_fd;
700 int r;
701 if(fd != ctx_fd)
702 {
703 //log << MSG::ERROR << "ERROR: handler does not get valid file descriptor. Aborting..." << endmsg;
704 }
705 if(pfm.pfm_read_pmds(fd, pd_smpl+1, 1) == -1)
706 {
707 //log << MSG::ERROR << "ERROR: pfm_read_pmds: " << strerror(errno) << ". Aborting..." << endmsg;
708 }
709 while(true)
710 {
711 r = read(fd, &msg, sizeof(msg));
712 if(r!=sizeof(msg))
713 {
714 if(r==-1 && errno==EINTR)
715 {
716 printf("read interrupted, retrying\n");
717 continue;
718 }
719 //log << MSG::ERROR << "ERROR: cannot read overflow message: " << strerror(errno) << ". Aborting..." << endmsg;
720 }
721 break;
722 }
723 switch(msg.type)
724 {
725 case PFM_MSG_OVFL: // the sampling buffer is full
726 process_smpl_buf(hdr, entry_size);
727 ovfl_count++;
728 if(pfm.pfm_restart(fd))
729 {
730 if(errno!=EBUSY)
731 {
732 //log << MSG::ERROR << "ERROR: pfm_restart error errno " << errno << ". Aborting..." << endmsg;
733 }
734 else
735 {
736 printf("pfm_restart: task probably terminated \n");
737 }
738 }
739 break;
740 default:
741 //log << MSG::ERROR << "ERROR: unknown message type " << msg.type << ". Aborting..." << endmsg;
742 break;
743 }
744
745 }
746
747
748
749
750
751
752
753 // start_smpl()
754 // const ModuleDescription& desc : description of the module that is just starting its execution (we are only interested in its name)
755 // initializes all the necessary structures to start the sampling, calling pfm_self_start()
756 void PerfMonAuditor::start_smpl()
757 {
758 MsgStream log(msgSvc(), name());
759 ovfl_count = 0;
760 num_smpl_pmds = 0;
761 last_overflow = ~0;
762 max_pmd = 0;
763 memset(&pfmlib_options, 0, sizeof(pfmlib_options));
764 pfmlib_options.pfm_debug = 0;
765 pfmlib_options.pfm_verbose = 0;
766 m_pfm.pfm_set_options(&pfmlib_options);
767 ret = m_pfm.pfm_initialize();
768 if(ret != PFMLIB_SUCCESS)
769 {
770 log << MSG::ERROR << "ERROR: Cannot initialize library: " << m_pfm.pfm_strerror(ret) << ". Aborting..." << endmsg;
771 }
772 struct sigaction act;
773 memset(&act, 0, sizeof(act));
774 act.sa_handler = (sig_t)sigio_handler; // dlopen() ==>
775 //act.sa_handler = (sig_t)&sigio_handler;
776 sigaction(SIGIO, &act, 0);
777 memset(&ctx, 0, sizeof(ctx));
778 memset(&buf_arg, 0, sizeof(buf_arg));
779 memset(&inp,0, sizeof(inp));
780 memset(&outp,0, sizeof(outp));
781 memset(pd_smpl, 0, sizeof(pd_smpl));
782 memset(pc, 0, sizeof(pc));
783 memset(&load_args, 0, sizeof(load_args));
784 m_pfm.pfm_get_num_counters(&num_counters);
785 memset(&params, 0, sizeof(params));
786 memset(&nhm_params, 0, sizeof(nhm_params));
787
788 for(int i=0; i<used_counters_number; i++)
789 {
790 ret = m_pfm.pfm_find_full_event(event_cstr[i], &inp.pfp_events[i]);
791 if(ret != PFMLIB_SUCCESS)
792 {
793 log << MSG::ERROR << "ERROR: cannot find event: " << event_cstr[i] << ". Aborting..." << endmsg;
794 }
795 }
796 inp.pfp_dfl_plm = PFM_PLM3;
797 inp.pfp_event_count = 4;
798 for(int i=0; i<used_counters_number; i++)
799 {
800 if(inv[i])
801 {
802 (params.pfp_core_counters[i]).flags |= PFM_CORE_SEL_INV;
803 (nhm_params.pfp_nhm_counters[i]).flags |= PFM_NHM_SEL_INV;
804 }
805 if(cmask[i]>0)
806 {
807 (params.pfp_core_counters[i]).cnt_mask = cmask[i];
808 (nhm_params.pfp_nhm_counters[i]).cnt_mask = cmask[i];
809 }
810 }
811 if(nehalem || westmere)
812 {
813 ret = m_pfm.pfm_dispatch_events(&inp, &nhm_params, &outp, NULL);
814 }
815 else
816 {
817 ret = m_pfm.pfm_dispatch_events(&inp, &params, &outp, NULL);
818 }
819 if(ret != PFMLIB_SUCCESS)
820 {
821 log << MSG::ERROR << "ERROR: cannot configure events: " << m_pfm.pfm_strerror(ret) << ". Aborting..." << endmsg;
822 }
823 for(unsigned int i=0; i<outp.pfp_pmc_count; i++)
824 {
825 pc[i].reg_num = outp.pfp_pmcs[i].reg_num;
826 pc[i].reg_value = outp.pfp_pmcs[i].reg_value;
827 }
828 for(unsigned int i=0; i<outp.pfp_pmd_count; i++)
829 {
830 pd_smpl[i].reg_num = outp.pfp_pmds[i].reg_num;
831 if(i)
832 {
833 pfm_bv_set(pd_smpl[0].reg_smpl_pmds, pd_smpl[i].reg_num);
834 if(pd_smpl[i].reg_num>max_pmd)
835 {
836 max_pmd = pd_smpl[i].reg_num;
837 }
838 num_smpl_pmds++;
839 }
840 }
841 for(int i=0; i<used_counters_number; i++)
842 {
843 pd_smpl[i].reg_flags |= PFM_REGFL_OVFL_NOTIFY | PFM_REGFL_RANDOM;
844 pfm_bv_copy(pd_smpl[i].reg_reset_pmds, pd_smpl[i].reg_smpl_pmds, max_pmd);
845 pd_smpl[i].reg_value = (uint64_t)(sp[i] * -1);
846 pd_smpl[i].reg_short_reset = (uint64_t)(sp[i] * -1);
847 pd_smpl[i].reg_long_reset = (uint64_t)(sp[i] * -1);
848 pd_smpl[i].reg_random_seed = 5; //tocheck
849 pd_smpl[i].reg_random_mask = 0xff; //tocheck
850 }
851 entry_size = sizeof(pfm_dfl_smpl_entry_t)+(num_smpl_pmds<<3);
852 ctx.ctx_flags = 0;
853 buf_arg.buf_size = 3*getpagesize()+512;
854 ctx_fd = m_pfm.pfm_create_context(&ctx, (char *)FMT_NAME, &buf_arg, sizeof(buf_arg));
855 if(ctx_fd==-1)
856 {
857 if(errno==ENOSYS)
858 {
859 log << MSG::ERROR << "ERROR: Your kernel does not have performance monitoring support! Aborting..." << endmsg;
860 }
861 log << MSG::ERROR << "ERROR: Can't create PFM context " << strerror(errno) << ". Aborting..." << endmsg;
862 }
863 buf_addr = mmap(NULL, (size_t)buf_arg.buf_size, PROT_READ, MAP_PRIVATE, ctx_fd, 0);
864 if(buf_addr==MAP_FAILED)
865 {
866 log << MSG::ERROR << "ERROR: cannot mmap sampling buffer: " << strerror(errno) << ". Aborting..." << endmsg;
867 }
868 hdr = (pfm_dfl_smpl_hdr_t *)buf_addr;
869 if(PFM_VERSION_MAJOR(hdr->hdr_version)<1)
870 {
871 log << MSG::ERROR << "ERROR: invalid buffer format version. Aborting..." << endmsg;
872 }
873 if(m_pfm.pfm_write_pmcs(ctx_fd, pc, outp.pfp_pmc_count))
874 {
875 log << MSG::ERROR << "ERROR: pfm_write_pmcs error errno " << strerror(errno) << ". Aborting..." << endmsg;
876 }
877 if(m_pfm.pfm_write_pmds(ctx_fd, pd_smpl, outp.pfp_pmd_count))
878 {
879 log << MSG::ERROR << "ERROR: pfm_write_pmds error errno " << strerror(errno) << ". Aborting..." << endmsg;
880 }
881 load_args.load_pid = getpid();
882 if(m_pfm.pfm_load_context(ctx_fd, &load_args))
883 {
884 log << MSG::ERROR << "ERROR: pfm_load_context error errno " << strerror(errno) << ". Aborting..." << endmsg;
885 }
886 ret = fcntl(ctx_fd, F_SETFL, fcntl(ctx_fd, F_GETFL, 0) | O_ASYNC);
887 if(ret == -1)
888 {
889 log << MSG::ERROR << "ERROR: cannot set ASYNC: " << strerror(errno) << ". Aborting..." << endmsg;
890 }
891 ret = fcntl(ctx_fd, F_SETOWN, getpid());
892 if(ret == -1)
893 {
894 log << MSG::ERROR << "ERROR: cannot setown: " << strerror(errno) << ". Aborting..." << endmsg;
895 }
896 //pfm_self_start(ctx_fd); ==>
897 m_pfm.pfm_start(ctx_fd, NULL);
898 }
899
900
901 // stop_smpl()
902 // const ModuleDescription& desc : description of the module that just finished its execution (we are only interested in its name)
903 // stops the sampling and calls process_smpl_buf() one last time to process all the remaining samples
904 void PerfMonAuditor::stop_smpl()
905 {
906 MsgStream log(msgSvc(), name());
907 m_pfm.pfm_self_stop(ctx_fd);
908 process_smpl_buf(hdr, entry_size);
909 close(ctx_fd);
910 ret = munmap(hdr, (size_t)buf_arg.buf_size);
911 if(ret)
912 {
913 log << MSG::ERROR << "Cannot unmap buffer: %s" << strerror(errno) << endmsg;
914 }
915 return;
916 }
917
918
919
920 // finalize_smpl()
921 // processes the sampling results in order to find library and offset of each sampled address, using the symbol() and tosymbol() functions,
922 // and then dumps the new found information into gzipped output files, to be processed later
923 void PerfMonAuditor::finalize_smpl()
924 {
925 MsgStream log(msgSvc(), name());
926 char filename[MAX_OUTPUT_FILENAME_LENGTH];
927 bzero(filename, MAX_OUTPUT_FILENAME_LENGTH);
928 char to_cat[50];
929 gzFile outfile;
930 int err;
931 for(int i=0; i<used_counters_number; i++)
932 {
933 sprintf(filename, "%s_%s", prefix_cstr, event_cstr[i]);
934 for(int j=0; j<(int)strlen(filename); j++)
935 {
936 if(filename[j]==':')
937 {
938 filename[j]='-';
939 }
940 }
941 bzero(to_cat, 50);
942 if(inv[i])
943 {
944 strcpy(to_cat, "_INV_1");
945 }
946 if(cmask[i]>0)
947 {
948 sprintf(to_cat, "%s_CMASK_%d", to_cat, cmask[i]);
949 }
950 sprintf(filename, "%s%s.txt.gz", filename, to_cat);
951 outfile = gzopen(filename, "wb");
952 if(outfile!=NULL)
953 {
954 if(nehalem)
955 {
956 gzprintf(outfile, "NHM ");
957 }
958 else if(westmere)
959 {
960 gzprintf(outfile, "WSM ");
961 }
962 else if(core)
963 {
964 gzprintf(outfile, "CORE ");
965 }
966 if(gzprintf(outfile, "%s %d %d %d\n", event_cstr[i], cmask[i], inv[i], sp[i]) < (int)strlen(event_cstr[i]))
967 {
968 log << MSG::ERROR << "ERROR: gzputs err: " << gzerror(outfile, &err) << ". Aborting..." << endmsg;
969 }
970 for(std::map<std::string, std::map<unsigned long, unsigned int> >::iterator it=samples[i].begin(); it!=samples[i].end(); it++)
971 {
972 unsigned long long sum = 0;
973 for(std::map<unsigned long, unsigned int>::iterator jt=(it->second).begin(); jt!=(it->second).end(); jt++)
974 {
975 sum += jt->second;
976 }
977 if(gzprintf(outfile, "%s%%%llu\n", (it->first).c_str(), sum) < (int)((it->first).length()))
978 {
979 log << MSG::ERROR << "ERROR: gzputs err: " << gzerror(outfile, &err) << ". Aborting..." << endmsg;
980 }
981 for(std::map<unsigned long, unsigned int>::iterator jt=(it->second).begin(); jt!=(it->second).end(); jt++)
982 {
983 char sym_name[SYM_NAME_MAX_LENGTH];
984 bzero(sym_name, SYM_NAME_MAX_LENGTH);
985 const char *libName;
986 const char *symbolName;
987 int libOffset = 0;
988 int offset = 0;
989 void *sym_addr = IgHookTrace::tosymbol((void *)(jt->first));
990 if(sym_addr != NULL)
991 {
992 bool success = IgHookTrace::symbol(sym_addr, symbolName, libName, offset, libOffset);
993 if(success)
994 {
995 if(symbolName!=NULL && strlen(symbolName)>0)
996 {
997 strcpy(sym_name, symbolName);
998 strcat(sym_name, " ");
999 }
1000 else
1001 {
1002 strcpy(sym_name, "??? ");
1003 }
1004 if(libName!=NULL && strlen(libName)>0)
1005 {
1006 strcat(sym_name, libName);
1007 strcat(sym_name, " ");
1008 }
1009 else
1010 {
1011 strcat(sym_name, "??? ");
1012 }
1013 sprintf(sym_name, "%s%d ", sym_name, libOffset);
1014 if(strlen(sym_name)<=0)
1015 {
1016 log << MSG::ERROR << "ERROR: Symbol name length is zero. Aborting..." << endmsg;
1017 }
1018 }
1019 else
1020 {
1021 strcpy(sym_name,"??? ??? 0 ");
1022 }
1023 }
1024 else
1025 {
1026 strcpy(sym_name,"??? ??? 0 ");
1027 }
1028 if(gzprintf(outfile, "%s %d\n", sym_name, jt->second) < (int)strlen(sym_name))
1029 {
1030 log << MSG::ERROR << "ERROR: gzputs err: " << gzerror(outfile, &err) << endmsg;
1031 }
1032 }
1033 }
1034 }
1035 else
1036 {
1037 log << MSG::ERROR << "ERROR: Could not open file: " << filename << ". Aborting..." << endmsg;
1038 }
1039 gzclose(outfile);
1040 }
1041 }
1042
1043
1044
1045 StatusCode PerfMonAuditor::finalize()
1046 {
1047 if(sampling == 0) finalizepm();
1048 else finalize_smpl();
1049 return Auditor::finalize();
1050 }
1051
1052
1053 void PerfMonAuditor::before(StandardEventType evt, INamedInterface *alg)
1054 {
1055 switch(evt)
1056 {
1057 case IAuditor::Initialize:
1058 i_beforeInitialize(alg);
1059 break;
1060 case IAuditor::Execute:
1061 i_beforeExecute(alg);
1062 break;
1063 default:
1064 break;
1065 }
1066 return;
1067 }
1068
1069 void PerfMonAuditor::after(StandardEventType evt, INamedInterface *alg, const StatusCode &)
1070 {
1071 switch(evt)
1072 {
1073 case IAuditor::Initialize:
1074 i_afterInitialize(alg);
1075 break;
1076 case IAuditor::Execute:
1077 i_afterExecute(alg);
1078 break;
1079 default:
1080 break;
1081 }
1082 return;
1083 }
1084
1085 void PerfMonAuditor::i_beforeInitialize(INamedInterface* alg)
1086 {
1087 if(alg == 0)
1088 {
1089 return;
1090 }
1091 return;
1092 }
1093
1094 void PerfMonAuditor::i_afterInitialize(INamedInterface* alg)
1095 {
1096 if(alg == 0)
1097 {
1098 return;
1099 }
1100 return;
1101 }
1102
1103 void PerfMonAuditor::i_beforeExecute(INamedInterface* alg)
1104 {
1105 MsgStream log(msgSvc(), name());
1106 if(alg == 0)
1107 {
1108 return;
1109 }
1110 //log << MSG::INFO << "before:inside! " << alg->name() << endmsg;
1111 if(first_alg)
1112 {
1113 first_alg = false;
1114 first_alg_name = alg->name();
1115 //log << MSG::INFO << "first_alg_name= " << alg->name() << endmsg;
1116 }
1117 if(!event_count_reached)
1118 {
1119 if(!first_alg_name.compare(alg->name()))
1120 {
1121 ph_ev_count++;
1122 //log << MSG::INFO << "EVENT COUNT: " << ph_ev_count << endmsg;
1123 if(ph_ev_count==start_at_event)
1124 {
1125 event_count_reached = true;
1126 //log << MSG::INFO << "!!! EVENT COUNT REACHED: " << ph_ev_count << endmsg;
1127 }
1128 }
1129 }
1130 if(event_count_reached)
1131 {
1132 //log << MSG::INFO << "before:inside! " << alg->name() << endmsg;
1133
1134 if(!alg_stack.empty())
1135 {
1136 if(sampling == 0) pausepm(); //pausing father algorithm counting
1137 else stop_smpl();
1138 }
1139 ++m_indent;
1140 std::vector <unsigned long int> zeroes(4,0);
1141 alg_stack.push(std::make_pair(alg, zeroes));
1142 if(sampling == 0) startpm();
1143 else start_smpl();
1144 }
1145 return;
1146 }
1147
1148 void PerfMonAuditor::i_afterExecute(INamedInterface* alg)
1149 {
1150 MsgStream log(msgSvc(), name());
1151 if(alg == 0)
1152 {
1153 return;
1154 }// log << MSG::INFO << "after:inside! " << alg->name() << endmsg;
1155
1156 if(event_count_reached)
1157 {
1158 //log << MSG::INFO << "after:inside! " << alg->name() << endmsg;
1159
1160 if(sampling == 0) stoppm();
1161 else stop_smpl();
1162 alg_stack.pop();
1163 --m_indent;
1164 if(!alg_stack.empty())
1165 {
1166 if(sampling == 0) startpm();
1167 else start_smpl(); //resuming father algorithm counting
1168 }
1169 }
1170 return;
1171 }
1172
1173 DECLARE_COMPONENT(PerfMonAuditor)