liblttng-ust/lttng-context-perf-counters.c

   1 /*
   2  * lttng-context-perf-counters.c
   3  *
   4  * LTTng UST performance monitoring counters (perf-counters) integration.
   5  *
   6  * Copyright (C) 2009-2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   7  *
   8  * This library is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License as published by the Free Software Foundation; only
  11  * version 2.1 of the License.
  12  *
  13  * This library is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16  * Lesser General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU Lesser General Public
  19  * License along with this library; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  21  */
  22
  23 #include <sys/types.h>
  24 #include <unistd.h>
  25 #include <string.h>
  26 #include <stdlib.h>
  27 #include <stdio.h>
  28 #include <stdbool.h>
  29 #include <sys/mman.h>
  30 #include <sys/syscall.h>
  31 #include <linux/perf_event.h>
  32 #include <lttng/ust-events.h>
  33 #include <lttng/ust-tracer.h>
  34 #include <lttng/ringbuffer-config.h>
  35 #include <urcu/system.h>
  36 #include <urcu/arch.h>
  37 #include <urcu/rculist.h>
  38 #include <helper.h>
  39 #include <urcu/ref.h>
  40 #include <usterr-signal-safe.h>
  41 #include <signal.h>
  42 #include "lttng-tracer-core.h"
  43
  44 /*
  45  * We use a global perf counter key and iterate on per-thread RCU lists
  46  * of fields in the fast path, even though this is not strictly speaking
  47  * what would provide the best fast-path complexity, to ensure teardown
  48  * of sessions vs thread exit is handled racelessly.
  49  *
  50  * Updates and traversals of thread_list are protected by UST lock.
  51  * Updates to rcu_field_list are protected by UST lock.
  52  */
  53
  54 struct lttng_perf_counter_thread_field {
  55         struct lttng_perf_counter_field *field; /* Back reference */
  56         struct perf_event_mmap_page *pc;
  57         struct cds_list_head thread_field_node; /* Per-field list of thread fields (node) */
  58         struct cds_list_head rcu_field_node;    /* RCU per-thread list of fields (node) */
  59         int fd;                                 /* Perf FD */
  60 };
  61
  62 struct lttng_perf_counter_thread {
  63         struct cds_list_head rcu_field_list;    /* RCU per-thread list of fields */
  64 };
  65
  66 struct lttng_perf_counter_field {
  67         struct perf_event_attr attr;
  68         struct cds_list_head thread_field_list; /* Per-field list of thread fields */
  69 };
  70
  71 static pthread_key_t perf_counter_key;
  72
  73 static
  74 size_t perf_counter_get_size(struct lttng_ctx_field *field, size_t offset)
  75 {
  76         size_t size = 0;
  77
  78         size += lib_ring_buffer_align(offset, lttng_alignof(uint64_t));
  79         size += sizeof(uint64_t);
  80         return size;
  81 }
  82
  83 static
  84 uint64_t read_perf_counter_syscall(
  85                 struct lttng_perf_counter_thread_field *thread_field)
  86 {
  87         uint64_t count;
  88
  89         if (caa_unlikely(thread_field->fd < 0))
  90                 return 0;
  91
  92         if (caa_unlikely(read(thread_field->fd, &count, sizeof(count))
  93                                 < sizeof(count)))
  94                 return 0;
  95
  96         return count;
  97 }
  98
  99 #if defined(__x86_64__) || defined(__i386__)
 100
 101 static
 102 uint64_t rdpmc(unsigned int counter)
 103 {
 104         unsigned int low, high;
 105
 106         asm volatile("rdpmc" : "=a" (low), "=d" (high) : "c" (counter));
 107
 108         return low | ((uint64_t) high) << 32;
 109 }
 110
 111 static
 112 uint64_t arch_read_perf_counter(
 113                 struct lttng_perf_counter_thread_field *thread_field)
 114 {
 115         uint32_t seq, idx;
 116         uint64_t count;
 117         struct perf_event_mmap_page *pc = thread_field->pc;
 118
 119         if (caa_unlikely(!pc))
 120                 return 0;
 121
 122         do {
 123                 seq = CMM_LOAD_SHARED(pc->lock);
 124                 cmm_barrier();
 125
 126                 idx = pc->index;
 127                 if (caa_likely(pc->cap_user_rdpmc && idx)) {
 128                         int64_t pmcval;
 129
 130                         pmcval = rdpmc(idx - 1);
 131                         /* Sign-extend the pmc register result. */
 132                         pmcval <<= 64 - pc->pmc_width;
 133                         pmcval >>= 64 - pc->pmc_width;
 134                         count = pc->offset + pmcval;
 135                 } else {
 136                         /* Fall-back on system call if rdpmc cannot be used. */
 137                         return read_perf_counter_syscall(thread_field);
 138                 }
 139                 cmm_barrier();
 140         } while (CMM_LOAD_SHARED(pc->lock) != seq);
 141
 142         return count;
 143 }
 144
 145 static
 146 int arch_perf_keep_fd(struct lttng_perf_counter_thread_field *thread_field)
 147 {
 148         struct perf_event_mmap_page *pc = thread_field->pc;
 149
 150         if (!pc)
 151                 return 0;
 152         return !pc->cap_user_rdpmc;
 153 }
 154
 155 #else
 156
 157 /* Generic (slow) implementation using a read system call. */
 158 static
 159 uint64_t arch_read_perf_counter(
 160                 struct lttng_perf_counter_thread_field *thread_field)
 161 {
 162         return read_perf_counter_syscall(thread_field);
 163 }
 164
 165 static
 166 int arch_perf_keep_fd(struct lttng_perf_counter_thread_field *thread_field)
 167 {
 168         return 1;
 169 }
 170
 171 #endif
 172
 173 static
 174 int sys_perf_event_open(struct perf_event_attr *attr,
 175                 pid_t pid, int cpu, int group_fd,
 176                 unsigned long flags)
 177 {
 178         return syscall(SYS_perf_event_open, attr, pid, cpu,
 179                         group_fd, flags);
 180 }
 181
 182 static
 183 int open_perf_fd(struct perf_event_attr *attr)
 184 {
 185         int fd;
 186
 187         fd = sys_perf_event_open(attr, 0, -1, -1, 0);
 188         if (fd < 0)
 189                 return -1;
 190
 191         return fd;
 192 }
 193
 194 static
 195 void close_perf_fd(int fd)
 196 {
 197         int ret;
 198
 199         if (fd < 0)
 200                 return;
 201
 202         ret = close(fd);
 203         if (ret) {
 204                 perror("Error closing LTTng-UST perf memory mapping FD");
 205         }
 206 }
 207
 208 static
 209 struct perf_event_mmap_page *setup_perf(
 210                 struct lttng_perf_counter_thread_field *thread_field)
 211 {
 212         void *perf_addr;
 213
 214         perf_addr = mmap(NULL, sizeof(struct perf_event_mmap_page),
 215                         PROT_READ, MAP_SHARED, thread_field->fd, 0);
 216         if (perf_addr == MAP_FAILED)
 217                 perf_addr = NULL;
 218
 219         if (!arch_perf_keep_fd(thread_field)) {
 220                 close_perf_fd(thread_field->fd);
 221                 thread_field->fd = -1;
 222         }
 223
 224         return perf_addr;
 225 }
 226
 227 static
 228 void unmap_perf_page(struct perf_event_mmap_page *pc)
 229 {
 230         int ret;
 231
 232         if (!pc)
 233                 return;
 234         ret = munmap(pc, sizeof(struct perf_event_mmap_page));
 235         if (ret < 0) {
 236                 PERROR("Error in munmap");
 237                 abort();
 238         }
 239 }
 240
 241 static
 242 struct lttng_perf_counter_thread *alloc_perf_counter_thread(void)
 243 {
 244         struct lttng_perf_counter_thread *perf_thread;
 245         sigset_t newmask, oldmask;
 246         int ret;
 247
 248         ret = sigfillset(&newmask);
 249         if (ret)
 250                 abort();
 251         ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
 252         if (ret)
 253                 abort();
 254         /* Check again with signals disabled */
 255         perf_thread = pthread_getspecific(perf_counter_key);
 256         if (perf_thread)
 257                 goto skip;
 258         perf_thread = zmalloc(sizeof(*perf_thread));
 259         if (!perf_thread)
 260                 abort();
 261         CDS_INIT_LIST_HEAD(&perf_thread->rcu_field_list);
 262         ret = pthread_setspecific(perf_counter_key, perf_thread);
 263         if (ret)
 264                 abort();
 265 skip:
 266         ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
 267         if (ret)
 268                 abort();
 269         return perf_thread;
 270 }
 271
 272 static
 273 struct lttng_perf_counter_thread_field *
 274         add_thread_field(struct lttng_perf_counter_field *perf_field,
 275                 struct lttng_perf_counter_thread *perf_thread)
 276 {
 277         struct lttng_perf_counter_thread_field *thread_field;
 278         sigset_t newmask, oldmask;
 279         int ret;
 280
 281         ret = sigfillset(&newmask);
 282         if (ret)
 283                 abort();
 284         ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
 285         if (ret)
 286                 abort();
 287         /* Check again with signals disabled */
 288         cds_list_for_each_entry_rcu(thread_field, &perf_thread->rcu_field_list,
 289                         rcu_field_node) {
 290                 if (thread_field->field == perf_field)
 291                         goto skip;
 292         }
 293         thread_field = zmalloc(sizeof(*thread_field));
 294         if (!thread_field)
 295                 abort();
 296         thread_field->field = perf_field;
 297         thread_field->fd = open_perf_fd(&perf_field->attr);
 298         if (thread_field->fd >= 0)
 299                 thread_field->pc = setup_perf(thread_field);
 300         /*
 301          * Note: thread_field->pc can be NULL if setup_perf() fails.
 302          * Also, thread_field->fd can be -1 if open_perf_fd() fails.
 303          */
 304         ust_lock_nocheck();
 305         cds_list_add_rcu(&thread_field->rcu_field_node,
 306                         &perf_thread->rcu_field_list);
 307         cds_list_add(&thread_field->thread_field_node,
 308                         &perf_field->thread_field_list);
 309         ust_unlock();
 310 skip:
 311         ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
 312         if (ret)
 313                 abort();
 314         return thread_field;
 315 }
 316
 317 static
 318 struct lttng_perf_counter_thread_field *
 319                 get_thread_field(struct lttng_perf_counter_field *field)
 320 {
 321         struct lttng_perf_counter_thread *perf_thread;
 322         struct lttng_perf_counter_thread_field *thread_field;
 323
 324         perf_thread = pthread_getspecific(perf_counter_key);
 325         if (!perf_thread)
 326                 perf_thread = alloc_perf_counter_thread();
 327         cds_list_for_each_entry_rcu(thread_field, &perf_thread->rcu_field_list,
 328                         rcu_field_node) {
 329                 if (thread_field->field == field)
 330                         return thread_field;
 331         }
 332         /* perf_counter_thread_field not found, need to add one */
 333         return add_thread_field(field, perf_thread);
 334 }
 335
 336 static
 337 uint64_t wrapper_perf_counter_read(struct lttng_ctx_field *field)
 338 {
 339         struct lttng_perf_counter_field *perf_field;
 340         struct lttng_perf_counter_thread_field *perf_thread_field;
 341
 342         perf_field = field->u.perf_counter;
 343         perf_thread_field = get_thread_field(perf_field);
 344         return arch_read_perf_counter(perf_thread_field);
 345 }
 346
 347 static
 348 void perf_counter_record(struct lttng_ctx_field *field,
 349                  struct lttng_ust_lib_ring_buffer_ctx *ctx,
 350                  struct lttng_channel *chan)
 351 {
 352         uint64_t value;
 353
 354         value = wrapper_perf_counter_read(field);
 355         lib_ring_buffer_align_ctx(ctx, lttng_alignof(value));
 356         chan->ops->event_write(ctx, &value, sizeof(value));
 357 }
 358
 359 static
 360 void perf_counter_get_value(struct lttng_ctx_field *field,
 361                 struct lttng_ctx_value *value)
 362 {
 363         uint64_t v;
 364
 365         v = wrapper_perf_counter_read(field);
 366         value->u.s64 = v;
 367 }
 368
 369 /* Called with UST lock held */
 370 static
 371 void lttng_destroy_perf_thread_field(
 372                 struct lttng_perf_counter_thread_field *thread_field)
 373 {
 374         close_perf_fd(thread_field->fd);
 375         unmap_perf_page(thread_field->pc);
 376         cds_list_del_rcu(&thread_field->rcu_field_node);
 377         cds_list_del(&thread_field->thread_field_node);
 378         free(thread_field);
 379 }
 380
 381 static
 382 void lttng_destroy_perf_thread_key(void *_key)
 383 {
 384         struct lttng_perf_counter_thread *perf_thread = _key;
 385         struct lttng_perf_counter_thread_field *pos, *p;
 386
 387         ust_lock_nocheck();
 388         cds_list_for_each_entry_safe(pos, p, &perf_thread->rcu_field_list,
 389                         rcu_field_node)
 390                 lttng_destroy_perf_thread_field(pos);
 391         ust_unlock();
 392         free(perf_thread);
 393 }
 394
 395 /* Called with UST lock held */
 396 static
 397 void lttng_destroy_perf_counter_field(struct lttng_ctx_field *field)
 398 {
 399         struct lttng_perf_counter_field *perf_field;
 400         struct lttng_perf_counter_thread_field *pos, *p;
 401
 402         free((char *) field->event_field.name);
 403         perf_field = field->u.perf_counter;
 404         /*
 405          * This put is performed when no threads can concurrently
 406          * perform a "get" concurrently, thanks to urcu-bp grace
 407          * period.
 408          */
 409         cds_list_for_each_entry_safe(pos, p, &perf_field->thread_field_list,
 410                         thread_field_node)
 411                 lttng_destroy_perf_thread_field(pos);
 412         free(perf_field);
 413 }
 414
 415 #ifdef __ARM_ARCH_7A__
 416
 417 static
 418 int perf_get_exclude_kernel(void)
 419 {
 420         return 0;
 421 }
 422
 423 #else /* __ARM_ARCH_7A__ */
 424
 425 static
 426 int perf_get_exclude_kernel(void)
 427 {
 428         return 1;
 429 }
 430
 431 #endif /* __ARM_ARCH_7A__ */
 432
 433 /* Called with UST lock held */
 434 int lttng_add_perf_counter_to_ctx(uint32_t type,
 435                                 uint64_t config,
 436                                 const char *name,
 437                                 struct lttng_ctx **ctx)
 438 {
 439         struct lttng_ctx_field *field;
 440         struct lttng_perf_counter_field *perf_field;
 441         char *name_alloc;
 442         int ret;
 443
 444         name_alloc = strdup(name);
 445         if (!name_alloc) {
 446                 ret = -ENOMEM;
 447                 goto name_alloc_error;
 448         }
 449         perf_field = zmalloc(sizeof(*perf_field));
 450         if (!perf_field) {
 451                 ret = -ENOMEM;
 452                 goto perf_field_alloc_error;
 453         }
 454         field = lttng_append_context(ctx);
 455         if (!field) {
 456                 ret = -ENOMEM;
 457                 goto append_context_error;
 458         }
 459         if (lttng_find_context(*ctx, name_alloc)) {
 460                 ret = -EEXIST;
 461                 goto find_error;
 462         }
 463
 464         field->destroy = lttng_destroy_perf_counter_field;
 465
 466         field->event_field.name = name_alloc;
 467         field->event_field.type.atype = atype_integer;
 468         field->event_field.type.u.basic.integer.size =
 469                         sizeof(uint64_t) * CHAR_BIT;
 470         field->event_field.type.u.basic.integer.alignment =
 471                         lttng_alignof(uint64_t) * CHAR_BIT;
 472         field->event_field.type.u.basic.integer.signedness =
 473                         lttng_is_signed_type(uint64_t);
 474         field->event_field.type.u.basic.integer.reverse_byte_order = 0;
 475         field->event_field.type.u.basic.integer.base = 10;
 476         field->event_field.type.u.basic.integer.encoding = lttng_encode_none;
 477         field->get_size = perf_counter_get_size;
 478         field->record = perf_counter_record;
 479         field->get_value = perf_counter_get_value;
 480
 481         perf_field->attr.type = type;
 482         perf_field->attr.config = config;
 483         perf_field->attr.exclude_kernel = perf_get_exclude_kernel();
 484         CDS_INIT_LIST_HEAD(&perf_field->thread_field_list);
 485         field->u.perf_counter = perf_field;
 486
 487         /* Ensure that this perf counter can be used in this process. */
 488         ret = open_perf_fd(&perf_field->attr);
 489         if (ret < 0) {
 490                 ret = -ENODEV;
 491                 goto setup_error;
 492         }
 493         close_perf_fd(ret);
 494
 495         /*
 496          * Contexts can only be added before tracing is started, so we
 497          * don't have to synchronize against concurrent threads using
 498          * the field here.
 499          */
 500
 501         lttng_context_update(*ctx);
 502         return 0;
 503
 504 setup_error:
 505 find_error:
 506         lttng_remove_context_field(ctx, field);
 507 append_context_error:
 508         free(perf_field);
 509 perf_field_alloc_error:
 510         free(name_alloc);
 511 name_alloc_error:
 512         return ret;
 513 }
 514
 515 int lttng_perf_counter_init(void)
 516 {
 517         int ret;
 518
 519         ret = pthread_key_create(&perf_counter_key,
 520                         lttng_destroy_perf_thread_key);
 521         if (ret)
 522                 ret = -ret;
 523         return ret;
 524 }
 525
 526 void lttng_perf_counter_exit(void)
 527 {
 528         int ret;
 529
 530         ret = pthread_key_delete(perf_counter_key);
 531         if (ret) {
 532                 errno = ret;
 533                 PERROR("Error in pthread_key_delete");
 534         }
 535 }