libust/tracepoint.c

   1 /*
   2  * Copyright (C) 2008-2011 Mathieu Desnoyers
   3  * Copyright (C) 2009 Pierre-Marc Fournier
   4  *
   5  * This library is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU Lesser General Public
   7  * License as published by the Free Software Foundation;
   8  * version 2.1 of the License.
   9  *
  10  * This library is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * Lesser General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU Lesser General Public
  16  * License along with this library; if not, write to the Free Software
  17  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
  18  *
  19  * Ported to userspace by Pierre-Marc Fournier.
  20  */
  21
  22 #define _LGPL_SOURCE
  23 #include <errno.h>
  24 #include <ust/tracepoint.h>
  25 #include <ust/tracepoint-internal.h>
  26 #include <ust/core.h>
  27 #include <ust/kcompat/kcompat.h>
  28 #include <urcu-bp.h>
  29 #include <urcu/hlist.h>
  30 #include <urcu/uatomic.h>
  31
  32 #include <ust/usterr-signal-safe.h>
  33
  34 extern struct tracepoint * const __start___tracepoints_ptrs[]
  35         __attribute__((visibility("hidden")));
  36 extern struct tracepoint * const __stop___tracepoints_ptrs[]
  37         __attribute__((visibility("hidden")));
  38
  39 static struct tracepoint * __tracepoint_ptr_dummy
  40         __attribute__((used, section("__tracepoints_ptrs")));
  41
  42 /* Set to 1 to enable tracepoint debug output */
  43 static const int tracepoint_debug;
  44 static int initialized;
  45 static void (*new_tracepoint_cb)(struct tracepoint *);
  46
  47 /* libraries that contain tracepoints (struct tracepoint_lib) */
  48 static CDS_LIST_HEAD(libs);
  49
  50 /*
  51  * Allow nested mutex for mutex listing and nested enable.
  52  */
  53 static __thread int nested_mutex;
  54
  55 /*
  56  * Tracepoints mutex protects the library tracepoints, the hash table,
  57  * and the library list.
  58  */
  59 static pthread_mutex_t tracepoints_mutex;
  60
  61 static
  62 void lock_tracepoints(void)
  63 {
  64         if (!(nested_mutex++))
  65                 pthread_mutex_lock(&tracepoints_mutex);
  66 }
  67
  68 static
  69 void unlock_tracepoints(void)
  70 {
  71         if (!(--nested_mutex))
  72                 pthread_mutex_unlock(&tracepoints_mutex);
  73 }
  74
  75 /*
  76  * Tracepoint hash table, containing the active tracepoints.
  77  * Protected by tracepoints_mutex.
  78  */
  79 #define TRACEPOINT_HASH_BITS 6
  80 #define TRACEPOINT_TABLE_SIZE (1 << TRACEPOINT_HASH_BITS)
  81 static struct cds_hlist_head tracepoint_table[TRACEPOINT_TABLE_SIZE];
  82
  83 static CDS_LIST_HEAD(old_probes);
  84 static int need_update;
  85
  86 /*
  87  * Note about RCU :
  88  * It is used to to delay the free of multiple probes array until a quiescent
  89  * state is reached.
  90  * Tracepoint entries modifications are protected by the tracepoints_mutex.
  91  */
  92 struct tracepoint_entry {
  93         struct cds_hlist_node hlist;
  94         struct tracepoint_probe *probes;
  95         int refcount;   /* Number of times armed. 0 if disarmed. */
  96         char name[0];
  97 };
  98
  99 struct tp_probes {
 100         union {
 101                 struct cds_list_head list;
 102         } u;
 103         struct tracepoint_probe probes[0];
 104 };
 105
 106 static inline void *allocate_probes(int count)
 107 {
 108         struct tp_probes *p  = zmalloc(count * sizeof(struct tracepoint_probe)
 109                         + sizeof(struct tp_probes));
 110         return p == NULL ? NULL : p->probes;
 111 }
 112
 113 static inline void release_probes(void *old)
 114 {
 115         if (old) {
 116                 struct tp_probes *tp_probes = _ust_container_of(old,
 117                         struct tp_probes, probes[0]);
 118                 synchronize_rcu();
 119                 free(tp_probes);
 120         }
 121 }
 122
 123 static void debug_print_probes(struct tracepoint_entry *entry)
 124 {
 125         int i;
 126
 127         if (!tracepoint_debug || !entry->probes)
 128                 return;
 129
 130         for (i = 0; entry->probes[i].func; i++)
 131                 DBG("Probe %d : %p", i, entry->probes[i].func);
 132 }
 133
 134 static void *
 135 tracepoint_entry_add_probe(struct tracepoint_entry *entry,
 136                            void *probe, void *data)
 137 {
 138         int nr_probes = 0;
 139         struct tracepoint_probe *old, *new;
 140
 141         WARN_ON(!probe);
 142
 143         debug_print_probes(entry);
 144         old = entry->probes;
 145         if (old) {
 146                 /* (N -> N+1), (N != 0, 1) probes */
 147                 for (nr_probes = 0; old[nr_probes].func; nr_probes++)
 148                         if (old[nr_probes].func == probe &&
 149                             old[nr_probes].data == data)
 150                                 return ERR_PTR(-EEXIST);
 151         }
 152         /* + 2 : one for new probe, one for NULL func */
 153         new = allocate_probes(nr_probes + 2);
 154         if (new == NULL)
 155                 return ERR_PTR(-ENOMEM);
 156         if (old)
 157                 memcpy(new, old, nr_probes * sizeof(struct tracepoint_probe));
 158         new[nr_probes].func = probe;
 159         new[nr_probes].data = data;
 160         new[nr_probes + 1].func = NULL;
 161         entry->refcount = nr_probes + 1;
 162         entry->probes = new;
 163         debug_print_probes(entry);
 164         return old;
 165 }
 166
 167 static void *
 168 tracepoint_entry_remove_probe(struct tracepoint_entry *entry, void *probe,
 169                               void *data)
 170 {
 171         int nr_probes = 0, nr_del = 0, i;
 172         struct tracepoint_probe *old, *new;
 173
 174         old = entry->probes;
 175
 176         if (!old)
 177                 return ERR_PTR(-ENOENT);
 178
 179         debug_print_probes(entry);
 180         /* (N -> M), (N > 1, M >= 0) probes */
 181         for (nr_probes = 0; old[nr_probes].func; nr_probes++) {
 182                 if (!probe ||
 183                      (old[nr_probes].func == probe &&
 184                      old[nr_probes].data == data))
 185                         nr_del++;
 186         }
 187
 188         if (nr_probes - nr_del == 0) {
 189                 /* N -> 0, (N > 1) */
 190                 entry->probes = NULL;
 191                 entry->refcount = 0;
 192                 debug_print_probes(entry);
 193                 return old;
 194         } else {
 195                 int j = 0;
 196                 /* N -> M, (N > 1, M > 0) */
 197                 /* + 1 for NULL */
 198                 new = allocate_probes(nr_probes - nr_del + 1);
 199                 if (new == NULL)
 200                         return ERR_PTR(-ENOMEM);
 201                 for (i = 0; old[i].func; i++)
 202                         if (probe &&
 203                             (old[i].func != probe || old[i].data != data))
 204                                 new[j++] = old[i];
 205                 new[nr_probes - nr_del].func = NULL;
 206                 entry->refcount = nr_probes - nr_del;
 207                 entry->probes = new;
 208         }
 209         debug_print_probes(entry);
 210         return old;
 211 }
 212
 213 /*
 214  * Get tracepoint if the tracepoint is present in the tracepoint hash table.
 215  * Must be called with tracepoints_mutex held.
 216  * Returns NULL if not present.
 217  */
 218 static struct tracepoint_entry *get_tracepoint(const char *name)
 219 {
 220         struct cds_hlist_head *head;
 221         struct cds_hlist_node *node;
 222         struct tracepoint_entry *e;
 223         u32 hash = jhash(name, strlen(name), 0);
 224
 225         head = &tracepoint_table[hash & (TRACEPOINT_TABLE_SIZE - 1)];
 226         cds_hlist_for_each_entry(e, node, head, hlist) {
 227                 if (!strcmp(name, e->name))
 228                         return e;
 229         }
 230         return NULL;
 231 }
 232
 233 /*
 234  * Add the tracepoint to the tracepoint hash table. Must be called with
 235  * tracepoints_mutex held.
 236  */
 237 static struct tracepoint_entry *add_tracepoint(const char *name)
 238 {
 239         struct cds_hlist_head *head;
 240         struct cds_hlist_node *node;
 241         struct tracepoint_entry *e;
 242         size_t name_len = strlen(name) + 1;
 243         u32 hash = jhash(name, name_len-1, 0);
 244
 245         head = &tracepoint_table[hash & (TRACEPOINT_TABLE_SIZE - 1)];
 246         cds_hlist_for_each_entry(e, node, head, hlist) {
 247                 if (!strcmp(name, e->name)) {
 248                         DBG("tracepoint %s busy", name);
 249                         return ERR_PTR(-EEXIST);        /* Already there */
 250                 }
 251         }
 252         /*
 253          * Using zmalloc here to allocate a variable length element. Could
 254          * cause some memory fragmentation if overused.
 255          */
 256         e = zmalloc(sizeof(struct tracepoint_entry) + name_len);
 257         if (!e)
 258                 return ERR_PTR(-ENOMEM);
 259         memcpy(&e->name[0], name, name_len);
 260         e->probes = NULL;
 261         e->refcount = 0;
 262         cds_hlist_add_head(&e->hlist, head);
 263         return e;
 264 }
 265
 266 /*
 267  * Remove the tracepoint from the tracepoint hash table. Must be called with
 268  * mutex_lock held.
 269  */
 270 static inline void remove_tracepoint(struct tracepoint_entry *e)
 271 {
 272         cds_hlist_del(&e->hlist);
 273         free(e);
 274 }
 275
 276 /*
 277  * Sets the probe callback corresponding to one tracepoint.
 278  */
 279 static void set_tracepoint(struct tracepoint_entry **entry,
 280         struct tracepoint *elem, int active)
 281 {
 282         WARN_ON(strcmp((*entry)->name, elem->name) != 0);
 283
 284         /*
 285          * rcu_assign_pointer has a cmm_smp_wmb() which makes sure that the new
 286          * probe callbacks array is consistent before setting a pointer to it.
 287          * This array is referenced by __DO_TRACE from
 288          * include/linux/tracepoints.h. A matching cmm_smp_read_barrier_depends()
 289          * is used.
 290          */
 291         rcu_assign_pointer(elem->probes, (*entry)->probes);
 292         elem->state = active;
 293 }
 294
 295 /*
 296  * Disable a tracepoint and its probe callback.
 297  * Note: only waiting an RCU period after setting elem->call to the empty
 298  * function insures that the original callback is not used anymore. This insured
 299  * by preempt_disable around the call site.
 300  */
 301 static void disable_tracepoint(struct tracepoint *elem)
 302 {
 303         elem->state = 0;
 304         rcu_assign_pointer(elem->probes, NULL);
 305 }
 306
 307 /**
 308  * tracepoint_update_probe_range - Update a probe range
 309  * @begin: beginning of the range
 310  * @end: end of the range
 311  *
 312  * Updates the probe callback corresponding to a range of tracepoints.
 313  */
 314 static
 315 void tracepoint_update_probe_range(struct tracepoint * const *begin,
 316                                    struct tracepoint * const *end)
 317 {
 318         struct tracepoint * const *iter;
 319         struct tracepoint_entry *mark_entry;
 320
 321         for (iter = begin; iter < end; iter++) {
 322                 if (!*iter)
 323                         continue;       /* skip dummy */
 324                 if (!(*iter)->name) {
 325                         disable_tracepoint(*iter);
 326                         continue;
 327                 }
 328                 mark_entry = get_tracepoint((*iter)->name);
 329                 if (mark_entry) {
 330                         set_tracepoint(&mark_entry, *iter,
 331                                         !!mark_entry->refcount);
 332                 } else {
 333                         disable_tracepoint(*iter);
 334                 }
 335         }
 336 }
 337
 338 static void lib_update_tracepoints(void)
 339 {
 340         struct tracepoint_lib *lib;
 341
 342         lock_tracepoints();
 343         cds_list_for_each_entry(lib, &libs, list) {
 344                 tracepoint_update_probe_range(lib->tracepoints_start,
 345                                 lib->tracepoints_start + lib->tracepoints_count);
 346         }
 347         unlock_tracepoints();
 348 }
 349
 350 /*
 351  * Update probes, removing the faulty probes.
 352  */
 353 static void tracepoint_update_probes(void)
 354 {
 355         /* tracepoints registered from libraries and executable. */
 356         lib_update_tracepoints();
 357 }
 358
 359 static struct tracepoint_probe *
 360 tracepoint_add_probe(const char *name, void *probe, void *data)
 361 {
 362         struct tracepoint_entry *entry;
 363         struct tracepoint_probe *old;
 364
 365         entry = get_tracepoint(name);
 366         if (!entry) {
 367                 entry = add_tracepoint(name);
 368                 if (IS_ERR(entry))
 369                         return (struct tracepoint_probe *)entry;
 370         }
 371         old = tracepoint_entry_add_probe(entry, probe, data);
 372         if (IS_ERR(old) && !entry->refcount)
 373                 remove_tracepoint(entry);
 374         return old;
 375 }
 376
 377 /**
 378  * __tracepoint_probe_register -  Connect a probe to a tracepoint
 379  * @name: tracepoint name
 380  * @probe: probe handler
 381  *
 382  * Returns 0 if ok, error value on error.
 383  * The probe address must at least be aligned on the architecture pointer size.
 384  */
 385 int __tracepoint_probe_register(const char *name, void *probe, void *data)
 386 {
 387         void *old;
 388
 389         lock_tracepoints();
 390         old = tracepoint_add_probe(name, probe, data);
 391         unlock_tracepoints();
 392         if (IS_ERR(old))
 393                 return PTR_ERR(old);
 394
 395         tracepoint_update_probes();             /* may update entry */
 396         release_probes(old);
 397         return 0;
 398 }
 399
 400 static void *tracepoint_remove_probe(const char *name, void *probe, void *data)
 401 {
 402         struct tracepoint_entry *entry;
 403         void *old;
 404
 405         entry = get_tracepoint(name);
 406         if (!entry)
 407                 return ERR_PTR(-ENOENT);
 408         old = tracepoint_entry_remove_probe(entry, probe, data);
 409         if (IS_ERR(old))
 410                 return old;
 411         if (!entry->refcount)
 412                 remove_tracepoint(entry);
 413         return old;
 414 }
 415
 416 /**
 417  * tracepoint_probe_unregister -  Disconnect a probe from a tracepoint
 418  * @name: tracepoint name
 419  * @probe: probe function pointer
 420  * @probe: probe data pointer
 421  *
 422  * We do not need to call a synchronize_sched to make sure the probes have
 423  * finished running before doing a module unload, because the module unload
 424  * itself uses stop_machine(), which insures that every preempt disabled section
 425  * have finished.
 426  */
 427 int __tracepoint_probe_unregister(const char *name, void *probe, void *data)
 428 {
 429         void *old;
 430
 431         lock_tracepoints();
 432         old = tracepoint_remove_probe(name, probe, data);
 433         unlock_tracepoints();
 434         if (IS_ERR(old))
 435                 return PTR_ERR(old);
 436
 437         tracepoint_update_probes();             /* may update entry */
 438         release_probes(old);
 439         return 0;
 440 }
 441
 442 static void tracepoint_add_old_probes(void *old)
 443 {
 444         need_update = 1;
 445         if (old) {
 446                 struct tp_probes *tp_probes = _ust_container_of(old,
 447                         struct tp_probes, probes[0]);
 448                 cds_list_add(&tp_probes->u.list, &old_probes);
 449         }
 450 }
 451
 452 /**
 453  * tracepoint_probe_register_noupdate -  register a probe but not connect
 454  * @name: tracepoint name
 455  * @probe: probe handler
 456  *
 457  * caller must call tracepoint_probe_update_all()
 458  */
 459 int tracepoint_probe_register_noupdate(const char *name, void *probe,
 460                                        void *data)
 461 {
 462         void *old;
 463
 464         lock_tracepoints();
 465         old = tracepoint_add_probe(name, probe, data);
 466         if (IS_ERR(old)) {
 467                 unlock_tracepoints();
 468                 return PTR_ERR(old);
 469         }
 470         tracepoint_add_old_probes(old);
 471         unlock_tracepoints();
 472         return 0;
 473 }
 474
 475 /**
 476  * tracepoint_probe_unregister_noupdate -  remove a probe but not disconnect
 477  * @name: tracepoint name
 478  * @probe: probe function pointer
 479  *
 480  * caller must call tracepoint_probe_update_all()
 481  */
 482 int tracepoint_probe_unregister_noupdate(const char *name, void *probe,
 483                                          void *data)
 484 {
 485         void *old;
 486
 487         lock_tracepoints();
 488         old = tracepoint_remove_probe(name, probe, data);
 489         if (IS_ERR(old)) {
 490                 unlock_tracepoints();
 491                 return PTR_ERR(old);
 492         }
 493         tracepoint_add_old_probes(old);
 494         unlock_tracepoints();
 495         return 0;
 496 }
 497
 498 /**
 499  * tracepoint_probe_update_all -  update tracepoints
 500  */
 501 void tracepoint_probe_update_all(void)
 502 {
 503         CDS_LIST_HEAD(release_probes);
 504         struct tp_probes *pos, *next;
 505
 506         lock_tracepoints();
 507         if (!need_update) {
 508                 unlock_tracepoints();
 509                 return;
 510         }
 511         if (!cds_list_empty(&old_probes))
 512                 cds_list_replace_init(&old_probes, &release_probes);
 513         need_update = 0;
 514         unlock_tracepoints();
 515
 516         tracepoint_update_probes();
 517         cds_list_for_each_entry_safe(pos, next, &release_probes, u.list) {
 518                 cds_list_del(&pos->u.list);
 519                 synchronize_rcu();
 520                 free(pos);
 521         }
 522 }
 523
 524 /*
 525  * Returns 0 if current not found.
 526  * Returns 1 if current found.
 527  *
 528  * Called with tracepoint mutex held
 529  */
 530 int lib_get_iter_tracepoints(struct tracepoint_iter *iter)
 531 {
 532         struct tracepoint_lib *iter_lib;
 533         int found = 0;
 534
 535         cds_list_for_each_entry(iter_lib, &libs, list) {
 536                 if (iter_lib < iter->lib)
 537                         continue;
 538                 else if (iter_lib > iter->lib)
 539                         iter->tracepoint = NULL;
 540                 found = tracepoint_get_iter_range(&iter->tracepoint,
 541                         iter_lib->tracepoints_start,
 542                         iter_lib->tracepoints_start + iter_lib->tracepoints_count);
 543                 if (found) {
 544                         iter->lib = iter_lib;
 545                         break;
 546                 }
 547         }
 548         return found;
 549 }
 550
 551 /**
 552  * tracepoint_get_iter_range - Get a next tracepoint iterator given a range.
 553  * @tracepoint: current tracepoints (in), next tracepoint (out)
 554  * @begin: beginning of the range
 555  * @end: end of the range
 556  *
 557  * Returns whether a next tracepoint has been found (1) or not (0).
 558  * Will return the first tracepoint in the range if the input tracepoint is
 559  * NULL.
 560  * Called with tracepoint mutex held.
 561  */
 562 int tracepoint_get_iter_range(struct tracepoint * const **tracepoint,
 563         struct tracepoint * const *begin, struct tracepoint * const *end)
 564 {
 565         if (!*tracepoint && begin != end)
 566                 *tracepoint = begin;
 567         while (*tracepoint >= begin && *tracepoint < end) {
 568                 if (!**tracepoint)
 569                         (*tracepoint)++;        /* skip dummy */
 570                 else
 571                         return 1;
 572         }
 573         return 0;
 574 }
 575
 576 /*
 577  * Called with tracepoint mutex held.
 578  */
 579 static void tracepoint_get_iter(struct tracepoint_iter *iter)
 580 {
 581         int found = 0;
 582
 583         /* tracepoints in libs. */
 584         found = lib_get_iter_tracepoints(iter);
 585         if (!found)
 586                 tracepoint_iter_reset(iter);
 587 }
 588
 589 void tracepoint_iter_start(struct tracepoint_iter *iter)
 590 {
 591         lock_tracepoints();
 592         tracepoint_get_iter(iter);
 593 }
 594
 595 /*
 596  * Called with tracepoint mutex held.
 597  */
 598 void tracepoint_iter_next(struct tracepoint_iter *iter)
 599 {
 600         iter->tracepoint++;
 601         /*
 602          * iter->tracepoint may be invalid because we blindly incremented it.
 603          * Make sure it is valid by marshalling on the tracepoints, getting the
 604          * tracepoints from following modules if necessary.
 605          */
 606         tracepoint_get_iter(iter);
 607 }
 608
 609 void tracepoint_iter_stop(struct tracepoint_iter *iter)
 610 {
 611         unlock_tracepoints();
 612 }
 613
 614 void tracepoint_iter_reset(struct tracepoint_iter *iter)
 615 {
 616         iter->tracepoint = NULL;
 617 }
 618
 619 void tracepoint_set_new_tracepoint_cb(void (*cb)(struct tracepoint *))
 620 {
 621         new_tracepoint_cb = cb;
 622 }
 623
 624 static void new_tracepoints(struct tracepoint * const *start, struct tracepoint * const *end)
 625 {
 626         if (new_tracepoint_cb) {
 627                 struct tracepoint * const *t;
 628
 629                 for (t = start; t < end; t++) {
 630                         if (*t)
 631                                 new_tracepoint_cb(*t);
 632                 }
 633         }
 634 }
 635
 636 int tracepoint_register_lib(struct tracepoint * const *tracepoints_start,
 637                             int tracepoints_count)
 638 {
 639         struct tracepoint_lib *pl, *iter;
 640
 641         init_tracepoint();
 642
 643         pl = (struct tracepoint_lib *) zmalloc(sizeof(struct tracepoint_lib));
 644
 645         pl->tracepoints_start = tracepoints_start;
 646         pl->tracepoints_count = tracepoints_count;
 647
 648         lock_tracepoints();
 649         /*
 650          * We sort the libs by struct lib pointer address.
 651          */
 652         cds_list_for_each_entry_reverse(iter, &libs, list) {
 653                 BUG_ON(iter == pl);    /* Should never be in the list twice */
 654                 if (iter < pl) {
 655                         /* We belong to the location right after iter. */
 656                         cds_list_add(&pl->list, &iter->list);
 657                         goto lib_added;
 658                 }
 659         }
 660         /* We should be added at the head of the list */
 661         cds_list_add(&pl->list, &libs);
 662 lib_added:
 663         unlock_tracepoints();
 664
 665         new_tracepoints(tracepoints_start, tracepoints_start + tracepoints_count);
 666
 667         /* TODO: update just the loaded lib */
 668         lib_update_tracepoints();
 669
 670         /* tracepoints_count - 1: skip dummy */
 671         DBG("just registered a tracepoints section from %p and having %d tracepoints (minus dummy tracepoints)", tracepoints_start, tracepoints_count);
 672
 673         return 0;
 674 }
 675
 676 int tracepoint_unregister_lib(struct tracepoint * const *tracepoints_start)
 677 {
 678         struct tracepoint_lib *lib;
 679
 680         lock_tracepoints();
 681         cds_list_for_each_entry(lib, &libs, list) {
 682                 if (lib->tracepoints_start == tracepoints_start) {
 683                         struct tracepoint_lib *lib2free = lib;
 684                         cds_list_del(&lib->list);
 685                         free(lib2free);
 686                         break;
 687                 }
 688         }
 689         unlock_tracepoints();
 690
 691         return 0;
 692 }
 693
 694 void init_tracepoint(void)
 695 {
 696         if (uatomic_xchg(&initialized, 1) == 1)
 697                 return;
 698         init_usterr();
 699         tracepoint_register_lib(__start___tracepoints_ptrs,
 700                 __stop___tracepoints_ptrs
 701                 - __start___tracepoints_ptrs);
 702 }
 703
 704 void exit_tracepoint(void)
 705 {
 706         tracepoint_unregister_lib(__start___tracepoints_ptrs);
 707 }