liblttng-ust/tracepoint.c

   1 /*
   2  * Copyright (C) 2008-2011 Mathieu Desnoyers
   3  * Copyright (C) 2009 Pierre-Marc Fournier
   4  *
   5  * This library is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU Lesser General Public
   7  * License as published by the Free Software Foundation;
   8  * version 2.1 of the License.
   9  *
  10  * This library is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * Lesser General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU Lesser General Public
  16  * License along with this library; if not, write to the Free Software
  17  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
  18  *
  19  * Ported to userspace by Pierre-Marc Fournier.
  20  */
  21
  22 #define _LGPL_SOURCE
  23 #include <errno.h>
  24 #include <stdint.h>
  25 #include <stddef.h>
  26
  27 #include <urcu/arch.h>
  28 #include <urcu-bp.h>
  29 #include <urcu/hlist.h>
  30 #include <urcu/uatomic.h>
  31 #include <urcu/compiler.h>
  32
  33 #include <lttng/tracepoint.h>
  34 #include <lttng/ust-abi.h>      /* for LTTNG_UST_SYM_NAME_LEN */
  35
  36 #include <usterr-signal-safe.h>
  37 #include <helper.h>
  38
  39 #include "tracepoint-internal.h"
  40 #include "ltt-tracer-core.h"
  41 #include "jhash.h"
  42 #include "error.h"
  43
  44 /* Set to 1 to enable tracepoint debug output */
  45 static const int tracepoint_debug;
  46 static int initialized;
  47 static void (*new_tracepoint_cb)(struct tracepoint *);
  48
  49 /*
  50  * libraries that contain tracepoints (struct tracepoint_lib).
  51  * Protected by UST lock.
  52  */
  53 static CDS_LIST_HEAD(libs);
  54
  55 /*
  56  * The UST lock protects the library tracepoints, the hash table, and
  57  * the library list.
  58  * All calls to the tracepoint API must be protected by the UST lock,
  59  * excepts calls to tracepoint_register_lib and
  60  * tracepoint_unregister_lib, which take the UST lock themselves.
  61  */
  62
  63 /*
  64  * Tracepoint hash table, containing the active tracepoints.
  65  * Protected by ust lock.
  66  */
  67 #define TRACEPOINT_HASH_BITS 6
  68 #define TRACEPOINT_TABLE_SIZE (1 << TRACEPOINT_HASH_BITS)
  69 static struct cds_hlist_head tracepoint_table[TRACEPOINT_TABLE_SIZE];
  70
  71 static CDS_LIST_HEAD(old_probes);
  72 static int need_update;
  73
  74 /*
  75  * Note about RCU :
  76  * It is used to to delay the free of multiple probes array until a quiescent
  77  * state is reached.
  78  * Tracepoint entries modifications are protected by the ust lock.
  79  */
  80 struct tracepoint_entry {
  81         struct cds_hlist_node hlist;
  82         struct tracepoint_probe *probes;
  83         int refcount;   /* Number of times armed. 0 if disarmed. */
  84         char name[0];
  85 };
  86
  87 struct tp_probes {
  88         union {
  89                 struct cds_list_head list;
  90                 /* Field below only used for call_rcu scheme */
  91                 /* struct rcu_head head; */
  92         } u;
  93         struct tracepoint_probe probes[0];
  94 };
  95
  96 static void *allocate_probes(int count)
  97 {
  98         struct tp_probes *p  = zmalloc(count * sizeof(struct tracepoint_probe)
  99                         + sizeof(struct tp_probes));
 100         return p == NULL ? NULL : p->probes;
 101 }
 102
 103 static void release_probes(void *old)
 104 {
 105         if (old) {
 106                 struct tp_probes *tp_probes = caa_container_of(old,
 107                         struct tp_probes, probes[0]);
 108                 synchronize_rcu();
 109                 free(tp_probes);
 110         }
 111 }
 112
 113 static void debug_print_probes(struct tracepoint_entry *entry)
 114 {
 115         int i;
 116
 117         if (!tracepoint_debug || !entry->probes)
 118                 return;
 119
 120         for (i = 0; entry->probes[i].func; i++)
 121                 DBG("Probe %d : %p", i, entry->probes[i].func);
 122 }
 123
 124 static void *
 125 tracepoint_entry_add_probe(struct tracepoint_entry *entry,
 126                            void *probe, void *data)
 127 {
 128         int nr_probes = 0;
 129         struct tracepoint_probe *old, *new;
 130
 131         WARN_ON(!probe);
 132
 133         debug_print_probes(entry);
 134         old = entry->probes;
 135         if (old) {
 136                 /* (N -> N+1), (N != 0, 1) probes */
 137                 for (nr_probes = 0; old[nr_probes].func; nr_probes++)
 138                         if (old[nr_probes].func == probe &&
 139                             old[nr_probes].data == data)
 140                                 return ERR_PTR(-EEXIST);
 141         }
 142         /* + 2 : one for new probe, one for NULL func */
 143         new = allocate_probes(nr_probes + 2);
 144         if (new == NULL)
 145                 return ERR_PTR(-ENOMEM);
 146         if (old)
 147                 memcpy(new, old, nr_probes * sizeof(struct tracepoint_probe));
 148         new[nr_probes].func = probe;
 149         new[nr_probes].data = data;
 150         new[nr_probes + 1].func = NULL;
 151         entry->refcount = nr_probes + 1;
 152         entry->probes = new;
 153         debug_print_probes(entry);
 154         return old;
 155 }
 156
 157 static void *
 158 tracepoint_entry_remove_probe(struct tracepoint_entry *entry, void *probe,
 159                               void *data)
 160 {
 161         int nr_probes = 0, nr_del = 0, i;
 162         struct tracepoint_probe *old, *new;
 163
 164         old = entry->probes;
 165
 166         if (!old)
 167                 return ERR_PTR(-ENOENT);
 168
 169         debug_print_probes(entry);
 170         /* (N -> M), (N > 1, M >= 0) probes */
 171         for (nr_probes = 0; old[nr_probes].func; nr_probes++) {
 172                 if (!probe ||
 173                      (old[nr_probes].func == probe &&
 174                      old[nr_probes].data == data))
 175                         nr_del++;
 176         }
 177
 178         if (nr_probes - nr_del == 0) {
 179                 /* N -> 0, (N > 1) */
 180                 entry->probes = NULL;
 181                 entry->refcount = 0;
 182                 debug_print_probes(entry);
 183                 return old;
 184         } else {
 185                 int j = 0;
 186                 /* N -> M, (N > 1, M > 0) */
 187                 /* + 1 for NULL */
 188                 new = allocate_probes(nr_probes - nr_del + 1);
 189                 if (new == NULL)
 190                         return ERR_PTR(-ENOMEM);
 191                 for (i = 0; old[i].func; i++)
 192                         if (probe &&
 193                             (old[i].func != probe || old[i].data != data))
 194                                 new[j++] = old[i];
 195                 new[nr_probes - nr_del].func = NULL;
 196                 entry->refcount = nr_probes - nr_del;
 197                 entry->probes = new;
 198         }
 199         debug_print_probes(entry);
 200         return old;
 201 }
 202
 203 /*
 204  * Get tracepoint if the tracepoint is present in the tracepoint hash table.
 205  * Must be called with ust lock held.
 206  * Returns NULL if not present.
 207  */
 208 static struct tracepoint_entry *get_tracepoint(const char *name)
 209 {
 210         struct cds_hlist_head *head;
 211         struct cds_hlist_node *node;
 212         struct tracepoint_entry *e;
 213         size_t name_len = strlen(name);
 214         uint32_t hash;
 215
 216         if (name_len > LTTNG_UST_SYM_NAME_LEN - 1) {
 217                 WARN("Truncating tracepoint name %s which exceeds size limits of %u chars", name, LTTNG_UST_SYM_NAME_LEN - 1);
 218                 name_len = LTTNG_UST_SYM_NAME_LEN - 1;
 219         }
 220         hash = jhash(name, name_len, 0);
 221         head = &tracepoint_table[hash & (TRACEPOINT_TABLE_SIZE - 1)];
 222         cds_hlist_for_each_entry(e, node, head, hlist) {
 223                 if (!strncmp(name, e->name, LTTNG_UST_SYM_NAME_LEN - 1))
 224                         return e;
 225         }
 226         return NULL;
 227 }
 228
 229 /*
 230  * Add the tracepoint to the tracepoint hash table. Must be called with
 231  * ust lock held.
 232  */
 233 static struct tracepoint_entry *add_tracepoint(const char *name)
 234 {
 235         struct cds_hlist_head *head;
 236         struct cds_hlist_node *node;
 237         struct tracepoint_entry *e;
 238         size_t name_len = strlen(name);
 239         uint32_t hash;
 240
 241         if (name_len > LTTNG_UST_SYM_NAME_LEN - 1) {
 242                 WARN("Truncating tracepoint name %s which exceeds size limits of %u chars", name, LTTNG_UST_SYM_NAME_LEN - 1);
 243                 name_len = LTTNG_UST_SYM_NAME_LEN - 1;
 244         }
 245         hash = jhash(name, name_len, 0);
 246         head = &tracepoint_table[hash & (TRACEPOINT_TABLE_SIZE - 1)];
 247         cds_hlist_for_each_entry(e, node, head, hlist) {
 248                 if (!strncmp(name, e->name, LTTNG_UST_SYM_NAME_LEN - 1)) {
 249                         DBG("tracepoint %s busy", name);
 250                         return ERR_PTR(-EEXIST);        /* Already there */
 251                 }
 252         }
 253         /*
 254          * Using zmalloc here to allocate a variable length element. Could
 255          * cause some memory fragmentation if overused.
 256          */
 257         e = zmalloc(sizeof(struct tracepoint_entry) + name_len + 1);
 258         if (!e)
 259                 return ERR_PTR(-ENOMEM);
 260         memcpy(&e->name[0], name, name_len + 1);
 261         e->name[name_len] = '\0';
 262         e->probes = NULL;
 263         e->refcount = 0;
 264         cds_hlist_add_head(&e->hlist, head);
 265         return e;
 266 }
 267
 268 /*
 269  * Remove the tracepoint from the tracepoint hash table. Must be called with
 270  * ust_lock held.
 271  */
 272 static void remove_tracepoint(struct tracepoint_entry *e)
 273 {
 274         cds_hlist_del(&e->hlist);
 275         free(e);
 276 }
 277
 278 /*
 279  * Sets the probe callback corresponding to one tracepoint.
 280  */
 281 static void set_tracepoint(struct tracepoint_entry **entry,
 282         struct tracepoint *elem, int active)
 283 {
 284         WARN_ON(strncmp((*entry)->name, elem->name, LTTNG_UST_SYM_NAME_LEN - 1) != 0);
 285
 286         /*
 287          * rcu_assign_pointer has a cmm_smp_wmb() which makes sure that the new
 288          * probe callbacks array is consistent before setting a pointer to it.
 289          * This array is referenced by __DO_TRACE from
 290          * include/linux/tracepoints.h. A matching cmm_smp_read_barrier_depends()
 291          * is used.
 292          */
 293         rcu_assign_pointer(elem->probes, (*entry)->probes);
 294         elem->state = active;
 295 }
 296
 297 /*
 298  * Disable a tracepoint and its probe callback.
 299  * Note: only waiting an RCU period after setting elem->call to the empty
 300  * function insures that the original callback is not used anymore. This insured
 301  * by preempt_disable around the call site.
 302  */
 303 static void disable_tracepoint(struct tracepoint *elem)
 304 {
 305         elem->state = 0;
 306         rcu_assign_pointer(elem->probes, NULL);
 307 }
 308
 309 /**
 310  * tracepoint_update_probe_range - Update a probe range
 311  * @begin: beginning of the range
 312  * @end: end of the range
 313  *
 314  * Updates the probe callback corresponding to a range of tracepoints.
 315  */
 316 static
 317 void tracepoint_update_probe_range(struct tracepoint * const *begin,
 318                                    struct tracepoint * const *end)
 319 {
 320         struct tracepoint * const *iter;
 321         struct tracepoint_entry *mark_entry;
 322
 323         for (iter = begin; iter < end; iter++) {
 324                 if (!*iter)
 325                         continue;       /* skip dummy */
 326                 if (!(*iter)->name) {
 327                         disable_tracepoint(*iter);
 328                         continue;
 329                 }
 330                 mark_entry = get_tracepoint((*iter)->name);
 331                 if (mark_entry) {
 332                         set_tracepoint(&mark_entry, *iter,
 333                                         !!mark_entry->refcount);
 334                 } else {
 335                         disable_tracepoint(*iter);
 336                 }
 337         }
 338 }
 339
 340 static void lib_update_tracepoints(void)
 341 {
 342         struct tracepoint_lib *lib;
 343
 344         cds_list_for_each_entry(lib, &libs, list) {
 345                 tracepoint_update_probe_range(lib->tracepoints_start,
 346                                 lib->tracepoints_start + lib->tracepoints_count);
 347         }
 348 }
 349
 350 /*
 351  * Update probes, removing the faulty probes.
 352  */
 353 static void tracepoint_update_probes(void)
 354 {
 355         /* tracepoints registered from libraries and executable. */
 356         lib_update_tracepoints();
 357 }
 358
 359 static struct tracepoint_probe *
 360 tracepoint_add_probe(const char *name, void *probe, void *data)
 361 {
 362         struct tracepoint_entry *entry;
 363         struct tracepoint_probe *old;
 364
 365         entry = get_tracepoint(name);
 366         if (!entry) {
 367                 entry = add_tracepoint(name);
 368                 if (IS_ERR(entry))
 369                         return (struct tracepoint_probe *)entry;
 370         }
 371         old = tracepoint_entry_add_probe(entry, probe, data);
 372         if (IS_ERR(old) && !entry->refcount)
 373                 remove_tracepoint(entry);
 374         return old;
 375 }
 376
 377 /**
 378  * __tracepoint_probe_register -  Connect a probe to a tracepoint
 379  * @name: tracepoint name
 380  * @probe: probe handler
 381  *
 382  * Returns 0 if ok, error value on error.
 383  * The probe address must at least be aligned on the architecture pointer size.
 384  * Called with the UST lock held.
 385  */
 386 int __tracepoint_probe_register(const char *name, void *probe, void *data)
 387 {
 388         void *old;
 389
 390         old = tracepoint_add_probe(name, probe, data);
 391         if (IS_ERR(old))
 392                 return PTR_ERR(old);
 393
 394         tracepoint_update_probes();             /* may update entry */
 395         release_probes(old);
 396         return 0;
 397 }
 398
 399 static void *tracepoint_remove_probe(const char *name, void *probe, void *data)
 400 {
 401         struct tracepoint_entry *entry;
 402         void *old;
 403
 404         entry = get_tracepoint(name);
 405         if (!entry)
 406                 return ERR_PTR(-ENOENT);
 407         old = tracepoint_entry_remove_probe(entry, probe, data);
 408         if (IS_ERR(old))
 409                 return old;
 410         if (!entry->refcount)
 411                 remove_tracepoint(entry);
 412         return old;
 413 }
 414
 415 /**
 416  * tracepoint_probe_unregister -  Disconnect a probe from a tracepoint
 417  * @name: tracepoint name
 418  * @probe: probe function pointer
 419  * @probe: probe data pointer
 420  *
 421  * Called with the UST lock held.
 422  */
 423 int __tracepoint_probe_unregister(const char *name, void *probe, void *data)
 424 {
 425         void *old;
 426
 427         old = tracepoint_remove_probe(name, probe, data);
 428         if (IS_ERR(old))
 429                 return PTR_ERR(old);
 430
 431         tracepoint_update_probes();             /* may update entry */
 432         release_probes(old);
 433         return 0;
 434 }
 435
 436 static void tracepoint_add_old_probes(void *old)
 437 {
 438         need_update = 1;
 439         if (old) {
 440                 struct tp_probes *tp_probes = caa_container_of(old,
 441                         struct tp_probes, probes[0]);
 442                 cds_list_add(&tp_probes->u.list, &old_probes);
 443         }
 444 }
 445
 446 /**
 447  * tracepoint_probe_register_noupdate -  register a probe but not connect
 448  * @name: tracepoint name
 449  * @probe: probe handler
 450  *
 451  * caller must call tracepoint_probe_update_all()
 452  * Called with the UST lock held.
 453  */
 454 int tracepoint_probe_register_noupdate(const char *name, void *probe,
 455                                        void *data)
 456 {
 457         void *old;
 458
 459         old = tracepoint_add_probe(name, probe, data);
 460         if (IS_ERR(old)) {
 461                 return PTR_ERR(old);
 462         }
 463         tracepoint_add_old_probes(old);
 464         return 0;
 465 }
 466
 467 /**
 468  * tracepoint_probe_unregister_noupdate -  remove a probe but not disconnect
 469  * @name: tracepoint name
 470  * @probe: probe function pointer
 471  *
 472  * caller must call tracepoint_probe_update_all()
 473  * Called with the UST lock held.
 474  */
 475 int tracepoint_probe_unregister_noupdate(const char *name, void *probe,
 476                                          void *data)
 477 {
 478         void *old;
 479
 480         old = tracepoint_remove_probe(name, probe, data);
 481         if (IS_ERR(old)) {
 482                 return PTR_ERR(old);
 483         }
 484         tracepoint_add_old_probes(old);
 485         return 0;
 486 }
 487
 488 /**
 489  * tracepoint_probe_update_all -  update tracepoints
 490  * Called with the UST lock held.
 491  */
 492 void tracepoint_probe_update_all(void)
 493 {
 494         CDS_LIST_HEAD(release_probes);
 495         struct tp_probes *pos, *next;
 496
 497         if (!need_update) {
 498                 return;
 499         }
 500         if (!cds_list_empty(&old_probes))
 501                 cds_list_replace_init(&old_probes, &release_probes);
 502         need_update = 0;
 503
 504         tracepoint_update_probes();
 505         cds_list_for_each_entry_safe(pos, next, &release_probes, u.list) {
 506                 cds_list_del(&pos->u.list);
 507                 synchronize_rcu();
 508                 free(pos);
 509         }
 510 }
 511
 512 void tracepoint_set_new_tracepoint_cb(void (*cb)(struct tracepoint *))
 513 {
 514         new_tracepoint_cb = cb;
 515 }
 516
 517 static void new_tracepoints(struct tracepoint * const *start, struct tracepoint * const *end)
 518 {
 519         if (new_tracepoint_cb) {
 520                 struct tracepoint * const *t;
 521
 522                 for (t = start; t < end; t++) {
 523                         if (*t)
 524                                 new_tracepoint_cb(*t);
 525                 }
 526         }
 527 }
 528
 529 static
 530 void lib_disable_tracepoints(struct tracepoint * const *begin,
 531                         struct tracepoint * const *end)
 532 {
 533         struct tracepoint * const *iter;
 534
 535         for (iter = begin; iter < end; iter++) {
 536                 if (!*iter)
 537                         continue;       /* skip dummy */
 538                 disable_tracepoint(*iter);
 539         }
 540
 541 }
 542
 543 int tracepoint_register_lib(struct tracepoint * const *tracepoints_start,
 544                             int tracepoints_count)
 545 {
 546         struct tracepoint_lib *pl, *iter;
 547
 548         init_tracepoint();
 549
 550         pl = (struct tracepoint_lib *) zmalloc(sizeof(struct tracepoint_lib));
 551
 552         pl->tracepoints_start = tracepoints_start;
 553         pl->tracepoints_count = tracepoints_count;
 554
 555         ust_lock();
 556         /*
 557          * We sort the libs by struct lib pointer address.
 558          */
 559         cds_list_for_each_entry_reverse(iter, &libs, list) {
 560                 BUG_ON(iter == pl);    /* Should never be in the list twice */
 561                 if (iter < pl) {
 562                         /* We belong to the location right after iter. */
 563                         cds_list_add(&pl->list, &iter->list);
 564                         goto lib_added;
 565                 }
 566         }
 567         /* We should be added at the head of the list */
 568         cds_list_add(&pl->list, &libs);
 569 lib_added:
 570         new_tracepoints(tracepoints_start, tracepoints_start + tracepoints_count);
 571
 572         /* TODO: update just the loaded lib */
 573         lib_update_tracepoints();
 574         ust_unlock();
 575
 576         DBG("just registered a tracepoints section from %p and having %d tracepoints",
 577                 tracepoints_start, tracepoints_count);
 578
 579         return 0;
 580 }
 581
 582 int tracepoint_unregister_lib(struct tracepoint * const *tracepoints_start)
 583 {
 584         struct tracepoint_lib *lib;
 585         int tracepoints_count;
 586
 587         ust_lock();
 588         cds_list_for_each_entry(lib, &libs, list) {
 589                 if (lib->tracepoints_start == tracepoints_start) {
 590                         struct tracepoint_lib *lib2free = lib;
 591
 592                         cds_list_del(&lib->list);
 593                         tracepoints_count = lib->tracepoints_count;
 594                         free(lib2free);
 595                         goto found;
 596                 }
 597         }
 598         goto end;
 599 found:
 600         /*
 601          * Force tracepoint disarm for all tracepoints of this lib.
 602          * This takes care of destructor of library that would leave a
 603          * LD_PRELOAD wrapper override function enabled for tracing, but
 604          * the session teardown would not be able to reach the
 605          * tracepoint anymore to disable it.
 606          */
 607         lib_disable_tracepoints(tracepoints_start,
 608                         tracepoints_start + tracepoints_count);
 609         DBG("just unregistered a tracepoints section from %p",
 610                 tracepoints_start);
 611 end:
 612         ust_unlock();
 613         return 0;
 614 }
 615
 616 void init_tracepoint(void)
 617 {
 618         if (uatomic_xchg(&initialized, 1) == 1)
 619                 return;
 620         init_usterr();
 621 }
 622
 623 void exit_tracepoint(void)
 624 {
 625         initialized = 0;
 626 }
 627
 628 /*
 629  * Create the wrapper symbols.
 630  */
 631 #undef tp_rcu_read_lock_bp
 632 #undef tp_rcu_read_unlock_bp
 633 #undef tp_rcu_dereference_bp
 634
 635 void tp_rcu_read_lock_bp(void)
 636 {
 637         rcu_read_lock_bp();
 638 }
 639
 640 void tp_rcu_read_unlock_bp(void)
 641 {
 642         rcu_read_unlock_bp();
 643 }
 644
 645 void *tp_rcu_dereference_sym_bp(void *p)
 646 {
 647         return rcu_dereference_bp(p);
 648 }