X-Git-Url: http://git.lttng.org/?p=ust.git;a=blobdiff_plain;f=libust%2Fmarker.c;h=a497f0664b7a5125ad0ebf43425fdb7b8cebe5d8;hp=794dd28fcfcf289d40cf2ad28755eda6fc38d3fb;hb=HEAD;hpb=34e4b7db790ca5ff678eae7cf5658455a8991870 diff --git a/libust/marker.c b/libust/marker.c index 794dd28..a497f06 100644 --- a/libust/marker.c +++ b/libust/marker.c @@ -1,10 +1,10 @@ /* - * Copyright (C) 2007 Mathieu Desnoyers + * Copyright (C) 2007-2011 Mathieu Desnoyers * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. + * License as published by the Free Software Foundation; + * version 2.1 of the License. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of @@ -15,82 +15,89 @@ * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ -//ust// #include -//ust// #include -//ust// #include -//#include "jhash.h" -//#include "list.h" -//#include "rcupdate.h" -//ust// #include -#include -//ust// #include -//ust// #include -//ust// #include -//ust// #include -//ust// #include -//ust// #include #define _LGPL_SOURCE -#include - -#include "kernelcompat.h" - -#include "marker.h" -#include "usterr.h" +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "usterr_signal_safe.h" #include "channels.h" #include "tracercore.h" #include "tracer.h" -extern struct marker __start___markers[] __attribute__((visibility("hidden"))); -extern struct marker __stop___markers[] __attribute__((visibility("hidden"))); +extern struct ust_marker * const __start___ust_marker_ptrs[] __attribute__((visibility("hidden"))); +extern struct ust_marker * const __stop___ust_marker_ptrs[] __attribute__((visibility("hidden"))); -/* Set to 1 to enable marker debug output */ -static const int marker_debug; +/* Set to 1 to enable ust_marker debug output */ +static const int ust_marker_debug; +static int initialized; +static void (*new_ust_marker_cb)(struct ust_marker *); /* - * markers_mutex nests inside module_mutex. Markers mutex protects the builtin - * and module markers and the hash table. + * ust_marker mutex protects the builtin and module ust_marker and the + * hash table, as well as the ust_marker_libs list. */ -static DEFINE_MUTEX(markers_mutex); +static DEFINE_MUTEX(ust_marker_mutex); +static CDS_LIST_HEAD(ust_marker_libs); -void lock_markers(void) +/* + * Allow nested mutex for mutex listing and nested enable. + */ +static __thread int nested_mutex; + +void lock_ust_marker(void) { - mutex_lock(&markers_mutex); + if (!(nested_mutex++)) + pthread_mutex_lock(&ust_marker_mutex); } -void unlock_markers(void) +void unlock_ust_marker(void) { - mutex_unlock(&markers_mutex); + if (!(--nested_mutex)) + pthread_mutex_unlock(&ust_marker_mutex); } /* - * Marker hash table, containing the active markers. - * Protected by module_mutex. + * ust_marker hash table, containing the active ust_marker. + * Protected by ust_marker mutex. */ -#define MARKER_HASH_BITS 6 -#define MARKER_TABLE_SIZE (1 << MARKER_HASH_BITS) -static struct hlist_head marker_table[MARKER_TABLE_SIZE]; +#define UST_MARKER_HASH_BITS 6 +#define UST_MARKER_TABLE_SIZE (1 << UST_MARKER_HASH_BITS) +static struct cds_hlist_head ust_marker_table[UST_MARKER_TABLE_SIZE]; + +struct ust_marker_probe_array { + struct rcu_head rcu; + struct ust_marker_probe_closure c[0]; +}; /* * Note about RCU : - * It is used to make sure every handler has finished using its private data - * between two consecutive operation (add or remove) on a given marker. It is - * also used to delay the free of multiple probes array until a quiescent state - * is reached. - * marker entries modifications are protected by the markers_mutex. + * It is used to make sure every handler has finished using its private + * data between two consecutive operation (add or remove) on a given + * ust_marker. It is also used to delay the free of multiple probes + * array until a quiescent state is reached. ust_marker entries + * modifications are protected by the ust_marker_mutex. */ -struct marker_entry { - struct hlist_node hlist; +struct ust_marker_entry { + struct cds_hlist_node hlist; char *format; char *name; /* Probe wrapper */ - void (*call)(const struct marker *mdata, void *call_private, ...); - struct marker_probe_closure single; - struct marker_probe_closure *multi; + void (*call)(const struct ust_marker *mdata, void *call_private, ...); + struct ust_marker_probe_closure single; + struct ust_marker_probe_array *multi; int refcount; /* Number of times armed. 0 if disarmed. */ - struct rcu_head rcu; - void *oldptr; - int rcu_pending; u16 channel_id; u16 event_id; unsigned char ptype:1; @@ -98,44 +105,36 @@ struct marker_entry { char channel[0]; /* Contains channel'\0'name'\0'format'\0' */ }; -#ifdef CONFIG_MARKERS_USERSPACE -static void marker_update_processes(void); -#else -static void marker_update_processes(void) -{ -} -#endif - /** - * __mark_empty_function - Empty probe callback - * @mdata: marker data + * __ust_marker_empty_function - Empty probe callback + * @mdata: ust_marker data * @probe_private: probe private data * @call_private: call site private data * @fmt: format string * @...: variable argument list * - * Empty callback provided as a probe to the markers. By providing this to a - * disabled marker, we make sure the execution flow is always valid even - * though the function pointer change and the marker enabling are two distinct - * operations that modifies the execution flow of preemptible code. + * Empty callback provided as a probe to the ust_marker. By providing + * this to a disabled ust_marker, we make sure the execution flow is + * always valid even though the function pointer change and the + * ust_marker enabling are two distinct operations that modifies the + * execution flow of preemptible code. */ -notrace void __mark_empty_function(const struct marker *mdata, +notrace void __ust_marker_empty_function(const struct ust_marker *mdata, void *probe_private, void *call_private, const char *fmt, va_list *args) { } -//ust// EXPORT_SYMBOL_GPL(__mark_empty_function); /* - * marker_probe_cb Callback that prepares the variable argument list for probes. - * @mdata: pointer of type struct marker + * ust_marker_probe_cb Callback that prepares the variable argument list for probes. + * @mdata: pointer of type struct ust_marker * @call_private: caller site private data * @...: Variable argument list. * * Since we do not use "typical" pointer based RCU in the 1 argument case, we - * need to put a full smp_rmb() in this branch. This is why we do not use + * need to put a full cmm_smp_rmb() in this branch. This is why we do not use * rcu_dereference() for the pointer read. */ -notrace void marker_probe_cb(const struct marker *mdata, +notrace void ust_marker_probe_cb(const struct ust_marker *mdata, void *call_private, ...) { va_list args; @@ -146,133 +145,129 @@ notrace void marker_probe_cb(const struct marker *mdata, * sure the teardown of the callbacks can be done correctly when they * are in modules and they insure RCU read coherency. */ -//ust// rcu_read_lock_sched_notrace(); + rcu_read_lock(); ptype = mdata->ptype; if (likely(!ptype)) { - marker_probe_func *func; + ust_marker_probe_func *func; /* Must read the ptype before ptr. They are not data dependant, - * so we put an explicit smp_rmb() here. */ - smp_rmb(); + * so we put an explicit cmm_smp_rmb() here. */ + cmm_smp_rmb(); func = mdata->single.func; /* Must read the ptr before private data. They are not data - * dependant, so we put an explicit smp_rmb() here. */ - smp_rmb(); + * dependant, so we put an explicit cmm_smp_rmb() here. */ + cmm_smp_rmb(); va_start(args, call_private); func(mdata, mdata->single.probe_private, call_private, mdata->format, &args); va_end(args); } else { - struct marker_probe_closure *multi; + struct ust_marker_probe_array *multi; int i; /* * Read mdata->ptype before mdata->multi. */ - smp_rmb(); + cmm_smp_rmb(); multi = mdata->multi; /* * multi points to an array, therefore accessing the array * depends on reading multi. However, even in this case, * we must insure that the pointer is read _before_ the array - * data. Same as rcu_dereference, but we need a full smp_rmb() - * in the fast path, so put the explicit barrier here. + * data. Same as rcu_dereference, but we need a full cmm_smp_rmb() + * in the fast path, so put the explicit cmm_barrier here. */ - smp_read_barrier_depends(); - for (i = 0; multi[i].func; i++) { + cmm_smp_read_barrier_depends(); + for (i = 0; multi->c[i].func; i++) { va_start(args, call_private); - multi[i].func(mdata, multi[i].probe_private, + multi->c[i].func(mdata, multi->c[i].probe_private, call_private, mdata->format, &args); va_end(args); } } -//ust// rcu_read_unlock_sched_notrace(); + rcu_read_unlock(); } -//ust// EXPORT_SYMBOL_GPL(marker_probe_cb); /* - * marker_probe_cb Callback that does not prepare the variable argument list. - * @mdata: pointer of type struct marker + * ust_marker_probe_cb Callback that does not prepare the variable argument list. + * @mdata: pointer of type struct ust_marker * @call_private: caller site private data * @...: Variable argument list. * - * Should be connected to markers "MARK_NOARGS". + * Should be connected to ust_marker "UST_MARKER_NOARGS". */ -static notrace void marker_probe_cb_noarg(const struct marker *mdata, +static notrace void ust_marker_probe_cb_noarg(const struct ust_marker *mdata, void *call_private, ...) { va_list args; /* not initialized */ char ptype; -//ust// rcu_read_lock_sched_notrace(); + rcu_read_lock(); ptype = mdata->ptype; if (likely(!ptype)) { - marker_probe_func *func; + ust_marker_probe_func *func; /* Must read the ptype before ptr. They are not data dependant, - * so we put an explicit smp_rmb() here. */ - smp_rmb(); + * so we put an explicit cmm_smp_rmb() here. */ + cmm_smp_rmb(); func = mdata->single.func; /* Must read the ptr before private data. They are not data - * dependant, so we put an explicit smp_rmb() here. */ - smp_rmb(); + * dependant, so we put an explicit cmm_smp_rmb() here. */ + cmm_smp_rmb(); func(mdata, mdata->single.probe_private, call_private, mdata->format, &args); } else { - struct marker_probe_closure *multi; + struct ust_marker_probe_array *multi; int i; /* * Read mdata->ptype before mdata->multi. */ - smp_rmb(); + cmm_smp_rmb(); multi = mdata->multi; /* * multi points to an array, therefore accessing the array * depends on reading multi. However, even in this case, * we must insure that the pointer is read _before_ the array - * data. Same as rcu_dereference, but we need a full smp_rmb() - * in the fast path, so put the explicit barrier here. + * data. Same as rcu_dereference, but we need a full cmm_smp_rmb() + * in the fast path, so put the explicit cmm_barrier here. */ - smp_read_barrier_depends(); - for (i = 0; multi[i].func; i++) - multi[i].func(mdata, multi[i].probe_private, + cmm_smp_read_barrier_depends(); + for (i = 0; multi->c[i].func; i++) + multi->c[i].func(mdata, multi->c[i].probe_private, call_private, mdata->format, &args); } -//ust// rcu_read_unlock_sched_notrace(); + rcu_read_unlock(); } static void free_old_closure(struct rcu_head *head) { - struct marker_entry *entry = container_of(head, - struct marker_entry, rcu); - kfree(entry->oldptr); - /* Make sure we free the data before setting the pending flag to 0 */ - smp_wmb(); - entry->rcu_pending = 0; + struct ust_marker_probe_array *multi = + _ust_container_of(head, struct ust_marker_probe_array, rcu); + free(multi); } -static void debug_print_probes(struct marker_entry *entry) +static void debug_print_probes(struct ust_marker_entry *entry) { int i; - if (!marker_debug) + if (!ust_marker_debug) return; if (!entry->ptype) { - printk(KERN_DEBUG "Single probe : %p %p\n", + DBG("Single probe : %p %p", entry->single.func, entry->single.probe_private); } else { - for (i = 0; entry->multi[i].func; i++) - printk(KERN_DEBUG "Multi probe %d : %p %p\n", i, - entry->multi[i].func, - entry->multi[i].probe_private); + for (i = 0; entry->multi->c[i].func; i++) + DBG("Multi probe %d : %p %p", i, + entry->multi->c[i].func, + entry->multi->c[i].probe_private); } } -static struct marker_probe_closure * -marker_entry_add_probe(struct marker_entry *entry, - marker_probe_func *probe, void *probe_private) +static struct ust_marker_probe_array * +ust_marker_entry_add_probe(struct ust_marker_entry *entry, + ust_marker_probe_func *probe, void *probe_private) { int nr_probes = 0; - struct marker_probe_closure *old, *new; + struct ust_marker_probe_array *old, *new; WARN_ON(!probe); @@ -282,7 +277,7 @@ marker_entry_add_probe(struct marker_entry *entry, if (entry->single.func == probe && entry->single.probe_private == probe_private) return ERR_PTR(-EBUSY); - if (entry->single.func == __mark_empty_function) { + if (entry->single.func == __ust_marker_empty_function) { /* 0 -> 1 probes */ entry->single.func = probe; entry->single.probe_private = probe_private; @@ -297,24 +292,24 @@ marker_entry_add_probe(struct marker_entry *entry, } } else { /* (N -> N+1), (N != 0, 1) probes */ - for (nr_probes = 0; old[nr_probes].func; nr_probes++) - if (old[nr_probes].func == probe - && old[nr_probes].probe_private + for (nr_probes = 0; old->c[nr_probes].func; nr_probes++) + if (old->c[nr_probes].func == probe + && old->c[nr_probes].probe_private == probe_private) return ERR_PTR(-EBUSY); } /* + 2 : one for new probe, one for NULL func */ - new = kzalloc((nr_probes + 2) * sizeof(struct marker_probe_closure), - GFP_KERNEL); + new = zmalloc(sizeof(struct ust_marker_probe_array) + + ((nr_probes + 2) * sizeof(struct ust_marker_probe_closure))); if (new == NULL) return ERR_PTR(-ENOMEM); if (!old) - new[0] = entry->single; + new->c[0] = entry->single; else - memcpy(new, old, - nr_probes * sizeof(struct marker_probe_closure)); - new[nr_probes].func = probe; - new[nr_probes].probe_private = probe_private; + memcpy(&new->c[0], &old->c[0], + nr_probes * sizeof(struct ust_marker_probe_closure)); + new->c[nr_probes].func = probe; + new->c[nr_probes].probe_private = probe_private; entry->refcount = nr_probes + 1; entry->multi = new; entry->ptype = 1; @@ -322,32 +317,32 @@ marker_entry_add_probe(struct marker_entry *entry, return old; } -static struct marker_probe_closure * -marker_entry_remove_probe(struct marker_entry *entry, - marker_probe_func *probe, void *probe_private) +static struct ust_marker_probe_array * +ust_marker_entry_remove_probe(struct ust_marker_entry *entry, + ust_marker_probe_func *probe, void *probe_private) { int nr_probes = 0, nr_del = 0, i; - struct marker_probe_closure *old, *new; + struct ust_marker_probe_array *old, *new; old = entry->multi; debug_print_probes(entry); if (!entry->ptype) { /* 0 -> N is an error */ - WARN_ON(entry->single.func == __mark_empty_function); + WARN_ON(entry->single.func == __ust_marker_empty_function); /* 1 -> 0 probes */ WARN_ON(probe && entry->single.func != probe); WARN_ON(entry->single.probe_private != probe_private); - entry->single.func = __mark_empty_function; + entry->single.func = __ust_marker_empty_function; entry->refcount = 0; entry->ptype = 0; debug_print_probes(entry); return NULL; } else { /* (N -> M), (N > 1, M >= 0) probes */ - for (nr_probes = 0; old[nr_probes].func; nr_probes++) { - if ((!probe || old[nr_probes].func == probe) - && old[nr_probes].probe_private + for (nr_probes = 0; old->c[nr_probes].func; nr_probes++) { + if ((!probe || old->c[nr_probes].func == probe) + && old->c[nr_probes].probe_private == probe_private) nr_del++; } @@ -355,29 +350,29 @@ marker_entry_remove_probe(struct marker_entry *entry, if (nr_probes - nr_del == 0) { /* N -> 0, (N > 1) */ - entry->single.func = __mark_empty_function; + entry->single.func = __ust_marker_empty_function; entry->refcount = 0; entry->ptype = 0; } else if (nr_probes - nr_del == 1) { /* N -> 1, (N > 1) */ - for (i = 0; old[i].func; i++) - if ((probe && old[i].func != probe) || - old[i].probe_private != probe_private) - entry->single = old[i]; + for (i = 0; old->c[i].func; i++) + if ((probe && old->c[i].func != probe) || + old->c[i].probe_private != probe_private) + entry->single = old->c[i]; entry->refcount = 1; entry->ptype = 0; } else { int j = 0; /* N -> M, (N > 1, M > 1) */ /* + 1 for NULL */ - new = kzalloc((nr_probes - nr_del + 1) - * sizeof(struct marker_probe_closure), GFP_KERNEL); + new = zmalloc(sizeof(struct ust_marker_probe_array) + + ((nr_probes - nr_del + 1) * sizeof(struct ust_marker_probe_closure))); if (new == NULL) return ERR_PTR(-ENOMEM); - for (i = 0; old[i].func; i++) - if ((probe && old[i].func != probe) || - old[i].probe_private != probe_private) - new[j++] = old[i]; + for (i = 0; old->c[i].func; i++) + if ((probe && old->c[i].func != probe) || + old->c[i].probe_private != probe_private) + new->c[j++] = old->c[i]; entry->refcount = nr_probes - nr_del; entry->ptype = 1; entry->multi = new; @@ -387,22 +382,22 @@ marker_entry_remove_probe(struct marker_entry *entry, } /* - * Get marker if the marker is present in the marker hash table. - * Must be called with markers_mutex held. + * Get ust_marker if the ust_marker is present in the ust_marker hash table. + * Must be called with ust_marker_mutex held. * Returns NULL if not present. */ -static struct marker_entry *get_marker(const char *channel, const char *name) +static struct ust_marker_entry *get_ust_marker(const char *channel, const char *name) { - struct hlist_head *head; - struct hlist_node *node; - struct marker_entry *e; + struct cds_hlist_head *head; + struct cds_hlist_node *node; + struct ust_marker_entry *e; size_t channel_len = strlen(channel) + 1; size_t name_len = strlen(name) + 1; u32 hash; hash = jhash(channel, channel_len-1, 0) ^ jhash(name, name_len-1, 0); - head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; - hlist_for_each_entry(e, node, head, hlist) { + head = &ust_marker_table[hash & ((1 << UST_MARKER_HASH_BITS)-1)]; + cds_hlist_for_each_entry(e, node, head, hlist) { if (!strcmp(channel, e->channel) && !strcmp(name, e->name)) return e; } @@ -410,15 +405,15 @@ static struct marker_entry *get_marker(const char *channel, const char *name) } /* - * Add the marker to the marker hash table. Must be called with markers_mutex - * held. + * Add the ust_marker to the ust_marker hash table. Must be called with + * ust_marker_mutex held. */ -static struct marker_entry *add_marker(const char *channel, const char *name, +static struct ust_marker_entry *add_ust_marker(const char *channel, const char *name, const char *format) { - struct hlist_head *head; - struct hlist_node *node; - struct marker_entry *e; + struct cds_hlist_head *head; + struct cds_hlist_node *node; + struct ust_marker_entry *e; size_t channel_len = strlen(channel) + 1; size_t name_len = strlen(name) + 1; size_t format_len = 0; @@ -427,60 +422,57 @@ static struct marker_entry *add_marker(const char *channel, const char *name, hash = jhash(channel, channel_len-1, 0) ^ jhash(name, name_len-1, 0); if (format) format_len = strlen(format) + 1; - head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; - hlist_for_each_entry(e, node, head, hlist) { + head = &ust_marker_table[hash & ((1 << UST_MARKER_HASH_BITS)-1)]; + cds_hlist_for_each_entry(e, node, head, hlist) { if (!strcmp(channel, e->channel) && !strcmp(name, e->name)) { - printk(KERN_NOTICE - "Marker %s.%s busy\n", channel, name); + DBG("ust_marker %s.%s busy", channel, name); return ERR_PTR(-EBUSY); /* Already there */ } } /* - * Using kmalloc here to allocate a variable length element. Could + * Using zmalloc here to allocate a variable length element. Could * cause some memory fragmentation if overused. */ - e = kmalloc(sizeof(struct marker_entry) - + channel_len + name_len + format_len, - GFP_KERNEL); + e = zmalloc(sizeof(struct ust_marker_entry) + + channel_len + name_len + format_len); if (!e) return ERR_PTR(-ENOMEM); memcpy(e->channel, channel, channel_len); e->name = &e->channel[channel_len]; memcpy(e->name, name, name_len); if (format) { - e->format = &e->name[channel_len + name_len]; + e->format = &e->name[name_len]; memcpy(e->format, format, format_len); - if (strcmp(e->format, MARK_NOARGS) == 0) - e->call = marker_probe_cb_noarg; + if (strcmp(e->format, UST_MARKER_NOARGS) == 0) + e->call = ust_marker_probe_cb_noarg; else - e->call = marker_probe_cb; - trace_mark(metadata, core_marker_format, + e->call = ust_marker_probe_cb; + __ust_marker(metadata, core_marker_format, NULL, "channel %s name %s format %s", e->channel, e->name, e->format); } else { e->format = NULL; - e->call = marker_probe_cb; + e->call = ust_marker_probe_cb; } - e->single.func = __mark_empty_function; + e->single.func = __ust_marker_empty_function; e->single.probe_private = NULL; e->multi = NULL; e->ptype = 0; e->format_allocated = 0; e->refcount = 0; - e->rcu_pending = 0; - hlist_add_head(&e->hlist, head); + cds_hlist_add_head(&e->hlist, head); return e; } /* - * Remove the marker from the marker hash table. Must be called with mutex_lock + * Remove the ust_marker from the ust_marker hash table. Must be called with mutex_lock * held. */ -static int remove_marker(const char *channel, const char *name) +static int remove_ust_marker(const char *channel, const char *name) { - struct hlist_head *head; - struct hlist_node *node; - struct marker_entry *e; + struct cds_hlist_head *head; + struct cds_hlist_node *node; + struct ust_marker_entry *e; int found = 0; size_t channel_len = strlen(channel) + 1; size_t name_len = strlen(name) + 1; @@ -488,8 +480,8 @@ static int remove_marker(const char *channel, const char *name) int ret; hash = jhash(channel, channel_len-1, 0) ^ jhash(name, name_len-1, 0); - head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; - hlist_for_each_entry(e, node, head, hlist) { + head = &ust_marker_table[hash & ((1 << UST_MARKER_HASH_BITS)-1)]; + cds_hlist_for_each_entry(e, node, head, hlist) { if (!strcmp(channel, e->channel) && !strcmp(name, e->name)) { found = 1; break; @@ -497,40 +489,37 @@ static int remove_marker(const char *channel, const char *name) } if (!found) return -ENOENT; - if (e->single.func != __mark_empty_function) + if (e->single.func != __ust_marker_empty_function) return -EBUSY; - hlist_del(&e->hlist); + cds_hlist_del(&e->hlist); if (e->format_allocated) - kfree(e->format); + free(e->format); ret = ltt_channels_unregister(e->channel); WARN_ON(ret); - /* Make sure the call_rcu has been executed */ -//ust// if (e->rcu_pending) -//ust// rcu_barrier_sched(); - kfree(e); + free(e); return 0; } /* * Set the mark_entry format to the format found in the element. */ -static int marker_set_format(struct marker_entry *entry, const char *format) +static int ust_marker_set_format(struct ust_marker_entry *entry, const char *format) { - entry->format = kstrdup(format, GFP_KERNEL); + entry->format = strdup(format); if (!entry->format) return -ENOMEM; entry->format_allocated = 1; - trace_mark(metadata, core_marker_format, + __ust_marker(metadata, core_marker_format, NULL, "channel %s name %s format %s", entry->channel, entry->name, entry->format); return 0; } /* - * Sets the probe callback corresponding to one marker. + * Sets the probe callback corresponding to one ust_marker. */ -static int set_marker(struct marker_entry *entry, struct marker *elem, +static int set_ust_marker(struct ust_marker_entry *entry, struct ust_marker *elem, int active) { int ret = 0; @@ -538,16 +527,14 @@ static int set_marker(struct marker_entry *entry, struct marker *elem, if (entry->format) { if (strcmp(entry->format, elem->format) != 0) { - printk(KERN_NOTICE - "Format mismatch for probe %s " - "(%s), marker (%s)\n", + ERR("Format mismatch for probe %s (%s), ust_marker (%s)", entry->name, entry->format, elem->format); return -EPERM; } } else { - ret = marker_set_format(entry, elem->format); + ret = ust_marker_set_format(entry, elem->format); if (ret) return ret; } @@ -566,7 +553,7 @@ static int set_marker(struct marker_entry *entry, struct marker *elem, * We only update the single probe private data when the ptr is * set to a _non_ single probe! (0 -> 1 and N -> 1, N != 1) */ - WARN_ON(elem->single.func != __mark_empty_function + WARN_ON(elem->single.func != __ust_marker_empty_function && elem->single.probe_private != entry->single.probe_private && !elem->ptype); elem->single.probe_private = entry->single.probe_private; @@ -574,7 +561,7 @@ static int set_marker(struct marker_entry *entry, struct marker *elem, * Make sure the private data is valid when we update the * single probe ptr. */ - smp_wmb(); + cmm_smp_wmb(); elem->single.func = entry->single.func; /* * We also make sure that the new probe callbacks array is consistent @@ -585,74 +572,64 @@ static int set_marker(struct marker_entry *entry, struct marker *elem, * Update the function or multi probe array pointer before setting the * ptype. */ - smp_wmb(); + cmm_smp_wmb(); elem->ptype = entry->ptype; -//ust// if (elem->tp_name && (active ^ _imv_read(elem->state))) { -//ust// WARN_ON(!elem->tp_cb); -//ust// /* -//ust// * It is ok to directly call the probe registration because type -//ust// * checking has been done in the __trace_mark_tp() macro. -//ust// */ -//ust// -//ust// if (active) { -//ust// /* -//ust// * try_module_get should always succeed because we hold -//ust// * markers_mutex to get the tp_cb address. -//ust// */ -//ust// ret = try_module_get(__module_text_address( -//ust// (unsigned long)elem->tp_cb)); -//ust// BUG_ON(!ret); -//ust// ret = tracepoint_probe_register_noupdate( -//ust// elem->tp_name, -//ust// elem->tp_cb); -//ust// } else { -//ust// ret = tracepoint_probe_unregister_noupdate( -//ust// elem->tp_name, -//ust// elem->tp_cb); -//ust// /* -//ust// * tracepoint_probe_update_all() must be called -//ust// * before the module containing tp_cb is unloaded. -//ust// */ -//ust// module_put(__module_text_address( -//ust// (unsigned long)elem->tp_cb)); -//ust// } -//ust// } - elem->state__imv = active; + if (elem->tp_name && (active ^ elem->state)) { + WARN_ON(!elem->tp_cb); + /* + * It is ok to directly call the probe registration because type + * checking has been done in the __ust_marker_tp() macro. + */ + + if (active) { + ret = tracepoint_probe_register_noupdate( + elem->tp_name, + elem->tp_cb, NULL); + } else { + /* + * tracepoint_probe_update_all() must be called + * before the library containing tp_cb is unloaded. + */ + ret = tracepoint_probe_unregister_noupdate( + elem->tp_name, + elem->tp_cb, NULL); + } + } + elem->state = active; return ret; } /* - * Disable a marker and its probe callback. + * Disable a ust_marker and its probe callback. * Note: only waiting an RCU period after setting elem->call to the empty * function insures that the original callback is not used anymore. This insured - * by rcu_read_lock_sched around the call site. + * by rcu_read_lock around the call site. */ -static void disable_marker(struct marker *elem) +static void disable_ust_marker(struct ust_marker *elem) { int ret; /* leave "call" as is. It is known statically. */ -//ust// if (elem->tp_name && _imv_read(elem->state)) { -//ust// WARN_ON(!elem->tp_cb); -//ust// /* -//ust// * It is ok to directly call the probe registration because type -//ust// * checking has been done in the __trace_mark_tp() macro. -//ust// */ -//ust// ret = tracepoint_probe_unregister_noupdate(elem->tp_name, -//ust// elem->tp_cb); -//ust// WARN_ON(ret); -//ust// /* -//ust// * tracepoint_probe_update_all() must be called -//ust// * before the module containing tp_cb is unloaded. -//ust// */ -//ust// module_put(__module_text_address((unsigned long)elem->tp_cb)); -//ust// } - elem->state__imv = 0; - elem->single.func = __mark_empty_function; + if (elem->tp_name && elem->state) { + WARN_ON(!elem->tp_cb); + /* + * It is ok to directly call the probe registration because type + * checking has been done in the __ust_marker_tp() macro. + */ + /* + * tracepoint_probe_update_all() must be called + * before the module containing tp_cb is unloaded. + */ + ret = tracepoint_probe_unregister_noupdate(elem->tp_name, + elem->tp_cb, NULL); + WARN_ON(ret); + } + elem->state = 0; + elem->single.func = __ust_marker_empty_function; /* Update the function before setting the ptype */ - smp_wmb(); + cmm_smp_wmb(); elem->ptype = 0; /* single probe */ /* * Leave the private data and channel_id/event_id there, because removal @@ -661,46 +638,62 @@ static void disable_marker(struct marker *elem) */ } +/* + * is_ust_marker_enabled - Check if a ust_marker is enabled + * @channel: channel name + * @name: ust_marker name + * + * Returns 1 if the ust_marker is enabled, 0 if disabled. + */ +int is_ust_marker_enabled(const char *channel, const char *name) +{ + struct ust_marker_entry *entry; + + lock_ust_marker(); + entry = get_ust_marker(channel, name); + unlock_ust_marker(); + + return entry && !!entry->refcount; +} + /** - * marker_update_probe_range - Update a probe range + * ust_marker_update_probe_range - Update a probe range * @begin: beginning of the range * @end: end of the range * - * Updates the probe callback corresponding to a range of markers. + * Updates the probe callback corresponding to a range of ust_marker. */ -void marker_update_probe_range(struct marker *begin, - struct marker *end) +static +void ust_marker_update_probe_range(struct ust_marker * const *begin, + struct ust_marker * const *end) { - struct marker *iter; - struct marker_entry *mark_entry; + struct ust_marker * const *iter; + struct ust_marker_entry *mark_entry; - mutex_lock(&markers_mutex); for (iter = begin; iter < end; iter++) { - mark_entry = get_marker(iter->channel, iter->name); + if (!*iter) + continue; /* skip dummy */ + mark_entry = get_ust_marker((*iter)->channel, (*iter)->name); if (mark_entry) { - set_marker(mark_entry, iter, !!mark_entry->refcount); + set_ust_marker(mark_entry, *iter, !!mark_entry->refcount); /* * ignore error, continue */ - - /* This is added for UST. We emit a core_marker_id event - * for markers that are already registered to a probe - * upon library load. Otherwise, no core_marker_id will - * be generated for these markers. Is this the right thing - * to do? - */ - trace_mark(metadata, core_marker_id, - "channel %s name %s event_id %hu " - "int #1u%zu long #1u%zu pointer #1u%zu " - "size_t #1u%zu alignment #1u%u", - iter->channel, iter->name, mark_entry->event_id, - sizeof(int), sizeof(long), sizeof(void *), - sizeof(size_t), ltt_get_alignment()); } else { - disable_marker(iter); + disable_ust_marker(*iter); } } - mutex_unlock(&markers_mutex); +} + +static void lib_update_ust_marker(void) +{ + struct ust_marker_lib *lib; + + lock_ust_marker(); + cds_list_for_each_entry(lib, &ust_marker_libs, list) + ust_marker_update_probe_range(lib->ust_marker_start, + lib->ust_marker_start + lib->ust_marker_count); + unlock_ust_marker(); } /* @@ -717,27 +710,19 @@ void marker_update_probe_range(struct marker *begin, * 1 -> 2 callbacks * 2 -> 1 callbacks * Other updates all behave the same, just like the 2 -> 3 or 3 -> 2 updates. - * Site effect : marker_set_format may delete the marker entry (creating a + * Site effect : ust_marker_set_format may delete the ust_marker entry (creating a * replacement). */ -static void marker_update_probes(void) +static void ust_marker_update_probes(void) { - /* Core kernel markers */ -//ust// marker_update_probe_range(__start___markers, __stop___markers); - /* Markers in modules. */ -//ust// module_update_markers(); - lib_update_markers(); -//ust// tracepoint_probe_update_all(); - /* Update immediate values */ - core_imv_update(); -//ust// module_imv_update(); /* FIXME: need to port for libs? */ - marker_update_processes(); + lib_update_ust_marker(); + tracepoint_probe_update_all(); } /** - * marker_probe_register - Connect a probe to a marker - * @channel: marker channel - * @name: marker name + * ust_marker_probe_register - Connect a probe to a ust_marker + * @channel: ust_marker channel + * @name: ust_marker name * @format: format string * @probe: probe handler * @probe_private: probe private data @@ -746,27 +731,27 @@ static void marker_update_probes(void) * Returns 0 if ok, error value on error. * The probe address must at least be aligned on the architecture pointer size. */ -int marker_probe_register(const char *channel, const char *name, - const char *format, marker_probe_func *probe, +int ust_marker_probe_register(const char *channel, const char *name, + const char *format, ust_marker_probe_func *probe, void *probe_private) { - struct marker_entry *entry; + struct ust_marker_entry *entry; int ret = 0, ret_err; - struct marker_probe_closure *old; + struct ust_marker_probe_array *old; int first_probe = 0; - mutex_lock(&markers_mutex); - entry = get_marker(channel, name); + lock_ust_marker(); + entry = get_ust_marker(channel, name); if (!entry) { first_probe = 1; - entry = add_marker(channel, name, format); + entry = add_ust_marker(channel, name, format); if (IS_ERR(entry)) ret = PTR_ERR(entry); if (ret) goto end; ret = ltt_channels_register(channel); if (ret) - goto error_remove_marker; + goto error_remove_ust_marker; ret = ltt_channels_get_index_from_name(channel); if (ret < 0) goto error_unregister_channel; @@ -776,7 +761,7 @@ int marker_probe_register(const char *channel, const char *name, goto error_unregister_channel; entry->event_id = ret; ret = 0; - trace_mark(metadata, core_marker_id, + __ust_marker(metadata, core_marker_id, NULL, "channel %s name %s event_id %hu " "int #1u%zu long #1u%zu pointer #1u%zu " "size_t #1u%zu alignment #1u%u", @@ -785,20 +770,14 @@ int marker_probe_register(const char *channel, const char *name, sizeof(size_t), ltt_get_alignment()); } else if (format) { if (!entry->format) - ret = marker_set_format(entry, format); + ret = ust_marker_set_format(entry, format); else if (strcmp(entry->format, format)) ret = -EPERM; if (ret) goto end; } - /* - * If we detect that a call_rcu is pending for this marker, - * make sure it's executed now. - */ -//ust// if (entry->rcu_pending) -//ust// rcu_barrier_sched(); - old = marker_entry_add_probe(entry, probe, probe_private); + old = ust_marker_entry_add_probe(entry, probe, probe_private); if (IS_ERR(old)) { ret = PTR_ERR(old); if (first_probe) @@ -806,109 +785,93 @@ int marker_probe_register(const char *channel, const char *name, else goto end; } - mutex_unlock(&markers_mutex); + unlock_ust_marker(); - marker_update_probes(); + /* Activate ust_marker if necessary */ + ust_marker_update_probes(); - mutex_lock(&markers_mutex); - entry = get_marker(channel, name); - if (!entry) - goto end; -//ust// if (entry->rcu_pending) -//ust// rcu_barrier_sched(); - entry->oldptr = old; - entry->rcu_pending = 1; - /* write rcu_pending before calling the RCU callback */ - smp_wmb(); -//ust// call_rcu_sched(&entry->rcu, free_old_closure); - synchronize_rcu(); free_old_closure(&entry->rcu); - goto end; + if (old) { + synchronize_rcu(); + free_old_closure(&old->rcu); + } + return ret; error_unregister_channel: ret_err = ltt_channels_unregister(channel); WARN_ON(ret_err); -error_remove_marker: - ret_err = remove_marker(channel, name); +error_remove_ust_marker: + ret_err = remove_ust_marker(channel, name); WARN_ON(ret_err); end: - mutex_unlock(&markers_mutex); + unlock_ust_marker(); return ret; } -//ust// EXPORT_SYMBOL_GPL(marker_probe_register); /** - * marker_probe_unregister - Disconnect a probe from a marker - * @channel: marker channel - * @name: marker name + * ust_marker_probe_unregister - Disconnect a probe from a ust_marker + * @channel: ust_marker channel + * @name: ust_marker name * @probe: probe function pointer * @probe_private: probe private data * - * Returns the private data given to marker_probe_register, or an ERR_PTR(). + * Returns the private data given to ust_marker_probe_register, or an ERR_PTR(). * We do not need to call a synchronize_sched to make sure the probes have * finished running before doing a module unload, because the module unload * itself uses stop_machine(), which insures that every preempt disabled section * have finished. */ -int marker_probe_unregister(const char *channel, const char *name, - marker_probe_func *probe, void *probe_private) +int ust_marker_probe_unregister(const char *channel, const char *name, + ust_marker_probe_func *probe, void *probe_private) { - struct marker_entry *entry; - struct marker_probe_closure *old; - int ret = -ENOENT; + struct ust_marker_entry *entry; + struct ust_marker_probe_array *old; + int ret = 0; - mutex_lock(&markers_mutex); - entry = get_marker(channel, name); - if (!entry) + lock_ust_marker(); + entry = get_ust_marker(channel, name); + if (!entry) { + ret = -ENOENT; goto end; -//ust// if (entry->rcu_pending) -//ust// rcu_barrier_sched(); - old = marker_entry_remove_probe(entry, probe, probe_private); - mutex_unlock(&markers_mutex); + } + old = ust_marker_entry_remove_probe(entry, probe, probe_private); + unlock_ust_marker(); - marker_update_probes(); + ust_marker_update_probes(); + + if (old) { + synchronize_rcu(); + free_old_closure(&old->rcu); + } + return ret; - mutex_lock(&markers_mutex); - entry = get_marker(channel, name); - if (!entry) - goto end; -//ust// if (entry->rcu_pending) -//ust// rcu_barrier_sched(); - entry->oldptr = old; - entry->rcu_pending = 1; - /* write rcu_pending before calling the RCU callback */ - smp_wmb(); -//ust// call_rcu_sched(&entry->rcu, free_old_closure); - synchronize_rcu(); free_old_closure(&entry->rcu); - remove_marker(channel, name); /* Ignore busy error message */ - ret = 0; end: - mutex_unlock(&markers_mutex); + unlock_ust_marker(); return ret; } -//ust// EXPORT_SYMBOL_GPL(marker_probe_unregister); -static struct marker_entry * -get_marker_from_private_data(marker_probe_func *probe, void *probe_private) +static struct ust_marker_entry * +get_ust_marker_from_private_data(ust_marker_probe_func *probe, + void *probe_private) { - struct marker_entry *entry; + struct ust_marker_entry *entry; unsigned int i; - struct hlist_head *head; - struct hlist_node *node; + struct cds_hlist_head *head; + struct cds_hlist_node *node; - for (i = 0; i < MARKER_TABLE_SIZE; i++) { - head = &marker_table[i]; - hlist_for_each_entry(entry, node, head, hlist) { + for (i = 0; i < UST_MARKER_TABLE_SIZE; i++) { + head = &ust_marker_table[i]; + cds_hlist_for_each_entry(entry, node, head, hlist) { if (!entry->ptype) { if (entry->single.func == probe && entry->single.probe_private == probe_private) return entry; } else { - struct marker_probe_closure *closure; + struct ust_marker_probe_array *closure; closure = entry->multi; - for (i = 0; closure[i].func; i++) { - if (closure[i].func == probe && - closure[i].probe_private + for (i = 0; closure->c[i].func; i++) { + if (closure->c[i].func == probe && + closure->c[i].probe_private == probe_private) return entry; } @@ -919,67 +882,59 @@ get_marker_from_private_data(marker_probe_func *probe, void *probe_private) } /** - * marker_probe_unregister_private_data - Disconnect a probe from a marker + * ust_marker_probe_unregister_private_data - Disconnect a probe from a ust_marker * @probe: probe function * @probe_private: probe private data * * Unregister a probe by providing the registered private data. - * Only removes the first marker found in hash table. + * Only removes the first ust_marker found in hash table. * Return 0 on success or error value. * We do not need to call a synchronize_sched to make sure the probes have * finished running before doing a module unload, because the module unload * itself uses stop_machine(), which insures that every preempt disabled section * have finished. */ -int marker_probe_unregister_private_data(marker_probe_func *probe, +int ust_marker_probe_unregister_private_data(ust_marker_probe_func *probe, void *probe_private) { - struct marker_entry *entry; + struct ust_marker_entry *entry; int ret = 0; - struct marker_probe_closure *old; - const char *channel = NULL, *name = NULL; + struct ust_marker_probe_array *old; + char *channel = NULL, *name = NULL; - mutex_lock(&markers_mutex); - entry = get_marker_from_private_data(probe, probe_private); + lock_ust_marker(); + entry = get_ust_marker_from_private_data(probe, probe_private); if (!entry) { ret = -ENOENT; - goto end; + goto unlock; } -//ust// if (entry->rcu_pending) -//ust// rcu_barrier_sched(); - old = marker_entry_remove_probe(entry, NULL, probe_private); - channel = kstrdup(entry->channel, GFP_KERNEL); - name = kstrdup(entry->name, GFP_KERNEL); - mutex_unlock(&markers_mutex); - - marker_update_probes(); - - mutex_lock(&markers_mutex); - entry = get_marker(channel, name); - if (!entry) - goto end; -//ust// if (entry->rcu_pending) -//ust// rcu_barrier_sched(); - entry->oldptr = old; - entry->rcu_pending = 1; - /* write rcu_pending before calling the RCU callback */ - smp_wmb(); -//ust// call_rcu_sched(&entry->rcu, free_old_closure); - synchronize_rcu(); free_old_closure(&entry->rcu); + old = ust_marker_entry_remove_probe(entry, NULL, probe_private); + channel = strdup(entry->channel); + name = strdup(entry->name); /* Ignore busy error message */ - remove_marker(channel, name); + remove_ust_marker(channel, name); + unlock_ust_marker(); + + ust_marker_update_probes(); + + if (old) { + synchronize_rcu(); + free_old_closure(&old->rcu); + } + goto end; + +unlock: + unlock_ust_marker(); end: - mutex_unlock(&markers_mutex); - kfree(channel); - kfree(name); + free(channel); + free(name); return ret; } -//ust// EXPORT_SYMBOL_GPL(marker_probe_unregister_private_data); /** - * marker_get_private_data - Get a marker's probe private data - * @channel: marker channel - * @name: marker name + * ust_marker_get_private_data - Get a ust_marker's probe private data + * @channel: ust_marker channel + * @name: ust_marker name * @probe: probe to match * @num: get the nth matching probe's private data * @@ -990,33 +945,33 @@ end: * owner of the data, or its content could vanish. This is mostly used to * confirm that a caller is the owner of a registered probe. */ -void *marker_get_private_data(const char *channel, const char *name, - marker_probe_func *probe, int num) +void *ust_marker_get_private_data(const char *channel, const char *name, + ust_marker_probe_func *probe, int num) { - struct hlist_head *head; - struct hlist_node *node; - struct marker_entry *e; + struct cds_hlist_head *head; + struct cds_hlist_node *node; + struct ust_marker_entry *e; size_t channel_len = strlen(channel) + 1; size_t name_len = strlen(name) + 1; int i; u32 hash; hash = jhash(channel, channel_len-1, 0) ^ jhash(name, name_len-1, 0); - head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; - hlist_for_each_entry(e, node, head, hlist) { + head = &ust_marker_table[hash & ((1 << UST_MARKER_HASH_BITS)-1)]; + cds_hlist_for_each_entry(e, node, head, hlist) { if (!strcmp(channel, e->channel) && !strcmp(name, e->name)) { if (!e->ptype) { if (num == 0 && e->single.func == probe) return e->single.probe_private; } else { - struct marker_probe_closure *closure; + struct ust_marker_probe_array *closure; int match = 0; closure = e->multi; - for (i = 0; closure[i].func; i++) { - if (closure[i].func != probe) + for (i = 0; closure->c[i].func; i++) { + if (closure->c[i].func != probe) continue; if (match++ == num) - return closure[i].probe_private; + return closure->c[i].probe_private; } } break; @@ -1024,515 +979,224 @@ void *marker_get_private_data(const char *channel, const char *name, } return ERR_PTR(-ENOENT); } -//ust// EXPORT_SYMBOL_GPL(marker_get_private_data); - -/** - * markers_compact_event_ids - Compact markers event IDs and reassign channels - * - * Called when no channel users are active by the channel infrastructure. - * Called with lock_markers() and channel mutex held. - */ -//ust// void markers_compact_event_ids(void) -//ust// { -//ust// struct marker_entry *entry; -//ust// unsigned int i; -//ust// struct hlist_head *head; -//ust// struct hlist_node *node; -//ust// int ret; -//ust// -//ust// for (i = 0; i < MARKER_TABLE_SIZE; i++) { -//ust// head = &marker_table[i]; -//ust// hlist_for_each_entry(entry, node, head, hlist) { -//ust// ret = ltt_channels_get_index_from_name(entry->channel); -//ust// WARN_ON(ret < 0); -//ust// entry->channel_id = ret; -//ust// ret = _ltt_channels_get_event_id(entry->channel, -//ust// entry->name); -//ust// WARN_ON(ret < 0); -//ust// entry->event_id = ret; -//ust// } -//ust// } -//ust// } - -//ust//#ifdef CONFIG_MODULES /** - * marker_get_iter_range - Get a next marker iterator given a range. - * @marker: current markers (in), next marker (out) + * ust_marker_get_iter_range - Get a next ust_marker iterator given a range. + * @ust_marker: current ust_marker (in), next ust_marker (out) * @begin: beginning of the range * @end: end of the range * - * Returns whether a next marker has been found (1) or not (0). - * Will return the first marker in the range if the input marker is NULL. + * Returns whether a next ust_marker has been found (1) or not (0). + * Will return the first ust_marker in the range if the input ust_marker is NULL. + * Called with markers mutex held. */ -int marker_get_iter_range(struct marker **marker, struct marker *begin, - struct marker *end) +static +int ust_marker_get_iter_range(struct ust_marker * const **ust_marker, + struct ust_marker * const *begin, + struct ust_marker * const *end) { - if (!*marker && begin != end) { - *marker = begin; - return 1; + if (!*ust_marker && begin != end) + *ust_marker = begin; + while (*ust_marker >= begin && *ust_marker < end) { + if (!**ust_marker) + (*ust_marker)++; /* skip dummy */ + else + return 1; } - if (*marker >= begin && *marker < end) - return 1; return 0; } -//ust// EXPORT_SYMBOL_GPL(marker_get_iter_range); -static void marker_get_iter(struct marker_iter *iter) +/* + * Returns 0 if current not found. + * Returns 1 if current found. + * Called with markers mutex held. + */ +static +int lib_get_iter_ust_marker(struct ust_marker_iter *iter) { + struct ust_marker_lib *iter_lib; int found = 0; - /* Core kernel markers */ - if (!iter->lib) { - /* ust FIXME: how come we cannot disable the following line? we shouldn't need core stuff */ - found = marker_get_iter_range(&iter->marker, - __start___markers, __stop___markers); - if (found) - goto end; + cds_list_for_each_entry(iter_lib, &ust_marker_libs, list) { + if (iter_lib < iter->lib) + continue; + else if (iter_lib > iter->lib) + iter->ust_marker = NULL; + found = ust_marker_get_iter_range(&iter->ust_marker, + iter_lib->ust_marker_start, + iter_lib->ust_marker_start + iter_lib->ust_marker_count); + if (found) { + iter->lib = iter_lib; + break; + } } - /* Markers in modules. */ - found = lib_get_iter_markers(iter); -end: - if (!found) - marker_iter_reset(iter); -} - -void marker_iter_start(struct marker_iter *iter) -{ - marker_get_iter(iter); -} -//ust// EXPORT_SYMBOL_GPL(marker_iter_start); - -void marker_iter_next(struct marker_iter *iter) -{ - iter->marker++; - /* - * iter->marker may be invalid because we blindly incremented it. - * Make sure it is valid by marshalling on the markers, getting the - * markers from following modules if necessary. - */ - marker_get_iter(iter); + return found; } -//ust// EXPORT_SYMBOL_GPL(marker_iter_next); -void marker_iter_stop(struct marker_iter *iter) +/* Called with markers mutex held. */ +static void ust_marker_get_iter(struct ust_marker_iter *iter) { -} -//ust// EXPORT_SYMBOL_GPL(marker_iter_stop); + int found = 0; -void marker_iter_reset(struct marker_iter *iter) -{ - iter->lib = NULL; - iter->marker = NULL; + found = lib_get_iter_ust_marker(iter); + if (!found) + ust_marker_iter_reset(iter); } -//ust// EXPORT_SYMBOL_GPL(marker_iter_reset); -#ifdef CONFIG_MARKERS_USERSPACE -/* - * must be called with current->user_markers_mutex held - */ -static void free_user_marker(char __user *state, struct hlist_head *head) +void ust_marker_iter_start(struct ust_marker_iter *iter) { - struct user_marker *umark; - struct hlist_node *pos, *n; - - hlist_for_each_entry_safe(umark, pos, n, head, hlist) { - if (umark->state == state) { - hlist_del(&umark->hlist); - kfree(umark); - } - } + lock_ust_marker(); + ust_marker_get_iter(iter); } -//ust// asmlinkage long sys_marker(char __user *name, char __user *format, -//ust// char __user *state, int reg) -//ust// { -//ust// struct user_marker *umark; -//ust// long len; -//ust// struct marker_entry *entry; -//ust// int ret = 0; -//ust// -//ust// printk(KERN_DEBUG "Program %s %s marker [%p, %p]\n", -//ust// current->comm, reg ? "registers" : "unregisters", -//ust// name, state); -//ust// if (reg) { -//ust// umark = kmalloc(sizeof(struct user_marker), GFP_KERNEL); -//ust// umark->name[MAX_USER_MARKER_NAME_LEN - 1] = '\0'; -//ust// umark->format[MAX_USER_MARKER_FORMAT_LEN - 1] = '\0'; -//ust// umark->state = state; -//ust// len = strncpy_from_user(umark->name, name, -//ust// MAX_USER_MARKER_NAME_LEN - 1); -//ust// if (len < 0) { -//ust// ret = -EFAULT; -//ust// goto error; -//ust// } -//ust// len = strncpy_from_user(umark->format, format, -//ust// MAX_USER_MARKER_FORMAT_LEN - 1); -//ust// if (len < 0) { -//ust// ret = -EFAULT; -//ust// goto error; -//ust// } -//ust// printk(KERN_DEBUG "Marker name : %s, format : %s", umark->name, -//ust// umark->format); -//ust// mutex_lock(&markers_mutex); -//ust// entry = get_marker("userspace", umark->name); -//ust// if (entry) { -//ust// if (entry->format && -//ust// strcmp(entry->format, umark->format) != 0) { -//ust// printk(" error, wrong format in process %s", -//ust// current->comm); -//ust// ret = -EPERM; -//ust// goto error_unlock; -//ust// } -//ust// printk(" %s", !!entry->refcount -//ust// ? "enabled" : "disabled"); -//ust// if (put_user(!!entry->refcount, state)) { -//ust// ret = -EFAULT; -//ust// goto error_unlock; -//ust// } -//ust// printk("\n"); -//ust// } else { -//ust// printk(" disabled\n"); -//ust// if (put_user(0, umark->state)) { -//ust// printk(KERN_WARNING -//ust// "Marker in %s caused a fault\n", -//ust// current->comm); -//ust// goto error_unlock; -//ust// } -//ust// } -//ust// mutex_lock(¤t->group_leader->user_markers_mutex); -//ust// hlist_add_head(&umark->hlist, -//ust// ¤t->group_leader->user_markers); -//ust// current->group_leader->user_markers_sequence++; -//ust// mutex_unlock(¤t->group_leader->user_markers_mutex); -//ust// mutex_unlock(&markers_mutex); -//ust// } else { -//ust// mutex_lock(¤t->group_leader->user_markers_mutex); -//ust// free_user_marker(state, -//ust// ¤t->group_leader->user_markers); -//ust// current->group_leader->user_markers_sequence++; -//ust// mutex_unlock(¤t->group_leader->user_markers_mutex); -//ust// } -//ust// goto end; -//ust// error_unlock: -//ust// mutex_unlock(&markers_mutex); -//ust// error: -//ust// kfree(umark); -//ust// end: -//ust// return ret; -//ust// } -//ust// -//ust// /* -//ust// * Types : -//ust// * string : 0 -//ust// */ -//ust// asmlinkage long sys_trace(int type, uint16_t id, -//ust// char __user *ubuf) -//ust// { -//ust// long ret = -EPERM; -//ust// char *page; -//ust// int len; -//ust// -//ust// switch (type) { -//ust// case 0: /* String */ -//ust// ret = -ENOMEM; -//ust// page = (char *)__get_free_page(GFP_TEMPORARY); -//ust// if (!page) -//ust// goto string_out; -//ust// len = strncpy_from_user(page, ubuf, PAGE_SIZE); -//ust// if (len < 0) { -//ust// ret = -EFAULT; -//ust// goto string_err; -//ust// } -//ust// trace_mark(userspace, string, "string %s", page); -//ust// string_err: -//ust// free_page((unsigned long) page); -//ust// string_out: -//ust// break; -//ust// default: -//ust// break; -//ust// } -//ust// return ret; -//ust// } - -//ust// static void marker_update_processes(void) -//ust// { -//ust// struct task_struct *g, *t; -//ust// -//ust// /* -//ust// * markers_mutex is taken to protect the p->user_markers read. -//ust// */ -//ust// mutex_lock(&markers_mutex); -//ust// read_lock(&tasklist_lock); -//ust// for_each_process(g) { -//ust// WARN_ON(!thread_group_leader(g)); -//ust// if (hlist_empty(&g->user_markers)) -//ust// continue; -//ust// if (strcmp(g->comm, "testprog") == 0) -//ust// printk(KERN_DEBUG "set update pending for testprog\n"); -//ust// t = g; -//ust// do { -//ust// /* TODO : implement this thread flag in each arch. */ -//ust// set_tsk_thread_flag(t, TIF_MARKER_PENDING); -//ust// } while ((t = next_thread(t)) != g); -//ust// } -//ust// read_unlock(&tasklist_lock); -//ust// mutex_unlock(&markers_mutex); -//ust// } - -/* - * Update current process. - * Note that we have to wait a whole scheduler period before we are sure that - * every running userspace threads have their markers updated. - * (synchronize_sched() can be used to insure this). - */ -void marker_update_process(void) +/* Called with markers mutex held. */ +void ust_marker_iter_next(struct ust_marker_iter *iter) { - struct user_marker *umark; - struct hlist_node *pos; - struct marker_entry *entry; - - mutex_lock(&markers_mutex); - mutex_lock(¤t->group_leader->user_markers_mutex); - if (strcmp(current->comm, "testprog") == 0) - printk(KERN_DEBUG "do update pending for testprog\n"); - hlist_for_each_entry(umark, pos, - ¤t->group_leader->user_markers, hlist) { - printk(KERN_DEBUG "Updating marker %s in %s\n", - umark->name, current->comm); - entry = get_marker("userspace", umark->name); - if (entry) { - if (entry->format && - strcmp(entry->format, umark->format) != 0) { - printk(KERN_WARNING - " error, wrong format in process %s\n", - current->comm); - break; - } - if (put_user(!!entry->refcount, umark->state)) { - printk(KERN_WARNING - "Marker in %s caused a fault\n", - current->comm); - break; - } - } else { - if (put_user(0, umark->state)) { - printk(KERN_WARNING - "Marker in %s caused a fault\n", - current->comm); - break; - } - } - } - clear_thread_flag(TIF_MARKER_PENDING); - mutex_unlock(¤t->group_leader->user_markers_mutex); - mutex_unlock(&markers_mutex); + iter->ust_marker++; + /* + * iter->ust_marker may be invalid because we blindly incremented it. + * Make sure it is valid by marshalling on the ust_marker, getting the + * ust_marker from following modules if necessary. + */ + ust_marker_get_iter(iter); } -/* - * Called at process exit and upon do_execve(). - * We assume that when the leader exits, no more references can be done to the - * leader structure by the other threads. - */ -void exit_user_markers(struct task_struct *p) +void ust_marker_iter_stop(struct ust_marker_iter *iter) { - struct user_marker *umark; - struct hlist_node *pos, *n; - - if (thread_group_leader(p)) { - mutex_lock(&markers_mutex); - mutex_lock(&p->user_markers_mutex); - hlist_for_each_entry_safe(umark, pos, n, &p->user_markers, - hlist) - kfree(umark); - INIT_HLIST_HEAD(&p->user_markers); - p->user_markers_sequence++; - mutex_unlock(&p->user_markers_mutex); - mutex_unlock(&markers_mutex); - } + unlock_ust_marker(); } -int is_marker_enabled(const char *channel, const char *name) +void ust_marker_iter_reset(struct ust_marker_iter *iter) { - struct marker_entry *entry; - - mutex_lock(&markers_mutex); - entry = get_marker(channel, name); - mutex_unlock(&markers_mutex); - - return entry && !!entry->refcount; + iter->lib = NULL; + iter->ust_marker = NULL; } -//ust// #endif -int marker_module_notify(struct notifier_block *self, - unsigned long val, void *data) +void ltt_dump_ust_marker_state(struct ust_trace *trace) { - struct module *mod = data; - - switch (val) { - case MODULE_STATE_COMING: - marker_update_probe_range(mod->markers, - mod->markers + mod->num_markers); - break; - case MODULE_STATE_GOING: - marker_update_probe_range(mod->markers, - mod->markers + mod->num_markers); - break; - } - return 0; -} - -struct notifier_block marker_module_nb = { - .notifier_call = marker_module_notify, - .priority = 0, -}; - -//ust// static int init_markers(void) -//ust// { -//ust// return register_module_notifier(&marker_module_nb); -//ust// } -//ust// __initcall(init_markers); -/* TODO: call marker_module_nb() when a library is linked at runtime (dlopen)? */ - -#endif /* CONFIG_MODULES */ - -void ltt_dump_marker_state(struct ltt_trace_struct *trace) -{ - struct marker_iter iter; + struct ust_marker_entry *entry; struct ltt_probe_private_data call_data; - const char *channel; + struct cds_hlist_head *head; + struct cds_hlist_node *node; + unsigned int i; + lock_ust_marker(); call_data.trace = trace; call_data.serializer = NULL; - marker_iter_reset(&iter); - marker_iter_start(&iter); - for (; iter.marker != NULL; marker_iter_next(&iter)) { - if (!_imv_read(iter.marker->state)) - continue; - channel = ltt_channels_get_name_from_index( - iter.marker->channel_id); - __trace_mark(0, metadata, core_marker_id, - &call_data, - "channel %s name %s event_id %hu " - "int #1u%zu long #1u%zu pointer #1u%zu " - "size_t #1u%zu alignment #1u%u", - channel, - iter.marker->name, - iter.marker->event_id, - sizeof(int), sizeof(long), - sizeof(void *), sizeof(size_t), - ltt_get_alignment()); - if (iter.marker->format) - __trace_mark(0, metadata, - core_marker_format, + for (i = 0; i < UST_MARKER_TABLE_SIZE; i++) { + head = &ust_marker_table[i]; + cds_hlist_for_each_entry(entry, node, head, hlist) { + __ust_marker(metadata, core_marker_id, &call_data, - "channel %s name %s format %s", - channel, - iter.marker->name, - iter.marker->format); - } - marker_iter_stop(&iter); -} -//ust// EXPORT_SYMBOL_GPL(ltt_dump_marker_state); - - -static LIST_HEAD(libs); - -/* - * Returns 0 if current not found. - * Returns 1 if current found. - */ -int lib_get_iter_markers(struct marker_iter *iter) -{ - struct lib *iter_lib; - int found = 0; - -//ust// mutex_lock(&module_mutex); - list_for_each_entry(iter_lib, &libs, list) { - if (iter_lib < iter->lib) - continue; - else if (iter_lib > iter->lib) - iter->marker = NULL; - found = marker_get_iter_range(&iter->marker, - iter_lib->markers_start, - iter_lib->markers_start + iter_lib->markers_count); - if (found) { - iter->lib = iter_lib; - break; + "channel %s name %s event_id %hu " + "int #1u%zu long #1u%zu pointer #1u%zu " + "size_t #1u%zu alignment #1u%u", + entry->channel, + entry->name, + entry->event_id, + sizeof(int), sizeof(long), + sizeof(void *), sizeof(size_t), + ltt_get_alignment()); + if (entry->format) + __ust_marker(metadata, + core_marker_format, + &call_data, + "channel %s name %s format %s", + entry->channel, + entry->name, + entry->format); } } -//ust// mutex_unlock(&module_mutex); - return found; + unlock_ust_marker(); } -void lib_update_markers(void) +void ust_marker_set_new_ust_marker_cb(void (*cb)(struct ust_marker *)) { - struct lib *lib; - -//ust// mutex_lock(&module_mutex); - list_for_each_entry(lib, &libs, list) - marker_update_probe_range(lib->markers_start, - lib->markers_start + lib->markers_count); -//ust// mutex_unlock(&module_mutex); + new_ust_marker_cb = cb; } -static void (*new_marker_cb)(struct marker *) = NULL; - -void marker_set_new_marker_cb(void (*cb)(struct marker *)) +static void new_ust_marker(struct ust_marker * const *start, + struct ust_marker * const *end) { - new_marker_cb = cb; -} + if (new_ust_marker_cb) { + struct ust_marker * const *m; -static void new_markers(struct marker *start, struct marker *end) -{ - if(new_marker_cb) { - struct marker *m; - for(m=start; m < end; m++) { - new_marker_cb(m); + for (m = start; m < end; m++) { + if (*m) + new_ust_marker_cb(*m); } } } -int marker_register_lib(struct marker *markers_start, int markers_count) +int ust_marker_register_lib(struct ust_marker * const *ust_marker_start, + int ust_marker_count) { - struct lib *pl; + struct ust_marker_lib *pl, *iter; - pl = (struct lib *) malloc(sizeof(struct lib)); + pl = (struct ust_marker_lib *) zmalloc(sizeof(struct ust_marker_lib)); - pl->markers_start = markers_start; - pl->markers_count = markers_count; + pl->ust_marker_start = ust_marker_start; + pl->ust_marker_count = ust_marker_count; - /* FIXME: maybe protect this with its own mutex? */ - lock_markers(); - list_add(&pl->list, &libs); - unlock_markers(); + lock_ust_marker(); + + /* + * We sort the libs by struct lib pointer address. + */ + cds_list_for_each_entry_reverse(iter, &ust_marker_libs, list) { + BUG_ON(iter == pl); /* Should never be in the list twice */ + if (iter < pl) { + /* We belong to the location right after iter. */ + cds_list_add(&pl->list, &iter->list); + goto lib_added; + } + } + /* We should be added at the head of the list */ + cds_list_add(&pl->list, &ust_marker_libs); +lib_added: + unlock_ust_marker(); - new_markers(markers_start, markers_start + markers_count); + new_ust_marker(ust_marker_start, ust_marker_start + ust_marker_count); - /* FIXME: update just the loaded lib */ - lib_update_markers(); + /* TODO: update just the loaded lib */ + lib_update_ust_marker(); - DBG("just registered a markers section from %p and having %d markers", markers_start, markers_count); + DBG("just registered a ust_marker section from %p and having %d ust_marker (minus dummy ust_marker)", ust_marker_start, ust_marker_count); return 0; } -int marker_unregister_lib(struct marker *markers_start, int markers_count) +int ust_marker_unregister_lib(struct ust_marker * const *ust_marker_start) { - /*FIXME: implement; but before implementing, marker_register_lib must - have appropriate locking. */ + struct ust_marker_lib *lib; + + lock_ust_marker(); + cds_list_for_each_entry(lib, &ust_marker_libs, list) { + if(lib->ust_marker_start == ust_marker_start) { + struct ust_marker_lib *lib2free = lib; + cds_list_del(&lib->list); + free(lib2free); + break; + } + } + unlock_ust_marker(); return 0; } -static int initialized = 0; - -void __attribute__((constructor)) init_markers(void) +void __attribute__((constructor)) init_ust_marker(void) { - if(!initialized) { - marker_register_lib(__start___markers, (((long)__stop___markers)-((long)__start___markers))/sizeof(struct marker)); - printf("markers_start: %p, markers_stop: %p\n", __start___markers, __stop___markers); + if (!initialized) { + init_tracepoint(); + ust_marker_register_lib(__start___ust_marker_ptrs, + __stop___ust_marker_ptrs + - __start___ust_marker_ptrs); initialized = 1; } } + +void __attribute__((destructor)) destroy_ust_marker(void) +{ + ust_marker_unregister_lib(__start___ust_marker_ptrs); +}