X-Git-Url: http://git.lttng.org/?a=blobdiff_plain;f=libust%2Fmarker.c;h=96d140996890c40c5b4a43da45a63a4180aad7d3;hb=8d6300d3b3cb0219e1109e931a2219dbd812b24d;hp=0c85cc3997f0d8669473c91055b7582ea4ad4f7c;hpb=1e4b909b682cc34aaa51ee13432c1c23f7e70bab;p=ust.git diff --git a/libust/marker.c b/libust/marker.c index 0c85cc3..96d1409 100644 --- a/libust/marker.c +++ b/libust/marker.c @@ -21,9 +21,11 @@ #define _LGPL_SOURCE #include #include +#include #include #include +#include #include "usterr.h" #include "channels.h" @@ -33,8 +35,8 @@ __thread long ust_reg_stack[500]; volatile __thread long *ust_reg_stack_ptr = (long *) 0; -extern struct marker __start___markers[] __attribute__((visibility("hidden"))); -extern struct marker __stop___markers[] __attribute__((visibility("hidden"))); +extern struct marker * const __start___markers_ptrs[] __attribute__((visibility("hidden"))); +extern struct marker * const __stop___markers_ptrs[] __attribute__((visibility("hidden"))); /* Set to 1 to enable marker debug output */ static const int marker_debug; @@ -45,7 +47,7 @@ static const int marker_debug; */ static DEFINE_MUTEX(markers_mutex); -static LIST_HEAD(libs); +static CDS_LIST_HEAD(libs); void lock_markers(void) @@ -64,7 +66,7 @@ void unlock_markers(void) */ #define MARKER_HASH_BITS 6 #define MARKER_TABLE_SIZE (1 << MARKER_HASH_BITS) -static struct hlist_head marker_table[MARKER_TABLE_SIZE]; +static struct cds_hlist_head marker_table[MARKER_TABLE_SIZE]; /* * Note about RCU : @@ -75,7 +77,7 @@ static struct hlist_head marker_table[MARKER_TABLE_SIZE]; * marker entries modifications are protected by the markers_mutex. */ struct marker_entry { - struct hlist_node hlist; + struct cds_hlist_node hlist; char *format; char *name; /* Probe wrapper */ @@ -127,7 +129,7 @@ notrace void __mark_empty_function(const struct marker *mdata, * @...: Variable argument list. * * Since we do not use "typical" pointer based RCU in the 1 argument case, we - * need to put a full smp_rmb() in this branch. This is why we do not use + * need to put a full cmm_smp_rmb() in this branch. This is why we do not use * rcu_dereference() for the pointer read. */ notrace void marker_probe_cb(const struct marker *mdata, @@ -146,12 +148,12 @@ notrace void marker_probe_cb(const struct marker *mdata, if (likely(!ptype)) { marker_probe_func *func; /* Must read the ptype before ptr. They are not data dependant, - * so we put an explicit smp_rmb() here. */ - smp_rmb(); + * so we put an explicit cmm_smp_rmb() here. */ + cmm_smp_rmb(); func = mdata->single.func; /* Must read the ptr before private data. They are not data - * dependant, so we put an explicit smp_rmb() here. */ - smp_rmb(); + * dependant, so we put an explicit cmm_smp_rmb() here. */ + cmm_smp_rmb(); va_start(args, regs); func(mdata, mdata->single.probe_private, regs, call_private, mdata->format, &args); @@ -162,16 +164,16 @@ notrace void marker_probe_cb(const struct marker *mdata, /* * Read mdata->ptype before mdata->multi. */ - smp_rmb(); + cmm_smp_rmb(); multi = mdata->multi; /* * multi points to an array, therefore accessing the array * depends on reading multi. However, even in this case, * we must insure that the pointer is read _before_ the array - * data. Same as rcu_dereference, but we need a full smp_rmb() - * in the fast path, so put the explicit barrier here. + * data. Same as rcu_dereference, but we need a full cmm_smp_rmb() + * in the fast path, so put the explicit cmm_barrier here. */ - smp_read_barrier_depends(); + cmm_smp_read_barrier_depends(); for (i = 0; multi[i].func; i++) { va_start(args, regs); multi[i].func(mdata, multi[i].probe_private, @@ -202,12 +204,12 @@ static notrace void marker_probe_cb_noarg(const struct marker *mdata, if (likely(!ptype)) { marker_probe_func *func; /* Must read the ptype before ptr. They are not data dependant, - * so we put an explicit smp_rmb() here. */ - smp_rmb(); + * so we put an explicit cmm_smp_rmb() here. */ + cmm_smp_rmb(); func = mdata->single.func; /* Must read the ptr before private data. They are not data - * dependant, so we put an explicit smp_rmb() here. */ - smp_rmb(); + * dependant, so we put an explicit cmm_smp_rmb() here. */ + cmm_smp_rmb(); func(mdata, mdata->single.probe_private, regs, call_private, mdata->format, &args); } else { @@ -216,16 +218,16 @@ static notrace void marker_probe_cb_noarg(const struct marker *mdata, /* * Read mdata->ptype before mdata->multi. */ - smp_rmb(); + cmm_smp_rmb(); multi = mdata->multi; /* * multi points to an array, therefore accessing the array * depends on reading multi. However, even in this case, * we must insure that the pointer is read _before_ the array - * data. Same as rcu_dereference, but we need a full smp_rmb() - * in the fast path, so put the explicit barrier here. + * data. Same as rcu_dereference, but we need a full cmm_smp_rmb() + * in the fast path, so put the explicit cmm_barrier here. */ - smp_read_barrier_depends(); + cmm_smp_read_barrier_depends(); for (i = 0; multi[i].func; i++) multi[i].func(mdata, multi[i].probe_private, regs, call_private, mdata->format, &args); @@ -239,7 +241,7 @@ static void free_old_closure(struct rcu_head *head) struct marker_entry, rcu); free(entry->oldptr); /* Make sure we free the data before setting the pending flag to 0 */ - smp_wmb(); + cmm_smp_wmb(); entry->rcu_pending = 0; } @@ -386,8 +388,8 @@ marker_entry_remove_probe(struct marker_entry *entry, */ static struct marker_entry *get_marker(const char *channel, const char *name) { - struct hlist_head *head; - struct hlist_node *node; + struct cds_hlist_head *head; + struct cds_hlist_node *node; struct marker_entry *e; size_t channel_len = strlen(channel) + 1; size_t name_len = strlen(name) + 1; @@ -395,7 +397,7 @@ static struct marker_entry *get_marker(const char *channel, const char *name) hash = jhash(channel, channel_len-1, 0) ^ jhash(name, name_len-1, 0); head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; - hlist_for_each_entry(e, node, head, hlist) { + cds_hlist_for_each_entry(e, node, head, hlist) { if (!strcmp(channel, e->channel) && !strcmp(name, e->name)) return e; } @@ -409,8 +411,8 @@ static struct marker_entry *get_marker(const char *channel, const char *name) static struct marker_entry *add_marker(const char *channel, const char *name, const char *format) { - struct hlist_head *head; - struct hlist_node *node; + struct cds_hlist_head *head; + struct cds_hlist_node *node; struct marker_entry *e; size_t channel_len = strlen(channel) + 1; size_t name_len = strlen(name) + 1; @@ -421,7 +423,7 @@ static struct marker_entry *add_marker(const char *channel, const char *name, if (format) format_len = strlen(format) + 1; head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; - hlist_for_each_entry(e, node, head, hlist) { + cds_hlist_for_each_entry(e, node, head, hlist) { if (!strcmp(channel, e->channel) && !strcmp(name, e->name)) { DBG("Marker %s.%s busy", channel, name); return ERR_PTR(-EBUSY); /* Already there */ @@ -459,7 +461,7 @@ static struct marker_entry *add_marker(const char *channel, const char *name, e->format_allocated = 0; e->refcount = 0; e->rcu_pending = 0; - hlist_add_head(&e->hlist, head); + cds_hlist_add_head(&e->hlist, head); return e; } @@ -469,8 +471,8 @@ static struct marker_entry *add_marker(const char *channel, const char *name, */ static int remove_marker(const char *channel, const char *name) { - struct hlist_head *head; - struct hlist_node *node; + struct cds_hlist_head *head; + struct cds_hlist_node *node; struct marker_entry *e; int found = 0; size_t channel_len = strlen(channel) + 1; @@ -480,7 +482,7 @@ static int remove_marker(const char *channel, const char *name) hash = jhash(channel, channel_len-1, 0) ^ jhash(name, name_len-1, 0); head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; - hlist_for_each_entry(e, node, head, hlist) { + cds_hlist_for_each_entry(e, node, head, hlist) { if (!strcmp(channel, e->channel) && !strcmp(name, e->name)) { found = 1; break; @@ -490,14 +492,14 @@ static int remove_marker(const char *channel, const char *name) return -ENOENT; if (e->single.func != __mark_empty_function) return -EBUSY; - hlist_del(&e->hlist); + cds_hlist_del(&e->hlist); if (e->format_allocated) free(e->format); ret = ltt_channels_unregister(e->channel); WARN_ON(ret); /* Make sure the call_rcu has been executed */ //ust// if (e->rcu_pending) -//ust// rcu_barrier_sched(); +//ust// rcu_cmm_barrier_sched(); free(e); return 0; } @@ -563,7 +565,7 @@ static int set_marker(struct marker_entry *entry, struct marker *elem, * Make sure the private data is valid when we update the * single probe ptr. */ - smp_wmb(); + cmm_smp_wmb(); elem->single.func = entry->single.func; /* * We also make sure that the new probe callbacks array is consistent @@ -574,7 +576,7 @@ static int set_marker(struct marker_entry *entry, struct marker *elem, * Update the function or multi probe array pointer before setting the * ptype. */ - smp_wmb(); + cmm_smp_wmb(); elem->ptype = entry->ptype; if (elem->tp_name && (active ^ _imv_read(elem->state))) { @@ -594,11 +596,11 @@ static int set_marker(struct marker_entry *entry, struct marker *elem, //ust// BUG_ON(!ret); ret = tracepoint_probe_register_noupdate( elem->tp_name, - elem->tp_cb); + elem->tp_cb, NULL); } else { ret = tracepoint_probe_unregister_noupdate( elem->tp_name, - elem->tp_cb); + elem->tp_cb, NULL); /* * tracepoint_probe_update_all() must be called * before the module containing tp_cb is unloaded. @@ -630,7 +632,7 @@ static void disable_marker(struct marker *elem) * checking has been done in the __trace_mark_tp() macro. */ ret = tracepoint_probe_unregister_noupdate(elem->tp_name, - elem->tp_cb); + elem->tp_cb, NULL); WARN_ON(ret); /* * tracepoint_probe_update_all() must be called @@ -641,7 +643,7 @@ static void disable_marker(struct marker *elem) elem->state__imv = 0; elem->single.func = __mark_empty_function; /* Update the function before setting the ptype */ - smp_wmb(); + cmm_smp_wmb(); elem->ptype = 0; /* single probe */ /* * Leave the private data and channel_id/event_id there, because removal @@ -675,36 +677,24 @@ int is_marker_enabled(const char *channel, const char *name) * * Updates the probe callback corresponding to a range of markers. */ -void marker_update_probe_range(struct marker *begin, - struct marker *end) +void marker_update_probe_range(struct marker * const *begin, + struct marker * const *end) { - struct marker *iter; + struct marker * const *iter; struct marker_entry *mark_entry; pthread_mutex_lock(&markers_mutex); for (iter = begin; iter < end; iter++) { - mark_entry = get_marker(iter->channel, iter->name); + if (!*iter) + continue; /* skip dummy */ + mark_entry = get_marker((*iter)->channel, (*iter)->name); if (mark_entry) { - set_marker(mark_entry, iter, !!mark_entry->refcount); + set_marker(mark_entry, *iter, !!mark_entry->refcount); /* * ignore error, continue */ - - /* This is added for UST. We emit a core_marker_id event - * for markers that are already registered to a probe - * upon library load. Otherwise, no core_marker_id will - * be generated for these markers. Is this the right thing - * to do? - */ - trace_mark(metadata, core_marker_id, - "channel %s name %s event_id %hu " - "int #1u%zu long #1u%zu pointer #1u%zu " - "size_t #1u%zu alignment #1u%u", - iter->channel, iter->name, mark_entry->event_id, - sizeof(int), sizeof(long), sizeof(void *), - sizeof(size_t), ltt_get_alignment()); } else { - disable_marker(iter); + disable_marker(*iter); } } pthread_mutex_unlock(&markers_mutex); @@ -716,7 +706,7 @@ static void lib_update_markers(void) /* FIXME: we should probably take a mutex here on libs */ //ust// pthread_mutex_lock(&module_mutex); - list_for_each_entry(lib, &libs, list) + cds_list_for_each_entry(lib, &libs, list) marker_update_probe_range(lib->markers_start, lib->markers_start + lib->markers_count); //ust// pthread_mutex_unlock(&module_mutex); @@ -741,10 +731,6 @@ static void lib_update_markers(void) */ static void marker_update_probes(void) { - /* Core kernel markers */ -//ust// marker_update_probe_range(__start___markers, __stop___markers); - /* Markers in modules. */ -//ust// module_update_markers(); lib_update_markers(); tracepoint_probe_update_all(); /* Update immediate values */ @@ -816,7 +802,7 @@ int marker_probe_register(const char *channel, const char *name, * make sure it's executed now. */ //ust// if (entry->rcu_pending) -//ust// rcu_barrier_sched(); +//ust// rcu_cmm_barrier_sched(); old = marker_entry_add_probe(entry, probe, probe_private); if (IS_ERR(old)) { ret = PTR_ERR(old); @@ -835,11 +821,11 @@ int marker_probe_register(const char *channel, const char *name, if (!entry) goto end; //ust// if (entry->rcu_pending) -//ust// rcu_barrier_sched(); +//ust// rcu_cmm_barrier_sched(); entry->oldptr = old; entry->rcu_pending = 1; /* write rcu_pending before calling the RCU callback */ - smp_wmb(); + cmm_smp_wmb(); //ust// call_rcu_sched(&entry->rcu, free_old_closure); synchronize_rcu(); free_old_closure(&entry->rcu); goto end; @@ -881,7 +867,7 @@ int marker_probe_unregister(const char *channel, const char *name, if (!entry) goto end; //ust// if (entry->rcu_pending) -//ust// rcu_barrier_sched(); +//ust// rcu_cmm_barrier_sched(); old = marker_entry_remove_probe(entry, probe, probe_private); pthread_mutex_unlock(&markers_mutex); @@ -892,11 +878,11 @@ int marker_probe_unregister(const char *channel, const char *name, if (!entry) goto end; //ust// if (entry->rcu_pending) -//ust// rcu_barrier_sched(); +//ust// rcu_cmm_barrier_sched(); entry->oldptr = old; entry->rcu_pending = 1; /* write rcu_pending before calling the RCU callback */ - smp_wmb(); + cmm_smp_wmb(); //ust// call_rcu_sched(&entry->rcu, free_old_closure); synchronize_rcu(); free_old_closure(&entry->rcu); remove_marker(channel, name); /* Ignore busy error message */ @@ -912,12 +898,12 @@ get_marker_from_private_data(marker_probe_func *probe, void *probe_private) { struct marker_entry *entry; unsigned int i; - struct hlist_head *head; - struct hlist_node *node; + struct cds_hlist_head *head; + struct cds_hlist_node *node; for (i = 0; i < MARKER_TABLE_SIZE; i++) { head = &marker_table[i]; - hlist_for_each_entry(entry, node, head, hlist) { + cds_hlist_for_each_entry(entry, node, head, hlist) { if (!entry->ptype) { if (entry->single.func == probe && entry->single.probe_private @@ -966,7 +952,7 @@ int marker_probe_unregister_private_data(marker_probe_func *probe, goto end; } //ust// if (entry->rcu_pending) -//ust// rcu_barrier_sched(); +//ust// rcu_cmm_barrier_sched(); old = marker_entry_remove_probe(entry, NULL, probe_private); channel = strdup(entry->channel); name = strdup(entry->name); @@ -979,11 +965,11 @@ int marker_probe_unregister_private_data(marker_probe_func *probe, if (!entry) goto end; //ust// if (entry->rcu_pending) -//ust// rcu_barrier_sched(); +//ust// rcu_cmm_barrier_sched(); entry->oldptr = old; entry->rcu_pending = 1; /* write rcu_pending before calling the RCU callback */ - smp_wmb(); + cmm_smp_wmb(); //ust// call_rcu_sched(&entry->rcu, free_old_closure); synchronize_rcu(); free_old_closure(&entry->rcu); /* Ignore busy error message */ @@ -1013,8 +999,8 @@ end: void *marker_get_private_data(const char *channel, const char *name, marker_probe_func *probe, int num) { - struct hlist_head *head; - struct hlist_node *node; + struct cds_hlist_head *head; + struct cds_hlist_node *node; struct marker_entry *e; size_t channel_len = strlen(channel) + 1; size_t name_len = strlen(name) + 1; @@ -1023,7 +1009,7 @@ void *marker_get_private_data(const char *channel, const char *name, hash = jhash(channel, channel_len-1, 0) ^ jhash(name, name_len-1, 0); head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; - hlist_for_each_entry(e, node, head, hlist) { + cds_hlist_for_each_entry(e, node, head, hlist) { if (!strcmp(channel, e->channel) && !strcmp(name, e->name)) { if (!e->ptype) { if (num == 0 && e->single.func == probe) @@ -1086,7 +1072,7 @@ int lib_get_iter_markers(struct marker_iter *iter) int found = 0; //ust// pthread_mutex_lock(&module_mutex); - list_for_each_entry(iter_lib, &libs, list) { + cds_list_for_each_entry(iter_lib, &libs, list) { if (iter_lib < iter->lib) continue; else if (iter_lib > iter->lib) @@ -1112,15 +1098,18 @@ int lib_get_iter_markers(struct marker_iter *iter) * Returns whether a next marker has been found (1) or not (0). * Will return the first marker in the range if the input marker is NULL. */ -int marker_get_iter_range(struct marker **marker, struct marker *begin, - struct marker *end) +int marker_get_iter_range(struct marker * const **marker, + struct marker * const *begin, + struct marker * const *end) { - if (!*marker && begin != end) { + if (!*marker && begin != end) *marker = begin; - return 1; + while (*marker >= begin && *marker < end) { + if (!**marker) + (*marker)++; /* skip dummy */ + else + return 1; } - if (*marker >= begin && *marker < end) - return 1; return 0; } //ust// EXPORT_SYMBOL_GPL(marker_get_iter_range); @@ -1129,17 +1118,7 @@ static void marker_get_iter(struct marker_iter *iter) { int found = 0; - /* Core kernel markers */ - if (!iter->lib) { - /* ust FIXME: how come we cannot disable the following line? we shouldn't need core stuff */ - found = marker_get_iter_range(&iter->marker, - __start___markers, __stop___markers); - if (found) - goto end; - } - /* Markers in modules. */ found = lib_get_iter_markers(iter); -end: if (!found) marker_iter_reset(iter); } @@ -1178,14 +1157,14 @@ void marker_iter_reset(struct marker_iter *iter) /* * must be called with current->user_markers_mutex held */ -static void free_user_marker(char __user *state, struct hlist_head *head) +static void free_user_marker(char __user *state, struct cds_hlist_head *head) { struct user_marker *umark; - struct hlist_node *pos, *n; + struct cds_hlist_node *pos, *n; - hlist_for_each_entry_safe(umark, pos, n, head, hlist) { + cds_hlist_for_each_entry_safe(umark, pos, n, head, hlist) { if (umark->state == state) { - hlist_del(&umark->hlist); + cds_hlist_del(&umark->hlist); free(umark); } } @@ -1243,12 +1222,12 @@ static void free_user_marker(char __user *state, struct hlist_head *head) void exit_user_markers(struct task_struct *p) { struct user_marker *umark; - struct hlist_node *pos, *n; + struct cds_hlist_node *pos, *n; if (thread_group_leader(p)) { pthread_mutex_lock(&markers_mutex); pthread_mutex_lock(&p->user_markers_mutex); - hlist_for_each_entry_safe(umark, pos, n, &p->user_markers, + cds_hlist_for_each_entry_safe(umark, pos, n, &p->user_markers, hlist) free(umark); INIT_HLIST_HEAD(&p->user_markers); @@ -1306,8 +1285,8 @@ void ltt_dump_marker_state(struct ust_trace *trace) { struct marker_entry *entry; struct ltt_probe_private_data call_data; - struct hlist_head *head; - struct hlist_node *node; + struct cds_hlist_head *head; + struct cds_hlist_node *node; unsigned int i; pthread_mutex_lock(&markers_mutex); @@ -1316,7 +1295,7 @@ void ltt_dump_marker_state(struct ust_trace *trace) for (i = 0; i < MARKER_TABLE_SIZE; i++) { head = &marker_table[i]; - hlist_for_each_entry(entry, node, head, hlist) { + cds_hlist_for_each_entry(entry, node, head, hlist) { __trace_mark(0, metadata, core_marker_id, &call_data, "channel %s name %s event_id %hu " @@ -1349,19 +1328,21 @@ void marker_set_new_marker_cb(void (*cb)(struct marker *)) new_marker_cb = cb; } -static void new_markers(struct marker *start, struct marker *end) +static void new_markers(struct marker * const *start, struct marker * const *end) { - if(new_marker_cb) { - struct marker *m; - for(m=start; m < end; m++) { - new_marker_cb(m); + if (new_marker_cb) { + struct marker * const *m; + + for(m = start; m < end; m++) { + if (*m) + new_marker_cb(*m); } } } -int marker_register_lib(struct marker *markers_start, int markers_count) +int marker_register_lib(struct marker * const *markers_start, int markers_count) { - struct lib *pl; + struct lib *pl, *iter; pl = (struct lib *) zmalloc(sizeof(struct lib)); @@ -1370,7 +1351,21 @@ int marker_register_lib(struct marker *markers_start, int markers_count) /* FIXME: maybe protect this with its own mutex? */ lock_markers(); - list_add(&pl->list, &libs); + + /* + * We sort the libs by struct lib pointer address. + */ + cds_list_for_each_entry_reverse(iter, &libs, list) { + BUG_ON(iter == pl); /* Should never be in the list twice */ + if (iter < pl) { + /* We belong to the location right after iter. */ + cds_list_add(&pl->list, &iter->list); + goto lib_added; + } + } + /* We should be added at the head of the list */ + cds_list_add(&pl->list, &libs); +lib_added: unlock_markers(); new_markers(markers_start, markers_start + markers_count); @@ -1378,12 +1373,12 @@ int marker_register_lib(struct marker *markers_start, int markers_count) /* FIXME: update just the loaded lib */ lib_update_markers(); - DBG("just registered a markers section from %p and having %d markers", markers_start, markers_count); + DBG("just registered a markers section from %p and having %d markers (minus dummy markers)", markers_start, markers_count); return 0; } -int marker_unregister_lib(struct marker *markers_start) +int marker_unregister_lib(struct marker * const *markers_start) { struct lib *lib; @@ -1394,10 +1389,10 @@ int marker_unregister_lib(struct marker *markers_start) /* FIXME: we should probably take a mutex here on libs */ //ust// pthread_mutex_lock(&module_mutex); - list_for_each_entry(lib, &libs, list) { + cds_list_for_each_entry(lib, &libs, list) { if(lib->markers_start == markers_start) { struct lib *lib2free = lib; - list_del(&lib->list); + cds_list_del(&lib->list); free(lib2free); break; } @@ -1412,13 +1407,15 @@ static int initialized = 0; void __attribute__((constructor)) init_markers(void) { - if(!initialized) { - marker_register_lib(__start___markers, (((long)__stop___markers)-((long)__start___markers))/sizeof(struct marker)); + if (!initialized) { + marker_register_lib(__start___markers_ptrs, + __stop___markers_ptrs + - __start___markers_ptrs); initialized = 1; } } -void __attribute__((constructor)) destroy_markers(void) +void __attribute__((destructor)) destroy_markers(void) { - marker_unregister_lib(__start___markers); + marker_unregister_lib(__start___markers_ptrs); }