2 * Copyright (C) 2007-2011 Mathieu Desnoyers
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation;
7 * version 2.1 of the License.
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
14 * You should have received a copy of the GNU Lesser General Public
15 * License along with this library; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23 #include <urcu/rculist.h>
24 #include <urcu/hlist.h>
27 #include <ust/marker.h>
28 #include <ust/marker-internal.h>
29 #include <ust/tracepoint.h>
30 #include <ust/tracepoint-internal.h>
32 #include "usterr_signal_safe.h"
34 #include "tracercore.h"
37 extern struct ust_marker
* const __start___ust_marker_ptrs
[] __attribute__((visibility("hidden")));
38 extern struct ust_marker
* const __stop___ust_marker_ptrs
[] __attribute__((visibility("hidden")));
40 /* Set to 1 to enable ust_marker debug output */
41 static const int ust_marker_debug
;
44 * ust_marker mutex protects the builtin and module ust_marker and the
45 * hash table, as well as the ust_marker_libs list.
47 static DEFINE_MUTEX(ust_marker_mutex
);
48 static CDS_LIST_HEAD(ust_marker_libs
);
50 void lock_ust_marker(void)
52 pthread_mutex_lock(&ust_marker_mutex
);
55 void unlock_ust_marker(void)
57 pthread_mutex_unlock(&ust_marker_mutex
);
61 * ust_marker hash table, containing the active ust_marker.
62 * Protected by ust_marker mutex.
64 #define UST_MARKER_HASH_BITS 6
65 #define UST_MARKER_TABLE_SIZE (1 << UST_MARKER_HASH_BITS)
66 static struct cds_hlist_head ust_marker_table
[UST_MARKER_TABLE_SIZE
];
68 struct ust_marker_probe_array
{
70 struct ust_marker_probe_closure c
[0];
75 * It is used to make sure every handler has finished using its private
76 * data between two consecutive operation (add or remove) on a given
77 * ust_marker. It is also used to delay the free of multiple probes
78 * array until a quiescent state is reached. ust_marker entries
79 * modifications are protected by the ust_marker_mutex.
81 struct ust_marker_entry
{
82 struct cds_hlist_node hlist
;
86 void (*call
)(const struct ust_marker
*mdata
, void *call_private
, ...);
87 struct ust_marker_probe_closure single
;
88 struct ust_marker_probe_array
*multi
;
89 int refcount
; /* Number of times armed. 0 if disarmed. */
92 unsigned char ptype
:1;
93 unsigned char format_allocated
:1;
94 char channel
[0]; /* Contains channel'\0'name'\0'format'\0' */
98 * __ust_marker_empty_function - Empty probe callback
99 * @mdata: ust_marker data
100 * @probe_private: probe private data
101 * @call_private: call site private data
102 * @fmt: format string
103 * @...: variable argument list
105 * Empty callback provided as a probe to the ust_marker. By providing
106 * this to a disabled ust_marker, we make sure the execution flow is
107 * always valid even though the function pointer change and the
108 * ust_marker enabling are two distinct operations that modifies the
109 * execution flow of preemptible code.
111 notrace
void __ust_marker_empty_function(const struct ust_marker
*mdata
,
112 void *probe_private
, void *call_private
, const char *fmt
, va_list *args
)
117 * ust_marker_probe_cb Callback that prepares the variable argument list for probes.
118 * @mdata: pointer of type struct ust_marker
119 * @call_private: caller site private data
120 * @...: Variable argument list.
122 * Since we do not use "typical" pointer based RCU in the 1 argument case, we
123 * need to put a full cmm_smp_rmb() in this branch. This is why we do not use
124 * rcu_dereference() for the pointer read.
126 notrace
void ust_marker_probe_cb(const struct ust_marker
*mdata
,
127 void *call_private
, ...)
133 * rcu_read_lock_sched does two things : disabling preemption to make
134 * sure the teardown of the callbacks can be done correctly when they
135 * are in modules and they insure RCU read coherency.
138 ptype
= mdata
->ptype
;
139 if (likely(!ptype
)) {
140 ust_marker_probe_func
*func
;
141 /* Must read the ptype before ptr. They are not data dependant,
142 * so we put an explicit cmm_smp_rmb() here. */
144 func
= mdata
->single
.func
;
145 /* Must read the ptr before private data. They are not data
146 * dependant, so we put an explicit cmm_smp_rmb() here. */
148 va_start(args
, call_private
);
149 func(mdata
, mdata
->single
.probe_private
, call_private
,
150 mdata
->format
, &args
);
153 struct ust_marker_probe_array
*multi
;
156 * Read mdata->ptype before mdata->multi.
159 multi
= mdata
->multi
;
161 * multi points to an array, therefore accessing the array
162 * depends on reading multi. However, even in this case,
163 * we must insure that the pointer is read _before_ the array
164 * data. Same as rcu_dereference, but we need a full cmm_smp_rmb()
165 * in the fast path, so put the explicit cmm_barrier here.
167 cmm_smp_read_barrier_depends();
168 for (i
= 0; multi
->c
[i
].func
; i
++) {
169 va_start(args
, call_private
);
170 multi
->c
[i
].func(mdata
, multi
->c
[i
].probe_private
,
171 call_private
, mdata
->format
, &args
);
179 * ust_marker_probe_cb Callback that does not prepare the variable argument list.
180 * @mdata: pointer of type struct ust_marker
181 * @call_private: caller site private data
182 * @...: Variable argument list.
184 * Should be connected to ust_marker "UST_MARKER_NOARGS".
186 static notrace
void ust_marker_probe_cb_noarg(const struct ust_marker
*mdata
,
187 void *call_private
, ...)
189 va_list args
; /* not initialized */
193 ptype
= mdata
->ptype
;
194 if (likely(!ptype
)) {
195 ust_marker_probe_func
*func
;
196 /* Must read the ptype before ptr. They are not data dependant,
197 * so we put an explicit cmm_smp_rmb() here. */
199 func
= mdata
->single
.func
;
200 /* Must read the ptr before private data. They are not data
201 * dependant, so we put an explicit cmm_smp_rmb() here. */
203 func(mdata
, mdata
->single
.probe_private
, call_private
,
204 mdata
->format
, &args
);
206 struct ust_marker_probe_array
*multi
;
209 * Read mdata->ptype before mdata->multi.
212 multi
= mdata
->multi
;
214 * multi points to an array, therefore accessing the array
215 * depends on reading multi. However, even in this case,
216 * we must insure that the pointer is read _before_ the array
217 * data. Same as rcu_dereference, but we need a full cmm_smp_rmb()
218 * in the fast path, so put the explicit cmm_barrier here.
220 cmm_smp_read_barrier_depends();
221 for (i
= 0; multi
->c
[i
].func
; i
++)
222 multi
->c
[i
].func(mdata
, multi
->c
[i
].probe_private
,
223 call_private
, mdata
->format
, &args
);
228 static void free_old_closure(struct rcu_head
*head
)
230 struct ust_marker_probe_array
*multi
=
231 _ust_container_of(head
, struct ust_marker_probe_array
, rcu
);
235 static void debug_print_probes(struct ust_marker_entry
*entry
)
239 if (!ust_marker_debug
)
243 DBG("Single probe : %p %p",
245 entry
->single
.probe_private
);
247 for (i
= 0; entry
->multi
->c
[i
].func
; i
++)
248 DBG("Multi probe %d : %p %p", i
,
249 entry
->multi
->c
[i
].func
,
250 entry
->multi
->c
[i
].probe_private
);
254 static struct ust_marker_probe_array
*
255 ust_marker_entry_add_probe(struct ust_marker_entry
*entry
,
256 ust_marker_probe_func
*probe
, void *probe_private
)
259 struct ust_marker_probe_array
*old
, *new;
263 debug_print_probes(entry
);
266 if (entry
->single
.func
== probe
&&
267 entry
->single
.probe_private
== probe_private
)
268 return ERR_PTR(-EBUSY
);
269 if (entry
->single
.func
== __ust_marker_empty_function
) {
271 entry
->single
.func
= probe
;
272 entry
->single
.probe_private
= probe_private
;
275 debug_print_probes(entry
);
283 /* (N -> N+1), (N != 0, 1) probes */
284 for (nr_probes
= 0; old
->c
[nr_probes
].func
; nr_probes
++)
285 if (old
->c
[nr_probes
].func
== probe
286 && old
->c
[nr_probes
].probe_private
288 return ERR_PTR(-EBUSY
);
290 /* + 2 : one for new probe, one for NULL func */
291 new = zmalloc(sizeof(struct ust_marker_probe_array
)
292 + ((nr_probes
+ 2) * sizeof(struct ust_marker_probe_closure
)));
294 return ERR_PTR(-ENOMEM
);
296 new->c
[0] = entry
->single
;
298 memcpy(&new->c
[0], &old
->c
[0],
299 nr_probes
* sizeof(struct ust_marker_probe_closure
));
300 new->c
[nr_probes
].func
= probe
;
301 new->c
[nr_probes
].probe_private
= probe_private
;
302 entry
->refcount
= nr_probes
+ 1;
305 debug_print_probes(entry
);
309 static struct ust_marker_probe_array
*
310 ust_marker_entry_remove_probe(struct ust_marker_entry
*entry
,
311 ust_marker_probe_func
*probe
, void *probe_private
)
313 int nr_probes
= 0, nr_del
= 0, i
;
314 struct ust_marker_probe_array
*old
, *new;
318 debug_print_probes(entry
);
320 /* 0 -> N is an error */
321 WARN_ON(entry
->single
.func
== __ust_marker_empty_function
);
323 WARN_ON(probe
&& entry
->single
.func
!= probe
);
324 WARN_ON(entry
->single
.probe_private
!= probe_private
);
325 entry
->single
.func
= __ust_marker_empty_function
;
328 debug_print_probes(entry
);
331 /* (N -> M), (N > 1, M >= 0) probes */
332 for (nr_probes
= 0; old
->c
[nr_probes
].func
; nr_probes
++) {
333 if ((!probe
|| old
->c
[nr_probes
].func
== probe
)
334 && old
->c
[nr_probes
].probe_private
340 if (nr_probes
- nr_del
== 0) {
341 /* N -> 0, (N > 1) */
342 entry
->single
.func
= __ust_marker_empty_function
;
345 } else if (nr_probes
- nr_del
== 1) {
346 /* N -> 1, (N > 1) */
347 for (i
= 0; old
->c
[i
].func
; i
++)
348 if ((probe
&& old
->c
[i
].func
!= probe
) ||
349 old
->c
[i
].probe_private
!= probe_private
)
350 entry
->single
= old
->c
[i
];
355 /* N -> M, (N > 1, M > 1) */
357 new = zmalloc(sizeof(struct ust_marker_probe_array
)
358 + ((nr_probes
- nr_del
+ 1) * sizeof(struct ust_marker_probe_closure
)));
360 return ERR_PTR(-ENOMEM
);
361 for (i
= 0; old
->c
[i
].func
; i
++)
362 if ((probe
&& old
->c
[i
].func
!= probe
) ||
363 old
->c
[i
].probe_private
!= probe_private
)
364 new->c
[j
++] = old
->c
[i
];
365 entry
->refcount
= nr_probes
- nr_del
;
369 debug_print_probes(entry
);
374 * Get ust_marker if the ust_marker is present in the ust_marker hash table.
375 * Must be called with ust_marker_mutex held.
376 * Returns NULL if not present.
378 static struct ust_marker_entry
*get_ust_marker(const char *channel
, const char *name
)
380 struct cds_hlist_head
*head
;
381 struct cds_hlist_node
*node
;
382 struct ust_marker_entry
*e
;
383 size_t channel_len
= strlen(channel
) + 1;
384 size_t name_len
= strlen(name
) + 1;
387 hash
= jhash(channel
, channel_len
-1, 0) ^ jhash(name
, name_len
-1, 0);
388 head
= &ust_marker_table
[hash
& ((1 << UST_MARKER_HASH_BITS
)-1)];
389 cds_hlist_for_each_entry(e
, node
, head
, hlist
) {
390 if (!strcmp(channel
, e
->channel
) && !strcmp(name
, e
->name
))
397 * Add the ust_marker to the ust_marker hash table. Must be called with
398 * ust_marker_mutex held.
400 static struct ust_marker_entry
*add_ust_marker(const char *channel
, const char *name
,
403 struct cds_hlist_head
*head
;
404 struct cds_hlist_node
*node
;
405 struct ust_marker_entry
*e
;
406 size_t channel_len
= strlen(channel
) + 1;
407 size_t name_len
= strlen(name
) + 1;
408 size_t format_len
= 0;
411 hash
= jhash(channel
, channel_len
-1, 0) ^ jhash(name
, name_len
-1, 0);
413 format_len
= strlen(format
) + 1;
414 head
= &ust_marker_table
[hash
& ((1 << UST_MARKER_HASH_BITS
)-1)];
415 cds_hlist_for_each_entry(e
, node
, head
, hlist
) {
416 if (!strcmp(channel
, e
->channel
) && !strcmp(name
, e
->name
)) {
417 DBG("ust_marker %s.%s busy", channel
, name
);
418 return ERR_PTR(-EBUSY
); /* Already there */
422 * Using zmalloc here to allocate a variable length element. Could
423 * cause some memory fragmentation if overused.
425 e
= zmalloc(sizeof(struct ust_marker_entry
)
426 + channel_len
+ name_len
+ format_len
);
428 return ERR_PTR(-ENOMEM
);
429 memcpy(e
->channel
, channel
, channel_len
);
430 e
->name
= &e
->channel
[channel_len
];
431 memcpy(e
->name
, name
, name_len
);
433 e
->format
= &e
->name
[name_len
];
434 memcpy(e
->format
, format
, format_len
);
435 if (strcmp(e
->format
, UST_MARKER_NOARGS
) == 0)
436 e
->call
= ust_marker_probe_cb_noarg
;
438 e
->call
= ust_marker_probe_cb
;
439 __ust_marker(metadata
, core_marker_format
, NULL
,
440 "channel %s name %s format %s",
441 e
->channel
, e
->name
, e
->format
);
444 e
->call
= ust_marker_probe_cb
;
446 e
->single
.func
= __ust_marker_empty_function
;
447 e
->single
.probe_private
= NULL
;
450 e
->format_allocated
= 0;
452 cds_hlist_add_head(&e
->hlist
, head
);
457 * Remove the ust_marker from the ust_marker hash table. Must be called with mutex_lock
460 static int remove_ust_marker(const char *channel
, const char *name
)
462 struct cds_hlist_head
*head
;
463 struct cds_hlist_node
*node
;
464 struct ust_marker_entry
*e
;
466 size_t channel_len
= strlen(channel
) + 1;
467 size_t name_len
= strlen(name
) + 1;
471 hash
= jhash(channel
, channel_len
-1, 0) ^ jhash(name
, name_len
-1, 0);
472 head
= &ust_marker_table
[hash
& ((1 << UST_MARKER_HASH_BITS
)-1)];
473 cds_hlist_for_each_entry(e
, node
, head
, hlist
) {
474 if (!strcmp(channel
, e
->channel
) && !strcmp(name
, e
->name
)) {
481 if (e
->single
.func
!= __ust_marker_empty_function
)
483 cds_hlist_del(&e
->hlist
);
484 if (e
->format_allocated
)
486 ret
= ltt_channels_unregister(e
->channel
);
493 * Set the mark_entry format to the format found in the element.
495 static int ust_marker_set_format(struct ust_marker_entry
*entry
, const char *format
)
497 entry
->format
= strdup(format
);
500 entry
->format_allocated
= 1;
502 __ust_marker(metadata
, core_marker_format
, NULL
,
503 "channel %s name %s format %s",
504 entry
->channel
, entry
->name
, entry
->format
);
509 * Sets the probe callback corresponding to one ust_marker.
511 static int set_ust_marker(struct ust_marker_entry
*entry
, struct ust_marker
*elem
,
515 WARN_ON(strcmp(entry
->name
, elem
->name
) != 0);
518 if (strcmp(entry
->format
, elem
->format
) != 0) {
519 ERR("Format mismatch for probe %s (%s), ust_marker (%s)",
526 ret
= ust_marker_set_format(entry
, elem
->format
);
532 * probe_cb setup (statically known) is done here. It is
533 * asynchronous with the rest of execution, therefore we only
534 * pass from a "safe" callback (with argument) to an "unsafe"
535 * callback (does not set arguments).
537 elem
->call
= entry
->call
;
538 elem
->channel_id
= entry
->channel_id
;
539 elem
->event_id
= entry
->event_id
;
542 * We only update the single probe private data when the ptr is
543 * set to a _non_ single probe! (0 -> 1 and N -> 1, N != 1)
545 WARN_ON(elem
->single
.func
!= __ust_marker_empty_function
546 && elem
->single
.probe_private
!= entry
->single
.probe_private
548 elem
->single
.probe_private
= entry
->single
.probe_private
;
550 * Make sure the private data is valid when we update the
554 elem
->single
.func
= entry
->single
.func
;
556 * We also make sure that the new probe callbacks array is consistent
557 * before setting a pointer to it.
559 rcu_assign_pointer(elem
->multi
, entry
->multi
);
561 * Update the function or multi probe array pointer before setting the
565 elem
->ptype
= entry
->ptype
;
567 if (elem
->tp_name
&& (active
^ elem
->state
)) {
568 WARN_ON(!elem
->tp_cb
);
570 * It is ok to directly call the probe registration because type
571 * checking has been done in the __ust_marker_tp() macro.
575 ret
= tracepoint_probe_register_noupdate(
580 * tracepoint_probe_update_all() must be called
581 * before the library containing tp_cb is unloaded.
583 ret
= tracepoint_probe_unregister_noupdate(
588 elem
->state
= active
;
594 * Disable a ust_marker and its probe callback.
595 * Note: only waiting an RCU period after setting elem->call to the empty
596 * function insures that the original callback is not used anymore. This insured
597 * by rcu_read_lock around the call site.
599 static void disable_ust_marker(struct ust_marker
*elem
)
603 /* leave "call" as is. It is known statically. */
604 if (elem
->tp_name
&& elem
->state
) {
605 WARN_ON(!elem
->tp_cb
);
607 * It is ok to directly call the probe registration because type
608 * checking has been done in the __ust_marker_tp() macro.
611 * tracepoint_probe_update_all() must be called
612 * before the module containing tp_cb is unloaded.
614 ret
= tracepoint_probe_unregister_noupdate(elem
->tp_name
,
619 elem
->single
.func
= __ust_marker_empty_function
;
620 /* Update the function before setting the ptype */
622 elem
->ptype
= 0; /* single probe */
624 * Leave the private data and channel_id/event_id there, because removal
625 * is racy and should be done only after an RCU period. These are never
626 * used until the next initialization anyway.
631 * is_ust_marker_enabled - Check if a ust_marker is enabled
632 * @channel: channel name
633 * @name: ust_marker name
635 * Returns 1 if the ust_marker is enabled, 0 if disabled.
637 int is_ust_marker_enabled(const char *channel
, const char *name
)
639 struct ust_marker_entry
*entry
;
641 pthread_mutex_lock(&ust_marker_mutex
);
642 entry
= get_ust_marker(channel
, name
);
643 pthread_mutex_unlock(&ust_marker_mutex
);
645 return entry
&& !!entry
->refcount
;
649 * ust_marker_update_probe_range - Update a probe range
650 * @begin: beginning of the range
651 * @end: end of the range
653 * Updates the probe callback corresponding to a range of ust_marker.
656 void ust_marker_update_probe_range(struct ust_marker
* const *begin
,
657 struct ust_marker
* const *end
)
659 struct ust_marker
* const *iter
;
660 struct ust_marker_entry
*mark_entry
;
662 for (iter
= begin
; iter
< end
; iter
++) {
664 continue; /* skip dummy */
665 mark_entry
= get_ust_marker((*iter
)->channel
, (*iter
)->name
);
667 set_ust_marker(mark_entry
, *iter
, !!mark_entry
->refcount
);
669 * ignore error, continue
672 disable_ust_marker(*iter
);
677 static void lib_update_ust_marker(void)
679 struct ust_marker_lib
*lib
;
681 pthread_mutex_lock(&ust_marker_mutex
);
682 cds_list_for_each_entry(lib
, &ust_marker_libs
, list
)
683 ust_marker_update_probe_range(lib
->ust_marker_start
,
684 lib
->ust_marker_start
+ lib
->ust_marker_count
);
685 pthread_mutex_unlock(&ust_marker_mutex
);
689 * Update probes, removing the faulty probes.
691 * Internal callback only changed before the first probe is connected to it.
692 * Single probe private data can only be changed on 0 -> 1 and 2 -> 1
693 * transitions. All other transitions will leave the old private data valid.
694 * This makes the non-atomicity of the callback/private data updates valid.
696 * "special case" updates :
701 * Other updates all behave the same, just like the 2 -> 3 or 3 -> 2 updates.
702 * Site effect : ust_marker_set_format may delete the ust_marker entry (creating a
705 static void ust_marker_update_probes(void)
707 lib_update_ust_marker();
708 tracepoint_probe_update_all();
712 * ust_marker_probe_register - Connect a probe to a ust_marker
713 * @channel: ust_marker channel
714 * @name: ust_marker name
715 * @format: format string
716 * @probe: probe handler
717 * @probe_private: probe private data
719 * private data must be a valid allocated memory address, or NULL.
720 * Returns 0 if ok, error value on error.
721 * The probe address must at least be aligned on the architecture pointer size.
723 int ust_marker_probe_register(const char *channel
, const char *name
,
724 const char *format
, ust_marker_probe_func
*probe
,
727 struct ust_marker_entry
*entry
;
728 int ret
= 0, ret_err
;
729 struct ust_marker_probe_array
*old
;
732 pthread_mutex_lock(&ust_marker_mutex
);
733 entry
= get_ust_marker(channel
, name
);
736 entry
= add_ust_marker(channel
, name
, format
);
738 ret
= PTR_ERR(entry
);
741 ret
= ltt_channels_register(channel
);
743 goto error_remove_ust_marker
;
744 ret
= ltt_channels_get_index_from_name(channel
);
746 goto error_unregister_channel
;
747 entry
->channel_id
= ret
;
748 ret
= ltt_channels_get_event_id(channel
, name
);
750 goto error_unregister_channel
;
751 entry
->event_id
= ret
;
753 __ust_marker(metadata
, core_marker_id
, NULL
,
754 "channel %s name %s event_id %hu "
755 "int #1u%zu long #1u%zu pointer #1u%zu "
756 "size_t #1u%zu alignment #1u%u",
757 channel
, name
, entry
->event_id
,
758 sizeof(int), sizeof(long), sizeof(void *),
759 sizeof(size_t), ltt_get_alignment());
762 ret
= ust_marker_set_format(entry
, format
);
763 else if (strcmp(entry
->format
, format
))
769 old
= ust_marker_entry_add_probe(entry
, probe
, probe_private
);
773 goto error_unregister_channel
;
777 pthread_mutex_unlock(&ust_marker_mutex
);
779 /* Activate ust_marker if necessary */
780 ust_marker_update_probes();
784 free_old_closure(&old
->rcu
);
788 error_unregister_channel
:
789 ret_err
= ltt_channels_unregister(channel
);
791 error_remove_ust_marker
:
792 ret_err
= remove_ust_marker(channel
, name
);
795 pthread_mutex_unlock(&ust_marker_mutex
);
800 * ust_marker_probe_unregister - Disconnect a probe from a ust_marker
801 * @channel: ust_marker channel
802 * @name: ust_marker name
803 * @probe: probe function pointer
804 * @probe_private: probe private data
806 * Returns the private data given to ust_marker_probe_register, or an ERR_PTR().
807 * We do not need to call a synchronize_sched to make sure the probes have
808 * finished running before doing a module unload, because the module unload
809 * itself uses stop_machine(), which insures that every preempt disabled section
812 int ust_marker_probe_unregister(const char *channel
, const char *name
,
813 ust_marker_probe_func
*probe
, void *probe_private
)
815 struct ust_marker_entry
*entry
;
816 struct ust_marker_probe_array
*old
;
819 pthread_mutex_lock(&ust_marker_mutex
);
820 entry
= get_ust_marker(channel
, name
);
825 old
= ust_marker_entry_remove_probe(entry
, probe
, probe_private
);
826 pthread_mutex_unlock(&ust_marker_mutex
);
828 ust_marker_update_probes();
832 free_old_closure(&old
->rcu
);
837 pthread_mutex_unlock(&ust_marker_mutex
);
841 static struct ust_marker_entry
*
842 get_ust_marker_from_private_data(ust_marker_probe_func
*probe
,
845 struct ust_marker_entry
*entry
;
847 struct cds_hlist_head
*head
;
848 struct cds_hlist_node
*node
;
850 for (i
= 0; i
< UST_MARKER_TABLE_SIZE
; i
++) {
851 head
= &ust_marker_table
[i
];
852 cds_hlist_for_each_entry(entry
, node
, head
, hlist
) {
854 if (entry
->single
.func
== probe
855 && entry
->single
.probe_private
859 struct ust_marker_probe_array
*closure
;
860 closure
= entry
->multi
;
861 for (i
= 0; closure
->c
[i
].func
; i
++) {
862 if (closure
->c
[i
].func
== probe
&&
863 closure
->c
[i
].probe_private
874 * ust_marker_probe_unregister_private_data - Disconnect a probe from a ust_marker
875 * @probe: probe function
876 * @probe_private: probe private data
878 * Unregister a probe by providing the registered private data.
879 * Only removes the first ust_marker found in hash table.
880 * Return 0 on success or error value.
881 * We do not need to call a synchronize_sched to make sure the probes have
882 * finished running before doing a module unload, because the module unload
883 * itself uses stop_machine(), which insures that every preempt disabled section
886 int ust_marker_probe_unregister_private_data(ust_marker_probe_func
*probe
,
889 struct ust_marker_entry
*entry
;
891 struct ust_marker_probe_array
*old
;
892 char *channel
= NULL
, *name
= NULL
;
894 pthread_mutex_lock(&ust_marker_mutex
);
895 entry
= get_ust_marker_from_private_data(probe
, probe_private
);
900 old
= ust_marker_entry_remove_probe(entry
, NULL
, probe_private
);
901 channel
= strdup(entry
->channel
);
902 name
= strdup(entry
->name
);
903 /* Ignore busy error message */
904 remove_ust_marker(channel
, name
);
905 pthread_mutex_unlock(&ust_marker_mutex
);
907 ust_marker_update_probes();
911 free_old_closure(&old
->rcu
);
916 pthread_mutex_unlock(&ust_marker_mutex
);
924 * ust_marker_get_private_data - Get a ust_marker's probe private data
925 * @channel: ust_marker channel
926 * @name: ust_marker name
927 * @probe: probe to match
928 * @num: get the nth matching probe's private data
930 * Returns the nth private data pointer (starting from 0) matching, or an
932 * Returns the private data pointer, or an ERR_PTR.
933 * The private data pointer should _only_ be dereferenced if the caller is the
934 * owner of the data, or its content could vanish. This is mostly used to
935 * confirm that a caller is the owner of a registered probe.
937 void *ust_marker_get_private_data(const char *channel
, const char *name
,
938 ust_marker_probe_func
*probe
, int num
)
940 struct cds_hlist_head
*head
;
941 struct cds_hlist_node
*node
;
942 struct ust_marker_entry
*e
;
943 size_t channel_len
= strlen(channel
) + 1;
944 size_t name_len
= strlen(name
) + 1;
948 hash
= jhash(channel
, channel_len
-1, 0) ^ jhash(name
, name_len
-1, 0);
949 head
= &ust_marker_table
[hash
& ((1 << UST_MARKER_HASH_BITS
)-1)];
950 cds_hlist_for_each_entry(e
, node
, head
, hlist
) {
951 if (!strcmp(channel
, e
->channel
) && !strcmp(name
, e
->name
)) {
953 if (num
== 0 && e
->single
.func
== probe
)
954 return e
->single
.probe_private
;
956 struct ust_marker_probe_array
*closure
;
959 for (i
= 0; closure
->c
[i
].func
; i
++) {
960 if (closure
->c
[i
].func
!= probe
)
963 return closure
->c
[i
].probe_private
;
969 return ERR_PTR(-ENOENT
);
973 * ust_marker_get_iter_range - Get a next ust_marker iterator given a range.
974 * @ust_marker: current ust_marker (in), next ust_marker (out)
975 * @begin: beginning of the range
976 * @end: end of the range
978 * Returns whether a next ust_marker has been found (1) or not (0).
979 * Will return the first ust_marker in the range if the input ust_marker is NULL.
980 * Called with markers mutex held.
983 int ust_marker_get_iter_range(struct ust_marker
* const **ust_marker
,
984 struct ust_marker
* const *begin
,
985 struct ust_marker
* const *end
)
987 if (!*ust_marker
&& begin
!= end
)
989 while (*ust_marker
>= begin
&& *ust_marker
< end
) {
991 (*ust_marker
)++; /* skip dummy */
999 * Returns 0 if current not found.
1000 * Returns 1 if current found.
1001 * Called with markers mutex held.
1004 int lib_get_iter_ust_marker(struct ust_marker_iter
*iter
)
1006 struct ust_marker_lib
*iter_lib
;
1009 cds_list_for_each_entry(iter_lib
, &ust_marker_libs
, list
) {
1010 if (iter_lib
< iter
->lib
)
1012 else if (iter_lib
> iter
->lib
)
1013 iter
->ust_marker
= NULL
;
1014 found
= ust_marker_get_iter_range(&iter
->ust_marker
,
1015 iter_lib
->ust_marker_start
,
1016 iter_lib
->ust_marker_start
+ iter_lib
->ust_marker_count
);
1018 iter
->lib
= iter_lib
;
1025 /* Called with markers mutex held. */
1026 static void ust_marker_get_iter(struct ust_marker_iter
*iter
)
1030 found
= lib_get_iter_ust_marker(iter
);
1032 ust_marker_iter_reset(iter
);
1035 /* Called with markers mutex held. */
1036 void ust_marker_iter_start(struct ust_marker_iter
*iter
)
1038 ust_marker_get_iter(iter
);
1041 /* Called with markers mutex held. */
1042 void ust_marker_iter_next(struct ust_marker_iter
*iter
)
1046 * iter->ust_marker may be invalid because we blindly incremented it.
1047 * Make sure it is valid by marshalling on the ust_marker, getting the
1048 * ust_marker from following modules if necessary.
1050 ust_marker_get_iter(iter
);
1053 /* Called with markers mutex held. */
1054 void ust_marker_iter_stop(struct ust_marker_iter
*iter
)
1058 /* Called with markers mutex held. */
1059 void ust_marker_iter_reset(struct ust_marker_iter
*iter
)
1062 iter
->ust_marker
= NULL
;
1065 void ltt_dump_ust_marker_state(struct ust_trace
*trace
)
1067 struct ust_marker_entry
*entry
;
1068 struct ltt_probe_private_data call_data
;
1069 struct cds_hlist_head
*head
;
1070 struct cds_hlist_node
*node
;
1073 pthread_mutex_lock(&ust_marker_mutex
);
1074 call_data
.trace
= trace
;
1075 call_data
.serializer
= NULL
;
1077 for (i
= 0; i
< UST_MARKER_TABLE_SIZE
; i
++) {
1078 head
= &ust_marker_table
[i
];
1079 cds_hlist_for_each_entry(entry
, node
, head
, hlist
) {
1080 __ust_marker(metadata
, core_marker_id
,
1082 "channel %s name %s event_id %hu "
1083 "int #1u%zu long #1u%zu pointer #1u%zu "
1084 "size_t #1u%zu alignment #1u%u",
1088 sizeof(int), sizeof(long),
1089 sizeof(void *), sizeof(size_t),
1090 ltt_get_alignment());
1092 __ust_marker(metadata
,
1095 "channel %s name %s format %s",
1101 pthread_mutex_unlock(&ust_marker_mutex
);
1104 static void (*new_ust_marker_cb
)(struct ust_marker
*) = NULL
;
1106 void ust_marker_set_new_ust_marker_cb(void (*cb
)(struct ust_marker
*))
1108 new_ust_marker_cb
= cb
;
1111 static void new_ust_marker(struct ust_marker
* const *start
,
1112 struct ust_marker
* const *end
)
1114 if (new_ust_marker_cb
) {
1115 struct ust_marker
* const *m
;
1117 for(m
= start
; m
< end
; m
++) {
1119 new_ust_marker_cb(*m
);
1124 int ust_marker_register_lib(struct ust_marker
* const *ust_marker_start
,
1125 int ust_marker_count
)
1127 struct ust_marker_lib
*pl
, *iter
;
1129 pl
= (struct ust_marker_lib
*) zmalloc(sizeof(struct ust_marker_lib
));
1131 pl
->ust_marker_start
= ust_marker_start
;
1132 pl
->ust_marker_count
= ust_marker_count
;
1137 * We sort the libs by struct lib pointer address.
1139 cds_list_for_each_entry_reverse(iter
, &ust_marker_libs
, list
) {
1140 BUG_ON(iter
== pl
); /* Should never be in the list twice */
1142 /* We belong to the location right after iter. */
1143 cds_list_add(&pl
->list
, &iter
->list
);
1147 /* We should be added at the head of the list */
1148 cds_list_add(&pl
->list
, &ust_marker_libs
);
1150 unlock_ust_marker();
1152 new_ust_marker(ust_marker_start
, ust_marker_start
+ ust_marker_count
);
1154 /* TODO: update just the loaded lib */
1155 lib_update_ust_marker();
1157 DBG("just registered a ust_marker section from %p and having %d ust_marker (minus dummy ust_marker)", ust_marker_start
, ust_marker_count
);
1162 int ust_marker_unregister_lib(struct ust_marker
* const *ust_marker_start
)
1164 struct ust_marker_lib
*lib
;
1167 cds_list_for_each_entry(lib
, &ust_marker_libs
, list
) {
1168 if(lib
->ust_marker_start
== ust_marker_start
) {
1169 struct ust_marker_lib
*lib2free
= lib
;
1170 cds_list_del(&lib
->list
);
1175 unlock_ust_marker();
1180 static int initialized
= 0;
1182 void __attribute__((constructor
)) init_ust_marker(void)
1185 ust_marker_register_lib(__start___ust_marker_ptrs
,
1186 __stop___ust_marker_ptrs
1187 - __start___ust_marker_ptrs
);
1192 void __attribute__((destructor
)) destroy_ust_marker(void)
1194 ust_marker_unregister_lib(__start___ust_marker_ptrs
);