libmarkers/marker.c

   1 /*
   2  * Copyright (C) 2007 Mathieu Desnoyers
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  17  */
  18 //ust// #include <linux/module.h>
  19 //ust// #include <linux/mutex.h>
  20 //ust// #include <linux/types.h>
  21 #include "jhash.h"
  22 #include "list.h"
  23 #include "rcupdate.h"
  24 //ust// #include <linux/marker.h>
  25 #include <errno.h>
  26 //ust// #include <linux/slab.h>
  27 //ust// #include <linux/immediate.h>
  28 //ust// #include <linux/sched.h>
  29 //ust// #include <linux/uaccess.h>
  30 //ust// #include <linux/user_marker.h>
  31 //ust// #include <linux/ltt-tracer.h>
  32
  33 #include "marker.h"
  34 #include "kernelcompat.h"
  35 #include "usterr.h"
  36 #include "channels.h"
  37 #include "tracercore.h"
  38 #include "tracer.h"
  39
  40 extern struct marker __start___markers[] __attribute__((visibility("hidden")));
  41 extern struct marker __stop___markers[] __attribute__((visibility("hidden")));
  42
  43 /* Set to 1 to enable marker debug output */
  44 static const int marker_debug;
  45
  46 /*
  47  * markers_mutex nests inside module_mutex. Markers mutex protects the builtin
  48  * and module markers and the hash table.
  49  */
  50 static DEFINE_MUTEX(markers_mutex);
  51
  52 void lock_markers(void)
  53 {
  54         mutex_lock(&markers_mutex);
  55 }
  56
  57 void unlock_markers(void)
  58 {
  59         mutex_unlock(&markers_mutex);
  60 }
  61
  62 /*
  63  * Marker hash table, containing the active markers.
  64  * Protected by module_mutex.
  65  */
  66 #define MARKER_HASH_BITS 6
  67 #define MARKER_TABLE_SIZE (1 << MARKER_HASH_BITS)
  68 static struct hlist_head marker_table[MARKER_TABLE_SIZE];
  69
  70 /*
  71  * Note about RCU :
  72  * It is used to make sure every handler has finished using its private data
  73  * between two consecutive operation (add or remove) on a given marker.  It is
  74  * also used to delay the free of multiple probes array until a quiescent state
  75  * is reached.
  76  * marker entries modifications are protected by the markers_mutex.
  77  */
  78 struct marker_entry {
  79         struct hlist_node hlist;
  80         char *format;
  81         char *name;
  82                         /* Probe wrapper */
  83         void (*call)(const struct marker *mdata, void *call_private, ...);
  84         struct marker_probe_closure single;
  85         struct marker_probe_closure *multi;
  86         int refcount;   /* Number of times armed. 0 if disarmed. */
  87         struct rcu_head rcu;
  88         void *oldptr;
  89         int rcu_pending;
  90         u16 channel_id;
  91         u16 event_id;
  92         unsigned char ptype:1;
  93         unsigned char format_allocated:1;
  94         char channel[0];        /* Contains channel'\0'name'\0'format'\0' */
  95 };
  96
  97 #ifdef CONFIG_MARKERS_USERSPACE
  98 static void marker_update_processes(void);
  99 #else
 100 static void marker_update_processes(void)
 101 {
 102 }
 103 #endif
 104
 105 /**
 106  * __mark_empty_function - Empty probe callback
 107  * @mdata: marker data
 108  * @probe_private: probe private data
 109  * @call_private: call site private data
 110  * @fmt: format string
 111  * @...: variable argument list
 112  *
 113  * Empty callback provided as a probe to the markers. By providing this to a
 114  * disabled marker, we make sure the  execution flow is always valid even
 115  * though the function pointer change and the marker enabling are two distinct
 116  * operations that modifies the execution flow of preemptible code.
 117  */
 118 notrace void __mark_empty_function(const struct marker *mdata,
 119         void *probe_private, void *call_private, const char *fmt, va_list *args)
 120 {
 121 }
 122 //ust// EXPORT_SYMBOL_GPL(__mark_empty_function);
 123
 124 /*
 125  * marker_probe_cb Callback that prepares the variable argument list for probes.
 126  * @mdata: pointer of type struct marker
 127  * @call_private: caller site private data
 128  * @...:  Variable argument list.
 129  *
 130  * Since we do not use "typical" pointer based RCU in the 1 argument case, we
 131  * need to put a full smp_rmb() in this branch. This is why we do not use
 132  * rcu_dereference() for the pointer read.
 133  */
 134 notrace void marker_probe_cb(const struct marker *mdata,
 135                 void *call_private, ...)
 136 {
 137         va_list args;
 138         char ptype;
 139
 140         /*
 141          * rcu_read_lock_sched does two things : disabling preemption to make
 142          * sure the teardown of the callbacks can be done correctly when they
 143          * are in modules and they insure RCU read coherency.
 144          */
 145 //ust// rcu_read_lock_sched_notrace();
 146         ptype = mdata->ptype;
 147         if (likely(!ptype)) {
 148                 marker_probe_func *func;
 149                 /* Must read the ptype before ptr. They are not data dependant,
 150                  * so we put an explicit smp_rmb() here. */
 151                 smp_rmb();
 152                 func = mdata->single.func;
 153                 /* Must read the ptr before private data. They are not data
 154                  * dependant, so we put an explicit smp_rmb() here. */
 155                 smp_rmb();
 156                 va_start(args, call_private);
 157                 func(mdata, mdata->single.probe_private, call_private,
 158                         mdata->format, &args);
 159                 va_end(args);
 160         } else {
 161                 struct marker_probe_closure *multi;
 162                 int i;
 163                 /*
 164                  * Read mdata->ptype before mdata->multi.
 165                  */
 166                 smp_rmb();
 167                 multi = mdata->multi;
 168                 /*
 169                  * multi points to an array, therefore accessing the array
 170                  * depends on reading multi. However, even in this case,
 171                  * we must insure that the pointer is read _before_ the array
 172                  * data. Same as rcu_dereference, but we need a full smp_rmb()
 173                  * in the fast path, so put the explicit barrier here.
 174                  */
 175                 smp_read_barrier_depends();
 176                 for (i = 0; multi[i].func; i++) {
 177                         va_start(args, call_private);
 178                         multi[i].func(mdata, multi[i].probe_private,
 179                                 call_private, mdata->format, &args);
 180                         va_end(args);
 181                 }
 182         }
 183 //ust// rcu_read_unlock_sched_notrace();
 184 }
 185 //ust// EXPORT_SYMBOL_GPL(marker_probe_cb);
 186
 187 /*
 188  * marker_probe_cb Callback that does not prepare the variable argument list.
 189  * @mdata: pointer of type struct marker
 190  * @call_private: caller site private data
 191  * @...:  Variable argument list.
 192  *
 193  * Should be connected to markers "MARK_NOARGS".
 194  */
 195 static notrace void marker_probe_cb_noarg(const struct marker *mdata,
 196                 void *call_private, ...)
 197 {
 198         va_list args;   /* not initialized */
 199         char ptype;
 200
 201 //ust// rcu_read_lock_sched_notrace();
 202         ptype = mdata->ptype;
 203         if (likely(!ptype)) {
 204                 marker_probe_func *func;
 205                 /* Must read the ptype before ptr. They are not data dependant,
 206                  * so we put an explicit smp_rmb() here. */
 207                 smp_rmb();
 208                 func = mdata->single.func;
 209                 /* Must read the ptr before private data. They are not data
 210                  * dependant, so we put an explicit smp_rmb() here. */
 211                 smp_rmb();
 212                 func(mdata, mdata->single.probe_private, call_private,
 213                         mdata->format, &args);
 214         } else {
 215                 struct marker_probe_closure *multi;
 216                 int i;
 217                 /*
 218                  * Read mdata->ptype before mdata->multi.
 219                  */
 220                 smp_rmb();
 221                 multi = mdata->multi;
 222                 /*
 223                  * multi points to an array, therefore accessing the array
 224                  * depends on reading multi. However, even in this case,
 225                  * we must insure that the pointer is read _before_ the array
 226                  * data. Same as rcu_dereference, but we need a full smp_rmb()
 227                  * in the fast path, so put the explicit barrier here.
 228                  */
 229                 smp_read_barrier_depends();
 230                 for (i = 0; multi[i].func; i++)
 231                         multi[i].func(mdata, multi[i].probe_private,
 232                                 call_private, mdata->format, &args);
 233         }
 234 //ust// rcu_read_unlock_sched_notrace();
 235 }
 236
 237 static void free_old_closure(struct rcu_head *head)
 238 {
 239         struct marker_entry *entry = container_of(head,
 240                 struct marker_entry, rcu);
 241         kfree(entry->oldptr);
 242         /* Make sure we free the data before setting the pending flag to 0 */
 243         smp_wmb();
 244         entry->rcu_pending = 0;
 245 }
 246
 247 static void debug_print_probes(struct marker_entry *entry)
 248 {
 249         int i;
 250
 251         if (!marker_debug)
 252                 return;
 253
 254         if (!entry->ptype) {
 255                 printk(KERN_DEBUG "Single probe : %p %p\n",
 256                         entry->single.func,
 257                         entry->single.probe_private);
 258         } else {
 259                 for (i = 0; entry->multi[i].func; i++)
 260                         printk(KERN_DEBUG "Multi probe %d : %p %p\n", i,
 261                                 entry->multi[i].func,
 262                                 entry->multi[i].probe_private);
 263         }
 264 }
 265
 266 static struct marker_probe_closure *
 267 marker_entry_add_probe(struct marker_entry *entry,
 268                 marker_probe_func *probe, void *probe_private)
 269 {
 270         int nr_probes = 0;
 271         struct marker_probe_closure *old, *new;
 272
 273         WARN_ON(!probe);
 274
 275         debug_print_probes(entry);
 276         old = entry->multi;
 277         if (!entry->ptype) {
 278                 if (entry->single.func == probe &&
 279                                 entry->single.probe_private == probe_private)
 280                         return ERR_PTR(-EBUSY);
 281                 if (entry->single.func == __mark_empty_function) {
 282                         /* 0 -> 1 probes */
 283                         entry->single.func = probe;
 284                         entry->single.probe_private = probe_private;
 285                         entry->refcount = 1;
 286                         entry->ptype = 0;
 287                         debug_print_probes(entry);
 288                         return NULL;
 289                 } else {
 290                         /* 1 -> 2 probes */
 291                         nr_probes = 1;
 292                         old = NULL;
 293                 }
 294         } else {
 295                 /* (N -> N+1), (N != 0, 1) probes */
 296                 for (nr_probes = 0; old[nr_probes].func; nr_probes++)
 297                         if (old[nr_probes].func == probe
 298                                         && old[nr_probes].probe_private
 299                                                 == probe_private)
 300                                 return ERR_PTR(-EBUSY);
 301         }
 302         /* + 2 : one for new probe, one for NULL func */
 303         new = kzalloc((nr_probes + 2) * sizeof(struct marker_probe_closure),
 304                         GFP_KERNEL);
 305         if (new == NULL)
 306                 return ERR_PTR(-ENOMEM);
 307         if (!old)
 308                 new[0] = entry->single;
 309         else
 310                 memcpy(new, old,
 311                         nr_probes * sizeof(struct marker_probe_closure));
 312         new[nr_probes].func = probe;
 313         new[nr_probes].probe_private = probe_private;
 314         entry->refcount = nr_probes + 1;
 315         entry->multi = new;
 316         entry->ptype = 1;
 317         debug_print_probes(entry);
 318         return old;
 319 }
 320
 321 static struct marker_probe_closure *
 322 marker_entry_remove_probe(struct marker_entry *entry,
 323                 marker_probe_func *probe, void *probe_private)
 324 {
 325         int nr_probes = 0, nr_del = 0, i;
 326         struct marker_probe_closure *old, *new;
 327
 328         old = entry->multi;
 329
 330         debug_print_probes(entry);
 331         if (!entry->ptype) {
 332                 /* 0 -> N is an error */
 333                 WARN_ON(entry->single.func == __mark_empty_function);
 334                 /* 1 -> 0 probes */
 335                 WARN_ON(probe && entry->single.func != probe);
 336                 WARN_ON(entry->single.probe_private != probe_private);
 337                 entry->single.func = __mark_empty_function;
 338                 entry->refcount = 0;
 339                 entry->ptype = 0;
 340                 debug_print_probes(entry);
 341                 return NULL;
 342         } else {
 343                 /* (N -> M), (N > 1, M >= 0) probes */
 344                 for (nr_probes = 0; old[nr_probes].func; nr_probes++) {
 345                         if ((!probe || old[nr_probes].func == probe)
 346                                         && old[nr_probes].probe_private
 347                                                 == probe_private)
 348                                 nr_del++;
 349                 }
 350         }
 351
 352         if (nr_probes - nr_del == 0) {
 353                 /* N -> 0, (N > 1) */
 354                 entry->single.func = __mark_empty_function;
 355                 entry->refcount = 0;
 356                 entry->ptype = 0;
 357         } else if (nr_probes - nr_del == 1) {
 358                 /* N -> 1, (N > 1) */
 359                 for (i = 0; old[i].func; i++)
 360                         if ((probe && old[i].func != probe) ||
 361                                         old[i].probe_private != probe_private)
 362                                 entry->single = old[i];
 363                 entry->refcount = 1;
 364                 entry->ptype = 0;
 365         } else {
 366                 int j = 0;
 367                 /* N -> M, (N > 1, M > 1) */
 368                 /* + 1 for NULL */
 369                 new = kzalloc((nr_probes - nr_del + 1)
 370                         * sizeof(struct marker_probe_closure), GFP_KERNEL);
 371                 if (new == NULL)
 372                         return ERR_PTR(-ENOMEM);
 373                 for (i = 0; old[i].func; i++)
 374                         if ((probe && old[i].func != probe) ||
 375                                         old[i].probe_private != probe_private)
 376                                 new[j++] = old[i];
 377                 entry->refcount = nr_probes - nr_del;
 378                 entry->ptype = 1;
 379                 entry->multi = new;
 380         }
 381         debug_print_probes(entry);
 382         return old;
 383 }
 384
 385 /*
 386  * Get marker if the marker is present in the marker hash table.
 387  * Must be called with markers_mutex held.
 388  * Returns NULL if not present.
 389  */
 390 static struct marker_entry *get_marker(const char *channel, const char *name)
 391 {
 392         struct hlist_head *head;
 393         struct hlist_node *node;
 394         struct marker_entry *e;
 395         size_t channel_len = strlen(channel) + 1;
 396         size_t name_len = strlen(name) + 1;
 397         u32 hash;
 398
 399         hash = jhash(channel, channel_len-1, 0) ^ jhash(name, name_len-1, 0);
 400         head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)];
 401         hlist_for_each_entry(e, node, head, hlist) {
 402                 if (!strcmp(channel, e->channel) && !strcmp(name, e->name))
 403                         return e;
 404         }
 405         return NULL;
 406 }
 407
 408 /*
 409  * Add the marker to the marker hash table. Must be called with markers_mutex
 410  * held.
 411  */
 412 static struct marker_entry *add_marker(const char *channel, const char *name,
 413                 const char *format)
 414 {
 415         struct hlist_head *head;
 416         struct hlist_node *node;
 417         struct marker_entry *e;
 418         size_t channel_len = strlen(channel) + 1;
 419         size_t name_len = strlen(name) + 1;
 420         size_t format_len = 0;
 421         u32 hash;
 422
 423         hash = jhash(channel, channel_len-1, 0) ^ jhash(name, name_len-1, 0);
 424         if (format)
 425                 format_len = strlen(format) + 1;
 426         head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)];
 427         hlist_for_each_entry(e, node, head, hlist) {
 428                 if (!strcmp(channel, e->channel) && !strcmp(name, e->name)) {
 429                         printk(KERN_NOTICE
 430                                 "Marker %s.%s busy\n", channel, name);
 431                         return ERR_PTR(-EBUSY); /* Already there */
 432                 }
 433         }
 434         /*
 435          * Using kmalloc here to allocate a variable length element. Could
 436          * cause some memory fragmentation if overused.
 437          */
 438         e = kmalloc(sizeof(struct marker_entry)
 439                     + channel_len + name_len + format_len,
 440                     GFP_KERNEL);
 441         if (!e)
 442                 return ERR_PTR(-ENOMEM);
 443         memcpy(e->channel, channel, channel_len);
 444         e->name = &e->channel[channel_len];
 445         memcpy(e->name, name, name_len);
 446         if (format) {
 447                 e->format = &e->name[channel_len + name_len];
 448                 memcpy(e->format, format, format_len);
 449                 if (strcmp(e->format, MARK_NOARGS) == 0)
 450                         e->call = marker_probe_cb_noarg;
 451                 else
 452                         e->call = marker_probe_cb;
 453                 trace_mark(metadata, core_marker_format,
 454                            "channel %s name %s format %s",
 455                            e->channel, e->name, e->format);
 456         } else {
 457                 e->format = NULL;
 458                 e->call = marker_probe_cb;
 459         }
 460         e->single.func = __mark_empty_function;
 461         e->single.probe_private = NULL;
 462         e->multi = NULL;
 463         e->ptype = 0;
 464         e->format_allocated = 0;
 465         e->refcount = 0;
 466         e->rcu_pending = 0;
 467         hlist_add_head(&e->hlist, head);
 468         return e;
 469 }
 470
 471 /*
 472  * Remove the marker from the marker hash table. Must be called with mutex_lock
 473  * held.
 474  */
 475 static int remove_marker(const char *channel, const char *name)
 476 {
 477         struct hlist_head *head;
 478         struct hlist_node *node;
 479         struct marker_entry *e;
 480         int found = 0;
 481         size_t channel_len = strlen(channel) + 1;
 482         size_t name_len = strlen(name) + 1;
 483         u32 hash;
 484         int ret;
 485
 486         hash = jhash(channel, channel_len-1, 0) ^ jhash(name, name_len-1, 0);
 487         head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)];
 488         hlist_for_each_entry(e, node, head, hlist) {
 489                 if (!strcmp(channel, e->channel) && !strcmp(name, e->name)) {
 490                         found = 1;
 491                         break;
 492                 }
 493         }
 494         if (!found)
 495                 return -ENOENT;
 496         if (e->single.func != __mark_empty_function)
 497                 return -EBUSY;
 498         hlist_del(&e->hlist);
 499         if (e->format_allocated)
 500                 kfree(e->format);
 501         ret = ltt_channels_unregister(e->channel);
 502         WARN_ON(ret);
 503         /* Make sure the call_rcu has been executed */
 504         if (e->rcu_pending)
 505                 rcu_barrier_sched();
 506         kfree(e);
 507         return 0;
 508 }
 509
 510 /*
 511  * Set the mark_entry format to the format found in the element.
 512  */
 513 static int marker_set_format(struct marker_entry *entry, const char *format)
 514 {
 515         entry->format = kstrdup(format, GFP_KERNEL);
 516         if (!entry->format)
 517                 return -ENOMEM;
 518         entry->format_allocated = 1;
 519
 520         trace_mark(metadata, core_marker_format,
 521                    "channel %s name %s format %s",
 522                    entry->channel, entry->name, entry->format);
 523         return 0;
 524 }
 525
 526 /*
 527  * Sets the probe callback corresponding to one marker.
 528  */
 529 static int set_marker(struct marker_entry *entry, struct marker *elem,
 530                 int active)
 531 {
 532         int ret = 0;
 533         WARN_ON(strcmp(entry->name, elem->name) != 0);
 534
 535         if (entry->format) {
 536                 if (strcmp(entry->format, elem->format) != 0) {
 537                         printk(KERN_NOTICE
 538                                 "Format mismatch for probe %s "
 539                                 "(%s), marker (%s)\n",
 540                                 entry->name,
 541                                 entry->format,
 542                                 elem->format);
 543                         return -EPERM;
 544                 }
 545         } else {
 546                 ret = marker_set_format(entry, elem->format);
 547                 if (ret)
 548                         return ret;
 549         }
 550
 551         /*
 552          * probe_cb setup (statically known) is done here. It is
 553          * asynchronous with the rest of execution, therefore we only
 554          * pass from a "safe" callback (with argument) to an "unsafe"
 555          * callback (does not set arguments).
 556          */
 557         elem->call = entry->call;
 558         elem->channel_id = entry->channel_id;
 559         elem->event_id = entry->event_id;
 560         /*
 561          * Sanity check :
 562          * We only update the single probe private data when the ptr is
 563          * set to a _non_ single probe! (0 -> 1 and N -> 1, N != 1)
 564          */
 565         WARN_ON(elem->single.func != __mark_empty_function
 566                 && elem->single.probe_private != entry->single.probe_private
 567                 && !elem->ptype);
 568         elem->single.probe_private = entry->single.probe_private;
 569         /*
 570          * Make sure the private data is valid when we update the
 571          * single probe ptr.
 572          */
 573         smp_wmb();
 574         elem->single.func = entry->single.func;
 575         /*
 576          * We also make sure that the new probe callbacks array is consistent
 577          * before setting a pointer to it.
 578          */
 579         rcu_assign_pointer(elem->multi, entry->multi);
 580         /*
 581          * Update the function or multi probe array pointer before setting the
 582          * ptype.
 583          */
 584         smp_wmb();
 585         elem->ptype = entry->ptype;
 586
 587 //ust// if (elem->tp_name && (active ^ _imv_read(elem->state))) {
 588 //ust//         WARN_ON(!elem->tp_cb);
 589 //ust//         /*
 590 //ust//          * It is ok to directly call the probe registration because type
 591 //ust//          * checking has been done in the __trace_mark_tp() macro.
 592 //ust//          */
 593 //ust//
 594 //ust//         if (active) {
 595 //ust//                 /*
 596 //ust//                  * try_module_get should always succeed because we hold
 597 //ust//                  * markers_mutex to get the tp_cb address.
 598 //ust//                  */
 599 //ust//                 ret = try_module_get(__module_text_address(
 600 //ust//                         (unsigned long)elem->tp_cb));
 601 //ust//                 BUG_ON(!ret);
 602 //ust//                 ret = tracepoint_probe_register_noupdate(
 603 //ust//                         elem->tp_name,
 604 //ust//                         elem->tp_cb);
 605 //ust//         } else {
 606 //ust//                 ret = tracepoint_probe_unregister_noupdate(
 607 //ust//                         elem->tp_name,
 608 //ust//                         elem->tp_cb);
 609 //ust//                 /*
 610 //ust//                  * tracepoint_probe_update_all() must be called
 611 //ust//                  * before the module containing tp_cb is unloaded.
 612 //ust//                  */
 613 //ust//                 module_put(__module_text_address(
 614 //ust//                         (unsigned long)elem->tp_cb));
 615 //ust//         }
 616 //ust// }
 617         elem->state__imv = active;
 618
 619         return ret;
 620 }
 621
 622 /*
 623  * Disable a marker and its probe callback.
 624  * Note: only waiting an RCU period after setting elem->call to the empty
 625  * function insures that the original callback is not used anymore. This insured
 626  * by rcu_read_lock_sched around the call site.
 627  */
 628 static void disable_marker(struct marker *elem)
 629 {
 630         int ret;
 631
 632         /* leave "call" as is. It is known statically. */
 633 //ust// if (elem->tp_name && _imv_read(elem->state)) {
 634 //ust//         WARN_ON(!elem->tp_cb);
 635 //ust//         /*
 636 //ust//          * It is ok to directly call the probe registration because type
 637 //ust//          * checking has been done in the __trace_mark_tp() macro.
 638 //ust//          */
 639 //ust//         ret = tracepoint_probe_unregister_noupdate(elem->tp_name,
 640 //ust//                 elem->tp_cb);
 641 //ust//         WARN_ON(ret);
 642 //ust//         /*
 643 //ust//          * tracepoint_probe_update_all() must be called
 644 //ust//          * before the module containing tp_cb is unloaded.
 645 //ust//          */
 646 //ust//         module_put(__module_text_address((unsigned long)elem->tp_cb));
 647 //ust// }
 648         elem->state__imv = 0;
 649         elem->single.func = __mark_empty_function;
 650         /* Update the function before setting the ptype */
 651         smp_wmb();
 652         elem->ptype = 0;        /* single probe */
 653         /*
 654          * Leave the private data and channel_id/event_id there, because removal
 655          * is racy and should be done only after an RCU period. These are never
 656          * used until the next initialization anyway.
 657          */
 658 }
 659
 660 /**
 661  * marker_update_probe_range - Update a probe range
 662  * @begin: beginning of the range
 663  * @end: end of the range
 664  *
 665  * Updates the probe callback corresponding to a range of markers.
 666  */
 667 void marker_update_probe_range(struct marker *begin,
 668         struct marker *end)
 669 {
 670         struct marker *iter;
 671         struct marker_entry *mark_entry;
 672
 673         mutex_lock(&markers_mutex);
 674         for (iter = begin; iter < end; iter++) {
 675                 mark_entry = get_marker(iter->channel, iter->name);
 676                 if (mark_entry) {
 677                         set_marker(mark_entry, iter, !!mark_entry->refcount);
 678                         /*
 679                          * ignore error, continue
 680                          */
 681                 } else {
 682                         disable_marker(iter);
 683                 }
 684         }
 685         mutex_unlock(&markers_mutex);
 686 }
 687
 688 /*
 689  * Update probes, removing the faulty probes.
 690  *
 691  * Internal callback only changed before the first probe is connected to it.
 692  * Single probe private data can only be changed on 0 -> 1 and 2 -> 1
 693  * transitions.  All other transitions will leave the old private data valid.
 694  * This makes the non-atomicity of the callback/private data updates valid.
 695  *
 696  * "special case" updates :
 697  * 0 -> 1 callback
 698  * 1 -> 0 callback
 699  * 1 -> 2 callbacks
 700  * 2 -> 1 callbacks
 701  * Other updates all behave the same, just like the 2 -> 3 or 3 -> 2 updates.
 702  * Site effect : marker_set_format may delete the marker entry (creating a
 703  * replacement).
 704  */
 705 static void marker_update_probes(void)
 706 {
 707         /* Core kernel markers */
 708 //ust// marker_update_probe_range(__start___markers, __stop___markers);
 709         /* Markers in modules. */
 710 //ust// module_update_markers();
 711         lib_update_markers();
 712 //ust// tracepoint_probe_update_all();
 713         /* Update immediate values */
 714         core_imv_update();
 715 //ust// module_imv_update();
 716         marker_update_processes();
 717 }
 718
 719 /**
 720  * marker_probe_register -  Connect a probe to a marker
 721  * @channel: marker channel
 722  * @name: marker name
 723  * @format: format string
 724  * @probe: probe handler
 725  * @probe_private: probe private data
 726  *
 727  * private data must be a valid allocated memory address, or NULL.
 728  * Returns 0 if ok, error value on error.
 729  * The probe address must at least be aligned on the architecture pointer size.
 730  */
 731 int marker_probe_register(const char *channel, const char *name,
 732                           const char *format, marker_probe_func *probe,
 733                           void *probe_private)
 734 {
 735         struct marker_entry *entry;
 736         int ret = 0, ret_err;
 737         struct marker_probe_closure *old;
 738         int first_probe = 0;
 739
 740         mutex_lock(&markers_mutex);
 741         entry = get_marker(channel, name);
 742         if (!entry) {
 743                 first_probe = 1;
 744                 entry = add_marker(channel, name, format);
 745                 if (IS_ERR(entry))
 746                         ret = PTR_ERR(entry);
 747                 if (ret)
 748                         goto end;
 749                 ret = ltt_channels_register(channel);
 750                 if (ret)
 751                         goto error_remove_marker;
 752                 ret = ltt_channels_get_index_from_name(channel);
 753                 if (ret < 0)
 754                         goto error_unregister_channel;
 755                 entry->channel_id = ret;
 756                 ret = ltt_channels_get_event_id(channel, name);
 757                 if (ret < 0)
 758                         goto error_unregister_channel;
 759                 entry->event_id = ret;
 760                 ret = 0;
 761                 trace_mark(metadata, core_marker_id,
 762                            "channel %s name %s event_id %hu "
 763                            "int #1u%zu long #1u%zu pointer #1u%zu "
 764                            "size_t #1u%zu alignment #1u%u",
 765                            channel, name, entry->event_id,
 766                            sizeof(int), sizeof(long), sizeof(void *),
 767                            sizeof(size_t), ltt_get_alignment());
 768         } else if (format) {
 769                 if (!entry->format)
 770                         ret = marker_set_format(entry, format);
 771                 else if (strcmp(entry->format, format))
 772                         ret = -EPERM;
 773                 if (ret)
 774                         goto end;
 775         }
 776
 777         /*
 778          * If we detect that a call_rcu is pending for this marker,
 779          * make sure it's executed now.
 780          */
 781         if (entry->rcu_pending)
 782                 rcu_barrier_sched();
 783         old = marker_entry_add_probe(entry, probe, probe_private);
 784         if (IS_ERR(old)) {
 785                 ret = PTR_ERR(old);
 786                 if (first_probe)
 787                         goto error_unregister_channel;
 788                 else
 789                         goto end;
 790         }
 791         mutex_unlock(&markers_mutex);
 792
 793         marker_update_probes();
 794
 795         mutex_lock(&markers_mutex);
 796         entry = get_marker(channel, name);
 797         if (!entry)
 798                 goto end;
 799         if (entry->rcu_pending)
 800                 rcu_barrier_sched();
 801         entry->oldptr = old;
 802         entry->rcu_pending = 1;
 803         /* write rcu_pending before calling the RCU callback */
 804         smp_wmb();
 805         call_rcu_sched(&entry->rcu, free_old_closure);
 806         goto end;
 807
 808 error_unregister_channel:
 809         ret_err = ltt_channels_unregister(channel);
 810         WARN_ON(ret_err);
 811 error_remove_marker:
 812         ret_err = remove_marker(channel, name);
 813         WARN_ON(ret_err);
 814 end:
 815         mutex_unlock(&markers_mutex);
 816         return ret;
 817 }
 818 //ust// EXPORT_SYMBOL_GPL(marker_probe_register);
 819
 820 /**
 821  * marker_probe_unregister -  Disconnect a probe from a marker
 822  * @channel: marker channel
 823  * @name: marker name
 824  * @probe: probe function pointer
 825  * @probe_private: probe private data
 826  *
 827  * Returns the private data given to marker_probe_register, or an ERR_PTR().
 828  * We do not need to call a synchronize_sched to make sure the probes have
 829  * finished running before doing a module unload, because the module unload
 830  * itself uses stop_machine(), which insures that every preempt disabled section
 831  * have finished.
 832  */
 833 int marker_probe_unregister(const char *channel, const char *name,
 834                             marker_probe_func *probe, void *probe_private)
 835 {
 836         struct marker_entry *entry;
 837         struct marker_probe_closure *old;
 838         int ret = -ENOENT;
 839
 840         mutex_lock(&markers_mutex);
 841         entry = get_marker(channel, name);
 842         if (!entry)
 843                 goto end;
 844         if (entry->rcu_pending)
 845                 rcu_barrier_sched();
 846         old = marker_entry_remove_probe(entry, probe, probe_private);
 847         mutex_unlock(&markers_mutex);
 848
 849         marker_update_probes();
 850
 851         mutex_lock(&markers_mutex);
 852         entry = get_marker(channel, name);
 853         if (!entry)
 854                 goto end;
 855         if (entry->rcu_pending)
 856                 rcu_barrier_sched();
 857         entry->oldptr = old;
 858         entry->rcu_pending = 1;
 859         /* write rcu_pending before calling the RCU callback */
 860         smp_wmb();
 861         call_rcu_sched(&entry->rcu, free_old_closure);
 862         remove_marker(channel, name);   /* Ignore busy error message */
 863         ret = 0;
 864 end:
 865         mutex_unlock(&markers_mutex);
 866         return ret;
 867 }
 868 //ust// EXPORT_SYMBOL_GPL(marker_probe_unregister);
 869
 870 static struct marker_entry *
 871 get_marker_from_private_data(marker_probe_func *probe, void *probe_private)
 872 {
 873         struct marker_entry *entry;
 874         unsigned int i;
 875         struct hlist_head *head;
 876         struct hlist_node *node;
 877
 878         for (i = 0; i < MARKER_TABLE_SIZE; i++) {
 879                 head = &marker_table[i];
 880                 hlist_for_each_entry(entry, node, head, hlist) {
 881                         if (!entry->ptype) {
 882                                 if (entry->single.func == probe
 883                                                 && entry->single.probe_private
 884                                                 == probe_private)
 885                                         return entry;
 886                         } else {
 887                                 struct marker_probe_closure *closure;
 888                                 closure = entry->multi;
 889                                 for (i = 0; closure[i].func; i++) {
 890                                         if (closure[i].func == probe &&
 891                                                         closure[i].probe_private
 892                                                         == probe_private)
 893                                                 return entry;
 894                                 }
 895                         }
 896                 }
 897         }
 898         return NULL;
 899 }
 900
 901 /**
 902  * marker_probe_unregister_private_data -  Disconnect a probe from a marker
 903  * @probe: probe function
 904  * @probe_private: probe private data
 905  *
 906  * Unregister a probe by providing the registered private data.
 907  * Only removes the first marker found in hash table.
 908  * Return 0 on success or error value.
 909  * We do not need to call a synchronize_sched to make sure the probes have
 910  * finished running before doing a module unload, because the module unload
 911  * itself uses stop_machine(), which insures that every preempt disabled section
 912  * have finished.
 913  */
 914 int marker_probe_unregister_private_data(marker_probe_func *probe,
 915                 void *probe_private)
 916 {
 917         struct marker_entry *entry;
 918         int ret = 0;
 919         struct marker_probe_closure *old;
 920         const char *channel = NULL, *name = NULL;
 921
 922         mutex_lock(&markers_mutex);
 923         entry = get_marker_from_private_data(probe, probe_private);
 924         if (!entry) {
 925                 ret = -ENOENT;
 926                 goto end;
 927         }
 928         if (entry->rcu_pending)
 929                 rcu_barrier_sched();
 930         old = marker_entry_remove_probe(entry, NULL, probe_private);
 931         channel = kstrdup(entry->channel, GFP_KERNEL);
 932         name = kstrdup(entry->name, GFP_KERNEL);
 933         mutex_unlock(&markers_mutex);
 934
 935         marker_update_probes();
 936
 937         mutex_lock(&markers_mutex);
 938         entry = get_marker(channel, name);
 939         if (!entry)
 940                 goto end;
 941         if (entry->rcu_pending)
 942                 rcu_barrier_sched();
 943         entry->oldptr = old;
 944         entry->rcu_pending = 1;
 945         /* write rcu_pending before calling the RCU callback */
 946         smp_wmb();
 947         call_rcu_sched(&entry->rcu, free_old_closure);
 948         /* Ignore busy error message */
 949         remove_marker(channel, name);
 950 end:
 951         mutex_unlock(&markers_mutex);
 952         kfree(channel);
 953         kfree(name);
 954         return ret;
 955 }
 956 //ust// EXPORT_SYMBOL_GPL(marker_probe_unregister_private_data);
 957
 958 /**
 959  * marker_get_private_data - Get a marker's probe private data
 960  * @channel: marker channel
 961  * @name: marker name
 962  * @probe: probe to match
 963  * @num: get the nth matching probe's private data
 964  *
 965  * Returns the nth private data pointer (starting from 0) matching, or an
 966  * ERR_PTR.
 967  * Returns the private data pointer, or an ERR_PTR.
 968  * The private data pointer should _only_ be dereferenced if the caller is the
 969  * owner of the data, or its content could vanish. This is mostly used to
 970  * confirm that a caller is the owner of a registered probe.
 971  */
 972 void *marker_get_private_data(const char *channel, const char *name,
 973                               marker_probe_func *probe, int num)
 974 {
 975         struct hlist_head *head;
 976         struct hlist_node *node;
 977         struct marker_entry *e;
 978         size_t channel_len = strlen(channel) + 1;
 979         size_t name_len = strlen(name) + 1;
 980         int i;
 981         u32 hash;
 982
 983         hash = jhash(channel, channel_len-1, 0) ^ jhash(name, name_len-1, 0);
 984         head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)];
 985         hlist_for_each_entry(e, node, head, hlist) {
 986                 if (!strcmp(channel, e->channel) && !strcmp(name, e->name)) {
 987                         if (!e->ptype) {
 988                                 if (num == 0 && e->single.func == probe)
 989                                         return e->single.probe_private;
 990                         } else {
 991                                 struct marker_probe_closure *closure;
 992                                 int match = 0;
 993                                 closure = e->multi;
 994                                 for (i = 0; closure[i].func; i++) {
 995                                         if (closure[i].func != probe)
 996                                                 continue;
 997                                         if (match++ == num)
 998                                                 return closure[i].probe_private;
 999                                 }
1000                         }
1001                         break;
1002                 }
1003         }
1004         return ERR_PTR(-ENOENT);
1005 }
1006 //ust// EXPORT_SYMBOL_GPL(marker_get_private_data);
1007
1008 /**
1009  * markers_compact_event_ids - Compact markers event IDs and reassign channels
1010  *
1011  * Called when no channel users are active by the channel infrastructure.
1012  * Called with lock_markers() and channel mutex held.
1013  */
1014 //ust// void markers_compact_event_ids(void)
1015 //ust// {
1016 //ust//         struct marker_entry *entry;
1017 //ust//         unsigned int i;
1018 //ust//         struct hlist_head *head;
1019 //ust//         struct hlist_node *node;
1020 //ust//         int ret;
1021 //ust//
1022 //ust//         for (i = 0; i < MARKER_TABLE_SIZE; i++) {
1023 //ust//                 head = &marker_table[i];
1024 //ust//                 hlist_for_each_entry(entry, node, head, hlist) {
1025 //ust//                         ret = ltt_channels_get_index_from_name(entry->channel);
1026 //ust//                         WARN_ON(ret < 0);
1027 //ust//                         entry->channel_id = ret;
1028 //ust//                         ret = _ltt_channels_get_event_id(entry->channel,
1029 //ust//                                                          entry->name);
1030 //ust//                         WARN_ON(ret < 0);
1031 //ust//                         entry->event_id = ret;
1032 //ust//                 }
1033 //ust//         }
1034 //ust// }
1035
1036 //ust//#ifdef CONFIG_MODULES
1037
1038 /**
1039  * marker_get_iter_range - Get a next marker iterator given a range.
1040  * @marker: current markers (in), next marker (out)
1041  * @begin: beginning of the range
1042  * @end: end of the range
1043  *
1044  * Returns whether a next marker has been found (1) or not (0).
1045  * Will return the first marker in the range if the input marker is NULL.
1046  */
1047 int marker_get_iter_range(struct marker **marker, struct marker *begin,
1048         struct marker *end)
1049 {
1050         if (!*marker && begin != end) {
1051                 *marker = begin;
1052                 return 1;
1053         }
1054         if (*marker >= begin && *marker < end)
1055                 return 1;
1056         return 0;
1057 }
1058 //ust// EXPORT_SYMBOL_GPL(marker_get_iter_range);
1059
1060 static void marker_get_iter(struct marker_iter *iter)
1061 {
1062         int found = 0;
1063
1064         /* Core kernel markers */
1065         if (!iter->lib) {
1066                 /* ust FIXME: how come we cannot disable the following line? we shouldn't need core stuff */
1067                 found = marker_get_iter_range(&iter->marker,
1068                                 __start___markers, __stop___markers);
1069                 if (found)
1070                         goto end;
1071         }
1072         /* Markers in modules. */
1073         found = lib_get_iter_markers(iter);
1074 end:
1075         if (!found)
1076                 marker_iter_reset(iter);
1077 }
1078
1079 void marker_iter_start(struct marker_iter *iter)
1080 {
1081         marker_get_iter(iter);
1082 }
1083 //ust// EXPORT_SYMBOL_GPL(marker_iter_start);
1084
1085 void marker_iter_next(struct marker_iter *iter)
1086 {
1087         iter->marker++;
1088         /*
1089          * iter->marker may be invalid because we blindly incremented it.
1090          * Make sure it is valid by marshalling on the markers, getting the
1091          * markers from following modules if necessary.
1092          */
1093         marker_get_iter(iter);
1094 }
1095 //ust// EXPORT_SYMBOL_GPL(marker_iter_next);
1096
1097 void marker_iter_stop(struct marker_iter *iter)
1098 {
1099 }
1100 //ust// EXPORT_SYMBOL_GPL(marker_iter_stop);
1101
1102 void marker_iter_reset(struct marker_iter *iter)
1103 {
1104         iter->lib = NULL;
1105         iter->marker = NULL;
1106 }
1107 //ust// EXPORT_SYMBOL_GPL(marker_iter_reset);
1108
1109 #ifdef CONFIG_MARKERS_USERSPACE
1110 /*
1111  * must be called with current->user_markers_mutex held
1112  */
1113 static void free_user_marker(char __user *state, struct hlist_head *head)
1114 {
1115         struct user_marker *umark;
1116         struct hlist_node *pos, *n;
1117
1118         hlist_for_each_entry_safe(umark, pos, n, head, hlist) {
1119                 if (umark->state == state) {
1120                         hlist_del(&umark->hlist);
1121                         kfree(umark);
1122                 }
1123         }
1124 }
1125
1126 //ust// asmlinkage long sys_marker(char __user *name, char __user *format,
1127 //ust//                 char __user *state, int reg)
1128 //ust// {
1129 //ust//         struct user_marker *umark;
1130 //ust//         long len;
1131 //ust//         struct marker_entry *entry;
1132 //ust//         int ret = 0;
1133 //ust//
1134 //ust//         printk(KERN_DEBUG "Program %s %s marker [%p, %p]\n",
1135 //ust//                 current->comm, reg ? "registers" : "unregisters",
1136 //ust//                 name, state);
1137 //ust//         if (reg) {
1138 //ust//                 umark = kmalloc(sizeof(struct user_marker), GFP_KERNEL);
1139 //ust//                 umark->name[MAX_USER_MARKER_NAME_LEN - 1] = '\0';
1140 //ust//                 umark->format[MAX_USER_MARKER_FORMAT_LEN - 1] = '\0';
1141 //ust//                 umark->state = state;
1142 //ust//                 len = strncpy_from_user(umark->name, name,
1143 //ust//                         MAX_USER_MARKER_NAME_LEN - 1);
1144 //ust//                 if (len < 0) {
1145 //ust//                         ret = -EFAULT;
1146 //ust//                         goto error;
1147 //ust//                 }
1148 //ust//                 len = strncpy_from_user(umark->format, format,
1149 //ust//                         MAX_USER_MARKER_FORMAT_LEN - 1);
1150 //ust//                 if (len < 0) {
1151 //ust//                         ret = -EFAULT;
1152 //ust//                         goto error;
1153 //ust//                 }
1154 //ust//                 printk(KERN_DEBUG "Marker name : %s, format : %s", umark->name,
1155 //ust//                         umark->format);
1156 //ust//                 mutex_lock(&markers_mutex);
1157 //ust//                 entry = get_marker("userspace", umark->name);
1158 //ust//                 if (entry) {
1159 //ust//                         if (entry->format &&
1160 //ust//                                 strcmp(entry->format, umark->format) != 0) {
1161 //ust//                                 printk(" error, wrong format in process %s",
1162 //ust//                                         current->comm);
1163 //ust//                                 ret = -EPERM;
1164 //ust//                                 goto error_unlock;
1165 //ust//                         }
1166 //ust//                         printk(" %s", !!entry->refcount
1167 //ust//                                         ? "enabled" : "disabled");
1168 //ust//                         if (put_user(!!entry->refcount, state)) {
1169 //ust//                                 ret = -EFAULT;
1170 //ust//                                 goto error_unlock;
1171 //ust//                         }
1172 //ust//                         printk("\n");
1173 //ust//                 } else {
1174 //ust//                         printk(" disabled\n");
1175 //ust//                         if (put_user(0, umark->state)) {
1176 //ust//                                 printk(KERN_WARNING
1177 //ust//                                         "Marker in %s caused a fault\n",
1178 //ust//                                         current->comm);
1179 //ust//                                 goto error_unlock;
1180 //ust//                         }
1181 //ust//                 }
1182 //ust//                 mutex_lock(&current->group_leader->user_markers_mutex);
1183 //ust//                 hlist_add_head(&umark->hlist,
1184 //ust//                         &current->group_leader->user_markers);
1185 //ust//                 current->group_leader->user_markers_sequence++;
1186 //ust//                 mutex_unlock(&current->group_leader->user_markers_mutex);
1187 //ust//                 mutex_unlock(&markers_mutex);
1188 //ust//         } else {
1189 //ust//                 mutex_lock(&current->group_leader->user_markers_mutex);
1190 //ust//                 free_user_marker(state,
1191 //ust//                         &current->group_leader->user_markers);
1192 //ust//                 current->group_leader->user_markers_sequence++;
1193 //ust//                 mutex_unlock(&current->group_leader->user_markers_mutex);
1194 //ust//         }
1195 //ust//         goto end;
1196 //ust// error_unlock:
1197 //ust//         mutex_unlock(&markers_mutex);
1198 //ust// error:
1199 //ust//         kfree(umark);
1200 //ust// end:
1201 //ust//         return ret;
1202 //ust// }
1203 //ust//
1204 //ust// /*
1205 //ust//  * Types :
1206 //ust//  * string : 0
1207 //ust//  */
1208 //ust// asmlinkage long sys_trace(int type, uint16_t id,
1209 //ust//                 char __user *ubuf)
1210 //ust// {
1211 //ust//         long ret = -EPERM;
1212 //ust//         char *page;
1213 //ust//         int len;
1214 //ust//
1215 //ust//         switch (type) {
1216 //ust//         case 0: /* String */
1217 //ust//                 ret = -ENOMEM;
1218 //ust//                 page = (char *)__get_free_page(GFP_TEMPORARY);
1219 //ust//                 if (!page)
1220 //ust//                         goto string_out;
1221 //ust//                 len = strncpy_from_user(page, ubuf, PAGE_SIZE);
1222 //ust//                 if (len < 0) {
1223 //ust//                         ret = -EFAULT;
1224 //ust//                         goto string_err;
1225 //ust//                 }
1226 //ust//                 trace_mark(userspace, string, "string %s", page);
1227 //ust// string_err:
1228 //ust//                 free_page((unsigned long) page);
1229 //ust// string_out:
1230 //ust//                 break;
1231 //ust//         default:
1232 //ust//                 break;
1233 //ust//         }
1234 //ust//         return ret;
1235 //ust// }
1236
1237 //ust// static void marker_update_processes(void)
1238 //ust// {
1239 //ust//         struct task_struct *g, *t;
1240 //ust//
1241 //ust//         /*
1242 //ust//          * markers_mutex is taken to protect the p->user_markers read.
1243 //ust//          */
1244 //ust//         mutex_lock(&markers_mutex);
1245 //ust//         read_lock(&tasklist_lock);
1246 //ust//         for_each_process(g) {
1247 //ust//                 WARN_ON(!thread_group_leader(g));
1248 //ust//                 if (hlist_empty(&g->user_markers))
1249 //ust//                         continue;
1250 //ust//                 if (strcmp(g->comm, "testprog") == 0)
1251 //ust//                         printk(KERN_DEBUG "set update pending for testprog\n");
1252 //ust//                 t = g;
1253 //ust//                 do {
1254 //ust//                         /* TODO : implement this thread flag in each arch. */
1255 //ust//                         set_tsk_thread_flag(t, TIF_MARKER_PENDING);
1256 //ust//                 } while ((t = next_thread(t)) != g);
1257 //ust//         }
1258 //ust//         read_unlock(&tasklist_lock);
1259 //ust//         mutex_unlock(&markers_mutex);
1260 //ust// }
1261
1262 /*
1263  * Update current process.
1264  * Note that we have to wait a whole scheduler period before we are sure that
1265  * every running userspace threads have their markers updated.
1266  * (synchronize_sched() can be used to insure this).
1267  */
1268 void marker_update_process(void)
1269 {
1270         struct user_marker *umark;
1271         struct hlist_node *pos;
1272         struct marker_entry *entry;
1273
1274         mutex_lock(&markers_mutex);
1275         mutex_lock(&current->group_leader->user_markers_mutex);
1276         if (strcmp(current->comm, "testprog") == 0)
1277                 printk(KERN_DEBUG "do update pending for testprog\n");
1278         hlist_for_each_entry(umark, pos,
1279                         &current->group_leader->user_markers, hlist) {
1280                 printk(KERN_DEBUG "Updating marker %s in %s\n",
1281                         umark->name, current->comm);
1282                 entry = get_marker("userspace", umark->name);
1283                 if (entry) {
1284                         if (entry->format &&
1285                                 strcmp(entry->format, umark->format) != 0) {
1286                                 printk(KERN_WARNING
1287                                         " error, wrong format in process %s\n",
1288                                         current->comm);
1289                                 break;
1290                         }
1291                         if (put_user(!!entry->refcount, umark->state)) {
1292                                 printk(KERN_WARNING
1293                                         "Marker in %s caused a fault\n",
1294                                         current->comm);
1295                                 break;
1296                         }
1297                 } else {
1298                         if (put_user(0, umark->state)) {
1299                                 printk(KERN_WARNING
1300                                         "Marker in %s caused a fault\n",
1301                                         current->comm);
1302                                 break;
1303                         }
1304                 }
1305         }
1306         clear_thread_flag(TIF_MARKER_PENDING);
1307         mutex_unlock(&current->group_leader->user_markers_mutex);
1308         mutex_unlock(&markers_mutex);
1309 }
1310
1311 /*
1312  * Called at process exit and upon do_execve().
1313  * We assume that when the leader exits, no more references can be done to the
1314  * leader structure by the other threads.
1315  */
1316 void exit_user_markers(struct task_struct *p)
1317 {
1318         struct user_marker *umark;
1319         struct hlist_node *pos, *n;
1320
1321         if (thread_group_leader(p)) {
1322                 mutex_lock(&markers_mutex);
1323                 mutex_lock(&p->user_markers_mutex);
1324                 hlist_for_each_entry_safe(umark, pos, n, &p->user_markers,
1325                         hlist)
1326                     kfree(umark);
1327                 INIT_HLIST_HEAD(&p->user_markers);
1328                 p->user_markers_sequence++;
1329                 mutex_unlock(&p->user_markers_mutex);
1330                 mutex_unlock(&markers_mutex);
1331         }
1332 }
1333
1334 int is_marker_enabled(const char *channel, const char *name)
1335 {
1336         struct marker_entry *entry;
1337
1338         mutex_lock(&markers_mutex);
1339         entry = get_marker(channel, name);
1340         mutex_unlock(&markers_mutex);
1341
1342         return entry && !!entry->refcount;
1343 }
1344 //ust// #endif
1345
1346 int marker_module_notify(struct notifier_block *self,
1347                          unsigned long val, void *data)
1348 {
1349         struct module *mod = data;
1350
1351         switch (val) {
1352         case MODULE_STATE_COMING:
1353                 marker_update_probe_range(mod->markers,
1354                         mod->markers + mod->num_markers);
1355                 break;
1356         case MODULE_STATE_GOING:
1357                 marker_update_probe_range(mod->markers,
1358                         mod->markers + mod->num_markers);
1359                 break;
1360         }
1361         return 0;
1362 }
1363
1364 struct notifier_block marker_module_nb = {
1365         .notifier_call = marker_module_notify,
1366         .priority = 0,
1367 };
1368
1369 //ust// static int init_markers(void)
1370 //ust// {
1371 //ust//         return register_module_notifier(&marker_module_nb);
1372 //ust// }
1373 //ust// __initcall(init_markers);
1374 /* TODO: call marker_module_nb() when a library is linked at runtime (dlopen)? */
1375
1376 #endif /* CONFIG_MODULES */
1377
1378 void ltt_dump_marker_state(struct ltt_trace_struct *trace)
1379 {
1380         struct marker_iter iter;
1381         struct ltt_probe_private_data call_data;
1382         const char *channel;
1383
1384         call_data.trace = trace;
1385         call_data.serializer = NULL;
1386
1387         marker_iter_reset(&iter);
1388         marker_iter_start(&iter);
1389         for (; iter.marker != NULL; marker_iter_next(&iter)) {
1390                 if (!_imv_read(iter.marker->state))
1391                         continue;
1392                 channel = ltt_channels_get_name_from_index(
1393                                 iter.marker->channel_id);
1394                 __trace_mark(0, metadata, core_marker_id,
1395                         &call_data,
1396                         "channel %s name %s event_id %hu "
1397                         "int #1u%zu long #1u%zu pointer #1u%zu "
1398                         "size_t #1u%zu alignment #1u%u",
1399                         channel,
1400                         iter.marker->name,
1401                         iter.marker->event_id,
1402                         sizeof(int), sizeof(long),
1403                         sizeof(void *), sizeof(size_t),
1404                         ltt_get_alignment());
1405                 if (iter.marker->format)
1406                         __trace_mark(0, metadata,
1407                                 core_marker_format,
1408                                 &call_data,
1409                                 "channel %s name %s format %s",
1410                                 channel,
1411                                 iter.marker->name,
1412                                 iter.marker->format);
1413         }
1414         marker_iter_stop(&iter);
1415 }
1416 //ust// EXPORT_SYMBOL_GPL(ltt_dump_marker_state);
1417
1418
1419 static LIST_HEAD(libs);
1420
1421 /*
1422  * Returns 0 if current not found.
1423  * Returns 1 if current found.
1424  */
1425 int lib_get_iter_markers(struct marker_iter *iter)
1426 {
1427         struct lib *iter_lib;
1428         int found = 0;
1429
1430 //ust// mutex_lock(&module_mutex);
1431         list_for_each_entry(iter_lib, &libs, list) {
1432                 if (iter_lib < iter->lib)
1433                         continue;
1434                 else if (iter_lib > iter->lib)
1435                         iter->marker = NULL;
1436                 found = marker_get_iter_range(&iter->marker,
1437                         iter_lib->markers_start,
1438                         iter_lib->markers_start + iter_lib->markers_count);
1439                 if (found) {
1440                         iter->lib = iter_lib;
1441                         break;
1442                 }
1443         }
1444 //ust// mutex_unlock(&module_mutex);
1445         return found;
1446 }
1447
1448 void lib_update_markers(void)
1449 {
1450         struct lib *lib;
1451
1452 //ust// mutex_lock(&module_mutex);
1453         list_for_each_entry(lib, &libs, list)
1454                 marker_update_probe_range(lib->markers_start,
1455                                 lib->markers_start + lib->markers_count);
1456 //ust// mutex_unlock(&module_mutex);
1457 }
1458
1459 int marker_register_lib(struct marker *markers_start, int markers_count)
1460 {
1461         struct lib *pl;
1462
1463         pl = (struct lib *) malloc(sizeof(struct lib));
1464
1465         pl->markers_start = markers_start;
1466         pl->markers_count = markers_count;
1467
1468         list_add(&pl->list, &libs);
1469
1470         printf("just registered a markers section from %p and having %d markers\n", markers_start, markers_count);
1471
1472         return 0;
1473 }
1474
1475 static void __attribute__((constructor)) __markers__init(void)
1476 {
1477         marker_register_lib(__start___markers, (((long)__stop___markers)-((long)__start___markers))/sizeof(struct marker));
1478         printf("markers_start: %p, markers_stop: %p\n", __start___markers, __stop___markers);
1479 }