libust/serialize.c

   1 /*
   2  * LTTng serializing code.
   3  *
   4  * Copyright Mathieu Desnoyers, March 2007.
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2.1 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
  19  *
  20  *
  21  * See this discussion about weirdness about passing va_list and then va_list to
  22  * functions. (related to array argument passing). va_list seems to be
  23  * implemented as an array on x86_64, but not on i386... This is why we pass a
  24  * va_list * to ltt_vtrace.
  25  */
  26
  27 #define _GNU_SOURCE
  28 #include <unistd.h>
  29 #include <sys/syscall.h>
  30 #include <stdarg.h>
  31 #include <string.h>
  32 #include <stdint.h>
  33 #include <stdio.h>
  34
  35 #include <ust/kernelcompat.h>
  36 #define _LGPL_SOURCE
  37 #include <urcu-bp.h>
  38 #include <urcu/rculist.h>
  39
  40 #include "buffers.h"
  41 #include "tracer.h"
  42 //#include "list.h"
  43 #include "usterr.h"
  44
  45 enum ltt_type {
  46         LTT_TYPE_SIGNED_INT,
  47         LTT_TYPE_UNSIGNED_INT,
  48         LTT_TYPE_STRING,
  49         LTT_TYPE_NONE,
  50 };
  51
  52 static int ust_get_cpu(void)
  53 {
  54         return sched_getcpu();
  55 }
  56
  57 #define LTT_ATTRIBUTE_NETWORK_BYTE_ORDER (1<<1)
  58
  59 /*
  60  * Inspired from vsnprintf
  61  *
  62  * The serialization format string supports the basic printf format strings.
  63  * In addition, it defines new formats that can be used to serialize more
  64  * complex/non portable data structures.
  65  *
  66  * Typical use:
  67  *
  68  * field_name %ctype
  69  * field_name #tracetype %ctype
  70  * field_name #tracetype %ctype1 %ctype2 ...
  71  *
  72  * A conversion is performed between format string types supported by GCC and
  73  * the trace type requested. GCC type is used to perform type checking on format
  74  * strings. Trace type is used to specify the exact binary representation
  75  * in the trace. A mapping is done between one or more GCC types to one trace
  76  * type. Sign extension, if required by the conversion, is performed following
  77  * the trace type.
  78  *
  79  * If a gcc format is not declared with a trace format, the gcc format is
  80  * also used as binary representation in the trace.
  81  *
  82  * Strings are supported with %s.
  83  * A single tracetype (sequence) can take multiple c types as parameter.
  84  *
  85  * c types:
  86  *
  87  * see printf(3).
  88  *
  89  * Note: to write a uint32_t in a trace, the following expression is recommended
  90  * si it can be portable:
  91  *
  92  * ("#4u%lu", (unsigned long)var)
  93  *
  94  * trace types:
  95  *
  96  * Serialization specific formats :
  97  *
  98  * Fixed size integers
  99  * #1u     writes uint8_t
 100  * #2u     writes uint16_t
 101  * #4u     writes uint32_t
 102  * #8u     writes uint64_t
 103  * #1d     writes int8_t
 104  * #2d     writes int16_t
 105  * #4d     writes int32_t
 106  * #8d     writes int64_t
 107  * i.e.:
 108  * #1u%lu #2u%lu #4d%lu #8d%lu #llu%hu #d%lu
 109  *
 110  * * Attributes:
 111  *
 112  * n:  (for network byte order)
 113  * #ntracetype%ctype
 114  *            is written in the trace in network byte order.
 115  *
 116  * i.e.: #bn4u%lu, #n%lu, #b%u
 117  *
 118  * TODO (eventually)
 119  * Variable length sequence
 120  * #a #tracetype1 #tracetype2 %array_ptr %elem_size %num_elems
 121  *            In the trace:
 122  *            #a specifies that this is a sequence
 123  *            #tracetype1 is the type of elements in the sequence
 124  *            #tracetype2 is the type of the element count
 125  *            GCC input:
 126  *            array_ptr is a pointer to an array that contains members of size
 127  *            elem_size.
 128  *            num_elems is the number of elements in the array.
 129  * i.e.: #a #lu #lu %p %lu %u
 130  *
 131  * Callback
 132  * #k         callback (taken from the probe data)
 133  *            The following % arguments are exepected by the callback
 134  *
 135  * i.e.: #a #lu #lu #k %p
 136  *
 137  * Note: No conversion is done from floats to integers, nor from integers to
 138  * floats between c types and trace types. float conversion from double to float
 139  * or from float to double is also not supported.
 140  *
 141  * REMOVE
 142  * %*b     expects sizeof(data), data
 143  *         where sizeof(data) is 1, 2, 4 or 8
 144  *
 145  * Fixed length struct, union or array.
 146  * FIXME: unable to extract those sizes statically.
 147  * %*r     expects sizeof(*ptr), ptr
 148  * %*.*r   expects sizeof(*ptr), __alignof__(*ptr), ptr
 149  * struct and unions removed.
 150  * Fixed length array:
 151  * [%p]#a[len #tracetype]
 152  * i.e.: [%p]#a[12 #lu]
 153  *
 154  * Variable length sequence
 155  * %*.*:*v expects sizeof(*ptr), __alignof__(*ptr), elem_num, ptr
 156  *         where elem_num is the number of elements in the sequence
 157  */
 158 static inline const char *parse_trace_type(const char *fmt,
 159                 char *trace_size, enum ltt_type *trace_type,
 160                 unsigned long *attributes)
 161 {
 162         int qualifier;          /* 'h', 'l', or 'L' for integer fields */
 163                                 /* 'z' support added 23/7/1999 S.H.    */
 164                                 /* 'z' changed to 'Z' --davidm 1/25/99 */
 165                                 /* 't' added for ptrdiff_t */
 166
 167         /* parse attributes. */
 168 repeat:
 169         switch (*fmt) {
 170         case 'n':
 171                 *attributes |= LTT_ATTRIBUTE_NETWORK_BYTE_ORDER;
 172                 ++fmt;
 173                 goto repeat;
 174         }
 175
 176         /* get the conversion qualifier */
 177         qualifier = -1;
 178         if (*fmt == 'h' || *fmt == 'l' || *fmt == 'L' ||
 179             *fmt == 'Z' || *fmt == 'z' || *fmt == 't' ||
 180             *fmt == 'S' || *fmt == '1' || *fmt == '2' ||
 181             *fmt == '4' || *fmt == 8) {
 182                 qualifier = *fmt;
 183                 ++fmt;
 184                 if (qualifier == 'l' && *fmt == 'l') {
 185                         qualifier = 'L';
 186                         ++fmt;
 187                 }
 188         }
 189
 190         switch (*fmt) {
 191         case 'c':
 192                 *trace_type = LTT_TYPE_UNSIGNED_INT;
 193                 *trace_size = sizeof(unsigned char);
 194                 goto parse_end;
 195         case 's':
 196                 *trace_type = LTT_TYPE_STRING;
 197                 goto parse_end;
 198         case 'p':
 199                 *trace_type = LTT_TYPE_UNSIGNED_INT;
 200                 *trace_size = sizeof(void *);
 201                 goto parse_end;
 202         case 'd':
 203         case 'i':
 204                 *trace_type = LTT_TYPE_SIGNED_INT;
 205                 break;
 206         case 'o':
 207         case 'u':
 208         case 'x':
 209         case 'X':
 210                 *trace_type = LTT_TYPE_UNSIGNED_INT;
 211                 break;
 212         default:
 213                 if (!*fmt)
 214                         --fmt;
 215                 goto parse_end;
 216         }
 217         switch (qualifier) {
 218         case 'L':
 219                 *trace_size = sizeof(long long);
 220                 break;
 221         case 'l':
 222                 *trace_size = sizeof(long);
 223                 break;
 224         case 'Z':
 225         case 'z':
 226                 *trace_size = sizeof(size_t);
 227                 break;
 228 //ust// case 't':
 229 //ust//         *trace_size = sizeof(ptrdiff_t);
 230 //ust//         break;
 231         case 'h':
 232                 *trace_size = sizeof(short);
 233                 break;
 234         case '1':
 235                 *trace_size = sizeof(uint8_t);
 236                 break;
 237         case '2':
 238                 *trace_size = sizeof(uint16_t);
 239                 break;
 240         case '4':
 241                 *trace_size = sizeof(uint32_t);
 242                 break;
 243         case '8':
 244                 *trace_size = sizeof(uint64_t);
 245                 break;
 246         default:
 247                 *trace_size = sizeof(int);
 248         }
 249
 250 parse_end:
 251         return fmt;
 252 }
 253
 254 /*
 255  * Restrictions:
 256  * Field width and precision are *not* supported.
 257  * %n not supported.
 258  */
 259 static inline const char *parse_c_type(const char *fmt,
 260                 char *c_size, enum ltt_type *c_type)
 261 {
 262         int qualifier;          /* 'h', 'l', or 'L' for integer fields */
 263                                 /* 'z' support added 23/7/1999 S.H.    */
 264                                 /* 'z' changed to 'Z' --davidm 1/25/99 */
 265                                 /* 't' added for ptrdiff_t */
 266
 267         /* process flags : ignore standard print formats for now. */
 268 repeat:
 269         switch (*fmt) {
 270         case '-':
 271         case '+':
 272         case ' ':
 273         case '#':
 274         case '0':
 275                 ++fmt;
 276                 goto repeat;
 277         }
 278
 279         /* get the conversion qualifier */
 280         qualifier = -1;
 281         if (*fmt == 'h' || *fmt == 'l' || *fmt == 'L' ||
 282             *fmt == 'Z' || *fmt == 'z' || *fmt == 't' ||
 283             *fmt == 'S') {
 284                 qualifier = *fmt;
 285                 ++fmt;
 286                 if (qualifier == 'l' && *fmt == 'l') {
 287                         qualifier = 'L';
 288                         ++fmt;
 289                 }
 290         }
 291
 292         switch (*fmt) {
 293         case 'c':
 294                 *c_type = LTT_TYPE_UNSIGNED_INT;
 295                 *c_size = sizeof(unsigned char);
 296                 goto parse_end;
 297         case 's':
 298                 *c_type = LTT_TYPE_STRING;
 299                 goto parse_end;
 300         case 'p':
 301                 *c_type = LTT_TYPE_UNSIGNED_INT;
 302                 *c_size = sizeof(void *);
 303                 goto parse_end;
 304         case 'd':
 305         case 'i':
 306                 *c_type = LTT_TYPE_SIGNED_INT;
 307                 break;
 308         case 'o':
 309         case 'u':
 310         case 'x':
 311         case 'X':
 312                 *c_type = LTT_TYPE_UNSIGNED_INT;
 313                 break;
 314         default:
 315                 if (!*fmt)
 316                         --fmt;
 317                 goto parse_end;
 318         }
 319         switch (qualifier) {
 320         case 'L':
 321                 *c_size = sizeof(long long);
 322                 break;
 323         case 'l':
 324                 *c_size = sizeof(long);
 325                 break;
 326         case 'Z':
 327         case 'z':
 328                 *c_size = sizeof(size_t);
 329                 break;
 330 //ust// case 't':
 331 //ust//         *c_size = sizeof(ptrdiff_t);
 332 //ust//         break;
 333         case 'h':
 334                 *c_size = sizeof(short);
 335                 break;
 336         default:
 337                 *c_size = sizeof(int);
 338         }
 339
 340 parse_end:
 341         return fmt;
 342 }
 343
 344 static inline size_t serialize_trace_data(struct ust_buffer *buf,
 345                 size_t buf_offset,
 346                 char trace_size, enum ltt_type trace_type,
 347                 char c_size, enum ltt_type c_type,
 348                 int *largest_align, va_list *args)
 349 {
 350         union {
 351                 unsigned long v_ulong;
 352                 uint64_t v_uint64;
 353                 struct {
 354                         const char *s;
 355                         size_t len;
 356                 } v_string;
 357         } tmp;
 358
 359         /*
 360          * Be careful about sign extension here.
 361          * Sign extension is done with the destination (trace) type.
 362          */
 363         switch (trace_type) {
 364         case LTT_TYPE_SIGNED_INT:
 365                 switch (c_size) {
 366                 case 1:
 367                         tmp.v_ulong = (long)(int8_t)va_arg(*args, int);
 368                         break;
 369                 case 2:
 370                         tmp.v_ulong = (long)(int16_t)va_arg(*args, int);
 371                         break;
 372                 case 4:
 373                         tmp.v_ulong = (long)(int32_t)va_arg(*args, int);
 374                         break;
 375                 case 8:
 376                         tmp.v_uint64 = va_arg(*args, int64_t);
 377                         break;
 378                 default:
 379                         BUG();
 380                 }
 381                 break;
 382         case LTT_TYPE_UNSIGNED_INT:
 383                 switch (c_size) {
 384                 case 1:
 385                         tmp.v_ulong = (unsigned long)(uint8_t)
 386                                         va_arg(*args, unsigned int);
 387                         break;
 388                 case 2:
 389                         tmp.v_ulong = (unsigned long)(uint16_t)
 390                                         va_arg(*args, unsigned int);
 391                         break;
 392                 case 4:
 393                         tmp.v_ulong = (unsigned long)(uint32_t)
 394                                         va_arg(*args, unsigned int);
 395                         break;
 396                 case 8:
 397                         tmp.v_uint64 = va_arg(*args, uint64_t);
 398                         break;
 399                 default:
 400                         BUG();
 401                 }
 402                 break;
 403         case LTT_TYPE_STRING:
 404                 tmp.v_string.s = va_arg(*args, const char *);
 405                 if ((unsigned long)tmp.v_string.s < PAGE_SIZE)
 406                         tmp.v_string.s = "<NULL>";
 407                 tmp.v_string.len = strlen(tmp.v_string.s)+1;
 408                 if (buf)
 409                         ust_buffers_write(buf, buf_offset, tmp.v_string.s,
 410                                 tmp.v_string.len);
 411                 buf_offset += tmp.v_string.len;
 412                 goto copydone;
 413         default:
 414                 BUG();
 415         }
 416
 417         /*
 418          * If trace_size is lower or equal to 4 bytes, there is no sign
 419          * extension to do because we are already encoded in a long. Therefore,
 420          * we can combine signed and unsigned ops. 4 bytes float also works
 421          * with this, because we do a simple copy of 4 bytes into 4 bytes
 422          * without manipulation (and we do not support conversion from integers
 423          * to floats).
 424          * It is also the case if c_size is 8 bytes, which is the largest
 425          * possible integer.
 426          */
 427         if (ltt_get_alignment()) {
 428                 buf_offset += ltt_align(buf_offset, trace_size);
 429                 if (largest_align)
 430                         *largest_align = max_t(int, *largest_align, trace_size);
 431         }
 432         if (trace_size <= 4 || c_size == 8) {
 433                 if (buf) {
 434                         switch (trace_size) {
 435                         case 1:
 436                                 if (c_size == 8)
 437                                         ust_buffers_write(buf, buf_offset,
 438                                         (uint8_t[]){ (uint8_t)tmp.v_uint64 },
 439                                         sizeof(uint8_t));
 440                                 else
 441                                         ust_buffers_write(buf, buf_offset,
 442                                         (uint8_t[]){ (uint8_t)tmp.v_ulong },
 443                                         sizeof(uint8_t));
 444                                 break;
 445                         case 2:
 446                                 if (c_size == 8)
 447                                         ust_buffers_write(buf, buf_offset,
 448                                         (uint16_t[]){ (uint16_t)tmp.v_uint64 },
 449                                         sizeof(uint16_t));
 450                                 else
 451                                         ust_buffers_write(buf, buf_offset,
 452                                         (uint16_t[]){ (uint16_t)tmp.v_ulong },
 453                                         sizeof(uint16_t));
 454                                 break;
 455                         case 4:
 456                                 if (c_size == 8)
 457                                         ust_buffers_write(buf, buf_offset,
 458                                         (uint32_t[]){ (uint32_t)tmp.v_uint64 },
 459                                         sizeof(uint32_t));
 460                                 else
 461                                         ust_buffers_write(buf, buf_offset,
 462                                         (uint32_t[]){ (uint32_t)tmp.v_ulong },
 463                                         sizeof(uint32_t));
 464                                 break;
 465                         case 8:
 466                                 /*
 467                                  * c_size cannot be other than 8 here because
 468                                  * trace_size > 4.
 469                                  */
 470                                 ust_buffers_write(buf, buf_offset,
 471                                 (uint64_t[]){ (uint64_t)tmp.v_uint64 },
 472                                 sizeof(uint64_t));
 473                                 break;
 474                         default:
 475                                 BUG();
 476                         }
 477                 }
 478                 buf_offset += trace_size;
 479                 goto copydone;
 480         } else {
 481                 /*
 482                  * Perform sign extension.
 483                  */
 484                 if (buf) {
 485                         switch (trace_type) {
 486                         case LTT_TYPE_SIGNED_INT:
 487                                 ust_buffers_write(buf, buf_offset,
 488                                         (int64_t[]){ (int64_t)tmp.v_ulong },
 489                                         sizeof(int64_t));
 490                                 break;
 491                         case LTT_TYPE_UNSIGNED_INT:
 492                                 ust_buffers_write(buf, buf_offset,
 493                                         (uint64_t[]){ (uint64_t)tmp.v_ulong },
 494                                         sizeof(uint64_t));
 495                                 break;
 496                         default:
 497                                 BUG();
 498                         }
 499                 }
 500                 buf_offset += trace_size;
 501                 goto copydone;
 502         }
 503
 504 copydone:
 505         return buf_offset;
 506 }
 507
 508 static notrace void skip_space(const char **ps)
 509 {
 510         while(**ps == ' ')
 511                 (*ps)++;
 512 }
 513
 514 static notrace void copy_token(char **out, const char **in)
 515 {
 516         while(**in != ' ' && **in != '\0') {
 517                 **out = **in;
 518                 (*out)++;
 519                 (*in)++;
 520         }
 521 }
 522
 523 /* serialize_to_text
 524  *
 525  * Given a format string and a va_list of arguments, convert them to a
 526  * human-readable string.
 527  *
 528  * @outbuf: the buffer to output the string to
 529  * @bufsize: the max size that can be used in outbuf
 530  * @fmt: the marker format string
 531  * @ap: a va_list that contains the arguments corresponding to fmt
 532  *
 533  * Return value: the number of chars that have been put in outbuf, excluding
 534  * the final \0, or, if the buffer was too small, the number of chars that
 535  * would have been written in outbuf if it had been large enough.
 536  *
 537  * outbuf may be NULL. The return value may then be used be allocate an
 538  * appropriate outbuf.
 539  *
 540  */
 541
 542 notrace
 543 int serialize_to_text(char *outbuf, int bufsize, const char *fmt, va_list ap)
 544 {
 545         int fmt_len = strlen(fmt);
 546         char *new_fmt = alloca(fmt_len + 1);
 547         const char *orig_fmt_p = fmt;
 548         char *new_fmt_p = new_fmt;
 549         char false_buf;
 550         int result;
 551         enum { none, cfmt, tracefmt, argname } prev_token = none;
 552
 553         while(*orig_fmt_p != '\0') {
 554                 if(*orig_fmt_p == '%') {
 555                         prev_token = cfmt;
 556                         copy_token(&new_fmt_p, &orig_fmt_p);
 557                 }
 558                 else if(*orig_fmt_p == '#') {
 559                         prev_token = tracefmt;
 560                         do {
 561                                 orig_fmt_p++;
 562                         } while(*orig_fmt_p != ' ' && *orig_fmt_p != '\0');
 563                 }
 564                 else if(*orig_fmt_p == ' ') {
 565                         if(prev_token == argname) {
 566                                 *new_fmt_p = '=';
 567                                 new_fmt_p++;
 568                         }
 569                         else if(prev_token == cfmt) {
 570                                 *new_fmt_p = ' ';
 571                                 new_fmt_p++;
 572                         }
 573
 574                         skip_space(&orig_fmt_p);
 575                 }
 576                 else {
 577                         prev_token = argname;
 578                         copy_token(&new_fmt_p, &orig_fmt_p);
 579                 }
 580         }
 581
 582         *new_fmt_p = '\0';
 583
 584         if(outbuf == NULL) {
 585                 /* use this false_buffer for compatibility with pre-C99 */
 586                 outbuf = &false_buf;
 587                 bufsize = 1;
 588         }
 589         result = vsnprintf(outbuf, bufsize, new_fmt, ap);
 590
 591         return result;
 592 }
 593
 594 notrace size_t ltt_serialize_data(struct ust_buffer *buf, size_t buf_offset,
 595                         struct ltt_serialize_closure *closure,
 596                         void *serialize_private, int *largest_align,
 597                         const char *fmt, va_list *args)
 598 {
 599         char trace_size = 0, c_size = 0;        /*
 600                                                  * 0 (unset), 1, 2, 4, 8 bytes.
 601                                                  */
 602         enum ltt_type trace_type = LTT_TYPE_NONE, c_type = LTT_TYPE_NONE;
 603         unsigned long attributes = 0;
 604
 605         for (; *fmt ; ++fmt) {
 606                 switch (*fmt) {
 607                 case '#':
 608                         /* tracetypes (#) */
 609                         ++fmt;                  /* skip first '#' */
 610                         if (*fmt == '#')        /* Escaped ## */
 611                                 break;
 612                         attributes = 0;
 613                         fmt = parse_trace_type(fmt, &trace_size, &trace_type,
 614                                 &attributes);
 615                         break;
 616                 case '%':
 617                         /* c types (%) */
 618                         ++fmt;                  /* skip first '%' */
 619                         if (*fmt == '%')        /* Escaped %% */
 620                                 break;
 621                         fmt = parse_c_type(fmt, &c_size, &c_type);
 622                         /*
 623                          * Output c types if no trace types has been
 624                          * specified.
 625                          */
 626                         if (!trace_size)
 627                                 trace_size = c_size;
 628                         if (trace_type == LTT_TYPE_NONE)
 629                                 trace_type = c_type;
 630                         if (c_type == LTT_TYPE_STRING)
 631                                 trace_type = LTT_TYPE_STRING;
 632                         /* perform trace write */
 633                         buf_offset = serialize_trace_data(buf,
 634                                                 buf_offset, trace_size,
 635                                                 trace_type, c_size, c_type,
 636                                                 largest_align, args);
 637                         trace_size = 0;
 638                         c_size = 0;
 639                         trace_type = LTT_TYPE_NONE;
 640                         c_size = LTT_TYPE_NONE;
 641                         attributes = 0;
 642                         break;
 643                         /* default is to skip the text, doing nothing */
 644                 }
 645         }
 646         return buf_offset;
 647 }
 648
 649 /*
 650  * Calculate data size
 651  * Assume that the padding for alignment starts at a sizeof(void *) address.
 652  */
 653 static notrace size_t ltt_get_data_size(struct ltt_serialize_closure *closure,
 654                                 void *serialize_private, int *largest_align,
 655                                 const char *fmt, va_list *args)
 656 {
 657         ltt_serialize_cb cb = closure->callbacks[0];
 658         closure->cb_idx = 0;
 659         return (size_t)cb(NULL, 0, closure, serialize_private,
 660                                 largest_align, fmt, args);
 661 }
 662
 663 static notrace
 664 void ltt_write_event_data(struct ust_buffer *buf, size_t buf_offset,
 665                                 struct ltt_serialize_closure *closure,
 666                                 void *serialize_private, int largest_align,
 667                                 const char *fmt, va_list *args)
 668 {
 669         ltt_serialize_cb cb = closure->callbacks[0];
 670         closure->cb_idx = 0;
 671         buf_offset += ltt_align(buf_offset, largest_align);
 672         cb(buf, buf_offset, closure, serialize_private, NULL, fmt, args);
 673 }
 674
 675
 676 notrace void ltt_vtrace(const struct marker *mdata, void *probe_data,
 677                         struct registers *regs, void *call_data,
 678                         const char *fmt, va_list *args)
 679 {
 680         int largest_align, ret;
 681         struct ltt_active_marker *pdata;
 682         uint16_t eID;
 683         size_t data_size, slot_size;
 684         unsigned int chan_index;
 685         struct ust_channel *channel;
 686         struct ltt_trace_struct *trace, *dest_trace = NULL;
 687         struct ust_buffer *buf;
 688         void *transport_data;
 689         u64 tsc;
 690         long buf_offset;
 691         va_list args_copy;
 692         struct ltt_serialize_closure closure;
 693         struct ltt_probe_private_data *private_data = call_data;
 694         void *serialize_private = NULL;
 695         int cpu;
 696         unsigned int rflags;
 697
 698         /*
 699          * This test is useful for quickly exiting static tracing when no trace
 700          * is active. We expect to have an active trace when we get here.
 701          */
 702         if (unlikely(ltt_traces.num_active_traces == 0))
 703                 return;
 704
 705         rcu_read_lock(); //ust// rcu_read_lock_sched_notrace();
 706 //ust// cpu = smp_processor_id();
 707         cpu = ust_get_cpu();
 708 //ust// __get_cpu_var(ltt_nesting)++;
 709         /* FIXME: should nesting be per-cpu? */
 710         ltt_nesting++;
 711
 712         pdata = (struct ltt_active_marker *)probe_data;
 713         eID = mdata->event_id;
 714         chan_index = mdata->channel_id;
 715         closure.callbacks = pdata->probe->callbacks;
 716
 717         if (unlikely(private_data)) {
 718                 dest_trace = private_data->trace;
 719                 if (private_data->serializer)
 720                         closure.callbacks = &private_data->serializer;
 721                 serialize_private = private_data->serialize_private;
 722         }
 723
 724         va_copy(args_copy, *args);
 725         /*
 726          * Assumes event payload to start on largest_align alignment.
 727          */
 728         largest_align = 1;      /* must be non-zero for ltt_align */
 729         data_size = ltt_get_data_size(&closure, serialize_private,
 730                                         &largest_align, fmt, &args_copy);
 731         largest_align = min_t(int, largest_align, sizeof(void *));
 732         va_end(args_copy);
 733
 734         /* Iterate on each trace */
 735         list_for_each_entry_rcu(trace, &ltt_traces.head, list) {
 736                 /*
 737                  * Expect the filter to filter out events. If we get here,
 738                  * we went through tracepoint activation as a first step.
 739                  */
 740                 if (unlikely(dest_trace && trace != dest_trace))
 741                         continue;
 742                 if (unlikely(!trace->active))
 743                         continue;
 744                 if (unlikely(!ltt_run_filter(trace, eID)))
 745                         continue;
 746 #ifdef CONFIG_LTT_DEBUG_EVENT_SIZE
 747                 rflags = LTT_RFLAG_ID_SIZE;
 748 #else
 749                 if (unlikely(eID >= LTT_FREE_EVENTS))
 750                         rflags = LTT_RFLAG_ID;
 751                 else
 752                         rflags = 0;
 753 #endif
 754                 /*
 755                  * Skip channels added after trace creation.
 756                  */
 757                 if (unlikely(chan_index >= trace->nr_channels))
 758                         continue;
 759                 channel = &trace->channels[chan_index];
 760                 if (!channel->active)
 761                         continue;
 762
 763                 /* If a new cpu was plugged since the trace was started, we did
 764                  * not add it to the trace, and therefore we write the event to
 765                  * cpu 0.
 766                  */
 767                 if(cpu >= channel->n_cpus) {
 768                         cpu = 0;
 769                 }
 770
 771                 /* reserve space : header and data */
 772                 ret = ltt_reserve_slot(trace, channel, &transport_data,
 773                                         data_size, &slot_size, &buf_offset,
 774                                         &tsc, &rflags,
 775                                         largest_align, cpu);
 776                 if (unlikely(ret < 0))
 777                         continue; /* buffer full */
 778
 779                 va_copy(args_copy, *args);
 780                 /* FIXME : could probably encapsulate transport better. */
 781 //ust//         buf = ((struct rchan *)channel->trans_channel_data)->buf[cpu];
 782                 buf = channel->buf[cpu];
 783                 /* Out-of-order write : header and data */
 784                 buf_offset = ltt_write_event_header(trace,
 785                                         buf, buf_offset,
 786                                         eID, data_size, tsc, rflags);
 787                 ltt_write_event_data(buf, buf_offset, &closure,
 788                                         serialize_private,
 789                                         largest_align, fmt, &args_copy);
 790                 va_end(args_copy);
 791                 /* Out-of-order commit */
 792                 ltt_commit_slot(channel, &transport_data, buf_offset,
 793                                 data_size, slot_size);
 794                 DBG("just commited event at offset %ld and size %zd", buf_offset, slot_size);
 795         }
 796 //ust// __get_cpu_var(ltt_nesting)--;
 797         ltt_nesting--;
 798         rcu_read_unlock(); //ust// rcu_read_unlock_sched_notrace();
 799 }
 800
 801 notrace void ltt_trace(const struct marker *mdata, void *probe_data,
 802                        struct registers *regs, void *call_data,
 803                        const char *fmt, ...)
 804 {
 805         va_list args;
 806
 807         va_start(args, fmt);
 808         ltt_vtrace(mdata, probe_data, regs, call_data, fmt, &args);
 809         va_end(args);
 810 }
 811
 812 //ust// MODULE_LICENSE("GPL");
 813 //ust// MODULE_AUTHOR("Mathieu Desnoyers");
 814 //ust// MODULE_DESCRIPTION("Linux Trace Toolkit Next Generation Serializer");