probes/lttng-events.h

   1 /*
   2  * lttng-events.h
   3  *
   4  * Copyright (C) 2009 Steven Rostedt <rostedt@goodmis.org>
   5  * Copyright (C) 2009-2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   6  *
   7  * This library is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU Lesser General Public
   9  * License as published by the Free Software Foundation; only
  10  * version 2.1 of the License.
  11  *
  12  * This library is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  15  * Lesser General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Lesser General Public
  18  * License along with this library; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  20  */
  21
  22 #include <linux/uaccess.h>
  23 #include <linux/debugfs.h>
  24 #include <linux/rculist.h>
  25 #include <asm/byteorder.h>
  26 #include "lttng.h"
  27 #include "lttng-types.h"
  28 #include "lttng-probe-user.h"
  29 #include "../wrapper/vmalloc.h" /* for wrapper_vmalloc_sync_all() */
  30 #include "../wrapper/ringbuffer/frontend_types.h"
  31 #include "../wrapper/rcu.h"
  32 #include "../lttng-events.h"
  33 #include "../lttng-tracer-core.h"
  34
  35 /*
  36  * Macro declarations used for all stages.
  37  */
  38
  39 /*
  40  * LTTng name mapping macros. LTTng remaps some of the kernel events to
  41  * enforce name-spacing.
  42  */
  43 #undef LTTNG_TRACEPOINT_EVENT_MAP
  44 #define LTTNG_TRACEPOINT_EVENT_MAP(name, map, proto, args, fields) \
  45         LTTNG_TRACEPOINT_EVENT_CLASS(map,                               \
  46                              PARAMS(proto),                             \
  47                              PARAMS(args),                              \
  48                              PARAMS(fields))                            \
  49         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(map, name, map, PARAMS(proto), PARAMS(args))
  50
  51 #undef LTTNG_TRACEPOINT_EVENT_MAP_NOARGS
  52 #define LTTNG_TRACEPOINT_EVENT_MAP_NOARGS(name, map, fields)            \
  53         LTTNG_TRACEPOINT_EVENT_CLASS_NOARGS(map,                        \
  54                              PARAMS(fields))                            \
  55         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(map, name, map)
  56
  57 #undef LTTNG_TRACEPOINT_EVENT_CODE_MAP
  58 #define LTTNG_TRACEPOINT_EVENT_CODE_MAP(name, map, proto, args, _locvar, _code, fields) \
  59         LTTNG_TRACEPOINT_EVENT_CLASS_CODE(map,                          \
  60                              PARAMS(proto),                             \
  61                              PARAMS(args),                              \
  62                              PARAMS(_locvar),                           \
  63                              PARAMS(_code),                             \
  64                              PARAMS(fields))                            \
  65         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(map, name, map, PARAMS(proto), PARAMS(args))
  66
  67 #undef LTTNG_TRACEPOINT_EVENT_CODE
  68 #define LTTNG_TRACEPOINT_EVENT_CODE(name, proto, args, _locvar, _code, fields) \
  69         LTTNG_TRACEPOINT_EVENT_CODE_MAP(name, name,                     \
  70                              PARAMS(proto),                             \
  71                              PARAMS(args),                              \
  72                              PARAMS(_locvar),                           \
  73                              PARAMS(_code),                             \
  74                              PARAMS(fields))
  75
  76 /*
  77  * LTTNG_TRACEPOINT_EVENT_CLASS can be used to add a generic function
  78  * handlers for events. That is, if all events have the same parameters
  79  * and just have distinct trace points.  Each tracepoint can be defined
  80  * with LTTNG_TRACEPOINT_EVENT_INSTANCE and that will map the
  81  * LTTNG_TRACEPOINT_EVENT_CLASS to the tracepoint.
  82  *
  83  * LTTNG_TRACEPOINT_EVENT is a one to one mapping between tracepoint and
  84  * template.
  85  */
  86
  87 #undef LTTNG_TRACEPOINT_EVENT
  88 #define LTTNG_TRACEPOINT_EVENT(name, proto, args, fields)               \
  89         LTTNG_TRACEPOINT_EVENT_MAP(name, name,                          \
  90                         PARAMS(proto),                                  \
  91                         PARAMS(args),                                   \
  92                         PARAMS(fields))
  93
  94 #undef LTTNG_TRACEPOINT_EVENT_NOARGS
  95 #define LTTNG_TRACEPOINT_EVENT_NOARGS(name, fields)                     \
  96         LTTNG_TRACEPOINT_EVENT_MAP_NOARGS(name, name, PARAMS(fields))
  97
  98 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE
  99 #define LTTNG_TRACEPOINT_EVENT_INSTANCE(template, name, proto, args)    \
 100         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(template, name, name, PARAMS(proto), PARAMS(args))
 101
 102 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_NOARGS
 103 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_NOARGS(template, name)  \
 104         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(template, name, name)
 105
 106 #undef LTTNG_TRACEPOINT_EVENT_CLASS
 107 #define LTTNG_TRACEPOINT_EVENT_CLASS(_name, _proto, _args, _fields) \
 108         LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, PARAMS(_proto), PARAMS(_args), , , \
 109                 PARAMS(_fields))
 110
 111 #undef LTTNG_TRACEPOINT_EVENT_CLASS_NOARGS
 112 #define LTTNG_TRACEPOINT_EVENT_CLASS_NOARGS(_name, _fields) \
 113         LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, , , PARAMS(_fields))
 114
 115
 116 /*
 117  * Stage 1 of the trace events.
 118  *
 119  * Create dummy trace calls for each events, verifying that the LTTng module
 120  * instrumentation headers match the kernel arguments. Will be optimized
 121  * out by the compiler.
 122  */
 123
 124 /* Reset all macros within TRACEPOINT_EVENT */
 125 #include "lttng-events-reset.h"
 126
 127 #undef TP_PROTO
 128 #define TP_PROTO(...)   __VA_ARGS__
 129
 130 #undef TP_ARGS
 131 #define TP_ARGS(...)    __VA_ARGS__
 132
 133 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
 134 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
 135 void trace_##_name(_proto);
 136
 137 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
 138 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map) \
 139 void trace_##_name(void);
 140
 141 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 142
 143 /*
 144  * Stage 1.1 of the trace events.
 145  *
 146  * Create dummy trace prototypes for each event class, and for each used
 147  * template. This will allow checking whether the prototypes from the
 148  * class and the instance using the class actually match.
 149  */
 150
 151 #include "lttng-events-reset.h" /* Reset all macros within TRACE_EVENT */
 152
 153 #undef TP_PROTO
 154 #define TP_PROTO(...)   __VA_ARGS__
 155
 156 #undef TP_ARGS
 157 #define TP_ARGS(...)    __VA_ARGS__
 158
 159 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
 160 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
 161 void __event_template_proto___##_template(_proto);
 162
 163 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
 164 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map) \
 165 void __event_template_proto___##_template(void);
 166
 167 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 168 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code, _fields) \
 169 void __event_template_proto___##_name(_proto);
 170
 171 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 172 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code, _fields) \
 173 void __event_template_proto___##_name(void);
 174
 175 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 176
 177 /*
 178  * Stage 2 of the trace events.
 179  *
 180  * Create event field type metadata section.
 181  * Each event produce an array of fields.
 182  */
 183
 184 /* Reset all macros within TRACEPOINT_EVENT */
 185 #include "lttng-events-reset.h"
 186 #include "lttng-events-write.h"
 187 #include "lttng-events-nowrite.h"
 188
 189 #undef _ctf_integer_ext
 190 #define _ctf_integer_ext(_type, _item, _src, _byte_order, _base, _user, _nowrite) \
 191         {                                                       \
 192           .name = #_item,                                       \
 193           .type = __type_integer(_type, 0, 0, -1, _byte_order, _base, none),\
 194           .nowrite = _nowrite,                                  \
 195           .user = _user,                                        \
 196         },
 197
 198 #undef _ctf_array_encoded
 199 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _user, _nowrite) \
 200         {                                                       \
 201           .name = #_item,                                       \
 202           .type =                                               \
 203                 {                                               \
 204                   .atype = atype_array,                         \
 205                   .u =                                          \
 206                         {                                       \
 207                           .array =                              \
 208                                 {                               \
 209                                   .elem_type = __type_integer(_type, 0, 0, 0, __BYTE_ORDER, 10, _encoding), \
 210                                   .length = _length,            \
 211                                 }                               \
 212                         }                                       \
 213                 },                                              \
 214           .nowrite = _nowrite,                                  \
 215           .user = _user,                                        \
 216         },
 217
 218 #undef _ctf_array_bitfield
 219 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 220         {                                                       \
 221           .name = #_item,                                       \
 222           .type =                                               \
 223                 {                                               \
 224                   .atype = atype_array,                         \
 225                   .u =                                          \
 226                         {                                       \
 227                           .array =                              \
 228                                 {                               \
 229                                   .elem_type = __type_integer(_type, 1, 1, 0, __LITTLE_ENDIAN, 10, none), \
 230                                   .length = (_length) * sizeof(_type) * CHAR_BIT, \
 231                                   .elem_alignment = lttng_alignof(_type), \
 232                                 }                               \
 233                         }                                       \
 234                 },                                              \
 235           .nowrite = _nowrite,                                  \
 236           .user = _user,                                        \
 237         },
 238
 239
 240 #undef _ctf_sequence_encoded
 241 #define _ctf_sequence_encoded(_type, _item, _src,               \
 242                         _length_type, _src_length, _encoding,   \
 243                         _byte_order, _base, _user, _nowrite)    \
 244         {                                                       \
 245           .name = #_item,                                       \
 246           .type =                                               \
 247                 {                                               \
 248                   .atype = atype_sequence,                      \
 249                   .u =                                          \
 250                         {                                       \
 251                           .sequence =                           \
 252                                 {                               \
 253                                   .length_type = __type_integer(_length_type, 0, 0, 0, __BYTE_ORDER, 10, none), \
 254                                   .elem_type = __type_integer(_type, 0, 0, -1, _byte_order, _base, _encoding), \
 255                                 },                              \
 256                         },                                      \
 257                 },                                              \
 258           .nowrite = _nowrite,                                  \
 259           .user = _user,                                        \
 260         },
 261
 262 #undef _ctf_sequence_bitfield
 263 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 264                         _length_type, _src_length,              \
 265                         _user, _nowrite)                        \
 266         {                                                       \
 267           .name = #_item,                                       \
 268           .type =                                               \
 269                 {                                               \
 270                   .atype = atype_sequence,                      \
 271                   .u =                                          \
 272                         {                                       \
 273                           .sequence =                           \
 274                                 {                               \
 275                                   .length_type = __type_integer(_length_type, 0, 0, 0, __BYTE_ORDER, 10, none), \
 276                                   .elem_type = __type_integer(_type, 1, 1, 0, __LITTLE_ENDIAN, 10, none), \
 277                                   .elem_alignment = lttng_alignof(_type), \
 278                                 },                              \
 279                         },                                      \
 280                 },                                              \
 281           .nowrite = _nowrite,                                  \
 282           .user = _user,                                        \
 283         },
 284
 285 #undef _ctf_string
 286 #define _ctf_string(_item, _src, _user, _nowrite)               \
 287         {                                                       \
 288           .name = #_item,                                       \
 289           .type =                                               \
 290                 {                                               \
 291                   .atype = atype_string,                        \
 292                   .u =                                          \
 293                         {                                       \
 294                           .basic = { .string = { .encoding = lttng_encode_UTF8 } } \
 295                         },                                      \
 296                 },                                              \
 297           .nowrite = _nowrite,                                  \
 298           .user = _user,                                        \
 299         },
 300
 301 #undef TP_FIELDS
 302 #define TP_FIELDS(...)  __VA_ARGS__     /* Only one used in this phase */
 303
 304 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 305 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code, _fields) \
 306         static const struct lttng_event_field __event_fields___##_name[] = { \
 307                 _fields                                                      \
 308         };
 309
 310 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 311 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code, _fields) \
 312         LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code, PARAMS(_fields))
 313
 314 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 315
 316 /*
 317  * Stage 3 of the trace events.
 318  *
 319  * Create probe callback prototypes.
 320  */
 321
 322 /* Reset all macros within TRACEPOINT_EVENT */
 323 #include "lttng-events-reset.h"
 324
 325 #undef TP_PROTO
 326 #define TP_PROTO(...)   __VA_ARGS__
 327
 328 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 329 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code, _fields) \
 330 static void __event_probe__##_name(void *__data, _proto);
 331
 332 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 333 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code, _fields) \
 334 static void __event_probe__##_name(void *__data);
 335
 336 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 337
 338 /*
 339  * Stage 4 of the trace events.
 340  *
 341  * Create static inline function that calculates event size.
 342  */
 343
 344 /* Reset all macros within TRACEPOINT_EVENT */
 345 #include "lttng-events-reset.h"
 346 #include "lttng-events-write.h"
 347
 348 #undef _ctf_integer_ext
 349 #define _ctf_integer_ext(_type, _item, _src, _byte_order, _base, _user, _nowrite) \
 350         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type)); \
 351         __event_len += sizeof(_type);
 352
 353 #undef _ctf_array_encoded
 354 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _user, _nowrite) \
 355         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type)); \
 356         __event_len += sizeof(_type) * (_length);
 357
 358 #undef _ctf_array_bitfield
 359 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 360         _ctf_array_encoded(_type, _item, _src, _length, none, _user, _nowrite)
 361
 362 #undef _ctf_sequence_encoded
 363 #define _ctf_sequence_encoded(_type, _item, _src, _length_type,                 \
 364                         _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
 365         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_length_type)); \
 366         __event_len += sizeof(_length_type);                                   \
 367         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type)); \
 368         __dynamic_len[__dynamic_len_idx] = (_src_length);                      \
 369         __event_len += sizeof(_type) * __dynamic_len[__dynamic_len_idx];       \
 370         __dynamic_len_idx++;
 371
 372 #undef _ctf_sequence_bitfield
 373 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 374                         _length_type, _src_length,              \
 375                         _user, _nowrite)                        \
 376         _ctf_sequence_encoded(_type, _item, _src, _length_type, _src_length, \
 377                 none, __LITTLE_ENDIAN, 10, _user, _nowrite)
 378
 379 /*
 380  * ctf_user_string includes \0. If returns 0, it faulted, so we set size to
 381  * 1 (\0 only).
 382  */
 383 #undef _ctf_string
 384 #define _ctf_string(_item, _src, _user, _nowrite)                              \
 385         if (_user)                                                             \
 386                 __event_len += __dynamic_len[__dynamic_len_idx++] =            \
 387                         max_t(size_t, lttng_strlen_user_inatomic(_src), 1);    \
 388         else                                                                   \
 389                 __event_len += __dynamic_len[__dynamic_len_idx++] =            \
 390                         strlen(_src) + 1;
 391
 392 #undef TP_PROTO
 393 #define TP_PROTO(...)   __VA_ARGS__
 394
 395 #undef TP_FIELDS
 396 #define TP_FIELDS(...)  __VA_ARGS__
 397
 398 #undef TP_locvar
 399 #define TP_locvar(...)  __VA_ARGS__
 400
 401 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 402 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code, _fields) \
 403 static inline size_t __event_get_size__##_name(size_t *__dynamic_len,         \
 404                 void *__tp_locvar, _proto)                                    \
 405 {                                                                             \
 406         size_t __event_len = 0;                                               \
 407         unsigned int __dynamic_len_idx __attribute__((unused)) = 0;           \
 408         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 409                                                                               \
 410         _fields                                                               \
 411         return __event_len;                                                   \
 412 }
 413
 414 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 415 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code, _fields) \
 416 static inline size_t __event_get_size__##_name(size_t *__dynamic_len,         \
 417                 void *__tp_locvar)                                            \
 418 {                                                                             \
 419         size_t __event_len = 0;                                               \
 420         unsigned int __dynamic_len_idx __attribute__((unused)) = 0;           \
 421         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 422                                                                               \
 423         _fields                                                               \
 424         return __event_len;                                                   \
 425 }
 426
 427 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 428
 429
 430 /*
 431  * Stage 4.1 of tracepoint event generation.
 432  *
 433  * Create static inline function that layout the filter stack data.
 434  * We make both write and nowrite data available to the filter.
 435  */
 436
 437 /* Reset all macros within TRACEPOINT_EVENT */
 438 #include "lttng-events-reset.h"
 439 #include "lttng-events-write.h"
 440 #include "lttng-events-nowrite.h"
 441
 442 #undef _ctf_integer_ext_fetched
 443 #define _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite) \
 444         if (lttng_is_signed_type(_type)) {                                     \
 445                 int64_t __ctf_tmp_int64;                                       \
 446                 switch (sizeof(_type)) {                                       \
 447                 case 1:                                                        \
 448                 {                                                              \
 449                         union { _type t; int8_t v; } __tmp = { (_type) (_src) }; \
 450                         __ctf_tmp_int64 = (int64_t) __tmp.v;                   \
 451                         break;                                                 \
 452                 }                                                              \
 453                 case 2:                                                        \
 454                 {                                                              \
 455                         union { _type t; int16_t v; } __tmp = { (_type) (_src) }; \
 456                         __ctf_tmp_int64 = (int64_t) __tmp.v;                   \
 457                         break;                                                 \
 458                 }                                                              \
 459                 case 4:                                                        \
 460                 {                                                              \
 461                         union { _type t; int32_t v; } __tmp = { (_type) (_src) }; \
 462                         __ctf_tmp_int64 = (int64_t) __tmp.v;                   \
 463                         break;                                                 \
 464                 }                                                              \
 465                 case 8:                                                        \
 466                 {                                                              \
 467                         union { _type t; int64_t v; } __tmp = { (_type) (_src) }; \
 468                         __ctf_tmp_int64 = (int64_t) __tmp.v;                   \
 469                         break;                                                 \
 470                 }                                                              \
 471                 default:                                                       \
 472                         BUG_ON(1);                                             \
 473                 };                                                             \
 474                 memcpy(__stack_data, &__ctf_tmp_int64, sizeof(int64_t));       \
 475         } else {                                                               \
 476                 uint64_t __ctf_tmp_uint64;                                     \
 477                 switch (sizeof(_type)) {                                       \
 478                 case 1:                                                        \
 479                 {                                                              \
 480                         union { _type t; uint8_t v; } __tmp = { (_type) (_src) }; \
 481                         __ctf_tmp_uint64 = (uint64_t) __tmp.v;                 \
 482                         break;                                                 \
 483                 }                                                              \
 484                 case 2:                                                        \
 485                 {                                                              \
 486                         union { _type t; uint16_t v; } __tmp = { (_type) (_src) }; \
 487                         __ctf_tmp_uint64 = (uint64_t) __tmp.v;                 \
 488                         break;                                                 \
 489                 }                                                              \
 490                 case 4:                                                        \
 491                 {                                                              \
 492                         union { _type t; uint32_t v; } __tmp = { (_type) (_src) }; \
 493                         __ctf_tmp_uint64 = (uint64_t) __tmp.v;                 \
 494                         break;                                                 \
 495                 }                                                              \
 496                 case 8:                                                        \
 497                 {                                                              \
 498                         union { _type t; uint64_t v; } __tmp = { (_type) (_src) }; \
 499                         __ctf_tmp_uint64 = (uint64_t) __tmp.v;                 \
 500                         break;                                                 \
 501                 }                                                              \
 502                 default:                                                       \
 503                         BUG_ON(1);                                             \
 504                 };                                                             \
 505                 memcpy(__stack_data, &__ctf_tmp_uint64, sizeof(uint64_t));     \
 506         }                                                                      \
 507         __stack_data += sizeof(int64_t);
 508
 509 #undef _ctf_integer_ext_isuser0
 510 #define _ctf_integer_ext_isuser0(_type, _item, _src, _byte_order, _base, _nowrite) \
 511         _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite)
 512
 513 #undef _ctf_integer_ext_isuser1
 514 #define _ctf_integer_ext_isuser1(_type, _item, _user_src, _byte_order, _base, _nowrite) \
 515 {                                                                              \
 516         __typeof__(_user_src) _src;                                            \
 517         if (get_user(_src, &(_user_src)))                                      \
 518                 _src = 0;                                                      \
 519         _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite) \
 520 }
 521
 522 #undef _ctf_integer_ext
 523 #define _ctf_integer_ext(_type, _item, _user_src, _byte_order, _base, _user, _nowrite) \
 524         _ctf_integer_ext_isuser##_user(_type, _item, _user_src, _byte_order, _base, _nowrite)
 525
 526 #undef _ctf_array_encoded
 527 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _user, _nowrite) \
 528         {                                                                      \
 529                 unsigned long __ctf_tmp_ulong = (unsigned long) (_length);     \
 530                 const void *__ctf_tmp_ptr = (_src);                            \
 531                 memcpy(__stack_data, &__ctf_tmp_ulong, sizeof(unsigned long)); \
 532                 __stack_data += sizeof(unsigned long);                         \
 533                 memcpy(__stack_data, &__ctf_tmp_ptr, sizeof(void *));          \
 534                 __stack_data += sizeof(void *);                                \
 535         }
 536
 537 #undef _ctf_array_bitfield
 538 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 539         _ctf_array_encoded(_type, _item, _src, _length, none, _user, _nowrite)
 540
 541 #undef _ctf_sequence_encoded
 542 #define _ctf_sequence_encoded(_type, _item, _src, _length_type,                \
 543                         _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
 544         {                                                                      \
 545                 unsigned long __ctf_tmp_ulong = (unsigned long) (_src_length); \
 546                 const void *__ctf_tmp_ptr = (_src);                            \
 547                 memcpy(__stack_data, &__ctf_tmp_ulong, sizeof(unsigned long)); \
 548                 __stack_data += sizeof(unsigned long);                         \
 549                 memcpy(__stack_data, &__ctf_tmp_ptr, sizeof(void *));          \
 550                 __stack_data += sizeof(void *);                                \
 551         }
 552
 553 #undef _ctf_sequence_bitfield
 554 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 555                         _length_type, _src_length,              \
 556                         _user, _nowrite)                        \
 557         _ctf_sequence_encoded(_type, _item, _src, _length_type, _src_length, \
 558                 none, __LITTLE_ENDIAN, 10, _user, _nowrite)
 559
 560 #undef _ctf_string
 561 #define _ctf_string(_item, _src, _user, _nowrite)                              \
 562         {                                                                      \
 563                 const void *__ctf_tmp_ptr = (_src);                            \
 564                 memcpy(__stack_data, &__ctf_tmp_ptr, sizeof(void *));          \
 565                 __stack_data += sizeof(void *);                                \
 566         }
 567
 568 #undef TP_PROTO
 569 #define TP_PROTO(...) __VA_ARGS__
 570
 571 #undef TP_FIELDS
 572 #define TP_FIELDS(...) __VA_ARGS__
 573
 574 #undef TP_locvar
 575 #define TP_locvar(...)  __VA_ARGS__
 576
 577 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 578 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code, _fields) \
 579 static inline                                                                 \
 580 void __event_prepare_filter_stack__##_name(char *__stack_data,                \
 581                 void *__tp_locvar)                                            \
 582 {                                                                             \
 583         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 584                                                                               \
 585         _fields                                                               \
 586 }
 587
 588 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 589 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code, _fields) \
 590 static inline                                                                 \
 591 void __event_prepare_filter_stack__##_name(char *__stack_data,                \
 592                 void *__tp_locvar, _proto)                                    \
 593 {                                                                             \
 594         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 595                                                                               \
 596         _fields                                                               \
 597 }
 598
 599 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 600
 601 /*
 602  * Stage 5 of the trace events.
 603  *
 604  * Create static inline function that calculates event payload alignment.
 605  */
 606
 607 /* Reset all macros within TRACEPOINT_EVENT */
 608 #include "lttng-events-reset.h"
 609 #include "lttng-events-write.h"
 610
 611 #undef _ctf_integer_ext
 612 #define _ctf_integer_ext(_type, _item, _src, _byte_order, _base, _user, _nowrite) \
 613         __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
 614
 615 #undef _ctf_array_encoded
 616 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _user, _nowrite) \
 617         __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
 618
 619 #undef _ctf_array_bitfield
 620 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 621         _ctf_array_encoded(_type, _item, _src, _length, none, _user, _nowrite)
 622
 623 #undef _ctf_sequence_encoded
 624 #define _ctf_sequence_encoded(_type, _item, _src, _length_type,                 \
 625                         _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
 626         __event_align = max_t(size_t, __event_align, lttng_alignof(_length_type)); \
 627         __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
 628
 629 #undef _ctf_sequence_bitfield
 630 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 631                         _length_type, _src_length,              \
 632                         _user, _nowrite)                        \
 633         _ctf_sequence_encoded(_type, _item, _src, _length_type, _src_length, \
 634                 none, __LITTLE_ENDIAN, 10, _user, _nowrite)
 635
 636 #undef _ctf_string
 637 #define _ctf_string(_item, _src, _user, _nowrite)
 638
 639 #undef TP_PROTO
 640 #define TP_PROTO(...)   __VA_ARGS__
 641
 642 #undef TP_FIELDS
 643 #define TP_FIELDS(...)  __VA_ARGS__
 644
 645 #undef TP_locvar
 646 #define TP_locvar(...)  __VA_ARGS__
 647
 648 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 649 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code, _fields) \
 650 static inline size_t __event_get_align__##_name(void *__tp_locvar, _proto)    \
 651 {                                                                             \
 652         size_t __event_align = 1;                                             \
 653         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 654                                                                               \
 655         _fields                                                               \
 656         return __event_align;                                                 \
 657 }
 658
 659 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 660 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code, _fields) \
 661 static inline size_t __event_get_align__##_name(void *__tp_locvar)            \
 662 {                                                                             \
 663         size_t __event_align = 1;                                             \
 664         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 665                                                                               \
 666         _fields                                                               \
 667         return __event_align;                                                 \
 668 }
 669
 670 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 671
 672 /*
 673  * Stage 6 of tracepoint event generation.
 674  *
 675  * Create the probe function. This function calls event size calculation
 676  * and writes event data into the buffer.
 677  */
 678
 679 /* Reset all macros within TRACEPOINT_EVENT */
 680 #include "lttng-events-reset.h"
 681 #include "lttng-events-write.h"
 682
 683 #undef _ctf_integer_ext_fetched
 684 #define _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite) \
 685         {                                                               \
 686                 _type __tmp = _src;                                     \
 687                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(__tmp));\
 688                 __chan->ops->event_write(&__ctx, &__tmp, sizeof(__tmp));\
 689         }
 690
 691 #undef _ctf_integer_ext_isuser0
 692 #define _ctf_integer_ext_isuser0(_type, _item, _src, _byte_order, _base, _nowrite) \
 693         _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite)
 694
 695 #undef _ctf_integer_ext_isuser1
 696 #define _ctf_integer_ext_isuser1(_type, _item, _user_src, _byte_order, _base, _nowrite) \
 697 {                                                                              \
 698         __typeof__(_user_src) _src;                                            \
 699         if (get_user(_src, &(_user_src)))                                      \
 700                 _src = 0;                                                      \
 701         _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite) \
 702 }
 703
 704 #undef _ctf_integer_ext
 705 #define _ctf_integer_ext(_type, _item, _user_src, _byte_order, _base, _user, _nowrite) \
 706         _ctf_integer_ext_isuser##_user(_type, _item, _user_src, _byte_order, _base, _nowrite)
 707
 708 #undef _ctf_array_encoded
 709 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _user, _nowrite) \
 710         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 711         if (_user) {                                                    \
 712                 __chan->ops->event_write_from_user(&__ctx, _src, sizeof(_type) * (_length)); \
 713         } else {                                                        \
 714                 __chan->ops->event_write(&__ctx, _src, sizeof(_type) * (_length)); \
 715         }
 716
 717 #if (__BYTE_ORDER == __LITTLE_ENDIAN)
 718 #undef _ctf_array_bitfield
 719 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 720         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 721         if (_user) {                                                    \
 722                 __chan->ops->event_write_from_user(&__ctx, _src, sizeof(_type) * (_length)); \
 723         } else {                                                        \
 724                 __chan->ops->event_write(&__ctx, _src, sizeof(_type) * (_length)); \
 725         }
 726 #else /* #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
 727 /*
 728  * For big endian, we need to byteswap into little endian.
 729  */
 730 #undef _ctf_array_bitfield
 731 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 732         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 733         {                                                               \
 734                 size_t _i;                                              \
 735                                                                         \
 736                 for (_i = 0; _i < (_length); _i++) {                    \
 737                         _type _tmp;                                     \
 738                                                                         \
 739                         if (_user) {                                    \
 740                                 if (get_user(_tmp, (_type *) _src + _i)) \
 741                                         _tmp = 0;                       \
 742                         } else {                                        \
 743                                 _tmp = ((_type *) _src)[_i];            \
 744                         }                                               \
 745                         switch (sizeof(_type)) {                        \
 746                         case 1:                                         \
 747                                 break;                                  \
 748                         case 2:                                         \
 749                                 _tmp = cpu_to_le16(_tmp);               \
 750                                 break;                                  \
 751                         case 4:                                         \
 752                                 _tmp = cpu_to_le32(_tmp);               \
 753                                 break;                                  \
 754                         case 8:                                         \
 755                                 _tmp = cpu_to_le64(_tmp);               \
 756                                 break;                                  \
 757                         default:                                        \
 758                                 BUG_ON(1);                              \
 759                         }                                               \
 760                         __chan->ops->event_write(&__ctx, &_tmp, sizeof(_type)); \
 761                 }                                                       \
 762         }
 763 #endif /* #else #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
 764
 765 #undef _ctf_sequence_encoded
 766 #define _ctf_sequence_encoded(_type, _item, _src, _length_type,         \
 767                         _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
 768         {                                                               \
 769                 _length_type __tmpl = __stackvar.__dynamic_len[__dynamic_len_idx]; \
 770                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_length_type));\
 771                 __chan->ops->event_write(&__ctx, &__tmpl, sizeof(_length_type));\
 772         }                                                               \
 773         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 774         if (_user) {                                                    \
 775                 __chan->ops->event_write_from_user(&__ctx, _src,        \
 776                         sizeof(_type) * __get_dynamic_len(dest));       \
 777         } else {                                                        \
 778                 __chan->ops->event_write(&__ctx, _src,                  \
 779                         sizeof(_type) * __get_dynamic_len(dest));       \
 780         }
 781
 782 #if (__BYTE_ORDER == __LITTLE_ENDIAN)
 783 #undef _ctf_sequence_bitfield
 784 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 785                         _length_type, _src_length,              \
 786                         _user, _nowrite)                        \
 787         {                                                               \
 788                 _length_type __tmpl = __stackvar.__dynamic_len[__dynamic_len_idx] * sizeof(_type) * CHAR_BIT; \
 789                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_length_type));\
 790                 __chan->ops->event_write(&__ctx, &__tmpl, sizeof(_length_type));\
 791         }                                                               \
 792         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 793         if (_user) {                                                    \
 794                 __chan->ops->event_write_from_user(&__ctx, _src,        \
 795                         sizeof(_type) * __get_dynamic_len(dest));       \
 796         } else {                                                        \
 797                 __chan->ops->event_write(&__ctx, _src,                  \
 798                         sizeof(_type) * __get_dynamic_len(dest));       \
 799         }
 800 #else /* #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
 801 /*
 802  * For big endian, we need to byteswap into little endian.
 803  */
 804 #undef _ctf_sequence_bitfield
 805 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 806                         _length_type, _src_length,              \
 807                         _user, _nowrite)                        \
 808         {                                                       \
 809                 _length_type __tmpl = __stackvar.__dynamic_len[__dynamic_len_idx] * sizeof(_type) * CHAR_BIT; \
 810                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_length_type));\
 811                 __chan->ops->event_write(&__ctx, &__tmpl, sizeof(_length_type));\
 812         }                                                               \
 813         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 814         {                                                               \
 815                 size_t _i, _length;                                     \
 816                                                                         \
 817                 _length = __get_dynamic_len(dest);                      \
 818                 for (_i = 0; _i < _length; _i++) {                      \
 819                         _type _tmp;                                     \
 820                                                                         \
 821                         if (_user) {                                    \
 822                                 if (get_user(_tmp, (_type *) _src + _i)) \
 823                                         _tmp = 0;                       \
 824                         } else {                                        \
 825                                 _tmp = ((_type *) _src)[_i];            \
 826                         }                                               \
 827                         switch (sizeof(_type)) {                        \
 828                         case 1:                                         \
 829                                 break;                                  \
 830                         case 2:                                         \
 831                                 _tmp = cpu_to_le16(_tmp);               \
 832                                 break;                                  \
 833                         case 4:                                         \
 834                                 _tmp = cpu_to_le32(_tmp);               \
 835                                 break;                                  \
 836                         case 8:                                         \
 837                                 _tmp = cpu_to_le64(_tmp);               \
 838                                 break;                                  \
 839                         default:                                        \
 840                                 BUG_ON(1);                              \
 841                         }                                               \
 842                         __chan->ops->event_write(&__ctx, &_tmp, sizeof(_type)); \
 843                 }                                                       \
 844         }
 845 #endif /* #else #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
 846
 847 #undef _ctf_string
 848 #define _ctf_string(_item, _src, _user, _nowrite)                       \
 849         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(*(_src)));      \
 850         if (_user) {                                                    \
 851                 __chan->ops->event_strcpy_from_user(&__ctx, _src,       \
 852                         __get_dynamic_len(dest));                       \
 853         } else {                                                        \
 854                 __chan->ops->event_strcpy(&__ctx, _src,                 \
 855                         __get_dynamic_len(dest));                       \
 856         }
 857
 858 /* Beware: this get len actually consumes the len value */
 859 #undef __get_dynamic_len
 860 #define __get_dynamic_len(field)        __stackvar.__dynamic_len[__dynamic_len_idx++]
 861
 862 #undef TP_PROTO
 863 #define TP_PROTO(...)   __VA_ARGS__
 864
 865 #undef TP_ARGS
 866 #define TP_ARGS(...)    __VA_ARGS__
 867
 868 #undef TP_FIELDS
 869 #define TP_FIELDS(...)  __VA_ARGS__
 870
 871 #undef TP_locvar
 872 #define TP_locvar(...)  __VA_ARGS__
 873
 874 #undef TP_code
 875 #define TP_code(...)    __VA_ARGS__
 876
 877 /*
 878  * For state dump, check that "session" argument (mandatory) matches the
 879  * session this event belongs to. Ensures that we write state dump data only
 880  * into the started session, not into all sessions.
 881  */
 882 #ifdef TP_SESSION_CHECK
 883 #define _TP_SESSION_CHECK(session, csession)    (session == csession)
 884 #else /* TP_SESSION_CHECK */
 885 #define _TP_SESSION_CHECK(session, csession)    1
 886 #endif /* TP_SESSION_CHECK */
 887
 888 /*
 889  * Using twice size for filter stack data to hold size and pointer for
 890  * each field (worse case). For integers, max size required is 64-bit.
 891  * Same for double-precision floats. Those fit within
 892  * 2*sizeof(unsigned long) for all supported architectures.
 893  * Perform UNION (||) of filter runtime list.
 894  */
 895 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 896 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code, _fields) \
 897 static void __event_probe__##_name(void *__data, _proto)                      \
 898 {                                                                             \
 899         struct probe_local_vars { _locvar };                                  \
 900         struct lttng_event *__event = __data;                                 \
 901         struct lttng_channel *__chan = __event->chan;                         \
 902         struct lttng_session *__session = __chan->session;                    \
 903         struct lib_ring_buffer_ctx __ctx;                                     \
 904         size_t __event_len, __event_align;                                    \
 905         size_t __dynamic_len_idx __attribute__((unused)) = 0;                 \
 906         union {                                                               \
 907                 size_t __dynamic_len[ARRAY_SIZE(__event_fields___##_name)];   \
 908                 char __filter_stack_data[2 * sizeof(unsigned long) * ARRAY_SIZE(__event_fields___##_name)]; \
 909         } __stackvar;                                                         \
 910         int __ret;                                                            \
 911         struct probe_local_vars __tp_locvar;                                  \
 912         struct probe_local_vars *tp_locvar __attribute__((unused)) =          \
 913                         &__tp_locvar;                                         \
 914         struct lttng_pid_tracker *__lpf;                                      \
 915                                                                               \
 916         if (!_TP_SESSION_CHECK(session, __session))                           \
 917                 return;                                                       \
 918         if (unlikely(!ACCESS_ONCE(__session->active)))                        \
 919                 return;                                                       \
 920         if (unlikely(!ACCESS_ONCE(__chan->enabled)))                          \
 921                 return;                                                       \
 922         if (unlikely(!ACCESS_ONCE(__event->enabled)))                         \
 923                 return;                                                       \
 924         __lpf = lttng_rcu_dereference(__session->pid_tracker);                \
 925         if (__lpf && likely(!lttng_pid_tracker_lookup(__lpf, current->pid)))  \
 926                 return;                                                       \
 927         _code                                                                 \
 928         if (unlikely(!list_empty(&__event->bytecode_runtime_head))) {         \
 929                 struct lttng_bytecode_runtime *bc_runtime;                    \
 930                 int __filter_record = __event->has_enablers_without_bytecode; \
 931                                                                               \
 932                 __event_prepare_filter_stack__##_name(__stackvar.__filter_stack_data, \
 933                                 tp_locvar, _args);                                    \
 934                 lttng_list_for_each_entry_rcu(bc_runtime, &__event->bytecode_runtime_head, node) { \
 935                         if (unlikely(bc_runtime->filter(bc_runtime,           \
 936                                         __stackvar.__filter_stack_data) & LTTNG_FILTER_RECORD_FLAG)) \
 937                                 __filter_record = 1;                          \
 938                 }                                                             \
 939                 if (likely(!__filter_record))                                 \
 940                         return;                                               \
 941         }                                                                     \
 942         __event_len = __event_get_size__##_name(__stackvar.__dynamic_len,     \
 943                                 tp_locvar, _args);                            \
 944         __event_align = __event_get_align__##_name(tp_locvar, _args);         \
 945         lib_ring_buffer_ctx_init(&__ctx, __chan->chan, __event, __event_len,  \
 946                                  __event_align, -1);                          \
 947         __ret = __chan->ops->event_reserve(&__ctx, __event->id);              \
 948         if (__ret < 0)                                                        \
 949                 return;                                                       \
 950         _fields                                                               \
 951         __chan->ops->event_commit(&__ctx);                                    \
 952 }
 953
 954 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 955 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code, _fields) \
 956 static void __event_probe__##_name(void *__data)                              \
 957 {                                                                             \
 958         struct probe_local_vars { _locvar };                                  \
 959         struct lttng_event *__event = __data;                                 \
 960         struct lttng_channel *__chan = __event->chan;                         \
 961         struct lttng_session *__session = __chan->session;                    \
 962         struct lib_ring_buffer_ctx __ctx;                                     \
 963         size_t __event_len, __event_align;                                    \
 964         size_t __dynamic_len_idx __attribute__((unused)) = 0;                 \
 965         union {                                                               \
 966                 size_t __dynamic_len[ARRAY_SIZE(__event_fields___##_name)];   \
 967                 char __filter_stack_data[2 * sizeof(unsigned long) * ARRAY_SIZE(__event_fields___##_name)]; \
 968         } __stackvar;                                                         \
 969         int __ret;                                                            \
 970         struct probe_local_vars __tp_locvar;                                  \
 971         struct probe_local_vars *tp_locvar __attribute__((unused)) =          \
 972                         &__tp_locvar;                                         \
 973         struct lttng_pid_tracker *__lpf;                                      \
 974                                                                               \
 975         if (!_TP_SESSION_CHECK(session, __session))                           \
 976                 return;                                                       \
 977         if (unlikely(!ACCESS_ONCE(__session->active)))                        \
 978                 return;                                                       \
 979         if (unlikely(!ACCESS_ONCE(__chan->enabled)))                          \
 980                 return;                                                       \
 981         if (unlikely(!ACCESS_ONCE(__event->enabled)))                         \
 982                 return;                                                       \
 983         __lpf = lttng_rcu_dereference(__session->pid_tracker);                \
 984         if (__lpf && likely(!lttng_pid_tracker_lookup(__lpf, current->pid)))  \
 985                 return;                                                       \
 986         _code                                                                 \
 987         if (unlikely(!list_empty(&__event->bytecode_runtime_head))) {         \
 988                 struct lttng_bytecode_runtime *bc_runtime;                    \
 989                 int __filter_record = __event->has_enablers_without_bytecode; \
 990                                                                               \
 991                 __event_prepare_filter_stack__##_name(__stackvar.__filter_stack_data, \
 992                                 tp_locvar);                                   \
 993                 lttng_list_for_each_entry_rcu(bc_runtime, &__event->bytecode_runtime_head, node) { \
 994                         if (unlikely(bc_runtime->filter(bc_runtime,           \
 995                                         __stackvar.__filter_stack_data) & LTTNG_FILTER_RECORD_FLAG)) \
 996                                 __filter_record = 1;                          \
 997                 }                                                             \
 998                 if (likely(!__filter_record))                                 \
 999                         return;                                               \
1000         }                                                                     \
1001         __event_len = __event_get_size__##_name(__stackvar.__dynamic_len, tp_locvar); \
1002         __event_align = __event_get_align__##_name(tp_locvar);                \
1003         lib_ring_buffer_ctx_init(&__ctx, __chan->chan, __event, __event_len,  \
1004                                  __event_align, -1);                          \
1005         __ret = __chan->ops->event_reserve(&__ctx, __event->id);              \
1006         if (__ret < 0)                                                        \
1007                 return;                                                       \
1008         _fields                                                               \
1009         __chan->ops->event_commit(&__ctx);                                    \
1010 }
1011
1012 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
1013
1014 #undef __get_dynamic_len
1015
1016 /*
1017  * Stage 7 of the trace events.
1018  *
1019  * Create event descriptions.
1020  */
1021
1022 /* Named field types must be defined in lttng-types.h */
1023
1024 #include "lttng-events-reset.h" /* Reset all macros within LTTNG_TRACEPOINT_EVENT */
1025
1026 #ifndef TP_PROBE_CB
1027 #define TP_PROBE_CB(_template)  &__event_probe__##_template
1028 #endif
1029
1030 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
1031 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map)      \
1032 static const struct lttng_event_desc __event_desc___##_map = {          \
1033         .fields = __event_fields___##_template,                         \
1034         .name = #_map,                                                  \
1035         .kname = #_name,                                                \
1036         .probe_callback = (void *) TP_PROBE_CB(_template),              \
1037         .nr_fields = ARRAY_SIZE(__event_fields___##_template),          \
1038         .owner = THIS_MODULE,                                           \
1039 };
1040
1041 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
1042 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
1043         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map)
1044
1045 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
1046
1047 /*
1048  * Stage 8 of the trace events.
1049  *
1050  * Create an array of event description pointers.
1051  */
1052
1053 #include "lttng-events-reset.h" /* Reset all macros within LTTNG_TRACEPOINT_EVENT */
1054
1055 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
1056 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map) \
1057                 &__event_desc___##_map,
1058
1059 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
1060 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
1061         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map)
1062
1063 #define TP_ID1(_token, _system) _token##_system
1064 #define TP_ID(_token, _system)  TP_ID1(_token, _system)
1065
1066 static const struct lttng_event_desc *TP_ID(__event_desc___, TRACE_SYSTEM)[] = {
1067 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
1068 };
1069
1070 #undef TP_ID1
1071 #undef TP_ID
1072
1073 /*
1074  * Stage 9 of the trace events.
1075  *
1076  * Create a toplevel descriptor for the whole probe.
1077  */
1078
1079 #define TP_ID1(_token, _system) _token##_system
1080 #define TP_ID(_token, _system)  TP_ID1(_token, _system)
1081
1082 /* non-const because list head will be modified when registered. */
1083 static __used struct lttng_probe_desc TP_ID(__probe_desc___, TRACE_SYSTEM) = {
1084         .provider = __stringify(TRACE_SYSTEM),
1085         .event_desc = TP_ID(__event_desc___, TRACE_SYSTEM),
1086         .nr_events = ARRAY_SIZE(TP_ID(__event_desc___, TRACE_SYSTEM)),
1087         .head = { NULL, NULL },
1088         .lazy_init_head = { NULL, NULL },
1089         .lazy = 0,
1090 };
1091
1092 #undef TP_ID1
1093 #undef TP_ID
1094
1095 /*
1096  * Stage 10 of the trace events.
1097  *
1098  * Register/unregister probes at module load/unload.
1099  */
1100
1101 #include "lttng-events-reset.h" /* Reset all macros within LTTNG_TRACEPOINT_EVENT */
1102
1103 #define TP_ID1(_token, _system) _token##_system
1104 #define TP_ID(_token, _system)  TP_ID1(_token, _system)
1105 #define module_init_eval1(_token, _system)      module_init(_token##_system)
1106 #define module_init_eval(_token, _system)       module_init_eval1(_token, _system)
1107 #define module_exit_eval1(_token, _system)      module_exit(_token##_system)
1108 #define module_exit_eval(_token, _system)       module_exit_eval1(_token, _system)
1109
1110 #ifndef TP_MODULE_NOINIT
1111 static int TP_ID(__lttng_events_init__, TRACE_SYSTEM)(void)
1112 {
1113         wrapper_vmalloc_sync_all();
1114         return lttng_probe_register(&TP_ID(__probe_desc___, TRACE_SYSTEM));
1115 }
1116
1117 static void TP_ID(__lttng_events_exit__, TRACE_SYSTEM)(void)
1118 {
1119         lttng_probe_unregister(&TP_ID(__probe_desc___, TRACE_SYSTEM));
1120 }
1121
1122 #ifndef TP_MODULE_NOAUTOLOAD
1123 module_init_eval(__lttng_events_init__, TRACE_SYSTEM);
1124 module_exit_eval(__lttng_events_exit__, TRACE_SYSTEM);
1125 #endif
1126
1127 #endif
1128
1129 #undef module_init_eval
1130 #undef module_exit_eval
1131 #undef TP_ID1
1132 #undef TP_ID
1133
1134 #undef TP_PROTO
1135 #undef TP_ARGS