include/lttng/tracepoint-event-impl.h

   1 /* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only)
   2  *
   3  * lttng/tracepoint-event-impl.h
   4  *
   5  * Copyright (C) 2009 Steven Rostedt <rostedt@goodmis.org>
   6  * Copyright (C) 2009-2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   7  */
   8
   9 #include <linux/uaccess.h>
  10 #include <linux/debugfs.h>
  11 #include <linux/rculist.h>
  12 #include <asm/byteorder.h>
  13 #include <linux/swab.h>
  14
  15 #include <wrapper/vmalloc.h>    /* for wrapper_vmalloc_sync_mappings() */
  16 #include <ringbuffer/frontend_types.h>
  17 #include <ringbuffer/backend.h>
  18 #include <wrapper/rcu.h>
  19 #include <wrapper/user_namespace.h>
  20 #include <lttng/types.h>
  21 #include <lttng/probe-user.h>
  22 #include <lttng/events.h>
  23 #include <lttng/tracer-core.h>
  24 #include <lttng/tp-mempool.h>
  25
  26 #define __LTTNG_NULL_STRING     "(null)"
  27
  28 #undef PARAMS
  29 #define PARAMS(args...)         args
  30
  31 /*
  32  * Macro declarations used for all stages.
  33  */
  34
  35 /*
  36  * LTTng name mapping macros. LTTng remaps some of the kernel events to
  37  * enforce name-spacing.
  38  */
  39 #undef LTTNG_TRACEPOINT_EVENT_MAP
  40 #define LTTNG_TRACEPOINT_EVENT_MAP(name, map, proto, args, fields) \
  41         LTTNG_TRACEPOINT_EVENT_CLASS(map,                               \
  42                              PARAMS(proto),                             \
  43                              PARAMS(args),                              \
  44                              PARAMS(fields))                            \
  45         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(map, name, map, PARAMS(proto), PARAMS(args))
  46
  47 #undef LTTNG_TRACEPOINT_EVENT_MAP_NOARGS
  48 #define LTTNG_TRACEPOINT_EVENT_MAP_NOARGS(name, map, fields)            \
  49         LTTNG_TRACEPOINT_EVENT_CLASS_NOARGS(map,                        \
  50                              PARAMS(fields))                            \
  51         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(map, name, map)
  52
  53 #undef LTTNG_TRACEPOINT_EVENT_CODE_MAP
  54 #define LTTNG_TRACEPOINT_EVENT_CODE_MAP(name, map, proto, args, _locvar, _code_pre, fields, _code_post) \
  55         LTTNG_TRACEPOINT_EVENT_CLASS_CODE(map,                          \
  56                              PARAMS(proto),                             \
  57                              PARAMS(args),                              \
  58                              PARAMS(_locvar),                           \
  59                              PARAMS(_code_pre),                         \
  60                              PARAMS(fields),                            \
  61                              PARAMS(_code_post))                        \
  62         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(map, name, map, PARAMS(proto), PARAMS(args))
  63
  64 #undef LTTNG_TRACEPOINT_EVENT_CODE
  65 #define LTTNG_TRACEPOINT_EVENT_CODE(name, proto, args, _locvar, _code_pre, fields, _code_post) \
  66         LTTNG_TRACEPOINT_EVENT_CODE_MAP(name, name,                     \
  67                              PARAMS(proto),                             \
  68                              PARAMS(args),                              \
  69                              PARAMS(_locvar),                           \
  70                              PARAMS(_code_pre),                         \
  71                              PARAMS(fields),                            \
  72                              PARAMS(_code_post))
  73
  74 /*
  75  * LTTNG_TRACEPOINT_EVENT_CLASS can be used to add a generic function
  76  * handlers for events. That is, if all events have the same parameters
  77  * and just have distinct trace points.  Each tracepoint can be defined
  78  * with LTTNG_TRACEPOINT_EVENT_INSTANCE and that will map the
  79  * LTTNG_TRACEPOINT_EVENT_CLASS to the tracepoint.
  80  *
  81  * LTTNG_TRACEPOINT_EVENT is a one to one mapping between tracepoint and
  82  * template.
  83  */
  84
  85 #undef LTTNG_TRACEPOINT_EVENT
  86 #define LTTNG_TRACEPOINT_EVENT(name, proto, args, fields)               \
  87         LTTNG_TRACEPOINT_EVENT_MAP(name, name,                          \
  88                         PARAMS(proto),                                  \
  89                         PARAMS(args),                                   \
  90                         PARAMS(fields))
  91
  92 #undef LTTNG_TRACEPOINT_EVENT_NOARGS
  93 #define LTTNG_TRACEPOINT_EVENT_NOARGS(name, fields)                     \
  94         LTTNG_TRACEPOINT_EVENT_MAP_NOARGS(name, name, PARAMS(fields))
  95
  96 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE
  97 #define LTTNG_TRACEPOINT_EVENT_INSTANCE(template, name, proto, args)    \
  98         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(template, name, name, PARAMS(proto), PARAMS(args))
  99
 100 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_NOARGS
 101 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_NOARGS(template, name)  \
 102         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(template, name, name)
 103
 104 #undef LTTNG_TRACEPOINT_EVENT_CLASS
 105 #define LTTNG_TRACEPOINT_EVENT_CLASS(_name, _proto, _args, _fields) \
 106         LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, PARAMS(_proto), PARAMS(_args), , , \
 107                 PARAMS(_fields), )
 108
 109 #undef LTTNG_TRACEPOINT_EVENT_CLASS_NOARGS
 110 #define LTTNG_TRACEPOINT_EVENT_CLASS_NOARGS(_name, _fields) \
 111         LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, , , PARAMS(_fields), )
 112
 113
 114 /*
 115  * Stage 1 of the trace events.
 116  *
 117  * Create dummy trace calls for each events, verifying that the LTTng module
 118  * instrumentation headers match the kernel arguments. Will be optimized
 119  * out by the compiler.
 120  */
 121
 122 /* Reset all macros within TRACEPOINT_EVENT */
 123 #include <lttng/events-reset.h>
 124
 125 #undef TP_PROTO
 126 #define TP_PROTO(...)   __VA_ARGS__
 127
 128 #undef TP_ARGS
 129 #define TP_ARGS(...)    __VA_ARGS__
 130
 131 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
 132 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
 133 void trace_##_name(_proto);
 134
 135 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
 136 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map) \
 137 void trace_##_name(void);
 138
 139 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 140
 141 /*
 142  * Stage 1.1 of the trace events.
 143  *
 144  * Create dummy trace prototypes for each event class, and for each used
 145  * template. This will allow checking whether the prototypes from the
 146  * class and the instance using the class actually match.
 147  */
 148
 149 #include <lttng/events-reset.h> /* Reset all macros within TRACE_EVENT */
 150
 151 #undef TP_PROTO
 152 #define TP_PROTO(...)   __VA_ARGS__
 153
 154 #undef TP_ARGS
 155 #define TP_ARGS(...)    __VA_ARGS__
 156
 157 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
 158 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
 159 void __event_template_proto___##_template(_proto);
 160
 161 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
 162 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map) \
 163 void __event_template_proto___##_template(void);
 164
 165 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 166 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 167 void __event_template_proto___##_name(_proto);
 168
 169 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 170 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 171 void __event_template_proto___##_name(void);
 172
 173 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 174
 175 /*
 176  * Stage 1.2 of tracepoint event generation
 177  *
 178  * Unfolding the enums
 179  */
 180 #include <lttng/events-reset.h> /* Reset all macros within TRACE_EVENT */
 181
 182 /* Enumeration entry (single value) */
 183 #undef ctf_enum_value
 184 #define ctf_enum_value(_string, _value)                                 \
 185         lttng_kernel_static_enum_entry_value(_string, _value)
 186
 187 /* Enumeration entry (range) */
 188 #undef ctf_enum_range
 189 #define ctf_enum_range(_string, _range_start, _range_end)               \
 190         lttng_kernel_static_enum_entry_range(_string, _range_start, _range_end)
 191
 192 /* Enumeration entry (automatic value; follows the rules of CTF) */
 193 #undef ctf_enum_auto
 194 #define ctf_enum_auto(_string)                                          \
 195         lttng_kernel_static_enum_entry_auto(_string)
 196
 197 #undef TP_ENUM_VALUES
 198 #define TP_ENUM_VALUES(...)                                             \
 199         __VA_ARGS__
 200
 201 #undef LTTNG_TRACEPOINT_ENUM
 202 #define LTTNG_TRACEPOINT_ENUM(_name, _values)                           \
 203         static const struct lttng_kernel_enum_entry *__enum_values__##_name[] = { \
 204                 _values                                                 \
 205         };
 206
 207 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 208
 209 /*
 210  * Stage 2 of the trace events.
 211  *
 212  * Create event field type metadata section.
 213  * Each event produce an array of fields.
 214  */
 215
 216 /* Reset all macros within TRACEPOINT_EVENT */
 217 #include <lttng/events-reset.h>
 218 #include <lttng/events-write.h>
 219 #include <lttng/events-nowrite.h>
 220
 221 #undef _ctf_integer_ext
 222 #define _ctf_integer_ext(_type, _item, _src, _byte_order, _base, _user, _nowrite)       \
 223         lttng_kernel_static_event_field(#_item,                                         \
 224                 lttng_kernel_static_type_integer_from_type(_type, _byte_order, _base),  \
 225                 _nowrite, _user, 0),
 226
 227 #undef _ctf_array_encoded
 228 #define _ctf_array_encoded(_type, _item, _src, _length,         \
 229                 _encoding, _byte_order, _elem_type_base, _user, _nowrite) \
 230         lttng_kernel_static_event_field(#_item,                 \
 231                 lttng_kernel_static_type_array(_length,         \
 232                         lttng_kernel_static_type_integer_from_type(_type, _byte_order, _elem_type_base), \
 233                         0,                                      \
 234                         _encoding),                             \
 235                 _nowrite, _user, 0),
 236
 237 #undef _ctf_array_bitfield
 238 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 239         lttng_kernel_static_event_field(#_item,                 \
 240                 lttng_kernel_static_type_array((_length) * sizeof(_type) * CHAR_BIT, \
 241                         lttng_kernel_static_type_integer(1, 1, 0, __LITTLE_ENDIAN, 10), \
 242                         lttng_alignof(_type),                   \
 243                         none),                                  \
 244                 _nowrite, _user, 0),
 245
 246 #undef _ctf_sequence_encoded
 247 #define _ctf_sequence_encoded(_type, _item, _src,               \
 248                         _length_type, _src_length, _encoding,   \
 249                         _byte_order, _elem_type_base, _user, _nowrite) \
 250         lttng_kernel_static_event_field("_" #_item "_length",   \
 251                 lttng_kernel_static_type_integer_from_type(_length_type, __BYTE_ORDER, 10), \
 252                 _nowrite, 0, 1),                                \
 253         lttng_kernel_static_event_field(#_item,                 \
 254                 lttng_kernel_static_type_sequence(NULL, /* Use previous field. */ \
 255                         lttng_kernel_static_type_integer_from_type(_type, _byte_order, _elem_type_base), \
 256                         0,                                      \
 257                         _encoding),                             \
 258                 _nowrite, _user, 0),
 259
 260 #undef _ctf_sequence_bitfield
 261 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 262                         _length_type, _src_length,              \
 263                         _user, _nowrite)                        \
 264         lttng_kernel_static_event_field("_" #_item "_length",   \
 265                 lttng_kernel_static_type_integer_from_type(_length_type, __BYTE_ORDER, 10), \
 266                 _nowrite, 0, 1),                                \
 267         lttng_kernel_static_event_field(#_item,                 \
 268                 lttng_kernel_static_type_sequence(NULL, /* Use previous field. */ \
 269                         lttng_kernel_static_type_integer(1, 1, 0, __LITTLE_ENDIAN, 10), \
 270                         lttng_alignof(_type),                   \
 271                         none),                                  \
 272                 _nowrite, _user, 0),
 273
 274 #undef _ctf_string
 275 #define _ctf_string(_item, _src, _user, _nowrite)               \
 276         lttng_kernel_static_event_field(#_item,                 \
 277                 lttng_kernel_static_type_string(UTF8),          \
 278                 _nowrite, _user, 0),
 279
 280 #undef _ctf_unused
 281 #define _ctf_unused(_src)
 282
 283 #undef _ctf_enum
 284 #define _ctf_enum(_name, _type, _item, _src, _user, _nowrite)   \
 285         lttng_kernel_static_event_field(#_item,                 \
 286                 lttng_kernel_static_type_enum(&__enum_##_name,  \
 287                         lttng_kernel_static_type_integer_from_type(_type, __BYTE_ORDER, 10)), \
 288                 _nowrite, _user, 0),
 289
 290 #undef ctf_custom_field
 291 #define ctf_custom_field(_type, _item, _code)                   \
 292         lttng_kernel_static_event_field(#_item, PARAMS(_type), 0, 0, 1),
 293
 294 #undef ctf_custom_type
 295 #define ctf_custom_type(...)    __VA_ARGS__
 296
 297 #undef TP_FIELDS
 298 #define TP_FIELDS(...)  __VA_ARGS__     /* Only one used in this phase */
 299
 300 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 301 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 302         static const struct lttng_kernel_event_field *__event_fields___##_name[] = { \
 303                 _fields                                                      \
 304         };
 305
 306 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 307 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 308         LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, PARAMS(_fields), _code_post)
 309
 310 #undef LTTNG_TRACEPOINT_ENUM
 311 #define LTTNG_TRACEPOINT_ENUM(_name, _values)                                           \
 312         static const struct lttng_kernel_enum_desc __enum_##_name = {                   \
 313                 .name = #_name,                                                         \
 314                 .entries = __enum_values__##_name,                                      \
 315                 .nr_entries = ARRAY_SIZE(__enum_values__##_name),                       \
 316         };
 317
 318 #define LTTNG_CREATE_FIELD_METADATA
 319 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 320 #undef LTTNG_CREATE_FIELD_METADATA
 321
 322 /*
 323  * Stage 3 of the trace events.
 324  *
 325  * Create probe callback prototypes.
 326  */
 327
 328 /* Reset all macros within TRACEPOINT_EVENT */
 329 #include <lttng/events-reset.h>
 330
 331 #undef TP_PROTO
 332 #define TP_PROTO(...)   __VA_ARGS__
 333
 334 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 335 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 336 static void __event_probe__##_name(void *__data, _proto);
 337
 338 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 339 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 340 static void __event_probe__##_name(void *__data);
 341
 342 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 343
 344 /*
 345  * Stage 4 of the trace events.
 346  *
 347  * Create static inline function that calculates event size.
 348  */
 349
 350 /* Reset all macros within TRACEPOINT_EVENT */
 351 #include <lttng/events-reset.h>
 352 #include <lttng/events-write.h>
 353
 354 #undef _ctf_integer_ext
 355 #define _ctf_integer_ext(_type, _item, _src, _byte_order, _base, _user, _nowrite) \
 356         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type)); \
 357         __event_len += sizeof(_type);
 358
 359 #undef _ctf_array_encoded
 360 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _byte_order, _base, _user, _nowrite) \
 361         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type)); \
 362         __event_len += sizeof(_type) * (_length);
 363
 364 #undef _ctf_array_bitfield
 365 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 366         _ctf_array_encoded(_type, _item, _src, _length, none, __LITTLE_ENDIAN, 0, _user, _nowrite)
 367
 368 #undef _ctf_sequence_encoded
 369 #define _ctf_sequence_encoded(_type, _item, _src, _length_type,                 \
 370                         _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
 371         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_length_type)); \
 372         __event_len += sizeof(_length_type);                                   \
 373         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type)); \
 374         {                                                                               \
 375                 size_t __seqlen = (_src_length);                                        \
 376                                                                                         \
 377                 if (unlikely(++this_cpu_ptr(&lttng_dynamic_len_stack)->offset >= LTTNG_DYNAMIC_LEN_STACK_SIZE)) \
 378                         goto error;                                                     \
 379                 barrier();      /* reserve before use. */                               \
 380                 this_cpu_ptr(&lttng_dynamic_len_stack)->stack[this_cpu_ptr(&lttng_dynamic_len_stack)->offset - 1] = __seqlen; \
 381                 __event_len += sizeof(_type) * __seqlen;                                \
 382         }
 383
 384 #undef _ctf_sequence_bitfield
 385 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 386                         _length_type, _src_length,              \
 387                         _user, _nowrite)                        \
 388         _ctf_sequence_encoded(_type, _item, _src, _length_type, _src_length, \
 389                 none, __LITTLE_ENDIAN, 10, _user, _nowrite)
 390
 391 /*
 392  * ctf_user_string includes \0. If returns 0, it faulted, so we set size to
 393  * 1 (\0 only).
 394  */
 395 #undef _ctf_string
 396 #define _ctf_string(_item, _src, _user, _nowrite)                              \
 397         if (unlikely(++this_cpu_ptr(&lttng_dynamic_len_stack)->offset >= LTTNG_DYNAMIC_LEN_STACK_SIZE)) \
 398                 goto error;                                                    \
 399         barrier();      /* reserve before use. */                              \
 400         if (_user) {                                                           \
 401                 __event_len += this_cpu_ptr(&lttng_dynamic_len_stack)->stack[this_cpu_ptr(&lttng_dynamic_len_stack)->offset - 1] = \
 402                         max_t(size_t, lttng_strlen_user_inatomic(_src), 1);    \
 403         } else {                                                               \
 404                 __event_len += this_cpu_ptr(&lttng_dynamic_len_stack)->stack[this_cpu_ptr(&lttng_dynamic_len_stack)->offset - 1] = \
 405                         strlen((_src) ? (_src) : __LTTNG_NULL_STRING) + 1; \
 406         }
 407
 408 #undef _ctf_enum
 409 #define _ctf_enum(_name, _type, _item, _src, _user, _nowrite)                  \
 410         _ctf_integer_ext(_type, _item, _src, __BYTE_ORDER, 10, _user, _nowrite)
 411
 412 #undef ctf_align
 413 #define ctf_align(_type)                                                \
 414         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type));
 415
 416 #undef ctf_custom_field
 417 #define ctf_custom_field(_type, _item, _code)                           \
 418         {                                                               \
 419                 _code                                                   \
 420         }
 421
 422 #undef ctf_custom_code
 423 #define ctf_custom_code(...)            __VA_ARGS__
 424
 425 #undef TP_PROTO
 426 #define TP_PROTO(...)   __VA_ARGS__
 427
 428 #undef TP_FIELDS
 429 #define TP_FIELDS(...)  __VA_ARGS__
 430
 431 #undef TP_locvar
 432 #define TP_locvar(...)  __VA_ARGS__
 433
 434 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 435 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 436 static inline ssize_t __event_get_size__##_name(void *__tp_locvar, _proto)     \
 437 {                                                                             \
 438         size_t __event_len = 0;                                               \
 439         unsigned int __dynamic_len_idx __attribute__((unused)) = 0;           \
 440         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 441                                                                               \
 442         _fields                                                               \
 443         return __event_len;                                                   \
 444                                                                               \
 445 error:                                                                        \
 446         __attribute__((unused));                                              \
 447         return -1;                                                            \
 448 }
 449
 450 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 451 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 452 static inline ssize_t __event_get_size__##_name(void *__tp_locvar)            \
 453 {                                                                             \
 454         size_t __event_len = 0;                                               \
 455         unsigned int __dynamic_len_idx __attribute__((unused)) = 0;           \
 456         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 457                                                                               \
 458         _fields                                                               \
 459         return __event_len;                                                   \
 460                                                                               \
 461 error:                                                                        \
 462         __attribute__((unused));                                              \
 463         return -1;                                                            \
 464 }
 465
 466 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 467
 468
 469 /*
 470  * Stage 4.1 of tracepoint event generation.
 471  *
 472  * Create static inline function that layout the filter stack data.
 473  * We make both write and nowrite data available to the filter.
 474  */
 475
 476 /* Reset all macros within TRACEPOINT_EVENT */
 477 #include <lttng/events-reset.h>
 478 #include <lttng/events-write.h>
 479 #include <lttng/events-nowrite.h>
 480
 481 #undef _ctf_integer_ext_fetched
 482 #define _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite) \
 483         if (lttng_is_signed_type(_type)) {                                     \
 484                 int64_t __ctf_tmp_int64;                                       \
 485                 switch (sizeof(_type)) {                                       \
 486                 case 1:                                                        \
 487                 {                                                              \
 488                         union { _type t; int8_t v; } __tmp = { (_type) (_src) }; \
 489                         __ctf_tmp_int64 = (int64_t) __tmp.v;                   \
 490                         break;                                                 \
 491                 }                                                              \
 492                 case 2:                                                        \
 493                 {                                                              \
 494                         union { _type t; int16_t v; } __tmp = { (_type) (_src) }; \
 495                         if (_byte_order != __BYTE_ORDER)                       \
 496                                 __swab16s(&__tmp.v);                           \
 497                         __ctf_tmp_int64 = (int64_t) __tmp.v;                   \
 498                         break;                                                 \
 499                 }                                                              \
 500                 case 4:                                                        \
 501                 {                                                              \
 502                         union { _type t; int32_t v; } __tmp = { (_type) (_src) }; \
 503                         if (_byte_order != __BYTE_ORDER)                       \
 504                                 __swab32s(&__tmp.v);                           \
 505                         __ctf_tmp_int64 = (int64_t) __tmp.v;                   \
 506                         break;                                                 \
 507                 }                                                              \
 508                 case 8:                                                        \
 509                 {                                                              \
 510                         union { _type t; int64_t v; } __tmp = { (_type) (_src) }; \
 511                         if (_byte_order != __BYTE_ORDER)                       \
 512                                 __swab64s(&__tmp.v);                           \
 513                         __ctf_tmp_int64 = (int64_t) __tmp.v;                   \
 514                         break;                                                 \
 515                 }                                                              \
 516                 default:                                                       \
 517                         BUG_ON(1);                                             \
 518                 };                                                             \
 519                 memcpy(__stack_data, &__ctf_tmp_int64, sizeof(int64_t));       \
 520         } else {                                                               \
 521                 uint64_t __ctf_tmp_uint64;                                     \
 522                 switch (sizeof(_type)) {                                       \
 523                 case 1:                                                        \
 524                 {                                                              \
 525                         union { _type t; uint8_t v; } __tmp = { (_type) (_src) }; \
 526                         __ctf_tmp_uint64 = (uint64_t) __tmp.v;                 \
 527                         break;                                                 \
 528                 }                                                              \
 529                 case 2:                                                        \
 530                 {                                                              \
 531                         union { _type t; uint16_t v; } __tmp = { (_type) (_src) }; \
 532                         if (_byte_order != __BYTE_ORDER)                       \
 533                                 __swab16s(&__tmp.v);                           \
 534                         __ctf_tmp_uint64 = (uint64_t) __tmp.v;                 \
 535                         break;                                                 \
 536                 }                                                              \
 537                 case 4:                                                        \
 538                 {                                                              \
 539                         union { _type t; uint32_t v; } __tmp = { (_type) (_src) }; \
 540                         if (_byte_order != __BYTE_ORDER)                       \
 541                                 __swab32s(&__tmp.v);                           \
 542                         __ctf_tmp_uint64 = (uint64_t) __tmp.v;                 \
 543                         break;                                                 \
 544                 }                                                              \
 545                 case 8:                                                        \
 546                 {                                                              \
 547                         union { _type t; uint64_t v; } __tmp = { (_type) (_src) }; \
 548                         if (_byte_order != __BYTE_ORDER)                       \
 549                                 __swab64s(&__tmp.v);                           \
 550                         __ctf_tmp_uint64 = (uint64_t) __tmp.v;                 \
 551                         break;                                                 \
 552                 }                                                              \
 553                 default:                                                       \
 554                         BUG_ON(1);                                             \
 555                 };                                                             \
 556                 memcpy(__stack_data, &__ctf_tmp_uint64, sizeof(uint64_t));     \
 557         }                                                                      \
 558         __stack_data += sizeof(int64_t);
 559
 560 #undef _ctf_integer_ext_isuser0
 561 #define _ctf_integer_ext_isuser0(_type, _item, _src, _byte_order, _base, _nowrite) \
 562         _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite)
 563
 564 #undef _ctf_integer_ext_isuser1
 565 #define _ctf_integer_ext_isuser1(_type, _item, _user_src, _byte_order, _base, _nowrite) \
 566 {                                                                                       \
 567         union {                                                                         \
 568                 char __array[sizeof(_user_src)];                                        \
 569                 __typeof__(_user_src) __v;                                              \
 570         } __tmp_fetch;                                                                  \
 571         if (lib_ring_buffer_copy_from_user_check_nofault(__tmp_fetch.__array,           \
 572                                 &(_user_src), sizeof(_user_src)))                       \
 573                 memset(__tmp_fetch.__array, 0, sizeof(__tmp_fetch.__array));            \
 574         _ctf_integer_ext_fetched(_type, _item, __tmp_fetch.__v, _byte_order, _base, _nowrite) \
 575 }
 576
 577 #undef _ctf_integer_ext
 578 #define _ctf_integer_ext(_type, _item, _user_src, _byte_order, _base, _user, _nowrite) \
 579         _ctf_integer_ext_isuser##_user(_type, _item, _user_src, _byte_order, _base, _nowrite)
 580
 581 #undef _ctf_array_encoded
 582 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _byte_order, _base, _user, _nowrite) \
 583         {                                                                      \
 584                 unsigned long __ctf_tmp_ulong = (unsigned long) (_length);     \
 585                 const void *__ctf_tmp_ptr = (_src);                            \
 586                 memcpy(__stack_data, &__ctf_tmp_ulong, sizeof(unsigned long)); \
 587                 __stack_data += sizeof(unsigned long);                         \
 588                 memcpy(__stack_data, &__ctf_tmp_ptr, sizeof(void *));          \
 589                 __stack_data += sizeof(void *);                                \
 590         }
 591
 592 #undef _ctf_array_bitfield
 593 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 594         _ctf_array_encoded(_type, _item, _src, _length, none, __LITTLE_ENDIAN, 0, _user, _nowrite)
 595
 596 #undef _ctf_sequence_encoded
 597 #define _ctf_sequence_encoded(_type, _item, _src, _length_type,                \
 598                         _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
 599         {                                                                      \
 600                 unsigned long __ctf_tmp_ulong = (unsigned long) (_src_length); \
 601                 const void *__ctf_tmp_ptr = (_src);                            \
 602                 memcpy(__stack_data, &__ctf_tmp_ulong, sizeof(unsigned long)); \
 603                 __stack_data += sizeof(unsigned long);                         \
 604                 memcpy(__stack_data, &__ctf_tmp_ptr, sizeof(void *));          \
 605                 __stack_data += sizeof(void *);                                \
 606         }
 607
 608 #undef _ctf_sequence_bitfield
 609 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 610                         _length_type, _src_length,              \
 611                         _user, _nowrite)                        \
 612         _ctf_sequence_encoded(_type, _item, _src, _length_type, _src_length, \
 613                 none, __LITTLE_ENDIAN, 10, _user, _nowrite)
 614
 615 #undef _ctf_string
 616 #define _ctf_string(_item, _src, _user, _nowrite)                              \
 617         {                                                                      \
 618                 const void *__ctf_tmp_ptr =                                    \
 619                         ((_src) ? (_src) : __LTTNG_NULL_STRING);               \
 620                 memcpy(__stack_data, &__ctf_tmp_ptr, sizeof(void *));          \
 621                 __stack_data += sizeof(void *);                                \
 622         }
 623
 624 #undef _ctf_enum
 625 #define _ctf_enum(_name, _type, _item, _src, _user, _nowrite)                  \
 626         _ctf_integer_ext(_type, _item, _src, __BYTE_ORDER, 10, _user, _nowrite)
 627
 628 #undef TP_PROTO
 629 #define TP_PROTO(...) __VA_ARGS__
 630
 631 #undef TP_FIELDS
 632 #define TP_FIELDS(...) __VA_ARGS__
 633
 634 #undef TP_locvar
 635 #define TP_locvar(...)  __VA_ARGS__
 636
 637 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 638 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 639 static inline                                                                 \
 640 void __event_prepare_interpreter_stack__##_name(char *__stack_data,                   \
 641                 void *__tp_locvar)                                            \
 642 {                                                                             \
 643         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 644                                                                               \
 645         _fields                                                               \
 646 }
 647
 648 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 649 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 650 static inline                                                                 \
 651 void __event_prepare_interpreter_stack__##_name(char *__stack_data,                   \
 652                 void *__tp_locvar, _proto)                                    \
 653 {                                                                             \
 654         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 655                                                                               \
 656         _fields                                                               \
 657 }
 658
 659 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 660
 661 /*
 662  * Stage 5 of the trace events.
 663  *
 664  * Create static inline function that calculates event payload alignment.
 665  */
 666
 667 /* Reset all macros within TRACEPOINT_EVENT */
 668 #include <lttng/events-reset.h>
 669 #include <lttng/events-write.h>
 670
 671 #undef _ctf_integer_ext
 672 #define _ctf_integer_ext(_type, _item, _src, _byte_order, _base, _user, _nowrite) \
 673         __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
 674
 675 #undef _ctf_array_encoded
 676 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _byte_order, _base, _user, _nowrite) \
 677         __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
 678
 679 #undef _ctf_array_bitfield
 680 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 681         _ctf_array_encoded(_type, _item, _src, _length, none, __LITTLE_ENDIAN, 0, _user, _nowrite)
 682
 683 #undef _ctf_sequence_encoded
 684 #define _ctf_sequence_encoded(_type, _item, _src, _length_type,                 \
 685                         _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
 686         __event_align = max_t(size_t, __event_align, lttng_alignof(_length_type)); \
 687         __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
 688
 689 #undef _ctf_sequence_bitfield
 690 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 691                         _length_type, _src_length,              \
 692                         _user, _nowrite)                        \
 693         _ctf_sequence_encoded(_type, _item, _src, _length_type, _src_length, \
 694                 none, __LITTLE_ENDIAN, 10, _user, _nowrite)
 695
 696 #undef _ctf_string
 697 #define _ctf_string(_item, _src, _user, _nowrite)
 698
 699 #undef _ctf_enum
 700 #define _ctf_enum(_name, _type, _item, _src, _user, _nowrite)   \
 701         _ctf_integer_ext(_type, _item, _src, __BYTE_ORDER, 10, _user, _nowrite)
 702
 703 #undef ctf_align
 704 #define ctf_align(_type)                                                \
 705         __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
 706
 707 #undef TP_PROTO
 708 #define TP_PROTO(...)   __VA_ARGS__
 709
 710 #undef TP_FIELDS
 711 #define TP_FIELDS(...)  __VA_ARGS__
 712
 713 #undef TP_locvar
 714 #define TP_locvar(...)  __VA_ARGS__
 715
 716 #undef ctf_custom_field
 717 #define ctf_custom_field(_type, _item, _code)           _code
 718
 719 #undef ctf_custom_code
 720 #define ctf_custom_code(...)                                            \
 721         {                                                               \
 722                 __VA_ARGS__                                             \
 723         }
 724
 725 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 726 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 727 static inline size_t __event_get_align__##_name(void *__tp_locvar, _proto)    \
 728 {                                                                             \
 729         size_t __event_align = 1;                                             \
 730         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 731                                                                               \
 732         _fields                                                               \
 733         return __event_align;                                                 \
 734 }
 735
 736 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 737 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 738 static inline size_t __event_get_align__##_name(void *__tp_locvar)            \
 739 {                                                                             \
 740         size_t __event_align = 1;                                             \
 741         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 742                                                                               \
 743         _fields                                                               \
 744         return __event_align;                                                 \
 745 }
 746
 747 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 748
 749 /*
 750  * Stage 6 of tracepoint event generation.
 751  *
 752  * Create the probe function. This function calls event size calculation
 753  * and writes event data into the buffer.
 754  */
 755
 756 /* Reset all macros within TRACEPOINT_EVENT */
 757 #include <lttng/events-reset.h>
 758 #include <lttng/events-write.h>
 759
 760 #undef _ctf_integer_ext_fetched
 761 #define _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite) \
 762         {                                                               \
 763                 _type __tmp = _src;                                     \
 764                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(__tmp));\
 765                 __chan->ops->event_write(&__ctx, &__tmp, sizeof(__tmp));\
 766         }
 767
 768 #undef _ctf_integer_ext_isuser0
 769 #define _ctf_integer_ext_isuser0(_type, _item, _src, _byte_order, _base, _nowrite) \
 770         _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite)
 771
 772 #undef _ctf_integer_ext_isuser1
 773 #define _ctf_integer_ext_isuser1(_type, _item, _user_src, _byte_order, _base, _nowrite) \
 774 {                                                                              \
 775         union {                                                                         \
 776                 char __array[sizeof(_user_src)];                                        \
 777                 __typeof__(_user_src) __v;                                              \
 778         } __tmp_fetch;                                                                  \
 779         if (lib_ring_buffer_copy_from_user_check_nofault(__tmp_fetch.__array,           \
 780                                 &(_user_src), sizeof(_user_src)))                       \
 781                 memset(__tmp_fetch.__array, 0, sizeof(__tmp_fetch.__array));            \
 782         _ctf_integer_ext_fetched(_type, _item, __tmp_fetch.__v, _byte_order, _base, _nowrite) \
 783 }
 784
 785 #undef _ctf_integer_ext
 786 #define _ctf_integer_ext(_type, _item, _user_src, _byte_order, _base, _user, _nowrite) \
 787         _ctf_integer_ext_isuser##_user(_type, _item, _user_src, _byte_order, _base, _nowrite)
 788
 789 #undef _ctf_array_encoded
 790 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _byte_order, _base, _user, _nowrite) \
 791         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 792         if (_user) {                                                    \
 793                 __chan->ops->event_write_from_user(&__ctx, _src, sizeof(_type) * (_length)); \
 794         } else {                                                        \
 795                 __chan->ops->event_write(&__ctx, _src, sizeof(_type) * (_length)); \
 796         }
 797
 798 #if (__BYTE_ORDER == __LITTLE_ENDIAN)
 799 #undef _ctf_array_bitfield
 800 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 801         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 802         if (_user) {                                                    \
 803                 __chan->ops->event_write_from_user(&__ctx, _src, sizeof(_type) * (_length)); \
 804         } else {                                                        \
 805                 __chan->ops->event_write(&__ctx, _src, sizeof(_type) * (_length)); \
 806         }
 807 #else /* #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
 808 /*
 809  * For big endian, we need to byteswap into little endian.
 810  */
 811 #undef _ctf_array_bitfield
 812 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 813         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 814         {                                                               \
 815                 size_t _i;                                              \
 816                                                                         \
 817                 for (_i = 0; _i < (_length); _i++) {                    \
 818                         _type _tmp;                                     \
 819                                                                         \
 820                         if (_user) {                                    \
 821                                 if (get_user(_tmp, (_type *) _src + _i)) \
 822                                         _tmp = 0;                       \
 823                         } else {                                        \
 824                                 _tmp = ((_type *) _src)[_i];            \
 825                         }                                               \
 826                         switch (sizeof(_type)) {                        \
 827                         case 1:                                         \
 828                                 break;                                  \
 829                         case 2:                                         \
 830                                 _tmp = cpu_to_le16(_tmp);               \
 831                                 break;                                  \
 832                         case 4:                                         \
 833                                 _tmp = cpu_to_le32(_tmp);               \
 834                                 break;                                  \
 835                         case 8:                                         \
 836                                 _tmp = cpu_to_le64(_tmp);               \
 837                                 break;                                  \
 838                         default:                                        \
 839                                 BUG_ON(1);                              \
 840                         }                                               \
 841                         __chan->ops->event_write(&__ctx, &_tmp, sizeof(_type)); \
 842                 }                                                       \
 843         }
 844 #endif /* #else #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
 845
 846 #undef _ctf_sequence_encoded
 847 #define _ctf_sequence_encoded(_type, _item, _src, _length_type,         \
 848                         _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
 849         {                                                               \
 850                 _length_type __tmpl = this_cpu_ptr(&lttng_dynamic_len_stack)->stack[__dynamic_len_idx]; \
 851                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_length_type));\
 852                 __chan->ops->event_write(&__ctx, &__tmpl, sizeof(_length_type));\
 853         }                                                               \
 854         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 855         if (_user) {                                                    \
 856                 __chan->ops->event_write_from_user(&__ctx, _src,        \
 857                         sizeof(_type) * __get_dynamic_len(dest));       \
 858         } else {                                                        \
 859                 __chan->ops->event_write(&__ctx, _src,                  \
 860                         sizeof(_type) * __get_dynamic_len(dest));       \
 861         }
 862
 863 #if (__BYTE_ORDER == __LITTLE_ENDIAN)
 864 #undef _ctf_sequence_bitfield
 865 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 866                         _length_type, _src_length,              \
 867                         _user, _nowrite)                        \
 868         {                                                               \
 869                 _length_type __tmpl = this_cpu_ptr(&lttng_dynamic_len_stack)->stack[__dynamic_len_idx] * sizeof(_type) * CHAR_BIT; \
 870                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_length_type));\
 871                 __chan->ops->event_write(&__ctx, &__tmpl, sizeof(_length_type));\
 872         }                                                               \
 873         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 874         if (_user) {                                                    \
 875                 __chan->ops->event_write_from_user(&__ctx, _src,        \
 876                         sizeof(_type) * __get_dynamic_len(dest));       \
 877         } else {                                                        \
 878                 __chan->ops->event_write(&__ctx, _src,                  \
 879                         sizeof(_type) * __get_dynamic_len(dest));       \
 880         }
 881 #else /* #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
 882 /*
 883  * For big endian, we need to byteswap into little endian.
 884  */
 885 #undef _ctf_sequence_bitfield
 886 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 887                         _length_type, _src_length,              \
 888                         _user, _nowrite)                        \
 889         {                                                       \
 890                 _length_type __tmpl = this_cpu_ptr(&lttng_dynamic_len_stack)->stack[__dynamic_len_idx] * sizeof(_type) * CHAR_BIT; \
 891                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_length_type));\
 892                 __chan->ops->event_write(&__ctx, &__tmpl, sizeof(_length_type));\
 893         }                                                               \
 894         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 895         {                                                               \
 896                 size_t _i, _length;                                     \
 897                                                                         \
 898                 _length = __get_dynamic_len(dest);                      \
 899                 for (_i = 0; _i < _length; _i++) {                      \
 900                         _type _tmp;                                     \
 901                                                                         \
 902                         if (_user) {                                    \
 903                                 if (get_user(_tmp, (_type *) _src + _i)) \
 904                                         _tmp = 0;                       \
 905                         } else {                                        \
 906                                 _tmp = ((_type *) _src)[_i];            \
 907                         }                                               \
 908                         switch (sizeof(_type)) {                        \
 909                         case 1:                                         \
 910                                 break;                                  \
 911                         case 2:                                         \
 912                                 _tmp = cpu_to_le16(_tmp);               \
 913                                 break;                                  \
 914                         case 4:                                         \
 915                                 _tmp = cpu_to_le32(_tmp);               \
 916                                 break;                                  \
 917                         case 8:                                         \
 918                                 _tmp = cpu_to_le64(_tmp);               \
 919                                 break;                                  \
 920                         default:                                        \
 921                                 BUG_ON(1);                              \
 922                         }                                               \
 923                         __chan->ops->event_write(&__ctx, &_tmp, sizeof(_type)); \
 924                 }                                                       \
 925         }
 926 #endif /* #else #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
 927
 928 #undef _ctf_string
 929 #define _ctf_string(_item, _src, _user, _nowrite)                       \
 930         if (_user) {                                                    \
 931                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(*(_src))); \
 932                 __chan->ops->event_strcpy_from_user(&__ctx, _src,       \
 933                         __get_dynamic_len(dest));                       \
 934         } else {                                                        \
 935                 const char *__ctf_tmp_string =                          \
 936                         ((_src) ? (_src) : __LTTNG_NULL_STRING);        \
 937                 lib_ring_buffer_align_ctx(&__ctx,                       \
 938                         lttng_alignof(*__ctf_tmp_string));              \
 939                 __chan->ops->event_strcpy(&__ctx, __ctf_tmp_string,     \
 940                         __get_dynamic_len(dest));                       \
 941         }
 942
 943 #undef _ctf_enum
 944 #define _ctf_enum(_name, _type, _item, _src, _user, _nowrite)           \
 945         _ctf_integer_ext(_type, _item, _src, __BYTE_ORDER, 10, _user, _nowrite)
 946
 947 #undef ctf_align
 948 #define ctf_align(_type)                                                \
 949         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));
 950
 951 #undef ctf_custom_field
 952 #define ctf_custom_field(_type, _item, _code)           _code
 953
 954 #undef ctf_custom_code
 955 #define ctf_custom_code(...)                                            \
 956         {                                                               \
 957                 __VA_ARGS__                                             \
 958         }
 959
 960 /* Beware: this get len actually consumes the len value */
 961 #undef __get_dynamic_len
 962 #define __get_dynamic_len(field)        this_cpu_ptr(&lttng_dynamic_len_stack)->stack[__dynamic_len_idx++]
 963
 964 #undef TP_PROTO
 965 #define TP_PROTO(...)   __VA_ARGS__
 966
 967 #undef TP_ARGS
 968 #define TP_ARGS(...)    __VA_ARGS__
 969
 970 #undef TP_FIELDS
 971 #define TP_FIELDS(...)  __VA_ARGS__
 972
 973 #undef TP_locvar
 974 #define TP_locvar(...)  __VA_ARGS__
 975
 976 #undef TP_code_pre
 977 #define TP_code_pre(...)        __VA_ARGS__
 978
 979 #undef TP_code_post
 980 #define TP_code_post(...)       __VA_ARGS__
 981
 982 /*
 983  * For state dump, check that "session" argument (mandatory) matches the
 984  * session this event belongs to. Ensures that we write state dump data only
 985  * into the started session, not into all sessions.
 986  */
 987 #ifdef TP_SESSION_CHECK
 988 #define _TP_SESSION_CHECK(session, csession)    (session == csession)
 989 #else /* TP_SESSION_CHECK */
 990 #define _TP_SESSION_CHECK(session, csession)    1
 991 #endif /* TP_SESSION_CHECK */
 992
 993 /*
 994  * Using twice size for filter stack data to hold size and pointer for
 995  * each field (worse case). For integers, max size required is 64-bit.
 996  * Same for double-precision floats. Those fit within
 997  * 2*sizeof(unsigned long) for all supported architectures.
 998  * Perform UNION (||) of filter runtime list.
 999  */
1000 #undef _LTTNG_TRACEPOINT_EVENT_CLASS_CODE
1001 #define _LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _data_proto, _locvar_args, _locvar, _code_pre, _fields, _code_post) \
1002 static void __event_probe__##_name(_data_proto)                                         \
1003 {                                                                                       \
1004         struct probe_local_vars { _locvar };                                            \
1005         struct lttng_kernel_event_common *__event = __data;                             \
1006         struct lttng_kernel_probe_ctx __lttng_probe_ctx = {                             \
1007                 .event = __event,                                                       \
1008                 .interruptible = !irqs_disabled(),                                      \
1009         };                                                                              \
1010         union {                                                                         \
1011                 size_t __dynamic_len_removed[ARRAY_SIZE(__event_fields___##_name)];     \
1012                 char __interpreter_stack_data[2 * sizeof(unsigned long) * ARRAY_SIZE(__event_fields___##_name)]; \
1013         } __stackvar;                                                                   \
1014         size_t __orig_dynamic_len_offset, __dynamic_len_idx __attribute__((unused));    \
1015         struct probe_local_vars __tp_locvar;                                            \
1016         struct probe_local_vars *tp_locvar __attribute__((unused)) =                    \
1017                         &__tp_locvar;                                                   \
1018         bool __interpreter_stack_prepared = false;                                      \
1019                                                                                         \
1020         switch (__event->type) {                                                        \
1021         case LTTNG_KERNEL_EVENT_TYPE_RECORDER:                                          \
1022         {                                                                               \
1023                 struct lttng_kernel_event_recorder *__event_recorder =                  \
1024                         container_of(__event, struct lttng_kernel_event_recorder, parent); \
1025                 struct lttng_channel *__chan = __event_recorder->chan;                  \
1026                 struct lttng_kernel_session *__session = __chan->session;               \
1027                 struct lttng_kernel_id_tracker_rcu *__lf;                               \
1028                                                                                         \
1029                 if (!_TP_SESSION_CHECK(session, __session))                             \
1030                         return;                                                         \
1031                 if (unlikely(!LTTNG_READ_ONCE(__session->active)))                      \
1032                         return;                                                         \
1033                 if (unlikely(!LTTNG_READ_ONCE(__chan->enabled)))                        \
1034                         return;                                                         \
1035                 __lf = lttng_rcu_dereference(__session->pid_tracker.p);                 \
1036                 if (__lf && likely(!lttng_id_tracker_lookup(__lf, current->tgid)))      \
1037                         return;                                                         \
1038                 __lf = lttng_rcu_dereference(__session->vpid_tracker.p);                \
1039                 if (__lf && likely(!lttng_id_tracker_lookup(__lf, task_tgid_vnr(current)))) \
1040                         return;                                                         \
1041                 __lf = lttng_rcu_dereference(__session->uid_tracker.p);                 \
1042                 if (__lf && likely(!lttng_id_tracker_lookup(__lf,                       \
1043                                 lttng_current_uid())))                                  \
1044                         return;                                                         \
1045                 __lf = lttng_rcu_dereference(__session->vuid_tracker.p);                \
1046                 if (__lf && likely(!lttng_id_tracker_lookup(__lf,                       \
1047                                 lttng_current_vuid())))                                 \
1048                         return;                                                         \
1049                 __lf = lttng_rcu_dereference(__session->gid_tracker.p);                 \
1050                 if (__lf && likely(!lttng_id_tracker_lookup(__lf,                       \
1051                                 lttng_current_gid())))                                  \
1052                         return;                                                         \
1053                 __lf = lttng_rcu_dereference(__session->vgid_tracker.p);                \
1054                 if (__lf && likely(!lttng_id_tracker_lookup(__lf,                       \
1055                                 lttng_current_vgid())))                                 \
1056                         return;                                                         \
1057                 break;                                                                  \
1058         }                                                                               \
1059         case LTTNG_KERNEL_EVENT_TYPE_NOTIFIER:                                          \
1060                 break;                                                                  \
1061         default:                                                                        \
1062                 WARN_ON_ONCE(1);                                                        \
1063         }                                                                               \
1064         if (unlikely(!READ_ONCE(__event->enabled)))                                     \
1065                 return;                                                                 \
1066         __orig_dynamic_len_offset = this_cpu_ptr(&lttng_dynamic_len_stack)->offset;     \
1067         __dynamic_len_idx = __orig_dynamic_len_offset;                                  \
1068         _code_pre                                                                       \
1069         if (unlikely(READ_ONCE(__event->eval_filter))) {                                \
1070                 __event_prepare_interpreter_stack__##_name(__stackvar.__interpreter_stack_data, \
1071                                 _locvar_args);                                          \
1072                 __interpreter_stack_prepared = true;                                    \
1073                 if (likely(__event->run_filter(__event,                                 \
1074                                 __stackvar.__interpreter_stack_data, &__lttng_probe_ctx, NULL) != LTTNG_KERNEL_EVENT_FILTER_ACCEPT)) \
1075                         goto __post;                                                    \
1076         }                                                                               \
1077         switch (__event->type) {                                                        \
1078         case LTTNG_KERNEL_EVENT_TYPE_RECORDER:                                          \
1079         {                                                                               \
1080                 struct lttng_kernel_event_recorder *__event_recorder =                  \
1081                         container_of(__event, struct lttng_kernel_event_recorder, parent); \
1082                 struct lttng_channel *__chan = __event_recorder->chan;                  \
1083                 struct lttng_kernel_ring_buffer_ctx __ctx;                              \
1084                 ssize_t __event_len;                                                    \
1085                 size_t __event_align;                                                   \
1086                 int __ret;                                                              \
1087                                                                                         \
1088                 __event_len = __event_get_size__##_name(_locvar_args);                  \
1089                 if (unlikely(__event_len < 0)) {                                        \
1090                         lib_ring_buffer_lost_event_too_big(__chan->chan);               \
1091                         goto __post;                                                    \
1092                 }                                                                       \
1093                 __event_align = __event_get_align__##_name(_locvar_args);               \
1094                 lib_ring_buffer_ctx_init(&__ctx, __event_recorder, __event_len,         \
1095                                          __event_align, &__lttng_probe_ctx);            \
1096                 __ret = __chan->ops->event_reserve(&__ctx);                             \
1097                 if (__ret < 0)                                                          \
1098                         goto __post;                                                    \
1099                 _fields                                                                 \
1100                 __chan->ops->event_commit(&__ctx);                                      \
1101                 break;                                                                  \
1102         }                                                                               \
1103         case LTTNG_KERNEL_EVENT_TYPE_NOTIFIER:                                          \
1104         {                                                                               \
1105                 struct lttng_kernel_event_notifier *__event_notifier =                  \
1106                         container_of(__event, struct lttng_kernel_event_notifier, parent); \
1107                 struct lttng_kernel_notification_ctx __notif_ctx;                       \
1108                                                                                         \
1109                 __notif_ctx.eval_capture = LTTNG_READ_ONCE(__event_notifier->eval_capture); \
1110                 if (unlikely(!__interpreter_stack_prepared && __notif_ctx.eval_capture)) \
1111                         __event_prepare_interpreter_stack__##_name(                     \
1112                                         __stackvar.__interpreter_stack_data,            \
1113                                         _locvar_args);                                  \
1114                                                                                         \
1115                 __event_notifier->notification_send(__event_notifier,                   \
1116                                 __stackvar.__interpreter_stack_data,                    \
1117                                 &__lttng_probe_ctx,                                     \
1118                                 &__notif_ctx);                                          \
1119                 break;                                                                  \
1120         }                                                                               \
1121         default:                                                                        \
1122                 WARN_ON_ONCE(1);                                                        \
1123         }                                                                               \
1124 __post:                                                                                 \
1125         _code_post                                                                      \
1126         barrier();      /* use before un-reserve. */                                    \
1127         this_cpu_ptr(&lttng_dynamic_len_stack)->offset = __orig_dynamic_len_offset;     \
1128         return;                                                                         \
1129 }
1130
1131 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
1132 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
1133         _LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, PARAMS(void *__data, _proto), PARAMS(tp_locvar, _args), \
1134                         PARAMS(_locvar), PARAMS(_code_pre), PARAMS(_fields), PARAMS(_code_post))
1135
1136 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
1137 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
1138         _LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, PARAMS(void *__data), PARAMS(tp_locvar), PARAMS(_locvar), \
1139                         PARAMS(_code_pre), PARAMS(_fields), PARAMS(_code_post))
1140
1141 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
1142
1143 #undef __get_dynamic_len
1144
1145 /*
1146  * Stage 7 of the trace events.
1147  *
1148  * Create event descriptions.
1149  */
1150
1151 /* Named field types must be defined in lttng-types.h */
1152
1153 #include <lttng/events-reset.h> /* Reset all macros within LTTNG_TRACEPOINT_EVENT */
1154
1155 #ifndef TP_PROBE_CB
1156 #define TP_PROBE_CB(_template)  &__event_probe__##_template
1157 #endif
1158
1159 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
1160 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map)      \
1161 static const struct lttng_kernel_event_desc __event_desc___##_map = {   \
1162         .event_name = #_map,                                            \
1163         .event_kname = #_name,                                          \
1164         .probe_callback = (void (*)(void)) TP_PROBE_CB(_template),      \
1165         .fields = __event_fields___##_template,                         \
1166         .nr_fields = ARRAY_SIZE(__event_fields___##_template),          \
1167         .owner = THIS_MODULE,                                           \
1168 };
1169
1170 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
1171 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
1172         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map)
1173
1174 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
1175
1176 /*
1177  * Stage 8 of the trace events.
1178  *
1179  * Create an array of event description pointers.
1180  */
1181
1182 #include <lttng/events-reset.h> /* Reset all macros within LTTNG_TRACEPOINT_EVENT */
1183
1184 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
1185 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map) \
1186                 &__event_desc___##_map,
1187
1188 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
1189 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
1190         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map)
1191
1192 #define TP_ID1(_token, _system) _token##_system
1193 #define TP_ID(_token, _system)  TP_ID1(_token, _system)
1194
1195 static const struct lttng_kernel_event_desc *TP_ID(__event_desc___, TRACE_SYSTEM)[] = {
1196 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
1197 };
1198
1199 #undef TP_ID1
1200 #undef TP_ID
1201
1202 /*
1203  * Stage 9 of the trace events.
1204  *
1205  * Create a toplevel descriptor for the whole probe.
1206  */
1207
1208 #define TP_ID1(_token, _system) _token##_system
1209 #define TP_ID(_token, _system)  TP_ID1(_token, _system)
1210
1211 /* non-const because list head will be modified when registered. */
1212 static __used struct lttng_kernel_probe_desc TP_ID(__probe_desc___, TRACE_SYSTEM) = {
1213         .provider_name = __stringify(TRACE_SYSTEM),
1214         .event_desc = TP_ID(__event_desc___, TRACE_SYSTEM),
1215         .nr_events = ARRAY_SIZE(TP_ID(__event_desc___, TRACE_SYSTEM)),
1216         .head = { NULL, NULL },
1217         .lazy_init_head = { NULL, NULL },
1218         .lazy = 0,
1219 };
1220
1221 #undef TP_ID1
1222 #undef TP_ID
1223
1224 /*
1225  * Stage 10 of the trace events.
1226  *
1227  * Register/unregister probes at module load/unload.
1228  */
1229
1230 #include <lttng/events-reset.h> /* Reset all macros within LTTNG_TRACEPOINT_EVENT */
1231
1232 #define TP_ID1(_token, _system) _token##_system
1233 #define TP_ID(_token, _system)  TP_ID1(_token, _system)
1234 #define module_init_eval1(_token, _system)      module_init(_token##_system)
1235 #define module_init_eval(_token, _system)       module_init_eval1(_token, _system)
1236 #define module_exit_eval1(_token, _system)      module_exit(_token##_system)
1237 #define module_exit_eval(_token, _system)       module_exit_eval1(_token, _system)
1238
1239 #ifndef TP_MODULE_NOINIT
1240 static int TP_ID(__lttng_events_init__, TRACE_SYSTEM)(void)
1241 {
1242         wrapper_vmalloc_sync_mappings();
1243         return lttng_kernel_probe_register(&TP_ID(__probe_desc___, TRACE_SYSTEM));
1244 }
1245
1246 static void TP_ID(__lttng_events_exit__, TRACE_SYSTEM)(void)
1247 {
1248         lttng_kernel_probe_unregister(&TP_ID(__probe_desc___, TRACE_SYSTEM));
1249 }
1250
1251 #ifndef TP_MODULE_NOAUTOLOAD
1252 module_init_eval(__lttng_events_init__, TRACE_SYSTEM);
1253 module_exit_eval(__lttng_events_exit__, TRACE_SYSTEM);
1254 #endif
1255
1256 #endif
1257
1258 #undef module_init_eval
1259 #undef module_exit_eval
1260 #undef TP_ID1
1261 #undef TP_ID
1262
1263 #undef TP_PROTO
1264 #undef TP_ARGS