instrumentation/events/lttng-module/rcu.h

   1 #undef TRACE_SYSTEM
   2 #define TRACE_SYSTEM rcu
   3
   4 #if !defined(LTTNG_TRACE_RCU_H) || defined(TRACE_HEADER_MULTI_READ)
   5 #define LTTNG_TRACE_RCU_H
   6
   7 #include <probes/lttng-tracepoint-event.h>
   8 #include <linux/version.h>
   9
  10 /*
  11  * Tracepoint for start/end markers used for utilization calculations.
  12  * By convention, the string is of the following forms:
  13  *
  14  * "Start <activity>" -- Mark the start of the specified activity,
  15  *                       such as "context switch".  Nesting is permitted.
  16  * "End <activity>" -- Mark the end of the specified activity.
  17  *
  18  * An "@" character within "<activity>" is a comment character: Data
  19  * reduction scripts will ignore the "@" and the remainder of the line.
  20  */
  21 LTTNG_TRACEPOINT_EVENT(rcu_utilization,
  22
  23 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,12,0))
  24         TP_PROTO(const char *s),
  25 #else /* #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,12,0)) */
  26         TP_PROTO(char *s),
  27 #endif /* #else #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,12,0)) */
  28
  29         TP_ARGS(s),
  30
  31         TP_FIELDS(
  32                 ctf_string(s, s)
  33         )
  34 )
  35
  36 #ifdef CONFIG_RCU_TRACE
  37
  38 #if defined(CONFIG_TREE_RCU) \
  39         || (LINUX_VERSION_CODE >= KERNEL_VERSION(3,19,0) \
  40                 && defined(CONFIG_PREEMPT_RCU)) \
  41         || defined(CONFIG_TREE_PREEMPT_RCU)
  42
  43 /*
  44  * Tracepoint for grace-period events: starting and ending a grace
  45  * period ("start" and "end", respectively), a CPU noting the start
  46  * of a new grace period or the end of an old grace period ("cpustart"
  47  * and "cpuend", respectively), a CPU passing through a quiescent
  48  * state ("cpuqs"), a CPU coming online or going offline ("cpuonl"
  49  * and "cpuofl", respectively), and a CPU being kicked for being too
  50  * long in dyntick-idle mode ("kick").
  51  */
  52 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,19,0) || \
  53         LTTNG_RHEL_KERNEL_RANGE(4,18,0,80,0,0, 4,19,0,0,0,0))
  54 LTTNG_TRACEPOINT_EVENT(rcu_grace_period,
  55
  56         TP_PROTO(const char *rcuname, unsigned long gp_seq, const char *gpevent),
  57
  58         TP_ARGS(rcuname, gp_seq, gpevent),
  59
  60         TP_FIELDS(
  61                 ctf_string(rcuname, rcuname)
  62                 ctf_integer(unsigned long, gp_seq, gp_seq)
  63                 ctf_string(gpevent, gpevent)
  64         )
  65 )
  66 #elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3,12,0))
  67 LTTNG_TRACEPOINT_EVENT(rcu_grace_period,
  68
  69         TP_PROTO(const char *rcuname, unsigned long gpnum, const char *gpevent),
  70
  71         TP_ARGS(rcuname, gpnum, gpevent),
  72
  73         TP_FIELDS(
  74                 ctf_string(rcuname, rcuname)
  75                 ctf_integer(unsigned long, gpnum, gpnum)
  76                 ctf_string(gpevent, gpevent)
  77         )
  78 )
  79 #else
  80 LTTNG_TRACEPOINT_EVENT(rcu_grace_period,
  81
  82         TP_PROTO(char *rcuname, unsigned long gpnum, char *gpevent),
  83
  84         TP_ARGS(rcuname, gpnum, gpevent),
  85
  86         TP_FIELDS(
  87                 ctf_string(rcuname, rcuname)
  88                 ctf_integer(unsigned long, gpnum, gpnum)
  89                 ctf_string(gpevent, gpevent)
  90         )
  91 )
  92 #endif
  93
  94 /*
  95  * Tracepoint for grace-period-initialization events.  These are
  96  * distinguished by the type of RCU, the new grace-period number, the
  97  * rcu_node structure level, the starting and ending CPU covered by the
  98  * rcu_node structure, and the mask of CPUs that will be waited for.
  99  * All but the type of RCU are extracted from the rcu_node structure.
 100  */
 101 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,19,0) || \
 102         LTTNG_RHEL_KERNEL_RANGE(4,18,0,80,0,0, 4,19,0,0,0,0))
 103 LTTNG_TRACEPOINT_EVENT(rcu_grace_period_init,
 104
 105         TP_PROTO(const char *rcuname, unsigned long gp_seq, u8 level,
 106                  int grplo, int grphi, unsigned long qsmask),
 107
 108         TP_ARGS(rcuname, gp_seq, level, grplo, grphi, qsmask),
 109
 110         TP_FIELDS(
 111                 ctf_string(rcuname, rcuname)
 112                 ctf_integer(unsigned long, gp_seq, gp_seq)
 113                 ctf_integer(u8, level, level)
 114                 ctf_integer(int, grplo, grplo)
 115                 ctf_integer(int, grphi, grphi)
 116                 ctf_integer(unsigned long, qsmask, qsmask)
 117         )
 118 )
 119 #elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3,12,0))
 120 LTTNG_TRACEPOINT_EVENT(rcu_grace_period_init,
 121
 122         TP_PROTO(const char *rcuname, unsigned long gpnum, u8 level,
 123                  int grplo, int grphi, unsigned long qsmask),
 124
 125         TP_ARGS(rcuname, gpnum, level, grplo, grphi, qsmask),
 126
 127         TP_FIELDS(
 128                 ctf_string(rcuname, rcuname)
 129                 ctf_integer(unsigned long, gpnum, gpnum)
 130                 ctf_integer(u8, level, level)
 131                 ctf_integer(int, grplo, grplo)
 132                 ctf_integer(int, grphi, grphi)
 133                 ctf_integer(unsigned long, qsmask, qsmask)
 134         )
 135 )
 136 #else
 137 LTTNG_TRACEPOINT_EVENT(rcu_grace_period_init,
 138
 139         TP_PROTO(char *rcuname, unsigned long gpnum, u8 level,
 140                  int grplo, int grphi, unsigned long qsmask),
 141
 142         TP_ARGS(rcuname, gpnum, level, grplo, grphi, qsmask),
 143
 144         TP_FIELDS(
 145                 ctf_string(rcuname, rcuname)
 146                 ctf_integer(unsigned long, gpnum, gpnum)
 147                 ctf_integer(u8, level, level)
 148                 ctf_integer(int, grplo, grplo)
 149                 ctf_integer(int, grphi, grphi)
 150                 ctf_integer(unsigned long, qsmask, qsmask)
 151         )
 152 )
 153 #endif
 154
 155 /*
 156  * Tracepoint for tasks blocking within preemptible-RCU read-side
 157  * critical sections.  Track the type of RCU (which one day might
 158  * include SRCU), the grace-period number that the task is blocking
 159  * (the current or the next), and the task's PID.
 160  */
 161 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,19,0) || \
 162         LTTNG_RHEL_KERNEL_RANGE(4,18,0,80,0,0, 4,19,0,0,0,0))
 163 LTTNG_TRACEPOINT_EVENT(rcu_preempt_task,
 164
 165         TP_PROTO(const char *rcuname, int pid, unsigned long gp_seq),
 166
 167         TP_ARGS(rcuname, pid, gp_seq),
 168
 169         TP_FIELDS(
 170                 ctf_string(rcuname, rcuname)
 171                 ctf_integer(unsigned long, gp_seq, gp_seq)
 172                 ctf_integer(int, pid, pid)
 173         )
 174 )
 175 #elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3,12,0))
 176 LTTNG_TRACEPOINT_EVENT(rcu_preempt_task,
 177
 178         TP_PROTO(const char *rcuname, int pid, unsigned long gpnum),
 179
 180         TP_ARGS(rcuname, pid, gpnum),
 181
 182         TP_FIELDS(
 183                 ctf_string(rcuname, rcuname)
 184                 ctf_integer(unsigned long, gpnum, gpnum)
 185                 ctf_integer(int, pid, pid)
 186         )
 187 )
 188 #else
 189 LTTNG_TRACEPOINT_EVENT(rcu_preempt_task,
 190
 191         TP_PROTO(char *rcuname, int pid, unsigned long gpnum),
 192
 193         TP_ARGS(rcuname, pid, gpnum),
 194
 195         TP_FIELDS(
 196                 ctf_string(rcuname, rcuname)
 197                 ctf_integer(unsigned long, gpnum, gpnum)
 198                 ctf_integer(int, pid, pid)
 199         )
 200 )
 201 #endif
 202
 203 /*
 204  * Tracepoint for tasks that blocked within a given preemptible-RCU
 205  * read-side critical section exiting that critical section.  Track the
 206  * type of RCU (which one day might include SRCU) and the task's PID.
 207  */
 208 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,19,0) || \
 209         LTTNG_RHEL_KERNEL_RANGE(4,18,0,80,0,0, 4,19,0,0,0,0))
 210 LTTNG_TRACEPOINT_EVENT(rcu_unlock_preempted_task,
 211
 212         TP_PROTO(const char *rcuname, unsigned long gp_seq, int pid),
 213
 214         TP_ARGS(rcuname, gp_seq, pid),
 215
 216         TP_FIELDS(
 217                 ctf_string(rcuname, rcuname)
 218                 ctf_integer(unsigned long, gp_seq, gp_seq)
 219                 ctf_integer(int, pid, pid)
 220         )
 221 )
 222 #elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3,12,0))
 223 LTTNG_TRACEPOINT_EVENT(rcu_unlock_preempted_task,
 224
 225         TP_PROTO(const char *rcuname, unsigned long gpnum, int pid),
 226
 227         TP_ARGS(rcuname, gpnum, pid),
 228
 229         TP_FIELDS(
 230                 ctf_string(rcuname, rcuname)
 231                 ctf_integer(unsigned long, gpnum, gpnum)
 232                 ctf_integer(int, pid, pid)
 233         )
 234 )
 235 #else
 236 LTTNG_TRACEPOINT_EVENT(rcu_unlock_preempted_task,
 237
 238         TP_PROTO(char *rcuname, unsigned long gpnum, int pid),
 239
 240         TP_ARGS(rcuname, gpnum, pid),
 241
 242         TP_FIELDS(
 243                 ctf_string(rcuname, rcuname)
 244                 ctf_integer(unsigned long, gpnum, gpnum)
 245                 ctf_integer(int, pid, pid)
 246         )
 247 )
 248 #endif
 249
 250 /*
 251  * Tracepoint for quiescent-state-reporting events.  These are
 252  * distinguished by the type of RCU, the grace-period number, the
 253  * mask of quiescent lower-level entities, the rcu_node structure level,
 254  * the starting and ending CPU covered by the rcu_node structure, and
 255  * whether there are any blocked tasks blocking the current grace period.
 256  * All but the type of RCU are extracted from the rcu_node structure.
 257  */
 258 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,19,0) || \
 259         LTTNG_RHEL_KERNEL_RANGE(4,18,0,80,0,0, 4,19,0,0,0,0))
 260 LTTNG_TRACEPOINT_EVENT(rcu_quiescent_state_report,
 261
 262         TP_PROTO(const char *rcuname, unsigned long gp_seq,
 263                  unsigned long mask, unsigned long qsmask,
 264                  u8 level, int grplo, int grphi, int gp_tasks),
 265
 266         TP_ARGS(rcuname, gp_seq, mask, qsmask, level, grplo, grphi, gp_tasks),
 267
 268         TP_FIELDS(
 269                 ctf_string(rcuname, rcuname)
 270                 ctf_integer(unsigned long, gp_seq, gp_seq)
 271                 ctf_integer(unsigned long, mask, mask)
 272                 ctf_integer(unsigned long, qsmask, qsmask)
 273                 ctf_integer(u8, level, level)
 274                 ctf_integer(int, grplo, grplo)
 275                 ctf_integer(int, grphi, grphi)
 276                 ctf_integer(u8, gp_tasks, gp_tasks)
 277         )
 278 )
 279 #elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3,12,0))
 280 LTTNG_TRACEPOINT_EVENT(rcu_quiescent_state_report,
 281
 282         TP_PROTO(const char *rcuname, unsigned long gpnum,
 283                  unsigned long mask, unsigned long qsmask,
 284                  u8 level, int grplo, int grphi, int gp_tasks),
 285
 286         TP_ARGS(rcuname, gpnum, mask, qsmask, level, grplo, grphi, gp_tasks),
 287
 288         TP_FIELDS(
 289                 ctf_string(rcuname, rcuname)
 290                 ctf_integer(unsigned long, gpnum, gpnum)
 291                 ctf_integer(unsigned long, mask, mask)
 292                 ctf_integer(unsigned long, qsmask, qsmask)
 293                 ctf_integer(u8, level, level)
 294                 ctf_integer(int, grplo, grplo)
 295                 ctf_integer(int, grphi, grphi)
 296                 ctf_integer(u8, gp_tasks, gp_tasks)
 297         )
 298 )
 299 #else
 300 LTTNG_TRACEPOINT_EVENT(rcu_quiescent_state_report,
 301
 302         TP_PROTO(char *rcuname, unsigned long gpnum,
 303                  unsigned long mask, unsigned long qsmask,
 304                  u8 level, int grplo, int grphi, int gp_tasks),
 305
 306         TP_ARGS(rcuname, gpnum, mask, qsmask, level, grplo, grphi, gp_tasks),
 307
 308         TP_FIELDS(
 309                 ctf_string(rcuname, rcuname)
 310                 ctf_integer(unsigned long, gpnum, gpnum)
 311                 ctf_integer(unsigned long, mask, mask)
 312                 ctf_integer(unsigned long, qsmask, qsmask)
 313                 ctf_integer(u8, level, level)
 314                 ctf_integer(int, grplo, grplo)
 315                 ctf_integer(int, grphi, grphi)
 316                 ctf_integer(u8, gp_tasks, gp_tasks)
 317         )
 318 )
 319 #endif
 320
 321 /*
 322  * Tracepoint for quiescent states detected by force_quiescent_state().
 323  * These trace events include the type of RCU, the grace-period number
 324  * that was blocked by the CPU, the CPU itself, and the type of quiescent
 325  * state, which can be "dti" for dyntick-idle mode, "ofl" for CPU offline,
 326  * or "kick" when kicking a CPU that has been in dyntick-idle mode for
 327  * too long.
 328  */
 329 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,19,0) || \
 330         LTTNG_RHEL_KERNEL_RANGE(4,18,0,80,0,0, 4,19,0,0,0,0))
 331 LTTNG_TRACEPOINT_EVENT(rcu_fqs,
 332
 333         TP_PROTO(const char *rcuname, unsigned long gp_seq, int cpu, const char *qsevent),
 334
 335         TP_ARGS(rcuname, gp_seq, cpu, qsevent),
 336
 337         TP_FIELDS(
 338                 ctf_integer(unsigned long, gp_seq, gp_seq)
 339                 ctf_integer(int, cpu, cpu)
 340                 ctf_string(rcuname, rcuname)
 341                 ctf_string(qsevent, qsevent)
 342         )
 343 )
 344 #elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3,12,0))
 345 LTTNG_TRACEPOINT_EVENT(rcu_fqs,
 346
 347         TP_PROTO(const char *rcuname, unsigned long gpnum, int cpu, const char *qsevent),
 348
 349         TP_ARGS(rcuname, gpnum, cpu, qsevent),
 350
 351         TP_FIELDS(
 352                 ctf_integer(unsigned long, gpnum, gpnum)
 353                 ctf_integer(int, cpu, cpu)
 354                 ctf_string(rcuname, rcuname)
 355                 ctf_string(qsevent, qsevent)
 356         )
 357 )
 358 #else
 359 LTTNG_TRACEPOINT_EVENT(rcu_fqs,
 360
 361         TP_PROTO(char *rcuname, unsigned long gpnum, int cpu, char *qsevent),
 362
 363         TP_ARGS(rcuname, gpnum, cpu, qsevent),
 364
 365         TP_FIELDS(
 366                 ctf_integer(unsigned long, gpnum, gpnum)
 367                 ctf_integer(int, cpu, cpu)
 368                 ctf_string(rcuname, rcuname)
 369                 ctf_string(qsevent, qsevent)
 370         )
 371 )
 372 #endif
 373
 374 #endif  /*
 375          * #if defined(CONFIG_TREE_RCU)
 376          *      || (LINUX_VERSION_CODE >= KERNEL_VERSION(3,19,0)
 377          *              && defined(CONFIG_PREEMPT_RCU))
 378          *      || defined(CONFIG_TREE_PREEMPT_RCU)
 379          */
 380
 381 /*
 382  * Tracepoint for dyntick-idle entry/exit events.  These take a string
 383  * as argument: "Start" for entering dyntick-idle mode, "End" for
 384  * leaving it, "--=" for events moving towards idle, and "++=" for events
 385  * moving away from idle.  "Error on entry: not idle task" and "Error on
 386  * exit: not idle task" indicate that a non-idle task is erroneously
 387  * toying with the idle loop.
 388  *
 389  * These events also take a pair of numbers, which indicate the nesting
 390  * depth before and after the event of interest.  Note that task-related
 391  * events use the upper bits of each number, while interrupt-related
 392  * events use the lower bits.
 393  */
 394 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0))       \
 395         || LTTNG_KERNEL_RANGE(5,5,6, 5,6,0)             \
 396         || LTTNG_KERNEL_RANGE(5,4,22, 5,5,0)
 397 LTTNG_TRACEPOINT_EVENT(rcu_dyntick,
 398
 399         TP_PROTO(const char *polarity, long oldnesting, long newnesting, int dynticks),
 400
 401         TP_ARGS(polarity, oldnesting, newnesting, dynticks),
 402
 403         TP_FIELDS(
 404                 ctf_string(polarity, polarity)
 405                 ctf_integer(long, oldnesting, oldnesting)
 406                 ctf_integer(long, newnesting, newnesting)
 407                 ctf_integer(int, dynticks, dynticks)
 408         )
 409 )
 410
 411 #elif (LINUX_VERSION_CODE >= KERNEL_VERSION(4,16,0))
 412 LTTNG_TRACEPOINT_EVENT(rcu_dyntick,
 413
 414         TP_PROTO(const char *polarity, long oldnesting, long newnesting, atomic_t dynticks),
 415
 416         TP_ARGS(polarity, oldnesting, newnesting, dynticks),
 417
 418         TP_FIELDS(
 419                 ctf_string(polarity, polarity)
 420                 ctf_integer(long, oldnesting, oldnesting)
 421                 ctf_integer(long, newnesting, newnesting)
 422                 ctf_integer(int, dynticks, atomic_read(&dynticks))
 423         )
 424 )
 425
 426 #elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3,12,0))
 427 LTTNG_TRACEPOINT_EVENT(rcu_dyntick,
 428
 429         TP_PROTO(const char *polarity, long long oldnesting, long long newnesting),
 430
 431         TP_ARGS(polarity, oldnesting, newnesting),
 432
 433         TP_FIELDS(
 434                 ctf_string(polarity, polarity)
 435                 ctf_integer(long long, oldnesting, oldnesting)
 436                 ctf_integer(long long, newnesting, newnesting)
 437         )
 438 )
 439 #elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3,3,0))
 440 LTTNG_TRACEPOINT_EVENT(rcu_dyntick,
 441
 442         TP_PROTO(char *polarity, long long oldnesting, long long newnesting),
 443
 444         TP_ARGS(polarity, oldnesting, newnesting),
 445
 446         TP_FIELDS(
 447                 ctf_string(polarity, polarity)
 448                 ctf_integer(long long, oldnesting, oldnesting)
 449                 ctf_integer(long long, newnesting, newnesting)
 450         )
 451 )
 452 #else
 453 LTTNG_TRACEPOINT_EVENT(rcu_dyntick,
 454
 455         TP_PROTO(char *polarity),
 456
 457         TP_ARGS(polarity),
 458
 459         TP_FIELDS(
 460                 ctf_string(polarity, polarity)
 461         )
 462 )
 463 #endif
 464
 465
 466 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,3,0))
 467 /*
 468  * Tracepoint for RCU preparation for idle, the goal being to get RCU
 469  * processing done so that the current CPU can shut off its scheduling
 470  * clock and enter dyntick-idle mode.  One way to accomplish this is
 471  * to drain all RCU callbacks from this CPU, and the other is to have
 472  * done everything RCU requires for the current grace period.  In this
 473  * latter case, the CPU will be awakened at the end of the current grace
 474  * period in order to process the remainder of its callbacks.
 475  *
 476  * These tracepoints take a string as argument:
 477  *
 478  *      "No callbacks": Nothing to do, no callbacks on this CPU.
 479  *      "In holdoff": Nothing to do, holding off after unsuccessful attempt.
 480  *      "Begin holdoff": Attempt failed, don't retry until next jiffy.
 481  *      "Dyntick with callbacks": Entering dyntick-idle despite callbacks.
 482  *      "Dyntick with lazy callbacks": Entering dyntick-idle w/lazy callbacks.
 483  *      "More callbacks": Still more callbacks, try again to clear them out.
 484  *      "Callbacks drained": All callbacks processed, off to dyntick idle!
 485  *      "Timer": Timer fired to cause CPU to continue processing callbacks.
 486  *      "Demigrate": Timer fired on wrong CPU, woke up correct CPU.
 487  *      "Cleanup after idle": Idle exited, timer canceled.
 488  */
 489 LTTNG_TRACEPOINT_EVENT(rcu_prep_idle,
 490
 491 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,12,0))
 492         TP_PROTO(const char *reason),
 493 #else /* #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,12,0)) */
 494         TP_PROTO(char *reason),
 495 #endif /* #else #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,12,0)) */
 496
 497         TP_ARGS(reason),
 498
 499         TP_FIELDS(
 500                 ctf_string(reason, reason)
 501         )
 502 )
 503 #endif
 504
 505 /*
 506  * Tracepoint for the registration of a single RCU callback function.
 507  * The first argument is the type of RCU, the second argument is
 508  * a pointer to the RCU callback itself, the third element is the
 509  * number of lazy callbacks queued, and the fourth element is the
 510  * total number of callbacks queued.
 511  */
 512 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0))
 513 LTTNG_TRACEPOINT_EVENT(rcu_callback,
 514
 515         TP_PROTO(const char *rcuname, struct rcu_head *rhp, long qlen),
 516
 517         TP_ARGS(rcuname, rhp, qlen),
 518
 519         TP_FIELDS(
 520                 ctf_string(rcuname, rcuname)
 521                 ctf_integer_hex(void *, rhp, rhp)
 522                 ctf_integer_hex(void *, func, rhp->func)
 523                 ctf_integer(long, qlen, qlen)
 524         )
 525 )
 526 #elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3,12,0))
 527 LTTNG_TRACEPOINT_EVENT(rcu_callback,
 528
 529         TP_PROTO(const char *rcuname, struct rcu_head *rhp, long qlen_lazy,
 530                  long qlen),
 531
 532         TP_ARGS(rcuname, rhp, qlen_lazy, qlen),
 533
 534         TP_FIELDS(
 535                 ctf_string(rcuname, rcuname)
 536                 ctf_integer_hex(void *, rhp, rhp)
 537                 ctf_integer_hex(void *, func, rhp->func)
 538                 ctf_integer(long, qlen_lazy, qlen_lazy)
 539                 ctf_integer(long, qlen, qlen)
 540         )
 541 )
 542 #elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3,4,0))
 543 LTTNG_TRACEPOINT_EVENT(rcu_callback,
 544
 545         TP_PROTO(char *rcuname, struct rcu_head *rhp, long qlen_lazy,
 546                  long qlen),
 547
 548         TP_ARGS(rcuname, rhp, qlen_lazy, qlen),
 549
 550         TP_FIELDS(
 551                 ctf_string(rcuname, rcuname)
 552                 ctf_integer_hex(void *, rhp, rhp)
 553                 ctf_integer_hex(void *, func, rhp->func)
 554                 ctf_integer(long, qlen_lazy, qlen_lazy)
 555                 ctf_integer(long, qlen, qlen)
 556         )
 557 )
 558 #else
 559 LTTNG_TRACEPOINT_EVENT(rcu_callback,
 560
 561         TP_PROTO(char *rcuname, struct rcu_head *rhp, long qlen),
 562
 563         TP_ARGS(rcuname, rhp, qlen),
 564
 565         TP_FIELDS(
 566                 ctf_string(rcuname, rcuname)
 567                 ctf_integer_hex(void *, rhp, rhp)
 568                 ctf_integer_hex(void *, func, rhp->func)
 569                 ctf_integer(long, qlen, qlen)
 570         )
 571 )
 572 #endif
 573
 574
 575 /*
 576  * Tracepoint for the registration of a single RCU callback of the special
 577  * kfree() form.  The first argument is the RCU type, the second argument
 578  * is a pointer to the RCU callback, the third argument is the offset
 579  * of the callback within the enclosing RCU-protected data structure,
 580  * the fourth argument is the number of lazy callbacks queued, and the
 581  * fifth argument is the total number of callbacks queued.
 582  */
 583 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0))
 584 LTTNG_TRACEPOINT_EVENT(rcu_kfree_callback,
 585
 586         TP_PROTO(const char *rcuname, struct rcu_head *rhp, unsigned long offset,
 587                  long qlen),
 588
 589         TP_ARGS(rcuname, rhp, offset, qlen),
 590
 591         TP_FIELDS(
 592                 ctf_string(rcuname, rcuname)
 593                 ctf_integer_hex(void *, rhp, rhp)
 594                 ctf_integer_hex(unsigned long, offset, offset)
 595                 ctf_integer(long, qlen, qlen)
 596         )
 597 )
 598 #elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3,12,0))
 599 LTTNG_TRACEPOINT_EVENT(rcu_kfree_callback,
 600
 601         TP_PROTO(const char *rcuname, struct rcu_head *rhp, unsigned long offset,
 602                  long qlen_lazy, long qlen),
 603
 604         TP_ARGS(rcuname, rhp, offset, qlen_lazy, qlen),
 605
 606         TP_FIELDS(
 607                 ctf_string(rcuname, rcuname)
 608                 ctf_integer_hex(void *, rhp, rhp)
 609                 ctf_integer_hex(unsigned long, offset, offset)
 610                 ctf_integer(long, qlen_lazy, qlen_lazy)
 611                 ctf_integer(long, qlen, qlen)
 612         )
 613 )
 614 #elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3,4,0))
 615 LTTNG_TRACEPOINT_EVENT(rcu_kfree_callback,
 616
 617         TP_PROTO(char *rcuname, struct rcu_head *rhp, unsigned long offset,
 618                  long qlen_lazy, long qlen),
 619
 620         TP_ARGS(rcuname, rhp, offset, qlen_lazy, qlen),
 621
 622         TP_FIELDS(
 623                 ctf_string(rcuname, rcuname)
 624                 ctf_integer_hex(void *, rhp, rhp)
 625                 ctf_integer_hex(unsigned long, offset, offset)
 626                 ctf_integer(long, qlen_lazy, qlen_lazy)
 627                 ctf_integer(long, qlen, qlen)
 628         )
 629 )
 630 #else
 631 LTTNG_TRACEPOINT_EVENT(rcu_kfree_callback,
 632
 633         TP_PROTO(char *rcuname, struct rcu_head *rhp, unsigned long offset,
 634                  long qlen),
 635
 636         TP_ARGS(rcuname, rhp, offset, qlen),
 637
 638         TP_FIELDS(
 639                 ctf_string(rcuname, rcuname)
 640                 ctf_integer_hex(void *, rhp, rhp)
 641                 ctf_integer_hex(unsigned long, offset, offset)
 642                 ctf_integer(long, qlen, qlen)
 643         )
 644 )
 645 #endif
 646
 647 /*
 648  * Tracepoint for marking the beginning rcu_do_batch, performed to start
 649  * RCU callback invocation.  The first argument is the RCU flavor,
 650  * the second is the number of lazy callbacks queued, the third is
 651  * the total number of callbacks queued, and the fourth argument is
 652  * the current RCU-callback batch limit.
 653  */
 654 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0))
 655 LTTNG_TRACEPOINT_EVENT(rcu_batch_start,
 656
 657         TP_PROTO(const char *rcuname, long qlen, long blimit),
 658
 659         TP_ARGS(rcuname, qlen, blimit),
 660
 661         TP_FIELDS(
 662                 ctf_string(rcuname, rcuname)
 663                 ctf_integer(long, qlen, qlen)
 664                 ctf_integer(long, blimit, blimit)
 665         )
 666 )
 667 #elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3,12,0))
 668 LTTNG_TRACEPOINT_EVENT(rcu_batch_start,
 669
 670         TP_PROTO(const char *rcuname, long qlen_lazy, long qlen, long blimit),
 671
 672         TP_ARGS(rcuname, qlen_lazy, qlen, blimit),
 673
 674         TP_FIELDS(
 675                 ctf_string(rcuname, rcuname)
 676                 ctf_integer(long, qlen_lazy, qlen_lazy)
 677                 ctf_integer(long, qlen, qlen)
 678                 ctf_integer(long, blimit, blimit)
 679         )
 680 )
 681 #elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3,9,0))
 682 LTTNG_TRACEPOINT_EVENT(rcu_batch_start,
 683
 684         TP_PROTO(char *rcuname, long qlen_lazy, long qlen, long blimit),
 685
 686         TP_ARGS(rcuname, qlen_lazy, qlen, blimit),
 687
 688         TP_FIELDS(
 689                 ctf_string(rcuname, rcuname)
 690                 ctf_integer(long, qlen_lazy, qlen_lazy)
 691                 ctf_integer(long, qlen, qlen)
 692                 ctf_integer(long, blimit, blimit)
 693         )
 694 )
 695 #elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3,4,0))
 696 LTTNG_TRACEPOINT_EVENT(rcu_batch_start,
 697
 698         TP_PROTO(char *rcuname, long qlen_lazy, long qlen, int blimit),
 699
 700         TP_ARGS(rcuname, qlen_lazy, qlen, blimit),
 701
 702         TP_FIELDS(
 703                 ctf_string(rcuname, rcuname)
 704                 ctf_integer(long, qlen_lazy, qlen_lazy)
 705                 ctf_integer(long, qlen, qlen)
 706                 ctf_integer(int, blimit, blimit)
 707         )
 708 )
 709 #else
 710 LTTNG_TRACEPOINT_EVENT(rcu_batch_start,
 711
 712         TP_PROTO(char *rcuname, long qlen, int blimit),
 713
 714         TP_ARGS(rcuname, qlen, blimit),
 715
 716         TP_FIELDS(
 717                 ctf_string(rcuname, rcuname)
 718                 ctf_integer(long, qlen, qlen)
 719                 ctf_integer(int, blimit, blimit)
 720         )
 721 )
 722 #endif
 723
 724 /*
 725  * Tracepoint for the invocation of a single RCU callback function.
 726  * The first argument is the type of RCU, and the second argument is
 727  * a pointer to the RCU callback itself.
 728  */
 729 LTTNG_TRACEPOINT_EVENT(rcu_invoke_callback,
 730
 731 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,12,0))
 732         TP_PROTO(const char *rcuname, struct rcu_head *rhp),
 733 #else /* #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,12,0)) */
 734         TP_PROTO(char *rcuname, struct rcu_head *rhp),
 735 #endif /* #else #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,12,0)) */
 736
 737         TP_ARGS(rcuname, rhp),
 738
 739         TP_FIELDS(
 740                 ctf_string(rcuname, rcuname)
 741                 ctf_integer_hex(void *, rhp, rhp)
 742                 ctf_integer_hex(void *, func, rhp->func)
 743         )
 744 )
 745
 746 /*
 747  * Tracepoint for the invocation of a single RCU callback of the special
 748  * kfree() form.  The first argument is the RCU flavor, the second
 749  * argument is a pointer to the RCU callback, and the third argument
 750  * is the offset of the callback within the enclosing RCU-protected
 751  * data structure.
 752  */
 753 LTTNG_TRACEPOINT_EVENT(rcu_invoke_kfree_callback,
 754
 755 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,12,0))
 756         TP_PROTO(const char *rcuname, struct rcu_head *rhp, unsigned long offset),
 757 #else /* #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,12,0)) */
 758         TP_PROTO(char *rcuname, struct rcu_head *rhp, unsigned long offset),
 759 #endif /* #else #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,12,0)) */
 760
 761         TP_ARGS(rcuname, rhp, offset),
 762
 763         TP_FIELDS(
 764                 ctf_string(rcuname, rcuname)
 765                 ctf_integer_hex(void *, rhp, rhp)
 766                 ctf_integer(unsigned long, offset, offset)
 767         )
 768 )
 769
 770 /*
 771  * Tracepoint for exiting rcu_do_batch after RCU callbacks have been
 772  * invoked.  The first argument is the name of the RCU flavor,
 773  * the second argument is number of callbacks actually invoked,
 774  * the third argument (cb) is whether or not any of the callbacks that
 775  * were ready to invoke at the beginning of this batch are still
 776  * queued, the fourth argument (nr) is the return value of need_resched(),
 777  * the fifth argument (iit) is 1 if the current task is the idle task,
 778  * and the sixth argument (risk) is the return value from
 779  * rcu_is_callbacks_kthread().
 780  */
 781 LTTNG_TRACEPOINT_EVENT(rcu_batch_end,
 782
 783 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,13,0))
 784         TP_PROTO(const char *rcuname, int callbacks_invoked,
 785                  char cb, char nr, char iit, char risk),
 786
 787         TP_ARGS(rcuname, callbacks_invoked, cb, nr, iit, risk),
 788 #elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3,12,0))
 789         TP_PROTO(const char *rcuname, int callbacks_invoked,
 790                  bool cb, bool nr, bool iit, bool risk),
 791
 792         TP_ARGS(rcuname, callbacks_invoked, cb, nr, iit, risk),
 793 #elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3,3,0))
 794         TP_PROTO(char *rcuname, int callbacks_invoked,
 795                  bool cb, bool nr, bool iit, bool risk),
 796
 797         TP_ARGS(rcuname, callbacks_invoked, cb, nr, iit, risk),
 798 #else
 799         TP_PROTO(char *rcuname, int callbacks_invoked),
 800
 801         TP_ARGS(rcuname, callbacks_invoked),
 802 #endif
 803
 804         TP_FIELDS(
 805                 ctf_string(rcuname, rcuname)
 806                 ctf_integer(int, callbacks_invoked, callbacks_invoked)
 807 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,13,0))
 808                 ctf_integer(char, cb, cb)
 809                 ctf_integer(char, nr, nr)
 810                 ctf_integer(char, iit, iit)
 811                 ctf_integer(char, risk, risk)
 812 #elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3,3,0))
 813                 ctf_integer(bool, cb, cb)
 814                 ctf_integer(bool, nr, nr)
 815                 ctf_integer(bool, iit, iit)
 816                 ctf_integer(bool, risk, risk)
 817 #endif
 818         )
 819 )
 820
 821 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,3,0))
 822 /*
 823  * Tracepoint for rcutorture readers.  The first argument is the name
 824  * of the RCU flavor from rcutorture's viewpoint and the second argument
 825  * is the callback address.
 826  */
 827 LTTNG_TRACEPOINT_EVENT(rcu_torture_read,
 828
 829 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,12,0))
 830         TP_PROTO(const char *rcutorturename, struct rcu_head *rhp,
 831                  unsigned long secs, unsigned long c_old, unsigned long c),
 832
 833         TP_ARGS(rcutorturename, rhp, secs, c_old, c),
 834 #elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3,9,0))
 835         TP_PROTO(char *rcutorturename, struct rcu_head *rhp,
 836                  unsigned long secs, unsigned long c_old, unsigned long c),
 837
 838         TP_ARGS(rcutorturename, rhp, secs, c_old, c),
 839 #else
 840         TP_PROTO(char *rcutorturename, struct rcu_head *rhp),
 841
 842         TP_ARGS(rcutorturename, rhp),
 843 #endif
 844
 845         TP_FIELDS(
 846                 ctf_string(rcutorturename, rcutorturename)
 847                 ctf_integer_hex(struct rcu_head *, rhp, rhp)
 848 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,9,0))
 849                 ctf_integer(unsigned long, secs, secs)
 850                 ctf_integer(unsigned long, c_old, c_old)
 851                 ctf_integer(unsigned long, c, c)
 852 #endif
 853         )
 854 )
 855 #endif
 856
 857 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,6,0))
 858 /*
 859  * Tracepoint for _rcu_barrier() execution.  The string "s" describes
 860  * the _rcu_barrier phase:
 861  *      "Begin": rcu_barrier_callback() started.
 862  *      "Check": rcu_barrier_callback() checking for piggybacking.
 863  *      "EarlyExit": rcu_barrier_callback() piggybacked, thus early exit.
 864  *      "Inc1": rcu_barrier_callback() piggyback check counter incremented.
 865  *      "Offline": rcu_barrier_callback() found offline CPU
 866  *      "OnlineQ": rcu_barrier_callback() found online CPU with callbacks.
 867  *      "OnlineNQ": rcu_barrier_callback() found online CPU, no callbacks.
 868  *      "IRQ": An rcu_barrier_callback() callback posted on remote CPU.
 869  *      "CB": An rcu_barrier_callback() invoked a callback, not the last.
 870  *      "LastCB": An rcu_barrier_callback() invoked the last callback.
 871  *      "Inc2": rcu_barrier_callback() piggyback check counter incremented.
 872  * The "cpu" argument is the CPU or -1 if meaningless, the "cnt" argument
 873  * is the count of remaining callbacks, and "done" is the piggybacking count.
 874  */
 875 LTTNG_TRACEPOINT_EVENT(rcu_barrier,
 876
 877 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,12,0))
 878         TP_PROTO(const char *rcuname, const char *s, int cpu, int cnt, unsigned long done),
 879 #else /* #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,12,0)) */
 880         TP_PROTO(char *rcuname, char *s, int cpu, int cnt, unsigned long done),
 881 #endif /* #else #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,12,0)) */
 882
 883         TP_ARGS(rcuname, s, cpu, cnt, done),
 884
 885         TP_FIELDS(
 886                 ctf_string(rcuname, rcuname)
 887                 ctf_string(s, s)
 888                 ctf_integer(int, cpu, cpu)
 889                 ctf_integer(int, cnt, cnt)
 890                 ctf_integer(unsigned long, done, done)
 891         )
 892 )
 893 #endif
 894
 895 #else /* #ifdef CONFIG_RCU_TRACE */
 896
 897 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,19,0) || \
 898         LTTNG_RHEL_KERNEL_RANGE(4,18,0,80,0,0, 4,19,0,0,0,0))
 899 #define trace_rcu_grace_period(rcuname, gp_seq, gpevent) do { } while (0)
 900 #define trace_rcu_grace_period_init(rcuname, gp_seq, level, grplo, grphi, \
 901                                     qsmask) do { } while (0)
 902 #define trace_rcu_preempt_task(rcuname, pid, gp_seq) do { } while (0)
 903 #define trace_rcu_unlock_preempted_task(rcuname, gp_seq, pid) do { } while (0)
 904 #define trace_rcu_quiescent_state_report(rcuname, gp_seq, mask, qsmask, level, \
 905                                          grplo, grphi, gp_tasks) do { } \
 906         while (0)
 907 #define trace_rcu_fqs(rcuname, gp_seq, cpu, qsevent) do { } while (0)
 908 #else
 909 #define trace_rcu_grace_period(rcuname, gpnum, gpevent) do { } while (0)
 910 #define trace_rcu_grace_period_init(rcuname, gpnum, level, grplo, grphi, \
 911                                     qsmask) do { } while (0)
 912 #define trace_rcu_preempt_task(rcuname, pid, gpnum) do { } while (0)
 913 #define trace_rcu_unlock_preempted_task(rcuname, gpnum, pid) do { } while (0)
 914 #define trace_rcu_quiescent_state_report(rcuname, gpnum, mask, qsmask, level, \
 915                                          grplo, grphi, gp_tasks) do { } \
 916         while (0)
 917 #define trace_rcu_fqs(rcuname, gpnum, cpu, qsevent) do { } while (0)
 918 #endif
 919
 920 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,16,0))
 921 #define trace_rcu_dyntick(polarity, oldnesting, newnesting, dyntick) do { } while (0)
 922 #elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3,3,0))
 923 #define trace_rcu_dyntick(polarity, oldnesting, newnesting) do { } while (0)
 924 #else
 925 #define trace_rcu_dyntick(polarity) do { } while (0)
 926 #endif
 927 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,3,0))
 928 #define trace_rcu_prep_idle(reason) do { } while (0)
 929 #endif
 930 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,4,0))
 931 #define trace_rcu_callback(rcuname, rhp, qlen_lazy, qlen) do { } while (0)
 932 #define trace_rcu_kfree_callback(rcuname, rhp, offset, qlen_lazy, qlen) \
 933         do { } while (0)
 934 #define trace_rcu_batch_start(rcuname, qlen_lazy, qlen, blimit) \
 935         do { } while (0)
 936 #else
 937 #define trace_rcu_callback(rcuname, rhp, qlen) do { } while (0)
 938 #define trace_rcu_kfree_callback(rcuname, rhp, offset, qlen) do { } while (0)
 939 #define trace_rcu_batch_start(rcuname, qlen, blimit) do { } while (0)
 940 #endif
 941 #define trace_rcu_invoke_callback(rcuname, rhp) do { } while (0)
 942 #define trace_rcu_invoke_kfree_callback(rcuname, rhp, offset) do { } while (0)
 943 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,3,0))
 944 #define trace_rcu_batch_end(rcuname, callbacks_invoked, cb, nr, iit, risk) \
 945         do { } while (0)
 946 #else
 947 #define trace_rcu_batch_end(rcuname, callbacks_invoked) do { } while (0)
 948 #endif
 949 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,9,0))
 950 #define trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c) \
 951         do { } while (0)
 952 #elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3,3,0))
 953 #define trace_rcu_torture_read(rcutorturename, rhp) do { } while (0)
 954 #endif
 955 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,6,0))
 956 #define trace_rcu_barrier(name, s, cpu, cnt, done) do { } while (0)
 957 #endif
 958 #endif /* #else #ifdef CONFIG_RCU_TRACE */
 959
 960 #endif /* LTTNG_TRACE_RCU_H */
 961
 962 /* This part must be outside protection */
 963 #include <probes/define_trace.h>