include/instrumentation/events/rcu.h

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 #undef TRACE_SYSTEM
   3 #define TRACE_SYSTEM rcu
   4
   5 #if !defined(LTTNG_TRACE_RCU_H) || defined(TRACE_HEADER_MULTI_READ)
   6 #define LTTNG_TRACE_RCU_H
   7
   8 #include <lttng/tracepoint-event.h>
   9 #include <lttng/kernel-version.h>
  10
  11 /*
  12  * Tracepoint for start/end markers used for utilization calculations.
  13  * By convention, the string is of the following forms:
  14  *
  15  * "Start <activity>" -- Mark the start of the specified activity,
  16  *                       such as "context switch".  Nesting is permitted.
  17  * "End <activity>" -- Mark the end of the specified activity.
  18  *
  19  * An "@" character within "<activity>" is a comment character: Data
  20  * reduction scripts will ignore the "@" and the remainder of the line.
  21  */
  22 LTTNG_TRACEPOINT_EVENT(rcu_utilization,
  23
  24 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,12,0))
  25         TP_PROTO(const char *s),
  26 #else /* #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,12,0)) */
  27         TP_PROTO(char *s),
  28 #endif /* #else #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,12,0)) */
  29
  30         TP_ARGS(s),
  31
  32         TP_FIELDS(
  33                 ctf_string(s, s)
  34         )
  35 )
  36
  37 #ifdef CONFIG_RCU_TRACE
  38
  39 #if defined(CONFIG_TREE_RCU) \
  40         || (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,19,0) \
  41                 && defined(CONFIG_PREEMPT_RCU)) \
  42         || defined(CONFIG_TREE_PREEMPT_RCU)
  43
  44 /*
  45  * Tracepoint for grace-period events: starting and ending a grace
  46  * period ("start" and "end", respectively), a CPU noting the start
  47  * of a new grace period or the end of an old grace period ("cpustart"
  48  * and "cpuend", respectively), a CPU passing through a quiescent
  49  * state ("cpuqs"), a CPU coming online or going offline ("cpuonl"
  50  * and "cpuofl", respectively), and a CPU being kicked for being too
  51  * long in dyntick-idle mode ("kick").
  52  */
  53 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,19,0) || \
  54         LTTNG_RHEL_KERNEL_RANGE(4,18,0,80,0,0, 4,19,0,0,0,0))
  55 LTTNG_TRACEPOINT_EVENT(rcu_grace_period,
  56
  57         TP_PROTO(const char *rcuname, unsigned long gp_seq, const char *gpevent),
  58
  59         TP_ARGS(rcuname, gp_seq, gpevent),
  60
  61         TP_FIELDS(
  62                 ctf_string(rcuname, rcuname)
  63                 ctf_integer(unsigned long, gp_seq, gp_seq)
  64                 ctf_string(gpevent, gpevent)
  65         )
  66 )
  67 #elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,12,0))
  68 LTTNG_TRACEPOINT_EVENT(rcu_grace_period,
  69
  70         TP_PROTO(const char *rcuname, unsigned long gpnum, const char *gpevent),
  71
  72         TP_ARGS(rcuname, gpnum, gpevent),
  73
  74         TP_FIELDS(
  75                 ctf_string(rcuname, rcuname)
  76                 ctf_integer(unsigned long, gpnum, gpnum)
  77                 ctf_string(gpevent, gpevent)
  78         )
  79 )
  80 #else
  81 LTTNG_TRACEPOINT_EVENT(rcu_grace_period,
  82
  83         TP_PROTO(char *rcuname, unsigned long gpnum, char *gpevent),
  84
  85         TP_ARGS(rcuname, gpnum, gpevent),
  86
  87         TP_FIELDS(
  88                 ctf_string(rcuname, rcuname)
  89                 ctf_integer(unsigned long, gpnum, gpnum)
  90                 ctf_string(gpevent, gpevent)
  91         )
  92 )
  93 #endif
  94
  95 /*
  96  * Tracepoint for grace-period-initialization events.  These are
  97  * distinguished by the type of RCU, the new grace-period number, the
  98  * rcu_node structure level, the starting and ending CPU covered by the
  99  * rcu_node structure, and the mask of CPUs that will be waited for.
 100  * All but the type of RCU are extracted from the rcu_node structure.
 101  */
 102 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,19,0) || \
 103         LTTNG_RHEL_KERNEL_RANGE(4,18,0,80,0,0, 4,19,0,0,0,0))
 104 LTTNG_TRACEPOINT_EVENT(rcu_grace_period_init,
 105
 106         TP_PROTO(const char *rcuname, unsigned long gp_seq, u8 level,
 107                  int grplo, int grphi, unsigned long qsmask),
 108
 109         TP_ARGS(rcuname, gp_seq, level, grplo, grphi, qsmask),
 110
 111         TP_FIELDS(
 112                 ctf_string(rcuname, rcuname)
 113                 ctf_integer(unsigned long, gp_seq, gp_seq)
 114                 ctf_integer(u8, level, level)
 115                 ctf_integer(int, grplo, grplo)
 116                 ctf_integer(int, grphi, grphi)
 117                 ctf_integer(unsigned long, qsmask, qsmask)
 118         )
 119 )
 120 #elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,12,0))
 121 LTTNG_TRACEPOINT_EVENT(rcu_grace_period_init,
 122
 123         TP_PROTO(const char *rcuname, unsigned long gpnum, u8 level,
 124                  int grplo, int grphi, unsigned long qsmask),
 125
 126         TP_ARGS(rcuname, gpnum, level, grplo, grphi, qsmask),
 127
 128         TP_FIELDS(
 129                 ctf_string(rcuname, rcuname)
 130                 ctf_integer(unsigned long, gpnum, gpnum)
 131                 ctf_integer(u8, level, level)
 132                 ctf_integer(int, grplo, grplo)
 133                 ctf_integer(int, grphi, grphi)
 134                 ctf_integer(unsigned long, qsmask, qsmask)
 135         )
 136 )
 137 #else
 138 LTTNG_TRACEPOINT_EVENT(rcu_grace_period_init,
 139
 140         TP_PROTO(char *rcuname, unsigned long gpnum, u8 level,
 141                  int grplo, int grphi, unsigned long qsmask),
 142
 143         TP_ARGS(rcuname, gpnum, level, grplo, grphi, qsmask),
 144
 145         TP_FIELDS(
 146                 ctf_string(rcuname, rcuname)
 147                 ctf_integer(unsigned long, gpnum, gpnum)
 148                 ctf_integer(u8, level, level)
 149                 ctf_integer(int, grplo, grplo)
 150                 ctf_integer(int, grphi, grphi)
 151                 ctf_integer(unsigned long, qsmask, qsmask)
 152         )
 153 )
 154 #endif
 155
 156 /*
 157  * Tracepoint for tasks blocking within preemptible-RCU read-side
 158  * critical sections.  Track the type of RCU (which one day might
 159  * include SRCU), the grace-period number that the task is blocking
 160  * (the current or the next), and the task's PID.
 161  */
 162 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,19,0) || \
 163         LTTNG_RHEL_KERNEL_RANGE(4,18,0,80,0,0, 4,19,0,0,0,0))
 164 LTTNG_TRACEPOINT_EVENT(rcu_preempt_task,
 165
 166         TP_PROTO(const char *rcuname, int pid, unsigned long gp_seq),
 167
 168         TP_ARGS(rcuname, pid, gp_seq),
 169
 170         TP_FIELDS(
 171                 ctf_string(rcuname, rcuname)
 172                 ctf_integer(unsigned long, gp_seq, gp_seq)
 173                 ctf_integer(int, pid, pid)
 174         )
 175 )
 176 #elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,12,0))
 177 LTTNG_TRACEPOINT_EVENT(rcu_preempt_task,
 178
 179         TP_PROTO(const char *rcuname, int pid, unsigned long gpnum),
 180
 181         TP_ARGS(rcuname, pid, gpnum),
 182
 183         TP_FIELDS(
 184                 ctf_string(rcuname, rcuname)
 185                 ctf_integer(unsigned long, gpnum, gpnum)
 186                 ctf_integer(int, pid, pid)
 187         )
 188 )
 189 #else
 190 LTTNG_TRACEPOINT_EVENT(rcu_preempt_task,
 191
 192         TP_PROTO(char *rcuname, int pid, unsigned long gpnum),
 193
 194         TP_ARGS(rcuname, pid, gpnum),
 195
 196         TP_FIELDS(
 197                 ctf_string(rcuname, rcuname)
 198                 ctf_integer(unsigned long, gpnum, gpnum)
 199                 ctf_integer(int, pid, pid)
 200         )
 201 )
 202 #endif
 203
 204 /*
 205  * Tracepoint for tasks that blocked within a given preemptible-RCU
 206  * read-side critical section exiting that critical section.  Track the
 207  * type of RCU (which one day might include SRCU) and the task's PID.
 208  */
 209 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,19,0) || \
 210         LTTNG_RHEL_KERNEL_RANGE(4,18,0,80,0,0, 4,19,0,0,0,0))
 211 LTTNG_TRACEPOINT_EVENT(rcu_unlock_preempted_task,
 212
 213         TP_PROTO(const char *rcuname, unsigned long gp_seq, int pid),
 214
 215         TP_ARGS(rcuname, gp_seq, pid),
 216
 217         TP_FIELDS(
 218                 ctf_string(rcuname, rcuname)
 219                 ctf_integer(unsigned long, gp_seq, gp_seq)
 220                 ctf_integer(int, pid, pid)
 221         )
 222 )
 223 #elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,12,0))
 224 LTTNG_TRACEPOINT_EVENT(rcu_unlock_preempted_task,
 225
 226         TP_PROTO(const char *rcuname, unsigned long gpnum, int pid),
 227
 228         TP_ARGS(rcuname, gpnum, pid),
 229
 230         TP_FIELDS(
 231                 ctf_string(rcuname, rcuname)
 232                 ctf_integer(unsigned long, gpnum, gpnum)
 233                 ctf_integer(int, pid, pid)
 234         )
 235 )
 236 #else
 237 LTTNG_TRACEPOINT_EVENT(rcu_unlock_preempted_task,
 238
 239         TP_PROTO(char *rcuname, unsigned long gpnum, int pid),
 240
 241         TP_ARGS(rcuname, gpnum, pid),
 242
 243         TP_FIELDS(
 244                 ctf_string(rcuname, rcuname)
 245                 ctf_integer(unsigned long, gpnum, gpnum)
 246                 ctf_integer(int, pid, pid)
 247         )
 248 )
 249 #endif
 250
 251 /*
 252  * Tracepoint for quiescent-state-reporting events.  These are
 253  * distinguished by the type of RCU, the grace-period number, the
 254  * mask of quiescent lower-level entities, the rcu_node structure level,
 255  * the starting and ending CPU covered by the rcu_node structure, and
 256  * whether there are any blocked tasks blocking the current grace period.
 257  * All but the type of RCU are extracted from the rcu_node structure.
 258  */
 259 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,19,0) || \
 260         LTTNG_RHEL_KERNEL_RANGE(4,18,0,80,0,0, 4,19,0,0,0,0))
 261 LTTNG_TRACEPOINT_EVENT(rcu_quiescent_state_report,
 262
 263         TP_PROTO(const char *rcuname, unsigned long gp_seq,
 264                  unsigned long mask, unsigned long qsmask,
 265                  u8 level, int grplo, int grphi, int gp_tasks),
 266
 267         TP_ARGS(rcuname, gp_seq, mask, qsmask, level, grplo, grphi, gp_tasks),
 268
 269         TP_FIELDS(
 270                 ctf_string(rcuname, rcuname)
 271                 ctf_integer(unsigned long, gp_seq, gp_seq)
 272                 ctf_integer(unsigned long, mask, mask)
 273                 ctf_integer(unsigned long, qsmask, qsmask)
 274                 ctf_integer(u8, level, level)
 275                 ctf_integer(int, grplo, grplo)
 276                 ctf_integer(int, grphi, grphi)
 277                 ctf_integer(u8, gp_tasks, gp_tasks)
 278         )
 279 )
 280 #elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,12,0))
 281 LTTNG_TRACEPOINT_EVENT(rcu_quiescent_state_report,
 282
 283         TP_PROTO(const char *rcuname, unsigned long gpnum,
 284                  unsigned long mask, unsigned long qsmask,
 285                  u8 level, int grplo, int grphi, int gp_tasks),
 286
 287         TP_ARGS(rcuname, gpnum, mask, qsmask, level, grplo, grphi, gp_tasks),
 288
 289         TP_FIELDS(
 290                 ctf_string(rcuname, rcuname)
 291                 ctf_integer(unsigned long, gpnum, gpnum)
 292                 ctf_integer(unsigned long, mask, mask)
 293                 ctf_integer(unsigned long, qsmask, qsmask)
 294                 ctf_integer(u8, level, level)
 295                 ctf_integer(int, grplo, grplo)
 296                 ctf_integer(int, grphi, grphi)
 297                 ctf_integer(u8, gp_tasks, gp_tasks)
 298         )
 299 )
 300 #else
 301 LTTNG_TRACEPOINT_EVENT(rcu_quiescent_state_report,
 302
 303         TP_PROTO(char *rcuname, unsigned long gpnum,
 304                  unsigned long mask, unsigned long qsmask,
 305                  u8 level, int grplo, int grphi, int gp_tasks),
 306
 307         TP_ARGS(rcuname, gpnum, mask, qsmask, level, grplo, grphi, gp_tasks),
 308
 309         TP_FIELDS(
 310                 ctf_string(rcuname, rcuname)
 311                 ctf_integer(unsigned long, gpnum, gpnum)
 312                 ctf_integer(unsigned long, mask, mask)
 313                 ctf_integer(unsigned long, qsmask, qsmask)
 314                 ctf_integer(u8, level, level)
 315                 ctf_integer(int, grplo, grplo)
 316                 ctf_integer(int, grphi, grphi)
 317                 ctf_integer(u8, gp_tasks, gp_tasks)
 318         )
 319 )
 320 #endif
 321
 322 /*
 323  * Tracepoint for quiescent states detected by force_quiescent_state().
 324  * These trace events include the type of RCU, the grace-period number
 325  * that was blocked by the CPU, the CPU itself, and the type of quiescent
 326  * state, which can be "dti" for dyntick-idle mode, "ofl" for CPU offline,
 327  * or "kick" when kicking a CPU that has been in dyntick-idle mode for
 328  * too long.
 329  */
 330 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,19,0) || \
 331         LTTNG_RHEL_KERNEL_RANGE(4,18,0,80,0,0, 4,19,0,0,0,0))
 332 LTTNG_TRACEPOINT_EVENT(rcu_fqs,
 333
 334         TP_PROTO(const char *rcuname, unsigned long gp_seq, int cpu, const char *qsevent),
 335
 336         TP_ARGS(rcuname, gp_seq, cpu, qsevent),
 337
 338         TP_FIELDS(
 339                 ctf_integer(unsigned long, gp_seq, gp_seq)
 340                 ctf_integer(int, cpu, cpu)
 341                 ctf_string(rcuname, rcuname)
 342                 ctf_string(qsevent, qsevent)
 343         )
 344 )
 345 #elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,12,0))
 346 LTTNG_TRACEPOINT_EVENT(rcu_fqs,
 347
 348         TP_PROTO(const char *rcuname, unsigned long gpnum, int cpu, const char *qsevent),
 349
 350         TP_ARGS(rcuname, gpnum, cpu, qsevent),
 351
 352         TP_FIELDS(
 353                 ctf_integer(unsigned long, gpnum, gpnum)
 354                 ctf_integer(int, cpu, cpu)
 355                 ctf_string(rcuname, rcuname)
 356                 ctf_string(qsevent, qsevent)
 357         )
 358 )
 359 #else
 360 LTTNG_TRACEPOINT_EVENT(rcu_fqs,
 361
 362         TP_PROTO(char *rcuname, unsigned long gpnum, int cpu, char *qsevent),
 363
 364         TP_ARGS(rcuname, gpnum, cpu, qsevent),
 365
 366         TP_FIELDS(
 367                 ctf_integer(unsigned long, gpnum, gpnum)
 368                 ctf_integer(int, cpu, cpu)
 369                 ctf_string(rcuname, rcuname)
 370                 ctf_string(qsevent, qsevent)
 371         )
 372 )
 373 #endif
 374
 375 #endif  /*
 376          * #if defined(CONFIG_TREE_RCU)
 377          *      || (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,19,0)
 378          *              && defined(CONFIG_PREEMPT_RCU))
 379          *      || defined(CONFIG_TREE_PREEMPT_RCU)
 380          */
 381
 382 /*
 383  * Tracepoint for dyntick-idle entry/exit events.  These take a string
 384  * as argument: "Start" for entering dyntick-idle mode, "End" for
 385  * leaving it, "--=" for events moving towards idle, and "++=" for events
 386  * moving away from idle.  "Error on entry: not idle task" and "Error on
 387  * exit: not idle task" indicate that a non-idle task is erroneously
 388  * toying with the idle loop.
 389  *
 390  * These events also take a pair of numbers, which indicate the nesting
 391  * depth before and after the event of interest.  Note that task-related
 392  * events use the upper bits of each number, while interrupt-related
 393  * events use the lower bits.
 394  */
 395 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,6,0))   \
 396         || LTTNG_KERNEL_RANGE(5,5,6, 5,6,0)             \
 397         || LTTNG_KERNEL_RANGE(5,4,22, 5,5,0)            \
 398         || LTTNG_UBUNTU_KERNEL_RANGE(5,0,21,46, 5,1,0,0)
 399 LTTNG_TRACEPOINT_EVENT(rcu_dyntick,
 400
 401         TP_PROTO(const char *polarity, long oldnesting, long newnesting, int dynticks),
 402
 403         TP_ARGS(polarity, oldnesting, newnesting, dynticks),
 404
 405         TP_FIELDS(
 406                 ctf_string(polarity, polarity)
 407                 ctf_integer(long, oldnesting, oldnesting)
 408                 ctf_integer(long, newnesting, newnesting)
 409                 ctf_integer(int, dynticks, dynticks)
 410         )
 411 )
 412
 413 #elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,16,0))
 414 LTTNG_TRACEPOINT_EVENT(rcu_dyntick,
 415
 416         TP_PROTO(const char *polarity, long oldnesting, long newnesting, atomic_t dynticks),
 417
 418         TP_ARGS(polarity, oldnesting, newnesting, dynticks),
 419
 420         TP_FIELDS(
 421                 ctf_string(polarity, polarity)
 422                 ctf_integer(long, oldnesting, oldnesting)
 423                 ctf_integer(long, newnesting, newnesting)
 424                 ctf_integer(int, dynticks, atomic_read(&dynticks))
 425         )
 426 )
 427
 428 #elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,12,0))
 429 LTTNG_TRACEPOINT_EVENT(rcu_dyntick,
 430
 431         TP_PROTO(const char *polarity, long long oldnesting, long long newnesting),
 432
 433         TP_ARGS(polarity, oldnesting, newnesting),
 434
 435         TP_FIELDS(
 436                 ctf_string(polarity, polarity)
 437                 ctf_integer(long long, oldnesting, oldnesting)
 438                 ctf_integer(long long, newnesting, newnesting)
 439         )
 440 )
 441 #elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,3,0))
 442 LTTNG_TRACEPOINT_EVENT(rcu_dyntick,
 443
 444         TP_PROTO(char *polarity, long long oldnesting, long long newnesting),
 445
 446         TP_ARGS(polarity, oldnesting, newnesting),
 447
 448         TP_FIELDS(
 449                 ctf_string(polarity, polarity)
 450                 ctf_integer(long long, oldnesting, oldnesting)
 451                 ctf_integer(long long, newnesting, newnesting)
 452         )
 453 )
 454 #else
 455 LTTNG_TRACEPOINT_EVENT(rcu_dyntick,
 456
 457         TP_PROTO(char *polarity),
 458
 459         TP_ARGS(polarity),
 460
 461         TP_FIELDS(
 462                 ctf_string(polarity, polarity)
 463         )
 464 )
 465 #endif
 466
 467
 468 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,3,0))
 469 /*
 470  * Tracepoint for RCU preparation for idle, the goal being to get RCU
 471  * processing done so that the current CPU can shut off its scheduling
 472  * clock and enter dyntick-idle mode.  One way to accomplish this is
 473  * to drain all RCU callbacks from this CPU, and the other is to have
 474  * done everything RCU requires for the current grace period.  In this
 475  * latter case, the CPU will be awakened at the end of the current grace
 476  * period in order to process the remainder of its callbacks.
 477  *
 478  * These tracepoints take a string as argument:
 479  *
 480  *      "No callbacks": Nothing to do, no callbacks on this CPU.
 481  *      "In holdoff": Nothing to do, holding off after unsuccessful attempt.
 482  *      "Begin holdoff": Attempt failed, don't retry until next jiffy.
 483  *      "Dyntick with callbacks": Entering dyntick-idle despite callbacks.
 484  *      "Dyntick with lazy callbacks": Entering dyntick-idle w/lazy callbacks.
 485  *      "More callbacks": Still more callbacks, try again to clear them out.
 486  *      "Callbacks drained": All callbacks processed, off to dyntick idle!
 487  *      "Timer": Timer fired to cause CPU to continue processing callbacks.
 488  *      "Demigrate": Timer fired on wrong CPU, woke up correct CPU.
 489  *      "Cleanup after idle": Idle exited, timer canceled.
 490  */
 491 LTTNG_TRACEPOINT_EVENT(rcu_prep_idle,
 492
 493 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,12,0))
 494         TP_PROTO(const char *reason),
 495 #else /* #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,12,0)) */
 496         TP_PROTO(char *reason),
 497 #endif /* #else #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,12,0)) */
 498
 499         TP_ARGS(reason),
 500
 501         TP_FIELDS(
 502                 ctf_string(reason, reason)
 503         )
 504 )
 505 #endif
 506
 507 /*
 508  * Tracepoint for the registration of a single RCU callback function.
 509  * The first argument is the type of RCU, the second argument is
 510  * a pointer to the RCU callback itself, the third element is the
 511  * number of lazy callbacks queued, and the fourth element is the
 512  * total number of callbacks queued.
 513  */
 514 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,6,0))
 515 LTTNG_TRACEPOINT_EVENT(rcu_callback,
 516
 517         TP_PROTO(const char *rcuname, struct rcu_head *rhp, long qlen),
 518
 519         TP_ARGS(rcuname, rhp, qlen),
 520
 521         TP_FIELDS(
 522                 ctf_string(rcuname, rcuname)
 523                 ctf_integer_hex(void *, rhp, rhp)
 524                 ctf_integer_hex(void *, func, rhp->func)
 525                 ctf_integer(long, qlen, qlen)
 526         )
 527 )
 528 #elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,12,0))
 529 LTTNG_TRACEPOINT_EVENT(rcu_callback,
 530
 531         TP_PROTO(const char *rcuname, struct rcu_head *rhp, long qlen_lazy,
 532                  long qlen),
 533
 534         TP_ARGS(rcuname, rhp, qlen_lazy, qlen),
 535
 536         TP_FIELDS(
 537                 ctf_string(rcuname, rcuname)
 538                 ctf_integer_hex(void *, rhp, rhp)
 539                 ctf_integer_hex(void *, func, rhp->func)
 540                 ctf_integer(long, qlen_lazy, qlen_lazy)
 541                 ctf_integer(long, qlen, qlen)
 542         )
 543 )
 544 #elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,4,0))
 545 LTTNG_TRACEPOINT_EVENT(rcu_callback,
 546
 547         TP_PROTO(char *rcuname, struct rcu_head *rhp, long qlen_lazy,
 548                  long qlen),
 549
 550         TP_ARGS(rcuname, rhp, qlen_lazy, qlen),
 551
 552         TP_FIELDS(
 553                 ctf_string(rcuname, rcuname)
 554                 ctf_integer_hex(void *, rhp, rhp)
 555                 ctf_integer_hex(void *, func, rhp->func)
 556                 ctf_integer(long, qlen_lazy, qlen_lazy)
 557                 ctf_integer(long, qlen, qlen)
 558         )
 559 )
 560 #else
 561 LTTNG_TRACEPOINT_EVENT(rcu_callback,
 562
 563         TP_PROTO(char *rcuname, struct rcu_head *rhp, long qlen),
 564
 565         TP_ARGS(rcuname, rhp, qlen),
 566
 567         TP_FIELDS(
 568                 ctf_string(rcuname, rcuname)
 569                 ctf_integer_hex(void *, rhp, rhp)
 570                 ctf_integer_hex(void *, func, rhp->func)
 571                 ctf_integer(long, qlen, qlen)
 572         )
 573 )
 574 #endif
 575
 576
 577 /*
 578  * Tracepoint for the registration of a single RCU callback of the special
 579  * kfree() form.  The first argument is the RCU type, the second argument
 580  * is a pointer to the RCU callback, the third argument is the offset
 581  * of the callback within the enclosing RCU-protected data structure,
 582  * the fourth argument is the number of lazy callbacks queued, and the
 583  * fifth argument is the total number of callbacks queued.
 584  */
 585 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,6,0))
 586 LTTNG_TRACEPOINT_EVENT(rcu_kfree_callback,
 587
 588         TP_PROTO(const char *rcuname, struct rcu_head *rhp, unsigned long offset,
 589                  long qlen),
 590
 591         TP_ARGS(rcuname, rhp, offset, qlen),
 592
 593         TP_FIELDS(
 594                 ctf_string(rcuname, rcuname)
 595                 ctf_integer_hex(void *, rhp, rhp)
 596                 ctf_integer_hex(unsigned long, offset, offset)
 597                 ctf_integer(long, qlen, qlen)
 598         )
 599 )
 600 #elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,12,0))
 601 LTTNG_TRACEPOINT_EVENT(rcu_kfree_callback,
 602
 603         TP_PROTO(const char *rcuname, struct rcu_head *rhp, unsigned long offset,
 604                  long qlen_lazy, long qlen),
 605
 606         TP_ARGS(rcuname, rhp, offset, qlen_lazy, qlen),
 607
 608         TP_FIELDS(
 609                 ctf_string(rcuname, rcuname)
 610                 ctf_integer_hex(void *, rhp, rhp)
 611                 ctf_integer_hex(unsigned long, offset, offset)
 612                 ctf_integer(long, qlen_lazy, qlen_lazy)
 613                 ctf_integer(long, qlen, qlen)
 614         )
 615 )
 616 #elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,4,0))
 617 LTTNG_TRACEPOINT_EVENT(rcu_kfree_callback,
 618
 619         TP_PROTO(char *rcuname, struct rcu_head *rhp, unsigned long offset,
 620                  long qlen_lazy, long qlen),
 621
 622         TP_ARGS(rcuname, rhp, offset, qlen_lazy, qlen),
 623
 624         TP_FIELDS(
 625                 ctf_string(rcuname, rcuname)
 626                 ctf_integer_hex(void *, rhp, rhp)
 627                 ctf_integer_hex(unsigned long, offset, offset)
 628                 ctf_integer(long, qlen_lazy, qlen_lazy)
 629                 ctf_integer(long, qlen, qlen)
 630         )
 631 )
 632 #else
 633 LTTNG_TRACEPOINT_EVENT(rcu_kfree_callback,
 634
 635         TP_PROTO(char *rcuname, struct rcu_head *rhp, unsigned long offset,
 636                  long qlen),
 637
 638         TP_ARGS(rcuname, rhp, offset, qlen),
 639
 640         TP_FIELDS(
 641                 ctf_string(rcuname, rcuname)
 642                 ctf_integer_hex(void *, rhp, rhp)
 643                 ctf_integer_hex(unsigned long, offset, offset)
 644                 ctf_integer(long, qlen, qlen)
 645         )
 646 )
 647 #endif
 648
 649 /*
 650  * Tracepoint for marking the beginning rcu_do_batch, performed to start
 651  * RCU callback invocation.  The first argument is the RCU flavor,
 652  * the second is the number of lazy callbacks queued, the third is
 653  * the total number of callbacks queued, and the fourth argument is
 654  * the current RCU-callback batch limit.
 655  */
 656 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,6,0))
 657 LTTNG_TRACEPOINT_EVENT(rcu_batch_start,
 658
 659         TP_PROTO(const char *rcuname, long qlen, long blimit),
 660
 661         TP_ARGS(rcuname, qlen, blimit),
 662
 663         TP_FIELDS(
 664                 ctf_string(rcuname, rcuname)
 665                 ctf_integer(long, qlen, qlen)
 666                 ctf_integer(long, blimit, blimit)
 667         )
 668 )
 669 #elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,12,0))
 670 LTTNG_TRACEPOINT_EVENT(rcu_batch_start,
 671
 672         TP_PROTO(const char *rcuname, long qlen_lazy, long qlen, long blimit),
 673
 674         TP_ARGS(rcuname, qlen_lazy, qlen, blimit),
 675
 676         TP_FIELDS(
 677                 ctf_string(rcuname, rcuname)
 678                 ctf_integer(long, qlen_lazy, qlen_lazy)
 679                 ctf_integer(long, qlen, qlen)
 680                 ctf_integer(long, blimit, blimit)
 681         )
 682 )
 683 #elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,9,0))
 684 LTTNG_TRACEPOINT_EVENT(rcu_batch_start,
 685
 686         TP_PROTO(char *rcuname, long qlen_lazy, long qlen, long blimit),
 687
 688         TP_ARGS(rcuname, qlen_lazy, qlen, blimit),
 689
 690         TP_FIELDS(
 691                 ctf_string(rcuname, rcuname)
 692                 ctf_integer(long, qlen_lazy, qlen_lazy)
 693                 ctf_integer(long, qlen, qlen)
 694                 ctf_integer(long, blimit, blimit)
 695         )
 696 )
 697 #elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,4,0))
 698 LTTNG_TRACEPOINT_EVENT(rcu_batch_start,
 699
 700         TP_PROTO(char *rcuname, long qlen_lazy, long qlen, int blimit),
 701
 702         TP_ARGS(rcuname, qlen_lazy, qlen, blimit),
 703
 704         TP_FIELDS(
 705                 ctf_string(rcuname, rcuname)
 706                 ctf_integer(long, qlen_lazy, qlen_lazy)
 707                 ctf_integer(long, qlen, qlen)
 708                 ctf_integer(int, blimit, blimit)
 709         )
 710 )
 711 #else
 712 LTTNG_TRACEPOINT_EVENT(rcu_batch_start,
 713
 714         TP_PROTO(char *rcuname, long qlen, int blimit),
 715
 716         TP_ARGS(rcuname, qlen, blimit),
 717
 718         TP_FIELDS(
 719                 ctf_string(rcuname, rcuname)
 720                 ctf_integer(long, qlen, qlen)
 721                 ctf_integer(int, blimit, blimit)
 722         )
 723 )
 724 #endif
 725
 726 /*
 727  * Tracepoint for the invocation of a single RCU callback function.
 728  * The first argument is the type of RCU, and the second argument is
 729  * a pointer to the RCU callback itself.
 730  */
 731 LTTNG_TRACEPOINT_EVENT(rcu_invoke_callback,
 732
 733 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,12,0))
 734         TP_PROTO(const char *rcuname, struct rcu_head *rhp),
 735 #else /* #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,12,0)) */
 736         TP_PROTO(char *rcuname, struct rcu_head *rhp),
 737 #endif /* #else #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,12,0)) */
 738
 739         TP_ARGS(rcuname, rhp),
 740
 741         TP_FIELDS(
 742                 ctf_string(rcuname, rcuname)
 743                 ctf_integer_hex(void *, rhp, rhp)
 744                 ctf_integer_hex(void *, func, rhp->func)
 745         )
 746 )
 747
 748 /*
 749  * Tracepoint for the invocation of a single RCU callback of the special
 750  * kfree() form.  The first argument is the RCU flavor, the second
 751  * argument is a pointer to the RCU callback, and the third argument
 752  * is the offset of the callback within the enclosing RCU-protected
 753  * data structure.
 754  */
 755 LTTNG_TRACEPOINT_EVENT(rcu_invoke_kfree_callback,
 756
 757 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,12,0))
 758         TP_PROTO(const char *rcuname, struct rcu_head *rhp, unsigned long offset),
 759 #else /* #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,12,0)) */
 760         TP_PROTO(char *rcuname, struct rcu_head *rhp, unsigned long offset),
 761 #endif /* #else #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,12,0)) */
 762
 763         TP_ARGS(rcuname, rhp, offset),
 764
 765         TP_FIELDS(
 766                 ctf_string(rcuname, rcuname)
 767                 ctf_integer_hex(void *, rhp, rhp)
 768                 ctf_integer(unsigned long, offset, offset)
 769         )
 770 )
 771
 772 /*
 773  * Tracepoint for exiting rcu_do_batch after RCU callbacks have been
 774  * invoked.  The first argument is the name of the RCU flavor,
 775  * the second argument is number of callbacks actually invoked,
 776  * the third argument (cb) is whether or not any of the callbacks that
 777  * were ready to invoke at the beginning of this batch are still
 778  * queued, the fourth argument (nr) is the return value of need_resched(),
 779  * the fifth argument (iit) is 1 if the current task is the idle task,
 780  * and the sixth argument (risk) is the return value from
 781  * rcu_is_callbacks_kthread().
 782  */
 783 LTTNG_TRACEPOINT_EVENT(rcu_batch_end,
 784
 785 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,13,0))
 786         TP_PROTO(const char *rcuname, int callbacks_invoked,
 787                  char cb, char nr, char iit, char risk),
 788
 789         TP_ARGS(rcuname, callbacks_invoked, cb, nr, iit, risk),
 790 #elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,12,0))
 791         TP_PROTO(const char *rcuname, int callbacks_invoked,
 792                  bool cb, bool nr, bool iit, bool risk),
 793
 794         TP_ARGS(rcuname, callbacks_invoked, cb, nr, iit, risk),
 795 #elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,3,0))
 796         TP_PROTO(char *rcuname, int callbacks_invoked,
 797                  bool cb, bool nr, bool iit, bool risk),
 798
 799         TP_ARGS(rcuname, callbacks_invoked, cb, nr, iit, risk),
 800 #else
 801         TP_PROTO(char *rcuname, int callbacks_invoked),
 802
 803         TP_ARGS(rcuname, callbacks_invoked),
 804 #endif
 805
 806         TP_FIELDS(
 807                 ctf_string(rcuname, rcuname)
 808                 ctf_integer(int, callbacks_invoked, callbacks_invoked)
 809 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,13,0))
 810                 ctf_integer(char, cb, cb)
 811                 ctf_integer(char, nr, nr)
 812                 ctf_integer(char, iit, iit)
 813                 ctf_integer(char, risk, risk)
 814 #elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,3,0))
 815                 ctf_integer(bool, cb, cb)
 816                 ctf_integer(bool, nr, nr)
 817                 ctf_integer(bool, iit, iit)
 818                 ctf_integer(bool, risk, risk)
 819 #endif
 820         )
 821 )
 822
 823 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,3,0))
 824 /*
 825  * Tracepoint for rcutorture readers.  The first argument is the name
 826  * of the RCU flavor from rcutorture's viewpoint and the second argument
 827  * is the callback address.
 828  */
 829 LTTNG_TRACEPOINT_EVENT(rcu_torture_read,
 830
 831 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,12,0))
 832         TP_PROTO(const char *rcutorturename, struct rcu_head *rhp,
 833                  unsigned long secs, unsigned long c_old, unsigned long c),
 834
 835         TP_ARGS(rcutorturename, rhp, secs, c_old, c),
 836 #elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,9,0))
 837         TP_PROTO(char *rcutorturename, struct rcu_head *rhp,
 838                  unsigned long secs, unsigned long c_old, unsigned long c),
 839
 840         TP_ARGS(rcutorturename, rhp, secs, c_old, c),
 841 #else
 842         TP_PROTO(char *rcutorturename, struct rcu_head *rhp),
 843
 844         TP_ARGS(rcutorturename, rhp),
 845 #endif
 846
 847         TP_FIELDS(
 848                 ctf_string(rcutorturename, rcutorturename)
 849                 ctf_integer_hex(struct rcu_head *, rhp, rhp)
 850 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,9,0))
 851                 ctf_integer(unsigned long, secs, secs)
 852                 ctf_integer(unsigned long, c_old, c_old)
 853                 ctf_integer(unsigned long, c, c)
 854 #endif
 855         )
 856 )
 857 #endif
 858
 859 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,6,0))
 860 /*
 861  * Tracepoint for _rcu_barrier() execution.  The string "s" describes
 862  * the _rcu_barrier phase:
 863  *      "Begin": rcu_barrier_callback() started.
 864  *      "Check": rcu_barrier_callback() checking for piggybacking.
 865  *      "EarlyExit": rcu_barrier_callback() piggybacked, thus early exit.
 866  *      "Inc1": rcu_barrier_callback() piggyback check counter incremented.
 867  *      "Offline": rcu_barrier_callback() found offline CPU
 868  *      "OnlineQ": rcu_barrier_callback() found online CPU with callbacks.
 869  *      "OnlineNQ": rcu_barrier_callback() found online CPU, no callbacks.
 870  *      "IRQ": An rcu_barrier_callback() callback posted on remote CPU.
 871  *      "CB": An rcu_barrier_callback() invoked a callback, not the last.
 872  *      "LastCB": An rcu_barrier_callback() invoked the last callback.
 873  *      "Inc2": rcu_barrier_callback() piggyback check counter incremented.
 874  * The "cpu" argument is the CPU or -1 if meaningless, the "cnt" argument
 875  * is the count of remaining callbacks, and "done" is the piggybacking count.
 876  */
 877 LTTNG_TRACEPOINT_EVENT(rcu_barrier,
 878
 879 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,12,0))
 880         TP_PROTO(const char *rcuname, const char *s, int cpu, int cnt, unsigned long done),
 881 #else /* #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,12,0)) */
 882         TP_PROTO(char *rcuname, char *s, int cpu, int cnt, unsigned long done),
 883 #endif /* #else #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,12,0)) */
 884
 885         TP_ARGS(rcuname, s, cpu, cnt, done),
 886
 887         TP_FIELDS(
 888                 ctf_string(rcuname, rcuname)
 889                 ctf_string(s, s)
 890                 ctf_integer(int, cpu, cpu)
 891                 ctf_integer(int, cnt, cnt)
 892                 ctf_integer(unsigned long, done, done)
 893         )
 894 )
 895 #endif
 896
 897 #else /* #ifdef CONFIG_RCU_TRACE */
 898
 899 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,19,0) || \
 900         LTTNG_RHEL_KERNEL_RANGE(4,18,0,80,0,0, 4,19,0,0,0,0))
 901 #define trace_rcu_grace_period(rcuname, gp_seq, gpevent) do { } while (0)
 902 #define trace_rcu_grace_period_init(rcuname, gp_seq, level, grplo, grphi, \
 903                                     qsmask) do { } while (0)
 904 #define trace_rcu_preempt_task(rcuname, pid, gp_seq) do { } while (0)
 905 #define trace_rcu_unlock_preempted_task(rcuname, gp_seq, pid) do { } while (0)
 906 #define trace_rcu_quiescent_state_report(rcuname, gp_seq, mask, qsmask, level, \
 907                                          grplo, grphi, gp_tasks) do { } \
 908         while (0)
 909 #define trace_rcu_fqs(rcuname, gp_seq, cpu, qsevent) do { } while (0)
 910 #else
 911 #define trace_rcu_grace_period(rcuname, gpnum, gpevent) do { } while (0)
 912 #define trace_rcu_grace_period_init(rcuname, gpnum, level, grplo, grphi, \
 913                                     qsmask) do { } while (0)
 914 #define trace_rcu_preempt_task(rcuname, pid, gpnum) do { } while (0)
 915 #define trace_rcu_unlock_preempted_task(rcuname, gpnum, pid) do { } while (0)
 916 #define trace_rcu_quiescent_state_report(rcuname, gpnum, mask, qsmask, level, \
 917                                          grplo, grphi, gp_tasks) do { } \
 918         while (0)
 919 #define trace_rcu_fqs(rcuname, gpnum, cpu, qsevent) do { } while (0)
 920 #endif
 921
 922 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,16,0))
 923 #define trace_rcu_dyntick(polarity, oldnesting, newnesting, dyntick) do { } while (0)
 924 #elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,3,0))
 925 #define trace_rcu_dyntick(polarity, oldnesting, newnesting) do { } while (0)
 926 #else
 927 #define trace_rcu_dyntick(polarity) do { } while (0)
 928 #endif
 929 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,3,0))
 930 #define trace_rcu_prep_idle(reason) do { } while (0)
 931 #endif
 932 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,4,0))
 933 #define trace_rcu_callback(rcuname, rhp, qlen_lazy, qlen) do { } while (0)
 934 #define trace_rcu_kfree_callback(rcuname, rhp, offset, qlen_lazy, qlen) \
 935         do { } while (0)
 936 #define trace_rcu_batch_start(rcuname, qlen_lazy, qlen, blimit) \
 937         do { } while (0)
 938 #else
 939 #define trace_rcu_callback(rcuname, rhp, qlen) do { } while (0)
 940 #define trace_rcu_kfree_callback(rcuname, rhp, offset, qlen) do { } while (0)
 941 #define trace_rcu_batch_start(rcuname, qlen, blimit) do { } while (0)
 942 #endif
 943 #define trace_rcu_invoke_callback(rcuname, rhp) do { } while (0)
 944 #define trace_rcu_invoke_kfree_callback(rcuname, rhp, offset) do { } while (0)
 945 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,3,0))
 946 #define trace_rcu_batch_end(rcuname, callbacks_invoked, cb, nr, iit, risk) \
 947         do { } while (0)
 948 #else
 949 #define trace_rcu_batch_end(rcuname, callbacks_invoked) do { } while (0)
 950 #endif
 951 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,9,0))
 952 #define trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c) \
 953         do { } while (0)
 954 #elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,3,0))
 955 #define trace_rcu_torture_read(rcutorturename, rhp) do { } while (0)
 956 #endif
 957 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,6,0))
 958 #define trace_rcu_barrier(name, s, cpu, cnt, done) do { } while (0)
 959 #endif
 960 #endif /* #else #ifdef CONFIG_RCU_TRACE */
 961
 962 #endif /* LTTNG_TRACE_RCU_H */
 963
 964 /* This part must be outside protection */
 965 #include <lttng/define_trace.h>